@apmantza/greedysearch-pi 1.0.23 → 1.1.0

package/extractors/gemini.mjs

@@ -108,15 +108,49 @@ async function extractAnswer(tab) {
   const answer = await cdp(['eval', tab, `window.__geminiClipboard || ''`]);
   if (!answer) throw new Error('Clipboard interceptor returned empty text');
 
-  // Sources: links rendered in the page (best-effort; Shadow DOM may hide some)
+  // Click "Sources" button to open the sidebar with proper source cards
+  await cdp(['eval', tab, `
+    (function() {
+      var btn = document.querySelector('button.legacy-sources-sidebar-button, button.mdс-button--outline');
+      if (!btn) btn = Array.from(document.querySelectorAll('button')).find(b => b.innerText?.trim() === 'Sources');
+      if (btn) { btn.click(); return 'clicked'; }
+      return 'not-found';
+    })()
+  `]).catch(() => 'not-found');
+
+  // Wait for the sources sidebar to populate
+  await new Promise(r => setTimeout(r, 1500));
+
+  // Extract sources from the sidebar panel (has proper URLs + titles)
   const raw = await cdp(['eval', tab, `
     (function() {
-      var sources = Array.from(document.querySelectorAll('a[href^="http"]'))
+      // Find the Sources sidebar container by heading
+      var headings = Array.from(document.querySelectorAll('h1, h2, h3, [class*="header"]'));
+      var sourceHeading = headings.find(h => h.innerText?.trim() === 'Sources');
+      if (sourceHeading) {
+        var container = sourceHeading.closest('.container') || sourceHeading.parentElement;
+        var links = Array.from(container.querySelectorAll('a[href^="http"]'))
+          .map(a => ({ url: a.href.split('#')[0], title: a.innerText?.trim().split('\\n')[0] || '' }))
+          .filter(s => s.url && !s.url.includes('gemini.google') && !s.url.includes('gstatic') && !s.url.includes('google.com/search'))
+          .filter((v, i, arr) => arr.findIndex(x => x.url === v.url) === i)
+          .slice(0, 8);
+        return JSON.stringify(links);
+      }
+      // Fallback: inline source cards with aria-labels
+      var cards = Array.from(document.querySelectorAll('button[aria-label*="citation from"]'));
+      if (cards.length) {
+        return JSON.stringify(cards.map(b => {
+          var label = b.getAttribute('aria-label') || '';
+          var name = label.match(/from\\s+(.+?)\\.\\s/)?.[1] || label;
+          return { url: '', title: name };
+        }));
+      }
+      // Last resort: page-wide links (may include footer junk)
+      return JSON.stringify(Array.from(document.querySelectorAll('a[href^="http"]'))
         .map(a => ({ url: a.href.split('#')[0], title: a.innerText?.trim().split('\\n')[0] || '' }))
         .filter(s => s.url && !s.url.includes('gemini.google') && !s.url.includes('gstatic') && !s.url.includes('google.com/search'))
         .filter((v, i, arr) => arr.findIndex(x => x.url === v.url) === i)
-        .slice(0, 8);
-      return JSON.stringify(sources);
+        .slice(0, 8));
     })()
   `]).catch(() => '[]');
   const sources = JSON.parse(raw);

package/index.ts

@@ -22,7 +22,7 @@ function cdpAvailable(): boolean {
 
 function runSearch(engine: string, query: string, flags: string[] = []): Promise<Record<string, unknown>> {
 	return new Promise((resolve, reject) => {
-		const proc = spawn("node", [__dir + "/search.mjs", engine, ...flags, query], {
+		const proc = spawn("node", [__dir + "/search.mjs", engine, "--inline", ...flags, query], {
 			stdio: ["ignore", "pipe", "pipe"],
 		});
 		let out = "";
@@ -137,9 +137,13 @@ export default function greedySearchExtension(pi: ExtensionAPI) {
 				description: 'When true and engine is "all", deduplicates sources across engines and feeds them to Gemini for a single grounded synthesis. Adds ~30s but saves tokens and improves answer quality.',
 				default: false,
 			})),
+			fullAnswer: Type.Optional(Type.Boolean({
+				description: 'When true, returns the complete answer instead of a truncated preview (default: false, answers are shortened to ~300 chars to save tokens).',
+				default: false,
+			})),
 		}),
 		execute: async (_toolCallId, params) => {
-			const { query, engine = "all", synthesize = false } = params as { query: string; engine: string; synthesize?: boolean };
+			const { query, engine = "all", synthesize = false, fullAnswer = false } = params as { query: string; engine: string; synthesize?: boolean; fullAnswer?: boolean };
 
 			if (!cdpAvailable()) {
 				return {
@@ -150,6 +154,7 @@ export default function greedySearchExtension(pi: ExtensionAPI) {
 
 			const flags: string[] = [];
 			if (synthesize && engine === "all") flags.push("--synthesize");
+			if (fullAnswer) flags.push("--full");
 
 			let data: Record<string, unknown>;
 			try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apmantza/greedysearch-pi",
-  "version": "1.0.23",
+  "version": "1.1.0",
   "description": "Pi extension: search Perplexity, Bing Copilot, and Google AI in parallel with optional Gemini synthesis — grounded AI answers, not just links",
   "type": "module",
   "keywords": [

package/search.mjs

@@ -22,7 +22,7 @@
 import { spawn } from 'child_process';
 import { fileURLToPath } from 'url';
 import { join, dirname } from 'path';
-import { readFileSync, existsSync, writeFileSync } from 'fs';
+import { readFileSync, existsSync, writeFileSync, mkdirSync } from 'fs';
 import { tmpdir, homedir } from 'os';
 import http from 'http';
 
@@ -41,6 +41,8 @@ const ENGINES = {
   b:          'bing-copilot.mjs',
   google:     'google-ai.mjs',
   g:          'google-ai.mjs',
+  gemini:     'gemini.mjs',
+  gem:        'gemini.mjs',
   stackoverflow: 'stackoverflow-ai.mjs',
   so:         'stackoverflow-ai.mjs',
   stack:      'stackoverflow-ai.mjs',
@@ -52,6 +54,7 @@ const ENGINE_DOMAINS = {
   perplexity: 'perplexity.ai',
   bing:       'copilot.microsoft.com',
   google:     'google.com',
+  gemini:     'gemini.google.com',
   stackoverflow: 'stackoverflow.com',
 };
 
@@ -155,7 +158,7 @@ async function fetchTopSource(url) {
       (function(){
         var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
         var text = (el || document.body).innerText;
-        return text.replace(/\\s+/g, ' ').trim().slice(0, 1500);
+        return text.replace(/\\s+/g, ' ').trim();
       })()
     `]);
     return { url, content };
@@ -214,7 +217,7 @@ async function synthesizeWithGemini(query, results) {
     if (r?.error) {
       prompt += `## ${engine} (failed)\nError: ${r.error}\n\n`;
     } else if (r?.answer) {
-      prompt += `## ${engine}\n${r.answer.slice(0, 2000)}\n\n`;
+      prompt += `## ${engine}\n${r.answer}\n\n`;
     }
   }
   
@@ -248,13 +251,41 @@ async function synthesizeWithGemini(query, results) {
   });
 }
 
-function writeOutput(data, outFile) {
+function slugify(query) {
+  return query.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').slice(0, 60);
+}
+
+function resultsDir() {
+  const dir = join(__dir, 'results');
+  mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+function writeOutput(data, outFile, { inline = false, synthesize = false, query = '' } = {}) {
   const json = JSON.stringify(data, null, 2) + '\n';
+
   if (outFile) {
     writeFileSync(outFile, json, 'utf8');
     process.stderr.write(`Results written to ${outFile}\n`);
-  } else {
+    return;
+  }
+
+  if (inline) {
     process.stdout.write(json);
+    return;
+  }
+
+  const ts   = new Date().toISOString().replace('T', '_').replace(/[:.]/g, '-').slice(0, 19);
+  const slug = slugify(query);
+  const base = join(resultsDir(), `${ts}_${slug}`);
+
+  writeFileSync(`${base}.json`, json, 'utf8');
+
+  if (synthesize && data._synthesis?.answer) {
+    writeFileSync(`${base}-synthesis.md`, data._synthesis.answer, 'utf8');
+    process.stdout.write(`${base}-synthesis.md\n`);
+  } else {
+    process.stdout.write(`${base}.json\n`);
   }
 }
 
@@ -332,6 +363,7 @@ async function main() {
   const short       = !full;   // brief by default; --full opts into complete answers
   const fetchSource = args.includes('--fetch-top-source');
   const synthesize  = args.includes('--synthesize');
+  const inline      = args.includes('--inline');
   const outIdx      = args.indexOf('--out');
   const outFile     = outIdx !== -1 ? args[outIdx + 1] : null;
   const rest        = args.filter((a, i) =>
@@ -339,6 +371,7 @@ async function main() {
     a !== '--short' &&  // keep accepting --short for back-compat
     a !== '--fetch-top-source' &&
     a !== '--synthesize' &&
+    a !== '--inline' &&
     a !== '--out' &&
     (outIdx === -1 || i !== outIdx + 1)
   );
@@ -410,7 +443,7 @@ async function main() {
       if (top) out._topSource = await fetchTopSource(top.url);
     }
 
-    writeOutput(out, outFile);
+    writeOutput(out, outFile, { inline, synthesize, query });
     return;
   }
 
@@ -425,7 +458,7 @@ async function main() {
     if (fetchSource && result.sources?.length > 0) {
       result.topSource = await fetchTopSource(result.sources[0].url);
     }
-    writeOutput(result, outFile);
+    writeOutput(result, outFile, { inline, synthesize, query });
   } catch (e) {
     process.stderr.write(`Error: ${e.message}\n`);
     process.exit(1);