mcp-scraper 0.40.2 → 0.40.3

package/dist/bin/mcp-stdio-server.js

@@ -7,19 +7,20 @@ import {
   registerMemoryMcpTools,
   registerPaaExtractorMcpTools,
   registerSerpIntelligenceCaptureTools
-} from "../chunk-LP6E462I.js";
+} from "../chunk-6SZ52BQ5.js";
 import "../chunk-345BQXZH.js";
-import "../chunk-3PIWJS6Y.js";
+import "../chunk-BBI7RGOT.js";
+import "../chunk-EHES33KB.js";
 import {
   renderInstallTerminal
 } from "../chunk-XELSA2MS.js";
 import "../chunk-CB5C3BPB.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-ZUJLSICT.js";
+} from "../chunk-XU4ZLIB2.js";
 import "../chunk-SBLGBZZB.js";
-import "../chunk-44HZLHDV.js";
-import "../chunk-GUVKHCKE.js";
+import "../chunk-5UN33CGU.js";
+import "../chunk-IQTT7CAB.js";
 
 // bin/mcp-stdio-server.ts
 import { readFileSync } from "fs";

package/dist/bin/mcp-stdio-server.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../bin/mcp-stdio-server.ts","../../src/mcp/hosted-memory-mcp-tool-executor.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools, registerSerpIntelligenceCaptureTools } from '../src/mcp/paa-mcp-server.js'\nimport { registerMemoryMcpTools } from '../src/mcp/memory-mcp-server.js'\nimport { HostedMemoryMcpToolExecutor } from '../src/mcp/hosted-memory-mcp-tool-executor.js'\nimport { SERVER_INSTRUCTIONS } from '../src/mcp/server-instructions.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst forceStdio =\n  process.argv.includes('--stdio') ||\n  process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n  const noColor =\n    process.argv.includes('--no-color') ||\n    process.env.NO_COLOR !== undefined ||\n    process.env.FORCE_COLOR === '0' ||\n    !process.stdout.isTTY\n\n  process.stdout.write(renderInstallTerminal({\n    version: PACKAGE_VERSION,\n    color: !noColor,\n    apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n  }))\n  process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n  const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n  for (const path of paths) {\n    try {\n      const value = readFileSync(path, 'utf8').trim()\n      if (value) return value\n    } catch {\n      void 0\n    }\n  }\n  return undefined\n}\n\nconst apiKey = (\n  process.env.MCP_SCRAPER_API_KEY ??\n  process.env.MCP_SCRAPER_KEY ??\n  process.env.MCP_API_KEY ??\n  readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n  process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n  process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS })\n\nconst httpExecutor = new HttpMcpToolExecutor(baseUrl, apiKey)\nregisterPaaExtractorMcpTools(server, httpExecutor)\nregisterSerpIntelligenceCaptureTools(server, httpExecutor)\nregisterBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl })\nregisterMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, apiKey))\n\nconst transport = new StdioServerTransport()\n\nasync function main() {\n  await server.connect(transport)\n}\n\nmain().catch(err => {\n  process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n  process.exit(1)\n})\n","import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'\nimport { MEMORY_TOOL_SCHEMAS } from './memory-tool-schemas.js'\n\nconst publicToolNameByUpstreamName = new Map(\n  MEMORY_TOOL_SCHEMAS.map(schema => [schema.upstreamName, schema.id]),\n)\n\ninterface JsonRpcEnvelope {\n  result?: CallToolResult\n  error?: { code?: number; message?: string }\n}\n\nfunction parseJsonRpcEnvelope(text: string): JsonRpcEnvelope {\n  const payloads = text\n    .split('\\n')\n    .filter(line => line.startsWith('data:'))\n    .map(line => line.slice(5).trim())\n    .filter(Boolean)\n\n  if (!payloads.length) return JSON.parse(text) as JsonRpcEnvelope\n\n  for (const payload of payloads) {\n    const parsed = JSON.parse(payload) as JsonRpcEnvelope\n    if (parsed.result || parsed.error) return parsed\n  }\n  throw new Error('hosted MCP returned no JSON-RPC result')\n}\n\nfunction errorResult(message: string): CallToolResult {\n  return {\n    content: [{ type: 'text', text: JSON.stringify({ ok: false, error: message }) }],\n    isError: true,\n  }\n}\n\n/**\n * Stdio-only bridge for memory tools. It deliberately calls the hosted aggregate\n * MCP surface so local clients execute the same registered tool implementation\n * and output contract as https://mcpscraper.dev/mcp.\n */\nexport class HostedMemoryMcpToolExecutor {\n  private readonly baseUrl: string\n  private readonly apiKey: string\n\n  constructor(baseUrl: string, apiKey: string) {\n    this.baseUrl = baseUrl.replace(/\\/$/, '')\n    this.apiKey = apiKey\n  }\n\n  async callMemoryTool(upstreamName: string, args: Record<string, unknown>): Promise<CallToolResult> {\n    const publicToolName = publicToolNameByUpstreamName.get(upstreamName)\n    if (!publicToolName) return errorResult(`unknown memory tool: ${upstreamName}`)\n\n    try {\n      const res = await fetch(`${this.baseUrl}/mcp`, {\n        method: 'POST',\n        headers: {\n          'content-type': 'application/json',\n          accept: 'application/json, text/event-stream',\n          'x-api-key': this.apiKey,\n        },\n        body: JSON.stringify({\n          jsonrpc: '2.0',\n          id: `memory:${publicToolName}`,\n          method: 'tools/call',\n          params: { name: publicToolName, arguments: args },\n        }),\n      })\n      const text = await res.text()\n      if (!res.ok) return errorResult(`hosted memory ${publicToolName} failed (HTTP ${res.status})`)\n\n      const envelope = parseJsonRpcEnvelope(text)\n      if (envelope.error) {\n        return errorResult(envelope.error.message ?? `hosted memory ${publicToolName} failed`)\n      }\n      return envelope.result ?? errorResult(`hosted memory ${publicToolName} returned no result`)\n    } catch (err) {\n      return errorResult(err instanceof Error ? err.message : `hosted memory ${publicToolName} call failed`)\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAC1B,SAAS,4BAA4B;;;ACFrC,IAAM,+BAA+B,IAAI;AAAA,EACvC,oBAAoB,IAAI,YAAU,CAAC,OAAO,cAAc,OAAO,EAAE,CAAC;AACpE;AAOA,SAAS,qBAAqB,MAA+B;AAC3D,QAAM,WAAW,KACd,MAAM,IAAI,EACV,OAAO,UAAQ,KAAK,WAAW,OAAO,CAAC,EACvC,IAAI,UAAQ,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,EAChC,OAAO,OAAO;AAEjB,MAAI,CAAC,SAAS,OAAQ,QAAO,KAAK,MAAM,IAAI;AAE5C,aAAW,WAAW,UAAU;AAC9B,UAAM,SAAS,KAAK,MAAM,OAAO;AACjC,QAAI,OAAO,UAAU,OAAO,MAAO,QAAO;AAAA,EAC5C;AACA,QAAM,IAAI,MAAM,wCAAwC;AAC1D;AAEA,SAAS,YAAY,SAAiC;AACpD,SAAO;AAAA,IACL,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,KAAK,UAAU,EAAE,IAAI,OAAO,OAAO,QAAQ,CAAC,EAAE,CAAC;AAAA,IAC/E,SAAS;AAAA,EACX;AACF;AAOO,IAAM,8BAAN,MAAkC;AAAA,EACtB;AAAA,EACA;AAAA,EAEjB,YAAYA,UAAiBC,SAAgB;AAC3C,SAAK,UAAUD,SAAQ,QAAQ,OAAO,EAAE;AACxC,SAAK,SAASC;AAAA,EAChB;AAAA,EAEA,MAAM,eAAe,cAAsB,MAAwD;AACjG,UAAM,iBAAiB,6BAA6B,IAAI,YAAY;AACpE,QAAI,CAAC,eAAgB,QAAO,YAAY,wBAAwB,YAAY,EAAE;AAE9E,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,GAAG,KAAK,OAAO,QAAQ;AAAA,QAC7C,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,gBAAgB;AAAA,UAChB,QAAQ;AAAA,UACR,aAAa,KAAK;AAAA,QACpB;AAAA,QACA,MAAM,KAAK,UAAU;AAAA,UACnB,SAAS;AAAA,UACT,IAAI,UAAU,cAAc;AAAA,UAC5B,QAAQ;AAAA,UACR,QAAQ,EAAE,MAAM,gBAAgB,WAAW,KAAK;AAAA,QAClD,CAAC;AAAA,MACH,CAAC;AACD,YAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAI,CAAC,IAAI,GAAI,QAAO,YAAY,iBAAiB,cAAc,iBAAiB,IAAI,MAAM,GAAG;AAE7F,YAAM,WAAW,qBAAqB,IAAI;AAC1C,UAAI,SAAS,OAAO;AAClB,eAAO,YAAY,SAAS,MAAM,WAAW,iBAAiB,cAAc,SAAS;AAAA,MACvF;AACA,aAAO,SAAS,UAAU,YAAY,iBAAiB,cAAc,qBAAqB;AAAA,IAC5F,SAAS,KAAK;AACZ,aAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,iBAAiB,cAAc,cAAc;AAAA,IACvG;AAAA,EACF;AACF;;;ADjEA,IAAM,aACJ,QAAQ,KAAK,SAAS,SAAS,KAC/B,QAAQ,IAAI,4BAA4B;AAC1C,IAAM,sBAAsB,QAAQ,QAAQ,MAAM,SAAS,QAAQ,OAAO,KAAK;AAC/E,IAAM,YAAY,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAE/E,IAAI,CAAC,eAAe,uBAAuB,YAAY;AACrD,QAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,UAAQ,OAAO,MAAM,sBAAsB;AAAA,IACzC,SAAS;AAAA,IACT,OAAO,CAAC;AAAA,IACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAAA,EACnE,CAAC,CAAC;AACF,UAAQ,KAAK,CAAC;AAChB;AAEA,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,IAAI,UAAU,EAAE,MAAM,eAAe,SAAS,gBAAgB,GAAG,EAAE,cAAc,oBAAoB,CAAC;AAErH,IAAM,eAAe,IAAI,oBAAoB,SAAS,MAAM;AAC5D,6BAA6B,QAAQ,YAAY;AACjD,qCAAqC,QAAQ,YAAY;AACzD,6BAA6B,QAAQ,EAAE,SAAS,QAAQ,eAAe,CAAC;AACxE,uBAAuB,QAAQ,IAAI,4BAA4B,SAAS,MAAM,CAAC;AAE/E,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["baseUrl","apiKey"]}
+{"version":3,"sources":["../../bin/mcp-stdio-server.ts","../../src/mcp/hosted-memory-mcp-tool-executor.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools, registerSerpIntelligenceCaptureTools } from '../src/mcp/paa-mcp-server.js'\nimport { registerMemoryMcpTools } from '../src/mcp/memory-mcp-server.js'\nimport { HostedMemoryMcpToolExecutor } from '../src/mcp/hosted-memory-mcp-tool-executor.js'\nimport { SERVER_INSTRUCTIONS } from '../src/mcp/server-instructions.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst forceStdio =\n  process.argv.includes('--stdio') ||\n  process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n  const noColor =\n    process.argv.includes('--no-color') ||\n    process.env.NO_COLOR !== undefined ||\n    process.env.FORCE_COLOR === '0' ||\n    !process.stdout.isTTY\n\n  process.stdout.write(renderInstallTerminal({\n    version: PACKAGE_VERSION,\n    color: !noColor,\n    apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n  }))\n  process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n  const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n  for (const path of paths) {\n    try {\n      const value = readFileSync(path, 'utf8').trim()\n      if (value) return value\n    } catch {\n      void 0\n    }\n  }\n  return undefined\n}\n\nconst apiKey = (\n  process.env.MCP_SCRAPER_API_KEY ??\n  process.env.MCP_SCRAPER_KEY ??\n  process.env.MCP_API_KEY ??\n  readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n  process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n  process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS })\n\nconst httpExecutor = new HttpMcpToolExecutor(baseUrl, apiKey)\nregisterPaaExtractorMcpTools(server, httpExecutor)\nregisterSerpIntelligenceCaptureTools(server, httpExecutor)\nregisterBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl })\nregisterMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, apiKey))\n\nconst transport = new StdioServerTransport()\n\nasync function main() {\n  await server.connect(transport)\n}\n\nmain().catch(err => {\n  process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n  process.exit(1)\n})\n","import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'\nimport { MEMORY_TOOL_SCHEMAS } from './memory-tool-schemas.js'\n\nconst publicToolNameByUpstreamName = new Map(\n  MEMORY_TOOL_SCHEMAS.map(schema => [schema.upstreamName, schema.id]),\n)\n\ninterface JsonRpcEnvelope {\n  result?: CallToolResult\n  error?: { code?: number; message?: string }\n}\n\nfunction parseJsonRpcEnvelope(text: string): JsonRpcEnvelope {\n  const payloads = text\n    .split('\\n')\n    .filter(line => line.startsWith('data:'))\n    .map(line => line.slice(5).trim())\n    .filter(Boolean)\n\n  if (!payloads.length) return JSON.parse(text) as JsonRpcEnvelope\n\n  for (const payload of payloads) {\n    const parsed = JSON.parse(payload) as JsonRpcEnvelope\n    if (parsed.result || parsed.error) return parsed\n  }\n  throw new Error('hosted MCP returned no JSON-RPC result')\n}\n\nfunction errorResult(message: string): CallToolResult {\n  return {\n    content: [{ type: 'text', text: JSON.stringify({ ok: false, error: message }) }],\n    isError: true,\n  }\n}\n\n/**\n * Stdio-only bridge for memory tools. It deliberately calls the hosted aggregate\n * MCP surface so local clients execute the same registered tool implementation\n * and output contract as https://mcpscraper.dev/mcp.\n */\nexport class HostedMemoryMcpToolExecutor {\n  private readonly baseUrl: string\n  private readonly apiKey: string\n\n  constructor(baseUrl: string, apiKey: string) {\n    this.baseUrl = baseUrl.replace(/\\/$/, '')\n    this.apiKey = apiKey\n  }\n\n  async callMemoryTool(upstreamName: string, args: Record<string, unknown>): Promise<CallToolResult> {\n    const publicToolName = publicToolNameByUpstreamName.get(upstreamName)\n    if (!publicToolName) return errorResult(`unknown memory tool: ${upstreamName}`)\n\n    try {\n      const res = await fetch(`${this.baseUrl}/mcp`, {\n        method: 'POST',\n        headers: {\n          'content-type': 'application/json',\n          accept: 'application/json, text/event-stream',\n          'x-api-key': this.apiKey,\n        },\n        body: JSON.stringify({\n          jsonrpc: '2.0',\n          id: `memory:${publicToolName}`,\n          method: 'tools/call',\n          params: { name: publicToolName, arguments: args },\n        }),\n      })\n      const text = await res.text()\n      if (!res.ok) return errorResult(`hosted memory ${publicToolName} failed (HTTP ${res.status})`)\n\n      const envelope = parseJsonRpcEnvelope(text)\n      if (envelope.error) {\n        return errorResult(envelope.error.message ?? `hosted memory ${publicToolName} failed`)\n      }\n      return envelope.result ?? errorResult(`hosted memory ${publicToolName} returned no result`)\n    } catch (err) {\n      return errorResult(err instanceof Error ? err.message : `hosted memory ${publicToolName} call failed`)\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAC1B,SAAS,4BAA4B;;;ACFrC,IAAM,+BAA+B,IAAI;AAAA,EACvC,oBAAoB,IAAI,YAAU,CAAC,OAAO,cAAc,OAAO,EAAE,CAAC;AACpE;AAOA,SAAS,qBAAqB,MAA+B;AAC3D,QAAM,WAAW,KACd,MAAM,IAAI,EACV,OAAO,UAAQ,KAAK,WAAW,OAAO,CAAC,EACvC,IAAI,UAAQ,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,EAChC,OAAO,OAAO;AAEjB,MAAI,CAAC,SAAS,OAAQ,QAAO,KAAK,MAAM,IAAI;AAE5C,aAAW,WAAW,UAAU;AAC9B,UAAM,SAAS,KAAK,MAAM,OAAO;AACjC,QAAI,OAAO,UAAU,OAAO,MAAO,QAAO;AAAA,EAC5C;AACA,QAAM,IAAI,MAAM,wCAAwC;AAC1D;AAEA,SAAS,YAAY,SAAiC;AACpD,SAAO;AAAA,IACL,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,KAAK,UAAU,EAAE,IAAI,OAAO,OAAO,QAAQ,CAAC,EAAE,CAAC;AAAA,IAC/E,SAAS;AAAA,EACX;AACF;AAOO,IAAM,8BAAN,MAAkC;AAAA,EACtB;AAAA,EACA;AAAA,EAEjB,YAAYA,UAAiBC,SAAgB;AAC3C,SAAK,UAAUD,SAAQ,QAAQ,OAAO,EAAE;AACxC,SAAK,SAASC;AAAA,EAChB;AAAA,EAEA,MAAM,eAAe,cAAsB,MAAwD;AACjG,UAAM,iBAAiB,6BAA6B,IAAI,YAAY;AACpE,QAAI,CAAC,eAAgB,QAAO,YAAY,wBAAwB,YAAY,EAAE;AAE9E,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,GAAG,KAAK,OAAO,QAAQ;AAAA,QAC7C,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,gBAAgB;AAAA,UAChB,QAAQ;AAAA,UACR,aAAa,KAAK;AAAA,QACpB;AAAA,QACA,MAAM,KAAK,UAAU;AAAA,UACnB,SAAS;AAAA,UACT,IAAI,UAAU,cAAc;AAAA,UAC5B,QAAQ;AAAA,UACR,QAAQ,EAAE,MAAM,gBAAgB,WAAW,KAAK;AAAA,QAClD,CAAC;AAAA,MACH,CAAC;AACD,YAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAI,CAAC,IAAI,GAAI,QAAO,YAAY,iBAAiB,cAAc,iBAAiB,IAAI,MAAM,GAAG;AAE7F,YAAM,WAAW,qBAAqB,IAAI;AAC1C,UAAI,SAAS,OAAO;AAClB,eAAO,YAAY,SAAS,MAAM,WAAW,iBAAiB,cAAc,SAAS;AAAA,MACvF;AACA,aAAO,SAAS,UAAU,YAAY,iBAAiB,cAAc,qBAAqB;AAAA,IAC5F,SAAS,KAAK;AACZ,aAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,iBAAiB,cAAc,cAAc;AAAA,IACvG;AAAA,EACF;AACF;;;ADjEA,IAAM,aACJ,QAAQ,KAAK,SAAS,SAAS,KAC/B,QAAQ,IAAI,4BAA4B;AAC1C,IAAM,sBAAsB,QAAQ,QAAQ,MAAM,SAAS,QAAQ,OAAO,KAAK;AAC/E,IAAM,YAAY,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAE/E,IAAI,CAAC,eAAe,uBAAuB,YAAY;AACrD,QAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,UAAQ,OAAO,MAAM,sBAAsB;AAAA,IACzC,SAAS;AAAA,IACT,OAAO,CAAC;AAAA,IACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAAA,EACnE,CAAC,CAAC;AACF,UAAQ,KAAK,CAAC;AAChB;AAEA,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,IAAI,UAAU,EAAE,MAAM,eAAe,SAAS,gBAAgB,GAAG,EAAE,cAAc,oBAAoB,CAAC;AAErH,IAAM,eAAe,IAAI,oBAAoB,SAAS,MAAM;AAC5D,6BAA6B,QAAQ,YAAY;AACjD,qCAAqC,QAAQ,YAAY;AACzD,6BAA6B,QAAQ,EAAE,SAAS,QAAQ,eAAe,CAAC;AACxE,uBAAuB,QAAQ,IAAI,4BAA4B,SAAS,MAAM,CAAC;AAE/E,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["baseUrl","apiKey"]}

package/dist/bin/paa-harvest.cjs

@@ -171,6 +171,9 @@ var PAASelectors = {
   itemDataInitQ: "data-initq",
   itemQuestionEl: ".JlqpRe",
   answerContainer: ".bCOlv, .hgKElc, .wDYxhc, .LGOjhe, .fo7IQd, .fmW3u",
+  aiAnswerBody: ".pWvJNd",
+  aiCitation: "a.PMDqCb[href]",
+  aiSourceCard: "a.vIWmYe[href], a.Zbfntb[href]",
   sourceTitle: "h3",
   sourceSite: ".VuuXrf",
   sourceCite: "cite",
@@ -1718,14 +1721,28 @@ var PAAExtractor = class {
         }
         return parts.join(" ").replace(/\s+/g, " ").trim();
       }
+      function cleanSourceLabel(value) {
+        return (value ?? "").replace(/\s*\(\+\d+\)\s*-\s*View related links.*$/i, "").replace(/\s*-\s*Opens in new tab.*$/i, "").trim();
+      }
       return Array.from(document.querySelectorAll(selectors.item)).map((pair) => {
-        let sourceTitle = pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0;
-        let sourceSite = pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0;
-        let sourceCite = pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0;
-        if (!sourceCite) {
+        const clickTarget = pair.querySelector(selectors.clickTarget);
+        const expanded = pair.classList.contains(selectors.expandedClass) || clickTarget?.getAttribute("aria-expanded") === "true";
+        const answerContainer = expanded ? pair.querySelector(selectors.answerContainer) : null;
+        const aiAnswerBody = answerContainer?.querySelector(selectors.aiAnswerBody);
+        const answerElement = aiAnswerBody ?? answerContainer;
+        let sourceTitle = expanded ? pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0 : void 0;
+        let sourceSite = expanded ? pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0 : void 0;
+        let sourceCite = expanded ? pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0 : void 0;
+        if (expanded && !sourceCite) {
           const seenHrefs = /* @__PURE__ */ new Set();
           const anchors = [];
-          for (const a of Array.from(pair.querySelectorAll("a[href]"))) {
+          const anchorRoot = answerElement ?? pair;
+          const prioritized = [
+            ...Array.from(anchorRoot.querySelectorAll(selectors.aiCitation)),
+            ...Array.from(anchorRoot.querySelectorAll(selectors.aiSourceCard)),
+            ...Array.from(anchorRoot.querySelectorAll("a[href]"))
+          ];
+          for (const a of prioritized) {
             const href = a.href;
             if (!/^https?:\/\//.test(href) || seenHrefs.has(href)) continue;
             let host = "";
@@ -1736,18 +1753,23 @@ var PAAExtractor = class {
             }
             if (/(^|\.)google(usercontent)?\.[a-z.]+$/.test(host)) continue;
             seenHrefs.add(href);
-            anchors.push({ text: (a.textContent || "").trim(), href, host });
+            anchors.push({
+              text: (a.textContent || "").trim(),
+              label: cleanSourceLabel(a.getAttribute("aria-label")),
+              href,
+              host
+            });
           }
-          const primary = anchors.find((x) => x.text.length > 2) ?? anchors[0];
+          const primary = anchors.find((x) => x.label.length > 2 || x.text.length > 2) ?? anchors[0];
           if (primary) {
-            sourceTitle = sourceTitle || primary.text || void 0;
+            sourceTitle = sourceTitle || primary.label || primary.text || void 0;
             sourceSite = sourceSite || primary.host.replace(/^www\./, "");
             sourceCite = primary.href;
           }
         }
         return {
           question: pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "",
-          answer: cleanText(pair.querySelector(selectors.answerContainer)) || void 0,
+          answer: cleanText(answerElement) || void 0,
           sourceTitle,
           sourceSite,
           sourceCite