mcp-scraper 0.6.0 → 0.8.0

package/dist/{chunk-BL7BBSYF.js → chunk-RUGJE5EB.js}

@@ -1,6 +1,6 @@
 import {
   LedgerOperation
-} from "./chunk-APJO2XV5.js";
+} from "./chunk-HL33CGJF.js";
 import {
   sanitizeVendorName
 } from "./chunk-M2S27J6Z.js";
@@ -127,4 +127,4 @@ export {
   failExtractJob,
   settleExtractJob
 };
-//# sourceMappingURL=chunk-BL7BBSYF.js.map
+//# sourceMappingURL=chunk-RUGJE5EB.js.map

package/dist/{extract-bundle-COS56ZDO.js → extract-bundle-U4D5LW5W.js}

@@ -4,7 +4,7 @@ import {
   renderImageSection,
   renderIssueReport,
   renderLinkReport
-} from "./chunk-GWRIO6JT.js";
+} from "./chunk-R7EETU7Z.js";
 import {
   getDb
 } from "./chunk-GL4BW4CP.js";
@@ -234,6 +234,7 @@ ${renderImageSection(extras.imageAudit)}`;
       writeFileSync(join(dir, "images.jsonl"), extras.imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"));
       writeFileSync(join(dir, "images-summary.json"), JSON.stringify(extras.imageAudit.summary, null, 2));
     }
+    if (extras.seoAudit) writeFileSync(join(dir, "seo-audit.json"), JSON.stringify(extras.seoAudit, null, 2));
     if (extras.branding) writeFileSync(join(dir, "branding.json"), JSON.stringify(extras.branding, null, 2));
     const artifactFiles = [
       { name: "report.md", type: "text/markdown" },
@@ -249,6 +250,7 @@ ${renderImageSection(extras.imageAudit)}`;
       artifactFiles.push({ name: "images.jsonl", type: "application/x-ndjson" });
       artifactFiles.push({ name: "images-summary.json", type: "application/json" });
     }
+    if (extras.seoAudit) artifactFiles.push({ name: "seo-audit.json", type: "application/json" });
     if (extras.branding) artifactFiles.push({ name: "branding.json", type: "application/json" });
     const zip = new ZipFile();
     const zipOut = createWriteStream(join(dir, "bundle.zip"));
@@ -273,4 +275,4 @@ ${renderImageSection(extras.imageAudit)}`;
 export {
   assembleExtractArtifacts
 };
-//# sourceMappingURL=extract-bundle-COS56ZDO.js.map
+//# sourceMappingURL=extract-bundle-U4D5LW5W.js.map

package/dist/extract-bundle-U4D5LW5W.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/api/extract-bundle.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'\nimport { join } from 'node:path'\nimport { tmpdir } from 'node:os'\nimport { createGzip } from 'node:zlib'\nimport { once } from 'node:events'\nimport { ZipFile } from 'yazl'\nimport { getDb } from './db.js'\nimport { getBlobStore, type StoredArtifact } from './blob-store.js'\nimport { computeIssues, renderIssueReport } from './seo-issues.js'\nimport { renderLinkReport, type ExtDomainRow, type LinkReport } from './seo-link-report.js'\nimport type { PageLinkMetrics } from './seo-link-graph.js'\nimport type { SeoAudit } from './seo-audit.js'\nimport type { ExtractJobRow } from './site-extract-repository.js'\nimport type { PageData } from './site-extractor.js'\nimport { renderImageSection, type ImageAudit } from './image-audit.js'\n\nconst CHUNK = 400\n\nfunction normalize(u: string): string {\n  return u.split('#')[0].replace(/\\/$/, '')\n}\n\nfunction registrableDomain(host: string): string {\n  const h = host.replace(/^www\\./, '').toLowerCase()\n  const parts = h.split('.')\n  return parts.length <= 2 ? h : parts.slice(-2).join('.')\n}\n\nfunction slugFactory() {\n  const counts = new Map<string, number>()\n  return (url: string): string => {\n    const base = url.replace(/^https?:\\/\\//, '').replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80) || 'page'\n    const n = counts.get(base) ?? 0\n    counts.set(base, n + 1)\n    return n ? `${base}-${n}` : base\n  }\n}\n\nasync function* pageChunks(jobId: string): AsyncGenerator<PageData[]> {\n  const db = getDb()\n  for (let offset = 0; ; offset += CHUNK) {\n    const res = await db.execute({\n      sql: `SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid LIMIT ${CHUNK} OFFSET ?`,\n      args: [jobId, offset],\n    })\n    if (!res.rows.length) return\n    yield res.rows.map(r => JSON.parse(String((r as unknown as Record<string, unknown>).page)) as PageData)\n  }\n}\n\nexport interface AssembleExtras {\n  branding?: unknown\n  imageAudit?: ImageAudit | null\n  seoAudit?: SeoAudit | null\n}\n\nexport async function assembleExtractArtifacts(job: ExtractJobRow, extras: AssembleExtras = {}): Promise<StoredArtifact[]> {\n  const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`)\n  mkdirSync(join(dir, 'pages'), { recursive: true })\n  try {\n    const metas: PageData[] = []\n    const statusByUrl = new Map<string, number | null>()\n    for await (const chunk of pageChunks(job.id)) {\n      for (const p of chunk) {\n        statusByUrl.set(normalize(p.url), p.status)\n        const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...slim } = p\n        metas.push(slim as unknown as PageData)\n      }\n    }\n\n    const gz = createGzip()\n    const linksOut = createWriteStream(join(dir, 'links.jsonl.gz'))\n    gz.pipe(linksOut)\n    const gzWrite = async (line: string) => { if (!gz.write(line)) await once(gz, 'drain') }\n\n    const slug = slugFactory()\n    const inboundFrom = new Map<string, Set<string>>()\n    const anchorCounts = new Map<string, Map<string, number>>()\n    const adjacency = new Map<string, Set<string>>()\n    const outCounts = new Map<string, { int: number; ext: number }>()\n    const domMap = new Map<string, { links: number; nofollow: number; pages: Set<string> }>()\n    const brokenLinkPages = new Set<string>()\n    let internalTotal = 0\n    let externalTotal = 0\n    let brokenInternal = 0\n    let pagesWithBody = 0\n    let siteReg = ''\n    try { siteReg = registrableDomain(new URL(job.startUrl).hostname) } catch { siteReg = '' }\n\n    for await (const chunk of pageChunks(job.id)) {\n      for (const p of chunk) {\n        if (p.bodyMarkdown) {\n          pagesWithBody++\n          writeFileSync(join(dir, 'pages', slug(p.url) + '.md'),\n            `# ${p.title ?? p.url}\\n\\nURL: ${p.url}\\nStatus: ${p.status}\\n\\n---\\n\\n${p.bodyMarkdown}\\n`)\n        }\n        const from = normalize(p.url)\n        const oc = { int: 0, ext: 0 }\n        for (const l of p.outlinks ?? []) {\n          const nofollow = (l.rel ?? '').split(/\\s+/).includes('nofollow')\n          const to = normalize(l.href)\n          const targetStatus = statusByUrl.get(to) ?? null\n          await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + '\\n')\n          if (l.internal) {\n            oc.int++\n            internalTotal++\n            if (!inboundFrom.has(to)) inboundFrom.set(to, new Set())\n            inboundFrom.get(to)!.add(from)\n            if (l.anchor) {\n              if (!anchorCounts.has(to)) anchorCounts.set(to, new Map())\n              const ac = anchorCounts.get(to)!\n              ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1)\n            }\n            if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {\n              if (!adjacency.has(from)) adjacency.set(from, new Set())\n              adjacency.get(from)!.add(to)\n            }\n            if (targetStatus != null && targetStatus >= 400) {\n              brokenInternal++\n              brokenLinkPages.add(p.url)\n            }\n          } else {\n            oc.ext++\n            let domain = ''\n            try { domain = registrableDomain(new URL(l.href).hostname) } catch { continue }\n            if (!domain || domain === siteReg) continue\n            externalTotal++\n            const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: new Set<string>() }\n            d.links++\n            if (nofollow) d.nofollow++\n            d.pages.add(p.url)\n            domMap.set(domain, d)\n          }\n        }\n        outCounts.set(p.url, oc)\n      }\n    }\n    gz.end()\n    await once(linksOut, 'finish')\n\n    const depth = new Map<string, number>()\n    const start = normalize(job.startUrl)\n    depth.set(start, 0)\n    const queue: string[] = [start]\n    while (queue.length > 0) {\n      const cur = queue.shift()!\n      const d = depth.get(cur)!\n      for (const next of adjacency.get(cur) ?? []) {\n        if (!depth.has(next)) { depth.set(next, d + 1); queue.push(next) }\n      }\n    }\n\n    const metrics = new Map<string, PageLinkMetrics>()\n    for (const p of metas) {\n      const key = normalize(p.url)\n      const inbound = inboundFrom.get(key) ?? new Set()\n      const ac = anchorCounts.get(key)\n      const topAnchors = ac\n        ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(e => e[0])\n        : []\n      const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 }\n      metrics.set(p.url, {\n        url: p.url,\n        inlinks: inbound.size,\n        uniqueInlinks: inbound.size,\n        outlinksInternal: oc.int,\n        outlinksExternal: oc.ext,\n        crawlDepth: depth.has(key) ? depth.get(key)! : null,\n        orphan: inbound.size === 0 && key !== start,\n        topAnchors,\n      })\n    }\n\n    const issues = computeIssues(metas, metrics, brokenLinkPages)\n    let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics)\n    if (extras.imageAudit) reportMd += `\\n\\n${renderImageSection(extras.imageAudit)}`\n\n    const metricValues = [...metrics.values()]\n    const round = (n: number) => Math.round(n * 10) / 10\n    const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 }\n    let sumInlinks = 0\n    let sumOutlinks = 0\n    for (const m of metricValues) {\n      sumInlinks += m.inlinks\n      sumOutlinks += m.outlinksInternal + m.outlinksExternal\n      if (m.inlinks === 0) distribution.zero++\n      else if (m.inlinks <= 2) distribution.oneToTwo++\n      else if (m.inlinks <= 10) distribution.threeToTen++\n      else distribution.elevenPlus++\n    }\n    const externalDomains: ExtDomainRow[] = [...domMap.entries()]\n      .map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size }))\n      .sort((a, b) => b.links - a.links)\n    const linkReport: LinkReport = {\n      summary: {\n        internal: {\n          totalLinks: internalTotal,\n          pages: metas.length,\n          orphans: metricValues.filter(m => m.orphan).length,\n          brokenInternal,\n          avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,\n          avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,\n          distribution,\n          topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20)\n            .map(m => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal })),\n        },\n        external: {\n          totalLinks: externalTotal,\n          uniqueDomains: externalDomains.length,\n          topDomains: externalDomains.slice(0, 20),\n        },\n      },\n      externalDomains,\n    }\n\n    const pagesOut = createWriteStream(join(dir, 'pages.jsonl'))\n    for (const p of metas) {\n      const m = metrics.get(p.url)\n      const row = { ...(p as unknown as Record<string, unknown>), inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false }\n      if (!pagesOut.write(JSON.stringify(row) + '\\n')) await once(pagesOut, 'drain')\n    }\n    pagesOut.end()\n    await once(pagesOut, 'finish')\n\n    writeFileSync(join(dir, 'report.md'), reportMd)\n    writeFileSync(join(dir, 'issues.json'), JSON.stringify(issues, null, 2))\n    writeFileSync(join(dir, 'link-metrics.jsonl'), metricValues.map(m => JSON.stringify(m)).join('\\n'))\n    writeFileSync(join(dir, 'link-report.md'), renderLinkReport(linkReport))\n    writeFileSync(join(dir, 'links-summary.json'), JSON.stringify(linkReport.summary, null, 2))\n    writeFileSync(join(dir, 'external-domains.json'), JSON.stringify(linkReport.externalDomains, null, 2))\n    if (extras.imageAudit) {\n      writeFileSync(join(dir, 'images.jsonl'), extras.imageAudit.rows.map(r => JSON.stringify(r)).join('\\n'))\n      writeFileSync(join(dir, 'images-summary.json'), JSON.stringify(extras.imageAudit.summary, null, 2))\n    }\n    if (extras.seoAudit) writeFileSync(join(dir, 'seo-audit.json'), JSON.stringify(extras.seoAudit, null, 2))\n    if (extras.branding) writeFileSync(join(dir, 'branding.json'), JSON.stringify(extras.branding, null, 2))\n\n    const artifactFiles: Array<{ name: string; type: string }> = [\n      { name: 'report.md', type: 'text/markdown' },\n      { name: 'issues.json', type: 'application/json' },\n      { name: 'pages.jsonl', type: 'application/x-ndjson' },\n      { name: 'links.jsonl.gz', type: 'application/gzip' },\n      { name: 'link-metrics.jsonl', type: 'application/x-ndjson' },\n      { name: 'link-report.md', type: 'text/markdown' },\n      { name: 'links-summary.json', type: 'application/json' },\n      { name: 'external-domains.json', type: 'application/json' },\n    ]\n    if (extras.imageAudit) {\n      artifactFiles.push({ name: 'images.jsonl', type: 'application/x-ndjson' })\n      artifactFiles.push({ name: 'images-summary.json', type: 'application/json' })\n    }\n    if (extras.seoAudit) artifactFiles.push({ name: 'seo-audit.json', type: 'application/json' })\n    if (extras.branding) artifactFiles.push({ name: 'branding.json', type: 'application/json' })\n\n    const zip = new ZipFile()\n    const zipOut = createWriteStream(join(dir, 'bundle.zip'))\n    zip.outputStream.pipe(zipOut)\n    for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name)\n    if (pagesWithBody > 0) {\n      const { readdirSync } = await import('node:fs')\n      for (const f of readdirSync(join(dir, 'pages'))) zip.addFile(join(dir, 'pages', f), `pages/${f}`)\n    }\n    zip.end()\n    await once(zipOut, 'finish')\n\n    const store = getBlobStore()\n    const stored: StoredArtifact[] = []\n    for (const f of [...artifactFiles, { name: 'bundle.zip', type: 'application/zip' }]) {\n      stored.push(await store.put(`extract/${job.id}/${f.name}`, readFileSync(join(dir, f.name)), f.type))\n    }\n    return stored\n  } finally {\n    rmSync(dir, { recursive: true, force: true })\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAAA,SAAS,mBAAmB,WAAW,cAAc,QAAQ,qBAAqB;AAClF,SAAS,YAAY;AACrB,SAAS,cAAc;AACvB,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AACrB,SAAS,eAAe;AAWxB,IAAM,QAAQ;AAEd,SAAS,UAAU,GAAmB;AACpC,SAAO,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,OAAO,EAAE;AAC1C;AAEA,SAAS,kBAAkB,MAAsB;AAC/C,QAAM,IAAI,KAAK,QAAQ,UAAU,EAAE,EAAE,YAAY;AACjD,QAAM,QAAQ,EAAE,MAAM,GAAG;AACzB,SAAO,MAAM,UAAU,IAAI,IAAI,MAAM,MAAM,EAAE,EAAE,KAAK,GAAG;AACzD;AAEA,SAAS,cAAc;AACrB,QAAM,SAAS,oBAAI,IAAoB;AACvC,SAAO,CAAC,QAAwB;AAC9B,UAAM,OAAO,IAAI,QAAQ,gBAAgB,EAAE,EAAE,QAAQ,kBAAkB,GAAG,EAAE,QAAQ,YAAY,EAAE,EAAE,MAAM,GAAG,EAAE,KAAK;AACpH,UAAM,IAAI,OAAO,IAAI,IAAI,KAAK;AAC9B,WAAO,IAAI,MAAM,IAAI,CAAC;AACtB,WAAO,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK;AAAA,EAC9B;AACF;AAEA,gBAAgB,WAAW,OAA2C;AACpE,QAAM,KAAK,MAAM;AACjB,WAAS,SAAS,KAAK,UAAU,OAAO;AACtC,UAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,MAC3B,KAAK,6EAA6E,KAAK;AAAA,MACvF,MAAM,CAAC,OAAO,MAAM;AAAA,IACtB,CAAC;AACD,QAAI,CAAC,IAAI,KAAK,OAAQ;AACtB,UAAM,IAAI,KAAK,IAAI,OAAK,KAAK,MAAM,OAAQ,EAAyC,IAAI,CAAC,CAAa;AAAA,EACxG;AACF;AAQA,eAAsB,yBAAyB,KAAoB,SAAyB,CAAC,GAA8B;AACzH,QAAM,MAAM,KAAK,OAAO,GAAG,kBAAkB,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE;AACnE,YAAU,KAAK,KAAK,OAAO,GAAG,EAAE,WAAW,KAAK,CAAC;AACjD,MAAI;AACF,UAAM,QAAoB,CAAC;AAC3B,UAAM,cAAc,oBAAI,IAA2B;AACnD,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,oBAAY,IAAI,UAAU,EAAE,GAAG,GAAG,EAAE,MAAM;AAC1C,cAAM,EAAE,cAAc,IAAI,QAAQ,IAAI,UAAU,IAAI,GAAG,KAAK,IAAI;AAChE,cAAM,KAAK,IAA2B;AAAA,MACxC;AAAA,IACF;AAEA,UAAM,KAAK,WAAW;AACtB,UAAM,WAAW,kBAAkB,KAAK,KAAK,gBAAgB,CAAC;AAC9D,OAAG,KAAK,QAAQ;AAChB,UAAM,UAAU,OAAO,SAAiB;AAAE,UAAI,CAAC,GAAG,MAAM,IAAI,EAAG,OAAM,KAAK,IAAI,OAAO;AAAA,IAAE;AAEvF,UAAM,OAAO,YAAY;AACzB,UAAM,cAAc,oBAAI,IAAyB;AACjD,UAAM,eAAe,oBAAI,IAAiC;AAC1D,UAAM,YAAY,oBAAI,IAAyB;AAC/C,UAAM,YAAY,oBAAI,IAA0C;AAChE,UAAM,SAAS,oBAAI,IAAqE;AACxF,UAAM,kBAAkB,oBAAI,IAAY;AACxC,QAAI,gBAAgB;AACpB,QAAI,gBAAgB;AACpB,QAAI,iBAAiB;AACrB,QAAI,gBAAgB;AACpB,QAAI,UAAU;AACd,QAAI;AAAE,gBAAU,kBAAkB,IAAI,IAAI,IAAI,QAAQ,EAAE,QAAQ;AAAA,IAAE,QAAQ;AAAE,gBAAU;AAAA,IAAG;AAEzF,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,YAAI,EAAE,cAAc;AAClB;AACA;AAAA,YAAc,KAAK,KAAK,SAAS,KAAK,EAAE,GAAG,IAAI,KAAK;AAAA,YAClD,KAAK,EAAE,SAAS,EAAE,GAAG;AAAA;AAAA,OAAY,EAAE,GAAG;AAAA,UAAa,EAAE,MAAM;AAAA;AAAA;AAAA;AAAA,EAAc,EAAE,YAAY;AAAA;AAAA,UAAI;AAAA,QAC/F;AACA,cAAM,OAAO,UAAU,EAAE,GAAG;AAC5B,cAAM,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AAC5B,mBAAW,KAAK,EAAE,YAAY,CAAC,GAAG;AAChC,gBAAM,YAAY,EAAE,OAAO,IAAI,MAAM,KAAK,EAAE,SAAS,UAAU;AAC/D,gBAAM,KAAK,UAAU,EAAE,IAAI;AAC3B,gBAAM,eAAe,YAAY,IAAI,EAAE,KAAK;AAC5C,gBAAM,QAAQ,KAAK,UAAU,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,UAAU,EAAE,UAAU,UAAU,aAAa,CAAC,IAAI,IAAI;AAC5I,cAAI,EAAE,UAAU;AACd,eAAG;AACH;AACA,gBAAI,CAAC,YAAY,IAAI,EAAE,EAAG,aAAY,IAAI,IAAI,oBAAI,IAAI,CAAC;AACvD,wBAAY,IAAI,EAAE,EAAG,IAAI,IAAI;AAC7B,gBAAI,EAAE,QAAQ;AACZ,kBAAI,CAAC,aAAa,IAAI,EAAE,EAAG,cAAa,IAAI,IAAI,oBAAI,IAAI,CAAC;AACzD,oBAAM,KAAK,aAAa,IAAI,EAAE;AAC9B,iBAAG,IAAI,EAAE,SAAS,GAAG,IAAI,EAAE,MAAM,KAAK,KAAK,CAAC;AAAA,YAC9C;AACA,gBAAI,CAAC,aAAa,YAAY,IAAI,EAAE,MAAM,OAAO,CAAC,YAAY,IAAI,EAAE,IAAI;AACtE,kBAAI,CAAC,UAAU,IAAI,IAAI,EAAG,WAAU,IAAI,MAAM,oBAAI,IAAI,CAAC;AACvD,wBAAU,IAAI,IAAI,EAAG,IAAI,EAAE;AAAA,YAC7B;AACA,gBAAI,gBAAgB,QAAQ,gBAAgB,KAAK;AAC/C;AACA,8BAAgB,IAAI,EAAE,GAAG;AAAA,YAC3B;AAAA,UACF,OAAO;AACL,eAAG;AACH,gBAAI,SAAS;AACb,gBAAI;AAAE,uBAAS,kBAAkB,IAAI,IAAI,EAAE,IAAI,EAAE,QAAQ;AAAA,YAAE,QAAQ;AAAE;AAAA,YAAS;AAC9E,gBAAI,CAAC,UAAU,WAAW,QAAS;AACnC;AACA,kBAAM,IAAI,OAAO,IAAI,MAAM,KAAK,EAAE,OAAO,GAAG,UAAU,GAAG,OAAO,oBAAI,IAAY,EAAE;AAClF,cAAE;AACF,gBAAI,SAAU,GAAE;AAChB,cAAE,MAAM,IAAI,EAAE,GAAG;AACjB,mBAAO,IAAI,QAAQ,CAAC;AAAA,UACtB;AAAA,QACF;AACA,kBAAU,IAAI,EAAE,KAAK,EAAE;AAAA,MACzB;AAAA,IACF;AACA,OAAG,IAAI;AACP,UAAM,KAAK,UAAU,QAAQ;AAE7B,UAAM,QAAQ,oBAAI,IAAoB;AACtC,UAAM,QAAQ,UAAU,IAAI,QAAQ;AACpC,UAAM,IAAI,OAAO,CAAC;AAClB,UAAM,QAAkB,CAAC,KAAK;AAC9B,WAAO,MAAM,SAAS,GAAG;AACvB,YAAM,MAAM,MAAM,MAAM;AACxB,YAAM,IAAI,MAAM,IAAI,GAAG;AACvB,iBAAW,QAAQ,UAAU,IAAI,GAAG,KAAK,CAAC,GAAG;AAC3C,YAAI,CAAC,MAAM,IAAI,IAAI,GAAG;AAAE,gBAAM,IAAI,MAAM,IAAI,CAAC;AAAG,gBAAM,KAAK,IAAI;AAAA,QAAE;AAAA,MACnE;AAAA,IACF;AAEA,UAAM,UAAU,oBAAI,IAA6B;AACjD,eAAW,KAAK,OAAO;AACrB,YAAM,MAAM,UAAU,EAAE,GAAG;AAC3B,YAAM,UAAU,YAAY,IAAI,GAAG,KAAK,oBAAI,IAAI;AAChD,YAAM,KAAK,aAAa,IAAI,GAAG;AAC/B,YAAM,aAAa,KACf,CAAC,GAAG,GAAG,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,IAAI,OAAK,EAAE,CAAC,CAAC,IACvE,CAAC;AACL,YAAM,KAAK,UAAU,IAAI,EAAE,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AACpD,cAAQ,IAAI,EAAE,KAAK;AAAA,QACjB,KAAK,EAAE;AAAA,QACP,SAAS,QAAQ;AAAA,QACjB,eAAe,QAAQ;AAAA,QACvB,kBAAkB,GAAG;AAAA,QACrB,kBAAkB,GAAG;AAAA,QACrB,YAAY,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,IAAK;AAAA,QAC/C,QAAQ,QAAQ,SAAS,KAAK,QAAQ;AAAA,QACtC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,cAAc,OAAO,SAAS,eAAe;AAC5D,QAAI,WAAW,kBAAkB,IAAI,UAAU,OAAO,QAAQ,OAAO;AACrE,QAAI,OAAO,WAAY,aAAY;AAAA;AAAA,EAAO,mBAAmB,OAAO,UAAU,CAAC;AAE/E,UAAM,eAAe,CAAC,GAAG,QAAQ,OAAO,CAAC;AACzC,UAAM,QAAQ,CAAC,MAAc,KAAK,MAAM,IAAI,EAAE,IAAI;AAClD,UAAM,eAAe,EAAE,MAAM,GAAG,UAAU,GAAG,YAAY,GAAG,YAAY,EAAE;AAC1E,QAAI,aAAa;AACjB,QAAI,cAAc;AAClB,eAAW,KAAK,cAAc;AAC5B,oBAAc,EAAE;AAChB,qBAAe,EAAE,mBAAmB,EAAE;AACtC,UAAI,EAAE,YAAY,EAAG,cAAa;AAAA,eACzB,EAAE,WAAW,EAAG,cAAa;AAAA,eAC7B,EAAE,WAAW,GAAI,cAAa;AAAA,UAClC,cAAa;AAAA,IACpB;AACA,UAAM,kBAAkC,CAAC,GAAG,OAAO,QAAQ,CAAC,EACzD,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,OAAO,EAAE,MAAM,KAAK,EAAE,EAC5F,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACnC,UAAM,aAAyB;AAAA,MAC7B,SAAS;AAAA,QACP,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,OAAO,MAAM;AAAA,UACb,SAAS,aAAa,OAAO,OAAK,EAAE,MAAM,EAAE;AAAA,UAC5C;AAAA,UACA,YAAY,MAAM,SAAS,MAAM,aAAa,MAAM,MAAM,IAAI;AAAA,UAC9D,aAAa,MAAM,SAAS,MAAM,cAAc,MAAM,MAAM,IAAI;AAAA,UAChE;AAAA,UACA,cAAc,CAAC,GAAG,YAAY,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,EAAE,EAC9E,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,SAAS,EAAE,SAAS,kBAAkB,EAAE,kBAAkB,kBAAkB,EAAE,iBAAiB,EAAE;AAAA,QAC9H;AAAA,QACA,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,eAAe,gBAAgB;AAAA,UAC/B,YAAY,gBAAgB,MAAM,GAAG,EAAE;AAAA,QACzC;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,UAAM,WAAW,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAC3D,eAAW,KAAK,OAAO;AACrB,YAAM,IAAI,QAAQ,IAAI,EAAE,GAAG;AAC3B,YAAM,MAAM,EAAE,GAAI,GAA0C,SAAS,GAAG,WAAW,GAAG,YAAY,GAAG,cAAc,MAAM,QAAQ,GAAG,UAAU,MAAM;AACpJ,UAAI,CAAC,SAAS,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,EAAG,OAAM,KAAK,UAAU,OAAO;AAAA,IAC/E;AACA,aAAS,IAAI;AACb,UAAM,KAAK,UAAU,QAAQ;AAE7B,kBAAc,KAAK,KAAK,WAAW,GAAG,QAAQ;AAC9C,kBAAc,KAAK,KAAK,aAAa,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,aAAa,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAClG,kBAAc,KAAK,KAAK,gBAAgB,GAAG,iBAAiB,UAAU,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,KAAK,UAAU,WAAW,SAAS,MAAM,CAAC,CAAC;AAC1F,kBAAc,KAAK,KAAK,uBAAuB,GAAG,KAAK,UAAU,WAAW,iBAAiB,MAAM,CAAC,CAAC;AACrG,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,KAAK,cAAc,GAAG,OAAO,WAAW,KAAK,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACtG,oBAAc,KAAK,KAAK,qBAAqB,GAAG,KAAK,UAAU,OAAO,WAAW,SAAS,MAAM,CAAC,CAAC;AAAA,IACpG;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,gBAAgB,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AACxG,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,eAAe,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AAEvG,UAAM,gBAAuD;AAAA,MAC3D,EAAE,MAAM,aAAa,MAAM,gBAAgB;AAAA,MAC3C,EAAE,MAAM,eAAe,MAAM,mBAAmB;AAAA,MAChD,EAAE,MAAM,eAAe,MAAM,uBAAuB;AAAA,MACpD,EAAE,MAAM,kBAAkB,MAAM,mBAAmB;AAAA,MACnD,EAAE,MAAM,sBAAsB,MAAM,uBAAuB;AAAA,MAC3D,EAAE,MAAM,kBAAkB,MAAM,gBAAgB;AAAA,MAChD,EAAE,MAAM,sBAAsB,MAAM,mBAAmB;AAAA,MACvD,EAAE,MAAM,yBAAyB,MAAM,mBAAmB;AAAA,IAC5D;AACA,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,EAAE,MAAM,gBAAgB,MAAM,uBAAuB,CAAC;AACzE,oBAAc,KAAK,EAAE,MAAM,uBAAuB,MAAM,mBAAmB,CAAC;AAAA,IAC9E;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,kBAAkB,MAAM,mBAAmB,CAAC;AAC5F,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,iBAAiB,MAAM,mBAAmB,CAAC;AAE3F,UAAM,MAAM,IAAI,QAAQ;AACxB,UAAM,SAAS,kBAAkB,KAAK,KAAK,YAAY,CAAC;AACxD,QAAI,aAAa,KAAK,MAAM;AAC5B,eAAW,KAAK,cAAe,KAAI,QAAQ,KAAK,KAAK,EAAE,IAAI,GAAG,EAAE,IAAI;AACpE,QAAI,gBAAgB,GAAG;AACrB,YAAM,EAAE,YAAY,IAAI,MAAM,OAAO,IAAS;AAC9C,iBAAW,KAAK,YAAY,KAAK,KAAK,OAAO,CAAC,EAAG,KAAI,QAAQ,KAAK,KAAK,SAAS,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAClG;AACA,QAAI,IAAI;AACR,UAAM,KAAK,QAAQ,QAAQ;AAE3B,UAAM,QAAQ,aAAa;AAC3B,UAAM,SAA2B,CAAC;AAClC,eAAW,KAAK,CAAC,GAAG,eAAe,EAAE,MAAM,cAAc,MAAM,kBAAkB,CAAC,GAAG;AACnF,aAAO,KAAK,MAAM,MAAM,IAAI,WAAW,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,aAAa,KAAK,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC;AAAA,IACrG;AACA,WAAO;AAAA,EACT,UAAE;AACA,WAAO,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAAA,EAC9C;AACF;","names":[]}

package/dist/index.cjs

@@ -1688,13 +1688,41 @@ var PAAExtractor = class {
         }
         return parts.join(" ").replace(/\s+/g, " ").trim();
       }
-      return Array.from(document.querySelectorAll(selectors.item)).map((pair) => ({
-        question: pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "",
-        answer: cleanText(pair.querySelector(selectors.answerContainer)) || void 0,
-        sourceTitle: pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0,
-        sourceSite: pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0,
-        sourceCite: pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0
-      }));
+      return Array.from(document.querySelectorAll(selectors.item)).map((pair) => {
+        let sourceTitle = pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0;
+        let sourceSite = pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0;
+        let sourceCite = pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0;
+        if (!sourceCite) {
+          const seenHrefs = /* @__PURE__ */ new Set();
+          const anchors = [];
+          for (const a of Array.from(pair.querySelectorAll("a[href]"))) {
+            const href = a.href;
+            if (!/^https?:\/\//.test(href) || seenHrefs.has(href)) continue;
+            let host = "";
+            try {
+              host = new URL(href).hostname;
+            } catch {
+              continue;
+            }
+            if (/(^|\.)google(usercontent)?\.[a-z.]+$/.test(host)) continue;
+            seenHrefs.add(href);
+            anchors.push({ text: (a.textContent || "").trim(), href, host });
+          }
+          const primary = anchors.find((x) => x.text.length > 2) ?? anchors[0];
+          if (primary) {
+            sourceTitle = sourceTitle || primary.text || void 0;
+            sourceSite = sourceSite || primary.host.replace(/^www\./, "");
+            sourceCite = primary.href;
+          }
+        }
+        return {
+          question: pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "",
+          answer: cleanText(pair.querySelector(selectors.answerContainer)) || void 0,
+          sourceTitle,
+          sourceSite,
+          sourceCite
+        };
+      });
     }, sels);
     return raw.flatMap((item) => {
       const cleaned = {
@@ -1736,54 +1764,115 @@ var PAAExtractor = class {
     const seenQs = /* @__PURE__ */ new Set();
     const orderedQs = [];
     const results = [];
-    const readAllQs = () => page.evaluate(
-      ({ sel, dataQ, dataInitQ, questionEl }) => Array.from(document.querySelectorAll(sel)).map(
-        (el) => el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || ""
-      ).filter(Boolean),
-      { sel: PAASelectors.item, dataQ: PAASelectors.itemDataQ, dataInitQ: PAASelectors.itemDataInitQ, questionEl: PAASelectors.itemQuestionEl }
+    const abandonedNeverReclick = /* @__PURE__ */ new Set();
+    const clickedOnceEver = /* @__PURE__ */ new Set();
+    const itemArgs = {
+      sel: PAASelectors.item,
+      dataQ: PAASelectors.itemDataQ,
+      dataInitQ: PAASelectors.itemDataInitQ,
+      questionEl: PAASelectors.itemQuestionEl,
+      answerSel: PAASelectors.answerContainer,
+      expandedClass: PAASelectors.expandedClass,
+      clickTarget: PAASelectors.clickTarget
+    };
+    const readItemStates = () => page.evaluate(
+      ({ sel, dataQ, dataInitQ, questionEl, expandedClass, answerSel }) => {
+        const answerText = (root) => {
+          if (!root) return "";
+          let out = "";
+          const walk = (n) => {
+            for (const c of Array.from(n.childNodes)) {
+              if (c.nodeType === Node.TEXT_NODE) out += c.textContent || "";
+              else if (c.nodeType === Node.ELEMENT_NODE && c.tagName !== "STYLE" && c.tagName !== "SCRIPT") walk(c);
+            }
+          };
+          walk(root);
+          return out.replace(/\s+/g, " ").trim();
+        };
+        return Array.from(document.querySelectorAll(sel)).map((el) => ({
+          q: el.getAttribute(dataQ) || el.getAttribute(dataInitQ) || el.querySelector(questionEl)?.innerText?.trim() || "",
+          done: el.classList.contains(expandedClass) || answerText(el.querySelector(answerSel)).length > 30
+        })).filter((s) => s.q);
+      },
+      itemArgs
     );
+    const expandOneItemSerially = async (q) => {
+      const clicked = await page.evaluate(
+        ({ sel, dataQ, dataInitQ, clickTarget, q: target }) => {
+          const item = Array.from(document.querySelectorAll(sel)).find((el) => (el.getAttribute(dataQ) || el.getAttribute(dataInitQ)) === target);
+          const btn = item?.querySelector(clickTarget);
+          if (!btn) return false;
+          btn.scrollIntoView({ block: "center" });
+          btn.click();
+          return true;
+        },
+        { ...itemArgs, q }
+      ).catch(() => false);
+      if (!clicked) return "failed";
+      try {
+        const outcome = await page.waitForFunction(
+          ({ sel, dataQ, dataInitQ, answerSel, q: target }) => {
+            const answerText = (root) => {
+              if (!root) return "";
+              let out = "";
+              const walk = (n) => {
+                for (const c of Array.from(n.childNodes)) {
+                  if (c.nodeType === Node.TEXT_NODE) out += c.textContent || "";
+                  else if (c.nodeType === Node.ELEMENT_NODE && c.tagName !== "STYLE" && c.tagName !== "SCRIPT") walk(c);
+                }
+              };
+              walk(root);
+              return out.replace(/\s+/g, " ").trim();
+            };
+            const item = Array.from(document.querySelectorAll(sel)).find((el) => (el.getAttribute(dataQ) || el.getAttribute(dataInitQ)) === target);
+            if (!item) return "gone";
+            const text = answerText(item.querySelector(answerSel));
+            if (/an error has occurred/i.test(text.slice(0, 80))) return "error";
+            if (text.length > 30) return "ok";
+            return false;
+          },
+          { ...itemArgs, q },
+          { timeout: 6500, polling: 200 }
+        );
+        return await outcome.jsonValue() === "ok" ? "ok" : "failed";
+      } catch {
+        return "failed";
+      }
+    };
     let round = 0;
-    while (seenQs.size < options.maxQuestions) {
+    let growthWaits = 0;
+    while (true) {
       if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
       this.throwIfAborted(signal);
-      await this.throwIfCaptcha(page, "Google PAA expansion");
-      const beforeQs = await readAllQs();
-      if (beforeQs.length >= options.maxQuestions) break;
-      const unexpandedSel = `${PAASelectors.item}:not(.${PAASelectors.expandedClass}) ${PAASelectors.clickTarget}`;
-      const unexpandedCount = await page.locator(unexpandedSel).count();
-      if (unexpandedCount === 0) break;
-      this.reporter.onDepth(++round);
-      for (let ci = 0; ci < unexpandedCount; ci++) {
-        if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
-        this.throwIfAborted(signal);
-        try {
-          const btn = page.locator(unexpandedSel).first();
-          await btn.scrollIntoViewIfNeeded();
-          await btn.hover({ force: true });
-          await page.waitForTimeout(100);
-          await btn.click({ force: true });
-          await page.waitForTimeout(500);
-        } catch {
+      const states = await readItemStates();
+      for (const s of states) {
+        if (!seenQs.has(s.q)) {
+          seenQs.add(s.q);
+          orderedQs.push(s.q);
         }
       }
-      await page.waitForFunction(
-        ({ sel, min }) => document.querySelectorAll(sel).length > min,
-        { sel: PAASelectors.item, min: beforeQs.length },
-        { timeout: 5e3 }
-      ).catch(() => {
-      });
-      await this.throwIfCaptcha(page, "Google PAA expansion");
-      const afterQs = await readAllQs();
-      if (afterQs.length === beforeQs.length) break;
-      for (const q of afterQs) {
-        if (!seenQs.has(q)) {
-          seenQs.add(q);
-          orderedQs.push(q);
-        }
+      const kept = new Set(orderedQs.slice(0, options.maxQuestions));
+      const clickable = states.filter((s) => kept.has(s.q) && !clickedOnceEver.has(s.q) && !abandonedNeverReclick.has(s.q));
+      const target = clickable.find((s) => !s.done) ?? (seenQs.size < options.maxQuestions ? clickable[0] : void 0);
+      if (!target) {
+        if (seenQs.size >= options.maxQuestions) break;
+        if (++growthWaits > 1) break;
+        const grew = await page.waitForFunction(
+          ({ sel, min }) => document.querySelectorAll(sel).length > min,
+          { sel: PAASelectors.item, min: states.length },
+          { timeout: 3e3 }
+        ).catch(() => null);
+        if (!grew) break;
+        continue;
       }
+      growthWaits = 0;
+      this.reporter.onDepth(++round);
+      await this.throwIfCaptcha(page, "Google PAA expansion");
+      clickedOnceEver.add(target.q);
+      if (await expandOneItemSerially(target.q) === "failed") abandonedNeverReclick.add(target.q);
     }
     const itemMap = new Map((await this.extractVisibleItems(page)).map((i) => [i.question, i]));
-    await this.fillIncompleteItems(page, orderedQs, itemMap, options);
+    await this.fillIncompleteItems(page, orderedQs, itemMap, options, abandonedNeverReclick);
     for (const q of orderedQs) {
       if (results.length >= options.maxQuestions) break;
       const key = this.normalizeQuestion(q);
@@ -1801,9 +1890,11 @@ var PAAExtractor = class {
     this.completeness.paaWithoutSource = results.filter((r) => !r.source_title && !r.source_site && !r.source_cite).length;
     return results;
   }
-  async fillIncompleteItems(page, orderedQs, itemMap, options) {
+  async fillIncompleteItems(page, orderedQs, itemMap, options, abandonedNeverReclick) {
     const kept = new Set(orderedQs.slice(0, options.maxQuestions));
-    const missingBefore = new Set([...kept].filter((q) => !itemMap.get(q)?.answer));
+    const missingBefore = new Set(
+      [...kept].filter((q) => !itemMap.get(q)?.answer && !abandonedNeverReclick?.has(q))
+    );
     if (missingBefore.size === 0) return;
     for (let round = 0; round < 2; round++) {
       if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
@@ -2195,40 +2286,41 @@ var PAAExtractor = class {
     };
   }
   async captureAIOverviewShareUrl(page, options) {
-    for (let attempt = 0; attempt < 3; attempt++) {
-      if (options?.softDeadlineMs && Date.now() >= options.softDeadlineMs) return null;
-      const clicked = await page.evaluate(() => {
-        const heads = Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"],div')).filter((h) => (h.textContent || "").trim() === "AI Overview");
-        for (const head of heads) {
-          let sec = head;
-          for (let up = 0; up < 8 && sec; up++) {
-            const btn = sec.querySelector('[aria-label="Share"]');
-            if (btn) {
-              btn.scrollIntoView({ block: "center" });
-              btn.click();
-              return true;
-            }
-            sec = sec.parentElement;
+    if (options?.softDeadlineMs && Date.now() >= options.softDeadlineMs) return null;
+    const clicked = await page.evaluate(() => {
+      const heads = Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"],div')).filter((h) => (h.textContent || "").trim() === "AI Overview");
+      for (const head of heads) {
+        let sec = head;
+        for (let up = 0; up < 8 && sec; up++) {
+          const btn = sec.querySelector('[aria-label="Share"]');
+          if (btn) {
+            btn.scrollIntoView({ block: "center" });
+            btn.click();
+            return true;
           }
+          sec = sec.parentElement;
         }
-        return false;
-      }).catch(() => false);
-      if (!clicked) return null;
-      await page.waitForTimeout(4500);
-      const url = await page.evaluate(() => {
+      }
+      return false;
+    }).catch(() => false);
+    if (!clicked) return null;
+    let url = null;
+    try {
+      const found = await page.waitForFunction(() => {
         const w = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
         while (w.nextNode()) {
           const m = (w.currentNode.textContent || "").match(/https:\/\/share\.google\/aimode\/\S+/);
           if (m) return m[0];
         }
-        return null;
-      }).catch(() => null);
-      await page.keyboard.press("Escape").catch(() => {
-      });
-      if (url) return url;
-      await page.waitForTimeout(600);
+        return false;
+      }, void 0, { timeout: 4500, polling: 250 });
+      url = await found.jsonValue();
+    } catch {
+      url = null;
     }
-    return null;
+    await page.keyboard.press("Escape").catch(() => {
+    });
+    return url;
   }
   buildTree(flat, _seed) {
     const roots = [];