mcp-scraper 0.3.11 → 0.3.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +2 -2
  2. package/dist/bin/api-server.cjs +86 -22
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +2 -2
  5. package/dist/bin/browser-agent-stdio-server.cjs +198 -27
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +3 -2
  8. package/dist/bin/browser-agent-stdio-server.js.map +1 -1
  9. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  10. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-cli.js +1 -1
  12. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +254 -56
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -3
  15. package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
  16. package/dist/bin/mcp-scraper-install.cjs +1 -1
  17. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  18. package/dist/bin/mcp-scraper-install.js +1 -1
  19. package/dist/bin/mcp-stdio-server.cjs +38 -11
  20. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  21. package/dist/bin/mcp-stdio-server.js +2 -2
  22. package/dist/bin/paa-harvest.cjs +21 -1
  23. package/dist/bin/paa-harvest.cjs.map +1 -1
  24. package/dist/bin/paa-harvest.js +1 -1
  25. package/dist/chunk-3YGKXXUG.js +133 -0
  26. package/dist/chunk-3YGKXXUG.js.map +1 -0
  27. package/dist/{chunk-UWSG3C5J.js → chunk-4OPKIDON.js} +22 -2
  28. package/dist/chunk-4OPKIDON.js.map +1 -0
  29. package/dist/{chunk-VMH7SRWY.js → chunk-7R7VBQRV.js} +39 -12
  30. package/dist/chunk-7R7VBQRV.js.map +1 -0
  31. package/dist/{chunk-L27GJQV7.js → chunk-BW3DGFNQ.js} +28 -5
  32. package/dist/chunk-BW3DGFNQ.js.map +1 -0
  33. package/dist/chunk-FB5ZJGFN.js +7 -0
  34. package/dist/chunk-FB5ZJGFN.js.map +1 -0
  35. package/dist/index.cjs +21 -1
  36. package/dist/index.cjs.map +1 -1
  37. package/dist/index.js +1 -1
  38. package/dist/{server-3SA5Q4OF.js → server-CUPDJLWM.js} +42 -151
  39. package/dist/server-CUPDJLWM.js.map +1 -0
  40. package/dist/{worker-56IXWOQU.js → worker-FG7ZWEGA.js} +2 -2
  41. package/docs/mcp-tool-craft-lint.generated.md +1 -1
  42. package/docs/mcp-tool-manifest.generated.json +1 -1
  43. package/package.json +1 -1
  44. package/dist/chunk-L27GJQV7.js.map +0 -1
  45. package/dist/chunk-RRE7WVHQ.js +0 -7
  46. package/dist/chunk-RRE7WVHQ.js.map +0 -1
  47. package/dist/chunk-UWSG3C5J.js.map +0 -1
  48. package/dist/chunk-VMH7SRWY.js.map +0 -1
  49. package/dist/server-3SA5Q4OF.js.map +0 -1
  50. /package/dist/{worker-56IXWOQU.js.map → worker-FG7ZWEGA.js.map} +0 -0
package/dist/bin/paa-harvest.js CHANGED
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
3
  harvest
4
- } from "../chunk-UWSG3C5J.js";
4
+ } from "../chunk-4OPKIDON.js";
5
5
  import "../chunk-M2S27J6Z.js";
6
6
  import {
7
7
  browserServiceApiKey
package/dist/chunk-3YGKXXUG.js ADDED
@@ -0,0 +1,133 @@
1
+ // src/services/fanout/export.ts
2
+ import { mkdirSync, writeFileSync } from "fs";
3
+ import { homedir } from "os";
4
+ import { join } from "path";
5
+ import Papa from "papaparse";
6
+ function outputBaseDir() {
7
+ return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join(homedir(), "Downloads", "mcp-scraper");
8
+ }
9
+ function safe(value) {
10
+ return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
11
+ }
12
+ function writeTable(path, rows, delimiter) {
13
+ writeFileSync(path, Papa.unparse(rows.length ? rows : [{}], { delimiter }));
14
+ }
15
+ function exportFanout(enriched) {
16
+ const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
17
+ const outputDir = outputBaseDir();
18
+ const relativeDir = join("fanout", `${stamp}-${safe(enriched.platform)}`);
19
+ const dir = join(outputDir, relativeDir);
20
+ mkdirSync(dir, { recursive: true });
21
+ const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
22
+ const citationRows = enriched.aggregates.citationOrder.map((c) => {
23
+ const src = enriched.citedUrls.find((s) => s.url === c.url);
24
+ return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
25
+ });
26
+ const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
27
+ const browsedOnlyRows = enriched.browsedOnly.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
28
+ const snippetRows = enriched.snippets.map((s) => ({ domain: s.domain, url: s.url, title: s.title, text: s.text }));
29
+ const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
30
+ const relativePaths = {
31
+ relativeTo: "MCP_SCRAPER_OUTPUT_DIR or ~/Downloads/mcp-scraper",
32
+ dir: relativeDir,
33
+ json: join(relativeDir, "fanout.json"),
34
+ queriesCsv: join(relativeDir, "queries.csv"),
35
+ queriesTsv: join(relativeDir, "queries.tsv"),
36
+ citationsCsv: join(relativeDir, "citations.csv"),
37
+ sourcesCsv: join(relativeDir, "sources.csv"),
38
+ browsedOnlyCsv: join(relativeDir, "browsed-only.csv"),
39
+ snippetsCsv: join(relativeDir, "snippets.csv"),
40
+ domainsCsv: join(relativeDir, "domains.csv"),
41
+ report: join(relativeDir, "report.html")
42
+ };
43
+ writeFileSync(join(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2));
44
+ writeTable(join(outputDir, relativePaths.queriesCsv), queryRows, ",");
45
+ writeTable(join(outputDir, relativePaths.queriesTsv), queryRows, " ");
46
+ writeTable(join(outputDir, relativePaths.citationsCsv), citationRows, ",");
47
+ writeTable(join(outputDir, relativePaths.sourcesCsv), sourceRows, ",");
48
+ writeTable(join(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ",");
49
+ writeTable(join(outputDir, relativePaths.snippetsCsv), snippetRows, ",");
50
+ writeTable(join(outputDir, relativePaths.domainsCsv), domainRows, ",");
51
+ writeFileSync(join(outputDir, relativePaths.report), renderReportHtml(enriched));
52
+ return relativePaths;
53
+ }
54
+ function esc(s) {
55
+ return String(s).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
56
+ }
57
+ function renderReportHtml(enriched) {
58
+ const data = JSON.stringify(enriched).replace(/</g, "\\u003c");
59
+ return `<!doctype html>
60
+ <html lang="en">
61
+ <head>
62
+ <meta charset="utf-8" />
63
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
64
+ <title>AI Search Fan-Out \u2014 ${esc(enriched.platform)}</title>
65
+ <style>
66
+ :root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}
67
+ *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}
68
+ .wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}
69
+ h1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}
70
+ .stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}
71
+ .cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}
72
+ .card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
73
+ .card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}
74
+ .bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}
75
+ .bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}
76
+ .bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}
77
+ .tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}
78
+ .tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}
79
+ .chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}
80
+ .chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}
81
+ .chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}
82
+ .row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}
83
+ .row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}
84
+ .tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}
85
+ .dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}
86
+ .pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}
87
+ .copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}
88
+ .hide{display:none}
89
+ </style>
90
+ </head>
91
+ <body><div class="wrap">
92
+ <h1>AI Search Fan-Out</h1>
93
+ <p class="sub" id="sub"></p>
94
+ <div class="stat" id="stat"></div>
95
+ <div class="cards" id="cards"></div>
96
+ <div id="charts"></div>
97
+ <div class="tabs"><button class="tab on" data-tab="q">Queries</button><button class="tab" data-tab="u">URLs</button></div>
98
+ <div class="chips" id="chips"></div>
99
+ <div id="list"></div>
100
+ </div>
101
+ <script>
102
+ const D=${data};
103
+ const el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};
104
+ document.getElementById('sub').textContent=D.platform+' \xB7 '+(D.meta.model||'model n/a')+' \xB7 '+D.meta.rounds+' search round(s) \xB7 prompt: '+(D.prompt||'');
105
+ document.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries \xB7 '+D.counts.browsed+' URLs researched \xB7 '+D.counts.cited+' cited</small>';
106
+ const cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];
107
+ const cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});
108
+ function barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}
109
+ const charts=document.getElementById('charts');
110
+ charts.appendChild(barChart('URL Categories',D.aggregates.byCategory));
111
+ let tab='q',filter=null;
112
+ const chips=document.getElementById('chips'),list=document.getElementById('list');
113
+ function render(){
114
+ chips.innerHTML='';list.innerHTML='';
115
+ if(tab==='q'){
116
+ D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});
117
+ } else {
118
+ const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});
119
+ Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});
120
+ D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class="dom">'+esc(s.domain)+'</span> <span class="pill '+(s.cited?'cite':'brow')+'">'+(s.cited?'cited '+s.timesCited+'\xD7':'browsed')+'</span> <span class="tag">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});
121
+ }
122
+ }
123
+ function esc(s){return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;')}
124
+ document.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});
125
+ render();
126
+ </script>
127
+ </body></html>`;
128
+ }
129
+
130
+ export {
131
+ exportFanout
132
+ };
133
+ //# sourceMappingURL=chunk-3YGKXXUG.js.map
package/dist/chunk-3YGKXXUG.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/services/fanout/export.ts"],"sourcesContent":["import { mkdirSync, writeFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport Papa from 'papaparse'\nimport type { EnrichedFanout } from './classify.js'\n\nexport interface FanoutExportPaths {\n relativeTo: string\n dir: string\n json: string\n queriesCsv: string\n queriesTsv: string\n citationsCsv: string\n sourcesCsv: string\n browsedOnlyCsv: string\n snippetsCsv: string\n domainsCsv: string\n report: string\n}\n\nfunction outputBaseDir(): string {\n return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join(homedir(), 'Downloads', 'mcp-scraper')\n}\n\nfunction safe(value: string): string {\n return value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80) || 'capture'\n}\n\nfunction writeTable(path: string, rows: Array<Record<string, unknown>>, delimiter: ','): void\nfunction writeTable(path: string, rows: Array<Record<string, unknown>>, delimiter: '\\t'): void\nfunction writeTable(path: string, rows: Array<Record<string, unknown>>, delimiter: string): void {\n writeFileSync(path, Papa.unparse(rows.length ? rows : [{}], { delimiter }))\n}\n\nexport function exportFanout(enriched: EnrichedFanout): FanoutExportPaths {\n const stamp = safe(enriched.capturedAt.replace(/[:.]/g, '-'))\n const outputDir = outputBaseDir()\n const relativeDir = join('fanout', `${stamp}-${safe(enriched.platform)}`)\n const dir = join(outputDir, relativeDir)\n mkdirSync(dir, { recursive: true })\n\n const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }))\n const citationRows = enriched.aggregates.citationOrder.map((c) => {\n const src = enriched.citedUrls.find((s) => s.url === c.url)\n return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? '', title: src?.title ?? '' }\n })\n const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? '' }))\n const browsedOnlyRows = enriched.browsedOnly.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? '' }))\n const snippetRows = enriched.snippets.map((s) => ({ domain: s.domain, url: s.url, title: s.title, text: s.text }))\n const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }))\n\n const relativePaths: FanoutExportPaths = {\n relativeTo: 'MCP_SCRAPER_OUTPUT_DIR or ~/Downloads/mcp-scraper',\n dir: relativeDir,\n json: join(relativeDir, 'fanout.json'),\n queriesCsv: join(relativeDir, 'queries.csv'),\n queriesTsv: join(relativeDir, 'queries.tsv'),\n citationsCsv: join(relativeDir, 'citations.csv'),\n sourcesCsv: join(relativeDir, 'sources.csv'),\n browsedOnlyCsv: join(relativeDir, 'browsed-only.csv'),\n snippetsCsv: join(relativeDir, 'snippets.csv'),\n domainsCsv: join(relativeDir, 'domains.csv'),\n report: join(relativeDir, 'report.html'),\n }\n\n writeFileSync(join(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2))\n writeTable(join(outputDir, relativePaths.queriesCsv), queryRows, ',')\n writeTable(join(outputDir, relativePaths.queriesTsv), queryRows, '\\t')\n writeTable(join(outputDir, relativePaths.citationsCsv), citationRows, ',')\n writeTable(join(outputDir, relativePaths.sourcesCsv), sourceRows, ',')\n writeTable(join(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ',')\n writeTable(join(outputDir, relativePaths.snippetsCsv), snippetRows, ',')\n writeTable(join(outputDir, relativePaths.domainsCsv), domainRows, ',')\n writeFileSync(join(outputDir, relativePaths.report), renderReportHtml(enriched))\n\n return relativePaths\n}\n\nfunction esc(s: string): string {\n return String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/\"/g, '&quot;')\n}\n\nexport function renderReportHtml(enriched: EnrichedFanout): string {\n const data = JSON.stringify(enriched).replace(/</g, '\\\\u003c')\n return `<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\" />\n<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n<title>AI Search Fan-Out — ${esc(enriched.platform)}</title>\n<style>\n:root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}\n*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}\n.wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}\nh1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}\n.stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}\n.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}\n.card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}\n.card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}\n.bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}\n.bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}\n.bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}\n.tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}\n.tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}\n.chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}\n.chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}\n.chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}\n.row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}\n.row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}\n.tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}\n.dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}\n.pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}\n.copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}\n.hide{display:none}\n</style>\n</head>\n<body><div class=\"wrap\">\n<h1>AI Search Fan-Out</h1>\n<p class=\"sub\" id=\"sub\"></p>\n<div class=\"stat\" id=\"stat\"></div>\n<div class=\"cards\" id=\"cards\"></div>\n<div id=\"charts\"></div>\n<div class=\"tabs\"><button class=\"tab on\" data-tab=\"q\">Queries</button><button class=\"tab\" data-tab=\"u\">URLs</button></div>\n<div class=\"chips\" id=\"chips\"></div>\n<div id=\"list\"></div>\n</div>\n<script>\nconst D=${data};\nconst el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};\ndocument.getElementById('sub').textContent=D.platform+' · '+(D.meta.model||'model n/a')+' · '+D.meta.rounds+' search round(s) · prompt: '+(D.prompt||'');\ndocument.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries · '+D.counts.browsed+' URLs researched · '+D.counts.cited+' cited</small>';\nconst cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];\nconst cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});\nfunction barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}\nconst charts=document.getElementById('charts');\ncharts.appendChild(barChart('URL Categories',D.aggregates.byCategory));\nlet tab='q',filter=null;\nconst chips=document.getElementById('chips'),list=document.getElementById('list');\nfunction render(){\n chips.innerHTML='';list.innerHTML='';\n if(tab==='q'){\n D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});\n } else {\n const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});\n Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});\n D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class=\"dom\">'+esc(s.domain)+'</span> <span class=\"pill '+(s.cited?'cite':'brow')+'\">'+(s.cited?'cited '+s.timesCited+'×':'browsed')+'</span> <span class=\"tag\">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});\n }\n}\nfunction esc(s){return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;')}\ndocument.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});\nrender();\n</script>\n</body></html>`\n}\n"],"mappings":";AAAA,SAAS,WAAW,qBAAqB;AACzC,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,OAAO,UAAU;AAiBjB,SAAS,gBAAwB;AAC/B,SAAO,QAAQ,IAAI,wBAAwB,KAAK,KAAK,KAAK,QAAQ,GAAG,aAAa,aAAa;AACjG;AAEA,SAAS,KAAK,OAAuB;AACnC,SAAO,MAAM,QAAQ,qBAAqB,GAAG,EAAE,QAAQ,YAAY,EAAE,EAAE,MAAM,GAAG,EAAE,KAAK;AACzF;AAIA,SAAS,WAAW,MAAc,MAAsC,WAAyB;AAC/F,gBAAc,MAAM,KAAK,QAAQ,KAAK,SAAS,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;AAC5E;AAEO,SAAS,aAAa,UAA6C;AACxE,QAAM,QAAQ,KAAK,SAAS,WAAW,QAAQ,SAAS,GAAG,CAAC;AAC5D,QAAM,YAAY,cAAc;AAChC,QAAM,cAAc,KAAK,UAAU,GAAG,KAAK,IAAI,KAAK,SAAS,QAAQ,CAAC,EAAE;AACxE,QAAM,MAAM,KAAK,WAAW,WAAW;AACvC,YAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAElC,QAAM,YAAY,SAAS,QAAQ,IAAI,CAAC,GAAG,OAAO,EAAE,OAAO,IAAI,GAAG,OAAO,EAAE,EAAE;AAC7E,QAAM,eAAe,SAAS,WAAW,cAAc,IAAI,CAAC,MAAM;AAChE,UAAM,MAAM,SAAS,UAAU,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG;AAC1D,WAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,YAAY,EAAE,YAAY,UAAU,KAAK,YAAY,IAAI,OAAO,KAAK,SAAS,GAAG;AAAA,EACxI,CAAC;AACD,QAAM,aAAa,SAAS,YAAY,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,YAAY,EAAE,YAAY,UAAU,EAAE,UAAU,OAAO,EAAE,OAAO,OAAO,EAAE,SAAS,GAAG,EAAE;AAC3L,QAAM,kBAAkB,SAAS,YAAY,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,YAAY,EAAE,YAAY,UAAU,EAAE,UAAU,OAAO,EAAE,OAAO,OAAO,EAAE,SAAS,GAAG,EAAE;AAChM,QAAM,cAAc,SAAS,SAAS,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,MAAM,EAAE,KAAK,EAAE;AACjH,QAAM,aAAa,SAAS,WAAW,SAAS,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,OAAO,EAAE,OAAO,OAAO,EAAE,OAAO,YAAY,EAAE,YAAY,UAAU,EAAE,SAAS,EAAE;AAEjK,QAAM,gBAAmC;AAAA,IACvC,YAAY;AAAA,IACZ,KAAK;AAAA,IACL,MAAM,KAAK,aAAa,aAAa;AAAA,IACrC,YAAY,KAAK,aAAa,aAAa;AAAA,IAC3C,YAAY,KAAK,aAAa,aAAa;AAAA,IAC3C,cAAc,KAAK,aAAa,eAAe;AAAA,IAC/C,YAAY,KAAK,aAAa,aAAa;AAAA,IAC3C,gBAAgB,KAAK,aAAa,kBAAkB;AAAA,IACpD,aAAa,KAAK,aAAa,cAAc;AAAA,IAC7C,YAAY,KAAK,aAAa,aAAa;AAAA,IAC3C,QAAQ,KAAK,aAAa,aAAa;AAAA,EACzC;AAEA,gBAAc,KAAK,WAAW,cAAc,IAAI,GAAG,KAAK,UAAU,UAAU,MAAM,CAAC,CAAC;AACpF,aAAW,KAAK,WAAW,cAAc,UAAU,GAAG,WAAW,GAAG;AACpE,aAAW,KAAK,WAAW,cAAc,UAAU,GAAG,WAAW,GAAI;AACrE,aAAW,KAAK,WAAW,cAAc,YAAY,GAAG,cAAc,GAAG;AACzE,aAAW,KAAK,WAAW,cAAc,UAAU,GAAG,YAAY,GAAG;AACrE,aAAW,KAAK,WAAW,cAAc,cAAc,GAAG,iBAAiB,GAAG;AAC9E,aAAW,KAAK,WAAW,cAAc,WAAW,GAAG,aAAa,GAAG;AACvE,aAAW,KAAK,WAAW,cAAc,UAAU,GAAG,YAAY,GAAG;AACrE,gBAAc,KAAK,WAAW,cAAc,MAAM,GAAG,iBAAiB,QAAQ,CAAC;AAE/E,SAAO;AACT;AAEA,SAAS,IAAI,GAAmB;AAC9B,SAAO,OAAO,CAAC,EAAE,QAAQ,MAAM,OAAO,EAAE,QAAQ,MAAM,MAAM,EAAE,QAAQ,MAAM,MAAM,EAAE,QAAQ,MAAM,QAAQ;AAC5G;AAEO,SAAS,iBAAiB,UAAkC;AACjE,QAAM,OAAO,KAAK,UAAU,QAAQ,EAAE,QAAQ,MAAM,SAAS;AAC7D,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA,kCAKoB,IAAI,SAAS,QAAQ,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,UAsCzC,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA0Bd;","names":[]}
package/dist/{chunk-UWSG3C5J.js → chunk-4OPKIDON.js} RENAMED
@@ -142,6 +142,16 @@ function proxyIdSuffix(proxyId) {
142
142
  function errorText(err) {
143
143
  return err instanceof Error ? err.message : String(err);
144
144
  }
145
+ function isKernelProfileConflict(err) {
146
+ return /\b409\b|conflict|already exists/i.test(errorText(err));
147
+ }
148
+ async function ensureKernelProfile(kernelClient, name) {
149
+ try {
150
+ await kernelClient.profiles.create({ name });
151
+ } catch (err) {
152
+ if (!isKernelProfileConflict(err)) throw err;
153
+ }
154
+ }
145
155
  function rankCheckContextOptions(config) {
146
156
  return {
147
157
  viewport: config.viewport,
@@ -226,10 +236,19 @@ var BrowserDriver = class {
226
236
  if (config.kernelApiKey) {
227
237
  this.kernelClient = new Kernel({ apiKey: config.kernelApiKey });
228
238
  const timeoutSeconds = positiveIntFromEnv("KERNEL_BROWSER_TIMEOUT_SECONDS", DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS);
239
+ if (config.kernelProfileName && config.kernelProfileSaveChanges === true) {
240
+ await ensureKernelProfile(this.kernelClient, config.kernelProfileName);
241
+ }
229
242
  const kernelBrowser = await this.kernelClient.browsers.create({
230
243
  stealth: true,
231
244
  timeout_seconds: timeoutSeconds,
232
- ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {}
245
+ ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {},
246
+ ...config.kernelProfileName ? {
247
+ profile: {
248
+ name: config.kernelProfileName,
249
+ ...typeof config.kernelProfileSaveChanges === "boolean" ? { save_changes: config.kernelProfileSaveChanges } : {}
250
+ }
251
+ } : {}
233
252
  });
234
253
  this.kernelSessionId = kernelBrowser.session_id;
235
254
  let defaultProxyDisabled = null;
@@ -273,6 +292,7 @@ var BrowserDriver = class {
273
292
  timeout_seconds: timeoutSeconds,
274
293
  proxy_mode: proxyMode,
275
294
  proxy_id_present: Boolean(config.kernelProxyId),
295
+ profile_name_present: Boolean(config.kernelProfileName),
276
296
  proxy_resolution_source: config.kernelProxyResolution?.source
277
297
  }));
278
298
  if (this.debugEnabled) {
@@ -2869,4 +2889,4 @@ export {
2869
2889
  resolveKernelProxyId,
2870
2890
  harvest
2871
2891
  };
2872
- //# sourceMappingURL=chunk-UWSG3C5J.js.map
2892
+ //# sourceMappingURL=chunk-4OPKIDON.js.map