mcp-scraper 0.3.17 → 0.3.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -58
- package/dist/bin/api-server.cjs +286 -49
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +4 -4
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +5 -5
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +12 -12
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +3 -3
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +2441 -785
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +1377 -9
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.js +2 -3
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-N33YEY6W.js → chunk-2XCNU22L.js} +210 -10
- package/dist/{chunk-N33YEY6W.js.map → chunk-2XCNU22L.js.map} +1 -1
- package/dist/chunk-C7ZNIXOA.js +7 -0
- package/dist/chunk-C7ZNIXOA.js.map +1 -0
- package/dist/{chunk-ACIUCZ27.js → chunk-DE7WP4UU.js} +11 -11
- package/dist/chunk-DE7WP4UU.js.map +1 -0
- package/dist/{chunk-ZAUMSBV3.js → chunk-HDPWLYK7.js} +6 -6
- package/dist/{chunk-4OPKIDON.js → chunk-IGOEAKC7.js} +4 -6
- package/dist/{chunk-4OPKIDON.js.map → chunk-IGOEAKC7.js.map} +1 -1
- package/dist/{chunk-M2S27J6Z.js → chunk-ZAP7MYVO.js} +26 -1
- package/dist/chunk-ZAP7MYVO.js.map +1 -0
- package/dist/index.js +2 -3
- package/dist/index.js.map +1 -1
- package/dist/{server-TKWY47JA.js → server-3M3XCVOT.js} +323 -158
- package/dist/server-3M3XCVOT.js.map +1 -0
- package/dist/{worker-2WVKKCC7.js → worker-J6I5XP2F.js} +9 -10
- package/dist/{worker-2WVKKCC7.js.map → worker-J6I5XP2F.js.map} +1 -1
- package/docs/mcp-tool-manifest.generated.json +28 -27
- package/package.json +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +0 -1523
- package/dist/bin/browser-agent-stdio-server.cjs.map +0 -1
- package/dist/bin/browser-agent-stdio-server.d.cts +0 -1
- package/dist/bin/browser-agent-stdio-server.d.ts +0 -1
- package/dist/bin/browser-agent-stdio-server.js +0 -43
- package/dist/bin/browser-agent-stdio-server.js.map +0 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +0 -5718
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +0 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.d.cts +0 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.d.ts +0 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +0 -70
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +0 -1
- package/dist/chunk-3YGKXXUG.js +0 -133
- package/dist/chunk-3YGKXXUG.js.map +0 -1
- package/dist/chunk-ACIUCZ27.js.map +0 -1
- package/dist/chunk-M2S27J6Z.js.map +0 -1
- package/dist/chunk-PVWWY5NV.js +0 -7
- package/dist/chunk-PVWWY5NV.js.map +0 -1
- package/dist/chunk-UFD7N36R.js +0 -1332
- package/dist/chunk-UFD7N36R.js.map +0 -1
- package/dist/chunk-WN7PBKMV.js +0 -28
- package/dist/chunk-WN7PBKMV.js.map +0 -1
- package/dist/server-TKWY47JA.js.map +0 -1
- /package/dist/{chunk-ZAUMSBV3.js.map → chunk-HDPWLYK7.js.map} +0 -0
|
@@ -1 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
HttpMcpToolExecutor,
|
|
4
|
-
SERVER_INSTRUCTIONS,
|
|
5
|
-
registerPaaExtractorMcpTools
|
|
6
|
-
} from "../chunk-N33YEY6W.js";
|
|
7
|
-
import "../chunk-M2S27J6Z.js";
|
|
8
|
-
import {
|
|
9
|
-
registerBrowserAgentMcpTools
|
|
10
|
-
} from "../chunk-UFD7N36R.js";
|
|
11
|
-
import "../chunk-3YGKXXUG.js";
|
|
12
|
-
import "../chunk-WN7PBKMV.js";
|
|
13
|
-
import {
|
|
14
|
-
renderInstallTerminal
|
|
15
|
-
} from "../chunk-ACIUCZ27.js";
|
|
16
|
-
import "../chunk-LFATOGDF.js";
|
|
17
|
-
import {
|
|
18
|
-
PACKAGE_VERSION
|
|
19
|
-
} from "../chunk-PVWWY5NV.js";
|
|
20
|
-
|
|
21
|
-
// bin/mcp-scraper-combined-stdio-server.ts
|
|
22
|
-
import { readFileSync } from "fs";
|
|
23
|
-
import { homedir } from "os";
|
|
24
|
-
import { join } from "path";
|
|
25
|
-
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
26
|
-
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
27
|
-
var forceStdio = process.argv.includes("--stdio") || process.env.MCP_SCRAPER_FORCE_STDIO === "1";
|
|
28
|
-
var interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY);
|
|
29
|
-
var wantsHelp = process.argv.includes("--help") || process.argv.includes("-h");
|
|
30
|
-
if (!forceStdio && (interactiveTerminal || wantsHelp)) {
|
|
31
|
-
const noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
32
|
-
process.stdout.write(renderInstallTerminal({
|
|
33
|
-
version: PACKAGE_VERSION,
|
|
34
|
-
color: !noColor,
|
|
35
|
-
apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim())
|
|
36
|
-
}));
|
|
37
|
-
process.exit(0);
|
|
38
|
-
}
|
|
39
|
-
function readApiKeyFile() {
|
|
40
|
-
const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
|
|
41
|
-
const paths = [explicitPath, join(homedir(), ".mcp-scraper-key")].filter(Boolean);
|
|
42
|
-
for (const path of paths) {
|
|
43
|
-
try {
|
|
44
|
-
const value = readFileSync(path, "utf8").trim();
|
|
45
|
-
if (value) return value;
|
|
46
|
-
} catch {
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
return void 0;
|
|
50
|
-
}
|
|
51
|
-
var apiKey = (process.env.MCP_SCRAPER_API_KEY ?? process.env.MCP_SCRAPER_KEY ?? process.env.MCP_API_KEY ?? readApiKeyFile())?.trim();
|
|
52
|
-
if (!apiKey) {
|
|
53
|
-
process.stderr.write("MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\n");
|
|
54
|
-
process.exit(1);
|
|
55
|
-
}
|
|
56
|
-
var baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || "https://mcpscraper.dev";
|
|
57
|
-
var consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
|
|
58
|
-
var server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS });
|
|
59
|
-
registerPaaExtractorMcpTools(server, new HttpMcpToolExecutor(baseUrl, apiKey));
|
|
60
|
-
registerBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl });
|
|
61
|
-
var transport = new StdioServerTransport();
|
|
62
|
-
async function main() {
|
|
63
|
-
await server.connect(transport);
|
|
64
|
-
}
|
|
65
|
-
main().catch((err) => {
|
|
66
|
-
process.stderr.write(`${err instanceof Error ? err.message : String(err)}
|
|
67
|
-
`);
|
|
68
|
-
process.exit(1);
|
|
69
|
-
});
|
|
70
|
-
//# sourceMappingURL=mcp-scraper-combined-stdio-server.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../bin/mcp-scraper-combined-stdio-server.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools } from '../src/mcp/paa-mcp-server.js'\nimport { SERVER_INSTRUCTIONS } from '../src/mcp/server-instructions.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst forceStdio =\n process.argv.includes('--stdio') ||\n process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n const noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\n process.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n }))\n process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n for (const path of paths) {\n try {\n const value = readFileSync(path, 'utf8').trim()\n if (value) return value\n } catch {\n void 0\n }\n }\n return undefined\n}\n\nconst apiKey = (\n process.env.MCP_SCRAPER_API_KEY ??\n process.env.MCP_SCRAPER_KEY ??\n process.env.MCP_API_KEY ??\n readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS })\n\nregisterPaaExtractorMcpTools(server, new HttpMcpToolExecutor(baseUrl, apiKey))\nregisterBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl })\n\nconst transport = new StdioServerTransport()\n\nasync function main() {\n await server.connect(transport)\n}\n\nmain().catch(err => {\n process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n process.exit(1)\n})\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAC1B,SAAS,4BAA4B;AAQrC,IAAM,aACJ,QAAQ,KAAK,SAAS,SAAS,KAC/B,QAAQ,IAAI,4BAA4B;AAC1C,IAAM,sBAAsB,QAAQ,QAAQ,MAAM,SAAS,QAAQ,OAAO,KAAK;AAC/E,IAAM,YAAY,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAE/E,IAAI,CAAC,eAAe,uBAAuB,YAAY;AACrD,QAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,UAAQ,OAAO,MAAM,sBAAsB;AAAA,IACzC,SAAS;AAAA,IACT,OAAO,CAAC;AAAA,IACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAAA,EACnE,CAAC,CAAC;AACF,UAAQ,KAAK,CAAC;AAChB;AAEA,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,IAAI,UAAU,EAAE,MAAM,eAAe,SAAS,gBAAgB,GAAG,EAAE,cAAc,oBAAoB,CAAC;AAErH,6BAA6B,QAAQ,IAAI,oBAAoB,SAAS,MAAM,CAAC;AAC7E,6BAA6B,QAAQ,EAAE,SAAS,QAAQ,eAAe,CAAC;AAExE,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
|
package/dist/chunk-3YGKXXUG.js
DELETED
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
// src/services/fanout/export.ts
|
|
2
|
-
import { mkdirSync, writeFileSync } from "fs";
|
|
3
|
-
import { homedir } from "os";
|
|
4
|
-
import { join } from "path";
|
|
5
|
-
import Papa from "papaparse";
|
|
6
|
-
function outputBaseDir() {
|
|
7
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join(homedir(), "Downloads", "mcp-scraper");
|
|
8
|
-
}
|
|
9
|
-
function safe(value) {
|
|
10
|
-
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
|
|
11
|
-
}
|
|
12
|
-
function writeTable(path, rows, delimiter) {
|
|
13
|
-
writeFileSync(path, Papa.unparse(rows.length ? rows : [{}], { delimiter }));
|
|
14
|
-
}
|
|
15
|
-
function exportFanout(enriched) {
|
|
16
|
-
const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
|
|
17
|
-
const outputDir = outputBaseDir();
|
|
18
|
-
const relativeDir = join("fanout", `${stamp}-${safe(enriched.platform)}`);
|
|
19
|
-
const dir = join(outputDir, relativeDir);
|
|
20
|
-
mkdirSync(dir, { recursive: true });
|
|
21
|
-
const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
|
|
22
|
-
const citationRows = enriched.aggregates.citationOrder.map((c) => {
|
|
23
|
-
const src = enriched.citedUrls.find((s) => s.url === c.url);
|
|
24
|
-
return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
|
|
25
|
-
});
|
|
26
|
-
const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
|
|
27
|
-
const browsedOnlyRows = enriched.browsedOnly.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
|
|
28
|
-
const snippetRows = enriched.snippets.map((s) => ({ domain: s.domain, url: s.url, title: s.title, text: s.text }));
|
|
29
|
-
const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
|
|
30
|
-
const relativePaths = {
|
|
31
|
-
relativeTo: "MCP_SCRAPER_OUTPUT_DIR or ~/Downloads/mcp-scraper",
|
|
32
|
-
dir: relativeDir,
|
|
33
|
-
json: join(relativeDir, "fanout.json"),
|
|
34
|
-
queriesCsv: join(relativeDir, "queries.csv"),
|
|
35
|
-
queriesTsv: join(relativeDir, "queries.tsv"),
|
|
36
|
-
citationsCsv: join(relativeDir, "citations.csv"),
|
|
37
|
-
sourcesCsv: join(relativeDir, "sources.csv"),
|
|
38
|
-
browsedOnlyCsv: join(relativeDir, "browsed-only.csv"),
|
|
39
|
-
snippetsCsv: join(relativeDir, "snippets.csv"),
|
|
40
|
-
domainsCsv: join(relativeDir, "domains.csv"),
|
|
41
|
-
report: join(relativeDir, "report.html")
|
|
42
|
-
};
|
|
43
|
-
writeFileSync(join(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2));
|
|
44
|
-
writeTable(join(outputDir, relativePaths.queriesCsv), queryRows, ",");
|
|
45
|
-
writeTable(join(outputDir, relativePaths.queriesTsv), queryRows, " ");
|
|
46
|
-
writeTable(join(outputDir, relativePaths.citationsCsv), citationRows, ",");
|
|
47
|
-
writeTable(join(outputDir, relativePaths.sourcesCsv), sourceRows, ",");
|
|
48
|
-
writeTable(join(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ",");
|
|
49
|
-
writeTable(join(outputDir, relativePaths.snippetsCsv), snippetRows, ",");
|
|
50
|
-
writeTable(join(outputDir, relativePaths.domainsCsv), domainRows, ",");
|
|
51
|
-
writeFileSync(join(outputDir, relativePaths.report), renderReportHtml(enriched));
|
|
52
|
-
return relativePaths;
|
|
53
|
-
}
|
|
54
|
-
function esc(s) {
|
|
55
|
-
return String(s).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
56
|
-
}
|
|
57
|
-
function renderReportHtml(enriched) {
|
|
58
|
-
const data = JSON.stringify(enriched).replace(/</g, "\\u003c");
|
|
59
|
-
return `<!doctype html>
|
|
60
|
-
<html lang="en">
|
|
61
|
-
<head>
|
|
62
|
-
<meta charset="utf-8" />
|
|
63
|
-
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
64
|
-
<title>AI Search Fan-Out \u2014 ${esc(enriched.platform)}</title>
|
|
65
|
-
<style>
|
|
66
|
-
:root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}
|
|
67
|
-
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}
|
|
68
|
-
.wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}
|
|
69
|
-
h1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}
|
|
70
|
-
.stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}
|
|
71
|
-
.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}
|
|
72
|
-
.card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
|
|
73
|
-
.card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}
|
|
74
|
-
.bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}
|
|
75
|
-
.bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}
|
|
76
|
-
.bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}
|
|
77
|
-
.tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}
|
|
78
|
-
.tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}
|
|
79
|
-
.chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}
|
|
80
|
-
.chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}
|
|
81
|
-
.chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}
|
|
82
|
-
.row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}
|
|
83
|
-
.row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}
|
|
84
|
-
.tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}
|
|
85
|
-
.dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}
|
|
86
|
-
.pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}
|
|
87
|
-
.copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}
|
|
88
|
-
.hide{display:none}
|
|
89
|
-
</style>
|
|
90
|
-
</head>
|
|
91
|
-
<body><div class="wrap">
|
|
92
|
-
<h1>AI Search Fan-Out</h1>
|
|
93
|
-
<p class="sub" id="sub"></p>
|
|
94
|
-
<div class="stat" id="stat"></div>
|
|
95
|
-
<div class="cards" id="cards"></div>
|
|
96
|
-
<div id="charts"></div>
|
|
97
|
-
<div class="tabs"><button class="tab on" data-tab="q">Queries</button><button class="tab" data-tab="u">URLs</button></div>
|
|
98
|
-
<div class="chips" id="chips"></div>
|
|
99
|
-
<div id="list"></div>
|
|
100
|
-
</div>
|
|
101
|
-
<script>
|
|
102
|
-
const D=${data};
|
|
103
|
-
const el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};
|
|
104
|
-
document.getElementById('sub').textContent=D.platform+' \xB7 '+(D.meta.model||'model n/a')+' \xB7 '+D.meta.rounds+' search round(s) \xB7 prompt: '+(D.prompt||'');
|
|
105
|
-
document.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries \xB7 '+D.counts.browsed+' URLs researched \xB7 '+D.counts.cited+' cited</small>';
|
|
106
|
-
const cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];
|
|
107
|
-
const cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});
|
|
108
|
-
function barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}
|
|
109
|
-
const charts=document.getElementById('charts');
|
|
110
|
-
charts.appendChild(barChart('URL Categories',D.aggregates.byCategory));
|
|
111
|
-
let tab='q',filter=null;
|
|
112
|
-
const chips=document.getElementById('chips'),list=document.getElementById('list');
|
|
113
|
-
function render(){
|
|
114
|
-
chips.innerHTML='';list.innerHTML='';
|
|
115
|
-
if(tab==='q'){
|
|
116
|
-
D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});
|
|
117
|
-
} else {
|
|
118
|
-
const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});
|
|
119
|
-
Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});
|
|
120
|
-
D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class="dom">'+esc(s.domain)+'</span> <span class="pill '+(s.cited?'cite':'brow')+'">'+(s.cited?'cited '+s.timesCited+'\xD7':'browsed')+'</span> <span class="tag">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
function esc(s){return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>')}
|
|
124
|
-
document.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});
|
|
125
|
-
render();
|
|
126
|
-
</script>
|
|
127
|
-
</body></html>`;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
export {
|
|
131
|
-
exportFanout
|
|
132
|
-
};
|
|
133
|
-
//# sourceMappingURL=chunk-3YGKXXUG.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/services/fanout/export.ts"],"sourcesContent":["import { mkdirSync, writeFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport Papa from 'papaparse'\nimport type { EnrichedFanout } from './classify.js'\n\nexport interface FanoutExportPaths {\n relativeTo: string\n dir: string\n json: string\n queriesCsv: string\n queriesTsv: string\n citationsCsv: string\n sourcesCsv: string\n browsedOnlyCsv: string\n snippetsCsv: string\n domainsCsv: string\n report: string\n}\n\nfunction outputBaseDir(): string {\n return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join(homedir(), 'Downloads', 'mcp-scraper')\n}\n\nfunction safe(value: string): string {\n return value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80) || 'capture'\n}\n\nfunction writeTable(path: string, rows: Array<Record<string, unknown>>, delimiter: ','): void\nfunction writeTable(path: string, rows: Array<Record<string, unknown>>, delimiter: '\\t'): void\nfunction writeTable(path: string, rows: Array<Record<string, unknown>>, delimiter: string): void {\n writeFileSync(path, Papa.unparse(rows.length ? rows : [{}], { delimiter }))\n}\n\nexport function exportFanout(enriched: EnrichedFanout): FanoutExportPaths {\n const stamp = safe(enriched.capturedAt.replace(/[:.]/g, '-'))\n const outputDir = outputBaseDir()\n const relativeDir = join('fanout', `${stamp}-${safe(enriched.platform)}`)\n const dir = join(outputDir, relativeDir)\n mkdirSync(dir, { recursive: true })\n\n const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }))\n const citationRows = enriched.aggregates.citationOrder.map((c) => {\n const src = enriched.citedUrls.find((s) => s.url === c.url)\n return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? '', title: src?.title ?? '' }\n })\n const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? '' }))\n const browsedOnlyRows = enriched.browsedOnly.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? '' }))\n const snippetRows = enriched.snippets.map((s) => ({ domain: s.domain, url: s.url, title: s.title, text: s.text }))\n const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }))\n\n const relativePaths: FanoutExportPaths = {\n relativeTo: 'MCP_SCRAPER_OUTPUT_DIR or ~/Downloads/mcp-scraper',\n dir: relativeDir,\n json: join(relativeDir, 'fanout.json'),\n queriesCsv: join(relativeDir, 'queries.csv'),\n queriesTsv: join(relativeDir, 'queries.tsv'),\n citationsCsv: join(relativeDir, 'citations.csv'),\n sourcesCsv: join(relativeDir, 'sources.csv'),\n browsedOnlyCsv: join(relativeDir, 'browsed-only.csv'),\n snippetsCsv: join(relativeDir, 'snippets.csv'),\n domainsCsv: join(relativeDir, 'domains.csv'),\n report: join(relativeDir, 'report.html'),\n }\n\n writeFileSync(join(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2))\n writeTable(join(outputDir, relativePaths.queriesCsv), queryRows, ',')\n writeTable(join(outputDir, relativePaths.queriesTsv), queryRows, '\\t')\n writeTable(join(outputDir, relativePaths.citationsCsv), citationRows, ',')\n writeTable(join(outputDir, relativePaths.sourcesCsv), sourceRows, ',')\n writeTable(join(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ',')\n writeTable(join(outputDir, relativePaths.snippetsCsv), snippetRows, ',')\n writeTable(join(outputDir, relativePaths.domainsCsv), domainRows, ',')\n writeFileSync(join(outputDir, relativePaths.report), renderReportHtml(enriched))\n\n return relativePaths\n}\n\nfunction esc(s: string): string {\n return String(s).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/\"/g, '"')\n}\n\nexport function renderReportHtml(enriched: EnrichedFanout): string {\n const data = JSON.stringify(enriched).replace(/</g, '\\\\u003c')\n return `<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\" />\n<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n<title>AI Search Fan-Out — ${esc(enriched.platform)}</title>\n<style>\n:root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}\n*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}\n.wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}\nh1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}\n.stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}\n.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}\n.card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}\n.card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}\n.bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}\n.bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}\n.bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}\n.tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}\n.tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}\n.chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}\n.chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}\n.chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}\n.row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}\n.row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}\n.tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}\n.dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}\n.pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}\n.copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}\n.hide{display:none}\n</style>\n</head>\n<body><div class=\"wrap\">\n<h1>AI Search Fan-Out</h1>\n<p class=\"sub\" id=\"sub\"></p>\n<div class=\"stat\" id=\"stat\"></div>\n<div class=\"cards\" id=\"cards\"></div>\n<div id=\"charts\"></div>\n<div class=\"tabs\"><button class=\"tab on\" data-tab=\"q\">Queries</button><button class=\"tab\" data-tab=\"u\">URLs</button></div>\n<div class=\"chips\" id=\"chips\"></div>\n<div id=\"list\"></div>\n</div>\n<script>\nconst D=${data};\nconst el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};\ndocument.getElementById('sub').textContent=D.platform+' · '+(D.meta.model||'model n/a')+' · '+D.meta.rounds+' search round(s) · prompt: '+(D.prompt||'');\ndocument.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries · '+D.counts.browsed+' URLs researched · '+D.counts.cited+' cited</small>';\nconst cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];\nconst cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});\nfunction barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}\nconst charts=document.getElementById('charts');\ncharts.appendChild(barChart('URL Categories',D.aggregates.byCategory));\nlet tab='q',filter=null;\nconst chips=document.getElementById('chips'),list=document.getElementById('list');\nfunction render(){\n chips.innerHTML='';list.innerHTML='';\n if(tab==='q'){\n D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});\n } else {\n const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});\n Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});\n D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class=\"dom\">'+esc(s.domain)+'</span> <span class=\"pill '+(s.cited?'cite':'brow')+'\">'+(s.cited?'cited '+s.timesCited+'×':'browsed')+'</span> <span class=\"tag\">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});\n }\n}\nfunction esc(s){return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>')}\ndocument.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});\nrender();\n</script>\n</body></html>`\n}\n"],"mappings":";AAAA,SAAS,WAAW,qBAAqB;AACzC,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,OAAO,UAAU;AAiBjB,SAAS,gBAAwB;AAC/B,SAAO,QAAQ,IAAI,wBAAwB,KAAK,KAAK,KAAK,QAAQ,GAAG,aAAa,aAAa;AACjG;AAEA,SAAS,KAAK,OAAuB;AACnC,SAAO,MAAM,QAAQ,qBAAqB,GAAG,EAAE,QAAQ,YAAY,EAAE,EAAE,MAAM,GAAG,EAAE,KAAK;AACzF;AAIA,SAAS,WAAW,MAAc,MAAsC,WAAyB;AAC/F,gBAAc,MAAM,KAAK,QAAQ,KAAK,SAAS,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;AAC5E;AAEO,SAAS,aAAa,UAA6C;AACxE,QAAM,QAAQ,KAAK,SAAS,WAAW,QAAQ,SAAS,GAAG,CAAC;AAC5D,QAAM,YAAY,cAAc;AAChC,QAAM,cAAc,KAAK,UAAU,GAAG,KAAK,IAAI,KAAK,SAAS,QAAQ,CAAC,EAAE;AACxE,QAAM,MAAM,KAAK,WAAW,WAAW;AACvC,YAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAElC,QAAM,YAAY,SAAS,QAAQ,IAAI,CAAC,GAAG,OAAO,EAAE,OAAO,IAAI,GAAG,OAAO,EAAE,EAAE;AAC7E,QAAM,eAAe,SAAS,WAAW,cAAc,IAAI,CAAC,MAAM;AAChE,UAAM,MAAM,SAAS,UAAU,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG;AAC1D,WAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,YAAY,EAAE,YAAY,UAAU,KAAK,YAAY,IAAI,OAAO,KAAK,SAAS,GAAG;AAAA,EACxI,CAAC;AACD,QAAM,aAAa,SAAS,YAAY,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,YAAY,EAAE,YAAY,UAAU,EAAE,UAAU,OAAO,EAAE,OAAO,OAAO,EAAE,SAAS,GAAG,EAAE;AAC3L,QAAM,kBAAkB,SAAS,YAAY,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,YAAY,EAAE,YAAY,UAAU,EAAE,UAAU,OAAO,EAAE,OAAO,OAAO,EAAE,SAAS,GAAG,EAAE;AAChM,QAAM,cAAc,SAAS,SAAS,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,OAAO,EAAE,OAAO,MAAM,EAAE,KAAK,EAAE;AACjH,QAAM,aAAa,SAAS,WAAW,SAAS,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,OAAO,EAAE,OAAO,OAAO,EAAE,OAAO,YAAY,EAAE,YAAY,UAAU,EAAE,SAAS,EAAE;AAEjK,QAAM,gBAAmC;AAAA,IACvC,YAAY;AAAA,IACZ,KAAK;AAAA,IACL,MAAM,KAAK,aAAa,aAAa;AAAA,IACrC,YAAY,KAAK,aAAa,aAAa;AAAA,IAC3C,YAAY,KAAK,aAAa,aAAa;AAAA,IAC3C,cAAc,KAAK,aAAa,eAAe;AAAA,IAC/C,YAAY,KAAK,aAAa,aAAa;AAAA,IAC3C,gBAAgB,KAAK,aAAa,kBAAkB;AAAA,IACpD,aAAa,KAAK,aAAa,cAAc;AAAA,IAC7C,YAAY,KAAK,aAAa,aAAa;AAAA,IAC3C,QAAQ,KAAK,aAAa,aAAa;AAAA,EACzC;AAEA,gBAAc,KAAK,WAAW,cAAc,IAAI,GAAG,KAAK,UAAU,UAAU,MAAM,CAAC,CAAC;AACpF,aAAW,KAAK,WAAW,cAAc,UAAU,GAAG,WAAW,GAAG;AACpE,aAAW,KAAK,WAAW,cAAc,UAAU,GAAG,WAAW,GAAI;AACrE,aAAW,KAAK,WAAW,cAAc,YAAY,GAAG,cAAc,GAAG;AACzE,aAAW,KAAK,WAAW,cAAc,UAAU,GAAG,YAAY,GAAG;AACrE,aAAW,KAAK,WAAW,cAAc,cAAc,GAAG,iBAAiB,GAAG;AAC9E,aAAW,KAAK,WAAW,cAAc,WAAW,GAAG,aAAa,GAAG;AACvE,aAAW,KAAK,WAAW,cAAc,UAAU,GAAG,YAAY,GAAG;AACrE,gBAAc,KAAK,WAAW,cAAc,MAAM,GAAG,iBAAiB,QAAQ,CAAC;AAE/E,SAAO;AACT;AAEA,SAAS,IAAI,GAAmB;AAC9B,SAAO,OAAO,CAAC,EAAE,QAAQ,MAAM,OAAO,EAAE,QAAQ,MAAM,MAAM,EAAE,QAAQ,MAAM,MAAM,EAAE,QAAQ,MAAM,QAAQ;AAC5G;AAEO,SAAS,iBAAiB,UAAkC;AACjE,QAAM,OAAO,KAAK,UAAU,QAAQ,EAAE,QAAQ,MAAM,SAAS;AAC7D,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA,kCAKoB,IAAI,SAAS,QAAQ,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,UAsCzC,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA0Bd;","names":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/install-terminal.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper-combined\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: saved hosted browser profiles for AI visibility. Share the watch_url, user signs in, then verify with browser_profile_status. Browser sessions are direct/no-proxy by default.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(45 MCP tools)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n toolRow('build', ['rank_tracker_workflow', 'cron plan', 'database prompt'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download'], color),\n toolRow('browser', ['browser_open', 'browser_profile_onboard', 'browser_profile_status', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n `${colorize('Workflows', 'cyan', color)} ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n '',\n colorize('Usage tips:', 'amber', color),\n 'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n 'Run mcp-scraper-combined in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n 'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n 'Customer auth setup: run browser_profile_onboard, send the watch_url, let the user sign in, then call browser_profile_status until AUTHENTICATED.',\n 'Reconnect setup: if a saved profile shows login controls later, run browser_profile_onboard again with the same profile name.',\n 'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n 'For MCP clients, use mcp-scraper-combined so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n 'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Hosted profile setup', 'amber', color),\n 'In your MCP client, call browser_profile_onboard with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n 'Give the returned watch_url to the user. After they sign in, call browser_profile_status, then browser_open with the returned profile.',\n '',\n colorize('Claude Code one-command setup', 'amber', color),\n claudeCommand,\n 'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'mcp-scraper-combined prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n 'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n '',\n ].join('\\n')\n}\n"],"mappings":";AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,0LAA0L,QAAQ,KAAK;AAAA,IAChN;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,kBAAkB,SAAS,KAAK,CAAC;AAAA,IAClF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,yBAAyB,aAAa,iBAAiB,GAAG,KAAK;AAAA,IACjF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,0BAA0B,GAAG,KAAK;AAAA,IAC9N,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,0BAA0B,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IAC1N,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;","names":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/errors.ts"],"sourcesContent":["export const RECAPTCHA_INSTRUCTIONS = 'Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.'\n\nexport function sanitizeVendorName(message: string): string {\n return message\n .replace(/kernel\\.sh\\s+sessions?/gi, 'sessions')\n .replace(/kernel\\.sh\\s+session/gi, 'this session')\n .replace(/kernel\\.sh/gi, 'the service')\n .replace(/kernel\\s+sessions?/gi, 'sessions')\n .replace(/kernel\\s+session/gi, 'this session')\n .replace(/\\bkernel\\b/gi, 'the service')\n .replace(/ +/g, ' ')\n .trim()\n}\n\nexport class CaptchaError extends Error {\n readonly name = 'CaptchaError'\n constructor(public readonly instructions: string) {\n super(`CAPTCHA detected. ${instructions}`)\n }\n}\n\nexport class ExtractionError extends Error {\n readonly name = 'ExtractionError'\n constructor(message: string, public readonly cause?: unknown) {\n super(message)\n }\n}\n\nexport class RequestAbortedError extends Error {\n readonly name = 'RequestAbortedError'\n constructor(message = 'Request aborted before harvest completed') {\n super(message)\n }\n}\n\nexport class LocationMismatchError extends Error {\n readonly name = 'LocationMismatchError'\n constructor(message = 'Google returned results for a different location than requested') {\n super(message)\n }\n}\n"],"mappings":";AAAO,IAAM,yBAAyB;AAE/B,SAAS,mBAAmB,SAAyB;AAC1D,SAAO,QACJ,QAAQ,4BAA4B,UAAU,EAC9C,QAAQ,0BAA0B,cAAc,EAChD,QAAQ,gBAAgB,aAAa,EACrC,QAAQ,wBAAwB,UAAU,EAC1C,QAAQ,sBAAsB,cAAc,EAC5C,QAAQ,gBAAgB,aAAa,EACrC,QAAQ,QAAQ,GAAG,EACnB,KAAK;AACV;AAEO,IAAM,eAAN,cAA2B,MAAM;AAAA,EAEtC,YAA4B,cAAsB;AAChD,UAAM,qBAAqB,YAAY,EAAE;AADf;AAAA,EAE5B;AAAA,EAF4B;AAAA,EADnB,OAAO;AAIlB;AAEO,IAAM,kBAAN,cAA8B,MAAM;AAAA,EAEzC,YAAY,SAAiC,OAAiB;AAC5D,UAAM,OAAO;AAD8B;AAAA,EAE7C;AAAA,EAF6C;AAAA,EADpC,OAAO;AAIlB;AAEO,IAAM,sBAAN,cAAkC,MAAM;AAAA,EACpC,OAAO;AAAA,EAChB,YAAY,UAAU,4CAA4C;AAChE,UAAM,OAAO;AAAA,EACf;AACF;AAEO,IAAM,wBAAN,cAAoC,MAAM;AAAA,EACtC,OAAO;AAAA,EAChB,YAAY,UAAU,mEAAmE;AACvF,UAAM,OAAO;AAAA,EACf;AACF;","names":[]}
|
package/dist/chunk-PVWWY5NV.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.3.17'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}
|