mcp-scraper 0.2.18 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -11
- package/dist/bin/api-server.cjs +1020 -348
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +314 -55
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +321 -167
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +3 -2
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +1230 -528
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +19 -3
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +8 -6
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +5 -92
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +726 -387
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -2
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +5 -3
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +2 -1
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-TL7YTFLH.js → chunk-DBQDG7EH.js} +99 -27
- package/dist/chunk-DBQDG7EH.js.map +1 -0
- package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js} +216 -108
- package/dist/chunk-H2R232HK.js.map +1 -0
- package/dist/{chunk-P5PLQU3H.js → chunk-L4OWOUGR.js} +315 -56
- package/dist/chunk-L4OWOUGR.js.map +1 -0
- package/dist/chunk-LFATOGDF.js +111 -0
- package/dist/chunk-LFATOGDF.js.map +1 -0
- package/dist/chunk-MLNCKQ2B.js +96 -0
- package/dist/chunk-MLNCKQ2B.js.map +1 -0
- package/dist/{chunk-RJMCASQH.js → chunk-O5J7E4VX.js} +394 -57
- package/dist/chunk-O5J7E4VX.js.map +1 -0
- package/dist/{chunk-BSYPATSM.js → chunk-SXTXMFEQ.js} +2 -2
- package/dist/{chunk-CQTAKXBN.js → chunk-XGUDTDZ2.js} +7 -104
- package/dist/chunk-XGUDTDZ2.js.map +1 -0
- package/dist/chunk-XKUDVN2E.js +7 -0
- package/dist/chunk-XKUDVN2E.js.map +1 -0
- package/dist/{db-P5X6UQ3E.js → db-BE4JVB3V.js} +8 -2
- package/dist/index.cjs +5 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/{server-VVY5K44O.js → server-7NE45K2A.js} +211 -37
- package/dist/server-7NE45K2A.js.map +1 -0
- package/dist/{worker-OZSWIS3F.js → worker-MIYG2B2I.js} +5 -4
- package/dist/{worker-OZSWIS3F.js.map → worker-MIYG2B2I.js.map} +1 -1
- package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +2 -2
- package/docs/mcp-tool-craft-lint.generated.md +51 -0
- package/docs/mcp-tool-manifest.generated.json +1406 -0
- package/docs/specs/local-competitive-audit-spec.md +1 -2
- package/docs/specs/mcp-tool-definition-quality-audit-spec.md +1602 -0
- package/package.json +4 -1
- package/dist/chunk-CQTAKXBN.js.map +0 -1
- package/dist/chunk-F44RBOJ5.js.map +0 -1
- package/dist/chunk-M5SAUO4K.js +0 -7
- package/dist/chunk-M5SAUO4K.js.map +0 -1
- package/dist/chunk-P5PLQU3H.js.map +0 -1
- package/dist/chunk-RJMCASQH.js.map +0 -1
- package/dist/chunk-TL7YTFLH.js.map +0 -1
- package/dist/server-VVY5K44O.js.map +0 -1
- /package/dist/{chunk-BSYPATSM.js.map → chunk-SXTXMFEQ.js.map} +0 -0
- /package/dist/{db-P5X6UQ3E.js.map → db-BE4JVB3V.js.map} +0 -0
|
@@ -4,11 +4,11 @@
|
|
|
4
4
|
// src/cli/human-cli.ts
|
|
5
5
|
var import_commander = require("commander");
|
|
6
6
|
var import_node_child_process2 = require("child_process");
|
|
7
|
-
var
|
|
7
|
+
var import_promises4 = require("fs/promises");
|
|
8
8
|
var import_node_path3 = require("path");
|
|
9
9
|
|
|
10
10
|
// src/version.ts
|
|
11
|
-
var PACKAGE_VERSION = "0.2.
|
|
11
|
+
var PACKAGE_VERSION = "0.2.20";
|
|
12
12
|
|
|
13
13
|
// src/cli/agent-config.ts
|
|
14
14
|
function apiKeyValue(options) {
|
|
@@ -441,7 +441,8 @@ async function openWorkflowReport(id, outputDir) {
|
|
|
441
441
|
}
|
|
442
442
|
|
|
443
443
|
// src/workflows/registry.ts
|
|
444
|
-
var
|
|
444
|
+
var import_promises3 = require("fs/promises");
|
|
445
|
+
var import_zod6 = require("zod");
|
|
445
446
|
|
|
446
447
|
// src/workflows/http-client.ts
|
|
447
448
|
var WorkflowHttpClient = class {
|
|
@@ -622,115 +623,254 @@ var agentPacketWorkflowDefinition = {
|
|
|
622
623
|
title: "Agent-Ready SEO Packet",
|
|
623
624
|
description: "Create an evidence folder for AI agents from live SERP/PAA/AI search surfaces.",
|
|
624
625
|
inputSchema: AgentPacketInputSchema,
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
626
|
+
createState: () => ({ serp: null, paa: null, warnings: [] }),
|
|
627
|
+
steps: [
|
|
628
|
+
{
|
|
629
|
+
id: "harvest-serp",
|
|
630
|
+
title: "Harvest organic SERP + AI Overview",
|
|
631
|
+
async run({ input, state, ctx }) {
|
|
632
|
+
if (!input.includeSerp) {
|
|
633
|
+
return { state, output: { skipped: true, reason: "includeSerp is false" } };
|
|
634
|
+
}
|
|
635
|
+
try {
|
|
636
|
+
const serp = await ctx.client.post("/harvest/sync", {
|
|
637
|
+
query: input.keyword,
|
|
638
|
+
location: input.location,
|
|
639
|
+
serpOnly: true,
|
|
640
|
+
maxQuestions: 1,
|
|
641
|
+
format: "json"
|
|
642
|
+
}, 18e4);
|
|
643
|
+
await ctx.artifacts.writeJson("SERP raw JSON", "raw/serp.json", serp);
|
|
644
|
+
return {
|
|
645
|
+
state: { ...state, serp },
|
|
646
|
+
output: {
|
|
647
|
+
organicResults: serp.organicResults?.length ?? 0,
|
|
648
|
+
localPack: serp.localPack?.length ?? 0,
|
|
649
|
+
aiOverviewDetected: Boolean(serp.aiOverview?.detected),
|
|
650
|
+
aiOverviewCitations: serp.aiOverview?.citations?.length ?? 0
|
|
651
|
+
}
|
|
652
|
+
};
|
|
653
|
+
} catch (err) {
|
|
654
|
+
const message = `SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
|
|
655
|
+
return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
|
|
656
|
+
}
|
|
642
657
|
}
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
658
|
+
},
|
|
659
|
+
{
|
|
660
|
+
id: "harvest-paa",
|
|
661
|
+
title: "Harvest People Also Ask",
|
|
662
|
+
async run({ input, state, ctx }) {
|
|
663
|
+
if (!input.includePaa) {
|
|
664
|
+
return { state, output: { skipped: true, reason: "includePaa is false" } };
|
|
665
|
+
}
|
|
666
|
+
try {
|
|
667
|
+
const paa = await ctx.client.post("/harvest/sync", {
|
|
668
|
+
query: input.keyword,
|
|
669
|
+
location: input.location,
|
|
670
|
+
maxQuestions: input.maxQuestions,
|
|
671
|
+
format: "json"
|
|
672
|
+
}, 28e4);
|
|
673
|
+
await ctx.artifacts.writeJson("PAA raw JSON", "raw/paa.json", paa);
|
|
674
|
+
return {
|
|
675
|
+
state: { ...state, paa },
|
|
676
|
+
output: { paaQuestions: paa.flat?.length ?? 0 }
|
|
677
|
+
};
|
|
678
|
+
} catch (err) {
|
|
679
|
+
const message = `PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
|
|
680
|
+
return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
},
|
|
684
|
+
{
|
|
685
|
+
id: "assemble",
|
|
686
|
+
title: "Assemble evidence packet",
|
|
687
|
+
async run({ input, state, ctx }) {
|
|
688
|
+
const { serp, paa } = state;
|
|
689
|
+
const warnings = state.warnings;
|
|
690
|
+
if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
|
|
691
|
+
const rows = sourceRows(input, serp, paa);
|
|
692
|
+
const target = normalizeDomain(input.domain);
|
|
693
|
+
const competitors = competitorRows(rows, target);
|
|
694
|
+
const evidence = {
|
|
695
|
+
input,
|
|
696
|
+
serp: serp ?? { status: "skipped" },
|
|
697
|
+
paa: paa ?? { status: "skipped" },
|
|
698
|
+
target: {
|
|
699
|
+
domain: target,
|
|
700
|
+
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
701
|
+
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
702
|
+
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
703
|
+
},
|
|
704
|
+
competitors,
|
|
705
|
+
warnings
|
|
706
|
+
};
|
|
707
|
+
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
708
|
+
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
709
|
+
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
710
|
+
const brief = [
|
|
711
|
+
`# SEO Evidence Brief: ${input.keyword}`,
|
|
712
|
+
"",
|
|
713
|
+
`Location: ${input.location ?? "not specified"}`,
|
|
714
|
+
`Target domain: ${target ?? "not specified"}`,
|
|
715
|
+
"",
|
|
716
|
+
"## Evidence Summary",
|
|
717
|
+
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
718
|
+
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
719
|
+
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
720
|
+
`- Competitor domains: ${competitors.length}`,
|
|
721
|
+
"",
|
|
722
|
+
"## Recommended Use",
|
|
723
|
+
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
724
|
+
].join("\n");
|
|
725
|
+
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
726
|
+
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
727
|
+
"# Agent Tasks",
|
|
728
|
+
"",
|
|
729
|
+
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
730
|
+
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
731
|
+
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
732
|
+
"- [ ] Mark unsupported recommendations as assumptions."
|
|
733
|
+
].join("\n"));
|
|
734
|
+
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
735
|
+
"# Agent Instructions",
|
|
736
|
+
"",
|
|
737
|
+
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
738
|
+
].join("\n"));
|
|
739
|
+
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
740
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
741
|
+
title: "Agent-Ready SEO Packet",
|
|
742
|
+
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
743
|
+
summary,
|
|
744
|
+
warnings,
|
|
745
|
+
tables: [
|
|
746
|
+
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
747
|
+
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
748
|
+
]
|
|
749
|
+
}));
|
|
750
|
+
const status2 = warnings.length ? "partial" : "succeeded";
|
|
751
|
+
const counts = { sources: rows.length, competitors: competitors.length };
|
|
752
|
+
await ctx.artifacts.writeManifest(status2, counts, warnings, []);
|
|
753
|
+
return {
|
|
754
|
+
state,
|
|
755
|
+
output: { sources: rows.length, competitors: competitors.length, status: status2 },
|
|
756
|
+
summary: { title: "Agent-Ready SEO Packet", summary, status: status2, counts, warnings, errors: [], reportPath }
|
|
757
|
+
};
|
|
654
758
|
}
|
|
655
759
|
}
|
|
656
|
-
|
|
657
|
-
const rows = sourceRows(input, serp, paa);
|
|
658
|
-
const target = normalizeDomain(input.domain);
|
|
659
|
-
const competitors = competitorRows(rows, target);
|
|
660
|
-
const evidence = {
|
|
661
|
-
input,
|
|
662
|
-
serp: serp ?? { status: "skipped" },
|
|
663
|
-
paa: paa ?? { status: "skipped" },
|
|
664
|
-
target: {
|
|
665
|
-
domain: target,
|
|
666
|
-
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
667
|
-
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
668
|
-
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
669
|
-
},
|
|
670
|
-
competitors,
|
|
671
|
-
warnings
|
|
672
|
-
};
|
|
673
|
-
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
674
|
-
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
675
|
-
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
676
|
-
const brief = [
|
|
677
|
-
`# SEO Evidence Brief: ${input.keyword}`,
|
|
678
|
-
"",
|
|
679
|
-
`Location: ${input.location ?? "not specified"}`,
|
|
680
|
-
`Target domain: ${target ?? "not specified"}`,
|
|
681
|
-
"",
|
|
682
|
-
"## Evidence Summary",
|
|
683
|
-
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
684
|
-
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
685
|
-
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
686
|
-
`- Competitor domains: ${competitors.length}`,
|
|
687
|
-
"",
|
|
688
|
-
"## Recommended Use",
|
|
689
|
-
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
690
|
-
].join("\n");
|
|
691
|
-
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
692
|
-
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
693
|
-
"# Agent Tasks",
|
|
694
|
-
"",
|
|
695
|
-
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
696
|
-
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
697
|
-
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
698
|
-
"- [ ] Mark unsupported recommendations as assumptions."
|
|
699
|
-
].join("\n"));
|
|
700
|
-
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
701
|
-
"# Agent Instructions",
|
|
702
|
-
"",
|
|
703
|
-
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
704
|
-
].join("\n"));
|
|
705
|
-
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
706
|
-
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
707
|
-
title: "Agent-Ready SEO Packet",
|
|
708
|
-
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
709
|
-
summary,
|
|
710
|
-
warnings,
|
|
711
|
-
tables: [
|
|
712
|
-
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
713
|
-
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
714
|
-
]
|
|
715
|
-
}));
|
|
716
|
-
const status2 = warnings.length ? "partial" : "succeeded";
|
|
717
|
-
const counts = { sources: rows.length, competitors: competitors.length };
|
|
718
|
-
await ctx.artifacts.writeManifest(status2, counts, warnings, errors);
|
|
719
|
-
return { title: "Agent-Ready SEO Packet", summary, status: status2, counts, warnings, errors, reportPath };
|
|
720
|
-
}
|
|
760
|
+
]
|
|
721
761
|
};
|
|
722
762
|
|
|
723
763
|
// src/workflows/workflows/directory.ts
|
|
764
|
+
var import_zod3 = require("zod");
|
|
765
|
+
|
|
766
|
+
// src/schemas.ts
|
|
724
767
|
var import_zod2 = require("zod");
|
|
725
|
-
var
|
|
768
|
+
var DEFAULT_PROXY_MODE = "configured";
|
|
769
|
+
var DEFAULT_MAPS_PROXY_MODE = "location";
|
|
770
|
+
var HarvestOptionsSchema = import_zod2.z.object({
|
|
726
771
|
query: import_zod2.z.string().min(1),
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
772
|
+
location: import_zod2.z.string().optional(),
|
|
773
|
+
gl: import_zod2.z.string().length(2).default("us"),
|
|
774
|
+
hl: import_zod2.z.string().length(2).default("en"),
|
|
775
|
+
device: import_zod2.z.enum(["desktop", "mobile"]).default("desktop"),
|
|
776
|
+
proxyMode: import_zod2.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
|
|
777
|
+
proxyZip: import_zod2.z.string().regex(/^\d{5}$/).optional(),
|
|
778
|
+
debug: import_zod2.z.boolean().default(false),
|
|
779
|
+
depth: import_zod2.z.number().int().min(1).max(30).default(3),
|
|
780
|
+
maxQuestions: import_zod2.z.number().int().min(1).max(1e3).default(100),
|
|
781
|
+
headless: import_zod2.z.boolean().default(false),
|
|
782
|
+
profileDir: import_zod2.z.string().optional(),
|
|
783
|
+
proxy: import_zod2.z.string().url().optional(),
|
|
784
|
+
kernelApiKey: import_zod2.z.string().optional(),
|
|
785
|
+
kernelProxyId: import_zod2.z.string().optional(),
|
|
786
|
+
kernelProxyResolution: import_zod2.z.unknown().optional(),
|
|
787
|
+
outputDir: import_zod2.z.string().default("./paa-output"),
|
|
788
|
+
format: import_zod2.z.enum(["json", "csv", "both"]).default("both"),
|
|
789
|
+
serpOnly: import_zod2.z.boolean().default(false),
|
|
790
|
+
pages: import_zod2.z.number().int().min(1).max(2).default(1)
|
|
791
|
+
});
|
|
792
|
+
var MapsPlaceOptionsSchema = import_zod2.z.object({
|
|
793
|
+
businessName: import_zod2.z.string().min(1),
|
|
794
|
+
location: import_zod2.z.string().min(1),
|
|
795
|
+
gl: import_zod2.z.string().length(2).default("us"),
|
|
796
|
+
hl: import_zod2.z.string().length(2).default("en"),
|
|
797
|
+
includeReviews: import_zod2.z.boolean().default(false),
|
|
798
|
+
maxReviews: import_zod2.z.number().int().min(1).max(500).default(50),
|
|
799
|
+
kernelApiKey: import_zod2.z.string().optional(),
|
|
800
|
+
kernelProxyId: import_zod2.z.string().optional(),
|
|
801
|
+
headless: import_zod2.z.boolean().default(true)
|
|
802
|
+
});
|
|
803
|
+
var MapsSearchOptionsSchema = import_zod2.z.object({
|
|
804
|
+
query: import_zod2.z.string().min(1),
|
|
805
|
+
location: import_zod2.z.string().optional(),
|
|
806
|
+
gl: import_zod2.z.string().length(2).default("us"),
|
|
807
|
+
hl: import_zod2.z.string().length(2).default("en"),
|
|
808
|
+
maxResults: import_zod2.z.number().int().min(1).max(50).default(10),
|
|
809
|
+
proxyMode: import_zod2.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
810
|
+
proxyZip: import_zod2.z.string().regex(/^\d{5}$/).optional(),
|
|
811
|
+
debug: import_zod2.z.boolean().default(false),
|
|
812
|
+
kernelApiKey: import_zod2.z.string().optional(),
|
|
813
|
+
kernelProxyId: import_zod2.z.string().optional(),
|
|
814
|
+
kernelProxyResolution: import_zod2.z.unknown().optional(),
|
|
815
|
+
headless: import_zod2.z.boolean().default(true)
|
|
816
|
+
});
|
|
817
|
+
var RawPAAItemSchema = import_zod2.z.object({
|
|
818
|
+
question: import_zod2.z.string().min(1),
|
|
819
|
+
answer: import_zod2.z.string().optional(),
|
|
820
|
+
sourceTitle: import_zod2.z.string().optional(),
|
|
821
|
+
sourceSite: import_zod2.z.string().optional(),
|
|
822
|
+
sourceCite: import_zod2.z.string().optional()
|
|
823
|
+
});
|
|
824
|
+
var RawMapsOverviewSchema = import_zod2.z.object({
|
|
825
|
+
name: import_zod2.z.string().nullable(),
|
|
826
|
+
rating: import_zod2.z.string().nullable(),
|
|
827
|
+
reviewCount: import_zod2.z.string().nullable(),
|
|
828
|
+
category: import_zod2.z.string().nullable(),
|
|
829
|
+
address: import_zod2.z.string().nullable(),
|
|
830
|
+
hoursSummary: import_zod2.z.string().nullable(),
|
|
831
|
+
phone: import_zod2.z.string().nullable(),
|
|
832
|
+
phoneDisplay: import_zod2.z.string().nullable(),
|
|
833
|
+
website: import_zod2.z.string().nullable(),
|
|
834
|
+
plusCode: import_zod2.z.string().nullable(),
|
|
835
|
+
bookingUrl: import_zod2.z.string().nullable()
|
|
836
|
+
});
|
|
837
|
+
var RawMapsHoursRowSchema = import_zod2.z.object({
|
|
838
|
+
day: import_zod2.z.string(),
|
|
839
|
+
hours: import_zod2.z.string()
|
|
840
|
+
});
|
|
841
|
+
var RawMapsReviewStatsSchema = import_zod2.z.object({
|
|
842
|
+
reviewHistogram: import_zod2.z.array(import_zod2.z.object({
|
|
843
|
+
stars: import_zod2.z.number(),
|
|
844
|
+
count: import_zod2.z.string()
|
|
845
|
+
})),
|
|
846
|
+
reviewTopics: import_zod2.z.array(import_zod2.z.object({
|
|
847
|
+
label: import_zod2.z.string(),
|
|
848
|
+
count: import_zod2.z.string()
|
|
849
|
+
}))
|
|
850
|
+
});
|
|
851
|
+
var RawMapsReviewCardSchema = import_zod2.z.object({
|
|
852
|
+
reviewId: import_zod2.z.string(),
|
|
853
|
+
author: import_zod2.z.string().nullable(),
|
|
854
|
+
stars: import_zod2.z.string().nullable(),
|
|
855
|
+
date: import_zod2.z.string().nullable(),
|
|
856
|
+
text: import_zod2.z.string().nullable(),
|
|
857
|
+
ownerResponse: import_zod2.z.string().nullable()
|
|
858
|
+
});
|
|
859
|
+
var RawMapsAboutAttributeSchema = import_zod2.z.object({
|
|
860
|
+
section: import_zod2.z.string(),
|
|
861
|
+
attribute: import_zod2.z.string()
|
|
862
|
+
});
|
|
863
|
+
|
|
864
|
+
// src/workflows/workflows/directory.ts
|
|
865
|
+
var DirectoryWorkflowCliInputSchema = import_zod3.z.object({
|
|
866
|
+
query: import_zod3.z.string().min(1),
|
|
867
|
+
state: import_zod3.z.string().min(2).default("TN"),
|
|
868
|
+
minPopulation: import_zod3.z.number().int().min(0).default(1e5),
|
|
869
|
+
maxCities: import_zod3.z.number().int().min(1).max(100).default(25),
|
|
870
|
+
maxResultsPerCity: import_zod3.z.number().int().min(1).max(50).default(20),
|
|
871
|
+
concurrency: import_zod3.z.number().int().min(1).max(5).default(5),
|
|
872
|
+
proxyMode: import_zod3.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
873
|
+
saveCsv: import_zod3.z.boolean().default(true)
|
|
734
874
|
});
|
|
735
875
|
function directoryRows(result) {
|
|
736
876
|
const rows = [];
|
|
@@ -854,18 +994,18 @@ ${summary}
|
|
|
854
994
|
};
|
|
855
995
|
|
|
856
996
|
// src/workflows/workflows/local-competitive-audit.ts
|
|
857
|
-
var
|
|
858
|
-
var LocalCompetitiveAuditInputSchema =
|
|
859
|
-
query:
|
|
860
|
-
state:
|
|
861
|
-
minPopulation:
|
|
862
|
-
maxCities:
|
|
863
|
-
maxResultsPerCity:
|
|
864
|
-
hydrateTop:
|
|
865
|
-
maxReviews:
|
|
866
|
-
concurrency:
|
|
867
|
-
proxyMode:
|
|
868
|
-
returnPartial:
|
|
997
|
+
var import_zod4 = require("zod");
|
|
998
|
+
var LocalCompetitiveAuditInputSchema = import_zod4.z.object({
|
|
999
|
+
query: import_zod4.z.string().min(1),
|
|
1000
|
+
state: import_zod4.z.string().min(2).default("TN"),
|
|
1001
|
+
minPopulation: import_zod4.z.number().int().min(0).default(1e5),
|
|
1002
|
+
maxCities: import_zod4.z.number().int().min(1).max(100).default(25),
|
|
1003
|
+
maxResultsPerCity: import_zod4.z.number().int().min(1).max(50).default(20),
|
|
1004
|
+
hydrateTop: import_zod4.z.number().int().min(0).max(10).default(5),
|
|
1005
|
+
maxReviews: import_zod4.z.number().int().min(0).max(500).default(50),
|
|
1006
|
+
concurrency: import_zod4.z.number().int().min(1).max(5).default(5),
|
|
1007
|
+
proxyMode: import_zod4.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
1008
|
+
returnPartial: import_zod4.z.boolean().default(true)
|
|
869
1009
|
});
|
|
870
1010
|
async function mapLimit(items, limit, fn) {
|
|
871
1011
|
const out = new Array(items.length);
|
|
@@ -1053,7 +1193,7 @@ ${summary}
|
|
|
1053
1193
|
};
|
|
1054
1194
|
|
|
1055
1195
|
// src/workflows/workflows/comparison-briefs.ts
|
|
1056
|
-
var
|
|
1196
|
+
var import_zod5 = require("zod");
|
|
1057
1197
|
|
|
1058
1198
|
// src/workflows/workflows/seo-workflow-utils.ts
|
|
1059
1199
|
var STOP_WORDS = /* @__PURE__ */ new Set([
|
|
@@ -1211,49 +1351,49 @@ async function mapLimit2(items, limit, fn) {
|
|
|
1211
1351
|
}
|
|
1212
1352
|
|
|
1213
1353
|
// src/workflows/workflows/comparison-briefs.ts
|
|
1214
|
-
var ProxyModeSchema =
|
|
1215
|
-
var MapComparisonInputSchema =
|
|
1216
|
-
query:
|
|
1217
|
-
location:
|
|
1218
|
-
state:
|
|
1219
|
-
minPopulation:
|
|
1220
|
-
maxCities:
|
|
1221
|
-
maxResultsPerCity:
|
|
1222
|
-
hydrateTop:
|
|
1223
|
-
maxReviews:
|
|
1224
|
-
concurrency:
|
|
1225
|
-
proxyMode: ProxyModeSchema.default(
|
|
1226
|
-
returnPartial:
|
|
1354
|
+
var ProxyModeSchema = import_zod5.z.enum(["location", "configured", "none"]);
|
|
1355
|
+
var MapComparisonInputSchema = import_zod5.z.object({
|
|
1356
|
+
query: import_zod5.z.string().min(1),
|
|
1357
|
+
location: import_zod5.z.string().optional(),
|
|
1358
|
+
state: import_zod5.z.string().optional(),
|
|
1359
|
+
minPopulation: import_zod5.z.number().int().min(0).default(1e5),
|
|
1360
|
+
maxCities: import_zod5.z.number().int().min(1).max(100).default(5),
|
|
1361
|
+
maxResultsPerCity: import_zod5.z.number().int().min(1).max(50).default(20),
|
|
1362
|
+
hydrateTop: import_zod5.z.number().int().min(0).max(10).default(5),
|
|
1363
|
+
maxReviews: import_zod5.z.number().int().min(0).max(500).default(25),
|
|
1364
|
+
concurrency: import_zod5.z.number().int().min(1).max(5).default(5),
|
|
1365
|
+
proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
|
|
1366
|
+
returnPartial: import_zod5.z.boolean().default(true)
|
|
1227
1367
|
}).refine((input) => input.location || input.state, {
|
|
1228
1368
|
message: "Either location or state is required for map-comparison"
|
|
1229
1369
|
});
|
|
1230
|
-
var SerpComparisonInputSchema =
|
|
1231
|
-
keyword:
|
|
1232
|
-
domain:
|
|
1233
|
-
url:
|
|
1234
|
-
location:
|
|
1235
|
-
maxResults:
|
|
1236
|
-
maxQuestions:
|
|
1237
|
-
extractTop:
|
|
1238
|
-
includePaa:
|
|
1239
|
-
includeAiOverview:
|
|
1240
|
-
returnPartial:
|
|
1370
|
+
var SerpComparisonInputSchema = import_zod5.z.object({
|
|
1371
|
+
keyword: import_zod5.z.string().min(1),
|
|
1372
|
+
domain: import_zod5.z.string().optional(),
|
|
1373
|
+
url: import_zod5.z.string().url().optional(),
|
|
1374
|
+
location: import_zod5.z.string().optional(),
|
|
1375
|
+
maxResults: import_zod5.z.number().int().min(1).max(20).default(10),
|
|
1376
|
+
maxQuestions: import_zod5.z.number().int().min(1).max(200).default(40),
|
|
1377
|
+
extractTop: import_zod5.z.number().int().min(0).max(10).default(5),
|
|
1378
|
+
includePaa: import_zod5.z.boolean().default(true),
|
|
1379
|
+
includeAiOverview: import_zod5.z.boolean().default(true),
|
|
1380
|
+
returnPartial: import_zod5.z.boolean().default(true)
|
|
1241
1381
|
});
|
|
1242
|
-
var PaaExpansionBriefInputSchema =
|
|
1243
|
-
keyword:
|
|
1244
|
-
location:
|
|
1245
|
-
maxQuestions:
|
|
1246
|
-
depth:
|
|
1247
|
-
returnPartial:
|
|
1382
|
+
var PaaExpansionBriefInputSchema = import_zod5.z.object({
|
|
1383
|
+
keyword: import_zod5.z.string().min(1),
|
|
1384
|
+
location: import_zod5.z.string().optional(),
|
|
1385
|
+
maxQuestions: import_zod5.z.number().int().min(1).max(300).default(80),
|
|
1386
|
+
depth: import_zod5.z.number().int().min(1).max(6).default(3),
|
|
1387
|
+
returnPartial: import_zod5.z.boolean().default(true)
|
|
1248
1388
|
});
|
|
1249
|
-
var AiOverviewLanguageInputSchema =
|
|
1250
|
-
keyword:
|
|
1251
|
-
domain:
|
|
1252
|
-
url:
|
|
1253
|
-
location:
|
|
1254
|
-
maxQuestions:
|
|
1255
|
-
extractTop:
|
|
1256
|
-
returnPartial:
|
|
1389
|
+
var AiOverviewLanguageInputSchema = import_zod5.z.object({
|
|
1390
|
+
keyword: import_zod5.z.string().min(1),
|
|
1391
|
+
domain: import_zod5.z.string().optional(),
|
|
1392
|
+
url: import_zod5.z.string().url().optional(),
|
|
1393
|
+
location: import_zod5.z.string().optional(),
|
|
1394
|
+
maxQuestions: import_zod5.z.number().int().min(1).max(200).default(40),
|
|
1395
|
+
extractTop: import_zod5.z.number().int().min(0).max(8).default(3),
|
|
1396
|
+
returnPartial: import_zod5.z.boolean().default(true)
|
|
1257
1397
|
});
|
|
1258
1398
|
function businessRowsFromMaps(location, query, results) {
|
|
1259
1399
|
return results.map((result) => ({
|
|
@@ -1769,22 +1909,36 @@ function workflowDefinition(id) {
|
|
|
1769
1909
|
if (!definition) throw new Error(`Unknown workflow "${id}". Available: ${DEFINITIONS.map((def) => def.id).join(", ")}`);
|
|
1770
1910
|
return definition;
|
|
1771
1911
|
}
|
|
1912
|
+
function resolveApiKey(options) {
|
|
1913
|
+
const apiKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim();
|
|
1914
|
+
if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
|
|
1915
|
+
return apiKey;
|
|
1916
|
+
}
|
|
1917
|
+
function resolveApiUrl(options) {
|
|
1918
|
+
return options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
1919
|
+
}
|
|
1772
1920
|
async function runWorkflow(id, rawInput, options = {}) {
|
|
1773
1921
|
const definition = workflowDefinition(id);
|
|
1774
1922
|
const input = definition.inputSchema.parse(rawInput);
|
|
1775
|
-
const apiKey = options
|
|
1776
|
-
|
|
1777
|
-
const apiUrl = options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
1923
|
+
const apiKey = resolveApiKey(options);
|
|
1924
|
+
const apiUrl = resolveApiUrl(options);
|
|
1778
1925
|
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
1779
1926
|
const client = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
|
|
1927
|
+
const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client, artifacts, signal: options.signal };
|
|
1780
1928
|
try {
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1929
|
+
if (definition.steps && definition.steps.length > 0) {
|
|
1930
|
+
let state = definition.createState ? definition.createState(input) : {};
|
|
1931
|
+
let summary = null;
|
|
1932
|
+
for (const step of definition.steps) {
|
|
1933
|
+
const outcome = await step.run({ input, state, ctx });
|
|
1934
|
+
state = outcome.state;
|
|
1935
|
+
if (outcome.summary) summary = outcome.summary;
|
|
1936
|
+
}
|
|
1937
|
+
if (!summary) throw new Error(`Workflow "${id}" produced no terminal step summary`);
|
|
1938
|
+
return summary;
|
|
1939
|
+
}
|
|
1940
|
+
if (definition.run) return await definition.run(input, ctx);
|
|
1941
|
+
throw new Error(`Workflow "${id}" has neither steps nor a run() implementation`);
|
|
1788
1942
|
} catch (err) {
|
|
1789
1943
|
const message = err instanceof Error ? err.message : String(err);
|
|
1790
1944
|
await artifacts.writeText("Failure", "summary.md", `# ${definition.title}
|
|
@@ -2038,13 +2192,13 @@ ${summary.warnings.map((w) => `- ${w}`).join("\n")}` : ""
|
|
|
2038
2192
|
const result = await apiRequest(`/workflows/runs/${id}`, "GET", opts);
|
|
2039
2193
|
const { apiUrl, apiKey } = apiOptions(opts);
|
|
2040
2194
|
const outDir = (0, import_node_path3.join)(opts.outputDir ?? workflowOutputBaseDir(), "workflow-downloads", id);
|
|
2041
|
-
await (0,
|
|
2195
|
+
await (0, import_promises4.mkdir)(outDir, { recursive: true });
|
|
2042
2196
|
const downloaded = [];
|
|
2043
2197
|
for (const artifact of result.run.artifacts ?? []) {
|
|
2044
2198
|
const res = await fetch(`${apiUrl}/workflows/runs/${id}/artifacts/${artifact.id}`, { headers: { "x-api-key": apiKey } });
|
|
2045
2199
|
if (!res.ok) throw new Error(`Failed to download ${artifact.label}: HTTP ${res.status}`);
|
|
2046
2200
|
const file = (0, import_node_path3.join)(outDir, (0, import_node_path3.basename)(artifact.path));
|
|
2047
|
-
await (0,
|
|
2201
|
+
await (0, import_promises4.writeFile)(file, Buffer.from(await res.arrayBuffer()));
|
|
2048
2202
|
downloaded.push(file);
|
|
2049
2203
|
}
|
|
2050
2204
|
writeOutput(opts.json ? { runId: id, files: downloaded } : downloaded.join("\n"), opts.json);
|