mcp-scraper 0.2.19 → 0.2.20

package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js}

@@ -1,3 +1,7 @@
+import {
+  DEFAULT_MAPS_PROXY_MODE
+} from "./chunk-LFATOGDF.js";
+
 // src/workflows/artifact-writer.ts
 import { mkdir, readFile, stat, writeFile } from "fs/promises";
 import { existsSync } from "fs";
@@ -206,6 +210,7 @@ async function openWorkflowReport(id, outputDir) {
 }
 
 // src/workflows/registry.ts
+import { readFile as readFile2 } from "fs/promises";
 import { z as z5 } from "zod";
 
 // src/workflows/http-client.ts
@@ -387,102 +392,141 @@ var agentPacketWorkflowDefinition = {
   title: "Agent-Ready SEO Packet",
   description: "Create an evidence folder for AI agents from live SERP/PAA/AI search surfaces.",
   inputSchema: AgentPacketInputSchema,
-  async run(input, ctx) {
-    await ctx.artifacts.writeManifest("running", {}, [], []);
-    const warnings = [];
-    const errors = [];
-    let serp = null;
-    let paa = null;
-    if (input.includeSerp) {
-      try {
-        serp = await ctx.client.post("/harvest/sync", {
-          query: input.keyword,
-          location: input.location,
-          serpOnly: true,
-          maxQuestions: 1,
-          format: "json"
-        }, 18e4);
-      } catch (err) {
-        warnings.push(`SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
+  createState: () => ({ serp: null, paa: null, warnings: [] }),
+  steps: [
+    {
+      id: "harvest-serp",
+      title: "Harvest organic SERP + AI Overview",
+      async run({ input, state, ctx }) {
+        if (!input.includeSerp) {
+          return { state, output: { skipped: true, reason: "includeSerp is false" } };
+        }
+        try {
+          const serp = await ctx.client.post("/harvest/sync", {
+            query: input.keyword,
+            location: input.location,
+            serpOnly: true,
+            maxQuestions: 1,
+            format: "json"
+          }, 18e4);
+          await ctx.artifacts.writeJson("SERP raw JSON", "raw/serp.json", serp);
+          return {
+            state: { ...state, serp },
+            output: {
+              organicResults: serp.organicResults?.length ?? 0,
+              localPack: serp.localPack?.length ?? 0,
+              aiOverviewDetected: Boolean(serp.aiOverview?.detected),
+              aiOverviewCitations: serp.aiOverview?.citations?.length ?? 0
+            }
+          };
+        } catch (err) {
+          const message = `SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
+          return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
+        }
       }
-    }
-    if (input.includePaa) {
-      try {
-        paa = await ctx.client.post("/harvest/sync", {
-          query: input.keyword,
-          location: input.location,
-          maxQuestions: input.maxQuestions,
-          format: "json"
-        }, 28e4);
-      } catch (err) {
-        warnings.push(`PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
+    },
+    {
+      id: "harvest-paa",
+      title: "Harvest People Also Ask",
+      async run({ input, state, ctx }) {
+        if (!input.includePaa) {
+          return { state, output: { skipped: true, reason: "includePaa is false" } };
+        }
+        try {
+          const paa = await ctx.client.post("/harvest/sync", {
+            query: input.keyword,
+            location: input.location,
+            maxQuestions: input.maxQuestions,
+            format: "json"
+          }, 28e4);
+          await ctx.artifacts.writeJson("PAA raw JSON", "raw/paa.json", paa);
+          return {
+            state: { ...state, paa },
+            output: { paaQuestions: paa.flat?.length ?? 0 }
+          };
+        } catch (err) {
+          const message = `PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
+          return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
+        }
+      }
+    },
+    {
+      id: "assemble",
+      title: "Assemble evidence packet",
+      async run({ input, state, ctx }) {
+        const { serp, paa } = state;
+        const warnings = state.warnings;
+        if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
+        const rows = sourceRows(input, serp, paa);
+        const target = normalizeDomain(input.domain);
+        const competitors = competitorRows(rows, target);
+        const evidence = {
+          input,
+          serp: serp ?? { status: "skipped" },
+          paa: paa ?? { status: "skipped" },
+          target: {
+            domain: target,
+            organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
+            citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
+            citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
+          },
+          competitors,
+          warnings
+        };
+        await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
+        await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
+        await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
+        const brief = [
+          `# SEO Evidence Brief: ${input.keyword}`,
+          "",
+          `Location: ${input.location ?? "not specified"}`,
+          `Target domain: ${target ?? "not specified"}`,
+          "",
+          "## Evidence Summary",
+          `- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
+          `- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
+          `- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
+          `- Competitor domains: ${competitors.length}`,
+          "",
+          "## Recommended Use",
+          "Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
+        ].join("\n");
+        await ctx.artifacts.writeText("Brief", "brief.md", brief);
+        await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
+          "# Agent Tasks",
+          "",
+          "- [ ] Read `evidence.json` before writing recommendations.",
+          "- [ ] Compare the target domain against `competitors.csv`.",
+          "- [ ] Use `sources.csv` for citations and source-grounded page sections.",
+          "- [ ] Mark unsupported recommendations as assumptions."
+        ].join("\n"));
+        await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
+          "# Agent Instructions",
+          "",
+          "You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
+        ].join("\n"));
+        const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
+        const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
+          title: "Agent-Ready SEO Packet",
+          subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
+          summary,
+          warnings,
+          tables: [
+            { title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
+            { title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
+          ]
+        }));
+        const status = warnings.length ? "partial" : "succeeded";
+        const counts = { sources: rows.length, competitors: competitors.length };
+        await ctx.artifacts.writeManifest(status, counts, warnings, []);
+        return {
+          state,
+          output: { sources: rows.length, competitors: competitors.length, status },
+          summary: { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors: [], reportPath }
+        };
       }
     }
-    if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
-    const rows = sourceRows(input, serp, paa);
-    const target = normalizeDomain(input.domain);
-    const competitors = competitorRows(rows, target);
-    const evidence = {
-      input,
-      serp: serp ?? { status: "skipped" },
-      paa: paa ?? { status: "skipped" },
-      target: {
-        domain: target,
-        organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
-        citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
-        citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
-      },
-      competitors,
-      warnings
-    };
-    await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
-    await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
-    await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
-    const brief = [
-      `# SEO Evidence Brief: ${input.keyword}`,
-      "",
-      `Location: ${input.location ?? "not specified"}`,
-      `Target domain: ${target ?? "not specified"}`,
-      "",
-      "## Evidence Summary",
-      `- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
-      `- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
-      `- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
-      `- Competitor domains: ${competitors.length}`,
-      "",
-      "## Recommended Use",
-      "Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
-    ].join("\n");
-    await ctx.artifacts.writeText("Brief", "brief.md", brief);
-    await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
-      "# Agent Tasks",
-      "",
-      "- [ ] Read `evidence.json` before writing recommendations.",
-      "- [ ] Compare the target domain against `competitors.csv`.",
-      "- [ ] Use `sources.csv` for citations and source-grounded page sections.",
-      "- [ ] Mark unsupported recommendations as assumptions."
-    ].join("\n"));
-    await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
-      "# Agent Instructions",
-      "",
-      "You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
-    ].join("\n"));
-    const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
-    const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
-      title: "Agent-Ready SEO Packet",
-      subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
-      summary,
-      warnings,
-      tables: [
-        { title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
-        { title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
-      ]
-    }));
-    const status = warnings.length ? "partial" : "succeeded";
-    const counts = { sources: rows.length, competitors: competitors.length };
-    await ctx.artifacts.writeManifest(status, counts, warnings, errors);
-    return { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors, reportPath };
-  }
+  ]
 };
 
 // src/workflows/workflows/directory.ts
@@ -494,7 +538,7 @@ var DirectoryWorkflowCliInputSchema = z2.object({
   maxCities: z2.number().int().min(1).max(100).default(25),
   maxResultsPerCity: z2.number().int().min(1).max(50).default(20),
   concurrency: z2.number().int().min(1).max(5).default(5),
-  proxyMode: z2.enum(["location", "configured", "none"]).default("location"),
+  proxyMode: z2.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
   saveCsv: z2.boolean().default(true)
 });
 function directoryRows(result) {
@@ -629,7 +673,7 @@ var LocalCompetitiveAuditInputSchema = z3.object({
   hydrateTop: z3.number().int().min(0).max(10).default(5),
   maxReviews: z3.number().int().min(0).max(500).default(50),
   concurrency: z3.number().int().min(1).max(5).default(5),
-  proxyMode: z3.enum(["location", "configured", "none"]).default("location"),
+  proxyMode: z3.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
   returnPartial: z3.boolean().default(true)
 });
 async function mapLimit(items, limit, fn) {
@@ -987,7 +1031,7 @@ var MapComparisonInputSchema = z4.object({
   hydrateTop: z4.number().int().min(0).max(10).default(5),
   maxReviews: z4.number().int().min(0).max(500).default(25),
   concurrency: z4.number().int().min(1).max(5).default(5),
-  proxyMode: ProxyModeSchema.default("location"),
+  proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
   returnPartial: z4.boolean().default(true)
 }).refine((input) => input.location || input.state, {
   message: "Either location or state is required for map-comparison"
@@ -1534,22 +1578,44 @@ function workflowDefinition(id) {
   if (!definition) throw new Error(`Unknown workflow "${id}". Available: ${DEFINITIONS.map((def) => def.id).join(", ")}`);
   return definition;
 }
-async function runWorkflow(id, rawInput, options = {}) {
+function workflowStepCount(id) {
   const definition = workflowDefinition(id);
-  const input = definition.inputSchema.parse(rawInput);
+  return definition.steps?.length ?? 1;
+}
+function workflowSupportsSteps(id) {
+  const definition = workflowDefinition(id);
+  return Boolean(definition.steps && definition.steps.length > 0);
+}
+function resolveApiKey(options) {
   const apiKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim();
   if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
-  const apiUrl = options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
+  return apiKey;
+}
+function resolveApiUrl(options) {
+  return options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
+}
+async function runWorkflow(id, rawInput, options = {}) {
+  const definition = workflowDefinition(id);
+  const input = definition.inputSchema.parse(rawInput);
+  const apiKey = resolveApiKey(options);
+  const apiUrl = resolveApiUrl(options);
   const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
   const client = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
+  const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client, artifacts, signal: options.signal };
   try {
-    return await definition.run(input, {
-      runId: artifacts.runId,
-      startedAt: artifacts.startedAt,
-      client,
-      artifacts,
-      signal: options.signal
-    });
+    if (definition.steps && definition.steps.length > 0) {
+      let state = definition.createState ? definition.createState(input) : {};
+      let summary = null;
+      for (const step of definition.steps) {
+        const outcome = await step.run({ input, state, ctx });
+        state = outcome.state;
+        if (outcome.summary) summary = outcome.summary;
+      }
+      if (!summary) throw new Error(`Workflow "${id}" produced no terminal step summary`);
+      return summary;
+    }
+    if (definition.run) return await definition.run(input, ctx);
+    throw new Error(`Workflow "${id}" has neither steps nor a run() implementation`);
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
     await artifacts.writeText("Failure", "summary.md", `# ${definition.title}
@@ -1560,6 +1626,45 @@ ${message}
     throw err;
   }
 }
+async function runWorkflowStep(id, rawInput, opts) {
+  const definition = workflowDefinition(id);
+  const steps = definition.steps;
+  if (!steps || steps.length === 0) throw new Error(`Workflow "${id}" does not support stepwise execution`);
+  if (opts.stepIndex < 0 || opts.stepIndex >= steps.length) {
+    throw new Error(`Step index ${opts.stepIndex} is out of range for "${id}" (${steps.length} steps)`);
+  }
+  const options = opts.options ?? {};
+  const input = definition.inputSchema.parse(rawInput);
+  const apiKey = resolveApiKey(options);
+  const apiUrl = resolveApiUrl(options);
+  const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
+  const client = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
+  const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client, artifacts, signal: options.signal };
+  const state = opts.state ?? (definition.createState ? definition.createState(input) : {});
+  const step = steps[opts.stepIndex];
+  const outcome = await step.run({ input, state, ctx });
+  const writtenArtifacts = await Promise.all(
+    artifacts.artifacts.map(async (artifact) => ({
+      ...artifact,
+      content: await readFile2(artifact.path, "utf8").catch(() => "")
+    }))
+  );
+  return {
+    runId: artifacts.runId,
+    workflowId: definition.id,
+    title: definition.title,
+    stepIndex: opts.stepIndex,
+    stepId: step.id,
+    stepTitle: step.title,
+    totalSteps: steps.length,
+    isLast: opts.stepIndex === steps.length - 1,
+    output: outcome.output,
+    state: outcome.state,
+    warnings: outcome.warnings ?? [],
+    summary: outcome.summary ?? null,
+    artifacts: writtenArtifacts
+  };
+}
 
 export {
   csvRecords,
@@ -1570,6 +1675,9 @@ export {
   openWorkflowReport,
   listWorkflowDefinitions,
   workflowDefinition,
-  runWorkflow
+  workflowStepCount,
+  workflowSupportsSteps,
+  runWorkflow,
+  runWorkflowStep
 };
-//# sourceMappingURL=chunk-F44RBOJ5.js.map
+//# sourceMappingURL=chunk-H2R232HK.js.map