mcp-scraper 0.2.19 → 0.2.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +3 -3
  2. package/dist/bin/api-server.cjs +1020 -348
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/browser-agent-stdio-server.cjs +314 -55
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +2 -2
  8. package/dist/bin/mcp-scraper-cli.cjs +321 -167
  9. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-cli.js +3 -2
  11. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  12. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +1128 -530
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-scraper-combined-stdio-server.js +5 -4
  15. package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
  16. package/dist/bin/mcp-scraper-install.cjs +3 -3
  17. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  18. package/dist/bin/mcp-scraper-install.js +2 -2
  19. package/dist/bin/mcp-stdio-server.cjs +726 -387
  20. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  21. package/dist/bin/mcp-stdio-server.js +3 -2
  22. package/dist/bin/mcp-stdio-server.js.map +1 -1
  23. package/dist/bin/paa-harvest.cjs +5 -3
  24. package/dist/bin/paa-harvest.cjs.map +1 -1
  25. package/dist/bin/paa-harvest.js +2 -1
  26. package/dist/bin/paa-harvest.js.map +1 -1
  27. package/dist/{chunk-TL7YTFLH.js → chunk-DBQDG7EH.js} +99 -27
  28. package/dist/chunk-DBQDG7EH.js.map +1 -0
  29. package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js} +216 -108
  30. package/dist/chunk-H2R232HK.js.map +1 -0
  31. package/dist/{chunk-76P4DKWR.js → chunk-L4OWOUGR.js} +315 -56
  32. package/dist/chunk-L4OWOUGR.js.map +1 -0
  33. package/dist/chunk-LFATOGDF.js +111 -0
  34. package/dist/chunk-LFATOGDF.js.map +1 -0
  35. package/dist/{chunk-MCBOZ2JF.js → chunk-MLNCKQ2B.js} +3 -3
  36. package/dist/chunk-MLNCKQ2B.js.map +1 -0
  37. package/dist/{chunk-4UEVYYDW.js → chunk-O5J7E4VX.js} +394 -57
  38. package/dist/chunk-O5J7E4VX.js.map +1 -0
  39. package/dist/{chunk-BSYPATSM.js → chunk-SXTXMFEQ.js} +2 -2
  40. package/dist/{chunk-CQTAKXBN.js → chunk-XGUDTDZ2.js} +7 -104
  41. package/dist/chunk-XGUDTDZ2.js.map +1 -0
  42. package/dist/chunk-XKUDVN2E.js +7 -0
  43. package/dist/chunk-XKUDVN2E.js.map +1 -0
  44. package/dist/{db-P5X6UQ3E.js → db-BE4JVB3V.js} +8 -2
  45. package/dist/index.cjs +5 -3
  46. package/dist/index.cjs.map +1 -1
  47. package/dist/index.d.cts +2 -2
  48. package/dist/index.d.ts +2 -2
  49. package/dist/index.js +2 -1
  50. package/dist/index.js.map +1 -1
  51. package/dist/{server-CRUSZA2Q.js → server-7NE45K2A.js} +211 -37
  52. package/dist/server-7NE45K2A.js.map +1 -0
  53. package/dist/{worker-OZSWIS3F.js → worker-MIYG2B2I.js} +5 -4
  54. package/dist/{worker-OZSWIS3F.js.map → worker-MIYG2B2I.js.map} +1 -1
  55. package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +2 -2
  56. package/docs/mcp-tool-craft-lint.generated.md +51 -0
  57. package/docs/mcp-tool-manifest.generated.json +1406 -0
  58. package/docs/specs/local-competitive-audit-spec.md +1 -2
  59. package/docs/specs/mcp-tool-definition-quality-audit-spec.md +1602 -0
  60. package/package.json +4 -1
  61. package/dist/chunk-4UEVYYDW.js.map +0 -1
  62. package/dist/chunk-76P4DKWR.js.map +0 -1
  63. package/dist/chunk-CQTAKXBN.js.map +0 -1
  64. package/dist/chunk-F44RBOJ5.js.map +0 -1
  65. package/dist/chunk-MCBOZ2JF.js.map +0 -1
  66. package/dist/chunk-RU2HZD55.js +0 -7
  67. package/dist/chunk-RU2HZD55.js.map +0 -1
  68. package/dist/chunk-TL7YTFLH.js.map +0 -1
  69. package/dist/server-CRUSZA2Q.js.map +0 -1
  70. /package/dist/{chunk-BSYPATSM.js.map → chunk-SXTXMFEQ.js.map} +0 -0
  71. /package/dist/{db-P5X6UQ3E.js.map → db-BE4JVB3V.js.map} +0 -0
@@ -4809,7 +4809,7 @@ function parsePageData(url, html, status, via) {
4809
4809
  }
4810
4810
  const h1 = headings.find((h) => h.level === 1)?.text ?? null;
4811
4811
  const bodyText = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
4812
- const wordCount = bodyText.split(" ").filter((w) => w.length > 2).length;
4812
+ const wordCount2 = bodyText.split(" ").filter((w) => w.length > 2).length;
4813
4813
  const schemaTypes = [];
4814
4814
  const schema = [];
4815
4815
  for (const m of html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
@@ -4845,7 +4845,7 @@ function parsePageData(url, html, status, via) {
4845
4845
  } catch {
4846
4846
  }
4847
4847
  }
4848
- return { url, status, via, title, metaDescription, h1, headings, wordCount, schemaTypes, canonicalUrl, internalLinks, externalLinks, bodyMarkdown, schema };
4848
+ return { url, status, via, title, metaDescription, h1, headings, wordCount: wordCount2, schemaTypes, canonicalUrl, internalLinks, externalLinks, bodyMarkdown, schema };
4849
4849
  }
4850
4850
  async function fetchAndParse(url, kernelApiKey) {
4851
4851
  try {
@@ -6906,6 +6906,8 @@ var db_exports = {};
6906
6906
  __export(db_exports, {
6907
6907
  SiteAuditJobRowSchema: () => SiteAuditJobRowSchema,
6908
6908
  SiteAuditPhaseLogRowSchema: () => SiteAuditPhaseLogRowSchema,
6909
+ advanceWorkflowRunStep: () => advanceWorkflowRunStep,
6910
+ appendWorkflowArtifacts: () => appendWorkflowArtifacts,
6909
6911
  cancelJob: () => cancelJob,
6910
6912
  checkRateLimit: () => checkRateLimit,
6911
6913
  claimMonthlyFreeRefresh: () => claimMonthlyFreeRefresh,
@@ -6946,6 +6948,7 @@ __export(db_exports, {
6946
6948
  getWorkflowArtifact: () => getWorkflowArtifact,
6947
6949
  getWorkflowRun: () => getWorkflowRun,
6948
6950
  getWorkflowRunByIdempotencyKey: () => getWorkflowRunByIdempotencyKey,
6951
+ getWorkflowRunStepState: () => getWorkflowRunStepState,
6949
6952
  getWorkflowSchedule: () => getWorkflowSchedule,
6950
6953
  hashPassword: () => hashPassword,
6951
6954
  ledgerExistsForOperation: () => ledgerExistsForOperation,
@@ -7227,9 +7230,34 @@ async function migrate() {
7227
7230
  queued_at TEXT NOT NULL DEFAULT (datetime('now')),
7228
7231
  started_at TEXT,
7229
7232
  completed_at TEXT,
7230
- idempotency_key TEXT
7233
+ idempotency_key TEXT,
7234
+ step_index INTEGER NOT NULL DEFAULT 0,
7235
+ total_steps INTEGER,
7236
+ step_state_json TEXT,
7237
+ step_log_json TEXT,
7238
+ webhook_url TEXT
7231
7239
  )
7232
7240
  `);
7241
+ try {
7242
+ await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_index INTEGER NOT NULL DEFAULT 0`);
7243
+ } catch {
7244
+ }
7245
+ try {
7246
+ await db.execute(`ALTER TABLE workflow_runs ADD COLUMN total_steps INTEGER`);
7247
+ } catch {
7248
+ }
7249
+ try {
7250
+ await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_state_json TEXT`);
7251
+ } catch {
7252
+ }
7253
+ try {
7254
+ await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_log_json TEXT`);
7255
+ } catch {
7256
+ }
7257
+ try {
7258
+ await db.execute(`ALTER TABLE workflow_runs ADD COLUMN webhook_url TEXT`);
7259
+ } catch {
7260
+ }
7233
7261
  await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_user_created ON workflow_runs(user_id, queued_at DESC)`);
7234
7262
  await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_schedule ON workflow_runs(schedule_id, queued_at DESC)`);
7235
7263
  await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS workflow_runs_idempotency ON workflow_runs(idempotency_key) WHERE idempotency_key IS NOT NULL`);
@@ -7244,9 +7272,14 @@ async function migrate() {
7244
7272
  content_type TEXT NOT NULL,
7245
7273
  bytes INTEGER,
7246
7274
  rows_count INTEGER,
7275
+ content TEXT,
7247
7276
  created_at TEXT NOT NULL DEFAULT (datetime('now'))
7248
7277
  )
7249
7278
  `);
7279
+ try {
7280
+ await db.execute(`ALTER TABLE workflow_artifacts ADD COLUMN content TEXT`);
7281
+ } catch {
7282
+ }
7250
7283
  await db.execute(`CREATE INDEX IF NOT EXISTS workflow_artifacts_run ON workflow_artifacts(run_id)`);
7251
7284
  await db.execute(`
7252
7285
  CREATE TABLE IF NOT EXISTS workflow_webhook_deliveries (
@@ -7692,9 +7725,22 @@ function rowToWorkflowRun(row) {
7692
7725
  queued_at: String(row.queued_at),
7693
7726
  started_at: row.started_at != null ? String(row.started_at) : null,
7694
7727
  completed_at: row.completed_at != null ? String(row.completed_at) : null,
7695
- idempotency_key: row.idempotency_key != null ? String(row.idempotency_key) : null
7728
+ idempotency_key: row.idempotency_key != null ? String(row.idempotency_key) : null,
7729
+ step_index: row.step_index != null ? Number(row.step_index) : 0,
7730
+ total_steps: row.total_steps != null ? Number(row.total_steps) : null,
7731
+ step_log: parseWorkflowStepLog(row.step_log_json),
7732
+ webhook_url: row.webhook_url != null ? String(row.webhook_url) : null
7696
7733
  };
7697
7734
  }
7735
+ function parseWorkflowStepLog(value) {
7736
+ if (value == null) return [];
7737
+ try {
7738
+ const parsed = JSON.parse(String(value));
7739
+ return Array.isArray(parsed) ? parsed : [];
7740
+ } catch {
7741
+ return [];
7742
+ }
7743
+ }
7698
7744
  function rowToWorkflowArtifact(row) {
7699
7745
  return {
7700
7746
  id: String(row.id),
@@ -7706,6 +7752,7 @@ function rowToWorkflowArtifact(row) {
7706
7752
  content_type: String(row.content_type),
7707
7753
  bytes: row.bytes != null ? Number(row.bytes) : null,
7708
7754
  rows_count: row.rows_count != null ? Number(row.rows_count) : null,
7755
+ content: row.content != null ? String(row.content) : null,
7709
7756
  created_at: String(row.created_at)
7710
7757
  };
7711
7758
  }
@@ -7817,13 +7864,15 @@ async function createWorkflowRun(input) {
7817
7864
  input.scheduleId ?? null,
7818
7865
  input.workflowId,
7819
7866
  JSON.stringify(input.workflowInput),
7820
- input.idempotencyKey ?? null
7867
+ input.idempotencyKey ?? null,
7868
+ input.webhookUrl ?? null,
7869
+ input.totalSteps ?? null
7821
7870
  ];
7822
7871
  try {
7823
7872
  await getDb().execute({
7824
7873
  sql: `
7825
- INSERT INTO workflow_runs (id, user_id, schedule_id, workflow_id, input_json, idempotency_key)
7826
- VALUES (?, ?, ?, ?, ?, ?)
7874
+ INSERT INTO workflow_runs (id, user_id, schedule_id, workflow_id, input_json, idempotency_key, webhook_url, total_steps)
7875
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
7827
7876
  `,
7828
7877
  args
7829
7878
  });
@@ -7877,30 +7926,53 @@ function workflowArtifactContentType(kind) {
7877
7926
  if (kind === "json") return "application/json; charset=utf-8";
7878
7927
  return "text/plain; charset=utf-8";
7879
7928
  }
7929
+ async function insertWorkflowArtifact(runId, userId, artifact) {
7930
+ await getDb().execute({
7931
+ sql: `
7932
+ INSERT INTO workflow_artifacts (id, run_id, user_id, kind, label, path, content_type, bytes, rows_count, content)
7933
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
7934
+ `,
7935
+ args: [
7936
+ (0, import_node_crypto.randomUUID)(),
7937
+ runId,
7938
+ userId,
7939
+ artifact.kind,
7940
+ artifact.label,
7941
+ artifact.path,
7942
+ workflowArtifactContentType(artifact.kind),
7943
+ artifact.bytes ?? null,
7944
+ artifact.rows ?? null,
7945
+ artifact.content ?? null
7946
+ ]
7947
+ });
7948
+ }
7880
7949
  async function replaceWorkflowArtifacts(runId, userId, artifacts) {
7881
- const db = getDb();
7882
- await db.execute({ sql: "DELETE FROM workflow_artifacts WHERE run_id = ? AND user_id = ?", args: [runId, userId] });
7883
- for (const artifact of artifacts) {
7884
- await db.execute({
7885
- sql: `
7886
- INSERT INTO workflow_artifacts (id, run_id, user_id, kind, label, path, content_type, bytes, rows_count)
7887
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
7888
- `,
7889
- args: [
7890
- (0, import_node_crypto.randomUUID)(),
7891
- runId,
7892
- userId,
7893
- artifact.kind,
7894
- artifact.label,
7895
- artifact.path,
7896
- workflowArtifactContentType(artifact.kind),
7897
- artifact.bytes ?? null,
7898
- artifact.rows ?? null
7899
- ]
7900
- });
7901
- }
7950
+ await getDb().execute({ sql: "DELETE FROM workflow_artifacts WHERE run_id = ? AND user_id = ?", args: [runId, userId] });
7951
+ for (const artifact of artifacts) await insertWorkflowArtifact(runId, userId, artifact);
7902
7952
  return listWorkflowArtifacts(runId, userId);
7903
7953
  }
7954
+ async function appendWorkflowArtifacts(runId, userId, artifacts) {
7955
+ for (const artifact of artifacts) await insertWorkflowArtifact(runId, userId, artifact);
7956
+ return listWorkflowArtifacts(runId, userId);
7957
+ }
7958
+ async function getWorkflowRunStepState(id, userId) {
7959
+ const res = userId == null ? await getDb().execute({ sql: "SELECT step_index, total_steps, step_state_json, step_log_json FROM workflow_runs WHERE id = ? LIMIT 1", args: [id] }) : await getDb().execute({ sql: "SELECT step_index, total_steps, step_state_json, step_log_json FROM workflow_runs WHERE id = ? AND user_id = ? LIMIT 1", args: [id, userId] });
7960
+ const row = res.rows[0];
7961
+ if (!row) return null;
7962
+ return {
7963
+ stepIndex: row.step_index != null ? Number(row.step_index) : 0,
7964
+ totalSteps: row.total_steps != null ? Number(row.total_steps) : null,
7965
+ state: row.step_state_json != null ? parseNullableJson(row.step_state_json) : null,
7966
+ stepLog: parseWorkflowStepLog(row.step_log_json)
7967
+ };
7968
+ }
7969
+ async function advanceWorkflowRunStep(id, input) {
7970
+ const stepLog = [...input.appendLog, input.logEntry];
7971
+ await getDb().execute({
7972
+ sql: `UPDATE workflow_runs SET status = ?, step_index = ?, total_steps = ?, step_state_json = ?, step_log_json = ?, started_at = COALESCE(started_at, datetime('now')) WHERE id = ?`,
7973
+ args: [input.status, input.stepIndex, input.totalSteps, JSON.stringify(input.state ?? null), JSON.stringify(stepLog), id]
7974
+ });
7975
+ }
7904
7976
  async function listWorkflowArtifacts(runId, userId) {
7905
7977
  const res = await getDb().execute({
7906
7978
  sql: "SELECT * FROM workflow_artifacts WHERE run_id = ? AND user_id = ? ORDER BY created_at ASC",
@@ -8909,6 +8981,7 @@ var init_schemas2 = __esm({
8909
8981
  downloadMp3: import_zod12.z.boolean().default(false),
8910
8982
  outputDir: import_zod12.z.string().default("./yt-output"),
8911
8983
  kernelApiKey: import_zod12.z.string().optional(),
8984
+ kernelProxyId: import_zod12.z.string().optional(),
8912
8985
  headless: import_zod12.z.boolean().default(false)
8913
8986
  }).refine(
8914
8987
  (o) => o.mode === "search" ? !!o.query : !!o.channelHandle,
@@ -9833,6 +9906,7 @@ var init_YouTubeExtractor = __esm({
9833
9906
  const config = {
9834
9907
  headless: options.headless,
9835
9908
  kernelApiKey: options.kernelApiKey,
9909
+ kernelProxyId: options.kernelProxyId,
9836
9910
  viewport: { width: 1280, height: 800 },
9837
9911
  locale: "en-US"
9838
9912
  };
@@ -10044,7 +10118,11 @@ async function ytHarvest(rawOptions) {
10044
10118
  throw new Error("A browser backend API key is required \u2014 YouTube harvesting requires a stealth session.");
10045
10119
  }
10046
10120
  const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};
10047
- const options = YouTubeHarvestOptionsSchema.parse({ kernelApiKey, ...raw });
10121
+ const options = YouTubeHarvestOptionsSchema.parse({
10122
+ kernelApiKey,
10123
+ kernelProxyId: browserServiceProxyId(),
10124
+ ...raw
10125
+ });
10048
10126
  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
10049
10127
  try {
10050
10128
  const result = await extractOnce(options);
@@ -10153,10 +10231,11 @@ function parseTimedtextXml(xml) {
10153
10231
  async function fetchViaKernelInnertube(videoId) {
10154
10232
  const kernelApiKey = browserServiceApiKey();
10155
10233
  if (!kernelApiKey) return null;
10234
+ const kernelProxyId = browserServiceProxyId();
10156
10235
  const driver = new BrowserDriver();
10157
10236
  const start = Date.now();
10158
10237
  try {
10159
- await driver.launch({ kernelApiKey, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
10238
+ await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
10160
10239
  const page = driver.getPage();
10161
10240
  await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
10162
10241
  await page.waitForFunction(
@@ -10250,10 +10329,10 @@ async function captureAudioViaMediaRecorder(page) {
10250
10329
  if (!result.ok || !result.base64 || !result.mimeType) return null;
10251
10330
  return { bytes: Buffer.from(result.base64, "base64"), mimeType: result.mimeType };
10252
10331
  }
10253
- async function attemptKernelWhisper(videoId, kernelApiKey, falKey, start) {
10332
+ async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start) {
10254
10333
  const driver = new BrowserDriver();
10255
10334
  try {
10256
- await driver.launch({ kernelApiKey, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
10335
+ await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
10257
10336
  const page = driver.getPage();
10258
10337
  await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
10259
10338
  await page.waitForFunction(
@@ -10295,12 +10374,13 @@ async function attemptKernelWhisper(videoId, kernelApiKey, falKey, start) {
10295
10374
  }
10296
10375
  async function fetchViaKernelWhisper(videoId) {
10297
10376
  const kernelApiKey = browserServiceApiKey();
10377
+ const kernelProxyId = browserServiceProxyId();
10298
10378
  const falKey = process.env.FAL_KEY;
10299
10379
  if (!kernelApiKey || !falKey) return null;
10300
10380
  const start = Date.now();
10301
10381
  for (let attempt = 0; attempt < 4; attempt++) {
10302
10382
  if (attempt > 0) await new Promise((r) => setTimeout(r, 2e3 * attempt));
10303
- const result = await attemptKernelWhisper(videoId, kernelApiKey, falKey, start);
10383
+ const result = await attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start);
10304
10384
  if (result) return result;
10305
10385
  }
10306
10386
  return null;
@@ -12283,6 +12363,110 @@ var init_kernel_proxy_resolver = __esm({
12283
12363
  }
12284
12364
  });
12285
12365
 
12366
+ // src/schemas.ts
12367
+ var import_zod15, DEFAULT_PROXY_MODE, DEFAULT_MAPS_PROXY_MODE, HarvestOptionsSchema, MapsPlaceOptionsSchema, MapsSearchOptionsSchema, RawPAAItemSchema, RawMapsOverviewSchema, RawMapsHoursRowSchema, RawMapsReviewStatsSchema, RawMapsReviewCardSchema, RawMapsAboutAttributeSchema;
12368
+ var init_schemas3 = __esm({
12369
+ "src/schemas.ts"() {
12370
+ "use strict";
12371
+ import_zod15 = require("zod");
12372
+ DEFAULT_PROXY_MODE = "configured";
12373
+ DEFAULT_MAPS_PROXY_MODE = "location";
12374
+ HarvestOptionsSchema = import_zod15.z.object({
12375
+ query: import_zod15.z.string().min(1),
12376
+ location: import_zod15.z.string().optional(),
12377
+ gl: import_zod15.z.string().length(2).default("us"),
12378
+ hl: import_zod15.z.string().length(2).default("en"),
12379
+ device: import_zod15.z.enum(["desktop", "mobile"]).default("desktop"),
12380
+ proxyMode: import_zod15.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
12381
+ proxyZip: import_zod15.z.string().regex(/^\d{5}$/).optional(),
12382
+ debug: import_zod15.z.boolean().default(false),
12383
+ depth: import_zod15.z.number().int().min(1).max(30).default(3),
12384
+ maxQuestions: import_zod15.z.number().int().min(1).max(1e3).default(100),
12385
+ headless: import_zod15.z.boolean().default(false),
12386
+ profileDir: import_zod15.z.string().optional(),
12387
+ proxy: import_zod15.z.string().url().optional(),
12388
+ kernelApiKey: import_zod15.z.string().optional(),
12389
+ kernelProxyId: import_zod15.z.string().optional(),
12390
+ kernelProxyResolution: import_zod15.z.unknown().optional(),
12391
+ outputDir: import_zod15.z.string().default("./paa-output"),
12392
+ format: import_zod15.z.enum(["json", "csv", "both"]).default("both"),
12393
+ serpOnly: import_zod15.z.boolean().default(false),
12394
+ pages: import_zod15.z.number().int().min(1).max(2).default(1)
12395
+ });
12396
+ MapsPlaceOptionsSchema = import_zod15.z.object({
12397
+ businessName: import_zod15.z.string().min(1),
12398
+ location: import_zod15.z.string().min(1),
12399
+ gl: import_zod15.z.string().length(2).default("us"),
12400
+ hl: import_zod15.z.string().length(2).default("en"),
12401
+ includeReviews: import_zod15.z.boolean().default(false),
12402
+ maxReviews: import_zod15.z.number().int().min(1).max(500).default(50),
12403
+ kernelApiKey: import_zod15.z.string().optional(),
12404
+ kernelProxyId: import_zod15.z.string().optional(),
12405
+ headless: import_zod15.z.boolean().default(true)
12406
+ });
12407
+ MapsSearchOptionsSchema = import_zod15.z.object({
12408
+ query: import_zod15.z.string().min(1),
12409
+ location: import_zod15.z.string().optional(),
12410
+ gl: import_zod15.z.string().length(2).default("us"),
12411
+ hl: import_zod15.z.string().length(2).default("en"),
12412
+ maxResults: import_zod15.z.number().int().min(1).max(50).default(10),
12413
+ proxyMode: import_zod15.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
12414
+ proxyZip: import_zod15.z.string().regex(/^\d{5}$/).optional(),
12415
+ debug: import_zod15.z.boolean().default(false),
12416
+ kernelApiKey: import_zod15.z.string().optional(),
12417
+ kernelProxyId: import_zod15.z.string().optional(),
12418
+ kernelProxyResolution: import_zod15.z.unknown().optional(),
12419
+ headless: import_zod15.z.boolean().default(true)
12420
+ });
12421
+ RawPAAItemSchema = import_zod15.z.object({
12422
+ question: import_zod15.z.string().min(1),
12423
+ answer: import_zod15.z.string().optional(),
12424
+ sourceTitle: import_zod15.z.string().optional(),
12425
+ sourceSite: import_zod15.z.string().optional(),
12426
+ sourceCite: import_zod15.z.string().optional()
12427
+ });
12428
+ RawMapsOverviewSchema = import_zod15.z.object({
12429
+ name: import_zod15.z.string().nullable(),
12430
+ rating: import_zod15.z.string().nullable(),
12431
+ reviewCount: import_zod15.z.string().nullable(),
12432
+ category: import_zod15.z.string().nullable(),
12433
+ address: import_zod15.z.string().nullable(),
12434
+ hoursSummary: import_zod15.z.string().nullable(),
12435
+ phone: import_zod15.z.string().nullable(),
12436
+ phoneDisplay: import_zod15.z.string().nullable(),
12437
+ website: import_zod15.z.string().nullable(),
12438
+ plusCode: import_zod15.z.string().nullable(),
12439
+ bookingUrl: import_zod15.z.string().nullable()
12440
+ });
12441
+ RawMapsHoursRowSchema = import_zod15.z.object({
12442
+ day: import_zod15.z.string(),
12443
+ hours: import_zod15.z.string()
12444
+ });
12445
+ RawMapsReviewStatsSchema = import_zod15.z.object({
12446
+ reviewHistogram: import_zod15.z.array(import_zod15.z.object({
12447
+ stars: import_zod15.z.number(),
12448
+ count: import_zod15.z.string()
12449
+ })),
12450
+ reviewTopics: import_zod15.z.array(import_zod15.z.object({
12451
+ label: import_zod15.z.string(),
12452
+ count: import_zod15.z.string()
12453
+ }))
12454
+ });
12455
+ RawMapsReviewCardSchema = import_zod15.z.object({
12456
+ reviewId: import_zod15.z.string(),
12457
+ author: import_zod15.z.string().nullable(),
12458
+ stars: import_zod15.z.string().nullable(),
12459
+ date: import_zod15.z.string().nullable(),
12460
+ text: import_zod15.z.string().nullable(),
12461
+ ownerResponse: import_zod15.z.string().nullable()
12462
+ });
12463
+ RawMapsAboutAttributeSchema = import_zod15.z.object({
12464
+ section: import_zod15.z.string(),
12465
+ attribute: import_zod15.z.string()
12466
+ });
12467
+ }
12468
+ });
12469
+
12286
12470
  // src/api/facebook-ad-routes.ts
12287
12471
  function invalidRequest(message) {
12288
12472
  return { error_code: "invalid_request", message };
@@ -12338,9 +12522,8 @@ async function kernelLaunchOptsResidential() {
12338
12522
  try {
12339
12523
  const resolution2 = await resolveKernelProxyId({
12340
12524
  kernelApiKey: browserServiceApiKey(),
12341
- proxyMode: "location",
12525
+ proxyMode: DEFAULT_PROXY_MODE,
12342
12526
  configuredKernelProxyId: browserServiceProxyId(),
12343
- location: "New York, NY",
12344
12527
  gl: "us"
12345
12528
  });
12346
12529
  if (resolution2.kernelProxyId) proxyId = resolution2.kernelProxyId;
@@ -12349,13 +12532,13 @@ async function kernelLaunchOptsResidential() {
12349
12532
  }
12350
12533
  return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
12351
12534
  }
12352
- var import_hono4, import_zod15, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
12535
+ var import_hono4, import_zod16, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
12353
12536
  var init_facebook_ad_routes = __esm({
12354
12537
  "src/api/facebook-ad-routes.ts"() {
12355
12538
  "use strict";
12356
12539
  import_hono4 = require("hono");
12357
12540
  init_browser_service_env();
12358
- import_zod15 = require("zod");
12541
+ import_zod16 = require("zod");
12359
12542
  init_db();
12360
12543
  init_rates();
12361
12544
  init_BrowserDriver();
@@ -12363,39 +12546,40 @@ var init_facebook_ad_routes = __esm({
12363
12546
  init_FacebookAdGraphql();
12364
12547
  init_FacebookOrganicVideoExtractor();
12365
12548
  init_kernel_proxy_resolver();
12549
+ init_schemas3();
12366
12550
  import_client3 = require("@fal-ai/client");
12367
12551
  init_api_auth();
12368
12552
  init_url_utils();
12369
12553
  init_concurrency_gates();
12370
- FacebookAdBodySchema = import_zod15.z.object({
12371
- url: import_zod15.z.string().trim().optional(),
12372
- libraryId: import_zod15.z.string().trim().optional(),
12373
- openModal: import_zod15.z.boolean().optional()
12554
+ FacebookAdBodySchema = import_zod16.z.object({
12555
+ url: import_zod16.z.string().trim().optional(),
12556
+ libraryId: import_zod16.z.string().trim().optional(),
12557
+ openModal: import_zod16.z.boolean().optional()
12374
12558
  }).refine((d) => !!d.url || !!d.libraryId, { message: "url or libraryId is required" });
12375
- FacebookPageIntelBodySchema = import_zod15.z.object({
12376
- pageId: import_zod15.z.string().trim().optional(),
12377
- query: import_zod15.z.string().trim().optional(),
12378
- libraryId: import_zod15.z.string().trim().optional(),
12379
- maxAds: import_zod15.z.number().int().min(1).max(200).optional(),
12380
- country: import_zod15.z.string().trim().toUpperCase().optional()
12559
+ FacebookPageIntelBodySchema = import_zod16.z.object({
12560
+ pageId: import_zod16.z.string().trim().optional(),
12561
+ query: import_zod16.z.string().trim().optional(),
12562
+ libraryId: import_zod16.z.string().trim().optional(),
12563
+ maxAds: import_zod16.z.number().int().min(1).max(200).optional(),
12564
+ country: import_zod16.z.string().trim().toUpperCase().optional()
12381
12565
  }).refine((d) => !!d.pageId || !!d.query || !!d.libraryId, {
12382
12566
  message: "pageId, libraryId, or query is required"
12383
12567
  });
12384
- FacebookTranscribeBodySchema = import_zod15.z.object({
12385
- videoUrl: import_zod15.z.string().trim().min(1, "videoUrl is required")
12568
+ FacebookTranscribeBodySchema = import_zod16.z.object({
12569
+ videoUrl: import_zod16.z.string().trim().min(1, "videoUrl is required")
12386
12570
  });
12387
- FacebookVideoTranscribeBodySchema = import_zod15.z.object({
12388
- url: import_zod15.z.string().trim().min(1, "url is required"),
12389
- quality: import_zod15.z.enum(["best", "hd", "sd"]).default("best")
12571
+ FacebookVideoTranscribeBodySchema = import_zod16.z.object({
12572
+ url: import_zod16.z.string().trim().min(1, "url is required"),
12573
+ quality: import_zod16.z.enum(["best", "hd", "sd"]).default("best")
12390
12574
  });
12391
- FacebookSearchBodySchema = import_zod15.z.object({
12392
- query: import_zod15.z.string().trim().min(1, "query is required"),
12393
- country: import_zod15.z.string().trim().toUpperCase().optional(),
12394
- maxResults: import_zod15.z.number().int().min(1).max(20).optional()
12575
+ FacebookSearchBodySchema = import_zod16.z.object({
12576
+ query: import_zod16.z.string().trim().min(1, "query is required"),
12577
+ country: import_zod16.z.string().trim().toUpperCase().optional(),
12578
+ maxResults: import_zod16.z.number().int().min(1).max(20).optional()
12395
12579
  });
12396
- FacebookMediaBodySchema = import_zod15.z.object({
12397
- url: import_zod15.z.string().trim().min(1, "url is required"),
12398
- filename: import_zod15.z.string().trim().optional()
12580
+ FacebookMediaBodySchema = import_zod16.z.object({
12581
+ url: import_zod16.z.string().trim().min(1, "url is required"),
12582
+ filename: import_zod16.z.string().trim().optional()
12399
12583
  });
12400
12584
  facebookAdApp = new import_hono4.Hono();
12401
12585
  facebookAdApp.post("/ad", createApiKeyAuth(), async (c) => {
@@ -12721,108 +12905,6 @@ var init_facebook_ad_routes = __esm({
12721
12905
  }
12722
12906
  });
12723
12907
 
12724
- // src/schemas.ts
12725
- var import_zod16, HarvestOptionsSchema, MapsPlaceOptionsSchema, MapsSearchOptionsSchema, RawPAAItemSchema, RawMapsOverviewSchema, RawMapsHoursRowSchema, RawMapsReviewStatsSchema, RawMapsReviewCardSchema, RawMapsAboutAttributeSchema;
12726
- var init_schemas3 = __esm({
12727
- "src/schemas.ts"() {
12728
- "use strict";
12729
- import_zod16 = require("zod");
12730
- HarvestOptionsSchema = import_zod16.z.object({
12731
- query: import_zod16.z.string().min(1),
12732
- location: import_zod16.z.string().optional(),
12733
- gl: import_zod16.z.string().length(2).default("us"),
12734
- hl: import_zod16.z.string().length(2).default("en"),
12735
- device: import_zod16.z.enum(["desktop", "mobile"]).default("desktop"),
12736
- proxyMode: import_zod16.z.enum(["location", "configured", "none"]).default("location"),
12737
- proxyZip: import_zod16.z.string().regex(/^\d{5}$/).optional(),
12738
- debug: import_zod16.z.boolean().default(false),
12739
- depth: import_zod16.z.number().int().min(1).max(30).default(3),
12740
- maxQuestions: import_zod16.z.number().int().min(1).max(1e3).default(100),
12741
- headless: import_zod16.z.boolean().default(false),
12742
- profileDir: import_zod16.z.string().optional(),
12743
- proxy: import_zod16.z.string().url().optional(),
12744
- kernelApiKey: import_zod16.z.string().optional(),
12745
- kernelProxyId: import_zod16.z.string().optional(),
12746
- kernelProxyResolution: import_zod16.z.unknown().optional(),
12747
- outputDir: import_zod16.z.string().default("./paa-output"),
12748
- format: import_zod16.z.enum(["json", "csv", "both"]).default("both"),
12749
- serpOnly: import_zod16.z.boolean().default(false),
12750
- pages: import_zod16.z.number().int().min(1).max(2).default(1)
12751
- });
12752
- MapsPlaceOptionsSchema = import_zod16.z.object({
12753
- businessName: import_zod16.z.string().min(1),
12754
- location: import_zod16.z.string().min(1),
12755
- gl: import_zod16.z.string().length(2).default("us"),
12756
- hl: import_zod16.z.string().length(2).default("en"),
12757
- includeReviews: import_zod16.z.boolean().default(false),
12758
- maxReviews: import_zod16.z.number().int().min(1).max(500).default(50),
12759
- kernelApiKey: import_zod16.z.string().optional(),
12760
- kernelProxyId: import_zod16.z.string().optional(),
12761
- headless: import_zod16.z.boolean().default(true)
12762
- });
12763
- MapsSearchOptionsSchema = import_zod16.z.object({
12764
- query: import_zod16.z.string().min(1),
12765
- location: import_zod16.z.string().optional(),
12766
- gl: import_zod16.z.string().length(2).default("us"),
12767
- hl: import_zod16.z.string().length(2).default("en"),
12768
- maxResults: import_zod16.z.number().int().min(1).max(50).default(10),
12769
- proxyMode: import_zod16.z.enum(["location", "configured", "none"]).default("location"),
12770
- proxyZip: import_zod16.z.string().regex(/^\d{5}$/).optional(),
12771
- debug: import_zod16.z.boolean().default(false),
12772
- kernelApiKey: import_zod16.z.string().optional(),
12773
- kernelProxyId: import_zod16.z.string().optional(),
12774
- kernelProxyResolution: import_zod16.z.unknown().optional(),
12775
- headless: import_zod16.z.boolean().default(true)
12776
- });
12777
- RawPAAItemSchema = import_zod16.z.object({
12778
- question: import_zod16.z.string().min(1),
12779
- answer: import_zod16.z.string().optional(),
12780
- sourceTitle: import_zod16.z.string().optional(),
12781
- sourceSite: import_zod16.z.string().optional(),
12782
- sourceCite: import_zod16.z.string().optional()
12783
- });
12784
- RawMapsOverviewSchema = import_zod16.z.object({
12785
- name: import_zod16.z.string().nullable(),
12786
- rating: import_zod16.z.string().nullable(),
12787
- reviewCount: import_zod16.z.string().nullable(),
12788
- category: import_zod16.z.string().nullable(),
12789
- address: import_zod16.z.string().nullable(),
12790
- hoursSummary: import_zod16.z.string().nullable(),
12791
- phone: import_zod16.z.string().nullable(),
12792
- phoneDisplay: import_zod16.z.string().nullable(),
12793
- website: import_zod16.z.string().nullable(),
12794
- plusCode: import_zod16.z.string().nullable(),
12795
- bookingUrl: import_zod16.z.string().nullable()
12796
- });
12797
- RawMapsHoursRowSchema = import_zod16.z.object({
12798
- day: import_zod16.z.string(),
12799
- hours: import_zod16.z.string()
12800
- });
12801
- RawMapsReviewStatsSchema = import_zod16.z.object({
12802
- reviewHistogram: import_zod16.z.array(import_zod16.z.object({
12803
- stars: import_zod16.z.number(),
12804
- count: import_zod16.z.string()
12805
- })),
12806
- reviewTopics: import_zod16.z.array(import_zod16.z.object({
12807
- label: import_zod16.z.string(),
12808
- count: import_zod16.z.string()
12809
- }))
12810
- });
12811
- RawMapsReviewCardSchema = import_zod16.z.object({
12812
- reviewId: import_zod16.z.string(),
12813
- author: import_zod16.z.string().nullable(),
12814
- stars: import_zod16.z.string().nullable(),
12815
- date: import_zod16.z.string().nullable(),
12816
- text: import_zod16.z.string().nullable(),
12817
- ownerResponse: import_zod16.z.string().nullable()
12818
- });
12819
- RawMapsAboutAttributeSchema = import_zod16.z.object({
12820
- section: import_zod16.z.string(),
12821
- attribute: import_zod16.z.string()
12822
- });
12823
- }
12824
- });
12825
-
12826
12908
  // src/extractor/MapsNavigator.ts
12827
12909
  var MapsNavigator;
12828
12910
  var init_MapsNavigator = __esm({
@@ -14432,13 +14514,25 @@ ${videoRows}`,
14432
14514
  }
14433
14515
  };
14434
14516
  }
14517
+ function structuredTranscriptChunks(chunks) {
14518
+ return chunks.map((c) => ({
14519
+ startSec: Number.isFinite(c.timestamp?.[0]) ? c.timestamp[0] : 0,
14520
+ endSec: Number.isFinite(c.timestamp?.[1]) ? c.timestamp[1] : 0,
14521
+ text: c.text
14522
+ }));
14523
+ }
14524
+ function wordCount(text) {
14525
+ return text.trim() ? text.trim().split(/\s+/).length : 0;
14526
+ }
14435
14527
  function formatYoutubeTranscribe(raw, input) {
14436
14528
  const parsed = parseData(raw);
14437
14529
  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
14438
14530
  const d = parsed.data;
14439
14531
  const text = d.text ?? "";
14440
14532
  const chunks = d.chunks ?? [];
14533
+ const videoId = d.videoId ?? input.videoId ?? null;
14441
14534
  const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
14535
+ const words = wordCount(text);
14442
14536
  const chunkRows = chunks.slice(0, 50).map((c) => {
14443
14537
  const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
14444
14538
  const mm = String(Math.floor(sec / 60)).padStart(2, "0");
@@ -14446,8 +14540,8 @@ function formatYoutubeTranscribe(raw, input) {
14446
14540
  return `| ${mm}:${ss} | ${cell(truncate(c.text, 120))} |`;
14447
14541
  }).join("\n");
14448
14542
  const full = [
14449
- `# YouTube Transcript: \`${input.videoId}\``,
14450
- `**Duration:** ${durSec}s \xB7 **${text.split(" ").length} words**`,
14543
+ `# YouTube Transcript: \`${videoId ?? input.url ?? "video"}\``,
14544
+ `**Duration:** ${durSec}s \xB7 **${words} words**`,
14451
14545
  `
14452
14546
  ## Full Transcript
14453
14547
  ${text}`,
@@ -14460,7 +14554,22 @@ ${chunkRows}` : "",
14460
14554
  ---
14461
14555
  \u{1F4A1} Harvest more from this channel: use \`youtube_harvest\` with \`mode: "channel"\``
14462
14556
  ].filter(Boolean).join("\n");
14463
- return oneBlock(full);
14557
+ return {
14558
+ ...oneBlock(full),
14559
+ structuredContent: {
14560
+ videoId,
14561
+ url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : input.url ?? null,
14562
+ wordCount: words,
14563
+ chunkCount: chunks.length,
14564
+ durationMs: typeof d.durationMs === "number" ? d.durationMs : null,
14565
+ transcriptText: text,
14566
+ chunks: structuredTranscriptChunks(chunks),
14567
+ resolvedInputs: {
14568
+ videoId,
14569
+ url: input.url ?? null
14570
+ }
14571
+ }
14572
+ };
14464
14573
  }
14465
14574
  function formatFacebookPageIntel(raw, input) {
14466
14575
  const parsed = parseData(raw);
@@ -14470,12 +14579,13 @@ function formatFacebookPageIntel(raw, input) {
14470
14579
  const ads = d.ads ?? [];
14471
14580
  const s = d.summary ?? { totalAds: 0, activeCount: 0, videoCount: 0, imageCount: 0 };
14472
14581
  const adBlocks = ads.map((ad, i) => [
14473
- `### Ad ${i + 1}${ad.libraryId ? ` \xB7 \`${ad.libraryId}\`` : ""} \u2014 ${ad.status ?? "\u2014"} \xB7 ${ad.creativeType ?? "\u2014"} \xB7 ${ad.startDate ?? "\u2014"}`,
14582
+ `### Ad ${i + 1}${ad.libraryId ? ` \xB7 \`${ad.libraryId}\`` : ""} \u2014 ${ad.status ?? "\u2014"} \xB7 ${ad.creativeType ?? "\u2014"} \xB7 ${ad.startDate ?? ad.started ?? "\u2014"}`,
14474
14583
  ad.headline ? `**Headline:** ${ad.headline}` : "",
14475
14584
  ad.primaryText ? `**Copy:** ${truncate(ad.primaryText, 200)}` : "",
14476
14585
  ad.cta ? `**CTA:** ${ad.cta}` : "",
14477
- ad.videoUrl ? `**Video URL:** \`${ad.videoUrl}\`` : "",
14478
- ad.variations ? `**Variations:** ${ad.variations}` : ""
14586
+ ad.landingUrl ? `**Landing URL:** ${ad.landingUrl}` : "",
14587
+ ad.videoUrl ?? ad.videoSrc ? `**Video URL:** \`${ad.videoUrl ?? ad.videoSrc}\`` : "",
14588
+ ad.variations ?? ad.clusterCount ? `**Variations:** ${ad.variations ?? ad.clusterCount}` : ""
14479
14589
  ].filter(Boolean).join("\n")).join("\n\n---\n\n");
14480
14590
  const full = [
14481
14591
  `# Facebook Ad Intel: ${advertiser}`,
@@ -14501,11 +14611,16 @@ ${adBlocks}`,
14501
14611
  libraryId: ad.libraryId ?? null,
14502
14612
  status: ad.status ?? null,
14503
14613
  creativeType: ad.creativeType ?? null,
14614
+ primaryText: ad.primaryText ?? null,
14504
14615
  headline: ad.headline ?? null,
14505
14616
  cta: ad.cta ?? null,
14506
- startDate: ad.startDate ?? null,
14507
- videoUrl: ad.videoUrl ?? null,
14508
- variations: typeof ad.variations === "number" ? ad.variations : null
14617
+ startDate: ad.startDate ?? ad.started ?? null,
14618
+ landingUrl: ad.landingUrl ?? null,
14619
+ domain: ad.domain ?? null,
14620
+ videoUrl: ad.videoUrl ?? ad.videoSrc ?? null,
14621
+ imageUrl: ad.imageUrl ?? ad.imageSrc ?? null,
14622
+ videoPoster: ad.videoPoster ?? null,
14623
+ variations: typeof ad.variations === "number" ? ad.variations : typeof ad.clusterCount === "number" ? ad.clusterCount : null
14509
14624
  }))
14510
14625
  }
14511
14626
  };
@@ -14539,8 +14654,11 @@ ${rows}`,
14539
14654
  advertiserCount: advertisers.length,
14540
14655
  advertisers: advertisers.map((a) => ({
14541
14656
  name: a.pageName ?? a.name ?? null,
14657
+ pageId: a.pageId ?? null,
14658
+ pageUrl: a.pageUrl ?? null,
14542
14659
  adCount: typeof a.adCount === "number" ? a.adCount : null,
14543
- libraryId: a.sampleLibraryId ?? a.libraryId ?? null
14660
+ libraryId: a.sampleLibraryId ?? a.libraryId ?? null,
14661
+ sampleLibraryId: a.sampleLibraryId ?? null
14544
14662
  }))
14545
14663
  }
14546
14664
  };
@@ -14638,6 +14756,34 @@ function formatWorkflowSuggest(input) {
14638
14756
  }
14639
14757
  };
14640
14758
  }
14759
+ function workflowStepLines(data) {
14760
+ const step = data.step;
14761
+ const nextStep = data.nextStep;
14762
+ const done = data.done === true;
14763
+ const lines = [];
14764
+ if (step) {
14765
+ const index = Number(step.index ?? 0);
14766
+ const totalSteps = step.totalSteps != null ? Number(step.totalSteps) : void 0;
14767
+ const stepLabel = totalSteps ? `${index + 1}/${totalSteps}` : `${index + 1}`;
14768
+ lines.push(`
14769
+ ## Step ${stepLabel}: ${step.title ?? step.id ?? ""}`);
14770
+ const output = step.output;
14771
+ if (output && Object.keys(output).length) {
14772
+ lines.push(Object.entries(output).map(([k, v]) => `- ${k}: ${typeof v === "object" ? JSON.stringify(v) : String(v)}`).join("\n"));
14773
+ }
14774
+ const warnings = Array.isArray(step.warnings) ? step.warnings : [];
14775
+ if (warnings.length) lines.push(`
14776
+ **Warnings:**
14777
+ ${warnings.map((w) => `- ${w}`).join("\n")}`);
14778
+ }
14779
+ if (done) {
14780
+ lines.push("\n**Done.** All steps complete.");
14781
+ } else if (nextStep && typeof nextStep === "object") {
14782
+ lines.push(`
14783
+ **Next step:** \`${nextStep.id ?? nextStep.index}\` \u2014 call \`workflow_step\` with this run id to continue.`);
14784
+ }
14785
+ return lines;
14786
+ }
14641
14787
  function formatWorkflowRun(raw, input) {
14642
14788
  const parsed = parseData(raw);
14643
14789
  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
@@ -14651,6 +14797,7 @@ function formatWorkflowRun(raw, input) {
14651
14797
  `**Run ID:** \`${runId || "unknown"}\``,
14652
14798
  `**Status:** ${status}`,
14653
14799
  summary?.title ? `**Title:** ${summary.title}` : "",
14800
+ ...workflowStepLines(parsed.data),
14654
14801
  summary?.summary ? `
14655
14802
  ## Summary
14656
14803
  ${summary.summary}` : "",
@@ -14666,6 +14813,42 @@ ${workflowArtifactRows(artifacts)}` : "",
14666
14813
  input: input.input ?? {},
14667
14814
  run,
14668
14815
  summary,
14816
+ step: parsed.data.step,
14817
+ nextStep: parsed.data.nextStep ?? null,
14818
+ done: parsed.data.done === true,
14819
+ artifacts
14820
+ }
14821
+ };
14822
+ }
14823
+ function formatWorkflowStep(raw, input) {
14824
+ const parsed = parseData(raw);
14825
+ if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
14826
+ const run = parsed.data.run;
14827
+ const summary = parsed.data.summary;
14828
+ const artifacts = workflowArtifactsFrom(run);
14829
+ const done = parsed.data.done === true;
14830
+ const full = [
14831
+ `# Workflow Step`,
14832
+ `**Run ID:** \`${input.runId}\``,
14833
+ `**Status:** ${run?.status ?? (done ? "done" : "running")}`,
14834
+ ...workflowStepLines(parsed.data),
14835
+ done && summary?.summary ? `
14836
+ ## Summary
14837
+ ${summary.summary}` : "",
14838
+ artifacts.length ? `
14839
+ ## Artifacts
14840
+ ${workflowArtifactRows(artifacts)}` : "",
14841
+ done && artifacts.length ? "\nUse `workflow_artifact_read` with the run id and artifact id to pull CSV, JSON, Markdown, or report content into context." : ""
14842
+ ].filter(Boolean).join("\n");
14843
+ return {
14844
+ ...oneBlock(full),
14845
+ structuredContent: {
14846
+ runId: input.runId,
14847
+ run,
14848
+ summary: summary ?? null,
14849
+ step: parsed.data.step,
14850
+ nextStep: parsed.data.nextStep ?? null,
14851
+ done,
14669
14852
  artifacts
14670
14853
  }
14671
14854
  };
@@ -15070,6 +15253,7 @@ function formatFacebookAdTranscribe(raw, input) {
15070
15253
  const text = d.text ?? "";
15071
15254
  const chunks = d.chunks ?? [];
15072
15255
  const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
15256
+ const words = wordCount(text);
15073
15257
  const chunkRows = chunks.slice(0, 50).map((c) => {
15074
15258
  const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
15075
15259
  const mm = String(Math.floor(sec / 60)).padStart(2, "0");
@@ -15078,7 +15262,7 @@ function formatFacebookAdTranscribe(raw, input) {
15078
15262
  }).join("\n");
15079
15263
  const full = [
15080
15264
  `# Facebook Ad Transcript`,
15081
- `**Duration:** ${durSec}s \xB7 **${text.split(" ").length} words**`,
15265
+ `**Duration:** ${durSec}s \xB7 **${words} words**`,
15082
15266
  `
15083
15267
  ## Full Transcript
15084
15268
  ${text}`,
@@ -15091,7 +15275,20 @@ ${chunkRows}` : "",
15091
15275
  ---
15092
15276
  \u{1F4A1} Get more ads from this advertiser: use \`facebook_page_intel\`. For public Facebook reel/post URLs, use \`facebook_video_transcribe\`.`
15093
15277
  ].filter(Boolean).join("\n");
15094
- return oneBlock(full);
15278
+ return {
15279
+ ...oneBlock(full),
15280
+ structuredContent: {
15281
+ videoUrl: input.videoUrl,
15282
+ wordCount: words,
15283
+ chunkCount: chunks.length,
15284
+ durationMs: typeof d.durationMs === "number" ? d.durationMs : null,
15285
+ transcriptText: text,
15286
+ chunks: structuredTranscriptChunks(chunks),
15287
+ resolvedInputs: {
15288
+ videoUrl: input.videoUrl
15289
+ }
15290
+ }
15291
+ };
15095
15292
  }
15096
15293
  function formatFacebookVideoTranscribe(raw, input) {
15097
15294
  const parsed = parseData(raw);
@@ -15099,7 +15296,7 @@ function formatFacebookVideoTranscribe(raw, input) {
15099
15296
  const d = parsed.data;
15100
15297
  const text = d.text ?? "";
15101
15298
  const chunks = d.chunks ?? [];
15102
- const wordCount = text.trim() ? text.trim().split(/\s+/).length : 0;
15299
+ const wordCount2 = text.trim() ? text.trim().split(/\s+/).length : 0;
15103
15300
  const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
15104
15301
  const videoDuration = typeof d.videoDurationSec === "number" ? `${Math.round(d.videoDurationSec)}s` : "\u2014";
15105
15302
  const label = d.videoId ? `Facebook Organic Video \`${d.videoId}\`` : "Facebook Organic Video";
@@ -15112,7 +15309,7 @@ function formatFacebookVideoTranscribe(raw, input) {
15112
15309
  const full = [
15113
15310
  `# ${label} Transcript`,
15114
15311
  d.ownerName ? `**Owner:** ${d.ownerName}` : "",
15115
- `**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${wordCount} words**`,
15312
+ `**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${wordCount2} words**`,
15116
15313
  d.pageUrl ? `**Page URL:** ${d.pageUrl}` : `**Page URL:** ${input.url}`,
15117
15314
  d.videoUrl ? `**Extracted MP4:** \`${d.videoUrl}\`` : "",
15118
15315
  `
@@ -15138,7 +15335,7 @@ ${chunkRows}` : "",
15138
15335
  bitrate: typeof d.bitrate === "number" ? d.bitrate : null,
15139
15336
  videoDurationSec: typeof d.videoDurationSec === "number" ? d.videoDurationSec : null,
15140
15337
  videoUrl: d.videoUrl ?? "",
15141
- wordCount,
15338
+ wordCount: wordCount2,
15142
15339
  chunkCount: chunks.length,
15143
15340
  transcriptText: text,
15144
15341
  chunks: chunks.map((c) => ({
@@ -15149,6 +15346,96 @@ ${chunkRows}` : "",
15149
15346
  }
15150
15347
  };
15151
15348
  }
15349
+ function formatCaptureSerpSnapshot(raw, input) {
15350
+ const parsed = parseData(raw);
15351
+ if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
15352
+ const d = parsed.data;
15353
+ const providerPayload = d;
15354
+ const organic = Array.isArray(d.organicResults) ? d.organicResults : Array.isArray(d.organic) ? d.organic : [];
15355
+ const localPack = Array.isArray(d.localPack) ? d.localPack : [];
15356
+ const artifacts = Array.isArray(d.artifacts) ? d.artifacts : [];
15357
+ const status = String(d.status ?? d.captureStatus ?? "captured");
15358
+ const query = typeof d.query === "string" ? d.query : typeof input.query === "string" ? input.query : null;
15359
+ const location2 = typeof d.location === "string" ? d.location : typeof input.location === "string" ? input.location : null;
15360
+ const capturedAt = typeof d.capturedAt === "string" ? d.capturedAt : typeof d.extractedAt === "string" ? d.extractedAt : null;
15361
+ const resultCount = organic.length + localPack.length;
15362
+ const full = [
15363
+ `# SERP Intelligence Snapshot: ${query ?? "query"}`,
15364
+ `**Status:** ${status}`,
15365
+ location2 ? `**Location:** ${location2}` : "",
15366
+ `**Result count:** ${resultCount}`,
15367
+ artifacts.length ? `**Artifacts:** ${artifacts.length}` : "",
15368
+ "",
15369
+ "Use `capture_serp_page_snapshots` when you need page-level evidence for ranking URLs."
15370
+ ].filter(Boolean).join("\n");
15371
+ return {
15372
+ ...oneBlock(full),
15373
+ structuredContent: {
15374
+ schemaVersion: "serp-intelligence.capture.v1",
15375
+ status,
15376
+ query,
15377
+ location: location2,
15378
+ capturedAt,
15379
+ resultCount,
15380
+ snapshotId: typeof d.snapshotId === "string" ? d.snapshotId : typeof d.snapshot_id === "string" ? d.snapshot_id : typeof d.id === "string" ? d.id : null,
15381
+ resolvedInputs: {
15382
+ query: input.query ?? null,
15383
+ location: input.location ?? null,
15384
+ gl: input.gl ?? null,
15385
+ hl: input.hl ?? null,
15386
+ device: input.device ?? null,
15387
+ proxyMode: input.proxyMode ?? null,
15388
+ proxyZip: input.proxyZip ?? null,
15389
+ pages: input.pages ?? null
15390
+ },
15391
+ artifacts,
15392
+ diagnostics: d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null,
15393
+ providerPayload
15394
+ }
15395
+ };
15396
+ }
15397
+ function formatCaptureSerpPageSnapshots(raw, input) {
15398
+ const parsed = parseData(raw);
15399
+ if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
15400
+ const d = parsed.data;
15401
+ const captures = Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
15402
+ const failedCount = captures.filter((c) => c.status === "failed" || c.error).length;
15403
+ const status = String(d.status ?? (failedCount ? "partial" : "captured"));
15404
+ const rows = captures.slice(0, 25).map((capture, i) => {
15405
+ const url = typeof capture.url === "string" ? capture.url : "";
15406
+ const sourceKind = typeof capture.sourceKind === "string" ? capture.sourceKind : typeof capture.source_kind === "string" ? capture.source_kind : "configured_target";
15407
+ return `| ${i + 1} | ${cell(url)} | ${cell(sourceKind)} | ${cell(String(capture.status ?? (capture.error ? "failed" : "ok")))} |`;
15408
+ }).join("\n");
15409
+ const full = [
15410
+ "# SERP Intelligence Page Snapshots",
15411
+ `**Status:** ${status}`,
15412
+ `**Captured:** ${captures.length}`,
15413
+ `**Failed:** ${failedCount}`,
15414
+ captures.length ? `
15415
+ | # | URL | Source | Status |
15416
+ |---|-----|--------|--------|
15417
+ ${rows}` : ""
15418
+ ].filter(Boolean).join("\n");
15419
+ return {
15420
+ ...oneBlock(full),
15421
+ structuredContent: {
15422
+ schemaVersion: "serp-intelligence.page-snapshots.v1",
15423
+ status,
15424
+ count: captures.length,
15425
+ failedCount,
15426
+ captures,
15427
+ resolvedInputs: {
15428
+ urls: input.urls ?? null,
15429
+ targets: input.targets ?? null,
15430
+ maxConcurrency: input.maxConcurrency ?? null,
15431
+ timeoutMs: input.timeoutMs ?? null,
15432
+ debug: input.debug ?? null
15433
+ },
15434
+ diagnostics: d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null,
15435
+ providerPayload: d
15436
+ }
15437
+ };
15438
+ }
15152
15439
  var import_node_fs3, import_node_os3, import_node_path5, reportSavingEnabled;
15153
15440
  var init_mcp_response_formatter = __esm({
15154
15441
  "src/mcp/mcp-response-formatter.ts"() {
@@ -15623,6 +15910,7 @@ var init_directory_workflow = __esm({
15623
15910
  init_mcp_response_formatter();
15624
15911
  init_browser_service_env();
15625
15912
  init_maps_search_rotation();
15913
+ init_schemas3();
15626
15914
  init_csv();
15627
15915
  init_location_db();
15628
15916
  DirectoryWorkflowOptionsSchema = import_zod18.z.object({
@@ -15638,7 +15926,7 @@ var init_directory_workflow = __esm({
15638
15926
  saveCsv: import_zod18.z.boolean().default(true),
15639
15927
  gl: import_zod18.z.string().length(2).default("us"),
15640
15928
  hl: import_zod18.z.string().length(2).default("en"),
15641
- proxyMode: import_zod18.z.enum(["location", "configured", "none"]).default("location"),
15929
+ proxyMode: import_zod18.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
15642
15930
  proxyZip: import_zod18.z.string().regex(/^\d{5}$/).optional(),
15643
15931
  debug: import_zod18.z.boolean().default(false),
15644
15932
  headless: import_zod18.z.boolean().default(true),
@@ -16062,102 +16350,141 @@ var init_agent_packet = __esm({
16062
16350
  title: "Agent-Ready SEO Packet",
16063
16351
  description: "Create an evidence folder for AI agents from live SERP/PAA/AI search surfaces.",
16064
16352
  inputSchema: AgentPacketInputSchema,
16065
- async run(input, ctx) {
16066
- await ctx.artifacts.writeManifest("running", {}, [], []);
16067
- const warnings = [];
16068
- const errors = [];
16069
- let serp = null;
16070
- let paa = null;
16071
- if (input.includeSerp) {
16072
- try {
16073
- serp = await ctx.client.post("/harvest/sync", {
16074
- query: input.keyword,
16075
- location: input.location,
16076
- serpOnly: true,
16077
- maxQuestions: 1,
16078
- format: "json"
16079
- }, 18e4);
16080
- } catch (err) {
16081
- warnings.push(`SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
16353
+ createState: () => ({ serp: null, paa: null, warnings: [] }),
16354
+ steps: [
16355
+ {
16356
+ id: "harvest-serp",
16357
+ title: "Harvest organic SERP + AI Overview",
16358
+ async run({ input, state, ctx }) {
16359
+ if (!input.includeSerp) {
16360
+ return { state, output: { skipped: true, reason: "includeSerp is false" } };
16361
+ }
16362
+ try {
16363
+ const serp = await ctx.client.post("/harvest/sync", {
16364
+ query: input.keyword,
16365
+ location: input.location,
16366
+ serpOnly: true,
16367
+ maxQuestions: 1,
16368
+ format: "json"
16369
+ }, 18e4);
16370
+ await ctx.artifacts.writeJson("SERP raw JSON", "raw/serp.json", serp);
16371
+ return {
16372
+ state: { ...state, serp },
16373
+ output: {
16374
+ organicResults: serp.organicResults?.length ?? 0,
16375
+ localPack: serp.localPack?.length ?? 0,
16376
+ aiOverviewDetected: Boolean(serp.aiOverview?.detected),
16377
+ aiOverviewCitations: serp.aiOverview?.citations?.length ?? 0
16378
+ }
16379
+ };
16380
+ } catch (err) {
16381
+ const message = `SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
16382
+ return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
16383
+ }
16082
16384
  }
16083
- }
16084
- if (input.includePaa) {
16085
- try {
16086
- paa = await ctx.client.post("/harvest/sync", {
16087
- query: input.keyword,
16088
- location: input.location,
16089
- maxQuestions: input.maxQuestions,
16090
- format: "json"
16091
- }, 28e4);
16092
- } catch (err) {
16093
- warnings.push(`PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
16385
+ },
16386
+ {
16387
+ id: "harvest-paa",
16388
+ title: "Harvest People Also Ask",
16389
+ async run({ input, state, ctx }) {
16390
+ if (!input.includePaa) {
16391
+ return { state, output: { skipped: true, reason: "includePaa is false" } };
16392
+ }
16393
+ try {
16394
+ const paa = await ctx.client.post("/harvest/sync", {
16395
+ query: input.keyword,
16396
+ location: input.location,
16397
+ maxQuestions: input.maxQuestions,
16398
+ format: "json"
16399
+ }, 28e4);
16400
+ await ctx.artifacts.writeJson("PAA raw JSON", "raw/paa.json", paa);
16401
+ return {
16402
+ state: { ...state, paa },
16403
+ output: { paaQuestions: paa.flat?.length ?? 0 }
16404
+ };
16405
+ } catch (err) {
16406
+ const message = `PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
16407
+ return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
16408
+ }
16409
+ }
16410
+ },
16411
+ {
16412
+ id: "assemble",
16413
+ title: "Assemble evidence packet",
16414
+ async run({ input, state, ctx }) {
16415
+ const { serp, paa } = state;
16416
+ const warnings = state.warnings;
16417
+ if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
16418
+ const rows = sourceRows(input, serp, paa);
16419
+ const target = normalizeDomain(input.domain);
16420
+ const competitors = competitorRows(rows, target);
16421
+ const evidence = {
16422
+ input,
16423
+ serp: serp ?? { status: "skipped" },
16424
+ paa: paa ?? { status: "skipped" },
16425
+ target: {
16426
+ domain: target,
16427
+ organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
16428
+ citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
16429
+ citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
16430
+ },
16431
+ competitors,
16432
+ warnings
16433
+ };
16434
+ await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
16435
+ await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
16436
+ await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
16437
+ const brief = [
16438
+ `# SEO Evidence Brief: ${input.keyword}`,
16439
+ "",
16440
+ `Location: ${input.location ?? "not specified"}`,
16441
+ `Target domain: ${target ?? "not specified"}`,
16442
+ "",
16443
+ "## Evidence Summary",
16444
+ `- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
16445
+ `- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
16446
+ `- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
16447
+ `- Competitor domains: ${competitors.length}`,
16448
+ "",
16449
+ "## Recommended Use",
16450
+ "Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
16451
+ ].join("\n");
16452
+ await ctx.artifacts.writeText("Brief", "brief.md", brief);
16453
+ await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
16454
+ "# Agent Tasks",
16455
+ "",
16456
+ "- [ ] Read `evidence.json` before writing recommendations.",
16457
+ "- [ ] Compare the target domain against `competitors.csv`.",
16458
+ "- [ ] Use `sources.csv` for citations and source-grounded page sections.",
16459
+ "- [ ] Mark unsupported recommendations as assumptions."
16460
+ ].join("\n"));
16461
+ await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
16462
+ "# Agent Instructions",
16463
+ "",
16464
+ "You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
16465
+ ].join("\n"));
16466
+ const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
16467
+ const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
16468
+ title: "Agent-Ready SEO Packet",
16469
+ subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
16470
+ summary,
16471
+ warnings,
16472
+ tables: [
16473
+ { title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
16474
+ { title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
16475
+ ]
16476
+ }));
16477
+ const status = warnings.length ? "partial" : "succeeded";
16478
+ const counts = { sources: rows.length, competitors: competitors.length };
16479
+ await ctx.artifacts.writeManifest(status, counts, warnings, []);
16480
+ return {
16481
+ state,
16482
+ output: { sources: rows.length, competitors: competitors.length, status },
16483
+ summary: { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors: [], reportPath }
16484
+ };
16094
16485
  }
16095
16486
  }
16096
- if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
16097
- const rows = sourceRows(input, serp, paa);
16098
- const target = normalizeDomain(input.domain);
16099
- const competitors = competitorRows(rows, target);
16100
- const evidence = {
16101
- input,
16102
- serp: serp ?? { status: "skipped" },
16103
- paa: paa ?? { status: "skipped" },
16104
- target: {
16105
- domain: target,
16106
- organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
16107
- citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
16108
- citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
16109
- },
16110
- competitors,
16111
- warnings
16112
- };
16113
- await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
16114
- await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
16115
- await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
16116
- const brief = [
16117
- `# SEO Evidence Brief: ${input.keyword}`,
16118
- "",
16119
- `Location: ${input.location ?? "not specified"}`,
16120
- `Target domain: ${target ?? "not specified"}`,
16121
- "",
16122
- "## Evidence Summary",
16123
- `- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
16124
- `- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
16125
- `- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
16126
- `- Competitor domains: ${competitors.length}`,
16127
- "",
16128
- "## Recommended Use",
16129
- "Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
16130
- ].join("\n");
16131
- await ctx.artifacts.writeText("Brief", "brief.md", brief);
16132
- await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
16133
- "# Agent Tasks",
16134
- "",
16135
- "- [ ] Read `evidence.json` before writing recommendations.",
16136
- "- [ ] Compare the target domain against `competitors.csv`.",
16137
- "- [ ] Use `sources.csv` for citations and source-grounded page sections.",
16138
- "- [ ] Mark unsupported recommendations as assumptions."
16139
- ].join("\n"));
16140
- await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
16141
- "# Agent Instructions",
16142
- "",
16143
- "You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
16144
- ].join("\n"));
16145
- const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
16146
- const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
16147
- title: "Agent-Ready SEO Packet",
16148
- subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
16149
- summary,
16150
- warnings,
16151
- tables: [
16152
- { title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
16153
- { title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
16154
- ]
16155
- }));
16156
- const status = warnings.length ? "partial" : "succeeded";
16157
- const counts = { sources: rows.length, competitors: competitors.length };
16158
- await ctx.artifacts.writeManifest(status, counts, warnings, errors);
16159
- return { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors, reportPath };
16160
- }
16487
+ ]
16161
16488
  };
16162
16489
  }
16163
16490
  });
@@ -16220,6 +16547,7 @@ var init_directory = __esm({
16220
16547
  "use strict";
16221
16548
  import_zod20 = require("zod");
16222
16549
  init_report_renderer();
16550
+ init_schemas3();
16223
16551
  DirectoryWorkflowCliInputSchema = import_zod20.z.object({
16224
16552
  query: import_zod20.z.string().min(1),
16225
16553
  state: import_zod20.z.string().min(2).default("TN"),
@@ -16227,7 +16555,7 @@ var init_directory = __esm({
16227
16555
  maxCities: import_zod20.z.number().int().min(1).max(100).default(25),
16228
16556
  maxResultsPerCity: import_zod20.z.number().int().min(1).max(50).default(20),
16229
16557
  concurrency: import_zod20.z.number().int().min(1).max(5).default(5),
16230
- proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location"),
16558
+ proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
16231
16559
  saveCsv: import_zod20.z.boolean().default(true)
16232
16560
  });
16233
16561
  DIRECTORY_CSV_HEADERS = [
@@ -16344,6 +16672,7 @@ var init_local_competitive_audit = __esm({
16344
16672
  import_zod21 = require("zod");
16345
16673
  init_report_renderer();
16346
16674
  init_directory();
16675
+ init_schemas3();
16347
16676
  LocalCompetitiveAuditInputSchema = import_zod21.z.object({
16348
16677
  query: import_zod21.z.string().min(1),
16349
16678
  state: import_zod21.z.string().min(2).default("TN"),
@@ -16353,7 +16682,7 @@ var init_local_competitive_audit = __esm({
16353
16682
  hydrateTop: import_zod21.z.number().int().min(0).max(10).default(5),
16354
16683
  maxReviews: import_zod21.z.number().int().min(0).max(500).default(50),
16355
16684
  concurrency: import_zod21.z.number().int().min(1).max(5).default(5),
16356
- proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default("location"),
16685
+ proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
16357
16686
  returnPartial: import_zod21.z.boolean().default(true)
16358
16687
  });
16359
16688
  localCompetitiveAuditWorkflowDefinition = {
@@ -16805,6 +17134,7 @@ var init_comparison_briefs = __esm({
16805
17134
  init_report_renderer();
16806
17135
  init_directory();
16807
17136
  init_seo_workflow_utils();
17137
+ init_schemas3();
16808
17138
  ProxyModeSchema = import_zod22.z.enum(["location", "configured", "none"]);
16809
17139
  MapComparisonInputSchema = import_zod22.z.object({
16810
17140
  query: import_zod22.z.string().min(1),
@@ -16816,7 +17146,7 @@ var init_comparison_briefs = __esm({
16816
17146
  hydrateTop: import_zod22.z.number().int().min(0).max(10).default(5),
16817
17147
  maxReviews: import_zod22.z.number().int().min(0).max(500).default(25),
16818
17148
  concurrency: import_zod22.z.number().int().min(1).max(5).default(5),
16819
- proxyMode: ProxyModeSchema.default("location"),
17149
+ proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
16820
17150
  returnPartial: import_zod22.z.boolean().default(true)
16821
17151
  }).refine((input) => input.location || input.state, {
16822
17152
  message: "Either location or state is required for map-comparison"
@@ -17230,22 +17560,44 @@ function workflowDefinition(id) {
17230
17560
  if (!definition) throw new Error(`Unknown workflow "${id}". Available: ${DEFINITIONS.map((def) => def.id).join(", ")}`);
17231
17561
  return definition;
17232
17562
  }
17233
- async function runWorkflow(id, rawInput, options = {}) {
17563
+ function workflowStepCount(id) {
17234
17564
  const definition = workflowDefinition(id);
17235
- const input = definition.inputSchema.parse(rawInput);
17565
+ return definition.steps?.length ?? 1;
17566
+ }
17567
+ function workflowSupportsSteps(id) {
17568
+ const definition = workflowDefinition(id);
17569
+ return Boolean(definition.steps && definition.steps.length > 0);
17570
+ }
17571
+ function resolveApiKey(options) {
17236
17572
  const apiKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim();
17237
17573
  if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
17238
- const apiUrl = options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
17574
+ return apiKey;
17575
+ }
17576
+ function resolveApiUrl(options) {
17577
+ return options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
17578
+ }
17579
+ async function runWorkflow(id, rawInput, options = {}) {
17580
+ const definition = workflowDefinition(id);
17581
+ const input = definition.inputSchema.parse(rawInput);
17582
+ const apiKey = resolveApiKey(options);
17583
+ const apiUrl = resolveApiUrl(options);
17239
17584
  const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
17240
17585
  const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
17586
+ const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client: client2, artifacts, signal: options.signal };
17241
17587
  try {
17242
- return await definition.run(input, {
17243
- runId: artifacts.runId,
17244
- startedAt: artifacts.startedAt,
17245
- client: client2,
17246
- artifacts,
17247
- signal: options.signal
17248
- });
17588
+ if (definition.steps && definition.steps.length > 0) {
17589
+ let state = definition.createState ? definition.createState(input) : {};
17590
+ let summary = null;
17591
+ for (const step of definition.steps) {
17592
+ const outcome = await step.run({ input, state, ctx });
17593
+ state = outcome.state;
17594
+ if (outcome.summary) summary = outcome.summary;
17595
+ }
17596
+ if (!summary) throw new Error(`Workflow "${id}" produced no terminal step summary`);
17597
+ return summary;
17598
+ }
17599
+ if (definition.run) return await definition.run(input, ctx);
17600
+ throw new Error(`Workflow "${id}" has neither steps nor a run() implementation`);
17249
17601
  } catch (err) {
17250
17602
  const message = err instanceof Error ? err.message : String(err);
17251
17603
  await artifacts.writeText("Failure", "summary.md", `# ${definition.title}
@@ -17256,10 +17608,50 @@ ${message}
17256
17608
  throw err;
17257
17609
  }
17258
17610
  }
17259
- var import_zod23, DEFINITIONS;
17611
+ async function runWorkflowStep(id, rawInput, opts) {
17612
+ const definition = workflowDefinition(id);
17613
+ const steps = definition.steps;
17614
+ if (!steps || steps.length === 0) throw new Error(`Workflow "${id}" does not support stepwise execution`);
17615
+ if (opts.stepIndex < 0 || opts.stepIndex >= steps.length) {
17616
+ throw new Error(`Step index ${opts.stepIndex} is out of range for "${id}" (${steps.length} steps)`);
17617
+ }
17618
+ const options = opts.options ?? {};
17619
+ const input = definition.inputSchema.parse(rawInput);
17620
+ const apiKey = resolveApiKey(options);
17621
+ const apiUrl = resolveApiUrl(options);
17622
+ const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
17623
+ const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
17624
+ const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client: client2, artifacts, signal: options.signal };
17625
+ const state = opts.state ?? (definition.createState ? definition.createState(input) : {});
17626
+ const step = steps[opts.stepIndex];
17627
+ const outcome = await step.run({ input, state, ctx });
17628
+ const writtenArtifacts = await Promise.all(
17629
+ artifacts.artifacts.map(async (artifact) => ({
17630
+ ...artifact,
17631
+ content: await (0, import_promises7.readFile)(artifact.path, "utf8").catch(() => "")
17632
+ }))
17633
+ );
17634
+ return {
17635
+ runId: artifacts.runId,
17636
+ workflowId: definition.id,
17637
+ title: definition.title,
17638
+ stepIndex: opts.stepIndex,
17639
+ stepId: step.id,
17640
+ stepTitle: step.title,
17641
+ totalSteps: steps.length,
17642
+ isLast: opts.stepIndex === steps.length - 1,
17643
+ output: outcome.output,
17644
+ state: outcome.state,
17645
+ warnings: outcome.warnings ?? [],
17646
+ summary: outcome.summary ?? null,
17647
+ artifacts: writtenArtifacts
17648
+ };
17649
+ }
17650
+ var import_promises7, import_zod23, DEFINITIONS;
17260
17651
  var init_registry2 = __esm({
17261
17652
  "src/workflows/registry.ts"() {
17262
17653
  "use strict";
17654
+ import_promises7 = require("fs/promises");
17263
17655
  import_zod23 = require("zod");
17264
17656
  init_artifact_writer();
17265
17657
  init_http_client2();
@@ -17328,7 +17720,7 @@ async function readManifestFromSummary(summary) {
17328
17720
  if (!summary.reportPath) return null;
17329
17721
  const manifestPath = summary.reportPath.replace(/report\.html$/, "manifest.json");
17330
17722
  try {
17331
- return JSON.parse(await (0, import_promises7.readFile)(manifestPath, "utf8"));
17723
+ return JSON.parse(await (0, import_promises8.readFile)(manifestPath, "utf8"));
17332
17724
  } catch {
17333
17725
  return null;
17334
17726
  }
@@ -17409,6 +17801,88 @@ async function executeWorkflowRun(input) {
17409
17801
  throw err;
17410
17802
  }
17411
17803
  }
17804
+ async function runWorkflowStepAndPersist(input) {
17805
+ const runId = input.run.id;
17806
+ const workflowId = input.run.workflow_id;
17807
+ await markWorkflowRunRunning(runId);
17808
+ let result;
17809
+ try {
17810
+ result = await runWorkflowStep(workflowId, input.workflowInput, {
17811
+ stepIndex: input.stepIndex,
17812
+ state: input.priorState ?? void 0,
17813
+ options: {
17814
+ apiKey: input.user.api_key,
17815
+ apiUrl: input.apiUrl,
17816
+ outputDir: hostedWorkflowOutputDir(),
17817
+ runId,
17818
+ headers: input.concurrencyLockId ? { "x-mcp-scraper-concurrency-lock": input.concurrencyLockId } : void 0
17819
+ }
17820
+ });
17821
+ } catch (err) {
17822
+ const message = err instanceof Error ? err.message : String(err);
17823
+ await failWorkflowRunRecord(runId, message, { error: message });
17824
+ const failedRun = await getWorkflowRun(runId, input.user.id);
17825
+ if (failedRun) await deliverWorkflowWebhook({ userId: input.user.id, run: failedRun, scheduleId: failedRun.schedule_id, webhookUrl: input.webhookUrl ?? null, summary: null, artifacts: [] });
17826
+ throw err;
17827
+ }
17828
+ if (result.artifacts.length) {
17829
+ await appendWorkflowArtifacts(runId, input.user.id, result.artifacts.map((a) => ({
17830
+ kind: a.kind,
17831
+ label: a.label,
17832
+ path: a.path,
17833
+ bytes: a.bytes,
17834
+ rows: a.rows,
17835
+ content: a.content
17836
+ })));
17837
+ }
17838
+ const logEntry = {
17839
+ stepIndex: result.stepIndex,
17840
+ stepId: result.stepId,
17841
+ stepTitle: result.stepTitle,
17842
+ output: result.output,
17843
+ warnings: result.warnings,
17844
+ at: (/* @__PURE__ */ new Date()).toISOString()
17845
+ };
17846
+ const status = result.isLast ? result.summary?.status ?? "succeeded" : "running";
17847
+ await advanceWorkflowRunStep(runId, {
17848
+ stepIndex: result.stepIndex + 1,
17849
+ totalSteps: result.totalSteps,
17850
+ state: result.state,
17851
+ status,
17852
+ logEntry,
17853
+ appendLog: input.priorLog
17854
+ });
17855
+ if (result.isLast && result.summary) {
17856
+ const manifest = await readManifestFromSummary(result.summary);
17857
+ await completeWorkflowRunRecord(runId, result.summary.status, manifest ?? { summary: result.summary });
17858
+ }
17859
+ const run = await getWorkflowRun(runId, input.user.id);
17860
+ const artifacts = await listWorkflowArtifacts(runId, input.user.id);
17861
+ if (result.isLast && run) {
17862
+ await deliverWorkflowWebhook({ userId: input.user.id, run, scheduleId: run.schedule_id, webhookUrl: input.webhookUrl ?? null, summary: result.summary, artifacts });
17863
+ }
17864
+ const definition = workflowDefinition(workflowId);
17865
+ const nextIndex = result.stepIndex + 1;
17866
+ const nextStep = result.isLast ? null : {
17867
+ index: nextIndex,
17868
+ id: definition.steps?.[nextIndex]?.id ?? null,
17869
+ title: definition.steps?.[nextIndex]?.title ?? null
17870
+ };
17871
+ return {
17872
+ run: run ? exposeRun(input.c, run, artifacts) : null,
17873
+ summary: result.isLast ? result.summary : null,
17874
+ step: {
17875
+ index: result.stepIndex,
17876
+ id: result.stepId,
17877
+ title: result.stepTitle,
17878
+ totalSteps: result.totalSteps,
17879
+ output: result.output,
17880
+ warnings: result.warnings
17881
+ },
17882
+ nextStep,
17883
+ done: result.isLast
17884
+ };
17885
+ }
17412
17886
  async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
17413
17887
  const now = (/* @__PURE__ */ new Date()).toISOString();
17414
17888
  const due = await listDueWorkflowSchedules(now, limit);
@@ -17461,12 +17935,12 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
17461
17935
  }
17462
17936
  return { dispatched: results.length, results };
17463
17937
  }
17464
- var import_node_crypto3, import_promises7, import_hono7, import_zod24, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema;
17938
+ var import_node_crypto3, import_promises8, import_hono7, import_zod24, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
17465
17939
  var init_workflow_routes = __esm({
17466
17940
  "src/api/workflow-routes.ts"() {
17467
17941
  "use strict";
17468
17942
  import_node_crypto3 = require("crypto");
17469
- import_promises7 = require("fs/promises");
17943
+ import_promises8 = require("fs/promises");
17470
17944
  import_hono7 = require("hono");
17471
17945
  import_zod24 = require("zod");
17472
17946
  init_artifact_writer();
@@ -17525,7 +17999,29 @@ var init_workflow_routes = __esm({
17525
17999
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
17526
18000
  let run = null;
17527
18001
  try {
17528
- run = await createWorkflowRun({ userId: user.id, workflowId: parsed.data.workflowId, workflowInput: input });
18002
+ const stepwise = workflowSupportsSteps(parsed.data.workflowId);
18003
+ run = await createWorkflowRun({
18004
+ userId: user.id,
18005
+ workflowId: parsed.data.workflowId,
18006
+ workflowInput: input,
18007
+ webhookUrl,
18008
+ totalSteps: stepwise ? workflowStepCount(parsed.data.workflowId) : 1
18009
+ });
18010
+ if (stepwise) {
18011
+ const response = await runWorkflowStepAndPersist({
18012
+ c,
18013
+ user,
18014
+ run,
18015
+ workflowInput: input,
18016
+ stepIndex: 0,
18017
+ priorState: null,
18018
+ priorLog: [],
18019
+ apiUrl: originFromUrl(c.req.url),
18020
+ webhookUrl,
18021
+ concurrencyLockId: gate.lockId
18022
+ });
18023
+ return c.json(response);
18024
+ }
17529
18025
  const executed = await executeWorkflowRun({
17530
18026
  runId: run.id,
17531
18027
  user,
@@ -17535,7 +18031,7 @@ var init_workflow_routes = __esm({
17535
18031
  webhookUrl,
17536
18032
  concurrencyLockId: gate.lockId
17537
18033
  });
17538
- return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
18034
+ return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary, done: true });
17539
18035
  } catch (err) {
17540
18036
  const failed = run ? await getWorkflowRun(run.id, user.id) : null;
17541
18037
  return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
@@ -17543,6 +18039,50 @@ var init_workflow_routes = __esm({
17543
18039
  await releaseConcurrencyGate(gate.lockId);
17544
18040
  }
17545
18041
  });
18042
+ TERMINAL_RUN_STATUSES = ["succeeded", "partial", "failed", "cancelled"];
18043
+ workflowApp.post("/runs/:id/step", createApiKeyAuth(), async (c) => {
18044
+ const user = c.get("user");
18045
+ const run = await getWorkflowRun(c.req.param("id"), user.id);
18046
+ if (!run) return c.json({ error: "Run not found" }, 404);
18047
+ if (!workflowSupportsSteps(run.workflow_id)) {
18048
+ return c.json({ error: `Workflow "${run.workflow_id}" does not support stepwise execution` }, 400);
18049
+ }
18050
+ const artifactsNow = await listWorkflowArtifacts(run.id, user.id);
18051
+ if (TERMINAL_RUN_STATUSES.includes(run.status)) {
18052
+ return c.json({ run: exposeRun(c, run, artifactsNow), done: true, error: `Run is already ${run.status}` }, 409);
18053
+ }
18054
+ const stepState = await getWorkflowRunStepState(run.id, user.id);
18055
+ const stepIndex = stepState?.stepIndex ?? run.step_index;
18056
+ const totalSteps = workflowStepCount(run.workflow_id);
18057
+ if (stepIndex >= totalSteps) {
18058
+ return c.json({ run: exposeRun(c, run, artifactsNow), done: true, error: "No remaining steps" }, 409);
18059
+ }
18060
+ const gate = await acquireConcurrencyGate(user, "workflow_run", {
18061
+ ttlSeconds: 60 * 60,
18062
+ metadata: { workflowId: run.workflow_id, step: stepIndex }
18063
+ });
18064
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
18065
+ try {
18066
+ const response = await runWorkflowStepAndPersist({
18067
+ c,
18068
+ user,
18069
+ run,
18070
+ workflowInput: run.input,
18071
+ stepIndex,
18072
+ priorState: stepState?.state ?? null,
18073
+ priorLog: stepState?.stepLog ?? run.step_log,
18074
+ apiUrl: originFromUrl(c.req.url),
18075
+ webhookUrl: run.webhook_url,
18076
+ concurrencyLockId: gate.lockId
18077
+ });
18078
+ return c.json(response);
18079
+ } catch (err) {
18080
+ const failed = await getWorkflowRun(run.id, user.id);
18081
+ return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
18082
+ } finally {
18083
+ await releaseConcurrencyGate(gate.lockId);
18084
+ }
18085
+ });
17546
18086
  workflowApp.get("/runs", createApiKeyAuth(), async (c) => {
17547
18087
  const user = c.get("user");
17548
18088
  const runs = await listWorkflowRuns(user.id);
@@ -17561,14 +18101,17 @@ var init_workflow_routes = __esm({
17561
18101
  if (!run) return c.json({ error: "Run not found" }, 404);
17562
18102
  const artifact = await getWorkflowArtifact(c.req.param("artifactId"), user.id);
17563
18103
  if (!artifact || artifact.run_id !== run.id) return c.json({ error: "Artifact not found" }, 404);
18104
+ const filename = artifact.path.split("/").pop() || artifact.id;
18105
+ const headers = {
18106
+ "Content-Type": artifact.content_type,
18107
+ "Content-Disposition": `attachment; filename="${filename}"`
18108
+ };
18109
+ if (artifact.content != null) {
18110
+ return new Response(artifact.content, { headers });
18111
+ }
17564
18112
  try {
17565
- const content = await (0, import_promises7.readFile)(artifact.path);
17566
- return new Response(content, {
17567
- headers: {
17568
- "Content-Type": artifact.content_type,
17569
- "Content-Disposition": `attachment; filename="${artifact.path.split("/").pop() || artifact.id}"`
17570
- }
17571
- });
18113
+ const content = await (0, import_promises8.readFile)(artifact.path);
18114
+ return new Response(content, { headers });
17572
18115
  } catch {
17573
18116
  return c.json({ error: "Artifact file is no longer available" }, 410);
17574
18117
  }
@@ -17856,7 +18399,7 @@ async function capturePageSnapshot(target, options = {}) {
17856
18399
  const textContent = truncateContent(text, maxContentChars);
17857
18400
  const schemaTypes = getSchemaTypes(extraction.schema, extraction.kpo.schemaTypes);
17858
18401
  const metrics = countLinks(extraction.bodyHtml, extraction.url);
17859
- const wordCount = countWords(extraction.bodyMarkdown);
18402
+ const wordCount2 = countWords(extraction.bodyMarkdown);
17860
18403
  const contentSha256 = sha256(extraction.bodyMarkdown);
17861
18404
  const canonicalUrl = extraction.kpo.canonicalUrl ? new URL(extraction.kpo.canonicalUrl, extraction.url).href : null;
17862
18405
  const contentIncluded = Boolean(options.includeContent);
@@ -17910,10 +18453,10 @@ async function capturePageSnapshot(target, options = {}) {
17910
18453
  },
17911
18454
  error: null,
17912
18455
  schemaTypes,
17913
- wordCount,
18456
+ wordCount: wordCount2,
17914
18457
  linkCounts: {
17915
18458
  ...metrics,
17916
- wordCount
18459
+ wordCount: wordCount2
17917
18460
  },
17918
18461
  hashes: {
17919
18462
  htmlSha256: sha256(extraction.bodyHtml),
@@ -19489,7 +20032,7 @@ async function harvest(rawOptions) {
19489
20032
  const signal = getAbortSignal(rawOptions);
19490
20033
  const onAttemptEvent = getAttemptLogSink(rawOptions);
19491
20034
  const requestedProxyMode = raw.proxyMode;
19492
- const proxyMode = requestedProxyMode === "none" ? "none" : requestedProxyMode === "configured" ? "configured" : "location";
20035
+ const proxyMode = requestedProxyMode === "location" || requestedProxyMode === "none" || requestedProxyMode === "configured" ? requestedProxyMode : DEFAULT_PROXY_MODE;
19493
20036
  const kernelApiKey = typeof raw.kernelApiKey === "string" ? raw.kernelApiKey.trim() : browserServiceApiKey();
19494
20037
  const configuredKernelProxyId = typeof raw.kernelProxyId === "string" ? raw.kernelProxyId.trim() : browserServiceProxyId();
19495
20038
  const proxyOpts = {
@@ -19863,6 +20406,7 @@ var init_schemas4 = __esm({
19863
20406
  "src/serp-intelligence/schemas.ts"() {
19864
20407
  "use strict";
19865
20408
  import_zod25 = require("zod");
20409
+ init_schemas3();
19866
20410
  SerpIntelligenceDeviceValues = ["desktop", "mobile"];
19867
20411
  SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
19868
20412
  SerpIntelligenceAttemptOutcomeValues = [
@@ -19903,7 +20447,7 @@ var init_schemas4 = __esm({
19903
20447
  gl: import_zod25.z.string().trim().length(2).default("us"),
19904
20448
  hl: import_zod25.z.string().trim().length(2).default("en"),
19905
20449
  device: import_zod25.z.enum(SerpIntelligenceDeviceValues).default("desktop"),
19906
- proxyMode: import_zod25.z.enum(SerpIntelligenceProxyModeValues).default("location"),
20450
+ proxyMode: import_zod25.z.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
19907
20451
  proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional(),
19908
20452
  pages: import_zod25.z.number().int().min(1).max(2).default(1),
19909
20453
  debug: import_zod25.z.boolean().default(false),
@@ -20433,16 +20977,17 @@ var PACKAGE_VERSION;
20433
20977
  var init_version = __esm({
20434
20978
  "src/version.ts"() {
20435
20979
  "use strict";
20436
- PACKAGE_VERSION = "0.2.19";
20980
+ PACKAGE_VERSION = "0.2.20";
20437
20981
  }
20438
20982
  });
20439
20983
 
20440
20984
  // src/mcp/mcp-tool-schemas.ts
20441
- var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
20985
+ var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
20442
20986
  var init_mcp_tool_schemas = __esm({
20443
20987
  "src/mcp/mcp-tool-schemas.ts"() {
20444
20988
  "use strict";
20445
20989
  import_zod26 = require("zod");
20990
+ init_schemas3();
20446
20991
  HarvestPaaInputSchema = {
20447
20992
  query: import_zod26.z.string().min(1).describe('Core search topic only. If the user says "best hvac company in Denver CO", use query="best hvac company" and location="Denver, CO". Do not include the location in query when it can be separated.'),
20448
20993
  location: import_zod26.z.string().optional().describe('City, region, or country for geo-targeted results, inferred from the user request when present, e.g. "Denver, CO", "Tokyo, Japan", "London, UK".'),
@@ -20450,8 +20995,8 @@ var init_mcp_tool_schemas = __esm({
20450
20995
  gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
20451
20996
  hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
20452
20997
  device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
20453
- proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
20454
- proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
20998
+ proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
20999
+ proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
20455
21000
  debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
20456
21001
  };
20457
21002
  ExtractUrlInputSchema = {
@@ -20475,22 +21020,23 @@ var init_mcp_tool_schemas = __esm({
20475
21020
  mode: import_zod26.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
20476
21021
  query: import_zod26.z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
20477
21022
  channelHandle: import_zod26.z.string().optional().describe("YouTube channel handle, channel ID, or URL. Examples: @mkbhd, UC..., https://youtube.com/@mkbhd."),
20478
- maxVideos: import_zod26.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50. Increase when user asks for full channel/history.")
21023
+ maxVideos: import_zod26.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50, maximum 500. Use 10-25 for quick topic discovery, 50 for normal channel/search harvests, and larger values only when the user asks for full channel/history because large responses consume more context.")
20479
21024
  };
20480
21025
  YoutubeTranscribeInputSchema = {
20481
- videoId: import_zod26.z.string().min(1).describe("YouTube video ID, e.g. dQw4w9WgXcQ")
21026
+ videoId: import_zod26.z.string().min(1).optional().describe("YouTube video ID, e.g. dQw4w9WgXcQ. Use only an ID returned by youtube_harvest or visible in a YouTube URL; do not invent one."),
21027
+ url: import_zod26.z.string().url().optional().describe("Full YouTube URL, e.g. https://www.youtube.com/watch?v=dQw4w9WgXcQ or https://youtu.be/dQw4w9WgXcQ. Use this when the user pasted a URL instead of an ID. Provide videoId or url.")
20482
21028
  };
20483
21029
  FacebookPageIntelInputSchema = {
20484
- pageId: import_zod26.z.string().optional(),
20485
- libraryId: import_zod26.z.string().optional(),
21030
+ pageId: import_zod26.z.string().optional().describe("Facebook advertiser/page ID. Use only a pageId returned by facebook_ad_search/facebook_page_intel or copied from Facebook Ad Library; do not construct one yourself."),
21031
+ libraryId: import_zod26.z.string().optional().describe("Facebook Ad Library archive ID for a known ad or advertiser sample. Use a libraryId returned by facebook_ad_search, or a libraryId/adArchiveId visible in Ad Library."),
20486
21032
  query: import_zod26.z.string().optional().describe("Advertiser or brand name when pageId/libraryId is not known. One of pageId, libraryId, or query is required."),
20487
- maxAds: import_zod26.z.number().int().min(1).max(200).default(50),
20488
- country: import_zod26.z.string().length(2).default("US")
21033
+ maxAds: import_zod26.z.number().int().min(1).max(200).default(50).describe("Maximum ads to inspect. Default 50, maximum 200. Prefer 25-50 for focused advertiser scans; use 100-200 only when the user asks for a broad ad archive sweep."),
21034
+ country: import_zod26.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU. Infer from the user request when they name a country.")
20489
21035
  };
20490
21036
  FacebookAdSearchInputSchema = {
20491
21037
  query: import_zod26.z.string().min(1).describe("Advertiser, brand, competitor, niche, or keyword to search in Facebook Ad Library."),
20492
- country: import_zod26.z.string().length(2).default("US"),
20493
- maxResults: import_zod26.z.number().int().min(1).max(20).default(10)
21038
+ country: import_zod26.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU."),
21039
+ maxResults: import_zod26.z.number().int().min(1).max(20).default(10).describe("Maximum advertisers to return. Default 10, maximum 20. Prefer tighter search terms over maxing this out.")
20494
21040
  };
20495
21041
  FacebookAdTranscribeInputSchema = {
20496
21042
  videoUrl: import_zod26.z.string().url().describe("Direct Facebook CDN video URL from a facebook_page_intel ad result. Do not pass a public Facebook reel/post/share URL here; use facebook_video_transcribe for organic Facebook URLs.")
@@ -20504,8 +21050,8 @@ var init_mcp_tool_schemas = __esm({
20504
21050
  location: import_zod26.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
20505
21051
  gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location."),
20506
21052
  hl: import_zod26.z.string().length(2).default("en").describe("Language inferred from user request."),
20507
- includeReviews: import_zod26.z.boolean().default(false).describe("Whether to fetch individual review cards"),
20508
- maxReviews: import_zod26.z.number().int().min(1).max(500).default(50).describe("Max review cards to return (requires includeReviews: true)")
21053
+ includeReviews: import_zod26.z.boolean().default(false).describe("Whether to fetch individual review cards. Use true when the user asks for reviews, customer pain, complaints, praise, themes, or review evidence."),
21054
+ maxReviews: import_zod26.z.number().int().min(1).max(500).default(50).describe("Max review cards to return when includeReviews is true. Default 50, maximum 500. Use 50 for normal review analysis and larger values only for deep review mining.")
20509
21055
  };
20510
21056
  MapsSearchInputSchema = {
20511
21057
  query: import_zod26.z.string().min(1).describe('Business category, niche, keyword, or search term. If the user says "roofers in Denver CO", use query="roofers" and location="Denver, CO". Do not put the location here when it can be separated.'),
@@ -20513,7 +21059,7 @@ var init_mcp_tool_schemas = __esm({
20513
21059
  gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location."),
20514
21060
  hl: import_zod26.z.string().length(2).default("en").describe("Language inferred from user request."),
20515
21061
  maxResults: import_zod26.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
20516
- proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state Maps searches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts. Use configured for the server proxy ID, and none only for local direct-network debugging."),
21062
+ proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode. Maps defaults to location so local market searches get city/state residential proxy targeting and rotation. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
20517
21063
  proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
20518
21064
  debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
20519
21065
  };
@@ -20528,8 +21074,8 @@ var init_mcp_tool_schemas = __esm({
20528
21074
  includeZipGroups: import_zod26.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
20529
21075
  usZipsCsvPath: import_zod26.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
20530
21076
  saveCsv: import_zod26.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
20531
- proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode for every city Maps search. Use location by default for US city/state batches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts per city. Use configured for the server proxy ID, and none only for local direct-network debugging."),
20532
- proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its Lead Magician ZIP group or city/state location."),
21077
+ proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode for every city Maps search. Maps workflows default to location so each city can use city/state or ZIP-group residential proxy targeting. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
21078
+ proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its ZIP group or city/state location."),
20533
21079
  debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
20534
21080
  };
20535
21081
  RankTrackerModeSchema = import_zod26.z.enum(["maps", "organic", "ai_overview", "paa"]);
@@ -20843,8 +21389,11 @@ var init_mcp_tool_schemas = __esm({
20843
21389
  advertiserCount: import_zod26.z.number().int().min(0),
20844
21390
  advertisers: import_zod26.z.array(import_zod26.z.object({
20845
21391
  name: NullableString,
21392
+ pageId: NullableString,
21393
+ pageUrl: NullableString,
20846
21394
  adCount: import_zod26.z.number().int().nullable(),
20847
- libraryId: NullableString
21395
+ libraryId: NullableString,
21396
+ sampleLibraryId: NullableString
20848
21397
  }))
20849
21398
  };
20850
21399
  FacebookPageIntelOutputSchema = {
@@ -20857,10 +21406,15 @@ var init_mcp_tool_schemas = __esm({
20857
21406
  libraryId: NullableString,
20858
21407
  status: NullableString,
20859
21408
  creativeType: NullableString,
21409
+ primaryText: NullableString,
20860
21410
  headline: NullableString,
20861
21411
  cta: NullableString,
20862
21412
  startDate: NullableString,
21413
+ landingUrl: NullableString,
21414
+ domain: NullableString,
20863
21415
  videoUrl: NullableString,
21416
+ imageUrl: NullableString,
21417
+ videoPoster: NullableString,
20864
21418
  variations: import_zod26.z.number().int().nullable()
20865
21419
  }))
20866
21420
  };
@@ -20882,6 +21436,58 @@ var init_mcp_tool_schemas = __esm({
20882
21436
  text: import_zod26.z.string()
20883
21437
  }))
20884
21438
  };
21439
+ TranscriptChunkOutput = import_zod26.z.object({
21440
+ startSec: import_zod26.z.number(),
21441
+ endSec: import_zod26.z.number(),
21442
+ text: import_zod26.z.string()
21443
+ });
21444
+ YoutubeTranscribeOutputSchema = {
21445
+ videoId: NullableString,
21446
+ url: NullableString,
21447
+ wordCount: import_zod26.z.number().int().min(0),
21448
+ chunkCount: import_zod26.z.number().int().min(0),
21449
+ durationMs: import_zod26.z.number().nullable(),
21450
+ transcriptText: import_zod26.z.string(),
21451
+ chunks: import_zod26.z.array(TranscriptChunkOutput),
21452
+ resolvedInputs: import_zod26.z.object({
21453
+ videoId: NullableString,
21454
+ url: NullableString
21455
+ })
21456
+ };
21457
+ FacebookAdTranscribeOutputSchema = {
21458
+ videoUrl: import_zod26.z.string().url(),
21459
+ wordCount: import_zod26.z.number().int().min(0),
21460
+ chunkCount: import_zod26.z.number().int().min(0),
21461
+ durationMs: import_zod26.z.number().nullable(),
21462
+ transcriptText: import_zod26.z.string(),
21463
+ chunks: import_zod26.z.array(TranscriptChunkOutput),
21464
+ resolvedInputs: import_zod26.z.object({
21465
+ videoUrl: import_zod26.z.string().url()
21466
+ })
21467
+ };
21468
+ CaptureSerpSnapshotOutputSchema = {
21469
+ schemaVersion: import_zod26.z.literal("serp-intelligence.capture.v1"),
21470
+ status: import_zod26.z.string(),
21471
+ query: NullableString,
21472
+ location: NullableString,
21473
+ capturedAt: NullableString,
21474
+ resultCount: import_zod26.z.number().int().min(0).nullable(),
21475
+ snapshotId: NullableString,
21476
+ resolvedInputs: import_zod26.z.record(import_zod26.z.unknown()),
21477
+ artifacts: import_zod26.z.array(import_zod26.z.record(import_zod26.z.unknown())),
21478
+ diagnostics: import_zod26.z.record(import_zod26.z.unknown()).nullable(),
21479
+ providerPayload: import_zod26.z.record(import_zod26.z.unknown())
21480
+ };
21481
+ CaptureSerpPageSnapshotsOutputSchema = {
21482
+ schemaVersion: import_zod26.z.literal("serp-intelligence.page-snapshots.v1"),
21483
+ status: import_zod26.z.string(),
21484
+ count: import_zod26.z.number().int().min(0),
21485
+ failedCount: import_zod26.z.number().int().min(0),
21486
+ captures: import_zod26.z.array(import_zod26.z.record(import_zod26.z.unknown())),
21487
+ resolvedInputs: import_zod26.z.record(import_zod26.z.unknown()),
21488
+ diagnostics: import_zod26.z.record(import_zod26.z.unknown()).nullable(),
21489
+ providerPayload: import_zod26.z.record(import_zod26.z.unknown())
21490
+ };
20885
21491
  CreditsInfoInputSchema = {
20886
21492
  item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
20887
21493
  includeLedger: import_zod26.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
@@ -20909,16 +21515,19 @@ var init_mcp_tool_schemas = __esm({
20909
21515
  maxSuggestions: import_zod26.z.number().int().min(1).max(8).default(3).describe("Number of matching workflow recipes to return.")
20910
21516
  };
20911
21517
  WorkflowRunInputSchema = {
20912
- workflowId: WorkflowIdSchema2.describe("Workflow to run. Use workflow_list or workflow_suggest first when unsure."),
21518
+ workflowId: WorkflowIdSchema2.describe("Workflow to run: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. Use only these values; call workflow_list or workflow_suggest first when unsure."),
20913
21519
  input: import_zod26.z.record(import_zod26.z.unknown()).default({}).describe("Workflow-specific input object. Examples: agent-packet uses {keyword, domain?, location?, maxQuestions?}; local-competitive-audit uses {query, state, minPopulation?, maxCities?, maxResultsPerCity?, hydrateTop?, maxReviews?}; serp-comparison uses {keyword, domain?, url?, location?, extractTop?}."),
20914
21520
  webhookUrl: import_zod26.z.string().url().optional().describe("Optional HTTPS webhook to receive the completed hosted workflow run event.")
20915
21521
  };
21522
+ WorkflowStepInputSchema = {
21523
+ runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself. Advances the run by exactly one step (one logical leg, e.g. one live harvest).")
21524
+ };
20916
21525
  WorkflowStatusInputSchema = {
20917
- runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run.")
21526
+ runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself.")
20918
21527
  };
20919
21528
  WorkflowArtifactReadInputSchema = {
20920
- runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run or workflow_status."),
20921
- artifactId: import_zod26.z.string().min(1).describe("Artifact id from the run artifact list."),
21529
+ runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself."),
21530
+ artifactId: import_zod26.z.string().min(1).describe("Artifact id from the run artifact list returned by workflow_run, workflow_step, or workflow_status. Use only a returned artifactId; do not construct one yourself."),
20922
21531
  maxBytes: import_zod26.z.number().int().min(1e3).max(1e6).default(2e5).describe("Maximum bytes of artifact text to return inline. Use lower values for large CSV/JSON artifacts; call again with the downloadUrl if needed outside MCP.")
20923
21532
  };
20924
21533
  WorkflowRecipeOutput = import_zod26.z.object({
@@ -20951,6 +21560,18 @@ var init_mcp_tool_schemas = __esm({
20951
21560
  input: import_zod26.z.record(import_zod26.z.unknown()),
20952
21561
  run: import_zod26.z.record(import_zod26.z.unknown()).optional(),
20953
21562
  summary: import_zod26.z.record(import_zod26.z.unknown()).optional(),
21563
+ step: import_zod26.z.record(import_zod26.z.unknown()).optional(),
21564
+ nextStep: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
21565
+ done: import_zod26.z.boolean().optional(),
21566
+ artifacts: import_zod26.z.array(WorkflowArtifactOutput)
21567
+ };
21568
+ WorkflowStepOutputSchema = {
21569
+ runId: import_zod26.z.string(),
21570
+ run: import_zod26.z.record(import_zod26.z.unknown()).optional(),
21571
+ summary: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
21572
+ step: import_zod26.z.record(import_zod26.z.unknown()).optional(),
21573
+ nextStep: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
21574
+ done: import_zod26.z.boolean(),
20954
21575
  artifacts: import_zod26.z.array(WorkflowArtifactOutput)
20955
21576
  };
20956
21577
  WorkflowStatusOutputSchema = {
@@ -20971,8 +21592,8 @@ var init_mcp_tool_schemas = __esm({
20971
21592
  gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
20972
21593
  hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from user request."),
20973
21594
  device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
20974
- proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
20975
- proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
21595
+ proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general SERP success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
21596
+ proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
20976
21597
  debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
20977
21598
  pages: import_zod26.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
20978
21599
  };
@@ -20982,8 +21603,8 @@ var init_mcp_tool_schemas = __esm({
20982
21603
  gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
20983
21604
  hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from the user request."),
20984
21605
  device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
20985
- proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy behavior for capture. Use location for localized US residential evidence; it creates a fresh proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static residential proxy, and none only for direct-network debugging."),
20986
- proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting when a precise city-center or ZIP proxy is needed. With proxyMode location this ZIP is used for each fresh proxy attempt."),
21606
+ proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy behavior for capture. Default configured uses the service proxy without city/ZIP targeting for the highest general capture success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
21607
+ proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when a precise city-center or ZIP proxy is needed."),
20987
21608
  pages: import_zod26.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
20988
21609
  debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
20989
21610
  includePageSnapshots: import_zod26.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs through the same product capture path."),
@@ -21402,70 +22023,72 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
21402
22023
  if (savesReports) registerSavedReportResources(server);
21403
22024
  server.registerTool("harvest_paa", {
21404
22025
  title: "Google PAA + SERP Harvest",
21405
- description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). For US local SERPs, leave proxyMode as location so the service uses fresh residential proxy IDs across retries, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear or the market evidence is wrong. Use maxQuestions 30 normally, 100-200 for "full", "deep", "all", or comprehensive research. Deep harvests above 100 questions can run for several minutes with no interim progress \u2014 warn the user before starting one and keep maxQuestions at or below 100 unless they explicitly want a deep harvest. Credits are charged by extracted question; unused request hold is refunded.'),
22026
+ description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use maxQuestions 30 normally, 100-200 for "full", "deep", "all", or comprehensive research. Deep harvests above 100 questions can run for several minutes with no interim progress \u2014 warn the user before starting one and keep maxQuestions at or below 100 unless they explicitly want a deep harvest. Credits are charged by extracted question; unused request hold is refunded.'),
21406
22027
  inputSchema: HarvestPaaInputSchema,
21407
22028
  outputSchema: HarvestPaaOutputSchema,
21408
22029
  annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
21409
22030
  }, async (input) => formatHarvestPaa(await executor.harvestPaa(input), input));
21410
22031
  server.registerTool("search_serp", {
21411
22032
  title: "Google SERP Lookup",
21412
- description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. For US city/state rankings, keep proxyMode as location and pass proxyZip when a city-center ZIP is known; location mode uses fresh residential proxy IDs, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear, the proxy tunnel fails, or location evidence is wrong."),
22033
+ description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting. Use proxyMode location and proxyZip only when the user explicitly needs city/ZIP-targeted residential proxy evidence."),
21413
22034
  inputSchema: SearchSerpInputSchema,
21414
22035
  outputSchema: SearchSerpOutputSchema,
21415
22036
  annotations: liveWebToolAnnotations("Google SERP Lookup")
21416
22037
  }, async (input) => formatSearchSerp(await executor.searchSerp(input), input));
21417
22038
  server.registerTool("extract_url", {
21418
22039
  title: "Single URL Extract",
21419
- description: withReportNote("Extract structured data from one public URL: page content as Markdown, heading structure, JSON-LD schema, entity details, NAP score, metadata, and missing schema fields. Use when the user provides a single URL or asks to inspect/scrape one page."),
22040
+ description: withReportNote("Extract structured data from one public URL when the user provides one page, asks to inspect/scrape a page, or needs page content, schema, headings, metadata, screenshots, branding, or media assets. Returns structured page fields plus artifact handles for saved reports/screenshots/media when requested. Use map_site_urls before extracting a site inventory; use extract_site for multi-page crawling."),
21420
22041
  inputSchema: ExtractUrlInputSchema,
21421
22042
  outputSchema: ExtractUrlOutputSchema,
21422
22043
  annotations: liveWebToolAnnotations("Single URL Extract")
21423
22044
  }, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
21424
22045
  server.registerTool("map_site_urls", {
21425
22046
  title: "Site URL Map",
21426
- description: withReportNote("Map/crawl a public website to build a URL inventory with HTTP status codes, broken links, redirects, and site scope. Use before extract_site for audits or when the user asks for a sitemap/URL inventory."),
22047
+ description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
21427
22048
  inputSchema: MapSiteUrlsInputSchema,
21428
22049
  outputSchema: MapSiteUrlsOutputSchema,
21429
22050
  annotations: liveWebToolAnnotations("Site URL Map")
21430
22051
  }, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
21431
22052
  server.registerTool("extract_site", {
21432
22053
  title: "Multi-Page Site Extract",
21433
- description: withReportNote("Run multi-page extraction across a public website. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use for website audits, competitor audits, and full-site extraction."),
22054
+ description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
21434
22055
  inputSchema: ExtractSiteInputSchema,
21435
22056
  outputSchema: ExtractSiteOutputSchema,
21436
22057
  annotations: liveWebToolAnnotations("Multi-Page Site Extract")
21437
22058
  }, async (input) => formatExtractSite(await executor.extractSite(input), input));
21438
22059
  server.registerTool("youtube_harvest", {
21439
22060
  title: "YouTube Video Harvest",
21440
- description: withReportNote('Harvest YouTube video metadata by search query or channel handle/ID/URL. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, dates, durations, URLs, thumbnails, and videoIds for follow-up transcription.'),
22061
+ description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
21441
22062
  inputSchema: YoutubeHarvestInputSchema,
21442
22063
  outputSchema: YoutubeHarvestOutputSchema,
21443
22064
  annotations: liveWebToolAnnotations("YouTube Video Harvest")
21444
22065
  }, async (input) => formatYoutubeHarvest(await executor.youtubeHarvest(input), input));
21445
22066
  server.registerTool("youtube_transcribe", {
21446
22067
  title: "YouTube Transcription",
21447
- description: withReportNote("Fetch and transcribe captions from a YouTube video. Returns full transcript, timestamped chunks, and word count. Pass a videoId from youtube_harvest results or infer it from a YouTube URL if the user provided one."),
22068
+ description: withReportNote("Fetch and transcribe captions from a YouTube video. Use this when the user asks what was said in a YouTube video, wants claims/offers/lessons extracted from a video, or provides a YouTube URL for transcript work. Returns full transcript, timestamped chunks, word count, and resolvedInputs. Pass videoId from youtube_harvest results, or pass url when the user pasted a YouTube URL. Use youtube_harvest first when you need to discover videos by topic/channel."),
21448
22069
  inputSchema: YoutubeTranscribeInputSchema,
22070
+ outputSchema: YoutubeTranscribeOutputSchema,
21449
22071
  annotations: liveWebToolAnnotations("YouTube Transcription")
21450
22072
  }, async (input) => formatYoutubeTranscribe(await executor.youtubeTranscribe(input), input));
21451
22073
  server.registerTool("facebook_page_intel", {
21452
22074
  title: "Facebook Advertiser Ad Intel",
21453
- description: withReportNote("Harvest ads from a Facebook advertiser. Returns ad copy, headlines, CTAs, creative type, status, landing URLs, and direct ad video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name as query. Use after facebook_ad_search when possible. For normal public Facebook reels/posts/watch/share URLs, use facebook_video_transcribe instead."),
22075
+ description: withReportNote("Harvest ads from a Facebook advertiser when the user wants current ad copy, creative angles, CTAs, video URLs, or competitive ad intelligence for one brand/page. Returns ad copy, headlines, CTAs, creative type, status, landing URLs, and direct ad video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name as query. Use facebook_ad_search first when the advertiser handle is unknown. For normal public Facebook reels/posts/watch/share URLs, use facebook_video_transcribe instead."),
21454
22076
  inputSchema: FacebookPageIntelInputSchema,
21455
22077
  outputSchema: FacebookPageIntelOutputSchema,
21456
22078
  annotations: liveWebToolAnnotations("Facebook Advertiser Ad Intel")
21457
22079
  }, async (input) => formatFacebookPageIntel(await executor.facebookPageIntel(input), input));
21458
22080
  server.registerTool("facebook_ad_search", {
21459
22081
  title: "Facebook Ad Library Search",
21460
- description: withReportNote("Search Facebook Ad Library by brand, advertiser, competitor, niche, or keyword. Returns advertisers with ad counts and library IDs. Use to discover competitors, then pass libraryId to facebook_page_intel."),
22082
+ description: withReportNote("Search Facebook Ad Library when the user wants to find advertisers by brand, competitor, niche, or keyword. Returns advertisers with ad counts and library IDs. Use this to discover competitor/page handles, then pass libraryId or pageId to facebook_page_intel for ad details."),
21461
22083
  inputSchema: FacebookAdSearchInputSchema,
21462
22084
  outputSchema: FacebookAdSearchOutputSchema,
21463
22085
  annotations: liveWebToolAnnotations("Facebook Ad Library Search")
21464
22086
  }, async (input) => formatFacebookAdSearch(await executor.facebookAdSearch(input), input));
21465
22087
  server.registerTool("facebook_ad_transcribe", {
21466
22088
  title: "Facebook Ad Transcription",
21467
- description: "Transcribe audio from a Facebook ad video CDN URL. Returns full transcript and timestamped chunks. Use only with the direct videoUrl value from facebook_page_intel results, not public Facebook post/reel/share URLs.",
22089
+ description: "Transcribe audio from a Facebook ad video CDN URL. Use this when facebook_page_intel returned a direct videoUrl and the user asks what the ad says, what claims it makes, or wants ad-message extraction. Returns full transcript, timestamped chunks, word count, and resolvedInputs. Use only with the direct videoUrl value from facebook_page_intel results; do not pass public Facebook post/reel/share URLs. Use facebook_video_transcribe for organic Facebook URLs, and facebook_page_intel first when you only have a brand/page/ad library handle.",
21468
22090
  inputSchema: FacebookAdTranscribeInputSchema,
22091
+ outputSchema: FacebookAdTranscribeOutputSchema,
21469
22092
  annotations: liveWebToolAnnotations("Facebook Ad Transcription")
21470
22093
  }, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
21471
22094
  server.registerTool("facebook_video_transcribe", {
@@ -21484,14 +22107,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
21484
22107
  }, async (input) => formatMapsPlaceIntel(await executor.mapsPlaceIntel(input), input));
21485
22108
  server.registerTool("maps_search", {
21486
22109
  title: "Google Maps Business Search",
21487
- description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." For US city/state Maps searches, keep proxyMode as location so the browser service creates a fresh residential proxy ID and browser session for the market; retryable failures rotate to a new proxy and new browser session for up to 5 attempts. Pass proxyZip only when a specific ZIP or city-center ZIP is known. Returns up to 50 candidates with names, place URLs, CIDs when available, ratings, review counts, profile metadata, and sanitized attempt telemetry. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.'),
22110
+ description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." Maps is the default location-targeted surface: omit proxyMode for normal Maps use so the service creates residential proxy evidence for the requested market and rotates on retryable failures. Pass proxyZip only when a specific ZIP or city-center ZIP is known. Use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting. Returns up to 50 candidates with names, place URLs, CIDs when available, ratings, review counts, profile metadata, and sanitized attempt telemetry. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.'),
21488
22111
  inputSchema: MapsSearchInputSchema,
21489
22112
  outputSchema: MapsSearchOutputSchema,
21490
22113
  annotations: liveWebToolAnnotations("Google Maps Business Search")
21491
22114
  }, async (input) => formatMapsSearch(await executor.mapsSearch(input), input));
21492
22115
  server.registerTool("directory_workflow", {
21493
22116
  title: "Directory Workflow: Markets + Maps",
21494
- description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Use concurrency up to 5 for parallel city sessions. Keep proxyMode as location so each city search creates fresh residential proxy IDs and browser sessions; retryable city failures rotate to a new proxy and new browser session for up to 5 attempts. Saved CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. Structured city results include sanitized attempt telemetry. Use maps_place_intel only when a selected profile needs deeper review topics, profile review count confirmation, or review cards. For local Lead Magician ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath only in local/test mode.'),
22117
+ description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Maps workflows default to location-targeted proxying so each city can use city/state or ZIP-group residential proxy evidence; use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting. Saved CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. Structured city results include sanitized attempt telemetry. Use maps_place_intel only when a selected profile needs deeper review topics, profile review count confirmation, or review cards. For local Lead Magician ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath only in local/test mode.'),
21495
22118
  inputSchema: DirectoryWorkflowInputSchema,
21496
22119
  outputSchema: DirectoryWorkflowOutputSchema,
21497
22120
  annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
@@ -21512,21 +22135,28 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
21512
22135
  }, async (input) => formatWorkflowSuggest(input));
21513
22136
  server.registerTool("workflow_run", {
21514
22137
  title: "Run Workflow",
21515
- description: withReportNote("Run a higher-level MCP Scraper workflow and return the run id, summary, and artifacts. Use after workflow_suggest or workflow_list. Runnable workflow ids: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. This is the main MCP tool for executing market analysis, ICP evidence packets, local competitive audits, Maps/SERP comparisons, content gap briefs, and AI Overview language guidance."),
22138
+ description: withReportNote("Start a higher-level MCP Scraper workflow. Use after workflow_suggest or workflow_list. Runnable workflow ids: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. This is the main MCP tool for market analysis, ICP evidence packets, local competitive audits, Maps/SERP comparisons, content gap briefs, and AI Overview language guidance. Stepwise workflows (e.g. agent-packet) run ONE leg per call and return runId, the step output, and nextStep \u2014 when nextStep is present, call workflow_step with the runId to run the next leg, and keep calling it until done is true. This keeps each call short instead of one long blocking run, so report each step result to the user as it arrives."),
21516
22139
  inputSchema: WorkflowRunInputSchema,
21517
22140
  outputSchema: WorkflowRunOutputSchema,
21518
22141
  annotations: liveWebToolAnnotations("Run Workflow")
21519
22142
  }, async (input) => formatWorkflowRun(await executor.workflowRun(input), input));
22143
+ server.registerTool("workflow_step", {
22144
+ title: "Advance Workflow Step",
22145
+ description: withReportNote("Run the next leg of a stepwise MCP Scraper workflow started with workflow_run. Pass the runId. Each call executes exactly one logical step (typically one live harvest), persists that step's artifacts, and returns the step output plus nextStep. Keep calling workflow_step with the same runId until done is true, reporting each step result to the user as it lands. Use this instead of waiting on one long workflow call \u2014 it avoids client timeouts on long multi-step jobs."),
22146
+ inputSchema: WorkflowStepInputSchema,
22147
+ outputSchema: WorkflowStepOutputSchema,
22148
+ annotations: liveWebToolAnnotations("Advance Workflow Step")
22149
+ }, async (input) => formatWorkflowStep(await executor.workflowStep(input), input));
21520
22150
  server.registerTool("workflow_status", {
21521
22151
  title: "Workflow Status",
21522
- description: "Fetch a hosted workflow run by id and list its current status and artifacts. Use after workflow_run when the model needs to re-open a run, inspect artifact ids, or recover from a long workflow.",
22152
+ description: "Fetch a hosted workflow run by id and list its current status and artifacts. Use when a workflow may still be running, when the model needs to re-open a run, inspect artifact ids, recover from a long workflow, or decide whether to call workflow_step or workflow_artifact_read next. Use only a runId returned by workflow_run/workflow_step/workflow_status; do not construct one yourself.",
21523
22153
  inputSchema: WorkflowStatusInputSchema,
21524
22154
  outputSchema: WorkflowStatusOutputSchema,
21525
22155
  annotations: liveWebToolAnnotations("Workflow Status")
21526
22156
  }, async (input) => formatWorkflowStatus(await executor.workflowStatus(input), input));
21527
22157
  server.registerTool("workflow_artifact_read", {
21528
22158
  title: "Read Workflow Artifact",
21529
- description: "Read a workflow artifact back into MCP context by run id and artifact id. Use this before writing final deliverables so the answer is grounded in generated evidence.json, CSVs, Markdown briefs, reports, and task files instead of memory. Use maxBytes to limit large CSV/JSON artifacts.",
22159
+ description: "Read a workflow artifact back into MCP context by run id and artifact id. Use this before writing final deliverables so the answer is grounded in generated evidence.json, CSVs, Markdown briefs, reports, and task files instead of memory. Use workflow_status first when artifact ids are unknown. Use only artifactId values returned by workflow_run/workflow_step/workflow_status; do not construct one yourself. Use maxBytes to limit large CSV/JSON artifacts.",
21530
22160
  inputSchema: WorkflowArtifactReadInputSchema,
21531
22161
  outputSchema: WorkflowArtifactReadOutputSchema,
21532
22162
  annotations: liveWebToolAnnotations("Read Workflow Artifact")
@@ -21568,6 +22198,27 @@ var init_paa_mcp_server = __esm({
21568
22198
  });
21569
22199
 
21570
22200
  // src/mcp/http-mcp-tool-executor.ts
22201
+ function youtubeVideoIdFromUrl(url) {
22202
+ if (!url) return null;
22203
+ try {
22204
+ const parsed = new URL(url);
22205
+ const host = parsed.hostname.replace(/^www\./, "").replace(/^m\./, "");
22206
+ if (host === "youtu.be") {
22207
+ const id = parsed.pathname.split("/").filter(Boolean)[0];
22208
+ return id || null;
22209
+ }
22210
+ if (host === "youtube.com" || host === "music.youtube.com") {
22211
+ const watchId = parsed.searchParams.get("v");
22212
+ if (watchId) return watchId;
22213
+ const parts = parsed.pathname.split("/").filter(Boolean);
22214
+ const markerIndex = parts.findIndex((part) => ["shorts", "embed", "live"].includes(part));
22215
+ if (markerIndex >= 0 && parts[markerIndex + 1]) return parts[markerIndex + 1];
22216
+ }
22217
+ } catch {
22218
+ return null;
22219
+ }
22220
+ return null;
22221
+ }
21571
22222
  var HttpMcpToolExecutor;
21572
22223
  var init_http_mcp_tool_executor = __esm({
21573
22224
  "src/mcp/http-mcp-tool-executor.ts"() {
@@ -21698,7 +22349,21 @@ var init_http_mcp_tool_executor = __esm({
21698
22349
  return this.call("/youtube/harvest", input);
21699
22350
  }
21700
22351
  youtubeTranscribe(input) {
21701
- return this.call("/youtube/transcribe", input);
22352
+ const videoId = input.videoId?.trim() || youtubeVideoIdFromUrl(input.url);
22353
+ if (!videoId) {
22354
+ return Promise.resolve({
22355
+ content: [{
22356
+ type: "text",
22357
+ text: JSON.stringify({
22358
+ error_code: "youtube_video_id_required",
22359
+ error: "Pass videoId from youtube_harvest or a YouTube url that contains a video id.",
22360
+ retryable: false
22361
+ })
22362
+ }],
22363
+ isError: true
22364
+ });
22365
+ }
22366
+ return this.call("/youtube/transcribe", { videoId });
21702
22367
  }
21703
22368
  facebookPageIntel(input) {
21704
22369
  return this.call("/facebook/page-intel", input);
@@ -21735,6 +22400,10 @@ var init_http_mcp_tool_executor = __esm({
21735
22400
  webhookUrl: input.webhookUrl
21736
22401
  }, Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 9e5);
21737
22402
  }
22403
+ workflowStep(input) {
22404
+ const timeoutMs = this.httpTimeoutOverrideMs ?? Number(process.env.MCP_SCRAPER_WORKFLOW_TIMEOUT_MS ?? 9e5);
22405
+ return this.call(`/workflows/runs/${encodeURIComponent(input.runId)}/step`, {}, Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 9e5);
22406
+ }
21738
22407
  workflowStatus(input) {
21739
22408
  return this.getJson(`/workflows/runs/${encodeURIComponent(input.runId)}`);
21740
22409
  }
@@ -21788,16 +22457,18 @@ async function requireMcpCallerKey(c) {
21788
22457
  function registerSerpIntelligenceCaptureTools(server, executor) {
21789
22458
  server.registerTool("capture_serp_snapshot", {
21790
22459
  title: "SERP Intelligence Snapshot",
21791
- description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized US residential evidence; location mode creates fresh proxy IDs across retries, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear or location evidence is wrong. Use configured only for the static residential proxy, and none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
22460
+ description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl. Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
21792
22461
  inputSchema: CaptureSerpSnapshotInputSchema,
22462
+ outputSchema: CaptureSerpSnapshotOutputSchema,
21793
22463
  annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
21794
- }, async (input) => executor.captureSerpSnapshot(input));
22464
+ }, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
21795
22465
  server.registerTool("capture_serp_page_snapshots", {
21796
22466
  title: "SERP Intelligence Page Snapshots",
21797
22467
  description: "Capture public ranking-page evidence through POST /serp-intelligence/page-snapshots, the same product page snapshot path used by Phoenix. Provide urls for simple captures or targets when preserving organic, AI citation, local-pack, configured target, or site-subject source metadata. Private IPs, localhost, file URLs, and internal URLs are rejected by the service. Use timeoutMs for slow pages and debug true for sanitized proxy/browser diagnostics.",
21798
22468
  inputSchema: CaptureSerpPageSnapshotsInputSchema,
22469
+ outputSchema: CaptureSerpPageSnapshotsOutputSchema,
21799
22470
  annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
21800
- }, async (input) => executor.captureSerpPageSnapshots(input));
22471
+ }, async (input) => formatCaptureSerpPageSnapshots(await executor.captureSerpPageSnapshots(input), input));
21801
22472
  }
21802
22473
  var import_hono9, import_webStandardStreamableHttp, mcpApp;
21803
22474
  var init_mcp_routes = __esm({
@@ -23518,6 +24189,7 @@ var init_server = __esm({
23518
24189
  init_rates();
23519
24190
  init_billing_schemas();
23520
24191
  init_server_schemas();
24192
+ init_schemas3();
23521
24193
  init_credit_operations();
23522
24194
  init_harvest();
23523
24195
  init_harvest_problems();
@@ -23747,7 +24419,7 @@ var init_server = __esm({
23747
24419
  gl: body.gl ?? "us",
23748
24420
  hl: body.hl ?? "en",
23749
24421
  device: body.device ?? "desktop",
23750
- proxyMode: body.proxyMode ?? "location",
24422
+ proxyMode: body.proxyMode ?? DEFAULT_PROXY_MODE,
23751
24423
  proxyZip: body.proxyZip,
23752
24424
  debug: body.debug ?? false,
23753
24425
  serpOnly: body.serpOnly ?? false,
@@ -23783,7 +24455,7 @@ var init_server = __esm({
23783
24455
  gl: body.gl ?? "us",
23784
24456
  hl: body.hl ?? "en",
23785
24457
  device: body.device ?? "desktop",
23786
- proxyMode: body.proxyMode ?? "location",
24458
+ proxyMode: body.proxyMode ?? DEFAULT_PROXY_MODE,
23787
24459
  proxyZip: body.proxyZip,
23788
24460
  debug: body.debug ?? false,
23789
24461
  serpOnly: body.serpOnly ?? false,