mcp-scraper 0.2.19 → 0.2.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +36 -5
  2. package/dist/bin/api-server.cjs +1072 -349
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/browser-agent-stdio-server.cjs +1274 -75
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +4 -2
  8. package/dist/bin/browser-agent-stdio-server.js.map +1 -1
  9. package/dist/bin/mcp-scraper-cli.cjs +659 -210
  10. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-cli.js +108 -9
  12. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +2104 -566
  14. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  15. package/dist/bin/mcp-scraper-combined-stdio-server.js +7 -4
  16. package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
  17. package/dist/bin/mcp-scraper-install.cjs +3 -3
  18. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  19. package/dist/bin/mcp-scraper-install.js +2 -2
  20. package/dist/bin/mcp-stdio-server.cjs +726 -387
  21. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  22. package/dist/bin/mcp-stdio-server.js +3 -2
  23. package/dist/bin/mcp-stdio-server.js.map +1 -1
  24. package/dist/bin/paa-harvest.cjs +5 -3
  25. package/dist/bin/paa-harvest.cjs.map +1 -1
  26. package/dist/bin/paa-harvest.js +5 -2
  27. package/dist/bin/paa-harvest.js.map +1 -1
  28. package/dist/{chunk-TL7YTFLH.js → chunk-DBQDG7EH.js} +99 -27
  29. package/dist/chunk-DBQDG7EH.js.map +1 -0
  30. package/dist/chunk-DUEW4EOO.js +214 -0
  31. package/dist/chunk-DUEW4EOO.js.map +1 -0
  32. package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js} +216 -108
  33. package/dist/chunk-H2R232HK.js.map +1 -0
  34. package/dist/chunk-HS6OKELW.js +1786 -0
  35. package/dist/chunk-HS6OKELW.js.map +1 -0
  36. package/dist/{chunk-CQTAKXBN.js → chunk-IPW4LFOT.js} +11 -116
  37. package/dist/chunk-IPW4LFOT.js.map +1 -0
  38. package/dist/chunk-LFATOGDF.js +111 -0
  39. package/dist/chunk-LFATOGDF.js.map +1 -0
  40. package/dist/{chunk-MCBOZ2JF.js → chunk-MLNCKQ2B.js} +3 -3
  41. package/dist/chunk-MLNCKQ2B.js.map +1 -0
  42. package/dist/{chunk-4UEVYYDW.js → chunk-O5J7E4VX.js} +394 -57
  43. package/dist/chunk-O5J7E4VX.js.map +1 -0
  44. package/dist/{chunk-BSYPATSM.js → chunk-SXTXMFEQ.js} +2 -2
  45. package/dist/chunk-WN7PBKMV.js +28 -0
  46. package/dist/chunk-WN7PBKMV.js.map +1 -0
  47. package/dist/chunk-XKUDVN2E.js +7 -0
  48. package/dist/chunk-XKUDVN2E.js.map +1 -0
  49. package/dist/{db-P5X6UQ3E.js → db-BE4JVB3V.js} +8 -2
  50. package/dist/index.cjs +5 -3
  51. package/dist/index.cjs.map +1 -1
  52. package/dist/index.d.cts +2 -2
  53. package/dist/index.d.ts +2 -2
  54. package/dist/index.js +3 -1
  55. package/dist/index.js.map +1 -1
  56. package/dist/{server-CRUSZA2Q.js → server-24XIQ6W6.js} +248 -40
  57. package/dist/server-24XIQ6W6.js.map +1 -0
  58. package/dist/{worker-OZSWIS3F.js → worker-SLQ375UG.js} +8 -5
  59. package/dist/{worker-OZSWIS3F.js.map → worker-SLQ375UG.js.map} +1 -1
  60. package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +2 -2
  61. package/docs/mcp-tool-craft-lint.generated.md +51 -0
  62. package/docs/mcp-tool-manifest.generated.json +1534 -0
  63. package/docs/specs/local-competitive-audit-spec.md +1 -2
  64. package/docs/specs/mcp-tool-definition-quality-audit-spec.md +1602 -0
  65. package/package.json +4 -1
  66. package/dist/chunk-4UEVYYDW.js.map +0 -1
  67. package/dist/chunk-76P4DKWR.js +0 -793
  68. package/dist/chunk-76P4DKWR.js.map +0 -1
  69. package/dist/chunk-CQTAKXBN.js.map +0 -1
  70. package/dist/chunk-F44RBOJ5.js.map +0 -1
  71. package/dist/chunk-MCBOZ2JF.js.map +0 -1
  72. package/dist/chunk-RU2HZD55.js +0 -7
  73. package/dist/chunk-RU2HZD55.js.map +0 -1
  74. package/dist/chunk-TL7YTFLH.js.map +0 -1
  75. package/dist/server-CRUSZA2Q.js.map +0 -1
  76. /package/dist/{chunk-BSYPATSM.js.map → chunk-SXTXMFEQ.js.map} +0 -0
  77. /package/dist/{db-P5X6UQ3E.js.map → db-BE4JVB3V.js.map} +0 -0
@@ -59,6 +59,17 @@ function browserServiceProxyId() {
59
59
  const value = (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim();
60
60
  return value || void 0;
61
61
  }
62
+ function browserServiceProfileName() {
63
+ const value = (process.env.BROWSER_AGENT_PROFILE_NAME ?? process.env.BROWSER_SERVICE_PROFILE_NAME ?? process.env.KERNEL_BROWSER_PROFILE_NAME ?? process.env.KERNEL_PROFILE_NAME)?.trim();
64
+ return value || void 0;
65
+ }
66
+ function browserServiceProfileSaveChanges() {
67
+ const value = (process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ?? process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();
68
+ if (!value) return void 0;
69
+ if (["1", "true", "yes", "on"].includes(value)) return true;
70
+ if (["0", "false", "no", "off"].includes(value)) return false;
71
+ return void 0;
72
+ }
62
73
  var init_browser_service_env = __esm({
63
74
  "src/lib/browser-service-env.ts"() {
64
75
  "use strict";
@@ -2465,6 +2476,16 @@ var init_registry = __esm({
2465
2476
  }
2466
2477
  });
2467
2478
 
2479
+ // src/attribution-pixel.ts
2480
+ var ATTRIBUTION_PIXEL_SRC, ATTRIBUTION_PIXEL_SCRIPT;
2481
+ var init_attribution_pixel = __esm({
2482
+ "src/attribution-pixel.ts"() {
2483
+ "use strict";
2484
+ ATTRIBUTION_PIXEL_SRC = "https://hyros-attribution-crm.vercel.app/api/track/sdk?token=35c27ba33dd0c6cc640f9fef415ee3dc9ead67fefe99d269be602ee269900e25";
2485
+ ATTRIBUTION_PIXEL_SCRIPT = `<script src="${ATTRIBUTION_PIXEL_SRC}" async></script>`;
2486
+ }
2487
+ });
2488
+
2468
2489
  // src/blog/template.ts
2469
2490
  function faqJsonLd(post) {
2470
2491
  const firstFive = post.sections.flatMap((s) => s.cards).slice(0, 5);
@@ -2599,6 +2620,7 @@ function renderPost(post) {
2599
2620
  <script type="application/ld+json">${faqJsonLd(post)}</script>
2600
2621
  <script type="application/ld+json">${breadcrumbJsonLd(post)}</script>
2601
2622
 
2623
+ ${ATTRIBUTION_PIXEL_SCRIPT}
2602
2624
  <style>${CSS}</style>
2603
2625
  </head>
2604
2626
  <body>
@@ -2742,6 +2764,7 @@ var CSS;
2742
2764
  var init_template = __esm({
2743
2765
  "src/blog/template.ts"() {
2744
2766
  "use strict";
2767
+ init_attribution_pixel();
2745
2768
  CSS = `
2746
2769
  @font-face {
2747
2770
  font-family: "GT Flexa";
@@ -4809,7 +4832,7 @@ function parsePageData(url, html, status, via) {
4809
4832
  }
4810
4833
  const h1 = headings.find((h) => h.level === 1)?.text ?? null;
4811
4834
  const bodyText = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
4812
- const wordCount = bodyText.split(" ").filter((w) => w.length > 2).length;
4835
+ const wordCount2 = bodyText.split(" ").filter((w) => w.length > 2).length;
4813
4836
  const schemaTypes = [];
4814
4837
  const schema = [];
4815
4838
  for (const m of html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
@@ -4845,7 +4868,7 @@ function parsePageData(url, html, status, via) {
4845
4868
  } catch {
4846
4869
  }
4847
4870
  }
4848
- return { url, status, via, title, metaDescription, h1, headings, wordCount, schemaTypes, canonicalUrl, internalLinks, externalLinks, bodyMarkdown, schema };
4871
+ return { url, status, via, title, metaDescription, h1, headings, wordCount: wordCount2, schemaTypes, canonicalUrl, internalLinks, externalLinks, bodyMarkdown, schema };
4849
4872
  }
4850
4873
  async function fetchAndParse(url, kernelApiKey) {
4851
4874
  try {
@@ -6906,6 +6929,8 @@ var db_exports = {};
6906
6929
  __export(db_exports, {
6907
6930
  SiteAuditJobRowSchema: () => SiteAuditJobRowSchema,
6908
6931
  SiteAuditPhaseLogRowSchema: () => SiteAuditPhaseLogRowSchema,
6932
+ advanceWorkflowRunStep: () => advanceWorkflowRunStep,
6933
+ appendWorkflowArtifacts: () => appendWorkflowArtifacts,
6909
6934
  cancelJob: () => cancelJob,
6910
6935
  checkRateLimit: () => checkRateLimit,
6911
6936
  claimMonthlyFreeRefresh: () => claimMonthlyFreeRefresh,
@@ -6946,6 +6971,7 @@ __export(db_exports, {
6946
6971
  getWorkflowArtifact: () => getWorkflowArtifact,
6947
6972
  getWorkflowRun: () => getWorkflowRun,
6948
6973
  getWorkflowRunByIdempotencyKey: () => getWorkflowRunByIdempotencyKey,
6974
+ getWorkflowRunStepState: () => getWorkflowRunStepState,
6949
6975
  getWorkflowSchedule: () => getWorkflowSchedule,
6950
6976
  hashPassword: () => hashPassword,
6951
6977
  ledgerExistsForOperation: () => ledgerExistsForOperation,
@@ -7227,9 +7253,34 @@ async function migrate() {
7227
7253
  queued_at TEXT NOT NULL DEFAULT (datetime('now')),
7228
7254
  started_at TEXT,
7229
7255
  completed_at TEXT,
7230
- idempotency_key TEXT
7256
+ idempotency_key TEXT,
7257
+ step_index INTEGER NOT NULL DEFAULT 0,
7258
+ total_steps INTEGER,
7259
+ step_state_json TEXT,
7260
+ step_log_json TEXT,
7261
+ webhook_url TEXT
7231
7262
  )
7232
7263
  `);
7264
+ try {
7265
+ await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_index INTEGER NOT NULL DEFAULT 0`);
7266
+ } catch {
7267
+ }
7268
+ try {
7269
+ await db.execute(`ALTER TABLE workflow_runs ADD COLUMN total_steps INTEGER`);
7270
+ } catch {
7271
+ }
7272
+ try {
7273
+ await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_state_json TEXT`);
7274
+ } catch {
7275
+ }
7276
+ try {
7277
+ await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_log_json TEXT`);
7278
+ } catch {
7279
+ }
7280
+ try {
7281
+ await db.execute(`ALTER TABLE workflow_runs ADD COLUMN webhook_url TEXT`);
7282
+ } catch {
7283
+ }
7233
7284
  await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_user_created ON workflow_runs(user_id, queued_at DESC)`);
7234
7285
  await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_schedule ON workflow_runs(schedule_id, queued_at DESC)`);
7235
7286
  await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS workflow_runs_idempotency ON workflow_runs(idempotency_key) WHERE idempotency_key IS NOT NULL`);
@@ -7244,9 +7295,14 @@ async function migrate() {
7244
7295
  content_type TEXT NOT NULL,
7245
7296
  bytes INTEGER,
7246
7297
  rows_count INTEGER,
7298
+ content TEXT,
7247
7299
  created_at TEXT NOT NULL DEFAULT (datetime('now'))
7248
7300
  )
7249
7301
  `);
7302
+ try {
7303
+ await db.execute(`ALTER TABLE workflow_artifacts ADD COLUMN content TEXT`);
7304
+ } catch {
7305
+ }
7250
7306
  await db.execute(`CREATE INDEX IF NOT EXISTS workflow_artifacts_run ON workflow_artifacts(run_id)`);
7251
7307
  await db.execute(`
7252
7308
  CREATE TABLE IF NOT EXISTS workflow_webhook_deliveries (
@@ -7692,9 +7748,22 @@ function rowToWorkflowRun(row) {
7692
7748
  queued_at: String(row.queued_at),
7693
7749
  started_at: row.started_at != null ? String(row.started_at) : null,
7694
7750
  completed_at: row.completed_at != null ? String(row.completed_at) : null,
7695
- idempotency_key: row.idempotency_key != null ? String(row.idempotency_key) : null
7751
+ idempotency_key: row.idempotency_key != null ? String(row.idempotency_key) : null,
7752
+ step_index: row.step_index != null ? Number(row.step_index) : 0,
7753
+ total_steps: row.total_steps != null ? Number(row.total_steps) : null,
7754
+ step_log: parseWorkflowStepLog(row.step_log_json),
7755
+ webhook_url: row.webhook_url != null ? String(row.webhook_url) : null
7696
7756
  };
7697
7757
  }
7758
+ function parseWorkflowStepLog(value) {
7759
+ if (value == null) return [];
7760
+ try {
7761
+ const parsed = JSON.parse(String(value));
7762
+ return Array.isArray(parsed) ? parsed : [];
7763
+ } catch {
7764
+ return [];
7765
+ }
7766
+ }
7698
7767
  function rowToWorkflowArtifact(row) {
7699
7768
  return {
7700
7769
  id: String(row.id),
@@ -7706,6 +7775,7 @@ function rowToWorkflowArtifact(row) {
7706
7775
  content_type: String(row.content_type),
7707
7776
  bytes: row.bytes != null ? Number(row.bytes) : null,
7708
7777
  rows_count: row.rows_count != null ? Number(row.rows_count) : null,
7778
+ content: row.content != null ? String(row.content) : null,
7709
7779
  created_at: String(row.created_at)
7710
7780
  };
7711
7781
  }
@@ -7817,13 +7887,15 @@ async function createWorkflowRun(input) {
7817
7887
  input.scheduleId ?? null,
7818
7888
  input.workflowId,
7819
7889
  JSON.stringify(input.workflowInput),
7820
- input.idempotencyKey ?? null
7890
+ input.idempotencyKey ?? null,
7891
+ input.webhookUrl ?? null,
7892
+ input.totalSteps ?? null
7821
7893
  ];
7822
7894
  try {
7823
7895
  await getDb().execute({
7824
7896
  sql: `
7825
- INSERT INTO workflow_runs (id, user_id, schedule_id, workflow_id, input_json, idempotency_key)
7826
- VALUES (?, ?, ?, ?, ?, ?)
7897
+ INSERT INTO workflow_runs (id, user_id, schedule_id, workflow_id, input_json, idempotency_key, webhook_url, total_steps)
7898
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
7827
7899
  `,
7828
7900
  args
7829
7901
  });
@@ -7877,30 +7949,53 @@ function workflowArtifactContentType(kind) {
7877
7949
  if (kind === "json") return "application/json; charset=utf-8";
7878
7950
  return "text/plain; charset=utf-8";
7879
7951
  }
7952
+ async function insertWorkflowArtifact(runId, userId, artifact) {
7953
+ await getDb().execute({
7954
+ sql: `
7955
+ INSERT INTO workflow_artifacts (id, run_id, user_id, kind, label, path, content_type, bytes, rows_count, content)
7956
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
7957
+ `,
7958
+ args: [
7959
+ (0, import_node_crypto.randomUUID)(),
7960
+ runId,
7961
+ userId,
7962
+ artifact.kind,
7963
+ artifact.label,
7964
+ artifact.path,
7965
+ workflowArtifactContentType(artifact.kind),
7966
+ artifact.bytes ?? null,
7967
+ artifact.rows ?? null,
7968
+ artifact.content ?? null
7969
+ ]
7970
+ });
7971
+ }
7880
7972
  async function replaceWorkflowArtifacts(runId, userId, artifacts) {
7881
- const db = getDb();
7882
- await db.execute({ sql: "DELETE FROM workflow_artifacts WHERE run_id = ? AND user_id = ?", args: [runId, userId] });
7883
- for (const artifact of artifacts) {
7884
- await db.execute({
7885
- sql: `
7886
- INSERT INTO workflow_artifacts (id, run_id, user_id, kind, label, path, content_type, bytes, rows_count)
7887
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
7888
- `,
7889
- args: [
7890
- (0, import_node_crypto.randomUUID)(),
7891
- runId,
7892
- userId,
7893
- artifact.kind,
7894
- artifact.label,
7895
- artifact.path,
7896
- workflowArtifactContentType(artifact.kind),
7897
- artifact.bytes ?? null,
7898
- artifact.rows ?? null
7899
- ]
7900
- });
7901
- }
7973
+ await getDb().execute({ sql: "DELETE FROM workflow_artifacts WHERE run_id = ? AND user_id = ?", args: [runId, userId] });
7974
+ for (const artifact of artifacts) await insertWorkflowArtifact(runId, userId, artifact);
7902
7975
  return listWorkflowArtifacts(runId, userId);
7903
7976
  }
7977
+ async function appendWorkflowArtifacts(runId, userId, artifacts) {
7978
+ for (const artifact of artifacts) await insertWorkflowArtifact(runId, userId, artifact);
7979
+ return listWorkflowArtifacts(runId, userId);
7980
+ }
7981
+ async function getWorkflowRunStepState(id, userId) {
7982
+ const res = userId == null ? await getDb().execute({ sql: "SELECT step_index, total_steps, step_state_json, step_log_json FROM workflow_runs WHERE id = ? LIMIT 1", args: [id] }) : await getDb().execute({ sql: "SELECT step_index, total_steps, step_state_json, step_log_json FROM workflow_runs WHERE id = ? AND user_id = ? LIMIT 1", args: [id, userId] });
7983
+ const row = res.rows[0];
7984
+ if (!row) return null;
7985
+ return {
7986
+ stepIndex: row.step_index != null ? Number(row.step_index) : 0,
7987
+ totalSteps: row.total_steps != null ? Number(row.total_steps) : null,
7988
+ state: row.step_state_json != null ? parseNullableJson(row.step_state_json) : null,
7989
+ stepLog: parseWorkflowStepLog(row.step_log_json)
7990
+ };
7991
+ }
7992
+ async function advanceWorkflowRunStep(id, input) {
7993
+ const stepLog = [...input.appendLog, input.logEntry];
7994
+ await getDb().execute({
7995
+ sql: `UPDATE workflow_runs SET status = ?, step_index = ?, total_steps = ?, step_state_json = ?, step_log_json = ?, started_at = COALESCE(started_at, datetime('now')) WHERE id = ?`,
7996
+ args: [input.status, input.stepIndex, input.totalSteps, JSON.stringify(input.state ?? null), JSON.stringify(stepLog), id]
7997
+ });
7998
+ }
7904
7999
  async function listWorkflowArtifacts(runId, userId) {
7905
8000
  const res = await getDb().execute({
7906
8001
  sql: "SELECT * FROM workflow_artifacts WHERE run_id = ? AND user_id = ? ORDER BY created_at ASC",
@@ -8909,6 +9004,7 @@ var init_schemas2 = __esm({
8909
9004
  downloadMp3: import_zod12.z.boolean().default(false),
8910
9005
  outputDir: import_zod12.z.string().default("./yt-output"),
8911
9006
  kernelApiKey: import_zod12.z.string().optional(),
9007
+ kernelProxyId: import_zod12.z.string().optional(),
8912
9008
  headless: import_zod12.z.boolean().default(false)
8913
9009
  }).refine(
8914
9010
  (o) => o.mode === "search" ? !!o.query : !!o.channelHandle,
@@ -9833,6 +9929,7 @@ var init_YouTubeExtractor = __esm({
9833
9929
  const config = {
9834
9930
  headless: options.headless,
9835
9931
  kernelApiKey: options.kernelApiKey,
9932
+ kernelProxyId: options.kernelProxyId,
9836
9933
  viewport: { width: 1280, height: 800 },
9837
9934
  locale: "en-US"
9838
9935
  };
@@ -10044,7 +10141,11 @@ async function ytHarvest(rawOptions) {
10044
10141
  throw new Error("A browser backend API key is required \u2014 YouTube harvesting requires a stealth session.");
10045
10142
  }
10046
10143
  const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};
10047
- const options = YouTubeHarvestOptionsSchema.parse({ kernelApiKey, ...raw });
10144
+ const options = YouTubeHarvestOptionsSchema.parse({
10145
+ kernelApiKey,
10146
+ kernelProxyId: browserServiceProxyId(),
10147
+ ...raw
10148
+ });
10048
10149
  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
10049
10150
  try {
10050
10151
  const result = await extractOnce(options);
@@ -10153,10 +10254,11 @@ function parseTimedtextXml(xml) {
10153
10254
  async function fetchViaKernelInnertube(videoId) {
10154
10255
  const kernelApiKey = browserServiceApiKey();
10155
10256
  if (!kernelApiKey) return null;
10257
+ const kernelProxyId = browserServiceProxyId();
10156
10258
  const driver = new BrowserDriver();
10157
10259
  const start = Date.now();
10158
10260
  try {
10159
- await driver.launch({ kernelApiKey, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
10261
+ await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
10160
10262
  const page = driver.getPage();
10161
10263
  await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
10162
10264
  await page.waitForFunction(
@@ -10250,10 +10352,10 @@ async function captureAudioViaMediaRecorder(page) {
10250
10352
  if (!result.ok || !result.base64 || !result.mimeType) return null;
10251
10353
  return { bytes: Buffer.from(result.base64, "base64"), mimeType: result.mimeType };
10252
10354
  }
10253
- async function attemptKernelWhisper(videoId, kernelApiKey, falKey, start) {
10355
+ async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start) {
10254
10356
  const driver = new BrowserDriver();
10255
10357
  try {
10256
- await driver.launch({ kernelApiKey, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
10358
+ await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
10257
10359
  const page = driver.getPage();
10258
10360
  await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
10259
10361
  await page.waitForFunction(
@@ -10295,12 +10397,13 @@ async function attemptKernelWhisper(videoId, kernelApiKey, falKey, start) {
10295
10397
  }
10296
10398
  async function fetchViaKernelWhisper(videoId) {
10297
10399
  const kernelApiKey = browserServiceApiKey();
10400
+ const kernelProxyId = browserServiceProxyId();
10298
10401
  const falKey = process.env.FAL_KEY;
10299
10402
  if (!kernelApiKey || !falKey) return null;
10300
10403
  const start = Date.now();
10301
10404
  for (let attempt = 0; attempt < 4; attempt++) {
10302
10405
  if (attempt > 0) await new Promise((r) => setTimeout(r, 2e3 * attempt));
10303
- const result = await attemptKernelWhisper(videoId, kernelApiKey, falKey, start);
10406
+ const result = await attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start);
10304
10407
  if (result) return result;
10305
10408
  }
10306
10409
  return null;
@@ -10626,6 +10729,7 @@ function buildTranscriptHtml(result) {
10626
10729
  (ch) => `<div class="chunk"><span class="ts">${fmtSecs(ch.timestamp[0])} \u2192 ${fmtSecs(ch.timestamp[1])}</span><span class="ct">${esc(ch.text.trim())}</span></div>`
10627
10730
  ).join("\n");
10628
10731
  return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
10732
+ ${ATTRIBUTION_PIXEL_SCRIPT}
10629
10733
  <style>body{font-family:system-ui,sans-serif;max-width:800px;margin:1.5rem auto;padding:0 1rem;color:#111;line-height:1.7}
10630
10734
  h1{font-size:1.2rem;margin-bottom:.25rem}.meta{color:#666;font-size:.85rem;margin-bottom:1.5rem}
10631
10735
  .full-text{background:#f5f5f3;border-radius:8px;padding:1rem 1.25rem;font-size:.875rem;white-space:pre-wrap;margin-bottom:2rem;word-break:break-word}
@@ -10655,6 +10759,7 @@ var init_youtube_routes = __esm({
10655
10759
  init_CaptionFetcher();
10656
10760
  init_BrowserDriver();
10657
10761
  init_errors();
10762
+ init_attribution_pixel();
10658
10763
  init_api_auth();
10659
10764
  init_server_schemas();
10660
10765
  init_concurrency_gates();
@@ -12283,6 +12388,110 @@ var init_kernel_proxy_resolver = __esm({
12283
12388
  }
12284
12389
  });
12285
12390
 
12391
+ // src/schemas.ts
12392
+ var import_zod15, DEFAULT_PROXY_MODE, DEFAULT_MAPS_PROXY_MODE, HarvestOptionsSchema, MapsPlaceOptionsSchema, MapsSearchOptionsSchema, RawPAAItemSchema, RawMapsOverviewSchema, RawMapsHoursRowSchema, RawMapsReviewStatsSchema, RawMapsReviewCardSchema, RawMapsAboutAttributeSchema;
12393
+ var init_schemas3 = __esm({
12394
+ "src/schemas.ts"() {
12395
+ "use strict";
12396
+ import_zod15 = require("zod");
12397
+ DEFAULT_PROXY_MODE = "configured";
12398
+ DEFAULT_MAPS_PROXY_MODE = "location";
12399
+ HarvestOptionsSchema = import_zod15.z.object({
12400
+ query: import_zod15.z.string().min(1),
12401
+ location: import_zod15.z.string().optional(),
12402
+ gl: import_zod15.z.string().length(2).default("us"),
12403
+ hl: import_zod15.z.string().length(2).default("en"),
12404
+ device: import_zod15.z.enum(["desktop", "mobile"]).default("desktop"),
12405
+ proxyMode: import_zod15.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
12406
+ proxyZip: import_zod15.z.string().regex(/^\d{5}$/).optional(),
12407
+ debug: import_zod15.z.boolean().default(false),
12408
+ depth: import_zod15.z.number().int().min(1).max(30).default(3),
12409
+ maxQuestions: import_zod15.z.number().int().min(1).max(1e3).default(100),
12410
+ headless: import_zod15.z.boolean().default(false),
12411
+ profileDir: import_zod15.z.string().optional(),
12412
+ proxy: import_zod15.z.string().url().optional(),
12413
+ kernelApiKey: import_zod15.z.string().optional(),
12414
+ kernelProxyId: import_zod15.z.string().optional(),
12415
+ kernelProxyResolution: import_zod15.z.unknown().optional(),
12416
+ outputDir: import_zod15.z.string().default("./paa-output"),
12417
+ format: import_zod15.z.enum(["json", "csv", "both"]).default("both"),
12418
+ serpOnly: import_zod15.z.boolean().default(false),
12419
+ pages: import_zod15.z.number().int().min(1).max(2).default(1)
12420
+ });
12421
+ MapsPlaceOptionsSchema = import_zod15.z.object({
12422
+ businessName: import_zod15.z.string().min(1),
12423
+ location: import_zod15.z.string().min(1),
12424
+ gl: import_zod15.z.string().length(2).default("us"),
12425
+ hl: import_zod15.z.string().length(2).default("en"),
12426
+ includeReviews: import_zod15.z.boolean().default(false),
12427
+ maxReviews: import_zod15.z.number().int().min(1).max(500).default(50),
12428
+ kernelApiKey: import_zod15.z.string().optional(),
12429
+ kernelProxyId: import_zod15.z.string().optional(),
12430
+ headless: import_zod15.z.boolean().default(true)
12431
+ });
12432
+ MapsSearchOptionsSchema = import_zod15.z.object({
12433
+ query: import_zod15.z.string().min(1),
12434
+ location: import_zod15.z.string().optional(),
12435
+ gl: import_zod15.z.string().length(2).default("us"),
12436
+ hl: import_zod15.z.string().length(2).default("en"),
12437
+ maxResults: import_zod15.z.number().int().min(1).max(50).default(10),
12438
+ proxyMode: import_zod15.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
12439
+ proxyZip: import_zod15.z.string().regex(/^\d{5}$/).optional(),
12440
+ debug: import_zod15.z.boolean().default(false),
12441
+ kernelApiKey: import_zod15.z.string().optional(),
12442
+ kernelProxyId: import_zod15.z.string().optional(),
12443
+ kernelProxyResolution: import_zod15.z.unknown().optional(),
12444
+ headless: import_zod15.z.boolean().default(true)
12445
+ });
12446
+ RawPAAItemSchema = import_zod15.z.object({
12447
+ question: import_zod15.z.string().min(1),
12448
+ answer: import_zod15.z.string().optional(),
12449
+ sourceTitle: import_zod15.z.string().optional(),
12450
+ sourceSite: import_zod15.z.string().optional(),
12451
+ sourceCite: import_zod15.z.string().optional()
12452
+ });
12453
+ RawMapsOverviewSchema = import_zod15.z.object({
12454
+ name: import_zod15.z.string().nullable(),
12455
+ rating: import_zod15.z.string().nullable(),
12456
+ reviewCount: import_zod15.z.string().nullable(),
12457
+ category: import_zod15.z.string().nullable(),
12458
+ address: import_zod15.z.string().nullable(),
12459
+ hoursSummary: import_zod15.z.string().nullable(),
12460
+ phone: import_zod15.z.string().nullable(),
12461
+ phoneDisplay: import_zod15.z.string().nullable(),
12462
+ website: import_zod15.z.string().nullable(),
12463
+ plusCode: import_zod15.z.string().nullable(),
12464
+ bookingUrl: import_zod15.z.string().nullable()
12465
+ });
12466
+ RawMapsHoursRowSchema = import_zod15.z.object({
12467
+ day: import_zod15.z.string(),
12468
+ hours: import_zod15.z.string()
12469
+ });
12470
+ RawMapsReviewStatsSchema = import_zod15.z.object({
12471
+ reviewHistogram: import_zod15.z.array(import_zod15.z.object({
12472
+ stars: import_zod15.z.number(),
12473
+ count: import_zod15.z.string()
12474
+ })),
12475
+ reviewTopics: import_zod15.z.array(import_zod15.z.object({
12476
+ label: import_zod15.z.string(),
12477
+ count: import_zod15.z.string()
12478
+ }))
12479
+ });
12480
+ RawMapsReviewCardSchema = import_zod15.z.object({
12481
+ reviewId: import_zod15.z.string(),
12482
+ author: import_zod15.z.string().nullable(),
12483
+ stars: import_zod15.z.string().nullable(),
12484
+ date: import_zod15.z.string().nullable(),
12485
+ text: import_zod15.z.string().nullable(),
12486
+ ownerResponse: import_zod15.z.string().nullable()
12487
+ });
12488
+ RawMapsAboutAttributeSchema = import_zod15.z.object({
12489
+ section: import_zod15.z.string(),
12490
+ attribute: import_zod15.z.string()
12491
+ });
12492
+ }
12493
+ });
12494
+
12286
12495
  // src/api/facebook-ad-routes.ts
12287
12496
  function invalidRequest(message) {
12288
12497
  return { error_code: "invalid_request", message };
@@ -12338,9 +12547,8 @@ async function kernelLaunchOptsResidential() {
12338
12547
  try {
12339
12548
  const resolution2 = await resolveKernelProxyId({
12340
12549
  kernelApiKey: browserServiceApiKey(),
12341
- proxyMode: "location",
12550
+ proxyMode: DEFAULT_PROXY_MODE,
12342
12551
  configuredKernelProxyId: browserServiceProxyId(),
12343
- location: "New York, NY",
12344
12552
  gl: "us"
12345
12553
  });
12346
12554
  if (resolution2.kernelProxyId) proxyId = resolution2.kernelProxyId;
@@ -12349,13 +12557,13 @@ async function kernelLaunchOptsResidential() {
12349
12557
  }
12350
12558
  return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
12351
12559
  }
12352
- var import_hono4, import_zod15, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
12560
+ var import_hono4, import_zod16, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
12353
12561
  var init_facebook_ad_routes = __esm({
12354
12562
  "src/api/facebook-ad-routes.ts"() {
12355
12563
  "use strict";
12356
12564
  import_hono4 = require("hono");
12357
12565
  init_browser_service_env();
12358
- import_zod15 = require("zod");
12566
+ import_zod16 = require("zod");
12359
12567
  init_db();
12360
12568
  init_rates();
12361
12569
  init_BrowserDriver();
@@ -12363,39 +12571,40 @@ var init_facebook_ad_routes = __esm({
12363
12571
  init_FacebookAdGraphql();
12364
12572
  init_FacebookOrganicVideoExtractor();
12365
12573
  init_kernel_proxy_resolver();
12574
+ init_schemas3();
12366
12575
  import_client3 = require("@fal-ai/client");
12367
12576
  init_api_auth();
12368
12577
  init_url_utils();
12369
12578
  init_concurrency_gates();
12370
- FacebookAdBodySchema = import_zod15.z.object({
12371
- url: import_zod15.z.string().trim().optional(),
12372
- libraryId: import_zod15.z.string().trim().optional(),
12373
- openModal: import_zod15.z.boolean().optional()
12579
+ FacebookAdBodySchema = import_zod16.z.object({
12580
+ url: import_zod16.z.string().trim().optional(),
12581
+ libraryId: import_zod16.z.string().trim().optional(),
12582
+ openModal: import_zod16.z.boolean().optional()
12374
12583
  }).refine((d) => !!d.url || !!d.libraryId, { message: "url or libraryId is required" });
12375
- FacebookPageIntelBodySchema = import_zod15.z.object({
12376
- pageId: import_zod15.z.string().trim().optional(),
12377
- query: import_zod15.z.string().trim().optional(),
12378
- libraryId: import_zod15.z.string().trim().optional(),
12379
- maxAds: import_zod15.z.number().int().min(1).max(200).optional(),
12380
- country: import_zod15.z.string().trim().toUpperCase().optional()
12584
+ FacebookPageIntelBodySchema = import_zod16.z.object({
12585
+ pageId: import_zod16.z.string().trim().optional(),
12586
+ query: import_zod16.z.string().trim().optional(),
12587
+ libraryId: import_zod16.z.string().trim().optional(),
12588
+ maxAds: import_zod16.z.number().int().min(1).max(200).optional(),
12589
+ country: import_zod16.z.string().trim().toUpperCase().optional()
12381
12590
  }).refine((d) => !!d.pageId || !!d.query || !!d.libraryId, {
12382
12591
  message: "pageId, libraryId, or query is required"
12383
12592
  });
12384
- FacebookTranscribeBodySchema = import_zod15.z.object({
12385
- videoUrl: import_zod15.z.string().trim().min(1, "videoUrl is required")
12593
+ FacebookTranscribeBodySchema = import_zod16.z.object({
12594
+ videoUrl: import_zod16.z.string().trim().min(1, "videoUrl is required")
12386
12595
  });
12387
- FacebookVideoTranscribeBodySchema = import_zod15.z.object({
12388
- url: import_zod15.z.string().trim().min(1, "url is required"),
12389
- quality: import_zod15.z.enum(["best", "hd", "sd"]).default("best")
12596
+ FacebookVideoTranscribeBodySchema = import_zod16.z.object({
12597
+ url: import_zod16.z.string().trim().min(1, "url is required"),
12598
+ quality: import_zod16.z.enum(["best", "hd", "sd"]).default("best")
12390
12599
  });
12391
- FacebookSearchBodySchema = import_zod15.z.object({
12392
- query: import_zod15.z.string().trim().min(1, "query is required"),
12393
- country: import_zod15.z.string().trim().toUpperCase().optional(),
12394
- maxResults: import_zod15.z.number().int().min(1).max(20).optional()
12600
+ FacebookSearchBodySchema = import_zod16.z.object({
12601
+ query: import_zod16.z.string().trim().min(1, "query is required"),
12602
+ country: import_zod16.z.string().trim().toUpperCase().optional(),
12603
+ maxResults: import_zod16.z.number().int().min(1).max(20).optional()
12395
12604
  });
12396
- FacebookMediaBodySchema = import_zod15.z.object({
12397
- url: import_zod15.z.string().trim().min(1, "url is required"),
12398
- filename: import_zod15.z.string().trim().optional()
12605
+ FacebookMediaBodySchema = import_zod16.z.object({
12606
+ url: import_zod16.z.string().trim().min(1, "url is required"),
12607
+ filename: import_zod16.z.string().trim().optional()
12399
12608
  });
12400
12609
  facebookAdApp = new import_hono4.Hono();
12401
12610
  facebookAdApp.post("/ad", createApiKeyAuth(), async (c) => {
@@ -12721,108 +12930,6 @@ var init_facebook_ad_routes = __esm({
12721
12930
  }
12722
12931
  });
12723
12932
 
12724
- // src/schemas.ts
12725
- var import_zod16, HarvestOptionsSchema, MapsPlaceOptionsSchema, MapsSearchOptionsSchema, RawPAAItemSchema, RawMapsOverviewSchema, RawMapsHoursRowSchema, RawMapsReviewStatsSchema, RawMapsReviewCardSchema, RawMapsAboutAttributeSchema;
12726
- var init_schemas3 = __esm({
12727
- "src/schemas.ts"() {
12728
- "use strict";
12729
- import_zod16 = require("zod");
12730
- HarvestOptionsSchema = import_zod16.z.object({
12731
- query: import_zod16.z.string().min(1),
12732
- location: import_zod16.z.string().optional(),
12733
- gl: import_zod16.z.string().length(2).default("us"),
12734
- hl: import_zod16.z.string().length(2).default("en"),
12735
- device: import_zod16.z.enum(["desktop", "mobile"]).default("desktop"),
12736
- proxyMode: import_zod16.z.enum(["location", "configured", "none"]).default("location"),
12737
- proxyZip: import_zod16.z.string().regex(/^\d{5}$/).optional(),
12738
- debug: import_zod16.z.boolean().default(false),
12739
- depth: import_zod16.z.number().int().min(1).max(30).default(3),
12740
- maxQuestions: import_zod16.z.number().int().min(1).max(1e3).default(100),
12741
- headless: import_zod16.z.boolean().default(false),
12742
- profileDir: import_zod16.z.string().optional(),
12743
- proxy: import_zod16.z.string().url().optional(),
12744
- kernelApiKey: import_zod16.z.string().optional(),
12745
- kernelProxyId: import_zod16.z.string().optional(),
12746
- kernelProxyResolution: import_zod16.z.unknown().optional(),
12747
- outputDir: import_zod16.z.string().default("./paa-output"),
12748
- format: import_zod16.z.enum(["json", "csv", "both"]).default("both"),
12749
- serpOnly: import_zod16.z.boolean().default(false),
12750
- pages: import_zod16.z.number().int().min(1).max(2).default(1)
12751
- });
12752
- MapsPlaceOptionsSchema = import_zod16.z.object({
12753
- businessName: import_zod16.z.string().min(1),
12754
- location: import_zod16.z.string().min(1),
12755
- gl: import_zod16.z.string().length(2).default("us"),
12756
- hl: import_zod16.z.string().length(2).default("en"),
12757
- includeReviews: import_zod16.z.boolean().default(false),
12758
- maxReviews: import_zod16.z.number().int().min(1).max(500).default(50),
12759
- kernelApiKey: import_zod16.z.string().optional(),
12760
- kernelProxyId: import_zod16.z.string().optional(),
12761
- headless: import_zod16.z.boolean().default(true)
12762
- });
12763
- MapsSearchOptionsSchema = import_zod16.z.object({
12764
- query: import_zod16.z.string().min(1),
12765
- location: import_zod16.z.string().optional(),
12766
- gl: import_zod16.z.string().length(2).default("us"),
12767
- hl: import_zod16.z.string().length(2).default("en"),
12768
- maxResults: import_zod16.z.number().int().min(1).max(50).default(10),
12769
- proxyMode: import_zod16.z.enum(["location", "configured", "none"]).default("location"),
12770
- proxyZip: import_zod16.z.string().regex(/^\d{5}$/).optional(),
12771
- debug: import_zod16.z.boolean().default(false),
12772
- kernelApiKey: import_zod16.z.string().optional(),
12773
- kernelProxyId: import_zod16.z.string().optional(),
12774
- kernelProxyResolution: import_zod16.z.unknown().optional(),
12775
- headless: import_zod16.z.boolean().default(true)
12776
- });
12777
- RawPAAItemSchema = import_zod16.z.object({
12778
- question: import_zod16.z.string().min(1),
12779
- answer: import_zod16.z.string().optional(),
12780
- sourceTitle: import_zod16.z.string().optional(),
12781
- sourceSite: import_zod16.z.string().optional(),
12782
- sourceCite: import_zod16.z.string().optional()
12783
- });
12784
- RawMapsOverviewSchema = import_zod16.z.object({
12785
- name: import_zod16.z.string().nullable(),
12786
- rating: import_zod16.z.string().nullable(),
12787
- reviewCount: import_zod16.z.string().nullable(),
12788
- category: import_zod16.z.string().nullable(),
12789
- address: import_zod16.z.string().nullable(),
12790
- hoursSummary: import_zod16.z.string().nullable(),
12791
- phone: import_zod16.z.string().nullable(),
12792
- phoneDisplay: import_zod16.z.string().nullable(),
12793
- website: import_zod16.z.string().nullable(),
12794
- plusCode: import_zod16.z.string().nullable(),
12795
- bookingUrl: import_zod16.z.string().nullable()
12796
- });
12797
- RawMapsHoursRowSchema = import_zod16.z.object({
12798
- day: import_zod16.z.string(),
12799
- hours: import_zod16.z.string()
12800
- });
12801
- RawMapsReviewStatsSchema = import_zod16.z.object({
12802
- reviewHistogram: import_zod16.z.array(import_zod16.z.object({
12803
- stars: import_zod16.z.number(),
12804
- count: import_zod16.z.string()
12805
- })),
12806
- reviewTopics: import_zod16.z.array(import_zod16.z.object({
12807
- label: import_zod16.z.string(),
12808
- count: import_zod16.z.string()
12809
- }))
12810
- });
12811
- RawMapsReviewCardSchema = import_zod16.z.object({
12812
- reviewId: import_zod16.z.string(),
12813
- author: import_zod16.z.string().nullable(),
12814
- stars: import_zod16.z.string().nullable(),
12815
- date: import_zod16.z.string().nullable(),
12816
- text: import_zod16.z.string().nullable(),
12817
- ownerResponse: import_zod16.z.string().nullable()
12818
- });
12819
- RawMapsAboutAttributeSchema = import_zod16.z.object({
12820
- section: import_zod16.z.string(),
12821
- attribute: import_zod16.z.string()
12822
- });
12823
- }
12824
- });
12825
-
12826
12933
  // src/extractor/MapsNavigator.ts
12827
12934
  var MapsNavigator;
12828
12935
  var init_MapsNavigator = __esm({
@@ -14432,13 +14539,25 @@ ${videoRows}`,
14432
14539
  }
14433
14540
  };
14434
14541
  }
14542
+ function structuredTranscriptChunks(chunks) {
14543
+ return chunks.map((c) => ({
14544
+ startSec: Number.isFinite(c.timestamp?.[0]) ? c.timestamp[0] : 0,
14545
+ endSec: Number.isFinite(c.timestamp?.[1]) ? c.timestamp[1] : 0,
14546
+ text: c.text
14547
+ }));
14548
+ }
14549
+ function wordCount(text) {
14550
+ return text.trim() ? text.trim().split(/\s+/).length : 0;
14551
+ }
14435
14552
  function formatYoutubeTranscribe(raw, input) {
14436
14553
  const parsed = parseData(raw);
14437
14554
  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
14438
14555
  const d = parsed.data;
14439
14556
  const text = d.text ?? "";
14440
14557
  const chunks = d.chunks ?? [];
14558
+ const videoId = d.videoId ?? input.videoId ?? null;
14441
14559
  const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
14560
+ const words = wordCount(text);
14442
14561
  const chunkRows = chunks.slice(0, 50).map((c) => {
14443
14562
  const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
14444
14563
  const mm = String(Math.floor(sec / 60)).padStart(2, "0");
@@ -14446,8 +14565,8 @@ function formatYoutubeTranscribe(raw, input) {
14446
14565
  return `| ${mm}:${ss} | ${cell(truncate(c.text, 120))} |`;
14447
14566
  }).join("\n");
14448
14567
  const full = [
14449
- `# YouTube Transcript: \`${input.videoId}\``,
14450
- `**Duration:** ${durSec}s \xB7 **${text.split(" ").length} words**`,
14568
+ `# YouTube Transcript: \`${videoId ?? input.url ?? "video"}\``,
14569
+ `**Duration:** ${durSec}s \xB7 **${words} words**`,
14451
14570
  `
14452
14571
  ## Full Transcript
14453
14572
  ${text}`,
@@ -14460,7 +14579,22 @@ ${chunkRows}` : "",
14460
14579
  ---
14461
14580
  \u{1F4A1} Harvest more from this channel: use \`youtube_harvest\` with \`mode: "channel"\``
14462
14581
  ].filter(Boolean).join("\n");
14463
- return oneBlock(full);
14582
+ return {
14583
+ ...oneBlock(full),
14584
+ structuredContent: {
14585
+ videoId,
14586
+ url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : input.url ?? null,
14587
+ wordCount: words,
14588
+ chunkCount: chunks.length,
14589
+ durationMs: typeof d.durationMs === "number" ? d.durationMs : null,
14590
+ transcriptText: text,
14591
+ chunks: structuredTranscriptChunks(chunks),
14592
+ resolvedInputs: {
14593
+ videoId,
14594
+ url: input.url ?? null
14595
+ }
14596
+ }
14597
+ };
14464
14598
  }
14465
14599
  function formatFacebookPageIntel(raw, input) {
14466
14600
  const parsed = parseData(raw);
@@ -14470,12 +14604,13 @@ function formatFacebookPageIntel(raw, input) {
14470
14604
  const ads = d.ads ?? [];
14471
14605
  const s = d.summary ?? { totalAds: 0, activeCount: 0, videoCount: 0, imageCount: 0 };
14472
14606
  const adBlocks = ads.map((ad, i) => [
14473
- `### Ad ${i + 1}${ad.libraryId ? ` \xB7 \`${ad.libraryId}\`` : ""} \u2014 ${ad.status ?? "\u2014"} \xB7 ${ad.creativeType ?? "\u2014"} \xB7 ${ad.startDate ?? "\u2014"}`,
14607
+ `### Ad ${i + 1}${ad.libraryId ? ` \xB7 \`${ad.libraryId}\`` : ""} \u2014 ${ad.status ?? "\u2014"} \xB7 ${ad.creativeType ?? "\u2014"} \xB7 ${ad.startDate ?? ad.started ?? "\u2014"}`,
14474
14608
  ad.headline ? `**Headline:** ${ad.headline}` : "",
14475
14609
  ad.primaryText ? `**Copy:** ${truncate(ad.primaryText, 200)}` : "",
14476
14610
  ad.cta ? `**CTA:** ${ad.cta}` : "",
14477
- ad.videoUrl ? `**Video URL:** \`${ad.videoUrl}\`` : "",
14478
- ad.variations ? `**Variations:** ${ad.variations}` : ""
14611
+ ad.landingUrl ? `**Landing URL:** ${ad.landingUrl}` : "",
14612
+ ad.videoUrl ?? ad.videoSrc ? `**Video URL:** \`${ad.videoUrl ?? ad.videoSrc}\`` : "",
14613
+ ad.variations ?? ad.clusterCount ? `**Variations:** ${ad.variations ?? ad.clusterCount}` : ""
14479
14614
  ].filter(Boolean).join("\n")).join("\n\n---\n\n");
14480
14615
  const full = [
14481
14616
  `# Facebook Ad Intel: ${advertiser}`,
@@ -14501,11 +14636,16 @@ ${adBlocks}`,
14501
14636
  libraryId: ad.libraryId ?? null,
14502
14637
  status: ad.status ?? null,
14503
14638
  creativeType: ad.creativeType ?? null,
14639
+ primaryText: ad.primaryText ?? null,
14504
14640
  headline: ad.headline ?? null,
14505
14641
  cta: ad.cta ?? null,
14506
- startDate: ad.startDate ?? null,
14507
- videoUrl: ad.videoUrl ?? null,
14508
- variations: typeof ad.variations === "number" ? ad.variations : null
14642
+ startDate: ad.startDate ?? ad.started ?? null,
14643
+ landingUrl: ad.landingUrl ?? null,
14644
+ domain: ad.domain ?? null,
14645
+ videoUrl: ad.videoUrl ?? ad.videoSrc ?? null,
14646
+ imageUrl: ad.imageUrl ?? ad.imageSrc ?? null,
14647
+ videoPoster: ad.videoPoster ?? null,
14648
+ variations: typeof ad.variations === "number" ? ad.variations : typeof ad.clusterCount === "number" ? ad.clusterCount : null
14509
14649
  }))
14510
14650
  }
14511
14651
  };
@@ -14539,8 +14679,11 @@ ${rows}`,
14539
14679
  advertiserCount: advertisers.length,
14540
14680
  advertisers: advertisers.map((a) => ({
14541
14681
  name: a.pageName ?? a.name ?? null,
14682
+ pageId: a.pageId ?? null,
14683
+ pageUrl: a.pageUrl ?? null,
14542
14684
  adCount: typeof a.adCount === "number" ? a.adCount : null,
14543
- libraryId: a.sampleLibraryId ?? a.libraryId ?? null
14685
+ libraryId: a.sampleLibraryId ?? a.libraryId ?? null,
14686
+ sampleLibraryId: a.sampleLibraryId ?? null
14544
14687
  }))
14545
14688
  }
14546
14689
  };
@@ -14638,6 +14781,34 @@ function formatWorkflowSuggest(input) {
14638
14781
  }
14639
14782
  };
14640
14783
  }
14784
+ function workflowStepLines(data) {
14785
+ const step = data.step;
14786
+ const nextStep = data.nextStep;
14787
+ const done = data.done === true;
14788
+ const lines = [];
14789
+ if (step) {
14790
+ const index = Number(step.index ?? 0);
14791
+ const totalSteps = step.totalSteps != null ? Number(step.totalSteps) : void 0;
14792
+ const stepLabel = totalSteps ? `${index + 1}/${totalSteps}` : `${index + 1}`;
14793
+ lines.push(`
14794
+ ## Step ${stepLabel}: ${step.title ?? step.id ?? ""}`);
14795
+ const output = step.output;
14796
+ if (output && Object.keys(output).length) {
14797
+ lines.push(Object.entries(output).map(([k, v]) => `- ${k}: ${typeof v === "object" ? JSON.stringify(v) : String(v)}`).join("\n"));
14798
+ }
14799
+ const warnings = Array.isArray(step.warnings) ? step.warnings : [];
14800
+ if (warnings.length) lines.push(`
14801
+ **Warnings:**
14802
+ ${warnings.map((w) => `- ${w}`).join("\n")}`);
14803
+ }
14804
+ if (done) {
14805
+ lines.push("\n**Done.** All steps complete.");
14806
+ } else if (nextStep && typeof nextStep === "object") {
14807
+ lines.push(`
14808
+ **Next step:** \`${nextStep.id ?? nextStep.index}\` \u2014 call \`workflow_step\` with this run id to continue.`);
14809
+ }
14810
+ return lines;
14811
+ }
14641
14812
  function formatWorkflowRun(raw, input) {
14642
14813
  const parsed = parseData(raw);
14643
14814
  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
@@ -14651,6 +14822,7 @@ function formatWorkflowRun(raw, input) {
14651
14822
  `**Run ID:** \`${runId || "unknown"}\``,
14652
14823
  `**Status:** ${status}`,
14653
14824
  summary?.title ? `**Title:** ${summary.title}` : "",
14825
+ ...workflowStepLines(parsed.data),
14654
14826
  summary?.summary ? `
14655
14827
  ## Summary
14656
14828
  ${summary.summary}` : "",
@@ -14666,6 +14838,42 @@ ${workflowArtifactRows(artifacts)}` : "",
14666
14838
  input: input.input ?? {},
14667
14839
  run,
14668
14840
  summary,
14841
+ step: parsed.data.step,
14842
+ nextStep: parsed.data.nextStep ?? null,
14843
+ done: parsed.data.done === true,
14844
+ artifacts
14845
+ }
14846
+ };
14847
+ }
14848
+ function formatWorkflowStep(raw, input) {
14849
+ const parsed = parseData(raw);
14850
+ if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
14851
+ const run = parsed.data.run;
14852
+ const summary = parsed.data.summary;
14853
+ const artifacts = workflowArtifactsFrom(run);
14854
+ const done = parsed.data.done === true;
14855
+ const full = [
14856
+ `# Workflow Step`,
14857
+ `**Run ID:** \`${input.runId}\``,
14858
+ `**Status:** ${run?.status ?? (done ? "done" : "running")}`,
14859
+ ...workflowStepLines(parsed.data),
14860
+ done && summary?.summary ? `
14861
+ ## Summary
14862
+ ${summary.summary}` : "",
14863
+ artifacts.length ? `
14864
+ ## Artifacts
14865
+ ${workflowArtifactRows(artifacts)}` : "",
14866
+ done && artifacts.length ? "\nUse `workflow_artifact_read` with the run id and artifact id to pull CSV, JSON, Markdown, or report content into context." : ""
14867
+ ].filter(Boolean).join("\n");
14868
+ return {
14869
+ ...oneBlock(full),
14870
+ structuredContent: {
14871
+ runId: input.runId,
14872
+ run,
14873
+ summary: summary ?? null,
14874
+ step: parsed.data.step,
14875
+ nextStep: parsed.data.nextStep ?? null,
14876
+ done,
14669
14877
  artifacts
14670
14878
  }
14671
14879
  };
@@ -15070,6 +15278,7 @@ function formatFacebookAdTranscribe(raw, input) {
15070
15278
  const text = d.text ?? "";
15071
15279
  const chunks = d.chunks ?? [];
15072
15280
  const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
15281
+ const words = wordCount(text);
15073
15282
  const chunkRows = chunks.slice(0, 50).map((c) => {
15074
15283
  const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
15075
15284
  const mm = String(Math.floor(sec / 60)).padStart(2, "0");
@@ -15078,7 +15287,7 @@ function formatFacebookAdTranscribe(raw, input) {
15078
15287
  }).join("\n");
15079
15288
  const full = [
15080
15289
  `# Facebook Ad Transcript`,
15081
- `**Duration:** ${durSec}s \xB7 **${text.split(" ").length} words**`,
15290
+ `**Duration:** ${durSec}s \xB7 **${words} words**`,
15082
15291
  `
15083
15292
  ## Full Transcript
15084
15293
  ${text}`,
@@ -15091,7 +15300,20 @@ ${chunkRows}` : "",
15091
15300
  ---
15092
15301
  \u{1F4A1} Get more ads from this advertiser: use \`facebook_page_intel\`. For public Facebook reel/post URLs, use \`facebook_video_transcribe\`.`
15093
15302
  ].filter(Boolean).join("\n");
15094
- return oneBlock(full);
15303
+ return {
15304
+ ...oneBlock(full),
15305
+ structuredContent: {
15306
+ videoUrl: input.videoUrl,
15307
+ wordCount: words,
15308
+ chunkCount: chunks.length,
15309
+ durationMs: typeof d.durationMs === "number" ? d.durationMs : null,
15310
+ transcriptText: text,
15311
+ chunks: structuredTranscriptChunks(chunks),
15312
+ resolvedInputs: {
15313
+ videoUrl: input.videoUrl
15314
+ }
15315
+ }
15316
+ };
15095
15317
  }
15096
15318
  function formatFacebookVideoTranscribe(raw, input) {
15097
15319
  const parsed = parseData(raw);
@@ -15099,7 +15321,7 @@ function formatFacebookVideoTranscribe(raw, input) {
15099
15321
  const d = parsed.data;
15100
15322
  const text = d.text ?? "";
15101
15323
  const chunks = d.chunks ?? [];
15102
- const wordCount = text.trim() ? text.trim().split(/\s+/).length : 0;
15324
+ const wordCount2 = text.trim() ? text.trim().split(/\s+/).length : 0;
15103
15325
  const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
15104
15326
  const videoDuration = typeof d.videoDurationSec === "number" ? `${Math.round(d.videoDurationSec)}s` : "\u2014";
15105
15327
  const label = d.videoId ? `Facebook Organic Video \`${d.videoId}\`` : "Facebook Organic Video";
@@ -15112,7 +15334,7 @@ function formatFacebookVideoTranscribe(raw, input) {
15112
15334
  const full = [
15113
15335
  `# ${label} Transcript`,
15114
15336
  d.ownerName ? `**Owner:** ${d.ownerName}` : "",
15115
- `**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${wordCount} words**`,
15337
+ `**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${wordCount2} words**`,
15116
15338
  d.pageUrl ? `**Page URL:** ${d.pageUrl}` : `**Page URL:** ${input.url}`,
15117
15339
  d.videoUrl ? `**Extracted MP4:** \`${d.videoUrl}\`` : "",
15118
15340
  `
@@ -15138,7 +15360,7 @@ ${chunkRows}` : "",
15138
15360
  bitrate: typeof d.bitrate === "number" ? d.bitrate : null,
15139
15361
  videoDurationSec: typeof d.videoDurationSec === "number" ? d.videoDurationSec : null,
15140
15362
  videoUrl: d.videoUrl ?? "",
15141
- wordCount,
15363
+ wordCount: wordCount2,
15142
15364
  chunkCount: chunks.length,
15143
15365
  transcriptText: text,
15144
15366
  chunks: chunks.map((c) => ({
@@ -15149,6 +15371,96 @@ ${chunkRows}` : "",
15149
15371
  }
15150
15372
  };
15151
15373
  }
15374
+ function formatCaptureSerpSnapshot(raw, input) {
15375
+ const parsed = parseData(raw);
15376
+ if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
15377
+ const d = parsed.data;
15378
+ const providerPayload = d;
15379
+ const organic = Array.isArray(d.organicResults) ? d.organicResults : Array.isArray(d.organic) ? d.organic : [];
15380
+ const localPack = Array.isArray(d.localPack) ? d.localPack : [];
15381
+ const artifacts = Array.isArray(d.artifacts) ? d.artifacts : [];
15382
+ const status = String(d.status ?? d.captureStatus ?? "captured");
15383
+ const query = typeof d.query === "string" ? d.query : typeof input.query === "string" ? input.query : null;
15384
+ const location2 = typeof d.location === "string" ? d.location : typeof input.location === "string" ? input.location : null;
15385
+ const capturedAt = typeof d.capturedAt === "string" ? d.capturedAt : typeof d.extractedAt === "string" ? d.extractedAt : null;
15386
+ const resultCount = organic.length + localPack.length;
15387
+ const full = [
15388
+ `# SERP Intelligence Snapshot: ${query ?? "query"}`,
15389
+ `**Status:** ${status}`,
15390
+ location2 ? `**Location:** ${location2}` : "",
15391
+ `**Result count:** ${resultCount}`,
15392
+ artifacts.length ? `**Artifacts:** ${artifacts.length}` : "",
15393
+ "",
15394
+ "Use `capture_serp_page_snapshots` when you need page-level evidence for ranking URLs."
15395
+ ].filter(Boolean).join("\n");
15396
+ return {
15397
+ ...oneBlock(full),
15398
+ structuredContent: {
15399
+ schemaVersion: "serp-intelligence.capture.v1",
15400
+ status,
15401
+ query,
15402
+ location: location2,
15403
+ capturedAt,
15404
+ resultCount,
15405
+ snapshotId: typeof d.snapshotId === "string" ? d.snapshotId : typeof d.snapshot_id === "string" ? d.snapshot_id : typeof d.id === "string" ? d.id : null,
15406
+ resolvedInputs: {
15407
+ query: input.query ?? null,
15408
+ location: input.location ?? null,
15409
+ gl: input.gl ?? null,
15410
+ hl: input.hl ?? null,
15411
+ device: input.device ?? null,
15412
+ proxyMode: input.proxyMode ?? null,
15413
+ proxyZip: input.proxyZip ?? null,
15414
+ pages: input.pages ?? null
15415
+ },
15416
+ artifacts,
15417
+ diagnostics: d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null,
15418
+ providerPayload
15419
+ }
15420
+ };
15421
+ }
15422
+ function formatCaptureSerpPageSnapshots(raw, input) {
15423
+ const parsed = parseData(raw);
15424
+ if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
15425
+ const d = parsed.data;
15426
+ const captures = Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
15427
+ const failedCount = captures.filter((c) => c.status === "failed" || c.error).length;
15428
+ const status = String(d.status ?? (failedCount ? "partial" : "captured"));
15429
+ const rows = captures.slice(0, 25).map((capture, i) => {
15430
+ const url = typeof capture.url === "string" ? capture.url : "";
15431
+ const sourceKind = typeof capture.sourceKind === "string" ? capture.sourceKind : typeof capture.source_kind === "string" ? capture.source_kind : "configured_target";
15432
+ return `| ${i + 1} | ${cell(url)} | ${cell(sourceKind)} | ${cell(String(capture.status ?? (capture.error ? "failed" : "ok")))} |`;
15433
+ }).join("\n");
15434
+ const full = [
15435
+ "# SERP Intelligence Page Snapshots",
15436
+ `**Status:** ${status}`,
15437
+ `**Captured:** ${captures.length}`,
15438
+ `**Failed:** ${failedCount}`,
15439
+ captures.length ? `
15440
+ | # | URL | Source | Status |
15441
+ |---|-----|--------|--------|
15442
+ ${rows}` : ""
15443
+ ].filter(Boolean).join("\n");
15444
+ return {
15445
+ ...oneBlock(full),
15446
+ structuredContent: {
15447
+ schemaVersion: "serp-intelligence.page-snapshots.v1",
15448
+ status,
15449
+ count: captures.length,
15450
+ failedCount,
15451
+ captures,
15452
+ resolvedInputs: {
15453
+ urls: input.urls ?? null,
15454
+ targets: input.targets ?? null,
15455
+ maxConcurrency: input.maxConcurrency ?? null,
15456
+ timeoutMs: input.timeoutMs ?? null,
15457
+ debug: input.debug ?? null
15458
+ },
15459
+ diagnostics: d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null,
15460
+ providerPayload: d
15461
+ }
15462
+ };
15463
+ }
15152
15464
  var import_node_fs3, import_node_os3, import_node_path5, reportSavingEnabled;
15153
15465
  var init_mcp_response_formatter = __esm({
15154
15466
  "src/mcp/mcp-response-formatter.ts"() {
@@ -15623,6 +15935,7 @@ var init_directory_workflow = __esm({
15623
15935
  init_mcp_response_formatter();
15624
15936
  init_browser_service_env();
15625
15937
  init_maps_search_rotation();
15938
+ init_schemas3();
15626
15939
  init_csv();
15627
15940
  init_location_db();
15628
15941
  DirectoryWorkflowOptionsSchema = import_zod18.z.object({
@@ -15638,7 +15951,7 @@ var init_directory_workflow = __esm({
15638
15951
  saveCsv: import_zod18.z.boolean().default(true),
15639
15952
  gl: import_zod18.z.string().length(2).default("us"),
15640
15953
  hl: import_zod18.z.string().length(2).default("en"),
15641
- proxyMode: import_zod18.z.enum(["location", "configured", "none"]).default("location"),
15954
+ proxyMode: import_zod18.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
15642
15955
  proxyZip: import_zod18.z.string().regex(/^\d{5}$/).optional(),
15643
15956
  debug: import_zod18.z.boolean().default(false),
15644
15957
  headless: import_zod18.z.boolean().default(true),
@@ -16062,102 +16375,141 @@ var init_agent_packet = __esm({
16062
16375
  title: "Agent-Ready SEO Packet",
16063
16376
  description: "Create an evidence folder for AI agents from live SERP/PAA/AI search surfaces.",
16064
16377
  inputSchema: AgentPacketInputSchema,
16065
- async run(input, ctx) {
16066
- await ctx.artifacts.writeManifest("running", {}, [], []);
16067
- const warnings = [];
16068
- const errors = [];
16069
- let serp = null;
16070
- let paa = null;
16071
- if (input.includeSerp) {
16072
- try {
16073
- serp = await ctx.client.post("/harvest/sync", {
16074
- query: input.keyword,
16075
- location: input.location,
16076
- serpOnly: true,
16077
- maxQuestions: 1,
16078
- format: "json"
16079
- }, 18e4);
16080
- } catch (err) {
16081
- warnings.push(`SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
16378
+ createState: () => ({ serp: null, paa: null, warnings: [] }),
16379
+ steps: [
16380
+ {
16381
+ id: "harvest-serp",
16382
+ title: "Harvest organic SERP + AI Overview",
16383
+ async run({ input, state, ctx }) {
16384
+ if (!input.includeSerp) {
16385
+ return { state, output: { skipped: true, reason: "includeSerp is false" } };
16386
+ }
16387
+ try {
16388
+ const serp = await ctx.client.post("/harvest/sync", {
16389
+ query: input.keyword,
16390
+ location: input.location,
16391
+ serpOnly: true,
16392
+ maxQuestions: 1,
16393
+ format: "json"
16394
+ }, 18e4);
16395
+ await ctx.artifacts.writeJson("SERP raw JSON", "raw/serp.json", serp);
16396
+ return {
16397
+ state: { ...state, serp },
16398
+ output: {
16399
+ organicResults: serp.organicResults?.length ?? 0,
16400
+ localPack: serp.localPack?.length ?? 0,
16401
+ aiOverviewDetected: Boolean(serp.aiOverview?.detected),
16402
+ aiOverviewCitations: serp.aiOverview?.citations?.length ?? 0
16403
+ }
16404
+ };
16405
+ } catch (err) {
16406
+ const message = `SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
16407
+ return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
16408
+ }
16082
16409
  }
16083
- }
16084
- if (input.includePaa) {
16085
- try {
16086
- paa = await ctx.client.post("/harvest/sync", {
16087
- query: input.keyword,
16088
- location: input.location,
16089
- maxQuestions: input.maxQuestions,
16090
- format: "json"
16091
- }, 28e4);
16092
- } catch (err) {
16093
- warnings.push(`PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
16410
+ },
16411
+ {
16412
+ id: "harvest-paa",
16413
+ title: "Harvest People Also Ask",
16414
+ async run({ input, state, ctx }) {
16415
+ if (!input.includePaa) {
16416
+ return { state, output: { skipped: true, reason: "includePaa is false" } };
16417
+ }
16418
+ try {
16419
+ const paa = await ctx.client.post("/harvest/sync", {
16420
+ query: input.keyword,
16421
+ location: input.location,
16422
+ maxQuestions: input.maxQuestions,
16423
+ format: "json"
16424
+ }, 28e4);
16425
+ await ctx.artifacts.writeJson("PAA raw JSON", "raw/paa.json", paa);
16426
+ return {
16427
+ state: { ...state, paa },
16428
+ output: { paaQuestions: paa.flat?.length ?? 0 }
16429
+ };
16430
+ } catch (err) {
16431
+ const message = `PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
16432
+ return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
16433
+ }
16434
+ }
16435
+ },
16436
+ {
16437
+ id: "assemble",
16438
+ title: "Assemble evidence packet",
16439
+ async run({ input, state, ctx }) {
16440
+ const { serp, paa } = state;
16441
+ const warnings = state.warnings;
16442
+ if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
16443
+ const rows = sourceRows(input, serp, paa);
16444
+ const target = normalizeDomain(input.domain);
16445
+ const competitors = competitorRows(rows, target);
16446
+ const evidence = {
16447
+ input,
16448
+ serp: serp ?? { status: "skipped" },
16449
+ paa: paa ?? { status: "skipped" },
16450
+ target: {
16451
+ domain: target,
16452
+ organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
16453
+ citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
16454
+ citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
16455
+ },
16456
+ competitors,
16457
+ warnings
16458
+ };
16459
+ await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
16460
+ await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
16461
+ await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
16462
+ const brief = [
16463
+ `# SEO Evidence Brief: ${input.keyword}`,
16464
+ "",
16465
+ `Location: ${input.location ?? "not specified"}`,
16466
+ `Target domain: ${target ?? "not specified"}`,
16467
+ "",
16468
+ "## Evidence Summary",
16469
+ `- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
16470
+ `- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
16471
+ `- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
16472
+ `- Competitor domains: ${competitors.length}`,
16473
+ "",
16474
+ "## Recommended Use",
16475
+ "Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
16476
+ ].join("\n");
16477
+ await ctx.artifacts.writeText("Brief", "brief.md", brief);
16478
+ await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
16479
+ "# Agent Tasks",
16480
+ "",
16481
+ "- [ ] Read `evidence.json` before writing recommendations.",
16482
+ "- [ ] Compare the target domain against `competitors.csv`.",
16483
+ "- [ ] Use `sources.csv` for citations and source-grounded page sections.",
16484
+ "- [ ] Mark unsupported recommendations as assumptions."
16485
+ ].join("\n"));
16486
+ await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
16487
+ "# Agent Instructions",
16488
+ "",
16489
+ "You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
16490
+ ].join("\n"));
16491
+ const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
16492
+ const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
16493
+ title: "Agent-Ready SEO Packet",
16494
+ subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
16495
+ summary,
16496
+ warnings,
16497
+ tables: [
16498
+ { title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
16499
+ { title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
16500
+ ]
16501
+ }));
16502
+ const status = warnings.length ? "partial" : "succeeded";
16503
+ const counts = { sources: rows.length, competitors: competitors.length };
16504
+ await ctx.artifacts.writeManifest(status, counts, warnings, []);
16505
+ return {
16506
+ state,
16507
+ output: { sources: rows.length, competitors: competitors.length, status },
16508
+ summary: { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors: [], reportPath }
16509
+ };
16094
16510
  }
16095
16511
  }
16096
- if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
16097
- const rows = sourceRows(input, serp, paa);
16098
- const target = normalizeDomain(input.domain);
16099
- const competitors = competitorRows(rows, target);
16100
- const evidence = {
16101
- input,
16102
- serp: serp ?? { status: "skipped" },
16103
- paa: paa ?? { status: "skipped" },
16104
- target: {
16105
- domain: target,
16106
- organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
16107
- citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
16108
- citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
16109
- },
16110
- competitors,
16111
- warnings
16112
- };
16113
- await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
16114
- await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
16115
- await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
16116
- const brief = [
16117
- `# SEO Evidence Brief: ${input.keyword}`,
16118
- "",
16119
- `Location: ${input.location ?? "not specified"}`,
16120
- `Target domain: ${target ?? "not specified"}`,
16121
- "",
16122
- "## Evidence Summary",
16123
- `- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
16124
- `- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
16125
- `- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
16126
- `- Competitor domains: ${competitors.length}`,
16127
- "",
16128
- "## Recommended Use",
16129
- "Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
16130
- ].join("\n");
16131
- await ctx.artifacts.writeText("Brief", "brief.md", brief);
16132
- await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
16133
- "# Agent Tasks",
16134
- "",
16135
- "- [ ] Read `evidence.json` before writing recommendations.",
16136
- "- [ ] Compare the target domain against `competitors.csv`.",
16137
- "- [ ] Use `sources.csv` for citations and source-grounded page sections.",
16138
- "- [ ] Mark unsupported recommendations as assumptions."
16139
- ].join("\n"));
16140
- await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
16141
- "# Agent Instructions",
16142
- "",
16143
- "You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
16144
- ].join("\n"));
16145
- const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
16146
- const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
16147
- title: "Agent-Ready SEO Packet",
16148
- subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
16149
- summary,
16150
- warnings,
16151
- tables: [
16152
- { title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
16153
- { title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
16154
- ]
16155
- }));
16156
- const status = warnings.length ? "partial" : "succeeded";
16157
- const counts = { sources: rows.length, competitors: competitors.length };
16158
- await ctx.artifacts.writeManifest(status, counts, warnings, errors);
16159
- return { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors, reportPath };
16160
- }
16512
+ ]
16161
16513
  };
16162
16514
  }
16163
16515
  });
@@ -16220,6 +16572,7 @@ var init_directory = __esm({
16220
16572
  "use strict";
16221
16573
  import_zod20 = require("zod");
16222
16574
  init_report_renderer();
16575
+ init_schemas3();
16223
16576
  DirectoryWorkflowCliInputSchema = import_zod20.z.object({
16224
16577
  query: import_zod20.z.string().min(1),
16225
16578
  state: import_zod20.z.string().min(2).default("TN"),
@@ -16227,7 +16580,7 @@ var init_directory = __esm({
16227
16580
  maxCities: import_zod20.z.number().int().min(1).max(100).default(25),
16228
16581
  maxResultsPerCity: import_zod20.z.number().int().min(1).max(50).default(20),
16229
16582
  concurrency: import_zod20.z.number().int().min(1).max(5).default(5),
16230
- proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location"),
16583
+ proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
16231
16584
  saveCsv: import_zod20.z.boolean().default(true)
16232
16585
  });
16233
16586
  DIRECTORY_CSV_HEADERS = [
@@ -16344,6 +16697,7 @@ var init_local_competitive_audit = __esm({
16344
16697
  import_zod21 = require("zod");
16345
16698
  init_report_renderer();
16346
16699
  init_directory();
16700
+ init_schemas3();
16347
16701
  LocalCompetitiveAuditInputSchema = import_zod21.z.object({
16348
16702
  query: import_zod21.z.string().min(1),
16349
16703
  state: import_zod21.z.string().min(2).default("TN"),
@@ -16353,7 +16707,7 @@ var init_local_competitive_audit = __esm({
16353
16707
  hydrateTop: import_zod21.z.number().int().min(0).max(10).default(5),
16354
16708
  maxReviews: import_zod21.z.number().int().min(0).max(500).default(50),
16355
16709
  concurrency: import_zod21.z.number().int().min(1).max(5).default(5),
16356
- proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default("location"),
16710
+ proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
16357
16711
  returnPartial: import_zod21.z.boolean().default(true)
16358
16712
  });
16359
16713
  localCompetitiveAuditWorkflowDefinition = {
@@ -16805,6 +17159,7 @@ var init_comparison_briefs = __esm({
16805
17159
  init_report_renderer();
16806
17160
  init_directory();
16807
17161
  init_seo_workflow_utils();
17162
+ init_schemas3();
16808
17163
  ProxyModeSchema = import_zod22.z.enum(["location", "configured", "none"]);
16809
17164
  MapComparisonInputSchema = import_zod22.z.object({
16810
17165
  query: import_zod22.z.string().min(1),
@@ -16816,7 +17171,7 @@ var init_comparison_briefs = __esm({
16816
17171
  hydrateTop: import_zod22.z.number().int().min(0).max(10).default(5),
16817
17172
  maxReviews: import_zod22.z.number().int().min(0).max(500).default(25),
16818
17173
  concurrency: import_zod22.z.number().int().min(1).max(5).default(5),
16819
- proxyMode: ProxyModeSchema.default("location"),
17174
+ proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
16820
17175
  returnPartial: import_zod22.z.boolean().default(true)
16821
17176
  }).refine((input) => input.location || input.state, {
16822
17177
  message: "Either location or state is required for map-comparison"
@@ -17230,22 +17585,44 @@ function workflowDefinition(id) {
17230
17585
  if (!definition) throw new Error(`Unknown workflow "${id}". Available: ${DEFINITIONS.map((def) => def.id).join(", ")}`);
17231
17586
  return definition;
17232
17587
  }
17233
- async function runWorkflow(id, rawInput, options = {}) {
17588
+ function workflowStepCount(id) {
17234
17589
  const definition = workflowDefinition(id);
17235
- const input = definition.inputSchema.parse(rawInput);
17590
+ return definition.steps?.length ?? 1;
17591
+ }
17592
+ function workflowSupportsSteps(id) {
17593
+ const definition = workflowDefinition(id);
17594
+ return Boolean(definition.steps && definition.steps.length > 0);
17595
+ }
17596
+ function resolveApiKey(options) {
17236
17597
  const apiKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim();
17237
17598
  if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
17238
- const apiUrl = options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
17599
+ return apiKey;
17600
+ }
17601
+ function resolveApiUrl(options) {
17602
+ return options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
17603
+ }
17604
+ async function runWorkflow(id, rawInput, options = {}) {
17605
+ const definition = workflowDefinition(id);
17606
+ const input = definition.inputSchema.parse(rawInput);
17607
+ const apiKey = resolveApiKey(options);
17608
+ const apiUrl = resolveApiUrl(options);
17239
17609
  const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
17240
17610
  const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
17611
+ const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client: client2, artifacts, signal: options.signal };
17241
17612
  try {
17242
- return await definition.run(input, {
17243
- runId: artifacts.runId,
17244
- startedAt: artifacts.startedAt,
17245
- client: client2,
17246
- artifacts,
17247
- signal: options.signal
17248
- });
17613
+ if (definition.steps && definition.steps.length > 0) {
17614
+ let state = definition.createState ? definition.createState(input) : {};
17615
+ let summary = null;
17616
+ for (const step of definition.steps) {
17617
+ const outcome = await step.run({ input, state, ctx });
17618
+ state = outcome.state;
17619
+ if (outcome.summary) summary = outcome.summary;
17620
+ }
17621
+ if (!summary) throw new Error(`Workflow "${id}" produced no terminal step summary`);
17622
+ return summary;
17623
+ }
17624
+ if (definition.run) return await definition.run(input, ctx);
17625
+ throw new Error(`Workflow "${id}" has neither steps nor a run() implementation`);
17249
17626
  } catch (err) {
17250
17627
  const message = err instanceof Error ? err.message : String(err);
17251
17628
  await artifacts.writeText("Failure", "summary.md", `# ${definition.title}
@@ -17256,10 +17633,50 @@ ${message}
17256
17633
  throw err;
17257
17634
  }
17258
17635
  }
17259
- var import_zod23, DEFINITIONS;
17636
+ async function runWorkflowStep(id, rawInput, opts) {
17637
+ const definition = workflowDefinition(id);
17638
+ const steps = definition.steps;
17639
+ if (!steps || steps.length === 0) throw new Error(`Workflow "${id}" does not support stepwise execution`);
17640
+ if (opts.stepIndex < 0 || opts.stepIndex >= steps.length) {
17641
+ throw new Error(`Step index ${opts.stepIndex} is out of range for "${id}" (${steps.length} steps)`);
17642
+ }
17643
+ const options = opts.options ?? {};
17644
+ const input = definition.inputSchema.parse(rawInput);
17645
+ const apiKey = resolveApiKey(options);
17646
+ const apiUrl = resolveApiUrl(options);
17647
+ const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
17648
+ const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
17649
+ const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client: client2, artifacts, signal: options.signal };
17650
+ const state = opts.state ?? (definition.createState ? definition.createState(input) : {});
17651
+ const step = steps[opts.stepIndex];
17652
+ const outcome = await step.run({ input, state, ctx });
17653
+ const writtenArtifacts = await Promise.all(
17654
+ artifacts.artifacts.map(async (artifact) => ({
17655
+ ...artifact,
17656
+ content: await (0, import_promises7.readFile)(artifact.path, "utf8").catch(() => "")
17657
+ }))
17658
+ );
17659
+ return {
17660
+ runId: artifacts.runId,
17661
+ workflowId: definition.id,
17662
+ title: definition.title,
17663
+ stepIndex: opts.stepIndex,
17664
+ stepId: step.id,
17665
+ stepTitle: step.title,
17666
+ totalSteps: steps.length,
17667
+ isLast: opts.stepIndex === steps.length - 1,
17668
+ output: outcome.output,
17669
+ state: outcome.state,
17670
+ warnings: outcome.warnings ?? [],
17671
+ summary: outcome.summary ?? null,
17672
+ artifacts: writtenArtifacts
17673
+ };
17674
+ }
17675
+ var import_promises7, import_zod23, DEFINITIONS;
17260
17676
  var init_registry2 = __esm({
17261
17677
  "src/workflows/registry.ts"() {
17262
17678
  "use strict";
17679
+ import_promises7 = require("fs/promises");
17263
17680
  import_zod23 = require("zod");
17264
17681
  init_artifact_writer();
17265
17682
  init_http_client2();
@@ -17328,7 +17745,7 @@ async function readManifestFromSummary(summary) {
17328
17745
  if (!summary.reportPath) return null;
17329
17746
  const manifestPath = summary.reportPath.replace(/report\.html$/, "manifest.json");
17330
17747
  try {
17331
- return JSON.parse(await (0, import_promises7.readFile)(manifestPath, "utf8"));
17748
+ return JSON.parse(await (0, import_promises8.readFile)(manifestPath, "utf8"));
17332
17749
  } catch {
17333
17750
  return null;
17334
17751
  }
@@ -17409,6 +17826,88 @@ async function executeWorkflowRun(input) {
17409
17826
  throw err;
17410
17827
  }
17411
17828
  }
17829
+ async function runWorkflowStepAndPersist(input) {
17830
+ const runId = input.run.id;
17831
+ const workflowId = input.run.workflow_id;
17832
+ await markWorkflowRunRunning(runId);
17833
+ let result;
17834
+ try {
17835
+ result = await runWorkflowStep(workflowId, input.workflowInput, {
17836
+ stepIndex: input.stepIndex,
17837
+ state: input.priorState ?? void 0,
17838
+ options: {
17839
+ apiKey: input.user.api_key,
17840
+ apiUrl: input.apiUrl,
17841
+ outputDir: hostedWorkflowOutputDir(),
17842
+ runId,
17843
+ headers: input.concurrencyLockId ? { "x-mcp-scraper-concurrency-lock": input.concurrencyLockId } : void 0
17844
+ }
17845
+ });
17846
+ } catch (err) {
17847
+ const message = err instanceof Error ? err.message : String(err);
17848
+ await failWorkflowRunRecord(runId, message, { error: message });
17849
+ const failedRun = await getWorkflowRun(runId, input.user.id);
17850
+ if (failedRun) await deliverWorkflowWebhook({ userId: input.user.id, run: failedRun, scheduleId: failedRun.schedule_id, webhookUrl: input.webhookUrl ?? null, summary: null, artifacts: [] });
17851
+ throw err;
17852
+ }
17853
+ if (result.artifacts.length) {
17854
+ await appendWorkflowArtifacts(runId, input.user.id, result.artifacts.map((a) => ({
17855
+ kind: a.kind,
17856
+ label: a.label,
17857
+ path: a.path,
17858
+ bytes: a.bytes,
17859
+ rows: a.rows,
17860
+ content: a.content
17861
+ })));
17862
+ }
17863
+ const logEntry = {
17864
+ stepIndex: result.stepIndex,
17865
+ stepId: result.stepId,
17866
+ stepTitle: result.stepTitle,
17867
+ output: result.output,
17868
+ warnings: result.warnings,
17869
+ at: (/* @__PURE__ */ new Date()).toISOString()
17870
+ };
17871
+ const status = result.isLast ? result.summary?.status ?? "succeeded" : "running";
17872
+ await advanceWorkflowRunStep(runId, {
17873
+ stepIndex: result.stepIndex + 1,
17874
+ totalSteps: result.totalSteps,
17875
+ state: result.state,
17876
+ status,
17877
+ logEntry,
17878
+ appendLog: input.priorLog
17879
+ });
17880
+ if (result.isLast && result.summary) {
17881
+ const manifest = await readManifestFromSummary(result.summary);
17882
+ await completeWorkflowRunRecord(runId, result.summary.status, manifest ?? { summary: result.summary });
17883
+ }
17884
+ const run = await getWorkflowRun(runId, input.user.id);
17885
+ const artifacts = await listWorkflowArtifacts(runId, input.user.id);
17886
+ if (result.isLast && run) {
17887
+ await deliverWorkflowWebhook({ userId: input.user.id, run, scheduleId: run.schedule_id, webhookUrl: input.webhookUrl ?? null, summary: result.summary, artifacts });
17888
+ }
17889
+ const definition = workflowDefinition(workflowId);
17890
+ const nextIndex = result.stepIndex + 1;
17891
+ const nextStep = result.isLast ? null : {
17892
+ index: nextIndex,
17893
+ id: definition.steps?.[nextIndex]?.id ?? null,
17894
+ title: definition.steps?.[nextIndex]?.title ?? null
17895
+ };
17896
+ return {
17897
+ run: run ? exposeRun(input.c, run, artifacts) : null,
17898
+ summary: result.isLast ? result.summary : null,
17899
+ step: {
17900
+ index: result.stepIndex,
17901
+ id: result.stepId,
17902
+ title: result.stepTitle,
17903
+ totalSteps: result.totalSteps,
17904
+ output: result.output,
17905
+ warnings: result.warnings
17906
+ },
17907
+ nextStep,
17908
+ done: result.isLast
17909
+ };
17910
+ }
17412
17911
  async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
17413
17912
  const now = (/* @__PURE__ */ new Date()).toISOString();
17414
17913
  const due = await listDueWorkflowSchedules(now, limit);
@@ -17461,12 +17960,12 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
17461
17960
  }
17462
17961
  return { dispatched: results.length, results };
17463
17962
  }
17464
- var import_node_crypto3, import_promises7, import_hono7, import_zod24, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema;
17963
+ var import_node_crypto3, import_promises8, import_hono7, import_zod24, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
17465
17964
  var init_workflow_routes = __esm({
17466
17965
  "src/api/workflow-routes.ts"() {
17467
17966
  "use strict";
17468
17967
  import_node_crypto3 = require("crypto");
17469
- import_promises7 = require("fs/promises");
17968
+ import_promises8 = require("fs/promises");
17470
17969
  import_hono7 = require("hono");
17471
17970
  import_zod24 = require("zod");
17472
17971
  init_artifact_writer();
@@ -17525,7 +18024,29 @@ var init_workflow_routes = __esm({
17525
18024
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
17526
18025
  let run = null;
17527
18026
  try {
17528
- run = await createWorkflowRun({ userId: user.id, workflowId: parsed.data.workflowId, workflowInput: input });
18027
+ const stepwise = workflowSupportsSteps(parsed.data.workflowId);
18028
+ run = await createWorkflowRun({
18029
+ userId: user.id,
18030
+ workflowId: parsed.data.workflowId,
18031
+ workflowInput: input,
18032
+ webhookUrl,
18033
+ totalSteps: stepwise ? workflowStepCount(parsed.data.workflowId) : 1
18034
+ });
18035
+ if (stepwise) {
18036
+ const response = await runWorkflowStepAndPersist({
18037
+ c,
18038
+ user,
18039
+ run,
18040
+ workflowInput: input,
18041
+ stepIndex: 0,
18042
+ priorState: null,
18043
+ priorLog: [],
18044
+ apiUrl: originFromUrl(c.req.url),
18045
+ webhookUrl,
18046
+ concurrencyLockId: gate.lockId
18047
+ });
18048
+ return c.json(response);
18049
+ }
17529
18050
  const executed = await executeWorkflowRun({
17530
18051
  runId: run.id,
17531
18052
  user,
@@ -17535,7 +18056,7 @@ var init_workflow_routes = __esm({
17535
18056
  webhookUrl,
17536
18057
  concurrencyLockId: gate.lockId
17537
18058
  });
17538
- return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
18059
+ return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary, done: true });
17539
18060
  } catch (err) {
17540
18061
  const failed = run ? await getWorkflowRun(run.id, user.id) : null;
17541
18062
  return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
@@ -17543,6 +18064,50 @@ var init_workflow_routes = __esm({
17543
18064
  await releaseConcurrencyGate(gate.lockId);
17544
18065
  }
17545
18066
  });
18067
+ TERMINAL_RUN_STATUSES = ["succeeded", "partial", "failed", "cancelled"];
18068
+ workflowApp.post("/runs/:id/step", createApiKeyAuth(), async (c) => {
18069
+ const user = c.get("user");
18070
+ const run = await getWorkflowRun(c.req.param("id"), user.id);
18071
+ if (!run) return c.json({ error: "Run not found" }, 404);
18072
+ if (!workflowSupportsSteps(run.workflow_id)) {
18073
+ return c.json({ error: `Workflow "${run.workflow_id}" does not support stepwise execution` }, 400);
18074
+ }
18075
+ const artifactsNow = await listWorkflowArtifacts(run.id, user.id);
18076
+ if (TERMINAL_RUN_STATUSES.includes(run.status)) {
18077
+ return c.json({ run: exposeRun(c, run, artifactsNow), done: true, error: `Run is already ${run.status}` }, 409);
18078
+ }
18079
+ const stepState = await getWorkflowRunStepState(run.id, user.id);
18080
+ const stepIndex = stepState?.stepIndex ?? run.step_index;
18081
+ const totalSteps = workflowStepCount(run.workflow_id);
18082
+ if (stepIndex >= totalSteps) {
18083
+ return c.json({ run: exposeRun(c, run, artifactsNow), done: true, error: "No remaining steps" }, 409);
18084
+ }
18085
+ const gate = await acquireConcurrencyGate(user, "workflow_run", {
18086
+ ttlSeconds: 60 * 60,
18087
+ metadata: { workflowId: run.workflow_id, step: stepIndex }
18088
+ });
18089
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
18090
+ try {
18091
+ const response = await runWorkflowStepAndPersist({
18092
+ c,
18093
+ user,
18094
+ run,
18095
+ workflowInput: run.input,
18096
+ stepIndex,
18097
+ priorState: stepState?.state ?? null,
18098
+ priorLog: stepState?.stepLog ?? run.step_log,
18099
+ apiUrl: originFromUrl(c.req.url),
18100
+ webhookUrl: run.webhook_url,
18101
+ concurrencyLockId: gate.lockId
18102
+ });
18103
+ return c.json(response);
18104
+ } catch (err) {
18105
+ const failed = await getWorkflowRun(run.id, user.id);
18106
+ return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
18107
+ } finally {
18108
+ await releaseConcurrencyGate(gate.lockId);
18109
+ }
18110
+ });
17546
18111
  workflowApp.get("/runs", createApiKeyAuth(), async (c) => {
17547
18112
  const user = c.get("user");
17548
18113
  const runs = await listWorkflowRuns(user.id);
@@ -17561,14 +18126,17 @@ var init_workflow_routes = __esm({
17561
18126
  if (!run) return c.json({ error: "Run not found" }, 404);
17562
18127
  const artifact = await getWorkflowArtifact(c.req.param("artifactId"), user.id);
17563
18128
  if (!artifact || artifact.run_id !== run.id) return c.json({ error: "Artifact not found" }, 404);
18129
+ const filename = artifact.path.split("/").pop() || artifact.id;
18130
+ const headers = {
18131
+ "Content-Type": artifact.content_type,
18132
+ "Content-Disposition": `attachment; filename="${filename}"`
18133
+ };
18134
+ if (artifact.content != null) {
18135
+ return new Response(artifact.content, { headers });
18136
+ }
17564
18137
  try {
17565
- const content = await (0, import_promises7.readFile)(artifact.path);
17566
- return new Response(content, {
17567
- headers: {
17568
- "Content-Type": artifact.content_type,
17569
- "Content-Disposition": `attachment; filename="${artifact.path.split("/").pop() || artifact.id}"`
17570
- }
17571
- });
18138
+ const content = await (0, import_promises8.readFile)(artifact.path);
18139
+ return new Response(content, { headers });
17572
18140
  } catch {
17573
18141
  return c.json({ error: "Artifact file is no longer available" }, 410);
17574
18142
  }
@@ -17856,7 +18424,7 @@ async function capturePageSnapshot(target, options = {}) {
17856
18424
  const textContent = truncateContent(text, maxContentChars);
17857
18425
  const schemaTypes = getSchemaTypes(extraction.schema, extraction.kpo.schemaTypes);
17858
18426
  const metrics = countLinks(extraction.bodyHtml, extraction.url);
17859
- const wordCount = countWords(extraction.bodyMarkdown);
18427
+ const wordCount2 = countWords(extraction.bodyMarkdown);
17860
18428
  const contentSha256 = sha256(extraction.bodyMarkdown);
17861
18429
  const canonicalUrl = extraction.kpo.canonicalUrl ? new URL(extraction.kpo.canonicalUrl, extraction.url).href : null;
17862
18430
  const contentIncluded = Boolean(options.includeContent);
@@ -17910,10 +18478,10 @@ async function capturePageSnapshot(target, options = {}) {
17910
18478
  },
17911
18479
  error: null,
17912
18480
  schemaTypes,
17913
- wordCount,
18481
+ wordCount: wordCount2,
17914
18482
  linkCounts: {
17915
18483
  ...metrics,
17916
- wordCount
18484
+ wordCount: wordCount2
17917
18485
  },
17918
18486
  hashes: {
17919
18487
  htmlSha256: sha256(extraction.bodyHtml),
@@ -19489,7 +20057,7 @@ async function harvest(rawOptions) {
19489
20057
  const signal = getAbortSignal(rawOptions);
19490
20058
  const onAttemptEvent = getAttemptLogSink(rawOptions);
19491
20059
  const requestedProxyMode = raw.proxyMode;
19492
- const proxyMode = requestedProxyMode === "none" ? "none" : requestedProxyMode === "configured" ? "configured" : "location";
20060
+ const proxyMode = requestedProxyMode === "location" || requestedProxyMode === "none" || requestedProxyMode === "configured" ? requestedProxyMode : DEFAULT_PROXY_MODE;
19493
20061
  const kernelApiKey = typeof raw.kernelApiKey === "string" ? raw.kernelApiKey.trim() : browserServiceApiKey();
19494
20062
  const configuredKernelProxyId = typeof raw.kernelProxyId === "string" ? raw.kernelProxyId.trim() : browserServiceProxyId();
19495
20063
  const proxyOpts = {
@@ -19863,6 +20431,7 @@ var init_schemas4 = __esm({
19863
20431
  "src/serp-intelligence/schemas.ts"() {
19864
20432
  "use strict";
19865
20433
  import_zod25 = require("zod");
20434
+ init_schemas3();
19866
20435
  SerpIntelligenceDeviceValues = ["desktop", "mobile"];
19867
20436
  SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
19868
20437
  SerpIntelligenceAttemptOutcomeValues = [
@@ -19903,7 +20472,7 @@ var init_schemas4 = __esm({
19903
20472
  gl: import_zod25.z.string().trim().length(2).default("us"),
19904
20473
  hl: import_zod25.z.string().trim().length(2).default("en"),
19905
20474
  device: import_zod25.z.enum(SerpIntelligenceDeviceValues).default("desktop"),
19906
- proxyMode: import_zod25.z.enum(SerpIntelligenceProxyModeValues).default("location"),
20475
+ proxyMode: import_zod25.z.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
19907
20476
  proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional(),
19908
20477
  pages: import_zod25.z.number().int().min(1).max(2).default(1),
19909
20478
  debug: import_zod25.z.boolean().default(false),
@@ -20433,16 +21002,17 @@ var PACKAGE_VERSION;
20433
21002
  var init_version = __esm({
20434
21003
  "src/version.ts"() {
20435
21004
  "use strict";
20436
- PACKAGE_VERSION = "0.2.19";
21005
+ PACKAGE_VERSION = "0.2.20";
20437
21006
  }
20438
21007
  });
20439
21008
 
20440
21009
  // src/mcp/mcp-tool-schemas.ts
20441
- var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
21010
+ var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
20442
21011
  var init_mcp_tool_schemas = __esm({
20443
21012
  "src/mcp/mcp-tool-schemas.ts"() {
20444
21013
  "use strict";
20445
21014
  import_zod26 = require("zod");
21015
+ init_schemas3();
20446
21016
  HarvestPaaInputSchema = {
20447
21017
  query: import_zod26.z.string().min(1).describe('Core search topic only. If the user says "best hvac company in Denver CO", use query="best hvac company" and location="Denver, CO". Do not include the location in query when it can be separated.'),
20448
21018
  location: import_zod26.z.string().optional().describe('City, region, or country for geo-targeted results, inferred from the user request when present, e.g. "Denver, CO", "Tokyo, Japan", "London, UK".'),
@@ -20450,8 +21020,8 @@ var init_mcp_tool_schemas = __esm({
20450
21020
  gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
20451
21021
  hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
20452
21022
  device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
20453
- proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
20454
- proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
21023
+ proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
21024
+ proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
20455
21025
  debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
20456
21026
  };
20457
21027
  ExtractUrlInputSchema = {
@@ -20475,22 +21045,23 @@ var init_mcp_tool_schemas = __esm({
20475
21045
  mode: import_zod26.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
20476
21046
  query: import_zod26.z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
20477
21047
  channelHandle: import_zod26.z.string().optional().describe("YouTube channel handle, channel ID, or URL. Examples: @mkbhd, UC..., https://youtube.com/@mkbhd."),
20478
- maxVideos: import_zod26.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50. Increase when user asks for full channel/history.")
21048
+ maxVideos: import_zod26.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50, maximum 500. Use 10-25 for quick topic discovery, 50 for normal channel/search harvests, and larger values only when the user asks for full channel/history because large responses consume more context.")
20479
21049
  };
20480
21050
  YoutubeTranscribeInputSchema = {
20481
- videoId: import_zod26.z.string().min(1).describe("YouTube video ID, e.g. dQw4w9WgXcQ")
21051
+ videoId: import_zod26.z.string().min(1).optional().describe("YouTube video ID, e.g. dQw4w9WgXcQ. Use only an ID returned by youtube_harvest or visible in a YouTube URL; do not invent one."),
21052
+ url: import_zod26.z.string().url().optional().describe("Full YouTube URL, e.g. https://www.youtube.com/watch?v=dQw4w9WgXcQ or https://youtu.be/dQw4w9WgXcQ. Use this when the user pasted a URL instead of an ID. Provide videoId or url.")
20482
21053
  };
20483
21054
  FacebookPageIntelInputSchema = {
20484
- pageId: import_zod26.z.string().optional(),
20485
- libraryId: import_zod26.z.string().optional(),
21055
+ pageId: import_zod26.z.string().optional().describe("Facebook advertiser/page ID. Use only a pageId returned by facebook_ad_search/facebook_page_intel or copied from Facebook Ad Library; do not construct one yourself."),
21056
+ libraryId: import_zod26.z.string().optional().describe("Facebook Ad Library archive ID for a known ad or advertiser sample. Use a libraryId returned by facebook_ad_search, or a libraryId/adArchiveId visible in Ad Library."),
20486
21057
  query: import_zod26.z.string().optional().describe("Advertiser or brand name when pageId/libraryId is not known. One of pageId, libraryId, or query is required."),
20487
- maxAds: import_zod26.z.number().int().min(1).max(200).default(50),
20488
- country: import_zod26.z.string().length(2).default("US")
21058
+ maxAds: import_zod26.z.number().int().min(1).max(200).default(50).describe("Maximum ads to inspect. Default 50, maximum 200. Prefer 25-50 for focused advertiser scans; use 100-200 only when the user asks for a broad ad archive sweep."),
21059
+ country: import_zod26.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU. Infer from the user request when they name a country.")
20489
21060
  };
20490
21061
  FacebookAdSearchInputSchema = {
20491
21062
  query: import_zod26.z.string().min(1).describe("Advertiser, brand, competitor, niche, or keyword to search in Facebook Ad Library."),
20492
- country: import_zod26.z.string().length(2).default("US"),
20493
- maxResults: import_zod26.z.number().int().min(1).max(20).default(10)
21063
+ country: import_zod26.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU."),
21064
+ maxResults: import_zod26.z.number().int().min(1).max(20).default(10).describe("Maximum advertisers to return. Default 10, maximum 20. Prefer tighter search terms over maxing this out.")
20494
21065
  };
20495
21066
  FacebookAdTranscribeInputSchema = {
20496
21067
  videoUrl: import_zod26.z.string().url().describe("Direct Facebook CDN video URL from a facebook_page_intel ad result. Do not pass a public Facebook reel/post/share URL here; use facebook_video_transcribe for organic Facebook URLs.")
@@ -20504,8 +21075,8 @@ var init_mcp_tool_schemas = __esm({
20504
21075
  location: import_zod26.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
20505
21076
  gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location."),
20506
21077
  hl: import_zod26.z.string().length(2).default("en").describe("Language inferred from user request."),
20507
- includeReviews: import_zod26.z.boolean().default(false).describe("Whether to fetch individual review cards"),
20508
- maxReviews: import_zod26.z.number().int().min(1).max(500).default(50).describe("Max review cards to return (requires includeReviews: true)")
21078
+ includeReviews: import_zod26.z.boolean().default(false).describe("Whether to fetch individual review cards. Use true when the user asks for reviews, customer pain, complaints, praise, themes, or review evidence."),
21079
+ maxReviews: import_zod26.z.number().int().min(1).max(500).default(50).describe("Max review cards to return when includeReviews is true. Default 50, maximum 500. Use 50 for normal review analysis and larger values only for deep review mining.")
20509
21080
  };
20510
21081
  MapsSearchInputSchema = {
20511
21082
  query: import_zod26.z.string().min(1).describe('Business category, niche, keyword, or search term. If the user says "roofers in Denver CO", use query="roofers" and location="Denver, CO". Do not put the location here when it can be separated.'),
@@ -20513,7 +21084,7 @@ var init_mcp_tool_schemas = __esm({
20513
21084
  gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location."),
20514
21085
  hl: import_zod26.z.string().length(2).default("en").describe("Language inferred from user request."),
20515
21086
  maxResults: import_zod26.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
20516
- proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state Maps searches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts. Use configured for the server proxy ID, and none only for local direct-network debugging."),
21087
+ proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode. Maps defaults to location so local market searches get city/state residential proxy targeting and rotation. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
20517
21088
  proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
20518
21089
  debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
20519
21090
  };
@@ -20528,8 +21099,8 @@ var init_mcp_tool_schemas = __esm({
20528
21099
  includeZipGroups: import_zod26.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
20529
21100
  usZipsCsvPath: import_zod26.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
20530
21101
  saveCsv: import_zod26.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
20531
- proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode for every city Maps search. Use location by default for US city/state batches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts per city. Use configured for the server proxy ID, and none only for local direct-network debugging."),
20532
- proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its Lead Magician ZIP group or city/state location."),
21102
+ proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode for every city Maps search. Maps workflows default to location so each city can use city/state or ZIP-group residential proxy targeting. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
21103
+ proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its ZIP group or city/state location."),
20533
21104
  debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
20534
21105
  };
20535
21106
  RankTrackerModeSchema = import_zod26.z.enum(["maps", "organic", "ai_overview", "paa"]);
@@ -20843,8 +21414,11 @@ var init_mcp_tool_schemas = __esm({
20843
21414
  advertiserCount: import_zod26.z.number().int().min(0),
20844
21415
  advertisers: import_zod26.z.array(import_zod26.z.object({
20845
21416
  name: NullableString,
21417
+ pageId: NullableString,
21418
+ pageUrl: NullableString,
20846
21419
  adCount: import_zod26.z.number().int().nullable(),
20847
- libraryId: NullableString
21420
+ libraryId: NullableString,
21421
+ sampleLibraryId: NullableString
20848
21422
  }))
20849
21423
  };
20850
21424
  FacebookPageIntelOutputSchema = {
@@ -20857,10 +21431,15 @@ var init_mcp_tool_schemas = __esm({
20857
21431
  libraryId: NullableString,
20858
21432
  status: NullableString,
20859
21433
  creativeType: NullableString,
21434
+ primaryText: NullableString,
20860
21435
  headline: NullableString,
20861
21436
  cta: NullableString,
20862
21437
  startDate: NullableString,
21438
+ landingUrl: NullableString,
21439
+ domain: NullableString,
20863
21440
  videoUrl: NullableString,
21441
+ imageUrl: NullableString,
21442
+ videoPoster: NullableString,
20864
21443
  variations: import_zod26.z.number().int().nullable()
20865
21444
  }))
20866
21445
  };
@@ -20882,6 +21461,58 @@ var init_mcp_tool_schemas = __esm({
20882
21461
  text: import_zod26.z.string()
20883
21462
  }))
20884
21463
  };
21464
+ TranscriptChunkOutput = import_zod26.z.object({
21465
+ startSec: import_zod26.z.number(),
21466
+ endSec: import_zod26.z.number(),
21467
+ text: import_zod26.z.string()
21468
+ });
21469
+ YoutubeTranscribeOutputSchema = {
21470
+ videoId: NullableString,
21471
+ url: NullableString,
21472
+ wordCount: import_zod26.z.number().int().min(0),
21473
+ chunkCount: import_zod26.z.number().int().min(0),
21474
+ durationMs: import_zod26.z.number().nullable(),
21475
+ transcriptText: import_zod26.z.string(),
21476
+ chunks: import_zod26.z.array(TranscriptChunkOutput),
21477
+ resolvedInputs: import_zod26.z.object({
21478
+ videoId: NullableString,
21479
+ url: NullableString
21480
+ })
21481
+ };
21482
+ FacebookAdTranscribeOutputSchema = {
21483
+ videoUrl: import_zod26.z.string().url(),
21484
+ wordCount: import_zod26.z.number().int().min(0),
21485
+ chunkCount: import_zod26.z.number().int().min(0),
21486
+ durationMs: import_zod26.z.number().nullable(),
21487
+ transcriptText: import_zod26.z.string(),
21488
+ chunks: import_zod26.z.array(TranscriptChunkOutput),
21489
+ resolvedInputs: import_zod26.z.object({
21490
+ videoUrl: import_zod26.z.string().url()
21491
+ })
21492
+ };
21493
+ CaptureSerpSnapshotOutputSchema = {
21494
+ schemaVersion: import_zod26.z.literal("serp-intelligence.capture.v1"),
21495
+ status: import_zod26.z.string(),
21496
+ query: NullableString,
21497
+ location: NullableString,
21498
+ capturedAt: NullableString,
21499
+ resultCount: import_zod26.z.number().int().min(0).nullable(),
21500
+ snapshotId: NullableString,
21501
+ resolvedInputs: import_zod26.z.record(import_zod26.z.unknown()),
21502
+ artifacts: import_zod26.z.array(import_zod26.z.record(import_zod26.z.unknown())),
21503
+ diagnostics: import_zod26.z.record(import_zod26.z.unknown()).nullable(),
21504
+ providerPayload: import_zod26.z.record(import_zod26.z.unknown())
21505
+ };
21506
+ CaptureSerpPageSnapshotsOutputSchema = {
21507
+ schemaVersion: import_zod26.z.literal("serp-intelligence.page-snapshots.v1"),
21508
+ status: import_zod26.z.string(),
21509
+ count: import_zod26.z.number().int().min(0),
21510
+ failedCount: import_zod26.z.number().int().min(0),
21511
+ captures: import_zod26.z.array(import_zod26.z.record(import_zod26.z.unknown())),
21512
+ resolvedInputs: import_zod26.z.record(import_zod26.z.unknown()),
21513
+ diagnostics: import_zod26.z.record(import_zod26.z.unknown()).nullable(),
21514
+ providerPayload: import_zod26.z.record(import_zod26.z.unknown())
21515
+ };
20885
21516
  CreditsInfoInputSchema = {
20886
21517
  item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
20887
21518
  includeLedger: import_zod26.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
@@ -20909,16 +21540,19 @@ var init_mcp_tool_schemas = __esm({
20909
21540
  maxSuggestions: import_zod26.z.number().int().min(1).max(8).default(3).describe("Number of matching workflow recipes to return.")
20910
21541
  };
20911
21542
  WorkflowRunInputSchema = {
20912
- workflowId: WorkflowIdSchema2.describe("Workflow to run. Use workflow_list or workflow_suggest first when unsure."),
21543
+ workflowId: WorkflowIdSchema2.describe("Workflow to run: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. Use only these values; call workflow_list or workflow_suggest first when unsure."),
20913
21544
  input: import_zod26.z.record(import_zod26.z.unknown()).default({}).describe("Workflow-specific input object. Examples: agent-packet uses {keyword, domain?, location?, maxQuestions?}; local-competitive-audit uses {query, state, minPopulation?, maxCities?, maxResultsPerCity?, hydrateTop?, maxReviews?}; serp-comparison uses {keyword, domain?, url?, location?, extractTop?}."),
20914
21545
  webhookUrl: import_zod26.z.string().url().optional().describe("Optional HTTPS webhook to receive the completed hosted workflow run event.")
20915
21546
  };
21547
+ WorkflowStepInputSchema = {
21548
+ runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself. Advances the run by exactly one step (one logical leg, e.g. one live harvest).")
21549
+ };
20916
21550
  WorkflowStatusInputSchema = {
20917
- runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run.")
21551
+ runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself.")
20918
21552
  };
20919
21553
  WorkflowArtifactReadInputSchema = {
20920
- runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run or workflow_status."),
20921
- artifactId: import_zod26.z.string().min(1).describe("Artifact id from the run artifact list."),
21554
+ runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself."),
21555
+ artifactId: import_zod26.z.string().min(1).describe("Artifact id from the run artifact list returned by workflow_run, workflow_step, or workflow_status. Use only a returned artifactId; do not construct one yourself."),
20922
21556
  maxBytes: import_zod26.z.number().int().min(1e3).max(1e6).default(2e5).describe("Maximum bytes of artifact text to return inline. Use lower values for large CSV/JSON artifacts; call again with the downloadUrl if needed outside MCP.")
20923
21557
  };
20924
21558
  WorkflowRecipeOutput = import_zod26.z.object({
@@ -20951,6 +21585,18 @@ var init_mcp_tool_schemas = __esm({
20951
21585
  input: import_zod26.z.record(import_zod26.z.unknown()),
20952
21586
  run: import_zod26.z.record(import_zod26.z.unknown()).optional(),
20953
21587
  summary: import_zod26.z.record(import_zod26.z.unknown()).optional(),
21588
+ step: import_zod26.z.record(import_zod26.z.unknown()).optional(),
21589
+ nextStep: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
21590
+ done: import_zod26.z.boolean().optional(),
21591
+ artifacts: import_zod26.z.array(WorkflowArtifactOutput)
21592
+ };
21593
+ WorkflowStepOutputSchema = {
21594
+ runId: import_zod26.z.string(),
21595
+ run: import_zod26.z.record(import_zod26.z.unknown()).optional(),
21596
+ summary: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
21597
+ step: import_zod26.z.record(import_zod26.z.unknown()).optional(),
21598
+ nextStep: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
21599
+ done: import_zod26.z.boolean(),
20954
21600
  artifacts: import_zod26.z.array(WorkflowArtifactOutput)
20955
21601
  };
20956
21602
  WorkflowStatusOutputSchema = {
@@ -20971,8 +21617,8 @@ var init_mcp_tool_schemas = __esm({
20971
21617
  gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
20972
21618
  hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from user request."),
20973
21619
  device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
20974
- proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
20975
- proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
21620
+ proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general SERP success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
21621
+ proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
20976
21622
  debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
20977
21623
  pages: import_zod26.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
20978
21624
  };
@@ -20982,8 +21628,8 @@ var init_mcp_tool_schemas = __esm({
20982
21628
  gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
20983
21629
  hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from the user request."),
20984
21630
  device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
20985
- proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default("location").describe("Proxy behavior for capture. Use location for localized US residential evidence; it creates a fresh proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static residential proxy, and none only for direct-network debugging."),
20986
- proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting when a precise city-center or ZIP proxy is needed. With proxyMode location this ZIP is used for each fresh proxy attempt."),
21631
+ proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy behavior for capture. Default configured uses the service proxy without city/ZIP targeting for the highest general capture success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
21632
+ proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when a precise city-center or ZIP proxy is needed."),
20987
21633
  pages: import_zod26.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
20988
21634
  debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
20989
21635
  includePageSnapshots: import_zod26.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs through the same product capture path."),
@@ -21402,70 +22048,72 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
21402
22048
  if (savesReports) registerSavedReportResources(server);
21403
22049
  server.registerTool("harvest_paa", {
21404
22050
  title: "Google PAA + SERP Harvest",
21405
- description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). For US local SERPs, leave proxyMode as location so the service uses fresh residential proxy IDs across retries, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear or the market evidence is wrong. Use maxQuestions 30 normally, 100-200 for "full", "deep", "all", or comprehensive research. Deep harvests above 100 questions can run for several minutes with no interim progress \u2014 warn the user before starting one and keep maxQuestions at or below 100 unless they explicitly want a deep harvest. Credits are charged by extracted question; unused request hold is refunded.'),
22051
+ description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use maxQuestions 30 normally, 100-200 for "full", "deep", "all", or comprehensive research. Deep harvests above 100 questions can run for several minutes with no interim progress \u2014 warn the user before starting one and keep maxQuestions at or below 100 unless they explicitly want a deep harvest. Credits are charged by extracted question; unused request hold is refunded.'),
21406
22052
  inputSchema: HarvestPaaInputSchema,
21407
22053
  outputSchema: HarvestPaaOutputSchema,
21408
22054
  annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
21409
22055
  }, async (input) => formatHarvestPaa(await executor.harvestPaa(input), input));
21410
22056
  server.registerTool("search_serp", {
21411
22057
  title: "Google SERP Lookup",
21412
- description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. For US city/state rankings, keep proxyMode as location and pass proxyZip when a city-center ZIP is known; location mode uses fresh residential proxy IDs, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear, the proxy tunnel fails, or location evidence is wrong."),
22058
+ description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting. Use proxyMode location and proxyZip only when the user explicitly needs city/ZIP-targeted residential proxy evidence."),
21413
22059
  inputSchema: SearchSerpInputSchema,
21414
22060
  outputSchema: SearchSerpOutputSchema,
21415
22061
  annotations: liveWebToolAnnotations("Google SERP Lookup")
21416
22062
  }, async (input) => formatSearchSerp(await executor.searchSerp(input), input));
21417
22063
  server.registerTool("extract_url", {
21418
22064
  title: "Single URL Extract",
21419
- description: withReportNote("Extract structured data from one public URL: page content as Markdown, heading structure, JSON-LD schema, entity details, NAP score, metadata, and missing schema fields. Use when the user provides a single URL or asks to inspect/scrape one page."),
22065
+ description: withReportNote("Extract structured data from one public URL when the user provides one page, asks to inspect/scrape a page, or needs page content, schema, headings, metadata, screenshots, branding, or media assets. Returns structured page fields plus artifact handles for saved reports/screenshots/media when requested. Use map_site_urls before extracting a site inventory; use extract_site for multi-page crawling."),
21420
22066
  inputSchema: ExtractUrlInputSchema,
21421
22067
  outputSchema: ExtractUrlOutputSchema,
21422
22068
  annotations: liveWebToolAnnotations("Single URL Extract")
21423
22069
  }, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
21424
22070
  server.registerTool("map_site_urls", {
21425
22071
  title: "Site URL Map",
21426
- description: withReportNote("Map/crawl a public website to build a URL inventory with HTTP status codes, broken links, redirects, and site scope. Use before extract_site for audits or when the user asks for a sitemap/URL inventory."),
22072
+ description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
21427
22073
  inputSchema: MapSiteUrlsInputSchema,
21428
22074
  outputSchema: MapSiteUrlsOutputSchema,
21429
22075
  annotations: liveWebToolAnnotations("Site URL Map")
21430
22076
  }, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
21431
22077
  server.registerTool("extract_site", {
21432
22078
  title: "Multi-Page Site Extract",
21433
- description: withReportNote("Run multi-page extraction across a public website. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use for website audits, competitor audits, and full-site extraction."),
22079
+ description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
21434
22080
  inputSchema: ExtractSiteInputSchema,
21435
22081
  outputSchema: ExtractSiteOutputSchema,
21436
22082
  annotations: liveWebToolAnnotations("Multi-Page Site Extract")
21437
22083
  }, async (input) => formatExtractSite(await executor.extractSite(input), input));
21438
22084
  server.registerTool("youtube_harvest", {
21439
22085
  title: "YouTube Video Harvest",
21440
- description: withReportNote('Harvest YouTube video metadata by search query or channel handle/ID/URL. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, dates, durations, URLs, thumbnails, and videoIds for follow-up transcription.'),
22086
+ description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
21441
22087
  inputSchema: YoutubeHarvestInputSchema,
21442
22088
  outputSchema: YoutubeHarvestOutputSchema,
21443
22089
  annotations: liveWebToolAnnotations("YouTube Video Harvest")
21444
22090
  }, async (input) => formatYoutubeHarvest(await executor.youtubeHarvest(input), input));
21445
22091
  server.registerTool("youtube_transcribe", {
21446
22092
  title: "YouTube Transcription",
21447
- description: withReportNote("Fetch and transcribe captions from a YouTube video. Returns full transcript, timestamped chunks, and word count. Pass a videoId from youtube_harvest results or infer it from a YouTube URL if the user provided one."),
22093
+ description: withReportNote("Fetch and transcribe captions from a YouTube video. Use this when the user asks what was said in a YouTube video, wants claims/offers/lessons extracted from a video, or provides a YouTube URL for transcript work. Returns full transcript, timestamped chunks, word count, and resolvedInputs. Pass videoId from youtube_harvest results, or pass url when the user pasted a YouTube URL. Use youtube_harvest first when you need to discover videos by topic/channel."),
21448
22094
  inputSchema: YoutubeTranscribeInputSchema,
22095
+ outputSchema: YoutubeTranscribeOutputSchema,
21449
22096
  annotations: liveWebToolAnnotations("YouTube Transcription")
21450
22097
  }, async (input) => formatYoutubeTranscribe(await executor.youtubeTranscribe(input), input));
21451
22098
  server.registerTool("facebook_page_intel", {
21452
22099
  title: "Facebook Advertiser Ad Intel",
21453
- description: withReportNote("Harvest ads from a Facebook advertiser. Returns ad copy, headlines, CTAs, creative type, status, landing URLs, and direct ad video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name as query. Use after facebook_ad_search when possible. For normal public Facebook reels/posts/watch/share URLs, use facebook_video_transcribe instead."),
22100
+ description: withReportNote("Harvest ads from a Facebook advertiser when the user wants current ad copy, creative angles, CTAs, video URLs, or competitive ad intelligence for one brand/page. Returns ad copy, headlines, CTAs, creative type, status, landing URLs, and direct ad video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name as query. Use facebook_ad_search first when the advertiser handle is unknown. For normal public Facebook reels/posts/watch/share URLs, use facebook_video_transcribe instead."),
21454
22101
  inputSchema: FacebookPageIntelInputSchema,
21455
22102
  outputSchema: FacebookPageIntelOutputSchema,
21456
22103
  annotations: liveWebToolAnnotations("Facebook Advertiser Ad Intel")
21457
22104
  }, async (input) => formatFacebookPageIntel(await executor.facebookPageIntel(input), input));
21458
22105
  server.registerTool("facebook_ad_search", {
21459
22106
  title: "Facebook Ad Library Search",
21460
- description: withReportNote("Search Facebook Ad Library by brand, advertiser, competitor, niche, or keyword. Returns advertisers with ad counts and library IDs. Use to discover competitors, then pass libraryId to facebook_page_intel."),
22107
+ description: withReportNote("Search Facebook Ad Library when the user wants to find advertisers by brand, competitor, niche, or keyword. Returns advertisers with ad counts and library IDs. Use this to discover competitor/page handles, then pass libraryId or pageId to facebook_page_intel for ad details."),
21461
22108
  inputSchema: FacebookAdSearchInputSchema,
21462
22109
  outputSchema: FacebookAdSearchOutputSchema,
21463
22110
  annotations: liveWebToolAnnotations("Facebook Ad Library Search")
21464
22111
  }, async (input) => formatFacebookAdSearch(await executor.facebookAdSearch(input), input));
21465
22112
  server.registerTool("facebook_ad_transcribe", {
21466
22113
  title: "Facebook Ad Transcription",
21467
- description: "Transcribe audio from a Facebook ad video CDN URL. Returns full transcript and timestamped chunks. Use only with the direct videoUrl value from facebook_page_intel results, not public Facebook post/reel/share URLs.",
22114
+ description: "Transcribe audio from a Facebook ad video CDN URL. Use this when facebook_page_intel returned a direct videoUrl and the user asks what the ad says, what claims it makes, or wants ad-message extraction. Returns full transcript, timestamped chunks, word count, and resolvedInputs. Use only with the direct videoUrl value from facebook_page_intel results; do not pass public Facebook post/reel/share URLs. Use facebook_video_transcribe for organic Facebook URLs, and facebook_page_intel first when you only have a brand/page/ad library handle.",
21468
22115
  inputSchema: FacebookAdTranscribeInputSchema,
22116
+ outputSchema: FacebookAdTranscribeOutputSchema,
21469
22117
  annotations: liveWebToolAnnotations("Facebook Ad Transcription")
21470
22118
  }, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
21471
22119
  server.registerTool("facebook_video_transcribe", {
@@ -21484,14 +22132,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
21484
22132
  }, async (input) => formatMapsPlaceIntel(await executor.mapsPlaceIntel(input), input));
21485
22133
  server.registerTool("maps_search", {
21486
22134
  title: "Google Maps Business Search",
21487
- description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." For US city/state Maps searches, keep proxyMode as location so the browser service creates a fresh residential proxy ID and browser session for the market; retryable failures rotate to a new proxy and new browser session for up to 5 attempts. Pass proxyZip only when a specific ZIP or city-center ZIP is known. Returns up to 50 candidates with names, place URLs, CIDs when available, ratings, review counts, profile metadata, and sanitized attempt telemetry. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.'),
22135
+ description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." Maps is the default location-targeted surface: omit proxyMode for normal Maps use so the service creates residential proxy evidence for the requested market and rotates on retryable failures. Pass proxyZip only when a specific ZIP or city-center ZIP is known. Use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting. Returns up to 50 candidates with names, place URLs, CIDs when available, ratings, review counts, profile metadata, and sanitized attempt telemetry. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.'),
21488
22136
  inputSchema: MapsSearchInputSchema,
21489
22137
  outputSchema: MapsSearchOutputSchema,
21490
22138
  annotations: liveWebToolAnnotations("Google Maps Business Search")
21491
22139
  }, async (input) => formatMapsSearch(await executor.mapsSearch(input), input));
21492
22140
  server.registerTool("directory_workflow", {
21493
22141
  title: "Directory Workflow: Markets + Maps",
21494
- description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Use concurrency up to 5 for parallel city sessions. Keep proxyMode as location so each city search creates fresh residential proxy IDs and browser sessions; retryable city failures rotate to a new proxy and new browser session for up to 5 attempts. Saved CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. Structured city results include sanitized attempt telemetry. Use maps_place_intel only when a selected profile needs deeper review topics, profile review count confirmation, or review cards. For local Lead Magician ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath only in local/test mode.'),
22142
+ description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Maps workflows default to location-targeted proxying so each city can use city/state or ZIP-group residential proxy evidence; use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting. Saved CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. Structured city results include sanitized attempt telemetry. Use maps_place_intel only when a selected profile needs deeper review topics, profile review count confirmation, or review cards. For local Lead Magician ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath only in local/test mode.'),
21495
22143
  inputSchema: DirectoryWorkflowInputSchema,
21496
22144
  outputSchema: DirectoryWorkflowOutputSchema,
21497
22145
  annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
@@ -21512,21 +22160,28 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
21512
22160
  }, async (input) => formatWorkflowSuggest(input));
21513
22161
  server.registerTool("workflow_run", {
21514
22162
  title: "Run Workflow",
21515
- description: withReportNote("Run a higher-level MCP Scraper workflow and return the run id, summary, and artifacts. Use after workflow_suggest or workflow_list. Runnable workflow ids: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. This is the main MCP tool for executing market analysis, ICP evidence packets, local competitive audits, Maps/SERP comparisons, content gap briefs, and AI Overview language guidance."),
22163
+ description: withReportNote("Start a higher-level MCP Scraper workflow. Use after workflow_suggest or workflow_list. Runnable workflow ids: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. This is the main MCP tool for market analysis, ICP evidence packets, local competitive audits, Maps/SERP comparisons, content gap briefs, and AI Overview language guidance. Stepwise workflows (e.g. agent-packet) run ONE leg per call and return runId, the step output, and nextStep \u2014 when nextStep is present, call workflow_step with the runId to run the next leg, and keep calling it until done is true. This keeps each call short instead of one long blocking run, so report each step result to the user as it arrives."),
21516
22164
  inputSchema: WorkflowRunInputSchema,
21517
22165
  outputSchema: WorkflowRunOutputSchema,
21518
22166
  annotations: liveWebToolAnnotations("Run Workflow")
21519
22167
  }, async (input) => formatWorkflowRun(await executor.workflowRun(input), input));
22168
+ server.registerTool("workflow_step", {
22169
+ title: "Advance Workflow Step",
22170
+ description: withReportNote("Run the next leg of a stepwise MCP Scraper workflow started with workflow_run. Pass the runId. Each call executes exactly one logical step (typically one live harvest), persists that step's artifacts, and returns the step output plus nextStep. Keep calling workflow_step with the same runId until done is true, reporting each step result to the user as it lands. Use this instead of waiting on one long workflow call \u2014 it avoids client timeouts on long multi-step jobs."),
22171
+ inputSchema: WorkflowStepInputSchema,
22172
+ outputSchema: WorkflowStepOutputSchema,
22173
+ annotations: liveWebToolAnnotations("Advance Workflow Step")
22174
+ }, async (input) => formatWorkflowStep(await executor.workflowStep(input), input));
21520
22175
  server.registerTool("workflow_status", {
21521
22176
  title: "Workflow Status",
21522
- description: "Fetch a hosted workflow run by id and list its current status and artifacts. Use after workflow_run when the model needs to re-open a run, inspect artifact ids, or recover from a long workflow.",
22177
+ description: "Fetch a hosted workflow run by id and list its current status and artifacts. Use when a workflow may still be running, when the model needs to re-open a run, inspect artifact ids, recover from a long workflow, or decide whether to call workflow_step or workflow_artifact_read next. Use only a runId returned by workflow_run/workflow_step/workflow_status; do not construct one yourself.",
21523
22178
  inputSchema: WorkflowStatusInputSchema,
21524
22179
  outputSchema: WorkflowStatusOutputSchema,
21525
22180
  annotations: liveWebToolAnnotations("Workflow Status")
21526
22181
  }, async (input) => formatWorkflowStatus(await executor.workflowStatus(input), input));
21527
22182
  server.registerTool("workflow_artifact_read", {
21528
22183
  title: "Read Workflow Artifact",
21529
- description: "Read a workflow artifact back into MCP context by run id and artifact id. Use this before writing final deliverables so the answer is grounded in generated evidence.json, CSVs, Markdown briefs, reports, and task files instead of memory. Use maxBytes to limit large CSV/JSON artifacts.",
22184
+ description: "Read a workflow artifact back into MCP context by run id and artifact id. Use this before writing final deliverables so the answer is grounded in generated evidence.json, CSVs, Markdown briefs, reports, and task files instead of memory. Use workflow_status first when artifact ids are unknown. Use only artifactId values returned by workflow_run/workflow_step/workflow_status; do not construct one yourself. Use maxBytes to limit large CSV/JSON artifacts.",
21530
22185
  inputSchema: WorkflowArtifactReadInputSchema,
21531
22186
  outputSchema: WorkflowArtifactReadOutputSchema,
21532
22187
  annotations: liveWebToolAnnotations("Read Workflow Artifact")
@@ -21568,6 +22223,27 @@ var init_paa_mcp_server = __esm({
21568
22223
  });
21569
22224
 
21570
22225
  // src/mcp/http-mcp-tool-executor.ts
22226
+ function youtubeVideoIdFromUrl(url) {
22227
+ if (!url) return null;
22228
+ try {
22229
+ const parsed = new URL(url);
22230
+ const host = parsed.hostname.replace(/^www\./, "").replace(/^m\./, "");
22231
+ if (host === "youtu.be") {
22232
+ const id = parsed.pathname.split("/").filter(Boolean)[0];
22233
+ return id || null;
22234
+ }
22235
+ if (host === "youtube.com" || host === "music.youtube.com") {
22236
+ const watchId = parsed.searchParams.get("v");
22237
+ if (watchId) return watchId;
22238
+ const parts = parsed.pathname.split("/").filter(Boolean);
22239
+ const markerIndex = parts.findIndex((part) => ["shorts", "embed", "live"].includes(part));
22240
+ if (markerIndex >= 0 && parts[markerIndex + 1]) return parts[markerIndex + 1];
22241
+ }
22242
+ } catch {
22243
+ return null;
22244
+ }
22245
+ return null;
22246
+ }
21571
22247
  var HttpMcpToolExecutor;
21572
22248
  var init_http_mcp_tool_executor = __esm({
21573
22249
  "src/mcp/http-mcp-tool-executor.ts"() {
@@ -21698,7 +22374,21 @@ var init_http_mcp_tool_executor = __esm({
21698
22374
  return this.call("/youtube/harvest", input);
21699
22375
  }
21700
22376
  youtubeTranscribe(input) {
21701
- return this.call("/youtube/transcribe", input);
22377
+ const videoId = input.videoId?.trim() || youtubeVideoIdFromUrl(input.url);
22378
+ if (!videoId) {
22379
+ return Promise.resolve({
22380
+ content: [{
22381
+ type: "text",
22382
+ text: JSON.stringify({
22383
+ error_code: "youtube_video_id_required",
22384
+ error: "Pass videoId from youtube_harvest or a YouTube url that contains a video id.",
22385
+ retryable: false
22386
+ })
22387
+ }],
22388
+ isError: true
22389
+ });
22390
+ }
22391
+ return this.call("/youtube/transcribe", { videoId });
21702
22392
  }
21703
22393
  facebookPageIntel(input) {
21704
22394
  return this.call("/facebook/page-intel", input);
@@ -21735,6 +22425,10 @@ var init_http_mcp_tool_executor = __esm({
21735
22425
  webhookUrl: input.webhookUrl
21736
22426
  }, Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 9e5);
21737
22427
  }
22428
+ workflowStep(input) {
22429
+ const timeoutMs = this.httpTimeoutOverrideMs ?? Number(process.env.MCP_SCRAPER_WORKFLOW_TIMEOUT_MS ?? 9e5);
22430
+ return this.call(`/workflows/runs/${encodeURIComponent(input.runId)}/step`, {}, Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 9e5);
22431
+ }
21738
22432
  workflowStatus(input) {
21739
22433
  return this.getJson(`/workflows/runs/${encodeURIComponent(input.runId)}`);
21740
22434
  }
@@ -21788,16 +22482,18 @@ async function requireMcpCallerKey(c) {
21788
22482
  function registerSerpIntelligenceCaptureTools(server, executor) {
21789
22483
  server.registerTool("capture_serp_snapshot", {
21790
22484
  title: "SERP Intelligence Snapshot",
21791
- description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized US residential evidence; location mode creates fresh proxy IDs across retries, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear or location evidence is wrong. Use configured only for the static residential proxy, and none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
22485
+ description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl. Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
21792
22486
  inputSchema: CaptureSerpSnapshotInputSchema,
22487
+ outputSchema: CaptureSerpSnapshotOutputSchema,
21793
22488
  annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
21794
- }, async (input) => executor.captureSerpSnapshot(input));
22489
+ }, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
21795
22490
  server.registerTool("capture_serp_page_snapshots", {
21796
22491
  title: "SERP Intelligence Page Snapshots",
21797
22492
  description: "Capture public ranking-page evidence through POST /serp-intelligence/page-snapshots, the same product page snapshot path used by Phoenix. Provide urls for simple captures or targets when preserving organic, AI citation, local-pack, configured target, or site-subject source metadata. Private IPs, localhost, file URLs, and internal URLs are rejected by the service. Use timeoutMs for slow pages and debug true for sanitized proxy/browser diagnostics.",
21798
22493
  inputSchema: CaptureSerpPageSnapshotsInputSchema,
22494
+ outputSchema: CaptureSerpPageSnapshotsOutputSchema,
21799
22495
  annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
21800
- }, async (input) => executor.captureSerpPageSnapshots(input));
22496
+ }, async (input) => formatCaptureSerpPageSnapshots(await executor.captureSerpPageSnapshots(input), input));
21801
22497
  }
21802
22498
  var import_hono9, import_webStandardStreamableHttp, mcpApp;
21803
22499
  var init_mcp_routes = __esm({
@@ -22006,14 +22702,35 @@ function client() {
22006
22702
  if (!apiKey) throw new Error("Browser backend API key is required");
22007
22703
  return new import_sdk6.default({ apiKey });
22008
22704
  }
22705
+ function isProfileConflict(err) {
22706
+ const message = err instanceof Error ? err.message : String(err);
22707
+ return /\b409\b|conflict|already exists/i.test(message);
22708
+ }
22709
+ async function ensureProfile(k, name) {
22710
+ try {
22711
+ await k.profiles.create({ name });
22712
+ } catch (err) {
22713
+ if (!isProfileConflict(err)) throw err;
22714
+ }
22715
+ }
22009
22716
  async function createSession(opts = {}) {
22010
22717
  const k = client();
22011
22718
  const resolvedProxyId = opts.proxyId ?? browserServiceProxyId();
22719
+ const resolvedProfileName = opts.profileName ?? browserServiceProfileName();
22720
+ const resolvedSaveProfileChanges = opts.saveProfileChanges ?? browserServiceProfileSaveChanges();
22721
+ if (resolvedProfileName && resolvedSaveProfileChanges === true) {
22722
+ await ensureProfile(k, resolvedProfileName);
22723
+ }
22012
22724
  const browser = await k.browsers.create({
22013
22725
  stealth: opts.stealth ?? true,
22014
22726
  timeout_seconds: opts.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS,
22015
22727
  ...resolvedProxyId ? { proxy_id: resolvedProxyId } : {},
22016
- ...opts.profileName ? { profile: { name: opts.profileName } } : {}
22728
+ ...resolvedProfileName ? {
22729
+ profile: {
22730
+ name: resolvedProfileName,
22731
+ ...typeof resolvedSaveProfileChanges === "boolean" ? { save_changes: resolvedSaveProfileChanges } : {}
22732
+ }
22733
+ } : {}
22017
22734
  });
22018
22735
  const runtimeSessionId = browser.session_id;
22019
22736
  if (opts.disableDefaultProxy) {
@@ -22404,6 +23121,7 @@ function buildBrowserAgentRoutes() {
22404
23121
  timeoutSeconds,
22405
23122
  proxyId: typeof body.proxy_id === "string" ? body.proxy_id : void 0,
22406
23123
  profileName: typeof body.profile === "string" ? body.profile : void 0,
23124
+ saveProfileChanges: typeof body.save_profile_changes === "boolean" ? body.save_profile_changes : void 0,
22407
23125
  disableDefaultProxy: body.disable_default_proxy === true,
22408
23126
  viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0
22409
23127
  });
@@ -22779,6 +23497,7 @@ function renderConsoleHtml(initialSessionId) {
22779
23497
  .gate button { width: 100%; }
22780
23498
  .muted { color: #6b7689; font-size: 12px; }
22781
23499
  </style>
23500
+ ${ATTRIBUTION_PIXEL_SCRIPT}
22782
23501
  </head>
22783
23502
  <body>
22784
23503
  <div id="app"></div>
@@ -22928,6 +23647,7 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
22928
23647
  var init_browser_agent_console = __esm({
22929
23648
  "src/api/browser-agent-console.ts"() {
22930
23649
  "use strict";
23650
+ init_attribution_pixel();
22931
23651
  }
22932
23652
  });
22933
23653
 
@@ -23486,6 +24206,7 @@ var init_server = __esm({
23486
24206
  init_outbound_sanitize();
23487
24207
  init_registry();
23488
24208
  init_template();
24209
+ init_attribution_pixel();
23489
24210
  init_og();
23490
24211
  import_resend = require("resend");
23491
24212
  init_url_utils();
@@ -23518,6 +24239,7 @@ var init_server = __esm({
23518
24239
  init_rates();
23519
24240
  init_billing_schemas();
23520
24241
  init_server_schemas();
24242
+ init_schemas3();
23521
24243
  init_credit_operations();
23522
24244
  init_harvest();
23523
24245
  init_harvest_problems();
@@ -23747,7 +24469,7 @@ var init_server = __esm({
23747
24469
  gl: body.gl ?? "us",
23748
24470
  hl: body.hl ?? "en",
23749
24471
  device: body.device ?? "desktop",
23750
- proxyMode: body.proxyMode ?? "location",
24472
+ proxyMode: body.proxyMode ?? DEFAULT_PROXY_MODE,
23751
24473
  proxyZip: body.proxyZip,
23752
24474
  debug: body.debug ?? false,
23753
24475
  serpOnly: body.serpOnly ?? false,
@@ -23783,7 +24505,7 @@ var init_server = __esm({
23783
24505
  gl: body.gl ?? "us",
23784
24506
  hl: body.hl ?? "en",
23785
24507
  device: body.device ?? "desktop",
23786
- proxyMode: body.proxyMode ?? "location",
24508
+ proxyMode: body.proxyMode ?? DEFAULT_PROXY_MODE,
23787
24509
  proxyZip: body.proxyZip,
23788
24510
  debug: body.debug ?? false,
23789
24511
  serpOnly: body.serpOnly ?? false,
@@ -24328,6 +25050,7 @@ var init_server = __esm({
24328
25050
  .i6 em{font-family:"GTF";font-weight:700;font-style:italic}
24329
25051
  .i7 em{font-family:"GTF";font-weight:900;font-style:italic}
24330
25052
  </style>
25053
+ ${ATTRIBUTION_PIXEL_SCRIPT}
24331
25054
  </head>
24332
25055
  <body>
24333
25056