mcp-scraper 0.2.19 → 0.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -5
- package/dist/bin/api-server.cjs +1072 -349
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1274 -75
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +4 -2
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +659 -210
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +108 -9
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +2104 -566
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +7 -4
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +726 -387
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -2
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +5 -3
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +5 -2
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-TL7YTFLH.js → chunk-DBQDG7EH.js} +99 -27
- package/dist/chunk-DBQDG7EH.js.map +1 -0
- package/dist/chunk-DUEW4EOO.js +214 -0
- package/dist/chunk-DUEW4EOO.js.map +1 -0
- package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js} +216 -108
- package/dist/chunk-H2R232HK.js.map +1 -0
- package/dist/chunk-HS6OKELW.js +1786 -0
- package/dist/chunk-HS6OKELW.js.map +1 -0
- package/dist/{chunk-CQTAKXBN.js → chunk-IPW4LFOT.js} +11 -116
- package/dist/chunk-IPW4LFOT.js.map +1 -0
- package/dist/chunk-LFATOGDF.js +111 -0
- package/dist/chunk-LFATOGDF.js.map +1 -0
- package/dist/{chunk-MCBOZ2JF.js → chunk-MLNCKQ2B.js} +3 -3
- package/dist/chunk-MLNCKQ2B.js.map +1 -0
- package/dist/{chunk-4UEVYYDW.js → chunk-O5J7E4VX.js} +394 -57
- package/dist/chunk-O5J7E4VX.js.map +1 -0
- package/dist/{chunk-BSYPATSM.js → chunk-SXTXMFEQ.js} +2 -2
- package/dist/chunk-WN7PBKMV.js +28 -0
- package/dist/chunk-WN7PBKMV.js.map +1 -0
- package/dist/chunk-XKUDVN2E.js +7 -0
- package/dist/chunk-XKUDVN2E.js.map +1 -0
- package/dist/{db-P5X6UQ3E.js → db-BE4JVB3V.js} +8 -2
- package/dist/index.cjs +5 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/{server-CRUSZA2Q.js → server-24XIQ6W6.js} +248 -40
- package/dist/server-24XIQ6W6.js.map +1 -0
- package/dist/{worker-OZSWIS3F.js → worker-SLQ375UG.js} +8 -5
- package/dist/{worker-OZSWIS3F.js.map → worker-SLQ375UG.js.map} +1 -1
- package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +2 -2
- package/docs/mcp-tool-craft-lint.generated.md +51 -0
- package/docs/mcp-tool-manifest.generated.json +1534 -0
- package/docs/specs/local-competitive-audit-spec.md +1 -2
- package/docs/specs/mcp-tool-definition-quality-audit-spec.md +1602 -0
- package/package.json +4 -1
- package/dist/chunk-4UEVYYDW.js.map +0 -1
- package/dist/chunk-76P4DKWR.js +0 -793
- package/dist/chunk-76P4DKWR.js.map +0 -1
- package/dist/chunk-CQTAKXBN.js.map +0 -1
- package/dist/chunk-F44RBOJ5.js.map +0 -1
- package/dist/chunk-MCBOZ2JF.js.map +0 -1
- package/dist/chunk-RU2HZD55.js +0 -7
- package/dist/chunk-RU2HZD55.js.map +0 -1
- package/dist/chunk-TL7YTFLH.js.map +0 -1
- package/dist/server-CRUSZA2Q.js.map +0 -1
- /package/dist/{chunk-BSYPATSM.js.map → chunk-SXTXMFEQ.js.map} +0 -0
- /package/dist/{db-P5X6UQ3E.js.map → db-BE4JVB3V.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -59,6 +59,17 @@ function browserServiceProxyId() {
|
|
|
59
59
|
const value = (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim();
|
|
60
60
|
return value || void 0;
|
|
61
61
|
}
|
|
62
|
+
function browserServiceProfileName() {
|
|
63
|
+
const value = (process.env.BROWSER_AGENT_PROFILE_NAME ?? process.env.BROWSER_SERVICE_PROFILE_NAME ?? process.env.KERNEL_BROWSER_PROFILE_NAME ?? process.env.KERNEL_PROFILE_NAME)?.trim();
|
|
64
|
+
return value || void 0;
|
|
65
|
+
}
|
|
66
|
+
function browserServiceProfileSaveChanges() {
|
|
67
|
+
const value = (process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ?? process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();
|
|
68
|
+
if (!value) return void 0;
|
|
69
|
+
if (["1", "true", "yes", "on"].includes(value)) return true;
|
|
70
|
+
if (["0", "false", "no", "off"].includes(value)) return false;
|
|
71
|
+
return void 0;
|
|
72
|
+
}
|
|
62
73
|
var init_browser_service_env = __esm({
|
|
63
74
|
"src/lib/browser-service-env.ts"() {
|
|
64
75
|
"use strict";
|
|
@@ -2465,6 +2476,16 @@ var init_registry = __esm({
|
|
|
2465
2476
|
}
|
|
2466
2477
|
});
|
|
2467
2478
|
|
|
2479
|
+
// src/attribution-pixel.ts
|
|
2480
|
+
var ATTRIBUTION_PIXEL_SRC, ATTRIBUTION_PIXEL_SCRIPT;
|
|
2481
|
+
var init_attribution_pixel = __esm({
|
|
2482
|
+
"src/attribution-pixel.ts"() {
|
|
2483
|
+
"use strict";
|
|
2484
|
+
ATTRIBUTION_PIXEL_SRC = "https://hyros-attribution-crm.vercel.app/api/track/sdk?token=35c27ba33dd0c6cc640f9fef415ee3dc9ead67fefe99d269be602ee269900e25";
|
|
2485
|
+
ATTRIBUTION_PIXEL_SCRIPT = `<script src="${ATTRIBUTION_PIXEL_SRC}" async></script>`;
|
|
2486
|
+
}
|
|
2487
|
+
});
|
|
2488
|
+
|
|
2468
2489
|
// src/blog/template.ts
|
|
2469
2490
|
function faqJsonLd(post) {
|
|
2470
2491
|
const firstFive = post.sections.flatMap((s) => s.cards).slice(0, 5);
|
|
@@ -2599,6 +2620,7 @@ function renderPost(post) {
|
|
|
2599
2620
|
<script type="application/ld+json">${faqJsonLd(post)}</script>
|
|
2600
2621
|
<script type="application/ld+json">${breadcrumbJsonLd(post)}</script>
|
|
2601
2622
|
|
|
2623
|
+
${ATTRIBUTION_PIXEL_SCRIPT}
|
|
2602
2624
|
<style>${CSS}</style>
|
|
2603
2625
|
</head>
|
|
2604
2626
|
<body>
|
|
@@ -2742,6 +2764,7 @@ var CSS;
|
|
|
2742
2764
|
var init_template = __esm({
|
|
2743
2765
|
"src/blog/template.ts"() {
|
|
2744
2766
|
"use strict";
|
|
2767
|
+
init_attribution_pixel();
|
|
2745
2768
|
CSS = `
|
|
2746
2769
|
@font-face {
|
|
2747
2770
|
font-family: "GT Flexa";
|
|
@@ -4809,7 +4832,7 @@ function parsePageData(url, html, status, via) {
|
|
|
4809
4832
|
}
|
|
4810
4833
|
const h1 = headings.find((h) => h.level === 1)?.text ?? null;
|
|
4811
4834
|
const bodyText = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
4812
|
-
const
|
|
4835
|
+
const wordCount2 = bodyText.split(" ").filter((w) => w.length > 2).length;
|
|
4813
4836
|
const schemaTypes = [];
|
|
4814
4837
|
const schema = [];
|
|
4815
4838
|
for (const m of html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
|
|
@@ -4845,7 +4868,7 @@ function parsePageData(url, html, status, via) {
|
|
|
4845
4868
|
} catch {
|
|
4846
4869
|
}
|
|
4847
4870
|
}
|
|
4848
|
-
return { url, status, via, title, metaDescription, h1, headings, wordCount, schemaTypes, canonicalUrl, internalLinks, externalLinks, bodyMarkdown, schema };
|
|
4871
|
+
return { url, status, via, title, metaDescription, h1, headings, wordCount: wordCount2, schemaTypes, canonicalUrl, internalLinks, externalLinks, bodyMarkdown, schema };
|
|
4849
4872
|
}
|
|
4850
4873
|
async function fetchAndParse(url, kernelApiKey) {
|
|
4851
4874
|
try {
|
|
@@ -6906,6 +6929,8 @@ var db_exports = {};
|
|
|
6906
6929
|
__export(db_exports, {
|
|
6907
6930
|
SiteAuditJobRowSchema: () => SiteAuditJobRowSchema,
|
|
6908
6931
|
SiteAuditPhaseLogRowSchema: () => SiteAuditPhaseLogRowSchema,
|
|
6932
|
+
advanceWorkflowRunStep: () => advanceWorkflowRunStep,
|
|
6933
|
+
appendWorkflowArtifacts: () => appendWorkflowArtifacts,
|
|
6909
6934
|
cancelJob: () => cancelJob,
|
|
6910
6935
|
checkRateLimit: () => checkRateLimit,
|
|
6911
6936
|
claimMonthlyFreeRefresh: () => claimMonthlyFreeRefresh,
|
|
@@ -6946,6 +6971,7 @@ __export(db_exports, {
|
|
|
6946
6971
|
getWorkflowArtifact: () => getWorkflowArtifact,
|
|
6947
6972
|
getWorkflowRun: () => getWorkflowRun,
|
|
6948
6973
|
getWorkflowRunByIdempotencyKey: () => getWorkflowRunByIdempotencyKey,
|
|
6974
|
+
getWorkflowRunStepState: () => getWorkflowRunStepState,
|
|
6949
6975
|
getWorkflowSchedule: () => getWorkflowSchedule,
|
|
6950
6976
|
hashPassword: () => hashPassword,
|
|
6951
6977
|
ledgerExistsForOperation: () => ledgerExistsForOperation,
|
|
@@ -7227,9 +7253,34 @@ async function migrate() {
|
|
|
7227
7253
|
queued_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
7228
7254
|
started_at TEXT,
|
|
7229
7255
|
completed_at TEXT,
|
|
7230
|
-
idempotency_key TEXT
|
|
7256
|
+
idempotency_key TEXT,
|
|
7257
|
+
step_index INTEGER NOT NULL DEFAULT 0,
|
|
7258
|
+
total_steps INTEGER,
|
|
7259
|
+
step_state_json TEXT,
|
|
7260
|
+
step_log_json TEXT,
|
|
7261
|
+
webhook_url TEXT
|
|
7231
7262
|
)
|
|
7232
7263
|
`);
|
|
7264
|
+
try {
|
|
7265
|
+
await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_index INTEGER NOT NULL DEFAULT 0`);
|
|
7266
|
+
} catch {
|
|
7267
|
+
}
|
|
7268
|
+
try {
|
|
7269
|
+
await db.execute(`ALTER TABLE workflow_runs ADD COLUMN total_steps INTEGER`);
|
|
7270
|
+
} catch {
|
|
7271
|
+
}
|
|
7272
|
+
try {
|
|
7273
|
+
await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_state_json TEXT`);
|
|
7274
|
+
} catch {
|
|
7275
|
+
}
|
|
7276
|
+
try {
|
|
7277
|
+
await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_log_json TEXT`);
|
|
7278
|
+
} catch {
|
|
7279
|
+
}
|
|
7280
|
+
try {
|
|
7281
|
+
await db.execute(`ALTER TABLE workflow_runs ADD COLUMN webhook_url TEXT`);
|
|
7282
|
+
} catch {
|
|
7283
|
+
}
|
|
7233
7284
|
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_user_created ON workflow_runs(user_id, queued_at DESC)`);
|
|
7234
7285
|
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_schedule ON workflow_runs(schedule_id, queued_at DESC)`);
|
|
7235
7286
|
await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS workflow_runs_idempotency ON workflow_runs(idempotency_key) WHERE idempotency_key IS NOT NULL`);
|
|
@@ -7244,9 +7295,14 @@ async function migrate() {
|
|
|
7244
7295
|
content_type TEXT NOT NULL,
|
|
7245
7296
|
bytes INTEGER,
|
|
7246
7297
|
rows_count INTEGER,
|
|
7298
|
+
content TEXT,
|
|
7247
7299
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
7248
7300
|
)
|
|
7249
7301
|
`);
|
|
7302
|
+
try {
|
|
7303
|
+
await db.execute(`ALTER TABLE workflow_artifacts ADD COLUMN content TEXT`);
|
|
7304
|
+
} catch {
|
|
7305
|
+
}
|
|
7250
7306
|
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_artifacts_run ON workflow_artifacts(run_id)`);
|
|
7251
7307
|
await db.execute(`
|
|
7252
7308
|
CREATE TABLE IF NOT EXISTS workflow_webhook_deliveries (
|
|
@@ -7692,9 +7748,22 @@ function rowToWorkflowRun(row) {
|
|
|
7692
7748
|
queued_at: String(row.queued_at),
|
|
7693
7749
|
started_at: row.started_at != null ? String(row.started_at) : null,
|
|
7694
7750
|
completed_at: row.completed_at != null ? String(row.completed_at) : null,
|
|
7695
|
-
idempotency_key: row.idempotency_key != null ? String(row.idempotency_key) : null
|
|
7751
|
+
idempotency_key: row.idempotency_key != null ? String(row.idempotency_key) : null,
|
|
7752
|
+
step_index: row.step_index != null ? Number(row.step_index) : 0,
|
|
7753
|
+
total_steps: row.total_steps != null ? Number(row.total_steps) : null,
|
|
7754
|
+
step_log: parseWorkflowStepLog(row.step_log_json),
|
|
7755
|
+
webhook_url: row.webhook_url != null ? String(row.webhook_url) : null
|
|
7696
7756
|
};
|
|
7697
7757
|
}
|
|
7758
|
+
function parseWorkflowStepLog(value) {
|
|
7759
|
+
if (value == null) return [];
|
|
7760
|
+
try {
|
|
7761
|
+
const parsed = JSON.parse(String(value));
|
|
7762
|
+
return Array.isArray(parsed) ? parsed : [];
|
|
7763
|
+
} catch {
|
|
7764
|
+
return [];
|
|
7765
|
+
}
|
|
7766
|
+
}
|
|
7698
7767
|
function rowToWorkflowArtifact(row) {
|
|
7699
7768
|
return {
|
|
7700
7769
|
id: String(row.id),
|
|
@@ -7706,6 +7775,7 @@ function rowToWorkflowArtifact(row) {
|
|
|
7706
7775
|
content_type: String(row.content_type),
|
|
7707
7776
|
bytes: row.bytes != null ? Number(row.bytes) : null,
|
|
7708
7777
|
rows_count: row.rows_count != null ? Number(row.rows_count) : null,
|
|
7778
|
+
content: row.content != null ? String(row.content) : null,
|
|
7709
7779
|
created_at: String(row.created_at)
|
|
7710
7780
|
};
|
|
7711
7781
|
}
|
|
@@ -7817,13 +7887,15 @@ async function createWorkflowRun(input) {
|
|
|
7817
7887
|
input.scheduleId ?? null,
|
|
7818
7888
|
input.workflowId,
|
|
7819
7889
|
JSON.stringify(input.workflowInput),
|
|
7820
|
-
input.idempotencyKey ?? null
|
|
7890
|
+
input.idempotencyKey ?? null,
|
|
7891
|
+
input.webhookUrl ?? null,
|
|
7892
|
+
input.totalSteps ?? null
|
|
7821
7893
|
];
|
|
7822
7894
|
try {
|
|
7823
7895
|
await getDb().execute({
|
|
7824
7896
|
sql: `
|
|
7825
|
-
INSERT INTO workflow_runs (id, user_id, schedule_id, workflow_id, input_json, idempotency_key)
|
|
7826
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
7897
|
+
INSERT INTO workflow_runs (id, user_id, schedule_id, workflow_id, input_json, idempotency_key, webhook_url, total_steps)
|
|
7898
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
7827
7899
|
`,
|
|
7828
7900
|
args
|
|
7829
7901
|
});
|
|
@@ -7877,30 +7949,53 @@ function workflowArtifactContentType(kind) {
|
|
|
7877
7949
|
if (kind === "json") return "application/json; charset=utf-8";
|
|
7878
7950
|
return "text/plain; charset=utf-8";
|
|
7879
7951
|
}
|
|
7952
|
+
async function insertWorkflowArtifact(runId, userId, artifact) {
|
|
7953
|
+
await getDb().execute({
|
|
7954
|
+
sql: `
|
|
7955
|
+
INSERT INTO workflow_artifacts (id, run_id, user_id, kind, label, path, content_type, bytes, rows_count, content)
|
|
7956
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
7957
|
+
`,
|
|
7958
|
+
args: [
|
|
7959
|
+
(0, import_node_crypto.randomUUID)(),
|
|
7960
|
+
runId,
|
|
7961
|
+
userId,
|
|
7962
|
+
artifact.kind,
|
|
7963
|
+
artifact.label,
|
|
7964
|
+
artifact.path,
|
|
7965
|
+
workflowArtifactContentType(artifact.kind),
|
|
7966
|
+
artifact.bytes ?? null,
|
|
7967
|
+
artifact.rows ?? null,
|
|
7968
|
+
artifact.content ?? null
|
|
7969
|
+
]
|
|
7970
|
+
});
|
|
7971
|
+
}
|
|
7880
7972
|
async function replaceWorkflowArtifacts(runId, userId, artifacts) {
|
|
7881
|
-
|
|
7882
|
-
|
|
7883
|
-
for (const artifact of artifacts) {
|
|
7884
|
-
await db.execute({
|
|
7885
|
-
sql: `
|
|
7886
|
-
INSERT INTO workflow_artifacts (id, run_id, user_id, kind, label, path, content_type, bytes, rows_count)
|
|
7887
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
7888
|
-
`,
|
|
7889
|
-
args: [
|
|
7890
|
-
(0, import_node_crypto.randomUUID)(),
|
|
7891
|
-
runId,
|
|
7892
|
-
userId,
|
|
7893
|
-
artifact.kind,
|
|
7894
|
-
artifact.label,
|
|
7895
|
-
artifact.path,
|
|
7896
|
-
workflowArtifactContentType(artifact.kind),
|
|
7897
|
-
artifact.bytes ?? null,
|
|
7898
|
-
artifact.rows ?? null
|
|
7899
|
-
]
|
|
7900
|
-
});
|
|
7901
|
-
}
|
|
7973
|
+
await getDb().execute({ sql: "DELETE FROM workflow_artifacts WHERE run_id = ? AND user_id = ?", args: [runId, userId] });
|
|
7974
|
+
for (const artifact of artifacts) await insertWorkflowArtifact(runId, userId, artifact);
|
|
7902
7975
|
return listWorkflowArtifacts(runId, userId);
|
|
7903
7976
|
}
|
|
7977
|
+
async function appendWorkflowArtifacts(runId, userId, artifacts) {
|
|
7978
|
+
for (const artifact of artifacts) await insertWorkflowArtifact(runId, userId, artifact);
|
|
7979
|
+
return listWorkflowArtifacts(runId, userId);
|
|
7980
|
+
}
|
|
7981
|
+
async function getWorkflowRunStepState(id, userId) {
|
|
7982
|
+
const res = userId == null ? await getDb().execute({ sql: "SELECT step_index, total_steps, step_state_json, step_log_json FROM workflow_runs WHERE id = ? LIMIT 1", args: [id] }) : await getDb().execute({ sql: "SELECT step_index, total_steps, step_state_json, step_log_json FROM workflow_runs WHERE id = ? AND user_id = ? LIMIT 1", args: [id, userId] });
|
|
7983
|
+
const row = res.rows[0];
|
|
7984
|
+
if (!row) return null;
|
|
7985
|
+
return {
|
|
7986
|
+
stepIndex: row.step_index != null ? Number(row.step_index) : 0,
|
|
7987
|
+
totalSteps: row.total_steps != null ? Number(row.total_steps) : null,
|
|
7988
|
+
state: row.step_state_json != null ? parseNullableJson(row.step_state_json) : null,
|
|
7989
|
+
stepLog: parseWorkflowStepLog(row.step_log_json)
|
|
7990
|
+
};
|
|
7991
|
+
}
|
|
7992
|
+
async function advanceWorkflowRunStep(id, input) {
|
|
7993
|
+
const stepLog = [...input.appendLog, input.logEntry];
|
|
7994
|
+
await getDb().execute({
|
|
7995
|
+
sql: `UPDATE workflow_runs SET status = ?, step_index = ?, total_steps = ?, step_state_json = ?, step_log_json = ?, started_at = COALESCE(started_at, datetime('now')) WHERE id = ?`,
|
|
7996
|
+
args: [input.status, input.stepIndex, input.totalSteps, JSON.stringify(input.state ?? null), JSON.stringify(stepLog), id]
|
|
7997
|
+
});
|
|
7998
|
+
}
|
|
7904
7999
|
async function listWorkflowArtifacts(runId, userId) {
|
|
7905
8000
|
const res = await getDb().execute({
|
|
7906
8001
|
sql: "SELECT * FROM workflow_artifacts WHERE run_id = ? AND user_id = ? ORDER BY created_at ASC",
|
|
@@ -8909,6 +9004,7 @@ var init_schemas2 = __esm({
|
|
|
8909
9004
|
downloadMp3: import_zod12.z.boolean().default(false),
|
|
8910
9005
|
outputDir: import_zod12.z.string().default("./yt-output"),
|
|
8911
9006
|
kernelApiKey: import_zod12.z.string().optional(),
|
|
9007
|
+
kernelProxyId: import_zod12.z.string().optional(),
|
|
8912
9008
|
headless: import_zod12.z.boolean().default(false)
|
|
8913
9009
|
}).refine(
|
|
8914
9010
|
(o) => o.mode === "search" ? !!o.query : !!o.channelHandle,
|
|
@@ -9833,6 +9929,7 @@ var init_YouTubeExtractor = __esm({
|
|
|
9833
9929
|
const config = {
|
|
9834
9930
|
headless: options.headless,
|
|
9835
9931
|
kernelApiKey: options.kernelApiKey,
|
|
9932
|
+
kernelProxyId: options.kernelProxyId,
|
|
9836
9933
|
viewport: { width: 1280, height: 800 },
|
|
9837
9934
|
locale: "en-US"
|
|
9838
9935
|
};
|
|
@@ -10044,7 +10141,11 @@ async function ytHarvest(rawOptions) {
|
|
|
10044
10141
|
throw new Error("A browser backend API key is required \u2014 YouTube harvesting requires a stealth session.");
|
|
10045
10142
|
}
|
|
10046
10143
|
const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};
|
|
10047
|
-
const options = YouTubeHarvestOptionsSchema.parse({
|
|
10144
|
+
const options = YouTubeHarvestOptionsSchema.parse({
|
|
10145
|
+
kernelApiKey,
|
|
10146
|
+
kernelProxyId: browserServiceProxyId(),
|
|
10147
|
+
...raw
|
|
10148
|
+
});
|
|
10048
10149
|
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
10049
10150
|
try {
|
|
10050
10151
|
const result = await extractOnce(options);
|
|
@@ -10153,10 +10254,11 @@ function parseTimedtextXml(xml) {
|
|
|
10153
10254
|
async function fetchViaKernelInnertube(videoId) {
|
|
10154
10255
|
const kernelApiKey = browserServiceApiKey();
|
|
10155
10256
|
if (!kernelApiKey) return null;
|
|
10257
|
+
const kernelProxyId = browserServiceProxyId();
|
|
10156
10258
|
const driver = new BrowserDriver();
|
|
10157
10259
|
const start = Date.now();
|
|
10158
10260
|
try {
|
|
10159
|
-
await driver.launch({ kernelApiKey, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
|
|
10261
|
+
await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
|
|
10160
10262
|
const page = driver.getPage();
|
|
10161
10263
|
await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
|
|
10162
10264
|
await page.waitForFunction(
|
|
@@ -10250,10 +10352,10 @@ async function captureAudioViaMediaRecorder(page) {
|
|
|
10250
10352
|
if (!result.ok || !result.base64 || !result.mimeType) return null;
|
|
10251
10353
|
return { bytes: Buffer.from(result.base64, "base64"), mimeType: result.mimeType };
|
|
10252
10354
|
}
|
|
10253
|
-
async function attemptKernelWhisper(videoId, kernelApiKey, falKey, start) {
|
|
10355
|
+
async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start) {
|
|
10254
10356
|
const driver = new BrowserDriver();
|
|
10255
10357
|
try {
|
|
10256
|
-
await driver.launch({ kernelApiKey, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
|
|
10358
|
+
await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
|
|
10257
10359
|
const page = driver.getPage();
|
|
10258
10360
|
await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
|
|
10259
10361
|
await page.waitForFunction(
|
|
@@ -10295,12 +10397,13 @@ async function attemptKernelWhisper(videoId, kernelApiKey, falKey, start) {
|
|
|
10295
10397
|
}
|
|
10296
10398
|
async function fetchViaKernelWhisper(videoId) {
|
|
10297
10399
|
const kernelApiKey = browserServiceApiKey();
|
|
10400
|
+
const kernelProxyId = browserServiceProxyId();
|
|
10298
10401
|
const falKey = process.env.FAL_KEY;
|
|
10299
10402
|
if (!kernelApiKey || !falKey) return null;
|
|
10300
10403
|
const start = Date.now();
|
|
10301
10404
|
for (let attempt = 0; attempt < 4; attempt++) {
|
|
10302
10405
|
if (attempt > 0) await new Promise((r) => setTimeout(r, 2e3 * attempt));
|
|
10303
|
-
const result = await attemptKernelWhisper(videoId, kernelApiKey, falKey, start);
|
|
10406
|
+
const result = await attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start);
|
|
10304
10407
|
if (result) return result;
|
|
10305
10408
|
}
|
|
10306
10409
|
return null;
|
|
@@ -10626,6 +10729,7 @@ function buildTranscriptHtml(result) {
|
|
|
10626
10729
|
(ch) => `<div class="chunk"><span class="ts">${fmtSecs(ch.timestamp[0])} \u2192 ${fmtSecs(ch.timestamp[1])}</span><span class="ct">${esc(ch.text.trim())}</span></div>`
|
|
10627
10730
|
).join("\n");
|
|
10628
10731
|
return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
10732
|
+
${ATTRIBUTION_PIXEL_SCRIPT}
|
|
10629
10733
|
<style>body{font-family:system-ui,sans-serif;max-width:800px;margin:1.5rem auto;padding:0 1rem;color:#111;line-height:1.7}
|
|
10630
10734
|
h1{font-size:1.2rem;margin-bottom:.25rem}.meta{color:#666;font-size:.85rem;margin-bottom:1.5rem}
|
|
10631
10735
|
.full-text{background:#f5f5f3;border-radius:8px;padding:1rem 1.25rem;font-size:.875rem;white-space:pre-wrap;margin-bottom:2rem;word-break:break-word}
|
|
@@ -10655,6 +10759,7 @@ var init_youtube_routes = __esm({
|
|
|
10655
10759
|
init_CaptionFetcher();
|
|
10656
10760
|
init_BrowserDriver();
|
|
10657
10761
|
init_errors();
|
|
10762
|
+
init_attribution_pixel();
|
|
10658
10763
|
init_api_auth();
|
|
10659
10764
|
init_server_schemas();
|
|
10660
10765
|
init_concurrency_gates();
|
|
@@ -12283,6 +12388,110 @@ var init_kernel_proxy_resolver = __esm({
|
|
|
12283
12388
|
}
|
|
12284
12389
|
});
|
|
12285
12390
|
|
|
12391
|
+
// src/schemas.ts
|
|
12392
|
+
var import_zod15, DEFAULT_PROXY_MODE, DEFAULT_MAPS_PROXY_MODE, HarvestOptionsSchema, MapsPlaceOptionsSchema, MapsSearchOptionsSchema, RawPAAItemSchema, RawMapsOverviewSchema, RawMapsHoursRowSchema, RawMapsReviewStatsSchema, RawMapsReviewCardSchema, RawMapsAboutAttributeSchema;
|
|
12393
|
+
var init_schemas3 = __esm({
|
|
12394
|
+
"src/schemas.ts"() {
|
|
12395
|
+
"use strict";
|
|
12396
|
+
import_zod15 = require("zod");
|
|
12397
|
+
DEFAULT_PROXY_MODE = "configured";
|
|
12398
|
+
DEFAULT_MAPS_PROXY_MODE = "location";
|
|
12399
|
+
HarvestOptionsSchema = import_zod15.z.object({
|
|
12400
|
+
query: import_zod15.z.string().min(1),
|
|
12401
|
+
location: import_zod15.z.string().optional(),
|
|
12402
|
+
gl: import_zod15.z.string().length(2).default("us"),
|
|
12403
|
+
hl: import_zod15.z.string().length(2).default("en"),
|
|
12404
|
+
device: import_zod15.z.enum(["desktop", "mobile"]).default("desktop"),
|
|
12405
|
+
proxyMode: import_zod15.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
|
|
12406
|
+
proxyZip: import_zod15.z.string().regex(/^\d{5}$/).optional(),
|
|
12407
|
+
debug: import_zod15.z.boolean().default(false),
|
|
12408
|
+
depth: import_zod15.z.number().int().min(1).max(30).default(3),
|
|
12409
|
+
maxQuestions: import_zod15.z.number().int().min(1).max(1e3).default(100),
|
|
12410
|
+
headless: import_zod15.z.boolean().default(false),
|
|
12411
|
+
profileDir: import_zod15.z.string().optional(),
|
|
12412
|
+
proxy: import_zod15.z.string().url().optional(),
|
|
12413
|
+
kernelApiKey: import_zod15.z.string().optional(),
|
|
12414
|
+
kernelProxyId: import_zod15.z.string().optional(),
|
|
12415
|
+
kernelProxyResolution: import_zod15.z.unknown().optional(),
|
|
12416
|
+
outputDir: import_zod15.z.string().default("./paa-output"),
|
|
12417
|
+
format: import_zod15.z.enum(["json", "csv", "both"]).default("both"),
|
|
12418
|
+
serpOnly: import_zod15.z.boolean().default(false),
|
|
12419
|
+
pages: import_zod15.z.number().int().min(1).max(2).default(1)
|
|
12420
|
+
});
|
|
12421
|
+
MapsPlaceOptionsSchema = import_zod15.z.object({
|
|
12422
|
+
businessName: import_zod15.z.string().min(1),
|
|
12423
|
+
location: import_zod15.z.string().min(1),
|
|
12424
|
+
gl: import_zod15.z.string().length(2).default("us"),
|
|
12425
|
+
hl: import_zod15.z.string().length(2).default("en"),
|
|
12426
|
+
includeReviews: import_zod15.z.boolean().default(false),
|
|
12427
|
+
maxReviews: import_zod15.z.number().int().min(1).max(500).default(50),
|
|
12428
|
+
kernelApiKey: import_zod15.z.string().optional(),
|
|
12429
|
+
kernelProxyId: import_zod15.z.string().optional(),
|
|
12430
|
+
headless: import_zod15.z.boolean().default(true)
|
|
12431
|
+
});
|
|
12432
|
+
MapsSearchOptionsSchema = import_zod15.z.object({
|
|
12433
|
+
query: import_zod15.z.string().min(1),
|
|
12434
|
+
location: import_zod15.z.string().optional(),
|
|
12435
|
+
gl: import_zod15.z.string().length(2).default("us"),
|
|
12436
|
+
hl: import_zod15.z.string().length(2).default("en"),
|
|
12437
|
+
maxResults: import_zod15.z.number().int().min(1).max(50).default(10),
|
|
12438
|
+
proxyMode: import_zod15.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
12439
|
+
proxyZip: import_zod15.z.string().regex(/^\d{5}$/).optional(),
|
|
12440
|
+
debug: import_zod15.z.boolean().default(false),
|
|
12441
|
+
kernelApiKey: import_zod15.z.string().optional(),
|
|
12442
|
+
kernelProxyId: import_zod15.z.string().optional(),
|
|
12443
|
+
kernelProxyResolution: import_zod15.z.unknown().optional(),
|
|
12444
|
+
headless: import_zod15.z.boolean().default(true)
|
|
12445
|
+
});
|
|
12446
|
+
RawPAAItemSchema = import_zod15.z.object({
|
|
12447
|
+
question: import_zod15.z.string().min(1),
|
|
12448
|
+
answer: import_zod15.z.string().optional(),
|
|
12449
|
+
sourceTitle: import_zod15.z.string().optional(),
|
|
12450
|
+
sourceSite: import_zod15.z.string().optional(),
|
|
12451
|
+
sourceCite: import_zod15.z.string().optional()
|
|
12452
|
+
});
|
|
12453
|
+
RawMapsOverviewSchema = import_zod15.z.object({
|
|
12454
|
+
name: import_zod15.z.string().nullable(),
|
|
12455
|
+
rating: import_zod15.z.string().nullable(),
|
|
12456
|
+
reviewCount: import_zod15.z.string().nullable(),
|
|
12457
|
+
category: import_zod15.z.string().nullable(),
|
|
12458
|
+
address: import_zod15.z.string().nullable(),
|
|
12459
|
+
hoursSummary: import_zod15.z.string().nullable(),
|
|
12460
|
+
phone: import_zod15.z.string().nullable(),
|
|
12461
|
+
phoneDisplay: import_zod15.z.string().nullable(),
|
|
12462
|
+
website: import_zod15.z.string().nullable(),
|
|
12463
|
+
plusCode: import_zod15.z.string().nullable(),
|
|
12464
|
+
bookingUrl: import_zod15.z.string().nullable()
|
|
12465
|
+
});
|
|
12466
|
+
RawMapsHoursRowSchema = import_zod15.z.object({
|
|
12467
|
+
day: import_zod15.z.string(),
|
|
12468
|
+
hours: import_zod15.z.string()
|
|
12469
|
+
});
|
|
12470
|
+
RawMapsReviewStatsSchema = import_zod15.z.object({
|
|
12471
|
+
reviewHistogram: import_zod15.z.array(import_zod15.z.object({
|
|
12472
|
+
stars: import_zod15.z.number(),
|
|
12473
|
+
count: import_zod15.z.string()
|
|
12474
|
+
})),
|
|
12475
|
+
reviewTopics: import_zod15.z.array(import_zod15.z.object({
|
|
12476
|
+
label: import_zod15.z.string(),
|
|
12477
|
+
count: import_zod15.z.string()
|
|
12478
|
+
}))
|
|
12479
|
+
});
|
|
12480
|
+
RawMapsReviewCardSchema = import_zod15.z.object({
|
|
12481
|
+
reviewId: import_zod15.z.string(),
|
|
12482
|
+
author: import_zod15.z.string().nullable(),
|
|
12483
|
+
stars: import_zod15.z.string().nullable(),
|
|
12484
|
+
date: import_zod15.z.string().nullable(),
|
|
12485
|
+
text: import_zod15.z.string().nullable(),
|
|
12486
|
+
ownerResponse: import_zod15.z.string().nullable()
|
|
12487
|
+
});
|
|
12488
|
+
RawMapsAboutAttributeSchema = import_zod15.z.object({
|
|
12489
|
+
section: import_zod15.z.string(),
|
|
12490
|
+
attribute: import_zod15.z.string()
|
|
12491
|
+
});
|
|
12492
|
+
}
|
|
12493
|
+
});
|
|
12494
|
+
|
|
12286
12495
|
// src/api/facebook-ad-routes.ts
|
|
12287
12496
|
function invalidRequest(message) {
|
|
12288
12497
|
return { error_code: "invalid_request", message };
|
|
@@ -12338,9 +12547,8 @@ async function kernelLaunchOptsResidential() {
|
|
|
12338
12547
|
try {
|
|
12339
12548
|
const resolution2 = await resolveKernelProxyId({
|
|
12340
12549
|
kernelApiKey: browserServiceApiKey(),
|
|
12341
|
-
proxyMode:
|
|
12550
|
+
proxyMode: DEFAULT_PROXY_MODE,
|
|
12342
12551
|
configuredKernelProxyId: browserServiceProxyId(),
|
|
12343
|
-
location: "New York, NY",
|
|
12344
12552
|
gl: "us"
|
|
12345
12553
|
});
|
|
12346
12554
|
if (resolution2.kernelProxyId) proxyId = resolution2.kernelProxyId;
|
|
@@ -12349,13 +12557,13 @@ async function kernelLaunchOptsResidential() {
|
|
|
12349
12557
|
}
|
|
12350
12558
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
12351
12559
|
}
|
|
12352
|
-
var import_hono4,
|
|
12560
|
+
var import_hono4, import_zod16, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
|
|
12353
12561
|
var init_facebook_ad_routes = __esm({
|
|
12354
12562
|
"src/api/facebook-ad-routes.ts"() {
|
|
12355
12563
|
"use strict";
|
|
12356
12564
|
import_hono4 = require("hono");
|
|
12357
12565
|
init_browser_service_env();
|
|
12358
|
-
|
|
12566
|
+
import_zod16 = require("zod");
|
|
12359
12567
|
init_db();
|
|
12360
12568
|
init_rates();
|
|
12361
12569
|
init_BrowserDriver();
|
|
@@ -12363,39 +12571,40 @@ var init_facebook_ad_routes = __esm({
|
|
|
12363
12571
|
init_FacebookAdGraphql();
|
|
12364
12572
|
init_FacebookOrganicVideoExtractor();
|
|
12365
12573
|
init_kernel_proxy_resolver();
|
|
12574
|
+
init_schemas3();
|
|
12366
12575
|
import_client3 = require("@fal-ai/client");
|
|
12367
12576
|
init_api_auth();
|
|
12368
12577
|
init_url_utils();
|
|
12369
12578
|
init_concurrency_gates();
|
|
12370
|
-
FacebookAdBodySchema =
|
|
12371
|
-
url:
|
|
12372
|
-
libraryId:
|
|
12373
|
-
openModal:
|
|
12579
|
+
FacebookAdBodySchema = import_zod16.z.object({
|
|
12580
|
+
url: import_zod16.z.string().trim().optional(),
|
|
12581
|
+
libraryId: import_zod16.z.string().trim().optional(),
|
|
12582
|
+
openModal: import_zod16.z.boolean().optional()
|
|
12374
12583
|
}).refine((d) => !!d.url || !!d.libraryId, { message: "url or libraryId is required" });
|
|
12375
|
-
FacebookPageIntelBodySchema =
|
|
12376
|
-
pageId:
|
|
12377
|
-
query:
|
|
12378
|
-
libraryId:
|
|
12379
|
-
maxAds:
|
|
12380
|
-
country:
|
|
12584
|
+
FacebookPageIntelBodySchema = import_zod16.z.object({
|
|
12585
|
+
pageId: import_zod16.z.string().trim().optional(),
|
|
12586
|
+
query: import_zod16.z.string().trim().optional(),
|
|
12587
|
+
libraryId: import_zod16.z.string().trim().optional(),
|
|
12588
|
+
maxAds: import_zod16.z.number().int().min(1).max(200).optional(),
|
|
12589
|
+
country: import_zod16.z.string().trim().toUpperCase().optional()
|
|
12381
12590
|
}).refine((d) => !!d.pageId || !!d.query || !!d.libraryId, {
|
|
12382
12591
|
message: "pageId, libraryId, or query is required"
|
|
12383
12592
|
});
|
|
12384
|
-
FacebookTranscribeBodySchema =
|
|
12385
|
-
videoUrl:
|
|
12593
|
+
FacebookTranscribeBodySchema = import_zod16.z.object({
|
|
12594
|
+
videoUrl: import_zod16.z.string().trim().min(1, "videoUrl is required")
|
|
12386
12595
|
});
|
|
12387
|
-
FacebookVideoTranscribeBodySchema =
|
|
12388
|
-
url:
|
|
12389
|
-
quality:
|
|
12596
|
+
FacebookVideoTranscribeBodySchema = import_zod16.z.object({
|
|
12597
|
+
url: import_zod16.z.string().trim().min(1, "url is required"),
|
|
12598
|
+
quality: import_zod16.z.enum(["best", "hd", "sd"]).default("best")
|
|
12390
12599
|
});
|
|
12391
|
-
FacebookSearchBodySchema =
|
|
12392
|
-
query:
|
|
12393
|
-
country:
|
|
12394
|
-
maxResults:
|
|
12600
|
+
FacebookSearchBodySchema = import_zod16.z.object({
|
|
12601
|
+
query: import_zod16.z.string().trim().min(1, "query is required"),
|
|
12602
|
+
country: import_zod16.z.string().trim().toUpperCase().optional(),
|
|
12603
|
+
maxResults: import_zod16.z.number().int().min(1).max(20).optional()
|
|
12395
12604
|
});
|
|
12396
|
-
FacebookMediaBodySchema =
|
|
12397
|
-
url:
|
|
12398
|
-
filename:
|
|
12605
|
+
FacebookMediaBodySchema = import_zod16.z.object({
|
|
12606
|
+
url: import_zod16.z.string().trim().min(1, "url is required"),
|
|
12607
|
+
filename: import_zod16.z.string().trim().optional()
|
|
12399
12608
|
});
|
|
12400
12609
|
facebookAdApp = new import_hono4.Hono();
|
|
12401
12610
|
facebookAdApp.post("/ad", createApiKeyAuth(), async (c) => {
|
|
@@ -12721,108 +12930,6 @@ var init_facebook_ad_routes = __esm({
|
|
|
12721
12930
|
}
|
|
12722
12931
|
});
|
|
12723
12932
|
|
|
12724
|
-
// src/schemas.ts
|
|
12725
|
-
var import_zod16, HarvestOptionsSchema, MapsPlaceOptionsSchema, MapsSearchOptionsSchema, RawPAAItemSchema, RawMapsOverviewSchema, RawMapsHoursRowSchema, RawMapsReviewStatsSchema, RawMapsReviewCardSchema, RawMapsAboutAttributeSchema;
|
|
12726
|
-
var init_schemas3 = __esm({
|
|
12727
|
-
"src/schemas.ts"() {
|
|
12728
|
-
"use strict";
|
|
12729
|
-
import_zod16 = require("zod");
|
|
12730
|
-
HarvestOptionsSchema = import_zod16.z.object({
|
|
12731
|
-
query: import_zod16.z.string().min(1),
|
|
12732
|
-
location: import_zod16.z.string().optional(),
|
|
12733
|
-
gl: import_zod16.z.string().length(2).default("us"),
|
|
12734
|
-
hl: import_zod16.z.string().length(2).default("en"),
|
|
12735
|
-
device: import_zod16.z.enum(["desktop", "mobile"]).default("desktop"),
|
|
12736
|
-
proxyMode: import_zod16.z.enum(["location", "configured", "none"]).default("location"),
|
|
12737
|
-
proxyZip: import_zod16.z.string().regex(/^\d{5}$/).optional(),
|
|
12738
|
-
debug: import_zod16.z.boolean().default(false),
|
|
12739
|
-
depth: import_zod16.z.number().int().min(1).max(30).default(3),
|
|
12740
|
-
maxQuestions: import_zod16.z.number().int().min(1).max(1e3).default(100),
|
|
12741
|
-
headless: import_zod16.z.boolean().default(false),
|
|
12742
|
-
profileDir: import_zod16.z.string().optional(),
|
|
12743
|
-
proxy: import_zod16.z.string().url().optional(),
|
|
12744
|
-
kernelApiKey: import_zod16.z.string().optional(),
|
|
12745
|
-
kernelProxyId: import_zod16.z.string().optional(),
|
|
12746
|
-
kernelProxyResolution: import_zod16.z.unknown().optional(),
|
|
12747
|
-
outputDir: import_zod16.z.string().default("./paa-output"),
|
|
12748
|
-
format: import_zod16.z.enum(["json", "csv", "both"]).default("both"),
|
|
12749
|
-
serpOnly: import_zod16.z.boolean().default(false),
|
|
12750
|
-
pages: import_zod16.z.number().int().min(1).max(2).default(1)
|
|
12751
|
-
});
|
|
12752
|
-
MapsPlaceOptionsSchema = import_zod16.z.object({
|
|
12753
|
-
businessName: import_zod16.z.string().min(1),
|
|
12754
|
-
location: import_zod16.z.string().min(1),
|
|
12755
|
-
gl: import_zod16.z.string().length(2).default("us"),
|
|
12756
|
-
hl: import_zod16.z.string().length(2).default("en"),
|
|
12757
|
-
includeReviews: import_zod16.z.boolean().default(false),
|
|
12758
|
-
maxReviews: import_zod16.z.number().int().min(1).max(500).default(50),
|
|
12759
|
-
kernelApiKey: import_zod16.z.string().optional(),
|
|
12760
|
-
kernelProxyId: import_zod16.z.string().optional(),
|
|
12761
|
-
headless: import_zod16.z.boolean().default(true)
|
|
12762
|
-
});
|
|
12763
|
-
MapsSearchOptionsSchema = import_zod16.z.object({
|
|
12764
|
-
query: import_zod16.z.string().min(1),
|
|
12765
|
-
location: import_zod16.z.string().optional(),
|
|
12766
|
-
gl: import_zod16.z.string().length(2).default("us"),
|
|
12767
|
-
hl: import_zod16.z.string().length(2).default("en"),
|
|
12768
|
-
maxResults: import_zod16.z.number().int().min(1).max(50).default(10),
|
|
12769
|
-
proxyMode: import_zod16.z.enum(["location", "configured", "none"]).default("location"),
|
|
12770
|
-
proxyZip: import_zod16.z.string().regex(/^\d{5}$/).optional(),
|
|
12771
|
-
debug: import_zod16.z.boolean().default(false),
|
|
12772
|
-
kernelApiKey: import_zod16.z.string().optional(),
|
|
12773
|
-
kernelProxyId: import_zod16.z.string().optional(),
|
|
12774
|
-
kernelProxyResolution: import_zod16.z.unknown().optional(),
|
|
12775
|
-
headless: import_zod16.z.boolean().default(true)
|
|
12776
|
-
});
|
|
12777
|
-
RawPAAItemSchema = import_zod16.z.object({
|
|
12778
|
-
question: import_zod16.z.string().min(1),
|
|
12779
|
-
answer: import_zod16.z.string().optional(),
|
|
12780
|
-
sourceTitle: import_zod16.z.string().optional(),
|
|
12781
|
-
sourceSite: import_zod16.z.string().optional(),
|
|
12782
|
-
sourceCite: import_zod16.z.string().optional()
|
|
12783
|
-
});
|
|
12784
|
-
RawMapsOverviewSchema = import_zod16.z.object({
|
|
12785
|
-
name: import_zod16.z.string().nullable(),
|
|
12786
|
-
rating: import_zod16.z.string().nullable(),
|
|
12787
|
-
reviewCount: import_zod16.z.string().nullable(),
|
|
12788
|
-
category: import_zod16.z.string().nullable(),
|
|
12789
|
-
address: import_zod16.z.string().nullable(),
|
|
12790
|
-
hoursSummary: import_zod16.z.string().nullable(),
|
|
12791
|
-
phone: import_zod16.z.string().nullable(),
|
|
12792
|
-
phoneDisplay: import_zod16.z.string().nullable(),
|
|
12793
|
-
website: import_zod16.z.string().nullable(),
|
|
12794
|
-
plusCode: import_zod16.z.string().nullable(),
|
|
12795
|
-
bookingUrl: import_zod16.z.string().nullable()
|
|
12796
|
-
});
|
|
12797
|
-
RawMapsHoursRowSchema = import_zod16.z.object({
|
|
12798
|
-
day: import_zod16.z.string(),
|
|
12799
|
-
hours: import_zod16.z.string()
|
|
12800
|
-
});
|
|
12801
|
-
RawMapsReviewStatsSchema = import_zod16.z.object({
|
|
12802
|
-
reviewHistogram: import_zod16.z.array(import_zod16.z.object({
|
|
12803
|
-
stars: import_zod16.z.number(),
|
|
12804
|
-
count: import_zod16.z.string()
|
|
12805
|
-
})),
|
|
12806
|
-
reviewTopics: import_zod16.z.array(import_zod16.z.object({
|
|
12807
|
-
label: import_zod16.z.string(),
|
|
12808
|
-
count: import_zod16.z.string()
|
|
12809
|
-
}))
|
|
12810
|
-
});
|
|
12811
|
-
RawMapsReviewCardSchema = import_zod16.z.object({
|
|
12812
|
-
reviewId: import_zod16.z.string(),
|
|
12813
|
-
author: import_zod16.z.string().nullable(),
|
|
12814
|
-
stars: import_zod16.z.string().nullable(),
|
|
12815
|
-
date: import_zod16.z.string().nullable(),
|
|
12816
|
-
text: import_zod16.z.string().nullable(),
|
|
12817
|
-
ownerResponse: import_zod16.z.string().nullable()
|
|
12818
|
-
});
|
|
12819
|
-
RawMapsAboutAttributeSchema = import_zod16.z.object({
|
|
12820
|
-
section: import_zod16.z.string(),
|
|
12821
|
-
attribute: import_zod16.z.string()
|
|
12822
|
-
});
|
|
12823
|
-
}
|
|
12824
|
-
});
|
|
12825
|
-
|
|
12826
12933
|
// src/extractor/MapsNavigator.ts
|
|
12827
12934
|
var MapsNavigator;
|
|
12828
12935
|
var init_MapsNavigator = __esm({
|
|
@@ -14432,13 +14539,25 @@ ${videoRows}`,
|
|
|
14432
14539
|
}
|
|
14433
14540
|
};
|
|
14434
14541
|
}
|
|
14542
|
+
function structuredTranscriptChunks(chunks) {
|
|
14543
|
+
return chunks.map((c) => ({
|
|
14544
|
+
startSec: Number.isFinite(c.timestamp?.[0]) ? c.timestamp[0] : 0,
|
|
14545
|
+
endSec: Number.isFinite(c.timestamp?.[1]) ? c.timestamp[1] : 0,
|
|
14546
|
+
text: c.text
|
|
14547
|
+
}));
|
|
14548
|
+
}
|
|
14549
|
+
function wordCount(text) {
|
|
14550
|
+
return text.trim() ? text.trim().split(/\s+/).length : 0;
|
|
14551
|
+
}
|
|
14435
14552
|
function formatYoutubeTranscribe(raw, input) {
|
|
14436
14553
|
const parsed = parseData(raw);
|
|
14437
14554
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
14438
14555
|
const d = parsed.data;
|
|
14439
14556
|
const text = d.text ?? "";
|
|
14440
14557
|
const chunks = d.chunks ?? [];
|
|
14558
|
+
const videoId = d.videoId ?? input.videoId ?? null;
|
|
14441
14559
|
const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
|
|
14560
|
+
const words = wordCount(text);
|
|
14442
14561
|
const chunkRows = chunks.slice(0, 50).map((c) => {
|
|
14443
14562
|
const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
|
|
14444
14563
|
const mm = String(Math.floor(sec / 60)).padStart(2, "0");
|
|
@@ -14446,8 +14565,8 @@ function formatYoutubeTranscribe(raw, input) {
|
|
|
14446
14565
|
return `| ${mm}:${ss} | ${cell(truncate(c.text, 120))} |`;
|
|
14447
14566
|
}).join("\n");
|
|
14448
14567
|
const full = [
|
|
14449
|
-
`# YouTube Transcript: \`${input.
|
|
14450
|
-
`**Duration:** ${durSec}s \xB7 **${
|
|
14568
|
+
`# YouTube Transcript: \`${videoId ?? input.url ?? "video"}\``,
|
|
14569
|
+
`**Duration:** ${durSec}s \xB7 **${words} words**`,
|
|
14451
14570
|
`
|
|
14452
14571
|
## Full Transcript
|
|
14453
14572
|
${text}`,
|
|
@@ -14460,7 +14579,22 @@ ${chunkRows}` : "",
|
|
|
14460
14579
|
---
|
|
14461
14580
|
\u{1F4A1} Harvest more from this channel: use \`youtube_harvest\` with \`mode: "channel"\``
|
|
14462
14581
|
].filter(Boolean).join("\n");
|
|
14463
|
-
return
|
|
14582
|
+
return {
|
|
14583
|
+
...oneBlock(full),
|
|
14584
|
+
structuredContent: {
|
|
14585
|
+
videoId,
|
|
14586
|
+
url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : input.url ?? null,
|
|
14587
|
+
wordCount: words,
|
|
14588
|
+
chunkCount: chunks.length,
|
|
14589
|
+
durationMs: typeof d.durationMs === "number" ? d.durationMs : null,
|
|
14590
|
+
transcriptText: text,
|
|
14591
|
+
chunks: structuredTranscriptChunks(chunks),
|
|
14592
|
+
resolvedInputs: {
|
|
14593
|
+
videoId,
|
|
14594
|
+
url: input.url ?? null
|
|
14595
|
+
}
|
|
14596
|
+
}
|
|
14597
|
+
};
|
|
14464
14598
|
}
|
|
14465
14599
|
function formatFacebookPageIntel(raw, input) {
|
|
14466
14600
|
const parsed = parseData(raw);
|
|
@@ -14470,12 +14604,13 @@ function formatFacebookPageIntel(raw, input) {
|
|
|
14470
14604
|
const ads = d.ads ?? [];
|
|
14471
14605
|
const s = d.summary ?? { totalAds: 0, activeCount: 0, videoCount: 0, imageCount: 0 };
|
|
14472
14606
|
const adBlocks = ads.map((ad, i) => [
|
|
14473
|
-
`### Ad ${i + 1}${ad.libraryId ? ` \xB7 \`${ad.libraryId}\`` : ""} \u2014 ${ad.status ?? "\u2014"} \xB7 ${ad.creativeType ?? "\u2014"} \xB7 ${ad.startDate ?? "\u2014"}`,
|
|
14607
|
+
`### Ad ${i + 1}${ad.libraryId ? ` \xB7 \`${ad.libraryId}\`` : ""} \u2014 ${ad.status ?? "\u2014"} \xB7 ${ad.creativeType ?? "\u2014"} \xB7 ${ad.startDate ?? ad.started ?? "\u2014"}`,
|
|
14474
14608
|
ad.headline ? `**Headline:** ${ad.headline}` : "",
|
|
14475
14609
|
ad.primaryText ? `**Copy:** ${truncate(ad.primaryText, 200)}` : "",
|
|
14476
14610
|
ad.cta ? `**CTA:** ${ad.cta}` : "",
|
|
14477
|
-
ad.
|
|
14478
|
-
ad.
|
|
14611
|
+
ad.landingUrl ? `**Landing URL:** ${ad.landingUrl}` : "",
|
|
14612
|
+
ad.videoUrl ?? ad.videoSrc ? `**Video URL:** \`${ad.videoUrl ?? ad.videoSrc}\`` : "",
|
|
14613
|
+
ad.variations ?? ad.clusterCount ? `**Variations:** ${ad.variations ?? ad.clusterCount}` : ""
|
|
14479
14614
|
].filter(Boolean).join("\n")).join("\n\n---\n\n");
|
|
14480
14615
|
const full = [
|
|
14481
14616
|
`# Facebook Ad Intel: ${advertiser}`,
|
|
@@ -14501,11 +14636,16 @@ ${adBlocks}`,
|
|
|
14501
14636
|
libraryId: ad.libraryId ?? null,
|
|
14502
14637
|
status: ad.status ?? null,
|
|
14503
14638
|
creativeType: ad.creativeType ?? null,
|
|
14639
|
+
primaryText: ad.primaryText ?? null,
|
|
14504
14640
|
headline: ad.headline ?? null,
|
|
14505
14641
|
cta: ad.cta ?? null,
|
|
14506
|
-
startDate: ad.startDate ?? null,
|
|
14507
|
-
|
|
14508
|
-
|
|
14642
|
+
startDate: ad.startDate ?? ad.started ?? null,
|
|
14643
|
+
landingUrl: ad.landingUrl ?? null,
|
|
14644
|
+
domain: ad.domain ?? null,
|
|
14645
|
+
videoUrl: ad.videoUrl ?? ad.videoSrc ?? null,
|
|
14646
|
+
imageUrl: ad.imageUrl ?? ad.imageSrc ?? null,
|
|
14647
|
+
videoPoster: ad.videoPoster ?? null,
|
|
14648
|
+
variations: typeof ad.variations === "number" ? ad.variations : typeof ad.clusterCount === "number" ? ad.clusterCount : null
|
|
14509
14649
|
}))
|
|
14510
14650
|
}
|
|
14511
14651
|
};
|
|
@@ -14539,8 +14679,11 @@ ${rows}`,
|
|
|
14539
14679
|
advertiserCount: advertisers.length,
|
|
14540
14680
|
advertisers: advertisers.map((a) => ({
|
|
14541
14681
|
name: a.pageName ?? a.name ?? null,
|
|
14682
|
+
pageId: a.pageId ?? null,
|
|
14683
|
+
pageUrl: a.pageUrl ?? null,
|
|
14542
14684
|
adCount: typeof a.adCount === "number" ? a.adCount : null,
|
|
14543
|
-
libraryId: a.sampleLibraryId ?? a.libraryId ?? null
|
|
14685
|
+
libraryId: a.sampleLibraryId ?? a.libraryId ?? null,
|
|
14686
|
+
sampleLibraryId: a.sampleLibraryId ?? null
|
|
14544
14687
|
}))
|
|
14545
14688
|
}
|
|
14546
14689
|
};
|
|
@@ -14638,6 +14781,34 @@ function formatWorkflowSuggest(input) {
|
|
|
14638
14781
|
}
|
|
14639
14782
|
};
|
|
14640
14783
|
}
|
|
14784
|
+
function workflowStepLines(data) {
|
|
14785
|
+
const step = data.step;
|
|
14786
|
+
const nextStep = data.nextStep;
|
|
14787
|
+
const done = data.done === true;
|
|
14788
|
+
const lines = [];
|
|
14789
|
+
if (step) {
|
|
14790
|
+
const index = Number(step.index ?? 0);
|
|
14791
|
+
const totalSteps = step.totalSteps != null ? Number(step.totalSteps) : void 0;
|
|
14792
|
+
const stepLabel = totalSteps ? `${index + 1}/${totalSteps}` : `${index + 1}`;
|
|
14793
|
+
lines.push(`
|
|
14794
|
+
## Step ${stepLabel}: ${step.title ?? step.id ?? ""}`);
|
|
14795
|
+
const output = step.output;
|
|
14796
|
+
if (output && Object.keys(output).length) {
|
|
14797
|
+
lines.push(Object.entries(output).map(([k, v]) => `- ${k}: ${typeof v === "object" ? JSON.stringify(v) : String(v)}`).join("\n"));
|
|
14798
|
+
}
|
|
14799
|
+
const warnings = Array.isArray(step.warnings) ? step.warnings : [];
|
|
14800
|
+
if (warnings.length) lines.push(`
|
|
14801
|
+
**Warnings:**
|
|
14802
|
+
${warnings.map((w) => `- ${w}`).join("\n")}`);
|
|
14803
|
+
}
|
|
14804
|
+
if (done) {
|
|
14805
|
+
lines.push("\n**Done.** All steps complete.");
|
|
14806
|
+
} else if (nextStep && typeof nextStep === "object") {
|
|
14807
|
+
lines.push(`
|
|
14808
|
+
**Next step:** \`${nextStep.id ?? nextStep.index}\` \u2014 call \`workflow_step\` with this run id to continue.`);
|
|
14809
|
+
}
|
|
14810
|
+
return lines;
|
|
14811
|
+
}
|
|
14641
14812
|
function formatWorkflowRun(raw, input) {
|
|
14642
14813
|
const parsed = parseData(raw);
|
|
14643
14814
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
@@ -14651,6 +14822,7 @@ function formatWorkflowRun(raw, input) {
|
|
|
14651
14822
|
`**Run ID:** \`${runId || "unknown"}\``,
|
|
14652
14823
|
`**Status:** ${status}`,
|
|
14653
14824
|
summary?.title ? `**Title:** ${summary.title}` : "",
|
|
14825
|
+
...workflowStepLines(parsed.data),
|
|
14654
14826
|
summary?.summary ? `
|
|
14655
14827
|
## Summary
|
|
14656
14828
|
${summary.summary}` : "",
|
|
@@ -14666,6 +14838,42 @@ ${workflowArtifactRows(artifacts)}` : "",
|
|
|
14666
14838
|
input: input.input ?? {},
|
|
14667
14839
|
run,
|
|
14668
14840
|
summary,
|
|
14841
|
+
step: parsed.data.step,
|
|
14842
|
+
nextStep: parsed.data.nextStep ?? null,
|
|
14843
|
+
done: parsed.data.done === true,
|
|
14844
|
+
artifacts
|
|
14845
|
+
}
|
|
14846
|
+
};
|
|
14847
|
+
}
|
|
14848
|
+
function formatWorkflowStep(raw, input) {
|
|
14849
|
+
const parsed = parseData(raw);
|
|
14850
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
14851
|
+
const run = parsed.data.run;
|
|
14852
|
+
const summary = parsed.data.summary;
|
|
14853
|
+
const artifacts = workflowArtifactsFrom(run);
|
|
14854
|
+
const done = parsed.data.done === true;
|
|
14855
|
+
const full = [
|
|
14856
|
+
`# Workflow Step`,
|
|
14857
|
+
`**Run ID:** \`${input.runId}\``,
|
|
14858
|
+
`**Status:** ${run?.status ?? (done ? "done" : "running")}`,
|
|
14859
|
+
...workflowStepLines(parsed.data),
|
|
14860
|
+
done && summary?.summary ? `
|
|
14861
|
+
## Summary
|
|
14862
|
+
${summary.summary}` : "",
|
|
14863
|
+
artifacts.length ? `
|
|
14864
|
+
## Artifacts
|
|
14865
|
+
${workflowArtifactRows(artifacts)}` : "",
|
|
14866
|
+
done && artifacts.length ? "\nUse `workflow_artifact_read` with the run id and artifact id to pull CSV, JSON, Markdown, or report content into context." : ""
|
|
14867
|
+
].filter(Boolean).join("\n");
|
|
14868
|
+
return {
|
|
14869
|
+
...oneBlock(full),
|
|
14870
|
+
structuredContent: {
|
|
14871
|
+
runId: input.runId,
|
|
14872
|
+
run,
|
|
14873
|
+
summary: summary ?? null,
|
|
14874
|
+
step: parsed.data.step,
|
|
14875
|
+
nextStep: parsed.data.nextStep ?? null,
|
|
14876
|
+
done,
|
|
14669
14877
|
artifacts
|
|
14670
14878
|
}
|
|
14671
14879
|
};
|
|
@@ -15070,6 +15278,7 @@ function formatFacebookAdTranscribe(raw, input) {
|
|
|
15070
15278
|
const text = d.text ?? "";
|
|
15071
15279
|
const chunks = d.chunks ?? [];
|
|
15072
15280
|
const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
|
|
15281
|
+
const words = wordCount(text);
|
|
15073
15282
|
const chunkRows = chunks.slice(0, 50).map((c) => {
|
|
15074
15283
|
const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
|
|
15075
15284
|
const mm = String(Math.floor(sec / 60)).padStart(2, "0");
|
|
@@ -15078,7 +15287,7 @@ function formatFacebookAdTranscribe(raw, input) {
|
|
|
15078
15287
|
}).join("\n");
|
|
15079
15288
|
const full = [
|
|
15080
15289
|
`# Facebook Ad Transcript`,
|
|
15081
|
-
`**Duration:** ${durSec}s \xB7 **${
|
|
15290
|
+
`**Duration:** ${durSec}s \xB7 **${words} words**`,
|
|
15082
15291
|
`
|
|
15083
15292
|
## Full Transcript
|
|
15084
15293
|
${text}`,
|
|
@@ -15091,7 +15300,20 @@ ${chunkRows}` : "",
|
|
|
15091
15300
|
---
|
|
15092
15301
|
\u{1F4A1} Get more ads from this advertiser: use \`facebook_page_intel\`. For public Facebook reel/post URLs, use \`facebook_video_transcribe\`.`
|
|
15093
15302
|
].filter(Boolean).join("\n");
|
|
15094
|
-
return
|
|
15303
|
+
return {
|
|
15304
|
+
...oneBlock(full),
|
|
15305
|
+
structuredContent: {
|
|
15306
|
+
videoUrl: input.videoUrl,
|
|
15307
|
+
wordCount: words,
|
|
15308
|
+
chunkCount: chunks.length,
|
|
15309
|
+
durationMs: typeof d.durationMs === "number" ? d.durationMs : null,
|
|
15310
|
+
transcriptText: text,
|
|
15311
|
+
chunks: structuredTranscriptChunks(chunks),
|
|
15312
|
+
resolvedInputs: {
|
|
15313
|
+
videoUrl: input.videoUrl
|
|
15314
|
+
}
|
|
15315
|
+
}
|
|
15316
|
+
};
|
|
15095
15317
|
}
|
|
15096
15318
|
function formatFacebookVideoTranscribe(raw, input) {
|
|
15097
15319
|
const parsed = parseData(raw);
|
|
@@ -15099,7 +15321,7 @@ function formatFacebookVideoTranscribe(raw, input) {
|
|
|
15099
15321
|
const d = parsed.data;
|
|
15100
15322
|
const text = d.text ?? "";
|
|
15101
15323
|
const chunks = d.chunks ?? [];
|
|
15102
|
-
const
|
|
15324
|
+
const wordCount2 = text.trim() ? text.trim().split(/\s+/).length : 0;
|
|
15103
15325
|
const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
|
|
15104
15326
|
const videoDuration = typeof d.videoDurationSec === "number" ? `${Math.round(d.videoDurationSec)}s` : "\u2014";
|
|
15105
15327
|
const label = d.videoId ? `Facebook Organic Video \`${d.videoId}\`` : "Facebook Organic Video";
|
|
@@ -15112,7 +15334,7 @@ function formatFacebookVideoTranscribe(raw, input) {
|
|
|
15112
15334
|
const full = [
|
|
15113
15335
|
`# ${label} Transcript`,
|
|
15114
15336
|
d.ownerName ? `**Owner:** ${d.ownerName}` : "",
|
|
15115
|
-
`**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${
|
|
15337
|
+
`**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${wordCount2} words**`,
|
|
15116
15338
|
d.pageUrl ? `**Page URL:** ${d.pageUrl}` : `**Page URL:** ${input.url}`,
|
|
15117
15339
|
d.videoUrl ? `**Extracted MP4:** \`${d.videoUrl}\`` : "",
|
|
15118
15340
|
`
|
|
@@ -15138,7 +15360,7 @@ ${chunkRows}` : "",
|
|
|
15138
15360
|
bitrate: typeof d.bitrate === "number" ? d.bitrate : null,
|
|
15139
15361
|
videoDurationSec: typeof d.videoDurationSec === "number" ? d.videoDurationSec : null,
|
|
15140
15362
|
videoUrl: d.videoUrl ?? "",
|
|
15141
|
-
wordCount,
|
|
15363
|
+
wordCount: wordCount2,
|
|
15142
15364
|
chunkCount: chunks.length,
|
|
15143
15365
|
transcriptText: text,
|
|
15144
15366
|
chunks: chunks.map((c) => ({
|
|
@@ -15149,6 +15371,96 @@ ${chunkRows}` : "",
|
|
|
15149
15371
|
}
|
|
15150
15372
|
};
|
|
15151
15373
|
}
|
|
15374
|
+
function formatCaptureSerpSnapshot(raw, input) {
|
|
15375
|
+
const parsed = parseData(raw);
|
|
15376
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
15377
|
+
const d = parsed.data;
|
|
15378
|
+
const providerPayload = d;
|
|
15379
|
+
const organic = Array.isArray(d.organicResults) ? d.organicResults : Array.isArray(d.organic) ? d.organic : [];
|
|
15380
|
+
const localPack = Array.isArray(d.localPack) ? d.localPack : [];
|
|
15381
|
+
const artifacts = Array.isArray(d.artifacts) ? d.artifacts : [];
|
|
15382
|
+
const status = String(d.status ?? d.captureStatus ?? "captured");
|
|
15383
|
+
const query = typeof d.query === "string" ? d.query : typeof input.query === "string" ? input.query : null;
|
|
15384
|
+
const location2 = typeof d.location === "string" ? d.location : typeof input.location === "string" ? input.location : null;
|
|
15385
|
+
const capturedAt = typeof d.capturedAt === "string" ? d.capturedAt : typeof d.extractedAt === "string" ? d.extractedAt : null;
|
|
15386
|
+
const resultCount = organic.length + localPack.length;
|
|
15387
|
+
const full = [
|
|
15388
|
+
`# SERP Intelligence Snapshot: ${query ?? "query"}`,
|
|
15389
|
+
`**Status:** ${status}`,
|
|
15390
|
+
location2 ? `**Location:** ${location2}` : "",
|
|
15391
|
+
`**Result count:** ${resultCount}`,
|
|
15392
|
+
artifacts.length ? `**Artifacts:** ${artifacts.length}` : "",
|
|
15393
|
+
"",
|
|
15394
|
+
"Use `capture_serp_page_snapshots` when you need page-level evidence for ranking URLs."
|
|
15395
|
+
].filter(Boolean).join("\n");
|
|
15396
|
+
return {
|
|
15397
|
+
...oneBlock(full),
|
|
15398
|
+
structuredContent: {
|
|
15399
|
+
schemaVersion: "serp-intelligence.capture.v1",
|
|
15400
|
+
status,
|
|
15401
|
+
query,
|
|
15402
|
+
location: location2,
|
|
15403
|
+
capturedAt,
|
|
15404
|
+
resultCount,
|
|
15405
|
+
snapshotId: typeof d.snapshotId === "string" ? d.snapshotId : typeof d.snapshot_id === "string" ? d.snapshot_id : typeof d.id === "string" ? d.id : null,
|
|
15406
|
+
resolvedInputs: {
|
|
15407
|
+
query: input.query ?? null,
|
|
15408
|
+
location: input.location ?? null,
|
|
15409
|
+
gl: input.gl ?? null,
|
|
15410
|
+
hl: input.hl ?? null,
|
|
15411
|
+
device: input.device ?? null,
|
|
15412
|
+
proxyMode: input.proxyMode ?? null,
|
|
15413
|
+
proxyZip: input.proxyZip ?? null,
|
|
15414
|
+
pages: input.pages ?? null
|
|
15415
|
+
},
|
|
15416
|
+
artifacts,
|
|
15417
|
+
diagnostics: d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null,
|
|
15418
|
+
providerPayload
|
|
15419
|
+
}
|
|
15420
|
+
};
|
|
15421
|
+
}
|
|
15422
|
+
function formatCaptureSerpPageSnapshots(raw, input) {
|
|
15423
|
+
const parsed = parseData(raw);
|
|
15424
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
15425
|
+
const d = parsed.data;
|
|
15426
|
+
const captures = Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
|
|
15427
|
+
const failedCount = captures.filter((c) => c.status === "failed" || c.error).length;
|
|
15428
|
+
const status = String(d.status ?? (failedCount ? "partial" : "captured"));
|
|
15429
|
+
const rows = captures.slice(0, 25).map((capture, i) => {
|
|
15430
|
+
const url = typeof capture.url === "string" ? capture.url : "";
|
|
15431
|
+
const sourceKind = typeof capture.sourceKind === "string" ? capture.sourceKind : typeof capture.source_kind === "string" ? capture.source_kind : "configured_target";
|
|
15432
|
+
return `| ${i + 1} | ${cell(url)} | ${cell(sourceKind)} | ${cell(String(capture.status ?? (capture.error ? "failed" : "ok")))} |`;
|
|
15433
|
+
}).join("\n");
|
|
15434
|
+
const full = [
|
|
15435
|
+
"# SERP Intelligence Page Snapshots",
|
|
15436
|
+
`**Status:** ${status}`,
|
|
15437
|
+
`**Captured:** ${captures.length}`,
|
|
15438
|
+
`**Failed:** ${failedCount}`,
|
|
15439
|
+
captures.length ? `
|
|
15440
|
+
| # | URL | Source | Status |
|
|
15441
|
+
|---|-----|--------|--------|
|
|
15442
|
+
${rows}` : ""
|
|
15443
|
+
].filter(Boolean).join("\n");
|
|
15444
|
+
return {
|
|
15445
|
+
...oneBlock(full),
|
|
15446
|
+
structuredContent: {
|
|
15447
|
+
schemaVersion: "serp-intelligence.page-snapshots.v1",
|
|
15448
|
+
status,
|
|
15449
|
+
count: captures.length,
|
|
15450
|
+
failedCount,
|
|
15451
|
+
captures,
|
|
15452
|
+
resolvedInputs: {
|
|
15453
|
+
urls: input.urls ?? null,
|
|
15454
|
+
targets: input.targets ?? null,
|
|
15455
|
+
maxConcurrency: input.maxConcurrency ?? null,
|
|
15456
|
+
timeoutMs: input.timeoutMs ?? null,
|
|
15457
|
+
debug: input.debug ?? null
|
|
15458
|
+
},
|
|
15459
|
+
diagnostics: d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null,
|
|
15460
|
+
providerPayload: d
|
|
15461
|
+
}
|
|
15462
|
+
};
|
|
15463
|
+
}
|
|
15152
15464
|
var import_node_fs3, import_node_os3, import_node_path5, reportSavingEnabled;
|
|
15153
15465
|
var init_mcp_response_formatter = __esm({
|
|
15154
15466
|
"src/mcp/mcp-response-formatter.ts"() {
|
|
@@ -15623,6 +15935,7 @@ var init_directory_workflow = __esm({
|
|
|
15623
15935
|
init_mcp_response_formatter();
|
|
15624
15936
|
init_browser_service_env();
|
|
15625
15937
|
init_maps_search_rotation();
|
|
15938
|
+
init_schemas3();
|
|
15626
15939
|
init_csv();
|
|
15627
15940
|
init_location_db();
|
|
15628
15941
|
DirectoryWorkflowOptionsSchema = import_zod18.z.object({
|
|
@@ -15638,7 +15951,7 @@ var init_directory_workflow = __esm({
|
|
|
15638
15951
|
saveCsv: import_zod18.z.boolean().default(true),
|
|
15639
15952
|
gl: import_zod18.z.string().length(2).default("us"),
|
|
15640
15953
|
hl: import_zod18.z.string().length(2).default("en"),
|
|
15641
|
-
proxyMode: import_zod18.z.enum(["location", "configured", "none"]).default(
|
|
15954
|
+
proxyMode: import_zod18.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
15642
15955
|
proxyZip: import_zod18.z.string().regex(/^\d{5}$/).optional(),
|
|
15643
15956
|
debug: import_zod18.z.boolean().default(false),
|
|
15644
15957
|
headless: import_zod18.z.boolean().default(true),
|
|
@@ -16062,102 +16375,141 @@ var init_agent_packet = __esm({
|
|
|
16062
16375
|
title: "Agent-Ready SEO Packet",
|
|
16063
16376
|
description: "Create an evidence folder for AI agents from live SERP/PAA/AI search surfaces.",
|
|
16064
16377
|
inputSchema: AgentPacketInputSchema,
|
|
16065
|
-
|
|
16066
|
-
|
|
16067
|
-
|
|
16068
|
-
|
|
16069
|
-
|
|
16070
|
-
|
|
16071
|
-
|
|
16072
|
-
|
|
16073
|
-
|
|
16074
|
-
|
|
16075
|
-
|
|
16076
|
-
|
|
16077
|
-
|
|
16078
|
-
|
|
16079
|
-
|
|
16080
|
-
|
|
16081
|
-
|
|
16378
|
+
createState: () => ({ serp: null, paa: null, warnings: [] }),
|
|
16379
|
+
steps: [
|
|
16380
|
+
{
|
|
16381
|
+
id: "harvest-serp",
|
|
16382
|
+
title: "Harvest organic SERP + AI Overview",
|
|
16383
|
+
async run({ input, state, ctx }) {
|
|
16384
|
+
if (!input.includeSerp) {
|
|
16385
|
+
return { state, output: { skipped: true, reason: "includeSerp is false" } };
|
|
16386
|
+
}
|
|
16387
|
+
try {
|
|
16388
|
+
const serp = await ctx.client.post("/harvest/sync", {
|
|
16389
|
+
query: input.keyword,
|
|
16390
|
+
location: input.location,
|
|
16391
|
+
serpOnly: true,
|
|
16392
|
+
maxQuestions: 1,
|
|
16393
|
+
format: "json"
|
|
16394
|
+
}, 18e4);
|
|
16395
|
+
await ctx.artifacts.writeJson("SERP raw JSON", "raw/serp.json", serp);
|
|
16396
|
+
return {
|
|
16397
|
+
state: { ...state, serp },
|
|
16398
|
+
output: {
|
|
16399
|
+
organicResults: serp.organicResults?.length ?? 0,
|
|
16400
|
+
localPack: serp.localPack?.length ?? 0,
|
|
16401
|
+
aiOverviewDetected: Boolean(serp.aiOverview?.detected),
|
|
16402
|
+
aiOverviewCitations: serp.aiOverview?.citations?.length ?? 0
|
|
16403
|
+
}
|
|
16404
|
+
};
|
|
16405
|
+
} catch (err) {
|
|
16406
|
+
const message = `SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
|
|
16407
|
+
return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
|
|
16408
|
+
}
|
|
16082
16409
|
}
|
|
16083
|
-
}
|
|
16084
|
-
|
|
16085
|
-
|
|
16086
|
-
|
|
16087
|
-
|
|
16088
|
-
|
|
16089
|
-
|
|
16090
|
-
|
|
16091
|
-
|
|
16092
|
-
|
|
16093
|
-
|
|
16410
|
+
},
|
|
16411
|
+
{
|
|
16412
|
+
id: "harvest-paa",
|
|
16413
|
+
title: "Harvest People Also Ask",
|
|
16414
|
+
async run({ input, state, ctx }) {
|
|
16415
|
+
if (!input.includePaa) {
|
|
16416
|
+
return { state, output: { skipped: true, reason: "includePaa is false" } };
|
|
16417
|
+
}
|
|
16418
|
+
try {
|
|
16419
|
+
const paa = await ctx.client.post("/harvest/sync", {
|
|
16420
|
+
query: input.keyword,
|
|
16421
|
+
location: input.location,
|
|
16422
|
+
maxQuestions: input.maxQuestions,
|
|
16423
|
+
format: "json"
|
|
16424
|
+
}, 28e4);
|
|
16425
|
+
await ctx.artifacts.writeJson("PAA raw JSON", "raw/paa.json", paa);
|
|
16426
|
+
return {
|
|
16427
|
+
state: { ...state, paa },
|
|
16428
|
+
output: { paaQuestions: paa.flat?.length ?? 0 }
|
|
16429
|
+
};
|
|
16430
|
+
} catch (err) {
|
|
16431
|
+
const message = `PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
|
|
16432
|
+
return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
|
|
16433
|
+
}
|
|
16434
|
+
}
|
|
16435
|
+
},
|
|
16436
|
+
{
|
|
16437
|
+
id: "assemble",
|
|
16438
|
+
title: "Assemble evidence packet",
|
|
16439
|
+
async run({ input, state, ctx }) {
|
|
16440
|
+
const { serp, paa } = state;
|
|
16441
|
+
const warnings = state.warnings;
|
|
16442
|
+
if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
|
|
16443
|
+
const rows = sourceRows(input, serp, paa);
|
|
16444
|
+
const target = normalizeDomain(input.domain);
|
|
16445
|
+
const competitors = competitorRows(rows, target);
|
|
16446
|
+
const evidence = {
|
|
16447
|
+
input,
|
|
16448
|
+
serp: serp ?? { status: "skipped" },
|
|
16449
|
+
paa: paa ?? { status: "skipped" },
|
|
16450
|
+
target: {
|
|
16451
|
+
domain: target,
|
|
16452
|
+
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
16453
|
+
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
16454
|
+
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
16455
|
+
},
|
|
16456
|
+
competitors,
|
|
16457
|
+
warnings
|
|
16458
|
+
};
|
|
16459
|
+
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
16460
|
+
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
16461
|
+
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
16462
|
+
const brief = [
|
|
16463
|
+
`# SEO Evidence Brief: ${input.keyword}`,
|
|
16464
|
+
"",
|
|
16465
|
+
`Location: ${input.location ?? "not specified"}`,
|
|
16466
|
+
`Target domain: ${target ?? "not specified"}`,
|
|
16467
|
+
"",
|
|
16468
|
+
"## Evidence Summary",
|
|
16469
|
+
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
16470
|
+
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
16471
|
+
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
16472
|
+
`- Competitor domains: ${competitors.length}`,
|
|
16473
|
+
"",
|
|
16474
|
+
"## Recommended Use",
|
|
16475
|
+
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
16476
|
+
].join("\n");
|
|
16477
|
+
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
16478
|
+
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
16479
|
+
"# Agent Tasks",
|
|
16480
|
+
"",
|
|
16481
|
+
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
16482
|
+
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
16483
|
+
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
16484
|
+
"- [ ] Mark unsupported recommendations as assumptions."
|
|
16485
|
+
].join("\n"));
|
|
16486
|
+
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
16487
|
+
"# Agent Instructions",
|
|
16488
|
+
"",
|
|
16489
|
+
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
16490
|
+
].join("\n"));
|
|
16491
|
+
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
16492
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
16493
|
+
title: "Agent-Ready SEO Packet",
|
|
16494
|
+
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
16495
|
+
summary,
|
|
16496
|
+
warnings,
|
|
16497
|
+
tables: [
|
|
16498
|
+
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
16499
|
+
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
16500
|
+
]
|
|
16501
|
+
}));
|
|
16502
|
+
const status = warnings.length ? "partial" : "succeeded";
|
|
16503
|
+
const counts = { sources: rows.length, competitors: competitors.length };
|
|
16504
|
+
await ctx.artifacts.writeManifest(status, counts, warnings, []);
|
|
16505
|
+
return {
|
|
16506
|
+
state,
|
|
16507
|
+
output: { sources: rows.length, competitors: competitors.length, status },
|
|
16508
|
+
summary: { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors: [], reportPath }
|
|
16509
|
+
};
|
|
16094
16510
|
}
|
|
16095
16511
|
}
|
|
16096
|
-
|
|
16097
|
-
const rows = sourceRows(input, serp, paa);
|
|
16098
|
-
const target = normalizeDomain(input.domain);
|
|
16099
|
-
const competitors = competitorRows(rows, target);
|
|
16100
|
-
const evidence = {
|
|
16101
|
-
input,
|
|
16102
|
-
serp: serp ?? { status: "skipped" },
|
|
16103
|
-
paa: paa ?? { status: "skipped" },
|
|
16104
|
-
target: {
|
|
16105
|
-
domain: target,
|
|
16106
|
-
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
16107
|
-
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
16108
|
-
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
16109
|
-
},
|
|
16110
|
-
competitors,
|
|
16111
|
-
warnings
|
|
16112
|
-
};
|
|
16113
|
-
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
16114
|
-
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
16115
|
-
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
16116
|
-
const brief = [
|
|
16117
|
-
`# SEO Evidence Brief: ${input.keyword}`,
|
|
16118
|
-
"",
|
|
16119
|
-
`Location: ${input.location ?? "not specified"}`,
|
|
16120
|
-
`Target domain: ${target ?? "not specified"}`,
|
|
16121
|
-
"",
|
|
16122
|
-
"## Evidence Summary",
|
|
16123
|
-
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
16124
|
-
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
16125
|
-
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
16126
|
-
`- Competitor domains: ${competitors.length}`,
|
|
16127
|
-
"",
|
|
16128
|
-
"## Recommended Use",
|
|
16129
|
-
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
16130
|
-
].join("\n");
|
|
16131
|
-
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
16132
|
-
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
16133
|
-
"# Agent Tasks",
|
|
16134
|
-
"",
|
|
16135
|
-
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
16136
|
-
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
16137
|
-
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
16138
|
-
"- [ ] Mark unsupported recommendations as assumptions."
|
|
16139
|
-
].join("\n"));
|
|
16140
|
-
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
16141
|
-
"# Agent Instructions",
|
|
16142
|
-
"",
|
|
16143
|
-
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
16144
|
-
].join("\n"));
|
|
16145
|
-
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
16146
|
-
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
16147
|
-
title: "Agent-Ready SEO Packet",
|
|
16148
|
-
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
16149
|
-
summary,
|
|
16150
|
-
warnings,
|
|
16151
|
-
tables: [
|
|
16152
|
-
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
16153
|
-
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
16154
|
-
]
|
|
16155
|
-
}));
|
|
16156
|
-
const status = warnings.length ? "partial" : "succeeded";
|
|
16157
|
-
const counts = { sources: rows.length, competitors: competitors.length };
|
|
16158
|
-
await ctx.artifacts.writeManifest(status, counts, warnings, errors);
|
|
16159
|
-
return { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors, reportPath };
|
|
16160
|
-
}
|
|
16512
|
+
]
|
|
16161
16513
|
};
|
|
16162
16514
|
}
|
|
16163
16515
|
});
|
|
@@ -16220,6 +16572,7 @@ var init_directory = __esm({
|
|
|
16220
16572
|
"use strict";
|
|
16221
16573
|
import_zod20 = require("zod");
|
|
16222
16574
|
init_report_renderer();
|
|
16575
|
+
init_schemas3();
|
|
16223
16576
|
DirectoryWorkflowCliInputSchema = import_zod20.z.object({
|
|
16224
16577
|
query: import_zod20.z.string().min(1),
|
|
16225
16578
|
state: import_zod20.z.string().min(2).default("TN"),
|
|
@@ -16227,7 +16580,7 @@ var init_directory = __esm({
|
|
|
16227
16580
|
maxCities: import_zod20.z.number().int().min(1).max(100).default(25),
|
|
16228
16581
|
maxResultsPerCity: import_zod20.z.number().int().min(1).max(50).default(20),
|
|
16229
16582
|
concurrency: import_zod20.z.number().int().min(1).max(5).default(5),
|
|
16230
|
-
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default(
|
|
16583
|
+
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
16231
16584
|
saveCsv: import_zod20.z.boolean().default(true)
|
|
16232
16585
|
});
|
|
16233
16586
|
DIRECTORY_CSV_HEADERS = [
|
|
@@ -16344,6 +16697,7 @@ var init_local_competitive_audit = __esm({
|
|
|
16344
16697
|
import_zod21 = require("zod");
|
|
16345
16698
|
init_report_renderer();
|
|
16346
16699
|
init_directory();
|
|
16700
|
+
init_schemas3();
|
|
16347
16701
|
LocalCompetitiveAuditInputSchema = import_zod21.z.object({
|
|
16348
16702
|
query: import_zod21.z.string().min(1),
|
|
16349
16703
|
state: import_zod21.z.string().min(2).default("TN"),
|
|
@@ -16353,7 +16707,7 @@ var init_local_competitive_audit = __esm({
|
|
|
16353
16707
|
hydrateTop: import_zod21.z.number().int().min(0).max(10).default(5),
|
|
16354
16708
|
maxReviews: import_zod21.z.number().int().min(0).max(500).default(50),
|
|
16355
16709
|
concurrency: import_zod21.z.number().int().min(1).max(5).default(5),
|
|
16356
|
-
proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default(
|
|
16710
|
+
proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
16357
16711
|
returnPartial: import_zod21.z.boolean().default(true)
|
|
16358
16712
|
});
|
|
16359
16713
|
localCompetitiveAuditWorkflowDefinition = {
|
|
@@ -16805,6 +17159,7 @@ var init_comparison_briefs = __esm({
|
|
|
16805
17159
|
init_report_renderer();
|
|
16806
17160
|
init_directory();
|
|
16807
17161
|
init_seo_workflow_utils();
|
|
17162
|
+
init_schemas3();
|
|
16808
17163
|
ProxyModeSchema = import_zod22.z.enum(["location", "configured", "none"]);
|
|
16809
17164
|
MapComparisonInputSchema = import_zod22.z.object({
|
|
16810
17165
|
query: import_zod22.z.string().min(1),
|
|
@@ -16816,7 +17171,7 @@ var init_comparison_briefs = __esm({
|
|
|
16816
17171
|
hydrateTop: import_zod22.z.number().int().min(0).max(10).default(5),
|
|
16817
17172
|
maxReviews: import_zod22.z.number().int().min(0).max(500).default(25),
|
|
16818
17173
|
concurrency: import_zod22.z.number().int().min(1).max(5).default(5),
|
|
16819
|
-
proxyMode: ProxyModeSchema.default(
|
|
17174
|
+
proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
|
|
16820
17175
|
returnPartial: import_zod22.z.boolean().default(true)
|
|
16821
17176
|
}).refine((input) => input.location || input.state, {
|
|
16822
17177
|
message: "Either location or state is required for map-comparison"
|
|
@@ -17230,22 +17585,44 @@ function workflowDefinition(id) {
|
|
|
17230
17585
|
if (!definition) throw new Error(`Unknown workflow "${id}". Available: ${DEFINITIONS.map((def) => def.id).join(", ")}`);
|
|
17231
17586
|
return definition;
|
|
17232
17587
|
}
|
|
17233
|
-
|
|
17588
|
+
function workflowStepCount(id) {
|
|
17234
17589
|
const definition = workflowDefinition(id);
|
|
17235
|
-
|
|
17590
|
+
return definition.steps?.length ?? 1;
|
|
17591
|
+
}
|
|
17592
|
+
function workflowSupportsSteps(id) {
|
|
17593
|
+
const definition = workflowDefinition(id);
|
|
17594
|
+
return Boolean(definition.steps && definition.steps.length > 0);
|
|
17595
|
+
}
|
|
17596
|
+
function resolveApiKey(options) {
|
|
17236
17597
|
const apiKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim();
|
|
17237
17598
|
if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
|
|
17238
|
-
|
|
17599
|
+
return apiKey;
|
|
17600
|
+
}
|
|
17601
|
+
function resolveApiUrl(options) {
|
|
17602
|
+
return options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
17603
|
+
}
|
|
17604
|
+
async function runWorkflow(id, rawInput, options = {}) {
|
|
17605
|
+
const definition = workflowDefinition(id);
|
|
17606
|
+
const input = definition.inputSchema.parse(rawInput);
|
|
17607
|
+
const apiKey = resolveApiKey(options);
|
|
17608
|
+
const apiUrl = resolveApiUrl(options);
|
|
17239
17609
|
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
17240
17610
|
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
|
|
17611
|
+
const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client: client2, artifacts, signal: options.signal };
|
|
17241
17612
|
try {
|
|
17242
|
-
|
|
17243
|
-
|
|
17244
|
-
|
|
17245
|
-
|
|
17246
|
-
|
|
17247
|
-
|
|
17248
|
-
|
|
17613
|
+
if (definition.steps && definition.steps.length > 0) {
|
|
17614
|
+
let state = definition.createState ? definition.createState(input) : {};
|
|
17615
|
+
let summary = null;
|
|
17616
|
+
for (const step of definition.steps) {
|
|
17617
|
+
const outcome = await step.run({ input, state, ctx });
|
|
17618
|
+
state = outcome.state;
|
|
17619
|
+
if (outcome.summary) summary = outcome.summary;
|
|
17620
|
+
}
|
|
17621
|
+
if (!summary) throw new Error(`Workflow "${id}" produced no terminal step summary`);
|
|
17622
|
+
return summary;
|
|
17623
|
+
}
|
|
17624
|
+
if (definition.run) return await definition.run(input, ctx);
|
|
17625
|
+
throw new Error(`Workflow "${id}" has neither steps nor a run() implementation`);
|
|
17249
17626
|
} catch (err) {
|
|
17250
17627
|
const message = err instanceof Error ? err.message : String(err);
|
|
17251
17628
|
await artifacts.writeText("Failure", "summary.md", `# ${definition.title}
|
|
@@ -17256,10 +17633,50 @@ ${message}
|
|
|
17256
17633
|
throw err;
|
|
17257
17634
|
}
|
|
17258
17635
|
}
|
|
17259
|
-
|
|
17636
|
+
async function runWorkflowStep(id, rawInput, opts) {
|
|
17637
|
+
const definition = workflowDefinition(id);
|
|
17638
|
+
const steps = definition.steps;
|
|
17639
|
+
if (!steps || steps.length === 0) throw new Error(`Workflow "${id}" does not support stepwise execution`);
|
|
17640
|
+
if (opts.stepIndex < 0 || opts.stepIndex >= steps.length) {
|
|
17641
|
+
throw new Error(`Step index ${opts.stepIndex} is out of range for "${id}" (${steps.length} steps)`);
|
|
17642
|
+
}
|
|
17643
|
+
const options = opts.options ?? {};
|
|
17644
|
+
const input = definition.inputSchema.parse(rawInput);
|
|
17645
|
+
const apiKey = resolveApiKey(options);
|
|
17646
|
+
const apiUrl = resolveApiUrl(options);
|
|
17647
|
+
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
17648
|
+
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
|
|
17649
|
+
const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client: client2, artifacts, signal: options.signal };
|
|
17650
|
+
const state = opts.state ?? (definition.createState ? definition.createState(input) : {});
|
|
17651
|
+
const step = steps[opts.stepIndex];
|
|
17652
|
+
const outcome = await step.run({ input, state, ctx });
|
|
17653
|
+
const writtenArtifacts = await Promise.all(
|
|
17654
|
+
artifacts.artifacts.map(async (artifact) => ({
|
|
17655
|
+
...artifact,
|
|
17656
|
+
content: await (0, import_promises7.readFile)(artifact.path, "utf8").catch(() => "")
|
|
17657
|
+
}))
|
|
17658
|
+
);
|
|
17659
|
+
return {
|
|
17660
|
+
runId: artifacts.runId,
|
|
17661
|
+
workflowId: definition.id,
|
|
17662
|
+
title: definition.title,
|
|
17663
|
+
stepIndex: opts.stepIndex,
|
|
17664
|
+
stepId: step.id,
|
|
17665
|
+
stepTitle: step.title,
|
|
17666
|
+
totalSteps: steps.length,
|
|
17667
|
+
isLast: opts.stepIndex === steps.length - 1,
|
|
17668
|
+
output: outcome.output,
|
|
17669
|
+
state: outcome.state,
|
|
17670
|
+
warnings: outcome.warnings ?? [],
|
|
17671
|
+
summary: outcome.summary ?? null,
|
|
17672
|
+
artifacts: writtenArtifacts
|
|
17673
|
+
};
|
|
17674
|
+
}
|
|
17675
|
+
var import_promises7, import_zod23, DEFINITIONS;
|
|
17260
17676
|
var init_registry2 = __esm({
|
|
17261
17677
|
"src/workflows/registry.ts"() {
|
|
17262
17678
|
"use strict";
|
|
17679
|
+
import_promises7 = require("fs/promises");
|
|
17263
17680
|
import_zod23 = require("zod");
|
|
17264
17681
|
init_artifact_writer();
|
|
17265
17682
|
init_http_client2();
|
|
@@ -17328,7 +17745,7 @@ async function readManifestFromSummary(summary) {
|
|
|
17328
17745
|
if (!summary.reportPath) return null;
|
|
17329
17746
|
const manifestPath = summary.reportPath.replace(/report\.html$/, "manifest.json");
|
|
17330
17747
|
try {
|
|
17331
|
-
return JSON.parse(await (0,
|
|
17748
|
+
return JSON.parse(await (0, import_promises8.readFile)(manifestPath, "utf8"));
|
|
17332
17749
|
} catch {
|
|
17333
17750
|
return null;
|
|
17334
17751
|
}
|
|
@@ -17409,6 +17826,88 @@ async function executeWorkflowRun(input) {
|
|
|
17409
17826
|
throw err;
|
|
17410
17827
|
}
|
|
17411
17828
|
}
|
|
17829
|
+
async function runWorkflowStepAndPersist(input) {
|
|
17830
|
+
const runId = input.run.id;
|
|
17831
|
+
const workflowId = input.run.workflow_id;
|
|
17832
|
+
await markWorkflowRunRunning(runId);
|
|
17833
|
+
let result;
|
|
17834
|
+
try {
|
|
17835
|
+
result = await runWorkflowStep(workflowId, input.workflowInput, {
|
|
17836
|
+
stepIndex: input.stepIndex,
|
|
17837
|
+
state: input.priorState ?? void 0,
|
|
17838
|
+
options: {
|
|
17839
|
+
apiKey: input.user.api_key,
|
|
17840
|
+
apiUrl: input.apiUrl,
|
|
17841
|
+
outputDir: hostedWorkflowOutputDir(),
|
|
17842
|
+
runId,
|
|
17843
|
+
headers: input.concurrencyLockId ? { "x-mcp-scraper-concurrency-lock": input.concurrencyLockId } : void 0
|
|
17844
|
+
}
|
|
17845
|
+
});
|
|
17846
|
+
} catch (err) {
|
|
17847
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
17848
|
+
await failWorkflowRunRecord(runId, message, { error: message });
|
|
17849
|
+
const failedRun = await getWorkflowRun(runId, input.user.id);
|
|
17850
|
+
if (failedRun) await deliverWorkflowWebhook({ userId: input.user.id, run: failedRun, scheduleId: failedRun.schedule_id, webhookUrl: input.webhookUrl ?? null, summary: null, artifacts: [] });
|
|
17851
|
+
throw err;
|
|
17852
|
+
}
|
|
17853
|
+
if (result.artifacts.length) {
|
|
17854
|
+
await appendWorkflowArtifacts(runId, input.user.id, result.artifacts.map((a) => ({
|
|
17855
|
+
kind: a.kind,
|
|
17856
|
+
label: a.label,
|
|
17857
|
+
path: a.path,
|
|
17858
|
+
bytes: a.bytes,
|
|
17859
|
+
rows: a.rows,
|
|
17860
|
+
content: a.content
|
|
17861
|
+
})));
|
|
17862
|
+
}
|
|
17863
|
+
const logEntry = {
|
|
17864
|
+
stepIndex: result.stepIndex,
|
|
17865
|
+
stepId: result.stepId,
|
|
17866
|
+
stepTitle: result.stepTitle,
|
|
17867
|
+
output: result.output,
|
|
17868
|
+
warnings: result.warnings,
|
|
17869
|
+
at: (/* @__PURE__ */ new Date()).toISOString()
|
|
17870
|
+
};
|
|
17871
|
+
const status = result.isLast ? result.summary?.status ?? "succeeded" : "running";
|
|
17872
|
+
await advanceWorkflowRunStep(runId, {
|
|
17873
|
+
stepIndex: result.stepIndex + 1,
|
|
17874
|
+
totalSteps: result.totalSteps,
|
|
17875
|
+
state: result.state,
|
|
17876
|
+
status,
|
|
17877
|
+
logEntry,
|
|
17878
|
+
appendLog: input.priorLog
|
|
17879
|
+
});
|
|
17880
|
+
if (result.isLast && result.summary) {
|
|
17881
|
+
const manifest = await readManifestFromSummary(result.summary);
|
|
17882
|
+
await completeWorkflowRunRecord(runId, result.summary.status, manifest ?? { summary: result.summary });
|
|
17883
|
+
}
|
|
17884
|
+
const run = await getWorkflowRun(runId, input.user.id);
|
|
17885
|
+
const artifacts = await listWorkflowArtifacts(runId, input.user.id);
|
|
17886
|
+
if (result.isLast && run) {
|
|
17887
|
+
await deliverWorkflowWebhook({ userId: input.user.id, run, scheduleId: run.schedule_id, webhookUrl: input.webhookUrl ?? null, summary: result.summary, artifacts });
|
|
17888
|
+
}
|
|
17889
|
+
const definition = workflowDefinition(workflowId);
|
|
17890
|
+
const nextIndex = result.stepIndex + 1;
|
|
17891
|
+
const nextStep = result.isLast ? null : {
|
|
17892
|
+
index: nextIndex,
|
|
17893
|
+
id: definition.steps?.[nextIndex]?.id ?? null,
|
|
17894
|
+
title: definition.steps?.[nextIndex]?.title ?? null
|
|
17895
|
+
};
|
|
17896
|
+
return {
|
|
17897
|
+
run: run ? exposeRun(input.c, run, artifacts) : null,
|
|
17898
|
+
summary: result.isLast ? result.summary : null,
|
|
17899
|
+
step: {
|
|
17900
|
+
index: result.stepIndex,
|
|
17901
|
+
id: result.stepId,
|
|
17902
|
+
title: result.stepTitle,
|
|
17903
|
+
totalSteps: result.totalSteps,
|
|
17904
|
+
output: result.output,
|
|
17905
|
+
warnings: result.warnings
|
|
17906
|
+
},
|
|
17907
|
+
nextStep,
|
|
17908
|
+
done: result.isLast
|
|
17909
|
+
};
|
|
17910
|
+
}
|
|
17412
17911
|
async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
17413
17912
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
17414
17913
|
const due = await listDueWorkflowSchedules(now, limit);
|
|
@@ -17461,12 +17960,12 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
17461
17960
|
}
|
|
17462
17961
|
return { dispatched: results.length, results };
|
|
17463
17962
|
}
|
|
17464
|
-
var import_node_crypto3,
|
|
17963
|
+
var import_node_crypto3, import_promises8, import_hono7, import_zod24, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
|
|
17465
17964
|
var init_workflow_routes = __esm({
|
|
17466
17965
|
"src/api/workflow-routes.ts"() {
|
|
17467
17966
|
"use strict";
|
|
17468
17967
|
import_node_crypto3 = require("crypto");
|
|
17469
|
-
|
|
17968
|
+
import_promises8 = require("fs/promises");
|
|
17470
17969
|
import_hono7 = require("hono");
|
|
17471
17970
|
import_zod24 = require("zod");
|
|
17472
17971
|
init_artifact_writer();
|
|
@@ -17525,7 +18024,29 @@ var init_workflow_routes = __esm({
|
|
|
17525
18024
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
17526
18025
|
let run = null;
|
|
17527
18026
|
try {
|
|
17528
|
-
|
|
18027
|
+
const stepwise = workflowSupportsSteps(parsed.data.workflowId);
|
|
18028
|
+
run = await createWorkflowRun({
|
|
18029
|
+
userId: user.id,
|
|
18030
|
+
workflowId: parsed.data.workflowId,
|
|
18031
|
+
workflowInput: input,
|
|
18032
|
+
webhookUrl,
|
|
18033
|
+
totalSteps: stepwise ? workflowStepCount(parsed.data.workflowId) : 1
|
|
18034
|
+
});
|
|
18035
|
+
if (stepwise) {
|
|
18036
|
+
const response = await runWorkflowStepAndPersist({
|
|
18037
|
+
c,
|
|
18038
|
+
user,
|
|
18039
|
+
run,
|
|
18040
|
+
workflowInput: input,
|
|
18041
|
+
stepIndex: 0,
|
|
18042
|
+
priorState: null,
|
|
18043
|
+
priorLog: [],
|
|
18044
|
+
apiUrl: originFromUrl(c.req.url),
|
|
18045
|
+
webhookUrl,
|
|
18046
|
+
concurrencyLockId: gate.lockId
|
|
18047
|
+
});
|
|
18048
|
+
return c.json(response);
|
|
18049
|
+
}
|
|
17529
18050
|
const executed = await executeWorkflowRun({
|
|
17530
18051
|
runId: run.id,
|
|
17531
18052
|
user,
|
|
@@ -17535,7 +18056,7 @@ var init_workflow_routes = __esm({
|
|
|
17535
18056
|
webhookUrl,
|
|
17536
18057
|
concurrencyLockId: gate.lockId
|
|
17537
18058
|
});
|
|
17538
|
-
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
18059
|
+
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary, done: true });
|
|
17539
18060
|
} catch (err) {
|
|
17540
18061
|
const failed = run ? await getWorkflowRun(run.id, user.id) : null;
|
|
17541
18062
|
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
@@ -17543,6 +18064,50 @@ var init_workflow_routes = __esm({
|
|
|
17543
18064
|
await releaseConcurrencyGate(gate.lockId);
|
|
17544
18065
|
}
|
|
17545
18066
|
});
|
|
18067
|
+
TERMINAL_RUN_STATUSES = ["succeeded", "partial", "failed", "cancelled"];
|
|
18068
|
+
workflowApp.post("/runs/:id/step", createApiKeyAuth(), async (c) => {
|
|
18069
|
+
const user = c.get("user");
|
|
18070
|
+
const run = await getWorkflowRun(c.req.param("id"), user.id);
|
|
18071
|
+
if (!run) return c.json({ error: "Run not found" }, 404);
|
|
18072
|
+
if (!workflowSupportsSteps(run.workflow_id)) {
|
|
18073
|
+
return c.json({ error: `Workflow "${run.workflow_id}" does not support stepwise execution` }, 400);
|
|
18074
|
+
}
|
|
18075
|
+
const artifactsNow = await listWorkflowArtifacts(run.id, user.id);
|
|
18076
|
+
if (TERMINAL_RUN_STATUSES.includes(run.status)) {
|
|
18077
|
+
return c.json({ run: exposeRun(c, run, artifactsNow), done: true, error: `Run is already ${run.status}` }, 409);
|
|
18078
|
+
}
|
|
18079
|
+
const stepState = await getWorkflowRunStepState(run.id, user.id);
|
|
18080
|
+
const stepIndex = stepState?.stepIndex ?? run.step_index;
|
|
18081
|
+
const totalSteps = workflowStepCount(run.workflow_id);
|
|
18082
|
+
if (stepIndex >= totalSteps) {
|
|
18083
|
+
return c.json({ run: exposeRun(c, run, artifactsNow), done: true, error: "No remaining steps" }, 409);
|
|
18084
|
+
}
|
|
18085
|
+
const gate = await acquireConcurrencyGate(user, "workflow_run", {
|
|
18086
|
+
ttlSeconds: 60 * 60,
|
|
18087
|
+
metadata: { workflowId: run.workflow_id, step: stepIndex }
|
|
18088
|
+
});
|
|
18089
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
18090
|
+
try {
|
|
18091
|
+
const response = await runWorkflowStepAndPersist({
|
|
18092
|
+
c,
|
|
18093
|
+
user,
|
|
18094
|
+
run,
|
|
18095
|
+
workflowInput: run.input,
|
|
18096
|
+
stepIndex,
|
|
18097
|
+
priorState: stepState?.state ?? null,
|
|
18098
|
+
priorLog: stepState?.stepLog ?? run.step_log,
|
|
18099
|
+
apiUrl: originFromUrl(c.req.url),
|
|
18100
|
+
webhookUrl: run.webhook_url,
|
|
18101
|
+
concurrencyLockId: gate.lockId
|
|
18102
|
+
});
|
|
18103
|
+
return c.json(response);
|
|
18104
|
+
} catch (err) {
|
|
18105
|
+
const failed = await getWorkflowRun(run.id, user.id);
|
|
18106
|
+
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
18107
|
+
} finally {
|
|
18108
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
18109
|
+
}
|
|
18110
|
+
});
|
|
17546
18111
|
workflowApp.get("/runs", createApiKeyAuth(), async (c) => {
|
|
17547
18112
|
const user = c.get("user");
|
|
17548
18113
|
const runs = await listWorkflowRuns(user.id);
|
|
@@ -17561,14 +18126,17 @@ var init_workflow_routes = __esm({
|
|
|
17561
18126
|
if (!run) return c.json({ error: "Run not found" }, 404);
|
|
17562
18127
|
const artifact = await getWorkflowArtifact(c.req.param("artifactId"), user.id);
|
|
17563
18128
|
if (!artifact || artifact.run_id !== run.id) return c.json({ error: "Artifact not found" }, 404);
|
|
18129
|
+
const filename = artifact.path.split("/").pop() || artifact.id;
|
|
18130
|
+
const headers = {
|
|
18131
|
+
"Content-Type": artifact.content_type,
|
|
18132
|
+
"Content-Disposition": `attachment; filename="${filename}"`
|
|
18133
|
+
};
|
|
18134
|
+
if (artifact.content != null) {
|
|
18135
|
+
return new Response(artifact.content, { headers });
|
|
18136
|
+
}
|
|
17564
18137
|
try {
|
|
17565
|
-
const content = await (0,
|
|
17566
|
-
return new Response(content, {
|
|
17567
|
-
headers: {
|
|
17568
|
-
"Content-Type": artifact.content_type,
|
|
17569
|
-
"Content-Disposition": `attachment; filename="${artifact.path.split("/").pop() || artifact.id}"`
|
|
17570
|
-
}
|
|
17571
|
-
});
|
|
18138
|
+
const content = await (0, import_promises8.readFile)(artifact.path);
|
|
18139
|
+
return new Response(content, { headers });
|
|
17572
18140
|
} catch {
|
|
17573
18141
|
return c.json({ error: "Artifact file is no longer available" }, 410);
|
|
17574
18142
|
}
|
|
@@ -17856,7 +18424,7 @@ async function capturePageSnapshot(target, options = {}) {
|
|
|
17856
18424
|
const textContent = truncateContent(text, maxContentChars);
|
|
17857
18425
|
const schemaTypes = getSchemaTypes(extraction.schema, extraction.kpo.schemaTypes);
|
|
17858
18426
|
const metrics = countLinks(extraction.bodyHtml, extraction.url);
|
|
17859
|
-
const
|
|
18427
|
+
const wordCount2 = countWords(extraction.bodyMarkdown);
|
|
17860
18428
|
const contentSha256 = sha256(extraction.bodyMarkdown);
|
|
17861
18429
|
const canonicalUrl = extraction.kpo.canonicalUrl ? new URL(extraction.kpo.canonicalUrl, extraction.url).href : null;
|
|
17862
18430
|
const contentIncluded = Boolean(options.includeContent);
|
|
@@ -17910,10 +18478,10 @@ async function capturePageSnapshot(target, options = {}) {
|
|
|
17910
18478
|
},
|
|
17911
18479
|
error: null,
|
|
17912
18480
|
schemaTypes,
|
|
17913
|
-
wordCount,
|
|
18481
|
+
wordCount: wordCount2,
|
|
17914
18482
|
linkCounts: {
|
|
17915
18483
|
...metrics,
|
|
17916
|
-
wordCount
|
|
18484
|
+
wordCount: wordCount2
|
|
17917
18485
|
},
|
|
17918
18486
|
hashes: {
|
|
17919
18487
|
htmlSha256: sha256(extraction.bodyHtml),
|
|
@@ -19489,7 +20057,7 @@ async function harvest(rawOptions) {
|
|
|
19489
20057
|
const signal = getAbortSignal(rawOptions);
|
|
19490
20058
|
const onAttemptEvent = getAttemptLogSink(rawOptions);
|
|
19491
20059
|
const requestedProxyMode = raw.proxyMode;
|
|
19492
|
-
const proxyMode = requestedProxyMode === "
|
|
20060
|
+
const proxyMode = requestedProxyMode === "location" || requestedProxyMode === "none" || requestedProxyMode === "configured" ? requestedProxyMode : DEFAULT_PROXY_MODE;
|
|
19493
20061
|
const kernelApiKey = typeof raw.kernelApiKey === "string" ? raw.kernelApiKey.trim() : browserServiceApiKey();
|
|
19494
20062
|
const configuredKernelProxyId = typeof raw.kernelProxyId === "string" ? raw.kernelProxyId.trim() : browserServiceProxyId();
|
|
19495
20063
|
const proxyOpts = {
|
|
@@ -19863,6 +20431,7 @@ var init_schemas4 = __esm({
|
|
|
19863
20431
|
"src/serp-intelligence/schemas.ts"() {
|
|
19864
20432
|
"use strict";
|
|
19865
20433
|
import_zod25 = require("zod");
|
|
20434
|
+
init_schemas3();
|
|
19866
20435
|
SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
19867
20436
|
SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
19868
20437
|
SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -19903,7 +20472,7 @@ var init_schemas4 = __esm({
|
|
|
19903
20472
|
gl: import_zod25.z.string().trim().length(2).default("us"),
|
|
19904
20473
|
hl: import_zod25.z.string().trim().length(2).default("en"),
|
|
19905
20474
|
device: import_zod25.z.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
19906
|
-
proxyMode: import_zod25.z.enum(SerpIntelligenceProxyModeValues).default(
|
|
20475
|
+
proxyMode: import_zod25.z.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
19907
20476
|
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional(),
|
|
19908
20477
|
pages: import_zod25.z.number().int().min(1).max(2).default(1),
|
|
19909
20478
|
debug: import_zod25.z.boolean().default(false),
|
|
@@ -20433,16 +21002,17 @@ var PACKAGE_VERSION;
|
|
|
20433
21002
|
var init_version = __esm({
|
|
20434
21003
|
"src/version.ts"() {
|
|
20435
21004
|
"use strict";
|
|
20436
|
-
PACKAGE_VERSION = "0.2.
|
|
21005
|
+
PACKAGE_VERSION = "0.2.20";
|
|
20437
21006
|
}
|
|
20438
21007
|
});
|
|
20439
21008
|
|
|
20440
21009
|
// src/mcp/mcp-tool-schemas.ts
|
|
20441
|
-
var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
21010
|
+
var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
20442
21011
|
var init_mcp_tool_schemas = __esm({
|
|
20443
21012
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
20444
21013
|
"use strict";
|
|
20445
21014
|
import_zod26 = require("zod");
|
|
21015
|
+
init_schemas3();
|
|
20446
21016
|
HarvestPaaInputSchema = {
|
|
20447
21017
|
query: import_zod26.z.string().min(1).describe('Core search topic only. If the user says "best hvac company in Denver CO", use query="best hvac company" and location="Denver, CO". Do not include the location in query when it can be separated.'),
|
|
20448
21018
|
location: import_zod26.z.string().optional().describe('City, region, or country for geo-targeted results, inferred from the user request when present, e.g. "Denver, CO", "Tokyo, Japan", "London, UK".'),
|
|
@@ -20450,8 +21020,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20450
21020
|
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
|
|
20451
21021
|
hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
|
|
20452
21022
|
device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
20453
|
-
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(
|
|
20454
|
-
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or
|
|
21023
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
21024
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
|
|
20455
21025
|
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
|
|
20456
21026
|
};
|
|
20457
21027
|
ExtractUrlInputSchema = {
|
|
@@ -20475,22 +21045,23 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20475
21045
|
mode: import_zod26.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
|
|
20476
21046
|
query: import_zod26.z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
|
|
20477
21047
|
channelHandle: import_zod26.z.string().optional().describe("YouTube channel handle, channel ID, or URL. Examples: @mkbhd, UC..., https://youtube.com/@mkbhd."),
|
|
20478
|
-
maxVideos: import_zod26.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50.
|
|
21048
|
+
maxVideos: import_zod26.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50, maximum 500. Use 10-25 for quick topic discovery, 50 for normal channel/search harvests, and larger values only when the user asks for full channel/history because large responses consume more context.")
|
|
20479
21049
|
};
|
|
20480
21050
|
YoutubeTranscribeInputSchema = {
|
|
20481
|
-
videoId: import_zod26.z.string().min(1).describe("YouTube video ID, e.g. dQw4w9WgXcQ")
|
|
21051
|
+
videoId: import_zod26.z.string().min(1).optional().describe("YouTube video ID, e.g. dQw4w9WgXcQ. Use only an ID returned by youtube_harvest or visible in a YouTube URL; do not invent one."),
|
|
21052
|
+
url: import_zod26.z.string().url().optional().describe("Full YouTube URL, e.g. https://www.youtube.com/watch?v=dQw4w9WgXcQ or https://youtu.be/dQw4w9WgXcQ. Use this when the user pasted a URL instead of an ID. Provide videoId or url.")
|
|
20482
21053
|
};
|
|
20483
21054
|
FacebookPageIntelInputSchema = {
|
|
20484
|
-
pageId: import_zod26.z.string().optional(),
|
|
20485
|
-
libraryId: import_zod26.z.string().optional(),
|
|
21055
|
+
pageId: import_zod26.z.string().optional().describe("Facebook advertiser/page ID. Use only a pageId returned by facebook_ad_search/facebook_page_intel or copied from Facebook Ad Library; do not construct one yourself."),
|
|
21056
|
+
libraryId: import_zod26.z.string().optional().describe("Facebook Ad Library archive ID for a known ad or advertiser sample. Use a libraryId returned by facebook_ad_search, or a libraryId/adArchiveId visible in Ad Library."),
|
|
20486
21057
|
query: import_zod26.z.string().optional().describe("Advertiser or brand name when pageId/libraryId is not known. One of pageId, libraryId, or query is required."),
|
|
20487
|
-
maxAds: import_zod26.z.number().int().min(1).max(200).default(50),
|
|
20488
|
-
country: import_zod26.z.string().length(2).default("US")
|
|
21058
|
+
maxAds: import_zod26.z.number().int().min(1).max(200).default(50).describe("Maximum ads to inspect. Default 50, maximum 200. Prefer 25-50 for focused advertiser scans; use 100-200 only when the user asks for a broad ad archive sweep."),
|
|
21059
|
+
country: import_zod26.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU. Infer from the user request when they name a country.")
|
|
20489
21060
|
};
|
|
20490
21061
|
FacebookAdSearchInputSchema = {
|
|
20491
21062
|
query: import_zod26.z.string().min(1).describe("Advertiser, brand, competitor, niche, or keyword to search in Facebook Ad Library."),
|
|
20492
|
-
country: import_zod26.z.string().length(2).default("US"),
|
|
20493
|
-
maxResults: import_zod26.z.number().int().min(1).max(20).default(10)
|
|
21063
|
+
country: import_zod26.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU."),
|
|
21064
|
+
maxResults: import_zod26.z.number().int().min(1).max(20).default(10).describe("Maximum advertisers to return. Default 10, maximum 20. Prefer tighter search terms over maxing this out.")
|
|
20494
21065
|
};
|
|
20495
21066
|
FacebookAdTranscribeInputSchema = {
|
|
20496
21067
|
videoUrl: import_zod26.z.string().url().describe("Direct Facebook CDN video URL from a facebook_page_intel ad result. Do not pass a public Facebook reel/post/share URL here; use facebook_video_transcribe for organic Facebook URLs.")
|
|
@@ -20504,8 +21075,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20504
21075
|
location: import_zod26.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
|
|
20505
21076
|
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
20506
21077
|
hl: import_zod26.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
20507
|
-
includeReviews: import_zod26.z.boolean().default(false).describe("Whether to fetch individual review cards"),
|
|
20508
|
-
maxReviews: import_zod26.z.number().int().min(1).max(500).default(50).describe("Max review cards to return
|
|
21078
|
+
includeReviews: import_zod26.z.boolean().default(false).describe("Whether to fetch individual review cards. Use true when the user asks for reviews, customer pain, complaints, praise, themes, or review evidence."),
|
|
21079
|
+
maxReviews: import_zod26.z.number().int().min(1).max(500).default(50).describe("Max review cards to return when includeReviews is true. Default 50, maximum 500. Use 50 for normal review analysis and larger values only for deep review mining.")
|
|
20509
21080
|
};
|
|
20510
21081
|
MapsSearchInputSchema = {
|
|
20511
21082
|
query: import_zod26.z.string().min(1).describe('Business category, niche, keyword, or search term. If the user says "roofers in Denver CO", use query="roofers" and location="Denver, CO". Do not put the location here when it can be separated.'),
|
|
@@ -20513,7 +21084,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20513
21084
|
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
20514
21085
|
hl: import_zod26.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
20515
21086
|
maxResults: import_zod26.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
|
|
20516
|
-
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(
|
|
21087
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode. Maps defaults to location so local market searches get city/state residential proxy targeting and rotation. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
|
|
20517
21088
|
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
|
|
20518
21089
|
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
|
|
20519
21090
|
};
|
|
@@ -20528,8 +21099,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20528
21099
|
includeZipGroups: import_zod26.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
|
|
20529
21100
|
usZipsCsvPath: import_zod26.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
20530
21101
|
saveCsv: import_zod26.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
|
|
20531
|
-
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(
|
|
20532
|
-
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its
|
|
21102
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode for every city Maps search. Maps workflows default to location so each city can use city/state or ZIP-group residential proxy targeting. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
|
|
21103
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its ZIP group or city/state location."),
|
|
20533
21104
|
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
|
|
20534
21105
|
};
|
|
20535
21106
|
RankTrackerModeSchema = import_zod26.z.enum(["maps", "organic", "ai_overview", "paa"]);
|
|
@@ -20843,8 +21414,11 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20843
21414
|
advertiserCount: import_zod26.z.number().int().min(0),
|
|
20844
21415
|
advertisers: import_zod26.z.array(import_zod26.z.object({
|
|
20845
21416
|
name: NullableString,
|
|
21417
|
+
pageId: NullableString,
|
|
21418
|
+
pageUrl: NullableString,
|
|
20846
21419
|
adCount: import_zod26.z.number().int().nullable(),
|
|
20847
|
-
libraryId: NullableString
|
|
21420
|
+
libraryId: NullableString,
|
|
21421
|
+
sampleLibraryId: NullableString
|
|
20848
21422
|
}))
|
|
20849
21423
|
};
|
|
20850
21424
|
FacebookPageIntelOutputSchema = {
|
|
@@ -20857,10 +21431,15 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20857
21431
|
libraryId: NullableString,
|
|
20858
21432
|
status: NullableString,
|
|
20859
21433
|
creativeType: NullableString,
|
|
21434
|
+
primaryText: NullableString,
|
|
20860
21435
|
headline: NullableString,
|
|
20861
21436
|
cta: NullableString,
|
|
20862
21437
|
startDate: NullableString,
|
|
21438
|
+
landingUrl: NullableString,
|
|
21439
|
+
domain: NullableString,
|
|
20863
21440
|
videoUrl: NullableString,
|
|
21441
|
+
imageUrl: NullableString,
|
|
21442
|
+
videoPoster: NullableString,
|
|
20864
21443
|
variations: import_zod26.z.number().int().nullable()
|
|
20865
21444
|
}))
|
|
20866
21445
|
};
|
|
@@ -20882,6 +21461,58 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20882
21461
|
text: import_zod26.z.string()
|
|
20883
21462
|
}))
|
|
20884
21463
|
};
|
|
21464
|
+
TranscriptChunkOutput = import_zod26.z.object({
|
|
21465
|
+
startSec: import_zod26.z.number(),
|
|
21466
|
+
endSec: import_zod26.z.number(),
|
|
21467
|
+
text: import_zod26.z.string()
|
|
21468
|
+
});
|
|
21469
|
+
YoutubeTranscribeOutputSchema = {
|
|
21470
|
+
videoId: NullableString,
|
|
21471
|
+
url: NullableString,
|
|
21472
|
+
wordCount: import_zod26.z.number().int().min(0),
|
|
21473
|
+
chunkCount: import_zod26.z.number().int().min(0),
|
|
21474
|
+
durationMs: import_zod26.z.number().nullable(),
|
|
21475
|
+
transcriptText: import_zod26.z.string(),
|
|
21476
|
+
chunks: import_zod26.z.array(TranscriptChunkOutput),
|
|
21477
|
+
resolvedInputs: import_zod26.z.object({
|
|
21478
|
+
videoId: NullableString,
|
|
21479
|
+
url: NullableString
|
|
21480
|
+
})
|
|
21481
|
+
};
|
|
21482
|
+
FacebookAdTranscribeOutputSchema = {
|
|
21483
|
+
videoUrl: import_zod26.z.string().url(),
|
|
21484
|
+
wordCount: import_zod26.z.number().int().min(0),
|
|
21485
|
+
chunkCount: import_zod26.z.number().int().min(0),
|
|
21486
|
+
durationMs: import_zod26.z.number().nullable(),
|
|
21487
|
+
transcriptText: import_zod26.z.string(),
|
|
21488
|
+
chunks: import_zod26.z.array(TranscriptChunkOutput),
|
|
21489
|
+
resolvedInputs: import_zod26.z.object({
|
|
21490
|
+
videoUrl: import_zod26.z.string().url()
|
|
21491
|
+
})
|
|
21492
|
+
};
|
|
21493
|
+
CaptureSerpSnapshotOutputSchema = {
|
|
21494
|
+
schemaVersion: import_zod26.z.literal("serp-intelligence.capture.v1"),
|
|
21495
|
+
status: import_zod26.z.string(),
|
|
21496
|
+
query: NullableString,
|
|
21497
|
+
location: NullableString,
|
|
21498
|
+
capturedAt: NullableString,
|
|
21499
|
+
resultCount: import_zod26.z.number().int().min(0).nullable(),
|
|
21500
|
+
snapshotId: NullableString,
|
|
21501
|
+
resolvedInputs: import_zod26.z.record(import_zod26.z.unknown()),
|
|
21502
|
+
artifacts: import_zod26.z.array(import_zod26.z.record(import_zod26.z.unknown())),
|
|
21503
|
+
diagnostics: import_zod26.z.record(import_zod26.z.unknown()).nullable(),
|
|
21504
|
+
providerPayload: import_zod26.z.record(import_zod26.z.unknown())
|
|
21505
|
+
};
|
|
21506
|
+
CaptureSerpPageSnapshotsOutputSchema = {
|
|
21507
|
+
schemaVersion: import_zod26.z.literal("serp-intelligence.page-snapshots.v1"),
|
|
21508
|
+
status: import_zod26.z.string(),
|
|
21509
|
+
count: import_zod26.z.number().int().min(0),
|
|
21510
|
+
failedCount: import_zod26.z.number().int().min(0),
|
|
21511
|
+
captures: import_zod26.z.array(import_zod26.z.record(import_zod26.z.unknown())),
|
|
21512
|
+
resolvedInputs: import_zod26.z.record(import_zod26.z.unknown()),
|
|
21513
|
+
diagnostics: import_zod26.z.record(import_zod26.z.unknown()).nullable(),
|
|
21514
|
+
providerPayload: import_zod26.z.record(import_zod26.z.unknown())
|
|
21515
|
+
};
|
|
20885
21516
|
CreditsInfoInputSchema = {
|
|
20886
21517
|
item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
|
|
20887
21518
|
includeLedger: import_zod26.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
@@ -20909,16 +21540,19 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20909
21540
|
maxSuggestions: import_zod26.z.number().int().min(1).max(8).default(3).describe("Number of matching workflow recipes to return.")
|
|
20910
21541
|
};
|
|
20911
21542
|
WorkflowRunInputSchema = {
|
|
20912
|
-
workflowId: WorkflowIdSchema2.describe("Workflow to run. Use workflow_list or workflow_suggest first when unsure."),
|
|
21543
|
+
workflowId: WorkflowIdSchema2.describe("Workflow to run: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. Use only these values; call workflow_list or workflow_suggest first when unsure."),
|
|
20913
21544
|
input: import_zod26.z.record(import_zod26.z.unknown()).default({}).describe("Workflow-specific input object. Examples: agent-packet uses {keyword, domain?, location?, maxQuestions?}; local-competitive-audit uses {query, state, minPopulation?, maxCities?, maxResultsPerCity?, hydrateTop?, maxReviews?}; serp-comparison uses {keyword, domain?, url?, location?, extractTop?}."),
|
|
20914
21545
|
webhookUrl: import_zod26.z.string().url().optional().describe("Optional HTTPS webhook to receive the completed hosted workflow run event.")
|
|
20915
21546
|
};
|
|
21547
|
+
WorkflowStepInputSchema = {
|
|
21548
|
+
runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself. Advances the run by exactly one step (one logical leg, e.g. one live harvest).")
|
|
21549
|
+
};
|
|
20916
21550
|
WorkflowStatusInputSchema = {
|
|
20917
|
-
runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run.")
|
|
21551
|
+
runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself.")
|
|
20918
21552
|
};
|
|
20919
21553
|
WorkflowArtifactReadInputSchema = {
|
|
20920
|
-
runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run or workflow_status."),
|
|
20921
|
-
artifactId: import_zod26.z.string().min(1).describe("Artifact id from the run artifact list."),
|
|
21554
|
+
runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself."),
|
|
21555
|
+
artifactId: import_zod26.z.string().min(1).describe("Artifact id from the run artifact list returned by workflow_run, workflow_step, or workflow_status. Use only a returned artifactId; do not construct one yourself."),
|
|
20922
21556
|
maxBytes: import_zod26.z.number().int().min(1e3).max(1e6).default(2e5).describe("Maximum bytes of artifact text to return inline. Use lower values for large CSV/JSON artifacts; call again with the downloadUrl if needed outside MCP.")
|
|
20923
21557
|
};
|
|
20924
21558
|
WorkflowRecipeOutput = import_zod26.z.object({
|
|
@@ -20951,6 +21585,18 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20951
21585
|
input: import_zod26.z.record(import_zod26.z.unknown()),
|
|
20952
21586
|
run: import_zod26.z.record(import_zod26.z.unknown()).optional(),
|
|
20953
21587
|
summary: import_zod26.z.record(import_zod26.z.unknown()).optional(),
|
|
21588
|
+
step: import_zod26.z.record(import_zod26.z.unknown()).optional(),
|
|
21589
|
+
nextStep: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
|
|
21590
|
+
done: import_zod26.z.boolean().optional(),
|
|
21591
|
+
artifacts: import_zod26.z.array(WorkflowArtifactOutput)
|
|
21592
|
+
};
|
|
21593
|
+
WorkflowStepOutputSchema = {
|
|
21594
|
+
runId: import_zod26.z.string(),
|
|
21595
|
+
run: import_zod26.z.record(import_zod26.z.unknown()).optional(),
|
|
21596
|
+
summary: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
|
|
21597
|
+
step: import_zod26.z.record(import_zod26.z.unknown()).optional(),
|
|
21598
|
+
nextStep: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
|
|
21599
|
+
done: import_zod26.z.boolean(),
|
|
20954
21600
|
artifacts: import_zod26.z.array(WorkflowArtifactOutput)
|
|
20955
21601
|
};
|
|
20956
21602
|
WorkflowStatusOutputSchema = {
|
|
@@ -20971,8 +21617,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20971
21617
|
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
20972
21618
|
hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from user request."),
|
|
20973
21619
|
device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
20974
|
-
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(
|
|
20975
|
-
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or
|
|
21620
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general SERP success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
21621
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
|
|
20976
21622
|
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
|
|
20977
21623
|
pages: import_zod26.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
|
|
20978
21624
|
};
|
|
@@ -20982,8 +21628,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20982
21628
|
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
|
|
20983
21629
|
hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
20984
21630
|
device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
|
|
20985
|
-
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(
|
|
20986
|
-
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting when a precise city-center or ZIP proxy is needed.
|
|
21631
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy behavior for capture. Default configured uses the service proxy without city/ZIP targeting for the highest general capture success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
21632
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when a precise city-center or ZIP proxy is needed."),
|
|
20987
21633
|
pages: import_zod26.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
|
|
20988
21634
|
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
|
|
20989
21635
|
includePageSnapshots: import_zod26.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs through the same product capture path."),
|
|
@@ -21402,70 +22048,72 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
21402
22048
|
if (savesReports) registerSavedReportResources(server);
|
|
21403
22049
|
server.registerTool("harvest_paa", {
|
|
21404
22050
|
title: "Google PAA + SERP Harvest",
|
|
21405
|
-
description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en").
|
|
22051
|
+
description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use maxQuestions 30 normally, 100-200 for "full", "deep", "all", or comprehensive research. Deep harvests above 100 questions can run for several minutes with no interim progress \u2014 warn the user before starting one and keep maxQuestions at or below 100 unless they explicitly want a deep harvest. Credits are charged by extracted question; unused request hold is refunded.'),
|
|
21406
22052
|
inputSchema: HarvestPaaInputSchema,
|
|
21407
22053
|
outputSchema: HarvestPaaOutputSchema,
|
|
21408
22054
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
21409
22055
|
}, async (input) => formatHarvestPaa(await executor.harvestPaa(input), input));
|
|
21410
22056
|
server.registerTool("search_serp", {
|
|
21411
22057
|
title: "Google SERP Lookup",
|
|
21412
|
-
description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request.
|
|
22058
|
+
description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting. Use proxyMode location and proxyZip only when the user explicitly needs city/ZIP-targeted residential proxy evidence."),
|
|
21413
22059
|
inputSchema: SearchSerpInputSchema,
|
|
21414
22060
|
outputSchema: SearchSerpOutputSchema,
|
|
21415
22061
|
annotations: liveWebToolAnnotations("Google SERP Lookup")
|
|
21416
22062
|
}, async (input) => formatSearchSerp(await executor.searchSerp(input), input));
|
|
21417
22063
|
server.registerTool("extract_url", {
|
|
21418
22064
|
title: "Single URL Extract",
|
|
21419
|
-
description: withReportNote("Extract structured data from one public URL
|
|
22065
|
+
description: withReportNote("Extract structured data from one public URL when the user provides one page, asks to inspect/scrape a page, or needs page content, schema, headings, metadata, screenshots, branding, or media assets. Returns structured page fields plus artifact handles for saved reports/screenshots/media when requested. Use map_site_urls before extracting a site inventory; use extract_site for multi-page crawling."),
|
|
21420
22066
|
inputSchema: ExtractUrlInputSchema,
|
|
21421
22067
|
outputSchema: ExtractUrlOutputSchema,
|
|
21422
22068
|
annotations: liveWebToolAnnotations("Single URL Extract")
|
|
21423
22069
|
}, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
|
|
21424
22070
|
server.registerTool("map_site_urls", {
|
|
21425
22071
|
title: "Site URL Map",
|
|
21426
|
-
description: withReportNote("Map/crawl a public website
|
|
22072
|
+
description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
|
|
21427
22073
|
inputSchema: MapSiteUrlsInputSchema,
|
|
21428
22074
|
outputSchema: MapSiteUrlsOutputSchema,
|
|
21429
22075
|
annotations: liveWebToolAnnotations("Site URL Map")
|
|
21430
22076
|
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
|
|
21431
22077
|
server.registerTool("extract_site", {
|
|
21432
22078
|
title: "Multi-Page Site Extract",
|
|
21433
|
-
description: withReportNote("Run multi-page extraction across a public website. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use
|
|
22079
|
+
description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
|
|
21434
22080
|
inputSchema: ExtractSiteInputSchema,
|
|
21435
22081
|
outputSchema: ExtractSiteOutputSchema,
|
|
21436
22082
|
annotations: liveWebToolAnnotations("Multi-Page Site Extract")
|
|
21437
22083
|
}, async (input) => formatExtractSite(await executor.extractSite(input), input));
|
|
21438
22084
|
server.registerTool("youtube_harvest", {
|
|
21439
22085
|
title: "YouTube Video Harvest",
|
|
21440
|
-
description: withReportNote('Harvest YouTube video metadata by
|
|
22086
|
+
description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
|
|
21441
22087
|
inputSchema: YoutubeHarvestInputSchema,
|
|
21442
22088
|
outputSchema: YoutubeHarvestOutputSchema,
|
|
21443
22089
|
annotations: liveWebToolAnnotations("YouTube Video Harvest")
|
|
21444
22090
|
}, async (input) => formatYoutubeHarvest(await executor.youtubeHarvest(input), input));
|
|
21445
22091
|
server.registerTool("youtube_transcribe", {
|
|
21446
22092
|
title: "YouTube Transcription",
|
|
21447
|
-
description: withReportNote("Fetch and transcribe captions from a YouTube video. Returns full transcript, timestamped chunks,
|
|
22093
|
+
description: withReportNote("Fetch and transcribe captions from a YouTube video. Use this when the user asks what was said in a YouTube video, wants claims/offers/lessons extracted from a video, or provides a YouTube URL for transcript work. Returns full transcript, timestamped chunks, word count, and resolvedInputs. Pass videoId from youtube_harvest results, or pass url when the user pasted a YouTube URL. Use youtube_harvest first when you need to discover videos by topic/channel."),
|
|
21448
22094
|
inputSchema: YoutubeTranscribeInputSchema,
|
|
22095
|
+
outputSchema: YoutubeTranscribeOutputSchema,
|
|
21449
22096
|
annotations: liveWebToolAnnotations("YouTube Transcription")
|
|
21450
22097
|
}, async (input) => formatYoutubeTranscribe(await executor.youtubeTranscribe(input), input));
|
|
21451
22098
|
server.registerTool("facebook_page_intel", {
|
|
21452
22099
|
title: "Facebook Advertiser Ad Intel",
|
|
21453
|
-
description: withReportNote("Harvest ads from a Facebook advertiser. Returns ad copy, headlines, CTAs, creative type, status, landing URLs, and direct ad video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name as query. Use
|
|
22100
|
+
description: withReportNote("Harvest ads from a Facebook advertiser when the user wants current ad copy, creative angles, CTAs, video URLs, or competitive ad intelligence for one brand/page. Returns ad copy, headlines, CTAs, creative type, status, landing URLs, and direct ad video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name as query. Use facebook_ad_search first when the advertiser handle is unknown. For normal public Facebook reels/posts/watch/share URLs, use facebook_video_transcribe instead."),
|
|
21454
22101
|
inputSchema: FacebookPageIntelInputSchema,
|
|
21455
22102
|
outputSchema: FacebookPageIntelOutputSchema,
|
|
21456
22103
|
annotations: liveWebToolAnnotations("Facebook Advertiser Ad Intel")
|
|
21457
22104
|
}, async (input) => formatFacebookPageIntel(await executor.facebookPageIntel(input), input));
|
|
21458
22105
|
server.registerTool("facebook_ad_search", {
|
|
21459
22106
|
title: "Facebook Ad Library Search",
|
|
21460
|
-
description: withReportNote("Search Facebook Ad Library by brand,
|
|
22107
|
+
description: withReportNote("Search Facebook Ad Library when the user wants to find advertisers by brand, competitor, niche, or keyword. Returns advertisers with ad counts and library IDs. Use this to discover competitor/page handles, then pass libraryId or pageId to facebook_page_intel for ad details."),
|
|
21461
22108
|
inputSchema: FacebookAdSearchInputSchema,
|
|
21462
22109
|
outputSchema: FacebookAdSearchOutputSchema,
|
|
21463
22110
|
annotations: liveWebToolAnnotations("Facebook Ad Library Search")
|
|
21464
22111
|
}, async (input) => formatFacebookAdSearch(await executor.facebookAdSearch(input), input));
|
|
21465
22112
|
server.registerTool("facebook_ad_transcribe", {
|
|
21466
22113
|
title: "Facebook Ad Transcription",
|
|
21467
|
-
description: "Transcribe audio from a Facebook ad video CDN URL. Returns full transcript
|
|
22114
|
+
description: "Transcribe audio from a Facebook ad video CDN URL. Use this when facebook_page_intel returned a direct videoUrl and the user asks what the ad says, what claims it makes, or wants ad-message extraction. Returns full transcript, timestamped chunks, word count, and resolvedInputs. Use only with the direct videoUrl value from facebook_page_intel results; do not pass public Facebook post/reel/share URLs. Use facebook_video_transcribe for organic Facebook URLs, and facebook_page_intel first when you only have a brand/page/ad library handle.",
|
|
21468
22115
|
inputSchema: FacebookAdTranscribeInputSchema,
|
|
22116
|
+
outputSchema: FacebookAdTranscribeOutputSchema,
|
|
21469
22117
|
annotations: liveWebToolAnnotations("Facebook Ad Transcription")
|
|
21470
22118
|
}, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
|
|
21471
22119
|
server.registerTool("facebook_video_transcribe", {
|
|
@@ -21484,14 +22132,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
21484
22132
|
}, async (input) => formatMapsPlaceIntel(await executor.mapsPlaceIntel(input), input));
|
|
21485
22133
|
server.registerTool("maps_search", {
|
|
21486
22134
|
title: "Google Maps Business Search",
|
|
21487
|
-
description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack."
|
|
22135
|
+
description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." Maps is the default location-targeted surface: omit proxyMode for normal Maps use so the service creates residential proxy evidence for the requested market and rotates on retryable failures. Pass proxyZip only when a specific ZIP or city-center ZIP is known. Use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting. Returns up to 50 candidates with names, place URLs, CIDs when available, ratings, review counts, profile metadata, and sanitized attempt telemetry. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.'),
|
|
21488
22136
|
inputSchema: MapsSearchInputSchema,
|
|
21489
22137
|
outputSchema: MapsSearchOutputSchema,
|
|
21490
22138
|
annotations: liveWebToolAnnotations("Google Maps Business Search")
|
|
21491
22139
|
}, async (input) => formatMapsSearch(await executor.mapsSearch(input), input));
|
|
21492
22140
|
server.registerTool("directory_workflow", {
|
|
21493
22141
|
title: "Directory Workflow: Markets + Maps",
|
|
21494
|
-
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency.
|
|
22142
|
+
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Maps workflows default to location-targeted proxying so each city can use city/state or ZIP-group residential proxy evidence; use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting. Saved CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. Structured city results include sanitized attempt telemetry. Use maps_place_intel only when a selected profile needs deeper review topics, profile review count confirmation, or review cards. For local Lead Magician ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath only in local/test mode.'),
|
|
21495
22143
|
inputSchema: DirectoryWorkflowInputSchema,
|
|
21496
22144
|
outputSchema: DirectoryWorkflowOutputSchema,
|
|
21497
22145
|
annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
|
|
@@ -21512,21 +22160,28 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
21512
22160
|
}, async (input) => formatWorkflowSuggest(input));
|
|
21513
22161
|
server.registerTool("workflow_run", {
|
|
21514
22162
|
title: "Run Workflow",
|
|
21515
|
-
description: withReportNote("
|
|
22163
|
+
description: withReportNote("Start a higher-level MCP Scraper workflow. Use after workflow_suggest or workflow_list. Runnable workflow ids: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. This is the main MCP tool for market analysis, ICP evidence packets, local competitive audits, Maps/SERP comparisons, content gap briefs, and AI Overview language guidance. Stepwise workflows (e.g. agent-packet) run ONE leg per call and return runId, the step output, and nextStep \u2014 when nextStep is present, call workflow_step with the runId to run the next leg, and keep calling it until done is true. This keeps each call short instead of one long blocking run, so report each step result to the user as it arrives."),
|
|
21516
22164
|
inputSchema: WorkflowRunInputSchema,
|
|
21517
22165
|
outputSchema: WorkflowRunOutputSchema,
|
|
21518
22166
|
annotations: liveWebToolAnnotations("Run Workflow")
|
|
21519
22167
|
}, async (input) => formatWorkflowRun(await executor.workflowRun(input), input));
|
|
22168
|
+
server.registerTool("workflow_step", {
|
|
22169
|
+
title: "Advance Workflow Step",
|
|
22170
|
+
description: withReportNote("Run the next leg of a stepwise MCP Scraper workflow started with workflow_run. Pass the runId. Each call executes exactly one logical step (typically one live harvest), persists that step's artifacts, and returns the step output plus nextStep. Keep calling workflow_step with the same runId until done is true, reporting each step result to the user as it lands. Use this instead of waiting on one long workflow call \u2014 it avoids client timeouts on long multi-step jobs."),
|
|
22171
|
+
inputSchema: WorkflowStepInputSchema,
|
|
22172
|
+
outputSchema: WorkflowStepOutputSchema,
|
|
22173
|
+
annotations: liveWebToolAnnotations("Advance Workflow Step")
|
|
22174
|
+
}, async (input) => formatWorkflowStep(await executor.workflowStep(input), input));
|
|
21520
22175
|
server.registerTool("workflow_status", {
|
|
21521
22176
|
title: "Workflow Status",
|
|
21522
|
-
description: "Fetch a hosted workflow run by id and list its current status and artifacts. Use
|
|
22177
|
+
description: "Fetch a hosted workflow run by id and list its current status and artifacts. Use when a workflow may still be running, when the model needs to re-open a run, inspect artifact ids, recover from a long workflow, or decide whether to call workflow_step or workflow_artifact_read next. Use only a runId returned by workflow_run/workflow_step/workflow_status; do not construct one yourself.",
|
|
21523
22178
|
inputSchema: WorkflowStatusInputSchema,
|
|
21524
22179
|
outputSchema: WorkflowStatusOutputSchema,
|
|
21525
22180
|
annotations: liveWebToolAnnotations("Workflow Status")
|
|
21526
22181
|
}, async (input) => formatWorkflowStatus(await executor.workflowStatus(input), input));
|
|
21527
22182
|
server.registerTool("workflow_artifact_read", {
|
|
21528
22183
|
title: "Read Workflow Artifact",
|
|
21529
|
-
description: "Read a workflow artifact back into MCP context by run id and artifact id. Use this before writing final deliverables so the answer is grounded in generated evidence.json, CSVs, Markdown briefs, reports, and task files instead of memory. Use maxBytes to limit large CSV/JSON artifacts.",
|
|
22184
|
+
description: "Read a workflow artifact back into MCP context by run id and artifact id. Use this before writing final deliverables so the answer is grounded in generated evidence.json, CSVs, Markdown briefs, reports, and task files instead of memory. Use workflow_status first when artifact ids are unknown. Use only artifactId values returned by workflow_run/workflow_step/workflow_status; do not construct one yourself. Use maxBytes to limit large CSV/JSON artifacts.",
|
|
21530
22185
|
inputSchema: WorkflowArtifactReadInputSchema,
|
|
21531
22186
|
outputSchema: WorkflowArtifactReadOutputSchema,
|
|
21532
22187
|
annotations: liveWebToolAnnotations("Read Workflow Artifact")
|
|
@@ -21568,6 +22223,27 @@ var init_paa_mcp_server = __esm({
|
|
|
21568
22223
|
});
|
|
21569
22224
|
|
|
21570
22225
|
// src/mcp/http-mcp-tool-executor.ts
|
|
22226
|
+
function youtubeVideoIdFromUrl(url) {
|
|
22227
|
+
if (!url) return null;
|
|
22228
|
+
try {
|
|
22229
|
+
const parsed = new URL(url);
|
|
22230
|
+
const host = parsed.hostname.replace(/^www\./, "").replace(/^m\./, "");
|
|
22231
|
+
if (host === "youtu.be") {
|
|
22232
|
+
const id = parsed.pathname.split("/").filter(Boolean)[0];
|
|
22233
|
+
return id || null;
|
|
22234
|
+
}
|
|
22235
|
+
if (host === "youtube.com" || host === "music.youtube.com") {
|
|
22236
|
+
const watchId = parsed.searchParams.get("v");
|
|
22237
|
+
if (watchId) return watchId;
|
|
22238
|
+
const parts = parsed.pathname.split("/").filter(Boolean);
|
|
22239
|
+
const markerIndex = parts.findIndex((part) => ["shorts", "embed", "live"].includes(part));
|
|
22240
|
+
if (markerIndex >= 0 && parts[markerIndex + 1]) return parts[markerIndex + 1];
|
|
22241
|
+
}
|
|
22242
|
+
} catch {
|
|
22243
|
+
return null;
|
|
22244
|
+
}
|
|
22245
|
+
return null;
|
|
22246
|
+
}
|
|
21571
22247
|
var HttpMcpToolExecutor;
|
|
21572
22248
|
var init_http_mcp_tool_executor = __esm({
|
|
21573
22249
|
"src/mcp/http-mcp-tool-executor.ts"() {
|
|
@@ -21698,7 +22374,21 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
21698
22374
|
return this.call("/youtube/harvest", input);
|
|
21699
22375
|
}
|
|
21700
22376
|
youtubeTranscribe(input) {
|
|
21701
|
-
|
|
22377
|
+
const videoId = input.videoId?.trim() || youtubeVideoIdFromUrl(input.url);
|
|
22378
|
+
if (!videoId) {
|
|
22379
|
+
return Promise.resolve({
|
|
22380
|
+
content: [{
|
|
22381
|
+
type: "text",
|
|
22382
|
+
text: JSON.stringify({
|
|
22383
|
+
error_code: "youtube_video_id_required",
|
|
22384
|
+
error: "Pass videoId from youtube_harvest or a YouTube url that contains a video id.",
|
|
22385
|
+
retryable: false
|
|
22386
|
+
})
|
|
22387
|
+
}],
|
|
22388
|
+
isError: true
|
|
22389
|
+
});
|
|
22390
|
+
}
|
|
22391
|
+
return this.call("/youtube/transcribe", { videoId });
|
|
21702
22392
|
}
|
|
21703
22393
|
facebookPageIntel(input) {
|
|
21704
22394
|
return this.call("/facebook/page-intel", input);
|
|
@@ -21735,6 +22425,10 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
21735
22425
|
webhookUrl: input.webhookUrl
|
|
21736
22426
|
}, Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 9e5);
|
|
21737
22427
|
}
|
|
22428
|
+
workflowStep(input) {
|
|
22429
|
+
const timeoutMs = this.httpTimeoutOverrideMs ?? Number(process.env.MCP_SCRAPER_WORKFLOW_TIMEOUT_MS ?? 9e5);
|
|
22430
|
+
return this.call(`/workflows/runs/${encodeURIComponent(input.runId)}/step`, {}, Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 9e5);
|
|
22431
|
+
}
|
|
21738
22432
|
workflowStatus(input) {
|
|
21739
22433
|
return this.getJson(`/workflows/runs/${encodeURIComponent(input.runId)}`);
|
|
21740
22434
|
}
|
|
@@ -21788,16 +22482,18 @@ async function requireMcpCallerKey(c) {
|
|
|
21788
22482
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
21789
22483
|
server.registerTool("capture_serp_snapshot", {
|
|
21790
22484
|
title: "SERP Intelligence Snapshot",
|
|
21791
|
-
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location
|
|
22485
|
+
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl. Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
|
|
21792
22486
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
22487
|
+
outputSchema: CaptureSerpSnapshotOutputSchema,
|
|
21793
22488
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
21794
|
-
}, async (input) => executor.captureSerpSnapshot(input));
|
|
22489
|
+
}, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
|
|
21795
22490
|
server.registerTool("capture_serp_page_snapshots", {
|
|
21796
22491
|
title: "SERP Intelligence Page Snapshots",
|
|
21797
22492
|
description: "Capture public ranking-page evidence through POST /serp-intelligence/page-snapshots, the same product page snapshot path used by Phoenix. Provide urls for simple captures or targets when preserving organic, AI citation, local-pack, configured target, or site-subject source metadata. Private IPs, localhost, file URLs, and internal URLs are rejected by the service. Use timeoutMs for slow pages and debug true for sanitized proxy/browser diagnostics.",
|
|
21798
22493
|
inputSchema: CaptureSerpPageSnapshotsInputSchema,
|
|
22494
|
+
outputSchema: CaptureSerpPageSnapshotsOutputSchema,
|
|
21799
22495
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
21800
|
-
}, async (input) => executor.captureSerpPageSnapshots(input));
|
|
22496
|
+
}, async (input) => formatCaptureSerpPageSnapshots(await executor.captureSerpPageSnapshots(input), input));
|
|
21801
22497
|
}
|
|
21802
22498
|
var import_hono9, import_webStandardStreamableHttp, mcpApp;
|
|
21803
22499
|
var init_mcp_routes = __esm({
|
|
@@ -22006,14 +22702,35 @@ function client() {
|
|
|
22006
22702
|
if (!apiKey) throw new Error("Browser backend API key is required");
|
|
22007
22703
|
return new import_sdk6.default({ apiKey });
|
|
22008
22704
|
}
|
|
22705
|
+
function isProfileConflict(err) {
|
|
22706
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
22707
|
+
return /\b409\b|conflict|already exists/i.test(message);
|
|
22708
|
+
}
|
|
22709
|
+
async function ensureProfile(k, name) {
|
|
22710
|
+
try {
|
|
22711
|
+
await k.profiles.create({ name });
|
|
22712
|
+
} catch (err) {
|
|
22713
|
+
if (!isProfileConflict(err)) throw err;
|
|
22714
|
+
}
|
|
22715
|
+
}
|
|
22009
22716
|
async function createSession(opts = {}) {
|
|
22010
22717
|
const k = client();
|
|
22011
22718
|
const resolvedProxyId = opts.proxyId ?? browserServiceProxyId();
|
|
22719
|
+
const resolvedProfileName = opts.profileName ?? browserServiceProfileName();
|
|
22720
|
+
const resolvedSaveProfileChanges = opts.saveProfileChanges ?? browserServiceProfileSaveChanges();
|
|
22721
|
+
if (resolvedProfileName && resolvedSaveProfileChanges === true) {
|
|
22722
|
+
await ensureProfile(k, resolvedProfileName);
|
|
22723
|
+
}
|
|
22012
22724
|
const browser = await k.browsers.create({
|
|
22013
22725
|
stealth: opts.stealth ?? true,
|
|
22014
22726
|
timeout_seconds: opts.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS,
|
|
22015
22727
|
...resolvedProxyId ? { proxy_id: resolvedProxyId } : {},
|
|
22016
|
-
...
|
|
22728
|
+
...resolvedProfileName ? {
|
|
22729
|
+
profile: {
|
|
22730
|
+
name: resolvedProfileName,
|
|
22731
|
+
...typeof resolvedSaveProfileChanges === "boolean" ? { save_changes: resolvedSaveProfileChanges } : {}
|
|
22732
|
+
}
|
|
22733
|
+
} : {}
|
|
22017
22734
|
});
|
|
22018
22735
|
const runtimeSessionId = browser.session_id;
|
|
22019
22736
|
if (opts.disableDefaultProxy) {
|
|
@@ -22404,6 +23121,7 @@ function buildBrowserAgentRoutes() {
|
|
|
22404
23121
|
timeoutSeconds,
|
|
22405
23122
|
proxyId: typeof body.proxy_id === "string" ? body.proxy_id : void 0,
|
|
22406
23123
|
profileName: typeof body.profile === "string" ? body.profile : void 0,
|
|
23124
|
+
saveProfileChanges: typeof body.save_profile_changes === "boolean" ? body.save_profile_changes : void 0,
|
|
22407
23125
|
disableDefaultProxy: body.disable_default_proxy === true,
|
|
22408
23126
|
viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0
|
|
22409
23127
|
});
|
|
@@ -22779,6 +23497,7 @@ function renderConsoleHtml(initialSessionId) {
|
|
|
22779
23497
|
.gate button { width: 100%; }
|
|
22780
23498
|
.muted { color: #6b7689; font-size: 12px; }
|
|
22781
23499
|
</style>
|
|
23500
|
+
${ATTRIBUTION_PIXEL_SCRIPT}
|
|
22782
23501
|
</head>
|
|
22783
23502
|
<body>
|
|
22784
23503
|
<div id="app"></div>
|
|
@@ -22928,6 +23647,7 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
|
|
|
22928
23647
|
var init_browser_agent_console = __esm({
|
|
22929
23648
|
"src/api/browser-agent-console.ts"() {
|
|
22930
23649
|
"use strict";
|
|
23650
|
+
init_attribution_pixel();
|
|
22931
23651
|
}
|
|
22932
23652
|
});
|
|
22933
23653
|
|
|
@@ -23486,6 +24206,7 @@ var init_server = __esm({
|
|
|
23486
24206
|
init_outbound_sanitize();
|
|
23487
24207
|
init_registry();
|
|
23488
24208
|
init_template();
|
|
24209
|
+
init_attribution_pixel();
|
|
23489
24210
|
init_og();
|
|
23490
24211
|
import_resend = require("resend");
|
|
23491
24212
|
init_url_utils();
|
|
@@ -23518,6 +24239,7 @@ var init_server = __esm({
|
|
|
23518
24239
|
init_rates();
|
|
23519
24240
|
init_billing_schemas();
|
|
23520
24241
|
init_server_schemas();
|
|
24242
|
+
init_schemas3();
|
|
23521
24243
|
init_credit_operations();
|
|
23522
24244
|
init_harvest();
|
|
23523
24245
|
init_harvest_problems();
|
|
@@ -23747,7 +24469,7 @@ var init_server = __esm({
|
|
|
23747
24469
|
gl: body.gl ?? "us",
|
|
23748
24470
|
hl: body.hl ?? "en",
|
|
23749
24471
|
device: body.device ?? "desktop",
|
|
23750
|
-
proxyMode: body.proxyMode ??
|
|
24472
|
+
proxyMode: body.proxyMode ?? DEFAULT_PROXY_MODE,
|
|
23751
24473
|
proxyZip: body.proxyZip,
|
|
23752
24474
|
debug: body.debug ?? false,
|
|
23753
24475
|
serpOnly: body.serpOnly ?? false,
|
|
@@ -23783,7 +24505,7 @@ var init_server = __esm({
|
|
|
23783
24505
|
gl: body.gl ?? "us",
|
|
23784
24506
|
hl: body.hl ?? "en",
|
|
23785
24507
|
device: body.device ?? "desktop",
|
|
23786
|
-
proxyMode: body.proxyMode ??
|
|
24508
|
+
proxyMode: body.proxyMode ?? DEFAULT_PROXY_MODE,
|
|
23787
24509
|
proxyZip: body.proxyZip,
|
|
23788
24510
|
debug: body.debug ?? false,
|
|
23789
24511
|
serpOnly: body.serpOnly ?? false,
|
|
@@ -24328,6 +25050,7 @@ var init_server = __esm({
|
|
|
24328
25050
|
.i6 em{font-family:"GTF";font-weight:700;font-style:italic}
|
|
24329
25051
|
.i7 em{font-family:"GTF";font-weight:900;font-style:italic}
|
|
24330
25052
|
</style>
|
|
25053
|
+
${ATTRIBUTION_PIXEL_SCRIPT}
|
|
24331
25054
|
</head>
|
|
24332
25055
|
<body>
|
|
24333
25056
|
|