mcp-scraper 0.2.19 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/bin/api-server.cjs +1020 -348
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +314 -55
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +321 -167
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +3 -2
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +1128 -530
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +5 -4
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +726 -387
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -2
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +5 -3
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +2 -1
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-TL7YTFLH.js → chunk-DBQDG7EH.js} +99 -27
- package/dist/chunk-DBQDG7EH.js.map +1 -0
- package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js} +216 -108
- package/dist/chunk-H2R232HK.js.map +1 -0
- package/dist/{chunk-76P4DKWR.js → chunk-L4OWOUGR.js} +315 -56
- package/dist/chunk-L4OWOUGR.js.map +1 -0
- package/dist/chunk-LFATOGDF.js +111 -0
- package/dist/chunk-LFATOGDF.js.map +1 -0
- package/dist/{chunk-MCBOZ2JF.js → chunk-MLNCKQ2B.js} +3 -3
- package/dist/chunk-MLNCKQ2B.js.map +1 -0
- package/dist/{chunk-4UEVYYDW.js → chunk-O5J7E4VX.js} +394 -57
- package/dist/chunk-O5J7E4VX.js.map +1 -0
- package/dist/{chunk-BSYPATSM.js → chunk-SXTXMFEQ.js} +2 -2
- package/dist/{chunk-CQTAKXBN.js → chunk-XGUDTDZ2.js} +7 -104
- package/dist/chunk-XGUDTDZ2.js.map +1 -0
- package/dist/chunk-XKUDVN2E.js +7 -0
- package/dist/chunk-XKUDVN2E.js.map +1 -0
- package/dist/{db-P5X6UQ3E.js → db-BE4JVB3V.js} +8 -2
- package/dist/index.cjs +5 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/{server-CRUSZA2Q.js → server-7NE45K2A.js} +211 -37
- package/dist/server-7NE45K2A.js.map +1 -0
- package/dist/{worker-OZSWIS3F.js → worker-MIYG2B2I.js} +5 -4
- package/dist/{worker-OZSWIS3F.js.map → worker-MIYG2B2I.js.map} +1 -1
- package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +2 -2
- package/docs/mcp-tool-craft-lint.generated.md +51 -0
- package/docs/mcp-tool-manifest.generated.json +1406 -0
- package/docs/specs/local-competitive-audit-spec.md +1 -2
- package/docs/specs/mcp-tool-definition-quality-audit-spec.md +1602 -0
- package/package.json +4 -1
- package/dist/chunk-4UEVYYDW.js.map +0 -1
- package/dist/chunk-76P4DKWR.js.map +0 -1
- package/dist/chunk-CQTAKXBN.js.map +0 -1
- package/dist/chunk-F44RBOJ5.js.map +0 -1
- package/dist/chunk-MCBOZ2JF.js.map +0 -1
- package/dist/chunk-RU2HZD55.js +0 -7
- package/dist/chunk-RU2HZD55.js.map +0 -1
- package/dist/chunk-TL7YTFLH.js.map +0 -1
- package/dist/server-CRUSZA2Q.js.map +0 -1
- /package/dist/{chunk-BSYPATSM.js.map → chunk-SXTXMFEQ.js.map} +0 -0
- /package/dist/{db-P5X6UQ3E.js.map → db-BE4JVB3V.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -4809,7 +4809,7 @@ function parsePageData(url, html, status, via) {
|
|
|
4809
4809
|
}
|
|
4810
4810
|
const h1 = headings.find((h) => h.level === 1)?.text ?? null;
|
|
4811
4811
|
const bodyText = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
4812
|
-
const
|
|
4812
|
+
const wordCount2 = bodyText.split(" ").filter((w) => w.length > 2).length;
|
|
4813
4813
|
const schemaTypes = [];
|
|
4814
4814
|
const schema = [];
|
|
4815
4815
|
for (const m of html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
|
|
@@ -4845,7 +4845,7 @@ function parsePageData(url, html, status, via) {
|
|
|
4845
4845
|
} catch {
|
|
4846
4846
|
}
|
|
4847
4847
|
}
|
|
4848
|
-
return { url, status, via, title, metaDescription, h1, headings, wordCount, schemaTypes, canonicalUrl, internalLinks, externalLinks, bodyMarkdown, schema };
|
|
4848
|
+
return { url, status, via, title, metaDescription, h1, headings, wordCount: wordCount2, schemaTypes, canonicalUrl, internalLinks, externalLinks, bodyMarkdown, schema };
|
|
4849
4849
|
}
|
|
4850
4850
|
async function fetchAndParse(url, kernelApiKey) {
|
|
4851
4851
|
try {
|
|
@@ -6906,6 +6906,8 @@ var db_exports = {};
|
|
|
6906
6906
|
__export(db_exports, {
|
|
6907
6907
|
SiteAuditJobRowSchema: () => SiteAuditJobRowSchema,
|
|
6908
6908
|
SiteAuditPhaseLogRowSchema: () => SiteAuditPhaseLogRowSchema,
|
|
6909
|
+
advanceWorkflowRunStep: () => advanceWorkflowRunStep,
|
|
6910
|
+
appendWorkflowArtifacts: () => appendWorkflowArtifacts,
|
|
6909
6911
|
cancelJob: () => cancelJob,
|
|
6910
6912
|
checkRateLimit: () => checkRateLimit,
|
|
6911
6913
|
claimMonthlyFreeRefresh: () => claimMonthlyFreeRefresh,
|
|
@@ -6946,6 +6948,7 @@ __export(db_exports, {
|
|
|
6946
6948
|
getWorkflowArtifact: () => getWorkflowArtifact,
|
|
6947
6949
|
getWorkflowRun: () => getWorkflowRun,
|
|
6948
6950
|
getWorkflowRunByIdempotencyKey: () => getWorkflowRunByIdempotencyKey,
|
|
6951
|
+
getWorkflowRunStepState: () => getWorkflowRunStepState,
|
|
6949
6952
|
getWorkflowSchedule: () => getWorkflowSchedule,
|
|
6950
6953
|
hashPassword: () => hashPassword,
|
|
6951
6954
|
ledgerExistsForOperation: () => ledgerExistsForOperation,
|
|
@@ -7227,9 +7230,34 @@ async function migrate() {
|
|
|
7227
7230
|
queued_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
7228
7231
|
started_at TEXT,
|
|
7229
7232
|
completed_at TEXT,
|
|
7230
|
-
idempotency_key TEXT
|
|
7233
|
+
idempotency_key TEXT,
|
|
7234
|
+
step_index INTEGER NOT NULL DEFAULT 0,
|
|
7235
|
+
total_steps INTEGER,
|
|
7236
|
+
step_state_json TEXT,
|
|
7237
|
+
step_log_json TEXT,
|
|
7238
|
+
webhook_url TEXT
|
|
7231
7239
|
)
|
|
7232
7240
|
`);
|
|
7241
|
+
try {
|
|
7242
|
+
await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_index INTEGER NOT NULL DEFAULT 0`);
|
|
7243
|
+
} catch {
|
|
7244
|
+
}
|
|
7245
|
+
try {
|
|
7246
|
+
await db.execute(`ALTER TABLE workflow_runs ADD COLUMN total_steps INTEGER`);
|
|
7247
|
+
} catch {
|
|
7248
|
+
}
|
|
7249
|
+
try {
|
|
7250
|
+
await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_state_json TEXT`);
|
|
7251
|
+
} catch {
|
|
7252
|
+
}
|
|
7253
|
+
try {
|
|
7254
|
+
await db.execute(`ALTER TABLE workflow_runs ADD COLUMN step_log_json TEXT`);
|
|
7255
|
+
} catch {
|
|
7256
|
+
}
|
|
7257
|
+
try {
|
|
7258
|
+
await db.execute(`ALTER TABLE workflow_runs ADD COLUMN webhook_url TEXT`);
|
|
7259
|
+
} catch {
|
|
7260
|
+
}
|
|
7233
7261
|
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_user_created ON workflow_runs(user_id, queued_at DESC)`);
|
|
7234
7262
|
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_schedule ON workflow_runs(schedule_id, queued_at DESC)`);
|
|
7235
7263
|
await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS workflow_runs_idempotency ON workflow_runs(idempotency_key) WHERE idempotency_key IS NOT NULL`);
|
|
@@ -7244,9 +7272,14 @@ async function migrate() {
|
|
|
7244
7272
|
content_type TEXT NOT NULL,
|
|
7245
7273
|
bytes INTEGER,
|
|
7246
7274
|
rows_count INTEGER,
|
|
7275
|
+
content TEXT,
|
|
7247
7276
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
7248
7277
|
)
|
|
7249
7278
|
`);
|
|
7279
|
+
try {
|
|
7280
|
+
await db.execute(`ALTER TABLE workflow_artifacts ADD COLUMN content TEXT`);
|
|
7281
|
+
} catch {
|
|
7282
|
+
}
|
|
7250
7283
|
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_artifacts_run ON workflow_artifacts(run_id)`);
|
|
7251
7284
|
await db.execute(`
|
|
7252
7285
|
CREATE TABLE IF NOT EXISTS workflow_webhook_deliveries (
|
|
@@ -7692,9 +7725,22 @@ function rowToWorkflowRun(row) {
|
|
|
7692
7725
|
queued_at: String(row.queued_at),
|
|
7693
7726
|
started_at: row.started_at != null ? String(row.started_at) : null,
|
|
7694
7727
|
completed_at: row.completed_at != null ? String(row.completed_at) : null,
|
|
7695
|
-
idempotency_key: row.idempotency_key != null ? String(row.idempotency_key) : null
|
|
7728
|
+
idempotency_key: row.idempotency_key != null ? String(row.idempotency_key) : null,
|
|
7729
|
+
step_index: row.step_index != null ? Number(row.step_index) : 0,
|
|
7730
|
+
total_steps: row.total_steps != null ? Number(row.total_steps) : null,
|
|
7731
|
+
step_log: parseWorkflowStepLog(row.step_log_json),
|
|
7732
|
+
webhook_url: row.webhook_url != null ? String(row.webhook_url) : null
|
|
7696
7733
|
};
|
|
7697
7734
|
}
|
|
7735
|
+
function parseWorkflowStepLog(value) {
|
|
7736
|
+
if (value == null) return [];
|
|
7737
|
+
try {
|
|
7738
|
+
const parsed = JSON.parse(String(value));
|
|
7739
|
+
return Array.isArray(parsed) ? parsed : [];
|
|
7740
|
+
} catch {
|
|
7741
|
+
return [];
|
|
7742
|
+
}
|
|
7743
|
+
}
|
|
7698
7744
|
function rowToWorkflowArtifact(row) {
|
|
7699
7745
|
return {
|
|
7700
7746
|
id: String(row.id),
|
|
@@ -7706,6 +7752,7 @@ function rowToWorkflowArtifact(row) {
|
|
|
7706
7752
|
content_type: String(row.content_type),
|
|
7707
7753
|
bytes: row.bytes != null ? Number(row.bytes) : null,
|
|
7708
7754
|
rows_count: row.rows_count != null ? Number(row.rows_count) : null,
|
|
7755
|
+
content: row.content != null ? String(row.content) : null,
|
|
7709
7756
|
created_at: String(row.created_at)
|
|
7710
7757
|
};
|
|
7711
7758
|
}
|
|
@@ -7817,13 +7864,15 @@ async function createWorkflowRun(input) {
|
|
|
7817
7864
|
input.scheduleId ?? null,
|
|
7818
7865
|
input.workflowId,
|
|
7819
7866
|
JSON.stringify(input.workflowInput),
|
|
7820
|
-
input.idempotencyKey ?? null
|
|
7867
|
+
input.idempotencyKey ?? null,
|
|
7868
|
+
input.webhookUrl ?? null,
|
|
7869
|
+
input.totalSteps ?? null
|
|
7821
7870
|
];
|
|
7822
7871
|
try {
|
|
7823
7872
|
await getDb().execute({
|
|
7824
7873
|
sql: `
|
|
7825
|
-
INSERT INTO workflow_runs (id, user_id, schedule_id, workflow_id, input_json, idempotency_key)
|
|
7826
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
7874
|
+
INSERT INTO workflow_runs (id, user_id, schedule_id, workflow_id, input_json, idempotency_key, webhook_url, total_steps)
|
|
7875
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
7827
7876
|
`,
|
|
7828
7877
|
args
|
|
7829
7878
|
});
|
|
@@ -7877,30 +7926,53 @@ function workflowArtifactContentType(kind) {
|
|
|
7877
7926
|
if (kind === "json") return "application/json; charset=utf-8";
|
|
7878
7927
|
return "text/plain; charset=utf-8";
|
|
7879
7928
|
}
|
|
7929
|
+
async function insertWorkflowArtifact(runId, userId, artifact) {
|
|
7930
|
+
await getDb().execute({
|
|
7931
|
+
sql: `
|
|
7932
|
+
INSERT INTO workflow_artifacts (id, run_id, user_id, kind, label, path, content_type, bytes, rows_count, content)
|
|
7933
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
7934
|
+
`,
|
|
7935
|
+
args: [
|
|
7936
|
+
(0, import_node_crypto.randomUUID)(),
|
|
7937
|
+
runId,
|
|
7938
|
+
userId,
|
|
7939
|
+
artifact.kind,
|
|
7940
|
+
artifact.label,
|
|
7941
|
+
artifact.path,
|
|
7942
|
+
workflowArtifactContentType(artifact.kind),
|
|
7943
|
+
artifact.bytes ?? null,
|
|
7944
|
+
artifact.rows ?? null,
|
|
7945
|
+
artifact.content ?? null
|
|
7946
|
+
]
|
|
7947
|
+
});
|
|
7948
|
+
}
|
|
7880
7949
|
async function replaceWorkflowArtifacts(runId, userId, artifacts) {
|
|
7881
|
-
|
|
7882
|
-
|
|
7883
|
-
for (const artifact of artifacts) {
|
|
7884
|
-
await db.execute({
|
|
7885
|
-
sql: `
|
|
7886
|
-
INSERT INTO workflow_artifacts (id, run_id, user_id, kind, label, path, content_type, bytes, rows_count)
|
|
7887
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
7888
|
-
`,
|
|
7889
|
-
args: [
|
|
7890
|
-
(0, import_node_crypto.randomUUID)(),
|
|
7891
|
-
runId,
|
|
7892
|
-
userId,
|
|
7893
|
-
artifact.kind,
|
|
7894
|
-
artifact.label,
|
|
7895
|
-
artifact.path,
|
|
7896
|
-
workflowArtifactContentType(artifact.kind),
|
|
7897
|
-
artifact.bytes ?? null,
|
|
7898
|
-
artifact.rows ?? null
|
|
7899
|
-
]
|
|
7900
|
-
});
|
|
7901
|
-
}
|
|
7950
|
+
await getDb().execute({ sql: "DELETE FROM workflow_artifacts WHERE run_id = ? AND user_id = ?", args: [runId, userId] });
|
|
7951
|
+
for (const artifact of artifacts) await insertWorkflowArtifact(runId, userId, artifact);
|
|
7902
7952
|
return listWorkflowArtifacts(runId, userId);
|
|
7903
7953
|
}
|
|
7954
|
+
async function appendWorkflowArtifacts(runId, userId, artifacts) {
|
|
7955
|
+
for (const artifact of artifacts) await insertWorkflowArtifact(runId, userId, artifact);
|
|
7956
|
+
return listWorkflowArtifacts(runId, userId);
|
|
7957
|
+
}
|
|
7958
|
+
async function getWorkflowRunStepState(id, userId) {
|
|
7959
|
+
const res = userId == null ? await getDb().execute({ sql: "SELECT step_index, total_steps, step_state_json, step_log_json FROM workflow_runs WHERE id = ? LIMIT 1", args: [id] }) : await getDb().execute({ sql: "SELECT step_index, total_steps, step_state_json, step_log_json FROM workflow_runs WHERE id = ? AND user_id = ? LIMIT 1", args: [id, userId] });
|
|
7960
|
+
const row = res.rows[0];
|
|
7961
|
+
if (!row) return null;
|
|
7962
|
+
return {
|
|
7963
|
+
stepIndex: row.step_index != null ? Number(row.step_index) : 0,
|
|
7964
|
+
totalSteps: row.total_steps != null ? Number(row.total_steps) : null,
|
|
7965
|
+
state: row.step_state_json != null ? parseNullableJson(row.step_state_json) : null,
|
|
7966
|
+
stepLog: parseWorkflowStepLog(row.step_log_json)
|
|
7967
|
+
};
|
|
7968
|
+
}
|
|
7969
|
+
async function advanceWorkflowRunStep(id, input) {
|
|
7970
|
+
const stepLog = [...input.appendLog, input.logEntry];
|
|
7971
|
+
await getDb().execute({
|
|
7972
|
+
sql: `UPDATE workflow_runs SET status = ?, step_index = ?, total_steps = ?, step_state_json = ?, step_log_json = ?, started_at = COALESCE(started_at, datetime('now')) WHERE id = ?`,
|
|
7973
|
+
args: [input.status, input.stepIndex, input.totalSteps, JSON.stringify(input.state ?? null), JSON.stringify(stepLog), id]
|
|
7974
|
+
});
|
|
7975
|
+
}
|
|
7904
7976
|
async function listWorkflowArtifacts(runId, userId) {
|
|
7905
7977
|
const res = await getDb().execute({
|
|
7906
7978
|
sql: "SELECT * FROM workflow_artifacts WHERE run_id = ? AND user_id = ? ORDER BY created_at ASC",
|
|
@@ -8909,6 +8981,7 @@ var init_schemas2 = __esm({
|
|
|
8909
8981
|
downloadMp3: import_zod12.z.boolean().default(false),
|
|
8910
8982
|
outputDir: import_zod12.z.string().default("./yt-output"),
|
|
8911
8983
|
kernelApiKey: import_zod12.z.string().optional(),
|
|
8984
|
+
kernelProxyId: import_zod12.z.string().optional(),
|
|
8912
8985
|
headless: import_zod12.z.boolean().default(false)
|
|
8913
8986
|
}).refine(
|
|
8914
8987
|
(o) => o.mode === "search" ? !!o.query : !!o.channelHandle,
|
|
@@ -9833,6 +9906,7 @@ var init_YouTubeExtractor = __esm({
|
|
|
9833
9906
|
const config = {
|
|
9834
9907
|
headless: options.headless,
|
|
9835
9908
|
kernelApiKey: options.kernelApiKey,
|
|
9909
|
+
kernelProxyId: options.kernelProxyId,
|
|
9836
9910
|
viewport: { width: 1280, height: 800 },
|
|
9837
9911
|
locale: "en-US"
|
|
9838
9912
|
};
|
|
@@ -10044,7 +10118,11 @@ async function ytHarvest(rawOptions) {
|
|
|
10044
10118
|
throw new Error("A browser backend API key is required \u2014 YouTube harvesting requires a stealth session.");
|
|
10045
10119
|
}
|
|
10046
10120
|
const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};
|
|
10047
|
-
const options = YouTubeHarvestOptionsSchema.parse({
|
|
10121
|
+
const options = YouTubeHarvestOptionsSchema.parse({
|
|
10122
|
+
kernelApiKey,
|
|
10123
|
+
kernelProxyId: browserServiceProxyId(),
|
|
10124
|
+
...raw
|
|
10125
|
+
});
|
|
10048
10126
|
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
10049
10127
|
try {
|
|
10050
10128
|
const result = await extractOnce(options);
|
|
@@ -10153,10 +10231,11 @@ function parseTimedtextXml(xml) {
|
|
|
10153
10231
|
async function fetchViaKernelInnertube(videoId) {
|
|
10154
10232
|
const kernelApiKey = browserServiceApiKey();
|
|
10155
10233
|
if (!kernelApiKey) return null;
|
|
10234
|
+
const kernelProxyId = browserServiceProxyId();
|
|
10156
10235
|
const driver = new BrowserDriver();
|
|
10157
10236
|
const start = Date.now();
|
|
10158
10237
|
try {
|
|
10159
|
-
await driver.launch({ kernelApiKey, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
|
|
10238
|
+
await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
|
|
10160
10239
|
const page = driver.getPage();
|
|
10161
10240
|
await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
|
|
10162
10241
|
await page.waitForFunction(
|
|
@@ -10250,10 +10329,10 @@ async function captureAudioViaMediaRecorder(page) {
|
|
|
10250
10329
|
if (!result.ok || !result.base64 || !result.mimeType) return null;
|
|
10251
10330
|
return { bytes: Buffer.from(result.base64, "base64"), mimeType: result.mimeType };
|
|
10252
10331
|
}
|
|
10253
|
-
async function attemptKernelWhisper(videoId, kernelApiKey, falKey, start) {
|
|
10332
|
+
async function attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start) {
|
|
10254
10333
|
const driver = new BrowserDriver();
|
|
10255
10334
|
try {
|
|
10256
|
-
await driver.launch({ kernelApiKey, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
|
|
10335
|
+
await driver.launch({ kernelApiKey, kernelProxyId, headless: true, viewport: { width: 1280, height: 800 }, locale: "en-US" });
|
|
10257
10336
|
const page = driver.getPage();
|
|
10258
10337
|
await driver.navigateTo(`https://www.youtube.com/watch?v=${videoId}`);
|
|
10259
10338
|
await page.waitForFunction(
|
|
@@ -10295,12 +10374,13 @@ async function attemptKernelWhisper(videoId, kernelApiKey, falKey, start) {
|
|
|
10295
10374
|
}
|
|
10296
10375
|
async function fetchViaKernelWhisper(videoId) {
|
|
10297
10376
|
const kernelApiKey = browserServiceApiKey();
|
|
10377
|
+
const kernelProxyId = browserServiceProxyId();
|
|
10298
10378
|
const falKey = process.env.FAL_KEY;
|
|
10299
10379
|
if (!kernelApiKey || !falKey) return null;
|
|
10300
10380
|
const start = Date.now();
|
|
10301
10381
|
for (let attempt = 0; attempt < 4; attempt++) {
|
|
10302
10382
|
if (attempt > 0) await new Promise((r) => setTimeout(r, 2e3 * attempt));
|
|
10303
|
-
const result = await attemptKernelWhisper(videoId, kernelApiKey, falKey, start);
|
|
10383
|
+
const result = await attemptKernelWhisper(videoId, kernelApiKey, kernelProxyId, falKey, start);
|
|
10304
10384
|
if (result) return result;
|
|
10305
10385
|
}
|
|
10306
10386
|
return null;
|
|
@@ -12283,6 +12363,110 @@ var init_kernel_proxy_resolver = __esm({
|
|
|
12283
12363
|
}
|
|
12284
12364
|
});
|
|
12285
12365
|
|
|
12366
|
+
// src/schemas.ts
|
|
12367
|
+
var import_zod15, DEFAULT_PROXY_MODE, DEFAULT_MAPS_PROXY_MODE, HarvestOptionsSchema, MapsPlaceOptionsSchema, MapsSearchOptionsSchema, RawPAAItemSchema, RawMapsOverviewSchema, RawMapsHoursRowSchema, RawMapsReviewStatsSchema, RawMapsReviewCardSchema, RawMapsAboutAttributeSchema;
|
|
12368
|
+
var init_schemas3 = __esm({
|
|
12369
|
+
"src/schemas.ts"() {
|
|
12370
|
+
"use strict";
|
|
12371
|
+
import_zod15 = require("zod");
|
|
12372
|
+
DEFAULT_PROXY_MODE = "configured";
|
|
12373
|
+
DEFAULT_MAPS_PROXY_MODE = "location";
|
|
12374
|
+
HarvestOptionsSchema = import_zod15.z.object({
|
|
12375
|
+
query: import_zod15.z.string().min(1),
|
|
12376
|
+
location: import_zod15.z.string().optional(),
|
|
12377
|
+
gl: import_zod15.z.string().length(2).default("us"),
|
|
12378
|
+
hl: import_zod15.z.string().length(2).default("en"),
|
|
12379
|
+
device: import_zod15.z.enum(["desktop", "mobile"]).default("desktop"),
|
|
12380
|
+
proxyMode: import_zod15.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
|
|
12381
|
+
proxyZip: import_zod15.z.string().regex(/^\d{5}$/).optional(),
|
|
12382
|
+
debug: import_zod15.z.boolean().default(false),
|
|
12383
|
+
depth: import_zod15.z.number().int().min(1).max(30).default(3),
|
|
12384
|
+
maxQuestions: import_zod15.z.number().int().min(1).max(1e3).default(100),
|
|
12385
|
+
headless: import_zod15.z.boolean().default(false),
|
|
12386
|
+
profileDir: import_zod15.z.string().optional(),
|
|
12387
|
+
proxy: import_zod15.z.string().url().optional(),
|
|
12388
|
+
kernelApiKey: import_zod15.z.string().optional(),
|
|
12389
|
+
kernelProxyId: import_zod15.z.string().optional(),
|
|
12390
|
+
kernelProxyResolution: import_zod15.z.unknown().optional(),
|
|
12391
|
+
outputDir: import_zod15.z.string().default("./paa-output"),
|
|
12392
|
+
format: import_zod15.z.enum(["json", "csv", "both"]).default("both"),
|
|
12393
|
+
serpOnly: import_zod15.z.boolean().default(false),
|
|
12394
|
+
pages: import_zod15.z.number().int().min(1).max(2).default(1)
|
|
12395
|
+
});
|
|
12396
|
+
MapsPlaceOptionsSchema = import_zod15.z.object({
|
|
12397
|
+
businessName: import_zod15.z.string().min(1),
|
|
12398
|
+
location: import_zod15.z.string().min(1),
|
|
12399
|
+
gl: import_zod15.z.string().length(2).default("us"),
|
|
12400
|
+
hl: import_zod15.z.string().length(2).default("en"),
|
|
12401
|
+
includeReviews: import_zod15.z.boolean().default(false),
|
|
12402
|
+
maxReviews: import_zod15.z.number().int().min(1).max(500).default(50),
|
|
12403
|
+
kernelApiKey: import_zod15.z.string().optional(),
|
|
12404
|
+
kernelProxyId: import_zod15.z.string().optional(),
|
|
12405
|
+
headless: import_zod15.z.boolean().default(true)
|
|
12406
|
+
});
|
|
12407
|
+
MapsSearchOptionsSchema = import_zod15.z.object({
|
|
12408
|
+
query: import_zod15.z.string().min(1),
|
|
12409
|
+
location: import_zod15.z.string().optional(),
|
|
12410
|
+
gl: import_zod15.z.string().length(2).default("us"),
|
|
12411
|
+
hl: import_zod15.z.string().length(2).default("en"),
|
|
12412
|
+
maxResults: import_zod15.z.number().int().min(1).max(50).default(10),
|
|
12413
|
+
proxyMode: import_zod15.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
12414
|
+
proxyZip: import_zod15.z.string().regex(/^\d{5}$/).optional(),
|
|
12415
|
+
debug: import_zod15.z.boolean().default(false),
|
|
12416
|
+
kernelApiKey: import_zod15.z.string().optional(),
|
|
12417
|
+
kernelProxyId: import_zod15.z.string().optional(),
|
|
12418
|
+
kernelProxyResolution: import_zod15.z.unknown().optional(),
|
|
12419
|
+
headless: import_zod15.z.boolean().default(true)
|
|
12420
|
+
});
|
|
12421
|
+
RawPAAItemSchema = import_zod15.z.object({
|
|
12422
|
+
question: import_zod15.z.string().min(1),
|
|
12423
|
+
answer: import_zod15.z.string().optional(),
|
|
12424
|
+
sourceTitle: import_zod15.z.string().optional(),
|
|
12425
|
+
sourceSite: import_zod15.z.string().optional(),
|
|
12426
|
+
sourceCite: import_zod15.z.string().optional()
|
|
12427
|
+
});
|
|
12428
|
+
RawMapsOverviewSchema = import_zod15.z.object({
|
|
12429
|
+
name: import_zod15.z.string().nullable(),
|
|
12430
|
+
rating: import_zod15.z.string().nullable(),
|
|
12431
|
+
reviewCount: import_zod15.z.string().nullable(),
|
|
12432
|
+
category: import_zod15.z.string().nullable(),
|
|
12433
|
+
address: import_zod15.z.string().nullable(),
|
|
12434
|
+
hoursSummary: import_zod15.z.string().nullable(),
|
|
12435
|
+
phone: import_zod15.z.string().nullable(),
|
|
12436
|
+
phoneDisplay: import_zod15.z.string().nullable(),
|
|
12437
|
+
website: import_zod15.z.string().nullable(),
|
|
12438
|
+
plusCode: import_zod15.z.string().nullable(),
|
|
12439
|
+
bookingUrl: import_zod15.z.string().nullable()
|
|
12440
|
+
});
|
|
12441
|
+
RawMapsHoursRowSchema = import_zod15.z.object({
|
|
12442
|
+
day: import_zod15.z.string(),
|
|
12443
|
+
hours: import_zod15.z.string()
|
|
12444
|
+
});
|
|
12445
|
+
RawMapsReviewStatsSchema = import_zod15.z.object({
|
|
12446
|
+
reviewHistogram: import_zod15.z.array(import_zod15.z.object({
|
|
12447
|
+
stars: import_zod15.z.number(),
|
|
12448
|
+
count: import_zod15.z.string()
|
|
12449
|
+
})),
|
|
12450
|
+
reviewTopics: import_zod15.z.array(import_zod15.z.object({
|
|
12451
|
+
label: import_zod15.z.string(),
|
|
12452
|
+
count: import_zod15.z.string()
|
|
12453
|
+
}))
|
|
12454
|
+
});
|
|
12455
|
+
RawMapsReviewCardSchema = import_zod15.z.object({
|
|
12456
|
+
reviewId: import_zod15.z.string(),
|
|
12457
|
+
author: import_zod15.z.string().nullable(),
|
|
12458
|
+
stars: import_zod15.z.string().nullable(),
|
|
12459
|
+
date: import_zod15.z.string().nullable(),
|
|
12460
|
+
text: import_zod15.z.string().nullable(),
|
|
12461
|
+
ownerResponse: import_zod15.z.string().nullable()
|
|
12462
|
+
});
|
|
12463
|
+
RawMapsAboutAttributeSchema = import_zod15.z.object({
|
|
12464
|
+
section: import_zod15.z.string(),
|
|
12465
|
+
attribute: import_zod15.z.string()
|
|
12466
|
+
});
|
|
12467
|
+
}
|
|
12468
|
+
});
|
|
12469
|
+
|
|
12286
12470
|
// src/api/facebook-ad-routes.ts
|
|
12287
12471
|
function invalidRequest(message) {
|
|
12288
12472
|
return { error_code: "invalid_request", message };
|
|
@@ -12338,9 +12522,8 @@ async function kernelLaunchOptsResidential() {
|
|
|
12338
12522
|
try {
|
|
12339
12523
|
const resolution2 = await resolveKernelProxyId({
|
|
12340
12524
|
kernelApiKey: browserServiceApiKey(),
|
|
12341
|
-
proxyMode:
|
|
12525
|
+
proxyMode: DEFAULT_PROXY_MODE,
|
|
12342
12526
|
configuredKernelProxyId: browserServiceProxyId(),
|
|
12343
|
-
location: "New York, NY",
|
|
12344
12527
|
gl: "us"
|
|
12345
12528
|
});
|
|
12346
12529
|
if (resolution2.kernelProxyId) proxyId = resolution2.kernelProxyId;
|
|
@@ -12349,13 +12532,13 @@ async function kernelLaunchOptsResidential() {
|
|
|
12349
12532
|
}
|
|
12350
12533
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
12351
12534
|
}
|
|
12352
|
-
var import_hono4,
|
|
12535
|
+
var import_hono4, import_zod16, import_client3, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
|
|
12353
12536
|
var init_facebook_ad_routes = __esm({
|
|
12354
12537
|
"src/api/facebook-ad-routes.ts"() {
|
|
12355
12538
|
"use strict";
|
|
12356
12539
|
import_hono4 = require("hono");
|
|
12357
12540
|
init_browser_service_env();
|
|
12358
|
-
|
|
12541
|
+
import_zod16 = require("zod");
|
|
12359
12542
|
init_db();
|
|
12360
12543
|
init_rates();
|
|
12361
12544
|
init_BrowserDriver();
|
|
@@ -12363,39 +12546,40 @@ var init_facebook_ad_routes = __esm({
|
|
|
12363
12546
|
init_FacebookAdGraphql();
|
|
12364
12547
|
init_FacebookOrganicVideoExtractor();
|
|
12365
12548
|
init_kernel_proxy_resolver();
|
|
12549
|
+
init_schemas3();
|
|
12366
12550
|
import_client3 = require("@fal-ai/client");
|
|
12367
12551
|
init_api_auth();
|
|
12368
12552
|
init_url_utils();
|
|
12369
12553
|
init_concurrency_gates();
|
|
12370
|
-
FacebookAdBodySchema =
|
|
12371
|
-
url:
|
|
12372
|
-
libraryId:
|
|
12373
|
-
openModal:
|
|
12554
|
+
FacebookAdBodySchema = import_zod16.z.object({
|
|
12555
|
+
url: import_zod16.z.string().trim().optional(),
|
|
12556
|
+
libraryId: import_zod16.z.string().trim().optional(),
|
|
12557
|
+
openModal: import_zod16.z.boolean().optional()
|
|
12374
12558
|
}).refine((d) => !!d.url || !!d.libraryId, { message: "url or libraryId is required" });
|
|
12375
|
-
FacebookPageIntelBodySchema =
|
|
12376
|
-
pageId:
|
|
12377
|
-
query:
|
|
12378
|
-
libraryId:
|
|
12379
|
-
maxAds:
|
|
12380
|
-
country:
|
|
12559
|
+
FacebookPageIntelBodySchema = import_zod16.z.object({
|
|
12560
|
+
pageId: import_zod16.z.string().trim().optional(),
|
|
12561
|
+
query: import_zod16.z.string().trim().optional(),
|
|
12562
|
+
libraryId: import_zod16.z.string().trim().optional(),
|
|
12563
|
+
maxAds: import_zod16.z.number().int().min(1).max(200).optional(),
|
|
12564
|
+
country: import_zod16.z.string().trim().toUpperCase().optional()
|
|
12381
12565
|
}).refine((d) => !!d.pageId || !!d.query || !!d.libraryId, {
|
|
12382
12566
|
message: "pageId, libraryId, or query is required"
|
|
12383
12567
|
});
|
|
12384
|
-
FacebookTranscribeBodySchema =
|
|
12385
|
-
videoUrl:
|
|
12568
|
+
FacebookTranscribeBodySchema = import_zod16.z.object({
|
|
12569
|
+
videoUrl: import_zod16.z.string().trim().min(1, "videoUrl is required")
|
|
12386
12570
|
});
|
|
12387
|
-
FacebookVideoTranscribeBodySchema =
|
|
12388
|
-
url:
|
|
12389
|
-
quality:
|
|
12571
|
+
FacebookVideoTranscribeBodySchema = import_zod16.z.object({
|
|
12572
|
+
url: import_zod16.z.string().trim().min(1, "url is required"),
|
|
12573
|
+
quality: import_zod16.z.enum(["best", "hd", "sd"]).default("best")
|
|
12390
12574
|
});
|
|
12391
|
-
FacebookSearchBodySchema =
|
|
12392
|
-
query:
|
|
12393
|
-
country:
|
|
12394
|
-
maxResults:
|
|
12575
|
+
FacebookSearchBodySchema = import_zod16.z.object({
|
|
12576
|
+
query: import_zod16.z.string().trim().min(1, "query is required"),
|
|
12577
|
+
country: import_zod16.z.string().trim().toUpperCase().optional(),
|
|
12578
|
+
maxResults: import_zod16.z.number().int().min(1).max(20).optional()
|
|
12395
12579
|
});
|
|
12396
|
-
FacebookMediaBodySchema =
|
|
12397
|
-
url:
|
|
12398
|
-
filename:
|
|
12580
|
+
FacebookMediaBodySchema = import_zod16.z.object({
|
|
12581
|
+
url: import_zod16.z.string().trim().min(1, "url is required"),
|
|
12582
|
+
filename: import_zod16.z.string().trim().optional()
|
|
12399
12583
|
});
|
|
12400
12584
|
facebookAdApp = new import_hono4.Hono();
|
|
12401
12585
|
facebookAdApp.post("/ad", createApiKeyAuth(), async (c) => {
|
|
@@ -12721,108 +12905,6 @@ var init_facebook_ad_routes = __esm({
|
|
|
12721
12905
|
}
|
|
12722
12906
|
});
|
|
12723
12907
|
|
|
12724
|
-
// src/schemas.ts
|
|
12725
|
-
var import_zod16, HarvestOptionsSchema, MapsPlaceOptionsSchema, MapsSearchOptionsSchema, RawPAAItemSchema, RawMapsOverviewSchema, RawMapsHoursRowSchema, RawMapsReviewStatsSchema, RawMapsReviewCardSchema, RawMapsAboutAttributeSchema;
|
|
12726
|
-
var init_schemas3 = __esm({
|
|
12727
|
-
"src/schemas.ts"() {
|
|
12728
|
-
"use strict";
|
|
12729
|
-
import_zod16 = require("zod");
|
|
12730
|
-
HarvestOptionsSchema = import_zod16.z.object({
|
|
12731
|
-
query: import_zod16.z.string().min(1),
|
|
12732
|
-
location: import_zod16.z.string().optional(),
|
|
12733
|
-
gl: import_zod16.z.string().length(2).default("us"),
|
|
12734
|
-
hl: import_zod16.z.string().length(2).default("en"),
|
|
12735
|
-
device: import_zod16.z.enum(["desktop", "mobile"]).default("desktop"),
|
|
12736
|
-
proxyMode: import_zod16.z.enum(["location", "configured", "none"]).default("location"),
|
|
12737
|
-
proxyZip: import_zod16.z.string().regex(/^\d{5}$/).optional(),
|
|
12738
|
-
debug: import_zod16.z.boolean().default(false),
|
|
12739
|
-
depth: import_zod16.z.number().int().min(1).max(30).default(3),
|
|
12740
|
-
maxQuestions: import_zod16.z.number().int().min(1).max(1e3).default(100),
|
|
12741
|
-
headless: import_zod16.z.boolean().default(false),
|
|
12742
|
-
profileDir: import_zod16.z.string().optional(),
|
|
12743
|
-
proxy: import_zod16.z.string().url().optional(),
|
|
12744
|
-
kernelApiKey: import_zod16.z.string().optional(),
|
|
12745
|
-
kernelProxyId: import_zod16.z.string().optional(),
|
|
12746
|
-
kernelProxyResolution: import_zod16.z.unknown().optional(),
|
|
12747
|
-
outputDir: import_zod16.z.string().default("./paa-output"),
|
|
12748
|
-
format: import_zod16.z.enum(["json", "csv", "both"]).default("both"),
|
|
12749
|
-
serpOnly: import_zod16.z.boolean().default(false),
|
|
12750
|
-
pages: import_zod16.z.number().int().min(1).max(2).default(1)
|
|
12751
|
-
});
|
|
12752
|
-
MapsPlaceOptionsSchema = import_zod16.z.object({
|
|
12753
|
-
businessName: import_zod16.z.string().min(1),
|
|
12754
|
-
location: import_zod16.z.string().min(1),
|
|
12755
|
-
gl: import_zod16.z.string().length(2).default("us"),
|
|
12756
|
-
hl: import_zod16.z.string().length(2).default("en"),
|
|
12757
|
-
includeReviews: import_zod16.z.boolean().default(false),
|
|
12758
|
-
maxReviews: import_zod16.z.number().int().min(1).max(500).default(50),
|
|
12759
|
-
kernelApiKey: import_zod16.z.string().optional(),
|
|
12760
|
-
kernelProxyId: import_zod16.z.string().optional(),
|
|
12761
|
-
headless: import_zod16.z.boolean().default(true)
|
|
12762
|
-
});
|
|
12763
|
-
MapsSearchOptionsSchema = import_zod16.z.object({
|
|
12764
|
-
query: import_zod16.z.string().min(1),
|
|
12765
|
-
location: import_zod16.z.string().optional(),
|
|
12766
|
-
gl: import_zod16.z.string().length(2).default("us"),
|
|
12767
|
-
hl: import_zod16.z.string().length(2).default("en"),
|
|
12768
|
-
maxResults: import_zod16.z.number().int().min(1).max(50).default(10),
|
|
12769
|
-
proxyMode: import_zod16.z.enum(["location", "configured", "none"]).default("location"),
|
|
12770
|
-
proxyZip: import_zod16.z.string().regex(/^\d{5}$/).optional(),
|
|
12771
|
-
debug: import_zod16.z.boolean().default(false),
|
|
12772
|
-
kernelApiKey: import_zod16.z.string().optional(),
|
|
12773
|
-
kernelProxyId: import_zod16.z.string().optional(),
|
|
12774
|
-
kernelProxyResolution: import_zod16.z.unknown().optional(),
|
|
12775
|
-
headless: import_zod16.z.boolean().default(true)
|
|
12776
|
-
});
|
|
12777
|
-
RawPAAItemSchema = import_zod16.z.object({
|
|
12778
|
-
question: import_zod16.z.string().min(1),
|
|
12779
|
-
answer: import_zod16.z.string().optional(),
|
|
12780
|
-
sourceTitle: import_zod16.z.string().optional(),
|
|
12781
|
-
sourceSite: import_zod16.z.string().optional(),
|
|
12782
|
-
sourceCite: import_zod16.z.string().optional()
|
|
12783
|
-
});
|
|
12784
|
-
RawMapsOverviewSchema = import_zod16.z.object({
|
|
12785
|
-
name: import_zod16.z.string().nullable(),
|
|
12786
|
-
rating: import_zod16.z.string().nullable(),
|
|
12787
|
-
reviewCount: import_zod16.z.string().nullable(),
|
|
12788
|
-
category: import_zod16.z.string().nullable(),
|
|
12789
|
-
address: import_zod16.z.string().nullable(),
|
|
12790
|
-
hoursSummary: import_zod16.z.string().nullable(),
|
|
12791
|
-
phone: import_zod16.z.string().nullable(),
|
|
12792
|
-
phoneDisplay: import_zod16.z.string().nullable(),
|
|
12793
|
-
website: import_zod16.z.string().nullable(),
|
|
12794
|
-
plusCode: import_zod16.z.string().nullable(),
|
|
12795
|
-
bookingUrl: import_zod16.z.string().nullable()
|
|
12796
|
-
});
|
|
12797
|
-
RawMapsHoursRowSchema = import_zod16.z.object({
|
|
12798
|
-
day: import_zod16.z.string(),
|
|
12799
|
-
hours: import_zod16.z.string()
|
|
12800
|
-
});
|
|
12801
|
-
RawMapsReviewStatsSchema = import_zod16.z.object({
|
|
12802
|
-
reviewHistogram: import_zod16.z.array(import_zod16.z.object({
|
|
12803
|
-
stars: import_zod16.z.number(),
|
|
12804
|
-
count: import_zod16.z.string()
|
|
12805
|
-
})),
|
|
12806
|
-
reviewTopics: import_zod16.z.array(import_zod16.z.object({
|
|
12807
|
-
label: import_zod16.z.string(),
|
|
12808
|
-
count: import_zod16.z.string()
|
|
12809
|
-
}))
|
|
12810
|
-
});
|
|
12811
|
-
RawMapsReviewCardSchema = import_zod16.z.object({
|
|
12812
|
-
reviewId: import_zod16.z.string(),
|
|
12813
|
-
author: import_zod16.z.string().nullable(),
|
|
12814
|
-
stars: import_zod16.z.string().nullable(),
|
|
12815
|
-
date: import_zod16.z.string().nullable(),
|
|
12816
|
-
text: import_zod16.z.string().nullable(),
|
|
12817
|
-
ownerResponse: import_zod16.z.string().nullable()
|
|
12818
|
-
});
|
|
12819
|
-
RawMapsAboutAttributeSchema = import_zod16.z.object({
|
|
12820
|
-
section: import_zod16.z.string(),
|
|
12821
|
-
attribute: import_zod16.z.string()
|
|
12822
|
-
});
|
|
12823
|
-
}
|
|
12824
|
-
});
|
|
12825
|
-
|
|
12826
12908
|
// src/extractor/MapsNavigator.ts
|
|
12827
12909
|
var MapsNavigator;
|
|
12828
12910
|
var init_MapsNavigator = __esm({
|
|
@@ -14432,13 +14514,25 @@ ${videoRows}`,
|
|
|
14432
14514
|
}
|
|
14433
14515
|
};
|
|
14434
14516
|
}
|
|
14517
|
+
function structuredTranscriptChunks(chunks) {
|
|
14518
|
+
return chunks.map((c) => ({
|
|
14519
|
+
startSec: Number.isFinite(c.timestamp?.[0]) ? c.timestamp[0] : 0,
|
|
14520
|
+
endSec: Number.isFinite(c.timestamp?.[1]) ? c.timestamp[1] : 0,
|
|
14521
|
+
text: c.text
|
|
14522
|
+
}));
|
|
14523
|
+
}
|
|
14524
|
+
function wordCount(text) {
|
|
14525
|
+
return text.trim() ? text.trim().split(/\s+/).length : 0;
|
|
14526
|
+
}
|
|
14435
14527
|
function formatYoutubeTranscribe(raw, input) {
|
|
14436
14528
|
const parsed = parseData(raw);
|
|
14437
14529
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
14438
14530
|
const d = parsed.data;
|
|
14439
14531
|
const text = d.text ?? "";
|
|
14440
14532
|
const chunks = d.chunks ?? [];
|
|
14533
|
+
const videoId = d.videoId ?? input.videoId ?? null;
|
|
14441
14534
|
const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
|
|
14535
|
+
const words = wordCount(text);
|
|
14442
14536
|
const chunkRows = chunks.slice(0, 50).map((c) => {
|
|
14443
14537
|
const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
|
|
14444
14538
|
const mm = String(Math.floor(sec / 60)).padStart(2, "0");
|
|
@@ -14446,8 +14540,8 @@ function formatYoutubeTranscribe(raw, input) {
|
|
|
14446
14540
|
return `| ${mm}:${ss} | ${cell(truncate(c.text, 120))} |`;
|
|
14447
14541
|
}).join("\n");
|
|
14448
14542
|
const full = [
|
|
14449
|
-
`# YouTube Transcript: \`${input.
|
|
14450
|
-
`**Duration:** ${durSec}s \xB7 **${
|
|
14543
|
+
`# YouTube Transcript: \`${videoId ?? input.url ?? "video"}\``,
|
|
14544
|
+
`**Duration:** ${durSec}s \xB7 **${words} words**`,
|
|
14451
14545
|
`
|
|
14452
14546
|
## Full Transcript
|
|
14453
14547
|
${text}`,
|
|
@@ -14460,7 +14554,22 @@ ${chunkRows}` : "",
|
|
|
14460
14554
|
---
|
|
14461
14555
|
\u{1F4A1} Harvest more from this channel: use \`youtube_harvest\` with \`mode: "channel"\``
|
|
14462
14556
|
].filter(Boolean).join("\n");
|
|
14463
|
-
return
|
|
14557
|
+
return {
|
|
14558
|
+
...oneBlock(full),
|
|
14559
|
+
structuredContent: {
|
|
14560
|
+
videoId,
|
|
14561
|
+
url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : input.url ?? null,
|
|
14562
|
+
wordCount: words,
|
|
14563
|
+
chunkCount: chunks.length,
|
|
14564
|
+
durationMs: typeof d.durationMs === "number" ? d.durationMs : null,
|
|
14565
|
+
transcriptText: text,
|
|
14566
|
+
chunks: structuredTranscriptChunks(chunks),
|
|
14567
|
+
resolvedInputs: {
|
|
14568
|
+
videoId,
|
|
14569
|
+
url: input.url ?? null
|
|
14570
|
+
}
|
|
14571
|
+
}
|
|
14572
|
+
};
|
|
14464
14573
|
}
|
|
14465
14574
|
function formatFacebookPageIntel(raw, input) {
|
|
14466
14575
|
const parsed = parseData(raw);
|
|
@@ -14470,12 +14579,13 @@ function formatFacebookPageIntel(raw, input) {
|
|
|
14470
14579
|
const ads = d.ads ?? [];
|
|
14471
14580
|
const s = d.summary ?? { totalAds: 0, activeCount: 0, videoCount: 0, imageCount: 0 };
|
|
14472
14581
|
const adBlocks = ads.map((ad, i) => [
|
|
14473
|
-
`### Ad ${i + 1}${ad.libraryId ? ` \xB7 \`${ad.libraryId}\`` : ""} \u2014 ${ad.status ?? "\u2014"} \xB7 ${ad.creativeType ?? "\u2014"} \xB7 ${ad.startDate ?? "\u2014"}`,
|
|
14582
|
+
`### Ad ${i + 1}${ad.libraryId ? ` \xB7 \`${ad.libraryId}\`` : ""} \u2014 ${ad.status ?? "\u2014"} \xB7 ${ad.creativeType ?? "\u2014"} \xB7 ${ad.startDate ?? ad.started ?? "\u2014"}`,
|
|
14474
14583
|
ad.headline ? `**Headline:** ${ad.headline}` : "",
|
|
14475
14584
|
ad.primaryText ? `**Copy:** ${truncate(ad.primaryText, 200)}` : "",
|
|
14476
14585
|
ad.cta ? `**CTA:** ${ad.cta}` : "",
|
|
14477
|
-
ad.
|
|
14478
|
-
ad.
|
|
14586
|
+
ad.landingUrl ? `**Landing URL:** ${ad.landingUrl}` : "",
|
|
14587
|
+
ad.videoUrl ?? ad.videoSrc ? `**Video URL:** \`${ad.videoUrl ?? ad.videoSrc}\`` : "",
|
|
14588
|
+
ad.variations ?? ad.clusterCount ? `**Variations:** ${ad.variations ?? ad.clusterCount}` : ""
|
|
14479
14589
|
].filter(Boolean).join("\n")).join("\n\n---\n\n");
|
|
14480
14590
|
const full = [
|
|
14481
14591
|
`# Facebook Ad Intel: ${advertiser}`,
|
|
@@ -14501,11 +14611,16 @@ ${adBlocks}`,
|
|
|
14501
14611
|
libraryId: ad.libraryId ?? null,
|
|
14502
14612
|
status: ad.status ?? null,
|
|
14503
14613
|
creativeType: ad.creativeType ?? null,
|
|
14614
|
+
primaryText: ad.primaryText ?? null,
|
|
14504
14615
|
headline: ad.headline ?? null,
|
|
14505
14616
|
cta: ad.cta ?? null,
|
|
14506
|
-
startDate: ad.startDate ?? null,
|
|
14507
|
-
|
|
14508
|
-
|
|
14617
|
+
startDate: ad.startDate ?? ad.started ?? null,
|
|
14618
|
+
landingUrl: ad.landingUrl ?? null,
|
|
14619
|
+
domain: ad.domain ?? null,
|
|
14620
|
+
videoUrl: ad.videoUrl ?? ad.videoSrc ?? null,
|
|
14621
|
+
imageUrl: ad.imageUrl ?? ad.imageSrc ?? null,
|
|
14622
|
+
videoPoster: ad.videoPoster ?? null,
|
|
14623
|
+
variations: typeof ad.variations === "number" ? ad.variations : typeof ad.clusterCount === "number" ? ad.clusterCount : null
|
|
14509
14624
|
}))
|
|
14510
14625
|
}
|
|
14511
14626
|
};
|
|
@@ -14539,8 +14654,11 @@ ${rows}`,
|
|
|
14539
14654
|
advertiserCount: advertisers.length,
|
|
14540
14655
|
advertisers: advertisers.map((a) => ({
|
|
14541
14656
|
name: a.pageName ?? a.name ?? null,
|
|
14657
|
+
pageId: a.pageId ?? null,
|
|
14658
|
+
pageUrl: a.pageUrl ?? null,
|
|
14542
14659
|
adCount: typeof a.adCount === "number" ? a.adCount : null,
|
|
14543
|
-
libraryId: a.sampleLibraryId ?? a.libraryId ?? null
|
|
14660
|
+
libraryId: a.sampleLibraryId ?? a.libraryId ?? null,
|
|
14661
|
+
sampleLibraryId: a.sampleLibraryId ?? null
|
|
14544
14662
|
}))
|
|
14545
14663
|
}
|
|
14546
14664
|
};
|
|
@@ -14638,6 +14756,34 @@ function formatWorkflowSuggest(input) {
|
|
|
14638
14756
|
}
|
|
14639
14757
|
};
|
|
14640
14758
|
}
|
|
14759
|
+
function workflowStepLines(data) {
|
|
14760
|
+
const step = data.step;
|
|
14761
|
+
const nextStep = data.nextStep;
|
|
14762
|
+
const done = data.done === true;
|
|
14763
|
+
const lines = [];
|
|
14764
|
+
if (step) {
|
|
14765
|
+
const index = Number(step.index ?? 0);
|
|
14766
|
+
const totalSteps = step.totalSteps != null ? Number(step.totalSteps) : void 0;
|
|
14767
|
+
const stepLabel = totalSteps ? `${index + 1}/${totalSteps}` : `${index + 1}`;
|
|
14768
|
+
lines.push(`
|
|
14769
|
+
## Step ${stepLabel}: ${step.title ?? step.id ?? ""}`);
|
|
14770
|
+
const output = step.output;
|
|
14771
|
+
if (output && Object.keys(output).length) {
|
|
14772
|
+
lines.push(Object.entries(output).map(([k, v]) => `- ${k}: ${typeof v === "object" ? JSON.stringify(v) : String(v)}`).join("\n"));
|
|
14773
|
+
}
|
|
14774
|
+
const warnings = Array.isArray(step.warnings) ? step.warnings : [];
|
|
14775
|
+
if (warnings.length) lines.push(`
|
|
14776
|
+
**Warnings:**
|
|
14777
|
+
${warnings.map((w) => `- ${w}`).join("\n")}`);
|
|
14778
|
+
}
|
|
14779
|
+
if (done) {
|
|
14780
|
+
lines.push("\n**Done.** All steps complete.");
|
|
14781
|
+
} else if (nextStep && typeof nextStep === "object") {
|
|
14782
|
+
lines.push(`
|
|
14783
|
+
**Next step:** \`${nextStep.id ?? nextStep.index}\` \u2014 call \`workflow_step\` with this run id to continue.`);
|
|
14784
|
+
}
|
|
14785
|
+
return lines;
|
|
14786
|
+
}
|
|
14641
14787
|
function formatWorkflowRun(raw, input) {
|
|
14642
14788
|
const parsed = parseData(raw);
|
|
14643
14789
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
@@ -14651,6 +14797,7 @@ function formatWorkflowRun(raw, input) {
|
|
|
14651
14797
|
`**Run ID:** \`${runId || "unknown"}\``,
|
|
14652
14798
|
`**Status:** ${status}`,
|
|
14653
14799
|
summary?.title ? `**Title:** ${summary.title}` : "",
|
|
14800
|
+
...workflowStepLines(parsed.data),
|
|
14654
14801
|
summary?.summary ? `
|
|
14655
14802
|
## Summary
|
|
14656
14803
|
${summary.summary}` : "",
|
|
@@ -14666,6 +14813,42 @@ ${workflowArtifactRows(artifacts)}` : "",
|
|
|
14666
14813
|
input: input.input ?? {},
|
|
14667
14814
|
run,
|
|
14668
14815
|
summary,
|
|
14816
|
+
step: parsed.data.step,
|
|
14817
|
+
nextStep: parsed.data.nextStep ?? null,
|
|
14818
|
+
done: parsed.data.done === true,
|
|
14819
|
+
artifacts
|
|
14820
|
+
}
|
|
14821
|
+
};
|
|
14822
|
+
}
|
|
14823
|
+
function formatWorkflowStep(raw, input) {
|
|
14824
|
+
const parsed = parseData(raw);
|
|
14825
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
14826
|
+
const run = parsed.data.run;
|
|
14827
|
+
const summary = parsed.data.summary;
|
|
14828
|
+
const artifacts = workflowArtifactsFrom(run);
|
|
14829
|
+
const done = parsed.data.done === true;
|
|
14830
|
+
const full = [
|
|
14831
|
+
`# Workflow Step`,
|
|
14832
|
+
`**Run ID:** \`${input.runId}\``,
|
|
14833
|
+
`**Status:** ${run?.status ?? (done ? "done" : "running")}`,
|
|
14834
|
+
...workflowStepLines(parsed.data),
|
|
14835
|
+
done && summary?.summary ? `
|
|
14836
|
+
## Summary
|
|
14837
|
+
${summary.summary}` : "",
|
|
14838
|
+
artifacts.length ? `
|
|
14839
|
+
## Artifacts
|
|
14840
|
+
${workflowArtifactRows(artifacts)}` : "",
|
|
14841
|
+
done && artifacts.length ? "\nUse `workflow_artifact_read` with the run id and artifact id to pull CSV, JSON, Markdown, or report content into context." : ""
|
|
14842
|
+
].filter(Boolean).join("\n");
|
|
14843
|
+
return {
|
|
14844
|
+
...oneBlock(full),
|
|
14845
|
+
structuredContent: {
|
|
14846
|
+
runId: input.runId,
|
|
14847
|
+
run,
|
|
14848
|
+
summary: summary ?? null,
|
|
14849
|
+
step: parsed.data.step,
|
|
14850
|
+
nextStep: parsed.data.nextStep ?? null,
|
|
14851
|
+
done,
|
|
14669
14852
|
artifacts
|
|
14670
14853
|
}
|
|
14671
14854
|
};
|
|
@@ -15070,6 +15253,7 @@ function formatFacebookAdTranscribe(raw, input) {
|
|
|
15070
15253
|
const text = d.text ?? "";
|
|
15071
15254
|
const chunks = d.chunks ?? [];
|
|
15072
15255
|
const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
|
|
15256
|
+
const words = wordCount(text);
|
|
15073
15257
|
const chunkRows = chunks.slice(0, 50).map((c) => {
|
|
15074
15258
|
const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
|
|
15075
15259
|
const mm = String(Math.floor(sec / 60)).padStart(2, "0");
|
|
@@ -15078,7 +15262,7 @@ function formatFacebookAdTranscribe(raw, input) {
|
|
|
15078
15262
|
}).join("\n");
|
|
15079
15263
|
const full = [
|
|
15080
15264
|
`# Facebook Ad Transcript`,
|
|
15081
|
-
`**Duration:** ${durSec}s \xB7 **${
|
|
15265
|
+
`**Duration:** ${durSec}s \xB7 **${words} words**`,
|
|
15082
15266
|
`
|
|
15083
15267
|
## Full Transcript
|
|
15084
15268
|
${text}`,
|
|
@@ -15091,7 +15275,20 @@ ${chunkRows}` : "",
|
|
|
15091
15275
|
---
|
|
15092
15276
|
\u{1F4A1} Get more ads from this advertiser: use \`facebook_page_intel\`. For public Facebook reel/post URLs, use \`facebook_video_transcribe\`.`
|
|
15093
15277
|
].filter(Boolean).join("\n");
|
|
15094
|
-
return
|
|
15278
|
+
return {
|
|
15279
|
+
...oneBlock(full),
|
|
15280
|
+
structuredContent: {
|
|
15281
|
+
videoUrl: input.videoUrl,
|
|
15282
|
+
wordCount: words,
|
|
15283
|
+
chunkCount: chunks.length,
|
|
15284
|
+
durationMs: typeof d.durationMs === "number" ? d.durationMs : null,
|
|
15285
|
+
transcriptText: text,
|
|
15286
|
+
chunks: structuredTranscriptChunks(chunks),
|
|
15287
|
+
resolvedInputs: {
|
|
15288
|
+
videoUrl: input.videoUrl
|
|
15289
|
+
}
|
|
15290
|
+
}
|
|
15291
|
+
};
|
|
15095
15292
|
}
|
|
15096
15293
|
function formatFacebookVideoTranscribe(raw, input) {
|
|
15097
15294
|
const parsed = parseData(raw);
|
|
@@ -15099,7 +15296,7 @@ function formatFacebookVideoTranscribe(raw, input) {
|
|
|
15099
15296
|
const d = parsed.data;
|
|
15100
15297
|
const text = d.text ?? "";
|
|
15101
15298
|
const chunks = d.chunks ?? [];
|
|
15102
|
-
const
|
|
15299
|
+
const wordCount2 = text.trim() ? text.trim().split(/\s+/).length : 0;
|
|
15103
15300
|
const durSec = d.durationMs ? (d.durationMs / 1e3).toFixed(0) : "\u2014";
|
|
15104
15301
|
const videoDuration = typeof d.videoDurationSec === "number" ? `${Math.round(d.videoDurationSec)}s` : "\u2014";
|
|
15105
15302
|
const label = d.videoId ? `Facebook Organic Video \`${d.videoId}\`` : "Facebook Organic Video";
|
|
@@ -15112,7 +15309,7 @@ function formatFacebookVideoTranscribe(raw, input) {
|
|
|
15112
15309
|
const full = [
|
|
15113
15310
|
`# ${label} Transcript`,
|
|
15114
15311
|
d.ownerName ? `**Owner:** ${d.ownerName}` : "",
|
|
15115
|
-
`**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${
|
|
15312
|
+
`**Video duration:** ${videoDuration} \xB7 **Transcribed in:** ${durSec}s \xB7 **${wordCount2} words**`,
|
|
15116
15313
|
d.pageUrl ? `**Page URL:** ${d.pageUrl}` : `**Page URL:** ${input.url}`,
|
|
15117
15314
|
d.videoUrl ? `**Extracted MP4:** \`${d.videoUrl}\`` : "",
|
|
15118
15315
|
`
|
|
@@ -15138,7 +15335,7 @@ ${chunkRows}` : "",
|
|
|
15138
15335
|
bitrate: typeof d.bitrate === "number" ? d.bitrate : null,
|
|
15139
15336
|
videoDurationSec: typeof d.videoDurationSec === "number" ? d.videoDurationSec : null,
|
|
15140
15337
|
videoUrl: d.videoUrl ?? "",
|
|
15141
|
-
wordCount,
|
|
15338
|
+
wordCount: wordCount2,
|
|
15142
15339
|
chunkCount: chunks.length,
|
|
15143
15340
|
transcriptText: text,
|
|
15144
15341
|
chunks: chunks.map((c) => ({
|
|
@@ -15149,6 +15346,96 @@ ${chunkRows}` : "",
|
|
|
15149
15346
|
}
|
|
15150
15347
|
};
|
|
15151
15348
|
}
|
|
15349
|
+
function formatCaptureSerpSnapshot(raw, input) {
|
|
15350
|
+
const parsed = parseData(raw);
|
|
15351
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
15352
|
+
const d = parsed.data;
|
|
15353
|
+
const providerPayload = d;
|
|
15354
|
+
const organic = Array.isArray(d.organicResults) ? d.organicResults : Array.isArray(d.organic) ? d.organic : [];
|
|
15355
|
+
const localPack = Array.isArray(d.localPack) ? d.localPack : [];
|
|
15356
|
+
const artifacts = Array.isArray(d.artifacts) ? d.artifacts : [];
|
|
15357
|
+
const status = String(d.status ?? d.captureStatus ?? "captured");
|
|
15358
|
+
const query = typeof d.query === "string" ? d.query : typeof input.query === "string" ? input.query : null;
|
|
15359
|
+
const location2 = typeof d.location === "string" ? d.location : typeof input.location === "string" ? input.location : null;
|
|
15360
|
+
const capturedAt = typeof d.capturedAt === "string" ? d.capturedAt : typeof d.extractedAt === "string" ? d.extractedAt : null;
|
|
15361
|
+
const resultCount = organic.length + localPack.length;
|
|
15362
|
+
const full = [
|
|
15363
|
+
`# SERP Intelligence Snapshot: ${query ?? "query"}`,
|
|
15364
|
+
`**Status:** ${status}`,
|
|
15365
|
+
location2 ? `**Location:** ${location2}` : "",
|
|
15366
|
+
`**Result count:** ${resultCount}`,
|
|
15367
|
+
artifacts.length ? `**Artifacts:** ${artifacts.length}` : "",
|
|
15368
|
+
"",
|
|
15369
|
+
"Use `capture_serp_page_snapshots` when you need page-level evidence for ranking URLs."
|
|
15370
|
+
].filter(Boolean).join("\n");
|
|
15371
|
+
return {
|
|
15372
|
+
...oneBlock(full),
|
|
15373
|
+
structuredContent: {
|
|
15374
|
+
schemaVersion: "serp-intelligence.capture.v1",
|
|
15375
|
+
status,
|
|
15376
|
+
query,
|
|
15377
|
+
location: location2,
|
|
15378
|
+
capturedAt,
|
|
15379
|
+
resultCount,
|
|
15380
|
+
snapshotId: typeof d.snapshotId === "string" ? d.snapshotId : typeof d.snapshot_id === "string" ? d.snapshot_id : typeof d.id === "string" ? d.id : null,
|
|
15381
|
+
resolvedInputs: {
|
|
15382
|
+
query: input.query ?? null,
|
|
15383
|
+
location: input.location ?? null,
|
|
15384
|
+
gl: input.gl ?? null,
|
|
15385
|
+
hl: input.hl ?? null,
|
|
15386
|
+
device: input.device ?? null,
|
|
15387
|
+
proxyMode: input.proxyMode ?? null,
|
|
15388
|
+
proxyZip: input.proxyZip ?? null,
|
|
15389
|
+
pages: input.pages ?? null
|
|
15390
|
+
},
|
|
15391
|
+
artifacts,
|
|
15392
|
+
diagnostics: d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null,
|
|
15393
|
+
providerPayload
|
|
15394
|
+
}
|
|
15395
|
+
};
|
|
15396
|
+
}
|
|
15397
|
+
function formatCaptureSerpPageSnapshots(raw, input) {
|
|
15398
|
+
const parsed = parseData(raw);
|
|
15399
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
15400
|
+
const d = parsed.data;
|
|
15401
|
+
const captures = Array.isArray(d.captures) ? d.captures : Array.isArray(d.results) ? d.results : [];
|
|
15402
|
+
const failedCount = captures.filter((c) => c.status === "failed" || c.error).length;
|
|
15403
|
+
const status = String(d.status ?? (failedCount ? "partial" : "captured"));
|
|
15404
|
+
const rows = captures.slice(0, 25).map((capture, i) => {
|
|
15405
|
+
const url = typeof capture.url === "string" ? capture.url : "";
|
|
15406
|
+
const sourceKind = typeof capture.sourceKind === "string" ? capture.sourceKind : typeof capture.source_kind === "string" ? capture.source_kind : "configured_target";
|
|
15407
|
+
return `| ${i + 1} | ${cell(url)} | ${cell(sourceKind)} | ${cell(String(capture.status ?? (capture.error ? "failed" : "ok")))} |`;
|
|
15408
|
+
}).join("\n");
|
|
15409
|
+
const full = [
|
|
15410
|
+
"# SERP Intelligence Page Snapshots",
|
|
15411
|
+
`**Status:** ${status}`,
|
|
15412
|
+
`**Captured:** ${captures.length}`,
|
|
15413
|
+
`**Failed:** ${failedCount}`,
|
|
15414
|
+
captures.length ? `
|
|
15415
|
+
| # | URL | Source | Status |
|
|
15416
|
+
|---|-----|--------|--------|
|
|
15417
|
+
${rows}` : ""
|
|
15418
|
+
].filter(Boolean).join("\n");
|
|
15419
|
+
return {
|
|
15420
|
+
...oneBlock(full),
|
|
15421
|
+
structuredContent: {
|
|
15422
|
+
schemaVersion: "serp-intelligence.page-snapshots.v1",
|
|
15423
|
+
status,
|
|
15424
|
+
count: captures.length,
|
|
15425
|
+
failedCount,
|
|
15426
|
+
captures,
|
|
15427
|
+
resolvedInputs: {
|
|
15428
|
+
urls: input.urls ?? null,
|
|
15429
|
+
targets: input.targets ?? null,
|
|
15430
|
+
maxConcurrency: input.maxConcurrency ?? null,
|
|
15431
|
+
timeoutMs: input.timeoutMs ?? null,
|
|
15432
|
+
debug: input.debug ?? null
|
|
15433
|
+
},
|
|
15434
|
+
diagnostics: d.diagnostics && typeof d.diagnostics === "object" ? d.diagnostics : null,
|
|
15435
|
+
providerPayload: d
|
|
15436
|
+
}
|
|
15437
|
+
};
|
|
15438
|
+
}
|
|
15152
15439
|
var import_node_fs3, import_node_os3, import_node_path5, reportSavingEnabled;
|
|
15153
15440
|
var init_mcp_response_formatter = __esm({
|
|
15154
15441
|
"src/mcp/mcp-response-formatter.ts"() {
|
|
@@ -15623,6 +15910,7 @@ var init_directory_workflow = __esm({
|
|
|
15623
15910
|
init_mcp_response_formatter();
|
|
15624
15911
|
init_browser_service_env();
|
|
15625
15912
|
init_maps_search_rotation();
|
|
15913
|
+
init_schemas3();
|
|
15626
15914
|
init_csv();
|
|
15627
15915
|
init_location_db();
|
|
15628
15916
|
DirectoryWorkflowOptionsSchema = import_zod18.z.object({
|
|
@@ -15638,7 +15926,7 @@ var init_directory_workflow = __esm({
|
|
|
15638
15926
|
saveCsv: import_zod18.z.boolean().default(true),
|
|
15639
15927
|
gl: import_zod18.z.string().length(2).default("us"),
|
|
15640
15928
|
hl: import_zod18.z.string().length(2).default("en"),
|
|
15641
|
-
proxyMode: import_zod18.z.enum(["location", "configured", "none"]).default(
|
|
15929
|
+
proxyMode: import_zod18.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
15642
15930
|
proxyZip: import_zod18.z.string().regex(/^\d{5}$/).optional(),
|
|
15643
15931
|
debug: import_zod18.z.boolean().default(false),
|
|
15644
15932
|
headless: import_zod18.z.boolean().default(true),
|
|
@@ -16062,102 +16350,141 @@ var init_agent_packet = __esm({
|
|
|
16062
16350
|
title: "Agent-Ready SEO Packet",
|
|
16063
16351
|
description: "Create an evidence folder for AI agents from live SERP/PAA/AI search surfaces.",
|
|
16064
16352
|
inputSchema: AgentPacketInputSchema,
|
|
16065
|
-
|
|
16066
|
-
|
|
16067
|
-
|
|
16068
|
-
|
|
16069
|
-
|
|
16070
|
-
|
|
16071
|
-
|
|
16072
|
-
|
|
16073
|
-
|
|
16074
|
-
|
|
16075
|
-
|
|
16076
|
-
|
|
16077
|
-
|
|
16078
|
-
|
|
16079
|
-
|
|
16080
|
-
|
|
16081
|
-
|
|
16353
|
+
createState: () => ({ serp: null, paa: null, warnings: [] }),
|
|
16354
|
+
steps: [
|
|
16355
|
+
{
|
|
16356
|
+
id: "harvest-serp",
|
|
16357
|
+
title: "Harvest organic SERP + AI Overview",
|
|
16358
|
+
async run({ input, state, ctx }) {
|
|
16359
|
+
if (!input.includeSerp) {
|
|
16360
|
+
return { state, output: { skipped: true, reason: "includeSerp is false" } };
|
|
16361
|
+
}
|
|
16362
|
+
try {
|
|
16363
|
+
const serp = await ctx.client.post("/harvest/sync", {
|
|
16364
|
+
query: input.keyword,
|
|
16365
|
+
location: input.location,
|
|
16366
|
+
serpOnly: true,
|
|
16367
|
+
maxQuestions: 1,
|
|
16368
|
+
format: "json"
|
|
16369
|
+
}, 18e4);
|
|
16370
|
+
await ctx.artifacts.writeJson("SERP raw JSON", "raw/serp.json", serp);
|
|
16371
|
+
return {
|
|
16372
|
+
state: { ...state, serp },
|
|
16373
|
+
output: {
|
|
16374
|
+
organicResults: serp.organicResults?.length ?? 0,
|
|
16375
|
+
localPack: serp.localPack?.length ?? 0,
|
|
16376
|
+
aiOverviewDetected: Boolean(serp.aiOverview?.detected),
|
|
16377
|
+
aiOverviewCitations: serp.aiOverview?.citations?.length ?? 0
|
|
16378
|
+
}
|
|
16379
|
+
};
|
|
16380
|
+
} catch (err) {
|
|
16381
|
+
const message = `SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
|
|
16382
|
+
return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
|
|
16383
|
+
}
|
|
16082
16384
|
}
|
|
16083
|
-
}
|
|
16084
|
-
|
|
16085
|
-
|
|
16086
|
-
|
|
16087
|
-
|
|
16088
|
-
|
|
16089
|
-
|
|
16090
|
-
|
|
16091
|
-
|
|
16092
|
-
|
|
16093
|
-
|
|
16385
|
+
},
|
|
16386
|
+
{
|
|
16387
|
+
id: "harvest-paa",
|
|
16388
|
+
title: "Harvest People Also Ask",
|
|
16389
|
+
async run({ input, state, ctx }) {
|
|
16390
|
+
if (!input.includePaa) {
|
|
16391
|
+
return { state, output: { skipped: true, reason: "includePaa is false" } };
|
|
16392
|
+
}
|
|
16393
|
+
try {
|
|
16394
|
+
const paa = await ctx.client.post("/harvest/sync", {
|
|
16395
|
+
query: input.keyword,
|
|
16396
|
+
location: input.location,
|
|
16397
|
+
maxQuestions: input.maxQuestions,
|
|
16398
|
+
format: "json"
|
|
16399
|
+
}, 28e4);
|
|
16400
|
+
await ctx.artifacts.writeJson("PAA raw JSON", "raw/paa.json", paa);
|
|
16401
|
+
return {
|
|
16402
|
+
state: { ...state, paa },
|
|
16403
|
+
output: { paaQuestions: paa.flat?.length ?? 0 }
|
|
16404
|
+
};
|
|
16405
|
+
} catch (err) {
|
|
16406
|
+
const message = `PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
|
|
16407
|
+
return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
|
|
16408
|
+
}
|
|
16409
|
+
}
|
|
16410
|
+
},
|
|
16411
|
+
{
|
|
16412
|
+
id: "assemble",
|
|
16413
|
+
title: "Assemble evidence packet",
|
|
16414
|
+
async run({ input, state, ctx }) {
|
|
16415
|
+
const { serp, paa } = state;
|
|
16416
|
+
const warnings = state.warnings;
|
|
16417
|
+
if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
|
|
16418
|
+
const rows = sourceRows(input, serp, paa);
|
|
16419
|
+
const target = normalizeDomain(input.domain);
|
|
16420
|
+
const competitors = competitorRows(rows, target);
|
|
16421
|
+
const evidence = {
|
|
16422
|
+
input,
|
|
16423
|
+
serp: serp ?? { status: "skipped" },
|
|
16424
|
+
paa: paa ?? { status: "skipped" },
|
|
16425
|
+
target: {
|
|
16426
|
+
domain: target,
|
|
16427
|
+
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
16428
|
+
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
16429
|
+
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
16430
|
+
},
|
|
16431
|
+
competitors,
|
|
16432
|
+
warnings
|
|
16433
|
+
};
|
|
16434
|
+
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
16435
|
+
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
16436
|
+
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
16437
|
+
const brief = [
|
|
16438
|
+
`# SEO Evidence Brief: ${input.keyword}`,
|
|
16439
|
+
"",
|
|
16440
|
+
`Location: ${input.location ?? "not specified"}`,
|
|
16441
|
+
`Target domain: ${target ?? "not specified"}`,
|
|
16442
|
+
"",
|
|
16443
|
+
"## Evidence Summary",
|
|
16444
|
+
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
16445
|
+
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
16446
|
+
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
16447
|
+
`- Competitor domains: ${competitors.length}`,
|
|
16448
|
+
"",
|
|
16449
|
+
"## Recommended Use",
|
|
16450
|
+
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
16451
|
+
].join("\n");
|
|
16452
|
+
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
16453
|
+
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
16454
|
+
"# Agent Tasks",
|
|
16455
|
+
"",
|
|
16456
|
+
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
16457
|
+
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
16458
|
+
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
16459
|
+
"- [ ] Mark unsupported recommendations as assumptions."
|
|
16460
|
+
].join("\n"));
|
|
16461
|
+
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
16462
|
+
"# Agent Instructions",
|
|
16463
|
+
"",
|
|
16464
|
+
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
16465
|
+
].join("\n"));
|
|
16466
|
+
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
16467
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
16468
|
+
title: "Agent-Ready SEO Packet",
|
|
16469
|
+
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
16470
|
+
summary,
|
|
16471
|
+
warnings,
|
|
16472
|
+
tables: [
|
|
16473
|
+
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
16474
|
+
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
16475
|
+
]
|
|
16476
|
+
}));
|
|
16477
|
+
const status = warnings.length ? "partial" : "succeeded";
|
|
16478
|
+
const counts = { sources: rows.length, competitors: competitors.length };
|
|
16479
|
+
await ctx.artifacts.writeManifest(status, counts, warnings, []);
|
|
16480
|
+
return {
|
|
16481
|
+
state,
|
|
16482
|
+
output: { sources: rows.length, competitors: competitors.length, status },
|
|
16483
|
+
summary: { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors: [], reportPath }
|
|
16484
|
+
};
|
|
16094
16485
|
}
|
|
16095
16486
|
}
|
|
16096
|
-
|
|
16097
|
-
const rows = sourceRows(input, serp, paa);
|
|
16098
|
-
const target = normalizeDomain(input.domain);
|
|
16099
|
-
const competitors = competitorRows(rows, target);
|
|
16100
|
-
const evidence = {
|
|
16101
|
-
input,
|
|
16102
|
-
serp: serp ?? { status: "skipped" },
|
|
16103
|
-
paa: paa ?? { status: "skipped" },
|
|
16104
|
-
target: {
|
|
16105
|
-
domain: target,
|
|
16106
|
-
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
16107
|
-
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
16108
|
-
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
16109
|
-
},
|
|
16110
|
-
competitors,
|
|
16111
|
-
warnings
|
|
16112
|
-
};
|
|
16113
|
-
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
16114
|
-
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
16115
|
-
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
16116
|
-
const brief = [
|
|
16117
|
-
`# SEO Evidence Brief: ${input.keyword}`,
|
|
16118
|
-
"",
|
|
16119
|
-
`Location: ${input.location ?? "not specified"}`,
|
|
16120
|
-
`Target domain: ${target ?? "not specified"}`,
|
|
16121
|
-
"",
|
|
16122
|
-
"## Evidence Summary",
|
|
16123
|
-
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
16124
|
-
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
16125
|
-
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
16126
|
-
`- Competitor domains: ${competitors.length}`,
|
|
16127
|
-
"",
|
|
16128
|
-
"## Recommended Use",
|
|
16129
|
-
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
16130
|
-
].join("\n");
|
|
16131
|
-
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
16132
|
-
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
16133
|
-
"# Agent Tasks",
|
|
16134
|
-
"",
|
|
16135
|
-
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
16136
|
-
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
16137
|
-
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
16138
|
-
"- [ ] Mark unsupported recommendations as assumptions."
|
|
16139
|
-
].join("\n"));
|
|
16140
|
-
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
16141
|
-
"# Agent Instructions",
|
|
16142
|
-
"",
|
|
16143
|
-
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
16144
|
-
].join("\n"));
|
|
16145
|
-
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
16146
|
-
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
16147
|
-
title: "Agent-Ready SEO Packet",
|
|
16148
|
-
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
16149
|
-
summary,
|
|
16150
|
-
warnings,
|
|
16151
|
-
tables: [
|
|
16152
|
-
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
16153
|
-
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
16154
|
-
]
|
|
16155
|
-
}));
|
|
16156
|
-
const status = warnings.length ? "partial" : "succeeded";
|
|
16157
|
-
const counts = { sources: rows.length, competitors: competitors.length };
|
|
16158
|
-
await ctx.artifacts.writeManifest(status, counts, warnings, errors);
|
|
16159
|
-
return { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors, reportPath };
|
|
16160
|
-
}
|
|
16487
|
+
]
|
|
16161
16488
|
};
|
|
16162
16489
|
}
|
|
16163
16490
|
});
|
|
@@ -16220,6 +16547,7 @@ var init_directory = __esm({
|
|
|
16220
16547
|
"use strict";
|
|
16221
16548
|
import_zod20 = require("zod");
|
|
16222
16549
|
init_report_renderer();
|
|
16550
|
+
init_schemas3();
|
|
16223
16551
|
DirectoryWorkflowCliInputSchema = import_zod20.z.object({
|
|
16224
16552
|
query: import_zod20.z.string().min(1),
|
|
16225
16553
|
state: import_zod20.z.string().min(2).default("TN"),
|
|
@@ -16227,7 +16555,7 @@ var init_directory = __esm({
|
|
|
16227
16555
|
maxCities: import_zod20.z.number().int().min(1).max(100).default(25),
|
|
16228
16556
|
maxResultsPerCity: import_zod20.z.number().int().min(1).max(50).default(20),
|
|
16229
16557
|
concurrency: import_zod20.z.number().int().min(1).max(5).default(5),
|
|
16230
|
-
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default(
|
|
16558
|
+
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
16231
16559
|
saveCsv: import_zod20.z.boolean().default(true)
|
|
16232
16560
|
});
|
|
16233
16561
|
DIRECTORY_CSV_HEADERS = [
|
|
@@ -16344,6 +16672,7 @@ var init_local_competitive_audit = __esm({
|
|
|
16344
16672
|
import_zod21 = require("zod");
|
|
16345
16673
|
init_report_renderer();
|
|
16346
16674
|
init_directory();
|
|
16675
|
+
init_schemas3();
|
|
16347
16676
|
LocalCompetitiveAuditInputSchema = import_zod21.z.object({
|
|
16348
16677
|
query: import_zod21.z.string().min(1),
|
|
16349
16678
|
state: import_zod21.z.string().min(2).default("TN"),
|
|
@@ -16353,7 +16682,7 @@ var init_local_competitive_audit = __esm({
|
|
|
16353
16682
|
hydrateTop: import_zod21.z.number().int().min(0).max(10).default(5),
|
|
16354
16683
|
maxReviews: import_zod21.z.number().int().min(0).max(500).default(50),
|
|
16355
16684
|
concurrency: import_zod21.z.number().int().min(1).max(5).default(5),
|
|
16356
|
-
proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default(
|
|
16685
|
+
proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
16357
16686
|
returnPartial: import_zod21.z.boolean().default(true)
|
|
16358
16687
|
});
|
|
16359
16688
|
localCompetitiveAuditWorkflowDefinition = {
|
|
@@ -16805,6 +17134,7 @@ var init_comparison_briefs = __esm({
|
|
|
16805
17134
|
init_report_renderer();
|
|
16806
17135
|
init_directory();
|
|
16807
17136
|
init_seo_workflow_utils();
|
|
17137
|
+
init_schemas3();
|
|
16808
17138
|
ProxyModeSchema = import_zod22.z.enum(["location", "configured", "none"]);
|
|
16809
17139
|
MapComparisonInputSchema = import_zod22.z.object({
|
|
16810
17140
|
query: import_zod22.z.string().min(1),
|
|
@@ -16816,7 +17146,7 @@ var init_comparison_briefs = __esm({
|
|
|
16816
17146
|
hydrateTop: import_zod22.z.number().int().min(0).max(10).default(5),
|
|
16817
17147
|
maxReviews: import_zod22.z.number().int().min(0).max(500).default(25),
|
|
16818
17148
|
concurrency: import_zod22.z.number().int().min(1).max(5).default(5),
|
|
16819
|
-
proxyMode: ProxyModeSchema.default(
|
|
17149
|
+
proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
|
|
16820
17150
|
returnPartial: import_zod22.z.boolean().default(true)
|
|
16821
17151
|
}).refine((input) => input.location || input.state, {
|
|
16822
17152
|
message: "Either location or state is required for map-comparison"
|
|
@@ -17230,22 +17560,44 @@ function workflowDefinition(id) {
|
|
|
17230
17560
|
if (!definition) throw new Error(`Unknown workflow "${id}". Available: ${DEFINITIONS.map((def) => def.id).join(", ")}`);
|
|
17231
17561
|
return definition;
|
|
17232
17562
|
}
|
|
17233
|
-
|
|
17563
|
+
function workflowStepCount(id) {
|
|
17234
17564
|
const definition = workflowDefinition(id);
|
|
17235
|
-
|
|
17565
|
+
return definition.steps?.length ?? 1;
|
|
17566
|
+
}
|
|
17567
|
+
function workflowSupportsSteps(id) {
|
|
17568
|
+
const definition = workflowDefinition(id);
|
|
17569
|
+
return Boolean(definition.steps && definition.steps.length > 0);
|
|
17570
|
+
}
|
|
17571
|
+
function resolveApiKey(options) {
|
|
17236
17572
|
const apiKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim();
|
|
17237
17573
|
if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
|
|
17238
|
-
|
|
17574
|
+
return apiKey;
|
|
17575
|
+
}
|
|
17576
|
+
function resolveApiUrl(options) {
|
|
17577
|
+
return options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
17578
|
+
}
|
|
17579
|
+
async function runWorkflow(id, rawInput, options = {}) {
|
|
17580
|
+
const definition = workflowDefinition(id);
|
|
17581
|
+
const input = definition.inputSchema.parse(rawInput);
|
|
17582
|
+
const apiKey = resolveApiKey(options);
|
|
17583
|
+
const apiUrl = resolveApiUrl(options);
|
|
17239
17584
|
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
17240
17585
|
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
|
|
17586
|
+
const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client: client2, artifacts, signal: options.signal };
|
|
17241
17587
|
try {
|
|
17242
|
-
|
|
17243
|
-
|
|
17244
|
-
|
|
17245
|
-
|
|
17246
|
-
|
|
17247
|
-
|
|
17248
|
-
|
|
17588
|
+
if (definition.steps && definition.steps.length > 0) {
|
|
17589
|
+
let state = definition.createState ? definition.createState(input) : {};
|
|
17590
|
+
let summary = null;
|
|
17591
|
+
for (const step of definition.steps) {
|
|
17592
|
+
const outcome = await step.run({ input, state, ctx });
|
|
17593
|
+
state = outcome.state;
|
|
17594
|
+
if (outcome.summary) summary = outcome.summary;
|
|
17595
|
+
}
|
|
17596
|
+
if (!summary) throw new Error(`Workflow "${id}" produced no terminal step summary`);
|
|
17597
|
+
return summary;
|
|
17598
|
+
}
|
|
17599
|
+
if (definition.run) return await definition.run(input, ctx);
|
|
17600
|
+
throw new Error(`Workflow "${id}" has neither steps nor a run() implementation`);
|
|
17249
17601
|
} catch (err) {
|
|
17250
17602
|
const message = err instanceof Error ? err.message : String(err);
|
|
17251
17603
|
await artifacts.writeText("Failure", "summary.md", `# ${definition.title}
|
|
@@ -17256,10 +17608,50 @@ ${message}
|
|
|
17256
17608
|
throw err;
|
|
17257
17609
|
}
|
|
17258
17610
|
}
|
|
17259
|
-
|
|
17611
|
+
async function runWorkflowStep(id, rawInput, opts) {
|
|
17612
|
+
const definition = workflowDefinition(id);
|
|
17613
|
+
const steps = definition.steps;
|
|
17614
|
+
if (!steps || steps.length === 0) throw new Error(`Workflow "${id}" does not support stepwise execution`);
|
|
17615
|
+
if (opts.stepIndex < 0 || opts.stepIndex >= steps.length) {
|
|
17616
|
+
throw new Error(`Step index ${opts.stepIndex} is out of range for "${id}" (${steps.length} steps)`);
|
|
17617
|
+
}
|
|
17618
|
+
const options = opts.options ?? {};
|
|
17619
|
+
const input = definition.inputSchema.parse(rawInput);
|
|
17620
|
+
const apiKey = resolveApiKey(options);
|
|
17621
|
+
const apiUrl = resolveApiUrl(options);
|
|
17622
|
+
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
17623
|
+
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
|
|
17624
|
+
const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client: client2, artifacts, signal: options.signal };
|
|
17625
|
+
const state = opts.state ?? (definition.createState ? definition.createState(input) : {});
|
|
17626
|
+
const step = steps[opts.stepIndex];
|
|
17627
|
+
const outcome = await step.run({ input, state, ctx });
|
|
17628
|
+
const writtenArtifacts = await Promise.all(
|
|
17629
|
+
artifacts.artifacts.map(async (artifact) => ({
|
|
17630
|
+
...artifact,
|
|
17631
|
+
content: await (0, import_promises7.readFile)(artifact.path, "utf8").catch(() => "")
|
|
17632
|
+
}))
|
|
17633
|
+
);
|
|
17634
|
+
return {
|
|
17635
|
+
runId: artifacts.runId,
|
|
17636
|
+
workflowId: definition.id,
|
|
17637
|
+
title: definition.title,
|
|
17638
|
+
stepIndex: opts.stepIndex,
|
|
17639
|
+
stepId: step.id,
|
|
17640
|
+
stepTitle: step.title,
|
|
17641
|
+
totalSteps: steps.length,
|
|
17642
|
+
isLast: opts.stepIndex === steps.length - 1,
|
|
17643
|
+
output: outcome.output,
|
|
17644
|
+
state: outcome.state,
|
|
17645
|
+
warnings: outcome.warnings ?? [],
|
|
17646
|
+
summary: outcome.summary ?? null,
|
|
17647
|
+
artifacts: writtenArtifacts
|
|
17648
|
+
};
|
|
17649
|
+
}
|
|
17650
|
+
var import_promises7, import_zod23, DEFINITIONS;
|
|
17260
17651
|
var init_registry2 = __esm({
|
|
17261
17652
|
"src/workflows/registry.ts"() {
|
|
17262
17653
|
"use strict";
|
|
17654
|
+
import_promises7 = require("fs/promises");
|
|
17263
17655
|
import_zod23 = require("zod");
|
|
17264
17656
|
init_artifact_writer();
|
|
17265
17657
|
init_http_client2();
|
|
@@ -17328,7 +17720,7 @@ async function readManifestFromSummary(summary) {
|
|
|
17328
17720
|
if (!summary.reportPath) return null;
|
|
17329
17721
|
const manifestPath = summary.reportPath.replace(/report\.html$/, "manifest.json");
|
|
17330
17722
|
try {
|
|
17331
|
-
return JSON.parse(await (0,
|
|
17723
|
+
return JSON.parse(await (0, import_promises8.readFile)(manifestPath, "utf8"));
|
|
17332
17724
|
} catch {
|
|
17333
17725
|
return null;
|
|
17334
17726
|
}
|
|
@@ -17409,6 +17801,88 @@ async function executeWorkflowRun(input) {
|
|
|
17409
17801
|
throw err;
|
|
17410
17802
|
}
|
|
17411
17803
|
}
|
|
17804
|
+
async function runWorkflowStepAndPersist(input) {
|
|
17805
|
+
const runId = input.run.id;
|
|
17806
|
+
const workflowId = input.run.workflow_id;
|
|
17807
|
+
await markWorkflowRunRunning(runId);
|
|
17808
|
+
let result;
|
|
17809
|
+
try {
|
|
17810
|
+
result = await runWorkflowStep(workflowId, input.workflowInput, {
|
|
17811
|
+
stepIndex: input.stepIndex,
|
|
17812
|
+
state: input.priorState ?? void 0,
|
|
17813
|
+
options: {
|
|
17814
|
+
apiKey: input.user.api_key,
|
|
17815
|
+
apiUrl: input.apiUrl,
|
|
17816
|
+
outputDir: hostedWorkflowOutputDir(),
|
|
17817
|
+
runId,
|
|
17818
|
+
headers: input.concurrencyLockId ? { "x-mcp-scraper-concurrency-lock": input.concurrencyLockId } : void 0
|
|
17819
|
+
}
|
|
17820
|
+
});
|
|
17821
|
+
} catch (err) {
|
|
17822
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
17823
|
+
await failWorkflowRunRecord(runId, message, { error: message });
|
|
17824
|
+
const failedRun = await getWorkflowRun(runId, input.user.id);
|
|
17825
|
+
if (failedRun) await deliverWorkflowWebhook({ userId: input.user.id, run: failedRun, scheduleId: failedRun.schedule_id, webhookUrl: input.webhookUrl ?? null, summary: null, artifacts: [] });
|
|
17826
|
+
throw err;
|
|
17827
|
+
}
|
|
17828
|
+
if (result.artifacts.length) {
|
|
17829
|
+
await appendWorkflowArtifacts(runId, input.user.id, result.artifacts.map((a) => ({
|
|
17830
|
+
kind: a.kind,
|
|
17831
|
+
label: a.label,
|
|
17832
|
+
path: a.path,
|
|
17833
|
+
bytes: a.bytes,
|
|
17834
|
+
rows: a.rows,
|
|
17835
|
+
content: a.content
|
|
17836
|
+
})));
|
|
17837
|
+
}
|
|
17838
|
+
const logEntry = {
|
|
17839
|
+
stepIndex: result.stepIndex,
|
|
17840
|
+
stepId: result.stepId,
|
|
17841
|
+
stepTitle: result.stepTitle,
|
|
17842
|
+
output: result.output,
|
|
17843
|
+
warnings: result.warnings,
|
|
17844
|
+
at: (/* @__PURE__ */ new Date()).toISOString()
|
|
17845
|
+
};
|
|
17846
|
+
const status = result.isLast ? result.summary?.status ?? "succeeded" : "running";
|
|
17847
|
+
await advanceWorkflowRunStep(runId, {
|
|
17848
|
+
stepIndex: result.stepIndex + 1,
|
|
17849
|
+
totalSteps: result.totalSteps,
|
|
17850
|
+
state: result.state,
|
|
17851
|
+
status,
|
|
17852
|
+
logEntry,
|
|
17853
|
+
appendLog: input.priorLog
|
|
17854
|
+
});
|
|
17855
|
+
if (result.isLast && result.summary) {
|
|
17856
|
+
const manifest = await readManifestFromSummary(result.summary);
|
|
17857
|
+
await completeWorkflowRunRecord(runId, result.summary.status, manifest ?? { summary: result.summary });
|
|
17858
|
+
}
|
|
17859
|
+
const run = await getWorkflowRun(runId, input.user.id);
|
|
17860
|
+
const artifacts = await listWorkflowArtifacts(runId, input.user.id);
|
|
17861
|
+
if (result.isLast && run) {
|
|
17862
|
+
await deliverWorkflowWebhook({ userId: input.user.id, run, scheduleId: run.schedule_id, webhookUrl: input.webhookUrl ?? null, summary: result.summary, artifacts });
|
|
17863
|
+
}
|
|
17864
|
+
const definition = workflowDefinition(workflowId);
|
|
17865
|
+
const nextIndex = result.stepIndex + 1;
|
|
17866
|
+
const nextStep = result.isLast ? null : {
|
|
17867
|
+
index: nextIndex,
|
|
17868
|
+
id: definition.steps?.[nextIndex]?.id ?? null,
|
|
17869
|
+
title: definition.steps?.[nextIndex]?.title ?? null
|
|
17870
|
+
};
|
|
17871
|
+
return {
|
|
17872
|
+
run: run ? exposeRun(input.c, run, artifacts) : null,
|
|
17873
|
+
summary: result.isLast ? result.summary : null,
|
|
17874
|
+
step: {
|
|
17875
|
+
index: result.stepIndex,
|
|
17876
|
+
id: result.stepId,
|
|
17877
|
+
title: result.stepTitle,
|
|
17878
|
+
totalSteps: result.totalSteps,
|
|
17879
|
+
output: result.output,
|
|
17880
|
+
warnings: result.warnings
|
|
17881
|
+
},
|
|
17882
|
+
nextStep,
|
|
17883
|
+
done: result.isLast
|
|
17884
|
+
};
|
|
17885
|
+
}
|
|
17412
17886
|
async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
17413
17887
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
17414
17888
|
const due = await listDueWorkflowSchedules(now, limit);
|
|
@@ -17461,12 +17935,12 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
17461
17935
|
}
|
|
17462
17936
|
return { dispatched: results.length, results };
|
|
17463
17937
|
}
|
|
17464
|
-
var import_node_crypto3,
|
|
17938
|
+
var import_node_crypto3, import_promises8, import_hono7, import_zod24, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
|
|
17465
17939
|
var init_workflow_routes = __esm({
|
|
17466
17940
|
"src/api/workflow-routes.ts"() {
|
|
17467
17941
|
"use strict";
|
|
17468
17942
|
import_node_crypto3 = require("crypto");
|
|
17469
|
-
|
|
17943
|
+
import_promises8 = require("fs/promises");
|
|
17470
17944
|
import_hono7 = require("hono");
|
|
17471
17945
|
import_zod24 = require("zod");
|
|
17472
17946
|
init_artifact_writer();
|
|
@@ -17525,7 +17999,29 @@ var init_workflow_routes = __esm({
|
|
|
17525
17999
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
17526
18000
|
let run = null;
|
|
17527
18001
|
try {
|
|
17528
|
-
|
|
18002
|
+
const stepwise = workflowSupportsSteps(parsed.data.workflowId);
|
|
18003
|
+
run = await createWorkflowRun({
|
|
18004
|
+
userId: user.id,
|
|
18005
|
+
workflowId: parsed.data.workflowId,
|
|
18006
|
+
workflowInput: input,
|
|
18007
|
+
webhookUrl,
|
|
18008
|
+
totalSteps: stepwise ? workflowStepCount(parsed.data.workflowId) : 1
|
|
18009
|
+
});
|
|
18010
|
+
if (stepwise) {
|
|
18011
|
+
const response = await runWorkflowStepAndPersist({
|
|
18012
|
+
c,
|
|
18013
|
+
user,
|
|
18014
|
+
run,
|
|
18015
|
+
workflowInput: input,
|
|
18016
|
+
stepIndex: 0,
|
|
18017
|
+
priorState: null,
|
|
18018
|
+
priorLog: [],
|
|
18019
|
+
apiUrl: originFromUrl(c.req.url),
|
|
18020
|
+
webhookUrl,
|
|
18021
|
+
concurrencyLockId: gate.lockId
|
|
18022
|
+
});
|
|
18023
|
+
return c.json(response);
|
|
18024
|
+
}
|
|
17529
18025
|
const executed = await executeWorkflowRun({
|
|
17530
18026
|
runId: run.id,
|
|
17531
18027
|
user,
|
|
@@ -17535,7 +18031,7 @@ var init_workflow_routes = __esm({
|
|
|
17535
18031
|
webhookUrl,
|
|
17536
18032
|
concurrencyLockId: gate.lockId
|
|
17537
18033
|
});
|
|
17538
|
-
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
18034
|
+
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary, done: true });
|
|
17539
18035
|
} catch (err) {
|
|
17540
18036
|
const failed = run ? await getWorkflowRun(run.id, user.id) : null;
|
|
17541
18037
|
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
@@ -17543,6 +18039,50 @@ var init_workflow_routes = __esm({
|
|
|
17543
18039
|
await releaseConcurrencyGate(gate.lockId);
|
|
17544
18040
|
}
|
|
17545
18041
|
});
|
|
18042
|
+
TERMINAL_RUN_STATUSES = ["succeeded", "partial", "failed", "cancelled"];
|
|
18043
|
+
workflowApp.post("/runs/:id/step", createApiKeyAuth(), async (c) => {
|
|
18044
|
+
const user = c.get("user");
|
|
18045
|
+
const run = await getWorkflowRun(c.req.param("id"), user.id);
|
|
18046
|
+
if (!run) return c.json({ error: "Run not found" }, 404);
|
|
18047
|
+
if (!workflowSupportsSteps(run.workflow_id)) {
|
|
18048
|
+
return c.json({ error: `Workflow "${run.workflow_id}" does not support stepwise execution` }, 400);
|
|
18049
|
+
}
|
|
18050
|
+
const artifactsNow = await listWorkflowArtifacts(run.id, user.id);
|
|
18051
|
+
if (TERMINAL_RUN_STATUSES.includes(run.status)) {
|
|
18052
|
+
return c.json({ run: exposeRun(c, run, artifactsNow), done: true, error: `Run is already ${run.status}` }, 409);
|
|
18053
|
+
}
|
|
18054
|
+
const stepState = await getWorkflowRunStepState(run.id, user.id);
|
|
18055
|
+
const stepIndex = stepState?.stepIndex ?? run.step_index;
|
|
18056
|
+
const totalSteps = workflowStepCount(run.workflow_id);
|
|
18057
|
+
if (stepIndex >= totalSteps) {
|
|
18058
|
+
return c.json({ run: exposeRun(c, run, artifactsNow), done: true, error: "No remaining steps" }, 409);
|
|
18059
|
+
}
|
|
18060
|
+
const gate = await acquireConcurrencyGate(user, "workflow_run", {
|
|
18061
|
+
ttlSeconds: 60 * 60,
|
|
18062
|
+
metadata: { workflowId: run.workflow_id, step: stepIndex }
|
|
18063
|
+
});
|
|
18064
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
18065
|
+
try {
|
|
18066
|
+
const response = await runWorkflowStepAndPersist({
|
|
18067
|
+
c,
|
|
18068
|
+
user,
|
|
18069
|
+
run,
|
|
18070
|
+
workflowInput: run.input,
|
|
18071
|
+
stepIndex,
|
|
18072
|
+
priorState: stepState?.state ?? null,
|
|
18073
|
+
priorLog: stepState?.stepLog ?? run.step_log,
|
|
18074
|
+
apiUrl: originFromUrl(c.req.url),
|
|
18075
|
+
webhookUrl: run.webhook_url,
|
|
18076
|
+
concurrencyLockId: gate.lockId
|
|
18077
|
+
});
|
|
18078
|
+
return c.json(response);
|
|
18079
|
+
} catch (err) {
|
|
18080
|
+
const failed = await getWorkflowRun(run.id, user.id);
|
|
18081
|
+
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
18082
|
+
} finally {
|
|
18083
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
18084
|
+
}
|
|
18085
|
+
});
|
|
17546
18086
|
workflowApp.get("/runs", createApiKeyAuth(), async (c) => {
|
|
17547
18087
|
const user = c.get("user");
|
|
17548
18088
|
const runs = await listWorkflowRuns(user.id);
|
|
@@ -17561,14 +18101,17 @@ var init_workflow_routes = __esm({
|
|
|
17561
18101
|
if (!run) return c.json({ error: "Run not found" }, 404);
|
|
17562
18102
|
const artifact = await getWorkflowArtifact(c.req.param("artifactId"), user.id);
|
|
17563
18103
|
if (!artifact || artifact.run_id !== run.id) return c.json({ error: "Artifact not found" }, 404);
|
|
18104
|
+
const filename = artifact.path.split("/").pop() || artifact.id;
|
|
18105
|
+
const headers = {
|
|
18106
|
+
"Content-Type": artifact.content_type,
|
|
18107
|
+
"Content-Disposition": `attachment; filename="${filename}"`
|
|
18108
|
+
};
|
|
18109
|
+
if (artifact.content != null) {
|
|
18110
|
+
return new Response(artifact.content, { headers });
|
|
18111
|
+
}
|
|
17564
18112
|
try {
|
|
17565
|
-
const content = await (0,
|
|
17566
|
-
return new Response(content, {
|
|
17567
|
-
headers: {
|
|
17568
|
-
"Content-Type": artifact.content_type,
|
|
17569
|
-
"Content-Disposition": `attachment; filename="${artifact.path.split("/").pop() || artifact.id}"`
|
|
17570
|
-
}
|
|
17571
|
-
});
|
|
18113
|
+
const content = await (0, import_promises8.readFile)(artifact.path);
|
|
18114
|
+
return new Response(content, { headers });
|
|
17572
18115
|
} catch {
|
|
17573
18116
|
return c.json({ error: "Artifact file is no longer available" }, 410);
|
|
17574
18117
|
}
|
|
@@ -17856,7 +18399,7 @@ async function capturePageSnapshot(target, options = {}) {
|
|
|
17856
18399
|
const textContent = truncateContent(text, maxContentChars);
|
|
17857
18400
|
const schemaTypes = getSchemaTypes(extraction.schema, extraction.kpo.schemaTypes);
|
|
17858
18401
|
const metrics = countLinks(extraction.bodyHtml, extraction.url);
|
|
17859
|
-
const
|
|
18402
|
+
const wordCount2 = countWords(extraction.bodyMarkdown);
|
|
17860
18403
|
const contentSha256 = sha256(extraction.bodyMarkdown);
|
|
17861
18404
|
const canonicalUrl = extraction.kpo.canonicalUrl ? new URL(extraction.kpo.canonicalUrl, extraction.url).href : null;
|
|
17862
18405
|
const contentIncluded = Boolean(options.includeContent);
|
|
@@ -17910,10 +18453,10 @@ async function capturePageSnapshot(target, options = {}) {
|
|
|
17910
18453
|
},
|
|
17911
18454
|
error: null,
|
|
17912
18455
|
schemaTypes,
|
|
17913
|
-
wordCount,
|
|
18456
|
+
wordCount: wordCount2,
|
|
17914
18457
|
linkCounts: {
|
|
17915
18458
|
...metrics,
|
|
17916
|
-
wordCount
|
|
18459
|
+
wordCount: wordCount2
|
|
17917
18460
|
},
|
|
17918
18461
|
hashes: {
|
|
17919
18462
|
htmlSha256: sha256(extraction.bodyHtml),
|
|
@@ -19489,7 +20032,7 @@ async function harvest(rawOptions) {
|
|
|
19489
20032
|
const signal = getAbortSignal(rawOptions);
|
|
19490
20033
|
const onAttemptEvent = getAttemptLogSink(rawOptions);
|
|
19491
20034
|
const requestedProxyMode = raw.proxyMode;
|
|
19492
|
-
const proxyMode = requestedProxyMode === "
|
|
20035
|
+
const proxyMode = requestedProxyMode === "location" || requestedProxyMode === "none" || requestedProxyMode === "configured" ? requestedProxyMode : DEFAULT_PROXY_MODE;
|
|
19493
20036
|
const kernelApiKey = typeof raw.kernelApiKey === "string" ? raw.kernelApiKey.trim() : browserServiceApiKey();
|
|
19494
20037
|
const configuredKernelProxyId = typeof raw.kernelProxyId === "string" ? raw.kernelProxyId.trim() : browserServiceProxyId();
|
|
19495
20038
|
const proxyOpts = {
|
|
@@ -19863,6 +20406,7 @@ var init_schemas4 = __esm({
|
|
|
19863
20406
|
"src/serp-intelligence/schemas.ts"() {
|
|
19864
20407
|
"use strict";
|
|
19865
20408
|
import_zod25 = require("zod");
|
|
20409
|
+
init_schemas3();
|
|
19866
20410
|
SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
19867
20411
|
SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
19868
20412
|
SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -19903,7 +20447,7 @@ var init_schemas4 = __esm({
|
|
|
19903
20447
|
gl: import_zod25.z.string().trim().length(2).default("us"),
|
|
19904
20448
|
hl: import_zod25.z.string().trim().length(2).default("en"),
|
|
19905
20449
|
device: import_zod25.z.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
19906
|
-
proxyMode: import_zod25.z.enum(SerpIntelligenceProxyModeValues).default(
|
|
20450
|
+
proxyMode: import_zod25.z.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
19907
20451
|
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional(),
|
|
19908
20452
|
pages: import_zod25.z.number().int().min(1).max(2).default(1),
|
|
19909
20453
|
debug: import_zod25.z.boolean().default(false),
|
|
@@ -20433,16 +20977,17 @@ var PACKAGE_VERSION;
|
|
|
20433
20977
|
var init_version = __esm({
|
|
20434
20978
|
"src/version.ts"() {
|
|
20435
20979
|
"use strict";
|
|
20436
|
-
PACKAGE_VERSION = "0.2.
|
|
20980
|
+
PACKAGE_VERSION = "0.2.20";
|
|
20437
20981
|
}
|
|
20438
20982
|
});
|
|
20439
20983
|
|
|
20440
20984
|
// src/mcp/mcp-tool-schemas.ts
|
|
20441
|
-
var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
20985
|
+
var import_zod26, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, FacebookVideoTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, FacebookVideoTranscribeOutputSchema, TranscriptChunkOutput, YoutubeTranscribeOutputSchema, FacebookAdTranscribeOutputSchema, CaptureSerpSnapshotOutputSchema, CaptureSerpPageSnapshotsOutputSchema, CreditsInfoInputSchema, WorkflowIdSchema2, WorkflowListInputSchema, WorkflowSuggestInputSchema, WorkflowRunInputSchema, WorkflowStepInputSchema, WorkflowStatusInputSchema, WorkflowArtifactReadInputSchema, WorkflowRecipeOutput, WorkflowDefinitionOutput, WorkflowArtifactOutput, WorkflowListOutputSchema, WorkflowSuggestOutputSchema, WorkflowRunOutputSchema, WorkflowStepOutputSchema, WorkflowStatusOutputSchema, WorkflowArtifactReadOutputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
20442
20986
|
var init_mcp_tool_schemas = __esm({
|
|
20443
20987
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
20444
20988
|
"use strict";
|
|
20445
20989
|
import_zod26 = require("zod");
|
|
20990
|
+
init_schemas3();
|
|
20446
20991
|
HarvestPaaInputSchema = {
|
|
20447
20992
|
query: import_zod26.z.string().min(1).describe('Core search topic only. If the user says "best hvac company in Denver CO", use query="best hvac company" and location="Denver, CO". Do not include the location in query when it can be separated.'),
|
|
20448
20993
|
location: import_zod26.z.string().optional().describe('City, region, or country for geo-targeted results, inferred from the user request when present, e.g. "Denver, CO", "Tokyo, Japan", "London, UK".'),
|
|
@@ -20450,8 +20995,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20450
20995
|
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
|
|
20451
20996
|
hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
|
|
20452
20997
|
device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
20453
|
-
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(
|
|
20454
|
-
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or
|
|
20998
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
20999
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
|
|
20455
21000
|
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
|
|
20456
21001
|
};
|
|
20457
21002
|
ExtractUrlInputSchema = {
|
|
@@ -20475,22 +21020,23 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20475
21020
|
mode: import_zod26.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
|
|
20476
21021
|
query: import_zod26.z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
|
|
20477
21022
|
channelHandle: import_zod26.z.string().optional().describe("YouTube channel handle, channel ID, or URL. Examples: @mkbhd, UC..., https://youtube.com/@mkbhd."),
|
|
20478
|
-
maxVideos: import_zod26.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50.
|
|
21023
|
+
maxVideos: import_zod26.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50, maximum 500. Use 10-25 for quick topic discovery, 50 for normal channel/search harvests, and larger values only when the user asks for full channel/history because large responses consume more context.")
|
|
20479
21024
|
};
|
|
20480
21025
|
YoutubeTranscribeInputSchema = {
|
|
20481
|
-
videoId: import_zod26.z.string().min(1).describe("YouTube video ID, e.g. dQw4w9WgXcQ")
|
|
21026
|
+
videoId: import_zod26.z.string().min(1).optional().describe("YouTube video ID, e.g. dQw4w9WgXcQ. Use only an ID returned by youtube_harvest or visible in a YouTube URL; do not invent one."),
|
|
21027
|
+
url: import_zod26.z.string().url().optional().describe("Full YouTube URL, e.g. https://www.youtube.com/watch?v=dQw4w9WgXcQ or https://youtu.be/dQw4w9WgXcQ. Use this when the user pasted a URL instead of an ID. Provide videoId or url.")
|
|
20482
21028
|
};
|
|
20483
21029
|
FacebookPageIntelInputSchema = {
|
|
20484
|
-
pageId: import_zod26.z.string().optional(),
|
|
20485
|
-
libraryId: import_zod26.z.string().optional(),
|
|
21030
|
+
pageId: import_zod26.z.string().optional().describe("Facebook advertiser/page ID. Use only a pageId returned by facebook_ad_search/facebook_page_intel or copied from Facebook Ad Library; do not construct one yourself."),
|
|
21031
|
+
libraryId: import_zod26.z.string().optional().describe("Facebook Ad Library archive ID for a known ad or advertiser sample. Use a libraryId returned by facebook_ad_search, or a libraryId/adArchiveId visible in Ad Library."),
|
|
20486
21032
|
query: import_zod26.z.string().optional().describe("Advertiser or brand name when pageId/libraryId is not known. One of pageId, libraryId, or query is required."),
|
|
20487
|
-
maxAds: import_zod26.z.number().int().min(1).max(200).default(50),
|
|
20488
|
-
country: import_zod26.z.string().length(2).default("US")
|
|
21033
|
+
maxAds: import_zod26.z.number().int().min(1).max(200).default(50).describe("Maximum ads to inspect. Default 50, maximum 200. Prefer 25-50 for focused advertiser scans; use 100-200 only when the user asks for a broad ad archive sweep."),
|
|
21034
|
+
country: import_zod26.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU. Infer from the user request when they name a country.")
|
|
20489
21035
|
};
|
|
20490
21036
|
FacebookAdSearchInputSchema = {
|
|
20491
21037
|
query: import_zod26.z.string().min(1).describe("Advertiser, brand, competitor, niche, or keyword to search in Facebook Ad Library."),
|
|
20492
|
-
country: import_zod26.z.string().length(2).default("US"),
|
|
20493
|
-
maxResults: import_zod26.z.number().int().min(1).max(20).default(10)
|
|
21038
|
+
country: import_zod26.z.string().length(2).default("US").describe("Two-letter Ad Library country code. Default US. Examples: US, CA, GB, AU."),
|
|
21039
|
+
maxResults: import_zod26.z.number().int().min(1).max(20).default(10).describe("Maximum advertisers to return. Default 10, maximum 20. Prefer tighter search terms over maxing this out.")
|
|
20494
21040
|
};
|
|
20495
21041
|
FacebookAdTranscribeInputSchema = {
|
|
20496
21042
|
videoUrl: import_zod26.z.string().url().describe("Direct Facebook CDN video URL from a facebook_page_intel ad result. Do not pass a public Facebook reel/post/share URL here; use facebook_video_transcribe for organic Facebook URLs.")
|
|
@@ -20504,8 +21050,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20504
21050
|
location: import_zod26.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
|
|
20505
21051
|
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
20506
21052
|
hl: import_zod26.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
20507
|
-
includeReviews: import_zod26.z.boolean().default(false).describe("Whether to fetch individual review cards"),
|
|
20508
|
-
maxReviews: import_zod26.z.number().int().min(1).max(500).default(50).describe("Max review cards to return
|
|
21053
|
+
includeReviews: import_zod26.z.boolean().default(false).describe("Whether to fetch individual review cards. Use true when the user asks for reviews, customer pain, complaints, praise, themes, or review evidence."),
|
|
21054
|
+
maxReviews: import_zod26.z.number().int().min(1).max(500).default(50).describe("Max review cards to return when includeReviews is true. Default 50, maximum 500. Use 50 for normal review analysis and larger values only for deep review mining.")
|
|
20509
21055
|
};
|
|
20510
21056
|
MapsSearchInputSchema = {
|
|
20511
21057
|
query: import_zod26.z.string().min(1).describe('Business category, niche, keyword, or search term. If the user says "roofers in Denver CO", use query="roofers" and location="Denver, CO". Do not put the location here when it can be separated.'),
|
|
@@ -20513,7 +21059,7 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20513
21059
|
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
20514
21060
|
hl: import_zod26.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
20515
21061
|
maxResults: import_zod26.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
|
|
20516
|
-
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(
|
|
21062
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode. Maps defaults to location so local market searches get city/state residential proxy targeting and rotation. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
|
|
20517
21063
|
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
|
|
20518
21064
|
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
|
|
20519
21065
|
};
|
|
@@ -20528,8 +21074,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20528
21074
|
includeZipGroups: import_zod26.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
|
|
20529
21075
|
usZipsCsvPath: import_zod26.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
20530
21076
|
saveCsv: import_zod26.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
|
|
20531
|
-
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(
|
|
20532
|
-
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its
|
|
21077
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting mode for every city Maps search. Maps workflows default to location so each city can use city/state or ZIP-group residential proxy targeting. Use configured to force the service proxy without city/ZIP targeting, and none only for local direct-network debugging."),
|
|
21078
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its ZIP group or city/state location."),
|
|
20533
21079
|
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
|
|
20534
21080
|
};
|
|
20535
21081
|
RankTrackerModeSchema = import_zod26.z.enum(["maps", "organic", "ai_overview", "paa"]);
|
|
@@ -20843,8 +21389,11 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20843
21389
|
advertiserCount: import_zod26.z.number().int().min(0),
|
|
20844
21390
|
advertisers: import_zod26.z.array(import_zod26.z.object({
|
|
20845
21391
|
name: NullableString,
|
|
21392
|
+
pageId: NullableString,
|
|
21393
|
+
pageUrl: NullableString,
|
|
20846
21394
|
adCount: import_zod26.z.number().int().nullable(),
|
|
20847
|
-
libraryId: NullableString
|
|
21395
|
+
libraryId: NullableString,
|
|
21396
|
+
sampleLibraryId: NullableString
|
|
20848
21397
|
}))
|
|
20849
21398
|
};
|
|
20850
21399
|
FacebookPageIntelOutputSchema = {
|
|
@@ -20857,10 +21406,15 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20857
21406
|
libraryId: NullableString,
|
|
20858
21407
|
status: NullableString,
|
|
20859
21408
|
creativeType: NullableString,
|
|
21409
|
+
primaryText: NullableString,
|
|
20860
21410
|
headline: NullableString,
|
|
20861
21411
|
cta: NullableString,
|
|
20862
21412
|
startDate: NullableString,
|
|
21413
|
+
landingUrl: NullableString,
|
|
21414
|
+
domain: NullableString,
|
|
20863
21415
|
videoUrl: NullableString,
|
|
21416
|
+
imageUrl: NullableString,
|
|
21417
|
+
videoPoster: NullableString,
|
|
20864
21418
|
variations: import_zod26.z.number().int().nullable()
|
|
20865
21419
|
}))
|
|
20866
21420
|
};
|
|
@@ -20882,6 +21436,58 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20882
21436
|
text: import_zod26.z.string()
|
|
20883
21437
|
}))
|
|
20884
21438
|
};
|
|
21439
|
+
TranscriptChunkOutput = import_zod26.z.object({
|
|
21440
|
+
startSec: import_zod26.z.number(),
|
|
21441
|
+
endSec: import_zod26.z.number(),
|
|
21442
|
+
text: import_zod26.z.string()
|
|
21443
|
+
});
|
|
21444
|
+
YoutubeTranscribeOutputSchema = {
|
|
21445
|
+
videoId: NullableString,
|
|
21446
|
+
url: NullableString,
|
|
21447
|
+
wordCount: import_zod26.z.number().int().min(0),
|
|
21448
|
+
chunkCount: import_zod26.z.number().int().min(0),
|
|
21449
|
+
durationMs: import_zod26.z.number().nullable(),
|
|
21450
|
+
transcriptText: import_zod26.z.string(),
|
|
21451
|
+
chunks: import_zod26.z.array(TranscriptChunkOutput),
|
|
21452
|
+
resolvedInputs: import_zod26.z.object({
|
|
21453
|
+
videoId: NullableString,
|
|
21454
|
+
url: NullableString
|
|
21455
|
+
})
|
|
21456
|
+
};
|
|
21457
|
+
FacebookAdTranscribeOutputSchema = {
|
|
21458
|
+
videoUrl: import_zod26.z.string().url(),
|
|
21459
|
+
wordCount: import_zod26.z.number().int().min(0),
|
|
21460
|
+
chunkCount: import_zod26.z.number().int().min(0),
|
|
21461
|
+
durationMs: import_zod26.z.number().nullable(),
|
|
21462
|
+
transcriptText: import_zod26.z.string(),
|
|
21463
|
+
chunks: import_zod26.z.array(TranscriptChunkOutput),
|
|
21464
|
+
resolvedInputs: import_zod26.z.object({
|
|
21465
|
+
videoUrl: import_zod26.z.string().url()
|
|
21466
|
+
})
|
|
21467
|
+
};
|
|
21468
|
+
CaptureSerpSnapshotOutputSchema = {
|
|
21469
|
+
schemaVersion: import_zod26.z.literal("serp-intelligence.capture.v1"),
|
|
21470
|
+
status: import_zod26.z.string(),
|
|
21471
|
+
query: NullableString,
|
|
21472
|
+
location: NullableString,
|
|
21473
|
+
capturedAt: NullableString,
|
|
21474
|
+
resultCount: import_zod26.z.number().int().min(0).nullable(),
|
|
21475
|
+
snapshotId: NullableString,
|
|
21476
|
+
resolvedInputs: import_zod26.z.record(import_zod26.z.unknown()),
|
|
21477
|
+
artifacts: import_zod26.z.array(import_zod26.z.record(import_zod26.z.unknown())),
|
|
21478
|
+
diagnostics: import_zod26.z.record(import_zod26.z.unknown()).nullable(),
|
|
21479
|
+
providerPayload: import_zod26.z.record(import_zod26.z.unknown())
|
|
21480
|
+
};
|
|
21481
|
+
CaptureSerpPageSnapshotsOutputSchema = {
|
|
21482
|
+
schemaVersion: import_zod26.z.literal("serp-intelligence.page-snapshots.v1"),
|
|
21483
|
+
status: import_zod26.z.string(),
|
|
21484
|
+
count: import_zod26.z.number().int().min(0),
|
|
21485
|
+
failedCount: import_zod26.z.number().int().min(0),
|
|
21486
|
+
captures: import_zod26.z.array(import_zod26.z.record(import_zod26.z.unknown())),
|
|
21487
|
+
resolvedInputs: import_zod26.z.record(import_zod26.z.unknown()),
|
|
21488
|
+
diagnostics: import_zod26.z.record(import_zod26.z.unknown()).nullable(),
|
|
21489
|
+
providerPayload: import_zod26.z.record(import_zod26.z.unknown())
|
|
21490
|
+
};
|
|
20885
21491
|
CreditsInfoInputSchema = {
|
|
20886
21492
|
item: import_zod26.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
|
|
20887
21493
|
includeLedger: import_zod26.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
@@ -20909,16 +21515,19 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20909
21515
|
maxSuggestions: import_zod26.z.number().int().min(1).max(8).default(3).describe("Number of matching workflow recipes to return.")
|
|
20910
21516
|
};
|
|
20911
21517
|
WorkflowRunInputSchema = {
|
|
20912
|
-
workflowId: WorkflowIdSchema2.describe("Workflow to run. Use workflow_list or workflow_suggest first when unsure."),
|
|
21518
|
+
workflowId: WorkflowIdSchema2.describe("Workflow to run: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. Use only these values; call workflow_list or workflow_suggest first when unsure."),
|
|
20913
21519
|
input: import_zod26.z.record(import_zod26.z.unknown()).default({}).describe("Workflow-specific input object. Examples: agent-packet uses {keyword, domain?, location?, maxQuestions?}; local-competitive-audit uses {query, state, minPopulation?, maxCities?, maxResultsPerCity?, hydrateTop?, maxReviews?}; serp-comparison uses {keyword, domain?, url?, location?, extractTop?}."),
|
|
20914
21520
|
webhookUrl: import_zod26.z.string().url().optional().describe("Optional HTTPS webhook to receive the completed hosted workflow run event.")
|
|
20915
21521
|
};
|
|
21522
|
+
WorkflowStepInputSchema = {
|
|
21523
|
+
runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself. Advances the run by exactly one step (one logical leg, e.g. one live harvest).")
|
|
21524
|
+
};
|
|
20916
21525
|
WorkflowStatusInputSchema = {
|
|
20917
|
-
runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run.")
|
|
21526
|
+
runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself.")
|
|
20918
21527
|
};
|
|
20919
21528
|
WorkflowArtifactReadInputSchema = {
|
|
20920
|
-
runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run or workflow_status."),
|
|
20921
|
-
artifactId: import_zod26.z.string().min(1).describe("Artifact id from the run artifact list."),
|
|
21529
|
+
runId: import_zod26.z.string().min(1).describe("Workflow run id returned by workflow_run, workflow_step, or workflow_status. Use only a returned runId; do not construct one yourself."),
|
|
21530
|
+
artifactId: import_zod26.z.string().min(1).describe("Artifact id from the run artifact list returned by workflow_run, workflow_step, or workflow_status. Use only a returned artifactId; do not construct one yourself."),
|
|
20922
21531
|
maxBytes: import_zod26.z.number().int().min(1e3).max(1e6).default(2e5).describe("Maximum bytes of artifact text to return inline. Use lower values for large CSV/JSON artifacts; call again with the downloadUrl if needed outside MCP.")
|
|
20923
21532
|
};
|
|
20924
21533
|
WorkflowRecipeOutput = import_zod26.z.object({
|
|
@@ -20951,6 +21560,18 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20951
21560
|
input: import_zod26.z.record(import_zod26.z.unknown()),
|
|
20952
21561
|
run: import_zod26.z.record(import_zod26.z.unknown()).optional(),
|
|
20953
21562
|
summary: import_zod26.z.record(import_zod26.z.unknown()).optional(),
|
|
21563
|
+
step: import_zod26.z.record(import_zod26.z.unknown()).optional(),
|
|
21564
|
+
nextStep: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
|
|
21565
|
+
done: import_zod26.z.boolean().optional(),
|
|
21566
|
+
artifacts: import_zod26.z.array(WorkflowArtifactOutput)
|
|
21567
|
+
};
|
|
21568
|
+
WorkflowStepOutputSchema = {
|
|
21569
|
+
runId: import_zod26.z.string(),
|
|
21570
|
+
run: import_zod26.z.record(import_zod26.z.unknown()).optional(),
|
|
21571
|
+
summary: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
|
|
21572
|
+
step: import_zod26.z.record(import_zod26.z.unknown()).optional(),
|
|
21573
|
+
nextStep: import_zod26.z.record(import_zod26.z.unknown()).nullable().optional(),
|
|
21574
|
+
done: import_zod26.z.boolean(),
|
|
20954
21575
|
artifacts: import_zod26.z.array(WorkflowArtifactOutput)
|
|
20955
21576
|
};
|
|
20956
21577
|
WorkflowStatusOutputSchema = {
|
|
@@ -20971,8 +21592,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20971
21592
|
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
20972
21593
|
hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from user request."),
|
|
20973
21594
|
device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
20974
|
-
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(
|
|
20975
|
-
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or
|
|
21595
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy targeting mode. Default configured uses the service proxy without city/ZIP targeting for the highest general SERP success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
21596
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when the user gives a specific ZIP or city-center targeting needs to be forced."),
|
|
20976
21597
|
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
|
|
20977
21598
|
pages: import_zod26.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
|
|
20978
21599
|
};
|
|
@@ -20982,8 +21603,8 @@ var init_mcp_tool_schemas = __esm({
|
|
|
20982
21603
|
gl: import_zod26.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
|
|
20983
21604
|
hl: import_zod26.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
20984
21605
|
device: import_zod26.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
|
|
20985
|
-
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(
|
|
20986
|
-
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting when a precise city-center or ZIP proxy is needed.
|
|
21606
|
+
proxyMode: import_zod26.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE).describe("Proxy behavior for capture. Default configured uses the service proxy without city/ZIP targeting for the highest general capture success rate. Use location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging."),
|
|
21607
|
+
proxyZip: import_zod26.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only with proxyMode location when a precise city-center or ZIP proxy is needed."),
|
|
20987
21608
|
pages: import_zod26.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
|
|
20988
21609
|
debug: import_zod26.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
|
|
20989
21610
|
includePageSnapshots: import_zod26.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs through the same product capture path."),
|
|
@@ -21402,70 +22023,72 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
21402
22023
|
if (savesReports) registerSavedReportResources(server);
|
|
21403
22024
|
server.registerTool("harvest_paa", {
|
|
21404
22025
|
title: "Google PAA + SERP Harvest",
|
|
21405
|
-
description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en").
|
|
22026
|
+
description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use maxQuestions 30 normally, 100-200 for "full", "deep", "all", or comprehensive research. Deep harvests above 100 questions can run for several minutes with no interim progress \u2014 warn the user before starting one and keep maxQuestions at or below 100 unless they explicitly want a deep harvest. Credits are charged by extracted question; unused request hold is refunded.'),
|
|
21406
22027
|
inputSchema: HarvestPaaInputSchema,
|
|
21407
22028
|
outputSchema: HarvestPaaOutputSchema,
|
|
21408
22029
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
21409
22030
|
}, async (input) => formatHarvestPaa(await executor.harvestPaa(input), input));
|
|
21410
22031
|
server.registerTool("search_serp", {
|
|
21411
22032
|
title: "Google SERP Lookup",
|
|
21412
|
-
description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request.
|
|
22033
|
+
description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting. Use proxyMode location and proxyZip only when the user explicitly needs city/ZIP-targeted residential proxy evidence."),
|
|
21413
22034
|
inputSchema: SearchSerpInputSchema,
|
|
21414
22035
|
outputSchema: SearchSerpOutputSchema,
|
|
21415
22036
|
annotations: liveWebToolAnnotations("Google SERP Lookup")
|
|
21416
22037
|
}, async (input) => formatSearchSerp(await executor.searchSerp(input), input));
|
|
21417
22038
|
server.registerTool("extract_url", {
|
|
21418
22039
|
title: "Single URL Extract",
|
|
21419
|
-
description: withReportNote("Extract structured data from one public URL
|
|
22040
|
+
description: withReportNote("Extract structured data from one public URL when the user provides one page, asks to inspect/scrape a page, or needs page content, schema, headings, metadata, screenshots, branding, or media assets. Returns structured page fields plus artifact handles for saved reports/screenshots/media when requested. Use map_site_urls before extracting a site inventory; use extract_site for multi-page crawling."),
|
|
21420
22041
|
inputSchema: ExtractUrlInputSchema,
|
|
21421
22042
|
outputSchema: ExtractUrlOutputSchema,
|
|
21422
22043
|
annotations: liveWebToolAnnotations("Single URL Extract")
|
|
21423
22044
|
}, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
|
|
21424
22045
|
server.registerTool("map_site_urls", {
|
|
21425
22046
|
title: "Site URL Map",
|
|
21426
|
-
description: withReportNote("Map/crawl a public website
|
|
22047
|
+
description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
|
|
21427
22048
|
inputSchema: MapSiteUrlsInputSchema,
|
|
21428
22049
|
outputSchema: MapSiteUrlsOutputSchema,
|
|
21429
22050
|
annotations: liveWebToolAnnotations("Site URL Map")
|
|
21430
22051
|
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
|
|
21431
22052
|
server.registerTool("extract_site", {
|
|
21432
22053
|
title: "Multi-Page Site Extract",
|
|
21433
|
-
description: withReportNote("Run multi-page extraction across a public website. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use
|
|
22054
|
+
description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
|
|
21434
22055
|
inputSchema: ExtractSiteInputSchema,
|
|
21435
22056
|
outputSchema: ExtractSiteOutputSchema,
|
|
21436
22057
|
annotations: liveWebToolAnnotations("Multi-Page Site Extract")
|
|
21437
22058
|
}, async (input) => formatExtractSite(await executor.extractSite(input), input));
|
|
21438
22059
|
server.registerTool("youtube_harvest", {
|
|
21439
22060
|
title: "YouTube Video Harvest",
|
|
21440
|
-
description: withReportNote('Harvest YouTube video metadata by
|
|
22061
|
+
description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
|
|
21441
22062
|
inputSchema: YoutubeHarvestInputSchema,
|
|
21442
22063
|
outputSchema: YoutubeHarvestOutputSchema,
|
|
21443
22064
|
annotations: liveWebToolAnnotations("YouTube Video Harvest")
|
|
21444
22065
|
}, async (input) => formatYoutubeHarvest(await executor.youtubeHarvest(input), input));
|
|
21445
22066
|
server.registerTool("youtube_transcribe", {
|
|
21446
22067
|
title: "YouTube Transcription",
|
|
21447
|
-
description: withReportNote("Fetch and transcribe captions from a YouTube video. Returns full transcript, timestamped chunks,
|
|
22068
|
+
description: withReportNote("Fetch and transcribe captions from a YouTube video. Use this when the user asks what was said in a YouTube video, wants claims/offers/lessons extracted from a video, or provides a YouTube URL for transcript work. Returns full transcript, timestamped chunks, word count, and resolvedInputs. Pass videoId from youtube_harvest results, or pass url when the user pasted a YouTube URL. Use youtube_harvest first when you need to discover videos by topic/channel."),
|
|
21448
22069
|
inputSchema: YoutubeTranscribeInputSchema,
|
|
22070
|
+
outputSchema: YoutubeTranscribeOutputSchema,
|
|
21449
22071
|
annotations: liveWebToolAnnotations("YouTube Transcription")
|
|
21450
22072
|
}, async (input) => formatYoutubeTranscribe(await executor.youtubeTranscribe(input), input));
|
|
21451
22073
|
server.registerTool("facebook_page_intel", {
|
|
21452
22074
|
title: "Facebook Advertiser Ad Intel",
|
|
21453
|
-
description: withReportNote("Harvest ads from a Facebook advertiser. Returns ad copy, headlines, CTAs, creative type, status, landing URLs, and direct ad video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name as query. Use
|
|
22075
|
+
description: withReportNote("Harvest ads from a Facebook advertiser when the user wants current ad copy, creative angles, CTAs, video URLs, or competitive ad intelligence for one brand/page. Returns ad copy, headlines, CTAs, creative type, status, landing URLs, and direct ad video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name as query. Use facebook_ad_search first when the advertiser handle is unknown. For normal public Facebook reels/posts/watch/share URLs, use facebook_video_transcribe instead."),
|
|
21454
22076
|
inputSchema: FacebookPageIntelInputSchema,
|
|
21455
22077
|
outputSchema: FacebookPageIntelOutputSchema,
|
|
21456
22078
|
annotations: liveWebToolAnnotations("Facebook Advertiser Ad Intel")
|
|
21457
22079
|
}, async (input) => formatFacebookPageIntel(await executor.facebookPageIntel(input), input));
|
|
21458
22080
|
server.registerTool("facebook_ad_search", {
|
|
21459
22081
|
title: "Facebook Ad Library Search",
|
|
21460
|
-
description: withReportNote("Search Facebook Ad Library by brand,
|
|
22082
|
+
description: withReportNote("Search Facebook Ad Library when the user wants to find advertisers by brand, competitor, niche, or keyword. Returns advertisers with ad counts and library IDs. Use this to discover competitor/page handles, then pass libraryId or pageId to facebook_page_intel for ad details."),
|
|
21461
22083
|
inputSchema: FacebookAdSearchInputSchema,
|
|
21462
22084
|
outputSchema: FacebookAdSearchOutputSchema,
|
|
21463
22085
|
annotations: liveWebToolAnnotations("Facebook Ad Library Search")
|
|
21464
22086
|
}, async (input) => formatFacebookAdSearch(await executor.facebookAdSearch(input), input));
|
|
21465
22087
|
server.registerTool("facebook_ad_transcribe", {
|
|
21466
22088
|
title: "Facebook Ad Transcription",
|
|
21467
|
-
description: "Transcribe audio from a Facebook ad video CDN URL. Returns full transcript
|
|
22089
|
+
description: "Transcribe audio from a Facebook ad video CDN URL. Use this when facebook_page_intel returned a direct videoUrl and the user asks what the ad says, what claims it makes, or wants ad-message extraction. Returns full transcript, timestamped chunks, word count, and resolvedInputs. Use only with the direct videoUrl value from facebook_page_intel results; do not pass public Facebook post/reel/share URLs. Use facebook_video_transcribe for organic Facebook URLs, and facebook_page_intel first when you only have a brand/page/ad library handle.",
|
|
21468
22090
|
inputSchema: FacebookAdTranscribeInputSchema,
|
|
22091
|
+
outputSchema: FacebookAdTranscribeOutputSchema,
|
|
21469
22092
|
annotations: liveWebToolAnnotations("Facebook Ad Transcription")
|
|
21470
22093
|
}, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
|
|
21471
22094
|
server.registerTool("facebook_video_transcribe", {
|
|
@@ -21484,14 +22107,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
21484
22107
|
}, async (input) => formatMapsPlaceIntel(await executor.mapsPlaceIntel(input), input));
|
|
21485
22108
|
server.registerTool("maps_search", {
|
|
21486
22109
|
title: "Google Maps Business Search",
|
|
21487
|
-
description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack."
|
|
22110
|
+
description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." Maps is the default location-targeted surface: omit proxyMode for normal Maps use so the service creates residential proxy evidence for the requested market and rotates on retryable failures. Pass proxyZip only when a specific ZIP or city-center ZIP is known. Use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting. Returns up to 50 candidates with names, place URLs, CIDs when available, ratings, review counts, profile metadata, and sanitized attempt telemetry. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.'),
|
|
21488
22111
|
inputSchema: MapsSearchInputSchema,
|
|
21489
22112
|
outputSchema: MapsSearchOutputSchema,
|
|
21490
22113
|
annotations: liveWebToolAnnotations("Google Maps Business Search")
|
|
21491
22114
|
}, async (input) => formatMapsSearch(await executor.mapsSearch(input), input));
|
|
21492
22115
|
server.registerTool("directory_workflow", {
|
|
21493
22116
|
title: "Directory Workflow: Markets + Maps",
|
|
21494
|
-
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency.
|
|
22117
|
+
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Maps workflows default to location-targeted proxying so each city can use city/state or ZIP-group residential proxy evidence; use proxyMode configured only when you explicitly do not want city/ZIP proxy targeting. Saved CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. Structured city results include sanitized attempt telemetry. Use maps_place_intel only when a selected profile needs deeper review topics, profile review count confirmation, or review cards. For local Lead Magician ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath only in local/test mode.'),
|
|
21495
22118
|
inputSchema: DirectoryWorkflowInputSchema,
|
|
21496
22119
|
outputSchema: DirectoryWorkflowOutputSchema,
|
|
21497
22120
|
annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
|
|
@@ -21512,21 +22135,28 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
21512
22135
|
}, async (input) => formatWorkflowSuggest(input));
|
|
21513
22136
|
server.registerTool("workflow_run", {
|
|
21514
22137
|
title: "Run Workflow",
|
|
21515
|
-
description: withReportNote("
|
|
22138
|
+
description: withReportNote("Start a higher-level MCP Scraper workflow. Use after workflow_suggest or workflow_list. Runnable workflow ids: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. This is the main MCP tool for market analysis, ICP evidence packets, local competitive audits, Maps/SERP comparisons, content gap briefs, and AI Overview language guidance. Stepwise workflows (e.g. agent-packet) run ONE leg per call and return runId, the step output, and nextStep \u2014 when nextStep is present, call workflow_step with the runId to run the next leg, and keep calling it until done is true. This keeps each call short instead of one long blocking run, so report each step result to the user as it arrives."),
|
|
21516
22139
|
inputSchema: WorkflowRunInputSchema,
|
|
21517
22140
|
outputSchema: WorkflowRunOutputSchema,
|
|
21518
22141
|
annotations: liveWebToolAnnotations("Run Workflow")
|
|
21519
22142
|
}, async (input) => formatWorkflowRun(await executor.workflowRun(input), input));
|
|
22143
|
+
server.registerTool("workflow_step", {
|
|
22144
|
+
title: "Advance Workflow Step",
|
|
22145
|
+
description: withReportNote("Run the next leg of a stepwise MCP Scraper workflow started with workflow_run. Pass the runId. Each call executes exactly one logical step (typically one live harvest), persists that step's artifacts, and returns the step output plus nextStep. Keep calling workflow_step with the same runId until done is true, reporting each step result to the user as it lands. Use this instead of waiting on one long workflow call \u2014 it avoids client timeouts on long multi-step jobs."),
|
|
22146
|
+
inputSchema: WorkflowStepInputSchema,
|
|
22147
|
+
outputSchema: WorkflowStepOutputSchema,
|
|
22148
|
+
annotations: liveWebToolAnnotations("Advance Workflow Step")
|
|
22149
|
+
}, async (input) => formatWorkflowStep(await executor.workflowStep(input), input));
|
|
21520
22150
|
server.registerTool("workflow_status", {
|
|
21521
22151
|
title: "Workflow Status",
|
|
21522
|
-
description: "Fetch a hosted workflow run by id and list its current status and artifacts. Use
|
|
22152
|
+
description: "Fetch a hosted workflow run by id and list its current status and artifacts. Use when a workflow may still be running, when the model needs to re-open a run, inspect artifact ids, recover from a long workflow, or decide whether to call workflow_step or workflow_artifact_read next. Use only a runId returned by workflow_run/workflow_step/workflow_status; do not construct one yourself.",
|
|
21523
22153
|
inputSchema: WorkflowStatusInputSchema,
|
|
21524
22154
|
outputSchema: WorkflowStatusOutputSchema,
|
|
21525
22155
|
annotations: liveWebToolAnnotations("Workflow Status")
|
|
21526
22156
|
}, async (input) => formatWorkflowStatus(await executor.workflowStatus(input), input));
|
|
21527
22157
|
server.registerTool("workflow_artifact_read", {
|
|
21528
22158
|
title: "Read Workflow Artifact",
|
|
21529
|
-
description: "Read a workflow artifact back into MCP context by run id and artifact id. Use this before writing final deliverables so the answer is grounded in generated evidence.json, CSVs, Markdown briefs, reports, and task files instead of memory. Use maxBytes to limit large CSV/JSON artifacts.",
|
|
22159
|
+
description: "Read a workflow artifact back into MCP context by run id and artifact id. Use this before writing final deliverables so the answer is grounded in generated evidence.json, CSVs, Markdown briefs, reports, and task files instead of memory. Use workflow_status first when artifact ids are unknown. Use only artifactId values returned by workflow_run/workflow_step/workflow_status; do not construct one yourself. Use maxBytes to limit large CSV/JSON artifacts.",
|
|
21530
22160
|
inputSchema: WorkflowArtifactReadInputSchema,
|
|
21531
22161
|
outputSchema: WorkflowArtifactReadOutputSchema,
|
|
21532
22162
|
annotations: liveWebToolAnnotations("Read Workflow Artifact")
|
|
@@ -21568,6 +22198,27 @@ var init_paa_mcp_server = __esm({
|
|
|
21568
22198
|
});
|
|
21569
22199
|
|
|
21570
22200
|
// src/mcp/http-mcp-tool-executor.ts
|
|
22201
|
+
function youtubeVideoIdFromUrl(url) {
|
|
22202
|
+
if (!url) return null;
|
|
22203
|
+
try {
|
|
22204
|
+
const parsed = new URL(url);
|
|
22205
|
+
const host = parsed.hostname.replace(/^www\./, "").replace(/^m\./, "");
|
|
22206
|
+
if (host === "youtu.be") {
|
|
22207
|
+
const id = parsed.pathname.split("/").filter(Boolean)[0];
|
|
22208
|
+
return id || null;
|
|
22209
|
+
}
|
|
22210
|
+
if (host === "youtube.com" || host === "music.youtube.com") {
|
|
22211
|
+
const watchId = parsed.searchParams.get("v");
|
|
22212
|
+
if (watchId) return watchId;
|
|
22213
|
+
const parts = parsed.pathname.split("/").filter(Boolean);
|
|
22214
|
+
const markerIndex = parts.findIndex((part) => ["shorts", "embed", "live"].includes(part));
|
|
22215
|
+
if (markerIndex >= 0 && parts[markerIndex + 1]) return parts[markerIndex + 1];
|
|
22216
|
+
}
|
|
22217
|
+
} catch {
|
|
22218
|
+
return null;
|
|
22219
|
+
}
|
|
22220
|
+
return null;
|
|
22221
|
+
}
|
|
21571
22222
|
var HttpMcpToolExecutor;
|
|
21572
22223
|
var init_http_mcp_tool_executor = __esm({
|
|
21573
22224
|
"src/mcp/http-mcp-tool-executor.ts"() {
|
|
@@ -21698,7 +22349,21 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
21698
22349
|
return this.call("/youtube/harvest", input);
|
|
21699
22350
|
}
|
|
21700
22351
|
youtubeTranscribe(input) {
|
|
21701
|
-
|
|
22352
|
+
const videoId = input.videoId?.trim() || youtubeVideoIdFromUrl(input.url);
|
|
22353
|
+
if (!videoId) {
|
|
22354
|
+
return Promise.resolve({
|
|
22355
|
+
content: [{
|
|
22356
|
+
type: "text",
|
|
22357
|
+
text: JSON.stringify({
|
|
22358
|
+
error_code: "youtube_video_id_required",
|
|
22359
|
+
error: "Pass videoId from youtube_harvest or a YouTube url that contains a video id.",
|
|
22360
|
+
retryable: false
|
|
22361
|
+
})
|
|
22362
|
+
}],
|
|
22363
|
+
isError: true
|
|
22364
|
+
});
|
|
22365
|
+
}
|
|
22366
|
+
return this.call("/youtube/transcribe", { videoId });
|
|
21702
22367
|
}
|
|
21703
22368
|
facebookPageIntel(input) {
|
|
21704
22369
|
return this.call("/facebook/page-intel", input);
|
|
@@ -21735,6 +22400,10 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
21735
22400
|
webhookUrl: input.webhookUrl
|
|
21736
22401
|
}, Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 9e5);
|
|
21737
22402
|
}
|
|
22403
|
+
workflowStep(input) {
|
|
22404
|
+
const timeoutMs = this.httpTimeoutOverrideMs ?? Number(process.env.MCP_SCRAPER_WORKFLOW_TIMEOUT_MS ?? 9e5);
|
|
22405
|
+
return this.call(`/workflows/runs/${encodeURIComponent(input.runId)}/step`, {}, Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 9e5);
|
|
22406
|
+
}
|
|
21738
22407
|
workflowStatus(input) {
|
|
21739
22408
|
return this.getJson(`/workflows/runs/${encodeURIComponent(input.runId)}`);
|
|
21740
22409
|
}
|
|
@@ -21788,16 +22457,18 @@ async function requireMcpCallerKey(c) {
|
|
|
21788
22457
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
21789
22458
|
server.registerTool("capture_serp_snapshot", {
|
|
21790
22459
|
title: "SERP Intelligence Snapshot",
|
|
21791
|
-
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location
|
|
22460
|
+
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location and infer gl/hl. Omit proxyMode for normal use; the service defaults to the configured browser-service proxy without city/ZIP targeting for the highest general success rate. Use proxyMode location only when the user explicitly needs city/ZIP-targeted residential proxy evidence. Use none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
|
|
21792
22461
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
22462
|
+
outputSchema: CaptureSerpSnapshotOutputSchema,
|
|
21793
22463
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
21794
|
-
}, async (input) => executor.captureSerpSnapshot(input));
|
|
22464
|
+
}, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
|
|
21795
22465
|
server.registerTool("capture_serp_page_snapshots", {
|
|
21796
22466
|
title: "SERP Intelligence Page Snapshots",
|
|
21797
22467
|
description: "Capture public ranking-page evidence through POST /serp-intelligence/page-snapshots, the same product page snapshot path used by Phoenix. Provide urls for simple captures or targets when preserving organic, AI citation, local-pack, configured target, or site-subject source metadata. Private IPs, localhost, file URLs, and internal URLs are rejected by the service. Use timeoutMs for slow pages and debug true for sanitized proxy/browser diagnostics.",
|
|
21798
22468
|
inputSchema: CaptureSerpPageSnapshotsInputSchema,
|
|
22469
|
+
outputSchema: CaptureSerpPageSnapshotsOutputSchema,
|
|
21799
22470
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
21800
|
-
}, async (input) => executor.captureSerpPageSnapshots(input));
|
|
22471
|
+
}, async (input) => formatCaptureSerpPageSnapshots(await executor.captureSerpPageSnapshots(input), input));
|
|
21801
22472
|
}
|
|
21802
22473
|
var import_hono9, import_webStandardStreamableHttp, mcpApp;
|
|
21803
22474
|
var init_mcp_routes = __esm({
|
|
@@ -23518,6 +24189,7 @@ var init_server = __esm({
|
|
|
23518
24189
|
init_rates();
|
|
23519
24190
|
init_billing_schemas();
|
|
23520
24191
|
init_server_schemas();
|
|
24192
|
+
init_schemas3();
|
|
23521
24193
|
init_credit_operations();
|
|
23522
24194
|
init_harvest();
|
|
23523
24195
|
init_harvest_problems();
|
|
@@ -23747,7 +24419,7 @@ var init_server = __esm({
|
|
|
23747
24419
|
gl: body.gl ?? "us",
|
|
23748
24420
|
hl: body.hl ?? "en",
|
|
23749
24421
|
device: body.device ?? "desktop",
|
|
23750
|
-
proxyMode: body.proxyMode ??
|
|
24422
|
+
proxyMode: body.proxyMode ?? DEFAULT_PROXY_MODE,
|
|
23751
24423
|
proxyZip: body.proxyZip,
|
|
23752
24424
|
debug: body.debug ?? false,
|
|
23753
24425
|
serpOnly: body.serpOnly ?? false,
|
|
@@ -23783,7 +24455,7 @@ var init_server = __esm({
|
|
|
23783
24455
|
gl: body.gl ?? "us",
|
|
23784
24456
|
hl: body.hl ?? "en",
|
|
23785
24457
|
device: body.device ?? "desktop",
|
|
23786
|
-
proxyMode: body.proxyMode ??
|
|
24458
|
+
proxyMode: body.proxyMode ?? DEFAULT_PROXY_MODE,
|
|
23787
24459
|
proxyZip: body.proxyZip,
|
|
23788
24460
|
debug: body.debug ?? false,
|
|
23789
24461
|
serpOnly: body.serpOnly ?? false,
|