mcp-scraper 0.2.7 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -5
- package/dist/bin/api-server.cjs +2417 -648
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1276 -0
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -0
- package/dist/bin/mcp-scraper-cli.d.cts +1 -0
- package/dist/bin/mcp-scraper-cli.d.ts +1 -0
- package/dist/bin/mcp-scraper-cli.js +476 -0
- package/dist/bin/mcp-scraper-cli.js.map +1 -0
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +69 -14
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +5 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +5 -1
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +69 -14
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/bin/paa-harvest.cjs +42 -6
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/{chunk-3TF6UT2P.js → chunk-5E6KJC26.js} +2 -2
- package/dist/{chunk-D4CJBZBY.js → chunk-7GCCOT3M.js} +372 -1
- package/dist/chunk-7GCCOT3M.js.map +1 -0
- package/dist/chunk-BJDCPCIE.js +7 -0
- package/dist/chunk-BJDCPCIE.js.map +1 -0
- package/dist/{chunk-MY3S7EX7.js → chunk-CQTAKXBN.js} +43 -7
- package/dist/chunk-CQTAKXBN.js.map +1 -0
- package/dist/chunk-L6IS63WS.js +869 -0
- package/dist/chunk-L6IS63WS.js.map +1 -0
- package/dist/{chunk-IQOCZGJJ.js → chunk-ROS67BNV.js} +5 -5
- package/dist/{chunk-7SI6XIR3.js → chunk-XAY5U67D.js} +70 -15
- package/dist/chunk-XAY5U67D.js.map +1 -0
- package/dist/{db-YWCNHBLH.js → db-BVHYI57K.js} +38 -2
- package/dist/index.cjs +42 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1 -1
- package/dist/{server-RIKRBDOI.js → server-BKJ7LULQ.js} +818 -394
- package/dist/server-BKJ7LULQ.js.map +1 -0
- package/dist/{worker-NAKGTIF5.js → worker-TDJQ6TH3.js} +8 -8
- package/docs/mcp-tool-quality-spec.md +3 -1
- package/docs/specs/agent-ready-seo-packet-spec.md +237 -0
- package/docs/specs/cli-agent-wiring-spec.md +203 -0
- package/docs/specs/deferred-work-spec.md +12 -0
- package/docs/specs/local-competitive-audit-spec.md +312 -0
- package/docs/specs/scheduled-workflows-api-spec.md +304 -0
- package/docs/specs/seo-cli-growth-roadmap-spec.md +179 -0
- package/docs/specs/seo-workflow-runner-and-reports-spec.md +241 -0
- package/package.json +3 -2
- package/dist/chunk-7SI6XIR3.js.map +0 -1
- package/dist/chunk-D4CJBZBY.js.map +0 -1
- package/dist/chunk-IZE5UW7Y.js +0 -7
- package/dist/chunk-IZE5UW7Y.js.map +0 -1
- package/dist/chunk-MY3S7EX7.js.map +0 -1
- package/dist/server-RIKRBDOI.js.map +0 -1
- /package/dist/{chunk-3TF6UT2P.js.map → chunk-5E6KJC26.js.map} +0 -0
- /package/dist/{chunk-IQOCZGJJ.js.map → chunk-ROS67BNV.js.map} +0 -0
- /package/dist/{db-YWCNHBLH.js.map → db-BVHYI57K.js.map} +0 -0
- /package/dist/{worker-NAKGTIF5.js.map → worker-TDJQ6TH3.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -6913,6 +6913,7 @@ __export(db_exports, {
|
|
|
6913
6913
|
claimPendingKpoJob: () => claimPendingKpoJob,
|
|
6914
6914
|
completeJob: () => completeJob,
|
|
6915
6915
|
completeKpoJob: () => completeKpoJob,
|
|
6916
|
+
completeWorkflowRunRecord: () => completeWorkflowRunRecord,
|
|
6916
6917
|
consumePasswordResetToken: () => consumePasswordResetToken,
|
|
6917
6918
|
countActiveJobsForUser: () => countActiveJobsForUser,
|
|
6918
6919
|
countActiveUsers: () => countActiveUsers,
|
|
@@ -6921,11 +6922,15 @@ __export(db_exports, {
|
|
|
6921
6922
|
createPasswordResetToken: () => createPasswordResetToken,
|
|
6922
6923
|
createRunningJob: () => createRunningJob,
|
|
6923
6924
|
createUser: () => createUser,
|
|
6925
|
+
createWorkflowRun: () => createWorkflowRun,
|
|
6926
|
+
createWorkflowSchedule: () => createWorkflowSchedule,
|
|
6924
6927
|
creditMc: () => creditMc,
|
|
6925
6928
|
deactivateUser: () => deactivateUser,
|
|
6926
6929
|
debitMc: () => debitMc,
|
|
6930
|
+
deleteWorkflowSchedule: () => deleteWorkflowSchedule,
|
|
6927
6931
|
failJob: () => failJob,
|
|
6928
6932
|
failKpoJob: () => failKpoJob,
|
|
6933
|
+
failWorkflowRunRecord: () => failWorkflowRunRecord,
|
|
6929
6934
|
finishHarvestAttempt: () => finishHarvestAttempt,
|
|
6930
6935
|
generateApiKey: () => generateApiKey,
|
|
6931
6936
|
getDb: () => getDb,
|
|
@@ -6938,19 +6943,32 @@ __export(db_exports, {
|
|
|
6938
6943
|
getUserById: () => getUserById,
|
|
6939
6944
|
getUserByStripeCustomerId: () => getUserByStripeCustomerId,
|
|
6940
6945
|
getUserStats: () => getUserStats,
|
|
6946
|
+
getWorkflowArtifact: () => getWorkflowArtifact,
|
|
6947
|
+
getWorkflowRun: () => getWorkflowRun,
|
|
6948
|
+
getWorkflowRunByIdempotencyKey: () => getWorkflowRunByIdempotencyKey,
|
|
6949
|
+
getWorkflowSchedule: () => getWorkflowSchedule,
|
|
6941
6950
|
hashPassword: () => hashPassword,
|
|
6942
6951
|
ledgerExistsForOperation: () => ledgerExistsForOperation,
|
|
6943
6952
|
ledgerExistsForStripePI: () => ledgerExistsForStripePI,
|
|
6953
|
+
listDueWorkflowSchedules: () => listDueWorkflowSchedules,
|
|
6944
6954
|
listHarvestAttempts: () => listHarvestAttempts,
|
|
6945
6955
|
listJobs: () => listJobs,
|
|
6946
6956
|
listKpoJobs: () => listKpoJobs,
|
|
6947
6957
|
listRequestEvents: () => listRequestEvents,
|
|
6948
6958
|
listUsers: () => listUsers,
|
|
6959
|
+
listWorkflowArtifacts: () => listWorkflowArtifacts,
|
|
6960
|
+
listWorkflowRuns: () => listWorkflowRuns,
|
|
6961
|
+
listWorkflowSchedules: () => listWorkflowSchedules,
|
|
6949
6962
|
logKpoPhaseComplete: () => logKpoPhaseComplete,
|
|
6950
6963
|
logRequestEvent: () => logRequestEvent,
|
|
6964
|
+
markWorkflowRunRunning: () => markWorkflowRunRunning,
|
|
6965
|
+
markWorkflowScheduleRan: () => markWorkflowScheduleRan,
|
|
6951
6966
|
migrate: () => migrate,
|
|
6967
|
+
patchWorkflowSchedule: () => patchWorkflowSchedule,
|
|
6952
6968
|
reconcileBalanceMc: () => reconcileBalanceMc,
|
|
6953
6969
|
recordStripeEvent: () => recordStripeEvent,
|
|
6970
|
+
recordWorkflowWebhookDelivery: () => recordWorkflowWebhookDelivery,
|
|
6971
|
+
replaceWorkflowArtifacts: () => replaceWorkflowArtifacts,
|
|
6954
6972
|
revokeApiKey: () => revokeApiKey,
|
|
6955
6973
|
rotateApiKey: () => rotateApiKey,
|
|
6956
6974
|
setConcurrencySubId: () => setConcurrencySubId,
|
|
@@ -7162,6 +7180,75 @@ async function migrate() {
|
|
|
7162
7180
|
)
|
|
7163
7181
|
`);
|
|
7164
7182
|
await db.execute(`CREATE INDEX IF NOT EXISTS request_events_user_created_at ON request_events(user_id, created_at DESC)`);
|
|
7183
|
+
await db.execute(`
|
|
7184
|
+
CREATE TABLE IF NOT EXISTS workflow_schedules (
|
|
7185
|
+
id TEXT PRIMARY KEY,
|
|
7186
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
7187
|
+
workflow_id TEXT NOT NULL,
|
|
7188
|
+
name TEXT NOT NULL,
|
|
7189
|
+
input_json TEXT NOT NULL,
|
|
7190
|
+
cadence TEXT NOT NULL,
|
|
7191
|
+
cron_expr TEXT,
|
|
7192
|
+
timezone TEXT NOT NULL DEFAULT 'UTC',
|
|
7193
|
+
webhook_url TEXT,
|
|
7194
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
7195
|
+
next_run_at TEXT,
|
|
7196
|
+
last_run_at TEXT,
|
|
7197
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
7198
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
7199
|
+
)
|
|
7200
|
+
`);
|
|
7201
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_schedules_user_status ON workflow_schedules(user_id, status, created_at DESC)`);
|
|
7202
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_schedules_due ON workflow_schedules(status, next_run_at)`);
|
|
7203
|
+
await db.execute(`
|
|
7204
|
+
CREATE TABLE IF NOT EXISTS workflow_runs (
|
|
7205
|
+
id TEXT PRIMARY KEY,
|
|
7206
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
7207
|
+
schedule_id TEXT,
|
|
7208
|
+
workflow_id TEXT NOT NULL,
|
|
7209
|
+
input_json TEXT NOT NULL,
|
|
7210
|
+
status TEXT NOT NULL DEFAULT 'queued',
|
|
7211
|
+
manifest_json TEXT,
|
|
7212
|
+
error_message TEXT,
|
|
7213
|
+
queued_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
7214
|
+
started_at TEXT,
|
|
7215
|
+
completed_at TEXT,
|
|
7216
|
+
idempotency_key TEXT
|
|
7217
|
+
)
|
|
7218
|
+
`);
|
|
7219
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_user_created ON workflow_runs(user_id, queued_at DESC)`);
|
|
7220
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_schedule ON workflow_runs(schedule_id, queued_at DESC)`);
|
|
7221
|
+
await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS workflow_runs_idempotency ON workflow_runs(idempotency_key) WHERE idempotency_key IS NOT NULL`);
|
|
7222
|
+
await db.execute(`
|
|
7223
|
+
CREATE TABLE IF NOT EXISTS workflow_artifacts (
|
|
7224
|
+
id TEXT PRIMARY KEY,
|
|
7225
|
+
run_id TEXT NOT NULL REFERENCES workflow_runs(id),
|
|
7226
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
7227
|
+
kind TEXT NOT NULL,
|
|
7228
|
+
label TEXT NOT NULL,
|
|
7229
|
+
path TEXT NOT NULL,
|
|
7230
|
+
content_type TEXT NOT NULL,
|
|
7231
|
+
bytes INTEGER,
|
|
7232
|
+
rows_count INTEGER,
|
|
7233
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
7234
|
+
)
|
|
7235
|
+
`);
|
|
7236
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_artifacts_run ON workflow_artifacts(run_id)`);
|
|
7237
|
+
await db.execute(`
|
|
7238
|
+
CREATE TABLE IF NOT EXISTS workflow_webhook_deliveries (
|
|
7239
|
+
id TEXT PRIMARY KEY,
|
|
7240
|
+
run_id TEXT NOT NULL REFERENCES workflow_runs(id),
|
|
7241
|
+
schedule_id TEXT,
|
|
7242
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
7243
|
+
webhook_url TEXT NOT NULL,
|
|
7244
|
+
status TEXT NOT NULL,
|
|
7245
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
7246
|
+
last_error TEXT,
|
|
7247
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
7248
|
+
delivered_at TEXT
|
|
7249
|
+
)
|
|
7250
|
+
`);
|
|
7251
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_webhook_deliveries_run ON workflow_webhook_deliveries(run_id)`);
|
|
7165
7252
|
await db.execute(`
|
|
7166
7253
|
CREATE TABLE IF NOT EXISTS stripe_events (
|
|
7167
7254
|
event_id TEXT PRIMARY KEY,
|
|
@@ -7551,6 +7638,290 @@ async function listRequestEvents(userId, limit = 100) {
|
|
|
7551
7638
|
});
|
|
7552
7639
|
return res.rows.map((r) => rowToRequestEvent(r));
|
|
7553
7640
|
}
|
|
7641
|
+
function parseJsonRecordValue(value) {
|
|
7642
|
+
if (value == null) return {};
|
|
7643
|
+
try {
|
|
7644
|
+
const parsed = JSON.parse(String(value));
|
|
7645
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
|
7646
|
+
} catch {
|
|
7647
|
+
return {};
|
|
7648
|
+
}
|
|
7649
|
+
}
|
|
7650
|
+
function rowToWorkflowSchedule(row) {
|
|
7651
|
+
return {
|
|
7652
|
+
id: String(row.id),
|
|
7653
|
+
user_id: Number(row.user_id),
|
|
7654
|
+
workflow_id: String(row.workflow_id),
|
|
7655
|
+
name: String(row.name),
|
|
7656
|
+
input: parseJsonRecordValue(row.input_json),
|
|
7657
|
+
cadence: String(row.cadence),
|
|
7658
|
+
cron_expr: row.cron_expr != null ? String(row.cron_expr) : null,
|
|
7659
|
+
timezone: String(row.timezone ?? "UTC"),
|
|
7660
|
+
webhook_url: row.webhook_url != null ? String(row.webhook_url) : null,
|
|
7661
|
+
status: String(row.status),
|
|
7662
|
+
next_run_at: row.next_run_at != null ? String(row.next_run_at) : null,
|
|
7663
|
+
last_run_at: row.last_run_at != null ? String(row.last_run_at) : null,
|
|
7664
|
+
created_at: String(row.created_at),
|
|
7665
|
+
updated_at: String(row.updated_at)
|
|
7666
|
+
};
|
|
7667
|
+
}
|
|
7668
|
+
function rowToWorkflowRun(row) {
|
|
7669
|
+
return {
|
|
7670
|
+
id: String(row.id),
|
|
7671
|
+
user_id: Number(row.user_id),
|
|
7672
|
+
schedule_id: row.schedule_id != null ? String(row.schedule_id) : null,
|
|
7673
|
+
workflow_id: String(row.workflow_id),
|
|
7674
|
+
input: parseJsonRecordValue(row.input_json),
|
|
7675
|
+
status: String(row.status),
|
|
7676
|
+
manifest: row.manifest_json != null ? parseNullableJson(row.manifest_json) : null,
|
|
7677
|
+
error_message: row.error_message != null ? String(row.error_message) : null,
|
|
7678
|
+
queued_at: String(row.queued_at),
|
|
7679
|
+
started_at: row.started_at != null ? String(row.started_at) : null,
|
|
7680
|
+
completed_at: row.completed_at != null ? String(row.completed_at) : null,
|
|
7681
|
+
idempotency_key: row.idempotency_key != null ? String(row.idempotency_key) : null
|
|
7682
|
+
};
|
|
7683
|
+
}
|
|
7684
|
+
function rowToWorkflowArtifact(row) {
|
|
7685
|
+
return {
|
|
7686
|
+
id: String(row.id),
|
|
7687
|
+
run_id: String(row.run_id),
|
|
7688
|
+
user_id: Number(row.user_id),
|
|
7689
|
+
kind: String(row.kind),
|
|
7690
|
+
label: String(row.label),
|
|
7691
|
+
path: String(row.path),
|
|
7692
|
+
content_type: String(row.content_type),
|
|
7693
|
+
bytes: row.bytes != null ? Number(row.bytes) : null,
|
|
7694
|
+
rows_count: row.rows_count != null ? Number(row.rows_count) : null,
|
|
7695
|
+
created_at: String(row.created_at)
|
|
7696
|
+
};
|
|
7697
|
+
}
|
|
7698
|
+
async function createWorkflowSchedule(input) {
|
|
7699
|
+
const id = (0, import_node_crypto.randomUUID)();
|
|
7700
|
+
await getDb().execute({
|
|
7701
|
+
sql: `
|
|
7702
|
+
INSERT INTO workflow_schedules (
|
|
7703
|
+
id, user_id, workflow_id, name, input_json, cadence, timezone, webhook_url, next_run_at
|
|
7704
|
+
)
|
|
7705
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
7706
|
+
`,
|
|
7707
|
+
args: [
|
|
7708
|
+
id,
|
|
7709
|
+
input.userId,
|
|
7710
|
+
input.workflowId,
|
|
7711
|
+
input.name,
|
|
7712
|
+
JSON.stringify(input.workflowInput),
|
|
7713
|
+
input.cadence,
|
|
7714
|
+
input.timezone ?? "UTC",
|
|
7715
|
+
input.webhookUrl ?? null,
|
|
7716
|
+
input.nextRunAt ?? null
|
|
7717
|
+
]
|
|
7718
|
+
});
|
|
7719
|
+
const schedule = await getWorkflowSchedule(id, input.userId);
|
|
7720
|
+
if (!schedule) throw new Error("Failed to create workflow schedule");
|
|
7721
|
+
return schedule;
|
|
7722
|
+
}
|
|
7723
|
+
async function getWorkflowSchedule(id, userId) {
|
|
7724
|
+
const res = userId == null ? await getDb().execute({ sql: "SELECT * FROM workflow_schedules WHERE id = ? AND status != ? LIMIT 1", args: [id, "deleted"] }) : await getDb().execute({ sql: "SELECT * FROM workflow_schedules WHERE id = ? AND user_id = ? AND status != ? LIMIT 1", args: [id, userId, "deleted"] });
|
|
7725
|
+
return res.rows[0] ? rowToWorkflowSchedule(res.rows[0]) : null;
|
|
7726
|
+
}
|
|
7727
|
+
async function listWorkflowSchedules(userId) {
|
|
7728
|
+
const res = await getDb().execute({
|
|
7729
|
+
sql: `SELECT * FROM workflow_schedules WHERE user_id = ? AND status != 'deleted' ORDER BY created_at DESC LIMIT 100`,
|
|
7730
|
+
args: [userId]
|
|
7731
|
+
});
|
|
7732
|
+
return res.rows.map((r) => rowToWorkflowSchedule(r));
|
|
7733
|
+
}
|
|
7734
|
+
async function patchWorkflowSchedule(input) {
|
|
7735
|
+
const sets = [];
|
|
7736
|
+
const args = [];
|
|
7737
|
+
if (input.name !== void 0) {
|
|
7738
|
+
sets.push("name = ?");
|
|
7739
|
+
args.push(input.name);
|
|
7740
|
+
}
|
|
7741
|
+
if (input.status !== void 0) {
|
|
7742
|
+
sets.push("status = ?");
|
|
7743
|
+
args.push(input.status);
|
|
7744
|
+
}
|
|
7745
|
+
if (input.workflowInput !== void 0) {
|
|
7746
|
+
sets.push("input_json = ?");
|
|
7747
|
+
args.push(JSON.stringify(input.workflowInput));
|
|
7748
|
+
}
|
|
7749
|
+
if (input.cadence !== void 0) {
|
|
7750
|
+
sets.push("cadence = ?");
|
|
7751
|
+
args.push(input.cadence);
|
|
7752
|
+
}
|
|
7753
|
+
if (input.timezone !== void 0) {
|
|
7754
|
+
sets.push("timezone = ?");
|
|
7755
|
+
args.push(input.timezone);
|
|
7756
|
+
}
|
|
7757
|
+
if (input.webhookUrl !== void 0) {
|
|
7758
|
+
sets.push("webhook_url = ?");
|
|
7759
|
+
args.push(input.webhookUrl);
|
|
7760
|
+
}
|
|
7761
|
+
if (input.nextRunAt !== void 0) {
|
|
7762
|
+
sets.push("next_run_at = ?");
|
|
7763
|
+
args.push(input.nextRunAt);
|
|
7764
|
+
}
|
|
7765
|
+
if (sets.length === 0) return getWorkflowSchedule(input.id, input.userId);
|
|
7766
|
+
sets.push(`updated_at = datetime('now')`);
|
|
7767
|
+
await getDb().execute({
|
|
7768
|
+
sql: `UPDATE workflow_schedules SET ${sets.join(", ")} WHERE id = ? AND user_id = ? AND status != 'deleted'`,
|
|
7769
|
+
args: [...args, input.id, input.userId]
|
|
7770
|
+
});
|
|
7771
|
+
return getWorkflowSchedule(input.id, input.userId);
|
|
7772
|
+
}
|
|
7773
|
+
async function deleteWorkflowSchedule(id, userId) {
|
|
7774
|
+
const res = await getDb().execute({
|
|
7775
|
+
sql: `UPDATE workflow_schedules SET status = 'deleted', updated_at = datetime('now') WHERE id = ? AND user_id = ? AND status != 'deleted'`,
|
|
7776
|
+
args: [id, userId]
|
|
7777
|
+
});
|
|
7778
|
+
return res.rowsAffected > 0;
|
|
7779
|
+
}
|
|
7780
|
+
async function listDueWorkflowSchedules(nowIso, limit = 10) {
|
|
7781
|
+
const res = await getDb().execute({
|
|
7782
|
+
sql: `
|
|
7783
|
+
SELECT * FROM workflow_schedules
|
|
7784
|
+
WHERE status = 'active' AND next_run_at IS NOT NULL AND next_run_at <= ?
|
|
7785
|
+
ORDER BY next_run_at ASC
|
|
7786
|
+
LIMIT ?
|
|
7787
|
+
`,
|
|
7788
|
+
args: [nowIso, limit]
|
|
7789
|
+
});
|
|
7790
|
+
return res.rows.map((r) => rowToWorkflowSchedule(r));
|
|
7791
|
+
}
|
|
7792
|
+
async function markWorkflowScheduleRan(id, lastRunAt, nextRunAt) {
|
|
7793
|
+
await getDb().execute({
|
|
7794
|
+
sql: `UPDATE workflow_schedules SET last_run_at = ?, next_run_at = ?, updated_at = datetime('now') WHERE id = ?`,
|
|
7795
|
+
args: [lastRunAt, nextRunAt, id]
|
|
7796
|
+
});
|
|
7797
|
+
}
|
|
7798
|
+
async function createWorkflowRun(input) {
|
|
7799
|
+
const id = input.id ?? (0, import_node_crypto.randomUUID)();
|
|
7800
|
+
const args = [
|
|
7801
|
+
id,
|
|
7802
|
+
input.userId,
|
|
7803
|
+
input.scheduleId ?? null,
|
|
7804
|
+
input.workflowId,
|
|
7805
|
+
JSON.stringify(input.workflowInput),
|
|
7806
|
+
input.idempotencyKey ?? null
|
|
7807
|
+
];
|
|
7808
|
+
try {
|
|
7809
|
+
await getDb().execute({
|
|
7810
|
+
sql: `
|
|
7811
|
+
INSERT INTO workflow_runs (id, user_id, schedule_id, workflow_id, input_json, idempotency_key)
|
|
7812
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
7813
|
+
`,
|
|
7814
|
+
args
|
|
7815
|
+
});
|
|
7816
|
+
} catch (err) {
|
|
7817
|
+
if (input.idempotencyKey) {
|
|
7818
|
+
const existing = await getWorkflowRunByIdempotencyKey(input.idempotencyKey);
|
|
7819
|
+
if (existing) return existing;
|
|
7820
|
+
}
|
|
7821
|
+
throw err;
|
|
7822
|
+
}
|
|
7823
|
+
const run = await getWorkflowRun(id, input.userId);
|
|
7824
|
+
if (!run) throw new Error("Failed to create workflow run");
|
|
7825
|
+
return run;
|
|
7826
|
+
}
|
|
7827
|
+
async function getWorkflowRun(id, userId) {
|
|
7828
|
+
const res = userId == null ? await getDb().execute({ sql: "SELECT * FROM workflow_runs WHERE id = ? LIMIT 1", args: [id] }) : await getDb().execute({ sql: "SELECT * FROM workflow_runs WHERE id = ? AND user_id = ? LIMIT 1", args: [id, userId] });
|
|
7829
|
+
return res.rows[0] ? rowToWorkflowRun(res.rows[0]) : null;
|
|
7830
|
+
}
|
|
7831
|
+
async function getWorkflowRunByIdempotencyKey(idempotencyKey) {
|
|
7832
|
+
const res = await getDb().execute({ sql: "SELECT * FROM workflow_runs WHERE idempotency_key = ? LIMIT 1", args: [idempotencyKey] });
|
|
7833
|
+
return res.rows[0] ? rowToWorkflowRun(res.rows[0]) : null;
|
|
7834
|
+
}
|
|
7835
|
+
async function listWorkflowRuns(userId, limit = 50) {
|
|
7836
|
+
const res = await getDb().execute({
|
|
7837
|
+
sql: "SELECT * FROM workflow_runs WHERE user_id = ? ORDER BY queued_at DESC LIMIT ?",
|
|
7838
|
+
args: [userId, limit]
|
|
7839
|
+
});
|
|
7840
|
+
return res.rows.map((r) => rowToWorkflowRun(r));
|
|
7841
|
+
}
|
|
7842
|
+
async function markWorkflowRunRunning(id) {
|
|
7843
|
+
await getDb().execute({
|
|
7844
|
+
sql: `UPDATE workflow_runs SET status = 'running', started_at = COALESCE(started_at, datetime('now')) WHERE id = ?`,
|
|
7845
|
+
args: [id]
|
|
7846
|
+
});
|
|
7847
|
+
}
|
|
7848
|
+
async function completeWorkflowRunRecord(id, status, manifest) {
|
|
7849
|
+
await getDb().execute({
|
|
7850
|
+
sql: `UPDATE workflow_runs SET status = ?, manifest_json = ?, completed_at = datetime('now') WHERE id = ?`,
|
|
7851
|
+
args: [status, JSON.stringify(manifest), id]
|
|
7852
|
+
});
|
|
7853
|
+
}
|
|
7854
|
+
async function failWorkflowRunRecord(id, errorMessage4, manifest) {
|
|
7855
|
+
await getDb().execute({
|
|
7856
|
+
sql: `UPDATE workflow_runs SET status = 'failed', error_message = ?, manifest_json = ?, completed_at = datetime('now') WHERE id = ?`,
|
|
7857
|
+
args: [errorMessage4, manifest == null ? null : JSON.stringify(manifest), id]
|
|
7858
|
+
});
|
|
7859
|
+
}
|
|
7860
|
+
function workflowArtifactContentType(kind) {
|
|
7861
|
+
if (kind === "html_report") return "text/html; charset=utf-8";
|
|
7862
|
+
if (kind === "csv") return "text/csv; charset=utf-8";
|
|
7863
|
+
if (kind === "json") return "application/json; charset=utf-8";
|
|
7864
|
+
return "text/plain; charset=utf-8";
|
|
7865
|
+
}
|
|
7866
|
+
async function replaceWorkflowArtifacts(runId, userId, artifacts) {
|
|
7867
|
+
const db = getDb();
|
|
7868
|
+
await db.execute({ sql: "DELETE FROM workflow_artifacts WHERE run_id = ? AND user_id = ?", args: [runId, userId] });
|
|
7869
|
+
for (const artifact of artifacts) {
|
|
7870
|
+
await db.execute({
|
|
7871
|
+
sql: `
|
|
7872
|
+
INSERT INTO workflow_artifacts (id, run_id, user_id, kind, label, path, content_type, bytes, rows_count)
|
|
7873
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
7874
|
+
`,
|
|
7875
|
+
args: [
|
|
7876
|
+
(0, import_node_crypto.randomUUID)(),
|
|
7877
|
+
runId,
|
|
7878
|
+
userId,
|
|
7879
|
+
artifact.kind,
|
|
7880
|
+
artifact.label,
|
|
7881
|
+
artifact.path,
|
|
7882
|
+
workflowArtifactContentType(artifact.kind),
|
|
7883
|
+
artifact.bytes ?? null,
|
|
7884
|
+
artifact.rows ?? null
|
|
7885
|
+
]
|
|
7886
|
+
});
|
|
7887
|
+
}
|
|
7888
|
+
return listWorkflowArtifacts(runId, userId);
|
|
7889
|
+
}
|
|
7890
|
+
async function listWorkflowArtifacts(runId, userId) {
|
|
7891
|
+
const res = await getDb().execute({
|
|
7892
|
+
sql: "SELECT * FROM workflow_artifacts WHERE run_id = ? AND user_id = ? ORDER BY created_at ASC",
|
|
7893
|
+
args: [runId, userId]
|
|
7894
|
+
});
|
|
7895
|
+
return res.rows.map((r) => rowToWorkflowArtifact(r));
|
|
7896
|
+
}
|
|
7897
|
+
async function getWorkflowArtifact(id, userId) {
|
|
7898
|
+
const res = await getDb().execute({
|
|
7899
|
+
sql: "SELECT * FROM workflow_artifacts WHERE id = ? AND user_id = ? LIMIT 1",
|
|
7900
|
+
args: [id, userId]
|
|
7901
|
+
});
|
|
7902
|
+
return res.rows[0] ? rowToWorkflowArtifact(res.rows[0]) : null;
|
|
7903
|
+
}
|
|
7904
|
+
async function recordWorkflowWebhookDelivery(input) {
|
|
7905
|
+
await getDb().execute({
|
|
7906
|
+
sql: `
|
|
7907
|
+
INSERT INTO workflow_webhook_deliveries (
|
|
7908
|
+
id, run_id, schedule_id, user_id, webhook_url, status, attempts, last_error, delivered_at
|
|
7909
|
+
)
|
|
7910
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
7911
|
+
`,
|
|
7912
|
+
args: [
|
|
7913
|
+
(0, import_node_crypto.randomUUID)(),
|
|
7914
|
+
input.runId,
|
|
7915
|
+
input.scheduleId ?? null,
|
|
7916
|
+
input.userId,
|
|
7917
|
+
input.webhookUrl,
|
|
7918
|
+
input.status,
|
|
7919
|
+
input.attempts,
|
|
7920
|
+
input.lastError ?? null,
|
|
7921
|
+
input.status === "delivered" ? (/* @__PURE__ */ new Date()).toISOString() : null
|
|
7922
|
+
]
|
|
7923
|
+
});
|
|
7924
|
+
}
|
|
7554
7925
|
async function countActiveUsers() {
|
|
7555
7926
|
const res = await getDb().execute(`SELECT COUNT(*) as n FROM users WHERE active = 1`);
|
|
7556
7927
|
return Number(res.rows[0]?.n ?? 0);
|
|
@@ -8682,7 +9053,7 @@ function buildYouTubeChannelVideosUrl(channelInput) {
|
|
|
8682
9053
|
}
|
|
8683
9054
|
return `https://www.youtube.com/${handle}/videos`;
|
|
8684
9055
|
}
|
|
8685
|
-
var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright3, import_sdk4, DESKTOP_USER_AGENT, MOBILE_USER_AGENT, DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS, KERNEL_BROWSER_CLOSE_TIMEOUT_MS, KERNEL_SESSION_DELETE_TIMEOUT_MS, BrowserDriver;
|
|
9056
|
+
var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright3, import_sdk4, DESKTOP_USER_AGENT, MOBILE_USER_AGENT, DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS, KERNEL_BROWSER_CLOSE_TIMEOUT_MS, KERNEL_SESSION_DELETE_TIMEOUT_MS, DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS, SERP_CHALLENGE_SOLVE_POLL_MS, BrowserDriver;
|
|
8686
9057
|
var init_BrowserDriver = __esm({
|
|
8687
9058
|
"src/driver/BrowserDriver.ts"() {
|
|
8688
9059
|
"use strict";
|
|
@@ -8698,6 +9069,8 @@ var init_BrowserDriver = __esm({
|
|
|
8698
9069
|
DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
|
|
8699
9070
|
KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
|
|
8700
9071
|
KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
|
|
9072
|
+
DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS = 2500;
|
|
9073
|
+
SERP_CHALLENGE_SOLVE_POLL_MS = 500;
|
|
8701
9074
|
BrowserDriver = class {
|
|
8702
9075
|
browser = null;
|
|
8703
9076
|
context = null;
|
|
@@ -8947,20 +9320,20 @@ var init_BrowserDriver = __esm({
|
|
|
8947
9320
|
const diag = await this.captureDiagnostics(url);
|
|
8948
9321
|
throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
|
|
8949
9322
|
}
|
|
8950
|
-
|
|
8951
|
-
if (captchaCount > 0) {
|
|
9323
|
+
if (await this.isSerpChallengeVisible()) {
|
|
8952
9324
|
await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
|
|
8953
|
-
|
|
9325
|
+
const solved = await this.waitForSerpChallengeToClear(navDebug, url);
|
|
9326
|
+
if (!solved) throw new CaptchaError(this.captchaMessage());
|
|
8954
9327
|
}
|
|
8955
9328
|
const fastFound = await this.page.waitForSelector(PAASelectors.item, { timeout: 4e3 }).catch(() => null);
|
|
8956
9329
|
if (fastFound) {
|
|
8957
9330
|
await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: true, captchaDetected: false });
|
|
8958
9331
|
return { hasPaa: true };
|
|
8959
9332
|
}
|
|
8960
|
-
|
|
8961
|
-
if (captchaAfter > 0) {
|
|
9333
|
+
if (await this.isSerpChallengeVisible()) {
|
|
8962
9334
|
await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: true });
|
|
8963
|
-
|
|
9335
|
+
const solved = await this.waitForSerpChallengeToClear(navDebug, url);
|
|
9336
|
+
if (!solved) throw new CaptchaError(this.captchaMessage());
|
|
8964
9337
|
}
|
|
8965
9338
|
for (let i = 1; i <= 6; i++) {
|
|
8966
9339
|
await this.page.evaluate((f) => {
|
|
@@ -8976,6 +9349,35 @@ var init_BrowserDriver = __esm({
|
|
|
8976
9349
|
await this.updateSerpNavigationDebug(navDebug, url, { hasPaa: false, captchaDetected: false });
|
|
8977
9350
|
return { hasPaa: false };
|
|
8978
9351
|
}
|
|
9352
|
+
async isSerpChallengeVisible() {
|
|
9353
|
+
if (!this.page) return false;
|
|
9354
|
+
const captchaCount = await this.page.locator(PAASelectors.captchaMarker).count().catch(() => 0);
|
|
9355
|
+
if (captchaCount > 0) return true;
|
|
9356
|
+
const finalUrl = this.page.url();
|
|
9357
|
+
if (/google\.[^/]+\/sorry\//i.test(finalUrl)) return true;
|
|
9358
|
+
const bodySnippet = await this.page.evaluate(() => {
|
|
9359
|
+
const text = (document.body?.innerText ?? "").replace(/\s+/g, " ").trim();
|
|
9360
|
+
return text.slice(0, 1e3);
|
|
9361
|
+
}).catch(() => "");
|
|
9362
|
+
return /recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(bodySnippet);
|
|
9363
|
+
}
|
|
9364
|
+
async waitForSerpChallengeToClear(navDebug, requestedUrl) {
|
|
9365
|
+
if (!this.kernelClient || !this.page) return false;
|
|
9366
|
+
const waitMs = positiveIntFromEnv("SERP_CHALLENGE_SOLVE_WAIT_MS", DEFAULT_SERP_CHALLENGE_SOLVE_WAIT_MS);
|
|
9367
|
+
const deadline = Date.now() + waitMs;
|
|
9368
|
+
while (Date.now() < deadline) {
|
|
9369
|
+
await this.page.waitForTimeout(Math.min(SERP_CHALLENGE_SOLVE_POLL_MS, Math.max(250, deadline - Date.now())));
|
|
9370
|
+
if (!await this.isSerpChallengeVisible()) {
|
|
9371
|
+
await this.page.waitForTimeout(1e3);
|
|
9372
|
+
if (!await this.isSerpChallengeVisible()) {
|
|
9373
|
+
await this.updateSerpNavigationDebug(navDebug, requestedUrl, { hasPaa: null, captchaDetected: false });
|
|
9374
|
+
return true;
|
|
9375
|
+
}
|
|
9376
|
+
}
|
|
9377
|
+
}
|
|
9378
|
+
await this.updateSerpNavigationDebug(navDebug, requestedUrl, { hasPaa: false, captchaDetected: true });
|
|
9379
|
+
return false;
|
|
9380
|
+
}
|
|
8979
9381
|
async updateSerpNavigationDebug(navDebug, requestedUrl, state) {
|
|
8980
9382
|
if (!navDebug || !this.page) return;
|
|
8981
9383
|
try {
|
|
@@ -9023,7 +9425,12 @@ var init_BrowserDriver = __esm({
|
|
|
9023
9425
|
async navigateTo(url) {
|
|
9024
9426
|
try {
|
|
9025
9427
|
await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
9428
|
+
if (/google\.[^/]+\/search\?/i.test(url) && await this.isSerpChallengeVisible()) {
|
|
9429
|
+
const solved = await this.waitForSerpChallengeToClear(null, url);
|
|
9430
|
+
if (!solved) throw new CaptchaError(this.captchaMessage());
|
|
9431
|
+
}
|
|
9026
9432
|
} catch (err) {
|
|
9433
|
+
if (err instanceof CaptchaError) throw err;
|
|
9027
9434
|
const diag = await this.captureDiagnostics(url);
|
|
9028
9435
|
throw new ExtractionError(`page.goto failed: ${err.message} | ${diag}`);
|
|
9029
9436
|
}
|
|
@@ -10359,8 +10766,8 @@ var init_FacebookAdExtractor = __esm({
|
|
|
10359
10766
|
const pos = st.getPropertyValue("mask-position") || st.getPropertyValue("-webkit-mask-position") || st.getPropertyValue("background-position") || "";
|
|
10360
10767
|
const m = pos.match(/(-?\d+)px\s+(-?\d+)px/);
|
|
10361
10768
|
if (!m) continue;
|
|
10362
|
-
const
|
|
10363
|
-
if (
|
|
10769
|
+
const platform2 = sm[parseInt(m[2])];
|
|
10770
|
+
if (platform2 && !platforms2.includes(platform2)) platforms2.push(platform2);
|
|
10364
10771
|
}
|
|
10365
10772
|
for (const el2 of Array.from(root2.querySelectorAll("span,div"))) {
|
|
10366
10773
|
const txt = t(el2);
|
|
@@ -12584,24 +12991,215 @@ var init_MapsSearchExtractor = __esm({
|
|
|
12584
12991
|
}
|
|
12585
12992
|
});
|
|
12586
12993
|
|
|
12587
|
-
// src/
|
|
12588
|
-
function
|
|
12589
|
-
|
|
12590
|
-
|
|
12591
|
-
|
|
12592
|
-
|
|
12593
|
-
|
|
12594
|
-
|
|
12994
|
+
// src/maps-search-rotation.ts
|
|
12995
|
+
function mapsSearchMaxAttemptsForProxyMode(proxyMode) {
|
|
12996
|
+
return proxyMode === "location" ? MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS : MAPS_SEARCH_MAX_ATTEMPTS;
|
|
12997
|
+
}
|
|
12998
|
+
function mapsSearchErrorMessage(err) {
|
|
12999
|
+
return sanitizeVendorName(err instanceof Error ? err.message : String(err));
|
|
13000
|
+
}
|
|
13001
|
+
function looksLikeProxyTunnelFailure(message) {
|
|
13002
|
+
return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
|
|
13003
|
+
}
|
|
13004
|
+
function looksLikeProxyUnavailable(message) {
|
|
13005
|
+
return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
|
|
13006
|
+
}
|
|
13007
|
+
function looksLikeBrowserClosed(message) {
|
|
13008
|
+
return /Target page, context or browser has been closed|browser has been closed|context has been closed|page has been closed|browser not found|CDP session closed|Connection closed|WebSocket.*closed|Protocol error.*closed|Browser closed|disconnected/i.test(message);
|
|
13009
|
+
}
|
|
13010
|
+
function mapsSearchFailureOutcome(err) {
|
|
13011
|
+
if (err instanceof CaptchaError) return "captcha";
|
|
13012
|
+
const message = mapsSearchErrorMessage(err);
|
|
13013
|
+
if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return "timeout";
|
|
13014
|
+
if (looksLikeProxyTunnelFailure(message)) return "proxy_tunnel_failed";
|
|
13015
|
+
if (looksLikeProxyUnavailable(message)) return "proxy_unavailable";
|
|
13016
|
+
if (looksLikeBrowserClosed(message)) return "browser_closed";
|
|
13017
|
+
return "unknown_error";
|
|
13018
|
+
}
|
|
13019
|
+
function retryableMapsSearchError(err, proxyMode) {
|
|
13020
|
+
if (err instanceof CaptchaError) return true;
|
|
13021
|
+
const message = mapsSearchErrorMessage(err);
|
|
13022
|
+
if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return true;
|
|
13023
|
+
if (looksLikeBrowserClosed(message)) return true;
|
|
13024
|
+
return proxyMode === "location" && (looksLikeProxyTunnelFailure(message) || looksLikeProxyUnavailable(message));
|
|
13025
|
+
}
|
|
13026
|
+
function suffix(value) {
|
|
13027
|
+
return value ? value.slice(-8) : null;
|
|
13028
|
+
}
|
|
13029
|
+
function attemptFromDebug(input) {
|
|
13030
|
+
const network = input.debug.networkLocation;
|
|
13031
|
+
const browserSessionId = input.debug.kernel?.sessionId ?? null;
|
|
13032
|
+
const target = input.resolution?.target ?? null;
|
|
13033
|
+
return {
|
|
13034
|
+
attemptNumber: input.attemptIndex + 1,
|
|
13035
|
+
maxAttempts: input.maxAttempts,
|
|
13036
|
+
status: input.status,
|
|
13037
|
+
outcome: input.outcome,
|
|
13038
|
+
willRetry: input.willRetry,
|
|
13039
|
+
durationMs: input.durationMs,
|
|
13040
|
+
resultCount: input.resultCount,
|
|
13041
|
+
error: input.error,
|
|
13042
|
+
proxyMode: input.proxyMode,
|
|
13043
|
+
proxyResolutionSource: input.resolution?.source ?? null,
|
|
13044
|
+
proxyIdSuffix: input.resolution?.proxyIdSuffix ?? null,
|
|
13045
|
+
proxyTargetLevel: target?.level ?? null,
|
|
13046
|
+
proxyTargetLocation: target?.canonicalLocation ?? null,
|
|
13047
|
+
proxyTargetZip: target?.zip ?? null,
|
|
13048
|
+
browserSessionIdSuffix: suffix(browserSessionId),
|
|
13049
|
+
observedIp: network?.ip ?? null,
|
|
13050
|
+
observedCity: network?.city ?? null,
|
|
13051
|
+
observedRegion: network?.region ?? null
|
|
13052
|
+
};
|
|
12595
13053
|
}
|
|
12596
|
-
async function cleanupDisposableProxy(kernelApiKey, proxyId) {
|
|
13054
|
+
async function cleanupDisposableProxy(kernelApiKey, proxyId, eventName) {
|
|
12597
13055
|
if (!kernelApiKey || !proxyId) return;
|
|
12598
|
-
|
|
13056
|
+
try {
|
|
13057
|
+
await deleteKernelProxyId(kernelApiKey, proxyId);
|
|
13058
|
+
} catch (err) {
|
|
12599
13059
|
console.warn(JSON.stringify({
|
|
12600
|
-
event:
|
|
12601
|
-
proxy_id_suffix: proxyId
|
|
12602
|
-
message:
|
|
13060
|
+
event: eventName,
|
|
13061
|
+
proxy_id_suffix: suffix(proxyId),
|
|
13062
|
+
message: mapsSearchErrorMessage(err)
|
|
12603
13063
|
}));
|
|
12604
|
-
}
|
|
13064
|
+
}
|
|
13065
|
+
}
|
|
13066
|
+
async function runMapsSearchWithRotation(options) {
|
|
13067
|
+
const attempts = [];
|
|
13068
|
+
const maxAttempts = mapsSearchMaxAttemptsForProxyMode(options.proxyMode);
|
|
13069
|
+
const started = Date.now();
|
|
13070
|
+
let lastError = null;
|
|
13071
|
+
for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
|
|
13072
|
+
const attemptStarted = Date.now();
|
|
13073
|
+
const driver = new BrowserDriver();
|
|
13074
|
+
const extractor = new MapsSearchExtractor(driver);
|
|
13075
|
+
const proxyZip = options.proxyZipForAttempt?.(attemptIndex) ?? options.proxyZip;
|
|
13076
|
+
let resolution2 = null;
|
|
13077
|
+
let disposableProxyId;
|
|
13078
|
+
try {
|
|
13079
|
+
resolution2 = await resolveKernelProxyId({
|
|
13080
|
+
kernelApiKey: options.kernelApiKey,
|
|
13081
|
+
proxyMode: options.proxyMode,
|
|
13082
|
+
configuredKernelProxyId: options.kernelProxyId ?? options.configuredKernelProxyId,
|
|
13083
|
+
location: options.location,
|
|
13084
|
+
proxyZip,
|
|
13085
|
+
gl: options.gl,
|
|
13086
|
+
attemptIndex,
|
|
13087
|
+
fresh: options.proxyMode === "location"
|
|
13088
|
+
});
|
|
13089
|
+
disposableProxyId = resolution2.disposableProxyId;
|
|
13090
|
+
const result = await extractor.extract({
|
|
13091
|
+
...options,
|
|
13092
|
+
proxyZip,
|
|
13093
|
+
kernelProxyId: options.proxyMode === "none" ? void 0 : resolution2.kernelProxyId,
|
|
13094
|
+
kernelProxyResolution: resolution2.resolution
|
|
13095
|
+
});
|
|
13096
|
+
const attempt = attemptFromDebug({
|
|
13097
|
+
attemptIndex,
|
|
13098
|
+
maxAttempts,
|
|
13099
|
+
status: "ok",
|
|
13100
|
+
outcome: "success",
|
|
13101
|
+
willRetry: false,
|
|
13102
|
+
durationMs: Date.now() - attemptStarted,
|
|
13103
|
+
resultCount: result.resultCount,
|
|
13104
|
+
error: null,
|
|
13105
|
+
proxyMode: options.proxyMode,
|
|
13106
|
+
resolution: resolution2.resolution,
|
|
13107
|
+
debug: driver.getDebugSnapshot()
|
|
13108
|
+
});
|
|
13109
|
+
attempts.push(attempt);
|
|
13110
|
+
return {
|
|
13111
|
+
...result,
|
|
13112
|
+
attempts: attempts.slice(),
|
|
13113
|
+
durationMs: Date.now() - started
|
|
13114
|
+
};
|
|
13115
|
+
} catch (err) {
|
|
13116
|
+
lastError = err;
|
|
13117
|
+
const retryable = retryableMapsSearchError(err, options.proxyMode);
|
|
13118
|
+
const willRetry = attemptIndex < maxAttempts - 1 && retryable;
|
|
13119
|
+
const attempt = attemptFromDebug({
|
|
13120
|
+
attemptIndex,
|
|
13121
|
+
maxAttempts,
|
|
13122
|
+
status: "failed",
|
|
13123
|
+
outcome: mapsSearchFailureOutcome(err),
|
|
13124
|
+
willRetry,
|
|
13125
|
+
durationMs: Date.now() - attemptStarted,
|
|
13126
|
+
resultCount: 0,
|
|
13127
|
+
error: mapsSearchErrorMessage(err),
|
|
13128
|
+
proxyMode: options.proxyMode,
|
|
13129
|
+
resolution: resolution2?.resolution ?? null,
|
|
13130
|
+
debug: driver.getDebugSnapshot()
|
|
13131
|
+
});
|
|
13132
|
+
attempts.push(attempt);
|
|
13133
|
+
console.warn(JSON.stringify({
|
|
13134
|
+
event: options.failureEventName ?? "maps_search_attempt_failed",
|
|
13135
|
+
...options.logContext,
|
|
13136
|
+
attempt_number: attempt.attemptNumber,
|
|
13137
|
+
max_attempts: attempt.maxAttempts,
|
|
13138
|
+
outcome: attempt.outcome,
|
|
13139
|
+
will_retry: attempt.willRetry,
|
|
13140
|
+
proxy_resolution_source: attempt.proxyResolutionSource,
|
|
13141
|
+
proxy_id_suffix: attempt.proxyIdSuffix,
|
|
13142
|
+
browser_session_id_suffix: attempt.browserSessionIdSuffix,
|
|
13143
|
+
observed_city: attempt.observedCity,
|
|
13144
|
+
observed_region: attempt.observedRegion,
|
|
13145
|
+
message: attempt.error
|
|
13146
|
+
}));
|
|
13147
|
+
if (!willRetry) {
|
|
13148
|
+
throw new MapsSearchRotationError(
|
|
13149
|
+
mapsSearchErrorMessage(err),
|
|
13150
|
+
attempts,
|
|
13151
|
+
retryable,
|
|
13152
|
+
err instanceof CaptchaError ? "captcha_or_blocked" : "maps_search_failed"
|
|
13153
|
+
);
|
|
13154
|
+
}
|
|
13155
|
+
} finally {
|
|
13156
|
+
await cleanupDisposableProxy(options.kernelApiKey, disposableProxyId, "maps_search_proxy_delete_failed");
|
|
13157
|
+
await driver.close();
|
|
13158
|
+
}
|
|
13159
|
+
}
|
|
13160
|
+
throw new MapsSearchRotationError(
|
|
13161
|
+
lastError ? mapsSearchErrorMessage(lastError) : "Maps search failed",
|
|
13162
|
+
attempts,
|
|
13163
|
+
true
|
|
13164
|
+
);
|
|
13165
|
+
}
|
|
13166
|
+
var MAPS_SEARCH_MAX_ATTEMPTS, MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS, MapsSearchRotationError;
|
|
13167
|
+
var init_maps_search_rotation = __esm({
|
|
13168
|
+
"src/maps-search-rotation.ts"() {
|
|
13169
|
+
"use strict";
|
|
13170
|
+
init_BrowserDriver();
|
|
13171
|
+
init_MapsSearchExtractor();
|
|
13172
|
+
init_errors();
|
|
13173
|
+
init_kernel_proxy_resolver();
|
|
13174
|
+
MAPS_SEARCH_MAX_ATTEMPTS = 3;
|
|
13175
|
+
MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS = 5;
|
|
13176
|
+
MapsSearchRotationError = class extends Error {
|
|
13177
|
+
constructor(message, attempts, retryable, errorCode = "maps_search_failed") {
|
|
13178
|
+
super(message);
|
|
13179
|
+
this.attempts = attempts;
|
|
13180
|
+
this.retryable = retryable;
|
|
13181
|
+
this.errorCode = errorCode;
|
|
13182
|
+
}
|
|
13183
|
+
attempts;
|
|
13184
|
+
retryable;
|
|
13185
|
+
errorCode;
|
|
13186
|
+
name = "MapsSearchRotationError";
|
|
13187
|
+
};
|
|
13188
|
+
}
|
|
13189
|
+
});
|
|
13190
|
+
|
|
13191
|
+
// src/api/maps-routes.ts
|
|
13192
|
+
function mapsErrorResponse(c, err, errorCode) {
|
|
13193
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
13194
|
+
const rotationError = err instanceof MapsSearchRotationError ? err : null;
|
|
13195
|
+
const blocked = msg.includes("CAPTCHA") || msg.includes("blocked");
|
|
13196
|
+
const retryable = rotationError?.retryable ?? blocked;
|
|
13197
|
+
return c.json({
|
|
13198
|
+
error: sanitizeVendorName(msg),
|
|
13199
|
+
error_code: rotationError?.errorCode ?? (blocked ? "captcha_or_blocked" : errorCode),
|
|
13200
|
+
retryable,
|
|
13201
|
+
attempts: rotationError?.attempts ?? void 0
|
|
13202
|
+
}, retryable ? 503 : 500);
|
|
12605
13203
|
}
|
|
12606
13204
|
var import_hono5, mapsApp;
|
|
12607
13205
|
var init_maps_routes = __esm({
|
|
@@ -12611,13 +13209,12 @@ var init_maps_routes = __esm({
|
|
|
12611
13209
|
init_db();
|
|
12612
13210
|
init_rates();
|
|
12613
13211
|
init_MapsExtractor();
|
|
12614
|
-
init_MapsSearchExtractor();
|
|
12615
13212
|
init_BrowserDriver();
|
|
12616
13213
|
init_schemas3();
|
|
12617
13214
|
init_api_auth();
|
|
12618
13215
|
init_errors();
|
|
12619
13216
|
init_browser_service_env();
|
|
12620
|
-
|
|
13217
|
+
init_maps_search_rotation();
|
|
12621
13218
|
mapsApp = new import_hono5.Hono();
|
|
12622
13219
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
12623
13220
|
const user = c.get("user");
|
|
@@ -12636,24 +13233,10 @@ var init_maps_routes = __esm({
|
|
|
12636
13233
|
[parsed.data.query, parsed.data.location].filter(Boolean).join(" ")
|
|
12637
13234
|
);
|
|
12638
13235
|
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.maps_search), 402);
|
|
12639
|
-
const driver = new BrowserDriver();
|
|
12640
|
-
const extractor = new MapsSearchExtractor(driver);
|
|
12641
|
-
let disposableProxyId;
|
|
12642
13236
|
try {
|
|
12643
|
-
const
|
|
12644
|
-
kernelApiKey: parsed.data.kernelApiKey,
|
|
12645
|
-
proxyMode: parsed.data.proxyMode,
|
|
12646
|
-
configuredKernelProxyId: browserServiceProxyId(),
|
|
12647
|
-
location: parsed.data.location,
|
|
12648
|
-
proxyZip: parsed.data.proxyZip,
|
|
12649
|
-
gl: parsed.data.gl,
|
|
12650
|
-
fresh: parsed.data.proxyMode === "location"
|
|
12651
|
-
});
|
|
12652
|
-
disposableProxyId = resolution2.disposableProxyId;
|
|
12653
|
-
const result = await extractor.extract({
|
|
13237
|
+
const result = await runMapsSearchWithRotation({
|
|
12654
13238
|
...parsed.data,
|
|
12655
|
-
|
|
12656
|
-
kernelProxyResolution: resolution2.resolution
|
|
13239
|
+
configuredKernelProxyId: browserServiceProxyId()
|
|
12657
13240
|
});
|
|
12658
13241
|
await logRequestEvent({
|
|
12659
13242
|
userId: user.id,
|
|
@@ -12674,12 +13257,10 @@ var init_maps_routes = __esm({
|
|
|
12674
13257
|
status: "failed",
|
|
12675
13258
|
query: [parsed.data.query, parsed.data.location].filter(Boolean).join(" "),
|
|
12676
13259
|
location: parsed.data.location,
|
|
12677
|
-
error: msg
|
|
13260
|
+
error: msg,
|
|
13261
|
+
result: err instanceof MapsSearchRotationError ? { attempts: err.attempts } : void 0
|
|
12678
13262
|
});
|
|
12679
|
-
return mapsErrorResponse(c,
|
|
12680
|
-
} finally {
|
|
12681
|
-
await cleanupDisposableProxy(parsed.data.kernelApiKey, disposableProxyId);
|
|
12682
|
-
await driver.close();
|
|
13263
|
+
return mapsErrorResponse(c, err, "maps_search_failed");
|
|
12683
13264
|
}
|
|
12684
13265
|
});
|
|
12685
13266
|
mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
@@ -12886,12 +13467,17 @@ function errorAttemptsSection(body) {
|
|
|
12886
13467
|
const browser = debug.browser ?? {};
|
|
12887
13468
|
const kernel = browser.browserRuntime ?? browser.kernel ?? {};
|
|
12888
13469
|
const proxyResolution = kernel.proxyResolution ?? {};
|
|
12889
|
-
const network = browser.networkLocation ?? {
|
|
13470
|
+
const network = browser.networkLocation ?? {
|
|
13471
|
+
ip: attempt.observedIp ?? attempt.observed_ip,
|
|
13472
|
+
city: attempt.observedCity ?? attempt.observed_city,
|
|
13473
|
+
region: attempt.observedRegion ?? attempt.observed_region
|
|
13474
|
+
};
|
|
12890
13475
|
const nav = browser.serpNavigation ?? {};
|
|
12891
13476
|
const geo = [network.ip, network.city, network.region].filter(Boolean).join(" / ") || "geo unknown";
|
|
12892
|
-
const sessionId = attempt.browser_session_id ?? attempt.kernel_session_id ?? kernel.sessionId ?? "unknown";
|
|
13477
|
+
const sessionId = attempt.browser_session_id ?? attempt.browserSessionIdSuffix ?? attempt.kernel_session_id ?? kernel.sessionId ?? "unknown";
|
|
12893
13478
|
const cleanupSucceeded2 = attempt.session_cleanup_succeeded ?? attempt.kernel_delete_succeeded;
|
|
12894
|
-
|
|
13479
|
+
const proxySource = proxyResolution.source ?? attempt.proxyResolutionSource;
|
|
13480
|
+
return `- Attempt ${attempt.attempt_number ?? attempt.attemptNumber ?? "?"}: ${attempt.outcome ?? attempt.status ?? "unknown"} \xB7 session ${sessionId} \xB7 proxy ${debug.request?.proxyMode ?? kernel.proxyMode ?? attempt.proxyMode ?? "unknown"}${proxySource ? `/${proxySource}` : ""} \xB7 ${geo} \xB7 CAPTCHA ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"} \xB7 cleanup ${cleanupSucceeded2 === true ? "yes" : cleanupSucceeded2 === false ? "no" : "unknown"}`;
|
|
12895
13481
|
});
|
|
12896
13482
|
return `
|
|
12897
13483
|
|
|
@@ -13352,6 +13938,29 @@ ${rows}`,
|
|
|
13352
13938
|
}
|
|
13353
13939
|
};
|
|
13354
13940
|
}
|
|
13941
|
+
function normalizeMapsAttempts(value) {
|
|
13942
|
+
const attempts = Array.isArray(value) ? value : [];
|
|
13943
|
+
return attempts.map((attempt, index) => ({
|
|
13944
|
+
attemptNumber: attempt.attemptNumber ?? attempt.attempt_number ?? index + 1,
|
|
13945
|
+
maxAttempts: attempt.maxAttempts ?? attempt.max_attempts ?? attempts.length,
|
|
13946
|
+
status: attempt.status === "ok" ? "ok" : "failed",
|
|
13947
|
+
outcome: attempt.outcome ?? attempt.status ?? "unknown",
|
|
13948
|
+
willRetry: attempt.willRetry ?? attempt.will_retry ?? false,
|
|
13949
|
+
durationMs: attempt.durationMs ?? attempt.duration_ms ?? 0,
|
|
13950
|
+
resultCount: attempt.resultCount ?? attempt.result_count ?? 0,
|
|
13951
|
+
error: attempt.error ? sanitizeVendorText(attempt.error) : null,
|
|
13952
|
+
proxyMode: attempt.proxyMode ?? attempt.proxy_mode ?? "location",
|
|
13953
|
+
proxyResolutionSource: attempt.proxyResolutionSource ?? attempt.proxy_resolution_source ?? null,
|
|
13954
|
+
proxyIdSuffix: attempt.proxyIdSuffix ?? attempt.proxy_id_suffix ?? null,
|
|
13955
|
+
proxyTargetLevel: attempt.proxyTargetLevel ?? attempt.proxy_target_level ?? null,
|
|
13956
|
+
proxyTargetLocation: attempt.proxyTargetLocation ?? attempt.proxy_target_location ?? null,
|
|
13957
|
+
proxyTargetZip: attempt.proxyTargetZip ?? attempt.proxy_target_zip ?? null,
|
|
13958
|
+
browserSessionIdSuffix: attempt.browserSessionIdSuffix ?? attempt.browser_session_id ?? null,
|
|
13959
|
+
observedIp: attempt.observedIp ?? attempt.observed_ip ?? null,
|
|
13960
|
+
observedCity: attempt.observedCity ?? attempt.observed_city ?? null,
|
|
13961
|
+
observedRegion: attempt.observedRegion ?? attempt.observed_region ?? null
|
|
13962
|
+
}));
|
|
13963
|
+
}
|
|
13355
13964
|
function formatCreditsInfo(raw, input) {
|
|
13356
13965
|
const parsed = parseData(raw);
|
|
13357
13966
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
@@ -13424,6 +14033,8 @@ function formatMapsSearch(raw, input) {
|
|
|
13424
14033
|
const searchQuery = d.searchQuery ?? [input.query, input.location].filter(Boolean).join(" ");
|
|
13425
14034
|
const requestedMax = d.requestedMaxResults ?? input.maxResults ?? 10;
|
|
13426
14035
|
const durationMs = d.durationMs;
|
|
14036
|
+
const attempts = normalizeMapsAttempts(d.attempts);
|
|
14037
|
+
const lastAttempt = attempts.at(-1);
|
|
13427
14038
|
const rows = results.map((r) => {
|
|
13428
14039
|
const rating = [r.rating, r.reviewCount ? `(${r.reviewCount})` : null].filter(Boolean).join(" ");
|
|
13429
14040
|
return `| ${r.position} | ${cell(r.name)} | ${cell(r.category)} | ${cell(rating)} | ${cell(r.address)} | ${r.cidDecimal ? `\`${r.cidDecimal}\`` : "\u2014"} | ${r.websiteUrl ? `[site](${r.websiteUrl})` : "\u2014"} | [maps](${r.placeUrl}) |`;
|
|
@@ -13438,6 +14049,7 @@ ${meta}`;
|
|
|
13438
14049
|
const full = [
|
|
13439
14050
|
`# Google Maps Search: "${searchQuery}"`,
|
|
13440
14051
|
`**Returned:** ${results.length} profile candidate${results.length === 1 ? "" : "s"} \xB7 **Requested max:** ${requestedMax} \xB7 **Limit:** 50`,
|
|
14052
|
+
attempts.length ? `**Attempts:** ${attempts.length}/${lastAttempt?.maxAttempts ?? attempts.length} \xB7 **Proxy:** ${lastAttempt?.proxyMode ?? "unknown"}${lastAttempt?.proxyResolutionSource ? `/${lastAttempt.proxyResolutionSource}` : ""} \xB7 **Observed:** ${[lastAttempt?.observedCity, lastAttempt?.observedRegion].filter(Boolean).join(", ") || "unknown"}` : null,
|
|
13441
14053
|
`
|
|
13442
14054
|
## Results
|
|
13443
14055
|
| # | Name | Category | Rating | Address | CID | Website | Maps |
|
|
@@ -13461,6 +14073,7 @@ ${rows}`,
|
|
|
13461
14073
|
requestedMaxResults: requestedMax,
|
|
13462
14074
|
resultCount: results.length,
|
|
13463
14075
|
results: normalizedResults,
|
|
14076
|
+
attempts,
|
|
13464
14077
|
durationMs: durationMs ?? 0
|
|
13465
14078
|
}
|
|
13466
14079
|
};
|
|
@@ -13471,6 +14084,7 @@ function formatDirectoryWorkflow(raw, input) {
|
|
|
13471
14084
|
const d = parsed.data;
|
|
13472
14085
|
const cities = (d.cities ?? []).map((city) => ({
|
|
13473
14086
|
...city,
|
|
14087
|
+
attempts: normalizeMapsAttempts(city.attempts),
|
|
13474
14088
|
results: city.results.map((result) => ({
|
|
13475
14089
|
...result,
|
|
13476
14090
|
phone: result.phone ?? null,
|
|
@@ -13934,36 +14548,9 @@ var init_location_db = __esm({
|
|
|
13934
14548
|
});
|
|
13935
14549
|
|
|
13936
14550
|
// src/directory/directory-workflow.ts
|
|
13937
|
-
async function cleanupDisposableProxy2(kernelApiKey, proxyId) {
|
|
13938
|
-
if (!kernelApiKey || !proxyId) return;
|
|
13939
|
-
try {
|
|
13940
|
-
await deleteKernelProxyId(kernelApiKey, proxyId);
|
|
13941
|
-
} catch (err) {
|
|
13942
|
-
console.warn(JSON.stringify({
|
|
13943
|
-
event: "directory_workflow_proxy_delete_failed",
|
|
13944
|
-
proxy_id_suffix: proxyId.slice(-6),
|
|
13945
|
-
message: err instanceof Error ? err.message : String(err)
|
|
13946
|
-
}));
|
|
13947
|
-
}
|
|
13948
|
-
}
|
|
13949
|
-
function maxAttemptsForProxyMode(proxyMode) {
|
|
13950
|
-
return proxyMode === "location" ? DIRECTORY_LOCATION_PROXY_MAX_ATTEMPTS : DIRECTORY_MAX_ATTEMPTS;
|
|
13951
|
-
}
|
|
13952
14551
|
function errorMessage(err) {
|
|
13953
14552
|
return err instanceof Error ? err.message : String(err);
|
|
13954
14553
|
}
|
|
13955
|
-
function looksLikeProxyTunnelFailure(message) {
|
|
13956
|
-
return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message);
|
|
13957
|
-
}
|
|
13958
|
-
function looksLikeProxyUnavailable(message) {
|
|
13959
|
-
return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
|
|
13960
|
-
}
|
|
13961
|
-
function retryableCitySearchError(err, proxyMode) {
|
|
13962
|
-
if (err instanceof CaptchaError) return true;
|
|
13963
|
-
const message = errorMessage(err);
|
|
13964
|
-
if (/timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message)) return true;
|
|
13965
|
-
return proxyMode === "location" && (looksLikeProxyTunnelFailure(message) || looksLikeProxyUnavailable(message));
|
|
13966
|
-
}
|
|
13967
14554
|
function proxyZipForAttempt(options, market, attemptIndex) {
|
|
13968
14555
|
if (options.proxyZip) return options.proxyZip;
|
|
13969
14556
|
if (!market.zips.length) return void 0;
|
|
@@ -13982,25 +14569,10 @@ async function mapLimit(items, limit, fn) {
|
|
|
13982
14569
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
13983
14570
|
return out;
|
|
13984
14571
|
}
|
|
13985
|
-
async function
|
|
13986
|
-
const
|
|
13987
|
-
|
|
13988
|
-
|
|
13989
|
-
let disposableProxyId;
|
|
13990
|
-
try {
|
|
13991
|
-
const proxyZip = proxyZipForAttempt(options, market, attemptIndex);
|
|
13992
|
-
const resolution2 = await resolveKernelProxyId({
|
|
13993
|
-
kernelApiKey: options.kernelApiKey,
|
|
13994
|
-
proxyMode: options.proxyMode,
|
|
13995
|
-
configuredKernelProxyId: browserServiceProxyId(),
|
|
13996
|
-
location: market.location,
|
|
13997
|
-
proxyZip,
|
|
13998
|
-
gl: options.gl,
|
|
13999
|
-
attemptIndex,
|
|
14000
|
-
fresh: options.proxyMode === "location"
|
|
14001
|
-
});
|
|
14002
|
-
disposableProxyId = resolution2.disposableProxyId;
|
|
14003
|
-
const result = await extractor.extract({
|
|
14572
|
+
async function searchCity(options, market) {
|
|
14573
|
+
const started = Date.now();
|
|
14574
|
+
try {
|
|
14575
|
+
const result = await runMapsSearchWithRotation({
|
|
14004
14576
|
query: options.query,
|
|
14005
14577
|
location: market.location,
|
|
14006
14578
|
gl: options.gl,
|
|
@@ -14008,11 +14580,16 @@ async function searchCityAttempt(options, market, attemptIndex) {
|
|
|
14008
14580
|
maxResults: options.maxResultsPerCity,
|
|
14009
14581
|
headless: options.headless,
|
|
14010
14582
|
kernelApiKey: options.kernelApiKey,
|
|
14011
|
-
kernelProxyId: options.proxyMode === "none" ? void 0 : resolution2.kernelProxyId,
|
|
14012
|
-
kernelProxyResolution: resolution2.resolution,
|
|
14013
14583
|
proxyMode: options.proxyMode,
|
|
14014
|
-
proxyZip,
|
|
14015
|
-
debug: options.debug
|
|
14584
|
+
proxyZip: options.proxyZip,
|
|
14585
|
+
debug: options.debug,
|
|
14586
|
+
configuredKernelProxyId: browserServiceProxyId(),
|
|
14587
|
+
proxyZipForAttempt: (attemptIndex) => proxyZipForAttempt(options, market, attemptIndex),
|
|
14588
|
+
failureEventName: "directory_workflow_city_attempt_failed",
|
|
14589
|
+
logContext: {
|
|
14590
|
+
city: market.city,
|
|
14591
|
+
state: market.state
|
|
14592
|
+
}
|
|
14016
14593
|
});
|
|
14017
14594
|
return {
|
|
14018
14595
|
city: market.city,
|
|
@@ -14028,50 +14605,29 @@ async function searchCityAttempt(options, market, attemptIndex) {
|
|
|
14028
14605
|
error: null,
|
|
14029
14606
|
resultCount: result.resultCount,
|
|
14030
14607
|
durationMs: result.durationMs,
|
|
14608
|
+
attempts: result.attempts ?? [],
|
|
14031
14609
|
results: result.results
|
|
14032
14610
|
};
|
|
14033
|
-
}
|
|
14034
|
-
|
|
14035
|
-
|
|
14036
|
-
|
|
14037
|
-
|
|
14038
|
-
|
|
14039
|
-
|
|
14040
|
-
|
|
14041
|
-
|
|
14042
|
-
|
|
14043
|
-
|
|
14044
|
-
|
|
14045
|
-
|
|
14046
|
-
|
|
14047
|
-
|
|
14048
|
-
|
|
14049
|
-
|
|
14050
|
-
|
|
14051
|
-
|
|
14052
|
-
max_attempts: maxAttempts,
|
|
14053
|
-
will_retry: willRetry,
|
|
14054
|
-
message: errorMessage(err)
|
|
14055
|
-
}));
|
|
14056
|
-
if (!willRetry) break;
|
|
14057
|
-
}
|
|
14611
|
+
} catch (err) {
|
|
14612
|
+
const attempts = err instanceof MapsSearchRotationError ? err.attempts : [];
|
|
14613
|
+
return {
|
|
14614
|
+
city: market.city,
|
|
14615
|
+
state: market.state,
|
|
14616
|
+
location: market.location,
|
|
14617
|
+
cityKey: market.cityKey,
|
|
14618
|
+
censusName: market.censusName,
|
|
14619
|
+
population: market.population,
|
|
14620
|
+
populationYear: market.populationYear,
|
|
14621
|
+
zips: market.zips,
|
|
14622
|
+
counties: market.counties,
|
|
14623
|
+
status: "failed",
|
|
14624
|
+
error: errorMessage(err),
|
|
14625
|
+
resultCount: 0,
|
|
14626
|
+
durationMs: Date.now() - started,
|
|
14627
|
+
attempts,
|
|
14628
|
+
results: []
|
|
14629
|
+
};
|
|
14058
14630
|
}
|
|
14059
|
-
return {
|
|
14060
|
-
city: market.city,
|
|
14061
|
-
state: market.state,
|
|
14062
|
-
location: market.location,
|
|
14063
|
-
cityKey: market.cityKey,
|
|
14064
|
-
censusName: market.censusName,
|
|
14065
|
-
population: market.population,
|
|
14066
|
-
populationYear: market.populationYear,
|
|
14067
|
-
zips: market.zips,
|
|
14068
|
-
counties: market.counties,
|
|
14069
|
-
status: "failed",
|
|
14070
|
-
error: lastError ? errorMessage(lastError) : "City Maps search failed",
|
|
14071
|
-
resultCount: 0,
|
|
14072
|
-
durationMs: Date.now() - started,
|
|
14073
|
-
results: []
|
|
14074
|
-
};
|
|
14075
14631
|
}
|
|
14076
14632
|
function csvRowsFor(result) {
|
|
14077
14633
|
const rows = [];
|
|
@@ -14092,6 +14648,7 @@ function csvRowsFor(result) {
|
|
|
14092
14648
|
result_position: null,
|
|
14093
14649
|
business_name: null,
|
|
14094
14650
|
review_stars: null,
|
|
14651
|
+
review_count: null,
|
|
14095
14652
|
category: null,
|
|
14096
14653
|
address: null,
|
|
14097
14654
|
phone: null,
|
|
@@ -14125,6 +14682,7 @@ function csvRowsFor(result) {
|
|
|
14125
14682
|
result_position: business.position,
|
|
14126
14683
|
business_name: business.name,
|
|
14127
14684
|
review_stars: business.rating,
|
|
14685
|
+
review_count: business.reviewCount,
|
|
14128
14686
|
category: business.category,
|
|
14129
14687
|
address: business.address,
|
|
14130
14688
|
phone: business.phone,
|
|
@@ -14165,6 +14723,7 @@ async function saveDirectoryCsv(result) {
|
|
|
14165
14723
|
"result_position",
|
|
14166
14724
|
"business_name",
|
|
14167
14725
|
"review_stars",
|
|
14726
|
+
"review_count",
|
|
14168
14727
|
"category",
|
|
14169
14728
|
"address",
|
|
14170
14729
|
"phone",
|
|
@@ -14206,23 +14765,18 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
14206
14765
|
const csvPath = options.saveCsv ? await saveDirectoryCsv(base) : null;
|
|
14207
14766
|
return { ...base, csvPath };
|
|
14208
14767
|
}
|
|
14209
|
-
var import_promises5, import_node_path6, import_zod18,
|
|
14768
|
+
var import_promises5, import_node_path6, import_zod18, DirectoryWorkflowOptionsSchema;
|
|
14210
14769
|
var init_directory_workflow = __esm({
|
|
14211
14770
|
"src/directory/directory-workflow.ts"() {
|
|
14212
14771
|
"use strict";
|
|
14213
14772
|
import_promises5 = require("fs/promises");
|
|
14214
14773
|
import_node_path6 = require("path");
|
|
14215
14774
|
import_zod18 = require("zod");
|
|
14216
|
-
init_MapsSearchExtractor();
|
|
14217
|
-
init_BrowserDriver();
|
|
14218
14775
|
init_mcp_response_formatter();
|
|
14219
14776
|
init_browser_service_env();
|
|
14220
|
-
|
|
14221
|
-
init_errors();
|
|
14777
|
+
init_maps_search_rotation();
|
|
14222
14778
|
init_csv();
|
|
14223
14779
|
init_location_db();
|
|
14224
|
-
DIRECTORY_MAX_ATTEMPTS = 3;
|
|
14225
|
-
DIRECTORY_LOCATION_PROXY_MAX_ATTEMPTS = 5;
|
|
14226
14780
|
DirectoryWorkflowOptionsSchema = import_zod18.z.object({
|
|
14227
14781
|
query: import_zod18.z.string().min(1),
|
|
14228
14782
|
state: import_zod18.z.string().min(2).default("TN"),
|
|
@@ -14318,10 +14872,1200 @@ var init_directory_routes = __esm({
|
|
|
14318
14872
|
}
|
|
14319
14873
|
});
|
|
14320
14874
|
|
|
14875
|
+
// src/lib/slugify.ts
|
|
14876
|
+
function slugify(s) {
|
|
14877
|
+
return s.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
|
|
14878
|
+
}
|
|
14879
|
+
var init_slugify = __esm({
|
|
14880
|
+
"src/lib/slugify.ts"() {
|
|
14881
|
+
"use strict";
|
|
14882
|
+
}
|
|
14883
|
+
});
|
|
14884
|
+
|
|
14885
|
+
// src/workflows/artifact-writer.ts
|
|
14886
|
+
function workflowOutputBaseDir(outputDir) {
|
|
14887
|
+
return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path7.join)((0, import_node_os4.homedir)(), "Downloads", "mcp-scraper");
|
|
14888
|
+
}
|
|
14889
|
+
function timestamp() {
|
|
14890
|
+
return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
14891
|
+
}
|
|
14892
|
+
function safeSlug(value) {
|
|
14893
|
+
return slugify(value).replace(/^-+|-+$/g, "").slice(0, 80) || "run";
|
|
14894
|
+
}
|
|
14895
|
+
function indexPath(baseDir = workflowOutputBaseDir()) {
|
|
14896
|
+
return (0, import_node_path7.join)(baseDir, "workflows", "index.json");
|
|
14897
|
+
}
|
|
14898
|
+
async function readIndex(baseDir) {
|
|
14899
|
+
const path6 = indexPath(baseDir);
|
|
14900
|
+
if (!(0, import_node_fs4.existsSync)(path6)) return { runs: [] };
|
|
14901
|
+
try {
|
|
14902
|
+
return JSON.parse(await (0, import_promises6.readFile)(path6, "utf8"));
|
|
14903
|
+
} catch {
|
|
14904
|
+
return { runs: [] };
|
|
14905
|
+
}
|
|
14906
|
+
}
|
|
14907
|
+
async function updateWorkflowIndex(manifest, manifestPath, baseDir) {
|
|
14908
|
+
if (manifest.status === "running") return;
|
|
14909
|
+
const path6 = indexPath(baseDir);
|
|
14910
|
+
const index = await readIndex(baseDir);
|
|
14911
|
+
const report = manifest.artifacts.find((a) => a.kind === "html_report")?.path ?? null;
|
|
14912
|
+
const entry = {
|
|
14913
|
+
workflow: manifest.workflow,
|
|
14914
|
+
runId: manifest.runId,
|
|
14915
|
+
status: manifest.status,
|
|
14916
|
+
startedAt: manifest.startedAt,
|
|
14917
|
+
reportPath: report,
|
|
14918
|
+
manifestPath,
|
|
14919
|
+
summary: `${manifest.title} (${manifest.status})`
|
|
14920
|
+
};
|
|
14921
|
+
index.runs = [entry, ...index.runs.filter((r) => r.runId !== manifest.runId)].slice(0, 200);
|
|
14922
|
+
await (0, import_promises6.mkdir)((0, import_node_path7.dirname)(path6), { recursive: true });
|
|
14923
|
+
await (0, import_promises6.writeFile)(path6, JSON.stringify(index, null, 2), "utf8");
|
|
14924
|
+
}
|
|
14925
|
+
var import_promises6, import_node_fs4, import_node_os4, import_node_path7, import_node_child_process2, ArtifactWriter;
|
|
14926
|
+
var init_artifact_writer = __esm({
|
|
14927
|
+
"src/workflows/artifact-writer.ts"() {
|
|
14928
|
+
"use strict";
|
|
14929
|
+
import_promises6 = require("fs/promises");
|
|
14930
|
+
import_node_fs4 = require("fs");
|
|
14931
|
+
import_node_os4 = require("os");
|
|
14932
|
+
import_node_path7 = require("path");
|
|
14933
|
+
import_node_child_process2 = require("child_process");
|
|
14934
|
+
init_csv();
|
|
14935
|
+
init_slugify();
|
|
14936
|
+
ArtifactWriter = class _ArtifactWriter {
|
|
14937
|
+
constructor(workflowId, title, runId, baseDir, runDir, startedAt, input) {
|
|
14938
|
+
this.workflowId = workflowId;
|
|
14939
|
+
this.title = title;
|
|
14940
|
+
this.runId = runId;
|
|
14941
|
+
this.baseDir = baseDir;
|
|
14942
|
+
this.runDir = runDir;
|
|
14943
|
+
this.startedAt = startedAt;
|
|
14944
|
+
this.input = input;
|
|
14945
|
+
}
|
|
14946
|
+
workflowId;
|
|
14947
|
+
title;
|
|
14948
|
+
runId;
|
|
14949
|
+
baseDir;
|
|
14950
|
+
runDir;
|
|
14951
|
+
startedAt;
|
|
14952
|
+
input;
|
|
14953
|
+
artifacts = [];
|
|
14954
|
+
static async create(workflowId, title, input, outputDir, forcedRunId) {
|
|
14955
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
14956
|
+
const runId = forcedRunId?.trim() || `${timestamp()}-${safeSlug(workflowId)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
14957
|
+
const nameSource = String(input.keyword ?? input.query ?? input.domain ?? input.state ?? workflowId);
|
|
14958
|
+
const baseDir = workflowOutputBaseDir(outputDir);
|
|
14959
|
+
const runDir = (0, import_node_path7.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
|
|
14960
|
+
await (0, import_promises6.mkdir)(runDir, { recursive: true });
|
|
14961
|
+
return new _ArtifactWriter(workflowId, title, runId, baseDir, runDir, startedAt, input);
|
|
14962
|
+
}
|
|
14963
|
+
async remember(kind, label, path6, rows) {
|
|
14964
|
+
const size = await (0, import_promises6.stat)(path6).then((s) => s.size).catch(() => void 0);
|
|
14965
|
+
this.artifacts.push({ kind, label, path: path6, bytes: size, rows });
|
|
14966
|
+
return path6;
|
|
14967
|
+
}
|
|
14968
|
+
async writeJson(label, relativePath, data) {
|
|
14969
|
+
const path6 = (0, import_node_path7.join)(this.runDir, relativePath);
|
|
14970
|
+
await (0, import_promises6.mkdir)((0, import_node_path7.dirname)(path6), { recursive: true });
|
|
14971
|
+
await (0, import_promises6.writeFile)(path6, JSON.stringify(data, null, 2), "utf8");
|
|
14972
|
+
return this.remember("json", label, path6);
|
|
14973
|
+
}
|
|
14974
|
+
async writeText(label, relativePath, text, kind = "markdown") {
|
|
14975
|
+
const path6 = (0, import_node_path7.join)(this.runDir, relativePath);
|
|
14976
|
+
await (0, import_promises6.mkdir)((0, import_node_path7.dirname)(path6), { recursive: true });
|
|
14977
|
+
await (0, import_promises6.writeFile)(path6, text, "utf8");
|
|
14978
|
+
return this.remember(kind, label, path6);
|
|
14979
|
+
}
|
|
14980
|
+
async writeCsv(label, relativePath, headers, rows) {
|
|
14981
|
+
const path6 = (0, import_node_path7.join)(this.runDir, relativePath);
|
|
14982
|
+
await (0, import_promises6.mkdir)((0, import_node_path7.dirname)(path6), { recursive: true });
|
|
14983
|
+
await (0, import_promises6.writeFile)(path6, rowsToCsv(headers, rows), "utf8");
|
|
14984
|
+
return this.remember("csv", label, path6, rows.length);
|
|
14985
|
+
}
|
|
14986
|
+
async writeHtml(label, relativePath, html) {
|
|
14987
|
+
return this.writeText(label, relativePath, html, "html_report");
|
|
14988
|
+
}
|
|
14989
|
+
async writeManifest(status, counts, warnings, errors) {
|
|
14990
|
+
const manifest = {
|
|
14991
|
+
workflow: this.workflowId,
|
|
14992
|
+
title: this.title,
|
|
14993
|
+
runId: this.runId,
|
|
14994
|
+
status,
|
|
14995
|
+
startedAt: this.startedAt,
|
|
14996
|
+
completedAt: status === "running" ? null : (/* @__PURE__ */ new Date()).toISOString(),
|
|
14997
|
+
input: this.input,
|
|
14998
|
+
artifacts: this.artifacts,
|
|
14999
|
+
warnings,
|
|
15000
|
+
errors,
|
|
15001
|
+
counts
|
|
15002
|
+
};
|
|
15003
|
+
const path6 = (0, import_node_path7.join)(this.runDir, "manifest.json");
|
|
15004
|
+
await (0, import_promises6.writeFile)(path6, JSON.stringify(manifest, null, 2), "utf8");
|
|
15005
|
+
await updateWorkflowIndex(manifest, path6, this.baseDir);
|
|
15006
|
+
return path6;
|
|
15007
|
+
}
|
|
15008
|
+
};
|
|
15009
|
+
}
|
|
15010
|
+
});
|
|
15011
|
+
|
|
15012
|
+
// src/workflows/http-client.ts
|
|
15013
|
+
var WorkflowHttpClient;
|
|
15014
|
+
var init_http_client2 = __esm({
|
|
15015
|
+
"src/workflows/http-client.ts"() {
|
|
15016
|
+
"use strict";
|
|
15017
|
+
WorkflowHttpClient = class {
|
|
15018
|
+
constructor(apiUrl, apiKey, fetchImpl = fetch) {
|
|
15019
|
+
this.apiUrl = apiUrl;
|
|
15020
|
+
this.apiKey = apiKey;
|
|
15021
|
+
this.fetchImpl = fetchImpl;
|
|
15022
|
+
}
|
|
15023
|
+
apiUrl;
|
|
15024
|
+
apiKey;
|
|
15025
|
+
fetchImpl;
|
|
15026
|
+
async post(path6, body, timeoutMs = 18e4) {
|
|
15027
|
+
const res = await this.fetchImpl(`${this.apiUrl.replace(/\/$/, "")}${path6}`, {
|
|
15028
|
+
method: "POST",
|
|
15029
|
+
headers: {
|
|
15030
|
+
"Content-Type": "application/json",
|
|
15031
|
+
"x-api-key": this.apiKey
|
|
15032
|
+
},
|
|
15033
|
+
body: JSON.stringify(body),
|
|
15034
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
15035
|
+
});
|
|
15036
|
+
const data = await res.json().catch(() => ({}));
|
|
15037
|
+
if (!res.ok) {
|
|
15038
|
+
const message = typeof data.error === "string" ? data.error : `Workflow API ${path6} failed with ${res.status}`;
|
|
15039
|
+
throw new Error(message);
|
|
15040
|
+
}
|
|
15041
|
+
return data;
|
|
15042
|
+
}
|
|
15043
|
+
};
|
|
15044
|
+
}
|
|
15045
|
+
});
|
|
15046
|
+
|
|
15047
|
+
// src/workflows/report-renderer.ts
|
|
15048
|
+
function escapeHtml(value) {
|
|
15049
|
+
return String(value ?? "").replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
15050
|
+
}
|
|
15051
|
+
function renderTable(table) {
|
|
15052
|
+
const rows = table.rows.map((row) => `<tr>${table.columns.map((col) => `<td>${escapeHtml(row[col])}</td>`).join("")}</tr>`).join("\n");
|
|
15053
|
+
return [
|
|
15054
|
+
`<section>`,
|
|
15055
|
+
`<h2>${escapeHtml(table.title)}</h2>`,
|
|
15056
|
+
`<div class="table-wrap"><table>`,
|
|
15057
|
+
`<thead><tr>${table.columns.map((col) => `<th>${escapeHtml(col)}</th>`).join("")}</tr></thead>`,
|
|
15058
|
+
`<tbody>${rows || `<tr><td colspan="${table.columns.length}">No rows</td></tr>`}</tbody>`,
|
|
15059
|
+
`</table></div>`,
|
|
15060
|
+
`</section>`
|
|
15061
|
+
].join("\n");
|
|
15062
|
+
}
|
|
15063
|
+
function renderWorkflowReport(input) {
|
|
15064
|
+
const warnings = input.warnings?.length ? `<section class="warnings"><h2>Warnings</h2><ul>${input.warnings.map((w) => `<li>${escapeHtml(w)}</li>`).join("")}</ul></section>` : "";
|
|
15065
|
+
const sections = (input.sections ?? []).map((section) => `<section><h2>${escapeHtml(section.title)}</h2><p>${escapeHtml(section.body)}</p></section>`).join("\n");
|
|
15066
|
+
const tables = (input.tables ?? []).map(renderTable).join("\n");
|
|
15067
|
+
return [
|
|
15068
|
+
"<!doctype html>",
|
|
15069
|
+
'<html lang="en">',
|
|
15070
|
+
"<head>",
|
|
15071
|
+
'<meta charset="utf-8">',
|
|
15072
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">',
|
|
15073
|
+
`<title>${escapeHtml(input.title)}</title>`,
|
|
15074
|
+
"<style>",
|
|
15075
|
+
':root{font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:#172026;background:#f6f7f8}',
|
|
15076
|
+
"body{margin:0;padding:32px}main{max-width:1180px;margin:0 auto;background:#fff;border:1px solid #d9dee3;border-radius:8px;padding:28px}",
|
|
15077
|
+
"h1{font-size:30px;line-height:1.15;margin:0 0 8px}h2{font-size:18px;margin:28px 0 12px}p{line-height:1.55;color:#3d4952}.sub{color:#66737f;margin:0 0 20px}",
|
|
15078
|
+
".summary{background:#eef6f8;border-left:4px solid #12829a;padding:14px 16px;margin:20px 0}.warnings{background:#fff7e8;border-left:4px solid #c87b00;padding:12px 16px}",
|
|
15079
|
+
".table-wrap{overflow:auto;border:1px solid #dfe5ea;border-radius:8px}table{border-collapse:collapse;width:100%;font-size:13px}th,td{border-bottom:1px solid #edf0f2;padding:9px 10px;text-align:left;vertical-align:top}th{background:#f3f5f7;color:#31404c;position:sticky;top:0}tr:last-child td{border-bottom:0}",
|
|
15080
|
+
"</style>",
|
|
15081
|
+
"</head>",
|
|
15082
|
+
"<body><main>",
|
|
15083
|
+
`<h1>${escapeHtml(input.title)}</h1>`,
|
|
15084
|
+
input.subtitle ? `<p class="sub">${escapeHtml(input.subtitle)}</p>` : "",
|
|
15085
|
+
`<div class="summary">${escapeHtml(input.summary)}</div>`,
|
|
15086
|
+
warnings,
|
|
15087
|
+
sections,
|
|
15088
|
+
tables,
|
|
15089
|
+
"</main></body></html>"
|
|
15090
|
+
].join("\n");
|
|
15091
|
+
}
|
|
15092
|
+
var init_report_renderer = __esm({
|
|
15093
|
+
"src/workflows/report-renderer.ts"() {
|
|
15094
|
+
"use strict";
|
|
15095
|
+
}
|
|
15096
|
+
});
|
|
15097
|
+
|
|
15098
|
+
// src/workflows/workflows/agent-packet.ts
|
|
15099
|
+
function normalizeDomain(value) {
|
|
15100
|
+
if (!value) return null;
|
|
15101
|
+
try {
|
|
15102
|
+
const url = new URL(value.includes("://") ? value : `https://${value}`);
|
|
15103
|
+
return url.hostname.replace(/^www\./, "").toLowerCase();
|
|
15104
|
+
} catch {
|
|
15105
|
+
return value.replace(/^https?:\/\//, "").replace(/^www\./, "").split("/")[0]?.toLowerCase() || null;
|
|
15106
|
+
}
|
|
15107
|
+
}
|
|
15108
|
+
function domainFromUrl(url) {
|
|
15109
|
+
return normalizeDomain(url) ?? "";
|
|
15110
|
+
}
|
|
15111
|
+
function sourceRows(input, serp, paa) {
|
|
15112
|
+
const target = normalizeDomain(input.domain);
|
|
15113
|
+
const rows = [];
|
|
15114
|
+
for (const result of serp?.organicResults ?? []) {
|
|
15115
|
+
const domain = normalizeDomain(result.domain) ?? domainFromUrl(result.url);
|
|
15116
|
+
rows.push({
|
|
15117
|
+
surface: "organic",
|
|
15118
|
+
position: result.position,
|
|
15119
|
+
title: result.title,
|
|
15120
|
+
url: result.url,
|
|
15121
|
+
domain,
|
|
15122
|
+
question: "",
|
|
15123
|
+
answer_excerpt: result.snippet ?? "",
|
|
15124
|
+
is_target: target ? domain === target : false,
|
|
15125
|
+
is_competitor: target ? domain !== target : true
|
|
15126
|
+
});
|
|
15127
|
+
}
|
|
15128
|
+
for (const row of paa?.flat ?? []) {
|
|
15129
|
+
const url = row.source_cite ?? "";
|
|
15130
|
+
const domain = normalizeDomain(row.source_site ?? "") ?? (url ? domainFromUrl(url) : "");
|
|
15131
|
+
rows.push({
|
|
15132
|
+
surface: "paa",
|
|
15133
|
+
position: "",
|
|
15134
|
+
title: row.source_title ?? row.source_site ?? "",
|
|
15135
|
+
url,
|
|
15136
|
+
domain,
|
|
15137
|
+
question: row.question,
|
|
15138
|
+
answer_excerpt: (row.answer ?? "").slice(0, 240),
|
|
15139
|
+
is_target: target ? domain === target : false,
|
|
15140
|
+
is_competitor: target ? Boolean(domain && domain !== target) : Boolean(domain)
|
|
15141
|
+
});
|
|
15142
|
+
}
|
|
15143
|
+
for (const [i, cite] of (serp?.aiOverview?.citations ?? []).entries()) {
|
|
15144
|
+
const url = cite.href ?? "";
|
|
15145
|
+
const domain = url ? domainFromUrl(url) : "";
|
|
15146
|
+
rows.push({
|
|
15147
|
+
surface: "ai_overview",
|
|
15148
|
+
position: i + 1,
|
|
15149
|
+
title: cite.text ?? "",
|
|
15150
|
+
url,
|
|
15151
|
+
domain,
|
|
15152
|
+
question: "",
|
|
15153
|
+
answer_excerpt: "",
|
|
15154
|
+
is_target: target ? domain === target : false,
|
|
15155
|
+
is_competitor: target ? Boolean(domain && domain !== target) : Boolean(domain)
|
|
15156
|
+
});
|
|
15157
|
+
}
|
|
15158
|
+
return rows;
|
|
15159
|
+
}
|
|
15160
|
+
function competitorRows(rows, targetDomain) {
|
|
15161
|
+
const byDomain = /* @__PURE__ */ new Map();
|
|
15162
|
+
for (const row of rows) {
|
|
15163
|
+
const domain = String(row.domain ?? "");
|
|
15164
|
+
if (!domain || domain === targetDomain) continue;
|
|
15165
|
+
const entry = byDomain.get(domain) ?? { organic: [], paa: 0, ai: 0, urls: /* @__PURE__ */ new Set() };
|
|
15166
|
+
if (row.surface === "organic" && row.position) entry.organic.push(Number(row.position));
|
|
15167
|
+
if (row.surface === "paa") entry.paa += 1;
|
|
15168
|
+
if (row.surface === "ai_overview") entry.ai += 1;
|
|
15169
|
+
if (row.url) entry.urls.add(String(row.url));
|
|
15170
|
+
byDomain.set(domain, entry);
|
|
15171
|
+
}
|
|
15172
|
+
return [...byDomain.entries()].map(([domain, entry]) => ({
|
|
15173
|
+
domain,
|
|
15174
|
+
organic_best_position: entry.organic.length ? Math.min(...entry.organic) : "",
|
|
15175
|
+
organic_count: entry.organic.length,
|
|
15176
|
+
paa_mentions: entry.paa,
|
|
15177
|
+
ai_overview_citations: entry.ai,
|
|
15178
|
+
source_url_count: entry.urls.size
|
|
15179
|
+
})).sort((a, b) => Number(a.organic_best_position || 999) - Number(b.organic_best_position || 999));
|
|
15180
|
+
}
|
|
15181
|
+
var import_zod19, AgentPacketInputSchema, agentPacketWorkflowDefinition;
|
|
15182
|
+
var init_agent_packet = __esm({
|
|
15183
|
+
"src/workflows/workflows/agent-packet.ts"() {
|
|
15184
|
+
"use strict";
|
|
15185
|
+
import_zod19 = require("zod");
|
|
15186
|
+
init_report_renderer();
|
|
15187
|
+
AgentPacketInputSchema = import_zod19.z.object({
|
|
15188
|
+
keyword: import_zod19.z.string().min(1),
|
|
15189
|
+
domain: import_zod19.z.string().optional(),
|
|
15190
|
+
location: import_zod19.z.string().optional(),
|
|
15191
|
+
maxQuestions: import_zod19.z.number().int().min(1).max(200).default(40),
|
|
15192
|
+
includeSerp: import_zod19.z.boolean().default(true),
|
|
15193
|
+
includePaa: import_zod19.z.boolean().default(true),
|
|
15194
|
+
includeAiOverview: import_zod19.z.boolean().default(true),
|
|
15195
|
+
returnPartial: import_zod19.z.boolean().default(true)
|
|
15196
|
+
});
|
|
15197
|
+
agentPacketWorkflowDefinition = {
|
|
15198
|
+
id: "agent-packet",
|
|
15199
|
+
title: "Agent-Ready SEO Packet",
|
|
15200
|
+
description: "Create an evidence folder for AI agents from live SERP/PAA/AI search surfaces.",
|
|
15201
|
+
inputSchema: AgentPacketInputSchema,
|
|
15202
|
+
async run(input, ctx) {
|
|
15203
|
+
await ctx.artifacts.writeManifest("running", {}, [], []);
|
|
15204
|
+
const warnings = [];
|
|
15205
|
+
const errors = [];
|
|
15206
|
+
let serp = null;
|
|
15207
|
+
let paa = null;
|
|
15208
|
+
if (input.includeSerp) {
|
|
15209
|
+
try {
|
|
15210
|
+
serp = await ctx.client.post("/harvest/sync", {
|
|
15211
|
+
query: input.keyword,
|
|
15212
|
+
location: input.location,
|
|
15213
|
+
serpOnly: true,
|
|
15214
|
+
maxQuestions: 1,
|
|
15215
|
+
format: "json"
|
|
15216
|
+
}, 18e4);
|
|
15217
|
+
} catch (err) {
|
|
15218
|
+
warnings.push(`SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
15219
|
+
}
|
|
15220
|
+
}
|
|
15221
|
+
if (input.includePaa) {
|
|
15222
|
+
try {
|
|
15223
|
+
paa = await ctx.client.post("/harvest/sync", {
|
|
15224
|
+
query: input.keyword,
|
|
15225
|
+
location: input.location,
|
|
15226
|
+
maxQuestions: input.maxQuestions,
|
|
15227
|
+
format: "json"
|
|
15228
|
+
}, 28e4);
|
|
15229
|
+
} catch (err) {
|
|
15230
|
+
warnings.push(`PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
15231
|
+
}
|
|
15232
|
+
}
|
|
15233
|
+
if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
|
|
15234
|
+
const rows = sourceRows(input, serp, paa);
|
|
15235
|
+
const target = normalizeDomain(input.domain);
|
|
15236
|
+
const competitors = competitorRows(rows, target);
|
|
15237
|
+
const evidence = {
|
|
15238
|
+
input,
|
|
15239
|
+
serp: serp ?? { status: "skipped" },
|
|
15240
|
+
paa: paa ?? { status: "skipped" },
|
|
15241
|
+
target: {
|
|
15242
|
+
domain: target,
|
|
15243
|
+
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
15244
|
+
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
15245
|
+
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
15246
|
+
},
|
|
15247
|
+
competitors,
|
|
15248
|
+
warnings
|
|
15249
|
+
};
|
|
15250
|
+
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
15251
|
+
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
15252
|
+
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
15253
|
+
const brief = [
|
|
15254
|
+
`# SEO Evidence Brief: ${input.keyword}`,
|
|
15255
|
+
"",
|
|
15256
|
+
`Location: ${input.location ?? "not specified"}`,
|
|
15257
|
+
`Target domain: ${target ?? "not specified"}`,
|
|
15258
|
+
"",
|
|
15259
|
+
"## Evidence Summary",
|
|
15260
|
+
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
15261
|
+
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
15262
|
+
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
15263
|
+
`- Competitor domains: ${competitors.length}`,
|
|
15264
|
+
"",
|
|
15265
|
+
"## Recommended Use",
|
|
15266
|
+
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
15267
|
+
].join("\n");
|
|
15268
|
+
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
15269
|
+
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
15270
|
+
"# Agent Tasks",
|
|
15271
|
+
"",
|
|
15272
|
+
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
15273
|
+
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
15274
|
+
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
15275
|
+
"- [ ] Mark unsupported recommendations as assumptions."
|
|
15276
|
+
].join("\n"));
|
|
15277
|
+
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
15278
|
+
"# Agent Instructions",
|
|
15279
|
+
"",
|
|
15280
|
+
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
15281
|
+
].join("\n"));
|
|
15282
|
+
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
15283
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
15284
|
+
title: "Agent-Ready SEO Packet",
|
|
15285
|
+
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
15286
|
+
summary,
|
|
15287
|
+
warnings,
|
|
15288
|
+
tables: [
|
|
15289
|
+
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
15290
|
+
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
15291
|
+
]
|
|
15292
|
+
}));
|
|
15293
|
+
const status = warnings.length ? "partial" : "succeeded";
|
|
15294
|
+
const counts = { sources: rows.length, competitors: competitors.length };
|
|
15295
|
+
await ctx.artifacts.writeManifest(status, counts, warnings, errors);
|
|
15296
|
+
return { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors, reportPath };
|
|
15297
|
+
}
|
|
15298
|
+
};
|
|
15299
|
+
}
|
|
15300
|
+
});
|
|
15301
|
+
|
|
15302
|
+
// src/workflows/workflows/directory.ts
|
|
15303
|
+
function directoryRows(result) {
|
|
15304
|
+
const rows = [];
|
|
15305
|
+
for (const city of result.cities) {
|
|
15306
|
+
if (!city.results.length) {
|
|
15307
|
+
rows.push({
|
|
15308
|
+
source_query: result.query,
|
|
15309
|
+
source_location: city.location,
|
|
15310
|
+
city: city.city,
|
|
15311
|
+
state: city.state,
|
|
15312
|
+
population: city.population,
|
|
15313
|
+
result_position: null,
|
|
15314
|
+
business_name: null,
|
|
15315
|
+
review_stars: null,
|
|
15316
|
+
review_count: null,
|
|
15317
|
+
category: null,
|
|
15318
|
+
address: null,
|
|
15319
|
+
phone: null,
|
|
15320
|
+
website_url: null,
|
|
15321
|
+
place_url: null,
|
|
15322
|
+
cid: null,
|
|
15323
|
+
cid_decimal: null,
|
|
15324
|
+
result_status: city.status,
|
|
15325
|
+
error: city.error
|
|
15326
|
+
});
|
|
15327
|
+
continue;
|
|
15328
|
+
}
|
|
15329
|
+
for (const business of city.results) {
|
|
15330
|
+
rows.push({
|
|
15331
|
+
source_query: result.query,
|
|
15332
|
+
source_location: city.location,
|
|
15333
|
+
city: city.city,
|
|
15334
|
+
state: city.state,
|
|
15335
|
+
population: city.population,
|
|
15336
|
+
result_position: business.position,
|
|
15337
|
+
business_name: business.name,
|
|
15338
|
+
review_stars: business.rating,
|
|
15339
|
+
review_count: business.reviewCount,
|
|
15340
|
+
category: business.category,
|
|
15341
|
+
address: business.address,
|
|
15342
|
+
phone: business.phone,
|
|
15343
|
+
website_url: business.websiteUrl,
|
|
15344
|
+
place_url: business.placeUrl,
|
|
15345
|
+
cid: business.cid,
|
|
15346
|
+
cid_decimal: business.cidDecimal,
|
|
15347
|
+
result_status: city.status,
|
|
15348
|
+
error: city.error
|
|
15349
|
+
});
|
|
15350
|
+
}
|
|
15351
|
+
}
|
|
15352
|
+
return rows;
|
|
15353
|
+
}
|
|
15354
|
+
var import_zod20, DirectoryWorkflowCliInputSchema, DIRECTORY_CSV_HEADERS, directoryWorkflowDefinition;
|
|
15355
|
+
var init_directory = __esm({
|
|
15356
|
+
"src/workflows/workflows/directory.ts"() {
|
|
15357
|
+
"use strict";
|
|
15358
|
+
import_zod20 = require("zod");
|
|
15359
|
+
init_report_renderer();
|
|
15360
|
+
DirectoryWorkflowCliInputSchema = import_zod20.z.object({
|
|
15361
|
+
query: import_zod20.z.string().min(1),
|
|
15362
|
+
state: import_zod20.z.string().min(2).default("TN"),
|
|
15363
|
+
minPopulation: import_zod20.z.number().int().min(0).default(1e5),
|
|
15364
|
+
maxCities: import_zod20.z.number().int().min(1).max(100).default(25),
|
|
15365
|
+
maxResultsPerCity: import_zod20.z.number().int().min(1).max(50).default(20),
|
|
15366
|
+
concurrency: import_zod20.z.number().int().min(1).max(5).default(5),
|
|
15367
|
+
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location"),
|
|
15368
|
+
saveCsv: import_zod20.z.boolean().default(true)
|
|
15369
|
+
});
|
|
15370
|
+
DIRECTORY_CSV_HEADERS = [
|
|
15371
|
+
"source_query",
|
|
15372
|
+
"source_location",
|
|
15373
|
+
"city",
|
|
15374
|
+
"state",
|
|
15375
|
+
"population",
|
|
15376
|
+
"result_position",
|
|
15377
|
+
"business_name",
|
|
15378
|
+
"review_stars",
|
|
15379
|
+
"review_count",
|
|
15380
|
+
"category",
|
|
15381
|
+
"address",
|
|
15382
|
+
"phone",
|
|
15383
|
+
"website_url",
|
|
15384
|
+
"place_url",
|
|
15385
|
+
"cid",
|
|
15386
|
+
"cid_decimal",
|
|
15387
|
+
"result_status",
|
|
15388
|
+
"error"
|
|
15389
|
+
];
|
|
15390
|
+
directoryWorkflowDefinition = {
|
|
15391
|
+
id: "directory",
|
|
15392
|
+
title: "Directory Workflow",
|
|
15393
|
+
description: "Select city markets and export Google Maps business candidates.",
|
|
15394
|
+
inputSchema: DirectoryWorkflowCliInputSchema,
|
|
15395
|
+
async run(input, ctx) {
|
|
15396
|
+
await ctx.artifacts.writeManifest("running", {}, [], []);
|
|
15397
|
+
const result = await ctx.client.post("/directory/run", input, 9e5);
|
|
15398
|
+
await ctx.artifacts.writeJson("Directory evidence", "evidence.json", result);
|
|
15399
|
+
const rows = directoryRows(result);
|
|
15400
|
+
await ctx.artifacts.writeCsv("Directory CSV", "exports/directory.csv", DIRECTORY_CSV_HEADERS, rows);
|
|
15401
|
+
const cityRows = result.cities.map((city) => ({
|
|
15402
|
+
city: city.city,
|
|
15403
|
+
state: city.state,
|
|
15404
|
+
population: city.population,
|
|
15405
|
+
status: city.status,
|
|
15406
|
+
result_count: city.resultCount,
|
|
15407
|
+
error: city.error ?? ""
|
|
15408
|
+
}));
|
|
15409
|
+
const topRows = rows.filter((row) => row.business_name).slice(0, 100).map((row) => ({
|
|
15410
|
+
city: row.city,
|
|
15411
|
+
position: row.result_position,
|
|
15412
|
+
business: row.business_name,
|
|
15413
|
+
rating: row.review_stars,
|
|
15414
|
+
reviews: row.review_count,
|
|
15415
|
+
category: row.category,
|
|
15416
|
+
website: row.website_url
|
|
15417
|
+
}));
|
|
15418
|
+
const summary = `${result.selectedCityCount} cities processed with ${result.totalResultCount} Maps results.`;
|
|
15419
|
+
await ctx.artifacts.writeText("Summary", "summary.md", `# Directory Workflow
|
|
15420
|
+
|
|
15421
|
+
${summary}
|
|
15422
|
+
`);
|
|
15423
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
15424
|
+
title: "Directory Workflow",
|
|
15425
|
+
subtitle: `${input.query} \xB7 ${input.state}`,
|
|
15426
|
+
summary,
|
|
15427
|
+
warnings: result.warnings,
|
|
15428
|
+
tables: [
|
|
15429
|
+
{ title: "Cities", columns: ["city", "state", "population", "status", "result_count", "error"], rows: cityRows },
|
|
15430
|
+
{ title: "Top Results", columns: ["city", "position", "business", "rating", "reviews", "category", "website"], rows: topRows }
|
|
15431
|
+
]
|
|
15432
|
+
}));
|
|
15433
|
+
const status = result.cities.some((city) => city.status === "failed") ? "partial" : "succeeded";
|
|
15434
|
+
const counts = { cities: result.selectedCityCount, results: result.totalResultCount, rows: rows.length };
|
|
15435
|
+
await ctx.artifacts.writeManifest(status, counts, result.warnings, []);
|
|
15436
|
+
return { title: "Directory Workflow", summary, status, counts, warnings: result.warnings, errors: [], reportPath };
|
|
15437
|
+
}
|
|
15438
|
+
};
|
|
15439
|
+
}
|
|
15440
|
+
});
|
|
15441
|
+
|
|
15442
|
+
// src/workflows/workflows/local-competitive-audit.ts
|
|
15443
|
+
async function mapLimit2(items, limit, fn) {
|
|
15444
|
+
const out = new Array(items.length);
|
|
15445
|
+
let next = 0;
|
|
15446
|
+
async function worker() {
|
|
15447
|
+
while (next < items.length) {
|
|
15448
|
+
const index = next;
|
|
15449
|
+
next += 1;
|
|
15450
|
+
out[index] = await fn(items[index], index);
|
|
15451
|
+
}
|
|
15452
|
+
}
|
|
15453
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
15454
|
+
return out;
|
|
15455
|
+
}
|
|
15456
|
+
function numberFrom(value) {
|
|
15457
|
+
if (value === null || value === void 0 || value === "") return null;
|
|
15458
|
+
const parsed = Number(String(value).replace(/[^\d.]/g, ""));
|
|
15459
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
15460
|
+
}
|
|
15461
|
+
function median(values) {
|
|
15462
|
+
const nums = values.filter((v) => v !== null).sort((a, b) => a - b);
|
|
15463
|
+
if (!nums.length) return null;
|
|
15464
|
+
return nums[Math.floor(nums.length / 2)] ?? null;
|
|
15465
|
+
}
|
|
15466
|
+
function termsFrom(texts) {
|
|
15467
|
+
const stop = /* @__PURE__ */ new Set(["the", "and", "for", "with", "that", "this", "was", "were", "are", "you", "our", "they", "had", "have", "not", "but", "from", "very", "great", "good"]);
|
|
15468
|
+
const counts = /* @__PURE__ */ new Map();
|
|
15469
|
+
for (const text of texts) {
|
|
15470
|
+
for (const token of text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/)) {
|
|
15471
|
+
if (token.length < 4 || stop.has(token)) continue;
|
|
15472
|
+
counts.set(token, (counts.get(token) ?? 0) + 1);
|
|
15473
|
+
}
|
|
15474
|
+
}
|
|
15475
|
+
return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8).map(([term, count]) => `${term} (${count})`).join("; ");
|
|
15476
|
+
}
|
|
15477
|
+
var import_zod21, LocalCompetitiveAuditInputSchema, localCompetitiveAuditWorkflowDefinition;
|
|
15478
|
+
var init_local_competitive_audit = __esm({
|
|
15479
|
+
"src/workflows/workflows/local-competitive-audit.ts"() {
|
|
15480
|
+
"use strict";
|
|
15481
|
+
import_zod21 = require("zod");
|
|
15482
|
+
init_report_renderer();
|
|
15483
|
+
init_directory();
|
|
15484
|
+
LocalCompetitiveAuditInputSchema = import_zod21.z.object({
|
|
15485
|
+
query: import_zod21.z.string().min(1),
|
|
15486
|
+
state: import_zod21.z.string().min(2).default("TN"),
|
|
15487
|
+
minPopulation: import_zod21.z.number().int().min(0).default(1e5),
|
|
15488
|
+
maxCities: import_zod21.z.number().int().min(1).max(100).default(25),
|
|
15489
|
+
maxResultsPerCity: import_zod21.z.number().int().min(1).max(50).default(20),
|
|
15490
|
+
hydrateTop: import_zod21.z.number().int().min(0).max(10).default(5),
|
|
15491
|
+
maxReviews: import_zod21.z.number().int().min(0).max(500).default(50),
|
|
15492
|
+
concurrency: import_zod21.z.number().int().min(1).max(5).default(5),
|
|
15493
|
+
proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default("location"),
|
|
15494
|
+
returnPartial: import_zod21.z.boolean().default(true)
|
|
15495
|
+
});
|
|
15496
|
+
localCompetitiveAuditWorkflowDefinition = {
|
|
15497
|
+
id: "local-competitive-audit",
|
|
15498
|
+
title: "Local Competitive Audit",
|
|
15499
|
+
description: "Audit local Maps competitors, categories, review counts, and review themes across city markets.",
|
|
15500
|
+
inputSchema: LocalCompetitiveAuditInputSchema,
|
|
15501
|
+
async run(input, ctx) {
|
|
15502
|
+
await ctx.artifacts.writeManifest("running", {}, [], []);
|
|
15503
|
+
const warnings = [];
|
|
15504
|
+
const errors = [];
|
|
15505
|
+
const directory = await ctx.client.post("/directory/run", {
|
|
15506
|
+
query: input.query,
|
|
15507
|
+
state: input.state,
|
|
15508
|
+
minPopulation: input.minPopulation,
|
|
15509
|
+
maxCities: input.maxCities,
|
|
15510
|
+
maxResultsPerCity: input.maxResultsPerCity,
|
|
15511
|
+
concurrency: input.concurrency,
|
|
15512
|
+
proxyMode: input.proxyMode,
|
|
15513
|
+
saveCsv: true
|
|
15514
|
+
}, 9e5);
|
|
15515
|
+
await ctx.artifacts.writeJson("Directory raw JSON", "raw/directory-workflow.json", directory);
|
|
15516
|
+
const baseRows = directoryRows(directory);
|
|
15517
|
+
await ctx.artifacts.writeCsv("Base directory CSV", "exports/directory.csv", DIRECTORY_CSV_HEADERS, baseRows);
|
|
15518
|
+
const selected = directory.cities.flatMap(
|
|
15519
|
+
(city) => city.results.slice(0, input.hydrateTop).map((result) => ({ city, result }))
|
|
15520
|
+
);
|
|
15521
|
+
const seen = /* @__PURE__ */ new Set();
|
|
15522
|
+
const deduped = selected.filter(({ city, result }) => {
|
|
15523
|
+
const key = result.cid ?? result.placeUrl ?? `${city.location}:${result.name.toLowerCase()}`;
|
|
15524
|
+
if (seen.has(key)) return false;
|
|
15525
|
+
seen.add(key);
|
|
15526
|
+
return true;
|
|
15527
|
+
});
|
|
15528
|
+
const hydrated = await mapLimit2(deduped, 3, async ({ city, result }, index) => {
|
|
15529
|
+
try {
|
|
15530
|
+
const detail = await ctx.client.post("/maps/place", {
|
|
15531
|
+
businessName: result.name,
|
|
15532
|
+
location: city.location,
|
|
15533
|
+
includeReviews: input.maxReviews > 0,
|
|
15534
|
+
maxReviews: Math.max(1, input.maxReviews)
|
|
15535
|
+
}, 18e4);
|
|
15536
|
+
await ctx.artifacts.writeJson(`${result.name} profile`, `raw/maps-place-intel/${index + 1}-${result.name.toLowerCase().replace(/[^a-z0-9]+/g, "-")}.json`, detail);
|
|
15537
|
+
return { city, result, detail, error: null };
|
|
15538
|
+
} catch (err) {
|
|
15539
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
15540
|
+
warnings.push(`Profile hydration failed for ${result.name} (${city.location}): ${message}`);
|
|
15541
|
+
return { city, result, detail: null, error: message };
|
|
15542
|
+
}
|
|
15543
|
+
});
|
|
15544
|
+
const competitorRows2 = hydrated.map(({ city, result, detail, error }) => {
|
|
15545
|
+
const reviews = detail?.reviews ?? [];
|
|
15546
|
+
const ownerResponses = reviews.filter((r) => r.ownerResponse).length;
|
|
15547
|
+
return {
|
|
15548
|
+
city: city.city,
|
|
15549
|
+
state: city.state,
|
|
15550
|
+
source_location: city.location,
|
|
15551
|
+
result_position: result.position,
|
|
15552
|
+
business_name: result.name,
|
|
15553
|
+
category: detail?.category ?? result.category,
|
|
15554
|
+
review_stars: detail?.rating ?? result.rating,
|
|
15555
|
+
review_count: detail?.reviewCount ?? result.reviewCount,
|
|
15556
|
+
phone: result.phone,
|
|
15557
|
+
website_url: detail?.website ?? result.websiteUrl,
|
|
15558
|
+
place_url: result.placeUrl,
|
|
15559
|
+
cid: result.cid,
|
|
15560
|
+
cid_decimal: result.cidDecimal,
|
|
15561
|
+
hydrated: detail ? "true" : "false",
|
|
15562
|
+
reviews_status: detail?.reviewsStatus ?? "",
|
|
15563
|
+
review_topics: (detail?.reviewTopics ?? []).map((t) => `${t.label} (${t.count})`).join("; "),
|
|
15564
|
+
owner_response_rate: reviews.length ? (ownerResponses / reviews.length).toFixed(2) : "",
|
|
15565
|
+
error: error ?? ""
|
|
15566
|
+
};
|
|
15567
|
+
});
|
|
15568
|
+
const reviewInsightRows = hydrated.map(({ city, result, detail }) => {
|
|
15569
|
+
const reviews = detail?.reviews ?? [];
|
|
15570
|
+
const positive = reviews.filter((r) => Number(r.stars) >= 5 && r.text).map((r) => r.text);
|
|
15571
|
+
const negative = reviews.filter((r) => Number(r.stars) > 0 && Number(r.stars) <= 3 && r.text).map((r) => r.text);
|
|
15572
|
+
return {
|
|
15573
|
+
city: city.city,
|
|
15574
|
+
business_name: result.name,
|
|
15575
|
+
cid: result.cid,
|
|
15576
|
+
review_count: detail?.reviewCount ?? result.reviewCount,
|
|
15577
|
+
average_rating: detail?.rating ?? result.rating,
|
|
15578
|
+
topics: (detail?.reviewTopics ?? []).map((t) => `${t.label} (${t.count})`).join("; "),
|
|
15579
|
+
praise_terms: termsFrom(positive),
|
|
15580
|
+
complaint_terms: termsFrom(negative),
|
|
15581
|
+
positive_sample: positive[0]?.slice(0, 280) ?? "",
|
|
15582
|
+
negative_sample: negative[0]?.slice(0, 280) ?? ""
|
|
15583
|
+
};
|
|
15584
|
+
});
|
|
15585
|
+
const citySummaryRows = directory.cities.map((city) => {
|
|
15586
|
+
const resultReviewCounts = city.results.map((r) => numberFrom(r.reviewCount));
|
|
15587
|
+
const resultRatings = city.results.map((r) => numberFrom(r.rating));
|
|
15588
|
+
const topThreeReviews = city.results.slice(0, 3).map((r) => numberFrom(r.reviewCount)).filter((v) => v !== null);
|
|
15589
|
+
const categories = /* @__PURE__ */ new Map();
|
|
15590
|
+
for (const result of city.results) {
|
|
15591
|
+
if (!result.category) continue;
|
|
15592
|
+
categories.set(result.category, (categories.get(result.category) ?? 0) + 1);
|
|
15593
|
+
}
|
|
15594
|
+
const medReviews = median(resultReviewCounts);
|
|
15595
|
+
const medRating = median(resultRatings);
|
|
15596
|
+
const topThreeAvg = topThreeReviews.length ? Math.round(topThreeReviews.reduce((a, b) => a + b, 0) / topThreeReviews.length) : null;
|
|
15597
|
+
const difficulty = Math.min(100, Math.round((topThreeAvg ?? 0) / 20 + (medRating ?? 0) * 10 + city.results.filter((r) => r.websiteUrl).length));
|
|
15598
|
+
const opportunity = Math.max(0, 100 - difficulty);
|
|
15599
|
+
return {
|
|
15600
|
+
city: city.city,
|
|
15601
|
+
state: city.state,
|
|
15602
|
+
population: city.population,
|
|
15603
|
+
result_count: city.resultCount,
|
|
15604
|
+
median_review_count: medReviews ?? "",
|
|
15605
|
+
median_rating: medRating ?? "",
|
|
15606
|
+
top_three_average_review_count: topThreeAvg ?? "",
|
|
15607
|
+
top_categories: [...categories.entries()].sort((a, b) => b[1] - a[1]).slice(0, 5).map(([cat, count]) => `${cat} (${count})`).join("; "),
|
|
15608
|
+
opportunity_score: opportunity,
|
|
15609
|
+
difficulty_score: difficulty
|
|
15610
|
+
};
|
|
15611
|
+
});
|
|
15612
|
+
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["city", "state", "source_location", "result_position", "business_name", "category", "review_stars", "review_count", "phone", "website_url", "place_url", "cid", "cid_decimal", "hydrated", "reviews_status", "review_topics", "owner_response_rate", "error"], competitorRows2);
|
|
15613
|
+
await ctx.artifacts.writeCsv("Review insights CSV", "review-insights.csv", ["city", "business_name", "cid", "review_count", "average_rating", "topics", "praise_terms", "complaint_terms", "positive_sample", "negative_sample"], reviewInsightRows);
|
|
15614
|
+
await ctx.artifacts.writeCsv("City summary CSV", "city-summary.csv", ["city", "state", "population", "result_count", "median_review_count", "median_rating", "top_three_average_review_count", "top_categories", "opportunity_score", "difficulty_score"], citySummaryRows);
|
|
15615
|
+
await ctx.artifacts.writeJson("Audit evidence", "evidence.json", { input, directory, hydrated, citySummaryRows, competitorRows: competitorRows2, reviewInsightRows });
|
|
15616
|
+
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
15617
|
+
"# Local Competitive Audit Tasks",
|
|
15618
|
+
"",
|
|
15619
|
+
"- [ ] Review city opportunity and difficulty scores as heuristics, not absolute truth.",
|
|
15620
|
+
"- [ ] Use review topics and samples as customer-language evidence.",
|
|
15621
|
+
"- [ ] Compare target GBP category, review count, and website quality against top competitors.",
|
|
15622
|
+
"- [ ] Identify cities with low review-count leaders and weak website coverage."
|
|
15623
|
+
].join("\n"));
|
|
15624
|
+
const summary = `${directory.selectedCityCount} cities, ${directory.totalResultCount} Maps results, ${hydrated.filter((h) => h.detail).length} hydrated profiles.`;
|
|
15625
|
+
await ctx.artifacts.writeText("Summary", "summary.md", `# Local Competitive Audit
|
|
15626
|
+
|
|
15627
|
+
${summary}
|
|
15628
|
+
`);
|
|
15629
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
15630
|
+
title: "Local Competitive Audit",
|
|
15631
|
+
subtitle: `${input.query} \xB7 ${input.state}`,
|
|
15632
|
+
summary,
|
|
15633
|
+
warnings,
|
|
15634
|
+
tables: [
|
|
15635
|
+
{ title: "City Opportunity", columns: ["city", "state", "population", "result_count", "median_review_count", "median_rating", "top_three_average_review_count", "top_categories", "opportunity_score", "difficulty_score"], rows: citySummaryRows },
|
|
15636
|
+
{ title: "Hydrated Competitors", columns: ["city", "result_position", "business_name", "category", "review_stars", "review_count", "hydrated", "review_topics"], rows: competitorRows2.slice(0, 100) },
|
|
15637
|
+
{ title: "Review Insights", columns: ["city", "business_name", "topics", "praise_terms", "complaint_terms"], rows: reviewInsightRows.slice(0, 100) }
|
|
15638
|
+
]
|
|
15639
|
+
}));
|
|
15640
|
+
const status = warnings.length || directory.cities.some((city) => city.status === "failed") ? "partial" : "succeeded";
|
|
15641
|
+
const counts = { cities: directory.selectedCityCount, mapsResults: directory.totalResultCount, hydratedProfiles: hydrated.filter((h) => h.detail).length };
|
|
15642
|
+
await ctx.artifacts.writeManifest(status, counts, warnings, errors);
|
|
15643
|
+
return { title: "Local Competitive Audit", summary, status, counts, warnings, errors, reportPath };
|
|
15644
|
+
}
|
|
15645
|
+
};
|
|
15646
|
+
}
|
|
15647
|
+
});
|
|
15648
|
+
|
|
15649
|
+
// src/workflows/registry.ts
|
|
15650
|
+
function listWorkflowDefinitions() {
|
|
15651
|
+
return DEFINITIONS.map(({ id, title, description }) => ({ id, title, description }));
|
|
15652
|
+
}
|
|
15653
|
+
function workflowDefinition(id) {
|
|
15654
|
+
const definition = DEFINITIONS.find((def) => def.id === id);
|
|
15655
|
+
if (!definition) throw new Error(`Unknown workflow "${id}". Available: ${DEFINITIONS.map((def) => def.id).join(", ")}`);
|
|
15656
|
+
return definition;
|
|
15657
|
+
}
|
|
15658
|
+
async function runWorkflow(id, rawInput, options = {}) {
|
|
15659
|
+
const definition = workflowDefinition(id);
|
|
15660
|
+
const input = definition.inputSchema.parse(rawInput);
|
|
15661
|
+
const apiKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim();
|
|
15662
|
+
if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
|
|
15663
|
+
const apiUrl = options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
15664
|
+
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
15665
|
+
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl);
|
|
15666
|
+
try {
|
|
15667
|
+
return await definition.run(input, {
|
|
15668
|
+
runId: artifacts.runId,
|
|
15669
|
+
startedAt: artifacts.startedAt,
|
|
15670
|
+
client: client2,
|
|
15671
|
+
artifacts,
|
|
15672
|
+
signal: options.signal
|
|
15673
|
+
});
|
|
15674
|
+
} catch (err) {
|
|
15675
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
15676
|
+
await artifacts.writeText("Failure", "summary.md", `# ${definition.title}
|
|
15677
|
+
|
|
15678
|
+
${message}
|
|
15679
|
+
`);
|
|
15680
|
+
await artifacts.writeManifest("failed", {}, [], [message]);
|
|
15681
|
+
throw err;
|
|
15682
|
+
}
|
|
15683
|
+
}
|
|
15684
|
+
var import_zod22, DEFINITIONS;
|
|
15685
|
+
var init_registry2 = __esm({
|
|
15686
|
+
"src/workflows/registry.ts"() {
|
|
15687
|
+
"use strict";
|
|
15688
|
+
import_zod22 = require("zod");
|
|
15689
|
+
init_artifact_writer();
|
|
15690
|
+
init_http_client2();
|
|
15691
|
+
init_agent_packet();
|
|
15692
|
+
init_directory();
|
|
15693
|
+
init_local_competitive_audit();
|
|
15694
|
+
DEFINITIONS = [
|
|
15695
|
+
directoryWorkflowDefinition,
|
|
15696
|
+
agentPacketWorkflowDefinition,
|
|
15697
|
+
localCompetitiveAuditWorkflowDefinition
|
|
15698
|
+
];
|
|
15699
|
+
}
|
|
15700
|
+
});
|
|
15701
|
+
|
|
15702
|
+
// src/api/workflow-routes.ts
|
|
15703
|
+
function hostedWorkflowOutputDir() {
|
|
15704
|
+
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
15705
|
+
}
|
|
15706
|
+
function originFromUrl(url) {
|
|
15707
|
+
const parsed = new URL(url);
|
|
15708
|
+
return `${parsed.protocol}//${parsed.host}`;
|
|
15709
|
+
}
|
|
15710
|
+
function addCadence(fromIso, cadence) {
|
|
15711
|
+
const next = new Date(fromIso);
|
|
15712
|
+
if (!Number.isFinite(next.getTime())) return new Date(Date.now() + 24 * 60 * 60 * 1e3).toISOString();
|
|
15713
|
+
if (cadence === "daily") next.setUTCDate(next.getUTCDate() + 1);
|
|
15714
|
+
else if (cadence === "weekly") next.setUTCDate(next.getUTCDate() + 7);
|
|
15715
|
+
else next.setUTCMonth(next.getUTCMonth() + 1);
|
|
15716
|
+
return next.toISOString();
|
|
15717
|
+
}
|
|
15718
|
+
function defaultNextRunAt(cadence) {
|
|
15719
|
+
return addCadence((/* @__PURE__ */ new Date()).toISOString(), cadence);
|
|
15720
|
+
}
|
|
15721
|
+
function validateWorkflowInput(workflowId, input) {
|
|
15722
|
+
const definition = workflowDefinition(workflowId);
|
|
15723
|
+
return definition.inputSchema.parse(input);
|
|
15724
|
+
}
|
|
15725
|
+
async function validateWebhookUrl(url) {
|
|
15726
|
+
if (!url) return null;
|
|
15727
|
+
const result = await validatePublicHttpUrl(url, { field: "webhookUrl", requireHttps: true });
|
|
15728
|
+
if (result.error) throw new Error(result.error);
|
|
15729
|
+
return url;
|
|
15730
|
+
}
|
|
15731
|
+
function runDownloadUrl(c, runId, artifactId) {
|
|
15732
|
+
return `${originFromUrl(c.req.url)}/workflows/runs/${runId}/artifacts/${artifactId}`;
|
|
15733
|
+
}
|
|
15734
|
+
function exposeRun(c, run, artifacts = []) {
|
|
15735
|
+
return {
|
|
15736
|
+
...run,
|
|
15737
|
+
artifacts: artifacts.map((artifact) => ({
|
|
15738
|
+
...artifact,
|
|
15739
|
+
downloadUrl: runDownloadUrl(c, run.id, artifact.id)
|
|
15740
|
+
}))
|
|
15741
|
+
};
|
|
15742
|
+
}
|
|
15743
|
+
async function persistRunArtifacts(runId, userId, manifest) {
|
|
15744
|
+
if (!manifest) return [];
|
|
15745
|
+
return replaceWorkflowArtifacts(runId, userId, manifest.artifacts);
|
|
15746
|
+
}
|
|
15747
|
+
async function readManifestFromSummary(summary) {
|
|
15748
|
+
if (!summary.reportPath) return null;
|
|
15749
|
+
const manifestPath = summary.reportPath.replace(/report\.html$/, "manifest.json");
|
|
15750
|
+
try {
|
|
15751
|
+
return JSON.parse(await (0, import_promises7.readFile)(manifestPath, "utf8"));
|
|
15752
|
+
} catch {
|
|
15753
|
+
return null;
|
|
15754
|
+
}
|
|
15755
|
+
}
|
|
15756
|
+
function webhookSignature(body, timestamp2) {
|
|
15757
|
+
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
15758
|
+
if (!secret2) return null;
|
|
15759
|
+
return (0, import_node_crypto2.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
|
|
15760
|
+
}
|
|
15761
|
+
async function deliverWorkflowWebhook(input) {
|
|
15762
|
+
if (!input.webhookUrl) return;
|
|
15763
|
+
const payload = {
|
|
15764
|
+
event: input.run.status === "failed" ? "workflow.run.failed" : input.run.status === "partial" ? "workflow.run.partial" : "workflow.run.succeeded",
|
|
15765
|
+
runId: input.run.id,
|
|
15766
|
+
scheduleId: input.scheduleId,
|
|
15767
|
+
workflowId: input.run.workflow_id,
|
|
15768
|
+
status: input.run.status,
|
|
15769
|
+
completedAt: input.run.completed_at ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
15770
|
+
summary: {
|
|
15771
|
+
title: input.summary?.title ?? input.run.workflow_id,
|
|
15772
|
+
counts: input.summary?.counts ?? {},
|
|
15773
|
+
warnings: input.summary?.warnings ?? [],
|
|
15774
|
+
errors: input.summary?.errors ?? (input.run.error_message ? [input.run.error_message] : [])
|
|
15775
|
+
},
|
|
15776
|
+
artifacts: input.artifacts.map((artifact) => ({
|
|
15777
|
+
kind: artifact.kind,
|
|
15778
|
+
label: artifact.label,
|
|
15779
|
+
path: artifact.path
|
|
15780
|
+
}))
|
|
15781
|
+
};
|
|
15782
|
+
const body = JSON.stringify(payload);
|
|
15783
|
+
const timestamp2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
15784
|
+
const headers = {
|
|
15785
|
+
"Content-Type": "application/json",
|
|
15786
|
+
"x-mcp-scraper-timestamp": timestamp2
|
|
15787
|
+
};
|
|
15788
|
+
const signature = webhookSignature(body, timestamp2);
|
|
15789
|
+
if (signature) headers["x-mcp-scraper-signature"] = signature;
|
|
15790
|
+
let lastError = null;
|
|
15791
|
+
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
15792
|
+
try {
|
|
15793
|
+
const res = await fetch(input.webhookUrl, { method: "POST", headers, body, signal: AbortSignal.timeout(15e3) });
|
|
15794
|
+
if (res.ok) {
|
|
15795
|
+
await recordWorkflowWebhookDelivery({ runId: input.run.id, scheduleId: input.scheduleId, userId: input.userId, webhookUrl: input.webhookUrl, status: "delivered", attempts: attempt });
|
|
15796
|
+
return;
|
|
15797
|
+
}
|
|
15798
|
+
lastError = `HTTP ${res.status}`;
|
|
15799
|
+
} catch (err) {
|
|
15800
|
+
lastError = err instanceof Error ? err.message : String(err);
|
|
15801
|
+
}
|
|
15802
|
+
await new Promise((resolve) => setTimeout(resolve, attempt * 500));
|
|
15803
|
+
}
|
|
15804
|
+
await recordWorkflowWebhookDelivery({ runId: input.run.id, scheduleId: input.scheduleId, userId: input.userId, webhookUrl: input.webhookUrl, status: "failed", attempts: 3, lastError });
|
|
15805
|
+
}
|
|
15806
|
+
async function executeWorkflowRun(input) {
|
|
15807
|
+
await markWorkflowRunRunning(input.runId);
|
|
15808
|
+
try {
|
|
15809
|
+
const summary = await runWorkflow(input.workflowId, input.workflowInput, {
|
|
15810
|
+
apiKey: input.user.api_key,
|
|
15811
|
+
apiUrl: input.apiUrl,
|
|
15812
|
+
outputDir: hostedWorkflowOutputDir(),
|
|
15813
|
+
runId: input.runId
|
|
15814
|
+
});
|
|
15815
|
+
const manifest = await readManifestFromSummary(summary);
|
|
15816
|
+
await completeWorkflowRunRecord(input.runId, summary.status, manifest ?? { summary });
|
|
15817
|
+
const run = await getWorkflowRun(input.runId, input.user.id);
|
|
15818
|
+
if (!run) throw new Error("Workflow run disappeared after execution");
|
|
15819
|
+
const artifacts = await persistRunArtifacts(input.runId, input.user.id, manifest);
|
|
15820
|
+
await deliverWorkflowWebhook({ userId: input.user.id, run, scheduleId: input.scheduleId ?? null, webhookUrl: input.webhookUrl ?? null, summary, artifacts });
|
|
15821
|
+
return { run, summary, artifacts };
|
|
15822
|
+
} catch (err) {
|
|
15823
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
15824
|
+
await failWorkflowRunRecord(input.runId, message, { error: message });
|
|
15825
|
+
const run = await getWorkflowRun(input.runId, input.user.id);
|
|
15826
|
+
if (!run) throw err;
|
|
15827
|
+
await deliverWorkflowWebhook({ userId: input.user.id, run, scheduleId: input.scheduleId ?? null, webhookUrl: input.webhookUrl ?? null, summary: null, artifacts: [] });
|
|
15828
|
+
throw err;
|
|
15829
|
+
}
|
|
15830
|
+
}
|
|
15831
|
+
async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
15832
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15833
|
+
const due = await listDueWorkflowSchedules(now, limit);
|
|
15834
|
+
const results = [];
|
|
15835
|
+
for (const schedule of due) {
|
|
15836
|
+
const user = await getUserById(schedule.user_id);
|
|
15837
|
+
if (!user) {
|
|
15838
|
+
results.push({ scheduleId: schedule.id, status: "failed", error: "User not found" });
|
|
15839
|
+
continue;
|
|
15840
|
+
}
|
|
15841
|
+
const scheduledFor = schedule.next_run_at ?? now;
|
|
15842
|
+
const run = await createWorkflowRun({
|
|
15843
|
+
userId: user.id,
|
|
15844
|
+
scheduleId: schedule.id,
|
|
15845
|
+
workflowId: schedule.workflow_id,
|
|
15846
|
+
workflowInput: schedule.input,
|
|
15847
|
+
idempotencyKey: `${schedule.id}:${scheduledFor}`
|
|
15848
|
+
});
|
|
15849
|
+
await markWorkflowScheduleRan(schedule.id, now, addCadence(scheduledFor, schedule.cadence));
|
|
15850
|
+
if (run.status !== "queued") {
|
|
15851
|
+
results.push({ scheduleId: schedule.id, runId: run.id, status: run.status });
|
|
15852
|
+
continue;
|
|
15853
|
+
}
|
|
15854
|
+
try {
|
|
15855
|
+
const executed = await executeWorkflowRun({
|
|
15856
|
+
runId: run.id,
|
|
15857
|
+
user,
|
|
15858
|
+
workflowId: schedule.workflow_id,
|
|
15859
|
+
workflowInput: schedule.input,
|
|
15860
|
+
apiUrl,
|
|
15861
|
+
scheduleId: schedule.id,
|
|
15862
|
+
webhookUrl: schedule.webhook_url
|
|
15863
|
+
});
|
|
15864
|
+
results.push({ scheduleId: schedule.id, runId: executed.run.id, status: executed.run.status });
|
|
15865
|
+
} catch (err) {
|
|
15866
|
+
results.push({ scheduleId: schedule.id, runId: run.id, status: "failed", error: err instanceof Error ? err.message : String(err) });
|
|
15867
|
+
}
|
|
15868
|
+
}
|
|
15869
|
+
return { dispatched: results.length, results };
|
|
15870
|
+
}
|
|
15871
|
+
var import_node_crypto2, import_promises7, import_hono7, import_zod23, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema;
|
|
15872
|
+
var init_workflow_routes = __esm({
|
|
15873
|
+
"src/api/workflow-routes.ts"() {
|
|
15874
|
+
"use strict";
|
|
15875
|
+
import_node_crypto2 = require("crypto");
|
|
15876
|
+
import_promises7 = require("fs/promises");
|
|
15877
|
+
import_hono7 = require("hono");
|
|
15878
|
+
import_zod23 = require("zod");
|
|
15879
|
+
init_artifact_writer();
|
|
15880
|
+
init_registry2();
|
|
15881
|
+
init_api_auth();
|
|
15882
|
+
init_db();
|
|
15883
|
+
init_url_utils();
|
|
15884
|
+
workflowApp = new import_hono7.Hono();
|
|
15885
|
+
WorkflowInputSchema = import_zod23.z.record(import_zod23.z.unknown()).default({});
|
|
15886
|
+
WorkflowIdSchema = import_zod23.z.string().min(1);
|
|
15887
|
+
CadenceSchema = import_zod23.z.enum(["daily", "weekly", "monthly"]);
|
|
15888
|
+
ScheduleStatusSchema = import_zod23.z.enum(["active", "paused"]);
|
|
15889
|
+
RunBodySchema = import_zod23.z.object({
|
|
15890
|
+
workflowId: WorkflowIdSchema,
|
|
15891
|
+
input: WorkflowInputSchema,
|
|
15892
|
+
webhookUrl: import_zod23.z.string().url().optional()
|
|
15893
|
+
});
|
|
15894
|
+
ScheduleCreateSchema = import_zod23.z.object({
|
|
15895
|
+
workflowId: WorkflowIdSchema,
|
|
15896
|
+
name: import_zod23.z.string().min(1).max(120).optional(),
|
|
15897
|
+
input: WorkflowInputSchema,
|
|
15898
|
+
cadence: CadenceSchema.default("weekly"),
|
|
15899
|
+
timezone: import_zod23.z.string().min(1).max(64).default("UTC"),
|
|
15900
|
+
webhookUrl: import_zod23.z.string().url().optional(),
|
|
15901
|
+
nextRunAt: import_zod23.z.string().datetime().optional()
|
|
15902
|
+
});
|
|
15903
|
+
SchedulePatchSchema = import_zod23.z.object({
|
|
15904
|
+
name: import_zod23.z.string().min(1).max(120).optional(),
|
|
15905
|
+
status: ScheduleStatusSchema.optional(),
|
|
15906
|
+
input: WorkflowInputSchema.optional(),
|
|
15907
|
+
cadence: CadenceSchema.optional(),
|
|
15908
|
+
timezone: import_zod23.z.string().min(1).max(64).optional(),
|
|
15909
|
+
webhookUrl: import_zod23.z.string().url().nullable().optional(),
|
|
15910
|
+
nextRunAt: import_zod23.z.string().datetime().nullable().optional()
|
|
15911
|
+
});
|
|
15912
|
+
workflowApp.get("/definitions", createApiKeyAuth(), (c) => {
|
|
15913
|
+
return c.json({ workflows: listWorkflowDefinitions() });
|
|
15914
|
+
});
|
|
15915
|
+
workflowApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
15916
|
+
const user = c.get("user");
|
|
15917
|
+
const parsed = RunBodySchema.safeParse(await c.req.json().catch(() => ({})));
|
|
15918
|
+
if (!parsed.success) return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
15919
|
+
let input;
|
|
15920
|
+
let webhookUrl;
|
|
15921
|
+
try {
|
|
15922
|
+
input = validateWorkflowInput(parsed.data.workflowId, parsed.data.input);
|
|
15923
|
+
webhookUrl = await validateWebhookUrl(parsed.data.webhookUrl);
|
|
15924
|
+
} catch (err) {
|
|
15925
|
+
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
15926
|
+
}
|
|
15927
|
+
const run = await createWorkflowRun({ userId: user.id, workflowId: parsed.data.workflowId, workflowInput: input });
|
|
15928
|
+
try {
|
|
15929
|
+
const executed = await executeWorkflowRun({
|
|
15930
|
+
runId: run.id,
|
|
15931
|
+
user,
|
|
15932
|
+
workflowId: parsed.data.workflowId,
|
|
15933
|
+
workflowInput: input,
|
|
15934
|
+
apiUrl: originFromUrl(c.req.url),
|
|
15935
|
+
webhookUrl
|
|
15936
|
+
});
|
|
15937
|
+
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
15938
|
+
} catch (err) {
|
|
15939
|
+
const failed = await getWorkflowRun(run.id, user.id);
|
|
15940
|
+
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
15941
|
+
}
|
|
15942
|
+
});
|
|
15943
|
+
workflowApp.get("/runs", createApiKeyAuth(), async (c) => {
|
|
15944
|
+
const user = c.get("user");
|
|
15945
|
+
const runs = await listWorkflowRuns(user.id);
|
|
15946
|
+
return c.json({ runs });
|
|
15947
|
+
});
|
|
15948
|
+
workflowApp.get("/runs/:id", createApiKeyAuth(), async (c) => {
|
|
15949
|
+
const user = c.get("user");
|
|
15950
|
+
const run = await getWorkflowRun(c.req.param("id"), user.id);
|
|
15951
|
+
if (!run) return c.json({ error: "Run not found" }, 404);
|
|
15952
|
+
const artifacts = await listWorkflowArtifacts(run.id, user.id);
|
|
15953
|
+
return c.json({ run: exposeRun(c, run, artifacts) });
|
|
15954
|
+
});
|
|
15955
|
+
workflowApp.get("/runs/:id/artifacts/:artifactId", createApiKeyAuth(), async (c) => {
|
|
15956
|
+
const user = c.get("user");
|
|
15957
|
+
const run = await getWorkflowRun(c.req.param("id"), user.id);
|
|
15958
|
+
if (!run) return c.json({ error: "Run not found" }, 404);
|
|
15959
|
+
const artifact = await getWorkflowArtifact(c.req.param("artifactId"), user.id);
|
|
15960
|
+
if (!artifact || artifact.run_id !== run.id) return c.json({ error: "Artifact not found" }, 404);
|
|
15961
|
+
try {
|
|
15962
|
+
const content = await (0, import_promises7.readFile)(artifact.path);
|
|
15963
|
+
return new Response(content, {
|
|
15964
|
+
headers: {
|
|
15965
|
+
"Content-Type": artifact.content_type,
|
|
15966
|
+
"Content-Disposition": `attachment; filename="${artifact.path.split("/").pop() || artifact.id}"`
|
|
15967
|
+
}
|
|
15968
|
+
});
|
|
15969
|
+
} catch {
|
|
15970
|
+
return c.json({ error: "Artifact file is no longer available" }, 410);
|
|
15971
|
+
}
|
|
15972
|
+
});
|
|
15973
|
+
workflowApp.post("/schedules", createApiKeyAuth(), async (c) => {
|
|
15974
|
+
const user = c.get("user");
|
|
15975
|
+
const parsed = ScheduleCreateSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
15976
|
+
if (!parsed.success) return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
15977
|
+
let input;
|
|
15978
|
+
let webhookUrl;
|
|
15979
|
+
try {
|
|
15980
|
+
input = validateWorkflowInput(parsed.data.workflowId, parsed.data.input);
|
|
15981
|
+
webhookUrl = await validateWebhookUrl(parsed.data.webhookUrl);
|
|
15982
|
+
} catch (err) {
|
|
15983
|
+
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
15984
|
+
}
|
|
15985
|
+
const schedule = await createWorkflowSchedule({
|
|
15986
|
+
userId: user.id,
|
|
15987
|
+
workflowId: parsed.data.workflowId,
|
|
15988
|
+
name: parsed.data.name ?? workflowDefinition(parsed.data.workflowId).title,
|
|
15989
|
+
workflowInput: input,
|
|
15990
|
+
cadence: parsed.data.cadence,
|
|
15991
|
+
timezone: parsed.data.timezone,
|
|
15992
|
+
webhookUrl,
|
|
15993
|
+
nextRunAt: parsed.data.nextRunAt ?? defaultNextRunAt(parsed.data.cadence)
|
|
15994
|
+
});
|
|
15995
|
+
return c.json({ schedule }, 201);
|
|
15996
|
+
});
|
|
15997
|
+
workflowApp.get("/schedules", createApiKeyAuth(), async (c) => {
|
|
15998
|
+
const user = c.get("user");
|
|
15999
|
+
return c.json({ schedules: await listWorkflowSchedules(user.id) });
|
|
16000
|
+
});
|
|
16001
|
+
workflowApp.patch("/schedules/:id", createApiKeyAuth(), async (c) => {
|
|
16002
|
+
const user = c.get("user");
|
|
16003
|
+
const existing = await getWorkflowSchedule(c.req.param("id"), user.id);
|
|
16004
|
+
if (!existing) return c.json({ error: "Schedule not found" }, 404);
|
|
16005
|
+
const parsed = SchedulePatchSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
16006
|
+
if (!parsed.success) return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
16007
|
+
let nextInput;
|
|
16008
|
+
let webhookUrl;
|
|
16009
|
+
try {
|
|
16010
|
+
nextInput = parsed.data.input ? validateWorkflowInput(existing.workflow_id, parsed.data.input) : void 0;
|
|
16011
|
+
webhookUrl = parsed.data.webhookUrl === void 0 ? void 0 : await validateWebhookUrl(parsed.data.webhookUrl);
|
|
16012
|
+
} catch (err) {
|
|
16013
|
+
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
16014
|
+
}
|
|
16015
|
+
const schedule = await patchWorkflowSchedule({
|
|
16016
|
+
id: existing.id,
|
|
16017
|
+
userId: user.id,
|
|
16018
|
+
name: parsed.data.name,
|
|
16019
|
+
status: parsed.data.status,
|
|
16020
|
+
workflowInput: nextInput,
|
|
16021
|
+
cadence: parsed.data.cadence,
|
|
16022
|
+
timezone: parsed.data.timezone,
|
|
16023
|
+
webhookUrl,
|
|
16024
|
+
nextRunAt: parsed.data.nextRunAt
|
|
16025
|
+
});
|
|
16026
|
+
return c.json({ schedule });
|
|
16027
|
+
});
|
|
16028
|
+
workflowApp.delete("/schedules/:id", createApiKeyAuth(), async (c) => {
|
|
16029
|
+
const user = c.get("user");
|
|
16030
|
+
const deleted = await deleteWorkflowSchedule(c.req.param("id"), user.id);
|
|
16031
|
+
if (!deleted) return c.json({ error: "Schedule not found" }, 404);
|
|
16032
|
+
return c.json({ ok: true });
|
|
16033
|
+
});
|
|
16034
|
+
workflowApp.post("/schedules/:id/run", createApiKeyAuth(), async (c) => {
|
|
16035
|
+
const user = c.get("user");
|
|
16036
|
+
const schedule = await getWorkflowSchedule(c.req.param("id"), user.id);
|
|
16037
|
+
if (!schedule) return c.json({ error: "Schedule not found" }, 404);
|
|
16038
|
+
const run = await createWorkflowRun({ userId: user.id, scheduleId: schedule.id, workflowId: schedule.workflow_id, workflowInput: schedule.input });
|
|
16039
|
+
try {
|
|
16040
|
+
const executed = await executeWorkflowRun({
|
|
16041
|
+
runId: run.id,
|
|
16042
|
+
user,
|
|
16043
|
+
workflowId: schedule.workflow_id,
|
|
16044
|
+
workflowInput: schedule.input,
|
|
16045
|
+
apiUrl: originFromUrl(c.req.url),
|
|
16046
|
+
scheduleId: schedule.id,
|
|
16047
|
+
webhookUrl: schedule.webhook_url
|
|
16048
|
+
});
|
|
16049
|
+
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
16050
|
+
} catch (err) {
|
|
16051
|
+
const failed = await getWorkflowRun(run.id, user.id);
|
|
16052
|
+
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
16053
|
+
}
|
|
16054
|
+
});
|
|
16055
|
+
workflowApp.post("/cron/dispatch", async (c) => {
|
|
16056
|
+
const secret2 = c.req.header("authorization");
|
|
16057
|
+
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
16058
|
+
return c.json({ error: "Unauthorized" }, 401);
|
|
16059
|
+
}
|
|
16060
|
+
return c.json(await dispatchDueWorkflowSchedules(originFromUrl(c.req.url)));
|
|
16061
|
+
});
|
|
16062
|
+
}
|
|
16063
|
+
});
|
|
16064
|
+
|
|
14321
16065
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
14322
16066
|
function sha256(value) {
|
|
14323
16067
|
if (!value) return null;
|
|
14324
|
-
return (0,
|
|
16068
|
+
return (0, import_node_crypto3.createHash)("sha256").update(value).digest("hex");
|
|
14325
16069
|
}
|
|
14326
16070
|
function countWords(markdown) {
|
|
14327
16071
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -14621,11 +16365,11 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
14621
16365
|
}
|
|
14622
16366
|
};
|
|
14623
16367
|
}
|
|
14624
|
-
var
|
|
16368
|
+
var import_node_crypto3, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
|
|
14625
16369
|
var init_page_snapshot_extractor = __esm({
|
|
14626
16370
|
"src/serp-intelligence/page-snapshot-extractor.ts"() {
|
|
14627
16371
|
"use strict";
|
|
14628
|
-
|
|
16372
|
+
import_node_crypto3 = require("crypto");
|
|
14629
16373
|
import_p_limit3 = __toESM(require("p-limit"), 1);
|
|
14630
16374
|
init_kpo_extractor();
|
|
14631
16375
|
init_url_utils();
|
|
@@ -15330,7 +17074,7 @@ var init_PAAExtractor = __esm({
|
|
|
15330
17074
|
const titleText = texts.find((t) => !/^\d+:\d+$/.test(t) && t.length > 5) ?? "";
|
|
15331
17075
|
if (!titleText) continue;
|
|
15332
17076
|
let title = titleText;
|
|
15333
|
-
let
|
|
17077
|
+
let platform2 = "";
|
|
15334
17078
|
let channel = "";
|
|
15335
17079
|
for (const p of sels.platforms) {
|
|
15336
17080
|
let lastIdx = -1;
|
|
@@ -15346,13 +17090,13 @@ var init_PAAExtractor = __esm({
|
|
|
15346
17090
|
const isSourceTag = /^[\s·]/.test(after) || after.trim() === "";
|
|
15347
17091
|
if (!isSourceTag) continue;
|
|
15348
17092
|
title = titleText.slice(0, lastIdx).trim();
|
|
15349
|
-
|
|
17093
|
+
platform2 = p;
|
|
15350
17094
|
const stripped = after.replace(/^[\s·]+/, "");
|
|
15351
17095
|
const dotIdx = stripped.indexOf("\xB7");
|
|
15352
17096
|
channel = (dotIdx === -1 ? stripped : stripped.slice(0, dotIdx)).trim();
|
|
15353
17097
|
break;
|
|
15354
17098
|
}
|
|
15355
|
-
if (title) results.push({ title, channel, platform, duration, url: href });
|
|
17099
|
+
if (title) results.push({ title, channel, platform: platform2, duration, url: href });
|
|
15356
17100
|
}
|
|
15357
17101
|
return results;
|
|
15358
17102
|
}, svSels);
|
|
@@ -15396,10 +17140,10 @@ var init_PAAExtractor = __esm({
|
|
|
15396
17140
|
const engagement = engagementParts[0] ?? "";
|
|
15397
17141
|
const dateCandidates = allSpans.map((s2) => s2.textContent?.trim() ?? "").filter((t) => /\d+ (day|week|month|year|hour)s? ago|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec/.test(t));
|
|
15398
17142
|
const date = ytDate || (dateCandidates[0] ?? "");
|
|
15399
|
-
const
|
|
17143
|
+
const platform2 = platformText || (ytChannel ? "YouTube" : "");
|
|
15400
17144
|
const source = ytChannel || sourceText;
|
|
15401
17145
|
let type = "unknown";
|
|
15402
|
-
const pl =
|
|
17146
|
+
const pl = platform2.toLowerCase();
|
|
15403
17147
|
const src = source.toLowerCase();
|
|
15404
17148
|
const srcRaw = sourceText.toLowerCase();
|
|
15405
17149
|
if (pl.includes("reddit") || src.startsWith("r/")) type = "reddit";
|
|
@@ -15408,7 +17152,7 @@ var init_PAAExtractor = __esm({
|
|
|
15408
17152
|
else if (pl.includes("tiktok") || srcRaw.includes("tiktok")) type = "tiktok";
|
|
15409
17153
|
else if (pl.includes("youtube") || !!ytChannel) type = "youtube";
|
|
15410
17154
|
else type = "news";
|
|
15411
|
-
return { type, title, url, source, platform, popularComment, engagement, date, duration, authorNote };
|
|
17155
|
+
return { type, title, url, source, platform: platform2, popularComment, engagement, date, duration, authorNote };
|
|
15412
17156
|
});
|
|
15413
17157
|
}, sels);
|
|
15414
17158
|
}
|
|
@@ -15870,52 +17614,52 @@ var init_PAAExtractor = __esm({
|
|
|
15870
17614
|
});
|
|
15871
17615
|
|
|
15872
17616
|
// src/output/OutputSerializer.ts
|
|
15873
|
-
var
|
|
17617
|
+
var import_node_fs5, import_node_path8, import_papaparse2, OutputSerializer;
|
|
15874
17618
|
var init_OutputSerializer = __esm({
|
|
15875
17619
|
"src/output/OutputSerializer.ts"() {
|
|
15876
17620
|
"use strict";
|
|
15877
|
-
|
|
15878
|
-
|
|
17621
|
+
import_node_fs5 = require("fs");
|
|
17622
|
+
import_node_path8 = __toESM(require("path"), 1);
|
|
15879
17623
|
import_papaparse2 = __toESM(require("papaparse"), 1);
|
|
15880
17624
|
OutputSerializer = class {
|
|
15881
17625
|
async writeJSON(result, outputDir) {
|
|
15882
|
-
await
|
|
17626
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
15883
17627
|
const slug = result.seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
15884
17628
|
const filename = `${slug}-${Date.now()}.json`;
|
|
15885
|
-
const fullPath =
|
|
15886
|
-
await
|
|
17629
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17630
|
+
await import_node_fs5.promises.writeFile(fullPath, JSON.stringify(result, null, 2), "utf8");
|
|
15887
17631
|
return fullPath;
|
|
15888
17632
|
}
|
|
15889
17633
|
async writeCSV(rows, outputDir) {
|
|
15890
|
-
await
|
|
17634
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
15891
17635
|
const seedRaw = rows[0]?.seed_query ?? "paa";
|
|
15892
17636
|
const slug = seedRaw.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
15893
17637
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
15894
17638
|
const filename = `${slug}-${Date.now()}.csv`;
|
|
15895
|
-
const fullPath =
|
|
15896
|
-
await
|
|
17639
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17640
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
15897
17641
|
return fullPath;
|
|
15898
17642
|
}
|
|
15899
17643
|
async writeVideoCSV(videos, seed, outputDir) {
|
|
15900
|
-
await
|
|
17644
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
15901
17645
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
15902
17646
|
const csv = import_papaparse2.default.unparse(videos, { header: true });
|
|
15903
17647
|
const filename = `${slug}-videos-${Date.now()}.csv`;
|
|
15904
|
-
const fullPath =
|
|
15905
|
-
await
|
|
17648
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17649
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
15906
17650
|
return fullPath;
|
|
15907
17651
|
}
|
|
15908
17652
|
async writeForumCSV(forums, seed, outputDir) {
|
|
15909
|
-
await
|
|
17653
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
15910
17654
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
15911
17655
|
const csv = import_papaparse2.default.unparse(forums, { header: true });
|
|
15912
17656
|
const filename = `${slug}-forums-${Date.now()}.csv`;
|
|
15913
|
-
const fullPath =
|
|
15914
|
-
await
|
|
17657
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17658
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
15915
17659
|
return fullPath;
|
|
15916
17660
|
}
|
|
15917
17661
|
async writeAIOverviewCSV(citations, text, seed, outputDir) {
|
|
15918
|
-
await
|
|
17662
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
15919
17663
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
15920
17664
|
const rows = citations.map((c, i) => ({
|
|
15921
17665
|
seed_query: seed,
|
|
@@ -15925,12 +17669,12 @@ var init_OutputSerializer = __esm({
|
|
|
15925
17669
|
}));
|
|
15926
17670
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
15927
17671
|
const filename = `${slug}-ai-overview-${Date.now()}.csv`;
|
|
15928
|
-
const fullPath =
|
|
15929
|
-
await
|
|
17672
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17673
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
15930
17674
|
return fullPath;
|
|
15931
17675
|
}
|
|
15932
17676
|
async writeAIModeCSV(citations, text, seed, outputDir) {
|
|
15933
|
-
await
|
|
17677
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
15934
17678
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
15935
17679
|
const rows = citations.map((c, i) => ({
|
|
15936
17680
|
seed_query: seed,
|
|
@@ -15940,18 +17684,18 @@ var init_OutputSerializer = __esm({
|
|
|
15940
17684
|
}));
|
|
15941
17685
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
15942
17686
|
const filename = `${slug}-ai-mode-${Date.now()}.csv`;
|
|
15943
|
-
const fullPath =
|
|
15944
|
-
await
|
|
17687
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17688
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
15945
17689
|
return fullPath;
|
|
15946
17690
|
}
|
|
15947
17691
|
async writeWhatPeopleSayingCSV(cards, seed, outputDir) {
|
|
15948
|
-
await
|
|
17692
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
15949
17693
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
15950
17694
|
const rows = cards.map((c) => ({ seed_query: seed, ...c }));
|
|
15951
17695
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
15952
17696
|
const filename = `${slug}-what-people-saying-${Date.now()}.csv`;
|
|
15953
|
-
const fullPath =
|
|
15954
|
-
await
|
|
17697
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17698
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
15955
17699
|
return fullPath;
|
|
15956
17700
|
}
|
|
15957
17701
|
};
|
|
@@ -16034,7 +17778,7 @@ function classifyAttemptResult(result) {
|
|
|
16034
17778
|
function errorMessage2(err) {
|
|
16035
17779
|
return err instanceof Error ? err.message : String(err);
|
|
16036
17780
|
}
|
|
16037
|
-
function
|
|
17781
|
+
function maxAttemptsForProxyMode(proxyMode) {
|
|
16038
17782
|
return proxyMode === "location" ? LOCATION_PROXY_MAX_ATTEMPTS : MAX_ATTEMPTS2;
|
|
16039
17783
|
}
|
|
16040
17784
|
function looksLikeProxyTunnelFailure2(message) {
|
|
@@ -16061,7 +17805,7 @@ function stripInternalDebug(result, keepDebug) {
|
|
|
16061
17805
|
delete diagnostics.debug;
|
|
16062
17806
|
return { ...result, diagnostics };
|
|
16063
17807
|
}
|
|
16064
|
-
async function
|
|
17808
|
+
async function cleanupDisposableProxy2(kernelApiKey, proxyId) {
|
|
16065
17809
|
if (!kernelApiKey || !proxyId) return;
|
|
16066
17810
|
try {
|
|
16067
17811
|
await deleteKernelProxyId(kernelApiKey, proxyId);
|
|
@@ -16146,7 +17890,7 @@ async function harvest(rawOptions) {
|
|
|
16146
17890
|
};
|
|
16147
17891
|
const requestedDebug = typeof raw.debug === "boolean" ? raw.debug : false;
|
|
16148
17892
|
const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
|
|
16149
|
-
const maxAttempts =
|
|
17893
|
+
const maxAttempts = maxAttemptsForProxyMode(proxyMode);
|
|
16150
17894
|
const serializer = new OutputSerializer();
|
|
16151
17895
|
let lastError = null;
|
|
16152
17896
|
for (let i = 0; i < maxAttempts; i++) {
|
|
@@ -16223,7 +17967,7 @@ async function harvest(rawOptions) {
|
|
|
16223
17967
|
debug: attempt.debug,
|
|
16224
17968
|
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
16225
17969
|
});
|
|
16226
|
-
await
|
|
17970
|
+
await cleanupDisposableProxy2(kernelApiKey, resolution2.disposableProxyId);
|
|
16227
17971
|
lastError = err;
|
|
16228
17972
|
if (willRetry) continue;
|
|
16229
17973
|
break;
|
|
@@ -16254,7 +17998,7 @@ async function harvest(rawOptions) {
|
|
|
16254
17998
|
debug: attempt.debug,
|
|
16255
17999
|
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
16256
18000
|
});
|
|
16257
|
-
await
|
|
18001
|
+
await cleanupDisposableProxy2(kernelApiKey, resolution2.disposableProxyId);
|
|
16258
18002
|
lastError = err;
|
|
16259
18003
|
if (willRetry) continue;
|
|
16260
18004
|
break;
|
|
@@ -16274,7 +18018,7 @@ async function harvest(rawOptions) {
|
|
|
16274
18018
|
debug: attempt.debug,
|
|
16275
18019
|
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
16276
18020
|
});
|
|
16277
|
-
await
|
|
18021
|
+
await cleanupDisposableProxy2(kernelApiKey, resolution2.disposableProxyId);
|
|
16278
18022
|
if (attemptOptions.format === "json" || attemptOptions.format === "both") {
|
|
16279
18023
|
await serializer.writeJSON(finalResult, attemptOptions.outputDir);
|
|
16280
18024
|
}
|
|
@@ -16502,11 +18246,11 @@ function isPublicHttpUrl(value) {
|
|
|
16502
18246
|
return false;
|
|
16503
18247
|
}
|
|
16504
18248
|
}
|
|
16505
|
-
var
|
|
18249
|
+
var import_zod24, SerpIntelligenceDeviceValues, SerpIntelligenceProxyModeValues, SerpIntelligenceAttemptOutcomeValues, SerpIntelligenceLocalizationStatusValues, SerpPageSnapshotSourceKindValues, SerpPageFetchStatusValues, SerpPageFetchedViaValues, HostnameSuffixPattern, Ipv4Pattern, SerpIntelligencePublicHttpUrlSchema, SerpIntelligenceCaptureBodySchema, SerpIntelligencePageSnapshotRequestSchema, SerpIntelligencePageSnapshotsBodySchema, SerpIntelligenceAICitationSchema, SerpIntelligenceOrganicResultSchema, SerpIntelligenceLocationEvidenceSchema, SerpIntelligenceHarvestResultSchema, SerpIntelligenceCaptureAttemptSchema, SerpPageSnapshotCaptureSchema, SerpIntelligenceCaptureResponseSchema, SerpIntelligencePageSnapshotsResponseSchema;
|
|
16506
18250
|
var init_schemas4 = __esm({
|
|
16507
18251
|
"src/serp-intelligence/schemas.ts"() {
|
|
16508
18252
|
"use strict";
|
|
16509
|
-
|
|
18253
|
+
import_zod24 = require("zod");
|
|
16510
18254
|
SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
16511
18255
|
SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
16512
18256
|
SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -16540,171 +18284,171 @@ var init_schemas4 = __esm({
|
|
|
16540
18284
|
SerpPageFetchedViaValues = ["fetch", "headless", "browser", "mcp"];
|
|
16541
18285
|
HostnameSuffixPattern = /(^|\.)localhost$/i;
|
|
16542
18286
|
Ipv4Pattern = /^\d{1,3}(?:\.\d{1,3}){3}$/;
|
|
16543
|
-
SerpIntelligencePublicHttpUrlSchema =
|
|
16544
|
-
SerpIntelligenceCaptureBodySchema =
|
|
16545
|
-
query:
|
|
16546
|
-
location:
|
|
16547
|
-
gl:
|
|
16548
|
-
hl:
|
|
16549
|
-
device:
|
|
16550
|
-
proxyMode:
|
|
16551
|
-
proxyZip:
|
|
16552
|
-
pages:
|
|
16553
|
-
debug:
|
|
16554
|
-
includePageSnapshots:
|
|
16555
|
-
pageSnapshotLimit:
|
|
18287
|
+
SerpIntelligencePublicHttpUrlSchema = import_zod24.z.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
18288
|
+
SerpIntelligenceCaptureBodySchema = import_zod24.z.object({
|
|
18289
|
+
query: import_zod24.z.string().trim().min(1, "query is required"),
|
|
18290
|
+
location: import_zod24.z.string().trim().min(1).optional(),
|
|
18291
|
+
gl: import_zod24.z.string().trim().length(2).default("us"),
|
|
18292
|
+
hl: import_zod24.z.string().trim().length(2).default("en"),
|
|
18293
|
+
device: import_zod24.z.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
18294
|
+
proxyMode: import_zod24.z.enum(SerpIntelligenceProxyModeValues).default("location"),
|
|
18295
|
+
proxyZip: import_zod24.z.string().regex(/^\d{5}$/).optional(),
|
|
18296
|
+
pages: import_zod24.z.number().int().min(1).max(2).default(1),
|
|
18297
|
+
debug: import_zod24.z.boolean().default(false),
|
|
18298
|
+
includePageSnapshots: import_zod24.z.boolean().default(false),
|
|
18299
|
+
pageSnapshotLimit: import_zod24.z.number().int().min(0).max(10).default(0)
|
|
16556
18300
|
}).strict();
|
|
16557
|
-
SerpIntelligencePageSnapshotRequestSchema =
|
|
18301
|
+
SerpIntelligencePageSnapshotRequestSchema = import_zod24.z.object({
|
|
16558
18302
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
16559
|
-
sourceKind:
|
|
16560
|
-
sourcePosition:
|
|
18303
|
+
sourceKind: import_zod24.z.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
18304
|
+
sourcePosition: import_zod24.z.number().int().min(1).optional()
|
|
16561
18305
|
}).strict();
|
|
16562
|
-
SerpIntelligencePageSnapshotsBodySchema =
|
|
16563
|
-
urls:
|
|
16564
|
-
targets:
|
|
16565
|
-
maxConcurrency:
|
|
16566
|
-
timeoutMs:
|
|
16567
|
-
debug:
|
|
18306
|
+
SerpIntelligencePageSnapshotsBodySchema = import_zod24.z.object({
|
|
18307
|
+
urls: import_zod24.z.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
18308
|
+
targets: import_zod24.z.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
18309
|
+
maxConcurrency: import_zod24.z.number().int().min(1).max(5).default(2),
|
|
18310
|
+
timeoutMs: import_zod24.z.number().int().min(1e3).max(6e4).default(15e3),
|
|
18311
|
+
debug: import_zod24.z.boolean().default(false)
|
|
16568
18312
|
}).strict();
|
|
16569
|
-
SerpIntelligenceAICitationSchema =
|
|
16570
|
-
text:
|
|
16571
|
-
href:
|
|
18313
|
+
SerpIntelligenceAICitationSchema = import_zod24.z.object({
|
|
18314
|
+
text: import_zod24.z.string(),
|
|
18315
|
+
href: import_zod24.z.string()
|
|
16572
18316
|
}).strict();
|
|
16573
|
-
SerpIntelligenceOrganicResultSchema =
|
|
16574
|
-
position:
|
|
16575
|
-
title:
|
|
16576
|
-
url:
|
|
16577
|
-
domain:
|
|
16578
|
-
cite:
|
|
16579
|
-
snippet:
|
|
16580
|
-
isRedditStyle:
|
|
16581
|
-
inlineRating:
|
|
16582
|
-
value:
|
|
16583
|
-
count:
|
|
18317
|
+
SerpIntelligenceOrganicResultSchema = import_zod24.z.object({
|
|
18318
|
+
position: import_zod24.z.number().int().min(1),
|
|
18319
|
+
title: import_zod24.z.string(),
|
|
18320
|
+
url: import_zod24.z.string(),
|
|
18321
|
+
domain: import_zod24.z.string(),
|
|
18322
|
+
cite: import_zod24.z.string().nullable(),
|
|
18323
|
+
snippet: import_zod24.z.string().nullable(),
|
|
18324
|
+
isRedditStyle: import_zod24.z.boolean(),
|
|
18325
|
+
inlineRating: import_zod24.z.object({
|
|
18326
|
+
value: import_zod24.z.string(),
|
|
18327
|
+
count: import_zod24.z.string()
|
|
16584
18328
|
}).strict().nullable()
|
|
16585
18329
|
}).strict();
|
|
16586
|
-
SerpIntelligenceLocationEvidenceSchema =
|
|
16587
|
-
status:
|
|
16588
|
-
expected:
|
|
16589
|
-
city:
|
|
16590
|
-
regionCode:
|
|
16591
|
-
canonicalLocation:
|
|
18330
|
+
SerpIntelligenceLocationEvidenceSchema = import_zod24.z.object({
|
|
18331
|
+
status: import_zod24.z.enum(SerpIntelligenceLocalizationStatusValues),
|
|
18332
|
+
expected: import_zod24.z.object({
|
|
18333
|
+
city: import_zod24.z.string(),
|
|
18334
|
+
regionCode: import_zod24.z.string().nullable(),
|
|
18335
|
+
canonicalLocation: import_zod24.z.string()
|
|
16592
18336
|
}).strict().nullable(),
|
|
16593
|
-
candidates:
|
|
16594
|
-
city:
|
|
16595
|
-
regionCode:
|
|
16596
|
-
count:
|
|
16597
|
-
examples:
|
|
18337
|
+
candidates: import_zod24.z.array(import_zod24.z.object({
|
|
18338
|
+
city: import_zod24.z.string(),
|
|
18339
|
+
regionCode: import_zod24.z.string(),
|
|
18340
|
+
count: import_zod24.z.number().int().min(0),
|
|
18341
|
+
examples: import_zod24.z.array(import_zod24.z.string())
|
|
16598
18342
|
}).strict())
|
|
16599
18343
|
}).strict();
|
|
16600
|
-
SerpIntelligenceHarvestResultSchema =
|
|
16601
|
-
seed:
|
|
16602
|
-
location:
|
|
16603
|
-
extractedAt:
|
|
16604
|
-
totalQuestions:
|
|
16605
|
-
surface:
|
|
16606
|
-
aiOverview:
|
|
16607
|
-
detected:
|
|
16608
|
-
text:
|
|
16609
|
-
citations:
|
|
16610
|
-
expanded:
|
|
16611
|
-
fullyExpanded:
|
|
16612
|
-
sections:
|
|
18344
|
+
SerpIntelligenceHarvestResultSchema = import_zod24.z.object({
|
|
18345
|
+
seed: import_zod24.z.string(),
|
|
18346
|
+
location: import_zod24.z.string().nullable(),
|
|
18347
|
+
extractedAt: import_zod24.z.string(),
|
|
18348
|
+
totalQuestions: import_zod24.z.number().int().min(0),
|
|
18349
|
+
surface: import_zod24.z.enum(["web", "aim", "unknown"]),
|
|
18350
|
+
aiOverview: import_zod24.z.object({
|
|
18351
|
+
detected: import_zod24.z.boolean(),
|
|
18352
|
+
text: import_zod24.z.string().nullable(),
|
|
18353
|
+
citations: import_zod24.z.array(SerpIntelligenceAICitationSchema),
|
|
18354
|
+
expanded: import_zod24.z.boolean().optional(),
|
|
18355
|
+
fullyExpanded: import_zod24.z.boolean().optional(),
|
|
18356
|
+
sections: import_zod24.z.array(import_zod24.z.string()).optional()
|
|
16613
18357
|
}).strict(),
|
|
16614
|
-
aiMode:
|
|
16615
|
-
detected:
|
|
16616
|
-
text:
|
|
16617
|
-
citations:
|
|
18358
|
+
aiMode: import_zod24.z.object({
|
|
18359
|
+
detected: import_zod24.z.boolean(),
|
|
18360
|
+
text: import_zod24.z.string().nullable(),
|
|
18361
|
+
citations: import_zod24.z.array(SerpIntelligenceAICitationSchema)
|
|
16618
18362
|
}).strict(),
|
|
16619
|
-
tree:
|
|
16620
|
-
flat:
|
|
16621
|
-
videos:
|
|
16622
|
-
forums:
|
|
16623
|
-
organicResults:
|
|
16624
|
-
localPack:
|
|
16625
|
-
entityIds:
|
|
16626
|
-
entities:
|
|
16627
|
-
name:
|
|
16628
|
-
kgId:
|
|
16629
|
-
cid:
|
|
16630
|
-
gcid:
|
|
18363
|
+
tree: import_zod24.z.array(import_zod24.z.unknown()),
|
|
18364
|
+
flat: import_zod24.z.array(import_zod24.z.unknown()),
|
|
18365
|
+
videos: import_zod24.z.array(import_zod24.z.unknown()),
|
|
18366
|
+
forums: import_zod24.z.array(import_zod24.z.unknown()),
|
|
18367
|
+
organicResults: import_zod24.z.array(SerpIntelligenceOrganicResultSchema),
|
|
18368
|
+
localPack: import_zod24.z.array(import_zod24.z.unknown()),
|
|
18369
|
+
entityIds: import_zod24.z.object({
|
|
18370
|
+
entities: import_zod24.z.array(import_zod24.z.object({
|
|
18371
|
+
name: import_zod24.z.string(),
|
|
18372
|
+
kgId: import_zod24.z.string().nullable(),
|
|
18373
|
+
cid: import_zod24.z.string().nullable(),
|
|
18374
|
+
gcid: import_zod24.z.string().nullable()
|
|
16631
18375
|
}).strict()),
|
|
16632
|
-
kgIds:
|
|
16633
|
-
cids:
|
|
16634
|
-
gcids:
|
|
18376
|
+
kgIds: import_zod24.z.array(import_zod24.z.string()),
|
|
18377
|
+
cids: import_zod24.z.array(import_zod24.z.string()),
|
|
18378
|
+
gcids: import_zod24.z.array(import_zod24.z.string())
|
|
16635
18379
|
}).strict(),
|
|
16636
|
-
stats:
|
|
16637
|
-
seed:
|
|
16638
|
-
totalQuestions:
|
|
16639
|
-
maxDepthReached:
|
|
16640
|
-
durationMs:
|
|
16641
|
-
errorCount:
|
|
18380
|
+
stats: import_zod24.z.object({
|
|
18381
|
+
seed: import_zod24.z.string(),
|
|
18382
|
+
totalQuestions: import_zod24.z.number().int().min(0),
|
|
18383
|
+
maxDepthReached: import_zod24.z.number().int().min(0),
|
|
18384
|
+
durationMs: import_zod24.z.number().min(0),
|
|
18385
|
+
errorCount: import_zod24.z.number().int().min(0)
|
|
16642
18386
|
}).strict(),
|
|
16643
|
-
diagnostics:
|
|
16644
|
-
completionStatus:
|
|
16645
|
-
problem:
|
|
16646
|
-
warnings:
|
|
16647
|
-
debug:
|
|
18387
|
+
diagnostics: import_zod24.z.object({
|
|
18388
|
+
completionStatus: import_zod24.z.enum(["paa_found", "no_paa", "serp_only"]),
|
|
18389
|
+
problem: import_zod24.z.null(),
|
|
18390
|
+
warnings: import_zod24.z.array(import_zod24.z.unknown()).optional(),
|
|
18391
|
+
debug: import_zod24.z.object({
|
|
16648
18392
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
16649
18393
|
}).passthrough().optional()
|
|
16650
18394
|
}).passthrough(),
|
|
16651
|
-
whatPeopleSaying:
|
|
18395
|
+
whatPeopleSaying: import_zod24.z.array(import_zod24.z.unknown())
|
|
16652
18396
|
}).strict();
|
|
16653
|
-
SerpIntelligenceCaptureAttemptSchema =
|
|
16654
|
-
attemptNumber:
|
|
16655
|
-
outcome:
|
|
16656
|
-
startedAt:
|
|
16657
|
-
completedAt:
|
|
16658
|
-
durationMs:
|
|
16659
|
-
problemCode:
|
|
16660
|
-
message:
|
|
16661
|
-
kernelSessionId:
|
|
16662
|
-
cleanupSucceeded:
|
|
18397
|
+
SerpIntelligenceCaptureAttemptSchema = import_zod24.z.object({
|
|
18398
|
+
attemptNumber: import_zod24.z.number().int().min(1),
|
|
18399
|
+
outcome: import_zod24.z.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
18400
|
+
startedAt: import_zod24.z.string().optional(),
|
|
18401
|
+
completedAt: import_zod24.z.string().optional(),
|
|
18402
|
+
durationMs: import_zod24.z.number().min(0).optional(),
|
|
18403
|
+
problemCode: import_zod24.z.string().optional(),
|
|
18404
|
+
message: import_zod24.z.string().optional(),
|
|
18405
|
+
kernelSessionId: import_zod24.z.string().nullable().optional(),
|
|
18406
|
+
cleanupSucceeded: import_zod24.z.boolean().nullable().optional()
|
|
16663
18407
|
}).strict();
|
|
16664
|
-
SerpPageSnapshotCaptureSchema =
|
|
18408
|
+
SerpPageSnapshotCaptureSchema = import_zod24.z.object({
|
|
16665
18409
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
16666
18410
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
16667
18411
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
16668
|
-
sourceKind:
|
|
16669
|
-
sourcePosition:
|
|
16670
|
-
status:
|
|
16671
|
-
fetchedVia:
|
|
16672
|
-
httpStatus:
|
|
16673
|
-
contentType:
|
|
16674
|
-
title:
|
|
18412
|
+
sourceKind: import_zod24.z.enum(SerpPageSnapshotSourceKindValues),
|
|
18413
|
+
sourcePosition: import_zod24.z.number().int().min(1).nullable(),
|
|
18414
|
+
status: import_zod24.z.enum(SerpPageFetchStatusValues),
|
|
18415
|
+
fetchedVia: import_zod24.z.enum(SerpPageFetchedViaValues).nullable(),
|
|
18416
|
+
httpStatus: import_zod24.z.number().int().min(100).max(599).nullable(),
|
|
18417
|
+
contentType: import_zod24.z.string().nullable(),
|
|
18418
|
+
title: import_zod24.z.string().nullable(),
|
|
16675
18419
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
16676
|
-
metaDescription:
|
|
16677
|
-
headings:
|
|
16678
|
-
level:
|
|
16679
|
-
text:
|
|
18420
|
+
metaDescription: import_zod24.z.string().nullable(),
|
|
18421
|
+
headings: import_zod24.z.array(import_zod24.z.object({
|
|
18422
|
+
level: import_zod24.z.number().int().min(1).max(6),
|
|
18423
|
+
text: import_zod24.z.string()
|
|
16680
18424
|
}).strict()).default([]),
|
|
16681
|
-
artifact:
|
|
16682
|
-
htmlBlobUrl:
|
|
16683
|
-
textBlobUrl:
|
|
16684
|
-
markdownBlobUrl:
|
|
16685
|
-
screenshotBlobUrl:
|
|
16686
|
-
contentSha256:
|
|
16687
|
-
capturedAt:
|
|
18425
|
+
artifact: import_zod24.z.object({
|
|
18426
|
+
htmlBlobUrl: import_zod24.z.string().url().nullable(),
|
|
18427
|
+
textBlobUrl: import_zod24.z.string().url().nullable(),
|
|
18428
|
+
markdownBlobUrl: import_zod24.z.string().url().nullable(),
|
|
18429
|
+
screenshotBlobUrl: import_zod24.z.string().url().nullable(),
|
|
18430
|
+
contentSha256: import_zod24.z.string().nullable(),
|
|
18431
|
+
capturedAt: import_zod24.z.string().nullable()
|
|
16688
18432
|
}).strict(),
|
|
16689
|
-
error:
|
|
16690
|
-
code:
|
|
16691
|
-
message:
|
|
18433
|
+
error: import_zod24.z.object({
|
|
18434
|
+
code: import_zod24.z.string(),
|
|
18435
|
+
message: import_zod24.z.string()
|
|
16692
18436
|
}).strict().nullable()
|
|
16693
18437
|
}).strict();
|
|
16694
|
-
SerpIntelligenceCaptureResponseSchema =
|
|
18438
|
+
SerpIntelligenceCaptureResponseSchema = import_zod24.z.object({
|
|
16695
18439
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
16696
|
-
attempts:
|
|
18440
|
+
attempts: import_zod24.z.array(SerpIntelligenceCaptureAttemptSchema),
|
|
16697
18441
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
16698
|
-
pageSnapshotArtifacts:
|
|
16699
|
-
billing:
|
|
16700
|
-
creditsUsed:
|
|
16701
|
-
requestId:
|
|
16702
|
-
jobId:
|
|
18442
|
+
pageSnapshotArtifacts: import_zod24.z.array(SerpPageSnapshotCaptureSchema),
|
|
18443
|
+
billing: import_zod24.z.object({
|
|
18444
|
+
creditsUsed: import_zod24.z.number().min(0).optional(),
|
|
18445
|
+
requestId: import_zod24.z.string().optional(),
|
|
18446
|
+
jobId: import_zod24.z.string().optional()
|
|
16703
18447
|
}).strict().optional()
|
|
16704
18448
|
}).strict();
|
|
16705
|
-
SerpIntelligencePageSnapshotsResponseSchema =
|
|
16706
|
-
pageSnapshotArtifacts:
|
|
16707
|
-
attempts:
|
|
18449
|
+
SerpIntelligencePageSnapshotsResponseSchema = import_zod24.z.object({
|
|
18450
|
+
pageSnapshotArtifacts: import_zod24.z.array(SerpPageSnapshotCaptureSchema),
|
|
18451
|
+
attempts: import_zod24.z.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
16708
18452
|
}).strict();
|
|
16709
18453
|
}
|
|
16710
18454
|
});
|
|
@@ -16948,11 +18692,11 @@ function pageSnapshotTargetsFromBody(body) {
|
|
|
16948
18692
|
sourcePosition: null
|
|
16949
18693
|
}));
|
|
16950
18694
|
}
|
|
16951
|
-
var
|
|
18695
|
+
var import_hono8, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, serpIntelligenceApp;
|
|
16952
18696
|
var init_serp_intelligence_routes = __esm({
|
|
16953
18697
|
"src/api/serp-intelligence-routes.ts"() {
|
|
16954
18698
|
"use strict";
|
|
16955
|
-
|
|
18699
|
+
import_hono8 = require("hono");
|
|
16956
18700
|
init_browser_service_env();
|
|
16957
18701
|
init_page_snapshot_extractor();
|
|
16958
18702
|
init_serp_capture_service();
|
|
@@ -16964,7 +18708,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
16964
18708
|
SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
16965
18709
|
POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
16966
18710
|
POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
16967
|
-
serpIntelligenceApp = new
|
|
18711
|
+
serpIntelligenceApp = new import_hono8.Hono();
|
|
16968
18712
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
16969
18713
|
serpIntelligenceApp.post("/capture", async (c) => {
|
|
16970
18714
|
void POST_CAPTURE_ROUTE_LABEL;
|
|
@@ -17082,132 +18826,152 @@ var PACKAGE_VERSION;
|
|
|
17082
18826
|
var init_version = __esm({
|
|
17083
18827
|
"src/version.ts"() {
|
|
17084
18828
|
"use strict";
|
|
17085
|
-
PACKAGE_VERSION = "0.2.
|
|
18829
|
+
PACKAGE_VERSION = "0.2.9";
|
|
17086
18830
|
}
|
|
17087
18831
|
});
|
|
17088
18832
|
|
|
17089
18833
|
// src/mcp/mcp-tool-schemas.ts
|
|
17090
|
-
var
|
|
18834
|
+
var import_zod25, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, CreditsInfoInputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
17091
18835
|
var init_mcp_tool_schemas = __esm({
|
|
17092
18836
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
17093
18837
|
"use strict";
|
|
17094
|
-
|
|
18838
|
+
import_zod25 = require("zod");
|
|
17095
18839
|
HarvestPaaInputSchema = {
|
|
17096
|
-
query:
|
|
17097
|
-
location:
|
|
17098
|
-
maxQuestions:
|
|
17099
|
-
gl:
|
|
17100
|
-
hl:
|
|
17101
|
-
device:
|
|
17102
|
-
proxyMode:
|
|
17103
|
-
proxyZip:
|
|
17104
|
-
debug:
|
|
18840
|
+
query: import_zod25.z.string().min(1).describe('Core search topic only. If the user says "best hvac company in Denver CO", use query="best hvac company" and location="Denver, CO". Do not include the location in query when it can be separated.'),
|
|
18841
|
+
location: import_zod25.z.string().optional().describe('City, region, or country for geo-targeted results, inferred from the user request when present, e.g. "Denver, CO", "Tokyo, Japan", "London, UK".'),
|
|
18842
|
+
maxQuestions: import_zod25.z.number().int().min(1).max(200).default(30).describe("Number of PAA questions to extract. Default 30. Maximum 200. Use 10 for quick probes, 30 for normal research, 100-200 when the user asks for everything/full/deep research. Larger harvests get a longer server time budget (151-200 questions \u2192 up to 280s). Credits are charged by extracted question; unused request hold is refunded."),
|
|
18843
|
+
gl: import_zod25.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
|
|
18844
|
+
hl: import_zod25.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
|
|
18845
|
+
device: import_zod25.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
18846
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
|
|
18847
|
+
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
18848
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
|
|
17105
18849
|
};
|
|
17106
18850
|
ExtractUrlInputSchema = {
|
|
17107
|
-
url:
|
|
17108
|
-
screenshot:
|
|
17109
|
-
screenshotDevice:
|
|
17110
|
-
extractBranding:
|
|
17111
|
-
downloadMedia:
|
|
17112
|
-
mediaTypes:
|
|
17113
|
-
allowLocal:
|
|
18851
|
+
url: import_zod25.z.string().url().describe("Public http/https URL to extract. Use this when the user provides one specific page URL."),
|
|
18852
|
+
screenshot: import_zod25.z.boolean().default(false).describe("Also capture a full-page screenshot of the URL. Saved to ~/Downloads/mcp-scraper/screenshots/ and returned inline. Use when the user asks to see or capture the page visually."),
|
|
18853
|
+
screenshotDevice: import_zod25.z.enum(["desktop", "mobile"]).default("desktop").describe("Viewport for screenshot. desktop = 1440\xD7900. mobile = 390\xD7844. Default desktop."),
|
|
18854
|
+
extractBranding: import_zod25.z.boolean().default(false).describe("Extract brand colors, fonts, logo, and favicon using a rendered browser session. Returns colorScheme (light/dark), colors (primary/accent/background/text/heading as hex), fonts (heading/body family names), and assets (logo URL, favicon URL). Use when the user asks about brand colors, site theme, or brand assets."),
|
|
18855
|
+
downloadMedia: import_zod25.z.boolean().default(false).describe("Extract and download all page media (images, video, audio) to ~/Downloads/mcp-scraper/media/. Ad networks, tracking pixels, and noise URLs are filtered automatically. Use when the user asks to download or harvest assets from a page."),
|
|
18856
|
+
mediaTypes: import_zod25.z.array(import_zod25.z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download. Default all three."),
|
|
18857
|
+
allowLocal: import_zod25.z.boolean().default(false).describe("Allow localhost and private-network URLs. For local development only.")
|
|
17114
18858
|
};
|
|
17115
18859
|
MapSiteUrlsInputSchema = {
|
|
17116
|
-
url:
|
|
17117
|
-
maxUrls:
|
|
18860
|
+
url: import_zod25.z.string().url().describe("Public website URL or domain to crawl for internal URLs. Use before extract_site when the user asks to audit/map/crawl a site."),
|
|
18861
|
+
maxUrls: import_zod25.z.number().int().min(1).max(500).optional().describe("Maximum URLs to discover. Use 100 for normal maps, higher when the user asks for a full inventory.")
|
|
17118
18862
|
};
|
|
17119
18863
|
ExtractSiteInputSchema = {
|
|
17120
|
-
url:
|
|
17121
|
-
maxPages:
|
|
18864
|
+
url: import_zod25.z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
|
|
18865
|
+
maxPages: import_zod25.z.number().int().min(1).max(50).optional().describe("Maximum pages to extract. Use 50 when the user asks for full results or a complete crawl within MCP limits.")
|
|
17122
18866
|
};
|
|
17123
18867
|
YoutubeHarvestInputSchema = {
|
|
17124
|
-
mode:
|
|
17125
|
-
query:
|
|
17126
|
-
channelHandle:
|
|
17127
|
-
maxVideos:
|
|
18868
|
+
mode: import_zod25.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
|
|
18869
|
+
query: import_zod25.z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
|
|
18870
|
+
channelHandle: import_zod25.z.string().optional().describe("YouTube channel handle, channel ID, or URL. Examples: @mkbhd, UC..., https://youtube.com/@mkbhd."),
|
|
18871
|
+
maxVideos: import_zod25.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50. Increase when user asks for full channel/history.")
|
|
17128
18872
|
};
|
|
17129
18873
|
YoutubeTranscribeInputSchema = {
|
|
17130
|
-
videoId:
|
|
18874
|
+
videoId: import_zod25.z.string().min(1).describe("YouTube video ID, e.g. dQw4w9WgXcQ")
|
|
17131
18875
|
};
|
|
17132
18876
|
FacebookPageIntelInputSchema = {
|
|
17133
|
-
pageId:
|
|
17134
|
-
libraryId:
|
|
17135
|
-
query:
|
|
17136
|
-
maxAds:
|
|
17137
|
-
country:
|
|
18877
|
+
pageId: import_zod25.z.string().optional(),
|
|
18878
|
+
libraryId: import_zod25.z.string().optional(),
|
|
18879
|
+
query: import_zod25.z.string().optional().describe("Advertiser or brand name when pageId/libraryId is not known. One of pageId, libraryId, or query is required."),
|
|
18880
|
+
maxAds: import_zod25.z.number().int().min(1).max(200).default(50),
|
|
18881
|
+
country: import_zod25.z.string().length(2).default("US")
|
|
17138
18882
|
};
|
|
17139
18883
|
FacebookAdSearchInputSchema = {
|
|
17140
|
-
query:
|
|
17141
|
-
country:
|
|
17142
|
-
maxResults:
|
|
18884
|
+
query: import_zod25.z.string().min(1).describe("Advertiser, brand, competitor, niche, or keyword to search in Facebook Ad Library."),
|
|
18885
|
+
country: import_zod25.z.string().length(2).default("US"),
|
|
18886
|
+
maxResults: import_zod25.z.number().int().min(1).max(20).default(10)
|
|
17143
18887
|
};
|
|
17144
18888
|
FacebookAdTranscribeInputSchema = {
|
|
17145
|
-
videoUrl:
|
|
18889
|
+
videoUrl: import_zod25.z.string().url().describe("Facebook CDN video URL from a facebook_page_intel result")
|
|
17146
18890
|
};
|
|
17147
18891
|
MapsPlaceIntelInputSchema = {
|
|
17148
|
-
businessName:
|
|
17149
|
-
location:
|
|
17150
|
-
gl:
|
|
17151
|
-
hl:
|
|
17152
|
-
includeReviews:
|
|
17153
|
-
maxReviews:
|
|
18892
|
+
businessName: import_zod25.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
18893
|
+
location: import_zod25.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
|
|
18894
|
+
gl: import_zod25.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
18895
|
+
hl: import_zod25.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
18896
|
+
includeReviews: import_zod25.z.boolean().default(false).describe("Whether to fetch individual review cards"),
|
|
18897
|
+
maxReviews: import_zod25.z.number().int().min(1).max(500).default(50).describe("Max review cards to return (requires includeReviews: true)")
|
|
17154
18898
|
};
|
|
17155
18899
|
MapsSearchInputSchema = {
|
|
17156
|
-
query:
|
|
17157
|
-
location:
|
|
17158
|
-
gl:
|
|
17159
|
-
hl:
|
|
17160
|
-
maxResults:
|
|
17161
|
-
proxyMode:
|
|
17162
|
-
proxyZip:
|
|
17163
|
-
debug:
|
|
18900
|
+
query: import_zod25.z.string().min(1).describe('Business category, niche, keyword, or search term. If the user says "roofers in Denver CO", use query="roofers" and location="Denver, CO". Do not put the location here when it can be separated.'),
|
|
18901
|
+
location: import_zod25.z.string().optional().describe('City, region, country, or service area for the Maps search, e.g. "Denver, CO". Infer from the user request when present.'),
|
|
18902
|
+
gl: import_zod25.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
18903
|
+
hl: import_zod25.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
18904
|
+
maxResults: import_zod25.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
|
|
18905
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state Maps searches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts. Use configured for the server proxy ID, and none only for local direct-network debugging."),
|
|
18906
|
+
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
|
|
18907
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
|
|
17164
18908
|
};
|
|
17165
18909
|
DirectoryWorkflowInputSchema = {
|
|
17166
|
-
query:
|
|
17167
|
-
state:
|
|
17168
|
-
minPopulation:
|
|
17169
|
-
populationYear:
|
|
17170
|
-
maxCities:
|
|
17171
|
-
maxResultsPerCity:
|
|
17172
|
-
concurrency:
|
|
17173
|
-
includeZipGroups:
|
|
17174
|
-
usZipsCsvPath:
|
|
17175
|
-
saveCsv:
|
|
17176
|
-
proxyMode:
|
|
17177
|
-
proxyZip:
|
|
17178
|
-
debug:
|
|
18910
|
+
query: import_zod25.z.string().min(1).describe("Business category, niche, or keyword to search on Google Maps for every selected market, e.g. roofers, dentists, med spas. Do not include the city here."),
|
|
18911
|
+
state: import_zod25.z.string().min(2).default("TN").describe("US state abbreviation or state name used to select Census places, e.g. TN or Tennessee."),
|
|
18912
|
+
minPopulation: import_zod25.z.number().int().min(0).default(1e5).describe('Minimum Census place population for market selection. Use 100000 for "cities above 100k population".'),
|
|
18913
|
+
populationYear: import_zod25.z.number().int().min(2020).max(2025).default(2025).describe("Census population estimate year from the 2020-2025 Population Estimates Program city/place dataset."),
|
|
18914
|
+
maxCities: import_zod25.z.number().int().min(1).max(100).default(25).describe("Maximum number of markets to process after sorting by population descending."),
|
|
18915
|
+
maxResultsPerCity: import_zod25.z.number().int().min(1).max(50).default(50).describe("Google Maps business/profile candidates to collect for each city. Maximum 50."),
|
|
18916
|
+
concurrency: import_zod25.z.number().int().min(1).max(5).default(5).describe("How many city Maps searches to run in parallel. Use 5 for broad directory batches unless debugging."),
|
|
18917
|
+
includeZipGroups: import_zod25.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
|
|
18918
|
+
usZipsCsvPath: import_zod25.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
18919
|
+
saveCsv: import_zod25.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
|
|
18920
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode for every city Maps search. Use location by default for US city/state batches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts per city. Use configured for the server proxy ID, and none only for local direct-network debugging."),
|
|
18921
|
+
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its Lead Magician ZIP group or city/state location."),
|
|
18922
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
|
|
17179
18923
|
};
|
|
17180
|
-
RankTrackerModeSchema =
|
|
18924
|
+
RankTrackerModeSchema = import_zod25.z.enum(["maps", "organic", "ai_overview", "paa"]);
|
|
17181
18925
|
RankTrackerBlueprintInputSchema = {
|
|
17182
|
-
projectName:
|
|
17183
|
-
targetDomain:
|
|
17184
|
-
targetBusinessName:
|
|
17185
|
-
trackingModes:
|
|
17186
|
-
keywords:
|
|
17187
|
-
locations:
|
|
17188
|
-
competitors:
|
|
17189
|
-
database:
|
|
17190
|
-
scheduleCadence:
|
|
17191
|
-
customCron:
|
|
17192
|
-
timezone:
|
|
17193
|
-
includeCron:
|
|
17194
|
-
includeDashboard:
|
|
17195
|
-
includeAlerts:
|
|
17196
|
-
notes:
|
|
18926
|
+
projectName: import_zod25.z.string().min(1).optional().describe("Optional name for the rank tracker project, client, or campaign."),
|
|
18927
|
+
targetDomain: import_zod25.z.string().min(1).optional().describe("Primary domain to track in organic results, AI Overview citations, and PAA sources, e.g. example.com."),
|
|
18928
|
+
targetBusinessName: import_zod25.z.string().min(1).optional().describe("Primary Google Business Profile or brand/business name to match in Maps results. Required for reliable Maps rank tracking."),
|
|
18929
|
+
trackingModes: import_zod25.z.array(RankTrackerModeSchema).min(1).max(4).default(["maps", "organic", "ai_overview", "paa"]).describe("Rank tracker surfaces to build. maps uses directory_workflow/maps_search. organic uses search_serp. ai_overview uses search_serp/harvest_paa. paa uses harvest_paa source presence."),
|
|
18930
|
+
keywords: import_zod25.z.array(import_zod25.z.string().min(1)).max(200).default([]).describe("Seed keywords or service queries to track. Leave empty when the downstream AI should create the keyword table from user input."),
|
|
18931
|
+
locations: import_zod25.z.array(import_zod25.z.string().min(1)).max(100).default([]).describe("Markets, cities, ZIPs, or service areas to track. Use city/state strings like Denver, CO for localized SERP and Maps checks."),
|
|
18932
|
+
competitors: import_zod25.z.array(import_zod25.z.string().min(1)).max(100).default([]).describe("Optional competitor domains or business names to persist as comparison targets."),
|
|
18933
|
+
database: import_zod25.z.enum(["postgres", "neon", "supabase", "sqlite", "mysql"]).default("postgres").describe("Database family the downstream AI should target when generating migrations."),
|
|
18934
|
+
scheduleCadence: import_zod25.z.enum(["daily", "weekly", "monthly", "custom"]).default("weekly").describe("Default recurring rank check cadence for the generated cron/heartbeat plan."),
|
|
18935
|
+
customCron: import_zod25.z.string().min(1).optional().describe("Cron expression to use when scheduleCadence is custom."),
|
|
18936
|
+
timezone: import_zod25.z.string().min(1).default("UTC").describe("IANA timezone for scheduled rank checks, e.g. America/Denver."),
|
|
18937
|
+
includeCron: import_zod25.z.boolean().default(true).describe("Include a cron or heartbeat worker plan. Keep true for production rank trackers."),
|
|
18938
|
+
includeDashboard: import_zod25.z.boolean().default(true).describe("Include dashboard/reporting requirements in the generated prompt."),
|
|
18939
|
+
includeAlerts: import_zod25.z.boolean().default(true).describe("Include alert rules for rank movement and SERP feature gains/losses."),
|
|
18940
|
+
notes: import_zod25.z.string().max(4e3).optional().describe("Extra product, client, stack, or hosting requirements to include in the implementation prompt.")
|
|
17197
18941
|
};
|
|
17198
|
-
NullableString =
|
|
18942
|
+
NullableString = import_zod25.z.string().nullable();
|
|
18943
|
+
MapsSearchAttemptOutput = import_zod25.z.object({
|
|
18944
|
+
attemptNumber: import_zod25.z.number().int().min(1),
|
|
18945
|
+
maxAttempts: import_zod25.z.number().int().min(1),
|
|
18946
|
+
status: import_zod25.z.enum(["ok", "failed"]),
|
|
18947
|
+
outcome: import_zod25.z.string(),
|
|
18948
|
+
willRetry: import_zod25.z.boolean(),
|
|
18949
|
+
durationMs: import_zod25.z.number().int().min(0),
|
|
18950
|
+
resultCount: import_zod25.z.number().int().min(0),
|
|
18951
|
+
error: NullableString,
|
|
18952
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]),
|
|
18953
|
+
proxyResolutionSource: import_zod25.z.enum(["disabled", "location_reused", "location_created", "configured_fallback", "unavailable"]).nullable(),
|
|
18954
|
+
proxyIdSuffix: NullableString,
|
|
18955
|
+
proxyTargetLevel: import_zod25.z.enum(["zip", "city", "state"]).nullable(),
|
|
18956
|
+
proxyTargetLocation: NullableString,
|
|
18957
|
+
proxyTargetZip: NullableString,
|
|
18958
|
+
browserSessionIdSuffix: NullableString,
|
|
18959
|
+
observedIp: NullableString,
|
|
18960
|
+
observedCity: NullableString,
|
|
18961
|
+
observedRegion: NullableString
|
|
18962
|
+
});
|
|
17199
18963
|
MapsSearchOutputSchema = {
|
|
17200
|
-
query:
|
|
17201
|
-
location:
|
|
17202
|
-
searchQuery:
|
|
17203
|
-
searchUrl:
|
|
17204
|
-
extractedAt:
|
|
17205
|
-
requestedMaxResults:
|
|
17206
|
-
resultCount:
|
|
17207
|
-
results:
|
|
17208
|
-
position:
|
|
17209
|
-
name:
|
|
17210
|
-
placeUrl:
|
|
18964
|
+
query: import_zod25.z.string(),
|
|
18965
|
+
location: import_zod25.z.string().nullable(),
|
|
18966
|
+
searchQuery: import_zod25.z.string(),
|
|
18967
|
+
searchUrl: import_zod25.z.string().url(),
|
|
18968
|
+
extractedAt: import_zod25.z.string(),
|
|
18969
|
+
requestedMaxResults: import_zod25.z.number().int().min(1).max(50),
|
|
18970
|
+
resultCount: import_zod25.z.number().int().min(0).max(50),
|
|
18971
|
+
results: import_zod25.z.array(import_zod25.z.object({
|
|
18972
|
+
position: import_zod25.z.number().int().min(1),
|
|
18973
|
+
name: import_zod25.z.string(),
|
|
18974
|
+
placeUrl: import_zod25.z.string().url(),
|
|
17211
18975
|
cid: NullableString,
|
|
17212
18976
|
cidDecimal: NullableString,
|
|
17213
18977
|
rating: NullableString,
|
|
@@ -17218,14 +18982,15 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17218
18982
|
hoursStatus: NullableString,
|
|
17219
18983
|
websiteUrl: NullableString,
|
|
17220
18984
|
directionsUrl: NullableString,
|
|
17221
|
-
metadata:
|
|
18985
|
+
metadata: import_zod25.z.array(import_zod25.z.string())
|
|
17222
18986
|
})),
|
|
17223
|
-
|
|
18987
|
+
attempts: import_zod25.z.array(MapsSearchAttemptOutput),
|
|
18988
|
+
durationMs: import_zod25.z.number().int().min(0)
|
|
17224
18989
|
};
|
|
17225
|
-
DirectoryMapsBusinessOutput =
|
|
17226
|
-
position:
|
|
17227
|
-
name:
|
|
17228
|
-
placeUrl:
|
|
18990
|
+
DirectoryMapsBusinessOutput = import_zod25.z.object({
|
|
18991
|
+
position: import_zod25.z.number().int().min(1),
|
|
18992
|
+
name: import_zod25.z.string(),
|
|
18993
|
+
placeUrl: import_zod25.z.string().url(),
|
|
17229
18994
|
cid: NullableString,
|
|
17230
18995
|
cidDecimal: NullableString,
|
|
17231
18996
|
rating: NullableString,
|
|
@@ -17236,112 +19001,113 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17236
19001
|
hoursStatus: NullableString,
|
|
17237
19002
|
websiteUrl: NullableString,
|
|
17238
19003
|
directionsUrl: NullableString,
|
|
17239
|
-
metadata:
|
|
19004
|
+
metadata: import_zod25.z.array(import_zod25.z.string())
|
|
17240
19005
|
});
|
|
17241
19006
|
DirectoryWorkflowOutputSchema = {
|
|
17242
|
-
query:
|
|
17243
|
-
state:
|
|
17244
|
-
minPopulation:
|
|
17245
|
-
populationYear:
|
|
17246
|
-
maxResultsPerCity:
|
|
17247
|
-
concurrency:
|
|
17248
|
-
censusSourceUrl:
|
|
19007
|
+
query: import_zod25.z.string(),
|
|
19008
|
+
state: import_zod25.z.string(),
|
|
19009
|
+
minPopulation: import_zod25.z.number().int().min(0),
|
|
19010
|
+
populationYear: import_zod25.z.number().int().min(2020).max(2025),
|
|
19011
|
+
maxResultsPerCity: import_zod25.z.number().int().min(1).max(50),
|
|
19012
|
+
concurrency: import_zod25.z.number().int().min(1).max(5),
|
|
19013
|
+
censusSourceUrl: import_zod25.z.string().url(),
|
|
17249
19014
|
usZipsSourcePath: NullableString,
|
|
17250
|
-
warnings:
|
|
17251
|
-
extractedAt:
|
|
17252
|
-
selectedCityCount:
|
|
17253
|
-
totalResultCount:
|
|
19015
|
+
warnings: import_zod25.z.array(import_zod25.z.string()),
|
|
19016
|
+
extractedAt: import_zod25.z.string(),
|
|
19017
|
+
selectedCityCount: import_zod25.z.number().int().min(0),
|
|
19018
|
+
totalResultCount: import_zod25.z.number().int().min(0),
|
|
17254
19019
|
csvPath: NullableString,
|
|
17255
|
-
cities:
|
|
17256
|
-
city:
|
|
17257
|
-
state:
|
|
17258
|
-
location:
|
|
17259
|
-
cityKey:
|
|
17260
|
-
censusName:
|
|
17261
|
-
population:
|
|
17262
|
-
populationYear:
|
|
17263
|
-
zips:
|
|
17264
|
-
counties:
|
|
17265
|
-
status:
|
|
19020
|
+
cities: import_zod25.z.array(import_zod25.z.object({
|
|
19021
|
+
city: import_zod25.z.string(),
|
|
19022
|
+
state: import_zod25.z.string(),
|
|
19023
|
+
location: import_zod25.z.string(),
|
|
19024
|
+
cityKey: import_zod25.z.string(),
|
|
19025
|
+
censusName: import_zod25.z.string(),
|
|
19026
|
+
population: import_zod25.z.number().int().min(0),
|
|
19027
|
+
populationYear: import_zod25.z.number().int().min(2020).max(2025),
|
|
19028
|
+
zips: import_zod25.z.array(import_zod25.z.string()),
|
|
19029
|
+
counties: import_zod25.z.array(import_zod25.z.string()),
|
|
19030
|
+
status: import_zod25.z.enum(["ok", "empty", "failed"]),
|
|
17266
19031
|
error: NullableString,
|
|
17267
|
-
resultCount:
|
|
17268
|
-
durationMs:
|
|
17269
|
-
|
|
19032
|
+
resultCount: import_zod25.z.number().int().min(0),
|
|
19033
|
+
durationMs: import_zod25.z.number().int().min(0),
|
|
19034
|
+
attempts: import_zod25.z.array(MapsSearchAttemptOutput),
|
|
19035
|
+
results: import_zod25.z.array(DirectoryMapsBusinessOutput)
|
|
17270
19036
|
})),
|
|
17271
|
-
durationMs:
|
|
19037
|
+
durationMs: import_zod25.z.number().int().min(0)
|
|
17272
19038
|
};
|
|
17273
|
-
RankTrackerToolPlanOutput =
|
|
17274
|
-
tool:
|
|
17275
|
-
purpose:
|
|
19039
|
+
RankTrackerToolPlanOutput = import_zod25.z.object({
|
|
19040
|
+
tool: import_zod25.z.string(),
|
|
19041
|
+
purpose: import_zod25.z.string()
|
|
17276
19042
|
});
|
|
17277
|
-
RankTrackerTableOutput =
|
|
17278
|
-
name:
|
|
17279
|
-
purpose:
|
|
17280
|
-
keyColumns:
|
|
19043
|
+
RankTrackerTableOutput = import_zod25.z.object({
|
|
19044
|
+
name: import_zod25.z.string(),
|
|
19045
|
+
purpose: import_zod25.z.string(),
|
|
19046
|
+
keyColumns: import_zod25.z.array(import_zod25.z.string())
|
|
17281
19047
|
});
|
|
17282
|
-
RankTrackerCronJobOutput =
|
|
17283
|
-
name:
|
|
17284
|
-
purpose:
|
|
17285
|
-
modes:
|
|
17286
|
-
recommendedTools:
|
|
19048
|
+
RankTrackerCronJobOutput = import_zod25.z.object({
|
|
19049
|
+
name: import_zod25.z.string(),
|
|
19050
|
+
purpose: import_zod25.z.string(),
|
|
19051
|
+
modes: import_zod25.z.array(RankTrackerModeSchema),
|
|
19052
|
+
recommendedTools: import_zod25.z.array(import_zod25.z.string())
|
|
17287
19053
|
});
|
|
17288
19054
|
RankTrackerBlueprintOutputSchema = {
|
|
17289
|
-
projectName:
|
|
19055
|
+
projectName: import_zod25.z.string(),
|
|
17290
19056
|
targetDomain: NullableString,
|
|
17291
19057
|
targetBusinessName: NullableString,
|
|
17292
|
-
trackingModes:
|
|
17293
|
-
database:
|
|
17294
|
-
recommendedTools:
|
|
17295
|
-
tables:
|
|
17296
|
-
cron:
|
|
17297
|
-
enabled:
|
|
17298
|
-
cadence:
|
|
17299
|
-
expression:
|
|
17300
|
-
timezone:
|
|
17301
|
-
jobs:
|
|
19058
|
+
trackingModes: import_zod25.z.array(RankTrackerModeSchema),
|
|
19059
|
+
database: import_zod25.z.string(),
|
|
19060
|
+
recommendedTools: import_zod25.z.array(RankTrackerToolPlanOutput),
|
|
19061
|
+
tables: import_zod25.z.array(RankTrackerTableOutput),
|
|
19062
|
+
cron: import_zod25.z.object({
|
|
19063
|
+
enabled: import_zod25.z.boolean(),
|
|
19064
|
+
cadence: import_zod25.z.string(),
|
|
19065
|
+
expression: import_zod25.z.string(),
|
|
19066
|
+
timezone: import_zod25.z.string(),
|
|
19067
|
+
jobs: import_zod25.z.array(RankTrackerCronJobOutput)
|
|
17302
19068
|
}),
|
|
17303
|
-
metrics:
|
|
17304
|
-
implementationPrompt:
|
|
19069
|
+
metrics: import_zod25.z.array(import_zod25.z.string()),
|
|
19070
|
+
implementationPrompt: import_zod25.z.string()
|
|
17305
19071
|
};
|
|
17306
|
-
OrganicResultOutput =
|
|
17307
|
-
position:
|
|
17308
|
-
title:
|
|
17309
|
-
url:
|
|
17310
|
-
domain:
|
|
19072
|
+
OrganicResultOutput = import_zod25.z.object({
|
|
19073
|
+
position: import_zod25.z.number().int(),
|
|
19074
|
+
title: import_zod25.z.string(),
|
|
19075
|
+
url: import_zod25.z.string(),
|
|
19076
|
+
domain: import_zod25.z.string(),
|
|
17311
19077
|
snippet: NullableString
|
|
17312
19078
|
});
|
|
17313
|
-
AiOverviewOutput =
|
|
17314
|
-
detected:
|
|
19079
|
+
AiOverviewOutput = import_zod25.z.object({
|
|
19080
|
+
detected: import_zod25.z.boolean(),
|
|
17315
19081
|
text: NullableString
|
|
17316
19082
|
}).nullable();
|
|
17317
|
-
EntityIdsOutput =
|
|
17318
|
-
kgIds:
|
|
17319
|
-
cids:
|
|
17320
|
-
gcids:
|
|
19083
|
+
EntityIdsOutput = import_zod25.z.object({
|
|
19084
|
+
kgIds: import_zod25.z.array(import_zod25.z.string()),
|
|
19085
|
+
cids: import_zod25.z.array(import_zod25.z.string()),
|
|
19086
|
+
gcids: import_zod25.z.array(import_zod25.z.string())
|
|
17321
19087
|
}).nullable();
|
|
17322
19088
|
HarvestPaaOutputSchema = {
|
|
17323
|
-
query:
|
|
19089
|
+
query: import_zod25.z.string(),
|
|
17324
19090
|
location: NullableString,
|
|
17325
|
-
questionCount:
|
|
19091
|
+
questionCount: import_zod25.z.number().int().min(0),
|
|
17326
19092
|
completionStatus: NullableString,
|
|
17327
|
-
questions:
|
|
17328
|
-
question:
|
|
19093
|
+
questions: import_zod25.z.array(import_zod25.z.object({
|
|
19094
|
+
question: import_zod25.z.string(),
|
|
17329
19095
|
answer: NullableString,
|
|
17330
19096
|
sourceTitle: NullableString,
|
|
17331
19097
|
sourceSite: NullableString
|
|
17332
19098
|
})),
|
|
17333
|
-
organicResults:
|
|
19099
|
+
organicResults: import_zod25.z.array(OrganicResultOutput),
|
|
17334
19100
|
aiOverview: AiOverviewOutput,
|
|
17335
19101
|
entityIds: EntityIdsOutput,
|
|
17336
|
-
durationMs:
|
|
19102
|
+
durationMs: import_zod25.z.number().min(0).nullable()
|
|
17337
19103
|
};
|
|
17338
19104
|
SearchSerpOutputSchema = {
|
|
17339
|
-
query:
|
|
19105
|
+
query: import_zod25.z.string(),
|
|
17340
19106
|
location: NullableString,
|
|
17341
|
-
organicResults:
|
|
17342
|
-
localPack:
|
|
17343
|
-
position:
|
|
17344
|
-
name:
|
|
19107
|
+
organicResults: import_zod25.z.array(OrganicResultOutput),
|
|
19108
|
+
localPack: import_zod25.z.array(import_zod25.z.object({
|
|
19109
|
+
position: import_zod25.z.number().int(),
|
|
19110
|
+
name: import_zod25.z.string(),
|
|
17345
19111
|
rating: NullableString,
|
|
17346
19112
|
reviewCount: NullableString,
|
|
17347
19113
|
websiteUrl: NullableString
|
|
@@ -17350,31 +19116,31 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17350
19116
|
entityIds: EntityIdsOutput
|
|
17351
19117
|
};
|
|
17352
19118
|
ExtractUrlOutputSchema = {
|
|
17353
|
-
url:
|
|
19119
|
+
url: import_zod25.z.string(),
|
|
17354
19120
|
title: NullableString,
|
|
17355
|
-
headings:
|
|
17356
|
-
level:
|
|
17357
|
-
text:
|
|
19121
|
+
headings: import_zod25.z.array(import_zod25.z.object({
|
|
19122
|
+
level: import_zod25.z.number().int(),
|
|
19123
|
+
text: import_zod25.z.string()
|
|
17358
19124
|
})),
|
|
17359
|
-
schemaBlockCount:
|
|
19125
|
+
schemaBlockCount: import_zod25.z.number().int().min(0),
|
|
17360
19126
|
entityName: NullableString,
|
|
17361
|
-
entityTypes:
|
|
17362
|
-
napScore:
|
|
17363
|
-
missingSchemaFields:
|
|
19127
|
+
entityTypes: import_zod25.z.array(import_zod25.z.string()),
|
|
19128
|
+
napScore: import_zod25.z.number().nullable(),
|
|
19129
|
+
missingSchemaFields: import_zod25.z.array(import_zod25.z.string()),
|
|
17364
19130
|
screenshotSaved: NullableString
|
|
17365
19131
|
};
|
|
17366
19132
|
ExtractSiteOutputSchema = {
|
|
17367
|
-
url:
|
|
17368
|
-
pageCount:
|
|
17369
|
-
pages:
|
|
17370
|
-
url:
|
|
19133
|
+
url: import_zod25.z.string(),
|
|
19134
|
+
pageCount: import_zod25.z.number().int().min(0),
|
|
19135
|
+
pages: import_zod25.z.array(import_zod25.z.object({
|
|
19136
|
+
url: import_zod25.z.string(),
|
|
17371
19137
|
title: NullableString,
|
|
17372
|
-
schemaTypes:
|
|
19138
|
+
schemaTypes: import_zod25.z.array(import_zod25.z.string())
|
|
17373
19139
|
})),
|
|
17374
|
-
durationMs:
|
|
19140
|
+
durationMs: import_zod25.z.number().min(0)
|
|
17375
19141
|
};
|
|
17376
19142
|
MapsPlaceIntelOutputSchema = {
|
|
17377
|
-
name:
|
|
19143
|
+
name: import_zod25.z.string(),
|
|
17378
19144
|
rating: NullableString,
|
|
17379
19145
|
reviewCount: NullableString,
|
|
17380
19146
|
category: NullableString,
|
|
@@ -17386,60 +19152,60 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17386
19152
|
kgmid: NullableString,
|
|
17387
19153
|
cidDecimal: NullableString,
|
|
17388
19154
|
cidUrl: NullableString,
|
|
17389
|
-
lat:
|
|
17390
|
-
lng:
|
|
17391
|
-
reviewsStatus:
|
|
17392
|
-
reviewsCollected:
|
|
17393
|
-
reviewTopics:
|
|
17394
|
-
label:
|
|
17395
|
-
count:
|
|
19155
|
+
lat: import_zod25.z.number().nullable(),
|
|
19156
|
+
lng: import_zod25.z.number().nullable(),
|
|
19157
|
+
reviewsStatus: import_zod25.z.string(),
|
|
19158
|
+
reviewsCollected: import_zod25.z.number().int().min(0),
|
|
19159
|
+
reviewTopics: import_zod25.z.array(import_zod25.z.object({
|
|
19160
|
+
label: import_zod25.z.string(),
|
|
19161
|
+
count: import_zod25.z.string()
|
|
17396
19162
|
}))
|
|
17397
19163
|
};
|
|
17398
19164
|
CreditsInfoOutputSchema = {
|
|
17399
|
-
balanceCredits:
|
|
17400
|
-
matchedCost:
|
|
17401
|
-
label:
|
|
17402
|
-
credits:
|
|
17403
|
-
unit:
|
|
19165
|
+
balanceCredits: import_zod25.z.number().nullable(),
|
|
19166
|
+
matchedCost: import_zod25.z.object({
|
|
19167
|
+
label: import_zod25.z.string(),
|
|
19168
|
+
credits: import_zod25.z.number(),
|
|
19169
|
+
unit: import_zod25.z.string(),
|
|
17404
19170
|
notes: NullableString
|
|
17405
19171
|
}).nullable(),
|
|
17406
|
-
costs:
|
|
17407
|
-
key:
|
|
17408
|
-
label:
|
|
17409
|
-
credits:
|
|
17410
|
-
unit:
|
|
19172
|
+
costs: import_zod25.z.array(import_zod25.z.object({
|
|
19173
|
+
key: import_zod25.z.string(),
|
|
19174
|
+
label: import_zod25.z.string(),
|
|
19175
|
+
credits: import_zod25.z.number(),
|
|
19176
|
+
unit: import_zod25.z.string(),
|
|
17411
19177
|
notes: NullableString
|
|
17412
19178
|
})),
|
|
17413
|
-
ledger:
|
|
17414
|
-
createdAt:
|
|
17415
|
-
operation:
|
|
17416
|
-
credits:
|
|
19179
|
+
ledger: import_zod25.z.array(import_zod25.z.object({
|
|
19180
|
+
createdAt: import_zod25.z.string(),
|
|
19181
|
+
operation: import_zod25.z.string(),
|
|
19182
|
+
credits: import_zod25.z.number(),
|
|
17417
19183
|
description: NullableString
|
|
17418
19184
|
}))
|
|
17419
19185
|
};
|
|
17420
19186
|
MapSiteUrlsOutputSchema = {
|
|
17421
|
-
startUrl:
|
|
17422
|
-
totalFound:
|
|
17423
|
-
truncated:
|
|
17424
|
-
okCount:
|
|
17425
|
-
redirectCount:
|
|
17426
|
-
brokenCount:
|
|
17427
|
-
urls:
|
|
17428
|
-
url:
|
|
17429
|
-
status:
|
|
19187
|
+
startUrl: import_zod25.z.string(),
|
|
19188
|
+
totalFound: import_zod25.z.number().int().min(0),
|
|
19189
|
+
truncated: import_zod25.z.boolean(),
|
|
19190
|
+
okCount: import_zod25.z.number().int().min(0),
|
|
19191
|
+
redirectCount: import_zod25.z.number().int().min(0),
|
|
19192
|
+
brokenCount: import_zod25.z.number().int().min(0),
|
|
19193
|
+
urls: import_zod25.z.array(import_zod25.z.object({
|
|
19194
|
+
url: import_zod25.z.string(),
|
|
19195
|
+
status: import_zod25.z.number().int().nullable()
|
|
17430
19196
|
})),
|
|
17431
|
-
durationMs:
|
|
19197
|
+
durationMs: import_zod25.z.number().min(0)
|
|
17432
19198
|
};
|
|
17433
19199
|
YoutubeHarvestOutputSchema = {
|
|
17434
|
-
mode:
|
|
17435
|
-
videoCount:
|
|
17436
|
-
channel:
|
|
19200
|
+
mode: import_zod25.z.string(),
|
|
19201
|
+
videoCount: import_zod25.z.number().int().min(0),
|
|
19202
|
+
channel: import_zod25.z.object({
|
|
17437
19203
|
title: NullableString,
|
|
17438
19204
|
subscriberCount: NullableString
|
|
17439
19205
|
}).nullable(),
|
|
17440
|
-
videos:
|
|
17441
|
-
videoId:
|
|
17442
|
-
title:
|
|
19206
|
+
videos: import_zod25.z.array(import_zod25.z.object({
|
|
19207
|
+
videoId: import_zod25.z.string(),
|
|
19208
|
+
title: import_zod25.z.string(),
|
|
17443
19209
|
channelName: NullableString,
|
|
17444
19210
|
views: NullableString,
|
|
17445
19211
|
duration: NullableString,
|
|
@@ -17447,21 +19213,21 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17447
19213
|
}))
|
|
17448
19214
|
};
|
|
17449
19215
|
FacebookAdSearchOutputSchema = {
|
|
17450
|
-
query:
|
|
17451
|
-
advertiserCount:
|
|
17452
|
-
advertisers:
|
|
19216
|
+
query: import_zod25.z.string(),
|
|
19217
|
+
advertiserCount: import_zod25.z.number().int().min(0),
|
|
19218
|
+
advertisers: import_zod25.z.array(import_zod25.z.object({
|
|
17453
19219
|
name: NullableString,
|
|
17454
|
-
adCount:
|
|
19220
|
+
adCount: import_zod25.z.number().int().nullable(),
|
|
17455
19221
|
libraryId: NullableString
|
|
17456
19222
|
}))
|
|
17457
19223
|
};
|
|
17458
19224
|
FacebookPageIntelOutputSchema = {
|
|
17459
19225
|
advertiserName: NullableString,
|
|
17460
|
-
totalAds:
|
|
17461
|
-
activeCount:
|
|
17462
|
-
videoCount:
|
|
17463
|
-
imageCount:
|
|
17464
|
-
ads:
|
|
19226
|
+
totalAds: import_zod25.z.number().int().min(0),
|
|
19227
|
+
activeCount: import_zod25.z.number().int().min(0),
|
|
19228
|
+
videoCount: import_zod25.z.number().int().min(0),
|
|
19229
|
+
imageCount: import_zod25.z.number().int().min(0),
|
|
19230
|
+
ads: import_zod25.z.array(import_zod25.z.object({
|
|
17465
19231
|
libraryId: NullableString,
|
|
17466
19232
|
status: NullableString,
|
|
17467
19233
|
creativeType: NullableString,
|
|
@@ -17469,58 +19235,58 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17469
19235
|
cta: NullableString,
|
|
17470
19236
|
startDate: NullableString,
|
|
17471
19237
|
videoUrl: NullableString,
|
|
17472
|
-
variations:
|
|
19238
|
+
variations: import_zod25.z.number().int().nullable()
|
|
17473
19239
|
}))
|
|
17474
19240
|
};
|
|
17475
19241
|
CreditsInfoInputSchema = {
|
|
17476
|
-
item:
|
|
17477
|
-
includeLedger:
|
|
19242
|
+
item: import_zod25.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", or "YouTube transcription"'),
|
|
19243
|
+
includeLedger: import_zod25.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
17478
19244
|
};
|
|
17479
19245
|
SearchSerpInputSchema = {
|
|
17480
|
-
query:
|
|
17481
|
-
location:
|
|
17482
|
-
gl:
|
|
17483
|
-
hl:
|
|
17484
|
-
device:
|
|
17485
|
-
proxyMode:
|
|
17486
|
-
proxyZip:
|
|
17487
|
-
debug:
|
|
17488
|
-
pages:
|
|
19246
|
+
query: import_zod25.z.string().min(1).describe('Core search topic only. Separate location when possible. If user says "best dentist in Brooklyn NY serp", use query="best dentist" and location="Brooklyn, NY".'),
|
|
19247
|
+
location: import_zod25.z.string().optional().describe("City, region, or country for geo-targeted results, inferred from user request when present."),
|
|
19248
|
+
gl: import_zod25.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
19249
|
+
hl: import_zod25.z.string().default("en").describe("Google interface/content language inferred from user request."),
|
|
19250
|
+
device: import_zod25.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
19251
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
|
|
19252
|
+
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
19253
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
|
|
19254
|
+
pages: import_zod25.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
|
|
17489
19255
|
};
|
|
17490
19256
|
CaptureSerpSnapshotInputSchema = {
|
|
17491
|
-
query:
|
|
17492
|
-
location:
|
|
17493
|
-
gl:
|
|
17494
|
-
hl:
|
|
17495
|
-
device:
|
|
17496
|
-
proxyMode:
|
|
17497
|
-
proxyZip:
|
|
17498
|
-
pages:
|
|
17499
|
-
debug:
|
|
17500
|
-
includePageSnapshots:
|
|
17501
|
-
pageSnapshotLimit:
|
|
19257
|
+
query: import_zod25.z.string().min(1).describe("Core search query to capture as a structured SERP Intelligence snapshot. Separate the place into location when the user gives a city, region, country, or ZIP."),
|
|
19258
|
+
location: import_zod25.z.string().optional().describe("City, region, country, or service area used for localized Google results. MCP Scraper records location evidence; UULE alone is not proof of localization."),
|
|
19259
|
+
gl: import_zod25.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
|
|
19260
|
+
hl: import_zod25.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
19261
|
+
device: import_zod25.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
|
|
19262
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]).default("location").describe("Proxy behavior for capture. Use location for localized US residential evidence; it creates a fresh proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static residential proxy, and none only for direct-network debugging."),
|
|
19263
|
+
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting when a precise city-center or ZIP proxy is needed. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
19264
|
+
pages: import_zod25.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
|
|
19265
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
|
|
19266
|
+
includePageSnapshots: import_zod25.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs through the same product capture path."),
|
|
19267
|
+
pageSnapshotLimit: import_zod25.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots to capture when includePageSnapshots is true. Use 0 when only SERP evidence is needed.")
|
|
17502
19268
|
};
|
|
17503
19269
|
ScreenshotInputSchema = {
|
|
17504
|
-
url:
|
|
17505
|
-
device:
|
|
17506
|
-
allowLocal:
|
|
19270
|
+
url: import_zod25.z.string().url().describe("URL to capture as a full-page screenshot. Use http or https. Pass allowLocal: true to capture localhost or private-network URLs during development."),
|
|
19271
|
+
device: import_zod25.z.enum(["desktop", "mobile"]).default("desktop").describe("Viewport profile. desktop = 1440\xD7900. mobile = 390\xD7844. Use desktop by default; use mobile when the user asks for a mobile view."),
|
|
19272
|
+
allowLocal: import_zod25.z.boolean().default(false).describe("Allow localhost and private-network URLs (127.x, 192.168.x, 10.x, etc.). For local development only \u2014 not for production use.")
|
|
17507
19273
|
};
|
|
17508
19274
|
CaptureSerpPageSnapshotsInputSchema = {
|
|
17509
|
-
urls:
|
|
17510
|
-
targets:
|
|
17511
|
-
url:
|
|
17512
|
-
sourceKind:
|
|
17513
|
-
sourcePosition:
|
|
19275
|
+
urls: import_zod25.z.array(import_zod25.z.string().url()).min(1).max(25).describe("Public HTTP/HTTPS URLs to capture as SERP Intelligence page snapshots. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
19276
|
+
targets: import_zod25.z.array(import_zod25.z.object({
|
|
19277
|
+
url: import_zod25.z.string().url().describe("Public HTTP/HTTPS URL to capture."),
|
|
19278
|
+
sourceKind: import_zod25.z.enum(["organic", "ai_citation", "local_pack_website", "configured_target", "site_subject"]).default("configured_target").describe("Why this page is being captured for SERP Intelligence evidence."),
|
|
19279
|
+
sourcePosition: import_zod25.z.number().int().min(1).optional().describe("Ranking or citation position when the page came from SERP evidence.")
|
|
17514
19280
|
}).strict()).min(1).max(25).optional().describe("Structured page snapshot targets. Use this instead of urls when source kind or position should be preserved."),
|
|
17515
|
-
maxConcurrency:
|
|
17516
|
-
timeoutMs:
|
|
17517
|
-
debug:
|
|
19281
|
+
maxConcurrency: import_zod25.z.number().int().min(1).max(5).default(2).describe("Parallel page captures. Use 2 normally; higher values can increase proxy/browser pressure."),
|
|
19282
|
+
timeoutMs: import_zod25.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds. Increase for slow pages; timeout artifacts are returned as structured capture failures."),
|
|
19283
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics for page snapshot debugging. Use true for capture, network, or proxy troubleshooting.")
|
|
17518
19284
|
};
|
|
17519
19285
|
}
|
|
17520
19286
|
});
|
|
17521
19287
|
|
|
17522
19288
|
// src/mcp/rank-tracker-blueprint.ts
|
|
17523
|
-
function
|
|
19289
|
+
function normalizeDomain2(value) {
|
|
17524
19290
|
const trimmed = value?.trim();
|
|
17525
19291
|
if (!trimmed) return null;
|
|
17526
19292
|
try {
|
|
@@ -17782,7 +19548,7 @@ function buildPrompt(input, modes, tools, tables, jobs, metrics, expression, tar
|
|
|
17782
19548
|
}
|
|
17783
19549
|
function buildRankTrackerBlueprint(input) {
|
|
17784
19550
|
const trackingModes = uniqueModes(input.trackingModes);
|
|
17785
|
-
const targetDomain =
|
|
19551
|
+
const targetDomain = normalizeDomain2(input.targetDomain);
|
|
17786
19552
|
const targetBusinessName = input.targetBusinessName?.trim() || null;
|
|
17787
19553
|
const projectName = input.projectName?.trim() || "MCP Scraper Rank Tracker";
|
|
17788
19554
|
const database = input.database || "postgres";
|
|
@@ -17870,7 +19636,7 @@ function localPlanningToolAnnotations(title) {
|
|
|
17870
19636
|
function listSavedReports() {
|
|
17871
19637
|
try {
|
|
17872
19638
|
const dir = outputBaseDir();
|
|
17873
|
-
return (0,
|
|
19639
|
+
return (0, import_node_fs6.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs6.statSync)((0, import_node_path9.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
|
|
17874
19640
|
} catch {
|
|
17875
19641
|
return [];
|
|
17876
19642
|
}
|
|
@@ -17894,9 +19660,9 @@ function registerSavedReportResources(server) {
|
|
|
17894
19660
|
},
|
|
17895
19661
|
async (uri, variables) => {
|
|
17896
19662
|
const requested = Array.isArray(variables.filename) ? variables.filename[0] : variables.filename;
|
|
17897
|
-
const filename = (0,
|
|
19663
|
+
const filename = (0, import_node_path9.basename)(decodeURIComponent(String(requested ?? "")));
|
|
17898
19664
|
if (!filename.endsWith(".md")) throw new Error("Only saved .md reports can be read");
|
|
17899
|
-
const text = (0,
|
|
19665
|
+
const text = (0, import_node_fs6.readFileSync)((0, import_node_path9.join)(outputBaseDir(), filename), "utf8");
|
|
17900
19666
|
return { contents: [{ uri: uri.href, mimeType: "text/markdown", text }] };
|
|
17901
19667
|
}
|
|
17902
19668
|
);
|
|
@@ -17913,14 +19679,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
17913
19679
|
if (savesReports) registerSavedReportResources(server);
|
|
17914
19680
|
server.registerTool("harvest_paa", {
|
|
17915
19681
|
title: "Google PAA + SERP Harvest",
|
|
17916
|
-
description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). For US local SERPs, leave proxyMode as location so the service uses fresh residential proxy IDs across retries
|
|
19682
|
+
description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). For US local SERPs, leave proxyMode as location so the service uses fresh residential proxy IDs across retries, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear or the market evidence is wrong. Use maxQuestions 30 normally, 100-200 for "full", "deep", "all", or comprehensive research. Deep harvests above 100 questions can run for several minutes with no interim progress \u2014 warn the user before starting one and keep maxQuestions at or below 100 unless they explicitly want a deep harvest. Credits are charged by extracted question; unused request hold is refunded.'),
|
|
17917
19683
|
inputSchema: HarvestPaaInputSchema,
|
|
17918
19684
|
outputSchema: HarvestPaaOutputSchema,
|
|
17919
19685
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
17920
19686
|
}, async (input) => formatHarvestPaa(await executor.harvestPaa(input), input));
|
|
17921
19687
|
server.registerTool("search_serp", {
|
|
17922
19688
|
title: "Google SERP Lookup",
|
|
17923
|
-
description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. For US city/state rankings, keep proxyMode as location and pass proxyZip when a city-center ZIP is known; location mode uses fresh residential proxy IDs
|
|
19689
|
+
description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. For US city/state rankings, keep proxyMode as location and pass proxyZip when a city-center ZIP is known; location mode uses fresh residential proxy IDs, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear, the proxy tunnel fails, or location evidence is wrong."),
|
|
17924
19690
|
inputSchema: SearchSerpInputSchema,
|
|
17925
19691
|
outputSchema: SearchSerpOutputSchema,
|
|
17926
19692
|
annotations: liveWebToolAnnotations("Google SERP Lookup")
|
|
@@ -17988,14 +19754,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
17988
19754
|
}, async (input) => formatMapsPlaceIntel(await executor.mapsPlaceIntel(input), input));
|
|
17989
19755
|
server.registerTool("maps_search", {
|
|
17990
19756
|
title: "Google Maps Business Search",
|
|
17991
|
-
description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." For US city/state Maps searches, keep proxyMode as location so the browser service
|
|
19757
|
+
description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." For US city/state Maps searches, keep proxyMode as location so the browser service creates a fresh residential proxy ID and browser session for the market; retryable failures rotate to a new proxy and new browser session for up to 5 attempts. Pass proxyZip only when a specific ZIP or city-center ZIP is known. Returns up to 50 candidates with names, place URLs, CIDs when available, ratings, review counts, profile metadata, and sanitized attempt telemetry. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.'),
|
|
17992
19758
|
inputSchema: MapsSearchInputSchema,
|
|
17993
19759
|
outputSchema: MapsSearchOutputSchema,
|
|
17994
19760
|
annotations: liveWebToolAnnotations("Google Maps Business Search")
|
|
17995
19761
|
}, async (input) => formatMapsSearch(await executor.mapsSearch(input), input));
|
|
17996
19762
|
server.registerTool("directory_workflow", {
|
|
17997
19763
|
title: "Directory Workflow: Markets + Maps",
|
|
17998
|
-
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Use concurrency up to 5 for parallel city sessions. Keep proxyMode as location so each city
|
|
19764
|
+
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Use concurrency up to 5 for parallel city sessions. Keep proxyMode as location so each city search creates fresh residential proxy IDs and browser sessions; retryable city failures rotate to a new proxy and new browser session for up to 5 attempts. Saved CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. Structured city results include sanitized attempt telemetry. Use maps_place_intel only when a selected profile needs deeper review topics, profile review count confirmation, or review cards. For local Lead Magician ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath only in local/test mode.'),
|
|
17999
19765
|
inputSchema: DirectoryWorkflowInputSchema,
|
|
18000
19766
|
outputSchema: DirectoryWorkflowOutputSchema,
|
|
18001
19767
|
annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
|
|
@@ -18021,13 +19787,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
18021
19787
|
}
|
|
18022
19788
|
}, async (input) => formatCreditsInfo(await executor.creditsInfo(input), input));
|
|
18023
19789
|
}
|
|
18024
|
-
var import_mcp,
|
|
19790
|
+
var import_mcp, import_node_fs6, import_node_path9;
|
|
18025
19791
|
var init_paa_mcp_server = __esm({
|
|
18026
19792
|
"src/mcp/paa-mcp-server.ts"() {
|
|
18027
19793
|
"use strict";
|
|
18028
19794
|
import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
18029
|
-
|
|
18030
|
-
|
|
19795
|
+
import_node_fs6 = require("fs");
|
|
19796
|
+
import_node_path9 = require("path");
|
|
18031
19797
|
init_version();
|
|
18032
19798
|
init_mcp_response_formatter();
|
|
18033
19799
|
init_mcp_tool_schemas();
|
|
@@ -18183,7 +19949,7 @@ async function requireMcpCallerKey(c) {
|
|
|
18183
19949
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
18184
19950
|
server.registerTool("capture_serp_snapshot", {
|
|
18185
19951
|
title: "SERP Intelligence Snapshot",
|
|
18186
|
-
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized US residential evidence; location mode creates fresh proxy IDs across retries and
|
|
19952
|
+
description: "Capture a structured SERP Intelligence Google snapshot through POST /serp-intelligence/capture, the same product capture path used by Phoenix. Split query from location, infer gl/hl, use proxyMode location for localized US residential evidence; location mode creates fresh proxy IDs across retries, briefly waits for browser-service automatic CAPTCHA/challenge solving when Google blocks a page, and rotates to a new proxy/session if the challenge does not clear or location evidence is wrong. Use configured only for the static residential proxy, and none only for direct-network debugging. Set debug true when investigating location evidence, proxy behavior, CAPTCHA, or capture reliability.",
|
|
18187
19953
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
18188
19954
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
18189
19955
|
}, async (input) => executor.captureSerpSnapshot(input));
|
|
@@ -18194,11 +19960,11 @@ function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
|
18194
19960
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
18195
19961
|
}, async (input) => executor.captureSerpPageSnapshots(input));
|
|
18196
19962
|
}
|
|
18197
|
-
var
|
|
19963
|
+
var import_hono9, import_webStandardStreamableHttp, mcpApp;
|
|
18198
19964
|
var init_mcp_routes = __esm({
|
|
18199
19965
|
"src/mcp/mcp-routes.ts"() {
|
|
18200
19966
|
"use strict";
|
|
18201
|
-
|
|
19967
|
+
import_hono9 = require("hono");
|
|
18202
19968
|
import_webStandardStreamableHttp = require("@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js");
|
|
18203
19969
|
init_paa_mcp_server();
|
|
18204
19970
|
init_http_mcp_tool_executor();
|
|
@@ -18206,7 +19972,7 @@ var init_mcp_routes = __esm({
|
|
|
18206
19972
|
init_db();
|
|
18207
19973
|
init_mcp_tool_schemas();
|
|
18208
19974
|
configureReportSaving(false);
|
|
18209
|
-
mcpApp = new
|
|
19975
|
+
mcpApp = new import_hono9.Hono();
|
|
18210
19976
|
mcpApp.all("/", async (c) => {
|
|
18211
19977
|
try {
|
|
18212
19978
|
const keyOrError = await requireMcpCallerKey(c);
|
|
@@ -18278,7 +20044,7 @@ async function migrateBrowserAgent() {
|
|
|
18278
20044
|
}
|
|
18279
20045
|
async function createSessionRow(input) {
|
|
18280
20046
|
const db = getDb();
|
|
18281
|
-
const id = `bas_${(0,
|
|
20047
|
+
const id = `bas_${(0, import_node_crypto4.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
18282
20048
|
await db.execute({
|
|
18283
20049
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, last_action_at)
|
|
18284
20050
|
VALUES (?, ?, ?, ?, 'open', ?, ?, datetime('now'))`,
|
|
@@ -18339,7 +20105,7 @@ async function recordAction(input) {
|
|
|
18339
20105
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
18340
20106
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
18341
20107
|
args: [
|
|
18342
|
-
`baa_${(0,
|
|
20108
|
+
`baa_${(0, import_node_crypto4.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
18343
20109
|
input.sessionId,
|
|
18344
20110
|
input.type,
|
|
18345
20111
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -18379,11 +20145,11 @@ async function listReplayRows(sessionId) {
|
|
|
18379
20145
|
});
|
|
18380
20146
|
return res.rows;
|
|
18381
20147
|
}
|
|
18382
|
-
var
|
|
20148
|
+
var import_node_crypto4, _ready;
|
|
18383
20149
|
var init_browser_agent_db = __esm({
|
|
18384
20150
|
"src/api/browser-agent-db.ts"() {
|
|
18385
20151
|
"use strict";
|
|
18386
|
-
|
|
20152
|
+
import_node_crypto4 = require("crypto");
|
|
18387
20153
|
init_db();
|
|
18388
20154
|
_ready = false;
|
|
18389
20155
|
}
|
|
@@ -18761,7 +20527,7 @@ async function loadOpenSession(id, userId) {
|
|
|
18761
20527
|
return row;
|
|
18762
20528
|
}
|
|
18763
20529
|
function buildBrowserAgentRoutes() {
|
|
18764
|
-
const app2 = new
|
|
20530
|
+
const app2 = new import_hono10.Hono();
|
|
18765
20531
|
app2.use("*", async (c, next) => {
|
|
18766
20532
|
await migrateBrowserAgent();
|
|
18767
20533
|
return next();
|
|
@@ -19077,11 +20843,11 @@ function buildBrowserAgentRoutes() {
|
|
|
19077
20843
|
});
|
|
19078
20844
|
return app2;
|
|
19079
20845
|
}
|
|
19080
|
-
var
|
|
20846
|
+
var import_hono10, auth;
|
|
19081
20847
|
var init_browser_agent_routes = __esm({
|
|
19082
20848
|
"src/api/browser-agent-routes.ts"() {
|
|
19083
20849
|
"use strict";
|
|
19084
|
-
|
|
20850
|
+
import_hono10 = require("hono");
|
|
19085
20851
|
init_api_auth();
|
|
19086
20852
|
init_errors();
|
|
19087
20853
|
init_db();
|
|
@@ -19294,16 +21060,16 @@ var init_browser_agent_console = __esm({
|
|
|
19294
21060
|
});
|
|
19295
21061
|
|
|
19296
21062
|
// src/api/stripe-routes.ts
|
|
19297
|
-
var import_stripe,
|
|
21063
|
+
var import_stripe, import_hono11, stripe, stripeApp;
|
|
19298
21064
|
var init_stripe_routes = __esm({
|
|
19299
21065
|
"src/api/stripe-routes.ts"() {
|
|
19300
21066
|
"use strict";
|
|
19301
21067
|
import_stripe = __toESM(require("stripe"), 1);
|
|
19302
|
-
|
|
21068
|
+
import_hono11 = require("hono");
|
|
19303
21069
|
init_db();
|
|
19304
21070
|
init_rates();
|
|
19305
21071
|
stripe = new import_stripe.default(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
19306
|
-
stripeApp = new
|
|
21072
|
+
stripeApp = new import_hono11.Hono();
|
|
19307
21073
|
stripeApp.post("/webhooks", async (c) => {
|
|
19308
21074
|
const sig = c.req.header("stripe-signature");
|
|
19309
21075
|
const body = await c.req.text();
|
|
@@ -19404,31 +21170,31 @@ var init_site_audit_worker = __esm({
|
|
|
19404
21170
|
});
|
|
19405
21171
|
|
|
19406
21172
|
// src/api/billing-schemas.ts
|
|
19407
|
-
var
|
|
21173
|
+
var import_zod26, BillingCheckoutBodySchema, FreeCreditBreakdownSchema, BillingBalanceResponseSchema, MonthlyRefreshSweepResultSchema;
|
|
19408
21174
|
var init_billing_schemas = __esm({
|
|
19409
21175
|
"src/api/billing-schemas.ts"() {
|
|
19410
21176
|
"use strict";
|
|
19411
|
-
|
|
19412
|
-
BillingCheckoutBodySchema =
|
|
19413
|
-
priceId:
|
|
21177
|
+
import_zod26 = require("zod");
|
|
21178
|
+
BillingCheckoutBodySchema = import_zod26.z.object({
|
|
21179
|
+
priceId: import_zod26.z.string().min(1)
|
|
19414
21180
|
});
|
|
19415
|
-
FreeCreditBreakdownSchema =
|
|
19416
|
-
signup_grant_mc:
|
|
19417
|
-
monthly_refresh_mc:
|
|
19418
|
-
total_free_mc:
|
|
19419
|
-
signup_grant_credits:
|
|
19420
|
-
monthly_refresh_credits:
|
|
19421
|
-
total_free_credits:
|
|
21181
|
+
FreeCreditBreakdownSchema = import_zod26.z.object({
|
|
21182
|
+
signup_grant_mc: import_zod26.z.number().int().nonnegative(),
|
|
21183
|
+
monthly_refresh_mc: import_zod26.z.number().int().nonnegative(),
|
|
21184
|
+
total_free_mc: import_zod26.z.number().int().nonnegative(),
|
|
21185
|
+
signup_grant_credits: import_zod26.z.number().nonnegative(),
|
|
21186
|
+
monthly_refresh_credits: import_zod26.z.number().nonnegative(),
|
|
21187
|
+
total_free_credits: import_zod26.z.number().nonnegative()
|
|
19422
21188
|
});
|
|
19423
|
-
BillingBalanceResponseSchema =
|
|
19424
|
-
balance_mc:
|
|
19425
|
-
balance_credits:
|
|
21189
|
+
BillingBalanceResponseSchema = import_zod26.z.object({
|
|
21190
|
+
balance_mc: import_zod26.z.number().int().nonnegative(),
|
|
21191
|
+
balance_credits: import_zod26.z.number().nonnegative(),
|
|
19426
21192
|
free_credits: FreeCreditBreakdownSchema,
|
|
19427
|
-
ledger:
|
|
21193
|
+
ledger: import_zod26.z.array(import_zod26.z.any())
|
|
19428
21194
|
});
|
|
19429
|
-
MonthlyRefreshSweepResultSchema =
|
|
19430
|
-
usersRefreshed:
|
|
19431
|
-
totalMcGranted:
|
|
21195
|
+
MonthlyRefreshSweepResultSchema = import_zod26.z.object({
|
|
21196
|
+
usersRefreshed: import_zod26.z.number().int().nonnegative(),
|
|
21197
|
+
totalMcGranted: import_zod26.z.number().int().nonnegative()
|
|
19432
21198
|
});
|
|
19433
21199
|
}
|
|
19434
21200
|
});
|
|
@@ -19590,14 +21356,14 @@ function getSessionSecret() {
|
|
|
19590
21356
|
function safeEqualHex(a, b) {
|
|
19591
21357
|
if (a.length !== b.length) return false;
|
|
19592
21358
|
try {
|
|
19593
|
-
return (0,
|
|
21359
|
+
return (0, import_node_crypto5.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
19594
21360
|
} catch {
|
|
19595
21361
|
return false;
|
|
19596
21362
|
}
|
|
19597
21363
|
}
|
|
19598
21364
|
function signSession(userId) {
|
|
19599
21365
|
const payload = String(userId);
|
|
19600
|
-
const sig = (0,
|
|
21366
|
+
const sig = (0, import_node_crypto5.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
19601
21367
|
return `${payload}.${sig}`;
|
|
19602
21368
|
}
|
|
19603
21369
|
function verifySession(token) {
|
|
@@ -19605,16 +21371,16 @@ function verifySession(token) {
|
|
|
19605
21371
|
if (dot === -1) return null;
|
|
19606
21372
|
const payload = token.slice(0, dot);
|
|
19607
21373
|
const sig = token.slice(dot + 1);
|
|
19608
|
-
const expected = (0,
|
|
21374
|
+
const expected = (0, import_node_crypto5.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
19609
21375
|
if (!safeEqualHex(sig, expected)) return null;
|
|
19610
21376
|
const id = parseInt(payload);
|
|
19611
21377
|
return isNaN(id) ? null : id;
|
|
19612
21378
|
}
|
|
19613
|
-
var
|
|
21379
|
+
var import_node_crypto5, isProduction, secret;
|
|
19614
21380
|
var init_session = __esm({
|
|
19615
21381
|
"src/api/session.ts"() {
|
|
19616
21382
|
"use strict";
|
|
19617
|
-
|
|
21383
|
+
import_node_crypto5 = require("crypto");
|
|
19618
21384
|
isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
19619
21385
|
secret = () => getSessionSecret();
|
|
19620
21386
|
}
|
|
@@ -19830,7 +21596,7 @@ async function checkHarvestLimits(userId, email, extraSlots = 0) {
|
|
|
19830
21596
|
if (active >= limit) return { error: `You have ${active} job${active !== 1 ? "s" : ""} running. Your account allows ${limit} concurrent job${limit !== 1 ? "s" : ""}. Wait for one to finish or add a concurrency slot at mcpscraper.dev/billing.` };
|
|
19831
21597
|
return null;
|
|
19832
21598
|
}
|
|
19833
|
-
var import_resend,
|
|
21599
|
+
var import_resend, import_hono12, import_hono13, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION, BYPASS_EMAILS, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
|
|
19834
21600
|
var init_server = __esm({
|
|
19835
21601
|
"src/api/server.ts"() {
|
|
19836
21602
|
"use strict";
|
|
@@ -19847,8 +21613,8 @@ var init_server = __esm({
|
|
|
19847
21613
|
init_media_extractor();
|
|
19848
21614
|
init_site_mapper();
|
|
19849
21615
|
init_site_extractor();
|
|
19850
|
-
|
|
19851
|
-
|
|
21616
|
+
import_hono12 = require("hono");
|
|
21617
|
+
import_hono13 = require("inngest/hono");
|
|
19852
21618
|
init_client();
|
|
19853
21619
|
init_site_audit();
|
|
19854
21620
|
init_site_audit_routes();
|
|
@@ -19857,6 +21623,7 @@ var init_server = __esm({
|
|
|
19857
21623
|
init_facebook_ad_routes();
|
|
19858
21624
|
init_maps_routes();
|
|
19859
21625
|
init_directory_routes();
|
|
21626
|
+
init_workflow_routes();
|
|
19860
21627
|
init_serp_intelligence_routes();
|
|
19861
21628
|
init_mcp_routes();
|
|
19862
21629
|
init_browser_agent_routes();
|
|
@@ -19919,7 +21686,7 @@ var init_server = __esm({
|
|
|
19919
21686
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
19920
21687
|
return next();
|
|
19921
21688
|
});
|
|
19922
|
-
app = new
|
|
21689
|
+
app = new import_hono12.Hono();
|
|
19923
21690
|
STRIPE_API_VERSION = "2026-02-25.clover";
|
|
19924
21691
|
app.use("*", async (c, next) => {
|
|
19925
21692
|
await next();
|
|
@@ -20517,19 +22284,21 @@ var init_server = __esm({
|
|
|
20517
22284
|
}
|
|
20518
22285
|
const { drainQueue: drainQueue2 } = await Promise.resolve().then(() => (init_worker(), worker_exports));
|
|
20519
22286
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
22287
|
+
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
20520
22288
|
const [results, sweepResult] = await Promise.all([
|
|
20521
22289
|
drainQueue2(budget),
|
|
20522
22290
|
runMonthlyRefreshSweep()
|
|
20523
22291
|
]);
|
|
20524
|
-
return c.json({ drained: results.length, results, sweepResult });
|
|
22292
|
+
return c.json({ drained: results.length, results, sweepResult, workflowDispatch: workflowDispatchResult });
|
|
20525
22293
|
});
|
|
20526
|
-
app.on(["GET", "POST", "PUT"], "/api/inngest", (0,
|
|
22294
|
+
app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono13.serve)({ client: inngest, functions: [siteAuditFn] }));
|
|
20527
22295
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
20528
22296
|
app.route("/youtube", youtubeApp);
|
|
20529
22297
|
app.route("/screenshot", screenshotApp);
|
|
20530
22298
|
app.route("/facebook", facebookAdApp);
|
|
20531
22299
|
app.route("/maps", mapsApp);
|
|
20532
22300
|
app.route("/directory", directoryApp);
|
|
22301
|
+
app.route("/workflows", workflowApp);
|
|
20533
22302
|
app.route("/serp-intelligence", serpIntelligenceApp);
|
|
20534
22303
|
app.route("/mcp", mcpApp);
|
|
20535
22304
|
app.route("/agent", buildBrowserAgentRoutes());
|
|
@@ -20642,10 +22411,10 @@ var init_server = __esm({
|
|
|
20642
22411
|
});
|
|
20643
22412
|
|
|
20644
22413
|
// bin/api-server.ts
|
|
20645
|
-
var
|
|
22414
|
+
var import_node_fs7 = require("fs");
|
|
20646
22415
|
function loadDotEnv() {
|
|
20647
22416
|
try {
|
|
20648
|
-
for (const line of (0,
|
|
22417
|
+
for (const line of (0, import_node_fs7.readFileSync)(".env", "utf8").split("\n")) {
|
|
20649
22418
|
const eq = line.indexOf("=");
|
|
20650
22419
|
if (eq < 1 || line.trimStart().startsWith("#")) continue;
|
|
20651
22420
|
const k = line.slice(0, eq).trim();
|