mcp-scraper 0.2.8 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -3
- package/dist/bin/api-server.cjs +2073 -506
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1276 -0
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -0
- package/dist/bin/mcp-scraper-cli.d.cts +1 -0
- package/dist/bin/mcp-scraper-cli.d.ts +1 -0
- package/dist/bin/mcp-scraper-cli.js +476 -0
- package/dist/bin/mcp-scraper-cli.js.map +1 -0
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +3 -3
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +4 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +4 -1
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +3 -3
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-5HMOPP76.js → chunk-2OHP6WWF.js} +2 -2
- package/dist/{chunk-D4CJBZBY.js → chunk-7GCCOT3M.js} +372 -1
- package/dist/chunk-7GCCOT3M.js.map +1 -0
- package/dist/chunk-L6IS63WS.js +869 -0
- package/dist/chunk-L6IS63WS.js.map +1 -0
- package/dist/{chunk-IQOCZGJJ.js → chunk-ROS67BNV.js} +5 -5
- package/dist/chunk-WKWFO45O.js +7 -0
- package/dist/chunk-WKWFO45O.js.map +1 -0
- package/dist/{chunk-6NEXSNSA.js → chunk-YAKKRQRH.js} +4 -4
- package/dist/chunk-YAKKRQRH.js.map +1 -0
- package/dist/{db-YWCNHBLH.js → db-BVHYI57K.js} +38 -2
- package/dist/{server-BTTDFPSQ.js → server-USOIPC2I.js} +585 -266
- package/dist/server-USOIPC2I.js.map +1 -0
- package/dist/{worker-5O44YBF4.js → worker-TDJQ6TH3.js} +8 -8
- package/docs/specs/agent-ready-seo-packet-spec.md +237 -0
- package/docs/specs/cli-agent-wiring-spec.md +203 -0
- package/docs/specs/deferred-work-spec.md +12 -0
- package/docs/specs/local-competitive-audit-spec.md +312 -0
- package/docs/specs/scheduled-workflows-api-spec.md +304 -0
- package/docs/specs/seo-cli-growth-roadmap-spec.md +179 -0
- package/docs/specs/seo-workflow-runner-and-reports-spec.md +241 -0
- package/package.json +20 -19
- package/dist/chunk-6NEXSNSA.js.map +0 -1
- package/dist/chunk-D4CJBZBY.js.map +0 -1
- package/dist/chunk-I26QN7WQ.js +0 -7
- package/dist/chunk-I26QN7WQ.js.map +0 -1
- package/dist/server-BTTDFPSQ.js.map +0 -1
- /package/dist/{chunk-5HMOPP76.js.map → chunk-2OHP6WWF.js.map} +0 -0
- /package/dist/{chunk-IQOCZGJJ.js.map → chunk-ROS67BNV.js.map} +0 -0
- /package/dist/{db-YWCNHBLH.js.map → db-BVHYI57K.js.map} +0 -0
- /package/dist/{worker-5O44YBF4.js.map → worker-TDJQ6TH3.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -6913,6 +6913,7 @@ __export(db_exports, {
|
|
|
6913
6913
|
claimPendingKpoJob: () => claimPendingKpoJob,
|
|
6914
6914
|
completeJob: () => completeJob,
|
|
6915
6915
|
completeKpoJob: () => completeKpoJob,
|
|
6916
|
+
completeWorkflowRunRecord: () => completeWorkflowRunRecord,
|
|
6916
6917
|
consumePasswordResetToken: () => consumePasswordResetToken,
|
|
6917
6918
|
countActiveJobsForUser: () => countActiveJobsForUser,
|
|
6918
6919
|
countActiveUsers: () => countActiveUsers,
|
|
@@ -6921,11 +6922,15 @@ __export(db_exports, {
|
|
|
6921
6922
|
createPasswordResetToken: () => createPasswordResetToken,
|
|
6922
6923
|
createRunningJob: () => createRunningJob,
|
|
6923
6924
|
createUser: () => createUser,
|
|
6925
|
+
createWorkflowRun: () => createWorkflowRun,
|
|
6926
|
+
createWorkflowSchedule: () => createWorkflowSchedule,
|
|
6924
6927
|
creditMc: () => creditMc,
|
|
6925
6928
|
deactivateUser: () => deactivateUser,
|
|
6926
6929
|
debitMc: () => debitMc,
|
|
6930
|
+
deleteWorkflowSchedule: () => deleteWorkflowSchedule,
|
|
6927
6931
|
failJob: () => failJob,
|
|
6928
6932
|
failKpoJob: () => failKpoJob,
|
|
6933
|
+
failWorkflowRunRecord: () => failWorkflowRunRecord,
|
|
6929
6934
|
finishHarvestAttempt: () => finishHarvestAttempt,
|
|
6930
6935
|
generateApiKey: () => generateApiKey,
|
|
6931
6936
|
getDb: () => getDb,
|
|
@@ -6938,19 +6943,32 @@ __export(db_exports, {
|
|
|
6938
6943
|
getUserById: () => getUserById,
|
|
6939
6944
|
getUserByStripeCustomerId: () => getUserByStripeCustomerId,
|
|
6940
6945
|
getUserStats: () => getUserStats,
|
|
6946
|
+
getWorkflowArtifact: () => getWorkflowArtifact,
|
|
6947
|
+
getWorkflowRun: () => getWorkflowRun,
|
|
6948
|
+
getWorkflowRunByIdempotencyKey: () => getWorkflowRunByIdempotencyKey,
|
|
6949
|
+
getWorkflowSchedule: () => getWorkflowSchedule,
|
|
6941
6950
|
hashPassword: () => hashPassword,
|
|
6942
6951
|
ledgerExistsForOperation: () => ledgerExistsForOperation,
|
|
6943
6952
|
ledgerExistsForStripePI: () => ledgerExistsForStripePI,
|
|
6953
|
+
listDueWorkflowSchedules: () => listDueWorkflowSchedules,
|
|
6944
6954
|
listHarvestAttempts: () => listHarvestAttempts,
|
|
6945
6955
|
listJobs: () => listJobs,
|
|
6946
6956
|
listKpoJobs: () => listKpoJobs,
|
|
6947
6957
|
listRequestEvents: () => listRequestEvents,
|
|
6948
6958
|
listUsers: () => listUsers,
|
|
6959
|
+
listWorkflowArtifacts: () => listWorkflowArtifacts,
|
|
6960
|
+
listWorkflowRuns: () => listWorkflowRuns,
|
|
6961
|
+
listWorkflowSchedules: () => listWorkflowSchedules,
|
|
6949
6962
|
logKpoPhaseComplete: () => logKpoPhaseComplete,
|
|
6950
6963
|
logRequestEvent: () => logRequestEvent,
|
|
6964
|
+
markWorkflowRunRunning: () => markWorkflowRunRunning,
|
|
6965
|
+
markWorkflowScheduleRan: () => markWorkflowScheduleRan,
|
|
6951
6966
|
migrate: () => migrate,
|
|
6967
|
+
patchWorkflowSchedule: () => patchWorkflowSchedule,
|
|
6952
6968
|
reconcileBalanceMc: () => reconcileBalanceMc,
|
|
6953
6969
|
recordStripeEvent: () => recordStripeEvent,
|
|
6970
|
+
recordWorkflowWebhookDelivery: () => recordWorkflowWebhookDelivery,
|
|
6971
|
+
replaceWorkflowArtifacts: () => replaceWorkflowArtifacts,
|
|
6954
6972
|
revokeApiKey: () => revokeApiKey,
|
|
6955
6973
|
rotateApiKey: () => rotateApiKey,
|
|
6956
6974
|
setConcurrencySubId: () => setConcurrencySubId,
|
|
@@ -7162,6 +7180,75 @@ async function migrate() {
|
|
|
7162
7180
|
)
|
|
7163
7181
|
`);
|
|
7164
7182
|
await db.execute(`CREATE INDEX IF NOT EXISTS request_events_user_created_at ON request_events(user_id, created_at DESC)`);
|
|
7183
|
+
await db.execute(`
|
|
7184
|
+
CREATE TABLE IF NOT EXISTS workflow_schedules (
|
|
7185
|
+
id TEXT PRIMARY KEY,
|
|
7186
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
7187
|
+
workflow_id TEXT NOT NULL,
|
|
7188
|
+
name TEXT NOT NULL,
|
|
7189
|
+
input_json TEXT NOT NULL,
|
|
7190
|
+
cadence TEXT NOT NULL,
|
|
7191
|
+
cron_expr TEXT,
|
|
7192
|
+
timezone TEXT NOT NULL DEFAULT 'UTC',
|
|
7193
|
+
webhook_url TEXT,
|
|
7194
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
7195
|
+
next_run_at TEXT,
|
|
7196
|
+
last_run_at TEXT,
|
|
7197
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
7198
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
7199
|
+
)
|
|
7200
|
+
`);
|
|
7201
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_schedules_user_status ON workflow_schedules(user_id, status, created_at DESC)`);
|
|
7202
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_schedules_due ON workflow_schedules(status, next_run_at)`);
|
|
7203
|
+
await db.execute(`
|
|
7204
|
+
CREATE TABLE IF NOT EXISTS workflow_runs (
|
|
7205
|
+
id TEXT PRIMARY KEY,
|
|
7206
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
7207
|
+
schedule_id TEXT,
|
|
7208
|
+
workflow_id TEXT NOT NULL,
|
|
7209
|
+
input_json TEXT NOT NULL,
|
|
7210
|
+
status TEXT NOT NULL DEFAULT 'queued',
|
|
7211
|
+
manifest_json TEXT,
|
|
7212
|
+
error_message TEXT,
|
|
7213
|
+
queued_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
7214
|
+
started_at TEXT,
|
|
7215
|
+
completed_at TEXT,
|
|
7216
|
+
idempotency_key TEXT
|
|
7217
|
+
)
|
|
7218
|
+
`);
|
|
7219
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_user_created ON workflow_runs(user_id, queued_at DESC)`);
|
|
7220
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_runs_schedule ON workflow_runs(schedule_id, queued_at DESC)`);
|
|
7221
|
+
await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS workflow_runs_idempotency ON workflow_runs(idempotency_key) WHERE idempotency_key IS NOT NULL`);
|
|
7222
|
+
await db.execute(`
|
|
7223
|
+
CREATE TABLE IF NOT EXISTS workflow_artifacts (
|
|
7224
|
+
id TEXT PRIMARY KEY,
|
|
7225
|
+
run_id TEXT NOT NULL REFERENCES workflow_runs(id),
|
|
7226
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
7227
|
+
kind TEXT NOT NULL,
|
|
7228
|
+
label TEXT NOT NULL,
|
|
7229
|
+
path TEXT NOT NULL,
|
|
7230
|
+
content_type TEXT NOT NULL,
|
|
7231
|
+
bytes INTEGER,
|
|
7232
|
+
rows_count INTEGER,
|
|
7233
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
7234
|
+
)
|
|
7235
|
+
`);
|
|
7236
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_artifacts_run ON workflow_artifacts(run_id)`);
|
|
7237
|
+
await db.execute(`
|
|
7238
|
+
CREATE TABLE IF NOT EXISTS workflow_webhook_deliveries (
|
|
7239
|
+
id TEXT PRIMARY KEY,
|
|
7240
|
+
run_id TEXT NOT NULL REFERENCES workflow_runs(id),
|
|
7241
|
+
schedule_id TEXT,
|
|
7242
|
+
user_id INTEGER NOT NULL REFERENCES users(id),
|
|
7243
|
+
webhook_url TEXT NOT NULL,
|
|
7244
|
+
status TEXT NOT NULL,
|
|
7245
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
7246
|
+
last_error TEXT,
|
|
7247
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
7248
|
+
delivered_at TEXT
|
|
7249
|
+
)
|
|
7250
|
+
`);
|
|
7251
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS workflow_webhook_deliveries_run ON workflow_webhook_deliveries(run_id)`);
|
|
7165
7252
|
await db.execute(`
|
|
7166
7253
|
CREATE TABLE IF NOT EXISTS stripe_events (
|
|
7167
7254
|
event_id TEXT PRIMARY KEY,
|
|
@@ -7551,6 +7638,290 @@ async function listRequestEvents(userId, limit = 100) {
|
|
|
7551
7638
|
});
|
|
7552
7639
|
return res.rows.map((r) => rowToRequestEvent(r));
|
|
7553
7640
|
}
|
|
7641
|
+
function parseJsonRecordValue(value) {
|
|
7642
|
+
if (value == null) return {};
|
|
7643
|
+
try {
|
|
7644
|
+
const parsed = JSON.parse(String(value));
|
|
7645
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
|
7646
|
+
} catch {
|
|
7647
|
+
return {};
|
|
7648
|
+
}
|
|
7649
|
+
}
|
|
7650
|
+
function rowToWorkflowSchedule(row) {
|
|
7651
|
+
return {
|
|
7652
|
+
id: String(row.id),
|
|
7653
|
+
user_id: Number(row.user_id),
|
|
7654
|
+
workflow_id: String(row.workflow_id),
|
|
7655
|
+
name: String(row.name),
|
|
7656
|
+
input: parseJsonRecordValue(row.input_json),
|
|
7657
|
+
cadence: String(row.cadence),
|
|
7658
|
+
cron_expr: row.cron_expr != null ? String(row.cron_expr) : null,
|
|
7659
|
+
timezone: String(row.timezone ?? "UTC"),
|
|
7660
|
+
webhook_url: row.webhook_url != null ? String(row.webhook_url) : null,
|
|
7661
|
+
status: String(row.status),
|
|
7662
|
+
next_run_at: row.next_run_at != null ? String(row.next_run_at) : null,
|
|
7663
|
+
last_run_at: row.last_run_at != null ? String(row.last_run_at) : null,
|
|
7664
|
+
created_at: String(row.created_at),
|
|
7665
|
+
updated_at: String(row.updated_at)
|
|
7666
|
+
};
|
|
7667
|
+
}
|
|
7668
|
+
function rowToWorkflowRun(row) {
|
|
7669
|
+
return {
|
|
7670
|
+
id: String(row.id),
|
|
7671
|
+
user_id: Number(row.user_id),
|
|
7672
|
+
schedule_id: row.schedule_id != null ? String(row.schedule_id) : null,
|
|
7673
|
+
workflow_id: String(row.workflow_id),
|
|
7674
|
+
input: parseJsonRecordValue(row.input_json),
|
|
7675
|
+
status: String(row.status),
|
|
7676
|
+
manifest: row.manifest_json != null ? parseNullableJson(row.manifest_json) : null,
|
|
7677
|
+
error_message: row.error_message != null ? String(row.error_message) : null,
|
|
7678
|
+
queued_at: String(row.queued_at),
|
|
7679
|
+
started_at: row.started_at != null ? String(row.started_at) : null,
|
|
7680
|
+
completed_at: row.completed_at != null ? String(row.completed_at) : null,
|
|
7681
|
+
idempotency_key: row.idempotency_key != null ? String(row.idempotency_key) : null
|
|
7682
|
+
};
|
|
7683
|
+
}
|
|
7684
|
+
function rowToWorkflowArtifact(row) {
|
|
7685
|
+
return {
|
|
7686
|
+
id: String(row.id),
|
|
7687
|
+
run_id: String(row.run_id),
|
|
7688
|
+
user_id: Number(row.user_id),
|
|
7689
|
+
kind: String(row.kind),
|
|
7690
|
+
label: String(row.label),
|
|
7691
|
+
path: String(row.path),
|
|
7692
|
+
content_type: String(row.content_type),
|
|
7693
|
+
bytes: row.bytes != null ? Number(row.bytes) : null,
|
|
7694
|
+
rows_count: row.rows_count != null ? Number(row.rows_count) : null,
|
|
7695
|
+
created_at: String(row.created_at)
|
|
7696
|
+
};
|
|
7697
|
+
}
|
|
7698
|
+
async function createWorkflowSchedule(input) {
|
|
7699
|
+
const id = (0, import_node_crypto.randomUUID)();
|
|
7700
|
+
await getDb().execute({
|
|
7701
|
+
sql: `
|
|
7702
|
+
INSERT INTO workflow_schedules (
|
|
7703
|
+
id, user_id, workflow_id, name, input_json, cadence, timezone, webhook_url, next_run_at
|
|
7704
|
+
)
|
|
7705
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
7706
|
+
`,
|
|
7707
|
+
args: [
|
|
7708
|
+
id,
|
|
7709
|
+
input.userId,
|
|
7710
|
+
input.workflowId,
|
|
7711
|
+
input.name,
|
|
7712
|
+
JSON.stringify(input.workflowInput),
|
|
7713
|
+
input.cadence,
|
|
7714
|
+
input.timezone ?? "UTC",
|
|
7715
|
+
input.webhookUrl ?? null,
|
|
7716
|
+
input.nextRunAt ?? null
|
|
7717
|
+
]
|
|
7718
|
+
});
|
|
7719
|
+
const schedule = await getWorkflowSchedule(id, input.userId);
|
|
7720
|
+
if (!schedule) throw new Error("Failed to create workflow schedule");
|
|
7721
|
+
return schedule;
|
|
7722
|
+
}
|
|
7723
|
+
async function getWorkflowSchedule(id, userId) {
|
|
7724
|
+
const res = userId == null ? await getDb().execute({ sql: "SELECT * FROM workflow_schedules WHERE id = ? AND status != ? LIMIT 1", args: [id, "deleted"] }) : await getDb().execute({ sql: "SELECT * FROM workflow_schedules WHERE id = ? AND user_id = ? AND status != ? LIMIT 1", args: [id, userId, "deleted"] });
|
|
7725
|
+
return res.rows[0] ? rowToWorkflowSchedule(res.rows[0]) : null;
|
|
7726
|
+
}
|
|
7727
|
+
async function listWorkflowSchedules(userId) {
|
|
7728
|
+
const res = await getDb().execute({
|
|
7729
|
+
sql: `SELECT * FROM workflow_schedules WHERE user_id = ? AND status != 'deleted' ORDER BY created_at DESC LIMIT 100`,
|
|
7730
|
+
args: [userId]
|
|
7731
|
+
});
|
|
7732
|
+
return res.rows.map((r) => rowToWorkflowSchedule(r));
|
|
7733
|
+
}
|
|
7734
|
+
async function patchWorkflowSchedule(input) {
|
|
7735
|
+
const sets = [];
|
|
7736
|
+
const args = [];
|
|
7737
|
+
if (input.name !== void 0) {
|
|
7738
|
+
sets.push("name = ?");
|
|
7739
|
+
args.push(input.name);
|
|
7740
|
+
}
|
|
7741
|
+
if (input.status !== void 0) {
|
|
7742
|
+
sets.push("status = ?");
|
|
7743
|
+
args.push(input.status);
|
|
7744
|
+
}
|
|
7745
|
+
if (input.workflowInput !== void 0) {
|
|
7746
|
+
sets.push("input_json = ?");
|
|
7747
|
+
args.push(JSON.stringify(input.workflowInput));
|
|
7748
|
+
}
|
|
7749
|
+
if (input.cadence !== void 0) {
|
|
7750
|
+
sets.push("cadence = ?");
|
|
7751
|
+
args.push(input.cadence);
|
|
7752
|
+
}
|
|
7753
|
+
if (input.timezone !== void 0) {
|
|
7754
|
+
sets.push("timezone = ?");
|
|
7755
|
+
args.push(input.timezone);
|
|
7756
|
+
}
|
|
7757
|
+
if (input.webhookUrl !== void 0) {
|
|
7758
|
+
sets.push("webhook_url = ?");
|
|
7759
|
+
args.push(input.webhookUrl);
|
|
7760
|
+
}
|
|
7761
|
+
if (input.nextRunAt !== void 0) {
|
|
7762
|
+
sets.push("next_run_at = ?");
|
|
7763
|
+
args.push(input.nextRunAt);
|
|
7764
|
+
}
|
|
7765
|
+
if (sets.length === 0) return getWorkflowSchedule(input.id, input.userId);
|
|
7766
|
+
sets.push(`updated_at = datetime('now')`);
|
|
7767
|
+
await getDb().execute({
|
|
7768
|
+
sql: `UPDATE workflow_schedules SET ${sets.join(", ")} WHERE id = ? AND user_id = ? AND status != 'deleted'`,
|
|
7769
|
+
args: [...args, input.id, input.userId]
|
|
7770
|
+
});
|
|
7771
|
+
return getWorkflowSchedule(input.id, input.userId);
|
|
7772
|
+
}
|
|
7773
|
+
async function deleteWorkflowSchedule(id, userId) {
|
|
7774
|
+
const res = await getDb().execute({
|
|
7775
|
+
sql: `UPDATE workflow_schedules SET status = 'deleted', updated_at = datetime('now') WHERE id = ? AND user_id = ? AND status != 'deleted'`,
|
|
7776
|
+
args: [id, userId]
|
|
7777
|
+
});
|
|
7778
|
+
return res.rowsAffected > 0;
|
|
7779
|
+
}
|
|
7780
|
+
async function listDueWorkflowSchedules(nowIso, limit = 10) {
|
|
7781
|
+
const res = await getDb().execute({
|
|
7782
|
+
sql: `
|
|
7783
|
+
SELECT * FROM workflow_schedules
|
|
7784
|
+
WHERE status = 'active' AND next_run_at IS NOT NULL AND next_run_at <= ?
|
|
7785
|
+
ORDER BY next_run_at ASC
|
|
7786
|
+
LIMIT ?
|
|
7787
|
+
`,
|
|
7788
|
+
args: [nowIso, limit]
|
|
7789
|
+
});
|
|
7790
|
+
return res.rows.map((r) => rowToWorkflowSchedule(r));
|
|
7791
|
+
}
|
|
7792
|
+
async function markWorkflowScheduleRan(id, lastRunAt, nextRunAt) {
|
|
7793
|
+
await getDb().execute({
|
|
7794
|
+
sql: `UPDATE workflow_schedules SET last_run_at = ?, next_run_at = ?, updated_at = datetime('now') WHERE id = ?`,
|
|
7795
|
+
args: [lastRunAt, nextRunAt, id]
|
|
7796
|
+
});
|
|
7797
|
+
}
|
|
7798
|
+
async function createWorkflowRun(input) {
|
|
7799
|
+
const id = input.id ?? (0, import_node_crypto.randomUUID)();
|
|
7800
|
+
const args = [
|
|
7801
|
+
id,
|
|
7802
|
+
input.userId,
|
|
7803
|
+
input.scheduleId ?? null,
|
|
7804
|
+
input.workflowId,
|
|
7805
|
+
JSON.stringify(input.workflowInput),
|
|
7806
|
+
input.idempotencyKey ?? null
|
|
7807
|
+
];
|
|
7808
|
+
try {
|
|
7809
|
+
await getDb().execute({
|
|
7810
|
+
sql: `
|
|
7811
|
+
INSERT INTO workflow_runs (id, user_id, schedule_id, workflow_id, input_json, idempotency_key)
|
|
7812
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
7813
|
+
`,
|
|
7814
|
+
args
|
|
7815
|
+
});
|
|
7816
|
+
} catch (err) {
|
|
7817
|
+
if (input.idempotencyKey) {
|
|
7818
|
+
const existing = await getWorkflowRunByIdempotencyKey(input.idempotencyKey);
|
|
7819
|
+
if (existing) return existing;
|
|
7820
|
+
}
|
|
7821
|
+
throw err;
|
|
7822
|
+
}
|
|
7823
|
+
const run = await getWorkflowRun(id, input.userId);
|
|
7824
|
+
if (!run) throw new Error("Failed to create workflow run");
|
|
7825
|
+
return run;
|
|
7826
|
+
}
|
|
7827
|
+
async function getWorkflowRun(id, userId) {
|
|
7828
|
+
const res = userId == null ? await getDb().execute({ sql: "SELECT * FROM workflow_runs WHERE id = ? LIMIT 1", args: [id] }) : await getDb().execute({ sql: "SELECT * FROM workflow_runs WHERE id = ? AND user_id = ? LIMIT 1", args: [id, userId] });
|
|
7829
|
+
return res.rows[0] ? rowToWorkflowRun(res.rows[0]) : null;
|
|
7830
|
+
}
|
|
7831
|
+
async function getWorkflowRunByIdempotencyKey(idempotencyKey) {
|
|
7832
|
+
const res = await getDb().execute({ sql: "SELECT * FROM workflow_runs WHERE idempotency_key = ? LIMIT 1", args: [idempotencyKey] });
|
|
7833
|
+
return res.rows[0] ? rowToWorkflowRun(res.rows[0]) : null;
|
|
7834
|
+
}
|
|
7835
|
+
async function listWorkflowRuns(userId, limit = 50) {
|
|
7836
|
+
const res = await getDb().execute({
|
|
7837
|
+
sql: "SELECT * FROM workflow_runs WHERE user_id = ? ORDER BY queued_at DESC LIMIT ?",
|
|
7838
|
+
args: [userId, limit]
|
|
7839
|
+
});
|
|
7840
|
+
return res.rows.map((r) => rowToWorkflowRun(r));
|
|
7841
|
+
}
|
|
7842
|
+
async function markWorkflowRunRunning(id) {
|
|
7843
|
+
await getDb().execute({
|
|
7844
|
+
sql: `UPDATE workflow_runs SET status = 'running', started_at = COALESCE(started_at, datetime('now')) WHERE id = ?`,
|
|
7845
|
+
args: [id]
|
|
7846
|
+
});
|
|
7847
|
+
}
|
|
7848
|
+
async function completeWorkflowRunRecord(id, status, manifest) {
|
|
7849
|
+
await getDb().execute({
|
|
7850
|
+
sql: `UPDATE workflow_runs SET status = ?, manifest_json = ?, completed_at = datetime('now') WHERE id = ?`,
|
|
7851
|
+
args: [status, JSON.stringify(manifest), id]
|
|
7852
|
+
});
|
|
7853
|
+
}
|
|
7854
|
+
async function failWorkflowRunRecord(id, errorMessage4, manifest) {
|
|
7855
|
+
await getDb().execute({
|
|
7856
|
+
sql: `UPDATE workflow_runs SET status = 'failed', error_message = ?, manifest_json = ?, completed_at = datetime('now') WHERE id = ?`,
|
|
7857
|
+
args: [errorMessage4, manifest == null ? null : JSON.stringify(manifest), id]
|
|
7858
|
+
});
|
|
7859
|
+
}
|
|
7860
|
+
function workflowArtifactContentType(kind) {
|
|
7861
|
+
if (kind === "html_report") return "text/html; charset=utf-8";
|
|
7862
|
+
if (kind === "csv") return "text/csv; charset=utf-8";
|
|
7863
|
+
if (kind === "json") return "application/json; charset=utf-8";
|
|
7864
|
+
return "text/plain; charset=utf-8";
|
|
7865
|
+
}
|
|
7866
|
+
async function replaceWorkflowArtifacts(runId, userId, artifacts) {
|
|
7867
|
+
const db = getDb();
|
|
7868
|
+
await db.execute({ sql: "DELETE FROM workflow_artifacts WHERE run_id = ? AND user_id = ?", args: [runId, userId] });
|
|
7869
|
+
for (const artifact of artifacts) {
|
|
7870
|
+
await db.execute({
|
|
7871
|
+
sql: `
|
|
7872
|
+
INSERT INTO workflow_artifacts (id, run_id, user_id, kind, label, path, content_type, bytes, rows_count)
|
|
7873
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
7874
|
+
`,
|
|
7875
|
+
args: [
|
|
7876
|
+
(0, import_node_crypto.randomUUID)(),
|
|
7877
|
+
runId,
|
|
7878
|
+
userId,
|
|
7879
|
+
artifact.kind,
|
|
7880
|
+
artifact.label,
|
|
7881
|
+
artifact.path,
|
|
7882
|
+
workflowArtifactContentType(artifact.kind),
|
|
7883
|
+
artifact.bytes ?? null,
|
|
7884
|
+
artifact.rows ?? null
|
|
7885
|
+
]
|
|
7886
|
+
});
|
|
7887
|
+
}
|
|
7888
|
+
return listWorkflowArtifacts(runId, userId);
|
|
7889
|
+
}
|
|
7890
|
+
async function listWorkflowArtifacts(runId, userId) {
|
|
7891
|
+
const res = await getDb().execute({
|
|
7892
|
+
sql: "SELECT * FROM workflow_artifacts WHERE run_id = ? AND user_id = ? ORDER BY created_at ASC",
|
|
7893
|
+
args: [runId, userId]
|
|
7894
|
+
});
|
|
7895
|
+
return res.rows.map((r) => rowToWorkflowArtifact(r));
|
|
7896
|
+
}
|
|
7897
|
+
async function getWorkflowArtifact(id, userId) {
|
|
7898
|
+
const res = await getDb().execute({
|
|
7899
|
+
sql: "SELECT * FROM workflow_artifacts WHERE id = ? AND user_id = ? LIMIT 1",
|
|
7900
|
+
args: [id, userId]
|
|
7901
|
+
});
|
|
7902
|
+
return res.rows[0] ? rowToWorkflowArtifact(res.rows[0]) : null;
|
|
7903
|
+
}
|
|
7904
|
+
async function recordWorkflowWebhookDelivery(input) {
|
|
7905
|
+
await getDb().execute({
|
|
7906
|
+
sql: `
|
|
7907
|
+
INSERT INTO workflow_webhook_deliveries (
|
|
7908
|
+
id, run_id, schedule_id, user_id, webhook_url, status, attempts, last_error, delivered_at
|
|
7909
|
+
)
|
|
7910
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
7911
|
+
`,
|
|
7912
|
+
args: [
|
|
7913
|
+
(0, import_node_crypto.randomUUID)(),
|
|
7914
|
+
input.runId,
|
|
7915
|
+
input.scheduleId ?? null,
|
|
7916
|
+
input.userId,
|
|
7917
|
+
input.webhookUrl,
|
|
7918
|
+
input.status,
|
|
7919
|
+
input.attempts,
|
|
7920
|
+
input.lastError ?? null,
|
|
7921
|
+
input.status === "delivered" ? (/* @__PURE__ */ new Date()).toISOString() : null
|
|
7922
|
+
]
|
|
7923
|
+
});
|
|
7924
|
+
}
|
|
7554
7925
|
async function countActiveUsers() {
|
|
7555
7926
|
const res = await getDb().execute(`SELECT COUNT(*) as n FROM users WHERE active = 1`);
|
|
7556
7927
|
return Number(res.rows[0]?.n ?? 0);
|
|
@@ -10395,8 +10766,8 @@ var init_FacebookAdExtractor = __esm({
|
|
|
10395
10766
|
const pos = st.getPropertyValue("mask-position") || st.getPropertyValue("-webkit-mask-position") || st.getPropertyValue("background-position") || "";
|
|
10396
10767
|
const m = pos.match(/(-?\d+)px\s+(-?\d+)px/);
|
|
10397
10768
|
if (!m) continue;
|
|
10398
|
-
const
|
|
10399
|
-
if (
|
|
10769
|
+
const platform2 = sm[parseInt(m[2])];
|
|
10770
|
+
if (platform2 && !platforms2.includes(platform2)) platforms2.push(platform2);
|
|
10400
10771
|
}
|
|
10401
10772
|
for (const el2 of Array.from(root2.querySelectorAll("span,div"))) {
|
|
10402
10773
|
const txt = t(el2);
|
|
@@ -14277,6 +14648,7 @@ function csvRowsFor(result) {
|
|
|
14277
14648
|
result_position: null,
|
|
14278
14649
|
business_name: null,
|
|
14279
14650
|
review_stars: null,
|
|
14651
|
+
review_count: null,
|
|
14280
14652
|
category: null,
|
|
14281
14653
|
address: null,
|
|
14282
14654
|
phone: null,
|
|
@@ -14310,6 +14682,7 @@ function csvRowsFor(result) {
|
|
|
14310
14682
|
result_position: business.position,
|
|
14311
14683
|
business_name: business.name,
|
|
14312
14684
|
review_stars: business.rating,
|
|
14685
|
+
review_count: business.reviewCount,
|
|
14313
14686
|
category: business.category,
|
|
14314
14687
|
address: business.address,
|
|
14315
14688
|
phone: business.phone,
|
|
@@ -14350,6 +14723,7 @@ async function saveDirectoryCsv(result) {
|
|
|
14350
14723
|
"result_position",
|
|
14351
14724
|
"business_name",
|
|
14352
14725
|
"review_stars",
|
|
14726
|
+
"review_count",
|
|
14353
14727
|
"category",
|
|
14354
14728
|
"address",
|
|
14355
14729
|
"phone",
|
|
@@ -14498,10 +14872,1200 @@ var init_directory_routes = __esm({
|
|
|
14498
14872
|
}
|
|
14499
14873
|
});
|
|
14500
14874
|
|
|
14875
|
+
// src/lib/slugify.ts
|
|
14876
|
+
function slugify(s) {
|
|
14877
|
+
return s.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
|
|
14878
|
+
}
|
|
14879
|
+
var init_slugify = __esm({
|
|
14880
|
+
"src/lib/slugify.ts"() {
|
|
14881
|
+
"use strict";
|
|
14882
|
+
}
|
|
14883
|
+
});
|
|
14884
|
+
|
|
14885
|
+
// src/workflows/artifact-writer.ts
|
|
14886
|
+
function workflowOutputBaseDir(outputDir) {
|
|
14887
|
+
return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path7.join)((0, import_node_os4.homedir)(), "Downloads", "mcp-scraper");
|
|
14888
|
+
}
|
|
14889
|
+
function timestamp() {
|
|
14890
|
+
return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
14891
|
+
}
|
|
14892
|
+
function safeSlug(value) {
|
|
14893
|
+
return slugify(value).replace(/^-+|-+$/g, "").slice(0, 80) || "run";
|
|
14894
|
+
}
|
|
14895
|
+
function indexPath(baseDir = workflowOutputBaseDir()) {
|
|
14896
|
+
return (0, import_node_path7.join)(baseDir, "workflows", "index.json");
|
|
14897
|
+
}
|
|
14898
|
+
async function readIndex(baseDir) {
|
|
14899
|
+
const path6 = indexPath(baseDir);
|
|
14900
|
+
if (!(0, import_node_fs4.existsSync)(path6)) return { runs: [] };
|
|
14901
|
+
try {
|
|
14902
|
+
return JSON.parse(await (0, import_promises6.readFile)(path6, "utf8"));
|
|
14903
|
+
} catch {
|
|
14904
|
+
return { runs: [] };
|
|
14905
|
+
}
|
|
14906
|
+
}
|
|
14907
|
+
async function updateWorkflowIndex(manifest, manifestPath, baseDir) {
|
|
14908
|
+
if (manifest.status === "running") return;
|
|
14909
|
+
const path6 = indexPath(baseDir);
|
|
14910
|
+
const index = await readIndex(baseDir);
|
|
14911
|
+
const report = manifest.artifacts.find((a) => a.kind === "html_report")?.path ?? null;
|
|
14912
|
+
const entry = {
|
|
14913
|
+
workflow: manifest.workflow,
|
|
14914
|
+
runId: manifest.runId,
|
|
14915
|
+
status: manifest.status,
|
|
14916
|
+
startedAt: manifest.startedAt,
|
|
14917
|
+
reportPath: report,
|
|
14918
|
+
manifestPath,
|
|
14919
|
+
summary: `${manifest.title} (${manifest.status})`
|
|
14920
|
+
};
|
|
14921
|
+
index.runs = [entry, ...index.runs.filter((r) => r.runId !== manifest.runId)].slice(0, 200);
|
|
14922
|
+
await (0, import_promises6.mkdir)((0, import_node_path7.dirname)(path6), { recursive: true });
|
|
14923
|
+
await (0, import_promises6.writeFile)(path6, JSON.stringify(index, null, 2), "utf8");
|
|
14924
|
+
}
|
|
14925
|
+
var import_promises6, import_node_fs4, import_node_os4, import_node_path7, import_node_child_process2, ArtifactWriter;
|
|
14926
|
+
var init_artifact_writer = __esm({
|
|
14927
|
+
"src/workflows/artifact-writer.ts"() {
|
|
14928
|
+
"use strict";
|
|
14929
|
+
import_promises6 = require("fs/promises");
|
|
14930
|
+
import_node_fs4 = require("fs");
|
|
14931
|
+
import_node_os4 = require("os");
|
|
14932
|
+
import_node_path7 = require("path");
|
|
14933
|
+
import_node_child_process2 = require("child_process");
|
|
14934
|
+
init_csv();
|
|
14935
|
+
init_slugify();
|
|
14936
|
+
ArtifactWriter = class _ArtifactWriter {
|
|
14937
|
+
constructor(workflowId, title, runId, baseDir, runDir, startedAt, input) {
|
|
14938
|
+
this.workflowId = workflowId;
|
|
14939
|
+
this.title = title;
|
|
14940
|
+
this.runId = runId;
|
|
14941
|
+
this.baseDir = baseDir;
|
|
14942
|
+
this.runDir = runDir;
|
|
14943
|
+
this.startedAt = startedAt;
|
|
14944
|
+
this.input = input;
|
|
14945
|
+
}
|
|
14946
|
+
workflowId;
|
|
14947
|
+
title;
|
|
14948
|
+
runId;
|
|
14949
|
+
baseDir;
|
|
14950
|
+
runDir;
|
|
14951
|
+
startedAt;
|
|
14952
|
+
input;
|
|
14953
|
+
artifacts = [];
|
|
14954
|
+
static async create(workflowId, title, input, outputDir, forcedRunId) {
|
|
14955
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
14956
|
+
const runId = forcedRunId?.trim() || `${timestamp()}-${safeSlug(workflowId)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
14957
|
+
const nameSource = String(input.keyword ?? input.query ?? input.domain ?? input.state ?? workflowId);
|
|
14958
|
+
const baseDir = workflowOutputBaseDir(outputDir);
|
|
14959
|
+
const runDir = (0, import_node_path7.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
|
|
14960
|
+
await (0, import_promises6.mkdir)(runDir, { recursive: true });
|
|
14961
|
+
return new _ArtifactWriter(workflowId, title, runId, baseDir, runDir, startedAt, input);
|
|
14962
|
+
}
|
|
14963
|
+
async remember(kind, label, path6, rows) {
|
|
14964
|
+
const size = await (0, import_promises6.stat)(path6).then((s) => s.size).catch(() => void 0);
|
|
14965
|
+
this.artifacts.push({ kind, label, path: path6, bytes: size, rows });
|
|
14966
|
+
return path6;
|
|
14967
|
+
}
|
|
14968
|
+
async writeJson(label, relativePath, data) {
|
|
14969
|
+
const path6 = (0, import_node_path7.join)(this.runDir, relativePath);
|
|
14970
|
+
await (0, import_promises6.mkdir)((0, import_node_path7.dirname)(path6), { recursive: true });
|
|
14971
|
+
await (0, import_promises6.writeFile)(path6, JSON.stringify(data, null, 2), "utf8");
|
|
14972
|
+
return this.remember("json", label, path6);
|
|
14973
|
+
}
|
|
14974
|
+
async writeText(label, relativePath, text, kind = "markdown") {
|
|
14975
|
+
const path6 = (0, import_node_path7.join)(this.runDir, relativePath);
|
|
14976
|
+
await (0, import_promises6.mkdir)((0, import_node_path7.dirname)(path6), { recursive: true });
|
|
14977
|
+
await (0, import_promises6.writeFile)(path6, text, "utf8");
|
|
14978
|
+
return this.remember(kind, label, path6);
|
|
14979
|
+
}
|
|
14980
|
+
async writeCsv(label, relativePath, headers, rows) {
|
|
14981
|
+
const path6 = (0, import_node_path7.join)(this.runDir, relativePath);
|
|
14982
|
+
await (0, import_promises6.mkdir)((0, import_node_path7.dirname)(path6), { recursive: true });
|
|
14983
|
+
await (0, import_promises6.writeFile)(path6, rowsToCsv(headers, rows), "utf8");
|
|
14984
|
+
return this.remember("csv", label, path6, rows.length);
|
|
14985
|
+
}
|
|
14986
|
+
async writeHtml(label, relativePath, html) {
|
|
14987
|
+
return this.writeText(label, relativePath, html, "html_report");
|
|
14988
|
+
}
|
|
14989
|
+
async writeManifest(status, counts, warnings, errors) {
|
|
14990
|
+
const manifest = {
|
|
14991
|
+
workflow: this.workflowId,
|
|
14992
|
+
title: this.title,
|
|
14993
|
+
runId: this.runId,
|
|
14994
|
+
status,
|
|
14995
|
+
startedAt: this.startedAt,
|
|
14996
|
+
completedAt: status === "running" ? null : (/* @__PURE__ */ new Date()).toISOString(),
|
|
14997
|
+
input: this.input,
|
|
14998
|
+
artifacts: this.artifacts,
|
|
14999
|
+
warnings,
|
|
15000
|
+
errors,
|
|
15001
|
+
counts
|
|
15002
|
+
};
|
|
15003
|
+
const path6 = (0, import_node_path7.join)(this.runDir, "manifest.json");
|
|
15004
|
+
await (0, import_promises6.writeFile)(path6, JSON.stringify(manifest, null, 2), "utf8");
|
|
15005
|
+
await updateWorkflowIndex(manifest, path6, this.baseDir);
|
|
15006
|
+
return path6;
|
|
15007
|
+
}
|
|
15008
|
+
};
|
|
15009
|
+
}
|
|
15010
|
+
});
|
|
15011
|
+
|
|
15012
|
+
// src/workflows/http-client.ts
|
|
15013
|
+
var WorkflowHttpClient;
|
|
15014
|
+
var init_http_client2 = __esm({
|
|
15015
|
+
"src/workflows/http-client.ts"() {
|
|
15016
|
+
"use strict";
|
|
15017
|
+
WorkflowHttpClient = class {
|
|
15018
|
+
constructor(apiUrl, apiKey, fetchImpl = fetch) {
|
|
15019
|
+
this.apiUrl = apiUrl;
|
|
15020
|
+
this.apiKey = apiKey;
|
|
15021
|
+
this.fetchImpl = fetchImpl;
|
|
15022
|
+
}
|
|
15023
|
+
apiUrl;
|
|
15024
|
+
apiKey;
|
|
15025
|
+
fetchImpl;
|
|
15026
|
+
async post(path6, body, timeoutMs = 18e4) {
|
|
15027
|
+
const res = await this.fetchImpl(`${this.apiUrl.replace(/\/$/, "")}${path6}`, {
|
|
15028
|
+
method: "POST",
|
|
15029
|
+
headers: {
|
|
15030
|
+
"Content-Type": "application/json",
|
|
15031
|
+
"x-api-key": this.apiKey
|
|
15032
|
+
},
|
|
15033
|
+
body: JSON.stringify(body),
|
|
15034
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
15035
|
+
});
|
|
15036
|
+
const data = await res.json().catch(() => ({}));
|
|
15037
|
+
if (!res.ok) {
|
|
15038
|
+
const message = typeof data.error === "string" ? data.error : `Workflow API ${path6} failed with ${res.status}`;
|
|
15039
|
+
throw new Error(message);
|
|
15040
|
+
}
|
|
15041
|
+
return data;
|
|
15042
|
+
}
|
|
15043
|
+
};
|
|
15044
|
+
}
|
|
15045
|
+
});
|
|
15046
|
+
|
|
15047
|
+
// src/workflows/report-renderer.ts
|
|
15048
|
+
function escapeHtml(value) {
|
|
15049
|
+
return String(value ?? "").replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
15050
|
+
}
|
|
15051
|
+
function renderTable(table) {
|
|
15052
|
+
const rows = table.rows.map((row) => `<tr>${table.columns.map((col) => `<td>${escapeHtml(row[col])}</td>`).join("")}</tr>`).join("\n");
|
|
15053
|
+
return [
|
|
15054
|
+
`<section>`,
|
|
15055
|
+
`<h2>${escapeHtml(table.title)}</h2>`,
|
|
15056
|
+
`<div class="table-wrap"><table>`,
|
|
15057
|
+
`<thead><tr>${table.columns.map((col) => `<th>${escapeHtml(col)}</th>`).join("")}</tr></thead>`,
|
|
15058
|
+
`<tbody>${rows || `<tr><td colspan="${table.columns.length}">No rows</td></tr>`}</tbody>`,
|
|
15059
|
+
`</table></div>`,
|
|
15060
|
+
`</section>`
|
|
15061
|
+
].join("\n");
|
|
15062
|
+
}
|
|
15063
|
+
function renderWorkflowReport(input) {
|
|
15064
|
+
const warnings = input.warnings?.length ? `<section class="warnings"><h2>Warnings</h2><ul>${input.warnings.map((w) => `<li>${escapeHtml(w)}</li>`).join("")}</ul></section>` : "";
|
|
15065
|
+
const sections = (input.sections ?? []).map((section) => `<section><h2>${escapeHtml(section.title)}</h2><p>${escapeHtml(section.body)}</p></section>`).join("\n");
|
|
15066
|
+
const tables = (input.tables ?? []).map(renderTable).join("\n");
|
|
15067
|
+
return [
|
|
15068
|
+
"<!doctype html>",
|
|
15069
|
+
'<html lang="en">',
|
|
15070
|
+
"<head>",
|
|
15071
|
+
'<meta charset="utf-8">',
|
|
15072
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">',
|
|
15073
|
+
`<title>${escapeHtml(input.title)}</title>`,
|
|
15074
|
+
"<style>",
|
|
15075
|
+
':root{font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:#172026;background:#f6f7f8}',
|
|
15076
|
+
"body{margin:0;padding:32px}main{max-width:1180px;margin:0 auto;background:#fff;border:1px solid #d9dee3;border-radius:8px;padding:28px}",
|
|
15077
|
+
"h1{font-size:30px;line-height:1.15;margin:0 0 8px}h2{font-size:18px;margin:28px 0 12px}p{line-height:1.55;color:#3d4952}.sub{color:#66737f;margin:0 0 20px}",
|
|
15078
|
+
".summary{background:#eef6f8;border-left:4px solid #12829a;padding:14px 16px;margin:20px 0}.warnings{background:#fff7e8;border-left:4px solid #c87b00;padding:12px 16px}",
|
|
15079
|
+
".table-wrap{overflow:auto;border:1px solid #dfe5ea;border-radius:8px}table{border-collapse:collapse;width:100%;font-size:13px}th,td{border-bottom:1px solid #edf0f2;padding:9px 10px;text-align:left;vertical-align:top}th{background:#f3f5f7;color:#31404c;position:sticky;top:0}tr:last-child td{border-bottom:0}",
|
|
15080
|
+
"</style>",
|
|
15081
|
+
"</head>",
|
|
15082
|
+
"<body><main>",
|
|
15083
|
+
`<h1>${escapeHtml(input.title)}</h1>`,
|
|
15084
|
+
input.subtitle ? `<p class="sub">${escapeHtml(input.subtitle)}</p>` : "",
|
|
15085
|
+
`<div class="summary">${escapeHtml(input.summary)}</div>`,
|
|
15086
|
+
warnings,
|
|
15087
|
+
sections,
|
|
15088
|
+
tables,
|
|
15089
|
+
"</main></body></html>"
|
|
15090
|
+
].join("\n");
|
|
15091
|
+
}
|
|
15092
|
+
var init_report_renderer = __esm({
|
|
15093
|
+
"src/workflows/report-renderer.ts"() {
|
|
15094
|
+
"use strict";
|
|
15095
|
+
}
|
|
15096
|
+
});
|
|
15097
|
+
|
|
15098
|
+
// src/workflows/workflows/agent-packet.ts
|
|
15099
|
+
function normalizeDomain(value) {
|
|
15100
|
+
if (!value) return null;
|
|
15101
|
+
try {
|
|
15102
|
+
const url = new URL(value.includes("://") ? value : `https://${value}`);
|
|
15103
|
+
return url.hostname.replace(/^www\./, "").toLowerCase();
|
|
15104
|
+
} catch {
|
|
15105
|
+
return value.replace(/^https?:\/\//, "").replace(/^www\./, "").split("/")[0]?.toLowerCase() || null;
|
|
15106
|
+
}
|
|
15107
|
+
}
|
|
15108
|
+
function domainFromUrl(url) {
|
|
15109
|
+
return normalizeDomain(url) ?? "";
|
|
15110
|
+
}
|
|
15111
|
+
function sourceRows(input, serp, paa) {
|
|
15112
|
+
const target = normalizeDomain(input.domain);
|
|
15113
|
+
const rows = [];
|
|
15114
|
+
for (const result of serp?.organicResults ?? []) {
|
|
15115
|
+
const domain = normalizeDomain(result.domain) ?? domainFromUrl(result.url);
|
|
15116
|
+
rows.push({
|
|
15117
|
+
surface: "organic",
|
|
15118
|
+
position: result.position,
|
|
15119
|
+
title: result.title,
|
|
15120
|
+
url: result.url,
|
|
15121
|
+
domain,
|
|
15122
|
+
question: "",
|
|
15123
|
+
answer_excerpt: result.snippet ?? "",
|
|
15124
|
+
is_target: target ? domain === target : false,
|
|
15125
|
+
is_competitor: target ? domain !== target : true
|
|
15126
|
+
});
|
|
15127
|
+
}
|
|
15128
|
+
for (const row of paa?.flat ?? []) {
|
|
15129
|
+
const url = row.source_cite ?? "";
|
|
15130
|
+
const domain = normalizeDomain(row.source_site ?? "") ?? (url ? domainFromUrl(url) : "");
|
|
15131
|
+
rows.push({
|
|
15132
|
+
surface: "paa",
|
|
15133
|
+
position: "",
|
|
15134
|
+
title: row.source_title ?? row.source_site ?? "",
|
|
15135
|
+
url,
|
|
15136
|
+
domain,
|
|
15137
|
+
question: row.question,
|
|
15138
|
+
answer_excerpt: (row.answer ?? "").slice(0, 240),
|
|
15139
|
+
is_target: target ? domain === target : false,
|
|
15140
|
+
is_competitor: target ? Boolean(domain && domain !== target) : Boolean(domain)
|
|
15141
|
+
});
|
|
15142
|
+
}
|
|
15143
|
+
for (const [i, cite] of (serp?.aiOverview?.citations ?? []).entries()) {
|
|
15144
|
+
const url = cite.href ?? "";
|
|
15145
|
+
const domain = url ? domainFromUrl(url) : "";
|
|
15146
|
+
rows.push({
|
|
15147
|
+
surface: "ai_overview",
|
|
15148
|
+
position: i + 1,
|
|
15149
|
+
title: cite.text ?? "",
|
|
15150
|
+
url,
|
|
15151
|
+
domain,
|
|
15152
|
+
question: "",
|
|
15153
|
+
answer_excerpt: "",
|
|
15154
|
+
is_target: target ? domain === target : false,
|
|
15155
|
+
is_competitor: target ? Boolean(domain && domain !== target) : Boolean(domain)
|
|
15156
|
+
});
|
|
15157
|
+
}
|
|
15158
|
+
return rows;
|
|
15159
|
+
}
|
|
15160
|
+
function competitorRows(rows, targetDomain) {
|
|
15161
|
+
const byDomain = /* @__PURE__ */ new Map();
|
|
15162
|
+
for (const row of rows) {
|
|
15163
|
+
const domain = String(row.domain ?? "");
|
|
15164
|
+
if (!domain || domain === targetDomain) continue;
|
|
15165
|
+
const entry = byDomain.get(domain) ?? { organic: [], paa: 0, ai: 0, urls: /* @__PURE__ */ new Set() };
|
|
15166
|
+
if (row.surface === "organic" && row.position) entry.organic.push(Number(row.position));
|
|
15167
|
+
if (row.surface === "paa") entry.paa += 1;
|
|
15168
|
+
if (row.surface === "ai_overview") entry.ai += 1;
|
|
15169
|
+
if (row.url) entry.urls.add(String(row.url));
|
|
15170
|
+
byDomain.set(domain, entry);
|
|
15171
|
+
}
|
|
15172
|
+
return [...byDomain.entries()].map(([domain, entry]) => ({
|
|
15173
|
+
domain,
|
|
15174
|
+
organic_best_position: entry.organic.length ? Math.min(...entry.organic) : "",
|
|
15175
|
+
organic_count: entry.organic.length,
|
|
15176
|
+
paa_mentions: entry.paa,
|
|
15177
|
+
ai_overview_citations: entry.ai,
|
|
15178
|
+
source_url_count: entry.urls.size
|
|
15179
|
+
})).sort((a, b) => Number(a.organic_best_position || 999) - Number(b.organic_best_position || 999));
|
|
15180
|
+
}
|
|
15181
|
+
var import_zod19, AgentPacketInputSchema, agentPacketWorkflowDefinition;
|
|
15182
|
+
var init_agent_packet = __esm({
|
|
15183
|
+
"src/workflows/workflows/agent-packet.ts"() {
|
|
15184
|
+
"use strict";
|
|
15185
|
+
import_zod19 = require("zod");
|
|
15186
|
+
init_report_renderer();
|
|
15187
|
+
AgentPacketInputSchema = import_zod19.z.object({
|
|
15188
|
+
keyword: import_zod19.z.string().min(1),
|
|
15189
|
+
domain: import_zod19.z.string().optional(),
|
|
15190
|
+
location: import_zod19.z.string().optional(),
|
|
15191
|
+
maxQuestions: import_zod19.z.number().int().min(1).max(200).default(40),
|
|
15192
|
+
includeSerp: import_zod19.z.boolean().default(true),
|
|
15193
|
+
includePaa: import_zod19.z.boolean().default(true),
|
|
15194
|
+
includeAiOverview: import_zod19.z.boolean().default(true),
|
|
15195
|
+
returnPartial: import_zod19.z.boolean().default(true)
|
|
15196
|
+
});
|
|
15197
|
+
agentPacketWorkflowDefinition = {
|
|
15198
|
+
id: "agent-packet",
|
|
15199
|
+
title: "Agent-Ready SEO Packet",
|
|
15200
|
+
description: "Create an evidence folder for AI agents from live SERP/PAA/AI search surfaces.",
|
|
15201
|
+
inputSchema: AgentPacketInputSchema,
|
|
15202
|
+
async run(input, ctx) {
|
|
15203
|
+
await ctx.artifacts.writeManifest("running", {}, [], []);
|
|
15204
|
+
const warnings = [];
|
|
15205
|
+
const errors = [];
|
|
15206
|
+
let serp = null;
|
|
15207
|
+
let paa = null;
|
|
15208
|
+
if (input.includeSerp) {
|
|
15209
|
+
try {
|
|
15210
|
+
serp = await ctx.client.post("/harvest/sync", {
|
|
15211
|
+
query: input.keyword,
|
|
15212
|
+
location: input.location,
|
|
15213
|
+
serpOnly: true,
|
|
15214
|
+
maxQuestions: 1,
|
|
15215
|
+
format: "json"
|
|
15216
|
+
}, 18e4);
|
|
15217
|
+
} catch (err) {
|
|
15218
|
+
warnings.push(`SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
15219
|
+
}
|
|
15220
|
+
}
|
|
15221
|
+
if (input.includePaa) {
|
|
15222
|
+
try {
|
|
15223
|
+
paa = await ctx.client.post("/harvest/sync", {
|
|
15224
|
+
query: input.keyword,
|
|
15225
|
+
location: input.location,
|
|
15226
|
+
maxQuestions: input.maxQuestions,
|
|
15227
|
+
format: "json"
|
|
15228
|
+
}, 28e4);
|
|
15229
|
+
} catch (err) {
|
|
15230
|
+
warnings.push(`PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
15231
|
+
}
|
|
15232
|
+
}
|
|
15233
|
+
if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
|
|
15234
|
+
const rows = sourceRows(input, serp, paa);
|
|
15235
|
+
const target = normalizeDomain(input.domain);
|
|
15236
|
+
const competitors = competitorRows(rows, target);
|
|
15237
|
+
const evidence = {
|
|
15238
|
+
input,
|
|
15239
|
+
serp: serp ?? { status: "skipped" },
|
|
15240
|
+
paa: paa ?? { status: "skipped" },
|
|
15241
|
+
target: {
|
|
15242
|
+
domain: target,
|
|
15243
|
+
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
15244
|
+
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
15245
|
+
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
15246
|
+
},
|
|
15247
|
+
competitors,
|
|
15248
|
+
warnings
|
|
15249
|
+
};
|
|
15250
|
+
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
15251
|
+
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
15252
|
+
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
15253
|
+
const brief = [
|
|
15254
|
+
`# SEO Evidence Brief: ${input.keyword}`,
|
|
15255
|
+
"",
|
|
15256
|
+
`Location: ${input.location ?? "not specified"}`,
|
|
15257
|
+
`Target domain: ${target ?? "not specified"}`,
|
|
15258
|
+
"",
|
|
15259
|
+
"## Evidence Summary",
|
|
15260
|
+
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
15261
|
+
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
15262
|
+
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
15263
|
+
`- Competitor domains: ${competitors.length}`,
|
|
15264
|
+
"",
|
|
15265
|
+
"## Recommended Use",
|
|
15266
|
+
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
15267
|
+
].join("\n");
|
|
15268
|
+
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
15269
|
+
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
15270
|
+
"# Agent Tasks",
|
|
15271
|
+
"",
|
|
15272
|
+
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
15273
|
+
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
15274
|
+
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
15275
|
+
"- [ ] Mark unsupported recommendations as assumptions."
|
|
15276
|
+
].join("\n"));
|
|
15277
|
+
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
15278
|
+
"# Agent Instructions",
|
|
15279
|
+
"",
|
|
15280
|
+
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
15281
|
+
].join("\n"));
|
|
15282
|
+
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
15283
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
15284
|
+
title: "Agent-Ready SEO Packet",
|
|
15285
|
+
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
15286
|
+
summary,
|
|
15287
|
+
warnings,
|
|
15288
|
+
tables: [
|
|
15289
|
+
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
15290
|
+
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
15291
|
+
]
|
|
15292
|
+
}));
|
|
15293
|
+
const status = warnings.length ? "partial" : "succeeded";
|
|
15294
|
+
const counts = { sources: rows.length, competitors: competitors.length };
|
|
15295
|
+
await ctx.artifacts.writeManifest(status, counts, warnings, errors);
|
|
15296
|
+
return { title: "Agent-Ready SEO Packet", summary, status, counts, warnings, errors, reportPath };
|
|
15297
|
+
}
|
|
15298
|
+
};
|
|
15299
|
+
}
|
|
15300
|
+
});
|
|
15301
|
+
|
|
15302
|
+
// src/workflows/workflows/directory.ts
|
|
15303
|
+
function directoryRows(result) {
|
|
15304
|
+
const rows = [];
|
|
15305
|
+
for (const city of result.cities) {
|
|
15306
|
+
if (!city.results.length) {
|
|
15307
|
+
rows.push({
|
|
15308
|
+
source_query: result.query,
|
|
15309
|
+
source_location: city.location,
|
|
15310
|
+
city: city.city,
|
|
15311
|
+
state: city.state,
|
|
15312
|
+
population: city.population,
|
|
15313
|
+
result_position: null,
|
|
15314
|
+
business_name: null,
|
|
15315
|
+
review_stars: null,
|
|
15316
|
+
review_count: null,
|
|
15317
|
+
category: null,
|
|
15318
|
+
address: null,
|
|
15319
|
+
phone: null,
|
|
15320
|
+
website_url: null,
|
|
15321
|
+
place_url: null,
|
|
15322
|
+
cid: null,
|
|
15323
|
+
cid_decimal: null,
|
|
15324
|
+
result_status: city.status,
|
|
15325
|
+
error: city.error
|
|
15326
|
+
});
|
|
15327
|
+
continue;
|
|
15328
|
+
}
|
|
15329
|
+
for (const business of city.results) {
|
|
15330
|
+
rows.push({
|
|
15331
|
+
source_query: result.query,
|
|
15332
|
+
source_location: city.location,
|
|
15333
|
+
city: city.city,
|
|
15334
|
+
state: city.state,
|
|
15335
|
+
population: city.population,
|
|
15336
|
+
result_position: business.position,
|
|
15337
|
+
business_name: business.name,
|
|
15338
|
+
review_stars: business.rating,
|
|
15339
|
+
review_count: business.reviewCount,
|
|
15340
|
+
category: business.category,
|
|
15341
|
+
address: business.address,
|
|
15342
|
+
phone: business.phone,
|
|
15343
|
+
website_url: business.websiteUrl,
|
|
15344
|
+
place_url: business.placeUrl,
|
|
15345
|
+
cid: business.cid,
|
|
15346
|
+
cid_decimal: business.cidDecimal,
|
|
15347
|
+
result_status: city.status,
|
|
15348
|
+
error: city.error
|
|
15349
|
+
});
|
|
15350
|
+
}
|
|
15351
|
+
}
|
|
15352
|
+
return rows;
|
|
15353
|
+
}
|
|
15354
|
+
var import_zod20, DirectoryWorkflowCliInputSchema, DIRECTORY_CSV_HEADERS, directoryWorkflowDefinition;
|
|
15355
|
+
var init_directory = __esm({
|
|
15356
|
+
"src/workflows/workflows/directory.ts"() {
|
|
15357
|
+
"use strict";
|
|
15358
|
+
import_zod20 = require("zod");
|
|
15359
|
+
init_report_renderer();
|
|
15360
|
+
DirectoryWorkflowCliInputSchema = import_zod20.z.object({
|
|
15361
|
+
query: import_zod20.z.string().min(1),
|
|
15362
|
+
state: import_zod20.z.string().min(2).default("TN"),
|
|
15363
|
+
minPopulation: import_zod20.z.number().int().min(0).default(1e5),
|
|
15364
|
+
maxCities: import_zod20.z.number().int().min(1).max(100).default(25),
|
|
15365
|
+
maxResultsPerCity: import_zod20.z.number().int().min(1).max(50).default(20),
|
|
15366
|
+
concurrency: import_zod20.z.number().int().min(1).max(5).default(5),
|
|
15367
|
+
proxyMode: import_zod20.z.enum(["location", "configured", "none"]).default("location"),
|
|
15368
|
+
saveCsv: import_zod20.z.boolean().default(true)
|
|
15369
|
+
});
|
|
15370
|
+
DIRECTORY_CSV_HEADERS = [
|
|
15371
|
+
"source_query",
|
|
15372
|
+
"source_location",
|
|
15373
|
+
"city",
|
|
15374
|
+
"state",
|
|
15375
|
+
"population",
|
|
15376
|
+
"result_position",
|
|
15377
|
+
"business_name",
|
|
15378
|
+
"review_stars",
|
|
15379
|
+
"review_count",
|
|
15380
|
+
"category",
|
|
15381
|
+
"address",
|
|
15382
|
+
"phone",
|
|
15383
|
+
"website_url",
|
|
15384
|
+
"place_url",
|
|
15385
|
+
"cid",
|
|
15386
|
+
"cid_decimal",
|
|
15387
|
+
"result_status",
|
|
15388
|
+
"error"
|
|
15389
|
+
];
|
|
15390
|
+
directoryWorkflowDefinition = {
|
|
15391
|
+
id: "directory",
|
|
15392
|
+
title: "Directory Workflow",
|
|
15393
|
+
description: "Select city markets and export Google Maps business candidates.",
|
|
15394
|
+
inputSchema: DirectoryWorkflowCliInputSchema,
|
|
15395
|
+
async run(input, ctx) {
|
|
15396
|
+
await ctx.artifacts.writeManifest("running", {}, [], []);
|
|
15397
|
+
const result = await ctx.client.post("/directory/run", input, 9e5);
|
|
15398
|
+
await ctx.artifacts.writeJson("Directory evidence", "evidence.json", result);
|
|
15399
|
+
const rows = directoryRows(result);
|
|
15400
|
+
await ctx.artifacts.writeCsv("Directory CSV", "exports/directory.csv", DIRECTORY_CSV_HEADERS, rows);
|
|
15401
|
+
const cityRows = result.cities.map((city) => ({
|
|
15402
|
+
city: city.city,
|
|
15403
|
+
state: city.state,
|
|
15404
|
+
population: city.population,
|
|
15405
|
+
status: city.status,
|
|
15406
|
+
result_count: city.resultCount,
|
|
15407
|
+
error: city.error ?? ""
|
|
15408
|
+
}));
|
|
15409
|
+
const topRows = rows.filter((row) => row.business_name).slice(0, 100).map((row) => ({
|
|
15410
|
+
city: row.city,
|
|
15411
|
+
position: row.result_position,
|
|
15412
|
+
business: row.business_name,
|
|
15413
|
+
rating: row.review_stars,
|
|
15414
|
+
reviews: row.review_count,
|
|
15415
|
+
category: row.category,
|
|
15416
|
+
website: row.website_url
|
|
15417
|
+
}));
|
|
15418
|
+
const summary = `${result.selectedCityCount} cities processed with ${result.totalResultCount} Maps results.`;
|
|
15419
|
+
await ctx.artifacts.writeText("Summary", "summary.md", `# Directory Workflow
|
|
15420
|
+
|
|
15421
|
+
${summary}
|
|
15422
|
+
`);
|
|
15423
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
15424
|
+
title: "Directory Workflow",
|
|
15425
|
+
subtitle: `${input.query} \xB7 ${input.state}`,
|
|
15426
|
+
summary,
|
|
15427
|
+
warnings: result.warnings,
|
|
15428
|
+
tables: [
|
|
15429
|
+
{ title: "Cities", columns: ["city", "state", "population", "status", "result_count", "error"], rows: cityRows },
|
|
15430
|
+
{ title: "Top Results", columns: ["city", "position", "business", "rating", "reviews", "category", "website"], rows: topRows }
|
|
15431
|
+
]
|
|
15432
|
+
}));
|
|
15433
|
+
const status = result.cities.some((city) => city.status === "failed") ? "partial" : "succeeded";
|
|
15434
|
+
const counts = { cities: result.selectedCityCount, results: result.totalResultCount, rows: rows.length };
|
|
15435
|
+
await ctx.artifacts.writeManifest(status, counts, result.warnings, []);
|
|
15436
|
+
return { title: "Directory Workflow", summary, status, counts, warnings: result.warnings, errors: [], reportPath };
|
|
15437
|
+
}
|
|
15438
|
+
};
|
|
15439
|
+
}
|
|
15440
|
+
});
|
|
15441
|
+
|
|
15442
|
+
// src/workflows/workflows/local-competitive-audit.ts
|
|
15443
|
+
async function mapLimit2(items, limit, fn) {
|
|
15444
|
+
const out = new Array(items.length);
|
|
15445
|
+
let next = 0;
|
|
15446
|
+
async function worker() {
|
|
15447
|
+
while (next < items.length) {
|
|
15448
|
+
const index = next;
|
|
15449
|
+
next += 1;
|
|
15450
|
+
out[index] = await fn(items[index], index);
|
|
15451
|
+
}
|
|
15452
|
+
}
|
|
15453
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
15454
|
+
return out;
|
|
15455
|
+
}
|
|
15456
|
+
function numberFrom(value) {
|
|
15457
|
+
if (value === null || value === void 0 || value === "") return null;
|
|
15458
|
+
const parsed = Number(String(value).replace(/[^\d.]/g, ""));
|
|
15459
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
15460
|
+
}
|
|
15461
|
+
function median(values) {
|
|
15462
|
+
const nums = values.filter((v) => v !== null).sort((a, b) => a - b);
|
|
15463
|
+
if (!nums.length) return null;
|
|
15464
|
+
return nums[Math.floor(nums.length / 2)] ?? null;
|
|
15465
|
+
}
|
|
15466
|
+
function termsFrom(texts) {
|
|
15467
|
+
const stop = /* @__PURE__ */ new Set(["the", "and", "for", "with", "that", "this", "was", "were", "are", "you", "our", "they", "had", "have", "not", "but", "from", "very", "great", "good"]);
|
|
15468
|
+
const counts = /* @__PURE__ */ new Map();
|
|
15469
|
+
for (const text of texts) {
|
|
15470
|
+
for (const token of text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/)) {
|
|
15471
|
+
if (token.length < 4 || stop.has(token)) continue;
|
|
15472
|
+
counts.set(token, (counts.get(token) ?? 0) + 1);
|
|
15473
|
+
}
|
|
15474
|
+
}
|
|
15475
|
+
return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8).map(([term, count]) => `${term} (${count})`).join("; ");
|
|
15476
|
+
}
|
|
15477
|
+
var import_zod21, LocalCompetitiveAuditInputSchema, localCompetitiveAuditWorkflowDefinition;
|
|
15478
|
+
var init_local_competitive_audit = __esm({
|
|
15479
|
+
"src/workflows/workflows/local-competitive-audit.ts"() {
|
|
15480
|
+
"use strict";
|
|
15481
|
+
import_zod21 = require("zod");
|
|
15482
|
+
init_report_renderer();
|
|
15483
|
+
init_directory();
|
|
15484
|
+
LocalCompetitiveAuditInputSchema = import_zod21.z.object({
|
|
15485
|
+
query: import_zod21.z.string().min(1),
|
|
15486
|
+
state: import_zod21.z.string().min(2).default("TN"),
|
|
15487
|
+
minPopulation: import_zod21.z.number().int().min(0).default(1e5),
|
|
15488
|
+
maxCities: import_zod21.z.number().int().min(1).max(100).default(25),
|
|
15489
|
+
maxResultsPerCity: import_zod21.z.number().int().min(1).max(50).default(20),
|
|
15490
|
+
hydrateTop: import_zod21.z.number().int().min(0).max(10).default(5),
|
|
15491
|
+
maxReviews: import_zod21.z.number().int().min(0).max(500).default(50),
|
|
15492
|
+
concurrency: import_zod21.z.number().int().min(1).max(5).default(5),
|
|
15493
|
+
proxyMode: import_zod21.z.enum(["location", "configured", "none"]).default("location"),
|
|
15494
|
+
returnPartial: import_zod21.z.boolean().default(true)
|
|
15495
|
+
});
|
|
15496
|
+
localCompetitiveAuditWorkflowDefinition = {
|
|
15497
|
+
id: "local-competitive-audit",
|
|
15498
|
+
title: "Local Competitive Audit",
|
|
15499
|
+
description: "Audit local Maps competitors, categories, review counts, and review themes across city markets.",
|
|
15500
|
+
inputSchema: LocalCompetitiveAuditInputSchema,
|
|
15501
|
+
async run(input, ctx) {
|
|
15502
|
+
await ctx.artifacts.writeManifest("running", {}, [], []);
|
|
15503
|
+
const warnings = [];
|
|
15504
|
+
const errors = [];
|
|
15505
|
+
const directory = await ctx.client.post("/directory/run", {
|
|
15506
|
+
query: input.query,
|
|
15507
|
+
state: input.state,
|
|
15508
|
+
minPopulation: input.minPopulation,
|
|
15509
|
+
maxCities: input.maxCities,
|
|
15510
|
+
maxResultsPerCity: input.maxResultsPerCity,
|
|
15511
|
+
concurrency: input.concurrency,
|
|
15512
|
+
proxyMode: input.proxyMode,
|
|
15513
|
+
saveCsv: true
|
|
15514
|
+
}, 9e5);
|
|
15515
|
+
await ctx.artifacts.writeJson("Directory raw JSON", "raw/directory-workflow.json", directory);
|
|
15516
|
+
const baseRows = directoryRows(directory);
|
|
15517
|
+
await ctx.artifacts.writeCsv("Base directory CSV", "exports/directory.csv", DIRECTORY_CSV_HEADERS, baseRows);
|
|
15518
|
+
const selected = directory.cities.flatMap(
|
|
15519
|
+
(city) => city.results.slice(0, input.hydrateTop).map((result) => ({ city, result }))
|
|
15520
|
+
);
|
|
15521
|
+
const seen = /* @__PURE__ */ new Set();
|
|
15522
|
+
const deduped = selected.filter(({ city, result }) => {
|
|
15523
|
+
const key = result.cid ?? result.placeUrl ?? `${city.location}:${result.name.toLowerCase()}`;
|
|
15524
|
+
if (seen.has(key)) return false;
|
|
15525
|
+
seen.add(key);
|
|
15526
|
+
return true;
|
|
15527
|
+
});
|
|
15528
|
+
const hydrated = await mapLimit2(deduped, 3, async ({ city, result }, index) => {
|
|
15529
|
+
try {
|
|
15530
|
+
const detail = await ctx.client.post("/maps/place", {
|
|
15531
|
+
businessName: result.name,
|
|
15532
|
+
location: city.location,
|
|
15533
|
+
includeReviews: input.maxReviews > 0,
|
|
15534
|
+
maxReviews: Math.max(1, input.maxReviews)
|
|
15535
|
+
}, 18e4);
|
|
15536
|
+
await ctx.artifacts.writeJson(`${result.name} profile`, `raw/maps-place-intel/${index + 1}-${result.name.toLowerCase().replace(/[^a-z0-9]+/g, "-")}.json`, detail);
|
|
15537
|
+
return { city, result, detail, error: null };
|
|
15538
|
+
} catch (err) {
|
|
15539
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
15540
|
+
warnings.push(`Profile hydration failed for ${result.name} (${city.location}): ${message}`);
|
|
15541
|
+
return { city, result, detail: null, error: message };
|
|
15542
|
+
}
|
|
15543
|
+
});
|
|
15544
|
+
const competitorRows2 = hydrated.map(({ city, result, detail, error }) => {
|
|
15545
|
+
const reviews = detail?.reviews ?? [];
|
|
15546
|
+
const ownerResponses = reviews.filter((r) => r.ownerResponse).length;
|
|
15547
|
+
return {
|
|
15548
|
+
city: city.city,
|
|
15549
|
+
state: city.state,
|
|
15550
|
+
source_location: city.location,
|
|
15551
|
+
result_position: result.position,
|
|
15552
|
+
business_name: result.name,
|
|
15553
|
+
category: detail?.category ?? result.category,
|
|
15554
|
+
review_stars: detail?.rating ?? result.rating,
|
|
15555
|
+
review_count: detail?.reviewCount ?? result.reviewCount,
|
|
15556
|
+
phone: result.phone,
|
|
15557
|
+
website_url: detail?.website ?? result.websiteUrl,
|
|
15558
|
+
place_url: result.placeUrl,
|
|
15559
|
+
cid: result.cid,
|
|
15560
|
+
cid_decimal: result.cidDecimal,
|
|
15561
|
+
hydrated: detail ? "true" : "false",
|
|
15562
|
+
reviews_status: detail?.reviewsStatus ?? "",
|
|
15563
|
+
review_topics: (detail?.reviewTopics ?? []).map((t) => `${t.label} (${t.count})`).join("; "),
|
|
15564
|
+
owner_response_rate: reviews.length ? (ownerResponses / reviews.length).toFixed(2) : "",
|
|
15565
|
+
error: error ?? ""
|
|
15566
|
+
};
|
|
15567
|
+
});
|
|
15568
|
+
const reviewInsightRows = hydrated.map(({ city, result, detail }) => {
|
|
15569
|
+
const reviews = detail?.reviews ?? [];
|
|
15570
|
+
const positive = reviews.filter((r) => Number(r.stars) >= 5 && r.text).map((r) => r.text);
|
|
15571
|
+
const negative = reviews.filter((r) => Number(r.stars) > 0 && Number(r.stars) <= 3 && r.text).map((r) => r.text);
|
|
15572
|
+
return {
|
|
15573
|
+
city: city.city,
|
|
15574
|
+
business_name: result.name,
|
|
15575
|
+
cid: result.cid,
|
|
15576
|
+
review_count: detail?.reviewCount ?? result.reviewCount,
|
|
15577
|
+
average_rating: detail?.rating ?? result.rating,
|
|
15578
|
+
topics: (detail?.reviewTopics ?? []).map((t) => `${t.label} (${t.count})`).join("; "),
|
|
15579
|
+
praise_terms: termsFrom(positive),
|
|
15580
|
+
complaint_terms: termsFrom(negative),
|
|
15581
|
+
positive_sample: positive[0]?.slice(0, 280) ?? "",
|
|
15582
|
+
negative_sample: negative[0]?.slice(0, 280) ?? ""
|
|
15583
|
+
};
|
|
15584
|
+
});
|
|
15585
|
+
const citySummaryRows = directory.cities.map((city) => {
|
|
15586
|
+
const resultReviewCounts = city.results.map((r) => numberFrom(r.reviewCount));
|
|
15587
|
+
const resultRatings = city.results.map((r) => numberFrom(r.rating));
|
|
15588
|
+
const topThreeReviews = city.results.slice(0, 3).map((r) => numberFrom(r.reviewCount)).filter((v) => v !== null);
|
|
15589
|
+
const categories = /* @__PURE__ */ new Map();
|
|
15590
|
+
for (const result of city.results) {
|
|
15591
|
+
if (!result.category) continue;
|
|
15592
|
+
categories.set(result.category, (categories.get(result.category) ?? 0) + 1);
|
|
15593
|
+
}
|
|
15594
|
+
const medReviews = median(resultReviewCounts);
|
|
15595
|
+
const medRating = median(resultRatings);
|
|
15596
|
+
const topThreeAvg = topThreeReviews.length ? Math.round(topThreeReviews.reduce((a, b) => a + b, 0) / topThreeReviews.length) : null;
|
|
15597
|
+
const difficulty = Math.min(100, Math.round((topThreeAvg ?? 0) / 20 + (medRating ?? 0) * 10 + city.results.filter((r) => r.websiteUrl).length));
|
|
15598
|
+
const opportunity = Math.max(0, 100 - difficulty);
|
|
15599
|
+
return {
|
|
15600
|
+
city: city.city,
|
|
15601
|
+
state: city.state,
|
|
15602
|
+
population: city.population,
|
|
15603
|
+
result_count: city.resultCount,
|
|
15604
|
+
median_review_count: medReviews ?? "",
|
|
15605
|
+
median_rating: medRating ?? "",
|
|
15606
|
+
top_three_average_review_count: topThreeAvg ?? "",
|
|
15607
|
+
top_categories: [...categories.entries()].sort((a, b) => b[1] - a[1]).slice(0, 5).map(([cat, count]) => `${cat} (${count})`).join("; "),
|
|
15608
|
+
opportunity_score: opportunity,
|
|
15609
|
+
difficulty_score: difficulty
|
|
15610
|
+
};
|
|
15611
|
+
});
|
|
15612
|
+
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["city", "state", "source_location", "result_position", "business_name", "category", "review_stars", "review_count", "phone", "website_url", "place_url", "cid", "cid_decimal", "hydrated", "reviews_status", "review_topics", "owner_response_rate", "error"], competitorRows2);
|
|
15613
|
+
await ctx.artifacts.writeCsv("Review insights CSV", "review-insights.csv", ["city", "business_name", "cid", "review_count", "average_rating", "topics", "praise_terms", "complaint_terms", "positive_sample", "negative_sample"], reviewInsightRows);
|
|
15614
|
+
await ctx.artifacts.writeCsv("City summary CSV", "city-summary.csv", ["city", "state", "population", "result_count", "median_review_count", "median_rating", "top_three_average_review_count", "top_categories", "opportunity_score", "difficulty_score"], citySummaryRows);
|
|
15615
|
+
await ctx.artifacts.writeJson("Audit evidence", "evidence.json", { input, directory, hydrated, citySummaryRows, competitorRows: competitorRows2, reviewInsightRows });
|
|
15616
|
+
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
15617
|
+
"# Local Competitive Audit Tasks",
|
|
15618
|
+
"",
|
|
15619
|
+
"- [ ] Review city opportunity and difficulty scores as heuristics, not absolute truth.",
|
|
15620
|
+
"- [ ] Use review topics and samples as customer-language evidence.",
|
|
15621
|
+
"- [ ] Compare target GBP category, review count, and website quality against top competitors.",
|
|
15622
|
+
"- [ ] Identify cities with low review-count leaders and weak website coverage."
|
|
15623
|
+
].join("\n"));
|
|
15624
|
+
const summary = `${directory.selectedCityCount} cities, ${directory.totalResultCount} Maps results, ${hydrated.filter((h) => h.detail).length} hydrated profiles.`;
|
|
15625
|
+
await ctx.artifacts.writeText("Summary", "summary.md", `# Local Competitive Audit
|
|
15626
|
+
|
|
15627
|
+
${summary}
|
|
15628
|
+
`);
|
|
15629
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
15630
|
+
title: "Local Competitive Audit",
|
|
15631
|
+
subtitle: `${input.query} \xB7 ${input.state}`,
|
|
15632
|
+
summary,
|
|
15633
|
+
warnings,
|
|
15634
|
+
tables: [
|
|
15635
|
+
{ title: "City Opportunity", columns: ["city", "state", "population", "result_count", "median_review_count", "median_rating", "top_three_average_review_count", "top_categories", "opportunity_score", "difficulty_score"], rows: citySummaryRows },
|
|
15636
|
+
{ title: "Hydrated Competitors", columns: ["city", "result_position", "business_name", "category", "review_stars", "review_count", "hydrated", "review_topics"], rows: competitorRows2.slice(0, 100) },
|
|
15637
|
+
{ title: "Review Insights", columns: ["city", "business_name", "topics", "praise_terms", "complaint_terms"], rows: reviewInsightRows.slice(0, 100) }
|
|
15638
|
+
]
|
|
15639
|
+
}));
|
|
15640
|
+
const status = warnings.length || directory.cities.some((city) => city.status === "failed") ? "partial" : "succeeded";
|
|
15641
|
+
const counts = { cities: directory.selectedCityCount, mapsResults: directory.totalResultCount, hydratedProfiles: hydrated.filter((h) => h.detail).length };
|
|
15642
|
+
await ctx.artifacts.writeManifest(status, counts, warnings, errors);
|
|
15643
|
+
return { title: "Local Competitive Audit", summary, status, counts, warnings, errors, reportPath };
|
|
15644
|
+
}
|
|
15645
|
+
};
|
|
15646
|
+
}
|
|
15647
|
+
});
|
|
15648
|
+
|
|
15649
|
+
// src/workflows/registry.ts
|
|
15650
|
+
function listWorkflowDefinitions() {
|
|
15651
|
+
return DEFINITIONS.map(({ id, title, description }) => ({ id, title, description }));
|
|
15652
|
+
}
|
|
15653
|
+
function workflowDefinition(id) {
|
|
15654
|
+
const definition = DEFINITIONS.find((def) => def.id === id);
|
|
15655
|
+
if (!definition) throw new Error(`Unknown workflow "${id}". Available: ${DEFINITIONS.map((def) => def.id).join(", ")}`);
|
|
15656
|
+
return definition;
|
|
15657
|
+
}
|
|
15658
|
+
async function runWorkflow(id, rawInput, options = {}) {
|
|
15659
|
+
const definition = workflowDefinition(id);
|
|
15660
|
+
const input = definition.inputSchema.parse(rawInput);
|
|
15661
|
+
const apiKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim();
|
|
15662
|
+
if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
|
|
15663
|
+
const apiUrl = options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
15664
|
+
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
15665
|
+
const client2 = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl);
|
|
15666
|
+
try {
|
|
15667
|
+
return await definition.run(input, {
|
|
15668
|
+
runId: artifacts.runId,
|
|
15669
|
+
startedAt: artifacts.startedAt,
|
|
15670
|
+
client: client2,
|
|
15671
|
+
artifacts,
|
|
15672
|
+
signal: options.signal
|
|
15673
|
+
});
|
|
15674
|
+
} catch (err) {
|
|
15675
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
15676
|
+
await artifacts.writeText("Failure", "summary.md", `# ${definition.title}
|
|
15677
|
+
|
|
15678
|
+
${message}
|
|
15679
|
+
`);
|
|
15680
|
+
await artifacts.writeManifest("failed", {}, [], [message]);
|
|
15681
|
+
throw err;
|
|
15682
|
+
}
|
|
15683
|
+
}
|
|
15684
|
+
var import_zod22, DEFINITIONS;
|
|
15685
|
+
var init_registry2 = __esm({
|
|
15686
|
+
"src/workflows/registry.ts"() {
|
|
15687
|
+
"use strict";
|
|
15688
|
+
import_zod22 = require("zod");
|
|
15689
|
+
init_artifact_writer();
|
|
15690
|
+
init_http_client2();
|
|
15691
|
+
init_agent_packet();
|
|
15692
|
+
init_directory();
|
|
15693
|
+
init_local_competitive_audit();
|
|
15694
|
+
DEFINITIONS = [
|
|
15695
|
+
directoryWorkflowDefinition,
|
|
15696
|
+
agentPacketWorkflowDefinition,
|
|
15697
|
+
localCompetitiveAuditWorkflowDefinition
|
|
15698
|
+
];
|
|
15699
|
+
}
|
|
15700
|
+
});
|
|
15701
|
+
|
|
15702
|
+
// src/api/workflow-routes.ts
|
|
15703
|
+
function hostedWorkflowOutputDir() {
|
|
15704
|
+
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
15705
|
+
}
|
|
15706
|
+
function originFromUrl(url) {
|
|
15707
|
+
const parsed = new URL(url);
|
|
15708
|
+
return `${parsed.protocol}//${parsed.host}`;
|
|
15709
|
+
}
|
|
15710
|
+
function addCadence(fromIso, cadence) {
|
|
15711
|
+
const next = new Date(fromIso);
|
|
15712
|
+
if (!Number.isFinite(next.getTime())) return new Date(Date.now() + 24 * 60 * 60 * 1e3).toISOString();
|
|
15713
|
+
if (cadence === "daily") next.setUTCDate(next.getUTCDate() + 1);
|
|
15714
|
+
else if (cadence === "weekly") next.setUTCDate(next.getUTCDate() + 7);
|
|
15715
|
+
else next.setUTCMonth(next.getUTCMonth() + 1);
|
|
15716
|
+
return next.toISOString();
|
|
15717
|
+
}
|
|
15718
|
+
function defaultNextRunAt(cadence) {
|
|
15719
|
+
return addCadence((/* @__PURE__ */ new Date()).toISOString(), cadence);
|
|
15720
|
+
}
|
|
15721
|
+
function validateWorkflowInput(workflowId, input) {
|
|
15722
|
+
const definition = workflowDefinition(workflowId);
|
|
15723
|
+
return definition.inputSchema.parse(input);
|
|
15724
|
+
}
|
|
15725
|
+
async function validateWebhookUrl(url) {
|
|
15726
|
+
if (!url) return null;
|
|
15727
|
+
const result = await validatePublicHttpUrl(url, { field: "webhookUrl", requireHttps: true });
|
|
15728
|
+
if (result.error) throw new Error(result.error);
|
|
15729
|
+
return url;
|
|
15730
|
+
}
|
|
15731
|
+
function runDownloadUrl(c, runId, artifactId) {
|
|
15732
|
+
return `${originFromUrl(c.req.url)}/workflows/runs/${runId}/artifacts/${artifactId}`;
|
|
15733
|
+
}
|
|
15734
|
+
function exposeRun(c, run, artifacts = []) {
|
|
15735
|
+
return {
|
|
15736
|
+
...run,
|
|
15737
|
+
artifacts: artifacts.map((artifact) => ({
|
|
15738
|
+
...artifact,
|
|
15739
|
+
downloadUrl: runDownloadUrl(c, run.id, artifact.id)
|
|
15740
|
+
}))
|
|
15741
|
+
};
|
|
15742
|
+
}
|
|
15743
|
+
async function persistRunArtifacts(runId, userId, manifest) {
|
|
15744
|
+
if (!manifest) return [];
|
|
15745
|
+
return replaceWorkflowArtifacts(runId, userId, manifest.artifacts);
|
|
15746
|
+
}
|
|
15747
|
+
async function readManifestFromSummary(summary) {
|
|
15748
|
+
if (!summary.reportPath) return null;
|
|
15749
|
+
const manifestPath = summary.reportPath.replace(/report\.html$/, "manifest.json");
|
|
15750
|
+
try {
|
|
15751
|
+
return JSON.parse(await (0, import_promises7.readFile)(manifestPath, "utf8"));
|
|
15752
|
+
} catch {
|
|
15753
|
+
return null;
|
|
15754
|
+
}
|
|
15755
|
+
}
|
|
15756
|
+
function webhookSignature(body, timestamp2) {
|
|
15757
|
+
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
15758
|
+
if (!secret2) return null;
|
|
15759
|
+
return (0, import_node_crypto2.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
|
|
15760
|
+
}
|
|
15761
|
+
async function deliverWorkflowWebhook(input) {
|
|
15762
|
+
if (!input.webhookUrl) return;
|
|
15763
|
+
const payload = {
|
|
15764
|
+
event: input.run.status === "failed" ? "workflow.run.failed" : input.run.status === "partial" ? "workflow.run.partial" : "workflow.run.succeeded",
|
|
15765
|
+
runId: input.run.id,
|
|
15766
|
+
scheduleId: input.scheduleId,
|
|
15767
|
+
workflowId: input.run.workflow_id,
|
|
15768
|
+
status: input.run.status,
|
|
15769
|
+
completedAt: input.run.completed_at ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
15770
|
+
summary: {
|
|
15771
|
+
title: input.summary?.title ?? input.run.workflow_id,
|
|
15772
|
+
counts: input.summary?.counts ?? {},
|
|
15773
|
+
warnings: input.summary?.warnings ?? [],
|
|
15774
|
+
errors: input.summary?.errors ?? (input.run.error_message ? [input.run.error_message] : [])
|
|
15775
|
+
},
|
|
15776
|
+
artifacts: input.artifacts.map((artifact) => ({
|
|
15777
|
+
kind: artifact.kind,
|
|
15778
|
+
label: artifact.label,
|
|
15779
|
+
path: artifact.path
|
|
15780
|
+
}))
|
|
15781
|
+
};
|
|
15782
|
+
const body = JSON.stringify(payload);
|
|
15783
|
+
const timestamp2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
15784
|
+
const headers = {
|
|
15785
|
+
"Content-Type": "application/json",
|
|
15786
|
+
"x-mcp-scraper-timestamp": timestamp2
|
|
15787
|
+
};
|
|
15788
|
+
const signature = webhookSignature(body, timestamp2);
|
|
15789
|
+
if (signature) headers["x-mcp-scraper-signature"] = signature;
|
|
15790
|
+
let lastError = null;
|
|
15791
|
+
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
15792
|
+
try {
|
|
15793
|
+
const res = await fetch(input.webhookUrl, { method: "POST", headers, body, signal: AbortSignal.timeout(15e3) });
|
|
15794
|
+
if (res.ok) {
|
|
15795
|
+
await recordWorkflowWebhookDelivery({ runId: input.run.id, scheduleId: input.scheduleId, userId: input.userId, webhookUrl: input.webhookUrl, status: "delivered", attempts: attempt });
|
|
15796
|
+
return;
|
|
15797
|
+
}
|
|
15798
|
+
lastError = `HTTP ${res.status}`;
|
|
15799
|
+
} catch (err) {
|
|
15800
|
+
lastError = err instanceof Error ? err.message : String(err);
|
|
15801
|
+
}
|
|
15802
|
+
await new Promise((resolve) => setTimeout(resolve, attempt * 500));
|
|
15803
|
+
}
|
|
15804
|
+
await recordWorkflowWebhookDelivery({ runId: input.run.id, scheduleId: input.scheduleId, userId: input.userId, webhookUrl: input.webhookUrl, status: "failed", attempts: 3, lastError });
|
|
15805
|
+
}
|
|
15806
|
+
async function executeWorkflowRun(input) {
|
|
15807
|
+
await markWorkflowRunRunning(input.runId);
|
|
15808
|
+
try {
|
|
15809
|
+
const summary = await runWorkflow(input.workflowId, input.workflowInput, {
|
|
15810
|
+
apiKey: input.user.api_key,
|
|
15811
|
+
apiUrl: input.apiUrl,
|
|
15812
|
+
outputDir: hostedWorkflowOutputDir(),
|
|
15813
|
+
runId: input.runId
|
|
15814
|
+
});
|
|
15815
|
+
const manifest = await readManifestFromSummary(summary);
|
|
15816
|
+
await completeWorkflowRunRecord(input.runId, summary.status, manifest ?? { summary });
|
|
15817
|
+
const run = await getWorkflowRun(input.runId, input.user.id);
|
|
15818
|
+
if (!run) throw new Error("Workflow run disappeared after execution");
|
|
15819
|
+
const artifacts = await persistRunArtifacts(input.runId, input.user.id, manifest);
|
|
15820
|
+
await deliverWorkflowWebhook({ userId: input.user.id, run, scheduleId: input.scheduleId ?? null, webhookUrl: input.webhookUrl ?? null, summary, artifacts });
|
|
15821
|
+
return { run, summary, artifacts };
|
|
15822
|
+
} catch (err) {
|
|
15823
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
15824
|
+
await failWorkflowRunRecord(input.runId, message, { error: message });
|
|
15825
|
+
const run = await getWorkflowRun(input.runId, input.user.id);
|
|
15826
|
+
if (!run) throw err;
|
|
15827
|
+
await deliverWorkflowWebhook({ userId: input.user.id, run, scheduleId: input.scheduleId ?? null, webhookUrl: input.webhookUrl ?? null, summary: null, artifacts: [] });
|
|
15828
|
+
throw err;
|
|
15829
|
+
}
|
|
15830
|
+
}
|
|
15831
|
+
async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
15832
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15833
|
+
const due = await listDueWorkflowSchedules(now, limit);
|
|
15834
|
+
const results = [];
|
|
15835
|
+
for (const schedule of due) {
|
|
15836
|
+
const user = await getUserById(schedule.user_id);
|
|
15837
|
+
if (!user) {
|
|
15838
|
+
results.push({ scheduleId: schedule.id, status: "failed", error: "User not found" });
|
|
15839
|
+
continue;
|
|
15840
|
+
}
|
|
15841
|
+
const scheduledFor = schedule.next_run_at ?? now;
|
|
15842
|
+
const run = await createWorkflowRun({
|
|
15843
|
+
userId: user.id,
|
|
15844
|
+
scheduleId: schedule.id,
|
|
15845
|
+
workflowId: schedule.workflow_id,
|
|
15846
|
+
workflowInput: schedule.input,
|
|
15847
|
+
idempotencyKey: `${schedule.id}:${scheduledFor}`
|
|
15848
|
+
});
|
|
15849
|
+
await markWorkflowScheduleRan(schedule.id, now, addCadence(scheduledFor, schedule.cadence));
|
|
15850
|
+
if (run.status !== "queued") {
|
|
15851
|
+
results.push({ scheduleId: schedule.id, runId: run.id, status: run.status });
|
|
15852
|
+
continue;
|
|
15853
|
+
}
|
|
15854
|
+
try {
|
|
15855
|
+
const executed = await executeWorkflowRun({
|
|
15856
|
+
runId: run.id,
|
|
15857
|
+
user,
|
|
15858
|
+
workflowId: schedule.workflow_id,
|
|
15859
|
+
workflowInput: schedule.input,
|
|
15860
|
+
apiUrl,
|
|
15861
|
+
scheduleId: schedule.id,
|
|
15862
|
+
webhookUrl: schedule.webhook_url
|
|
15863
|
+
});
|
|
15864
|
+
results.push({ scheduleId: schedule.id, runId: executed.run.id, status: executed.run.status });
|
|
15865
|
+
} catch (err) {
|
|
15866
|
+
results.push({ scheduleId: schedule.id, runId: run.id, status: "failed", error: err instanceof Error ? err.message : String(err) });
|
|
15867
|
+
}
|
|
15868
|
+
}
|
|
15869
|
+
return { dispatched: results.length, results };
|
|
15870
|
+
}
|
|
15871
|
+
var import_node_crypto2, import_promises7, import_hono7, import_zod23, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema;
|
|
15872
|
+
var init_workflow_routes = __esm({
|
|
15873
|
+
"src/api/workflow-routes.ts"() {
|
|
15874
|
+
"use strict";
|
|
15875
|
+
import_node_crypto2 = require("crypto");
|
|
15876
|
+
import_promises7 = require("fs/promises");
|
|
15877
|
+
import_hono7 = require("hono");
|
|
15878
|
+
import_zod23 = require("zod");
|
|
15879
|
+
init_artifact_writer();
|
|
15880
|
+
init_registry2();
|
|
15881
|
+
init_api_auth();
|
|
15882
|
+
init_db();
|
|
15883
|
+
init_url_utils();
|
|
15884
|
+
workflowApp = new import_hono7.Hono();
|
|
15885
|
+
WorkflowInputSchema = import_zod23.z.record(import_zod23.z.unknown()).default({});
|
|
15886
|
+
WorkflowIdSchema = import_zod23.z.string().min(1);
|
|
15887
|
+
CadenceSchema = import_zod23.z.enum(["daily", "weekly", "monthly"]);
|
|
15888
|
+
ScheduleStatusSchema = import_zod23.z.enum(["active", "paused"]);
|
|
15889
|
+
RunBodySchema = import_zod23.z.object({
|
|
15890
|
+
workflowId: WorkflowIdSchema,
|
|
15891
|
+
input: WorkflowInputSchema,
|
|
15892
|
+
webhookUrl: import_zod23.z.string().url().optional()
|
|
15893
|
+
});
|
|
15894
|
+
ScheduleCreateSchema = import_zod23.z.object({
|
|
15895
|
+
workflowId: WorkflowIdSchema,
|
|
15896
|
+
name: import_zod23.z.string().min(1).max(120).optional(),
|
|
15897
|
+
input: WorkflowInputSchema,
|
|
15898
|
+
cadence: CadenceSchema.default("weekly"),
|
|
15899
|
+
timezone: import_zod23.z.string().min(1).max(64).default("UTC"),
|
|
15900
|
+
webhookUrl: import_zod23.z.string().url().optional(),
|
|
15901
|
+
nextRunAt: import_zod23.z.string().datetime().optional()
|
|
15902
|
+
});
|
|
15903
|
+
SchedulePatchSchema = import_zod23.z.object({
|
|
15904
|
+
name: import_zod23.z.string().min(1).max(120).optional(),
|
|
15905
|
+
status: ScheduleStatusSchema.optional(),
|
|
15906
|
+
input: WorkflowInputSchema.optional(),
|
|
15907
|
+
cadence: CadenceSchema.optional(),
|
|
15908
|
+
timezone: import_zod23.z.string().min(1).max(64).optional(),
|
|
15909
|
+
webhookUrl: import_zod23.z.string().url().nullable().optional(),
|
|
15910
|
+
nextRunAt: import_zod23.z.string().datetime().nullable().optional()
|
|
15911
|
+
});
|
|
15912
|
+
workflowApp.get("/definitions", createApiKeyAuth(), (c) => {
|
|
15913
|
+
return c.json({ workflows: listWorkflowDefinitions() });
|
|
15914
|
+
});
|
|
15915
|
+
workflowApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
15916
|
+
const user = c.get("user");
|
|
15917
|
+
const parsed = RunBodySchema.safeParse(await c.req.json().catch(() => ({})));
|
|
15918
|
+
if (!parsed.success) return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
15919
|
+
let input;
|
|
15920
|
+
let webhookUrl;
|
|
15921
|
+
try {
|
|
15922
|
+
input = validateWorkflowInput(parsed.data.workflowId, parsed.data.input);
|
|
15923
|
+
webhookUrl = await validateWebhookUrl(parsed.data.webhookUrl);
|
|
15924
|
+
} catch (err) {
|
|
15925
|
+
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
15926
|
+
}
|
|
15927
|
+
const run = await createWorkflowRun({ userId: user.id, workflowId: parsed.data.workflowId, workflowInput: input });
|
|
15928
|
+
try {
|
|
15929
|
+
const executed = await executeWorkflowRun({
|
|
15930
|
+
runId: run.id,
|
|
15931
|
+
user,
|
|
15932
|
+
workflowId: parsed.data.workflowId,
|
|
15933
|
+
workflowInput: input,
|
|
15934
|
+
apiUrl: originFromUrl(c.req.url),
|
|
15935
|
+
webhookUrl
|
|
15936
|
+
});
|
|
15937
|
+
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
15938
|
+
} catch (err) {
|
|
15939
|
+
const failed = await getWorkflowRun(run.id, user.id);
|
|
15940
|
+
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
15941
|
+
}
|
|
15942
|
+
});
|
|
15943
|
+
workflowApp.get("/runs", createApiKeyAuth(), async (c) => {
|
|
15944
|
+
const user = c.get("user");
|
|
15945
|
+
const runs = await listWorkflowRuns(user.id);
|
|
15946
|
+
return c.json({ runs });
|
|
15947
|
+
});
|
|
15948
|
+
workflowApp.get("/runs/:id", createApiKeyAuth(), async (c) => {
|
|
15949
|
+
const user = c.get("user");
|
|
15950
|
+
const run = await getWorkflowRun(c.req.param("id"), user.id);
|
|
15951
|
+
if (!run) return c.json({ error: "Run not found" }, 404);
|
|
15952
|
+
const artifacts = await listWorkflowArtifacts(run.id, user.id);
|
|
15953
|
+
return c.json({ run: exposeRun(c, run, artifacts) });
|
|
15954
|
+
});
|
|
15955
|
+
workflowApp.get("/runs/:id/artifacts/:artifactId", createApiKeyAuth(), async (c) => {
|
|
15956
|
+
const user = c.get("user");
|
|
15957
|
+
const run = await getWorkflowRun(c.req.param("id"), user.id);
|
|
15958
|
+
if (!run) return c.json({ error: "Run not found" }, 404);
|
|
15959
|
+
const artifact = await getWorkflowArtifact(c.req.param("artifactId"), user.id);
|
|
15960
|
+
if (!artifact || artifact.run_id !== run.id) return c.json({ error: "Artifact not found" }, 404);
|
|
15961
|
+
try {
|
|
15962
|
+
const content = await (0, import_promises7.readFile)(artifact.path);
|
|
15963
|
+
return new Response(content, {
|
|
15964
|
+
headers: {
|
|
15965
|
+
"Content-Type": artifact.content_type,
|
|
15966
|
+
"Content-Disposition": `attachment; filename="${artifact.path.split("/").pop() || artifact.id}"`
|
|
15967
|
+
}
|
|
15968
|
+
});
|
|
15969
|
+
} catch {
|
|
15970
|
+
return c.json({ error: "Artifact file is no longer available" }, 410);
|
|
15971
|
+
}
|
|
15972
|
+
});
|
|
15973
|
+
workflowApp.post("/schedules", createApiKeyAuth(), async (c) => {
|
|
15974
|
+
const user = c.get("user");
|
|
15975
|
+
const parsed = ScheduleCreateSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
15976
|
+
if (!parsed.success) return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
15977
|
+
let input;
|
|
15978
|
+
let webhookUrl;
|
|
15979
|
+
try {
|
|
15980
|
+
input = validateWorkflowInput(parsed.data.workflowId, parsed.data.input);
|
|
15981
|
+
webhookUrl = await validateWebhookUrl(parsed.data.webhookUrl);
|
|
15982
|
+
} catch (err) {
|
|
15983
|
+
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
15984
|
+
}
|
|
15985
|
+
const schedule = await createWorkflowSchedule({
|
|
15986
|
+
userId: user.id,
|
|
15987
|
+
workflowId: parsed.data.workflowId,
|
|
15988
|
+
name: parsed.data.name ?? workflowDefinition(parsed.data.workflowId).title,
|
|
15989
|
+
workflowInput: input,
|
|
15990
|
+
cadence: parsed.data.cadence,
|
|
15991
|
+
timezone: parsed.data.timezone,
|
|
15992
|
+
webhookUrl,
|
|
15993
|
+
nextRunAt: parsed.data.nextRunAt ?? defaultNextRunAt(parsed.data.cadence)
|
|
15994
|
+
});
|
|
15995
|
+
return c.json({ schedule }, 201);
|
|
15996
|
+
});
|
|
15997
|
+
workflowApp.get("/schedules", createApiKeyAuth(), async (c) => {
|
|
15998
|
+
const user = c.get("user");
|
|
15999
|
+
return c.json({ schedules: await listWorkflowSchedules(user.id) });
|
|
16000
|
+
});
|
|
16001
|
+
workflowApp.patch("/schedules/:id", createApiKeyAuth(), async (c) => {
|
|
16002
|
+
const user = c.get("user");
|
|
16003
|
+
const existing = await getWorkflowSchedule(c.req.param("id"), user.id);
|
|
16004
|
+
if (!existing) return c.json({ error: "Schedule not found" }, 404);
|
|
16005
|
+
const parsed = SchedulePatchSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
16006
|
+
if (!parsed.success) return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
16007
|
+
let nextInput;
|
|
16008
|
+
let webhookUrl;
|
|
16009
|
+
try {
|
|
16010
|
+
nextInput = parsed.data.input ? validateWorkflowInput(existing.workflow_id, parsed.data.input) : void 0;
|
|
16011
|
+
webhookUrl = parsed.data.webhookUrl === void 0 ? void 0 : await validateWebhookUrl(parsed.data.webhookUrl);
|
|
16012
|
+
} catch (err) {
|
|
16013
|
+
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
16014
|
+
}
|
|
16015
|
+
const schedule = await patchWorkflowSchedule({
|
|
16016
|
+
id: existing.id,
|
|
16017
|
+
userId: user.id,
|
|
16018
|
+
name: parsed.data.name,
|
|
16019
|
+
status: parsed.data.status,
|
|
16020
|
+
workflowInput: nextInput,
|
|
16021
|
+
cadence: parsed.data.cadence,
|
|
16022
|
+
timezone: parsed.data.timezone,
|
|
16023
|
+
webhookUrl,
|
|
16024
|
+
nextRunAt: parsed.data.nextRunAt
|
|
16025
|
+
});
|
|
16026
|
+
return c.json({ schedule });
|
|
16027
|
+
});
|
|
16028
|
+
workflowApp.delete("/schedules/:id", createApiKeyAuth(), async (c) => {
|
|
16029
|
+
const user = c.get("user");
|
|
16030
|
+
const deleted = await deleteWorkflowSchedule(c.req.param("id"), user.id);
|
|
16031
|
+
if (!deleted) return c.json({ error: "Schedule not found" }, 404);
|
|
16032
|
+
return c.json({ ok: true });
|
|
16033
|
+
});
|
|
16034
|
+
workflowApp.post("/schedules/:id/run", createApiKeyAuth(), async (c) => {
|
|
16035
|
+
const user = c.get("user");
|
|
16036
|
+
const schedule = await getWorkflowSchedule(c.req.param("id"), user.id);
|
|
16037
|
+
if (!schedule) return c.json({ error: "Schedule not found" }, 404);
|
|
16038
|
+
const run = await createWorkflowRun({ userId: user.id, scheduleId: schedule.id, workflowId: schedule.workflow_id, workflowInput: schedule.input });
|
|
16039
|
+
try {
|
|
16040
|
+
const executed = await executeWorkflowRun({
|
|
16041
|
+
runId: run.id,
|
|
16042
|
+
user,
|
|
16043
|
+
workflowId: schedule.workflow_id,
|
|
16044
|
+
workflowInput: schedule.input,
|
|
16045
|
+
apiUrl: originFromUrl(c.req.url),
|
|
16046
|
+
scheduleId: schedule.id,
|
|
16047
|
+
webhookUrl: schedule.webhook_url
|
|
16048
|
+
});
|
|
16049
|
+
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
16050
|
+
} catch (err) {
|
|
16051
|
+
const failed = await getWorkflowRun(run.id, user.id);
|
|
16052
|
+
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
16053
|
+
}
|
|
16054
|
+
});
|
|
16055
|
+
workflowApp.post("/cron/dispatch", async (c) => {
|
|
16056
|
+
const secret2 = c.req.header("authorization");
|
|
16057
|
+
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
16058
|
+
return c.json({ error: "Unauthorized" }, 401);
|
|
16059
|
+
}
|
|
16060
|
+
return c.json(await dispatchDueWorkflowSchedules(originFromUrl(c.req.url)));
|
|
16061
|
+
});
|
|
16062
|
+
}
|
|
16063
|
+
});
|
|
16064
|
+
|
|
14501
16065
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
14502
16066
|
function sha256(value) {
|
|
14503
16067
|
if (!value) return null;
|
|
14504
|
-
return (0,
|
|
16068
|
+
return (0, import_node_crypto3.createHash)("sha256").update(value).digest("hex");
|
|
14505
16069
|
}
|
|
14506
16070
|
function countWords(markdown) {
|
|
14507
16071
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -14801,11 +16365,11 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
14801
16365
|
}
|
|
14802
16366
|
};
|
|
14803
16367
|
}
|
|
14804
|
-
var
|
|
16368
|
+
var import_node_crypto3, import_p_limit3, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
|
|
14805
16369
|
var init_page_snapshot_extractor = __esm({
|
|
14806
16370
|
"src/serp-intelligence/page-snapshot-extractor.ts"() {
|
|
14807
16371
|
"use strict";
|
|
14808
|
-
|
|
16372
|
+
import_node_crypto3 = require("crypto");
|
|
14809
16373
|
import_p_limit3 = __toESM(require("p-limit"), 1);
|
|
14810
16374
|
init_kpo_extractor();
|
|
14811
16375
|
init_url_utils();
|
|
@@ -15510,7 +17074,7 @@ var init_PAAExtractor = __esm({
|
|
|
15510
17074
|
const titleText = texts.find((t) => !/^\d+:\d+$/.test(t) && t.length > 5) ?? "";
|
|
15511
17075
|
if (!titleText) continue;
|
|
15512
17076
|
let title = titleText;
|
|
15513
|
-
let
|
|
17077
|
+
let platform2 = "";
|
|
15514
17078
|
let channel = "";
|
|
15515
17079
|
for (const p of sels.platforms) {
|
|
15516
17080
|
let lastIdx = -1;
|
|
@@ -15526,13 +17090,13 @@ var init_PAAExtractor = __esm({
|
|
|
15526
17090
|
const isSourceTag = /^[\s·]/.test(after) || after.trim() === "";
|
|
15527
17091
|
if (!isSourceTag) continue;
|
|
15528
17092
|
title = titleText.slice(0, lastIdx).trim();
|
|
15529
|
-
|
|
17093
|
+
platform2 = p;
|
|
15530
17094
|
const stripped = after.replace(/^[\s·]+/, "");
|
|
15531
17095
|
const dotIdx = stripped.indexOf("\xB7");
|
|
15532
17096
|
channel = (dotIdx === -1 ? stripped : stripped.slice(0, dotIdx)).trim();
|
|
15533
17097
|
break;
|
|
15534
17098
|
}
|
|
15535
|
-
if (title) results.push({ title, channel, platform, duration, url: href });
|
|
17099
|
+
if (title) results.push({ title, channel, platform: platform2, duration, url: href });
|
|
15536
17100
|
}
|
|
15537
17101
|
return results;
|
|
15538
17102
|
}, svSels);
|
|
@@ -15576,10 +17140,10 @@ var init_PAAExtractor = __esm({
|
|
|
15576
17140
|
const engagement = engagementParts[0] ?? "";
|
|
15577
17141
|
const dateCandidates = allSpans.map((s2) => s2.textContent?.trim() ?? "").filter((t) => /\d+ (day|week|month|year|hour)s? ago|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec/.test(t));
|
|
15578
17142
|
const date = ytDate || (dateCandidates[0] ?? "");
|
|
15579
|
-
const
|
|
17143
|
+
const platform2 = platformText || (ytChannel ? "YouTube" : "");
|
|
15580
17144
|
const source = ytChannel || sourceText;
|
|
15581
17145
|
let type = "unknown";
|
|
15582
|
-
const pl =
|
|
17146
|
+
const pl = platform2.toLowerCase();
|
|
15583
17147
|
const src = source.toLowerCase();
|
|
15584
17148
|
const srcRaw = sourceText.toLowerCase();
|
|
15585
17149
|
if (pl.includes("reddit") || src.startsWith("r/")) type = "reddit";
|
|
@@ -15588,7 +17152,7 @@ var init_PAAExtractor = __esm({
|
|
|
15588
17152
|
else if (pl.includes("tiktok") || srcRaw.includes("tiktok")) type = "tiktok";
|
|
15589
17153
|
else if (pl.includes("youtube") || !!ytChannel) type = "youtube";
|
|
15590
17154
|
else type = "news";
|
|
15591
|
-
return { type, title, url, source, platform, popularComment, engagement, date, duration, authorNote };
|
|
17155
|
+
return { type, title, url, source, platform: platform2, popularComment, engagement, date, duration, authorNote };
|
|
15592
17156
|
});
|
|
15593
17157
|
}, sels);
|
|
15594
17158
|
}
|
|
@@ -16050,52 +17614,52 @@ var init_PAAExtractor = __esm({
|
|
|
16050
17614
|
});
|
|
16051
17615
|
|
|
16052
17616
|
// src/output/OutputSerializer.ts
|
|
16053
|
-
var
|
|
17617
|
+
var import_node_fs5, import_node_path8, import_papaparse2, OutputSerializer;
|
|
16054
17618
|
var init_OutputSerializer = __esm({
|
|
16055
17619
|
"src/output/OutputSerializer.ts"() {
|
|
16056
17620
|
"use strict";
|
|
16057
|
-
|
|
16058
|
-
|
|
17621
|
+
import_node_fs5 = require("fs");
|
|
17622
|
+
import_node_path8 = __toESM(require("path"), 1);
|
|
16059
17623
|
import_papaparse2 = __toESM(require("papaparse"), 1);
|
|
16060
17624
|
OutputSerializer = class {
|
|
16061
17625
|
async writeJSON(result, outputDir) {
|
|
16062
|
-
await
|
|
17626
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
16063
17627
|
const slug = result.seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
16064
17628
|
const filename = `${slug}-${Date.now()}.json`;
|
|
16065
|
-
const fullPath =
|
|
16066
|
-
await
|
|
17629
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17630
|
+
await import_node_fs5.promises.writeFile(fullPath, JSON.stringify(result, null, 2), "utf8");
|
|
16067
17631
|
return fullPath;
|
|
16068
17632
|
}
|
|
16069
17633
|
async writeCSV(rows, outputDir) {
|
|
16070
|
-
await
|
|
17634
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
16071
17635
|
const seedRaw = rows[0]?.seed_query ?? "paa";
|
|
16072
17636
|
const slug = seedRaw.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
16073
17637
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
16074
17638
|
const filename = `${slug}-${Date.now()}.csv`;
|
|
16075
|
-
const fullPath =
|
|
16076
|
-
await
|
|
17639
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17640
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
16077
17641
|
return fullPath;
|
|
16078
17642
|
}
|
|
16079
17643
|
async writeVideoCSV(videos, seed, outputDir) {
|
|
16080
|
-
await
|
|
17644
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
16081
17645
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
16082
17646
|
const csv = import_papaparse2.default.unparse(videos, { header: true });
|
|
16083
17647
|
const filename = `${slug}-videos-${Date.now()}.csv`;
|
|
16084
|
-
const fullPath =
|
|
16085
|
-
await
|
|
17648
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17649
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
16086
17650
|
return fullPath;
|
|
16087
17651
|
}
|
|
16088
17652
|
async writeForumCSV(forums, seed, outputDir) {
|
|
16089
|
-
await
|
|
17653
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
16090
17654
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
16091
17655
|
const csv = import_papaparse2.default.unparse(forums, { header: true });
|
|
16092
17656
|
const filename = `${slug}-forums-${Date.now()}.csv`;
|
|
16093
|
-
const fullPath =
|
|
16094
|
-
await
|
|
17657
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17658
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
16095
17659
|
return fullPath;
|
|
16096
17660
|
}
|
|
16097
17661
|
async writeAIOverviewCSV(citations, text, seed, outputDir) {
|
|
16098
|
-
await
|
|
17662
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
16099
17663
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
16100
17664
|
const rows = citations.map((c, i) => ({
|
|
16101
17665
|
seed_query: seed,
|
|
@@ -16105,12 +17669,12 @@ var init_OutputSerializer = __esm({
|
|
|
16105
17669
|
}));
|
|
16106
17670
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
16107
17671
|
const filename = `${slug}-ai-overview-${Date.now()}.csv`;
|
|
16108
|
-
const fullPath =
|
|
16109
|
-
await
|
|
17672
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17673
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
16110
17674
|
return fullPath;
|
|
16111
17675
|
}
|
|
16112
17676
|
async writeAIModeCSV(citations, text, seed, outputDir) {
|
|
16113
|
-
await
|
|
17677
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
16114
17678
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
16115
17679
|
const rows = citations.map((c, i) => ({
|
|
16116
17680
|
seed_query: seed,
|
|
@@ -16120,18 +17684,18 @@ var init_OutputSerializer = __esm({
|
|
|
16120
17684
|
}));
|
|
16121
17685
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
16122
17686
|
const filename = `${slug}-ai-mode-${Date.now()}.csv`;
|
|
16123
|
-
const fullPath =
|
|
16124
|
-
await
|
|
17687
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17688
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
16125
17689
|
return fullPath;
|
|
16126
17690
|
}
|
|
16127
17691
|
async writeWhatPeopleSayingCSV(cards, seed, outputDir) {
|
|
16128
|
-
await
|
|
17692
|
+
await import_node_fs5.promises.mkdir(outputDir, { recursive: true });
|
|
16129
17693
|
const slug = seed.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
|
|
16130
17694
|
const rows = cards.map((c) => ({ seed_query: seed, ...c }));
|
|
16131
17695
|
const csv = import_papaparse2.default.unparse(rows, { header: true });
|
|
16132
17696
|
const filename = `${slug}-what-people-saying-${Date.now()}.csv`;
|
|
16133
|
-
const fullPath =
|
|
16134
|
-
await
|
|
17697
|
+
const fullPath = import_node_path8.default.join(outputDir, filename);
|
|
17698
|
+
await import_node_fs5.promises.writeFile(fullPath, csv, "utf8");
|
|
16135
17699
|
return fullPath;
|
|
16136
17700
|
}
|
|
16137
17701
|
};
|
|
@@ -16682,11 +18246,11 @@ function isPublicHttpUrl(value) {
|
|
|
16682
18246
|
return false;
|
|
16683
18247
|
}
|
|
16684
18248
|
}
|
|
16685
|
-
var
|
|
18249
|
+
var import_zod24, SerpIntelligenceDeviceValues, SerpIntelligenceProxyModeValues, SerpIntelligenceAttemptOutcomeValues, SerpIntelligenceLocalizationStatusValues, SerpPageSnapshotSourceKindValues, SerpPageFetchStatusValues, SerpPageFetchedViaValues, HostnameSuffixPattern, Ipv4Pattern, SerpIntelligencePublicHttpUrlSchema, SerpIntelligenceCaptureBodySchema, SerpIntelligencePageSnapshotRequestSchema, SerpIntelligencePageSnapshotsBodySchema, SerpIntelligenceAICitationSchema, SerpIntelligenceOrganicResultSchema, SerpIntelligenceLocationEvidenceSchema, SerpIntelligenceHarvestResultSchema, SerpIntelligenceCaptureAttemptSchema, SerpPageSnapshotCaptureSchema, SerpIntelligenceCaptureResponseSchema, SerpIntelligencePageSnapshotsResponseSchema;
|
|
16686
18250
|
var init_schemas4 = __esm({
|
|
16687
18251
|
"src/serp-intelligence/schemas.ts"() {
|
|
16688
18252
|
"use strict";
|
|
16689
|
-
|
|
18253
|
+
import_zod24 = require("zod");
|
|
16690
18254
|
SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
16691
18255
|
SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
16692
18256
|
SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -16720,171 +18284,171 @@ var init_schemas4 = __esm({
|
|
|
16720
18284
|
SerpPageFetchedViaValues = ["fetch", "headless", "browser", "mcp"];
|
|
16721
18285
|
HostnameSuffixPattern = /(^|\.)localhost$/i;
|
|
16722
18286
|
Ipv4Pattern = /^\d{1,3}(?:\.\d{1,3}){3}$/;
|
|
16723
|
-
SerpIntelligencePublicHttpUrlSchema =
|
|
16724
|
-
SerpIntelligenceCaptureBodySchema =
|
|
16725
|
-
query:
|
|
16726
|
-
location:
|
|
16727
|
-
gl:
|
|
16728
|
-
hl:
|
|
16729
|
-
device:
|
|
16730
|
-
proxyMode:
|
|
16731
|
-
proxyZip:
|
|
16732
|
-
pages:
|
|
16733
|
-
debug:
|
|
16734
|
-
includePageSnapshots:
|
|
16735
|
-
pageSnapshotLimit:
|
|
18287
|
+
SerpIntelligencePublicHttpUrlSchema = import_zod24.z.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
18288
|
+
SerpIntelligenceCaptureBodySchema = import_zod24.z.object({
|
|
18289
|
+
query: import_zod24.z.string().trim().min(1, "query is required"),
|
|
18290
|
+
location: import_zod24.z.string().trim().min(1).optional(),
|
|
18291
|
+
gl: import_zod24.z.string().trim().length(2).default("us"),
|
|
18292
|
+
hl: import_zod24.z.string().trim().length(2).default("en"),
|
|
18293
|
+
device: import_zod24.z.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
18294
|
+
proxyMode: import_zod24.z.enum(SerpIntelligenceProxyModeValues).default("location"),
|
|
18295
|
+
proxyZip: import_zod24.z.string().regex(/^\d{5}$/).optional(),
|
|
18296
|
+
pages: import_zod24.z.number().int().min(1).max(2).default(1),
|
|
18297
|
+
debug: import_zod24.z.boolean().default(false),
|
|
18298
|
+
includePageSnapshots: import_zod24.z.boolean().default(false),
|
|
18299
|
+
pageSnapshotLimit: import_zod24.z.number().int().min(0).max(10).default(0)
|
|
16736
18300
|
}).strict();
|
|
16737
|
-
SerpIntelligencePageSnapshotRequestSchema =
|
|
18301
|
+
SerpIntelligencePageSnapshotRequestSchema = import_zod24.z.object({
|
|
16738
18302
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
16739
|
-
sourceKind:
|
|
16740
|
-
sourcePosition:
|
|
18303
|
+
sourceKind: import_zod24.z.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
18304
|
+
sourcePosition: import_zod24.z.number().int().min(1).optional()
|
|
16741
18305
|
}).strict();
|
|
16742
|
-
SerpIntelligencePageSnapshotsBodySchema =
|
|
16743
|
-
urls:
|
|
16744
|
-
targets:
|
|
16745
|
-
maxConcurrency:
|
|
16746
|
-
timeoutMs:
|
|
16747
|
-
debug:
|
|
18306
|
+
SerpIntelligencePageSnapshotsBodySchema = import_zod24.z.object({
|
|
18307
|
+
urls: import_zod24.z.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
18308
|
+
targets: import_zod24.z.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
18309
|
+
maxConcurrency: import_zod24.z.number().int().min(1).max(5).default(2),
|
|
18310
|
+
timeoutMs: import_zod24.z.number().int().min(1e3).max(6e4).default(15e3),
|
|
18311
|
+
debug: import_zod24.z.boolean().default(false)
|
|
16748
18312
|
}).strict();
|
|
16749
|
-
SerpIntelligenceAICitationSchema =
|
|
16750
|
-
text:
|
|
16751
|
-
href:
|
|
18313
|
+
SerpIntelligenceAICitationSchema = import_zod24.z.object({
|
|
18314
|
+
text: import_zod24.z.string(),
|
|
18315
|
+
href: import_zod24.z.string()
|
|
16752
18316
|
}).strict();
|
|
16753
|
-
SerpIntelligenceOrganicResultSchema =
|
|
16754
|
-
position:
|
|
16755
|
-
title:
|
|
16756
|
-
url:
|
|
16757
|
-
domain:
|
|
16758
|
-
cite:
|
|
16759
|
-
snippet:
|
|
16760
|
-
isRedditStyle:
|
|
16761
|
-
inlineRating:
|
|
16762
|
-
value:
|
|
16763
|
-
count:
|
|
18317
|
+
SerpIntelligenceOrganicResultSchema = import_zod24.z.object({
|
|
18318
|
+
position: import_zod24.z.number().int().min(1),
|
|
18319
|
+
title: import_zod24.z.string(),
|
|
18320
|
+
url: import_zod24.z.string(),
|
|
18321
|
+
domain: import_zod24.z.string(),
|
|
18322
|
+
cite: import_zod24.z.string().nullable(),
|
|
18323
|
+
snippet: import_zod24.z.string().nullable(),
|
|
18324
|
+
isRedditStyle: import_zod24.z.boolean(),
|
|
18325
|
+
inlineRating: import_zod24.z.object({
|
|
18326
|
+
value: import_zod24.z.string(),
|
|
18327
|
+
count: import_zod24.z.string()
|
|
16764
18328
|
}).strict().nullable()
|
|
16765
18329
|
}).strict();
|
|
16766
|
-
SerpIntelligenceLocationEvidenceSchema =
|
|
16767
|
-
status:
|
|
16768
|
-
expected:
|
|
16769
|
-
city:
|
|
16770
|
-
regionCode:
|
|
16771
|
-
canonicalLocation:
|
|
18330
|
+
SerpIntelligenceLocationEvidenceSchema = import_zod24.z.object({
|
|
18331
|
+
status: import_zod24.z.enum(SerpIntelligenceLocalizationStatusValues),
|
|
18332
|
+
expected: import_zod24.z.object({
|
|
18333
|
+
city: import_zod24.z.string(),
|
|
18334
|
+
regionCode: import_zod24.z.string().nullable(),
|
|
18335
|
+
canonicalLocation: import_zod24.z.string()
|
|
16772
18336
|
}).strict().nullable(),
|
|
16773
|
-
candidates:
|
|
16774
|
-
city:
|
|
16775
|
-
regionCode:
|
|
16776
|
-
count:
|
|
16777
|
-
examples:
|
|
18337
|
+
candidates: import_zod24.z.array(import_zod24.z.object({
|
|
18338
|
+
city: import_zod24.z.string(),
|
|
18339
|
+
regionCode: import_zod24.z.string(),
|
|
18340
|
+
count: import_zod24.z.number().int().min(0),
|
|
18341
|
+
examples: import_zod24.z.array(import_zod24.z.string())
|
|
16778
18342
|
}).strict())
|
|
16779
18343
|
}).strict();
|
|
16780
|
-
SerpIntelligenceHarvestResultSchema =
|
|
16781
|
-
seed:
|
|
16782
|
-
location:
|
|
16783
|
-
extractedAt:
|
|
16784
|
-
totalQuestions:
|
|
16785
|
-
surface:
|
|
16786
|
-
aiOverview:
|
|
16787
|
-
detected:
|
|
16788
|
-
text:
|
|
16789
|
-
citations:
|
|
16790
|
-
expanded:
|
|
16791
|
-
fullyExpanded:
|
|
16792
|
-
sections:
|
|
18344
|
+
SerpIntelligenceHarvestResultSchema = import_zod24.z.object({
|
|
18345
|
+
seed: import_zod24.z.string(),
|
|
18346
|
+
location: import_zod24.z.string().nullable(),
|
|
18347
|
+
extractedAt: import_zod24.z.string(),
|
|
18348
|
+
totalQuestions: import_zod24.z.number().int().min(0),
|
|
18349
|
+
surface: import_zod24.z.enum(["web", "aim", "unknown"]),
|
|
18350
|
+
aiOverview: import_zod24.z.object({
|
|
18351
|
+
detected: import_zod24.z.boolean(),
|
|
18352
|
+
text: import_zod24.z.string().nullable(),
|
|
18353
|
+
citations: import_zod24.z.array(SerpIntelligenceAICitationSchema),
|
|
18354
|
+
expanded: import_zod24.z.boolean().optional(),
|
|
18355
|
+
fullyExpanded: import_zod24.z.boolean().optional(),
|
|
18356
|
+
sections: import_zod24.z.array(import_zod24.z.string()).optional()
|
|
16793
18357
|
}).strict(),
|
|
16794
|
-
aiMode:
|
|
16795
|
-
detected:
|
|
16796
|
-
text:
|
|
16797
|
-
citations:
|
|
18358
|
+
aiMode: import_zod24.z.object({
|
|
18359
|
+
detected: import_zod24.z.boolean(),
|
|
18360
|
+
text: import_zod24.z.string().nullable(),
|
|
18361
|
+
citations: import_zod24.z.array(SerpIntelligenceAICitationSchema)
|
|
16798
18362
|
}).strict(),
|
|
16799
|
-
tree:
|
|
16800
|
-
flat:
|
|
16801
|
-
videos:
|
|
16802
|
-
forums:
|
|
16803
|
-
organicResults:
|
|
16804
|
-
localPack:
|
|
16805
|
-
entityIds:
|
|
16806
|
-
entities:
|
|
16807
|
-
name:
|
|
16808
|
-
kgId:
|
|
16809
|
-
cid:
|
|
16810
|
-
gcid:
|
|
18363
|
+
tree: import_zod24.z.array(import_zod24.z.unknown()),
|
|
18364
|
+
flat: import_zod24.z.array(import_zod24.z.unknown()),
|
|
18365
|
+
videos: import_zod24.z.array(import_zod24.z.unknown()),
|
|
18366
|
+
forums: import_zod24.z.array(import_zod24.z.unknown()),
|
|
18367
|
+
organicResults: import_zod24.z.array(SerpIntelligenceOrganicResultSchema),
|
|
18368
|
+
localPack: import_zod24.z.array(import_zod24.z.unknown()),
|
|
18369
|
+
entityIds: import_zod24.z.object({
|
|
18370
|
+
entities: import_zod24.z.array(import_zod24.z.object({
|
|
18371
|
+
name: import_zod24.z.string(),
|
|
18372
|
+
kgId: import_zod24.z.string().nullable(),
|
|
18373
|
+
cid: import_zod24.z.string().nullable(),
|
|
18374
|
+
gcid: import_zod24.z.string().nullable()
|
|
16811
18375
|
}).strict()),
|
|
16812
|
-
kgIds:
|
|
16813
|
-
cids:
|
|
16814
|
-
gcids:
|
|
18376
|
+
kgIds: import_zod24.z.array(import_zod24.z.string()),
|
|
18377
|
+
cids: import_zod24.z.array(import_zod24.z.string()),
|
|
18378
|
+
gcids: import_zod24.z.array(import_zod24.z.string())
|
|
16815
18379
|
}).strict(),
|
|
16816
|
-
stats:
|
|
16817
|
-
seed:
|
|
16818
|
-
totalQuestions:
|
|
16819
|
-
maxDepthReached:
|
|
16820
|
-
durationMs:
|
|
16821
|
-
errorCount:
|
|
18380
|
+
stats: import_zod24.z.object({
|
|
18381
|
+
seed: import_zod24.z.string(),
|
|
18382
|
+
totalQuestions: import_zod24.z.number().int().min(0),
|
|
18383
|
+
maxDepthReached: import_zod24.z.number().int().min(0),
|
|
18384
|
+
durationMs: import_zod24.z.number().min(0),
|
|
18385
|
+
errorCount: import_zod24.z.number().int().min(0)
|
|
16822
18386
|
}).strict(),
|
|
16823
|
-
diagnostics:
|
|
16824
|
-
completionStatus:
|
|
16825
|
-
problem:
|
|
16826
|
-
warnings:
|
|
16827
|
-
debug:
|
|
18387
|
+
diagnostics: import_zod24.z.object({
|
|
18388
|
+
completionStatus: import_zod24.z.enum(["paa_found", "no_paa", "serp_only"]),
|
|
18389
|
+
problem: import_zod24.z.null(),
|
|
18390
|
+
warnings: import_zod24.z.array(import_zod24.z.unknown()).optional(),
|
|
18391
|
+
debug: import_zod24.z.object({
|
|
16828
18392
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
16829
18393
|
}).passthrough().optional()
|
|
16830
18394
|
}).passthrough(),
|
|
16831
|
-
whatPeopleSaying:
|
|
18395
|
+
whatPeopleSaying: import_zod24.z.array(import_zod24.z.unknown())
|
|
16832
18396
|
}).strict();
|
|
16833
|
-
SerpIntelligenceCaptureAttemptSchema =
|
|
16834
|
-
attemptNumber:
|
|
16835
|
-
outcome:
|
|
16836
|
-
startedAt:
|
|
16837
|
-
completedAt:
|
|
16838
|
-
durationMs:
|
|
16839
|
-
problemCode:
|
|
16840
|
-
message:
|
|
16841
|
-
kernelSessionId:
|
|
16842
|
-
cleanupSucceeded:
|
|
18397
|
+
SerpIntelligenceCaptureAttemptSchema = import_zod24.z.object({
|
|
18398
|
+
attemptNumber: import_zod24.z.number().int().min(1),
|
|
18399
|
+
outcome: import_zod24.z.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
18400
|
+
startedAt: import_zod24.z.string().optional(),
|
|
18401
|
+
completedAt: import_zod24.z.string().optional(),
|
|
18402
|
+
durationMs: import_zod24.z.number().min(0).optional(),
|
|
18403
|
+
problemCode: import_zod24.z.string().optional(),
|
|
18404
|
+
message: import_zod24.z.string().optional(),
|
|
18405
|
+
kernelSessionId: import_zod24.z.string().nullable().optional(),
|
|
18406
|
+
cleanupSucceeded: import_zod24.z.boolean().nullable().optional()
|
|
16843
18407
|
}).strict();
|
|
16844
|
-
SerpPageSnapshotCaptureSchema =
|
|
18408
|
+
SerpPageSnapshotCaptureSchema = import_zod24.z.object({
|
|
16845
18409
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
16846
18410
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
16847
18411
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
16848
|
-
sourceKind:
|
|
16849
|
-
sourcePosition:
|
|
16850
|
-
status:
|
|
16851
|
-
fetchedVia:
|
|
16852
|
-
httpStatus:
|
|
16853
|
-
contentType:
|
|
16854
|
-
title:
|
|
18412
|
+
sourceKind: import_zod24.z.enum(SerpPageSnapshotSourceKindValues),
|
|
18413
|
+
sourcePosition: import_zod24.z.number().int().min(1).nullable(),
|
|
18414
|
+
status: import_zod24.z.enum(SerpPageFetchStatusValues),
|
|
18415
|
+
fetchedVia: import_zod24.z.enum(SerpPageFetchedViaValues).nullable(),
|
|
18416
|
+
httpStatus: import_zod24.z.number().int().min(100).max(599).nullable(),
|
|
18417
|
+
contentType: import_zod24.z.string().nullable(),
|
|
18418
|
+
title: import_zod24.z.string().nullable(),
|
|
16855
18419
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
16856
|
-
metaDescription:
|
|
16857
|
-
headings:
|
|
16858
|
-
level:
|
|
16859
|
-
text:
|
|
18420
|
+
metaDescription: import_zod24.z.string().nullable(),
|
|
18421
|
+
headings: import_zod24.z.array(import_zod24.z.object({
|
|
18422
|
+
level: import_zod24.z.number().int().min(1).max(6),
|
|
18423
|
+
text: import_zod24.z.string()
|
|
16860
18424
|
}).strict()).default([]),
|
|
16861
|
-
artifact:
|
|
16862
|
-
htmlBlobUrl:
|
|
16863
|
-
textBlobUrl:
|
|
16864
|
-
markdownBlobUrl:
|
|
16865
|
-
screenshotBlobUrl:
|
|
16866
|
-
contentSha256:
|
|
16867
|
-
capturedAt:
|
|
18425
|
+
artifact: import_zod24.z.object({
|
|
18426
|
+
htmlBlobUrl: import_zod24.z.string().url().nullable(),
|
|
18427
|
+
textBlobUrl: import_zod24.z.string().url().nullable(),
|
|
18428
|
+
markdownBlobUrl: import_zod24.z.string().url().nullable(),
|
|
18429
|
+
screenshotBlobUrl: import_zod24.z.string().url().nullable(),
|
|
18430
|
+
contentSha256: import_zod24.z.string().nullable(),
|
|
18431
|
+
capturedAt: import_zod24.z.string().nullable()
|
|
16868
18432
|
}).strict(),
|
|
16869
|
-
error:
|
|
16870
|
-
code:
|
|
16871
|
-
message:
|
|
18433
|
+
error: import_zod24.z.object({
|
|
18434
|
+
code: import_zod24.z.string(),
|
|
18435
|
+
message: import_zod24.z.string()
|
|
16872
18436
|
}).strict().nullable()
|
|
16873
18437
|
}).strict();
|
|
16874
|
-
SerpIntelligenceCaptureResponseSchema =
|
|
18438
|
+
SerpIntelligenceCaptureResponseSchema = import_zod24.z.object({
|
|
16875
18439
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
16876
|
-
attempts:
|
|
18440
|
+
attempts: import_zod24.z.array(SerpIntelligenceCaptureAttemptSchema),
|
|
16877
18441
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
16878
|
-
pageSnapshotArtifacts:
|
|
16879
|
-
billing:
|
|
16880
|
-
creditsUsed:
|
|
16881
|
-
requestId:
|
|
16882
|
-
jobId:
|
|
18442
|
+
pageSnapshotArtifacts: import_zod24.z.array(SerpPageSnapshotCaptureSchema),
|
|
18443
|
+
billing: import_zod24.z.object({
|
|
18444
|
+
creditsUsed: import_zod24.z.number().min(0).optional(),
|
|
18445
|
+
requestId: import_zod24.z.string().optional(),
|
|
18446
|
+
jobId: import_zod24.z.string().optional()
|
|
16883
18447
|
}).strict().optional()
|
|
16884
18448
|
}).strict();
|
|
16885
|
-
SerpIntelligencePageSnapshotsResponseSchema =
|
|
16886
|
-
pageSnapshotArtifacts:
|
|
16887
|
-
attempts:
|
|
18449
|
+
SerpIntelligencePageSnapshotsResponseSchema = import_zod24.z.object({
|
|
18450
|
+
pageSnapshotArtifacts: import_zod24.z.array(SerpPageSnapshotCaptureSchema),
|
|
18451
|
+
attempts: import_zod24.z.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
16888
18452
|
}).strict();
|
|
16889
18453
|
}
|
|
16890
18454
|
});
|
|
@@ -17128,11 +18692,11 @@ function pageSnapshotTargetsFromBody(body) {
|
|
|
17128
18692
|
sourcePosition: null
|
|
17129
18693
|
}));
|
|
17130
18694
|
}
|
|
17131
|
-
var
|
|
18695
|
+
var import_hono8, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, serpIntelligenceApp;
|
|
17132
18696
|
var init_serp_intelligence_routes = __esm({
|
|
17133
18697
|
"src/api/serp-intelligence-routes.ts"() {
|
|
17134
18698
|
"use strict";
|
|
17135
|
-
|
|
18699
|
+
import_hono8 = require("hono");
|
|
17136
18700
|
init_browser_service_env();
|
|
17137
18701
|
init_page_snapshot_extractor();
|
|
17138
18702
|
init_serp_capture_service();
|
|
@@ -17144,7 +18708,7 @@ var init_serp_intelligence_routes = __esm({
|
|
|
17144
18708
|
SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
17145
18709
|
POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
17146
18710
|
POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
17147
|
-
serpIntelligenceApp = new
|
|
18711
|
+
serpIntelligenceApp = new import_hono8.Hono();
|
|
17148
18712
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
17149
18713
|
serpIntelligenceApp.post("/capture", async (c) => {
|
|
17150
18714
|
void POST_CAPTURE_ROUTE_LABEL;
|
|
@@ -17262,133 +18826,133 @@ var PACKAGE_VERSION;
|
|
|
17262
18826
|
var init_version = __esm({
|
|
17263
18827
|
"src/version.ts"() {
|
|
17264
18828
|
"use strict";
|
|
17265
|
-
PACKAGE_VERSION = "0.2.
|
|
18829
|
+
PACKAGE_VERSION = "0.2.10";
|
|
17266
18830
|
}
|
|
17267
18831
|
});
|
|
17268
18832
|
|
|
17269
18833
|
// src/mcp/mcp-tool-schemas.ts
|
|
17270
|
-
var
|
|
18834
|
+
var import_zod25, HarvestPaaInputSchema, ExtractUrlInputSchema, MapSiteUrlsInputSchema, ExtractSiteInputSchema, YoutubeHarvestInputSchema, YoutubeTranscribeInputSchema, FacebookPageIntelInputSchema, FacebookAdSearchInputSchema, FacebookAdTranscribeInputSchema, MapsPlaceIntelInputSchema, MapsSearchInputSchema, DirectoryWorkflowInputSchema, RankTrackerModeSchema, RankTrackerBlueprintInputSchema, NullableString, MapsSearchAttemptOutput, MapsSearchOutputSchema, DirectoryMapsBusinessOutput, DirectoryWorkflowOutputSchema, RankTrackerToolPlanOutput, RankTrackerTableOutput, RankTrackerCronJobOutput, RankTrackerBlueprintOutputSchema, OrganicResultOutput, AiOverviewOutput, EntityIdsOutput, HarvestPaaOutputSchema, SearchSerpOutputSchema, ExtractUrlOutputSchema, ExtractSiteOutputSchema, MapsPlaceIntelOutputSchema, CreditsInfoOutputSchema, MapSiteUrlsOutputSchema, YoutubeHarvestOutputSchema, FacebookAdSearchOutputSchema, FacebookPageIntelOutputSchema, CreditsInfoInputSchema, SearchSerpInputSchema, CaptureSerpSnapshotInputSchema, ScreenshotInputSchema, CaptureSerpPageSnapshotsInputSchema;
|
|
17271
18835
|
var init_mcp_tool_schemas = __esm({
|
|
17272
18836
|
"src/mcp/mcp-tool-schemas.ts"() {
|
|
17273
18837
|
"use strict";
|
|
17274
|
-
|
|
18838
|
+
import_zod25 = require("zod");
|
|
17275
18839
|
HarvestPaaInputSchema = {
|
|
17276
|
-
query:
|
|
17277
|
-
location:
|
|
17278
|
-
maxQuestions:
|
|
17279
|
-
gl:
|
|
17280
|
-
hl:
|
|
17281
|
-
device:
|
|
17282
|
-
proxyMode:
|
|
17283
|
-
proxyZip:
|
|
17284
|
-
debug:
|
|
18840
|
+
query: import_zod25.z.string().min(1).describe('Core search topic only. If the user says "best hvac company in Denver CO", use query="best hvac company" and location="Denver, CO". Do not include the location in query when it can be separated.'),
|
|
18841
|
+
location: import_zod25.z.string().optional().describe('City, region, or country for geo-targeted results, inferred from the user request when present, e.g. "Denver, CO", "Tokyo, Japan", "London, UK".'),
|
|
18842
|
+
maxQuestions: import_zod25.z.number().int().min(1).max(200).default(30).describe("Number of PAA questions to extract. Default 30. Maximum 200. Use 10 for quick probes, 30 for normal research, 100-200 when the user asks for everything/full/deep research. Larger harvests get a longer server time budget (151-200 questions \u2192 up to 280s). Credits are charged by extracted question; unused request hold is refunded."),
|
|
18843
|
+
gl: import_zod25.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
|
|
18844
|
+
hl: import_zod25.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
|
|
18845
|
+
device: import_zod25.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
18846
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
|
|
18847
|
+
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
18848
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
|
|
17285
18849
|
};
|
|
17286
18850
|
ExtractUrlInputSchema = {
|
|
17287
|
-
url:
|
|
17288
|
-
screenshot:
|
|
17289
|
-
screenshotDevice:
|
|
17290
|
-
extractBranding:
|
|
17291
|
-
downloadMedia:
|
|
17292
|
-
mediaTypes:
|
|
17293
|
-
allowLocal:
|
|
18851
|
+
url: import_zod25.z.string().url().describe("Public http/https URL to extract. Use this when the user provides one specific page URL."),
|
|
18852
|
+
screenshot: import_zod25.z.boolean().default(false).describe("Also capture a full-page screenshot of the URL. Saved to ~/Downloads/mcp-scraper/screenshots/ and returned inline. Use when the user asks to see or capture the page visually."),
|
|
18853
|
+
screenshotDevice: import_zod25.z.enum(["desktop", "mobile"]).default("desktop").describe("Viewport for screenshot. desktop = 1440\xD7900. mobile = 390\xD7844. Default desktop."),
|
|
18854
|
+
extractBranding: import_zod25.z.boolean().default(false).describe("Extract brand colors, fonts, logo, and favicon using a rendered browser session. Returns colorScheme (light/dark), colors (primary/accent/background/text/heading as hex), fonts (heading/body family names), and assets (logo URL, favicon URL). Use when the user asks about brand colors, site theme, or brand assets."),
|
|
18855
|
+
downloadMedia: import_zod25.z.boolean().default(false).describe("Extract and download all page media (images, video, audio) to ~/Downloads/mcp-scraper/media/. Ad networks, tracking pixels, and noise URLs are filtered automatically. Use when the user asks to download or harvest assets from a page."),
|
|
18856
|
+
mediaTypes: import_zod25.z.array(import_zod25.z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download. Default all three."),
|
|
18857
|
+
allowLocal: import_zod25.z.boolean().default(false).describe("Allow localhost and private-network URLs. For local development only.")
|
|
17294
18858
|
};
|
|
17295
18859
|
MapSiteUrlsInputSchema = {
|
|
17296
|
-
url:
|
|
17297
|
-
maxUrls:
|
|
18860
|
+
url: import_zod25.z.string().url().describe("Public website URL or domain to crawl for internal URLs. Use before extract_site when the user asks to audit/map/crawl a site."),
|
|
18861
|
+
maxUrls: import_zod25.z.number().int().min(1).max(500).optional().describe("Maximum URLs to discover. Use 100 for normal maps, higher when the user asks for a full inventory.")
|
|
17298
18862
|
};
|
|
17299
18863
|
ExtractSiteInputSchema = {
|
|
17300
|
-
url:
|
|
17301
|
-
maxPages:
|
|
18864
|
+
url: import_zod25.z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
|
|
18865
|
+
maxPages: import_zod25.z.number().int().min(1).max(50).optional().describe("Maximum pages to extract. Use 50 when the user asks for full results or a complete crawl within MCP limits.")
|
|
17302
18866
|
};
|
|
17303
18867
|
YoutubeHarvestInputSchema = {
|
|
17304
|
-
mode:
|
|
17305
|
-
query:
|
|
17306
|
-
channelHandle:
|
|
17307
|
-
maxVideos:
|
|
18868
|
+
mode: import_zod25.z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
|
|
18869
|
+
query: import_zod25.z.string().optional().describe("Required when mode is search. The YouTube search topic in the user\u2019s words."),
|
|
18870
|
+
channelHandle: import_zod25.z.string().optional().describe("YouTube channel handle, channel ID, or URL. Examples: @mkbhd, UC..., https://youtube.com/@mkbhd."),
|
|
18871
|
+
maxVideos: import_zod25.z.number().int().min(1).max(500).default(50).describe("Number of videos to return. Default 50. Increase when user asks for full channel/history.")
|
|
17308
18872
|
};
|
|
17309
18873
|
YoutubeTranscribeInputSchema = {
|
|
17310
|
-
videoId:
|
|
18874
|
+
videoId: import_zod25.z.string().min(1).describe("YouTube video ID, e.g. dQw4w9WgXcQ")
|
|
17311
18875
|
};
|
|
17312
18876
|
FacebookPageIntelInputSchema = {
|
|
17313
|
-
pageId:
|
|
17314
|
-
libraryId:
|
|
17315
|
-
query:
|
|
17316
|
-
maxAds:
|
|
17317
|
-
country:
|
|
18877
|
+
pageId: import_zod25.z.string().optional(),
|
|
18878
|
+
libraryId: import_zod25.z.string().optional(),
|
|
18879
|
+
query: import_zod25.z.string().optional().describe("Advertiser or brand name when pageId/libraryId is not known. One of pageId, libraryId, or query is required."),
|
|
18880
|
+
maxAds: import_zod25.z.number().int().min(1).max(200).default(50),
|
|
18881
|
+
country: import_zod25.z.string().length(2).default("US")
|
|
17318
18882
|
};
|
|
17319
18883
|
FacebookAdSearchInputSchema = {
|
|
17320
|
-
query:
|
|
17321
|
-
country:
|
|
17322
|
-
maxResults:
|
|
18884
|
+
query: import_zod25.z.string().min(1).describe("Advertiser, brand, competitor, niche, or keyword to search in Facebook Ad Library."),
|
|
18885
|
+
country: import_zod25.z.string().length(2).default("US"),
|
|
18886
|
+
maxResults: import_zod25.z.number().int().min(1).max(20).default(10)
|
|
17323
18887
|
};
|
|
17324
18888
|
FacebookAdTranscribeInputSchema = {
|
|
17325
|
-
videoUrl:
|
|
18889
|
+
videoUrl: import_zod25.z.string().url().describe("Facebook CDN video URL from a facebook_page_intel result")
|
|
17326
18890
|
};
|
|
17327
18891
|
MapsPlaceIntelInputSchema = {
|
|
17328
|
-
businessName:
|
|
17329
|
-
location:
|
|
17330
|
-
gl:
|
|
17331
|
-
hl:
|
|
17332
|
-
includeReviews:
|
|
17333
|
-
maxReviews:
|
|
18892
|
+
businessName: import_zod25.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
|
|
18893
|
+
location: import_zod25.z.string().min(1).describe('City/region/country where the business should be searched, e.g. "Denver, CO". Infer from the user request when possible.'),
|
|
18894
|
+
gl: import_zod25.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
18895
|
+
hl: import_zod25.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
18896
|
+
includeReviews: import_zod25.z.boolean().default(false).describe("Whether to fetch individual review cards"),
|
|
18897
|
+
maxReviews: import_zod25.z.number().int().min(1).max(500).default(50).describe("Max review cards to return (requires includeReviews: true)")
|
|
17334
18898
|
};
|
|
17335
18899
|
MapsSearchInputSchema = {
|
|
17336
|
-
query:
|
|
17337
|
-
location:
|
|
17338
|
-
gl:
|
|
17339
|
-
hl:
|
|
17340
|
-
maxResults:
|
|
17341
|
-
proxyMode:
|
|
17342
|
-
proxyZip:
|
|
17343
|
-
debug:
|
|
18900
|
+
query: import_zod25.z.string().min(1).describe('Business category, niche, keyword, or search term. If the user says "roofers in Denver CO", use query="roofers" and location="Denver, CO". Do not put the location here when it can be separated.'),
|
|
18901
|
+
location: import_zod25.z.string().optional().describe('City, region, country, or service area for the Maps search, e.g. "Denver, CO". Infer from the user request when present.'),
|
|
18902
|
+
gl: import_zod25.z.string().length(2).default("us").describe("Google country code inferred from location."),
|
|
18903
|
+
hl: import_zod25.z.string().length(2).default("en").describe("Language inferred from user request."),
|
|
18904
|
+
maxResults: import_zod25.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
|
|
18905
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state Maps searches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts. Use configured for the server proxy ID, and none only for local direct-network debugging."),
|
|
18906
|
+
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
|
|
18907
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
|
|
17344
18908
|
};
|
|
17345
18909
|
DirectoryWorkflowInputSchema = {
|
|
17346
|
-
query:
|
|
17347
|
-
state:
|
|
17348
|
-
minPopulation:
|
|
17349
|
-
populationYear:
|
|
17350
|
-
maxCities:
|
|
17351
|
-
maxResultsPerCity:
|
|
17352
|
-
concurrency:
|
|
17353
|
-
includeZipGroups:
|
|
17354
|
-
usZipsCsvPath:
|
|
17355
|
-
saveCsv:
|
|
17356
|
-
proxyMode:
|
|
17357
|
-
proxyZip:
|
|
17358
|
-
debug:
|
|
18910
|
+
query: import_zod25.z.string().min(1).describe("Business category, niche, or keyword to search on Google Maps for every selected market, e.g. roofers, dentists, med spas. Do not include the city here."),
|
|
18911
|
+
state: import_zod25.z.string().min(2).default("TN").describe("US state abbreviation or state name used to select Census places, e.g. TN or Tennessee."),
|
|
18912
|
+
minPopulation: import_zod25.z.number().int().min(0).default(1e5).describe('Minimum Census place population for market selection. Use 100000 for "cities above 100k population".'),
|
|
18913
|
+
populationYear: import_zod25.z.number().int().min(2020).max(2025).default(2025).describe("Census population estimate year from the 2020-2025 Population Estimates Program city/place dataset."),
|
|
18914
|
+
maxCities: import_zod25.z.number().int().min(1).max(100).default(25).describe("Maximum number of markets to process after sorting by population descending."),
|
|
18915
|
+
maxResultsPerCity: import_zod25.z.number().int().min(1).max(50).default(50).describe("Google Maps business/profile candidates to collect for each city. Maximum 50."),
|
|
18916
|
+
concurrency: import_zod25.z.number().int().min(1).max(5).default(5).describe("How many city Maps searches to run in parallel. Use 5 for broad directory batches unless debugging."),
|
|
18917
|
+
includeZipGroups: import_zod25.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
|
|
18918
|
+
usZipsCsvPath: import_zod25.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
18919
|
+
saveCsv: import_zod25.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
|
|
18920
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode for every city Maps search. Use location by default for US city/state batches; retryable failures create a new residential proxy ID and new browser session for up to 5 attempts per city. Use configured for the server proxy ID, and none only for local direct-network debugging."),
|
|
18921
|
+
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its Lead Magician ZIP group or city/state location."),
|
|
18922
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
|
|
17359
18923
|
};
|
|
17360
|
-
RankTrackerModeSchema =
|
|
18924
|
+
RankTrackerModeSchema = import_zod25.z.enum(["maps", "organic", "ai_overview", "paa"]);
|
|
17361
18925
|
RankTrackerBlueprintInputSchema = {
|
|
17362
|
-
projectName:
|
|
17363
|
-
targetDomain:
|
|
17364
|
-
targetBusinessName:
|
|
17365
|
-
trackingModes:
|
|
17366
|
-
keywords:
|
|
17367
|
-
locations:
|
|
17368
|
-
competitors:
|
|
17369
|
-
database:
|
|
17370
|
-
scheduleCadence:
|
|
17371
|
-
customCron:
|
|
17372
|
-
timezone:
|
|
17373
|
-
includeCron:
|
|
17374
|
-
includeDashboard:
|
|
17375
|
-
includeAlerts:
|
|
17376
|
-
notes:
|
|
18926
|
+
projectName: import_zod25.z.string().min(1).optional().describe("Optional name for the rank tracker project, client, or campaign."),
|
|
18927
|
+
targetDomain: import_zod25.z.string().min(1).optional().describe("Primary domain to track in organic results, AI Overview citations, and PAA sources, e.g. example.com."),
|
|
18928
|
+
targetBusinessName: import_zod25.z.string().min(1).optional().describe("Primary Google Business Profile or brand/business name to match in Maps results. Required for reliable Maps rank tracking."),
|
|
18929
|
+
trackingModes: import_zod25.z.array(RankTrackerModeSchema).min(1).max(4).default(["maps", "organic", "ai_overview", "paa"]).describe("Rank tracker surfaces to build. maps uses directory_workflow/maps_search. organic uses search_serp. ai_overview uses search_serp/harvest_paa. paa uses harvest_paa source presence."),
|
|
18930
|
+
keywords: import_zod25.z.array(import_zod25.z.string().min(1)).max(200).default([]).describe("Seed keywords or service queries to track. Leave empty when the downstream AI should create the keyword table from user input."),
|
|
18931
|
+
locations: import_zod25.z.array(import_zod25.z.string().min(1)).max(100).default([]).describe("Markets, cities, ZIPs, or service areas to track. Use city/state strings like Denver, CO for localized SERP and Maps checks."),
|
|
18932
|
+
competitors: import_zod25.z.array(import_zod25.z.string().min(1)).max(100).default([]).describe("Optional competitor domains or business names to persist as comparison targets."),
|
|
18933
|
+
database: import_zod25.z.enum(["postgres", "neon", "supabase", "sqlite", "mysql"]).default("postgres").describe("Database family the downstream AI should target when generating migrations."),
|
|
18934
|
+
scheduleCadence: import_zod25.z.enum(["daily", "weekly", "monthly", "custom"]).default("weekly").describe("Default recurring rank check cadence for the generated cron/heartbeat plan."),
|
|
18935
|
+
customCron: import_zod25.z.string().min(1).optional().describe("Cron expression to use when scheduleCadence is custom."),
|
|
18936
|
+
timezone: import_zod25.z.string().min(1).default("UTC").describe("IANA timezone for scheduled rank checks, e.g. America/Denver."),
|
|
18937
|
+
includeCron: import_zod25.z.boolean().default(true).describe("Include a cron or heartbeat worker plan. Keep true for production rank trackers."),
|
|
18938
|
+
includeDashboard: import_zod25.z.boolean().default(true).describe("Include dashboard/reporting requirements in the generated prompt."),
|
|
18939
|
+
includeAlerts: import_zod25.z.boolean().default(true).describe("Include alert rules for rank movement and SERP feature gains/losses."),
|
|
18940
|
+
notes: import_zod25.z.string().max(4e3).optional().describe("Extra product, client, stack, or hosting requirements to include in the implementation prompt.")
|
|
17377
18941
|
};
|
|
17378
|
-
NullableString =
|
|
17379
|
-
MapsSearchAttemptOutput =
|
|
17380
|
-
attemptNumber:
|
|
17381
|
-
maxAttempts:
|
|
17382
|
-
status:
|
|
17383
|
-
outcome:
|
|
17384
|
-
willRetry:
|
|
17385
|
-
durationMs:
|
|
17386
|
-
resultCount:
|
|
18942
|
+
NullableString = import_zod25.z.string().nullable();
|
|
18943
|
+
MapsSearchAttemptOutput = import_zod25.z.object({
|
|
18944
|
+
attemptNumber: import_zod25.z.number().int().min(1),
|
|
18945
|
+
maxAttempts: import_zod25.z.number().int().min(1),
|
|
18946
|
+
status: import_zod25.z.enum(["ok", "failed"]),
|
|
18947
|
+
outcome: import_zod25.z.string(),
|
|
18948
|
+
willRetry: import_zod25.z.boolean(),
|
|
18949
|
+
durationMs: import_zod25.z.number().int().min(0),
|
|
18950
|
+
resultCount: import_zod25.z.number().int().min(0),
|
|
17387
18951
|
error: NullableString,
|
|
17388
|
-
proxyMode:
|
|
17389
|
-
proxyResolutionSource:
|
|
18952
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]),
|
|
18953
|
+
proxyResolutionSource: import_zod25.z.enum(["disabled", "location_reused", "location_created", "configured_fallback", "unavailable"]).nullable(),
|
|
17390
18954
|
proxyIdSuffix: NullableString,
|
|
17391
|
-
proxyTargetLevel:
|
|
18955
|
+
proxyTargetLevel: import_zod25.z.enum(["zip", "city", "state"]).nullable(),
|
|
17392
18956
|
proxyTargetLocation: NullableString,
|
|
17393
18957
|
proxyTargetZip: NullableString,
|
|
17394
18958
|
browserSessionIdSuffix: NullableString,
|
|
@@ -17397,17 +18961,17 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17397
18961
|
observedRegion: NullableString
|
|
17398
18962
|
});
|
|
17399
18963
|
MapsSearchOutputSchema = {
|
|
17400
|
-
query:
|
|
17401
|
-
location:
|
|
17402
|
-
searchQuery:
|
|
17403
|
-
searchUrl:
|
|
17404
|
-
extractedAt:
|
|
17405
|
-
requestedMaxResults:
|
|
17406
|
-
resultCount:
|
|
17407
|
-
results:
|
|
17408
|
-
position:
|
|
17409
|
-
name:
|
|
17410
|
-
placeUrl:
|
|
18964
|
+
query: import_zod25.z.string(),
|
|
18965
|
+
location: import_zod25.z.string().nullable(),
|
|
18966
|
+
searchQuery: import_zod25.z.string(),
|
|
18967
|
+
searchUrl: import_zod25.z.string().url(),
|
|
18968
|
+
extractedAt: import_zod25.z.string(),
|
|
18969
|
+
requestedMaxResults: import_zod25.z.number().int().min(1).max(50),
|
|
18970
|
+
resultCount: import_zod25.z.number().int().min(0).max(50),
|
|
18971
|
+
results: import_zod25.z.array(import_zod25.z.object({
|
|
18972
|
+
position: import_zod25.z.number().int().min(1),
|
|
18973
|
+
name: import_zod25.z.string(),
|
|
18974
|
+
placeUrl: import_zod25.z.string().url(),
|
|
17411
18975
|
cid: NullableString,
|
|
17412
18976
|
cidDecimal: NullableString,
|
|
17413
18977
|
rating: NullableString,
|
|
@@ -17418,15 +18982,15 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17418
18982
|
hoursStatus: NullableString,
|
|
17419
18983
|
websiteUrl: NullableString,
|
|
17420
18984
|
directionsUrl: NullableString,
|
|
17421
|
-
metadata:
|
|
18985
|
+
metadata: import_zod25.z.array(import_zod25.z.string())
|
|
17422
18986
|
})),
|
|
17423
|
-
attempts:
|
|
17424
|
-
durationMs:
|
|
18987
|
+
attempts: import_zod25.z.array(MapsSearchAttemptOutput),
|
|
18988
|
+
durationMs: import_zod25.z.number().int().min(0)
|
|
17425
18989
|
};
|
|
17426
|
-
DirectoryMapsBusinessOutput =
|
|
17427
|
-
position:
|
|
17428
|
-
name:
|
|
17429
|
-
placeUrl:
|
|
18990
|
+
DirectoryMapsBusinessOutput = import_zod25.z.object({
|
|
18991
|
+
position: import_zod25.z.number().int().min(1),
|
|
18992
|
+
name: import_zod25.z.string(),
|
|
18993
|
+
placeUrl: import_zod25.z.string().url(),
|
|
17430
18994
|
cid: NullableString,
|
|
17431
18995
|
cidDecimal: NullableString,
|
|
17432
18996
|
rating: NullableString,
|
|
@@ -17437,113 +19001,113 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17437
19001
|
hoursStatus: NullableString,
|
|
17438
19002
|
websiteUrl: NullableString,
|
|
17439
19003
|
directionsUrl: NullableString,
|
|
17440
|
-
metadata:
|
|
19004
|
+
metadata: import_zod25.z.array(import_zod25.z.string())
|
|
17441
19005
|
});
|
|
17442
19006
|
DirectoryWorkflowOutputSchema = {
|
|
17443
|
-
query:
|
|
17444
|
-
state:
|
|
17445
|
-
minPopulation:
|
|
17446
|
-
populationYear:
|
|
17447
|
-
maxResultsPerCity:
|
|
17448
|
-
concurrency:
|
|
17449
|
-
censusSourceUrl:
|
|
19007
|
+
query: import_zod25.z.string(),
|
|
19008
|
+
state: import_zod25.z.string(),
|
|
19009
|
+
minPopulation: import_zod25.z.number().int().min(0),
|
|
19010
|
+
populationYear: import_zod25.z.number().int().min(2020).max(2025),
|
|
19011
|
+
maxResultsPerCity: import_zod25.z.number().int().min(1).max(50),
|
|
19012
|
+
concurrency: import_zod25.z.number().int().min(1).max(5),
|
|
19013
|
+
censusSourceUrl: import_zod25.z.string().url(),
|
|
17450
19014
|
usZipsSourcePath: NullableString,
|
|
17451
|
-
warnings:
|
|
17452
|
-
extractedAt:
|
|
17453
|
-
selectedCityCount:
|
|
17454
|
-
totalResultCount:
|
|
19015
|
+
warnings: import_zod25.z.array(import_zod25.z.string()),
|
|
19016
|
+
extractedAt: import_zod25.z.string(),
|
|
19017
|
+
selectedCityCount: import_zod25.z.number().int().min(0),
|
|
19018
|
+
totalResultCount: import_zod25.z.number().int().min(0),
|
|
17455
19019
|
csvPath: NullableString,
|
|
17456
|
-
cities:
|
|
17457
|
-
city:
|
|
17458
|
-
state:
|
|
17459
|
-
location:
|
|
17460
|
-
cityKey:
|
|
17461
|
-
censusName:
|
|
17462
|
-
population:
|
|
17463
|
-
populationYear:
|
|
17464
|
-
zips:
|
|
17465
|
-
counties:
|
|
17466
|
-
status:
|
|
19020
|
+
cities: import_zod25.z.array(import_zod25.z.object({
|
|
19021
|
+
city: import_zod25.z.string(),
|
|
19022
|
+
state: import_zod25.z.string(),
|
|
19023
|
+
location: import_zod25.z.string(),
|
|
19024
|
+
cityKey: import_zod25.z.string(),
|
|
19025
|
+
censusName: import_zod25.z.string(),
|
|
19026
|
+
population: import_zod25.z.number().int().min(0),
|
|
19027
|
+
populationYear: import_zod25.z.number().int().min(2020).max(2025),
|
|
19028
|
+
zips: import_zod25.z.array(import_zod25.z.string()),
|
|
19029
|
+
counties: import_zod25.z.array(import_zod25.z.string()),
|
|
19030
|
+
status: import_zod25.z.enum(["ok", "empty", "failed"]),
|
|
17467
19031
|
error: NullableString,
|
|
17468
|
-
resultCount:
|
|
17469
|
-
durationMs:
|
|
17470
|
-
attempts:
|
|
17471
|
-
results:
|
|
19032
|
+
resultCount: import_zod25.z.number().int().min(0),
|
|
19033
|
+
durationMs: import_zod25.z.number().int().min(0),
|
|
19034
|
+
attempts: import_zod25.z.array(MapsSearchAttemptOutput),
|
|
19035
|
+
results: import_zod25.z.array(DirectoryMapsBusinessOutput)
|
|
17472
19036
|
})),
|
|
17473
|
-
durationMs:
|
|
19037
|
+
durationMs: import_zod25.z.number().int().min(0)
|
|
17474
19038
|
};
|
|
17475
|
-
RankTrackerToolPlanOutput =
|
|
17476
|
-
tool:
|
|
17477
|
-
purpose:
|
|
19039
|
+
RankTrackerToolPlanOutput = import_zod25.z.object({
|
|
19040
|
+
tool: import_zod25.z.string(),
|
|
19041
|
+
purpose: import_zod25.z.string()
|
|
17478
19042
|
});
|
|
17479
|
-
RankTrackerTableOutput =
|
|
17480
|
-
name:
|
|
17481
|
-
purpose:
|
|
17482
|
-
keyColumns:
|
|
19043
|
+
RankTrackerTableOutput = import_zod25.z.object({
|
|
19044
|
+
name: import_zod25.z.string(),
|
|
19045
|
+
purpose: import_zod25.z.string(),
|
|
19046
|
+
keyColumns: import_zod25.z.array(import_zod25.z.string())
|
|
17483
19047
|
});
|
|
17484
|
-
RankTrackerCronJobOutput =
|
|
17485
|
-
name:
|
|
17486
|
-
purpose:
|
|
17487
|
-
modes:
|
|
17488
|
-
recommendedTools:
|
|
19048
|
+
RankTrackerCronJobOutput = import_zod25.z.object({
|
|
19049
|
+
name: import_zod25.z.string(),
|
|
19050
|
+
purpose: import_zod25.z.string(),
|
|
19051
|
+
modes: import_zod25.z.array(RankTrackerModeSchema),
|
|
19052
|
+
recommendedTools: import_zod25.z.array(import_zod25.z.string())
|
|
17489
19053
|
});
|
|
17490
19054
|
RankTrackerBlueprintOutputSchema = {
|
|
17491
|
-
projectName:
|
|
19055
|
+
projectName: import_zod25.z.string(),
|
|
17492
19056
|
targetDomain: NullableString,
|
|
17493
19057
|
targetBusinessName: NullableString,
|
|
17494
|
-
trackingModes:
|
|
17495
|
-
database:
|
|
17496
|
-
recommendedTools:
|
|
17497
|
-
tables:
|
|
17498
|
-
cron:
|
|
17499
|
-
enabled:
|
|
17500
|
-
cadence:
|
|
17501
|
-
expression:
|
|
17502
|
-
timezone:
|
|
17503
|
-
jobs:
|
|
19058
|
+
trackingModes: import_zod25.z.array(RankTrackerModeSchema),
|
|
19059
|
+
database: import_zod25.z.string(),
|
|
19060
|
+
recommendedTools: import_zod25.z.array(RankTrackerToolPlanOutput),
|
|
19061
|
+
tables: import_zod25.z.array(RankTrackerTableOutput),
|
|
19062
|
+
cron: import_zod25.z.object({
|
|
19063
|
+
enabled: import_zod25.z.boolean(),
|
|
19064
|
+
cadence: import_zod25.z.string(),
|
|
19065
|
+
expression: import_zod25.z.string(),
|
|
19066
|
+
timezone: import_zod25.z.string(),
|
|
19067
|
+
jobs: import_zod25.z.array(RankTrackerCronJobOutput)
|
|
17504
19068
|
}),
|
|
17505
|
-
metrics:
|
|
17506
|
-
implementationPrompt:
|
|
19069
|
+
metrics: import_zod25.z.array(import_zod25.z.string()),
|
|
19070
|
+
implementationPrompt: import_zod25.z.string()
|
|
17507
19071
|
};
|
|
17508
|
-
OrganicResultOutput =
|
|
17509
|
-
position:
|
|
17510
|
-
title:
|
|
17511
|
-
url:
|
|
17512
|
-
domain:
|
|
19072
|
+
OrganicResultOutput = import_zod25.z.object({
|
|
19073
|
+
position: import_zod25.z.number().int(),
|
|
19074
|
+
title: import_zod25.z.string(),
|
|
19075
|
+
url: import_zod25.z.string(),
|
|
19076
|
+
domain: import_zod25.z.string(),
|
|
17513
19077
|
snippet: NullableString
|
|
17514
19078
|
});
|
|
17515
|
-
AiOverviewOutput =
|
|
17516
|
-
detected:
|
|
19079
|
+
AiOverviewOutput = import_zod25.z.object({
|
|
19080
|
+
detected: import_zod25.z.boolean(),
|
|
17517
19081
|
text: NullableString
|
|
17518
19082
|
}).nullable();
|
|
17519
|
-
EntityIdsOutput =
|
|
17520
|
-
kgIds:
|
|
17521
|
-
cids:
|
|
17522
|
-
gcids:
|
|
19083
|
+
EntityIdsOutput = import_zod25.z.object({
|
|
19084
|
+
kgIds: import_zod25.z.array(import_zod25.z.string()),
|
|
19085
|
+
cids: import_zod25.z.array(import_zod25.z.string()),
|
|
19086
|
+
gcids: import_zod25.z.array(import_zod25.z.string())
|
|
17523
19087
|
}).nullable();
|
|
17524
19088
|
HarvestPaaOutputSchema = {
|
|
17525
|
-
query:
|
|
19089
|
+
query: import_zod25.z.string(),
|
|
17526
19090
|
location: NullableString,
|
|
17527
|
-
questionCount:
|
|
19091
|
+
questionCount: import_zod25.z.number().int().min(0),
|
|
17528
19092
|
completionStatus: NullableString,
|
|
17529
|
-
questions:
|
|
17530
|
-
question:
|
|
19093
|
+
questions: import_zod25.z.array(import_zod25.z.object({
|
|
19094
|
+
question: import_zod25.z.string(),
|
|
17531
19095
|
answer: NullableString,
|
|
17532
19096
|
sourceTitle: NullableString,
|
|
17533
19097
|
sourceSite: NullableString
|
|
17534
19098
|
})),
|
|
17535
|
-
organicResults:
|
|
19099
|
+
organicResults: import_zod25.z.array(OrganicResultOutput),
|
|
17536
19100
|
aiOverview: AiOverviewOutput,
|
|
17537
19101
|
entityIds: EntityIdsOutput,
|
|
17538
|
-
durationMs:
|
|
19102
|
+
durationMs: import_zod25.z.number().min(0).nullable()
|
|
17539
19103
|
};
|
|
17540
19104
|
SearchSerpOutputSchema = {
|
|
17541
|
-
query:
|
|
19105
|
+
query: import_zod25.z.string(),
|
|
17542
19106
|
location: NullableString,
|
|
17543
|
-
organicResults:
|
|
17544
|
-
localPack:
|
|
17545
|
-
position:
|
|
17546
|
-
name:
|
|
19107
|
+
organicResults: import_zod25.z.array(OrganicResultOutput),
|
|
19108
|
+
localPack: import_zod25.z.array(import_zod25.z.object({
|
|
19109
|
+
position: import_zod25.z.number().int(),
|
|
19110
|
+
name: import_zod25.z.string(),
|
|
17547
19111
|
rating: NullableString,
|
|
17548
19112
|
reviewCount: NullableString,
|
|
17549
19113
|
websiteUrl: NullableString
|
|
@@ -17552,31 +19116,31 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17552
19116
|
entityIds: EntityIdsOutput
|
|
17553
19117
|
};
|
|
17554
19118
|
ExtractUrlOutputSchema = {
|
|
17555
|
-
url:
|
|
19119
|
+
url: import_zod25.z.string(),
|
|
17556
19120
|
title: NullableString,
|
|
17557
|
-
headings:
|
|
17558
|
-
level:
|
|
17559
|
-
text:
|
|
19121
|
+
headings: import_zod25.z.array(import_zod25.z.object({
|
|
19122
|
+
level: import_zod25.z.number().int(),
|
|
19123
|
+
text: import_zod25.z.string()
|
|
17560
19124
|
})),
|
|
17561
|
-
schemaBlockCount:
|
|
19125
|
+
schemaBlockCount: import_zod25.z.number().int().min(0),
|
|
17562
19126
|
entityName: NullableString,
|
|
17563
|
-
entityTypes:
|
|
17564
|
-
napScore:
|
|
17565
|
-
missingSchemaFields:
|
|
19127
|
+
entityTypes: import_zod25.z.array(import_zod25.z.string()),
|
|
19128
|
+
napScore: import_zod25.z.number().nullable(),
|
|
19129
|
+
missingSchemaFields: import_zod25.z.array(import_zod25.z.string()),
|
|
17566
19130
|
screenshotSaved: NullableString
|
|
17567
19131
|
};
|
|
17568
19132
|
ExtractSiteOutputSchema = {
|
|
17569
|
-
url:
|
|
17570
|
-
pageCount:
|
|
17571
|
-
pages:
|
|
17572
|
-
url:
|
|
19133
|
+
url: import_zod25.z.string(),
|
|
19134
|
+
pageCount: import_zod25.z.number().int().min(0),
|
|
19135
|
+
pages: import_zod25.z.array(import_zod25.z.object({
|
|
19136
|
+
url: import_zod25.z.string(),
|
|
17573
19137
|
title: NullableString,
|
|
17574
|
-
schemaTypes:
|
|
19138
|
+
schemaTypes: import_zod25.z.array(import_zod25.z.string())
|
|
17575
19139
|
})),
|
|
17576
|
-
durationMs:
|
|
19140
|
+
durationMs: import_zod25.z.number().min(0)
|
|
17577
19141
|
};
|
|
17578
19142
|
MapsPlaceIntelOutputSchema = {
|
|
17579
|
-
name:
|
|
19143
|
+
name: import_zod25.z.string(),
|
|
17580
19144
|
rating: NullableString,
|
|
17581
19145
|
reviewCount: NullableString,
|
|
17582
19146
|
category: NullableString,
|
|
@@ -17588,60 +19152,60 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17588
19152
|
kgmid: NullableString,
|
|
17589
19153
|
cidDecimal: NullableString,
|
|
17590
19154
|
cidUrl: NullableString,
|
|
17591
|
-
lat:
|
|
17592
|
-
lng:
|
|
17593
|
-
reviewsStatus:
|
|
17594
|
-
reviewsCollected:
|
|
17595
|
-
reviewTopics:
|
|
17596
|
-
label:
|
|
17597
|
-
count:
|
|
19155
|
+
lat: import_zod25.z.number().nullable(),
|
|
19156
|
+
lng: import_zod25.z.number().nullable(),
|
|
19157
|
+
reviewsStatus: import_zod25.z.string(),
|
|
19158
|
+
reviewsCollected: import_zod25.z.number().int().min(0),
|
|
19159
|
+
reviewTopics: import_zod25.z.array(import_zod25.z.object({
|
|
19160
|
+
label: import_zod25.z.string(),
|
|
19161
|
+
count: import_zod25.z.string()
|
|
17598
19162
|
}))
|
|
17599
19163
|
};
|
|
17600
19164
|
CreditsInfoOutputSchema = {
|
|
17601
|
-
balanceCredits:
|
|
17602
|
-
matchedCost:
|
|
17603
|
-
label:
|
|
17604
|
-
credits:
|
|
17605
|
-
unit:
|
|
19165
|
+
balanceCredits: import_zod25.z.number().nullable(),
|
|
19166
|
+
matchedCost: import_zod25.z.object({
|
|
19167
|
+
label: import_zod25.z.string(),
|
|
19168
|
+
credits: import_zod25.z.number(),
|
|
19169
|
+
unit: import_zod25.z.string(),
|
|
17606
19170
|
notes: NullableString
|
|
17607
19171
|
}).nullable(),
|
|
17608
|
-
costs:
|
|
17609
|
-
key:
|
|
17610
|
-
label:
|
|
17611
|
-
credits:
|
|
17612
|
-
unit:
|
|
19172
|
+
costs: import_zod25.z.array(import_zod25.z.object({
|
|
19173
|
+
key: import_zod25.z.string(),
|
|
19174
|
+
label: import_zod25.z.string(),
|
|
19175
|
+
credits: import_zod25.z.number(),
|
|
19176
|
+
unit: import_zod25.z.string(),
|
|
17613
19177
|
notes: NullableString
|
|
17614
19178
|
})),
|
|
17615
|
-
ledger:
|
|
17616
|
-
createdAt:
|
|
17617
|
-
operation:
|
|
17618
|
-
credits:
|
|
19179
|
+
ledger: import_zod25.z.array(import_zod25.z.object({
|
|
19180
|
+
createdAt: import_zod25.z.string(),
|
|
19181
|
+
operation: import_zod25.z.string(),
|
|
19182
|
+
credits: import_zod25.z.number(),
|
|
17619
19183
|
description: NullableString
|
|
17620
19184
|
}))
|
|
17621
19185
|
};
|
|
17622
19186
|
MapSiteUrlsOutputSchema = {
|
|
17623
|
-
startUrl:
|
|
17624
|
-
totalFound:
|
|
17625
|
-
truncated:
|
|
17626
|
-
okCount:
|
|
17627
|
-
redirectCount:
|
|
17628
|
-
brokenCount:
|
|
17629
|
-
urls:
|
|
17630
|
-
url:
|
|
17631
|
-
status:
|
|
19187
|
+
startUrl: import_zod25.z.string(),
|
|
19188
|
+
totalFound: import_zod25.z.number().int().min(0),
|
|
19189
|
+
truncated: import_zod25.z.boolean(),
|
|
19190
|
+
okCount: import_zod25.z.number().int().min(0),
|
|
19191
|
+
redirectCount: import_zod25.z.number().int().min(0),
|
|
19192
|
+
brokenCount: import_zod25.z.number().int().min(0),
|
|
19193
|
+
urls: import_zod25.z.array(import_zod25.z.object({
|
|
19194
|
+
url: import_zod25.z.string(),
|
|
19195
|
+
status: import_zod25.z.number().int().nullable()
|
|
17632
19196
|
})),
|
|
17633
|
-
durationMs:
|
|
19197
|
+
durationMs: import_zod25.z.number().min(0)
|
|
17634
19198
|
};
|
|
17635
19199
|
YoutubeHarvestOutputSchema = {
|
|
17636
|
-
mode:
|
|
17637
|
-
videoCount:
|
|
17638
|
-
channel:
|
|
19200
|
+
mode: import_zod25.z.string(),
|
|
19201
|
+
videoCount: import_zod25.z.number().int().min(0),
|
|
19202
|
+
channel: import_zod25.z.object({
|
|
17639
19203
|
title: NullableString,
|
|
17640
19204
|
subscriberCount: NullableString
|
|
17641
19205
|
}).nullable(),
|
|
17642
|
-
videos:
|
|
17643
|
-
videoId:
|
|
17644
|
-
title:
|
|
19206
|
+
videos: import_zod25.z.array(import_zod25.z.object({
|
|
19207
|
+
videoId: import_zod25.z.string(),
|
|
19208
|
+
title: import_zod25.z.string(),
|
|
17645
19209
|
channelName: NullableString,
|
|
17646
19210
|
views: NullableString,
|
|
17647
19211
|
duration: NullableString,
|
|
@@ -17649,21 +19213,21 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17649
19213
|
}))
|
|
17650
19214
|
};
|
|
17651
19215
|
FacebookAdSearchOutputSchema = {
|
|
17652
|
-
query:
|
|
17653
|
-
advertiserCount:
|
|
17654
|
-
advertisers:
|
|
19216
|
+
query: import_zod25.z.string(),
|
|
19217
|
+
advertiserCount: import_zod25.z.number().int().min(0),
|
|
19218
|
+
advertisers: import_zod25.z.array(import_zod25.z.object({
|
|
17655
19219
|
name: NullableString,
|
|
17656
|
-
adCount:
|
|
19220
|
+
adCount: import_zod25.z.number().int().nullable(),
|
|
17657
19221
|
libraryId: NullableString
|
|
17658
19222
|
}))
|
|
17659
19223
|
};
|
|
17660
19224
|
FacebookPageIntelOutputSchema = {
|
|
17661
19225
|
advertiserName: NullableString,
|
|
17662
|
-
totalAds:
|
|
17663
|
-
activeCount:
|
|
17664
|
-
videoCount:
|
|
17665
|
-
imageCount:
|
|
17666
|
-
ads:
|
|
19226
|
+
totalAds: import_zod25.z.number().int().min(0),
|
|
19227
|
+
activeCount: import_zod25.z.number().int().min(0),
|
|
19228
|
+
videoCount: import_zod25.z.number().int().min(0),
|
|
19229
|
+
imageCount: import_zod25.z.number().int().min(0),
|
|
19230
|
+
ads: import_zod25.z.array(import_zod25.z.object({
|
|
17667
19231
|
libraryId: NullableString,
|
|
17668
19232
|
status: NullableString,
|
|
17669
19233
|
creativeType: NullableString,
|
|
@@ -17671,58 +19235,58 @@ var init_mcp_tool_schemas = __esm({
|
|
|
17671
19235
|
cta: NullableString,
|
|
17672
19236
|
startDate: NullableString,
|
|
17673
19237
|
videoUrl: NullableString,
|
|
17674
|
-
variations:
|
|
19238
|
+
variations: import_zod25.z.number().int().nullable()
|
|
17675
19239
|
}))
|
|
17676
19240
|
};
|
|
17677
19241
|
CreditsInfoInputSchema = {
|
|
17678
|
-
item:
|
|
17679
|
-
includeLedger:
|
|
19242
|
+
item: import_zod25.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", or "YouTube transcription"'),
|
|
19243
|
+
includeLedger: import_zod25.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
17680
19244
|
};
|
|
17681
19245
|
SearchSerpInputSchema = {
|
|
17682
|
-
query:
|
|
17683
|
-
location:
|
|
17684
|
-
gl:
|
|
17685
|
-
hl:
|
|
17686
|
-
device:
|
|
17687
|
-
proxyMode:
|
|
17688
|
-
proxyZip:
|
|
17689
|
-
debug:
|
|
17690
|
-
pages:
|
|
19246
|
+
query: import_zod25.z.string().min(1).describe('Core search topic only. Separate location when possible. If user says "best dentist in Brooklyn NY serp", use query="best dentist" and location="Brooklyn, NY".'),
|
|
19247
|
+
location: import_zod25.z.string().optional().describe("City, region, or country for geo-targeted results, inferred from user request when present."),
|
|
19248
|
+
gl: import_zod25.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
|
|
19249
|
+
hl: import_zod25.z.string().default("en").describe("Google interface/content language inferred from user request."),
|
|
19250
|
+
device: import_zod25.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
|
|
19251
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
|
|
19252
|
+
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
19253
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
|
|
19254
|
+
pages: import_zod25.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
|
|
17691
19255
|
};
|
|
17692
19256
|
CaptureSerpSnapshotInputSchema = {
|
|
17693
|
-
query:
|
|
17694
|
-
location:
|
|
17695
|
-
gl:
|
|
17696
|
-
hl:
|
|
17697
|
-
device:
|
|
17698
|
-
proxyMode:
|
|
17699
|
-
proxyZip:
|
|
17700
|
-
pages:
|
|
17701
|
-
debug:
|
|
17702
|
-
includePageSnapshots:
|
|
17703
|
-
pageSnapshotLimit:
|
|
19257
|
+
query: import_zod25.z.string().min(1).describe("Core search query to capture as a structured SERP Intelligence snapshot. Separate the place into location when the user gives a city, region, country, or ZIP."),
|
|
19258
|
+
location: import_zod25.z.string().optional().describe("City, region, country, or service area used for localized Google results. MCP Scraper records location evidence; UULE alone is not proof of localization."),
|
|
19259
|
+
gl: import_zod25.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
|
|
19260
|
+
hl: import_zod25.z.string().default("en").describe("Google interface/content language inferred from the user request."),
|
|
19261
|
+
device: import_zod25.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
|
|
19262
|
+
proxyMode: import_zod25.z.enum(["location", "configured", "none"]).default("location").describe("Proxy behavior for capture. Use location for localized US residential evidence; it creates a fresh proxy ID per attempt. If Google shows a CAPTCHA/challenge, browser-service sessions briefly wait for the automatic solver first, then retry with a fresh proxy/session if the challenge does not clear. Also retries proxy tunnel failure and wrong-location evidence before returning. Use configured only for the static residential proxy, and none only for direct-network debugging."),
|
|
19263
|
+
proxyZip: import_zod25.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting when a precise city-center or ZIP proxy is needed. With proxyMode location this ZIP is used for each fresh proxy attempt."),
|
|
19264
|
+
pages: import_zod25.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
|
|
19265
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
|
|
19266
|
+
includePageSnapshots: import_zod25.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs through the same product capture path."),
|
|
19267
|
+
pageSnapshotLimit: import_zod25.z.number().int().min(0).max(10).default(0).describe("Maximum ranking-page snapshots to capture when includePageSnapshots is true. Use 0 when only SERP evidence is needed.")
|
|
17704
19268
|
};
|
|
17705
19269
|
ScreenshotInputSchema = {
|
|
17706
|
-
url:
|
|
17707
|
-
device:
|
|
17708
|
-
allowLocal:
|
|
19270
|
+
url: import_zod25.z.string().url().describe("URL to capture as a full-page screenshot. Use http or https. Pass allowLocal: true to capture localhost or private-network URLs during development."),
|
|
19271
|
+
device: import_zod25.z.enum(["desktop", "mobile"]).default("desktop").describe("Viewport profile. desktop = 1440\xD7900. mobile = 390\xD7844. Use desktop by default; use mobile when the user asks for a mobile view."),
|
|
19272
|
+
allowLocal: import_zod25.z.boolean().default(false).describe("Allow localhost and private-network URLs (127.x, 192.168.x, 10.x, etc.). For local development only \u2014 not for production use.")
|
|
17709
19273
|
};
|
|
17710
19274
|
CaptureSerpPageSnapshotsInputSchema = {
|
|
17711
|
-
urls:
|
|
17712
|
-
targets:
|
|
17713
|
-
url:
|
|
17714
|
-
sourceKind:
|
|
17715
|
-
sourcePosition:
|
|
19275
|
+
urls: import_zod25.z.array(import_zod25.z.string().url()).min(1).max(25).describe("Public HTTP/HTTPS URLs to capture as SERP Intelligence page snapshots. Do not pass localhost, private IPs, file URLs, or internal admin URLs."),
|
|
19276
|
+
targets: import_zod25.z.array(import_zod25.z.object({
|
|
19277
|
+
url: import_zod25.z.string().url().describe("Public HTTP/HTTPS URL to capture."),
|
|
19278
|
+
sourceKind: import_zod25.z.enum(["organic", "ai_citation", "local_pack_website", "configured_target", "site_subject"]).default("configured_target").describe("Why this page is being captured for SERP Intelligence evidence."),
|
|
19279
|
+
sourcePosition: import_zod25.z.number().int().min(1).optional().describe("Ranking or citation position when the page came from SERP evidence.")
|
|
17716
19280
|
}).strict()).min(1).max(25).optional().describe("Structured page snapshot targets. Use this instead of urls when source kind or position should be preserved."),
|
|
17717
|
-
maxConcurrency:
|
|
17718
|
-
timeoutMs:
|
|
17719
|
-
debug:
|
|
19281
|
+
maxConcurrency: import_zod25.z.number().int().min(1).max(5).default(2).describe("Parallel page captures. Use 2 normally; higher values can increase proxy/browser pressure."),
|
|
19282
|
+
timeoutMs: import_zod25.z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds. Increase for slow pages; timeout artifacts are returned as structured capture failures."),
|
|
19283
|
+
debug: import_zod25.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics for page snapshot debugging. Use true for capture, network, or proxy troubleshooting.")
|
|
17720
19284
|
};
|
|
17721
19285
|
}
|
|
17722
19286
|
});
|
|
17723
19287
|
|
|
17724
19288
|
// src/mcp/rank-tracker-blueprint.ts
|
|
17725
|
-
function
|
|
19289
|
+
function normalizeDomain2(value) {
|
|
17726
19290
|
const trimmed = value?.trim();
|
|
17727
19291
|
if (!trimmed) return null;
|
|
17728
19292
|
try {
|
|
@@ -17984,7 +19548,7 @@ function buildPrompt(input, modes, tools, tables, jobs, metrics, expression, tar
|
|
|
17984
19548
|
}
|
|
17985
19549
|
function buildRankTrackerBlueprint(input) {
|
|
17986
19550
|
const trackingModes = uniqueModes(input.trackingModes);
|
|
17987
|
-
const targetDomain =
|
|
19551
|
+
const targetDomain = normalizeDomain2(input.targetDomain);
|
|
17988
19552
|
const targetBusinessName = input.targetBusinessName?.trim() || null;
|
|
17989
19553
|
const projectName = input.projectName?.trim() || "MCP Scraper Rank Tracker";
|
|
17990
19554
|
const database = input.database || "postgres";
|
|
@@ -18072,7 +19636,7 @@ function localPlanningToolAnnotations(title) {
|
|
|
18072
19636
|
function listSavedReports() {
|
|
18073
19637
|
try {
|
|
18074
19638
|
const dir = outputBaseDir();
|
|
18075
|
-
return (0,
|
|
19639
|
+
return (0, import_node_fs6.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs6.statSync)((0, import_node_path9.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
|
|
18076
19640
|
} catch {
|
|
18077
19641
|
return [];
|
|
18078
19642
|
}
|
|
@@ -18096,9 +19660,9 @@ function registerSavedReportResources(server) {
|
|
|
18096
19660
|
},
|
|
18097
19661
|
async (uri, variables) => {
|
|
18098
19662
|
const requested = Array.isArray(variables.filename) ? variables.filename[0] : variables.filename;
|
|
18099
|
-
const filename = (0,
|
|
19663
|
+
const filename = (0, import_node_path9.basename)(decodeURIComponent(String(requested ?? "")));
|
|
18100
19664
|
if (!filename.endsWith(".md")) throw new Error("Only saved .md reports can be read");
|
|
18101
|
-
const text = (0,
|
|
19665
|
+
const text = (0, import_node_fs6.readFileSync)((0, import_node_path9.join)(outputBaseDir(), filename), "utf8");
|
|
18102
19666
|
return { contents: [{ uri: uri.href, mimeType: "text/markdown", text }] };
|
|
18103
19667
|
}
|
|
18104
19668
|
);
|
|
@@ -18197,7 +19761,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
18197
19761
|
}, async (input) => formatMapsSearch(await executor.mapsSearch(input), input));
|
|
18198
19762
|
server.registerTool("directory_workflow", {
|
|
18199
19763
|
title: "Directory Workflow: Markets + Maps",
|
|
18200
|
-
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Use concurrency up to 5 for parallel city sessions. Keep proxyMode as location so each city search creates fresh residential proxy IDs and browser sessions; retryable city failures rotate to a new proxy and new browser session for up to 5 attempts. Saved CSV rows include source_location, result_position, business_name, review_stars, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. Structured city results include sanitized attempt telemetry.
|
|
19764
|
+
description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Use concurrency up to 5 for parallel city sessions. Keep proxyMode as location so each city search creates fresh residential proxy IDs and browser sessions; retryable city failures rotate to a new proxy and new browser session for up to 5 attempts. Saved CSV rows include source_location, result_position, business_name, review_stars, review_count, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. Structured city results include sanitized attempt telemetry. Use maps_place_intel only when a selected profile needs deeper review topics, profile review count confirmation, or review cards. For local Lead Magician ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath only in local/test mode.'),
|
|
18201
19765
|
inputSchema: DirectoryWorkflowInputSchema,
|
|
18202
19766
|
outputSchema: DirectoryWorkflowOutputSchema,
|
|
18203
19767
|
annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
|
|
@@ -18223,13 +19787,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
18223
19787
|
}
|
|
18224
19788
|
}, async (input) => formatCreditsInfo(await executor.creditsInfo(input), input));
|
|
18225
19789
|
}
|
|
18226
|
-
var import_mcp,
|
|
19790
|
+
var import_mcp, import_node_fs6, import_node_path9;
|
|
18227
19791
|
var init_paa_mcp_server = __esm({
|
|
18228
19792
|
"src/mcp/paa-mcp-server.ts"() {
|
|
18229
19793
|
"use strict";
|
|
18230
19794
|
import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
18231
|
-
|
|
18232
|
-
|
|
19795
|
+
import_node_fs6 = require("fs");
|
|
19796
|
+
import_node_path9 = require("path");
|
|
18233
19797
|
init_version();
|
|
18234
19798
|
init_mcp_response_formatter();
|
|
18235
19799
|
init_mcp_tool_schemas();
|
|
@@ -18396,11 +19960,11 @@ function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
|
18396
19960
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
18397
19961
|
}, async (input) => executor.captureSerpPageSnapshots(input));
|
|
18398
19962
|
}
|
|
18399
|
-
var
|
|
19963
|
+
var import_hono9, import_webStandardStreamableHttp, mcpApp;
|
|
18400
19964
|
var init_mcp_routes = __esm({
|
|
18401
19965
|
"src/mcp/mcp-routes.ts"() {
|
|
18402
19966
|
"use strict";
|
|
18403
|
-
|
|
19967
|
+
import_hono9 = require("hono");
|
|
18404
19968
|
import_webStandardStreamableHttp = require("@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js");
|
|
18405
19969
|
init_paa_mcp_server();
|
|
18406
19970
|
init_http_mcp_tool_executor();
|
|
@@ -18408,7 +19972,7 @@ var init_mcp_routes = __esm({
|
|
|
18408
19972
|
init_db();
|
|
18409
19973
|
init_mcp_tool_schemas();
|
|
18410
19974
|
configureReportSaving(false);
|
|
18411
|
-
mcpApp = new
|
|
19975
|
+
mcpApp = new import_hono9.Hono();
|
|
18412
19976
|
mcpApp.all("/", async (c) => {
|
|
18413
19977
|
try {
|
|
18414
19978
|
const keyOrError = await requireMcpCallerKey(c);
|
|
@@ -18480,7 +20044,7 @@ async function migrateBrowserAgent() {
|
|
|
18480
20044
|
}
|
|
18481
20045
|
async function createSessionRow(input) {
|
|
18482
20046
|
const db = getDb();
|
|
18483
|
-
const id = `bas_${(0,
|
|
20047
|
+
const id = `bas_${(0, import_node_crypto4.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
18484
20048
|
await db.execute({
|
|
18485
20049
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, last_action_at)
|
|
18486
20050
|
VALUES (?, ?, ?, ?, 'open', ?, ?, datetime('now'))`,
|
|
@@ -18541,7 +20105,7 @@ async function recordAction(input) {
|
|
|
18541
20105
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
18542
20106
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
18543
20107
|
args: [
|
|
18544
|
-
`baa_${(0,
|
|
20108
|
+
`baa_${(0, import_node_crypto4.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
18545
20109
|
input.sessionId,
|
|
18546
20110
|
input.type,
|
|
18547
20111
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -18581,11 +20145,11 @@ async function listReplayRows(sessionId) {
|
|
|
18581
20145
|
});
|
|
18582
20146
|
return res.rows;
|
|
18583
20147
|
}
|
|
18584
|
-
var
|
|
20148
|
+
var import_node_crypto4, _ready;
|
|
18585
20149
|
var init_browser_agent_db = __esm({
|
|
18586
20150
|
"src/api/browser-agent-db.ts"() {
|
|
18587
20151
|
"use strict";
|
|
18588
|
-
|
|
20152
|
+
import_node_crypto4 = require("crypto");
|
|
18589
20153
|
init_db();
|
|
18590
20154
|
_ready = false;
|
|
18591
20155
|
}
|
|
@@ -18963,7 +20527,7 @@ async function loadOpenSession(id, userId) {
|
|
|
18963
20527
|
return row;
|
|
18964
20528
|
}
|
|
18965
20529
|
function buildBrowserAgentRoutes() {
|
|
18966
|
-
const app2 = new
|
|
20530
|
+
const app2 = new import_hono10.Hono();
|
|
18967
20531
|
app2.use("*", async (c, next) => {
|
|
18968
20532
|
await migrateBrowserAgent();
|
|
18969
20533
|
return next();
|
|
@@ -19279,11 +20843,11 @@ function buildBrowserAgentRoutes() {
|
|
|
19279
20843
|
});
|
|
19280
20844
|
return app2;
|
|
19281
20845
|
}
|
|
19282
|
-
var
|
|
20846
|
+
var import_hono10, auth;
|
|
19283
20847
|
var init_browser_agent_routes = __esm({
|
|
19284
20848
|
"src/api/browser-agent-routes.ts"() {
|
|
19285
20849
|
"use strict";
|
|
19286
|
-
|
|
20850
|
+
import_hono10 = require("hono");
|
|
19287
20851
|
init_api_auth();
|
|
19288
20852
|
init_errors();
|
|
19289
20853
|
init_db();
|
|
@@ -19496,16 +21060,16 @@ var init_browser_agent_console = __esm({
|
|
|
19496
21060
|
});
|
|
19497
21061
|
|
|
19498
21062
|
// src/api/stripe-routes.ts
|
|
19499
|
-
var import_stripe,
|
|
21063
|
+
var import_stripe, import_hono11, stripe, stripeApp;
|
|
19500
21064
|
var init_stripe_routes = __esm({
|
|
19501
21065
|
"src/api/stripe-routes.ts"() {
|
|
19502
21066
|
"use strict";
|
|
19503
21067
|
import_stripe = __toESM(require("stripe"), 1);
|
|
19504
|
-
|
|
21068
|
+
import_hono11 = require("hono");
|
|
19505
21069
|
init_db();
|
|
19506
21070
|
init_rates();
|
|
19507
21071
|
stripe = new import_stripe.default(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
19508
|
-
stripeApp = new
|
|
21072
|
+
stripeApp = new import_hono11.Hono();
|
|
19509
21073
|
stripeApp.post("/webhooks", async (c) => {
|
|
19510
21074
|
const sig = c.req.header("stripe-signature");
|
|
19511
21075
|
const body = await c.req.text();
|
|
@@ -19606,31 +21170,31 @@ var init_site_audit_worker = __esm({
|
|
|
19606
21170
|
});
|
|
19607
21171
|
|
|
19608
21172
|
// src/api/billing-schemas.ts
|
|
19609
|
-
var
|
|
21173
|
+
var import_zod26, BillingCheckoutBodySchema, FreeCreditBreakdownSchema, BillingBalanceResponseSchema, MonthlyRefreshSweepResultSchema;
|
|
19610
21174
|
var init_billing_schemas = __esm({
|
|
19611
21175
|
"src/api/billing-schemas.ts"() {
|
|
19612
21176
|
"use strict";
|
|
19613
|
-
|
|
19614
|
-
BillingCheckoutBodySchema =
|
|
19615
|
-
priceId:
|
|
21177
|
+
import_zod26 = require("zod");
|
|
21178
|
+
BillingCheckoutBodySchema = import_zod26.z.object({
|
|
21179
|
+
priceId: import_zod26.z.string().min(1)
|
|
19616
21180
|
});
|
|
19617
|
-
FreeCreditBreakdownSchema =
|
|
19618
|
-
signup_grant_mc:
|
|
19619
|
-
monthly_refresh_mc:
|
|
19620
|
-
total_free_mc:
|
|
19621
|
-
signup_grant_credits:
|
|
19622
|
-
monthly_refresh_credits:
|
|
19623
|
-
total_free_credits:
|
|
21181
|
+
FreeCreditBreakdownSchema = import_zod26.z.object({
|
|
21182
|
+
signup_grant_mc: import_zod26.z.number().int().nonnegative(),
|
|
21183
|
+
monthly_refresh_mc: import_zod26.z.number().int().nonnegative(),
|
|
21184
|
+
total_free_mc: import_zod26.z.number().int().nonnegative(),
|
|
21185
|
+
signup_grant_credits: import_zod26.z.number().nonnegative(),
|
|
21186
|
+
monthly_refresh_credits: import_zod26.z.number().nonnegative(),
|
|
21187
|
+
total_free_credits: import_zod26.z.number().nonnegative()
|
|
19624
21188
|
});
|
|
19625
|
-
BillingBalanceResponseSchema =
|
|
19626
|
-
balance_mc:
|
|
19627
|
-
balance_credits:
|
|
21189
|
+
BillingBalanceResponseSchema = import_zod26.z.object({
|
|
21190
|
+
balance_mc: import_zod26.z.number().int().nonnegative(),
|
|
21191
|
+
balance_credits: import_zod26.z.number().nonnegative(),
|
|
19628
21192
|
free_credits: FreeCreditBreakdownSchema,
|
|
19629
|
-
ledger:
|
|
21193
|
+
ledger: import_zod26.z.array(import_zod26.z.any())
|
|
19630
21194
|
});
|
|
19631
|
-
MonthlyRefreshSweepResultSchema =
|
|
19632
|
-
usersRefreshed:
|
|
19633
|
-
totalMcGranted:
|
|
21195
|
+
MonthlyRefreshSweepResultSchema = import_zod26.z.object({
|
|
21196
|
+
usersRefreshed: import_zod26.z.number().int().nonnegative(),
|
|
21197
|
+
totalMcGranted: import_zod26.z.number().int().nonnegative()
|
|
19634
21198
|
});
|
|
19635
21199
|
}
|
|
19636
21200
|
});
|
|
@@ -19792,14 +21356,14 @@ function getSessionSecret() {
|
|
|
19792
21356
|
function safeEqualHex(a, b) {
|
|
19793
21357
|
if (a.length !== b.length) return false;
|
|
19794
21358
|
try {
|
|
19795
|
-
return (0,
|
|
21359
|
+
return (0, import_node_crypto5.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
19796
21360
|
} catch {
|
|
19797
21361
|
return false;
|
|
19798
21362
|
}
|
|
19799
21363
|
}
|
|
19800
21364
|
function signSession(userId) {
|
|
19801
21365
|
const payload = String(userId);
|
|
19802
|
-
const sig = (0,
|
|
21366
|
+
const sig = (0, import_node_crypto5.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
19803
21367
|
return `${payload}.${sig}`;
|
|
19804
21368
|
}
|
|
19805
21369
|
function verifySession(token) {
|
|
@@ -19807,16 +21371,16 @@ function verifySession(token) {
|
|
|
19807
21371
|
if (dot === -1) return null;
|
|
19808
21372
|
const payload = token.slice(0, dot);
|
|
19809
21373
|
const sig = token.slice(dot + 1);
|
|
19810
|
-
const expected = (0,
|
|
21374
|
+
const expected = (0, import_node_crypto5.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
19811
21375
|
if (!safeEqualHex(sig, expected)) return null;
|
|
19812
21376
|
const id = parseInt(payload);
|
|
19813
21377
|
return isNaN(id) ? null : id;
|
|
19814
21378
|
}
|
|
19815
|
-
var
|
|
21379
|
+
var import_node_crypto5, isProduction, secret;
|
|
19816
21380
|
var init_session = __esm({
|
|
19817
21381
|
"src/api/session.ts"() {
|
|
19818
21382
|
"use strict";
|
|
19819
|
-
|
|
21383
|
+
import_node_crypto5 = require("crypto");
|
|
19820
21384
|
isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
19821
21385
|
secret = () => getSessionSecret();
|
|
19822
21386
|
}
|
|
@@ -20032,7 +21596,7 @@ async function checkHarvestLimits(userId, email, extraSlots = 0) {
|
|
|
20032
21596
|
if (active >= limit) return { error: `You have ${active} job${active !== 1 ? "s" : ""} running. Your account allows ${limit} concurrent job${limit !== 1 ? "s" : ""}. Wait for one to finish or add a concurrency slot at mcpscraper.dev/billing.` };
|
|
20033
21597
|
return null;
|
|
20034
21598
|
}
|
|
20035
|
-
var import_resend,
|
|
21599
|
+
var import_resend, import_hono12, import_hono13, import_factory6, import_cookie, import_stripe2, secureCookies, isProduction2, sessionCookieOptions, requireAllowedOrigin, auth2, adminAuth, sessionAuth, app, STRIPE_API_VERSION, BYPASS_EMAILS, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
|
|
20036
21600
|
var init_server = __esm({
|
|
20037
21601
|
"src/api/server.ts"() {
|
|
20038
21602
|
"use strict";
|
|
@@ -20049,8 +21613,8 @@ var init_server = __esm({
|
|
|
20049
21613
|
init_media_extractor();
|
|
20050
21614
|
init_site_mapper();
|
|
20051
21615
|
init_site_extractor();
|
|
20052
|
-
|
|
20053
|
-
|
|
21616
|
+
import_hono12 = require("hono");
|
|
21617
|
+
import_hono13 = require("inngest/hono");
|
|
20054
21618
|
init_client();
|
|
20055
21619
|
init_site_audit();
|
|
20056
21620
|
init_site_audit_routes();
|
|
@@ -20059,6 +21623,7 @@ var init_server = __esm({
|
|
|
20059
21623
|
init_facebook_ad_routes();
|
|
20060
21624
|
init_maps_routes();
|
|
20061
21625
|
init_directory_routes();
|
|
21626
|
+
init_workflow_routes();
|
|
20062
21627
|
init_serp_intelligence_routes();
|
|
20063
21628
|
init_mcp_routes();
|
|
20064
21629
|
init_browser_agent_routes();
|
|
@@ -20121,7 +21686,7 @@ var init_server = __esm({
|
|
|
20121
21686
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
20122
21687
|
return next();
|
|
20123
21688
|
});
|
|
20124
|
-
app = new
|
|
21689
|
+
app = new import_hono12.Hono();
|
|
20125
21690
|
STRIPE_API_VERSION = "2026-02-25.clover";
|
|
20126
21691
|
app.use("*", async (c, next) => {
|
|
20127
21692
|
await next();
|
|
@@ -20719,19 +22284,21 @@ var init_server = __esm({
|
|
|
20719
22284
|
}
|
|
20720
22285
|
const { drainQueue: drainQueue2 } = await Promise.resolve().then(() => (init_worker(), worker_exports));
|
|
20721
22286
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
22287
|
+
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
20722
22288
|
const [results, sweepResult] = await Promise.all([
|
|
20723
22289
|
drainQueue2(budget),
|
|
20724
22290
|
runMonthlyRefreshSweep()
|
|
20725
22291
|
]);
|
|
20726
|
-
return c.json({ drained: results.length, results, sweepResult });
|
|
22292
|
+
return c.json({ drained: results.length, results, sweepResult, workflowDispatch: workflowDispatchResult });
|
|
20727
22293
|
});
|
|
20728
|
-
app.on(["GET", "POST", "PUT"], "/api/inngest", (0,
|
|
22294
|
+
app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono13.serve)({ client: inngest, functions: [siteAuditFn] }));
|
|
20729
22295
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
20730
22296
|
app.route("/youtube", youtubeApp);
|
|
20731
22297
|
app.route("/screenshot", screenshotApp);
|
|
20732
22298
|
app.route("/facebook", facebookAdApp);
|
|
20733
22299
|
app.route("/maps", mapsApp);
|
|
20734
22300
|
app.route("/directory", directoryApp);
|
|
22301
|
+
app.route("/workflows", workflowApp);
|
|
20735
22302
|
app.route("/serp-intelligence", serpIntelligenceApp);
|
|
20736
22303
|
app.route("/mcp", mcpApp);
|
|
20737
22304
|
app.route("/agent", buildBrowserAgentRoutes());
|
|
@@ -20844,10 +22411,10 @@ var init_server = __esm({
|
|
|
20844
22411
|
});
|
|
20845
22412
|
|
|
20846
22413
|
// bin/api-server.ts
|
|
20847
|
-
var
|
|
22414
|
+
var import_node_fs7 = require("fs");
|
|
20848
22415
|
function loadDotEnv() {
|
|
20849
22416
|
try {
|
|
20850
|
-
for (const line of (0,
|
|
22417
|
+
for (const line of (0, import_node_fs7.readFileSync)(".env", "utf8").split("\n")) {
|
|
20851
22418
|
const eq = line.indexOf("=");
|
|
20852
22419
|
if (eq < 1 || line.trimStart().startsWith("#")) continue;
|
|
20853
22420
|
const k = line.slice(0, eq).trim();
|