mcp-scraper 0.2.8 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -3
- package/dist/bin/api-server.cjs +2073 -506
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1276 -0
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -0
- package/dist/bin/mcp-scraper-cli.d.cts +1 -0
- package/dist/bin/mcp-scraper-cli.d.ts +1 -0
- package/dist/bin/mcp-scraper-cli.js +476 -0
- package/dist/bin/mcp-scraper-cli.js.map +1 -0
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +3 -3
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +4 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +4 -1
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +3 -3
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-5HMOPP76.js → chunk-2OHP6WWF.js} +2 -2
- package/dist/{chunk-D4CJBZBY.js → chunk-7GCCOT3M.js} +372 -1
- package/dist/chunk-7GCCOT3M.js.map +1 -0
- package/dist/chunk-L6IS63WS.js +869 -0
- package/dist/chunk-L6IS63WS.js.map +1 -0
- package/dist/{chunk-IQOCZGJJ.js → chunk-ROS67BNV.js} +5 -5
- package/dist/chunk-WKWFO45O.js +7 -0
- package/dist/chunk-WKWFO45O.js.map +1 -0
- package/dist/{chunk-6NEXSNSA.js → chunk-YAKKRQRH.js} +4 -4
- package/dist/chunk-YAKKRQRH.js.map +1 -0
- package/dist/{db-YWCNHBLH.js → db-BVHYI57K.js} +38 -2
- package/dist/{server-BTTDFPSQ.js → server-USOIPC2I.js} +585 -266
- package/dist/server-USOIPC2I.js.map +1 -0
- package/dist/{worker-5O44YBF4.js → worker-TDJQ6TH3.js} +8 -8
- package/docs/specs/agent-ready-seo-packet-spec.md +237 -0
- package/docs/specs/cli-agent-wiring-spec.md +203 -0
- package/docs/specs/deferred-work-spec.md +12 -0
- package/docs/specs/local-competitive-audit-spec.md +312 -0
- package/docs/specs/scheduled-workflows-api-spec.md +304 -0
- package/docs/specs/seo-cli-growth-roadmap-spec.md +179 -0
- package/docs/specs/seo-workflow-runner-and-reports-spec.md +241 -0
- package/package.json +20 -19
- package/dist/chunk-6NEXSNSA.js.map +0 -1
- package/dist/chunk-D4CJBZBY.js.map +0 -1
- package/dist/chunk-I26QN7WQ.js +0 -7
- package/dist/chunk-I26QN7WQ.js.map +0 -1
- package/dist/server-BTTDFPSQ.js.map +0 -1
- /package/dist/{chunk-5HMOPP76.js.map → chunk-2OHP6WWF.js.map} +0 -0
- /package/dist/{chunk-IQOCZGJJ.js.map → chunk-ROS67BNV.js.map} +0 -0
- /package/dist/{db-YWCNHBLH.js.map → db-BVHYI57K.js.map} +0 -0
- /package/dist/{worker-5O44YBF4.js.map → worker-TDJQ6TH3.js.map} +0 -0
|
@@ -15,45 +15,14 @@ import {
|
|
|
15
15
|
harvestProblemResponse,
|
|
16
16
|
insufficientBalanceResponse,
|
|
17
17
|
serializeHarvestProblem
|
|
18
|
-
} from "./chunk-
|
|
19
|
-
import {
|
|
20
|
-
BrowserDriver,
|
|
21
|
-
MapsPlaceOptionsSchema,
|
|
22
|
-
MapsSearchOptionsSchema,
|
|
23
|
-
MapsSelectors,
|
|
24
|
-
RawMapsAboutAttributeSchema,
|
|
25
|
-
RawMapsHoursRowSchema,
|
|
26
|
-
RawMapsOverviewSchema,
|
|
27
|
-
RawMapsReviewStatsSchema,
|
|
28
|
-
browserServiceApiKey,
|
|
29
|
-
browserServiceProxyId,
|
|
30
|
-
buildYouTubeChannelVideosUrl,
|
|
31
|
-
deleteKernelProxyId,
|
|
32
|
-
harvest,
|
|
33
|
-
resolveKernelProxyId
|
|
34
|
-
} from "./chunk-CQTAKXBN.js";
|
|
35
|
-
import {
|
|
36
|
-
CaptureSerpPageSnapshotsInputSchema,
|
|
37
|
-
CaptureSerpSnapshotInputSchema,
|
|
38
|
-
HttpMcpToolExecutor,
|
|
39
|
-
buildPaaExtractorMcpServer,
|
|
40
|
-
configureReportSaving,
|
|
41
|
-
harvestTimeoutBudget,
|
|
42
|
-
liveWebToolAnnotations,
|
|
43
|
-
outputBaseDir
|
|
44
|
-
} from "./chunk-6NEXSNSA.js";
|
|
45
|
-
import {
|
|
46
|
-
CaptchaError,
|
|
47
|
-
RECAPTCHA_INSTRUCTIONS,
|
|
48
|
-
sanitizeVendorName
|
|
49
|
-
} from "./chunk-M2S27J6Z.js";
|
|
50
|
-
import "./chunk-I26QN7WQ.js";
|
|
18
|
+
} from "./chunk-ROS67BNV.js";
|
|
51
19
|
import {
|
|
52
20
|
SiteAuditJobRowSchema,
|
|
53
21
|
cancelJob,
|
|
54
22
|
checkRateLimit,
|
|
55
23
|
claimMonthlyFreeRefresh,
|
|
56
24
|
completeJob,
|
|
25
|
+
completeWorkflowRunRecord,
|
|
57
26
|
consumePasswordResetToken,
|
|
58
27
|
countActiveJobsForUser,
|
|
59
28
|
countActiveUsers,
|
|
@@ -61,10 +30,14 @@ import {
|
|
|
61
30
|
createPasswordResetToken,
|
|
62
31
|
createRunningJob,
|
|
63
32
|
createUser,
|
|
33
|
+
createWorkflowRun,
|
|
34
|
+
createWorkflowSchedule,
|
|
64
35
|
creditMc,
|
|
65
36
|
deactivateUser,
|
|
66
37
|
debitMc,
|
|
38
|
+
deleteWorkflowSchedule,
|
|
67
39
|
failJob,
|
|
40
|
+
failWorkflowRunRecord,
|
|
68
41
|
getDb,
|
|
69
42
|
getJob,
|
|
70
43
|
getLedger,
|
|
@@ -73,15 +46,27 @@ import {
|
|
|
73
46
|
getUserById,
|
|
74
47
|
getUserByStripeCustomerId,
|
|
75
48
|
getUserStats,
|
|
49
|
+
getWorkflowArtifact,
|
|
50
|
+
getWorkflowRun,
|
|
51
|
+
getWorkflowSchedule,
|
|
76
52
|
ledgerExistsForOperation,
|
|
77
53
|
ledgerExistsForStripePI,
|
|
54
|
+
listDueWorkflowSchedules,
|
|
78
55
|
listHarvestAttempts,
|
|
79
56
|
listJobs,
|
|
80
57
|
listRequestEvents,
|
|
81
58
|
listUsers,
|
|
59
|
+
listWorkflowArtifacts,
|
|
60
|
+
listWorkflowRuns,
|
|
61
|
+
listWorkflowSchedules,
|
|
82
62
|
logRequestEvent,
|
|
63
|
+
markWorkflowRunRunning,
|
|
64
|
+
markWorkflowScheduleRan,
|
|
65
|
+
patchWorkflowSchedule,
|
|
83
66
|
reconcileBalanceMc,
|
|
84
67
|
recordStripeEvent,
|
|
68
|
+
recordWorkflowWebhookDelivery,
|
|
69
|
+
replaceWorkflowArtifacts,
|
|
85
70
|
revokeApiKey,
|
|
86
71
|
rotateApiKey,
|
|
87
72
|
setConcurrencySubId,
|
|
@@ -89,7 +74,47 @@ import {
|
|
|
89
74
|
setPassword,
|
|
90
75
|
setStripeCustomerId,
|
|
91
76
|
verifyPassword
|
|
92
|
-
} from "./chunk-
|
|
77
|
+
} from "./chunk-7GCCOT3M.js";
|
|
78
|
+
import {
|
|
79
|
+
BrowserDriver,
|
|
80
|
+
MapsPlaceOptionsSchema,
|
|
81
|
+
MapsSearchOptionsSchema,
|
|
82
|
+
MapsSelectors,
|
|
83
|
+
RawMapsAboutAttributeSchema,
|
|
84
|
+
RawMapsHoursRowSchema,
|
|
85
|
+
RawMapsOverviewSchema,
|
|
86
|
+
RawMapsReviewStatsSchema,
|
|
87
|
+
browserServiceApiKey,
|
|
88
|
+
browserServiceProxyId,
|
|
89
|
+
buildYouTubeChannelVideosUrl,
|
|
90
|
+
deleteKernelProxyId,
|
|
91
|
+
harvest,
|
|
92
|
+
resolveKernelProxyId
|
|
93
|
+
} from "./chunk-CQTAKXBN.js";
|
|
94
|
+
import {
|
|
95
|
+
CaptureSerpPageSnapshotsInputSchema,
|
|
96
|
+
CaptureSerpSnapshotInputSchema,
|
|
97
|
+
HttpMcpToolExecutor,
|
|
98
|
+
buildPaaExtractorMcpServer,
|
|
99
|
+
configureReportSaving,
|
|
100
|
+
harvestTimeoutBudget,
|
|
101
|
+
liveWebToolAnnotations,
|
|
102
|
+
outputBaseDir
|
|
103
|
+
} from "./chunk-YAKKRQRH.js";
|
|
104
|
+
import {
|
|
105
|
+
CaptchaError,
|
|
106
|
+
RECAPTCHA_INSTRUCTIONS,
|
|
107
|
+
sanitizeVendorName
|
|
108
|
+
} from "./chunk-M2S27J6Z.js";
|
|
109
|
+
import {
|
|
110
|
+
csvRecords,
|
|
111
|
+
listWorkflowDefinitions,
|
|
112
|
+
rowsToCsv,
|
|
113
|
+
runWorkflow,
|
|
114
|
+
workflowDefinition,
|
|
115
|
+
workflowOutputBaseDir
|
|
116
|
+
} from "./chunk-L6IS63WS.js";
|
|
117
|
+
import "./chunk-WKWFO45O.js";
|
|
93
118
|
|
|
94
119
|
// src/api/outbound-sanitize.ts
|
|
95
120
|
var KEY_RENAMES = {
|
|
@@ -4848,7 +4873,7 @@ async function extractSite(opts) {
|
|
|
4848
4873
|
}
|
|
4849
4874
|
|
|
4850
4875
|
// src/api/server.ts
|
|
4851
|
-
import { Hono as
|
|
4876
|
+
import { Hono as Hono12 } from "hono";
|
|
4852
4877
|
import { serve as serveInngest } from "inngest/hono";
|
|
4853
4878
|
|
|
4854
4879
|
// src/inngest/client.ts
|
|
@@ -10221,69 +10246,6 @@ import { mkdir as mkdir2, writeFile } from "fs/promises";
|
|
|
10221
10246
|
import { join as join4 } from "path";
|
|
10222
10247
|
import { z as z15 } from "zod";
|
|
10223
10248
|
|
|
10224
|
-
// src/directory/csv.ts
|
|
10225
|
-
function parseCsv(text) {
|
|
10226
|
-
const rows = [];
|
|
10227
|
-
let row = [];
|
|
10228
|
-
let field = "";
|
|
10229
|
-
let quoted = false;
|
|
10230
|
-
for (let i = 0; i < text.length; i += 1) {
|
|
10231
|
-
const ch = text[i];
|
|
10232
|
-
const next = text[i + 1];
|
|
10233
|
-
if (quoted) {
|
|
10234
|
-
if (ch === '"' && next === '"') {
|
|
10235
|
-
field += '"';
|
|
10236
|
-
i += 1;
|
|
10237
|
-
} else if (ch === '"') {
|
|
10238
|
-
quoted = false;
|
|
10239
|
-
} else {
|
|
10240
|
-
field += ch;
|
|
10241
|
-
}
|
|
10242
|
-
continue;
|
|
10243
|
-
}
|
|
10244
|
-
if (ch === '"') {
|
|
10245
|
-
quoted = true;
|
|
10246
|
-
} else if (ch === ",") {
|
|
10247
|
-
row.push(field);
|
|
10248
|
-
field = "";
|
|
10249
|
-
} else if (ch === "\n") {
|
|
10250
|
-
row.push(field);
|
|
10251
|
-
rows.push(row);
|
|
10252
|
-
row = [];
|
|
10253
|
-
field = "";
|
|
10254
|
-
} else if (ch !== "\r") {
|
|
10255
|
-
field += ch;
|
|
10256
|
-
}
|
|
10257
|
-
}
|
|
10258
|
-
if (field.length > 0 || row.length > 0) {
|
|
10259
|
-
row.push(field);
|
|
10260
|
-
rows.push(row);
|
|
10261
|
-
}
|
|
10262
|
-
return rows;
|
|
10263
|
-
}
|
|
10264
|
-
function csvRecords(text) {
|
|
10265
|
-
const rows = parseCsv(text).filter((row) => row.some((cell) => cell.trim() !== ""));
|
|
10266
|
-
const header = rows[0]?.map((cell) => cell.trim()) ?? [];
|
|
10267
|
-
return rows.slice(1).map((row) => {
|
|
10268
|
-
const record = {};
|
|
10269
|
-
for (let i = 0; i < header.length; i += 1) {
|
|
10270
|
-
record[header[i]] = row[i] ?? "";
|
|
10271
|
-
}
|
|
10272
|
-
return record;
|
|
10273
|
-
});
|
|
10274
|
-
}
|
|
10275
|
-
function csvCell(value) {
|
|
10276
|
-
if (value === null || value === void 0) return "";
|
|
10277
|
-
const text = String(value);
|
|
10278
|
-
return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
|
|
10279
|
-
}
|
|
10280
|
-
function rowsToCsv(headers, rows) {
|
|
10281
|
-
return [
|
|
10282
|
-
headers.join(","),
|
|
10283
|
-
...rows.map((row) => headers.map((header) => csvCell(row[header])).join(","))
|
|
10284
|
-
].join("\n") + "\n";
|
|
10285
|
-
}
|
|
10286
|
-
|
|
10287
10249
|
// src/directory/location-db.ts
|
|
10288
10250
|
import { access, readFile } from "fs/promises";
|
|
10289
10251
|
var POPULATION_YEARS = [2020, 2021, 2022, 2023, 2024, 2025];
|
|
@@ -10562,6 +10524,7 @@ function csvRowsFor(result) {
|
|
|
10562
10524
|
result_position: null,
|
|
10563
10525
|
business_name: null,
|
|
10564
10526
|
review_stars: null,
|
|
10527
|
+
review_count: null,
|
|
10565
10528
|
category: null,
|
|
10566
10529
|
address: null,
|
|
10567
10530
|
phone: null,
|
|
@@ -10595,6 +10558,7 @@ function csvRowsFor(result) {
|
|
|
10595
10558
|
result_position: business.position,
|
|
10596
10559
|
business_name: business.name,
|
|
10597
10560
|
review_stars: business.rating,
|
|
10561
|
+
review_count: business.reviewCount,
|
|
10598
10562
|
category: business.category,
|
|
10599
10563
|
address: business.address,
|
|
10600
10564
|
phone: business.phone,
|
|
@@ -10635,6 +10599,7 @@ async function saveDirectoryCsv(result) {
|
|
|
10635
10599
|
"result_position",
|
|
10636
10600
|
"business_name",
|
|
10637
10601
|
"review_stars",
|
|
10602
|
+
"review_count",
|
|
10638
10603
|
"category",
|
|
10639
10604
|
"address",
|
|
10640
10605
|
"phone",
|
|
@@ -10737,8 +10702,360 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
|
10737
10702
|
}
|
|
10738
10703
|
});
|
|
10739
10704
|
|
|
10740
|
-
// src/api/
|
|
10705
|
+
// src/api/workflow-routes.ts
|
|
10706
|
+
import { createHmac } from "crypto";
|
|
10707
|
+
import { readFile as readFile2 } from "fs/promises";
|
|
10741
10708
|
import { Hono as Hono7 } from "hono";
|
|
10709
|
+
import { z as z16 } from "zod";
|
|
10710
|
+
var workflowApp = new Hono7();
|
|
10711
|
+
var WorkflowInputSchema = z16.record(z16.unknown()).default({});
|
|
10712
|
+
var WorkflowIdSchema = z16.string().min(1);
|
|
10713
|
+
var CadenceSchema = z16.enum(["daily", "weekly", "monthly"]);
|
|
10714
|
+
var ScheduleStatusSchema = z16.enum(["active", "paused"]);
|
|
10715
|
+
var RunBodySchema = z16.object({
|
|
10716
|
+
workflowId: WorkflowIdSchema,
|
|
10717
|
+
input: WorkflowInputSchema,
|
|
10718
|
+
webhookUrl: z16.string().url().optional()
|
|
10719
|
+
});
|
|
10720
|
+
var ScheduleCreateSchema = z16.object({
|
|
10721
|
+
workflowId: WorkflowIdSchema,
|
|
10722
|
+
name: z16.string().min(1).max(120).optional(),
|
|
10723
|
+
input: WorkflowInputSchema,
|
|
10724
|
+
cadence: CadenceSchema.default("weekly"),
|
|
10725
|
+
timezone: z16.string().min(1).max(64).default("UTC"),
|
|
10726
|
+
webhookUrl: z16.string().url().optional(),
|
|
10727
|
+
nextRunAt: z16.string().datetime().optional()
|
|
10728
|
+
});
|
|
10729
|
+
var SchedulePatchSchema = z16.object({
|
|
10730
|
+
name: z16.string().min(1).max(120).optional(),
|
|
10731
|
+
status: ScheduleStatusSchema.optional(),
|
|
10732
|
+
input: WorkflowInputSchema.optional(),
|
|
10733
|
+
cadence: CadenceSchema.optional(),
|
|
10734
|
+
timezone: z16.string().min(1).max(64).optional(),
|
|
10735
|
+
webhookUrl: z16.string().url().nullable().optional(),
|
|
10736
|
+
nextRunAt: z16.string().datetime().nullable().optional()
|
|
10737
|
+
});
|
|
10738
|
+
function hostedWorkflowOutputDir() {
|
|
10739
|
+
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
10740
|
+
}
|
|
10741
|
+
function originFromUrl(url) {
|
|
10742
|
+
const parsed = new URL(url);
|
|
10743
|
+
return `${parsed.protocol}//${parsed.host}`;
|
|
10744
|
+
}
|
|
10745
|
+
function addCadence(fromIso, cadence) {
|
|
10746
|
+
const next = new Date(fromIso);
|
|
10747
|
+
if (!Number.isFinite(next.getTime())) return new Date(Date.now() + 24 * 60 * 60 * 1e3).toISOString();
|
|
10748
|
+
if (cadence === "daily") next.setUTCDate(next.getUTCDate() + 1);
|
|
10749
|
+
else if (cadence === "weekly") next.setUTCDate(next.getUTCDate() + 7);
|
|
10750
|
+
else next.setUTCMonth(next.getUTCMonth() + 1);
|
|
10751
|
+
return next.toISOString();
|
|
10752
|
+
}
|
|
10753
|
+
function defaultNextRunAt(cadence) {
|
|
10754
|
+
return addCadence((/* @__PURE__ */ new Date()).toISOString(), cadence);
|
|
10755
|
+
}
|
|
10756
|
+
function validateWorkflowInput(workflowId, input) {
|
|
10757
|
+
const definition = workflowDefinition(workflowId);
|
|
10758
|
+
return definition.inputSchema.parse(input);
|
|
10759
|
+
}
|
|
10760
|
+
async function validateWebhookUrl(url) {
|
|
10761
|
+
if (!url) return null;
|
|
10762
|
+
const result = await validatePublicHttpUrl(url, { field: "webhookUrl", requireHttps: true });
|
|
10763
|
+
if (result.error) throw new Error(result.error);
|
|
10764
|
+
return url;
|
|
10765
|
+
}
|
|
10766
|
+
function runDownloadUrl(c, runId, artifactId) {
|
|
10767
|
+
return `${originFromUrl(c.req.url)}/workflows/runs/${runId}/artifacts/${artifactId}`;
|
|
10768
|
+
}
|
|
10769
|
+
function exposeRun(c, run, artifacts = []) {
|
|
10770
|
+
return {
|
|
10771
|
+
...run,
|
|
10772
|
+
artifacts: artifacts.map((artifact) => ({
|
|
10773
|
+
...artifact,
|
|
10774
|
+
downloadUrl: runDownloadUrl(c, run.id, artifact.id)
|
|
10775
|
+
}))
|
|
10776
|
+
};
|
|
10777
|
+
}
|
|
10778
|
+
async function persistRunArtifacts(runId, userId, manifest) {
|
|
10779
|
+
if (!manifest) return [];
|
|
10780
|
+
return replaceWorkflowArtifacts(runId, userId, manifest.artifacts);
|
|
10781
|
+
}
|
|
10782
|
+
async function readManifestFromSummary(summary) {
|
|
10783
|
+
if (!summary.reportPath) return null;
|
|
10784
|
+
const manifestPath = summary.reportPath.replace(/report\.html$/, "manifest.json");
|
|
10785
|
+
try {
|
|
10786
|
+
return JSON.parse(await readFile2(manifestPath, "utf8"));
|
|
10787
|
+
} catch {
|
|
10788
|
+
return null;
|
|
10789
|
+
}
|
|
10790
|
+
}
|
|
10791
|
+
function webhookSignature(body, timestamp) {
|
|
10792
|
+
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
10793
|
+
if (!secret2) return null;
|
|
10794
|
+
return createHmac("sha256", secret2).update(`${timestamp}.${body}`).digest("hex");
|
|
10795
|
+
}
|
|
10796
|
+
async function deliverWorkflowWebhook(input) {
|
|
10797
|
+
if (!input.webhookUrl) return;
|
|
10798
|
+
const payload = {
|
|
10799
|
+
event: input.run.status === "failed" ? "workflow.run.failed" : input.run.status === "partial" ? "workflow.run.partial" : "workflow.run.succeeded",
|
|
10800
|
+
runId: input.run.id,
|
|
10801
|
+
scheduleId: input.scheduleId,
|
|
10802
|
+
workflowId: input.run.workflow_id,
|
|
10803
|
+
status: input.run.status,
|
|
10804
|
+
completedAt: input.run.completed_at ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
10805
|
+
summary: {
|
|
10806
|
+
title: input.summary?.title ?? input.run.workflow_id,
|
|
10807
|
+
counts: input.summary?.counts ?? {},
|
|
10808
|
+
warnings: input.summary?.warnings ?? [],
|
|
10809
|
+
errors: input.summary?.errors ?? (input.run.error_message ? [input.run.error_message] : [])
|
|
10810
|
+
},
|
|
10811
|
+
artifacts: input.artifacts.map((artifact) => ({
|
|
10812
|
+
kind: artifact.kind,
|
|
10813
|
+
label: artifact.label,
|
|
10814
|
+
path: artifact.path
|
|
10815
|
+
}))
|
|
10816
|
+
};
|
|
10817
|
+
const body = JSON.stringify(payload);
|
|
10818
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
10819
|
+
const headers = {
|
|
10820
|
+
"Content-Type": "application/json",
|
|
10821
|
+
"x-mcp-scraper-timestamp": timestamp
|
|
10822
|
+
};
|
|
10823
|
+
const signature = webhookSignature(body, timestamp);
|
|
10824
|
+
if (signature) headers["x-mcp-scraper-signature"] = signature;
|
|
10825
|
+
let lastError = null;
|
|
10826
|
+
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
10827
|
+
try {
|
|
10828
|
+
const res = await fetch(input.webhookUrl, { method: "POST", headers, body, signal: AbortSignal.timeout(15e3) });
|
|
10829
|
+
if (res.ok) {
|
|
10830
|
+
await recordWorkflowWebhookDelivery({ runId: input.run.id, scheduleId: input.scheduleId, userId: input.userId, webhookUrl: input.webhookUrl, status: "delivered", attempts: attempt });
|
|
10831
|
+
return;
|
|
10832
|
+
}
|
|
10833
|
+
lastError = `HTTP ${res.status}`;
|
|
10834
|
+
} catch (err) {
|
|
10835
|
+
lastError = err instanceof Error ? err.message : String(err);
|
|
10836
|
+
}
|
|
10837
|
+
await new Promise((resolve) => setTimeout(resolve, attempt * 500));
|
|
10838
|
+
}
|
|
10839
|
+
await recordWorkflowWebhookDelivery({ runId: input.run.id, scheduleId: input.scheduleId, userId: input.userId, webhookUrl: input.webhookUrl, status: "failed", attempts: 3, lastError });
|
|
10840
|
+
}
|
|
10841
|
+
async function executeWorkflowRun(input) {
|
|
10842
|
+
await markWorkflowRunRunning(input.runId);
|
|
10843
|
+
try {
|
|
10844
|
+
const summary = await runWorkflow(input.workflowId, input.workflowInput, {
|
|
10845
|
+
apiKey: input.user.api_key,
|
|
10846
|
+
apiUrl: input.apiUrl,
|
|
10847
|
+
outputDir: hostedWorkflowOutputDir(),
|
|
10848
|
+
runId: input.runId
|
|
10849
|
+
});
|
|
10850
|
+
const manifest = await readManifestFromSummary(summary);
|
|
10851
|
+
await completeWorkflowRunRecord(input.runId, summary.status, manifest ?? { summary });
|
|
10852
|
+
const run = await getWorkflowRun(input.runId, input.user.id);
|
|
10853
|
+
if (!run) throw new Error("Workflow run disappeared after execution");
|
|
10854
|
+
const artifacts = await persistRunArtifacts(input.runId, input.user.id, manifest);
|
|
10855
|
+
await deliverWorkflowWebhook({ userId: input.user.id, run, scheduleId: input.scheduleId ?? null, webhookUrl: input.webhookUrl ?? null, summary, artifacts });
|
|
10856
|
+
return { run, summary, artifacts };
|
|
10857
|
+
} catch (err) {
|
|
10858
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
10859
|
+
await failWorkflowRunRecord(input.runId, message, { error: message });
|
|
10860
|
+
const run = await getWorkflowRun(input.runId, input.user.id);
|
|
10861
|
+
if (!run) throw err;
|
|
10862
|
+
await deliverWorkflowWebhook({ userId: input.user.id, run, scheduleId: input.scheduleId ?? null, webhookUrl: input.webhookUrl ?? null, summary: null, artifacts: [] });
|
|
10863
|
+
throw err;
|
|
10864
|
+
}
|
|
10865
|
+
}
|
|
10866
|
+
workflowApp.get("/definitions", createApiKeyAuth(), (c) => {
|
|
10867
|
+
return c.json({ workflows: listWorkflowDefinitions() });
|
|
10868
|
+
});
|
|
10869
|
+
workflowApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
10870
|
+
const user = c.get("user");
|
|
10871
|
+
const parsed = RunBodySchema.safeParse(await c.req.json().catch(() => ({})));
|
|
10872
|
+
if (!parsed.success) return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
10873
|
+
let input;
|
|
10874
|
+
let webhookUrl;
|
|
10875
|
+
try {
|
|
10876
|
+
input = validateWorkflowInput(parsed.data.workflowId, parsed.data.input);
|
|
10877
|
+
webhookUrl = await validateWebhookUrl(parsed.data.webhookUrl);
|
|
10878
|
+
} catch (err) {
|
|
10879
|
+
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
10880
|
+
}
|
|
10881
|
+
const run = await createWorkflowRun({ userId: user.id, workflowId: parsed.data.workflowId, workflowInput: input });
|
|
10882
|
+
try {
|
|
10883
|
+
const executed = await executeWorkflowRun({
|
|
10884
|
+
runId: run.id,
|
|
10885
|
+
user,
|
|
10886
|
+
workflowId: parsed.data.workflowId,
|
|
10887
|
+
workflowInput: input,
|
|
10888
|
+
apiUrl: originFromUrl(c.req.url),
|
|
10889
|
+
webhookUrl
|
|
10890
|
+
});
|
|
10891
|
+
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
10892
|
+
} catch (err) {
|
|
10893
|
+
const failed = await getWorkflowRun(run.id, user.id);
|
|
10894
|
+
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
10895
|
+
}
|
|
10896
|
+
});
|
|
10897
|
+
workflowApp.get("/runs", createApiKeyAuth(), async (c) => {
|
|
10898
|
+
const user = c.get("user");
|
|
10899
|
+
const runs = await listWorkflowRuns(user.id);
|
|
10900
|
+
return c.json({ runs });
|
|
10901
|
+
});
|
|
10902
|
+
workflowApp.get("/runs/:id", createApiKeyAuth(), async (c) => {
|
|
10903
|
+
const user = c.get("user");
|
|
10904
|
+
const run = await getWorkflowRun(c.req.param("id"), user.id);
|
|
10905
|
+
if (!run) return c.json({ error: "Run not found" }, 404);
|
|
10906
|
+
const artifacts = await listWorkflowArtifacts(run.id, user.id);
|
|
10907
|
+
return c.json({ run: exposeRun(c, run, artifacts) });
|
|
10908
|
+
});
|
|
10909
|
+
workflowApp.get("/runs/:id/artifacts/:artifactId", createApiKeyAuth(), async (c) => {
|
|
10910
|
+
const user = c.get("user");
|
|
10911
|
+
const run = await getWorkflowRun(c.req.param("id"), user.id);
|
|
10912
|
+
if (!run) return c.json({ error: "Run not found" }, 404);
|
|
10913
|
+
const artifact = await getWorkflowArtifact(c.req.param("artifactId"), user.id);
|
|
10914
|
+
if (!artifact || artifact.run_id !== run.id) return c.json({ error: "Artifact not found" }, 404);
|
|
10915
|
+
try {
|
|
10916
|
+
const content = await readFile2(artifact.path);
|
|
10917
|
+
return new Response(content, {
|
|
10918
|
+
headers: {
|
|
10919
|
+
"Content-Type": artifact.content_type,
|
|
10920
|
+
"Content-Disposition": `attachment; filename="${artifact.path.split("/").pop() || artifact.id}"`
|
|
10921
|
+
}
|
|
10922
|
+
});
|
|
10923
|
+
} catch {
|
|
10924
|
+
return c.json({ error: "Artifact file is no longer available" }, 410);
|
|
10925
|
+
}
|
|
10926
|
+
});
|
|
10927
|
+
workflowApp.post("/schedules", createApiKeyAuth(), async (c) => {
|
|
10928
|
+
const user = c.get("user");
|
|
10929
|
+
const parsed = ScheduleCreateSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
10930
|
+
if (!parsed.success) return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
10931
|
+
let input;
|
|
10932
|
+
let webhookUrl;
|
|
10933
|
+
try {
|
|
10934
|
+
input = validateWorkflowInput(parsed.data.workflowId, parsed.data.input);
|
|
10935
|
+
webhookUrl = await validateWebhookUrl(parsed.data.webhookUrl);
|
|
10936
|
+
} catch (err) {
|
|
10937
|
+
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
10938
|
+
}
|
|
10939
|
+
const schedule = await createWorkflowSchedule({
|
|
10940
|
+
userId: user.id,
|
|
10941
|
+
workflowId: parsed.data.workflowId,
|
|
10942
|
+
name: parsed.data.name ?? workflowDefinition(parsed.data.workflowId).title,
|
|
10943
|
+
workflowInput: input,
|
|
10944
|
+
cadence: parsed.data.cadence,
|
|
10945
|
+
timezone: parsed.data.timezone,
|
|
10946
|
+
webhookUrl,
|
|
10947
|
+
nextRunAt: parsed.data.nextRunAt ?? defaultNextRunAt(parsed.data.cadence)
|
|
10948
|
+
});
|
|
10949
|
+
return c.json({ schedule }, 201);
|
|
10950
|
+
});
|
|
10951
|
+
workflowApp.get("/schedules", createApiKeyAuth(), async (c) => {
|
|
10952
|
+
const user = c.get("user");
|
|
10953
|
+
return c.json({ schedules: await listWorkflowSchedules(user.id) });
|
|
10954
|
+
});
|
|
10955
|
+
workflowApp.patch("/schedules/:id", createApiKeyAuth(), async (c) => {
|
|
10956
|
+
const user = c.get("user");
|
|
10957
|
+
const existing = await getWorkflowSchedule(c.req.param("id"), user.id);
|
|
10958
|
+
if (!existing) return c.json({ error: "Schedule not found" }, 404);
|
|
10959
|
+
const parsed = SchedulePatchSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
10960
|
+
if (!parsed.success) return c.json({ error: parsed.error.issues[0]?.message ?? "Invalid request" }, 400);
|
|
10961
|
+
let nextInput;
|
|
10962
|
+
let webhookUrl;
|
|
10963
|
+
try {
|
|
10964
|
+
nextInput = parsed.data.input ? validateWorkflowInput(existing.workflow_id, parsed.data.input) : void 0;
|
|
10965
|
+
webhookUrl = parsed.data.webhookUrl === void 0 ? void 0 : await validateWebhookUrl(parsed.data.webhookUrl);
|
|
10966
|
+
} catch (err) {
|
|
10967
|
+
return c.json({ error: err instanceof Error ? err.message : String(err) }, 400);
|
|
10968
|
+
}
|
|
10969
|
+
const schedule = await patchWorkflowSchedule({
|
|
10970
|
+
id: existing.id,
|
|
10971
|
+
userId: user.id,
|
|
10972
|
+
name: parsed.data.name,
|
|
10973
|
+
status: parsed.data.status,
|
|
10974
|
+
workflowInput: nextInput,
|
|
10975
|
+
cadence: parsed.data.cadence,
|
|
10976
|
+
timezone: parsed.data.timezone,
|
|
10977
|
+
webhookUrl,
|
|
10978
|
+
nextRunAt: parsed.data.nextRunAt
|
|
10979
|
+
});
|
|
10980
|
+
return c.json({ schedule });
|
|
10981
|
+
});
|
|
10982
|
+
workflowApp.delete("/schedules/:id", createApiKeyAuth(), async (c) => {
|
|
10983
|
+
const user = c.get("user");
|
|
10984
|
+
const deleted = await deleteWorkflowSchedule(c.req.param("id"), user.id);
|
|
10985
|
+
if (!deleted) return c.json({ error: "Schedule not found" }, 404);
|
|
10986
|
+
return c.json({ ok: true });
|
|
10987
|
+
});
|
|
10988
|
+
workflowApp.post("/schedules/:id/run", createApiKeyAuth(), async (c) => {
|
|
10989
|
+
const user = c.get("user");
|
|
10990
|
+
const schedule = await getWorkflowSchedule(c.req.param("id"), user.id);
|
|
10991
|
+
if (!schedule) return c.json({ error: "Schedule not found" }, 404);
|
|
10992
|
+
const run = await createWorkflowRun({ userId: user.id, scheduleId: schedule.id, workflowId: schedule.workflow_id, workflowInput: schedule.input });
|
|
10993
|
+
try {
|
|
10994
|
+
const executed = await executeWorkflowRun({
|
|
10995
|
+
runId: run.id,
|
|
10996
|
+
user,
|
|
10997
|
+
workflowId: schedule.workflow_id,
|
|
10998
|
+
workflowInput: schedule.input,
|
|
10999
|
+
apiUrl: originFromUrl(c.req.url),
|
|
11000
|
+
scheduleId: schedule.id,
|
|
11001
|
+
webhookUrl: schedule.webhook_url
|
|
11002
|
+
});
|
|
11003
|
+
return c.json({ run: exposeRun(c, executed.run, executed.artifacts), summary: executed.summary });
|
|
11004
|
+
} catch (err) {
|
|
11005
|
+
const failed = await getWorkflowRun(run.id, user.id);
|
|
11006
|
+
return c.json({ run: failed, error: err instanceof Error ? err.message : String(err) }, 500);
|
|
11007
|
+
}
|
|
11008
|
+
});
|
|
11009
|
+
async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
11010
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11011
|
+
const due = await listDueWorkflowSchedules(now, limit);
|
|
11012
|
+
const results = [];
|
|
11013
|
+
for (const schedule of due) {
|
|
11014
|
+
const user = await getUserById(schedule.user_id);
|
|
11015
|
+
if (!user) {
|
|
11016
|
+
results.push({ scheduleId: schedule.id, status: "failed", error: "User not found" });
|
|
11017
|
+
continue;
|
|
11018
|
+
}
|
|
11019
|
+
const scheduledFor = schedule.next_run_at ?? now;
|
|
11020
|
+
const run = await createWorkflowRun({
|
|
11021
|
+
userId: user.id,
|
|
11022
|
+
scheduleId: schedule.id,
|
|
11023
|
+
workflowId: schedule.workflow_id,
|
|
11024
|
+
workflowInput: schedule.input,
|
|
11025
|
+
idempotencyKey: `${schedule.id}:${scheduledFor}`
|
|
11026
|
+
});
|
|
11027
|
+
await markWorkflowScheduleRan(schedule.id, now, addCadence(scheduledFor, schedule.cadence));
|
|
11028
|
+
if (run.status !== "queued") {
|
|
11029
|
+
results.push({ scheduleId: schedule.id, runId: run.id, status: run.status });
|
|
11030
|
+
continue;
|
|
11031
|
+
}
|
|
11032
|
+
try {
|
|
11033
|
+
const executed = await executeWorkflowRun({
|
|
11034
|
+
runId: run.id,
|
|
11035
|
+
user,
|
|
11036
|
+
workflowId: schedule.workflow_id,
|
|
11037
|
+
workflowInput: schedule.input,
|
|
11038
|
+
apiUrl,
|
|
11039
|
+
scheduleId: schedule.id,
|
|
11040
|
+
webhookUrl: schedule.webhook_url
|
|
11041
|
+
});
|
|
11042
|
+
results.push({ scheduleId: schedule.id, runId: executed.run.id, status: executed.run.status });
|
|
11043
|
+
} catch (err) {
|
|
11044
|
+
results.push({ scheduleId: schedule.id, runId: run.id, status: "failed", error: err instanceof Error ? err.message : String(err) });
|
|
11045
|
+
}
|
|
11046
|
+
}
|
|
11047
|
+
return { dispatched: results.length, results };
|
|
11048
|
+
}
|
|
11049
|
+
workflowApp.post("/cron/dispatch", async (c) => {
|
|
11050
|
+
const secret2 = c.req.header("authorization");
|
|
11051
|
+
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
11052
|
+
return c.json({ error: "Unauthorized" }, 401);
|
|
11053
|
+
}
|
|
11054
|
+
return c.json(await dispatchDueWorkflowSchedules(originFromUrl(c.req.url)));
|
|
11055
|
+
});
|
|
11056
|
+
|
|
11057
|
+
// src/api/serp-intelligence-routes.ts
|
|
11058
|
+
import { Hono as Hono8 } from "hono";
|
|
10742
11059
|
|
|
10743
11060
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
10744
11061
|
import { createHash } from "crypto";
|
|
@@ -11050,7 +11367,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
11050
11367
|
}
|
|
11051
11368
|
|
|
11052
11369
|
// src/serp-intelligence/schemas.ts
|
|
11053
|
-
import { z as
|
|
11370
|
+
import { z as z17 } from "zod";
|
|
11054
11371
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
11055
11372
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
11056
11373
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -11112,171 +11429,171 @@ function isPublicHttpUrl(value) {
|
|
|
11112
11429
|
return false;
|
|
11113
11430
|
}
|
|
11114
11431
|
}
|
|
11115
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
11116
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
11117
|
-
query:
|
|
11118
|
-
location:
|
|
11119
|
-
gl:
|
|
11120
|
-
hl:
|
|
11121
|
-
device:
|
|
11122
|
-
proxyMode:
|
|
11123
|
-
proxyZip:
|
|
11124
|
-
pages:
|
|
11125
|
-
debug:
|
|
11126
|
-
includePageSnapshots:
|
|
11127
|
-
pageSnapshotLimit:
|
|
11432
|
+
var SerpIntelligencePublicHttpUrlSchema = z17.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
11433
|
+
var SerpIntelligenceCaptureBodySchema = z17.object({
|
|
11434
|
+
query: z17.string().trim().min(1, "query is required"),
|
|
11435
|
+
location: z17.string().trim().min(1).optional(),
|
|
11436
|
+
gl: z17.string().trim().length(2).default("us"),
|
|
11437
|
+
hl: z17.string().trim().length(2).default("en"),
|
|
11438
|
+
device: z17.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
11439
|
+
proxyMode: z17.enum(SerpIntelligenceProxyModeValues).default("location"),
|
|
11440
|
+
proxyZip: z17.string().regex(/^\d{5}$/).optional(),
|
|
11441
|
+
pages: z17.number().int().min(1).max(2).default(1),
|
|
11442
|
+
debug: z17.boolean().default(false),
|
|
11443
|
+
includePageSnapshots: z17.boolean().default(false),
|
|
11444
|
+
pageSnapshotLimit: z17.number().int().min(0).max(10).default(0)
|
|
11128
11445
|
}).strict();
|
|
11129
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
11446
|
+
var SerpIntelligencePageSnapshotRequestSchema = z17.object({
|
|
11130
11447
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
11131
|
-
sourceKind:
|
|
11132
|
-
sourcePosition:
|
|
11448
|
+
sourceKind: z17.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
11449
|
+
sourcePosition: z17.number().int().min(1).optional()
|
|
11133
11450
|
}).strict();
|
|
11134
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
11135
|
-
urls:
|
|
11136
|
-
targets:
|
|
11137
|
-
maxConcurrency:
|
|
11138
|
-
timeoutMs:
|
|
11139
|
-
debug:
|
|
11451
|
+
var SerpIntelligencePageSnapshotsBodySchema = z17.object({
|
|
11452
|
+
urls: z17.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
11453
|
+
targets: z17.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
11454
|
+
maxConcurrency: z17.number().int().min(1).max(5).default(2),
|
|
11455
|
+
timeoutMs: z17.number().int().min(1e3).max(6e4).default(15e3),
|
|
11456
|
+
debug: z17.boolean().default(false)
|
|
11140
11457
|
}).strict();
|
|
11141
|
-
var SerpIntelligenceAICitationSchema =
|
|
11142
|
-
text:
|
|
11143
|
-
href:
|
|
11458
|
+
var SerpIntelligenceAICitationSchema = z17.object({
|
|
11459
|
+
text: z17.string(),
|
|
11460
|
+
href: z17.string()
|
|
11144
11461
|
}).strict();
|
|
11145
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
11146
|
-
position:
|
|
11147
|
-
title:
|
|
11148
|
-
url:
|
|
11149
|
-
domain:
|
|
11150
|
-
cite:
|
|
11151
|
-
snippet:
|
|
11152
|
-
isRedditStyle:
|
|
11153
|
-
inlineRating:
|
|
11154
|
-
value:
|
|
11155
|
-
count:
|
|
11462
|
+
var SerpIntelligenceOrganicResultSchema = z17.object({
|
|
11463
|
+
position: z17.number().int().min(1),
|
|
11464
|
+
title: z17.string(),
|
|
11465
|
+
url: z17.string(),
|
|
11466
|
+
domain: z17.string(),
|
|
11467
|
+
cite: z17.string().nullable(),
|
|
11468
|
+
snippet: z17.string().nullable(),
|
|
11469
|
+
isRedditStyle: z17.boolean(),
|
|
11470
|
+
inlineRating: z17.object({
|
|
11471
|
+
value: z17.string(),
|
|
11472
|
+
count: z17.string()
|
|
11156
11473
|
}).strict().nullable()
|
|
11157
11474
|
}).strict();
|
|
11158
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
11159
|
-
status:
|
|
11160
|
-
expected:
|
|
11161
|
-
city:
|
|
11162
|
-
regionCode:
|
|
11163
|
-
canonicalLocation:
|
|
11475
|
+
var SerpIntelligenceLocationEvidenceSchema = z17.object({
|
|
11476
|
+
status: z17.enum(SerpIntelligenceLocalizationStatusValues),
|
|
11477
|
+
expected: z17.object({
|
|
11478
|
+
city: z17.string(),
|
|
11479
|
+
regionCode: z17.string().nullable(),
|
|
11480
|
+
canonicalLocation: z17.string()
|
|
11164
11481
|
}).strict().nullable(),
|
|
11165
|
-
candidates:
|
|
11166
|
-
city:
|
|
11167
|
-
regionCode:
|
|
11168
|
-
count:
|
|
11169
|
-
examples:
|
|
11482
|
+
candidates: z17.array(z17.object({
|
|
11483
|
+
city: z17.string(),
|
|
11484
|
+
regionCode: z17.string(),
|
|
11485
|
+
count: z17.number().int().min(0),
|
|
11486
|
+
examples: z17.array(z17.string())
|
|
11170
11487
|
}).strict())
|
|
11171
11488
|
}).strict();
|
|
11172
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
11173
|
-
seed:
|
|
11174
|
-
location:
|
|
11175
|
-
extractedAt:
|
|
11176
|
-
totalQuestions:
|
|
11177
|
-
surface:
|
|
11178
|
-
aiOverview:
|
|
11179
|
-
detected:
|
|
11180
|
-
text:
|
|
11181
|
-
citations:
|
|
11182
|
-
expanded:
|
|
11183
|
-
fullyExpanded:
|
|
11184
|
-
sections:
|
|
11489
|
+
var SerpIntelligenceHarvestResultSchema = z17.object({
|
|
11490
|
+
seed: z17.string(),
|
|
11491
|
+
location: z17.string().nullable(),
|
|
11492
|
+
extractedAt: z17.string(),
|
|
11493
|
+
totalQuestions: z17.number().int().min(0),
|
|
11494
|
+
surface: z17.enum(["web", "aim", "unknown"]),
|
|
11495
|
+
aiOverview: z17.object({
|
|
11496
|
+
detected: z17.boolean(),
|
|
11497
|
+
text: z17.string().nullable(),
|
|
11498
|
+
citations: z17.array(SerpIntelligenceAICitationSchema),
|
|
11499
|
+
expanded: z17.boolean().optional(),
|
|
11500
|
+
fullyExpanded: z17.boolean().optional(),
|
|
11501
|
+
sections: z17.array(z17.string()).optional()
|
|
11185
11502
|
}).strict(),
|
|
11186
|
-
aiMode:
|
|
11187
|
-
detected:
|
|
11188
|
-
text:
|
|
11189
|
-
citations:
|
|
11503
|
+
aiMode: z17.object({
|
|
11504
|
+
detected: z17.boolean(),
|
|
11505
|
+
text: z17.string().nullable(),
|
|
11506
|
+
citations: z17.array(SerpIntelligenceAICitationSchema)
|
|
11190
11507
|
}).strict(),
|
|
11191
|
-
tree:
|
|
11192
|
-
flat:
|
|
11193
|
-
videos:
|
|
11194
|
-
forums:
|
|
11195
|
-
organicResults:
|
|
11196
|
-
localPack:
|
|
11197
|
-
entityIds:
|
|
11198
|
-
entities:
|
|
11199
|
-
name:
|
|
11200
|
-
kgId:
|
|
11201
|
-
cid:
|
|
11202
|
-
gcid:
|
|
11508
|
+
tree: z17.array(z17.unknown()),
|
|
11509
|
+
flat: z17.array(z17.unknown()),
|
|
11510
|
+
videos: z17.array(z17.unknown()),
|
|
11511
|
+
forums: z17.array(z17.unknown()),
|
|
11512
|
+
organicResults: z17.array(SerpIntelligenceOrganicResultSchema),
|
|
11513
|
+
localPack: z17.array(z17.unknown()),
|
|
11514
|
+
entityIds: z17.object({
|
|
11515
|
+
entities: z17.array(z17.object({
|
|
11516
|
+
name: z17.string(),
|
|
11517
|
+
kgId: z17.string().nullable(),
|
|
11518
|
+
cid: z17.string().nullable(),
|
|
11519
|
+
gcid: z17.string().nullable()
|
|
11203
11520
|
}).strict()),
|
|
11204
|
-
kgIds:
|
|
11205
|
-
cids:
|
|
11206
|
-
gcids:
|
|
11521
|
+
kgIds: z17.array(z17.string()),
|
|
11522
|
+
cids: z17.array(z17.string()),
|
|
11523
|
+
gcids: z17.array(z17.string())
|
|
11207
11524
|
}).strict(),
|
|
11208
|
-
stats:
|
|
11209
|
-
seed:
|
|
11210
|
-
totalQuestions:
|
|
11211
|
-
maxDepthReached:
|
|
11212
|
-
durationMs:
|
|
11213
|
-
errorCount:
|
|
11525
|
+
stats: z17.object({
|
|
11526
|
+
seed: z17.string(),
|
|
11527
|
+
totalQuestions: z17.number().int().min(0),
|
|
11528
|
+
maxDepthReached: z17.number().int().min(0),
|
|
11529
|
+
durationMs: z17.number().min(0),
|
|
11530
|
+
errorCount: z17.number().int().min(0)
|
|
11214
11531
|
}).strict(),
|
|
11215
|
-
diagnostics:
|
|
11216
|
-
completionStatus:
|
|
11217
|
-
problem:
|
|
11218
|
-
warnings:
|
|
11219
|
-
debug:
|
|
11532
|
+
diagnostics: z17.object({
|
|
11533
|
+
completionStatus: z17.enum(["paa_found", "no_paa", "serp_only"]),
|
|
11534
|
+
problem: z17.null(),
|
|
11535
|
+
warnings: z17.array(z17.unknown()).optional(),
|
|
11536
|
+
debug: z17.object({
|
|
11220
11537
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
11221
11538
|
}).passthrough().optional()
|
|
11222
11539
|
}).passthrough(),
|
|
11223
|
-
whatPeopleSaying:
|
|
11540
|
+
whatPeopleSaying: z17.array(z17.unknown())
|
|
11224
11541
|
}).strict();
|
|
11225
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
11226
|
-
attemptNumber:
|
|
11227
|
-
outcome:
|
|
11228
|
-
startedAt:
|
|
11229
|
-
completedAt:
|
|
11230
|
-
durationMs:
|
|
11231
|
-
problemCode:
|
|
11232
|
-
message:
|
|
11233
|
-
kernelSessionId:
|
|
11234
|
-
cleanupSucceeded:
|
|
11542
|
+
var SerpIntelligenceCaptureAttemptSchema = z17.object({
|
|
11543
|
+
attemptNumber: z17.number().int().min(1),
|
|
11544
|
+
outcome: z17.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
11545
|
+
startedAt: z17.string().optional(),
|
|
11546
|
+
completedAt: z17.string().optional(),
|
|
11547
|
+
durationMs: z17.number().min(0).optional(),
|
|
11548
|
+
problemCode: z17.string().optional(),
|
|
11549
|
+
message: z17.string().optional(),
|
|
11550
|
+
kernelSessionId: z17.string().nullable().optional(),
|
|
11551
|
+
cleanupSucceeded: z17.boolean().nullable().optional()
|
|
11235
11552
|
}).strict();
|
|
11236
|
-
var SerpPageSnapshotCaptureSchema =
|
|
11553
|
+
var SerpPageSnapshotCaptureSchema = z17.object({
|
|
11237
11554
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
11238
11555
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
11239
11556
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
11240
|
-
sourceKind:
|
|
11241
|
-
sourcePosition:
|
|
11242
|
-
status:
|
|
11243
|
-
fetchedVia:
|
|
11244
|
-
httpStatus:
|
|
11245
|
-
contentType:
|
|
11246
|
-
title:
|
|
11557
|
+
sourceKind: z17.enum(SerpPageSnapshotSourceKindValues),
|
|
11558
|
+
sourcePosition: z17.number().int().min(1).nullable(),
|
|
11559
|
+
status: z17.enum(SerpPageFetchStatusValues),
|
|
11560
|
+
fetchedVia: z17.enum(SerpPageFetchedViaValues).nullable(),
|
|
11561
|
+
httpStatus: z17.number().int().min(100).max(599).nullable(),
|
|
11562
|
+
contentType: z17.string().nullable(),
|
|
11563
|
+
title: z17.string().nullable(),
|
|
11247
11564
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
11248
|
-
metaDescription:
|
|
11249
|
-
headings:
|
|
11250
|
-
level:
|
|
11251
|
-
text:
|
|
11565
|
+
metaDescription: z17.string().nullable(),
|
|
11566
|
+
headings: z17.array(z17.object({
|
|
11567
|
+
level: z17.number().int().min(1).max(6),
|
|
11568
|
+
text: z17.string()
|
|
11252
11569
|
}).strict()).default([]),
|
|
11253
|
-
artifact:
|
|
11254
|
-
htmlBlobUrl:
|
|
11255
|
-
textBlobUrl:
|
|
11256
|
-
markdownBlobUrl:
|
|
11257
|
-
screenshotBlobUrl:
|
|
11258
|
-
contentSha256:
|
|
11259
|
-
capturedAt:
|
|
11570
|
+
artifact: z17.object({
|
|
11571
|
+
htmlBlobUrl: z17.string().url().nullable(),
|
|
11572
|
+
textBlobUrl: z17.string().url().nullable(),
|
|
11573
|
+
markdownBlobUrl: z17.string().url().nullable(),
|
|
11574
|
+
screenshotBlobUrl: z17.string().url().nullable(),
|
|
11575
|
+
contentSha256: z17.string().nullable(),
|
|
11576
|
+
capturedAt: z17.string().nullable()
|
|
11260
11577
|
}).strict(),
|
|
11261
|
-
error:
|
|
11262
|
-
code:
|
|
11263
|
-
message:
|
|
11578
|
+
error: z17.object({
|
|
11579
|
+
code: z17.string(),
|
|
11580
|
+
message: z17.string()
|
|
11264
11581
|
}).strict().nullable()
|
|
11265
11582
|
}).strict();
|
|
11266
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
11583
|
+
var SerpIntelligenceCaptureResponseSchema = z17.object({
|
|
11267
11584
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
11268
|
-
attempts:
|
|
11585
|
+
attempts: z17.array(SerpIntelligenceCaptureAttemptSchema),
|
|
11269
11586
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
11270
|
-
pageSnapshotArtifacts:
|
|
11271
|
-
billing:
|
|
11272
|
-
creditsUsed:
|
|
11273
|
-
requestId:
|
|
11274
|
-
jobId:
|
|
11587
|
+
pageSnapshotArtifacts: z17.array(SerpPageSnapshotCaptureSchema),
|
|
11588
|
+
billing: z17.object({
|
|
11589
|
+
creditsUsed: z17.number().min(0).optional(),
|
|
11590
|
+
requestId: z17.string().optional(),
|
|
11591
|
+
jobId: z17.string().optional()
|
|
11275
11592
|
}).strict().optional()
|
|
11276
11593
|
}).strict();
|
|
11277
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
11278
|
-
pageSnapshotArtifacts:
|
|
11279
|
-
attempts:
|
|
11594
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z17.object({
|
|
11595
|
+
pageSnapshotArtifacts: z17.array(SerpPageSnapshotCaptureSchema),
|
|
11596
|
+
attempts: z17.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
11280
11597
|
}).strict();
|
|
11281
11598
|
|
|
11282
11599
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -11448,7 +11765,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
|
11448
11765
|
var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
11449
11766
|
var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
11450
11767
|
var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
11451
|
-
var serpIntelligenceApp = new
|
|
11768
|
+
var serpIntelligenceApp = new Hono8();
|
|
11452
11769
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
11453
11770
|
function structuredError(input) {
|
|
11454
11771
|
return {
|
|
@@ -11623,7 +11940,7 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
11623
11940
|
});
|
|
11624
11941
|
|
|
11625
11942
|
// src/mcp/mcp-routes.ts
|
|
11626
|
-
import { Hono as
|
|
11943
|
+
import { Hono as Hono9 } from "hono";
|
|
11627
11944
|
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
|
11628
11945
|
configureReportSaving(false);
|
|
11629
11946
|
function mcpAuthError() {
|
|
@@ -11653,7 +11970,7 @@ async function requireMcpCallerKey(c) {
|
|
|
11653
11970
|
if (!user) return mcpAuthError();
|
|
11654
11971
|
return callerKey;
|
|
11655
11972
|
}
|
|
11656
|
-
var mcpApp = new
|
|
11973
|
+
var mcpApp = new Hono9();
|
|
11657
11974
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
11658
11975
|
server.registerTool("capture_serp_snapshot", {
|
|
11659
11976
|
title: "SERP Intelligence Snapshot",
|
|
@@ -11689,7 +12006,7 @@ mcpApp.all("/", async (c) => {
|
|
|
11689
12006
|
});
|
|
11690
12007
|
|
|
11691
12008
|
// src/api/browser-agent-routes.ts
|
|
11692
|
-
import { Hono as
|
|
12009
|
+
import { Hono as Hono10 } from "hono";
|
|
11693
12010
|
|
|
11694
12011
|
// src/api/browser-agent-db.ts
|
|
11695
12012
|
import { randomUUID } from "crypto";
|
|
@@ -12210,7 +12527,7 @@ async function loadOpenSession(id, userId) {
|
|
|
12210
12527
|
return row;
|
|
12211
12528
|
}
|
|
12212
12529
|
function buildBrowserAgentRoutes() {
|
|
12213
|
-
const app2 = new
|
|
12530
|
+
const app2 = new Hono10();
|
|
12214
12531
|
app2.use("*", async (c, next) => {
|
|
12215
12532
|
await migrateBrowserAgent();
|
|
12216
12533
|
return next();
|
|
@@ -12725,9 +13042,9 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
|
|
|
12725
13042
|
|
|
12726
13043
|
// src/api/stripe-routes.ts
|
|
12727
13044
|
import Stripe from "stripe";
|
|
12728
|
-
import { Hono as
|
|
13045
|
+
import { Hono as Hono11 } from "hono";
|
|
12729
13046
|
var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
12730
|
-
var stripeApp = new
|
|
13047
|
+
var stripeApp = new Hono11();
|
|
12731
13048
|
stripeApp.post("/webhooks", async (c) => {
|
|
12732
13049
|
const sig = c.req.header("stripe-signature");
|
|
12733
13050
|
const body = await c.req.text();
|
|
@@ -12822,27 +13139,27 @@ import { getCookie, setCookie, deleteCookie } from "hono/cookie";
|
|
|
12822
13139
|
import Stripe2 from "stripe";
|
|
12823
13140
|
|
|
12824
13141
|
// src/api/billing-schemas.ts
|
|
12825
|
-
import { z as
|
|
12826
|
-
var BillingCheckoutBodySchema =
|
|
12827
|
-
priceId:
|
|
12828
|
-
});
|
|
12829
|
-
var FreeCreditBreakdownSchema =
|
|
12830
|
-
signup_grant_mc:
|
|
12831
|
-
monthly_refresh_mc:
|
|
12832
|
-
total_free_mc:
|
|
12833
|
-
signup_grant_credits:
|
|
12834
|
-
monthly_refresh_credits:
|
|
12835
|
-
total_free_credits:
|
|
12836
|
-
});
|
|
12837
|
-
var BillingBalanceResponseSchema =
|
|
12838
|
-
balance_mc:
|
|
12839
|
-
balance_credits:
|
|
13142
|
+
import { z as z18 } from "zod";
|
|
13143
|
+
var BillingCheckoutBodySchema = z18.object({
|
|
13144
|
+
priceId: z18.string().min(1)
|
|
13145
|
+
});
|
|
13146
|
+
var FreeCreditBreakdownSchema = z18.object({
|
|
13147
|
+
signup_grant_mc: z18.number().int().nonnegative(),
|
|
13148
|
+
monthly_refresh_mc: z18.number().int().nonnegative(),
|
|
13149
|
+
total_free_mc: z18.number().int().nonnegative(),
|
|
13150
|
+
signup_grant_credits: z18.number().nonnegative(),
|
|
13151
|
+
monthly_refresh_credits: z18.number().nonnegative(),
|
|
13152
|
+
total_free_credits: z18.number().nonnegative()
|
|
13153
|
+
});
|
|
13154
|
+
var BillingBalanceResponseSchema = z18.object({
|
|
13155
|
+
balance_mc: z18.number().int().nonnegative(),
|
|
13156
|
+
balance_credits: z18.number().nonnegative(),
|
|
12840
13157
|
free_credits: FreeCreditBreakdownSchema,
|
|
12841
|
-
ledger:
|
|
13158
|
+
ledger: z18.array(z18.any())
|
|
12842
13159
|
});
|
|
12843
|
-
var MonthlyRefreshSweepResultSchema =
|
|
12844
|
-
usersRefreshed:
|
|
12845
|
-
totalMcGranted:
|
|
13160
|
+
var MonthlyRefreshSweepResultSchema = z18.object({
|
|
13161
|
+
usersRefreshed: z18.number().int().nonnegative(),
|
|
13162
|
+
totalMcGranted: z18.number().int().nonnegative()
|
|
12846
13163
|
});
|
|
12847
13164
|
|
|
12848
13165
|
// src/api/credit-operations.ts
|
|
@@ -12943,7 +13260,7 @@ async function getFreeCreditBreakdown(userId) {
|
|
|
12943
13260
|
}
|
|
12944
13261
|
|
|
12945
13262
|
// src/api/session.ts
|
|
12946
|
-
import { createHmac, timingSafeEqual } from "crypto";
|
|
13263
|
+
import { createHmac as createHmac2, timingSafeEqual } from "crypto";
|
|
12947
13264
|
var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
12948
13265
|
function getSessionSecret() {
|
|
12949
13266
|
const configured = process.env.SESSION_SECRET?.trim();
|
|
@@ -12962,7 +13279,7 @@ function safeEqualHex(a, b) {
|
|
|
12962
13279
|
}
|
|
12963
13280
|
function signSession(userId) {
|
|
12964
13281
|
const payload = String(userId);
|
|
12965
|
-
const sig =
|
|
13282
|
+
const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
12966
13283
|
return `${payload}.${sig}`;
|
|
12967
13284
|
}
|
|
12968
13285
|
function verifySession(token) {
|
|
@@ -12970,7 +13287,7 @@ function verifySession(token) {
|
|
|
12970
13287
|
if (dot === -1) return null;
|
|
12971
13288
|
const payload = token.slice(0, dot);
|
|
12972
13289
|
const sig = token.slice(dot + 1);
|
|
12973
|
-
const expected =
|
|
13290
|
+
const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
12974
13291
|
if (!safeEqualHex(sig, expected)) return null;
|
|
12975
13292
|
const id = parseInt(payload);
|
|
12976
13293
|
return isNaN(id) ? null : id;
|
|
@@ -13053,7 +13370,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
|
|
|
13053
13370
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
13054
13371
|
return next();
|
|
13055
13372
|
});
|
|
13056
|
-
var app = new
|
|
13373
|
+
var app = new Hono12();
|
|
13057
13374
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
13058
13375
|
function requireStripeSecret() {
|
|
13059
13376
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
@@ -13699,13 +14016,14 @@ app.get("/cron/tick", async (c) => {
|
|
|
13699
14016
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
13700
14017
|
return c.json({ error: "Unauthorized" }, 401);
|
|
13701
14018
|
}
|
|
13702
|
-
const { drainQueue } = await import("./worker-
|
|
14019
|
+
const { drainQueue } = await import("./worker-TDJQ6TH3.js");
|
|
13703
14020
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
14021
|
+
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
13704
14022
|
const [results, sweepResult] = await Promise.all([
|
|
13705
14023
|
drainQueue(budget),
|
|
13706
14024
|
runMonthlyRefreshSweep()
|
|
13707
14025
|
]);
|
|
13708
|
-
return c.json({ drained: results.length, results, sweepResult });
|
|
14026
|
+
return c.json({ drained: results.length, results, sweepResult, workflowDispatch: workflowDispatchResult });
|
|
13709
14027
|
});
|
|
13710
14028
|
app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn] }));
|
|
13711
14029
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
@@ -13714,6 +14032,7 @@ app.route("/screenshot", screenshotApp);
|
|
|
13714
14032
|
app.route("/facebook", facebookAdApp);
|
|
13715
14033
|
app.route("/maps", mapsApp);
|
|
13716
14034
|
app.route("/directory", directoryApp);
|
|
14035
|
+
app.route("/workflows", workflowApp);
|
|
13717
14036
|
app.route("/serp-intelligence", serpIntelligenceApp);
|
|
13718
14037
|
app.route("/mcp", mcpApp);
|
|
13719
14038
|
app.route("/agent", buildBrowserAgentRoutes());
|
|
@@ -13825,4 +14144,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
13825
14144
|
export {
|
|
13826
14145
|
app
|
|
13827
14146
|
};
|
|
13828
|
-
//# sourceMappingURL=server-
|
|
14147
|
+
//# sourceMappingURL=server-USOIPC2I.js.map
|