mcp-scraper 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +3342 -2655
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +113 -19
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-GL4BW4CP.js → chunk-DZBGULWW.js} +61 -9
- package/dist/chunk-DZBGULWW.js.map +1 -0
- package/dist/{chunk-D7ZT27HY.js → chunk-GMZFQRH2.js} +2 -2
- package/dist/{chunk-HL33CGJF.js → chunk-HXGZMMW5.js} +29 -2
- package/dist/chunk-HXGZMMW5.js.map +1 -0
- package/dist/{chunk-HE2LQPJ2.js → chunk-JBME5WJR.js} +2 -2
- package/dist/chunk-JWIE5NCR.js +284 -0
- package/dist/chunk-JWIE5NCR.js.map +1 -0
- package/dist/chunk-L4LMSXLO.js +7 -0
- package/dist/chunk-L4LMSXLO.js.map +1 -0
- package/dist/{chunk-RUGJE5EB.js → chunk-TUSLATZY.js} +3 -3
- package/dist/{chunk-O2S5TOCG.js → chunk-XHJJQRNQ.js} +111 -22
- package/dist/chunk-XHJJQRNQ.js.map +1 -0
- package/dist/{chunk-2HDMYW4B.js → chunk-ZJDANZTQ.js} +2 -2
- package/dist/{db-LIOTIWVN.js → db-EONGTNJM.js} +2 -2
- package/dist/{extract-bundle-U4D5LW5W.js → extract-bundle-HMCYUVBJ.js} +58 -5
- package/dist/extract-bundle-HMCYUVBJ.js.map +1 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -3
- package/dist/{server-VWXDE64Y.js → server-PH3VJ74X.js} +937 -745
- package/dist/server-PH3VJ74X.js.map +1 -0
- package/dist/{site-extract-repository-NVSZH35Y.js → site-extract-repository-YNPXEZCB.js} +4 -4
- package/dist/{worker-AM2DHUWG.js → worker-45WEWCMQ.js} +6 -6
- package/docs/mcp-tool-manifest.generated.json +181 -28
- package/docs/specs/meta-ad-creative-media-resolution-spec.md +3 -3
- package/docs/specs/unified-credit-and-scheduled-execution-billing-spec.md +1007 -0
- package/package.json +2 -2
- package/dist/chunk-GL4BW4CP.js.map +0 -1
- package/dist/chunk-HL33CGJF.js.map +0 -1
- package/dist/chunk-O2S5TOCG.js.map +0 -1
- package/dist/chunk-OQHYDW4Q.js +0 -7
- package/dist/chunk-OQHYDW4Q.js.map +0 -1
- package/dist/extract-bundle-U4D5LW5W.js.map +0 -1
- package/dist/server-VWXDE64Y.js.map +0 -1
- /package/dist/{chunk-D7ZT27HY.js.map → chunk-GMZFQRH2.js.map} +0 -0
- /package/dist/{chunk-HE2LQPJ2.js.map → chunk-JBME5WJR.js.map} +0 -0
- /package/dist/{chunk-RUGJE5EB.js.map → chunk-TUSLATZY.js.map} +0 -0
- /package/dist/{chunk-2HDMYW4B.js.map → chunk-ZJDANZTQ.js.map} +0 -0
- /package/dist/{db-LIOTIWVN.js.map → db-EONGTNJM.js.map} +0 -0
- /package/dist/{site-extract-repository-NVSZH35Y.js.map → site-extract-repository-YNPXEZCB.js.map} +0 -0
- /package/dist/{worker-AM2DHUWG.js.map → worker-45WEWCMQ.js.map} +0 -0
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
import {
|
|
2
|
+
harvestPageMedia
|
|
3
|
+
} from "./chunk-JWIE5NCR.js";
|
|
1
4
|
import {
|
|
2
5
|
classifyHarvestProblem,
|
|
3
6
|
createHarvestAttemptRecorder,
|
|
4
7
|
harvestProblemResponse,
|
|
5
8
|
serializeHarvestProblem
|
|
6
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-JBME5WJR.js";
|
|
7
10
|
import {
|
|
8
11
|
BrowserDriver,
|
|
9
12
|
MapsSelectors,
|
|
@@ -11,7 +14,7 @@ import {
|
|
|
11
14
|
deleteKernelProxyId,
|
|
12
15
|
harvest,
|
|
13
16
|
resolveKernelProxyId
|
|
14
|
-
} from "./chunk-
|
|
17
|
+
} from "./chunk-ZJDANZTQ.js";
|
|
15
18
|
import {
|
|
16
19
|
HttpMcpToolExecutor,
|
|
17
20
|
MemoryMcpToolExecutor,
|
|
@@ -32,7 +35,7 @@ import {
|
|
|
32
35
|
sanitizeAttempts,
|
|
33
36
|
sanitizeHarvestResult,
|
|
34
37
|
transcribeMediaUrl
|
|
35
|
-
} from "./chunk-
|
|
38
|
+
} from "./chunk-XHJJQRNQ.js";
|
|
36
39
|
import {
|
|
37
40
|
auditImages,
|
|
38
41
|
buildLinkReport,
|
|
@@ -50,7 +53,7 @@ import {
|
|
|
50
53
|
recordKernelSession,
|
|
51
54
|
recordVendorUsage,
|
|
52
55
|
runWithCostContext
|
|
53
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-GMZFQRH2.js";
|
|
54
57
|
import {
|
|
55
58
|
csvRecords,
|
|
56
59
|
listWorkflowDefinitions,
|
|
@@ -74,7 +77,7 @@ import {
|
|
|
74
77
|
RawMapsOverviewSchema,
|
|
75
78
|
RawMapsReviewStatsSchema
|
|
76
79
|
} from "./chunk-XGIPATLV.js";
|
|
77
|
-
import "./chunk-
|
|
80
|
+
import "./chunk-L4LMSXLO.js";
|
|
78
81
|
import {
|
|
79
82
|
completeExtractJob,
|
|
80
83
|
countSuccessfulPages,
|
|
@@ -85,7 +88,7 @@ import {
|
|
|
85
88
|
saveExtractPages,
|
|
86
89
|
setExtractJobTotal,
|
|
87
90
|
settleExtractJob
|
|
88
|
-
} from "./chunk-
|
|
91
|
+
} from "./chunk-TUSLATZY.js";
|
|
89
92
|
import {
|
|
90
93
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
91
94
|
CONCURRENCY_PRICE_ID,
|
|
@@ -100,15 +103,22 @@ import {
|
|
|
100
103
|
MEMORY_AI_MARGIN_MULTIPLE,
|
|
101
104
|
MEMORY_PLANS,
|
|
102
105
|
MEMORY_PLAN_QUOTA,
|
|
106
|
+
SCHEDULED_AGENT_LLM_MARKUP,
|
|
107
|
+
SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
108
|
+
SCHEDULED_RUN_BASE_CREDITS,
|
|
109
|
+
SCHEDULED_RUN_BASE_MC,
|
|
110
|
+
SCHEDULED_RUN_DEFAULT_MAX_CREDITS,
|
|
103
111
|
SCHEDULING_PLANS,
|
|
104
112
|
SITE_AUDIT_LLM_MARGIN_MULTIPLE,
|
|
105
113
|
SUBSCRIPTION_TIERS,
|
|
106
114
|
SUBSCRIPTION_TIER_BY_KEY,
|
|
115
|
+
USD_NANOS,
|
|
107
116
|
VIDEO_ANALYSIS_LLM_MARGIN_MULTIPLE,
|
|
108
117
|
browserActiveCostMc,
|
|
109
118
|
concurrencySlotBillingInfo,
|
|
110
|
-
insufficientBalanceResponse
|
|
111
|
-
|
|
119
|
+
insufficientBalanceResponse,
|
|
120
|
+
vendorCostUsdToMc
|
|
121
|
+
} from "./chunk-HXGZMMW5.js";
|
|
112
122
|
import {
|
|
113
123
|
CaptchaError,
|
|
114
124
|
RECAPTCHA_INSTRUCTIONS,
|
|
@@ -191,7 +201,7 @@ import {
|
|
|
191
201
|
setUserMemoryCreds,
|
|
192
202
|
upsertPageSnapshot,
|
|
193
203
|
verifyPassword
|
|
194
|
-
} from "./chunk-
|
|
204
|
+
} from "./chunk-DZBGULWW.js";
|
|
195
205
|
|
|
196
206
|
// src/blog/registry.ts
|
|
197
207
|
var posts = [
|
|
@@ -4226,285 +4236,6 @@ async function capturePageData(url, opts = {}) {
|
|
|
4226
4236
|
}
|
|
4227
4237
|
}
|
|
4228
4238
|
|
|
4229
|
-
// src/lib/media-extractor.ts
|
|
4230
|
-
import { createWriteStream, mkdirSync } from "fs";
|
|
4231
|
-
import { homedir } from "os";
|
|
4232
|
-
import { join as join2, extname, basename } from "path";
|
|
4233
|
-
import { pipeline } from "stream/promises";
|
|
4234
|
-
import { Readable } from "stream";
|
|
4235
|
-
var AD_PATTERNS = [
|
|
4236
|
-
"doubleclick.net",
|
|
4237
|
-
"googlesyndication.com",
|
|
4238
|
-
"googletagmanager.com",
|
|
4239
|
-
"google-analytics.com",
|
|
4240
|
-
"googletagservices.com",
|
|
4241
|
-
"adservice.google",
|
|
4242
|
-
"googletag.",
|
|
4243
|
-
"pagead2.googlesyndication",
|
|
4244
|
-
"facebook.net/tr",
|
|
4245
|
-
"connect.facebook.net",
|
|
4246
|
-
"fbcdn.net/rsrc",
|
|
4247
|
-
"analytics.twitter.com",
|
|
4248
|
-
"static.ads-twitter.com",
|
|
4249
|
-
"ads.twitter.com",
|
|
4250
|
-
"t.co/i/adsct",
|
|
4251
|
-
"pixel.advertising.com",
|
|
4252
|
-
"hotjar.com",
|
|
4253
|
-
"clarity.ms",
|
|
4254
|
-
"quantserve.com",
|
|
4255
|
-
"scorecardresearch.com",
|
|
4256
|
-
"newrelic.com",
|
|
4257
|
-
"nr-data.net",
|
|
4258
|
-
"segment.io",
|
|
4259
|
-
"segment.com",
|
|
4260
|
-
"amplitude.com",
|
|
4261
|
-
"mixpanel.com",
|
|
4262
|
-
"heap.io",
|
|
4263
|
-
"fullstory.com",
|
|
4264
|
-
"moatads.com",
|
|
4265
|
-
"criteo.com",
|
|
4266
|
-
"adsrvr.org",
|
|
4267
|
-
"rubiconproject.com",
|
|
4268
|
-
"pubmatic.com",
|
|
4269
|
-
"openx.net",
|
|
4270
|
-
"appnexus.com",
|
|
4271
|
-
"amazon-adsystem.com",
|
|
4272
|
-
"media.net",
|
|
4273
|
-
"yieldmo.com",
|
|
4274
|
-
"triplelift.com",
|
|
4275
|
-
"sharethrough.com",
|
|
4276
|
-
"prebid.",
|
|
4277
|
-
"smaato.net",
|
|
4278
|
-
"indexworm.com",
|
|
4279
|
-
"casalemedia.com",
|
|
4280
|
-
"outbrain.com",
|
|
4281
|
-
"taboola.com",
|
|
4282
|
-
"revcontent.com",
|
|
4283
|
-
"mgid.com",
|
|
4284
|
-
"tawk.to",
|
|
4285
|
-
"intercom.io",
|
|
4286
|
-
"drift.com",
|
|
4287
|
-
"hs-scripts.com",
|
|
4288
|
-
"zopim.com",
|
|
4289
|
-
"livechatinc.com",
|
|
4290
|
-
"userlike.com",
|
|
4291
|
-
"onetrust.com",
|
|
4292
|
-
"cookielaw.org",
|
|
4293
|
-
"cookieinformation.com",
|
|
4294
|
-
"trustarc.com",
|
|
4295
|
-
"/ads/",
|
|
4296
|
-
"/ad/",
|
|
4297
|
-
"/banner/",
|
|
4298
|
-
"/banners/",
|
|
4299
|
-
"/pixel/",
|
|
4300
|
-
"/beacon/",
|
|
4301
|
-
"/tracking/",
|
|
4302
|
-
"/tracker/",
|
|
4303
|
-
"/remarketing/",
|
|
4304
|
-
"/conversion/",
|
|
4305
|
-
"1x1.gif",
|
|
4306
|
-
"spacer.gif",
|
|
4307
|
-
"blank.gif",
|
|
4308
|
-
"transparent.gif"
|
|
4309
|
-
];
|
|
4310
|
-
var IMAGE_EXTS = /* @__PURE__ */ new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif", ".svg", ".tiff"]);
|
|
4311
|
-
var VIDEO_EXTS = /* @__PURE__ */ new Set([".mp4", ".webm", ".mov", ".avi", ".m4v", ".ogv", ".mkv"]);
|
|
4312
|
-
var AUDIO_EXTS = /* @__PURE__ */ new Set([".mp3", ".wav", ".ogg", ".aac", ".m4a", ".flac", ".opus"]);
|
|
4313
|
-
function isAdUrl(url) {
|
|
4314
|
-
const lower = url.toLowerCase();
|
|
4315
|
-
return AD_PATTERNS.some((p) => lower.includes(p));
|
|
4316
|
-
}
|
|
4317
|
-
function isDataUri(url) {
|
|
4318
|
-
return url.startsWith("data:");
|
|
4319
|
-
}
|
|
4320
|
-
function typeFromUrl(url) {
|
|
4321
|
-
try {
|
|
4322
|
-
const ext = extname(new URL(url).pathname).toLowerCase();
|
|
4323
|
-
if (IMAGE_EXTS.has(ext)) return "image";
|
|
4324
|
-
if (VIDEO_EXTS.has(ext)) return "video";
|
|
4325
|
-
if (AUDIO_EXTS.has(ext)) return "audio";
|
|
4326
|
-
} catch {
|
|
4327
|
-
}
|
|
4328
|
-
return null;
|
|
4329
|
-
}
|
|
4330
|
-
function typeFromMime(mime) {
|
|
4331
|
-
const lower = mime.toLowerCase();
|
|
4332
|
-
if (lower.startsWith("image/")) return "image";
|
|
4333
|
-
if (lower.startsWith("video/")) return "video";
|
|
4334
|
-
if (lower.startsWith("audio/")) return "audio";
|
|
4335
|
-
return null;
|
|
4336
|
-
}
|
|
4337
|
-
function resolveUrl(raw, base) {
|
|
4338
|
-
if (!raw || isDataUri(raw)) return null;
|
|
4339
|
-
try {
|
|
4340
|
-
return new URL(raw, base).href;
|
|
4341
|
-
} catch {
|
|
4342
|
-
return null;
|
|
4343
|
-
}
|
|
4344
|
-
}
|
|
4345
|
-
function safeFilename(url, index) {
|
|
4346
|
-
try {
|
|
4347
|
-
const u = new URL(url);
|
|
4348
|
-
const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
|
|
4349
|
-
return base || `asset-${index}`;
|
|
4350
|
-
} catch {
|
|
4351
|
-
return `asset-${index}`;
|
|
4352
|
-
}
|
|
4353
|
-
}
|
|
4354
|
-
function extractMediaUrls(html, baseUrl) {
|
|
4355
|
-
const seen = /* @__PURE__ */ new Set();
|
|
4356
|
-
const urls = [];
|
|
4357
|
-
const add = (raw) => {
|
|
4358
|
-
const resolved = resolveUrl(raw.trim(), baseUrl);
|
|
4359
|
-
if (!resolved || seen.has(resolved) || isAdUrl(resolved)) return;
|
|
4360
|
-
seen.add(resolved);
|
|
4361
|
-
urls.push(resolved);
|
|
4362
|
-
};
|
|
4363
|
-
for (const m of html.matchAll(/<img\s[^>]*>/gi)) {
|
|
4364
|
-
const tag = m[0];
|
|
4365
|
-
for (const attr of ["src", "data-src", "data-lazy-src", "data-original"]) {
|
|
4366
|
-
const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
|
|
4367
|
-
if (v) add(v);
|
|
4368
|
-
}
|
|
4369
|
-
const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
|
|
4370
|
-
if (srcset) {
|
|
4371
|
-
for (const part of srcset.split(",")) {
|
|
4372
|
-
const u = part.trim().split(/\s+/)[0];
|
|
4373
|
-
if (u) add(u);
|
|
4374
|
-
}
|
|
4375
|
-
}
|
|
4376
|
-
}
|
|
4377
|
-
for (const m of html.matchAll(/<source\s[^>]*>/gi)) {
|
|
4378
|
-
const tag = m[0];
|
|
4379
|
-
const src = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
|
|
4380
|
-
if (src) add(src);
|
|
4381
|
-
const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
|
|
4382
|
-
if (srcset) {
|
|
4383
|
-
for (const part of srcset.split(",")) {
|
|
4384
|
-
const u = part.trim().split(/\s+/)[0];
|
|
4385
|
-
if (u) add(u);
|
|
4386
|
-
}
|
|
4387
|
-
}
|
|
4388
|
-
}
|
|
4389
|
-
for (const m of html.matchAll(/<video\s[^>]*>/gi)) {
|
|
4390
|
-
const tag = m[0];
|
|
4391
|
-
for (const attr of ["src", "poster"]) {
|
|
4392
|
-
const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
|
|
4393
|
-
if (v) add(v);
|
|
4394
|
-
}
|
|
4395
|
-
}
|
|
4396
|
-
for (const m of html.matchAll(/<audio\s[^>]*>/gi)) {
|
|
4397
|
-
const tag = m[0];
|
|
4398
|
-
const v = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
|
|
4399
|
-
if (v) add(v);
|
|
4400
|
-
}
|
|
4401
|
-
for (const m of html.matchAll(/<meta\s[^>]*>/gi)) {
|
|
4402
|
-
const tag = m[0];
|
|
4403
|
-
const prop = (tag.match(/\b(?:property|name)\s*=\s*["']([^"']+)["']/i) ?? [])[1]?.toLowerCase();
|
|
4404
|
-
if (prop === "og:image" || prop === "twitter:image") {
|
|
4405
|
-
const content = (tag.match(/\bcontent\s*=\s*["']([^"']+)["']/i) ?? [])[1];
|
|
4406
|
-
if (content) add(content);
|
|
4407
|
-
}
|
|
4408
|
-
}
|
|
4409
|
-
for (const m of html.matchAll(/background(?:-image)?\s*:\s*url\(\s*["']?([^"')]+)["']?\s*\)/gi)) {
|
|
4410
|
-
add(m[1]);
|
|
4411
|
-
}
|
|
4412
|
-
return urls;
|
|
4413
|
-
}
|
|
4414
|
-
async function downloadAsset(url, destDir, filename) {
|
|
4415
|
-
const res = await fetch(url, {
|
|
4416
|
-
headers: { "User-Agent": "Mozilla/5.0 (compatible; ThorbitBot/1.0)" },
|
|
4417
|
-
signal: AbortSignal.timeout(15e3)
|
|
4418
|
-
});
|
|
4419
|
-
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
4420
|
-
if (!res.body) throw new Error("Empty response body");
|
|
4421
|
-
const mimeType = res.headers.get("content-type")?.split(";")[0].trim() ?? null;
|
|
4422
|
-
let dest = join2(destDir, filename);
|
|
4423
|
-
if (mimeType && !extname(filename)) {
|
|
4424
|
-
const mimeExt = {
|
|
4425
|
-
"image/jpeg": ".jpg",
|
|
4426
|
-
"image/png": ".png",
|
|
4427
|
-
"image/webp": ".webp",
|
|
4428
|
-
"image/gif": ".gif",
|
|
4429
|
-
"image/svg+xml": ".svg",
|
|
4430
|
-
"image/avif": ".avif",
|
|
4431
|
-
"video/mp4": ".mp4",
|
|
4432
|
-
"video/webm": ".webm",
|
|
4433
|
-
"audio/mpeg": ".mp3",
|
|
4434
|
-
"audio/ogg": ".ogg",
|
|
4435
|
-
"audio/wav": ".wav"
|
|
4436
|
-
};
|
|
4437
|
-
const ext = mimeExt[mimeType];
|
|
4438
|
-
if (ext) dest = dest + ext;
|
|
4439
|
-
}
|
|
4440
|
-
const writer = createWriteStream(dest);
|
|
4441
|
-
await pipeline(Readable.fromWeb(res.body), writer);
|
|
4442
|
-
const { statSync } = await import("fs");
|
|
4443
|
-
const sizeBytes = statSync(dest).size;
|
|
4444
|
-
return { savedPath: dest, sizeBytes, mimeType };
|
|
4445
|
-
}
|
|
4446
|
-
async function harvestPageMedia(html, pageUrl, options = {}) {
|
|
4447
|
-
const types = options.types ?? ["image", "video", "audio"];
|
|
4448
|
-
const typesSet = new Set(types);
|
|
4449
|
-
const rawUrls = extractMediaUrls(html, pageUrl);
|
|
4450
|
-
const totalFound = rawUrls.length;
|
|
4451
|
-
const filteredUrls = [];
|
|
4452
|
-
const kept = [];
|
|
4453
|
-
for (const url of rawUrls) {
|
|
4454
|
-
const type = typeFromUrl(url);
|
|
4455
|
-
if (!type || !typesSet.has(type)) {
|
|
4456
|
-
filteredUrls.push(url);
|
|
4457
|
-
continue;
|
|
4458
|
-
}
|
|
4459
|
-
kept.push({ url, type });
|
|
4460
|
-
}
|
|
4461
|
-
if (options.outputDir === null) {
|
|
4462
|
-
return {
|
|
4463
|
-
pageUrl,
|
|
4464
|
-
outputDir: null,
|
|
4465
|
-
assets: kept.map(({ url, type }, i) => ({
|
|
4466
|
-
url,
|
|
4467
|
-
type,
|
|
4468
|
-
mimeType: null,
|
|
4469
|
-
filename: safeFilename(url, i),
|
|
4470
|
-
savedPath: null,
|
|
4471
|
-
sizeBytes: null
|
|
4472
|
-
})),
|
|
4473
|
-
filteredCount: filteredUrls.length,
|
|
4474
|
-
totalFound
|
|
4475
|
-
};
|
|
4476
|
-
}
|
|
4477
|
-
const domain = (() => {
|
|
4478
|
-
try {
|
|
4479
|
-
return new URL(pageUrl).hostname.replace(/^www\./, "");
|
|
4480
|
-
} catch {
|
|
4481
|
-
return "unknown";
|
|
4482
|
-
}
|
|
4483
|
-
})();
|
|
4484
|
-
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
4485
|
-
const outDir = options.outputDir ?? join2(homedir(), "Downloads", "mcp-scraper", "media", `${stamp}-${domain}`);
|
|
4486
|
-
mkdirSync(outDir, { recursive: true });
|
|
4487
|
-
const assets = [];
|
|
4488
|
-
await Promise.allSettled(
|
|
4489
|
-
kept.map(async ({ url, type }, i) => {
|
|
4490
|
-
const filename = safeFilename(url, i);
|
|
4491
|
-
try {
|
|
4492
|
-
const { savedPath, sizeBytes, mimeType } = await downloadAsset(url, outDir, filename);
|
|
4493
|
-
const resolvedType = mimeType ? typeFromMime(mimeType) ?? type : type;
|
|
4494
|
-
assets.push({ url, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes });
|
|
4495
|
-
} catch {
|
|
4496
|
-
assets.push({ url, type, mimeType: null, filename, savedPath: null, sizeBytes: null });
|
|
4497
|
-
}
|
|
4498
|
-
})
|
|
4499
|
-
);
|
|
4500
|
-
assets.sort((a, b) => {
|
|
4501
|
-
if (a.savedPath && !b.savedPath) return -1;
|
|
4502
|
-
if (!a.savedPath && b.savedPath) return 1;
|
|
4503
|
-
return a.url.localeCompare(b.url);
|
|
4504
|
-
});
|
|
4505
|
-
return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound };
|
|
4506
|
-
}
|
|
4507
|
-
|
|
4508
4239
|
// src/api/site-mapper.ts
|
|
4509
4240
|
var SKIP_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
4510
4241
|
"jpg",
|
|
@@ -5322,7 +5053,7 @@ async function extractSite(opts) {
|
|
|
5322
5053
|
}
|
|
5323
5054
|
|
|
5324
5055
|
// src/api/server.ts
|
|
5325
|
-
import { Hono as
|
|
5056
|
+
import { Hono as Hono24 } from "hono";
|
|
5326
5057
|
import { serve as serveInngest } from "inngest/hono";
|
|
5327
5058
|
|
|
5328
5059
|
// src/inngest/client.ts
|
|
@@ -7628,7 +7359,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
7628
7359
|
return buildSeoAudit(pages, job.startUrl);
|
|
7629
7360
|
}) : null;
|
|
7630
7361
|
const artifacts = await step.run("finalize", async () => {
|
|
7631
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
7362
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-HMCYUVBJ.js");
|
|
7632
7363
|
const stored = await assembleExtractArtifacts(job, { branding, imageAudit, seoAudit });
|
|
7633
7364
|
await completeExtractJob(jobId, stored);
|
|
7634
7365
|
return stored;
|
|
@@ -8248,8 +7979,479 @@ internalMemoryApp.post("/ai-debit", async (c) => {
|
|
|
8248
7979
|
return c.json({ ok, mc, balance_mc, insufficient: !ok }, 200);
|
|
8249
7980
|
});
|
|
8250
7981
|
|
|
8251
|
-
// src/api/
|
|
7982
|
+
// src/api/internal-billing-routes.ts
|
|
8252
7983
|
import { Hono as Hono3 } from "hono";
|
|
7984
|
+
import { z as z10 } from "zod";
|
|
7985
|
+
|
|
7986
|
+
// src/api/unified-billing.ts
|
|
7987
|
+
import { createHash as createHash2, createHmac, randomUUID, timingSafeEqual } from "crypto";
|
|
7988
|
+
var SCHEDULED_RUN_BILLING_CLASS = "scheduled_agent_run";
|
|
7989
|
+
var SCHEDULED_RUN_SOURCE_SURFACE = "mastra_scheduler";
|
|
7990
|
+
var AUTHORIZATION_TTL_MS = 30 * 60 * 1e3;
|
|
7991
|
+
var UnifiedBillingError = class extends Error {
|
|
7992
|
+
constructor(code, message, httpStatus, details = {}) {
|
|
7993
|
+
super(message);
|
|
7994
|
+
this.code = code;
|
|
7995
|
+
this.httpStatus = httpStatus;
|
|
7996
|
+
this.details = details;
|
|
7997
|
+
this.name = "UnifiedBillingError";
|
|
7998
|
+
}
|
|
7999
|
+
code;
|
|
8000
|
+
httpStatus;
|
|
8001
|
+
details;
|
|
8002
|
+
};
|
|
8003
|
+
function billingSecret() {
|
|
8004
|
+
const secret2 = process.env.INTERNAL_BILLING_SECRET ?? process.env.INTERNAL_MEMORY_DEBIT_SECRET;
|
|
8005
|
+
if (!secret2) throw new UnifiedBillingError("unauthorized", "internal billing secret is not configured", 503);
|
|
8006
|
+
return secret2;
|
|
8007
|
+
}
|
|
8008
|
+
function normalizeIdentity(identity) {
|
|
8009
|
+
const normalized = identity.trim().toLowerCase();
|
|
8010
|
+
if (!normalized || normalized.length > 320) {
|
|
8011
|
+
throw new UnifiedBillingError("invalid_request", "identity is required", 400);
|
|
8012
|
+
}
|
|
8013
|
+
return normalized;
|
|
8014
|
+
}
|
|
8015
|
+
function normalizeIdempotencyKey(value) {
|
|
8016
|
+
const key = value.trim();
|
|
8017
|
+
if (!key || key.length > 500) {
|
|
8018
|
+
throw new UnifiedBillingError("invalid_request", "a bounded idempotency key is required", 400);
|
|
8019
|
+
}
|
|
8020
|
+
return key;
|
|
8021
|
+
}
|
|
8022
|
+
function tokenFor(id, idempotencyKey) {
|
|
8023
|
+
return createHmac("sha256", billingSecret()).update(id).update(":").update(idempotencyKey).digest("base64url");
|
|
8024
|
+
}
|
|
8025
|
+
function tokenHash(token) {
|
|
8026
|
+
return createHash2("sha256").update(token).digest("hex");
|
|
8027
|
+
}
|
|
8028
|
+
function verifyToken(row, token) {
|
|
8029
|
+
const expected = Buffer.from(row.token_hash, "hex");
|
|
8030
|
+
const actual = Buffer.from(tokenHash(token), "hex");
|
|
8031
|
+
if (expected.length !== actual.length || !timingSafeEqual(expected, actual)) {
|
|
8032
|
+
throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
|
|
8033
|
+
}
|
|
8034
|
+
}
|
|
8035
|
+
function safeMetadata(value) {
|
|
8036
|
+
if (!value) return "{}";
|
|
8037
|
+
return JSON.stringify(value).slice(0, 4e3);
|
|
8038
|
+
}
|
|
8039
|
+
function mapAuthorization(row, balanceMc) {
|
|
8040
|
+
return {
|
|
8041
|
+
id: row.id,
|
|
8042
|
+
token: tokenFor(row.id, row.idempotency_key),
|
|
8043
|
+
status: row.status,
|
|
8044
|
+
minimumMc: Number(row.minimum_amount_mc),
|
|
8045
|
+
maximumMc: Number(row.maximum_amount_mc),
|
|
8046
|
+
minimumCredits: Number(row.minimum_amount_mc) / MC_PER_CREDIT,
|
|
8047
|
+
maximumCredits: Number(row.maximum_amount_mc) / MC_PER_CREDIT,
|
|
8048
|
+
balanceMc,
|
|
8049
|
+
expiresAt: row.expires_at,
|
|
8050
|
+
reservationMode: "balance_check"
|
|
8051
|
+
};
|
|
8052
|
+
}
|
|
8053
|
+
async function authorizationById(id) {
|
|
8054
|
+
const result = await getDb().execute({
|
|
8055
|
+
sql: "SELECT id, user_id, idempotency_key, status, minimum_amount_mc, maximum_amount_mc, token_hash, expires_at FROM billing_authorizations WHERE id = ? LIMIT 1",
|
|
8056
|
+
args: [id]
|
|
8057
|
+
});
|
|
8058
|
+
return result.rows[0] ? result.rows[0] : null;
|
|
8059
|
+
}
|
|
8060
|
+
async function eventByKey(idempotencyKey) {
|
|
8061
|
+
const result = await getDb().execute({
|
|
8062
|
+
sql: "SELECT id, authorization_id, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, metadata FROM billing_events WHERE idempotency_key = ? LIMIT 1",
|
|
8063
|
+
args: [idempotencyKey]
|
|
8064
|
+
});
|
|
8065
|
+
return result.rows[0] ? result.rows[0] : null;
|
|
8066
|
+
}
|
|
8067
|
+
function mapEvent(row, balanceMc, duplicate) {
|
|
8068
|
+
let reason;
|
|
8069
|
+
if (row.metadata) {
|
|
8070
|
+
try {
|
|
8071
|
+
const metadata = JSON.parse(row.metadata);
|
|
8072
|
+
if (typeof metadata.reason === "string") reason = metadata.reason;
|
|
8073
|
+
} catch {
|
|
8074
|
+
}
|
|
8075
|
+
}
|
|
8076
|
+
const rawNanos = row.raw_cost_usd_nanos == null ? null : Number(row.raw_cost_usd_nanos);
|
|
8077
|
+
const modelMc = Math.max(0, Number(row.amount_mc) - SCHEDULED_RUN_BASE_MC);
|
|
8078
|
+
return {
|
|
8079
|
+
id: row.id,
|
|
8080
|
+
authorizationId: row.authorization_id,
|
|
8081
|
+
status: row.status === "settled" ? "settled" : "cost_pending",
|
|
8082
|
+
amountMc: Number(row.amount_mc),
|
|
8083
|
+
credits: Number(row.amount_mc) / MC_PER_CREDIT,
|
|
8084
|
+
baseMc: SCHEDULED_RUN_BASE_MC,
|
|
8085
|
+
modelMc,
|
|
8086
|
+
rawCostUsd: rawNanos == null ? null : rawNanos / USD_NANOS,
|
|
8087
|
+
multiplierBps: Number(row.multiplier_bps ?? SCHEDULED_AGENT_LLM_MARKUP_BPS),
|
|
8088
|
+
balanceMc,
|
|
8089
|
+
...reason ? { reason } : {},
|
|
8090
|
+
duplicate
|
|
8091
|
+
};
|
|
8092
|
+
}
|
|
8093
|
+
async function authorizeScheduledRun(args) {
|
|
8094
|
+
const identity = normalizeIdentity(args.identity);
|
|
8095
|
+
const idempotencyKey = normalizeIdempotencyKey(args.idempotencyKey);
|
|
8096
|
+
const user = await getUserByEmail(identity);
|
|
8097
|
+
if (!user) throw new UnifiedBillingError("user_not_found", "billing identity was not found", 404);
|
|
8098
|
+
const pending = await getDb().execute({
|
|
8099
|
+
sql: "SELECT id FROM billing_events WHERE user_id = ? AND billing_class = ? AND status = 'cost_pending' LIMIT 1",
|
|
8100
|
+
args: [user.id, SCHEDULED_RUN_BILLING_CLASS]
|
|
8101
|
+
});
|
|
8102
|
+
if (pending.rows.length > 0) {
|
|
8103
|
+
throw new UnifiedBillingError("pending_cost", "a previous scheduled run has an unresolved cost", 409);
|
|
8104
|
+
}
|
|
8105
|
+
const existing = await getDb().execute({
|
|
8106
|
+
sql: "SELECT id, user_id, idempotency_key, status, minimum_amount_mc, maximum_amount_mc, token_hash, expires_at FROM billing_authorizations WHERE idempotency_key = ? LIMIT 1",
|
|
8107
|
+
args: [idempotencyKey]
|
|
8108
|
+
});
|
|
8109
|
+
if (existing.rows[0]) {
|
|
8110
|
+
const row = existing.rows[0];
|
|
8111
|
+
if (Number(row.user_id) !== user.id) {
|
|
8112
|
+
throw new UnifiedBillingError(
|
|
8113
|
+
"invalid_request",
|
|
8114
|
+
"the idempotency key is already in use by another billing identity",
|
|
8115
|
+
409
|
|
8116
|
+
);
|
|
8117
|
+
}
|
|
8118
|
+
const balanceMc2 = await reconcileBalanceMc(user.id, user.balance_mc);
|
|
8119
|
+
return mapAuthorization(row, balanceMc2);
|
|
8120
|
+
}
|
|
8121
|
+
const maximumCredits = args.maximumCredits ?? SCHEDULED_RUN_DEFAULT_MAX_CREDITS;
|
|
8122
|
+
if (!Number.isFinite(maximumCredits) || maximumCredits < SCHEDULED_RUN_BASE_MC / MC_PER_CREDIT) {
|
|
8123
|
+
throw new UnifiedBillingError("invalid_request", "maximumCredits must cover the scheduled-run base charge", 400);
|
|
8124
|
+
}
|
|
8125
|
+
const maximumMc = Math.ceil(maximumCredits * MC_PER_CREDIT);
|
|
8126
|
+
const balanceMc = await reconcileBalanceMc(user.id, user.balance_mc);
|
|
8127
|
+
if (balanceMc < SCHEDULED_RUN_BASE_MC) {
|
|
8128
|
+
throw new UnifiedBillingError(
|
|
8129
|
+
"insufficient_credits",
|
|
8130
|
+
"the wallet cannot cover the scheduled-run base charge",
|
|
8131
|
+
402,
|
|
8132
|
+
{ balanceMc, requiredMc: SCHEDULED_RUN_BASE_MC }
|
|
8133
|
+
);
|
|
8134
|
+
}
|
|
8135
|
+
const id = randomUUID();
|
|
8136
|
+
const token = tokenFor(id, idempotencyKey);
|
|
8137
|
+
const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
|
|
8138
|
+
const inserted = await getDb().execute({
|
|
8139
|
+
sql: "INSERT OR IGNORE INTO billing_authorizations (id, user_id, idempotency_key, billing_class, status, minimum_amount_mc, maximum_amount_mc, token_hash, metadata, expires_at) VALUES (?, ?, ?, ?, 'authorized', ?, ?, ?, ?, ?)",
|
|
8140
|
+
args: [
|
|
8141
|
+
id,
|
|
8142
|
+
user.id,
|
|
8143
|
+
idempotencyKey,
|
|
8144
|
+
SCHEDULED_RUN_BILLING_CLASS,
|
|
8145
|
+
SCHEDULED_RUN_BASE_MC,
|
|
8146
|
+
maximumMc,
|
|
8147
|
+
tokenHash(token),
|
|
8148
|
+
safeMetadata(args.metadata),
|
|
8149
|
+
expiresAt
|
|
8150
|
+
]
|
|
8151
|
+
});
|
|
8152
|
+
if (inserted.rowsAffected === 0) {
|
|
8153
|
+
const raced = await getDb().execute({
|
|
8154
|
+
sql: "SELECT id, user_id, idempotency_key, status, minimum_amount_mc, maximum_amount_mc, token_hash, expires_at FROM billing_authorizations WHERE idempotency_key = ? LIMIT 1",
|
|
8155
|
+
args: [idempotencyKey]
|
|
8156
|
+
});
|
|
8157
|
+
if (!raced.rows[0]) throw new Error("billing authorization insert raced without a readable winner");
|
|
8158
|
+
const row = raced.rows[0];
|
|
8159
|
+
if (Number(row.user_id) !== user.id) {
|
|
8160
|
+
throw new UnifiedBillingError(
|
|
8161
|
+
"invalid_request",
|
|
8162
|
+
"the idempotency key is already in use by another billing identity",
|
|
8163
|
+
409
|
|
8164
|
+
);
|
|
8165
|
+
}
|
|
8166
|
+
return mapAuthorization(row, balanceMc);
|
|
8167
|
+
}
|
|
8168
|
+
return {
|
|
8169
|
+
id,
|
|
8170
|
+
token,
|
|
8171
|
+
status: "authorized",
|
|
8172
|
+
minimumMc: SCHEDULED_RUN_BASE_MC,
|
|
8173
|
+
maximumMc,
|
|
8174
|
+
minimumCredits: SCHEDULED_RUN_BASE_MC / MC_PER_CREDIT,
|
|
8175
|
+
maximumCredits,
|
|
8176
|
+
balanceMc,
|
|
8177
|
+
expiresAt,
|
|
8178
|
+
reservationMode: "balance_check"
|
|
8179
|
+
};
|
|
8180
|
+
}
|
|
8181
|
+
async function settleScheduledRun(args) {
|
|
8182
|
+
const identity = normalizeIdentity(args.identity);
|
|
8183
|
+
const user = await getUserByEmail(identity);
|
|
8184
|
+
if (!user) throw new UnifiedBillingError("user_not_found", "billing identity was not found", 404);
|
|
8185
|
+
const authorization = await authorizationById(args.authorizationId);
|
|
8186
|
+
if (!authorization || Number(authorization.user_id) !== user.id) {
|
|
8187
|
+
throw new UnifiedBillingError("authorization_not_found", "billing authorization was not found", 404);
|
|
8188
|
+
}
|
|
8189
|
+
verifyToken(authorization, args.authorizationToken);
|
|
8190
|
+
const settlementKey = authorization.id + ":settlement";
|
|
8191
|
+
const existing = await eventByKey(settlementKey);
|
|
8192
|
+
if (existing) {
|
|
8193
|
+
const balanceMc = await reconcileBalanceMc(user.id, user.balance_mc);
|
|
8194
|
+
if (existing.status === "settling") {
|
|
8195
|
+
throw new UnifiedBillingError("settlement_in_progress", "scheduled-run settlement is already in progress", 409);
|
|
8196
|
+
}
|
|
8197
|
+
return mapEvent(existing, balanceMc, true);
|
|
8198
|
+
}
|
|
8199
|
+
if (authorization.status !== "authorized") {
|
|
8200
|
+
throw new UnifiedBillingError("authorization_closed", "billing authorization is no longer open", 409);
|
|
8201
|
+
}
|
|
8202
|
+
if (Date.parse(authorization.expires_at) <= Date.now()) {
|
|
8203
|
+
await getDb().execute({
|
|
8204
|
+
sql: "UPDATE billing_authorizations SET status = 'expired' WHERE id = ? AND status = 'authorized'",
|
|
8205
|
+
args: [authorization.id]
|
|
8206
|
+
});
|
|
8207
|
+
throw new UnifiedBillingError("authorization_expired", "billing authorization expired before settlement", 409);
|
|
8208
|
+
}
|
|
8209
|
+
const rawCostUsd = args.rawCostUsd ?? null;
|
|
8210
|
+
if (args.modelCostStatus === "reported" && (rawCostUsd == null || !Number.isFinite(rawCostUsd) || rawCostUsd < 0)) {
|
|
8211
|
+
throw new UnifiedBillingError("invalid_request", "reported model cost must be a finite non-negative number", 400);
|
|
8212
|
+
}
|
|
8213
|
+
if (args.modelCostStatus === "not_applicable" && rawCostUsd != null && rawCostUsd !== 0) {
|
|
8214
|
+
throw new UnifiedBillingError("invalid_request", "not_applicable model cost cannot include a non-zero cost", 400);
|
|
8215
|
+
}
|
|
8216
|
+
const eventId = randomUUID();
|
|
8217
|
+
const rawCostNanos = rawCostUsd == null ? null : Math.round(rawCostUsd * USD_NANOS);
|
|
8218
|
+
const modelMc = rawCostUsd == null ? 0 : vendorCostUsdToMc(rawCostUsd, SCHEDULED_AGENT_LLM_MARKUP_BPS);
|
|
8219
|
+
const totalMc = SCHEDULED_RUN_BASE_MC + modelMc;
|
|
8220
|
+
const pendingReason = args.modelCostStatus === "missing" ? "openrouter_cost_missing" : totalMc > Number(authorization.maximum_amount_mc) ? "max_credits_per_run_exceeded" : null;
|
|
8221
|
+
if (pendingReason) {
|
|
8222
|
+
await getDb().execute({
|
|
8223
|
+
sql: "INSERT INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, input_tokens, output_tokens, cached_tokens, model_steps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'cost_pending', ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
8224
|
+
args: [
|
|
8225
|
+
eventId,
|
|
8226
|
+
user.id,
|
|
8227
|
+
authorization.id,
|
|
8228
|
+
settlementKey,
|
|
8229
|
+
SCHEDULED_RUN_BILLING_CLASS,
|
|
8230
|
+
SCHEDULED_RUN_SOURCE_SURFACE,
|
|
8231
|
+
totalMc,
|
|
8232
|
+
rawCostNanos,
|
|
8233
|
+
SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
8234
|
+
Math.max(0, Math.round(args.inputTokens ?? 0)),
|
|
8235
|
+
Math.max(0, Math.round(args.outputTokens ?? 0)),
|
|
8236
|
+
Math.max(0, Math.round(args.cachedTokens ?? 0)),
|
|
8237
|
+
Math.max(0, Math.round(args.modelSteps ?? 0)),
|
|
8238
|
+
safeMetadata({ ...args.metadata, reason: pendingReason })
|
|
8239
|
+
]
|
|
8240
|
+
});
|
|
8241
|
+
await getDb().execute({
|
|
8242
|
+
sql: "UPDATE billing_authorizations SET status = 'cost_pending', settled_at = datetime('now') WHERE id = ?",
|
|
8243
|
+
args: [authorization.id]
|
|
8244
|
+
});
|
|
8245
|
+
return {
|
|
8246
|
+
id: eventId,
|
|
8247
|
+
authorizationId: authorization.id,
|
|
8248
|
+
status: "cost_pending",
|
|
8249
|
+
amountMc: totalMc,
|
|
8250
|
+
credits: totalMc / MC_PER_CREDIT,
|
|
8251
|
+
baseMc: SCHEDULED_RUN_BASE_MC,
|
|
8252
|
+
modelMc,
|
|
8253
|
+
rawCostUsd,
|
|
8254
|
+
multiplierBps: SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
8255
|
+
balanceMc: await reconcileBalanceMc(user.id, user.balance_mc),
|
|
8256
|
+
reason: pendingReason,
|
|
8257
|
+
duplicate: false
|
|
8258
|
+
};
|
|
8259
|
+
}
|
|
8260
|
+
const claimed = await getDb().execute({
|
|
8261
|
+
sql: "INSERT OR IGNORE INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, raw_cost_usd_nanos, multiplier_bps, input_tokens, output_tokens, cached_tokens, model_steps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'settling', ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
8262
|
+
args: [
|
|
8263
|
+
eventId,
|
|
8264
|
+
user.id,
|
|
8265
|
+
authorization.id,
|
|
8266
|
+
settlementKey,
|
|
8267
|
+
SCHEDULED_RUN_BILLING_CLASS,
|
|
8268
|
+
SCHEDULED_RUN_SOURCE_SURFACE,
|
|
8269
|
+
totalMc,
|
|
8270
|
+
rawCostNanos,
|
|
8271
|
+
SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
8272
|
+
Math.max(0, Math.round(args.inputTokens ?? 0)),
|
|
8273
|
+
Math.max(0, Math.round(args.outputTokens ?? 0)),
|
|
8274
|
+
Math.max(0, Math.round(args.cachedTokens ?? 0)),
|
|
8275
|
+
Math.max(0, Math.round(args.modelSteps ?? 0)),
|
|
8276
|
+
safeMetadata(args.metadata)
|
|
8277
|
+
]
|
|
8278
|
+
});
|
|
8279
|
+
if (claimed.rowsAffected === 0) {
|
|
8280
|
+
const raced = await eventByKey(settlementKey);
|
|
8281
|
+
if (!raced) throw new Error("billing settlement raced without a readable winner");
|
|
8282
|
+
if (raced.status === "settling") {
|
|
8283
|
+
throw new UnifiedBillingError("settlement_in_progress", "scheduled-run settlement is already in progress", 409);
|
|
8284
|
+
}
|
|
8285
|
+
return mapEvent(raced, await reconcileBalanceMc(user.id, user.balance_mc), true);
|
|
8286
|
+
}
|
|
8287
|
+
const debit = await debitMc(
|
|
8288
|
+
user.id,
|
|
8289
|
+
totalMc,
|
|
8290
|
+
LedgerOperation.SCHEDULED_RUN,
|
|
8291
|
+
"scheduled-run:" + authorization.id
|
|
8292
|
+
);
|
|
8293
|
+
if (!debit.ok) {
|
|
8294
|
+
const metadata = safeMetadata({ ...args.metadata, reason: "insufficient_credits_at_settlement" });
|
|
8295
|
+
await getDb().execute({
|
|
8296
|
+
sql: "UPDATE billing_events SET status = 'cost_pending', metadata = ? WHERE id = ?",
|
|
8297
|
+
args: [metadata, eventId]
|
|
8298
|
+
});
|
|
8299
|
+
await getDb().execute({
|
|
8300
|
+
sql: "UPDATE billing_authorizations SET status = 'cost_pending', settled_at = datetime('now') WHERE id = ?",
|
|
8301
|
+
args: [authorization.id]
|
|
8302
|
+
});
|
|
8303
|
+
return {
|
|
8304
|
+
id: eventId,
|
|
8305
|
+
authorizationId: authorization.id,
|
|
8306
|
+
status: "cost_pending",
|
|
8307
|
+
amountMc: totalMc,
|
|
8308
|
+
credits: totalMc / MC_PER_CREDIT,
|
|
8309
|
+
baseMc: SCHEDULED_RUN_BASE_MC,
|
|
8310
|
+
modelMc,
|
|
8311
|
+
rawCostUsd,
|
|
8312
|
+
multiplierBps: SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
8313
|
+
balanceMc: debit.balance_mc,
|
|
8314
|
+
reason: "insufficient_credits_at_settlement",
|
|
8315
|
+
duplicate: false
|
|
8316
|
+
};
|
|
8317
|
+
}
|
|
8318
|
+
await getDb().execute({
|
|
8319
|
+
sql: "UPDATE billing_events SET status = 'settled', settled_at = datetime('now') WHERE id = ?",
|
|
8320
|
+
args: [eventId]
|
|
8321
|
+
});
|
|
8322
|
+
await getDb().execute({
|
|
8323
|
+
sql: "UPDATE billing_authorizations SET status = 'settled', settled_at = datetime('now') WHERE id = ?",
|
|
8324
|
+
args: [authorization.id]
|
|
8325
|
+
});
|
|
8326
|
+
return {
|
|
8327
|
+
id: eventId,
|
|
8328
|
+
authorizationId: authorization.id,
|
|
8329
|
+
status: "settled",
|
|
8330
|
+
amountMc: totalMc,
|
|
8331
|
+
credits: totalMc / MC_PER_CREDIT,
|
|
8332
|
+
baseMc: SCHEDULED_RUN_BASE_MC,
|
|
8333
|
+
modelMc,
|
|
8334
|
+
rawCostUsd,
|
|
8335
|
+
multiplierBps: SCHEDULED_AGENT_LLM_MARKUP_BPS,
|
|
8336
|
+
balanceMc: debit.balance_mc,
|
|
8337
|
+
duplicate: false
|
|
8338
|
+
};
|
|
8339
|
+
}
|
|
8340
|
+
async function voidScheduledRunAuthorization(args) {
|
|
8341
|
+
const identity = normalizeIdentity(args.identity);
|
|
8342
|
+
const user = await getUserByEmail(identity);
|
|
8343
|
+
if (!user) throw new UnifiedBillingError("user_not_found", "billing identity was not found", 404);
|
|
8344
|
+
const authorization = await authorizationById(args.authorizationId);
|
|
8345
|
+
if (!authorization || Number(authorization.user_id) !== user.id) {
|
|
8346
|
+
throw new UnifiedBillingError("authorization_not_found", "billing authorization was not found", 404);
|
|
8347
|
+
}
|
|
8348
|
+
verifyToken(authorization, args.authorizationToken);
|
|
8349
|
+
if (authorization.status === "authorized") {
|
|
8350
|
+
await getDb().execute({
|
|
8351
|
+
sql: "UPDATE billing_authorizations SET status = 'void', metadata = ?, settled_at = datetime('now') WHERE id = ? AND status = 'authorized'",
|
|
8352
|
+
args: [safeMetadata({ reason: args.reason ?? "voided_before_execution" }), authorization.id]
|
|
8353
|
+
});
|
|
8354
|
+
return { ok: true, status: "void" };
|
|
8355
|
+
}
|
|
8356
|
+
return { ok: true, status: authorization.status };
|
|
8357
|
+
}
|
|
8358
|
+
|
|
8359
|
+
// src/api/internal-billing-routes.ts
|
|
8360
|
+
var internalBillingApp = new Hono3();
|
|
8361
|
+
var metadataSchema = z10.record(z10.unknown()).optional();
|
|
8362
|
+
var authorizeSchema = z10.object({
|
|
8363
|
+
identity: z10.string().min(1).max(320),
|
|
8364
|
+
idempotencyKey: z10.string().min(1).max(500),
|
|
8365
|
+
maximumCredits: z10.number().finite().positive().optional(),
|
|
8366
|
+
metadata: metadataSchema
|
|
8367
|
+
});
|
|
8368
|
+
var settleSchema = z10.object({
|
|
8369
|
+
identity: z10.string().min(1).max(320),
|
|
8370
|
+
authorizationId: z10.string().uuid(),
|
|
8371
|
+
authorizationToken: z10.string().min(20).max(500),
|
|
8372
|
+
modelCostStatus: z10.enum(["reported", "missing", "not_applicable"]),
|
|
8373
|
+
rawCostUsd: z10.number().finite().nonnegative().nullable().optional(),
|
|
8374
|
+
inputTokens: z10.number().finite().nonnegative().optional(),
|
|
8375
|
+
outputTokens: z10.number().finite().nonnegative().optional(),
|
|
8376
|
+
cachedTokens: z10.number().finite().nonnegative().optional(),
|
|
8377
|
+
modelSteps: z10.number().finite().nonnegative().optional(),
|
|
8378
|
+
metadata: metadataSchema
|
|
8379
|
+
});
|
|
8380
|
+
var voidSchema = z10.object({
|
|
8381
|
+
identity: z10.string().min(1).max(320),
|
|
8382
|
+
authorizationId: z10.string().uuid(),
|
|
8383
|
+
authorizationToken: z10.string().min(20).max(500),
|
|
8384
|
+
reason: z10.string().max(500).optional()
|
|
8385
|
+
});
|
|
8386
|
+
function internalRequestAuthorized(header) {
|
|
8387
|
+
const expected = process.env.INTERNAL_BILLING_SECRET ?? process.env.INTERNAL_MEMORY_DEBIT_SECRET;
|
|
8388
|
+
return Boolean(expected && header && header === expected);
|
|
8389
|
+
}
|
|
8390
|
+
function billingError(error) {
|
|
8391
|
+
if (error instanceof UnifiedBillingError) {
|
|
8392
|
+
return {
|
|
8393
|
+
body: {
|
|
8394
|
+
ok: false,
|
|
8395
|
+
error_code: error.code,
|
|
8396
|
+
error: error.message,
|
|
8397
|
+
...error.details
|
|
8398
|
+
},
|
|
8399
|
+
status: error.httpStatus
|
|
8400
|
+
};
|
|
8401
|
+
}
|
|
8402
|
+
console.error("[internal-billing] unexpected error", error instanceof Error ? error.message : error);
|
|
8403
|
+
return {
|
|
8404
|
+
body: { ok: false, error_code: "internal_error", error: "internal billing operation failed" },
|
|
8405
|
+
status: 500
|
|
8406
|
+
};
|
|
8407
|
+
}
|
|
8408
|
+
internalBillingApp.use("*", async (c, next) => {
|
|
8409
|
+
if (!internalRequestAuthorized(c.req.header("x-internal-secret"))) {
|
|
8410
|
+
return c.json({ ok: false, error_code: "unauthorized", error: "unauthorized" }, 401);
|
|
8411
|
+
}
|
|
8412
|
+
await next();
|
|
8413
|
+
});
|
|
8414
|
+
internalBillingApp.post("/scheduled-run/authorize", async (c) => {
|
|
8415
|
+
const parsed = authorizeSchema.safeParse(await c.req.json().catch(() => null));
|
|
8416
|
+
if (!parsed.success) {
|
|
8417
|
+
return c.json({ ok: false, error_code: "invalid_request", error: "invalid billing authorization request" }, 400);
|
|
8418
|
+
}
|
|
8419
|
+
try {
|
|
8420
|
+
const authorization = await authorizeScheduledRun(parsed.data);
|
|
8421
|
+
return c.json({ ok: true, authorization });
|
|
8422
|
+
} catch (error) {
|
|
8423
|
+
const response = billingError(error);
|
|
8424
|
+
return c.json(response.body, response.status);
|
|
8425
|
+
}
|
|
8426
|
+
});
|
|
8427
|
+
internalBillingApp.post("/scheduled-run/settle", async (c) => {
|
|
8428
|
+
const parsed = settleSchema.safeParse(await c.req.json().catch(() => null));
|
|
8429
|
+
if (!parsed.success) {
|
|
8430
|
+
return c.json({ ok: false, error_code: "invalid_request", error: "invalid scheduled-run settlement request" }, 400);
|
|
8431
|
+
}
|
|
8432
|
+
try {
|
|
8433
|
+
const settlement = await settleScheduledRun(parsed.data);
|
|
8434
|
+
return c.json({ ok: true, settlement });
|
|
8435
|
+
} catch (error) {
|
|
8436
|
+
const response = billingError(error);
|
|
8437
|
+
return c.json(response.body, response.status);
|
|
8438
|
+
}
|
|
8439
|
+
});
|
|
8440
|
+
internalBillingApp.post("/scheduled-run/void", async (c) => {
|
|
8441
|
+
const parsed = voidSchema.safeParse(await c.req.json().catch(() => null));
|
|
8442
|
+
if (!parsed.success) {
|
|
8443
|
+
return c.json({ ok: false, error_code: "invalid_request", error: "invalid billing void request" }, 400);
|
|
8444
|
+
}
|
|
8445
|
+
try {
|
|
8446
|
+
return c.json(await voidScheduledRunAuthorization(parsed.data));
|
|
8447
|
+
} catch (error) {
|
|
8448
|
+
const response = billingError(error);
|
|
8449
|
+
return c.json(response.body, response.status);
|
|
8450
|
+
}
|
|
8451
|
+
});
|
|
8452
|
+
|
|
8453
|
+
// src/api/youtube-routes.ts
|
|
8454
|
+
import { Hono as Hono4 } from "hono";
|
|
8253
8455
|
|
|
8254
8456
|
// src/youtube/youtube-harvest.ts
|
|
8255
8457
|
import { promises as fs } from "fs";
|
|
@@ -8257,17 +8459,17 @@ import path4 from "path";
|
|
|
8257
8459
|
import Papa from "papaparse";
|
|
8258
8460
|
|
|
8259
8461
|
// src/youtube/schemas.ts
|
|
8260
|
-
import { z as
|
|
8261
|
-
var YouTubeHarvestOptionsSchema =
|
|
8262
|
-
mode:
|
|
8263
|
-
query:
|
|
8264
|
-
channelHandle:
|
|
8265
|
-
maxVideos:
|
|
8266
|
-
downloadMp3:
|
|
8267
|
-
outputDir:
|
|
8268
|
-
kernelApiKey:
|
|
8269
|
-
kernelProxyId:
|
|
8270
|
-
headless:
|
|
8462
|
+
import { z as z11 } from "zod";
|
|
8463
|
+
var YouTubeHarvestOptionsSchema = z11.object({
|
|
8464
|
+
mode: z11.enum(["search", "channel"]),
|
|
8465
|
+
query: z11.string().optional(),
|
|
8466
|
+
channelHandle: z11.string().optional(),
|
|
8467
|
+
maxVideos: z11.number().int().min(1).max(5e3).default(100),
|
|
8468
|
+
downloadMp3: z11.boolean().default(false),
|
|
8469
|
+
outputDir: z11.string().default("./yt-output"),
|
|
8470
|
+
kernelApiKey: z11.string().optional(),
|
|
8471
|
+
kernelProxyId: z11.string().optional(),
|
|
8472
|
+
headless: z11.boolean().default(false)
|
|
8271
8473
|
}).refine(
|
|
8272
8474
|
(o) => o.mode === "search" ? !!o.query : !!o.channelHandle,
|
|
8273
8475
|
{ message: "search mode requires query; channel mode requires channelHandle" }
|
|
@@ -9053,67 +9255,68 @@ function createApiKeyAuth() {
|
|
|
9053
9255
|
}
|
|
9054
9256
|
|
|
9055
9257
|
// src/api/server-schemas.ts
|
|
9056
|
-
import { z as
|
|
9057
|
-
var HarvestBodySchema =
|
|
9058
|
-
query:
|
|
9059
|
-
location:
|
|
9060
|
-
depth:
|
|
9061
|
-
maxQuestions:
|
|
9062
|
-
callback_url:
|
|
9063
|
-
gl:
|
|
9064
|
-
hl:
|
|
9065
|
-
device:
|
|
9066
|
-
proxyMode:
|
|
9067
|
-
proxyZip:
|
|
9068
|
-
debug:
|
|
9069
|
-
serpOnly:
|
|
9070
|
-
pages:
|
|
9071
|
-
});
|
|
9072
|
-
var ExtractUrlBodySchema =
|
|
9073
|
-
url:
|
|
9074
|
-
screenshot:
|
|
9075
|
-
screenshotDevice:
|
|
9076
|
-
extractBranding:
|
|
9077
|
-
downloadMedia:
|
|
9078
|
-
mediaTypes:
|
|
9079
|
-
allowLocal:
|
|
9080
|
-
depositToVault:
|
|
9081
|
-
vaultName:
|
|
9082
|
-
});
|
|
9083
|
-
var DiffPageBodySchema = z11.object({
|
|
9084
|
-
url: z11.string().min(1, "url is required"),
|
|
9085
|
-
allowLocal: z11.boolean().optional(),
|
|
9086
|
-
resetBaseline: z11.boolean().optional()
|
|
9087
|
-
});
|
|
9088
|
-
var MapUrlsBodySchema = z11.object({
|
|
9089
|
-
url: z11.string().min(1, "url is required"),
|
|
9090
|
-
maxUrls: z11.number().int().min(1).max(2e3).optional(),
|
|
9091
|
-
concurrency: z11.number().int().min(1).max(20).optional(),
|
|
9092
|
-
browserFallback: z11.boolean().optional(),
|
|
9093
|
-
kernelFallback: z11.boolean().optional()
|
|
9094
|
-
});
|
|
9095
|
-
var ExtractSiteBodySchema = z11.object({
|
|
9096
|
-
url: z11.string().min(1, "url is required"),
|
|
9097
|
-
maxPages: z11.number().int().min(1).max(1e4).optional(),
|
|
9098
|
-
browserFallback: z11.boolean().optional(),
|
|
9099
|
-
kernelFallback: z11.boolean().optional(),
|
|
9100
|
-
rotateProxies: z11.boolean().optional(),
|
|
9101
|
-
rotateProxyEvery: z11.number().int().min(1).max(100).optional(),
|
|
9102
|
-
background: z11.boolean().optional(),
|
|
9103
|
-
formats: z11.array(z11.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
|
|
9104
|
-
});
|
|
9105
|
-
var YoutubeHarvestBodySchema = z11.object({
|
|
9106
|
-
mode: z11.enum(["search", "channel"]),
|
|
9107
|
-
query: z11.string().optional(),
|
|
9108
|
-
channelHandle: z11.string().optional(),
|
|
9109
|
-
maxVideos: z11.number().int().min(1).max(500).optional()
|
|
9258
|
+
import { z as z12 } from "zod";
|
|
9259
|
+
var HarvestBodySchema = z12.object({
|
|
9260
|
+
query: z12.string().min(1, "query is required"),
|
|
9261
|
+
location: z12.string().optional(),
|
|
9262
|
+
depth: z12.number().int().min(1).max(4).optional(),
|
|
9263
|
+
maxQuestions: z12.number().int().min(1).optional(),
|
|
9264
|
+
callback_url: z12.string().url().optional(),
|
|
9265
|
+
gl: z12.string().optional(),
|
|
9266
|
+
hl: z12.string().optional(),
|
|
9267
|
+
device: z12.enum(["desktop", "mobile"]).optional(),
|
|
9268
|
+
proxyMode: z12.enum(["location", "configured", "none"]).optional(),
|
|
9269
|
+
proxyZip: z12.string().regex(/^\d{5}$/).optional(),
|
|
9270
|
+
debug: z12.boolean().optional(),
|
|
9271
|
+
serpOnly: z12.boolean().optional(),
|
|
9272
|
+
pages: z12.number().int().min(1).max(2).optional()
|
|
9273
|
+
});
|
|
9274
|
+
var ExtractUrlBodySchema = z12.object({
|
|
9275
|
+
url: z12.string().min(1, "url is required"),
|
|
9276
|
+
screenshot: z12.boolean().optional(),
|
|
9277
|
+
screenshotDevice: z12.enum(["desktop", "mobile"]).optional(),
|
|
9278
|
+
extractBranding: z12.boolean().optional(),
|
|
9279
|
+
downloadMedia: z12.boolean().optional(),
|
|
9280
|
+
mediaTypes: z12.array(z12.enum(["image", "video", "audio"])).optional(),
|
|
9281
|
+
allowLocal: z12.boolean().optional(),
|
|
9282
|
+
depositToVault: z12.boolean().optional(),
|
|
9283
|
+
vaultName: z12.string().trim().min(1).max(120).optional()
|
|
9110
9284
|
});
|
|
9111
|
-
var
|
|
9112
|
-
|
|
9285
|
+
var DiffPageBodySchema = z12.object({
|
|
9286
|
+
url: z12.string().min(1, "url is required"),
|
|
9287
|
+
allowLocal: z12.boolean().optional(),
|
|
9288
|
+
resetBaseline: z12.boolean().optional()
|
|
9289
|
+
});
|
|
9290
|
+
var MapUrlsBodySchema = z12.object({
|
|
9291
|
+
url: z12.string().min(1, "url is required"),
|
|
9292
|
+
maxUrls: z12.number().int().min(1).max(2e3).optional(),
|
|
9293
|
+
concurrency: z12.number().int().min(1).max(20).optional(),
|
|
9294
|
+
browserFallback: z12.boolean().optional(),
|
|
9295
|
+
kernelFallback: z12.boolean().optional()
|
|
9296
|
+
});
|
|
9297
|
+
var ExtractSiteBodySchema = z12.object({
|
|
9298
|
+
url: z12.string().min(1, "url is required"),
|
|
9299
|
+
maxPages: z12.number().int().min(1).max(1e4).optional(),
|
|
9300
|
+
browserFallback: z12.boolean().optional(),
|
|
9301
|
+
kernelFallback: z12.boolean().optional(),
|
|
9302
|
+
rotateProxies: z12.boolean().optional(),
|
|
9303
|
+
rotateProxyEvery: z12.number().int().min(1).max(100).optional(),
|
|
9304
|
+
background: z12.boolean().optional(),
|
|
9305
|
+
downloadImages: z12.boolean().optional(),
|
|
9306
|
+
formats: z12.array(z12.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
|
|
9307
|
+
});
|
|
9308
|
+
var YoutubeHarvestBodySchema = z12.object({
|
|
9309
|
+
mode: z12.enum(["search", "channel"]),
|
|
9310
|
+
query: z12.string().optional(),
|
|
9311
|
+
channelHandle: z12.string().optional(),
|
|
9312
|
+
maxVideos: z12.number().int().min(1).max(500).optional()
|
|
9313
|
+
});
|
|
9314
|
+
var YoutubeTranscribeBodySchema = z12.object({
|
|
9315
|
+
videoId: z12.string().min(1, "videoId is required")
|
|
9113
9316
|
});
|
|
9114
9317
|
|
|
9115
9318
|
// src/api/concurrency-gates.ts
|
|
9116
|
-
import { randomUUID } from "crypto";
|
|
9319
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
9117
9320
|
var DEFAULT_LOCK_TTL_SECONDS = 15 * 60;
|
|
9118
9321
|
var DEFAULT_RETRY_AFTER_SECONDS = 30;
|
|
9119
9322
|
var MAX_LOCK_TTL_SECONDS = 24 * 60 * 60;
|
|
@@ -9147,7 +9350,7 @@ function lockTtlModifier(ttlSeconds) {
|
|
|
9147
9350
|
const ttl = Number.isFinite(ttlSeconds) ? Math.max(30, Math.min(Math.trunc(Number(ttlSeconds)), MAX_LOCK_TTL_SECONDS)) : DEFAULT_LOCK_TTL_SECONDS;
|
|
9148
9351
|
return `+${ttl} seconds`;
|
|
9149
9352
|
}
|
|
9150
|
-
function
|
|
9353
|
+
function safeMetadata2(metadata) {
|
|
9151
9354
|
if (!metadata) return null;
|
|
9152
9355
|
try {
|
|
9153
9356
|
return JSON.stringify(metadata).slice(0, 4e3);
|
|
@@ -9231,7 +9434,7 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
9231
9434
|
return { ok: true, lockId: null, active: await countActiveUsageForUser(user.id), limit, operation, reused: true };
|
|
9232
9435
|
}
|
|
9233
9436
|
await expireConcurrencyLocksForUser(user.id);
|
|
9234
|
-
const lockId = `cl_${
|
|
9437
|
+
const lockId = `cl_${randomUUID2().replace(/-/g, "").slice(0, 24)}`;
|
|
9235
9438
|
const res = await getDb().execute({
|
|
9236
9439
|
sql: `INSERT INTO concurrency_locks (id, user_id, operation, status, expires_at, metadata)
|
|
9237
9440
|
SELECT ?, ?, ?, 'active', datetime('now', ?), ?
|
|
@@ -9253,7 +9456,7 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
9253
9456
|
user.id,
|
|
9254
9457
|
operation,
|
|
9255
9458
|
lockTtlModifier(options.ttlSeconds),
|
|
9256
|
-
|
|
9459
|
+
safeMetadata2(options.metadata),
|
|
9257
9460
|
user.id,
|
|
9258
9461
|
user.id,
|
|
9259
9462
|
limit
|
|
@@ -9287,7 +9490,7 @@ async function extendConcurrencyGate(lockId, ttlSeconds = DEFAULT_LOCK_TTL_SECON
|
|
|
9287
9490
|
}
|
|
9288
9491
|
|
|
9289
9492
|
// src/api/youtube-routes.ts
|
|
9290
|
-
var youtubeApp = new
|
|
9493
|
+
var youtubeApp = new Hono4();
|
|
9291
9494
|
youtubeApp.post("/harvest", createApiKeyAuth(), async (c) => {
|
|
9292
9495
|
const raw = await c.req.json().catch(() => ({}));
|
|
9293
9496
|
const parsed = YoutubeHarvestBodySchema.safeParse(raw);
|
|
@@ -9446,14 +9649,14 @@ function fmtTs(secs) {
|
|
|
9446
9649
|
}
|
|
9447
9650
|
|
|
9448
9651
|
// src/api/screenshot-routes.ts
|
|
9449
|
-
import { Hono as
|
|
9450
|
-
import { z as
|
|
9451
|
-
var ScreenshotBodySchema =
|
|
9452
|
-
url:
|
|
9453
|
-
device:
|
|
9454
|
-
allowLocal:
|
|
9652
|
+
import { Hono as Hono5 } from "hono";
|
|
9653
|
+
import { z as z13 } from "zod";
|
|
9654
|
+
var ScreenshotBodySchema = z13.object({
|
|
9655
|
+
url: z13.string().trim().min(1, "url is required"),
|
|
9656
|
+
device: z13.string().trim().optional(),
|
|
9657
|
+
allowLocal: z13.boolean().optional()
|
|
9455
9658
|
});
|
|
9456
|
-
var screenshotApp = new
|
|
9659
|
+
var screenshotApp = new Hono5();
|
|
9457
9660
|
screenshotApp.use("*", createApiKeyAuth());
|
|
9458
9661
|
screenshotApp.post("/", async (c) => {
|
|
9459
9662
|
const raw = await c.req.json().catch(() => ({}));
|
|
@@ -9510,8 +9713,8 @@ screenshotApp.post("/", async (c) => {
|
|
|
9510
9713
|
});
|
|
9511
9714
|
|
|
9512
9715
|
// src/api/facebook-ad-routes.ts
|
|
9513
|
-
import { Hono as
|
|
9514
|
-
import { z as
|
|
9716
|
+
import { Hono as Hono6 } from "hono";
|
|
9717
|
+
import { z as z14 } from "zod";
|
|
9515
9718
|
|
|
9516
9719
|
// src/extractor/FacebookAdExtractor.ts
|
|
9517
9720
|
var SPRITE_Y_TO_PLATFORM = {
|
|
@@ -10404,35 +10607,35 @@ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "b
|
|
|
10404
10607
|
|
|
10405
10608
|
// src/api/facebook-ad-routes.ts
|
|
10406
10609
|
import { fal } from "@fal-ai/client";
|
|
10407
|
-
var FacebookAdBodySchema =
|
|
10408
|
-
url:
|
|
10409
|
-
libraryId:
|
|
10410
|
-
openModal:
|
|
10610
|
+
var FacebookAdBodySchema = z14.object({
|
|
10611
|
+
url: z14.string().trim().optional(),
|
|
10612
|
+
libraryId: z14.string().trim().optional(),
|
|
10613
|
+
openModal: z14.boolean().optional()
|
|
10411
10614
|
}).refine((d) => !!d.url || !!d.libraryId, { message: "url or libraryId is required" });
|
|
10412
|
-
var FacebookPageIntelBodySchema =
|
|
10413
|
-
pageId:
|
|
10414
|
-
query:
|
|
10415
|
-
libraryId:
|
|
10416
|
-
maxAds:
|
|
10417
|
-
country:
|
|
10615
|
+
var FacebookPageIntelBodySchema = z14.object({
|
|
10616
|
+
pageId: z14.string().trim().optional(),
|
|
10617
|
+
query: z14.string().trim().optional(),
|
|
10618
|
+
libraryId: z14.string().trim().optional(),
|
|
10619
|
+
maxAds: z14.number().int().min(1).max(200).optional(),
|
|
10620
|
+
country: z14.string().trim().toUpperCase().optional()
|
|
10418
10621
|
}).refine((d) => !!d.pageId || !!d.query || !!d.libraryId, {
|
|
10419
10622
|
message: "pageId, libraryId, or query is required"
|
|
10420
10623
|
});
|
|
10421
|
-
var FacebookTranscribeBodySchema =
|
|
10422
|
-
videoUrl:
|
|
10624
|
+
var FacebookTranscribeBodySchema = z14.object({
|
|
10625
|
+
videoUrl: z14.string().trim().min(1, "videoUrl is required")
|
|
10423
10626
|
});
|
|
10424
|
-
var FacebookVideoTranscribeBodySchema =
|
|
10425
|
-
url:
|
|
10426
|
-
quality:
|
|
10627
|
+
var FacebookVideoTranscribeBodySchema = z14.object({
|
|
10628
|
+
url: z14.string().trim().min(1, "url is required"),
|
|
10629
|
+
quality: z14.enum(["best", "hd", "sd"]).default("best")
|
|
10427
10630
|
});
|
|
10428
|
-
var FacebookSearchBodySchema =
|
|
10429
|
-
query:
|
|
10430
|
-
country:
|
|
10431
|
-
maxResults:
|
|
10631
|
+
var FacebookSearchBodySchema = z14.object({
|
|
10632
|
+
query: z14.string().trim().min(1, "query is required"),
|
|
10633
|
+
country: z14.string().trim().toUpperCase().optional(),
|
|
10634
|
+
maxResults: z14.number().int().min(1).max(20).optional()
|
|
10432
10635
|
});
|
|
10433
|
-
var FacebookMediaBodySchema =
|
|
10434
|
-
url:
|
|
10435
|
-
filename:
|
|
10636
|
+
var FacebookMediaBodySchema = z14.object({
|
|
10637
|
+
url: z14.string().trim().min(1, "url is required"),
|
|
10638
|
+
filename: z14.string().trim().optional()
|
|
10436
10639
|
});
|
|
10437
10640
|
function invalidRequest(message) {
|
|
10438
10641
|
return { error_code: "invalid_request", message };
|
|
@@ -10514,7 +10717,7 @@ async function kernelLaunchOptsResidential() {
|
|
|
10514
10717
|
}
|
|
10515
10718
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
10516
10719
|
}
|
|
10517
|
-
var facebookAdApp = new
|
|
10720
|
+
var facebookAdApp = new Hono6();
|
|
10518
10721
|
facebookAdApp.post("/ad", createApiKeyAuth(), async (c) => {
|
|
10519
10722
|
const raw = await c.req.json().catch(() => ({}));
|
|
10520
10723
|
const parsed = FacebookAdBodySchema.safeParse(raw);
|
|
@@ -10858,8 +11061,8 @@ facebookAdApp.post("/media", createApiKeyAuth(), async (c) => {
|
|
|
10858
11061
|
});
|
|
10859
11062
|
|
|
10860
11063
|
// src/api/google-ads-routes.ts
|
|
10861
|
-
import { Hono as
|
|
10862
|
-
import { z as
|
|
11064
|
+
import { Hono as Hono7 } from "hono";
|
|
11065
|
+
import { z as z15 } from "zod";
|
|
10863
11066
|
|
|
10864
11067
|
// src/extractor/GoogleAdsExtractor.ts
|
|
10865
11068
|
var ORIGIN = "https://adstransparency.google.com";
|
|
@@ -11029,19 +11232,19 @@ var GoogleAdsExtractor = class {
|
|
|
11029
11232
|
|
|
11030
11233
|
// src/api/google-ads-routes.ts
|
|
11031
11234
|
import { fal as fal2 } from "@fal-ai/client";
|
|
11032
|
-
var GoogleAdsSearchBodySchema =
|
|
11033
|
-
query:
|
|
11034
|
-
region:
|
|
11035
|
-
maxResults:
|
|
11235
|
+
var GoogleAdsSearchBodySchema = z15.object({
|
|
11236
|
+
query: z15.string().trim().min(1, "query is required"),
|
|
11237
|
+
region: z15.string().trim().toUpperCase().length(2).optional(),
|
|
11238
|
+
maxResults: z15.number().int().min(1).max(20).optional()
|
|
11036
11239
|
});
|
|
11037
|
-
var GoogleAdsPageIntelBodySchema =
|
|
11038
|
-
advertiserId:
|
|
11039
|
-
domain:
|
|
11040
|
-
region:
|
|
11041
|
-
maxAds:
|
|
11240
|
+
var GoogleAdsPageIntelBodySchema = z15.object({
|
|
11241
|
+
advertiserId: z15.string().trim().optional(),
|
|
11242
|
+
domain: z15.string().trim().optional(),
|
|
11243
|
+
region: z15.string().trim().toUpperCase().length(2).optional(),
|
|
11244
|
+
maxAds: z15.number().int().min(1).max(200).optional()
|
|
11042
11245
|
}).refine((d) => !!d.advertiserId || !!d.domain, { message: "advertiserId or domain is required" });
|
|
11043
|
-
var GoogleAdsTranscribeBodySchema =
|
|
11044
|
-
videoUrl:
|
|
11246
|
+
var GoogleAdsTranscribeBodySchema = z15.object({
|
|
11247
|
+
videoUrl: z15.string().trim().min(1, "videoUrl is required")
|
|
11045
11248
|
});
|
|
11046
11249
|
function invalidRequest2(message) {
|
|
11047
11250
|
return { error_code: "invalid_request", message };
|
|
@@ -11067,7 +11270,7 @@ async function kernelLaunchOptsResidential2() {
|
|
|
11067
11270
|
function isBlockedMessage(msg) {
|
|
11068
11271
|
return msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha");
|
|
11069
11272
|
}
|
|
11070
|
-
var googleAdsApp = new
|
|
11273
|
+
var googleAdsApp = new Hono7();
|
|
11071
11274
|
googleAdsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
11072
11275
|
const raw = await c.req.json().catch(() => ({}));
|
|
11073
11276
|
const parsed = GoogleAdsSearchBodySchema.safeParse(raw);
|
|
@@ -11183,11 +11386,11 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
|
11183
11386
|
});
|
|
11184
11387
|
|
|
11185
11388
|
// src/api/instagram-routes.ts
|
|
11186
|
-
import { Hono as
|
|
11187
|
-
import { z as
|
|
11389
|
+
import { Hono as Hono8 } from "hono";
|
|
11390
|
+
import { z as z16 } from "zod";
|
|
11188
11391
|
import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
|
|
11189
11392
|
import { tmpdir } from "os";
|
|
11190
|
-
import { join as
|
|
11393
|
+
import { join as join3 } from "path";
|
|
11191
11394
|
import { spawn } from "child_process";
|
|
11192
11395
|
|
|
11193
11396
|
// src/extractor/InstagramContentExtractor.ts
|
|
@@ -11546,32 +11749,32 @@ async function extractInstagramMediaFromPage(page, sourceUrl, capturedMediaUrls
|
|
|
11546
11749
|
}
|
|
11547
11750
|
|
|
11548
11751
|
// src/api/instagram-routes.ts
|
|
11549
|
-
var InstagramProfileContentBodySchema =
|
|
11550
|
-
handle:
|
|
11551
|
-
url:
|
|
11552
|
-
profile:
|
|
11553
|
-
saveProfileChanges:
|
|
11554
|
-
save_profile_changes:
|
|
11555
|
-
maxItems:
|
|
11556
|
-
maxScrolls:
|
|
11557
|
-
scrollDelayMs:
|
|
11558
|
-
stableScrollLimit:
|
|
11752
|
+
var InstagramProfileContentBodySchema = z16.object({
|
|
11753
|
+
handle: z16.string().trim().optional(),
|
|
11754
|
+
url: z16.string().trim().optional(),
|
|
11755
|
+
profile: z16.string().trim().min(1).optional(),
|
|
11756
|
+
saveProfileChanges: z16.boolean().optional(),
|
|
11757
|
+
save_profile_changes: z16.boolean().optional(),
|
|
11758
|
+
maxItems: z16.number().int().min(1).max(2e3).default(50),
|
|
11759
|
+
maxScrolls: z16.number().int().min(0).max(250).default(10),
|
|
11760
|
+
scrollDelayMs: z16.number().int().min(250).max(5e3).default(1200),
|
|
11761
|
+
stableScrollLimit: z16.number().int().min(1).max(10).default(4)
|
|
11559
11762
|
}).refine((d) => !!d.handle || !!d.url, {
|
|
11560
11763
|
message: "handle or url is required"
|
|
11561
11764
|
});
|
|
11562
|
-
var InstagramMediaTypeSchema =
|
|
11563
|
-
var InstagramMediaDownloadBodySchema =
|
|
11564
|
-
url:
|
|
11565
|
-
profile:
|
|
11566
|
-
saveProfileChanges:
|
|
11567
|
-
save_profile_changes:
|
|
11568
|
-
mediaTypes:
|
|
11569
|
-
downloadMedia:
|
|
11570
|
-
downloadAllTracks:
|
|
11571
|
-
includeTranscript:
|
|
11572
|
-
mux:
|
|
11573
|
-
});
|
|
11574
|
-
var instagramApp = new
|
|
11765
|
+
var InstagramMediaTypeSchema = z16.enum(["image", "video", "audio"]);
|
|
11766
|
+
var InstagramMediaDownloadBodySchema = z16.object({
|
|
11767
|
+
url: z16.string().trim().min(1, "url is required"),
|
|
11768
|
+
profile: z16.string().trim().min(1).optional(),
|
|
11769
|
+
saveProfileChanges: z16.boolean().optional(),
|
|
11770
|
+
save_profile_changes: z16.boolean().optional(),
|
|
11771
|
+
mediaTypes: z16.array(InstagramMediaTypeSchema).default(["image", "video", "audio"]),
|
|
11772
|
+
downloadMedia: z16.boolean().default(true),
|
|
11773
|
+
downloadAllTracks: z16.boolean().default(false),
|
|
11774
|
+
includeTranscript: z16.boolean().default(false),
|
|
11775
|
+
mux: z16.boolean().default(true)
|
|
11776
|
+
});
|
|
11777
|
+
var instagramApp = new Hono8();
|
|
11575
11778
|
function invalidRequest3(message) {
|
|
11576
11779
|
return { error_code: "invalid_request", message };
|
|
11577
11780
|
}
|
|
@@ -11785,8 +11988,8 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
11785
11988
|
downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
|
|
11786
11989
|
const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
|
|
11787
11990
|
if (isMuxInput) {
|
|
11788
|
-
if (!muxWorkDir) muxWorkDir = mkdtempSync(
|
|
11789
|
-
const localPath =
|
|
11991
|
+
if (!muxWorkDir) muxWorkDir = mkdtempSync(join3(tmpdir(), "ig-mux-"));
|
|
11992
|
+
const localPath = join3(muxWorkDir, trackFilename(track, index, extraction.shortcode));
|
|
11790
11993
|
writeFileSync(localPath, bytes);
|
|
11791
11994
|
muxInputs.set(track.url, localPath);
|
|
11792
11995
|
}
|
|
@@ -11795,7 +11998,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
11795
11998
|
}
|
|
11796
11999
|
}
|
|
11797
12000
|
if (muxEligible && muxWorkDir && muxInputs.has(extraction.selectedVideoTrack.url) && muxInputs.has(extraction.selectedAudioTrack.url)) {
|
|
11798
|
-
const outPath =
|
|
12001
|
+
const outPath = join3(muxWorkDir, "muxed.mp4");
|
|
11799
12002
|
const muxed = await runFfmpegMux(muxInputs.get(extraction.selectedVideoTrack.url), muxInputs.get(extraction.selectedAudioTrack.url), outPath);
|
|
11800
12003
|
if (muxed.ok) {
|
|
11801
12004
|
const stored = await store.put(`${prefix}/muxed.mp4`, readFileSync2(outPath), "video/mp4");
|
|
@@ -11851,11 +12054,11 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
11851
12054
|
});
|
|
11852
12055
|
|
|
11853
12056
|
// src/api/reddit-routes.ts
|
|
11854
|
-
import { Hono as
|
|
11855
|
-
import { z as
|
|
11856
|
-
var RedditThreadBodySchema =
|
|
11857
|
-
url:
|
|
11858
|
-
maxComments:
|
|
12057
|
+
import { Hono as Hono9 } from "hono";
|
|
12058
|
+
import { z as z17 } from "zod";
|
|
12059
|
+
var RedditThreadBodySchema = z17.object({
|
|
12060
|
+
url: z17.string().trim().min(1, "url is required"),
|
|
12061
|
+
maxComments: z17.number().int().min(1).max(2e3).optional()
|
|
11859
12062
|
});
|
|
11860
12063
|
function invalidRequest4(message) {
|
|
11861
12064
|
return { error_code: "invalid_request", message };
|
|
@@ -11911,7 +12114,7 @@ var PARSE_REDDIT = `(() => {
|
|
|
11911
12114
|
});
|
|
11912
12115
|
return { title: title, author: author, score: score, postBody: postBody, blocked: blocked, numComments: comments.length, comments: comments };
|
|
11913
12116
|
})()`;
|
|
11914
|
-
var redditApp = new
|
|
12117
|
+
var redditApp = new Hono9();
|
|
11915
12118
|
redditApp.post("/thread", createApiKeyAuth(), async (c) => {
|
|
11916
12119
|
const raw = await c.req.json().catch(() => ({}));
|
|
11917
12120
|
const parsed = RedditThreadBodySchema.safeParse(raw);
|
|
@@ -11993,14 +12196,14 @@ redditApp.post("/thread", createApiKeyAuth(), async (c) => {
|
|
|
11993
12196
|
});
|
|
11994
12197
|
|
|
11995
12198
|
// src/api/video-routes.ts
|
|
11996
|
-
import { Hono as
|
|
11997
|
-
import { z as
|
|
12199
|
+
import { Hono as Hono10 } from "hono";
|
|
12200
|
+
import { z as z18 } from "zod";
|
|
11998
12201
|
|
|
11999
12202
|
// src/api/memory.ts
|
|
12000
12203
|
import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from "crypto";
|
|
12001
12204
|
|
|
12002
12205
|
// src/api/session.ts
|
|
12003
|
-
import { createHmac, timingSafeEqual } from "crypto";
|
|
12206
|
+
import { createHmac as createHmac2, timingSafeEqual as timingSafeEqual2 } from "crypto";
|
|
12004
12207
|
var isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
12005
12208
|
function getSessionSecret() {
|
|
12006
12209
|
const configured = process.env.SESSION_SECRET?.trim();
|
|
@@ -12012,14 +12215,14 @@ var secret = () => getSessionSecret();
|
|
|
12012
12215
|
function safeEqualHex(a, b) {
|
|
12013
12216
|
if (a.length !== b.length) return false;
|
|
12014
12217
|
try {
|
|
12015
|
-
return
|
|
12218
|
+
return timingSafeEqual2(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
12016
12219
|
} catch {
|
|
12017
12220
|
return false;
|
|
12018
12221
|
}
|
|
12019
12222
|
}
|
|
12020
12223
|
function signSession(userId) {
|
|
12021
12224
|
const payload = String(userId);
|
|
12022
|
-
const sig =
|
|
12225
|
+
const sig = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
12023
12226
|
return `${payload}.${sig}`;
|
|
12024
12227
|
}
|
|
12025
12228
|
function verifySession(token) {
|
|
@@ -12027,7 +12230,7 @@ function verifySession(token) {
|
|
|
12027
12230
|
if (dot === -1) return null;
|
|
12028
12231
|
const payload = token.slice(0, dot);
|
|
12029
12232
|
const sig = token.slice(dot + 1);
|
|
12030
|
-
const expected =
|
|
12233
|
+
const expected = createHmac2("sha256", secret()).update(payload).digest("hex");
|
|
12031
12234
|
if (!safeEqualHex(sig, expected)) return null;
|
|
12032
12235
|
const id = parseInt(payload);
|
|
12033
12236
|
return isNaN(id) ? null : id;
|
|
@@ -12173,13 +12376,13 @@ async function syncScheduleEntitlement(user, enabled, quotaPerPeriod) {
|
|
|
12173
12376
|
}
|
|
12174
12377
|
|
|
12175
12378
|
// src/api/video-routes.ts
|
|
12176
|
-
var videoApp = new
|
|
12177
|
-
var AnalyzeBodySchema =
|
|
12178
|
-
sourceUrl:
|
|
12179
|
-
intervalS:
|
|
12180
|
-
maxFrames:
|
|
12181
|
-
detail:
|
|
12182
|
-
vault:
|
|
12379
|
+
var videoApp = new Hono10();
|
|
12380
|
+
var AnalyzeBodySchema = z18.object({
|
|
12381
|
+
sourceUrl: z18.string().trim().url("sourceUrl must be a direct video file URL"),
|
|
12382
|
+
intervalS: z18.number().min(1).max(30).optional(),
|
|
12383
|
+
maxFrames: z18.number().int().min(1).max(480).optional(),
|
|
12384
|
+
detail: z18.enum(["fast", "standard", "deep"]).optional(),
|
|
12385
|
+
vault: z18.string().trim().min(1).optional()
|
|
12183
12386
|
});
|
|
12184
12387
|
function invalidRequest5(message) {
|
|
12185
12388
|
return { error_code: "invalid_request", message };
|
|
@@ -12283,7 +12486,7 @@ videoApp.post("/status", createApiKeyAuth(), async (c) => {
|
|
|
12283
12486
|
});
|
|
12284
12487
|
|
|
12285
12488
|
// src/api/maps-routes.ts
|
|
12286
|
-
import { Hono as
|
|
12489
|
+
import { Hono as Hono11 } from "hono";
|
|
12287
12490
|
|
|
12288
12491
|
// src/extractor/MapsNavigator.ts
|
|
12289
12492
|
var MapsNavigator = class {
|
|
@@ -12485,7 +12688,7 @@ var MapsReviewCollector = class {
|
|
|
12485
12688
|
};
|
|
12486
12689
|
|
|
12487
12690
|
// src/extractor/MapsExtractor.ts
|
|
12488
|
-
import { z as
|
|
12691
|
+
import { z as z19 } from "zod";
|
|
12489
12692
|
var MapsExtractor = class {
|
|
12490
12693
|
constructor(driver) {
|
|
12491
12694
|
this.driver = driver;
|
|
@@ -12693,7 +12896,7 @@ var MapsExtractor = class {
|
|
|
12693
12896
|
});
|
|
12694
12897
|
return rows;
|
|
12695
12898
|
}, { hoursTable: MapsSelectors.hoursTable, hoursTableAlt: MapsSelectors.hoursTableAlt });
|
|
12696
|
-
const result =
|
|
12899
|
+
const result = z19.array(RawMapsHoursRowSchema).safeParse(raw);
|
|
12697
12900
|
if (!result.success) {
|
|
12698
12901
|
console.warn("[MapsExtractor] hours parse failed", result.error.flatten());
|
|
12699
12902
|
return [];
|
|
@@ -12757,7 +12960,7 @@ var MapsExtractor = class {
|
|
|
12757
12960
|
});
|
|
12758
12961
|
return results;
|
|
12759
12962
|
});
|
|
12760
|
-
const result =
|
|
12963
|
+
const result = z19.array(RawMapsAboutAttributeSchema).safeParse(raw);
|
|
12761
12964
|
if (!result.success) {
|
|
12762
12965
|
console.warn("[MapsExtractor] about parse failed", result.error.flatten());
|
|
12763
12966
|
return [];
|
|
@@ -13208,7 +13411,7 @@ function mapsErrorResponse(c, err, errorCode) {
|
|
|
13208
13411
|
attempts: rotationError?.attempts ?? void 0
|
|
13209
13412
|
}, retryable ? 503 : 500);
|
|
13210
13413
|
}
|
|
13211
|
-
var mapsApp = new
|
|
13414
|
+
var mapsApp = new Hono11();
|
|
13212
13415
|
mapsApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
13213
13416
|
const user = c.get("user");
|
|
13214
13417
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -13346,11 +13549,11 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
13346
13549
|
});
|
|
13347
13550
|
|
|
13348
13551
|
// src/api/trustpilot-routes.ts
|
|
13349
|
-
import { Hono as
|
|
13350
|
-
import { z as
|
|
13351
|
-
var TrustpilotReviewsBodySchema =
|
|
13352
|
-
domain:
|
|
13353
|
-
maxPages:
|
|
13552
|
+
import { Hono as Hono12 } from "hono";
|
|
13553
|
+
import { z as z20 } from "zod";
|
|
13554
|
+
var TrustpilotReviewsBodySchema = z20.object({
|
|
13555
|
+
domain: z20.string().trim().min(1, "domain is required"),
|
|
13556
|
+
maxPages: z20.number().int().min(1).max(50).default(5)
|
|
13354
13557
|
});
|
|
13355
13558
|
function invalidRequest6(message) {
|
|
13356
13559
|
return { error_code: "invalid_request", message };
|
|
@@ -13415,7 +13618,7 @@ async function extractReviewCards(page) {
|
|
|
13415
13618
|
});
|
|
13416
13619
|
});
|
|
13417
13620
|
}
|
|
13418
|
-
var trustpilotApp = new
|
|
13621
|
+
var trustpilotApp = new Hono12();
|
|
13419
13622
|
trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
13420
13623
|
const raw = await c.req.json().catch(() => ({}));
|
|
13421
13624
|
const parsed = TrustpilotReviewsBodySchema.safeParse(raw);
|
|
@@ -13496,11 +13699,11 @@ trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
13496
13699
|
});
|
|
13497
13700
|
|
|
13498
13701
|
// src/api/g2-routes.ts
|
|
13499
|
-
import { Hono as
|
|
13500
|
-
import { z as
|
|
13501
|
-
var G2ReviewsBodySchema =
|
|
13502
|
-
product:
|
|
13503
|
-
maxPages:
|
|
13702
|
+
import { Hono as Hono13 } from "hono";
|
|
13703
|
+
import { z as z21 } from "zod";
|
|
13704
|
+
var G2ReviewsBodySchema = z21.object({
|
|
13705
|
+
product: z21.string().trim().min(1, "product is required"),
|
|
13706
|
+
maxPages: z21.number().int().min(1).max(50).default(5)
|
|
13504
13707
|
});
|
|
13505
13708
|
function invalidRequest7(message) {
|
|
13506
13709
|
return { error_code: "invalid_request", message };
|
|
@@ -13605,7 +13808,7 @@ async function extractG2ReviewCards(page) {
|
|
|
13605
13808
|
});
|
|
13606
13809
|
});
|
|
13607
13810
|
}
|
|
13608
|
-
var g2App = new
|
|
13811
|
+
var g2App = new Hono13();
|
|
13609
13812
|
g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
13610
13813
|
const raw = await c.req.json().catch(() => ({}));
|
|
13611
13814
|
const parsed = G2ReviewsBodySchema.safeParse(raw);
|
|
@@ -13702,12 +13905,12 @@ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
13702
13905
|
});
|
|
13703
13906
|
|
|
13704
13907
|
// src/api/directory-routes.ts
|
|
13705
|
-
import { Hono as
|
|
13908
|
+
import { Hono as Hono14 } from "hono";
|
|
13706
13909
|
|
|
13707
13910
|
// src/directory/directory-workflow.ts
|
|
13708
13911
|
import { mkdir as mkdir2, writeFile } from "fs/promises";
|
|
13709
|
-
import { join as
|
|
13710
|
-
import { z as
|
|
13912
|
+
import { join as join4 } from "path";
|
|
13913
|
+
import { z as z22 } from "zod";
|
|
13711
13914
|
|
|
13712
13915
|
// src/directory/location-db.ts
|
|
13713
13916
|
import { access, readFile } from "fs/promises";
|
|
@@ -13868,24 +14071,24 @@ async function resolveDirectoryMarkets(options) {
|
|
|
13868
14071
|
}
|
|
13869
14072
|
|
|
13870
14073
|
// src/directory/directory-workflow.ts
|
|
13871
|
-
var DirectoryWorkflowOptionsSchema =
|
|
13872
|
-
query:
|
|
13873
|
-
state:
|
|
13874
|
-
minPopulation:
|
|
13875
|
-
populationYear:
|
|
13876
|
-
maxCities:
|
|
13877
|
-
maxResultsPerCity:
|
|
13878
|
-
concurrency:
|
|
13879
|
-
includeZipGroups:
|
|
13880
|
-
usZipsCsvPath:
|
|
13881
|
-
saveCsv:
|
|
13882
|
-
gl:
|
|
13883
|
-
hl:
|
|
13884
|
-
proxyMode:
|
|
13885
|
-
proxyZip:
|
|
13886
|
-
debug:
|
|
13887
|
-
headless:
|
|
13888
|
-
kernelApiKey:
|
|
14074
|
+
var DirectoryWorkflowOptionsSchema = z22.object({
|
|
14075
|
+
query: z22.string().min(1),
|
|
14076
|
+
state: z22.string().min(2).default("TN"),
|
|
14077
|
+
minPopulation: z22.number().int().min(0).default(1e5),
|
|
14078
|
+
populationYear: z22.union(POPULATION_YEARS.map((year) => z22.literal(year))).default(2025),
|
|
14079
|
+
maxCities: z22.number().int().min(1).max(100).default(25),
|
|
14080
|
+
maxResultsPerCity: z22.number().int().min(1).max(50).default(50),
|
|
14081
|
+
concurrency: z22.number().int().min(1).max(5).default(5),
|
|
14082
|
+
includeZipGroups: z22.boolean().default(true),
|
|
14083
|
+
usZipsCsvPath: z22.string().optional(),
|
|
14084
|
+
saveCsv: z22.boolean().default(true),
|
|
14085
|
+
gl: z22.string().length(2).default("us"),
|
|
14086
|
+
hl: z22.string().length(2).default("en"),
|
|
14087
|
+
proxyMode: z22.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
14088
|
+
proxyZip: z22.string().regex(/^\d{5}$/).optional(),
|
|
14089
|
+
debug: z22.boolean().default(false),
|
|
14090
|
+
headless: z22.boolean().default(true),
|
|
14091
|
+
kernelApiKey: z22.string().optional()
|
|
13889
14092
|
});
|
|
13890
14093
|
function errorMessage(err) {
|
|
13891
14094
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -14042,11 +14245,11 @@ function csvRowsFor(result) {
|
|
|
14042
14245
|
return rows;
|
|
14043
14246
|
}
|
|
14044
14247
|
async function saveDirectoryCsv(result) {
|
|
14045
|
-
const outDir =
|
|
14248
|
+
const outDir = join4(outputBaseDir(), "directory-workflows");
|
|
14046
14249
|
await mkdir2(outDir, { recursive: true });
|
|
14047
14250
|
const stamp = result.extractedAt.replace(/[:.]/g, "-");
|
|
14048
14251
|
const slug = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
14049
|
-
const path5 =
|
|
14252
|
+
const path5 = join4(outDir, `${stamp}-${slug}-directory-workflow.csv`);
|
|
14050
14253
|
const headers = [
|
|
14051
14254
|
"source_query",
|
|
14052
14255
|
"source_location",
|
|
@@ -14106,7 +14309,7 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
14106
14309
|
}
|
|
14107
14310
|
|
|
14108
14311
|
// src/api/directory-routes.ts
|
|
14109
|
-
var directoryApp = new
|
|
14312
|
+
var directoryApp = new Hono14();
|
|
14110
14313
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
14111
14314
|
const user = c.get("user");
|
|
14112
14315
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -14177,37 +14380,37 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
|
14177
14380
|
});
|
|
14178
14381
|
|
|
14179
14382
|
// src/api/workflow-routes.ts
|
|
14180
|
-
import { createHmac as
|
|
14383
|
+
import { createHmac as createHmac3 } from "crypto";
|
|
14181
14384
|
import { readFile as readFile2 } from "fs/promises";
|
|
14182
|
-
import { Hono as
|
|
14183
|
-
import { z as
|
|
14184
|
-
var workflowApp = new
|
|
14185
|
-
var WorkflowInputSchema =
|
|
14186
|
-
var WorkflowIdSchema =
|
|
14187
|
-
var CadenceSchema =
|
|
14188
|
-
var ScheduleStatusSchema =
|
|
14189
|
-
var RunBodySchema =
|
|
14385
|
+
import { Hono as Hono15 } from "hono";
|
|
14386
|
+
import { z as z23 } from "zod";
|
|
14387
|
+
var workflowApp = new Hono15();
|
|
14388
|
+
var WorkflowInputSchema = z23.record(z23.unknown()).default({});
|
|
14389
|
+
var WorkflowIdSchema = z23.string().min(1);
|
|
14390
|
+
var CadenceSchema = z23.enum(["daily", "weekly", "monthly"]);
|
|
14391
|
+
var ScheduleStatusSchema = z23.enum(["active", "paused"]);
|
|
14392
|
+
var RunBodySchema = z23.object({
|
|
14190
14393
|
workflowId: WorkflowIdSchema,
|
|
14191
14394
|
input: WorkflowInputSchema,
|
|
14192
|
-
webhookUrl:
|
|
14395
|
+
webhookUrl: z23.string().url().optional()
|
|
14193
14396
|
});
|
|
14194
|
-
var ScheduleCreateSchema =
|
|
14397
|
+
var ScheduleCreateSchema = z23.object({
|
|
14195
14398
|
workflowId: WorkflowIdSchema,
|
|
14196
|
-
name:
|
|
14399
|
+
name: z23.string().min(1).max(120).optional(),
|
|
14197
14400
|
input: WorkflowInputSchema,
|
|
14198
14401
|
cadence: CadenceSchema.default("weekly"),
|
|
14199
|
-
timezone:
|
|
14200
|
-
webhookUrl:
|
|
14201
|
-
nextRunAt:
|
|
14402
|
+
timezone: z23.string().min(1).max(64).default("UTC"),
|
|
14403
|
+
webhookUrl: z23.string().url().optional(),
|
|
14404
|
+
nextRunAt: z23.string().datetime().optional()
|
|
14202
14405
|
});
|
|
14203
|
-
var SchedulePatchSchema =
|
|
14204
|
-
name:
|
|
14406
|
+
var SchedulePatchSchema = z23.object({
|
|
14407
|
+
name: z23.string().min(1).max(120).optional(),
|
|
14205
14408
|
status: ScheduleStatusSchema.optional(),
|
|
14206
14409
|
input: WorkflowInputSchema.optional(),
|
|
14207
14410
|
cadence: CadenceSchema.optional(),
|
|
14208
|
-
timezone:
|
|
14209
|
-
webhookUrl:
|
|
14210
|
-
nextRunAt:
|
|
14411
|
+
timezone: z23.string().min(1).max(64).optional(),
|
|
14412
|
+
webhookUrl: z23.string().url().nullable().optional(),
|
|
14413
|
+
nextRunAt: z23.string().datetime().nullable().optional()
|
|
14211
14414
|
});
|
|
14212
14415
|
function hostedWorkflowOutputDir() {
|
|
14213
14416
|
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
@@ -14265,7 +14468,7 @@ async function readManifestFromSummary(summary) {
|
|
|
14265
14468
|
function webhookSignature(body, timestamp) {
|
|
14266
14469
|
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
14267
14470
|
if (!secret2) return null;
|
|
14268
|
-
return
|
|
14471
|
+
return createHmac3("sha256", secret2).update(`${timestamp}.${body}`).digest("hex");
|
|
14269
14472
|
}
|
|
14270
14473
|
async function deliverWorkflowWebhook(input) {
|
|
14271
14474
|
if (!input.webhookUrl) return;
|
|
@@ -14711,17 +14914,17 @@ workflowApp.post("/cron/dispatch", async (c) => {
|
|
|
14711
14914
|
});
|
|
14712
14915
|
|
|
14713
14916
|
// src/api/serp-intelligence-routes.ts
|
|
14714
|
-
import { Hono as
|
|
14917
|
+
import { Hono as Hono16 } from "hono";
|
|
14715
14918
|
|
|
14716
14919
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
14717
|
-
import { createHash as
|
|
14920
|
+
import { createHash as createHash3 } from "crypto";
|
|
14718
14921
|
import pLimit3 from "p-limit";
|
|
14719
14922
|
var DEFAULT_TIMEOUT_MS = 15e3;
|
|
14720
14923
|
var DEFAULT_MAX_CONCURRENCY = 2;
|
|
14721
14924
|
var DEFAULT_MAX_CONTENT_CHARS = 25e4;
|
|
14722
14925
|
function sha256(value) {
|
|
14723
14926
|
if (!value) return null;
|
|
14724
|
-
return
|
|
14927
|
+
return createHash3("sha256").update(value).digest("hex");
|
|
14725
14928
|
}
|
|
14726
14929
|
function countWords(markdown) {
|
|
14727
14930
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -15023,7 +15226,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
15023
15226
|
}
|
|
15024
15227
|
|
|
15025
15228
|
// src/serp-intelligence/schemas.ts
|
|
15026
|
-
import { z as
|
|
15229
|
+
import { z as z24 } from "zod";
|
|
15027
15230
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
15028
15231
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
15029
15232
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -15085,171 +15288,171 @@ function isPublicHttpUrl(value) {
|
|
|
15085
15288
|
return false;
|
|
15086
15289
|
}
|
|
15087
15290
|
}
|
|
15088
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
15089
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
15090
|
-
query:
|
|
15091
|
-
location:
|
|
15092
|
-
gl:
|
|
15093
|
-
hl:
|
|
15094
|
-
device:
|
|
15095
|
-
proxyMode:
|
|
15096
|
-
proxyZip:
|
|
15097
|
-
pages:
|
|
15098
|
-
debug:
|
|
15099
|
-
includePageSnapshots:
|
|
15100
|
-
pageSnapshotLimit:
|
|
15291
|
+
var SerpIntelligencePublicHttpUrlSchema = z24.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
15292
|
+
var SerpIntelligenceCaptureBodySchema = z24.object({
|
|
15293
|
+
query: z24.string().trim().min(1, "query is required"),
|
|
15294
|
+
location: z24.string().trim().min(1).optional(),
|
|
15295
|
+
gl: z24.string().trim().length(2).default("us"),
|
|
15296
|
+
hl: z24.string().trim().length(2).default("en"),
|
|
15297
|
+
device: z24.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
15298
|
+
proxyMode: z24.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
15299
|
+
proxyZip: z24.string().regex(/^\d{5}$/).optional(),
|
|
15300
|
+
pages: z24.number().int().min(1).max(2).default(1),
|
|
15301
|
+
debug: z24.boolean().default(false),
|
|
15302
|
+
includePageSnapshots: z24.boolean().default(false),
|
|
15303
|
+
pageSnapshotLimit: z24.number().int().min(0).max(10).default(0)
|
|
15101
15304
|
}).strict();
|
|
15102
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
15305
|
+
var SerpIntelligencePageSnapshotRequestSchema = z24.object({
|
|
15103
15306
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
15104
|
-
sourceKind:
|
|
15105
|
-
sourcePosition:
|
|
15307
|
+
sourceKind: z24.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
15308
|
+
sourcePosition: z24.number().int().min(1).optional()
|
|
15106
15309
|
}).strict();
|
|
15107
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
15108
|
-
urls:
|
|
15109
|
-
targets:
|
|
15110
|
-
maxConcurrency:
|
|
15111
|
-
timeoutMs:
|
|
15112
|
-
debug:
|
|
15310
|
+
var SerpIntelligencePageSnapshotsBodySchema = z24.object({
|
|
15311
|
+
urls: z24.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
15312
|
+
targets: z24.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
15313
|
+
maxConcurrency: z24.number().int().min(1).max(5).default(2),
|
|
15314
|
+
timeoutMs: z24.number().int().min(1e3).max(6e4).default(15e3),
|
|
15315
|
+
debug: z24.boolean().default(false)
|
|
15113
15316
|
}).strict();
|
|
15114
|
-
var SerpIntelligenceAICitationSchema =
|
|
15115
|
-
text:
|
|
15116
|
-
href:
|
|
15317
|
+
var SerpIntelligenceAICitationSchema = z24.object({
|
|
15318
|
+
text: z24.string(),
|
|
15319
|
+
href: z24.string()
|
|
15117
15320
|
}).strict();
|
|
15118
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
15119
|
-
position:
|
|
15120
|
-
title:
|
|
15121
|
-
url:
|
|
15122
|
-
domain:
|
|
15123
|
-
cite:
|
|
15124
|
-
snippet:
|
|
15125
|
-
isRedditStyle:
|
|
15126
|
-
inlineRating:
|
|
15127
|
-
value:
|
|
15128
|
-
count:
|
|
15321
|
+
var SerpIntelligenceOrganicResultSchema = z24.object({
|
|
15322
|
+
position: z24.number().int().min(1),
|
|
15323
|
+
title: z24.string(),
|
|
15324
|
+
url: z24.string(),
|
|
15325
|
+
domain: z24.string(),
|
|
15326
|
+
cite: z24.string().nullable(),
|
|
15327
|
+
snippet: z24.string().nullable(),
|
|
15328
|
+
isRedditStyle: z24.boolean(),
|
|
15329
|
+
inlineRating: z24.object({
|
|
15330
|
+
value: z24.string(),
|
|
15331
|
+
count: z24.string()
|
|
15129
15332
|
}).strict().nullable()
|
|
15130
15333
|
}).strict();
|
|
15131
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
15132
|
-
status:
|
|
15133
|
-
expected:
|
|
15134
|
-
city:
|
|
15135
|
-
regionCode:
|
|
15136
|
-
canonicalLocation:
|
|
15334
|
+
var SerpIntelligenceLocationEvidenceSchema = z24.object({
|
|
15335
|
+
status: z24.enum(SerpIntelligenceLocalizationStatusValues),
|
|
15336
|
+
expected: z24.object({
|
|
15337
|
+
city: z24.string(),
|
|
15338
|
+
regionCode: z24.string().nullable(),
|
|
15339
|
+
canonicalLocation: z24.string()
|
|
15137
15340
|
}).strict().nullable(),
|
|
15138
|
-
candidates:
|
|
15139
|
-
city:
|
|
15140
|
-
regionCode:
|
|
15141
|
-
count:
|
|
15142
|
-
examples:
|
|
15341
|
+
candidates: z24.array(z24.object({
|
|
15342
|
+
city: z24.string(),
|
|
15343
|
+
regionCode: z24.string(),
|
|
15344
|
+
count: z24.number().int().min(0),
|
|
15345
|
+
examples: z24.array(z24.string())
|
|
15143
15346
|
}).strict())
|
|
15144
15347
|
}).strict();
|
|
15145
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
15146
|
-
seed:
|
|
15147
|
-
location:
|
|
15148
|
-
extractedAt:
|
|
15149
|
-
totalQuestions:
|
|
15150
|
-
surface:
|
|
15151
|
-
aiOverview:
|
|
15152
|
-
detected:
|
|
15153
|
-
text:
|
|
15154
|
-
citations:
|
|
15155
|
-
expanded:
|
|
15156
|
-
fullyExpanded:
|
|
15157
|
-
sections:
|
|
15348
|
+
var SerpIntelligenceHarvestResultSchema = z24.object({
|
|
15349
|
+
seed: z24.string(),
|
|
15350
|
+
location: z24.string().nullable(),
|
|
15351
|
+
extractedAt: z24.string(),
|
|
15352
|
+
totalQuestions: z24.number().int().min(0),
|
|
15353
|
+
surface: z24.enum(["web", "aim", "unknown"]),
|
|
15354
|
+
aiOverview: z24.object({
|
|
15355
|
+
detected: z24.boolean(),
|
|
15356
|
+
text: z24.string().nullable(),
|
|
15357
|
+
citations: z24.array(SerpIntelligenceAICitationSchema),
|
|
15358
|
+
expanded: z24.boolean().optional(),
|
|
15359
|
+
fullyExpanded: z24.boolean().optional(),
|
|
15360
|
+
sections: z24.array(z24.string()).optional()
|
|
15158
15361
|
}).strict(),
|
|
15159
|
-
aiMode:
|
|
15160
|
-
detected:
|
|
15161
|
-
text:
|
|
15162
|
-
citations:
|
|
15362
|
+
aiMode: z24.object({
|
|
15363
|
+
detected: z24.boolean(),
|
|
15364
|
+
text: z24.string().nullable(),
|
|
15365
|
+
citations: z24.array(SerpIntelligenceAICitationSchema)
|
|
15163
15366
|
}).strict(),
|
|
15164
|
-
tree:
|
|
15165
|
-
flat:
|
|
15166
|
-
videos:
|
|
15167
|
-
forums:
|
|
15168
|
-
organicResults:
|
|
15169
|
-
localPack:
|
|
15170
|
-
entityIds:
|
|
15171
|
-
entities:
|
|
15172
|
-
name:
|
|
15173
|
-
kgId:
|
|
15174
|
-
cid:
|
|
15175
|
-
gcid:
|
|
15367
|
+
tree: z24.array(z24.unknown()),
|
|
15368
|
+
flat: z24.array(z24.unknown()),
|
|
15369
|
+
videos: z24.array(z24.unknown()),
|
|
15370
|
+
forums: z24.array(z24.unknown()),
|
|
15371
|
+
organicResults: z24.array(SerpIntelligenceOrganicResultSchema),
|
|
15372
|
+
localPack: z24.array(z24.unknown()),
|
|
15373
|
+
entityIds: z24.object({
|
|
15374
|
+
entities: z24.array(z24.object({
|
|
15375
|
+
name: z24.string(),
|
|
15376
|
+
kgId: z24.string().nullable(),
|
|
15377
|
+
cid: z24.string().nullable(),
|
|
15378
|
+
gcid: z24.string().nullable()
|
|
15176
15379
|
}).strict()),
|
|
15177
|
-
kgIds:
|
|
15178
|
-
cids:
|
|
15179
|
-
gcids:
|
|
15380
|
+
kgIds: z24.array(z24.string()),
|
|
15381
|
+
cids: z24.array(z24.string()),
|
|
15382
|
+
gcids: z24.array(z24.string())
|
|
15180
15383
|
}).strict(),
|
|
15181
|
-
stats:
|
|
15182
|
-
seed:
|
|
15183
|
-
totalQuestions:
|
|
15184
|
-
maxDepthReached:
|
|
15185
|
-
durationMs:
|
|
15186
|
-
errorCount:
|
|
15384
|
+
stats: z24.object({
|
|
15385
|
+
seed: z24.string(),
|
|
15386
|
+
totalQuestions: z24.number().int().min(0),
|
|
15387
|
+
maxDepthReached: z24.number().int().min(0),
|
|
15388
|
+
durationMs: z24.number().min(0),
|
|
15389
|
+
errorCount: z24.number().int().min(0)
|
|
15187
15390
|
}).strict(),
|
|
15188
|
-
diagnostics:
|
|
15189
|
-
completionStatus:
|
|
15190
|
-
problem:
|
|
15191
|
-
warnings:
|
|
15192
|
-
debug:
|
|
15391
|
+
diagnostics: z24.object({
|
|
15392
|
+
completionStatus: z24.enum(["paa_found", "no_paa", "serp_only"]),
|
|
15393
|
+
problem: z24.null(),
|
|
15394
|
+
warnings: z24.array(z24.unknown()).optional(),
|
|
15395
|
+
debug: z24.object({
|
|
15193
15396
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
15194
15397
|
}).passthrough().optional()
|
|
15195
15398
|
}).passthrough(),
|
|
15196
|
-
whatPeopleSaying:
|
|
15399
|
+
whatPeopleSaying: z24.array(z24.unknown())
|
|
15197
15400
|
}).strict();
|
|
15198
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
15199
|
-
attemptNumber:
|
|
15200
|
-
outcome:
|
|
15201
|
-
startedAt:
|
|
15202
|
-
completedAt:
|
|
15203
|
-
durationMs:
|
|
15204
|
-
problemCode:
|
|
15205
|
-
message:
|
|
15206
|
-
kernelSessionId:
|
|
15207
|
-
cleanupSucceeded:
|
|
15401
|
+
var SerpIntelligenceCaptureAttemptSchema = z24.object({
|
|
15402
|
+
attemptNumber: z24.number().int().min(1),
|
|
15403
|
+
outcome: z24.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
15404
|
+
startedAt: z24.string().optional(),
|
|
15405
|
+
completedAt: z24.string().optional(),
|
|
15406
|
+
durationMs: z24.number().min(0).optional(),
|
|
15407
|
+
problemCode: z24.string().optional(),
|
|
15408
|
+
message: z24.string().optional(),
|
|
15409
|
+
kernelSessionId: z24.string().nullable().optional(),
|
|
15410
|
+
cleanupSucceeded: z24.boolean().nullable().optional()
|
|
15208
15411
|
}).strict();
|
|
15209
|
-
var SerpPageSnapshotCaptureSchema =
|
|
15412
|
+
var SerpPageSnapshotCaptureSchema = z24.object({
|
|
15210
15413
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
15211
15414
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
15212
15415
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
15213
|
-
sourceKind:
|
|
15214
|
-
sourcePosition:
|
|
15215
|
-
status:
|
|
15216
|
-
fetchedVia:
|
|
15217
|
-
httpStatus:
|
|
15218
|
-
contentType:
|
|
15219
|
-
title:
|
|
15416
|
+
sourceKind: z24.enum(SerpPageSnapshotSourceKindValues),
|
|
15417
|
+
sourcePosition: z24.number().int().min(1).nullable(),
|
|
15418
|
+
status: z24.enum(SerpPageFetchStatusValues),
|
|
15419
|
+
fetchedVia: z24.enum(SerpPageFetchedViaValues).nullable(),
|
|
15420
|
+
httpStatus: z24.number().int().min(100).max(599).nullable(),
|
|
15421
|
+
contentType: z24.string().nullable(),
|
|
15422
|
+
title: z24.string().nullable(),
|
|
15220
15423
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
15221
|
-
metaDescription:
|
|
15222
|
-
headings:
|
|
15223
|
-
level:
|
|
15224
|
-
text:
|
|
15424
|
+
metaDescription: z24.string().nullable(),
|
|
15425
|
+
headings: z24.array(z24.object({
|
|
15426
|
+
level: z24.number().int().min(1).max(6),
|
|
15427
|
+
text: z24.string()
|
|
15225
15428
|
}).strict()).default([]),
|
|
15226
|
-
artifact:
|
|
15227
|
-
htmlBlobUrl:
|
|
15228
|
-
textBlobUrl:
|
|
15229
|
-
markdownBlobUrl:
|
|
15230
|
-
screenshotBlobUrl:
|
|
15231
|
-
contentSha256:
|
|
15232
|
-
capturedAt:
|
|
15429
|
+
artifact: z24.object({
|
|
15430
|
+
htmlBlobUrl: z24.string().url().nullable(),
|
|
15431
|
+
textBlobUrl: z24.string().url().nullable(),
|
|
15432
|
+
markdownBlobUrl: z24.string().url().nullable(),
|
|
15433
|
+
screenshotBlobUrl: z24.string().url().nullable(),
|
|
15434
|
+
contentSha256: z24.string().nullable(),
|
|
15435
|
+
capturedAt: z24.string().nullable()
|
|
15233
15436
|
}).strict(),
|
|
15234
|
-
error:
|
|
15235
|
-
code:
|
|
15236
|
-
message:
|
|
15437
|
+
error: z24.object({
|
|
15438
|
+
code: z24.string(),
|
|
15439
|
+
message: z24.string()
|
|
15237
15440
|
}).strict().nullable()
|
|
15238
15441
|
}).strict();
|
|
15239
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
15442
|
+
var SerpIntelligenceCaptureResponseSchema = z24.object({
|
|
15240
15443
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
15241
|
-
attempts:
|
|
15444
|
+
attempts: z24.array(SerpIntelligenceCaptureAttemptSchema),
|
|
15242
15445
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
15243
|
-
pageSnapshotArtifacts:
|
|
15244
|
-
billing:
|
|
15245
|
-
creditsUsed:
|
|
15246
|
-
requestId:
|
|
15247
|
-
jobId:
|
|
15446
|
+
pageSnapshotArtifacts: z24.array(SerpPageSnapshotCaptureSchema),
|
|
15447
|
+
billing: z24.object({
|
|
15448
|
+
creditsUsed: z24.number().min(0).optional(),
|
|
15449
|
+
requestId: z24.string().optional(),
|
|
15450
|
+
jobId: z24.string().optional()
|
|
15248
15451
|
}).strict().optional()
|
|
15249
15452
|
}).strict();
|
|
15250
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
15251
|
-
pageSnapshotArtifacts:
|
|
15252
|
-
attempts:
|
|
15453
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z24.object({
|
|
15454
|
+
pageSnapshotArtifacts: z24.array(SerpPageSnapshotCaptureSchema),
|
|
15455
|
+
attempts: z24.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
15253
15456
|
}).strict();
|
|
15254
15457
|
|
|
15255
15458
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -15421,7 +15624,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
|
15421
15624
|
var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
15422
15625
|
var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
15423
15626
|
var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
15424
|
-
var serpIntelligenceApp = new
|
|
15627
|
+
var serpIntelligenceApp = new Hono16();
|
|
15425
15628
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
15426
15629
|
function structuredError(input) {
|
|
15427
15630
|
return {
|
|
@@ -15590,7 +15793,7 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
15590
15793
|
});
|
|
15591
15794
|
|
|
15592
15795
|
// src/mcp/mcp-routes.ts
|
|
15593
|
-
import { Hono as
|
|
15796
|
+
import { Hono as Hono17 } from "hono";
|
|
15594
15797
|
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
|
15595
15798
|
|
|
15596
15799
|
// src/mcp/memory-inprocess-executor.ts
|
|
@@ -16172,7 +16375,7 @@ async function requireMcpCallerKey(c) {
|
|
|
16172
16375
|
if (!user) return mcpAuthError();
|
|
16173
16376
|
return { key: callerKey, user };
|
|
16174
16377
|
}
|
|
16175
|
-
var mcpApp = new
|
|
16378
|
+
var mcpApp = new Hono17();
|
|
16176
16379
|
mcpApp.all("/", async (c) => {
|
|
16177
16380
|
try {
|
|
16178
16381
|
const keyOrError = await requireMcpCallerKey(c);
|
|
@@ -16198,11 +16401,11 @@ mcpApp.all("/", async (c) => {
|
|
|
16198
16401
|
});
|
|
16199
16402
|
|
|
16200
16403
|
// src/api/browser-agent-routes.ts
|
|
16201
|
-
import { randomUUID as
|
|
16202
|
-
import { Hono as
|
|
16404
|
+
import { randomUUID as randomUUID4 } from "crypto";
|
|
16405
|
+
import { Hono as Hono18 } from "hono";
|
|
16203
16406
|
|
|
16204
16407
|
// src/api/browser-agent-db.ts
|
|
16205
|
-
import { randomUUID as
|
|
16408
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
16206
16409
|
var _ready = false;
|
|
16207
16410
|
async function migrateBrowserAgent() {
|
|
16208
16411
|
if (_ready) return;
|
|
@@ -16306,7 +16509,7 @@ async function migrateBrowserAgent() {
|
|
|
16306
16509
|
}
|
|
16307
16510
|
async function createExtensionRow(input) {
|
|
16308
16511
|
const db = getDb();
|
|
16309
|
-
const id = `bext_${
|
|
16512
|
+
const id = `bext_${randomUUID3().replace(/-/g, "").slice(0, 20)}`;
|
|
16310
16513
|
await db.execute({
|
|
16311
16514
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
16312
16515
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -16341,7 +16544,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
16341
16544
|
}
|
|
16342
16545
|
async function createAuthConnectionRow(input) {
|
|
16343
16546
|
const db = getDb();
|
|
16344
|
-
const connectionId = `authc_${
|
|
16547
|
+
const connectionId = `authc_${randomUUID3().replace(/-/g, "").slice(0, 20)}`;
|
|
16345
16548
|
await db.execute({
|
|
16346
16549
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
16347
16550
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -16375,7 +16578,7 @@ async function listAuthConnectionsByProfile(profile) {
|
|
|
16375
16578
|
}
|
|
16376
16579
|
async function createSessionRow(input) {
|
|
16377
16580
|
const db = getDb();
|
|
16378
|
-
const id = `bas_${
|
|
16581
|
+
const id = `bas_${randomUUID3().replace(/-/g, "").slice(0, 20)}`;
|
|
16379
16582
|
await db.execute({
|
|
16380
16583
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
16381
16584
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -16444,7 +16647,7 @@ async function recordAction(input) {
|
|
|
16444
16647
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
16445
16648
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
16446
16649
|
args: [
|
|
16447
|
-
`baa_${
|
|
16650
|
+
`baa_${randomUUID3().replace(/-/g, "").slice(0, 20)}`,
|
|
16448
16651
|
input.sessionId,
|
|
16449
16652
|
input.type,
|
|
16450
16653
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -17674,7 +17877,7 @@ async function loadOpenSession(id, userId) {
|
|
|
17674
17877
|
return row;
|
|
17675
17878
|
}
|
|
17676
17879
|
function buildBrowserAgentRoutes() {
|
|
17677
|
-
const app2 = new
|
|
17880
|
+
const app2 = new Hono18();
|
|
17678
17881
|
app2.use("*", async (c, next) => {
|
|
17679
17882
|
await migrateBrowserAgent();
|
|
17680
17883
|
return next();
|
|
@@ -18132,7 +18335,7 @@ function buildBrowserAgentRoutes() {
|
|
|
18132
18335
|
}
|
|
18133
18336
|
const existing = await getExtensionRow(user.id, name);
|
|
18134
18337
|
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
18135
|
-
const backendName = `u${user.id}_${
|
|
18338
|
+
const backendName = `u${user.id}_${randomUUID4().replace(/-/g, "")}`;
|
|
18136
18339
|
try {
|
|
18137
18340
|
const imported = await importExtensionFromStore(storeUrl, backendName);
|
|
18138
18341
|
const row = await createExtensionRow({
|
|
@@ -18464,9 +18667,9 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
|
|
|
18464
18667
|
|
|
18465
18668
|
// src/api/stripe-routes.ts
|
|
18466
18669
|
import Stripe from "stripe";
|
|
18467
|
-
import { Hono as
|
|
18670
|
+
import { Hono as Hono19 } from "hono";
|
|
18468
18671
|
var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
18469
|
-
var stripeApp = new
|
|
18672
|
+
var stripeApp = new Hono19();
|
|
18470
18673
|
stripeApp.post("/webhooks", async (c) => {
|
|
18471
18674
|
const sig = c.req.header("stripe-signature");
|
|
18472
18675
|
const body = await c.req.text();
|
|
@@ -18568,9 +18771,9 @@ async function resolveUser(customerId, emailFallback) {
|
|
|
18568
18771
|
}
|
|
18569
18772
|
|
|
18570
18773
|
// src/api/oauth-routes.ts
|
|
18571
|
-
import { Hono as
|
|
18774
|
+
import { Hono as Hono20 } from "hono";
|
|
18572
18775
|
import { getCookie, setCookie } from "hono/cookie";
|
|
18573
|
-
import { createHash as
|
|
18776
|
+
import { createHash as createHash4, randomBytes as randomBytes2, randomUUID as randomUUID5 } from "crypto";
|
|
18574
18777
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
18575
18778
|
var ISSUER = "https://mcpscraper.dev";
|
|
18576
18779
|
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
|
|
@@ -18603,7 +18806,7 @@ async function getKeys() {
|
|
|
18603
18806
|
const privateKey = await importPKCS8(pem, "RS256", { extractable: true });
|
|
18604
18807
|
const full = await exportJWK(privateKey);
|
|
18605
18808
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
18606
|
-
const kid =
|
|
18809
|
+
const kid = createHash4("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
18607
18810
|
publicJwk.kid = kid;
|
|
18608
18811
|
publicJwk.alg = "RS256";
|
|
18609
18812
|
publicJwk.use = "sig";
|
|
@@ -18737,7 +18940,7 @@ function redirectWithError(redirectUri, state, error) {
|
|
|
18737
18940
|
if (state) u.searchParams.set("state", state);
|
|
18738
18941
|
return new Response(null, { status: 302, headers: { Location: u.toString() } });
|
|
18739
18942
|
}
|
|
18740
|
-
var oauthApp = new
|
|
18943
|
+
var oauthApp = new Hono20();
|
|
18741
18944
|
oauthApp.use("*", async (c, next) => {
|
|
18742
18945
|
c.header("Access-Control-Allow-Origin", "*");
|
|
18743
18946
|
c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
@@ -18913,12 +19116,12 @@ oauthApp.post("/oauth/authorize", async (c) => {
|
|
|
18913
19116
|
});
|
|
18914
19117
|
function pkceMatches(verifier, challenge) {
|
|
18915
19118
|
if (!verifier) return false;
|
|
18916
|
-
const computed =
|
|
19119
|
+
const computed = createHash4("sha256").update(verifier).digest("base64url");
|
|
18917
19120
|
return computed === challenge;
|
|
18918
19121
|
}
|
|
18919
19122
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
18920
19123
|
const { privateKey, kid } = await getKeys();
|
|
18921
|
-
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(
|
|
19124
|
+
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID5()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
18922
19125
|
}
|
|
18923
19126
|
function tokenErrorResponse(c, error, description, status) {
|
|
18924
19127
|
return c.json({ error, error_description: description }, status);
|
|
@@ -18991,8 +19194,8 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
18991
19194
|
});
|
|
18992
19195
|
|
|
18993
19196
|
// src/api/chat-routes.ts
|
|
18994
|
-
import { Hono as
|
|
18995
|
-
var chatApp = new
|
|
19197
|
+
import { Hono as Hono21 } from "hono";
|
|
19198
|
+
var chatApp = new Hono21();
|
|
18996
19199
|
function tokenOk(token) {
|
|
18997
19200
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
18998
19201
|
}
|
|
@@ -19662,8 +19865,8 @@ loadChannels();
|
|
|
19662
19865
|
}
|
|
19663
19866
|
|
|
19664
19867
|
// src/api/schedule-routes.ts
|
|
19665
|
-
import { Hono as
|
|
19666
|
-
var scheduleApp = new
|
|
19868
|
+
import { Hono as Hono22 } from "hono";
|
|
19869
|
+
var scheduleApp = new Hono22();
|
|
19667
19870
|
function tokenOk2(token) {
|
|
19668
19871
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
19669
19872
|
}
|
|
@@ -19854,8 +20057,15 @@ async function loadStatus() {
|
|
|
19854
20057
|
const r = await get('/api/status');
|
|
19855
20058
|
const text = document.getElementById('quota-text');
|
|
19856
20059
|
const fill = document.getElementById('quota-fill');
|
|
19857
|
-
if (!r.ok
|
|
19858
|
-
text.textContent = '
|
|
20060
|
+
if (!r.ok) {
|
|
20061
|
+
text.textContent = 'Scheduled Actions status is temporarily unavailable.';
|
|
20062
|
+
text.classList.remove('warn');
|
|
20063
|
+
fill.style.width = '0%';
|
|
20064
|
+
fill.classList.remove('warn');
|
|
20065
|
+
return;
|
|
20066
|
+
}
|
|
20067
|
+
if (r.billingMode === 'credits') {
|
|
20068
|
+
text.textContent = (r.runBaseCredits || 75) + ' Credits per run + AI at ' + (r.llmCostMultiplier || 1.5) + '\xD7 actual OpenRouter cost';
|
|
19859
20069
|
text.classList.remove('warn');
|
|
19860
20070
|
fill.style.width = '0%';
|
|
19861
20071
|
fill.classList.remove('warn');
|
|
@@ -19987,7 +20197,7 @@ loadActions();
|
|
|
19987
20197
|
}
|
|
19988
20198
|
|
|
19989
20199
|
// src/api/resend-inbound-routes.ts
|
|
19990
|
-
import { Hono as
|
|
20200
|
+
import { Hono as Hono23 } from "hono";
|
|
19991
20201
|
import { Resend } from "resend";
|
|
19992
20202
|
var SUPPORT_ADDRESS = "support@mcpscraper.dev";
|
|
19993
20203
|
var DMARC_ADDRESS = "dmarc@mcpscraper.dev";
|
|
@@ -20163,7 +20373,7 @@ async function forwardReceivedEmail(resend, email, destination, label) {
|
|
|
20163
20373
|
if (sent.error) throw new Error(`forward failed: ${sent.error.message}`);
|
|
20164
20374
|
return sent.data.id;
|
|
20165
20375
|
}
|
|
20166
|
-
var resendInboundApp = new
|
|
20376
|
+
var resendInboundApp = new Hono23();
|
|
20167
20377
|
resendInboundApp.post("/webhooks", async (c) => {
|
|
20168
20378
|
const apiKey = process.env.RESEND_API_KEY?.trim();
|
|
20169
20379
|
const webhookSecret = process.env.RESEND_WEBHOOK_SECRET?.trim();
|
|
@@ -20257,13 +20467,13 @@ import { getCookie as getCookie2, setCookie as setCookie2, deleteCookie } from "
|
|
|
20257
20467
|
import Stripe2 from "stripe";
|
|
20258
20468
|
|
|
20259
20469
|
// src/api/page-diff.ts
|
|
20260
|
-
import { createHash as
|
|
20470
|
+
import { createHash as createHash5 } from "crypto";
|
|
20261
20471
|
import { diffLines } from "diff";
|
|
20262
20472
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
20263
20473
|
var MAX_DIFF_HUNKS = 200;
|
|
20264
20474
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
20265
20475
|
function sha256Hex(value) {
|
|
20266
|
-
return
|
|
20476
|
+
return createHash5("sha256").update(value).digest("hex");
|
|
20267
20477
|
}
|
|
20268
20478
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
20269
20479
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -20370,7 +20580,7 @@ async function persistScrapeBody(user, opts) {
|
|
|
20370
20580
|
}
|
|
20371
20581
|
|
|
20372
20582
|
// src/api/connection-memory-import.ts
|
|
20373
|
-
import { createHash as
|
|
20583
|
+
import { createHash as createHash6 } from "crypto";
|
|
20374
20584
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
20375
20585
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
20376
20586
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -20400,7 +20610,7 @@ function canonicalJson(value) {
|
|
|
20400
20610
|
return JSON.stringify(value);
|
|
20401
20611
|
}
|
|
20402
20612
|
function sha2562(value) {
|
|
20403
|
-
return
|
|
20613
|
+
return createHash6("sha256").update(value).digest("hex");
|
|
20404
20614
|
}
|
|
20405
20615
|
function sensitiveKey(key) {
|
|
20406
20616
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -20618,8 +20828,8 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
|
|
|
20618
20828
|
|
|
20619
20829
|
// src/api/scrape-blob-cleanup.ts
|
|
20620
20830
|
import { readdir, stat, unlink } from "fs/promises";
|
|
20621
|
-
import { homedir
|
|
20622
|
-
import { join as
|
|
20831
|
+
import { homedir } from "os";
|
|
20832
|
+
import { join as join5 } from "path";
|
|
20623
20833
|
async function cleanupVercel(token, cutoff) {
|
|
20624
20834
|
const { list, del } = await import("@vercel/blob");
|
|
20625
20835
|
let cursor;
|
|
@@ -20636,8 +20846,8 @@ async function cleanupVercel(token, cutoff) {
|
|
|
20636
20846
|
return { deleted, store: "vercel-blob" };
|
|
20637
20847
|
}
|
|
20638
20848
|
async function cleanupLocal(cutoff) {
|
|
20639
|
-
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() ||
|
|
20640
|
-
const dir =
|
|
20849
|
+
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join5(homedir(), "Downloads", "mcp-scraper");
|
|
20850
|
+
const dir = join5(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
|
|
20641
20851
|
let deleted = 0;
|
|
20642
20852
|
let entries;
|
|
20643
20853
|
try {
|
|
@@ -20646,7 +20856,7 @@ async function cleanupLocal(cutoff) {
|
|
|
20646
20856
|
return { deleted: 0, store: "local" };
|
|
20647
20857
|
}
|
|
20648
20858
|
for (const name of entries) {
|
|
20649
|
-
const path5 =
|
|
20859
|
+
const path5 = join5(dir, name);
|
|
20650
20860
|
try {
|
|
20651
20861
|
const s = await stat(path5);
|
|
20652
20862
|
if (s.isFile() && s.mtimeMs < cutoff) {
|
|
@@ -20669,7 +20879,7 @@ async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
|
20669
20879
|
}
|
|
20670
20880
|
|
|
20671
20881
|
// src/api/connected-data-export.ts
|
|
20672
|
-
import { randomUUID as
|
|
20882
|
+
import { randomUUID as randomUUID6 } from "crypto";
|
|
20673
20883
|
var CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
20674
20884
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
20675
20885
|
);
|
|
@@ -20794,7 +21004,7 @@ function finitePositive(value, fallback) {
|
|
|
20794
21004
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
20795
21005
|
}
|
|
20796
21006
|
async function collectConnectedDataExport(args) {
|
|
20797
|
-
const exportId =
|
|
21007
|
+
const exportId = randomUUID6();
|
|
20798
21008
|
const now = args.now ?? Date.now;
|
|
20799
21009
|
const startedAt = now();
|
|
20800
21010
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -21123,7 +21333,7 @@ async function dbUsage(identity, plan) {
|
|
|
21123
21333
|
}
|
|
21124
21334
|
|
|
21125
21335
|
// src/api/nango-control.ts
|
|
21126
|
-
import { createHash as
|
|
21336
|
+
import { createHash as createHash7 } from "crypto";
|
|
21127
21337
|
var DEFAULT_NANGO_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
21128
21338
|
var DISABLED_NANGO_TOOLS = {
|
|
21129
21339
|
"google-mail": /* @__PURE__ */ new Set(["list-filters"]),
|
|
@@ -21722,7 +21932,7 @@ function canonicalJson2(value) {
|
|
|
21722
21932
|
return JSON.stringify(value);
|
|
21723
21933
|
}
|
|
21724
21934
|
function projectedToolSchemaHash(tool) {
|
|
21725
|
-
return
|
|
21935
|
+
return createHash7("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
21726
21936
|
}
|
|
21727
21937
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
21728
21938
|
const body = await controlRequest("/api/internal/nango/connections/actions/describe", {
|
|
@@ -22219,7 +22429,7 @@ var auth2 = createMiddleware3(async (c, next) => {
|
|
|
22219
22429
|
const user = await getUserByApiKey(key);
|
|
22220
22430
|
if (!user) return c.json({ error: "Invalid or inactive API key" }, 401);
|
|
22221
22431
|
const refreshed = await applyMonthlyFreeRefresh(user);
|
|
22222
|
-
const balanceMc = await reconcileBalanceMc(refreshed.id);
|
|
22432
|
+
const balanceMc = await reconcileBalanceMc(refreshed.id, refreshed.balance_mc);
|
|
22223
22433
|
c.set("user", { ...refreshed, balance_mc: balanceMc });
|
|
22224
22434
|
return next();
|
|
22225
22435
|
});
|
|
@@ -22238,7 +22448,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
|
|
|
22238
22448
|
const user = await getUserById(userId);
|
|
22239
22449
|
if (!user) return c.json({ error: "User not found" }, 401);
|
|
22240
22450
|
const refreshed = await applyMonthlyFreeRefresh(user);
|
|
22241
|
-
const balanceMc = await reconcileBalanceMc(refreshed.id);
|
|
22451
|
+
const balanceMc = await reconcileBalanceMc(refreshed.id, refreshed.balance_mc);
|
|
22242
22452
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
22243
22453
|
return next();
|
|
22244
22454
|
});
|
|
@@ -22253,7 +22463,7 @@ var requireIntegrationsTier = createMiddleware3(async (c, next) => {
|
|
|
22253
22463
|
}
|
|
22254
22464
|
return next();
|
|
22255
22465
|
});
|
|
22256
|
-
var app = new
|
|
22466
|
+
var app = new Hono24();
|
|
22257
22467
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
22258
22468
|
function requireStripeSecret() {
|
|
22259
22469
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
@@ -22478,7 +22688,7 @@ app.get("/me", async (c) => {
|
|
|
22478
22688
|
}
|
|
22479
22689
|
const refreshed = await applyMonthlyFreeRefresh(foundUser);
|
|
22480
22690
|
const [balanceMc, stats] = await Promise.all([
|
|
22481
|
-
reconcileBalanceMc(refreshed.id),
|
|
22691
|
+
reconcileBalanceMc(refreshed.id, refreshed.balance_mc),
|
|
22482
22692
|
getUserStats(refreshed.id)
|
|
22483
22693
|
]);
|
|
22484
22694
|
const user = { ...refreshed, balance_mc: balanceMc };
|
|
@@ -22754,41 +22964,13 @@ app.post("/memory/portal", auth2, async (c) => {
|
|
|
22754
22964
|
}
|
|
22755
22965
|
});
|
|
22756
22966
|
app.post("/schedule-checkout", auth2, async (c) => {
|
|
22757
|
-
|
|
22758
|
-
|
|
22759
|
-
|
|
22760
|
-
|
|
22761
|
-
|
|
22762
|
-
|
|
22763
|
-
|
|
22764
|
-
const stripeClient = new Stripe2(secret2, { apiVersion: STRIPE_API_VERSION });
|
|
22765
|
-
let customerId = user.stripe_customer_id;
|
|
22766
|
-
if (!customerId) {
|
|
22767
|
-
const customer = await stripeClient.customers.create({ email: user.email });
|
|
22768
|
-
customerId = customer.id;
|
|
22769
|
-
await setStripeCustomerId(user.id, customerId);
|
|
22770
|
-
}
|
|
22771
|
-
const session = await stripeClient.checkout.sessions.create({
|
|
22772
|
-
mode: "subscription",
|
|
22773
|
-
customer: customerId,
|
|
22774
|
-
line_items: [{ price: priceId, quantity: 1 }],
|
|
22775
|
-
client_reference_id: String(user.id),
|
|
22776
|
-
metadata: { product: "mcp-scheduling", userId: String(user.id) },
|
|
22777
|
-
subscription_data: { metadata: { product: "mcp-scheduling", userId: String(user.id) } },
|
|
22778
|
-
automatic_tax: { enabled: true },
|
|
22779
|
-
billing_address_collection: "required",
|
|
22780
|
-
customer_update: { address: "auto", name: "auto" },
|
|
22781
|
-
tax_id_collection: { enabled: true },
|
|
22782
|
-
success_url: `${appOrigin()}/memory?billing=success`,
|
|
22783
|
-
cancel_url: `${appOrigin()}/memory?billing=cancel`
|
|
22784
|
-
});
|
|
22785
|
-
if (!session.url) return c.json({ error: "Stripe did not return a checkout URL." }, 502);
|
|
22786
|
-
return c.json({ url: session.url });
|
|
22787
|
-
} catch (err) {
|
|
22788
|
-
const message = err instanceof Error ? err.message : "Unable to start scheduling checkout.";
|
|
22789
|
-
console.error("[schedule-checkout]", message);
|
|
22790
|
-
return c.json({ error: message }, 500);
|
|
22791
|
-
}
|
|
22967
|
+
return c.json({
|
|
22968
|
+
error_code: "scheduled_actions_now_credit_metered",
|
|
22969
|
+
error: "Scheduled Actions is included and no longer requires a separate subscription.",
|
|
22970
|
+
billingMode: "credits",
|
|
22971
|
+
runBaseCredits: SCHEDULED_RUN_BASE_CREDITS,
|
|
22972
|
+
llmCostMultiplier: SCHEDULED_AGENT_LLM_MARKUP
|
|
22973
|
+
}, 410);
|
|
22792
22974
|
});
|
|
22793
22975
|
function scheduleConnectionError(c, err, fallback) {
|
|
22794
22976
|
if (err instanceof ScheduleConnectionValidationError) return c.json({ ok: false, error: err.message }, 400);
|
|
@@ -22902,7 +23084,14 @@ app.get("/schedule-status", auth2, async (c) => {
|
|
|
22902
23084
|
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
22903
23085
|
if (!key) return c.json({ error: error ?? "memory unavailable" }, 503);
|
|
22904
23086
|
const r = await memoryCall("getScheduleStatusTool", {}, key);
|
|
22905
|
-
return c.json(
|
|
23087
|
+
return c.json({
|
|
23088
|
+
...r,
|
|
23089
|
+
ok: true,
|
|
23090
|
+
enabled: true,
|
|
23091
|
+
billingMode: "credits",
|
|
23092
|
+
runBaseCredits: SCHEDULED_RUN_BASE_CREDITS,
|
|
23093
|
+
llmCostMultiplier: SCHEDULED_AGENT_LLM_MARKUP
|
|
23094
|
+
});
|
|
22906
23095
|
} catch (err) {
|
|
22907
23096
|
const message = err instanceof Error ? err.message : "Unable to get schedule status.";
|
|
22908
23097
|
console.error("[schedule-status]", message);
|
|
@@ -23902,7 +24091,8 @@ app.post("/extract-site", auth2, async (c) => {
|
|
|
23902
24091
|
if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
|
|
23903
24092
|
const parsed = checked.parsed;
|
|
23904
24093
|
const user = c.get("user");
|
|
23905
|
-
|
|
24094
|
+
const downloadImages = body.downloadImages === true;
|
|
24095
|
+
if (body.background === true || downloadImages) {
|
|
23906
24096
|
const affordablePages = Math.floor(user.balance_mc / MC_COSTS.page_scrape);
|
|
23907
24097
|
if (affordablePages < 1) return c.json(insufficientBalanceResponse(user.balance_mc, MC_COSTS.page_scrape), 402);
|
|
23908
24098
|
const maxPages = Math.min(Math.min(1e4, Math.max(1, body.maxPages ?? 1e4)), affordablePages);
|
|
@@ -23917,6 +24107,7 @@ app.post("/extract-site", auth2, async (c) => {
|
|
|
23917
24107
|
concurrency: concurrencyLimitForUser(user),
|
|
23918
24108
|
urlsPerBrowser: body.rotateProxyEvery ?? 10,
|
|
23919
24109
|
formats: body.formats,
|
|
24110
|
+
downloadImages,
|
|
23920
24111
|
heldMc
|
|
23921
24112
|
});
|
|
23922
24113
|
jobCreated = true;
|
|
@@ -24225,7 +24416,7 @@ app.post("/billing/concurrency/cancel", requireAllowedOrigin, sessionAuth, async
|
|
|
24225
24416
|
});
|
|
24226
24417
|
app.get("/billing/balance", auth2, async (c) => {
|
|
24227
24418
|
const user = c.get("user");
|
|
24228
|
-
const balanceMc =
|
|
24419
|
+
const balanceMc = user.balance_mc;
|
|
24229
24420
|
const ledger = await getLedger(user.id, 20);
|
|
24230
24421
|
const freeCredits = await getFreeCreditBreakdown(user.id);
|
|
24231
24422
|
return c.json({
|
|
@@ -24237,7 +24428,7 @@ app.get("/billing/balance", auth2, async (c) => {
|
|
|
24237
24428
|
});
|
|
24238
24429
|
app.post("/billing/credits", auth2, async (c) => {
|
|
24239
24430
|
const user = c.get("user");
|
|
24240
|
-
const balanceMc =
|
|
24431
|
+
const balanceMc = user.balance_mc;
|
|
24241
24432
|
const body = await c.req.json().catch(() => ({}));
|
|
24242
24433
|
const query = body.item?.trim().toLowerCase();
|
|
24243
24434
|
const costs = CREDIT_COST_CATALOG.map(({ aliases, ...cost }) => cost);
|
|
@@ -24270,7 +24461,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
24270
24461
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
24271
24462
|
return c.json({ error: "Unauthorized" }, 401);
|
|
24272
24463
|
}
|
|
24273
|
-
const { drainQueue } = await import("./worker-
|
|
24464
|
+
const { drainQueue } = await import("./worker-45WEWCMQ.js");
|
|
24274
24465
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
24275
24466
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
24276
24467
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup] = await Promise.all([
|
|
@@ -24298,15 +24489,15 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
24298
24489
|
return c.json({ error: "Unauthorized" }, 401);
|
|
24299
24490
|
}
|
|
24300
24491
|
const jobId = c.req.param("id");
|
|
24301
|
-
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-
|
|
24302
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
24492
|
+
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-YNPXEZCB.js");
|
|
24493
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-HMCYUVBJ.js");
|
|
24303
24494
|
const job = await getExtractJob2(jobId);
|
|
24304
24495
|
if (!job) return c.json({ error: "job not found" }, 404);
|
|
24305
24496
|
const stored = await assembleExtractArtifacts(job, {});
|
|
24306
24497
|
await completeExtractJob2(jobId, stored);
|
|
24307
24498
|
let settlement = "already_settled_or_refunded";
|
|
24308
24499
|
if (job.billedMc == null && job.userId != null) {
|
|
24309
|
-
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
24500
|
+
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-YNPXEZCB.js");
|
|
24310
24501
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
24311
24502
|
const successful = await countSuccessfulPages2(jobId);
|
|
24312
24503
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -24324,6 +24515,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
24324
24515
|
app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
|
|
24325
24516
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
24326
24517
|
app.route("/api/internal/memory", internalMemoryApp);
|
|
24518
|
+
app.route("/api/internal/billing", internalBillingApp);
|
|
24327
24519
|
app.route("/youtube", youtubeApp);
|
|
24328
24520
|
app.route("/screenshot", screenshotApp);
|
|
24329
24521
|
app.route("/facebook", facebookAdApp);
|
|
@@ -24471,4 +24663,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
24471
24663
|
export {
|
|
24472
24664
|
app
|
|
24473
24665
|
};
|
|
24474
|
-
//# sourceMappingURL=server-
|
|
24666
|
+
//# sourceMappingURL=server-PH3VJ74X.js.map
|