mcp-scraper 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +2394 -1851
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +179 -3
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-NVUBFCJK.js → chunk-4ZZWFI6L.js} +2 -2
- package/dist/{chunk-4MFYMUIF.js → chunk-5LO6EHEH.js} +25 -3
- package/dist/chunk-5LO6EHEH.js.map +1 -0
- package/dist/chunk-UGQC2FOX.js +7 -0
- package/dist/chunk-UGQC2FOX.js.map +1 -0
- package/dist/{chunk-BAMVP57P.js → chunk-WI4A3KFQ.js} +180 -4
- package/dist/chunk-WI4A3KFQ.js.map +1 -0
- package/dist/{server-LBEYRKM2.js → server-XCDFHHLE.js} +551 -220
- package/dist/server-XCDFHHLE.js.map +1 -0
- package/dist/{site-extract-repository-4NVO5KQA.js → site-extract-repository-THBVEXMP.js} +3 -3
- package/dist/{worker-7XRZNLKB.js → worker-YHKUJR5P.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-4MFYMUIF.js.map +0 -1
- package/dist/chunk-BAMVP57P.js.map +0 -1
- package/dist/chunk-SRMGSCC3.js +0 -7
- package/dist/chunk-SRMGSCC3.js.map +0 -1
- package/dist/server-LBEYRKM2.js.map +0 -1
- /package/dist/{chunk-NVUBFCJK.js.map → chunk-4ZZWFI6L.js.map} +0 -0
- /package/dist/{site-extract-repository-4NVO5KQA.js.map → site-extract-repository-THBVEXMP.js.map} +0 -0
- /package/dist/{worker-7XRZNLKB.js.map → worker-YHKUJR5P.js.map} +0 -0
|
@@ -31,7 +31,7 @@ import {
|
|
|
31
31
|
registerSerpIntelligenceCaptureTools,
|
|
32
32
|
sanitizeAttempts,
|
|
33
33
|
sanitizeHarvestResult
|
|
34
|
-
} from "./chunk-
|
|
34
|
+
} from "./chunk-WI4A3KFQ.js";
|
|
35
35
|
import {
|
|
36
36
|
auditImages,
|
|
37
37
|
getBlobStore
|
|
@@ -65,7 +65,7 @@ import {
|
|
|
65
65
|
RawMapsOverviewSchema,
|
|
66
66
|
RawMapsReviewStatsSchema
|
|
67
67
|
} from "./chunk-XGIPATLV.js";
|
|
68
|
-
import "./chunk-
|
|
68
|
+
import "./chunk-UGQC2FOX.js";
|
|
69
69
|
import {
|
|
70
70
|
completeExtractJob,
|
|
71
71
|
countSuccessfulPages,
|
|
@@ -76,7 +76,7 @@ import {
|
|
|
76
76
|
saveExtractPages,
|
|
77
77
|
setExtractJobTotal,
|
|
78
78
|
settleExtractJob
|
|
79
|
-
} from "./chunk-
|
|
79
|
+
} from "./chunk-4ZZWFI6L.js";
|
|
80
80
|
import {
|
|
81
81
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
82
82
|
CONCURRENCY_PRICE_ID,
|
|
@@ -95,7 +95,7 @@ import {
|
|
|
95
95
|
browserActiveCostMc,
|
|
96
96
|
concurrencySlotBillingInfo,
|
|
97
97
|
insufficientBalanceResponse
|
|
98
|
-
} from "./chunk-
|
|
98
|
+
} from "./chunk-5LO6EHEH.js";
|
|
99
99
|
import {
|
|
100
100
|
CaptchaError,
|
|
101
101
|
RECAPTCHA_INSTRUCTIONS,
|
|
@@ -5274,7 +5274,7 @@ async function extractSite(opts) {
|
|
|
5274
5274
|
}
|
|
5275
5275
|
|
|
5276
5276
|
// src/api/server.ts
|
|
5277
|
-
import { Hono as
|
|
5277
|
+
import { Hono as Hono22 } from "hono";
|
|
5278
5278
|
import { serve as serveInngest } from "inngest/hono";
|
|
5279
5279
|
|
|
5280
5280
|
// src/inngest/client.ts
|
|
@@ -11935,23 +11935,8 @@ async function provisionMemoryForUser(user) {
|
|
|
11935
11935
|
} catch {
|
|
11936
11936
|
}
|
|
11937
11937
|
}
|
|
11938
|
-
function memoryPlanForUser(user) {
|
|
11939
|
-
switch (user.subscription_tier) {
|
|
11940
|
-
case "starter":
|
|
11941
|
-
return "pro";
|
|
11942
|
-
case "growth":
|
|
11943
|
-
return "team";
|
|
11944
|
-
case "scale":
|
|
11945
|
-
return "enterprise";
|
|
11946
|
-
default:
|
|
11947
|
-
return "free";
|
|
11948
|
-
}
|
|
11949
|
-
}
|
|
11950
|
-
var MEMORY_PLAN_RANK = { free: 0, pro: 1, team: 2, enterprise: 3 };
|
|
11951
11938
|
function resolveEffectiveMemoryPlan(user) {
|
|
11952
|
-
|
|
11953
|
-
const memoryProPlan = user.memory_plan === "pro" || user.memory_plan === "team" ? user.memory_plan : "free";
|
|
11954
|
-
return MEMORY_PLAN_RANK[tierPlan] >= MEMORY_PLAN_RANK[memoryProPlan] ? tierPlan : memoryProPlan;
|
|
11939
|
+
return user.memory_plan === "pro" || user.memory_plan === "team" ? user.memory_plan : "free";
|
|
11955
11940
|
}
|
|
11956
11941
|
async function getOrCreateUserMemoryKey(user) {
|
|
11957
11942
|
const creds = await getUserMemoryCreds(user.id);
|
|
@@ -13310,13 +13295,219 @@ trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
13310
13295
|
}
|
|
13311
13296
|
});
|
|
13312
13297
|
|
|
13313
|
-
// src/api/
|
|
13298
|
+
// src/api/g2-routes.ts
|
|
13314
13299
|
import { Hono as Hono12 } from "hono";
|
|
13300
|
+
import { z as z20 } from "zod";
|
|
13301
|
+
var G2ReviewsBodySchema = z20.object({
|
|
13302
|
+
product: z20.string().trim().min(1, "product is required"),
|
|
13303
|
+
maxPages: z20.number().int().min(1).max(50).default(5)
|
|
13304
|
+
});
|
|
13305
|
+
function invalidRequest7(message) {
|
|
13306
|
+
return { error_code: "invalid_request", message };
|
|
13307
|
+
}
|
|
13308
|
+
function normalizeProductSlug(input) {
|
|
13309
|
+
return input.trim().replace(/^https?:\/\/(www\.)?g2\.com\/products\//i, "").replace(/\/.*$/, "").replace(/\s+/g, "-").toLowerCase();
|
|
13310
|
+
}
|
|
13311
|
+
function reviewPageUrl2(product, page) {
|
|
13312
|
+
const base = `https://www.g2.com/products/${encodeURIComponent(product)}/reviews`;
|
|
13313
|
+
return page <= 1 ? base : `${base}?page=${page}`;
|
|
13314
|
+
}
|
|
13315
|
+
async function residentialProxyId2(attemptIndex) {
|
|
13316
|
+
try {
|
|
13317
|
+
const r = await resolveKernelProxyId({
|
|
13318
|
+
kernelApiKey: browserServiceApiKey(),
|
|
13319
|
+
proxyMode: "location",
|
|
13320
|
+
location: "Austin, TX",
|
|
13321
|
+
gl: "us",
|
|
13322
|
+
attemptIndex,
|
|
13323
|
+
fresh: true
|
|
13324
|
+
});
|
|
13325
|
+
return r.kernelProxyId;
|
|
13326
|
+
} catch {
|
|
13327
|
+
return void 0;
|
|
13328
|
+
}
|
|
13329
|
+
}
|
|
13330
|
+
async function extractG2ReviewCards(page) {
|
|
13331
|
+
return page.evaluate(() => {
|
|
13332
|
+
const cards = Array.from(document.querySelectorAll("article[data-track-in-viewport-options]"));
|
|
13333
|
+
return cards.map((card) => {
|
|
13334
|
+
let meta = {};
|
|
13335
|
+
try {
|
|
13336
|
+
meta = JSON.parse(card.getAttribute("data-track-in-viewport-options") ?? "{}");
|
|
13337
|
+
} catch {
|
|
13338
|
+
meta = {};
|
|
13339
|
+
}
|
|
13340
|
+
const reviewId = meta.survey_response_id != null ? String(meta.survey_response_id) : null;
|
|
13341
|
+
const publishedDateRaw = typeof meta.published_date === "string" ? meta.published_date : null;
|
|
13342
|
+
const date = publishedDateRaw && /^\d{8}$/.test(publishedDateRaw) ? `${publishedDateRaw.slice(0, 4)}-${publishedDateRaw.slice(4, 6)}-${publishedDateRaw.slice(6, 8)}` : null;
|
|
13343
|
+
const profileLink = card.querySelector('a[href*="/users/"]');
|
|
13344
|
+
const profileHref = profileLink?.getAttribute("href") ?? "";
|
|
13345
|
+
const profileMatch = profileHref.match(/\/users\/([a-f0-9-]+)/i);
|
|
13346
|
+
const profileId = profileMatch ? profileMatch[1] : null;
|
|
13347
|
+
const infoWrapper = profileLink?.parentElement;
|
|
13348
|
+
const infoDiv = infoWrapper ? Array.from(infoWrapper.children).find((el2) => el2 !== profileLink) : null;
|
|
13349
|
+
const name = infoDiv?.querySelector('div[class*="font-bold"]')?.textContent?.trim() ?? null;
|
|
13350
|
+
const subtleDivs = infoDiv ? Array.from(infoDiv.querySelectorAll('div[class*="text-subtle"]')) : [];
|
|
13351
|
+
const reviewerTitle = subtleDivs[0]?.textContent?.trim() ?? null;
|
|
13352
|
+
const companySegment = subtleDivs[1]?.textContent?.trim() ?? null;
|
|
13353
|
+
const ratingEl = card.querySelector('[class*="star-wrapper__desc__rating"]');
|
|
13354
|
+
const ratingText = ratingEl?.textContent?.trim() ?? "";
|
|
13355
|
+
const ratingParts = ratingText.split("/");
|
|
13356
|
+
const rating = ratingParts[0] ? parseFloat(ratingParts[0]) : null;
|
|
13357
|
+
const ratingScale = ratingParts[1] ? parseFloat(ratingParts[1]) : 5;
|
|
13358
|
+
const titleEl = card.querySelector('div[class*="text-lg"][class*="font-bold"]');
|
|
13359
|
+
const title = titleEl?.textContent?.trim().replace(/^["“]+|["”]+$/g, "") ?? null;
|
|
13360
|
+
const sections = Array.from(card.querySelectorAll("section"));
|
|
13361
|
+
const body = sections.map((sec) => {
|
|
13362
|
+
const question = sec.querySelector('div[class*="font-bold"]')?.textContent?.trim() ?? null;
|
|
13363
|
+
const paragraphs = Array.from(sec.querySelectorAll("p")).map((p) => p.textContent?.trim() ?? "");
|
|
13364
|
+
const answer = paragraphs.join("\n\n").replace(/Review collected by and hosted on G2\.com\.\s*$/, "").trim();
|
|
13365
|
+
return { question, answer };
|
|
13366
|
+
}).filter((b) => b.answer.length > 0);
|
|
13367
|
+
const flagButtons = Array.from(card.querySelectorAll("button[title]"));
|
|
13368
|
+
let currentUser = false;
|
|
13369
|
+
let validated = false;
|
|
13370
|
+
let incentivized = false;
|
|
13371
|
+
let origin = null;
|
|
13372
|
+
for (const btn of flagButtons) {
|
|
13373
|
+
const label = btn.querySelector("label")?.textContent?.trim() ?? "";
|
|
13374
|
+
if (/current user/i.test(label)) currentUser = true;
|
|
13375
|
+
else if (/validated/i.test(label)) validated = true;
|
|
13376
|
+
else if (/^incentivized$/i.test(label)) incentivized = true;
|
|
13377
|
+
else if (/^source:/i.test(label)) origin = label.replace(/^source:\s*/i, "").trim();
|
|
13378
|
+
}
|
|
13379
|
+
const copyBtn = card.querySelector('[data-clipboard-text*="/reviews/"]');
|
|
13380
|
+
const reviewUrl = copyBtn?.getAttribute("data-clipboard-text") ?? null;
|
|
13381
|
+
return {
|
|
13382
|
+
source: "g2",
|
|
13383
|
+
sourceReviewId: reviewId,
|
|
13384
|
+
reviewUrl,
|
|
13385
|
+
reviewer: {
|
|
13386
|
+
name,
|
|
13387
|
+
profileId,
|
|
13388
|
+
title: reviewerTitle,
|
|
13389
|
+
companySegment
|
|
13390
|
+
},
|
|
13391
|
+
rating,
|
|
13392
|
+
ratingScale,
|
|
13393
|
+
title,
|
|
13394
|
+
date,
|
|
13395
|
+
body: body.length > 0 ? body : [{ question: null, answer: "" }],
|
|
13396
|
+
truncated: false,
|
|
13397
|
+
flags: {
|
|
13398
|
+
origin,
|
|
13399
|
+
incentivized,
|
|
13400
|
+
validated,
|
|
13401
|
+
currentUser,
|
|
13402
|
+
companyReplied: null
|
|
13403
|
+
}
|
|
13404
|
+
};
|
|
13405
|
+
});
|
|
13406
|
+
});
|
|
13407
|
+
}
|
|
13408
|
+
var g2App = new Hono12();
|
|
13409
|
+
g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
13410
|
+
const raw = await c.req.json().catch(() => ({}));
|
|
13411
|
+
const parsed = G2ReviewsBodySchema.safeParse(raw);
|
|
13412
|
+
if (!parsed.success) return c.json(invalidRequest7(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
|
|
13413
|
+
const body = parsed.data;
|
|
13414
|
+
const product = normalizeProductSlug(body.product);
|
|
13415
|
+
const user = c.get("user");
|
|
13416
|
+
const gate = await acquireConcurrencyGate(user, "g2_reviews", {
|
|
13417
|
+
reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
|
|
13418
|
+
metadata: { product, maxPages: body.maxPages }
|
|
13419
|
+
});
|
|
13420
|
+
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
13421
|
+
let debited = false;
|
|
13422
|
+
let reviewDebitMc = 0;
|
|
13423
|
+
try {
|
|
13424
|
+
const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.g2_reviews, LedgerOperation.G2_REVIEWS, product);
|
|
13425
|
+
if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.g2_reviews), 402);
|
|
13426
|
+
debited = true;
|
|
13427
|
+
const startMs = Date.now();
|
|
13428
|
+
const allReviews = [];
|
|
13429
|
+
let pagesFetched = 0;
|
|
13430
|
+
let blocked = false;
|
|
13431
|
+
for (let attempt = 0; attempt < 3 && pagesFetched === 0; attempt++) {
|
|
13432
|
+
const proxyId = await residentialProxyId2(attempt);
|
|
13433
|
+
const driver = new BrowserDriver();
|
|
13434
|
+
try {
|
|
13435
|
+
await driver.launch({
|
|
13436
|
+
headless: true,
|
|
13437
|
+
kernelApiKey: browserServiceApiKey(),
|
|
13438
|
+
kernelProxyId: proxyId,
|
|
13439
|
+
viewport: { width: 1280, height: 900 },
|
|
13440
|
+
locale: "en-US"
|
|
13441
|
+
});
|
|
13442
|
+
const page = driver.getPage();
|
|
13443
|
+
for (let pageNum = 1; pageNum <= body.maxPages; pageNum++) {
|
|
13444
|
+
const url = reviewPageUrl2(product, pageNum);
|
|
13445
|
+
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
13446
|
+
await page.waitForTimeout(5e3);
|
|
13447
|
+
await page.evaluate(() => window.scrollTo(0, 1200)).catch(() => {
|
|
13448
|
+
});
|
|
13449
|
+
await page.waitForTimeout(3e3);
|
|
13450
|
+
let cards;
|
|
13451
|
+
try {
|
|
13452
|
+
cards = await extractG2ReviewCards(page);
|
|
13453
|
+
} catch {
|
|
13454
|
+
await page.waitForTimeout(3e3);
|
|
13455
|
+
cards = await extractG2ReviewCards(page);
|
|
13456
|
+
}
|
|
13457
|
+
if (cards.length === 0) break;
|
|
13458
|
+
allReviews.push(...cards);
|
|
13459
|
+
pagesFetched = pageNum;
|
|
13460
|
+
}
|
|
13461
|
+
} catch {
|
|
13462
|
+
blocked = true;
|
|
13463
|
+
} finally {
|
|
13464
|
+
await driver.close();
|
|
13465
|
+
}
|
|
13466
|
+
}
|
|
13467
|
+
if (pagesFetched === 0 || allReviews.length === 0) {
|
|
13468
|
+
await creditMc(user.id, MC_COSTS.g2_reviews, LedgerOperation.G2_REVIEWS_REFUND, blocked ? "blocked after retries" : "no reviews found");
|
|
13469
|
+
await logRequestEvent({ userId: user.id, source: "g2_reviews", status: "failed", query: product, error: blocked ? "blocked after retries" : "no reviews found" });
|
|
13470
|
+
return c.json({ error: "No reviews found for this product (refunded)", product }, 404);
|
|
13471
|
+
}
|
|
13472
|
+
reviewDebitMc = allReviews.length * MC_COSTS.g2_review;
|
|
13473
|
+
if (reviewDebitMc > 0) {
|
|
13474
|
+
const reviewDebit = await debitMc(user.id, reviewDebitMc, LedgerOperation.G2_REVIEW, product);
|
|
13475
|
+
if (!reviewDebit.ok) {
|
|
13476
|
+
allReviews.length = Math.floor((reviewDebit.balance_mc ?? 0) / MC_COSTS.g2_review);
|
|
13477
|
+
reviewDebitMc = allReviews.length * MC_COSTS.g2_review;
|
|
13478
|
+
if (reviewDebitMc > 0) await debitMc(user.id, reviewDebitMc, LedgerOperation.G2_REVIEW, product);
|
|
13479
|
+
}
|
|
13480
|
+
}
|
|
13481
|
+
const result = {
|
|
13482
|
+
product,
|
|
13483
|
+
reviewUrl: reviewPageUrl2(product, 1),
|
|
13484
|
+
extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13485
|
+
requestedMaxPages: body.maxPages,
|
|
13486
|
+
pagesFetched,
|
|
13487
|
+
reviewCount: allReviews.length,
|
|
13488
|
+
reviews: allReviews,
|
|
13489
|
+
durationMs: Date.now() - startMs
|
|
13490
|
+
};
|
|
13491
|
+
await logRequestEvent({ userId: user.id, source: "g2_reviews", status: "done", query: product, resultCount: allReviews.length, result });
|
|
13492
|
+
return c.json(result);
|
|
13493
|
+
} catch (err) {
|
|
13494
|
+
if (debited) await creditMc(user.id, MC_COSTS.g2_reviews, LedgerOperation.G2_REVIEWS_REFUND, "failed call");
|
|
13495
|
+
if (reviewDebitMc > 0) await creditMc(user.id, reviewDebitMc, LedgerOperation.G2_REVIEW_REFUND, "failed call");
|
|
13496
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
13497
|
+
await logRequestEvent({ userId: user.id, source: "g2_reviews", status: "failed", query: product, error: msg });
|
|
13498
|
+
return c.json({ error: msg }, 500);
|
|
13499
|
+
} finally {
|
|
13500
|
+
await releaseConcurrencyGate(gate.lockId);
|
|
13501
|
+
}
|
|
13502
|
+
});
|
|
13503
|
+
|
|
13504
|
+
// src/api/directory-routes.ts
|
|
13505
|
+
import { Hono as Hono13 } from "hono";
|
|
13315
13506
|
|
|
13316
13507
|
// src/directory/directory-workflow.ts
|
|
13317
13508
|
import { mkdir as mkdir2, writeFile } from "fs/promises";
|
|
13318
13509
|
import { join as join5 } from "path";
|
|
13319
|
-
import { z as
|
|
13510
|
+
import { z as z21 } from "zod";
|
|
13320
13511
|
|
|
13321
13512
|
// src/directory/location-db.ts
|
|
13322
13513
|
import { access, readFile } from "fs/promises";
|
|
@@ -13477,24 +13668,24 @@ async function resolveDirectoryMarkets(options) {
|
|
|
13477
13668
|
}
|
|
13478
13669
|
|
|
13479
13670
|
// src/directory/directory-workflow.ts
|
|
13480
|
-
var DirectoryWorkflowOptionsSchema =
|
|
13481
|
-
query:
|
|
13482
|
-
state:
|
|
13483
|
-
minPopulation:
|
|
13484
|
-
populationYear:
|
|
13485
|
-
maxCities:
|
|
13486
|
-
maxResultsPerCity:
|
|
13487
|
-
concurrency:
|
|
13488
|
-
includeZipGroups:
|
|
13489
|
-
usZipsCsvPath:
|
|
13490
|
-
saveCsv:
|
|
13491
|
-
gl:
|
|
13492
|
-
hl:
|
|
13493
|
-
proxyMode:
|
|
13494
|
-
proxyZip:
|
|
13495
|
-
debug:
|
|
13496
|
-
headless:
|
|
13497
|
-
kernelApiKey:
|
|
13671
|
+
var DirectoryWorkflowOptionsSchema = z21.object({
|
|
13672
|
+
query: z21.string().min(1),
|
|
13673
|
+
state: z21.string().min(2).default("TN"),
|
|
13674
|
+
minPopulation: z21.number().int().min(0).default(1e5),
|
|
13675
|
+
populationYear: z21.union(POPULATION_YEARS.map((year) => z21.literal(year))).default(2025),
|
|
13676
|
+
maxCities: z21.number().int().min(1).max(100).default(25),
|
|
13677
|
+
maxResultsPerCity: z21.number().int().min(1).max(50).default(50),
|
|
13678
|
+
concurrency: z21.number().int().min(1).max(5).default(5),
|
|
13679
|
+
includeZipGroups: z21.boolean().default(true),
|
|
13680
|
+
usZipsCsvPath: z21.string().optional(),
|
|
13681
|
+
saveCsv: z21.boolean().default(true),
|
|
13682
|
+
gl: z21.string().length(2).default("us"),
|
|
13683
|
+
hl: z21.string().length(2).default("en"),
|
|
13684
|
+
proxyMode: z21.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
13685
|
+
proxyZip: z21.string().regex(/^\d{5}$/).optional(),
|
|
13686
|
+
debug: z21.boolean().default(false),
|
|
13687
|
+
headless: z21.boolean().default(true),
|
|
13688
|
+
kernelApiKey: z21.string().optional()
|
|
13498
13689
|
});
|
|
13499
13690
|
function errorMessage(err) {
|
|
13500
13691
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -13715,7 +13906,7 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
|
|
|
13715
13906
|
}
|
|
13716
13907
|
|
|
13717
13908
|
// src/api/directory-routes.ts
|
|
13718
|
-
var directoryApp = new
|
|
13909
|
+
var directoryApp = new Hono13();
|
|
13719
13910
|
directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
13720
13911
|
const user = c.get("user");
|
|
13721
13912
|
const body = await c.req.json().catch(() => ({}));
|
|
@@ -13788,35 +13979,35 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
|
|
|
13788
13979
|
// src/api/workflow-routes.ts
|
|
13789
13980
|
import { createHmac as createHmac2 } from "crypto";
|
|
13790
13981
|
import { readFile as readFile2 } from "fs/promises";
|
|
13791
|
-
import { Hono as
|
|
13792
|
-
import { z as
|
|
13793
|
-
var workflowApp = new
|
|
13794
|
-
var WorkflowInputSchema =
|
|
13795
|
-
var WorkflowIdSchema =
|
|
13796
|
-
var CadenceSchema =
|
|
13797
|
-
var ScheduleStatusSchema =
|
|
13798
|
-
var RunBodySchema =
|
|
13982
|
+
import { Hono as Hono14 } from "hono";
|
|
13983
|
+
import { z as z22 } from "zod";
|
|
13984
|
+
var workflowApp = new Hono14();
|
|
13985
|
+
var WorkflowInputSchema = z22.record(z22.unknown()).default({});
|
|
13986
|
+
var WorkflowIdSchema = z22.string().min(1);
|
|
13987
|
+
var CadenceSchema = z22.enum(["daily", "weekly", "monthly"]);
|
|
13988
|
+
var ScheduleStatusSchema = z22.enum(["active", "paused"]);
|
|
13989
|
+
var RunBodySchema = z22.object({
|
|
13799
13990
|
workflowId: WorkflowIdSchema,
|
|
13800
13991
|
input: WorkflowInputSchema,
|
|
13801
|
-
webhookUrl:
|
|
13992
|
+
webhookUrl: z22.string().url().optional()
|
|
13802
13993
|
});
|
|
13803
|
-
var ScheduleCreateSchema =
|
|
13994
|
+
var ScheduleCreateSchema = z22.object({
|
|
13804
13995
|
workflowId: WorkflowIdSchema,
|
|
13805
|
-
name:
|
|
13996
|
+
name: z22.string().min(1).max(120).optional(),
|
|
13806
13997
|
input: WorkflowInputSchema,
|
|
13807
13998
|
cadence: CadenceSchema.default("weekly"),
|
|
13808
|
-
timezone:
|
|
13809
|
-
webhookUrl:
|
|
13810
|
-
nextRunAt:
|
|
13999
|
+
timezone: z22.string().min(1).max(64).default("UTC"),
|
|
14000
|
+
webhookUrl: z22.string().url().optional(),
|
|
14001
|
+
nextRunAt: z22.string().datetime().optional()
|
|
13811
14002
|
});
|
|
13812
|
-
var SchedulePatchSchema =
|
|
13813
|
-
name:
|
|
14003
|
+
var SchedulePatchSchema = z22.object({
|
|
14004
|
+
name: z22.string().min(1).max(120).optional(),
|
|
13814
14005
|
status: ScheduleStatusSchema.optional(),
|
|
13815
14006
|
input: WorkflowInputSchema.optional(),
|
|
13816
14007
|
cadence: CadenceSchema.optional(),
|
|
13817
|
-
timezone:
|
|
13818
|
-
webhookUrl:
|
|
13819
|
-
nextRunAt:
|
|
14008
|
+
timezone: z22.string().min(1).max(64).optional(),
|
|
14009
|
+
webhookUrl: z22.string().url().nullable().optional(),
|
|
14010
|
+
nextRunAt: z22.string().datetime().nullable().optional()
|
|
13820
14011
|
});
|
|
13821
14012
|
function hostedWorkflowOutputDir() {
|
|
13822
14013
|
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
@@ -14320,7 +14511,7 @@ workflowApp.post("/cron/dispatch", async (c) => {
|
|
|
14320
14511
|
});
|
|
14321
14512
|
|
|
14322
14513
|
// src/api/serp-intelligence-routes.ts
|
|
14323
|
-
import { Hono as
|
|
14514
|
+
import { Hono as Hono15 } from "hono";
|
|
14324
14515
|
|
|
14325
14516
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
14326
14517
|
import { createHash as createHash2 } from "crypto";
|
|
@@ -14632,7 +14823,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
14632
14823
|
}
|
|
14633
14824
|
|
|
14634
14825
|
// src/serp-intelligence/schemas.ts
|
|
14635
|
-
import { z as
|
|
14826
|
+
import { z as z23 } from "zod";
|
|
14636
14827
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
14637
14828
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
14638
14829
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -14694,171 +14885,171 @@ function isPublicHttpUrl(value) {
|
|
|
14694
14885
|
return false;
|
|
14695
14886
|
}
|
|
14696
14887
|
}
|
|
14697
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
14698
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
14699
|
-
query:
|
|
14700
|
-
location:
|
|
14701
|
-
gl:
|
|
14702
|
-
hl:
|
|
14703
|
-
device:
|
|
14704
|
-
proxyMode:
|
|
14705
|
-
proxyZip:
|
|
14706
|
-
pages:
|
|
14707
|
-
debug:
|
|
14708
|
-
includePageSnapshots:
|
|
14709
|
-
pageSnapshotLimit:
|
|
14888
|
+
var SerpIntelligencePublicHttpUrlSchema = z23.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
14889
|
+
var SerpIntelligenceCaptureBodySchema = z23.object({
|
|
14890
|
+
query: z23.string().trim().min(1, "query is required"),
|
|
14891
|
+
location: z23.string().trim().min(1).optional(),
|
|
14892
|
+
gl: z23.string().trim().length(2).default("us"),
|
|
14893
|
+
hl: z23.string().trim().length(2).default("en"),
|
|
14894
|
+
device: z23.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
14895
|
+
proxyMode: z23.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
14896
|
+
proxyZip: z23.string().regex(/^\d{5}$/).optional(),
|
|
14897
|
+
pages: z23.number().int().min(1).max(2).default(1),
|
|
14898
|
+
debug: z23.boolean().default(false),
|
|
14899
|
+
includePageSnapshots: z23.boolean().default(false),
|
|
14900
|
+
pageSnapshotLimit: z23.number().int().min(0).max(10).default(0)
|
|
14710
14901
|
}).strict();
|
|
14711
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
14902
|
+
var SerpIntelligencePageSnapshotRequestSchema = z23.object({
|
|
14712
14903
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
14713
|
-
sourceKind:
|
|
14714
|
-
sourcePosition:
|
|
14904
|
+
sourceKind: z23.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
14905
|
+
sourcePosition: z23.number().int().min(1).optional()
|
|
14715
14906
|
}).strict();
|
|
14716
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
14717
|
-
urls:
|
|
14718
|
-
targets:
|
|
14719
|
-
maxConcurrency:
|
|
14720
|
-
timeoutMs:
|
|
14721
|
-
debug:
|
|
14907
|
+
var SerpIntelligencePageSnapshotsBodySchema = z23.object({
|
|
14908
|
+
urls: z23.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
|
|
14909
|
+
targets: z23.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
14910
|
+
maxConcurrency: z23.number().int().min(1).max(5).default(2),
|
|
14911
|
+
timeoutMs: z23.number().int().min(1e3).max(6e4).default(15e3),
|
|
14912
|
+
debug: z23.boolean().default(false)
|
|
14722
14913
|
}).strict();
|
|
14723
|
-
var SerpIntelligenceAICitationSchema =
|
|
14724
|
-
text:
|
|
14725
|
-
href:
|
|
14914
|
+
var SerpIntelligenceAICitationSchema = z23.object({
|
|
14915
|
+
text: z23.string(),
|
|
14916
|
+
href: z23.string()
|
|
14726
14917
|
}).strict();
|
|
14727
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
14728
|
-
position:
|
|
14729
|
-
title:
|
|
14730
|
-
url:
|
|
14731
|
-
domain:
|
|
14732
|
-
cite:
|
|
14733
|
-
snippet:
|
|
14734
|
-
isRedditStyle:
|
|
14735
|
-
inlineRating:
|
|
14736
|
-
value:
|
|
14737
|
-
count:
|
|
14918
|
+
var SerpIntelligenceOrganicResultSchema = z23.object({
|
|
14919
|
+
position: z23.number().int().min(1),
|
|
14920
|
+
title: z23.string(),
|
|
14921
|
+
url: z23.string(),
|
|
14922
|
+
domain: z23.string(),
|
|
14923
|
+
cite: z23.string().nullable(),
|
|
14924
|
+
snippet: z23.string().nullable(),
|
|
14925
|
+
isRedditStyle: z23.boolean(),
|
|
14926
|
+
inlineRating: z23.object({
|
|
14927
|
+
value: z23.string(),
|
|
14928
|
+
count: z23.string()
|
|
14738
14929
|
}).strict().nullable()
|
|
14739
14930
|
}).strict();
|
|
14740
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
14741
|
-
status:
|
|
14742
|
-
expected:
|
|
14743
|
-
city:
|
|
14744
|
-
regionCode:
|
|
14745
|
-
canonicalLocation:
|
|
14931
|
+
var SerpIntelligenceLocationEvidenceSchema = z23.object({
|
|
14932
|
+
status: z23.enum(SerpIntelligenceLocalizationStatusValues),
|
|
14933
|
+
expected: z23.object({
|
|
14934
|
+
city: z23.string(),
|
|
14935
|
+
regionCode: z23.string().nullable(),
|
|
14936
|
+
canonicalLocation: z23.string()
|
|
14746
14937
|
}).strict().nullable(),
|
|
14747
|
-
candidates:
|
|
14748
|
-
city:
|
|
14749
|
-
regionCode:
|
|
14750
|
-
count:
|
|
14751
|
-
examples:
|
|
14938
|
+
candidates: z23.array(z23.object({
|
|
14939
|
+
city: z23.string(),
|
|
14940
|
+
regionCode: z23.string(),
|
|
14941
|
+
count: z23.number().int().min(0),
|
|
14942
|
+
examples: z23.array(z23.string())
|
|
14752
14943
|
}).strict())
|
|
14753
14944
|
}).strict();
|
|
14754
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
14755
|
-
seed:
|
|
14756
|
-
location:
|
|
14757
|
-
extractedAt:
|
|
14758
|
-
totalQuestions:
|
|
14759
|
-
surface:
|
|
14760
|
-
aiOverview:
|
|
14761
|
-
detected:
|
|
14762
|
-
text:
|
|
14763
|
-
citations:
|
|
14764
|
-
expanded:
|
|
14765
|
-
fullyExpanded:
|
|
14766
|
-
sections:
|
|
14945
|
+
var SerpIntelligenceHarvestResultSchema = z23.object({
|
|
14946
|
+
seed: z23.string(),
|
|
14947
|
+
location: z23.string().nullable(),
|
|
14948
|
+
extractedAt: z23.string(),
|
|
14949
|
+
totalQuestions: z23.number().int().min(0),
|
|
14950
|
+
surface: z23.enum(["web", "aim", "unknown"]),
|
|
14951
|
+
aiOverview: z23.object({
|
|
14952
|
+
detected: z23.boolean(),
|
|
14953
|
+
text: z23.string().nullable(),
|
|
14954
|
+
citations: z23.array(SerpIntelligenceAICitationSchema),
|
|
14955
|
+
expanded: z23.boolean().optional(),
|
|
14956
|
+
fullyExpanded: z23.boolean().optional(),
|
|
14957
|
+
sections: z23.array(z23.string()).optional()
|
|
14767
14958
|
}).strict(),
|
|
14768
|
-
aiMode:
|
|
14769
|
-
detected:
|
|
14770
|
-
text:
|
|
14771
|
-
citations:
|
|
14959
|
+
aiMode: z23.object({
|
|
14960
|
+
detected: z23.boolean(),
|
|
14961
|
+
text: z23.string().nullable(),
|
|
14962
|
+
citations: z23.array(SerpIntelligenceAICitationSchema)
|
|
14772
14963
|
}).strict(),
|
|
14773
|
-
tree:
|
|
14774
|
-
flat:
|
|
14775
|
-
videos:
|
|
14776
|
-
forums:
|
|
14777
|
-
organicResults:
|
|
14778
|
-
localPack:
|
|
14779
|
-
entityIds:
|
|
14780
|
-
entities:
|
|
14781
|
-
name:
|
|
14782
|
-
kgId:
|
|
14783
|
-
cid:
|
|
14784
|
-
gcid:
|
|
14964
|
+
tree: z23.array(z23.unknown()),
|
|
14965
|
+
flat: z23.array(z23.unknown()),
|
|
14966
|
+
videos: z23.array(z23.unknown()),
|
|
14967
|
+
forums: z23.array(z23.unknown()),
|
|
14968
|
+
organicResults: z23.array(SerpIntelligenceOrganicResultSchema),
|
|
14969
|
+
localPack: z23.array(z23.unknown()),
|
|
14970
|
+
entityIds: z23.object({
|
|
14971
|
+
entities: z23.array(z23.object({
|
|
14972
|
+
name: z23.string(),
|
|
14973
|
+
kgId: z23.string().nullable(),
|
|
14974
|
+
cid: z23.string().nullable(),
|
|
14975
|
+
gcid: z23.string().nullable()
|
|
14785
14976
|
}).strict()),
|
|
14786
|
-
kgIds:
|
|
14787
|
-
cids:
|
|
14788
|
-
gcids:
|
|
14977
|
+
kgIds: z23.array(z23.string()),
|
|
14978
|
+
cids: z23.array(z23.string()),
|
|
14979
|
+
gcids: z23.array(z23.string())
|
|
14789
14980
|
}).strict(),
|
|
14790
|
-
stats:
|
|
14791
|
-
seed:
|
|
14792
|
-
totalQuestions:
|
|
14793
|
-
maxDepthReached:
|
|
14794
|
-
durationMs:
|
|
14795
|
-
errorCount:
|
|
14981
|
+
stats: z23.object({
|
|
14982
|
+
seed: z23.string(),
|
|
14983
|
+
totalQuestions: z23.number().int().min(0),
|
|
14984
|
+
maxDepthReached: z23.number().int().min(0),
|
|
14985
|
+
durationMs: z23.number().min(0),
|
|
14986
|
+
errorCount: z23.number().int().min(0)
|
|
14796
14987
|
}).strict(),
|
|
14797
|
-
diagnostics:
|
|
14798
|
-
completionStatus:
|
|
14799
|
-
problem:
|
|
14800
|
-
warnings:
|
|
14801
|
-
debug:
|
|
14988
|
+
diagnostics: z23.object({
|
|
14989
|
+
completionStatus: z23.enum(["paa_found", "no_paa", "serp_only"]),
|
|
14990
|
+
problem: z23.null(),
|
|
14991
|
+
warnings: z23.array(z23.unknown()).optional(),
|
|
14992
|
+
debug: z23.object({
|
|
14802
14993
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
14803
14994
|
}).passthrough().optional()
|
|
14804
14995
|
}).passthrough(),
|
|
14805
|
-
whatPeopleSaying:
|
|
14996
|
+
whatPeopleSaying: z23.array(z23.unknown())
|
|
14806
14997
|
}).strict();
|
|
14807
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
14808
|
-
attemptNumber:
|
|
14809
|
-
outcome:
|
|
14810
|
-
startedAt:
|
|
14811
|
-
completedAt:
|
|
14812
|
-
durationMs:
|
|
14813
|
-
problemCode:
|
|
14814
|
-
message:
|
|
14815
|
-
kernelSessionId:
|
|
14816
|
-
cleanupSucceeded:
|
|
14998
|
+
var SerpIntelligenceCaptureAttemptSchema = z23.object({
|
|
14999
|
+
attemptNumber: z23.number().int().min(1),
|
|
15000
|
+
outcome: z23.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
15001
|
+
startedAt: z23.string().optional(),
|
|
15002
|
+
completedAt: z23.string().optional(),
|
|
15003
|
+
durationMs: z23.number().min(0).optional(),
|
|
15004
|
+
problemCode: z23.string().optional(),
|
|
15005
|
+
message: z23.string().optional(),
|
|
15006
|
+
kernelSessionId: z23.string().nullable().optional(),
|
|
15007
|
+
cleanupSucceeded: z23.boolean().nullable().optional()
|
|
14817
15008
|
}).strict();
|
|
14818
|
-
var SerpPageSnapshotCaptureSchema =
|
|
15009
|
+
var SerpPageSnapshotCaptureSchema = z23.object({
|
|
14819
15010
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
14820
15011
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
14821
15012
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
14822
|
-
sourceKind:
|
|
14823
|
-
sourcePosition:
|
|
14824
|
-
status:
|
|
14825
|
-
fetchedVia:
|
|
14826
|
-
httpStatus:
|
|
14827
|
-
contentType:
|
|
14828
|
-
title:
|
|
15013
|
+
sourceKind: z23.enum(SerpPageSnapshotSourceKindValues),
|
|
15014
|
+
sourcePosition: z23.number().int().min(1).nullable(),
|
|
15015
|
+
status: z23.enum(SerpPageFetchStatusValues),
|
|
15016
|
+
fetchedVia: z23.enum(SerpPageFetchedViaValues).nullable(),
|
|
15017
|
+
httpStatus: z23.number().int().min(100).max(599).nullable(),
|
|
15018
|
+
contentType: z23.string().nullable(),
|
|
15019
|
+
title: z23.string().nullable(),
|
|
14829
15020
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
14830
|
-
metaDescription:
|
|
14831
|
-
headings:
|
|
14832
|
-
level:
|
|
14833
|
-
text:
|
|
15021
|
+
metaDescription: z23.string().nullable(),
|
|
15022
|
+
headings: z23.array(z23.object({
|
|
15023
|
+
level: z23.number().int().min(1).max(6),
|
|
15024
|
+
text: z23.string()
|
|
14834
15025
|
}).strict()).default([]),
|
|
14835
|
-
artifact:
|
|
14836
|
-
htmlBlobUrl:
|
|
14837
|
-
textBlobUrl:
|
|
14838
|
-
markdownBlobUrl:
|
|
14839
|
-
screenshotBlobUrl:
|
|
14840
|
-
contentSha256:
|
|
14841
|
-
capturedAt:
|
|
15026
|
+
artifact: z23.object({
|
|
15027
|
+
htmlBlobUrl: z23.string().url().nullable(),
|
|
15028
|
+
textBlobUrl: z23.string().url().nullable(),
|
|
15029
|
+
markdownBlobUrl: z23.string().url().nullable(),
|
|
15030
|
+
screenshotBlobUrl: z23.string().url().nullable(),
|
|
15031
|
+
contentSha256: z23.string().nullable(),
|
|
15032
|
+
capturedAt: z23.string().nullable()
|
|
14842
15033
|
}).strict(),
|
|
14843
|
-
error:
|
|
14844
|
-
code:
|
|
14845
|
-
message:
|
|
15034
|
+
error: z23.object({
|
|
15035
|
+
code: z23.string(),
|
|
15036
|
+
message: z23.string()
|
|
14846
15037
|
}).strict().nullable()
|
|
14847
15038
|
}).strict();
|
|
14848
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
15039
|
+
var SerpIntelligenceCaptureResponseSchema = z23.object({
|
|
14849
15040
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
14850
|
-
attempts:
|
|
15041
|
+
attempts: z23.array(SerpIntelligenceCaptureAttemptSchema),
|
|
14851
15042
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
14852
|
-
pageSnapshotArtifacts:
|
|
14853
|
-
billing:
|
|
14854
|
-
creditsUsed:
|
|
14855
|
-
requestId:
|
|
14856
|
-
jobId:
|
|
15043
|
+
pageSnapshotArtifacts: z23.array(SerpPageSnapshotCaptureSchema),
|
|
15044
|
+
billing: z23.object({
|
|
15045
|
+
creditsUsed: z23.number().min(0).optional(),
|
|
15046
|
+
requestId: z23.string().optional(),
|
|
15047
|
+
jobId: z23.string().optional()
|
|
14857
15048
|
}).strict().optional()
|
|
14858
15049
|
}).strict();
|
|
14859
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
14860
|
-
pageSnapshotArtifacts:
|
|
14861
|
-
attempts:
|
|
15050
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z23.object({
|
|
15051
|
+
pageSnapshotArtifacts: z23.array(SerpPageSnapshotCaptureSchema),
|
|
15052
|
+
attempts: z23.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
14862
15053
|
}).strict();
|
|
14863
15054
|
|
|
14864
15055
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -15030,7 +15221,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
|
|
|
15030
15221
|
var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
|
|
15031
15222
|
var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
|
|
15032
15223
|
var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
|
|
15033
|
-
var serpIntelligenceApp = new
|
|
15224
|
+
var serpIntelligenceApp = new Hono15();
|
|
15034
15225
|
serpIntelligenceApp.use("*", createApiKeyAuth());
|
|
15035
15226
|
function structuredError(input) {
|
|
15036
15227
|
return {
|
|
@@ -15199,7 +15390,7 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
|
|
|
15199
15390
|
});
|
|
15200
15391
|
|
|
15201
15392
|
// src/mcp/mcp-routes.ts
|
|
15202
|
-
import { Hono as
|
|
15393
|
+
import { Hono as Hono16 } from "hono";
|
|
15203
15394
|
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
|
15204
15395
|
|
|
15205
15396
|
// src/mcp/memory-inprocess-executor.ts
|
|
@@ -15726,7 +15917,7 @@ async function requireMcpCallerKey(c) {
|
|
|
15726
15917
|
if (!user) return mcpAuthError();
|
|
15727
15918
|
return { key: callerKey, user };
|
|
15728
15919
|
}
|
|
15729
|
-
var mcpApp = new
|
|
15920
|
+
var mcpApp = new Hono16();
|
|
15730
15921
|
mcpApp.all("/", async (c) => {
|
|
15731
15922
|
try {
|
|
15732
15923
|
const keyOrError = await requireMcpCallerKey(c);
|
|
@@ -15752,7 +15943,8 @@ mcpApp.all("/", async (c) => {
|
|
|
15752
15943
|
});
|
|
15753
15944
|
|
|
15754
15945
|
// src/api/browser-agent-routes.ts
|
|
15755
|
-
import {
|
|
15946
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
15947
|
+
import { Hono as Hono17 } from "hono";
|
|
15756
15948
|
|
|
15757
15949
|
// src/api/browser-agent-db.ts
|
|
15758
15950
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
@@ -15840,8 +16032,58 @@ async function migrateBrowserAgent() {
|
|
|
15840
16032
|
} catch {
|
|
15841
16033
|
}
|
|
15842
16034
|
await db.execute(`CREATE INDEX IF NOT EXISTS browser_auth_connections_profile ON browser_auth_connections(profile)`);
|
|
16035
|
+
await db.execute(`
|
|
16036
|
+
CREATE TABLE IF NOT EXISTS browser_agent_extensions (
|
|
16037
|
+
id TEXT PRIMARY KEY,
|
|
16038
|
+
user_id INTEGER NOT NULL,
|
|
16039
|
+
name TEXT NOT NULL,
|
|
16040
|
+
backend_id TEXT NOT NULL,
|
|
16041
|
+
backend_name TEXT NOT NULL,
|
|
16042
|
+
source TEXT NOT NULL DEFAULT 'store',
|
|
16043
|
+
source_url TEXT,
|
|
16044
|
+
size_bytes INTEGER,
|
|
16045
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
16046
|
+
)
|
|
16047
|
+
`);
|
|
16048
|
+
await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS browser_agent_extensions_user_name ON browser_agent_extensions(user_id, name)`);
|
|
16049
|
+
await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_extensions_user ON browser_agent_extensions(user_id)`);
|
|
15843
16050
|
_ready = true;
|
|
15844
16051
|
}
|
|
16052
|
+
async function createExtensionRow(input) {
|
|
16053
|
+
const db = getDb();
|
|
16054
|
+
const id = `bext_${randomUUID2().replace(/-/g, "").slice(0, 20)}`;
|
|
16055
|
+
await db.execute({
|
|
16056
|
+
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
16057
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
16058
|
+
args: [id, input.userId, input.name, input.backendId, input.backendName, input.source, input.sourceUrl, input.sizeBytes]
|
|
16059
|
+
});
|
|
16060
|
+
const row = await getExtensionRow(input.userId, input.name);
|
|
16061
|
+
if (!row) throw new Error("extension insert failed");
|
|
16062
|
+
return row;
|
|
16063
|
+
}
|
|
16064
|
+
async function getExtensionRow(userId, name) {
|
|
16065
|
+
const db = getDb();
|
|
16066
|
+
const res = await db.execute({
|
|
16067
|
+
sql: `SELECT * FROM browser_agent_extensions WHERE user_id = ? AND name = ?`,
|
|
16068
|
+
args: [userId, name]
|
|
16069
|
+
});
|
|
16070
|
+
return res.rows[0] ?? null;
|
|
16071
|
+
}
|
|
16072
|
+
async function listExtensionRows(userId) {
|
|
16073
|
+
const db = getDb();
|
|
16074
|
+
const res = await db.execute({
|
|
16075
|
+
sql: `SELECT * FROM browser_agent_extensions WHERE user_id = ? ORDER BY created_at DESC`,
|
|
16076
|
+
args: [userId]
|
|
16077
|
+
});
|
|
16078
|
+
return res.rows;
|
|
16079
|
+
}
|
|
16080
|
+
async function deleteExtensionRow(userId, name) {
|
|
16081
|
+
const db = getDb();
|
|
16082
|
+
await db.execute({
|
|
16083
|
+
sql: `DELETE FROM browser_agent_extensions WHERE user_id = ? AND name = ?`,
|
|
16084
|
+
args: [userId, name]
|
|
16085
|
+
});
|
|
16086
|
+
}
|
|
15845
16087
|
async function createAuthConnectionRow(input) {
|
|
15846
16088
|
const db = getDb();
|
|
15847
16089
|
const connectionId = `authc_${randomUUID2().replace(/-/g, "").slice(0, 20)}`;
|
|
@@ -16698,6 +16940,28 @@ function isProfileConflict(err) {
|
|
|
16698
16940
|
const message = err instanceof Error ? err.message : String(err);
|
|
16699
16941
|
return /\b409\b|conflict|already exists/i.test(message);
|
|
16700
16942
|
}
|
|
16943
|
+
function isNotFound(err) {
|
|
16944
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
16945
|
+
return /\b404\b|not found/i.test(message);
|
|
16946
|
+
}
|
|
16947
|
+
async function importExtensionFromStore(storeUrl, backendName) {
|
|
16948
|
+
const k = client();
|
|
16949
|
+
const archive = await k.extensions.downloadFromChromeStore({ url: storeUrl });
|
|
16950
|
+
const uploaded = await k.extensions.upload({ file: archive, name: backendName });
|
|
16951
|
+
return {
|
|
16952
|
+
backendId: uploaded.id,
|
|
16953
|
+
backendName: uploaded.name ?? backendName,
|
|
16954
|
+
sizeBytes: typeof uploaded.size_bytes === "number" ? uploaded.size_bytes : null
|
|
16955
|
+
};
|
|
16956
|
+
}
|
|
16957
|
+
async function deleteExtensionBackend(backendName) {
|
|
16958
|
+
const k = client();
|
|
16959
|
+
try {
|
|
16960
|
+
await k.extensions.delete(backendName);
|
|
16961
|
+
} catch (err) {
|
|
16962
|
+
if (!isNotFound(err)) throw err;
|
|
16963
|
+
}
|
|
16964
|
+
}
|
|
16701
16965
|
async function ensureProfile(k, name) {
|
|
16702
16966
|
try {
|
|
16703
16967
|
await k.profiles.create({ name });
|
|
@@ -16716,6 +16980,7 @@ async function createSession(opts = {}) {
|
|
|
16716
16980
|
const resolvedSaveProfileChanges = opts.saveProfileChanges ?? browserServiceProfileSaveChanges();
|
|
16717
16981
|
const disableDefaultProxy = opts.disableDefaultProxy ?? !resolvedProxyId;
|
|
16718
16982
|
const startUrl = opts.startUrl?.trim();
|
|
16983
|
+
const extensionNames = (opts.extensionNames ?? []).map((name) => name.trim()).filter(Boolean);
|
|
16719
16984
|
if (resolvedProfileName && resolvedSaveProfileChanges === true) {
|
|
16720
16985
|
await ensureProfile(k, resolvedProfileName);
|
|
16721
16986
|
}
|
|
@@ -16729,7 +16994,8 @@ async function createSession(opts = {}) {
|
|
|
16729
16994
|
name: resolvedProfileName,
|
|
16730
16995
|
...typeof resolvedSaveProfileChanges === "boolean" ? { save_changes: resolvedSaveProfileChanges } : {}
|
|
16731
16996
|
}
|
|
16732
|
-
} : {}
|
|
16997
|
+
} : {},
|
|
16998
|
+
...extensionNames.length ? { extensions: extensionNames.map((name) => ({ name })) } : {}
|
|
16733
16999
|
};
|
|
16734
17000
|
const browser = await k.browsers.create(createPayload);
|
|
16735
17001
|
const runtimeSessionId = browser.session_id;
|
|
@@ -17117,6 +17383,16 @@ function parseAgentDate(value) {
|
|
|
17117
17383
|
const legacyParsed = Date.parse(`${value.replace(" ", "T")}Z`);
|
|
17118
17384
|
return Number.isFinite(legacyParsed) ? legacyParsed : null;
|
|
17119
17385
|
}
|
|
17386
|
+
var EXTENSION_NAME_RE = /^[\w .-]{1,64}$/;
|
|
17387
|
+
function publicExtension(row) {
|
|
17388
|
+
return {
|
|
17389
|
+
name: row.name,
|
|
17390
|
+
source: row.source,
|
|
17391
|
+
source_url: row.source_url,
|
|
17392
|
+
size_bytes: row.size_bytes,
|
|
17393
|
+
created_at: row.created_at
|
|
17394
|
+
};
|
|
17395
|
+
}
|
|
17120
17396
|
function replayClock(row) {
|
|
17121
17397
|
if (!row) return null;
|
|
17122
17398
|
const startedAtMs = parseAgentDate(row.started_at);
|
|
@@ -17143,7 +17419,7 @@ async function loadOpenSession(id, userId) {
|
|
|
17143
17419
|
return row;
|
|
17144
17420
|
}
|
|
17145
17421
|
function buildBrowserAgentRoutes() {
|
|
17146
|
-
const app2 = new
|
|
17422
|
+
const app2 = new Hono17();
|
|
17147
17423
|
app2.use("*", async (c, next) => {
|
|
17148
17424
|
await migrateBrowserAgent();
|
|
17149
17425
|
return next();
|
|
@@ -17263,6 +17539,15 @@ function buildBrowserAgentRoutes() {
|
|
|
17263
17539
|
const body = await c.req.json().catch(() => ({}));
|
|
17264
17540
|
const timeoutSeconds = typeof body.timeout_seconds === "number" ? body.timeout_seconds : void 0;
|
|
17265
17541
|
const startUrl = typeof body.url === "string" ? body.url : void 0;
|
|
17542
|
+
const extensionNames = Array.isArray(body.extension_names) ? body.extension_names.filter((v) => typeof v === "string") : void 0;
|
|
17543
|
+
let resolvedExtensionBackendNames;
|
|
17544
|
+
if (extensionNames && extensionNames.length) {
|
|
17545
|
+
const names = extensionNames;
|
|
17546
|
+
const rows = await Promise.all(names.map((name) => getExtensionRow(user.id, name)));
|
|
17547
|
+
const missing = names.filter((_, i) => !rows[i]);
|
|
17548
|
+
if (missing.length) return c.json({ error: `extension(s) not found: ${missing.join(", ")}` }, 400);
|
|
17549
|
+
resolvedExtensionBackendNames = rows.map((row) => row.backend_name);
|
|
17550
|
+
}
|
|
17266
17551
|
const gate = await acquireConcurrencyGate(user, "browser_agent_session", {
|
|
17267
17552
|
ttlSeconds: browserSessionLockTtlSeconds(timeoutSeconds),
|
|
17268
17553
|
metadata: { label: typeof body.label === "string" ? body.label : null }
|
|
@@ -17277,7 +17562,8 @@ function buildBrowserAgentRoutes() {
|
|
|
17277
17562
|
saveProfileChanges: typeof body.save_profile_changes === "boolean" ? body.save_profile_changes : void 0,
|
|
17278
17563
|
startUrl,
|
|
17279
17564
|
disableDefaultProxy: typeof body.disable_default_proxy === "boolean" ? body.disable_default_proxy : void 0,
|
|
17280
|
-
viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0
|
|
17565
|
+
viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0,
|
|
17566
|
+
extensionNames: resolvedExtensionBackendNames
|
|
17281
17567
|
});
|
|
17282
17568
|
runtimeSessionId = created.runtimeSessionId;
|
|
17283
17569
|
const row = await createSessionRow({
|
|
@@ -17580,6 +17866,52 @@ function buildBrowserAgentRoutes() {
|
|
|
17580
17866
|
}))
|
|
17581
17867
|
});
|
|
17582
17868
|
});
|
|
17869
|
+
app2.post("/extensions/import", async (c) => {
|
|
17870
|
+
const user = c.get("user");
|
|
17871
|
+
const body = await c.req.json().catch(() => ({}));
|
|
17872
|
+
const storeUrl = typeof body.store_url === "string" ? body.store_url.trim() : "";
|
|
17873
|
+
const name = typeof body.name === "string" ? body.name.trim() : "";
|
|
17874
|
+
if (!storeUrl) return c.json({ error: "store_url is required" }, 400);
|
|
17875
|
+
if (!name || !EXTENSION_NAME_RE.test(name)) {
|
|
17876
|
+
return c.json({ error: "name is required (1-64 characters: letters, numbers, spaces, dots, underscores, hyphens)" }, 400);
|
|
17877
|
+
}
|
|
17878
|
+
const existing = await getExtensionRow(user.id, name);
|
|
17879
|
+
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
17880
|
+
const backendName = `u${user.id}_${randomUUID3().replace(/-/g, "")}`;
|
|
17881
|
+
try {
|
|
17882
|
+
const imported = await importExtensionFromStore(storeUrl, backendName);
|
|
17883
|
+
const row = await createExtensionRow({
|
|
17884
|
+
userId: user.id,
|
|
17885
|
+
name,
|
|
17886
|
+
backendId: imported.backendId,
|
|
17887
|
+
backendName: imported.backendName,
|
|
17888
|
+
source: "store",
|
|
17889
|
+
sourceUrl: storeUrl,
|
|
17890
|
+
sizeBytes: imported.sizeBytes
|
|
17891
|
+
});
|
|
17892
|
+
return c.json(publicExtension(row));
|
|
17893
|
+
} catch (err) {
|
|
17894
|
+
return c.json(failure(err), 502);
|
|
17895
|
+
}
|
|
17896
|
+
});
|
|
17897
|
+
app2.get("/extensions", async (c) => {
|
|
17898
|
+
const user = c.get("user");
|
|
17899
|
+
const rows = await listExtensionRows(user.id);
|
|
17900
|
+
return c.json({ extensions: rows.map(publicExtension) });
|
|
17901
|
+
});
|
|
17902
|
+
app2.delete("/extensions/:name", async (c) => {
|
|
17903
|
+
const user = c.get("user");
|
|
17904
|
+
const name = c.req.param("name");
|
|
17905
|
+
const row = await getExtensionRow(user.id, name);
|
|
17906
|
+
if (!row) return c.json({ error: "not found" }, 404);
|
|
17907
|
+
try {
|
|
17908
|
+
await deleteExtensionBackend(row.backend_name);
|
|
17909
|
+
} catch (err) {
|
|
17910
|
+
return c.json(failure(err), 502);
|
|
17911
|
+
}
|
|
17912
|
+
await deleteExtensionRow(user.id, name);
|
|
17913
|
+
return c.json({ ok: true });
|
|
17914
|
+
});
|
|
17583
17915
|
app2.get("/sessions/:id/replays/:replayId/download", async (c) => {
|
|
17584
17916
|
const user = c.get("user");
|
|
17585
17917
|
const row = await loadOpenSession(c.req.param("id"), user.id);
|
|
@@ -17877,9 +18209,9 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
|
|
|
17877
18209
|
|
|
17878
18210
|
// src/api/stripe-routes.ts
|
|
17879
18211
|
import Stripe from "stripe";
|
|
17880
|
-
import { Hono as
|
|
18212
|
+
import { Hono as Hono18 } from "hono";
|
|
17881
18213
|
var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
|
|
17882
|
-
var stripeApp = new
|
|
18214
|
+
var stripeApp = new Hono18();
|
|
17883
18215
|
stripeApp.post("/webhooks", async (c) => {
|
|
17884
18216
|
const sig = c.req.header("stripe-signature");
|
|
17885
18217
|
const body = await c.req.text();
|
|
@@ -17935,7 +18267,6 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
17935
18267
|
const live = sub.status === "active" || sub.status === "trialing";
|
|
17936
18268
|
const newTier = live ? tier.tier : null;
|
|
17937
18269
|
await setSubscriptionTier(user.id, newTier, live ? tier.concurrency : 0, live ? sub.id : null);
|
|
17938
|
-
await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, subscription_tier: newTier }));
|
|
17939
18270
|
} else if (priceId === CONCURRENCY_PRICE_ID && event.type === "customer.subscription.created") {
|
|
17940
18271
|
await setExtraConcurrencySlots(user.id, user.extra_concurrency_slots + 1);
|
|
17941
18272
|
await setConcurrencySubId(user.id, sub.id);
|
|
@@ -17961,7 +18292,6 @@ stripeApp.post("/webhooks", async (c) => {
|
|
|
17961
18292
|
if (!user) return c.json({ received: true });
|
|
17962
18293
|
if (priceId && priceId in SUBSCRIPTION_TIERS) {
|
|
17963
18294
|
await setSubscriptionTier(user.id, null, 0, null);
|
|
17964
|
-
await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, subscription_tier: null }));
|
|
17965
18295
|
} else if (priceId === CONCURRENCY_PRICE_ID) {
|
|
17966
18296
|
await setExtraConcurrencySlots(user.id, Math.max(0, user.extra_concurrency_slots - 1));
|
|
17967
18297
|
await setConcurrencySubId(user.id, null);
|
|
@@ -17983,9 +18313,9 @@ async function resolveUser(customerId, emailFallback) {
|
|
|
17983
18313
|
}
|
|
17984
18314
|
|
|
17985
18315
|
// src/api/oauth-routes.ts
|
|
17986
|
-
import { Hono as
|
|
18316
|
+
import { Hono as Hono19 } from "hono";
|
|
17987
18317
|
import { getCookie, setCookie } from "hono/cookie";
|
|
17988
|
-
import { createHash as createHash3, randomBytes as randomBytes2, randomUUID as
|
|
18318
|
+
import { createHash as createHash3, randomBytes as randomBytes2, randomUUID as randomUUID4 } from "crypto";
|
|
17989
18319
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
17990
18320
|
var ISSUER = "https://mcpscraper.dev";
|
|
17991
18321
|
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
|
|
@@ -18152,7 +18482,7 @@ function redirectWithError(redirectUri, state, error) {
|
|
|
18152
18482
|
if (state) u.searchParams.set("state", state);
|
|
18153
18483
|
return new Response(null, { status: 302, headers: { Location: u.toString() } });
|
|
18154
18484
|
}
|
|
18155
|
-
var oauthApp = new
|
|
18485
|
+
var oauthApp = new Hono19();
|
|
18156
18486
|
oauthApp.use("*", async (c, next) => {
|
|
18157
18487
|
c.header("Access-Control-Allow-Origin", "*");
|
|
18158
18488
|
c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
|
|
@@ -18333,7 +18663,7 @@ function pkceMatches(verifier, challenge) {
|
|
|
18333
18663
|
}
|
|
18334
18664
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
18335
18665
|
const { privateKey, kid } = await getKeys();
|
|
18336
|
-
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(
|
|
18666
|
+
return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID4()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
18337
18667
|
}
|
|
18338
18668
|
function tokenErrorResponse(c, error, description, status) {
|
|
18339
18669
|
return c.json({ error, error_description: description }, status);
|
|
@@ -18406,8 +18736,8 @@ oauthApp.post("/oauth/token", async (c) => {
|
|
|
18406
18736
|
});
|
|
18407
18737
|
|
|
18408
18738
|
// src/api/chat-routes.ts
|
|
18409
|
-
import { Hono as
|
|
18410
|
-
var chatApp = new
|
|
18739
|
+
import { Hono as Hono20 } from "hono";
|
|
18740
|
+
var chatApp = new Hono20();
|
|
18411
18741
|
function tokenOk(token) {
|
|
18412
18742
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
18413
18743
|
}
|
|
@@ -19077,8 +19407,8 @@ loadChannels();
|
|
|
19077
19407
|
}
|
|
19078
19408
|
|
|
19079
19409
|
// src/api/schedule-routes.ts
|
|
19080
|
-
import { Hono as
|
|
19081
|
-
var scheduleApp = new
|
|
19410
|
+
import { Hono as Hono21 } from "hono";
|
|
19411
|
+
var scheduleApp = new Hono21();
|
|
19082
19412
|
function tokenOk2(token) {
|
|
19083
19413
|
return !!token && token.startsWith("mk_") && token.length > 10;
|
|
19084
19414
|
}
|
|
@@ -19833,7 +20163,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
|
|
|
19833
20163
|
c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
|
|
19834
20164
|
return next();
|
|
19835
20165
|
});
|
|
19836
|
-
var app = new
|
|
20166
|
+
var app = new Hono22();
|
|
19837
20167
|
var STRIPE_API_VERSION = "2026-02-25.clover";
|
|
19838
20168
|
function requireStripeSecret() {
|
|
19839
20169
|
const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
|
|
@@ -21194,7 +21524,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
21194
21524
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
21195
21525
|
return c.json({ error: "Unauthorized" }, 401);
|
|
21196
21526
|
}
|
|
21197
|
-
const { drainQueue } = await import("./worker-
|
|
21527
|
+
const { drainQueue } = await import("./worker-YHKUJR5P.js");
|
|
21198
21528
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
21199
21529
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
21200
21530
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
|
|
@@ -21212,7 +21542,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
21212
21542
|
return c.json({ error: "Unauthorized" }, 401);
|
|
21213
21543
|
}
|
|
21214
21544
|
const jobId = c.req.param("id");
|
|
21215
|
-
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-
|
|
21545
|
+
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-THBVEXMP.js");
|
|
21216
21546
|
const { assembleExtractArtifacts } = await import("./extract-bundle-UKE273TV.js");
|
|
21217
21547
|
const job = await getExtractJob2(jobId);
|
|
21218
21548
|
if (!job) return c.json({ error: "job not found" }, 404);
|
|
@@ -21220,7 +21550,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
21220
21550
|
await completeExtractJob2(jobId, stored);
|
|
21221
21551
|
let settlement = "already_settled_or_refunded";
|
|
21222
21552
|
if (job.billedMc == null && job.userId != null) {
|
|
21223
|
-
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
21553
|
+
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-THBVEXMP.js");
|
|
21224
21554
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
21225
21555
|
const successful = await countSuccessfulPages2(jobId);
|
|
21226
21556
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -21247,6 +21577,7 @@ app.route("/reddit", redditApp);
|
|
|
21247
21577
|
app.route("/video", videoApp);
|
|
21248
21578
|
app.route("/maps", mapsApp);
|
|
21249
21579
|
app.route("/trustpilot", trustpilotApp);
|
|
21580
|
+
app.route("/g2", g2App);
|
|
21250
21581
|
app.route("/directory", directoryApp);
|
|
21251
21582
|
app.route("/workflows", workflowApp);
|
|
21252
21583
|
app.route("/serp-intelligence", serpIntelligenceApp);
|
|
@@ -21383,4 +21714,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
21383
21714
|
export {
|
|
21384
21715
|
app
|
|
21385
21716
|
};
|
|
21386
|
-
//# sourceMappingURL=server-
|
|
21717
|
+
//# sourceMappingURL=server-XCDFHHLE.js.map
|