mcp-scraper 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/bin/api-server.cjs +2394 -1851
  2. package/dist/bin/api-server.cjs.map +1 -1
  3. package/dist/bin/api-server.js +2 -2
  4. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  5. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  6. package/dist/bin/mcp-scraper-cli.js +1 -1
  7. package/dist/bin/mcp-scraper-install.cjs +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  9. package/dist/bin/mcp-scraper-install.js +1 -1
  10. package/dist/bin/mcp-stdio-server.cjs +179 -3
  11. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  12. package/dist/bin/mcp-stdio-server.js +2 -2
  13. package/dist/{chunk-NVUBFCJK.js → chunk-4ZZWFI6L.js} +2 -2
  14. package/dist/{chunk-4MFYMUIF.js → chunk-5LO6EHEH.js} +25 -3
  15. package/dist/chunk-5LO6EHEH.js.map +1 -0
  16. package/dist/chunk-UGQC2FOX.js +7 -0
  17. package/dist/chunk-UGQC2FOX.js.map +1 -0
  18. package/dist/{chunk-BAMVP57P.js → chunk-WI4A3KFQ.js} +180 -4
  19. package/dist/chunk-WI4A3KFQ.js.map +1 -0
  20. package/dist/{server-LBEYRKM2.js → server-XCDFHHLE.js} +551 -220
  21. package/dist/server-XCDFHHLE.js.map +1 -0
  22. package/dist/{site-extract-repository-4NVO5KQA.js → site-extract-repository-THBVEXMP.js} +3 -3
  23. package/dist/{worker-7XRZNLKB.js → worker-YHKUJR5P.js} +2 -2
  24. package/package.json +1 -1
  25. package/dist/chunk-4MFYMUIF.js.map +0 -1
  26. package/dist/chunk-BAMVP57P.js.map +0 -1
  27. package/dist/chunk-SRMGSCC3.js +0 -7
  28. package/dist/chunk-SRMGSCC3.js.map +0 -1
  29. package/dist/server-LBEYRKM2.js.map +0 -1
  30. /package/dist/{chunk-NVUBFCJK.js.map → chunk-4ZZWFI6L.js.map} +0 -0
  31. /package/dist/{site-extract-repository-4NVO5KQA.js.map → site-extract-repository-THBVEXMP.js.map} +0 -0
  32. /package/dist/{worker-7XRZNLKB.js.map → worker-YHKUJR5P.js.map} +0 -0
@@ -31,7 +31,7 @@ import {
31
31
  registerSerpIntelligenceCaptureTools,
32
32
  sanitizeAttempts,
33
33
  sanitizeHarvestResult
34
- } from "./chunk-BAMVP57P.js";
34
+ } from "./chunk-WI4A3KFQ.js";
35
35
  import {
36
36
  auditImages,
37
37
  getBlobStore
@@ -65,7 +65,7 @@ import {
65
65
  RawMapsOverviewSchema,
66
66
  RawMapsReviewStatsSchema
67
67
  } from "./chunk-XGIPATLV.js";
68
- import "./chunk-SRMGSCC3.js";
68
+ import "./chunk-UGQC2FOX.js";
69
69
  import {
70
70
  completeExtractJob,
71
71
  countSuccessfulPages,
@@ -76,7 +76,7 @@ import {
76
76
  saveExtractPages,
77
77
  setExtractJobTotal,
78
78
  settleExtractJob
79
- } from "./chunk-NVUBFCJK.js";
79
+ } from "./chunk-4ZZWFI6L.js";
80
80
  import {
81
81
  BROWSER_OPEN_MIN_BALANCE_MC,
82
82
  CONCURRENCY_PRICE_ID,
@@ -95,7 +95,7 @@ import {
95
95
  browserActiveCostMc,
96
96
  concurrencySlotBillingInfo,
97
97
  insufficientBalanceResponse
98
- } from "./chunk-4MFYMUIF.js";
98
+ } from "./chunk-5LO6EHEH.js";
99
99
  import {
100
100
  CaptchaError,
101
101
  RECAPTCHA_INSTRUCTIONS,
@@ -5274,7 +5274,7 @@ async function extractSite(opts) {
5274
5274
  }
5275
5275
 
5276
5276
  // src/api/server.ts
5277
- import { Hono as Hono21 } from "hono";
5277
+ import { Hono as Hono22 } from "hono";
5278
5278
  import { serve as serveInngest } from "inngest/hono";
5279
5279
 
5280
5280
  // src/inngest/client.ts
@@ -11935,23 +11935,8 @@ async function provisionMemoryForUser(user) {
11935
11935
  } catch {
11936
11936
  }
11937
11937
  }
11938
- function memoryPlanForUser(user) {
11939
- switch (user.subscription_tier) {
11940
- case "starter":
11941
- return "pro";
11942
- case "growth":
11943
- return "team";
11944
- case "scale":
11945
- return "enterprise";
11946
- default:
11947
- return "free";
11948
- }
11949
- }
11950
- var MEMORY_PLAN_RANK = { free: 0, pro: 1, team: 2, enterprise: 3 };
11951
11938
  function resolveEffectiveMemoryPlan(user) {
11952
- const tierPlan = memoryPlanForUser(user);
11953
- const memoryProPlan = user.memory_plan === "pro" || user.memory_plan === "team" ? user.memory_plan : "free";
11954
- return MEMORY_PLAN_RANK[tierPlan] >= MEMORY_PLAN_RANK[memoryProPlan] ? tierPlan : memoryProPlan;
11939
+ return user.memory_plan === "pro" || user.memory_plan === "team" ? user.memory_plan : "free";
11955
11940
  }
11956
11941
  async function getOrCreateUserMemoryKey(user) {
11957
11942
  const creds = await getUserMemoryCreds(user.id);
@@ -13310,13 +13295,219 @@ trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
13310
13295
  }
13311
13296
  });
13312
13297
 
13313
- // src/api/directory-routes.ts
13298
+ // src/api/g2-routes.ts
13314
13299
  import { Hono as Hono12 } from "hono";
13300
+ import { z as z20 } from "zod";
13301
+ var G2ReviewsBodySchema = z20.object({
13302
+ product: z20.string().trim().min(1, "product is required"),
13303
+ maxPages: z20.number().int().min(1).max(50).default(5)
13304
+ });
13305
+ function invalidRequest7(message) {
13306
+ return { error_code: "invalid_request", message };
13307
+ }
13308
+ function normalizeProductSlug(input) {
13309
+ return input.trim().replace(/^https?:\/\/(www\.)?g2\.com\/products\//i, "").replace(/\/.*$/, "").replace(/\s+/g, "-").toLowerCase();
13310
+ }
13311
+ function reviewPageUrl2(product, page) {
13312
+ const base = `https://www.g2.com/products/${encodeURIComponent(product)}/reviews`;
13313
+ return page <= 1 ? base : `${base}?page=${page}`;
13314
+ }
13315
+ async function residentialProxyId2(attemptIndex) {
13316
+ try {
13317
+ const r = await resolveKernelProxyId({
13318
+ kernelApiKey: browserServiceApiKey(),
13319
+ proxyMode: "location",
13320
+ location: "Austin, TX",
13321
+ gl: "us",
13322
+ attemptIndex,
13323
+ fresh: true
13324
+ });
13325
+ return r.kernelProxyId;
13326
+ } catch {
13327
+ return void 0;
13328
+ }
13329
+ }
13330
+ async function extractG2ReviewCards(page) {
13331
+ return page.evaluate(() => {
13332
+ const cards = Array.from(document.querySelectorAll("article[data-track-in-viewport-options]"));
13333
+ return cards.map((card) => {
13334
+ let meta = {};
13335
+ try {
13336
+ meta = JSON.parse(card.getAttribute("data-track-in-viewport-options") ?? "{}");
13337
+ } catch {
13338
+ meta = {};
13339
+ }
13340
+ const reviewId = meta.survey_response_id != null ? String(meta.survey_response_id) : null;
13341
+ const publishedDateRaw = typeof meta.published_date === "string" ? meta.published_date : null;
13342
+ const date = publishedDateRaw && /^\d{8}$/.test(publishedDateRaw) ? `${publishedDateRaw.slice(0, 4)}-${publishedDateRaw.slice(4, 6)}-${publishedDateRaw.slice(6, 8)}` : null;
13343
+ const profileLink = card.querySelector('a[href*="/users/"]');
13344
+ const profileHref = profileLink?.getAttribute("href") ?? "";
13345
+ const profileMatch = profileHref.match(/\/users\/([a-f0-9-]+)/i);
13346
+ const profileId = profileMatch ? profileMatch[1] : null;
13347
+ const infoWrapper = profileLink?.parentElement;
13348
+ const infoDiv = infoWrapper ? Array.from(infoWrapper.children).find((el2) => el2 !== profileLink) : null;
13349
+ const name = infoDiv?.querySelector('div[class*="font-bold"]')?.textContent?.trim() ?? null;
13350
+ const subtleDivs = infoDiv ? Array.from(infoDiv.querySelectorAll('div[class*="text-subtle"]')) : [];
13351
+ const reviewerTitle = subtleDivs[0]?.textContent?.trim() ?? null;
13352
+ const companySegment = subtleDivs[1]?.textContent?.trim() ?? null;
13353
+ const ratingEl = card.querySelector('[class*="star-wrapper__desc__rating"]');
13354
+ const ratingText = ratingEl?.textContent?.trim() ?? "";
13355
+ const ratingParts = ratingText.split("/");
13356
+ const rating = ratingParts[0] ? parseFloat(ratingParts[0]) : null;
13357
+ const ratingScale = ratingParts[1] ? parseFloat(ratingParts[1]) : 5;
13358
+ const titleEl = card.querySelector('div[class*="text-lg"][class*="font-bold"]');
13359
+ const title = titleEl?.textContent?.trim().replace(/^["“]+|["”]+$/g, "") ?? null;
13360
+ const sections = Array.from(card.querySelectorAll("section"));
13361
+ const body = sections.map((sec) => {
13362
+ const question = sec.querySelector('div[class*="font-bold"]')?.textContent?.trim() ?? null;
13363
+ const paragraphs = Array.from(sec.querySelectorAll("p")).map((p) => p.textContent?.trim() ?? "");
13364
+ const answer = paragraphs.join("\n\n").replace(/Review collected by and hosted on G2\.com\.\s*$/, "").trim();
13365
+ return { question, answer };
13366
+ }).filter((b) => b.answer.length > 0);
13367
+ const flagButtons = Array.from(card.querySelectorAll("button[title]"));
13368
+ let currentUser = false;
13369
+ let validated = false;
13370
+ let incentivized = false;
13371
+ let origin = null;
13372
+ for (const btn of flagButtons) {
13373
+ const label = btn.querySelector("label")?.textContent?.trim() ?? "";
13374
+ if (/current user/i.test(label)) currentUser = true;
13375
+ else if (/validated/i.test(label)) validated = true;
13376
+ else if (/^incentivized$/i.test(label)) incentivized = true;
13377
+ else if (/^source:/i.test(label)) origin = label.replace(/^source:\s*/i, "").trim();
13378
+ }
13379
+ const copyBtn = card.querySelector('[data-clipboard-text*="/reviews/"]');
13380
+ const reviewUrl = copyBtn?.getAttribute("data-clipboard-text") ?? null;
13381
+ return {
13382
+ source: "g2",
13383
+ sourceReviewId: reviewId,
13384
+ reviewUrl,
13385
+ reviewer: {
13386
+ name,
13387
+ profileId,
13388
+ title: reviewerTitle,
13389
+ companySegment
13390
+ },
13391
+ rating,
13392
+ ratingScale,
13393
+ title,
13394
+ date,
13395
+ body: body.length > 0 ? body : [{ question: null, answer: "" }],
13396
+ truncated: false,
13397
+ flags: {
13398
+ origin,
13399
+ incentivized,
13400
+ validated,
13401
+ currentUser,
13402
+ companyReplied: null
13403
+ }
13404
+ };
13405
+ });
13406
+ });
13407
+ }
13408
+ var g2App = new Hono12();
13409
+ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
13410
+ const raw = await c.req.json().catch(() => ({}));
13411
+ const parsed = G2ReviewsBodySchema.safeParse(raw);
13412
+ if (!parsed.success) return c.json(invalidRequest7(parsed.error.issues[0]?.message ?? "Invalid request"), 400);
13413
+ const body = parsed.data;
13414
+ const product = normalizeProductSlug(body.product);
13415
+ const user = c.get("user");
13416
+ const gate = await acquireConcurrencyGate(user, "g2_reviews", {
13417
+ reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
13418
+ metadata: { product, maxPages: body.maxPages }
13419
+ });
13420
+ if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
13421
+ let debited = false;
13422
+ let reviewDebitMc = 0;
13423
+ try {
13424
+ const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.g2_reviews, LedgerOperation.G2_REVIEWS, product);
13425
+ if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.g2_reviews), 402);
13426
+ debited = true;
13427
+ const startMs = Date.now();
13428
+ const allReviews = [];
13429
+ let pagesFetched = 0;
13430
+ let blocked = false;
13431
+ for (let attempt = 0; attempt < 3 && pagesFetched === 0; attempt++) {
13432
+ const proxyId = await residentialProxyId2(attempt);
13433
+ const driver = new BrowserDriver();
13434
+ try {
13435
+ await driver.launch({
13436
+ headless: true,
13437
+ kernelApiKey: browserServiceApiKey(),
13438
+ kernelProxyId: proxyId,
13439
+ viewport: { width: 1280, height: 900 },
13440
+ locale: "en-US"
13441
+ });
13442
+ const page = driver.getPage();
13443
+ for (let pageNum = 1; pageNum <= body.maxPages; pageNum++) {
13444
+ const url = reviewPageUrl2(product, pageNum);
13445
+ await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
13446
+ await page.waitForTimeout(5e3);
13447
+ await page.evaluate(() => window.scrollTo(0, 1200)).catch(() => {
13448
+ });
13449
+ await page.waitForTimeout(3e3);
13450
+ let cards;
13451
+ try {
13452
+ cards = await extractG2ReviewCards(page);
13453
+ } catch {
13454
+ await page.waitForTimeout(3e3);
13455
+ cards = await extractG2ReviewCards(page);
13456
+ }
13457
+ if (cards.length === 0) break;
13458
+ allReviews.push(...cards);
13459
+ pagesFetched = pageNum;
13460
+ }
13461
+ } catch {
13462
+ blocked = true;
13463
+ } finally {
13464
+ await driver.close();
13465
+ }
13466
+ }
13467
+ if (pagesFetched === 0 || allReviews.length === 0) {
13468
+ await creditMc(user.id, MC_COSTS.g2_reviews, LedgerOperation.G2_REVIEWS_REFUND, blocked ? "blocked after retries" : "no reviews found");
13469
+ await logRequestEvent({ userId: user.id, source: "g2_reviews", status: "failed", query: product, error: blocked ? "blocked after retries" : "no reviews found" });
13470
+ return c.json({ error: "No reviews found for this product (refunded)", product }, 404);
13471
+ }
13472
+ reviewDebitMc = allReviews.length * MC_COSTS.g2_review;
13473
+ if (reviewDebitMc > 0) {
13474
+ const reviewDebit = await debitMc(user.id, reviewDebitMc, LedgerOperation.G2_REVIEW, product);
13475
+ if (!reviewDebit.ok) {
13476
+ allReviews.length = Math.floor((reviewDebit.balance_mc ?? 0) / MC_COSTS.g2_review);
13477
+ reviewDebitMc = allReviews.length * MC_COSTS.g2_review;
13478
+ if (reviewDebitMc > 0) await debitMc(user.id, reviewDebitMc, LedgerOperation.G2_REVIEW, product);
13479
+ }
13480
+ }
13481
+ const result = {
13482
+ product,
13483
+ reviewUrl: reviewPageUrl2(product, 1),
13484
+ extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
13485
+ requestedMaxPages: body.maxPages,
13486
+ pagesFetched,
13487
+ reviewCount: allReviews.length,
13488
+ reviews: allReviews,
13489
+ durationMs: Date.now() - startMs
13490
+ };
13491
+ await logRequestEvent({ userId: user.id, source: "g2_reviews", status: "done", query: product, resultCount: allReviews.length, result });
13492
+ return c.json(result);
13493
+ } catch (err) {
13494
+ if (debited) await creditMc(user.id, MC_COSTS.g2_reviews, LedgerOperation.G2_REVIEWS_REFUND, "failed call");
13495
+ if (reviewDebitMc > 0) await creditMc(user.id, reviewDebitMc, LedgerOperation.G2_REVIEW_REFUND, "failed call");
13496
+ const msg = err instanceof Error ? err.message : String(err);
13497
+ await logRequestEvent({ userId: user.id, source: "g2_reviews", status: "failed", query: product, error: msg });
13498
+ return c.json({ error: msg }, 500);
13499
+ } finally {
13500
+ await releaseConcurrencyGate(gate.lockId);
13501
+ }
13502
+ });
13503
+
13504
+ // src/api/directory-routes.ts
13505
+ import { Hono as Hono13 } from "hono";
13315
13506
 
13316
13507
  // src/directory/directory-workflow.ts
13317
13508
  import { mkdir as mkdir2, writeFile } from "fs/promises";
13318
13509
  import { join as join5 } from "path";
13319
- import { z as z20 } from "zod";
13510
+ import { z as z21 } from "zod";
13320
13511
 
13321
13512
  // src/directory/location-db.ts
13322
13513
  import { access, readFile } from "fs/promises";
@@ -13477,24 +13668,24 @@ async function resolveDirectoryMarkets(options) {
13477
13668
  }
13478
13669
 
13479
13670
  // src/directory/directory-workflow.ts
13480
- var DirectoryWorkflowOptionsSchema = z20.object({
13481
- query: z20.string().min(1),
13482
- state: z20.string().min(2).default("TN"),
13483
- minPopulation: z20.number().int().min(0).default(1e5),
13484
- populationYear: z20.union(POPULATION_YEARS.map((year) => z20.literal(year))).default(2025),
13485
- maxCities: z20.number().int().min(1).max(100).default(25),
13486
- maxResultsPerCity: z20.number().int().min(1).max(50).default(50),
13487
- concurrency: z20.number().int().min(1).max(5).default(5),
13488
- includeZipGroups: z20.boolean().default(true),
13489
- usZipsCsvPath: z20.string().optional(),
13490
- saveCsv: z20.boolean().default(true),
13491
- gl: z20.string().length(2).default("us"),
13492
- hl: z20.string().length(2).default("en"),
13493
- proxyMode: z20.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
13494
- proxyZip: z20.string().regex(/^\d{5}$/).optional(),
13495
- debug: z20.boolean().default(false),
13496
- headless: z20.boolean().default(true),
13497
- kernelApiKey: z20.string().optional()
13671
+ var DirectoryWorkflowOptionsSchema = z21.object({
13672
+ query: z21.string().min(1),
13673
+ state: z21.string().min(2).default("TN"),
13674
+ minPopulation: z21.number().int().min(0).default(1e5),
13675
+ populationYear: z21.union(POPULATION_YEARS.map((year) => z21.literal(year))).default(2025),
13676
+ maxCities: z21.number().int().min(1).max(100).default(25),
13677
+ maxResultsPerCity: z21.number().int().min(1).max(50).default(50),
13678
+ concurrency: z21.number().int().min(1).max(5).default(5),
13679
+ includeZipGroups: z21.boolean().default(true),
13680
+ usZipsCsvPath: z21.string().optional(),
13681
+ saveCsv: z21.boolean().default(true),
13682
+ gl: z21.string().length(2).default("us"),
13683
+ hl: z21.string().length(2).default("en"),
13684
+ proxyMode: z21.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
13685
+ proxyZip: z21.string().regex(/^\d{5}$/).optional(),
13686
+ debug: z21.boolean().default(false),
13687
+ headless: z21.boolean().default(true),
13688
+ kernelApiKey: z21.string().optional()
13498
13689
  });
13499
13690
  function errorMessage(err) {
13500
13691
  return err instanceof Error ? err.message : String(err);
@@ -13715,7 +13906,7 @@ async function runDirectoryWorkflowFromPlan(options, plan) {
13715
13906
  }
13716
13907
 
13717
13908
  // src/api/directory-routes.ts
13718
- var directoryApp = new Hono12();
13909
+ var directoryApp = new Hono13();
13719
13910
  directoryApp.post("/run", createApiKeyAuth(), async (c) => {
13720
13911
  const user = c.get("user");
13721
13912
  const body = await c.req.json().catch(() => ({}));
@@ -13788,35 +13979,35 @@ directoryApp.post("/run", createApiKeyAuth(), async (c) => {
13788
13979
  // src/api/workflow-routes.ts
13789
13980
  import { createHmac as createHmac2 } from "crypto";
13790
13981
  import { readFile as readFile2 } from "fs/promises";
13791
- import { Hono as Hono13 } from "hono";
13792
- import { z as z21 } from "zod";
13793
- var workflowApp = new Hono13();
13794
- var WorkflowInputSchema = z21.record(z21.unknown()).default({});
13795
- var WorkflowIdSchema = z21.string().min(1);
13796
- var CadenceSchema = z21.enum(["daily", "weekly", "monthly"]);
13797
- var ScheduleStatusSchema = z21.enum(["active", "paused"]);
13798
- var RunBodySchema = z21.object({
13982
+ import { Hono as Hono14 } from "hono";
13983
+ import { z as z22 } from "zod";
13984
+ var workflowApp = new Hono14();
13985
+ var WorkflowInputSchema = z22.record(z22.unknown()).default({});
13986
+ var WorkflowIdSchema = z22.string().min(1);
13987
+ var CadenceSchema = z22.enum(["daily", "weekly", "monthly"]);
13988
+ var ScheduleStatusSchema = z22.enum(["active", "paused"]);
13989
+ var RunBodySchema = z22.object({
13799
13990
  workflowId: WorkflowIdSchema,
13800
13991
  input: WorkflowInputSchema,
13801
- webhookUrl: z21.string().url().optional()
13992
+ webhookUrl: z22.string().url().optional()
13802
13993
  });
13803
- var ScheduleCreateSchema = z21.object({
13994
+ var ScheduleCreateSchema = z22.object({
13804
13995
  workflowId: WorkflowIdSchema,
13805
- name: z21.string().min(1).max(120).optional(),
13996
+ name: z22.string().min(1).max(120).optional(),
13806
13997
  input: WorkflowInputSchema,
13807
13998
  cadence: CadenceSchema.default("weekly"),
13808
- timezone: z21.string().min(1).max(64).default("UTC"),
13809
- webhookUrl: z21.string().url().optional(),
13810
- nextRunAt: z21.string().datetime().optional()
13999
+ timezone: z22.string().min(1).max(64).default("UTC"),
14000
+ webhookUrl: z22.string().url().optional(),
14001
+ nextRunAt: z22.string().datetime().optional()
13811
14002
  });
13812
- var SchedulePatchSchema = z21.object({
13813
- name: z21.string().min(1).max(120).optional(),
14003
+ var SchedulePatchSchema = z22.object({
14004
+ name: z22.string().min(1).max(120).optional(),
13814
14005
  status: ScheduleStatusSchema.optional(),
13815
14006
  input: WorkflowInputSchema.optional(),
13816
14007
  cadence: CadenceSchema.optional(),
13817
- timezone: z21.string().min(1).max(64).optional(),
13818
- webhookUrl: z21.string().url().nullable().optional(),
13819
- nextRunAt: z21.string().datetime().nullable().optional()
14008
+ timezone: z22.string().min(1).max(64).optional(),
14009
+ webhookUrl: z22.string().url().nullable().optional(),
14010
+ nextRunAt: z22.string().datetime().nullable().optional()
13820
14011
  });
13821
14012
  function hostedWorkflowOutputDir() {
13822
14013
  return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
@@ -14320,7 +14511,7 @@ workflowApp.post("/cron/dispatch", async (c) => {
14320
14511
  });
14321
14512
 
14322
14513
  // src/api/serp-intelligence-routes.ts
14323
- import { Hono as Hono14 } from "hono";
14514
+ import { Hono as Hono15 } from "hono";
14324
14515
 
14325
14516
  // src/serp-intelligence/page-snapshot-extractor.ts
14326
14517
  import { createHash as createHash2 } from "crypto";
@@ -14632,7 +14823,7 @@ async function capturePageSnapshots(targets, options = {}) {
14632
14823
  }
14633
14824
 
14634
14825
  // src/serp-intelligence/schemas.ts
14635
- import { z as z22 } from "zod";
14826
+ import { z as z23 } from "zod";
14636
14827
  var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
14637
14828
  var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
14638
14829
  var SerpIntelligenceAttemptOutcomeValues = [
@@ -14694,171 +14885,171 @@ function isPublicHttpUrl(value) {
14694
14885
  return false;
14695
14886
  }
14696
14887
  }
14697
- var SerpIntelligencePublicHttpUrlSchema = z22.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
14698
- var SerpIntelligenceCaptureBodySchema = z22.object({
14699
- query: z22.string().trim().min(1, "query is required"),
14700
- location: z22.string().trim().min(1).optional(),
14701
- gl: z22.string().trim().length(2).default("us"),
14702
- hl: z22.string().trim().length(2).default("en"),
14703
- device: z22.enum(SerpIntelligenceDeviceValues).default("desktop"),
14704
- proxyMode: z22.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
14705
- proxyZip: z22.string().regex(/^\d{5}$/).optional(),
14706
- pages: z22.number().int().min(1).max(2).default(1),
14707
- debug: z22.boolean().default(false),
14708
- includePageSnapshots: z22.boolean().default(false),
14709
- pageSnapshotLimit: z22.number().int().min(0).max(10).default(0)
14888
+ var SerpIntelligencePublicHttpUrlSchema = z23.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
14889
+ var SerpIntelligenceCaptureBodySchema = z23.object({
14890
+ query: z23.string().trim().min(1, "query is required"),
14891
+ location: z23.string().trim().min(1).optional(),
14892
+ gl: z23.string().trim().length(2).default("us"),
14893
+ hl: z23.string().trim().length(2).default("en"),
14894
+ device: z23.enum(SerpIntelligenceDeviceValues).default("desktop"),
14895
+ proxyMode: z23.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
14896
+ proxyZip: z23.string().regex(/^\d{5}$/).optional(),
14897
+ pages: z23.number().int().min(1).max(2).default(1),
14898
+ debug: z23.boolean().default(false),
14899
+ includePageSnapshots: z23.boolean().default(false),
14900
+ pageSnapshotLimit: z23.number().int().min(0).max(10).default(0)
14710
14901
  }).strict();
14711
- var SerpIntelligencePageSnapshotRequestSchema = z22.object({
14902
+ var SerpIntelligencePageSnapshotRequestSchema = z23.object({
14712
14903
  url: SerpIntelligencePublicHttpUrlSchema,
14713
- sourceKind: z22.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
14714
- sourcePosition: z22.number().int().min(1).optional()
14904
+ sourceKind: z23.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
14905
+ sourcePosition: z23.number().int().min(1).optional()
14715
14906
  }).strict();
14716
- var SerpIntelligencePageSnapshotsBodySchema = z22.object({
14717
- urls: z22.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
14718
- targets: z22.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
14719
- maxConcurrency: z22.number().int().min(1).max(5).default(2),
14720
- timeoutMs: z22.number().int().min(1e3).max(6e4).default(15e3),
14721
- debug: z22.boolean().default(false)
14907
+ var SerpIntelligencePageSnapshotsBodySchema = z23.object({
14908
+ urls: z23.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25),
14909
+ targets: z23.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
14910
+ maxConcurrency: z23.number().int().min(1).max(5).default(2),
14911
+ timeoutMs: z23.number().int().min(1e3).max(6e4).default(15e3),
14912
+ debug: z23.boolean().default(false)
14722
14913
  }).strict();
14723
- var SerpIntelligenceAICitationSchema = z22.object({
14724
- text: z22.string(),
14725
- href: z22.string()
14914
+ var SerpIntelligenceAICitationSchema = z23.object({
14915
+ text: z23.string(),
14916
+ href: z23.string()
14726
14917
  }).strict();
14727
- var SerpIntelligenceOrganicResultSchema = z22.object({
14728
- position: z22.number().int().min(1),
14729
- title: z22.string(),
14730
- url: z22.string(),
14731
- domain: z22.string(),
14732
- cite: z22.string().nullable(),
14733
- snippet: z22.string().nullable(),
14734
- isRedditStyle: z22.boolean(),
14735
- inlineRating: z22.object({
14736
- value: z22.string(),
14737
- count: z22.string()
14918
+ var SerpIntelligenceOrganicResultSchema = z23.object({
14919
+ position: z23.number().int().min(1),
14920
+ title: z23.string(),
14921
+ url: z23.string(),
14922
+ domain: z23.string(),
14923
+ cite: z23.string().nullable(),
14924
+ snippet: z23.string().nullable(),
14925
+ isRedditStyle: z23.boolean(),
14926
+ inlineRating: z23.object({
14927
+ value: z23.string(),
14928
+ count: z23.string()
14738
14929
  }).strict().nullable()
14739
14930
  }).strict();
14740
- var SerpIntelligenceLocationEvidenceSchema = z22.object({
14741
- status: z22.enum(SerpIntelligenceLocalizationStatusValues),
14742
- expected: z22.object({
14743
- city: z22.string(),
14744
- regionCode: z22.string().nullable(),
14745
- canonicalLocation: z22.string()
14931
+ var SerpIntelligenceLocationEvidenceSchema = z23.object({
14932
+ status: z23.enum(SerpIntelligenceLocalizationStatusValues),
14933
+ expected: z23.object({
14934
+ city: z23.string(),
14935
+ regionCode: z23.string().nullable(),
14936
+ canonicalLocation: z23.string()
14746
14937
  }).strict().nullable(),
14747
- candidates: z22.array(z22.object({
14748
- city: z22.string(),
14749
- regionCode: z22.string(),
14750
- count: z22.number().int().min(0),
14751
- examples: z22.array(z22.string())
14938
+ candidates: z23.array(z23.object({
14939
+ city: z23.string(),
14940
+ regionCode: z23.string(),
14941
+ count: z23.number().int().min(0),
14942
+ examples: z23.array(z23.string())
14752
14943
  }).strict())
14753
14944
  }).strict();
14754
- var SerpIntelligenceHarvestResultSchema = z22.object({
14755
- seed: z22.string(),
14756
- location: z22.string().nullable(),
14757
- extractedAt: z22.string(),
14758
- totalQuestions: z22.number().int().min(0),
14759
- surface: z22.enum(["web", "aim", "unknown"]),
14760
- aiOverview: z22.object({
14761
- detected: z22.boolean(),
14762
- text: z22.string().nullable(),
14763
- citations: z22.array(SerpIntelligenceAICitationSchema),
14764
- expanded: z22.boolean().optional(),
14765
- fullyExpanded: z22.boolean().optional(),
14766
- sections: z22.array(z22.string()).optional()
14945
+ var SerpIntelligenceHarvestResultSchema = z23.object({
14946
+ seed: z23.string(),
14947
+ location: z23.string().nullable(),
14948
+ extractedAt: z23.string(),
14949
+ totalQuestions: z23.number().int().min(0),
14950
+ surface: z23.enum(["web", "aim", "unknown"]),
14951
+ aiOverview: z23.object({
14952
+ detected: z23.boolean(),
14953
+ text: z23.string().nullable(),
14954
+ citations: z23.array(SerpIntelligenceAICitationSchema),
14955
+ expanded: z23.boolean().optional(),
14956
+ fullyExpanded: z23.boolean().optional(),
14957
+ sections: z23.array(z23.string()).optional()
14767
14958
  }).strict(),
14768
- aiMode: z22.object({
14769
- detected: z22.boolean(),
14770
- text: z22.string().nullable(),
14771
- citations: z22.array(SerpIntelligenceAICitationSchema)
14959
+ aiMode: z23.object({
14960
+ detected: z23.boolean(),
14961
+ text: z23.string().nullable(),
14962
+ citations: z23.array(SerpIntelligenceAICitationSchema)
14772
14963
  }).strict(),
14773
- tree: z22.array(z22.unknown()),
14774
- flat: z22.array(z22.unknown()),
14775
- videos: z22.array(z22.unknown()),
14776
- forums: z22.array(z22.unknown()),
14777
- organicResults: z22.array(SerpIntelligenceOrganicResultSchema),
14778
- localPack: z22.array(z22.unknown()),
14779
- entityIds: z22.object({
14780
- entities: z22.array(z22.object({
14781
- name: z22.string(),
14782
- kgId: z22.string().nullable(),
14783
- cid: z22.string().nullable(),
14784
- gcid: z22.string().nullable()
14964
+ tree: z23.array(z23.unknown()),
14965
+ flat: z23.array(z23.unknown()),
14966
+ videos: z23.array(z23.unknown()),
14967
+ forums: z23.array(z23.unknown()),
14968
+ organicResults: z23.array(SerpIntelligenceOrganicResultSchema),
14969
+ localPack: z23.array(z23.unknown()),
14970
+ entityIds: z23.object({
14971
+ entities: z23.array(z23.object({
14972
+ name: z23.string(),
14973
+ kgId: z23.string().nullable(),
14974
+ cid: z23.string().nullable(),
14975
+ gcid: z23.string().nullable()
14785
14976
  }).strict()),
14786
- kgIds: z22.array(z22.string()),
14787
- cids: z22.array(z22.string()),
14788
- gcids: z22.array(z22.string())
14977
+ kgIds: z23.array(z23.string()),
14978
+ cids: z23.array(z23.string()),
14979
+ gcids: z23.array(z23.string())
14789
14980
  }).strict(),
14790
- stats: z22.object({
14791
- seed: z22.string(),
14792
- totalQuestions: z22.number().int().min(0),
14793
- maxDepthReached: z22.number().int().min(0),
14794
- durationMs: z22.number().min(0),
14795
- errorCount: z22.number().int().min(0)
14981
+ stats: z23.object({
14982
+ seed: z23.string(),
14983
+ totalQuestions: z23.number().int().min(0),
14984
+ maxDepthReached: z23.number().int().min(0),
14985
+ durationMs: z23.number().min(0),
14986
+ errorCount: z23.number().int().min(0)
14796
14987
  }).strict(),
14797
- diagnostics: z22.object({
14798
- completionStatus: z22.enum(["paa_found", "no_paa", "serp_only"]),
14799
- problem: z22.null(),
14800
- warnings: z22.array(z22.unknown()).optional(),
14801
- debug: z22.object({
14988
+ diagnostics: z23.object({
14989
+ completionStatus: z23.enum(["paa_found", "no_paa", "serp_only"]),
14990
+ problem: z23.null(),
14991
+ warnings: z23.array(z23.unknown()).optional(),
14992
+ debug: z23.object({
14802
14993
  locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
14803
14994
  }).passthrough().optional()
14804
14995
  }).passthrough(),
14805
- whatPeopleSaying: z22.array(z22.unknown())
14996
+ whatPeopleSaying: z23.array(z23.unknown())
14806
14997
  }).strict();
14807
- var SerpIntelligenceCaptureAttemptSchema = z22.object({
14808
- attemptNumber: z22.number().int().min(1),
14809
- outcome: z22.enum(SerpIntelligenceAttemptOutcomeValues),
14810
- startedAt: z22.string().optional(),
14811
- completedAt: z22.string().optional(),
14812
- durationMs: z22.number().min(0).optional(),
14813
- problemCode: z22.string().optional(),
14814
- message: z22.string().optional(),
14815
- kernelSessionId: z22.string().nullable().optional(),
14816
- cleanupSucceeded: z22.boolean().nullable().optional()
14998
+ var SerpIntelligenceCaptureAttemptSchema = z23.object({
14999
+ attemptNumber: z23.number().int().min(1),
15000
+ outcome: z23.enum(SerpIntelligenceAttemptOutcomeValues),
15001
+ startedAt: z23.string().optional(),
15002
+ completedAt: z23.string().optional(),
15003
+ durationMs: z23.number().min(0).optional(),
15004
+ problemCode: z23.string().optional(),
15005
+ message: z23.string().optional(),
15006
+ kernelSessionId: z23.string().nullable().optional(),
15007
+ cleanupSucceeded: z23.boolean().nullable().optional()
14817
15008
  }).strict();
14818
- var SerpPageSnapshotCaptureSchema = z22.object({
15009
+ var SerpPageSnapshotCaptureSchema = z23.object({
14819
15010
  url: SerpIntelligencePublicHttpUrlSchema,
14820
15011
  requestedUrl: SerpIntelligencePublicHttpUrlSchema,
14821
15012
  finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
14822
- sourceKind: z22.enum(SerpPageSnapshotSourceKindValues),
14823
- sourcePosition: z22.number().int().min(1).nullable(),
14824
- status: z22.enum(SerpPageFetchStatusValues),
14825
- fetchedVia: z22.enum(SerpPageFetchedViaValues).nullable(),
14826
- httpStatus: z22.number().int().min(100).max(599).nullable(),
14827
- contentType: z22.string().nullable(),
14828
- title: z22.string().nullable(),
15013
+ sourceKind: z23.enum(SerpPageSnapshotSourceKindValues),
15014
+ sourcePosition: z23.number().int().min(1).nullable(),
15015
+ status: z23.enum(SerpPageFetchStatusValues),
15016
+ fetchedVia: z23.enum(SerpPageFetchedViaValues).nullable(),
15017
+ httpStatus: z23.number().int().min(100).max(599).nullable(),
15018
+ contentType: z23.string().nullable(),
15019
+ title: z23.string().nullable(),
14829
15020
  canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
14830
- metaDescription: z22.string().nullable(),
14831
- headings: z22.array(z22.object({
14832
- level: z22.number().int().min(1).max(6),
14833
- text: z22.string()
15021
+ metaDescription: z23.string().nullable(),
15022
+ headings: z23.array(z23.object({
15023
+ level: z23.number().int().min(1).max(6),
15024
+ text: z23.string()
14834
15025
  }).strict()).default([]),
14835
- artifact: z22.object({
14836
- htmlBlobUrl: z22.string().url().nullable(),
14837
- textBlobUrl: z22.string().url().nullable(),
14838
- markdownBlobUrl: z22.string().url().nullable(),
14839
- screenshotBlobUrl: z22.string().url().nullable(),
14840
- contentSha256: z22.string().nullable(),
14841
- capturedAt: z22.string().nullable()
15026
+ artifact: z23.object({
15027
+ htmlBlobUrl: z23.string().url().nullable(),
15028
+ textBlobUrl: z23.string().url().nullable(),
15029
+ markdownBlobUrl: z23.string().url().nullable(),
15030
+ screenshotBlobUrl: z23.string().url().nullable(),
15031
+ contentSha256: z23.string().nullable(),
15032
+ capturedAt: z23.string().nullable()
14842
15033
  }).strict(),
14843
- error: z22.object({
14844
- code: z22.string(),
14845
- message: z22.string()
15034
+ error: z23.object({
15035
+ code: z23.string(),
15036
+ message: z23.string()
14846
15037
  }).strict().nullable()
14847
15038
  }).strict();
14848
- var SerpIntelligenceCaptureResponseSchema = z22.object({
15039
+ var SerpIntelligenceCaptureResponseSchema = z23.object({
14849
15040
  harvestResult: SerpIntelligenceHarvestResultSchema,
14850
- attempts: z22.array(SerpIntelligenceCaptureAttemptSchema),
15041
+ attempts: z23.array(SerpIntelligenceCaptureAttemptSchema),
14851
15042
  locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
14852
- pageSnapshotArtifacts: z22.array(SerpPageSnapshotCaptureSchema),
14853
- billing: z22.object({
14854
- creditsUsed: z22.number().min(0).optional(),
14855
- requestId: z22.string().optional(),
14856
- jobId: z22.string().optional()
15043
+ pageSnapshotArtifacts: z23.array(SerpPageSnapshotCaptureSchema),
15044
+ billing: z23.object({
15045
+ creditsUsed: z23.number().min(0).optional(),
15046
+ requestId: z23.string().optional(),
15047
+ jobId: z23.string().optional()
14857
15048
  }).strict().optional()
14858
15049
  }).strict();
14859
- var SerpIntelligencePageSnapshotsResponseSchema = z22.object({
14860
- pageSnapshotArtifacts: z22.array(SerpPageSnapshotCaptureSchema),
14861
- attempts: z22.array(SerpIntelligenceCaptureAttemptSchema).default([])
15050
+ var SerpIntelligencePageSnapshotsResponseSchema = z23.object({
15051
+ pageSnapshotArtifacts: z23.array(SerpPageSnapshotCaptureSchema),
15052
+ attempts: z23.array(SerpIntelligenceCaptureAttemptSchema).default([])
14862
15053
  }).strict();
14863
15054
 
14864
15055
  // src/serp-intelligence/serp-capture-service.ts
@@ -15030,7 +15221,7 @@ var SERP_INTELLIGENCE_RATE_LIMIT = 60;
15030
15221
  var SERP_INTELLIGENCE_RATE_WINDOW_SECONDS = 60;
15031
15222
  var POST_CAPTURE_ROUTE_LABEL = "POST /capture";
15032
15223
  var POST_PAGE_SNAPSHOTS_ROUTE_LABEL = "POST /page-snapshots";
15033
- var serpIntelligenceApp = new Hono14();
15224
+ var serpIntelligenceApp = new Hono15();
15034
15225
  serpIntelligenceApp.use("*", createApiKeyAuth());
15035
15226
  function structuredError(input) {
15036
15227
  return {
@@ -15199,7 +15390,7 @@ serpIntelligenceApp.post("/page-snapshots", async (c) => {
15199
15390
  });
15200
15391
 
15201
15392
  // src/mcp/mcp-routes.ts
15202
- import { Hono as Hono15 } from "hono";
15393
+ import { Hono as Hono16 } from "hono";
15203
15394
  import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
15204
15395
 
15205
15396
  // src/mcp/memory-inprocess-executor.ts
@@ -15726,7 +15917,7 @@ async function requireMcpCallerKey(c) {
15726
15917
  if (!user) return mcpAuthError();
15727
15918
  return { key: callerKey, user };
15728
15919
  }
15729
- var mcpApp = new Hono15();
15920
+ var mcpApp = new Hono16();
15730
15921
  mcpApp.all("/", async (c) => {
15731
15922
  try {
15732
15923
  const keyOrError = await requireMcpCallerKey(c);
@@ -15752,7 +15943,8 @@ mcpApp.all("/", async (c) => {
15752
15943
  });
15753
15944
 
15754
15945
  // src/api/browser-agent-routes.ts
15755
- import { Hono as Hono16 } from "hono";
15946
+ import { randomUUID as randomUUID3 } from "crypto";
15947
+ import { Hono as Hono17 } from "hono";
15756
15948
 
15757
15949
  // src/api/browser-agent-db.ts
15758
15950
  import { randomUUID as randomUUID2 } from "crypto";
@@ -15840,8 +16032,58 @@ async function migrateBrowserAgent() {
15840
16032
  } catch {
15841
16033
  }
15842
16034
  await db.execute(`CREATE INDEX IF NOT EXISTS browser_auth_connections_profile ON browser_auth_connections(profile)`);
16035
+ await db.execute(`
16036
+ CREATE TABLE IF NOT EXISTS browser_agent_extensions (
16037
+ id TEXT PRIMARY KEY,
16038
+ user_id INTEGER NOT NULL,
16039
+ name TEXT NOT NULL,
16040
+ backend_id TEXT NOT NULL,
16041
+ backend_name TEXT NOT NULL,
16042
+ source TEXT NOT NULL DEFAULT 'store',
16043
+ source_url TEXT,
16044
+ size_bytes INTEGER,
16045
+ created_at TEXT NOT NULL DEFAULT (datetime('now'))
16046
+ )
16047
+ `);
16048
+ await db.execute(`CREATE UNIQUE INDEX IF NOT EXISTS browser_agent_extensions_user_name ON browser_agent_extensions(user_id, name)`);
16049
+ await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_extensions_user ON browser_agent_extensions(user_id)`);
15843
16050
  _ready = true;
15844
16051
  }
16052
+ async function createExtensionRow(input) {
16053
+ const db = getDb();
16054
+ const id = `bext_${randomUUID2().replace(/-/g, "").slice(0, 20)}`;
16055
+ await db.execute({
16056
+ sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
16057
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
16058
+ args: [id, input.userId, input.name, input.backendId, input.backendName, input.source, input.sourceUrl, input.sizeBytes]
16059
+ });
16060
+ const row = await getExtensionRow(input.userId, input.name);
16061
+ if (!row) throw new Error("extension insert failed");
16062
+ return row;
16063
+ }
16064
+ async function getExtensionRow(userId, name) {
16065
+ const db = getDb();
16066
+ const res = await db.execute({
16067
+ sql: `SELECT * FROM browser_agent_extensions WHERE user_id = ? AND name = ?`,
16068
+ args: [userId, name]
16069
+ });
16070
+ return res.rows[0] ?? null;
16071
+ }
16072
+ async function listExtensionRows(userId) {
16073
+ const db = getDb();
16074
+ const res = await db.execute({
16075
+ sql: `SELECT * FROM browser_agent_extensions WHERE user_id = ? ORDER BY created_at DESC`,
16076
+ args: [userId]
16077
+ });
16078
+ return res.rows;
16079
+ }
16080
+ async function deleteExtensionRow(userId, name) {
16081
+ const db = getDb();
16082
+ await db.execute({
16083
+ sql: `DELETE FROM browser_agent_extensions WHERE user_id = ? AND name = ?`,
16084
+ args: [userId, name]
16085
+ });
16086
+ }
15845
16087
  async function createAuthConnectionRow(input) {
15846
16088
  const db = getDb();
15847
16089
  const connectionId = `authc_${randomUUID2().replace(/-/g, "").slice(0, 20)}`;
@@ -16698,6 +16940,28 @@ function isProfileConflict(err) {
16698
16940
  const message = err instanceof Error ? err.message : String(err);
16699
16941
  return /\b409\b|conflict|already exists/i.test(message);
16700
16942
  }
16943
+ function isNotFound(err) {
16944
+ const message = err instanceof Error ? err.message : String(err);
16945
+ return /\b404\b|not found/i.test(message);
16946
+ }
16947
+ async function importExtensionFromStore(storeUrl, backendName) {
16948
+ const k = client();
16949
+ const archive = await k.extensions.downloadFromChromeStore({ url: storeUrl });
16950
+ const uploaded = await k.extensions.upload({ file: archive, name: backendName });
16951
+ return {
16952
+ backendId: uploaded.id,
16953
+ backendName: uploaded.name ?? backendName,
16954
+ sizeBytes: typeof uploaded.size_bytes === "number" ? uploaded.size_bytes : null
16955
+ };
16956
+ }
16957
+ async function deleteExtensionBackend(backendName) {
16958
+ const k = client();
16959
+ try {
16960
+ await k.extensions.delete(backendName);
16961
+ } catch (err) {
16962
+ if (!isNotFound(err)) throw err;
16963
+ }
16964
+ }
16701
16965
  async function ensureProfile(k, name) {
16702
16966
  try {
16703
16967
  await k.profiles.create({ name });
@@ -16716,6 +16980,7 @@ async function createSession(opts = {}) {
16716
16980
  const resolvedSaveProfileChanges = opts.saveProfileChanges ?? browserServiceProfileSaveChanges();
16717
16981
  const disableDefaultProxy = opts.disableDefaultProxy ?? !resolvedProxyId;
16718
16982
  const startUrl = opts.startUrl?.trim();
16983
+ const extensionNames = (opts.extensionNames ?? []).map((name) => name.trim()).filter(Boolean);
16719
16984
  if (resolvedProfileName && resolvedSaveProfileChanges === true) {
16720
16985
  await ensureProfile(k, resolvedProfileName);
16721
16986
  }
@@ -16729,7 +16994,8 @@ async function createSession(opts = {}) {
16729
16994
  name: resolvedProfileName,
16730
16995
  ...typeof resolvedSaveProfileChanges === "boolean" ? { save_changes: resolvedSaveProfileChanges } : {}
16731
16996
  }
16732
- } : {}
16997
+ } : {},
16998
+ ...extensionNames.length ? { extensions: extensionNames.map((name) => ({ name })) } : {}
16733
16999
  };
16734
17000
  const browser = await k.browsers.create(createPayload);
16735
17001
  const runtimeSessionId = browser.session_id;
@@ -17117,6 +17383,16 @@ function parseAgentDate(value) {
17117
17383
  const legacyParsed = Date.parse(`${value.replace(" ", "T")}Z`);
17118
17384
  return Number.isFinite(legacyParsed) ? legacyParsed : null;
17119
17385
  }
17386
+ var EXTENSION_NAME_RE = /^[\w .-]{1,64}$/;
17387
+ function publicExtension(row) {
17388
+ return {
17389
+ name: row.name,
17390
+ source: row.source,
17391
+ source_url: row.source_url,
17392
+ size_bytes: row.size_bytes,
17393
+ created_at: row.created_at
17394
+ };
17395
+ }
17120
17396
  function replayClock(row) {
17121
17397
  if (!row) return null;
17122
17398
  const startedAtMs = parseAgentDate(row.started_at);
@@ -17143,7 +17419,7 @@ async function loadOpenSession(id, userId) {
17143
17419
  return row;
17144
17420
  }
17145
17421
  function buildBrowserAgentRoutes() {
17146
- const app2 = new Hono16();
17422
+ const app2 = new Hono17();
17147
17423
  app2.use("*", async (c, next) => {
17148
17424
  await migrateBrowserAgent();
17149
17425
  return next();
@@ -17263,6 +17539,15 @@ function buildBrowserAgentRoutes() {
17263
17539
  const body = await c.req.json().catch(() => ({}));
17264
17540
  const timeoutSeconds = typeof body.timeout_seconds === "number" ? body.timeout_seconds : void 0;
17265
17541
  const startUrl = typeof body.url === "string" ? body.url : void 0;
17542
+ const extensionNames = Array.isArray(body.extension_names) ? body.extension_names.filter((v) => typeof v === "string") : void 0;
17543
+ let resolvedExtensionBackendNames;
17544
+ if (extensionNames && extensionNames.length) {
17545
+ const names = extensionNames;
17546
+ const rows = await Promise.all(names.map((name) => getExtensionRow(user.id, name)));
17547
+ const missing = names.filter((_, i) => !rows[i]);
17548
+ if (missing.length) return c.json({ error: `extension(s) not found: ${missing.join(", ")}` }, 400);
17549
+ resolvedExtensionBackendNames = rows.map((row) => row.backend_name);
17550
+ }
17266
17551
  const gate = await acquireConcurrencyGate(user, "browser_agent_session", {
17267
17552
  ttlSeconds: browserSessionLockTtlSeconds(timeoutSeconds),
17268
17553
  metadata: { label: typeof body.label === "string" ? body.label : null }
@@ -17277,7 +17562,8 @@ function buildBrowserAgentRoutes() {
17277
17562
  saveProfileChanges: typeof body.save_profile_changes === "boolean" ? body.save_profile_changes : void 0,
17278
17563
  startUrl,
17279
17564
  disableDefaultProxy: typeof body.disable_default_proxy === "boolean" ? body.disable_default_proxy : void 0,
17280
- viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0
17565
+ viewport: body.viewport && typeof body.viewport === "object" ? body.viewport : void 0,
17566
+ extensionNames: resolvedExtensionBackendNames
17281
17567
  });
17282
17568
  runtimeSessionId = created.runtimeSessionId;
17283
17569
  const row = await createSessionRow({
@@ -17580,6 +17866,52 @@ function buildBrowserAgentRoutes() {
17580
17866
  }))
17581
17867
  });
17582
17868
  });
17869
+ app2.post("/extensions/import", async (c) => {
17870
+ const user = c.get("user");
17871
+ const body = await c.req.json().catch(() => ({}));
17872
+ const storeUrl = typeof body.store_url === "string" ? body.store_url.trim() : "";
17873
+ const name = typeof body.name === "string" ? body.name.trim() : "";
17874
+ if (!storeUrl) return c.json({ error: "store_url is required" }, 400);
17875
+ if (!name || !EXTENSION_NAME_RE.test(name)) {
17876
+ return c.json({ error: "name is required (1-64 characters: letters, numbers, spaces, dots, underscores, hyphens)" }, 400);
17877
+ }
17878
+ const existing = await getExtensionRow(user.id, name);
17879
+ if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
17880
+ const backendName = `u${user.id}_${randomUUID3().replace(/-/g, "")}`;
17881
+ try {
17882
+ const imported = await importExtensionFromStore(storeUrl, backendName);
17883
+ const row = await createExtensionRow({
17884
+ userId: user.id,
17885
+ name,
17886
+ backendId: imported.backendId,
17887
+ backendName: imported.backendName,
17888
+ source: "store",
17889
+ sourceUrl: storeUrl,
17890
+ sizeBytes: imported.sizeBytes
17891
+ });
17892
+ return c.json(publicExtension(row));
17893
+ } catch (err) {
17894
+ return c.json(failure(err), 502);
17895
+ }
17896
+ });
17897
+ app2.get("/extensions", async (c) => {
17898
+ const user = c.get("user");
17899
+ const rows = await listExtensionRows(user.id);
17900
+ return c.json({ extensions: rows.map(publicExtension) });
17901
+ });
17902
+ app2.delete("/extensions/:name", async (c) => {
17903
+ const user = c.get("user");
17904
+ const name = c.req.param("name");
17905
+ const row = await getExtensionRow(user.id, name);
17906
+ if (!row) return c.json({ error: "not found" }, 404);
17907
+ try {
17908
+ await deleteExtensionBackend(row.backend_name);
17909
+ } catch (err) {
17910
+ return c.json(failure(err), 502);
17911
+ }
17912
+ await deleteExtensionRow(user.id, name);
17913
+ return c.json({ ok: true });
17914
+ });
17583
17915
  app2.get("/sessions/:id/replays/:replayId/download", async (c) => {
17584
17916
  const user = c.get("user");
17585
17917
  const row = await loadOpenSession(c.req.param("id"), user.id);
@@ -17877,9 +18209,9 @@ if (state.key) { refreshSessions(); if (state.current) selectSession(state.curre
17877
18209
 
17878
18210
  // src/api/stripe-routes.ts
17879
18211
  import Stripe from "stripe";
17880
- import { Hono as Hono17 } from "hono";
18212
+ import { Hono as Hono18 } from "hono";
17881
18213
  var stripe = new Stripe(process.env.STRIPE_SECRET_KEY, { apiVersion: "2026-02-25.clover" });
17882
- var stripeApp = new Hono17();
18214
+ var stripeApp = new Hono18();
17883
18215
  stripeApp.post("/webhooks", async (c) => {
17884
18216
  const sig = c.req.header("stripe-signature");
17885
18217
  const body = await c.req.text();
@@ -17935,7 +18267,6 @@ stripeApp.post("/webhooks", async (c) => {
17935
18267
  const live = sub.status === "active" || sub.status === "trialing";
17936
18268
  const newTier = live ? tier.tier : null;
17937
18269
  await setSubscriptionTier(user.id, newTier, live ? tier.concurrency : 0, live ? sub.id : null);
17938
- await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, subscription_tier: newTier }));
17939
18270
  } else if (priceId === CONCURRENCY_PRICE_ID && event.type === "customer.subscription.created") {
17940
18271
  await setExtraConcurrencySlots(user.id, user.extra_concurrency_slots + 1);
17941
18272
  await setConcurrencySubId(user.id, sub.id);
@@ -17961,7 +18292,6 @@ stripeApp.post("/webhooks", async (c) => {
17961
18292
  if (!user) return c.json({ received: true });
17962
18293
  if (priceId && priceId in SUBSCRIPTION_TIERS) {
17963
18294
  await setSubscriptionTier(user.id, null, 0, null);
17964
- await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, subscription_tier: null }));
17965
18295
  } else if (priceId === CONCURRENCY_PRICE_ID) {
17966
18296
  await setExtraConcurrencySlots(user.id, Math.max(0, user.extra_concurrency_slots - 1));
17967
18297
  await setConcurrencySubId(user.id, null);
@@ -17983,9 +18313,9 @@ async function resolveUser(customerId, emailFallback) {
17983
18313
  }
17984
18314
 
17985
18315
  // src/api/oauth-routes.ts
17986
- import { Hono as Hono18 } from "hono";
18316
+ import { Hono as Hono19 } from "hono";
17987
18317
  import { getCookie, setCookie } from "hono/cookie";
17988
- import { createHash as createHash3, randomBytes as randomBytes2, randomUUID as randomUUID3 } from "crypto";
18318
+ import { createHash as createHash3, randomBytes as randomBytes2, randomUUID as randomUUID4 } from "crypto";
17989
18319
  import { importPKCS8, exportJWK, SignJWT } from "jose";
17990
18320
  var ISSUER = "https://mcpscraper.dev";
17991
18321
  var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
@@ -18152,7 +18482,7 @@ function redirectWithError(redirectUri, state, error) {
18152
18482
  if (state) u.searchParams.set("state", state);
18153
18483
  return new Response(null, { status: 302, headers: { Location: u.toString() } });
18154
18484
  }
18155
- var oauthApp = new Hono18();
18485
+ var oauthApp = new Hono19();
18156
18486
  oauthApp.use("*", async (c, next) => {
18157
18487
  c.header("Access-Control-Allow-Origin", "*");
18158
18488
  c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
@@ -18333,7 +18663,7 @@ function pkceMatches(verifier, challenge) {
18333
18663
  }
18334
18664
  async function mintAccessToken(identity, scope, plan, audience) {
18335
18665
  const { privateKey, kid } = await getKeys();
18336
- return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID3()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
18666
+ return new SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti(randomUUID4()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
18337
18667
  }
18338
18668
  function tokenErrorResponse(c, error, description, status) {
18339
18669
  return c.json({ error, error_description: description }, status);
@@ -18406,8 +18736,8 @@ oauthApp.post("/oauth/token", async (c) => {
18406
18736
  });
18407
18737
 
18408
18738
  // src/api/chat-routes.ts
18409
- import { Hono as Hono19 } from "hono";
18410
- var chatApp = new Hono19();
18739
+ import { Hono as Hono20 } from "hono";
18740
+ var chatApp = new Hono20();
18411
18741
  function tokenOk(token) {
18412
18742
  return !!token && token.startsWith("mk_") && token.length > 10;
18413
18743
  }
@@ -19077,8 +19407,8 @@ loadChannels();
19077
19407
  }
19078
19408
 
19079
19409
  // src/api/schedule-routes.ts
19080
- import { Hono as Hono20 } from "hono";
19081
- var scheduleApp = new Hono20();
19410
+ import { Hono as Hono21 } from "hono";
19411
+ var scheduleApp = new Hono21();
19082
19412
  function tokenOk2(token) {
19083
19413
  return !!token && token.startsWith("mk_") && token.length > 10;
19084
19414
  }
@@ -19833,7 +20163,7 @@ var sessionAuth = createMiddleware3(async (c, next) => {
19833
20163
  c.set("sessionUser", { ...refreshed, balance_mc: balanceMc });
19834
20164
  return next();
19835
20165
  });
19836
- var app = new Hono21();
20166
+ var app = new Hono22();
19837
20167
  var STRIPE_API_VERSION = "2026-02-25.clover";
19838
20168
  function requireStripeSecret() {
19839
20169
  const secret2 = process.env.STRIPE_SECRET_KEY?.trim();
@@ -21194,7 +21524,7 @@ app.get("/cron/tick", async (c) => {
21194
21524
  if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
21195
21525
  return c.json({ error: "Unauthorized" }, 401);
21196
21526
  }
21197
- const { drainQueue } = await import("./worker-7XRZNLKB.js");
21527
+ const { drainQueue } = await import("./worker-YHKUJR5P.js");
21198
21528
  const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
21199
21529
  const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
21200
21530
  const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
@@ -21212,7 +21542,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
21212
21542
  return c.json({ error: "Unauthorized" }, 401);
21213
21543
  }
21214
21544
  const jobId = c.req.param("id");
21215
- const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-4NVO5KQA.js");
21545
+ const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-THBVEXMP.js");
21216
21546
  const { assembleExtractArtifacts } = await import("./extract-bundle-UKE273TV.js");
21217
21547
  const job = await getExtractJob2(jobId);
21218
21548
  if (!job) return c.json({ error: "job not found" }, 404);
@@ -21220,7 +21550,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
21220
21550
  await completeExtractJob2(jobId, stored);
21221
21551
  let settlement = "already_settled_or_refunded";
21222
21552
  if (job.billedMc == null && job.userId != null) {
21223
- const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-4NVO5KQA.js");
21553
+ const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-THBVEXMP.js");
21224
21554
  const heldMc = Number(job.options.heldMc ?? 0);
21225
21555
  const successful = await countSuccessfulPages2(jobId);
21226
21556
  const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -21247,6 +21577,7 @@ app.route("/reddit", redditApp);
21247
21577
  app.route("/video", videoApp);
21248
21578
  app.route("/maps", mapsApp);
21249
21579
  app.route("/trustpilot", trustpilotApp);
21580
+ app.route("/g2", g2App);
21250
21581
  app.route("/directory", directoryApp);
21251
21582
  app.route("/workflows", workflowApp);
21252
21583
  app.route("/serp-intelligence", serpIntelligenceApp);
@@ -21383,4 +21714,4 @@ app.get("/blog/:slug/", (c) => {
21383
21714
  export {
21384
21715
  app
21385
21716
  };
21386
- //# sourceMappingURL=server-LBEYRKM2.js.map
21717
+ //# sourceMappingURL=server-XCDFHHLE.js.map