mcp-scraper 0.51.1 → 0.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +4 -4
  2. package/dist/bin/api-server.cjs +2912 -904
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +2 -2
  5. package/dist/bin/mcp-scraper-cli.cjs +5 -1
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +3 -3
  8. package/dist/bin/mcp-scraper-install.cjs +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +1 -1
  11. package/dist/bin/mcp-stdio-server.cjs +384 -43
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +2 -2
  14. package/dist/bin/paa-harvest.cjs +4 -0
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +2 -2
  17. package/dist/{chunk-ZSJHRZY5.js → chunk-J7TH5KU7.js} +2 -2
  18. package/dist/{chunk-2K4RCBYY.js → chunk-OT2AF7SH.js} +385 -44
  19. package/dist/chunk-OT2AF7SH.js.map +1 -0
  20. package/dist/chunk-SG2PEHR3.js +562 -0
  21. package/dist/chunk-SG2PEHR3.js.map +1 -0
  22. package/dist/{chunk-GOZIG6HD.js → chunk-TVZD37SE.js} +2 -2
  23. package/dist/{chunk-3ZUBQQPQ.js → chunk-V7CEOUBF.js} +5 -1
  24. package/dist/chunk-V7CEOUBF.js.map +1 -0
  25. package/dist/chunk-VIAXQTZK.js +7 -0
  26. package/dist/chunk-VIAXQTZK.js.map +1 -0
  27. package/dist/{extract-bundle-RCTNANCH.js → extract-bundle-GUEUCTAE.js} +2 -2
  28. package/dist/index.cjs +4 -0
  29. package/dist/index.cjs.map +1 -1
  30. package/dist/index.js +2 -2
  31. package/dist/{server-JR5XPTXS.js → server-7EXEDKAU.js} +1920 -420
  32. package/dist/server-7EXEDKAU.js.map +1 -0
  33. package/dist/{worker-KQN673JF.js → worker-JL4TG6IF.js} +3 -3
  34. package/package.json +1 -1
  35. package/dist/chunk-27FMOD6S.js +0 -430
  36. package/dist/chunk-27FMOD6S.js.map +0 -1
  37. package/dist/chunk-2K4RCBYY.js.map +0 -1
  38. package/dist/chunk-3ZUBQQPQ.js.map +0 -1
  39. package/dist/chunk-XLUITZUM.js +0 -7
  40. package/dist/chunk-XLUITZUM.js.map +0 -1
  41. package/dist/server-JR5XPTXS.js.map +0 -1
  42. /package/dist/{chunk-ZSJHRZY5.js.map → chunk-J7TH5KU7.js.map} +0 -0
  43. /package/dist/{chunk-GOZIG6HD.js.map → chunk-TVZD37SE.js.map} +0 -0
  44. /package/dist/{extract-bundle-RCTNANCH.js.map → extract-bundle-GUEUCTAE.js.map} +0 -0
  45. /package/dist/{worker-KQN673JF.js.map → worker-JL4TG6IF.js.map} +0 -0
@@ -1,10 +1,12 @@
1
1
  import {
2
2
  SITE_EXTRACT_ARTIFACT_PREFIX,
3
3
  cleanupExpiredSiteExtractArtifacts,
4
+ discoverStaticMedia,
4
5
  harvestPageMedia,
5
6
  readOwnedSiteExtractArtifactBuffer,
6
- renewSiteExtractArtifactDownload
7
- } from "./chunk-27FMOD6S.js";
7
+ renewSiteExtractArtifactDownload,
8
+ typeFromMime
9
+ } from "./chunk-SG2PEHR3.js";
8
10
  import {
9
11
  REQUIRED_LOCATION_STATE_CODES,
10
12
  importHostedCensusPlacesCsv,
@@ -100,7 +102,7 @@ import {
100
102
  resolveKernelProxyId,
101
103
  serpArrivalUrl,
102
104
  withRuntimeUserAgent
103
- } from "./chunk-GOZIG6HD.js";
105
+ } from "./chunk-TVZD37SE.js";
104
106
  import {
105
107
  ApiErrorResponseSchema,
106
108
  ArtifactTemplatePresetKeySchema,
@@ -158,7 +160,7 @@ import {
158
160
  resolveDeploymentProfile,
159
161
  resolveLocalSourcebookSchemaType,
160
162
  transcribeMediaUrl
161
- } from "./chunk-2K4RCBYY.js";
163
+ } from "./chunk-OT2AF7SH.js";
162
164
  import {
163
165
  auditImageUrls,
164
166
  auditImages,
@@ -196,7 +198,7 @@ import {
196
198
  workflowOutputBaseDir,
197
199
  workflowStepCount,
198
200
  workflowSupportsSteps
199
- } from "./chunk-ZSJHRZY5.js";
201
+ } from "./chunk-J7TH5KU7.js";
200
202
  import {
201
203
  csvRecords,
202
204
  parseCsv,
@@ -211,10 +213,10 @@ import {
211
213
  RawMapsHoursRowSchema,
212
214
  RawMapsOverviewSchema,
213
215
  RawMapsReviewStatsSchema
214
- } from "./chunk-3ZUBQQPQ.js";
216
+ } from "./chunk-V7CEOUBF.js";
215
217
  import {
216
218
  PACKAGE_VERSION
217
- } from "./chunk-XLUITZUM.js";
219
+ } from "./chunk-VIAXQTZK.js";
218
220
  import {
219
221
  abandonExtractSettlement,
220
222
  countSuccessfulPages,
@@ -3813,40 +3815,136 @@ async function readResponseTextWithLimit(response, maxBytes, label) {
3813
3815
  }
3814
3816
 
3815
3817
  // src/api/kernel-fetch.ts
3816
- async function fetchWithKernel(url) {
3817
- const checked = await validatePublicHttpUrl(url, { field: "browser URL" });
3818
- if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Browser URL was rejected");
3819
- const auditedUrl = checked.parsed.href;
3820
- const apiKey = browserServiceApiKey();
3821
- if (!apiKey) throw new Error("Browser backend API key not set");
3822
- const client2 = new Kernel({ apiKey });
3818
+ var CHALLENGE_PATTERNS = [
3819
+ { reason: "cloudflare_title", pattern: /<title[^>]*>\s*just a moment(?:\.\.\.)?\s*<\/title>/i },
3820
+ { reason: "cloudflare_challenge_platform", pattern: /(?:cdn-cgi\/challenge-platform|challenge-platform\/h\/|cf-chl-|cf_chl_)/i },
3821
+ { reason: "cloudflare_browser_verification", pattern: /(?:cf-browser-verification|checking your browser before accessing)/i },
3822
+ { reason: "cloudflare_cookie_message", pattern: /enable javascript and cookies to continue/i },
3823
+ { reason: "generic_security_check", pattern: /performing security verification|verify you are human/i }
3824
+ ];
3825
+ function detectBrowserChallenge(input) {
3826
+ if (input.status === 403 || input.status === 429) {
3827
+ return { detected: true, reason: `http_${input.status}` };
3828
+ }
3829
+ const sample = `${input.title ?? ""}
3830
+ ${input.html ?? ""}`.slice(0, 25e4);
3831
+ for (const candidate of CHALLENGE_PATTERNS) {
3832
+ if (candidate.pattern.test(sample)) return { detected: true, reason: candidate.reason };
3833
+ }
3834
+ return { detected: false, reason: null };
3835
+ }
3836
+ async function runKernelAttempt(client2, auditedUrl, mode) {
3823
3837
  const openedAtMs = Date.now();
3824
- const useHeadless = currentCostContext()?.forceHeadful !== true;
3825
- const kb = await client2.browsers.create({ stealth: true, ...useHeadless ? { headless: true } : {}, timeout_seconds: 60 });
3838
+ const headlessSent = mode === "stealth_headless" ? true : null;
3839
+ const kb = await client2.browsers.create({
3840
+ stealth: true,
3841
+ ...headlessSent ? { headless: true } : {},
3842
+ timeout_seconds: 60
3843
+ });
3826
3844
  const browser = await chromium.connectOverCDP(kb.cdp_ws_url);
3845
+ let attemptError = null;
3827
3846
  try {
3828
3847
  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
3829
- const context = await browser.newContext(withRuntimeUserAgent({
3830
- serviceWorkers: "block"
3831
- }, runtimeUserAgent));
3848
+ const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
3832
3849
  const page = context.pages()[0] ?? await context.newPage();
3833
3850
  const guard = await installBrowserRequestGuard(page, auditedUrl);
3834
- await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
3851
+ const response = await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
3835
3852
  if (guard.blockedReason && page.url() === "about:blank") throw new Error(guard.blockedReason);
3836
3853
  await page.waitForTimeout(2500);
3837
- return await page.evaluate((maxBytes) => {
3838
- const value = document.documentElement?.outerHTML ?? "";
3839
- if (new TextEncoder().encode(value).byteLength > maxBytes) throw new Error(`Browser HTML exceeds ${maxBytes} byte limit`);
3840
- return value;
3841
- }, MAX_PAGE_HTML_BYTES);
3854
+ let status = response?.status() ?? null;
3855
+ let title = await page.title().catch(() => "");
3856
+ let html = await page.content();
3857
+ let challenge = detectBrowserChallenge({ status, title, html });
3858
+ if (challenge.detected && mode === "stealth_headful") {
3859
+ const deadline = Date.now() + 2e4;
3860
+ while (challenge.detected && Date.now() < deadline) {
3861
+ await page.waitForTimeout(1e3);
3862
+ title = await page.title().catch(() => "");
3863
+ html = await page.content();
3864
+ challenge = detectBrowserChallenge({ status, title, html });
3865
+ }
3866
+ status = response?.status() ?? status;
3867
+ }
3868
+ const byteLength2 = new TextEncoder().encode(html).byteLength;
3869
+ if (byteLength2 > MAX_PAGE_HTML_BYTES) throw new Error(`Browser HTML exceeds ${MAX_PAGE_HTML_BYTES} byte limit`);
3870
+ return { html, status, challenge, durationMs: Date.now() - openedAtMs };
3871
+ } catch (error) {
3872
+ attemptError = error instanceof Error ? error.message : String(error);
3873
+ throw error;
3842
3874
  } finally {
3843
3875
  await browser.close().catch(() => {
3844
3876
  });
3845
3877
  await client2.browsers.deleteByID(kb.session_id).catch(() => {
3846
3878
  });
3847
- void recordKernelSession({ kernelSessionId: kb.session_id, source: "kernel_fetch", stealth: typeof kb.stealth === "boolean" ? kb.stealth : null, headlessSent: useHeadless, proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null, fallback: true, openedAtMs, closedAtMs: Date.now() });
3879
+ void recordKernelSession({
3880
+ kernelSessionId: kb.session_id,
3881
+ source: "kernel_fetch",
3882
+ stealth: typeof kb.stealth === "boolean" ? kb.stealth : null,
3883
+ headlessSent,
3884
+ proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null,
3885
+ fallback: true,
3886
+ openedAtMs,
3887
+ closedAtMs: Date.now(),
3888
+ error: attemptError
3889
+ });
3848
3890
  }
3849
3891
  }
3892
+ async function fetchWithKernelDetailed(url) {
3893
+ const checked = await validatePublicHttpUrl(url, { field: "browser URL" });
3894
+ if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Browser URL was rejected");
3895
+ const auditedUrl = checked.parsed.href;
3896
+ const apiKey = browserServiceApiKey();
3897
+ if (!apiKey) throw new Error("Browser backend API key not set");
3898
+ const client2 = new Kernel({ apiKey });
3899
+ const startsHeadful = currentCostContext()?.forceHeadful === true;
3900
+ const modes = startsHeadful ? ["stealth_headful"] : ["stealth_headless", "stealth_headful"];
3901
+ const attempts = [];
3902
+ for (const mode of modes) {
3903
+ const result = await runKernelAttempt(client2, auditedUrl, mode);
3904
+ if (result.challenge.detected) {
3905
+ const completion = mode === "stealth_headful" ? "challenge_unresolved" : "challenge_detected";
3906
+ attempts.push({
3907
+ mode,
3908
+ status: result.status,
3909
+ challengeDetected: true,
3910
+ challengeReason: result.challenge.reason,
3911
+ completion,
3912
+ durationMs: result.durationMs
3913
+ });
3914
+ if (mode === "stealth_headful") {
3915
+ const error = new Error(`browser_challenge_unresolved: ${result.challenge.reason ?? "challenge page remained after bounded wait"}`);
3916
+ error.code = "browser_challenge_unresolved";
3917
+ error.attempts = attempts;
3918
+ throw error;
3919
+ }
3920
+ continue;
3921
+ }
3922
+ if (result.status != null && (result.status < 200 || result.status >= 400)) {
3923
+ attempts.push({
3924
+ mode,
3925
+ status: result.status,
3926
+ challengeDetected: false,
3927
+ challengeReason: null,
3928
+ completion: "http_error",
3929
+ durationMs: result.durationMs
3930
+ });
3931
+ throw new Error(`browser_http_${result.status}: Browser fetch returned HTTP ${result.status}`);
3932
+ }
3933
+ attempts.push({
3934
+ mode,
3935
+ status: result.status,
3936
+ challengeDetected: false,
3937
+ challengeReason: null,
3938
+ completion: "usable_page",
3939
+ durationMs: result.durationMs
3940
+ });
3941
+ return { html: result.html, mode, attempts };
3942
+ }
3943
+ throw new Error("browser_challenge_unresolved: no usable browser response");
3944
+ }
3945
+ async function fetchWithKernel(url) {
3946
+ return (await fetchWithKernelDetailed(url)).html;
3947
+ }
3850
3948
 
3851
3949
  // src/lib/http-user-agent.ts
3852
3950
  var MCP_SCRAPER_CRAWLER_USER_AGENT = "MCPScraperBot/1.0 (+https://mcpscraper.dev)";
@@ -3875,6 +3973,8 @@ async function extractKpo(opts) {
3875
3973
  }
3876
3974
  if (!res.ok) return null;
3877
3975
  const text2 = await res.text();
3976
+ const title2 = text2.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? null;
3977
+ if (detectBrowserChallenge({ status: res.status, title: title2, html: text2 }).detected) return null;
3878
3978
  if (text2.length < 2e3) return null;
3879
3979
  const stripped = text2.replace(/<(script|style|noscript|svg|iframe|canvas)[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<(nav|header|footer|aside)[^>]*>[\s\S]*?<\/\1>/gi, "");
3880
3980
  const contentBlock = stripped.match(/<main[^>]*>([\s\S]*?)<\/main>/i)?.[1] ?? stripped.match(/<article[^>]*>([\s\S]*?)<\/article>/i)?.[1] ?? stripped.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? "";
@@ -3889,13 +3989,16 @@ async function extractKpo(opts) {
3889
3989
  };
3890
3990
  let html;
3891
3991
  let fetchedVia = "fetch";
3992
+ let browserFetch;
3892
3993
  const plainFetchStart = Date.now();
3893
3994
  const plainHtml = await tryPlainFetch();
3894
3995
  const plainFetchMs = Date.now() - plainFetchStart;
3895
3996
  if (plainHtml) {
3896
3997
  html = plainHtml;
3897
3998
  } else if (opts.kernelApiKey) {
3898
- html = await fetchWithKernel(parsed.href);
3999
+ const browserResult = await fetchWithKernelDetailed(parsed.href);
4000
+ html = browserResult.html;
4001
+ browserFetch = { mode: browserResult.mode, attempts: browserResult.attempts };
3899
4002
  fetchedVia = "headless";
3900
4003
  } else {
3901
4004
  throw new Error("Fetch failed: site blocked plain HTTP and no headless browser backend is configured");
@@ -4195,6 +4298,7 @@ async function extractKpo(opts) {
4195
4298
  return {
4196
4299
  url: parsed.href,
4197
4300
  fetchedVia,
4301
+ browserFetch,
4198
4302
  durationMs,
4199
4303
  plainFetchMs,
4200
4304
  title,
@@ -4212,13 +4316,47 @@ import { chromium as chromium2 } from "playwright";
4212
4316
  import Kernel2 from "@onkernel/sdk";
4213
4317
 
4214
4318
  // src/lib/branding-extractor.ts
4215
- function rgbToHex(rgb) {
4216
- const m = rgb.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)/);
4217
- if (!m) return null;
4218
- const r = parseInt(m[1], 10);
4219
- const g = parseInt(m[2], 10);
4220
- const b = parseInt(m[3], 10);
4221
- return "#" + [r, g, b].map((v) => v.toString(16).padStart(2, "0")).join("");
4319
+ function selectRankedLogoCandidates(candidates) {
4320
+ const ranked = candidates.filter((candidate) => {
4321
+ if (!/^https?:\/\//i.test(candidate.url) || !Number.isFinite(candidate.score) || candidate.region === "body") return false;
4322
+ if (candidate.region === "json_ld" || candidate.region === "favicon") return true;
4323
+ return candidate.evidence.includes("links_to_homepage") || candidate.evidence.includes("site_identity_marker");
4324
+ }).map((candidate) => ({
4325
+ ...candidate,
4326
+ confidence: candidate.score >= 85 ? "high" : candidate.score >= 60 ? "medium" : "low",
4327
+ evidence: [...new Set(candidate.evidence)]
4328
+ })).sort((a, b) => b.score - a.score || a.url.localeCompare(b.url)).slice(0, 20);
4329
+ const primary = ranked.find((candidate) => candidate.score >= 55 && candidate.region !== "favicon") ?? null;
4330
+ return {
4331
+ primary,
4332
+ variants: primary ? ranked.filter((candidate) => candidate.url !== primary.url && candidate.score >= 55 && candidate.region !== "favicon" && logoFamilyKey(candidate.url) === logoFamilyKey(primary.url)).slice(0, 5).map((candidate) => candidate.url) : [],
4333
+ candidates: ranked,
4334
+ selectionReason: primary?.evidence.join(", ") ?? null
4335
+ };
4336
+ }
4337
+ function logoFamilyKey(value) {
4338
+ try {
4339
+ const url = new URL(value);
4340
+ const host2 = url.hostname.replace(/^www\./, "").toLowerCase();
4341
+ const path5 = decodeURIComponent(url.pathname).toLowerCase().replace(/-\d+x\d+(?=\.[a-z0-9]+$)/, "");
4342
+ return `${host2}${path5}`;
4343
+ } catch {
4344
+ return value.toLowerCase();
4345
+ }
4346
+ }
4347
+ function selectProofImages(candidates) {
4348
+ const byUrl = /* @__PURE__ */ new Map();
4349
+ for (const candidate of candidates) {
4350
+ if (!/^https?:\/\//i.test(candidate.url) || !Number.isFinite(candidate.score) || candidate.score < 45) continue;
4351
+ const normalized = {
4352
+ ...candidate,
4353
+ confidence: candidate.score >= 85 ? "high" : candidate.score >= 60 ? "medium" : "low",
4354
+ evidence: [...new Set(candidate.evidence)]
4355
+ };
4356
+ const existing = byUrl.get(normalized.url);
4357
+ if (!existing || normalized.score > existing.score) byUrl.set(normalized.url, normalized);
4358
+ }
4359
+ return [...byUrl.values()].sort((a, b) => b.score - a.score || a.url.localeCompare(b.url)).slice(0, 20);
4222
4360
  }
4223
4361
  function luminance(hex) {
4224
4362
  const m = hex.match(/^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})$/i);
@@ -4232,6 +4370,12 @@ function isTransparentOrWhite(hex) {
4232
4370
  const lum = luminance(hex);
4233
4371
  return lum > 240;
4234
4372
  }
4373
+ function isChromatic(hex) {
4374
+ const match = hex?.match(/^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})$/i);
4375
+ if (!match) return false;
4376
+ const channels = [parseInt(match[1], 16), parseInt(match[2], 16), parseInt(match[3], 16)];
4377
+ return Math.max(...channels) - Math.min(...channels) >= 24 && luminance(hex) > 18;
4378
+ }
4235
4379
  function firstFont(fontFamily) {
4236
4380
  if (!fontFamily) return null;
4237
4381
  const first = fontFamily.split(",")[0].trim().replace(/['"]/g, "");
@@ -4240,14 +4384,34 @@ function firstFont(fontFamily) {
4240
4384
  function dominantColor(freq) {
4241
4385
  return Object.entries(freq).filter(([hex]) => !isTransparentOrWhite(hex) && hex !== "#000000" && hex !== "#020101").sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
4242
4386
  }
4387
+ function dominantChromaticColor(freq) {
4388
+ return Object.entries(freq).filter(([hex]) => isChromatic(hex)).sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
4389
+ }
4243
4390
  async function extractBrandingFromPage(page) {
4391
+ const challenge = detectBrowserChallenge({
4392
+ title: await page.title().catch(() => ""),
4393
+ html: await page.content().catch(() => "")
4394
+ });
4395
+ if (challenge.detected) {
4396
+ const error = new Error(`browser_challenge_unresolved: ${challenge.reason ?? "challenge page is not branding evidence"}`);
4397
+ error.code = "browser_challenge_unresolved";
4398
+ throw error;
4399
+ }
4244
4400
  const evalScript = `
4245
4401
  (function() {
4246
4402
  function cs(el) { return el ? window.getComputedStyle(el) : null; }
4247
- function toHex(rgb) {
4248
- var m = rgb && rgb.match(/rgba?\\((\\d+),\\s*(\\d+),\\s*(\\d+)/);
4249
- if (!m) return null;
4250
- return '#' + [m[1],m[2],m[3]].map(function(v){ return ('0'+parseInt(v).toString(16)).slice(-2); }).join('');
4403
+ var colorCanvas = document.createElement('canvas');
4404
+ colorCanvas.width = 1; colorCanvas.height = 1;
4405
+ var colorContext = colorCanvas.getContext('2d', { willReadFrequently: true });
4406
+ function toHex(value) {
4407
+ if (!value || !colorContext) return null;
4408
+ colorContext.clearRect(0, 0, 1, 1);
4409
+ colorContext.fillStyle = 'rgba(1, 2, 3, 0.001)';
4410
+ colorContext.fillStyle = value;
4411
+ colorContext.fillRect(0, 0, 1, 1);
4412
+ var data = colorContext.getImageData(0, 0, 1, 1).data;
4413
+ if (data[3] < 64) return null;
4414
+ return '#' + [data[0], data[1], data[2]].map(function(v){ return ('0'+v.toString(16)).slice(-2); }).join('');
4251
4415
  }
4252
4416
  function isUsable(hex) {
4253
4417
  if (!hex) return false;
@@ -4256,30 +4420,63 @@ async function extractBrandingFromPage(page) {
4256
4420
  return (0.2126*r + 0.7152*g + 0.0722*b) <= 230;
4257
4421
  }
4258
4422
 
4423
+ function isChromatic(hex) {
4424
+ if (!hex) return false;
4425
+ var r=parseInt(hex.slice(1,3),16), g=parseInt(hex.slice(3,5),16), b=parseInt(hex.slice(5,7),16);
4426
+ return Math.max(r,g,b) - Math.min(r,g,b) >= 24 && (0.2126*r + 0.7152*g + 0.0722*b) > 18;
4427
+ }
4428
+
4429
+ function visibleRect(el) {
4430
+ if (!el) return null;
4431
+ var style = cs(el);
4432
+ var rect = el.getBoundingClientRect();
4433
+ if (!style || style.display === 'none' || style.visibility === 'hidden' || parseFloat(style.opacity || '1') <= 0 || rect.width < 2 || rect.height < 2) return null;
4434
+ return rect;
4435
+ }
4436
+
4259
4437
  var navEl = document.querySelector('nav, header, [role="banner"]');
4260
4438
  var bodyEl = document.body;
4261
4439
  var h1El = document.querySelector('h1');
4262
- var btnEl = document.querySelector(
4263
- 'a.btn-primary, button.btn-primary, .btn-primary, .cta-btn,' +
4264
- 'a.button--primary, button.button--primary, [class*="btn-cta"],' +
4265
- '[class*="cta-button"], .wp-block-button__link, [class*="hero"] a,' +
4266
- '.elementor-button, .elementor-button-link,' +
4267
- '.et_pb_button,' +
4268
- '.fl-button,' +
4269
- '.vc_btn,' +
4270
- '[class*="cta"][href], [class*="get-started"], [class*="contact-btn"]'
4271
- );
4440
+ var btnEl = null;
4441
+ var btnScore = -Infinity;
4442
+ var interactive = document.querySelectorAll('a[href], button, [role="button"]');
4443
+ for (var bi = 0; bi < interactive.length; bi++) {
4444
+ var candidateButton = interactive[bi];
4445
+ var buttonRect = visibleRect(candidateButton);
4446
+ if (!buttonRect || buttonRect.width < 44 || buttonRect.height < 20) continue;
4447
+ if (buttonRect.bottom < 0 || buttonRect.top > window.innerHeight) continue;
4448
+ var buttonStyle = cs(candidateButton);
4449
+ var buttonBg = buttonStyle ? toHex(buttonStyle.backgroundColor) : null;
4450
+ if (!buttonBg) continue;
4451
+ var buttonDescriptor = [candidateButton.textContent, candidateButton.id, candidateButton.className, candidateButton.getAttribute('aria-label')].join(' ');
4452
+ if (/\\b(?:skip|screen-reader|sr-only|visually-hidden)\\b/i.test(buttonDescriptor)) continue;
4453
+ var buttonText = String(candidateButton.textContent || candidateButton.getAttribute('aria-label') || '').replace(/\\s+/g, ' ').trim();
4454
+ var explicitButtonEvidence = /\\b(?:btn[-_ ]?primary|button[-_ ]?primary|cta|call[-_ ]?to[-_ ]?action)\\b/i.test(buttonDescriptor);
4455
+ var actionTextEvidence = /\\b(?:get started|start|try|book|contact|schedule|request|sign up|join|buy|shop|call|quote|estimate|verify)\\b/i.test(buttonDescriptor);
4456
+ if (!explicitButtonEvidence && !actionTextEvidence && (buttonRect.height > 100 || buttonText.length > 80)) continue;
4457
+ var score = 0;
4458
+ if (isChromatic(buttonBg)) score += 35;
4459
+ if (explicitButtonEvidence) score += 30;
4460
+ if (actionTextEvidence) score += 20;
4461
+ if (candidateButton.closest('main, [role="main"], [class*="hero"]')) score += 12;
4462
+ if (candidateButton.closest('header, nav, [role="banner"]')) score += 4;
4463
+ score += Math.min(10, Math.round((buttonRect.width * buttonRect.height) / 5000));
4464
+ if (score > btnScore) { btnEl = candidateButton; btnScore = score; }
4465
+ }
4272
4466
  var navStyle = cs(navEl);
4273
4467
  var bodyStyle = cs(bodyEl);
4274
4468
  var h1Style = cs(h1El);
4275
4469
  var btnStyle = cs(btnEl);
4276
4470
 
4277
4471
  var svgFreq = {};
4278
- var svgScope = navEl || document.querySelector('header, [role="banner"]');
4472
+ var svgScope = document;
4279
4473
  if (svgScope) {
4280
- var svgEls = svgScope.querySelectorAll('svg *, [fill], path, circle, rect, polygon, polyline');
4474
+ var svgEls = svgScope.querySelectorAll('svg *, svg[fill], svg path, svg circle, svg rect, svg polygon, svg polyline');
4281
4475
  for (var si = 0; si < svgEls.length; si++) {
4282
4476
  var svgEl = svgEls[si];
4477
+ var owningSvg = svgEl.closest('svg');
4478
+ var owningRect = owningSvg ? owningSvg.getBoundingClientRect() : null;
4479
+ if (!owningRect || owningRect.bottom < 0 || owningRect.top > 300 || owningRect.width < 12 || owningRect.height < 12) continue;
4283
4480
  var fillComp = cs(svgEl) ? cs(svgEl).fill : null;
4284
4481
  var fillAttr = svgEl.getAttribute('fill');
4285
4482
  var fillHex = null;
@@ -4301,55 +4498,251 @@ async function extractBrandingFromPage(page) {
4301
4498
  }
4302
4499
  }
4303
4500
 
4501
+ var cssPrimaryCandidates = [];
4502
+ var cssAccentCandidates = [];
4503
+ var rootStyle = cs(document.documentElement);
4504
+ if (rootStyle) {
4505
+ for (var ci = 0; ci < rootStyle.length; ci++) {
4506
+ var propertyName = rootStyle[ci];
4507
+ if (!propertyName || propertyName.indexOf('--') !== 0) continue;
4508
+ var normalizedName = propertyName.toLowerCase();
4509
+ if (!/(?:^|[-_])(?:brand|primary|accent)(?:$|[-_])/.test(normalizedName)) continue;
4510
+ if (/(?:on[-_]?primary|primary[-_]?(?:text|foreground)|accent[-_]?(?:text|foreground))/.test(normalizedName)) continue;
4511
+ var propertyValue = rootStyle.getPropertyValue(propertyName).trim();
4512
+ var propertyHex = toHex(propertyValue);
4513
+ if (!propertyHex || !isChromatic(propertyHex)) continue;
4514
+ var item = { hex: propertyHex, name: propertyName };
4515
+ if (/(?:^|[-_])primary(?:$|[-_])|brand/.test(normalizedName)) cssPrimaryCandidates.push(item);
4516
+ if (/(?:^|[-_])accent(?:$|[-_])/.test(normalizedName)) cssAccentCandidates.push(item);
4517
+ }
4518
+ }
4519
+
4520
+ var themeColorEl = document.querySelector('meta[name="theme-color"]');
4521
+ var themeColor = themeColorEl ? toHex(themeColorEl.getAttribute('content')) : null;
4522
+
4523
+ var visibleBgFreq = {};
4524
+ var topBgFreq = {};
4525
+ var visibleElements = document.querySelectorAll('body *');
4526
+ for (var vi = 0; vi < visibleElements.length && vi < 1500; vi++) {
4527
+ var visibleElement = visibleElements[vi];
4528
+ var visibleElementRect = visibleRect(visibleElement);
4529
+ if (!visibleElementRect || visibleElementRect.width < 20 || visibleElementRect.height < 12) continue;
4530
+ var visibleStyle = cs(visibleElement);
4531
+ var visibleHex = visibleStyle ? toHex(visibleStyle.backgroundColor) : null;
4532
+ if (!visibleHex || !isChromatic(visibleHex)) continue;
4533
+ var areaWeight = Math.max(1, Math.min(20, Math.round(Math.sqrt(visibleElementRect.width * visibleElementRect.height) / 40)));
4534
+ visibleBgFreq[visibleHex] = (visibleBgFreq[visibleHex] || 0) + areaWeight;
4535
+ if (visibleElementRect.top < 300 && visibleElementRect.bottom > 0 && visibleElementRect.width >= window.innerWidth * 0.5) {
4536
+ topBgFreq[visibleHex] = (topBgFreq[visibleHex] || 0) + areaWeight;
4537
+ }
4538
+ }
4539
+
4304
4540
  var pageHost = window.location.hostname.replace(/^www\\./, '');
4305
4541
  function isSameDomain(src) {
4306
- try { return new URL(src).hostname.replace(/^www\\./, '').endsWith(pageHost); } catch { return false; }
4542
+ try {
4543
+ var host = new URL(src).hostname.replace(/^www\\./, '');
4544
+ return host === pageHost || host.endsWith('.' + pageHost);
4545
+ } catch { return false; }
4307
4546
  }
4308
- var logoSelectors = [
4309
- 'header img[class*="logo"]', 'nav img[class*="logo"]',
4310
- 'img[class*="site-logo"]', 'img[class*="brand-logo"]',
4311
- 'img[alt*="logo" i]', '.logo img', '#logo img',
4312
- 'header img:first-of-type', 'nav img:first-of-type'
4313
- ];
4314
- var logoSrc = null;
4315
- for (var i = 0; i < logoSelectors.length; i++) {
4316
- var el = document.querySelector(logoSelectors[i]);
4317
- if (el && el.src && isSameDomain(el.src)) { logoSrc = el.src; break; }
4318
- }
4319
- if (!logoSrc) {
4320
- for (var j = 0; j < logoSelectors.length; j++) {
4321
- var el2 = document.querySelector(logoSelectors[j]);
4322
- if (el2 && el2.src) { logoSrc = el2.src; break; }
4547
+ var logoCandidates = {};
4548
+ function addLogo(raw, region, evidence, score, alt, width, height) {
4549
+ if (!raw || raw.startsWith('data:') || raw.startsWith('blob:')) return;
4550
+ var url;
4551
+ try { url = new URL(raw, document.baseURI); } catch (_) { return; }
4552
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') return;
4553
+ url.hash = '';
4554
+ var key = url.href;
4555
+ var existing = logoCandidates[key] || {
4556
+ url: key, score: 0, region: region, evidence: [], alt: null, width: null, height: null
4557
+ };
4558
+ existing.score = Math.max(existing.score, score);
4559
+ existing.evidence = Array.from(new Set(existing.evidence.concat(evidence)));
4560
+ if (region === 'json_ld' || (existing.region === 'body' && region !== 'body')) existing.region = region;
4561
+ if (alt && !existing.alt) existing.alt = alt.replace(/\\s+/g, ' ').trim().slice(0, 300) || null;
4562
+ if ((width || 0) * (height || 0) > (existing.width || 0) * (existing.height || 0)) {
4563
+ existing.width = width || null; existing.height = height || null;
4564
+ }
4565
+ logoCandidates[key] = existing;
4566
+ }
4567
+
4568
+ function jsonLogo(value) {
4569
+ if (typeof value === 'string') return value;
4570
+ if (!value || typeof value !== 'object') return null;
4571
+ return typeof value.url === 'string' ? value.url : typeof value.contentUrl === 'string' ? value.contentUrl : null;
4572
+ }
4573
+ var jsonScripts = document.querySelectorAll('script[type="application/ld+json"]');
4574
+ for (var ji = 0; ji < jsonScripts.length; ji++) {
4575
+ try {
4576
+ var parsed = JSON.parse(jsonScripts[ji].textContent || 'null');
4577
+ var stack = Array.isArray(parsed) ? parsed.slice() : [parsed];
4578
+ var visited = 0;
4579
+ while (stack.length && visited++ < 1000) {
4580
+ var node = stack.pop();
4581
+ if (!node || typeof node !== 'object') continue;
4582
+ var types = Array.isArray(node['@type']) ? node['@type'] : [node['@type']];
4583
+ var entity = types.some(function(t) { return /^(Organization|Corporation|LocalBusiness|WebSite|Brand)$/i.test(String(t || '')); });
4584
+ var logo = entity ? jsonLogo(node.logo) : null;
4585
+ if (logo) addLogo(logo, 'json_ld', ['json_ld_logo'], 100, null, null, null);
4586
+ Object.keys(node).forEach(function(key) {
4587
+ if (key === '@context') return;
4588
+ var child = node[key];
4589
+ if (Array.isArray(child)) stack = stack.concat(child.filter(function(value) { return value && typeof value === 'object'; }));
4590
+ else if (child && typeof child === 'object') stack.push(child);
4591
+ });
4592
+ }
4593
+ } catch (_) {}
4594
+ }
4595
+
4596
+ var proofCandidates = {};
4597
+ var pageHeadings = Array.from(document.querySelectorAll('h1,h2,h3,h4,h5,h6'));
4598
+ function nearestHeadingText(element) {
4599
+ var nearest = null;
4600
+ for (var hi = 0; hi < pageHeadings.length; hi++) {
4601
+ if (pageHeadings[hi].compareDocumentPosition(element) & 4) nearest = pageHeadings[hi];
4602
+ else break;
4603
+ }
4604
+ return nearest && nearest.textContent ? nearest.textContent.replace(/\\s+/g, ' ').trim().slice(0, 300) : null;
4605
+ }
4606
+ function proofTypeFor(descriptor, context) {
4607
+ if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation))\\b/i.test(descriptor)) return 'accreditation';
4608
+ if (/\\b(?:certifi(?:ed|cation)|compliance|legitscript|hipaa)\\b/i.test(descriptor)) return 'certification';
4609
+ if (/\\b(?:award|winner|best of|top[-_ ]?rated)\\b/i.test(descriptor)) return 'award';
4610
+ if (/\\b(?:member|membership|association|professionals?)\\b/i.test(descriptor)) return 'membership';
4611
+ if (/\\b(?:press|featured|as seen|mentioned)\\b/i.test(descriptor)) return 'press_mention';
4612
+ if (/\\b(?:partner|customer|client|trusted by|used by|works with)\\b/i.test(descriptor)) return 'partner_or_customer';
4613
+ if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation))\\b/i.test(context || '')) return 'accreditation';
4614
+ if (/\\b(?:certifi(?:ed|cation)|compliance)\\b/i.test(context || '')) return 'certification';
4615
+ if (/\\b(?:award|winner|best of|top[-_ ]?rated)\\b/i.test(context || '')) return 'award';
4616
+ if (/\\b(?:member|membership|association)\\b/i.test(context || '')) return 'membership';
4617
+ if (/\\b(?:press|featured|as seen|mentioned)\\b/i.test(context || '')) return 'press_mention';
4618
+ if (/\\b(?:partner|customer|client|trusted by|used by|works with)\\b/i.test(context || '')) return 'partner_or_customer';
4619
+ return 'trust_mark';
4620
+ }
4621
+ function addProof(raw, proofType, score, evidence, alt, context, width, height) {
4622
+ if (!raw || raw.startsWith('data:') || raw.startsWith('blob:')) return;
4623
+ var url;
4624
+ try { url = new URL(raw, document.baseURI); } catch (_) { return; }
4625
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') return;
4626
+ url.hash = '';
4627
+ var key = url.href;
4628
+ var existing = proofCandidates[key];
4629
+ if (!existing || score > existing.score) {
4630
+ proofCandidates[key] = {
4631
+ url: key, proofType: proofType, score: score, confidence: 'low',
4632
+ evidence: Array.from(new Set(evidence)), alt: alt || null, context: context || null,
4633
+ width: width || null, height: height || null
4634
+ };
4635
+ }
4636
+ }
4637
+
4638
+ var imageEls = document.querySelectorAll('img');
4639
+ for (var li = 0; li < imageEls.length; li++) {
4640
+ var image = imageEls[li];
4641
+ var src = image.currentSrc || image.src || image.getAttribute('data-src') || image.getAttribute('data-lazy-src');
4642
+ if (!src) continue;
4643
+ var header = image.closest('header,[role="banner"]');
4644
+ var nav = image.closest('nav,[role="navigation"]');
4645
+ var footer = image.closest('footer,[role="contentinfo"]');
4646
+ var region = header ? 'header' : nav ? 'nav' : footer ? 'footer' : 'body';
4647
+ var evidence = [];
4648
+ var score = 0;
4649
+ if (header) { score += 35; evidence.push('header_or_banner'); }
4650
+ else if (nav) { score += 28; evidence.push('navigation'); }
4651
+ else if (footer) { score += 18; evidence.push('footer_or_contentinfo'); }
4652
+ var descriptor = [src, image.id, image.className, image.alt, image.getAttribute('itemprop'), image.parentElement && image.parentElement.className].join(' ');
4653
+ var identityDescriptor = [image.id, image.className, image.getAttribute('itemprop'), image.parentElement && image.parentElement.className].join(' ');
4654
+ if (/\\b(?:site[-_ ]?logo|brand[-_ ]?logo|custom-logo|navbar[-_ ]?brand|site[-_ ]?branding|site[-_ ]?identity|logo[-_ ]?link)\\b/i.test(identityDescriptor)) { score += 30; evidence.push('site_identity_marker'); }
4655
+ else if (/\\blogo\\b/i.test(descriptor)) { score += 12; evidence.push('generic_logo_marker'); }
4656
+ var anchor = image.closest('a[href]');
4657
+ if (anchor) {
4658
+ try {
4659
+ var destination = new URL(anchor.href, document.baseURI);
4660
+ if (destination.origin === location.origin && destination.pathname.replace(/\\/+$/, '') === '') {
4661
+ score += 22; evidence.push('links_to_homepage');
4662
+ }
4663
+ } catch (_) {}
4664
+ }
4665
+ if (isSameDomain(src)) { score += 8; evidence.push('same_domain'); }
4666
+ var style = cs(image);
4667
+ var visible = !!style && style.display !== 'none' && style.visibility !== 'hidden' && parseFloat(style.opacity || '1') > 0;
4668
+ if (visible) { score += 5; evidence.push('visible'); }
4669
+ var width = image.naturalWidth || image.width || null;
4670
+ var height = image.naturalHeight || image.height || null;
4671
+ if (width && height && width >= 80 && height >= 24) { score += 5; evidence.push('plausible_logo_dimensions'); }
4672
+ if ((width && width < 32) || (height && height < 16)) { score -= 45; evidence.push('tiny_asset_penalty'); }
4673
+ if (/\\b(?:icon|avatar|profile|search|menu|hamburger|social|badge)\\b/i.test(descriptor)) { score -= 35; evidence.push('icon_or_avatar_penalty'); }
4674
+ addLogo(src, region, evidence, score, image.alt || null, width, height);
4675
+
4676
+ if (region === 'body' && visible && width && height && width >= 80 && height >= 40) {
4677
+ var context = nearestHeadingText(image);
4678
+ var proofDescriptor = [src, image.alt, image.getAttribute('title')].filter(Boolean).join(' ');
4679
+ var proofEvidence = [];
4680
+ var proofScore = 0;
4681
+ var commercialContext = !!context && /\\b(?:insurances?|payment|financing|we accept|accepted health)\\b/i.test(context);
4682
+ if (/\\b(?:logo|seal|trust[-_ ]?mark|certification[-_ ]?mark)\\b/i.test(proofDescriptor)) { proofScore += 30; proofEvidence.push('proof_asset_marker'); }
4683
+ if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation)|certifi(?:ed|cation)|compliance|legitscript|hipaa|award|winner|best of|top[-_ ]?rated|member|membership|association|professionals?|press|featured|as seen|mentioned|partner|customer|client|trusted by|used by|works with)\\b/i.test(proofDescriptor)) { proofScore += 30; proofEvidence.push('proof_descriptor'); }
4684
+ if (context && /\\b(?:recognized|accredit(?:ed|ation)|certifi(?:ed|cation)|award|winner|member|membership|association|featured|as seen|mentioned|partner|customer|client|trusted|trust)\\b/i.test(context)) { proofScore += 25; proofEvidence.push('proof_section_context'); }
4685
+ if (commercialContext) { proofScore -= 100; proofEvidence.push('commercial_acceptance_context_exclusion'); }
4686
+ proofScore += 10; proofEvidence.push('proof_dimensions');
4687
+ proofScore += 5; proofEvidence.push('visible');
4688
+ if (isSameDomain(src)) { proofScore += 5; proofEvidence.push('same_domain'); }
4689
+ else { proofScore += 3; proofEvidence.push('external_asset_domain'); }
4690
+ if (!commercialContext && proofScore >= 45) addProof(src, proofTypeFor(proofDescriptor, context), proofScore, proofEvidence, image.alt || null, context, width, height);
4323
4691
  }
4324
4692
  }
4325
4693
  var faviconEl = document.querySelector(
4326
4694
  'link[rel~="icon"], link[rel="shortcut icon"], link[rel="apple-touch-icon"]'
4327
4695
  );
4696
+ if (faviconEl && faviconEl.href) addLogo(faviconEl.href, 'favicon', ['favicon_fallback_only'], 10, null, null, null);
4697
+ var rankedLogos = Object.values(logoCandidates).filter(function(candidate) { return candidate.region !== 'body'; }).sort(function(a, b) { return b.score - a.score || a.url.localeCompare(b.url); }).slice(0, 20);
4698
+ for (var ri = 0; ri < rankedLogos.length; ri++) {
4699
+ rankedLogos[ri].confidence = rankedLogos[ri].score >= 85 ? 'high' : rankedLogos[ri].score >= 60 ? 'medium' : 'low';
4700
+ }
4701
+ var primaryLogo = rankedLogos.length && rankedLogos[0].score >= 55 && rankedLogos[0].region !== 'favicon' ? rankedLogos[0] : null;
4328
4702
  return {
4329
- navBg: navStyle ? navStyle.backgroundColor : null,
4330
- bodyBg: bodyStyle ? bodyStyle.backgroundColor : null,
4331
- bodyColor: bodyStyle ? bodyStyle.color : null,
4332
- h1Color: h1Style ? h1Style.color : null,
4333
- btnBg: btnStyle ? btnStyle.backgroundColor : null,
4703
+ navBg: navStyle ? toHex(navStyle.backgroundColor) : null,
4704
+ bodyBg: (bodyStyle ? toHex(bodyStyle.backgroundColor) : null) || (cs(document.documentElement) ? toHex(cs(document.documentElement).backgroundColor) : null) || '#ffffff',
4705
+ bodyColor: bodyStyle ? toHex(bodyStyle.color) : null,
4706
+ h1Color: h1Style ? toHex(h1Style.color) : null,
4707
+ btnBg: btnStyle ? toHex(btnStyle.backgroundColor) : null,
4708
+ btnDetail: btnEl ? String(btnEl.textContent || btnEl.getAttribute('aria-label') || '').replace(/\\s+/g, ' ').trim().slice(0, 120) : null,
4334
4709
  bodyFont: bodyStyle ? bodyStyle.fontFamily : null,
4335
4710
  h1Font: h1Style ? h1Style.fontFamily : null,
4336
- logoSrc: logoSrc,
4711
+ logoSrc: primaryLogo ? primaryLogo.url : null,
4712
+ logoConfidence: primaryLogo ? primaryLogo.confidence : null,
4713
+ logoSelectionReason: primaryLogo ? primaryLogo.evidence.join(', ') : null,
4714
+ logoCandidates: rankedLogos,
4715
+ proofCandidates: Object.values(proofCandidates),
4337
4716
  faviconHref: faviconEl ? faviconEl.href : null,
4338
4717
  svgFreq: svgFreq,
4339
4718
  navChildBgFreq: navChildBgFreq,
4719
+ visibleBgFreq: visibleBgFreq,
4720
+ topBgFreq: topBgFreq,
4721
+ cssPrimaryCandidates: cssPrimaryCandidates,
4722
+ cssAccentCandidates: cssAccentCandidates,
4723
+ themeColor: themeColor,
4340
4724
  };
4341
4725
  })()
4342
4726
  `;
4343
4727
  const raw = await page.evaluate(evalScript);
4344
- const navBgHex = rgbToHex(raw.navBg ?? "");
4345
- const bodyBgHex = rgbToHex(raw.bodyBg ?? "");
4728
+ const logoSelection = selectRankedLogoCandidates(raw.logoCandidates ?? []);
4729
+ const navBgHex = raw.navBg;
4730
+ const bodyBgHex = raw.bodyBg;
4346
4731
  const navBgUsable = navBgHex && !isTransparentOrWhite(navBgHex) && navBgHex !== "#000000" && navBgHex !== "#020101" ? navBgHex : null;
4347
- const svgPrimary = dominantColor(raw.svgFreq ?? {});
4732
+ const svgPrimary = dominantChromaticColor(raw.svgFreq ?? {});
4348
4733
  const navChildBg = dominantColor(raw.navChildBgFreq ?? {});
4349
- const primary = navBgUsable ?? svgPrimary ?? navChildBg ?? bodyBgHex;
4350
- const accent = rgbToHex(raw.btnBg ?? "");
4351
- const text2 = rgbToHex(raw.bodyColor ?? "");
4352
- const heading = rgbToHex(raw.h1Color ?? "");
4734
+ const visibleBg = dominantColor(raw.visibleBgFreq ?? {});
4735
+ const topBg = dominantColor(raw.topBgFreq ?? {});
4736
+ const cssPrimary = raw.cssPrimaryCandidates?.[0] ?? null;
4737
+ const cssAccent = raw.cssAccentCandidates?.[0] ?? null;
4738
+ const themeColor = isChromatic(raw.themeColor) ? raw.themeColor : null;
4739
+ const ctaColor = isChromatic(raw.btnBg) ? raw.btnBg : null;
4740
+ const primaryChoice = cssPrimary ? { value: cssPrimary.hex, source: "css_variable", confidence: "high", detail: cssPrimary.name } : themeColor ? { value: themeColor, source: "theme_color", confidence: "high", detail: 'meta[name="theme-color"]' } : navBgUsable && isChromatic(navBgUsable) ? { value: navBgUsable, source: "navigation_background", confidence: "medium", detail: "nav, header, or banner computed background" } : topBg && isChromatic(topBg) ? { value: topBg, source: "navigation_background", confidence: "medium", detail: "dominant wide rendered background near the top of the page" } : svgPrimary && isChromatic(svgPrimary) ? { value: svgPrimary, source: "header_svg", confidence: "medium", detail: "dominant rendered identity SVG fill near the top of the page" } : navChildBg && isChromatic(navChildBg) ? { value: navChildBg, source: "navigation_background", confidence: "medium", detail: "dominant rendered navigation child background" } : ctaColor ? { value: ctaColor, source: "visible_cta", confidence: "medium", detail: raw.btnDetail } : visibleBg && isChromatic(visibleBg) ? { value: visibleBg, source: "visible_background", confidence: "low", detail: "area-weighted rendered element background" } : bodyBgHex && !isTransparentOrWhite(bodyBgHex) ? { value: bodyBgHex, source: "body_background", confidence: "low", detail: "computed body background fallback" } : null;
4741
+ const accentChoice = cssAccent ? { value: cssAccent.hex, source: "css_variable", confidence: "high", detail: cssAccent.name } : ctaColor ? { value: ctaColor, source: "visible_cta", confidence: "medium", detail: raw.btnDetail } : null;
4742
+ const primary = primaryChoice?.value ?? null;
4743
+ const accent = accentChoice?.value ?? null;
4744
+ const text2 = raw.bodyColor;
4745
+ const heading = raw.h1Color;
4353
4746
  const background = bodyBgHex;
4354
4747
  const scheme = background ? luminance(background) > 128 ? "light" : "dark" : null;
4355
4748
  return {
@@ -4361,14 +4754,200 @@ async function extractBrandingFromPage(page) {
4361
4754
  text: text2 ?? null,
4362
4755
  heading: heading ?? null
4363
4756
  },
4757
+ colorEvidence: {
4758
+ primary: primaryChoice,
4759
+ accent: accentChoice,
4760
+ background: background ? { value: background, source: "body_background", confidence: "high", detail: "computed body background" } : null,
4761
+ text: text2 ? { value: text2, source: "body_text", confidence: "high", detail: "computed body text color" } : null,
4762
+ heading: heading ? { value: heading, source: "heading_text", confidence: "high", detail: "computed first H1 text color" } : null
4763
+ },
4364
4764
  fonts: {
4365
4765
  body: firstFont(raw.bodyFont ?? ""),
4366
4766
  heading: firstFont(raw.h1Font ?? "")
4367
4767
  },
4368
4768
  assets: {
4369
- logo: raw.logoSrc || null,
4370
- favicon: raw.faviconHref || null
4769
+ logo: logoSelection.primary?.url ?? null,
4770
+ favicon: raw.faviconHref || null,
4771
+ logoConfidence: logoSelection.primary?.confidence ?? null,
4772
+ logoSelectionReason: logoSelection.selectionReason,
4773
+ logoVariants: logoSelection.variants,
4774
+ logoCandidates: logoSelection.candidates,
4775
+ proofImages: selectProofImages(raw.proofCandidates ?? [])
4776
+ }
4777
+ };
4778
+ }
4779
+
4780
+ // src/lib/rendered-media.ts
4781
+ var SETTLE_MS = 450;
4782
+ var STABLE_CHECKS_REQUIRED = 3;
4783
+ var MAX_SCROLL_ROUNDS = 40;
4784
+ var MAX_DOM_NODES = 15e3;
4785
+ function mergeAsset(target, incoming) {
4786
+ const existing = target.get(incoming.url);
4787
+ if (!existing) {
4788
+ target.set(incoming.url, incoming);
4789
+ return;
4790
+ }
4791
+ existing.discoveryMethods = [.../* @__PURE__ */ new Set([...existing.discoveryMethods, ...incoming.discoveryMethods])];
4792
+ existing.altTexts = [.../* @__PURE__ */ new Set([...existing.altTexts, ...incoming.altTexts])];
4793
+ existing.contexts = [.../* @__PURE__ */ new Set([...existing.contexts, ...incoming.contexts])];
4794
+ if ((incoming.width ?? 0) * (incoming.height ?? 0) > (existing.width ?? 0) * (existing.height ?? 0)) {
4795
+ existing.width = incoming.width;
4796
+ existing.height = incoming.height;
4797
+ }
4798
+ }
4799
+ async function readRenderedSnapshot(page) {
4800
+ await page.evaluate("globalThis.__name = globalThis.__name || ((target) => target)");
4801
+ return page.evaluate(({ maxDomNodes }) => {
4802
+ const rows = /* @__PURE__ */ new Map();
4803
+ const resolve2 = (raw) => {
4804
+ if (!raw || raw.startsWith("data:") || raw.startsWith("blob:")) return null;
4805
+ try {
4806
+ const parsed = new URL(raw, document.baseURI);
4807
+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
4808
+ parsed.hash = "";
4809
+ return parsed.href;
4810
+ } catch {
4811
+ return null;
4812
+ }
4813
+ };
4814
+ const contextFor = (element) => {
4815
+ const contexts = [];
4816
+ const section2 = element.closest('section,article,main,header,footer,aside,[role="region"]');
4817
+ if (section2?.id) contexts.push(`#${section2.id}`);
4818
+ const labelled = section2?.getAttribute("aria-label")?.trim();
4819
+ if (labelled) contexts.push(labelled.slice(0, 200));
4820
+ const heading = section2?.querySelector("h1,h2,h3,h4,h5,h6")?.textContent?.replace(/\s+/g, " ").trim();
4821
+ if (heading) contexts.push(heading.slice(0, 200));
4822
+ return contexts;
4823
+ };
4824
+ const add = (raw, type, method, element, dimensions, alt) => {
4825
+ const url = resolve2(raw);
4826
+ if (!url) return;
4827
+ const existing = rows.get(url) ?? {
4828
+ url,
4829
+ type,
4830
+ discoveryMethods: [],
4831
+ altTexts: [],
4832
+ contexts: [],
4833
+ width: null,
4834
+ height: null
4835
+ };
4836
+ existing.discoveryMethods = [.../* @__PURE__ */ new Set([...existing.discoveryMethods, method])];
4837
+ const cleanAlt = alt?.replace(/\s+/g, " ").trim();
4838
+ if (cleanAlt) existing.altTexts = [.../* @__PURE__ */ new Set([...existing.altTexts, cleanAlt.slice(0, 500)])];
4839
+ existing.contexts = [.../* @__PURE__ */ new Set([...existing.contexts, ...contextFor(element)])];
4840
+ if (dimensions && dimensions.width > 0 && dimensions.height > 0) {
4841
+ const currentArea = (existing.width ?? 0) * (existing.height ?? 0);
4842
+ if (dimensions.width * dimensions.height > currentArea) {
4843
+ existing.width = dimensions.width;
4844
+ existing.height = dimensions.height;
4845
+ }
4846
+ }
4847
+ rows.set(url, existing);
4848
+ };
4849
+ const srcsetUrls = (value) => (value ?? "").split(",").map((part) => part.trim().split(/\s+/)[0] ?? "").filter(Boolean);
4850
+ for (const img of Array.from(document.querySelectorAll("img"))) {
4851
+ const image = img;
4852
+ const dimensions = { width: image.naturalWidth, height: image.naturalHeight };
4853
+ add(image.currentSrc, "image", "rendered-current-src", image, dimensions, image.alt);
4854
+ add(image.getAttribute("src"), "image", "rendered-src", image, dimensions, image.alt);
4855
+ for (const attr of ["data-src", "data-lazy-src", "data-original", "data-bg", "data-background"]) {
4856
+ add(image.getAttribute(attr), "image", `rendered-${attr}`, image, dimensions, image.alt);
4857
+ }
4858
+ for (const url of srcsetUrls(image.getAttribute("srcset"))) add(url, "image", "rendered-srcset", image, dimensions, image.alt);
4859
+ const anchor = image.closest("a[href]");
4860
+ const anchorAttrs = anchor ? Array.from(anchor.attributes).map((attribute) => `${attribute.name}=${attribute.value}`).join(" ") : "";
4861
+ if (anchor && (/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(anchorAttrs) || /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(anchor.href))) add(anchor.href, "image", "rendered-lightbox-target", image, void 0, image.alt);
4862
+ }
4863
+ for (const source of Array.from(document.querySelectorAll("source"))) {
4864
+ const parent = source.closest("picture,video,audio");
4865
+ const type = parent?.tagName === "VIDEO" ? "video" : parent?.tagName === "AUDIO" ? "audio" : "image";
4866
+ add(source.getAttribute("src"), type, "rendered-source", source);
4867
+ for (const url of srcsetUrls(source.getAttribute("srcset"))) add(url, type, "rendered-source-srcset", source);
4868
+ }
4869
+ for (const video of Array.from(document.querySelectorAll("video"))) {
4870
+ const media = video;
4871
+ add(media.currentSrc || media.src, "video", "rendered-video", media);
4872
+ add(media.poster, "image", "rendered-video-poster", media, { width: media.videoWidth, height: media.videoHeight });
4873
+ }
4874
+ for (const audio of Array.from(document.querySelectorAll("audio"))) {
4875
+ const media = audio;
4876
+ add(media.currentSrc || media.src, "audio", "rendered-audio", media);
4877
+ }
4878
+ for (const svgImage of Array.from(document.querySelectorAll("svg image"))) {
4879
+ add(svgImage.getAttribute("href") || svgImage.getAttribute("xlink:href"), "image", "rendered-svg-image", svgImage);
4880
+ }
4881
+ const all = Array.from(document.querySelectorAll("*")).slice(0, maxDomNodes);
4882
+ for (const element of all) {
4883
+ const background = getComputedStyle(element).backgroundImage;
4884
+ if (!background || background === "none") continue;
4885
+ for (const match of background.matchAll(/url\(["']?(.+?)["']?\)/gi)) {
4886
+ add(match[1], "image", "rendered-computed-background", element);
4887
+ }
4371
4888
  }
4889
+ const documentHeight = Math.max(document.body?.scrollHeight ?? 0, document.documentElement?.scrollHeight ?? 0);
4890
+ const viewportHeight = window.innerHeight;
4891
+ return {
4892
+ assets: [...rows.values()],
4893
+ documentHeight,
4894
+ scrollY: window.scrollY,
4895
+ viewportHeight,
4896
+ atBottom: window.scrollY + viewportHeight >= documentHeight - 5
4897
+ };
4898
+ }, { maxDomNodes: MAX_DOM_NODES });
4899
+ }
4900
+ async function collectRenderedPageMedia(page, maxAssets = 1e3) {
4901
+ const boundedMax = Math.max(1, Math.min(maxAssets, 2e3));
4902
+ await page.evaluate(() => {
4903
+ for (const image of Array.from(document.querySelectorAll('img[loading="lazy"]'))) image.loading = "eager";
4904
+ }).catch(() => void 0);
4905
+ const assets = /* @__PURE__ */ new Map();
4906
+ let stableChecks = 0;
4907
+ let previousHeight = -1;
4908
+ let rounds = 0;
4909
+ let documentHeight = 0;
4910
+ while (rounds < MAX_SCROLL_ROUNDS) {
4911
+ rounds += 1;
4912
+ const before = assets.size;
4913
+ const snapshot = await readRenderedSnapshot(page);
4914
+ documentHeight = snapshot.documentHeight;
4915
+ for (const asset of snapshot.assets) mergeAsset(assets, asset);
4916
+ if (assets.size >= boundedMax) {
4917
+ return {
4918
+ assets: [...assets.values()].slice(0, boundedMax),
4919
+ exhausted: false,
4920
+ stopReason: "asset_limit",
4921
+ scrollRounds: rounds,
4922
+ documentHeight
4923
+ };
4924
+ }
4925
+ const unchanged = snapshot.atBottom && before === assets.size && previousHeight === snapshot.documentHeight;
4926
+ stableChecks = unchanged ? stableChecks + 1 : 0;
4927
+ previousHeight = snapshot.documentHeight;
4928
+ if (stableChecks >= STABLE_CHECKS_REQUIRED) {
4929
+ await page.evaluate(() => window.scrollTo(0, 0)).catch(() => void 0);
4930
+ return {
4931
+ assets: [...assets.values()],
4932
+ exhausted: true,
4933
+ stopReason: "page_exhausted",
4934
+ scrollRounds: rounds,
4935
+ documentHeight
4936
+ };
4937
+ }
4938
+ await page.evaluate(({ viewportHeight, documentHeight: height }) => {
4939
+ const next = Math.min(Math.max(0, height - viewportHeight), window.scrollY + Math.max(800, Math.floor(viewportHeight * 0.9)));
4940
+ window.scrollTo(0, next);
4941
+ }, { viewportHeight: snapshot.viewportHeight, documentHeight: snapshot.documentHeight });
4942
+ await page.waitForTimeout(SETTLE_MS);
4943
+ }
4944
+ await page.evaluate(() => window.scrollTo(0, 0)).catch(() => void 0);
4945
+ return {
4946
+ assets: [...assets.values()],
4947
+ exhausted: false,
4948
+ stopReason: "scroll_round_limit",
4949
+ scrollRounds: rounds,
4950
+ documentHeight
4372
4951
  };
4373
4952
  }
4374
4953
 
@@ -4376,7 +4955,7 @@ async function extractBrandingFromPage(page) {
4376
4955
  var DESKTOP_VIEWPORT = { width: 1440, height: 900 };
4377
4956
  var MOBILE_VIEWPORT = { width: 390, height: 844 };
4378
4957
  var SCROLL_STEP = 900;
4379
- var SETTLE_MS = 400;
4958
+ var SETTLE_MS2 = 400;
4380
4959
  async function scrollAndStitch(page, device = "desktop") {
4381
4960
  const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
4382
4961
  await page.setViewportSize(vp);
@@ -4392,7 +4971,7 @@ async function scrollAndStitch(page, device = "desktop") {
4392
4971
  let scrollY = 0;
4393
4972
  while (scrollY < totalH) {
4394
4973
  await page.evaluate((y) => window.scrollTo(0, y), scrollY);
4395
- await page.waitForTimeout(SETTLE_MS);
4974
+ await page.waitForTimeout(SETTLE_MS2);
4396
4975
  const remaining = totalH - scrollY;
4397
4976
  const stripH = Math.min(SCROLL_STEP, remaining);
4398
4977
  const buf = await page.screenshot({
@@ -4423,7 +5002,7 @@ async function scrollAndStitch(page, device = "desktop") {
4423
5002
  );
4424
5003
  return Buffer.from(base64, "base64");
4425
5004
  }
4426
- async function captureLocal(url, device, wantScreenshot, wantBranding) {
5005
+ async function captureLocal(url, device, wantScreenshot, wantBranding, wantMedia, maxMediaAssets) {
4427
5006
  const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
4428
5007
  const browser = await chromium2.launch({ headless: true });
4429
5008
  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
@@ -4434,17 +5013,18 @@ async function captureLocal(url, device, wantScreenshot, wantBranding) {
4434
5013
  }, runtimeUserAgent));
4435
5014
  const page = await context.newPage();
4436
5015
  try {
4437
- await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
5016
+ await page.goto(url, { waitUntil: "domcontentloaded", timeout: 6e4 });
4438
5017
  await page.waitForTimeout(1500);
4439
5018
  const result = {};
4440
5019
  if (wantBranding) result.branding = await extractBrandingFromPage(page);
5020
+ if (wantMedia) result.renderedMedia = await collectRenderedPageMedia(page, maxMediaAssets);
4441
5021
  if (wantScreenshot) result.screenshot = await scrollAndStitch(page, device);
4442
5022
  return result;
4443
5023
  } finally {
4444
5024
  await browser.close();
4445
5025
  }
4446
5026
  }
4447
- async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding) {
5027
+ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding, wantMedia, maxMediaAssets) {
4448
5028
  const kernel = new Kernel2({ apiKey: kernelApiKey });
4449
5029
  const openedAtMs = Date.now();
4450
5030
  const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
@@ -4457,10 +5037,11 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
4457
5037
  }, runtimeUserAgent));
4458
5038
  const page = context.pages()[0] ?? await context.newPage();
4459
5039
  try {
4460
- await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
5040
+ await page.goto(url, { waitUntil: "domcontentloaded", timeout: 6e4 });
4461
5041
  await page.waitForTimeout(1500);
4462
5042
  const result = {};
4463
5043
  if (wantBranding) result.branding = await extractBrandingFromPage(page);
5044
+ if (wantMedia) result.renderedMedia = await collectRenderedPageMedia(page, maxMediaAssets);
4464
5045
  if (wantScreenshot) result.screenshot = await scrollAndStitch(page, device);
4465
5046
  return result;
4466
5047
  } finally {
@@ -4480,12 +5061,14 @@ async function capturePageData(url, opts = {}) {
4480
5061
  const device = opts.device ?? "desktop";
4481
5062
  const wantShot = opts.screenshot ?? false;
4482
5063
  const wantBranding = opts.branding ?? false;
5064
+ const wantMedia = opts.media ?? false;
5065
+ const maxMediaAssets = opts.maxMediaAssets ?? 1e3;
4483
5066
  try {
4484
- return await captureLocal(url, device, wantShot, wantBranding);
5067
+ return await captureLocal(url, device, wantShot, wantBranding, wantMedia, maxMediaAssets);
4485
5068
  } catch (localErr) {
4486
5069
  const msg = localErr instanceof Error ? localErr.message : String(localErr);
4487
5070
  if (!opts.kernelApiKey) throw new Error(`Local capture failed (${msg}) and no headless browser backend is configured`);
4488
- return await captureKernel(url, opts.kernelApiKey, device, wantShot, wantBranding);
5071
+ return await captureKernel(url, opts.kernelApiKey, device, wantShot, wantBranding, wantMedia, maxMediaAssets);
4489
5072
  }
4490
5073
  }
4491
5074
 
@@ -4890,7 +5473,7 @@ function withTimeout(p, ms) {
4890
5473
  async function createBrowserWithRetry(client2, headless) {
4891
5474
  for (let attempt = 0; attempt < 3; attempt++) {
4892
5475
  try {
4893
- return await client2.browsers.create({ stealth: true, headless, timeout_seconds: 120 });
5476
+ return await client2.browsers.create({ stealth: true, ...headless ? { headless: true } : {}, timeout_seconds: 120 });
4894
5477
  } catch (err) {
4895
5478
  if (attempt === 2) throw err;
4896
5479
  await new Promise((r) => setTimeout(r, 5e3));
@@ -4899,7 +5482,7 @@ async function createBrowserWithRetry(client2, headless) {
4899
5482
  throw new Error("unreachable");
4900
5483
  }
4901
5484
  var PAGE_HARD_TIMEOUT_MS = 35e3;
4902
- async function crawlPage(context, url) {
5485
+ async function crawlPage(context, url, headless) {
4903
5486
  const startedAt = Date.now();
4904
5487
  let page = null;
4905
5488
  let guardState = null;
@@ -4908,13 +5491,38 @@ async function crawlPage(context, url) {
4908
5491
  guardState = await installBrowserRequestGuard(page, url);
4909
5492
  const work = (async (p) => {
4910
5493
  const resp = await p.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 });
4911
- const html = await p.evaluate((maxBytes) => {
5494
+ let html = await p.evaluate((maxBytes) => {
4912
5495
  const value = document.documentElement?.outerHTML ?? "";
4913
5496
  if (new TextEncoder().encode(value).byteLength > maxBytes) throw new Error(`Browser HTML exceeds ${maxBytes} byte limit`);
4914
5497
  return value;
4915
5498
  }, MAX_PAGE_HTML_BYTES);
4916
5499
  const headers = resp ? Object.fromEntries(Object.entries(await resp.headers()).map(([k, v]) => [k.toLowerCase(), v])) : void 0;
4917
5500
  const finalUrl = p.url();
5501
+ let challenge = detectBrowserChallenge({ status: resp?.status() ?? null, title: await p.title().catch(() => ""), html });
5502
+ if (challenge.detected && !headless) {
5503
+ const deadline = Date.now() + 2e4;
5504
+ while (challenge.detected && Date.now() < deadline) {
5505
+ await p.waitForTimeout(1e3);
5506
+ html = await p.content();
5507
+ challenge = detectBrowserChallenge({ status: resp?.status() ?? null, title: await p.title().catch(() => ""), html });
5508
+ }
5509
+ }
5510
+ if (challenge.detected) {
5511
+ return {
5512
+ url,
5513
+ html: null,
5514
+ status: resp?.status() ?? 0,
5515
+ via: "browser",
5516
+ headers,
5517
+ responseTimeMs: Date.now() - startedAt,
5518
+ redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null,
5519
+ errorCode: "browser_challenge_unresolved",
5520
+ error: `Browser challenge remained unresolved (${challenge.reason ?? "unknown challenge"}).`,
5521
+ browserMode: headless ? "stealth_headless" : "stealth_headful",
5522
+ challengeDetected: true,
5523
+ stopReason: "challenge_unresolved"
5524
+ };
5525
+ }
4918
5526
  return {
4919
5527
  url,
4920
5528
  html,
@@ -4922,7 +5530,10 @@ async function crawlPage(context, url) {
4922
5530
  via: "browser",
4923
5531
  headers,
4924
5532
  responseTimeMs: Date.now() - startedAt,
4925
- redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null
5533
+ redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null,
5534
+ browserMode: headless ? "stealth_headless" : "stealth_headful",
5535
+ challengeDetected: false,
5536
+ stopReason: resp && (resp.status() < 200 || resp.status() >= 300) ? "http_error" : "usable_page"
4926
5537
  };
4927
5538
  })(page);
4928
5539
  const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("page-hard-timeout")), PAGE_HARD_TIMEOUT_MS));
@@ -4937,7 +5548,10 @@ async function crawlPage(context, url) {
4937
5548
  via: "browser",
4938
5549
  responseTimeMs: Date.now() - startedAt,
4939
5550
  errorCode,
4940
- error: message.slice(0, 500)
5551
+ error: message.slice(0, 500),
5552
+ browserMode: headless ? "stealth_headless" : "stealth_headful",
5553
+ challengeDetected: false,
5554
+ stopReason: "navigation_failed"
4941
5555
  };
4942
5556
  } finally {
4943
5557
  if (page) await page.close().catch(() => {
@@ -4951,7 +5565,7 @@ async function crawlWithRotation(urls, opts) {
4951
5565
  const headless = opts.headless ?? DEFAULT_HEADLESS;
4952
5566
  const uniqueUrls = [...new Set(urls)];
4953
5567
  const byUrl = /* @__PURE__ */ new Map();
4954
- const runPass = async (targets, phase) => {
5568
+ const runPass = async (targets, phase, passHeadless) => {
4955
5569
  if (targets.length === 0) return;
4956
5570
  let next = 0;
4957
5571
  let done = 0;
@@ -4974,7 +5588,7 @@ async function crawlWithRotation(urls, opts) {
4974
5588
  const closedAtMs = Date.now();
4975
5589
  await client2.browsers.deleteByID(k.session_id).catch(() => {
4976
5590
  });
4977
- await recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent: headless, proxyUsed: typeof k.proxy_id === "string" ? Boolean(k.proxy_id) : null, openedAtMs, closedAtMs });
5591
+ await recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent: passHeadless ? true : null, proxyUsed: typeof k.proxy_id === "string" ? Boolean(k.proxy_id) : null, fallback: phase === "retry", openedAtMs, closedAtMs });
4978
5592
  }
4979
5593
  };
4980
5594
  try {
@@ -4986,7 +5600,7 @@ async function crawlWithRotation(urls, opts) {
4986
5600
  try {
4987
5601
  if (!browser || inBrowser >= urlsPerBrowser) {
4988
5602
  await retire();
4989
- kb = await createBrowserWithRetry(client2, headless);
5603
+ kb = await createBrowserWithRetry(client2, passHeadless);
4990
5604
  kbOpenedAtMs = Date.now();
4991
5605
  browser = await withTimeout(chromium3.connectOverCDP(kb.cdp_ws_url), 3e4);
4992
5606
  runtimeUserAgent = await detectRuntimeUserAgent(browser);
@@ -4994,7 +5608,7 @@ async function crawlWithRotation(urls, opts) {
4994
5608
  }
4995
5609
  const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
4996
5610
  try {
4997
- r = await crawlPage(context, url);
5611
+ r = await crawlPage(context, url, passHeadless);
4998
5612
  } finally {
4999
5613
  await context.close().catch(() => {
5000
5614
  });
@@ -5008,7 +5622,10 @@ async function crawlWithRotation(urls, opts) {
5008
5622
  status: 0,
5009
5623
  via: "browser",
5010
5624
  errorCode: "browser_session_failed",
5011
- error: message.slice(0, 500)
5625
+ error: message.slice(0, 500),
5626
+ browserMode: passHeadless ? "stealth_headless" : "stealth_headful",
5627
+ challengeDetected: false,
5628
+ stopReason: "navigation_failed"
5012
5629
  };
5013
5630
  await retire();
5014
5631
  }
@@ -5032,12 +5649,12 @@ async function crawlWithRotation(urls, opts) {
5032
5649
  };
5033
5650
  await Promise.all(Array.from({ length: workers }, () => worker()));
5034
5651
  };
5035
- await runPass(uniqueUrls, "main");
5652
+ await runPass(uniqueUrls, "main", headless);
5036
5653
  const failed = uniqueUrls.filter((u) => {
5037
5654
  const r = byUrl.get(u);
5038
5655
  return !r || !r.html || r.status < 200 || r.status >= 300;
5039
5656
  });
5040
- if (failed.length > 0) await runPass(failed, "retry");
5657
+ if (failed.length > 0) await runPass(failed, "retry", headless ? false : headless);
5041
5658
  return urls.map((u) => byUrl.get(u) ?? {
5042
5659
  url: u,
5043
5660
  html: null,
@@ -5749,8 +6366,6 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5749
6366
  if (text2) headings.push({ level: Number(m[1]), text: text2.slice(0, 200) });
5750
6367
  }
5751
6368
  const h1 = headings.find((h) => h.level === 1)?.text ?? null;
5752
- const bodyText = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
5753
- const wordCount2 = bodyText.split(" ").filter((w) => w.length > 2).length;
5754
6369
  const schemaTypes = [];
5755
6370
  let schemaTypeBytes = 0;
5756
6371
  const addSchemaType = (value) => {
@@ -5807,11 +6422,14 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5807
6422
  }
5808
6423
  }
5809
6424
  const mainHtml = (html.match(/<main[^>]*>([\s\S]*?)<\/main>/i)?.[1] ?? html.match(/<article[^>]*>([\s\S]*?)<\/article>/i)?.[1] ?? html.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? html).replace(/<(script|style|nav|header|footer|aside|noscript|svg|iframe)[^>]*>[\s\S]*?<\/\1>/gi, "");
6425
+ const mainText = mainHtml.replace(/<[^>]+>/g, " ").replace(/&(?:nbsp|#160);/gi, " ").replace(/&(?:amp|#38);/gi, "&").replace(/&#(?:x[0-9a-f]+|\d+);/gi, " ").replace(/\s+/g, " ").trim();
6426
+ const mainWords = mainText.match(/[\p{L}\p{N}][\p{L}\p{N}'’.-]*/gu) ?? [];
6427
+ const wordCount2 = mainWords.filter((word) => word.length > 2).length;
5810
6428
  let bodyMarkdown = "";
5811
6429
  try {
5812
6430
  bodyMarkdown = turndown.turndown(mainHtml).replace(/\n{3,}/g, "\n\n").trim().slice(0, MAX_PAGE_MARKDOWN);
5813
6431
  } catch {
5814
- bodyMarkdown = bodyText.slice(0, MAX_PAGE_MARKDOWN);
6432
+ bodyMarkdown = mainText.slice(0, MAX_PAGE_MARKDOWN);
5815
6433
  }
5816
6434
  const outlinks = [];
5817
6435
  const discoveryLinkSet = /* @__PURE__ */ new Set();
@@ -5839,10 +6457,11 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5839
6457
  const internal = abs.origin === origin;
5840
6458
  if (internal) {
5841
6459
  internalLinks++;
5842
- if (!discoveryLinkSet.has(abs.href)) {
5843
- const bytes2 = Buffer.byteLength(abs.href);
6460
+ const crawlable = normalizeUrl(abs.href, documentUrl);
6461
+ if (crawlable && !discoveryLinkSet.has(crawlable)) {
6462
+ const bytes2 = Buffer.byteLength(crawlable);
5844
6463
  if (discoveryLinkSet.size < PAGE_DATA_LIMITS.discoveryLinks && discoveryBytes + bytes2 <= PAGE_DATA_LIMITS.discoveryBytes) {
5845
- discoveryLinkSet.add(abs.href);
6464
+ discoveryLinkSet.add(crawlable);
5846
6465
  discoveryBytes += bytes2;
5847
6466
  } else truncatedFields.add("discoveryLinks");
5848
6467
  }
@@ -5864,8 +6483,8 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5864
6483
  const h1List = headings.filter((h) => h.level === 1);
5865
6484
  const h1_2 = h1List[1]?.text ?? null;
5866
6485
  const h2Count = headings.filter((h) => h.level === 2).length;
5867
- const textRatio = Number((bodyText.length / Math.max(1, html.length)).toFixed(3));
5868
- const contentHash = createHash("sha1").update(bodyText).digest("hex");
6486
+ const textRatio = Number((mainText.length / Math.max(1, html.length)).toFixed(3));
6487
+ const contentHash = createHash("sha1").update(mainText).digest("hex");
5869
6488
  const metaRobots = boundedText(html.match(/<meta[^>]+name=["']robots["'][^>]+content=["']([^"']*)["']/i)?.[1], 512)?.toLowerCase() ?? null;
5870
6489
  const relNext = boundedUrl(html.match(/<link[^>]+rel=["']next["'][^>]+href=["']([^"']*)["']/i)?.[1]);
5871
6490
  const relPrev = boundedUrl(html.match(/<link[^>]+rel=["']prev["'][^>]+href=["']([^"']*)["']/i)?.[1]);
@@ -5919,11 +6538,8 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5919
6538
  }
5920
6539
  }
5921
6540
  }
5922
- if (og?.image) {
5923
- try {
5924
- addImageLink(new URL(og.image, documentUrl).href);
5925
- } catch {
5926
- }
6541
+ for (const asset of discoverStaticMedia(html, documentUrl)) {
6542
+ if (asset.type === "image") addImageLink(asset.url);
5927
6543
  }
5928
6544
  const imageLinks = [...imageLinkSet];
5929
6545
  const canonNorm = (u) => {
@@ -5950,12 +6566,19 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5950
6566
  indexable = false;
5951
6567
  indexabilityReason = "canonicalised";
5952
6568
  }
5953
- const extractionSucceeded = bodyMarkdown.trim().length > 0;
6569
+ const bodyClass = html.match(/<body\b[^>]*\bclass\s*=\s*["']([^"']*)["']/i)?.[1] ?? "";
6570
+ const wordpressAttachment = /(?:^|\s)(?:attachment|attachment-template-default)(?:\s|$)/i.test(bodyClass);
6571
+ const meaningfulText = mainText.replace(/https?:\/\/\S+/gi, " ").replace(/\s+/g, " ").trim();
6572
+ const meaningfulWords = meaningfulText.match(/[\p{L}\p{N}][\p{L}\p{N}'’.-]*/gu) ?? [];
6573
+ const hasMeaningfulText = meaningfulText.length >= 24 && meaningfulWords.length >= 4;
6574
+ const extractionSucceeded = !wordpressAttachment && bodyMarkdown.trim().length > 0 && hasMeaningfulText;
6575
+ const failureCode = wordpressAttachment ? "wordpress_attachment_page" : bodyMarkdown.trim().length === 0 ? "empty_rendered_content" : "insufficient_text_content";
6576
+ const failureReason = wordpressAttachment ? "The HTML is a WordPress media attachment page, not a text-content page." : bodyMarkdown.trim().length === 0 ? "The HTML response produced no usable page content." : `The main content contained only ${meaningfulWords.length} word(s) and ${meaningfulText.length} text character(s).`;
5954
6577
  if (!extractionSucceeded) {
5955
6578
  indexable = false;
5956
6579
  indexabilityReason = "extraction-failed";
5957
6580
  }
5958
- const normalizedAttempts = extractionSucceeded ? fetchAttempts : fetchAttempts.map((attempt, index) => index === fetchAttempts.length - 1 && attempt.ok ? { ...attempt, ok: false, errorCode: "empty_rendered_content", error: "The HTML response produced no usable page content." } : attempt);
6581
+ const normalizedAttempts = extractionSucceeded ? fetchAttempts : fetchAttempts.map((attempt, index) => index === fetchAttempts.length - 1 && attempt.ok ? { ...attempt, ok: false, errorCode: failureCode, error: failureReason } : attempt);
5959
6582
  return {
5960
6583
  url,
5961
6584
  status,
@@ -6002,9 +6625,12 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
6002
6625
  discoveryLinks,
6003
6626
  bodyMarkdown,
6004
6627
  schema,
6628
+ contentKind: extractionSucceeded ? "text_page" : "insufficient_text",
6629
+ mainTextChars: meaningfulText.length,
6630
+ mainWordCount: meaningfulWords.length,
6005
6631
  extractionStatus: extractionSucceeded ? "successful" : "failed",
6006
- failureCode: extractionSucceeded ? null : "empty_rendered_content",
6007
- failureReason: extractionSucceeded ? null : "The HTML response produced no usable page content.",
6632
+ failureCode: extractionSucceeded ? null : failureCode,
6633
+ failureReason: extractionSucceeded ? null : failureReason,
6008
6634
  fetchAttempts: normalizedAttempts,
6009
6635
  truncatedFields: [...truncatedFields]
6010
6636
  };
@@ -6092,6 +6718,7 @@ async function fetchPagePlain(url) {
6092
6718
  const attempt3 = failedAttempt("fetch", res.status, responseTimeMs2, "non_html_response", `Plain fetch returned ${contentType}, not HTML.`);
6093
6719
  const page = emptyPageData(url, res.status, "fetch", { code: attempt3.errorCode, message: attempt3.error, attempts: [attempt3] });
6094
6720
  page.contentType = contentType;
6721
+ page.contentKind = "non_html";
6095
6722
  return page;
6096
6723
  }
6097
6724
  const html = await readResponseTextWithLimit(res, MAX_PAGE_HTML_BYTES, "Page HTML");
@@ -6134,9 +6761,24 @@ async function mapWithConcurrency(items, concurrency, fn) {
6134
6761
  }
6135
6762
  async function extractPagesRotating(urls, opts) {
6136
6763
  const uniqueUrls = [...new Set(urls)];
6137
- const plainPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY, fetchPagePlain);
6764
+ const plainPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY, async (url) => {
6765
+ const checked = await validatePublicHttpUrl(url, { field: "page URL" });
6766
+ if (checked.error || !checked.parsed) return fetchPagePlain(url);
6767
+ if (normalizeUrl(checked.parsed.href, checked.parsed.href)) return fetchPagePlain(url);
6768
+ const attempt = failedAttempt("fetch", null, 0, "non_page_url", "The URL points to an asset, feed, API, account route, or other non-page path excluded from text-page extraction.");
6769
+ const page = emptyPageData(url, null, "fetch", { code: attempt.errorCode, message: attempt.error, attempts: [attempt] });
6770
+ page.contentKind = "excluded_url";
6771
+ return page;
6772
+ });
6138
6773
  const byUrl = new Map(plainPages.map((page) => [page.url, page]));
6139
- const browserTargets = plainPages.filter((page) => !isPageExtractionSuccessful(page) && !["unsafe_url", "unsafe_redirect", "cross_site_redirect"].includes(page.failureCode ?? "")).map((page) => page.url);
6774
+ const browserTargets = plainPages.filter((page) => !isPageExtractionSuccessful(page) && ![
6775
+ "unsafe_url",
6776
+ "unsafe_redirect",
6777
+ "cross_site_redirect",
6778
+ "non_html_response",
6779
+ "non_page_url",
6780
+ "wordpress_attachment_page"
6781
+ ].includes(page.failureCode ?? "")).map((page) => page.url);
6140
6782
  if (opts.kernelApiKey && browserTargets.length > 0) {
6141
6783
  const fetched = await crawlWithRotation(browserTargets, {
6142
6784
  apiKey: opts.kernelApiKey,
@@ -8740,7 +9382,7 @@ var siteExtractFn = inngest.createFunction(
8740
9382
  });
8741
9383
  }) : null;
8742
9384
  const finalized = await step.run("finalize", async () => {
8743
- const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
9385
+ const { assembleExtractArtifacts } = await import("./extract-bundle-GUEUCTAE.js");
8744
9386
  const current = await getExtractJob(jobId2);
8745
9387
  if (!current) throw new Error("extract job disappeared before finalization");
8746
9388
  const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -10071,8 +10713,8 @@ async function saveDirectoryCsvLocal(result) {
10071
10713
  const outDir = join3(outputBaseDir(), "directory-workflows");
10072
10714
  await mkdir(outDir, { recursive: true });
10073
10715
  const stamp = result.extractedAt.replace(/[:.]/g, "-");
10074
- const slug2 = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
10075
- const path5 = join3(outDir, `${stamp}-${slug2}-directory-workflow.csv`);
10716
+ const slug4 = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
10717
+ const path5 = join3(outDir, `${stamp}-${slug4}-directory-workflow.csv`);
10076
10718
  await writeFile(path5, renderDirectoryWorkflowCsv(result), "utf8");
10077
10719
  return path5;
10078
10720
  }
@@ -12214,9 +12856,9 @@ async function listAllPublicLocalSourcebook(category) {
12214
12856
  }) : await getDb().execute(`SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') ORDER BY s.category, s.state, s.business_name`);
12215
12857
  return result.rows.map((row) => parseJson2(row.payload_json));
12216
12858
  }
12217
- async function getPublicLocalSourcebook(category, state, slug2) {
12859
+ async function getPublicLocalSourcebook(category, state, slug4) {
12218
12860
  await ensureLocalSourcebookSchema();
12219
- const result = await getDb().execute({ sql: `SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.category = ? AND s.state = ? AND s.slug = ? AND s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') LIMIT 1`, args: [category, state, slug2] });
12861
+ const result = await getDb().execute({ sql: `SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.category = ? AND s.state = ? AND s.slug = ? AND s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') LIMIT 1`, args: [category, state, slug4] });
12220
12862
  return result.rows[0] ? parseJson2(result.rows[0].payload_json) : null;
12221
12863
  }
12222
12864
 
@@ -12520,6 +13162,235 @@ var MapsReviewCollector = class {
12520
13162
  }
12521
13163
  };
12522
13164
 
13165
+ // src/extractor/MapsImageCollector.ts
13166
+ var QUIESCENCE_MS = 700;
13167
+ var STABLE_CHECKS_REQUIRED2 = 3;
13168
+ var MAX_SCROLL_ROUNDS2 = 80;
13169
+ var OWNER_DISCOVERY_LIMIT = 500;
13170
+ function mapsImageIdentity(rawUrl) {
13171
+ try {
13172
+ const url = new URL(rawUrl);
13173
+ const path5 = url.hostname.toLowerCase().endsWith("googleusercontent.com") ? url.pathname.replace(/=[^/]+$/, "") : url.pathname;
13174
+ return `${url.hostname.toLowerCase()}${path5}`;
13175
+ } catch {
13176
+ return rawUrl.split("=")[0] ?? rawUrl;
13177
+ }
13178
+ }
13179
+ function isMapsPhotoUrl(rawUrl) {
13180
+ try {
13181
+ const host2 = new URL(rawUrl).hostname.toLowerCase().replace(/\.$/, "");
13182
+ return host2 === "googleusercontent.com" || host2.endsWith(".googleusercontent.com") || host2 === "streetviewpixels-pa.googleapis.com";
13183
+ } catch {
13184
+ return false;
13185
+ }
13186
+ }
13187
+ async function openPhotoGallery(page) {
13188
+ const seePhotos = page.getByRole("button", { name: /^see photos$/i }).first();
13189
+ const exact = await seePhotos.count() > 0 ? seePhotos : page.locator('[aria-label^="See photos" i], button[aria-label^="Photo of" i]').first();
13190
+ if (await exact.count() === 0) {
13191
+ return false;
13192
+ }
13193
+ const clicked = await exact.click({ timeout: 5e3 }).then(() => true, () => false);
13194
+ if (!clicked) return false;
13195
+ const galleryControl = page.locator('button[aria-label="By owner" i], [role="tab"][aria-label*="owner" i]').first();
13196
+ await galleryControl.waitFor({ state: "visible", timeout: 5e3 }).catch(() => void 0);
13197
+ return galleryControl.isVisible().catch(() => false);
13198
+ }
13199
+ async function selectGalleryTab(page, scope) {
13200
+ const matcher = scope === "owner" ? /by owner/i : /^all(?:\s|$)/i;
13201
+ const direct = page.locator(scope === "owner" ? 'button[aria-label="By owner" i]' : 'button[aria-label="All" i]');
13202
+ const roleTab = page.getByRole("tab", { name: matcher });
13203
+ const tabs = await direct.count() > 0 ? direct : roleTab;
13204
+ if (await tabs.count() === 0) return false;
13205
+ const visible = tabs.filter({ visible: true }).first();
13206
+ if (await visible.count() === 0) return false;
13207
+ const clicked = await visible.click({ timeout: 5e3 }).then(() => true, () => false);
13208
+ if (!clicked) return false;
13209
+ await page.waitForTimeout(QUIESCENCE_MS);
13210
+ return true;
13211
+ }
13212
+ async function readAndAdvanceGallery(page) {
13213
+ return page.evaluate(() => {
13214
+ const mediaNodes = Array.from(document.querySelectorAll('[aria-label^="Photo " i], [aria-label^="Video " i]'));
13215
+ const imageUrl = (root) => {
13216
+ const candidates = [root, ...Array.from(root.querySelectorAll("*"))];
13217
+ for (const node of candidates) {
13218
+ if (node instanceof HTMLImageElement) {
13219
+ const value = node.currentSrc || node.src;
13220
+ if (value) return value;
13221
+ }
13222
+ const inline = node.style.backgroundImage;
13223
+ const computed = inline || window.getComputedStyle(node).backgroundImage;
13224
+ const match = computed?.match(/url\(["']?(.+?)["']?\)/i);
13225
+ if (match?.[1]) return match[1];
13226
+ }
13227
+ return null;
13228
+ };
13229
+ const items = mediaNodes.map((node) => {
13230
+ const label = node.getAttribute("aria-label") ?? "";
13231
+ const match = label.match(/^(Photo|Video)\s+(\d+)/i);
13232
+ return {
13233
+ kind: match?.[1]?.toLowerCase() === "video" ? "video" : "photo",
13234
+ position: match?.[2] ? Number(match[2]) : null,
13235
+ url: imageUrl(node)
13236
+ };
13237
+ });
13238
+ const first = mediaNodes[0] ?? null;
13239
+ const ancestors = [];
13240
+ let current = first?.parentElement ?? null;
13241
+ while (current) {
13242
+ if (current.scrollHeight > current.clientHeight + 40) ancestors.push(current);
13243
+ current = current.parentElement;
13244
+ }
13245
+ const fallback = Array.from(document.querySelectorAll("div")).filter((node) => {
13246
+ const style = window.getComputedStyle(node);
13247
+ return /(auto|scroll)/.test(style.overflowY) && node.scrollHeight > node.clientHeight + 40 && mediaNodes.some((media) => node.contains(media));
13248
+ });
13249
+ const preferred = Array.from(document.querySelectorAll("div.m6QErb.DxyBCb.kA9KIf.dS8AEf")).find((node) => mediaNodes.some((media) => node.contains(media)) && node.scrollHeight > node.clientHeight + 40);
13250
+ const scroller = preferred ?? [...ancestors, ...fallback].sort((a, b) => b.scrollHeight - a.scrollHeight)[0] ?? null;
13251
+ const atBottom = scroller ? scroller.scrollTop + scroller.clientHeight >= scroller.scrollHeight - 5 : true;
13252
+ const metrics = {
13253
+ scrollHeight: scroller?.scrollHeight ?? 0,
13254
+ scrollTop: scroller?.scrollTop ?? 0,
13255
+ clientHeight: scroller?.clientHeight ?? 0,
13256
+ atBottom
13257
+ };
13258
+ if (scroller && !atBottom) {
13259
+ scroller.scrollTop = Math.min(
13260
+ scroller.scrollHeight - scroller.clientHeight,
13261
+ scroller.scrollTop + Math.max(1200, Math.floor(scroller.clientHeight * 1.75))
13262
+ );
13263
+ }
13264
+ return { items, ...metrics };
13265
+ });
13266
+ }
13267
+ async function collectGalleryPass(page, limit) {
13268
+ const photos = /* @__PURE__ */ new Map();
13269
+ const videos = /* @__PURE__ */ new Set();
13270
+ let stableChecks = 0;
13271
+ let previousHeight = -1;
13272
+ let rounds = 0;
13273
+ while (rounds < MAX_SCROLL_ROUNDS2) {
13274
+ rounds += 1;
13275
+ const before = photos.size + videos.size;
13276
+ const snapshot = await readAndAdvanceGallery(page);
13277
+ for (const item of snapshot.items) {
13278
+ if (item.kind === "video") {
13279
+ videos.add(item.url ? mapsImageIdentity(item.url) : `position:${item.position ?? videos.size}`);
13280
+ continue;
13281
+ }
13282
+ if (!item.url || !isMapsPhotoUrl(item.url)) continue;
13283
+ const key = mapsImageIdentity(item.url);
13284
+ const existing = photos.get(key);
13285
+ if (!existing || (item.position ?? Infinity) < (existing.position ?? Infinity)) photos.set(key, item);
13286
+ }
13287
+ if (photos.size >= limit) {
13288
+ return {
13289
+ items: [...photos.values()].slice(0, limit),
13290
+ exhausted: false,
13291
+ stopReason: "limit_reached",
13292
+ photosDiscovered: photos.size
13293
+ };
13294
+ }
13295
+ await page.waitForTimeout(QUIESCENCE_MS);
13296
+ const unchanged = snapshot.atBottom && before === photos.size + videos.size && previousHeight === snapshot.scrollHeight;
13297
+ stableChecks = unchanged ? stableChecks + 1 : 0;
13298
+ previousHeight = snapshot.scrollHeight;
13299
+ if (stableChecks >= STABLE_CHECKS_REQUIRED2) {
13300
+ return {
13301
+ items: [...photos.values()],
13302
+ exhausted: true,
13303
+ stopReason: "gallery_exhausted",
13304
+ photosDiscovered: photos.size
13305
+ };
13306
+ }
13307
+ }
13308
+ return {
13309
+ items: [...photos.values()],
13310
+ exhausted: false,
13311
+ stopReason: "scroll_round_limit",
13312
+ photosDiscovered: photos.size
13313
+ };
13314
+ }
13315
+ var MapsImageCollector = class {
13316
+ constructor(page) {
13317
+ this.page = page;
13318
+ }
13319
+ page;
13320
+ async collect(scope, maxImages) {
13321
+ if (!await openPhotoGallery(this.page)) {
13322
+ return {
13323
+ images: [],
13324
+ scope,
13325
+ exhausted: false,
13326
+ stopReason: "gallery_unavailable",
13327
+ ownerGalleryAvailable: false,
13328
+ ownerGalleryExhausted: false,
13329
+ ownerPhotosDiscovered: 0,
13330
+ allPhotosDiscovered: 0
13331
+ };
13332
+ }
13333
+ const ownerGalleryAvailable = await selectGalleryTab(this.page, "owner");
13334
+ const ownerPass = ownerGalleryAvailable ? await collectGalleryPass(this.page, scope === "owner" ? maxImages : OWNER_DISCOVERY_LIMIT) : { items: [], exhausted: false, stopReason: "gallery_unavailable", photosDiscovered: 0 };
13335
+ const ownerKeys = new Set(ownerPass.items.flatMap((item) => item.url ? [mapsImageIdentity(item.url)] : []));
13336
+ if (scope === "owner") {
13337
+ return {
13338
+ images: ownerPass.items.flatMap((item, index) => item.url ? [{
13339
+ index: index + 1,
13340
+ galleryPosition: item.position,
13341
+ sourceUrl: item.url,
13342
+ mediaKey: mapsImageIdentity(item.url),
13343
+ origin: "owner",
13344
+ originConfidence: "owner_gallery"
13345
+ }] : []),
13346
+ scope,
13347
+ exhausted: ownerPass.exhausted,
13348
+ stopReason: ownerGalleryAvailable ? ownerPass.stopReason : "gallery_unavailable",
13349
+ ownerGalleryAvailable,
13350
+ ownerGalleryExhausted: ownerPass.exhausted,
13351
+ ownerPhotosDiscovered: ownerPass.photosDiscovered,
13352
+ allPhotosDiscovered: ownerPass.photosDiscovered
13353
+ };
13354
+ }
13355
+ if (!await selectGalleryTab(this.page, "all")) {
13356
+ return {
13357
+ images: [],
13358
+ scope,
13359
+ exhausted: false,
13360
+ stopReason: "gallery_unavailable",
13361
+ ownerGalleryAvailable,
13362
+ ownerGalleryExhausted: ownerPass.exhausted,
13363
+ ownerPhotosDiscovered: ownerPass.photosDiscovered,
13364
+ allPhotosDiscovered: 0
13365
+ };
13366
+ }
13367
+ const allPass = await collectGalleryPass(this.page, maxImages);
13368
+ const images = allPass.items.flatMap((item, index) => {
13369
+ if (!item.url) return [];
13370
+ const mediaKey = mapsImageIdentity(item.url);
13371
+ const isOwner = ownerKeys.has(mediaKey);
13372
+ return [{
13373
+ index: index + 1,
13374
+ galleryPosition: item.position,
13375
+ sourceUrl: item.url,
13376
+ mediaKey,
13377
+ origin: isOwner ? "owner" : ownerPass.exhausted ? "other" : "unknown",
13378
+ originConfidence: isOwner ? "owner_gallery_match" : ownerPass.exhausted ? "not_in_exhausted_owner_gallery" : "owner_gallery_incomplete"
13379
+ }];
13380
+ });
13381
+ return {
13382
+ images,
13383
+ scope,
13384
+ exhausted: allPass.exhausted,
13385
+ stopReason: allPass.stopReason,
13386
+ ownerGalleryAvailable,
13387
+ ownerGalleryExhausted: ownerPass.exhausted,
13388
+ ownerPhotosDiscovered: ownerPass.photosDiscovered,
13389
+ allPhotosDiscovered: allPass.photosDiscovered
13390
+ };
13391
+ }
13392
+ };
13393
+
12523
13394
  // src/extractor/MapsExtractor.ts
12524
13395
  import { z as z11 } from "zod";
12525
13396
  var MapsExtractor = class {
@@ -12544,6 +13415,48 @@ var MapsExtractor = class {
12544
13415
  const identifiers = this.parseUrlIdentifiers(placeUrl);
12545
13416
  const overview = await this.extractOverview(page);
12546
13417
  const hoursTable = await this.extractFullHours(page);
13418
+ let media = {
13419
+ status: "not_requested",
13420
+ scope: options.imageScope,
13421
+ requestedMaxImages: options.maxImages,
13422
+ images: [],
13423
+ imagesCollected: 0,
13424
+ imagesDownloaded: 0,
13425
+ ownerImagesCollected: 0,
13426
+ otherImagesCollected: 0,
13427
+ unknownOriginImagesCollected: 0,
13428
+ ownerGalleryAvailable: false,
13429
+ ownerGalleryExhausted: false,
13430
+ ownerPhotosDiscovered: 0,
13431
+ allPhotosDiscovered: 0,
13432
+ exhausted: false,
13433
+ stopReason: "not_requested",
13434
+ artifact: null,
13435
+ warnings: []
13436
+ };
13437
+ if (options.includeImages) {
13438
+ try {
13439
+ await page.goto(placeUrl, { waitUntil: "domcontentloaded", timeout: 2e4 });
13440
+ await page.waitForTimeout(500);
13441
+ const collected = await new MapsImageCollector(page).collect(options.imageScope, options.maxImages);
13442
+ const ownerImagesCollected = collected.images.filter((image) => image.origin === "owner").length;
13443
+ const otherImagesCollected = collected.images.filter((image) => image.origin === "other").length;
13444
+ const unknownOriginImagesCollected = collected.images.filter((image) => image.origin === "unknown").length;
13445
+ media = {
13446
+ ...media,
13447
+ ...collected,
13448
+ status: collected.images.length > 0 ? "collected" : collected.stopReason === "gallery_unavailable" ? "unavailable" : "none_exist",
13449
+ imagesCollected: collected.images.length,
13450
+ ownerImagesCollected,
13451
+ otherImagesCollected,
13452
+ unknownOriginImagesCollected
13453
+ };
13454
+ await page.goto(placeUrl, { waitUntil: "domcontentloaded", timeout: 2e4 }).catch(() => void 0);
13455
+ } catch (err) {
13456
+ console.warn("[MapsExtractor] image gallery extraction failed:", err instanceof Error ? err.message : String(err));
13457
+ media = { ...media, status: "unavailable", stopReason: "gallery_unavailable", warnings: ["Google Maps image gallery could not be read in this run."] };
13458
+ }
13459
+ }
12547
13460
  let reviewStats = { reviewHistogram: [], reviewTopics: [] };
12548
13461
  let reviews = [];
12549
13462
  let reviewsStatus = "not_requested";
@@ -12631,6 +13544,7 @@ var MapsExtractor = class {
12631
13544
  services,
12632
13545
  areasServed,
12633
13546
  servicesStatus,
13547
+ media,
12634
13548
  durationMs: Date.now() - startMs
12635
13549
  };
12636
13550
  } finally {
@@ -12913,11 +13827,11 @@ function rootDomain() {
12913
13827
  const configured = process.env.LOCAL_SOURCEBOOK_PUBLIC_ROOT_DOMAIN?.trim() || DEFAULT_ROOT_DOMAIN;
12914
13828
  return configured.replace(/^https?:\/\//i, "").replace(/^www\./i, "").replace(/\/.*$/, "").replace(/\.+$/, "").toLowerCase();
12915
13829
  }
12916
- function localSourcebookPublicUrls(category, state, slug2) {
13830
+ function localSourcebookPublicUrls(category, state, slug4) {
12917
13831
  const categoryUrl = `https://${CATEGORY_PUBLIC_SLUGS[category]}.${rootDomain()}`;
12918
13832
  const statePath = `/${state.toLowerCase()}`;
12919
- const profilePath = `${statePath}/${slug2}/`;
12920
- const reviewsPath = `${statePath}/${slug2}/reviews`;
13833
+ const profilePath = `${statePath}/${slug4}/`;
13834
+ const reviewsPath = `${statePath}/${slug4}/reviews`;
12921
13835
  return {
12922
13836
  categoryUrl,
12923
13837
  nationwideDirectoryUrl: `${categoryUrl}/directory`,
@@ -13465,13 +14379,13 @@ function host(value) {
13465
14379
  }
13466
14380
  }
13467
14381
  function tokens(value) {
13468
- return new Set(value.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((token4) => token4.length > 1 && !BUSINESS_SUFFIXES.has(token4)));
14382
+ return new Set(value.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((token6) => token6.length > 1 && !BUSINESS_SUFFIXES.has(token6)));
13469
14383
  }
13470
14384
  function nameOverlap(left, right) {
13471
14385
  const a = tokens(left);
13472
14386
  const b = tokens(right);
13473
14387
  if (!a.size || !b.size) return 0;
13474
- const intersection = [...a].filter((token4) => b.has(token4)).length;
14388
+ const intersection = [...a].filter((token6) => b.has(token6)).length;
13475
14389
  return intersection / Math.max(a.size, b.size);
13476
14390
  }
13477
14391
  function validateLocalSourcebookMapsIdentity(submission, maps) {
@@ -13520,6 +14434,10 @@ function createDefaultLocalSourcebookProvider() {
13520
14434
  includeReviews: true,
13521
14435
  maxReviews,
13522
14436
  includeServices: true,
14437
+ includeImages: false,
14438
+ imageScope: "all",
14439
+ maxImages: 100,
14440
+ maxInlineImages: 3,
13523
14441
  kernelApiKey,
13524
14442
  headless: true
13525
14443
  });
@@ -14405,12 +15323,12 @@ function normalizeIdempotencyKey(value) {
14405
15323
  function tokenFor(id, idempotencyKey3) {
14406
15324
  return createHmac("sha256", billingSecret()).update(id).update(":").update(idempotencyKey3).digest("base64url");
14407
15325
  }
14408
- function tokenHash(token4) {
14409
- return createHash4("sha256").update(token4).digest("hex");
15326
+ function tokenHash(token6) {
15327
+ return createHash4("sha256").update(token6).digest("hex");
14410
15328
  }
14411
- function verifyToken(row, token4) {
15329
+ function verifyToken(row, token6) {
14412
15330
  const expected = Buffer.from(row.token_hash, "hex");
14413
- const actual = Buffer.from(tokenHash(token4), "hex");
15331
+ const actual = Buffer.from(tokenHash(token6), "hex");
14414
15332
  if (expected.length !== actual.length || !timingSafeEqual(expected, actual)) {
14415
15333
  throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
14416
15334
  }
@@ -14527,11 +15445,11 @@ async function authorizeScheduledRun(args) {
14527
15445
  );
14528
15446
  }
14529
15447
  const id = randomUUID4();
14530
- const token4 = tokenFor(id, idempotencyKey3);
15448
+ const token6 = tokenFor(id, idempotencyKey3);
14531
15449
  const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
14532
15450
  const inserted = await getDb().execute({
14533
15451
  sql: "INSERT OR IGNORE INTO billing_authorizations (id, user_id, idempotency_key, billing_class, status, minimum_amount_mc, maximum_amount_mc, token_hash, metadata, expires_at) VALUES (?, ?, ?, ?, 'authorized', ?, ?, ?, ?, ?)",
14534
- args: [id, user.id, idempotencyKey3, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_BASE_MC, maximumMc, tokenHash(token4), safeMetadata2(args.metadata), expiresAt]
15452
+ args: [id, user.id, idempotencyKey3, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_BASE_MC, maximumMc, tokenHash(token6), safeMetadata2(args.metadata), expiresAt]
14535
15453
  });
14536
15454
  if (inserted.rowsAffected === 0) {
14537
15455
  const raced = await getDb().execute({
@@ -14547,7 +15465,7 @@ async function authorizeScheduledRun(args) {
14547
15465
  }
14548
15466
  return {
14549
15467
  id,
14550
- token: token4,
15468
+ token: token6,
14551
15469
  status: "authorized",
14552
15470
  minimumMc: SCHEDULED_RUN_BASE_MC,
14553
15471
  maximumMc,
@@ -16157,9 +17075,9 @@ var YouTubeExtractor = class {
16157
17075
  }
16158
17076
  async fetchContinuation(page, endpoint, contToken, context) {
16159
17077
  const result = await page.evaluate(
16160
- async ({ ep, token: token4, ctx }) => {
17078
+ async ({ ep, token: token6, ctx }) => {
16161
17079
  const body = {
16162
- continuation: token4,
17080
+ continuation: token6,
16163
17081
  context: {
16164
17082
  client: {
16165
17083
  clientName: ctx.clientName,
@@ -16403,23 +17321,23 @@ async function extractOnce(options) {
16403
17321
  }
16404
17322
  async function writeOutputs(result, outputDir) {
16405
17323
  await fs.mkdir(outputDir, { recursive: true });
16406
- const slug2 = result.target.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
17324
+ const slug4 = result.target.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
16407
17325
  const ts = Date.now();
16408
17326
  await fs.writeFile(
16409
- path4.join(outputDir, `${slug2}-${ts}.json`),
17327
+ path4.join(outputDir, `${slug4}-${ts}.json`),
16410
17328
  JSON.stringify(result, null, 2),
16411
17329
  "utf8"
16412
17330
  );
16413
17331
  if (result.videos.length > 0) {
16414
17332
  await fs.writeFile(
16415
- path4.join(outputDir, `${slug2}-videos-${ts}.csv`),
17333
+ path4.join(outputDir, `${slug4}-videos-${ts}.csv`),
16416
17334
  Papa.unparse(result.videos, { header: true }),
16417
17335
  "utf8"
16418
17336
  );
16419
17337
  }
16420
17338
  if (result.downloads.length > 0) {
16421
17339
  await fs.writeFile(
16422
- path4.join(outputDir, `${slug2}-downloads-${ts}.csv`),
17340
+ path4.join(outputDir, `${slug4}-downloads-${ts}.csv`),
16423
17341
  Papa.unparse(result.downloads, { header: true }),
16424
17342
  "utf8"
16425
17343
  );
@@ -16783,6 +17701,8 @@ var ExtractUrlBodyShape = {
16783
17701
  includeFeaturedImage: z16.boolean().optional(),
16784
17702
  downloadMedia: z16.boolean().optional(),
16785
17703
  mediaTypes: z16.array(z16.enum(["image", "video", "audio"])).optional(),
17704
+ maxMediaAssets: z16.number().int().min(1).max(250).optional(),
17705
+ maxInlineImages: z16.number().int().min(0).max(5).optional(),
16786
17706
  delivery: z16.enum(["auto", "inline", "artifact", "memory"]).optional(),
16787
17707
  preserveMedia: z16.boolean().optional(),
16788
17708
  depositToVault: z16.boolean().optional(),
@@ -18059,7 +18979,7 @@ function advertiserTokenOverlap(query, candidate) {
18059
18979
  const candidateTokens = new Set(normalizeAdvertiserName(candidate).split(" ").filter(Boolean));
18060
18980
  if (queryTokens.size === 0 || candidateTokens.size === 0) return 0;
18061
18981
  let overlap = 0;
18062
- for (const token4 of queryTokens) if (candidateTokens.has(token4)) overlap++;
18982
+ for (const token6 of queryTokens) if (candidateTokens.has(token6)) overlap++;
18063
18983
  return overlap / queryTokens.size;
18064
18984
  }
18065
18985
  function assessPageIntelMatch(body, result) {
@@ -19294,8 +20214,8 @@ function getBlobStore() {
19294
20214
  return cached;
19295
20215
  }
19296
20216
  var VercelBlobStore = class {
19297
- constructor(token4) {
19298
- this.token = token4;
20217
+ constructor(token6) {
20218
+ this.token = token6;
19299
20219
  }
19300
20220
  token;
19301
20221
  kind = "vercel-blob";
@@ -19751,8 +20671,8 @@ function safeFilePart(input) {
19751
20671
  }
19752
20672
  function blobKeyPrefix(shortcode, sourceUrl) {
19753
20673
  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
19754
- const slug2 = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
19755
- return `instagram/${stamp}-${slug2}`;
20674
+ const slug4 = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
20675
+ return `instagram/${stamp}-${slug4}`;
19756
20676
  }
19757
20677
  async function fetchMediaBytes(url, referer, runtimeUserAgent) {
19758
20678
  const res = await fetch(url, {
@@ -20664,6 +21584,212 @@ videoApp.post("/status", createApiKeyAuth(), async (c) => {
20664
21584
 
20665
21585
  // src/api/maps-routes.ts
20666
21586
  import { Hono as Hono13 } from "hono";
21587
+
21588
+ // src/api/maps-media-artifacts.ts
21589
+ import { homedir as homedir2 } from "os";
21590
+ import { join as join6 } from "path";
21591
+ import { randomBytes as randomBytes2 } from "crypto";
21592
+ import { ZipFile } from "yazl";
21593
+ import pLimit4 from "p-limit";
21594
+ var MAPS_MEDIA_ARTIFACT_PREFIX = "maps-media/";
21595
+ var MAPS_MEDIA_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
21596
+ var MAPS_MEDIA_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
21597
+ var MAX_IMAGE_BYTES = 5 * 1024 * 1024;
21598
+ var MAX_ARCHIVE_IMAGE_BYTES = 40 * 1024 * 1024;
21599
+ var MAX_INLINE_IMAGE_BYTES = 15e5;
21600
+ var MAX_INLINE_TOTAL_BYTES = 25e5;
21601
+ var DOWNLOAD_CONCURRENCY = 6;
21602
+ var MAX_REDIRECTS = 4;
21603
+ function token() {
21604
+ return process.env.MAPS_MEDIA_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
21605
+ }
21606
+ function policy() {
21607
+ return {
21608
+ prefix: MAPS_MEDIA_ARTIFACT_PREFIX,
21609
+ artifactTtlMs: MAPS_MEDIA_ARTIFACT_TTL_MS,
21610
+ downloadTtlMs: MAPS_MEDIA_DOWNLOAD_TTL_MS,
21611
+ token: token()
21612
+ };
21613
+ }
21614
+ function allowedMapsImageUrl(raw) {
21615
+ try {
21616
+ const url = new URL(raw);
21617
+ if (url.protocol !== "https:") return null;
21618
+ const host2 = url.hostname.toLowerCase().replace(/\.$/, "");
21619
+ if (host2 !== "googleusercontent.com" && !host2.endsWith(".googleusercontent.com") && host2 !== "streetviewpixels-pa.googleapis.com") return null;
21620
+ return url;
21621
+ } catch {
21622
+ return null;
21623
+ }
21624
+ }
21625
+ function preferredMapsImageRendition(rawUrl) {
21626
+ const url = allowedMapsImageUrl(rawUrl);
21627
+ if (!url) return rawUrl;
21628
+ if (url.hostname.toLowerCase().endsWith("googleusercontent.com") && /=[^/]+$/.test(url.pathname)) {
21629
+ url.pathname = url.pathname.replace(/=[^/]+$/, "=s2048");
21630
+ }
21631
+ return url.href;
21632
+ }
21633
+ function imageMime(bytes) {
21634
+ if (bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return { mimeType: "image/jpeg", extension: "jpg" };
21635
+ if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80 && bytes[2] === 78 && bytes[3] === 71) return { mimeType: "image/png", extension: "png" };
21636
+ if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(0, 4)) === "RIFF" && String.fromCharCode(...bytes.slice(8, 12)) === "WEBP") return { mimeType: "image/webp", extension: "webp" };
21637
+ if (bytes.length >= 6 && ["GIF87a", "GIF89a"].includes(String.fromCharCode(...bytes.slice(0, 6)))) return { mimeType: "image/gif", extension: "gif" };
21638
+ return null;
21639
+ }
21640
+ async function readBounded(response) {
21641
+ const declared = Number(response.headers.get("content-length") ?? "");
21642
+ if (Number.isFinite(declared) && declared > MAX_IMAGE_BYTES) throw new Error("image_too_large");
21643
+ if (!response.body) {
21644
+ const bytes2 = new Uint8Array(await response.arrayBuffer());
21645
+ if (bytes2.byteLength > MAX_IMAGE_BYTES) throw new Error("image_too_large");
21646
+ return bytes2;
21647
+ }
21648
+ const reader = response.body.getReader();
21649
+ const chunks = [];
21650
+ let total = 0;
21651
+ try {
21652
+ while (true) {
21653
+ const { done, value } = await reader.read();
21654
+ if (done) break;
21655
+ if (!value) continue;
21656
+ total += value.byteLength;
21657
+ if (total > MAX_IMAGE_BYTES) throw new Error("image_too_large");
21658
+ chunks.push(value);
21659
+ }
21660
+ } finally {
21661
+ reader.releaseLock();
21662
+ }
21663
+ const bytes = new Uint8Array(total);
21664
+ let offset = 0;
21665
+ for (const chunk of chunks) {
21666
+ bytes.set(chunk, offset);
21667
+ offset += chunk.byteLength;
21668
+ }
21669
+ return bytes;
21670
+ }
21671
+ async function fetchMapsImage(rawUrl) {
21672
+ let url = allowedMapsImageUrl(preferredMapsImageRendition(rawUrl));
21673
+ if (!url) throw new Error("image_host_not_allowed");
21674
+ for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {
21675
+ const checked = await validatePublicHttpUrl(url.href, { field: "Maps image URL" });
21676
+ if (checked.error || !checked.parsed) throw new Error("image_url_rejected");
21677
+ const response = await fetch(checked.parsed.href, {
21678
+ redirect: "manual",
21679
+ headers: { Accept: "image/jpeg,image/png,image/webp,image/gif;q=0.8" },
21680
+ signal: AbortSignal.timeout(15e3)
21681
+ });
21682
+ if ([301, 302, 303, 307, 308].includes(response.status)) {
21683
+ const location2 = response.headers.get("location");
21684
+ if (!location2 || redirect === MAX_REDIRECTS) throw new Error("image_redirect_rejected");
21685
+ url = allowedMapsImageUrl(new URL(location2, url).href);
21686
+ if (!url) throw new Error("image_redirect_host_not_allowed");
21687
+ continue;
21688
+ }
21689
+ if (!response.ok) throw new Error(`image_fetch_${response.status}`);
21690
+ const body = await readBounded(response);
21691
+ const detected = imageMime(body);
21692
+ if (!detected) throw new Error("image_content_invalid");
21693
+ return { bytes: Buffer.from(body), ...detected, finalUrl: url.href };
21694
+ }
21695
+ throw new Error("image_redirect_rejected");
21696
+ }
21697
+ function zipBuffer(files) {
21698
+ return new Promise((resolve2, reject) => {
21699
+ const zip = new ZipFile();
21700
+ const chunks = [];
21701
+ zip.outputStream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
21702
+ zip.outputStream.once("error", reject);
21703
+ zip.outputStream.once("end", () => resolve2(Buffer.concat(chunks)));
21704
+ for (const file of files) zip.addBuffer(file.content, file.path);
21705
+ zip.end();
21706
+ });
21707
+ }
21708
+ function slug(value) {
21709
+ return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 70) || "google-maps-place";
21710
+ }
21711
+ function mapsMediaArtifactOwnerId(artifactId) {
21712
+ return privateArtifactOwnerId(artifactId, MAPS_MEDIA_ARTIFACT_PREFIX);
21713
+ }
21714
+ async function readOwnedMapsMediaArtifactBuffer(args) {
21715
+ if (mapsMediaArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
21716
+ return readPrivateArtifactBuffer({ policy: policy(), artifactId: args.artifactId, maxBytes: 50 * 1024 * 1024 });
21717
+ }
21718
+ async function packageMapsMedia(args) {
21719
+ const limit = pLimit4(DOWNLOAD_CONCURRENCY);
21720
+ const files = [];
21721
+ let storedBytes = 0;
21722
+ let inlineBytes = 0;
21723
+ let inlineCount = 0;
21724
+ await Promise.all(args.media.images.map((image) => limit(async () => {
21725
+ try {
21726
+ const fetched = await fetchMapsImage(image.sourceUrl);
21727
+ if (storedBytes + fetched.bytes.length > MAX_ARCHIVE_IMAGE_BYTES) throw new Error("archive_byte_limit_reached");
21728
+ storedBytes += fetched.bytes.length;
21729
+ const filename = `images/${String(image.index).padStart(4, "0")}-${image.origin}.${fetched.extension}`;
21730
+ files.push({ path: filename, content: fetched.bytes });
21731
+ image.sourceUrl = fetched.finalUrl;
21732
+ image.filename = filename;
21733
+ image.mimeType = fetched.mimeType;
21734
+ image.bytes = fetched.bytes.length;
21735
+ image.downloadStatus = "downloaded";
21736
+ image.downloadError = null;
21737
+ if (inlineCount < args.maxInlineImages && fetched.bytes.length <= MAX_INLINE_IMAGE_BYTES && inlineBytes + fetched.bytes.length <= MAX_INLINE_TOTAL_BYTES) {
21738
+ image.inlinePreview = { data: fetched.bytes.toString("base64"), mimeType: fetched.mimeType };
21739
+ inlineBytes += fetched.bytes.length;
21740
+ inlineCount += 1;
21741
+ }
21742
+ } catch (error) {
21743
+ image.filename = null;
21744
+ image.mimeType = null;
21745
+ image.bytes = null;
21746
+ image.downloadStatus = "failed";
21747
+ image.downloadError = error instanceof Error ? error.message : "image_download_failed";
21748
+ image.inlinePreview = null;
21749
+ }
21750
+ })));
21751
+ args.media.imagesDownloaded = args.media.images.filter((image) => image.downloadStatus === "downloaded").length;
21752
+ const failed = args.media.images.length - args.media.imagesDownloaded;
21753
+ if (failed > 0) args.media.warnings.push(`${failed} image(s) could not be downloaded; their manifest rows include a bounded error code.`);
21754
+ if (args.media.imagesDownloaded === 0) return { media: args.media, artifact: null };
21755
+ const cleanImages = args.media.images.map(({ inlinePreview: _preview, ...image }) => image);
21756
+ const summary = {
21757
+ businessName: args.businessName,
21758
+ placeUrl: args.placeUrl,
21759
+ extractedAt: args.extractedAt,
21760
+ scope: args.media.scope,
21761
+ imagesCollected: args.media.imagesCollected,
21762
+ imagesDownloaded: args.media.imagesDownloaded,
21763
+ ownerImagesCollected: args.media.ownerImagesCollected,
21764
+ otherImagesCollected: args.media.otherImagesCollected,
21765
+ unknownOriginImagesCollected: args.media.unknownOriginImagesCollected,
21766
+ ownerGalleryExhausted: args.media.ownerGalleryExhausted,
21767
+ galleryExhausted: args.media.exhausted,
21768
+ stopReason: args.media.stopReason,
21769
+ notes: [
21770
+ "origin=owner is backed by the Google Maps By owner gallery or an exact media-key match to it.",
21771
+ "origin=other means absent from an exhausted owner gallery; it can include customer, review, Street View, Google, or unattributed media.",
21772
+ "Google-hosted photo files use a bounded s2048 rendition when that CDN form is available; each file is still subject to byte and signature validation."
21773
+ ]
21774
+ };
21775
+ files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
21776
+ files.push({ path: "images.jsonl", content: Buffer.from(cleanImages.map((image) => JSON.stringify(image)).join("\n") + "\n") });
21777
+ const archive = await zipBuffer(files);
21778
+ const id = randomBytes2(6).toString("hex");
21779
+ const pointer = await createPrivateArtifact({
21780
+ policy: policy(),
21781
+ ownerId: args.ownerId,
21782
+ artifactKey: `${id}.zip`,
21783
+ createdAt: new Date(args.extractedAt),
21784
+ filename: `${slug(args.businessName)}-maps-images.zip`,
21785
+ contentType: "application/zip",
21786
+ content: archive
21787
+ });
21788
+ const localPath = token() ? null : join6(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir2(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
21789
+ return { media: args.media, artifact: { ...pointer, localPath } };
21790
+ }
21791
+
21792
+ // src/api/maps-routes.ts
20667
21793
  function mapsErrorResponse(c, err, errorCode) {
20668
21794
  const msg = err instanceof Error ? err.message : String(err);
20669
21795
  const rotationError = err instanceof MapsSearchRotationError ? err : null;
@@ -20782,6 +21908,30 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
20782
21908
  result.reviews = [];
20783
21909
  }
20784
21910
  }
21911
+ if (parsed.data.includeImages && result.media?.images.length > 0) {
21912
+ try {
21913
+ const packaged = await packageMapsMedia({
21914
+ ownerId: String(user.id),
21915
+ businessName: result.name ?? parsed.data.businessName,
21916
+ placeUrl: result.placeUrl,
21917
+ extractedAt: result.extractedAt,
21918
+ media: result.media,
21919
+ maxInlineImages: parsed.data.maxInlineImages
21920
+ });
21921
+ result.media = packaged.media;
21922
+ result.media.artifact = packaged.artifact;
21923
+ } catch (error) {
21924
+ console.warn("[maps/place] image artifact packaging failed:", error instanceof Error ? error.message : String(error));
21925
+ result.media.warnings.push("The image manifest was collected, but the downloadable ZIP artifact could not be created.");
21926
+ }
21927
+ }
21928
+ const logResult = result.media ? {
21929
+ ...result,
21930
+ media: {
21931
+ ...result.media,
21932
+ images: result.media.images.map(({ inlinePreview: _preview, ...image }) => image)
21933
+ }
21934
+ } : result;
20785
21935
  await logRequestEvent({
20786
21936
  userId: user.id,
20787
21937
  source: "maps_place",
@@ -20789,7 +21939,7 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
20789
21939
  query: `${parsed.data.businessName} ${parsed.data.location}`,
20790
21940
  location: parsed.data.location,
20791
21941
  resultCount: reviewCount,
20792
- result
21942
+ result: logResult
20793
21943
  });
20794
21944
  return c.json(result);
20795
21945
  } catch (err) {
@@ -21729,7 +22879,7 @@ function getLocalSourcebookContract(category) {
21729
22879
  }
21730
22880
  async function prepareLocalSourcebookWrite(input) {
21731
22881
  const parsed = DirectoryIdentitySchema.parse(input);
21732
- const slug2 = parsed.slug ?? parsed.businessName.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
22882
+ const slug4 = parsed.slug ?? parsed.businessName.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
21733
22883
  const categoryTag = CATEGORY_TAG[parsed.category];
21734
22884
  const candidates = [
21735
22885
  { tag: categoryTag, central: true, reusable: true },
@@ -21737,12 +22887,12 @@ async function prepareLocalSourcebookWrite(input) {
21737
22887
  ...parsed.tags.filter((tag) => tag !== categoryTag).map((tag) => ({ tag }))
21738
22888
  ];
21739
22889
  const tagResolutions = await resolveLocalSourcebookTags(candidates);
21740
- const publicUrls = localSourcebookPublicUrls(parsed.category, parsed.state, slug2);
22890
+ const publicUrls = localSourcebookPublicUrls(parsed.category, parsed.state, slug4);
21741
22891
  const schemaOrgType = parsed.schemaOrgType ?? defaultLocalSourcebookSchemaType(parsed.category);
21742
22892
  return {
21743
22893
  ok: true,
21744
- route: { category: parsed.category, state: parsed.state, slug: slug2, ...publicUrls },
21745
- proposedWrite: { ...parsed, schemaOrgType, slug: slug2, tags: tagResolutions.filter((item) => item.action === "reuse").map((item) => item.tag) },
22894
+ route: { category: parsed.category, state: parsed.state, slug: slug4, ...publicUrls },
22895
+ proposedWrite: { ...parsed, schemaOrgType, slug: slug4, tags: tagResolutions.filter((item) => item.action === "reuse").map((item) => item.tag) },
21746
22896
  contract: getLocalSourcebookContract(parsed.category),
21747
22897
  tagResolutions,
21748
22898
  instructions: [
@@ -21807,7 +22957,7 @@ async function persistLocalSourcebookTags(submissionId, userId, tags, decisions
21807
22957
  }
21808
22958
 
21809
22959
  // src/api/local-sourcebook-routes.ts
21810
- var slug = (value) => value.toLowerCase().trim().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
22960
+ var slug2 = (value) => value.toLowerCase().trim().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
21811
22961
  var SubmitSchema = z26.object({
21812
22962
  category: z26.enum(LOCAL_SOURCEBOOK_CATEGORIES),
21813
22963
  state: z26.string().trim().length(2).transform((value) => value.toLowerCase()),
@@ -21878,7 +23028,7 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
21878
23028
  if (!idempotencyKey3) {
21879
23029
  return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook capture." }, 400);
21880
23030
  }
21881
- const listingSlug = identity.slug ?? slug(identity.businessName);
23031
+ const listingSlug = identity.slug ?? slug2(identity.businessName);
21882
23032
  const publicUrls = localSourcebookPublicUrls(identity.category, identity.state, listingSlug);
21883
23033
  const charged = await chargeListingWrite({ surface: "local_sourcebook", kind: "create", userId: Number(user.id), idempotencyKey: idempotencyKey3 });
21884
23034
  if ("insufficient" in charged) return c.json(charged.insufficient, 402);
@@ -21929,7 +23079,7 @@ localSourcebookApp.post("/submissions", createApiKeyAuth(), async (c) => {
21929
23079
  if (!parsed.success) return c.json({ error: "invalid_submission", issues: parsed.error.issues }, 400);
21930
23080
  const input = parsed.data;
21931
23081
  if (!input.idempotencyKey) return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook listing." }, 400);
21932
- const listingSlug = input.slug ?? slug(input.businessName);
23082
+ const listingSlug = input.slug ?? slug2(input.businessName);
21933
23083
  const schemaOrgType = input.schemaOrgType ?? defaultLocalSourcebookSchemaType(input.category);
21934
23084
  const publicUrls = localSourcebookPublicUrls(input.category, input.state, listingSlug);
21935
23085
  let billing = null;
@@ -22986,7 +24136,7 @@ import { createHash as createHash8, randomUUID as randomUUID9 } from "crypto";
22986
24136
 
22987
24137
  // src/serp-intelligence/page-snapshot-extractor.ts
22988
24138
  import { createHash as createHash7 } from "crypto";
22989
- import pLimit4 from "p-limit";
24139
+ import pLimit5 from "p-limit";
22990
24140
  var DEFAULT_TIMEOUT_MS = 15e3;
22991
24141
  var DEFAULT_MAX_CONCURRENCY = 2;
22992
24142
  var DEFAULT_MAX_CONTENT_CHARS = 25e4;
@@ -23267,7 +24417,7 @@ async function capturePageSnapshot(target, options = {}) {
23267
24417
  async function capturePageSnapshots(targets, options = {}) {
23268
24418
  const timeoutMs = normalizeTimeoutMs(options.timeoutMs);
23269
24419
  const maxConcurrency = normalizeMaxConcurrency(options.maxConcurrency);
23270
- const limit = pLimit4(maxConcurrency);
24420
+ const limit = pLimit5(maxConcurrency);
23271
24421
  const pageSnapshotArtifacts = await Promise.all(
23272
24422
  targets.map((target) => limit(() => capturePageSnapshot(target, { ...options, timeoutMs })))
23273
24423
  );
@@ -24010,7 +25160,50 @@ function folderInventory(result) {
24010
25160
  }
24011
25161
  function enrichMemoryResult(toolName, result) {
24012
25162
  if (toolName !== "listTool" || result.ok !== true) return result;
24013
- return { ...result, folders: folderInventory(result) };
25163
+ if (result.scope === "account") return result;
25164
+ return { ...result, scope: "vault", folders: folderInventory(result) };
25165
+ }
25166
+ async function buildAccountMemoryInventory(args, listOne) {
25167
+ const filters = { ...args };
25168
+ delete filters.allVaults;
25169
+ delete filters.vault;
25170
+ const initial = enrichMemoryResult("listTool", await listOne(filters));
25171
+ if (initial.ok !== true) return initial;
25172
+ const vaults = Array.isArray(initial.vaults) ? initial.vaults.filter((vault) => typeof vault === "string") : [];
25173
+ const initialVault = typeof initial.vault === "string" ? initial.vault : null;
25174
+ if (!vaults.length && initialVault) vaults.push(initialVault);
25175
+ const vaultInventories = [];
25176
+ for (const vault of vaults) {
25177
+ const listed = vault === initialVault ? initial : enrichMemoryResult("listTool", await listOne({ ...filters, vault }));
25178
+ if (listed.ok !== true) {
25179
+ return {
25180
+ ok: false,
25181
+ scope: "account",
25182
+ partial: true,
25183
+ vaults,
25184
+ vaultInventories,
25185
+ error: typeof listed.error === "string" ? `Could not list vault ${vault}: ${listed.error}` : `Could not list vault ${vault}`
25186
+ };
25187
+ }
25188
+ const notes = Array.isArray(listed.notes) ? listed.notes : [];
25189
+ const folders = Array.isArray(listed.folders) ? listed.folders : [];
25190
+ vaultInventories.push({
25191
+ vault,
25192
+ noteCount: notes.length,
25193
+ folderCount: folders.length,
25194
+ notes,
25195
+ folders
25196
+ });
25197
+ }
25198
+ return {
25199
+ ok: true,
25200
+ scope: "account",
25201
+ partial: false,
25202
+ vaults,
25203
+ totalNotes: vaultInventories.reduce((sum, inventory) => sum + Number(inventory.noteCount), 0),
25204
+ totalFolders: vaultInventories.reduce((sum, inventory) => sum + Number(inventory.folderCount), 0),
25205
+ vaultInventories
25206
+ };
24014
25207
  }
24015
25208
  var MemoryMcpToolExecutor = class {
24016
25209
  baseUrl;
@@ -24021,22 +25214,25 @@ var MemoryMcpToolExecutor = class {
24021
25214
  }
24022
25215
  async callMemoryTool(toolName, args) {
24023
25216
  try {
24024
- const res = await fetch(`${this.baseUrl}/memory/mcp-call`, {
24025
- method: "POST",
24026
- headers: {
24027
- "content-type": "application/json",
24028
- "x-api-key": this.apiKey
24029
- },
24030
- body: JSON.stringify({ toolName, args })
24031
- });
24032
- const data = await res.json().catch(() => null);
24033
- if (!res.ok) {
24034
- const message = data?.error ?? `memory ${toolName} failed (HTTP ${res.status})`;
24035
- return { content: [{ type: "text", text: message }], isError: true };
24036
- }
25217
+ const callOne = async (input) => {
25218
+ const res = await fetch(`${this.baseUrl}/memory/mcp-call`, {
25219
+ method: "POST",
25220
+ headers: {
25221
+ "content-type": "application/json",
25222
+ "x-api-key": this.apiKey
25223
+ },
25224
+ body: JSON.stringify({ toolName, args: input })
25225
+ });
25226
+ const data2 = await res.json().catch(() => null);
25227
+ if (!res.ok) {
25228
+ throw new Error(data2?.error ?? `memory ${toolName} failed (HTTP ${res.status})`);
25229
+ }
25230
+ return data2 ?? { ok: false, error: `memory ${toolName} returned no result` };
25231
+ };
25232
+ const data = toolName === "listTool" && args.allVaults === true ? await buildAccountMemoryInventory(args, callOne) : await callOne(args);
24037
25233
  const result = enrichMemoryResult(
24038
25234
  toolName,
24039
- data ?? { ok: false, error: `memory ${toolName} returned no result` }
25235
+ data
24040
25236
  );
24041
25237
  return {
24042
25238
  content: [{ type: "text", text: JSON.stringify(result) }],
@@ -24062,15 +25258,15 @@ var ALLOWED_IMAGE_TYPES = /* @__PURE__ */ new Map([
24062
25258
  ["image/webp", "webp"],
24063
25259
  ["image/gif", "gif"]
24064
25260
  ]);
24065
- function token() {
25261
+ function token2() {
24066
25262
  return process.env.IMAGE_SOURCE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
24067
25263
  }
24068
- function policy() {
25264
+ function policy2() {
24069
25265
  return {
24070
25266
  prefix: IMAGE_SOURCE_ARTIFACT_PREFIX,
24071
25267
  artifactTtlMs: IMAGE_SOURCE_ARTIFACT_TTL_MS,
24072
25268
  downloadTtlMs: IMAGE_SOURCE_DOWNLOAD_TTL_MS,
24073
- token: token()
25269
+ token: token2()
24074
25270
  };
24075
25271
  }
24076
25272
  function imageSourceArtifactOwnerId(artifactId) {
@@ -24084,7 +25280,7 @@ async function createImageSourceArtifact(args) {
24084
25280
  }
24085
25281
  const id = randomUUID10().replaceAll("-", "");
24086
25282
  return createPrivateArtifact({
24087
- policy: policy(),
25283
+ policy: policy2(),
24088
25284
  ownerId: args.ownerId,
24089
25285
  artifactKey: `${id}.${extension}`,
24090
25286
  createdAt: args.createdAt ?? /* @__PURE__ */ new Date(),
@@ -24096,7 +25292,7 @@ async function createImageSourceArtifact(args) {
24096
25292
  async function readOwnedImageSourceArtifact(args) {
24097
25293
  if (imageSourceArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
24098
25294
  return readPrivateArtifactBuffer({
24099
- policy: policy(),
25295
+ policy: policy2(),
24100
25296
  artifactId: args.artifactId,
24101
25297
  maxBytes: IMAGE_SOURCE_MAX_BYTES
24102
25298
  });
@@ -24253,10 +25449,8 @@ function registerReadCutovers() {
24253
25449
  return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
24254
25450
  });
24255
25451
  registerCutover("listTool", async (input, apiKey) => {
24256
- const result = enrichMemoryResult(
24257
- "listTool",
24258
- await listTool.execute({ ...input, apiKey }, {}) ?? {}
24259
- );
25452
+ const listOne = async (args) => await listTool.execute({ ...args, apiKey }, {}) ?? {};
25453
+ const result = input.allVaults === true ? await buildAccountMemoryInventory(input, listOne) : enrichMemoryResult("listTool", await listOne(input));
24260
25454
  return { content: [{ type: "text", text: JSON.stringify(result) }], structuredContent: result, isError: false };
24261
25455
  });
24262
25456
  registerCutover("searchTool", async (input, apiKey) => {
@@ -24684,8 +25878,8 @@ function protectedResourceMetadata() {
24684
25878
  function wwwAuthenticateValue() {
24685
25879
  return `Bearer resource_metadata="${oauthResourceBaseUrl()}/.well-known/oauth-protected-resource"`;
24686
25880
  }
24687
- function isBearerJwt(token4) {
24688
- return !token4.startsWith("sk_") && token4.split(".").length === 3;
25881
+ function isBearerJwt(token6) {
25882
+ return !token6.startsWith("sk_") && token6.split(".").length === 3;
24689
25883
  }
24690
25884
  var jwksSingleton = null;
24691
25885
  function getJwks() {
@@ -24693,8 +25887,8 @@ function getJwks() {
24693
25887
  jwksSingleton = createRemoteJWKSet(new URL(`${oauthIssuer()}/.well-known/jwks.json`));
24694
25888
  return jwksSingleton;
24695
25889
  }
24696
- async function verifyScraperOAuthJwt(token4) {
24697
- const { payload } = await jwtVerify(token4, getJwks(), {
25890
+ async function verifyScraperOAuthJwt(token6) {
25891
+ const { payload } = await jwtVerify(token6, getJwks(), {
24698
25892
  issuer: oauthIssuer(),
24699
25893
  audience: oauthResourceUrl()
24700
25894
  });
@@ -27417,7 +28611,7 @@ async function resolveUser3(customerId, emailFallback) {
27417
28611
  // src/api/oauth-routes.ts
27418
28612
  import { Hono as Hono24 } from "hono";
27419
28613
  import { getCookie, setCookie } from "hono/cookie";
27420
- import { createHash as createHash9, randomBytes as randomBytes2, randomUUID as randomUUID13 } from "crypto";
28614
+ import { createHash as createHash9, randomBytes as randomBytes3, randomUUID as randomUUID13 } from "crypto";
27421
28615
  import { importPKCS8, exportJWK, SignJWT } from "jose";
27422
28616
  var ISSUER = "https://mcpscraper.dev";
27423
28617
  var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
@@ -27516,9 +28710,9 @@ function negotiateScope(requested, user, resource) {
27516
28710
  if (operator && !unique3.includes("memory.admin")) unique3.push("memory.admin");
27517
28711
  return unique3.join(" ");
27518
28712
  }
27519
- async function sessionUser(token4) {
27520
- if (!token4) return null;
27521
- const userId = verifySession(token4);
28713
+ async function sessionUser(token6) {
28714
+ if (!token6) return null;
28715
+ const userId = verifySession(token6);
27522
28716
  if (!userId) return null;
27523
28717
  const user = await withOAuthDatabaseDeadline("session-user", getUserById(userId));
27524
28718
  return user ?? null;
@@ -27703,7 +28897,7 @@ oauthApp.post("/oauth/register", async (c) => {
27703
28897
  }
27704
28898
  }
27705
28899
  const clientName = typeof body.client_name === "string" ? body.client_name : null;
27706
- const clientId = `client_${randomBytes2(16).toString("hex")}`;
28900
+ const clientId = `client_${randomBytes3(16).toString("hex")}`;
27707
28901
  await withOAuthDatabaseDeadline("register-client", registerClient(clientId, redirectUris, clientName));
27708
28902
  console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
27709
28903
  return c.json({
@@ -27778,7 +28972,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
27778
28972
  if (action2 === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
27779
28973
  if (action2 !== "approve") return c.text("unsupported action", 400);
27780
28974
  const scope = negotiateScope(p.scope, user, p.resource);
27781
- const code = `code_${randomBytes2(32).toString("base64url")}`;
28975
+ const code = `code_${randomBytes3(32).toString("base64url")}`;
27782
28976
  const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
27783
28977
  await withOAuthDatabaseDeadline("put-authorization-code", putCode({
27784
28978
  code,
@@ -27829,7 +29023,7 @@ oauthApp.post("/oauth/token", async (c) => {
27829
29023
  const plan = user ? resolvePlan(user) : "free";
27830
29024
  const audience = record.resource ?? RESOURCE();
27831
29025
  const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
27832
- const refreshToken = `rt_${randomBytes2(40).toString("base64url")}`;
29026
+ const refreshToken = `rt_${randomBytes3(40).toString("base64url")}`;
27833
29027
  await withOAuthDatabaseDeadline("put-refresh-token", putRefresh({
27834
29028
  refresh_token: refreshToken,
27835
29029
  client_id: clientId,
@@ -27859,7 +29053,7 @@ oauthApp.post("/oauth/token", async (c) => {
27859
29053
  const plan = user ? resolvePlan(user) : "free";
27860
29054
  const audience = record.resource ?? RESOURCE();
27861
29055
  const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
27862
- const nextRefresh = `rt_${randomBytes2(40).toString("base64url")}`;
29056
+ const nextRefresh = `rt_${randomBytes3(40).toString("base64url")}`;
27863
29057
  await withOAuthDatabaseDeadline("rotate-refresh-token", rotateRefresh(refreshToken, {
27864
29058
  refresh_token: nextRefresh,
27865
29059
  client_id: record.client_id,
@@ -27877,35 +29071,35 @@ oauthApp.post("/oauth/token", async (c) => {
27877
29071
  // src/api/chat-routes.ts
27878
29072
  import { Hono as Hono25 } from "hono";
27879
29073
  var chatApp = new Hono25();
27880
- function tokenOk(token4) {
27881
- return !!token4 && token4.startsWith("mk_") && token4.length > 10;
29074
+ function tokenOk(token6) {
29075
+ return !!token6 && token6.startsWith("mk_") && token6.length > 10;
27882
29076
  }
27883
29077
  function jsonError(message, status = 400) {
27884
29078
  return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
27885
29079
  }
27886
29080
  chatApp.get("/api/channels", async (c) => {
27887
- const token4 = c.req.query("token");
27888
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
27889
- const r = await memoryCall("listVaultsTool", {}, token4);
29081
+ const token6 = c.req.query("token");
29082
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
29083
+ const r = await memoryCall("listVaultsTool", {}, token6);
27890
29084
  if (!r.ok) return c.json(r);
27891
29085
  const vaults = (r.vaults ?? []).filter((v) => v.kind === "channel");
27892
29086
  return c.json({ ok: true, channels: vaults });
27893
29087
  });
27894
29088
  chatApp.get("/api/messages", async (c) => {
27895
- const token4 = c.req.query("token");
27896
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
29089
+ const token6 = c.req.query("token");
29090
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
27897
29091
  const vault = c.req.query("vault");
27898
29092
  if (!vault) return jsonError("vault is required");
27899
29093
  const parentMessageId = c.req.query("parentMessageId") || void 0;
27900
- const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId }, token4);
29094
+ const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId }, token6);
27901
29095
  return c.json(r);
27902
29096
  });
27903
29097
  chatApp.get("/api/poll", async (c) => {
27904
- const token4 = c.req.query("token");
27905
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
29098
+ const token6 = c.req.query("token");
29099
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
27906
29100
  const vault = c.req.query("vault");
27907
29101
  if (!vault) return jsonError("vault is required");
27908
- const r = await memoryCall("pollChannelTool", { vault }, token4);
29102
+ const r = await memoryCall("pollChannelTool", { vault }, token6);
27909
29103
  return c.json(r);
27910
29104
  });
27911
29105
  function parseAttachNote(v) {
@@ -27930,12 +29124,12 @@ chatApp.post("/api/reply", async (c) => {
27930
29124
  return c.json(r);
27931
29125
  });
27932
29126
  chatApp.get("/api/notes-own", async (c) => {
27933
- const token4 = c.req.query("token");
27934
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
27935
- const vaultsRes = await memoryCall("listVaultsTool", {}, token4);
29127
+ const token6 = c.req.query("token");
29128
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
29129
+ const vaultsRes = await memoryCall("listVaultsTool", {}, token6);
27936
29130
  if (!vaultsRes.ok) return c.json(vaultsRes);
27937
29131
  const ownVaults = (vaultsRes.vaults ?? []).filter((v) => v.role === "owner" && v.kind === "notes");
27938
- const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle }, token4)));
29132
+ const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle }, token6)));
27939
29133
  const notes = ownVaults.flatMap((v, i) => {
27940
29134
  const r = lists[i];
27941
29135
  return r.ok ? (r.notes ?? []).map((n) => ({ vault: v.handle, path: n.path, title: n.title })) : [];
@@ -27943,12 +29137,12 @@ chatApp.get("/api/notes-own", async (c) => {
27943
29137
  return c.json({ ok: true, notes });
27944
29138
  });
27945
29139
  chatApp.get("/api/message-note", async (c) => {
27946
- const token4 = c.req.query("token");
27947
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
29140
+ const token6 = c.req.query("token");
29141
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
27948
29142
  const vault = c.req.query("vault");
27949
29143
  const messageId = c.req.query("messageId");
27950
29144
  if (!vault || !messageId) return jsonError("vault and messageId are required");
27951
- const r = await memoryCall("getMessageNoteTool", { vault, messageId }, token4);
29145
+ const r = await memoryCall("getMessageNoteTool", { vault, messageId }, token6);
27952
29146
  return c.json(r);
27953
29147
  });
27954
29148
  chatApp.post("/api/react", async (c) => {
@@ -27959,17 +29153,17 @@ chatApp.post("/api/react", async (c) => {
27959
29153
  return c.json(r);
27960
29154
  });
27961
29155
  chatApp.get("/api/mentions", async (c) => {
27962
- const token4 = c.req.query("token");
27963
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
27964
- const r = await memoryCall("myMentionsTool", {}, token4);
29156
+ const token6 = c.req.query("token");
29157
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
29158
+ const r = await memoryCall("myMentionsTool", {}, token6);
27965
29159
  return c.json(r);
27966
29160
  });
27967
29161
  chatApp.get("/api/members", async (c) => {
27968
- const token4 = c.req.query("token");
27969
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
29162
+ const token6 = c.req.query("token");
29163
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
27970
29164
  const vault = c.req.query("vault");
27971
29165
  if (!vault) return jsonError("vault is required");
27972
- const r = await memoryCall("listChannelMembersTool", { vault }, token4);
29166
+ const r = await memoryCall("listChannelMembersTool", { vault }, token6);
27973
29167
  return c.json(r);
27974
29168
  });
27975
29169
  chatApp.post("/api/channel", async (c) => {
@@ -28005,13 +29199,13 @@ chatApp.post("/api/member-remove", async (c) => {
28005
29199
  return c.json(r);
28006
29200
  });
28007
29201
  chatApp.get("/:token", (c) => {
28008
- const token4 = c.req.param("token");
28009
- if (!tokenOk(token4)) {
29202
+ const token6 = c.req.param("token");
29203
+ if (!tokenOk(token6)) {
28010
29204
  return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This chat link isn't valid.</h2><p>Ask the AI to run get-chat-link again, or revoke-chat-link then get-chat-link if you think it was compromised.</p></body></html>`, 401);
28011
29205
  }
28012
- return c.html(renderChatPage(token4));
29206
+ return c.html(renderChatPage(token6));
28013
29207
  });
28014
- function renderChatPage(token4) {
29208
+ function renderChatPage(token6) {
28015
29209
  return `<!doctype html>
28016
29210
  <html>
28017
29211
  <head>
@@ -28196,7 +29390,7 @@ function renderChatPage(token4) {
28196
29390
  </div>
28197
29391
 
28198
29392
  <script>
28199
- const TOKEN = ${JSON.stringify(token4)};
29393
+ const TOKEN = ${JSON.stringify(token6)};
28200
29394
  const state = { channels: [], activeVault: null, activeName: null, members: [], threadParent: null, pollTimer: null, pendingAttach: null, threadPendingAttach: null, ownNotes: null, pickerTarget: 'main' };
28201
29395
 
28202
29396
  function api(path, opts) {
@@ -28566,8 +29760,8 @@ var MAX_VAULT_RECORDS = 200;
28566
29760
  var MAX_MEMORY_READ_CONCURRENCY = 4;
28567
29761
  var MAX_MEMORY_READ_ATTEMPTS = 3;
28568
29762
  var vaultApp = new Hono26();
28569
- function validToken(token4) {
28570
- return !!token4 && token4.startsWith("mk_") && token4.length > 20 && token4.length < 256;
29763
+ function validToken(token6) {
29764
+ return !!token6 && token6.startsWith("mk_") && token6.length > 20 && token6.length < 256;
28571
29765
  }
28572
29766
  function isVault(value) {
28573
29767
  return typeof value === "string" && VAULTS.includes(value);
@@ -28693,11 +29887,11 @@ function shouldRetryMemoryRead(error) {
28693
29887
  function sleep(ms) {
28694
29888
  return new Promise((resolve2) => setTimeout(resolve2, ms));
28695
29889
  }
28696
- async function readMemory(tool, args, token4) {
28697
- let result = await memoryCall(tool, args, token4);
29890
+ async function readMemory(tool, args, token6) {
29891
+ let result = await memoryCall(tool, args, token6);
28698
29892
  for (let attempt = 1; !result.ok && attempt < MAX_MEMORY_READ_ATTEMPTS && shouldRetryMemoryRead(result.error); attempt += 1) {
28699
29893
  await sleep(attempt * 150);
28700
- result = await memoryCall(tool, args, token4);
29894
+ result = await memoryCall(tool, args, token6);
28701
29895
  }
28702
29896
  return result;
28703
29897
  }
@@ -28714,27 +29908,27 @@ async function mapWithConcurrency3(items, limit, map) {
28714
29908
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
28715
29909
  return values;
28716
29910
  }
28717
- async function readVault(vault, token4) {
28718
- const list = await readMemory("listTool", { vault }, token4);
29911
+ async function readVault(vault, token6) {
29912
+ const list = await readMemory("listTool", { vault }, token6);
28719
29913
  if (!list.ok) throw new Error(list.error ?? `unable to read ${vault}`);
28720
29914
  const noteRefs = (list.notes ?? []).slice(0, MAX_VAULT_RECORDS);
28721
29915
  const notes = await mapWithConcurrency3(
28722
29916
  noteRefs,
28723
29917
  MAX_MEMORY_READ_CONCURRENCY,
28724
- ({ path: path5 }) => readMemory("getTool", { vault, path: path5 }, token4)
29918
+ ({ path: path5 }) => readMemory("getTool", { vault, path: path5 }, token6)
28725
29919
  );
28726
29920
  const failed = notes.find((result) => !result.ok);
28727
29921
  if (failed && !failed.ok) throw new Error(failed.error ?? `unable to read ${vault} record`);
28728
29922
  return notes.filter((result) => result.ok && !!result.note).map((result) => workspaceRecord(vault, result.note));
28729
29923
  }
28730
29924
  vaultApp.get("/api/workspace", async (c) => {
28731
- const token4 = c.req.query("token");
28732
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
29925
+ const token6 = c.req.query("token");
29926
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28733
29927
  try {
28734
29928
  const requested = c.req.query("vault");
28735
29929
  const vaults = isVault(requested) ? [requested] : VAULTS;
28736
29930
  const results = [];
28737
- for (const vault of vaults) results.push(await readVault(vault, token4));
29931
+ for (const vault of vaults) results.push(await readVault(vault, token6));
28738
29932
  const records = results.flat().sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
28739
29933
  return c.json({ ok: true, records, limited: results.some((result) => result.length >= 200) });
28740
29934
  } catch (error) {
@@ -28743,8 +29937,8 @@ vaultApp.get("/api/workspace", async (c) => {
28743
29937
  });
28744
29938
  vaultApp.post("/api/records", async (c) => {
28745
29939
  const body = await c.req.json().catch(() => ({}));
28746
- const token4 = typeof body.token === "string" ? body.token : void 0;
28747
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
29940
+ const token6 = typeof body.token === "string" ? body.token : void 0;
29941
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28748
29942
  const vault = body.vault;
28749
29943
  const path5 = stringValue(body.path, 320);
28750
29944
  const title = stringValue(body.title, 320);
@@ -28755,19 +29949,19 @@ vaultApp.post("/api/records", async (c) => {
28755
29949
  const validationError2 = validateNewRecord(vault, normalizedProps);
28756
29950
  if (validationError2) return c.json({ ok: false, error: validationError2 }, 400);
28757
29951
  const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
28758
- const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } }, token4);
29952
+ const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } }, token6);
28759
29953
  return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
28760
29954
  });
28761
29955
  vaultApp.post("/api/records/transition", async (c) => {
28762
29956
  const body = await c.req.json().catch(() => ({}));
28763
- const token4 = typeof body.token === "string" ? body.token : void 0;
28764
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
29957
+ const token6 = typeof body.token === "string" ? body.token : void 0;
29958
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28765
29959
  const vault = typeof body.vault === "string" ? body.vault : "";
28766
29960
  const path5 = typeof body.path === "string" ? body.path : "";
28767
29961
  const status = stringValue(body.status, 80);
28768
29962
  const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
28769
29963
  if (!isVault(vault) || !path5 || !status) return c.json({ ok: false, error: "valid vault, path, and status are required" }, 400);
28770
- const current = await readMemory("getTool", { vault, path: path5 }, token4);
29964
+ const current = await readMemory("getTool", { vault, path: path5 }, token6);
28771
29965
  if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
28772
29966
  const result = await memoryCall("putTool", {
28773
29967
  vault,
@@ -28776,13 +29970,13 @@ vaultApp.post("/api/records/transition", async (c) => {
28776
29970
  content: current.note.content,
28777
29971
  props: { ...current.note.props ?? {}, status },
28778
29972
  baseRevision: baseRevision ?? current.note.revision
28779
- }, token4);
29973
+ }, token6);
28780
29974
  return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
28781
29975
  });
28782
29976
  vaultApp.post("/api/records/update", async (c) => {
28783
29977
  const body = await c.req.json().catch(() => ({}));
28784
- const token4 = typeof body.token === "string" ? body.token : void 0;
28785
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
29978
+ const token6 = typeof body.token === "string" ? body.token : void 0;
29979
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28786
29980
  const vault = body.vault;
28787
29981
  const path5 = stringValue(body.path, 320);
28788
29982
  const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
@@ -28792,7 +29986,7 @@ vaultApp.post("/api/records/update", async (c) => {
28792
29986
  if (!isVault(vault) || !path5 || body.title !== void 0 && !title || body.props !== void 0 && !props) {
28793
29987
  return c.json({ ok: false, error: "valid vault, path, and optional record fields are required" }, 400);
28794
29988
  }
28795
- const current = await readMemory("getTool", { vault, path: path5 }, token4);
29989
+ const current = await readMemory("getTool", { vault, path: path5 }, token6);
28796
29990
  if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
28797
29991
  const nextProps = normalizeRelationshipProps(vault, { ...current.note.props ?? {}, ...props ?? {} });
28798
29992
  const validationError2 = validateNewRecord(vault, nextProps);
@@ -28804,18 +29998,18 @@ vaultApp.post("/api/records/update", async (c) => {
28804
29998
  content: content ?? current.note.content,
28805
29999
  props: nextProps,
28806
30000
  baseRevision: baseRevision ?? current.note.revision
28807
- }, token4);
30001
+ }, token6);
28808
30002
  return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
28809
30003
  });
28810
30004
  vaultApp.delete("/api/records", async (c) => {
28811
30005
  const body = await c.req.json().catch(() => ({}));
28812
- const token4 = typeof body.token === "string" ? body.token : void 0;
28813
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
30006
+ const token6 = typeof body.token === "string" ? body.token : void 0;
30007
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28814
30008
  const vault = body.vault;
28815
30009
  const path5 = stringValue(body.path, 320);
28816
30010
  const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
28817
30011
  if (vault !== "Tasks" || !path5) return c.json({ ok: false, error: "only a valid task record can be deleted" }, 400);
28818
- const current = await readMemory("getTool", { vault, path: path5 }, token4);
30012
+ const current = await readMemory("getTool", { vault, path: path5 }, token6);
28819
30013
  if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
28820
30014
  if (!["task", "milestone"].includes(typeof current.note.props?.type === "string" ? current.note.props.type : defaultType("Tasks"))) {
28821
30015
  return c.json({ ok: false, error: "only task records can be deleted" }, 400);
@@ -28824,13 +30018,13 @@ vaultApp.delete("/api/records", async (c) => {
28824
30018
  vault,
28825
30019
  path: path5,
28826
30020
  baseRevision: baseRevision ?? current.note.revision
28827
- }, token4);
30021
+ }, token6);
28828
30022
  return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
28829
30023
  });
28830
30024
  vaultApp.post("/api/communications/review", async (c) => {
28831
30025
  const body = await c.req.json().catch(() => ({}));
28832
- const token4 = typeof body.token === "string" ? body.token : void 0;
28833
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
30026
+ const token6 = typeof body.token === "string" ? body.token : void 0;
30027
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28834
30028
  const path5 = stringValue(body.path, 320);
28835
30029
  const action2 = stringValue(body.action, 24);
28836
30030
  const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
@@ -28841,7 +30035,7 @@ vaultApp.post("/api/communications/review", async (c) => {
28841
30035
  if (!path5 || !normalizedAction || !["approve", "deny", "edit"].includes(normalizedAction) || body.title !== void 0 && !title) {
28842
30036
  return c.json({ ok: false, error: "valid path, action, and optional draft fields are required" }, 400);
28843
30037
  }
28844
- const current = await readMemory("getTool", { vault: "Communications", path: path5 }, token4);
30038
+ const current = await readMemory("getTool", { vault: "Communications", path: path5 }, token6);
28845
30039
  if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
28846
30040
  if (current.note.props?.type !== "email") return c.json({ ok: false, error: "only email communications can be reviewed" }, 400);
28847
30041
  const reviewed = normalizedAction === "approve" || normalizedAction === "deny";
@@ -28865,13 +30059,13 @@ vaultApp.post("/api/communications/review", async (c) => {
28865
30059
  content: content ?? current.note.content,
28866
30060
  props,
28867
30061
  baseRevision: baseRevision ?? current.note.revision
28868
- }, token4);
30062
+ }, token6);
28869
30063
  return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
28870
30064
  });
28871
30065
  vaultApp.post("/api/activity", async (c) => {
28872
30066
  const body = await c.req.json().catch(() => ({}));
28873
- const token4 = typeof body.token === "string" ? body.token : void 0;
28874
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
30067
+ const token6 = typeof body.token === "string" ? body.token : void 0;
30068
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28875
30069
  const title = stringValue(body.title, 320);
28876
30070
  const content = typeof body.content === "string" && body.content.length <= 2e4 ? body.content : "";
28877
30071
  const props = propsValue(body.props) ?? {};
@@ -28881,7 +30075,7 @@ vaultApp.post("/api/activity", async (c) => {
28881
30075
  return c.json({ ok: false, error: "Activities require at least one canonical People reference in person_refs" }, 400);
28882
30076
  }
28883
30077
  for (const ref of personRefs) {
28884
- const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") }, token4);
30078
+ const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") }, token6);
28885
30079
  if (!person.ok || !person.note || !["person", "organization"].includes(String(person.note.props?.type ?? ""))) {
28886
30080
  return c.json({ ok: false, error: `Activity person reference is not a known People record: ${ref}` }, 400);
28887
30081
  }
@@ -28909,37 +30103,37 @@ vaultApp.post("/api/activity", async (c) => {
28909
30103
  ...props,
28910
30104
  person_refs: personRefs
28911
30105
  }
28912
- }, token4);
30106
+ }, token6);
28913
30107
  return c.json(result, result.ok ? 200 : 502);
28914
30108
  });
28915
30109
 
28916
30110
  // src/api/schedule-routes.ts
28917
30111
  import { Hono as Hono27 } from "hono";
28918
30112
  var scheduleApp = new Hono27();
28919
- function tokenOk2(token4) {
28920
- return !!token4 && token4.startsWith("mk_") && token4.length > 10;
30113
+ function tokenOk2(token6) {
30114
+ return !!token6 && token6.startsWith("mk_") && token6.length > 10;
28921
30115
  }
28922
30116
  function jsonError2(message, status = 400) {
28923
30117
  return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
28924
30118
  }
28925
30119
  scheduleApp.get("/api/status", async (c) => {
28926
- const token4 = c.req.query("token");
28927
- if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
28928
- const r = await memoryCall("getScheduleStatusTool", {}, token4);
30120
+ const token6 = c.req.query("token");
30121
+ if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
30122
+ const r = await memoryCall("getScheduleStatusTool", {}, token6);
28929
30123
  return c.json(r);
28930
30124
  });
28931
30125
  scheduleApp.get("/api/vaults", async (c) => {
28932
- const token4 = c.req.query("token");
28933
- if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
28934
- const r = await memoryCall("listVaultsTool", {}, token4);
30126
+ const token6 = c.req.query("token");
30127
+ if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
30128
+ const r = await memoryCall("listVaultsTool", {}, token6);
28935
30129
  if (!r.ok) return c.json(r);
28936
30130
  const vaults = (r.vaults ?? []).filter((v) => v.kind !== "channel");
28937
30131
  return c.json({ ok: true, vaults });
28938
30132
  });
28939
30133
  scheduleApp.get("/api/actions", async (c) => {
28940
- const token4 = c.req.query("token");
28941
- if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
28942
- const r = await memoryCall("listScheduledActionsTool", {}, token4);
30134
+ const token6 = c.req.query("token");
30135
+ if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
30136
+ const r = await memoryCall("listScheduledActionsTool", {}, token6);
28943
30137
  return c.json(r);
28944
30138
  });
28945
30139
  scheduleApp.post("/api/propose", async (c) => {
@@ -28975,13 +30169,13 @@ scheduleApp.post("/api/actions/:id/delete", async (c) => {
28975
30169
  return c.json(r);
28976
30170
  });
28977
30171
  scheduleApp.get("/:token", (c) => {
28978
- const token4 = c.req.param("token");
28979
- if (!tokenOk2(token4)) {
30172
+ const token6 = c.req.param("token");
30173
+ if (!tokenOk2(token6)) {
28980
30174
  return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This automations link isn't valid.</h2><p>Ask the AI to run get-schedule-link again, or revoke-schedule-link then get-schedule-link if you think it was compromised.</p></body></html>`, 401);
28981
30175
  }
28982
- return c.html(renderSchedulePage(token4));
30176
+ return c.html(renderSchedulePage(token6));
28983
30177
  });
28984
- function renderSchedulePage(token4) {
30178
+ function renderSchedulePage(token6) {
28985
30179
  return `<!doctype html>
28986
30180
  <html>
28987
30181
  <head>
@@ -29079,7 +30273,7 @@ function renderSchedulePage(token4) {
29079
30273
  </div>
29080
30274
 
29081
30275
  <script>
29082
- const TOKEN = ${JSON.stringify(token4)};
30276
+ const TOKEN = ${JSON.stringify(token6)};
29083
30277
  const state = { actions: [], vaults: [] };
29084
30278
 
29085
30279
  function api(path, opts) { return fetch('/schedule' + path, opts).then(r => r.json()); }
@@ -29322,8 +30516,8 @@ function supportThreadToken(customerEmail, normalizedSubject) {
29322
30516
  return shortHash(`${customerEmail.trim().toLowerCase()}
29323
30517
  ${normalizeSupportSubject(normalizedSubject).toLowerCase()}`);
29324
30518
  }
29325
- function supportThreadAddress(token4, receivingDomain) {
29326
- const safeToken = token4.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
30519
+ function supportThreadAddress(token6, receivingDomain) {
30520
+ const safeToken = token6.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
29327
30521
  const domain = receivingDomain.trim().toLowerCase();
29328
30522
  if (!safeToken || !/^[a-z0-9.-]+\.[a-z]{2,}$/i.test(domain)) throw new Error("invalid support thread address");
29329
30523
  return `support+${safeToken}@${domain}`;
@@ -29350,8 +30544,8 @@ function personPath(identity) {
29350
30544
  const local = identity.email.split("@")[0];
29351
30545
  return `contacts/${safePathPart(identity.name === identity.email ? local : identity.name)}-${shortHash(identity.email, 12)}`;
29352
30546
  }
29353
- function issuePath(token4) {
29354
- return `support/threads/${token4}`;
30547
+ function issuePath(token6) {
30548
+ return `support/threads/${token6}`;
29355
30549
  }
29356
30550
  function communicationPath(email) {
29357
30551
  const date = cleanLine(email.created_at, (/* @__PURE__ */ new Date()).toISOString()).slice(0, 10).replace(/[^0-9-]/g, "");
@@ -29535,8 +30729,8 @@ function buildIssueContent(email, customer, personPathValue, threadAddressValue)
29535
30729
  `- Person: ${noteRef("People", personPathValue)}`
29536
30730
  ].join("\n");
29537
30731
  }
29538
- async function findOrCreateIssue(memoryKey, email, customer, personPathValue, token4, threadAddressValue) {
29539
- const path5 = issuePath(token4);
30732
+ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, token6, threadAddressValue) {
30733
+ const path5 = issuePath(token6);
29540
30734
  const existing = await getNote(memoryKey, "Issues", path5);
29541
30735
  if (existing) return existing;
29542
30736
  const subject = normalizeSupportSubject(email.subject);
@@ -29546,7 +30740,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
29546
30740
  path: path5,
29547
30741
  title: `Support thread \u2014 ${subject} \u2014 ${customer.email}`.slice(0, 240),
29548
30742
  content: buildIssueContent(email, customer, personPathValue, threadAddressValue),
29549
- source: `resend:thread:${token4}`,
30743
+ source: `resend:thread:${token6}`,
29550
30744
  type: "user_friction",
29551
30745
  tagCandidates: ISSUE_TAGS,
29552
30746
  props: {
@@ -29555,7 +30749,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
29555
30749
  summary,
29556
30750
  pinned: false,
29557
30751
  source_type: "thread",
29558
- source_ref: `resend:thread:${token4}`,
30752
+ source_ref: `resend:thread:${token6}`,
29559
30753
  related: [],
29560
30754
  related_vault_notes: [noteRef("People", personPathValue)],
29561
30755
  embed: true,
@@ -29570,7 +30764,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
29570
30764
  customer_email: customer.email,
29571
30765
  customer_name: customer.name,
29572
30766
  normalized_subject: subject,
29573
- thread_token: token4,
30767
+ thread_token: token6,
29574
30768
  thread_address: threadAddressValue,
29575
30769
  communication_refs: [],
29576
30770
  message_count: 0,
@@ -29613,7 +30807,7 @@ function buildCommunicationContent(email, customer, direction, personPathValue,
29613
30807
  ...parentPath ? [`- Previous message: [[${parentPath}]]`] : []
29614
30808
  ].join("\n");
29615
30809
  }
29616
- async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue, token4, parentPath) {
30810
+ async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue, token6, parentPath) {
29617
30811
  const path5 = communicationPath(email);
29618
30812
  const existing = await getNote(memoryKey, "Communication", path5);
29619
30813
  if (existing) return existing;
@@ -29652,8 +30846,8 @@ async function captureCommunication(memoryKey, email, customer, direction, perso
29652
30846
  provider: "resend",
29653
30847
  provider_connection_ref: "resend:updates.mcpscraper.dev",
29654
30848
  provider_message_ref: providerMessageRef,
29655
- provider_thread_ref: token4,
29656
- thread_ref: token4,
30849
+ provider_thread_ref: token6,
30850
+ thread_ref: token6,
29657
30851
  issue_ref: issuePathValue,
29658
30852
  sender_email: supportSender(email)?.email ?? email.from,
29659
30853
  recipient_emails: email.to ?? [],
@@ -29713,8 +30907,8 @@ async function updateIssueThread(memoryKey, issuePathValue, email, communication
29713
30907
  if (!result.conflict || attempt === 1) throw new Error(result.error ?? "support thread issue update failed");
29714
30908
  }
29715
30909
  }
29716
- async function getSupportThreadByToken(memoryKey, token4) {
29717
- const safeToken = token4.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
30910
+ async function getSupportThreadByToken(memoryKey, token6) {
30911
+ const safeToken = token6.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
29718
30912
  if (safeToken.length !== 24) return null;
29719
30913
  const path5 = issuePath(safeToken);
29720
30914
  const issue = await getNote(memoryKey, "Issues", path5);
@@ -29742,8 +30936,8 @@ async function captureSupportMessage(memoryKey, email, options) {
29742
30936
  const customer = options.customer ?? supportSender(email);
29743
30937
  if (!customer) throw new Error("support sender does not contain a usable email address");
29744
30938
  const normalizedSubject = options.existingThread?.normalizedSubject ?? normalizeSupportSubject(email.subject);
29745
- const token4 = options.existingThread?.threadToken ?? supportThreadToken(customer.email, normalizedSubject);
29746
- const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(token4, options.receivingDomain);
30939
+ const token6 = options.existingThread?.threadToken ?? supportThreadToken(customer.email, normalizedSubject);
30940
+ const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(token6, options.receivingDomain);
29747
30941
  const person = options.existingThread ? await getNote(memoryKey, "People", options.existingThread.personPath) : await findOrCreatePerson(memoryKey, customer, email.created_at);
29748
30942
  if (!person) throw new Error("support person could not be resolved");
29749
30943
  const issue = options.existingThread?.issue ?? await findOrCreateIssue(
@@ -29751,7 +30945,7 @@ async function captureSupportMessage(memoryKey, email, options) {
29751
30945
  email,
29752
30946
  customer,
29753
30947
  person.path,
29754
- token4,
30948
+ token6,
29755
30949
  threadAddressValue
29756
30950
  );
29757
30951
  const parentPath = typeof issue.props?.last_communication_ref === "string" ? issue.props.last_communication_ref : void 0;
@@ -29762,7 +30956,7 @@ async function captureSupportMessage(memoryKey, email, options) {
29762
30956
  options.direction,
29763
30957
  person.path,
29764
30958
  issue.path,
29765
- token4,
30959
+ token6,
29766
30960
  parentPath
29767
30961
  );
29768
30962
  await updateIssueThread(memoryKey, issue.path, email, communication.path, options.direction, person.path);
@@ -29771,7 +30965,7 @@ async function captureSupportMessage(memoryKey, email, options) {
29771
30965
  communicationPath: communication.path,
29772
30966
  personPath: person.path,
29773
30967
  parentCommunicationPath: parentPath,
29774
- threadToken: token4,
30968
+ threadToken: token6,
29775
30969
  threadAddress: threadAddressValue,
29776
30970
  customerEmail: customer.email,
29777
30971
  normalizedSubject,
@@ -29944,8 +31138,8 @@ resendInboundApp.post("/webhooks", async (c) => {
29944
31138
  threadAddress: captured.threadAddress
29945
31139
  });
29946
31140
  }
29947
- const token4 = extractSupportThreadToken(receivedEvent.data.to ?? [], ...domains);
29948
- const thread = token4 ? await getSupportThreadByToken(memoryKey, token4) : null;
31141
+ const token6 = extractSupportThreadToken(receivedEvent.data.to ?? [], ...domains);
31142
+ const thread = token6 ? await getSupportThreadByToken(memoryKey, token6) : null;
29949
31143
  if (!thread) return c.json({ ok: true, route, ignored: true, reason: "unknown-support-thread" });
29950
31144
  const operator = parseMailbox(destination);
29951
31145
  if (operator && sender.email === operator.email) {
@@ -30231,8 +31425,8 @@ var workshopFieldNotes = [
30231
31425
  function escapeHtml3(value) {
30232
31426
  return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&#039;");
30233
31427
  }
30234
- function workshopFieldNoteUrl(slug2) {
30235
- return `${WORKSHOP_FIELD_NOTES_URL}?article=${encodeURIComponent(slug2)}`;
31428
+ function workshopFieldNoteUrl(slug4) {
31429
+ return `${WORKSHOP_FIELD_NOTES_URL}?article=${encodeURIComponent(slug4)}`;
30236
31430
  }
30237
31431
  function renderWorkshopFieldNotesBody() {
30238
31432
  let activeCategory = "";
@@ -30791,16 +31985,16 @@ import { z as z31 } from "zod";
30791
31985
  // src/editorial-reading-room/render.ts
30792
31986
  import { createHash as createHash11 } from "crypto";
30793
31987
  import { readFileSync as readFileSync3 } from "fs";
30794
- import { dirname as dirname3, join as join6, resolve } from "path";
31988
+ import { dirname as dirname3, join as join7, resolve } from "path";
30795
31989
  import { Marked, Renderer } from "marked";
30796
31990
  var runtimeEntryDir = dirname3(resolve(process.argv[1] || process.cwd()));
30797
31991
  var assetCache = /* @__PURE__ */ new Map();
30798
31992
  function assetPath(name) {
30799
31993
  const candidates = [
30800
- join6(runtimeEntryDir, "editorial-reading-room", "assets", name),
30801
- join6(runtimeEntryDir, "..", "editorial-reading-room", "assets", name),
30802
- join6(process.cwd(), "src", "editorial-reading-room", "assets", name),
30803
- join6(process.cwd(), "dist", "editorial-reading-room", "assets", name)
31994
+ join7(runtimeEntryDir, "editorial-reading-room", "assets", name),
31995
+ join7(runtimeEntryDir, "..", "editorial-reading-room", "assets", name),
31996
+ join7(process.cwd(), "src", "editorial-reading-room", "assets", name),
31997
+ join7(process.cwd(), "dist", "editorial-reading-room", "assets", name)
30804
31998
  ];
30805
31999
  for (const candidate of candidates) {
30806
32000
  try {
@@ -30874,8 +32068,8 @@ function addHeadingIds(html) {
30874
32068
  });
30875
32069
  return { rendered, toc };
30876
32070
  }
30877
- function safeFilename(requested, slug2) {
30878
- const stem = (requested || slug2).replace(/\.html?$/i, "").replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 110) || "editorial-reading-room";
32071
+ function safeFilename(requested, slug4) {
32072
+ const stem = (requested || slug4).replace(/\.html?$/i, "").replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 110) || "editorial-reading-room";
30879
32073
  return `${stem}.html`;
30880
32074
  }
30881
32075
  function escapedJsonForHtml(value) {
@@ -31031,20 +32225,20 @@ ${provenance}`;
31031
32225
  var EDITORIAL_READING_ROOM_ARTIFACT_PREFIX = "editorial-reading-rooms/";
31032
32226
  var EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
31033
32227
  var EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
31034
- function token2() {
32228
+ function token3() {
31035
32229
  return process.env.EDITORIAL_READING_ROOM_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
31036
32230
  }
31037
- function policy2() {
32231
+ function policy3() {
31038
32232
  return {
31039
32233
  prefix: EDITORIAL_READING_ROOM_ARTIFACT_PREFIX,
31040
32234
  artifactTtlMs: EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS,
31041
32235
  downloadTtlMs: EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS,
31042
- token: token2()
32236
+ token: token3()
31043
32237
  };
31044
32238
  }
31045
32239
  function createEditorialReadingRoomArtifact(args) {
31046
32240
  return createPrivateArtifact({
31047
- policy: policy2(),
32241
+ policy: policy3(),
31048
32242
  ownerId: args.ownerId,
31049
32243
  artifactKey: `${args.rendered.sha256.slice(0, 12)}-${args.rendered.filename}`,
31050
32244
  createdAt: args.rendered.generatedAt,
@@ -31055,7 +32249,7 @@ function createEditorialReadingRoomArtifact(args) {
31055
32249
  }
31056
32250
  function renewEditorialReadingRoomArtifactDownload(args) {
31057
32251
  return renewPrivateArtifactDownload({
31058
- policy: policy2(),
32252
+ policy: policy3(),
31059
32253
  ownerId: args.ownerId,
31060
32254
  artifactId: args.artifactId
31061
32255
  });
@@ -31835,7 +33029,7 @@ async function searchCommonsEntities(filters = {}, userId) {
31835
33029
  async function prepareCommonsEntity(input, user) {
31836
33030
  await ensureCommonsSchema();
31837
33031
  const title = optionalText2(input.title, 180) ?? "";
31838
- const slug2 = normalizeSlug(input.slug || title);
33032
+ const slug4 = normalizeSlug(input.slug || title);
31839
33033
  const schemaOrgType = optionalText2(input.schemaOrgType, 80);
31840
33034
  const contract = commonsEntityProfileFor(input.entityType, schemaOrgType);
31841
33035
  const entityType = contract.entityType;
@@ -31843,7 +33037,7 @@ async function prepareCommonsEntity(input, user) {
31843
33037
  const headings = extractMarkdownHeadings(bodyMarkdown);
31844
33038
  const duplicateCandidates = await potentialDuplicateCandidates({
31845
33039
  title,
31846
- slug: slug2,
33040
+ slug: slug4,
31847
33041
  source: normalizeSource(input.source),
31848
33042
  maxCandidates: clampNumber(input.maxCandidates, 1, 20, 8)
31849
33043
  });
@@ -31860,8 +33054,8 @@ async function prepareCommonsEntity(input, user) {
31860
33054
  entityIdPrefix: "TPW-Q",
31861
33055
  userId: user?.id ?? null,
31862
33056
  normalizedEntityType: entityType,
31863
- proposedSlug: slug2 || null,
31864
- publicUrl: slug2 ? publicUrlForSlug(slug2) : null,
33057
+ proposedSlug: slug4 || null,
33058
+ publicUrl: slug4 ? publicUrlForSlug(slug4) : null,
31865
33059
  writePath: "commons_prepare_entity -> commons_validate_entity -> commons_submit_entity"
31866
33060
  },
31867
33061
  contract,
@@ -31896,7 +33090,7 @@ async function validateCommonsEntity(input, user) {
31896
33090
  const warnings = [];
31897
33091
  const title = optionalText2(input.title, 180);
31898
33092
  const description = optionalText2(input.description, 600);
31899
- const slug2 = normalizeSlug(input.slug || title || "");
33093
+ const slug4 = normalizeSlug(input.slug || title || "");
31900
33094
  const contract = commonsEntityProfileFor(input.entityType, input.seo?.schemaOrgType);
31901
33095
  const source = normalizeSource(input.source);
31902
33096
  const featuredImage = normalizeFeaturedImage(input.featuredImage);
@@ -31905,11 +33099,11 @@ async function validateCommonsEntity(input, user) {
31905
33099
  const placeholderSections = unsupportedPlaceholderSections(bodyMarkdown);
31906
33100
  const duplicateCandidates = await potentialDuplicateCandidates({
31907
33101
  title: title ?? "",
31908
- slug: slug2,
33102
+ slug: slug4,
31909
33103
  source,
31910
33104
  maxCandidates: 8
31911
33105
  });
31912
- const existing = slug2 || input.entityId ? await findEntityBySlugOrIdAnyStatus(slug2, input.entityId) : null;
33106
+ const existing = slug4 || input.entityId ? await findEntityBySlugOrIdAnyStatus(slug4, input.entityId) : null;
31913
33107
  try {
31914
33108
  normalizeCommonsClaims(input.claims);
31915
33109
  } catch (error) {
@@ -31917,7 +33111,7 @@ async function validateCommonsEntity(input, user) {
31917
33111
  }
31918
33112
  if (!title || title.length < 3) errors.push("title must be at least 3 characters.");
31919
33113
  if (!description || description.length < 8) errors.push("description must be at least 8 characters.");
31920
- if (!slug2) errors.push("A Commons entity needs a URL slug.");
33114
+ if (!slug4) errors.push("A Commons entity needs a URL slug.");
31921
33115
  if (!featuredImage?.url) errors.push("A featured image is required before a Commons entity can publish.");
31922
33116
  if (!bodyMarkdown.trim() && !source.originalUrl && !source.resolvedUrl) {
31923
33117
  errors.push("A public entity needs article body text or source evidence before publishing.");
@@ -32378,13 +33572,13 @@ async function findProposalByIdempotency(userId, idempotencyKey3) {
32378
33572
  });
32379
33573
  return result.rows[0] ? rowToProposal(result.rows[0]) : null;
32380
33574
  }
32381
- async function findEntityBySlugOrIdAnyStatus(slug2, entityId) {
33575
+ async function findEntityBySlugOrIdAnyStatus(slug4, entityId) {
32382
33576
  const result = entityId ? await getDb().execute({
32383
33577
  sql: "SELECT * FROM commons_entities WHERE slug = ? OR id = ? LIMIT 1",
32384
- args: [slug2, entityId]
33578
+ args: [slug4, entityId]
32385
33579
  }) : await getDb().execute({
32386
33580
  sql: "SELECT * FROM commons_entities WHERE slug = ? LIMIT 1",
32387
- args: [slug2]
33581
+ args: [slug4]
32388
33582
  });
32389
33583
  return result.rows[0] ? rowToEntity(result.rows[0]) : null;
32390
33584
  }
@@ -32576,13 +33770,13 @@ function canonicalizeSourceUrl(value) {
32576
33770
  function normalizeSubmitInput(input) {
32577
33771
  const title = requiredText(input.title, "title", 3, 180);
32578
33772
  const description = requiredText(input.description, "description", 8, 600);
32579
- const slug2 = normalizeSlug(input.slug || title);
32580
- if (!slug2) throw new CommonsRepositoryError("slug_required", "A Commons entity needs a URL slug.");
33773
+ const slug4 = normalizeSlug(input.slug || title);
33774
+ if (!slug4) throw new CommonsRepositoryError("slug_required", "A Commons entity needs a URL slug.");
32581
33775
  return {
32582
33776
  idempotencyKey: requiredText(input.idempotencyKey, "idempotencyKey", 8, 200),
32583
33777
  title,
32584
33778
  description,
32585
- slug: slug2,
33779
+ slug: slug4,
32586
33780
  entityId: normalizeEntityId(input.entityId),
32587
33781
  entityType: normalizeEntityType(input.entityType),
32588
33782
  disambiguationName: optionalText2(input.disambiguationName, 240) ?? void 0,
@@ -33131,13 +34325,13 @@ function normalizeRelatedLinks(relatedLinks) {
33131
34325
  return relatedLinks.map((link) => {
33132
34326
  const title = requiredText(link.title, "relatedLinks.title", 1, 180);
33133
34327
  const url = optionalText2(link.url, 2e3);
33134
- const slug2 = link.slug ? normalizeSlug(link.slug) : void 0;
34328
+ const slug4 = link.slug ? normalizeSlug(link.slug) : void 0;
33135
34329
  const entityId = normalizeEntityId(link.entityId);
33136
- const needsLink = link.needsLink === true || !url && !slug2 && !entityId;
34330
+ const needsLink = link.needsLink === true || !url && !slug4 && !entityId;
33137
34331
  return {
33138
34332
  title,
33139
34333
  url,
33140
- slug: slug2,
34334
+ slug: slug4,
33141
34335
  entityId,
33142
34336
  relationship: optionalText2(link.relationship, 120) ?? void 0,
33143
34337
  summary: optionalText2(link.summary, 300) ?? void 0,
@@ -33368,8 +34562,8 @@ function actorLabel(input, user) {
33368
34562
  function publicBaseUrl() {
33369
34563
  return (process.env.TRANSPARENT_COMMONS_PUBLIC_BASE_URL?.trim() || DEFAULT_COMMONS_BASE_URL).replace(/\/+$/, "");
33370
34564
  }
33371
- function publicUrlForSlug(slug2) {
33372
- return `${publicBaseUrl()}/wiki/${encodeURIComponent(slug2)}`;
34565
+ function publicUrlForSlug(slug4) {
34566
+ return `${publicBaseUrl()}/wiki/${encodeURIComponent(slug4)}`;
33373
34567
  }
33374
34568
  function allocateEntityId() {
33375
34569
  return `TPW-Q${Date.now()}${Math.floor(Math.random() * 1e3).toString().padStart(3, "0")}`;
@@ -33571,11 +34765,11 @@ async function validateCommonsPublication(input, user) {
33571
34765
  } catch (error) {
33572
34766
  errors.push(error instanceof Error ? error.message : String(error));
33573
34767
  }
33574
- const slug2 = normalizeEditionSlug(input.editionSlug || input.edition.site.slug);
33575
- const latest = publication ? await getLatestEdition(publication.id, slug2) : null;
33576
- if (latest && input.baseRevision === void 0) errors.push(`Edition ${slug2} already exists at revision ${latest.revision}; supply baseRevision to edit it.`);
34768
+ const slug4 = normalizeEditionSlug(input.editionSlug || input.edition.site.slug);
34769
+ const latest = publication ? await getLatestEdition(publication.id, slug4) : null;
34770
+ if (latest && input.baseRevision === void 0) errors.push(`Edition ${slug4} already exists at revision ${latest.revision}; supply baseRevision to edit it.`);
33577
34771
  if (latest && input.baseRevision !== void 0 && latest.revision !== input.baseRevision) {
33578
- errors.push(`Edition ${slug2} is revision ${latest.revision}; the proposed edit targeted ${input.baseRevision}.`);
34772
+ errors.push(`Edition ${slug4} is revision ${latest.revision}; the proposed edit targeted ${input.baseRevision}.`);
33579
34773
  }
33580
34774
  }
33581
34775
  }
@@ -34324,7 +35518,7 @@ import Papa2 from "papaparse";
34324
35518
  import {
34325
35519
  createHash as createHash15,
34326
35520
  createHmac as createHmac3,
34327
- randomBytes as randomBytes3,
35521
+ randomBytes as randomBytes4,
34328
35522
  timingSafeEqual as timingSafeEqual2
34329
35523
  } from "crypto";
34330
35524
  import { z as z33 } from "zod";
@@ -34536,7 +35730,7 @@ function dashboardCallbackUrl() {
34536
35730
  }
34537
35731
  async function createThorbitConnectUrl(userId) {
34538
35732
  if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
34539
- const state = randomBytes3(32).toString("base64url");
35733
+ const state = randomBytes4(32).toString("base64url");
34540
35734
  const stateHash = createHash15("sha256").update(state).digest("hex");
34541
35735
  await getAnalyticsPool().query(
34542
35736
  `INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
@@ -34549,8 +35743,8 @@ async function createThorbitConnectUrl(userId) {
34549
35743
  url.searchParams.set("state", state);
34550
35744
  return url.toString();
34551
35745
  }
34552
- function verifyThorbitAssertion(token4) {
34553
- const [encoded, signature] = token4.split(".");
35746
+ function verifyThorbitAssertion(token6) {
35747
+ const [encoded, signature] = token6.split(".");
34554
35748
  if (!encoded || !signature || !bridgeConfigured()) throw new Error("Invalid Thorbit assertion");
34555
35749
  const expected = createHmac3("sha256", bridgeSecret()).update(encoded).digest();
34556
35750
  const actual = Buffer.from(signature, "base64url");
@@ -36028,7 +37222,7 @@ function scheduledArtifactOwnerIdForApiKey(apiKey) {
36028
37222
  }
36029
37223
 
36030
37224
  // src/api/scheduled-run-view-links.ts
36031
- import { createHash as createHash18, randomBytes as randomBytes4, randomUUID as randomUUID17 } from "crypto";
37225
+ import { createHash as createHash18, randomBytes as randomBytes5, randomUUID as randomUUID17 } from "crypto";
36032
37226
  var schemaReady2 = false;
36033
37227
  async function ensureScheduledRunViewLinksSchema() {
36034
37228
  if (schemaReady2) return;
@@ -36081,8 +37275,8 @@ async function ensureScheduledRunViewLinksSchema() {
36081
37275
  `);
36082
37276
  schemaReady2 = true;
36083
37277
  }
36084
- function tokenHash2(token4) {
36085
- return createHash18("sha256").update(token4).digest("hex");
37278
+ function tokenHash2(token6) {
37279
+ return createHash18("sha256").update(token6).digest("hex");
36086
37280
  }
36087
37281
  function mapRow(row) {
36088
37282
  return {
@@ -36102,7 +37296,7 @@ function mapRow(row) {
36102
37296
  async function createScheduledRunViewLink(input) {
36103
37297
  await ensureScheduledRunViewLinksSchema();
36104
37298
  const now = input.now ?? /* @__PURE__ */ new Date();
36105
- const token4 = randomBytes4(32).toString("base64url");
37299
+ const token6 = randomBytes5(32).toString("base64url");
36106
37300
  const record = {
36107
37301
  shareId: randomUUID17(),
36108
37302
  ownerId: input.ownerId,
@@ -36111,7 +37305,7 @@ async function createScheduledRunViewLink(input) {
36111
37305
  title: input.title.slice(0, 300),
36112
37306
  summary: input.summary.slice(0, 2e3),
36113
37307
  rendererKey: input.rendererKey,
36114
- tokenHash: tokenHash2(token4),
37308
+ tokenHash: tokenHash2(token6),
36115
37309
  expiresAt: new Date(now.getTime() + input.expiresInDays * 24 * 60 * 60 * 1e3).toISOString(),
36116
37310
  revokedAt: null,
36117
37311
  createdAt: now.toISOString()
@@ -36136,9 +37330,9 @@ async function createScheduledRunViewLink(input) {
36136
37330
  record.createdAt
36137
37331
  ]
36138
37332
  });
36139
- return { record, token: token4 };
37333
+ return { record, token: token6 };
36140
37334
  }
36141
- async function resolveScheduledRunViewLink(token4, now = /* @__PURE__ */ new Date()) {
37335
+ async function resolveScheduledRunViewLink(token6, now = /* @__PURE__ */ new Date()) {
36142
37336
  await ensureScheduledRunViewLinksSchema();
36143
37337
  const result = await getDb().execute({
36144
37338
  sql: `
@@ -36150,7 +37344,7 @@ async function resolveScheduledRunViewLink(token4, now = /* @__PURE__ */ new Dat
36150
37344
  AND expires_at > ?
36151
37345
  LIMIT 1
36152
37346
  `,
36153
- args: [tokenHash2(token4), now.toISOString()]
37347
+ args: [tokenHash2(token6), now.toISOString()]
36154
37348
  });
36155
37349
  const row = result.rows[0];
36156
37350
  return row ? mapRow(row) : null;
@@ -36565,7 +37759,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
36565
37759
  const artifact = result.data.run.artifactOutputs.find((output) => output.artifactId === body.data.artifactId);
36566
37760
  if (!artifact) return c.json(apiError("The scheduled result artifact was not found.", "not_found"), 404);
36567
37761
  const ownerId = scheduledArtifactOwnerIdForApiKey(user.api_key);
36568
- const { record, token: token4 } = await createScheduledRunViewLink({
37762
+ const { record, token: token6 } = await createScheduledRunViewLink({
36569
37763
  ownerId,
36570
37764
  runId: result.data.run.runId,
36571
37765
  artifactId: artifact.artifactId,
@@ -36580,7 +37774,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
36580
37774
  shareId: record.shareId,
36581
37775
  runId: record.runId,
36582
37776
  artifactId: record.artifactId,
36583
- url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${token4}`,
37777
+ url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${token6}`,
36584
37778
  expiresAt: record.expiresAt,
36585
37779
  createdAt: record.createdAt
36586
37780
  }, 201);
@@ -36606,15 +37800,15 @@ import { createHash as createHash21 } from "crypto";
36606
37800
  var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
36607
37801
  var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
36608
37802
  var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
36609
- function token3() {
37803
+ function token4() {
36610
37804
  return process.env.SCHEDULED_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
36611
37805
  }
36612
- function policy3() {
37806
+ function policy4() {
36613
37807
  return {
36614
37808
  prefix: SCHEDULED_RUN_ARTIFACT_PREFIX,
36615
37809
  artifactTtlMs: null,
36616
37810
  downloadTtlMs: SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS,
36617
- token: token3()
37811
+ token: token4()
36618
37812
  };
36619
37813
  }
36620
37814
  function runStorageSegment(runId) {
@@ -36625,7 +37819,7 @@ async function createScheduledRunArtifact(args) {
36625
37819
  throw new Error("Scheduled artifact HTML must be 2,000,000 bytes or fewer.");
36626
37820
  }
36627
37821
  const stored = await createPrivateArtifact({
36628
- policy: policy3(),
37822
+ policy: policy4(),
36629
37823
  ownerId: args.request.ownerId,
36630
37824
  scopeSegments: [runStorageSegment(args.request.runId)],
36631
37825
  artifactKey: `${args.rendered.sha256.slice(0, 16)}-${args.rendered.filename}`,
@@ -36651,7 +37845,7 @@ async function createScheduledRunArtifact(args) {
36651
37845
  async function readScheduledRunArtifact(args) {
36652
37846
  if (!args.artifactId.startsWith(`${SCHEDULED_RUN_ARTIFACT_PREFIX}${args.ownerId}/`)) return null;
36653
37847
  const window2 = await readPrivateArtifactWindow({
36654
- policy: policy3(),
37848
+ policy: policy4(),
36655
37849
  artifactId: args.artifactId,
36656
37850
  offset: 0,
36657
37851
  maxBytes: SCHEDULED_RUN_ARTIFACT_MAX_BYTES + 1
@@ -37214,6 +38408,282 @@ function shouldRunSiteExtractInBackground(input) {
37214
38408
  return input.background === true || input.downloadImages === true || requestedSiteMaxPages(input.maxPages) > MAX_SYNCHRONOUS_SITE_PAGES;
37215
38409
  }
37216
38410
 
38411
+ // src/api/page-media-artifacts.ts
38412
+ import { createHash as createHash23, randomBytes as randomBytes6 } from "crypto";
38413
+ import { homedir as homedir3 } from "os";
38414
+ import { basename, extname, join as join8 } from "path";
38415
+ import pLimit6 from "p-limit";
38416
+ import { ZipFile as ZipFile2 } from "yazl";
38417
+ var PAGE_MEDIA_ARTIFACT_PREFIX = "page-media/";
38418
+ var PAGE_MEDIA_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
38419
+ var PAGE_MEDIA_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
38420
+ var MAX_FILE_BYTES = 10 * 1024 * 1024;
38421
+ var MAX_ARCHIVE_MEDIA_BYTES = 45 * 1024 * 1024;
38422
+ var MAX_INLINE_IMAGE_BYTES2 = 15e5;
38423
+ var MAX_INLINE_TOTAL_BYTES2 = 25e5;
38424
+ var DOWNLOAD_CONCURRENCY2 = 6;
38425
+ var MAX_REDIRECTS2 = 5;
38426
+ function token5() {
38427
+ return process.env.PAGE_MEDIA_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
38428
+ }
38429
+ function policy5() {
38430
+ return {
38431
+ prefix: PAGE_MEDIA_ARTIFACT_PREFIX,
38432
+ artifactTtlMs: PAGE_MEDIA_ARTIFACT_TTL_MS,
38433
+ downloadTtlMs: PAGE_MEDIA_DOWNLOAD_TTL_MS,
38434
+ token: token5()
38435
+ };
38436
+ }
38437
+ function imageSignature(bytes, contentType) {
38438
+ if (bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return { mimeType: "image/jpeg", extension: "jpg" };
38439
+ if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80 && bytes[2] === 78 && bytes[3] === 71) return { mimeType: "image/png", extension: "png" };
38440
+ if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(0, 4)) === "RIFF" && String.fromCharCode(...bytes.slice(8, 12)) === "WEBP") return { mimeType: "image/webp", extension: "webp" };
38441
+ if (bytes.length >= 6 && ["GIF87a", "GIF89a"].includes(String.fromCharCode(...bytes.slice(0, 6)))) return { mimeType: "image/gif", extension: "gif" };
38442
+ if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(4, 12)).match(/^ftyp(?:avif|avis)$/)) return { mimeType: "image/avif", extension: "avif" };
38443
+ const prefix = Buffer.from(bytes.slice(0, Math.min(bytes.length, 1024))).toString("utf8").replace(/^\uFEFF/, "").trimStart();
38444
+ if ((contentType === "image/svg+xml" || prefix.startsWith("<svg") || /^<\?xml[\s\S]*?<svg/i.test(prefix)) && /<svg[\s>]/i.test(prefix)) {
38445
+ return { mimeType: "image/svg+xml", extension: "svg" };
38446
+ }
38447
+ return null;
38448
+ }
38449
+ function mediaSignature(bytes, contentType, expectedType) {
38450
+ if (expectedType === "image") return imageSignature(bytes, contentType);
38451
+ if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(4, 8)) === "ftyp") return { mimeType: "video/mp4", extension: "mp4" };
38452
+ if (bytes.length >= 4 && bytes[0] === 26 && bytes[1] === 69 && bytes[2] === 223 && bytes[3] === 163) return { mimeType: "video/webm", extension: "webm" };
38453
+ if (bytes.length >= 4 && String.fromCharCode(...bytes.slice(0, 4)) === "OggS") {
38454
+ return expectedType === "audio" ? { mimeType: "audio/ogg", extension: "ogg" } : { mimeType: "video/ogg", extension: "ogv" };
38455
+ }
38456
+ if (expectedType === "audio" && bytes.length >= 3 && String.fromCharCode(...bytes.slice(0, 3)) === "ID3") return { mimeType: "audio/mpeg", extension: "mp3" };
38457
+ if (contentType && typeFromMime(contentType) === expectedType) {
38458
+ const extension = contentType.split("/")[1]?.replace("jpeg", "jpg").replace("mpeg", "mp3").replace("svg+xml", "svg");
38459
+ if (extension && /^[a-z0-9]+$/i.test(extension)) return { mimeType: contentType, extension };
38460
+ }
38461
+ return null;
38462
+ }
38463
+ function imageDimensions(bytes, mimeType) {
38464
+ const buffer = Buffer.from(bytes);
38465
+ if (mimeType === "image/png" && buffer.length >= 24) return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) };
38466
+ if (mimeType === "image/gif" && buffer.length >= 10) return { width: buffer.readUInt16LE(6), height: buffer.readUInt16LE(8) };
38467
+ if (mimeType === "image/webp" && buffer.length >= 30) {
38468
+ const chunk = buffer.toString("ascii", 12, 16);
38469
+ if (chunk === "VP8X") return { width: 1 + buffer.readUIntLE(24, 3), height: 1 + buffer.readUIntLE(27, 3) };
38470
+ if (chunk === "VP8L" && buffer[20] === 47) {
38471
+ const bits = buffer.readUInt32LE(21);
38472
+ return { width: (bits & 16383) + 1, height: (bits >> 14 & 16383) + 1 };
38473
+ }
38474
+ }
38475
+ if (mimeType === "image/jpeg") {
38476
+ let offset = 2;
38477
+ while (offset + 9 < buffer.length) {
38478
+ if (buffer[offset] !== 255) {
38479
+ offset += 1;
38480
+ continue;
38481
+ }
38482
+ const marker = buffer[offset + 1];
38483
+ if (marker === 216 || marker === 217) {
38484
+ offset += 2;
38485
+ continue;
38486
+ }
38487
+ const length = buffer.readUInt16BE(offset + 2);
38488
+ if (length < 2 || offset + length + 2 > buffer.length) break;
38489
+ if ([192, 193, 194, 195, 197, 198, 199, 201, 202, 203, 205, 206, 207].includes(marker)) {
38490
+ return { height: buffer.readUInt16BE(offset + 5), width: buffer.readUInt16BE(offset + 7) };
38491
+ }
38492
+ offset += length + 2;
38493
+ }
38494
+ }
38495
+ if (mimeType === "image/svg+xml") {
38496
+ const source = buffer.toString("utf8", 0, Math.min(buffer.length, 64 * 1024));
38497
+ const tag = source.match(/<svg\b[^>]*>/i)?.[0] ?? "";
38498
+ const width = Number(tag.match(/\bwidth=["']?([\d.]+)/i)?.[1] ?? 0);
38499
+ const height = Number(tag.match(/\bheight=["']?([\d.]+)/i)?.[1] ?? 0);
38500
+ if (width > 0 && height > 0) return { width: Math.round(width), height: Math.round(height) };
38501
+ const viewBox = tag.match(/\bviewBox=["']([^"']+)["']/i)?.[1]?.trim().split(/[\s,]+/).map(Number);
38502
+ if (viewBox?.length === 4 && viewBox[2] > 0 && viewBox[3] > 0) return { width: Math.round(viewBox[2]), height: Math.round(viewBox[3]) };
38503
+ }
38504
+ return null;
38505
+ }
38506
+ async function readBounded2(response) {
38507
+ const declared = Number(response.headers.get("content-length") ?? "");
38508
+ if (Number.isFinite(declared) && declared > MAX_FILE_BYTES) throw new Error("media_too_large");
38509
+ if (!response.body) {
38510
+ const bytes = Buffer.from(await response.arrayBuffer());
38511
+ if (bytes.length > MAX_FILE_BYTES) throw new Error("media_too_large");
38512
+ return bytes;
38513
+ }
38514
+ const reader = response.body.getReader();
38515
+ const chunks = [];
38516
+ let total = 0;
38517
+ try {
38518
+ for (; ; ) {
38519
+ const { done, value } = await reader.read();
38520
+ if (done) break;
38521
+ if (!value) continue;
38522
+ total += value.byteLength;
38523
+ if (total > MAX_FILE_BYTES) throw new Error("media_too_large");
38524
+ chunks.push(Buffer.from(value));
38525
+ }
38526
+ } finally {
38527
+ reader.releaseLock();
38528
+ }
38529
+ return Buffer.concat(chunks);
38530
+ }
38531
+ async function fetchMedia(rawUrl, expectedType) {
38532
+ let target = rawUrl;
38533
+ for (let redirects = 0; redirects <= MAX_REDIRECTS2; redirects += 1) {
38534
+ const checked = await validatePublicHttpUrl(target, { field: "website media URL" });
38535
+ if (checked.error || !checked.parsed) throw new Error("media_url_rejected");
38536
+ const response = await fetch(checked.parsed.href, {
38537
+ redirect: "manual",
38538
+ headers: { Accept: expectedType === "image" ? "image/*" : `${expectedType}/*` },
38539
+ signal: AbortSignal.timeout(15e3)
38540
+ });
38541
+ if ([301, 302, 303, 307, 308].includes(response.status)) {
38542
+ const location2 = response.headers.get("location");
38543
+ if (!location2 || redirects === MAX_REDIRECTS2) throw new Error("media_redirect_rejected");
38544
+ target = new URL(location2, checked.parsed).href;
38545
+ continue;
38546
+ }
38547
+ if (!response.ok) throw new Error(`media_fetch_${response.status}`);
38548
+ const bytes = await readBounded2(response);
38549
+ const declared = response.headers.get("content-type")?.split(";")[0].trim().toLowerCase() ?? null;
38550
+ const detected = mediaSignature(bytes, declared, expectedType);
38551
+ if (!detected || typeFromMime(detected.mimeType) !== expectedType) throw new Error("media_content_invalid");
38552
+ const dimensions = expectedType === "image" ? imageDimensions(bytes, detected.mimeType) : null;
38553
+ return {
38554
+ bytes,
38555
+ ...detected,
38556
+ finalUrl: checked.parsed.href,
38557
+ width: dimensions?.width ?? null,
38558
+ height: dimensions?.height ?? null,
38559
+ sha256: createHash23("sha256").update(bytes).digest("hex")
38560
+ };
38561
+ }
38562
+ throw new Error("media_redirect_rejected");
38563
+ }
38564
+ function zipBuffer2(files) {
38565
+ return new Promise((resolve2, reject) => {
38566
+ const zip = new ZipFile2();
38567
+ const chunks = [];
38568
+ zip.outputStream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
38569
+ zip.outputStream.once("error", reject);
38570
+ zip.outputStream.once("end", () => resolve2(Buffer.concat(chunks)));
38571
+ for (const file of files) zip.addBuffer(file.content, file.path);
38572
+ zip.end();
38573
+ });
38574
+ }
38575
+ function slug3(value) {
38576
+ return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 70) || "website";
38577
+ }
38578
+ function safeOriginalName(rawUrl) {
38579
+ try {
38580
+ const name = basename(new URL(rawUrl).pathname).replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
38581
+ return name.slice(0, 80);
38582
+ } catch {
38583
+ return "";
38584
+ }
38585
+ }
38586
+ function pageMediaArtifactOwnerId(artifactId) {
38587
+ return privateArtifactOwnerId(artifactId, PAGE_MEDIA_ARTIFACT_PREFIX);
38588
+ }
38589
+ async function readOwnedPageMediaArtifactBuffer(args) {
38590
+ if (pageMediaArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
38591
+ return readPrivateArtifactBuffer({ policy: policy5(), artifactId: args.artifactId, maxBytes: 60 * 1024 * 1024 });
38592
+ }
38593
+ async function packagePageMedia(args) {
38594
+ const files = [];
38595
+ const limit = pLimit6(DOWNLOAD_CONCURRENCY2);
38596
+ const digestToFilename = /* @__PURE__ */ new Map();
38597
+ let storedBytes = 0;
38598
+ let inlineBytes = 0;
38599
+ let inlineCount = 0;
38600
+ await Promise.all(args.media.assets.map((asset, position) => limit(async () => {
38601
+ try {
38602
+ const fetched = await fetchMedia(asset.url, asset.type);
38603
+ asset.finalUrl = fetched.finalUrl;
38604
+ asset.mimeType = fetched.mimeType;
38605
+ asset.sizeBytes = fetched.bytes.length;
38606
+ asset.sha256 = fetched.sha256;
38607
+ asset.width = fetched.width ?? asset.width;
38608
+ asset.height = fetched.height ?? asset.height;
38609
+ const duplicate = digestToFilename.get(fetched.sha256);
38610
+ if (duplicate) {
38611
+ asset.filename = duplicate;
38612
+ asset.duplicateOf = duplicate;
38613
+ asset.downloadStatus = "downloaded";
38614
+ asset.downloadError = null;
38615
+ return;
38616
+ }
38617
+ if (storedBytes + fetched.bytes.length > MAX_ARCHIVE_MEDIA_BYTES) throw new Error("archive_byte_limit_reached");
38618
+ storedBytes += fetched.bytes.length;
38619
+ const original = safeOriginalName(fetched.finalUrl);
38620
+ const extension = extname(original);
38621
+ const stem = original ? extension ? original.slice(0, -extension.length) : original : asset.type;
38622
+ const filename = `media/${String(position + 1).padStart(4, "0")}-${slug3(stem)}.${fetched.extension}`;
38623
+ digestToFilename.set(fetched.sha256, filename);
38624
+ files.push({ path: filename, content: fetched.bytes });
38625
+ asset.filename = filename;
38626
+ asset.duplicateOf = null;
38627
+ asset.downloadStatus = "downloaded";
38628
+ asset.downloadError = null;
38629
+ if (asset.type === "image" && inlineCount < args.maxInlineImages && fetched.bytes.length <= MAX_INLINE_IMAGE_BYTES2 && inlineBytes + fetched.bytes.length <= MAX_INLINE_TOTAL_BYTES2) {
38630
+ asset.inlinePreview = { data: fetched.bytes.toString("base64"), mimeType: fetched.mimeType };
38631
+ inlineBytes += fetched.bytes.length;
38632
+ inlineCount += 1;
38633
+ }
38634
+ } catch (error) {
38635
+ asset.finalUrl = null;
38636
+ asset.mimeType = null;
38637
+ asset.sizeBytes = null;
38638
+ asset.sha256 = null;
38639
+ asset.savedPath = null;
38640
+ asset.downloadStatus = "failed";
38641
+ asset.downloadError = error instanceof Error ? error.message : "media_download_failed";
38642
+ asset.inlinePreview = null;
38643
+ }
38644
+ })));
38645
+ const downloaded = args.media.assets.filter((asset) => asset.downloadStatus === "downloaded").length;
38646
+ const failed = args.media.assets.length - downloaded;
38647
+ if (failed > 0) args.media.warnings.push(`${failed} media asset(s) could not be downloaded; media.jsonl contains bounded error codes.`);
38648
+ if (downloaded === 0) return { media: args.media, artifact: null };
38649
+ const cleanAssets = args.media.assets.map(({ inlinePreview: _preview, ...asset }) => asset);
38650
+ const summary = {
38651
+ pageUrl: args.pageUrl,
38652
+ title: args.title,
38653
+ extractedAt: args.extractedAt,
38654
+ completeness: args.media.completeness,
38655
+ exhausted: args.media.exhausted,
38656
+ stopReason: args.media.stopReason,
38657
+ scrollRounds: args.media.scrollRounds,
38658
+ staticFound: args.media.staticFound,
38659
+ renderedFound: args.media.renderedFound,
38660
+ totalFound: args.media.totalFound,
38661
+ retainedAfterFilteringAndVariantCollapse: args.media.assets.length,
38662
+ downloaded,
38663
+ failed,
38664
+ notes: [
38665
+ "The inventory unions static-source and rendered-page discovery.",
38666
+ "complete means rendered scrolling reached three stable document-height checks; it does not claim every interactive control was opened.",
38667
+ "Media records preserve discovery evidence but do not assert ownership, rights, identity, or publishability."
38668
+ ]
38669
+ };
38670
+ files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
38671
+ files.push({ path: "media.jsonl", content: Buffer.from(cleanAssets.map((asset) => JSON.stringify(asset)).join("\n") + "\n") });
38672
+ const archive = await zipBuffer2(files);
38673
+ const id = randomBytes6(6).toString("hex");
38674
+ const pointer = await createPrivateArtifact({
38675
+ policy: policy5(),
38676
+ ownerId: args.ownerId,
38677
+ artifactKey: `${id}.zip`,
38678
+ createdAt: new Date(args.extractedAt),
38679
+ filename: `${slug3(args.title || new URL(args.pageUrl).hostname)}-website-media.zip`,
38680
+ contentType: "application/zip",
38681
+ content: archive
38682
+ });
38683
+ const localPath = token5() ? null : join8(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join8(homedir3(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
38684
+ return { media: args.media, artifact: { ...pointer, localPath } };
38685
+ }
38686
+
37217
38687
  // src/api/site-extract-reconciliation.ts
37218
38688
  async function redispatchFundedSiteExtractJobs(limit = 25) {
37219
38689
  const jobs = await listFundedPendingExtractJobs(limit);
@@ -37279,13 +38749,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
37279
38749
  }
37280
38750
 
37281
38751
  // src/api/page-diff.ts
37282
- import { createHash as createHash23 } from "crypto";
38752
+ import { createHash as createHash24 } from "crypto";
37283
38753
  import { diffLines } from "diff";
37284
38754
  var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
37285
38755
  var MAX_DIFF_HUNKS = 200;
37286
38756
  var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
37287
38757
  function sha256Hex(value) {
37288
- return createHash23("sha256").update(value).digest("hex");
38758
+ return createHash24("sha256").update(value).digest("hex");
37289
38759
  }
37290
38760
  function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
37291
38761
  if (value.length <= maxChars) return { value, truncated: false };
@@ -37378,8 +38848,8 @@ async function uploadFallbackBlob(opts) {
37378
38848
  try {
37379
38849
  const content = (opts.content ?? "").trim();
37380
38850
  if (!content) return null;
37381
- const slug2 = (slugify(opts.title || opts.source) || "scrape").slice(0, 60);
37382
- const key = `${SCRAPE_FALLBACK_PREFIX}${slug2}.md`;
38851
+ const slug4 = (slugify(opts.title || opts.source) || "scrape").slice(0, 60);
38852
+ const key = `${SCRAPE_FALLBACK_PREFIX}${slug4}.md`;
37383
38853
  const doc = `# ${opts.title}
37384
38854
  Source: ${opts.source}
37385
38855
 
@@ -37399,10 +38869,10 @@ async function persistScrapeBody(user, opts) {
37399
38869
  }
37400
38870
 
37401
38871
  // src/api/scrape-image-sink.ts
37402
- import { createHash as createHash24 } from "crypto";
38872
+ import { createHash as createHash25 } from "crypto";
37403
38873
  var MAX_IMAGES_PER_SCRAPE = 25;
37404
38874
  function idempotencyKey2(userId, vault, input) {
37405
- return `scrape-image-${createHash24("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
38875
+ return `scrape-image-${createHash25("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
37406
38876
  }
37407
38877
  async function persistScrapeImagesToMemory(user, inputs, vault) {
37408
38878
  const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
@@ -37446,7 +38916,7 @@ async function persistScrapeImagesToMemory(user, inputs, vault) {
37446
38916
  }
37447
38917
 
37448
38918
  // src/api/archive-reader.ts
37449
- import { basename, extname } from "path";
38919
+ import { basename as basename2, extname as extname2 } from "path";
37450
38920
  import yauzl from "yauzl";
37451
38921
  var MAX_ARCHIVE_DOWNLOAD_BYTES = 50 * 1024 * 1024;
37452
38922
  var MAX_ARCHIVE_ENTRIES = 1e4;
@@ -37501,9 +38971,9 @@ var ArchiveReadError = class extends Error {
37501
38971
  retryable;
37502
38972
  };
37503
38973
  function readableContentType(path5) {
37504
- const extensionType = READABLE_EXTENSIONS.get(extname(path5).toLowerCase());
38974
+ const extensionType = READABLE_EXTENSIONS.get(extname2(path5).toLowerCase());
37505
38975
  if (extensionType) return extensionType;
37506
- return READABLE_FILENAMES.has(basename(path5).toLowerCase()) ? "text/plain" : null;
38976
+ return READABLE_FILENAMES.has(basename2(path5).toLowerCase()) ? "text/plain" : null;
37507
38977
  }
37508
38978
  function safeEntryPath(path5) {
37509
38979
  const normalized = path5.replaceAll("\\", "/");
@@ -37864,7 +39334,7 @@ function archiveLibrarySource(archiveUrl, path5) {
37864
39334
  return source.href;
37865
39335
  }
37866
39336
  function archiveEntryTitle(path5) {
37867
- return basename(path5).slice(0, 200) || "Archive entry";
39337
+ return basename2(path5).slice(0, 200) || "Archive entry";
37868
39338
  }
37869
39339
  function inferredWaybackCapturedAt(content) {
37870
39340
  const timestamp = content.slice(0, 4e3).match(/^Archive timestamp:\s*(\d{14})\s*$/m)?.[1];
@@ -37874,7 +39344,7 @@ function inferredWaybackCapturedAt(content) {
37874
39344
  }
37875
39345
 
37876
39346
  // src/api/connection-memory-import.ts
37877
- import { createHash as createHash25 } from "crypto";
39347
+ import { createHash as createHash26 } from "crypto";
37878
39348
  var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
37879
39349
  var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
37880
39350
  var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
@@ -37904,7 +39374,7 @@ function canonicalJson(value) {
37904
39374
  return JSON.stringify(value);
37905
39375
  }
37906
39376
  function sha2562(value) {
37907
- return createHash25("sha256").update(value).digest("hex");
39377
+ return createHash26("sha256").update(value).digest("hex");
37908
39378
  }
37909
39379
  function sensitiveKey(key) {
37910
39380
  const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
@@ -38122,17 +39592,17 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
38122
39592
 
38123
39593
  // src/api/scrape-blob-cleanup.ts
38124
39594
  import { readdir, stat, unlink } from "fs/promises";
38125
- import { homedir as homedir2 } from "os";
38126
- import { join as join7 } from "path";
38127
- async function cleanupVercel(token4, cutoff) {
39595
+ import { homedir as homedir4 } from "os";
39596
+ import { join as join9 } from "path";
39597
+ async function cleanupVercel(token6, cutoff) {
38128
39598
  const { list, del } = await import("@vercel/blob");
38129
39599
  let cursor;
38130
39600
  let deleted = 0;
38131
39601
  do {
38132
- const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token: token4 });
39602
+ const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token: token6 });
38133
39603
  const expired = res.blobs.filter((b) => new Date(b.uploadedAt).getTime() < cutoff);
38134
39604
  if (expired.length) {
38135
- await del(expired.map((b) => b.url), { token: token4 });
39605
+ await del(expired.map((b) => b.url), { token: token6 });
38136
39606
  deleted += expired.length;
38137
39607
  }
38138
39608
  cursor = res.hasMore ? res.cursor : void 0;
@@ -38140,8 +39610,8 @@ async function cleanupVercel(token4, cutoff) {
38140
39610
  return { deleted, store: "vercel-blob" };
38141
39611
  }
38142
39612
  async function cleanupLocal(cutoff) {
38143
- const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join7(homedir2(), "Downloads", "mcp-scraper");
38144
- const dir = join7(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
39613
+ const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join9(homedir4(), "Downloads", "mcp-scraper");
39614
+ const dir = join9(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
38145
39615
  let deleted = 0;
38146
39616
  let entries;
38147
39617
  try {
@@ -38150,7 +39620,7 @@ async function cleanupLocal(cutoff) {
38150
39620
  return { deleted: 0, store: "local" };
38151
39621
  }
38152
39622
  for (const name of entries) {
38153
- const path5 = join7(dir, name);
39623
+ const path5 = join9(dir, name);
38154
39624
  try {
38155
39625
  const s = await stat(path5);
38156
39626
  if (s.isFile() && s.mtimeMs < cutoff) {
@@ -38165,8 +39635,8 @@ async function cleanupLocal(cutoff) {
38165
39635
  async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
38166
39636
  const cutoff = Date.now() - maxAgeMs;
38167
39637
  try {
38168
- const token4 = process.env.BLOB_READ_WRITE_TOKEN;
38169
- return token4 ? await cleanupVercel(token4, cutoff) : await cleanupLocal(cutoff);
39638
+ const token6 = process.env.BLOB_READ_WRITE_TOKEN;
39639
+ return token6 ? await cleanupVercel(token6, cutoff) : await cleanupLocal(cutoff);
38170
39640
  } catch {
38171
39641
  return { deleted: 0, store: "none" };
38172
39642
  }
@@ -38915,7 +40385,7 @@ async function dbUsage(identity, plan) {
38915
40385
  }
38916
40386
 
38917
40387
  // src/api/nango-control.ts
38918
- import { createHash as createHash27, randomUUID as randomUUID22 } from "crypto";
40388
+ import { createHash as createHash28, randomUUID as randomUUID22 } from "crypto";
38919
40389
 
38920
40390
  // src/api/slack-archive-analysis.ts
38921
40391
  function isRecord2(value) {
@@ -39289,7 +40759,7 @@ async function exportSlackChannelPage(input, dependencies) {
39289
40759
  }
39290
40760
 
39291
40761
  // src/api/main-nango-transport.ts
39292
- import { createHash as createHash26, randomUUID as randomUUID21 } from "crypto";
40762
+ import { createHash as createHash27, randomUUID as randomUUID21 } from "crypto";
39293
40763
  import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
39294
40764
 
39295
40765
  // src/api/connected-tool-policy.ts
@@ -39332,9 +40802,9 @@ function buildConnectedToolPolicyRegistry(args) {
39332
40802
  return policies;
39333
40803
  }
39334
40804
  function connectedToolPolicyOrThrow(registry, toolName) {
39335
- const policy4 = registry.get(toolName);
39336
- if (!policy4) throw new Error("connected_tool_policy_missing");
39337
- return policy4;
40805
+ const policy6 = registry.get(toolName);
40806
+ if (!policy6) throw new Error("connected_tool_policy_missing");
40807
+ return policy6;
39338
40808
  }
39339
40809
 
39340
40810
  // src/api/main-nango-transport.ts
@@ -39680,9 +41150,9 @@ async function listNangoToolsDirect(identity, connectionId) {
39680
41150
  connectionActive: connection.lifecycleStatus === "connected",
39681
41151
  actionsEnabled: connection.actionsEnabled
39682
41152
  });
39683
- const readTools = [...policies.values()].filter((policy4) => policy4.classification === "read").map((policy4) => policy4.name);
39684
- const actionTools = [...policies.values()].filter((policy4) => policy4.classification === "action").map((policy4) => policy4.name);
39685
- const revision = createHash26("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
41153
+ const readTools = [...policies.values()].filter((policy6) => policy6.classification === "read").map((policy6) => policy6.name);
41154
+ const actionTools = [...policies.values()].filter((policy6) => policy6.classification === "action").map((policy6) => policy6.name);
41155
+ const revision = createHash27("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
39686
41156
  await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
39687
41157
  const refreshed = await getOwnedServiceConnection(identity, connection.id);
39688
41158
  return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
@@ -39691,18 +41161,18 @@ async function callNangoToolDirect(args) {
39691
41161
  const { connection, tools } = await listNangoToolsDirect(args.identity, args.connectionId);
39692
41162
  const selected = tools.find((tool) => tool.name === args.tool);
39693
41163
  if (!selected) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
39694
- const policy4 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
41164
+ const policy6 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
39695
41165
  tools,
39696
41166
  connectionActive: connection.lifecycleStatus === "connected",
39697
41167
  actionsEnabled: connection.actionsEnabled
39698
41168
  }), selected.name);
39699
- if (policy4.classification !== args.classification) {
41169
+ if (policy6.classification !== args.classification) {
39700
41170
  throw new MainNangoTransportError("The provider tool classification does not match this operation.", 403, "tool_not_allowed", false);
39701
41171
  }
39702
- if (!policy4.available && policy4.blockedReason === "actions_disabled") {
41172
+ if (!policy6.available && policy6.blockedReason === "actions_disabled") {
39703
41173
  throw new MainNangoTransportError("Actions are disabled for this service connection.", 403, "actions_disabled", false);
39704
41174
  }
39705
- if (!policy4.available) {
41175
+ if (!policy6.available) {
39706
41176
  throw new MainNangoTransportError("The provider tool is not authorized for this connection.", 403, "tool_not_allowed", false);
39707
41177
  }
39708
41178
  await preflightConnectedUsage({
@@ -39710,7 +41180,7 @@ async function callNangoToolDirect(args) {
39710
41180
  ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
39711
41181
  });
39712
41182
  const requestId = args.requestId?.trim() || randomUUID21();
39713
- const idempotencyKey3 = `main-nango:${createHash26("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
41183
+ const idempotencyKey3 = `main-nango:${createHash27("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
39714
41184
  const startedAt = /* @__PURE__ */ new Date();
39715
41185
  const started = performance.now();
39716
41186
  let result;
@@ -39735,7 +41205,7 @@ async function callNangoToolDirect(args) {
39735
41205
  toolName: args.tool,
39736
41206
  operationKind: args.operationKind ?? args.classification,
39737
41207
  outcome: providerError ? "error" : "partial",
39738
- requestId: requestId.length <= 200 ? requestId : createHash26("sha256").update(requestId).digest("hex"),
41208
+ requestId: requestId.length <= 200 ? requestId : createHash27("sha256").update(requestId).digest("hex"),
39739
41209
  startedAt: startedAt.toISOString(),
39740
41210
  completedAt: completedAt.toISOString()
39741
41211
  }
@@ -39747,13 +41217,13 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
39747
41217
  const { connection, tools } = await listNangoToolsDirect(identity, connectionId);
39748
41218
  const tool = tools.find((candidate) => candidate.name === toolName);
39749
41219
  if (!tool) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
39750
- const policy4 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
41220
+ const policy6 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
39751
41221
  tools,
39752
41222
  connectionActive: connection.lifecycleStatus === "connected",
39753
41223
  actionsEnabled: connection.actionsEnabled
39754
41224
  }), tool.name);
39755
41225
  const classification = classifyConnectedTool(tool);
39756
- const callable = policy4.available;
41226
+ const callable = policy6.available;
39757
41227
  const projected = {
39758
41228
  name: tool.name,
39759
41229
  ...tool.title ? { title: tool.title } : {},
@@ -39766,18 +41236,18 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
39766
41236
  ...projected,
39767
41237
  classification,
39768
41238
  callable,
39769
- blockedReason: policy4.blockedReason,
39770
- requiredPermissions: policy4.requiredPermissions,
39771
- missingPermissions: policy4.missingPermissions,
39772
- requiredFeatures: policy4.requiredFeatures,
39773
- missingFeatures: policy4.missingFeatures,
41239
+ blockedReason: policy6.blockedReason,
41240
+ requiredPermissions: policy6.requiredPermissions,
41241
+ missingPermissions: policy6.missingPermissions,
41242
+ requiredFeatures: policy6.requiredFeatures,
41243
+ missingFeatures: policy6.missingFeatures,
39774
41244
  transport: "nango",
39775
41245
  providerConfigKey: connection.providerConfigKey,
39776
41246
  providerContractVersion: MAIN_INTEGRATION_CONTRACT_VERSION,
39777
41247
  providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
39778
41248
  protocolVersion: null,
39779
41249
  schemaSource: "live_tools_list",
39780
- schemaHash: createHash26("sha256").update(JSON.stringify(projected)).digest("hex"),
41250
+ schemaHash: createHash27("sha256").update(JSON.stringify(projected)).digest("hex"),
39781
41251
  fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
39782
41252
  };
39783
41253
  }
@@ -40501,8 +41971,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
40501
41971
  return data.connection.actionsEnabled === true;
40502
41972
  }
40503
41973
  async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
40504
- const requestId = `main-connected-action:${createHash27("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID22()).digest("hex")}`;
40505
- const requestDigest = createHash27("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
41974
+ const requestId = `main-connected-action:${createHash28("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID22()).digest("hex")}`;
41975
+ const requestDigest = createHash28("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
40506
41976
  if (mainOwnsIntegrations()) {
40507
41977
  const selectedTool = tool?.trim();
40508
41978
  if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
@@ -40653,7 +42123,7 @@ function canonicalJson2(value) {
40653
42123
  return JSON.stringify(value);
40654
42124
  }
40655
42125
  function projectedToolSchemaHash(tool) {
40656
- return createHash27("sha256").update(canonicalJson2(tool)).digest("hex");
42126
+ return createHash28("sha256").update(canonicalJson2(tool)).digest("hex");
40657
42127
  }
40658
42128
  async function describeNangoTool(identity, connectionId, tool, fresh) {
40659
42129
  if (mainOwnsIntegrations()) {
@@ -40932,7 +42402,7 @@ async function callMainOwnedExportPage(identity, input) {
40932
42402
  }
40933
42403
 
40934
42404
  // src/api/resend-control.ts
40935
- import { createHash as createHash28 } from "crypto";
42405
+ import { createHash as createHash29 } from "crypto";
40936
42406
  var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
40937
42407
  var RESEND_PROVIDER_CONFIG_KEY = "resend";
40938
42408
  var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
@@ -41191,7 +42661,7 @@ async function callResendRead(identity, connectionId, tool, args) {
41191
42661
  return isRecord5(data) ? data.result ?? data : data;
41192
42662
  }
41193
42663
  async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
41194
- const requestId = `main-resend-action:${createHash28("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
42664
+ const requestId = `main-resend-action:${createHash29("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
41195
42665
  const body = await controlRequest2("/api/internal/resend/actions/call", {
41196
42666
  method: "POST",
41197
42667
  headers: { "x-request-id": requestId },
@@ -41318,9 +42788,9 @@ var auth4 = createMiddleware5(async (c, next) => {
41318
42788
  return next();
41319
42789
  });
41320
42790
  var sessionAuth = createMiddleware5(async (c, next) => {
41321
- const token4 = getCookie2(c, "session");
41322
- if (!token4) return c.json({ error: "Not authenticated" }, 401);
41323
- const userId = verifySession(token4);
42791
+ const token6 = getCookie2(c, "session");
42792
+ if (!token6) return c.json({ error: "Not authenticated" }, 401);
42793
+ const userId = verifySession(token6);
41324
42794
  if (!userId) return c.json({ error: "Session expired" }, 401);
41325
42795
  const user = await getUserById(userId);
41326
42796
  if (!user) return c.json({ error: "User not found" }, 401);
@@ -41465,8 +42935,8 @@ app.post("/auth/register", requireAllowedOrigin, async (c) => {
41465
42935
  console.warn("[auth/register] memory provision failed (will retry on first write):", err instanceof Error ? err.message : String(err));
41466
42936
  }
41467
42937
  })();
41468
- const token4 = signSession(user.id);
41469
- setCookie2(c, "session", token4, sessionCookieOptions2);
42938
+ const token6 = signSession(user.id);
42939
+ setCookie2(c, "session", token6, sessionCookieOptions2);
41470
42940
  return c.json({ email: user.email, api_key: newKey }, 201);
41471
42941
  } catch (err) {
41472
42942
  const msg = err instanceof Error ? err.message : "";
@@ -41484,8 +42954,8 @@ app.post("/auth/login", requireAllowedOrigin, async (c) => {
41484
42954
  if (!user || !user.password_hash || !verifyPassword(password, user.password_hash)) {
41485
42955
  return c.json({ error: "Invalid email or password" }, 401);
41486
42956
  }
41487
- const token4 = signSession(user.id);
41488
- setCookie2(c, "session", token4, sessionCookieOptions2);
42957
+ const token6 = signSession(user.id);
42958
+ setCookie2(c, "session", token6, sessionCookieOptions2);
41489
42959
  return c.json({ email: user.email, api_key: user.key_active ? user.api_key : null });
41490
42960
  });
41491
42961
  app.post("/auth/logout", requireAllowedOrigin, (c) => {
@@ -41540,9 +43010,9 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
41540
43010
  if (emailLimited) return emailLimited;
41541
43011
  const user = await getUserByEmail(normalizedEmail);
41542
43012
  if (!user) return c.json({ ok: true });
41543
- const token4 = await createPasswordResetToken(user.id);
43013
+ const token6 = await createPasswordResetToken(user.id);
41544
43014
  const appUrl = process.env.APP_URL ?? "https://mcpscraper.dev";
41545
- const resetUrl = `${appUrl}/reset-password?token=${token4}`;
43015
+ const resetUrl = `${appUrl}/reset-password?token=${token6}`;
41546
43016
  if (process.env.RESEND_API_KEY) {
41547
43017
  try {
41548
43018
  const resend = new Resend3(process.env.RESEND_API_KEY);
@@ -41562,12 +43032,12 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
41562
43032
  return c.json({ ok: true });
41563
43033
  });
41564
43034
  app.post("/auth/reset-password", requireAllowedOrigin, async (c) => {
41565
- const { token: token4, password } = await c.req.json();
41566
- if (!token4 || !password) return c.json({ error: "Token and password required" }, 400);
43035
+ const { token: token6, password } = await c.req.json();
43036
+ if (!token6 || !password) return c.json({ error: "Token and password required" }, 400);
41567
43037
  if (password.length < 8) return c.json({ error: "Password must be at least 8 characters" }, 400);
41568
43038
  const limited = await enforceRateLimit(c, "auth_reset", rateLimitKey(c), 10, 60 * 60);
41569
43039
  if (limited) return limited;
41570
- const userId = await consumePasswordResetToken(token4);
43040
+ const userId = await consumePasswordResetToken(token6);
41571
43041
  if (!userId) return c.json({ error: "Invalid or expired reset link" }, 400);
41572
43042
  await setPassword(userId, password);
41573
43043
  return c.json({ ok: true });
@@ -41604,9 +43074,9 @@ function sessionBootstrapPayload(user) {
41604
43074
  }
41605
43075
  app.get("/me", async (c) => {
41606
43076
  c.header("Cache-Control", "no-store");
41607
- const token4 = getCookie2(c, "session");
41608
- if (!token4) return c.json({ authenticated: false });
41609
- const userId = verifySession(token4);
43077
+ const token6 = getCookie2(c, "session");
43078
+ if (!token6) return c.json({ authenticated: false });
43079
+ const userId = verifySession(token6);
41610
43080
  if (!userId) {
41611
43081
  deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
41612
43082
  return c.json({ authenticated: false });
@@ -43232,6 +44702,8 @@ app.post("/extract-url", auth4, async (c) => {
43232
44702
  }
43233
44703
  const effectiveDelivery = bodyResult.data.depositToVault ? "memory" : requestedDelivery;
43234
44704
  const preserveMedia = bodyResult.data.preserveMedia ?? bodyResult.data.downloadMedia ?? false;
44705
+ const maxMediaAssets = bodyResult.data.maxMediaAssets ?? 100;
44706
+ const maxInlineImages = bodyResult.data.maxInlineImages ?? 3;
43235
44707
  if (!allowLocal) {
43236
44708
  const checked = await validatePublicHttpUrl(url, { field: "URL" });
43237
44709
  if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
@@ -43266,7 +44738,14 @@ app.post("/extract-url", auth4, async (c) => {
43266
44738
  const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
43267
44739
  const [result, pageData] = await Promise.all([
43268
44740
  extractKpo({ url: extractionUrl, kernelApiKey }),
43269
- screenshot2 || extractBranding2 ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding2 }).catch((err) => {
44741
+ screenshot2 || extractBranding2 || preserveMedia ? capturePageData(canonicalUrl, {
44742
+ kernelApiKey,
44743
+ device,
44744
+ screenshot: !!screenshot2,
44745
+ branding: !!extractBranding2,
44746
+ media: preserveMedia,
44747
+ maxMediaAssets: Math.min(1e3, Math.max(250, maxMediaAssets * 4))
44748
+ }).catch((err) => {
43270
44749
  console.error("[extract-url] capturePageData failed:", err instanceof Error ? err.message : err);
43271
44750
  return null;
43272
44751
  }) : null
@@ -43277,7 +44756,28 @@ app.post("/extract-url", auth4, async (c) => {
43277
44756
  if (screenshotBuf) {
43278
44757
  screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
43279
44758
  }
43280
- const mediaMeta = preserveMedia ? await harvestPageMedia(result.bodyHtml, waybackReplay?.originalUrl ?? canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
44759
+ let mediaMeta = preserveMedia ? await harvestPageMedia(result.bodyHtml, waybackReplay?.originalUrl ?? canonicalUrl, {
44760
+ types: mediaTypes ?? ["image", "video", "audio"],
44761
+ outputDir: null,
44762
+ rendered: pageData?.renderedMedia ?? null,
44763
+ maxAssets: maxMediaAssets
44764
+ }) : null;
44765
+ if (mediaMeta) {
44766
+ try {
44767
+ const packaged = await packagePageMedia({
44768
+ ownerId: String(user.id),
44769
+ pageUrl: canonicalUrl,
44770
+ title: result.title ?? null,
44771
+ extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
44772
+ media: mediaMeta,
44773
+ maxInlineImages
44774
+ });
44775
+ mediaMeta = packaged.media;
44776
+ mediaMeta.artifact = packaged.artifact;
44777
+ } catch (error) {
44778
+ mediaMeta.warnings.push(`Media packaging failed: ${error instanceof Error ? error.message : "media_packaging_failed"}`);
44779
+ }
44780
+ }
43281
44781
  const featuredImage = includeFeaturedImage ? resolveFeaturedImage({
43282
44782
  html: result.bodyHtml,
43283
44783
  meta: result.meta,
@@ -43460,7 +44960,7 @@ app.post("/archive/read", auth4, async (c) => {
43460
44960
  try {
43461
44961
  let ownedArchive = null;
43462
44962
  if (artifactId) {
43463
- const buffer = await readOwnedSiteExtractArtifactBuffer({ artifactId, ownerId: String(user.id) });
44963
+ const buffer = artifactId.startsWith(MAPS_MEDIA_ARTIFACT_PREFIX) ? await readOwnedMapsMediaArtifactBuffer({ artifactId, ownerId: String(user.id) }) : artifactId.startsWith(PAGE_MEDIA_ARTIFACT_PREFIX) ? await readOwnedPageMediaArtifactBuffer({ artifactId, ownerId: String(user.id) }) : await readOwnedSiteExtractArtifactBuffer({ artifactId, ownerId: String(user.id) });
43464
44964
  if (!buffer) throw new ArchiveReadError("archive_not_found", "Archive artifact was not found or has expired.", 404);
43465
44965
  ownedArchive = { buffer, ref: artifactId };
43466
44966
  }
@@ -44270,7 +45770,7 @@ app.get("/cron/tick", async (c) => {
44270
45770
  return c.json({ error: "Unauthorized" }, 401);
44271
45771
  }
44272
45772
  const startedAt = Date.now();
44273
- const { drainQueue } = await import("./worker-KQN673JF.js");
45773
+ const { drainQueue } = await import("./worker-JL4TG6IF.js");
44274
45774
  const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
44275
45775
  const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
44276
45776
  const unfinished = [];
@@ -44341,7 +45841,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
44341
45841
  terminalExtractJobStatus: terminalExtractJobStatus2,
44342
45842
  extractJobLimitInfo: extractJobLimitInfo2
44343
45843
  } = await import("./site-extract-repository-3VQAALNR.js");
44344
- const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
45844
+ const { assembleExtractArtifacts } = await import("./extract-bundle-GUEUCTAE.js");
44345
45845
  const existing = await getExtractJob2(jobId2);
44346
45846
  if (!existing) return c.json({ error: "job not found" }, 404);
44347
45847
  if (existing.status !== "failed") {
@@ -44553,4 +46053,4 @@ app.get("/blog/:slug/", (c) => {
44553
46053
  export {
44554
46054
  app
44555
46055
  };
44556
- //# sourceMappingURL=server-JR5XPTXS.js.map
46056
+ //# sourceMappingURL=server-7EXEDKAU.js.map