mcp-scraper 0.51.2 → 0.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +3 -3
  2. package/dist/bin/api-server.cjs +2819 -869
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +2 -2
  5. package/dist/bin/mcp-scraper-cli.cjs +5 -1
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +3 -3
  8. package/dist/bin/mcp-scraper-install.cjs +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +1 -1
  11. package/dist/bin/mcp-stdio-server.cjs +355 -28
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +2 -2
  14. package/dist/bin/paa-harvest.cjs +4 -0
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +2 -2
  17. package/dist/{chunk-ZSJHRZY5.js → chunk-J7TH5KU7.js} +2 -2
  18. package/dist/{chunk-FCZ3QCZP.js → chunk-OT2AF7SH.js} +356 -29
  19. package/dist/chunk-OT2AF7SH.js.map +1 -0
  20. package/dist/chunk-SG2PEHR3.js +562 -0
  21. package/dist/chunk-SG2PEHR3.js.map +1 -0
  22. package/dist/{chunk-GOZIG6HD.js → chunk-TVZD37SE.js} +2 -2
  23. package/dist/{chunk-3ZUBQQPQ.js → chunk-V7CEOUBF.js} +5 -1
  24. package/dist/chunk-V7CEOUBF.js.map +1 -0
  25. package/dist/chunk-VIAXQTZK.js +7 -0
  26. package/dist/chunk-VIAXQTZK.js.map +1 -0
  27. package/dist/{extract-bundle-RCTNANCH.js → extract-bundle-GUEUCTAE.js} +2 -2
  28. package/dist/index.cjs +4 -0
  29. package/dist/index.cjs.map +1 -1
  30. package/dist/index.js +2 -2
  31. package/dist/{server-KAEHVDY7.js → server-7EXEDKAU.js} +1857 -401
  32. package/dist/server-7EXEDKAU.js.map +1 -0
  33. package/dist/{worker-KQN673JF.js → worker-JL4TG6IF.js} +3 -3
  34. package/package.json +1 -1
  35. package/dist/chunk-27FMOD6S.js +0 -430
  36. package/dist/chunk-27FMOD6S.js.map +0 -1
  37. package/dist/chunk-3ZUBQQPQ.js.map +0 -1
  38. package/dist/chunk-FCZ3QCZP.js.map +0 -1
  39. package/dist/chunk-JNRSR5ZJ.js +0 -7
  40. package/dist/chunk-JNRSR5ZJ.js.map +0 -1
  41. package/dist/server-KAEHVDY7.js.map +0 -1
  42. /package/dist/{chunk-ZSJHRZY5.js.map → chunk-J7TH5KU7.js.map} +0 -0
  43. /package/dist/{chunk-GOZIG6HD.js.map → chunk-TVZD37SE.js.map} +0 -0
  44. /package/dist/{extract-bundle-RCTNANCH.js.map → extract-bundle-GUEUCTAE.js.map} +0 -0
  45. /package/dist/{worker-KQN673JF.js.map → worker-JL4TG6IF.js.map} +0 -0
@@ -1,10 +1,12 @@
1
1
  import {
2
2
  SITE_EXTRACT_ARTIFACT_PREFIX,
3
3
  cleanupExpiredSiteExtractArtifacts,
4
+ discoverStaticMedia,
4
5
  harvestPageMedia,
5
6
  readOwnedSiteExtractArtifactBuffer,
6
- renewSiteExtractArtifactDownload
7
- } from "./chunk-27FMOD6S.js";
7
+ renewSiteExtractArtifactDownload,
8
+ typeFromMime
9
+ } from "./chunk-SG2PEHR3.js";
8
10
  import {
9
11
  REQUIRED_LOCATION_STATE_CODES,
10
12
  importHostedCensusPlacesCsv,
@@ -100,7 +102,7 @@ import {
100
102
  resolveKernelProxyId,
101
103
  serpArrivalUrl,
102
104
  withRuntimeUserAgent
103
- } from "./chunk-GOZIG6HD.js";
105
+ } from "./chunk-TVZD37SE.js";
104
106
  import {
105
107
  ApiErrorResponseSchema,
106
108
  ArtifactTemplatePresetKeySchema,
@@ -158,7 +160,7 @@ import {
158
160
  resolveDeploymentProfile,
159
161
  resolveLocalSourcebookSchemaType,
160
162
  transcribeMediaUrl
161
- } from "./chunk-FCZ3QCZP.js";
163
+ } from "./chunk-OT2AF7SH.js";
162
164
  import {
163
165
  auditImageUrls,
164
166
  auditImages,
@@ -196,7 +198,7 @@ import {
196
198
  workflowOutputBaseDir,
197
199
  workflowStepCount,
198
200
  workflowSupportsSteps
199
- } from "./chunk-ZSJHRZY5.js";
201
+ } from "./chunk-J7TH5KU7.js";
200
202
  import {
201
203
  csvRecords,
202
204
  parseCsv,
@@ -211,10 +213,10 @@ import {
211
213
  RawMapsHoursRowSchema,
212
214
  RawMapsOverviewSchema,
213
215
  RawMapsReviewStatsSchema
214
- } from "./chunk-3ZUBQQPQ.js";
216
+ } from "./chunk-V7CEOUBF.js";
215
217
  import {
216
218
  PACKAGE_VERSION
217
- } from "./chunk-JNRSR5ZJ.js";
219
+ } from "./chunk-VIAXQTZK.js";
218
220
  import {
219
221
  abandonExtractSettlement,
220
222
  countSuccessfulPages,
@@ -3813,39 +3815,135 @@ async function readResponseTextWithLimit(response, maxBytes, label) {
3813
3815
  }
3814
3816
 
3815
3817
  // src/api/kernel-fetch.ts
3816
- async function fetchWithKernel(url) {
3817
- const checked = await validatePublicHttpUrl(url, { field: "browser URL" });
3818
- if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Browser URL was rejected");
3819
- const auditedUrl = checked.parsed.href;
3820
- const apiKey = browserServiceApiKey();
3821
- if (!apiKey) throw new Error("Browser backend API key not set");
3822
- const client2 = new Kernel({ apiKey });
3818
+ var CHALLENGE_PATTERNS = [
3819
+ { reason: "cloudflare_title", pattern: /<title[^>]*>\s*just a moment(?:\.\.\.)?\s*<\/title>/i },
3820
+ { reason: "cloudflare_challenge_platform", pattern: /(?:cdn-cgi\/challenge-platform|challenge-platform\/h\/|cf-chl-|cf_chl_)/i },
3821
+ { reason: "cloudflare_browser_verification", pattern: /(?:cf-browser-verification|checking your browser before accessing)/i },
3822
+ { reason: "cloudflare_cookie_message", pattern: /enable javascript and cookies to continue/i },
3823
+ { reason: "generic_security_check", pattern: /performing security verification|verify you are human/i }
3824
+ ];
3825
+ function detectBrowserChallenge(input) {
3826
+ if (input.status === 403 || input.status === 429) {
3827
+ return { detected: true, reason: `http_${input.status}` };
3828
+ }
3829
+ const sample = `${input.title ?? ""}
3830
+ ${input.html ?? ""}`.slice(0, 25e4);
3831
+ for (const candidate of CHALLENGE_PATTERNS) {
3832
+ if (candidate.pattern.test(sample)) return { detected: true, reason: candidate.reason };
3833
+ }
3834
+ return { detected: false, reason: null };
3835
+ }
3836
+ async function runKernelAttempt(client2, auditedUrl, mode) {
3823
3837
  const openedAtMs = Date.now();
3824
- const useHeadless = currentCostContext()?.forceHeadful !== true;
3825
- const kb = await client2.browsers.create({ stealth: true, ...useHeadless ? { headless: true } : {}, timeout_seconds: 60 });
3838
+ const headlessSent = mode === "stealth_headless" ? true : null;
3839
+ const kb = await client2.browsers.create({
3840
+ stealth: true,
3841
+ ...headlessSent ? { headless: true } : {},
3842
+ timeout_seconds: 60
3843
+ });
3826
3844
  const browser = await chromium.connectOverCDP(kb.cdp_ws_url);
3845
+ let attemptError = null;
3827
3846
  try {
3828
3847
  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
3829
- const context = await browser.newContext(withRuntimeUserAgent({
3830
- serviceWorkers: "block"
3831
- }, runtimeUserAgent));
3848
+ const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
3832
3849
  const page = context.pages()[0] ?? await context.newPage();
3833
3850
  const guard = await installBrowserRequestGuard(page, auditedUrl);
3834
- await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
3851
+ const response = await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
3835
3852
  if (guard.blockedReason && page.url() === "about:blank") throw new Error(guard.blockedReason);
3836
3853
  await page.waitForTimeout(2500);
3837
- return await page.evaluate((maxBytes) => {
3838
- const value = document.documentElement?.outerHTML ?? "";
3839
- if (new TextEncoder().encode(value).byteLength > maxBytes) throw new Error(`Browser HTML exceeds ${maxBytes} byte limit`);
3840
- return value;
3841
- }, MAX_PAGE_HTML_BYTES);
3854
+ let status = response?.status() ?? null;
3855
+ let title = await page.title().catch(() => "");
3856
+ let html = await page.content();
3857
+ let challenge = detectBrowserChallenge({ status, title, html });
3858
+ if (challenge.detected && mode === "stealth_headful") {
3859
+ const deadline = Date.now() + 2e4;
3860
+ while (challenge.detected && Date.now() < deadline) {
3861
+ await page.waitForTimeout(1e3);
3862
+ title = await page.title().catch(() => "");
3863
+ html = await page.content();
3864
+ challenge = detectBrowserChallenge({ status, title, html });
3865
+ }
3866
+ status = response?.status() ?? status;
3867
+ }
3868
+ const byteLength2 = new TextEncoder().encode(html).byteLength;
3869
+ if (byteLength2 > MAX_PAGE_HTML_BYTES) throw new Error(`Browser HTML exceeds ${MAX_PAGE_HTML_BYTES} byte limit`);
3870
+ return { html, status, challenge, durationMs: Date.now() - openedAtMs };
3871
+ } catch (error) {
3872
+ attemptError = error instanceof Error ? error.message : String(error);
3873
+ throw error;
3842
3874
  } finally {
3843
3875
  await browser.close().catch(() => {
3844
3876
  });
3845
3877
  await client2.browsers.deleteByID(kb.session_id).catch(() => {
3846
3878
  });
3847
- void recordKernelSession({ kernelSessionId: kb.session_id, source: "kernel_fetch", stealth: typeof kb.stealth === "boolean" ? kb.stealth : null, headlessSent: useHeadless, proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null, fallback: true, openedAtMs, closedAtMs: Date.now() });
3879
+ void recordKernelSession({
3880
+ kernelSessionId: kb.session_id,
3881
+ source: "kernel_fetch",
3882
+ stealth: typeof kb.stealth === "boolean" ? kb.stealth : null,
3883
+ headlessSent,
3884
+ proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null,
3885
+ fallback: true,
3886
+ openedAtMs,
3887
+ closedAtMs: Date.now(),
3888
+ error: attemptError
3889
+ });
3890
+ }
3891
+ }
3892
+ async function fetchWithKernelDetailed(url) {
3893
+ const checked = await validatePublicHttpUrl(url, { field: "browser URL" });
3894
+ if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Browser URL was rejected");
3895
+ const auditedUrl = checked.parsed.href;
3896
+ const apiKey = browserServiceApiKey();
3897
+ if (!apiKey) throw new Error("Browser backend API key not set");
3898
+ const client2 = new Kernel({ apiKey });
3899
+ const startsHeadful = currentCostContext()?.forceHeadful === true;
3900
+ const modes = startsHeadful ? ["stealth_headful"] : ["stealth_headless", "stealth_headful"];
3901
+ const attempts = [];
3902
+ for (const mode of modes) {
3903
+ const result = await runKernelAttempt(client2, auditedUrl, mode);
3904
+ if (result.challenge.detected) {
3905
+ const completion = mode === "stealth_headful" ? "challenge_unresolved" : "challenge_detected";
3906
+ attempts.push({
3907
+ mode,
3908
+ status: result.status,
3909
+ challengeDetected: true,
3910
+ challengeReason: result.challenge.reason,
3911
+ completion,
3912
+ durationMs: result.durationMs
3913
+ });
3914
+ if (mode === "stealth_headful") {
3915
+ const error = new Error(`browser_challenge_unresolved: ${result.challenge.reason ?? "challenge page remained after bounded wait"}`);
3916
+ error.code = "browser_challenge_unresolved";
3917
+ error.attempts = attempts;
3918
+ throw error;
3919
+ }
3920
+ continue;
3921
+ }
3922
+ if (result.status != null && (result.status < 200 || result.status >= 400)) {
3923
+ attempts.push({
3924
+ mode,
3925
+ status: result.status,
3926
+ challengeDetected: false,
3927
+ challengeReason: null,
3928
+ completion: "http_error",
3929
+ durationMs: result.durationMs
3930
+ });
3931
+ throw new Error(`browser_http_${result.status}: Browser fetch returned HTTP ${result.status}`);
3932
+ }
3933
+ attempts.push({
3934
+ mode,
3935
+ status: result.status,
3936
+ challengeDetected: false,
3937
+ challengeReason: null,
3938
+ completion: "usable_page",
3939
+ durationMs: result.durationMs
3940
+ });
3941
+ return { html: result.html, mode, attempts };
3848
3942
  }
3943
+ throw new Error("browser_challenge_unresolved: no usable browser response");
3944
+ }
3945
+ async function fetchWithKernel(url) {
3946
+ return (await fetchWithKernelDetailed(url)).html;
3849
3947
  }
3850
3948
 
3851
3949
  // src/lib/http-user-agent.ts
@@ -3875,6 +3973,8 @@ async function extractKpo(opts) {
3875
3973
  }
3876
3974
  if (!res.ok) return null;
3877
3975
  const text2 = await res.text();
3976
+ const title2 = text2.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? null;
3977
+ if (detectBrowserChallenge({ status: res.status, title: title2, html: text2 }).detected) return null;
3878
3978
  if (text2.length < 2e3) return null;
3879
3979
  const stripped = text2.replace(/<(script|style|noscript|svg|iframe|canvas)[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<(nav|header|footer|aside)[^>]*>[\s\S]*?<\/\1>/gi, "");
3880
3980
  const contentBlock = stripped.match(/<main[^>]*>([\s\S]*?)<\/main>/i)?.[1] ?? stripped.match(/<article[^>]*>([\s\S]*?)<\/article>/i)?.[1] ?? stripped.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? "";
@@ -3889,13 +3989,16 @@ async function extractKpo(opts) {
3889
3989
  };
3890
3990
  let html;
3891
3991
  let fetchedVia = "fetch";
3992
+ let browserFetch;
3892
3993
  const plainFetchStart = Date.now();
3893
3994
  const plainHtml = await tryPlainFetch();
3894
3995
  const plainFetchMs = Date.now() - plainFetchStart;
3895
3996
  if (plainHtml) {
3896
3997
  html = plainHtml;
3897
3998
  } else if (opts.kernelApiKey) {
3898
- html = await fetchWithKernel(parsed.href);
3999
+ const browserResult = await fetchWithKernelDetailed(parsed.href);
4000
+ html = browserResult.html;
4001
+ browserFetch = { mode: browserResult.mode, attempts: browserResult.attempts };
3899
4002
  fetchedVia = "headless";
3900
4003
  } else {
3901
4004
  throw new Error("Fetch failed: site blocked plain HTTP and no headless browser backend is configured");
@@ -4195,6 +4298,7 @@ async function extractKpo(opts) {
4195
4298
  return {
4196
4299
  url: parsed.href,
4197
4300
  fetchedVia,
4301
+ browserFetch,
4198
4302
  durationMs,
4199
4303
  plainFetchMs,
4200
4304
  title,
@@ -4212,13 +4316,47 @@ import { chromium as chromium2 } from "playwright";
4212
4316
  import Kernel2 from "@onkernel/sdk";
4213
4317
 
4214
4318
  // src/lib/branding-extractor.ts
4215
- function rgbToHex(rgb) {
4216
- const m = rgb.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)/);
4217
- if (!m) return null;
4218
- const r = parseInt(m[1], 10);
4219
- const g = parseInt(m[2], 10);
4220
- const b = parseInt(m[3], 10);
4221
- return "#" + [r, g, b].map((v) => v.toString(16).padStart(2, "0")).join("");
4319
+ function selectRankedLogoCandidates(candidates) {
4320
+ const ranked = candidates.filter((candidate) => {
4321
+ if (!/^https?:\/\//i.test(candidate.url) || !Number.isFinite(candidate.score) || candidate.region === "body") return false;
4322
+ if (candidate.region === "json_ld" || candidate.region === "favicon") return true;
4323
+ return candidate.evidence.includes("links_to_homepage") || candidate.evidence.includes("site_identity_marker");
4324
+ }).map((candidate) => ({
4325
+ ...candidate,
4326
+ confidence: candidate.score >= 85 ? "high" : candidate.score >= 60 ? "medium" : "low",
4327
+ evidence: [...new Set(candidate.evidence)]
4328
+ })).sort((a, b) => b.score - a.score || a.url.localeCompare(b.url)).slice(0, 20);
4329
+ const primary = ranked.find((candidate) => candidate.score >= 55 && candidate.region !== "favicon") ?? null;
4330
+ return {
4331
+ primary,
4332
+ variants: primary ? ranked.filter((candidate) => candidate.url !== primary.url && candidate.score >= 55 && candidate.region !== "favicon" && logoFamilyKey(candidate.url) === logoFamilyKey(primary.url)).slice(0, 5).map((candidate) => candidate.url) : [],
4333
+ candidates: ranked,
4334
+ selectionReason: primary?.evidence.join(", ") ?? null
4335
+ };
4336
+ }
4337
+ function logoFamilyKey(value) {
4338
+ try {
4339
+ const url = new URL(value);
4340
+ const host2 = url.hostname.replace(/^www\./, "").toLowerCase();
4341
+ const path5 = decodeURIComponent(url.pathname).toLowerCase().replace(/-\d+x\d+(?=\.[a-z0-9]+$)/, "");
4342
+ return `${host2}${path5}`;
4343
+ } catch {
4344
+ return value.toLowerCase();
4345
+ }
4346
+ }
4347
+ function selectProofImages(candidates) {
4348
+ const byUrl = /* @__PURE__ */ new Map();
4349
+ for (const candidate of candidates) {
4350
+ if (!/^https?:\/\//i.test(candidate.url) || !Number.isFinite(candidate.score) || candidate.score < 45) continue;
4351
+ const normalized = {
4352
+ ...candidate,
4353
+ confidence: candidate.score >= 85 ? "high" : candidate.score >= 60 ? "medium" : "low",
4354
+ evidence: [...new Set(candidate.evidence)]
4355
+ };
4356
+ const existing = byUrl.get(normalized.url);
4357
+ if (!existing || normalized.score > existing.score) byUrl.set(normalized.url, normalized);
4358
+ }
4359
+ return [...byUrl.values()].sort((a, b) => b.score - a.score || a.url.localeCompare(b.url)).slice(0, 20);
4222
4360
  }
4223
4361
  function luminance(hex) {
4224
4362
  const m = hex.match(/^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})$/i);
@@ -4232,6 +4370,12 @@ function isTransparentOrWhite(hex) {
4232
4370
  const lum = luminance(hex);
4233
4371
  return lum > 240;
4234
4372
  }
4373
+ function isChromatic(hex) {
4374
+ const match = hex?.match(/^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})$/i);
4375
+ if (!match) return false;
4376
+ const channels = [parseInt(match[1], 16), parseInt(match[2], 16), parseInt(match[3], 16)];
4377
+ return Math.max(...channels) - Math.min(...channels) >= 24 && luminance(hex) > 18;
4378
+ }
4235
4379
  function firstFont(fontFamily) {
4236
4380
  if (!fontFamily) return null;
4237
4381
  const first = fontFamily.split(",")[0].trim().replace(/['"]/g, "");
@@ -4240,14 +4384,34 @@ function firstFont(fontFamily) {
4240
4384
  function dominantColor(freq) {
4241
4385
  return Object.entries(freq).filter(([hex]) => !isTransparentOrWhite(hex) && hex !== "#000000" && hex !== "#020101").sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
4242
4386
  }
4387
+ function dominantChromaticColor(freq) {
4388
+ return Object.entries(freq).filter(([hex]) => isChromatic(hex)).sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
4389
+ }
4243
4390
  async function extractBrandingFromPage(page) {
4391
+ const challenge = detectBrowserChallenge({
4392
+ title: await page.title().catch(() => ""),
4393
+ html: await page.content().catch(() => "")
4394
+ });
4395
+ if (challenge.detected) {
4396
+ const error = new Error(`browser_challenge_unresolved: ${challenge.reason ?? "challenge page is not branding evidence"}`);
4397
+ error.code = "browser_challenge_unresolved";
4398
+ throw error;
4399
+ }
4244
4400
  const evalScript = `
4245
4401
  (function() {
4246
4402
  function cs(el) { return el ? window.getComputedStyle(el) : null; }
4247
- function toHex(rgb) {
4248
- var m = rgb && rgb.match(/rgba?\\((\\d+),\\s*(\\d+),\\s*(\\d+)/);
4249
- if (!m) return null;
4250
- return '#' + [m[1],m[2],m[3]].map(function(v){ return ('0'+parseInt(v).toString(16)).slice(-2); }).join('');
4403
+ var colorCanvas = document.createElement('canvas');
4404
+ colorCanvas.width = 1; colorCanvas.height = 1;
4405
+ var colorContext = colorCanvas.getContext('2d', { willReadFrequently: true });
4406
+ function toHex(value) {
4407
+ if (!value || !colorContext) return null;
4408
+ colorContext.clearRect(0, 0, 1, 1);
4409
+ colorContext.fillStyle = 'rgba(1, 2, 3, 0.001)';
4410
+ colorContext.fillStyle = value;
4411
+ colorContext.fillRect(0, 0, 1, 1);
4412
+ var data = colorContext.getImageData(0, 0, 1, 1).data;
4413
+ if (data[3] < 64) return null;
4414
+ return '#' + [data[0], data[1], data[2]].map(function(v){ return ('0'+v.toString(16)).slice(-2); }).join('');
4251
4415
  }
4252
4416
  function isUsable(hex) {
4253
4417
  if (!hex) return false;
@@ -4256,30 +4420,63 @@ async function extractBrandingFromPage(page) {
4256
4420
  return (0.2126*r + 0.7152*g + 0.0722*b) <= 230;
4257
4421
  }
4258
4422
 
4423
+ function isChromatic(hex) {
4424
+ if (!hex) return false;
4425
+ var r=parseInt(hex.slice(1,3),16), g=parseInt(hex.slice(3,5),16), b=parseInt(hex.slice(5,7),16);
4426
+ return Math.max(r,g,b) - Math.min(r,g,b) >= 24 && (0.2126*r + 0.7152*g + 0.0722*b) > 18;
4427
+ }
4428
+
4429
+ function visibleRect(el) {
4430
+ if (!el) return null;
4431
+ var style = cs(el);
4432
+ var rect = el.getBoundingClientRect();
4433
+ if (!style || style.display === 'none' || style.visibility === 'hidden' || parseFloat(style.opacity || '1') <= 0 || rect.width < 2 || rect.height < 2) return null;
4434
+ return rect;
4435
+ }
4436
+
4259
4437
  var navEl = document.querySelector('nav, header, [role="banner"]');
4260
4438
  var bodyEl = document.body;
4261
4439
  var h1El = document.querySelector('h1');
4262
- var btnEl = document.querySelector(
4263
- 'a.btn-primary, button.btn-primary, .btn-primary, .cta-btn,' +
4264
- 'a.button--primary, button.button--primary, [class*="btn-cta"],' +
4265
- '[class*="cta-button"], .wp-block-button__link, [class*="hero"] a,' +
4266
- '.elementor-button, .elementor-button-link,' +
4267
- '.et_pb_button,' +
4268
- '.fl-button,' +
4269
- '.vc_btn,' +
4270
- '[class*="cta"][href], [class*="get-started"], [class*="contact-btn"]'
4271
- );
4440
+ var btnEl = null;
4441
+ var btnScore = -Infinity;
4442
+ var interactive = document.querySelectorAll('a[href], button, [role="button"]');
4443
+ for (var bi = 0; bi < interactive.length; bi++) {
4444
+ var candidateButton = interactive[bi];
4445
+ var buttonRect = visibleRect(candidateButton);
4446
+ if (!buttonRect || buttonRect.width < 44 || buttonRect.height < 20) continue;
4447
+ if (buttonRect.bottom < 0 || buttonRect.top > window.innerHeight) continue;
4448
+ var buttonStyle = cs(candidateButton);
4449
+ var buttonBg = buttonStyle ? toHex(buttonStyle.backgroundColor) : null;
4450
+ if (!buttonBg) continue;
4451
+ var buttonDescriptor = [candidateButton.textContent, candidateButton.id, candidateButton.className, candidateButton.getAttribute('aria-label')].join(' ');
4452
+ if (/\\b(?:skip|screen-reader|sr-only|visually-hidden)\\b/i.test(buttonDescriptor)) continue;
4453
+ var buttonText = String(candidateButton.textContent || candidateButton.getAttribute('aria-label') || '').replace(/\\s+/g, ' ').trim();
4454
+ var explicitButtonEvidence = /\\b(?:btn[-_ ]?primary|button[-_ ]?primary|cta|call[-_ ]?to[-_ ]?action)\\b/i.test(buttonDescriptor);
4455
+ var actionTextEvidence = /\\b(?:get started|start|try|book|contact|schedule|request|sign up|join|buy|shop|call|quote|estimate|verify)\\b/i.test(buttonDescriptor);
4456
+ if (!explicitButtonEvidence && !actionTextEvidence && (buttonRect.height > 100 || buttonText.length > 80)) continue;
4457
+ var score = 0;
4458
+ if (isChromatic(buttonBg)) score += 35;
4459
+ if (explicitButtonEvidence) score += 30;
4460
+ if (actionTextEvidence) score += 20;
4461
+ if (candidateButton.closest('main, [role="main"], [class*="hero"]')) score += 12;
4462
+ if (candidateButton.closest('header, nav, [role="banner"]')) score += 4;
4463
+ score += Math.min(10, Math.round((buttonRect.width * buttonRect.height) / 5000));
4464
+ if (score > btnScore) { btnEl = candidateButton; btnScore = score; }
4465
+ }
4272
4466
  var navStyle = cs(navEl);
4273
4467
  var bodyStyle = cs(bodyEl);
4274
4468
  var h1Style = cs(h1El);
4275
4469
  var btnStyle = cs(btnEl);
4276
4470
 
4277
4471
  var svgFreq = {};
4278
- var svgScope = navEl || document.querySelector('header, [role="banner"]');
4472
+ var svgScope = document;
4279
4473
  if (svgScope) {
4280
- var svgEls = svgScope.querySelectorAll('svg *, [fill], path, circle, rect, polygon, polyline');
4474
+ var svgEls = svgScope.querySelectorAll('svg *, svg[fill], svg path, svg circle, svg rect, svg polygon, svg polyline');
4281
4475
  for (var si = 0; si < svgEls.length; si++) {
4282
4476
  var svgEl = svgEls[si];
4477
+ var owningSvg = svgEl.closest('svg');
4478
+ var owningRect = owningSvg ? owningSvg.getBoundingClientRect() : null;
4479
+ if (!owningRect || owningRect.bottom < 0 || owningRect.top > 300 || owningRect.width < 12 || owningRect.height < 12) continue;
4283
4480
  var fillComp = cs(svgEl) ? cs(svgEl).fill : null;
4284
4481
  var fillAttr = svgEl.getAttribute('fill');
4285
4482
  var fillHex = null;
@@ -4301,55 +4498,251 @@ async function extractBrandingFromPage(page) {
4301
4498
  }
4302
4499
  }
4303
4500
 
4501
+ var cssPrimaryCandidates = [];
4502
+ var cssAccentCandidates = [];
4503
+ var rootStyle = cs(document.documentElement);
4504
+ if (rootStyle) {
4505
+ for (var ci = 0; ci < rootStyle.length; ci++) {
4506
+ var propertyName = rootStyle[ci];
4507
+ if (!propertyName || propertyName.indexOf('--') !== 0) continue;
4508
+ var normalizedName = propertyName.toLowerCase();
4509
+ if (!/(?:^|[-_])(?:brand|primary|accent)(?:$|[-_])/.test(normalizedName)) continue;
4510
+ if (/(?:on[-_]?primary|primary[-_]?(?:text|foreground)|accent[-_]?(?:text|foreground))/.test(normalizedName)) continue;
4511
+ var propertyValue = rootStyle.getPropertyValue(propertyName).trim();
4512
+ var propertyHex = toHex(propertyValue);
4513
+ if (!propertyHex || !isChromatic(propertyHex)) continue;
4514
+ var item = { hex: propertyHex, name: propertyName };
4515
+ if (/(?:^|[-_])primary(?:$|[-_])|brand/.test(normalizedName)) cssPrimaryCandidates.push(item);
4516
+ if (/(?:^|[-_])accent(?:$|[-_])/.test(normalizedName)) cssAccentCandidates.push(item);
4517
+ }
4518
+ }
4519
+
4520
+ var themeColorEl = document.querySelector('meta[name="theme-color"]');
4521
+ var themeColor = themeColorEl ? toHex(themeColorEl.getAttribute('content')) : null;
4522
+
4523
+ var visibleBgFreq = {};
4524
+ var topBgFreq = {};
4525
+ var visibleElements = document.querySelectorAll('body *');
4526
+ for (var vi = 0; vi < visibleElements.length && vi < 1500; vi++) {
4527
+ var visibleElement = visibleElements[vi];
4528
+ var visibleElementRect = visibleRect(visibleElement);
4529
+ if (!visibleElementRect || visibleElementRect.width < 20 || visibleElementRect.height < 12) continue;
4530
+ var visibleStyle = cs(visibleElement);
4531
+ var visibleHex = visibleStyle ? toHex(visibleStyle.backgroundColor) : null;
4532
+ if (!visibleHex || !isChromatic(visibleHex)) continue;
4533
+ var areaWeight = Math.max(1, Math.min(20, Math.round(Math.sqrt(visibleElementRect.width * visibleElementRect.height) / 40)));
4534
+ visibleBgFreq[visibleHex] = (visibleBgFreq[visibleHex] || 0) + areaWeight;
4535
+ if (visibleElementRect.top < 300 && visibleElementRect.bottom > 0 && visibleElementRect.width >= window.innerWidth * 0.5) {
4536
+ topBgFreq[visibleHex] = (topBgFreq[visibleHex] || 0) + areaWeight;
4537
+ }
4538
+ }
4539
+
4304
4540
  var pageHost = window.location.hostname.replace(/^www\\./, '');
4305
4541
  function isSameDomain(src) {
4306
- try { return new URL(src).hostname.replace(/^www\\./, '').endsWith(pageHost); } catch { return false; }
4542
+ try {
4543
+ var host = new URL(src).hostname.replace(/^www\\./, '');
4544
+ return host === pageHost || host.endsWith('.' + pageHost);
4545
+ } catch { return false; }
4307
4546
  }
4308
- var logoSelectors = [
4309
- 'header img[class*="logo"]', 'nav img[class*="logo"]',
4310
- 'img[class*="site-logo"]', 'img[class*="brand-logo"]',
4311
- 'img[alt*="logo" i]', '.logo img', '#logo img',
4312
- 'header img:first-of-type', 'nav img:first-of-type'
4313
- ];
4314
- var logoSrc = null;
4315
- for (var i = 0; i < logoSelectors.length; i++) {
4316
- var el = document.querySelector(logoSelectors[i]);
4317
- if (el && el.src && isSameDomain(el.src)) { logoSrc = el.src; break; }
4318
- }
4319
- if (!logoSrc) {
4320
- for (var j = 0; j < logoSelectors.length; j++) {
4321
- var el2 = document.querySelector(logoSelectors[j]);
4322
- if (el2 && el2.src) { logoSrc = el2.src; break; }
4547
+ var logoCandidates = {};
4548
+ function addLogo(raw, region, evidence, score, alt, width, height) {
4549
+ if (!raw || raw.startsWith('data:') || raw.startsWith('blob:')) return;
4550
+ var url;
4551
+ try { url = new URL(raw, document.baseURI); } catch (_) { return; }
4552
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') return;
4553
+ url.hash = '';
4554
+ var key = url.href;
4555
+ var existing = logoCandidates[key] || {
4556
+ url: key, score: 0, region: region, evidence: [], alt: null, width: null, height: null
4557
+ };
4558
+ existing.score = Math.max(existing.score, score);
4559
+ existing.evidence = Array.from(new Set(existing.evidence.concat(evidence)));
4560
+ if (region === 'json_ld' || (existing.region === 'body' && region !== 'body')) existing.region = region;
4561
+ if (alt && !existing.alt) existing.alt = alt.replace(/\\s+/g, ' ').trim().slice(0, 300) || null;
4562
+ if ((width || 0) * (height || 0) > (existing.width || 0) * (existing.height || 0)) {
4563
+ existing.width = width || null; existing.height = height || null;
4564
+ }
4565
+ logoCandidates[key] = existing;
4566
+ }
4567
+
4568
+ function jsonLogo(value) {
4569
+ if (typeof value === 'string') return value;
4570
+ if (!value || typeof value !== 'object') return null;
4571
+ return typeof value.url === 'string' ? value.url : typeof value.contentUrl === 'string' ? value.contentUrl : null;
4572
+ }
4573
+ var jsonScripts = document.querySelectorAll('script[type="application/ld+json"]');
4574
+ for (var ji = 0; ji < jsonScripts.length; ji++) {
4575
+ try {
4576
+ var parsed = JSON.parse(jsonScripts[ji].textContent || 'null');
4577
+ var stack = Array.isArray(parsed) ? parsed.slice() : [parsed];
4578
+ var visited = 0;
4579
+ while (stack.length && visited++ < 1000) {
4580
+ var node = stack.pop();
4581
+ if (!node || typeof node !== 'object') continue;
4582
+ var types = Array.isArray(node['@type']) ? node['@type'] : [node['@type']];
4583
+ var entity = types.some(function(t) { return /^(Organization|Corporation|LocalBusiness|WebSite|Brand)$/i.test(String(t || '')); });
4584
+ var logo = entity ? jsonLogo(node.logo) : null;
4585
+ if (logo) addLogo(logo, 'json_ld', ['json_ld_logo'], 100, null, null, null);
4586
+ Object.keys(node).forEach(function(key) {
4587
+ if (key === '@context') return;
4588
+ var child = node[key];
4589
+ if (Array.isArray(child)) stack = stack.concat(child.filter(function(value) { return value && typeof value === 'object'; }));
4590
+ else if (child && typeof child === 'object') stack.push(child);
4591
+ });
4592
+ }
4593
+ } catch (_) {}
4594
+ }
4595
+
4596
+ var proofCandidates = {};
4597
+ var pageHeadings = Array.from(document.querySelectorAll('h1,h2,h3,h4,h5,h6'));
4598
+ function nearestHeadingText(element) {
4599
+ var nearest = null;
4600
+ for (var hi = 0; hi < pageHeadings.length; hi++) {
4601
+ if (pageHeadings[hi].compareDocumentPosition(element) & 4) nearest = pageHeadings[hi];
4602
+ else break;
4603
+ }
4604
+ return nearest && nearest.textContent ? nearest.textContent.replace(/\\s+/g, ' ').trim().slice(0, 300) : null;
4605
+ }
4606
+ function proofTypeFor(descriptor, context) {
4607
+ if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation))\\b/i.test(descriptor)) return 'accreditation';
4608
+ if (/\\b(?:certifi(?:ed|cation)|compliance|legitscript|hipaa)\\b/i.test(descriptor)) return 'certification';
4609
+ if (/\\b(?:award|winner|best of|top[-_ ]?rated)\\b/i.test(descriptor)) return 'award';
4610
+ if (/\\b(?:member|membership|association|professionals?)\\b/i.test(descriptor)) return 'membership';
4611
+ if (/\\b(?:press|featured|as seen|mentioned)\\b/i.test(descriptor)) return 'press_mention';
4612
+ if (/\\b(?:partner|customer|client|trusted by|used by|works with)\\b/i.test(descriptor)) return 'partner_or_customer';
4613
+ if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation))\\b/i.test(context || '')) return 'accreditation';
4614
+ if (/\\b(?:certifi(?:ed|cation)|compliance)\\b/i.test(context || '')) return 'certification';
4615
+ if (/\\b(?:award|winner|best of|top[-_ ]?rated)\\b/i.test(context || '')) return 'award';
4616
+ if (/\\b(?:member|membership|association)\\b/i.test(context || '')) return 'membership';
4617
+ if (/\\b(?:press|featured|as seen|mentioned)\\b/i.test(context || '')) return 'press_mention';
4618
+ if (/\\b(?:partner|customer|client|trusted by|used by|works with)\\b/i.test(context || '')) return 'partner_or_customer';
4619
+ return 'trust_mark';
4620
+ }
4621
+ function addProof(raw, proofType, score, evidence, alt, context, width, height) {
4622
+ if (!raw || raw.startsWith('data:') || raw.startsWith('blob:')) return;
4623
+ var url;
4624
+ try { url = new URL(raw, document.baseURI); } catch (_) { return; }
4625
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') return;
4626
+ url.hash = '';
4627
+ var key = url.href;
4628
+ var existing = proofCandidates[key];
4629
+ if (!existing || score > existing.score) {
4630
+ proofCandidates[key] = {
4631
+ url: key, proofType: proofType, score: score, confidence: 'low',
4632
+ evidence: Array.from(new Set(evidence)), alt: alt || null, context: context || null,
4633
+ width: width || null, height: height || null
4634
+ };
4635
+ }
4636
+ }
4637
+
4638
+ var imageEls = document.querySelectorAll('img');
4639
+ for (var li = 0; li < imageEls.length; li++) {
4640
+ var image = imageEls[li];
4641
+ var src = image.currentSrc || image.src || image.getAttribute('data-src') || image.getAttribute('data-lazy-src');
4642
+ if (!src) continue;
4643
+ var header = image.closest('header,[role="banner"]');
4644
+ var nav = image.closest('nav,[role="navigation"]');
4645
+ var footer = image.closest('footer,[role="contentinfo"]');
4646
+ var region = header ? 'header' : nav ? 'nav' : footer ? 'footer' : 'body';
4647
+ var evidence = [];
4648
+ var score = 0;
4649
+ if (header) { score += 35; evidence.push('header_or_banner'); }
4650
+ else if (nav) { score += 28; evidence.push('navigation'); }
4651
+ else if (footer) { score += 18; evidence.push('footer_or_contentinfo'); }
4652
+ var descriptor = [src, image.id, image.className, image.alt, image.getAttribute('itemprop'), image.parentElement && image.parentElement.className].join(' ');
4653
+ var identityDescriptor = [image.id, image.className, image.getAttribute('itemprop'), image.parentElement && image.parentElement.className].join(' ');
4654
+ if (/\\b(?:site[-_ ]?logo|brand[-_ ]?logo|custom-logo|navbar[-_ ]?brand|site[-_ ]?branding|site[-_ ]?identity|logo[-_ ]?link)\\b/i.test(identityDescriptor)) { score += 30; evidence.push('site_identity_marker'); }
4655
+ else if (/\\blogo\\b/i.test(descriptor)) { score += 12; evidence.push('generic_logo_marker'); }
4656
+ var anchor = image.closest('a[href]');
4657
+ if (anchor) {
4658
+ try {
4659
+ var destination = new URL(anchor.href, document.baseURI);
4660
+ if (destination.origin === location.origin && destination.pathname.replace(/\\/+$/, '') === '') {
4661
+ score += 22; evidence.push('links_to_homepage');
4662
+ }
4663
+ } catch (_) {}
4664
+ }
4665
+ if (isSameDomain(src)) { score += 8; evidence.push('same_domain'); }
4666
+ var style = cs(image);
4667
+ var visible = !!style && style.display !== 'none' && style.visibility !== 'hidden' && parseFloat(style.opacity || '1') > 0;
4668
+ if (visible) { score += 5; evidence.push('visible'); }
4669
+ var width = image.naturalWidth || image.width || null;
4670
+ var height = image.naturalHeight || image.height || null;
4671
+ if (width && height && width >= 80 && height >= 24) { score += 5; evidence.push('plausible_logo_dimensions'); }
4672
+ if ((width && width < 32) || (height && height < 16)) { score -= 45; evidence.push('tiny_asset_penalty'); }
4673
+ if (/\\b(?:icon|avatar|profile|search|menu|hamburger|social|badge)\\b/i.test(descriptor)) { score -= 35; evidence.push('icon_or_avatar_penalty'); }
4674
+ addLogo(src, region, evidence, score, image.alt || null, width, height);
4675
+
4676
+ if (region === 'body' && visible && width && height && width >= 80 && height >= 40) {
4677
+ var context = nearestHeadingText(image);
4678
+ var proofDescriptor = [src, image.alt, image.getAttribute('title')].filter(Boolean).join(' ');
4679
+ var proofEvidence = [];
4680
+ var proofScore = 0;
4681
+ var commercialContext = !!context && /\\b(?:insurances?|payment|financing|we accept|accepted health)\\b/i.test(context);
4682
+ if (/\\b(?:logo|seal|trust[-_ ]?mark|certification[-_ ]?mark)\\b/i.test(proofDescriptor)) { proofScore += 30; proofEvidence.push('proof_asset_marker'); }
4683
+ if (/\\b(?:joint[-_ ]commission|accredit(?:ed|ation)|certifi(?:ed|cation)|compliance|legitscript|hipaa|award|winner|best of|top[-_ ]?rated|member|membership|association|professionals?|press|featured|as seen|mentioned|partner|customer|client|trusted by|used by|works with)\\b/i.test(proofDescriptor)) { proofScore += 30; proofEvidence.push('proof_descriptor'); }
4684
+ if (context && /\\b(?:recognized|accredit(?:ed|ation)|certifi(?:ed|cation)|award|winner|member|membership|association|featured|as seen|mentioned|partner|customer|client|trusted|trust)\\b/i.test(context)) { proofScore += 25; proofEvidence.push('proof_section_context'); }
4685
+ if (commercialContext) { proofScore -= 100; proofEvidence.push('commercial_acceptance_context_exclusion'); }
4686
+ proofScore += 10; proofEvidence.push('proof_dimensions');
4687
+ proofScore += 5; proofEvidence.push('visible');
4688
+ if (isSameDomain(src)) { proofScore += 5; proofEvidence.push('same_domain'); }
4689
+ else { proofScore += 3; proofEvidence.push('external_asset_domain'); }
4690
+ if (!commercialContext && proofScore >= 45) addProof(src, proofTypeFor(proofDescriptor, context), proofScore, proofEvidence, image.alt || null, context, width, height);
4323
4691
  }
4324
4692
  }
4325
4693
  var faviconEl = document.querySelector(
4326
4694
  'link[rel~="icon"], link[rel="shortcut icon"], link[rel="apple-touch-icon"]'
4327
4695
  );
4696
+ if (faviconEl && faviconEl.href) addLogo(faviconEl.href, 'favicon', ['favicon_fallback_only'], 10, null, null, null);
4697
+ var rankedLogos = Object.values(logoCandidates).filter(function(candidate) { return candidate.region !== 'body'; }).sort(function(a, b) { return b.score - a.score || a.url.localeCompare(b.url); }).slice(0, 20);
4698
+ for (var ri = 0; ri < rankedLogos.length; ri++) {
4699
+ rankedLogos[ri].confidence = rankedLogos[ri].score >= 85 ? 'high' : rankedLogos[ri].score >= 60 ? 'medium' : 'low';
4700
+ }
4701
+ var primaryLogo = rankedLogos.length && rankedLogos[0].score >= 55 && rankedLogos[0].region !== 'favicon' ? rankedLogos[0] : null;
4328
4702
  return {
4329
- navBg: navStyle ? navStyle.backgroundColor : null,
4330
- bodyBg: bodyStyle ? bodyStyle.backgroundColor : null,
4331
- bodyColor: bodyStyle ? bodyStyle.color : null,
4332
- h1Color: h1Style ? h1Style.color : null,
4333
- btnBg: btnStyle ? btnStyle.backgroundColor : null,
4703
+ navBg: navStyle ? toHex(navStyle.backgroundColor) : null,
4704
+ bodyBg: (bodyStyle ? toHex(bodyStyle.backgroundColor) : null) || (cs(document.documentElement) ? toHex(cs(document.documentElement).backgroundColor) : null) || '#ffffff',
4705
+ bodyColor: bodyStyle ? toHex(bodyStyle.color) : null,
4706
+ h1Color: h1Style ? toHex(h1Style.color) : null,
4707
+ btnBg: btnStyle ? toHex(btnStyle.backgroundColor) : null,
4708
+ btnDetail: btnEl ? String(btnEl.textContent || btnEl.getAttribute('aria-label') || '').replace(/\\s+/g, ' ').trim().slice(0, 120) : null,
4334
4709
  bodyFont: bodyStyle ? bodyStyle.fontFamily : null,
4335
4710
  h1Font: h1Style ? h1Style.fontFamily : null,
4336
- logoSrc: logoSrc,
4711
+ logoSrc: primaryLogo ? primaryLogo.url : null,
4712
+ logoConfidence: primaryLogo ? primaryLogo.confidence : null,
4713
+ logoSelectionReason: primaryLogo ? primaryLogo.evidence.join(', ') : null,
4714
+ logoCandidates: rankedLogos,
4715
+ proofCandidates: Object.values(proofCandidates),
4337
4716
  faviconHref: faviconEl ? faviconEl.href : null,
4338
4717
  svgFreq: svgFreq,
4339
4718
  navChildBgFreq: navChildBgFreq,
4719
+ visibleBgFreq: visibleBgFreq,
4720
+ topBgFreq: topBgFreq,
4721
+ cssPrimaryCandidates: cssPrimaryCandidates,
4722
+ cssAccentCandidates: cssAccentCandidates,
4723
+ themeColor: themeColor,
4340
4724
  };
4341
4725
  })()
4342
4726
  `;
4343
4727
  const raw = await page.evaluate(evalScript);
4344
- const navBgHex = rgbToHex(raw.navBg ?? "");
4345
- const bodyBgHex = rgbToHex(raw.bodyBg ?? "");
4728
+ const logoSelection = selectRankedLogoCandidates(raw.logoCandidates ?? []);
4729
+ const navBgHex = raw.navBg;
4730
+ const bodyBgHex = raw.bodyBg;
4346
4731
  const navBgUsable = navBgHex && !isTransparentOrWhite(navBgHex) && navBgHex !== "#000000" && navBgHex !== "#020101" ? navBgHex : null;
4347
- const svgPrimary = dominantColor(raw.svgFreq ?? {});
4732
+ const svgPrimary = dominantChromaticColor(raw.svgFreq ?? {});
4348
4733
  const navChildBg = dominantColor(raw.navChildBgFreq ?? {});
4349
- const primary = navBgUsable ?? svgPrimary ?? navChildBg ?? bodyBgHex;
4350
- const accent = rgbToHex(raw.btnBg ?? "");
4351
- const text2 = rgbToHex(raw.bodyColor ?? "");
4352
- const heading = rgbToHex(raw.h1Color ?? "");
4734
+ const visibleBg = dominantColor(raw.visibleBgFreq ?? {});
4735
+ const topBg = dominantColor(raw.topBgFreq ?? {});
4736
+ const cssPrimary = raw.cssPrimaryCandidates?.[0] ?? null;
4737
+ const cssAccent = raw.cssAccentCandidates?.[0] ?? null;
4738
+ const themeColor = isChromatic(raw.themeColor) ? raw.themeColor : null;
4739
+ const ctaColor = isChromatic(raw.btnBg) ? raw.btnBg : null;
4740
+ const primaryChoice = cssPrimary ? { value: cssPrimary.hex, source: "css_variable", confidence: "high", detail: cssPrimary.name } : themeColor ? { value: themeColor, source: "theme_color", confidence: "high", detail: 'meta[name="theme-color"]' } : navBgUsable && isChromatic(navBgUsable) ? { value: navBgUsable, source: "navigation_background", confidence: "medium", detail: "nav, header, or banner computed background" } : topBg && isChromatic(topBg) ? { value: topBg, source: "navigation_background", confidence: "medium", detail: "dominant wide rendered background near the top of the page" } : svgPrimary && isChromatic(svgPrimary) ? { value: svgPrimary, source: "header_svg", confidence: "medium", detail: "dominant rendered identity SVG fill near the top of the page" } : navChildBg && isChromatic(navChildBg) ? { value: navChildBg, source: "navigation_background", confidence: "medium", detail: "dominant rendered navigation child background" } : ctaColor ? { value: ctaColor, source: "visible_cta", confidence: "medium", detail: raw.btnDetail } : visibleBg && isChromatic(visibleBg) ? { value: visibleBg, source: "visible_background", confidence: "low", detail: "area-weighted rendered element background" } : bodyBgHex && !isTransparentOrWhite(bodyBgHex) ? { value: bodyBgHex, source: "body_background", confidence: "low", detail: "computed body background fallback" } : null;
4741
+ const accentChoice = cssAccent ? { value: cssAccent.hex, source: "css_variable", confidence: "high", detail: cssAccent.name } : ctaColor ? { value: ctaColor, source: "visible_cta", confidence: "medium", detail: raw.btnDetail } : null;
4742
+ const primary = primaryChoice?.value ?? null;
4743
+ const accent = accentChoice?.value ?? null;
4744
+ const text2 = raw.bodyColor;
4745
+ const heading = raw.h1Color;
4353
4746
  const background = bodyBgHex;
4354
4747
  const scheme = background ? luminance(background) > 128 ? "light" : "dark" : null;
4355
4748
  return {
@@ -4361,22 +4754,208 @@ async function extractBrandingFromPage(page) {
4361
4754
  text: text2 ?? null,
4362
4755
  heading: heading ?? null
4363
4756
  },
4757
+ colorEvidence: {
4758
+ primary: primaryChoice,
4759
+ accent: accentChoice,
4760
+ background: background ? { value: background, source: "body_background", confidence: "high", detail: "computed body background" } : null,
4761
+ text: text2 ? { value: text2, source: "body_text", confidence: "high", detail: "computed body text color" } : null,
4762
+ heading: heading ? { value: heading, source: "heading_text", confidence: "high", detail: "computed first H1 text color" } : null
4763
+ },
4364
4764
  fonts: {
4365
4765
  body: firstFont(raw.bodyFont ?? ""),
4366
4766
  heading: firstFont(raw.h1Font ?? "")
4367
4767
  },
4368
4768
  assets: {
4369
- logo: raw.logoSrc || null,
4370
- favicon: raw.faviconHref || null
4769
+ logo: logoSelection.primary?.url ?? null,
4770
+ favicon: raw.faviconHref || null,
4771
+ logoConfidence: logoSelection.primary?.confidence ?? null,
4772
+ logoSelectionReason: logoSelection.selectionReason,
4773
+ logoVariants: logoSelection.variants,
4774
+ logoCandidates: logoSelection.candidates,
4775
+ proofImages: selectProofImages(raw.proofCandidates ?? [])
4371
4776
  }
4372
4777
  };
4373
4778
  }
4374
4779
 
4780
+ // src/lib/rendered-media.ts
4781
+ var SETTLE_MS = 450;
4782
+ var STABLE_CHECKS_REQUIRED = 3;
4783
+ var MAX_SCROLL_ROUNDS = 40;
4784
+ var MAX_DOM_NODES = 15e3;
4785
+ function mergeAsset(target, incoming) {
4786
+ const existing = target.get(incoming.url);
4787
+ if (!existing) {
4788
+ target.set(incoming.url, incoming);
4789
+ return;
4790
+ }
4791
+ existing.discoveryMethods = [.../* @__PURE__ */ new Set([...existing.discoveryMethods, ...incoming.discoveryMethods])];
4792
+ existing.altTexts = [.../* @__PURE__ */ new Set([...existing.altTexts, ...incoming.altTexts])];
4793
+ existing.contexts = [.../* @__PURE__ */ new Set([...existing.contexts, ...incoming.contexts])];
4794
+ if ((incoming.width ?? 0) * (incoming.height ?? 0) > (existing.width ?? 0) * (existing.height ?? 0)) {
4795
+ existing.width = incoming.width;
4796
+ existing.height = incoming.height;
4797
+ }
4798
+ }
4799
+ async function readRenderedSnapshot(page) {
4800
+ await page.evaluate("globalThis.__name = globalThis.__name || ((target) => target)");
4801
+ return page.evaluate(({ maxDomNodes }) => {
4802
+ const rows = /* @__PURE__ */ new Map();
4803
+ const resolve2 = (raw) => {
4804
+ if (!raw || raw.startsWith("data:") || raw.startsWith("blob:")) return null;
4805
+ try {
4806
+ const parsed = new URL(raw, document.baseURI);
4807
+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
4808
+ parsed.hash = "";
4809
+ return parsed.href;
4810
+ } catch {
4811
+ return null;
4812
+ }
4813
+ };
4814
+ const contextFor = (element) => {
4815
+ const contexts = [];
4816
+ const section2 = element.closest('section,article,main,header,footer,aside,[role="region"]');
4817
+ if (section2?.id) contexts.push(`#${section2.id}`);
4818
+ const labelled = section2?.getAttribute("aria-label")?.trim();
4819
+ if (labelled) contexts.push(labelled.slice(0, 200));
4820
+ const heading = section2?.querySelector("h1,h2,h3,h4,h5,h6")?.textContent?.replace(/\s+/g, " ").trim();
4821
+ if (heading) contexts.push(heading.slice(0, 200));
4822
+ return contexts;
4823
+ };
4824
+ const add = (raw, type, method, element, dimensions, alt) => {
4825
+ const url = resolve2(raw);
4826
+ if (!url) return;
4827
+ const existing = rows.get(url) ?? {
4828
+ url,
4829
+ type,
4830
+ discoveryMethods: [],
4831
+ altTexts: [],
4832
+ contexts: [],
4833
+ width: null,
4834
+ height: null
4835
+ };
4836
+ existing.discoveryMethods = [.../* @__PURE__ */ new Set([...existing.discoveryMethods, method])];
4837
+ const cleanAlt = alt?.replace(/\s+/g, " ").trim();
4838
+ if (cleanAlt) existing.altTexts = [.../* @__PURE__ */ new Set([...existing.altTexts, cleanAlt.slice(0, 500)])];
4839
+ existing.contexts = [.../* @__PURE__ */ new Set([...existing.contexts, ...contextFor(element)])];
4840
+ if (dimensions && dimensions.width > 0 && dimensions.height > 0) {
4841
+ const currentArea = (existing.width ?? 0) * (existing.height ?? 0);
4842
+ if (dimensions.width * dimensions.height > currentArea) {
4843
+ existing.width = dimensions.width;
4844
+ existing.height = dimensions.height;
4845
+ }
4846
+ }
4847
+ rows.set(url, existing);
4848
+ };
4849
+ const srcsetUrls = (value) => (value ?? "").split(",").map((part) => part.trim().split(/\s+/)[0] ?? "").filter(Boolean);
4850
+ for (const img of Array.from(document.querySelectorAll("img"))) {
4851
+ const image = img;
4852
+ const dimensions = { width: image.naturalWidth, height: image.naturalHeight };
4853
+ add(image.currentSrc, "image", "rendered-current-src", image, dimensions, image.alt);
4854
+ add(image.getAttribute("src"), "image", "rendered-src", image, dimensions, image.alt);
4855
+ for (const attr of ["data-src", "data-lazy-src", "data-original", "data-bg", "data-background"]) {
4856
+ add(image.getAttribute(attr), "image", `rendered-${attr}`, image, dimensions, image.alt);
4857
+ }
4858
+ for (const url of srcsetUrls(image.getAttribute("srcset"))) add(url, "image", "rendered-srcset", image, dimensions, image.alt);
4859
+ const anchor = image.closest("a[href]");
4860
+ const anchorAttrs = anchor ? Array.from(anchor.attributes).map((attribute) => `${attribute.name}=${attribute.value}`).join(" ") : "";
4861
+ if (anchor && (/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(anchorAttrs) || /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(anchor.href))) add(anchor.href, "image", "rendered-lightbox-target", image, void 0, image.alt);
4862
+ }
4863
+ for (const source of Array.from(document.querySelectorAll("source"))) {
4864
+ const parent = source.closest("picture,video,audio");
4865
+ const type = parent?.tagName === "VIDEO" ? "video" : parent?.tagName === "AUDIO" ? "audio" : "image";
4866
+ add(source.getAttribute("src"), type, "rendered-source", source);
4867
+ for (const url of srcsetUrls(source.getAttribute("srcset"))) add(url, type, "rendered-source-srcset", source);
4868
+ }
4869
+ for (const video of Array.from(document.querySelectorAll("video"))) {
4870
+ const media = video;
4871
+ add(media.currentSrc || media.src, "video", "rendered-video", media);
4872
+ add(media.poster, "image", "rendered-video-poster", media, { width: media.videoWidth, height: media.videoHeight });
4873
+ }
4874
+ for (const audio of Array.from(document.querySelectorAll("audio"))) {
4875
+ const media = audio;
4876
+ add(media.currentSrc || media.src, "audio", "rendered-audio", media);
4877
+ }
4878
+ for (const svgImage of Array.from(document.querySelectorAll("svg image"))) {
4879
+ add(svgImage.getAttribute("href") || svgImage.getAttribute("xlink:href"), "image", "rendered-svg-image", svgImage);
4880
+ }
4881
+ const all = Array.from(document.querySelectorAll("*")).slice(0, maxDomNodes);
4882
+ for (const element of all) {
4883
+ const background = getComputedStyle(element).backgroundImage;
4884
+ if (!background || background === "none") continue;
4885
+ for (const match of background.matchAll(/url\(["']?(.+?)["']?\)/gi)) {
4886
+ add(match[1], "image", "rendered-computed-background", element);
4887
+ }
4888
+ }
4889
+ const documentHeight = Math.max(document.body?.scrollHeight ?? 0, document.documentElement?.scrollHeight ?? 0);
4890
+ const viewportHeight = window.innerHeight;
4891
+ return {
4892
+ assets: [...rows.values()],
4893
+ documentHeight,
4894
+ scrollY: window.scrollY,
4895
+ viewportHeight,
4896
+ atBottom: window.scrollY + viewportHeight >= documentHeight - 5
4897
+ };
4898
+ }, { maxDomNodes: MAX_DOM_NODES });
4899
+ }
4900
+ async function collectRenderedPageMedia(page, maxAssets = 1e3) {
4901
+ const boundedMax = Math.max(1, Math.min(maxAssets, 2e3));
4902
+ await page.evaluate(() => {
4903
+ for (const image of Array.from(document.querySelectorAll('img[loading="lazy"]'))) image.loading = "eager";
4904
+ }).catch(() => void 0);
4905
+ const assets = /* @__PURE__ */ new Map();
4906
+ let stableChecks = 0;
4907
+ let previousHeight = -1;
4908
+ let rounds = 0;
4909
+ let documentHeight = 0;
4910
+ while (rounds < MAX_SCROLL_ROUNDS) {
4911
+ rounds += 1;
4912
+ const before = assets.size;
4913
+ const snapshot = await readRenderedSnapshot(page);
4914
+ documentHeight = snapshot.documentHeight;
4915
+ for (const asset of snapshot.assets) mergeAsset(assets, asset);
4916
+ if (assets.size >= boundedMax) {
4917
+ return {
4918
+ assets: [...assets.values()].slice(0, boundedMax),
4919
+ exhausted: false,
4920
+ stopReason: "asset_limit",
4921
+ scrollRounds: rounds,
4922
+ documentHeight
4923
+ };
4924
+ }
4925
+ const unchanged = snapshot.atBottom && before === assets.size && previousHeight === snapshot.documentHeight;
4926
+ stableChecks = unchanged ? stableChecks + 1 : 0;
4927
+ previousHeight = snapshot.documentHeight;
4928
+ if (stableChecks >= STABLE_CHECKS_REQUIRED) {
4929
+ await page.evaluate(() => window.scrollTo(0, 0)).catch(() => void 0);
4930
+ return {
4931
+ assets: [...assets.values()],
4932
+ exhausted: true,
4933
+ stopReason: "page_exhausted",
4934
+ scrollRounds: rounds,
4935
+ documentHeight
4936
+ };
4937
+ }
4938
+ await page.evaluate(({ viewportHeight, documentHeight: height }) => {
4939
+ const next = Math.min(Math.max(0, height - viewportHeight), window.scrollY + Math.max(800, Math.floor(viewportHeight * 0.9)));
4940
+ window.scrollTo(0, next);
4941
+ }, { viewportHeight: snapshot.viewportHeight, documentHeight: snapshot.documentHeight });
4942
+ await page.waitForTimeout(SETTLE_MS);
4943
+ }
4944
+ await page.evaluate(() => window.scrollTo(0, 0)).catch(() => void 0);
4945
+ return {
4946
+ assets: [...assets.values()],
4947
+ exhausted: false,
4948
+ stopReason: "scroll_round_limit",
4949
+ scrollRounds: rounds,
4950
+ documentHeight
4951
+ };
4952
+ }
4953
+
4375
4954
  // src/lib/screenshot.ts
4376
4955
  var DESKTOP_VIEWPORT = { width: 1440, height: 900 };
4377
4956
  var MOBILE_VIEWPORT = { width: 390, height: 844 };
4378
4957
  var SCROLL_STEP = 900;
4379
- var SETTLE_MS = 400;
4958
+ var SETTLE_MS2 = 400;
4380
4959
  async function scrollAndStitch(page, device = "desktop") {
4381
4960
  const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
4382
4961
  await page.setViewportSize(vp);
@@ -4392,7 +4971,7 @@ async function scrollAndStitch(page, device = "desktop") {
4392
4971
  let scrollY = 0;
4393
4972
  while (scrollY < totalH) {
4394
4973
  await page.evaluate((y) => window.scrollTo(0, y), scrollY);
4395
- await page.waitForTimeout(SETTLE_MS);
4974
+ await page.waitForTimeout(SETTLE_MS2);
4396
4975
  const remaining = totalH - scrollY;
4397
4976
  const stripH = Math.min(SCROLL_STEP, remaining);
4398
4977
  const buf = await page.screenshot({
@@ -4423,7 +5002,7 @@ async function scrollAndStitch(page, device = "desktop") {
4423
5002
  );
4424
5003
  return Buffer.from(base64, "base64");
4425
5004
  }
4426
- async function captureLocal(url, device, wantScreenshot, wantBranding) {
5005
+ async function captureLocal(url, device, wantScreenshot, wantBranding, wantMedia, maxMediaAssets) {
4427
5006
  const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
4428
5007
  const browser = await chromium2.launch({ headless: true });
4429
5008
  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
@@ -4434,17 +5013,18 @@ async function captureLocal(url, device, wantScreenshot, wantBranding) {
4434
5013
  }, runtimeUserAgent));
4435
5014
  const page = await context.newPage();
4436
5015
  try {
4437
- await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
5016
+ await page.goto(url, { waitUntil: "domcontentloaded", timeout: 6e4 });
4438
5017
  await page.waitForTimeout(1500);
4439
5018
  const result = {};
4440
5019
  if (wantBranding) result.branding = await extractBrandingFromPage(page);
5020
+ if (wantMedia) result.renderedMedia = await collectRenderedPageMedia(page, maxMediaAssets);
4441
5021
  if (wantScreenshot) result.screenshot = await scrollAndStitch(page, device);
4442
5022
  return result;
4443
5023
  } finally {
4444
5024
  await browser.close();
4445
5025
  }
4446
5026
  }
4447
- async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding) {
5027
+ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBranding, wantMedia, maxMediaAssets) {
4448
5028
  const kernel = new Kernel2({ apiKey: kernelApiKey });
4449
5029
  const openedAtMs = Date.now();
4450
5030
  const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
@@ -4457,10 +5037,11 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
4457
5037
  }, runtimeUserAgent));
4458
5038
  const page = context.pages()[0] ?? await context.newPage();
4459
5039
  try {
4460
- await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
5040
+ await page.goto(url, { waitUntil: "domcontentloaded", timeout: 6e4 });
4461
5041
  await page.waitForTimeout(1500);
4462
5042
  const result = {};
4463
5043
  if (wantBranding) result.branding = await extractBrandingFromPage(page);
5044
+ if (wantMedia) result.renderedMedia = await collectRenderedPageMedia(page, maxMediaAssets);
4464
5045
  if (wantScreenshot) result.screenshot = await scrollAndStitch(page, device);
4465
5046
  return result;
4466
5047
  } finally {
@@ -4480,12 +5061,14 @@ async function capturePageData(url, opts = {}) {
4480
5061
  const device = opts.device ?? "desktop";
4481
5062
  const wantShot = opts.screenshot ?? false;
4482
5063
  const wantBranding = opts.branding ?? false;
5064
+ const wantMedia = opts.media ?? false;
5065
+ const maxMediaAssets = opts.maxMediaAssets ?? 1e3;
4483
5066
  try {
4484
- return await captureLocal(url, device, wantShot, wantBranding);
5067
+ return await captureLocal(url, device, wantShot, wantBranding, wantMedia, maxMediaAssets);
4485
5068
  } catch (localErr) {
4486
5069
  const msg = localErr instanceof Error ? localErr.message : String(localErr);
4487
5070
  if (!opts.kernelApiKey) throw new Error(`Local capture failed (${msg}) and no headless browser backend is configured`);
4488
- return await captureKernel(url, opts.kernelApiKey, device, wantShot, wantBranding);
5071
+ return await captureKernel(url, opts.kernelApiKey, device, wantShot, wantBranding, wantMedia, maxMediaAssets);
4489
5072
  }
4490
5073
  }
4491
5074
 
@@ -4890,7 +5473,7 @@ function withTimeout(p, ms) {
4890
5473
  async function createBrowserWithRetry(client2, headless) {
4891
5474
  for (let attempt = 0; attempt < 3; attempt++) {
4892
5475
  try {
4893
- return await client2.browsers.create({ stealth: true, headless, timeout_seconds: 120 });
5476
+ return await client2.browsers.create({ stealth: true, ...headless ? { headless: true } : {}, timeout_seconds: 120 });
4894
5477
  } catch (err) {
4895
5478
  if (attempt === 2) throw err;
4896
5479
  await new Promise((r) => setTimeout(r, 5e3));
@@ -4899,7 +5482,7 @@ async function createBrowserWithRetry(client2, headless) {
4899
5482
  throw new Error("unreachable");
4900
5483
  }
4901
5484
  var PAGE_HARD_TIMEOUT_MS = 35e3;
4902
- async function crawlPage(context, url) {
5485
+ async function crawlPage(context, url, headless) {
4903
5486
  const startedAt = Date.now();
4904
5487
  let page = null;
4905
5488
  let guardState = null;
@@ -4908,13 +5491,38 @@ async function crawlPage(context, url) {
4908
5491
  guardState = await installBrowserRequestGuard(page, url);
4909
5492
  const work = (async (p) => {
4910
5493
  const resp = await p.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 });
4911
- const html = await p.evaluate((maxBytes) => {
5494
+ let html = await p.evaluate((maxBytes) => {
4912
5495
  const value = document.documentElement?.outerHTML ?? "";
4913
5496
  if (new TextEncoder().encode(value).byteLength > maxBytes) throw new Error(`Browser HTML exceeds ${maxBytes} byte limit`);
4914
5497
  return value;
4915
5498
  }, MAX_PAGE_HTML_BYTES);
4916
5499
  const headers = resp ? Object.fromEntries(Object.entries(await resp.headers()).map(([k, v]) => [k.toLowerCase(), v])) : void 0;
4917
5500
  const finalUrl = p.url();
5501
+ let challenge = detectBrowserChallenge({ status: resp?.status() ?? null, title: await p.title().catch(() => ""), html });
5502
+ if (challenge.detected && !headless) {
5503
+ const deadline = Date.now() + 2e4;
5504
+ while (challenge.detected && Date.now() < deadline) {
5505
+ await p.waitForTimeout(1e3);
5506
+ html = await p.content();
5507
+ challenge = detectBrowserChallenge({ status: resp?.status() ?? null, title: await p.title().catch(() => ""), html });
5508
+ }
5509
+ }
5510
+ if (challenge.detected) {
5511
+ return {
5512
+ url,
5513
+ html: null,
5514
+ status: resp?.status() ?? 0,
5515
+ via: "browser",
5516
+ headers,
5517
+ responseTimeMs: Date.now() - startedAt,
5518
+ redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null,
5519
+ errorCode: "browser_challenge_unresolved",
5520
+ error: `Browser challenge remained unresolved (${challenge.reason ?? "unknown challenge"}).`,
5521
+ browserMode: headless ? "stealth_headless" : "stealth_headful",
5522
+ challengeDetected: true,
5523
+ stopReason: "challenge_unresolved"
5524
+ };
5525
+ }
4918
5526
  return {
4919
5527
  url,
4920
5528
  html,
@@ -4922,7 +5530,10 @@ async function crawlPage(context, url) {
4922
5530
  via: "browser",
4923
5531
  headers,
4924
5532
  responseTimeMs: Date.now() - startedAt,
4925
- redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null
5533
+ redirectUrl: finalUrl && finalUrl !== url ? finalUrl : null,
5534
+ browserMode: headless ? "stealth_headless" : "stealth_headful",
5535
+ challengeDetected: false,
5536
+ stopReason: resp && (resp.status() < 200 || resp.status() >= 300) ? "http_error" : "usable_page"
4926
5537
  };
4927
5538
  })(page);
4928
5539
  const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("page-hard-timeout")), PAGE_HARD_TIMEOUT_MS));
@@ -4937,7 +5548,10 @@ async function crawlPage(context, url) {
4937
5548
  via: "browser",
4938
5549
  responseTimeMs: Date.now() - startedAt,
4939
5550
  errorCode,
4940
- error: message.slice(0, 500)
5551
+ error: message.slice(0, 500),
5552
+ browserMode: headless ? "stealth_headless" : "stealth_headful",
5553
+ challengeDetected: false,
5554
+ stopReason: "navigation_failed"
4941
5555
  };
4942
5556
  } finally {
4943
5557
  if (page) await page.close().catch(() => {
@@ -4951,7 +5565,7 @@ async function crawlWithRotation(urls, opts) {
4951
5565
  const headless = opts.headless ?? DEFAULT_HEADLESS;
4952
5566
  const uniqueUrls = [...new Set(urls)];
4953
5567
  const byUrl = /* @__PURE__ */ new Map();
4954
- const runPass = async (targets, phase) => {
5568
+ const runPass = async (targets, phase, passHeadless) => {
4955
5569
  if (targets.length === 0) return;
4956
5570
  let next = 0;
4957
5571
  let done = 0;
@@ -4974,7 +5588,7 @@ async function crawlWithRotation(urls, opts) {
4974
5588
  const closedAtMs = Date.now();
4975
5589
  await client2.browsers.deleteByID(k.session_id).catch(() => {
4976
5590
  });
4977
- await recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent: headless, proxyUsed: typeof k.proxy_id === "string" ? Boolean(k.proxy_id) : null, openedAtMs, closedAtMs });
5591
+ await recordKernelSession({ kernelSessionId: k.session_id, source: "rotating_crawl", stealth: typeof k.stealth === "boolean" ? k.stealth : null, headlessSent: passHeadless ? true : null, proxyUsed: typeof k.proxy_id === "string" ? Boolean(k.proxy_id) : null, fallback: phase === "retry", openedAtMs, closedAtMs });
4978
5592
  }
4979
5593
  };
4980
5594
  try {
@@ -4986,7 +5600,7 @@ async function crawlWithRotation(urls, opts) {
4986
5600
  try {
4987
5601
  if (!browser || inBrowser >= urlsPerBrowser) {
4988
5602
  await retire();
4989
- kb = await createBrowserWithRetry(client2, headless);
5603
+ kb = await createBrowserWithRetry(client2, passHeadless);
4990
5604
  kbOpenedAtMs = Date.now();
4991
5605
  browser = await withTimeout(chromium3.connectOverCDP(kb.cdp_ws_url), 3e4);
4992
5606
  runtimeUserAgent = await detectRuntimeUserAgent(browser);
@@ -4994,7 +5608,7 @@ async function crawlWithRotation(urls, opts) {
4994
5608
  }
4995
5609
  const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
4996
5610
  try {
4997
- r = await crawlPage(context, url);
5611
+ r = await crawlPage(context, url, passHeadless);
4998
5612
  } finally {
4999
5613
  await context.close().catch(() => {
5000
5614
  });
@@ -5008,7 +5622,10 @@ async function crawlWithRotation(urls, opts) {
5008
5622
  status: 0,
5009
5623
  via: "browser",
5010
5624
  errorCode: "browser_session_failed",
5011
- error: message.slice(0, 500)
5625
+ error: message.slice(0, 500),
5626
+ browserMode: passHeadless ? "stealth_headless" : "stealth_headful",
5627
+ challengeDetected: false,
5628
+ stopReason: "navigation_failed"
5012
5629
  };
5013
5630
  await retire();
5014
5631
  }
@@ -5032,12 +5649,12 @@ async function crawlWithRotation(urls, opts) {
5032
5649
  };
5033
5650
  await Promise.all(Array.from({ length: workers }, () => worker()));
5034
5651
  };
5035
- await runPass(uniqueUrls, "main");
5652
+ await runPass(uniqueUrls, "main", headless);
5036
5653
  const failed = uniqueUrls.filter((u) => {
5037
5654
  const r = byUrl.get(u);
5038
5655
  return !r || !r.html || r.status < 200 || r.status >= 300;
5039
5656
  });
5040
- if (failed.length > 0) await runPass(failed, "retry");
5657
+ if (failed.length > 0) await runPass(failed, "retry", headless ? false : headless);
5041
5658
  return urls.map((u) => byUrl.get(u) ?? {
5042
5659
  url: u,
5043
5660
  html: null,
@@ -5749,8 +6366,6 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5749
6366
  if (text2) headings.push({ level: Number(m[1]), text: text2.slice(0, 200) });
5750
6367
  }
5751
6368
  const h1 = headings.find((h) => h.level === 1)?.text ?? null;
5752
- const bodyText = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
5753
- const wordCount2 = bodyText.split(" ").filter((w) => w.length > 2).length;
5754
6369
  const schemaTypes = [];
5755
6370
  let schemaTypeBytes = 0;
5756
6371
  const addSchemaType = (value) => {
@@ -5807,11 +6422,14 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5807
6422
  }
5808
6423
  }
5809
6424
  const mainHtml = (html.match(/<main[^>]*>([\s\S]*?)<\/main>/i)?.[1] ?? html.match(/<article[^>]*>([\s\S]*?)<\/article>/i)?.[1] ?? html.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? html).replace(/<(script|style|nav|header|footer|aside|noscript|svg|iframe)[^>]*>[\s\S]*?<\/\1>/gi, "");
6425
+ const mainText = mainHtml.replace(/<[^>]+>/g, " ").replace(/&(?:nbsp|#160);/gi, " ").replace(/&(?:amp|#38);/gi, "&").replace(/&#(?:x[0-9a-f]+|\d+);/gi, " ").replace(/\s+/g, " ").trim();
6426
+ const mainWords = mainText.match(/[\p{L}\p{N}][\p{L}\p{N}'’.-]*/gu) ?? [];
6427
+ const wordCount2 = mainWords.filter((word) => word.length > 2).length;
5810
6428
  let bodyMarkdown = "";
5811
6429
  try {
5812
6430
  bodyMarkdown = turndown.turndown(mainHtml).replace(/\n{3,}/g, "\n\n").trim().slice(0, MAX_PAGE_MARKDOWN);
5813
6431
  } catch {
5814
- bodyMarkdown = bodyText.slice(0, MAX_PAGE_MARKDOWN);
6432
+ bodyMarkdown = mainText.slice(0, MAX_PAGE_MARKDOWN);
5815
6433
  }
5816
6434
  const outlinks = [];
5817
6435
  const discoveryLinkSet = /* @__PURE__ */ new Set();
@@ -5839,10 +6457,11 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5839
6457
  const internal = abs.origin === origin;
5840
6458
  if (internal) {
5841
6459
  internalLinks++;
5842
- if (!discoveryLinkSet.has(abs.href)) {
5843
- const bytes2 = Buffer.byteLength(abs.href);
6460
+ const crawlable = normalizeUrl(abs.href, documentUrl);
6461
+ if (crawlable && !discoveryLinkSet.has(crawlable)) {
6462
+ const bytes2 = Buffer.byteLength(crawlable);
5844
6463
  if (discoveryLinkSet.size < PAGE_DATA_LIMITS.discoveryLinks && discoveryBytes + bytes2 <= PAGE_DATA_LIMITS.discoveryBytes) {
5845
- discoveryLinkSet.add(abs.href);
6464
+ discoveryLinkSet.add(crawlable);
5846
6465
  discoveryBytes += bytes2;
5847
6466
  } else truncatedFields.add("discoveryLinks");
5848
6467
  }
@@ -5864,8 +6483,8 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5864
6483
  const h1List = headings.filter((h) => h.level === 1);
5865
6484
  const h1_2 = h1List[1]?.text ?? null;
5866
6485
  const h2Count = headings.filter((h) => h.level === 2).length;
5867
- const textRatio = Number((bodyText.length / Math.max(1, html.length)).toFixed(3));
5868
- const contentHash = createHash("sha1").update(bodyText).digest("hex");
6486
+ const textRatio = Number((mainText.length / Math.max(1, html.length)).toFixed(3));
6487
+ const contentHash = createHash("sha1").update(mainText).digest("hex");
5869
6488
  const metaRobots = boundedText(html.match(/<meta[^>]+name=["']robots["'][^>]+content=["']([^"']*)["']/i)?.[1], 512)?.toLowerCase() ?? null;
5870
6489
  const relNext = boundedUrl(html.match(/<link[^>]+rel=["']next["'][^>]+href=["']([^"']*)["']/i)?.[1]);
5871
6490
  const relPrev = boundedUrl(html.match(/<link[^>]+rel=["']prev["'][^>]+href=["']([^"']*)["']/i)?.[1]);
@@ -5919,11 +6538,8 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5919
6538
  }
5920
6539
  }
5921
6540
  }
5922
- if (og?.image) {
5923
- try {
5924
- addImageLink(new URL(og.image, documentUrl).href);
5925
- } catch {
5926
- }
6541
+ for (const asset of discoverStaticMedia(html, documentUrl)) {
6542
+ if (asset.type === "image") addImageLink(asset.url);
5927
6543
  }
5928
6544
  const imageLinks = [...imageLinkSet];
5929
6545
  const canonNorm = (u) => {
@@ -5950,12 +6566,19 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
5950
6566
  indexable = false;
5951
6567
  indexabilityReason = "canonicalised";
5952
6568
  }
5953
- const extractionSucceeded = bodyMarkdown.trim().length > 0;
6569
+ const bodyClass = html.match(/<body\b[^>]*\bclass\s*=\s*["']([^"']*)["']/i)?.[1] ?? "";
6570
+ const wordpressAttachment = /(?:^|\s)(?:attachment|attachment-template-default)(?:\s|$)/i.test(bodyClass);
6571
+ const meaningfulText = mainText.replace(/https?:\/\/\S+/gi, " ").replace(/\s+/g, " ").trim();
6572
+ const meaningfulWords = meaningfulText.match(/[\p{L}\p{N}][\p{L}\p{N}'’.-]*/gu) ?? [];
6573
+ const hasMeaningfulText = meaningfulText.length >= 24 && meaningfulWords.length >= 4;
6574
+ const extractionSucceeded = !wordpressAttachment && bodyMarkdown.trim().length > 0 && hasMeaningfulText;
6575
+ const failureCode = wordpressAttachment ? "wordpress_attachment_page" : bodyMarkdown.trim().length === 0 ? "empty_rendered_content" : "insufficient_text_content";
6576
+ const failureReason = wordpressAttachment ? "The HTML is a WordPress media attachment page, not a text-content page." : bodyMarkdown.trim().length === 0 ? "The HTML response produced no usable page content." : `The main content contained only ${meaningfulWords.length} word(s) and ${meaningfulText.length} text character(s).`;
5954
6577
  if (!extractionSucceeded) {
5955
6578
  indexable = false;
5956
6579
  indexabilityReason = "extraction-failed";
5957
6580
  }
5958
- const normalizedAttempts = extractionSucceeded ? fetchAttempts : fetchAttempts.map((attempt, index) => index === fetchAttempts.length - 1 && attempt.ok ? { ...attempt, ok: false, errorCode: "empty_rendered_content", error: "The HTML response produced no usable page content." } : attempt);
6581
+ const normalizedAttempts = extractionSucceeded ? fetchAttempts : fetchAttempts.map((attempt, index) => index === fetchAttempts.length - 1 && attempt.ok ? { ...attempt, ok: false, errorCode: failureCode, error: failureReason } : attempt);
5959
6582
  return {
5960
6583
  url,
5961
6584
  status,
@@ -6002,9 +6625,12 @@ function parsePageData(url, html, status, via, resp, fetchAttempts = []) {
6002
6625
  discoveryLinks,
6003
6626
  bodyMarkdown,
6004
6627
  schema,
6628
+ contentKind: extractionSucceeded ? "text_page" : "insufficient_text",
6629
+ mainTextChars: meaningfulText.length,
6630
+ mainWordCount: meaningfulWords.length,
6005
6631
  extractionStatus: extractionSucceeded ? "successful" : "failed",
6006
- failureCode: extractionSucceeded ? null : "empty_rendered_content",
6007
- failureReason: extractionSucceeded ? null : "The HTML response produced no usable page content.",
6632
+ failureCode: extractionSucceeded ? null : failureCode,
6633
+ failureReason: extractionSucceeded ? null : failureReason,
6008
6634
  fetchAttempts: normalizedAttempts,
6009
6635
  truncatedFields: [...truncatedFields]
6010
6636
  };
@@ -6092,6 +6718,7 @@ async function fetchPagePlain(url) {
6092
6718
  const attempt3 = failedAttempt("fetch", res.status, responseTimeMs2, "non_html_response", `Plain fetch returned ${contentType}, not HTML.`);
6093
6719
  const page = emptyPageData(url, res.status, "fetch", { code: attempt3.errorCode, message: attempt3.error, attempts: [attempt3] });
6094
6720
  page.contentType = contentType;
6721
+ page.contentKind = "non_html";
6095
6722
  return page;
6096
6723
  }
6097
6724
  const html = await readResponseTextWithLimit(res, MAX_PAGE_HTML_BYTES, "Page HTML");
@@ -6134,9 +6761,24 @@ async function mapWithConcurrency(items, concurrency, fn) {
6134
6761
  }
6135
6762
  async function extractPagesRotating(urls, opts) {
6136
6763
  const uniqueUrls = [...new Set(urls)];
6137
- const plainPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY, fetchPagePlain);
6764
+ const plainPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY, async (url) => {
6765
+ const checked = await validatePublicHttpUrl(url, { field: "page URL" });
6766
+ if (checked.error || !checked.parsed) return fetchPagePlain(url);
6767
+ if (normalizeUrl(checked.parsed.href, checked.parsed.href)) return fetchPagePlain(url);
6768
+ const attempt = failedAttempt("fetch", null, 0, "non_page_url", "The URL points to an asset, feed, API, account route, or other non-page path excluded from text-page extraction.");
6769
+ const page = emptyPageData(url, null, "fetch", { code: attempt.errorCode, message: attempt.error, attempts: [attempt] });
6770
+ page.contentKind = "excluded_url";
6771
+ return page;
6772
+ });
6138
6773
  const byUrl = new Map(plainPages.map((page) => [page.url, page]));
6139
- const browserTargets = plainPages.filter((page) => !isPageExtractionSuccessful(page) && !["unsafe_url", "unsafe_redirect", "cross_site_redirect"].includes(page.failureCode ?? "")).map((page) => page.url);
6774
+ const browserTargets = plainPages.filter((page) => !isPageExtractionSuccessful(page) && ![
6775
+ "unsafe_url",
6776
+ "unsafe_redirect",
6777
+ "cross_site_redirect",
6778
+ "non_html_response",
6779
+ "non_page_url",
6780
+ "wordpress_attachment_page"
6781
+ ].includes(page.failureCode ?? "")).map((page) => page.url);
6140
6782
  if (opts.kernelApiKey && browserTargets.length > 0) {
6141
6783
  const fetched = await crawlWithRotation(browserTargets, {
6142
6784
  apiKey: opts.kernelApiKey,
@@ -8740,7 +9382,7 @@ var siteExtractFn = inngest.createFunction(
8740
9382
  });
8741
9383
  }) : null;
8742
9384
  const finalized = await step.run("finalize", async () => {
8743
- const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
9385
+ const { assembleExtractArtifacts } = await import("./extract-bundle-GUEUCTAE.js");
8744
9386
  const current = await getExtractJob(jobId2);
8745
9387
  if (!current) throw new Error("extract job disappeared before finalization");
8746
9388
  const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -10071,8 +10713,8 @@ async function saveDirectoryCsvLocal(result) {
10071
10713
  const outDir = join3(outputBaseDir(), "directory-workflows");
10072
10714
  await mkdir(outDir, { recursive: true });
10073
10715
  const stamp = result.extractedAt.replace(/[:.]/g, "-");
10074
- const slug2 = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
10075
- const path5 = join3(outDir, `${stamp}-${slug2}-directory-workflow.csv`);
10716
+ const slug4 = `${result.state}-${result.query}`.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
10717
+ const path5 = join3(outDir, `${stamp}-${slug4}-directory-workflow.csv`);
10076
10718
  await writeFile(path5, renderDirectoryWorkflowCsv(result), "utf8");
10077
10719
  return path5;
10078
10720
  }
@@ -12214,9 +12856,9 @@ async function listAllPublicLocalSourcebook(category) {
12214
12856
  }) : await getDb().execute(`SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') ORDER BY s.category, s.state, s.business_name`);
12215
12857
  return result.rows.map((row) => parseJson2(row.payload_json));
12216
12858
  }
12217
- async function getPublicLocalSourcebook(category, state, slug2) {
12859
+ async function getPublicLocalSourcebook(category, state, slug4) {
12218
12860
  await ensureLocalSourcebookSchema();
12219
- const result = await getDb().execute({ sql: `SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.category = ? AND s.state = ? AND s.slug = ? AND s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') LIMIT 1`, args: [category, state, slug2] });
12861
+ const result = await getDb().execute({ sql: `SELECT r.payload_json FROM local_sourcebook_submissions s JOIN local_sourcebook_revisions r ON r.submission_id = s.id AND r.revision = s.published_revision WHERE s.category = ? AND s.state = ? AND s.slug = ? AND s.published_revision IS NOT NULL AND s.status NOT IN ('unpublished', 'rejected') LIMIT 1`, args: [category, state, slug4] });
12220
12862
  return result.rows[0] ? parseJson2(result.rows[0].payload_json) : null;
12221
12863
  }
12222
12864
 
@@ -12520,6 +13162,235 @@ var MapsReviewCollector = class {
12520
13162
  }
12521
13163
  };
12522
13164
 
13165
+ // src/extractor/MapsImageCollector.ts
13166
+ var QUIESCENCE_MS = 700;
13167
+ var STABLE_CHECKS_REQUIRED2 = 3;
13168
+ var MAX_SCROLL_ROUNDS2 = 80;
13169
+ var OWNER_DISCOVERY_LIMIT = 500;
13170
+ function mapsImageIdentity(rawUrl) {
13171
+ try {
13172
+ const url = new URL(rawUrl);
13173
+ const path5 = url.hostname.toLowerCase().endsWith("googleusercontent.com") ? url.pathname.replace(/=[^/]+$/, "") : url.pathname;
13174
+ return `${url.hostname.toLowerCase()}${path5}`;
13175
+ } catch {
13176
+ return rawUrl.split("=")[0] ?? rawUrl;
13177
+ }
13178
+ }
13179
+ function isMapsPhotoUrl(rawUrl) {
13180
+ try {
13181
+ const host2 = new URL(rawUrl).hostname.toLowerCase().replace(/\.$/, "");
13182
+ return host2 === "googleusercontent.com" || host2.endsWith(".googleusercontent.com") || host2 === "streetviewpixels-pa.googleapis.com";
13183
+ } catch {
13184
+ return false;
13185
+ }
13186
+ }
13187
+ async function openPhotoGallery(page) {
13188
+ const seePhotos = page.getByRole("button", { name: /^see photos$/i }).first();
13189
+ const exact = await seePhotos.count() > 0 ? seePhotos : page.locator('[aria-label^="See photos" i], button[aria-label^="Photo of" i]').first();
13190
+ if (await exact.count() === 0) {
13191
+ return false;
13192
+ }
13193
+ const clicked = await exact.click({ timeout: 5e3 }).then(() => true, () => false);
13194
+ if (!clicked) return false;
13195
+ const galleryControl = page.locator('button[aria-label="By owner" i], [role="tab"][aria-label*="owner" i]').first();
13196
+ await galleryControl.waitFor({ state: "visible", timeout: 5e3 }).catch(() => void 0);
13197
+ return galleryControl.isVisible().catch(() => false);
13198
+ }
13199
+ async function selectGalleryTab(page, scope) {
13200
+ const matcher = scope === "owner" ? /by owner/i : /^all(?:\s|$)/i;
13201
+ const direct = page.locator(scope === "owner" ? 'button[aria-label="By owner" i]' : 'button[aria-label="All" i]');
13202
+ const roleTab = page.getByRole("tab", { name: matcher });
13203
+ const tabs = await direct.count() > 0 ? direct : roleTab;
13204
+ if (await tabs.count() === 0) return false;
13205
+ const visible = tabs.filter({ visible: true }).first();
13206
+ if (await visible.count() === 0) return false;
13207
+ const clicked = await visible.click({ timeout: 5e3 }).then(() => true, () => false);
13208
+ if (!clicked) return false;
13209
+ await page.waitForTimeout(QUIESCENCE_MS);
13210
+ return true;
13211
+ }
13212
+ async function readAndAdvanceGallery(page) {
13213
+ return page.evaluate(() => {
13214
+ const mediaNodes = Array.from(document.querySelectorAll('[aria-label^="Photo " i], [aria-label^="Video " i]'));
13215
+ const imageUrl = (root) => {
13216
+ const candidates = [root, ...Array.from(root.querySelectorAll("*"))];
13217
+ for (const node of candidates) {
13218
+ if (node instanceof HTMLImageElement) {
13219
+ const value = node.currentSrc || node.src;
13220
+ if (value) return value;
13221
+ }
13222
+ const inline = node.style.backgroundImage;
13223
+ const computed = inline || window.getComputedStyle(node).backgroundImage;
13224
+ const match = computed?.match(/url\(["']?(.+?)["']?\)/i);
13225
+ if (match?.[1]) return match[1];
13226
+ }
13227
+ return null;
13228
+ };
13229
+ const items = mediaNodes.map((node) => {
13230
+ const label = node.getAttribute("aria-label") ?? "";
13231
+ const match = label.match(/^(Photo|Video)\s+(\d+)/i);
13232
+ return {
13233
+ kind: match?.[1]?.toLowerCase() === "video" ? "video" : "photo",
13234
+ position: match?.[2] ? Number(match[2]) : null,
13235
+ url: imageUrl(node)
13236
+ };
13237
+ });
13238
+ const first = mediaNodes[0] ?? null;
13239
+ const ancestors = [];
13240
+ let current = first?.parentElement ?? null;
13241
+ while (current) {
13242
+ if (current.scrollHeight > current.clientHeight + 40) ancestors.push(current);
13243
+ current = current.parentElement;
13244
+ }
13245
+ const fallback = Array.from(document.querySelectorAll("div")).filter((node) => {
13246
+ const style = window.getComputedStyle(node);
13247
+ return /(auto|scroll)/.test(style.overflowY) && node.scrollHeight > node.clientHeight + 40 && mediaNodes.some((media) => node.contains(media));
13248
+ });
13249
+ const preferred = Array.from(document.querySelectorAll("div.m6QErb.DxyBCb.kA9KIf.dS8AEf")).find((node) => mediaNodes.some((media) => node.contains(media)) && node.scrollHeight > node.clientHeight + 40);
13250
+ const scroller = preferred ?? [...ancestors, ...fallback].sort((a, b) => b.scrollHeight - a.scrollHeight)[0] ?? null;
13251
+ const atBottom = scroller ? scroller.scrollTop + scroller.clientHeight >= scroller.scrollHeight - 5 : true;
13252
+ const metrics = {
13253
+ scrollHeight: scroller?.scrollHeight ?? 0,
13254
+ scrollTop: scroller?.scrollTop ?? 0,
13255
+ clientHeight: scroller?.clientHeight ?? 0,
13256
+ atBottom
13257
+ };
13258
+ if (scroller && !atBottom) {
13259
+ scroller.scrollTop = Math.min(
13260
+ scroller.scrollHeight - scroller.clientHeight,
13261
+ scroller.scrollTop + Math.max(1200, Math.floor(scroller.clientHeight * 1.75))
13262
+ );
13263
+ }
13264
+ return { items, ...metrics };
13265
+ });
13266
+ }
13267
+ async function collectGalleryPass(page, limit) {
13268
+ const photos = /* @__PURE__ */ new Map();
13269
+ const videos = /* @__PURE__ */ new Set();
13270
+ let stableChecks = 0;
13271
+ let previousHeight = -1;
13272
+ let rounds = 0;
13273
+ while (rounds < MAX_SCROLL_ROUNDS2) {
13274
+ rounds += 1;
13275
+ const before = photos.size + videos.size;
13276
+ const snapshot = await readAndAdvanceGallery(page);
13277
+ for (const item of snapshot.items) {
13278
+ if (item.kind === "video") {
13279
+ videos.add(item.url ? mapsImageIdentity(item.url) : `position:${item.position ?? videos.size}`);
13280
+ continue;
13281
+ }
13282
+ if (!item.url || !isMapsPhotoUrl(item.url)) continue;
13283
+ const key = mapsImageIdentity(item.url);
13284
+ const existing = photos.get(key);
13285
+ if (!existing || (item.position ?? Infinity) < (existing.position ?? Infinity)) photos.set(key, item);
13286
+ }
13287
+ if (photos.size >= limit) {
13288
+ return {
13289
+ items: [...photos.values()].slice(0, limit),
13290
+ exhausted: false,
13291
+ stopReason: "limit_reached",
13292
+ photosDiscovered: photos.size
13293
+ };
13294
+ }
13295
+ await page.waitForTimeout(QUIESCENCE_MS);
13296
+ const unchanged = snapshot.atBottom && before === photos.size + videos.size && previousHeight === snapshot.scrollHeight;
13297
+ stableChecks = unchanged ? stableChecks + 1 : 0;
13298
+ previousHeight = snapshot.scrollHeight;
13299
+ if (stableChecks >= STABLE_CHECKS_REQUIRED2) {
13300
+ return {
13301
+ items: [...photos.values()],
13302
+ exhausted: true,
13303
+ stopReason: "gallery_exhausted",
13304
+ photosDiscovered: photos.size
13305
+ };
13306
+ }
13307
+ }
13308
+ return {
13309
+ items: [...photos.values()],
13310
+ exhausted: false,
13311
+ stopReason: "scroll_round_limit",
13312
+ photosDiscovered: photos.size
13313
+ };
13314
+ }
13315
+ var MapsImageCollector = class {
13316
+ constructor(page) {
13317
+ this.page = page;
13318
+ }
13319
+ page;
13320
+ async collect(scope, maxImages) {
13321
+ if (!await openPhotoGallery(this.page)) {
13322
+ return {
13323
+ images: [],
13324
+ scope,
13325
+ exhausted: false,
13326
+ stopReason: "gallery_unavailable",
13327
+ ownerGalleryAvailable: false,
13328
+ ownerGalleryExhausted: false,
13329
+ ownerPhotosDiscovered: 0,
13330
+ allPhotosDiscovered: 0
13331
+ };
13332
+ }
13333
+ const ownerGalleryAvailable = await selectGalleryTab(this.page, "owner");
13334
+ const ownerPass = ownerGalleryAvailable ? await collectGalleryPass(this.page, scope === "owner" ? maxImages : OWNER_DISCOVERY_LIMIT) : { items: [], exhausted: false, stopReason: "gallery_unavailable", photosDiscovered: 0 };
13335
+ const ownerKeys = new Set(ownerPass.items.flatMap((item) => item.url ? [mapsImageIdentity(item.url)] : []));
13336
+ if (scope === "owner") {
13337
+ return {
13338
+ images: ownerPass.items.flatMap((item, index) => item.url ? [{
13339
+ index: index + 1,
13340
+ galleryPosition: item.position,
13341
+ sourceUrl: item.url,
13342
+ mediaKey: mapsImageIdentity(item.url),
13343
+ origin: "owner",
13344
+ originConfidence: "owner_gallery"
13345
+ }] : []),
13346
+ scope,
13347
+ exhausted: ownerPass.exhausted,
13348
+ stopReason: ownerGalleryAvailable ? ownerPass.stopReason : "gallery_unavailable",
13349
+ ownerGalleryAvailable,
13350
+ ownerGalleryExhausted: ownerPass.exhausted,
13351
+ ownerPhotosDiscovered: ownerPass.photosDiscovered,
13352
+ allPhotosDiscovered: ownerPass.photosDiscovered
13353
+ };
13354
+ }
13355
+ if (!await selectGalleryTab(this.page, "all")) {
13356
+ return {
13357
+ images: [],
13358
+ scope,
13359
+ exhausted: false,
13360
+ stopReason: "gallery_unavailable",
13361
+ ownerGalleryAvailable,
13362
+ ownerGalleryExhausted: ownerPass.exhausted,
13363
+ ownerPhotosDiscovered: ownerPass.photosDiscovered,
13364
+ allPhotosDiscovered: 0
13365
+ };
13366
+ }
13367
+ const allPass = await collectGalleryPass(this.page, maxImages);
13368
+ const images = allPass.items.flatMap((item, index) => {
13369
+ if (!item.url) return [];
13370
+ const mediaKey = mapsImageIdentity(item.url);
13371
+ const isOwner = ownerKeys.has(mediaKey);
13372
+ return [{
13373
+ index: index + 1,
13374
+ galleryPosition: item.position,
13375
+ sourceUrl: item.url,
13376
+ mediaKey,
13377
+ origin: isOwner ? "owner" : ownerPass.exhausted ? "other" : "unknown",
13378
+ originConfidence: isOwner ? "owner_gallery_match" : ownerPass.exhausted ? "not_in_exhausted_owner_gallery" : "owner_gallery_incomplete"
13379
+ }];
13380
+ });
13381
+ return {
13382
+ images,
13383
+ scope,
13384
+ exhausted: allPass.exhausted,
13385
+ stopReason: allPass.stopReason,
13386
+ ownerGalleryAvailable,
13387
+ ownerGalleryExhausted: ownerPass.exhausted,
13388
+ ownerPhotosDiscovered: ownerPass.photosDiscovered,
13389
+ allPhotosDiscovered: allPass.photosDiscovered
13390
+ };
13391
+ }
13392
+ };
13393
+
12523
13394
  // src/extractor/MapsExtractor.ts
12524
13395
  import { z as z11 } from "zod";
12525
13396
  var MapsExtractor = class {
@@ -12544,6 +13415,48 @@ var MapsExtractor = class {
12544
13415
  const identifiers = this.parseUrlIdentifiers(placeUrl);
12545
13416
  const overview = await this.extractOverview(page);
12546
13417
  const hoursTable = await this.extractFullHours(page);
13418
+ let media = {
13419
+ status: "not_requested",
13420
+ scope: options.imageScope,
13421
+ requestedMaxImages: options.maxImages,
13422
+ images: [],
13423
+ imagesCollected: 0,
13424
+ imagesDownloaded: 0,
13425
+ ownerImagesCollected: 0,
13426
+ otherImagesCollected: 0,
13427
+ unknownOriginImagesCollected: 0,
13428
+ ownerGalleryAvailable: false,
13429
+ ownerGalleryExhausted: false,
13430
+ ownerPhotosDiscovered: 0,
13431
+ allPhotosDiscovered: 0,
13432
+ exhausted: false,
13433
+ stopReason: "not_requested",
13434
+ artifact: null,
13435
+ warnings: []
13436
+ };
13437
+ if (options.includeImages) {
13438
+ try {
13439
+ await page.goto(placeUrl, { waitUntil: "domcontentloaded", timeout: 2e4 });
13440
+ await page.waitForTimeout(500);
13441
+ const collected = await new MapsImageCollector(page).collect(options.imageScope, options.maxImages);
13442
+ const ownerImagesCollected = collected.images.filter((image) => image.origin === "owner").length;
13443
+ const otherImagesCollected = collected.images.filter((image) => image.origin === "other").length;
13444
+ const unknownOriginImagesCollected = collected.images.filter((image) => image.origin === "unknown").length;
13445
+ media = {
13446
+ ...media,
13447
+ ...collected,
13448
+ status: collected.images.length > 0 ? "collected" : collected.stopReason === "gallery_unavailable" ? "unavailable" : "none_exist",
13449
+ imagesCollected: collected.images.length,
13450
+ ownerImagesCollected,
13451
+ otherImagesCollected,
13452
+ unknownOriginImagesCollected
13453
+ };
13454
+ await page.goto(placeUrl, { waitUntil: "domcontentloaded", timeout: 2e4 }).catch(() => void 0);
13455
+ } catch (err) {
13456
+ console.warn("[MapsExtractor] image gallery extraction failed:", err instanceof Error ? err.message : String(err));
13457
+ media = { ...media, status: "unavailable", stopReason: "gallery_unavailable", warnings: ["Google Maps image gallery could not be read in this run."] };
13458
+ }
13459
+ }
12547
13460
  let reviewStats = { reviewHistogram: [], reviewTopics: [] };
12548
13461
  let reviews = [];
12549
13462
  let reviewsStatus = "not_requested";
@@ -12631,6 +13544,7 @@ var MapsExtractor = class {
12631
13544
  services,
12632
13545
  areasServed,
12633
13546
  servicesStatus,
13547
+ media,
12634
13548
  durationMs: Date.now() - startMs
12635
13549
  };
12636
13550
  } finally {
@@ -12913,11 +13827,11 @@ function rootDomain() {
12913
13827
  const configured = process.env.LOCAL_SOURCEBOOK_PUBLIC_ROOT_DOMAIN?.trim() || DEFAULT_ROOT_DOMAIN;
12914
13828
  return configured.replace(/^https?:\/\//i, "").replace(/^www\./i, "").replace(/\/.*$/, "").replace(/\.+$/, "").toLowerCase();
12915
13829
  }
12916
- function localSourcebookPublicUrls(category, state, slug2) {
13830
+ function localSourcebookPublicUrls(category, state, slug4) {
12917
13831
  const categoryUrl = `https://${CATEGORY_PUBLIC_SLUGS[category]}.${rootDomain()}`;
12918
13832
  const statePath = `/${state.toLowerCase()}`;
12919
- const profilePath = `${statePath}/${slug2}/`;
12920
- const reviewsPath = `${statePath}/${slug2}/reviews`;
13833
+ const profilePath = `${statePath}/${slug4}/`;
13834
+ const reviewsPath = `${statePath}/${slug4}/reviews`;
12921
13835
  return {
12922
13836
  categoryUrl,
12923
13837
  nationwideDirectoryUrl: `${categoryUrl}/directory`,
@@ -13465,13 +14379,13 @@ function host(value) {
13465
14379
  }
13466
14380
  }
13467
14381
  function tokens(value) {
13468
- return new Set(value.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((token4) => token4.length > 1 && !BUSINESS_SUFFIXES.has(token4)));
14382
+ return new Set(value.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((token6) => token6.length > 1 && !BUSINESS_SUFFIXES.has(token6)));
13469
14383
  }
13470
14384
  function nameOverlap(left, right) {
13471
14385
  const a = tokens(left);
13472
14386
  const b = tokens(right);
13473
14387
  if (!a.size || !b.size) return 0;
13474
- const intersection = [...a].filter((token4) => b.has(token4)).length;
14388
+ const intersection = [...a].filter((token6) => b.has(token6)).length;
13475
14389
  return intersection / Math.max(a.size, b.size);
13476
14390
  }
13477
14391
  function validateLocalSourcebookMapsIdentity(submission, maps) {
@@ -13520,6 +14434,10 @@ function createDefaultLocalSourcebookProvider() {
13520
14434
  includeReviews: true,
13521
14435
  maxReviews,
13522
14436
  includeServices: true,
14437
+ includeImages: false,
14438
+ imageScope: "all",
14439
+ maxImages: 100,
14440
+ maxInlineImages: 3,
13523
14441
  kernelApiKey,
13524
14442
  headless: true
13525
14443
  });
@@ -14405,12 +15323,12 @@ function normalizeIdempotencyKey(value) {
14405
15323
  function tokenFor(id, idempotencyKey3) {
14406
15324
  return createHmac("sha256", billingSecret()).update(id).update(":").update(idempotencyKey3).digest("base64url");
14407
15325
  }
14408
- function tokenHash(token4) {
14409
- return createHash4("sha256").update(token4).digest("hex");
15326
+ function tokenHash(token6) {
15327
+ return createHash4("sha256").update(token6).digest("hex");
14410
15328
  }
14411
- function verifyToken(row, token4) {
15329
+ function verifyToken(row, token6) {
14412
15330
  const expected = Buffer.from(row.token_hash, "hex");
14413
- const actual = Buffer.from(tokenHash(token4), "hex");
15331
+ const actual = Buffer.from(tokenHash(token6), "hex");
14414
15332
  if (expected.length !== actual.length || !timingSafeEqual(expected, actual)) {
14415
15333
  throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
14416
15334
  }
@@ -14527,11 +15445,11 @@ async function authorizeScheduledRun(args) {
14527
15445
  );
14528
15446
  }
14529
15447
  const id = randomUUID4();
14530
- const token4 = tokenFor(id, idempotencyKey3);
15448
+ const token6 = tokenFor(id, idempotencyKey3);
14531
15449
  const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
14532
15450
  const inserted = await getDb().execute({
14533
15451
  sql: "INSERT OR IGNORE INTO billing_authorizations (id, user_id, idempotency_key, billing_class, status, minimum_amount_mc, maximum_amount_mc, token_hash, metadata, expires_at) VALUES (?, ?, ?, ?, 'authorized', ?, ?, ?, ?, ?)",
14534
- args: [id, user.id, idempotencyKey3, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_BASE_MC, maximumMc, tokenHash(token4), safeMetadata2(args.metadata), expiresAt]
15452
+ args: [id, user.id, idempotencyKey3, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_BASE_MC, maximumMc, tokenHash(token6), safeMetadata2(args.metadata), expiresAt]
14535
15453
  });
14536
15454
  if (inserted.rowsAffected === 0) {
14537
15455
  const raced = await getDb().execute({
@@ -14547,7 +15465,7 @@ async function authorizeScheduledRun(args) {
14547
15465
  }
14548
15466
  return {
14549
15467
  id,
14550
- token: token4,
15468
+ token: token6,
14551
15469
  status: "authorized",
14552
15470
  minimumMc: SCHEDULED_RUN_BASE_MC,
14553
15471
  maximumMc,
@@ -16157,9 +17075,9 @@ var YouTubeExtractor = class {
16157
17075
  }
16158
17076
  async fetchContinuation(page, endpoint, contToken, context) {
16159
17077
  const result = await page.evaluate(
16160
- async ({ ep, token: token4, ctx }) => {
17078
+ async ({ ep, token: token6, ctx }) => {
16161
17079
  const body = {
16162
- continuation: token4,
17080
+ continuation: token6,
16163
17081
  context: {
16164
17082
  client: {
16165
17083
  clientName: ctx.clientName,
@@ -16403,23 +17321,23 @@ async function extractOnce(options) {
16403
17321
  }
16404
17322
  async function writeOutputs(result, outputDir) {
16405
17323
  await fs.mkdir(outputDir, { recursive: true });
16406
- const slug2 = result.target.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
17324
+ const slug4 = result.target.toLowerCase().replace(/\W+/g, "-").slice(0, 40);
16407
17325
  const ts = Date.now();
16408
17326
  await fs.writeFile(
16409
- path4.join(outputDir, `${slug2}-${ts}.json`),
17327
+ path4.join(outputDir, `${slug4}-${ts}.json`),
16410
17328
  JSON.stringify(result, null, 2),
16411
17329
  "utf8"
16412
17330
  );
16413
17331
  if (result.videos.length > 0) {
16414
17332
  await fs.writeFile(
16415
- path4.join(outputDir, `${slug2}-videos-${ts}.csv`),
17333
+ path4.join(outputDir, `${slug4}-videos-${ts}.csv`),
16416
17334
  Papa.unparse(result.videos, { header: true }),
16417
17335
  "utf8"
16418
17336
  );
16419
17337
  }
16420
17338
  if (result.downloads.length > 0) {
16421
17339
  await fs.writeFile(
16422
- path4.join(outputDir, `${slug2}-downloads-${ts}.csv`),
17340
+ path4.join(outputDir, `${slug4}-downloads-${ts}.csv`),
16423
17341
  Papa.unparse(result.downloads, { header: true }),
16424
17342
  "utf8"
16425
17343
  );
@@ -16783,6 +17701,8 @@ var ExtractUrlBodyShape = {
16783
17701
  includeFeaturedImage: z16.boolean().optional(),
16784
17702
  downloadMedia: z16.boolean().optional(),
16785
17703
  mediaTypes: z16.array(z16.enum(["image", "video", "audio"])).optional(),
17704
+ maxMediaAssets: z16.number().int().min(1).max(250).optional(),
17705
+ maxInlineImages: z16.number().int().min(0).max(5).optional(),
16786
17706
  delivery: z16.enum(["auto", "inline", "artifact", "memory"]).optional(),
16787
17707
  preserveMedia: z16.boolean().optional(),
16788
17708
  depositToVault: z16.boolean().optional(),
@@ -18059,7 +18979,7 @@ function advertiserTokenOverlap(query, candidate) {
18059
18979
  const candidateTokens = new Set(normalizeAdvertiserName(candidate).split(" ").filter(Boolean));
18060
18980
  if (queryTokens.size === 0 || candidateTokens.size === 0) return 0;
18061
18981
  let overlap = 0;
18062
- for (const token4 of queryTokens) if (candidateTokens.has(token4)) overlap++;
18982
+ for (const token6 of queryTokens) if (candidateTokens.has(token6)) overlap++;
18063
18983
  return overlap / queryTokens.size;
18064
18984
  }
18065
18985
  function assessPageIntelMatch(body, result) {
@@ -19294,8 +20214,8 @@ function getBlobStore() {
19294
20214
  return cached;
19295
20215
  }
19296
20216
  var VercelBlobStore = class {
19297
- constructor(token4) {
19298
- this.token = token4;
20217
+ constructor(token6) {
20218
+ this.token = token6;
19299
20219
  }
19300
20220
  token;
19301
20221
  kind = "vercel-blob";
@@ -19751,8 +20671,8 @@ function safeFilePart(input) {
19751
20671
  }
19752
20672
  function blobKeyPrefix(shortcode, sourceUrl) {
19753
20673
  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
19754
- const slug2 = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
19755
- return `instagram/${stamp}-${slug2}`;
20674
+ const slug4 = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
20675
+ return `instagram/${stamp}-${slug4}`;
19756
20676
  }
19757
20677
  async function fetchMediaBytes(url, referer, runtimeUserAgent) {
19758
20678
  const res = await fetch(url, {
@@ -20664,6 +21584,212 @@ videoApp.post("/status", createApiKeyAuth(), async (c) => {
20664
21584
 
20665
21585
  // src/api/maps-routes.ts
20666
21586
  import { Hono as Hono13 } from "hono";
21587
+
21588
+ // src/api/maps-media-artifacts.ts
21589
+ import { homedir as homedir2 } from "os";
21590
+ import { join as join6 } from "path";
21591
+ import { randomBytes as randomBytes2 } from "crypto";
21592
+ import { ZipFile } from "yazl";
21593
+ import pLimit4 from "p-limit";
21594
+ var MAPS_MEDIA_ARTIFACT_PREFIX = "maps-media/";
21595
+ var MAPS_MEDIA_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
21596
+ var MAPS_MEDIA_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
21597
+ var MAX_IMAGE_BYTES = 5 * 1024 * 1024;
21598
+ var MAX_ARCHIVE_IMAGE_BYTES = 40 * 1024 * 1024;
21599
+ var MAX_INLINE_IMAGE_BYTES = 15e5;
21600
+ var MAX_INLINE_TOTAL_BYTES = 25e5;
21601
+ var DOWNLOAD_CONCURRENCY = 6;
21602
+ var MAX_REDIRECTS = 4;
21603
+ function token() {
21604
+ return process.env.MAPS_MEDIA_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
21605
+ }
21606
+ function policy() {
21607
+ return {
21608
+ prefix: MAPS_MEDIA_ARTIFACT_PREFIX,
21609
+ artifactTtlMs: MAPS_MEDIA_ARTIFACT_TTL_MS,
21610
+ downloadTtlMs: MAPS_MEDIA_DOWNLOAD_TTL_MS,
21611
+ token: token()
21612
+ };
21613
+ }
21614
+ function allowedMapsImageUrl(raw) {
21615
+ try {
21616
+ const url = new URL(raw);
21617
+ if (url.protocol !== "https:") return null;
21618
+ const host2 = url.hostname.toLowerCase().replace(/\.$/, "");
21619
+ if (host2 !== "googleusercontent.com" && !host2.endsWith(".googleusercontent.com") && host2 !== "streetviewpixels-pa.googleapis.com") return null;
21620
+ return url;
21621
+ } catch {
21622
+ return null;
21623
+ }
21624
+ }
21625
+ function preferredMapsImageRendition(rawUrl) {
21626
+ const url = allowedMapsImageUrl(rawUrl);
21627
+ if (!url) return rawUrl;
21628
+ if (url.hostname.toLowerCase().endsWith("googleusercontent.com") && /=[^/]+$/.test(url.pathname)) {
21629
+ url.pathname = url.pathname.replace(/=[^/]+$/, "=s2048");
21630
+ }
21631
+ return url.href;
21632
+ }
21633
+ function imageMime(bytes) {
21634
+ if (bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return { mimeType: "image/jpeg", extension: "jpg" };
21635
+ if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80 && bytes[2] === 78 && bytes[3] === 71) return { mimeType: "image/png", extension: "png" };
21636
+ if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(0, 4)) === "RIFF" && String.fromCharCode(...bytes.slice(8, 12)) === "WEBP") return { mimeType: "image/webp", extension: "webp" };
21637
+ if (bytes.length >= 6 && ["GIF87a", "GIF89a"].includes(String.fromCharCode(...bytes.slice(0, 6)))) return { mimeType: "image/gif", extension: "gif" };
21638
+ return null;
21639
+ }
21640
+ async function readBounded(response) {
21641
+ const declared = Number(response.headers.get("content-length") ?? "");
21642
+ if (Number.isFinite(declared) && declared > MAX_IMAGE_BYTES) throw new Error("image_too_large");
21643
+ if (!response.body) {
21644
+ const bytes2 = new Uint8Array(await response.arrayBuffer());
21645
+ if (bytes2.byteLength > MAX_IMAGE_BYTES) throw new Error("image_too_large");
21646
+ return bytes2;
21647
+ }
21648
+ const reader = response.body.getReader();
21649
+ const chunks = [];
21650
+ let total = 0;
21651
+ try {
21652
+ while (true) {
21653
+ const { done, value } = await reader.read();
21654
+ if (done) break;
21655
+ if (!value) continue;
21656
+ total += value.byteLength;
21657
+ if (total > MAX_IMAGE_BYTES) throw new Error("image_too_large");
21658
+ chunks.push(value);
21659
+ }
21660
+ } finally {
21661
+ reader.releaseLock();
21662
+ }
21663
+ const bytes = new Uint8Array(total);
21664
+ let offset = 0;
21665
+ for (const chunk of chunks) {
21666
+ bytes.set(chunk, offset);
21667
+ offset += chunk.byteLength;
21668
+ }
21669
+ return bytes;
21670
+ }
21671
+ async function fetchMapsImage(rawUrl) {
21672
+ let url = allowedMapsImageUrl(preferredMapsImageRendition(rawUrl));
21673
+ if (!url) throw new Error("image_host_not_allowed");
21674
+ for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {
21675
+ const checked = await validatePublicHttpUrl(url.href, { field: "Maps image URL" });
21676
+ if (checked.error || !checked.parsed) throw new Error("image_url_rejected");
21677
+ const response = await fetch(checked.parsed.href, {
21678
+ redirect: "manual",
21679
+ headers: { Accept: "image/jpeg,image/png,image/webp,image/gif;q=0.8" },
21680
+ signal: AbortSignal.timeout(15e3)
21681
+ });
21682
+ if ([301, 302, 303, 307, 308].includes(response.status)) {
21683
+ const location2 = response.headers.get("location");
21684
+ if (!location2 || redirect === MAX_REDIRECTS) throw new Error("image_redirect_rejected");
21685
+ url = allowedMapsImageUrl(new URL(location2, url).href);
21686
+ if (!url) throw new Error("image_redirect_host_not_allowed");
21687
+ continue;
21688
+ }
21689
+ if (!response.ok) throw new Error(`image_fetch_${response.status}`);
21690
+ const body = await readBounded(response);
21691
+ const detected = imageMime(body);
21692
+ if (!detected) throw new Error("image_content_invalid");
21693
+ return { bytes: Buffer.from(body), ...detected, finalUrl: url.href };
21694
+ }
21695
+ throw new Error("image_redirect_rejected");
21696
+ }
21697
+ function zipBuffer(files) {
21698
+ return new Promise((resolve2, reject) => {
21699
+ const zip = new ZipFile();
21700
+ const chunks = [];
21701
+ zip.outputStream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
21702
+ zip.outputStream.once("error", reject);
21703
+ zip.outputStream.once("end", () => resolve2(Buffer.concat(chunks)));
21704
+ for (const file of files) zip.addBuffer(file.content, file.path);
21705
+ zip.end();
21706
+ });
21707
+ }
21708
+ function slug(value) {
21709
+ return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 70) || "google-maps-place";
21710
+ }
21711
+ function mapsMediaArtifactOwnerId(artifactId) {
21712
+ return privateArtifactOwnerId(artifactId, MAPS_MEDIA_ARTIFACT_PREFIX);
21713
+ }
21714
+ async function readOwnedMapsMediaArtifactBuffer(args) {
21715
+ if (mapsMediaArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
21716
+ return readPrivateArtifactBuffer({ policy: policy(), artifactId: args.artifactId, maxBytes: 50 * 1024 * 1024 });
21717
+ }
21718
+ async function packageMapsMedia(args) {
21719
+ const limit = pLimit4(DOWNLOAD_CONCURRENCY);
21720
+ const files = [];
21721
+ let storedBytes = 0;
21722
+ let inlineBytes = 0;
21723
+ let inlineCount = 0;
21724
+ await Promise.all(args.media.images.map((image) => limit(async () => {
21725
+ try {
21726
+ const fetched = await fetchMapsImage(image.sourceUrl);
21727
+ if (storedBytes + fetched.bytes.length > MAX_ARCHIVE_IMAGE_BYTES) throw new Error("archive_byte_limit_reached");
21728
+ storedBytes += fetched.bytes.length;
21729
+ const filename = `images/${String(image.index).padStart(4, "0")}-${image.origin}.${fetched.extension}`;
21730
+ files.push({ path: filename, content: fetched.bytes });
21731
+ image.sourceUrl = fetched.finalUrl;
21732
+ image.filename = filename;
21733
+ image.mimeType = fetched.mimeType;
21734
+ image.bytes = fetched.bytes.length;
21735
+ image.downloadStatus = "downloaded";
21736
+ image.downloadError = null;
21737
+ if (inlineCount < args.maxInlineImages && fetched.bytes.length <= MAX_INLINE_IMAGE_BYTES && inlineBytes + fetched.bytes.length <= MAX_INLINE_TOTAL_BYTES) {
21738
+ image.inlinePreview = { data: fetched.bytes.toString("base64"), mimeType: fetched.mimeType };
21739
+ inlineBytes += fetched.bytes.length;
21740
+ inlineCount += 1;
21741
+ }
21742
+ } catch (error) {
21743
+ image.filename = null;
21744
+ image.mimeType = null;
21745
+ image.bytes = null;
21746
+ image.downloadStatus = "failed";
21747
+ image.downloadError = error instanceof Error ? error.message : "image_download_failed";
21748
+ image.inlinePreview = null;
21749
+ }
21750
+ })));
21751
+ args.media.imagesDownloaded = args.media.images.filter((image) => image.downloadStatus === "downloaded").length;
21752
+ const failed = args.media.images.length - args.media.imagesDownloaded;
21753
+ if (failed > 0) args.media.warnings.push(`${failed} image(s) could not be downloaded; their manifest rows include a bounded error code.`);
21754
+ if (args.media.imagesDownloaded === 0) return { media: args.media, artifact: null };
21755
+ const cleanImages = args.media.images.map(({ inlinePreview: _preview, ...image }) => image);
21756
+ const summary = {
21757
+ businessName: args.businessName,
21758
+ placeUrl: args.placeUrl,
21759
+ extractedAt: args.extractedAt,
21760
+ scope: args.media.scope,
21761
+ imagesCollected: args.media.imagesCollected,
21762
+ imagesDownloaded: args.media.imagesDownloaded,
21763
+ ownerImagesCollected: args.media.ownerImagesCollected,
21764
+ otherImagesCollected: args.media.otherImagesCollected,
21765
+ unknownOriginImagesCollected: args.media.unknownOriginImagesCollected,
21766
+ ownerGalleryExhausted: args.media.ownerGalleryExhausted,
21767
+ galleryExhausted: args.media.exhausted,
21768
+ stopReason: args.media.stopReason,
21769
+ notes: [
21770
+ "origin=owner is backed by the Google Maps By owner gallery or an exact media-key match to it.",
21771
+ "origin=other means absent from an exhausted owner gallery; it can include customer, review, Street View, Google, or unattributed media.",
21772
+ "Google-hosted photo files use a bounded s2048 rendition when that CDN form is available; each file is still subject to byte and signature validation."
21773
+ ]
21774
+ };
21775
+ files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
21776
+ files.push({ path: "images.jsonl", content: Buffer.from(cleanImages.map((image) => JSON.stringify(image)).join("\n") + "\n") });
21777
+ const archive = await zipBuffer(files);
21778
+ const id = randomBytes2(6).toString("hex");
21779
+ const pointer = await createPrivateArtifact({
21780
+ policy: policy(),
21781
+ ownerId: args.ownerId,
21782
+ artifactKey: `${id}.zip`,
21783
+ createdAt: new Date(args.extractedAt),
21784
+ filename: `${slug(args.businessName)}-maps-images.zip`,
21785
+ contentType: "application/zip",
21786
+ content: archive
21787
+ });
21788
+ const localPath = token() ? null : join6(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir2(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
21789
+ return { media: args.media, artifact: { ...pointer, localPath } };
21790
+ }
21791
+
21792
+ // src/api/maps-routes.ts
20667
21793
  function mapsErrorResponse(c, err, errorCode) {
20668
21794
  const msg = err instanceof Error ? err.message : String(err);
20669
21795
  const rotationError = err instanceof MapsSearchRotationError ? err : null;
@@ -20782,6 +21908,30 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
20782
21908
  result.reviews = [];
20783
21909
  }
20784
21910
  }
21911
+ if (parsed.data.includeImages && result.media?.images.length > 0) {
21912
+ try {
21913
+ const packaged = await packageMapsMedia({
21914
+ ownerId: String(user.id),
21915
+ businessName: result.name ?? parsed.data.businessName,
21916
+ placeUrl: result.placeUrl,
21917
+ extractedAt: result.extractedAt,
21918
+ media: result.media,
21919
+ maxInlineImages: parsed.data.maxInlineImages
21920
+ });
21921
+ result.media = packaged.media;
21922
+ result.media.artifact = packaged.artifact;
21923
+ } catch (error) {
21924
+ console.warn("[maps/place] image artifact packaging failed:", error instanceof Error ? error.message : String(error));
21925
+ result.media.warnings.push("The image manifest was collected, but the downloadable ZIP artifact could not be created.");
21926
+ }
21927
+ }
21928
+ const logResult = result.media ? {
21929
+ ...result,
21930
+ media: {
21931
+ ...result.media,
21932
+ images: result.media.images.map(({ inlinePreview: _preview, ...image }) => image)
21933
+ }
21934
+ } : result;
20785
21935
  await logRequestEvent({
20786
21936
  userId: user.id,
20787
21937
  source: "maps_place",
@@ -20789,7 +21939,7 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
20789
21939
  query: `${parsed.data.businessName} ${parsed.data.location}`,
20790
21940
  location: parsed.data.location,
20791
21941
  resultCount: reviewCount,
20792
- result
21942
+ result: logResult
20793
21943
  });
20794
21944
  return c.json(result);
20795
21945
  } catch (err) {
@@ -21729,7 +22879,7 @@ function getLocalSourcebookContract(category) {
21729
22879
  }
21730
22880
  async function prepareLocalSourcebookWrite(input) {
21731
22881
  const parsed = DirectoryIdentitySchema.parse(input);
21732
- const slug2 = parsed.slug ?? parsed.businessName.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
22882
+ const slug4 = parsed.slug ?? parsed.businessName.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
21733
22883
  const categoryTag = CATEGORY_TAG[parsed.category];
21734
22884
  const candidates = [
21735
22885
  { tag: categoryTag, central: true, reusable: true },
@@ -21737,12 +22887,12 @@ async function prepareLocalSourcebookWrite(input) {
21737
22887
  ...parsed.tags.filter((tag) => tag !== categoryTag).map((tag) => ({ tag }))
21738
22888
  ];
21739
22889
  const tagResolutions = await resolveLocalSourcebookTags(candidates);
21740
- const publicUrls = localSourcebookPublicUrls(parsed.category, parsed.state, slug2);
22890
+ const publicUrls = localSourcebookPublicUrls(parsed.category, parsed.state, slug4);
21741
22891
  const schemaOrgType = parsed.schemaOrgType ?? defaultLocalSourcebookSchemaType(parsed.category);
21742
22892
  return {
21743
22893
  ok: true,
21744
- route: { category: parsed.category, state: parsed.state, slug: slug2, ...publicUrls },
21745
- proposedWrite: { ...parsed, schemaOrgType, slug: slug2, tags: tagResolutions.filter((item) => item.action === "reuse").map((item) => item.tag) },
22894
+ route: { category: parsed.category, state: parsed.state, slug: slug4, ...publicUrls },
22895
+ proposedWrite: { ...parsed, schemaOrgType, slug: slug4, tags: tagResolutions.filter((item) => item.action === "reuse").map((item) => item.tag) },
21746
22896
  contract: getLocalSourcebookContract(parsed.category),
21747
22897
  tagResolutions,
21748
22898
  instructions: [
@@ -21807,7 +22957,7 @@ async function persistLocalSourcebookTags(submissionId, userId, tags, decisions
21807
22957
  }
21808
22958
 
21809
22959
  // src/api/local-sourcebook-routes.ts
21810
- var slug = (value) => value.toLowerCase().trim().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
22960
+ var slug2 = (value) => value.toLowerCase().trim().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
21811
22961
  var SubmitSchema = z26.object({
21812
22962
  category: z26.enum(LOCAL_SOURCEBOOK_CATEGORIES),
21813
22963
  state: z26.string().trim().length(2).transform((value) => value.toLowerCase()),
@@ -21878,7 +23028,7 @@ localSourcebookApp.post("/capture", createApiKeyAuth(), async (c) => {
21878
23028
  if (!idempotencyKey3) {
21879
23029
  return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook capture." }, 400);
21880
23030
  }
21881
- const listingSlug = identity.slug ?? slug(identity.businessName);
23031
+ const listingSlug = identity.slug ?? slug2(identity.businessName);
21882
23032
  const publicUrls = localSourcebookPublicUrls(identity.category, identity.state, listingSlug);
21883
23033
  const charged = await chargeListingWrite({ surface: "local_sourcebook", kind: "create", userId: Number(user.id), idempotencyKey: idempotencyKey3 });
21884
23034
  if ("insufficient" in charged) return c.json(charged.insufficient, 402);
@@ -21929,7 +23079,7 @@ localSourcebookApp.post("/submissions", createApiKeyAuth(), async (c) => {
21929
23079
  if (!parsed.success) return c.json({ error: "invalid_submission", issues: parsed.error.issues }, 400);
21930
23080
  const input = parsed.data;
21931
23081
  if (!input.idempotencyKey) return c.json({ error: "idempotency_key_required", message: "A stable idempotencyKey is required for a new Local Sourcebook listing." }, 400);
21932
- const listingSlug = input.slug ?? slug(input.businessName);
23082
+ const listingSlug = input.slug ?? slug2(input.businessName);
21933
23083
  const schemaOrgType = input.schemaOrgType ?? defaultLocalSourcebookSchemaType(input.category);
21934
23084
  const publicUrls = localSourcebookPublicUrls(input.category, input.state, listingSlug);
21935
23085
  let billing = null;
@@ -22986,7 +24136,7 @@ import { createHash as createHash8, randomUUID as randomUUID9 } from "crypto";
22986
24136
 
22987
24137
  // src/serp-intelligence/page-snapshot-extractor.ts
22988
24138
  import { createHash as createHash7 } from "crypto";
22989
- import pLimit4 from "p-limit";
24139
+ import pLimit5 from "p-limit";
22990
24140
  var DEFAULT_TIMEOUT_MS = 15e3;
22991
24141
  var DEFAULT_MAX_CONCURRENCY = 2;
22992
24142
  var DEFAULT_MAX_CONTENT_CHARS = 25e4;
@@ -23267,7 +24417,7 @@ async function capturePageSnapshot(target, options = {}) {
23267
24417
  async function capturePageSnapshots(targets, options = {}) {
23268
24418
  const timeoutMs = normalizeTimeoutMs(options.timeoutMs);
23269
24419
  const maxConcurrency = normalizeMaxConcurrency(options.maxConcurrency);
23270
- const limit = pLimit4(maxConcurrency);
24420
+ const limit = pLimit5(maxConcurrency);
23271
24421
  const pageSnapshotArtifacts = await Promise.all(
23272
24422
  targets.map((target) => limit(() => capturePageSnapshot(target, { ...options, timeoutMs })))
23273
24423
  );
@@ -24108,15 +25258,15 @@ var ALLOWED_IMAGE_TYPES = /* @__PURE__ */ new Map([
24108
25258
  ["image/webp", "webp"],
24109
25259
  ["image/gif", "gif"]
24110
25260
  ]);
24111
- function token() {
25261
+ function token2() {
24112
25262
  return process.env.IMAGE_SOURCE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
24113
25263
  }
24114
- function policy() {
25264
+ function policy2() {
24115
25265
  return {
24116
25266
  prefix: IMAGE_SOURCE_ARTIFACT_PREFIX,
24117
25267
  artifactTtlMs: IMAGE_SOURCE_ARTIFACT_TTL_MS,
24118
25268
  downloadTtlMs: IMAGE_SOURCE_DOWNLOAD_TTL_MS,
24119
- token: token()
25269
+ token: token2()
24120
25270
  };
24121
25271
  }
24122
25272
  function imageSourceArtifactOwnerId(artifactId) {
@@ -24130,7 +25280,7 @@ async function createImageSourceArtifact(args) {
24130
25280
  }
24131
25281
  const id = randomUUID10().replaceAll("-", "");
24132
25282
  return createPrivateArtifact({
24133
- policy: policy(),
25283
+ policy: policy2(),
24134
25284
  ownerId: args.ownerId,
24135
25285
  artifactKey: `${id}.${extension}`,
24136
25286
  createdAt: args.createdAt ?? /* @__PURE__ */ new Date(),
@@ -24142,7 +25292,7 @@ async function createImageSourceArtifact(args) {
24142
25292
  async function readOwnedImageSourceArtifact(args) {
24143
25293
  if (imageSourceArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
24144
25294
  return readPrivateArtifactBuffer({
24145
- policy: policy(),
25295
+ policy: policy2(),
24146
25296
  artifactId: args.artifactId,
24147
25297
  maxBytes: IMAGE_SOURCE_MAX_BYTES
24148
25298
  });
@@ -24728,8 +25878,8 @@ function protectedResourceMetadata() {
24728
25878
  function wwwAuthenticateValue() {
24729
25879
  return `Bearer resource_metadata="${oauthResourceBaseUrl()}/.well-known/oauth-protected-resource"`;
24730
25880
  }
24731
- function isBearerJwt(token4) {
24732
- return !token4.startsWith("sk_") && token4.split(".").length === 3;
25881
+ function isBearerJwt(token6) {
25882
+ return !token6.startsWith("sk_") && token6.split(".").length === 3;
24733
25883
  }
24734
25884
  var jwksSingleton = null;
24735
25885
  function getJwks() {
@@ -24737,8 +25887,8 @@ function getJwks() {
24737
25887
  jwksSingleton = createRemoteJWKSet(new URL(`${oauthIssuer()}/.well-known/jwks.json`));
24738
25888
  return jwksSingleton;
24739
25889
  }
24740
- async function verifyScraperOAuthJwt(token4) {
24741
- const { payload } = await jwtVerify(token4, getJwks(), {
25890
+ async function verifyScraperOAuthJwt(token6) {
25891
+ const { payload } = await jwtVerify(token6, getJwks(), {
24742
25892
  issuer: oauthIssuer(),
24743
25893
  audience: oauthResourceUrl()
24744
25894
  });
@@ -27461,7 +28611,7 @@ async function resolveUser3(customerId, emailFallback) {
27461
28611
  // src/api/oauth-routes.ts
27462
28612
  import { Hono as Hono24 } from "hono";
27463
28613
  import { getCookie, setCookie } from "hono/cookie";
27464
- import { createHash as createHash9, randomBytes as randomBytes2, randomUUID as randomUUID13 } from "crypto";
28614
+ import { createHash as createHash9, randomBytes as randomBytes3, randomUUID as randomUUID13 } from "crypto";
27465
28615
  import { importPKCS8, exportJWK, SignJWT } from "jose";
27466
28616
  var ISSUER = "https://mcpscraper.dev";
27467
28617
  var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
@@ -27560,9 +28710,9 @@ function negotiateScope(requested, user, resource) {
27560
28710
  if (operator && !unique3.includes("memory.admin")) unique3.push("memory.admin");
27561
28711
  return unique3.join(" ");
27562
28712
  }
27563
- async function sessionUser(token4) {
27564
- if (!token4) return null;
27565
- const userId = verifySession(token4);
28713
+ async function sessionUser(token6) {
28714
+ if (!token6) return null;
28715
+ const userId = verifySession(token6);
27566
28716
  if (!userId) return null;
27567
28717
  const user = await withOAuthDatabaseDeadline("session-user", getUserById(userId));
27568
28718
  return user ?? null;
@@ -27747,7 +28897,7 @@ oauthApp.post("/oauth/register", async (c) => {
27747
28897
  }
27748
28898
  }
27749
28899
  const clientName = typeof body.client_name === "string" ? body.client_name : null;
27750
- const clientId = `client_${randomBytes2(16).toString("hex")}`;
28900
+ const clientId = `client_${randomBytes3(16).toString("hex")}`;
27751
28901
  await withOAuthDatabaseDeadline("register-client", registerClient(clientId, redirectUris, clientName));
27752
28902
  console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
27753
28903
  return c.json({
@@ -27822,7 +28972,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
27822
28972
  if (action2 === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
27823
28973
  if (action2 !== "approve") return c.text("unsupported action", 400);
27824
28974
  const scope = negotiateScope(p.scope, user, p.resource);
27825
- const code = `code_${randomBytes2(32).toString("base64url")}`;
28975
+ const code = `code_${randomBytes3(32).toString("base64url")}`;
27826
28976
  const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
27827
28977
  await withOAuthDatabaseDeadline("put-authorization-code", putCode({
27828
28978
  code,
@@ -27873,7 +29023,7 @@ oauthApp.post("/oauth/token", async (c) => {
27873
29023
  const plan = user ? resolvePlan(user) : "free";
27874
29024
  const audience = record.resource ?? RESOURCE();
27875
29025
  const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
27876
- const refreshToken = `rt_${randomBytes2(40).toString("base64url")}`;
29026
+ const refreshToken = `rt_${randomBytes3(40).toString("base64url")}`;
27877
29027
  await withOAuthDatabaseDeadline("put-refresh-token", putRefresh({
27878
29028
  refresh_token: refreshToken,
27879
29029
  client_id: clientId,
@@ -27903,7 +29053,7 @@ oauthApp.post("/oauth/token", async (c) => {
27903
29053
  const plan = user ? resolvePlan(user) : "free";
27904
29054
  const audience = record.resource ?? RESOURCE();
27905
29055
  const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
27906
- const nextRefresh = `rt_${randomBytes2(40).toString("base64url")}`;
29056
+ const nextRefresh = `rt_${randomBytes3(40).toString("base64url")}`;
27907
29057
  await withOAuthDatabaseDeadline("rotate-refresh-token", rotateRefresh(refreshToken, {
27908
29058
  refresh_token: nextRefresh,
27909
29059
  client_id: record.client_id,
@@ -27921,35 +29071,35 @@ oauthApp.post("/oauth/token", async (c) => {
27921
29071
  // src/api/chat-routes.ts
27922
29072
  import { Hono as Hono25 } from "hono";
27923
29073
  var chatApp = new Hono25();
27924
- function tokenOk(token4) {
27925
- return !!token4 && token4.startsWith("mk_") && token4.length > 10;
29074
+ function tokenOk(token6) {
29075
+ return !!token6 && token6.startsWith("mk_") && token6.length > 10;
27926
29076
  }
27927
29077
  function jsonError(message, status = 400) {
27928
29078
  return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
27929
29079
  }
27930
29080
  chatApp.get("/api/channels", async (c) => {
27931
- const token4 = c.req.query("token");
27932
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
27933
- const r = await memoryCall("listVaultsTool", {}, token4);
29081
+ const token6 = c.req.query("token");
29082
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
29083
+ const r = await memoryCall("listVaultsTool", {}, token6);
27934
29084
  if (!r.ok) return c.json(r);
27935
29085
  const vaults = (r.vaults ?? []).filter((v) => v.kind === "channel");
27936
29086
  return c.json({ ok: true, channels: vaults });
27937
29087
  });
27938
29088
  chatApp.get("/api/messages", async (c) => {
27939
- const token4 = c.req.query("token");
27940
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
29089
+ const token6 = c.req.query("token");
29090
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
27941
29091
  const vault = c.req.query("vault");
27942
29092
  if (!vault) return jsonError("vault is required");
27943
29093
  const parentMessageId = c.req.query("parentMessageId") || void 0;
27944
- const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId }, token4);
29094
+ const r = await memoryCall("listChannelMessagesTool", { vault, parentMessageId }, token6);
27945
29095
  return c.json(r);
27946
29096
  });
27947
29097
  chatApp.get("/api/poll", async (c) => {
27948
- const token4 = c.req.query("token");
27949
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
29098
+ const token6 = c.req.query("token");
29099
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
27950
29100
  const vault = c.req.query("vault");
27951
29101
  if (!vault) return jsonError("vault is required");
27952
- const r = await memoryCall("pollChannelTool", { vault }, token4);
29102
+ const r = await memoryCall("pollChannelTool", { vault }, token6);
27953
29103
  return c.json(r);
27954
29104
  });
27955
29105
  function parseAttachNote(v) {
@@ -27974,12 +29124,12 @@ chatApp.post("/api/reply", async (c) => {
27974
29124
  return c.json(r);
27975
29125
  });
27976
29126
  chatApp.get("/api/notes-own", async (c) => {
27977
- const token4 = c.req.query("token");
27978
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
27979
- const vaultsRes = await memoryCall("listVaultsTool", {}, token4);
29127
+ const token6 = c.req.query("token");
29128
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
29129
+ const vaultsRes = await memoryCall("listVaultsTool", {}, token6);
27980
29130
  if (!vaultsRes.ok) return c.json(vaultsRes);
27981
29131
  const ownVaults = (vaultsRes.vaults ?? []).filter((v) => v.role === "owner" && v.kind === "notes");
27982
- const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle }, token4)));
29132
+ const lists = await Promise.all(ownVaults.map((v) => memoryCall("listTool", { vault: v.handle }, token6)));
27983
29133
  const notes = ownVaults.flatMap((v, i) => {
27984
29134
  const r = lists[i];
27985
29135
  return r.ok ? (r.notes ?? []).map((n) => ({ vault: v.handle, path: n.path, title: n.title })) : [];
@@ -27987,12 +29137,12 @@ chatApp.get("/api/notes-own", async (c) => {
27987
29137
  return c.json({ ok: true, notes });
27988
29138
  });
27989
29139
  chatApp.get("/api/message-note", async (c) => {
27990
- const token4 = c.req.query("token");
27991
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
29140
+ const token6 = c.req.query("token");
29141
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
27992
29142
  const vault = c.req.query("vault");
27993
29143
  const messageId = c.req.query("messageId");
27994
29144
  if (!vault || !messageId) return jsonError("vault and messageId are required");
27995
- const r = await memoryCall("getMessageNoteTool", { vault, messageId }, token4);
29145
+ const r = await memoryCall("getMessageNoteTool", { vault, messageId }, token6);
27996
29146
  return c.json(r);
27997
29147
  });
27998
29148
  chatApp.post("/api/react", async (c) => {
@@ -28003,17 +29153,17 @@ chatApp.post("/api/react", async (c) => {
28003
29153
  return c.json(r);
28004
29154
  });
28005
29155
  chatApp.get("/api/mentions", async (c) => {
28006
- const token4 = c.req.query("token");
28007
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
28008
- const r = await memoryCall("myMentionsTool", {}, token4);
29156
+ const token6 = c.req.query("token");
29157
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
29158
+ const r = await memoryCall("myMentionsTool", {}, token6);
28009
29159
  return c.json(r);
28010
29160
  });
28011
29161
  chatApp.get("/api/members", async (c) => {
28012
- const token4 = c.req.query("token");
28013
- if (!tokenOk(token4)) return jsonError("missing or malformed token", 401);
29162
+ const token6 = c.req.query("token");
29163
+ if (!tokenOk(token6)) return jsonError("missing or malformed token", 401);
28014
29164
  const vault = c.req.query("vault");
28015
29165
  if (!vault) return jsonError("vault is required");
28016
- const r = await memoryCall("listChannelMembersTool", { vault }, token4);
29166
+ const r = await memoryCall("listChannelMembersTool", { vault }, token6);
28017
29167
  return c.json(r);
28018
29168
  });
28019
29169
  chatApp.post("/api/channel", async (c) => {
@@ -28049,13 +29199,13 @@ chatApp.post("/api/member-remove", async (c) => {
28049
29199
  return c.json(r);
28050
29200
  });
28051
29201
  chatApp.get("/:token", (c) => {
28052
- const token4 = c.req.param("token");
28053
- if (!tokenOk(token4)) {
29202
+ const token6 = c.req.param("token");
29203
+ if (!tokenOk(token6)) {
28054
29204
  return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This chat link isn't valid.</h2><p>Ask the AI to run get-chat-link again, or revoke-chat-link then get-chat-link if you think it was compromised.</p></body></html>`, 401);
28055
29205
  }
28056
- return c.html(renderChatPage(token4));
29206
+ return c.html(renderChatPage(token6));
28057
29207
  });
28058
- function renderChatPage(token4) {
29208
+ function renderChatPage(token6) {
28059
29209
  return `<!doctype html>
28060
29210
  <html>
28061
29211
  <head>
@@ -28240,7 +29390,7 @@ function renderChatPage(token4) {
28240
29390
  </div>
28241
29391
 
28242
29392
  <script>
28243
- const TOKEN = ${JSON.stringify(token4)};
29393
+ const TOKEN = ${JSON.stringify(token6)};
28244
29394
  const state = { channels: [], activeVault: null, activeName: null, members: [], threadParent: null, pollTimer: null, pendingAttach: null, threadPendingAttach: null, ownNotes: null, pickerTarget: 'main' };
28245
29395
 
28246
29396
  function api(path, opts) {
@@ -28610,8 +29760,8 @@ var MAX_VAULT_RECORDS = 200;
28610
29760
  var MAX_MEMORY_READ_CONCURRENCY = 4;
28611
29761
  var MAX_MEMORY_READ_ATTEMPTS = 3;
28612
29762
  var vaultApp = new Hono26();
28613
- function validToken(token4) {
28614
- return !!token4 && token4.startsWith("mk_") && token4.length > 20 && token4.length < 256;
29763
+ function validToken(token6) {
29764
+ return !!token6 && token6.startsWith("mk_") && token6.length > 20 && token6.length < 256;
28615
29765
  }
28616
29766
  function isVault(value) {
28617
29767
  return typeof value === "string" && VAULTS.includes(value);
@@ -28737,11 +29887,11 @@ function shouldRetryMemoryRead(error) {
28737
29887
  function sleep(ms) {
28738
29888
  return new Promise((resolve2) => setTimeout(resolve2, ms));
28739
29889
  }
28740
- async function readMemory(tool, args, token4) {
28741
- let result = await memoryCall(tool, args, token4);
29890
+ async function readMemory(tool, args, token6) {
29891
+ let result = await memoryCall(tool, args, token6);
28742
29892
  for (let attempt = 1; !result.ok && attempt < MAX_MEMORY_READ_ATTEMPTS && shouldRetryMemoryRead(result.error); attempt += 1) {
28743
29893
  await sleep(attempt * 150);
28744
- result = await memoryCall(tool, args, token4);
29894
+ result = await memoryCall(tool, args, token6);
28745
29895
  }
28746
29896
  return result;
28747
29897
  }
@@ -28758,27 +29908,27 @@ async function mapWithConcurrency3(items, limit, map) {
28758
29908
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
28759
29909
  return values;
28760
29910
  }
28761
- async function readVault(vault, token4) {
28762
- const list = await readMemory("listTool", { vault }, token4);
29911
+ async function readVault(vault, token6) {
29912
+ const list = await readMemory("listTool", { vault }, token6);
28763
29913
  if (!list.ok) throw new Error(list.error ?? `unable to read ${vault}`);
28764
29914
  const noteRefs = (list.notes ?? []).slice(0, MAX_VAULT_RECORDS);
28765
29915
  const notes = await mapWithConcurrency3(
28766
29916
  noteRefs,
28767
29917
  MAX_MEMORY_READ_CONCURRENCY,
28768
- ({ path: path5 }) => readMemory("getTool", { vault, path: path5 }, token4)
29918
+ ({ path: path5 }) => readMemory("getTool", { vault, path: path5 }, token6)
28769
29919
  );
28770
29920
  const failed = notes.find((result) => !result.ok);
28771
29921
  if (failed && !failed.ok) throw new Error(failed.error ?? `unable to read ${vault} record`);
28772
29922
  return notes.filter((result) => result.ok && !!result.note).map((result) => workspaceRecord(vault, result.note));
28773
29923
  }
28774
29924
  vaultApp.get("/api/workspace", async (c) => {
28775
- const token4 = c.req.query("token");
28776
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
29925
+ const token6 = c.req.query("token");
29926
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28777
29927
  try {
28778
29928
  const requested = c.req.query("vault");
28779
29929
  const vaults = isVault(requested) ? [requested] : VAULTS;
28780
29930
  const results = [];
28781
- for (const vault of vaults) results.push(await readVault(vault, token4));
29931
+ for (const vault of vaults) results.push(await readVault(vault, token6));
28782
29932
  const records = results.flat().sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
28783
29933
  return c.json({ ok: true, records, limited: results.some((result) => result.length >= 200) });
28784
29934
  } catch (error) {
@@ -28787,8 +29937,8 @@ vaultApp.get("/api/workspace", async (c) => {
28787
29937
  });
28788
29938
  vaultApp.post("/api/records", async (c) => {
28789
29939
  const body = await c.req.json().catch(() => ({}));
28790
- const token4 = typeof body.token === "string" ? body.token : void 0;
28791
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
29940
+ const token6 = typeof body.token === "string" ? body.token : void 0;
29941
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28792
29942
  const vault = body.vault;
28793
29943
  const path5 = stringValue(body.path, 320);
28794
29944
  const title = stringValue(body.title, 320);
@@ -28799,19 +29949,19 @@ vaultApp.post("/api/records", async (c) => {
28799
29949
  const validationError2 = validateNewRecord(vault, normalizedProps);
28800
29950
  if (validationError2) return c.json({ ok: false, error: validationError2 }, 400);
28801
29951
  const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
28802
- const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } }, token4);
29952
+ const result = await memoryCall("putTool", { vault, path: path5, title, content: content || title, props: normalizedProps, ...baseRevision === void 0 ? {} : { baseRevision } }, token6);
28803
29953
  return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
28804
29954
  });
28805
29955
  vaultApp.post("/api/records/transition", async (c) => {
28806
29956
  const body = await c.req.json().catch(() => ({}));
28807
- const token4 = typeof body.token === "string" ? body.token : void 0;
28808
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
29957
+ const token6 = typeof body.token === "string" ? body.token : void 0;
29958
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28809
29959
  const vault = typeof body.vault === "string" ? body.vault : "";
28810
29960
  const path5 = typeof body.path === "string" ? body.path : "";
28811
29961
  const status = stringValue(body.status, 80);
28812
29962
  const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
28813
29963
  if (!isVault(vault) || !path5 || !status) return c.json({ ok: false, error: "valid vault, path, and status are required" }, 400);
28814
- const current = await readMemory("getTool", { vault, path: path5 }, token4);
29964
+ const current = await readMemory("getTool", { vault, path: path5 }, token6);
28815
29965
  if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
28816
29966
  const result = await memoryCall("putTool", {
28817
29967
  vault,
@@ -28820,13 +29970,13 @@ vaultApp.post("/api/records/transition", async (c) => {
28820
29970
  content: current.note.content,
28821
29971
  props: { ...current.note.props ?? {}, status },
28822
29972
  baseRevision: baseRevision ?? current.note.revision
28823
- }, token4);
29973
+ }, token6);
28824
29974
  return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
28825
29975
  });
28826
29976
  vaultApp.post("/api/records/update", async (c) => {
28827
29977
  const body = await c.req.json().catch(() => ({}));
28828
- const token4 = typeof body.token === "string" ? body.token : void 0;
28829
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
29978
+ const token6 = typeof body.token === "string" ? body.token : void 0;
29979
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28830
29980
  const vault = body.vault;
28831
29981
  const path5 = stringValue(body.path, 320);
28832
29982
  const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
@@ -28836,7 +29986,7 @@ vaultApp.post("/api/records/update", async (c) => {
28836
29986
  if (!isVault(vault) || !path5 || body.title !== void 0 && !title || body.props !== void 0 && !props) {
28837
29987
  return c.json({ ok: false, error: "valid vault, path, and optional record fields are required" }, 400);
28838
29988
  }
28839
- const current = await readMemory("getTool", { vault, path: path5 }, token4);
29989
+ const current = await readMemory("getTool", { vault, path: path5 }, token6);
28840
29990
  if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
28841
29991
  const nextProps = normalizeRelationshipProps(vault, { ...current.note.props ?? {}, ...props ?? {} });
28842
29992
  const validationError2 = validateNewRecord(vault, nextProps);
@@ -28848,18 +29998,18 @@ vaultApp.post("/api/records/update", async (c) => {
28848
29998
  content: content ?? current.note.content,
28849
29999
  props: nextProps,
28850
30000
  baseRevision: baseRevision ?? current.note.revision
28851
- }, token4);
30001
+ }, token6);
28852
30002
  return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
28853
30003
  });
28854
30004
  vaultApp.delete("/api/records", async (c) => {
28855
30005
  const body = await c.req.json().catch(() => ({}));
28856
- const token4 = typeof body.token === "string" ? body.token : void 0;
28857
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
30006
+ const token6 = typeof body.token === "string" ? body.token : void 0;
30007
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28858
30008
  const vault = body.vault;
28859
30009
  const path5 = stringValue(body.path, 320);
28860
30010
  const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
28861
30011
  if (vault !== "Tasks" || !path5) return c.json({ ok: false, error: "only a valid task record can be deleted" }, 400);
28862
- const current = await readMemory("getTool", { vault, path: path5 }, token4);
30012
+ const current = await readMemory("getTool", { vault, path: path5 }, token6);
28863
30013
  if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
28864
30014
  if (!["task", "milestone"].includes(typeof current.note.props?.type === "string" ? current.note.props.type : defaultType("Tasks"))) {
28865
30015
  return c.json({ ok: false, error: "only task records can be deleted" }, 400);
@@ -28868,13 +30018,13 @@ vaultApp.delete("/api/records", async (c) => {
28868
30018
  vault,
28869
30019
  path: path5,
28870
30020
  baseRevision: baseRevision ?? current.note.revision
28871
- }, token4);
30021
+ }, token6);
28872
30022
  return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
28873
30023
  });
28874
30024
  vaultApp.post("/api/communications/review", async (c) => {
28875
30025
  const body = await c.req.json().catch(() => ({}));
28876
- const token4 = typeof body.token === "string" ? body.token : void 0;
28877
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
30026
+ const token6 = typeof body.token === "string" ? body.token : void 0;
30027
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28878
30028
  const path5 = stringValue(body.path, 320);
28879
30029
  const action2 = stringValue(body.action, 24);
28880
30030
  const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
@@ -28885,7 +30035,7 @@ vaultApp.post("/api/communications/review", async (c) => {
28885
30035
  if (!path5 || !normalizedAction || !["approve", "deny", "edit"].includes(normalizedAction) || body.title !== void 0 && !title) {
28886
30036
  return c.json({ ok: false, error: "valid path, action, and optional draft fields are required" }, 400);
28887
30037
  }
28888
- const current = await readMemory("getTool", { vault: "Communications", path: path5 }, token4);
30038
+ const current = await readMemory("getTool", { vault: "Communications", path: path5 }, token6);
28889
30039
  if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
28890
30040
  if (current.note.props?.type !== "email") return c.json({ ok: false, error: "only email communications can be reviewed" }, 400);
28891
30041
  const reviewed = normalizedAction === "approve" || normalizedAction === "deny";
@@ -28909,13 +30059,13 @@ vaultApp.post("/api/communications/review", async (c) => {
28909
30059
  content: content ?? current.note.content,
28910
30060
  props,
28911
30061
  baseRevision: baseRevision ?? current.note.revision
28912
- }, token4);
30062
+ }, token6);
28913
30063
  return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
28914
30064
  });
28915
30065
  vaultApp.post("/api/activity", async (c) => {
28916
30066
  const body = await c.req.json().catch(() => ({}));
28917
- const token4 = typeof body.token === "string" ? body.token : void 0;
28918
- if (!validToken(token4)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
30067
+ const token6 = typeof body.token === "string" ? body.token : void 0;
30068
+ if (!validToken(token6)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
28919
30069
  const title = stringValue(body.title, 320);
28920
30070
  const content = typeof body.content === "string" && body.content.length <= 2e4 ? body.content : "";
28921
30071
  const props = propsValue(body.props) ?? {};
@@ -28925,7 +30075,7 @@ vaultApp.post("/api/activity", async (c) => {
28925
30075
  return c.json({ ok: false, error: "Activities require at least one canonical People reference in person_refs" }, 400);
28926
30076
  }
28927
30077
  for (const ref of personRefs) {
28928
- const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") }, token4);
30078
+ const person = await readMemory("getTool", { vault: "People", path: ref.replace(/^People::/, "") }, token6);
28929
30079
  if (!person.ok || !person.note || !["person", "organization"].includes(String(person.note.props?.type ?? ""))) {
28930
30080
  return c.json({ ok: false, error: `Activity person reference is not a known People record: ${ref}` }, 400);
28931
30081
  }
@@ -28953,37 +30103,37 @@ vaultApp.post("/api/activity", async (c) => {
28953
30103
  ...props,
28954
30104
  person_refs: personRefs
28955
30105
  }
28956
- }, token4);
30106
+ }, token6);
28957
30107
  return c.json(result, result.ok ? 200 : 502);
28958
30108
  });
28959
30109
 
28960
30110
  // src/api/schedule-routes.ts
28961
30111
  import { Hono as Hono27 } from "hono";
28962
30112
  var scheduleApp = new Hono27();
28963
- function tokenOk2(token4) {
28964
- return !!token4 && token4.startsWith("mk_") && token4.length > 10;
30113
+ function tokenOk2(token6) {
30114
+ return !!token6 && token6.startsWith("mk_") && token6.length > 10;
28965
30115
  }
28966
30116
  function jsonError2(message, status = 400) {
28967
30117
  return new Response(JSON.stringify({ ok: false, error: message }), { status, headers: { "content-type": "application/json" } });
28968
30118
  }
28969
30119
  scheduleApp.get("/api/status", async (c) => {
28970
- const token4 = c.req.query("token");
28971
- if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
28972
- const r = await memoryCall("getScheduleStatusTool", {}, token4);
30120
+ const token6 = c.req.query("token");
30121
+ if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
30122
+ const r = await memoryCall("getScheduleStatusTool", {}, token6);
28973
30123
  return c.json(r);
28974
30124
  });
28975
30125
  scheduleApp.get("/api/vaults", async (c) => {
28976
- const token4 = c.req.query("token");
28977
- if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
28978
- const r = await memoryCall("listVaultsTool", {}, token4);
30126
+ const token6 = c.req.query("token");
30127
+ if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
30128
+ const r = await memoryCall("listVaultsTool", {}, token6);
28979
30129
  if (!r.ok) return c.json(r);
28980
30130
  const vaults = (r.vaults ?? []).filter((v) => v.kind !== "channel");
28981
30131
  return c.json({ ok: true, vaults });
28982
30132
  });
28983
30133
  scheduleApp.get("/api/actions", async (c) => {
28984
- const token4 = c.req.query("token");
28985
- if (!tokenOk2(token4)) return jsonError2("missing or malformed token", 401);
28986
- const r = await memoryCall("listScheduledActionsTool", {}, token4);
30134
+ const token6 = c.req.query("token");
30135
+ if (!tokenOk2(token6)) return jsonError2("missing or malformed token", 401);
30136
+ const r = await memoryCall("listScheduledActionsTool", {}, token6);
28987
30137
  return c.json(r);
28988
30138
  });
28989
30139
  scheduleApp.post("/api/propose", async (c) => {
@@ -29019,13 +30169,13 @@ scheduleApp.post("/api/actions/:id/delete", async (c) => {
29019
30169
  return c.json(r);
29020
30170
  });
29021
30171
  scheduleApp.get("/:token", (c) => {
29022
- const token4 = c.req.param("token");
29023
- if (!tokenOk2(token4)) {
30172
+ const token6 = c.req.param("token");
30173
+ if (!tokenOk2(token6)) {
29024
30174
  return c.html(`<!doctype html><html><body style="font-family:system-ui;padding:40px;color:#3D3548"><h2>This automations link isn't valid.</h2><p>Ask the AI to run get-schedule-link again, or revoke-schedule-link then get-schedule-link if you think it was compromised.</p></body></html>`, 401);
29025
30175
  }
29026
- return c.html(renderSchedulePage(token4));
30176
+ return c.html(renderSchedulePage(token6));
29027
30177
  });
29028
- function renderSchedulePage(token4) {
30178
+ function renderSchedulePage(token6) {
29029
30179
  return `<!doctype html>
29030
30180
  <html>
29031
30181
  <head>
@@ -29123,7 +30273,7 @@ function renderSchedulePage(token4) {
29123
30273
  </div>
29124
30274
 
29125
30275
  <script>
29126
- const TOKEN = ${JSON.stringify(token4)};
30276
+ const TOKEN = ${JSON.stringify(token6)};
29127
30277
  const state = { actions: [], vaults: [] };
29128
30278
 
29129
30279
  function api(path, opts) { return fetch('/schedule' + path, opts).then(r => r.json()); }
@@ -29366,8 +30516,8 @@ function supportThreadToken(customerEmail, normalizedSubject) {
29366
30516
  return shortHash(`${customerEmail.trim().toLowerCase()}
29367
30517
  ${normalizeSupportSubject(normalizedSubject).toLowerCase()}`);
29368
30518
  }
29369
- function supportThreadAddress(token4, receivingDomain) {
29370
- const safeToken = token4.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
30519
+ function supportThreadAddress(token6, receivingDomain) {
30520
+ const safeToken = token6.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
29371
30521
  const domain = receivingDomain.trim().toLowerCase();
29372
30522
  if (!safeToken || !/^[a-z0-9.-]+\.[a-z]{2,}$/i.test(domain)) throw new Error("invalid support thread address");
29373
30523
  return `support+${safeToken}@${domain}`;
@@ -29394,8 +30544,8 @@ function personPath(identity) {
29394
30544
  const local = identity.email.split("@")[0];
29395
30545
  return `contacts/${safePathPart(identity.name === identity.email ? local : identity.name)}-${shortHash(identity.email, 12)}`;
29396
30546
  }
29397
- function issuePath(token4) {
29398
- return `support/threads/${token4}`;
30547
+ function issuePath(token6) {
30548
+ return `support/threads/${token6}`;
29399
30549
  }
29400
30550
  function communicationPath(email) {
29401
30551
  const date = cleanLine(email.created_at, (/* @__PURE__ */ new Date()).toISOString()).slice(0, 10).replace(/[^0-9-]/g, "");
@@ -29579,8 +30729,8 @@ function buildIssueContent(email, customer, personPathValue, threadAddressValue)
29579
30729
  `- Person: ${noteRef("People", personPathValue)}`
29580
30730
  ].join("\n");
29581
30731
  }
29582
- async function findOrCreateIssue(memoryKey, email, customer, personPathValue, token4, threadAddressValue) {
29583
- const path5 = issuePath(token4);
30732
+ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, token6, threadAddressValue) {
30733
+ const path5 = issuePath(token6);
29584
30734
  const existing = await getNote(memoryKey, "Issues", path5);
29585
30735
  if (existing) return existing;
29586
30736
  const subject = normalizeSupportSubject(email.subject);
@@ -29590,7 +30740,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
29590
30740
  path: path5,
29591
30741
  title: `Support thread \u2014 ${subject} \u2014 ${customer.email}`.slice(0, 240),
29592
30742
  content: buildIssueContent(email, customer, personPathValue, threadAddressValue),
29593
- source: `resend:thread:${token4}`,
30743
+ source: `resend:thread:${token6}`,
29594
30744
  type: "user_friction",
29595
30745
  tagCandidates: ISSUE_TAGS,
29596
30746
  props: {
@@ -29599,7 +30749,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
29599
30749
  summary,
29600
30750
  pinned: false,
29601
30751
  source_type: "thread",
29602
- source_ref: `resend:thread:${token4}`,
30752
+ source_ref: `resend:thread:${token6}`,
29603
30753
  related: [],
29604
30754
  related_vault_notes: [noteRef("People", personPathValue)],
29605
30755
  embed: true,
@@ -29614,7 +30764,7 @@ async function findOrCreateIssue(memoryKey, email, customer, personPathValue, to
29614
30764
  customer_email: customer.email,
29615
30765
  customer_name: customer.name,
29616
30766
  normalized_subject: subject,
29617
- thread_token: token4,
30767
+ thread_token: token6,
29618
30768
  thread_address: threadAddressValue,
29619
30769
  communication_refs: [],
29620
30770
  message_count: 0,
@@ -29657,7 +30807,7 @@ function buildCommunicationContent(email, customer, direction, personPathValue,
29657
30807
  ...parentPath ? [`- Previous message: [[${parentPath}]]`] : []
29658
30808
  ].join("\n");
29659
30809
  }
29660
- async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue, token4, parentPath) {
30810
+ async function captureCommunication(memoryKey, email, customer, direction, personPathValue, issuePathValue, token6, parentPath) {
29661
30811
  const path5 = communicationPath(email);
29662
30812
  const existing = await getNote(memoryKey, "Communication", path5);
29663
30813
  if (existing) return existing;
@@ -29696,8 +30846,8 @@ async function captureCommunication(memoryKey, email, customer, direction, perso
29696
30846
  provider: "resend",
29697
30847
  provider_connection_ref: "resend:updates.mcpscraper.dev",
29698
30848
  provider_message_ref: providerMessageRef,
29699
- provider_thread_ref: token4,
29700
- thread_ref: token4,
30849
+ provider_thread_ref: token6,
30850
+ thread_ref: token6,
29701
30851
  issue_ref: issuePathValue,
29702
30852
  sender_email: supportSender(email)?.email ?? email.from,
29703
30853
  recipient_emails: email.to ?? [],
@@ -29757,8 +30907,8 @@ async function updateIssueThread(memoryKey, issuePathValue, email, communication
29757
30907
  if (!result.conflict || attempt === 1) throw new Error(result.error ?? "support thread issue update failed");
29758
30908
  }
29759
30909
  }
29760
- async function getSupportThreadByToken(memoryKey, token4) {
29761
- const safeToken = token4.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
30910
+ async function getSupportThreadByToken(memoryKey, token6) {
30911
+ const safeToken = token6.toLowerCase().replace(/[^a-f0-9]/g, "").slice(0, 24);
29762
30912
  if (safeToken.length !== 24) return null;
29763
30913
  const path5 = issuePath(safeToken);
29764
30914
  const issue = await getNote(memoryKey, "Issues", path5);
@@ -29786,8 +30936,8 @@ async function captureSupportMessage(memoryKey, email, options) {
29786
30936
  const customer = options.customer ?? supportSender(email);
29787
30937
  if (!customer) throw new Error("support sender does not contain a usable email address");
29788
30938
  const normalizedSubject = options.existingThread?.normalizedSubject ?? normalizeSupportSubject(email.subject);
29789
- const token4 = options.existingThread?.threadToken ?? supportThreadToken(customer.email, normalizedSubject);
29790
- const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(token4, options.receivingDomain);
30939
+ const token6 = options.existingThread?.threadToken ?? supportThreadToken(customer.email, normalizedSubject);
30940
+ const threadAddressValue = options.existingThread?.threadAddress ?? supportThreadAddress(token6, options.receivingDomain);
29791
30941
  const person = options.existingThread ? await getNote(memoryKey, "People", options.existingThread.personPath) : await findOrCreatePerson(memoryKey, customer, email.created_at);
29792
30942
  if (!person) throw new Error("support person could not be resolved");
29793
30943
  const issue = options.existingThread?.issue ?? await findOrCreateIssue(
@@ -29795,7 +30945,7 @@ async function captureSupportMessage(memoryKey, email, options) {
29795
30945
  email,
29796
30946
  customer,
29797
30947
  person.path,
29798
- token4,
30948
+ token6,
29799
30949
  threadAddressValue
29800
30950
  );
29801
30951
  const parentPath = typeof issue.props?.last_communication_ref === "string" ? issue.props.last_communication_ref : void 0;
@@ -29806,7 +30956,7 @@ async function captureSupportMessage(memoryKey, email, options) {
29806
30956
  options.direction,
29807
30957
  person.path,
29808
30958
  issue.path,
29809
- token4,
30959
+ token6,
29810
30960
  parentPath
29811
30961
  );
29812
30962
  await updateIssueThread(memoryKey, issue.path, email, communication.path, options.direction, person.path);
@@ -29815,7 +30965,7 @@ async function captureSupportMessage(memoryKey, email, options) {
29815
30965
  communicationPath: communication.path,
29816
30966
  personPath: person.path,
29817
30967
  parentCommunicationPath: parentPath,
29818
- threadToken: token4,
30968
+ threadToken: token6,
29819
30969
  threadAddress: threadAddressValue,
29820
30970
  customerEmail: customer.email,
29821
30971
  normalizedSubject,
@@ -29988,8 +31138,8 @@ resendInboundApp.post("/webhooks", async (c) => {
29988
31138
  threadAddress: captured.threadAddress
29989
31139
  });
29990
31140
  }
29991
- const token4 = extractSupportThreadToken(receivedEvent.data.to ?? [], ...domains);
29992
- const thread = token4 ? await getSupportThreadByToken(memoryKey, token4) : null;
31141
+ const token6 = extractSupportThreadToken(receivedEvent.data.to ?? [], ...domains);
31142
+ const thread = token6 ? await getSupportThreadByToken(memoryKey, token6) : null;
29993
31143
  if (!thread) return c.json({ ok: true, route, ignored: true, reason: "unknown-support-thread" });
29994
31144
  const operator = parseMailbox(destination);
29995
31145
  if (operator && sender.email === operator.email) {
@@ -30275,8 +31425,8 @@ var workshopFieldNotes = [
30275
31425
  function escapeHtml3(value) {
30276
31426
  return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&#039;");
30277
31427
  }
30278
- function workshopFieldNoteUrl(slug2) {
30279
- return `${WORKSHOP_FIELD_NOTES_URL}?article=${encodeURIComponent(slug2)}`;
31428
+ function workshopFieldNoteUrl(slug4) {
31429
+ return `${WORKSHOP_FIELD_NOTES_URL}?article=${encodeURIComponent(slug4)}`;
30280
31430
  }
30281
31431
  function renderWorkshopFieldNotesBody() {
30282
31432
  let activeCategory = "";
@@ -30835,16 +31985,16 @@ import { z as z31 } from "zod";
30835
31985
  // src/editorial-reading-room/render.ts
30836
31986
  import { createHash as createHash11 } from "crypto";
30837
31987
  import { readFileSync as readFileSync3 } from "fs";
30838
- import { dirname as dirname3, join as join6, resolve } from "path";
31988
+ import { dirname as dirname3, join as join7, resolve } from "path";
30839
31989
  import { Marked, Renderer } from "marked";
30840
31990
  var runtimeEntryDir = dirname3(resolve(process.argv[1] || process.cwd()));
30841
31991
  var assetCache = /* @__PURE__ */ new Map();
30842
31992
  function assetPath(name) {
30843
31993
  const candidates = [
30844
- join6(runtimeEntryDir, "editorial-reading-room", "assets", name),
30845
- join6(runtimeEntryDir, "..", "editorial-reading-room", "assets", name),
30846
- join6(process.cwd(), "src", "editorial-reading-room", "assets", name),
30847
- join6(process.cwd(), "dist", "editorial-reading-room", "assets", name)
31994
+ join7(runtimeEntryDir, "editorial-reading-room", "assets", name),
31995
+ join7(runtimeEntryDir, "..", "editorial-reading-room", "assets", name),
31996
+ join7(process.cwd(), "src", "editorial-reading-room", "assets", name),
31997
+ join7(process.cwd(), "dist", "editorial-reading-room", "assets", name)
30848
31998
  ];
30849
31999
  for (const candidate of candidates) {
30850
32000
  try {
@@ -30918,8 +32068,8 @@ function addHeadingIds(html) {
30918
32068
  });
30919
32069
  return { rendered, toc };
30920
32070
  }
30921
- function safeFilename(requested, slug2) {
30922
- const stem = (requested || slug2).replace(/\.html?$/i, "").replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 110) || "editorial-reading-room";
32071
+ function safeFilename(requested, slug4) {
32072
+ const stem = (requested || slug4).replace(/\.html?$/i, "").replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 110) || "editorial-reading-room";
30923
32073
  return `${stem}.html`;
30924
32074
  }
30925
32075
  function escapedJsonForHtml(value) {
@@ -31075,20 +32225,20 @@ ${provenance}`;
31075
32225
  var EDITORIAL_READING_ROOM_ARTIFACT_PREFIX = "editorial-reading-rooms/";
31076
32226
  var EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
31077
32227
  var EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
31078
- function token2() {
32228
+ function token3() {
31079
32229
  return process.env.EDITORIAL_READING_ROOM_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
31080
32230
  }
31081
- function policy2() {
32231
+ function policy3() {
31082
32232
  return {
31083
32233
  prefix: EDITORIAL_READING_ROOM_ARTIFACT_PREFIX,
31084
32234
  artifactTtlMs: EDITORIAL_READING_ROOM_ARTIFACT_TTL_MS,
31085
32235
  downloadTtlMs: EDITORIAL_READING_ROOM_DOWNLOAD_TTL_MS,
31086
- token: token2()
32236
+ token: token3()
31087
32237
  };
31088
32238
  }
31089
32239
  function createEditorialReadingRoomArtifact(args) {
31090
32240
  return createPrivateArtifact({
31091
- policy: policy2(),
32241
+ policy: policy3(),
31092
32242
  ownerId: args.ownerId,
31093
32243
  artifactKey: `${args.rendered.sha256.slice(0, 12)}-${args.rendered.filename}`,
31094
32244
  createdAt: args.rendered.generatedAt,
@@ -31099,7 +32249,7 @@ function createEditorialReadingRoomArtifact(args) {
31099
32249
  }
31100
32250
  function renewEditorialReadingRoomArtifactDownload(args) {
31101
32251
  return renewPrivateArtifactDownload({
31102
- policy: policy2(),
32252
+ policy: policy3(),
31103
32253
  ownerId: args.ownerId,
31104
32254
  artifactId: args.artifactId
31105
32255
  });
@@ -31879,7 +33029,7 @@ async function searchCommonsEntities(filters = {}, userId) {
31879
33029
  async function prepareCommonsEntity(input, user) {
31880
33030
  await ensureCommonsSchema();
31881
33031
  const title = optionalText2(input.title, 180) ?? "";
31882
- const slug2 = normalizeSlug(input.slug || title);
33032
+ const slug4 = normalizeSlug(input.slug || title);
31883
33033
  const schemaOrgType = optionalText2(input.schemaOrgType, 80);
31884
33034
  const contract = commonsEntityProfileFor(input.entityType, schemaOrgType);
31885
33035
  const entityType = contract.entityType;
@@ -31887,7 +33037,7 @@ async function prepareCommonsEntity(input, user) {
31887
33037
  const headings = extractMarkdownHeadings(bodyMarkdown);
31888
33038
  const duplicateCandidates = await potentialDuplicateCandidates({
31889
33039
  title,
31890
- slug: slug2,
33040
+ slug: slug4,
31891
33041
  source: normalizeSource(input.source),
31892
33042
  maxCandidates: clampNumber(input.maxCandidates, 1, 20, 8)
31893
33043
  });
@@ -31904,8 +33054,8 @@ async function prepareCommonsEntity(input, user) {
31904
33054
  entityIdPrefix: "TPW-Q",
31905
33055
  userId: user?.id ?? null,
31906
33056
  normalizedEntityType: entityType,
31907
- proposedSlug: slug2 || null,
31908
- publicUrl: slug2 ? publicUrlForSlug(slug2) : null,
33057
+ proposedSlug: slug4 || null,
33058
+ publicUrl: slug4 ? publicUrlForSlug(slug4) : null,
31909
33059
  writePath: "commons_prepare_entity -> commons_validate_entity -> commons_submit_entity"
31910
33060
  },
31911
33061
  contract,
@@ -31940,7 +33090,7 @@ async function validateCommonsEntity(input, user) {
31940
33090
  const warnings = [];
31941
33091
  const title = optionalText2(input.title, 180);
31942
33092
  const description = optionalText2(input.description, 600);
31943
- const slug2 = normalizeSlug(input.slug || title || "");
33093
+ const slug4 = normalizeSlug(input.slug || title || "");
31944
33094
  const contract = commonsEntityProfileFor(input.entityType, input.seo?.schemaOrgType);
31945
33095
  const source = normalizeSource(input.source);
31946
33096
  const featuredImage = normalizeFeaturedImage(input.featuredImage);
@@ -31949,11 +33099,11 @@ async function validateCommonsEntity(input, user) {
31949
33099
  const placeholderSections = unsupportedPlaceholderSections(bodyMarkdown);
31950
33100
  const duplicateCandidates = await potentialDuplicateCandidates({
31951
33101
  title: title ?? "",
31952
- slug: slug2,
33102
+ slug: slug4,
31953
33103
  source,
31954
33104
  maxCandidates: 8
31955
33105
  });
31956
- const existing = slug2 || input.entityId ? await findEntityBySlugOrIdAnyStatus(slug2, input.entityId) : null;
33106
+ const existing = slug4 || input.entityId ? await findEntityBySlugOrIdAnyStatus(slug4, input.entityId) : null;
31957
33107
  try {
31958
33108
  normalizeCommonsClaims(input.claims);
31959
33109
  } catch (error) {
@@ -31961,7 +33111,7 @@ async function validateCommonsEntity(input, user) {
31961
33111
  }
31962
33112
  if (!title || title.length < 3) errors.push("title must be at least 3 characters.");
31963
33113
  if (!description || description.length < 8) errors.push("description must be at least 8 characters.");
31964
- if (!slug2) errors.push("A Commons entity needs a URL slug.");
33114
+ if (!slug4) errors.push("A Commons entity needs a URL slug.");
31965
33115
  if (!featuredImage?.url) errors.push("A featured image is required before a Commons entity can publish.");
31966
33116
  if (!bodyMarkdown.trim() && !source.originalUrl && !source.resolvedUrl) {
31967
33117
  errors.push("A public entity needs article body text or source evidence before publishing.");
@@ -32422,13 +33572,13 @@ async function findProposalByIdempotency(userId, idempotencyKey3) {
32422
33572
  });
32423
33573
  return result.rows[0] ? rowToProposal(result.rows[0]) : null;
32424
33574
  }
32425
- async function findEntityBySlugOrIdAnyStatus(slug2, entityId) {
33575
+ async function findEntityBySlugOrIdAnyStatus(slug4, entityId) {
32426
33576
  const result = entityId ? await getDb().execute({
32427
33577
  sql: "SELECT * FROM commons_entities WHERE slug = ? OR id = ? LIMIT 1",
32428
- args: [slug2, entityId]
33578
+ args: [slug4, entityId]
32429
33579
  }) : await getDb().execute({
32430
33580
  sql: "SELECT * FROM commons_entities WHERE slug = ? LIMIT 1",
32431
- args: [slug2]
33581
+ args: [slug4]
32432
33582
  });
32433
33583
  return result.rows[0] ? rowToEntity(result.rows[0]) : null;
32434
33584
  }
@@ -32620,13 +33770,13 @@ function canonicalizeSourceUrl(value) {
32620
33770
  function normalizeSubmitInput(input) {
32621
33771
  const title = requiredText(input.title, "title", 3, 180);
32622
33772
  const description = requiredText(input.description, "description", 8, 600);
32623
- const slug2 = normalizeSlug(input.slug || title);
32624
- if (!slug2) throw new CommonsRepositoryError("slug_required", "A Commons entity needs a URL slug.");
33773
+ const slug4 = normalizeSlug(input.slug || title);
33774
+ if (!slug4) throw new CommonsRepositoryError("slug_required", "A Commons entity needs a URL slug.");
32625
33775
  return {
32626
33776
  idempotencyKey: requiredText(input.idempotencyKey, "idempotencyKey", 8, 200),
32627
33777
  title,
32628
33778
  description,
32629
- slug: slug2,
33779
+ slug: slug4,
32630
33780
  entityId: normalizeEntityId(input.entityId),
32631
33781
  entityType: normalizeEntityType(input.entityType),
32632
33782
  disambiguationName: optionalText2(input.disambiguationName, 240) ?? void 0,
@@ -33175,13 +34325,13 @@ function normalizeRelatedLinks(relatedLinks) {
33175
34325
  return relatedLinks.map((link) => {
33176
34326
  const title = requiredText(link.title, "relatedLinks.title", 1, 180);
33177
34327
  const url = optionalText2(link.url, 2e3);
33178
- const slug2 = link.slug ? normalizeSlug(link.slug) : void 0;
34328
+ const slug4 = link.slug ? normalizeSlug(link.slug) : void 0;
33179
34329
  const entityId = normalizeEntityId(link.entityId);
33180
- const needsLink = link.needsLink === true || !url && !slug2 && !entityId;
34330
+ const needsLink = link.needsLink === true || !url && !slug4 && !entityId;
33181
34331
  return {
33182
34332
  title,
33183
34333
  url,
33184
- slug: slug2,
34334
+ slug: slug4,
33185
34335
  entityId,
33186
34336
  relationship: optionalText2(link.relationship, 120) ?? void 0,
33187
34337
  summary: optionalText2(link.summary, 300) ?? void 0,
@@ -33412,8 +34562,8 @@ function actorLabel(input, user) {
33412
34562
  function publicBaseUrl() {
33413
34563
  return (process.env.TRANSPARENT_COMMONS_PUBLIC_BASE_URL?.trim() || DEFAULT_COMMONS_BASE_URL).replace(/\/+$/, "");
33414
34564
  }
33415
- function publicUrlForSlug(slug2) {
33416
- return `${publicBaseUrl()}/wiki/${encodeURIComponent(slug2)}`;
34565
+ function publicUrlForSlug(slug4) {
34566
+ return `${publicBaseUrl()}/wiki/${encodeURIComponent(slug4)}`;
33417
34567
  }
33418
34568
  function allocateEntityId() {
33419
34569
  return `TPW-Q${Date.now()}${Math.floor(Math.random() * 1e3).toString().padStart(3, "0")}`;
@@ -33615,11 +34765,11 @@ async function validateCommonsPublication(input, user) {
33615
34765
  } catch (error) {
33616
34766
  errors.push(error instanceof Error ? error.message : String(error));
33617
34767
  }
33618
- const slug2 = normalizeEditionSlug(input.editionSlug || input.edition.site.slug);
33619
- const latest = publication ? await getLatestEdition(publication.id, slug2) : null;
33620
- if (latest && input.baseRevision === void 0) errors.push(`Edition ${slug2} already exists at revision ${latest.revision}; supply baseRevision to edit it.`);
34768
+ const slug4 = normalizeEditionSlug(input.editionSlug || input.edition.site.slug);
34769
+ const latest = publication ? await getLatestEdition(publication.id, slug4) : null;
34770
+ if (latest && input.baseRevision === void 0) errors.push(`Edition ${slug4} already exists at revision ${latest.revision}; supply baseRevision to edit it.`);
33621
34771
  if (latest && input.baseRevision !== void 0 && latest.revision !== input.baseRevision) {
33622
- errors.push(`Edition ${slug2} is revision ${latest.revision}; the proposed edit targeted ${input.baseRevision}.`);
34772
+ errors.push(`Edition ${slug4} is revision ${latest.revision}; the proposed edit targeted ${input.baseRevision}.`);
33623
34773
  }
33624
34774
  }
33625
34775
  }
@@ -34368,7 +35518,7 @@ import Papa2 from "papaparse";
34368
35518
  import {
34369
35519
  createHash as createHash15,
34370
35520
  createHmac as createHmac3,
34371
- randomBytes as randomBytes3,
35521
+ randomBytes as randomBytes4,
34372
35522
  timingSafeEqual as timingSafeEqual2
34373
35523
  } from "crypto";
34374
35524
  import { z as z33 } from "zod";
@@ -34580,7 +35730,7 @@ function dashboardCallbackUrl() {
34580
35730
  }
34581
35731
  async function createThorbitConnectUrl(userId) {
34582
35732
  if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
34583
- const state = randomBytes3(32).toString("base64url");
35733
+ const state = randomBytes4(32).toString("base64url");
34584
35734
  const stateHash = createHash15("sha256").update(state).digest("hex");
34585
35735
  await getAnalyticsPool().query(
34586
35736
  `INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
@@ -34593,8 +35743,8 @@ async function createThorbitConnectUrl(userId) {
34593
35743
  url.searchParams.set("state", state);
34594
35744
  return url.toString();
34595
35745
  }
34596
- function verifyThorbitAssertion(token4) {
34597
- const [encoded, signature] = token4.split(".");
35746
+ function verifyThorbitAssertion(token6) {
35747
+ const [encoded, signature] = token6.split(".");
34598
35748
  if (!encoded || !signature || !bridgeConfigured()) throw new Error("Invalid Thorbit assertion");
34599
35749
  const expected = createHmac3("sha256", bridgeSecret()).update(encoded).digest();
34600
35750
  const actual = Buffer.from(signature, "base64url");
@@ -36072,7 +37222,7 @@ function scheduledArtifactOwnerIdForApiKey(apiKey) {
36072
37222
  }
36073
37223
 
36074
37224
  // src/api/scheduled-run-view-links.ts
36075
- import { createHash as createHash18, randomBytes as randomBytes4, randomUUID as randomUUID17 } from "crypto";
37225
+ import { createHash as createHash18, randomBytes as randomBytes5, randomUUID as randomUUID17 } from "crypto";
36076
37226
  var schemaReady2 = false;
36077
37227
  async function ensureScheduledRunViewLinksSchema() {
36078
37228
  if (schemaReady2) return;
@@ -36125,8 +37275,8 @@ async function ensureScheduledRunViewLinksSchema() {
36125
37275
  `);
36126
37276
  schemaReady2 = true;
36127
37277
  }
36128
- function tokenHash2(token4) {
36129
- return createHash18("sha256").update(token4).digest("hex");
37278
+ function tokenHash2(token6) {
37279
+ return createHash18("sha256").update(token6).digest("hex");
36130
37280
  }
36131
37281
  function mapRow(row) {
36132
37282
  return {
@@ -36146,7 +37296,7 @@ function mapRow(row) {
36146
37296
  async function createScheduledRunViewLink(input) {
36147
37297
  await ensureScheduledRunViewLinksSchema();
36148
37298
  const now = input.now ?? /* @__PURE__ */ new Date();
36149
- const token4 = randomBytes4(32).toString("base64url");
37299
+ const token6 = randomBytes5(32).toString("base64url");
36150
37300
  const record = {
36151
37301
  shareId: randomUUID17(),
36152
37302
  ownerId: input.ownerId,
@@ -36155,7 +37305,7 @@ async function createScheduledRunViewLink(input) {
36155
37305
  title: input.title.slice(0, 300),
36156
37306
  summary: input.summary.slice(0, 2e3),
36157
37307
  rendererKey: input.rendererKey,
36158
- tokenHash: tokenHash2(token4),
37308
+ tokenHash: tokenHash2(token6),
36159
37309
  expiresAt: new Date(now.getTime() + input.expiresInDays * 24 * 60 * 60 * 1e3).toISOString(),
36160
37310
  revokedAt: null,
36161
37311
  createdAt: now.toISOString()
@@ -36180,9 +37330,9 @@ async function createScheduledRunViewLink(input) {
36180
37330
  record.createdAt
36181
37331
  ]
36182
37332
  });
36183
- return { record, token: token4 };
37333
+ return { record, token: token6 };
36184
37334
  }
36185
- async function resolveScheduledRunViewLink(token4, now = /* @__PURE__ */ new Date()) {
37335
+ async function resolveScheduledRunViewLink(token6, now = /* @__PURE__ */ new Date()) {
36186
37336
  await ensureScheduledRunViewLinksSchema();
36187
37337
  const result = await getDb().execute({
36188
37338
  sql: `
@@ -36194,7 +37344,7 @@ async function resolveScheduledRunViewLink(token4, now = /* @__PURE__ */ new Dat
36194
37344
  AND expires_at > ?
36195
37345
  LIMIT 1
36196
37346
  `,
36197
- args: [tokenHash2(token4), now.toISOString()]
37347
+ args: [tokenHash2(token6), now.toISOString()]
36198
37348
  });
36199
37349
  const row = result.rows[0];
36200
37350
  return row ? mapRow(row) : null;
@@ -36609,7 +37759,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
36609
37759
  const artifact = result.data.run.artifactOutputs.find((output) => output.artifactId === body.data.artifactId);
36610
37760
  if (!artifact) return c.json(apiError("The scheduled result artifact was not found.", "not_found"), 404);
36611
37761
  const ownerId = scheduledArtifactOwnerIdForApiKey(user.api_key);
36612
- const { record, token: token4 } = await createScheduledRunViewLink({
37762
+ const { record, token: token6 } = await createScheduledRunViewLink({
36613
37763
  ownerId,
36614
37764
  runId: result.data.run.runId,
36615
37765
  artifactId: artifact.artifactId,
@@ -36624,7 +37774,7 @@ scheduledResultApp.post("/schedule-runs/:runId/view-links", async (c) => {
36624
37774
  shareId: record.shareId,
36625
37775
  runId: record.runId,
36626
37776
  artifactId: record.artifactId,
36627
- url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${token4}`,
37777
+ url: `${origin.replace(/\/$/, "")}/view/scheduled-run/${token6}`,
36628
37778
  expiresAt: record.expiresAt,
36629
37779
  createdAt: record.createdAt
36630
37780
  }, 201);
@@ -36650,15 +37800,15 @@ import { createHash as createHash21 } from "crypto";
36650
37800
  var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
36651
37801
  var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
36652
37802
  var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
36653
- function token3() {
37803
+ function token4() {
36654
37804
  return process.env.SCHEDULED_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
36655
37805
  }
36656
- function policy3() {
37806
+ function policy4() {
36657
37807
  return {
36658
37808
  prefix: SCHEDULED_RUN_ARTIFACT_PREFIX,
36659
37809
  artifactTtlMs: null,
36660
37810
  downloadTtlMs: SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS,
36661
- token: token3()
37811
+ token: token4()
36662
37812
  };
36663
37813
  }
36664
37814
  function runStorageSegment(runId) {
@@ -36669,7 +37819,7 @@ async function createScheduledRunArtifact(args) {
36669
37819
  throw new Error("Scheduled artifact HTML must be 2,000,000 bytes or fewer.");
36670
37820
  }
36671
37821
  const stored = await createPrivateArtifact({
36672
- policy: policy3(),
37822
+ policy: policy4(),
36673
37823
  ownerId: args.request.ownerId,
36674
37824
  scopeSegments: [runStorageSegment(args.request.runId)],
36675
37825
  artifactKey: `${args.rendered.sha256.slice(0, 16)}-${args.rendered.filename}`,
@@ -36695,7 +37845,7 @@ async function createScheduledRunArtifact(args) {
36695
37845
  async function readScheduledRunArtifact(args) {
36696
37846
  if (!args.artifactId.startsWith(`${SCHEDULED_RUN_ARTIFACT_PREFIX}${args.ownerId}/`)) return null;
36697
37847
  const window2 = await readPrivateArtifactWindow({
36698
- policy: policy3(),
37848
+ policy: policy4(),
36699
37849
  artifactId: args.artifactId,
36700
37850
  offset: 0,
36701
37851
  maxBytes: SCHEDULED_RUN_ARTIFACT_MAX_BYTES + 1
@@ -37258,6 +38408,282 @@ function shouldRunSiteExtractInBackground(input) {
37258
38408
  return input.background === true || input.downloadImages === true || requestedSiteMaxPages(input.maxPages) > MAX_SYNCHRONOUS_SITE_PAGES;
37259
38409
  }
37260
38410
 
38411
+ // src/api/page-media-artifacts.ts
38412
+ import { createHash as createHash23, randomBytes as randomBytes6 } from "crypto";
38413
+ import { homedir as homedir3 } from "os";
38414
+ import { basename, extname, join as join8 } from "path";
38415
+ import pLimit6 from "p-limit";
38416
+ import { ZipFile as ZipFile2 } from "yazl";
38417
+ var PAGE_MEDIA_ARTIFACT_PREFIX = "page-media/";
38418
+ var PAGE_MEDIA_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
38419
+ var PAGE_MEDIA_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
38420
+ var MAX_FILE_BYTES = 10 * 1024 * 1024;
38421
+ var MAX_ARCHIVE_MEDIA_BYTES = 45 * 1024 * 1024;
38422
+ var MAX_INLINE_IMAGE_BYTES2 = 15e5;
38423
+ var MAX_INLINE_TOTAL_BYTES2 = 25e5;
38424
+ var DOWNLOAD_CONCURRENCY2 = 6;
38425
+ var MAX_REDIRECTS2 = 5;
38426
+ function token5() {
38427
+ return process.env.PAGE_MEDIA_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
38428
+ }
38429
+ function policy5() {
38430
+ return {
38431
+ prefix: PAGE_MEDIA_ARTIFACT_PREFIX,
38432
+ artifactTtlMs: PAGE_MEDIA_ARTIFACT_TTL_MS,
38433
+ downloadTtlMs: PAGE_MEDIA_DOWNLOAD_TTL_MS,
38434
+ token: token5()
38435
+ };
38436
+ }
38437
+ function imageSignature(bytes, contentType) {
38438
+ if (bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return { mimeType: "image/jpeg", extension: "jpg" };
38439
+ if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80 && bytes[2] === 78 && bytes[3] === 71) return { mimeType: "image/png", extension: "png" };
38440
+ if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(0, 4)) === "RIFF" && String.fromCharCode(...bytes.slice(8, 12)) === "WEBP") return { mimeType: "image/webp", extension: "webp" };
38441
+ if (bytes.length >= 6 && ["GIF87a", "GIF89a"].includes(String.fromCharCode(...bytes.slice(0, 6)))) return { mimeType: "image/gif", extension: "gif" };
38442
+ if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(4, 12)).match(/^ftyp(?:avif|avis)$/)) return { mimeType: "image/avif", extension: "avif" };
38443
+ const prefix = Buffer.from(bytes.slice(0, Math.min(bytes.length, 1024))).toString("utf8").replace(/^\uFEFF/, "").trimStart();
38444
+ if ((contentType === "image/svg+xml" || prefix.startsWith("<svg") || /^<\?xml[\s\S]*?<svg/i.test(prefix)) && /<svg[\s>]/i.test(prefix)) {
38445
+ return { mimeType: "image/svg+xml", extension: "svg" };
38446
+ }
38447
+ return null;
38448
+ }
38449
+ function mediaSignature(bytes, contentType, expectedType) {
38450
+ if (expectedType === "image") return imageSignature(bytes, contentType);
38451
+ if (bytes.length >= 12 && String.fromCharCode(...bytes.slice(4, 8)) === "ftyp") return { mimeType: "video/mp4", extension: "mp4" };
38452
+ if (bytes.length >= 4 && bytes[0] === 26 && bytes[1] === 69 && bytes[2] === 223 && bytes[3] === 163) return { mimeType: "video/webm", extension: "webm" };
38453
+ if (bytes.length >= 4 && String.fromCharCode(...bytes.slice(0, 4)) === "OggS") {
38454
+ return expectedType === "audio" ? { mimeType: "audio/ogg", extension: "ogg" } : { mimeType: "video/ogg", extension: "ogv" };
38455
+ }
38456
+ if (expectedType === "audio" && bytes.length >= 3 && String.fromCharCode(...bytes.slice(0, 3)) === "ID3") return { mimeType: "audio/mpeg", extension: "mp3" };
38457
+ if (contentType && typeFromMime(contentType) === expectedType) {
38458
+ const extension = contentType.split("/")[1]?.replace("jpeg", "jpg").replace("mpeg", "mp3").replace("svg+xml", "svg");
38459
+ if (extension && /^[a-z0-9]+$/i.test(extension)) return { mimeType: contentType, extension };
38460
+ }
38461
+ return null;
38462
+ }
38463
+ function imageDimensions(bytes, mimeType) {
38464
+ const buffer = Buffer.from(bytes);
38465
+ if (mimeType === "image/png" && buffer.length >= 24) return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) };
38466
+ if (mimeType === "image/gif" && buffer.length >= 10) return { width: buffer.readUInt16LE(6), height: buffer.readUInt16LE(8) };
38467
+ if (mimeType === "image/webp" && buffer.length >= 30) {
38468
+ const chunk = buffer.toString("ascii", 12, 16);
38469
+ if (chunk === "VP8X") return { width: 1 + buffer.readUIntLE(24, 3), height: 1 + buffer.readUIntLE(27, 3) };
38470
+ if (chunk === "VP8L" && buffer[20] === 47) {
38471
+ const bits = buffer.readUInt32LE(21);
38472
+ return { width: (bits & 16383) + 1, height: (bits >> 14 & 16383) + 1 };
38473
+ }
38474
+ }
38475
+ if (mimeType === "image/jpeg") {
38476
+ let offset = 2;
38477
+ while (offset + 9 < buffer.length) {
38478
+ if (buffer[offset] !== 255) {
38479
+ offset += 1;
38480
+ continue;
38481
+ }
38482
+ const marker = buffer[offset + 1];
38483
+ if (marker === 216 || marker === 217) {
38484
+ offset += 2;
38485
+ continue;
38486
+ }
38487
+ const length = buffer.readUInt16BE(offset + 2);
38488
+ if (length < 2 || offset + length + 2 > buffer.length) break;
38489
+ if ([192, 193, 194, 195, 197, 198, 199, 201, 202, 203, 205, 206, 207].includes(marker)) {
38490
+ return { height: buffer.readUInt16BE(offset + 5), width: buffer.readUInt16BE(offset + 7) };
38491
+ }
38492
+ offset += length + 2;
38493
+ }
38494
+ }
38495
+ if (mimeType === "image/svg+xml") {
38496
+ const source = buffer.toString("utf8", 0, Math.min(buffer.length, 64 * 1024));
38497
+ const tag = source.match(/<svg\b[^>]*>/i)?.[0] ?? "";
38498
+ const width = Number(tag.match(/\bwidth=["']?([\d.]+)/i)?.[1] ?? 0);
38499
+ const height = Number(tag.match(/\bheight=["']?([\d.]+)/i)?.[1] ?? 0);
38500
+ if (width > 0 && height > 0) return { width: Math.round(width), height: Math.round(height) };
38501
+ const viewBox = tag.match(/\bviewBox=["']([^"']+)["']/i)?.[1]?.trim().split(/[\s,]+/).map(Number);
38502
+ if (viewBox?.length === 4 && viewBox[2] > 0 && viewBox[3] > 0) return { width: Math.round(viewBox[2]), height: Math.round(viewBox[3]) };
38503
+ }
38504
+ return null;
38505
+ }
38506
+ async function readBounded2(response) {
38507
+ const declared = Number(response.headers.get("content-length") ?? "");
38508
+ if (Number.isFinite(declared) && declared > MAX_FILE_BYTES) throw new Error("media_too_large");
38509
+ if (!response.body) {
38510
+ const bytes = Buffer.from(await response.arrayBuffer());
38511
+ if (bytes.length > MAX_FILE_BYTES) throw new Error("media_too_large");
38512
+ return bytes;
38513
+ }
38514
+ const reader = response.body.getReader();
38515
+ const chunks = [];
38516
+ let total = 0;
38517
+ try {
38518
+ for (; ; ) {
38519
+ const { done, value } = await reader.read();
38520
+ if (done) break;
38521
+ if (!value) continue;
38522
+ total += value.byteLength;
38523
+ if (total > MAX_FILE_BYTES) throw new Error("media_too_large");
38524
+ chunks.push(Buffer.from(value));
38525
+ }
38526
+ } finally {
38527
+ reader.releaseLock();
38528
+ }
38529
+ return Buffer.concat(chunks);
38530
+ }
38531
+ async function fetchMedia(rawUrl, expectedType) {
38532
+ let target = rawUrl;
38533
+ for (let redirects = 0; redirects <= MAX_REDIRECTS2; redirects += 1) {
38534
+ const checked = await validatePublicHttpUrl(target, { field: "website media URL" });
38535
+ if (checked.error || !checked.parsed) throw new Error("media_url_rejected");
38536
+ const response = await fetch(checked.parsed.href, {
38537
+ redirect: "manual",
38538
+ headers: { Accept: expectedType === "image" ? "image/*" : `${expectedType}/*` },
38539
+ signal: AbortSignal.timeout(15e3)
38540
+ });
38541
+ if ([301, 302, 303, 307, 308].includes(response.status)) {
38542
+ const location2 = response.headers.get("location");
38543
+ if (!location2 || redirects === MAX_REDIRECTS2) throw new Error("media_redirect_rejected");
38544
+ target = new URL(location2, checked.parsed).href;
38545
+ continue;
38546
+ }
38547
+ if (!response.ok) throw new Error(`media_fetch_${response.status}`);
38548
+ const bytes = await readBounded2(response);
38549
+ const declared = response.headers.get("content-type")?.split(";")[0].trim().toLowerCase() ?? null;
38550
+ const detected = mediaSignature(bytes, declared, expectedType);
38551
+ if (!detected || typeFromMime(detected.mimeType) !== expectedType) throw new Error("media_content_invalid");
38552
+ const dimensions = expectedType === "image" ? imageDimensions(bytes, detected.mimeType) : null;
38553
+ return {
38554
+ bytes,
38555
+ ...detected,
38556
+ finalUrl: checked.parsed.href,
38557
+ width: dimensions?.width ?? null,
38558
+ height: dimensions?.height ?? null,
38559
+ sha256: createHash23("sha256").update(bytes).digest("hex")
38560
+ };
38561
+ }
38562
+ throw new Error("media_redirect_rejected");
38563
+ }
38564
+ function zipBuffer2(files) {
38565
+ return new Promise((resolve2, reject) => {
38566
+ const zip = new ZipFile2();
38567
+ const chunks = [];
38568
+ zip.outputStream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
38569
+ zip.outputStream.once("error", reject);
38570
+ zip.outputStream.once("end", () => resolve2(Buffer.concat(chunks)));
38571
+ for (const file of files) zip.addBuffer(file.content, file.path);
38572
+ zip.end();
38573
+ });
38574
+ }
38575
+ function slug3(value) {
38576
+ return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 70) || "website";
38577
+ }
38578
+ function safeOriginalName(rawUrl) {
38579
+ try {
38580
+ const name = basename(new URL(rawUrl).pathname).replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
38581
+ return name.slice(0, 80);
38582
+ } catch {
38583
+ return "";
38584
+ }
38585
+ }
38586
+ function pageMediaArtifactOwnerId(artifactId) {
38587
+ return privateArtifactOwnerId(artifactId, PAGE_MEDIA_ARTIFACT_PREFIX);
38588
+ }
38589
+ async function readOwnedPageMediaArtifactBuffer(args) {
38590
+ if (pageMediaArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
38591
+ return readPrivateArtifactBuffer({ policy: policy5(), artifactId: args.artifactId, maxBytes: 60 * 1024 * 1024 });
38592
+ }
38593
+ async function packagePageMedia(args) {
38594
+ const files = [];
38595
+ const limit = pLimit6(DOWNLOAD_CONCURRENCY2);
38596
+ const digestToFilename = /* @__PURE__ */ new Map();
38597
+ let storedBytes = 0;
38598
+ let inlineBytes = 0;
38599
+ let inlineCount = 0;
38600
+ await Promise.all(args.media.assets.map((asset, position) => limit(async () => {
38601
+ try {
38602
+ const fetched = await fetchMedia(asset.url, asset.type);
38603
+ asset.finalUrl = fetched.finalUrl;
38604
+ asset.mimeType = fetched.mimeType;
38605
+ asset.sizeBytes = fetched.bytes.length;
38606
+ asset.sha256 = fetched.sha256;
38607
+ asset.width = fetched.width ?? asset.width;
38608
+ asset.height = fetched.height ?? asset.height;
38609
+ const duplicate = digestToFilename.get(fetched.sha256);
38610
+ if (duplicate) {
38611
+ asset.filename = duplicate;
38612
+ asset.duplicateOf = duplicate;
38613
+ asset.downloadStatus = "downloaded";
38614
+ asset.downloadError = null;
38615
+ return;
38616
+ }
38617
+ if (storedBytes + fetched.bytes.length > MAX_ARCHIVE_MEDIA_BYTES) throw new Error("archive_byte_limit_reached");
38618
+ storedBytes += fetched.bytes.length;
38619
+ const original = safeOriginalName(fetched.finalUrl);
38620
+ const extension = extname(original);
38621
+ const stem = original ? extension ? original.slice(0, -extension.length) : original : asset.type;
38622
+ const filename = `media/${String(position + 1).padStart(4, "0")}-${slug3(stem)}.${fetched.extension}`;
38623
+ digestToFilename.set(fetched.sha256, filename);
38624
+ files.push({ path: filename, content: fetched.bytes });
38625
+ asset.filename = filename;
38626
+ asset.duplicateOf = null;
38627
+ asset.downloadStatus = "downloaded";
38628
+ asset.downloadError = null;
38629
+ if (asset.type === "image" && inlineCount < args.maxInlineImages && fetched.bytes.length <= MAX_INLINE_IMAGE_BYTES2 && inlineBytes + fetched.bytes.length <= MAX_INLINE_TOTAL_BYTES2) {
38630
+ asset.inlinePreview = { data: fetched.bytes.toString("base64"), mimeType: fetched.mimeType };
38631
+ inlineBytes += fetched.bytes.length;
38632
+ inlineCount += 1;
38633
+ }
38634
+ } catch (error) {
38635
+ asset.finalUrl = null;
38636
+ asset.mimeType = null;
38637
+ asset.sizeBytes = null;
38638
+ asset.sha256 = null;
38639
+ asset.savedPath = null;
38640
+ asset.downloadStatus = "failed";
38641
+ asset.downloadError = error instanceof Error ? error.message : "media_download_failed";
38642
+ asset.inlinePreview = null;
38643
+ }
38644
+ })));
38645
+ const downloaded = args.media.assets.filter((asset) => asset.downloadStatus === "downloaded").length;
38646
+ const failed = args.media.assets.length - downloaded;
38647
+ if (failed > 0) args.media.warnings.push(`${failed} media asset(s) could not be downloaded; media.jsonl contains bounded error codes.`);
38648
+ if (downloaded === 0) return { media: args.media, artifact: null };
38649
+ const cleanAssets = args.media.assets.map(({ inlinePreview: _preview, ...asset }) => asset);
38650
+ const summary = {
38651
+ pageUrl: args.pageUrl,
38652
+ title: args.title,
38653
+ extractedAt: args.extractedAt,
38654
+ completeness: args.media.completeness,
38655
+ exhausted: args.media.exhausted,
38656
+ stopReason: args.media.stopReason,
38657
+ scrollRounds: args.media.scrollRounds,
38658
+ staticFound: args.media.staticFound,
38659
+ renderedFound: args.media.renderedFound,
38660
+ totalFound: args.media.totalFound,
38661
+ retainedAfterFilteringAndVariantCollapse: args.media.assets.length,
38662
+ downloaded,
38663
+ failed,
38664
+ notes: [
38665
+ "The inventory unions static-source and rendered-page discovery.",
38666
+ "complete means rendered scrolling reached three stable document-height checks; it does not claim every interactive control was opened.",
38667
+ "Media records preserve discovery evidence but do not assert ownership, rights, identity, or publishability."
38668
+ ]
38669
+ };
38670
+ files.push({ path: "summary.json", content: Buffer.from(JSON.stringify(summary, null, 2)) });
38671
+ files.push({ path: "media.jsonl", content: Buffer.from(cleanAssets.map((asset) => JSON.stringify(asset)).join("\n") + "\n") });
38672
+ const archive = await zipBuffer2(files);
38673
+ const id = randomBytes6(6).toString("hex");
38674
+ const pointer = await createPrivateArtifact({
38675
+ policy: policy5(),
38676
+ ownerId: args.ownerId,
38677
+ artifactKey: `${id}.zip`,
38678
+ createdAt: new Date(args.extractedAt),
38679
+ filename: `${slug3(args.title || new URL(args.pageUrl).hostname)}-website-media.zip`,
38680
+ contentType: "application/zip",
38681
+ content: archive
38682
+ });
38683
+ const localPath = token5() ? null : join8(process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join8(homedir3(), "Downloads", "mcp-scraper"), "blobs", pointer.artifactId);
38684
+ return { media: args.media, artifact: { ...pointer, localPath } };
38685
+ }
38686
+
37261
38687
  // src/api/site-extract-reconciliation.ts
37262
38688
  async function redispatchFundedSiteExtractJobs(limit = 25) {
37263
38689
  const jobs = await listFundedPendingExtractJobs(limit);
@@ -37323,13 +38749,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
37323
38749
  }
37324
38750
 
37325
38751
  // src/api/page-diff.ts
37326
- import { createHash as createHash23 } from "crypto";
38752
+ import { createHash as createHash24 } from "crypto";
37327
38753
  import { diffLines } from "diff";
37328
38754
  var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
37329
38755
  var MAX_DIFF_HUNKS = 200;
37330
38756
  var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
37331
38757
  function sha256Hex(value) {
37332
- return createHash23("sha256").update(value).digest("hex");
38758
+ return createHash24("sha256").update(value).digest("hex");
37333
38759
  }
37334
38760
  function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
37335
38761
  if (value.length <= maxChars) return { value, truncated: false };
@@ -37422,8 +38848,8 @@ async function uploadFallbackBlob(opts) {
37422
38848
  try {
37423
38849
  const content = (opts.content ?? "").trim();
37424
38850
  if (!content) return null;
37425
- const slug2 = (slugify(opts.title || opts.source) || "scrape").slice(0, 60);
37426
- const key = `${SCRAPE_FALLBACK_PREFIX}${slug2}.md`;
38851
+ const slug4 = (slugify(opts.title || opts.source) || "scrape").slice(0, 60);
38852
+ const key = `${SCRAPE_FALLBACK_PREFIX}${slug4}.md`;
37427
38853
  const doc = `# ${opts.title}
37428
38854
  Source: ${opts.source}
37429
38855
 
@@ -37443,10 +38869,10 @@ async function persistScrapeBody(user, opts) {
37443
38869
  }
37444
38870
 
37445
38871
  // src/api/scrape-image-sink.ts
37446
- import { createHash as createHash24 } from "crypto";
38872
+ import { createHash as createHash25 } from "crypto";
37447
38873
  var MAX_IMAGES_PER_SCRAPE = 25;
37448
38874
  function idempotencyKey2(userId, vault, input) {
37449
- return `scrape-image-${createHash24("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
38875
+ return `scrape-image-${createHash25("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
37450
38876
  }
37451
38877
  async function persistScrapeImagesToMemory(user, inputs, vault) {
37452
38878
  const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
@@ -37490,7 +38916,7 @@ async function persistScrapeImagesToMemory(user, inputs, vault) {
37490
38916
  }
37491
38917
 
37492
38918
  // src/api/archive-reader.ts
37493
- import { basename, extname } from "path";
38919
+ import { basename as basename2, extname as extname2 } from "path";
37494
38920
  import yauzl from "yauzl";
37495
38921
  var MAX_ARCHIVE_DOWNLOAD_BYTES = 50 * 1024 * 1024;
37496
38922
  var MAX_ARCHIVE_ENTRIES = 1e4;
@@ -37545,9 +38971,9 @@ var ArchiveReadError = class extends Error {
37545
38971
  retryable;
37546
38972
  };
37547
38973
  function readableContentType(path5) {
37548
- const extensionType = READABLE_EXTENSIONS.get(extname(path5).toLowerCase());
38974
+ const extensionType = READABLE_EXTENSIONS.get(extname2(path5).toLowerCase());
37549
38975
  if (extensionType) return extensionType;
37550
- return READABLE_FILENAMES.has(basename(path5).toLowerCase()) ? "text/plain" : null;
38976
+ return READABLE_FILENAMES.has(basename2(path5).toLowerCase()) ? "text/plain" : null;
37551
38977
  }
37552
38978
  function safeEntryPath(path5) {
37553
38979
  const normalized = path5.replaceAll("\\", "/");
@@ -37908,7 +39334,7 @@ function archiveLibrarySource(archiveUrl, path5) {
37908
39334
  return source.href;
37909
39335
  }
37910
39336
  function archiveEntryTitle(path5) {
37911
- return basename(path5).slice(0, 200) || "Archive entry";
39337
+ return basename2(path5).slice(0, 200) || "Archive entry";
37912
39338
  }
37913
39339
  function inferredWaybackCapturedAt(content) {
37914
39340
  const timestamp = content.slice(0, 4e3).match(/^Archive timestamp:\s*(\d{14})\s*$/m)?.[1];
@@ -37918,7 +39344,7 @@ function inferredWaybackCapturedAt(content) {
37918
39344
  }
37919
39345
 
37920
39346
  // src/api/connection-memory-import.ts
37921
- import { createHash as createHash25 } from "crypto";
39347
+ import { createHash as createHash26 } from "crypto";
37922
39348
  var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
37923
39349
  var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
37924
39350
  var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
@@ -37948,7 +39374,7 @@ function canonicalJson(value) {
37948
39374
  return JSON.stringify(value);
37949
39375
  }
37950
39376
  function sha2562(value) {
37951
- return createHash25("sha256").update(value).digest("hex");
39377
+ return createHash26("sha256").update(value).digest("hex");
37952
39378
  }
37953
39379
  function sensitiveKey(key) {
37954
39380
  const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
@@ -38166,17 +39592,17 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
38166
39592
 
38167
39593
  // src/api/scrape-blob-cleanup.ts
38168
39594
  import { readdir, stat, unlink } from "fs/promises";
38169
- import { homedir as homedir2 } from "os";
38170
- import { join as join7 } from "path";
38171
- async function cleanupVercel(token4, cutoff) {
39595
+ import { homedir as homedir4 } from "os";
39596
+ import { join as join9 } from "path";
39597
+ async function cleanupVercel(token6, cutoff) {
38172
39598
  const { list, del } = await import("@vercel/blob");
38173
39599
  let cursor;
38174
39600
  let deleted = 0;
38175
39601
  do {
38176
- const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token: token4 });
39602
+ const res = await list({ prefix: SCRAPE_FALLBACK_PREFIX, cursor, token: token6 });
38177
39603
  const expired = res.blobs.filter((b) => new Date(b.uploadedAt).getTime() < cutoff);
38178
39604
  if (expired.length) {
38179
- await del(expired.map((b) => b.url), { token: token4 });
39605
+ await del(expired.map((b) => b.url), { token: token6 });
38180
39606
  deleted += expired.length;
38181
39607
  }
38182
39608
  cursor = res.hasMore ? res.cursor : void 0;
@@ -38184,8 +39610,8 @@ async function cleanupVercel(token4, cutoff) {
38184
39610
  return { deleted, store: "vercel-blob" };
38185
39611
  }
38186
39612
  async function cleanupLocal(cutoff) {
38187
- const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join7(homedir2(), "Downloads", "mcp-scraper");
38188
- const dir = join7(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
39613
+ const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join9(homedir4(), "Downloads", "mcp-scraper");
39614
+ const dir = join9(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
38189
39615
  let deleted = 0;
38190
39616
  let entries;
38191
39617
  try {
@@ -38194,7 +39620,7 @@ async function cleanupLocal(cutoff) {
38194
39620
  return { deleted: 0, store: "local" };
38195
39621
  }
38196
39622
  for (const name of entries) {
38197
- const path5 = join7(dir, name);
39623
+ const path5 = join9(dir, name);
38198
39624
  try {
38199
39625
  const s = await stat(path5);
38200
39626
  if (s.isFile() && s.mtimeMs < cutoff) {
@@ -38209,8 +39635,8 @@ async function cleanupLocal(cutoff) {
38209
39635
  async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
38210
39636
  const cutoff = Date.now() - maxAgeMs;
38211
39637
  try {
38212
- const token4 = process.env.BLOB_READ_WRITE_TOKEN;
38213
- return token4 ? await cleanupVercel(token4, cutoff) : await cleanupLocal(cutoff);
39638
+ const token6 = process.env.BLOB_READ_WRITE_TOKEN;
39639
+ return token6 ? await cleanupVercel(token6, cutoff) : await cleanupLocal(cutoff);
38214
39640
  } catch {
38215
39641
  return { deleted: 0, store: "none" };
38216
39642
  }
@@ -38959,7 +40385,7 @@ async function dbUsage(identity, plan) {
38959
40385
  }
38960
40386
 
38961
40387
  // src/api/nango-control.ts
38962
- import { createHash as createHash27, randomUUID as randomUUID22 } from "crypto";
40388
+ import { createHash as createHash28, randomUUID as randomUUID22 } from "crypto";
38963
40389
 
38964
40390
  // src/api/slack-archive-analysis.ts
38965
40391
  function isRecord2(value) {
@@ -39333,7 +40759,7 @@ async function exportSlackChannelPage(input, dependencies) {
39333
40759
  }
39334
40760
 
39335
40761
  // src/api/main-nango-transport.ts
39336
- import { createHash as createHash26, randomUUID as randomUUID21 } from "crypto";
40762
+ import { createHash as createHash27, randomUUID as randomUUID21 } from "crypto";
39337
40763
  import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
39338
40764
 
39339
40765
  // src/api/connected-tool-policy.ts
@@ -39376,9 +40802,9 @@ function buildConnectedToolPolicyRegistry(args) {
39376
40802
  return policies;
39377
40803
  }
39378
40804
  function connectedToolPolicyOrThrow(registry, toolName) {
39379
- const policy4 = registry.get(toolName);
39380
- if (!policy4) throw new Error("connected_tool_policy_missing");
39381
- return policy4;
40805
+ const policy6 = registry.get(toolName);
40806
+ if (!policy6) throw new Error("connected_tool_policy_missing");
40807
+ return policy6;
39382
40808
  }
39383
40809
 
39384
40810
  // src/api/main-nango-transport.ts
@@ -39724,9 +41150,9 @@ async function listNangoToolsDirect(identity, connectionId) {
39724
41150
  connectionActive: connection.lifecycleStatus === "connected",
39725
41151
  actionsEnabled: connection.actionsEnabled
39726
41152
  });
39727
- const readTools = [...policies.values()].filter((policy4) => policy4.classification === "read").map((policy4) => policy4.name);
39728
- const actionTools = [...policies.values()].filter((policy4) => policy4.classification === "action").map((policy4) => policy4.name);
39729
- const revision = createHash26("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
41153
+ const readTools = [...policies.values()].filter((policy6) => policy6.classification === "read").map((policy6) => policy6.name);
41154
+ const actionTools = [...policies.values()].filter((policy6) => policy6.classification === "action").map((policy6) => policy6.name);
41155
+ const revision = createHash27("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
39730
41156
  await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
39731
41157
  const refreshed = await getOwnedServiceConnection(identity, connection.id);
39732
41158
  return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
@@ -39735,18 +41161,18 @@ async function callNangoToolDirect(args) {
39735
41161
  const { connection, tools } = await listNangoToolsDirect(args.identity, args.connectionId);
39736
41162
  const selected = tools.find((tool) => tool.name === args.tool);
39737
41163
  if (!selected) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
39738
- const policy4 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
41164
+ const policy6 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
39739
41165
  tools,
39740
41166
  connectionActive: connection.lifecycleStatus === "connected",
39741
41167
  actionsEnabled: connection.actionsEnabled
39742
41168
  }), selected.name);
39743
- if (policy4.classification !== args.classification) {
41169
+ if (policy6.classification !== args.classification) {
39744
41170
  throw new MainNangoTransportError("The provider tool classification does not match this operation.", 403, "tool_not_allowed", false);
39745
41171
  }
39746
- if (!policy4.available && policy4.blockedReason === "actions_disabled") {
41172
+ if (!policy6.available && policy6.blockedReason === "actions_disabled") {
39747
41173
  throw new MainNangoTransportError("Actions are disabled for this service connection.", 403, "actions_disabled", false);
39748
41174
  }
39749
- if (!policy4.available) {
41175
+ if (!policy6.available) {
39750
41176
  throw new MainNangoTransportError("The provider tool is not authorized for this connection.", 403, "tool_not_allowed", false);
39751
41177
  }
39752
41178
  await preflightConnectedUsage({
@@ -39754,7 +41180,7 @@ async function callNangoToolDirect(args) {
39754
41180
  ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
39755
41181
  });
39756
41182
  const requestId = args.requestId?.trim() || randomUUID21();
39757
- const idempotencyKey3 = `main-nango:${createHash26("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
41183
+ const idempotencyKey3 = `main-nango:${createHash27("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
39758
41184
  const startedAt = /* @__PURE__ */ new Date();
39759
41185
  const started = performance.now();
39760
41186
  let result;
@@ -39779,7 +41205,7 @@ async function callNangoToolDirect(args) {
39779
41205
  toolName: args.tool,
39780
41206
  operationKind: args.operationKind ?? args.classification,
39781
41207
  outcome: providerError ? "error" : "partial",
39782
- requestId: requestId.length <= 200 ? requestId : createHash26("sha256").update(requestId).digest("hex"),
41208
+ requestId: requestId.length <= 200 ? requestId : createHash27("sha256").update(requestId).digest("hex"),
39783
41209
  startedAt: startedAt.toISOString(),
39784
41210
  completedAt: completedAt.toISOString()
39785
41211
  }
@@ -39791,13 +41217,13 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
39791
41217
  const { connection, tools } = await listNangoToolsDirect(identity, connectionId);
39792
41218
  const tool = tools.find((candidate) => candidate.name === toolName);
39793
41219
  if (!tool) throw new MainNangoTransportError("The requested provider tool is unavailable.", 404, "live_tool_missing", false);
39794
- const policy4 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
41220
+ const policy6 = connectedToolPolicyOrThrow(buildConnectedToolPolicyRegistry({
39795
41221
  tools,
39796
41222
  connectionActive: connection.lifecycleStatus === "connected",
39797
41223
  actionsEnabled: connection.actionsEnabled
39798
41224
  }), tool.name);
39799
41225
  const classification = classifyConnectedTool(tool);
39800
- const callable = policy4.available;
41226
+ const callable = policy6.available;
39801
41227
  const projected = {
39802
41228
  name: tool.name,
39803
41229
  ...tool.title ? { title: tool.title } : {},
@@ -39810,18 +41236,18 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
39810
41236
  ...projected,
39811
41237
  classification,
39812
41238
  callable,
39813
- blockedReason: policy4.blockedReason,
39814
- requiredPermissions: policy4.requiredPermissions,
39815
- missingPermissions: policy4.missingPermissions,
39816
- requiredFeatures: policy4.requiredFeatures,
39817
- missingFeatures: policy4.missingFeatures,
41239
+ blockedReason: policy6.blockedReason,
41240
+ requiredPermissions: policy6.requiredPermissions,
41241
+ missingPermissions: policy6.missingPermissions,
41242
+ requiredFeatures: policy6.requiredFeatures,
41243
+ missingFeatures: policy6.missingFeatures,
39818
41244
  transport: "nango",
39819
41245
  providerConfigKey: connection.providerConfigKey,
39820
41246
  providerContractVersion: MAIN_INTEGRATION_CONTRACT_VERSION,
39821
41247
  providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
39822
41248
  protocolVersion: null,
39823
41249
  schemaSource: "live_tools_list",
39824
- schemaHash: createHash26("sha256").update(JSON.stringify(projected)).digest("hex"),
41250
+ schemaHash: createHash27("sha256").update(JSON.stringify(projected)).digest("hex"),
39825
41251
  fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
39826
41252
  };
39827
41253
  }
@@ -40545,8 +41971,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
40545
41971
  return data.connection.actionsEnabled === true;
40546
41972
  }
40547
41973
  async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
40548
- const requestId = `main-connected-action:${createHash27("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID22()).digest("hex")}`;
40549
- const requestDigest = createHash27("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
41974
+ const requestId = `main-connected-action:${createHash28("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID22()).digest("hex")}`;
41975
+ const requestDigest = createHash28("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
40550
41976
  if (mainOwnsIntegrations()) {
40551
41977
  const selectedTool = tool?.trim();
40552
41978
  if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
@@ -40697,7 +42123,7 @@ function canonicalJson2(value) {
40697
42123
  return JSON.stringify(value);
40698
42124
  }
40699
42125
  function projectedToolSchemaHash(tool) {
40700
- return createHash27("sha256").update(canonicalJson2(tool)).digest("hex");
42126
+ return createHash28("sha256").update(canonicalJson2(tool)).digest("hex");
40701
42127
  }
40702
42128
  async function describeNangoTool(identity, connectionId, tool, fresh) {
40703
42129
  if (mainOwnsIntegrations()) {
@@ -40976,7 +42402,7 @@ async function callMainOwnedExportPage(identity, input) {
40976
42402
  }
40977
42403
 
40978
42404
  // src/api/resend-control.ts
40979
- import { createHash as createHash28 } from "crypto";
42405
+ import { createHash as createHash29 } from "crypto";
40980
42406
  var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
40981
42407
  var RESEND_PROVIDER_CONFIG_KEY = "resend";
40982
42408
  var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
@@ -41235,7 +42661,7 @@ async function callResendRead(identity, connectionId, tool, args) {
41235
42661
  return isRecord5(data) ? data.result ?? data : data;
41236
42662
  }
41237
42663
  async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
41238
- const requestId = `main-resend-action:${createHash28("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
42664
+ const requestId = `main-resend-action:${createHash29("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
41239
42665
  const body = await controlRequest2("/api/internal/resend/actions/call", {
41240
42666
  method: "POST",
41241
42667
  headers: { "x-request-id": requestId },
@@ -41362,9 +42788,9 @@ var auth4 = createMiddleware5(async (c, next) => {
41362
42788
  return next();
41363
42789
  });
41364
42790
  var sessionAuth = createMiddleware5(async (c, next) => {
41365
- const token4 = getCookie2(c, "session");
41366
- if (!token4) return c.json({ error: "Not authenticated" }, 401);
41367
- const userId = verifySession(token4);
42791
+ const token6 = getCookie2(c, "session");
42792
+ if (!token6) return c.json({ error: "Not authenticated" }, 401);
42793
+ const userId = verifySession(token6);
41368
42794
  if (!userId) return c.json({ error: "Session expired" }, 401);
41369
42795
  const user = await getUserById(userId);
41370
42796
  if (!user) return c.json({ error: "User not found" }, 401);
@@ -41509,8 +42935,8 @@ app.post("/auth/register", requireAllowedOrigin, async (c) => {
41509
42935
  console.warn("[auth/register] memory provision failed (will retry on first write):", err instanceof Error ? err.message : String(err));
41510
42936
  }
41511
42937
  })();
41512
- const token4 = signSession(user.id);
41513
- setCookie2(c, "session", token4, sessionCookieOptions2);
42938
+ const token6 = signSession(user.id);
42939
+ setCookie2(c, "session", token6, sessionCookieOptions2);
41514
42940
  return c.json({ email: user.email, api_key: newKey }, 201);
41515
42941
  } catch (err) {
41516
42942
  const msg = err instanceof Error ? err.message : "";
@@ -41528,8 +42954,8 @@ app.post("/auth/login", requireAllowedOrigin, async (c) => {
41528
42954
  if (!user || !user.password_hash || !verifyPassword(password, user.password_hash)) {
41529
42955
  return c.json({ error: "Invalid email or password" }, 401);
41530
42956
  }
41531
- const token4 = signSession(user.id);
41532
- setCookie2(c, "session", token4, sessionCookieOptions2);
42957
+ const token6 = signSession(user.id);
42958
+ setCookie2(c, "session", token6, sessionCookieOptions2);
41533
42959
  return c.json({ email: user.email, api_key: user.key_active ? user.api_key : null });
41534
42960
  });
41535
42961
  app.post("/auth/logout", requireAllowedOrigin, (c) => {
@@ -41584,9 +43010,9 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
41584
43010
  if (emailLimited) return emailLimited;
41585
43011
  const user = await getUserByEmail(normalizedEmail);
41586
43012
  if (!user) return c.json({ ok: true });
41587
- const token4 = await createPasswordResetToken(user.id);
43013
+ const token6 = await createPasswordResetToken(user.id);
41588
43014
  const appUrl = process.env.APP_URL ?? "https://mcpscraper.dev";
41589
- const resetUrl = `${appUrl}/reset-password?token=${token4}`;
43015
+ const resetUrl = `${appUrl}/reset-password?token=${token6}`;
41590
43016
  if (process.env.RESEND_API_KEY) {
41591
43017
  try {
41592
43018
  const resend = new Resend3(process.env.RESEND_API_KEY);
@@ -41606,12 +43032,12 @@ app.post("/auth/forgot-password", requireAllowedOrigin, async (c) => {
41606
43032
  return c.json({ ok: true });
41607
43033
  });
41608
43034
  app.post("/auth/reset-password", requireAllowedOrigin, async (c) => {
41609
- const { token: token4, password } = await c.req.json();
41610
- if (!token4 || !password) return c.json({ error: "Token and password required" }, 400);
43035
+ const { token: token6, password } = await c.req.json();
43036
+ if (!token6 || !password) return c.json({ error: "Token and password required" }, 400);
41611
43037
  if (password.length < 8) return c.json({ error: "Password must be at least 8 characters" }, 400);
41612
43038
  const limited = await enforceRateLimit(c, "auth_reset", rateLimitKey(c), 10, 60 * 60);
41613
43039
  if (limited) return limited;
41614
- const userId = await consumePasswordResetToken(token4);
43040
+ const userId = await consumePasswordResetToken(token6);
41615
43041
  if (!userId) return c.json({ error: "Invalid or expired reset link" }, 400);
41616
43042
  await setPassword(userId, password);
41617
43043
  return c.json({ ok: true });
@@ -41648,9 +43074,9 @@ function sessionBootstrapPayload(user) {
41648
43074
  }
41649
43075
  app.get("/me", async (c) => {
41650
43076
  c.header("Cache-Control", "no-store");
41651
- const token4 = getCookie2(c, "session");
41652
- if (!token4) return c.json({ authenticated: false });
41653
- const userId = verifySession(token4);
43077
+ const token6 = getCookie2(c, "session");
43078
+ if (!token6) return c.json({ authenticated: false });
43079
+ const userId = verifySession(token6);
41654
43080
  if (!userId) {
41655
43081
  deleteCookie(c, "session", { path: "/", secure: secureCookies2, sameSite: "Strict" });
41656
43082
  return c.json({ authenticated: false });
@@ -43276,6 +44702,8 @@ app.post("/extract-url", auth4, async (c) => {
43276
44702
  }
43277
44703
  const effectiveDelivery = bodyResult.data.depositToVault ? "memory" : requestedDelivery;
43278
44704
  const preserveMedia = bodyResult.data.preserveMedia ?? bodyResult.data.downloadMedia ?? false;
44705
+ const maxMediaAssets = bodyResult.data.maxMediaAssets ?? 100;
44706
+ const maxInlineImages = bodyResult.data.maxInlineImages ?? 3;
43279
44707
  if (!allowLocal) {
43280
44708
  const checked = await validatePublicHttpUrl(url, { field: "URL" });
43281
44709
  if (checked.error || !checked.parsed) return c.json({ error: checked.error ?? "Invalid URL" }, 400);
@@ -43310,7 +44738,14 @@ app.post("/extract-url", auth4, async (c) => {
43310
44738
  const device = screenshotDevice === "mobile" ? "mobile" : "desktop";
43311
44739
  const [result, pageData] = await Promise.all([
43312
44740
  extractKpo({ url: extractionUrl, kernelApiKey }),
43313
- screenshot2 || extractBranding2 ? capturePageData(canonicalUrl, { kernelApiKey, device, screenshot: !!screenshot2, branding: !!extractBranding2 }).catch((err) => {
44741
+ screenshot2 || extractBranding2 || preserveMedia ? capturePageData(canonicalUrl, {
44742
+ kernelApiKey,
44743
+ device,
44744
+ screenshot: !!screenshot2,
44745
+ branding: !!extractBranding2,
44746
+ media: preserveMedia,
44747
+ maxMediaAssets: Math.min(1e3, Math.max(250, maxMediaAssets * 4))
44748
+ }).catch((err) => {
43314
44749
  console.error("[extract-url] capturePageData failed:", err instanceof Error ? err.message : err);
43315
44750
  return null;
43316
44751
  }) : null
@@ -43321,7 +44756,28 @@ app.post("/extract-url", auth4, async (c) => {
43321
44756
  if (screenshotBuf) {
43322
44757
  screenshotMeta = { base64: screenshotBuf.toString("base64"), sizeBytes: screenshotBuf.length, device };
43323
44758
  }
43324
- const mediaMeta = preserveMedia ? await harvestPageMedia(result.bodyHtml, waybackReplay?.originalUrl ?? canonicalUrl, { types: mediaTypes ?? ["image", "video", "audio"], outputDir: null }) : null;
44759
+ let mediaMeta = preserveMedia ? await harvestPageMedia(result.bodyHtml, waybackReplay?.originalUrl ?? canonicalUrl, {
44760
+ types: mediaTypes ?? ["image", "video", "audio"],
44761
+ outputDir: null,
44762
+ rendered: pageData?.renderedMedia ?? null,
44763
+ maxAssets: maxMediaAssets
44764
+ }) : null;
44765
+ if (mediaMeta) {
44766
+ try {
44767
+ const packaged = await packagePageMedia({
44768
+ ownerId: String(user.id),
44769
+ pageUrl: canonicalUrl,
44770
+ title: result.title ?? null,
44771
+ extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
44772
+ media: mediaMeta,
44773
+ maxInlineImages
44774
+ });
44775
+ mediaMeta = packaged.media;
44776
+ mediaMeta.artifact = packaged.artifact;
44777
+ } catch (error) {
44778
+ mediaMeta.warnings.push(`Media packaging failed: ${error instanceof Error ? error.message : "media_packaging_failed"}`);
44779
+ }
44780
+ }
43325
44781
  const featuredImage = includeFeaturedImage ? resolveFeaturedImage({
43326
44782
  html: result.bodyHtml,
43327
44783
  meta: result.meta,
@@ -43504,7 +44960,7 @@ app.post("/archive/read", auth4, async (c) => {
43504
44960
  try {
43505
44961
  let ownedArchive = null;
43506
44962
  if (artifactId) {
43507
- const buffer = await readOwnedSiteExtractArtifactBuffer({ artifactId, ownerId: String(user.id) });
44963
+ const buffer = artifactId.startsWith(MAPS_MEDIA_ARTIFACT_PREFIX) ? await readOwnedMapsMediaArtifactBuffer({ artifactId, ownerId: String(user.id) }) : artifactId.startsWith(PAGE_MEDIA_ARTIFACT_PREFIX) ? await readOwnedPageMediaArtifactBuffer({ artifactId, ownerId: String(user.id) }) : await readOwnedSiteExtractArtifactBuffer({ artifactId, ownerId: String(user.id) });
43508
44964
  if (!buffer) throw new ArchiveReadError("archive_not_found", "Archive artifact was not found or has expired.", 404);
43509
44965
  ownedArchive = { buffer, ref: artifactId };
43510
44966
  }
@@ -44314,7 +45770,7 @@ app.get("/cron/tick", async (c) => {
44314
45770
  return c.json({ error: "Unauthorized" }, 401);
44315
45771
  }
44316
45772
  const startedAt = Date.now();
44317
- const { drainQueue } = await import("./worker-KQN673JF.js");
45773
+ const { drainQueue } = await import("./worker-JL4TG6IF.js");
44318
45774
  const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
44319
45775
  const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
44320
45776
  const unfinished = [];
@@ -44385,7 +45841,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
44385
45841
  terminalExtractJobStatus: terminalExtractJobStatus2,
44386
45842
  extractJobLimitInfo: extractJobLimitInfo2
44387
45843
  } = await import("./site-extract-repository-3VQAALNR.js");
44388
- const { assembleExtractArtifacts } = await import("./extract-bundle-RCTNANCH.js");
45844
+ const { assembleExtractArtifacts } = await import("./extract-bundle-GUEUCTAE.js");
44389
45845
  const existing = await getExtractJob2(jobId2);
44390
45846
  if (!existing) return c.json({ error: "job not found" }, 404);
44391
45847
  if (existing.status !== "failed") {
@@ -44597,4 +46053,4 @@ app.get("/blog/:slug/", (c) => {
44597
46053
  export {
44598
46054
  app
44599
46055
  };
44600
- //# sourceMappingURL=server-KAEHVDY7.js.map
46056
+ //# sourceMappingURL=server-7EXEDKAU.js.map