@aglyn/plugins-ai 1.0.0-beta.289 → 1.0.0-beta.290

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aglyn/plugins-ai",
-  "version": "1.0.0-beta.289",
+  "version": "1.0.0-beta.290",
   "license": "Apache-2.0",
   "homepage": "https://aglyn.com",
   "repository": {
@@ -25,22 +25,22 @@
     "./package.json": "./package.json"
   },
   "dependencies": {
-    "@aglyn/aglyn": "1.0.0-beta.289",
-    "@aglyn/shared-data-enums": "1.0.0-beta.289",
-    "@aglyn/shared-data-mdi": "1.0.0-beta.289",
-    "@aglyn/shared-data-types": "1.0.0-beta.289",
-    "@aglyn/shared-ui-email-campaigns": "1.0.0-beta.289",
-    "@aglyn/shared-ui-jsx": "1.0.0-beta.289",
-    "@aglyn/shared-ui-snackstack": "1.0.0-beta.289",
-    "@aglyn/shared-ui-theme": "1.0.0-beta.289",
-    "@aglyn/shared-util-email": "1.0.0-beta.289",
-    "@aglyn/shared-util-errors": "1.0.0-beta.289",
-    "@aglyn/shared-util-http": "1.0.0-beta.289",
-    "@aglyn/shared-util-timestamp": "1.0.0-beta.289",
-    "@aglyn/shared-util-tools": "1.0.0-beta.289",
-    "@aglyn/tenant-data-admin": "1.0.0-beta.289",
-    "@aglyn/tenant-feature-instance": "1.0.0-beta.289",
-    "@aglyn/tenant-runtime": "1.0.0-beta.289",
+    "@aglyn/aglyn": "1.0.0-beta.290",
+    "@aglyn/shared-data-enums": "1.0.0-beta.290",
+    "@aglyn/shared-data-mdi": "1.0.0-beta.290",
+    "@aglyn/shared-data-types": "1.0.0-beta.290",
+    "@aglyn/shared-ui-email-campaigns": "1.0.0-beta.290",
+    "@aglyn/shared-ui-jsx": "1.0.0-beta.290",
+    "@aglyn/shared-ui-snackstack": "1.0.0-beta.290",
+    "@aglyn/shared-ui-theme": "1.0.0-beta.290",
+    "@aglyn/shared-util-email": "1.0.0-beta.290",
+    "@aglyn/shared-util-errors": "1.0.0-beta.290",
+    "@aglyn/shared-util-http": "1.0.0-beta.290",
+    "@aglyn/shared-util-timestamp": "1.0.0-beta.290",
+    "@aglyn/shared-util-tools": "1.0.0-beta.290",
+    "@aglyn/tenant-data-admin": "1.0.0-beta.290",
+    "@aglyn/tenant-feature-instance": "1.0.0-beta.290",
+    "@aglyn/tenant-runtime": "1.0.0-beta.290",
     "@swc/helpers": "0.5.23"
   },
   "peerDependencies": {

package/src/lib/jobs/ai-theme-brand-inputs.d.ts

@@ -155,18 +155,20 @@ export interface PublicFetchOptions {
     userAgent?: string;
     /** Sent as `Accept`. */
     acceptHeader?: string;
+    /**
+     * More request headers for one hop, asked again for EVERY hop's own address
+     * (AGL-4083, AGL-4085), so a header meant for one host never rides a
+     * redirect to another. Sent under `Accept` and `User-Agent`, and neither
+     * can be changed through here. The site import's edge header goes this
+     * way, and only to the hosts `siteImportEdgeHeadersFor` decided are ours.
+     */
+    headersFor?: (url: URL) => Readonly<Record<string, string>> | undefined;
     /** Asked of every redirect hop before it is followed; the SSRF checks run as well. */
     redirectAllowed?: (from: URL, to: URL) => boolean;
     /** Refuse a body over `maxBytes` rather than cut it. */
     refuseOver?: boolean;
     /** Keep the body's bytes as `data` rather than decode them as text (a picture, AGL-3976). */
     binary?: boolean;
-    /**
-     * More request headers (AGL-4083), sent under `Accept` and `User-Agent`:
-     * neither can be changed through here. The site import's edge header goes
-     * this way, and only to the hosts its caller names.
-     */
-    headers?: Readonly<Record<string, string>>;
 }
 /**
  * A public https resource as text, through the plugin fetch proxy's SSRF
@@ -184,7 +186,7 @@ export declare function fetchPublicText(url: string, options: {
     accept: RegExp;
     signal?: AbortSignal;
     timeoutMs?: number;
-    headers?: PublicFetchOptions['headers'];
+    headersFor?: PublicFetchOptions['headersFor'];
 }): Promise<{
     text: string;
     url: string;
@@ -203,11 +205,13 @@ export interface AiReferencePage {
 }
 /**
  * A page and its first stylesheets through {@link fetchPublicText}; `null` when the page cannot be read.
- * `headers` go with the page's own request and with a stylesheet's only when
- * it is on the page's host (AGL-4083): the site import's edge header is for
- * the site it was minted for, never a CDN's.
+ * `headersFor` is asked for every request and every redirect hop, the page's
+ * and each stylesheet's, by its own address (AGL-4083, AGL-4085): the site
+ * import's edge header, built by `siteImportEdgeHeadersFor`, answers only for
+ * the site's own origins we serve, so a hop or a stylesheet anywhere else (a
+ * CDN's, another site's) carries nothing.
  */
-export declare function readReferencePage(url: URL, signal: AbortSignal, headers?: PublicFetchOptions['headers']): Promise<AiReferencePage | null>;
+export declare function readReferencePage(url: URL, signal: AbortSignal, headersFor?: PublicFetchOptions['headersFor']): Promise<AiReferencePage | null>;
 /** A read page's brand colors: every color its markup and stylesheets use, ranked. */
 export declare function referencePageColors(page: Pick<AiReferencePage, 'html' | 'stylesheets'>): string[];
 /** The two reads that leave the process, replaceable where a spec needs to. */

package/src/lib/jobs/ai-theme-brand-inputs.js

@@ -362,7 +362,7 @@ function refusedFetch(refusal, url, extra = {}) {
                 redirect: 'manual',
                 signal: clock.signal,
                 dispatcher,
-                headers: _extends({}, publicFetchExtraHeaders(options.headers), {
+                headers: _extends({}, publicFetchExtraHeaders(options.headersFor == null ? void 0 : options.headersFor.call(options, parsed)), {
                     Accept: (_options_acceptHeader = options.acceptHeader) != null ? _options_acceptHeader : 'text/html, text/css;q=0.9'
                 }, options.userAgent ? {
                     'User-Agent': options.userAgent
@@ -448,34 +448,28 @@ function refusedFetch(refusal, url, extra = {}) {
 }
 /**
  * A page and its first stylesheets through {@link fetchPublicText}; `null` when the page cannot be read.
- * `headers` go with the page's own request and with a stylesheet's only when
- * it is on the page's host (AGL-4083): the site import's edge header is for
- * the site it was minted for, never a CDN's.
- */ export async function readReferencePage(url, signal, headers) {
+ * `headersFor` is asked for every request and every redirect hop, the page's
+ * and each stylesheet's, by its own address (AGL-4083, AGL-4085): the site
+ * import's edge header, built by `siteImportEdgeHeadersFor`, answers only for
+ * the site's own origins we serve, so a hop or a stylesheet anywhere else (a
+ * CDN's, another site's) carries nothing.
+ */ export async function readReferencePage(url, signal, headersFor) {
     const page = await fetchPublicText(url.toString(), _extends({
         maxBytes: AI_THEME_REFERENCE_MAX_BYTES,
         accept: /text\/html|application\/xhtml\+xml/i,
         signal
-    }, headers ? {
-        headers
+    }, headersFor ? {
+        headersFor
     } : {}));
     if (!page) return null;
-    const host = new URL(page.url).host;
     const stylesheets = [];
     for (const sheet of stylesheetUrlsOf(page.text, page.url)){
-        const sameHost = (()=>{
-            try {
-                return new URL(sheet).host === host;
-            } catch (unused) {
-                return false;
-            }
-        })();
         const css = await fetchPublicText(sheet, _extends({
             maxBytes: AI_THEME_STYLESHEET_MAX_BYTES,
             accept: /text\/css/i,
             signal
-        }, headers && sameHost ? {
-            headers
+        }, headersFor ? {
+            headersFor
         } : {}));
         if (css) stylesheets.push(css.text);
     }

package/src/lib/jobs/ai-theme-brand-inputs.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/jobs/ai-theme-brand-inputs.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { checkEntitlement } from '@aglyn/aglyn/app-utils/plan-entitlements'\nimport type {\n  AglynOrgBilling,\n  OrgBrandingProfile,\n} from '@aglyn/aglyn/foundation/definitions/org-billing.types'\nimport { normalizeThemeHex } from '../tools/ai-theme-tool'\nimport { AI_THEME_BRAND_BUDGET_MS } from './ai-job-theme-budget'\nimport {\n  aiMediaAssetLocation,\n  loadAiMediaSharp,\n  readAiMediaAssetBytes,\n  type AiMediaAssetLocation,\n} from './ai-media-asset'\n\n/**\n * The brand a theme job builds on (AGL-2938): the colors that belong to the\n * site before anyone describes them.\n *\n * - **The workspace's brand color** — a white-label workspace's, and only\n *   theirs: without the entitlement the resolved profile is the platform's,\n *   which is no customer's brand.\n * - **The site logo** — when the logo is an asset of the site's own media\n *   library or its org's, read from storage and reduced to its dominant\n *   colors. Never fetched from a URL: a logo that is not a library asset is\n *   left alone.\n * - **A page the brief links to** — fetched through the same SSRF guard the\n *   plugin fetch proxy uses (a public address, pinned; https only; every\n *   redirect re-validated), capped in bytes and time, and reduced to the\n *   colors its markup and stylesheets use.\n *\n * Only hex colors leave this module. Nothing a page or a logo SAYS reaches a\n * prompt, so there is nothing in them to follow. Each source is best effort:\n * one that cannot be read is named in a note and the proposal is made\n * without it.\n */\n\nexport type AiThemeBrandSource = 'organization' | 'logo' | 'reference'\n\n/** How the prompt names where each brand color came from. */\nexport const AI_THEME_BRAND_SOURCE_WORDS: Record<AiThemeBrandSource, string> = {\n  organization: 'the workspace brand color',\n  logo: 'from the site logo',\n  reference: 'from the page the brief links to',\n}\n\nexport interface AiThemeBrandColor {\n  hex: string\n  source: AiThemeBrandSource\n}\n\nexport interface AiThemeBrandInputs {\n  colors: AiThemeBrandColor[]\n  /** A source that could not be read, in customer-safe words. */\n  notes: string[]\n}\n\n/** The most colors one source contributes. */\nexport const AI_THEME_BRAND_COLORS_PER_SOURCE = 4\n\n/** The largest logo decoded for its colors. */\nexport const AI_THEME_LOGO_MAX_BYTES = 5 * 1024 * 1024\n\n/** The most of a linked page, and of each of its stylesheets, that is read. */\nexport const AI_THEME_REFERENCE_MAX_BYTES = 512 * 1024\nexport const AI_THEME_STYLESHEET_MAX_BYTES = 256 * 1024\n\n/** The stylesheets of a linked page that are read beside it. */\nexport const AI_THEME_REFERENCE_STYLESHEETS = 2\n\n/**\n * The wall clock brand colors may take, declared with the theme step's time\n * (`ai-job-theme-budget.ts`), which counts it (AGL-3035).\n */\nexport { AI_THEME_BRAND_BUDGET_MS }\n\nconst FETCH_TIMEOUT_MS = 4_000\nconst MAX_REDIRECT_HOPS = 3\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])\n\ntype Rgb = [red: number, green: number, blue: number]\n\nconst toHex = ([red, green, blue]: Rgb) =>\n  `#${[red, green, blue]\n    .map((channel) => Math.round(Math.min(255, Math.max(0, channel))).toString(16).padStart(2, '0'))\n    .join('')}`\n\nconst channelsOf = (hex: string): Rgb => [\n  parseInt(hex.slice(1, 3), 16),\n  parseInt(hex.slice(3, 5), 16),\n  parseInt(hex.slice(5, 7), 16),\n]\n\nconst distance = (a: Rgb, b: Rgb) => Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2])\n\n/** How far a color is from grey: the spread of its channels. */\nconst chroma = ([red, green, blue]: Rgb) => Math.max(red, green, blue) - Math.min(red, green, blue)\n\n/** Colors closer than this read as the same color in a palette. */\nconst SAME_COLOR_DISTANCE = 40\n\n/** A spread under this reads as a grey, which says little about a brand. */\nconst GREY_CHROMA = 38\n\n/** The distinct colors of a ranked list, colorful ones first when there are any. */\nfunction distinctColors(ranked: readonly Rgb[], count: number): string[] {\n  const colorful = ranked.filter((color) => chroma(color) > GREY_CHROMA)\n  const chosen: Rgb[] = []\n  for (const color of colorful.length ? colorful : ranked) {\n    if (chosen.some((other) => distance(other, color) < SAME_COLOR_DISTANCE)) continue\n    chosen.push(color)\n    if (chosen.length === count) break\n  }\n  return chosen.map(toHex)\n}\n\n/* ------------------------------------------------------------------------ *\n * The workspace's brand color\n * ------------------------------------------------------------------------ */\n\n/** A white-label workspace's own brand color; nothing for any other workspace. */\nexport function organizationBrandColors(\n  org: Partial<AglynOrgBilling> | null | undefined,\n): AiThemeBrandColor[] {\n  if (!checkEntitlement(org, 'whiteLabel')) return []\n  const raw = (org?.brandingProfile as OrgBrandingProfile | undefined)?.primaryColor\n  const hex = typeof raw === 'string' ? normalizeThemeHex(raw) : null\n  return hex ? [{ hex, source: 'organization' }] : []\n}\n\n/* ------------------------------------------------------------------------ *\n * The site logo\n * ------------------------------------------------------------------------ */\n\nexport type AiThemeLogoLocation = AiMediaAssetLocation\n\n/**\n * Where a site's logo lives in a media library: a media reference or a CDN\n * path naming this site's library, or its org's. Anything else — an external\n * URL, an asset of some other site or org — has no location, and the logo is\n * left alone. The media library's own rule, shared with every job that reads\n * an asset (`ai-media-asset.ts`).\n */\nexport const logoMediaLocation = aiMediaAssetLocation\n\n/**\n * The dominant colors of an image's pixels: transparent pixels skipped, the\n * rest grouped into coarse buckets, the busiest distinct buckets first, and\n * the near-white and near-black edges a logo sits on dropped while anything\n * else is there.\n */\nexport function dominantColorsFromPixels(\n  data: ArrayLike<number>,\n  channels: number,\n  count = AI_THEME_BRAND_COLORS_PER_SOURCE,\n): string[] {\n  const buckets = new Map<number, { sum: Rgb; pixels: number }>()\n  for (let index = 0; index + channels <= data.length; index += channels) {\n    if (channels >= 4 && data[index + 3] < 128) continue\n    const pixel: Rgb = [data[index], data[index + 1], data[index + 2]]\n    const key = ((pixel[0] >> 4) << 8) | ((pixel[1] >> 4) << 4) | (pixel[2] >> 4)\n    const bucket = buckets.get(key) ?? { sum: [0, 0, 0] as Rgb, pixels: 0 }\n    bucket.sum = [bucket.sum[0] + pixel[0], bucket.sum[1] + pixel[1], bucket.sum[2] + pixel[2]]\n    bucket.pixels += 1\n    buckets.set(key, bucket)\n  }\n  const ranked = [...buckets.values()]\n    .sort((a, b) => b.pixels - a.pixels)\n    .map(({ sum, pixels }): Rgb => [sum[0] / pixels, sum[1] / pixels, sum[2] / pixels])\n  const edge = (color: Rgb) => Math.min(...color) > 235 || Math.max(...color) < 20\n  const inside = ranked.filter((color) => !edge(color))\n  return distinctColors(inside.length ? inside : ranked, count)\n}\n\n/** The part of `sharp` the logo read uses. */\ninterface SharpPixelPipeline {\n  resize(width: number, height: number, options: { fit: 'inside' }): SharpPixelPipeline\n  ensureAlpha(): SharpPixelPipeline\n  raw(): SharpPixelPipeline\n  toBuffer(options: { resolveWithObject: true }): Promise<{\n    data: Buffer\n    info: { channels: number }\n  }>\n}\ntype SharpPixels = (input: Buffer, options?: { limitInputPixels?: number }) => SharpPixelPipeline\n\nconst LOGO_TYPES = /^image\\/(png|jpeg|webp|gif|avif|svg\\+xml)$/\n\n/**\n * A library logo's colors, read the way the media CDN would serve it\n * (`readAiMediaAssetBytes`). `null` when the asset may not be read or the bytes\n * cannot be decoded.\n *\n * On Vercel the read and the decode run in the plugins' image function, the\n * one function that ships `sharp`'s native library (`ai-image-reads.ts`,\n * AGL-4048), inside the brand reads' own bound (`signal`); anywhere else they\n * run here. Either way an answer that does not come is a logo whose colors\n * could not be read.\n */\nasync function readLogoColors(\n  firestore: FirebaseFirestore.Firestore,\n  location: AiThemeLogoLocation,\n  hostId: string,\n  signal?: AbortSignal,\n): Promise<string[] | null> {\n  const images = await import('./ai-image-reads')\n  const origin = images.aiImageReadsOrigin()\n  if (!origin) return readAiThemeLogoColorsAt(firestore, location, hostId)\n  // The location was held to the site's org already: an org asset's scope\n  // IS that org, and a site's own asset needs none.\n  const orgId = location.scope.isOrg ? location.scope.scopeId : ''\n  const payload = images.aiImageReadsRequest('logo-colors', location, hostId, orgId)\n  if (!payload) return null\n  const answer = await images.requestAiImageReads<{ colors?: unknown }>(\n    origin,\n    payload,\n    signal ?? AbortSignal.timeout(AI_THEME_BRAND_BUDGET_MS),\n  )\n  return logoColorsAnswer(answer?.colors)\n}\n\n/**\n * A library logo's colors read and decoded in THIS process, from a location\n * already held to the media library's rule: what `readLogoColors` does off\n * Vercel, and what the images function does for it on Vercel.\n */\nexport async function readAiThemeLogoColorsAt(\n  firestore: FirebaseFirestore.Firestore,\n  location: AiThemeLogoLocation,\n  hostId: string,\n): Promise<string[] | null> {\n  const asset = await readAiMediaAssetBytes(firestore, location, hostId, {\n    maxBytes: AI_THEME_LOGO_MAX_BYTES,\n    types: LOGO_TYPES,\n  })\n  if (!asset) return null\n  const sharp = (await loadAiMediaSharp()) as SharpPixels\n  const { data, info } = await sharp(asset.buffer, { limitInputPixels: 64_000_000 })\n    .resize(48, 48, { fit: 'inside' })\n    .ensureAlpha()\n    .raw()\n    .toBuffer({ resolveWithObject: true })\n  return dominantColorsFromPixels(data, info.channels)\n}\n\n/** What the images function answered, as hex colors: only colors leave it. */\nfunction logoColorsAnswer(colors: unknown): string[] | null {\n  if (!Array.isArray(colors)) return null\n  return colors\n    .map((color) => (typeof color === 'string' ? normalizeThemeHex(color) : null))\n    .filter((hex): hex is string => Boolean(hex))\n    .slice(0, AI_THEME_BRAND_COLORS_PER_SOURCE)\n}\n\n/* ------------------------------------------------------------------------ *\n * A page the brief links to\n * ------------------------------------------------------------------------ */\n\nconst URL_IN_BRIEF = /\\bhttps?:\\/\\/[^\\s<>\"'`)\\]]+/i\n\n/** The first https link in a brief, or `null` — an http link is not followed. */\nexport function referenceUrlOf(brief: string): URL | null {\n  const match = URL_IN_BRIEF.exec(brief)\n  if (!match) return null\n  try {\n    const url = new URL(match[0].replace(/[.,;:!?]+$/, ''))\n    return url.protocol === 'https:' ? url : null\n  } catch {\n    return null\n  }\n}\n\n/** How much a page's declared `theme-color` counts beside one use of a color. */\nexport const AI_THEME_THEME_COLOR_WEIGHT = 25\n\nconst THEME_COLOR_META = /<meta\\b[^>]*\\bname\\s*=\\s*[\"']?theme-color\\b[^>]*>/gi\nconst CONTENT_ATTRIBUTE = /\\bcontent\\s*=\\s*[\"']?\\s*([^\"'\\s>]+)/i\n// A `#` after `&` is a character reference (`&#039;`), not a color.\nconst HEX_LITERAL = /(?<!&)#(?:[0-9a-f]{6}|[0-9a-f]{3})(?![0-9a-z])/gi\nconst RGB_LITERAL = /rgba?\\(\\s*(\\d{1,3})\\s*[,\\s]\\s*(\\d{1,3})\\s*[,\\s]\\s*(\\d{1,3})/gi\n\n/** Every color a page's markup or a stylesheet uses, weighted by how often. */\nexport function colorsFromMarkup(text: string): Map<string, number> {\n  const weights = new Map<string, number>()\n  const add = (hex: string | null, weight: number) => {\n    if (hex) weights.set(hex, (weights.get(hex) ?? 0) + weight)\n  }\n  for (const tag of text.match(THEME_COLOR_META) ?? []) {\n    add(normalizeThemeHex(CONTENT_ATTRIBUTE.exec(tag)?.[1] ?? ''), AI_THEME_THEME_COLOR_WEIGHT)\n  }\n  for (const match of text.matchAll(HEX_LITERAL)) add(normalizeThemeHex(match[0]), 1)\n  for (const match of text.matchAll(RGB_LITERAL)) {\n    add(toHex([Number(match[1]), Number(match[2]), Number(match[3])]), 1)\n  }\n  return weights\n}\n\nconst STYLESHEET_LINK = /<link\\b[^>]*\\brel\\s*=\\s*[\"']?stylesheet\\b[^>]*>/gi\nconst HREF_ATTRIBUTE = /\\bhref\\s*=\\s*[\"']?\\s*([^\"'\\s>]+)/i\n\n/** The https stylesheets a page links, resolved against it, at most a few. */\nexport function stylesheetUrlsOf(html: string, pageUrl: string): string[] {\n  const urls: string[] = []\n  for (const tag of html.match(STYLESHEET_LINK) ?? []) {\n    const href = HREF_ATTRIBUTE.exec(tag)?.[1]\n    if (!href) continue\n    try {\n      const url = new URL(href, pageUrl)\n      if (url.protocol === 'https:' && !urls.includes(url.toString())) urls.push(url.toString())\n    } catch {\n      continue\n    }\n    if (urls.length === AI_THEME_REFERENCE_STYLESHEETS) break\n  }\n  return urls\n}\n\n/** The most used distinct colors, the colorful before the greys. */\nexport function rankBrandColors(\n  weights: ReadonlyMap<string, number>,\n  count = AI_THEME_BRAND_COLORS_PER_SOURCE,\n): string[] {\n  const ranked = [...weights.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .map(([hex]) => channelsOf(hex))\n  return distinctColors(ranked, count)\n}\n\n/** One abort for the caller's signal and a clock, whichever ends first. */\nfunction deadline(signal: AbortSignal | undefined, ms: number) {\n  const controller = new AbortController()\n  const timer = setTimeout(() => controller.abort(), ms)\n  const onAbort = () => controller.abort()\n  if (signal?.aborted) controller.abort()\n  signal?.addEventListener('abort', onAbort)\n  return {\n    signal: controller.signal,\n    done: () => {\n      clearTimeout(timer)\n      signal?.removeEventListener('abort', onAbort)\n    },\n  }\n}\n\n/**\n * The body as text, to `maxBytes`. Past the cap it is cut there, or with\n * `refuseOver` it is not read at all: `null`, the read canceled.\n */\nasync function readCapped(\n  body: ReadableStream<Uint8Array>,\n  maxBytes: number,\n  refuseOver = false,\n  binary = false,\n): Promise<{ text: string; bytes: number; truncated: boolean; data?: Uint8Array } | null> {\n  const reader = body.getReader()\n  const chunks: Uint8Array[] = []\n  let total = 0\n  let truncated = false\n  for (;;) {\n    const { done, value } = await reader.read()\n    if (done || !value) break\n    const room = maxBytes - total\n    if (value.byteLength > room) {\n      await reader.cancel().catch(() => undefined)\n      if (refuseOver) return null\n      chunks.push(value.subarray(0, room))\n      total += room\n      truncated = true\n      break\n    }\n    chunks.push(value)\n    total += value.byteLength\n  }\n  const joined = new Uint8Array(total)\n  let offset = 0\n  for (const chunk of chunks) {\n    joined.set(chunk, offset)\n    offset += chunk.byteLength\n  }\n  // A picture's bytes are kept as they came (AGL-3976), never decoded as text.\n  if (binary) return { text: '', bytes: total, truncated, data: joined }\n  return { text: new TextDecoder().decode(joined), bytes: total, truncated }\n}\n\n/** Why {@link fetchPublicTextDetailed} read nothing. */\nexport type PublicFetchRefusal =\n  /** Not https, a port, credentials, or not a URL at all. */\n  | 'refused-url'\n  /** The name resolves to a private address, or not at all. */\n  | 'private-address'\n  /** A redirect the caller's `redirectAllowed` refused. */\n  | 'redirect-refused'\n  | 'too-many-redirects'\n  /** An answer that was not 2xx; `status` says which. */\n  | 'http-status'\n  /** A 2xx whose content type the caller does not accept. */\n  | 'content-type'\n  /** Past `maxBytes` with `refuseOver` set. */\n  | 'too-large'\n  /** The socket, the clock or the caller's signal ended it. */\n  | 'network'\n\n/**\n * One shape for both outcomes, read by `ok`: a union does not narrow across\n * this repo's library boundaries (`strictNullChecks` is off), so a caller\n * could not reach either arm's fields.\n */\nexport interface PublicFetchResult {\n  ok: boolean\n  /** The address read, after redirects; or the one that was refused. */\n  url: string\n  /** Empty unless `ok`. */\n  text: string\n  bytes: number\n  /** Cut at `maxBytes` (never with `refuseOver`). */\n  truncated: boolean\n  contentType: string\n  /** Null when `ok`. */\n  refusal: PublicFetchRefusal | null\n  status?: number\n  /** Where a refused redirect pointed. */\n  location?: string\n  /** The body's bytes, with `binary`; absent otherwise. */\n  data?: Uint8Array\n  /**\n   * Whose edge refused an error answer, when its headers say (AGL-4083): the\n   * bot protection a site owner opens for AglynBot is that edge's setting.\n   */\n  edge?: PublicFetchEdge\n}\n\n/** An edge network that names itself on the answers it refuses with. */\nexport type PublicFetchEdge = 'vercel' | 'cloudflare'\n\n/** The edge an answer came through, from the headers each one sets; `null` for neither. */\nexport function publicFetchEdgeOf(headers: Pick<Headers, 'get'>): PublicFetchEdge | null {\n  const server = (headers.get('server') ?? '').toLowerCase()\n  if (headers.get('x-vercel-mitigated') || headers.get('x-vercel-id') || server === 'vercel') return 'vercel'\n  if (headers.get('cf-mitigated') || headers.get('cf-ray') || server === 'cloudflare') return 'cloudflare'\n  return null\n}\n\nfunction refusedFetch(\n  refusal: PublicFetchRefusal,\n  url: string,\n  extra: { status?: number; location?: string; edge?: PublicFetchEdge } = {},\n): PublicFetchResult {\n  return { ok: false, url, text: '', bytes: 0, truncated: false, contentType: '', refusal, ...extra }\n}\n\nexport interface PublicFetchOptions {\n  maxBytes: number\n  accept: RegExp\n  signal?: AbortSignal\n  timeoutMs?: number\n  /** Sent as `User-Agent`; the runtime's own when absent. */\n  userAgent?: string\n  /** Sent as `Accept`. */\n  acceptHeader?: string\n  /** Asked of every redirect hop before it is followed; the SSRF checks run as well. */\n  redirectAllowed?: (from: URL, to: URL) => boolean\n  /** Refuse a body over `maxBytes` rather than cut it. */\n  refuseOver?: boolean\n  /** Keep the body's bytes as `data` rather than decode them as text (a picture, AGL-3976). */\n  binary?: boolean\n  /**\n   * More request headers (AGL-4083), sent under `Accept` and `User-Agent`:\n   * neither can be changed through here. The site import's edge header goes\n   * this way, and only to the hosts its caller names.\n   */\n  headers?: Readonly<Record<string, string>>\n}\n\n/** The caller's extra headers, without any that would say what the request is: those are the fetch's own. */\nfunction publicFetchExtraHeaders(headers: PublicFetchOptions['headers']): Record<string, string> {\n  const out: Record<string, string> = {}\n  for (const [name, value] of Object.entries(headers ?? {})) {\n    if (/^(?:accept|user-agent|host|cookie|authorization|content-length)$/i.test(name)) continue\n    if (typeof value === 'string') out[name] = value\n  }\n  return out\n}\n\n/**\n * A public https resource as text, through the plugin fetch proxy's SSRF\n * guard: the host must resolve only to public addresses and the socket is\n * pinned to the address checked; no credentials in the URL and no port but\n * the default; every redirect re-enters the same checks and the caller's own\n * `redirectAllowed`; the body is read to a byte cap under a clock, and only a\n * content type the caller accepts is read at all. Says why when it reads\n * nothing.\n */\nexport async function fetchPublicTextDetailed(\n  url: string,\n  options: PublicFetchOptions,\n): Promise<PublicFetchResult> {\n  const { createPinnedDispatcher, resolvePublicIp } = await import('./ai-theme-brand-server')\n  let current = url\n  for (let hop = 0; hop <= MAX_REDIRECT_HOPS; hop += 1) {\n    let parsed: URL\n    try {\n      parsed = new URL(current)\n    } catch {\n      return refusedFetch('refused-url', current)\n    }\n    if (parsed.protocol !== 'https:' || parsed.username || parsed.password) {\n      return refusedFetch('refused-url', current)\n    }\n    if (parsed.port && parsed.port !== '443') {\n      return refusedFetch('refused-url', current)\n    }\n    const pinned = await resolvePublicIp(parsed.hostname)\n    if (!pinned) return refusedFetch('private-address', parsed.toString())\n    const dispatcher = createPinnedDispatcher(pinned)\n    const clock = deadline(options.signal, options.timeoutMs ?? FETCH_TIMEOUT_MS)\n    try {\n      // `dispatcher` is undici's, which the global fetch honors; a variable\n      // rather than a literal keeps the DOM `RequestInit` check off it.\n      const init = {\n        method: 'GET',\n        redirect: 'manual' as const,\n        signal: clock.signal,\n        dispatcher,\n        headers: {\n          ...publicFetchExtraHeaders(options.headers),\n          Accept: options.acceptHeader ?? 'text/html, text/css;q=0.9',\n          ...(options.userAgent ? { 'User-Agent': options.userAgent } : {}),\n        },\n      }\n      const response = await fetch(parsed.toString(), init as RequestInit)\n      if (REDIRECT_STATUSES.has(response.status)) {\n        await response.body?.cancel().catch(() => undefined)\n        const location = response.headers.get('location')\n        if (!location) {\n          return refusedFetch('http-status', parsed.toString(), { status: response.status })\n        }\n        let next: URL\n        try {\n          next = new URL(location, parsed)\n        } catch {\n          return refusedFetch('redirect-refused', parsed.toString(), { location })\n        }\n        if (options.redirectAllowed && !options.redirectAllowed(parsed, next)) {\n          return refusedFetch('redirect-refused', parsed.toString(), { location: next.toString() })\n        }\n        current = next.toString()\n        continue\n      }\n      if (!response.ok || !response.body) {\n        await response.body?.cancel().catch(() => undefined)\n        const edge = publicFetchEdgeOf(response.headers)\n        return refusedFetch('http-status', parsed.toString(), { status: response.status, ...(edge ? { edge } : {}) })\n      }\n      const contentType = response.headers.get('content-type') ?? ''\n      if (!options.accept.test(contentType)) {\n        await response.body.cancel().catch(() => undefined)\n        return refusedFetch('content-type', parsed.toString(), { status: response.status })\n      }\n      const declared = Number(response.headers.get('content-length') ?? '')\n      if (options.refuseOver && Number.isFinite(declared) && declared > options.maxBytes) {\n        await response.body.cancel().catch(() => undefined)\n        return refusedFetch('too-large', parsed.toString(), { status: response.status })\n      }\n      const body = await readCapped(response.body, options.maxBytes, options.refuseOver, options.binary === true)\n      if (!body) {\n        return refusedFetch('too-large', parsed.toString(), { status: response.status })\n      }\n      return { ok: true, url: parsed.toString(), contentType, refusal: null, ...body }\n    } catch {\n      return refusedFetch('network', parsed.toString())\n    } finally {\n      clock.done()\n      await dispatcher.close().catch(() => undefined)\n    }\n  }\n  return refusedFetch('too-many-redirects', current)\n}\n\n/** {@link fetchPublicTextDetailed}, answering `null` for anything it refused. */\nexport async function fetchPublicText(\n  url: string,\n  options: { maxBytes: number; accept: RegExp; signal?: AbortSignal; timeoutMs?: number; headers?: PublicFetchOptions['headers'] },\n): Promise<{ text: string; url: string } | null> {\n  const result = await fetchPublicTextDetailed(url, options)\n  return result.ok ? { text: result.text, url: result.url } : null\n}\n\n/**\n * A public page as it was read: its markup and the text of its first\n * stylesheets ({@link AI_THEME_REFERENCE_STYLESHEETS}), each under its byte\n * cap. What a theme job takes its colors from and a site start its\n * inspiration signals (AGL-3975); never a crawl, one page and its sheets.\n */\nexport interface AiReferencePage {\n  /** The address the markup was finally read at, after redirects. */\n  url: string\n  html: string\n  stylesheets: string[]\n}\n\n/**\n * A page and its first stylesheets through {@link fetchPublicText}; `null` when the page cannot be read.\n * `headers` go with the page's own request and with a stylesheet's only when\n * it is on the page's host (AGL-4083): the site import's edge header is for\n * the site it was minted for, never a CDN's.\n */\nexport async function readReferencePage(\n  url: URL,\n  signal: AbortSignal,\n  headers?: PublicFetchOptions['headers'],\n): Promise<AiReferencePage | null> {\n  const page = await fetchPublicText(url.toString(), {\n    maxBytes: AI_THEME_REFERENCE_MAX_BYTES,\n    accept: /text\\/html|application\\/xhtml\\+xml/i,\n    signal,\n    ...(headers ? { headers } : {}),\n  })\n  if (!page) return null\n  const host = new URL(page.url).host\n  const stylesheets: string[] = []\n  for (const sheet of stylesheetUrlsOf(page.text, page.url)) {\n    const sameHost = (() => {\n      try {\n        return new URL(sheet).host === host\n      } catch {\n        return false\n      }\n    })()\n    const css = await fetchPublicText(sheet, {\n      maxBytes: AI_THEME_STYLESHEET_MAX_BYTES,\n      accept: /text\\/css/i,\n      signal,\n      ...(headers && sameHost ? { headers } : {}),\n    })\n    if (css) stylesheets.push(css.text)\n  }\n  return { url: page.url, html: page.text, stylesheets }\n}\n\n/** A read page's brand colors: every color its markup and stylesheets use, ranked. */\nexport function referencePageColors(page: Pick<AiReferencePage, 'html' | 'stylesheets'>): string[] {\n  const weights = colorsFromMarkup(page.html)\n  for (const css of page.stylesheets) {\n    for (const [hex, weight] of colorsFromMarkup(css)) {\n      weights.set(hex, (weights.get(hex) ?? 0) + weight)\n    }\n  }\n  return rankBrandColors(weights)\n}\n\n/** A linked page's colors, from its markup and its first stylesheets. */\nasync function readReferenceColors(url: URL, signal: AbortSignal): Promise<string[] | null> {\n  const page = await readReferencePage(url, signal)\n  return page ? referencePageColors(page) : null\n}\n\n/* ------------------------------------------------------------------------ *\n * Gathering\n * ------------------------------------------------------------------------ */\n\n/** The two reads that leave the process, replaceable where a spec needs to. */\nexport interface AiThemeBrandSeams {\n  logoColors?: (\n    firestore: FirebaseFirestore.Firestore,\n    location: AiThemeLogoLocation,\n    hostId: string,\n    /** The brand reads' bound, which a read made elsewhere waits no longer than. */\n    signal?: AbortSignal,\n  ) => Promise<string[] | null>\n  referenceColors?: (url: URL, signal: AbortSignal) => Promise<string[] | null>\n}\n\nexport async function gatherAiThemeBrandInputs(\n  input: {\n    firestore: FirebaseFirestore.Firestore\n    org: Partial<AglynOrgBilling> | null\n    hostId: string\n    /** The site's host document. */\n    host: Record<string, unknown>\n    brief: string\n    signal?: AbortSignal\n  },\n  seams: AiThemeBrandSeams = {},\n): Promise<AiThemeBrandInputs> {\n  const colors: AiThemeBrandColor[] = [...organizationBrandColors(input.org)]\n  const notes: string[] = []\n  const clock = deadline(input.signal, AI_THEME_BRAND_BUDGET_MS)\n  const add = (found: readonly string[], source: AiThemeBrandSource) => {\n    for (const hex of found.slice(0, AI_THEME_BRAND_COLORS_PER_SOURCE)) {\n      if (!colors.some((color) => color.hex === hex)) colors.push({ hex, source })\n    }\n  }\n  try {\n    const location = logoMediaLocation(\n      input.host['logoUrl'],\n      input.hostId,\n      String(input.host['orgId'] ?? ''),\n    )\n    if (location) {\n      const found = await (seams.logoColors ?? readLogoColors)(\n        input.firestore,\n        location,\n        input.hostId,\n        clock.signal,\n      ).catch(() => null)\n      if (found) add(found, 'logo')\n      else notes.push(\"The site logo's colors could not be read, so the proposal does not use them.\")\n    }\n    const reference = referenceUrlOf(input.brief)\n    if (reference) {\n      const found = clock.signal.aborted\n        ? null\n        : await (seams.referenceColors ?? readReferenceColors)(reference, clock.signal).catch(\n            () => null,\n          )\n      if (found?.length) add(found, 'reference')\n      else {\n        notes.push(\n          `The page at ${reference.hostname} could not be read for its colors, so the proposal does not use them.`,\n        )\n      }\n    }\n  } finally {\n    clock.done()\n  }\n  return { colors, notes }\n}\n"],"names":["checkEntitlement","normalizeThemeHex","AI_THEME_BRAND_BUDGET_MS","aiMediaAssetLocation","loadAiMediaSharp","readAiMediaAssetBytes","AI_THEME_BRAND_SOURCE_WORDS","organization","logo","reference","AI_THEME_BRAND_COLORS_PER_SOURCE","AI_THEME_LOGO_MAX_BYTES","AI_THEME_REFERENCE_MAX_BYTES","AI_THEME_STYLESHEET_MAX_BYTES","AI_THEME_REFERENCE_STYLESHEETS","FETCH_TIMEOUT_MS","MAX_REDIRECT_HOPS","REDIRECT_STATUSES","Set","toHex","red","green","blue","map","channel","Math","round","min","max","toString","padStart","join","channelsOf","hex","parseInt","slice","distance","a","b","hypot","chroma","SAME_COLOR_DISTANCE","GREY_CHROMA","distinctColors","ranked","count","colorful","filter","color","chosen","length","some","other","push","organizationBrandColors","org","raw","brandingProfile","primaryColor","source","logoMediaLocation","dominantColorsFromPixels","data","channels","buckets","Map","index","pixel","key","bucket","get","sum","pixels","set","values","sort","edge","inside","LOGO_TYPES","readLogoColors","firestore","location","hostId","signal","images","origin","aiImageReadsOrigin","readAiThemeLogoColorsAt","orgId","scope","isOrg","scopeId","payload","aiImageReadsRequest","answer","requestAiImageReads","AbortSignal","timeout","logoColorsAnswer","colors","asset","maxBytes","types","sharp","info","buffer","limitInputPixels","resize","fit","ensureAlpha","toBuffer","resolveWithObject","Array","isArray","Boolean","URL_IN_BRIEF","referenceUrlOf","brief","match","exec","url","URL","replace","protocol","AI_THEME_THEME_COLOR_WEIGHT","THEME_COLOR_META","CONTENT_ATTRIBUTE","HEX_LITERAL","RGB_LITERAL","colorsFromMarkup","text","weights","add","weight","tag","matchAll","Number","STYLESHEET_LINK","HREF_ATTRIBUTE","stylesheetUrlsOf","html","pageUrl","urls","href","includes","rankBrandColors","entries","deadline","ms","controller","AbortController","timer","setTimeout","abort","onAbort","aborted","addEventListener","done","clearTimeout","removeEventListener","readCapped","body","refuseOver","binary","reader","getReader","chunks","total","truncated","value","read","room","byteLength","cancel","catch","undefined","subarray","joined","Uint8Array","offset","chunk","bytes","TextDecoder","decode","publicFetchEdgeOf","headers","server","toLowerCase","refusedFetch","refusal","extra","ok","contentType","publicFetchExtraHeaders","out","name","Object","test","fetchPublicTextDetailed","options","createPinnedDispatcher","resolvePublicIp","current","hop","parsed","username","password","port","pinned","hostname","dispatcher","clock","timeoutMs","response","init","method","redirect","Accept","acceptHeader","userAgent","fetch","has","status","next","redirectAllowed","accept","declared","isFinite","close","fetchPublicText","result","readReferencePage","page","host","stylesheets","sheet","sameHost","css","referencePageColors","readReferenceColors","gatherAiThemeBrandInputs","input","seams","notes","found","String","logoColors","referenceColors"],"mappings":";AAAA;;;;;;;;;;;;;;;CAeC,GAED,SAASA,gBAAgB,QAAQ,2CAA0C;AAK3E,SAASC,iBAAiB,QAAQ,4BAAwB;AAC1D,SAASC,wBAAwB,QAAQ,2BAAuB;AAChE,SACEC,oBAAoB,EACpBC,gBAAgB,EAChBC,qBAAqB,QAEhB,sBAAkB;AA0BzB,2DAA2D,GAC3D,OAAO,MAAMC,8BAAkE;IAC7EC,cAAc;IACdC,MAAM;IACNC,WAAW;AACb,EAAC;AAaD,4CAA4C,GAC5C,OAAO,MAAMC,mCAAmC,EAAC;AAEjD,6CAA6C,GAC7C,OAAO,MAAMC,0BAA0B,IAAI,OAAO,KAAI;AAEtD,6EAA6E,GAC7E,OAAO,MAAMC,+BAA+B,MAAM,KAAI;AACtD,OAAO,MAAMC,gCAAgC,MAAM,KAAI;AAEvD,8DAA8D,GAC9D,OAAO,MAAMC,iCAAiC,EAAC;AAE/C;;;CAGC,GACD,SAASZ,wBAAwB,GAAE;AAEnC,MAAMa,mBAAmB;AACzB,MAAMC,oBAAoB;AAC1B,MAAMC,oBAAoB,IAAIC,IAAI;IAAC;IAAK;IAAK;IAAK;IAAK;CAAI;AAI3D,MAAMC,QAAQ,CAAC,CAACC,KAAKC,OAAOC,KAAU,GACpC,CAAC,CAAC,EAAE;QAACF;QAAKC;QAAOC;KAAK,CACnBC,GAAG,CAAC,CAACC,UAAYC,KAAKC,KAAK,CAACD,KAAKE,GAAG,CAAC,KAAKF,KAAKG,GAAG,CAAC,GAAGJ,WAAWK,QAAQ,CAAC,IAAIC,QAAQ,CAAC,GAAG,MAC1FC,IAAI,CAAC,KAAK;AAEf,MAAMC,aAAa,CAACC,MAAqB;QACvCC,SAASD,IAAIE,KAAK,CAAC,GAAG,IAAI;QAC1BD,SAASD,IAAIE,KAAK,CAAC,GAAG,IAAI;QAC1BD,SAASD,IAAIE,KAAK,CAAC,GAAG,IAAI;KAC3B;AAED,MAAMC,WAAW,CAACC,GAAQC,IAAWb,KAAKc,KAAK,CAACF,CAAC,CAAC,EAAE,GAAGC,CAAC,CAAC,EAAE,EAAED,CAAC,CAAC,EAAE,GAAGC,CAAC,CAAC,EAAE,EAAED,CAAC,CAAC,EAAE,GAAGC,CAAC,CAAC,EAAE;AAErF,8DAA8D,GAC9D,MAAME,SAAS,CAAC,CAACpB,KAAKC,OAAOC,KAAU,GAAKG,KAAKG,GAAG,CAACR,KAAKC,OAAOC,QAAQG,KAAKE,GAAG,CAACP,KAAKC,OAAOC;AAE9F,iEAAiE,GACjE,MAAMmB,sBAAsB;AAE5B,0EAA0E,GAC1E,MAAMC,cAAc;AAEpB,kFAAkF,GAClF,SAASC,eAAeC,MAAsB,EAAEC,KAAa;IAC3D,MAAMC,WAAWF,OAAOG,MAAM,CAAC,CAACC,QAAUR,OAAOQ,SAASN;IAC1D,MAAMO,SAAgB,EAAE;IACxB,KAAK,MAAMD,SAASF,SAASI,MAAM,GAAGJ,WAAWF,OAAQ;QACvD,IAAIK,OAAOE,IAAI,CAAC,CAACC,QAAUhB,SAASgB,OAAOJ,SAASP,sBAAsB;QAC1EQ,OAAOI,IAAI,CAACL;QACZ,IAAIC,OAAOC,MAAM,KAAKL,OAAO;IAC/B;IACA,OAAOI,OAAO1B,GAAG,CAACJ;AACpB;AAEA;;4EAE4E,GAE5E,gFAAgF,GAChF,OAAO,SAASmC,wBACdC,GAAgD;QAGnCA;IADb,IAAI,CAACvD,iBAAiBuD,KAAK,eAAe,OAAO,EAAE;IACnD,MAAMC,MAAOD,wBAAAA,uBAAAA,IAAKE,eAAe,qBAArB,AAACF,qBAAyDG,YAAY;IAClF,MAAMzB,MAAM,OAAOuB,QAAQ,WAAWvD,kBAAkBuD,OAAO;IAC/D,OAAOvB,MAAM;QAAC;YAAEA;YAAK0B,QAAQ;QAAe;KAAE,GAAG,EAAE;AACrD;AAQA;;;;;;CAMC,GACD,OAAO,MAAMC,oBAAoBzD,qBAAoB;AAErD;;;;;CAKC,GACD,OAAO,SAAS0D,yBACdC,IAAuB,EACvBC,QAAgB,EAChBlB,QAAQnC,gCAAgC;IAExC,MAAMsD,UAAU,IAAIC;IACpB,IAAK,IAAIC,QAAQ,GAAGA,QAAQH,YAAYD,KAAKZ,MAAM,EAAEgB,SAASH,SAAU;YAIvDC;QAHf,IAAID,YAAY,KAAKD,IAAI,CAACI,QAAQ,EAAE,GAAG,KAAK;QAC5C,MAAMC,QAAa;YAACL,IAAI,CAACI,MAAM;YAAEJ,IAAI,CAACI,QAAQ,EAAE;YAAEJ,IAAI,CAACI,QAAQ,EAAE;SAAC;QAClE,MAAME,MAAM,AAAED,KAAK,CAAC,EAAE,IAAI,KAAM,IAAM,AAACA,KAAK,CAAC,EAAE,IAAI,KAAM,IAAMA,KAAK,CAAC,EAAE,IAAI;QAC3E,MAAME,UAASL,eAAAA,QAAQM,GAAG,CAACF,gBAAZJ,eAAoB;YAAEO,KAAK;gBAAC;gBAAG;gBAAG;aAAE;YAASC,QAAQ;QAAE;QACtEH,OAAOE,GAAG,GAAG;YAACF,OAAOE,GAAG,CAAC,EAAE,GAAGJ,KAAK,CAAC,EAAE;YAAEE,OAAOE,GAAG,CAAC,EAAE,GAAGJ,KAAK,CAAC,EAAE;YAAEE,OAAOE,GAAG,CAAC,EAAE,GAAGJ,KAAK,CAAC,EAAE;SAAC;QAC3FE,OAAOG,MAAM,IAAI;QACjBR,QAAQS,GAAG,CAACL,KAAKC;IACnB;IACA,MAAMzB,SAAS;WAAIoB,QAAQU,MAAM;KAAG,CACjCC,IAAI,CAAC,CAACtC,GAAGC,IAAMA,EAAEkC,MAAM,GAAGnC,EAAEmC,MAAM,EAClCjD,GAAG,CAAC,CAAC,EAAEgD,GAAG,EAAEC,MAAM,EAAE,GAAU;YAACD,GAAG,CAAC,EAAE,GAAGC;YAAQD,GAAG,CAAC,EAAE,GAAGC;YAAQD,GAAG,CAAC,EAAE,GAAGC;SAAO;IACpF,MAAMI,OAAO,CAAC5B,QAAevB,KAAKE,GAAG,IAAIqB,SAAS,OAAOvB,KAAKG,GAAG,IAAIoB,SAAS;IAC9E,MAAM6B,SAASjC,OAAOG,MAAM,CAAC,CAACC,QAAU,CAAC4B,KAAK5B;IAC9C,OAAOL,eAAekC,OAAO3B,MAAM,GAAG2B,SAASjC,QAAQC;AACzD;AAcA,MAAMiC,aAAa;AAEnB;;;;;;;;;;CAUC,GACD,eAAeC,eACbC,SAAsC,EACtCC,QAA6B,EAC7BC,MAAc,EACdC,MAAoB;IAEpB,MAAMC,SAAS,MAAM,MAAM,CAAC;IAC5B,MAAMC,SAASD,OAAOE,kBAAkB;IACxC,IAAI,CAACD,QAAQ,OAAOE,wBAAwBP,WAAWC,UAAUC;IACjE,wEAAwE;IACxE,kDAAkD;IAClD,MAAMM,QAAQP,SAASQ,KAAK,CAACC,KAAK,GAAGT,SAASQ,KAAK,CAACE,OAAO,GAAG;IAC9D,MAAMC,UAAUR,OAAOS,mBAAmB,CAAC,eAAeZ,UAAUC,QAAQM;IAC5E,IAAI,CAACI,SAAS,OAAO;IACrB,MAAME,SAAS,MAAMV,OAAOW,mBAAmB,CAC7CV,QACAO,SACAT,iBAAAA,SAAUa,YAAYC,OAAO,CAAC/F;IAEhC,OAAOgG,iBAAiBJ,0BAAAA,OAAQK,MAAM;AACxC;AAEA;;;;CAIC,GACD,OAAO,eAAeZ,wBACpBP,SAAsC,EACtCC,QAA6B,EAC7BC,MAAc;IAEd,MAAMkB,QAAQ,MAAM/F,sBAAsB2E,WAAWC,UAAUC,QAAQ;QACrEmB,UAAU1F;QACV2F,OAAOxB;IACT;IACA,IAAI,CAACsB,OAAO,OAAO;IACnB,MAAMG,QAAS,MAAMnG;IACrB,MAAM,EAAE0D,IAAI,EAAE0C,IAAI,EAAE,GAAG,MAAMD,MAAMH,MAAMK,MAAM,EAAE;QAAEC,kBAAkB;IAAW,GAC7EC,MAAM,CAAC,IAAI,IAAI;QAAEC,KAAK;IAAS,GAC/BC,WAAW,GACXrD,GAAG,GACHsD,QAAQ,CAAC;QAAEC,mBAAmB;IAAK;IACtC,OAAOlD,yBAAyBC,MAAM0C,KAAKzC,QAAQ;AACrD;AAEA,4EAA4E,GAC5E,SAASmC,iBAAiBC,MAAe;IACvC,IAAI,CAACa,MAAMC,OAAO,CAACd,SAAS,OAAO;IACnC,OAAOA,OACJ5E,GAAG,CAAC,CAACyB,QAAW,OAAOA,UAAU,WAAW/C,kBAAkB+C,SAAS,MACvED,MAAM,CAAC,CAACd,MAAuBiF,QAAQjF,MACvCE,KAAK,CAAC,GAAGzB;AACd;AAEA;;4EAE4E,GAE5E,MAAMyG,eAAe;AAErB,+EAA+E,GAC/E,OAAO,SAASC,eAAeC,KAAa;IAC1C,MAAMC,QAAQH,aAAaI,IAAI,CAACF;IAChC,IAAI,CAACC,OAAO,OAAO;IACnB,IAAI;QACF,MAAME,MAAM,IAAIC,IAAIH,KAAK,CAAC,EAAE,CAACI,OAAO,CAAC,cAAc;QACnD,OAAOF,IAAIG,QAAQ,KAAK,WAAWH,MAAM;IAC3C,EAAE,eAAM;QACN,OAAO;IACT;AACF;AAEA,+EAA+E,GAC/E,OAAO,MAAMI,8BAA8B,GAAE;AAE7C,MAAMC,mBAAmB;AACzB,MAAMC,oBAAoB;AAC1B,oEAAoE;AACpE,MAAMC,cAAc;AACpB,MAAMC,cAAc;AAEpB,6EAA6E,GAC7E,OAAO,SAASC,iBAAiBC,IAAY;QAKzBA;IAJlB,MAAMC,UAAU,IAAIlE;IACpB,MAAMmE,MAAM,CAACnG,KAAoBoG;YACJF;QAA3B,IAAIlG,KAAKkG,QAAQ1D,GAAG,CAACxC,KAAK,EAACkG,eAAAA,QAAQ7D,GAAG,CAACrC,gBAAZkG,eAAoB,KAAKE;IACtD;IACA,KAAK,MAAMC,QAAOJ,cAAAA,KAAKZ,KAAK,CAACO,6BAAXK,cAAgC,EAAE,CAAE;;YAC9BJ;QAAtBM,IAAInI,2BAAkB6H,0BAAAA,kBAAkBP,IAAI,CAACe,yBAAvBR,uBAA6B,CAAC,EAAE,mBAAI,KAAKF;IACjE;IACA,KAAK,MAAMN,SAASY,KAAKK,QAAQ,CAACR,aAAcK,IAAInI,kBAAkBqH,KAAK,CAAC,EAAE,GAAG;IACjF,KAAK,MAAMA,SAASY,KAAKK,QAAQ,CAACP,aAAc;QAC9CI,IAAIjH,MAAM;YAACqH,OAAOlB,KAAK,CAAC,EAAE;YAAGkB,OAAOlB,KAAK,CAAC,EAAE;YAAGkB,OAAOlB,KAAK,CAAC,EAAE;SAAE,GAAG;IACrE;IACA,OAAOa;AACT;AAEA,MAAMM,kBAAkB;AACxB,MAAMC,iBAAiB;AAEvB,4EAA4E,GAC5E,OAAO,SAASC,iBAAiBC,IAAY,EAAEC,OAAe;QAE1CD;IADlB,MAAME,OAAiB,EAAE;IACzB,KAAK,MAAMR,QAAOM,cAAAA,KAAKtB,KAAK,CAACmB,4BAAXG,cAA+B,EAAE,CAAE;YACtCF;QAAb,MAAMK,QAAOL,uBAAAA,eAAenB,IAAI,CAACe,yBAApBI,oBAA0B,CAAC,EAAE;QAC1C,IAAI,CAACK,MAAM;QACX,IAAI;YACF,MAAMvB,MAAM,IAAIC,IAAIsB,MAAMF;YAC1B,IAAIrB,IAAIG,QAAQ,KAAK,YAAY,CAACmB,KAAKE,QAAQ,CAACxB,IAAI3F,QAAQ,KAAKiH,KAAKzF,IAAI,CAACmE,IAAI3F,QAAQ;QACzF,EAAE,eAAM;YACN;QACF;QACA,IAAIiH,KAAK5F,MAAM,KAAKpC,gCAAgC;IACtD;IACA,OAAOgI;AACT;AAEA,kEAAkE,GAClE,OAAO,SAASG,gBACdd,OAAoC,EACpCtF,QAAQnC,gCAAgC;IAExC,MAAMkC,SAAS;WAAIuF,QAAQe,OAAO;KAAG,CAClCvE,IAAI,CAAC,CAACtC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1Bd,GAAG,CAAC,CAAC,CAACU,IAAI,GAAKD,WAAWC;IAC7B,OAAOU,eAAeC,QAAQC;AAChC;AAEA,yEAAyE,GACzE,SAASsG,SAAShE,MAA+B,EAAEiE,EAAU;IAC3D,MAAMC,aAAa,IAAIC;IACvB,MAAMC,QAAQC,WAAW,IAAMH,WAAWI,KAAK,IAAIL;IACnD,MAAMM,UAAU,IAAML,WAAWI,KAAK;IACtC,IAAItE,0BAAAA,OAAQwE,OAAO,EAAEN,WAAWI,KAAK;IACrCtE,0BAAAA,OAAQyE,gBAAgB,CAAC,SAASF;IAClC,OAAO;QACLvE,QAAQkE,WAAWlE,MAAM;QACzB0E,MAAM;YACJC,aAAaP;YACbpE,0BAAAA,OAAQ4E,mBAAmB,CAAC,SAASL;QACvC;IACF;AACF;AAEA;;;CAGC,GACD,eAAeM,WACbC,IAAgC,EAChC5D,QAAgB,EAChB6D,aAAa,KAAK,EAClBC,SAAS,KAAK;IAEd,MAAMC,SAASH,KAAKI,SAAS;IAC7B,MAAMC,SAAuB,EAAE;IAC/B,IAAIC,QAAQ;IACZ,IAAIC,YAAY;IAChB,OAAS;QACP,MAAM,EAAEX,IAAI,EAAEY,KAAK,EAAE,GAAG,MAAML,OAAOM,IAAI;QACzC,IAAIb,QAAQ,CAACY,OAAO;QACpB,MAAME,OAAOtE,WAAWkE;QACxB,IAAIE,MAAMG,UAAU,GAAGD,MAAM;YAC3B,MAAMP,OAAOS,MAAM,GAAGC,KAAK,CAAC,IAAMC;YAClC,IAAIb,YAAY,OAAO;YACvBI,OAAOjH,IAAI,CAACoH,MAAMO,QAAQ,CAAC,GAAGL;YAC9BJ,SAASI;YACTH,YAAY;YACZ;QACF;QACAF,OAAOjH,IAAI,CAACoH;QACZF,SAASE,MAAMG,UAAU;IAC3B;IACA,MAAMK,SAAS,IAAIC,WAAWX;IAC9B,IAAIY,SAAS;IACb,KAAK,MAAMC,SAASd,OAAQ;QAC1BW,OAAOxG,GAAG,CAAC2G,OAAOD;QAClBA,UAAUC,MAAMR,UAAU;IAC5B;IACA,6EAA6E;IAC7E,IAAIT,QAAQ,OAAO;QAAEjC,MAAM;QAAImD,OAAOd;QAAOC;QAAW1G,MAAMmH;IAAO;IACrE,OAAO;QAAE/C,MAAM,IAAIoD,cAAcC,MAAM,CAACN;QAASI,OAAOd;QAAOC;IAAU;AAC3E;AAoDA,yFAAyF,GACzF,OAAO,SAASgB,kBAAkBC,OAA6B;QAC7CA;IAAhB,MAAMC,SAAS,EAACD,eAAAA,QAAQnH,GAAG,CAAC,qBAAZmH,eAAyB,IAAIE,WAAW;IACxD,IAAIF,QAAQnH,GAAG,CAAC,yBAAyBmH,QAAQnH,GAAG,CAAC,kBAAkBoH,WAAW,UAAU,OAAO;IACnG,IAAID,QAAQnH,GAAG,CAAC,mBAAmBmH,QAAQnH,GAAG,CAAC,aAAaoH,WAAW,cAAc,OAAO;IAC5F,OAAO;AACT;AAEA,SAASE,aACPC,OAA2B,EAC3BrE,GAAW,EACXsE,QAAwE,CAAC,CAAC;IAE1E,OAAO;QAAEC,IAAI;QAAOvE;QAAKU,MAAM;QAAImD,OAAO;QAAGb,WAAW;QAAOwB,aAAa;QAAIH;OAAYC;AAC9F;AAyBA,2GAA2G,GAC3G,SAASG,wBAAwBR,OAAsC;IACrE,MAAMS,MAA8B,CAAC;IACrC,KAAK,MAAM,CAACC,MAAM1B,MAAM,IAAI2B,OAAOlD,OAAO,CAACuC,kBAAAA,UAAW,CAAC,GAAI;QACzD,IAAI,oEAAoEY,IAAI,CAACF,OAAO;QACpF,IAAI,OAAO1B,UAAU,UAAUyB,GAAG,CAACC,KAAK,GAAG1B;IAC7C;IACA,OAAOyB;AACT;AAEA;;;;;;;;CAQC,GACD,OAAO,eAAeI,wBACpB9E,GAAW,EACX+E,OAA2B;IAE3B,MAAM,EAAEC,sBAAsB,EAAEC,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC;IACjE,IAAIC,UAAUlF;IACd,IAAK,IAAImF,MAAM,GAAGA,OAAO3L,mBAAmB2L,OAAO,EAAG;YAgBbJ;QAfvC,IAAIK;QACJ,IAAI;YACFA,SAAS,IAAInF,IAAIiF;QACnB,EAAE,eAAM;YACN,OAAOd,aAAa,eAAec;QACrC;QACA,IAAIE,OAAOjF,QAAQ,KAAK,YAAYiF,OAAOC,QAAQ,IAAID,OAAOE,QAAQ,EAAE;YACtE,OAAOlB,aAAa,eAAec;QACrC;QACA,IAAIE,OAAOG,IAAI,IAAIH,OAAOG,IAAI,KAAK,OAAO;YACxC,OAAOnB,aAAa,eAAec;QACrC;QACA,MAAMM,SAAS,MAAMP,gBAAgBG,OAAOK,QAAQ;QACpD,IAAI,CAACD,QAAQ,OAAOpB,aAAa,mBAAmBgB,OAAO/K,QAAQ;QACnE,MAAMqL,aAAaV,uBAAuBQ;QAC1C,MAAMG,QAAQhE,SAASoD,QAAQpH,MAAM,GAAEoH,qBAAAA,QAAQa,SAAS,YAAjBb,qBAAqBxL;QAC5D,IAAI;gBAUUwL,uBA4BQc,uBAKIA;YA1CxB,sEAAsE;YACtE,kEAAkE;YAClE,MAAMC,OAAO;gBACXC,QAAQ;gBACRC,UAAU;gBACVrI,QAAQgI,MAAMhI,MAAM;gBACpB+H;gBACAzB,SAAS,aACJQ,wBAAwBM,QAAQd,OAAO;oBAC1CgC,MAAM,GAAElB,wBAAAA,QAAQmB,YAAY,YAApBnB,wBAAwB;mBAC5BA,QAAQoB,SAAS,GAAG;oBAAE,cAAcpB,QAAQoB,SAAS;gBAAC,IAAI,CAAC;YAEnE;YACA,MAAMN,WAAW,MAAMO,MAAMhB,OAAO/K,QAAQ,IAAIyL;YAChD,IAAIrM,kBAAkB4M,GAAG,CAACR,SAASS,MAAM,GAAG;oBACpCT;gBAAN,QAAMA,iBAAAA,SAASpD,IAAI,qBAAboD,eAAexC,MAAM,GAAGC,KAAK,CAAC,IAAMC;gBAC1C,MAAM9F,WAAWoI,SAAS5B,OAAO,CAACnH,GAAG,CAAC;gBACtC,IAAI,CAACW,UAAU;oBACb,OAAO2G,aAAa,eAAegB,OAAO/K,QAAQ,IAAI;wBAAEiM,QAAQT,SAASS,MAAM;oBAAC;gBAClF;gBACA,IAAIC;gBACJ,IAAI;oBACFA,OAAO,IAAItG,IAAIxC,UAAU2H;gBAC3B,EAAE,eAAM;oBACN,OAAOhB,aAAa,oBAAoBgB,OAAO/K,QAAQ,IAAI;wBAAEoD;oBAAS;gBACxE;gBACA,IAAIsH,QAAQyB,eAAe,IAAI,CAACzB,QAAQyB,eAAe,CAACpB,QAAQmB,OAAO;oBACrE,OAAOnC,aAAa,oBAAoBgB,OAAO/K,QAAQ,IAAI;wBAAEoD,UAAU8I,KAAKlM,QAAQ;oBAAG;gBACzF;gBACA6K,UAAUqB,KAAKlM,QAAQ;gBACvB;YACF;YACA,IAAI,CAACwL,SAAStB,EAAE,IAAI,CAACsB,SAASpD,IAAI,EAAE;oBAC5BoD;gBAAN,QAAMA,kBAAAA,SAASpD,IAAI,qBAAboD,gBAAexC,MAAM,GAAGC,KAAK,CAAC,IAAMC;gBAC1C,MAAMnG,OAAO4G,kBAAkB6B,SAAS5B,OAAO;gBAC/C,OAAOG,aAAa,eAAegB,OAAO/K,QAAQ,IAAI;oBAAEiM,QAAQT,SAASS,MAAM;mBAAMlJ,OAAO;oBAAEA;gBAAK,IAAI,CAAC;YAC1G;YACA,MAAMoH,eAAcqB,wBAAAA,SAAS5B,OAAO,CAACnH,GAAG,CAAC,2BAArB+I,wBAAwC;YAC5D,IAAI,CAACd,QAAQ0B,MAAM,CAAC5B,IAAI,CAACL,cAAc;gBACrC,MAAMqB,SAASpD,IAAI,CAACY,MAAM,GAAGC,KAAK,CAAC,IAAMC;gBACzC,OAAOa,aAAa,gBAAgBgB,OAAO/K,QAAQ,IAAI;oBAAEiM,QAAQT,SAASS,MAAM;gBAAC;YACnF;YACA,MAAMI,WAAW1F,QAAO6E,yBAAAA,SAAS5B,OAAO,CAACnH,GAAG,CAAC,6BAArB+I,yBAA0C;YAClE,IAAId,QAAQrC,UAAU,IAAI1B,OAAO2F,QAAQ,CAACD,aAAaA,WAAW3B,QAAQlG,QAAQ,EAAE;gBAClF,MAAMgH,SAASpD,IAAI,CAACY,MAAM,GAAGC,KAAK,CAAC,IAAMC;gBACzC,OAAOa,aAAa,aAAagB,OAAO/K,QAAQ,IAAI;oBAAEiM,QAAQT,SAASS,MAAM;gBAAC;YAChF;YACA,MAAM7D,OAAO,MAAMD,WAAWqD,SAASpD,IAAI,EAAEsC,QAAQlG,QAAQ,EAAEkG,QAAQrC,UAAU,EAAEqC,QAAQpC,MAAM,KAAK;YACtG,IAAI,CAACF,MAAM;gBACT,OAAO2B,aAAa,aAAagB,OAAO/K,QAAQ,IAAI;oBAAEiM,QAAQT,SAASS,MAAM;gBAAC;YAChF;YACA,OAAO;gBAAE/B,IAAI;gBAAMvE,KAAKoF,OAAO/K,QAAQ;gBAAImK;gBAAaH,SAAS;eAAS5B;QAC5E,EAAE,eAAM;YACN,OAAO2B,aAAa,WAAWgB,OAAO/K,QAAQ;QAChD,SAAU;YACRsL,MAAMtD,IAAI;YACV,MAAMqD,WAAWkB,KAAK,GAAGtD,KAAK,CAAC,IAAMC;QACvC;IACF;IACA,OAAOa,aAAa,sBAAsBc;AAC5C;AAEA,+EAA+E,GAC/E,OAAO,eAAe2B,gBACpB7G,GAAW,EACX+E,OAAgI;IAEhI,MAAM+B,SAAS,MAAMhC,wBAAwB9E,KAAK+E;IAClD,OAAO+B,OAAOvC,EAAE,GAAG;QAAE7D,MAAMoG,OAAOpG,IAAI;QAAEV,KAAK8G,OAAO9G,GAAG;IAAC,IAAI;AAC9D;AAeA;;;;;CAKC,GACD,OAAO,eAAe+G,kBACpB/G,GAAQ,EACRrC,MAAmB,EACnBsG,OAAuC;IAEvC,MAAM+C,OAAO,MAAMH,gBAAgB7G,IAAI3F,QAAQ,IAAI;QACjDwE,UAAUzF;QACVqN,QAAQ;QACR9I;OACIsG,UAAU;QAAEA;IAAQ,IAAI,CAAC;IAE/B,IAAI,CAAC+C,MAAM,OAAO;IAClB,MAAMC,OAAO,IAAIhH,IAAI+G,KAAKhH,GAAG,EAAEiH,IAAI;IACnC,MAAMC,cAAwB,EAAE;IAChC,KAAK,MAAMC,SAAShG,iBAAiB6F,KAAKtG,IAAI,EAAEsG,KAAKhH,GAAG,EAAG;QACzD,MAAMoH,WAAW,AAAC,CAAA;YAChB,IAAI;gBACF,OAAO,IAAInH,IAAIkH,OAAOF,IAAI,KAAKA;YACjC,EAAE,eAAM;gBACN,OAAO;YACT;QACF,CAAA;QACA,MAAMI,MAAM,MAAMR,gBAAgBM,OAAO;YACvCtI,UAAUxF;YACVoN,QAAQ;YACR9I;WACIsG,WAAWmD,WAAW;YAAEnD;QAAQ,IAAI,CAAC;QAE3C,IAAIoD,KAAKH,YAAYrL,IAAI,CAACwL,IAAI3G,IAAI;IACpC;IACA,OAAO;QAAEV,KAAKgH,KAAKhH,GAAG;QAAEoB,MAAM4F,KAAKtG,IAAI;QAAEwG;IAAY;AACvD;AAEA,oFAAoF,GACpF,OAAO,SAASI,oBAAoBN,IAAmD;IACrF,MAAMrG,UAAUF,iBAAiBuG,KAAK5F,IAAI;IAC1C,KAAK,MAAMiG,OAAOL,KAAKE,WAAW,CAAE;QAClC,KAAK,MAAM,CAACzM,KAAKoG,OAAO,IAAIJ,iBAAiB4G,KAAM;gBAC/B1G;YAAlBA,QAAQ1D,GAAG,CAACxC,KAAK,EAACkG,eAAAA,QAAQ7D,GAAG,CAACrC,gBAAZkG,eAAoB,KAAKE;QAC7C;IACF;IACA,OAAOY,gBAAgBd;AACzB;AAEA,uEAAuE,GACvE,eAAe4G,oBAAoBvH,GAAQ,EAAErC,MAAmB;IAC9D,MAAMqJ,OAAO,MAAMD,kBAAkB/G,KAAKrC;IAC1C,OAAOqJ,OAAOM,oBAAoBN,QAAQ;AAC5C;AAkBA,OAAO,eAAeQ,yBACpBC,KAQC,EACDC,QAA2B,CAAC,CAAC;IAE7B,MAAM/I,SAA8B;WAAI7C,wBAAwB2L,MAAM1L,GAAG;KAAE;IAC3E,MAAM4L,QAAkB,EAAE;IAC1B,MAAMhC,QAAQhE,SAAS8F,MAAM9J,MAAM,EAAEjF;IACrC,MAAMkI,MAAM,CAACgH,OAA0BzL;QACrC,KAAK,MAAM1B,OAAOmN,MAAMjN,KAAK,CAAC,GAAGzB,kCAAmC;YAClE,IAAI,CAACyF,OAAOhD,IAAI,CAAC,CAACH,QAAUA,MAAMf,GAAG,KAAKA,MAAMkE,OAAO9C,IAAI,CAAC;gBAAEpB;gBAAK0B;YAAO;QAC5E;IACF;IACA,IAAI;YAIOsL;QAHT,MAAMhK,WAAWrB,kBACfqL,MAAMR,IAAI,CAAC,UAAU,EACrBQ,MAAM/J,MAAM,EACZmK,QAAOJ,oBAAAA,MAAMR,IAAI,CAAC,QAAQ,YAAnBQ,oBAAuB;QAEhC,IAAIhK,UAAU;gBACSiK;YAArB,MAAME,QAAQ,MAAM,EAACF,oBAAAA,MAAMI,UAAU,YAAhBJ,oBAAoBnK,gBACvCkK,MAAMjK,SAAS,EACfC,UACAgK,MAAM/J,MAAM,EACZiI,MAAMhI,MAAM,EACZ2F,KAAK,CAAC,IAAM;YACd,IAAIsE,OAAOhH,IAAIgH,OAAO;iBACjBD,MAAM9L,IAAI,CAAC;QAClB;QACA,MAAM5C,YAAY2G,eAAe6H,MAAM5H,KAAK;QAC5C,IAAI5G,WAAW;gBAGFyO;YAFX,MAAME,QAAQjC,MAAMhI,MAAM,CAACwE,OAAO,GAC9B,OACA,MAAM,EAACuF,yBAAAA,MAAMK,eAAe,YAArBL,yBAAyBH,qBAAqBtO,WAAW0M,MAAMhI,MAAM,EAAE2F,KAAK,CACjF,IAAM;YAEZ,IAAIsE,yBAAAA,MAAOlM,MAAM,EAAEkF,IAAIgH,OAAO;iBACzB;gBACHD,MAAM9L,IAAI,CACR,CAAC,YAAY,EAAE5C,UAAUwM,QAAQ,CAAC,qEAAqE,CAAC;YAE5G;QACF;IACF,SAAU;QACRE,MAAMtD,IAAI;IACZ;IACA,OAAO;QAAE1D;QAAQgJ;IAAM;AACzB"}
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/jobs/ai-theme-brand-inputs.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { checkEntitlement } from '@aglyn/aglyn/app-utils/plan-entitlements'\nimport type {\n  AglynOrgBilling,\n  OrgBrandingProfile,\n} from '@aglyn/aglyn/foundation/definitions/org-billing.types'\nimport { normalizeThemeHex } from '../tools/ai-theme-tool'\nimport { AI_THEME_BRAND_BUDGET_MS } from './ai-job-theme-budget'\nimport {\n  aiMediaAssetLocation,\n  loadAiMediaSharp,\n  readAiMediaAssetBytes,\n  type AiMediaAssetLocation,\n} from './ai-media-asset'\n\n/**\n * The brand a theme job builds on (AGL-2938): the colors that belong to the\n * site before anyone describes them.\n *\n * - **The workspace's brand color** — a white-label workspace's, and only\n *   theirs: without the entitlement the resolved profile is the platform's,\n *   which is no customer's brand.\n * - **The site logo** — when the logo is an asset of the site's own media\n *   library or its org's, read from storage and reduced to its dominant\n *   colors. Never fetched from a URL: a logo that is not a library asset is\n *   left alone.\n * - **A page the brief links to** — fetched through the same SSRF guard the\n *   plugin fetch proxy uses (a public address, pinned; https only; every\n *   redirect re-validated), capped in bytes and time, and reduced to the\n *   colors its markup and stylesheets use.\n *\n * Only hex colors leave this module. Nothing a page or a logo SAYS reaches a\n * prompt, so there is nothing in them to follow. Each source is best effort:\n * one that cannot be read is named in a note and the proposal is made\n * without it.\n */\n\nexport type AiThemeBrandSource = 'organization' | 'logo' | 'reference'\n\n/** How the prompt names where each brand color came from. */\nexport const AI_THEME_BRAND_SOURCE_WORDS: Record<AiThemeBrandSource, string> = {\n  organization: 'the workspace brand color',\n  logo: 'from the site logo',\n  reference: 'from the page the brief links to',\n}\n\nexport interface AiThemeBrandColor {\n  hex: string\n  source: AiThemeBrandSource\n}\n\nexport interface AiThemeBrandInputs {\n  colors: AiThemeBrandColor[]\n  /** A source that could not be read, in customer-safe words. */\n  notes: string[]\n}\n\n/** The most colors one source contributes. */\nexport const AI_THEME_BRAND_COLORS_PER_SOURCE = 4\n\n/** The largest logo decoded for its colors. */\nexport const AI_THEME_LOGO_MAX_BYTES = 5 * 1024 * 1024\n\n/** The most of a linked page, and of each of its stylesheets, that is read. */\nexport const AI_THEME_REFERENCE_MAX_BYTES = 512 * 1024\nexport const AI_THEME_STYLESHEET_MAX_BYTES = 256 * 1024\n\n/** The stylesheets of a linked page that are read beside it. */\nexport const AI_THEME_REFERENCE_STYLESHEETS = 2\n\n/**\n * The wall clock brand colors may take, declared with the theme step's time\n * (`ai-job-theme-budget.ts`), which counts it (AGL-3035).\n */\nexport { AI_THEME_BRAND_BUDGET_MS }\n\nconst FETCH_TIMEOUT_MS = 4_000\nconst MAX_REDIRECT_HOPS = 3\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])\n\ntype Rgb = [red: number, green: number, blue: number]\n\nconst toHex = ([red, green, blue]: Rgb) =>\n  `#${[red, green, blue]\n    .map((channel) => Math.round(Math.min(255, Math.max(0, channel))).toString(16).padStart(2, '0'))\n    .join('')}`\n\nconst channelsOf = (hex: string): Rgb => [\n  parseInt(hex.slice(1, 3), 16),\n  parseInt(hex.slice(3, 5), 16),\n  parseInt(hex.slice(5, 7), 16),\n]\n\nconst distance = (a: Rgb, b: Rgb) => Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2])\n\n/** How far a color is from grey: the spread of its channels. */\nconst chroma = ([red, green, blue]: Rgb) => Math.max(red, green, blue) - Math.min(red, green, blue)\n\n/** Colors closer than this read as the same color in a palette. */\nconst SAME_COLOR_DISTANCE = 40\n\n/** A spread under this reads as a grey, which says little about a brand. */\nconst GREY_CHROMA = 38\n\n/** The distinct colors of a ranked list, colorful ones first when there are any. */\nfunction distinctColors(ranked: readonly Rgb[], count: number): string[] {\n  const colorful = ranked.filter((color) => chroma(color) > GREY_CHROMA)\n  const chosen: Rgb[] = []\n  for (const color of colorful.length ? colorful : ranked) {\n    if (chosen.some((other) => distance(other, color) < SAME_COLOR_DISTANCE)) continue\n    chosen.push(color)\n    if (chosen.length === count) break\n  }\n  return chosen.map(toHex)\n}\n\n/* ------------------------------------------------------------------------ *\n * The workspace's brand color\n * ------------------------------------------------------------------------ */\n\n/** A white-label workspace's own brand color; nothing for any other workspace. */\nexport function organizationBrandColors(\n  org: Partial<AglynOrgBilling> | null | undefined,\n): AiThemeBrandColor[] {\n  if (!checkEntitlement(org, 'whiteLabel')) return []\n  const raw = (org?.brandingProfile as OrgBrandingProfile | undefined)?.primaryColor\n  const hex = typeof raw === 'string' ? normalizeThemeHex(raw) : null\n  return hex ? [{ hex, source: 'organization' }] : []\n}\n\n/* ------------------------------------------------------------------------ *\n * The site logo\n * ------------------------------------------------------------------------ */\n\nexport type AiThemeLogoLocation = AiMediaAssetLocation\n\n/**\n * Where a site's logo lives in a media library: a media reference or a CDN\n * path naming this site's library, or its org's. Anything else — an external\n * URL, an asset of some other site or org — has no location, and the logo is\n * left alone. The media library's own rule, shared with every job that reads\n * an asset (`ai-media-asset.ts`).\n */\nexport const logoMediaLocation = aiMediaAssetLocation\n\n/**\n * The dominant colors of an image's pixels: transparent pixels skipped, the\n * rest grouped into coarse buckets, the busiest distinct buckets first, and\n * the near-white and near-black edges a logo sits on dropped while anything\n * else is there.\n */\nexport function dominantColorsFromPixels(\n  data: ArrayLike<number>,\n  channels: number,\n  count = AI_THEME_BRAND_COLORS_PER_SOURCE,\n): string[] {\n  const buckets = new Map<number, { sum: Rgb; pixels: number }>()\n  for (let index = 0; index + channels <= data.length; index += channels) {\n    if (channels >= 4 && data[index + 3] < 128) continue\n    const pixel: Rgb = [data[index], data[index + 1], data[index + 2]]\n    const key = ((pixel[0] >> 4) << 8) | ((pixel[1] >> 4) << 4) | (pixel[2] >> 4)\n    const bucket = buckets.get(key) ?? { sum: [0, 0, 0] as Rgb, pixels: 0 }\n    bucket.sum = [bucket.sum[0] + pixel[0], bucket.sum[1] + pixel[1], bucket.sum[2] + pixel[2]]\n    bucket.pixels += 1\n    buckets.set(key, bucket)\n  }\n  const ranked = [...buckets.values()]\n    .sort((a, b) => b.pixels - a.pixels)\n    .map(({ sum, pixels }): Rgb => [sum[0] / pixels, sum[1] / pixels, sum[2] / pixels])\n  const edge = (color: Rgb) => Math.min(...color) > 235 || Math.max(...color) < 20\n  const inside = ranked.filter((color) => !edge(color))\n  return distinctColors(inside.length ? inside : ranked, count)\n}\n\n/** The part of `sharp` the logo read uses. */\ninterface SharpPixelPipeline {\n  resize(width: number, height: number, options: { fit: 'inside' }): SharpPixelPipeline\n  ensureAlpha(): SharpPixelPipeline\n  raw(): SharpPixelPipeline\n  toBuffer(options: { resolveWithObject: true }): Promise<{\n    data: Buffer\n    info: { channels: number }\n  }>\n}\ntype SharpPixels = (input: Buffer, options?: { limitInputPixels?: number }) => SharpPixelPipeline\n\nconst LOGO_TYPES = /^image\\/(png|jpeg|webp|gif|avif|svg\\+xml)$/\n\n/**\n * A library logo's colors, read the way the media CDN would serve it\n * (`readAiMediaAssetBytes`). `null` when the asset may not be read or the bytes\n * cannot be decoded.\n *\n * On Vercel the read and the decode run in the plugins' image function, the\n * one function that ships `sharp`'s native library (`ai-image-reads.ts`,\n * AGL-4048), inside the brand reads' own bound (`signal`); anywhere else they\n * run here. Either way an answer that does not come is a logo whose colors\n * could not be read.\n */\nasync function readLogoColors(\n  firestore: FirebaseFirestore.Firestore,\n  location: AiThemeLogoLocation,\n  hostId: string,\n  signal?: AbortSignal,\n): Promise<string[] | null> {\n  const images = await import('./ai-image-reads')\n  const origin = images.aiImageReadsOrigin()\n  if (!origin) return readAiThemeLogoColorsAt(firestore, location, hostId)\n  // The location was held to the site's org already: an org asset's scope\n  // IS that org, and a site's own asset needs none.\n  const orgId = location.scope.isOrg ? location.scope.scopeId : ''\n  const payload = images.aiImageReadsRequest('logo-colors', location, hostId, orgId)\n  if (!payload) return null\n  const answer = await images.requestAiImageReads<{ colors?: unknown }>(\n    origin,\n    payload,\n    signal ?? AbortSignal.timeout(AI_THEME_BRAND_BUDGET_MS),\n  )\n  return logoColorsAnswer(answer?.colors)\n}\n\n/**\n * A library logo's colors read and decoded in THIS process, from a location\n * already held to the media library's rule: what `readLogoColors` does off\n * Vercel, and what the images function does for it on Vercel.\n */\nexport async function readAiThemeLogoColorsAt(\n  firestore: FirebaseFirestore.Firestore,\n  location: AiThemeLogoLocation,\n  hostId: string,\n): Promise<string[] | null> {\n  const asset = await readAiMediaAssetBytes(firestore, location, hostId, {\n    maxBytes: AI_THEME_LOGO_MAX_BYTES,\n    types: LOGO_TYPES,\n  })\n  if (!asset) return null\n  const sharp = (await loadAiMediaSharp()) as SharpPixels\n  const { data, info } = await sharp(asset.buffer, { limitInputPixels: 64_000_000 })\n    .resize(48, 48, { fit: 'inside' })\n    .ensureAlpha()\n    .raw()\n    .toBuffer({ resolveWithObject: true })\n  return dominantColorsFromPixels(data, info.channels)\n}\n\n/** What the images function answered, as hex colors: only colors leave it. */\nfunction logoColorsAnswer(colors: unknown): string[] | null {\n  if (!Array.isArray(colors)) return null\n  return colors\n    .map((color) => (typeof color === 'string' ? normalizeThemeHex(color) : null))\n    .filter((hex): hex is string => Boolean(hex))\n    .slice(0, AI_THEME_BRAND_COLORS_PER_SOURCE)\n}\n\n/* ------------------------------------------------------------------------ *\n * A page the brief links to\n * ------------------------------------------------------------------------ */\n\nconst URL_IN_BRIEF = /\\bhttps?:\\/\\/[^\\s<>\"'`)\\]]+/i\n\n/** The first https link in a brief, or `null` — an http link is not followed. */\nexport function referenceUrlOf(brief: string): URL | null {\n  const match = URL_IN_BRIEF.exec(brief)\n  if (!match) return null\n  try {\n    const url = new URL(match[0].replace(/[.,;:!?]+$/, ''))\n    return url.protocol === 'https:' ? url : null\n  } catch {\n    return null\n  }\n}\n\n/** How much a page's declared `theme-color` counts beside one use of a color. */\nexport const AI_THEME_THEME_COLOR_WEIGHT = 25\n\nconst THEME_COLOR_META = /<meta\\b[^>]*\\bname\\s*=\\s*[\"']?theme-color\\b[^>]*>/gi\nconst CONTENT_ATTRIBUTE = /\\bcontent\\s*=\\s*[\"']?\\s*([^\"'\\s>]+)/i\n// A `#` after `&` is a character reference (`&#039;`), not a color.\nconst HEX_LITERAL = /(?<!&)#(?:[0-9a-f]{6}|[0-9a-f]{3})(?![0-9a-z])/gi\nconst RGB_LITERAL = /rgba?\\(\\s*(\\d{1,3})\\s*[,\\s]\\s*(\\d{1,3})\\s*[,\\s]\\s*(\\d{1,3})/gi\n\n/** Every color a page's markup or a stylesheet uses, weighted by how often. */\nexport function colorsFromMarkup(text: string): Map<string, number> {\n  const weights = new Map<string, number>()\n  const add = (hex: string | null, weight: number) => {\n    if (hex) weights.set(hex, (weights.get(hex) ?? 0) + weight)\n  }\n  for (const tag of text.match(THEME_COLOR_META) ?? []) {\n    add(normalizeThemeHex(CONTENT_ATTRIBUTE.exec(tag)?.[1] ?? ''), AI_THEME_THEME_COLOR_WEIGHT)\n  }\n  for (const match of text.matchAll(HEX_LITERAL)) add(normalizeThemeHex(match[0]), 1)\n  for (const match of text.matchAll(RGB_LITERAL)) {\n    add(toHex([Number(match[1]), Number(match[2]), Number(match[3])]), 1)\n  }\n  return weights\n}\n\nconst STYLESHEET_LINK = /<link\\b[^>]*\\brel\\s*=\\s*[\"']?stylesheet\\b[^>]*>/gi\nconst HREF_ATTRIBUTE = /\\bhref\\s*=\\s*[\"']?\\s*([^\"'\\s>]+)/i\n\n/** The https stylesheets a page links, resolved against it, at most a few. */\nexport function stylesheetUrlsOf(html: string, pageUrl: string): string[] {\n  const urls: string[] = []\n  for (const tag of html.match(STYLESHEET_LINK) ?? []) {\n    const href = HREF_ATTRIBUTE.exec(tag)?.[1]\n    if (!href) continue\n    try {\n      const url = new URL(href, pageUrl)\n      if (url.protocol === 'https:' && !urls.includes(url.toString())) urls.push(url.toString())\n    } catch {\n      continue\n    }\n    if (urls.length === AI_THEME_REFERENCE_STYLESHEETS) break\n  }\n  return urls\n}\n\n/** The most used distinct colors, the colorful before the greys. */\nexport function rankBrandColors(\n  weights: ReadonlyMap<string, number>,\n  count = AI_THEME_BRAND_COLORS_PER_SOURCE,\n): string[] {\n  const ranked = [...weights.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .map(([hex]) => channelsOf(hex))\n  return distinctColors(ranked, count)\n}\n\n/** One abort for the caller's signal and a clock, whichever ends first. */\nfunction deadline(signal: AbortSignal | undefined, ms: number) {\n  const controller = new AbortController()\n  const timer = setTimeout(() => controller.abort(), ms)\n  const onAbort = () => controller.abort()\n  if (signal?.aborted) controller.abort()\n  signal?.addEventListener('abort', onAbort)\n  return {\n    signal: controller.signal,\n    done: () => {\n      clearTimeout(timer)\n      signal?.removeEventListener('abort', onAbort)\n    },\n  }\n}\n\n/**\n * The body as text, to `maxBytes`. Past the cap it is cut there, or with\n * `refuseOver` it is not read at all: `null`, the read canceled.\n */\nasync function readCapped(\n  body: ReadableStream<Uint8Array>,\n  maxBytes: number,\n  refuseOver = false,\n  binary = false,\n): Promise<{ text: string; bytes: number; truncated: boolean; data?: Uint8Array } | null> {\n  const reader = body.getReader()\n  const chunks: Uint8Array[] = []\n  let total = 0\n  let truncated = false\n  for (;;) {\n    const { done, value } = await reader.read()\n    if (done || !value) break\n    const room = maxBytes - total\n    if (value.byteLength > room) {\n      await reader.cancel().catch(() => undefined)\n      if (refuseOver) return null\n      chunks.push(value.subarray(0, room))\n      total += room\n      truncated = true\n      break\n    }\n    chunks.push(value)\n    total += value.byteLength\n  }\n  const joined = new Uint8Array(total)\n  let offset = 0\n  for (const chunk of chunks) {\n    joined.set(chunk, offset)\n    offset += chunk.byteLength\n  }\n  // A picture's bytes are kept as they came (AGL-3976), never decoded as text.\n  if (binary) return { text: '', bytes: total, truncated, data: joined }\n  return { text: new TextDecoder().decode(joined), bytes: total, truncated }\n}\n\n/** Why {@link fetchPublicTextDetailed} read nothing. */\nexport type PublicFetchRefusal =\n  /** Not https, a port, credentials, or not a URL at all. */\n  | 'refused-url'\n  /** The name resolves to a private address, or not at all. */\n  | 'private-address'\n  /** A redirect the caller's `redirectAllowed` refused. */\n  | 'redirect-refused'\n  | 'too-many-redirects'\n  /** An answer that was not 2xx; `status` says which. */\n  | 'http-status'\n  /** A 2xx whose content type the caller does not accept. */\n  | 'content-type'\n  /** Past `maxBytes` with `refuseOver` set. */\n  | 'too-large'\n  /** The socket, the clock or the caller's signal ended it. */\n  | 'network'\n\n/**\n * One shape for both outcomes, read by `ok`: a union does not narrow across\n * this repo's library boundaries (`strictNullChecks` is off), so a caller\n * could not reach either arm's fields.\n */\nexport interface PublicFetchResult {\n  ok: boolean\n  /** The address read, after redirects; or the one that was refused. */\n  url: string\n  /** Empty unless `ok`. */\n  text: string\n  bytes: number\n  /** Cut at `maxBytes` (never with `refuseOver`). */\n  truncated: boolean\n  contentType: string\n  /** Null when `ok`. */\n  refusal: PublicFetchRefusal | null\n  status?: number\n  /** Where a refused redirect pointed. */\n  location?: string\n  /** The body's bytes, with `binary`; absent otherwise. */\n  data?: Uint8Array\n  /**\n   * Whose edge refused an error answer, when its headers say (AGL-4083): the\n   * bot protection a site owner opens for AglynBot is that edge's setting.\n   */\n  edge?: PublicFetchEdge\n}\n\n/** An edge network that names itself on the answers it refuses with. */\nexport type PublicFetchEdge = 'vercel' | 'cloudflare'\n\n/** The edge an answer came through, from the headers each one sets; `null` for neither. */\nexport function publicFetchEdgeOf(headers: Pick<Headers, 'get'>): PublicFetchEdge | null {\n  const server = (headers.get('server') ?? '').toLowerCase()\n  if (headers.get('x-vercel-mitigated') || headers.get('x-vercel-id') || server === 'vercel') return 'vercel'\n  if (headers.get('cf-mitigated') || headers.get('cf-ray') || server === 'cloudflare') return 'cloudflare'\n  return null\n}\n\nfunction refusedFetch(\n  refusal: PublicFetchRefusal,\n  url: string,\n  extra: { status?: number; location?: string; edge?: PublicFetchEdge } = {},\n): PublicFetchResult {\n  return { ok: false, url, text: '', bytes: 0, truncated: false, contentType: '', refusal, ...extra }\n}\n\nexport interface PublicFetchOptions {\n  maxBytes: number\n  accept: RegExp\n  signal?: AbortSignal\n  timeoutMs?: number\n  /** Sent as `User-Agent`; the runtime's own when absent. */\n  userAgent?: string\n  /** Sent as `Accept`. */\n  acceptHeader?: string\n  /**\n   * More request headers for one hop, asked again for EVERY hop's own address\n   * (AGL-4083, AGL-4085), so a header meant for one host never rides a\n   * redirect to another. Sent under `Accept` and `User-Agent`, and neither\n   * can be changed through here. The site import's edge header goes this\n   * way, and only to the hosts `siteImportEdgeHeadersFor` decided are ours.\n   */\n  headersFor?: (url: URL) => Readonly<Record<string, string>> | undefined\n  /** Asked of every redirect hop before it is followed; the SSRF checks run as well. */\n  redirectAllowed?: (from: URL, to: URL) => boolean\n  /** Refuse a body over `maxBytes` rather than cut it. */\n  refuseOver?: boolean\n  /** Keep the body's bytes as `data` rather than decode them as text (a picture, AGL-3976). */\n  binary?: boolean\n}\n\n/** The caller's extra headers, without any that would say what the request is: those are the fetch's own. */\nfunction publicFetchExtraHeaders(headers: Readonly<Record<string, string>> | undefined): Record<string, string> {\n  const out: Record<string, string> = {}\n  for (const [name, value] of Object.entries(headers ?? {})) {\n    if (/^(?:accept|user-agent|host|cookie|authorization|content-length)$/i.test(name)) continue\n    if (typeof value === 'string') out[name] = value\n  }\n  return out\n}\n\n/**\n * A public https resource as text, through the plugin fetch proxy's SSRF\n * guard: the host must resolve only to public addresses and the socket is\n * pinned to the address checked; no credentials in the URL and no port but\n * the default; every redirect re-enters the same checks and the caller's own\n * `redirectAllowed`; the body is read to a byte cap under a clock, and only a\n * content type the caller accepts is read at all. Says why when it reads\n * nothing.\n */\nexport async function fetchPublicTextDetailed(\n  url: string,\n  options: PublicFetchOptions,\n): Promise<PublicFetchResult> {\n  const { createPinnedDispatcher, resolvePublicIp } = await import('./ai-theme-brand-server')\n  let current = url\n  for (let hop = 0; hop <= MAX_REDIRECT_HOPS; hop += 1) {\n    let parsed: URL\n    try {\n      parsed = new URL(current)\n    } catch {\n      return refusedFetch('refused-url', current)\n    }\n    if (parsed.protocol !== 'https:' || parsed.username || parsed.password) {\n      return refusedFetch('refused-url', current)\n    }\n    if (parsed.port && parsed.port !== '443') {\n      return refusedFetch('refused-url', current)\n    }\n    const pinned = await resolvePublicIp(parsed.hostname)\n    if (!pinned) return refusedFetch('private-address', parsed.toString())\n    const dispatcher = createPinnedDispatcher(pinned)\n    const clock = deadline(options.signal, options.timeoutMs ?? FETCH_TIMEOUT_MS)\n    try {\n      // `dispatcher` is undici's, which the global fetch honors; a variable\n      // rather than a literal keeps the DOM `RequestInit` check off it.\n      const init = {\n        method: 'GET',\n        redirect: 'manual' as const,\n        signal: clock.signal,\n        dispatcher,\n        headers: {\n          ...publicFetchExtraHeaders(options.headersFor?.(parsed)),\n          Accept: options.acceptHeader ?? 'text/html, text/css;q=0.9',\n          ...(options.userAgent ? { 'User-Agent': options.userAgent } : {}),\n        },\n      }\n      const response = await fetch(parsed.toString(), init as RequestInit)\n      if (REDIRECT_STATUSES.has(response.status)) {\n        await response.body?.cancel().catch(() => undefined)\n        const location = response.headers.get('location')\n        if (!location) {\n          return refusedFetch('http-status', parsed.toString(), { status: response.status })\n        }\n        let next: URL\n        try {\n          next = new URL(location, parsed)\n        } catch {\n          return refusedFetch('redirect-refused', parsed.toString(), { location })\n        }\n        if (options.redirectAllowed && !options.redirectAllowed(parsed, next)) {\n          return refusedFetch('redirect-refused', parsed.toString(), { location: next.toString() })\n        }\n        current = next.toString()\n        continue\n      }\n      if (!response.ok || !response.body) {\n        await response.body?.cancel().catch(() => undefined)\n        const edge = publicFetchEdgeOf(response.headers)\n        return refusedFetch('http-status', parsed.toString(), { status: response.status, ...(edge ? { edge } : {}) })\n      }\n      const contentType = response.headers.get('content-type') ?? ''\n      if (!options.accept.test(contentType)) {\n        await response.body.cancel().catch(() => undefined)\n        return refusedFetch('content-type', parsed.toString(), { status: response.status })\n      }\n      const declared = Number(response.headers.get('content-length') ?? '')\n      if (options.refuseOver && Number.isFinite(declared) && declared > options.maxBytes) {\n        await response.body.cancel().catch(() => undefined)\n        return refusedFetch('too-large', parsed.toString(), { status: response.status })\n      }\n      const body = await readCapped(response.body, options.maxBytes, options.refuseOver, options.binary === true)\n      if (!body) {\n        return refusedFetch('too-large', parsed.toString(), { status: response.status })\n      }\n      return { ok: true, url: parsed.toString(), contentType, refusal: null, ...body }\n    } catch {\n      return refusedFetch('network', parsed.toString())\n    } finally {\n      clock.done()\n      await dispatcher.close().catch(() => undefined)\n    }\n  }\n  return refusedFetch('too-many-redirects', current)\n}\n\n/** {@link fetchPublicTextDetailed}, answering `null` for anything it refused. */\nexport async function fetchPublicText(\n  url: string,\n  options: { maxBytes: number; accept: RegExp; signal?: AbortSignal; timeoutMs?: number; headersFor?: PublicFetchOptions['headersFor'] },\n): Promise<{ text: string; url: string } | null> {\n  const result = await fetchPublicTextDetailed(url, options)\n  return result.ok ? { text: result.text, url: result.url } : null\n}\n\n/**\n * A public page as it was read: its markup and the text of its first\n * stylesheets ({@link AI_THEME_REFERENCE_STYLESHEETS}), each under its byte\n * cap. What a theme job takes its colors from and a site start its\n * inspiration signals (AGL-3975); never a crawl, one page and its sheets.\n */\nexport interface AiReferencePage {\n  /** The address the markup was finally read at, after redirects. */\n  url: string\n  html: string\n  stylesheets: string[]\n}\n\n/**\n * A page and its first stylesheets through {@link fetchPublicText}; `null` when the page cannot be read.\n * `headersFor` is asked for every request and every redirect hop, the page's\n * and each stylesheet's, by its own address (AGL-4083, AGL-4085): the site\n * import's edge header, built by `siteImportEdgeHeadersFor`, answers only for\n * the site's own origins we serve, so a hop or a stylesheet anywhere else (a\n * CDN's, another site's) carries nothing.\n */\nexport async function readReferencePage(\n  url: URL,\n  signal: AbortSignal,\n  headersFor?: PublicFetchOptions['headersFor'],\n): Promise<AiReferencePage | null> {\n  const page = await fetchPublicText(url.toString(), {\n    maxBytes: AI_THEME_REFERENCE_MAX_BYTES,\n    accept: /text\\/html|application\\/xhtml\\+xml/i,\n    signal,\n    ...(headersFor ? { headersFor } : {}),\n  })\n  if (!page) return null\n  const stylesheets: string[] = []\n  for (const sheet of stylesheetUrlsOf(page.text, page.url)) {\n    const css = await fetchPublicText(sheet, {\n      maxBytes: AI_THEME_STYLESHEET_MAX_BYTES,\n      accept: /text\\/css/i,\n      signal,\n      ...(headersFor ? { headersFor } : {}),\n    })\n    if (css) stylesheets.push(css.text)\n  }\n  return { url: page.url, html: page.text, stylesheets }\n}\n\n/** A read page's brand colors: every color its markup and stylesheets use, ranked. */\nexport function referencePageColors(page: Pick<AiReferencePage, 'html' | 'stylesheets'>): string[] {\n  const weights = colorsFromMarkup(page.html)\n  for (const css of page.stylesheets) {\n    for (const [hex, weight] of colorsFromMarkup(css)) {\n      weights.set(hex, (weights.get(hex) ?? 0) + weight)\n    }\n  }\n  return rankBrandColors(weights)\n}\n\n/** A linked page's colors, from its markup and its first stylesheets. */\nasync function readReferenceColors(url: URL, signal: AbortSignal): Promise<string[] | null> {\n  const page = await readReferencePage(url, signal)\n  return page ? referencePageColors(page) : null\n}\n\n/* ------------------------------------------------------------------------ *\n * Gathering\n * ------------------------------------------------------------------------ */\n\n/** The two reads that leave the process, replaceable where a spec needs to. */\nexport interface AiThemeBrandSeams {\n  logoColors?: (\n    firestore: FirebaseFirestore.Firestore,\n    location: AiThemeLogoLocation,\n    hostId: string,\n    /** The brand reads' bound, which a read made elsewhere waits no longer than. */\n    signal?: AbortSignal,\n  ) => Promise<string[] | null>\n  referenceColors?: (url: URL, signal: AbortSignal) => Promise<string[] | null>\n}\n\nexport async function gatherAiThemeBrandInputs(\n  input: {\n    firestore: FirebaseFirestore.Firestore\n    org: Partial<AglynOrgBilling> | null\n    hostId: string\n    /** The site's host document. */\n    host: Record<string, unknown>\n    brief: string\n    signal?: AbortSignal\n  },\n  seams: AiThemeBrandSeams = {},\n): Promise<AiThemeBrandInputs> {\n  const colors: AiThemeBrandColor[] = [...organizationBrandColors(input.org)]\n  const notes: string[] = []\n  const clock = deadline(input.signal, AI_THEME_BRAND_BUDGET_MS)\n  const add = (found: readonly string[], source: AiThemeBrandSource) => {\n    for (const hex of found.slice(0, AI_THEME_BRAND_COLORS_PER_SOURCE)) {\n      if (!colors.some((color) => color.hex === hex)) colors.push({ hex, source })\n    }\n  }\n  try {\n    const location = logoMediaLocation(\n      input.host['logoUrl'],\n      input.hostId,\n      String(input.host['orgId'] ?? ''),\n    )\n    if (location) {\n      const found = await (seams.logoColors ?? readLogoColors)(\n        input.firestore,\n        location,\n        input.hostId,\n        clock.signal,\n      ).catch(() => null)\n      if (found) add(found, 'logo')\n      else notes.push(\"The site logo's colors could not be read, so the proposal does not use them.\")\n    }\n    const reference = referenceUrlOf(input.brief)\n    if (reference) {\n      const found = clock.signal.aborted\n        ? null\n        : await (seams.referenceColors ?? readReferenceColors)(reference, clock.signal).catch(\n            () => null,\n          )\n      if (found?.length) add(found, 'reference')\n      else {\n        notes.push(\n          `The page at ${reference.hostname} could not be read for its colors, so the proposal does not use them.`,\n        )\n      }\n    }\n  } finally {\n    clock.done()\n  }\n  return { colors, notes }\n}\n"],"names":["checkEntitlement","normalizeThemeHex","AI_THEME_BRAND_BUDGET_MS","aiMediaAssetLocation","loadAiMediaSharp","readAiMediaAssetBytes","AI_THEME_BRAND_SOURCE_WORDS","organization","logo","reference","AI_THEME_BRAND_COLORS_PER_SOURCE","AI_THEME_LOGO_MAX_BYTES","AI_THEME_REFERENCE_MAX_BYTES","AI_THEME_STYLESHEET_MAX_BYTES","AI_THEME_REFERENCE_STYLESHEETS","FETCH_TIMEOUT_MS","MAX_REDIRECT_HOPS","REDIRECT_STATUSES","Set","toHex","red","green","blue","map","channel","Math","round","min","max","toString","padStart","join","channelsOf","hex","parseInt","slice","distance","a","b","hypot","chroma","SAME_COLOR_DISTANCE","GREY_CHROMA","distinctColors","ranked","count","colorful","filter","color","chosen","length","some","other","push","organizationBrandColors","org","raw","brandingProfile","primaryColor","source","logoMediaLocation","dominantColorsFromPixels","data","channels","buckets","Map","index","pixel","key","bucket","get","sum","pixels","set","values","sort","edge","inside","LOGO_TYPES","readLogoColors","firestore","location","hostId","signal","images","origin","aiImageReadsOrigin","readAiThemeLogoColorsAt","orgId","scope","isOrg","scopeId","payload","aiImageReadsRequest","answer","requestAiImageReads","AbortSignal","timeout","logoColorsAnswer","colors","asset","maxBytes","types","sharp","info","buffer","limitInputPixels","resize","fit","ensureAlpha","toBuffer","resolveWithObject","Array","isArray","Boolean","URL_IN_BRIEF","referenceUrlOf","brief","match","exec","url","URL","replace","protocol","AI_THEME_THEME_COLOR_WEIGHT","THEME_COLOR_META","CONTENT_ATTRIBUTE","HEX_LITERAL","RGB_LITERAL","colorsFromMarkup","text","weights","add","weight","tag","matchAll","Number","STYLESHEET_LINK","HREF_ATTRIBUTE","stylesheetUrlsOf","html","pageUrl","urls","href","includes","rankBrandColors","entries","deadline","ms","controller","AbortController","timer","setTimeout","abort","onAbort","aborted","addEventListener","done","clearTimeout","removeEventListener","readCapped","body","refuseOver","binary","reader","getReader","chunks","total","truncated","value","read","room","byteLength","cancel","catch","undefined","subarray","joined","Uint8Array","offset","chunk","bytes","TextDecoder","decode","publicFetchEdgeOf","headers","server","toLowerCase","refusedFetch","refusal","extra","ok","contentType","publicFetchExtraHeaders","out","name","Object","test","fetchPublicTextDetailed","options","createPinnedDispatcher","resolvePublicIp","current","hop","parsed","username","password","port","pinned","hostname","dispatcher","clock","timeoutMs","response","init","method","redirect","headersFor","Accept","acceptHeader","userAgent","fetch","has","status","next","redirectAllowed","accept","declared","isFinite","close","fetchPublicText","result","readReferencePage","page","stylesheets","sheet","css","referencePageColors","readReferenceColors","gatherAiThemeBrandInputs","input","seams","notes","found","host","String","logoColors","referenceColors"],"mappings":";AAAA;;;;;;;;;;;;;;;CAeC,GAED,SAASA,gBAAgB,QAAQ,2CAA0C;AAK3E,SAASC,iBAAiB,QAAQ,4BAAwB;AAC1D,SAASC,wBAAwB,QAAQ,2BAAuB;AAChE,SACEC,oBAAoB,EACpBC,gBAAgB,EAChBC,qBAAqB,QAEhB,sBAAkB;AA0BzB,2DAA2D,GAC3D,OAAO,MAAMC,8BAAkE;IAC7EC,cAAc;IACdC,MAAM;IACNC,WAAW;AACb,EAAC;AAaD,4CAA4C,GAC5C,OAAO,MAAMC,mCAAmC,EAAC;AAEjD,6CAA6C,GAC7C,OAAO,MAAMC,0BAA0B,IAAI,OAAO,KAAI;AAEtD,6EAA6E,GAC7E,OAAO,MAAMC,+BAA+B,MAAM,KAAI;AACtD,OAAO,MAAMC,gCAAgC,MAAM,KAAI;AAEvD,8DAA8D,GAC9D,OAAO,MAAMC,iCAAiC,EAAC;AAE/C;;;CAGC,GACD,SAASZ,wBAAwB,GAAE;AAEnC,MAAMa,mBAAmB;AACzB,MAAMC,oBAAoB;AAC1B,MAAMC,oBAAoB,IAAIC,IAAI;IAAC;IAAK;IAAK;IAAK;IAAK;CAAI;AAI3D,MAAMC,QAAQ,CAAC,CAACC,KAAKC,OAAOC,KAAU,GACpC,CAAC,CAAC,EAAE;QAACF;QAAKC;QAAOC;KAAK,CACnBC,GAAG,CAAC,CAACC,UAAYC,KAAKC,KAAK,CAACD,KAAKE,GAAG,CAAC,KAAKF,KAAKG,GAAG,CAAC,GAAGJ,WAAWK,QAAQ,CAAC,IAAIC,QAAQ,CAAC,GAAG,MAC1FC,IAAI,CAAC,KAAK;AAEf,MAAMC,aAAa,CAACC,MAAqB;QACvCC,SAASD,IAAIE,KAAK,CAAC,GAAG,IAAI;QAC1BD,SAASD,IAAIE,KAAK,CAAC,GAAG,IAAI;QAC1BD,SAASD,IAAIE,KAAK,CAAC,GAAG,IAAI;KAC3B;AAED,MAAMC,WAAW,CAACC,GAAQC,IAAWb,KAAKc,KAAK,CAACF,CAAC,CAAC,EAAE,GAAGC,CAAC,CAAC,EAAE,EAAED,CAAC,CAAC,EAAE,GAAGC,CAAC,CAAC,EAAE,EAAED,CAAC,CAAC,EAAE,GAAGC,CAAC,CAAC,EAAE;AAErF,8DAA8D,GAC9D,MAAME,SAAS,CAAC,CAACpB,KAAKC,OAAOC,KAAU,GAAKG,KAAKG,GAAG,CAACR,KAAKC,OAAOC,QAAQG,KAAKE,GAAG,CAACP,KAAKC,OAAOC;AAE9F,iEAAiE,GACjE,MAAMmB,sBAAsB;AAE5B,0EAA0E,GAC1E,MAAMC,cAAc;AAEpB,kFAAkF,GAClF,SAASC,eAAeC,MAAsB,EAAEC,KAAa;IAC3D,MAAMC,WAAWF,OAAOG,MAAM,CAAC,CAACC,QAAUR,OAAOQ,SAASN;IAC1D,MAAMO,SAAgB,EAAE;IACxB,KAAK,MAAMD,SAASF,SAASI,MAAM,GAAGJ,WAAWF,OAAQ;QACvD,IAAIK,OAAOE,IAAI,CAAC,CAACC,QAAUhB,SAASgB,OAAOJ,SAASP,sBAAsB;QAC1EQ,OAAOI,IAAI,CAACL;QACZ,IAAIC,OAAOC,MAAM,KAAKL,OAAO;IAC/B;IACA,OAAOI,OAAO1B,GAAG,CAACJ;AACpB;AAEA;;4EAE4E,GAE5E,gFAAgF,GAChF,OAAO,SAASmC,wBACdC,GAAgD;QAGnCA;IADb,IAAI,CAACvD,iBAAiBuD,KAAK,eAAe,OAAO,EAAE;IACnD,MAAMC,MAAOD,wBAAAA,uBAAAA,IAAKE,eAAe,qBAArB,AAACF,qBAAyDG,YAAY;IAClF,MAAMzB,MAAM,OAAOuB,QAAQ,WAAWvD,kBAAkBuD,OAAO;IAC/D,OAAOvB,MAAM;QAAC;YAAEA;YAAK0B,QAAQ;QAAe;KAAE,GAAG,EAAE;AACrD;AAQA;;;;;;CAMC,GACD,OAAO,MAAMC,oBAAoBzD,qBAAoB;AAErD;;;;;CAKC,GACD,OAAO,SAAS0D,yBACdC,IAAuB,EACvBC,QAAgB,EAChBlB,QAAQnC,gCAAgC;IAExC,MAAMsD,UAAU,IAAIC;IACpB,IAAK,IAAIC,QAAQ,GAAGA,QAAQH,YAAYD,KAAKZ,MAAM,EAAEgB,SAASH,SAAU;YAIvDC;QAHf,IAAID,YAAY,KAAKD,IAAI,CAACI,QAAQ,EAAE,GAAG,KAAK;QAC5C,MAAMC,QAAa;YAACL,IAAI,CAACI,MAAM;YAAEJ,IAAI,CAACI,QAAQ,EAAE;YAAEJ,IAAI,CAACI,QAAQ,EAAE;SAAC;QAClE,MAAME,MAAM,AAAED,KAAK,CAAC,EAAE,IAAI,KAAM,IAAM,AAACA,KAAK,CAAC,EAAE,IAAI,KAAM,IAAMA,KAAK,CAAC,EAAE,IAAI;QAC3E,MAAME,UAASL,eAAAA,QAAQM,GAAG,CAACF,gBAAZJ,eAAoB;YAAEO,KAAK;gBAAC;gBAAG;gBAAG;aAAE;YAASC,QAAQ;QAAE;QACtEH,OAAOE,GAAG,GAAG;YAACF,OAAOE,GAAG,CAAC,EAAE,GAAGJ,KAAK,CAAC,EAAE;YAAEE,OAAOE,GAAG,CAAC,EAAE,GAAGJ,KAAK,CAAC,EAAE;YAAEE,OAAOE,GAAG,CAAC,EAAE,GAAGJ,KAAK,CAAC,EAAE;SAAC;QAC3FE,OAAOG,MAAM,IAAI;QACjBR,QAAQS,GAAG,CAACL,KAAKC;IACnB;IACA,MAAMzB,SAAS;WAAIoB,QAAQU,MAAM;KAAG,CACjCC,IAAI,CAAC,CAACtC,GAAGC,IAAMA,EAAEkC,MAAM,GAAGnC,EAAEmC,MAAM,EAClCjD,GAAG,CAAC,CAAC,EAAEgD,GAAG,EAAEC,MAAM,EAAE,GAAU;YAACD,GAAG,CAAC,EAAE,GAAGC;YAAQD,GAAG,CAAC,EAAE,GAAGC;YAAQD,GAAG,CAAC,EAAE,GAAGC;SAAO;IACpF,MAAMI,OAAO,CAAC5B,QAAevB,KAAKE,GAAG,IAAIqB,SAAS,OAAOvB,KAAKG,GAAG,IAAIoB,SAAS;IAC9E,MAAM6B,SAASjC,OAAOG,MAAM,CAAC,CAACC,QAAU,CAAC4B,KAAK5B;IAC9C,OAAOL,eAAekC,OAAO3B,MAAM,GAAG2B,SAASjC,QAAQC;AACzD;AAcA,MAAMiC,aAAa;AAEnB;;;;;;;;;;CAUC,GACD,eAAeC,eACbC,SAAsC,EACtCC,QAA6B,EAC7BC,MAAc,EACdC,MAAoB;IAEpB,MAAMC,SAAS,MAAM,MAAM,CAAC;IAC5B,MAAMC,SAASD,OAAOE,kBAAkB;IACxC,IAAI,CAACD,QAAQ,OAAOE,wBAAwBP,WAAWC,UAAUC;IACjE,wEAAwE;IACxE,kDAAkD;IAClD,MAAMM,QAAQP,SAASQ,KAAK,CAACC,KAAK,GAAGT,SAASQ,KAAK,CAACE,OAAO,GAAG;IAC9D,MAAMC,UAAUR,OAAOS,mBAAmB,CAAC,eAAeZ,UAAUC,QAAQM;IAC5E,IAAI,CAACI,SAAS,OAAO;IACrB,MAAME,SAAS,MAAMV,OAAOW,mBAAmB,CAC7CV,QACAO,SACAT,iBAAAA,SAAUa,YAAYC,OAAO,CAAC/F;IAEhC,OAAOgG,iBAAiBJ,0BAAAA,OAAQK,MAAM;AACxC;AAEA;;;;CAIC,GACD,OAAO,eAAeZ,wBACpBP,SAAsC,EACtCC,QAA6B,EAC7BC,MAAc;IAEd,MAAMkB,QAAQ,MAAM/F,sBAAsB2E,WAAWC,UAAUC,QAAQ;QACrEmB,UAAU1F;QACV2F,OAAOxB;IACT;IACA,IAAI,CAACsB,OAAO,OAAO;IACnB,MAAMG,QAAS,MAAMnG;IACrB,MAAM,EAAE0D,IAAI,EAAE0C,IAAI,EAAE,GAAG,MAAMD,MAAMH,MAAMK,MAAM,EAAE;QAAEC,kBAAkB;IAAW,GAC7EC,MAAM,CAAC,IAAI,IAAI;QAAEC,KAAK;IAAS,GAC/BC,WAAW,GACXrD,GAAG,GACHsD,QAAQ,CAAC;QAAEC,mBAAmB;IAAK;IACtC,OAAOlD,yBAAyBC,MAAM0C,KAAKzC,QAAQ;AACrD;AAEA,4EAA4E,GAC5E,SAASmC,iBAAiBC,MAAe;IACvC,IAAI,CAACa,MAAMC,OAAO,CAACd,SAAS,OAAO;IACnC,OAAOA,OACJ5E,GAAG,CAAC,CAACyB,QAAW,OAAOA,UAAU,WAAW/C,kBAAkB+C,SAAS,MACvED,MAAM,CAAC,CAACd,MAAuBiF,QAAQjF,MACvCE,KAAK,CAAC,GAAGzB;AACd;AAEA;;4EAE4E,GAE5E,MAAMyG,eAAe;AAErB,+EAA+E,GAC/E,OAAO,SAASC,eAAeC,KAAa;IAC1C,MAAMC,QAAQH,aAAaI,IAAI,CAACF;IAChC,IAAI,CAACC,OAAO,OAAO;IACnB,IAAI;QACF,MAAME,MAAM,IAAIC,IAAIH,KAAK,CAAC,EAAE,CAACI,OAAO,CAAC,cAAc;QACnD,OAAOF,IAAIG,QAAQ,KAAK,WAAWH,MAAM;IAC3C,EAAE,eAAM;QACN,OAAO;IACT;AACF;AAEA,+EAA+E,GAC/E,OAAO,MAAMI,8BAA8B,GAAE;AAE7C,MAAMC,mBAAmB;AACzB,MAAMC,oBAAoB;AAC1B,oEAAoE;AACpE,MAAMC,cAAc;AACpB,MAAMC,cAAc;AAEpB,6EAA6E,GAC7E,OAAO,SAASC,iBAAiBC,IAAY;QAKzBA;IAJlB,MAAMC,UAAU,IAAIlE;IACpB,MAAMmE,MAAM,CAACnG,KAAoBoG;YACJF;QAA3B,IAAIlG,KAAKkG,QAAQ1D,GAAG,CAACxC,KAAK,EAACkG,eAAAA,QAAQ7D,GAAG,CAACrC,gBAAZkG,eAAoB,KAAKE;IACtD;IACA,KAAK,MAAMC,QAAOJ,cAAAA,KAAKZ,KAAK,CAACO,6BAAXK,cAAgC,EAAE,CAAE;;YAC9BJ;QAAtBM,IAAInI,2BAAkB6H,0BAAAA,kBAAkBP,IAAI,CAACe,yBAAvBR,uBAA6B,CAAC,EAAE,mBAAI,KAAKF;IACjE;IACA,KAAK,MAAMN,SAASY,KAAKK,QAAQ,CAACR,aAAcK,IAAInI,kBAAkBqH,KAAK,CAAC,EAAE,GAAG;IACjF,KAAK,MAAMA,SAASY,KAAKK,QAAQ,CAACP,aAAc;QAC9CI,IAAIjH,MAAM;YAACqH,OAAOlB,KAAK,CAAC,EAAE;YAAGkB,OAAOlB,KAAK,CAAC,EAAE;YAAGkB,OAAOlB,KAAK,CAAC,EAAE;SAAE,GAAG;IACrE;IACA,OAAOa;AACT;AAEA,MAAMM,kBAAkB;AACxB,MAAMC,iBAAiB;AAEvB,4EAA4E,GAC5E,OAAO,SAASC,iBAAiBC,IAAY,EAAEC,OAAe;QAE1CD;IADlB,MAAME,OAAiB,EAAE;IACzB,KAAK,MAAMR,QAAOM,cAAAA,KAAKtB,KAAK,CAACmB,4BAAXG,cAA+B,EAAE,CAAE;YACtCF;QAAb,MAAMK,QAAOL,uBAAAA,eAAenB,IAAI,CAACe,yBAApBI,oBAA0B,CAAC,EAAE;QAC1C,IAAI,CAACK,MAAM;QACX,IAAI;YACF,MAAMvB,MAAM,IAAIC,IAAIsB,MAAMF;YAC1B,IAAIrB,IAAIG,QAAQ,KAAK,YAAY,CAACmB,KAAKE,QAAQ,CAACxB,IAAI3F,QAAQ,KAAKiH,KAAKzF,IAAI,CAACmE,IAAI3F,QAAQ;QACzF,EAAE,eAAM;YACN;QACF;QACA,IAAIiH,KAAK5F,MAAM,KAAKpC,gCAAgC;IACtD;IACA,OAAOgI;AACT;AAEA,kEAAkE,GAClE,OAAO,SAASG,gBACdd,OAAoC,EACpCtF,QAAQnC,gCAAgC;IAExC,MAAMkC,SAAS;WAAIuF,QAAQe,OAAO;KAAG,CAClCvE,IAAI,CAAC,CAACtC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1Bd,GAAG,CAAC,CAAC,CAACU,IAAI,GAAKD,WAAWC;IAC7B,OAAOU,eAAeC,QAAQC;AAChC;AAEA,yEAAyE,GACzE,SAASsG,SAAShE,MAA+B,EAAEiE,EAAU;IAC3D,MAAMC,aAAa,IAAIC;IACvB,MAAMC,QAAQC,WAAW,IAAMH,WAAWI,KAAK,IAAIL;IACnD,MAAMM,UAAU,IAAML,WAAWI,KAAK;IACtC,IAAItE,0BAAAA,OAAQwE,OAAO,EAAEN,WAAWI,KAAK;IACrCtE,0BAAAA,OAAQyE,gBAAgB,CAAC,SAASF;IAClC,OAAO;QACLvE,QAAQkE,WAAWlE,MAAM;QACzB0E,MAAM;YACJC,aAAaP;YACbpE,0BAAAA,OAAQ4E,mBAAmB,CAAC,SAASL;QACvC;IACF;AACF;AAEA;;;CAGC,GACD,eAAeM,WACbC,IAAgC,EAChC5D,QAAgB,EAChB6D,aAAa,KAAK,EAClBC,SAAS,KAAK;IAEd,MAAMC,SAASH,KAAKI,SAAS;IAC7B,MAAMC,SAAuB,EAAE;IAC/B,IAAIC,QAAQ;IACZ,IAAIC,YAAY;IAChB,OAAS;QACP,MAAM,EAAEX,IAAI,EAAEY,KAAK,EAAE,GAAG,MAAML,OAAOM,IAAI;QACzC,IAAIb,QAAQ,CAACY,OAAO;QACpB,MAAME,OAAOtE,WAAWkE;QACxB,IAAIE,MAAMG,UAAU,GAAGD,MAAM;YAC3B,MAAMP,OAAOS,MAAM,GAAGC,KAAK,CAAC,IAAMC;YAClC,IAAIb,YAAY,OAAO;YACvBI,OAAOjH,IAAI,CAACoH,MAAMO,QAAQ,CAAC,GAAGL;YAC9BJ,SAASI;YACTH,YAAY;YACZ;QACF;QACAF,OAAOjH,IAAI,CAACoH;QACZF,SAASE,MAAMG,UAAU;IAC3B;IACA,MAAMK,SAAS,IAAIC,WAAWX;IAC9B,IAAIY,SAAS;IACb,KAAK,MAAMC,SAASd,OAAQ;QAC1BW,OAAOxG,GAAG,CAAC2G,OAAOD;QAClBA,UAAUC,MAAMR,UAAU;IAC5B;IACA,6EAA6E;IAC7E,IAAIT,QAAQ,OAAO;QAAEjC,MAAM;QAAImD,OAAOd;QAAOC;QAAW1G,MAAMmH;IAAO;IACrE,OAAO;QAAE/C,MAAM,IAAIoD,cAAcC,MAAM,CAACN;QAASI,OAAOd;QAAOC;IAAU;AAC3E;AAoDA,yFAAyF,GACzF,OAAO,SAASgB,kBAAkBC,OAA6B;QAC7CA;IAAhB,MAAMC,SAAS,EAACD,eAAAA,QAAQnH,GAAG,CAAC,qBAAZmH,eAAyB,IAAIE,WAAW;IACxD,IAAIF,QAAQnH,GAAG,CAAC,yBAAyBmH,QAAQnH,GAAG,CAAC,kBAAkBoH,WAAW,UAAU,OAAO;IACnG,IAAID,QAAQnH,GAAG,CAAC,mBAAmBmH,QAAQnH,GAAG,CAAC,aAAaoH,WAAW,cAAc,OAAO;IAC5F,OAAO;AACT;AAEA,SAASE,aACPC,OAA2B,EAC3BrE,GAAW,EACXsE,QAAwE,CAAC,CAAC;IAE1E,OAAO;QAAEC,IAAI;QAAOvE;QAAKU,MAAM;QAAImD,OAAO;QAAGb,WAAW;QAAOwB,aAAa;QAAIH;OAAYC;AAC9F;AA2BA,2GAA2G,GAC3G,SAASG,wBAAwBR,OAAqD;IACpF,MAAMS,MAA8B,CAAC;IACrC,KAAK,MAAM,CAACC,MAAM1B,MAAM,IAAI2B,OAAOlD,OAAO,CAACuC,kBAAAA,UAAW,CAAC,GAAI;QACzD,IAAI,oEAAoEY,IAAI,CAACF,OAAO;QACpF,IAAI,OAAO1B,UAAU,UAAUyB,GAAG,CAACC,KAAK,GAAG1B;IAC7C;IACA,OAAOyB;AACT;AAEA;;;;;;;;CAQC,GACD,OAAO,eAAeI,wBACpB9E,GAAW,EACX+E,OAA2B;IAE3B,MAAM,EAAEC,sBAAsB,EAAEC,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC;IACjE,IAAIC,UAAUlF;IACd,IAAK,IAAImF,MAAM,GAAGA,OAAO3L,mBAAmB2L,OAAO,EAAG;YAgBbJ;QAfvC,IAAIK;QACJ,IAAI;YACFA,SAAS,IAAInF,IAAIiF;QACnB,EAAE,eAAM;YACN,OAAOd,aAAa,eAAec;QACrC;QACA,IAAIE,OAAOjF,QAAQ,KAAK,YAAYiF,OAAOC,QAAQ,IAAID,OAAOE,QAAQ,EAAE;YACtE,OAAOlB,aAAa,eAAec;QACrC;QACA,IAAIE,OAAOG,IAAI,IAAIH,OAAOG,IAAI,KAAK,OAAO;YACxC,OAAOnB,aAAa,eAAec;QACrC;QACA,MAAMM,SAAS,MAAMP,gBAAgBG,OAAOK,QAAQ;QACpD,IAAI,CAACD,QAAQ,OAAOpB,aAAa,mBAAmBgB,OAAO/K,QAAQ;QACnE,MAAMqL,aAAaV,uBAAuBQ;QAC1C,MAAMG,QAAQhE,SAASoD,QAAQpH,MAAM,GAAEoH,qBAAAA,QAAQa,SAAS,YAAjBb,qBAAqBxL;QAC5D,IAAI;gBAUUwL,uBA4BQc,uBAKIA;YA1CxB,sEAAsE;YACtE,kEAAkE;YAClE,MAAMC,OAAO;gBACXC,QAAQ;gBACRC,UAAU;gBACVrI,QAAQgI,MAAMhI,MAAM;gBACpB+H;gBACAzB,SAAS,aACJQ,wBAAwBM,QAAQkB,UAAU,oBAAlBlB,QAAQkB,UAAU,MAAlBlB,SAAqBK;oBAChDc,MAAM,GAAEnB,wBAAAA,QAAQoB,YAAY,YAApBpB,wBAAwB;mBAC5BA,QAAQqB,SAAS,GAAG;oBAAE,cAAcrB,QAAQqB,SAAS;gBAAC,IAAI,CAAC;YAEnE;YACA,MAAMP,WAAW,MAAMQ,MAAMjB,OAAO/K,QAAQ,IAAIyL;YAChD,IAAIrM,kBAAkB6M,GAAG,CAACT,SAASU,MAAM,GAAG;oBACpCV;gBAAN,QAAMA,iBAAAA,SAASpD,IAAI,qBAAboD,eAAexC,MAAM,GAAGC,KAAK,CAAC,IAAMC;gBAC1C,MAAM9F,WAAWoI,SAAS5B,OAAO,CAACnH,GAAG,CAAC;gBACtC,IAAI,CAACW,UAAU;oBACb,OAAO2G,aAAa,eAAegB,OAAO/K,QAAQ,IAAI;wBAAEkM,QAAQV,SAASU,MAAM;oBAAC;gBAClF;gBACA,IAAIC;gBACJ,IAAI;oBACFA,OAAO,IAAIvG,IAAIxC,UAAU2H;gBAC3B,EAAE,eAAM;oBACN,OAAOhB,aAAa,oBAAoBgB,OAAO/K,QAAQ,IAAI;wBAAEoD;oBAAS;gBACxE;gBACA,IAAIsH,QAAQ0B,eAAe,IAAI,CAAC1B,QAAQ0B,eAAe,CAACrB,QAAQoB,OAAO;oBACrE,OAAOpC,aAAa,oBAAoBgB,OAAO/K,QAAQ,IAAI;wBAAEoD,UAAU+I,KAAKnM,QAAQ;oBAAG;gBACzF;gBACA6K,UAAUsB,KAAKnM,QAAQ;gBACvB;YACF;YACA,IAAI,CAACwL,SAAStB,EAAE,IAAI,CAACsB,SAASpD,IAAI,EAAE;oBAC5BoD;gBAAN,QAAMA,kBAAAA,SAASpD,IAAI,qBAAboD,gBAAexC,MAAM,GAAGC,KAAK,CAAC,IAAMC;gBAC1C,MAAMnG,OAAO4G,kBAAkB6B,SAAS5B,OAAO;gBAC/C,OAAOG,aAAa,eAAegB,OAAO/K,QAAQ,IAAI;oBAAEkM,QAAQV,SAASU,MAAM;mBAAMnJ,OAAO;oBAAEA;gBAAK,IAAI,CAAC;YAC1G;YACA,MAAMoH,eAAcqB,wBAAAA,SAAS5B,OAAO,CAACnH,GAAG,CAAC,2BAArB+I,wBAAwC;YAC5D,IAAI,CAACd,QAAQ2B,MAAM,CAAC7B,IAAI,CAACL,cAAc;gBACrC,MAAMqB,SAASpD,IAAI,CAACY,MAAM,GAAGC,KAAK,CAAC,IAAMC;gBACzC,OAAOa,aAAa,gBAAgBgB,OAAO/K,QAAQ,IAAI;oBAAEkM,QAAQV,SAASU,MAAM;gBAAC;YACnF;YACA,MAAMI,WAAW3F,QAAO6E,yBAAAA,SAAS5B,OAAO,CAACnH,GAAG,CAAC,6BAArB+I,yBAA0C;YAClE,IAAId,QAAQrC,UAAU,IAAI1B,OAAO4F,QAAQ,CAACD,aAAaA,WAAW5B,QAAQlG,QAAQ,EAAE;gBAClF,MAAMgH,SAASpD,IAAI,CAACY,MAAM,GAAGC,KAAK,CAAC,IAAMC;gBACzC,OAAOa,aAAa,aAAagB,OAAO/K,QAAQ,IAAI;oBAAEkM,QAAQV,SAASU,MAAM;gBAAC;YAChF;YACA,MAAM9D,OAAO,MAAMD,WAAWqD,SAASpD,IAAI,EAAEsC,QAAQlG,QAAQ,EAAEkG,QAAQrC,UAAU,EAAEqC,QAAQpC,MAAM,KAAK;YACtG,IAAI,CAACF,MAAM;gBACT,OAAO2B,aAAa,aAAagB,OAAO/K,QAAQ,IAAI;oBAAEkM,QAAQV,SAASU,MAAM;gBAAC;YAChF;YACA,OAAO;gBAAEhC,IAAI;gBAAMvE,KAAKoF,OAAO/K,QAAQ;gBAAImK;gBAAaH,SAAS;eAAS5B;QAC5E,EAAE,eAAM;YACN,OAAO2B,aAAa,WAAWgB,OAAO/K,QAAQ;QAChD,SAAU;YACRsL,MAAMtD,IAAI;YACV,MAAMqD,WAAWmB,KAAK,GAAGvD,KAAK,CAAC,IAAMC;QACvC;IACF;IACA,OAAOa,aAAa,sBAAsBc;AAC5C;AAEA,+EAA+E,GAC/E,OAAO,eAAe4B,gBACpB9G,GAAW,EACX+E,OAAsI;IAEtI,MAAMgC,SAAS,MAAMjC,wBAAwB9E,KAAK+E;IAClD,OAAOgC,OAAOxC,EAAE,GAAG;QAAE7D,MAAMqG,OAAOrG,IAAI;QAAEV,KAAK+G,OAAO/G,GAAG;IAAC,IAAI;AAC9D;AAeA;;;;;;;CAOC,GACD,OAAO,eAAegH,kBACpBhH,GAAQ,EACRrC,MAAmB,EACnBsI,UAA6C;IAE7C,MAAMgB,OAAO,MAAMH,gBAAgB9G,IAAI3F,QAAQ,IAAI;QACjDwE,UAAUzF;QACVsN,QAAQ;QACR/I;OACIsI,aAAa;QAAEA;IAAW,IAAI,CAAC;IAErC,IAAI,CAACgB,MAAM,OAAO;IAClB,MAAMC,cAAwB,EAAE;IAChC,KAAK,MAAMC,SAAShG,iBAAiB8F,KAAKvG,IAAI,EAAEuG,KAAKjH,GAAG,EAAG;QACzD,MAAMoH,MAAM,MAAMN,gBAAgBK,OAAO;YACvCtI,UAAUxF;YACVqN,QAAQ;YACR/I;WACIsI,aAAa;YAAEA;QAAW,IAAI,CAAC;QAErC,IAAImB,KAAKF,YAAYrL,IAAI,CAACuL,IAAI1G,IAAI;IACpC;IACA,OAAO;QAAEV,KAAKiH,KAAKjH,GAAG;QAAEoB,MAAM6F,KAAKvG,IAAI;QAAEwG;IAAY;AACvD;AAEA,oFAAoF,GACpF,OAAO,SAASG,oBAAoBJ,IAAmD;IACrF,MAAMtG,UAAUF,iBAAiBwG,KAAK7F,IAAI;IAC1C,KAAK,MAAMgG,OAAOH,KAAKC,WAAW,CAAE;QAClC,KAAK,MAAM,CAACzM,KAAKoG,OAAO,IAAIJ,iBAAiB2G,KAAM;gBAC/BzG;YAAlBA,QAAQ1D,GAAG,CAACxC,KAAK,EAACkG,eAAAA,QAAQ7D,GAAG,CAACrC,gBAAZkG,eAAoB,KAAKE;QAC7C;IACF;IACA,OAAOY,gBAAgBd;AACzB;AAEA,uEAAuE,GACvE,eAAe2G,oBAAoBtH,GAAQ,EAAErC,MAAmB;IAC9D,MAAMsJ,OAAO,MAAMD,kBAAkBhH,KAAKrC;IAC1C,OAAOsJ,OAAOI,oBAAoBJ,QAAQ;AAC5C;AAkBA,OAAO,eAAeM,yBACpBC,KAQC,EACDC,QAA2B,CAAC,CAAC;IAE7B,MAAM9I,SAA8B;WAAI7C,wBAAwB0L,MAAMzL,GAAG;KAAE;IAC3E,MAAM2L,QAAkB,EAAE;IAC1B,MAAM/B,QAAQhE,SAAS6F,MAAM7J,MAAM,EAAEjF;IACrC,MAAMkI,MAAM,CAAC+G,OAA0BxL;QACrC,KAAK,MAAM1B,OAAOkN,MAAMhN,KAAK,CAAC,GAAGzB,kCAAmC;YAClE,IAAI,CAACyF,OAAOhD,IAAI,CAAC,CAACH,QAAUA,MAAMf,GAAG,KAAKA,MAAMkE,OAAO9C,IAAI,CAAC;gBAAEpB;gBAAK0B;YAAO;QAC5E;IACF;IACA,IAAI;YAIOqL;QAHT,MAAM/J,WAAWrB,kBACfoL,MAAMI,IAAI,CAAC,UAAU,EACrBJ,MAAM9J,MAAM,EACZmK,QAAOL,oBAAAA,MAAMI,IAAI,CAAC,QAAQ,YAAnBJ,oBAAuB;QAEhC,IAAI/J,UAAU;gBACSgK;YAArB,MAAME,QAAQ,MAAM,EAACF,oBAAAA,MAAMK,UAAU,YAAhBL,oBAAoBlK,gBACvCiK,MAAMhK,SAAS,EACfC,UACA+J,MAAM9J,MAAM,EACZiI,MAAMhI,MAAM,EACZ2F,KAAK,CAAC,IAAM;YACd,IAAIqE,OAAO/G,IAAI+G,OAAO;iBACjBD,MAAM7L,IAAI,CAAC;QAClB;QACA,MAAM5C,YAAY2G,eAAe4H,MAAM3H,KAAK;QAC5C,IAAI5G,WAAW;gBAGFwO;YAFX,MAAME,QAAQhC,MAAMhI,MAAM,CAACwE,OAAO,GAC9B,OACA,MAAM,EAACsF,yBAAAA,MAAMM,eAAe,YAArBN,yBAAyBH,qBAAqBrO,WAAW0M,MAAMhI,MAAM,EAAE2F,KAAK,CACjF,IAAM;YAEZ,IAAIqE,yBAAAA,MAAOjM,MAAM,EAAEkF,IAAI+G,OAAO;iBACzB;gBACHD,MAAM7L,IAAI,CACR,CAAC,YAAY,EAAE5C,UAAUwM,QAAQ,CAAC,qEAAqE,CAAC;YAE5G;QACF;IACF,SAAU;QACRE,MAAMtD,IAAI;IACZ;IACA,OAAO;QAAE1D;QAAQ+I;IAAM;AACzB"}

package/src/lib/site-import/site-import-crawl.d.ts

@@ -155,10 +155,12 @@ export interface SiteImportCrawlSeams {
     now?: () => number;
     /**
      * Request headers for the site being read, asked once per crawl with its
-     * domain (AGL-4083): the edge header a site this platform serves lets
-     * AglynBot past its own bot protection with. Sent on every request to the
-     * site, under the crawler's own `Accept` and `User-Agent`, and only to the
-     * site: every address the crawler asks for is on it. None when absent.
+     * domain (AGL-4083, AGL-4085). Absent, the crawl asks the edge
+     * (`siteImportEdgeHeadersFor`): Aglyn's edge secret for each of the site's
+     * two origins this platform serves, decided origin by origin, and nothing
+     * anywhere else. Either way the headers go only to the site's two origins,
+     * and the fetch asks again on every redirect hop, so a hop anywhere else
+     * carries nothing. Always under the crawler's own `Accept` and `User-Agent`.
      */
     headersFor?: (domain: string) => Promise<Readonly<Record<string, string>>> | Readonly<Record<string, string>>;
 }

package/src/lib/site-import/site-import-crawl.js

@@ -36,6 +36,10 @@ var _process_env_SITE_IMPORT_BOT_INFO_URL;
  *   group, else `*`. A robots.txt that answers 404 allows everything; one
  *   that cannot be read at all (5xx, no answer) stops the crawl, because
  *   "could not ask" is not "allowed".
+ * - **Our own edge.** A site this platform serves gets the edge secret
+ *   (`site-import-edge.ts`, AGL-4085), which lifts our bot challenge for
+ *   AglynBot; it is asked per request and per redirect hop, and a request
+ *   that leaves the site's two origins never carries it.
  * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an
  *   index's children included), then same-site links found on the pages read,
  *   for whatever the sitemap missed.
@@ -61,6 +65,7 @@ import { fenceUntrustedText } from "@aglyn/shared-util-tools/untrusted-text";
 import { fetchPublicTextDetailed } from "../jobs/ai-theme-brand-inputs.js";
 import { extractSiteImportPage, readMetaValues } from "./site-import-html.js";
 import { AGLYN_BOT_TOKEN, parseRobotsTxt, parseSitemap, robotsAllows, robotsRulesFor } from "./site-import-robots.js";
+import { NO_SITE_IMPORT_HEADERS, siteImportEdgeHeadersFor } from "./site-import-edge.js";
 /**
  * Where the bot's user agent points a site owner who wants to know what it is
  * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on
@@ -88,18 +93,25 @@ export const SITE_IMPORT_DEFAULTS = {
 /** Never a page: files a link may point at that are not HTML. */ const NON_PAGE_EXTENSION = /\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i;
 const TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i;
 const DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\.[a-z0-9-]{1,63})+$/;
-/** The seam's headers for a domain; none when it has none or cannot say. */ async function siteImportHeadersFor(seams, domain) {
-    if (!seams.headersFor) return null;
+/**
+ * The per-request headers of a crawl of `domain`: the seam's, else the edge's,
+ * and none for any address off the site's two origins. Never throws.
+ */ async function siteRequestHeaders(seams, domain, origins) {
+    let inner = NO_SITE_IMPORT_HEADERS;
     try {
-        const headers = await seams.headersFor(domain);
-        return headers && Object.keys(headers).length ? headers : null;
+        if (seams.headersFor) {
+            const headers = await seams.headersFor(domain);
+            if (headers && Object.keys(headers).length) inner = ()=>headers;
+        } else {
+            inner = await siteImportEdgeHeadersFor(origins);
+        }
     } catch (error) {
         console.warn('[site-import] the edge headers could not be read; reading without them', {
             domain,
             error
         });
-        return null;
     }
+    return (url)=>origins.includes(url.origin) ? inner(url) : {};
 }
 /** The site's two origins for a domain. */ export function siteImportOrigins(domain) {
     const bare = domain.replace(/^www\./, '');
@@ -174,6 +186,8 @@ function skipReasonFor(result) {
         inventory.refused = 'lookalike';
         return finish();
     }
+    // Asked once, before anything else, and held to the site's two origins.
+    const headersFor = await siteRequestHeaders(seams, domain, origins);
     const maxPages = Math.max(0, Math.floor(options.maxPages));
     const maxPageBytes = (_options_maxPageBytes = options.maxPageBytes) != null ? _options_maxPageBytes : SITE_IMPORT_DEFAULTS.maxPageBytes;
     const maxTotalBytes = (_options_maxTotalBytes = options.maxTotalBytes) != null ? _options_maxTotalBytes : SITE_IMPORT_DEFAULTS.maxTotalBytes;
@@ -186,20 +200,17 @@ function skipReasonFor(result) {
     const sameSite = (url)=>origins.includes(url.origin) && url.protocol === 'https:';
     const remainingBytes = ()=>maxTotalBytes - inventory.bytesRead;
     const outOfTime = ()=>controller.signal.aborted || now() >= deadlineAt;
-    // Asked once, before anything else; every address below is on the site.
-    const extraHeaders = await siteImportHeadersFor(seams, domain);
-    const request = (url, accept, acceptHeader, maxBytes)=>fetchText(url, _extends({
+    const request = (url, accept, acceptHeader, maxBytes)=>fetchText(url, {
             maxBytes,
             accept,
             acceptHeader,
             userAgent: AGLYN_BOT_USER_AGENT,
+            headersFor,
             signal: controller.signal,
             timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),
             redirectAllowed: (_from, to)=>sameSite(to),
             refuseOver: true
-        }, extraHeaders ? {
-            headers: extraHeaders
-        } : {}));
+        });
     const skip = (entry)=>{
         if (inventory.skipped.length < SITE_IMPORT_DEFAULTS.maxSkipped) inventory.skipped.push(entry);
         else inventory.skippedOverflow += 1;
@@ -494,17 +505,15 @@ function skipReasonFor(result) {
         values: []
     };
     const origins = siteImportOrigins(bare);
-    const extraHeaders = await siteImportHeadersFor(seams, bare);
-    const result = await fetchText(`${origins[0]}/`, _extends({
+    const result = await fetchText(`${origins[0]}/`, {
         maxBytes: SITE_IMPORT_DEFAULTS.maxPageBytes,
         accept: /text\/html|application\/xhtml\+xml/i,
         acceptHeader: 'text/html, application/xhtml+xml;q=0.9',
         userAgent: AGLYN_BOT_USER_AGENT,
+        headersFor: await siteRequestHeaders(seams, bare, origins),
         timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,
         redirectAllowed: (_from, to)=>to.protocol === 'https:' && origins.includes(to.origin)
-    }, extraHeaders ? {
-        headers: extraHeaders
-    } : {}));
+    });
     if (!result.ok) {
         // An answer that is not the page (a 404, a redirect off the site) is
         // still an answer: the tag is not where it must be.

package/src/lib/site-import/site-import-crawl.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-crawl.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * THE SITE-IMPORT CRAWLER (AGL-3974, parent AGL-3973).\n *\n * Reads a website a workspace has PROVED it owns (domain-ownership.ts in\n * tenant-data-admin; the doors check the proof before calling this) into a\n * typed inventory: each page's title, description, headings and text in\n * order, images and menu links. It writes nothing; the import job decides\n * what becomes a page.\n *\n * Every request goes through `fetchPublicTextDetailed`, the theme job's\n * SSRF-guarded fetch: https on the default port only, public addresses only,\n * the socket pinned to the address checked. On top of that:\n *\n * - **One site.** The site is `https://<domain>` and `https://www.<domain>`,\n *   one site to its owner. Every redirect hop and every discovered link must\n *   stay on it; a page that redirects anywhere else is refused, not followed.\n * - **AglynBot.** Every request says `AglynBot/1.0 (+<the operator's bot page>)`,\n *   and robots.txt is read first and honored for every page: the AglynBot\n *   group, else `*`. A robots.txt that answers 404 allows everything; one\n *   that cannot be read at all (5xx, no answer) stops the crawl, because\n *   \"could not ask\" is not \"allowed\".\n * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an\n *   index's children included), then same-site links found on the pages read,\n *   for whatever the sitemap missed.\n * - **Budgets.** Pages (the caller's, from the plan), bytes per page (a page\n *   over it is refused, not cut), bytes in total, and wall-clock time. A\n *   whole site's worth fits (AGL-4083): aglyn.com's 123 pages average 440 KB,\n *   54 MB in all, so the total is 160 MB; a page is parsed and dropped as it\n *   is read, so what is held is its blocks, never its bytes.\n * - **Four at a time.** robots.txt, the sitemaps and the home page are read\n *   one after another; the rest four at a time, and the inventory keeps the\n *   order they were asked in, which is the sitemap's.\n * - **Nothing vanishes.** An address a stop (the clock, the bytes, the page\n *   budget) left unread is listed as `not-reached`, so the inventory's\n *   counts add up to the site. A wave's own read leaves its unread pages for\n *   its next pass instead.\n * - **No lookalikes.** A domain that wears a brand it is not\n *   (`lookalikeBrandForHost`) is refused before any request.\n * - **No sign-in pages.** A page with a password field is refused and\n *   reported; forms with payment fields, scripts, iframes and embeds never\n *   reach the inventory (site-import-html.ts).\n *=========================================*/\n\nimport { lookalikeBrandForHost } from '@aglyn/shared-util-email/outbound-phishing-screen'\nimport { fenceUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n  fetchPublicTextDetailed,\n  type PublicFetchEdge,\n  type PublicFetchOptions,\n  type PublicFetchResult,\n} from '../jobs/ai-theme-brand-inputs'\nimport {\n  extractSiteImportPage,\n  readMetaValues,\n  type SiteImportBlock,\n  type SiteImportBrandMark,\n  type SiteImportForm,\n  type SiteImportIcon,\n  type SiteImportImage,\n  type SiteImportNavLink,\n  type SiteImportOrganization,\n} from './site-import-html'\nimport {\n  AGLYN_BOT_TOKEN,\n  parseRobotsTxt,\n  parseSitemap,\n  robotsAllows,\n  robotsRulesFor,\n  type RobotsRule,\n} from './site-import-robots'\n\n/**\n * Where the bot's user agent points a site owner who wants to know what it is\n * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on\n * a self-hosted deployment, so its crawler never names Aglyn's address; the\n * literal is only that variable's default.\n *\n * Aglyn's is the Bring my site guide, which says what AglynBot reads and how\n * robots.txt keeps it out. There is no page of its own (Zach, 10/20: \"we don't\n * need a bot page at all\"), and the site it reads is the reader's own: an\n * import starts only once its owner has proved the site is theirs.\n */\nexport const AGLYN_BOT_INFO_URL =\n  process.env['SITE_IMPORT_BOT_INFO_URL'] ?? 'https://docs.aglyn.com/ai/bring-my-site'\n\n/** What every crawler request says it is. */\nexport const AGLYN_BOT_USER_AGENT = `AglynBot/1.0 (+${AGLYN_BOT_INFO_URL})`\n\nexport const SITE_IMPORT_DEFAULTS = {\n  maxPageBytes: 2 * 1024 * 1024,\n  /** A whole site's worth (AGL-4083): 54 MB for aglyn.com's 123 pages, with room. */\n  maxTotalBytes: 160 * 1024 * 1024,\n  timeBudgetMs: 60_000,\n  /** One request's own clock, inside the whole crawl's. */\n  requestTimeoutMs: 8_000,\n  robotsMaxBytes: 256 * 1024,\n  sitemapMaxBytes: 4 * 1024 * 1024,\n  /** Sitemap documents read, an index's children included: an index of six children is seven. */\n  maxSitemaps: 24,\n  /** URLs a sitemap may contribute to the queue. */\n  maxSitemapUrls: 2_000,\n  /** Reasons kept in `skipped` before the list stops growing: every address a sitemap may name. */\n  maxSkipped: 2_000,\n  /** Pages read at once, after robots.txt, the sitemaps and the home page. */\n  concurrency: 4,\n} as const\n\n/** Never a page: files a link may point at that are not HTML. */\nconst NON_PAGE_EXTENSION =\n  /\\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i\n\nconst TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i\n\nconst DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\\.[a-z0-9-]{1,63})+$/\n\nexport type SiteImportSkipReason =\n  /** The page has a password field: a sign-in page, refused outright. */\n  | 'login-form'\n  /** robots.txt keeps AglynBot off this path. */\n  | 'robots'\n  /** A redirect left the site. */\n  | 'off-origin-redirect'\n  /** Over the per-page byte cap. */\n  | 'too-large'\n  /** Answered with something that is not HTML. */\n  | 'not-html'\n  /** Answered 4xx or 5xx. */\n  | 'http-error'\n  /** No answer, a private address, or a refused address. */\n  | 'unreachable'\n  /** Never asked for: a stop (time, bytes, the page budget) came first (AGL-4083). */\n  | 'not-reached'\n\nexport interface SiteImportSkipped {\n  url: string\n  reason: SiteImportSkipReason\n  status?: number\n  /** Where an off-origin redirect pointed. */\n  location?: string\n  /** Whose edge turned an error answer away, when it said (AGL-4083). */\n  edge?: PublicFetchEdge\n}\n\nexport interface SiteImportPage {\n  /** The address the page was read from, after any same-site redirect. */\n  url: string\n  path: string\n  /** Where the crawler learned of the page. */\n  foundBy: 'home' | 'sitemap' | 'link'\n  title: string\n  description: string\n  lang: string | null\n  canonical: string | null\n  /** The share card (AGL-4083). */\n  ogImage: string | null\n  /** The site's own name, as its head gives it. */\n  siteName: string\n  icons: SiteImportIcon[]\n  /** The header's link home: the brand's logo and words. */\n  brand: SiteImportBrandMark | null\n  organization: SiteImportOrganization | null\n  blocks: SiteImportBlock[]\n  images: SiteImportImage[]\n  navLinks: SiteImportNavLink[]\n  /** Same-site page addresses this page links to. */\n  links: string[]\n  forms: { kept: SiteImportForm[]; removed: number }\n  bytes: number\n}\n\nexport type SiteImportStop =\n  | 'page-budget'\n  | 'total-bytes'\n  | 'time'\n  /** robots.txt could not be read, so nothing was asked of the site. */\n  | 'robots-unreachable'\n\nexport interface SiteImportInventory {\n  domain: string\n  /** The site's origins: the bare name and its `www.` twin. */\n  origins: string[]\n  /** Set when nothing was fetched at all. */\n  refused: 'invalid-domain' | 'lookalike' | null\n  pages: SiteImportPage[]\n  skipped: SiteImportSkipped[]\n  /** Skips past {@link SITE_IMPORT_DEFAULTS.maxSkipped}, counted only. */\n  skippedOverflow: number\n  robots: { found: boolean; sitemaps: string[] }\n  /** Page addresses the sitemaps listed (same-site, before robots and budget). */\n  sitemapUrls: number\n  stoppedBy: SiteImportStop | null\n  bytesRead: number\n  startedAt: number\n  finishedAt: number\n}\n\nexport interface SiteImportCrawlOptions {\n  /** The proved domain; `www.` is folded in. */\n  domain: string\n  /** The most pages the inventory may hold: the plan's page count. */\n  maxPages: number\n  maxPageBytes?: number\n  maxTotalBytes?: number\n  timeBudgetMs?: number\n  signal?: AbortSignal\n  /**\n   * Read the sitemap's shallow addresses before its deep ones (AGL-3976), so a\n   * crawl the page budget stops reads a site's main pages before the tail of\n   * its blog. A sitemap's own order is kept among addresses of one depth.\n   */\n  shallowFirst?: boolean\n  /**\n   * Read exactly these addresses of the site, in order, and nothing else\n   * (AGL-3976): an import's wave reads its own pages again when it starts. No\n   * sitemap is read and no link is followed; robots.txt is read and honored\n   * as for any crawl, and every other rule holds.\n   */\n  only?: readonly string[]\n}\n\nexport interface SiteImportCrawlSeams {\n  fetch?: (url: string, options: PublicFetchOptions) => Promise<PublicFetchResult>\n  now?: () => number\n  /**\n   * Request headers for the site being read, asked once per crawl with its\n   * domain (AGL-4083): the edge header a site this platform serves lets\n   * AglynBot past its own bot protection with. Sent on every request to the\n   * site, under the crawler's own `Accept` and `User-Agent`, and only to the\n   * site: every address the crawler asks for is on it. None when absent.\n   */\n  headersFor?: (domain: string) => Promise<Readonly<Record<string, string>>> | Readonly<Record<string, string>>\n}\n\n/** The seam's headers for a domain; none when it has none or cannot say. */\nasync function siteImportHeadersFor(seams: SiteImportCrawlSeams, domain: string): Promise<Readonly<Record<string, string>> | null> {\n  if (!seams.headersFor) return null\n  try {\n    const headers = await seams.headersFor(domain)\n    return headers && Object.keys(headers).length ? headers : null\n  } catch (error) {\n    console.warn('[site-import] the edge headers could not be read; reading without them', { domain, error })\n    return null\n  }\n}\n\n/** The site's two origins for a domain. */\nexport function siteImportOrigins(domain: string): string[] {\n  const bare = domain.replace(/^www\\./, '')\n  return [`https://${bare}`, `https://www.${bare}`]\n}\n\n/** A crawlable page address on the site, normalized for de-duplication; else null. */\nexport function siteImportPageKey(url: URL, origins: readonly string[]): string | null {\n  if (url.protocol !== 'https:' || url.username || url.password) return null\n  if (url.port && url.port !== '443') return null\n  if (!origins.includes(url.origin)) return null\n  if (NON_PAGE_EXTENSION.test(url.pathname)) return null\n  const clean = new URL(url.toString())\n  clean.hash = ''\n  for (const key of [...clean.searchParams.keys()]) {\n    if (TRACKING_PARAM.test(key)) clean.searchParams.delete(key)\n  }\n  clean.search = clean.searchParams.toString() ? `?${clean.searchParams.toString()}` : ''\n  return clean.toString()\n}\n\nfunction skipReasonFor(result: PublicFetchResult): SiteImportSkipReason {\n  switch (result.refusal) {\n    case 'redirect-refused':\n      return 'off-origin-redirect'\n    case 'content-type':\n      return 'not-html'\n    case 'too-large':\n      return 'too-large'\n    case 'http-status':\n      return 'http-error'\n    default:\n      return 'unreachable'\n  }\n}\n\n/** Read a proved site into an inventory. Never throws for anything the site does. */\nexport async function crawlSiteForImport(\n  options: SiteImportCrawlOptions,\n  seams: SiteImportCrawlSeams = {},\n): Promise<SiteImportInventory> {\n  const now = seams.now ?? Date.now\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const domain = String(options.domain ?? '').trim().toLowerCase().replace(/\\.+$/, '').replace(/^www\\./, '')\n  const origins = DOMAIN_PATTERN.test(domain) ? siteImportOrigins(domain) : []\n  const startedAt = now()\n  const inventory: SiteImportInventory = {\n    domain,\n    origins,\n    refused: null,\n    pages: [],\n    skipped: [],\n    skippedOverflow: 0,\n    robots: { found: false, sitemaps: [] },\n    sitemapUrls: 0,\n    stoppedBy: null,\n    bytesRead: 0,\n    startedAt,\n    finishedAt: startedAt,\n  }\n  const finish = () => {\n    inventory.finishedAt = now()\n    return inventory\n  }\n  if (!origins.length) {\n    inventory.refused = 'invalid-domain'\n    return finish()\n  }\n  if (lookalikeBrandForHost(domain)) {\n    inventory.refused = 'lookalike'\n    return finish()\n  }\n\n  const maxPages = Math.max(0, Math.floor(options.maxPages))\n  const maxPageBytes = options.maxPageBytes ?? SITE_IMPORT_DEFAULTS.maxPageBytes\n  const maxTotalBytes = options.maxTotalBytes ?? SITE_IMPORT_DEFAULTS.maxTotalBytes\n  const deadlineAt = startedAt + (options.timeBudgetMs ?? SITE_IMPORT_DEFAULTS.timeBudgetMs)\n  const controller = new AbortController()\n  const onAbort = () => controller.abort()\n  options.signal?.addEventListener('abort', onAbort)\n  if (options.signal?.aborted) controller.abort()\n  const timer = setTimeout(() => controller.abort(), Math.max(0, deadlineAt - startedAt))\n\n  const sameSite = (url: URL) => origins.includes(url.origin) && url.protocol === 'https:'\n  const remainingBytes = () => maxTotalBytes - inventory.bytesRead\n  const outOfTime = () => controller.signal.aborted || now() >= deadlineAt\n  // Asked once, before anything else; every address below is on the site.\n  const extraHeaders = await siteImportHeadersFor(seams, domain)\n  const request = (url: string, accept: RegExp, acceptHeader: string, maxBytes: number) =>\n    fetchText(url, {\n      maxBytes,\n      accept,\n      acceptHeader,\n      userAgent: AGLYN_BOT_USER_AGENT,\n      signal: controller.signal,\n      timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),\n      redirectAllowed: (_from, to) => sameSite(to),\n      refuseOver: true,\n      ...(extraHeaders ? { headers: extraHeaders } : {}),\n    })\n  const skip = (entry: SiteImportSkipped) => {\n    if (inventory.skipped.length < SITE_IMPORT_DEFAULTS.maxSkipped) inventory.skipped.push(entry)\n    else inventory.skippedOverflow += 1\n  }\n\n  try {\n    // robots.txt, before anything else is asked of the site.\n    const home = `${origins[0]}/`\n    let rules: RobotsRule[] = []\n    const robots = await request(\n      `${origins[0]}/robots.txt`,\n      /text\\/plain|text\\/html|^$/i,\n      'text/plain',\n      Math.min(SITE_IMPORT_DEFAULTS.robotsMaxBytes, remainingBytes()),\n    )\n    if (robots.ok) {\n      inventory.bytesRead += robots.bytes\n      const parsed = parseRobotsTxt(robots.text)\n      rules = robotsRulesFor(parsed, AGLYN_BOT_TOKEN)\n      inventory.robots = { found: true, sitemaps: parsed.sitemaps }\n    } else if (\n      !(robots.refusal === 'http-status' && (robots.status ?? 0) >= 400 && (robots.status ?? 0) < 500)\n    ) {\n      inventory.stoppedBy = outOfTime() ? 'time' : 'robots-unreachable'\n      return finish()\n    }\n    const allowed = (key: string) => {\n      const url = new URL(key)\n      return robotsAllows(rules, `${url.pathname}${url.search}`)\n    }\n\n    // A wave's own pages (AGL-3976): exactly those, after robots.txt.\n    const only = options.only\n      ? options.only\n          .map((raw) => {\n            try {\n              return siteImportPageKey(new URL(raw), origins)\n            } catch {\n              return null\n            }\n          })\n          .filter((key): key is string => !!key)\n      : null\n\n    // Sitemaps: the ones robots.txt names on this site, else /sitemap.xml.\n    const sitemapQueue: string[] = []\n    for (const raw of inventory.robots.sitemaps) {\n      try {\n        const url = new URL(raw)\n        if (sameSite(url)) sitemapQueue.push(url.toString())\n      } catch {\n        // A malformed Sitemap line names nothing.\n      }\n    }\n    if (!sitemapQueue.length && !only) sitemapQueue.push(`${origins[0]}/sitemap.xml`)\n    if (only) sitemapQueue.length = 0\n    const sitemapSeen = new Set<string>()\n    const fromSitemap: string[] = []\n    let sitemapsRead = 0\n    while (sitemapQueue.length && sitemapsRead < SITE_IMPORT_DEFAULTS.maxSitemaps && !outOfTime()) {\n      const next = sitemapQueue.shift() as string\n      if (sitemapSeen.has(next)) continue\n      sitemapSeen.add(next)\n      if (/\\.gz$/i.test(new URL(next).pathname)) continue\n      if (!allowed(next)) continue\n      const room = Math.min(SITE_IMPORT_DEFAULTS.sitemapMaxBytes, remainingBytes())\n      if (room <= 0) break\n      sitemapsRead += 1\n      const sitemap = await request(next, /xml|text\\/plain/i, 'application/xml, text/xml;q=0.9', room)\n      if (!sitemap.ok) continue\n      inventory.bytesRead += sitemap.bytes\n      const entries = parseSitemap(sitemap.text)\n      for (const location of entries.locations) {\n        let url: URL\n        try {\n          url = new URL(location)\n        } catch {\n          continue\n        }\n        if (entries.index) {\n          if (sameSite(url)) sitemapQueue.push(url.toString())\n          continue\n        }\n        const key = siteImportPageKey(url, origins)\n        if (key && fromSitemap.length < SITE_IMPORT_DEFAULTS.maxSitemapUrls) fromSitemap.push(key)\n      }\n    }\n    inventory.sitemapUrls = fromSitemap.length\n    if (options.shallowFirst) {\n      const depth = (key: string) => new URL(key).pathname.split('/').filter(Boolean).length\n      const ranked = fromSitemap.map((key, order) => ({ key, order, depth: depth(key) }))\n      ranked.sort((a, b) => a.depth - b.depth || a.order - b.order)\n      fromSitemap.splice(0, fromSitemap.length, ...ranked.map((entry) => entry.key))\n    }\n\n    // Pages: home, then the sitemap's, then links the pages themselves name.\n    // A wave's read is its own pages alone.\n    const queue: { key: string; foundBy: SiteImportPage['foundBy'] }[] = only\n      ? only.map((key) => ({ key, foundBy: (new URL(key).pathname === '/' ? 'home' : 'link') as SiteImportPage['foundBy'] }))\n      : [{ key: home, foundBy: 'home' }]\n    const queued = new Set<string>(only ?? [home])\n    for (const key of fromSitemap) {\n      if (queued.has(key)) continue\n      queued.add(key)\n      queue.push({ key, foundBy: 'sitemap' })\n    }\n    const read = new Set<string>()\n    // The site's two names are one site: once the home page says which one\n    // it answers on, every address is read on that one, so /about is not\n    // read twice as example.com/about and www.example.com/about.\n    // A wave's read starts on the name its pages were read on the first time.\n    let preferred = only?.length ? new URL(only[0]).origin : origins[0]\n    const canon = (raw: string) => {\n      const url = new URL(raw)\n      return origins.includes(url.origin) && url.origin !== preferred\n        ? `${preferred}${url.pathname}${url.search}`\n        : raw\n    }\n    const maxAttempts = maxPages * 3 + 10\n    let attempts = 0\n    type Asked = { next: (typeof queue)[number]; key: string; cap: number; page: PublicFetchResult }\n    /** The pages asked for and not yet taken in, in the order they were asked. */\n    const window: Array<Promise<Asked>> = []\n    /** The bytes the asked pages may still read: what is left is shared, never promised twice. */\n    let reserved = 0\n    let stop: SiteImportStop | null = null\n    /** Asked for and never read, because a stop came first: listed as not reached. */\n    const cutOff: string[] = []\n    /** Home alone first, so its answer settles which of the site's two names the rest are read on. */\n    let homeIn = !!only\n    /** Why no more may be asked now: a stop, `wait` for an asked page to come in first, or nothing. */\n    const held = (): SiteImportStop | 'wait' | null => {\n      if (inventory.pages.length >= maxPages) return 'page-budget'\n      if (outOfTime()) return 'time'\n      if (remainingBytes() <= 0) return 'total-bytes'\n      if (attempts >= maxAttempts) return 'page-budget'\n      if (inventory.pages.length + window.length >= maxPages) return 'wait'\n      if (remainingBytes() - reserved <= 0) return 'wait'\n      return null\n    }\n    /** Takes one asked page in: the inventory's next page, a skip, or the reason the read must stop. */\n    const takeIn = ({ next, key, cap, page }: Asked): SiteImportStop | null => {\n      const foundBy = next.foundBy\n      if (foundBy === 'home') homeIn = true\n      if (!page.ok) {\n        if (page.refusal === 'too-large' && cap < maxPageBytes) {\n          cutOff.push(key)\n          return 'total-bytes'\n        }\n        if (page.refusal === 'network' && outOfTime()) {\n          cutOff.push(key)\n          return 'time'\n        }\n        skip({\n          url: key,\n          reason: skipReasonFor(page),\n          ...(page.status ? { status: page.status } : {}),\n          ...(page.location ? { location: page.location } : {}),\n          ...(page.edge ? { edge: page.edge } : {}),\n        })\n        return null\n      }\n      inventory.bytesRead += page.bytes\n      if (foundBy === 'home') preferred = new URL(page.url).origin\n      const finalKey = canon(siteImportPageKey(new URL(page.url), origins) ?? page.url)\n      if (finalKey !== key) {\n        if (read.has(finalKey)) return null\n        read.add(finalKey)\n        if (!allowed(finalKey)) {\n          skip({ url: finalKey, reason: 'robots' })\n          return null\n        }\n      }\n      const extract = extractSiteImportPage(page.text, page.url, sameSite)\n      if (extract.loginForm) {\n        skip({ url: finalKey, reason: 'login-form' })\n        return null\n      }\n      const links: string[] = []\n      for (const link of extract.links) {\n        const linkKey = siteImportPageKey(new URL(link), origins)\n        if (!linkKey) continue\n        links.push(linkKey)\n        if (!only && !queued.has(linkKey)) {\n          queued.add(linkKey)\n          queue.push({ key: linkKey, foundBy: 'link' })\n        }\n      }\n      inventory.pages.push({\n        url: finalKey,\n        path: new URL(finalKey).pathname,\n        foundBy,\n        title: extract.title,\n        description: extract.description,\n        lang: extract.lang,\n        canonical: extract.canonical,\n        ogImage: extract.ogImage,\n        siteName: extract.siteName,\n        icons: extract.icons,\n        brand: extract.brand,\n        organization: extract.organization,\n        blocks: extract.blocks,\n        images: extract.images,\n        navLinks: extract.navLinks,\n        links: [...new Set(links.map(canon))],\n        forms: extract.forms,\n        bytes: page.bytes,\n      })\n      return null\n    }\n    for (;;) {\n      const width = homeIn ? SITE_IMPORT_DEFAULTS.concurrency : 1\n      while (!stop && queue.length && window.length < width) {\n        const why = held()\n        if (why === 'wait') break\n        if (why) {\n          stop = why\n          break\n        }\n        const next = queue.shift() as (typeof queue)[number]\n        const key = canon(next.key)\n        if (read.has(key)) continue\n        if (!allowed(key)) {\n          skip({ url: key, reason: 'robots' })\n          if (next.foundBy === 'home') homeIn = true\n          continue\n        }\n        attempts += 1\n        read.add(key)\n        const cap = Math.min(maxPageBytes, remainingBytes() - reserved)\n        reserved += cap\n        window.push(\n          request(key, /text\\/html|application\\/xhtml\\+xml/i, 'text/html, application/xhtml+xml;q=0.9', cap).then((page) => ({ next, key, cap, page })),\n        )\n      }\n      if (!window.length) break\n      const asked = await (window.shift() as Promise<Asked>)\n      reserved -= asked.cap\n      stop = takeIn(asked) ?? stop\n    }\n    inventory.stoppedBy = stop\n    // Whatever a stop left unread is said, never dropped; a wave's own read\n    // leaves its unread pages for its next pass instead.\n    if (stop && !only) {\n      const unread = [...cutOff]\n      for (const next of queue) {\n        const key = canon(next.key)\n        if (!read.has(key) && !unread.includes(key)) unread.push(key)\n      }\n      for (const url of unread) skip({ url, reason: 'not-reached' })\n    }\n    return finish()\n  } finally {\n    clearTimeout(timer)\n    options.signal?.removeEventListener('abort', onAbort)\n  }\n}\n\n/**\n * The home page's `aglyn-site-verification` values, for the meta-tag proof:\n * the same fetch, user agent and same-site redirects as the crawl, but no\n * robots.txt, because the person asked for this one page to be read.\n * `answered: false` when the page could not be read at all.\n */\nexport async function readSiteVerificationMeta(\n  domain: string,\n  metaName: string,\n  seams: SiteImportCrawlSeams = {},\n): Promise<{ answered: boolean; values: string[]; url?: string }> {\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const bare = String(domain ?? '').trim().toLowerCase().replace(/^www\\./, '')\n  if (!DOMAIN_PATTERN.test(bare) || lookalikeBrandForHost(bare)) return { answered: false, values: [] }\n  const origins = siteImportOrigins(bare)\n  const extraHeaders = await siteImportHeadersFor(seams, bare)\n  const result = await fetchText(`${origins[0]}/`, {\n    maxBytes: SITE_IMPORT_DEFAULTS.maxPageBytes,\n    accept: /text\\/html|application\\/xhtml\\+xml/i,\n    acceptHeader: 'text/html, application/xhtml+xml;q=0.9',\n    userAgent: AGLYN_BOT_USER_AGENT,\n    timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,\n    redirectAllowed: (_from, to) => to.protocol === 'https:' && origins.includes(to.origin),\n    ...(extraHeaders ? { headers: extraHeaders } : {}),\n  })\n  if (!result.ok) {\n    // An answer that is not the page (a 404, a redirect off the site) is\n    // still an answer: the tag is not where it must be.\n    const answered = result.refusal === 'http-status' || result.refusal === 'redirect-refused' || result.refusal === 'content-type'\n    return { answered, values: [], url: result.url }\n  }\n  const head = result.text.split(/<body\\b/i)[0] ?? ''\n  return { answered: true, values: readMetaValues(head, metaName), url: result.url }\n}\n\n/**\n * One page as text for a prompt: its title, description and blocks in order,\n * fenced as untrusted data. The importer's prompt carries `UNTRUSTED_TEXT_RULE`.\n */\nexport function siteImportPagePromptText(page: SiteImportPage, maxChars = 12_000): string {\n  const lines: string[] = []\n  if (page.title) lines.push(`Title: ${page.title}`)\n  if (page.description) lines.push(`Description: ${page.description}`)\n  for (const block of page.blocks) {\n    if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n    else if (block.kind === 'image') lines.push(`[image: ${block.alt || 'no description'}]`)\n    else if (block.kind === 'list-item') lines.push(`- ${block.text}`)\n    else if (block.kind === 'quote') lines.push(`> ${block.text}`)\n    else if (block.kind === 'link') {\n      // A link inside a sentence is in its block's words already.\n      if (!block.inline) lines.push(block.text)\n    } else lines.push(block.text)\n  }\n  return fenceUntrustedText(`Page ${page.path} of the site being imported`, lines.join('\\n'), { maxChars })\n}\n"],"names":["process","lookalikeBrandForHost","fenceUntrustedText","fetchPublicTextDetailed","extractSiteImportPage","readMetaValues","AGLYN_BOT_TOKEN","parseRobotsTxt","parseSitemap","robotsAllows","robotsRulesFor","AGLYN_BOT_INFO_URL","env","AGLYN_BOT_USER_AGENT","SITE_IMPORT_DEFAULTS","maxPageBytes","maxTotalBytes","timeBudgetMs","requestTimeoutMs","robotsMaxBytes","sitemapMaxBytes","maxSitemaps","maxSitemapUrls","maxSkipped","concurrency","NON_PAGE_EXTENSION","TRACKING_PARAM","DOMAIN_PATTERN","siteImportHeadersFor","seams","domain","headersFor","headers","Object","keys","length","error","console","warn","siteImportOrigins","bare","replace","siteImportPageKey","url","origins","protocol","username","password","port","includes","origin","test","pathname","clean","URL","toString","hash","key","searchParams","delete","search","skipReasonFor","result","refusal","crawlSiteForImport","options","now","Date","fetchText","fetch","String","trim","toLowerCase","startedAt","inventory","refused","pages","skipped","skippedOverflow","robots","found","sitemaps","sitemapUrls","stoppedBy","bytesRead","finishedAt","finish","maxPages","Math","max","floor","deadlineAt","controller","AbortController","onAbort","abort","signal","addEventListener","aborted","timer","setTimeout","sameSite","remainingBytes","outOfTime","extraHeaders","request","accept","acceptHeader","maxBytes","userAgent","timeoutMs","min","redirectAllowed","_from","to","refuseOver","skip","entry","push","home","rules","ok","bytes","parsed","text","status","allowed","only","map","raw","filter","sitemapQueue","sitemapSeen","Set","fromSitemap","sitemapsRead","next","shift","has","add","room","sitemap","entries","location","locations","index","shallowFirst","depth","split","Boolean","ranked","order","sort","a","b","splice","queue","foundBy","queued","read","preferred","canon","maxAttempts","attempts","window","reserved","stop","cutOff","homeIn","held","takeIn","cap","page","reason","edge","finalKey","extract","loginForm","links","link","linkKey","path","title","description","lang","canonical","ogImage","siteName","icons","brand","organization","blocks","images","navLinks","forms","width","why","then","asked","unread","clearTimeout","removeEventListener","readSiteVerificationMeta","metaName","answered","values","head","siteImportPagePromptText","maxChars","lines","block","kind","repeat","level","alt","inline","join"],"mappings":";IAoGEA;AApGF;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2CAyC2C,GAE3C,SAASC,qBAAqB,QAAQ,oDAAmD;AACzF,SAASC,kBAAkB,QAAQ,0CAAyC;AAC5E,SACEC,uBAAuB,QAIlB,mCAA+B;AACtC,SACEC,qBAAqB,EACrBC,cAAc,QAQT,wBAAoB;AAC3B,SACEC,eAAe,EACfC,cAAc,EACdC,YAAY,EACZC,YAAY,EACZC,cAAc,QAET,0BAAsB;AAE7B;;;;;;;;;;CAUC,GACD,OAAO,MAAMC,sBACXX,wCAAAA,QAAQY,GAAG,CAAC,2BAA2B,YAAvCZ,wCAA2C,0CAAyC;AAEtF,2CAA2C,GAC3C,OAAO,MAAMa,uBAAuB,CAAC,eAAe,EAAEF,mBAAmB,CAAC,CAAC,CAAA;AAE3E,OAAO,MAAMG,uBAAuB;IAClCC,cAAc,IAAI,OAAO;IACzB,iFAAiF,GACjFC,eAAe,MAAM,OAAO;IAC5BC,cAAc;IACd,uDAAuD,GACvDC,kBAAkB;IAClBC,gBAAgB,MAAM;IACtBC,iBAAiB,IAAI,OAAO;IAC5B,6FAA6F,GAC7FC,aAAa;IACb,gDAAgD,GAChDC,gBAAgB;IAChB,+FAA+F,GAC/FC,YAAY;IACZ,0EAA0E,GAC1EC,aAAa;AACf,EAAU;AAEV,+DAA+D,GAC/D,MAAMC,qBACJ;AAEF,MAAMC,iBAAiB;AAEvB,MAAMC,iBAAiB;AAwHvB,0EAA0E,GAC1E,eAAeC,qBAAqBC,KAA2B,EAAEC,MAAc;IAC7E,IAAI,CAACD,MAAME,UAAU,EAAE,OAAO;IAC9B,IAAI;QACF,MAAMC,UAAU,MAAMH,MAAME,UAAU,CAACD;QACvC,OAAOE,WAAWC,OAAOC,IAAI,CAACF,SAASG,MAAM,GAAGH,UAAU;IAC5D,EAAE,OAAOI,OAAO;QACdC,QAAQC,IAAI,CAAC,0EAA0E;YAAER;YAAQM;QAAM;QACvG,OAAO;IACT;AACF;AAEA,yCAAyC,GACzC,OAAO,SAASG,kBAAkBT,MAAc;IAC9C,MAAMU,OAAOV,OAAOW,OAAO,CAAC,UAAU;IACtC,OAAO;QAAC,CAAC,QAAQ,EAAED,MAAM;QAAE,CAAC,YAAY,EAAEA,MAAM;KAAC;AACnD;AAEA,oFAAoF,GACpF,OAAO,SAASE,kBAAkBC,GAAQ,EAAEC,OAA0B;IACpE,IAAID,IAAIE,QAAQ,KAAK,YAAYF,IAAIG,QAAQ,IAAIH,IAAII,QAAQ,EAAE,OAAO;IACtE,IAAIJ,IAAIK,IAAI,IAAIL,IAAIK,IAAI,KAAK,OAAO,OAAO;IAC3C,IAAI,CAACJ,QAAQK,QAAQ,CAACN,IAAIO,MAAM,GAAG,OAAO;IAC1C,IAAIzB,mBAAmB0B,IAAI,CAACR,IAAIS,QAAQ,GAAG,OAAO;IAClD,MAAMC,QAAQ,IAAIC,IAAIX,IAAIY,QAAQ;IAClCF,MAAMG,IAAI,GAAG;IACb,KAAK,MAAMC,OAAO;WAAIJ,MAAMK,YAAY,CAACxB,IAAI;KAAG,CAAE;QAChD,IAAIR,eAAeyB,IAAI,CAACM,MAAMJ,MAAMK,YAAY,CAACC,MAAM,CAACF;IAC1D;IACAJ,MAAMO,MAAM,GAAGP,MAAMK,YAAY,CAACH,QAAQ,KAAK,CAAC,CAAC,EAAEF,MAAMK,YAAY,CAACH,QAAQ,IAAI,GAAG;IACrF,OAAOF,MAAME,QAAQ;AACvB;AAEA,SAASM,cAAcC,MAAyB;IAC9C,OAAQA,OAAOC,OAAO;QACpB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,mFAAmF,GACnF,OAAO,eAAeC,mBACpBC,OAA+B,EAC/BpC,QAA8B,CAAC,CAAC;QAEpBA,YACMA,cACIoC,iBA+BDA,uBACCA,wBACUA;QAGhCA,iBACIA;IAvCJ,MAAMC,OAAMrC,aAAAA,MAAMqC,GAAG,YAATrC,aAAasC,KAAKD,GAAG;IACjC,MAAME,aAAYvC,eAAAA,MAAMwC,KAAK,YAAXxC,eAAe1B;IACjC,MAAM2B,SAASwC,QAAOL,kBAAAA,QAAQnC,MAAM,YAAdmC,kBAAkB,IAAIM,IAAI,GAAGC,WAAW,GAAG/B,OAAO,CAAC,QAAQ,IAAIA,OAAO,CAAC,UAAU;IACvG,MAAMG,UAAUjB,eAAewB,IAAI,CAACrB,UAAUS,kBAAkBT,UAAU,EAAE;IAC5E,MAAM2C,YAAYP;IAClB,MAAMQ,YAAiC;QACrC5C;QACAc;QACA+B,SAAS;QACTC,OAAO,EAAE;QACTC,SAAS,EAAE;QACXC,iBAAiB;QACjBC,QAAQ;YAAEC,OAAO;YAAOC,UAAU,EAAE;QAAC;QACrCC,aAAa;QACbC,WAAW;QACXC,WAAW;QACXX;QACAY,YAAYZ;IACd;IACA,MAAMa,SAAS;QACbZ,UAAUW,UAAU,GAAGnB;QACvB,OAAOQ;IACT;IACA,IAAI,CAAC9B,QAAQT,MAAM,EAAE;QACnBuC,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IACA,IAAIrF,sBAAsB6B,SAAS;QACjC4C,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IAEA,MAAMC,WAAWC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAACzB,QAAQsB,QAAQ;IACxD,MAAMxE,gBAAekD,wBAAAA,QAAQlD,YAAY,YAApBkD,wBAAwBnD,qBAAqBC,YAAY;IAC9E,MAAMC,iBAAgBiD,yBAAAA,QAAQjD,aAAa,YAArBiD,yBAAyBnD,qBAAqBE,aAAa;IACjF,MAAM2E,aAAalB,cAAaR,wBAAAA,QAAQhD,YAAY,YAApBgD,wBAAwBnD,qBAAqBG,YAAY;IACzF,MAAM2E,aAAa,IAAIC;IACvB,MAAMC,UAAU,IAAMF,WAAWG,KAAK;KACtC9B,kBAAAA,QAAQ+B,MAAM,qBAAd/B,gBAAgBgC,gBAAgB,CAAC,SAASH;IAC1C,KAAI7B,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBiC,OAAO,EAAEN,WAAWG,KAAK;IAC7C,MAAMI,QAAQC,WAAW,IAAMR,WAAWG,KAAK,IAAIP,KAAKC,GAAG,CAAC,GAAGE,aAAalB;IAE5E,MAAM4B,WAAW,CAAC1D,MAAaC,QAAQK,QAAQ,CAACN,IAAIO,MAAM,KAAKP,IAAIE,QAAQ,KAAK;IAChF,MAAMyD,iBAAiB,IAAMtF,gBAAgB0D,UAAUU,SAAS;IAChE,MAAMmB,YAAY,IAAMX,WAAWI,MAAM,CAACE,OAAO,IAAIhC,SAASyB;IAC9D,wEAAwE;IACxE,MAAMa,eAAe,MAAM5E,qBAAqBC,OAAOC;IACvD,MAAM2E,UAAU,CAAC9D,KAAa+D,QAAgBC,cAAsBC,WAClExC,UAAUzB,KAAK;YACbiE;YACAF;YACAC;YACAE,WAAWhG;YACXmF,QAAQJ,WAAWI,MAAM;YACzBc,WAAWtB,KAAKC,GAAG,CAAC,GAAGD,KAAKuB,GAAG,CAACjG,qBAAqBI,gBAAgB,EAAEyE,aAAazB;YACpF8C,iBAAiB,CAACC,OAAOC,KAAOb,SAASa;YACzCC,YAAY;WACRX,eAAe;YAAExE,SAASwE;QAAa,IAAI,CAAC;IAEpD,MAAMY,OAAO,CAACC;QACZ,IAAI3C,UAAUG,OAAO,CAAC1C,MAAM,GAAGrB,qBAAqBS,UAAU,EAAEmD,UAAUG,OAAO,CAACyC,IAAI,CAACD;aAClF3C,UAAUI,eAAe,IAAI;IACpC;IAEA,IAAI;YAgBuCC,gBAA+BA;QAfxE,yDAAyD;QACzD,MAAMwC,OAAO,GAAG3E,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,IAAI4E,QAAsB,EAAE;QAC5B,MAAMzC,SAAS,MAAM0B,QACnB,GAAG7D,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAC1B,8BACA,cACA4C,KAAKuB,GAAG,CAACjG,qBAAqBK,cAAc,EAAEmF;QAEhD,IAAIvB,OAAO0C,EAAE,EAAE;YACb/C,UAAUU,SAAS,IAAIL,OAAO2C,KAAK;YACnC,MAAMC,SAASpH,eAAewE,OAAO6C,IAAI;YACzCJ,QAAQ9G,eAAeiH,QAAQrH;YAC/BoE,UAAUK,MAAM,GAAG;gBAAEC,OAAO;gBAAMC,UAAU0C,OAAO1C,QAAQ;YAAC;QAC9D,OAAO,IACL,CAAEF,CAAAA,OAAOhB,OAAO,KAAK,iBAAiB,EAACgB,iBAAAA,OAAO8C,MAAM,YAAb9C,iBAAiB,MAAM,OAAO,EAACA,kBAAAA,OAAO8C,MAAM,YAAb9C,kBAAiB,KAAK,GAAE,GAC9F;YACAL,UAAUS,SAAS,GAAGoB,cAAc,SAAS;YAC7C,OAAOjB;QACT;QACA,MAAMwC,UAAU,CAACrE;YACf,MAAMd,MAAM,IAAIW,IAAIG;YACpB,OAAOhD,aAAa+G,OAAO,GAAG7E,IAAIS,QAAQ,GAAGT,IAAIiB,MAAM,EAAE;QAC3D;QAEA,kEAAkE;QAClE,MAAMmE,OAAO9D,QAAQ8D,IAAI,GACrB9D,QAAQ8D,IAAI,CACTC,GAAG,CAAC,CAACC;YACJ,IAAI;gBACF,OAAOvF,kBAAkB,IAAIY,IAAI2E,MAAMrF;YACzC,EAAE,eAAM;gBACN,OAAO;YACT;QACF,GACCsF,MAAM,CAAC,CAACzE,MAAuB,CAAC,CAACA,OACpC;QAEJ,uEAAuE;QACvE,MAAM0E,eAAyB,EAAE;QACjC,KAAK,MAAMF,OAAOvD,UAAUK,MAAM,CAACE,QAAQ,CAAE;YAC3C,IAAI;gBACF,MAAMtC,MAAM,IAAIW,IAAI2E;gBACpB,IAAI5B,SAAS1D,MAAMwF,aAAab,IAAI,CAAC3E,IAAIY,QAAQ;YACnD,EAAE,eAAM;YACN,0CAA0C;YAC5C;QACF;QACA,IAAI,CAAC4E,aAAahG,MAAM,IAAI,CAAC4F,MAAMI,aAAab,IAAI,CAAC,GAAG1E,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC;QAChF,IAAImF,MAAMI,aAAahG,MAAM,GAAG;QAChC,MAAMiG,cAAc,IAAIC;QACxB,MAAMC,cAAwB,EAAE;QAChC,IAAIC,eAAe;QACnB,MAAOJ,aAAahG,MAAM,IAAIoG,eAAezH,qBAAqBO,WAAW,IAAI,CAACkF,YAAa;YAC7F,MAAMiC,OAAOL,aAAaM,KAAK;YAC/B,IAAIL,YAAYM,GAAG,CAACF,OAAO;YAC3BJ,YAAYO,GAAG,CAACH;YAChB,IAAI,SAASrF,IAAI,CAAC,IAAIG,IAAIkF,MAAMpF,QAAQ,GAAG;YAC3C,IAAI,CAAC0E,QAAQU,OAAO;YACpB,MAAMI,OAAOpD,KAAKuB,GAAG,CAACjG,qBAAqBM,eAAe,EAAEkF;YAC5D,IAAIsC,QAAQ,GAAG;YACfL,gBAAgB;YAChB,MAAMM,UAAU,MAAMpC,QAAQ+B,MAAM,oBAAoB,mCAAmCI;YAC3F,IAAI,CAACC,QAAQpB,EAAE,EAAE;YACjB/C,UAAUU,SAAS,IAAIyD,QAAQnB,KAAK;YACpC,MAAMoB,UAAUtI,aAAaqI,QAAQjB,IAAI;YACzC,KAAK,MAAMmB,YAAYD,QAAQE,SAAS,CAAE;gBACxC,IAAIrG;gBACJ,IAAI;oBACFA,MAAM,IAAIW,IAAIyF;gBAChB,EAAE,eAAM;oBACN;gBACF;gBACA,IAAID,QAAQG,KAAK,EAAE;oBACjB,IAAI5C,SAAS1D,MAAMwF,aAAab,IAAI,CAAC3E,IAAIY,QAAQ;oBACjD;gBACF;gBACA,MAAME,MAAMf,kBAAkBC,KAAKC;gBACnC,IAAIa,OAAO6E,YAAYnG,MAAM,GAAGrB,qBAAqBQ,cAAc,EAAEgH,YAAYhB,IAAI,CAAC7D;YACxF;QACF;QACAiB,UAAUQ,WAAW,GAAGoD,YAAYnG,MAAM;QAC1C,IAAI8B,QAAQiF,YAAY,EAAE;YACxB,MAAMC,QAAQ,CAAC1F,MAAgB,IAAIH,IAAIG,KAAKL,QAAQ,CAACgG,KAAK,CAAC,KAAKlB,MAAM,CAACmB,SAASlH,MAAM;YACtF,MAAMmH,SAAShB,YAAYN,GAAG,CAAC,CAACvE,KAAK8F,QAAW,CAAA;oBAAE9F;oBAAK8F;oBAAOJ,OAAOA,MAAM1F;gBAAK,CAAA;YAChF6F,OAAOE,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEN,KAAK,GAAGO,EAAEP,KAAK,IAAIM,EAAEF,KAAK,GAAGG,EAAEH,KAAK;YAC5DjB,YAAYqB,MAAM,CAAC,GAAGrB,YAAYnG,MAAM,KAAKmH,OAAOtB,GAAG,CAAC,CAACX,QAAUA,MAAM5D,GAAG;QAC9E;QAEA,yEAAyE;QACzE,wCAAwC;QACxC,MAAMmG,QAA+D7B,OACjEA,KAAKC,GAAG,CAAC,CAACvE,MAAS,CAAA;gBAAEA;gBAAKoG,SAAU,IAAIvG,IAAIG,KAAKL,QAAQ,KAAK,MAAM,SAAS;YAAqC,CAAA,KAClH;YAAC;gBAAEK,KAAK8D;gBAAMsC,SAAS;YAAO;SAAE;QACpC,MAAMC,SAAS,IAAIzB,IAAYN,eAAAA,OAAQ;YAACR;SAAK;QAC7C,KAAK,MAAM9D,OAAO6E,YAAa;YAC7B,IAAIwB,OAAOpB,GAAG,CAACjF,MAAM;YACrBqG,OAAOnB,GAAG,CAAClF;YACXmG,MAAMtC,IAAI,CAAC;gBAAE7D;gBAAKoG,SAAS;YAAU;QACvC;QACA,MAAME,OAAO,IAAI1B;QACjB,uEAAuE;QACvE,qEAAqE;QACrE,6DAA6D;QAC7D,0EAA0E;QAC1E,IAAI2B,YAAYjC,CAAAA,wBAAAA,KAAM5F,MAAM,IAAG,IAAImB,IAAIyE,IAAI,CAAC,EAAE,EAAE7E,MAAM,GAAGN,OAAO,CAAC,EAAE;QACnE,MAAMqH,QAAQ,CAAChC;YACb,MAAMtF,MAAM,IAAIW,IAAI2E;YACpB,OAAOrF,QAAQK,QAAQ,CAACN,IAAIO,MAAM,KAAKP,IAAIO,MAAM,KAAK8G,YAClD,GAAGA,YAAYrH,IAAIS,QAAQ,GAAGT,IAAIiB,MAAM,EAAE,GAC1CqE;QACN;QACA,MAAMiC,cAAc3E,WAAW,IAAI;QACnC,IAAI4E,WAAW;QAEf,4EAA4E,GAC5E,MAAMC,SAAgC,EAAE;QACxC,4FAA4F,GAC5F,IAAIC,WAAW;QACf,IAAIC,OAA8B;QAClC,gFAAgF,GAChF,MAAMC,SAAmB,EAAE;QAC3B,gGAAgG,GAChG,IAAIC,SAAS,CAAC,CAACzC;QACf,iGAAiG,GACjG,MAAM0C,OAAO;YACX,IAAI/F,UAAUE,KAAK,CAACzC,MAAM,IAAIoD,UAAU,OAAO;YAC/C,IAAIgB,aAAa,OAAO;YACxB,IAAID,oBAAoB,GAAG,OAAO;YAClC,IAAI6D,YAAYD,aAAa,OAAO;YACpC,IAAIxF,UAAUE,KAAK,CAACzC,MAAM,GAAGiI,OAAOjI,MAAM,IAAIoD,UAAU,OAAO;YAC/D,IAAIe,mBAAmB+D,YAAY,GAAG,OAAO;YAC7C,OAAO;QACT;QACA,kGAAkG,GAClG,MAAMK,SAAS,CAAC,EAAElC,IAAI,EAAE/E,GAAG,EAAEkH,GAAG,EAAEC,IAAI,EAAS;gBAuBtBlI;YAtBvB,MAAMmH,UAAUrB,KAAKqB,OAAO;YAC5B,IAAIA,YAAY,QAAQW,SAAS;YACjC,IAAI,CAACI,KAAKnD,EAAE,EAAE;gBACZ,IAAImD,KAAK7G,OAAO,KAAK,eAAe4G,MAAM5J,cAAc;oBACtDwJ,OAAOjD,IAAI,CAAC7D;oBACZ,OAAO;gBACT;gBACA,IAAImH,KAAK7G,OAAO,KAAK,aAAawC,aAAa;oBAC7CgE,OAAOjD,IAAI,CAAC7D;oBACZ,OAAO;gBACT;gBACA2D,KAAK;oBACHzE,KAAKc;oBACLoH,QAAQhH,cAAc+G;mBAClBA,KAAK/C,MAAM,GAAG;oBAAEA,QAAQ+C,KAAK/C,MAAM;gBAAC,IAAI,CAAC,GACzC+C,KAAK7B,QAAQ,GAAG;oBAAEA,UAAU6B,KAAK7B,QAAQ;gBAAC,IAAI,CAAC,GAC/C6B,KAAKE,IAAI,GAAG;oBAAEA,MAAMF,KAAKE,IAAI;gBAAC,IAAI,CAAC;gBAEzC,OAAO;YACT;YACApG,UAAUU,SAAS,IAAIwF,KAAKlD,KAAK;YACjC,IAAImC,YAAY,QAAQG,YAAY,IAAI1G,IAAIsH,KAAKjI,GAAG,EAAEO,MAAM;YAC5D,MAAM6H,WAAWd,OAAMvH,qBAAAA,kBAAkB,IAAIY,IAAIsH,KAAKjI,GAAG,GAAGC,oBAArCF,qBAAiDkI,KAAKjI,GAAG;YAChF,IAAIoI,aAAatH,KAAK;gBACpB,IAAIsG,KAAKrB,GAAG,CAACqC,WAAW,OAAO;gBAC/BhB,KAAKpB,GAAG,CAACoC;gBACT,IAAI,CAACjD,QAAQiD,WAAW;oBACtB3D,KAAK;wBAAEzE,KAAKoI;wBAAUF,QAAQ;oBAAS;oBACvC,OAAO;gBACT;YACF;YACA,MAAMG,UAAU5K,sBAAsBwK,KAAKhD,IAAI,EAAEgD,KAAKjI,GAAG,EAAE0D;YAC3D,IAAI2E,QAAQC,SAAS,EAAE;gBACrB7D,KAAK;oBAAEzE,KAAKoI;oBAAUF,QAAQ;gBAAa;gBAC3C,OAAO;YACT;YACA,MAAMK,QAAkB,EAAE;YAC1B,KAAK,MAAMC,QAAQH,QAAQE,KAAK,CAAE;gBAChC,MAAME,UAAU1I,kBAAkB,IAAIY,IAAI6H,OAAOvI;gBACjD,IAAI,CAACwI,SAAS;gBACdF,MAAM5D,IAAI,CAAC8D;gBACX,IAAI,CAACrD,QAAQ,CAAC+B,OAAOpB,GAAG,CAAC0C,UAAU;oBACjCtB,OAAOnB,GAAG,CAACyC;oBACXxB,MAAMtC,IAAI,CAAC;wBAAE7D,KAAK2H;wBAASvB,SAAS;oBAAO;gBAC7C;YACF;YACAnF,UAAUE,KAAK,CAAC0C,IAAI,CAAC;gBACnB3E,KAAKoI;gBACLM,MAAM,IAAI/H,IAAIyH,UAAU3H,QAAQ;gBAChCyG;gBACAyB,OAAON,QAAQM,KAAK;gBACpBC,aAAaP,QAAQO,WAAW;gBAChCC,MAAMR,QAAQQ,IAAI;gBAClBC,WAAWT,QAAQS,SAAS;gBAC5BC,SAASV,QAAQU,OAAO;gBACxBC,UAAUX,QAAQW,QAAQ;gBAC1BC,OAAOZ,QAAQY,KAAK;gBACpBC,OAAOb,QAAQa,KAAK;gBACpBC,cAAcd,QAAQc,YAAY;gBAClCC,QAAQf,QAAQe,MAAM;gBACtBC,QAAQhB,QAAQgB,MAAM;gBACtBC,UAAUjB,QAAQiB,QAAQ;gBAC1Bf,OAAO;uBAAI,IAAI7C,IAAI6C,MAAMlD,GAAG,CAACiC;iBAAQ;gBACrCiC,OAAOlB,QAAQkB,KAAK;gBACpBxE,OAAOkD,KAAKlD,KAAK;YACnB;YACA,OAAO;QACT;QACA,OAAS;gBA4BAgD;YA3BP,MAAMyB,QAAQ3B,SAAS1J,qBAAqBU,WAAW,GAAG;YAC1D,MAAO,CAAC8I,QAAQV,MAAMzH,MAAM,IAAIiI,OAAOjI,MAAM,GAAGgK,MAAO;gBACrD,MAAMC,MAAM3B;gBACZ,IAAI2B,QAAQ,QAAQ;gBACpB,IAAIA,KAAK;oBACP9B,OAAO8B;oBACP;gBACF;gBACA,MAAM5D,OAAOoB,MAAMnB,KAAK;gBACxB,MAAMhF,MAAMwG,MAAMzB,KAAK/E,GAAG;gBAC1B,IAAIsG,KAAKrB,GAAG,CAACjF,MAAM;gBACnB,IAAI,CAACqE,QAAQrE,MAAM;oBACjB2D,KAAK;wBAAEzE,KAAKc;wBAAKoH,QAAQ;oBAAS;oBAClC,IAAIrC,KAAKqB,OAAO,KAAK,QAAQW,SAAS;oBACtC;gBACF;gBACAL,YAAY;gBACZJ,KAAKpB,GAAG,CAAClF;gBACT,MAAMkH,MAAMnF,KAAKuB,GAAG,CAAChG,cAAcuF,mBAAmB+D;gBACtDA,YAAYM;gBACZP,OAAO9C,IAAI,CACTb,QAAQhD,KAAK,uCAAuC,0CAA0CkH,KAAK0B,IAAI,CAAC,CAACzB,OAAU,CAAA;wBAAEpC;wBAAM/E;wBAAKkH;wBAAKC;oBAAK,CAAA;YAE9I;YACA,IAAI,CAACR,OAAOjI,MAAM,EAAE;YACpB,MAAMmK,QAAQ,MAAOlC,OAAO3B,KAAK;YACjC4B,YAAYiC,MAAM3B,GAAG;YACrBL,QAAOI,UAAAA,OAAO4B,kBAAP5B,UAAiBJ;QAC1B;QACA5F,UAAUS,SAAS,GAAGmF;QACtB,wEAAwE;QACxE,qDAAqD;QACrD,IAAIA,QAAQ,CAACvC,MAAM;YACjB,MAAMwE,SAAS;mBAAIhC;aAAO;YAC1B,KAAK,MAAM/B,QAAQoB,MAAO;gBACxB,MAAMnG,MAAMwG,MAAMzB,KAAK/E,GAAG;gBAC1B,IAAI,CAACsG,KAAKrB,GAAG,CAACjF,QAAQ,CAAC8I,OAAOtJ,QAAQ,CAACQ,MAAM8I,OAAOjF,IAAI,CAAC7D;YAC3D;YACA,KAAK,MAAMd,OAAO4J,OAAQnF,KAAK;gBAAEzE;gBAAKkI,QAAQ;YAAc;QAC9D;QACA,OAAOvF;IACT,SAAU;YAERrB;QADAuI,aAAarG;SACblC,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBwI,mBAAmB,CAAC,SAAS3G;IAC/C;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAe4G,yBACpB5K,MAAc,EACd6K,QAAgB,EAChB9K,QAA8B,CAAC,CAAC;QAEdA,cAoBLiC;IApBb,MAAMM,aAAYvC,eAAAA,MAAMwC,KAAK,YAAXxC,eAAe1B;IACjC,MAAMqC,OAAO8B,OAAOxC,iBAAAA,SAAU,IAAIyC,IAAI,GAAGC,WAAW,GAAG/B,OAAO,CAAC,UAAU;IACzE,IAAI,CAACd,eAAewB,IAAI,CAACX,SAASvC,sBAAsBuC,OAAO,OAAO;QAAEoK,UAAU;QAAOC,QAAQ,EAAE;IAAC;IACpG,MAAMjK,UAAUL,kBAAkBC;IAClC,MAAMgE,eAAe,MAAM5E,qBAAqBC,OAAOW;IACvD,MAAMsB,SAAS,MAAMM,UAAU,GAAGxB,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;QAC/CgE,UAAU9F,qBAAqBC,YAAY;QAC3C2F,QAAQ;QACRC,cAAc;QACdE,WAAWhG;QACXiG,WAAWhG,qBAAqBI,gBAAgB;QAChD8F,iBAAiB,CAACC,OAAOC,KAAOA,GAAGrE,QAAQ,KAAK,YAAYD,QAAQK,QAAQ,CAACiE,GAAGhE,MAAM;OAClFsD,eAAe;QAAExE,SAASwE;IAAa,IAAI,CAAC;IAElD,IAAI,CAAC1C,OAAO2D,EAAE,EAAE;QACd,qEAAqE;QACrE,oDAAoD;QACpD,MAAMmF,WAAW9I,OAAOC,OAAO,KAAK,iBAAiBD,OAAOC,OAAO,KAAK,sBAAsBD,OAAOC,OAAO,KAAK;QACjH,OAAO;YAAE6I;YAAUC,QAAQ,EAAE;YAAElK,KAAKmB,OAAOnB,GAAG;QAAC;IACjD;IACA,MAAMmK,QAAOhJ,sBAAAA,OAAO8D,IAAI,CAACwB,KAAK,CAAC,WAAW,CAAC,EAAE,YAAhCtF,sBAAoC;IACjD,OAAO;QAAE8I,UAAU;QAAMC,QAAQxM,eAAeyM,MAAMH;QAAWhK,KAAKmB,OAAOnB,GAAG;IAAC;AACnF;AAEA;;;CAGC,GACD,OAAO,SAASoK,yBAAyBnC,IAAoB,EAAEoC,WAAW,KAAM;IAC9E,MAAMC,QAAkB,EAAE;IAC1B,IAAIrC,KAAKU,KAAK,EAAE2B,MAAM3F,IAAI,CAAC,CAAC,OAAO,EAAEsD,KAAKU,KAAK,EAAE;IACjD,IAAIV,KAAKW,WAAW,EAAE0B,MAAM3F,IAAI,CAAC,CAAC,aAAa,EAAEsD,KAAKW,WAAW,EAAE;IACnE,KAAK,MAAM2B,SAAStC,KAAKmB,MAAM,CAAE;QAC/B,IAAImB,MAAMC,IAAI,KAAK,WAAWF,MAAM3F,IAAI,CAAC,GAAG,IAAI8F,MAAM,CAACF,MAAMG,KAAK,EAAE,CAAC,EAAEH,MAAMtF,IAAI,EAAE;aAC9E,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,QAAQ,EAAE4F,MAAMI,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAClF,IAAIJ,MAAMC,IAAI,KAAK,aAAaF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aAC5D,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aACxD,IAAIsF,MAAMC,IAAI,KAAK,QAAQ;YAC9B,4DAA4D;YAC5D,IAAI,CAACD,MAAMK,MAAM,EAAEN,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;QAC1C,OAAOqF,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;IAC9B;IACA,OAAO1H,mBAAmB,CAAC,KAAK,EAAE0K,KAAKS,IAAI,CAAC,2BAA2B,CAAC,EAAE4B,MAAMO,IAAI,CAAC,OAAO;QAAER;IAAS;AACzG"}
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-crawl.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * THE SITE-IMPORT CRAWLER (AGL-3974, parent AGL-3973).\n *\n * Reads a website a workspace has PROVED it owns (domain-ownership.ts in\n * tenant-data-admin; the doors check the proof before calling this) into a\n * typed inventory: each page's title, description, headings and text in\n * order, images and menu links. It writes nothing; the import job decides\n * what becomes a page.\n *\n * Every request goes through `fetchPublicTextDetailed`, the theme job's\n * SSRF-guarded fetch: https on the default port only, public addresses only,\n * the socket pinned to the address checked. On top of that:\n *\n * - **One site.** The site is `https://<domain>` and `https://www.<domain>`,\n *   one site to its owner. Every redirect hop and every discovered link must\n *   stay on it; a page that redirects anywhere else is refused, not followed.\n * - **AglynBot.** Every request says `AglynBot/1.0 (+<the operator's bot page>)`,\n *   and robots.txt is read first and honored for every page: the AglynBot\n *   group, else `*`. A robots.txt that answers 404 allows everything; one\n *   that cannot be read at all (5xx, no answer) stops the crawl, because\n *   \"could not ask\" is not \"allowed\".\n * - **Our own edge.** A site this platform serves gets the edge secret\n *   (`site-import-edge.ts`, AGL-4085), which lifts our bot challenge for\n *   AglynBot; it is asked per request and per redirect hop, and a request\n *   that leaves the site's two origins never carries it.\n * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an\n *   index's children included), then same-site links found on the pages read,\n *   for whatever the sitemap missed.\n * - **Budgets.** Pages (the caller's, from the plan), bytes per page (a page\n *   over it is refused, not cut), bytes in total, and wall-clock time. A\n *   whole site's worth fits (AGL-4083): aglyn.com's 123 pages average 440 KB,\n *   54 MB in all, so the total is 160 MB; a page is parsed and dropped as it\n *   is read, so what is held is its blocks, never its bytes.\n * - **Four at a time.** robots.txt, the sitemaps and the home page are read\n *   one after another; the rest four at a time, and the inventory keeps the\n *   order they were asked in, which is the sitemap's.\n * - **Nothing vanishes.** An address a stop (the clock, the bytes, the page\n *   budget) left unread is listed as `not-reached`, so the inventory's\n *   counts add up to the site. A wave's own read leaves its unread pages for\n *   its next pass instead.\n * - **No lookalikes.** A domain that wears a brand it is not\n *   (`lookalikeBrandForHost`) is refused before any request.\n * - **No sign-in pages.** A page with a password field is refused and\n *   reported; forms with payment fields, scripts, iframes and embeds never\n *   reach the inventory (site-import-html.ts).\n *=========================================*/\n\nimport { lookalikeBrandForHost } from '@aglyn/shared-util-email/outbound-phishing-screen'\nimport { fenceUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n  fetchPublicTextDetailed,\n  type PublicFetchEdge,\n  type PublicFetchOptions,\n  type PublicFetchResult,\n} from '../jobs/ai-theme-brand-inputs'\nimport {\n  extractSiteImportPage,\n  readMetaValues,\n  type SiteImportBlock,\n  type SiteImportBrandMark,\n  type SiteImportForm,\n  type SiteImportIcon,\n  type SiteImportImage,\n  type SiteImportNavLink,\n  type SiteImportOrganization,\n} from './site-import-html'\nimport {\n  AGLYN_BOT_TOKEN,\n  parseRobotsTxt,\n  parseSitemap,\n  robotsAllows,\n  robotsRulesFor,\n  type RobotsRule,\n} from './site-import-robots'\nimport { NO_SITE_IMPORT_HEADERS, siteImportEdgeHeadersFor, type SiteImportHeadersFor } from './site-import-edge'\n\n/**\n * Where the bot's user agent points a site owner who wants to know what it is\n * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on\n * a self-hosted deployment, so its crawler never names Aglyn's address; the\n * literal is only that variable's default.\n *\n * Aglyn's is the Bring my site guide, which says what AglynBot reads and how\n * robots.txt keeps it out. There is no page of its own (Zach, 10/20: \"we don't\n * need a bot page at all\"), and the site it reads is the reader's own: an\n * import starts only once its owner has proved the site is theirs.\n */\nexport const AGLYN_BOT_INFO_URL =\n  process.env['SITE_IMPORT_BOT_INFO_URL'] ?? 'https://docs.aglyn.com/ai/bring-my-site'\n\n/** What every crawler request says it is. */\nexport const AGLYN_BOT_USER_AGENT = `AglynBot/1.0 (+${AGLYN_BOT_INFO_URL})`\n\nexport const SITE_IMPORT_DEFAULTS = {\n  maxPageBytes: 2 * 1024 * 1024,\n  /** A whole site's worth (AGL-4083): 54 MB for aglyn.com's 123 pages, with room. */\n  maxTotalBytes: 160 * 1024 * 1024,\n  timeBudgetMs: 60_000,\n  /** One request's own clock, inside the whole crawl's. */\n  requestTimeoutMs: 8_000,\n  robotsMaxBytes: 256 * 1024,\n  sitemapMaxBytes: 4 * 1024 * 1024,\n  /** Sitemap documents read, an index's children included: an index of six children is seven. */\n  maxSitemaps: 24,\n  /** URLs a sitemap may contribute to the queue. */\n  maxSitemapUrls: 2_000,\n  /** Reasons kept in `skipped` before the list stops growing: every address a sitemap may name. */\n  maxSkipped: 2_000,\n  /** Pages read at once, after robots.txt, the sitemaps and the home page. */\n  concurrency: 4,\n} as const\n\n/** Never a page: files a link may point at that are not HTML. */\nconst NON_PAGE_EXTENSION =\n  /\\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i\n\nconst TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i\n\nconst DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\\.[a-z0-9-]{1,63})+$/\n\nexport type SiteImportSkipReason =\n  /** The page has a password field: a sign-in page, refused outright. */\n  | 'login-form'\n  /** robots.txt keeps AglynBot off this path. */\n  | 'robots'\n  /** A redirect left the site. */\n  | 'off-origin-redirect'\n  /** Over the per-page byte cap. */\n  | 'too-large'\n  /** Answered with something that is not HTML. */\n  | 'not-html'\n  /** Answered 4xx or 5xx. */\n  | 'http-error'\n  /** No answer, a private address, or a refused address. */\n  | 'unreachable'\n  /** Never asked for: a stop (time, bytes, the page budget) came first (AGL-4083). */\n  | 'not-reached'\n\nexport interface SiteImportSkipped {\n  url: string\n  reason: SiteImportSkipReason\n  status?: number\n  /** Where an off-origin redirect pointed. */\n  location?: string\n  /** Whose edge turned an error answer away, when it said (AGL-4083). */\n  edge?: PublicFetchEdge\n}\n\nexport interface SiteImportPage {\n  /** The address the page was read from, after any same-site redirect. */\n  url: string\n  path: string\n  /** Where the crawler learned of the page. */\n  foundBy: 'home' | 'sitemap' | 'link'\n  title: string\n  description: string\n  lang: string | null\n  canonical: string | null\n  /** The share card (AGL-4083). */\n  ogImage: string | null\n  /** The site's own name, as its head gives it. */\n  siteName: string\n  icons: SiteImportIcon[]\n  /** The header's link home: the brand's logo and words. */\n  brand: SiteImportBrandMark | null\n  organization: SiteImportOrganization | null\n  blocks: SiteImportBlock[]\n  images: SiteImportImage[]\n  navLinks: SiteImportNavLink[]\n  /** Same-site page addresses this page links to. */\n  links: string[]\n  forms: { kept: SiteImportForm[]; removed: number }\n  bytes: number\n}\n\nexport type SiteImportStop =\n  | 'page-budget'\n  | 'total-bytes'\n  | 'time'\n  /** robots.txt could not be read, so nothing was asked of the site. */\n  | 'robots-unreachable'\n\nexport interface SiteImportInventory {\n  domain: string\n  /** The site's origins: the bare name and its `www.` twin. */\n  origins: string[]\n  /** Set when nothing was fetched at all. */\n  refused: 'invalid-domain' | 'lookalike' | null\n  pages: SiteImportPage[]\n  skipped: SiteImportSkipped[]\n  /** Skips past {@link SITE_IMPORT_DEFAULTS.maxSkipped}, counted only. */\n  skippedOverflow: number\n  robots: { found: boolean; sitemaps: string[] }\n  /** Page addresses the sitemaps listed (same-site, before robots and budget). */\n  sitemapUrls: number\n  stoppedBy: SiteImportStop | null\n  bytesRead: number\n  startedAt: number\n  finishedAt: number\n}\n\nexport interface SiteImportCrawlOptions {\n  /** The proved domain; `www.` is folded in. */\n  domain: string\n  /** The most pages the inventory may hold: the plan's page count. */\n  maxPages: number\n  maxPageBytes?: number\n  maxTotalBytes?: number\n  timeBudgetMs?: number\n  signal?: AbortSignal\n  /**\n   * Read the sitemap's shallow addresses before its deep ones (AGL-3976), so a\n   * crawl the page budget stops reads a site's main pages before the tail of\n   * its blog. A sitemap's own order is kept among addresses of one depth.\n   */\n  shallowFirst?: boolean\n  /**\n   * Read exactly these addresses of the site, in order, and nothing else\n   * (AGL-3976): an import's wave reads its own pages again when it starts. No\n   * sitemap is read and no link is followed; robots.txt is read and honored\n   * as for any crawl, and every other rule holds.\n   */\n  only?: readonly string[]\n}\n\nexport interface SiteImportCrawlSeams {\n  fetch?: (url: string, options: PublicFetchOptions) => Promise<PublicFetchResult>\n  now?: () => number\n  /**\n   * Request headers for the site being read, asked once per crawl with its\n   * domain (AGL-4083, AGL-4085). Absent, the crawl asks the edge\n   * (`siteImportEdgeHeadersFor`): Aglyn's edge secret for each of the site's\n   * two origins this platform serves, decided origin by origin, and nothing\n   * anywhere else. Either way the headers go only to the site's two origins,\n   * and the fetch asks again on every redirect hop, so a hop anywhere else\n   * carries nothing. Always under the crawler's own `Accept` and `User-Agent`.\n   */\n  headersFor?: (domain: string) => Promise<Readonly<Record<string, string>>> | Readonly<Record<string, string>>\n}\n\n/**\n * The per-request headers of a crawl of `domain`: the seam's, else the edge's,\n * and none for any address off the site's two origins. Never throws.\n */\nasync function siteRequestHeaders(\n  seams: SiteImportCrawlSeams,\n  domain: string,\n  origins: readonly string[],\n): Promise<SiteImportHeadersFor> {\n  let inner: SiteImportHeadersFor = NO_SITE_IMPORT_HEADERS\n  try {\n    if (seams.headersFor) {\n      const headers = await seams.headersFor(domain)\n      if (headers && Object.keys(headers).length) inner = () => headers\n    } else {\n      inner = await siteImportEdgeHeadersFor(origins)\n    }\n  } catch (error) {\n    console.warn('[site-import] the edge headers could not be read; reading without them', { domain, error })\n  }\n  return (url: URL) => (origins.includes(url.origin) ? inner(url) : {})\n}\n\n/** The site's two origins for a domain. */\nexport function siteImportOrigins(domain: string): string[] {\n  const bare = domain.replace(/^www\\./, '')\n  return [`https://${bare}`, `https://www.${bare}`]\n}\n\n/** A crawlable page address on the site, normalized for de-duplication; else null. */\nexport function siteImportPageKey(url: URL, origins: readonly string[]): string | null {\n  if (url.protocol !== 'https:' || url.username || url.password) return null\n  if (url.port && url.port !== '443') return null\n  if (!origins.includes(url.origin)) return null\n  if (NON_PAGE_EXTENSION.test(url.pathname)) return null\n  const clean = new URL(url.toString())\n  clean.hash = ''\n  for (const key of [...clean.searchParams.keys()]) {\n    if (TRACKING_PARAM.test(key)) clean.searchParams.delete(key)\n  }\n  clean.search = clean.searchParams.toString() ? `?${clean.searchParams.toString()}` : ''\n  return clean.toString()\n}\n\nfunction skipReasonFor(result: PublicFetchResult): SiteImportSkipReason {\n  switch (result.refusal) {\n    case 'redirect-refused':\n      return 'off-origin-redirect'\n    case 'content-type':\n      return 'not-html'\n    case 'too-large':\n      return 'too-large'\n    case 'http-status':\n      return 'http-error'\n    default:\n      return 'unreachable'\n  }\n}\n\n/** Read a proved site into an inventory. Never throws for anything the site does. */\nexport async function crawlSiteForImport(\n  options: SiteImportCrawlOptions,\n  seams: SiteImportCrawlSeams = {},\n): Promise<SiteImportInventory> {\n  const now = seams.now ?? Date.now\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const domain = String(options.domain ?? '').trim().toLowerCase().replace(/\\.+$/, '').replace(/^www\\./, '')\n  const origins = DOMAIN_PATTERN.test(domain) ? siteImportOrigins(domain) : []\n  const startedAt = now()\n  const inventory: SiteImportInventory = {\n    domain,\n    origins,\n    refused: null,\n    pages: [],\n    skipped: [],\n    skippedOverflow: 0,\n    robots: { found: false, sitemaps: [] },\n    sitemapUrls: 0,\n    stoppedBy: null,\n    bytesRead: 0,\n    startedAt,\n    finishedAt: startedAt,\n  }\n  const finish = () => {\n    inventory.finishedAt = now()\n    return inventory\n  }\n  if (!origins.length) {\n    inventory.refused = 'invalid-domain'\n    return finish()\n  }\n  if (lookalikeBrandForHost(domain)) {\n    inventory.refused = 'lookalike'\n    return finish()\n  }\n\n  // Asked once, before anything else, and held to the site's two origins.\n  const headersFor = await siteRequestHeaders(seams, domain, origins)\n  const maxPages = Math.max(0, Math.floor(options.maxPages))\n  const maxPageBytes = options.maxPageBytes ?? SITE_IMPORT_DEFAULTS.maxPageBytes\n  const maxTotalBytes = options.maxTotalBytes ?? SITE_IMPORT_DEFAULTS.maxTotalBytes\n  const deadlineAt = startedAt + (options.timeBudgetMs ?? SITE_IMPORT_DEFAULTS.timeBudgetMs)\n  const controller = new AbortController()\n  const onAbort = () => controller.abort()\n  options.signal?.addEventListener('abort', onAbort)\n  if (options.signal?.aborted) controller.abort()\n  const timer = setTimeout(() => controller.abort(), Math.max(0, deadlineAt - startedAt))\n\n  const sameSite = (url: URL) => origins.includes(url.origin) && url.protocol === 'https:'\n  const remainingBytes = () => maxTotalBytes - inventory.bytesRead\n  const outOfTime = () => controller.signal.aborted || now() >= deadlineAt\n  const request = (url: string, accept: RegExp, acceptHeader: string, maxBytes: number) =>\n    fetchText(url, {\n      maxBytes,\n      accept,\n      acceptHeader,\n      userAgent: AGLYN_BOT_USER_AGENT,\n      headersFor,\n      signal: controller.signal,\n      timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),\n      redirectAllowed: (_from, to) => sameSite(to),\n      refuseOver: true,\n    })\n  const skip = (entry: SiteImportSkipped) => {\n    if (inventory.skipped.length < SITE_IMPORT_DEFAULTS.maxSkipped) inventory.skipped.push(entry)\n    else inventory.skippedOverflow += 1\n  }\n\n  try {\n    // robots.txt, before anything else is asked of the site.\n    const home = `${origins[0]}/`\n    let rules: RobotsRule[] = []\n    const robots = await request(\n      `${origins[0]}/robots.txt`,\n      /text\\/plain|text\\/html|^$/i,\n      'text/plain',\n      Math.min(SITE_IMPORT_DEFAULTS.robotsMaxBytes, remainingBytes()),\n    )\n    if (robots.ok) {\n      inventory.bytesRead += robots.bytes\n      const parsed = parseRobotsTxt(robots.text)\n      rules = robotsRulesFor(parsed, AGLYN_BOT_TOKEN)\n      inventory.robots = { found: true, sitemaps: parsed.sitemaps }\n    } else if (\n      !(robots.refusal === 'http-status' && (robots.status ?? 0) >= 400 && (robots.status ?? 0) < 500)\n    ) {\n      inventory.stoppedBy = outOfTime() ? 'time' : 'robots-unreachable'\n      return finish()\n    }\n    const allowed = (key: string) => {\n      const url = new URL(key)\n      return robotsAllows(rules, `${url.pathname}${url.search}`)\n    }\n\n    // A wave's own pages (AGL-3976): exactly those, after robots.txt.\n    const only = options.only\n      ? options.only\n          .map((raw) => {\n            try {\n              return siteImportPageKey(new URL(raw), origins)\n            } catch {\n              return null\n            }\n          })\n          .filter((key): key is string => !!key)\n      : null\n\n    // Sitemaps: the ones robots.txt names on this site, else /sitemap.xml.\n    const sitemapQueue: string[] = []\n    for (const raw of inventory.robots.sitemaps) {\n      try {\n        const url = new URL(raw)\n        if (sameSite(url)) sitemapQueue.push(url.toString())\n      } catch {\n        // A malformed Sitemap line names nothing.\n      }\n    }\n    if (!sitemapQueue.length && !only) sitemapQueue.push(`${origins[0]}/sitemap.xml`)\n    if (only) sitemapQueue.length = 0\n    const sitemapSeen = new Set<string>()\n    const fromSitemap: string[] = []\n    let sitemapsRead = 0\n    while (sitemapQueue.length && sitemapsRead < SITE_IMPORT_DEFAULTS.maxSitemaps && !outOfTime()) {\n      const next = sitemapQueue.shift() as string\n      if (sitemapSeen.has(next)) continue\n      sitemapSeen.add(next)\n      if (/\\.gz$/i.test(new URL(next).pathname)) continue\n      if (!allowed(next)) continue\n      const room = Math.min(SITE_IMPORT_DEFAULTS.sitemapMaxBytes, remainingBytes())\n      if (room <= 0) break\n      sitemapsRead += 1\n      const sitemap = await request(next, /xml|text\\/plain/i, 'application/xml, text/xml;q=0.9', room)\n      if (!sitemap.ok) continue\n      inventory.bytesRead += sitemap.bytes\n      const entries = parseSitemap(sitemap.text)\n      for (const location of entries.locations) {\n        let url: URL\n        try {\n          url = new URL(location)\n        } catch {\n          continue\n        }\n        if (entries.index) {\n          if (sameSite(url)) sitemapQueue.push(url.toString())\n          continue\n        }\n        const key = siteImportPageKey(url, origins)\n        if (key && fromSitemap.length < SITE_IMPORT_DEFAULTS.maxSitemapUrls) fromSitemap.push(key)\n      }\n    }\n    inventory.sitemapUrls = fromSitemap.length\n    if (options.shallowFirst) {\n      const depth = (key: string) => new URL(key).pathname.split('/').filter(Boolean).length\n      const ranked = fromSitemap.map((key, order) => ({ key, order, depth: depth(key) }))\n      ranked.sort((a, b) => a.depth - b.depth || a.order - b.order)\n      fromSitemap.splice(0, fromSitemap.length, ...ranked.map((entry) => entry.key))\n    }\n\n    // Pages: home, then the sitemap's, then links the pages themselves name.\n    // A wave's read is its own pages alone.\n    const queue: { key: string; foundBy: SiteImportPage['foundBy'] }[] = only\n      ? only.map((key) => ({ key, foundBy: (new URL(key).pathname === '/' ? 'home' : 'link') as SiteImportPage['foundBy'] }))\n      : [{ key: home, foundBy: 'home' }]\n    const queued = new Set<string>(only ?? [home])\n    for (const key of fromSitemap) {\n      if (queued.has(key)) continue\n      queued.add(key)\n      queue.push({ key, foundBy: 'sitemap' })\n    }\n    const read = new Set<string>()\n    // The site's two names are one site: once the home page says which one\n    // it answers on, every address is read on that one, so /about is not\n    // read twice as example.com/about and www.example.com/about.\n    // A wave's read starts on the name its pages were read on the first time.\n    let preferred = only?.length ? new URL(only[0]).origin : origins[0]\n    const canon = (raw: string) => {\n      const url = new URL(raw)\n      return origins.includes(url.origin) && url.origin !== preferred\n        ? `${preferred}${url.pathname}${url.search}`\n        : raw\n    }\n    const maxAttempts = maxPages * 3 + 10\n    let attempts = 0\n    type Asked = { next: (typeof queue)[number]; key: string; cap: number; page: PublicFetchResult }\n    /** The pages asked for and not yet taken in, in the order they were asked. */\n    const window: Array<Promise<Asked>> = []\n    /** The bytes the asked pages may still read: what is left is shared, never promised twice. */\n    let reserved = 0\n    let stop: SiteImportStop | null = null\n    /** Asked for and never read, because a stop came first: listed as not reached. */\n    const cutOff: string[] = []\n    /** Home alone first, so its answer settles which of the site's two names the rest are read on. */\n    let homeIn = !!only\n    /** Why no more may be asked now: a stop, `wait` for an asked page to come in first, or nothing. */\n    const held = (): SiteImportStop | 'wait' | null => {\n      if (inventory.pages.length >= maxPages) return 'page-budget'\n      if (outOfTime()) return 'time'\n      if (remainingBytes() <= 0) return 'total-bytes'\n      if (attempts >= maxAttempts) return 'page-budget'\n      if (inventory.pages.length + window.length >= maxPages) return 'wait'\n      if (remainingBytes() - reserved <= 0) return 'wait'\n      return null\n    }\n    /** Takes one asked page in: the inventory's next page, a skip, or the reason the read must stop. */\n    const takeIn = ({ next, key, cap, page }: Asked): SiteImportStop | null => {\n      const foundBy = next.foundBy\n      if (foundBy === 'home') homeIn = true\n      if (!page.ok) {\n        if (page.refusal === 'too-large' && cap < maxPageBytes) {\n          cutOff.push(key)\n          return 'total-bytes'\n        }\n        if (page.refusal === 'network' && outOfTime()) {\n          cutOff.push(key)\n          return 'time'\n        }\n        skip({\n          url: key,\n          reason: skipReasonFor(page),\n          ...(page.status ? { status: page.status } : {}),\n          ...(page.location ? { location: page.location } : {}),\n          ...(page.edge ? { edge: page.edge } : {}),\n        })\n        return null\n      }\n      inventory.bytesRead += page.bytes\n      if (foundBy === 'home') preferred = new URL(page.url).origin\n      const finalKey = canon(siteImportPageKey(new URL(page.url), origins) ?? page.url)\n      if (finalKey !== key) {\n        if (read.has(finalKey)) return null\n        read.add(finalKey)\n        if (!allowed(finalKey)) {\n          skip({ url: finalKey, reason: 'robots' })\n          return null\n        }\n      }\n      const extract = extractSiteImportPage(page.text, page.url, sameSite)\n      if (extract.loginForm) {\n        skip({ url: finalKey, reason: 'login-form' })\n        return null\n      }\n      const links: string[] = []\n      for (const link of extract.links) {\n        const linkKey = siteImportPageKey(new URL(link), origins)\n        if (!linkKey) continue\n        links.push(linkKey)\n        if (!only && !queued.has(linkKey)) {\n          queued.add(linkKey)\n          queue.push({ key: linkKey, foundBy: 'link' })\n        }\n      }\n      inventory.pages.push({\n        url: finalKey,\n        path: new URL(finalKey).pathname,\n        foundBy,\n        title: extract.title,\n        description: extract.description,\n        lang: extract.lang,\n        canonical: extract.canonical,\n        ogImage: extract.ogImage,\n        siteName: extract.siteName,\n        icons: extract.icons,\n        brand: extract.brand,\n        organization: extract.organization,\n        blocks: extract.blocks,\n        images: extract.images,\n        navLinks: extract.navLinks,\n        links: [...new Set(links.map(canon))],\n        forms: extract.forms,\n        bytes: page.bytes,\n      })\n      return null\n    }\n    for (;;) {\n      const width = homeIn ? SITE_IMPORT_DEFAULTS.concurrency : 1\n      while (!stop && queue.length && window.length < width) {\n        const why = held()\n        if (why === 'wait') break\n        if (why) {\n          stop = why\n          break\n        }\n        const next = queue.shift() as (typeof queue)[number]\n        const key = canon(next.key)\n        if (read.has(key)) continue\n        if (!allowed(key)) {\n          skip({ url: key, reason: 'robots' })\n          if (next.foundBy === 'home') homeIn = true\n          continue\n        }\n        attempts += 1\n        read.add(key)\n        const cap = Math.min(maxPageBytes, remainingBytes() - reserved)\n        reserved += cap\n        window.push(\n          request(key, /text\\/html|application\\/xhtml\\+xml/i, 'text/html, application/xhtml+xml;q=0.9', cap).then((page) => ({ next, key, cap, page })),\n        )\n      }\n      if (!window.length) break\n      const asked = await (window.shift() as Promise<Asked>)\n      reserved -= asked.cap\n      stop = takeIn(asked) ?? stop\n    }\n    inventory.stoppedBy = stop\n    // Whatever a stop left unread is said, never dropped; a wave's own read\n    // leaves its unread pages for its next pass instead.\n    if (stop && !only) {\n      const unread = [...cutOff]\n      for (const next of queue) {\n        const key = canon(next.key)\n        if (!read.has(key) && !unread.includes(key)) unread.push(key)\n      }\n      for (const url of unread) skip({ url, reason: 'not-reached' })\n    }\n    return finish()\n  } finally {\n    clearTimeout(timer)\n    options.signal?.removeEventListener('abort', onAbort)\n  }\n}\n\n/**\n * The home page's `aglyn-site-verification` values, for the meta-tag proof:\n * the same fetch, user agent and same-site redirects as the crawl, but no\n * robots.txt, because the person asked for this one page to be read.\n * `answered: false` when the page could not be read at all.\n */\nexport async function readSiteVerificationMeta(\n  domain: string,\n  metaName: string,\n  seams: SiteImportCrawlSeams = {},\n): Promise<{ answered: boolean; values: string[]; url?: string }> {\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const bare = String(domain ?? '').trim().toLowerCase().replace(/^www\\./, '')\n  if (!DOMAIN_PATTERN.test(bare) || lookalikeBrandForHost(bare)) return { answered: false, values: [] }\n  const origins = siteImportOrigins(bare)\n  const result = await fetchText(`${origins[0]}/`, {\n    maxBytes: SITE_IMPORT_DEFAULTS.maxPageBytes,\n    accept: /text\\/html|application\\/xhtml\\+xml/i,\n    acceptHeader: 'text/html, application/xhtml+xml;q=0.9',\n    userAgent: AGLYN_BOT_USER_AGENT,\n    headersFor: await siteRequestHeaders(seams, bare, origins),\n    timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,\n    redirectAllowed: (_from, to) => to.protocol === 'https:' && origins.includes(to.origin),\n  })\n  if (!result.ok) {\n    // An answer that is not the page (a 404, a redirect off the site) is\n    // still an answer: the tag is not where it must be.\n    const answered = result.refusal === 'http-status' || result.refusal === 'redirect-refused' || result.refusal === 'content-type'\n    return { answered, values: [], url: result.url }\n  }\n  const head = result.text.split(/<body\\b/i)[0] ?? ''\n  return { answered: true, values: readMetaValues(head, metaName), url: result.url }\n}\n\n/**\n * One page as text for a prompt: its title, description and blocks in order,\n * fenced as untrusted data. The importer's prompt carries `UNTRUSTED_TEXT_RULE`.\n */\nexport function siteImportPagePromptText(page: SiteImportPage, maxChars = 12_000): string {\n  const lines: string[] = []\n  if (page.title) lines.push(`Title: ${page.title}`)\n  if (page.description) lines.push(`Description: ${page.description}`)\n  for (const block of page.blocks) {\n    if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n    else if (block.kind === 'image') lines.push(`[image: ${block.alt || 'no description'}]`)\n    else if (block.kind === 'list-item') lines.push(`- ${block.text}`)\n    else if (block.kind === 'quote') lines.push(`> ${block.text}`)\n    else if (block.kind === 'link') {\n      // A link inside a sentence is in its block's words already.\n      if (!block.inline) lines.push(block.text)\n    } else lines.push(block.text)\n  }\n  return fenceUntrustedText(`Page ${page.path} of the site being imported`, lines.join('\\n'), { maxChars })\n}\n"],"names":["process","lookalikeBrandForHost","fenceUntrustedText","fetchPublicTextDetailed","extractSiteImportPage","readMetaValues","AGLYN_BOT_TOKEN","parseRobotsTxt","parseSitemap","robotsAllows","robotsRulesFor","NO_SITE_IMPORT_HEADERS","siteImportEdgeHeadersFor","AGLYN_BOT_INFO_URL","env","AGLYN_BOT_USER_AGENT","SITE_IMPORT_DEFAULTS","maxPageBytes","maxTotalBytes","timeBudgetMs","requestTimeoutMs","robotsMaxBytes","sitemapMaxBytes","maxSitemaps","maxSitemapUrls","maxSkipped","concurrency","NON_PAGE_EXTENSION","TRACKING_PARAM","DOMAIN_PATTERN","siteRequestHeaders","seams","domain","origins","inner","headersFor","headers","Object","keys","length","error","console","warn","url","includes","origin","siteImportOrigins","bare","replace","siteImportPageKey","protocol","username","password","port","test","pathname","clean","URL","toString","hash","key","searchParams","delete","search","skipReasonFor","result","refusal","crawlSiteForImport","options","now","Date","fetchText","fetch","String","trim","toLowerCase","startedAt","inventory","refused","pages","skipped","skippedOverflow","robots","found","sitemaps","sitemapUrls","stoppedBy","bytesRead","finishedAt","finish","maxPages","Math","max","floor","deadlineAt","controller","AbortController","onAbort","abort","signal","addEventListener","aborted","timer","setTimeout","sameSite","remainingBytes","outOfTime","request","accept","acceptHeader","maxBytes","userAgent","timeoutMs","min","redirectAllowed","_from","to","refuseOver","skip","entry","push","home","rules","ok","bytes","parsed","text","status","allowed","only","map","raw","filter","sitemapQueue","sitemapSeen","Set","fromSitemap","sitemapsRead","next","shift","has","add","room","sitemap","entries","location","locations","index","shallowFirst","depth","split","Boolean","ranked","order","sort","a","b","splice","queue","foundBy","queued","read","preferred","canon","maxAttempts","attempts","window","reserved","stop","cutOff","homeIn","held","takeIn","cap","page","reason","edge","finalKey","extract","loginForm","links","link","linkKey","path","title","description","lang","canonical","ogImage","siteName","icons","brand","organization","blocks","images","navLinks","forms","width","why","then","asked","unread","clearTimeout","removeEventListener","readSiteVerificationMeta","metaName","answered","values","head","siteImportPagePromptText","maxChars","lines","block","kind","repeat","level","alt","inline","join"],"mappings":";IAyGEA;AAzGF;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2CA6C2C,GAE3C,SAASC,qBAAqB,QAAQ,oDAAmD;AACzF,SAASC,kBAAkB,QAAQ,0CAAyC;AAC5E,SACEC,uBAAuB,QAIlB,mCAA+B;AACtC,SACEC,qBAAqB,EACrBC,cAAc,QAQT,wBAAoB;AAC3B,SACEC,eAAe,EACfC,cAAc,EACdC,YAAY,EACZC,YAAY,EACZC,cAAc,QAET,0BAAsB;AAC7B,SAASC,sBAAsB,EAAEC,wBAAwB,QAAmC,wBAAoB;AAEhH;;;;;;;;;;CAUC,GACD,OAAO,MAAMC,sBACXb,wCAAAA,QAAQc,GAAG,CAAC,2BAA2B,YAAvCd,wCAA2C,0CAAyC;AAEtF,2CAA2C,GAC3C,OAAO,MAAMe,uBAAuB,CAAC,eAAe,EAAEF,mBAAmB,CAAC,CAAC,CAAA;AAE3E,OAAO,MAAMG,uBAAuB;IAClCC,cAAc,IAAI,OAAO;IACzB,iFAAiF,GACjFC,eAAe,MAAM,OAAO;IAC5BC,cAAc;IACd,uDAAuD,GACvDC,kBAAkB;IAClBC,gBAAgB,MAAM;IACtBC,iBAAiB,IAAI,OAAO;IAC5B,6FAA6F,GAC7FC,aAAa;IACb,gDAAgD,GAChDC,gBAAgB;IAChB,+FAA+F,GAC/FC,YAAY;IACZ,0EAA0E,GAC1EC,aAAa;AACf,EAAU;AAEV,+DAA+D,GAC/D,MAAMC,qBACJ;AAEF,MAAMC,iBAAiB;AAEvB,MAAMC,iBAAiB;AA0HvB;;;CAGC,GACD,eAAeC,mBACbC,KAA2B,EAC3BC,MAAc,EACdC,OAA0B;IAE1B,IAAIC,QAA8BvB;IAClC,IAAI;QACF,IAAIoB,MAAMI,UAAU,EAAE;YACpB,MAAMC,UAAU,MAAML,MAAMI,UAAU,CAACH;YACvC,IAAII,WAAWC,OAAOC,IAAI,CAACF,SAASG,MAAM,EAAEL,QAAQ,IAAME;QAC5D,OAAO;YACLF,QAAQ,MAAMtB,yBAAyBqB;QACzC;IACF,EAAE,OAAOO,OAAO;QACdC,QAAQC,IAAI,CAAC,0EAA0E;YAAEV;YAAQQ;QAAM;IACzG;IACA,OAAO,CAACG,MAAcV,QAAQW,QAAQ,CAACD,IAAIE,MAAM,IAAIX,MAAMS,OAAO,CAAC;AACrE;AAEA,yCAAyC,GACzC,OAAO,SAASG,kBAAkBd,MAAc;IAC9C,MAAMe,OAAOf,OAAOgB,OAAO,CAAC,UAAU;IACtC,OAAO;QAAC,CAAC,QAAQ,EAAED,MAAM;QAAE,CAAC,YAAY,EAAEA,MAAM;KAAC;AACnD;AAEA,oFAAoF,GACpF,OAAO,SAASE,kBAAkBN,GAAQ,EAAEV,OAA0B;IACpE,IAAIU,IAAIO,QAAQ,KAAK,YAAYP,IAAIQ,QAAQ,IAAIR,IAAIS,QAAQ,EAAE,OAAO;IACtE,IAAIT,IAAIU,IAAI,IAAIV,IAAIU,IAAI,KAAK,OAAO,OAAO;IAC3C,IAAI,CAACpB,QAAQW,QAAQ,CAACD,IAAIE,MAAM,GAAG,OAAO;IAC1C,IAAIlB,mBAAmB2B,IAAI,CAACX,IAAIY,QAAQ,GAAG,OAAO;IAClD,MAAMC,QAAQ,IAAIC,IAAId,IAAIe,QAAQ;IAClCF,MAAMG,IAAI,GAAG;IACb,KAAK,MAAMC,OAAO;WAAIJ,MAAMK,YAAY,CAACvB,IAAI;KAAG,CAAE;QAChD,IAAIV,eAAe0B,IAAI,CAACM,MAAMJ,MAAMK,YAAY,CAACC,MAAM,CAACF;IAC1D;IACAJ,MAAMO,MAAM,GAAGP,MAAMK,YAAY,CAACH,QAAQ,KAAK,CAAC,CAAC,EAAEF,MAAMK,YAAY,CAACH,QAAQ,IAAI,GAAG;IACrF,OAAOF,MAAME,QAAQ;AACvB;AAEA,SAASM,cAAcC,MAAyB;IAC9C,OAAQA,OAAOC,OAAO;QACpB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,mFAAmF,GACnF,OAAO,eAAeC,mBACpBC,OAA+B,EAC/BrC,QAA8B,CAAC,CAAC;QAEpBA,YACMA,cACIqC,iBAiCDA,uBACCA,wBACUA;QAGhCA,iBACIA;IAzCJ,MAAMC,OAAMtC,aAAAA,MAAMsC,GAAG,YAATtC,aAAauC,KAAKD,GAAG;IACjC,MAAME,aAAYxC,eAAAA,MAAMyC,KAAK,YAAXzC,eAAe5B;IACjC,MAAM6B,SAASyC,QAAOL,kBAAAA,QAAQpC,MAAM,YAAdoC,kBAAkB,IAAIM,IAAI,GAAGC,WAAW,GAAG3B,OAAO,CAAC,QAAQ,IAAIA,OAAO,CAAC,UAAU;IACvG,MAAMf,UAAUJ,eAAeyB,IAAI,CAACtB,UAAUc,kBAAkBd,UAAU,EAAE;IAC5E,MAAM4C,YAAYP;IAClB,MAAMQ,YAAiC;QACrC7C;QACAC;QACA6C,SAAS;QACTC,OAAO,EAAE;QACTC,SAAS,EAAE;QACXC,iBAAiB;QACjBC,QAAQ;YAAEC,OAAO;YAAOC,UAAU,EAAE;QAAC;QACrCC,aAAa;QACbC,WAAW;QACXC,WAAW;QACXX;QACAY,YAAYZ;IACd;IACA,MAAMa,SAAS;QACbZ,UAAUW,UAAU,GAAGnB;QACvB,OAAOQ;IACT;IACA,IAAI,CAAC5C,QAAQM,MAAM,EAAE;QACnBsC,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IACA,IAAIxF,sBAAsB+B,SAAS;QACjC6C,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IAEA,wEAAwE;IACxE,MAAMtD,aAAa,MAAML,mBAAmBC,OAAOC,QAAQC;IAC3D,MAAMyD,WAAWC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAACzB,QAAQsB,QAAQ;IACxD,MAAMzE,gBAAemD,wBAAAA,QAAQnD,YAAY,YAApBmD,wBAAwBpD,qBAAqBC,YAAY;IAC9E,MAAMC,iBAAgBkD,yBAAAA,QAAQlD,aAAa,YAArBkD,yBAAyBpD,qBAAqBE,aAAa;IACjF,MAAM4E,aAAalB,cAAaR,wBAAAA,QAAQjD,YAAY,YAApBiD,wBAAwBpD,qBAAqBG,YAAY;IACzF,MAAM4E,aAAa,IAAIC;IACvB,MAAMC,UAAU,IAAMF,WAAWG,KAAK;KACtC9B,kBAAAA,QAAQ+B,MAAM,qBAAd/B,gBAAgBgC,gBAAgB,CAAC,SAASH;IAC1C,KAAI7B,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBiC,OAAO,EAAEN,WAAWG,KAAK;IAC7C,MAAMI,QAAQC,WAAW,IAAMR,WAAWG,KAAK,IAAIP,KAAKC,GAAG,CAAC,GAAGE,aAAalB;IAE5E,MAAM4B,WAAW,CAAC7D,MAAaV,QAAQW,QAAQ,CAACD,IAAIE,MAAM,KAAKF,IAAIO,QAAQ,KAAK;IAChF,MAAMuD,iBAAiB,IAAMvF,gBAAgB2D,UAAUU,SAAS;IAChE,MAAMmB,YAAY,IAAMX,WAAWI,MAAM,CAACE,OAAO,IAAIhC,SAASyB;IAC9D,MAAMa,UAAU,CAAChE,KAAaiE,QAAgBC,cAAsBC,WAClEvC,UAAU5B,KAAK;YACbmE;YACAF;YACAC;YACAE,WAAWhG;YACXoB;YACAgE,QAAQJ,WAAWI,MAAM;YACzBa,WAAWrB,KAAKC,GAAG,CAAC,GAAGD,KAAKsB,GAAG,CAACjG,qBAAqBI,gBAAgB,EAAE0E,aAAazB;YACpF6C,iBAAiB,CAACC,OAAOC,KAAOZ,SAASY;YACzCC,YAAY;QACd;IACF,MAAMC,OAAO,CAACC;QACZ,IAAI1C,UAAUG,OAAO,CAACzC,MAAM,GAAGvB,qBAAqBS,UAAU,EAAEoD,UAAUG,OAAO,CAACwC,IAAI,CAACD;aAClF1C,UAAUI,eAAe,IAAI;IACpC;IAEA,IAAI;YAgBuCC,gBAA+BA;QAfxE,yDAAyD;QACzD,MAAMuC,OAAO,GAAGxF,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,IAAIyF,QAAsB,EAAE;QAC5B,MAAMxC,SAAS,MAAMyB,QACnB,GAAG1E,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAC1B,8BACA,cACA0D,KAAKsB,GAAG,CAACjG,qBAAqBK,cAAc,EAAEoF;QAEhD,IAAIvB,OAAOyC,EAAE,EAAE;YACb9C,UAAUU,SAAS,IAAIL,OAAO0C,KAAK;YACnC,MAAMC,SAAStH,eAAe2E,OAAO4C,IAAI;YACzCJ,QAAQhH,eAAemH,QAAQvH;YAC/BuE,UAAUK,MAAM,GAAG;gBAAEC,OAAO;gBAAMC,UAAUyC,OAAOzC,QAAQ;YAAC;QAC9D,OAAO,IACL,CAAEF,CAAAA,OAAOhB,OAAO,KAAK,iBAAiB,EAACgB,iBAAAA,OAAO6C,MAAM,YAAb7C,iBAAiB,MAAM,OAAO,EAACA,kBAAAA,OAAO6C,MAAM,YAAb7C,kBAAiB,KAAK,GAAE,GAC9F;YACAL,UAAUS,SAAS,GAAGoB,cAAc,SAAS;YAC7C,OAAOjB;QACT;QACA,MAAMuC,UAAU,CAACpE;YACf,MAAMjB,MAAM,IAAIc,IAAIG;YACpB,OAAOnD,aAAaiH,OAAO,GAAG/E,IAAIY,QAAQ,GAAGZ,IAAIoB,MAAM,EAAE;QAC3D;QAEA,kEAAkE;QAClE,MAAMkE,OAAO7D,QAAQ6D,IAAI,GACrB7D,QAAQ6D,IAAI,CACTC,GAAG,CAAC,CAACC;YACJ,IAAI;gBACF,OAAOlF,kBAAkB,IAAIQ,IAAI0E,MAAMlG;YACzC,EAAE,eAAM;gBACN,OAAO;YACT;QACF,GACCmG,MAAM,CAAC,CAACxE,MAAuB,CAAC,CAACA,OACpC;QAEJ,uEAAuE;QACvE,MAAMyE,eAAyB,EAAE;QACjC,KAAK,MAAMF,OAAOtD,UAAUK,MAAM,CAACE,QAAQ,CAAE;YAC3C,IAAI;gBACF,MAAMzC,MAAM,IAAIc,IAAI0E;gBACpB,IAAI3B,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIe,QAAQ;YACnD,EAAE,eAAM;YACN,0CAA0C;YAC5C;QACF;QACA,IAAI,CAAC2E,aAAa9F,MAAM,IAAI,CAAC0F,MAAMI,aAAab,IAAI,CAAC,GAAGvF,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC;QAChF,IAAIgG,MAAMI,aAAa9F,MAAM,GAAG;QAChC,MAAM+F,cAAc,IAAIC;QACxB,MAAMC,cAAwB,EAAE;QAChC,IAAIC,eAAe;QACnB,MAAOJ,aAAa9F,MAAM,IAAIkG,eAAezH,qBAAqBO,WAAW,IAAI,CAACmF,YAAa;YAC7F,MAAMgC,OAAOL,aAAaM,KAAK;YAC/B,IAAIL,YAAYM,GAAG,CAACF,OAAO;YAC3BJ,YAAYO,GAAG,CAACH;YAChB,IAAI,SAASpF,IAAI,CAAC,IAAIG,IAAIiF,MAAMnF,QAAQ,GAAG;YAC3C,IAAI,CAACyE,QAAQU,OAAO;YACpB,MAAMI,OAAOnD,KAAKsB,GAAG,CAACjG,qBAAqBM,eAAe,EAAEmF;YAC5D,IAAIqC,QAAQ,GAAG;YACfL,gBAAgB;YAChB,MAAMM,UAAU,MAAMpC,QAAQ+B,MAAM,oBAAoB,mCAAmCI;YAC3F,IAAI,CAACC,QAAQpB,EAAE,EAAE;YACjB9C,UAAUU,SAAS,IAAIwD,QAAQnB,KAAK;YACpC,MAAMoB,UAAUxI,aAAauI,QAAQjB,IAAI;YACzC,KAAK,MAAMmB,YAAYD,QAAQE,SAAS,CAAE;gBACxC,IAAIvG;gBACJ,IAAI;oBACFA,MAAM,IAAIc,IAAIwF;gBAChB,EAAE,eAAM;oBACN;gBACF;gBACA,IAAID,QAAQG,KAAK,EAAE;oBACjB,IAAI3C,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIe,QAAQ;oBACjD;gBACF;gBACA,MAAME,MAAMX,kBAAkBN,KAAKV;gBACnC,IAAI2B,OAAO4E,YAAYjG,MAAM,GAAGvB,qBAAqBQ,cAAc,EAAEgH,YAAYhB,IAAI,CAAC5D;YACxF;QACF;QACAiB,UAAUQ,WAAW,GAAGmD,YAAYjG,MAAM;QAC1C,IAAI6B,QAAQgF,YAAY,EAAE;YACxB,MAAMC,QAAQ,CAACzF,MAAgB,IAAIH,IAAIG,KAAKL,QAAQ,CAAC+F,KAAK,CAAC,KAAKlB,MAAM,CAACmB,SAAShH,MAAM;YACtF,MAAMiH,SAAShB,YAAYN,GAAG,CAAC,CAACtE,KAAK6F,QAAW,CAAA;oBAAE7F;oBAAK6F;oBAAOJ,OAAOA,MAAMzF;gBAAK,CAAA;YAChF4F,OAAOE,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEN,KAAK,GAAGO,EAAEP,KAAK,IAAIM,EAAEF,KAAK,GAAGG,EAAEH,KAAK;YAC5DjB,YAAYqB,MAAM,CAAC,GAAGrB,YAAYjG,MAAM,KAAKiH,OAAOtB,GAAG,CAAC,CAACX,QAAUA,MAAM3D,GAAG;QAC9E;QAEA,yEAAyE;QACzE,wCAAwC;QACxC,MAAMkG,QAA+D7B,OACjEA,KAAKC,GAAG,CAAC,CAACtE,MAAS,CAAA;gBAAEA;gBAAKmG,SAAU,IAAItG,IAAIG,KAAKL,QAAQ,KAAK,MAAM,SAAS;YAAqC,CAAA,KAClH;YAAC;gBAAEK,KAAK6D;gBAAMsC,SAAS;YAAO;SAAE;QACpC,MAAMC,SAAS,IAAIzB,IAAYN,eAAAA,OAAQ;YAACR;SAAK;QAC7C,KAAK,MAAM7D,OAAO4E,YAAa;YAC7B,IAAIwB,OAAOpB,GAAG,CAAChF,MAAM;YACrBoG,OAAOnB,GAAG,CAACjF;YACXkG,MAAMtC,IAAI,CAAC;gBAAE5D;gBAAKmG,SAAS;YAAU;QACvC;QACA,MAAME,OAAO,IAAI1B;QACjB,uEAAuE;QACvE,qEAAqE;QACrE,6DAA6D;QAC7D,0EAA0E;QAC1E,IAAI2B,YAAYjC,CAAAA,wBAAAA,KAAM1F,MAAM,IAAG,IAAIkB,IAAIwE,IAAI,CAAC,EAAE,EAAEpF,MAAM,GAAGZ,OAAO,CAAC,EAAE;QACnE,MAAMkI,QAAQ,CAAChC;YACb,MAAMxF,MAAM,IAAIc,IAAI0E;YACpB,OAAOlG,QAAQW,QAAQ,CAACD,IAAIE,MAAM,KAAKF,IAAIE,MAAM,KAAKqH,YAClD,GAAGA,YAAYvH,IAAIY,QAAQ,GAAGZ,IAAIoB,MAAM,EAAE,GAC1CoE;QACN;QACA,MAAMiC,cAAc1E,WAAW,IAAI;QACnC,IAAI2E,WAAW;QAEf,4EAA4E,GAC5E,MAAMC,SAAgC,EAAE;QACxC,4FAA4F,GAC5F,IAAIC,WAAW;QACf,IAAIC,OAA8B;QAClC,gFAAgF,GAChF,MAAMC,SAAmB,EAAE;QAC3B,gGAAgG,GAChG,IAAIC,SAAS,CAAC,CAACzC;QACf,iGAAiG,GACjG,MAAM0C,OAAO;YACX,IAAI9F,UAAUE,KAAK,CAACxC,MAAM,IAAImD,UAAU,OAAO;YAC/C,IAAIgB,aAAa,OAAO;YACxB,IAAID,oBAAoB,GAAG,OAAO;YAClC,IAAI4D,YAAYD,aAAa,OAAO;YACpC,IAAIvF,UAAUE,KAAK,CAACxC,MAAM,GAAG+H,OAAO/H,MAAM,IAAImD,UAAU,OAAO;YAC/D,IAAIe,mBAAmB8D,YAAY,GAAG,OAAO;YAC7C,OAAO;QACT;QACA,kGAAkG,GAClG,MAAMK,SAAS,CAAC,EAAElC,IAAI,EAAE9E,GAAG,EAAEiH,GAAG,EAAEC,IAAI,EAAS;gBAuBtB7H;YAtBvB,MAAM8G,UAAUrB,KAAKqB,OAAO;YAC5B,IAAIA,YAAY,QAAQW,SAAS;YACjC,IAAI,CAACI,KAAKnD,EAAE,EAAE;gBACZ,IAAImD,KAAK5G,OAAO,KAAK,eAAe2G,MAAM5J,cAAc;oBACtDwJ,OAAOjD,IAAI,CAAC5D;oBACZ,OAAO;gBACT;gBACA,IAAIkH,KAAK5G,OAAO,KAAK,aAAawC,aAAa;oBAC7C+D,OAAOjD,IAAI,CAAC5D;oBACZ,OAAO;gBACT;gBACA0D,KAAK;oBACH3E,KAAKiB;oBACLmH,QAAQ/G,cAAc8G;mBAClBA,KAAK/C,MAAM,GAAG;oBAAEA,QAAQ+C,KAAK/C,MAAM;gBAAC,IAAI,CAAC,GACzC+C,KAAK7B,QAAQ,GAAG;oBAAEA,UAAU6B,KAAK7B,QAAQ;gBAAC,IAAI,CAAC,GAC/C6B,KAAKE,IAAI,GAAG;oBAAEA,MAAMF,KAAKE,IAAI;gBAAC,IAAI,CAAC;gBAEzC,OAAO;YACT;YACAnG,UAAUU,SAAS,IAAIuF,KAAKlD,KAAK;YACjC,IAAImC,YAAY,QAAQG,YAAY,IAAIzG,IAAIqH,KAAKnI,GAAG,EAAEE,MAAM;YAC5D,MAAMoI,WAAWd,OAAMlH,qBAAAA,kBAAkB,IAAIQ,IAAIqH,KAAKnI,GAAG,GAAGV,oBAArCgB,qBAAiD6H,KAAKnI,GAAG;YAChF,IAAIsI,aAAarH,KAAK;gBACpB,IAAIqG,KAAKrB,GAAG,CAACqC,WAAW,OAAO;gBAC/BhB,KAAKpB,GAAG,CAACoC;gBACT,IAAI,CAACjD,QAAQiD,WAAW;oBACtB3D,KAAK;wBAAE3E,KAAKsI;wBAAUF,QAAQ;oBAAS;oBACvC,OAAO;gBACT;YACF;YACA,MAAMG,UAAU9K,sBAAsB0K,KAAKhD,IAAI,EAAEgD,KAAKnI,GAAG,EAAE6D;YAC3D,IAAI0E,QAAQC,SAAS,EAAE;gBACrB7D,KAAK;oBAAE3E,KAAKsI;oBAAUF,QAAQ;gBAAa;gBAC3C,OAAO;YACT;YACA,MAAMK,QAAkB,EAAE;YAC1B,KAAK,MAAMC,QAAQH,QAAQE,KAAK,CAAE;gBAChC,MAAME,UAAUrI,kBAAkB,IAAIQ,IAAI4H,OAAOpJ;gBACjD,IAAI,CAACqJ,SAAS;gBACdF,MAAM5D,IAAI,CAAC8D;gBACX,IAAI,CAACrD,QAAQ,CAAC+B,OAAOpB,GAAG,CAAC0C,UAAU;oBACjCtB,OAAOnB,GAAG,CAACyC;oBACXxB,MAAMtC,IAAI,CAAC;wBAAE5D,KAAK0H;wBAASvB,SAAS;oBAAO;gBAC7C;YACF;YACAlF,UAAUE,KAAK,CAACyC,IAAI,CAAC;gBACnB7E,KAAKsI;gBACLM,MAAM,IAAI9H,IAAIwH,UAAU1H,QAAQ;gBAChCwG;gBACAyB,OAAON,QAAQM,KAAK;gBACpBC,aAAaP,QAAQO,WAAW;gBAChCC,MAAMR,QAAQQ,IAAI;gBAClBC,WAAWT,QAAQS,SAAS;gBAC5BC,SAASV,QAAQU,OAAO;gBACxBC,UAAUX,QAAQW,QAAQ;gBAC1BC,OAAOZ,QAAQY,KAAK;gBACpBC,OAAOb,QAAQa,KAAK;gBACpBC,cAAcd,QAAQc,YAAY;gBAClCC,QAAQf,QAAQe,MAAM;gBACtBC,QAAQhB,QAAQgB,MAAM;gBACtBC,UAAUjB,QAAQiB,QAAQ;gBAC1Bf,OAAO;uBAAI,IAAI7C,IAAI6C,MAAMlD,GAAG,CAACiC;iBAAQ;gBACrCiC,OAAOlB,QAAQkB,KAAK;gBACpBxE,OAAOkD,KAAKlD,KAAK;YACnB;YACA,OAAO;QACT;QACA,OAAS;gBA4BAgD;YA3BP,MAAMyB,QAAQ3B,SAAS1J,qBAAqBU,WAAW,GAAG;YAC1D,MAAO,CAAC8I,QAAQV,MAAMvH,MAAM,IAAI+H,OAAO/H,MAAM,GAAG8J,MAAO;gBACrD,MAAMC,MAAM3B;gBACZ,IAAI2B,QAAQ,QAAQ;gBACpB,IAAIA,KAAK;oBACP9B,OAAO8B;oBACP;gBACF;gBACA,MAAM5D,OAAOoB,MAAMnB,KAAK;gBACxB,MAAM/E,MAAMuG,MAAMzB,KAAK9E,GAAG;gBAC1B,IAAIqG,KAAKrB,GAAG,CAAChF,MAAM;gBACnB,IAAI,CAACoE,QAAQpE,MAAM;oBACjB0D,KAAK;wBAAE3E,KAAKiB;wBAAKmH,QAAQ;oBAAS;oBAClC,IAAIrC,KAAKqB,OAAO,KAAK,QAAQW,SAAS;oBACtC;gBACF;gBACAL,YAAY;gBACZJ,KAAKpB,GAAG,CAACjF;gBACT,MAAMiH,MAAMlF,KAAKsB,GAAG,CAAChG,cAAcwF,mBAAmB8D;gBACtDA,YAAYM;gBACZP,OAAO9C,IAAI,CACTb,QAAQ/C,KAAK,uCAAuC,0CAA0CiH,KAAK0B,IAAI,CAAC,CAACzB,OAAU,CAAA;wBAAEpC;wBAAM9E;wBAAKiH;wBAAKC;oBAAK,CAAA;YAE9I;YACA,IAAI,CAACR,OAAO/H,MAAM,EAAE;YACpB,MAAMiK,QAAQ,MAAOlC,OAAO3B,KAAK;YACjC4B,YAAYiC,MAAM3B,GAAG;YACrBL,QAAOI,UAAAA,OAAO4B,kBAAP5B,UAAiBJ;QAC1B;QACA3F,UAAUS,SAAS,GAAGkF;QACtB,wEAAwE;QACxE,qDAAqD;QACrD,IAAIA,QAAQ,CAACvC,MAAM;YACjB,MAAMwE,SAAS;mBAAIhC;aAAO;YAC1B,KAAK,MAAM/B,QAAQoB,MAAO;gBACxB,MAAMlG,MAAMuG,MAAMzB,KAAK9E,GAAG;gBAC1B,IAAI,CAACqG,KAAKrB,GAAG,CAAChF,QAAQ,CAAC6I,OAAO7J,QAAQ,CAACgB,MAAM6I,OAAOjF,IAAI,CAAC5D;YAC3D;YACA,KAAK,MAAMjB,OAAO8J,OAAQnF,KAAK;gBAAE3E;gBAAKoI,QAAQ;YAAc;QAC9D;QACA,OAAOtF;IACT,SAAU;YAERrB;QADAsI,aAAapG;SACblC,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBuI,mBAAmB,CAAC,SAAS1G;IAC/C;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAe2G,yBACpB5K,MAAc,EACd6K,QAAgB,EAChB9K,QAA8B,CAAC,CAAC;QAEdA,cAmBLkC;IAnBb,MAAMM,aAAYxC,eAAAA,MAAMyC,KAAK,YAAXzC,eAAe5B;IACjC,MAAM4C,OAAO0B,OAAOzC,iBAAAA,SAAU,IAAI0C,IAAI,GAAGC,WAAW,GAAG3B,OAAO,CAAC,UAAU;IACzE,IAAI,CAACnB,eAAeyB,IAAI,CAACP,SAAS9C,sBAAsB8C,OAAO,OAAO;QAAE+J,UAAU;QAAOC,QAAQ,EAAE;IAAC;IACpG,MAAM9K,UAAUa,kBAAkBC;IAClC,MAAMkB,SAAS,MAAMM,UAAU,GAAGtC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;QAC/C6E,UAAU9F,qBAAqBC,YAAY;QAC3C2F,QAAQ;QACRC,cAAc;QACdE,WAAWhG;QACXoB,YAAY,MAAML,mBAAmBC,OAAOgB,MAAMd;QAClD+E,WAAWhG,qBAAqBI,gBAAgB;QAChD8F,iBAAiB,CAACC,OAAOC,KAAOA,GAAGlE,QAAQ,KAAK,YAAYjB,QAAQW,QAAQ,CAACwE,GAAGvE,MAAM;IACxF;IACA,IAAI,CAACoB,OAAO0D,EAAE,EAAE;QACd,qEAAqE;QACrE,oDAAoD;QACpD,MAAMmF,WAAW7I,OAAOC,OAAO,KAAK,iBAAiBD,OAAOC,OAAO,KAAK,sBAAsBD,OAAOC,OAAO,KAAK;QACjH,OAAO;YAAE4I;YAAUC,QAAQ,EAAE;YAAEpK,KAAKsB,OAAOtB,GAAG;QAAC;IACjD;IACA,MAAMqK,QAAO/I,sBAAAA,OAAO6D,IAAI,CAACwB,KAAK,CAAC,WAAW,CAAC,EAAE,YAAhCrF,sBAAoC;IACjD,OAAO;QAAE6I,UAAU;QAAMC,QAAQ1M,eAAe2M,MAAMH;QAAWlK,KAAKsB,OAAOtB,GAAG;IAAC;AACnF;AAEA;;;CAGC,GACD,OAAO,SAASsK,yBAAyBnC,IAAoB,EAAEoC,WAAW,KAAM;IAC9E,MAAMC,QAAkB,EAAE;IAC1B,IAAIrC,KAAKU,KAAK,EAAE2B,MAAM3F,IAAI,CAAC,CAAC,OAAO,EAAEsD,KAAKU,KAAK,EAAE;IACjD,IAAIV,KAAKW,WAAW,EAAE0B,MAAM3F,IAAI,CAAC,CAAC,aAAa,EAAEsD,KAAKW,WAAW,EAAE;IACnE,KAAK,MAAM2B,SAAStC,KAAKmB,MAAM,CAAE;QAC/B,IAAImB,MAAMC,IAAI,KAAK,WAAWF,MAAM3F,IAAI,CAAC,GAAG,IAAI8F,MAAM,CAACF,MAAMG,KAAK,EAAE,CAAC,EAAEH,MAAMtF,IAAI,EAAE;aAC9E,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,QAAQ,EAAE4F,MAAMI,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAClF,IAAIJ,MAAMC,IAAI,KAAK,aAAaF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aAC5D,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aACxD,IAAIsF,MAAMC,IAAI,KAAK,QAAQ;YAC9B,4DAA4D;YAC5D,IAAI,CAACD,MAAMK,MAAM,EAAEN,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;QAC1C,OAAOqF,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;IAC9B;IACA,OAAO5H,mBAAmB,CAAC,KAAK,EAAE4K,KAAKS,IAAI,CAAC,2BAA2B,CAAC,EAAE4B,MAAMO,IAAI,CAAC,OAAO;QAAER;IAAS;AACzG"}

package/src/lib/site-import/site-import-edge-server.d.ts

@@ -0,0 +1,31 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * The server read the site-import edge defers (AGL-4085): whether this
+ * platform serves a hostname, which opens Firestore and the hosting
+ * platform's API. `site-import-edge.ts` loads it only once a secret is
+ * configured, so a crawl without one never opens either.
+ *
+ * The deferral targets THIS module by relative path, for the reason
+ * `ai-theme-brand-server.ts` gives: an `import()` of `@aglyn/tenant-data-admin`
+ * by its package specifier makes nx treat that lib as lazy-loaded everywhere,
+ * and every static import of it across the plugin, the console and the
+ * tenant then fails lint.
+ *
+ * Import this module dynamically, and only from `site-import-edge.ts`.
+ */
+export { isPlatformServedDomain } from '@aglyn/tenant-data-admin/server/platform-served-domain';

package/src/lib/site-import/site-import-edge-server.js

@@ -0,0 +1,31 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ /**
+ * The server read the site-import edge defers (AGL-4085): whether this
+ * platform serves a hostname, which opens Firestore and the hosting
+ * platform's API. `site-import-edge.ts` loads it only once a secret is
+ * configured, so a crawl without one never opens either.
+ *
+ * The deferral targets THIS module by relative path, for the reason
+ * `ai-theme-brand-server.ts` gives: an `import()` of `@aglyn/tenant-data-admin`
+ * by its package specifier makes nx treat that lib as lazy-loaded everywhere,
+ * and every static import of it across the plugin, the console and the
+ * tenant then fails lint.
+ *
+ * Import this module dynamically, and only from `site-import-edge.ts`.
+ */ export { isPlatformServedDomain } from "@aglyn/tenant-data-admin/server/platform-served-domain";
+
+//# sourceMappingURL=site-import-edge-server.js.map

package/src/lib/site-import/site-import-edge-server.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-edge-server.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * The server read the site-import edge defers (AGL-4085): whether this\n * platform serves a hostname, which opens Firestore and the hosting\n * platform's API. `site-import-edge.ts` loads it only once a secret is\n * configured, so a crawl without one never opens either.\n *\n * The deferral targets THIS module by relative path, for the reason\n * `ai-theme-brand-server.ts` gives: an `import()` of `@aglyn/tenant-data-admin`\n * by its package specifier makes nx treat that lib as lazy-loaded everywhere,\n * and every static import of it across the plugin, the console and the\n * tenant then fails lint.\n *\n * Import this module dynamically, and only from `site-import-edge.ts`.\n */\nexport { isPlatformServedDomain } from '@aglyn/tenant-data-admin/server/platform-served-domain'\n"],"names":["isPlatformServedDomain"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;CAaC,GACD,SAASA,sBAAsB,QAAQ,yDAAwD"}

package/src/lib/site-import/site-import-edge.d.ts

@@ -0,0 +1,44 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** The header the edge rule matches. Lowercase, as the rule declares it. */
+export declare const SITE_IMPORT_EDGE_HEADER = "x-aglyn-import";
+/** A secret shorter than this is treated as unset: `openssl rand -hex 32` makes 64. */
+export declare const SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH = 32;
+/** Extra headers for one request, asked per address. */
+export type SiteImportHeadersFor = (url: URL) => Readonly<Record<string, string>>;
+/** Sends nothing anywhere. */
+export declare const NO_SITE_IMPORT_HEADERS: SiteImportHeadersFor;
+export interface SiteImportEdgeDeps {
+    /** Where the secret is read from; `process.env` by default. */
+    env?: Readonly<Record<string, string | undefined>>;
+    /** Whether this platform serves a hostname; `isPlatformServedDomain` by default. */
+    served?: (hostname: string) => Promise<boolean>;
+}
+/** The edge secret, or null when it is unset or not shaped like one. Never logged. */
+export declare function siteImportEdgeToken(env?: SiteImportEdgeDeps['env']): string | null;
+/**
+ * The edge header for requests to one hostname: the secret when this platform
+ * serves it, else nothing. Never throws.
+ */
+export declare function siteImportEdgeHeaders(hostname: string, deps?: SiteImportEdgeDeps): Promise<Readonly<Record<string, string>>>;
+/**
+ * Per-request headers for a crawl of these origins: each origin's hostname is
+ * decided once, and a request is given the header only when it is https on
+ * the default port to an origin decided "served". Anything else, a hop to
+ * another host included, gets nothing.
+ */
+export declare function siteImportEdgeHeadersFor(origins: readonly string[], deps?: SiteImportEdgeDeps): Promise<SiteImportHeadersFor>;

package/src/lib/site-import/site-import-edge.js

@@ -0,0 +1,108 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ /*==========================================
+ * AGLYNBOT THROUGH AGLYN'S OWN EDGE (AGL-4085).
+ *
+ * The tenant deployment's bot protection answers a client that runs no
+ * JavaScript with a 429 challenge page, and AglynBot is such a client. So a
+ * site we host, aglyn.com included, could not be brought in: on 2026-10-20 the
+ * import read 10 of 121 pages, the ten the store-review bypass happens to
+ * admit, and no home page.
+ *
+ * The edge rule "Site import crawler bypass" (declared in
+ * `tools/scripts/lib/firewall-posture.mjs`) lifts that challenge, and nothing
+ * else, for a request carrying BOTH this header with the secret AND a
+ * user agent starting `AglynBot/`. This module is the only place that holds
+ * the secret, and it gives it out for one reason: a request to a hostname this
+ * platform serves (`isPlatformServedDomain`: our host records, then the hosting
+ * platform's live view). A site anywhere else never sees it.
+ *
+ * - The secret is `SITE_IMPORT_EDGE_TOKEN`, a server variable on the console,
+ *   where the crawl runs. Read in the bracket form, so Next never substitutes
+ *   it into a bundle. Unset, or not shaped like a secret, nothing is sent and
+ *   nothing is read: the crawler is challenged as it was before.
+ * - The decision is per hostname and per request: the fetch asks
+ *   `headersFor(url)` again on every redirect hop, so a hop to another host
+ *   carries nothing even where the caller would follow it.
+ * - Rotation: a new value in the console's env and the same value in the
+ *   rule, together. See `apps/docs/docs/operations/firewall-posture.md`.
+ *=========================================*/ /** The header the edge rule matches. Lowercase, as the rule declares it. */ export const SITE_IMPORT_EDGE_HEADER = 'x-aglyn-import';
+/** A secret shorter than this is treated as unset: `openssl rand -hex 32` makes 64. */ export const SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH = 32;
+/** Header-safe characters only, so the value can neither break a header nor fail the rule's match. */ const TOKEN_PATTERN = /^[A-Za-z0-9_-]+$/;
+/** Sends nothing anywhere. */ export const NO_SITE_IMPORT_HEADERS = ()=>({});
+/** The edge secret, or null when it is unset or not shaped like one. Never logged. */ export function siteImportEdgeToken(env) {
+    var _ref;
+    // The direct read is spelled out so the self-host env reference guard sees it.
+    const token = String((_ref = env ? env['SITE_IMPORT_EDGE_TOKEN'] : process.env['SITE_IMPORT_EDGE_TOKEN']) != null ? _ref : '').trim();
+    if (token.length < SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH || token.length > 256) return null;
+    return TOKEN_PATTERN.test(token) ? token : null;
+}
+async function defaultServed(hostname) {
+    // Loaded only once a secret is configured, so a crawl that has none (every
+    // spec, every self-hosted deployment without the rule) never opens Firestore.
+    // By relative path: see `site-import-edge-server.ts`.
+    const { isPlatformServedDomain } = await import("./site-import-edge-server.js");
+    return isPlatformServedDomain(hostname);
+}
+/**
+ * The edge header for requests to one hostname: the secret when this platform
+ * serves it, else nothing. Never throws.
+ */ export async function siteImportEdgeHeaders(hostname, deps = {}) {
+    const token = siteImportEdgeToken(deps.env);
+    if (!token) return {};
+    const name = String(hostname != null ? hostname : '').trim().toLowerCase();
+    if (!name) return {};
+    try {
+        var _deps_served;
+        const served = await ((_deps_served = deps.served) != null ? _deps_served : defaultServed)(name);
+        return served ? {
+            [SITE_IMPORT_EDGE_HEADER]: token
+        } : {};
+    } catch (error) {
+        console.error('[site-import-edge] could not tell whether the platform serves', name, error);
+        return {};
+    }
+}
+/**
+ * Per-request headers for a crawl of these origins: each origin's hostname is
+ * decided once, and a request is given the header only when it is https on
+ * the default port to an origin decided "served". Anything else, a hop to
+ * another host included, gets nothing.
+ */ export async function siteImportEdgeHeadersFor(origins, deps = {}) {
+    if (!siteImportEdgeToken(deps.env)) return NO_SITE_IMPORT_HEADERS;
+    const decided = new Map();
+    for (const raw of origins){
+        let origin;
+        try {
+            origin = new URL(raw);
+        } catch (unused) {
+            continue;
+        }
+        if (!isPlainHttpsOrigin(origin) || decided.has(origin.origin)) continue;
+        const headers = await siteImportEdgeHeaders(origin.hostname, deps);
+        if (Object.keys(headers).length) decided.set(origin.origin, headers);
+    }
+    if (!decided.size) return NO_SITE_IMPORT_HEADERS;
+    return (url)=>{
+        var _decided_get;
+        return isPlainHttpsOrigin(url) ? (_decided_get = decided.get(url.origin)) != null ? _decided_get : {} : {};
+    };
+}
+function isPlainHttpsOrigin(url) {
+    return url.protocol === 'https:' && (!url.port || url.port === '443') && !url.username && !url.password;
+}
+
+//# sourceMappingURL=site-import-edge.js.map

package/src/lib/site-import/site-import-edge.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-edge.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * AGLYNBOT THROUGH AGLYN'S OWN EDGE (AGL-4085).\n *\n * The tenant deployment's bot protection answers a client that runs no\n * JavaScript with a 429 challenge page, and AglynBot is such a client. So a\n * site we host, aglyn.com included, could not be brought in: on 2026-10-20 the\n * import read 10 of 121 pages, the ten the store-review bypass happens to\n * admit, and no home page.\n *\n * The edge rule \"Site import crawler bypass\" (declared in\n * `tools/scripts/lib/firewall-posture.mjs`) lifts that challenge, and nothing\n * else, for a request carrying BOTH this header with the secret AND a\n * user agent starting `AglynBot/`. This module is the only place that holds\n * the secret, and it gives it out for one reason: a request to a hostname this\n * platform serves (`isPlatformServedDomain`: our host records, then the hosting\n * platform's live view). A site anywhere else never sees it.\n *\n * - The secret is `SITE_IMPORT_EDGE_TOKEN`, a server variable on the console,\n *   where the crawl runs. Read in the bracket form, so Next never substitutes\n *   it into a bundle. Unset, or not shaped like a secret, nothing is sent and\n *   nothing is read: the crawler is challenged as it was before.\n * - The decision is per hostname and per request: the fetch asks\n *   `headersFor(url)` again on every redirect hop, so a hop to another host\n *   carries nothing even where the caller would follow it.\n * - Rotation: a new value in the console's env and the same value in the\n *   rule, together. See `apps/docs/docs/operations/firewall-posture.md`.\n *=========================================*/\n\n/** The header the edge rule matches. Lowercase, as the rule declares it. */\nexport const SITE_IMPORT_EDGE_HEADER = 'x-aglyn-import'\n\n/** A secret shorter than this is treated as unset: `openssl rand -hex 32` makes 64. */\nexport const SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH = 32\n\n/** Header-safe characters only, so the value can neither break a header nor fail the rule's match. */\nconst TOKEN_PATTERN = /^[A-Za-z0-9_-]+$/\n\n/** Extra headers for one request, asked per address. */\nexport type SiteImportHeadersFor = (url: URL) => Readonly<Record<string, string>>\n\n/** Sends nothing anywhere. */\nexport const NO_SITE_IMPORT_HEADERS: SiteImportHeadersFor = () => ({})\n\nexport interface SiteImportEdgeDeps {\n  /** Where the secret is read from; `process.env` by default. */\n  env?: Readonly<Record<string, string | undefined>>\n  /** Whether this platform serves a hostname; `isPlatformServedDomain` by default. */\n  served?: (hostname: string) => Promise<boolean>\n}\n\n/** The edge secret, or null when it is unset or not shaped like one. Never logged. */\nexport function siteImportEdgeToken(env?: SiteImportEdgeDeps['env']): string | null {\n  // The direct read is spelled out so the self-host env reference guard sees it.\n  const token = String((env ? env['SITE_IMPORT_EDGE_TOKEN'] : process.env['SITE_IMPORT_EDGE_TOKEN']) ?? '').trim()\n  if (token.length < SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH || token.length > 256) return null\n  return TOKEN_PATTERN.test(token) ? token : null\n}\n\nasync function defaultServed(hostname: string): Promise<boolean> {\n  // Loaded only once a secret is configured, so a crawl that has none (every\n  // spec, every self-hosted deployment without the rule) never opens Firestore.\n  // By relative path: see `site-import-edge-server.ts`.\n  const { isPlatformServedDomain } = await import('./site-import-edge-server')\n  return isPlatformServedDomain(hostname)\n}\n\n/**\n * The edge header for requests to one hostname: the secret when this platform\n * serves it, else nothing. Never throws.\n */\nexport async function siteImportEdgeHeaders(\n  hostname: string,\n  deps: SiteImportEdgeDeps = {},\n): Promise<Readonly<Record<string, string>>> {\n  const token = siteImportEdgeToken(deps.env)\n  if (!token) return {}\n  const name = String(hostname ?? '').trim().toLowerCase()\n  if (!name) return {}\n  try {\n    const served = await (deps.served ?? defaultServed)(name)\n    return served ? { [SITE_IMPORT_EDGE_HEADER]: token } : {}\n  } catch (error) {\n    console.error('[site-import-edge] could not tell whether the platform serves', name, error)\n    return {}\n  }\n}\n\n/**\n * Per-request headers for a crawl of these origins: each origin's hostname is\n * decided once, and a request is given the header only when it is https on\n * the default port to an origin decided \"served\". Anything else, a hop to\n * another host included, gets nothing.\n */\nexport async function siteImportEdgeHeadersFor(\n  origins: readonly string[],\n  deps: SiteImportEdgeDeps = {},\n): Promise<SiteImportHeadersFor> {\n  if (!siteImportEdgeToken(deps.env)) return NO_SITE_IMPORT_HEADERS\n  const decided = new Map<string, Readonly<Record<string, string>>>()\n  for (const raw of origins) {\n    let origin: URL\n    try {\n      origin = new URL(raw)\n    } catch {\n      continue\n    }\n    if (!isPlainHttpsOrigin(origin) || decided.has(origin.origin)) continue\n    const headers = await siteImportEdgeHeaders(origin.hostname, deps)\n    if (Object.keys(headers).length) decided.set(origin.origin, headers)\n  }\n  if (!decided.size) return NO_SITE_IMPORT_HEADERS\n  return (url: URL) => (isPlainHttpsOrigin(url) ? decided.get(url.origin) ?? {} : {})\n}\n\nfunction isPlainHttpsOrigin(url: URL): boolean {\n  return url.protocol === 'https:' && (!url.port || url.port === '443') && !url.username && !url.password\n}\n"],"names":["SITE_IMPORT_EDGE_HEADER","SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH","TOKEN_PATTERN","NO_SITE_IMPORT_HEADERS","siteImportEdgeToken","env","token","String","process","trim","length","test","defaultServed","hostname","isPlatformServedDomain","siteImportEdgeHeaders","deps","name","toLowerCase","served","error","console","siteImportEdgeHeadersFor","origins","decided","Map","raw","origin","URL","isPlainHttpsOrigin","has","headers","Object","keys","set","size","url","get","protocol","port","username","password"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;2CA0B2C,GAE3C,0EAA0E,GAC1E,OAAO,MAAMA,0BAA0B,iBAAgB;AAEvD,qFAAqF,GACrF,OAAO,MAAMC,oCAAoC,GAAE;AAEnD,oGAAoG,GACpG,MAAMC,gBAAgB;AAKtB,4BAA4B,GAC5B,OAAO,MAAMC,yBAA+C,IAAO,CAAA,CAAC,CAAA,EAAE;AAStE,oFAAoF,GACpF,OAAO,SAASC,oBAAoBC,GAA+B;QAE3CA;IADtB,+EAA+E;IAC/E,MAAMC,QAAQC,QAAQF,OAAAA,MAAMA,GAAG,CAAC,yBAAyB,GAAGG,QAAQH,GAAG,CAAC,yBAAyB,YAA3EA,OAAgF,IAAII,IAAI;IAC9G,IAAIH,MAAMI,MAAM,GAAGT,qCAAqCK,MAAMI,MAAM,GAAG,KAAK,OAAO;IACnF,OAAOR,cAAcS,IAAI,CAACL,SAASA,QAAQ;AAC7C;AAEA,eAAeM,cAAcC,QAAgB;IAC3C,2EAA2E;IAC3E,8EAA8E;IAC9E,sDAAsD;IACtD,MAAM,EAAEC,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC;IAChD,OAAOA,uBAAuBD;AAChC;AAEA;;;CAGC,GACD,OAAO,eAAeE,sBACpBF,QAAgB,EAChBG,OAA2B,CAAC,CAAC;IAE7B,MAAMV,QAAQF,oBAAoBY,KAAKX,GAAG;IAC1C,IAAI,CAACC,OAAO,OAAO,CAAC;IACpB,MAAMW,OAAOV,OAAOM,mBAAAA,WAAY,IAAIJ,IAAI,GAAGS,WAAW;IACtD,IAAI,CAACD,MAAM,OAAO,CAAC;IACnB,IAAI;YACoBD;QAAtB,MAAMG,SAAS,MAAM,EAACH,eAAAA,KAAKG,MAAM,YAAXH,eAAeJ,eAAeK;QACpD,OAAOE,SAAS;YAAE,CAACnB,wBAAwB,EAAEM;QAAM,IAAI,CAAC;IAC1D,EAAE,OAAOc,OAAO;QACdC,QAAQD,KAAK,CAAC,iEAAiEH,MAAMG;QACrF,OAAO,CAAC;IACV;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAeE,yBACpBC,OAA0B,EAC1BP,OAA2B,CAAC,CAAC;IAE7B,IAAI,CAACZ,oBAAoBY,KAAKX,GAAG,GAAG,OAAOF;IAC3C,MAAMqB,UAAU,IAAIC;IACpB,KAAK,MAAMC,OAAOH,QAAS;QACzB,IAAII;QACJ,IAAI;YACFA,SAAS,IAAIC,IAAIF;QACnB,EAAE,eAAM;YACN;QACF;QACA,IAAI,CAACG,mBAAmBF,WAAWH,QAAQM,GAAG,CAACH,OAAOA,MAAM,GAAG;QAC/D,MAAMI,UAAU,MAAMhB,sBAAsBY,OAAOd,QAAQ,EAAEG;QAC7D,IAAIgB,OAAOC,IAAI,CAACF,SAASrB,MAAM,EAAEc,QAAQU,GAAG,CAACP,OAAOA,MAAM,EAAEI;IAC9D;IACA,IAAI,CAACP,QAAQW,IAAI,EAAE,OAAOhC;IAC1B,OAAO,CAACiC;YAAwCZ;eAA1BK,mBAAmBO,QAAOZ,eAAAA,QAAQa,GAAG,CAACD,IAAIT,MAAM,aAAtBH,eAA2B,CAAC,IAAI,CAAC;;AACnF;AAEA,SAASK,mBAAmBO,GAAQ;IAClC,OAAOA,IAAIE,QAAQ,KAAK,YAAa,CAAA,CAACF,IAAIG,IAAI,IAAIH,IAAIG,IAAI,KAAK,KAAI,KAAM,CAACH,IAAII,QAAQ,IAAI,CAACJ,IAAIK,QAAQ;AACzG"}