@aglyn/plugins-ai 1.0.0-beta.289 → 1.0.0-beta.291

package/src/lib/site-import/site-import-crawl.d.ts

@@ -155,10 +155,12 @@ export interface SiteImportCrawlSeams {
     now?: () => number;
     /**
      * Request headers for the site being read, asked once per crawl with its
-     * domain (AGL-4083): the edge header a site this platform serves lets
-     * AglynBot past its own bot protection with. Sent on every request to the
-     * site, under the crawler's own `Accept` and `User-Agent`, and only to the
-     * site: every address the crawler asks for is on it. None when absent.
+     * domain (AGL-4083, AGL-4085). Absent, the crawl asks the edge
+     * (`siteImportEdgeHeadersFor`): Aglyn's edge secret for each of the site's
+     * two origins this platform serves, decided origin by origin, and nothing
+     * anywhere else. Either way the headers go only to the site's two origins,
+     * and the fetch asks again on every redirect hop, so a hop anywhere else
+     * carries nothing. Always under the crawler's own `Accept` and `User-Agent`.
      */
     headersFor?: (domain: string) => Promise<Readonly<Record<string, string>>> | Readonly<Record<string, string>>;
 }

package/src/lib/site-import/site-import-crawl.js

@@ -36,6 +36,10 @@ var _process_env_SITE_IMPORT_BOT_INFO_URL;
  *   group, else `*`. A robots.txt that answers 404 allows everything; one
  *   that cannot be read at all (5xx, no answer) stops the crawl, because
  *   "could not ask" is not "allowed".
+ * - **Our own edge.** A site this platform serves gets the edge secret
+ *   (`site-import-edge.ts`, AGL-4085), which lifts our bot challenge for
+ *   AglynBot; it is asked per request and per redirect hop, and a request
+ *   that leaves the site's two origins never carries it.
  * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an
  *   index's children included), then same-site links found on the pages read,
  *   for whatever the sitemap missed.
@@ -61,6 +65,7 @@ import { fenceUntrustedText } from "@aglyn/shared-util-tools/untrusted-text";
 import { fetchPublicTextDetailed } from "../jobs/ai-theme-brand-inputs.js";
 import { extractSiteImportPage, readMetaValues } from "./site-import-html.js";
 import { AGLYN_BOT_TOKEN, parseRobotsTxt, parseSitemap, robotsAllows, robotsRulesFor } from "./site-import-robots.js";
+import { NO_SITE_IMPORT_HEADERS, siteImportEdgeHeadersFor } from "./site-import-edge.js";
 /**
  * Where the bot's user agent points a site owner who wants to know what it is
  * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on
@@ -88,18 +93,25 @@ export const SITE_IMPORT_DEFAULTS = {
 /** Never a page: files a link may point at that are not HTML. */ const NON_PAGE_EXTENSION = /\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i;
 const TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i;
 const DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\.[a-z0-9-]{1,63})+$/;
-/** The seam's headers for a domain; none when it has none or cannot say. */ async function siteImportHeadersFor(seams, domain) {
-    if (!seams.headersFor) return null;
+/**
+ * The per-request headers of a crawl of `domain`: the seam's, else the edge's,
+ * and none for any address off the site's two origins. Never throws.
+ */ async function siteRequestHeaders(seams, domain, origins) {
+    let inner = NO_SITE_IMPORT_HEADERS;
     try {
-        const headers = await seams.headersFor(domain);
-        return headers && Object.keys(headers).length ? headers : null;
+        if (seams.headersFor) {
+            const headers = await seams.headersFor(domain);
+            if (headers && Object.keys(headers).length) inner = ()=>headers;
+        } else {
+            inner = await siteImportEdgeHeadersFor(origins);
+        }
     } catch (error) {
         console.warn('[site-import] the edge headers could not be read; reading without them', {
             domain,
             error
         });
-        return null;
     }
+    return (url)=>origins.includes(url.origin) ? inner(url) : {};
 }
 /** The site's two origins for a domain. */ export function siteImportOrigins(domain) {
     const bare = domain.replace(/^www\./, '');
@@ -174,6 +186,8 @@ function skipReasonFor(result) {
         inventory.refused = 'lookalike';
         return finish();
     }
+    // Asked once, before anything else, and held to the site's two origins.
+    const headersFor = await siteRequestHeaders(seams, domain, origins);
     const maxPages = Math.max(0, Math.floor(options.maxPages));
     const maxPageBytes = (_options_maxPageBytes = options.maxPageBytes) != null ? _options_maxPageBytes : SITE_IMPORT_DEFAULTS.maxPageBytes;
     const maxTotalBytes = (_options_maxTotalBytes = options.maxTotalBytes) != null ? _options_maxTotalBytes : SITE_IMPORT_DEFAULTS.maxTotalBytes;
@@ -186,20 +200,17 @@ function skipReasonFor(result) {
     const sameSite = (url)=>origins.includes(url.origin) && url.protocol === 'https:';
     const remainingBytes = ()=>maxTotalBytes - inventory.bytesRead;
     const outOfTime = ()=>controller.signal.aborted || now() >= deadlineAt;
-    // Asked once, before anything else; every address below is on the site.
-    const extraHeaders = await siteImportHeadersFor(seams, domain);
-    const request = (url, accept, acceptHeader, maxBytes)=>fetchText(url, _extends({
+    const request = (url, accept, acceptHeader, maxBytes)=>fetchText(url, {
             maxBytes,
             accept,
             acceptHeader,
             userAgent: AGLYN_BOT_USER_AGENT,
+            headersFor,
             signal: controller.signal,
             timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),
             redirectAllowed: (_from, to)=>sameSite(to),
             refuseOver: true
-        }, extraHeaders ? {
-            headers: extraHeaders
-        } : {}));
+        });
     const skip = (entry)=>{
         if (inventory.skipped.length < SITE_IMPORT_DEFAULTS.maxSkipped) inventory.skipped.push(entry);
         else inventory.skippedOverflow += 1;
@@ -494,17 +505,15 @@ function skipReasonFor(result) {
         values: []
     };
     const origins = siteImportOrigins(bare);
-    const extraHeaders = await siteImportHeadersFor(seams, bare);
-    const result = await fetchText(`${origins[0]}/`, _extends({
+    const result = await fetchText(`${origins[0]}/`, {
         maxBytes: SITE_IMPORT_DEFAULTS.maxPageBytes,
         accept: /text\/html|application\/xhtml\+xml/i,
         acceptHeader: 'text/html, application/xhtml+xml;q=0.9',
         userAgent: AGLYN_BOT_USER_AGENT,
+        headersFor: await siteRequestHeaders(seams, bare, origins),
         timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,
         redirectAllowed: (_from, to)=>to.protocol === 'https:' && origins.includes(to.origin)
-    }, extraHeaders ? {
-        headers: extraHeaders
-    } : {}));
+    });
     if (!result.ok) {
         // An answer that is not the page (a 404, a redirect off the site) is
         // still an answer: the tag is not where it must be.

package/src/lib/site-import/site-import-crawl.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-crawl.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * THE SITE-IMPORT CRAWLER (AGL-3974, parent AGL-3973).\n *\n * Reads a website a workspace has PROVED it owns (domain-ownership.ts in\n * tenant-data-admin; the doors check the proof before calling this) into a\n * typed inventory: each page's title, description, headings and text in\n * order, images and menu links. It writes nothing; the import job decides\n * what becomes a page.\n *\n * Every request goes through `fetchPublicTextDetailed`, the theme job's\n * SSRF-guarded fetch: https on the default port only, public addresses only,\n * the socket pinned to the address checked. On top of that:\n *\n * - **One site.** The site is `https://<domain>` and `https://www.<domain>`,\n *   one site to its owner. Every redirect hop and every discovered link must\n *   stay on it; a page that redirects anywhere else is refused, not followed.\n * - **AglynBot.** Every request says `AglynBot/1.0 (+<the operator's bot page>)`,\n *   and robots.txt is read first and honored for every page: the AglynBot\n *   group, else `*`. A robots.txt that answers 404 allows everything; one\n *   that cannot be read at all (5xx, no answer) stops the crawl, because\n *   \"could not ask\" is not \"allowed\".\n * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an\n *   index's children included), then same-site links found on the pages read,\n *   for whatever the sitemap missed.\n * - **Budgets.** Pages (the caller's, from the plan), bytes per page (a page\n *   over it is refused, not cut), bytes in total, and wall-clock time. A\n *   whole site's worth fits (AGL-4083): aglyn.com's 123 pages average 440 KB,\n *   54 MB in all, so the total is 160 MB; a page is parsed and dropped as it\n *   is read, so what is held is its blocks, never its bytes.\n * - **Four at a time.** robots.txt, the sitemaps and the home page are read\n *   one after another; the rest four at a time, and the inventory keeps the\n *   order they were asked in, which is the sitemap's.\n * - **Nothing vanishes.** An address a stop (the clock, the bytes, the page\n *   budget) left unread is listed as `not-reached`, so the inventory's\n *   counts add up to the site. A wave's own read leaves its unread pages for\n *   its next pass instead.\n * - **No lookalikes.** A domain that wears a brand it is not\n *   (`lookalikeBrandForHost`) is refused before any request.\n * - **No sign-in pages.** A page with a password field is refused and\n *   reported; forms with payment fields, scripts, iframes and embeds never\n *   reach the inventory (site-import-html.ts).\n *=========================================*/\n\nimport { lookalikeBrandForHost } from '@aglyn/shared-util-email/outbound-phishing-screen'\nimport { fenceUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n  fetchPublicTextDetailed,\n  type PublicFetchEdge,\n  type PublicFetchOptions,\n  type PublicFetchResult,\n} from '../jobs/ai-theme-brand-inputs'\nimport {\n  extractSiteImportPage,\n  readMetaValues,\n  type SiteImportBlock,\n  type SiteImportBrandMark,\n  type SiteImportForm,\n  type SiteImportIcon,\n  type SiteImportImage,\n  type SiteImportNavLink,\n  type SiteImportOrganization,\n} from './site-import-html'\nimport {\n  AGLYN_BOT_TOKEN,\n  parseRobotsTxt,\n  parseSitemap,\n  robotsAllows,\n  robotsRulesFor,\n  type RobotsRule,\n} from './site-import-robots'\n\n/**\n * Where the bot's user agent points a site owner who wants to know what it is\n * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on\n * a self-hosted deployment, so its crawler never names Aglyn's address; the\n * literal is only that variable's default.\n *\n * Aglyn's is the Bring my site guide, which says what AglynBot reads and how\n * robots.txt keeps it out. There is no page of its own (Zach, 10/20: \"we don't\n * need a bot page at all\"), and the site it reads is the reader's own: an\n * import starts only once its owner has proved the site is theirs.\n */\nexport const AGLYN_BOT_INFO_URL =\n  process.env['SITE_IMPORT_BOT_INFO_URL'] ?? 'https://docs.aglyn.com/ai/bring-my-site'\n\n/** What every crawler request says it is. */\nexport const AGLYN_BOT_USER_AGENT = `AglynBot/1.0 (+${AGLYN_BOT_INFO_URL})`\n\nexport const SITE_IMPORT_DEFAULTS = {\n  maxPageBytes: 2 * 1024 * 1024,\n  /** A whole site's worth (AGL-4083): 54 MB for aglyn.com's 123 pages, with room. */\n  maxTotalBytes: 160 * 1024 * 1024,\n  timeBudgetMs: 60_000,\n  /** One request's own clock, inside the whole crawl's. */\n  requestTimeoutMs: 8_000,\n  robotsMaxBytes: 256 * 1024,\n  sitemapMaxBytes: 4 * 1024 * 1024,\n  /** Sitemap documents read, an index's children included: an index of six children is seven. */\n  maxSitemaps: 24,\n  /** URLs a sitemap may contribute to the queue. */\n  maxSitemapUrls: 2_000,\n  /** Reasons kept in `skipped` before the list stops growing: every address a sitemap may name. */\n  maxSkipped: 2_000,\n  /** Pages read at once, after robots.txt, the sitemaps and the home page. */\n  concurrency: 4,\n} as const\n\n/** Never a page: files a link may point at that are not HTML. */\nconst NON_PAGE_EXTENSION =\n  /\\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i\n\nconst TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i\n\nconst DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\\.[a-z0-9-]{1,63})+$/\n\nexport type SiteImportSkipReason =\n  /** The page has a password field: a sign-in page, refused outright. */\n  | 'login-form'\n  /** robots.txt keeps AglynBot off this path. */\n  | 'robots'\n  /** A redirect left the site. */\n  | 'off-origin-redirect'\n  /** Over the per-page byte cap. */\n  | 'too-large'\n  /** Answered with something that is not HTML. */\n  | 'not-html'\n  /** Answered 4xx or 5xx. */\n  | 'http-error'\n  /** No answer, a private address, or a refused address. */\n  | 'unreachable'\n  /** Never asked for: a stop (time, bytes, the page budget) came first (AGL-4083). */\n  | 'not-reached'\n\nexport interface SiteImportSkipped {\n  url: string\n  reason: SiteImportSkipReason\n  status?: number\n  /** Where an off-origin redirect pointed. */\n  location?: string\n  /** Whose edge turned an error answer away, when it said (AGL-4083). */\n  edge?: PublicFetchEdge\n}\n\nexport interface SiteImportPage {\n  /** The address the page was read from, after any same-site redirect. */\n  url: string\n  path: string\n  /** Where the crawler learned of the page. */\n  foundBy: 'home' | 'sitemap' | 'link'\n  title: string\n  description: string\n  lang: string | null\n  canonical: string | null\n  /** The share card (AGL-4083). */\n  ogImage: string | null\n  /** The site's own name, as its head gives it. */\n  siteName: string\n  icons: SiteImportIcon[]\n  /** The header's link home: the brand's logo and words. */\n  brand: SiteImportBrandMark | null\n  organization: SiteImportOrganization | null\n  blocks: SiteImportBlock[]\n  images: SiteImportImage[]\n  navLinks: SiteImportNavLink[]\n  /** Same-site page addresses this page links to. */\n  links: string[]\n  forms: { kept: SiteImportForm[]; removed: number }\n  bytes: number\n}\n\nexport type SiteImportStop =\n  | 'page-budget'\n  | 'total-bytes'\n  | 'time'\n  /** robots.txt could not be read, so nothing was asked of the site. */\n  | 'robots-unreachable'\n\nexport interface SiteImportInventory {\n  domain: string\n  /** The site's origins: the bare name and its `www.` twin. */\n  origins: string[]\n  /** Set when nothing was fetched at all. */\n  refused: 'invalid-domain' | 'lookalike' | null\n  pages: SiteImportPage[]\n  skipped: SiteImportSkipped[]\n  /** Skips past {@link SITE_IMPORT_DEFAULTS.maxSkipped}, counted only. */\n  skippedOverflow: number\n  robots: { found: boolean; sitemaps: string[] }\n  /** Page addresses the sitemaps listed (same-site, before robots and budget). */\n  sitemapUrls: number\n  stoppedBy: SiteImportStop | null\n  bytesRead: number\n  startedAt: number\n  finishedAt: number\n}\n\nexport interface SiteImportCrawlOptions {\n  /** The proved domain; `www.` is folded in. */\n  domain: string\n  /** The most pages the inventory may hold: the plan's page count. */\n  maxPages: number\n  maxPageBytes?: number\n  maxTotalBytes?: number\n  timeBudgetMs?: number\n  signal?: AbortSignal\n  /**\n   * Read the sitemap's shallow addresses before its deep ones (AGL-3976), so a\n   * crawl the page budget stops reads a site's main pages before the tail of\n   * its blog. A sitemap's own order is kept among addresses of one depth.\n   */\n  shallowFirst?: boolean\n  /**\n   * Read exactly these addresses of the site, in order, and nothing else\n   * (AGL-3976): an import's wave reads its own pages again when it starts. No\n   * sitemap is read and no link is followed; robots.txt is read and honored\n   * as for any crawl, and every other rule holds.\n   */\n  only?: readonly string[]\n}\n\nexport interface SiteImportCrawlSeams {\n  fetch?: (url: string, options: PublicFetchOptions) => Promise<PublicFetchResult>\n  now?: () => number\n  /**\n   * Request headers for the site being read, asked once per crawl with its\n   * domain (AGL-4083): the edge header a site this platform serves lets\n   * AglynBot past its own bot protection with. Sent on every request to the\n   * site, under the crawler's own `Accept` and `User-Agent`, and only to the\n   * site: every address the crawler asks for is on it. None when absent.\n   */\n  headersFor?: (domain: string) => Promise<Readonly<Record<string, string>>> | Readonly<Record<string, string>>\n}\n\n/** The seam's headers for a domain; none when it has none or cannot say. */\nasync function siteImportHeadersFor(seams: SiteImportCrawlSeams, domain: string): Promise<Readonly<Record<string, string>> | null> {\n  if (!seams.headersFor) return null\n  try {\n    const headers = await seams.headersFor(domain)\n    return headers && Object.keys(headers).length ? headers : null\n  } catch (error) {\n    console.warn('[site-import] the edge headers could not be read; reading without them', { domain, error })\n    return null\n  }\n}\n\n/** The site's two origins for a domain. */\nexport function siteImportOrigins(domain: string): string[] {\n  const bare = domain.replace(/^www\\./, '')\n  return [`https://${bare}`, `https://www.${bare}`]\n}\n\n/** A crawlable page address on the site, normalized for de-duplication; else null. */\nexport function siteImportPageKey(url: URL, origins: readonly string[]): string | null {\n  if (url.protocol !== 'https:' || url.username || url.password) return null\n  if (url.port && url.port !== '443') return null\n  if (!origins.includes(url.origin)) return null\n  if (NON_PAGE_EXTENSION.test(url.pathname)) return null\n  const clean = new URL(url.toString())\n  clean.hash = ''\n  for (const key of [...clean.searchParams.keys()]) {\n    if (TRACKING_PARAM.test(key)) clean.searchParams.delete(key)\n  }\n  clean.search = clean.searchParams.toString() ? `?${clean.searchParams.toString()}` : ''\n  return clean.toString()\n}\n\nfunction skipReasonFor(result: PublicFetchResult): SiteImportSkipReason {\n  switch (result.refusal) {\n    case 'redirect-refused':\n      return 'off-origin-redirect'\n    case 'content-type':\n      return 'not-html'\n    case 'too-large':\n      return 'too-large'\n    case 'http-status':\n      return 'http-error'\n    default:\n      return 'unreachable'\n  }\n}\n\n/** Read a proved site into an inventory. Never throws for anything the site does. */\nexport async function crawlSiteForImport(\n  options: SiteImportCrawlOptions,\n  seams: SiteImportCrawlSeams = {},\n): Promise<SiteImportInventory> {\n  const now = seams.now ?? Date.now\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const domain = String(options.domain ?? '').trim().toLowerCase().replace(/\\.+$/, '').replace(/^www\\./, '')\n  const origins = DOMAIN_PATTERN.test(domain) ? siteImportOrigins(domain) : []\n  const startedAt = now()\n  const inventory: SiteImportInventory = {\n    domain,\n    origins,\n    refused: null,\n    pages: [],\n    skipped: [],\n    skippedOverflow: 0,\n    robots: { found: false, sitemaps: [] },\n    sitemapUrls: 0,\n    stoppedBy: null,\n    bytesRead: 0,\n    startedAt,\n    finishedAt: startedAt,\n  }\n  const finish = () => {\n    inventory.finishedAt = now()\n    return inventory\n  }\n  if (!origins.length) {\n    inventory.refused = 'invalid-domain'\n    return finish()\n  }\n  if (lookalikeBrandForHost(domain)) {\n    inventory.refused = 'lookalike'\n    return finish()\n  }\n\n  const maxPages = Math.max(0, Math.floor(options.maxPages))\n  const maxPageBytes = options.maxPageBytes ?? SITE_IMPORT_DEFAULTS.maxPageBytes\n  const maxTotalBytes = options.maxTotalBytes ?? SITE_IMPORT_DEFAULTS.maxTotalBytes\n  const deadlineAt = startedAt + (options.timeBudgetMs ?? SITE_IMPORT_DEFAULTS.timeBudgetMs)\n  const controller = new AbortController()\n  const onAbort = () => controller.abort()\n  options.signal?.addEventListener('abort', onAbort)\n  if (options.signal?.aborted) controller.abort()\n  const timer = setTimeout(() => controller.abort(), Math.max(0, deadlineAt - startedAt))\n\n  const sameSite = (url: URL) => origins.includes(url.origin) && url.protocol === 'https:'\n  const remainingBytes = () => maxTotalBytes - inventory.bytesRead\n  const outOfTime = () => controller.signal.aborted || now() >= deadlineAt\n  // Asked once, before anything else; every address below is on the site.\n  const extraHeaders = await siteImportHeadersFor(seams, domain)\n  const request = (url: string, accept: RegExp, acceptHeader: string, maxBytes: number) =>\n    fetchText(url, {\n      maxBytes,\n      accept,\n      acceptHeader,\n      userAgent: AGLYN_BOT_USER_AGENT,\n      signal: controller.signal,\n      timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),\n      redirectAllowed: (_from, to) => sameSite(to),\n      refuseOver: true,\n      ...(extraHeaders ? { headers: extraHeaders } : {}),\n    })\n  const skip = (entry: SiteImportSkipped) => {\n    if (inventory.skipped.length < SITE_IMPORT_DEFAULTS.maxSkipped) inventory.skipped.push(entry)\n    else inventory.skippedOverflow += 1\n  }\n\n  try {\n    // robots.txt, before anything else is asked of the site.\n    const home = `${origins[0]}/`\n    let rules: RobotsRule[] = []\n    const robots = await request(\n      `${origins[0]}/robots.txt`,\n      /text\\/plain|text\\/html|^$/i,\n      'text/plain',\n      Math.min(SITE_IMPORT_DEFAULTS.robotsMaxBytes, remainingBytes()),\n    )\n    if (robots.ok) {\n      inventory.bytesRead += robots.bytes\n      const parsed = parseRobotsTxt(robots.text)\n      rules = robotsRulesFor(parsed, AGLYN_BOT_TOKEN)\n      inventory.robots = { found: true, sitemaps: parsed.sitemaps }\n    } else if (\n      !(robots.refusal === 'http-status' && (robots.status ?? 0) >= 400 && (robots.status ?? 0) < 500)\n    ) {\n      inventory.stoppedBy = outOfTime() ? 'time' : 'robots-unreachable'\n      return finish()\n    }\n    const allowed = (key: string) => {\n      const url = new URL(key)\n      return robotsAllows(rules, `${url.pathname}${url.search}`)\n    }\n\n    // A wave's own pages (AGL-3976): exactly those, after robots.txt.\n    const only = options.only\n      ? options.only\n          .map((raw) => {\n            try {\n              return siteImportPageKey(new URL(raw), origins)\n            } catch {\n              return null\n            }\n          })\n          .filter((key): key is string => !!key)\n      : null\n\n    // Sitemaps: the ones robots.txt names on this site, else /sitemap.xml.\n    const sitemapQueue: string[] = []\n    for (const raw of inventory.robots.sitemaps) {\n      try {\n        const url = new URL(raw)\n        if (sameSite(url)) sitemapQueue.push(url.toString())\n      } catch {\n        // A malformed Sitemap line names nothing.\n      }\n    }\n    if (!sitemapQueue.length && !only) sitemapQueue.push(`${origins[0]}/sitemap.xml`)\n    if (only) sitemapQueue.length = 0\n    const sitemapSeen = new Set<string>()\n    const fromSitemap: string[] = []\n    let sitemapsRead = 0\n    while (sitemapQueue.length && sitemapsRead < SITE_IMPORT_DEFAULTS.maxSitemaps && !outOfTime()) {\n      const next = sitemapQueue.shift() as string\n      if (sitemapSeen.has(next)) continue\n      sitemapSeen.add(next)\n      if (/\\.gz$/i.test(new URL(next).pathname)) continue\n      if (!allowed(next)) continue\n      const room = Math.min(SITE_IMPORT_DEFAULTS.sitemapMaxBytes, remainingBytes())\n      if (room <= 0) break\n      sitemapsRead += 1\n      const sitemap = await request(next, /xml|text\\/plain/i, 'application/xml, text/xml;q=0.9', room)\n      if (!sitemap.ok) continue\n      inventory.bytesRead += sitemap.bytes\n      const entries = parseSitemap(sitemap.text)\n      for (const location of entries.locations) {\n        let url: URL\n        try {\n          url = new URL(location)\n        } catch {\n          continue\n        }\n        if (entries.index) {\n          if (sameSite(url)) sitemapQueue.push(url.toString())\n          continue\n        }\n        const key = siteImportPageKey(url, origins)\n        if (key && fromSitemap.length < SITE_IMPORT_DEFAULTS.maxSitemapUrls) fromSitemap.push(key)\n      }\n    }\n    inventory.sitemapUrls = fromSitemap.length\n    if (options.shallowFirst) {\n      const depth = (key: string) => new URL(key).pathname.split('/').filter(Boolean).length\n      const ranked = fromSitemap.map((key, order) => ({ key, order, depth: depth(key) }))\n      ranked.sort((a, b) => a.depth - b.depth || a.order - b.order)\n      fromSitemap.splice(0, fromSitemap.length, ...ranked.map((entry) => entry.key))\n    }\n\n    // Pages: home, then the sitemap's, then links the pages themselves name.\n    // A wave's read is its own pages alone.\n    const queue: { key: string; foundBy: SiteImportPage['foundBy'] }[] = only\n      ? only.map((key) => ({ key, foundBy: (new URL(key).pathname === '/' ? 'home' : 'link') as SiteImportPage['foundBy'] }))\n      : [{ key: home, foundBy: 'home' }]\n    const queued = new Set<string>(only ?? [home])\n    for (const key of fromSitemap) {\n      if (queued.has(key)) continue\n      queued.add(key)\n      queue.push({ key, foundBy: 'sitemap' })\n    }\n    const read = new Set<string>()\n    // The site's two names are one site: once the home page says which one\n    // it answers on, every address is read on that one, so /about is not\n    // read twice as example.com/about and www.example.com/about.\n    // A wave's read starts on the name its pages were read on the first time.\n    let preferred = only?.length ? new URL(only[0]).origin : origins[0]\n    const canon = (raw: string) => {\n      const url = new URL(raw)\n      return origins.includes(url.origin) && url.origin !== preferred\n        ? `${preferred}${url.pathname}${url.search}`\n        : raw\n    }\n    const maxAttempts = maxPages * 3 + 10\n    let attempts = 0\n    type Asked = { next: (typeof queue)[number]; key: string; cap: number; page: PublicFetchResult }\n    /** The pages asked for and not yet taken in, in the order they were asked. */\n    const window: Array<Promise<Asked>> = []\n    /** The bytes the asked pages may still read: what is left is shared, never promised twice. */\n    let reserved = 0\n    let stop: SiteImportStop | null = null\n    /** Asked for and never read, because a stop came first: listed as not reached. */\n    const cutOff: string[] = []\n    /** Home alone first, so its answer settles which of the site's two names the rest are read on. */\n    let homeIn = !!only\n    /** Why no more may be asked now: a stop, `wait` for an asked page to come in first, or nothing. */\n    const held = (): SiteImportStop | 'wait' | null => {\n      if (inventory.pages.length >= maxPages) return 'page-budget'\n      if (outOfTime()) return 'time'\n      if (remainingBytes() <= 0) return 'total-bytes'\n      if (attempts >= maxAttempts) return 'page-budget'\n      if (inventory.pages.length + window.length >= maxPages) return 'wait'\n      if (remainingBytes() - reserved <= 0) return 'wait'\n      return null\n    }\n    /** Takes one asked page in: the inventory's next page, a skip, or the reason the read must stop. */\n    const takeIn = ({ next, key, cap, page }: Asked): SiteImportStop | null => {\n      const foundBy = next.foundBy\n      if (foundBy === 'home') homeIn = true\n      if (!page.ok) {\n        if (page.refusal === 'too-large' && cap < maxPageBytes) {\n          cutOff.push(key)\n          return 'total-bytes'\n        }\n        if (page.refusal === 'network' && outOfTime()) {\n          cutOff.push(key)\n          return 'time'\n        }\n        skip({\n          url: key,\n          reason: skipReasonFor(page),\n          ...(page.status ? { status: page.status } : {}),\n          ...(page.location ? { location: page.location } : {}),\n          ...(page.edge ? { edge: page.edge } : {}),\n        })\n        return null\n      }\n      inventory.bytesRead += page.bytes\n      if (foundBy === 'home') preferred = new URL(page.url).origin\n      const finalKey = canon(siteImportPageKey(new URL(page.url), origins) ?? page.url)\n      if (finalKey !== key) {\n        if (read.has(finalKey)) return null\n        read.add(finalKey)\n        if (!allowed(finalKey)) {\n          skip({ url: finalKey, reason: 'robots' })\n          return null\n        }\n      }\n      const extract = extractSiteImportPage(page.text, page.url, sameSite)\n      if (extract.loginForm) {\n        skip({ url: finalKey, reason: 'login-form' })\n        return null\n      }\n      const links: string[] = []\n      for (const link of extract.links) {\n        const linkKey = siteImportPageKey(new URL(link), origins)\n        if (!linkKey) continue\n        links.push(linkKey)\n        if (!only && !queued.has(linkKey)) {\n          queued.add(linkKey)\n          queue.push({ key: linkKey, foundBy: 'link' })\n        }\n      }\n      inventory.pages.push({\n        url: finalKey,\n        path: new URL(finalKey).pathname,\n        foundBy,\n        title: extract.title,\n        description: extract.description,\n        lang: extract.lang,\n        canonical: extract.canonical,\n        ogImage: extract.ogImage,\n        siteName: extract.siteName,\n        icons: extract.icons,\n        brand: extract.brand,\n        organization: extract.organization,\n        blocks: extract.blocks,\n        images: extract.images,\n        navLinks: extract.navLinks,\n        links: [...new Set(links.map(canon))],\n        forms: extract.forms,\n        bytes: page.bytes,\n      })\n      return null\n    }\n    for (;;) {\n      const width = homeIn ? SITE_IMPORT_DEFAULTS.concurrency : 1\n      while (!stop && queue.length && window.length < width) {\n        const why = held()\n        if (why === 'wait') break\n        if (why) {\n          stop = why\n          break\n        }\n        const next = queue.shift() as (typeof queue)[number]\n        const key = canon(next.key)\n        if (read.has(key)) continue\n        if (!allowed(key)) {\n          skip({ url: key, reason: 'robots' })\n          if (next.foundBy === 'home') homeIn = true\n          continue\n        }\n        attempts += 1\n        read.add(key)\n        const cap = Math.min(maxPageBytes, remainingBytes() - reserved)\n        reserved += cap\n        window.push(\n          request(key, /text\\/html|application\\/xhtml\\+xml/i, 'text/html, application/xhtml+xml;q=0.9', cap).then((page) => ({ next, key, cap, page })),\n        )\n      }\n      if (!window.length) break\n      const asked = await (window.shift() as Promise<Asked>)\n      reserved -= asked.cap\n      stop = takeIn(asked) ?? stop\n    }\n    inventory.stoppedBy = stop\n    // Whatever a stop left unread is said, never dropped; a wave's own read\n    // leaves its unread pages for its next pass instead.\n    if (stop && !only) {\n      const unread = [...cutOff]\n      for (const next of queue) {\n        const key = canon(next.key)\n        if (!read.has(key) && !unread.includes(key)) unread.push(key)\n      }\n      for (const url of unread) skip({ url, reason: 'not-reached' })\n    }\n    return finish()\n  } finally {\n    clearTimeout(timer)\n    options.signal?.removeEventListener('abort', onAbort)\n  }\n}\n\n/**\n * The home page's `aglyn-site-verification` values, for the meta-tag proof:\n * the same fetch, user agent and same-site redirects as the crawl, but no\n * robots.txt, because the person asked for this one page to be read.\n * `answered: false` when the page could not be read at all.\n */\nexport async function readSiteVerificationMeta(\n  domain: string,\n  metaName: string,\n  seams: SiteImportCrawlSeams = {},\n): Promise<{ answered: boolean; values: string[]; url?: string }> {\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const bare = String(domain ?? '').trim().toLowerCase().replace(/^www\\./, '')\n  if (!DOMAIN_PATTERN.test(bare) || lookalikeBrandForHost(bare)) return { answered: false, values: [] }\n  const origins = siteImportOrigins(bare)\n  const extraHeaders = await siteImportHeadersFor(seams, bare)\n  const result = await fetchText(`${origins[0]}/`, {\n    maxBytes: SITE_IMPORT_DEFAULTS.maxPageBytes,\n    accept: /text\\/html|application\\/xhtml\\+xml/i,\n    acceptHeader: 'text/html, application/xhtml+xml;q=0.9',\n    userAgent: AGLYN_BOT_USER_AGENT,\n    timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,\n    redirectAllowed: (_from, to) => to.protocol === 'https:' && origins.includes(to.origin),\n    ...(extraHeaders ? { headers: extraHeaders } : {}),\n  })\n  if (!result.ok) {\n    // An answer that is not the page (a 404, a redirect off the site) is\n    // still an answer: the tag is not where it must be.\n    const answered = result.refusal === 'http-status' || result.refusal === 'redirect-refused' || result.refusal === 'content-type'\n    return { answered, values: [], url: result.url }\n  }\n  const head = result.text.split(/<body\\b/i)[0] ?? ''\n  return { answered: true, values: readMetaValues(head, metaName), url: result.url }\n}\n\n/**\n * One page as text for a prompt: its title, description and blocks in order,\n * fenced as untrusted data. The importer's prompt carries `UNTRUSTED_TEXT_RULE`.\n */\nexport function siteImportPagePromptText(page: SiteImportPage, maxChars = 12_000): string {\n  const lines: string[] = []\n  if (page.title) lines.push(`Title: ${page.title}`)\n  if (page.description) lines.push(`Description: ${page.description}`)\n  for (const block of page.blocks) {\n    if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n    else if (block.kind === 'image') lines.push(`[image: ${block.alt || 'no description'}]`)\n    else if (block.kind === 'list-item') lines.push(`- ${block.text}`)\n    else if (block.kind === 'quote') lines.push(`> ${block.text}`)\n    else if (block.kind === 'link') {\n      // A link inside a sentence is in its block's words already.\n      if (!block.inline) lines.push(block.text)\n    } else lines.push(block.text)\n  }\n  return fenceUntrustedText(`Page ${page.path} of the site being imported`, lines.join('\\n'), { maxChars })\n}\n"],"names":["process","lookalikeBrandForHost","fenceUntrustedText","fetchPublicTextDetailed","extractSiteImportPage","readMetaValues","AGLYN_BOT_TOKEN","parseRobotsTxt","parseSitemap","robotsAllows","robotsRulesFor","AGLYN_BOT_INFO_URL","env","AGLYN_BOT_USER_AGENT","SITE_IMPORT_DEFAULTS","maxPageBytes","maxTotalBytes","timeBudgetMs","requestTimeoutMs","robotsMaxBytes","sitemapMaxBytes","maxSitemaps","maxSitemapUrls","maxSkipped","concurrency","NON_PAGE_EXTENSION","TRACKING_PARAM","DOMAIN_PATTERN","siteImportHeadersFor","seams","domain","headersFor","headers","Object","keys","length","error","console","warn","siteImportOrigins","bare","replace","siteImportPageKey","url","origins","protocol","username","password","port","includes","origin","test","pathname","clean","URL","toString","hash","key","searchParams","delete","search","skipReasonFor","result","refusal","crawlSiteForImport","options","now","Date","fetchText","fetch","String","trim","toLowerCase","startedAt","inventory","refused","pages","skipped","skippedOverflow","robots","found","sitemaps","sitemapUrls","stoppedBy","bytesRead","finishedAt","finish","maxPages","Math","max","floor","deadlineAt","controller","AbortController","onAbort","abort","signal","addEventListener","aborted","timer","setTimeout","sameSite","remainingBytes","outOfTime","extraHeaders","request","accept","acceptHeader","maxBytes","userAgent","timeoutMs","min","redirectAllowed","_from","to","refuseOver","skip","entry","push","home","rules","ok","bytes","parsed","text","status","allowed","only","map","raw","filter","sitemapQueue","sitemapSeen","Set","fromSitemap","sitemapsRead","next","shift","has","add","room","sitemap","entries","location","locations","index","shallowFirst","depth","split","Boolean","ranked","order","sort","a","b","splice","queue","foundBy","queued","read","preferred","canon","maxAttempts","attempts","window","reserved","stop","cutOff","homeIn","held","takeIn","cap","page","reason","edge","finalKey","extract","loginForm","links","link","linkKey","path","title","description","lang","canonical","ogImage","siteName","icons","brand","organization","blocks","images","navLinks","forms","width","why","then","asked","unread","clearTimeout","removeEventListener","readSiteVerificationMeta","metaName","answered","values","head","siteImportPagePromptText","maxChars","lines","block","kind","repeat","level","alt","inline","join"],"mappings":";IAoGEA;AApGF;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2CAyC2C,GAE3C,SAASC,qBAAqB,QAAQ,oDAAmD;AACzF,SAASC,kBAAkB,QAAQ,0CAAyC;AAC5E,SACEC,uBAAuB,QAIlB,mCAA+B;AACtC,SACEC,qBAAqB,EACrBC,cAAc,QAQT,wBAAoB;AAC3B,SACEC,eAAe,EACfC,cAAc,EACdC,YAAY,EACZC,YAAY,EACZC,cAAc,QAET,0BAAsB;AAE7B;;;;;;;;;;CAUC,GACD,OAAO,MAAMC,sBACXX,wCAAAA,QAAQY,GAAG,CAAC,2BAA2B,YAAvCZ,wCAA2C,0CAAyC;AAEtF,2CAA2C,GAC3C,OAAO,MAAMa,uBAAuB,CAAC,eAAe,EAAEF,mBAAmB,CAAC,CAAC,CAAA;AAE3E,OAAO,MAAMG,uBAAuB;IAClCC,cAAc,IAAI,OAAO;IACzB,iFAAiF,GACjFC,eAAe,MAAM,OAAO;IAC5BC,cAAc;IACd,uDAAuD,GACvDC,kBAAkB;IAClBC,gBAAgB,MAAM;IACtBC,iBAAiB,IAAI,OAAO;IAC5B,6FAA6F,GAC7FC,aAAa;IACb,gDAAgD,GAChDC,gBAAgB;IAChB,+FAA+F,GAC/FC,YAAY;IACZ,0EAA0E,GAC1EC,aAAa;AACf,EAAU;AAEV,+DAA+D,GAC/D,MAAMC,qBACJ;AAEF,MAAMC,iBAAiB;AAEvB,MAAMC,iBAAiB;AAwHvB,0EAA0E,GAC1E,eAAeC,qBAAqBC,KAA2B,EAAEC,MAAc;IAC7E,IAAI,CAACD,MAAME,UAAU,EAAE,OAAO;IAC9B,IAAI;QACF,MAAMC,UAAU,MAAMH,MAAME,UAAU,CAACD;QACvC,OAAOE,WAAWC,OAAOC,IAAI,CAACF,SAASG,MAAM,GAAGH,UAAU;IAC5D,EAAE,OAAOI,OAAO;QACdC,QAAQC,IAAI,CAAC,0EAA0E;YAAER;YAAQM;QAAM;QACvG,OAAO;IACT;AACF;AAEA,yCAAyC,GACzC,OAAO,SAASG,kBAAkBT,MAAc;IAC9C,MAAMU,OAAOV,OAAOW,OAAO,CAAC,UAAU;IACtC,OAAO;QAAC,CAAC,QAAQ,EAAED,MAAM;QAAE,CAAC,YAAY,EAAEA,MAAM;KAAC;AACnD;AAEA,oFAAoF,GACpF,OAAO,SAASE,kBAAkBC,GAAQ,EAAEC,OAA0B;IACpE,IAAID,IAAIE,QAAQ,KAAK,YAAYF,IAAIG,QAAQ,IAAIH,IAAII,QAAQ,EAAE,OAAO;IACtE,IAAIJ,IAAIK,IAAI,IAAIL,IAAIK,IAAI,KAAK,OAAO,OAAO;IAC3C,IAAI,CAACJ,QAAQK,QAAQ,CAACN,IAAIO,MAAM,GAAG,OAAO;IAC1C,IAAIzB,mBAAmB0B,IAAI,CAACR,IAAIS,QAAQ,GAAG,OAAO;IAClD,MAAMC,QAAQ,IAAIC,IAAIX,IAAIY,QAAQ;IAClCF,MAAMG,IAAI,GAAG;IACb,KAAK,MAAMC,OAAO;WAAIJ,MAAMK,YAAY,CAACxB,IAAI;KAAG,CAAE;QAChD,IAAIR,eAAeyB,IAAI,CAACM,MAAMJ,MAAMK,YAAY,CAACC,MAAM,CAACF;IAC1D;IACAJ,MAAMO,MAAM,GAAGP,MAAMK,YAAY,CAACH,QAAQ,KAAK,CAAC,CAAC,EAAEF,MAAMK,YAAY,CAACH,QAAQ,IAAI,GAAG;IACrF,OAAOF,MAAME,QAAQ;AACvB;AAEA,SAASM,cAAcC,MAAyB;IAC9C,OAAQA,OAAOC,OAAO;QACpB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,mFAAmF,GACnF,OAAO,eAAeC,mBACpBC,OAA+B,EAC/BpC,QAA8B,CAAC,CAAC;QAEpBA,YACMA,cACIoC,iBA+BDA,uBACCA,wBACUA;QAGhCA,iBACIA;IAvCJ,MAAMC,OAAMrC,aAAAA,MAAMqC,GAAG,YAATrC,aAAasC,KAAKD,GAAG;IACjC,MAAME,aAAYvC,eAAAA,MAAMwC,KAAK,YAAXxC,eAAe1B;IACjC,MAAM2B,SAASwC,QAAOL,kBAAAA,QAAQnC,MAAM,YAAdmC,kBAAkB,IAAIM,IAAI,GAAGC,WAAW,GAAG/B,OAAO,CAAC,QAAQ,IAAIA,OAAO,CAAC,UAAU;IACvG,MAAMG,UAAUjB,eAAewB,IAAI,CAACrB,UAAUS,kBAAkBT,UAAU,EAAE;IAC5E,MAAM2C,YAAYP;IAClB,MAAMQ,YAAiC;QACrC5C;QACAc;QACA+B,SAAS;QACTC,OAAO,EAAE;QACTC,SAAS,EAAE;QACXC,iBAAiB;QACjBC,QAAQ;YAAEC,OAAO;YAAOC,UAAU,EAAE;QAAC;QACrCC,aAAa;QACbC,WAAW;QACXC,WAAW;QACXX;QACAY,YAAYZ;IACd;IACA,MAAMa,SAAS;QACbZ,UAAUW,UAAU,GAAGnB;QACvB,OAAOQ;IACT;IACA,IAAI,CAAC9B,QAAQT,MAAM,EAAE;QACnBuC,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IACA,IAAIrF,sBAAsB6B,SAAS;QACjC4C,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IAEA,MAAMC,WAAWC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAACzB,QAAQsB,QAAQ;IACxD,MAAMxE,gBAAekD,wBAAAA,QAAQlD,YAAY,YAApBkD,wBAAwBnD,qBAAqBC,YAAY;IAC9E,MAAMC,iBAAgBiD,yBAAAA,QAAQjD,aAAa,YAArBiD,yBAAyBnD,qBAAqBE,aAAa;IACjF,MAAM2E,aAAalB,cAAaR,wBAAAA,QAAQhD,YAAY,YAApBgD,wBAAwBnD,qBAAqBG,YAAY;IACzF,MAAM2E,aAAa,IAAIC;IACvB,MAAMC,UAAU,IAAMF,WAAWG,KAAK;KACtC9B,kBAAAA,QAAQ+B,MAAM,qBAAd/B,gBAAgBgC,gBAAgB,CAAC,SAASH;IAC1C,KAAI7B,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBiC,OAAO,EAAEN,WAAWG,KAAK;IAC7C,MAAMI,QAAQC,WAAW,IAAMR,WAAWG,KAAK,IAAIP,KAAKC,GAAG,CAAC,GAAGE,aAAalB;IAE5E,MAAM4B,WAAW,CAAC1D,MAAaC,QAAQK,QAAQ,CAACN,IAAIO,MAAM,KAAKP,IAAIE,QAAQ,KAAK;IAChF,MAAMyD,iBAAiB,IAAMtF,gBAAgB0D,UAAUU,SAAS;IAChE,MAAMmB,YAAY,IAAMX,WAAWI,MAAM,CAACE,OAAO,IAAIhC,SAASyB;IAC9D,wEAAwE;IACxE,MAAMa,eAAe,MAAM5E,qBAAqBC,OAAOC;IACvD,MAAM2E,UAAU,CAAC9D,KAAa+D,QAAgBC,cAAsBC,WAClExC,UAAUzB,KAAK;YACbiE;YACAF;YACAC;YACAE,WAAWhG;YACXmF,QAAQJ,WAAWI,MAAM;YACzBc,WAAWtB,KAAKC,GAAG,CAAC,GAAGD,KAAKuB,GAAG,CAACjG,qBAAqBI,gBAAgB,EAAEyE,aAAazB;YACpF8C,iBAAiB,CAACC,OAAOC,KAAOb,SAASa;YACzCC,YAAY;WACRX,eAAe;YAAExE,SAASwE;QAAa,IAAI,CAAC;IAEpD,MAAMY,OAAO,CAACC;QACZ,IAAI3C,UAAUG,OAAO,CAAC1C,MAAM,GAAGrB,qBAAqBS,UAAU,EAAEmD,UAAUG,OAAO,CAACyC,IAAI,CAACD;aAClF3C,UAAUI,eAAe,IAAI;IACpC;IAEA,IAAI;YAgBuCC,gBAA+BA;QAfxE,yDAAyD;QACzD,MAAMwC,OAAO,GAAG3E,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,IAAI4E,QAAsB,EAAE;QAC5B,MAAMzC,SAAS,MAAM0B,QACnB,GAAG7D,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAC1B,8BACA,cACA4C,KAAKuB,GAAG,CAACjG,qBAAqBK,cAAc,EAAEmF;QAEhD,IAAIvB,OAAO0C,EAAE,EAAE;YACb/C,UAAUU,SAAS,IAAIL,OAAO2C,KAAK;YACnC,MAAMC,SAASpH,eAAewE,OAAO6C,IAAI;YACzCJ,QAAQ9G,eAAeiH,QAAQrH;YAC/BoE,UAAUK,MAAM,GAAG;gBAAEC,OAAO;gBAAMC,UAAU0C,OAAO1C,QAAQ;YAAC;QAC9D,OAAO,IACL,CAAEF,CAAAA,OAAOhB,OAAO,KAAK,iBAAiB,EAACgB,iBAAAA,OAAO8C,MAAM,YAAb9C,iBAAiB,MAAM,OAAO,EAACA,kBAAAA,OAAO8C,MAAM,YAAb9C,kBAAiB,KAAK,GAAE,GAC9F;YACAL,UAAUS,SAAS,GAAGoB,cAAc,SAAS;YAC7C,OAAOjB;QACT;QACA,MAAMwC,UAAU,CAACrE;YACf,MAAMd,MAAM,IAAIW,IAAIG;YACpB,OAAOhD,aAAa+G,OAAO,GAAG7E,IAAIS,QAAQ,GAAGT,IAAIiB,MAAM,EAAE;QAC3D;QAEA,kEAAkE;QAClE,MAAMmE,OAAO9D,QAAQ8D,IAAI,GACrB9D,QAAQ8D,IAAI,CACTC,GAAG,CAAC,CAACC;YACJ,IAAI;gBACF,OAAOvF,kBAAkB,IAAIY,IAAI2E,MAAMrF;YACzC,EAAE,eAAM;gBACN,OAAO;YACT;QACF,GACCsF,MAAM,CAAC,CAACzE,MAAuB,CAAC,CAACA,OACpC;QAEJ,uEAAuE;QACvE,MAAM0E,eAAyB,EAAE;QACjC,KAAK,MAAMF,OAAOvD,UAAUK,MAAM,CAACE,QAAQ,CAAE;YAC3C,IAAI;gBACF,MAAMtC,MAAM,IAAIW,IAAI2E;gBACpB,IAAI5B,SAAS1D,MAAMwF,aAAab,IAAI,CAAC3E,IAAIY,QAAQ;YACnD,EAAE,eAAM;YACN,0CAA0C;YAC5C;QACF;QACA,IAAI,CAAC4E,aAAahG,MAAM,IAAI,CAAC4F,MAAMI,aAAab,IAAI,CAAC,GAAG1E,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC;QAChF,IAAImF,MAAMI,aAAahG,MAAM,GAAG;QAChC,MAAMiG,cAAc,IAAIC;QACxB,MAAMC,cAAwB,EAAE;QAChC,IAAIC,eAAe;QACnB,MAAOJ,aAAahG,MAAM,IAAIoG,eAAezH,qBAAqBO,WAAW,IAAI,CAACkF,YAAa;YAC7F,MAAMiC,OAAOL,aAAaM,KAAK;YAC/B,IAAIL,YAAYM,GAAG,CAACF,OAAO;YAC3BJ,YAAYO,GAAG,CAACH;YAChB,IAAI,SAASrF,IAAI,CAAC,IAAIG,IAAIkF,MAAMpF,QAAQ,GAAG;YAC3C,IAAI,CAAC0E,QAAQU,OAAO;YACpB,MAAMI,OAAOpD,KAAKuB,GAAG,CAACjG,qBAAqBM,eAAe,EAAEkF;YAC5D,IAAIsC,QAAQ,GAAG;YACfL,gBAAgB;YAChB,MAAMM,UAAU,MAAMpC,QAAQ+B,MAAM,oBAAoB,mCAAmCI;YAC3F,IAAI,CAACC,QAAQpB,EAAE,EAAE;YACjB/C,UAAUU,SAAS,IAAIyD,QAAQnB,KAAK;YACpC,MAAMoB,UAAUtI,aAAaqI,QAAQjB,IAAI;YACzC,KAAK,MAAMmB,YAAYD,QAAQE,SAAS,CAAE;gBACxC,IAAIrG;gBACJ,IAAI;oBACFA,MAAM,IAAIW,IAAIyF;gBAChB,EAAE,eAAM;oBACN;gBACF;gBACA,IAAID,QAAQG,KAAK,EAAE;oBACjB,IAAI5C,SAAS1D,MAAMwF,aAAab,IAAI,CAAC3E,IAAIY,QAAQ;oBACjD;gBACF;gBACA,MAAME,MAAMf,kBAAkBC,KAAKC;gBACnC,IAAIa,OAAO6E,YAAYnG,MAAM,GAAGrB,qBAAqBQ,cAAc,EAAEgH,YAAYhB,IAAI,CAAC7D;YACxF;QACF;QACAiB,UAAUQ,WAAW,GAAGoD,YAAYnG,MAAM;QAC1C,IAAI8B,QAAQiF,YAAY,EAAE;YACxB,MAAMC,QAAQ,CAAC1F,MAAgB,IAAIH,IAAIG,KAAKL,QAAQ,CAACgG,KAAK,CAAC,KAAKlB,MAAM,CAACmB,SAASlH,MAAM;YACtF,MAAMmH,SAAShB,YAAYN,GAAG,CAAC,CAACvE,KAAK8F,QAAW,CAAA;oBAAE9F;oBAAK8F;oBAAOJ,OAAOA,MAAM1F;gBAAK,CAAA;YAChF6F,OAAOE,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEN,KAAK,GAAGO,EAAEP,KAAK,IAAIM,EAAEF,KAAK,GAAGG,EAAEH,KAAK;YAC5DjB,YAAYqB,MAAM,CAAC,GAAGrB,YAAYnG,MAAM,KAAKmH,OAAOtB,GAAG,CAAC,CAACX,QAAUA,MAAM5D,GAAG;QAC9E;QAEA,yEAAyE;QACzE,wCAAwC;QACxC,MAAMmG,QAA+D7B,OACjEA,KAAKC,GAAG,CAAC,CAACvE,MAAS,CAAA;gBAAEA;gBAAKoG,SAAU,IAAIvG,IAAIG,KAAKL,QAAQ,KAAK,MAAM,SAAS;YAAqC,CAAA,KAClH;YAAC;gBAAEK,KAAK8D;gBAAMsC,SAAS;YAAO;SAAE;QACpC,MAAMC,SAAS,IAAIzB,IAAYN,eAAAA,OAAQ;YAACR;SAAK;QAC7C,KAAK,MAAM9D,OAAO6E,YAAa;YAC7B,IAAIwB,OAAOpB,GAAG,CAACjF,MAAM;YACrBqG,OAAOnB,GAAG,CAAClF;YACXmG,MAAMtC,IAAI,CAAC;gBAAE7D;gBAAKoG,SAAS;YAAU;QACvC;QACA,MAAME,OAAO,IAAI1B;QACjB,uEAAuE;QACvE,qEAAqE;QACrE,6DAA6D;QAC7D,0EAA0E;QAC1E,IAAI2B,YAAYjC,CAAAA,wBAAAA,KAAM5F,MAAM,IAAG,IAAImB,IAAIyE,IAAI,CAAC,EAAE,EAAE7E,MAAM,GAAGN,OAAO,CAAC,EAAE;QACnE,MAAMqH,QAAQ,CAAChC;YACb,MAAMtF,MAAM,IAAIW,IAAI2E;YACpB,OAAOrF,QAAQK,QAAQ,CAACN,IAAIO,MAAM,KAAKP,IAAIO,MAAM,KAAK8G,YAClD,GAAGA,YAAYrH,IAAIS,QAAQ,GAAGT,IAAIiB,MAAM,EAAE,GAC1CqE;QACN;QACA,MAAMiC,cAAc3E,WAAW,IAAI;QACnC,IAAI4E,WAAW;QAEf,4EAA4E,GAC5E,MAAMC,SAAgC,EAAE;QACxC,4FAA4F,GAC5F,IAAIC,WAAW;QACf,IAAIC,OAA8B;QAClC,gFAAgF,GAChF,MAAMC,SAAmB,EAAE;QAC3B,gGAAgG,GAChG,IAAIC,SAAS,CAAC,CAACzC;QACf,iGAAiG,GACjG,MAAM0C,OAAO;YACX,IAAI/F,UAAUE,KAAK,CAACzC,MAAM,IAAIoD,UAAU,OAAO;YAC/C,IAAIgB,aAAa,OAAO;YACxB,IAAID,oBAAoB,GAAG,OAAO;YAClC,IAAI6D,YAAYD,aAAa,OAAO;YACpC,IAAIxF,UAAUE,KAAK,CAACzC,MAAM,GAAGiI,OAAOjI,MAAM,IAAIoD,UAAU,OAAO;YAC/D,IAAIe,mBAAmB+D,YAAY,GAAG,OAAO;YAC7C,OAAO;QACT;QACA,kGAAkG,GAClG,MAAMK,SAAS,CAAC,EAAElC,IAAI,EAAE/E,GAAG,EAAEkH,GAAG,EAAEC,IAAI,EAAS;gBAuBtBlI;YAtBvB,MAAMmH,UAAUrB,KAAKqB,OAAO;YAC5B,IAAIA,YAAY,QAAQW,SAAS;YACjC,IAAI,CAACI,KAAKnD,EAAE,EAAE;gBACZ,IAAImD,KAAK7G,OAAO,KAAK,eAAe4G,MAAM5J,cAAc;oBACtDwJ,OAAOjD,IAAI,CAAC7D;oBACZ,OAAO;gBACT;gBACA,IAAImH,KAAK7G,OAAO,KAAK,aAAawC,aAAa;oBAC7CgE,OAAOjD,IAAI,CAAC7D;oBACZ,OAAO;gBACT;gBACA2D,KAAK;oBACHzE,KAAKc;oBACLoH,QAAQhH,cAAc+G;mBAClBA,KAAK/C,MAAM,GAAG;oBAAEA,QAAQ+C,KAAK/C,MAAM;gBAAC,IAAI,CAAC,GACzC+C,KAAK7B,QAAQ,GAAG;oBAAEA,UAAU6B,KAAK7B,QAAQ;gBAAC,IAAI,CAAC,GAC/C6B,KAAKE,IAAI,GAAG;oBAAEA,MAAMF,KAAKE,IAAI;gBAAC,IAAI,CAAC;gBAEzC,OAAO;YACT;YACApG,UAAUU,SAAS,IAAIwF,KAAKlD,KAAK;YACjC,IAAImC,YAAY,QAAQG,YAAY,IAAI1G,IAAIsH,KAAKjI,GAAG,EAAEO,MAAM;YAC5D,MAAM6H,WAAWd,OAAMvH,qBAAAA,kBAAkB,IAAIY,IAAIsH,KAAKjI,GAAG,GAAGC,oBAArCF,qBAAiDkI,KAAKjI,GAAG;YAChF,IAAIoI,aAAatH,KAAK;gBACpB,IAAIsG,KAAKrB,GAAG,CAACqC,WAAW,OAAO;gBAC/BhB,KAAKpB,GAAG,CAACoC;gBACT,IAAI,CAACjD,QAAQiD,WAAW;oBACtB3D,KAAK;wBAAEzE,KAAKoI;wBAAUF,QAAQ;oBAAS;oBACvC,OAAO;gBACT;YACF;YACA,MAAMG,UAAU5K,sBAAsBwK,KAAKhD,IAAI,EAAEgD,KAAKjI,GAAG,EAAE0D;YAC3D,IAAI2E,QAAQC,SAAS,EAAE;gBACrB7D,KAAK;oBAAEzE,KAAKoI;oBAAUF,QAAQ;gBAAa;gBAC3C,OAAO;YACT;YACA,MAAMK,QAAkB,EAAE;YAC1B,KAAK,MAAMC,QAAQH,QAAQE,KAAK,CAAE;gBAChC,MAAME,UAAU1I,kBAAkB,IAAIY,IAAI6H,OAAOvI;gBACjD,IAAI,CAACwI,SAAS;gBACdF,MAAM5D,IAAI,CAAC8D;gBACX,IAAI,CAACrD,QAAQ,CAAC+B,OAAOpB,GAAG,CAAC0C,UAAU;oBACjCtB,OAAOnB,GAAG,CAACyC;oBACXxB,MAAMtC,IAAI,CAAC;wBAAE7D,KAAK2H;wBAASvB,SAAS;oBAAO;gBAC7C;YACF;YACAnF,UAAUE,KAAK,CAAC0C,IAAI,CAAC;gBACnB3E,KAAKoI;gBACLM,MAAM,IAAI/H,IAAIyH,UAAU3H,QAAQ;gBAChCyG;gBACAyB,OAAON,QAAQM,KAAK;gBACpBC,aAAaP,QAAQO,WAAW;gBAChCC,MAAMR,QAAQQ,IAAI;gBAClBC,WAAWT,QAAQS,SAAS;gBAC5BC,SAASV,QAAQU,OAAO;gBACxBC,UAAUX,QAAQW,QAAQ;gBAC1BC,OAAOZ,QAAQY,KAAK;gBACpBC,OAAOb,QAAQa,KAAK;gBACpBC,cAAcd,QAAQc,YAAY;gBAClCC,QAAQf,QAAQe,MAAM;gBACtBC,QAAQhB,QAAQgB,MAAM;gBACtBC,UAAUjB,QAAQiB,QAAQ;gBAC1Bf,OAAO;uBAAI,IAAI7C,IAAI6C,MAAMlD,GAAG,CAACiC;iBAAQ;gBACrCiC,OAAOlB,QAAQkB,KAAK;gBACpBxE,OAAOkD,KAAKlD,KAAK;YACnB;YACA,OAAO;QACT;QACA,OAAS;gBA4BAgD;YA3BP,MAAMyB,QAAQ3B,SAAS1J,qBAAqBU,WAAW,GAAG;YAC1D,MAAO,CAAC8I,QAAQV,MAAMzH,MAAM,IAAIiI,OAAOjI,MAAM,GAAGgK,MAAO;gBACrD,MAAMC,MAAM3B;gBACZ,IAAI2B,QAAQ,QAAQ;gBACpB,IAAIA,KAAK;oBACP9B,OAAO8B;oBACP;gBACF;gBACA,MAAM5D,OAAOoB,MAAMnB,KAAK;gBACxB,MAAMhF,MAAMwG,MAAMzB,KAAK/E,GAAG;gBAC1B,IAAIsG,KAAKrB,GAAG,CAACjF,MAAM;gBACnB,IAAI,CAACqE,QAAQrE,MAAM;oBACjB2D,KAAK;wBAAEzE,KAAKc;wBAAKoH,QAAQ;oBAAS;oBAClC,IAAIrC,KAAKqB,OAAO,KAAK,QAAQW,SAAS;oBACtC;gBACF;gBACAL,YAAY;gBACZJ,KAAKpB,GAAG,CAAClF;gBACT,MAAMkH,MAAMnF,KAAKuB,GAAG,CAAChG,cAAcuF,mBAAmB+D;gBACtDA,YAAYM;gBACZP,OAAO9C,IAAI,CACTb,QAAQhD,KAAK,uCAAuC,0CAA0CkH,KAAK0B,IAAI,CAAC,CAACzB,OAAU,CAAA;wBAAEpC;wBAAM/E;wBAAKkH;wBAAKC;oBAAK,CAAA;YAE9I;YACA,IAAI,CAACR,OAAOjI,MAAM,EAAE;YACpB,MAAMmK,QAAQ,MAAOlC,OAAO3B,KAAK;YACjC4B,YAAYiC,MAAM3B,GAAG;YACrBL,QAAOI,UAAAA,OAAO4B,kBAAP5B,UAAiBJ;QAC1B;QACA5F,UAAUS,SAAS,GAAGmF;QACtB,wEAAwE;QACxE,qDAAqD;QACrD,IAAIA,QAAQ,CAACvC,MAAM;YACjB,MAAMwE,SAAS;mBAAIhC;aAAO;YAC1B,KAAK,MAAM/B,QAAQoB,MAAO;gBACxB,MAAMnG,MAAMwG,MAAMzB,KAAK/E,GAAG;gBAC1B,IAAI,CAACsG,KAAKrB,GAAG,CAACjF,QAAQ,CAAC8I,OAAOtJ,QAAQ,CAACQ,MAAM8I,OAAOjF,IAAI,CAAC7D;YAC3D;YACA,KAAK,MAAMd,OAAO4J,OAAQnF,KAAK;gBAAEzE;gBAAKkI,QAAQ;YAAc;QAC9D;QACA,OAAOvF;IACT,SAAU;YAERrB;QADAuI,aAAarG;SACblC,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBwI,mBAAmB,CAAC,SAAS3G;IAC/C;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAe4G,yBACpB5K,MAAc,EACd6K,QAAgB,EAChB9K,QAA8B,CAAC,CAAC;QAEdA,cAoBLiC;IApBb,MAAMM,aAAYvC,eAAAA,MAAMwC,KAAK,YAAXxC,eAAe1B;IACjC,MAAMqC,OAAO8B,OAAOxC,iBAAAA,SAAU,IAAIyC,IAAI,GAAGC,WAAW,GAAG/B,OAAO,CAAC,UAAU;IACzE,IAAI,CAACd,eAAewB,IAAI,CAACX,SAASvC,sBAAsBuC,OAAO,OAAO;QAAEoK,UAAU;QAAOC,QAAQ,EAAE;IAAC;IACpG,MAAMjK,UAAUL,kBAAkBC;IAClC,MAAMgE,eAAe,MAAM5E,qBAAqBC,OAAOW;IACvD,MAAMsB,SAAS,MAAMM,UAAU,GAAGxB,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;QAC/CgE,UAAU9F,qBAAqBC,YAAY;QAC3C2F,QAAQ;QACRC,cAAc;QACdE,WAAWhG;QACXiG,WAAWhG,qBAAqBI,gBAAgB;QAChD8F,iBAAiB,CAACC,OAAOC,KAAOA,GAAGrE,QAAQ,KAAK,YAAYD,QAAQK,QAAQ,CAACiE,GAAGhE,MAAM;OAClFsD,eAAe;QAAExE,SAASwE;IAAa,IAAI,CAAC;IAElD,IAAI,CAAC1C,OAAO2D,EAAE,EAAE;QACd,qEAAqE;QACrE,oDAAoD;QACpD,MAAMmF,WAAW9I,OAAOC,OAAO,KAAK,iBAAiBD,OAAOC,OAAO,KAAK,sBAAsBD,OAAOC,OAAO,KAAK;QACjH,OAAO;YAAE6I;YAAUC,QAAQ,EAAE;YAAElK,KAAKmB,OAAOnB,GAAG;QAAC;IACjD;IACA,MAAMmK,QAAOhJ,sBAAAA,OAAO8D,IAAI,CAACwB,KAAK,CAAC,WAAW,CAAC,EAAE,YAAhCtF,sBAAoC;IACjD,OAAO;QAAE8I,UAAU;QAAMC,QAAQxM,eAAeyM,MAAMH;QAAWhK,KAAKmB,OAAOnB,GAAG;IAAC;AACnF;AAEA;;;CAGC,GACD,OAAO,SAASoK,yBAAyBnC,IAAoB,EAAEoC,WAAW,KAAM;IAC9E,MAAMC,QAAkB,EAAE;IAC1B,IAAIrC,KAAKU,KAAK,EAAE2B,MAAM3F,IAAI,CAAC,CAAC,OAAO,EAAEsD,KAAKU,KAAK,EAAE;IACjD,IAAIV,KAAKW,WAAW,EAAE0B,MAAM3F,IAAI,CAAC,CAAC,aAAa,EAAEsD,KAAKW,WAAW,EAAE;IACnE,KAAK,MAAM2B,SAAStC,KAAKmB,MAAM,CAAE;QAC/B,IAAImB,MAAMC,IAAI,KAAK,WAAWF,MAAM3F,IAAI,CAAC,GAAG,IAAI8F,MAAM,CAACF,MAAMG,KAAK,EAAE,CAAC,EAAEH,MAAMtF,IAAI,EAAE;aAC9E,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,QAAQ,EAAE4F,MAAMI,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAClF,IAAIJ,MAAMC,IAAI,KAAK,aAAaF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aAC5D,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aACxD,IAAIsF,MAAMC,IAAI,KAAK,QAAQ;YAC9B,4DAA4D;YAC5D,IAAI,CAACD,MAAMK,MAAM,EAAEN,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;QAC1C,OAAOqF,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;IAC9B;IACA,OAAO1H,mBAAmB,CAAC,KAAK,EAAE0K,KAAKS,IAAI,CAAC,2BAA2B,CAAC,EAAE4B,MAAMO,IAAI,CAAC,OAAO;QAAER;IAAS;AACzG"}
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-crawl.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * THE SITE-IMPORT CRAWLER (AGL-3974, parent AGL-3973).\n *\n * Reads a website a workspace has PROVED it owns (domain-ownership.ts in\n * tenant-data-admin; the doors check the proof before calling this) into a\n * typed inventory: each page's title, description, headings and text in\n * order, images and menu links. It writes nothing; the import job decides\n * what becomes a page.\n *\n * Every request goes through `fetchPublicTextDetailed`, the theme job's\n * SSRF-guarded fetch: https on the default port only, public addresses only,\n * the socket pinned to the address checked. On top of that:\n *\n * - **One site.** The site is `https://<domain>` and `https://www.<domain>`,\n *   one site to its owner. Every redirect hop and every discovered link must\n *   stay on it; a page that redirects anywhere else is refused, not followed.\n * - **AglynBot.** Every request says `AglynBot/1.0 (+<the operator's bot page>)`,\n *   and robots.txt is read first and honored for every page: the AglynBot\n *   group, else `*`. A robots.txt that answers 404 allows everything; one\n *   that cannot be read at all (5xx, no answer) stops the crawl, because\n *   \"could not ask\" is not \"allowed\".\n * - **Our own edge.** A site this platform serves gets the edge secret\n *   (`site-import-edge.ts`, AGL-4085), which lifts our bot challenge for\n *   AglynBot; it is asked per request and per redirect hop, and a request\n *   that leaves the site's two origins never carries it.\n * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an\n *   index's children included), then same-site links found on the pages read,\n *   for whatever the sitemap missed.\n * - **Budgets.** Pages (the caller's, from the plan), bytes per page (a page\n *   over it is refused, not cut), bytes in total, and wall-clock time. A\n *   whole site's worth fits (AGL-4083): aglyn.com's 123 pages average 440 KB,\n *   54 MB in all, so the total is 160 MB; a page is parsed and dropped as it\n *   is read, so what is held is its blocks, never its bytes.\n * - **Four at a time.** robots.txt, the sitemaps and the home page are read\n *   one after another; the rest four at a time, and the inventory keeps the\n *   order they were asked in, which is the sitemap's.\n * - **Nothing vanishes.** An address a stop (the clock, the bytes, the page\n *   budget) left unread is listed as `not-reached`, so the inventory's\n *   counts add up to the site. A wave's own read leaves its unread pages for\n *   its next pass instead.\n * - **No lookalikes.** A domain that wears a brand it is not\n *   (`lookalikeBrandForHost`) is refused before any request.\n * - **No sign-in pages.** A page with a password field is refused and\n *   reported; forms with payment fields, scripts, iframes and embeds never\n *   reach the inventory (site-import-html.ts).\n *=========================================*/\n\nimport { lookalikeBrandForHost } from '@aglyn/shared-util-email/outbound-phishing-screen'\nimport { fenceUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n  fetchPublicTextDetailed,\n  type PublicFetchEdge,\n  type PublicFetchOptions,\n  type PublicFetchResult,\n} from '../jobs/ai-theme-brand-inputs'\nimport {\n  extractSiteImportPage,\n  readMetaValues,\n  type SiteImportBlock,\n  type SiteImportBrandMark,\n  type SiteImportForm,\n  type SiteImportIcon,\n  type SiteImportImage,\n  type SiteImportNavLink,\n  type SiteImportOrganization,\n} from './site-import-html'\nimport {\n  AGLYN_BOT_TOKEN,\n  parseRobotsTxt,\n  parseSitemap,\n  robotsAllows,\n  robotsRulesFor,\n  type RobotsRule,\n} from './site-import-robots'\nimport { NO_SITE_IMPORT_HEADERS, siteImportEdgeHeadersFor, type SiteImportHeadersFor } from './site-import-edge'\n\n/**\n * Where the bot's user agent points a site owner who wants to know what it is\n * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on\n * a self-hosted deployment, so its crawler never names Aglyn's address; the\n * literal is only that variable's default.\n *\n * Aglyn's is the Bring my site guide, which says what AglynBot reads and how\n * robots.txt keeps it out. There is no page of its own (Zach, 10/20: \"we don't\n * need a bot page at all\"), and the site it reads is the reader's own: an\n * import starts only once its owner has proved the site is theirs.\n */\nexport const AGLYN_BOT_INFO_URL =\n  process.env['SITE_IMPORT_BOT_INFO_URL'] ?? 'https://docs.aglyn.com/ai/bring-my-site'\n\n/** What every crawler request says it is. */\nexport const AGLYN_BOT_USER_AGENT = `AglynBot/1.0 (+${AGLYN_BOT_INFO_URL})`\n\nexport const SITE_IMPORT_DEFAULTS = {\n  maxPageBytes: 2 * 1024 * 1024,\n  /** A whole site's worth (AGL-4083): 54 MB for aglyn.com's 123 pages, with room. */\n  maxTotalBytes: 160 * 1024 * 1024,\n  timeBudgetMs: 60_000,\n  /** One request's own clock, inside the whole crawl's. */\n  requestTimeoutMs: 8_000,\n  robotsMaxBytes: 256 * 1024,\n  sitemapMaxBytes: 4 * 1024 * 1024,\n  /** Sitemap documents read, an index's children included: an index of six children is seven. */\n  maxSitemaps: 24,\n  /** URLs a sitemap may contribute to the queue. */\n  maxSitemapUrls: 2_000,\n  /** Reasons kept in `skipped` before the list stops growing: every address a sitemap may name. */\n  maxSkipped: 2_000,\n  /** Pages read at once, after robots.txt, the sitemaps and the home page. */\n  concurrency: 4,\n} as const\n\n/** Never a page: files a link may point at that are not HTML. */\nconst NON_PAGE_EXTENSION =\n  /\\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i\n\nconst TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i\n\nconst DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\\.[a-z0-9-]{1,63})+$/\n\nexport type SiteImportSkipReason =\n  /** The page has a password field: a sign-in page, refused outright. */\n  | 'login-form'\n  /** robots.txt keeps AglynBot off this path. */\n  | 'robots'\n  /** A redirect left the site. */\n  | 'off-origin-redirect'\n  /** Over the per-page byte cap. */\n  | 'too-large'\n  /** Answered with something that is not HTML. */\n  | 'not-html'\n  /** Answered 4xx or 5xx. */\n  | 'http-error'\n  /** No answer, a private address, or a refused address. */\n  | 'unreachable'\n  /** Never asked for: a stop (time, bytes, the page budget) came first (AGL-4083). */\n  | 'not-reached'\n\nexport interface SiteImportSkipped {\n  url: string\n  reason: SiteImportSkipReason\n  status?: number\n  /** Where an off-origin redirect pointed. */\n  location?: string\n  /** Whose edge turned an error answer away, when it said (AGL-4083). */\n  edge?: PublicFetchEdge\n}\n\nexport interface SiteImportPage {\n  /** The address the page was read from, after any same-site redirect. */\n  url: string\n  path: string\n  /** Where the crawler learned of the page. */\n  foundBy: 'home' | 'sitemap' | 'link'\n  title: string\n  description: string\n  lang: string | null\n  canonical: string | null\n  /** The share card (AGL-4083). */\n  ogImage: string | null\n  /** The site's own name, as its head gives it. */\n  siteName: string\n  icons: SiteImportIcon[]\n  /** The header's link home: the brand's logo and words. */\n  brand: SiteImportBrandMark | null\n  organization: SiteImportOrganization | null\n  blocks: SiteImportBlock[]\n  images: SiteImportImage[]\n  navLinks: SiteImportNavLink[]\n  /** Same-site page addresses this page links to. */\n  links: string[]\n  forms: { kept: SiteImportForm[]; removed: number }\n  bytes: number\n}\n\nexport type SiteImportStop =\n  | 'page-budget'\n  | 'total-bytes'\n  | 'time'\n  /** robots.txt could not be read, so nothing was asked of the site. */\n  | 'robots-unreachable'\n\nexport interface SiteImportInventory {\n  domain: string\n  /** The site's origins: the bare name and its `www.` twin. */\n  origins: string[]\n  /** Set when nothing was fetched at all. */\n  refused: 'invalid-domain' | 'lookalike' | null\n  pages: SiteImportPage[]\n  skipped: SiteImportSkipped[]\n  /** Skips past {@link SITE_IMPORT_DEFAULTS.maxSkipped}, counted only. */\n  skippedOverflow: number\n  robots: { found: boolean; sitemaps: string[] }\n  /** Page addresses the sitemaps listed (same-site, before robots and budget). */\n  sitemapUrls: number\n  stoppedBy: SiteImportStop | null\n  bytesRead: number\n  startedAt: number\n  finishedAt: number\n}\n\nexport interface SiteImportCrawlOptions {\n  /** The proved domain; `www.` is folded in. */\n  domain: string\n  /** The most pages the inventory may hold: the plan's page count. */\n  maxPages: number\n  maxPageBytes?: number\n  maxTotalBytes?: number\n  timeBudgetMs?: number\n  signal?: AbortSignal\n  /**\n   * Read the sitemap's shallow addresses before its deep ones (AGL-3976), so a\n   * crawl the page budget stops reads a site's main pages before the tail of\n   * its blog. A sitemap's own order is kept among addresses of one depth.\n   */\n  shallowFirst?: boolean\n  /**\n   * Read exactly these addresses of the site, in order, and nothing else\n   * (AGL-3976): an import's wave reads its own pages again when it starts. No\n   * sitemap is read and no link is followed; robots.txt is read and honored\n   * as for any crawl, and every other rule holds.\n   */\n  only?: readonly string[]\n}\n\nexport interface SiteImportCrawlSeams {\n  fetch?: (url: string, options: PublicFetchOptions) => Promise<PublicFetchResult>\n  now?: () => number\n  /**\n   * Request headers for the site being read, asked once per crawl with its\n   * domain (AGL-4083, AGL-4085). Absent, the crawl asks the edge\n   * (`siteImportEdgeHeadersFor`): Aglyn's edge secret for each of the site's\n   * two origins this platform serves, decided origin by origin, and nothing\n   * anywhere else. Either way the headers go only to the site's two origins,\n   * and the fetch asks again on every redirect hop, so a hop anywhere else\n   * carries nothing. Always under the crawler's own `Accept` and `User-Agent`.\n   */\n  headersFor?: (domain: string) => Promise<Readonly<Record<string, string>>> | Readonly<Record<string, string>>\n}\n\n/**\n * The per-request headers of a crawl of `domain`: the seam's, else the edge's,\n * and none for any address off the site's two origins. Never throws.\n */\nasync function siteRequestHeaders(\n  seams: SiteImportCrawlSeams,\n  domain: string,\n  origins: readonly string[],\n): Promise<SiteImportHeadersFor> {\n  let inner: SiteImportHeadersFor = NO_SITE_IMPORT_HEADERS\n  try {\n    if (seams.headersFor) {\n      const headers = await seams.headersFor(domain)\n      if (headers && Object.keys(headers).length) inner = () => headers\n    } else {\n      inner = await siteImportEdgeHeadersFor(origins)\n    }\n  } catch (error) {\n    console.warn('[site-import] the edge headers could not be read; reading without them', { domain, error })\n  }\n  return (url: URL) => (origins.includes(url.origin) ? inner(url) : {})\n}\n\n/** The site's two origins for a domain. */\nexport function siteImportOrigins(domain: string): string[] {\n  const bare = domain.replace(/^www\\./, '')\n  return [`https://${bare}`, `https://www.${bare}`]\n}\n\n/** A crawlable page address on the site, normalized for de-duplication; else null. */\nexport function siteImportPageKey(url: URL, origins: readonly string[]): string | null {\n  if (url.protocol !== 'https:' || url.username || url.password) return null\n  if (url.port && url.port !== '443') return null\n  if (!origins.includes(url.origin)) return null\n  if (NON_PAGE_EXTENSION.test(url.pathname)) return null\n  const clean = new URL(url.toString())\n  clean.hash = ''\n  for (const key of [...clean.searchParams.keys()]) {\n    if (TRACKING_PARAM.test(key)) clean.searchParams.delete(key)\n  }\n  clean.search = clean.searchParams.toString() ? `?${clean.searchParams.toString()}` : ''\n  return clean.toString()\n}\n\nfunction skipReasonFor(result: PublicFetchResult): SiteImportSkipReason {\n  switch (result.refusal) {\n    case 'redirect-refused':\n      return 'off-origin-redirect'\n    case 'content-type':\n      return 'not-html'\n    case 'too-large':\n      return 'too-large'\n    case 'http-status':\n      return 'http-error'\n    default:\n      return 'unreachable'\n  }\n}\n\n/** Read a proved site into an inventory. Never throws for anything the site does. */\nexport async function crawlSiteForImport(\n  options: SiteImportCrawlOptions,\n  seams: SiteImportCrawlSeams = {},\n): Promise<SiteImportInventory> {\n  const now = seams.now ?? Date.now\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const domain = String(options.domain ?? '').trim().toLowerCase().replace(/\\.+$/, '').replace(/^www\\./, '')\n  const origins = DOMAIN_PATTERN.test(domain) ? siteImportOrigins(domain) : []\n  const startedAt = now()\n  const inventory: SiteImportInventory = {\n    domain,\n    origins,\n    refused: null,\n    pages: [],\n    skipped: [],\n    skippedOverflow: 0,\n    robots: { found: false, sitemaps: [] },\n    sitemapUrls: 0,\n    stoppedBy: null,\n    bytesRead: 0,\n    startedAt,\n    finishedAt: startedAt,\n  }\n  const finish = () => {\n    inventory.finishedAt = now()\n    return inventory\n  }\n  if (!origins.length) {\n    inventory.refused = 'invalid-domain'\n    return finish()\n  }\n  if (lookalikeBrandForHost(domain)) {\n    inventory.refused = 'lookalike'\n    return finish()\n  }\n\n  // Asked once, before anything else, and held to the site's two origins.\n  const headersFor = await siteRequestHeaders(seams, domain, origins)\n  const maxPages = Math.max(0, Math.floor(options.maxPages))\n  const maxPageBytes = options.maxPageBytes ?? SITE_IMPORT_DEFAULTS.maxPageBytes\n  const maxTotalBytes = options.maxTotalBytes ?? SITE_IMPORT_DEFAULTS.maxTotalBytes\n  const deadlineAt = startedAt + (options.timeBudgetMs ?? SITE_IMPORT_DEFAULTS.timeBudgetMs)\n  const controller = new AbortController()\n  const onAbort = () => controller.abort()\n  options.signal?.addEventListener('abort', onAbort)\n  if (options.signal?.aborted) controller.abort()\n  const timer = setTimeout(() => controller.abort(), Math.max(0, deadlineAt - startedAt))\n\n  const sameSite = (url: URL) => origins.includes(url.origin) && url.protocol === 'https:'\n  const remainingBytes = () => maxTotalBytes - inventory.bytesRead\n  const outOfTime = () => controller.signal.aborted || now() >= deadlineAt\n  const request = (url: string, accept: RegExp, acceptHeader: string, maxBytes: number) =>\n    fetchText(url, {\n      maxBytes,\n      accept,\n      acceptHeader,\n      userAgent: AGLYN_BOT_USER_AGENT,\n      headersFor,\n      signal: controller.signal,\n      timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),\n      redirectAllowed: (_from, to) => sameSite(to),\n      refuseOver: true,\n    })\n  const skip = (entry: SiteImportSkipped) => {\n    if (inventory.skipped.length < SITE_IMPORT_DEFAULTS.maxSkipped) inventory.skipped.push(entry)\n    else inventory.skippedOverflow += 1\n  }\n\n  try {\n    // robots.txt, before anything else is asked of the site.\n    const home = `${origins[0]}/`\n    let rules: RobotsRule[] = []\n    const robots = await request(\n      `${origins[0]}/robots.txt`,\n      /text\\/plain|text\\/html|^$/i,\n      'text/plain',\n      Math.min(SITE_IMPORT_DEFAULTS.robotsMaxBytes, remainingBytes()),\n    )\n    if (robots.ok) {\n      inventory.bytesRead += robots.bytes\n      const parsed = parseRobotsTxt(robots.text)\n      rules = robotsRulesFor(parsed, AGLYN_BOT_TOKEN)\n      inventory.robots = { found: true, sitemaps: parsed.sitemaps }\n    } else if (\n      !(robots.refusal === 'http-status' && (robots.status ?? 0) >= 400 && (robots.status ?? 0) < 500)\n    ) {\n      inventory.stoppedBy = outOfTime() ? 'time' : 'robots-unreachable'\n      return finish()\n    }\n    const allowed = (key: string) => {\n      const url = new URL(key)\n      return robotsAllows(rules, `${url.pathname}${url.search}`)\n    }\n\n    // A wave's own pages (AGL-3976): exactly those, after robots.txt.\n    const only = options.only\n      ? options.only\n          .map((raw) => {\n            try {\n              return siteImportPageKey(new URL(raw), origins)\n            } catch {\n              return null\n            }\n          })\n          .filter((key): key is string => !!key)\n      : null\n\n    // Sitemaps: the ones robots.txt names on this site, else /sitemap.xml.\n    const sitemapQueue: string[] = []\n    for (const raw of inventory.robots.sitemaps) {\n      try {\n        const url = new URL(raw)\n        if (sameSite(url)) sitemapQueue.push(url.toString())\n      } catch {\n        // A malformed Sitemap line names nothing.\n      }\n    }\n    if (!sitemapQueue.length && !only) sitemapQueue.push(`${origins[0]}/sitemap.xml`)\n    if (only) sitemapQueue.length = 0\n    const sitemapSeen = new Set<string>()\n    const fromSitemap: string[] = []\n    let sitemapsRead = 0\n    while (sitemapQueue.length && sitemapsRead < SITE_IMPORT_DEFAULTS.maxSitemaps && !outOfTime()) {\n      const next = sitemapQueue.shift() as string\n      if (sitemapSeen.has(next)) continue\n      sitemapSeen.add(next)\n      if (/\\.gz$/i.test(new URL(next).pathname)) continue\n      if (!allowed(next)) continue\n      const room = Math.min(SITE_IMPORT_DEFAULTS.sitemapMaxBytes, remainingBytes())\n      if (room <= 0) break\n      sitemapsRead += 1\n      const sitemap = await request(next, /xml|text\\/plain/i, 'application/xml, text/xml;q=0.9', room)\n      if (!sitemap.ok) continue\n      inventory.bytesRead += sitemap.bytes\n      const entries = parseSitemap(sitemap.text)\n      for (const location of entries.locations) {\n        let url: URL\n        try {\n          url = new URL(location)\n        } catch {\n          continue\n        }\n        if (entries.index) {\n          if (sameSite(url)) sitemapQueue.push(url.toString())\n          continue\n        }\n        const key = siteImportPageKey(url, origins)\n        if (key && fromSitemap.length < SITE_IMPORT_DEFAULTS.maxSitemapUrls) fromSitemap.push(key)\n      }\n    }\n    inventory.sitemapUrls = fromSitemap.length\n    if (options.shallowFirst) {\n      const depth = (key: string) => new URL(key).pathname.split('/').filter(Boolean).length\n      const ranked = fromSitemap.map((key, order) => ({ key, order, depth: depth(key) }))\n      ranked.sort((a, b) => a.depth - b.depth || a.order - b.order)\n      fromSitemap.splice(0, fromSitemap.length, ...ranked.map((entry) => entry.key))\n    }\n\n    // Pages: home, then the sitemap's, then links the pages themselves name.\n    // A wave's read is its own pages alone.\n    const queue: { key: string; foundBy: SiteImportPage['foundBy'] }[] = only\n      ? only.map((key) => ({ key, foundBy: (new URL(key).pathname === '/' ? 'home' : 'link') as SiteImportPage['foundBy'] }))\n      : [{ key: home, foundBy: 'home' }]\n    const queued = new Set<string>(only ?? [home])\n    for (const key of fromSitemap) {\n      if (queued.has(key)) continue\n      queued.add(key)\n      queue.push({ key, foundBy: 'sitemap' })\n    }\n    const read = new Set<string>()\n    // The site's two names are one site: once the home page says which one\n    // it answers on, every address is read on that one, so /about is not\n    // read twice as example.com/about and www.example.com/about.\n    // A wave's read starts on the name its pages were read on the first time.\n    let preferred = only?.length ? new URL(only[0]).origin : origins[0]\n    const canon = (raw: string) => {\n      const url = new URL(raw)\n      return origins.includes(url.origin) && url.origin !== preferred\n        ? `${preferred}${url.pathname}${url.search}`\n        : raw\n    }\n    const maxAttempts = maxPages * 3 + 10\n    let attempts = 0\n    type Asked = { next: (typeof queue)[number]; key: string; cap: number; page: PublicFetchResult }\n    /** The pages asked for and not yet taken in, in the order they were asked. */\n    const window: Array<Promise<Asked>> = []\n    /** The bytes the asked pages may still read: what is left is shared, never promised twice. */\n    let reserved = 0\n    let stop: SiteImportStop | null = null\n    /** Asked for and never read, because a stop came first: listed as not reached. */\n    const cutOff: string[] = []\n    /** Home alone first, so its answer settles which of the site's two names the rest are read on. */\n    let homeIn = !!only\n    /** Why no more may be asked now: a stop, `wait` for an asked page to come in first, or nothing. */\n    const held = (): SiteImportStop | 'wait' | null => {\n      if (inventory.pages.length >= maxPages) return 'page-budget'\n      if (outOfTime()) return 'time'\n      if (remainingBytes() <= 0) return 'total-bytes'\n      if (attempts >= maxAttempts) return 'page-budget'\n      if (inventory.pages.length + window.length >= maxPages) return 'wait'\n      if (remainingBytes() - reserved <= 0) return 'wait'\n      return null\n    }\n    /** Takes one asked page in: the inventory's next page, a skip, or the reason the read must stop. */\n    const takeIn = ({ next, key, cap, page }: Asked): SiteImportStop | null => {\n      const foundBy = next.foundBy\n      if (foundBy === 'home') homeIn = true\n      if (!page.ok) {\n        if (page.refusal === 'too-large' && cap < maxPageBytes) {\n          cutOff.push(key)\n          return 'total-bytes'\n        }\n        if (page.refusal === 'network' && outOfTime()) {\n          cutOff.push(key)\n          return 'time'\n        }\n        skip({\n          url: key,\n          reason: skipReasonFor(page),\n          ...(page.status ? { status: page.status } : {}),\n          ...(page.location ? { location: page.location } : {}),\n          ...(page.edge ? { edge: page.edge } : {}),\n        })\n        return null\n      }\n      inventory.bytesRead += page.bytes\n      if (foundBy === 'home') preferred = new URL(page.url).origin\n      const finalKey = canon(siteImportPageKey(new URL(page.url), origins) ?? page.url)\n      if (finalKey !== key) {\n        if (read.has(finalKey)) return null\n        read.add(finalKey)\n        if (!allowed(finalKey)) {\n          skip({ url: finalKey, reason: 'robots' })\n          return null\n        }\n      }\n      const extract = extractSiteImportPage(page.text, page.url, sameSite)\n      if (extract.loginForm) {\n        skip({ url: finalKey, reason: 'login-form' })\n        return null\n      }\n      const links: string[] = []\n      for (const link of extract.links) {\n        const linkKey = siteImportPageKey(new URL(link), origins)\n        if (!linkKey) continue\n        links.push(linkKey)\n        if (!only && !queued.has(linkKey)) {\n          queued.add(linkKey)\n          queue.push({ key: linkKey, foundBy: 'link' })\n        }\n      }\n      inventory.pages.push({\n        url: finalKey,\n        path: new URL(finalKey).pathname,\n        foundBy,\n        title: extract.title,\n        description: extract.description,\n        lang: extract.lang,\n        canonical: extract.canonical,\n        ogImage: extract.ogImage,\n        siteName: extract.siteName,\n        icons: extract.icons,\n        brand: extract.brand,\n        organization: extract.organization,\n        blocks: extract.blocks,\n        images: extract.images,\n        navLinks: extract.navLinks,\n        links: [...new Set(links.map(canon))],\n        forms: extract.forms,\n        bytes: page.bytes,\n      })\n      return null\n    }\n    for (;;) {\n      const width = homeIn ? SITE_IMPORT_DEFAULTS.concurrency : 1\n      while (!stop && queue.length && window.length < width) {\n        const why = held()\n        if (why === 'wait') break\n        if (why) {\n          stop = why\n          break\n        }\n        const next = queue.shift() as (typeof queue)[number]\n        const key = canon(next.key)\n        if (read.has(key)) continue\n        if (!allowed(key)) {\n          skip({ url: key, reason: 'robots' })\n          if (next.foundBy === 'home') homeIn = true\n          continue\n        }\n        attempts += 1\n        read.add(key)\n        const cap = Math.min(maxPageBytes, remainingBytes() - reserved)\n        reserved += cap\n        window.push(\n          request(key, /text\\/html|application\\/xhtml\\+xml/i, 'text/html, application/xhtml+xml;q=0.9', cap).then((page) => ({ next, key, cap, page })),\n        )\n      }\n      if (!window.length) break\n      const asked = await (window.shift() as Promise<Asked>)\n      reserved -= asked.cap\n      stop = takeIn(asked) ?? stop\n    }\n    inventory.stoppedBy = stop\n    // Whatever a stop left unread is said, never dropped; a wave's own read\n    // leaves its unread pages for its next pass instead.\n    if (stop && !only) {\n      const unread = [...cutOff]\n      for (const next of queue) {\n        const key = canon(next.key)\n        if (!read.has(key) && !unread.includes(key)) unread.push(key)\n      }\n      for (const url of unread) skip({ url, reason: 'not-reached' })\n    }\n    return finish()\n  } finally {\n    clearTimeout(timer)\n    options.signal?.removeEventListener('abort', onAbort)\n  }\n}\n\n/**\n * The home page's `aglyn-site-verification` values, for the meta-tag proof:\n * the same fetch, user agent and same-site redirects as the crawl, but no\n * robots.txt, because the person asked for this one page to be read.\n * `answered: false` when the page could not be read at all.\n */\nexport async function readSiteVerificationMeta(\n  domain: string,\n  metaName: string,\n  seams: SiteImportCrawlSeams = {},\n): Promise<{ answered: boolean; values: string[]; url?: string }> {\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const bare = String(domain ?? '').trim().toLowerCase().replace(/^www\\./, '')\n  if (!DOMAIN_PATTERN.test(bare) || lookalikeBrandForHost(bare)) return { answered: false, values: [] }\n  const origins = siteImportOrigins(bare)\n  const result = await fetchText(`${origins[0]}/`, {\n    maxBytes: SITE_IMPORT_DEFAULTS.maxPageBytes,\n    accept: /text\\/html|application\\/xhtml\\+xml/i,\n    acceptHeader: 'text/html, application/xhtml+xml;q=0.9',\n    userAgent: AGLYN_BOT_USER_AGENT,\n    headersFor: await siteRequestHeaders(seams, bare, origins),\n    timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,\n    redirectAllowed: (_from, to) => to.protocol === 'https:' && origins.includes(to.origin),\n  })\n  if (!result.ok) {\n    // An answer that is not the page (a 404, a redirect off the site) is\n    // still an answer: the tag is not where it must be.\n    const answered = result.refusal === 'http-status' || result.refusal === 'redirect-refused' || result.refusal === 'content-type'\n    return { answered, values: [], url: result.url }\n  }\n  const head = result.text.split(/<body\\b/i)[0] ?? ''\n  return { answered: true, values: readMetaValues(head, metaName), url: result.url }\n}\n\n/**\n * One page as text for a prompt: its title, description and blocks in order,\n * fenced as untrusted data. The importer's prompt carries `UNTRUSTED_TEXT_RULE`.\n */\nexport function siteImportPagePromptText(page: SiteImportPage, maxChars = 12_000): string {\n  const lines: string[] = []\n  if (page.title) lines.push(`Title: ${page.title}`)\n  if (page.description) lines.push(`Description: ${page.description}`)\n  for (const block of page.blocks) {\n    if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n    else if (block.kind === 'image') lines.push(`[image: ${block.alt || 'no description'}]`)\n    else if (block.kind === 'list-item') lines.push(`- ${block.text}`)\n    else if (block.kind === 'quote') lines.push(`> ${block.text}`)\n    else if (block.kind === 'link') {\n      // A link inside a sentence is in its block's words already.\n      if (!block.inline) lines.push(block.text)\n    } else lines.push(block.text)\n  }\n  return fenceUntrustedText(`Page ${page.path} of the site being imported`, lines.join('\\n'), { maxChars })\n}\n"],"names":["process","lookalikeBrandForHost","fenceUntrustedText","fetchPublicTextDetailed","extractSiteImportPage","readMetaValues","AGLYN_BOT_TOKEN","parseRobotsTxt","parseSitemap","robotsAllows","robotsRulesFor","NO_SITE_IMPORT_HEADERS","siteImportEdgeHeadersFor","AGLYN_BOT_INFO_URL","env","AGLYN_BOT_USER_AGENT","SITE_IMPORT_DEFAULTS","maxPageBytes","maxTotalBytes","timeBudgetMs","requestTimeoutMs","robotsMaxBytes","sitemapMaxBytes","maxSitemaps","maxSitemapUrls","maxSkipped","concurrency","NON_PAGE_EXTENSION","TRACKING_PARAM","DOMAIN_PATTERN","siteRequestHeaders","seams","domain","origins","inner","headersFor","headers","Object","keys","length","error","console","warn","url","includes","origin","siteImportOrigins","bare","replace","siteImportPageKey","protocol","username","password","port","test","pathname","clean","URL","toString","hash","key","searchParams","delete","search","skipReasonFor","result","refusal","crawlSiteForImport","options","now","Date","fetchText","fetch","String","trim","toLowerCase","startedAt","inventory","refused","pages","skipped","skippedOverflow","robots","found","sitemaps","sitemapUrls","stoppedBy","bytesRead","finishedAt","finish","maxPages","Math","max","floor","deadlineAt","controller","AbortController","onAbort","abort","signal","addEventListener","aborted","timer","setTimeout","sameSite","remainingBytes","outOfTime","request","accept","acceptHeader","maxBytes","userAgent","timeoutMs","min","redirectAllowed","_from","to","refuseOver","skip","entry","push","home","rules","ok","bytes","parsed","text","status","allowed","only","map","raw","filter","sitemapQueue","sitemapSeen","Set","fromSitemap","sitemapsRead","next","shift","has","add","room","sitemap","entries","location","locations","index","shallowFirst","depth","split","Boolean","ranked","order","sort","a","b","splice","queue","foundBy","queued","read","preferred","canon","maxAttempts","attempts","window","reserved","stop","cutOff","homeIn","held","takeIn","cap","page","reason","edge","finalKey","extract","loginForm","links","link","linkKey","path","title","description","lang","canonical","ogImage","siteName","icons","brand","organization","blocks","images","navLinks","forms","width","why","then","asked","unread","clearTimeout","removeEventListener","readSiteVerificationMeta","metaName","answered","values","head","siteImportPagePromptText","maxChars","lines","block","kind","repeat","level","alt","inline","join"],"mappings":";IAyGEA;AAzGF;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2CA6C2C,GAE3C,SAASC,qBAAqB,QAAQ,oDAAmD;AACzF,SAASC,kBAAkB,QAAQ,0CAAyC;AAC5E,SACEC,uBAAuB,QAIlB,mCAA+B;AACtC,SACEC,qBAAqB,EACrBC,cAAc,QAQT,wBAAoB;AAC3B,SACEC,eAAe,EACfC,cAAc,EACdC,YAAY,EACZC,YAAY,EACZC,cAAc,QAET,0BAAsB;AAC7B,SAASC,sBAAsB,EAAEC,wBAAwB,QAAmC,wBAAoB;AAEhH;;;;;;;;;;CAUC,GACD,OAAO,MAAMC,sBACXb,wCAAAA,QAAQc,GAAG,CAAC,2BAA2B,YAAvCd,wCAA2C,0CAAyC;AAEtF,2CAA2C,GAC3C,OAAO,MAAMe,uBAAuB,CAAC,eAAe,EAAEF,mBAAmB,CAAC,CAAC,CAAA;AAE3E,OAAO,MAAMG,uBAAuB;IAClCC,cAAc,IAAI,OAAO;IACzB,iFAAiF,GACjFC,eAAe,MAAM,OAAO;IAC5BC,cAAc;IACd,uDAAuD,GACvDC,kBAAkB;IAClBC,gBAAgB,MAAM;IACtBC,iBAAiB,IAAI,OAAO;IAC5B,6FAA6F,GAC7FC,aAAa;IACb,gDAAgD,GAChDC,gBAAgB;IAChB,+FAA+F,GAC/FC,YAAY;IACZ,0EAA0E,GAC1EC,aAAa;AACf,EAAU;AAEV,+DAA+D,GAC/D,MAAMC,qBACJ;AAEF,MAAMC,iBAAiB;AAEvB,MAAMC,iBAAiB;AA0HvB;;;CAGC,GACD,eAAeC,mBACbC,KAA2B,EAC3BC,MAAc,EACdC,OAA0B;IAE1B,IAAIC,QAA8BvB;IAClC,IAAI;QACF,IAAIoB,MAAMI,UAAU,EAAE;YACpB,MAAMC,UAAU,MAAML,MAAMI,UAAU,CAACH;YACvC,IAAII,WAAWC,OAAOC,IAAI,CAACF,SAASG,MAAM,EAAEL,QAAQ,IAAME;QAC5D,OAAO;YACLF,QAAQ,MAAMtB,yBAAyBqB;QACzC;IACF,EAAE,OAAOO,OAAO;QACdC,QAAQC,IAAI,CAAC,0EAA0E;YAAEV;YAAQQ;QAAM;IACzG;IACA,OAAO,CAACG,MAAcV,QAAQW,QAAQ,CAACD,IAAIE,MAAM,IAAIX,MAAMS,OAAO,CAAC;AACrE;AAEA,yCAAyC,GACzC,OAAO,SAASG,kBAAkBd,MAAc;IAC9C,MAAMe,OAAOf,OAAOgB,OAAO,CAAC,UAAU;IACtC,OAAO;QAAC,CAAC,QAAQ,EAAED,MAAM;QAAE,CAAC,YAAY,EAAEA,MAAM;KAAC;AACnD;AAEA,oFAAoF,GACpF,OAAO,SAASE,kBAAkBN,GAAQ,EAAEV,OAA0B;IACpE,IAAIU,IAAIO,QAAQ,KAAK,YAAYP,IAAIQ,QAAQ,IAAIR,IAAIS,QAAQ,EAAE,OAAO;IACtE,IAAIT,IAAIU,IAAI,IAAIV,IAAIU,IAAI,KAAK,OAAO,OAAO;IAC3C,IAAI,CAACpB,QAAQW,QAAQ,CAACD,IAAIE,MAAM,GAAG,OAAO;IAC1C,IAAIlB,mBAAmB2B,IAAI,CAACX,IAAIY,QAAQ,GAAG,OAAO;IAClD,MAAMC,QAAQ,IAAIC,IAAId,IAAIe,QAAQ;IAClCF,MAAMG,IAAI,GAAG;IACb,KAAK,MAAMC,OAAO;WAAIJ,MAAMK,YAAY,CAACvB,IAAI;KAAG,CAAE;QAChD,IAAIV,eAAe0B,IAAI,CAACM,MAAMJ,MAAMK,YAAY,CAACC,MAAM,CAACF;IAC1D;IACAJ,MAAMO,MAAM,GAAGP,MAAMK,YAAY,CAACH,QAAQ,KAAK,CAAC,CAAC,EAAEF,MAAMK,YAAY,CAACH,QAAQ,IAAI,GAAG;IACrF,OAAOF,MAAME,QAAQ;AACvB;AAEA,SAASM,cAAcC,MAAyB;IAC9C,OAAQA,OAAOC,OAAO;QACpB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,mFAAmF,GACnF,OAAO,eAAeC,mBACpBC,OAA+B,EAC/BrC,QAA8B,CAAC,CAAC;QAEpBA,YACMA,cACIqC,iBAiCDA,uBACCA,wBACUA;QAGhCA,iBACIA;IAzCJ,MAAMC,OAAMtC,aAAAA,MAAMsC,GAAG,YAATtC,aAAauC,KAAKD,GAAG;IACjC,MAAME,aAAYxC,eAAAA,MAAMyC,KAAK,YAAXzC,eAAe5B;IACjC,MAAM6B,SAASyC,QAAOL,kBAAAA,QAAQpC,MAAM,YAAdoC,kBAAkB,IAAIM,IAAI,GAAGC,WAAW,GAAG3B,OAAO,CAAC,QAAQ,IAAIA,OAAO,CAAC,UAAU;IACvG,MAAMf,UAAUJ,eAAeyB,IAAI,CAACtB,UAAUc,kBAAkBd,UAAU,EAAE;IAC5E,MAAM4C,YAAYP;IAClB,MAAMQ,YAAiC;QACrC7C;QACAC;QACA6C,SAAS;QACTC,OAAO,EAAE;QACTC,SAAS,EAAE;QACXC,iBAAiB;QACjBC,QAAQ;YAAEC,OAAO;YAAOC,UAAU,EAAE;QAAC;QACrCC,aAAa;QACbC,WAAW;QACXC,WAAW;QACXX;QACAY,YAAYZ;IACd;IACA,MAAMa,SAAS;QACbZ,UAAUW,UAAU,GAAGnB;QACvB,OAAOQ;IACT;IACA,IAAI,CAAC5C,QAAQM,MAAM,EAAE;QACnBsC,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IACA,IAAIxF,sBAAsB+B,SAAS;QACjC6C,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IAEA,wEAAwE;IACxE,MAAMtD,aAAa,MAAML,mBAAmBC,OAAOC,QAAQC;IAC3D,MAAMyD,WAAWC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAACzB,QAAQsB,QAAQ;IACxD,MAAMzE,gBAAemD,wBAAAA,QAAQnD,YAAY,YAApBmD,wBAAwBpD,qBAAqBC,YAAY;IAC9E,MAAMC,iBAAgBkD,yBAAAA,QAAQlD,aAAa,YAArBkD,yBAAyBpD,qBAAqBE,aAAa;IACjF,MAAM4E,aAAalB,cAAaR,wBAAAA,QAAQjD,YAAY,YAApBiD,wBAAwBpD,qBAAqBG,YAAY;IACzF,MAAM4E,aAAa,IAAIC;IACvB,MAAMC,UAAU,IAAMF,WAAWG,KAAK;KACtC9B,kBAAAA,QAAQ+B,MAAM,qBAAd/B,gBAAgBgC,gBAAgB,CAAC,SAASH;IAC1C,KAAI7B,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBiC,OAAO,EAAEN,WAAWG,KAAK;IAC7C,MAAMI,QAAQC,WAAW,IAAMR,WAAWG,KAAK,IAAIP,KAAKC,GAAG,CAAC,GAAGE,aAAalB;IAE5E,MAAM4B,WAAW,CAAC7D,MAAaV,QAAQW,QAAQ,CAACD,IAAIE,MAAM,KAAKF,IAAIO,QAAQ,KAAK;IAChF,MAAMuD,iBAAiB,IAAMvF,gBAAgB2D,UAAUU,SAAS;IAChE,MAAMmB,YAAY,IAAMX,WAAWI,MAAM,CAACE,OAAO,IAAIhC,SAASyB;IAC9D,MAAMa,UAAU,CAAChE,KAAaiE,QAAgBC,cAAsBC,WAClEvC,UAAU5B,KAAK;YACbmE;YACAF;YACAC;YACAE,WAAWhG;YACXoB;YACAgE,QAAQJ,WAAWI,MAAM;YACzBa,WAAWrB,KAAKC,GAAG,CAAC,GAAGD,KAAKsB,GAAG,CAACjG,qBAAqBI,gBAAgB,EAAE0E,aAAazB;YACpF6C,iBAAiB,CAACC,OAAOC,KAAOZ,SAASY;YACzCC,YAAY;QACd;IACF,MAAMC,OAAO,CAACC;QACZ,IAAI1C,UAAUG,OAAO,CAACzC,MAAM,GAAGvB,qBAAqBS,UAAU,EAAEoD,UAAUG,OAAO,CAACwC,IAAI,CAACD;aAClF1C,UAAUI,eAAe,IAAI;IACpC;IAEA,IAAI;YAgBuCC,gBAA+BA;QAfxE,yDAAyD;QACzD,MAAMuC,OAAO,GAAGxF,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,IAAIyF,QAAsB,EAAE;QAC5B,MAAMxC,SAAS,MAAMyB,QACnB,GAAG1E,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAC1B,8BACA,cACA0D,KAAKsB,GAAG,CAACjG,qBAAqBK,cAAc,EAAEoF;QAEhD,IAAIvB,OAAOyC,EAAE,EAAE;YACb9C,UAAUU,SAAS,IAAIL,OAAO0C,KAAK;YACnC,MAAMC,SAAStH,eAAe2E,OAAO4C,IAAI;YACzCJ,QAAQhH,eAAemH,QAAQvH;YAC/BuE,UAAUK,MAAM,GAAG;gBAAEC,OAAO;gBAAMC,UAAUyC,OAAOzC,QAAQ;YAAC;QAC9D,OAAO,IACL,CAAEF,CAAAA,OAAOhB,OAAO,KAAK,iBAAiB,EAACgB,iBAAAA,OAAO6C,MAAM,YAAb7C,iBAAiB,MAAM,OAAO,EAACA,kBAAAA,OAAO6C,MAAM,YAAb7C,kBAAiB,KAAK,GAAE,GAC9F;YACAL,UAAUS,SAAS,GAAGoB,cAAc,SAAS;YAC7C,OAAOjB;QACT;QACA,MAAMuC,UAAU,CAACpE;YACf,MAAMjB,MAAM,IAAIc,IAAIG;YACpB,OAAOnD,aAAaiH,OAAO,GAAG/E,IAAIY,QAAQ,GAAGZ,IAAIoB,MAAM,EAAE;QAC3D;QAEA,kEAAkE;QAClE,MAAMkE,OAAO7D,QAAQ6D,IAAI,GACrB7D,QAAQ6D,IAAI,CACTC,GAAG,CAAC,CAACC;YACJ,IAAI;gBACF,OAAOlF,kBAAkB,IAAIQ,IAAI0E,MAAMlG;YACzC,EAAE,eAAM;gBACN,OAAO;YACT;QACF,GACCmG,MAAM,CAAC,CAACxE,MAAuB,CAAC,CAACA,OACpC;QAEJ,uEAAuE;QACvE,MAAMyE,eAAyB,EAAE;QACjC,KAAK,MAAMF,OAAOtD,UAAUK,MAAM,CAACE,QAAQ,CAAE;YAC3C,IAAI;gBACF,MAAMzC,MAAM,IAAIc,IAAI0E;gBACpB,IAAI3B,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIe,QAAQ;YACnD,EAAE,eAAM;YACN,0CAA0C;YAC5C;QACF;QACA,IAAI,CAAC2E,aAAa9F,MAAM,IAAI,CAAC0F,MAAMI,aAAab,IAAI,CAAC,GAAGvF,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC;QAChF,IAAIgG,MAAMI,aAAa9F,MAAM,GAAG;QAChC,MAAM+F,cAAc,IAAIC;QACxB,MAAMC,cAAwB,EAAE;QAChC,IAAIC,eAAe;QACnB,MAAOJ,aAAa9F,MAAM,IAAIkG,eAAezH,qBAAqBO,WAAW,IAAI,CAACmF,YAAa;YAC7F,MAAMgC,OAAOL,aAAaM,KAAK;YAC/B,IAAIL,YAAYM,GAAG,CAACF,OAAO;YAC3BJ,YAAYO,GAAG,CAACH;YAChB,IAAI,SAASpF,IAAI,CAAC,IAAIG,IAAIiF,MAAMnF,QAAQ,GAAG;YAC3C,IAAI,CAACyE,QAAQU,OAAO;YACpB,MAAMI,OAAOnD,KAAKsB,GAAG,CAACjG,qBAAqBM,eAAe,EAAEmF;YAC5D,IAAIqC,QAAQ,GAAG;YACfL,gBAAgB;YAChB,MAAMM,UAAU,MAAMpC,QAAQ+B,MAAM,oBAAoB,mCAAmCI;YAC3F,IAAI,CAACC,QAAQpB,EAAE,EAAE;YACjB9C,UAAUU,SAAS,IAAIwD,QAAQnB,KAAK;YACpC,MAAMoB,UAAUxI,aAAauI,QAAQjB,IAAI;YACzC,KAAK,MAAMmB,YAAYD,QAAQE,SAAS,CAAE;gBACxC,IAAIvG;gBACJ,IAAI;oBACFA,MAAM,IAAIc,IAAIwF;gBAChB,EAAE,eAAM;oBACN;gBACF;gBACA,IAAID,QAAQG,KAAK,EAAE;oBACjB,IAAI3C,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIe,QAAQ;oBACjD;gBACF;gBACA,MAAME,MAAMX,kBAAkBN,KAAKV;gBACnC,IAAI2B,OAAO4E,YAAYjG,MAAM,GAAGvB,qBAAqBQ,cAAc,EAAEgH,YAAYhB,IAAI,CAAC5D;YACxF;QACF;QACAiB,UAAUQ,WAAW,GAAGmD,YAAYjG,MAAM;QAC1C,IAAI6B,QAAQgF,YAAY,EAAE;YACxB,MAAMC,QAAQ,CAACzF,MAAgB,IAAIH,IAAIG,KAAKL,QAAQ,CAAC+F,KAAK,CAAC,KAAKlB,MAAM,CAACmB,SAAShH,MAAM;YACtF,MAAMiH,SAAShB,YAAYN,GAAG,CAAC,CAACtE,KAAK6F,QAAW,CAAA;oBAAE7F;oBAAK6F;oBAAOJ,OAAOA,MAAMzF;gBAAK,CAAA;YAChF4F,OAAOE,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEN,KAAK,GAAGO,EAAEP,KAAK,IAAIM,EAAEF,KAAK,GAAGG,EAAEH,KAAK;YAC5DjB,YAAYqB,MAAM,CAAC,GAAGrB,YAAYjG,MAAM,KAAKiH,OAAOtB,GAAG,CAAC,CAACX,QAAUA,MAAM3D,GAAG;QAC9E;QAEA,yEAAyE;QACzE,wCAAwC;QACxC,MAAMkG,QAA+D7B,OACjEA,KAAKC,GAAG,CAAC,CAACtE,MAAS,CAAA;gBAAEA;gBAAKmG,SAAU,IAAItG,IAAIG,KAAKL,QAAQ,KAAK,MAAM,SAAS;YAAqC,CAAA,KAClH;YAAC;gBAAEK,KAAK6D;gBAAMsC,SAAS;YAAO;SAAE;QACpC,MAAMC,SAAS,IAAIzB,IAAYN,eAAAA,OAAQ;YAACR;SAAK;QAC7C,KAAK,MAAM7D,OAAO4E,YAAa;YAC7B,IAAIwB,OAAOpB,GAAG,CAAChF,MAAM;YACrBoG,OAAOnB,GAAG,CAACjF;YACXkG,MAAMtC,IAAI,CAAC;gBAAE5D;gBAAKmG,SAAS;YAAU;QACvC;QACA,MAAME,OAAO,IAAI1B;QACjB,uEAAuE;QACvE,qEAAqE;QACrE,6DAA6D;QAC7D,0EAA0E;QAC1E,IAAI2B,YAAYjC,CAAAA,wBAAAA,KAAM1F,MAAM,IAAG,IAAIkB,IAAIwE,IAAI,CAAC,EAAE,EAAEpF,MAAM,GAAGZ,OAAO,CAAC,EAAE;QACnE,MAAMkI,QAAQ,CAAChC;YACb,MAAMxF,MAAM,IAAIc,IAAI0E;YACpB,OAAOlG,QAAQW,QAAQ,CAACD,IAAIE,MAAM,KAAKF,IAAIE,MAAM,KAAKqH,YAClD,GAAGA,YAAYvH,IAAIY,QAAQ,GAAGZ,IAAIoB,MAAM,EAAE,GAC1CoE;QACN;QACA,MAAMiC,cAAc1E,WAAW,IAAI;QACnC,IAAI2E,WAAW;QAEf,4EAA4E,GAC5E,MAAMC,SAAgC,EAAE;QACxC,4FAA4F,GAC5F,IAAIC,WAAW;QACf,IAAIC,OAA8B;QAClC,gFAAgF,GAChF,MAAMC,SAAmB,EAAE;QAC3B,gGAAgG,GAChG,IAAIC,SAAS,CAAC,CAACzC;QACf,iGAAiG,GACjG,MAAM0C,OAAO;YACX,IAAI9F,UAAUE,KAAK,CAACxC,MAAM,IAAImD,UAAU,OAAO;YAC/C,IAAIgB,aAAa,OAAO;YACxB,IAAID,oBAAoB,GAAG,OAAO;YAClC,IAAI4D,YAAYD,aAAa,OAAO;YACpC,IAAIvF,UAAUE,KAAK,CAACxC,MAAM,GAAG+H,OAAO/H,MAAM,IAAImD,UAAU,OAAO;YAC/D,IAAIe,mBAAmB8D,YAAY,GAAG,OAAO;YAC7C,OAAO;QACT;QACA,kGAAkG,GAClG,MAAMK,SAAS,CAAC,EAAElC,IAAI,EAAE9E,GAAG,EAAEiH,GAAG,EAAEC,IAAI,EAAS;gBAuBtB7H;YAtBvB,MAAM8G,UAAUrB,KAAKqB,OAAO;YAC5B,IAAIA,YAAY,QAAQW,SAAS;YACjC,IAAI,CAACI,KAAKnD,EAAE,EAAE;gBACZ,IAAImD,KAAK5G,OAAO,KAAK,eAAe2G,MAAM5J,cAAc;oBACtDwJ,OAAOjD,IAAI,CAAC5D;oBACZ,OAAO;gBACT;gBACA,IAAIkH,KAAK5G,OAAO,KAAK,aAAawC,aAAa;oBAC7C+D,OAAOjD,IAAI,CAAC5D;oBACZ,OAAO;gBACT;gBACA0D,KAAK;oBACH3E,KAAKiB;oBACLmH,QAAQ/G,cAAc8G;mBAClBA,KAAK/C,MAAM,GAAG;oBAAEA,QAAQ+C,KAAK/C,MAAM;gBAAC,IAAI,CAAC,GACzC+C,KAAK7B,QAAQ,GAAG;oBAAEA,UAAU6B,KAAK7B,QAAQ;gBAAC,IAAI,CAAC,GAC/C6B,KAAKE,IAAI,GAAG;oBAAEA,MAAMF,KAAKE,IAAI;gBAAC,IAAI,CAAC;gBAEzC,OAAO;YACT;YACAnG,UAAUU,SAAS,IAAIuF,KAAKlD,KAAK;YACjC,IAAImC,YAAY,QAAQG,YAAY,IAAIzG,IAAIqH,KAAKnI,GAAG,EAAEE,MAAM;YAC5D,MAAMoI,WAAWd,OAAMlH,qBAAAA,kBAAkB,IAAIQ,IAAIqH,KAAKnI,GAAG,GAAGV,oBAArCgB,qBAAiD6H,KAAKnI,GAAG;YAChF,IAAIsI,aAAarH,KAAK;gBACpB,IAAIqG,KAAKrB,GAAG,CAACqC,WAAW,OAAO;gBAC/BhB,KAAKpB,GAAG,CAACoC;gBACT,IAAI,CAACjD,QAAQiD,WAAW;oBACtB3D,KAAK;wBAAE3E,KAAKsI;wBAAUF,QAAQ;oBAAS;oBACvC,OAAO;gBACT;YACF;YACA,MAAMG,UAAU9K,sBAAsB0K,KAAKhD,IAAI,EAAEgD,KAAKnI,GAAG,EAAE6D;YAC3D,IAAI0E,QAAQC,SAAS,EAAE;gBACrB7D,KAAK;oBAAE3E,KAAKsI;oBAAUF,QAAQ;gBAAa;gBAC3C,OAAO;YACT;YACA,MAAMK,QAAkB,EAAE;YAC1B,KAAK,MAAMC,QAAQH,QAAQE,KAAK,CAAE;gBAChC,MAAME,UAAUrI,kBAAkB,IAAIQ,IAAI4H,OAAOpJ;gBACjD,IAAI,CAACqJ,SAAS;gBACdF,MAAM5D,IAAI,CAAC8D;gBACX,IAAI,CAACrD,QAAQ,CAAC+B,OAAOpB,GAAG,CAAC0C,UAAU;oBACjCtB,OAAOnB,GAAG,CAACyC;oBACXxB,MAAMtC,IAAI,CAAC;wBAAE5D,KAAK0H;wBAASvB,SAAS;oBAAO;gBAC7C;YACF;YACAlF,UAAUE,KAAK,CAACyC,IAAI,CAAC;gBACnB7E,KAAKsI;gBACLM,MAAM,IAAI9H,IAAIwH,UAAU1H,QAAQ;gBAChCwG;gBACAyB,OAAON,QAAQM,KAAK;gBACpBC,aAAaP,QAAQO,WAAW;gBAChCC,MAAMR,QAAQQ,IAAI;gBAClBC,WAAWT,QAAQS,SAAS;gBAC5BC,SAASV,QAAQU,OAAO;gBACxBC,UAAUX,QAAQW,QAAQ;gBAC1BC,OAAOZ,QAAQY,KAAK;gBACpBC,OAAOb,QAAQa,KAAK;gBACpBC,cAAcd,QAAQc,YAAY;gBAClCC,QAAQf,QAAQe,MAAM;gBACtBC,QAAQhB,QAAQgB,MAAM;gBACtBC,UAAUjB,QAAQiB,QAAQ;gBAC1Bf,OAAO;uBAAI,IAAI7C,IAAI6C,MAAMlD,GAAG,CAACiC;iBAAQ;gBACrCiC,OAAOlB,QAAQkB,KAAK;gBACpBxE,OAAOkD,KAAKlD,KAAK;YACnB;YACA,OAAO;QACT;QACA,OAAS;gBA4BAgD;YA3BP,MAAMyB,QAAQ3B,SAAS1J,qBAAqBU,WAAW,GAAG;YAC1D,MAAO,CAAC8I,QAAQV,MAAMvH,MAAM,IAAI+H,OAAO/H,MAAM,GAAG8J,MAAO;gBACrD,MAAMC,MAAM3B;gBACZ,IAAI2B,QAAQ,QAAQ;gBACpB,IAAIA,KAAK;oBACP9B,OAAO8B;oBACP;gBACF;gBACA,MAAM5D,OAAOoB,MAAMnB,KAAK;gBACxB,MAAM/E,MAAMuG,MAAMzB,KAAK9E,GAAG;gBAC1B,IAAIqG,KAAKrB,GAAG,CAAChF,MAAM;gBACnB,IAAI,CAACoE,QAAQpE,MAAM;oBACjB0D,KAAK;wBAAE3E,KAAKiB;wBAAKmH,QAAQ;oBAAS;oBAClC,IAAIrC,KAAKqB,OAAO,KAAK,QAAQW,SAAS;oBACtC;gBACF;gBACAL,YAAY;gBACZJ,KAAKpB,GAAG,CAACjF;gBACT,MAAMiH,MAAMlF,KAAKsB,GAAG,CAAChG,cAAcwF,mBAAmB8D;gBACtDA,YAAYM;gBACZP,OAAO9C,IAAI,CACTb,QAAQ/C,KAAK,uCAAuC,0CAA0CiH,KAAK0B,IAAI,CAAC,CAACzB,OAAU,CAAA;wBAAEpC;wBAAM9E;wBAAKiH;wBAAKC;oBAAK,CAAA;YAE9I;YACA,IAAI,CAACR,OAAO/H,MAAM,EAAE;YACpB,MAAMiK,QAAQ,MAAOlC,OAAO3B,KAAK;YACjC4B,YAAYiC,MAAM3B,GAAG;YACrBL,QAAOI,UAAAA,OAAO4B,kBAAP5B,UAAiBJ;QAC1B;QACA3F,UAAUS,SAAS,GAAGkF;QACtB,wEAAwE;QACxE,qDAAqD;QACrD,IAAIA,QAAQ,CAACvC,MAAM;YACjB,MAAMwE,SAAS;mBAAIhC;aAAO;YAC1B,KAAK,MAAM/B,QAAQoB,MAAO;gBACxB,MAAMlG,MAAMuG,MAAMzB,KAAK9E,GAAG;gBAC1B,IAAI,CAACqG,KAAKrB,GAAG,CAAChF,QAAQ,CAAC6I,OAAO7J,QAAQ,CAACgB,MAAM6I,OAAOjF,IAAI,CAAC5D;YAC3D;YACA,KAAK,MAAMjB,OAAO8J,OAAQnF,KAAK;gBAAE3E;gBAAKoI,QAAQ;YAAc;QAC9D;QACA,OAAOtF;IACT,SAAU;YAERrB;QADAsI,aAAapG;SACblC,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBuI,mBAAmB,CAAC,SAAS1G;IAC/C;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAe2G,yBACpB5K,MAAc,EACd6K,QAAgB,EAChB9K,QAA8B,CAAC,CAAC;QAEdA,cAmBLkC;IAnBb,MAAMM,aAAYxC,eAAAA,MAAMyC,KAAK,YAAXzC,eAAe5B;IACjC,MAAM4C,OAAO0B,OAAOzC,iBAAAA,SAAU,IAAI0C,IAAI,GAAGC,WAAW,GAAG3B,OAAO,CAAC,UAAU;IACzE,IAAI,CAACnB,eAAeyB,IAAI,CAACP,SAAS9C,sBAAsB8C,OAAO,OAAO;QAAE+J,UAAU;QAAOC,QAAQ,EAAE;IAAC;IACpG,MAAM9K,UAAUa,kBAAkBC;IAClC,MAAMkB,SAAS,MAAMM,UAAU,GAAGtC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;QAC/C6E,UAAU9F,qBAAqBC,YAAY;QAC3C2F,QAAQ;QACRC,cAAc;QACdE,WAAWhG;QACXoB,YAAY,MAAML,mBAAmBC,OAAOgB,MAAMd;QAClD+E,WAAWhG,qBAAqBI,gBAAgB;QAChD8F,iBAAiB,CAACC,OAAOC,KAAOA,GAAGlE,QAAQ,KAAK,YAAYjB,QAAQW,QAAQ,CAACwE,GAAGvE,MAAM;IACxF;IACA,IAAI,CAACoB,OAAO0D,EAAE,EAAE;QACd,qEAAqE;QACrE,oDAAoD;QACpD,MAAMmF,WAAW7I,OAAOC,OAAO,KAAK,iBAAiBD,OAAOC,OAAO,KAAK,sBAAsBD,OAAOC,OAAO,KAAK;QACjH,OAAO;YAAE4I;YAAUC,QAAQ,EAAE;YAAEpK,KAAKsB,OAAOtB,GAAG;QAAC;IACjD;IACA,MAAMqK,QAAO/I,sBAAAA,OAAO6D,IAAI,CAACwB,KAAK,CAAC,WAAW,CAAC,EAAE,YAAhCrF,sBAAoC;IACjD,OAAO;QAAE6I,UAAU;QAAMC,QAAQ1M,eAAe2M,MAAMH;QAAWlK,KAAKsB,OAAOtB,GAAG;IAAC;AACnF;AAEA;;;CAGC,GACD,OAAO,SAASsK,yBAAyBnC,IAAoB,EAAEoC,WAAW,KAAM;IAC9E,MAAMC,QAAkB,EAAE;IAC1B,IAAIrC,KAAKU,KAAK,EAAE2B,MAAM3F,IAAI,CAAC,CAAC,OAAO,EAAEsD,KAAKU,KAAK,EAAE;IACjD,IAAIV,KAAKW,WAAW,EAAE0B,MAAM3F,IAAI,CAAC,CAAC,aAAa,EAAEsD,KAAKW,WAAW,EAAE;IACnE,KAAK,MAAM2B,SAAStC,KAAKmB,MAAM,CAAE;QAC/B,IAAImB,MAAMC,IAAI,KAAK,WAAWF,MAAM3F,IAAI,CAAC,GAAG,IAAI8F,MAAM,CAACF,MAAMG,KAAK,EAAE,CAAC,EAAEH,MAAMtF,IAAI,EAAE;aAC9E,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,QAAQ,EAAE4F,MAAMI,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAClF,IAAIJ,MAAMC,IAAI,KAAK,aAAaF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aAC5D,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aACxD,IAAIsF,MAAMC,IAAI,KAAK,QAAQ;YAC9B,4DAA4D;YAC5D,IAAI,CAACD,MAAMK,MAAM,EAAEN,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;QAC1C,OAAOqF,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;IAC9B;IACA,OAAO5H,mBAAmB,CAAC,KAAK,EAAE4K,KAAKS,IAAI,CAAC,2BAA2B,CAAC,EAAE4B,MAAMO,IAAI,CAAC,OAAO;QAAER;IAAS;AACzG"}

package/src/lib/site-import/site-import-edge-server.d.ts

@@ -0,0 +1,31 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * The server read the site-import edge defers (AGL-4085): whether this
+ * platform serves a hostname, which opens Firestore and the hosting
+ * platform's API. `site-import-edge.ts` loads it only once a secret is
+ * configured, so a crawl without one never opens either.
+ *
+ * The deferral targets THIS module by relative path, for the reason
+ * `ai-theme-brand-server.ts` gives: an `import()` of `@aglyn/tenant-data-admin`
+ * by its package specifier makes nx treat that lib as lazy-loaded everywhere,
+ * and every static import of it across the plugin, the console and the
+ * tenant then fails lint.
+ *
+ * Import this module dynamically, and only from `site-import-edge.ts`.
+ */
+export { isPlatformServedDomain } from '@aglyn/tenant-data-admin/server/platform-served-domain';

package/src/lib/site-import/site-import-edge-server.js

@@ -0,0 +1,31 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ /**
+ * The server read the site-import edge defers (AGL-4085): whether this
+ * platform serves a hostname, which opens Firestore and the hosting
+ * platform's API. `site-import-edge.ts` loads it only once a secret is
+ * configured, so a crawl without one never opens either.
+ *
+ * The deferral targets THIS module by relative path, for the reason
+ * `ai-theme-brand-server.ts` gives: an `import()` of `@aglyn/tenant-data-admin`
+ * by its package specifier makes nx treat that lib as lazy-loaded everywhere,
+ * and every static import of it across the plugin, the console and the
+ * tenant then fails lint.
+ *
+ * Import this module dynamically, and only from `site-import-edge.ts`.
+ */ export { isPlatformServedDomain } from "@aglyn/tenant-data-admin/server/platform-served-domain";
+
+//# sourceMappingURL=site-import-edge-server.js.map

package/src/lib/site-import/site-import-edge-server.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-edge-server.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * The server read the site-import edge defers (AGL-4085): whether this\n * platform serves a hostname, which opens Firestore and the hosting\n * platform's API. `site-import-edge.ts` loads it only once a secret is\n * configured, so a crawl without one never opens either.\n *\n * The deferral targets THIS module by relative path, for the reason\n * `ai-theme-brand-server.ts` gives: an `import()` of `@aglyn/tenant-data-admin`\n * by its package specifier makes nx treat that lib as lazy-loaded everywhere,\n * and every static import of it across the plugin, the console and the\n * tenant then fails lint.\n *\n * Import this module dynamically, and only from `site-import-edge.ts`.\n */\nexport { isPlatformServedDomain } from '@aglyn/tenant-data-admin/server/platform-served-domain'\n"],"names":["isPlatformServedDomain"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;CAaC,GACD,SAASA,sBAAsB,QAAQ,yDAAwD"}

package/src/lib/site-import/site-import-edge.d.ts

@@ -0,0 +1,44 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** The header the edge rule matches. Lowercase, as the rule declares it. */
+export declare const SITE_IMPORT_EDGE_HEADER = "x-aglyn-import";
+/** A secret shorter than this is treated as unset: `openssl rand -hex 32` makes 64. */
+export declare const SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH = 32;
+/** Extra headers for one request, asked per address. */
+export type SiteImportHeadersFor = (url: URL) => Readonly<Record<string, string>>;
+/** Sends nothing anywhere. */
+export declare const NO_SITE_IMPORT_HEADERS: SiteImportHeadersFor;
+export interface SiteImportEdgeDeps {
+    /** Where the secret is read from; `process.env` by default. */
+    env?: Readonly<Record<string, string | undefined>>;
+    /** Whether this platform serves a hostname; `isPlatformServedDomain` by default. */
+    served?: (hostname: string) => Promise<boolean>;
+}
+/** The edge secret, or null when it is unset or not shaped like one. Never logged. */
+export declare function siteImportEdgeToken(env?: SiteImportEdgeDeps['env']): string | null;
+/**
+ * The edge header for requests to one hostname: the secret when this platform
+ * serves it, else nothing. Never throws.
+ */
+export declare function siteImportEdgeHeaders(hostname: string, deps?: SiteImportEdgeDeps): Promise<Readonly<Record<string, string>>>;
+/**
+ * Per-request headers for a crawl of these origins: each origin's hostname is
+ * decided once, and a request is given the header only when it is https on
+ * the default port to an origin decided "served". Anything else, a hop to
+ * another host included, gets nothing.
+ */
+export declare function siteImportEdgeHeadersFor(origins: readonly string[], deps?: SiteImportEdgeDeps): Promise<SiteImportHeadersFor>;

package/src/lib/site-import/site-import-edge.js

@@ -0,0 +1,108 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ /*==========================================
+ * AGLYNBOT THROUGH AGLYN'S OWN EDGE (AGL-4085).
+ *
+ * The tenant deployment's bot protection answers a client that runs no
+ * JavaScript with a 429 challenge page, and AglynBot is such a client. So a
+ * site we host, aglyn.com included, could not be brought in: on 2026-10-20 the
+ * import read 10 of 121 pages, the ten the store-review bypass happens to
+ * admit, and no home page.
+ *
+ * The edge rule "Site import crawler bypass" (declared in
+ * `tools/scripts/lib/firewall-posture.mjs`) lifts that challenge, and nothing
+ * else, for a request carrying BOTH this header with the secret AND a
+ * user agent starting `AglynBot/`. This module is the only place that holds
+ * the secret, and it gives it out for one reason: a request to a hostname this
+ * platform serves (`isPlatformServedDomain`: our host records, then the hosting
+ * platform's live view). A site anywhere else never sees it.
+ *
+ * - The secret is `SITE_IMPORT_EDGE_TOKEN`, a server variable on the console,
+ *   where the crawl runs. Read in the bracket form, so Next never substitutes
+ *   it into a bundle. Unset, or not shaped like a secret, nothing is sent and
+ *   nothing is read: the crawler is challenged as it was before.
+ * - The decision is per hostname and per request: the fetch asks
+ *   `headersFor(url)` again on every redirect hop, so a hop to another host
+ *   carries nothing even where the caller would follow it.
+ * - Rotation: a new value in the console's env and the same value in the
+ *   rule, together. See `apps/docs/docs/operations/firewall-posture.md`.
+ *=========================================*/ /** The header the edge rule matches. Lowercase, as the rule declares it. */ export const SITE_IMPORT_EDGE_HEADER = 'x-aglyn-import';
+/** A secret shorter than this is treated as unset: `openssl rand -hex 32` makes 64. */ export const SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH = 32;
+/** Header-safe characters only, so the value can neither break a header nor fail the rule's match. */ const TOKEN_PATTERN = /^[A-Za-z0-9_-]+$/;
+/** Sends nothing anywhere. */ export const NO_SITE_IMPORT_HEADERS = ()=>({});
+/** The edge secret, or null when it is unset or not shaped like one. Never logged. */ export function siteImportEdgeToken(env) {
+    var _ref;
+    // The direct read is spelled out so the self-host env reference guard sees it.
+    const token = String((_ref = env ? env['SITE_IMPORT_EDGE_TOKEN'] : process.env['SITE_IMPORT_EDGE_TOKEN']) != null ? _ref : '').trim();
+    if (token.length < SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH || token.length > 256) return null;
+    return TOKEN_PATTERN.test(token) ? token : null;
+}
+async function defaultServed(hostname) {
+    // Loaded only once a secret is configured, so a crawl that has none (every
+    // spec, every self-hosted deployment without the rule) never opens Firestore.
+    // By relative path: see `site-import-edge-server.ts`.
+    const { isPlatformServedDomain } = await import("./site-import-edge-server.js");
+    return isPlatformServedDomain(hostname);
+}
+/**
+ * The edge header for requests to one hostname: the secret when this platform
+ * serves it, else nothing. Never throws.
+ */ export async function siteImportEdgeHeaders(hostname, deps = {}) {
+    const token = siteImportEdgeToken(deps.env);
+    if (!token) return {};
+    const name = String(hostname != null ? hostname : '').trim().toLowerCase();
+    if (!name) return {};
+    try {
+        var _deps_served;
+        const served = await ((_deps_served = deps.served) != null ? _deps_served : defaultServed)(name);
+        return served ? {
+            [SITE_IMPORT_EDGE_HEADER]: token
+        } : {};
+    } catch (error) {
+        console.error('[site-import-edge] could not tell whether the platform serves', name, error);
+        return {};
+    }
+}
+/**
+ * Per-request headers for a crawl of these origins: each origin's hostname is
+ * decided once, and a request is given the header only when it is https on
+ * the default port to an origin decided "served". Anything else, a hop to
+ * another host included, gets nothing.
+ */ export async function siteImportEdgeHeadersFor(origins, deps = {}) {
+    if (!siteImportEdgeToken(deps.env)) return NO_SITE_IMPORT_HEADERS;
+    const decided = new Map();
+    for (const raw of origins){
+        let origin;
+        try {
+            origin = new URL(raw);
+        } catch (unused) {
+            continue;
+        }
+        if (!isPlainHttpsOrigin(origin) || decided.has(origin.origin)) continue;
+        const headers = await siteImportEdgeHeaders(origin.hostname, deps);
+        if (Object.keys(headers).length) decided.set(origin.origin, headers);
+    }
+    if (!decided.size) return NO_SITE_IMPORT_HEADERS;
+    return (url)=>{
+        var _decided_get;
+        return isPlainHttpsOrigin(url) ? (_decided_get = decided.get(url.origin)) != null ? _decided_get : {} : {};
+    };
+}
+function isPlainHttpsOrigin(url) {
+    return url.protocol === 'https:' && (!url.port || url.port === '443') && !url.username && !url.password;
+}
+
+//# sourceMappingURL=site-import-edge.js.map

package/src/lib/site-import/site-import-edge.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-edge.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * AGLYNBOT THROUGH AGLYN'S OWN EDGE (AGL-4085).\n *\n * The tenant deployment's bot protection answers a client that runs no\n * JavaScript with a 429 challenge page, and AglynBot is such a client. So a\n * site we host, aglyn.com included, could not be brought in: on 2026-10-20 the\n * import read 10 of 121 pages, the ten the store-review bypass happens to\n * admit, and no home page.\n *\n * The edge rule \"Site import crawler bypass\" (declared in\n * `tools/scripts/lib/firewall-posture.mjs`) lifts that challenge, and nothing\n * else, for a request carrying BOTH this header with the secret AND a\n * user agent starting `AglynBot/`. This module is the only place that holds\n * the secret, and it gives it out for one reason: a request to a hostname this\n * platform serves (`isPlatformServedDomain`: our host records, then the hosting\n * platform's live view). A site anywhere else never sees it.\n *\n * - The secret is `SITE_IMPORT_EDGE_TOKEN`, a server variable on the console,\n *   where the crawl runs. Read in the bracket form, so Next never substitutes\n *   it into a bundle. Unset, or not shaped like a secret, nothing is sent and\n *   nothing is read: the crawler is challenged as it was before.\n * - The decision is per hostname and per request: the fetch asks\n *   `headersFor(url)` again on every redirect hop, so a hop to another host\n *   carries nothing even where the caller would follow it.\n * - Rotation: a new value in the console's env and the same value in the\n *   rule, together. See `apps/docs/docs/operations/firewall-posture.md`.\n *=========================================*/\n\n/** The header the edge rule matches. Lowercase, as the rule declares it. */\nexport const SITE_IMPORT_EDGE_HEADER = 'x-aglyn-import'\n\n/** A secret shorter than this is treated as unset: `openssl rand -hex 32` makes 64. */\nexport const SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH = 32\n\n/** Header-safe characters only, so the value can neither break a header nor fail the rule's match. */\nconst TOKEN_PATTERN = /^[A-Za-z0-9_-]+$/\n\n/** Extra headers for one request, asked per address. */\nexport type SiteImportHeadersFor = (url: URL) => Readonly<Record<string, string>>\n\n/** Sends nothing anywhere. */\nexport const NO_SITE_IMPORT_HEADERS: SiteImportHeadersFor = () => ({})\n\nexport interface SiteImportEdgeDeps {\n  /** Where the secret is read from; `process.env` by default. */\n  env?: Readonly<Record<string, string | undefined>>\n  /** Whether this platform serves a hostname; `isPlatformServedDomain` by default. */\n  served?: (hostname: string) => Promise<boolean>\n}\n\n/** The edge secret, or null when it is unset or not shaped like one. Never logged. */\nexport function siteImportEdgeToken(env?: SiteImportEdgeDeps['env']): string | null {\n  // The direct read is spelled out so the self-host env reference guard sees it.\n  const token = String((env ? env['SITE_IMPORT_EDGE_TOKEN'] : process.env['SITE_IMPORT_EDGE_TOKEN']) ?? '').trim()\n  if (token.length < SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH || token.length > 256) return null\n  return TOKEN_PATTERN.test(token) ? token : null\n}\n\nasync function defaultServed(hostname: string): Promise<boolean> {\n  // Loaded only once a secret is configured, so a crawl that has none (every\n  // spec, every self-hosted deployment without the rule) never opens Firestore.\n  // By relative path: see `site-import-edge-server.ts`.\n  const { isPlatformServedDomain } = await import('./site-import-edge-server')\n  return isPlatformServedDomain(hostname)\n}\n\n/**\n * The edge header for requests to one hostname: the secret when this platform\n * serves it, else nothing. Never throws.\n */\nexport async function siteImportEdgeHeaders(\n  hostname: string,\n  deps: SiteImportEdgeDeps = {},\n): Promise<Readonly<Record<string, string>>> {\n  const token = siteImportEdgeToken(deps.env)\n  if (!token) return {}\n  const name = String(hostname ?? '').trim().toLowerCase()\n  if (!name) return {}\n  try {\n    const served = await (deps.served ?? defaultServed)(name)\n    return served ? { [SITE_IMPORT_EDGE_HEADER]: token } : {}\n  } catch (error) {\n    console.error('[site-import-edge] could not tell whether the platform serves', name, error)\n    return {}\n  }\n}\n\n/**\n * Per-request headers for a crawl of these origins: each origin's hostname is\n * decided once, and a request is given the header only when it is https on\n * the default port to an origin decided \"served\". Anything else, a hop to\n * another host included, gets nothing.\n */\nexport async function siteImportEdgeHeadersFor(\n  origins: readonly string[],\n  deps: SiteImportEdgeDeps = {},\n): Promise<SiteImportHeadersFor> {\n  if (!siteImportEdgeToken(deps.env)) return NO_SITE_IMPORT_HEADERS\n  const decided = new Map<string, Readonly<Record<string, string>>>()\n  for (const raw of origins) {\n    let origin: URL\n    try {\n      origin = new URL(raw)\n    } catch {\n      continue\n    }\n    if (!isPlainHttpsOrigin(origin) || decided.has(origin.origin)) continue\n    const headers = await siteImportEdgeHeaders(origin.hostname, deps)\n    if (Object.keys(headers).length) decided.set(origin.origin, headers)\n  }\n  if (!decided.size) return NO_SITE_IMPORT_HEADERS\n  return (url: URL) => (isPlainHttpsOrigin(url) ? decided.get(url.origin) ?? {} : {})\n}\n\nfunction isPlainHttpsOrigin(url: URL): boolean {\n  return url.protocol === 'https:' && (!url.port || url.port === '443') && !url.username && !url.password\n}\n"],"names":["SITE_IMPORT_EDGE_HEADER","SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH","TOKEN_PATTERN","NO_SITE_IMPORT_HEADERS","siteImportEdgeToken","env","token","String","process","trim","length","test","defaultServed","hostname","isPlatformServedDomain","siteImportEdgeHeaders","deps","name","toLowerCase","served","error","console","siteImportEdgeHeadersFor","origins","decided","Map","raw","origin","URL","isPlainHttpsOrigin","has","headers","Object","keys","set","size","url","get","protocol","port","username","password"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;2CA0B2C,GAE3C,0EAA0E,GAC1E,OAAO,MAAMA,0BAA0B,iBAAgB;AAEvD,qFAAqF,GACrF,OAAO,MAAMC,oCAAoC,GAAE;AAEnD,oGAAoG,GACpG,MAAMC,gBAAgB;AAKtB,4BAA4B,GAC5B,OAAO,MAAMC,yBAA+C,IAAO,CAAA,CAAC,CAAA,EAAE;AAStE,oFAAoF,GACpF,OAAO,SAASC,oBAAoBC,GAA+B;QAE3CA;IADtB,+EAA+E;IAC/E,MAAMC,QAAQC,QAAQF,OAAAA,MAAMA,GAAG,CAAC,yBAAyB,GAAGG,QAAQH,GAAG,CAAC,yBAAyB,YAA3EA,OAAgF,IAAII,IAAI;IAC9G,IAAIH,MAAMI,MAAM,GAAGT,qCAAqCK,MAAMI,MAAM,GAAG,KAAK,OAAO;IACnF,OAAOR,cAAcS,IAAI,CAACL,SAASA,QAAQ;AAC7C;AAEA,eAAeM,cAAcC,QAAgB;IAC3C,2EAA2E;IAC3E,8EAA8E;IAC9E,sDAAsD;IACtD,MAAM,EAAEC,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC;IAChD,OAAOA,uBAAuBD;AAChC;AAEA;;;CAGC,GACD,OAAO,eAAeE,sBACpBF,QAAgB,EAChBG,OAA2B,CAAC,CAAC;IAE7B,MAAMV,QAAQF,oBAAoBY,KAAKX,GAAG;IAC1C,IAAI,CAACC,OAAO,OAAO,CAAC;IACpB,MAAMW,OAAOV,OAAOM,mBAAAA,WAAY,IAAIJ,IAAI,GAAGS,WAAW;IACtD,IAAI,CAACD,MAAM,OAAO,CAAC;IACnB,IAAI;YACoBD;QAAtB,MAAMG,SAAS,MAAM,EAACH,eAAAA,KAAKG,MAAM,YAAXH,eAAeJ,eAAeK;QACpD,OAAOE,SAAS;YAAE,CAACnB,wBAAwB,EAAEM;QAAM,IAAI,CAAC;IAC1D,EAAE,OAAOc,OAAO;QACdC,QAAQD,KAAK,CAAC,iEAAiEH,MAAMG;QACrF,OAAO,CAAC;IACV;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAeE,yBACpBC,OAA0B,EAC1BP,OAA2B,CAAC,CAAC;IAE7B,IAAI,CAACZ,oBAAoBY,KAAKX,GAAG,GAAG,OAAOF;IAC3C,MAAMqB,UAAU,IAAIC;IACpB,KAAK,MAAMC,OAAOH,QAAS;QACzB,IAAII;QACJ,IAAI;YACFA,SAAS,IAAIC,IAAIF;QACnB,EAAE,eAAM;YACN;QACF;QACA,IAAI,CAACG,mBAAmBF,WAAWH,QAAQM,GAAG,CAACH,OAAOA,MAAM,GAAG;QAC/D,MAAMI,UAAU,MAAMhB,sBAAsBY,OAAOd,QAAQ,EAAEG;QAC7D,IAAIgB,OAAOC,IAAI,CAACF,SAASrB,MAAM,EAAEc,QAAQU,GAAG,CAACP,OAAOA,MAAM,EAAEI;IAC9D;IACA,IAAI,CAACP,QAAQW,IAAI,EAAE,OAAOhC;IAC1B,OAAO,CAACiC;YAAwCZ;eAA1BK,mBAAmBO,QAAOZ,eAAAA,QAAQa,GAAG,CAACD,IAAIT,MAAM,aAAtBH,eAA2B,CAAC,IAAI,CAAC;;AACnF;AAEA,SAASK,mBAAmBO,GAAQ;IAClC,OAAOA,IAAIE,QAAQ,KAAK,YAAa,CAAA,CAACF,IAAIG,IAAI,IAAIH,IAAIG,IAAI,KAAK,KAAI,KAAM,CAACH,IAAII,QAAQ,IAAI,CAACJ,IAAIK,QAAQ;AACzG"}