@aglyn/plugins-ai 1.0.0-beta.291 → 1.0.0-beta.293

package/src/lib/site-import/site-import-crawl.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-crawl.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * THE SITE-IMPORT CRAWLER (AGL-3974, parent AGL-3973).\n *\n * Reads a website a workspace has PROVED it owns (domain-ownership.ts in\n * tenant-data-admin; the doors check the proof before calling this) into a\n * typed inventory: each page's title, description, headings and text in\n * order, images and menu links. It writes nothing; the import job decides\n * what becomes a page.\n *\n * Every request goes through `fetchPublicTextDetailed`, the theme job's\n * SSRF-guarded fetch: https on the default port only, public addresses only,\n * the socket pinned to the address checked. On top of that:\n *\n * - **One site.** The site is `https://<domain>` and `https://www.<domain>`,\n *   one site to its owner. Every redirect hop and every discovered link must\n *   stay on it; a page that redirects anywhere else is refused, not followed.\n * - **AglynBot.** Every request says `AglynBot/1.0 (+<the operator's bot page>)`,\n *   and robots.txt is read first and honored for every page: the AglynBot\n *   group, else `*`. A robots.txt that answers 404 allows everything; one\n *   that cannot be read at all (5xx, no answer) stops the crawl, because\n *   \"could not ask\" is not \"allowed\".\n * - **Our own edge.** A site this platform serves gets the edge secret\n *   (`site-import-edge.ts`, AGL-4085), which lifts our bot challenge for\n *   AglynBot; it is asked per request and per redirect hop, and a request\n *   that leaves the site's two origins never carries it.\n * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an\n *   index's children included), then same-site links found on the pages read,\n *   for whatever the sitemap missed.\n * - **Budgets.** Pages (the caller's, from the plan), bytes per page (a page\n *   over it is refused, not cut), bytes in total, and wall-clock time. A\n *   whole site's worth fits (AGL-4083): aglyn.com's 123 pages average 440 KB,\n *   54 MB in all, so the total is 160 MB; a page is parsed and dropped as it\n *   is read, so what is held is its blocks, never its bytes.\n * - **Four at a time.** robots.txt, the sitemaps and the home page are read\n *   one after another; the rest four at a time, and the inventory keeps the\n *   order they were asked in, which is the sitemap's.\n * - **Nothing vanishes.** An address a stop (the clock, the bytes, the page\n *   budget) left unread is listed as `not-reached`, so the inventory's\n *   counts add up to the site. A wave's own read leaves its unread pages for\n *   its next pass instead.\n * - **No lookalikes.** A domain that wears a brand it is not\n *   (`lookalikeBrandForHost`) is refused before any request.\n * - **No sign-in pages.** A page with a password field is refused and\n *   reported; forms with payment fields, scripts, iframes and embeds never\n *   reach the inventory (site-import-html.ts).\n *=========================================*/\n\nimport { lookalikeBrandForHost } from '@aglyn/shared-util-email/outbound-phishing-screen'\nimport { fenceUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n  fetchPublicTextDetailed,\n  type PublicFetchEdge,\n  type PublicFetchOptions,\n  type PublicFetchResult,\n} from '../jobs/ai-theme-brand-inputs'\nimport {\n  extractSiteImportPage,\n  readMetaValues,\n  type SiteImportBlock,\n  type SiteImportBrandMark,\n  type SiteImportForm,\n  type SiteImportIcon,\n  type SiteImportImage,\n  type SiteImportNavLink,\n  type SiteImportOrganization,\n} from './site-import-html'\nimport {\n  AGLYN_BOT_TOKEN,\n  parseRobotsTxt,\n  parseSitemap,\n  robotsAllows,\n  robotsRulesFor,\n  type RobotsRule,\n} from './site-import-robots'\nimport { NO_SITE_IMPORT_HEADERS, siteImportEdgeHeadersFor, type SiteImportHeadersFor } from './site-import-edge'\n\n/**\n * Where the bot's user agent points a site owner who wants to know what it is\n * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on\n * a self-hosted deployment, so its crawler never names Aglyn's address; the\n * literal is only that variable's default.\n *\n * Aglyn's is the Bring my site guide, which says what AglynBot reads and how\n * robots.txt keeps it out. There is no page of its own (Zach, 10/20: \"we don't\n * need a bot page at all\"), and the site it reads is the reader's own: an\n * import starts only once its owner has proved the site is theirs.\n */\nexport const AGLYN_BOT_INFO_URL =\n  process.env['SITE_IMPORT_BOT_INFO_URL'] ?? 'https://docs.aglyn.com/ai/bring-my-site'\n\n/** What every crawler request says it is. */\nexport const AGLYN_BOT_USER_AGENT = `AglynBot/1.0 (+${AGLYN_BOT_INFO_URL})`\n\nexport const SITE_IMPORT_DEFAULTS = {\n  maxPageBytes: 2 * 1024 * 1024,\n  /** A whole site's worth (AGL-4083): 54 MB for aglyn.com's 123 pages, with room. */\n  maxTotalBytes: 160 * 1024 * 1024,\n  timeBudgetMs: 60_000,\n  /** One request's own clock, inside the whole crawl's. */\n  requestTimeoutMs: 8_000,\n  robotsMaxBytes: 256 * 1024,\n  sitemapMaxBytes: 4 * 1024 * 1024,\n  /** Sitemap documents read, an index's children included: an index of six children is seven. */\n  maxSitemaps: 24,\n  /** URLs a sitemap may contribute to the queue. */\n  maxSitemapUrls: 2_000,\n  /** Reasons kept in `skipped` before the list stops growing: every address a sitemap may name. */\n  maxSkipped: 2_000,\n  /** Pages read at once, after robots.txt, the sitemaps and the home page. */\n  concurrency: 4,\n} as const\n\n/** Never a page: files a link may point at that are not HTML. */\nconst NON_PAGE_EXTENSION =\n  /\\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i\n\nconst TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i\n\nconst DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\\.[a-z0-9-]{1,63})+$/\n\nexport type SiteImportSkipReason =\n  /** The page has a password field: a sign-in page, refused outright. */\n  | 'login-form'\n  /** robots.txt keeps AglynBot off this path. */\n  | 'robots'\n  /** A redirect left the site. */\n  | 'off-origin-redirect'\n  /** Over the per-page byte cap. */\n  | 'too-large'\n  /** Answered with something that is not HTML. */\n  | 'not-html'\n  /** Answered 4xx or 5xx. */\n  | 'http-error'\n  /** No answer, a private address, or a refused address. */\n  | 'unreachable'\n  /** Never asked for: a stop (time, bytes, the page budget) came first (AGL-4083). */\n  | 'not-reached'\n\nexport interface SiteImportSkipped {\n  url: string\n  reason: SiteImportSkipReason\n  status?: number\n  /** Where an off-origin redirect pointed. */\n  location?: string\n  /** Whose edge turned an error answer away, when it said (AGL-4083). */\n  edge?: PublicFetchEdge\n}\n\nexport interface SiteImportPage {\n  /** The address the page was read from, after any same-site redirect. */\n  url: string\n  path: string\n  /** Where the crawler learned of the page. */\n  foundBy: 'home' | 'sitemap' | 'link'\n  title: string\n  description: string\n  lang: string | null\n  canonical: string | null\n  /** The share card (AGL-4083). */\n  ogImage: string | null\n  /** The site's own name, as its head gives it. */\n  siteName: string\n  icons: SiteImportIcon[]\n  /** The header's link home: the brand's logo and words. */\n  brand: SiteImportBrandMark | null\n  organization: SiteImportOrganization | null\n  blocks: SiteImportBlock[]\n  images: SiteImportImage[]\n  navLinks: SiteImportNavLink[]\n  /** Same-site page addresses this page links to. */\n  links: string[]\n  forms: { kept: SiteImportForm[]; removed: number }\n  bytes: number\n}\n\nexport type SiteImportStop =\n  | 'page-budget'\n  | 'total-bytes'\n  | 'time'\n  /** robots.txt could not be read, so nothing was asked of the site. */\n  | 'robots-unreachable'\n\nexport interface SiteImportInventory {\n  domain: string\n  /** The site's origins: the bare name and its `www.` twin. */\n  origins: string[]\n  /** Set when nothing was fetched at all. */\n  refused: 'invalid-domain' | 'lookalike' | null\n  pages: SiteImportPage[]\n  skipped: SiteImportSkipped[]\n  /** Skips past {@link SITE_IMPORT_DEFAULTS.maxSkipped}, counted only. */\n  skippedOverflow: number\n  robots: { found: boolean; sitemaps: string[] }\n  /** Page addresses the sitemaps listed (same-site, before robots and budget). */\n  sitemapUrls: number\n  stoppedBy: SiteImportStop | null\n  bytesRead: number\n  startedAt: number\n  finishedAt: number\n}\n\nexport interface SiteImportCrawlOptions {\n  /** The proved domain; `www.` is folded in. */\n  domain: string\n  /** The most pages the inventory may hold: the plan's page count. */\n  maxPages: number\n  maxPageBytes?: number\n  maxTotalBytes?: number\n  timeBudgetMs?: number\n  signal?: AbortSignal\n  /**\n   * Read the sitemap's shallow addresses before its deep ones (AGL-3976), so a\n   * crawl the page budget stops reads a site's main pages before the tail of\n   * its blog. A sitemap's own order is kept among addresses of one depth.\n   */\n  shallowFirst?: boolean\n  /**\n   * Read exactly these addresses of the site, in order, and nothing else\n   * (AGL-3976): an import's wave reads its own pages again when it starts. No\n   * sitemap is read and no link is followed; robots.txt is read and honored\n   * as for any crawl, and every other rule holds.\n   */\n  only?: readonly string[]\n}\n\nexport interface SiteImportCrawlSeams {\n  fetch?: (url: string, options: PublicFetchOptions) => Promise<PublicFetchResult>\n  now?: () => number\n  /**\n   * Request headers for the site being read, asked once per crawl with its\n   * domain (AGL-4083, AGL-4085). Absent, the crawl asks the edge\n   * (`siteImportEdgeHeadersFor`): Aglyn's edge secret for each of the site's\n   * two origins this platform serves, decided origin by origin, and nothing\n   * anywhere else. Either way the headers go only to the site's two origins,\n   * and the fetch asks again on every redirect hop, so a hop anywhere else\n   * carries nothing. Always under the crawler's own `Accept` and `User-Agent`.\n   */\n  headersFor?: (domain: string) => Promise<Readonly<Record<string, string>>> | Readonly<Record<string, string>>\n}\n\n/**\n * The per-request headers of a crawl of `domain`: the seam's, else the edge's,\n * and none for any address off the site's two origins. Never throws.\n */\nasync function siteRequestHeaders(\n  seams: SiteImportCrawlSeams,\n  domain: string,\n  origins: readonly string[],\n): Promise<SiteImportHeadersFor> {\n  let inner: SiteImportHeadersFor = NO_SITE_IMPORT_HEADERS\n  try {\n    if (seams.headersFor) {\n      const headers = await seams.headersFor(domain)\n      if (headers && Object.keys(headers).length) inner = () => headers\n    } else {\n      inner = await siteImportEdgeHeadersFor(origins)\n    }\n  } catch (error) {\n    console.warn('[site-import] the edge headers could not be read; reading without them', { domain, error })\n  }\n  return (url: URL) => (origins.includes(url.origin) ? inner(url) : {})\n}\n\n/** The site's two origins for a domain. */\nexport function siteImportOrigins(domain: string): string[] {\n  const bare = domain.replace(/^www\\./, '')\n  return [`https://${bare}`, `https://www.${bare}`]\n}\n\n/** A crawlable page address on the site, normalized for de-duplication; else null. */\nexport function siteImportPageKey(url: URL, origins: readonly string[]): string | null {\n  if (url.protocol !== 'https:' || url.username || url.password) return null\n  if (url.port && url.port !== '443') return null\n  if (!origins.includes(url.origin)) return null\n  if (NON_PAGE_EXTENSION.test(url.pathname)) return null\n  const clean = new URL(url.toString())\n  clean.hash = ''\n  for (const key of [...clean.searchParams.keys()]) {\n    if (TRACKING_PARAM.test(key)) clean.searchParams.delete(key)\n  }\n  clean.search = clean.searchParams.toString() ? `?${clean.searchParams.toString()}` : ''\n  return clean.toString()\n}\n\nfunction skipReasonFor(result: PublicFetchResult): SiteImportSkipReason {\n  switch (result.refusal) {\n    case 'redirect-refused':\n      return 'off-origin-redirect'\n    case 'content-type':\n      return 'not-html'\n    case 'too-large':\n      return 'too-large'\n    case 'http-status':\n      return 'http-error'\n    default:\n      return 'unreachable'\n  }\n}\n\n/** Read a proved site into an inventory. Never throws for anything the site does. */\nexport async function crawlSiteForImport(\n  options: SiteImportCrawlOptions,\n  seams: SiteImportCrawlSeams = {},\n): Promise<SiteImportInventory> {\n  const now = seams.now ?? Date.now\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const domain = String(options.domain ?? '').trim().toLowerCase().replace(/\\.+$/, '').replace(/^www\\./, '')\n  const origins = DOMAIN_PATTERN.test(domain) ? siteImportOrigins(domain) : []\n  const startedAt = now()\n  const inventory: SiteImportInventory = {\n    domain,\n    origins,\n    refused: null,\n    pages: [],\n    skipped: [],\n    skippedOverflow: 0,\n    robots: { found: false, sitemaps: [] },\n    sitemapUrls: 0,\n    stoppedBy: null,\n    bytesRead: 0,\n    startedAt,\n    finishedAt: startedAt,\n  }\n  const finish = () => {\n    inventory.finishedAt = now()\n    return inventory\n  }\n  if (!origins.length) {\n    inventory.refused = 'invalid-domain'\n    return finish()\n  }\n  if (lookalikeBrandForHost(domain)) {\n    inventory.refused = 'lookalike'\n    return finish()\n  }\n\n  // Asked once, before anything else, and held to the site's two origins.\n  const headersFor = await siteRequestHeaders(seams, domain, origins)\n  const maxPages = Math.max(0, Math.floor(options.maxPages))\n  const maxPageBytes = options.maxPageBytes ?? SITE_IMPORT_DEFAULTS.maxPageBytes\n  const maxTotalBytes = options.maxTotalBytes ?? SITE_IMPORT_DEFAULTS.maxTotalBytes\n  const deadlineAt = startedAt + (options.timeBudgetMs ?? SITE_IMPORT_DEFAULTS.timeBudgetMs)\n  const controller = new AbortController()\n  const onAbort = () => controller.abort()\n  options.signal?.addEventListener('abort', onAbort)\n  if (options.signal?.aborted) controller.abort()\n  const timer = setTimeout(() => controller.abort(), Math.max(0, deadlineAt - startedAt))\n\n  const sameSite = (url: URL) => origins.includes(url.origin) && url.protocol === 'https:'\n  const remainingBytes = () => maxTotalBytes - inventory.bytesRead\n  const outOfTime = () => controller.signal.aborted || now() >= deadlineAt\n  const request = (url: string, accept: RegExp, acceptHeader: string, maxBytes: number) =>\n    fetchText(url, {\n      maxBytes,\n      accept,\n      acceptHeader,\n      userAgent: AGLYN_BOT_USER_AGENT,\n      headersFor,\n      signal: controller.signal,\n      timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),\n      redirectAllowed: (_from, to) => sameSite(to),\n      refuseOver: true,\n    })\n  const skip = (entry: SiteImportSkipped) => {\n    if (inventory.skipped.length < SITE_IMPORT_DEFAULTS.maxSkipped) inventory.skipped.push(entry)\n    else inventory.skippedOverflow += 1\n  }\n\n  try {\n    // robots.txt, before anything else is asked of the site.\n    const home = `${origins[0]}/`\n    let rules: RobotsRule[] = []\n    const robots = await request(\n      `${origins[0]}/robots.txt`,\n      /text\\/plain|text\\/html|^$/i,\n      'text/plain',\n      Math.min(SITE_IMPORT_DEFAULTS.robotsMaxBytes, remainingBytes()),\n    )\n    if (robots.ok) {\n      inventory.bytesRead += robots.bytes\n      const parsed = parseRobotsTxt(robots.text)\n      rules = robotsRulesFor(parsed, AGLYN_BOT_TOKEN)\n      inventory.robots = { found: true, sitemaps: parsed.sitemaps }\n    } else if (\n      !(robots.refusal === 'http-status' && (robots.status ?? 0) >= 400 && (robots.status ?? 0) < 500)\n    ) {\n      inventory.stoppedBy = outOfTime() ? 'time' : 'robots-unreachable'\n      return finish()\n    }\n    const allowed = (key: string) => {\n      const url = new URL(key)\n      return robotsAllows(rules, `${url.pathname}${url.search}`)\n    }\n\n    // A wave's own pages (AGL-3976): exactly those, after robots.txt.\n    const only = options.only\n      ? options.only\n          .map((raw) => {\n            try {\n              return siteImportPageKey(new URL(raw), origins)\n            } catch {\n              return null\n            }\n          })\n          .filter((key): key is string => !!key)\n      : null\n\n    // Sitemaps: the ones robots.txt names on this site, else /sitemap.xml.\n    const sitemapQueue: string[] = []\n    for (const raw of inventory.robots.sitemaps) {\n      try {\n        const url = new URL(raw)\n        if (sameSite(url)) sitemapQueue.push(url.toString())\n      } catch {\n        // A malformed Sitemap line names nothing.\n      }\n    }\n    if (!sitemapQueue.length && !only) sitemapQueue.push(`${origins[0]}/sitemap.xml`)\n    if (only) sitemapQueue.length = 0\n    const sitemapSeen = new Set<string>()\n    const fromSitemap: string[] = []\n    let sitemapsRead = 0\n    while (sitemapQueue.length && sitemapsRead < SITE_IMPORT_DEFAULTS.maxSitemaps && !outOfTime()) {\n      const next = sitemapQueue.shift() as string\n      if (sitemapSeen.has(next)) continue\n      sitemapSeen.add(next)\n      if (/\\.gz$/i.test(new URL(next).pathname)) continue\n      if (!allowed(next)) continue\n      const room = Math.min(SITE_IMPORT_DEFAULTS.sitemapMaxBytes, remainingBytes())\n      if (room <= 0) break\n      sitemapsRead += 1\n      const sitemap = await request(next, /xml|text\\/plain/i, 'application/xml, text/xml;q=0.9', room)\n      if (!sitemap.ok) continue\n      inventory.bytesRead += sitemap.bytes\n      const entries = parseSitemap(sitemap.text)\n      for (const location of entries.locations) {\n        let url: URL\n        try {\n          url = new URL(location)\n        } catch {\n          continue\n        }\n        if (entries.index) {\n          if (sameSite(url)) sitemapQueue.push(url.toString())\n          continue\n        }\n        const key = siteImportPageKey(url, origins)\n        if (key && fromSitemap.length < SITE_IMPORT_DEFAULTS.maxSitemapUrls) fromSitemap.push(key)\n      }\n    }\n    inventory.sitemapUrls = fromSitemap.length\n    if (options.shallowFirst) {\n      const depth = (key: string) => new URL(key).pathname.split('/').filter(Boolean).length\n      const ranked = fromSitemap.map((key, order) => ({ key, order, depth: depth(key) }))\n      ranked.sort((a, b) => a.depth - b.depth || a.order - b.order)\n      fromSitemap.splice(0, fromSitemap.length, ...ranked.map((entry) => entry.key))\n    }\n\n    // Pages: home, then the sitemap's, then links the pages themselves name.\n    // A wave's read is its own pages alone.\n    const queue: { key: string; foundBy: SiteImportPage['foundBy'] }[] = only\n      ? only.map((key) => ({ key, foundBy: (new URL(key).pathname === '/' ? 'home' : 'link') as SiteImportPage['foundBy'] }))\n      : [{ key: home, foundBy: 'home' }]\n    const queued = new Set<string>(only ?? [home])\n    for (const key of fromSitemap) {\n      if (queued.has(key)) continue\n      queued.add(key)\n      queue.push({ key, foundBy: 'sitemap' })\n    }\n    const read = new Set<string>()\n    // The site's two names are one site: once the home page says which one\n    // it answers on, every address is read on that one, so /about is not\n    // read twice as example.com/about and www.example.com/about.\n    // A wave's read starts on the name its pages were read on the first time.\n    let preferred = only?.length ? new URL(only[0]).origin : origins[0]\n    const canon = (raw: string) => {\n      const url = new URL(raw)\n      return origins.includes(url.origin) && url.origin !== preferred\n        ? `${preferred}${url.pathname}${url.search}`\n        : raw\n    }\n    const maxAttempts = maxPages * 3 + 10\n    let attempts = 0\n    type Asked = { next: (typeof queue)[number]; key: string; cap: number; page: PublicFetchResult }\n    /** The pages asked for and not yet taken in, in the order they were asked. */\n    const window: Array<Promise<Asked>> = []\n    /** The bytes the asked pages may still read: what is left is shared, never promised twice. */\n    let reserved = 0\n    let stop: SiteImportStop | null = null\n    /** Asked for and never read, because a stop came first: listed as not reached. */\n    const cutOff: string[] = []\n    /** Home alone first, so its answer settles which of the site's two names the rest are read on. */\n    let homeIn = !!only\n    /** Why no more may be asked now: a stop, `wait` for an asked page to come in first, or nothing. */\n    const held = (): SiteImportStop | 'wait' | null => {\n      if (inventory.pages.length >= maxPages) return 'page-budget'\n      if (outOfTime()) return 'time'\n      if (remainingBytes() <= 0) return 'total-bytes'\n      if (attempts >= maxAttempts) return 'page-budget'\n      if (inventory.pages.length + window.length >= maxPages) return 'wait'\n      if (remainingBytes() - reserved <= 0) return 'wait'\n      return null\n    }\n    /** Takes one asked page in: the inventory's next page, a skip, or the reason the read must stop. */\n    const takeIn = ({ next, key, cap, page }: Asked): SiteImportStop | null => {\n      const foundBy = next.foundBy\n      if (foundBy === 'home') homeIn = true\n      if (!page.ok) {\n        if (page.refusal === 'too-large' && cap < maxPageBytes) {\n          cutOff.push(key)\n          return 'total-bytes'\n        }\n        if (page.refusal === 'network' && outOfTime()) {\n          cutOff.push(key)\n          return 'time'\n        }\n        skip({\n          url: key,\n          reason: skipReasonFor(page),\n          ...(page.status ? { status: page.status } : {}),\n          ...(page.location ? { location: page.location } : {}),\n          ...(page.edge ? { edge: page.edge } : {}),\n        })\n        return null\n      }\n      inventory.bytesRead += page.bytes\n      if (foundBy === 'home') preferred = new URL(page.url).origin\n      const finalKey = canon(siteImportPageKey(new URL(page.url), origins) ?? page.url)\n      if (finalKey !== key) {\n        if (read.has(finalKey)) return null\n        read.add(finalKey)\n        if (!allowed(finalKey)) {\n          skip({ url: finalKey, reason: 'robots' })\n          return null\n        }\n      }\n      const extract = extractSiteImportPage(page.text, page.url, sameSite)\n      if (extract.loginForm) {\n        skip({ url: finalKey, reason: 'login-form' })\n        return null\n      }\n      const links: string[] = []\n      for (const link of extract.links) {\n        const linkKey = siteImportPageKey(new URL(link), origins)\n        if (!linkKey) continue\n        links.push(linkKey)\n        if (!only && !queued.has(linkKey)) {\n          queued.add(linkKey)\n          queue.push({ key: linkKey, foundBy: 'link' })\n        }\n      }\n      inventory.pages.push({\n        url: finalKey,\n        path: new URL(finalKey).pathname,\n        foundBy,\n        title: extract.title,\n        description: extract.description,\n        lang: extract.lang,\n        canonical: extract.canonical,\n        ogImage: extract.ogImage,\n        siteName: extract.siteName,\n        icons: extract.icons,\n        brand: extract.brand,\n        organization: extract.organization,\n        blocks: extract.blocks,\n        images: extract.images,\n        navLinks: extract.navLinks,\n        links: [...new Set(links.map(canon))],\n        forms: extract.forms,\n        bytes: page.bytes,\n      })\n      return null\n    }\n    for (;;) {\n      const width = homeIn ? SITE_IMPORT_DEFAULTS.concurrency : 1\n      while (!stop && queue.length && window.length < width) {\n        const why = held()\n        if (why === 'wait') break\n        if (why) {\n          stop = why\n          break\n        }\n        const next = queue.shift() as (typeof queue)[number]\n        const key = canon(next.key)\n        if (read.has(key)) continue\n        if (!allowed(key)) {\n          skip({ url: key, reason: 'robots' })\n          if (next.foundBy === 'home') homeIn = true\n          continue\n        }\n        attempts += 1\n        read.add(key)\n        const cap = Math.min(maxPageBytes, remainingBytes() - reserved)\n        reserved += cap\n        window.push(\n          request(key, /text\\/html|application\\/xhtml\\+xml/i, 'text/html, application/xhtml+xml;q=0.9', cap).then((page) => ({ next, key, cap, page })),\n        )\n      }\n      if (!window.length) break\n      const asked = await (window.shift() as Promise<Asked>)\n      reserved -= asked.cap\n      stop = takeIn(asked) ?? stop\n    }\n    inventory.stoppedBy = stop\n    // Whatever a stop left unread is said, never dropped; a wave's own read\n    // leaves its unread pages for its next pass instead.\n    if (stop && !only) {\n      const unread = [...cutOff]\n      for (const next of queue) {\n        const key = canon(next.key)\n        if (!read.has(key) && !unread.includes(key)) unread.push(key)\n      }\n      for (const url of unread) skip({ url, reason: 'not-reached' })\n    }\n    return finish()\n  } finally {\n    clearTimeout(timer)\n    options.signal?.removeEventListener('abort', onAbort)\n  }\n}\n\n/**\n * The home page's `aglyn-site-verification` values, for the meta-tag proof:\n * the same fetch, user agent and same-site redirects as the crawl, but no\n * robots.txt, because the person asked for this one page to be read.\n * `answered: false` when the page could not be read at all.\n */\nexport async function readSiteVerificationMeta(\n  domain: string,\n  metaName: string,\n  seams: SiteImportCrawlSeams = {},\n): Promise<{ answered: boolean; values: string[]; url?: string }> {\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const bare = String(domain ?? '').trim().toLowerCase().replace(/^www\\./, '')\n  if (!DOMAIN_PATTERN.test(bare) || lookalikeBrandForHost(bare)) return { answered: false, values: [] }\n  const origins = siteImportOrigins(bare)\n  const result = await fetchText(`${origins[0]}/`, {\n    maxBytes: SITE_IMPORT_DEFAULTS.maxPageBytes,\n    accept: /text\\/html|application\\/xhtml\\+xml/i,\n    acceptHeader: 'text/html, application/xhtml+xml;q=0.9',\n    userAgent: AGLYN_BOT_USER_AGENT,\n    headersFor: await siteRequestHeaders(seams, bare, origins),\n    timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,\n    redirectAllowed: (_from, to) => to.protocol === 'https:' && origins.includes(to.origin),\n  })\n  if (!result.ok) {\n    // An answer that is not the page (a 404, a redirect off the site) is\n    // still an answer: the tag is not where it must be.\n    const answered = result.refusal === 'http-status' || result.refusal === 'redirect-refused' || result.refusal === 'content-type'\n    return { answered, values: [], url: result.url }\n  }\n  const head = result.text.split(/<body\\b/i)[0] ?? ''\n  return { answered: true, values: readMetaValues(head, metaName), url: result.url }\n}\n\n/**\n * One page as text for a prompt: its title, description and blocks in order,\n * fenced as untrusted data. The importer's prompt carries `UNTRUSTED_TEXT_RULE`.\n */\nexport function siteImportPagePromptText(page: SiteImportPage, maxChars = 12_000): string {\n  const lines: string[] = []\n  if (page.title) lines.push(`Title: ${page.title}`)\n  if (page.description) lines.push(`Description: ${page.description}`)\n  for (const block of page.blocks) {\n    if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n    else if (block.kind === 'image') lines.push(`[image: ${block.alt || 'no description'}]`)\n    else if (block.kind === 'list-item') lines.push(`- ${block.text}`)\n    else if (block.kind === 'quote') lines.push(`> ${block.text}`)\n    else if (block.kind === 'link') {\n      // A link inside a sentence is in its block's words already.\n      if (!block.inline) lines.push(block.text)\n    } else lines.push(block.text)\n  }\n  return fenceUntrustedText(`Page ${page.path} of the site being imported`, lines.join('\\n'), { maxChars })\n}\n"],"names":["process","lookalikeBrandForHost","fenceUntrustedText","fetchPublicTextDetailed","extractSiteImportPage","readMetaValues","AGLYN_BOT_TOKEN","parseRobotsTxt","parseSitemap","robotsAllows","robotsRulesFor","NO_SITE_IMPORT_HEADERS","siteImportEdgeHeadersFor","AGLYN_BOT_INFO_URL","env","AGLYN_BOT_USER_AGENT","SITE_IMPORT_DEFAULTS","maxPageBytes","maxTotalBytes","timeBudgetMs","requestTimeoutMs","robotsMaxBytes","sitemapMaxBytes","maxSitemaps","maxSitemapUrls","maxSkipped","concurrency","NON_PAGE_EXTENSION","TRACKING_PARAM","DOMAIN_PATTERN","siteRequestHeaders","seams","domain","origins","inner","headersFor","headers","Object","keys","length","error","console","warn","url","includes","origin","siteImportOrigins","bare","replace","siteImportPageKey","protocol","username","password","port","test","pathname","clean","URL","toString","hash","key","searchParams","delete","search","skipReasonFor","result","refusal","crawlSiteForImport","options","now","Date","fetchText","fetch","String","trim","toLowerCase","startedAt","inventory","refused","pages","skipped","skippedOverflow","robots","found","sitemaps","sitemapUrls","stoppedBy","bytesRead","finishedAt","finish","maxPages","Math","max","floor","deadlineAt","controller","AbortController","onAbort","abort","signal","addEventListener","aborted","timer","setTimeout","sameSite","remainingBytes","outOfTime","request","accept","acceptHeader","maxBytes","userAgent","timeoutMs","min","redirectAllowed","_from","to","refuseOver","skip","entry","push","home","rules","ok","bytes","parsed","text","status","allowed","only","map","raw","filter","sitemapQueue","sitemapSeen","Set","fromSitemap","sitemapsRead","next","shift","has","add","room","sitemap","entries","location","locations","index","shallowFirst","depth","split","Boolean","ranked","order","sort","a","b","splice","queue","foundBy","queued","read","preferred","canon","maxAttempts","attempts","window","reserved","stop","cutOff","homeIn","held","takeIn","cap","page","reason","edge","finalKey","extract","loginForm","links","link","linkKey","path","title","description","lang","canonical","ogImage","siteName","icons","brand","organization","blocks","images","navLinks","forms","width","why","then","asked","unread","clearTimeout","removeEventListener","readSiteVerificationMeta","metaName","answered","values","head","siteImportPagePromptText","maxChars","lines","block","kind","repeat","level","alt","inline","join"],"mappings":";IAyGEA;AAzGF;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2CA6C2C,GAE3C,SAASC,qBAAqB,QAAQ,oDAAmD;AACzF,SAASC,kBAAkB,QAAQ,0CAAyC;AAC5E,SACEC,uBAAuB,QAIlB,mCAA+B;AACtC,SACEC,qBAAqB,EACrBC,cAAc,QAQT,wBAAoB;AAC3B,SACEC,eAAe,EACfC,cAAc,EACdC,YAAY,EACZC,YAAY,EACZC,cAAc,QAET,0BAAsB;AAC7B,SAASC,sBAAsB,EAAEC,wBAAwB,QAAmC,wBAAoB;AAEhH;;;;;;;;;;CAUC,GACD,OAAO,MAAMC,sBACXb,wCAAAA,QAAQc,GAAG,CAAC,2BAA2B,YAAvCd,wCAA2C,0CAAyC;AAEtF,2CAA2C,GAC3C,OAAO,MAAMe,uBAAuB,CAAC,eAAe,EAAEF,mBAAmB,CAAC,CAAC,CAAA;AAE3E,OAAO,MAAMG,uBAAuB;IAClCC,cAAc,IAAI,OAAO;IACzB,iFAAiF,GACjFC,eAAe,MAAM,OAAO;IAC5BC,cAAc;IACd,uDAAuD,GACvDC,kBAAkB;IAClBC,gBAAgB,MAAM;IACtBC,iBAAiB,IAAI,OAAO;IAC5B,6FAA6F,GAC7FC,aAAa;IACb,gDAAgD,GAChDC,gBAAgB;IAChB,+FAA+F,GAC/FC,YAAY;IACZ,0EAA0E,GAC1EC,aAAa;AACf,EAAU;AAEV,+DAA+D,GAC/D,MAAMC,qBACJ;AAEF,MAAMC,iBAAiB;AAEvB,MAAMC,iBAAiB;AA0HvB;;;CAGC,GACD,eAAeC,mBACbC,KAA2B,EAC3BC,MAAc,EACdC,OAA0B;IAE1B,IAAIC,QAA8BvB;IAClC,IAAI;QACF,IAAIoB,MAAMI,UAAU,EAAE;YACpB,MAAMC,UAAU,MAAML,MAAMI,UAAU,CAACH;YACvC,IAAII,WAAWC,OAAOC,IAAI,CAACF,SAASG,MAAM,EAAEL,QAAQ,IAAME;QAC5D,OAAO;YACLF,QAAQ,MAAMtB,yBAAyBqB;QACzC;IACF,EAAE,OAAOO,OAAO;QACdC,QAAQC,IAAI,CAAC,0EAA0E;YAAEV;YAAQQ;QAAM;IACzG;IACA,OAAO,CAACG,MAAcV,QAAQW,QAAQ,CAACD,IAAIE,MAAM,IAAIX,MAAMS,OAAO,CAAC;AACrE;AAEA,yCAAyC,GACzC,OAAO,SAASG,kBAAkBd,MAAc;IAC9C,MAAMe,OAAOf,OAAOgB,OAAO,CAAC,UAAU;IACtC,OAAO;QAAC,CAAC,QAAQ,EAAED,MAAM;QAAE,CAAC,YAAY,EAAEA,MAAM;KAAC;AACnD;AAEA,oFAAoF,GACpF,OAAO,SAASE,kBAAkBN,GAAQ,EAAEV,OAA0B;IACpE,IAAIU,IAAIO,QAAQ,KAAK,YAAYP,IAAIQ,QAAQ,IAAIR,IAAIS,QAAQ,EAAE,OAAO;IACtE,IAAIT,IAAIU,IAAI,IAAIV,IAAIU,IAAI,KAAK,OAAO,OAAO;IAC3C,IAAI,CAACpB,QAAQW,QAAQ,CAACD,IAAIE,MAAM,GAAG,OAAO;IAC1C,IAAIlB,mBAAmB2B,IAAI,CAACX,IAAIY,QAAQ,GAAG,OAAO;IAClD,MAAMC,QAAQ,IAAIC,IAAId,IAAIe,QAAQ;IAClCF,MAAMG,IAAI,GAAG;IACb,KAAK,MAAMC,OAAO;WAAIJ,MAAMK,YAAY,CAACvB,IAAI;KAAG,CAAE;QAChD,IAAIV,eAAe0B,IAAI,CAACM,MAAMJ,MAAMK,YAAY,CAACC,MAAM,CAACF;IAC1D;IACAJ,MAAMO,MAAM,GAAGP,MAAMK,YAAY,CAACH,QAAQ,KAAK,CAAC,CAAC,EAAEF,MAAMK,YAAY,CAACH,QAAQ,IAAI,GAAG;IACrF,OAAOF,MAAME,QAAQ;AACvB;AAEA,SAASM,cAAcC,MAAyB;IAC9C,OAAQA,OAAOC,OAAO;QACpB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,mFAAmF,GACnF,OAAO,eAAeC,mBACpBC,OAA+B,EAC/BrC,QAA8B,CAAC,CAAC;QAEpBA,YACMA,cACIqC,iBAiCDA,uBACCA,wBACUA;QAGhCA,iBACIA;IAzCJ,MAAMC,OAAMtC,aAAAA,MAAMsC,GAAG,YAATtC,aAAauC,KAAKD,GAAG;IACjC,MAAME,aAAYxC,eAAAA,MAAMyC,KAAK,YAAXzC,eAAe5B;IACjC,MAAM6B,SAASyC,QAAOL,kBAAAA,QAAQpC,MAAM,YAAdoC,kBAAkB,IAAIM,IAAI,GAAGC,WAAW,GAAG3B,OAAO,CAAC,QAAQ,IAAIA,OAAO,CAAC,UAAU;IACvG,MAAMf,UAAUJ,eAAeyB,IAAI,CAACtB,UAAUc,kBAAkBd,UAAU,EAAE;IAC5E,MAAM4C,YAAYP;IAClB,MAAMQ,YAAiC;QACrC7C;QACAC;QACA6C,SAAS;QACTC,OAAO,EAAE;QACTC,SAAS,EAAE;QACXC,iBAAiB;QACjBC,QAAQ;YAAEC,OAAO;YAAOC,UAAU,EAAE;QAAC;QACrCC,aAAa;QACbC,WAAW;QACXC,WAAW;QACXX;QACAY,YAAYZ;IACd;IACA,MAAMa,SAAS;QACbZ,UAAUW,UAAU,GAAGnB;QACvB,OAAOQ;IACT;IACA,IAAI,CAAC5C,QAAQM,MAAM,EAAE;QACnBsC,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IACA,IAAIxF,sBAAsB+B,SAAS;QACjC6C,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IAEA,wEAAwE;IACxE,MAAMtD,aAAa,MAAML,mBAAmBC,OAAOC,QAAQC;IAC3D,MAAMyD,WAAWC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAACzB,QAAQsB,QAAQ;IACxD,MAAMzE,gBAAemD,wBAAAA,QAAQnD,YAAY,YAApBmD,wBAAwBpD,qBAAqBC,YAAY;IAC9E,MAAMC,iBAAgBkD,yBAAAA,QAAQlD,aAAa,YAArBkD,yBAAyBpD,qBAAqBE,aAAa;IACjF,MAAM4E,aAAalB,cAAaR,wBAAAA,QAAQjD,YAAY,YAApBiD,wBAAwBpD,qBAAqBG,YAAY;IACzF,MAAM4E,aAAa,IAAIC;IACvB,MAAMC,UAAU,IAAMF,WAAWG,KAAK;KACtC9B,kBAAAA,QAAQ+B,MAAM,qBAAd/B,gBAAgBgC,gBAAgB,CAAC,SAASH;IAC1C,KAAI7B,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBiC,OAAO,EAAEN,WAAWG,KAAK;IAC7C,MAAMI,QAAQC,WAAW,IAAMR,WAAWG,KAAK,IAAIP,KAAKC,GAAG,CAAC,GAAGE,aAAalB;IAE5E,MAAM4B,WAAW,CAAC7D,MAAaV,QAAQW,QAAQ,CAACD,IAAIE,MAAM,KAAKF,IAAIO,QAAQ,KAAK;IAChF,MAAMuD,iBAAiB,IAAMvF,gBAAgB2D,UAAUU,SAAS;IAChE,MAAMmB,YAAY,IAAMX,WAAWI,MAAM,CAACE,OAAO,IAAIhC,SAASyB;IAC9D,MAAMa,UAAU,CAAChE,KAAaiE,QAAgBC,cAAsBC,WAClEvC,UAAU5B,KAAK;YACbmE;YACAF;YACAC;YACAE,WAAWhG;YACXoB;YACAgE,QAAQJ,WAAWI,MAAM;YACzBa,WAAWrB,KAAKC,GAAG,CAAC,GAAGD,KAAKsB,GAAG,CAACjG,qBAAqBI,gBAAgB,EAAE0E,aAAazB;YACpF6C,iBAAiB,CAACC,OAAOC,KAAOZ,SAASY;YACzCC,YAAY;QACd;IACF,MAAMC,OAAO,CAACC;QACZ,IAAI1C,UAAUG,OAAO,CAACzC,MAAM,GAAGvB,qBAAqBS,UAAU,EAAEoD,UAAUG,OAAO,CAACwC,IAAI,CAACD;aAClF1C,UAAUI,eAAe,IAAI;IACpC;IAEA,IAAI;YAgBuCC,gBAA+BA;QAfxE,yDAAyD;QACzD,MAAMuC,OAAO,GAAGxF,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,IAAIyF,QAAsB,EAAE;QAC5B,MAAMxC,SAAS,MAAMyB,QACnB,GAAG1E,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAC1B,8BACA,cACA0D,KAAKsB,GAAG,CAACjG,qBAAqBK,cAAc,EAAEoF;QAEhD,IAAIvB,OAAOyC,EAAE,EAAE;YACb9C,UAAUU,SAAS,IAAIL,OAAO0C,KAAK;YACnC,MAAMC,SAAStH,eAAe2E,OAAO4C,IAAI;YACzCJ,QAAQhH,eAAemH,QAAQvH;YAC/BuE,UAAUK,MAAM,GAAG;gBAAEC,OAAO;gBAAMC,UAAUyC,OAAOzC,QAAQ;YAAC;QAC9D,OAAO,IACL,CAAEF,CAAAA,OAAOhB,OAAO,KAAK,iBAAiB,EAACgB,iBAAAA,OAAO6C,MAAM,YAAb7C,iBAAiB,MAAM,OAAO,EAACA,kBAAAA,OAAO6C,MAAM,YAAb7C,kBAAiB,KAAK,GAAE,GAC9F;YACAL,UAAUS,SAAS,GAAGoB,cAAc,SAAS;YAC7C,OAAOjB;QACT;QACA,MAAMuC,UAAU,CAACpE;YACf,MAAMjB,MAAM,IAAIc,IAAIG;YACpB,OAAOnD,aAAaiH,OAAO,GAAG/E,IAAIY,QAAQ,GAAGZ,IAAIoB,MAAM,EAAE;QAC3D;QAEA,kEAAkE;QAClE,MAAMkE,OAAO7D,QAAQ6D,IAAI,GACrB7D,QAAQ6D,IAAI,CACTC,GAAG,CAAC,CAACC;YACJ,IAAI;gBACF,OAAOlF,kBAAkB,IAAIQ,IAAI0E,MAAMlG;YACzC,EAAE,eAAM;gBACN,OAAO;YACT;QACF,GACCmG,MAAM,CAAC,CAACxE,MAAuB,CAAC,CAACA,OACpC;QAEJ,uEAAuE;QACvE,MAAMyE,eAAyB,EAAE;QACjC,KAAK,MAAMF,OAAOtD,UAAUK,MAAM,CAACE,QAAQ,CAAE;YAC3C,IAAI;gBACF,MAAMzC,MAAM,IAAIc,IAAI0E;gBACpB,IAAI3B,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIe,QAAQ;YACnD,EAAE,eAAM;YACN,0CAA0C;YAC5C;QACF;QACA,IAAI,CAAC2E,aAAa9F,MAAM,IAAI,CAAC0F,MAAMI,aAAab,IAAI,CAAC,GAAGvF,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC;QAChF,IAAIgG,MAAMI,aAAa9F,MAAM,GAAG;QAChC,MAAM+F,cAAc,IAAIC;QACxB,MAAMC,cAAwB,EAAE;QAChC,IAAIC,eAAe;QACnB,MAAOJ,aAAa9F,MAAM,IAAIkG,eAAezH,qBAAqBO,WAAW,IAAI,CAACmF,YAAa;YAC7F,MAAMgC,OAAOL,aAAaM,KAAK;YAC/B,IAAIL,YAAYM,GAAG,CAACF,OAAO;YAC3BJ,YAAYO,GAAG,CAACH;YAChB,IAAI,SAASpF,IAAI,CAAC,IAAIG,IAAIiF,MAAMnF,QAAQ,GAAG;YAC3C,IAAI,CAACyE,QAAQU,OAAO;YACpB,MAAMI,OAAOnD,KAAKsB,GAAG,CAACjG,qBAAqBM,eAAe,EAAEmF;YAC5D,IAAIqC,QAAQ,GAAG;YACfL,gBAAgB;YAChB,MAAMM,UAAU,MAAMpC,QAAQ+B,MAAM,oBAAoB,mCAAmCI;YAC3F,IAAI,CAACC,QAAQpB,EAAE,EAAE;YACjB9C,UAAUU,SAAS,IAAIwD,QAAQnB,KAAK;YACpC,MAAMoB,UAAUxI,aAAauI,QAAQjB,IAAI;YACzC,KAAK,MAAMmB,YAAYD,QAAQE,SAAS,CAAE;gBACxC,IAAIvG;gBACJ,IAAI;oBACFA,MAAM,IAAIc,IAAIwF;gBAChB,EAAE,eAAM;oBACN;gBACF;gBACA,IAAID,QAAQG,KAAK,EAAE;oBACjB,IAAI3C,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIe,QAAQ;oBACjD;gBACF;gBACA,MAAME,MAAMX,kBAAkBN,KAAKV;gBACnC,IAAI2B,OAAO4E,YAAYjG,MAAM,GAAGvB,qBAAqBQ,cAAc,EAAEgH,YAAYhB,IAAI,CAAC5D;YACxF;QACF;QACAiB,UAAUQ,WAAW,GAAGmD,YAAYjG,MAAM;QAC1C,IAAI6B,QAAQgF,YAAY,EAAE;YACxB,MAAMC,QAAQ,CAACzF,MAAgB,IAAIH,IAAIG,KAAKL,QAAQ,CAAC+F,KAAK,CAAC,KAAKlB,MAAM,CAACmB,SAAShH,MAAM;YACtF,MAAMiH,SAAShB,YAAYN,GAAG,CAAC,CAACtE,KAAK6F,QAAW,CAAA;oBAAE7F;oBAAK6F;oBAAOJ,OAAOA,MAAMzF;gBAAK,CAAA;YAChF4F,OAAOE,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEN,KAAK,GAAGO,EAAEP,KAAK,IAAIM,EAAEF,KAAK,GAAGG,EAAEH,KAAK;YAC5DjB,YAAYqB,MAAM,CAAC,GAAGrB,YAAYjG,MAAM,KAAKiH,OAAOtB,GAAG,CAAC,CAACX,QAAUA,MAAM3D,GAAG;QAC9E;QAEA,yEAAyE;QACzE,wCAAwC;QACxC,MAAMkG,QAA+D7B,OACjEA,KAAKC,GAAG,CAAC,CAACtE,MAAS,CAAA;gBAAEA;gBAAKmG,SAAU,IAAItG,IAAIG,KAAKL,QAAQ,KAAK,MAAM,SAAS;YAAqC,CAAA,KAClH;YAAC;gBAAEK,KAAK6D;gBAAMsC,SAAS;YAAO;SAAE;QACpC,MAAMC,SAAS,IAAIzB,IAAYN,eAAAA,OAAQ;YAACR;SAAK;QAC7C,KAAK,MAAM7D,OAAO4E,YAAa;YAC7B,IAAIwB,OAAOpB,GAAG,CAAChF,MAAM;YACrBoG,OAAOnB,GAAG,CAACjF;YACXkG,MAAMtC,IAAI,CAAC;gBAAE5D;gBAAKmG,SAAS;YAAU;QACvC;QACA,MAAME,OAAO,IAAI1B;QACjB,uEAAuE;QACvE,qEAAqE;QACrE,6DAA6D;QAC7D,0EAA0E;QAC1E,IAAI2B,YAAYjC,CAAAA,wBAAAA,KAAM1F,MAAM,IAAG,IAAIkB,IAAIwE,IAAI,CAAC,EAAE,EAAEpF,MAAM,GAAGZ,OAAO,CAAC,EAAE;QACnE,MAAMkI,QAAQ,CAAChC;YACb,MAAMxF,MAAM,IAAIc,IAAI0E;YACpB,OAAOlG,QAAQW,QAAQ,CAACD,IAAIE,MAAM,KAAKF,IAAIE,MAAM,KAAKqH,YAClD,GAAGA,YAAYvH,IAAIY,QAAQ,GAAGZ,IAAIoB,MAAM,EAAE,GAC1CoE;QACN;QACA,MAAMiC,cAAc1E,WAAW,IAAI;QACnC,IAAI2E,WAAW;QAEf,4EAA4E,GAC5E,MAAMC,SAAgC,EAAE;QACxC,4FAA4F,GAC5F,IAAIC,WAAW;QACf,IAAIC,OAA8B;QAClC,gFAAgF,GAChF,MAAMC,SAAmB,EAAE;QAC3B,gGAAgG,GAChG,IAAIC,SAAS,CAAC,CAACzC;QACf,iGAAiG,GACjG,MAAM0C,OAAO;YACX,IAAI9F,UAAUE,KAAK,CAACxC,MAAM,IAAImD,UAAU,OAAO;YAC/C,IAAIgB,aAAa,OAAO;YACxB,IAAID,oBAAoB,GAAG,OAAO;YAClC,IAAI4D,YAAYD,aAAa,OAAO;YACpC,IAAIvF,UAAUE,KAAK,CAACxC,MAAM,GAAG+H,OAAO/H,MAAM,IAAImD,UAAU,OAAO;YAC/D,IAAIe,mBAAmB8D,YAAY,GAAG,OAAO;YAC7C,OAAO;QACT;QACA,kGAAkG,GAClG,MAAMK,SAAS,CAAC,EAAElC,IAAI,EAAE9E,GAAG,EAAEiH,GAAG,EAAEC,IAAI,EAAS;gBAuBtB7H;YAtBvB,MAAM8G,UAAUrB,KAAKqB,OAAO;YAC5B,IAAIA,YAAY,QAAQW,SAAS;YACjC,IAAI,CAACI,KAAKnD,EAAE,EAAE;gBACZ,IAAImD,KAAK5G,OAAO,KAAK,eAAe2G,MAAM5J,cAAc;oBACtDwJ,OAAOjD,IAAI,CAAC5D;oBACZ,OAAO;gBACT;gBACA,IAAIkH,KAAK5G,OAAO,KAAK,aAAawC,aAAa;oBAC7C+D,OAAOjD,IAAI,CAAC5D;oBACZ,OAAO;gBACT;gBACA0D,KAAK;oBACH3E,KAAKiB;oBACLmH,QAAQ/G,cAAc8G;mBAClBA,KAAK/C,MAAM,GAAG;oBAAEA,QAAQ+C,KAAK/C,MAAM;gBAAC,IAAI,CAAC,GACzC+C,KAAK7B,QAAQ,GAAG;oBAAEA,UAAU6B,KAAK7B,QAAQ;gBAAC,IAAI,CAAC,GAC/C6B,KAAKE,IAAI,GAAG;oBAAEA,MAAMF,KAAKE,IAAI;gBAAC,IAAI,CAAC;gBAEzC,OAAO;YACT;YACAnG,UAAUU,SAAS,IAAIuF,KAAKlD,KAAK;YACjC,IAAImC,YAAY,QAAQG,YAAY,IAAIzG,IAAIqH,KAAKnI,GAAG,EAAEE,MAAM;YAC5D,MAAMoI,WAAWd,OAAMlH,qBAAAA,kBAAkB,IAAIQ,IAAIqH,KAAKnI,GAAG,GAAGV,oBAArCgB,qBAAiD6H,KAAKnI,GAAG;YAChF,IAAIsI,aAAarH,KAAK;gBACpB,IAAIqG,KAAKrB,GAAG,CAACqC,WAAW,OAAO;gBAC/BhB,KAAKpB,GAAG,CAACoC;gBACT,IAAI,CAACjD,QAAQiD,WAAW;oBACtB3D,KAAK;wBAAE3E,KAAKsI;wBAAUF,QAAQ;oBAAS;oBACvC,OAAO;gBACT;YACF;YACA,MAAMG,UAAU9K,sBAAsB0K,KAAKhD,IAAI,EAAEgD,KAAKnI,GAAG,EAAE6D;YAC3D,IAAI0E,QAAQC,SAAS,EAAE;gBACrB7D,KAAK;oBAAE3E,KAAKsI;oBAAUF,QAAQ;gBAAa;gBAC3C,OAAO;YACT;YACA,MAAMK,QAAkB,EAAE;YAC1B,KAAK,MAAMC,QAAQH,QAAQE,KAAK,CAAE;gBAChC,MAAME,UAAUrI,kBAAkB,IAAIQ,IAAI4H,OAAOpJ;gBACjD,IAAI,CAACqJ,SAAS;gBACdF,MAAM5D,IAAI,CAAC8D;gBACX,IAAI,CAACrD,QAAQ,CAAC+B,OAAOpB,GAAG,CAAC0C,UAAU;oBACjCtB,OAAOnB,GAAG,CAACyC;oBACXxB,MAAMtC,IAAI,CAAC;wBAAE5D,KAAK0H;wBAASvB,SAAS;oBAAO;gBAC7C;YACF;YACAlF,UAAUE,KAAK,CAACyC,IAAI,CAAC;gBACnB7E,KAAKsI;gBACLM,MAAM,IAAI9H,IAAIwH,UAAU1H,QAAQ;gBAChCwG;gBACAyB,OAAON,QAAQM,KAAK;gBACpBC,aAAaP,QAAQO,WAAW;gBAChCC,MAAMR,QAAQQ,IAAI;gBAClBC,WAAWT,QAAQS,SAAS;gBAC5BC,SAASV,QAAQU,OAAO;gBACxBC,UAAUX,QAAQW,QAAQ;gBAC1BC,OAAOZ,QAAQY,KAAK;gBACpBC,OAAOb,QAAQa,KAAK;gBACpBC,cAAcd,QAAQc,YAAY;gBAClCC,QAAQf,QAAQe,MAAM;gBACtBC,QAAQhB,QAAQgB,MAAM;gBACtBC,UAAUjB,QAAQiB,QAAQ;gBAC1Bf,OAAO;uBAAI,IAAI7C,IAAI6C,MAAMlD,GAAG,CAACiC;iBAAQ;gBACrCiC,OAAOlB,QAAQkB,KAAK;gBACpBxE,OAAOkD,KAAKlD,KAAK;YACnB;YACA,OAAO;QACT;QACA,OAAS;gBA4BAgD;YA3BP,MAAMyB,QAAQ3B,SAAS1J,qBAAqBU,WAAW,GAAG;YAC1D,MAAO,CAAC8I,QAAQV,MAAMvH,MAAM,IAAI+H,OAAO/H,MAAM,GAAG8J,MAAO;gBACrD,MAAMC,MAAM3B;gBACZ,IAAI2B,QAAQ,QAAQ;gBACpB,IAAIA,KAAK;oBACP9B,OAAO8B;oBACP;gBACF;gBACA,MAAM5D,OAAOoB,MAAMnB,KAAK;gBACxB,MAAM/E,MAAMuG,MAAMzB,KAAK9E,GAAG;gBAC1B,IAAIqG,KAAKrB,GAAG,CAAChF,MAAM;gBACnB,IAAI,CAACoE,QAAQpE,MAAM;oBACjB0D,KAAK;wBAAE3E,KAAKiB;wBAAKmH,QAAQ;oBAAS;oBAClC,IAAIrC,KAAKqB,OAAO,KAAK,QAAQW,SAAS;oBACtC;gBACF;gBACAL,YAAY;gBACZJ,KAAKpB,GAAG,CAACjF;gBACT,MAAMiH,MAAMlF,KAAKsB,GAAG,CAAChG,cAAcwF,mBAAmB8D;gBACtDA,YAAYM;gBACZP,OAAO9C,IAAI,CACTb,QAAQ/C,KAAK,uCAAuC,0CAA0CiH,KAAK0B,IAAI,CAAC,CAACzB,OAAU,CAAA;wBAAEpC;wBAAM9E;wBAAKiH;wBAAKC;oBAAK,CAAA;YAE9I;YACA,IAAI,CAACR,OAAO/H,MAAM,EAAE;YACpB,MAAMiK,QAAQ,MAAOlC,OAAO3B,KAAK;YACjC4B,YAAYiC,MAAM3B,GAAG;YACrBL,QAAOI,UAAAA,OAAO4B,kBAAP5B,UAAiBJ;QAC1B;QACA3F,UAAUS,SAAS,GAAGkF;QACtB,wEAAwE;QACxE,qDAAqD;QACrD,IAAIA,QAAQ,CAACvC,MAAM;YACjB,MAAMwE,SAAS;mBAAIhC;aAAO;YAC1B,KAAK,MAAM/B,QAAQoB,MAAO;gBACxB,MAAMlG,MAAMuG,MAAMzB,KAAK9E,GAAG;gBAC1B,IAAI,CAACqG,KAAKrB,GAAG,CAAChF,QAAQ,CAAC6I,OAAO7J,QAAQ,CAACgB,MAAM6I,OAAOjF,IAAI,CAAC5D;YAC3D;YACA,KAAK,MAAMjB,OAAO8J,OAAQnF,KAAK;gBAAE3E;gBAAKoI,QAAQ;YAAc;QAC9D;QACA,OAAOtF;IACT,SAAU;YAERrB;QADAsI,aAAapG;SACblC,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBuI,mBAAmB,CAAC,SAAS1G;IAC/C;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAe2G,yBACpB5K,MAAc,EACd6K,QAAgB,EAChB9K,QAA8B,CAAC,CAAC;QAEdA,cAmBLkC;IAnBb,MAAMM,aAAYxC,eAAAA,MAAMyC,KAAK,YAAXzC,eAAe5B;IACjC,MAAM4C,OAAO0B,OAAOzC,iBAAAA,SAAU,IAAI0C,IAAI,GAAGC,WAAW,GAAG3B,OAAO,CAAC,UAAU;IACzE,IAAI,CAACnB,eAAeyB,IAAI,CAACP,SAAS9C,sBAAsB8C,OAAO,OAAO;QAAE+J,UAAU;QAAOC,QAAQ,EAAE;IAAC;IACpG,MAAM9K,UAAUa,kBAAkBC;IAClC,MAAMkB,SAAS,MAAMM,UAAU,GAAGtC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;QAC/C6E,UAAU9F,qBAAqBC,YAAY;QAC3C2F,QAAQ;QACRC,cAAc;QACdE,WAAWhG;QACXoB,YAAY,MAAML,mBAAmBC,OAAOgB,MAAMd;QAClD+E,WAAWhG,qBAAqBI,gBAAgB;QAChD8F,iBAAiB,CAACC,OAAOC,KAAOA,GAAGlE,QAAQ,KAAK,YAAYjB,QAAQW,QAAQ,CAACwE,GAAGvE,MAAM;IACxF;IACA,IAAI,CAACoB,OAAO0D,EAAE,EAAE;QACd,qEAAqE;QACrE,oDAAoD;QACpD,MAAMmF,WAAW7I,OAAOC,OAAO,KAAK,iBAAiBD,OAAOC,OAAO,KAAK,sBAAsBD,OAAOC,OAAO,KAAK;QACjH,OAAO;YAAE4I;YAAUC,QAAQ,EAAE;YAAEpK,KAAKsB,OAAOtB,GAAG;QAAC;IACjD;IACA,MAAMqK,QAAO/I,sBAAAA,OAAO6D,IAAI,CAACwB,KAAK,CAAC,WAAW,CAAC,EAAE,YAAhCrF,sBAAoC;IACjD,OAAO;QAAE6I,UAAU;QAAMC,QAAQ1M,eAAe2M,MAAMH;QAAWlK,KAAKsB,OAAOtB,GAAG;IAAC;AACnF;AAEA;;;CAGC,GACD,OAAO,SAASsK,yBAAyBnC,IAAoB,EAAEoC,WAAW,KAAM;IAC9E,MAAMC,QAAkB,EAAE;IAC1B,IAAIrC,KAAKU,KAAK,EAAE2B,MAAM3F,IAAI,CAAC,CAAC,OAAO,EAAEsD,KAAKU,KAAK,EAAE;IACjD,IAAIV,KAAKW,WAAW,EAAE0B,MAAM3F,IAAI,CAAC,CAAC,aAAa,EAAEsD,KAAKW,WAAW,EAAE;IACnE,KAAK,MAAM2B,SAAStC,KAAKmB,MAAM,CAAE;QAC/B,IAAImB,MAAMC,IAAI,KAAK,WAAWF,MAAM3F,IAAI,CAAC,GAAG,IAAI8F,MAAM,CAACF,MAAMG,KAAK,EAAE,CAAC,EAAEH,MAAMtF,IAAI,EAAE;aAC9E,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,QAAQ,EAAE4F,MAAMI,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAClF,IAAIJ,MAAMC,IAAI,KAAK,aAAaF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aAC5D,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aACxD,IAAIsF,MAAMC,IAAI,KAAK,QAAQ;YAC9B,4DAA4D;YAC5D,IAAI,CAACD,MAAMK,MAAM,EAAEN,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;QAC1C,OAAOqF,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;IAC9B;IACA,OAAO5H,mBAAmB,CAAC,KAAK,EAAE4K,KAAKS,IAAI,CAAC,2BAA2B,CAAC,EAAE4B,MAAMO,IAAI,CAAC,OAAO;QAAER;IAAS;AACzG"}
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-crawl.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * THE SITE-IMPORT CRAWLER (AGL-3974, parent AGL-3973).\n *\n * Reads a website a workspace has PROVED it owns (domain-ownership.ts in\n * tenant-data-admin; the doors check the proof before calling this) into a\n * typed inventory: each page's title, description, headings and text in\n * order, images and menu links. It writes nothing; the import job decides\n * what becomes a page.\n *\n * Every request goes through `fetchPublicTextDetailed`, the theme job's\n * SSRF-guarded fetch: https on the default port only, public addresses only,\n * the socket pinned to the address checked. On top of that:\n *\n * - **One site.** The site is `https://<domain>` and `https://www.<domain>`,\n *   one site to its owner. Every redirect hop and every discovered link must\n *   stay on it; a page that redirects anywhere else is refused, not followed.\n * - **AglynBot.** Every request says `AglynBot/1.0 (+<the operator's bot page>)`,\n *   and robots.txt is read first and honored for every page: the AglynBot\n *   group, else `*`. A robots.txt that answers 404 allows everything; one\n *   that cannot be read at all (5xx, no answer) stops the crawl, because\n *   \"could not ask\" is not \"allowed\".\n * - **Our own edge.** A site this platform serves gets the edge secret\n *   (`site-import-edge.ts`, AGL-4085), which lifts our bot challenge for\n *   AglynBot; it is asked per request and per redirect hop, and a request\n *   that leaves the site's two origins never carries it.\n * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an\n *   index's children included), then same-site links found on the pages read,\n *   for whatever the sitemap missed.\n * - **Budgets.** Pages (the caller's, from the plan), bytes per page (a page\n *   over it is refused, not cut), bytes in total, and wall-clock time. A\n *   whole site's worth fits (AGL-4083): aglyn.com's 123 pages average 440 KB,\n *   54 MB in all, so the total is 160 MB; a page is parsed and dropped as it\n *   is read, so what is held is its blocks, never its bytes.\n * - **Four at a time.** robots.txt, the sitemaps and the home page are read\n *   one after another; the rest four at a time, and the inventory keeps the\n *   order they were asked in, which is the sitemap's.\n * - **Nothing vanishes.** An address a stop (the clock, the bytes, the page\n *   budget) left unread is listed as `not-reached`, so the inventory's\n *   counts add up to the site. A wave's own read leaves its unread pages for\n *   its next pass instead.\n * - **No lookalikes.** A domain that wears a brand it is not\n *   (`lookalikeBrandForHost`) is refused before any request.\n * - **No sign-in pages.** A page with a password field is refused and\n *   reported; forms with payment fields, scripts, iframes and embeds never\n *   reach the inventory (site-import-html.ts).\n *=========================================*/\n\nimport { lookalikeBrandForHost } from '@aglyn/shared-util-email/outbound-phishing-screen'\nimport { fenceUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n  AI_THEME_REFERENCE_STYLESHEETS,\n  AI_THEME_STYLESHEET_MAX_BYTES,\n  fetchPublicTextDetailed,\n  type PublicFetchEdge,\n  type PublicFetchOptions,\n  type PublicFetchResult,\n} from '../jobs/ai-theme-brand-inputs'\nimport type { SiteImportFooter, SiteImportLayout, SiteImportLookFacts, SiteImportNavTree } from '../model/ai-site-import'\nimport {\n  extractSiteImportPage,\n  readMetaValues,\n  type SiteImportBlock,\n  type SiteImportBrandMark,\n  type SiteImportForm,\n  type SiteImportIcon,\n  type SiteImportImage,\n  type SiteImportNavLink,\n  type SiteImportOrganization,\n} from './site-import-html'\nimport {\n  AGLYN_BOT_TOKEN,\n  parseRobotsTxt,\n  parseSitemap,\n  robotsAllows,\n  robotsRulesFor,\n  type RobotsRule,\n} from './site-import-robots'\nimport { NO_SITE_IMPORT_HEADERS, siteImportEdgeHeadersFor, type SiteImportHeadersFor } from './site-import-edge'\nimport { siteImportLayout, siteImportLookOf } from './site-import-layout'\nimport { siteImportFooterColumns, siteImportNavTree } from './site-import-nav'\n\n/**\n * Where the bot's user agent points a site owner who wants to know what it is\n * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on\n * a self-hosted deployment, so its crawler never names Aglyn's address; the\n * literal is only that variable's default.\n *\n * Aglyn's is the Bring my site guide, which says what AglynBot reads and how\n * robots.txt keeps it out. There is no page of its own (Zach, 10/20: \"we don't\n * need a bot page at all\"), and the site it reads is the reader's own: an\n * import starts only once its owner has proved the site is theirs.\n */\nexport const AGLYN_BOT_INFO_URL =\n  process.env['SITE_IMPORT_BOT_INFO_URL'] ?? 'https://docs.aglyn.com/ai/bring-my-site'\n\n/** What every crawler request says it is. */\nexport const AGLYN_BOT_USER_AGENT = `AglynBot/1.0 (+${AGLYN_BOT_INFO_URL})`\n\nexport const SITE_IMPORT_DEFAULTS = {\n  maxPageBytes: 2 * 1024 * 1024,\n  /**\n   * A whole site's worth (AGL-4083): 54 MB for aglyn.com's 123 pages, with\n   * room; 400 pages at its 440 KB average are 176 MB (AGL-4106).\n   */\n  maxTotalBytes: 320 * 1024 * 1024,\n  timeBudgetMs: 60_000,\n  /** One request's own clock, inside the whole crawl's. */\n  requestTimeoutMs: 8_000,\n  robotsMaxBytes: 256 * 1024,\n  sitemapMaxBytes: 4 * 1024 * 1024,\n  /** Sitemap documents read, an index's children included: an index of six children is seven. */\n  maxSitemaps: 24,\n  /** URLs a sitemap may contribute to the queue. */\n  maxSitemapUrls: 2_000,\n  /** Reasons kept in `skipped` before the list stops growing: every address a sitemap may name. */\n  maxSkipped: 2_000,\n  /** Pages read at once, after robots.txt, the sitemaps and the home page. */\n  concurrency: 4,\n} as const\n\n/** Never a page: files a link may point at that are not HTML. */\nconst NON_PAGE_EXTENSION =\n  /\\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i\n\nconst TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i\n\n/**\n * Never a page, whatever its extension (AGL-4106): an API's or a CDN's path\n * (aglyn.com's press kit links 136 `/api/media/cdn/…?download=1` files) and\n * any address that asks for a download.\n */\nconst NON_PAGE_PATH = /^\\/(?:api|cdn-cgi|wp-json|_next\\/static|_next\\/image)(?:\\/|$)/i\nconst DOWNLOAD_PARAM = /^(?:download|dl|attachment)$/i\n\nconst DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\\.[a-z0-9-]{1,63})+$/\n\nexport type SiteImportSkipReason =\n  /** The page has a password field: a sign-in page, refused outright. */\n  | 'login-form'\n  /** robots.txt keeps AglynBot off this path. */\n  | 'robots'\n  /** A redirect left the site. */\n  | 'off-origin-redirect'\n  /** Over the per-page byte cap. */\n  | 'too-large'\n  /** Answered with something that is not HTML. */\n  | 'not-html'\n  /** Answered 4xx or 5xx. */\n  | 'http-error'\n  /** No answer, a private address, or a refused address. */\n  | 'unreachable'\n  /** Never asked for: a stop (time, bytes, the page budget) came first (AGL-4083). */\n  | 'not-reached'\n\nexport interface SiteImportSkipped {\n  url: string\n  reason: SiteImportSkipReason\n  status?: number\n  /** Where an off-origin redirect pointed. */\n  location?: string\n  /** Whose edge turned an error answer away, when it said (AGL-4083). */\n  edge?: PublicFetchEdge\n}\n\nexport interface SiteImportPage {\n  /** The address the page was read from, after any same-site redirect. */\n  url: string\n  path: string\n  /** Where the crawler learned of the page. */\n  foundBy: 'home' | 'sitemap' | 'link'\n  title: string\n  description: string\n  lang: string | null\n  canonical: string | null\n  /** The share card (AGL-4083). */\n  ogImage: string | null\n  /** The site's own name, as its head gives it. */\n  siteName: string\n  icons: SiteImportIcon[]\n  /** The header's link home: the brand's logo and words. */\n  brand: SiteImportBrandMark | null\n  organization: SiteImportOrganization | null\n  blocks: SiteImportBlock[]\n  images: SiteImportImage[]\n  navLinks: SiteImportNavLink[]\n  /** Same-site page addresses this page links to. */\n  links: string[]\n  forms: { kept: SiteImportForm[]; removed: number }\n  bytes: number\n  /** How the source lays the page out (AGL-4106), on a read asked for it (`layout`): a wave's. */\n  layout?: SiteImportLayout | null\n  /** The home page's menu, footer and look, as its source builds them (AGL-4106). */\n  nav?: SiteImportNavTree | null\n  footer?: SiteImportFooter | null\n  look?: SiteImportLookFacts | null\n}\n\nexport type SiteImportStop =\n  | 'page-budget'\n  | 'total-bytes'\n  | 'time'\n  /** robots.txt could not be read, so nothing was asked of the site. */\n  | 'robots-unreachable'\n\nexport interface SiteImportInventory {\n  domain: string\n  /** The site's origins: the bare name and its `www.` twin. */\n  origins: string[]\n  /** Set when nothing was fetched at all. */\n  refused: 'invalid-domain' | 'lookalike' | null\n  pages: SiteImportPage[]\n  skipped: SiteImportSkipped[]\n  /** Skips past {@link SITE_IMPORT_DEFAULTS.maxSkipped}, counted only. */\n  skippedOverflow: number\n  robots: { found: boolean; sitemaps: string[] }\n  /** Page addresses the sitemaps listed (same-site, before robots and budget). */\n  sitemapUrls: number\n  stoppedBy: SiteImportStop | null\n  bytesRead: number\n  startedAt: number\n  finishedAt: number\n}\n\nexport interface SiteImportCrawlOptions {\n  /** The proved domain; `www.` is folded in. */\n  domain: string\n  /** The most pages the inventory may hold: the plan's page count. */\n  maxPages: number\n  maxPageBytes?: number\n  maxTotalBytes?: number\n  timeBudgetMs?: number\n  signal?: AbortSignal\n  /**\n   * Read the sitemap's shallow addresses before its deep ones (AGL-3976), so a\n   * crawl the page budget stops reads a site's main pages before the tail of\n   * its blog. A sitemap's own order is kept among addresses of one depth.\n   */\n  shallowFirst?: boolean\n  /**\n   * Read exactly these addresses of the site, in order, and nothing else\n   * (AGL-3976): an import's wave reads its own pages again when it starts. No\n   * sitemap is read and no link is followed; robots.txt is read and honored\n   * as for any crawl, and every other rule holds.\n   */\n  only?: readonly string[]\n  /**\n   * Read how each page is laid out (AGL-4106): its structure, through its own\n   * stylesheets (the first two it links, each asked once a crawl). On by\n   * default for a wave's read (`only`), whose pages are built; the home page's\n   * menu, footer and look are read on every crawl.\n   */\n  layout?: boolean\n}\n\nexport interface SiteImportCrawlSeams {\n  fetch?: (url: string, options: PublicFetchOptions) => Promise<PublicFetchResult>\n  now?: () => number\n  /**\n   * Request headers for the site being read, asked once per crawl with its\n   * domain (AGL-4083, AGL-4085). Absent, the crawl asks the edge\n   * (`siteImportEdgeHeadersFor`): Aglyn's edge secret for each of the site's\n   * two origins this platform serves, decided origin by origin, and nothing\n   * anywhere else. Either way the headers go only to the site's two origins,\n   * and the fetch asks again on every redirect hop, so a hop anywhere else\n   * carries nothing. Always under the crawler's own `Accept` and `User-Agent`.\n   */\n  headersFor?: (domain: string) => Promise<Readonly<Record<string, string>>> | Readonly<Record<string, string>>\n}\n\n/**\n * The per-request headers of a crawl of `domain`: the seam's, else the edge's,\n * and none for any address off the site's two origins. Never throws.\n */\nasync function siteRequestHeaders(\n  seams: SiteImportCrawlSeams,\n  domain: string,\n  origins: readonly string[],\n): Promise<SiteImportHeadersFor> {\n  let inner: SiteImportHeadersFor = NO_SITE_IMPORT_HEADERS\n  try {\n    if (seams.headersFor) {\n      const headers = await seams.headersFor(domain)\n      if (headers && Object.keys(headers).length) inner = () => headers\n    } else {\n      inner = await siteImportEdgeHeadersFor(origins)\n    }\n  } catch (error) {\n    console.warn('[site-import] the edge headers could not be read; reading without them', { domain, error })\n  }\n  return (url: URL) => (origins.includes(url.origin) ? inner(url) : {})\n}\n\n/** The site's two origins for a domain. */\nexport function siteImportOrigins(domain: string): string[] {\n  const bare = domain.replace(/^www\\./, '')\n  return [`https://${bare}`, `https://www.${bare}`]\n}\n\n/** A crawlable page address on the site, normalized for de-duplication; else null. */\nexport function siteImportPageKey(url: URL, origins: readonly string[]): string | null {\n  if (url.protocol !== 'https:' || url.username || url.password) return null\n  if (url.port && url.port !== '443') return null\n  if (!origins.includes(url.origin)) return null\n  if (NON_PAGE_EXTENSION.test(url.pathname) || NON_PAGE_PATH.test(url.pathname)) return null\n  if ([...url.searchParams.keys()].some((key) => DOWNLOAD_PARAM.test(key))) return null\n  const clean = new URL(url.toString())\n  clean.hash = ''\n  for (const key of [...clean.searchParams.keys()]) {\n    if (TRACKING_PARAM.test(key)) clean.searchParams.delete(key)\n  }\n  clean.search = clean.searchParams.toString() ? `?${clean.searchParams.toString()}` : ''\n  return clean.toString()\n}\n\nfunction skipReasonFor(result: PublicFetchResult): SiteImportSkipReason {\n  switch (result.refusal) {\n    case 'redirect-refused':\n      return 'off-origin-redirect'\n    case 'content-type':\n      return 'not-html'\n    case 'too-large':\n      return 'too-large'\n    case 'http-status':\n      return 'http-error'\n    default:\n      return 'unreachable'\n  }\n}\n\n/** Read a proved site into an inventory. Never throws for anything the site does. */\nexport async function crawlSiteForImport(\n  options: SiteImportCrawlOptions,\n  seams: SiteImportCrawlSeams = {},\n): Promise<SiteImportInventory> {\n  const now = seams.now ?? Date.now\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const domain = String(options.domain ?? '').trim().toLowerCase().replace(/\\.+$/, '').replace(/^www\\./, '')\n  const origins = DOMAIN_PATTERN.test(domain) ? siteImportOrigins(domain) : []\n  const startedAt = now()\n  const inventory: SiteImportInventory = {\n    domain,\n    origins,\n    refused: null,\n    pages: [],\n    skipped: [],\n    skippedOverflow: 0,\n    robots: { found: false, sitemaps: [] },\n    sitemapUrls: 0,\n    stoppedBy: null,\n    bytesRead: 0,\n    startedAt,\n    finishedAt: startedAt,\n  }\n  const finish = () => {\n    inventory.finishedAt = now()\n    return inventory\n  }\n  if (!origins.length) {\n    inventory.refused = 'invalid-domain'\n    return finish()\n  }\n  if (lookalikeBrandForHost(domain)) {\n    inventory.refused = 'lookalike'\n    return finish()\n  }\n\n  // Asked once, before anything else, and held to the site's two origins.\n  const headersFor = await siteRequestHeaders(seams, domain, origins)\n  const maxPages = Math.max(0, Math.floor(options.maxPages))\n  const maxPageBytes = options.maxPageBytes ?? SITE_IMPORT_DEFAULTS.maxPageBytes\n  const maxTotalBytes = options.maxTotalBytes ?? SITE_IMPORT_DEFAULTS.maxTotalBytes\n  const deadlineAt = startedAt + (options.timeBudgetMs ?? SITE_IMPORT_DEFAULTS.timeBudgetMs)\n  const controller = new AbortController()\n  const onAbort = () => controller.abort()\n  options.signal?.addEventListener('abort', onAbort)\n  if (options.signal?.aborted) controller.abort()\n  const timer = setTimeout(() => controller.abort(), Math.max(0, deadlineAt - startedAt))\n\n  const sameSite = (url: URL) => origins.includes(url.origin) && url.protocol === 'https:'\n  const remainingBytes = () => maxTotalBytes - inventory.bytesRead\n  const outOfTime = () => controller.signal.aborted || now() >= deadlineAt\n  const request = (url: string, accept: RegExp, acceptHeader: string, maxBytes: number) =>\n    fetchText(url, {\n      maxBytes,\n      accept,\n      acceptHeader,\n      userAgent: AGLYN_BOT_USER_AGENT,\n      headersFor,\n      signal: controller.signal,\n      timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),\n      redirectAllowed: (_from, to) => sameSite(to),\n      refuseOver: true,\n    })\n  const skip = (entry: SiteImportSkipped) => {\n    if (inventory.skipped.length < SITE_IMPORT_DEFAULTS.maxSkipped) inventory.skipped.push(entry)\n    else inventory.skippedOverflow += 1\n  }\n\n  try {\n    // robots.txt, before anything else is asked of the site.\n    const home = `${origins[0]}/`\n    let rules: RobotsRule[] = []\n    const robots = await request(\n      `${origins[0]}/robots.txt`,\n      /text\\/plain|text\\/html|^$/i,\n      'text/plain',\n      Math.min(SITE_IMPORT_DEFAULTS.robotsMaxBytes, remainingBytes()),\n    )\n    if (robots.ok) {\n      inventory.bytesRead += robots.bytes\n      const parsed = parseRobotsTxt(robots.text)\n      rules = robotsRulesFor(parsed, AGLYN_BOT_TOKEN)\n      inventory.robots = { found: true, sitemaps: parsed.sitemaps }\n    } else if (\n      !(robots.refusal === 'http-status' && (robots.status ?? 0) >= 400 && (robots.status ?? 0) < 500)\n    ) {\n      inventory.stoppedBy = outOfTime() ? 'time' : 'robots-unreachable'\n      return finish()\n    }\n    const allowed = (key: string) => {\n      const url = new URL(key)\n      return robotsAllows(rules, `${url.pathname}${url.search}`)\n    }\n\n    // A wave's own pages (AGL-3976): exactly those, after robots.txt.\n    const only = options.only\n      ? options.only\n          .map((raw) => {\n            try {\n              return siteImportPageKey(new URL(raw), origins)\n            } catch {\n              return null\n            }\n          })\n          .filter((key): key is string => !!key)\n      : null\n\n    // Sitemaps: the ones robots.txt names on this site, else /sitemap.xml.\n    const sitemapQueue: string[] = []\n    for (const raw of inventory.robots.sitemaps) {\n      try {\n        const url = new URL(raw)\n        if (sameSite(url)) sitemapQueue.push(url.toString())\n      } catch {\n        // A malformed Sitemap line names nothing.\n      }\n    }\n    if (!sitemapQueue.length && !only) sitemapQueue.push(`${origins[0]}/sitemap.xml`)\n    if (only) sitemapQueue.length = 0\n    const sitemapSeen = new Set<string>()\n    const fromSitemap: string[] = []\n    let sitemapsRead = 0\n    while (sitemapQueue.length && sitemapsRead < SITE_IMPORT_DEFAULTS.maxSitemaps && !outOfTime()) {\n      const next = sitemapQueue.shift() as string\n      if (sitemapSeen.has(next)) continue\n      sitemapSeen.add(next)\n      if (/\\.gz$/i.test(new URL(next).pathname)) continue\n      if (!allowed(next)) continue\n      const room = Math.min(SITE_IMPORT_DEFAULTS.sitemapMaxBytes, remainingBytes())\n      if (room <= 0) break\n      sitemapsRead += 1\n      const sitemap = await request(next, /xml|text\\/plain/i, 'application/xml, text/xml;q=0.9', room)\n      if (!sitemap.ok) continue\n      inventory.bytesRead += sitemap.bytes\n      const entries = parseSitemap(sitemap.text)\n      for (const location of entries.locations) {\n        let url: URL\n        try {\n          url = new URL(location)\n        } catch {\n          continue\n        }\n        if (entries.index) {\n          if (sameSite(url)) sitemapQueue.push(url.toString())\n          continue\n        }\n        const key = siteImportPageKey(url, origins)\n        if (key && fromSitemap.length < SITE_IMPORT_DEFAULTS.maxSitemapUrls) fromSitemap.push(key)\n      }\n    }\n    inventory.sitemapUrls = fromSitemap.length\n    if (options.shallowFirst) {\n      const depth = (key: string) => new URL(key).pathname.split('/').filter(Boolean).length\n      const ranked = fromSitemap.map((key, order) => ({ key, order, depth: depth(key) }))\n      ranked.sort((a, b) => a.depth - b.depth || a.order - b.order)\n      fromSitemap.splice(0, fromSitemap.length, ...ranked.map((entry) => entry.key))\n    }\n\n    // Pages: home, then the sitemap's, then links the pages themselves name.\n    // A wave's read is its own pages alone.\n    const queue: { key: string; foundBy: SiteImportPage['foundBy'] }[] = only\n      ? only.map((key) => ({ key, foundBy: (new URL(key).pathname === '/' ? 'home' : 'link') as SiteImportPage['foundBy'] }))\n      : [{ key: home, foundBy: 'home' }]\n    const queued = new Set<string>(only ?? [home])\n    for (const key of fromSitemap) {\n      if (queued.has(key)) continue\n      queued.add(key)\n      queue.push({ key, foundBy: 'sitemap' })\n    }\n    const read = new Set<string>()\n    // The site's two names are one site: once the home page says which one\n    // it answers on, every address is read on that one, so /about is not\n    // read twice as example.com/about and www.example.com/about.\n    // A wave's read starts on the name its pages were read on the first time.\n    let preferred = only?.length ? new URL(only[0]).origin : origins[0]\n    const canon = (raw: string) => {\n      const url = new URL(raw)\n      return origins.includes(url.origin) && url.origin !== preferred\n        ? `${preferred}${url.pathname}${url.search}`\n        : raw\n    }\n    const maxAttempts = maxPages * 3 + 10\n    let attempts = 0\n    type Asked = { next: (typeof queue)[number]; key: string; cap: number; page: PublicFetchResult }\n    /** The pages asked for and not yet taken in, in the order they were asked. */\n    const window: Array<Promise<Asked>> = []\n    /** The bytes the asked pages may still read: what is left is shared, never promised twice. */\n    let reserved = 0\n    let stop: SiteImportStop | null = null\n    /** Asked for and never read, because a stop came first: listed as not reached. */\n    const cutOff: string[] = []\n    /** Home alone first, so its answer settles which of the site's two names the rest are read on. */\n    let homeIn = !!only\n    /** Why no more may be asked now: a stop, `wait` for an asked page to come in first, or nothing. */\n    const held = (): SiteImportStop | 'wait' | null => {\n      if (inventory.pages.length >= maxPages) return 'page-budget'\n      if (outOfTime()) return 'time'\n      if (remainingBytes() <= 0) return 'total-bytes'\n      if (attempts >= maxAttempts) return 'page-budget'\n      if (inventory.pages.length + window.length >= maxPages) return 'wait'\n      if (remainingBytes() - reserved <= 0) return 'wait'\n      return null\n    }\n    const readLayout = options.layout ?? !!only\n    /** Linked stylesheets, each asked once a crawl. */\n    const sheets = new Map<string, Promise<string | null>>()\n    const stylesheet = (url: string): Promise<string | null> => {\n      const held = sheets.get(url)\n      if (held) return held\n      const room = Math.min(AI_THEME_STYLESHEET_MAX_BYTES, remainingBytes() - reserved)\n      const asked: Promise<string | null> =\n        room <= 0 || outOfTime()\n          ? Promise.resolve(null)\n          : request(url, /text\\/css|text\\/plain/i, 'text/css,*/*;q=0.1', room).then(\n              (answer) => {\n                if (!answer.ok) return null\n                inventory.bytesRead += answer.bytes\n                return answer.text\n              },\n              () => null,\n            )\n      sheets.set(url, asked)\n      return asked\n    }\n    /** Takes one asked page in: the inventory's next page, a skip, or the reason the read must stop. */\n    const takeIn = async ({ next, key, cap, page }: Asked): Promise<SiteImportStop | null> => {\n      const foundBy = next.foundBy\n      if (foundBy === 'home') homeIn = true\n      if (!page.ok) {\n        if (page.refusal === 'too-large' && cap < maxPageBytes) {\n          cutOff.push(key)\n          return 'total-bytes'\n        }\n        if (page.refusal === 'network' && outOfTime()) {\n          cutOff.push(key)\n          return 'time'\n        }\n        // A file is not a page: it takes nothing from the page budget's attempts (AGL-4106).\n        if (page.refusal === 'content-type') attempts = Math.max(0, attempts - 1)\n        skip({\n          url: key,\n          reason: skipReasonFor(page),\n          ...(page.status ? { status: page.status } : {}),\n          ...(page.location ? { location: page.location } : {}),\n          ...(page.edge ? { edge: page.edge } : {}),\n        })\n        return null\n      }\n      inventory.bytesRead += page.bytes\n      if (foundBy === 'home') preferred = new URL(page.url).origin\n      const finalKey = canon(siteImportPageKey(new URL(page.url), origins) ?? page.url)\n      if (finalKey !== key) {\n        if (read.has(finalKey)) return null\n        read.add(finalKey)\n        if (!allowed(finalKey)) {\n          skip({ url: finalKey, reason: 'robots' })\n          return null\n        }\n      }\n      const isHome = foundBy === 'home'\n      const structured = readLayout || isHome\n      const extract = extractSiteImportPage(page.text, page.url, sameSite, { structure: structured })\n      if (extract.loginForm) {\n        skip({ url: finalKey, reason: 'login-form' })\n        return null\n      }\n      const links: string[] = []\n      for (const link of extract.links) {\n        const linkKey = siteImportPageKey(new URL(link), origins)\n        if (!linkKey) continue\n        links.push(linkKey)\n        if (!only && !queued.has(linkKey)) {\n          queued.add(linkKey)\n          queue.push({ key: linkKey, foundBy: 'link' })\n        }\n      }\n      // How the page is built (AGL-4106), through its own stylesheets.\n      let built: Pick<SiteImportPage, 'layout' | 'nav' | 'footer' | 'look'> = {}\n      if (extract.structure) {\n        const linked = await Promise.all(extract.structure.stylesheets.filter((url) => !/fonts\\.googleapis\\.com/i.test(url)).slice(0, AI_THEME_REFERENCE_STYLESHEETS).map(stylesheet))\n        const css = linked.filter((text): text is string => typeof text === 'string')\n        const source = { blocks: extract.blocks, organization: extract.organization, structure: extract.structure }\n        try {\n          built = {\n            ...(readLayout ? { layout: siteImportLayout(source, css) } : {}),\n            ...(isHome ? { nav: siteImportNavTree(source, css), footer: siteImportFooterColumns(source, css), look: siteImportLookOf(source, css) } : {}),\n          }\n        } catch (error) {\n          // A page the reader cannot lay out is still its words.\n          console.warn('[site-import] the layout could not be read', { url: page.url, error })\n        }\n      }\n      inventory.pages.push({\n        ...built,\n        url: finalKey,\n        path: new URL(finalKey).pathname,\n        foundBy,\n        title: extract.title,\n        description: extract.description,\n        lang: extract.lang,\n        canonical: extract.canonical,\n        ogImage: extract.ogImage,\n        siteName: extract.siteName,\n        icons: extract.icons,\n        brand: extract.brand,\n        organization: extract.organization,\n        blocks: extract.blocks,\n        images: extract.images,\n        navLinks: extract.navLinks,\n        links: [...new Set(links.map(canon))],\n        forms: extract.forms,\n        bytes: page.bytes,\n      })\n      return null\n    }\n    for (;;) {\n      const width = homeIn ? SITE_IMPORT_DEFAULTS.concurrency : 1\n      while (!stop && queue.length && window.length < width) {\n        const why = held()\n        if (why === 'wait') break\n        if (why) {\n          stop = why\n          break\n        }\n        const next = queue.shift() as (typeof queue)[number]\n        const key = canon(next.key)\n        if (read.has(key)) continue\n        if (!allowed(key)) {\n          skip({ url: key, reason: 'robots' })\n          if (next.foundBy === 'home') homeIn = true\n          continue\n        }\n        attempts += 1\n        read.add(key)\n        const cap = Math.min(maxPageBytes, remainingBytes() - reserved)\n        reserved += cap\n        window.push(\n          request(key, /text\\/html|application\\/xhtml\\+xml/i, 'text/html, application/xhtml+xml;q=0.9', cap).then((page) => ({ next, key, cap, page })),\n        )\n      }\n      if (!window.length) break\n      const asked = await (window.shift() as Promise<Asked>)\n      reserved -= asked.cap\n      stop = (await takeIn(asked)) ?? stop\n    }\n    inventory.stoppedBy = stop\n    // Whatever a stop left unread is said, never dropped; a wave's own read\n    // leaves its unread pages for its next pass instead.\n    if (stop && !only) {\n      const unread = [...cutOff]\n      for (const next of queue) {\n        const key = canon(next.key)\n        if (!read.has(key) && !unread.includes(key)) unread.push(key)\n      }\n      for (const url of unread) skip({ url, reason: 'not-reached' })\n    }\n    return finish()\n  } finally {\n    clearTimeout(timer)\n    options.signal?.removeEventListener('abort', onAbort)\n  }\n}\n\n/**\n * The home page's `aglyn-site-verification` values, for the meta-tag proof:\n * the same fetch, user agent and same-site redirects as the crawl, but no\n * robots.txt, because the person asked for this one page to be read.\n * `answered: false` when the page could not be read at all.\n */\nexport async function readSiteVerificationMeta(\n  domain: string,\n  metaName: string,\n  seams: SiteImportCrawlSeams = {},\n): Promise<{ answered: boolean; values: string[]; url?: string }> {\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const bare = String(domain ?? '').trim().toLowerCase().replace(/^www\\./, '')\n  if (!DOMAIN_PATTERN.test(bare) || lookalikeBrandForHost(bare)) return { answered: false, values: [] }\n  const origins = siteImportOrigins(bare)\n  const result = await fetchText(`${origins[0]}/`, {\n    maxBytes: SITE_IMPORT_DEFAULTS.maxPageBytes,\n    accept: /text\\/html|application\\/xhtml\\+xml/i,\n    acceptHeader: 'text/html, application/xhtml+xml;q=0.9',\n    userAgent: AGLYN_BOT_USER_AGENT,\n    headersFor: await siteRequestHeaders(seams, bare, origins),\n    timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,\n    redirectAllowed: (_from, to) => to.protocol === 'https:' && origins.includes(to.origin),\n  })\n  if (!result.ok) {\n    // An answer that is not the page (a 404, a redirect off the site) is\n    // still an answer: the tag is not where it must be.\n    const answered = result.refusal === 'http-status' || result.refusal === 'redirect-refused' || result.refusal === 'content-type'\n    return { answered, values: [], url: result.url }\n  }\n  const head = result.text.split(/<body\\b/i)[0] ?? ''\n  return { answered: true, values: readMetaValues(head, metaName), url: result.url }\n}\n\n/**\n * One page as text for a prompt: its title, description and blocks in order,\n * fenced as untrusted data. The importer's prompt carries `UNTRUSTED_TEXT_RULE`.\n */\nexport function siteImportPagePromptText(page: SiteImportPage, maxChars = 12_000): string {\n  const lines: string[] = []\n  if (page.title) lines.push(`Title: ${page.title}`)\n  if (page.description) lines.push(`Description: ${page.description}`)\n  for (const block of page.blocks) {\n    if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n    else if (block.kind === 'image') lines.push(`[image: ${block.alt || 'no description'}]`)\n    else if (block.kind === 'list-item') lines.push(`- ${block.text}`)\n    else if (block.kind === 'quote') lines.push(`> ${block.text}`)\n    else if (block.kind === 'link') {\n      // A link inside a sentence is in its block's words already.\n      if (!block.inline) lines.push(block.text)\n    } else lines.push(block.text)\n  }\n  return fenceUntrustedText(`Page ${page.path} of the site being imported`, lines.join('\\n'), { maxChars })\n}\n"],"names":["process","lookalikeBrandForHost","fenceUntrustedText","AI_THEME_REFERENCE_STYLESHEETS","AI_THEME_STYLESHEET_MAX_BYTES","fetchPublicTextDetailed","extractSiteImportPage","readMetaValues","AGLYN_BOT_TOKEN","parseRobotsTxt","parseSitemap","robotsAllows","robotsRulesFor","NO_SITE_IMPORT_HEADERS","siteImportEdgeHeadersFor","siteImportLayout","siteImportLookOf","siteImportFooterColumns","siteImportNavTree","AGLYN_BOT_INFO_URL","env","AGLYN_BOT_USER_AGENT","SITE_IMPORT_DEFAULTS","maxPageBytes","maxTotalBytes","timeBudgetMs","requestTimeoutMs","robotsMaxBytes","sitemapMaxBytes","maxSitemaps","maxSitemapUrls","maxSkipped","concurrency","NON_PAGE_EXTENSION","TRACKING_PARAM","NON_PAGE_PATH","DOWNLOAD_PARAM","DOMAIN_PATTERN","siteRequestHeaders","seams","domain","origins","inner","headersFor","headers","Object","keys","length","error","console","warn","url","includes","origin","siteImportOrigins","bare","replace","siteImportPageKey","protocol","username","password","port","test","pathname","searchParams","some","key","clean","URL","toString","hash","delete","search","skipReasonFor","result","refusal","crawlSiteForImport","options","now","Date","fetchText","fetch","String","trim","toLowerCase","startedAt","inventory","refused","pages","skipped","skippedOverflow","robots","found","sitemaps","sitemapUrls","stoppedBy","bytesRead","finishedAt","finish","maxPages","Math","max","floor","deadlineAt","controller","AbortController","onAbort","abort","signal","addEventListener","aborted","timer","setTimeout","sameSite","remainingBytes","outOfTime","request","accept","acceptHeader","maxBytes","userAgent","timeoutMs","min","redirectAllowed","_from","to","refuseOver","skip","entry","push","home","rules","ok","bytes","parsed","text","status","allowed","only","map","raw","filter","sitemapQueue","sitemapSeen","Set","fromSitemap","sitemapsRead","next","shift","has","add","room","sitemap","entries","location","locations","index","shallowFirst","depth","split","Boolean","ranked","order","sort","a","b","splice","queue","foundBy","queued","read","preferred","canon","maxAttempts","attempts","window","reserved","stop","cutOff","homeIn","held","readLayout","layout","sheets","Map","stylesheet","get","asked","Promise","resolve","then","answer","set","takeIn","cap","page","reason","edge","finalKey","isHome","structured","extract","structure","loginForm","links","link","linkKey","built","linked","all","stylesheets","slice","css","source","blocks","organization","nav","footer","look","path","title","description","lang","canonical","ogImage","siteName","icons","brand","images","navLinks","forms","width","why","unread","clearTimeout","removeEventListener","readSiteVerificationMeta","metaName","answered","values","head","siteImportPagePromptText","maxChars","lines","block","kind","repeat","level","alt","inline","join"],"mappings":";IA8GEA;AA9GF;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2CA6C2C,GAE3C,SAASC,qBAAqB,QAAQ,oDAAmD;AACzF,SAASC,kBAAkB,QAAQ,0CAAyC;AAC5E,SACEC,8BAA8B,EAC9BC,6BAA6B,EAC7BC,uBAAuB,QAIlB,mCAA+B;AAEtC,SACEC,qBAAqB,EACrBC,cAAc,QAQT,wBAAoB;AAC3B,SACEC,eAAe,EACfC,cAAc,EACdC,YAAY,EACZC,YAAY,EACZC,cAAc,QAET,0BAAsB;AAC7B,SAASC,sBAAsB,EAAEC,wBAAwB,QAAmC,wBAAoB;AAChH,SAASC,gBAAgB,EAAEC,gBAAgB,QAAQ,0BAAsB;AACzE,SAASC,uBAAuB,EAAEC,iBAAiB,QAAQ,uBAAmB;AAE9E;;;;;;;;;;CAUC,GACD,OAAO,MAAMC,sBACXnB,wCAAAA,QAAQoB,GAAG,CAAC,2BAA2B,YAAvCpB,wCAA2C,0CAAyC;AAEtF,2CAA2C,GAC3C,OAAO,MAAMqB,uBAAuB,CAAC,eAAe,EAAEF,mBAAmB,CAAC,CAAC,CAAA;AAE3E,OAAO,MAAMG,uBAAuB;IAClCC,cAAc,IAAI,OAAO;IACzB;;;GAGC,GACDC,eAAe,MAAM,OAAO;IAC5BC,cAAc;IACd,uDAAuD,GACvDC,kBAAkB;IAClBC,gBAAgB,MAAM;IACtBC,iBAAiB,IAAI,OAAO;IAC5B,6FAA6F,GAC7FC,aAAa;IACb,gDAAgD,GAChDC,gBAAgB;IAChB,+FAA+F,GAC/FC,YAAY;IACZ,0EAA0E,GAC1EC,aAAa;AACf,EAAU;AAEV,+DAA+D,GAC/D,MAAMC,qBACJ;AAEF,MAAMC,iBAAiB;AAEvB;;;;CAIC,GACD,MAAMC,gBAAgB;AACtB,MAAMC,iBAAiB;AAEvB,MAAMC,iBAAiB;AAuIvB;;;CAGC,GACD,eAAeC,mBACbC,KAA2B,EAC3BC,MAAc,EACdC,OAA0B;IAE1B,IAAIC,QAA8B7B;IAClC,IAAI;QACF,IAAI0B,MAAMI,UAAU,EAAE;YACpB,MAAMC,UAAU,MAAML,MAAMI,UAAU,CAACH;YACvC,IAAII,WAAWC,OAAOC,IAAI,CAACF,SAASG,MAAM,EAAEL,QAAQ,IAAME;QAC5D,OAAO;YACLF,QAAQ,MAAM5B,yBAAyB2B;QACzC;IACF,EAAE,OAAOO,OAAO;QACdC,QAAQC,IAAI,CAAC,0EAA0E;YAAEV;YAAQQ;QAAM;IACzG;IACA,OAAO,CAACG,MAAcV,QAAQW,QAAQ,CAACD,IAAIE,MAAM,IAAIX,MAAMS,OAAO,CAAC;AACrE;AAEA,yCAAyC,GACzC,OAAO,SAASG,kBAAkBd,MAAc;IAC9C,MAAMe,OAAOf,OAAOgB,OAAO,CAAC,UAAU;IACtC,OAAO;QAAC,CAAC,QAAQ,EAAED,MAAM;QAAE,CAAC,YAAY,EAAEA,MAAM;KAAC;AACnD;AAEA,oFAAoF,GACpF,OAAO,SAASE,kBAAkBN,GAAQ,EAAEV,OAA0B;IACpE,IAAIU,IAAIO,QAAQ,KAAK,YAAYP,IAAIQ,QAAQ,IAAIR,IAAIS,QAAQ,EAAE,OAAO;IACtE,IAAIT,IAAIU,IAAI,IAAIV,IAAIU,IAAI,KAAK,OAAO,OAAO;IAC3C,IAAI,CAACpB,QAAQW,QAAQ,CAACD,IAAIE,MAAM,GAAG,OAAO;IAC1C,IAAIpB,mBAAmB6B,IAAI,CAACX,IAAIY,QAAQ,KAAK5B,cAAc2B,IAAI,CAACX,IAAIY,QAAQ,GAAG,OAAO;IACtF,IAAI;WAAIZ,IAAIa,YAAY,CAAClB,IAAI;KAAG,CAACmB,IAAI,CAAC,CAACC,MAAQ9B,eAAe0B,IAAI,CAACI,OAAO,OAAO;IACjF,MAAMC,QAAQ,IAAIC,IAAIjB,IAAIkB,QAAQ;IAClCF,MAAMG,IAAI,GAAG;IACb,KAAK,MAAMJ,OAAO;WAAIC,MAAMH,YAAY,CAAClB,IAAI;KAAG,CAAE;QAChD,IAAIZ,eAAe4B,IAAI,CAACI,MAAMC,MAAMH,YAAY,CAACO,MAAM,CAACL;IAC1D;IACAC,MAAMK,MAAM,GAAGL,MAAMH,YAAY,CAACK,QAAQ,KAAK,CAAC,CAAC,EAAEF,MAAMH,YAAY,CAACK,QAAQ,IAAI,GAAG;IACrF,OAAOF,MAAME,QAAQ;AACvB;AAEA,SAASI,cAAcC,MAAyB;IAC9C,OAAQA,OAAOC,OAAO;QACpB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,mFAAmF,GACnF,OAAO,eAAeC,mBACpBC,OAA+B,EAC/BtC,QAA8B,CAAC,CAAC;QAEpBA,YACMA,cACIsC,iBAiCDA,uBACCA,wBACUA;QAGhCA,iBACIA;IAzCJ,MAAMC,OAAMvC,aAAAA,MAAMuC,GAAG,YAATvC,aAAawC,KAAKD,GAAG;IACjC,MAAME,aAAYzC,eAAAA,MAAM0C,KAAK,YAAX1C,eAAelC;IACjC,MAAMmC,SAAS0C,QAAOL,kBAAAA,QAAQrC,MAAM,YAAdqC,kBAAkB,IAAIM,IAAI,GAAGC,WAAW,GAAG5B,OAAO,CAAC,QAAQ,IAAIA,OAAO,CAAC,UAAU;IACvG,MAAMf,UAAUJ,eAAeyB,IAAI,CAACtB,UAAUc,kBAAkBd,UAAU,EAAE;IAC5E,MAAM6C,YAAYP;IAClB,MAAMQ,YAAiC;QACrC9C;QACAC;QACA8C,SAAS;QACTC,OAAO,EAAE;QACTC,SAAS,EAAE;QACXC,iBAAiB;QACjBC,QAAQ;YAAEC,OAAO;YAAOC,UAAU,EAAE;QAAC;QACrCC,aAAa;QACbC,WAAW;QACXC,WAAW;QACXX;QACAY,YAAYZ;IACd;IACA,MAAMa,SAAS;QACbZ,UAAUW,UAAU,GAAGnB;QACvB,OAAOQ;IACT;IACA,IAAI,CAAC7C,QAAQM,MAAM,EAAE;QACnBuC,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IACA,IAAIjG,sBAAsBuC,SAAS;QACjC8C,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IAEA,wEAAwE;IACxE,MAAMvD,aAAa,MAAML,mBAAmBC,OAAOC,QAAQC;IAC3D,MAAM0D,WAAWC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAACzB,QAAQsB,QAAQ;IACxD,MAAM5E,gBAAesD,wBAAAA,QAAQtD,YAAY,YAApBsD,wBAAwBvD,qBAAqBC,YAAY;IAC9E,MAAMC,iBAAgBqD,yBAAAA,QAAQrD,aAAa,YAArBqD,yBAAyBvD,qBAAqBE,aAAa;IACjF,MAAM+E,aAAalB,cAAaR,wBAAAA,QAAQpD,YAAY,YAApBoD,wBAAwBvD,qBAAqBG,YAAY;IACzF,MAAM+E,aAAa,IAAIC;IACvB,MAAMC,UAAU,IAAMF,WAAWG,KAAK;KACtC9B,kBAAAA,QAAQ+B,MAAM,qBAAd/B,gBAAgBgC,gBAAgB,CAAC,SAASH;IAC1C,KAAI7B,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBiC,OAAO,EAAEN,WAAWG,KAAK;IAC7C,MAAMI,QAAQC,WAAW,IAAMR,WAAWG,KAAK,IAAIP,KAAKC,GAAG,CAAC,GAAGE,aAAalB;IAE5E,MAAM4B,WAAW,CAAC9D,MAAaV,QAAQW,QAAQ,CAACD,IAAIE,MAAM,KAAKF,IAAIO,QAAQ,KAAK;IAChF,MAAMwD,iBAAiB,IAAM1F,gBAAgB8D,UAAUU,SAAS;IAChE,MAAMmB,YAAY,IAAMX,WAAWI,MAAM,CAACE,OAAO,IAAIhC,SAASyB;IAC9D,MAAMa,UAAU,CAACjE,KAAakE,QAAgBC,cAAsBC,WAClEvC,UAAU7B,KAAK;YACboE;YACAF;YACAC;YACAE,WAAWnG;YACXsB;YACAiE,QAAQJ,WAAWI,MAAM;YACzBa,WAAWrB,KAAKC,GAAG,CAAC,GAAGD,KAAKsB,GAAG,CAACpG,qBAAqBI,gBAAgB,EAAE6E,aAAazB;YACpF6C,iBAAiB,CAACC,OAAOC,KAAOZ,SAASY;YACzCC,YAAY;QACd;IACF,MAAMC,OAAO,CAACC;QACZ,IAAI1C,UAAUG,OAAO,CAAC1C,MAAM,GAAGzB,qBAAqBS,UAAU,EAAEuD,UAAUG,OAAO,CAACwC,IAAI,CAACD;aAClF1C,UAAUI,eAAe,IAAI;IACpC;IAEA,IAAI;YAgBuCC,gBAA+BA,iBAuHrDd;QAtInB,yDAAyD;QACzD,MAAMqD,OAAO,GAAGzF,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,IAAI0F,QAAsB,EAAE;QAC5B,MAAMxC,SAAS,MAAMyB,QACnB,GAAG3E,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAC1B,8BACA,cACA2D,KAAKsB,GAAG,CAACpG,qBAAqBK,cAAc,EAAEuF;QAEhD,IAAIvB,OAAOyC,EAAE,EAAE;YACb9C,UAAUU,SAAS,IAAIL,OAAO0C,KAAK;YACnC,MAAMC,SAAS7H,eAAekF,OAAO4C,IAAI;YACzCJ,QAAQvH,eAAe0H,QAAQ9H;YAC/B8E,UAAUK,MAAM,GAAG;gBAAEC,OAAO;gBAAMC,UAAUyC,OAAOzC,QAAQ;YAAC;QAC9D,OAAO,IACL,CAAEF,CAAAA,OAAOhB,OAAO,KAAK,iBAAiB,EAACgB,iBAAAA,OAAO6C,MAAM,YAAb7C,iBAAiB,MAAM,OAAO,EAACA,kBAAAA,OAAO6C,MAAM,YAAb7C,kBAAiB,KAAK,GAAE,GAC9F;YACAL,UAAUS,SAAS,GAAGoB,cAAc,SAAS;YAC7C,OAAOjB;QACT;QACA,MAAMuC,UAAU,CAACvE;YACf,MAAMf,MAAM,IAAIiB,IAAIF;YACpB,OAAOvD,aAAawH,OAAO,GAAGhF,IAAIY,QAAQ,GAAGZ,IAAIqB,MAAM,EAAE;QAC3D;QAEA,kEAAkE;QAClE,MAAMkE,OAAO7D,QAAQ6D,IAAI,GACrB7D,QAAQ6D,IAAI,CACTC,GAAG,CAAC,CAACC;YACJ,IAAI;gBACF,OAAOnF,kBAAkB,IAAIW,IAAIwE,MAAMnG;YACzC,EAAE,eAAM;gBACN,OAAO;YACT;QACF,GACCoG,MAAM,CAAC,CAAC3E,MAAuB,CAAC,CAACA,OACpC;QAEJ,uEAAuE;QACvE,MAAM4E,eAAyB,EAAE;QACjC,KAAK,MAAMF,OAAOtD,UAAUK,MAAM,CAACE,QAAQ,CAAE;YAC3C,IAAI;gBACF,MAAM1C,MAAM,IAAIiB,IAAIwE;gBACpB,IAAI3B,SAAS9D,MAAM2F,aAAab,IAAI,CAAC9E,IAAIkB,QAAQ;YACnD,EAAE,eAAM;YACN,0CAA0C;YAC5C;QACF;QACA,IAAI,CAACyE,aAAa/F,MAAM,IAAI,CAAC2F,MAAMI,aAAab,IAAI,CAAC,GAAGxF,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC;QAChF,IAAIiG,MAAMI,aAAa/F,MAAM,GAAG;QAChC,MAAMgG,cAAc,IAAIC;QACxB,MAAMC,cAAwB,EAAE;QAChC,IAAIC,eAAe;QACnB,MAAOJ,aAAa/F,MAAM,IAAImG,eAAe5H,qBAAqBO,WAAW,IAAI,CAACsF,YAAa;YAC7F,MAAMgC,OAAOL,aAAaM,KAAK;YAC/B,IAAIL,YAAYM,GAAG,CAACF,OAAO;YAC3BJ,YAAYO,GAAG,CAACH;YAChB,IAAI,SAASrF,IAAI,CAAC,IAAIM,IAAI+E,MAAMpF,QAAQ,GAAG;YAC3C,IAAI,CAAC0E,QAAQU,OAAO;YACpB,MAAMI,OAAOnD,KAAKsB,GAAG,CAACpG,qBAAqBM,eAAe,EAAEsF;YAC5D,IAAIqC,QAAQ,GAAG;YACfL,gBAAgB;YAChB,MAAMM,UAAU,MAAMpC,QAAQ+B,MAAM,oBAAoB,mCAAmCI;YAC3F,IAAI,CAACC,QAAQpB,EAAE,EAAE;YACjB9C,UAAUU,SAAS,IAAIwD,QAAQnB,KAAK;YACpC,MAAMoB,UAAU/I,aAAa8I,QAAQjB,IAAI;YACzC,KAAK,MAAMmB,YAAYD,QAAQE,SAAS,CAAE;gBACxC,IAAIxG;gBACJ,IAAI;oBACFA,MAAM,IAAIiB,IAAIsF;gBAChB,EAAE,eAAM;oBACN;gBACF;gBACA,IAAID,QAAQG,KAAK,EAAE;oBACjB,IAAI3C,SAAS9D,MAAM2F,aAAab,IAAI,CAAC9E,IAAIkB,QAAQ;oBACjD;gBACF;gBACA,MAAMH,MAAMT,kBAAkBN,KAAKV;gBACnC,IAAIyB,OAAO+E,YAAYlG,MAAM,GAAGzB,qBAAqBQ,cAAc,EAAEmH,YAAYhB,IAAI,CAAC/D;YACxF;QACF;QACAoB,UAAUQ,WAAW,GAAGmD,YAAYlG,MAAM;QAC1C,IAAI8B,QAAQgF,YAAY,EAAE;YACxB,MAAMC,QAAQ,CAAC5F,MAAgB,IAAIE,IAAIF,KAAKH,QAAQ,CAACgG,KAAK,CAAC,KAAKlB,MAAM,CAACmB,SAASjH,MAAM;YACtF,MAAMkH,SAAShB,YAAYN,GAAG,CAAC,CAACzE,KAAKgG,QAAW,CAAA;oBAAEhG;oBAAKgG;oBAAOJ,OAAOA,MAAM5F;gBAAK,CAAA;YAChF+F,OAAOE,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEN,KAAK,GAAGO,EAAEP,KAAK,IAAIM,EAAEF,KAAK,GAAGG,EAAEH,KAAK;YAC5DjB,YAAYqB,MAAM,CAAC,GAAGrB,YAAYlG,MAAM,KAAKkH,OAAOtB,GAAG,CAAC,CAACX,QAAUA,MAAM9D,GAAG;QAC9E;QAEA,yEAAyE;QACzE,wCAAwC;QACxC,MAAMqG,QAA+D7B,OACjEA,KAAKC,GAAG,CAAC,CAACzE,MAAS,CAAA;gBAAEA;gBAAKsG,SAAU,IAAIpG,IAAIF,KAAKH,QAAQ,KAAK,MAAM,SAAS;YAAqC,CAAA,KAClH;YAAC;gBAAEG,KAAKgE;gBAAMsC,SAAS;YAAO;SAAE;QACpC,MAAMC,SAAS,IAAIzB,IAAYN,eAAAA,OAAQ;YAACR;SAAK;QAC7C,KAAK,MAAMhE,OAAO+E,YAAa;YAC7B,IAAIwB,OAAOpB,GAAG,CAACnF,MAAM;YACrBuG,OAAOnB,GAAG,CAACpF;YACXqG,MAAMtC,IAAI,CAAC;gBAAE/D;gBAAKsG,SAAS;YAAU;QACvC;QACA,MAAME,OAAO,IAAI1B;QACjB,uEAAuE;QACvE,qEAAqE;QACrE,6DAA6D;QAC7D,0EAA0E;QAC1E,IAAI2B,YAAYjC,CAAAA,wBAAAA,KAAM3F,MAAM,IAAG,IAAIqB,IAAIsE,IAAI,CAAC,EAAE,EAAErF,MAAM,GAAGZ,OAAO,CAAC,EAAE;QACnE,MAAMmI,QAAQ,CAAChC;YACb,MAAMzF,MAAM,IAAIiB,IAAIwE;YACpB,OAAOnG,QAAQW,QAAQ,CAACD,IAAIE,MAAM,KAAKF,IAAIE,MAAM,KAAKsH,YAClD,GAAGA,YAAYxH,IAAIY,QAAQ,GAAGZ,IAAIqB,MAAM,EAAE,GAC1CoE;QACN;QACA,MAAMiC,cAAc1E,WAAW,IAAI;QACnC,IAAI2E,WAAW;QAEf,4EAA4E,GAC5E,MAAMC,SAAgC,EAAE;QACxC,4FAA4F,GAC5F,IAAIC,WAAW;QACf,IAAIC,OAA8B;QAClC,gFAAgF,GAChF,MAAMC,SAAmB,EAAE;QAC3B,gGAAgG,GAChG,IAAIC,SAAS,CAAC,CAACzC;QACf,iGAAiG,GACjG,MAAM0C,OAAO;YACX,IAAI9F,UAAUE,KAAK,CAACzC,MAAM,IAAIoD,UAAU,OAAO;YAC/C,IAAIgB,aAAa,OAAO;YACxB,IAAID,oBAAoB,GAAG,OAAO;YAClC,IAAI4D,YAAYD,aAAa,OAAO;YACpC,IAAIvF,UAAUE,KAAK,CAACzC,MAAM,GAAGgI,OAAOhI,MAAM,IAAIoD,UAAU,OAAO;YAC/D,IAAIe,mBAAmB8D,YAAY,GAAG,OAAO;YAC7C,OAAO;QACT;QACA,MAAMK,cAAaxG,kBAAAA,QAAQyG,MAAM,YAAdzG,kBAAkB,CAAC,CAAC6D;QACvC,iDAAiD,GACjD,MAAM6C,SAAS,IAAIC;QACnB,MAAMC,aAAa,CAACtI;YAClB,MAAMiI,OAAOG,OAAOG,GAAG,CAACvI;YACxB,IAAIiI,MAAM,OAAOA;YACjB,MAAM7B,OAAOnD,KAAKsB,GAAG,CAACtH,+BAA+B8G,mBAAmB8D;YACxE,MAAMW,QACJpC,QAAQ,KAAKpC,cACTyE,QAAQC,OAAO,CAAC,QAChBzE,QAAQjE,KAAK,0BAA0B,sBAAsBoG,MAAMuC,IAAI,CACrE,CAACC;gBACC,IAAI,CAACA,OAAO3D,EAAE,EAAE,OAAO;gBACvB9C,UAAUU,SAAS,IAAI+F,OAAO1D,KAAK;gBACnC,OAAO0D,OAAOxD,IAAI;YACpB,GACA,IAAM;YAEdgD,OAAOS,GAAG,CAAC7I,KAAKwI;YAChB,OAAOA;QACT;QACA,kGAAkG,GAClG,MAAMM,SAAS,OAAO,EAAE9C,IAAI,EAAEjF,GAAG,EAAEgI,GAAG,EAAEC,IAAI,EAAS;gBAyB5B1I;YAxBvB,MAAM+G,UAAUrB,KAAKqB,OAAO;YAC5B,IAAIA,YAAY,QAAQW,SAAS;YACjC,IAAI,CAACgB,KAAK/D,EAAE,EAAE;gBACZ,IAAI+D,KAAKxH,OAAO,KAAK,eAAeuH,MAAM3K,cAAc;oBACtD2J,OAAOjD,IAAI,CAAC/D;oBACZ,OAAO;gBACT;gBACA,IAAIiI,KAAKxH,OAAO,KAAK,aAAawC,aAAa;oBAC7C+D,OAAOjD,IAAI,CAAC/D;oBACZ,OAAO;gBACT;gBACA,qFAAqF;gBACrF,IAAIiI,KAAKxH,OAAO,KAAK,gBAAgBmG,WAAW1E,KAAKC,GAAG,CAAC,GAAGyE,WAAW;gBACvE/C,KAAK;oBACH5E,KAAKe;oBACLkI,QAAQ3H,cAAc0H;mBAClBA,KAAK3D,MAAM,GAAG;oBAAEA,QAAQ2D,KAAK3D,MAAM;gBAAC,IAAI,CAAC,GACzC2D,KAAKzC,QAAQ,GAAG;oBAAEA,UAAUyC,KAAKzC,QAAQ;gBAAC,IAAI,CAAC,GAC/CyC,KAAKE,IAAI,GAAG;oBAAEA,MAAMF,KAAKE,IAAI;gBAAC,IAAI,CAAC;gBAEzC,OAAO;YACT;YACA/G,UAAUU,SAAS,IAAImG,KAAK9D,KAAK;YACjC,IAAImC,YAAY,QAAQG,YAAY,IAAIvG,IAAI+H,KAAKhJ,GAAG,EAAEE,MAAM;YAC5D,MAAMiJ,WAAW1B,OAAMnH,qBAAAA,kBAAkB,IAAIW,IAAI+H,KAAKhJ,GAAG,GAAGV,oBAArCgB,qBAAiD0I,KAAKhJ,GAAG;YAChF,IAAImJ,aAAapI,KAAK;gBACpB,IAAIwG,KAAKrB,GAAG,CAACiD,WAAW,OAAO;gBAC/B5B,KAAKpB,GAAG,CAACgD;gBACT,IAAI,CAAC7D,QAAQ6D,WAAW;oBACtBvE,KAAK;wBAAE5E,KAAKmJ;wBAAUF,QAAQ;oBAAS;oBACvC,OAAO;gBACT;YACF;YACA,MAAMG,SAAS/B,YAAY;YAC3B,MAAMgC,aAAanB,cAAckB;YACjC,MAAME,UAAUnM,sBAAsB6L,KAAK5D,IAAI,EAAE4D,KAAKhJ,GAAG,EAAE8D,UAAU;gBAAEyF,WAAWF;YAAW;YAC7F,IAAIC,QAAQE,SAAS,EAAE;gBACrB5E,KAAK;oBAAE5E,KAAKmJ;oBAAUF,QAAQ;gBAAa;gBAC3C,OAAO;YACT;YACA,MAAMQ,QAAkB,EAAE;YAC1B,KAAK,MAAMC,QAAQJ,QAAQG,KAAK,CAAE;gBAChC,MAAME,UAAUrJ,kBAAkB,IAAIW,IAAIyI,OAAOpK;gBACjD,IAAI,CAACqK,SAAS;gBACdF,MAAM3E,IAAI,CAAC6E;gBACX,IAAI,CAACpE,QAAQ,CAAC+B,OAAOpB,GAAG,CAACyD,UAAU;oBACjCrC,OAAOnB,GAAG,CAACwD;oBACXvC,MAAMtC,IAAI,CAAC;wBAAE/D,KAAK4I;wBAAStC,SAAS;oBAAO;gBAC7C;YACF;YACA,iEAAiE;YACjE,IAAIuC,QAAoE,CAAC;YACzE,IAAIN,QAAQC,SAAS,EAAE;gBACrB,MAAMM,SAAS,MAAMpB,QAAQqB,GAAG,CAACR,QAAQC,SAAS,CAACQ,WAAW,CAACrE,MAAM,CAAC,CAAC1F,MAAQ,CAAC,0BAA0BW,IAAI,CAACX,MAAMgK,KAAK,CAAC,GAAGhN,gCAAgCwI,GAAG,CAAC8C;gBAClK,MAAM2B,MAAMJ,OAAOnE,MAAM,CAAC,CAACN,OAAyB,OAAOA,SAAS;gBACpE,MAAM8E,SAAS;oBAAEC,QAAQb,QAAQa,MAAM;oBAAEC,cAAcd,QAAQc,YAAY;oBAAEb,WAAWD,QAAQC,SAAS;gBAAC;gBAC1G,IAAI;oBACFK,QAAQ,aACF1B,aAAa;wBAAEC,QAAQvK,iBAAiBsM,QAAQD;oBAAK,IAAI,CAAC,GAC1Db,SAAS;wBAAEiB,KAAKtM,kBAAkBmM,QAAQD;wBAAMK,QAAQxM,wBAAwBoM,QAAQD;wBAAMM,MAAM1M,iBAAiBqM,QAAQD;oBAAK,IAAI,CAAC;gBAE/I,EAAE,OAAOpK,OAAO;oBACd,uDAAuD;oBACvDC,QAAQC,IAAI,CAAC,8CAA8C;wBAAEC,KAAKgJ,KAAKhJ,GAAG;wBAAEH;oBAAM;gBACpF;YACF;YACAsC,UAAUE,KAAK,CAACyC,IAAI,CAAC,aAChB8E;gBACH5J,KAAKmJ;gBACLqB,MAAM,IAAIvJ,IAAIkI,UAAUvI,QAAQ;gBAChCyG;gBACAoD,OAAOnB,QAAQmB,KAAK;gBACpBC,aAAapB,QAAQoB,WAAW;gBAChCC,MAAMrB,QAAQqB,IAAI;gBAClBC,WAAWtB,QAAQsB,SAAS;gBAC5BC,SAASvB,QAAQuB,OAAO;gBACxBC,UAAUxB,QAAQwB,QAAQ;gBAC1BC,OAAOzB,QAAQyB,KAAK;gBACpBC,OAAO1B,QAAQ0B,KAAK;gBACpBZ,cAAcd,QAAQc,YAAY;gBAClCD,QAAQb,QAAQa,MAAM;gBACtBc,QAAQ3B,QAAQ2B,MAAM;gBACtBC,UAAU5B,QAAQ4B,QAAQ;gBAC1BzB,OAAO;uBAAI,IAAI5D,IAAI4D,MAAMjE,GAAG,CAACiC;iBAAQ;gBACrC0D,OAAO7B,QAAQ6B,KAAK;gBACpBjG,OAAO8D,KAAK9D,KAAK;;YAEnB,OAAO;QACT;QACA,OAAS;gBA4BC;YA3BR,MAAMkG,QAAQpD,SAAS7J,qBAAqBU,WAAW,GAAG;YAC1D,MAAO,CAACiJ,QAAQV,MAAMxH,MAAM,IAAIgI,OAAOhI,MAAM,GAAGwL,MAAO;gBACrD,MAAMC,MAAMpD;gBACZ,IAAIoD,QAAQ,QAAQ;gBACpB,IAAIA,KAAK;oBACPvD,OAAOuD;oBACP;gBACF;gBACA,MAAMrF,OAAOoB,MAAMnB,KAAK;gBACxB,MAAMlF,MAAM0G,MAAMzB,KAAKjF,GAAG;gBAC1B,IAAIwG,KAAKrB,GAAG,CAACnF,MAAM;gBACnB,IAAI,CAACuE,QAAQvE,MAAM;oBACjB6D,KAAK;wBAAE5E,KAAKe;wBAAKkI,QAAQ;oBAAS;oBAClC,IAAIjD,KAAKqB,OAAO,KAAK,QAAQW,SAAS;oBACtC;gBACF;gBACAL,YAAY;gBACZJ,KAAKpB,GAAG,CAACpF;gBACT,MAAMgI,MAAM9F,KAAKsB,GAAG,CAACnG,cAAc2F,mBAAmB8D;gBACtDA,YAAYkB;gBACZnB,OAAO9C,IAAI,CACTb,QAAQlD,KAAK,uCAAuC,0CAA0CgI,KAAKJ,IAAI,CAAC,CAACK,OAAU,CAAA;wBAAEhD;wBAAMjF;wBAAKgI;wBAAKC;oBAAK,CAAA;YAE9I;YACA,IAAI,CAACpB,OAAOhI,MAAM,EAAE;YACpB,MAAM4I,QAAQ,MAAOZ,OAAO3B,KAAK;YACjC4B,YAAYW,MAAMO,GAAG;YACrBjB,QAAQ,OAAA,MAAMgB,OAAON,kBAAb,OAAwBV;QAClC;QACA3F,UAAUS,SAAS,GAAGkF;QACtB,wEAAwE;QACxE,qDAAqD;QACrD,IAAIA,QAAQ,CAACvC,MAAM;YACjB,MAAM+F,SAAS;mBAAIvD;aAAO;YAC1B,KAAK,MAAM/B,QAAQoB,MAAO;gBACxB,MAAMrG,MAAM0G,MAAMzB,KAAKjF,GAAG;gBAC1B,IAAI,CAACwG,KAAKrB,GAAG,CAACnF,QAAQ,CAACuK,OAAOrL,QAAQ,CAACc,MAAMuK,OAAOxG,IAAI,CAAC/D;YAC3D;YACA,KAAK,MAAMf,OAAOsL,OAAQ1G,KAAK;gBAAE5E;gBAAKiJ,QAAQ;YAAc;QAC9D;QACA,OAAOlG;IACT,SAAU;YAERrB;QADA6J,aAAa3H;SACblC,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgB8J,mBAAmB,CAAC,SAASjI;IAC/C;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAekI,yBACpBpM,MAAc,EACdqM,QAAgB,EAChBtM,QAA8B,CAAC,CAAC;QAEdA,cAmBLmC;IAnBb,MAAMM,aAAYzC,eAAAA,MAAM0C,KAAK,YAAX1C,eAAelC;IACjC,MAAMkD,OAAO2B,OAAO1C,iBAAAA,SAAU,IAAI2C,IAAI,GAAGC,WAAW,GAAG5B,OAAO,CAAC,UAAU;IACzE,IAAI,CAACnB,eAAeyB,IAAI,CAACP,SAAStD,sBAAsBsD,OAAO,OAAO;QAAEuL,UAAU;QAAOC,QAAQ,EAAE;IAAC;IACpG,MAAMtM,UAAUa,kBAAkBC;IAClC,MAAMmB,SAAS,MAAMM,UAAU,GAAGvC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;QAC/C8E,UAAUjG,qBAAqBC,YAAY;QAC3C8F,QAAQ;QACRC,cAAc;QACdE,WAAWnG;QACXsB,YAAY,MAAML,mBAAmBC,OAAOgB,MAAMd;QAClDgF,WAAWnG,qBAAqBI,gBAAgB;QAChDiG,iBAAiB,CAACC,OAAOC,KAAOA,GAAGnE,QAAQ,KAAK,YAAYjB,QAAQW,QAAQ,CAACyE,GAAGxE,MAAM;IACxF;IACA,IAAI,CAACqB,OAAO0D,EAAE,EAAE;QACd,qEAAqE;QACrE,oDAAoD;QACpD,MAAM0G,WAAWpK,OAAOC,OAAO,KAAK,iBAAiBD,OAAOC,OAAO,KAAK,sBAAsBD,OAAOC,OAAO,KAAK;QACjH,OAAO;YAAEmK;YAAUC,QAAQ,EAAE;YAAE5L,KAAKuB,OAAOvB,GAAG;QAAC;IACjD;IACA,MAAM6L,QAAOtK,sBAAAA,OAAO6D,IAAI,CAACwB,KAAK,CAAC,WAAW,CAAC,EAAE,YAAhCrF,sBAAoC;IACjD,OAAO;QAAEoK,UAAU;QAAMC,QAAQxO,eAAeyO,MAAMH;QAAW1L,KAAKuB,OAAOvB,GAAG;IAAC;AACnF;AAEA;;;CAGC,GACD,OAAO,SAAS8L,yBAAyB9C,IAAoB,EAAE+C,WAAW,KAAM;IAC9E,MAAMC,QAAkB,EAAE;IAC1B,IAAIhD,KAAKyB,KAAK,EAAEuB,MAAMlH,IAAI,CAAC,CAAC,OAAO,EAAEkE,KAAKyB,KAAK,EAAE;IACjD,IAAIzB,KAAK0B,WAAW,EAAEsB,MAAMlH,IAAI,CAAC,CAAC,aAAa,EAAEkE,KAAK0B,WAAW,EAAE;IACnE,KAAK,MAAMuB,SAASjD,KAAKmB,MAAM,CAAE;QAC/B,IAAI8B,MAAMC,IAAI,KAAK,WAAWF,MAAMlH,IAAI,CAAC,GAAG,IAAIqH,MAAM,CAACF,MAAMG,KAAK,EAAE,CAAC,EAAEH,MAAM7G,IAAI,EAAE;aAC9E,IAAI6G,MAAMC,IAAI,KAAK,SAASF,MAAMlH,IAAI,CAAC,CAAC,QAAQ,EAAEmH,MAAMI,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAClF,IAAIJ,MAAMC,IAAI,KAAK,aAAaF,MAAMlH,IAAI,CAAC,CAAC,EAAE,EAAEmH,MAAM7G,IAAI,EAAE;aAC5D,IAAI6G,MAAMC,IAAI,KAAK,SAASF,MAAMlH,IAAI,CAAC,CAAC,EAAE,EAAEmH,MAAM7G,IAAI,EAAE;aACxD,IAAI6G,MAAMC,IAAI,KAAK,QAAQ;YAC9B,4DAA4D;YAC5D,IAAI,CAACD,MAAMK,MAAM,EAAEN,MAAMlH,IAAI,CAACmH,MAAM7G,IAAI;QAC1C,OAAO4G,MAAMlH,IAAI,CAACmH,MAAM7G,IAAI;IAC9B;IACA,OAAOrI,mBAAmB,CAAC,KAAK,EAAEiM,KAAKwB,IAAI,CAAC,2BAA2B,CAAC,EAAEwB,MAAMO,IAAI,CAAC,OAAO;QAAER;IAAS;AACzG"}

package/src/lib/site-import/site-import-css.d.ts

@@ -0,0 +1,110 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type { SiteImportStructure } from './site-import-html';
+/** The width a page is read at: a laptop's, where a desktop layout shows. */
+export declare const SITE_IMPORT_CSS_VIEWPORT: {
+    readonly width: 1280;
+    readonly height: 800;
+};
+interface Compound {
+    tag: string | null;
+    id: string | null;
+    classes: string[];
+    attrs: Array<{
+        name: string;
+        op: string | null;
+        value: string;
+    }>;
+    /** `:root`: only the document's root element. */
+    root: boolean;
+}
+interface ComplexSelector {
+    /** Rightmost last. */
+    compounds: Compound[];
+    /** `combinators[i]` joins `compounds[i]` and `compounds[i + 1]`: a space, a child, an adjacent or a sibling combinator. */
+    combinators: string[];
+    specificity: number;
+}
+interface CssRule {
+    selector: ComplexSelector;
+    order: number;
+    decls: Array<[property: string, value: string, important: boolean]>;
+}
+/** The page's sheets, parsed for one width. */
+export interface SiteImportCss {
+    rules: CssRule[];
+    /** Rules by their rightmost compound's id (`#x`), first class (`.x`), tag, or `*`. */
+    index: Map<string, CssRule[]>;
+    /** Families its `@font-face` rules name, in order. */
+    fontFaces: string[];
+}
+/**
+ * The page's sheets parsed for a wide screen. Malformed CSS is read as far as
+ * it makes sense, as a browser would; nothing throws.
+ */
+export declare function siteImportCssOf(sheets: readonly string[]): SiteImportCss;
+/** One element's resolved style: the kept properties, `var()` read, lowercased. */
+export type SiteImportComputedStyle = ReadonlyMap<string, string>;
+/** Reads elements' styles over a page's structure, each once. */
+export interface SiteImportStyles {
+    of(element: number): SiteImportComputedStyle;
+    /** The root's (`<html>`) element id, which `:root` matches; -1 for none. */
+    readonly root: number;
+}
+/**
+ * Elements' styles over a page's structure and its sheets. Styles are
+ * resolved on demand, parent first, each element once.
+ */
+export declare function siteImportStylesOf(structure: Pick<SiteImportStructure, 'elements' | 'root'>, css: SiteImportCss): SiteImportStyles;
+export interface SiteImportRgba {
+    r: number;
+    g: number;
+    b: number;
+    a: number;
+}
+/** The first color a value names, or `null`; `transparent` is a color of no strength. */
+export declare function siteImportColorOf(value: string | undefined): SiteImportRgba | null;
+/** `#rrggbb` of a color, its strength ignored. */
+export declare function siteImportHex(color: SiteImportRgba): string;
+/** Relative luminance, 0 (black) to 1 (white). */
+export declare function siteImportLuminance(color: SiteImportRgba): number;
+/** How colorful a color is, 0 (grey) to 1. */
+export declare function siteImportSaturation(color: SiteImportRgba): number;
+/** What an element draws behind itself: a color, a gradient and its colors, or a picture. */
+export interface SiteImportBackground {
+    color: SiteImportRgba | null;
+    gradient: SiteImportRgba[] | null;
+    /** A picture's address, as written. */
+    image: string | null;
+}
+/** An element's own background, from the shorthand or its parts. */
+export declare function siteImportBackgroundOf(style: SiteImportComputedStyle): SiteImportBackground;
+/** Whether an element draws a border on any side. */
+export declare function siteImportHasBorder(style: SiteImportComputedStyle): boolean;
+/** Whether an element draws a shadow. */
+export declare function siteImportHasShadow(style: SiteImportComputedStyle): boolean;
+/** An element's corner radius in px, or `null`. */
+export declare function siteImportRadiusPx(style: SiteImportComputedStyle): number | null;
+/** A font size in px, or `null`. */
+export declare function siteImportFontSizePx(style: SiteImportComputedStyle): number | null;
+/** The relative widths a grid template draws its columns at, or `null` for none. */
+export declare function siteImportGridColumns(template: string | undefined): number[] | null;
+/** Weights scaled to twelfths, each at least 1: `[420, 788]` is `[4, 8]`. */
+export declare function siteImportTwelfths(weights: readonly number[]): number[];
+/** The families a `font-family` value names, quotes dropped, generic ones left out. */
+export declare function siteImportFontFamilies(value: string | undefined): string[];
+export {};