@aglyn/plugins-ai 1.0.0-beta.288 → 1.0.0-beta.290

package/src/lib/site-import/site-import-crawl.js

@@ -36,11 +36,25 @@ var _process_env_SITE_IMPORT_BOT_INFO_URL;
  *   group, else `*`. A robots.txt that answers 404 allows everything; one
  *   that cannot be read at all (5xx, no answer) stops the crawl, because
  *   "could not ask" is not "allowed".
+ * - **Our own edge.** A site this platform serves gets the edge secret
+ *   (`site-import-edge.ts`, AGL-4085), which lifts our bot challenge for
+ *   AglynBot; it is asked per request and per redirect hop, and a request
+ *   that leaves the site's two origins never carries it.
  * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an
  *   index's children included), then same-site links found on the pages read,
  *   for whatever the sitemap missed.
  * - **Budgets.** Pages (the caller's, from the plan), bytes per page (a page
- *   over it is refused, not cut), bytes in total, and wall-clock time.
+ *   over it is refused, not cut), bytes in total, and wall-clock time. A
+ *   whole site's worth fits (AGL-4083): aglyn.com's 123 pages average 440 KB,
+ *   54 MB in all, so the total is 160 MB; a page is parsed and dropped as it
+ *   is read, so what is held is its blocks, never its bytes.
+ * - **Four at a time.** robots.txt, the sitemaps and the home page are read
+ *   one after another; the rest four at a time, and the inventory keeps the
+ *   order they were asked in, which is the sitemap's.
+ * - **Nothing vanishes.** An address a stop (the clock, the bytes, the page
+ *   budget) left unread is listed as `not-reached`, so the inventory's
+ *   counts add up to the site. A wave's own read leaves its unread pages for
+ *   its next pass instead.
  * - **No lookalikes.** A domain that wears a brand it is not
  *   (`lookalikeBrandForHost`) is refused before any request.
  * - **No sign-in pages.** A page with a password field is refused and
@@ -51,6 +65,7 @@ import { fenceUntrustedText } from "@aglyn/shared-util-tools/untrusted-text";
 import { fetchPublicTextDetailed } from "../jobs/ai-theme-brand-inputs.js";
 import { extractSiteImportPage, readMetaValues } from "./site-import-html.js";
 import { AGLYN_BOT_TOKEN, parseRobotsTxt, parseSitemap, robotsAllows, robotsRulesFor } from "./site-import-robots.js";
+import { NO_SITE_IMPORT_HEADERS, siteImportEdgeHeadersFor } from "./site-import-edge.js";
 /**
  * Where the bot's user agent points a site owner who wants to know what it is
  * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on
@@ -65,18 +80,39 @@ import { AGLYN_BOT_TOKEN, parseRobotsTxt, parseSitemap, robotsAllows, robotsRule
 /** What every crawler request says it is. */ export const AGLYN_BOT_USER_AGENT = `AglynBot/1.0 (+${AGLYN_BOT_INFO_URL})`;
 export const SITE_IMPORT_DEFAULTS = {
     maxPageBytes: 2 * 1024 * 1024,
-    maxTotalBytes: 24 * 1024 * 1024,
+    /** A whole site's worth (AGL-4083): 54 MB for aglyn.com's 123 pages, with room. */ maxTotalBytes: 160 * 1024 * 1024,
     timeBudgetMs: 60000,
     /** One request's own clock, inside the whole crawl's. */ requestTimeoutMs: 8000,
     robotsMaxBytes: 256 * 1024,
     sitemapMaxBytes: 4 * 1024 * 1024,
-    /** Sitemap documents read, an index's children included. */ maxSitemaps: 6,
+    /** Sitemap documents read, an index's children included: an index of six children is seven. */ maxSitemaps: 24,
     /** URLs a sitemap may contribute to the queue. */ maxSitemapUrls: 2000,
-    /** Reasons kept in `skipped` before the list stops growing. */ maxSkipped: 200
+    /** Reasons kept in `skipped` before the list stops growing: every address a sitemap may name. */ maxSkipped: 2000,
+    /** Pages read at once, after robots.txt, the sitemaps and the home page. */ concurrency: 4
 };
 /** Never a page: files a link may point at that are not HTML. */ const NON_PAGE_EXTENSION = /\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i;
 const TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i;
 const DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\.[a-z0-9-]{1,63})+$/;
+/**
+ * The per-request headers of a crawl of `domain`: the seam's, else the edge's,
+ * and none for any address off the site's two origins. Never throws.
+ */ async function siteRequestHeaders(seams, domain, origins) {
+    let inner = NO_SITE_IMPORT_HEADERS;
+    try {
+        if (seams.headersFor) {
+            const headers = await seams.headersFor(domain);
+            if (headers && Object.keys(headers).length) inner = ()=>headers;
+        } else {
+            inner = await siteImportEdgeHeadersFor(origins);
+        }
+    } catch (error) {
+        console.warn('[site-import] the edge headers could not be read; reading without them', {
+            domain,
+            error
+        });
+    }
+    return (url)=>origins.includes(url.origin) ? inner(url) : {};
+}
 /** The site's two origins for a domain. */ export function siteImportOrigins(domain) {
     const bare = domain.replace(/^www\./, '');
     return [
@@ -150,6 +186,8 @@ function skipReasonFor(result) {
         inventory.refused = 'lookalike';
         return finish();
     }
+    // Asked once, before anything else, and held to the site's two origins.
+    const headersFor = await siteRequestHeaders(seams, domain, origins);
     const maxPages = Math.max(0, Math.floor(options.maxPages));
     const maxPageBytes = (_options_maxPageBytes = options.maxPageBytes) != null ? _options_maxPageBytes : SITE_IMPORT_DEFAULTS.maxPageBytes;
     const maxTotalBytes = (_options_maxTotalBytes = options.maxTotalBytes) != null ? _options_maxTotalBytes : SITE_IMPORT_DEFAULTS.maxTotalBytes;
@@ -167,6 +205,7 @@ function skipReasonFor(result) {
             accept,
             acceptHeader,
             userAgent: AGLYN_BOT_USER_AGENT,
+            headersFor,
             signal: controller.signal,
             timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),
             redirectAllowed: (_from, to)=>sameSite(to),
@@ -294,47 +333,32 @@ function skipReasonFor(result) {
         };
         const maxAttempts = maxPages * 3 + 10;
         let attempts = 0;
-        while(queue.length){
+        /** The pages asked for and not yet taken in, in the order they were asked. */ const window = [];
+        /** The bytes the asked pages may still read: what is left is shared, never promised twice. */ let reserved = 0;
+        let stop = null;
+        /** Asked for and never read, because a stop came first: listed as not reached. */ const cutOff = [];
+        /** Home alone first, so its answer settles which of the site's two names the rest are read on. */ let homeIn = !!only;
+        /** Why no more may be asked now: a stop, `wait` for an asked page to come in first, or nothing. */ const held = ()=>{
+            if (inventory.pages.length >= maxPages) return 'page-budget';
+            if (outOfTime()) return 'time';
+            if (remainingBytes() <= 0) return 'total-bytes';
+            if (attempts >= maxAttempts) return 'page-budget';
+            if (inventory.pages.length + window.length >= maxPages) return 'wait';
+            if (remainingBytes() - reserved <= 0) return 'wait';
+            return null;
+        };
+        /** Takes one asked page in: the inventory's next page, a skip, or the reason the read must stop. */ const takeIn = ({ next, key, cap, page })=>{
             var _siteImportPageKey;
-            if (inventory.pages.length >= maxPages) {
-                inventory.stoppedBy = 'page-budget';
-                break;
-            }
-            if (outOfTime()) {
-                inventory.stoppedBy = 'time';
-                break;
-            }
-            if (remainingBytes() <= 0) {
-                inventory.stoppedBy = 'total-bytes';
-                break;
-            }
-            if (attempts >= maxAttempts) {
-                inventory.stoppedBy = 'page-budget';
-                break;
-            }
-            const next = queue.shift();
-            const key = canon(next.key);
             const foundBy = next.foundBy;
-            if (read.has(key)) continue;
-            if (!allowed(key)) {
-                skip({
-                    url: key,
-                    reason: 'robots'
-                });
-                continue;
-            }
-            attempts += 1;
-            read.add(key);
-            const cap = Math.min(maxPageBytes, remainingBytes());
-            const page = await request(key, /text\/html|application\/xhtml\+xml/i, 'text/html, application/xhtml+xml;q=0.9', cap);
+            if (foundBy === 'home') homeIn = true;
             if (!page.ok) {
                 if (page.refusal === 'too-large' && cap < maxPageBytes) {
-                    inventory.stoppedBy = 'total-bytes';
-                    break;
+                    cutOff.push(key);
+                    return 'total-bytes';
                 }
                 if (page.refusal === 'network' && outOfTime()) {
-                    inventory.stoppedBy = 'time';
-                    break;
+                    cutOff.push(key);
+                    return 'time';
                 }
                 skip(_extends({
                     url: key,
@@ -343,21 +367,23 @@ function skipReasonFor(result) {
                     status: page.status
                 } : {}, page.location ? {
                     location: page.location
+                } : {}, page.edge ? {
+                    edge: page.edge
                 } : {}));
-                continue;
+                return null;
             }
             inventory.bytesRead += page.bytes;
             if (foundBy === 'home') preferred = new URL(page.url).origin;
             const finalKey = canon((_siteImportPageKey = siteImportPageKey(new URL(page.url), origins)) != null ? _siteImportPageKey : page.url);
             if (finalKey !== key) {
-                if (read.has(finalKey)) continue;
+                if (read.has(finalKey)) return null;
                 read.add(finalKey);
                 if (!allowed(finalKey)) {
                     skip({
                         url: finalKey,
                         reason: 'robots'
                     });
-                    continue;
+                    return null;
                 }
             }
             const extract = extractSiteImportPage(page.text, page.url, sameSite);
@@ -366,7 +392,7 @@ function skipReasonFor(result) {
                     url: finalKey,
                     reason: 'login-form'
                 });
-                continue;
+                return null;
             }
             const links = [];
             for (const link of extract.links){
@@ -389,6 +415,11 @@ function skipReasonFor(result) {
                 description: extract.description,
                 lang: extract.lang,
                 canonical: extract.canonical,
+                ogImage: extract.ogImage,
+                siteName: extract.siteName,
+                icons: extract.icons,
+                brand: extract.brand,
+                organization: extract.organization,
                 blocks: extract.blocks,
                 images: extract.images,
                 navLinks: extract.navLinks,
@@ -398,6 +429,60 @@ function skipReasonFor(result) {
                 forms: extract.forms,
                 bytes: page.bytes
             });
+            return null;
+        };
+        for(;;){
+            var _takeIn;
+            const width = homeIn ? SITE_IMPORT_DEFAULTS.concurrency : 1;
+            while(!stop && queue.length && window.length < width){
+                const why = held();
+                if (why === 'wait') break;
+                if (why) {
+                    stop = why;
+                    break;
+                }
+                const next = queue.shift();
+                const key = canon(next.key);
+                if (read.has(key)) continue;
+                if (!allowed(key)) {
+                    skip({
+                        url: key,
+                        reason: 'robots'
+                    });
+                    if (next.foundBy === 'home') homeIn = true;
+                    continue;
+                }
+                attempts += 1;
+                read.add(key);
+                const cap = Math.min(maxPageBytes, remainingBytes() - reserved);
+                reserved += cap;
+                window.push(request(key, /text\/html|application\/xhtml\+xml/i, 'text/html, application/xhtml+xml;q=0.9', cap).then((page)=>({
+                        next,
+                        key,
+                        cap,
+                        page
+                    })));
+            }
+            if (!window.length) break;
+            const asked = await window.shift();
+            reserved -= asked.cap;
+            stop = (_takeIn = takeIn(asked)) != null ? _takeIn : stop;
+        }
+        inventory.stoppedBy = stop;
+        // Whatever a stop left unread is said, never dropped; a wave's own read
+        // leaves its unread pages for its next pass instead.
+        if (stop && !only) {
+            const unread = [
+                ...cutOff
+            ];
+            for (const next of queue){
+                const key = canon(next.key);
+                if (!read.has(key) && !unread.includes(key)) unread.push(key);
+            }
+            for (const url of unread)skip({
+                url,
+                reason: 'not-reached'
+            });
         }
         return finish();
     } finally{
@@ -425,6 +510,7 @@ function skipReasonFor(result) {
         accept: /text\/html|application\/xhtml\+xml/i,
         acceptHeader: 'text/html, application/xhtml+xml;q=0.9',
         userAgent: AGLYN_BOT_USER_AGENT,
+        headersFor: await siteRequestHeaders(seams, bare, origins),
         timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,
         redirectAllowed: (_from, to)=>to.protocol === 'https:' && origins.includes(to.origin)
     });
@@ -457,7 +543,10 @@ function skipReasonFor(result) {
         else if (block.kind === 'image') lines.push(`[image: ${block.alt || 'no description'}]`);
         else if (block.kind === 'list-item') lines.push(`- ${block.text}`);
         else if (block.kind === 'quote') lines.push(`> ${block.text}`);
-        else lines.push(block.text);
+        else if (block.kind === 'link') {
+            // A link inside a sentence is in its block's words already.
+            if (!block.inline) lines.push(block.text);
+        } else lines.push(block.text);
     }
     return fenceUntrustedText(`Page ${page.path} of the site being imported`, lines.join('\n'), {
         maxChars

package/src/lib/site-import/site-import-crawl.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-crawl.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * THE SITE-IMPORT CRAWLER (AGL-3974, parent AGL-3973).\n *\n * Reads a website a workspace has PROVED it owns (domain-ownership.ts in\n * tenant-data-admin; the doors check the proof before calling this) into a\n * typed inventory: each page's title, description, headings and text in\n * order, images and menu links. It writes nothing; the import job decides\n * what becomes a page.\n *\n * Every request goes through `fetchPublicTextDetailed`, the theme job's\n * SSRF-guarded fetch: https on the default port only, public addresses only,\n * the socket pinned to the address checked. On top of that:\n *\n * - **One site.** The site is `https://<domain>` and `https://www.<domain>`,\n *   one site to its owner. Every redirect hop and every discovered link must\n *   stay on it; a page that redirects anywhere else is refused, not followed.\n * - **AglynBot.** Every request says `AglynBot/1.0 (+<the operator's bot page>)`,\n *   and robots.txt is read first and honored for every page: the AglynBot\n *   group, else `*`. A robots.txt that answers 404 allows everything; one\n *   that cannot be read at all (5xx, no answer) stops the crawl, because\n *   \"could not ask\" is not \"allowed\".\n * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an\n *   index's children included), then same-site links found on the pages read,\n *   for whatever the sitemap missed.\n * - **Budgets.** Pages (the caller's, from the plan), bytes per page (a page\n *   over it is refused, not cut), bytes in total, and wall-clock time.\n * - **No lookalikes.** A domain that wears a brand it is not\n *   (`lookalikeBrandForHost`) is refused before any request.\n * - **No sign-in pages.** A page with a password field is refused and\n *   reported; forms with payment fields, scripts, iframes and embeds never\n *   reach the inventory (site-import-html.ts).\n *=========================================*/\n\nimport { lookalikeBrandForHost } from '@aglyn/shared-util-email/outbound-phishing-screen'\nimport { fenceUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n  fetchPublicTextDetailed,\n  type PublicFetchOptions,\n  type PublicFetchResult,\n} from '../jobs/ai-theme-brand-inputs'\nimport {\n  extractSiteImportPage,\n  readMetaValues,\n  type SiteImportBlock,\n  type SiteImportForm,\n  type SiteImportImage,\n  type SiteImportNavLink,\n} from './site-import-html'\nimport {\n  AGLYN_BOT_TOKEN,\n  parseRobotsTxt,\n  parseSitemap,\n  robotsAllows,\n  robotsRulesFor,\n  type RobotsRule,\n} from './site-import-robots'\n\n/**\n * Where the bot's user agent points a site owner who wants to know what it is\n * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on\n * a self-hosted deployment, so its crawler never names Aglyn's address; the\n * literal is only that variable's default.\n *\n * Aglyn's is the Bring my site guide, which says what AglynBot reads and how\n * robots.txt keeps it out. There is no page of its own (Zach, 10/20: \"we don't\n * need a bot page at all\"), and the site it reads is the reader's own: an\n * import starts only once its owner has proved the site is theirs.\n */\nexport const AGLYN_BOT_INFO_URL =\n  process.env['SITE_IMPORT_BOT_INFO_URL'] ?? 'https://docs.aglyn.com/ai/bring-my-site'\n\n/** What every crawler request says it is. */\nexport const AGLYN_BOT_USER_AGENT = `AglynBot/1.0 (+${AGLYN_BOT_INFO_URL})`\n\nexport const SITE_IMPORT_DEFAULTS = {\n  maxPageBytes: 2 * 1024 * 1024,\n  maxTotalBytes: 24 * 1024 * 1024,\n  timeBudgetMs: 60_000,\n  /** One request's own clock, inside the whole crawl's. */\n  requestTimeoutMs: 8_000,\n  robotsMaxBytes: 256 * 1024,\n  sitemapMaxBytes: 4 * 1024 * 1024,\n  /** Sitemap documents read, an index's children included. */\n  maxSitemaps: 6,\n  /** URLs a sitemap may contribute to the queue. */\n  maxSitemapUrls: 2_000,\n  /** Reasons kept in `skipped` before the list stops growing. */\n  maxSkipped: 200,\n} as const\n\n/** Never a page: files a link may point at that are not HTML. */\nconst NON_PAGE_EXTENSION =\n  /\\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i\n\nconst TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i\n\nconst DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\\.[a-z0-9-]{1,63})+$/\n\nexport type SiteImportSkipReason =\n  /** The page has a password field: a sign-in page, refused outright. */\n  | 'login-form'\n  /** robots.txt keeps AglynBot off this path. */\n  | 'robots'\n  /** A redirect left the site. */\n  | 'off-origin-redirect'\n  /** Over the per-page byte cap. */\n  | 'too-large'\n  /** Answered with something that is not HTML. */\n  | 'not-html'\n  /** Answered 4xx or 5xx. */\n  | 'http-error'\n  /** No answer, a private address, or a refused address. */\n  | 'unreachable'\n\nexport interface SiteImportSkipped {\n  url: string\n  reason: SiteImportSkipReason\n  status?: number\n  /** Where an off-origin redirect pointed. */\n  location?: string\n}\n\nexport interface SiteImportPage {\n  /** The address the page was read from, after any same-site redirect. */\n  url: string\n  path: string\n  /** Where the crawler learned of the page. */\n  foundBy: 'home' | 'sitemap' | 'link'\n  title: string\n  description: string\n  lang: string | null\n  canonical: string | null\n  blocks: SiteImportBlock[]\n  images: SiteImportImage[]\n  navLinks: SiteImportNavLink[]\n  /** Same-site page addresses this page links to. */\n  links: string[]\n  forms: { kept: SiteImportForm[]; removed: number }\n  bytes: number\n}\n\nexport type SiteImportStop =\n  | 'page-budget'\n  | 'total-bytes'\n  | 'time'\n  /** robots.txt could not be read, so nothing was asked of the site. */\n  | 'robots-unreachable'\n\nexport interface SiteImportInventory {\n  domain: string\n  /** The site's origins: the bare name and its `www.` twin. */\n  origins: string[]\n  /** Set when nothing was fetched at all. */\n  refused: 'invalid-domain' | 'lookalike' | null\n  pages: SiteImportPage[]\n  skipped: SiteImportSkipped[]\n  /** Skips past {@link SITE_IMPORT_DEFAULTS.maxSkipped}, counted only. */\n  skippedOverflow: number\n  robots: { found: boolean; sitemaps: string[] }\n  /** Page addresses the sitemaps listed (same-site, before robots and budget). */\n  sitemapUrls: number\n  stoppedBy: SiteImportStop | null\n  bytesRead: number\n  startedAt: number\n  finishedAt: number\n}\n\nexport interface SiteImportCrawlOptions {\n  /** The proved domain; `www.` is folded in. */\n  domain: string\n  /** The most pages the inventory may hold: the plan's page count. */\n  maxPages: number\n  maxPageBytes?: number\n  maxTotalBytes?: number\n  timeBudgetMs?: number\n  signal?: AbortSignal\n  /**\n   * Read the sitemap's shallow addresses before its deep ones (AGL-3976), so a\n   * crawl the page budget stops reads a site's main pages before the tail of\n   * its blog. A sitemap's own order is kept among addresses of one depth.\n   */\n  shallowFirst?: boolean\n  /**\n   * Read exactly these addresses of the site, in order, and nothing else\n   * (AGL-3976): an import's wave reads its own pages again when it starts. No\n   * sitemap is read and no link is followed; robots.txt is read and honored\n   * as for any crawl, and every other rule holds.\n   */\n  only?: readonly string[]\n}\n\nexport interface SiteImportCrawlSeams {\n  fetch?: (url: string, options: PublicFetchOptions) => Promise<PublicFetchResult>\n  now?: () => number\n}\n\n/** The site's two origins for a domain. */\nexport function siteImportOrigins(domain: string): string[] {\n  const bare = domain.replace(/^www\\./, '')\n  return [`https://${bare}`, `https://www.${bare}`]\n}\n\n/** A crawlable page address on the site, normalized for de-duplication; else null. */\nexport function siteImportPageKey(url: URL, origins: readonly string[]): string | null {\n  if (url.protocol !== 'https:' || url.username || url.password) return null\n  if (url.port && url.port !== '443') return null\n  if (!origins.includes(url.origin)) return null\n  if (NON_PAGE_EXTENSION.test(url.pathname)) return null\n  const clean = new URL(url.toString())\n  clean.hash = ''\n  for (const key of [...clean.searchParams.keys()]) {\n    if (TRACKING_PARAM.test(key)) clean.searchParams.delete(key)\n  }\n  clean.search = clean.searchParams.toString() ? `?${clean.searchParams.toString()}` : ''\n  return clean.toString()\n}\n\nfunction skipReasonFor(result: PublicFetchResult): SiteImportSkipReason {\n  switch (result.refusal) {\n    case 'redirect-refused':\n      return 'off-origin-redirect'\n    case 'content-type':\n      return 'not-html'\n    case 'too-large':\n      return 'too-large'\n    case 'http-status':\n      return 'http-error'\n    default:\n      return 'unreachable'\n  }\n}\n\n/** Read a proved site into an inventory. Never throws for anything the site does. */\nexport async function crawlSiteForImport(\n  options: SiteImportCrawlOptions,\n  seams: SiteImportCrawlSeams = {},\n): Promise<SiteImportInventory> {\n  const now = seams.now ?? Date.now\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const domain = String(options.domain ?? '').trim().toLowerCase().replace(/\\.+$/, '').replace(/^www\\./, '')\n  const origins = DOMAIN_PATTERN.test(domain) ? siteImportOrigins(domain) : []\n  const startedAt = now()\n  const inventory: SiteImportInventory = {\n    domain,\n    origins,\n    refused: null,\n    pages: [],\n    skipped: [],\n    skippedOverflow: 0,\n    robots: { found: false, sitemaps: [] },\n    sitemapUrls: 0,\n    stoppedBy: null,\n    bytesRead: 0,\n    startedAt,\n    finishedAt: startedAt,\n  }\n  const finish = () => {\n    inventory.finishedAt = now()\n    return inventory\n  }\n  if (!origins.length) {\n    inventory.refused = 'invalid-domain'\n    return finish()\n  }\n  if (lookalikeBrandForHost(domain)) {\n    inventory.refused = 'lookalike'\n    return finish()\n  }\n\n  const maxPages = Math.max(0, Math.floor(options.maxPages))\n  const maxPageBytes = options.maxPageBytes ?? SITE_IMPORT_DEFAULTS.maxPageBytes\n  const maxTotalBytes = options.maxTotalBytes ?? SITE_IMPORT_DEFAULTS.maxTotalBytes\n  const deadlineAt = startedAt + (options.timeBudgetMs ?? SITE_IMPORT_DEFAULTS.timeBudgetMs)\n  const controller = new AbortController()\n  const onAbort = () => controller.abort()\n  options.signal?.addEventListener('abort', onAbort)\n  if (options.signal?.aborted) controller.abort()\n  const timer = setTimeout(() => controller.abort(), Math.max(0, deadlineAt - startedAt))\n\n  const sameSite = (url: URL) => origins.includes(url.origin) && url.protocol === 'https:'\n  const remainingBytes = () => maxTotalBytes - inventory.bytesRead\n  const outOfTime = () => controller.signal.aborted || now() >= deadlineAt\n  const request = (url: string, accept: RegExp, acceptHeader: string, maxBytes: number) =>\n    fetchText(url, {\n      maxBytes,\n      accept,\n      acceptHeader,\n      userAgent: AGLYN_BOT_USER_AGENT,\n      signal: controller.signal,\n      timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),\n      redirectAllowed: (_from, to) => sameSite(to),\n      refuseOver: true,\n    })\n  const skip = (entry: SiteImportSkipped) => {\n    if (inventory.skipped.length < SITE_IMPORT_DEFAULTS.maxSkipped) inventory.skipped.push(entry)\n    else inventory.skippedOverflow += 1\n  }\n\n  try {\n    // robots.txt, before anything else is asked of the site.\n    const home = `${origins[0]}/`\n    let rules: RobotsRule[] = []\n    const robots = await request(\n      `${origins[0]}/robots.txt`,\n      /text\\/plain|text\\/html|^$/i,\n      'text/plain',\n      Math.min(SITE_IMPORT_DEFAULTS.robotsMaxBytes, remainingBytes()),\n    )\n    if (robots.ok) {\n      inventory.bytesRead += robots.bytes\n      const parsed = parseRobotsTxt(robots.text)\n      rules = robotsRulesFor(parsed, AGLYN_BOT_TOKEN)\n      inventory.robots = { found: true, sitemaps: parsed.sitemaps }\n    } else if (\n      !(robots.refusal === 'http-status' && (robots.status ?? 0) >= 400 && (robots.status ?? 0) < 500)\n    ) {\n      inventory.stoppedBy = outOfTime() ? 'time' : 'robots-unreachable'\n      return finish()\n    }\n    const allowed = (key: string) => {\n      const url = new URL(key)\n      return robotsAllows(rules, `${url.pathname}${url.search}`)\n    }\n\n    // A wave's own pages (AGL-3976): exactly those, after robots.txt.\n    const only = options.only\n      ? options.only\n          .map((raw) => {\n            try {\n              return siteImportPageKey(new URL(raw), origins)\n            } catch {\n              return null\n            }\n          })\n          .filter((key): key is string => !!key)\n      : null\n\n    // Sitemaps: the ones robots.txt names on this site, else /sitemap.xml.\n    const sitemapQueue: string[] = []\n    for (const raw of inventory.robots.sitemaps) {\n      try {\n        const url = new URL(raw)\n        if (sameSite(url)) sitemapQueue.push(url.toString())\n      } catch {\n        // A malformed Sitemap line names nothing.\n      }\n    }\n    if (!sitemapQueue.length && !only) sitemapQueue.push(`${origins[0]}/sitemap.xml`)\n    if (only) sitemapQueue.length = 0\n    const sitemapSeen = new Set<string>()\n    const fromSitemap: string[] = []\n    let sitemapsRead = 0\n    while (sitemapQueue.length && sitemapsRead < SITE_IMPORT_DEFAULTS.maxSitemaps && !outOfTime()) {\n      const next = sitemapQueue.shift() as string\n      if (sitemapSeen.has(next)) continue\n      sitemapSeen.add(next)\n      if (/\\.gz$/i.test(new URL(next).pathname)) continue\n      if (!allowed(next)) continue\n      const room = Math.min(SITE_IMPORT_DEFAULTS.sitemapMaxBytes, remainingBytes())\n      if (room <= 0) break\n      sitemapsRead += 1\n      const sitemap = await request(next, /xml|text\\/plain/i, 'application/xml, text/xml;q=0.9', room)\n      if (!sitemap.ok) continue\n      inventory.bytesRead += sitemap.bytes\n      const entries = parseSitemap(sitemap.text)\n      for (const location of entries.locations) {\n        let url: URL\n        try {\n          url = new URL(location)\n        } catch {\n          continue\n        }\n        if (entries.index) {\n          if (sameSite(url)) sitemapQueue.push(url.toString())\n          continue\n        }\n        const key = siteImportPageKey(url, origins)\n        if (key && fromSitemap.length < SITE_IMPORT_DEFAULTS.maxSitemapUrls) fromSitemap.push(key)\n      }\n    }\n    inventory.sitemapUrls = fromSitemap.length\n    if (options.shallowFirst) {\n      const depth = (key: string) => new URL(key).pathname.split('/').filter(Boolean).length\n      const ranked = fromSitemap.map((key, order) => ({ key, order, depth: depth(key) }))\n      ranked.sort((a, b) => a.depth - b.depth || a.order - b.order)\n      fromSitemap.splice(0, fromSitemap.length, ...ranked.map((entry) => entry.key))\n    }\n\n    // Pages: home, then the sitemap's, then links the pages themselves name.\n    // A wave's read is its own pages alone.\n    const queue: { key: string; foundBy: SiteImportPage['foundBy'] }[] = only\n      ? only.map((key) => ({ key, foundBy: (new URL(key).pathname === '/' ? 'home' : 'link') as SiteImportPage['foundBy'] }))\n      : [{ key: home, foundBy: 'home' }]\n    const queued = new Set<string>(only ?? [home])\n    for (const key of fromSitemap) {\n      if (queued.has(key)) continue\n      queued.add(key)\n      queue.push({ key, foundBy: 'sitemap' })\n    }\n    const read = new Set<string>()\n    // The site's two names are one site: once the home page says which one\n    // it answers on, every address is read on that one, so /about is not\n    // read twice as example.com/about and www.example.com/about.\n    // A wave's read starts on the name its pages were read on the first time.\n    let preferred = only?.length ? new URL(only[0]).origin : origins[0]\n    const canon = (raw: string) => {\n      const url = new URL(raw)\n      return origins.includes(url.origin) && url.origin !== preferred\n        ? `${preferred}${url.pathname}${url.search}`\n        : raw\n    }\n    const maxAttempts = maxPages * 3 + 10\n    let attempts = 0\n    while (queue.length) {\n      if (inventory.pages.length >= maxPages) {\n        inventory.stoppedBy = 'page-budget'\n        break\n      }\n      if (outOfTime()) {\n        inventory.stoppedBy = 'time'\n        break\n      }\n      if (remainingBytes() <= 0) {\n        inventory.stoppedBy = 'total-bytes'\n        break\n      }\n      if (attempts >= maxAttempts) {\n        inventory.stoppedBy = 'page-budget'\n        break\n      }\n      const next = queue.shift() as (typeof queue)[number]\n      const key = canon(next.key)\n      const foundBy = next.foundBy\n      if (read.has(key)) continue\n      if (!allowed(key)) {\n        skip({ url: key, reason: 'robots' })\n        continue\n      }\n      attempts += 1\n      read.add(key)\n      const cap = Math.min(maxPageBytes, remainingBytes())\n      const page = await request(key, /text\\/html|application\\/xhtml\\+xml/i, 'text/html, application/xhtml+xml;q=0.9', cap)\n      if (!page.ok) {\n        if (page.refusal === 'too-large' && cap < maxPageBytes) {\n          inventory.stoppedBy = 'total-bytes'\n          break\n        }\n        if (page.refusal === 'network' && outOfTime()) {\n          inventory.stoppedBy = 'time'\n          break\n        }\n        skip({\n          url: key,\n          reason: skipReasonFor(page),\n          ...(page.status ? { status: page.status } : {}),\n          ...(page.location ? { location: page.location } : {}),\n        })\n        continue\n      }\n      inventory.bytesRead += page.bytes\n      if (foundBy === 'home') preferred = new URL(page.url).origin\n      const finalKey = canon(siteImportPageKey(new URL(page.url), origins) ?? page.url)\n      if (finalKey !== key) {\n        if (read.has(finalKey)) continue\n        read.add(finalKey)\n        if (!allowed(finalKey)) {\n          skip({ url: finalKey, reason: 'robots' })\n          continue\n        }\n      }\n      const extract = extractSiteImportPage(page.text, page.url, sameSite)\n      if (extract.loginForm) {\n        skip({ url: finalKey, reason: 'login-form' })\n        continue\n      }\n      const links: string[] = []\n      for (const link of extract.links) {\n        const linkKey = siteImportPageKey(new URL(link), origins)\n        if (!linkKey) continue\n        links.push(linkKey)\n        if (!only && !queued.has(linkKey)) {\n          queued.add(linkKey)\n          queue.push({ key: linkKey, foundBy: 'link' })\n        }\n      }\n      inventory.pages.push({\n        url: finalKey,\n        path: new URL(finalKey).pathname,\n        foundBy,\n        title: extract.title,\n        description: extract.description,\n        lang: extract.lang,\n        canonical: extract.canonical,\n        blocks: extract.blocks,\n        images: extract.images,\n        navLinks: extract.navLinks,\n        links: [...new Set(links.map(canon))],\n        forms: extract.forms,\n        bytes: page.bytes,\n      })\n    }\n    return finish()\n  } finally {\n    clearTimeout(timer)\n    options.signal?.removeEventListener('abort', onAbort)\n  }\n}\n\n/**\n * The home page's `aglyn-site-verification` values, for the meta-tag proof:\n * the same fetch, user agent and same-site redirects as the crawl, but no\n * robots.txt, because the person asked for this one page to be read.\n * `answered: false` when the page could not be read at all.\n */\nexport async function readSiteVerificationMeta(\n  domain: string,\n  metaName: string,\n  seams: SiteImportCrawlSeams = {},\n): Promise<{ answered: boolean; values: string[]; url?: string }> {\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const bare = String(domain ?? '').trim().toLowerCase().replace(/^www\\./, '')\n  if (!DOMAIN_PATTERN.test(bare) || lookalikeBrandForHost(bare)) return { answered: false, values: [] }\n  const origins = siteImportOrigins(bare)\n  const result = await fetchText(`${origins[0]}/`, {\n    maxBytes: SITE_IMPORT_DEFAULTS.maxPageBytes,\n    accept: /text\\/html|application\\/xhtml\\+xml/i,\n    acceptHeader: 'text/html, application/xhtml+xml;q=0.9',\n    userAgent: AGLYN_BOT_USER_AGENT,\n    timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,\n    redirectAllowed: (_from, to) => to.protocol === 'https:' && origins.includes(to.origin),\n  })\n  if (!result.ok) {\n    // An answer that is not the page (a 404, a redirect off the site) is\n    // still an answer: the tag is not where it must be.\n    const answered = result.refusal === 'http-status' || result.refusal === 'redirect-refused' || result.refusal === 'content-type'\n    return { answered, values: [], url: result.url }\n  }\n  const head = result.text.split(/<body\\b/i)[0] ?? ''\n  return { answered: true, values: readMetaValues(head, metaName), url: result.url }\n}\n\n/**\n * One page as text for a prompt: its title, description and blocks in order,\n * fenced as untrusted data. The importer's prompt carries `UNTRUSTED_TEXT_RULE`.\n */\nexport function siteImportPagePromptText(page: SiteImportPage, maxChars = 12_000): string {\n  const lines: string[] = []\n  if (page.title) lines.push(`Title: ${page.title}`)\n  if (page.description) lines.push(`Description: ${page.description}`)\n  for (const block of page.blocks) {\n    if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n    else if (block.kind === 'image') lines.push(`[image: ${block.alt || 'no description'}]`)\n    else if (block.kind === 'list-item') lines.push(`- ${block.text}`)\n    else if (block.kind === 'quote') lines.push(`> ${block.text}`)\n    else lines.push(block.text)\n  }\n  return fenceUntrustedText(`Page ${page.path} of the site being imported`, lines.join('\\n'), { maxChars })\n}\n"],"names":["process","lookalikeBrandForHost","fenceUntrustedText","fetchPublicTextDetailed","extractSiteImportPage","readMetaValues","AGLYN_BOT_TOKEN","parseRobotsTxt","parseSitemap","robotsAllows","robotsRulesFor","AGLYN_BOT_INFO_URL","env","AGLYN_BOT_USER_AGENT","SITE_IMPORT_DEFAULTS","maxPageBytes","maxTotalBytes","timeBudgetMs","requestTimeoutMs","robotsMaxBytes","sitemapMaxBytes","maxSitemaps","maxSitemapUrls","maxSkipped","NON_PAGE_EXTENSION","TRACKING_PARAM","DOMAIN_PATTERN","siteImportOrigins","domain","bare","replace","siteImportPageKey","url","origins","protocol","username","password","port","includes","origin","test","pathname","clean","URL","toString","hash","key","searchParams","keys","delete","search","skipReasonFor","result","refusal","crawlSiteForImport","options","seams","now","Date","fetchText","fetch","String","trim","toLowerCase","startedAt","inventory","refused","pages","skipped","skippedOverflow","robots","found","sitemaps","sitemapUrls","stoppedBy","bytesRead","finishedAt","finish","length","maxPages","Math","max","floor","deadlineAt","controller","AbortController","onAbort","abort","signal","addEventListener","aborted","timer","setTimeout","sameSite","remainingBytes","outOfTime","request","accept","acceptHeader","maxBytes","userAgent","timeoutMs","min","redirectAllowed","_from","to","refuseOver","skip","entry","push","home","rules","ok","bytes","parsed","text","status","allowed","only","map","raw","filter","sitemapQueue","sitemapSeen","Set","fromSitemap","sitemapsRead","next","shift","has","add","room","sitemap","entries","location","locations","index","shallowFirst","depth","split","Boolean","ranked","order","sort","a","b","splice","queue","foundBy","queued","read","preferred","canon","maxAttempts","attempts","reason","cap","page","finalKey","extract","loginForm","links","link","linkKey","path","title","description","lang","canonical","blocks","images","navLinks","forms","clearTimeout","removeEventListener","readSiteVerificationMeta","metaName","answered","values","head","siteImportPagePromptText","maxChars","lines","block","kind","repeat","level","alt","join"],"mappings":";IAsFEA;AAtFF;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2CA+B2C,GAE3C,SAASC,qBAAqB,QAAQ,oDAAmD;AACzF,SAASC,kBAAkB,QAAQ,0CAAyC;AAC5E,SACEC,uBAAuB,QAGlB,mCAA+B;AACtC,SACEC,qBAAqB,EACrBC,cAAc,QAKT,wBAAoB;AAC3B,SACEC,eAAe,EACfC,cAAc,EACdC,YAAY,EACZC,YAAY,EACZC,cAAc,QAET,0BAAsB;AAE7B;;;;;;;;;;CAUC,GACD,OAAO,MAAMC,sBACXX,wCAAAA,QAAQY,GAAG,CAAC,2BAA2B,YAAvCZ,wCAA2C,0CAAyC;AAEtF,2CAA2C,GAC3C,OAAO,MAAMa,uBAAuB,CAAC,eAAe,EAAEF,mBAAmB,CAAC,CAAC,CAAA;AAE3E,OAAO,MAAMG,uBAAuB;IAClCC,cAAc,IAAI,OAAO;IACzBC,eAAe,KAAK,OAAO;IAC3BC,cAAc;IACd,uDAAuD,GACvDC,kBAAkB;IAClBC,gBAAgB,MAAM;IACtBC,iBAAiB,IAAI,OAAO;IAC5B,0DAA0D,GAC1DC,aAAa;IACb,gDAAgD,GAChDC,gBAAgB;IAChB,6DAA6D,GAC7DC,YAAY;AACd,EAAU;AAEV,+DAA+D,GAC/D,MAAMC,qBACJ;AAEF,MAAMC,iBAAiB;AAEvB,MAAMC,iBAAiB;AAoGvB,yCAAyC,GACzC,OAAO,SAASC,kBAAkBC,MAAc;IAC9C,MAAMC,OAAOD,OAAOE,OAAO,CAAC,UAAU;IACtC,OAAO;QAAC,CAAC,QAAQ,EAAED,MAAM;QAAE,CAAC,YAAY,EAAEA,MAAM;KAAC;AACnD;AAEA,oFAAoF,GACpF,OAAO,SAASE,kBAAkBC,GAAQ,EAAEC,OAA0B;IACpE,IAAID,IAAIE,QAAQ,KAAK,YAAYF,IAAIG,QAAQ,IAAIH,IAAII,QAAQ,EAAE,OAAO;IACtE,IAAIJ,IAAIK,IAAI,IAAIL,IAAIK,IAAI,KAAK,OAAO,OAAO;IAC3C,IAAI,CAACJ,QAAQK,QAAQ,CAACN,IAAIO,MAAM,GAAG,OAAO;IAC1C,IAAIf,mBAAmBgB,IAAI,CAACR,IAAIS,QAAQ,GAAG,OAAO;IAClD,MAAMC,QAAQ,IAAIC,IAAIX,IAAIY,QAAQ;IAClCF,MAAMG,IAAI,GAAG;IACb,KAAK,MAAMC,OAAO;WAAIJ,MAAMK,YAAY,CAACC,IAAI;KAAG,CAAE;QAChD,IAAIvB,eAAee,IAAI,CAACM,MAAMJ,MAAMK,YAAY,CAACE,MAAM,CAACH;IAC1D;IACAJ,MAAMQ,MAAM,GAAGR,MAAMK,YAAY,CAACH,QAAQ,KAAK,CAAC,CAAC,EAAEF,MAAMK,YAAY,CAACH,QAAQ,IAAI,GAAG;IACrF,OAAOF,MAAME,QAAQ;AACvB;AAEA,SAASO,cAAcC,MAAyB;IAC9C,OAAQA,OAAOC,OAAO;QACpB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,mFAAmF,GACnF,OAAO,eAAeC,mBACpBC,OAA+B,EAC/BC,QAA8B,CAAC,CAAC;QAEpBA,YACMA,cACID,iBA+BDA,uBACCA,wBACUA;QAGhCA,iBACIA;IAvCJ,MAAME,OAAMD,aAAAA,MAAMC,GAAG,YAATD,aAAaE,KAAKD,GAAG;IACjC,MAAME,aAAYH,eAAAA,MAAMI,KAAK,YAAXJ,eAAerD;IACjC,MAAMyB,SAASiC,QAAON,kBAAAA,QAAQ3B,MAAM,YAAd2B,kBAAkB,IAAIO,IAAI,GAAGC,WAAW,GAAGjC,OAAO,CAAC,QAAQ,IAAIA,OAAO,CAAC,UAAU;IACvG,MAAMG,UAAUP,eAAec,IAAI,CAACZ,UAAUD,kBAAkBC,UAAU,EAAE;IAC5E,MAAMoC,YAAYP;IAClB,MAAMQ,YAAiC;QACrCrC;QACAK;QACAiC,SAAS;QACTC,OAAO,EAAE;QACTC,SAAS,EAAE;QACXC,iBAAiB;QACjBC,QAAQ;YAAEC,OAAO;YAAOC,UAAU,EAAE;QAAC;QACrCC,aAAa;QACbC,WAAW;QACXC,WAAW;QACXX;QACAY,YAAYZ;IACd;IACA,MAAMa,SAAS;QACbZ,UAAUW,UAAU,GAAGnB;QACvB,OAAOQ;IACT;IACA,IAAI,CAAChC,QAAQ6C,MAAM,EAAE;QACnBb,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IACA,IAAI5E,sBAAsB2B,SAAS;QACjCqC,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IAEA,MAAME,WAAWC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAAC3B,QAAQwB,QAAQ;IACxD,MAAMhE,gBAAewC,wBAAAA,QAAQxC,YAAY,YAApBwC,wBAAwBzC,qBAAqBC,YAAY;IAC9E,MAAMC,iBAAgBuC,yBAAAA,QAAQvC,aAAa,YAArBuC,yBAAyBzC,qBAAqBE,aAAa;IACjF,MAAMmE,aAAanB,cAAaT,wBAAAA,QAAQtC,YAAY,YAApBsC,wBAAwBzC,qBAAqBG,YAAY;IACzF,MAAMmE,aAAa,IAAIC;IACvB,MAAMC,UAAU,IAAMF,WAAWG,KAAK;KACtChC,kBAAAA,QAAQiC,MAAM,qBAAdjC,gBAAgBkC,gBAAgB,CAAC,SAASH;IAC1C,KAAI/B,mBAAAA,QAAQiC,MAAM,qBAAdjC,iBAAgBmC,OAAO,EAAEN,WAAWG,KAAK;IAC7C,MAAMI,QAAQC,WAAW,IAAMR,WAAWG,KAAK,IAAIP,KAAKC,GAAG,CAAC,GAAGE,aAAanB;IAE5E,MAAM6B,WAAW,CAAC7D,MAAaC,QAAQK,QAAQ,CAACN,IAAIO,MAAM,KAAKP,IAAIE,QAAQ,KAAK;IAChF,MAAM4D,iBAAiB,IAAM9E,gBAAgBiD,UAAUU,SAAS;IAChE,MAAMoB,YAAY,IAAMX,WAAWI,MAAM,CAACE,OAAO,IAAIjC,SAAS0B;IAC9D,MAAMa,UAAU,CAAChE,KAAaiE,QAAgBC,cAAsBC,WAClExC,UAAU3B,KAAK;YACbmE;YACAF;YACAC;YACAE,WAAWvF;YACX2E,QAAQJ,WAAWI,MAAM;YACzBa,WAAWrB,KAAKC,GAAG,CAAC,GAAGD,KAAKsB,GAAG,CAACxF,qBAAqBI,gBAAgB,EAAEiE,aAAa1B;YACpF8C,iBAAiB,CAACC,OAAOC,KAAOZ,SAASY;YACzCC,YAAY;QACd;IACF,MAAMC,OAAO,CAACC;QACZ,IAAI3C,UAAUG,OAAO,CAACU,MAAM,GAAGhE,qBAAqBS,UAAU,EAAE0C,UAAUG,OAAO,CAACyC,IAAI,CAACD;aAClF3C,UAAUI,eAAe,IAAI;IACpC;IAEA,IAAI;YAgBuCC,gBAA+BA;QAfxE,yDAAyD;QACzD,MAAMwC,OAAO,GAAG7E,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,IAAI8E,QAAsB,EAAE;QAC5B,MAAMzC,SAAS,MAAM0B,QACnB,GAAG/D,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAC1B,8BACA,cACA+C,KAAKsB,GAAG,CAACxF,qBAAqBK,cAAc,EAAE2E;QAEhD,IAAIxB,OAAO0C,EAAE,EAAE;YACb/C,UAAUU,SAAS,IAAIL,OAAO2C,KAAK;YACnC,MAAMC,SAAS3G,eAAe+D,OAAO6C,IAAI;YACzCJ,QAAQrG,eAAewG,QAAQ5G;YAC/B2D,UAAUK,MAAM,GAAG;gBAAEC,OAAO;gBAAMC,UAAU0C,OAAO1C,QAAQ;YAAC;QAC9D,OAAO,IACL,CAAEF,CAAAA,OAAOjB,OAAO,KAAK,iBAAiB,EAACiB,iBAAAA,OAAO8C,MAAM,YAAb9C,iBAAiB,MAAM,OAAO,EAACA,kBAAAA,OAAO8C,MAAM,YAAb9C,kBAAiB,KAAK,GAAE,GAC9F;YACAL,UAAUS,SAAS,GAAGqB,cAAc,SAAS;YAC7C,OAAOlB;QACT;QACA,MAAMwC,UAAU,CAACvE;YACf,MAAMd,MAAM,IAAIW,IAAIG;YACpB,OAAOrC,aAAasG,OAAO,GAAG/E,IAAIS,QAAQ,GAAGT,IAAIkB,MAAM,EAAE;QAC3D;QAEA,kEAAkE;QAClE,MAAMoE,OAAO/D,QAAQ+D,IAAI,GACrB/D,QAAQ+D,IAAI,CACTC,GAAG,CAAC,CAACC;YACJ,IAAI;gBACF,OAAOzF,kBAAkB,IAAIY,IAAI6E,MAAMvF;YACzC,EAAE,eAAM;gBACN,OAAO;YACT;QACF,GACCwF,MAAM,CAAC,CAAC3E,MAAuB,CAAC,CAACA,OACpC;QAEJ,uEAAuE;QACvE,MAAM4E,eAAyB,EAAE;QACjC,KAAK,MAAMF,OAAOvD,UAAUK,MAAM,CAACE,QAAQ,CAAE;YAC3C,IAAI;gBACF,MAAMxC,MAAM,IAAIW,IAAI6E;gBACpB,IAAI3B,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIY,QAAQ;YACnD,EAAE,eAAM;YACN,0CAA0C;YAC5C;QACF;QACA,IAAI,CAAC8E,aAAa5C,MAAM,IAAI,CAACwC,MAAMI,aAAab,IAAI,CAAC,GAAG5E,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC;QAChF,IAAIqF,MAAMI,aAAa5C,MAAM,GAAG;QAChC,MAAM6C,cAAc,IAAIC;QACxB,MAAMC,cAAwB,EAAE;QAChC,IAAIC,eAAe;QACnB,MAAOJ,aAAa5C,MAAM,IAAIgD,eAAehH,qBAAqBO,WAAW,IAAI,CAAC0E,YAAa;YAC7F,MAAMgC,OAAOL,aAAaM,KAAK;YAC/B,IAAIL,YAAYM,GAAG,CAACF,OAAO;YAC3BJ,YAAYO,GAAG,CAACH;YAChB,IAAI,SAASvF,IAAI,CAAC,IAAIG,IAAIoF,MAAMtF,QAAQ,GAAG;YAC3C,IAAI,CAAC4E,QAAQU,OAAO;YACpB,MAAMI,OAAOnD,KAAKsB,GAAG,CAACxF,qBAAqBM,eAAe,EAAE0E;YAC5D,IAAIqC,QAAQ,GAAG;YACfL,gBAAgB;YAChB,MAAMM,UAAU,MAAMpC,QAAQ+B,MAAM,oBAAoB,mCAAmCI;YAC3F,IAAI,CAACC,QAAQpB,EAAE,EAAE;YACjB/C,UAAUU,SAAS,IAAIyD,QAAQnB,KAAK;YACpC,MAAMoB,UAAU7H,aAAa4H,QAAQjB,IAAI;YACzC,KAAK,MAAMmB,YAAYD,QAAQE,SAAS,CAAE;gBACxC,IAAIvG;gBACJ,IAAI;oBACFA,MAAM,IAAIW,IAAI2F;gBAChB,EAAE,eAAM;oBACN;gBACF;gBACA,IAAID,QAAQG,KAAK,EAAE;oBACjB,IAAI3C,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIY,QAAQ;oBACjD;gBACF;gBACA,MAAME,MAAMf,kBAAkBC,KAAKC;gBACnC,IAAIa,OAAO+E,YAAY/C,MAAM,GAAGhE,qBAAqBQ,cAAc,EAAEuG,YAAYhB,IAAI,CAAC/D;YACxF;QACF;QACAmB,UAAUQ,WAAW,GAAGoD,YAAY/C,MAAM;QAC1C,IAAIvB,QAAQkF,YAAY,EAAE;YACxB,MAAMC,QAAQ,CAAC5F,MAAgB,IAAIH,IAAIG,KAAKL,QAAQ,CAACkG,KAAK,CAAC,KAAKlB,MAAM,CAACmB,SAAS9D,MAAM;YACtF,MAAM+D,SAAShB,YAAYN,GAAG,CAAC,CAACzE,KAAKgG,QAAW,CAAA;oBAAEhG;oBAAKgG;oBAAOJ,OAAOA,MAAM5F;gBAAK,CAAA;YAChF+F,OAAOE,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEN,KAAK,GAAGO,EAAEP,KAAK,IAAIM,EAAEF,KAAK,GAAGG,EAAEH,KAAK;YAC5DjB,YAAYqB,MAAM,CAAC,GAAGrB,YAAY/C,MAAM,KAAK+D,OAAOtB,GAAG,CAAC,CAACX,QAAUA,MAAM9D,GAAG;QAC9E;QAEA,yEAAyE;QACzE,wCAAwC;QACxC,MAAMqG,QAA+D7B,OACjEA,KAAKC,GAAG,CAAC,CAACzE,MAAS,CAAA;gBAAEA;gBAAKsG,SAAU,IAAIzG,IAAIG,KAAKL,QAAQ,KAAK,MAAM,SAAS;YAAqC,CAAA,KAClH;YAAC;gBAAEK,KAAKgE;gBAAMsC,SAAS;YAAO;SAAE;QACpC,MAAMC,SAAS,IAAIzB,IAAYN,eAAAA,OAAQ;YAACR;SAAK;QAC7C,KAAK,MAAMhE,OAAO+E,YAAa;YAC7B,IAAIwB,OAAOpB,GAAG,CAACnF,MAAM;YACrBuG,OAAOnB,GAAG,CAACpF;YACXqG,MAAMtC,IAAI,CAAC;gBAAE/D;gBAAKsG,SAAS;YAAU;QACvC;QACA,MAAME,OAAO,IAAI1B;QACjB,uEAAuE;QACvE,qEAAqE;QACrE,6DAA6D;QAC7D,0EAA0E;QAC1E,IAAI2B,YAAYjC,CAAAA,wBAAAA,KAAMxC,MAAM,IAAG,IAAInC,IAAI2E,IAAI,CAAC,EAAE,EAAE/E,MAAM,GAAGN,OAAO,CAAC,EAAE;QACnE,MAAMuH,QAAQ,CAAChC;YACb,MAAMxF,MAAM,IAAIW,IAAI6E;YACpB,OAAOvF,QAAQK,QAAQ,CAACN,IAAIO,MAAM,KAAKP,IAAIO,MAAM,KAAKgH,YAClD,GAAGA,YAAYvH,IAAIS,QAAQ,GAAGT,IAAIkB,MAAM,EAAE,GAC1CsE;QACN;QACA,MAAMiC,cAAc1E,WAAW,IAAI;QACnC,IAAI2E,WAAW;QACf,MAAOP,MAAMrE,MAAM,CAAE;gBAgDI/C;YA/CvB,IAAIkC,UAAUE,KAAK,CAACW,MAAM,IAAIC,UAAU;gBACtCd,UAAUS,SAAS,GAAG;gBACtB;YACF;YACA,IAAIqB,aAAa;gBACf9B,UAAUS,SAAS,GAAG;gBACtB;YACF;YACA,IAAIoB,oBAAoB,GAAG;gBACzB7B,UAAUS,SAAS,GAAG;gBACtB;YACF;YACA,IAAIgF,YAAYD,aAAa;gBAC3BxF,UAAUS,SAAS,GAAG;gBACtB;YACF;YACA,MAAMqD,OAAOoB,MAAMnB,KAAK;YACxB,MAAMlF,MAAM0G,MAAMzB,KAAKjF,GAAG;YAC1B,MAAMsG,UAAUrB,KAAKqB,OAAO;YAC5B,IAAIE,KAAKrB,GAAG,CAACnF,MAAM;YACnB,IAAI,CAACuE,QAAQvE,MAAM;gBACjB6D,KAAK;oBAAE3E,KAAKc;oBAAK6G,QAAQ;gBAAS;gBAClC;YACF;YACAD,YAAY;YACZJ,KAAKpB,GAAG,CAACpF;YACT,MAAM8G,MAAM5E,KAAKsB,GAAG,CAACvF,cAAc+E;YACnC,MAAM+D,OAAO,MAAM7D,QAAQlD,KAAK,uCAAuC,0CAA0C8G;YACjH,IAAI,CAACC,KAAK7C,EAAE,EAAE;gBACZ,IAAI6C,KAAKxG,OAAO,KAAK,eAAeuG,MAAM7I,cAAc;oBACtDkD,UAAUS,SAAS,GAAG;oBACtB;gBACF;gBACA,IAAImF,KAAKxG,OAAO,KAAK,aAAa0C,aAAa;oBAC7C9B,UAAUS,SAAS,GAAG;oBACtB;gBACF;gBACAiC,KAAK;oBACH3E,KAAKc;oBACL6G,QAAQxG,cAAc0G;mBAClBA,KAAKzC,MAAM,GAAG;oBAAEA,QAAQyC,KAAKzC,MAAM;gBAAC,IAAI,CAAC,GACzCyC,KAAKvB,QAAQ,GAAG;oBAAEA,UAAUuB,KAAKvB,QAAQ;gBAAC,IAAI,CAAC;gBAErD;YACF;YACArE,UAAUU,SAAS,IAAIkF,KAAK5C,KAAK;YACjC,IAAImC,YAAY,QAAQG,YAAY,IAAI5G,IAAIkH,KAAK7H,GAAG,EAAEO,MAAM;YAC5D,MAAMuH,WAAWN,OAAMzH,qBAAAA,kBAAkB,IAAIY,IAAIkH,KAAK7H,GAAG,GAAGC,oBAArCF,qBAAiD8H,KAAK7H,GAAG;YAChF,IAAI8H,aAAahH,KAAK;gBACpB,IAAIwG,KAAKrB,GAAG,CAAC6B,WAAW;gBACxBR,KAAKpB,GAAG,CAAC4B;gBACT,IAAI,CAACzC,QAAQyC,WAAW;oBACtBnD,KAAK;wBAAE3E,KAAK8H;wBAAUH,QAAQ;oBAAS;oBACvC;gBACF;YACF;YACA,MAAMI,UAAU3J,sBAAsByJ,KAAK1C,IAAI,EAAE0C,KAAK7H,GAAG,EAAE6D;YAC3D,IAAIkE,QAAQC,SAAS,EAAE;gBACrBrD,KAAK;oBAAE3E,KAAK8H;oBAAUH,QAAQ;gBAAa;gBAC3C;YACF;YACA,MAAMM,QAAkB,EAAE;YAC1B,KAAK,MAAMC,QAAQH,QAAQE,KAAK,CAAE;gBAChC,MAAME,UAAUpI,kBAAkB,IAAIY,IAAIuH,OAAOjI;gBACjD,IAAI,CAACkI,SAAS;gBACdF,MAAMpD,IAAI,CAACsD;gBACX,IAAI,CAAC7C,QAAQ,CAAC+B,OAAOpB,GAAG,CAACkC,UAAU;oBACjCd,OAAOnB,GAAG,CAACiC;oBACXhB,MAAMtC,IAAI,CAAC;wBAAE/D,KAAKqH;wBAASf,SAAS;oBAAO;gBAC7C;YACF;YACAnF,UAAUE,KAAK,CAAC0C,IAAI,CAAC;gBACnB7E,KAAK8H;gBACLM,MAAM,IAAIzH,IAAImH,UAAUrH,QAAQ;gBAChC2G;gBACAiB,OAAON,QAAQM,KAAK;gBACpBC,aAAaP,QAAQO,WAAW;gBAChCC,MAAMR,QAAQQ,IAAI;gBAClBC,WAAWT,QAAQS,SAAS;gBAC5BC,QAAQV,QAAQU,MAAM;gBACtBC,QAAQX,QAAQW,MAAM;gBACtBC,UAAUZ,QAAQY,QAAQ;gBAC1BV,OAAO;uBAAI,IAAIrC,IAAIqC,MAAM1C,GAAG,CAACiC;iBAAQ;gBACrCoB,OAAOb,QAAQa,KAAK;gBACpB3D,OAAO4C,KAAK5C,KAAK;YACnB;QACF;QACA,OAAOpC;IACT,SAAU;YAERtB;QADAsH,aAAalF;SACbpC,mBAAAA,QAAQiC,MAAM,qBAAdjC,iBAAgBuH,mBAAmB,CAAC,SAASxF;IAC/C;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAeyF,yBACpBnJ,MAAc,EACdoJ,QAAgB,EAChBxH,QAA8B,CAAC,CAAC;QAEdA,cAkBLJ;IAlBb,MAAMO,aAAYH,eAAAA,MAAMI,KAAK,YAAXJ,eAAerD;IACjC,MAAM0B,OAAOgC,OAAOjC,iBAAAA,SAAU,IAAIkC,IAAI,GAAGC,WAAW,GAAGjC,OAAO,CAAC,UAAU;IACzE,IAAI,CAACJ,eAAec,IAAI,CAACX,SAAS5B,sBAAsB4B,OAAO,OAAO;QAAEoJ,UAAU;QAAOC,QAAQ,EAAE;IAAC;IACpG,MAAMjJ,UAAUN,kBAAkBE;IAClC,MAAMuB,SAAS,MAAMO,UAAU,GAAG1B,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;QAC/CkE,UAAUrF,qBAAqBC,YAAY;QAC3CkF,QAAQ;QACRC,cAAc;QACdE,WAAWvF;QACXwF,WAAWvF,qBAAqBI,gBAAgB;QAChDqF,iBAAiB,CAACC,OAAOC,KAAOA,GAAGvE,QAAQ,KAAK,YAAYD,QAAQK,QAAQ,CAACmE,GAAGlE,MAAM;IACxF;IACA,IAAI,CAACa,OAAO4D,EAAE,EAAE;QACd,qEAAqE;QACrE,oDAAoD;QACpD,MAAMiE,WAAW7H,OAAOC,OAAO,KAAK,iBAAiBD,OAAOC,OAAO,KAAK,sBAAsBD,OAAOC,OAAO,KAAK;QACjH,OAAO;YAAE4H;YAAUC,QAAQ,EAAE;YAAElJ,KAAKoB,OAAOpB,GAAG;QAAC;IACjD;IACA,MAAMmJ,QAAO/H,sBAAAA,OAAO+D,IAAI,CAACwB,KAAK,CAAC,WAAW,CAAC,EAAE,YAAhCvF,sBAAoC;IACjD,OAAO;QAAE6H,UAAU;QAAMC,QAAQ7K,eAAe8K,MAAMH;QAAWhJ,KAAKoB,OAAOpB,GAAG;IAAC;AACnF;AAEA;;;CAGC,GACD,OAAO,SAASoJ,yBAAyBvB,IAAoB,EAAEwB,WAAW,KAAM;IAC9E,MAAMC,QAAkB,EAAE;IAC1B,IAAIzB,KAAKQ,KAAK,EAAEiB,MAAMzE,IAAI,CAAC,CAAC,OAAO,EAAEgD,KAAKQ,KAAK,EAAE;IACjD,IAAIR,KAAKS,WAAW,EAAEgB,MAAMzE,IAAI,CAAC,CAAC,aAAa,EAAEgD,KAAKS,WAAW,EAAE;IACnE,KAAK,MAAMiB,SAAS1B,KAAKY,MAAM,CAAE;QAC/B,IAAIc,MAAMC,IAAI,KAAK,WAAWF,MAAMzE,IAAI,CAAC,GAAG,IAAI4E,MAAM,CAACF,MAAMG,KAAK,EAAE,CAAC,EAAEH,MAAMpE,IAAI,EAAE;aAC9E,IAAIoE,MAAMC,IAAI,KAAK,SAASF,MAAMzE,IAAI,CAAC,CAAC,QAAQ,EAAE0E,MAAMI,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAClF,IAAIJ,MAAMC,IAAI,KAAK,aAAaF,MAAMzE,IAAI,CAAC,CAAC,EAAE,EAAE0E,MAAMpE,IAAI,EAAE;aAC5D,IAAIoE,MAAMC,IAAI,KAAK,SAASF,MAAMzE,IAAI,CAAC,CAAC,EAAE,EAAE0E,MAAMpE,IAAI,EAAE;aACxDmE,MAAMzE,IAAI,CAAC0E,MAAMpE,IAAI;IAC5B;IACA,OAAOjH,mBAAmB,CAAC,KAAK,EAAE2J,KAAKO,IAAI,CAAC,2BAA2B,CAAC,EAAEkB,MAAMM,IAAI,CAAC,OAAO;QAAEP;IAAS;AACzG"}
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-crawl.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * THE SITE-IMPORT CRAWLER (AGL-3974, parent AGL-3973).\n *\n * Reads a website a workspace has PROVED it owns (domain-ownership.ts in\n * tenant-data-admin; the doors check the proof before calling this) into a\n * typed inventory: each page's title, description, headings and text in\n * order, images and menu links. It writes nothing; the import job decides\n * what becomes a page.\n *\n * Every request goes through `fetchPublicTextDetailed`, the theme job's\n * SSRF-guarded fetch: https on the default port only, public addresses only,\n * the socket pinned to the address checked. On top of that:\n *\n * - **One site.** The site is `https://<domain>` and `https://www.<domain>`,\n *   one site to its owner. Every redirect hop and every discovered link must\n *   stay on it; a page that redirects anywhere else is refused, not followed.\n * - **AglynBot.** Every request says `AglynBot/1.0 (+<the operator's bot page>)`,\n *   and robots.txt is read first and honored for every page: the AglynBot\n *   group, else `*`. A robots.txt that answers 404 allows everything; one\n *   that cannot be read at all (5xx, no answer) stops the crawl, because\n *   \"could not ask\" is not \"allowed\".\n * - **Our own edge.** A site this platform serves gets the edge secret\n *   (`site-import-edge.ts`, AGL-4085), which lifts our bot challenge for\n *   AglynBot; it is asked per request and per redirect hop, and a request\n *   that leaves the site's two origins never carries it.\n * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an\n *   index's children included), then same-site links found on the pages read,\n *   for whatever the sitemap missed.\n * - **Budgets.** Pages (the caller's, from the plan), bytes per page (a page\n *   over it is refused, not cut), bytes in total, and wall-clock time. A\n *   whole site's worth fits (AGL-4083): aglyn.com's 123 pages average 440 KB,\n *   54 MB in all, so the total is 160 MB; a page is parsed and dropped as it\n *   is read, so what is held is its blocks, never its bytes.\n * - **Four at a time.** robots.txt, the sitemaps and the home page are read\n *   one after another; the rest four at a time, and the inventory keeps the\n *   order they were asked in, which is the sitemap's.\n * - **Nothing vanishes.** An address a stop (the clock, the bytes, the page\n *   budget) left unread is listed as `not-reached`, so the inventory's\n *   counts add up to the site. A wave's own read leaves its unread pages for\n *   its next pass instead.\n * - **No lookalikes.** A domain that wears a brand it is not\n *   (`lookalikeBrandForHost`) is refused before any request.\n * - **No sign-in pages.** A page with a password field is refused and\n *   reported; forms with payment fields, scripts, iframes and embeds never\n *   reach the inventory (site-import-html.ts).\n *=========================================*/\n\nimport { lookalikeBrandForHost } from '@aglyn/shared-util-email/outbound-phishing-screen'\nimport { fenceUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n  fetchPublicTextDetailed,\n  type PublicFetchEdge,\n  type PublicFetchOptions,\n  type PublicFetchResult,\n} from '../jobs/ai-theme-brand-inputs'\nimport {\n  extractSiteImportPage,\n  readMetaValues,\n  type SiteImportBlock,\n  type SiteImportBrandMark,\n  type SiteImportForm,\n  type SiteImportIcon,\n  type SiteImportImage,\n  type SiteImportNavLink,\n  type SiteImportOrganization,\n} from './site-import-html'\nimport {\n  AGLYN_BOT_TOKEN,\n  parseRobotsTxt,\n  parseSitemap,\n  robotsAllows,\n  robotsRulesFor,\n  type RobotsRule,\n} from './site-import-robots'\nimport { NO_SITE_IMPORT_HEADERS, siteImportEdgeHeadersFor, type SiteImportHeadersFor } from './site-import-edge'\n\n/**\n * Where the bot's user agent points a site owner who wants to know what it is\n * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on\n * a self-hosted deployment, so its crawler never names Aglyn's address; the\n * literal is only that variable's default.\n *\n * Aglyn's is the Bring my site guide, which says what AglynBot reads and how\n * robots.txt keeps it out. There is no page of its own (Zach, 10/20: \"we don't\n * need a bot page at all\"), and the site it reads is the reader's own: an\n * import starts only once its owner has proved the site is theirs.\n */\nexport const AGLYN_BOT_INFO_URL =\n  process.env['SITE_IMPORT_BOT_INFO_URL'] ?? 'https://docs.aglyn.com/ai/bring-my-site'\n\n/** What every crawler request says it is. */\nexport const AGLYN_BOT_USER_AGENT = `AglynBot/1.0 (+${AGLYN_BOT_INFO_URL})`\n\nexport const SITE_IMPORT_DEFAULTS = {\n  maxPageBytes: 2 * 1024 * 1024,\n  /** A whole site's worth (AGL-4083): 54 MB for aglyn.com's 123 pages, with room. */\n  maxTotalBytes: 160 * 1024 * 1024,\n  timeBudgetMs: 60_000,\n  /** One request's own clock, inside the whole crawl's. */\n  requestTimeoutMs: 8_000,\n  robotsMaxBytes: 256 * 1024,\n  sitemapMaxBytes: 4 * 1024 * 1024,\n  /** Sitemap documents read, an index's children included: an index of six children is seven. */\n  maxSitemaps: 24,\n  /** URLs a sitemap may contribute to the queue. */\n  maxSitemapUrls: 2_000,\n  /** Reasons kept in `skipped` before the list stops growing: every address a sitemap may name. */\n  maxSkipped: 2_000,\n  /** Pages read at once, after robots.txt, the sitemaps and the home page. */\n  concurrency: 4,\n} as const\n\n/** Never a page: files a link may point at that are not HTML. */\nconst NON_PAGE_EXTENSION =\n  /\\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i\n\nconst TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i\n\nconst DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\\.[a-z0-9-]{1,63})+$/\n\nexport type SiteImportSkipReason =\n  /** The page has a password field: a sign-in page, refused outright. */\n  | 'login-form'\n  /** robots.txt keeps AglynBot off this path. */\n  | 'robots'\n  /** A redirect left the site. */\n  | 'off-origin-redirect'\n  /** Over the per-page byte cap. */\n  | 'too-large'\n  /** Answered with something that is not HTML. */\n  | 'not-html'\n  /** Answered 4xx or 5xx. */\n  | 'http-error'\n  /** No answer, a private address, or a refused address. */\n  | 'unreachable'\n  /** Never asked for: a stop (time, bytes, the page budget) came first (AGL-4083). */\n  | 'not-reached'\n\nexport interface SiteImportSkipped {\n  url: string\n  reason: SiteImportSkipReason\n  status?: number\n  /** Where an off-origin redirect pointed. */\n  location?: string\n  /** Whose edge turned an error answer away, when it said (AGL-4083). */\n  edge?: PublicFetchEdge\n}\n\nexport interface SiteImportPage {\n  /** The address the page was read from, after any same-site redirect. */\n  url: string\n  path: string\n  /** Where the crawler learned of the page. */\n  foundBy: 'home' | 'sitemap' | 'link'\n  title: string\n  description: string\n  lang: string | null\n  canonical: string | null\n  /** The share card (AGL-4083). */\n  ogImage: string | null\n  /** The site's own name, as its head gives it. */\n  siteName: string\n  icons: SiteImportIcon[]\n  /** The header's link home: the brand's logo and words. */\n  brand: SiteImportBrandMark | null\n  organization: SiteImportOrganization | null\n  blocks: SiteImportBlock[]\n  images: SiteImportImage[]\n  navLinks: SiteImportNavLink[]\n  /** Same-site page addresses this page links to. */\n  links: string[]\n  forms: { kept: SiteImportForm[]; removed: number }\n  bytes: number\n}\n\nexport type SiteImportStop =\n  | 'page-budget'\n  | 'total-bytes'\n  | 'time'\n  /** robots.txt could not be read, so nothing was asked of the site. */\n  | 'robots-unreachable'\n\nexport interface SiteImportInventory {\n  domain: string\n  /** The site's origins: the bare name and its `www.` twin. */\n  origins: string[]\n  /** Set when nothing was fetched at all. */\n  refused: 'invalid-domain' | 'lookalike' | null\n  pages: SiteImportPage[]\n  skipped: SiteImportSkipped[]\n  /** Skips past {@link SITE_IMPORT_DEFAULTS.maxSkipped}, counted only. */\n  skippedOverflow: number\n  robots: { found: boolean; sitemaps: string[] }\n  /** Page addresses the sitemaps listed (same-site, before robots and budget). */\n  sitemapUrls: number\n  stoppedBy: SiteImportStop | null\n  bytesRead: number\n  startedAt: number\n  finishedAt: number\n}\n\nexport interface SiteImportCrawlOptions {\n  /** The proved domain; `www.` is folded in. */\n  domain: string\n  /** The most pages the inventory may hold: the plan's page count. */\n  maxPages: number\n  maxPageBytes?: number\n  maxTotalBytes?: number\n  timeBudgetMs?: number\n  signal?: AbortSignal\n  /**\n   * Read the sitemap's shallow addresses before its deep ones (AGL-3976), so a\n   * crawl the page budget stops reads a site's main pages before the tail of\n   * its blog. A sitemap's own order is kept among addresses of one depth.\n   */\n  shallowFirst?: boolean\n  /**\n   * Read exactly these addresses of the site, in order, and nothing else\n   * (AGL-3976): an import's wave reads its own pages again when it starts. No\n   * sitemap is read and no link is followed; robots.txt is read and honored\n   * as for any crawl, and every other rule holds.\n   */\n  only?: readonly string[]\n}\n\nexport interface SiteImportCrawlSeams {\n  fetch?: (url: string, options: PublicFetchOptions) => Promise<PublicFetchResult>\n  now?: () => number\n  /**\n   * Request headers for the site being read, asked once per crawl with its\n   * domain (AGL-4083, AGL-4085). Absent, the crawl asks the edge\n   * (`siteImportEdgeHeadersFor`): Aglyn's edge secret for each of the site's\n   * two origins this platform serves, decided origin by origin, and nothing\n   * anywhere else. Either way the headers go only to the site's two origins,\n   * and the fetch asks again on every redirect hop, so a hop anywhere else\n   * carries nothing. Always under the crawler's own `Accept` and `User-Agent`.\n   */\n  headersFor?: (domain: string) => Promise<Readonly<Record<string, string>>> | Readonly<Record<string, string>>\n}\n\n/**\n * The per-request headers of a crawl of `domain`: the seam's, else the edge's,\n * and none for any address off the site's two origins. Never throws.\n */\nasync function siteRequestHeaders(\n  seams: SiteImportCrawlSeams,\n  domain: string,\n  origins: readonly string[],\n): Promise<SiteImportHeadersFor> {\n  let inner: SiteImportHeadersFor = NO_SITE_IMPORT_HEADERS\n  try {\n    if (seams.headersFor) {\n      const headers = await seams.headersFor(domain)\n      if (headers && Object.keys(headers).length) inner = () => headers\n    } else {\n      inner = await siteImportEdgeHeadersFor(origins)\n    }\n  } catch (error) {\n    console.warn('[site-import] the edge headers could not be read; reading without them', { domain, error })\n  }\n  return (url: URL) => (origins.includes(url.origin) ? inner(url) : {})\n}\n\n/** The site's two origins for a domain. */\nexport function siteImportOrigins(domain: string): string[] {\n  const bare = domain.replace(/^www\\./, '')\n  return [`https://${bare}`, `https://www.${bare}`]\n}\n\n/** A crawlable page address on the site, normalized for de-duplication; else null. */\nexport function siteImportPageKey(url: URL, origins: readonly string[]): string | null {\n  if (url.protocol !== 'https:' || url.username || url.password) return null\n  if (url.port && url.port !== '443') return null\n  if (!origins.includes(url.origin)) return null\n  if (NON_PAGE_EXTENSION.test(url.pathname)) return null\n  const clean = new URL(url.toString())\n  clean.hash = ''\n  for (const key of [...clean.searchParams.keys()]) {\n    if (TRACKING_PARAM.test(key)) clean.searchParams.delete(key)\n  }\n  clean.search = clean.searchParams.toString() ? `?${clean.searchParams.toString()}` : ''\n  return clean.toString()\n}\n\nfunction skipReasonFor(result: PublicFetchResult): SiteImportSkipReason {\n  switch (result.refusal) {\n    case 'redirect-refused':\n      return 'off-origin-redirect'\n    case 'content-type':\n      return 'not-html'\n    case 'too-large':\n      return 'too-large'\n    case 'http-status':\n      return 'http-error'\n    default:\n      return 'unreachable'\n  }\n}\n\n/** Read a proved site into an inventory. Never throws for anything the site does. */\nexport async function crawlSiteForImport(\n  options: SiteImportCrawlOptions,\n  seams: SiteImportCrawlSeams = {},\n): Promise<SiteImportInventory> {\n  const now = seams.now ?? Date.now\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const domain = String(options.domain ?? '').trim().toLowerCase().replace(/\\.+$/, '').replace(/^www\\./, '')\n  const origins = DOMAIN_PATTERN.test(domain) ? siteImportOrigins(domain) : []\n  const startedAt = now()\n  const inventory: SiteImportInventory = {\n    domain,\n    origins,\n    refused: null,\n    pages: [],\n    skipped: [],\n    skippedOverflow: 0,\n    robots: { found: false, sitemaps: [] },\n    sitemapUrls: 0,\n    stoppedBy: null,\n    bytesRead: 0,\n    startedAt,\n    finishedAt: startedAt,\n  }\n  const finish = () => {\n    inventory.finishedAt = now()\n    return inventory\n  }\n  if (!origins.length) {\n    inventory.refused = 'invalid-domain'\n    return finish()\n  }\n  if (lookalikeBrandForHost(domain)) {\n    inventory.refused = 'lookalike'\n    return finish()\n  }\n\n  // Asked once, before anything else, and held to the site's two origins.\n  const headersFor = await siteRequestHeaders(seams, domain, origins)\n  const maxPages = Math.max(0, Math.floor(options.maxPages))\n  const maxPageBytes = options.maxPageBytes ?? SITE_IMPORT_DEFAULTS.maxPageBytes\n  const maxTotalBytes = options.maxTotalBytes ?? SITE_IMPORT_DEFAULTS.maxTotalBytes\n  const deadlineAt = startedAt + (options.timeBudgetMs ?? SITE_IMPORT_DEFAULTS.timeBudgetMs)\n  const controller = new AbortController()\n  const onAbort = () => controller.abort()\n  options.signal?.addEventListener('abort', onAbort)\n  if (options.signal?.aborted) controller.abort()\n  const timer = setTimeout(() => controller.abort(), Math.max(0, deadlineAt - startedAt))\n\n  const sameSite = (url: URL) => origins.includes(url.origin) && url.protocol === 'https:'\n  const remainingBytes = () => maxTotalBytes - inventory.bytesRead\n  const outOfTime = () => controller.signal.aborted || now() >= deadlineAt\n  const request = (url: string, accept: RegExp, acceptHeader: string, maxBytes: number) =>\n    fetchText(url, {\n      maxBytes,\n      accept,\n      acceptHeader,\n      userAgent: AGLYN_BOT_USER_AGENT,\n      headersFor,\n      signal: controller.signal,\n      timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),\n      redirectAllowed: (_from, to) => sameSite(to),\n      refuseOver: true,\n    })\n  const skip = (entry: SiteImportSkipped) => {\n    if (inventory.skipped.length < SITE_IMPORT_DEFAULTS.maxSkipped) inventory.skipped.push(entry)\n    else inventory.skippedOverflow += 1\n  }\n\n  try {\n    // robots.txt, before anything else is asked of the site.\n    const home = `${origins[0]}/`\n    let rules: RobotsRule[] = []\n    const robots = await request(\n      `${origins[0]}/robots.txt`,\n      /text\\/plain|text\\/html|^$/i,\n      'text/plain',\n      Math.min(SITE_IMPORT_DEFAULTS.robotsMaxBytes, remainingBytes()),\n    )\n    if (robots.ok) {\n      inventory.bytesRead += robots.bytes\n      const parsed = parseRobotsTxt(robots.text)\n      rules = robotsRulesFor(parsed, AGLYN_BOT_TOKEN)\n      inventory.robots = { found: true, sitemaps: parsed.sitemaps }\n    } else if (\n      !(robots.refusal === 'http-status' && (robots.status ?? 0) >= 400 && (robots.status ?? 0) < 500)\n    ) {\n      inventory.stoppedBy = outOfTime() ? 'time' : 'robots-unreachable'\n      return finish()\n    }\n    const allowed = (key: string) => {\n      const url = new URL(key)\n      return robotsAllows(rules, `${url.pathname}${url.search}`)\n    }\n\n    // A wave's own pages (AGL-3976): exactly those, after robots.txt.\n    const only = options.only\n      ? options.only\n          .map((raw) => {\n            try {\n              return siteImportPageKey(new URL(raw), origins)\n            } catch {\n              return null\n            }\n          })\n          .filter((key): key is string => !!key)\n      : null\n\n    // Sitemaps: the ones robots.txt names on this site, else /sitemap.xml.\n    const sitemapQueue: string[] = []\n    for (const raw of inventory.robots.sitemaps) {\n      try {\n        const url = new URL(raw)\n        if (sameSite(url)) sitemapQueue.push(url.toString())\n      } catch {\n        // A malformed Sitemap line names nothing.\n      }\n    }\n    if (!sitemapQueue.length && !only) sitemapQueue.push(`${origins[0]}/sitemap.xml`)\n    if (only) sitemapQueue.length = 0\n    const sitemapSeen = new Set<string>()\n    const fromSitemap: string[] = []\n    let sitemapsRead = 0\n    while (sitemapQueue.length && sitemapsRead < SITE_IMPORT_DEFAULTS.maxSitemaps && !outOfTime()) {\n      const next = sitemapQueue.shift() as string\n      if (sitemapSeen.has(next)) continue\n      sitemapSeen.add(next)\n      if (/\\.gz$/i.test(new URL(next).pathname)) continue\n      if (!allowed(next)) continue\n      const room = Math.min(SITE_IMPORT_DEFAULTS.sitemapMaxBytes, remainingBytes())\n      if (room <= 0) break\n      sitemapsRead += 1\n      const sitemap = await request(next, /xml|text\\/plain/i, 'application/xml, text/xml;q=0.9', room)\n      if (!sitemap.ok) continue\n      inventory.bytesRead += sitemap.bytes\n      const entries = parseSitemap(sitemap.text)\n      for (const location of entries.locations) {\n        let url: URL\n        try {\n          url = new URL(location)\n        } catch {\n          continue\n        }\n        if (entries.index) {\n          if (sameSite(url)) sitemapQueue.push(url.toString())\n          continue\n        }\n        const key = siteImportPageKey(url, origins)\n        if (key && fromSitemap.length < SITE_IMPORT_DEFAULTS.maxSitemapUrls) fromSitemap.push(key)\n      }\n    }\n    inventory.sitemapUrls = fromSitemap.length\n    if (options.shallowFirst) {\n      const depth = (key: string) => new URL(key).pathname.split('/').filter(Boolean).length\n      const ranked = fromSitemap.map((key, order) => ({ key, order, depth: depth(key) }))\n      ranked.sort((a, b) => a.depth - b.depth || a.order - b.order)\n      fromSitemap.splice(0, fromSitemap.length, ...ranked.map((entry) => entry.key))\n    }\n\n    // Pages: home, then the sitemap's, then links the pages themselves name.\n    // A wave's read is its own pages alone.\n    const queue: { key: string; foundBy: SiteImportPage['foundBy'] }[] = only\n      ? only.map((key) => ({ key, foundBy: (new URL(key).pathname === '/' ? 'home' : 'link') as SiteImportPage['foundBy'] }))\n      : [{ key: home, foundBy: 'home' }]\n    const queued = new Set<string>(only ?? [home])\n    for (const key of fromSitemap) {\n      if (queued.has(key)) continue\n      queued.add(key)\n      queue.push({ key, foundBy: 'sitemap' })\n    }\n    const read = new Set<string>()\n    // The site's two names are one site: once the home page says which one\n    // it answers on, every address is read on that one, so /about is not\n    // read twice as example.com/about and www.example.com/about.\n    // A wave's read starts on the name its pages were read on the first time.\n    let preferred = only?.length ? new URL(only[0]).origin : origins[0]\n    const canon = (raw: string) => {\n      const url = new URL(raw)\n      return origins.includes(url.origin) && url.origin !== preferred\n        ? `${preferred}${url.pathname}${url.search}`\n        : raw\n    }\n    const maxAttempts = maxPages * 3 + 10\n    let attempts = 0\n    type Asked = { next: (typeof queue)[number]; key: string; cap: number; page: PublicFetchResult }\n    /** The pages asked for and not yet taken in, in the order they were asked. */\n    const window: Array<Promise<Asked>> = []\n    /** The bytes the asked pages may still read: what is left is shared, never promised twice. */\n    let reserved = 0\n    let stop: SiteImportStop | null = null\n    /** Asked for and never read, because a stop came first: listed as not reached. */\n    const cutOff: string[] = []\n    /** Home alone first, so its answer settles which of the site's two names the rest are read on. */\n    let homeIn = !!only\n    /** Why no more may be asked now: a stop, `wait` for an asked page to come in first, or nothing. */\n    const held = (): SiteImportStop | 'wait' | null => {\n      if (inventory.pages.length >= maxPages) return 'page-budget'\n      if (outOfTime()) return 'time'\n      if (remainingBytes() <= 0) return 'total-bytes'\n      if (attempts >= maxAttempts) return 'page-budget'\n      if (inventory.pages.length + window.length >= maxPages) return 'wait'\n      if (remainingBytes() - reserved <= 0) return 'wait'\n      return null\n    }\n    /** Takes one asked page in: the inventory's next page, a skip, or the reason the read must stop. */\n    const takeIn = ({ next, key, cap, page }: Asked): SiteImportStop | null => {\n      const foundBy = next.foundBy\n      if (foundBy === 'home') homeIn = true\n      if (!page.ok) {\n        if (page.refusal === 'too-large' && cap < maxPageBytes) {\n          cutOff.push(key)\n          return 'total-bytes'\n        }\n        if (page.refusal === 'network' && outOfTime()) {\n          cutOff.push(key)\n          return 'time'\n        }\n        skip({\n          url: key,\n          reason: skipReasonFor(page),\n          ...(page.status ? { status: page.status } : {}),\n          ...(page.location ? { location: page.location } : {}),\n          ...(page.edge ? { edge: page.edge } : {}),\n        })\n        return null\n      }\n      inventory.bytesRead += page.bytes\n      if (foundBy === 'home') preferred = new URL(page.url).origin\n      const finalKey = canon(siteImportPageKey(new URL(page.url), origins) ?? page.url)\n      if (finalKey !== key) {\n        if (read.has(finalKey)) return null\n        read.add(finalKey)\n        if (!allowed(finalKey)) {\n          skip({ url: finalKey, reason: 'robots' })\n          return null\n        }\n      }\n      const extract = extractSiteImportPage(page.text, page.url, sameSite)\n      if (extract.loginForm) {\n        skip({ url: finalKey, reason: 'login-form' })\n        return null\n      }\n      const links: string[] = []\n      for (const link of extract.links) {\n        const linkKey = siteImportPageKey(new URL(link), origins)\n        if (!linkKey) continue\n        links.push(linkKey)\n        if (!only && !queued.has(linkKey)) {\n          queued.add(linkKey)\n          queue.push({ key: linkKey, foundBy: 'link' })\n        }\n      }\n      inventory.pages.push({\n        url: finalKey,\n        path: new URL(finalKey).pathname,\n        foundBy,\n        title: extract.title,\n        description: extract.description,\n        lang: extract.lang,\n        canonical: extract.canonical,\n        ogImage: extract.ogImage,\n        siteName: extract.siteName,\n        icons: extract.icons,\n        brand: extract.brand,\n        organization: extract.organization,\n        blocks: extract.blocks,\n        images: extract.images,\n        navLinks: extract.navLinks,\n        links: [...new Set(links.map(canon))],\n        forms: extract.forms,\n        bytes: page.bytes,\n      })\n      return null\n    }\n    for (;;) {\n      const width = homeIn ? SITE_IMPORT_DEFAULTS.concurrency : 1\n      while (!stop && queue.length && window.length < width) {\n        const why = held()\n        if (why === 'wait') break\n        if (why) {\n          stop = why\n          break\n        }\n        const next = queue.shift() as (typeof queue)[number]\n        const key = canon(next.key)\n        if (read.has(key)) continue\n        if (!allowed(key)) {\n          skip({ url: key, reason: 'robots' })\n          if (next.foundBy === 'home') homeIn = true\n          continue\n        }\n        attempts += 1\n        read.add(key)\n        const cap = Math.min(maxPageBytes, remainingBytes() - reserved)\n        reserved += cap\n        window.push(\n          request(key, /text\\/html|application\\/xhtml\\+xml/i, 'text/html, application/xhtml+xml;q=0.9', cap).then((page) => ({ next, key, cap, page })),\n        )\n      }\n      if (!window.length) break\n      const asked = await (window.shift() as Promise<Asked>)\n      reserved -= asked.cap\n      stop = takeIn(asked) ?? stop\n    }\n    inventory.stoppedBy = stop\n    // Whatever a stop left unread is said, never dropped; a wave's own read\n    // leaves its unread pages for its next pass instead.\n    if (stop && !only) {\n      const unread = [...cutOff]\n      for (const next of queue) {\n        const key = canon(next.key)\n        if (!read.has(key) && !unread.includes(key)) unread.push(key)\n      }\n      for (const url of unread) skip({ url, reason: 'not-reached' })\n    }\n    return finish()\n  } finally {\n    clearTimeout(timer)\n    options.signal?.removeEventListener('abort', onAbort)\n  }\n}\n\n/**\n * The home page's `aglyn-site-verification` values, for the meta-tag proof:\n * the same fetch, user agent and same-site redirects as the crawl, but no\n * robots.txt, because the person asked for this one page to be read.\n * `answered: false` when the page could not be read at all.\n */\nexport async function readSiteVerificationMeta(\n  domain: string,\n  metaName: string,\n  seams: SiteImportCrawlSeams = {},\n): Promise<{ answered: boolean; values: string[]; url?: string }> {\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const bare = String(domain ?? '').trim().toLowerCase().replace(/^www\\./, '')\n  if (!DOMAIN_PATTERN.test(bare) || lookalikeBrandForHost(bare)) return { answered: false, values: [] }\n  const origins = siteImportOrigins(bare)\n  const result = await fetchText(`${origins[0]}/`, {\n    maxBytes: SITE_IMPORT_DEFAULTS.maxPageBytes,\n    accept: /text\\/html|application\\/xhtml\\+xml/i,\n    acceptHeader: 'text/html, application/xhtml+xml;q=0.9',\n    userAgent: AGLYN_BOT_USER_AGENT,\n    headersFor: await siteRequestHeaders(seams, bare, origins),\n    timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,\n    redirectAllowed: (_from, to) => to.protocol === 'https:' && origins.includes(to.origin),\n  })\n  if (!result.ok) {\n    // An answer that is not the page (a 404, a redirect off the site) is\n    // still an answer: the tag is not where it must be.\n    const answered = result.refusal === 'http-status' || result.refusal === 'redirect-refused' || result.refusal === 'content-type'\n    return { answered, values: [], url: result.url }\n  }\n  const head = result.text.split(/<body\\b/i)[0] ?? ''\n  return { answered: true, values: readMetaValues(head, metaName), url: result.url }\n}\n\n/**\n * One page as text for a prompt: its title, description and blocks in order,\n * fenced as untrusted data. The importer's prompt carries `UNTRUSTED_TEXT_RULE`.\n */\nexport function siteImportPagePromptText(page: SiteImportPage, maxChars = 12_000): string {\n  const lines: string[] = []\n  if (page.title) lines.push(`Title: ${page.title}`)\n  if (page.description) lines.push(`Description: ${page.description}`)\n  for (const block of page.blocks) {\n    if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n    else if (block.kind === 'image') lines.push(`[image: ${block.alt || 'no description'}]`)\n    else if (block.kind === 'list-item') lines.push(`- ${block.text}`)\n    else if (block.kind === 'quote') lines.push(`> ${block.text}`)\n    else if (block.kind === 'link') {\n      // A link inside a sentence is in its block's words already.\n      if (!block.inline) lines.push(block.text)\n    } else lines.push(block.text)\n  }\n  return fenceUntrustedText(`Page ${page.path} of the site being imported`, lines.join('\\n'), { maxChars })\n}\n"],"names":["process","lookalikeBrandForHost","fenceUntrustedText","fetchPublicTextDetailed","extractSiteImportPage","readMetaValues","AGLYN_BOT_TOKEN","parseRobotsTxt","parseSitemap","robotsAllows","robotsRulesFor","NO_SITE_IMPORT_HEADERS","siteImportEdgeHeadersFor","AGLYN_BOT_INFO_URL","env","AGLYN_BOT_USER_AGENT","SITE_IMPORT_DEFAULTS","maxPageBytes","maxTotalBytes","timeBudgetMs","requestTimeoutMs","robotsMaxBytes","sitemapMaxBytes","maxSitemaps","maxSitemapUrls","maxSkipped","concurrency","NON_PAGE_EXTENSION","TRACKING_PARAM","DOMAIN_PATTERN","siteRequestHeaders","seams","domain","origins","inner","headersFor","headers","Object","keys","length","error","console","warn","url","includes","origin","siteImportOrigins","bare","replace","siteImportPageKey","protocol","username","password","port","test","pathname","clean","URL","toString","hash","key","searchParams","delete","search","skipReasonFor","result","refusal","crawlSiteForImport","options","now","Date","fetchText","fetch","String","trim","toLowerCase","startedAt","inventory","refused","pages","skipped","skippedOverflow","robots","found","sitemaps","sitemapUrls","stoppedBy","bytesRead","finishedAt","finish","maxPages","Math","max","floor","deadlineAt","controller","AbortController","onAbort","abort","signal","addEventListener","aborted","timer","setTimeout","sameSite","remainingBytes","outOfTime","request","accept","acceptHeader","maxBytes","userAgent","timeoutMs","min","redirectAllowed","_from","to","refuseOver","skip","entry","push","home","rules","ok","bytes","parsed","text","status","allowed","only","map","raw","filter","sitemapQueue","sitemapSeen","Set","fromSitemap","sitemapsRead","next","shift","has","add","room","sitemap","entries","location","locations","index","shallowFirst","depth","split","Boolean","ranked","order","sort","a","b","splice","queue","foundBy","queued","read","preferred","canon","maxAttempts","attempts","window","reserved","stop","cutOff","homeIn","held","takeIn","cap","page","reason","edge","finalKey","extract","loginForm","links","link","linkKey","path","title","description","lang","canonical","ogImage","siteName","icons","brand","organization","blocks","images","navLinks","forms","width","why","then","asked","unread","clearTimeout","removeEventListener","readSiteVerificationMeta","metaName","answered","values","head","siteImportPagePromptText","maxChars","lines","block","kind","repeat","level","alt","inline","join"],"mappings":";IAyGEA;AAzGF;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2CA6C2C,GAE3C,SAASC,qBAAqB,QAAQ,oDAAmD;AACzF,SAASC,kBAAkB,QAAQ,0CAAyC;AAC5E,SACEC,uBAAuB,QAIlB,mCAA+B;AACtC,SACEC,qBAAqB,EACrBC,cAAc,QAQT,wBAAoB;AAC3B,SACEC,eAAe,EACfC,cAAc,EACdC,YAAY,EACZC,YAAY,EACZC,cAAc,QAET,0BAAsB;AAC7B,SAASC,sBAAsB,EAAEC,wBAAwB,QAAmC,wBAAoB;AAEhH;;;;;;;;;;CAUC,GACD,OAAO,MAAMC,sBACXb,wCAAAA,QAAQc,GAAG,CAAC,2BAA2B,YAAvCd,wCAA2C,0CAAyC;AAEtF,2CAA2C,GAC3C,OAAO,MAAMe,uBAAuB,CAAC,eAAe,EAAEF,mBAAmB,CAAC,CAAC,CAAA;AAE3E,OAAO,MAAMG,uBAAuB;IAClCC,cAAc,IAAI,OAAO;IACzB,iFAAiF,GACjFC,eAAe,MAAM,OAAO;IAC5BC,cAAc;IACd,uDAAuD,GACvDC,kBAAkB;IAClBC,gBAAgB,MAAM;IACtBC,iBAAiB,IAAI,OAAO;IAC5B,6FAA6F,GAC7FC,aAAa;IACb,gDAAgD,GAChDC,gBAAgB;IAChB,+FAA+F,GAC/FC,YAAY;IACZ,0EAA0E,GAC1EC,aAAa;AACf,EAAU;AAEV,+DAA+D,GAC/D,MAAMC,qBACJ;AAEF,MAAMC,iBAAiB;AAEvB,MAAMC,iBAAiB;AA0HvB;;;CAGC,GACD,eAAeC,mBACbC,KAA2B,EAC3BC,MAAc,EACdC,OAA0B;IAE1B,IAAIC,QAA8BvB;IAClC,IAAI;QACF,IAAIoB,MAAMI,UAAU,EAAE;YACpB,MAAMC,UAAU,MAAML,MAAMI,UAAU,CAACH;YACvC,IAAII,WAAWC,OAAOC,IAAI,CAACF,SAASG,MAAM,EAAEL,QAAQ,IAAME;QAC5D,OAAO;YACLF,QAAQ,MAAMtB,yBAAyBqB;QACzC;IACF,EAAE,OAAOO,OAAO;QACdC,QAAQC,IAAI,CAAC,0EAA0E;YAAEV;YAAQQ;QAAM;IACzG;IACA,OAAO,CAACG,MAAcV,QAAQW,QAAQ,CAACD,IAAIE,MAAM,IAAIX,MAAMS,OAAO,CAAC;AACrE;AAEA,yCAAyC,GACzC,OAAO,SAASG,kBAAkBd,MAAc;IAC9C,MAAMe,OAAOf,OAAOgB,OAAO,CAAC,UAAU;IACtC,OAAO;QAAC,CAAC,QAAQ,EAAED,MAAM;QAAE,CAAC,YAAY,EAAEA,MAAM;KAAC;AACnD;AAEA,oFAAoF,GACpF,OAAO,SAASE,kBAAkBN,GAAQ,EAAEV,OAA0B;IACpE,IAAIU,IAAIO,QAAQ,KAAK,YAAYP,IAAIQ,QAAQ,IAAIR,IAAIS,QAAQ,EAAE,OAAO;IACtE,IAAIT,IAAIU,IAAI,IAAIV,IAAIU,IAAI,KAAK,OAAO,OAAO;IAC3C,IAAI,CAACpB,QAAQW,QAAQ,CAACD,IAAIE,MAAM,GAAG,OAAO;IAC1C,IAAIlB,mBAAmB2B,IAAI,CAACX,IAAIY,QAAQ,GAAG,OAAO;IAClD,MAAMC,QAAQ,IAAIC,IAAId,IAAIe,QAAQ;IAClCF,MAAMG,IAAI,GAAG;IACb,KAAK,MAAMC,OAAO;WAAIJ,MAAMK,YAAY,CAACvB,IAAI;KAAG,CAAE;QAChD,IAAIV,eAAe0B,IAAI,CAACM,MAAMJ,MAAMK,YAAY,CAACC,MAAM,CAACF;IAC1D;IACAJ,MAAMO,MAAM,GAAGP,MAAMK,YAAY,CAACH,QAAQ,KAAK,CAAC,CAAC,EAAEF,MAAMK,YAAY,CAACH,QAAQ,IAAI,GAAG;IACrF,OAAOF,MAAME,QAAQ;AACvB;AAEA,SAASM,cAAcC,MAAyB;IAC9C,OAAQA,OAAOC,OAAO;QACpB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,mFAAmF,GACnF,OAAO,eAAeC,mBACpBC,OAA+B,EAC/BrC,QAA8B,CAAC,CAAC;QAEpBA,YACMA,cACIqC,iBAiCDA,uBACCA,wBACUA;QAGhCA,iBACIA;IAzCJ,MAAMC,OAAMtC,aAAAA,MAAMsC,GAAG,YAATtC,aAAauC,KAAKD,GAAG;IACjC,MAAME,aAAYxC,eAAAA,MAAMyC,KAAK,YAAXzC,eAAe5B;IACjC,MAAM6B,SAASyC,QAAOL,kBAAAA,QAAQpC,MAAM,YAAdoC,kBAAkB,IAAIM,IAAI,GAAGC,WAAW,GAAG3B,OAAO,CAAC,QAAQ,IAAIA,OAAO,CAAC,UAAU;IACvG,MAAMf,UAAUJ,eAAeyB,IAAI,CAACtB,UAAUc,kBAAkBd,UAAU,EAAE;IAC5E,MAAM4C,YAAYP;IAClB,MAAMQ,YAAiC;QACrC7C;QACAC;QACA6C,SAAS;QACTC,OAAO,EAAE;QACTC,SAAS,EAAE;QACXC,iBAAiB;QACjBC,QAAQ;YAAEC,OAAO;YAAOC,UAAU,EAAE;QAAC;QACrCC,aAAa;QACbC,WAAW;QACXC,WAAW;QACXX;QACAY,YAAYZ;IACd;IACA,MAAMa,SAAS;QACbZ,UAAUW,UAAU,GAAGnB;QACvB,OAAOQ;IACT;IACA,IAAI,CAAC5C,QAAQM,MAAM,EAAE;QACnBsC,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IACA,IAAIxF,sBAAsB+B,SAAS;QACjC6C,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IAEA,wEAAwE;IACxE,MAAMtD,aAAa,MAAML,mBAAmBC,OAAOC,QAAQC;IAC3D,MAAMyD,WAAWC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAACzB,QAAQsB,QAAQ;IACxD,MAAMzE,gBAAemD,wBAAAA,QAAQnD,YAAY,YAApBmD,wBAAwBpD,qBAAqBC,YAAY;IAC9E,MAAMC,iBAAgBkD,yBAAAA,QAAQlD,aAAa,YAArBkD,yBAAyBpD,qBAAqBE,aAAa;IACjF,MAAM4E,aAAalB,cAAaR,wBAAAA,QAAQjD,YAAY,YAApBiD,wBAAwBpD,qBAAqBG,YAAY;IACzF,MAAM4E,aAAa,IAAIC;IACvB,MAAMC,UAAU,IAAMF,WAAWG,KAAK;KACtC9B,kBAAAA,QAAQ+B,MAAM,qBAAd/B,gBAAgBgC,gBAAgB,CAAC,SAASH;IAC1C,KAAI7B,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBiC,OAAO,EAAEN,WAAWG,KAAK;IAC7C,MAAMI,QAAQC,WAAW,IAAMR,WAAWG,KAAK,IAAIP,KAAKC,GAAG,CAAC,GAAGE,aAAalB;IAE5E,MAAM4B,WAAW,CAAC7D,MAAaV,QAAQW,QAAQ,CAACD,IAAIE,MAAM,KAAKF,IAAIO,QAAQ,KAAK;IAChF,MAAMuD,iBAAiB,IAAMvF,gBAAgB2D,UAAUU,SAAS;IAChE,MAAMmB,YAAY,IAAMX,WAAWI,MAAM,CAACE,OAAO,IAAIhC,SAASyB;IAC9D,MAAMa,UAAU,CAAChE,KAAaiE,QAAgBC,cAAsBC,WAClEvC,UAAU5B,KAAK;YACbmE;YACAF;YACAC;YACAE,WAAWhG;YACXoB;YACAgE,QAAQJ,WAAWI,MAAM;YACzBa,WAAWrB,KAAKC,GAAG,CAAC,GAAGD,KAAKsB,GAAG,CAACjG,qBAAqBI,gBAAgB,EAAE0E,aAAazB;YACpF6C,iBAAiB,CAACC,OAAOC,KAAOZ,SAASY;YACzCC,YAAY;QACd;IACF,MAAMC,OAAO,CAACC;QACZ,IAAI1C,UAAUG,OAAO,CAACzC,MAAM,GAAGvB,qBAAqBS,UAAU,EAAEoD,UAAUG,OAAO,CAACwC,IAAI,CAACD;aAClF1C,UAAUI,eAAe,IAAI;IACpC;IAEA,IAAI;YAgBuCC,gBAA+BA;QAfxE,yDAAyD;QACzD,MAAMuC,OAAO,GAAGxF,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,IAAIyF,QAAsB,EAAE;QAC5B,MAAMxC,SAAS,MAAMyB,QACnB,GAAG1E,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAC1B,8BACA,cACA0D,KAAKsB,GAAG,CAACjG,qBAAqBK,cAAc,EAAEoF;QAEhD,IAAIvB,OAAOyC,EAAE,EAAE;YACb9C,UAAUU,SAAS,IAAIL,OAAO0C,KAAK;YACnC,MAAMC,SAAStH,eAAe2E,OAAO4C,IAAI;YACzCJ,QAAQhH,eAAemH,QAAQvH;YAC/BuE,UAAUK,MAAM,GAAG;gBAAEC,OAAO;gBAAMC,UAAUyC,OAAOzC,QAAQ;YAAC;QAC9D,OAAO,IACL,CAAEF,CAAAA,OAAOhB,OAAO,KAAK,iBAAiB,EAACgB,iBAAAA,OAAO6C,MAAM,YAAb7C,iBAAiB,MAAM,OAAO,EAACA,kBAAAA,OAAO6C,MAAM,YAAb7C,kBAAiB,KAAK,GAAE,GAC9F;YACAL,UAAUS,SAAS,GAAGoB,cAAc,SAAS;YAC7C,OAAOjB;QACT;QACA,MAAMuC,UAAU,CAACpE;YACf,MAAMjB,MAAM,IAAIc,IAAIG;YACpB,OAAOnD,aAAaiH,OAAO,GAAG/E,IAAIY,QAAQ,GAAGZ,IAAIoB,MAAM,EAAE;QAC3D;QAEA,kEAAkE;QAClE,MAAMkE,OAAO7D,QAAQ6D,IAAI,GACrB7D,QAAQ6D,IAAI,CACTC,GAAG,CAAC,CAACC;YACJ,IAAI;gBACF,OAAOlF,kBAAkB,IAAIQ,IAAI0E,MAAMlG;YACzC,EAAE,eAAM;gBACN,OAAO;YACT;QACF,GACCmG,MAAM,CAAC,CAACxE,MAAuB,CAAC,CAACA,OACpC;QAEJ,uEAAuE;QACvE,MAAMyE,eAAyB,EAAE;QACjC,KAAK,MAAMF,OAAOtD,UAAUK,MAAM,CAACE,QAAQ,CAAE;YAC3C,IAAI;gBACF,MAAMzC,MAAM,IAAIc,IAAI0E;gBACpB,IAAI3B,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIe,QAAQ;YACnD,EAAE,eAAM;YACN,0CAA0C;YAC5C;QACF;QACA,IAAI,CAAC2E,aAAa9F,MAAM,IAAI,CAAC0F,MAAMI,aAAab,IAAI,CAAC,GAAGvF,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC;QAChF,IAAIgG,MAAMI,aAAa9F,MAAM,GAAG;QAChC,MAAM+F,cAAc,IAAIC;QACxB,MAAMC,cAAwB,EAAE;QAChC,IAAIC,eAAe;QACnB,MAAOJ,aAAa9F,MAAM,IAAIkG,eAAezH,qBAAqBO,WAAW,IAAI,CAACmF,YAAa;YAC7F,MAAMgC,OAAOL,aAAaM,KAAK;YAC/B,IAAIL,YAAYM,GAAG,CAACF,OAAO;YAC3BJ,YAAYO,GAAG,CAACH;YAChB,IAAI,SAASpF,IAAI,CAAC,IAAIG,IAAIiF,MAAMnF,QAAQ,GAAG;YAC3C,IAAI,CAACyE,QAAQU,OAAO;YACpB,MAAMI,OAAOnD,KAAKsB,GAAG,CAACjG,qBAAqBM,eAAe,EAAEmF;YAC5D,IAAIqC,QAAQ,GAAG;YACfL,gBAAgB;YAChB,MAAMM,UAAU,MAAMpC,QAAQ+B,MAAM,oBAAoB,mCAAmCI;YAC3F,IAAI,CAACC,QAAQpB,EAAE,EAAE;YACjB9C,UAAUU,SAAS,IAAIwD,QAAQnB,KAAK;YACpC,MAAMoB,UAAUxI,aAAauI,QAAQjB,IAAI;YACzC,KAAK,MAAMmB,YAAYD,QAAQE,SAAS,CAAE;gBACxC,IAAIvG;gBACJ,IAAI;oBACFA,MAAM,IAAIc,IAAIwF;gBAChB,EAAE,eAAM;oBACN;gBACF;gBACA,IAAID,QAAQG,KAAK,EAAE;oBACjB,IAAI3C,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIe,QAAQ;oBACjD;gBACF;gBACA,MAAME,MAAMX,kBAAkBN,KAAKV;gBACnC,IAAI2B,OAAO4E,YAAYjG,MAAM,GAAGvB,qBAAqBQ,cAAc,EAAEgH,YAAYhB,IAAI,CAAC5D;YACxF;QACF;QACAiB,UAAUQ,WAAW,GAAGmD,YAAYjG,MAAM;QAC1C,IAAI6B,QAAQgF,YAAY,EAAE;YACxB,MAAMC,QAAQ,CAACzF,MAAgB,IAAIH,IAAIG,KAAKL,QAAQ,CAAC+F,KAAK,CAAC,KAAKlB,MAAM,CAACmB,SAAShH,MAAM;YACtF,MAAMiH,SAAShB,YAAYN,GAAG,CAAC,CAACtE,KAAK6F,QAAW,CAAA;oBAAE7F;oBAAK6F;oBAAOJ,OAAOA,MAAMzF;gBAAK,CAAA;YAChF4F,OAAOE,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEN,KAAK,GAAGO,EAAEP,KAAK,IAAIM,EAAEF,KAAK,GAAGG,EAAEH,KAAK;YAC5DjB,YAAYqB,MAAM,CAAC,GAAGrB,YAAYjG,MAAM,KAAKiH,OAAOtB,GAAG,CAAC,CAACX,QAAUA,MAAM3D,GAAG;QAC9E;QAEA,yEAAyE;QACzE,wCAAwC;QACxC,MAAMkG,QAA+D7B,OACjEA,KAAKC,GAAG,CAAC,CAACtE,MAAS,CAAA;gBAAEA;gBAAKmG,SAAU,IAAItG,IAAIG,KAAKL,QAAQ,KAAK,MAAM,SAAS;YAAqC,CAAA,KAClH;YAAC;gBAAEK,KAAK6D;gBAAMsC,SAAS;YAAO;SAAE;QACpC,MAAMC,SAAS,IAAIzB,IAAYN,eAAAA,OAAQ;YAACR;SAAK;QAC7C,KAAK,MAAM7D,OAAO4E,YAAa;YAC7B,IAAIwB,OAAOpB,GAAG,CAAChF,MAAM;YACrBoG,OAAOnB,GAAG,CAACjF;YACXkG,MAAMtC,IAAI,CAAC;gBAAE5D;gBAAKmG,SAAS;YAAU;QACvC;QACA,MAAME,OAAO,IAAI1B;QACjB,uEAAuE;QACvE,qEAAqE;QACrE,6DAA6D;QAC7D,0EAA0E;QAC1E,IAAI2B,YAAYjC,CAAAA,wBAAAA,KAAM1F,MAAM,IAAG,IAAIkB,IAAIwE,IAAI,CAAC,EAAE,EAAEpF,MAAM,GAAGZ,OAAO,CAAC,EAAE;QACnE,MAAMkI,QAAQ,CAAChC;YACb,MAAMxF,MAAM,IAAIc,IAAI0E;YACpB,OAAOlG,QAAQW,QAAQ,CAACD,IAAIE,MAAM,KAAKF,IAAIE,MAAM,KAAKqH,YAClD,GAAGA,YAAYvH,IAAIY,QAAQ,GAAGZ,IAAIoB,MAAM,EAAE,GAC1CoE;QACN;QACA,MAAMiC,cAAc1E,WAAW,IAAI;QACnC,IAAI2E,WAAW;QAEf,4EAA4E,GAC5E,MAAMC,SAAgC,EAAE;QACxC,4FAA4F,GAC5F,IAAIC,WAAW;QACf,IAAIC,OAA8B;QAClC,gFAAgF,GAChF,MAAMC,SAAmB,EAAE;QAC3B,gGAAgG,GAChG,IAAIC,SAAS,CAAC,CAACzC;QACf,iGAAiG,GACjG,MAAM0C,OAAO;YACX,IAAI9F,UAAUE,KAAK,CAACxC,MAAM,IAAImD,UAAU,OAAO;YAC/C,IAAIgB,aAAa,OAAO;YACxB,IAAID,oBAAoB,GAAG,OAAO;YAClC,IAAI4D,YAAYD,aAAa,OAAO;YACpC,IAAIvF,UAAUE,KAAK,CAACxC,MAAM,GAAG+H,OAAO/H,MAAM,IAAImD,UAAU,OAAO;YAC/D,IAAIe,mBAAmB8D,YAAY,GAAG,OAAO;YAC7C,OAAO;QACT;QACA,kGAAkG,GAClG,MAAMK,SAAS,CAAC,EAAElC,IAAI,EAAE9E,GAAG,EAAEiH,GAAG,EAAEC,IAAI,EAAS;gBAuBtB7H;YAtBvB,MAAM8G,UAAUrB,KAAKqB,OAAO;YAC5B,IAAIA,YAAY,QAAQW,SAAS;YACjC,IAAI,CAACI,KAAKnD,EAAE,EAAE;gBACZ,IAAImD,KAAK5G,OAAO,KAAK,eAAe2G,MAAM5J,cAAc;oBACtDwJ,OAAOjD,IAAI,CAAC5D;oBACZ,OAAO;gBACT;gBACA,IAAIkH,KAAK5G,OAAO,KAAK,aAAawC,aAAa;oBAC7C+D,OAAOjD,IAAI,CAAC5D;oBACZ,OAAO;gBACT;gBACA0D,KAAK;oBACH3E,KAAKiB;oBACLmH,QAAQ/G,cAAc8G;mBAClBA,KAAK/C,MAAM,GAAG;oBAAEA,QAAQ+C,KAAK/C,MAAM;gBAAC,IAAI,CAAC,GACzC+C,KAAK7B,QAAQ,GAAG;oBAAEA,UAAU6B,KAAK7B,QAAQ;gBAAC,IAAI,CAAC,GAC/C6B,KAAKE,IAAI,GAAG;oBAAEA,MAAMF,KAAKE,IAAI;gBAAC,IAAI,CAAC;gBAEzC,OAAO;YACT;YACAnG,UAAUU,SAAS,IAAIuF,KAAKlD,KAAK;YACjC,IAAImC,YAAY,QAAQG,YAAY,IAAIzG,IAAIqH,KAAKnI,GAAG,EAAEE,MAAM;YAC5D,MAAMoI,WAAWd,OAAMlH,qBAAAA,kBAAkB,IAAIQ,IAAIqH,KAAKnI,GAAG,GAAGV,oBAArCgB,qBAAiD6H,KAAKnI,GAAG;YAChF,IAAIsI,aAAarH,KAAK;gBACpB,IAAIqG,KAAKrB,GAAG,CAACqC,WAAW,OAAO;gBAC/BhB,KAAKpB,GAAG,CAACoC;gBACT,IAAI,CAACjD,QAAQiD,WAAW;oBACtB3D,KAAK;wBAAE3E,KAAKsI;wBAAUF,QAAQ;oBAAS;oBACvC,OAAO;gBACT;YACF;YACA,MAAMG,UAAU9K,sBAAsB0K,KAAKhD,IAAI,EAAEgD,KAAKnI,GAAG,EAAE6D;YAC3D,IAAI0E,QAAQC,SAAS,EAAE;gBACrB7D,KAAK;oBAAE3E,KAAKsI;oBAAUF,QAAQ;gBAAa;gBAC3C,OAAO;YACT;YACA,MAAMK,QAAkB,EAAE;YAC1B,KAAK,MAAMC,QAAQH,QAAQE,KAAK,CAAE;gBAChC,MAAME,UAAUrI,kBAAkB,IAAIQ,IAAI4H,OAAOpJ;gBACjD,IAAI,CAACqJ,SAAS;gBACdF,MAAM5D,IAAI,CAAC8D;gBACX,IAAI,CAACrD,QAAQ,CAAC+B,OAAOpB,GAAG,CAAC0C,UAAU;oBACjCtB,OAAOnB,GAAG,CAACyC;oBACXxB,MAAMtC,IAAI,CAAC;wBAAE5D,KAAK0H;wBAASvB,SAAS;oBAAO;gBAC7C;YACF;YACAlF,UAAUE,KAAK,CAACyC,IAAI,CAAC;gBACnB7E,KAAKsI;gBACLM,MAAM,IAAI9H,IAAIwH,UAAU1H,QAAQ;gBAChCwG;gBACAyB,OAAON,QAAQM,KAAK;gBACpBC,aAAaP,QAAQO,WAAW;gBAChCC,MAAMR,QAAQQ,IAAI;gBAClBC,WAAWT,QAAQS,SAAS;gBAC5BC,SAASV,QAAQU,OAAO;gBACxBC,UAAUX,QAAQW,QAAQ;gBAC1BC,OAAOZ,QAAQY,KAAK;gBACpBC,OAAOb,QAAQa,KAAK;gBACpBC,cAAcd,QAAQc,YAAY;gBAClCC,QAAQf,QAAQe,MAAM;gBACtBC,QAAQhB,QAAQgB,MAAM;gBACtBC,UAAUjB,QAAQiB,QAAQ;gBAC1Bf,OAAO;uBAAI,IAAI7C,IAAI6C,MAAMlD,GAAG,CAACiC;iBAAQ;gBACrCiC,OAAOlB,QAAQkB,KAAK;gBACpBxE,OAAOkD,KAAKlD,KAAK;YACnB;YACA,OAAO;QACT;QACA,OAAS;gBA4BAgD;YA3BP,MAAMyB,QAAQ3B,SAAS1J,qBAAqBU,WAAW,GAAG;YAC1D,MAAO,CAAC8I,QAAQV,MAAMvH,MAAM,IAAI+H,OAAO/H,MAAM,GAAG8J,MAAO;gBACrD,MAAMC,MAAM3B;gBACZ,IAAI2B,QAAQ,QAAQ;gBACpB,IAAIA,KAAK;oBACP9B,OAAO8B;oBACP;gBACF;gBACA,MAAM5D,OAAOoB,MAAMnB,KAAK;gBACxB,MAAM/E,MAAMuG,MAAMzB,KAAK9E,GAAG;gBAC1B,IAAIqG,KAAKrB,GAAG,CAAChF,MAAM;gBACnB,IAAI,CAACoE,QAAQpE,MAAM;oBACjB0D,KAAK;wBAAE3E,KAAKiB;wBAAKmH,QAAQ;oBAAS;oBAClC,IAAIrC,KAAKqB,OAAO,KAAK,QAAQW,SAAS;oBACtC;gBACF;gBACAL,YAAY;gBACZJ,KAAKpB,GAAG,CAACjF;gBACT,MAAMiH,MAAMlF,KAAKsB,GAAG,CAAChG,cAAcwF,mBAAmB8D;gBACtDA,YAAYM;gBACZP,OAAO9C,IAAI,CACTb,QAAQ/C,KAAK,uCAAuC,0CAA0CiH,KAAK0B,IAAI,CAAC,CAACzB,OAAU,CAAA;wBAAEpC;wBAAM9E;wBAAKiH;wBAAKC;oBAAK,CAAA;YAE9I;YACA,IAAI,CAACR,OAAO/H,MAAM,EAAE;YACpB,MAAMiK,QAAQ,MAAOlC,OAAO3B,KAAK;YACjC4B,YAAYiC,MAAM3B,GAAG;YACrBL,QAAOI,UAAAA,OAAO4B,kBAAP5B,UAAiBJ;QAC1B;QACA3F,UAAUS,SAAS,GAAGkF;QACtB,wEAAwE;QACxE,qDAAqD;QACrD,IAAIA,QAAQ,CAACvC,MAAM;YACjB,MAAMwE,SAAS;mBAAIhC;aAAO;YAC1B,KAAK,MAAM/B,QAAQoB,MAAO;gBACxB,MAAMlG,MAAMuG,MAAMzB,KAAK9E,GAAG;gBAC1B,IAAI,CAACqG,KAAKrB,GAAG,CAAChF,QAAQ,CAAC6I,OAAO7J,QAAQ,CAACgB,MAAM6I,OAAOjF,IAAI,CAAC5D;YAC3D;YACA,KAAK,MAAMjB,OAAO8J,OAAQnF,KAAK;gBAAE3E;gBAAKoI,QAAQ;YAAc;QAC9D;QACA,OAAOtF;IACT,SAAU;YAERrB;QADAsI,aAAapG;SACblC,mBAAAA,QAAQ+B,MAAM,qBAAd/B,iBAAgBuI,mBAAmB,CAAC,SAAS1G;IAC/C;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAe2G,yBACpB5K,MAAc,EACd6K,QAAgB,EAChB9K,QAA8B,CAAC,CAAC;QAEdA,cAmBLkC;IAnBb,MAAMM,aAAYxC,eAAAA,MAAMyC,KAAK,YAAXzC,eAAe5B;IACjC,MAAM4C,OAAO0B,OAAOzC,iBAAAA,SAAU,IAAI0C,IAAI,GAAGC,WAAW,GAAG3B,OAAO,CAAC,UAAU;IACzE,IAAI,CAACnB,eAAeyB,IAAI,CAACP,SAAS9C,sBAAsB8C,OAAO,OAAO;QAAE+J,UAAU;QAAOC,QAAQ,EAAE;IAAC;IACpG,MAAM9K,UAAUa,kBAAkBC;IAClC,MAAMkB,SAAS,MAAMM,UAAU,GAAGtC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;QAC/C6E,UAAU9F,qBAAqBC,YAAY;QAC3C2F,QAAQ;QACRC,cAAc;QACdE,WAAWhG;QACXoB,YAAY,MAAML,mBAAmBC,OAAOgB,MAAMd;QAClD+E,WAAWhG,qBAAqBI,gBAAgB;QAChD8F,iBAAiB,CAACC,OAAOC,KAAOA,GAAGlE,QAAQ,KAAK,YAAYjB,QAAQW,QAAQ,CAACwE,GAAGvE,MAAM;IACxF;IACA,IAAI,CAACoB,OAAO0D,EAAE,EAAE;QACd,qEAAqE;QACrE,oDAAoD;QACpD,MAAMmF,WAAW7I,OAAOC,OAAO,KAAK,iBAAiBD,OAAOC,OAAO,KAAK,sBAAsBD,OAAOC,OAAO,KAAK;QACjH,OAAO;YAAE4I;YAAUC,QAAQ,EAAE;YAAEpK,KAAKsB,OAAOtB,GAAG;QAAC;IACjD;IACA,MAAMqK,QAAO/I,sBAAAA,OAAO6D,IAAI,CAACwB,KAAK,CAAC,WAAW,CAAC,EAAE,YAAhCrF,sBAAoC;IACjD,OAAO;QAAE6I,UAAU;QAAMC,QAAQ1M,eAAe2M,MAAMH;QAAWlK,KAAKsB,OAAOtB,GAAG;IAAC;AACnF;AAEA;;;CAGC,GACD,OAAO,SAASsK,yBAAyBnC,IAAoB,EAAEoC,WAAW,KAAM;IAC9E,MAAMC,QAAkB,EAAE;IAC1B,IAAIrC,KAAKU,KAAK,EAAE2B,MAAM3F,IAAI,CAAC,CAAC,OAAO,EAAEsD,KAAKU,KAAK,EAAE;IACjD,IAAIV,KAAKW,WAAW,EAAE0B,MAAM3F,IAAI,CAAC,CAAC,aAAa,EAAEsD,KAAKW,WAAW,EAAE;IACnE,KAAK,MAAM2B,SAAStC,KAAKmB,MAAM,CAAE;QAC/B,IAAImB,MAAMC,IAAI,KAAK,WAAWF,MAAM3F,IAAI,CAAC,GAAG,IAAI8F,MAAM,CAACF,MAAMG,KAAK,EAAE,CAAC,EAAEH,MAAMtF,IAAI,EAAE;aAC9E,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,QAAQ,EAAE4F,MAAMI,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAClF,IAAIJ,MAAMC,IAAI,KAAK,aAAaF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aAC5D,IAAIsF,MAAMC,IAAI,KAAK,SAASF,MAAM3F,IAAI,CAAC,CAAC,EAAE,EAAE4F,MAAMtF,IAAI,EAAE;aACxD,IAAIsF,MAAMC,IAAI,KAAK,QAAQ;YAC9B,4DAA4D;YAC5D,IAAI,CAACD,MAAMK,MAAM,EAAEN,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;QAC1C,OAAOqF,MAAM3F,IAAI,CAAC4F,MAAMtF,IAAI;IAC9B;IACA,OAAO5H,mBAAmB,CAAC,KAAK,EAAE4K,KAAKS,IAAI,CAAC,2BAA2B,CAAC,EAAE4B,MAAMO,IAAI,CAAC,OAAO;QAAER;IAAS;AACzG"}

package/src/lib/site-import/site-import-edge-server.d.ts

@@ -0,0 +1,31 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * The server read the site-import edge defers (AGL-4085): whether this
+ * platform serves a hostname, which opens Firestore and the hosting
+ * platform's API. `site-import-edge.ts` loads it only once a secret is
+ * configured, so a crawl without one never opens either.
+ *
+ * The deferral targets THIS module by relative path, for the reason
+ * `ai-theme-brand-server.ts` gives: an `import()` of `@aglyn/tenant-data-admin`
+ * by its package specifier makes nx treat that lib as lazy-loaded everywhere,
+ * and every static import of it across the plugin, the console and the
+ * tenant then fails lint.
+ *
+ * Import this module dynamically, and only from `site-import-edge.ts`.
+ */
+export { isPlatformServedDomain } from '@aglyn/tenant-data-admin/server/platform-served-domain';

package/src/lib/site-import/site-import-edge-server.js

@@ -0,0 +1,31 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ /**
+ * The server read the site-import edge defers (AGL-4085): whether this
+ * platform serves a hostname, which opens Firestore and the hosting
+ * platform's API. `site-import-edge.ts` loads it only once a secret is
+ * configured, so a crawl without one never opens either.
+ *
+ * The deferral targets THIS module by relative path, for the reason
+ * `ai-theme-brand-server.ts` gives: an `import()` of `@aglyn/tenant-data-admin`
+ * by its package specifier makes nx treat that lib as lazy-loaded everywhere,
+ * and every static import of it across the plugin, the console and the
+ * tenant then fails lint.
+ *
+ * Import this module dynamically, and only from `site-import-edge.ts`.
+ */ export { isPlatformServedDomain } from "@aglyn/tenant-data-admin/server/platform-served-domain";
+
+//# sourceMappingURL=site-import-edge-server.js.map

package/src/lib/site-import/site-import-edge-server.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-edge-server.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * The server read the site-import edge defers (AGL-4085): whether this\n * platform serves a hostname, which opens Firestore and the hosting\n * platform's API. `site-import-edge.ts` loads it only once a secret is\n * configured, so a crawl without one never opens either.\n *\n * The deferral targets THIS module by relative path, for the reason\n * `ai-theme-brand-server.ts` gives: an `import()` of `@aglyn/tenant-data-admin`\n * by its package specifier makes nx treat that lib as lazy-loaded everywhere,\n * and every static import of it across the plugin, the console and the\n * tenant then fails lint.\n *\n * Import this module dynamically, and only from `site-import-edge.ts`.\n */\nexport { isPlatformServedDomain } from '@aglyn/tenant-data-admin/server/platform-served-domain'\n"],"names":["isPlatformServedDomain"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;CAaC,GACD,SAASA,sBAAsB,QAAQ,yDAAwD"}

package/src/lib/site-import/site-import-edge.d.ts

@@ -0,0 +1,44 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** The header the edge rule matches. Lowercase, as the rule declares it. */
+export declare const SITE_IMPORT_EDGE_HEADER = "x-aglyn-import";
+/** A secret shorter than this is treated as unset: `openssl rand -hex 32` makes 64. */
+export declare const SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH = 32;
+/** Extra headers for one request, asked per address. */
+export type SiteImportHeadersFor = (url: URL) => Readonly<Record<string, string>>;
+/** Sends nothing anywhere. */
+export declare const NO_SITE_IMPORT_HEADERS: SiteImportHeadersFor;
+export interface SiteImportEdgeDeps {
+    /** Where the secret is read from; `process.env` by default. */
+    env?: Readonly<Record<string, string | undefined>>;
+    /** Whether this platform serves a hostname; `isPlatformServedDomain` by default. */
+    served?: (hostname: string) => Promise<boolean>;
+}
+/** The edge secret, or null when it is unset or not shaped like one. Never logged. */
+export declare function siteImportEdgeToken(env?: SiteImportEdgeDeps['env']): string | null;
+/**
+ * The edge header for requests to one hostname: the secret when this platform
+ * serves it, else nothing. Never throws.
+ */
+export declare function siteImportEdgeHeaders(hostname: string, deps?: SiteImportEdgeDeps): Promise<Readonly<Record<string, string>>>;
+/**
+ * Per-request headers for a crawl of these origins: each origin's hostname is
+ * decided once, and a request is given the header only when it is https on
+ * the default port to an origin decided "served". Anything else, a hop to
+ * another host included, gets nothing.
+ */
+export declare function siteImportEdgeHeadersFor(origins: readonly string[], deps?: SiteImportEdgeDeps): Promise<SiteImportHeadersFor>;