@aglyn/plugins-ai 1.0.0-beta.271 → 1.0.0-beta.273

package/src/lib/site-import/site-import-crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-crawl.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * THE SITE-IMPORT CRAWLER (AGL-3974, parent AGL-3973).\n *\n * Reads a website a workspace has PROVED it owns (domain-ownership.ts in\n * tenant-data-admin; the doors check the proof before calling this) into a\n * typed inventory: each page's title, description, headings and text in\n * order, images and menu links. It writes nothing; the import job decides\n * what becomes a page.\n *\n * Every request goes through `fetchPublicTextDetailed`, the theme job's\n * SSRF-guarded fetch: https on the default port only, public addresses only,\n * the socket pinned to the address checked. On top of that:\n *\n * - **One site.** The site is `https://<domain>` and `https://www.<domain>`,\n *   one site to its owner. Every redirect hop and every discovered link must\n *   stay on it; a page that redirects anywhere else is refused, not followed.\n * - **AglynBot.** Every request says `AglynBot/1.0 (+<the operator's bot page>)`,\n *   and robots.txt is read first and honored for every page: the AglynBot\n *   group, else `*`. A robots.txt that answers 404 allows everything; one\n *   that cannot be read at all (5xx, no answer) stops the crawl, because\n *   \"could not ask\" is not \"allowed\".\n * - **Sitemap first.** The sitemaps robots.txt names, else /sitemap.xml (an\n *   index's children included), then same-site links found on the pages read,\n *   for whatever the sitemap missed.\n * - **Budgets.** Pages (the caller's, from the plan), bytes per page (a page\n *   over it is refused, not cut), bytes in total, and wall-clock time.\n * - **No lookalikes.** A domain that wears a brand it is not\n *   (`lookalikeBrandForHost`) is refused before any request.\n * - **No sign-in pages.** A page with a password field is refused and\n *   reported; forms with payment fields, scripts, iframes and embeds never\n *   reach the inventory (site-import-html.ts).\n *=========================================*/\n\nimport { lookalikeBrandForHost } from '@aglyn/shared-util-email/outbound-phishing-screen'\nimport { fenceUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n  fetchPublicTextDetailed,\n  type PublicFetchOptions,\n  type PublicFetchResult,\n} from '../jobs/ai-theme-brand-inputs'\nimport {\n  extractSiteImportPage,\n  readMetaValues,\n  type SiteImportBlock,\n  type SiteImportForm,\n  type SiteImportImage,\n  type SiteImportNavLink,\n} from './site-import-html'\nimport {\n  AGLYN_BOT_TOKEN,\n  parseRobotsTxt,\n  parseSitemap,\n  robotsAllows,\n  robotsRulesFor,\n  type RobotsRule,\n} from './site-import-robots'\n\n/**\n * Where the bot's user agent points a site owner who wants to know what it is\n * (AGL-3983): the operator's own page about it. `SITE_IMPORT_BOT_INFO_URL` on\n * a self-hosted deployment, so its crawler never names Aglyn's address; the\n * literal is only that variable's default.\n *\n * Aglyn's is the Bring my site guide, which says what AglynBot reads and how\n * robots.txt keeps it out. There is no page of its own (Zach, 10/20: \"we don't\n * need a bot page at all\"), and the site it reads is the reader's own: an\n * import starts only once its owner has proved the site is theirs.\n */\nexport const AGLYN_BOT_INFO_URL =\n  process.env['SITE_IMPORT_BOT_INFO_URL'] ?? 'https://docs.aglyn.com/ai/bring-my-site'\n\n/** What every crawler request says it is. */\nexport const AGLYN_BOT_USER_AGENT = `AglynBot/1.0 (+${AGLYN_BOT_INFO_URL})`\n\nexport const SITE_IMPORT_DEFAULTS = {\n  maxPageBytes: 2 * 1024 * 1024,\n  maxTotalBytes: 24 * 1024 * 1024,\n  timeBudgetMs: 60_000,\n  /** One request's own clock, inside the whole crawl's. */\n  requestTimeoutMs: 8_000,\n  robotsMaxBytes: 256 * 1024,\n  sitemapMaxBytes: 4 * 1024 * 1024,\n  /** Sitemap documents read, an index's children included. */\n  maxSitemaps: 6,\n  /** URLs a sitemap may contribute to the queue. */\n  maxSitemapUrls: 2_000,\n  /** Reasons kept in `skipped` before the list stops growing. */\n  maxSkipped: 200,\n} as const\n\n/** Never a page: files a link may point at that are not HTML. */\nconst NON_PAGE_EXTENSION =\n  /\\.(?:pdf|jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|zip|gz|tgz|rar|7z|mp4|m4v|mov|webm|mp3|wav|ogg|m4a|docx?|xlsx?|pptx?|csv|txt|css|js|mjs|json|xml|rss|atom|woff2?|ttf|otf|eot|exe|dmg|apk)$/i\n\nconst TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|mc_cid|mc_eid|_ga|ref)$/i\n\nconst DOMAIN_PATTERN = /^(?!-)[a-z0-9-]{1,63}(\\.[a-z0-9-]{1,63})+$/\n\nexport type SiteImportSkipReason =\n  /** The page has a password field: a sign-in page, refused outright. */\n  | 'login-form'\n  /** robots.txt keeps AglynBot off this path. */\n  | 'robots'\n  /** A redirect left the site. */\n  | 'off-origin-redirect'\n  /** Over the per-page byte cap. */\n  | 'too-large'\n  /** Answered with something that is not HTML. */\n  | 'not-html'\n  /** Answered 4xx or 5xx. */\n  | 'http-error'\n  /** No answer, a private address, or a refused address. */\n  | 'unreachable'\n\nexport interface SiteImportSkipped {\n  url: string\n  reason: SiteImportSkipReason\n  status?: number\n  /** Where an off-origin redirect pointed. */\n  location?: string\n}\n\nexport interface SiteImportPage {\n  /** The address the page was read from, after any same-site redirect. */\n  url: string\n  path: string\n  /** Where the crawler learned of the page. */\n  foundBy: 'home' | 'sitemap' | 'link'\n  title: string\n  description: string\n  lang: string | null\n  canonical: string | null\n  blocks: SiteImportBlock[]\n  images: SiteImportImage[]\n  navLinks: SiteImportNavLink[]\n  /** Same-site page addresses this page links to. */\n  links: string[]\n  forms: { kept: SiteImportForm[]; removed: number }\n  bytes: number\n}\n\nexport type SiteImportStop =\n  | 'page-budget'\n  | 'total-bytes'\n  | 'time'\n  /** robots.txt could not be read, so nothing was asked of the site. */\n  | 'robots-unreachable'\n\nexport interface SiteImportInventory {\n  domain: string\n  /** The site's origins: the bare name and its `www.` twin. */\n  origins: string[]\n  /** Set when nothing was fetched at all. */\n  refused: 'invalid-domain' | 'lookalike' | null\n  pages: SiteImportPage[]\n  skipped: SiteImportSkipped[]\n  /** Skips past {@link SITE_IMPORT_DEFAULTS.maxSkipped}, counted only. */\n  skippedOverflow: number\n  robots: { found: boolean; sitemaps: string[] }\n  /** Page addresses the sitemaps listed (same-site, before robots and budget). */\n  sitemapUrls: number\n  stoppedBy: SiteImportStop | null\n  bytesRead: number\n  startedAt: number\n  finishedAt: number\n}\n\nexport interface SiteImportCrawlOptions {\n  /** The proved domain; `www.` is folded in. */\n  domain: string\n  /** The most pages the inventory may hold: the plan's page count. */\n  maxPages: number\n  maxPageBytes?: number\n  maxTotalBytes?: number\n  timeBudgetMs?: number\n  signal?: AbortSignal\n  /**\n   * Read the sitemap's shallow addresses before its deep ones (AGL-3976), so a\n   * crawl the page budget stops reads a site's main pages before the tail of\n   * its blog. A sitemap's own order is kept among addresses of one depth.\n   */\n  shallowFirst?: boolean\n  /**\n   * Read exactly these addresses of the site, in order, and nothing else\n   * (AGL-3976): an import's wave reads its own pages again when it starts. No\n   * sitemap is read and no link is followed; robots.txt is read and honored\n   * as for any crawl, and every other rule holds.\n   */\n  only?: readonly string[]\n}\n\nexport interface SiteImportCrawlSeams {\n  fetch?: (url: string, options: PublicFetchOptions) => Promise<PublicFetchResult>\n  now?: () => number\n}\n\n/** The site's two origins for a domain. */\nexport function siteImportOrigins(domain: string): string[] {\n  const bare = domain.replace(/^www\\./, '')\n  return [`https://${bare}`, `https://www.${bare}`]\n}\n\n/** A crawlable page address on the site, normalized for de-duplication; else null. */\nexport function siteImportPageKey(url: URL, origins: readonly string[]): string | null {\n  if (url.protocol !== 'https:' || url.username || url.password) return null\n  if (url.port && url.port !== '443') return null\n  if (!origins.includes(url.origin)) return null\n  if (NON_PAGE_EXTENSION.test(url.pathname)) return null\n  const clean = new URL(url.toString())\n  clean.hash = ''\n  for (const key of [...clean.searchParams.keys()]) {\n    if (TRACKING_PARAM.test(key)) clean.searchParams.delete(key)\n  }\n  clean.search = clean.searchParams.toString() ? `?${clean.searchParams.toString()}` : ''\n  return clean.toString()\n}\n\nfunction skipReasonFor(result: PublicFetchResult): SiteImportSkipReason {\n  switch (result.refusal) {\n    case 'redirect-refused':\n      return 'off-origin-redirect'\n    case 'content-type':\n      return 'not-html'\n    case 'too-large':\n      return 'too-large'\n    case 'http-status':\n      return 'http-error'\n    default:\n      return 'unreachable'\n  }\n}\n\n/** Read a proved site into an inventory. Never throws for anything the site does. */\nexport async function crawlSiteForImport(\n  options: SiteImportCrawlOptions,\n  seams: SiteImportCrawlSeams = {},\n): Promise<SiteImportInventory> {\n  const now = seams.now ?? Date.now\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const domain = String(options.domain ?? '').trim().toLowerCase().replace(/\\.+$/, '').replace(/^www\\./, '')\n  const origins = DOMAIN_PATTERN.test(domain) ? siteImportOrigins(domain) : []\n  const startedAt = now()\n  const inventory: SiteImportInventory = {\n    domain,\n    origins,\n    refused: null,\n    pages: [],\n    skipped: [],\n    skippedOverflow: 0,\n    robots: { found: false, sitemaps: [] },\n    sitemapUrls: 0,\n    stoppedBy: null,\n    bytesRead: 0,\n    startedAt,\n    finishedAt: startedAt,\n  }\n  const finish = () => {\n    inventory.finishedAt = now()\n    return inventory\n  }\n  if (!origins.length) {\n    inventory.refused = 'invalid-domain'\n    return finish()\n  }\n  if (lookalikeBrandForHost(domain)) {\n    inventory.refused = 'lookalike'\n    return finish()\n  }\n\n  const maxPages = Math.max(0, Math.floor(options.maxPages))\n  const maxPageBytes = options.maxPageBytes ?? SITE_IMPORT_DEFAULTS.maxPageBytes\n  const maxTotalBytes = options.maxTotalBytes ?? SITE_IMPORT_DEFAULTS.maxTotalBytes\n  const deadlineAt = startedAt + (options.timeBudgetMs ?? SITE_IMPORT_DEFAULTS.timeBudgetMs)\n  const controller = new AbortController()\n  const onAbort = () => controller.abort()\n  options.signal?.addEventListener('abort', onAbort)\n  if (options.signal?.aborted) controller.abort()\n  const timer = setTimeout(() => controller.abort(), Math.max(0, deadlineAt - startedAt))\n\n  const sameSite = (url: URL) => origins.includes(url.origin) && url.protocol === 'https:'\n  const remainingBytes = () => maxTotalBytes - inventory.bytesRead\n  const outOfTime = () => controller.signal.aborted || now() >= deadlineAt\n  const request = (url: string, accept: RegExp, acceptHeader: string, maxBytes: number) =>\n    fetchText(url, {\n      maxBytes,\n      accept,\n      acceptHeader,\n      userAgent: AGLYN_BOT_USER_AGENT,\n      signal: controller.signal,\n      timeoutMs: Math.max(1, Math.min(SITE_IMPORT_DEFAULTS.requestTimeoutMs, deadlineAt - now())),\n      redirectAllowed: (_from, to) => sameSite(to),\n      refuseOver: true,\n    })\n  const skip = (entry: SiteImportSkipped) => {\n    if (inventory.skipped.length < SITE_IMPORT_DEFAULTS.maxSkipped) inventory.skipped.push(entry)\n    else inventory.skippedOverflow += 1\n  }\n\n  try {\n    // robots.txt, before anything else is asked of the site.\n    const home = `${origins[0]}/`\n    let rules: RobotsRule[] = []\n    const robots = await request(\n      `${origins[0]}/robots.txt`,\n      /text\\/plain|text\\/html|^$/i,\n      'text/plain',\n      Math.min(SITE_IMPORT_DEFAULTS.robotsMaxBytes, remainingBytes()),\n    )\n    if (robots.ok) {\n      inventory.bytesRead += robots.bytes\n      const parsed = parseRobotsTxt(robots.text)\n      rules = robotsRulesFor(parsed, AGLYN_BOT_TOKEN)\n      inventory.robots = { found: true, sitemaps: parsed.sitemaps }\n    } else if (\n      !(robots.refusal === 'http-status' && (robots.status ?? 0) >= 400 && (robots.status ?? 0) < 500)\n    ) {\n      inventory.stoppedBy = outOfTime() ? 'time' : 'robots-unreachable'\n      return finish()\n    }\n    const allowed = (key: string) => {\n      const url = new URL(key)\n      return robotsAllows(rules, `${url.pathname}${url.search}`)\n    }\n\n    // A wave's own pages (AGL-3976): exactly those, after robots.txt.\n    const only = options.only\n      ? options.only\n          .map((raw) => {\n            try {\n              return siteImportPageKey(new URL(raw), origins)\n            } catch {\n              return null\n            }\n          })\n          .filter((key): key is string => !!key)\n      : null\n\n    // Sitemaps: the ones robots.txt names on this site, else /sitemap.xml.\n    const sitemapQueue: string[] = []\n    for (const raw of inventory.robots.sitemaps) {\n      try {\n        const url = new URL(raw)\n        if (sameSite(url)) sitemapQueue.push(url.toString())\n      } catch {\n        // A malformed Sitemap line names nothing.\n      }\n    }\n    if (!sitemapQueue.length && !only) sitemapQueue.push(`${origins[0]}/sitemap.xml`)\n    if (only) sitemapQueue.length = 0\n    const sitemapSeen = new Set<string>()\n    const fromSitemap: string[] = []\n    let sitemapsRead = 0\n    while (sitemapQueue.length && sitemapsRead < SITE_IMPORT_DEFAULTS.maxSitemaps && !outOfTime()) {\n      const next = sitemapQueue.shift() as string\n      if (sitemapSeen.has(next)) continue\n      sitemapSeen.add(next)\n      if (/\\.gz$/i.test(new URL(next).pathname)) continue\n      if (!allowed(next)) continue\n      const room = Math.min(SITE_IMPORT_DEFAULTS.sitemapMaxBytes, remainingBytes())\n      if (room <= 0) break\n      sitemapsRead += 1\n      const sitemap = await request(next, /xml|text\\/plain/i, 'application/xml, text/xml;q=0.9', room)\n      if (!sitemap.ok) continue\n      inventory.bytesRead += sitemap.bytes\n      const entries = parseSitemap(sitemap.text)\n      for (const location of entries.locations) {\n        let url: URL\n        try {\n          url = new URL(location)\n        } catch {\n          continue\n        }\n        if (entries.index) {\n          if (sameSite(url)) sitemapQueue.push(url.toString())\n          continue\n        }\n        const key = siteImportPageKey(url, origins)\n        if (key && fromSitemap.length < SITE_IMPORT_DEFAULTS.maxSitemapUrls) fromSitemap.push(key)\n      }\n    }\n    inventory.sitemapUrls = fromSitemap.length\n    if (options.shallowFirst) {\n      const depth = (key: string) => new URL(key).pathname.split('/').filter(Boolean).length\n      const ranked = fromSitemap.map((key, order) => ({ key, order, depth: depth(key) }))\n      ranked.sort((a, b) => a.depth - b.depth || a.order - b.order)\n      fromSitemap.splice(0, fromSitemap.length, ...ranked.map((entry) => entry.key))\n    }\n\n    // Pages: home, then the sitemap's, then links the pages themselves name.\n    // A wave's read is its own pages alone.\n    const queue: { key: string; foundBy: SiteImportPage['foundBy'] }[] = only\n      ? only.map((key) => ({ key, foundBy: (new URL(key).pathname === '/' ? 'home' : 'link') as SiteImportPage['foundBy'] }))\n      : [{ key: home, foundBy: 'home' }]\n    const queued = new Set<string>(only ?? [home])\n    for (const key of fromSitemap) {\n      if (queued.has(key)) continue\n      queued.add(key)\n      queue.push({ key, foundBy: 'sitemap' })\n    }\n    const read = new Set<string>()\n    // The site's two names are one site: once the home page says which one\n    // it answers on, every address is read on that one, so /about is not\n    // read twice as example.com/about and www.example.com/about.\n    // A wave's read starts on the name its pages were read on the first time.\n    let preferred = only?.length ? new URL(only[0]).origin : origins[0]\n    const canon = (raw: string) => {\n      const url = new URL(raw)\n      return origins.includes(url.origin) && url.origin !== preferred\n        ? `${preferred}${url.pathname}${url.search}`\n        : raw\n    }\n    const maxAttempts = maxPages * 3 + 10\n    let attempts = 0\n    while (queue.length) {\n      if (inventory.pages.length >= maxPages) {\n        inventory.stoppedBy = 'page-budget'\n        break\n      }\n      if (outOfTime()) {\n        inventory.stoppedBy = 'time'\n        break\n      }\n      if (remainingBytes() <= 0) {\n        inventory.stoppedBy = 'total-bytes'\n        break\n      }\n      if (attempts >= maxAttempts) {\n        inventory.stoppedBy = 'page-budget'\n        break\n      }\n      const next = queue.shift() as (typeof queue)[number]\n      const key = canon(next.key)\n      const foundBy = next.foundBy\n      if (read.has(key)) continue\n      if (!allowed(key)) {\n        skip({ url: key, reason: 'robots' })\n        continue\n      }\n      attempts += 1\n      read.add(key)\n      const cap = Math.min(maxPageBytes, remainingBytes())\n      const page = await request(key, /text\\/html|application\\/xhtml\\+xml/i, 'text/html, application/xhtml+xml;q=0.9', cap)\n      if (!page.ok) {\n        if (page.refusal === 'too-large' && cap < maxPageBytes) {\n          inventory.stoppedBy = 'total-bytes'\n          break\n        }\n        if (page.refusal === 'network' && outOfTime()) {\n          inventory.stoppedBy = 'time'\n          break\n        }\n        skip({\n          url: key,\n          reason: skipReasonFor(page),\n          ...(page.status ? { status: page.status } : {}),\n          ...(page.location ? { location: page.location } : {}),\n        })\n        continue\n      }\n      inventory.bytesRead += page.bytes\n      if (foundBy === 'home') preferred = new URL(page.url).origin\n      const finalKey = canon(siteImportPageKey(new URL(page.url), origins) ?? page.url)\n      if (finalKey !== key) {\n        if (read.has(finalKey)) continue\n        read.add(finalKey)\n        if (!allowed(finalKey)) {\n          skip({ url: finalKey, reason: 'robots' })\n          continue\n        }\n      }\n      const extract = extractSiteImportPage(page.text, page.url, sameSite)\n      if (extract.loginForm) {\n        skip({ url: finalKey, reason: 'login-form' })\n        continue\n      }\n      const links: string[] = []\n      for (const link of extract.links) {\n        const linkKey = siteImportPageKey(new URL(link), origins)\n        if (!linkKey) continue\n        links.push(linkKey)\n        if (!only && !queued.has(linkKey)) {\n          queued.add(linkKey)\n          queue.push({ key: linkKey, foundBy: 'link' })\n        }\n      }\n      inventory.pages.push({\n        url: finalKey,\n        path: new URL(finalKey).pathname,\n        foundBy,\n        title: extract.title,\n        description: extract.description,\n        lang: extract.lang,\n        canonical: extract.canonical,\n        blocks: extract.blocks,\n        images: extract.images,\n        navLinks: extract.navLinks,\n        links: [...new Set(links.map(canon))],\n        forms: extract.forms,\n        bytes: page.bytes,\n      })\n    }\n    return finish()\n  } finally {\n    clearTimeout(timer)\n    options.signal?.removeEventListener('abort', onAbort)\n  }\n}\n\n/**\n * The home page's `aglyn-site-verification` values, for the meta-tag proof:\n * the same fetch, user agent and same-site redirects as the crawl, but no\n * robots.txt, because the person asked for this one page to be read.\n * `answered: false` when the page could not be read at all.\n */\nexport async function readSiteVerificationMeta(\n  domain: string,\n  metaName: string,\n  seams: SiteImportCrawlSeams = {},\n): Promise<{ answered: boolean; values: string[]; url?: string }> {\n  const fetchText = seams.fetch ?? fetchPublicTextDetailed\n  const bare = String(domain ?? '').trim().toLowerCase().replace(/^www\\./, '')\n  if (!DOMAIN_PATTERN.test(bare) || lookalikeBrandForHost(bare)) return { answered: false, values: [] }\n  const origins = siteImportOrigins(bare)\n  const result = await fetchText(`${origins[0]}/`, {\n    maxBytes: SITE_IMPORT_DEFAULTS.maxPageBytes,\n    accept: /text\\/html|application\\/xhtml\\+xml/i,\n    acceptHeader: 'text/html, application/xhtml+xml;q=0.9',\n    userAgent: AGLYN_BOT_USER_AGENT,\n    timeoutMs: SITE_IMPORT_DEFAULTS.requestTimeoutMs,\n    redirectAllowed: (_from, to) => to.protocol === 'https:' && origins.includes(to.origin),\n  })\n  if (!result.ok) {\n    // An answer that is not the page (a 404, a redirect off the site) is\n    // still an answer: the tag is not where it must be.\n    const answered = result.refusal === 'http-status' || result.refusal === 'redirect-refused' || result.refusal === 'content-type'\n    return { answered, values: [], url: result.url }\n  }\n  const head = result.text.split(/<body\\b/i)[0] ?? ''\n  return { answered: true, values: readMetaValues(head, metaName), url: result.url }\n}\n\n/**\n * One page as text for a prompt: its title, description and blocks in order,\n * fenced as untrusted data. The importer's prompt carries `UNTRUSTED_TEXT_RULE`.\n */\nexport function siteImportPagePromptText(page: SiteImportPage, maxChars = 12_000): string {\n  const lines: string[] = []\n  if (page.title) lines.push(`Title: ${page.title}`)\n  if (page.description) lines.push(`Description: ${page.description}`)\n  for (const block of page.blocks) {\n    if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n    else if (block.kind === 'image') lines.push(`[image: ${block.alt || 'no description'}]`)\n    else if (block.kind === 'list-item') lines.push(`- ${block.text}`)\n    else if (block.kind === 'quote') lines.push(`> ${block.text}`)\n    else lines.push(block.text)\n  }\n  return fenceUntrustedText(`Page ${page.path} of the site being imported`, lines.join('\\n'), { maxChars })\n}\n"],"names":["process","lookalikeBrandForHost","fenceUntrustedText","fetchPublicTextDetailed","extractSiteImportPage","readMetaValues","AGLYN_BOT_TOKEN","parseRobotsTxt","parseSitemap","robotsAllows","robotsRulesFor","AGLYN_BOT_INFO_URL","env","AGLYN_BOT_USER_AGENT","SITE_IMPORT_DEFAULTS","maxPageBytes","maxTotalBytes","timeBudgetMs","requestTimeoutMs","robotsMaxBytes","sitemapMaxBytes","maxSitemaps","maxSitemapUrls","maxSkipped","NON_PAGE_EXTENSION","TRACKING_PARAM","DOMAIN_PATTERN","siteImportOrigins","domain","bare","replace","siteImportPageKey","url","origins","protocol","username","password","port","includes","origin","test","pathname","clean","URL","toString","hash","key","searchParams","keys","delete","search","skipReasonFor","result","refusal","crawlSiteForImport","options","seams","now","Date","fetchText","fetch","String","trim","toLowerCase","startedAt","inventory","refused","pages","skipped","skippedOverflow","robots","found","sitemaps","sitemapUrls","stoppedBy","bytesRead","finishedAt","finish","length","maxPages","Math","max","floor","deadlineAt","controller","AbortController","onAbort","abort","signal","addEventListener","aborted","timer","setTimeout","sameSite","remainingBytes","outOfTime","request","accept","acceptHeader","maxBytes","userAgent","timeoutMs","min","redirectAllowed","_from","to","refuseOver","skip","entry","push","home","rules","ok","bytes","parsed","text","status","allowed","only","map","raw","filter","sitemapQueue","sitemapSeen","Set","fromSitemap","sitemapsRead","next","shift","has","add","room","sitemap","entries","location","locations","index","shallowFirst","depth","split","Boolean","ranked","order","sort","a","b","splice","queue","foundBy","queued","read","preferred","canon","maxAttempts","attempts","reason","cap","page","finalKey","extract","loginForm","links","link","linkKey","path","title","description","lang","canonical","blocks","images","navLinks","forms","clearTimeout","removeEventListener","readSiteVerificationMeta","metaName","answered","values","head","siteImportPagePromptText","maxChars","lines","block","kind","repeat","level","alt","join"],"mappings":";IAsFEA;AAtFF;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2CA+B2C,GAE3C,SAASC,qBAAqB,QAAQ,oDAAmD;AACzF,SAASC,kBAAkB,QAAQ,0CAAyC;AAC5E,SACEC,uBAAuB,QAGlB,mCAA+B;AACtC,SACEC,qBAAqB,EACrBC,cAAc,QAKT,wBAAoB;AAC3B,SACEC,eAAe,EACfC,cAAc,EACdC,YAAY,EACZC,YAAY,EACZC,cAAc,QAET,0BAAsB;AAE7B;;;;;;;;;;CAUC,GACD,OAAO,MAAMC,sBACXX,wCAAAA,QAAQY,GAAG,CAAC,2BAA2B,YAAvCZ,wCAA2C,0CAAyC;AAEtF,2CAA2C,GAC3C,OAAO,MAAMa,uBAAuB,CAAC,eAAe,EAAEF,mBAAmB,CAAC,CAAC,CAAA;AAE3E,OAAO,MAAMG,uBAAuB;IAClCC,cAAc,IAAI,OAAO;IACzBC,eAAe,KAAK,OAAO;IAC3BC,cAAc;IACd,uDAAuD,GACvDC,kBAAkB;IAClBC,gBAAgB,MAAM;IACtBC,iBAAiB,IAAI,OAAO;IAC5B,0DAA0D,GAC1DC,aAAa;IACb,gDAAgD,GAChDC,gBAAgB;IAChB,6DAA6D,GAC7DC,YAAY;AACd,EAAU;AAEV,+DAA+D,GAC/D,MAAMC,qBACJ;AAEF,MAAMC,iBAAiB;AAEvB,MAAMC,iBAAiB;AAoGvB,yCAAyC,GACzC,OAAO,SAASC,kBAAkBC,MAAc;IAC9C,MAAMC,OAAOD,OAAOE,OAAO,CAAC,UAAU;IACtC,OAAO;QAAC,CAAC,QAAQ,EAAED,MAAM;QAAE,CAAC,YAAY,EAAEA,MAAM;KAAC;AACnD;AAEA,oFAAoF,GACpF,OAAO,SAASE,kBAAkBC,GAAQ,EAAEC,OAA0B;IACpE,IAAID,IAAIE,QAAQ,KAAK,YAAYF,IAAIG,QAAQ,IAAIH,IAAII,QAAQ,EAAE,OAAO;IACtE,IAAIJ,IAAIK,IAAI,IAAIL,IAAIK,IAAI,KAAK,OAAO,OAAO;IAC3C,IAAI,CAACJ,QAAQK,QAAQ,CAACN,IAAIO,MAAM,GAAG,OAAO;IAC1C,IAAIf,mBAAmBgB,IAAI,CAACR,IAAIS,QAAQ,GAAG,OAAO;IAClD,MAAMC,QAAQ,IAAIC,IAAIX,IAAIY,QAAQ;IAClCF,MAAMG,IAAI,GAAG;IACb,KAAK,MAAMC,OAAO;WAAIJ,MAAMK,YAAY,CAACC,IAAI;KAAG,CAAE;QAChD,IAAIvB,eAAee,IAAI,CAACM,MAAMJ,MAAMK,YAAY,CAACE,MAAM,CAACH;IAC1D;IACAJ,MAAMQ,MAAM,GAAGR,MAAMK,YAAY,CAACH,QAAQ,KAAK,CAAC,CAAC,EAAEF,MAAMK,YAAY,CAACH,QAAQ,IAAI,GAAG;IACrF,OAAOF,MAAME,QAAQ;AACvB;AAEA,SAASO,cAAcC,MAAyB;IAC9C,OAAQA,OAAOC,OAAO;QACpB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,mFAAmF,GACnF,OAAO,eAAeC,mBACpBC,OAA+B,EAC/BC,QAA8B,CAAC,CAAC;QAEpBA,YACMA,cACID,iBA+BDA,uBACCA,wBACUA;QAGhCA,iBACIA;IAvCJ,MAAME,OAAMD,aAAAA,MAAMC,GAAG,YAATD,aAAaE,KAAKD,GAAG;IACjC,MAAME,aAAYH,eAAAA,MAAMI,KAAK,YAAXJ,eAAerD;IACjC,MAAMyB,SAASiC,QAAON,kBAAAA,QAAQ3B,MAAM,YAAd2B,kBAAkB,IAAIO,IAAI,GAAGC,WAAW,GAAGjC,OAAO,CAAC,QAAQ,IAAIA,OAAO,CAAC,UAAU;IACvG,MAAMG,UAAUP,eAAec,IAAI,CAACZ,UAAUD,kBAAkBC,UAAU,EAAE;IAC5E,MAAMoC,YAAYP;IAClB,MAAMQ,YAAiC;QACrCrC;QACAK;QACAiC,SAAS;QACTC,OAAO,EAAE;QACTC,SAAS,EAAE;QACXC,iBAAiB;QACjBC,QAAQ;YAAEC,OAAO;YAAOC,UAAU,EAAE;QAAC;QACrCC,aAAa;QACbC,WAAW;QACXC,WAAW;QACXX;QACAY,YAAYZ;IACd;IACA,MAAMa,SAAS;QACbZ,UAAUW,UAAU,GAAGnB;QACvB,OAAOQ;IACT;IACA,IAAI,CAAChC,QAAQ6C,MAAM,EAAE;QACnBb,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IACA,IAAI5E,sBAAsB2B,SAAS;QACjCqC,UAAUC,OAAO,GAAG;QACpB,OAAOW;IACT;IAEA,MAAME,WAAWC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAAC3B,QAAQwB,QAAQ;IACxD,MAAMhE,gBAAewC,wBAAAA,QAAQxC,YAAY,YAApBwC,wBAAwBzC,qBAAqBC,YAAY;IAC9E,MAAMC,iBAAgBuC,yBAAAA,QAAQvC,aAAa,YAArBuC,yBAAyBzC,qBAAqBE,aAAa;IACjF,MAAMmE,aAAanB,cAAaT,wBAAAA,QAAQtC,YAAY,YAApBsC,wBAAwBzC,qBAAqBG,YAAY;IACzF,MAAMmE,aAAa,IAAIC;IACvB,MAAMC,UAAU,IAAMF,WAAWG,KAAK;KACtChC,kBAAAA,QAAQiC,MAAM,qBAAdjC,gBAAgBkC,gBAAgB,CAAC,SAASH;IAC1C,KAAI/B,mBAAAA,QAAQiC,MAAM,qBAAdjC,iBAAgBmC,OAAO,EAAEN,WAAWG,KAAK;IAC7C,MAAMI,QAAQC,WAAW,IAAMR,WAAWG,KAAK,IAAIP,KAAKC,GAAG,CAAC,GAAGE,aAAanB;IAE5E,MAAM6B,WAAW,CAAC7D,MAAaC,QAAQK,QAAQ,CAACN,IAAIO,MAAM,KAAKP,IAAIE,QAAQ,KAAK;IAChF,MAAM4D,iBAAiB,IAAM9E,gBAAgBiD,UAAUU,SAAS;IAChE,MAAMoB,YAAY,IAAMX,WAAWI,MAAM,CAACE,OAAO,IAAIjC,SAAS0B;IAC9D,MAAMa,UAAU,CAAChE,KAAaiE,QAAgBC,cAAsBC,WAClExC,UAAU3B,KAAK;YACbmE;YACAF;YACAC;YACAE,WAAWvF;YACX2E,QAAQJ,WAAWI,MAAM;YACzBa,WAAWrB,KAAKC,GAAG,CAAC,GAAGD,KAAKsB,GAAG,CAACxF,qBAAqBI,gBAAgB,EAAEiE,aAAa1B;YACpF8C,iBAAiB,CAACC,OAAOC,KAAOZ,SAASY;YACzCC,YAAY;QACd;IACF,MAAMC,OAAO,CAACC;QACZ,IAAI3C,UAAUG,OAAO,CAACU,MAAM,GAAGhE,qBAAqBS,UAAU,EAAE0C,UAAUG,OAAO,CAACyC,IAAI,CAACD;aAClF3C,UAAUI,eAAe,IAAI;IACpC;IAEA,IAAI;YAgBuCC,gBAA+BA;QAfxE,yDAAyD;QACzD,MAAMwC,OAAO,GAAG7E,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,IAAI8E,QAAsB,EAAE;QAC5B,MAAMzC,SAAS,MAAM0B,QACnB,GAAG/D,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,EAC1B,8BACA,cACA+C,KAAKsB,GAAG,CAACxF,qBAAqBK,cAAc,EAAE2E;QAEhD,IAAIxB,OAAO0C,EAAE,EAAE;YACb/C,UAAUU,SAAS,IAAIL,OAAO2C,KAAK;YACnC,MAAMC,SAAS3G,eAAe+D,OAAO6C,IAAI;YACzCJ,QAAQrG,eAAewG,QAAQ5G;YAC/B2D,UAAUK,MAAM,GAAG;gBAAEC,OAAO;gBAAMC,UAAU0C,OAAO1C,QAAQ;YAAC;QAC9D,OAAO,IACL,CAAEF,CAAAA,OAAOjB,OAAO,KAAK,iBAAiB,EAACiB,iBAAAA,OAAO8C,MAAM,YAAb9C,iBAAiB,MAAM,OAAO,EAACA,kBAAAA,OAAO8C,MAAM,YAAb9C,kBAAiB,KAAK,GAAE,GAC9F;YACAL,UAAUS,SAAS,GAAGqB,cAAc,SAAS;YAC7C,OAAOlB;QACT;QACA,MAAMwC,UAAU,CAACvE;YACf,MAAMd,MAAM,IAAIW,IAAIG;YACpB,OAAOrC,aAAasG,OAAO,GAAG/E,IAAIS,QAAQ,GAAGT,IAAIkB,MAAM,EAAE;QAC3D;QAEA,kEAAkE;QAClE,MAAMoE,OAAO/D,QAAQ+D,IAAI,GACrB/D,QAAQ+D,IAAI,CACTC,GAAG,CAAC,CAACC;YACJ,IAAI;gBACF,OAAOzF,kBAAkB,IAAIY,IAAI6E,MAAMvF;YACzC,EAAE,eAAM;gBACN,OAAO;YACT;QACF,GACCwF,MAAM,CAAC,CAAC3E,MAAuB,CAAC,CAACA,OACpC;QAEJ,uEAAuE;QACvE,MAAM4E,eAAyB,EAAE;QACjC,KAAK,MAAMF,OAAOvD,UAAUK,MAAM,CAACE,QAAQ,CAAE;YAC3C,IAAI;gBACF,MAAMxC,MAAM,IAAIW,IAAI6E;gBACpB,IAAI3B,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIY,QAAQ;YACnD,EAAE,eAAM;YACN,0CAA0C;YAC5C;QACF;QACA,IAAI,CAAC8E,aAAa5C,MAAM,IAAI,CAACwC,MAAMI,aAAab,IAAI,CAAC,GAAG5E,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC;QAChF,IAAIqF,MAAMI,aAAa5C,MAAM,GAAG;QAChC,MAAM6C,cAAc,IAAIC;QACxB,MAAMC,cAAwB,EAAE;QAChC,IAAIC,eAAe;QACnB,MAAOJ,aAAa5C,MAAM,IAAIgD,eAAehH,qBAAqBO,WAAW,IAAI,CAAC0E,YAAa;YAC7F,MAAMgC,OAAOL,aAAaM,KAAK;YAC/B,IAAIL,YAAYM,GAAG,CAACF,OAAO;YAC3BJ,YAAYO,GAAG,CAACH;YAChB,IAAI,SAASvF,IAAI,CAAC,IAAIG,IAAIoF,MAAMtF,QAAQ,GAAG;YAC3C,IAAI,CAAC4E,QAAQU,OAAO;YACpB,MAAMI,OAAOnD,KAAKsB,GAAG,CAACxF,qBAAqBM,eAAe,EAAE0E;YAC5D,IAAIqC,QAAQ,GAAG;YACfL,gBAAgB;YAChB,MAAMM,UAAU,MAAMpC,QAAQ+B,MAAM,oBAAoB,mCAAmCI;YAC3F,IAAI,CAACC,QAAQpB,EAAE,EAAE;YACjB/C,UAAUU,SAAS,IAAIyD,QAAQnB,KAAK;YACpC,MAAMoB,UAAU7H,aAAa4H,QAAQjB,IAAI;YACzC,KAAK,MAAMmB,YAAYD,QAAQE,SAAS,CAAE;gBACxC,IAAIvG;gBACJ,IAAI;oBACFA,MAAM,IAAIW,IAAI2F;gBAChB,EAAE,eAAM;oBACN;gBACF;gBACA,IAAID,QAAQG,KAAK,EAAE;oBACjB,IAAI3C,SAAS7D,MAAM0F,aAAab,IAAI,CAAC7E,IAAIY,QAAQ;oBACjD;gBACF;gBACA,MAAME,MAAMf,kBAAkBC,KAAKC;gBACnC,IAAIa,OAAO+E,YAAY/C,MAAM,GAAGhE,qBAAqBQ,cAAc,EAAEuG,YAAYhB,IAAI,CAAC/D;YACxF;QACF;QACAmB,UAAUQ,WAAW,GAAGoD,YAAY/C,MAAM;QAC1C,IAAIvB,QAAQkF,YAAY,EAAE;YACxB,MAAMC,QAAQ,CAAC5F,MAAgB,IAAIH,IAAIG,KAAKL,QAAQ,CAACkG,KAAK,CAAC,KAAKlB,MAAM,CAACmB,SAAS9D,MAAM;YACtF,MAAM+D,SAAShB,YAAYN,GAAG,CAAC,CAACzE,KAAKgG,QAAW,CAAA;oBAAEhG;oBAAKgG;oBAAOJ,OAAOA,MAAM5F;gBAAK,CAAA;YAChF+F,OAAOE,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEN,KAAK,GAAGO,EAAEP,KAAK,IAAIM,EAAEF,KAAK,GAAGG,EAAEH,KAAK;YAC5DjB,YAAYqB,MAAM,CAAC,GAAGrB,YAAY/C,MAAM,KAAK+D,OAAOtB,GAAG,CAAC,CAACX,QAAUA,MAAM9D,GAAG;QAC9E;QAEA,yEAAyE;QACzE,wCAAwC;QACxC,MAAMqG,QAA+D7B,OACjEA,KAAKC,GAAG,CAAC,CAACzE,MAAS,CAAA;gBAAEA;gBAAKsG,SAAU,IAAIzG,IAAIG,KAAKL,QAAQ,KAAK,MAAM,SAAS;YAAqC,CAAA,KAClH;YAAC;gBAAEK,KAAKgE;gBAAMsC,SAAS;YAAO;SAAE;QACpC,MAAMC,SAAS,IAAIzB,IAAYN,eAAAA,OAAQ;YAACR;SAAK;QAC7C,KAAK,MAAMhE,OAAO+E,YAAa;YAC7B,IAAIwB,OAAOpB,GAAG,CAACnF,MAAM;YACrBuG,OAAOnB,GAAG,CAACpF;YACXqG,MAAMtC,IAAI,CAAC;gBAAE/D;gBAAKsG,SAAS;YAAU;QACvC;QACA,MAAME,OAAO,IAAI1B;QACjB,uEAAuE;QACvE,qEAAqE;QACrE,6DAA6D;QAC7D,0EAA0E;QAC1E,IAAI2B,YAAYjC,CAAAA,wBAAAA,KAAMxC,MAAM,IAAG,IAAInC,IAAI2E,IAAI,CAAC,EAAE,EAAE/E,MAAM,GAAGN,OAAO,CAAC,EAAE;QACnE,MAAMuH,QAAQ,CAAChC;YACb,MAAMxF,MAAM,IAAIW,IAAI6E;YACpB,OAAOvF,QAAQK,QAAQ,CAACN,IAAIO,MAAM,KAAKP,IAAIO,MAAM,KAAKgH,YAClD,GAAGA,YAAYvH,IAAIS,QAAQ,GAAGT,IAAIkB,MAAM,EAAE,GAC1CsE;QACN;QACA,MAAMiC,cAAc1E,WAAW,IAAI;QACnC,IAAI2E,WAAW;QACf,MAAOP,MAAMrE,MAAM,CAAE;gBAgDI/C;YA/CvB,IAAIkC,UAAUE,KAAK,CAACW,MAAM,IAAIC,UAAU;gBACtCd,UAAUS,SAAS,GAAG;gBACtB;YACF;YACA,IAAIqB,aAAa;gBACf9B,UAAUS,SAAS,GAAG;gBACtB;YACF;YACA,IAAIoB,oBAAoB,GAAG;gBACzB7B,UAAUS,SAAS,GAAG;gBACtB;YACF;YACA,IAAIgF,YAAYD,aAAa;gBAC3BxF,UAAUS,SAAS,GAAG;gBACtB;YACF;YACA,MAAMqD,OAAOoB,MAAMnB,KAAK;YACxB,MAAMlF,MAAM0G,MAAMzB,KAAKjF,GAAG;YAC1B,MAAMsG,UAAUrB,KAAKqB,OAAO;YAC5B,IAAIE,KAAKrB,GAAG,CAACnF,MAAM;YACnB,IAAI,CAACuE,QAAQvE,MAAM;gBACjB6D,KAAK;oBAAE3E,KAAKc;oBAAK6G,QAAQ;gBAAS;gBAClC;YACF;YACAD,YAAY;YACZJ,KAAKpB,GAAG,CAACpF;YACT,MAAM8G,MAAM5E,KAAKsB,GAAG,CAACvF,cAAc+E;YACnC,MAAM+D,OAAO,MAAM7D,QAAQlD,KAAK,uCAAuC,0CAA0C8G;YACjH,IAAI,CAACC,KAAK7C,EAAE,EAAE;gBACZ,IAAI6C,KAAKxG,OAAO,KAAK,eAAeuG,MAAM7I,cAAc;oBACtDkD,UAAUS,SAAS,GAAG;oBACtB;gBACF;gBACA,IAAImF,KAAKxG,OAAO,KAAK,aAAa0C,aAAa;oBAC7C9B,UAAUS,SAAS,GAAG;oBACtB;gBACF;gBACAiC,KAAK;oBACH3E,KAAKc;oBACL6G,QAAQxG,cAAc0G;mBAClBA,KAAKzC,MAAM,GAAG;oBAAEA,QAAQyC,KAAKzC,MAAM;gBAAC,IAAI,CAAC,GACzCyC,KAAKvB,QAAQ,GAAG;oBAAEA,UAAUuB,KAAKvB,QAAQ;gBAAC,IAAI,CAAC;gBAErD;YACF;YACArE,UAAUU,SAAS,IAAIkF,KAAK5C,KAAK;YACjC,IAAImC,YAAY,QAAQG,YAAY,IAAI5G,IAAIkH,KAAK7H,GAAG,EAAEO,MAAM;YAC5D,MAAMuH,WAAWN,OAAMzH,qBAAAA,kBAAkB,IAAIY,IAAIkH,KAAK7H,GAAG,GAAGC,oBAArCF,qBAAiD8H,KAAK7H,GAAG;YAChF,IAAI8H,aAAahH,KAAK;gBACpB,IAAIwG,KAAKrB,GAAG,CAAC6B,WAAW;gBACxBR,KAAKpB,GAAG,CAAC4B;gBACT,IAAI,CAACzC,QAAQyC,WAAW;oBACtBnD,KAAK;wBAAE3E,KAAK8H;wBAAUH,QAAQ;oBAAS;oBACvC;gBACF;YACF;YACA,MAAMI,UAAU3J,sBAAsByJ,KAAK1C,IAAI,EAAE0C,KAAK7H,GAAG,EAAE6D;YAC3D,IAAIkE,QAAQC,SAAS,EAAE;gBACrBrD,KAAK;oBAAE3E,KAAK8H;oBAAUH,QAAQ;gBAAa;gBAC3C;YACF;YACA,MAAMM,QAAkB,EAAE;YAC1B,KAAK,MAAMC,QAAQH,QAAQE,KAAK,CAAE;gBAChC,MAAME,UAAUpI,kBAAkB,IAAIY,IAAIuH,OAAOjI;gBACjD,IAAI,CAACkI,SAAS;gBACdF,MAAMpD,IAAI,CAACsD;gBACX,IAAI,CAAC7C,QAAQ,CAAC+B,OAAOpB,GAAG,CAACkC,UAAU;oBACjCd,OAAOnB,GAAG,CAACiC;oBACXhB,MAAMtC,IAAI,CAAC;wBAAE/D,KAAKqH;wBAASf,SAAS;oBAAO;gBAC7C;YACF;YACAnF,UAAUE,KAAK,CAAC0C,IAAI,CAAC;gBACnB7E,KAAK8H;gBACLM,MAAM,IAAIzH,IAAImH,UAAUrH,QAAQ;gBAChC2G;gBACAiB,OAAON,QAAQM,KAAK;gBACpBC,aAAaP,QAAQO,WAAW;gBAChCC,MAAMR,QAAQQ,IAAI;gBAClBC,WAAWT,QAAQS,SAAS;gBAC5BC,QAAQV,QAAQU,MAAM;gBACtBC,QAAQX,QAAQW,MAAM;gBACtBC,UAAUZ,QAAQY,QAAQ;gBAC1BV,OAAO;uBAAI,IAAIrC,IAAIqC,MAAM1C,GAAG,CAACiC;iBAAQ;gBACrCoB,OAAOb,QAAQa,KAAK;gBACpB3D,OAAO4C,KAAK5C,KAAK;YACnB;QACF;QACA,OAAOpC;IACT,SAAU;YAERtB;QADAsH,aAAalF;SACbpC,mBAAAA,QAAQiC,MAAM,qBAAdjC,iBAAgBuH,mBAAmB,CAAC,SAASxF;IAC/C;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAeyF,yBACpBnJ,MAAc,EACdoJ,QAAgB,EAChBxH,QAA8B,CAAC,CAAC;QAEdA,cAkBLJ;IAlBb,MAAMO,aAAYH,eAAAA,MAAMI,KAAK,YAAXJ,eAAerD;IACjC,MAAM0B,OAAOgC,OAAOjC,iBAAAA,SAAU,IAAIkC,IAAI,GAAGC,WAAW,GAAGjC,OAAO,CAAC,UAAU;IACzE,IAAI,CAACJ,eAAec,IAAI,CAACX,SAAS5B,sBAAsB4B,OAAO,OAAO;QAAEoJ,UAAU;QAAOC,QAAQ,EAAE;IAAC;IACpG,MAAMjJ,UAAUN,kBAAkBE;IAClC,MAAMuB,SAAS,MAAMO,UAAU,GAAG1B,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;QAC/CkE,UAAUrF,qBAAqBC,YAAY;QAC3CkF,QAAQ;QACRC,cAAc;QACdE,WAAWvF;QACXwF,WAAWvF,qBAAqBI,gBAAgB;QAChDqF,iBAAiB,CAACC,OAAOC,KAAOA,GAAGvE,QAAQ,KAAK,YAAYD,QAAQK,QAAQ,CAACmE,GAAGlE,MAAM;IACxF;IACA,IAAI,CAACa,OAAO4D,EAAE,EAAE;QACd,qEAAqE;QACrE,oDAAoD;QACpD,MAAMiE,WAAW7H,OAAOC,OAAO,KAAK,iBAAiBD,OAAOC,OAAO,KAAK,sBAAsBD,OAAOC,OAAO,KAAK;QACjH,OAAO;YAAE4H;YAAUC,QAAQ,EAAE;YAAElJ,KAAKoB,OAAOpB,GAAG;QAAC;IACjD;IACA,MAAMmJ,QAAO/H,sBAAAA,OAAO+D,IAAI,CAACwB,KAAK,CAAC,WAAW,CAAC,EAAE,YAAhCvF,sBAAoC;IACjD,OAAO;QAAE6H,UAAU;QAAMC,QAAQ7K,eAAe8K,MAAMH;QAAWhJ,KAAKoB,OAAOpB,GAAG;IAAC;AACnF;AAEA;;;CAGC,GACD,OAAO,SAASoJ,yBAAyBvB,IAAoB,EAAEwB,WAAW,KAAM;IAC9E,MAAMC,QAAkB,EAAE;IAC1B,IAAIzB,KAAKQ,KAAK,EAAEiB,MAAMzE,IAAI,CAAC,CAAC,OAAO,EAAEgD,KAAKQ,KAAK,EAAE;IACjD,IAAIR,KAAKS,WAAW,EAAEgB,MAAMzE,IAAI,CAAC,CAAC,aAAa,EAAEgD,KAAKS,WAAW,EAAE;IACnE,KAAK,MAAMiB,SAAS1B,KAAKY,MAAM,CAAE;QAC/B,IAAIc,MAAMC,IAAI,KAAK,WAAWF,MAAMzE,IAAI,CAAC,GAAG,IAAI4E,MAAM,CAACF,MAAMG,KAAK,EAAE,CAAC,EAAEH,MAAMpE,IAAI,EAAE;aAC9E,IAAIoE,MAAMC,IAAI,KAAK,SAASF,MAAMzE,IAAI,CAAC,CAAC,QAAQ,EAAE0E,MAAMI,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAClF,IAAIJ,MAAMC,IAAI,KAAK,aAAaF,MAAMzE,IAAI,CAAC,CAAC,EAAE,EAAE0E,MAAMpE,IAAI,EAAE;aAC5D,IAAIoE,MAAMC,IAAI,KAAK,SAASF,MAAMzE,IAAI,CAAC,CAAC,EAAE,EAAE0E,MAAMpE,IAAI,EAAE;aACxDmE,MAAMzE,IAAI,CAAC0E,MAAMpE,IAAI;IAC5B;IACA,OAAOjH,mBAAmB,CAAC,KAAK,EAAE2J,KAAKO,IAAI,CAAC,2BAA2B,CAAC,EAAEkB,MAAMM,IAAI,CAAC,OAAO;QAAEP;IAAS;AACzG"}

package/src/lib/site-import/site-import-flag.d.ts

@@ -0,0 +1,23 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * The release flag every "Bring my site" surface closes on (AGL-3974,
+ * AGL-3976): the ownership and crawl doors, the import job's admission and
+ * each of its waves, and the console's choice card. Off by default; a
+ * module of its own so the job and the doors read it without each other.
+ */
+export declare const SITE_IMPORT_RELEASE_FLAG: "release_ai_site_import";

package/src/lib/site-import/site-import-flag.js

@@ -0,0 +1,23 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ /**
+ * The release flag every "Bring my site" surface closes on (AGL-3974,
+ * AGL-3976): the ownership and crawl doors, the import job's admission and
+ * each of its waves, and the console's choice card. Off by default; a
+ * module of its own so the job and the doors read it without each other.
+ */ export const SITE_IMPORT_RELEASE_FLAG = 'release_ai_site_import';
+
+//# sourceMappingURL=site-import-flag.js.map

package/src/lib/site-import/site-import-flag.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-flag.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * The release flag every \"Bring my site\" surface closes on (AGL-3974,\n * AGL-3976): the ownership and crawl doors, the import job's admission and\n * each of its waves, and the console's choice card. Off by default; a\n * module of its own so the job and the doors read it without each other.\n */\nexport const SITE_IMPORT_RELEASE_FLAG = 'release_ai_site_import' as const\n"],"names":["SITE_IMPORT_RELEASE_FLAG"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;CAKC,GACD,OAAO,MAAMA,2BAA2B,yBAAiC"}

package/src/lib/site-import/site-import-html.d.ts

@@ -0,0 +1,94 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** One run of a page's content, in the order the page draws it. */
+export type SiteImportBlock = {
+    kind: 'heading';
+    level: 1 | 2 | 3 | 4 | 5 | 6;
+    text: string;
+    region: SiteImportRegion;
+} | {
+    kind: 'paragraph' | 'list-item' | 'quote';
+    text: string;
+    region: SiteImportRegion;
+} | {
+    kind: 'image';
+    url: string;
+    alt: string;
+    region: SiteImportRegion;
+};
+/** Where on the page a block sat: the header and footer carry business details. */
+export type SiteImportRegion = 'header' | 'main' | 'footer';
+export interface SiteImportImage {
+    /** Absolute https (or http) address. */
+    url: string;
+    alt: string;
+    /** On the site's own origin; otherwise a CDN or a third party. The importer decides. */
+    sameSite: boolean;
+}
+export interface SiteImportNavLink {
+    text: string;
+    url: string;
+}
+/** A form that was kept: what it asks, never what was typed. */
+export interface SiteImportForm {
+    fields: {
+        type: string;
+        name: string;
+    }[];
+}
+export interface SiteImportExtract {
+    title: string;
+    description: string;
+    lang: string | null;
+    canonical: string | null;
+    blocks: SiteImportBlock[];
+    images: SiteImportImage[];
+    navLinks: SiteImportNavLink[];
+    /** Same-site page addresses, fragment-free, in first-seen order. */
+    links: string[];
+    forms: {
+        kept: SiteImportForm[];
+        removed: number;
+    };
+    /** True when the page has a password field; nothing else is extracted. */
+    loginForm: boolean;
+}
+/** Ceilings that keep one odd page from becoming the whole inventory. */
+export declare const SITE_IMPORT_PAGE_LIMITS: {
+    readonly blocks: 400;
+    readonly blockChars: 2000;
+    readonly images: 100;
+    readonly links: 300;
+    readonly navLinks: 60;
+    readonly forms: 10;
+    readonly fieldsPerForm: 30;
+};
+/** Character references to characters; an unknown name stays as written. */
+export declare function decodeHtmlEntities(text: string): string;
+/** A tag's attributes, names lowercased, values decoded. */
+export declare function parseAttributes(source: string): Record<string, string>;
+/** The `content` of every `<meta name="…">` with this name. */
+export declare function readMetaValues(html: string, name: string): string[];
+/** `raw` against the page, as an absolute http(s) address without a fragment. */
+export declare function resolveSiteUrl(raw: string | undefined, base: string): URL | null;
+/**
+ * The words, structure, pictures and links of one page.
+ *
+ * `isSameSite(url)` says whether an address is on the site being imported;
+ * only those become `links`, and images are marked by it.
+ */
+export declare function extractSiteImportPage(html: string, pageUrl: string, isSameSite: (url: URL) => boolean): SiteImportExtract;