@aglyn/plugins-ai 1.0.0-beta.271 → 1.0.0-beta.273

package/src/lib/site-import/site-import-images.js

@@ -0,0 +1,204 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ /*==========================================
+ * AN IMPORTED PAGE'S PHOTOS, INTO THE SITE'S OWN MEDIA LIBRARY (AGL-3976).
+ *
+ * Every picture goes through the media library's server door
+ * (`pluginMediaIngest().ingest`), which checks the member, the site's
+ * lockdown, the bytes' structure, the takedown list and the storage band
+ * exactly as an upload does. This module only fetches and names:
+ *
+ * - **Never a second copy.** A picture's SOURCE KEY is `import:` and a hash
+ *   of its address with any fragment and tracking parameter removed. The
+ *   library is asked for that key first (`findImportedPhoto`); a logo every
+ *   page shows is stored once and placed everywhere. An ingest that cannot
+ *   answer the question stores nothing, rather than risk the second copy.
+ * - **The safe fetch.** Bytes are read through `fetchPublicTextDetailed`,
+ *   the SSRF-guarded fetch the crawler uses (public https only, the socket
+ *   pinned, every redirect checked again), as AglynBot, capped in size and
+ *   time, and only as JPEG, PNG, WebP or GIF: the raster types the library
+ *   takes from a server.
+ *=========================================*/ import { _ as _extends } from "@swc/helpers/_/_extends";
+import { createHash } from "node:crypto";
+import { fetchPublicTextDetailed } from "../jobs/ai-theme-brand-inputs.js";
+import { AGLYN_BOT_USER_AGENT } from "./site-import-crawl.js";
+/** The largest picture brought in. */ export const SITE_IMPORT_IMAGE_MAX_BYTES = 8 * 1024 * 1024;
+/** A picture smaller than this is a spacer or a tracking pixel, not a photo. */ export const SITE_IMPORT_IMAGE_MIN_BYTES = 1024;
+/** One picture's own clock. */ export const SITE_IMPORT_IMAGE_TIMEOUT_MS = 8000;
+const RASTER = /^image\/(?:jpeg|jpg|pjpeg|png|webp|gif)\b/i;
+const NOT_A_PHOTO = /\.(?:svg|ico|bmp|tiff?|avif|heic)$/i;
+const TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|_ga|ref)$/i;
+const EXTENSION = {
+    'image/jpeg': 'jpg',
+    'image/jpg': 'jpg',
+    'image/pjpeg': 'jpg',
+    'image/png': 'png',
+    'image/webp': 'webp',
+    'image/gif': 'gif'
+};
+/** A picture's address as it is fetched and keyed: https, no fragment or tracking; `null` for one that is not a photo. */ export function siteImportImageUrl(raw) {
+    let url;
+    try {
+        url = new URL(String(raw != null ? raw : '').trim());
+    } catch (unused) {
+        return null;
+    }
+    if (url.protocol === 'http:') url.protocol = 'https:';
+    if (url.protocol !== 'https:' || url.username || url.password) return null;
+    if (url.port && url.port !== '443') return null;
+    if (NOT_A_PHOTO.test(url.pathname)) return null;
+    url.hash = '';
+    for (const key of [
+        ...url.searchParams.keys()
+    ]){
+        if (TRACKING_PARAM.test(key)) url.searchParams.delete(key);
+    }
+    return url.toString();
+}
+/** The key a site's library holds an imported picture by. */ export function siteImportImageKey(url) {
+    return `import:${createHash('sha256').update(url).digest('hex').slice(0, 32)}`;
+}
+/** A file name for the library, from the picture's address and type. */ export function siteImportImageFileName(url, contentType) {
+    var _EXTENSION_type;
+    let base = 'photo';
+    try {
+        var _pathname_split_filter_pop;
+        const last = decodeURIComponent((_pathname_split_filter_pop = new URL(url).pathname.split('/').filter(Boolean).pop()) != null ? _pathname_split_filter_pop : '');
+        base = last.replace(/\.[a-z0-9]+$/i, '').replace(/[^a-zA-Z0-9 _-]+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '') || 'photo';
+    } catch (unused) {
+    // An address that does not decode keeps the plain name.
+    }
+    const type = contentType.split(';')[0].trim().toLowerCase();
+    return `${base.slice(0, 80)}.${(_EXTENSION_type = EXTENSION[type]) != null ? _EXTENSION_type : 'jpg'}`;
+}
+/**
+ * Brings one page's pictures into the site's library, reusing any the
+ * library already holds by key. Never throws for anything a picture does.
+ */ export async function bringInSiteImportPhotos(input, deps) {
+    var _deps_now, _deps_fetch;
+    const now = (_deps_now = deps.now) != null ? _deps_now : Date.now;
+    const fetchBytes = (_deps_fetch = deps.fetch) != null ? _deps_fetch : fetchPublicTextDetailed;
+    const assets = [];
+    let brought = 0;
+    let reused = 0;
+    let skipped = 0;
+    const ingest = deps.ingest;
+    // No library door, or one that cannot say what it already holds: nothing
+    // is stored, because a picture stored without that check could be a copy.
+    if (!(ingest == null ? void 0 : ingest.findImportedPhoto)) {
+        return {
+            brought,
+            reused,
+            skipped: input.images.length,
+            assets
+        };
+    }
+    const seen = new Set();
+    for (const image of input.images){
+        var _input_signal;
+        const url = siteImportImageUrl(image.url);
+        if (!url || seen.has(url)) {
+            if (!url) skipped += 1;
+            continue;
+        }
+        seen.add(url);
+        if (((_input_signal = input.signal) == null ? void 0 : _input_signal.aborted) || now() >= input.deadlineAt) {
+            skipped += 1;
+            continue;
+        }
+        const sourceKey = siteImportImageKey(url);
+        try {
+            const held = await ingest.findImportedPhoto({
+                hostId: input.hostId,
+                sourceKey
+            });
+            if (held) {
+                reused += 1;
+                assets.push(_extends({
+                    mediaId: held.mediaId,
+                    src: held.src
+                }, held.width ? {
+                    width: held.width
+                } : {}, held.height ? {
+                    height: held.height
+                } : {}, held.alt || image.alt ? {
+                    alt: held.alt || image.alt
+                } : {}));
+                continue;
+            }
+            const fetched = await fetchBytes(url, _extends({
+                maxBytes: SITE_IMPORT_IMAGE_MAX_BYTES,
+                accept: RASTER,
+                acceptHeader: 'image/webp,image/png,image/jpeg,image/gif;q=0.9',
+                userAgent: AGLYN_BOT_USER_AGENT,
+                timeoutMs: Math.max(1, Math.min(SITE_IMPORT_IMAGE_TIMEOUT_MS, input.deadlineAt - now())),
+                refuseOver: true,
+                binary: true
+            }, input.signal ? {
+                signal: input.signal
+            } : {}));
+            if (!fetched.ok || !fetched.data || fetched.data.byteLength < SITE_IMPORT_IMAGE_MIN_BYTES) {
+                skipped += 1;
+                continue;
+            }
+            const contentType = fetched.contentType.split(';')[0].trim().toLowerCase().replace('image/jpg', 'image/jpeg').replace('image/pjpeg', 'image/jpeg');
+            const stored = await ingest.ingest(_extends({
+                hostId: input.hostId,
+                uid: input.uid,
+                fileName: siteImportImageFileName(url, contentType),
+                contentType,
+                bytes: fetched.data
+            }, image.alt ? {
+                alt: image.alt.slice(0, 250)
+            } : {}, {
+                importedFrom: {
+                    key: sourceKey,
+                    url,
+                    domain: input.domain
+                }
+            }));
+            if (!stored.ok) {
+                skipped += 1;
+                continue;
+            }
+            brought += 1;
+            assets.push(_extends({
+                mediaId: stored.mediaId,
+                src: stored.src
+            }, stored.width ? {
+                width: stored.width
+            } : {}, stored.height ? {
+                height: stored.height
+            } : {}, image.alt ? {
+                alt: image.alt.slice(0, 250)
+            } : {}));
+        } catch (error) {
+            console.warn('[site-import] a photo could not be brought in', {
+                hostId: input.hostId,
+                error
+            });
+            skipped += 1;
+        }
+    }
+    return {
+        brought,
+        reused,
+        skipped,
+        assets
+    };
+}
+
+//# sourceMappingURL=site-import-images.js.map

package/src/lib/site-import/site-import-images.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-images.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * AN IMPORTED PAGE'S PHOTOS, INTO THE SITE'S OWN MEDIA LIBRARY (AGL-3976).\n *\n * Every picture goes through the media library's server door\n * (`pluginMediaIngest().ingest`), which checks the member, the site's\n * lockdown, the bytes' structure, the takedown list and the storage band\n * exactly as an upload does. This module only fetches and names:\n *\n * - **Never a second copy.** A picture's SOURCE KEY is `import:` and a hash\n *   of its address with any fragment and tracking parameter removed. The\n *   library is asked for that key first (`findImportedPhoto`); a logo every\n *   page shows is stored once and placed everywhere. An ingest that cannot\n *   answer the question stores nothing, rather than risk the second copy.\n * - **The safe fetch.** Bytes are read through `fetchPublicTextDetailed`,\n *   the SSRF-guarded fetch the crawler uses (public https only, the socket\n *   pinned, every redirect checked again), as AglynBot, capped in size and\n *   time, and only as JPEG, PNG, WebP or GIF: the raster types the library\n *   takes from a server.\n *=========================================*/\n\nimport type { PluginMediaIngest } from '@aglyn/aglyn/plugin-manager/plugin-media-ingest'\nimport { createHash } from 'node:crypto'\nimport {\n  fetchPublicTextDetailed,\n  type PublicFetchOptions,\n  type PublicFetchResult,\n} from '../jobs/ai-theme-brand-inputs'\nimport type { AiSiteImportPage, AiSiteImportPhoto } from '../model/ai-site-import'\nimport { AGLYN_BOT_USER_AGENT } from './site-import-crawl'\n\n/** The largest picture brought in. */\nexport const SITE_IMPORT_IMAGE_MAX_BYTES = 8 * 1024 * 1024\n\n/** A picture smaller than this is a spacer or a tracking pixel, not a photo. */\nexport const SITE_IMPORT_IMAGE_MIN_BYTES = 1_024\n\n/** One picture's own clock. */\nexport const SITE_IMPORT_IMAGE_TIMEOUT_MS = 8_000\n\nconst RASTER = /^image\\/(?:jpeg|jpg|pjpeg|png|webp|gif)\\b/i\n\nconst NOT_A_PHOTO = /\\.(?:svg|ico|bmp|tiff?|avif|heic)$/i\n\nconst TRACKING_PARAM = /^(?:utm_[a-z]+|fbclid|gclid|msclkid|_ga|ref)$/i\n\nconst EXTENSION: Record<string, string> = {\n  'image/jpeg': 'jpg',\n  'image/jpg': 'jpg',\n  'image/pjpeg': 'jpg',\n  'image/png': 'png',\n  'image/webp': 'webp',\n  'image/gif': 'gif',\n}\n\n/** A picture's address as it is fetched and keyed: https, no fragment or tracking; `null` for one that is not a photo. */\nexport function siteImportImageUrl(raw: string): string | null {\n  let url: URL\n  try {\n    url = new URL(String(raw ?? '').trim())\n  } catch {\n    return null\n  }\n  if (url.protocol === 'http:') url.protocol = 'https:'\n  if (url.protocol !== 'https:' || url.username || url.password) return null\n  if (url.port && url.port !== '443') return null\n  if (NOT_A_PHOTO.test(url.pathname)) return null\n  url.hash = ''\n  for (const key of [...url.searchParams.keys()]) {\n    if (TRACKING_PARAM.test(key)) url.searchParams.delete(key)\n  }\n  return url.toString()\n}\n\n/** The key a site's library holds an imported picture by. */\nexport function siteImportImageKey(url: string): string {\n  return `import:${createHash('sha256').update(url).digest('hex').slice(0, 32)}`\n}\n\n/** A file name for the library, from the picture's address and type. */\nexport function siteImportImageFileName(url: string, contentType: string): string {\n  let base = 'photo'\n  try {\n    const last = decodeURIComponent(new URL(url).pathname.split('/').filter(Boolean).pop() ?? '')\n    base = last.replace(/\\.[a-z0-9]+$/i, '').replace(/[^a-zA-Z0-9 _-]+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '') || 'photo'\n  } catch {\n    // An address that does not decode keeps the plain name.\n  }\n  const type = contentType.split(';')[0].trim().toLowerCase()\n  return `${base.slice(0, 80)}.${EXTENSION[type] ?? 'jpg'}`\n}\n\nexport interface SiteImportPhotosInput {\n  hostId: string\n  /** The member the photos are stored as: the job's creator. */\n  uid: string\n  domain: string\n  images: ReadonlyArray<{ url: string; alt: string }>\n  signal?: AbortSignal\n  /** When the pass must stop asking for more. */\n  deadlineAt: number\n}\n\nexport interface SiteImportPhotosDeps {\n  ingest: PluginMediaIngest | null\n  fetch?: (url: string, options: PublicFetchOptions) => Promise<PublicFetchResult>\n  now?: () => number\n}\n\n/**\n * Brings one page's pictures into the site's library, reusing any the\n * library already holds by key. Never throws for anything a picture does.\n */\nexport async function bringInSiteImportPhotos(\n  input: SiteImportPhotosInput,\n  deps: SiteImportPhotosDeps,\n): Promise<NonNullable<AiSiteImportPage['photos']>> {\n  const now = deps.now ?? Date.now\n  const fetchBytes = deps.fetch ?? fetchPublicTextDetailed\n  const assets: AiSiteImportPhoto[] = []\n  let brought = 0\n  let reused = 0\n  let skipped = 0\n  const ingest = deps.ingest\n  // No library door, or one that cannot say what it already holds: nothing\n  // is stored, because a picture stored without that check could be a copy.\n  if (!ingest?.findImportedPhoto) {\n    return { brought, reused, skipped: input.images.length, assets }\n  }\n  const seen = new Set<string>()\n  for (const image of input.images) {\n    const url = siteImportImageUrl(image.url)\n    if (!url || seen.has(url)) {\n      if (!url) skipped += 1\n      continue\n    }\n    seen.add(url)\n    if (input.signal?.aborted || now() >= input.deadlineAt) {\n      skipped += 1\n      continue\n    }\n    const sourceKey = siteImportImageKey(url)\n    try {\n      const held = await ingest.findImportedPhoto({ hostId: input.hostId, sourceKey })\n      if (held) {\n        reused += 1\n        assets.push({\n          mediaId: held.mediaId,\n          src: held.src,\n          ...(held.width ? { width: held.width } : {}),\n          ...(held.height ? { height: held.height } : {}),\n          ...(held.alt || image.alt ? { alt: held.alt || image.alt } : {}),\n        })\n        continue\n      }\n      const fetched = await fetchBytes(url, {\n        maxBytes: SITE_IMPORT_IMAGE_MAX_BYTES,\n        accept: RASTER,\n        acceptHeader: 'image/webp,image/png,image/jpeg,image/gif;q=0.9',\n        userAgent: AGLYN_BOT_USER_AGENT,\n        timeoutMs: Math.max(1, Math.min(SITE_IMPORT_IMAGE_TIMEOUT_MS, input.deadlineAt - now())),\n        refuseOver: true,\n        binary: true,\n        ...(input.signal ? { signal: input.signal } : {}),\n      })\n      if (!fetched.ok || !fetched.data || fetched.data.byteLength < SITE_IMPORT_IMAGE_MIN_BYTES) {\n        skipped += 1\n        continue\n      }\n      const contentType = fetched.contentType.split(';')[0].trim().toLowerCase().replace('image/jpg', 'image/jpeg').replace('image/pjpeg', 'image/jpeg')\n      const stored = await ingest.ingest({\n        hostId: input.hostId,\n        uid: input.uid,\n        fileName: siteImportImageFileName(url, contentType),\n        contentType,\n        bytes: fetched.data,\n        ...(image.alt ? { alt: image.alt.slice(0, 250) } : {}),\n        importedFrom: { key: sourceKey, url, domain: input.domain },\n      })\n      if (!stored.ok) {\n        skipped += 1\n        continue\n      }\n      brought += 1\n      assets.push({\n        mediaId: stored.mediaId,\n        src: stored.src,\n        ...(stored.width ? { width: stored.width } : {}),\n        ...(stored.height ? { height: stored.height } : {}),\n        ...(image.alt ? { alt: image.alt.slice(0, 250) } : {}),\n      })\n    } catch (error) {\n      console.warn('[site-import] a photo could not be brought in', { hostId: input.hostId, error })\n      skipped += 1\n    }\n  }\n  return { brought, reused, skipped, assets }\n}\n"],"names":["createHash","fetchPublicTextDetailed","AGLYN_BOT_USER_AGENT","SITE_IMPORT_IMAGE_MAX_BYTES","SITE_IMPORT_IMAGE_MIN_BYTES","SITE_IMPORT_IMAGE_TIMEOUT_MS","RASTER","NOT_A_PHOTO","TRACKING_PARAM","EXTENSION","siteImportImageUrl","raw","url","URL","String","trim","protocol","username","password","port","test","pathname","hash","key","searchParams","keys","delete","toString","siteImportImageKey","update","digest","slice","siteImportImageFileName","contentType","base","last","decodeURIComponent","split","filter","Boolean","pop","replace","type","toLowerCase","bringInSiteImportPhotos","input","deps","now","Date","fetchBytes","fetch","assets","brought","reused","skipped","ingest","findImportedPhoto","images","length","seen","Set","image","has","add","signal","aborted","deadlineAt","sourceKey","held","hostId","push","mediaId","src","width","height","alt","fetched","maxBytes","accept","acceptHeader","userAgent","timeoutMs","Math","max","min","refuseOver","binary","ok","data","byteLength","stored","uid","fileName","bytes","importedFrom","domain","error","console","warn"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;2CAkB2C;AAG3C,SAASA,UAAU,QAAQ,cAAa;AACxC,SACEC,uBAAuB,QAGlB,mCAA+B;AAEtC,SAASC,oBAAoB,QAAQ,yBAAqB;AAE1D,oCAAoC,GACpC,OAAO,MAAMC,8BAA8B,IAAI,OAAO,KAAI;AAE1D,8EAA8E,GAC9E,OAAO,MAAMC,8BAA8B,KAAK;AAEhD,6BAA6B,GAC7B,OAAO,MAAMC,+BAA+B,KAAK;AAEjD,MAAMC,SAAS;AAEf,MAAMC,cAAc;AAEpB,MAAMC,iBAAiB;AAEvB,MAAMC,YAAoC;IACxC,cAAc;IACd,aAAa;IACb,eAAe;IACf,aAAa;IACb,cAAc;IACd,aAAa;AACf;AAEA,wHAAwH,GACxH,OAAO,SAASC,mBAAmBC,GAAW;IAC5C,IAAIC;IACJ,IAAI;QACFA,MAAM,IAAIC,IAAIC,OAAOH,cAAAA,MAAO,IAAII,IAAI;IACtC,EAAE,eAAM;QACN,OAAO;IACT;IACA,IAAIH,IAAII,QAAQ,KAAK,SAASJ,IAAII,QAAQ,GAAG;IAC7C,IAAIJ,IAAII,QAAQ,KAAK,YAAYJ,IAAIK,QAAQ,IAAIL,IAAIM,QAAQ,EAAE,OAAO;IACtE,IAAIN,IAAIO,IAAI,IAAIP,IAAIO,IAAI,KAAK,OAAO,OAAO;IAC3C,IAAIZ,YAAYa,IAAI,CAACR,IAAIS,QAAQ,GAAG,OAAO;IAC3CT,IAAIU,IAAI,GAAG;IACX,KAAK,MAAMC,OAAO;WAAIX,IAAIY,YAAY,CAACC,IAAI;KAAG,CAAE;QAC9C,IAAIjB,eAAeY,IAAI,CAACG,MAAMX,IAAIY,YAAY,CAACE,MAAM,CAACH;IACxD;IACA,OAAOX,IAAIe,QAAQ;AACrB;AAEA,2DAA2D,GAC3D,OAAO,SAASC,mBAAmBhB,GAAW;IAC5C,OAAO,CAAC,OAAO,EAAEZ,WAAW,UAAU6B,MAAM,CAACjB,KAAKkB,MAAM,CAAC,OAAOC,KAAK,CAAC,GAAG,KAAK;AAChF;AAEA,sEAAsE,GACtE,OAAO,SAASC,wBAAwBpB,GAAW,EAAEqB,WAAmB;QASvCxB;IAR/B,IAAIyB,OAAO;IACX,IAAI;YAC8B;QAAhC,MAAMC,OAAOC,oBAAmB,6BAAA,IAAIvB,IAAID,KAAKS,QAAQ,CAACgB,KAAK,CAAC,KAAKC,MAAM,CAACC,SAASC,GAAG,cAApD,6BAA0D;QAC1FN,OAAOC,KAAKM,OAAO,CAAC,iBAAiB,IAAIA,OAAO,CAAC,qBAAqB,KAAKA,OAAO,CAAC,OAAO,KAAKA,OAAO,CAAC,UAAU,OAAO;IAC1H,EAAE,eAAM;IACN,wDAAwD;IAC1D;IACA,MAAMC,OAAOT,YAAYI,KAAK,CAAC,IAAI,CAAC,EAAE,CAACtB,IAAI,GAAG4B,WAAW;IACzD,OAAO,GAAGT,KAAKH,KAAK,CAAC,GAAG,IAAI,CAAC,GAAEtB,kBAAAA,SAAS,CAACiC,KAAK,YAAfjC,kBAAmB,OAAO;AAC3D;AAmBA;;;CAGC,GACD,OAAO,eAAemC,wBACpBC,KAA4B,EAC5BC,IAA0B;QAEdA,WACOA;IADnB,MAAMC,OAAMD,YAAAA,KAAKC,GAAG,YAARD,YAAYE,KAAKD,GAAG;IAChC,MAAME,cAAaH,cAAAA,KAAKI,KAAK,YAAVJ,cAAc7C;IACjC,MAAMkD,SAA8B,EAAE;IACtC,IAAIC,UAAU;IACd,IAAIC,SAAS;IACb,IAAIC,UAAU;IACd,MAAMC,SAAST,KAAKS,MAAM;IAC1B,yEAAyE;IACzE,0EAA0E;IAC1E,IAAI,EAACA,0BAAAA,OAAQC,iBAAiB,GAAE;QAC9B,OAAO;YAAEJ;YAASC;YAAQC,SAAST,MAAMY,MAAM,CAACC,MAAM;YAAEP;QAAO;IACjE;IACA,MAAMQ,OAAO,IAAIC;IACjB,KAAK,MAAMC,SAAShB,MAAMY,MAAM,CAAE;YAO5BZ;QANJ,MAAMjC,MAAMF,mBAAmBmD,MAAMjD,GAAG;QACxC,IAAI,CAACA,OAAO+C,KAAKG,GAAG,CAAClD,MAAM;YACzB,IAAI,CAACA,KAAK0C,WAAW;YACrB;QACF;QACAK,KAAKI,GAAG,CAACnD;QACT,IAAIiC,EAAAA,gBAAAA,MAAMmB,MAAM,qBAAZnB,cAAcoB,OAAO,KAAIlB,SAASF,MAAMqB,UAAU,EAAE;YACtDZ,WAAW;YACX;QACF;QACA,MAAMa,YAAYvC,mBAAmBhB;QACrC,IAAI;YACF,MAAMwD,OAAO,MAAMb,OAAOC,iBAAiB,CAAC;gBAAEa,QAAQxB,MAAMwB,MAAM;gBAAEF;YAAU;YAC9E,IAAIC,MAAM;gBACRf,UAAU;gBACVF,OAAOmB,IAAI,CAAC;oBACVC,SAASH,KAAKG,OAAO;oBACrBC,KAAKJ,KAAKI,GAAG;mBACTJ,KAAKK,KAAK,GAAG;oBAAEA,OAAOL,KAAKK,KAAK;gBAAC,IAAI,CAAC,GACtCL,KAAKM,MAAM,GAAG;oBAAEA,QAAQN,KAAKM,MAAM;gBAAC,IAAI,CAAC,GACzCN,KAAKO,GAAG,IAAId,MAAMc,GAAG,GAAG;oBAAEA,KAAKP,KAAKO,GAAG,IAAId,MAAMc,GAAG;gBAAC,IAAI,CAAC;gBAEhE;YACF;YACA,MAAMC,UAAU,MAAM3B,WAAWrC,KAAK;gBACpCiE,UAAU1E;gBACV2E,QAAQxE;gBACRyE,cAAc;gBACdC,WAAW9E;gBACX+E,WAAWC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,GAAG,CAAC/E,8BAA8BwC,MAAMqB,UAAU,GAAGnB;gBACjFsC,YAAY;gBACZC,QAAQ;eACJzC,MAAMmB,MAAM,GAAG;gBAAEA,QAAQnB,MAAMmB,MAAM;YAAC,IAAI,CAAC;YAEjD,IAAI,CAACY,QAAQW,EAAE,IAAI,CAACX,QAAQY,IAAI,IAAIZ,QAAQY,IAAI,CAACC,UAAU,GAAGrF,6BAA6B;gBACzFkD,WAAW;gBACX;YACF;YACA,MAAMrB,cAAc2C,QAAQ3C,WAAW,CAACI,KAAK,CAAC,IAAI,CAAC,EAAE,CAACtB,IAAI,GAAG4B,WAAW,GAAGF,OAAO,CAAC,aAAa,cAAcA,OAAO,CAAC,eAAe;YACrI,MAAMiD,SAAS,MAAMnC,OAAOA,MAAM,CAAC;gBACjCc,QAAQxB,MAAMwB,MAAM;gBACpBsB,KAAK9C,MAAM8C,GAAG;gBACdC,UAAU5D,wBAAwBpB,KAAKqB;gBACvCA;gBACA4D,OAAOjB,QAAQY,IAAI;eACf3B,MAAMc,GAAG,GAAG;gBAAEA,KAAKd,MAAMc,GAAG,CAAC5C,KAAK,CAAC,GAAG;YAAK,IAAI,CAAC;gBACpD+D,cAAc;oBAAEvE,KAAK4C;oBAAWvD;oBAAKmF,QAAQlD,MAAMkD,MAAM;gBAAC;;YAE5D,IAAI,CAACL,OAAOH,EAAE,EAAE;gBACdjC,WAAW;gBACX;YACF;YACAF,WAAW;YACXD,OAAOmB,IAAI,CAAC;gBACVC,SAASmB,OAAOnB,OAAO;gBACvBC,KAAKkB,OAAOlB,GAAG;eACXkB,OAAOjB,KAAK,GAAG;gBAAEA,OAAOiB,OAAOjB,KAAK;YAAC,IAAI,CAAC,GAC1CiB,OAAOhB,MAAM,GAAG;gBAAEA,QAAQgB,OAAOhB,MAAM;YAAC,IAAI,CAAC,GAC7Cb,MAAMc,GAAG,GAAG;gBAAEA,KAAKd,MAAMc,GAAG,CAAC5C,KAAK,CAAC,GAAG;YAAK,IAAI,CAAC;QAExD,EAAE,OAAOiE,OAAO;YACdC,QAAQC,IAAI,CAAC,iDAAiD;gBAAE7B,QAAQxB,MAAMwB,MAAM;gBAAE2B;YAAM;YAC5F1C,WAAW;QACb;IACF;IACA,OAAO;QAAEF;QAASC;QAAQC;QAASH;IAAO;AAC5C"}

package/src/lib/site-import/site-import-page.d.ts

@@ -0,0 +1,48 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { type AiSiteImportLeftOutReason, type AiSiteImportListedPage, type AiSiteImportPageRead } from '../model/ai-site-import';
+import type { SiteImportPage, SiteImportSkipReason } from './site-import-crawl';
+/** A line of untrusted text safe to show outside a fence: neutralized, one line, cut short. */
+export declare function siteImportLabel(text: string, max?: number): string;
+/** How many words a page says, its title and headings included. */
+export declare function siteImportPageWords(page: Pick<SiteImportPage, 'title' | 'blocks'>): number;
+/**
+ * A page's name on the new site: "Home" for the home page, else its browser
+ * title without the old site's name after a separator ("About | Rosie's
+ * Bakery" is "About"), else its first heading.
+ */
+export declare function siteImportPageTitle(page: Pick<SiteImportPage, 'path' | 'title' | 'blocks'>): string;
+/** The page as the inventory lists it. */
+export declare function siteImportListedPage(page: SiteImportPage): AiSiteImportListedPage;
+/** The page's photos, the ones in its body first, each once. */
+export declare function siteImportPageImages(page: Pick<SiteImportPage, 'blocks' | 'images'>): Array<{
+    url: string;
+    alt: string;
+}>;
+/**
+ * The page's sections, from its headings: the first is its opening (its
+ * title or first heading), then one for each second-level heading of its
+ * body, at most `AI_SITE_IMPORT_MAX_SECTIONS`. A page with no headings below
+ * its title but more to say than an opening gets a second section for it.
+ */
+export declare function siteImportPageSections(page: Pick<SiteImportPage, 'title' | 'blocks'>): string[];
+/** The page's text in order, as a prompt shows it inside its fence. */
+export declare function siteImportPageText(page: Pick<SiteImportPage, 'title' | 'description' | 'blocks'>): string;
+/** The page as a wave reads it. */
+export declare function siteImportPageRead(page: SiteImportPage, at: string): AiSiteImportPageRead;
+/** Why the crawler could not read a page, as an import leaves it out. */
+export declare function siteImportLeftOutReason(reason: SiteImportSkipReason): AiSiteImportLeftOutReason;

package/src/lib/site-import/site-import-page.js

@@ -0,0 +1,150 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ /*==========================================
+ * ONE CRAWLED PAGE, AS AN IMPORT KEEPS IT (AGL-3976).
+ *
+ * The crawler's page (`SiteImportPage`) holds everything it extracted; an
+ * import keeps much less, and keeps it in two shapes:
+ *
+ * - **Listed**, for the inventory: its address, title, how it was found and
+ *   how many words and photos it has. No text: an inventory of a whole site
+ *   rides the job document.
+ * - **Read**, for a wave: its text in order, its sections from its headings,
+ *   and its photos' addresses. The text is kept as the page says it and is
+ *   fenced as untrusted data only where a prompt shows it; the section names
+ *   and the title travel outside the fence, so they are neutralized (no
+ *   fence marker, no hidden character) and cut short.
+ *=========================================*/ import { neutralizeUntrustedText } from "@aglyn/shared-util-tools/untrusted-text";
+import { AI_SITE_IMPORT_IMAGES_PER_PAGE, AI_SITE_IMPORT_LABEL_MAX_CHARS, AI_SITE_IMPORT_MAX_SECTIONS, AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS } from "../model/ai-site-import.js";
+/** A line of untrusted text safe to show outside a fence: neutralized, one line, cut short. */ export function siteImportLabel(text, max = AI_SITE_IMPORT_LABEL_MAX_CHARS) {
+    const line = neutralizeUntrustedText(text).replace(/\s+/g, ' ').trim();
+    return line.length > max ? `${line.slice(0, max - 1).trimEnd()}…` : line;
+}
+const wordsIn = (text)=>{
+    var _text_match;
+    return ((_text_match = text.match(RegExp("[\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376-\\u0377\\u037A-\\u037D\\u037F\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u052F\\u0531-\\u0556\\u0559\\u0560-\\u0588\\u05D0-\\u05EA\\u05EF-\\u05F2\\u0620-\\u064A\\u066E-\\u066F\\u0671-\\u06D3\\u06D5\\u06E5-\\u06E6\\u06EE-\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4-\\u07F5\\u07FA\\u0800-\\u0815\\u081A\\u0824\\u0828\\u0840-\\u0858\\u0860-\\u086A\\u0870-\\u0887\\u0889-\\u088F\\u08A0-\\u08C9\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971-\\u0980\\u0985-\\u098C\\u098F-\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC-\\u09DD\\u09DF-\\u09E1\\u09F0-\\u09F1\\u09FC\\u0A05-\\u0A0A\\u0A0F-\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32-\\u0A33\\u0A35-\\u0A36\\u0A38-\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2-\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0-\\u0AE1\\u0AF9\\u0B05-\\u0B0C\\u0B0F-\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32-\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C-\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99-\\u0B9A\\u0B9C\\u0B9E-\\u0B9F\\u0BA3-\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C39\\u0C3D\\u0C58-\\u0C5A\\u0C5C-\\u0C5D\\u0C60-\\u0C61\\u0C80\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDC-\\u0CDE\\u0CE0-\\u0CE1\\u0CF1-\\u0CF2\\u0D04-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D3A\\u0D3D\\u0D4E\\u0D54-\\u0D56\\u0D5F-\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32-\\u0E33\\u0E40-\\u0E46\\u0E81-\\u0E82\\u0E84\\u0E86-\\u0E8A\\u0E8C-\\u0EA3\\u0EA5\\u0EA7-\\u0EB0\\u0EB2-\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC-\\u0EDF\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8C\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065-\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10C7\\u10CD\\u10D0-\\u10FA\\u10FC-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F5\\u13F8-\\u13FD\\u1401-\\u166C\\u166F-\\u167F\\u1681-\\u169A\\u16A0-\\u16EA\\u16F1-\\u16F8\\u1700-\\u1711\\u171F-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1878\\u1880-\\u1884\\u1887-\\u18A8\\u18AA\\u18B0-\\u18F5\\u1900-\\u191E\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19AB\\u19B0-\\u19C9\\u1A00-\\u1A16\\u1A20-\\u1A54\\u1AA7\\u1B05-\\u1B33\\u1B45-\\u1B4C\\u1B83-\\u1BA0\\u1BAE-\\u1BAF\\u1BBA-\\u1BE5\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1C80-\\u1C8A\\u1C90-\\u1CBA\\u1CBD-\\u1CBF\\u1CE9-\\u1CEC\\u1CEE-\\u1CF3\\u1CF5-\\u1CF6\\u1CFA\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u209C\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2183-\\u2184\\u2C00-\\u2CE4\\u2CEB-\\u2CEE\\u2CF2-\\u2CF3\\u2D00-\\u2D25\\u2D27\\u2D2D\\u2D30-\\u2D67\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005-\\u3006\\u3031-\\u3035\\u303B-\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312F\\u3131-\\u318E\\u31A0-\\u31BF\\u31F0-\\u31FF\\u3400-\\u4DBF\\u4E00-\\uA48C\\uA4D0-\\uA4FD\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A-\\uA62B\\uA640-\\uA66E\\uA67F-\\uA69D\\uA6A0-\\uA6E5\\uA717-\\uA71F\\uA722-\\uA788\\uA78B-\\uA7DC\\uA7F1-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA8F2-\\uA8F7\\uA8FB\\uA8FD-\\uA8FE\\uA90A-\\uA925\\uA930-\\uA946\\uA960-\\uA97C\\uA984-\\uA9B2\\uA9CF\\uA9E0-\\uA9E4\\uA9E6-\\uA9EF\\uA9FA-\\uA9FE\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAA60-\\uAA76\\uAA7A\\uAA7E-\\uAAAF\\uAAB1\\uAAB5-\\uAAB6\\uAAB9-\\uAABD\\uAAC0\\uAAC2\\uAADB-\\uAADD\\uAAE0-\\uAAEA\\uAAF2-\\uAAF4\\uAB01-\\uAB06\\uAB09-\\uAB0E\\uAB11-\\uAB16\\uAB20-\\uAB26\\uAB28-\\uAB2E\\uAB30-\\uAB5A\\uAB5C-\\uAB69\\uAB70-\\uABE2\\uAC00-\\uD7A3\\uD7B0-\\uD7C6\\uD7CB-\\uD7FB\\uF900-\\uFA6D\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40-\\uFB41\\uFB43-\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC\\u{10000}-\\u{1000B}\\u{1000D}-\\u{10026}\\u{10028}-\\u{1003A}\\u{1003C}-\\u{1003D}\\u{1003F}-\\u{1004D}\\u{10050}-\\u{1005D}\\u{10080}-\\u{100FA}\\u{10280}-\\u{1029C}\\u{102A0}-\\u{102D0}\\u{10300}-\\u{1031F}\\u{1032D}-\\u{10340}\\u{10342}-\\u{10349}\\u{10350}-\\u{10375}\\u{10380}-\\u{1039D}\\u{103A0}-\\u{103C3}\\u{103C8}-\\u{103CF}\\u{10400}-\\u{1049D}\\u{104B0}-\\u{104D3}\\u{104D8}-\\u{104FB}\\u{10500}-\\u{10527}\\u{10530}-\\u{10563}\\u{10570}-\\u{1057A}\\u{1057C}-\\u{1058A}\\u{1058C}-\\u{10592}\\u{10594}-\\u{10595}\\u{10597}-\\u{105A1}\\u{105A3}-\\u{105B1}\\u{105B3}-\\u{105B9}\\u{105BB}-\\u{105BC}\\u{105C0}-\\u{105F3}\\u{10600}-\\u{10736}\\u{10740}-\\u{10755}\\u{10760}-\\u{10767}\\u{10780}-\\u{10785}\\u{10787}-\\u{107B0}\\u{107B2}-\\u{107BA}\\u{10800}-\\u{10805}\\u{10808}\\u{1080A}-\\u{10835}\\u{10837}-\\u{10838}\\u{1083C}\\u{1083F}-\\u{10855}\\u{10860}-\\u{10876}\\u{10880}-\\u{1089E}\\u{108E0}-\\u{108F2}\\u{108F4}-\\u{108F5}\\u{10900}-\\u{10915}\\u{10920}-\\u{10939}\\u{10940}-\\u{10959}\\u{10980}-\\u{109B7}\\u{109BE}-\\u{109BF}\\u{10A00}\\u{10A10}-\\u{10A13}\\u{10A15}-\\u{10A17}\\u{10A19}-\\u{10A35}\\u{10A60}-\\u{10A7C}\\u{10A80}-\\u{10A9C}\\u{10AC0}-\\u{10AC7}\\u{10AC9}-\\u{10AE4}\\u{10B00}-\\u{10B35}\\u{10B40}-\\u{10B55}\\u{10B60}-\\u{10B72}\\u{10B80}-\\u{10B91}\\u{10C00}-\\u{10C48}\\u{10C80}-\\u{10CB2}\\u{10CC0}-\\u{10CF2}\\u{10D00}-\\u{10D23}\\u{10D4A}-\\u{10D65}\\u{10D6F}-\\u{10D85}\\u{10E80}-\\u{10EA9}\\u{10EB0}-\\u{10EB1}\\u{10EC2}-\\u{10EC7}\\u{10F00}-\\u{10F1C}\\u{10F27}\\u{10F30}-\\u{10F45}\\u{10F70}-\\u{10F81}\\u{10FB0}-\\u{10FC4}\\u{10FE0}-\\u{10FF6}\\u{11003}-\\u{11037}\\u{11071}-\\u{11072}\\u{11075}\\u{11083}-\\u{110AF}\\u{110D0}-\\u{110E8}\\u{11103}-\\u{11126}\\u{11144}\\u{11147}\\u{11150}-\\u{11172}\\u{11176}\\u{11183}-\\u{111B2}\\u{111C1}-\\u{111C4}\\u{111DA}\\u{111DC}\\u{11200}-\\u{11211}\\u{11213}-\\u{1122B}\\u{1123F}-\\u{11240}\\u{11280}-\\u{11286}\\u{11288}\\u{1128A}-\\u{1128D}\\u{1128F}-\\u{1129D}\\u{1129F}-\\u{112A8}\\u{112B0}-\\u{112DE}\\u{11305}-\\u{1130C}\\u{1130F}-\\u{11310}\\u{11313}-\\u{11328}\\u{1132A}-\\u{11330}\\u{11332}-\\u{11333}\\u{11335}-\\u{11339}\\u{1133D}\\u{11350}\\u{1135D}-\\u{11361}\\u{11380}-\\u{11389}\\u{1138B}\\u{1138E}\\u{11390}-\\u{113B5}\\u{113B7}\\u{113D1}\\u{113D3}\\u{11400}-\\u{11434}\\u{11447}-\\u{1144A}\\u{1145F}-\\u{11461}\\u{11480}-\\u{114AF}\\u{114C4}-\\u{114C5}\\u{114C7}\\u{11580}-\\u{115AE}\\u{115D8}-\\u{115DB}\\u{11600}-\\u{1162F}\\u{11644}\\u{11680}-\\u{116AA}\\u{116B8}\\u{11700}-\\u{1171A}\\u{11740}-\\u{11746}\\u{11800}-\\u{1182B}\\u{118A0}-\\u{118DF}\\u{118FF}-\\u{11906}\\u{11909}\\u{1190C}-\\u{11913}\\u{11915}-\\u{11916}\\u{11918}-\\u{1192F}\\u{1193F}\\u{11941}\\u{119A0}-\\u{119A7}\\u{119AA}-\\u{119D0}\\u{119E1}\\u{119E3}\\u{11A00}\\u{11A0B}-\\u{11A32}\\u{11A3A}\\u{11A50}\\u{11A5C}-\\u{11A89}\\u{11A9D}\\u{11AB0}-\\u{11AF8}\\u{11BC0}-\\u{11BE0}\\u{11C00}-\\u{11C08}\\u{11C0A}-\\u{11C2E}\\u{11C40}\\u{11C72}-\\u{11C8F}\\u{11D00}-\\u{11D06}\\u{11D08}-\\u{11D09}\\u{11D0B}-\\u{11D30}\\u{11D46}\\u{11D60}-\\u{11D65}\\u{11D67}-\\u{11D68}\\u{11D6A}-\\u{11D89}\\u{11D98}\\u{11DB0}-\\u{11DDB}\\u{11EE0}-\\u{11EF2}\\u{11F02}\\u{11F04}-\\u{11F10}\\u{11F12}-\\u{11F33}\\u{11FB0}\\u{12000}-\\u{12399}\\u{12480}-\\u{12543}\\u{12F90}-\\u{12FF0}\\u{13000}-\\u{1342F}\\u{13441}-\\u{13446}\\u{13460}-\\u{143FA}\\u{14400}-\\u{14646}\\u{16100}-\\u{1611D}\\u{16800}-\\u{16A38}\\u{16A40}-\\u{16A5E}\\u{16A70}-\\u{16ABE}\\u{16AD0}-\\u{16AED}\\u{16B00}-\\u{16B2F}\\u{16B40}-\\u{16B43}\\u{16B63}-\\u{16B77}\\u{16B7D}-\\u{16B8F}\\u{16D40}-\\u{16D6C}\\u{16E40}-\\u{16E7F}\\u{16EA0}-\\u{16EB8}\\u{16EBB}-\\u{16ED3}\\u{16F00}-\\u{16F4A}\\u{16F50}\\u{16F93}-\\u{16F9F}\\u{16FE0}-\\u{16FE1}\\u{16FE3}\\u{16FF2}-\\u{16FF3}\\u{17000}-\\u{18CD5}\\u{18CFF}-\\u{18D1E}\\u{18D80}-\\u{18DF2}\\u{1AFF0}-\\u{1AFF3}\\u{1AFF5}-\\u{1AFFB}\\u{1AFFD}-\\u{1AFFE}\\u{1B000}-\\u{1B122}\\u{1B132}\\u{1B150}-\\u{1B152}\\u{1B155}\\u{1B164}-\\u{1B167}\\u{1B170}-\\u{1B2FB}\\u{1BC00}-\\u{1BC6A}\\u{1BC70}-\\u{1BC7C}\\u{1BC80}-\\u{1BC88}\\u{1BC90}-\\u{1BC99}\\u{1D400}-\\u{1D454}\\u{1D456}-\\u{1D49C}\\u{1D49E}-\\u{1D49F}\\u{1D4A2}\\u{1D4A5}-\\u{1D4A6}\\u{1D4A9}-\\u{1D4AC}\\u{1D4AE}-\\u{1D4B9}\\u{1D4BB}\\u{1D4BD}-\\u{1D4C3}\\u{1D4C5}-\\u{1D505}\\u{1D507}-\\u{1D50A}\\u{1D50D}-\\u{1D514}\\u{1D516}-\\u{1D51C}\\u{1D51E}-\\u{1D539}\\u{1D53B}-\\u{1D53E}\\u{1D540}-\\u{1D544}\\u{1D546}\\u{1D54A}-\\u{1D550}\\u{1D552}-\\u{1D6A5}\\u{1D6A8}-\\u{1D6C0}\\u{1D6C2}-\\u{1D6DA}\\u{1D6DC}-\\u{1D6FA}\\u{1D6FC}-\\u{1D714}\\u{1D716}-\\u{1D734}\\u{1D736}-\\u{1D74E}\\u{1D750}-\\u{1D76E}\\u{1D770}-\\u{1D788}\\u{1D78A}-\\u{1D7A8}\\u{1D7AA}-\\u{1D7C2}\\u{1D7C4}-\\u{1D7CB}\\u{1DF00}-\\u{1DF1E}\\u{1DF25}-\\u{1DF2A}\\u{1E030}-\\u{1E06D}\\u{1E100}-\\u{1E12C}\\u{1E137}-\\u{1E13D}\\u{1E14E}\\u{1E290}-\\u{1E2AD}\\u{1E2C0}-\\u{1E2EB}\\u{1E4D0}-\\u{1E4EB}\\u{1E5D0}-\\u{1E5ED}\\u{1E5F0}\\u{1E6C0}-\\u{1E6DE}\\u{1E6E0}-\\u{1E6E2}\\u{1E6E4}-\\u{1E6E5}\\u{1E6E7}-\\u{1E6ED}\\u{1E6F0}-\\u{1E6F4}\\u{1E6FE}-\\u{1E6FF}\\u{1E7E0}-\\u{1E7E6}\\u{1E7E8}-\\u{1E7EB}\\u{1E7ED}-\\u{1E7EE}\\u{1E7F0}-\\u{1E7FE}\\u{1E800}-\\u{1E8C4}\\u{1E900}-\\u{1E943}\\u{1E94B}\\u{1EE00}-\\u{1EE03}\\u{1EE05}-\\u{1EE1F}\\u{1EE21}-\\u{1EE22}\\u{1EE24}\\u{1EE27}\\u{1EE29}-\\u{1EE32}\\u{1EE34}-\\u{1EE37}\\u{1EE39}\\u{1EE3B}\\u{1EE42}\\u{1EE47}\\u{1EE49}\\u{1EE4B}\\u{1EE4D}-\\u{1EE4F}\\u{1EE51}-\\u{1EE52}\\u{1EE54}\\u{1EE57}\\u{1EE59}\\u{1EE5B}\\u{1EE5D}\\u{1EE5F}\\u{1EE61}-\\u{1EE62}\\u{1EE64}\\u{1EE67}-\\u{1EE6A}\\u{1EE6C}-\\u{1EE72}\\u{1EE74}-\\u{1EE77}\\u{1EE79}-\\u{1EE7C}\\u{1EE7E}\\u{1EE80}-\\u{1EE89}\\u{1EE8B}-\\u{1EE9B}\\u{1EEA1}-\\u{1EEA3}\\u{1EEA5}-\\u{1EEA9}\\u{1EEAB}-\\u{1EEBB}\\u{20000}-\\u{2A6DF}\\u{2A700}-\\u{2B81D}\\u{2B820}-\\u{2CEAD}\\u{2CEB0}-\\u{2EBE0}\\u{2EBF0}-\\u{2EE5D}\\u{2F800}-\\u{2FA1D}\\u{30000}-\\u{3134A}\\u{31350}-\\u{33479}\\u0030-\\u0039\\u00B2-\\u00B3\\u00B9\\u00BC-\\u00BE\\u0660-\\u0669\\u06F0-\\u06F9\\u07C0-\\u07C9\\u0966-\\u096F\\u09E6-\\u09EF\\u09F4-\\u09F9\\u0A66-\\u0A6F\\u0AE6-\\u0AEF\\u0B66-\\u0B6F\\u0B72-\\u0B77\\u0BE6-\\u0BF2\\u0C66-\\u0C6F\\u0C78-\\u0C7E\\u0CE6-\\u0CEF\\u0D58-\\u0D5E\\u0D66-\\u0D78\\u0DE6-\\u0DEF\\u0E50-\\u0E59\\u0ED0-\\u0ED9\\u0F20-\\u0F33\\u1040-\\u1049\\u1090-\\u1099\\u1369-\\u137C\\u16EE-\\u16F0\\u17E0-\\u17E9\\u17F0-\\u17F9\\u1810-\\u1819\\u1946-\\u194F\\u19D0-\\u19DA\\u1A80-\\u1A89\\u1A90-\\u1A99\\u1B50-\\u1B59\\u1BB0-\\u1BB9\\u1C40-\\u1C49\\u1C50-\\u1C59\\u2070\\u2074-\\u2079\\u2080-\\u2089\\u2150-\\u2182\\u2185-\\u2189\\u2460-\\u249B\\u24EA-\\u24FF\\u2776-\\u2793\\u2CFD\\u3007\\u3021-\\u3029\\u3038-\\u303A\\u3192-\\u3195\\u3220-\\u3229\\u3248-\\u324F\\u3251-\\u325F\\u3280-\\u3289\\u32B1-\\u32BF\\uA620-\\uA629\\uA6E6-\\uA6EF\\uA830-\\uA835\\uA8D0-\\uA8D9\\uA900-\\uA909\\uA9D0-\\uA9D9\\uA9F0-\\uA9F9\\uAA50-\\uAA59\\uABF0-\\uABF9\\uFF10-\\uFF19\\u{10107}-\\u{10133}\\u{10140}-\\u{10178}\\u{1018A}-\\u{1018B}\\u{102E1}-\\u{102FB}\\u{10320}-\\u{10323}\\u{10341}\\u{1034A}\\u{103D1}-\\u{103D5}\\u{104A0}-\\u{104A9}\\u{10858}-\\u{1085F}\\u{10879}-\\u{1087F}\\u{108A7}-\\u{108AF}\\u{108FB}-\\u{108FF}\\u{10916}-\\u{1091B}\\u{109BC}-\\u{109BD}\\u{109C0}-\\u{109CF}\\u{109D2}-\\u{109FF}\\u{10A40}-\\u{10A48}\\u{10A7D}-\\u{10A7E}\\u{10A9D}-\\u{10A9F}\\u{10AEB}-\\u{10AEF}\\u{10B58}-\\u{10B5F}\\u{10B78}-\\u{10B7F}\\u{10BA9}-\\u{10BAF}\\u{10CFA}-\\u{10CFF}\\u{10D30}-\\u{10D39}\\u{10D40}-\\u{10D49}\\u{10E60}-\\u{10E7E}\\u{10F1D}-\\u{10F26}\\u{10F51}-\\u{10F54}\\u{10FC5}-\\u{10FCB}\\u{11052}-\\u{1106F}\\u{110F0}-\\u{110F9}\\u{11136}-\\u{1113F}\\u{111D0}-\\u{111D9}\\u{111E1}-\\u{111F4}\\u{112F0}-\\u{112F9}\\u{11450}-\\u{11459}\\u{114D0}-\\u{114D9}\\u{11650}-\\u{11659}\\u{116C0}-\\u{116C9}\\u{116D0}-\\u{116E3}\\u{11730}-\\u{1173B}\\u{118E0}-\\u{118F2}\\u{11950}-\\u{11959}\\u{11BF0}-\\u{11BF9}\\u{11C50}-\\u{11C6C}\\u{11D50}-\\u{11D59}\\u{11DA0}-\\u{11DA9}\\u{11DE0}-\\u{11DE9}\\u{11F50}-\\u{11F59}\\u{11FC0}-\\u{11FD4}\\u{12400}-\\u{1246E}\\u{16130}-\\u{16139}\\u{16A60}-\\u{16A69}\\u{16AC0}-\\u{16AC9}\\u{16B50}-\\u{16B59}\\u{16B5B}-\\u{16B61}\\u{16D70}-\\u{16D79}\\u{16E80}-\\u{16E96}\\u{16FF4}-\\u{16FF6}\\u{1CCF0}-\\u{1CCF9}\\u{1D2C0}-\\u{1D2D3}\\u{1D2E0}-\\u{1D2F3}\\u{1D360}-\\u{1D378}\\u{1D7CE}-\\u{1D7FF}\\u{1E140}-\\u{1E149}\\u{1E2F0}-\\u{1E2F9}\\u{1E4F0}-\\u{1E4F9}\\u{1E5F1}-\\u{1E5FA}\\u{1E8C7}-\\u{1E8CF}\\u{1E950}-\\u{1E959}\\u{1EC71}-\\u{1ECAB}\\u{1ECAD}-\\u{1ECAF}\\u{1ECB1}-\\u{1ECB4}\\u{1ED01}-\\u{1ED2D}\\u{1ED2F}-\\u{1ED3D}\\u{1F100}-\\u{1F10C}\\u{1FBF0}-\\u{1FBF9}]+", "gu"))) != null ? _text_match : []).length;
+};
+/** How many words a page says, its title and headings included. */ export function siteImportPageWords(page) {
+    return page.blocks.reduce((total, block)=>total + (block.kind === 'image' ? 0 : wordsIn(block.text)), 0);
+}
+/** Between a page's own name and its site's in a browser title: "About | Rosie's Bakery". */ const TITLE_SEPARATOR = /\s+[|\u2013\u2014\u00b7:-]\s+/;
+/**
+ * A page's name on the new site: "Home" for the home page, else its browser
+ * title without the old site's name after a separator ("About | Rosie's
+ * Bakery" is "About"), else its first heading.
+ */ export function siteImportPageTitle(page) {
+    var _split_, _page_title, _ref;
+    var _page_blocks_find;
+    if ((page.path.replace(/\/+$/, '') || '/') === '/') return 'Home';
+    const own = siteImportLabel((_split_ = ((_page_title = page.title) != null ? _page_title : '').split(TITLE_SEPARATOR)[0]) != null ? _split_ : '');
+    if (own.length >= 2) return own;
+    return siteImportLabel((_ref = (_page_blocks_find = page.blocks.find((block)=>block.kind === 'heading')) == null ? void 0 : _page_blocks_find.text) != null ? _ref : '');
+}
+/** The page as the inventory lists it. */ export function siteImportListedPage(page) {
+    return {
+        url: page.url,
+        path: page.path,
+        title: siteImportPageTitle(page),
+        foundBy: page.foundBy,
+        words: siteImportPageWords(page),
+        images: page.images.length
+    };
+}
+/** The page's photos, the ones in its body first, each once. */ export function siteImportPageImages(page) {
+    const seen = new Set();
+    const images = [];
+    const add = (url, alt)=>{
+        if (!url || seen.has(url) || images.length >= AI_SITE_IMPORT_IMAGES_PER_PAGE) return;
+        seen.add(url);
+        images.push({
+            url,
+            alt: siteImportLabel(alt, 250)
+        });
+    };
+    for (const block of page.blocks)if (block.kind === 'image' && block.region === 'main') add(block.url, block.alt);
+    for (const image of page.images)add(image.url, image.alt);
+    return images;
+}
+/**
+ * The page's sections, from its headings: the first is its opening (its
+ * title or first heading), then one for each second-level heading of its
+ * body, at most `AI_SITE_IMPORT_MAX_SECTIONS`. A page with no headings below
+ * its title but more to say than an opening gets a second section for it.
+ */ export function siteImportPageSections(page) {
+    var _headings_find;
+    const main = page.blocks.filter((block)=>block.region === 'main');
+    const headings = main.filter((block)=>block.kind === 'heading');
+    const opening = siteImportLabel(((_headings_find = headings.find((block)=>block.level === 1)) == null ? void 0 : _headings_find.text) || page.title || '') || 'Introduction';
+    const sections = [
+        opening
+    ];
+    const seen = new Set([
+        opening.toLowerCase()
+    ]);
+    for (const heading of headings){
+        if (heading.level !== 2) continue;
+        const name = siteImportLabel(heading.text);
+        if (!name || seen.has(name.toLowerCase())) continue;
+        seen.add(name.toLowerCase());
+        sections.push(name);
+        if (sections.length >= AI_SITE_IMPORT_MAX_SECTIONS) break;
+    }
+    if (sections.length === 1 && siteImportPageWords({
+        title: '',
+        blocks: main
+    }) > 80) sections.push('Details');
+    return sections;
+}
+/** The page's text in order, as a prompt shows it inside its fence. */ export function siteImportPageText(page) {
+    const lines = [];
+    if (page.title) lines.push(`Title: ${page.title}`);
+    if (page.description) lines.push(`Description: ${page.description}`);
+    for (const block of page.blocks){
+        const where = block.region === 'main' ? '' : `(${block.region}) `;
+        if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`);
+        else if (block.kind === 'image') lines.push(`${where}[photo: ${block.alt || 'no description'}]`);
+        else if (block.kind === 'list-item') lines.push(`${where}- ${block.text}`);
+        else if (block.kind === 'quote') lines.push(`${where}> ${block.text}`);
+        else lines.push(`${where}${block.text}`);
+    }
+    const text = lines.join('\n');
+    return text.length > AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS ? text.slice(0, AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS) : text;
+}
+/** The page as a wave reads it. */ export function siteImportPageRead(page, at) {
+    return {
+        at,
+        title: siteImportPageTitle(page),
+        text: siteImportPageText(page),
+        sections: siteImportPageSections(page),
+        images: siteImportPageImages(page)
+    };
+}
+/** Why the crawler could not read a page, as an import leaves it out. */ export function siteImportLeftOutReason(reason) {
+    switch(reason){
+        case 'login-form':
+            return 'login-form';
+        case 'robots':
+            return 'robots';
+        case 'too-large':
+            return 'too-large';
+        case 'off-origin-redirect':
+            return 'off-site';
+        default:
+            return 'unreadable';
+    }
+}
+
+//# sourceMappingURL=site-import-page.js.map

package/src/lib/site-import/site-import-page.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-page.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * ONE CRAWLED PAGE, AS AN IMPORT KEEPS IT (AGL-3976).\n *\n * The crawler's page (`SiteImportPage`) holds everything it extracted; an\n * import keeps much less, and keeps it in two shapes:\n *\n * - **Listed**, for the inventory: its address, title, how it was found and\n *   how many words and photos it has. No text: an inventory of a whole site\n *   rides the job document.\n * - **Read**, for a wave: its text in order, its sections from its headings,\n *   and its photos' addresses. The text is kept as the page says it and is\n *   fenced as untrusted data only where a prompt shows it; the section names\n *   and the title travel outside the fence, so they are neutralized (no\n *   fence marker, no hidden character) and cut short.\n *=========================================*/\n\nimport { neutralizeUntrustedText } from '@aglyn/shared-util-tools/untrusted-text'\nimport {\n  AI_SITE_IMPORT_IMAGES_PER_PAGE,\n  AI_SITE_IMPORT_LABEL_MAX_CHARS,\n  AI_SITE_IMPORT_MAX_SECTIONS,\n  AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS,\n  type AiSiteImportLeftOutReason,\n  type AiSiteImportListedPage,\n  type AiSiteImportPageRead,\n} from '../model/ai-site-import'\nimport type { SiteImportPage, SiteImportSkipReason } from './site-import-crawl'\n\n/** A line of untrusted text safe to show outside a fence: neutralized, one line, cut short. */\nexport function siteImportLabel(text: string, max = AI_SITE_IMPORT_LABEL_MAX_CHARS): string {\n  const line = neutralizeUntrustedText(text).replace(/\\s+/g, ' ').trim()\n  return line.length > max ? `${line.slice(0, max - 1).trimEnd()}…` : line\n}\n\nconst wordsIn = (text: string) => (text.match(/[\\p{L}\\p{N}]+/gu) ?? []).length\n\n/** How many words a page says, its title and headings included. */\nexport function siteImportPageWords(page: Pick<SiteImportPage, 'title' | 'blocks'>): number {\n  return page.blocks.reduce((total, block) => total + (block.kind === 'image' ? 0 : wordsIn(block.text)), 0)\n}\n\n/** Between a page's own name and its site's in a browser title: \"About | Rosie's Bakery\". */\nconst TITLE_SEPARATOR = /\\s+[|\\u2013\\u2014\\u00b7:-]\\s+/\n\n/**\n * A page's name on the new site: \"Home\" for the home page, else its browser\n * title without the old site's name after a separator (\"About | Rosie's\n * Bakery\" is \"About\"), else its first heading.\n */\nexport function siteImportPageTitle(page: Pick<SiteImportPage, 'path' | 'title' | 'blocks'>): string {\n  if ((page.path.replace(/\\/+$/, '') || '/') === '/') return 'Home'\n  const own = siteImportLabel((page.title ?? '').split(TITLE_SEPARATOR)[0] ?? '')\n  if (own.length >= 2) return own\n  return siteImportLabel(page.blocks.find((block) => block.kind === 'heading')?.text ?? '')\n}\n\n/** The page as the inventory lists it. */\nexport function siteImportListedPage(page: SiteImportPage): AiSiteImportListedPage {\n  return {\n    url: page.url,\n    path: page.path,\n    title: siteImportPageTitle(page),\n    foundBy: page.foundBy,\n    words: siteImportPageWords(page),\n    images: page.images.length,\n  }\n}\n\n/** The page's photos, the ones in its body first, each once. */\nexport function siteImportPageImages(page: Pick<SiteImportPage, 'blocks' | 'images'>): Array<{ url: string; alt: string }> {\n  const seen = new Set<string>()\n  const images: Array<{ url: string; alt: string }> = []\n  const add = (url: string, alt: string) => {\n    if (!url || seen.has(url) || images.length >= AI_SITE_IMPORT_IMAGES_PER_PAGE) return\n    seen.add(url)\n    images.push({ url, alt: siteImportLabel(alt, 250) })\n  }\n  for (const block of page.blocks) if (block.kind === 'image' && block.region === 'main') add(block.url, block.alt)\n  for (const image of page.images) add(image.url, image.alt)\n  return images\n}\n\n/**\n * The page's sections, from its headings: the first is its opening (its\n * title or first heading), then one for each second-level heading of its\n * body, at most `AI_SITE_IMPORT_MAX_SECTIONS`. A page with no headings below\n * its title but more to say than an opening gets a second section for it.\n */\nexport function siteImportPageSections(page: Pick<SiteImportPage, 'title' | 'blocks'>): string[] {\n  const main = page.blocks.filter((block) => block.region === 'main')\n  const headings = main.filter((block): block is Extract<typeof block, { kind: 'heading' }> => block.kind === 'heading')\n  const opening = siteImportLabel(headings.find((block) => block.level === 1)?.text || page.title || '') || 'Introduction'\n  const sections = [opening]\n  const seen = new Set([opening.toLowerCase()])\n  for (const heading of headings) {\n    if (heading.level !== 2) continue\n    const name = siteImportLabel(heading.text)\n    if (!name || seen.has(name.toLowerCase())) continue\n    seen.add(name.toLowerCase())\n    sections.push(name)\n    if (sections.length >= AI_SITE_IMPORT_MAX_SECTIONS) break\n  }\n  if (sections.length === 1 && siteImportPageWords({ title: '', blocks: main }) > 80) sections.push('Details')\n  return sections\n}\n\n/** The page's text in order, as a prompt shows it inside its fence. */\nexport function siteImportPageText(page: Pick<SiteImportPage, 'title' | 'description' | 'blocks'>): string {\n  const lines: string[] = []\n  if (page.title) lines.push(`Title: ${page.title}`)\n  if (page.description) lines.push(`Description: ${page.description}`)\n  for (const block of page.blocks) {\n    const where = block.region === 'main' ? '' : `(${block.region}) `\n    if (block.kind === 'heading') lines.push(`${'#'.repeat(block.level)} ${block.text}`)\n    else if (block.kind === 'image') lines.push(`${where}[photo: ${block.alt || 'no description'}]`)\n    else if (block.kind === 'list-item') lines.push(`${where}- ${block.text}`)\n    else if (block.kind === 'quote') lines.push(`${where}> ${block.text}`)\n    else lines.push(`${where}${block.text}`)\n  }\n  const text = lines.join('\\n')\n  return text.length > AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS ? text.slice(0, AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS) : text\n}\n\n/** The page as a wave reads it. */\nexport function siteImportPageRead(page: SiteImportPage, at: string): AiSiteImportPageRead {\n  return {\n    at,\n    title: siteImportPageTitle(page),\n    text: siteImportPageText(page),\n    sections: siteImportPageSections(page),\n    images: siteImportPageImages(page),\n  }\n}\n\n/** Why the crawler could not read a page, as an import leaves it out. */\nexport function siteImportLeftOutReason(reason: SiteImportSkipReason): AiSiteImportLeftOutReason {\n  switch (reason) {\n    case 'login-form':\n      return 'login-form'\n    case 'robots':\n      return 'robots'\n    case 'too-large':\n      return 'too-large'\n    case 'off-origin-redirect':\n      return 'off-site'\n    default:\n      return 'unreadable'\n  }\n}\n"],"names":["neutralizeUntrustedText","AI_SITE_IMPORT_IMAGES_PER_PAGE","AI_SITE_IMPORT_LABEL_MAX_CHARS","AI_SITE_IMPORT_MAX_SECTIONS","AI_SITE_IMPORT_PAGE_TEXT_MAX_CHARS","siteImportLabel","text","max","line","replace","trim","length","slice","trimEnd","wordsIn","match","siteImportPageWords","page","blocks","reduce","total","block","kind","TITLE_SEPARATOR","siteImportPageTitle","path","own","title","split","find","siteImportListedPage","url","foundBy","words","images","siteImportPageImages","seen","Set","add","alt","has","push","region","image","siteImportPageSections","headings","main","filter","opening","level","sections","toLowerCase","heading","name","siteImportPageText","lines","description","where","repeat","join","siteImportPageRead","at","siteImportLeftOutReason","reason"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;2CAc2C,GAE3C,SAASA,uBAAuB,QAAQ,0CAAyC;AACjF,SACEC,8BAA8B,EAC9BC,8BAA8B,EAC9BC,2BAA2B,EAC3BC,kCAAkC,QAI7B,6BAAyB;AAGhC,6FAA6F,GAC7F,OAAO,SAASC,gBAAgBC,IAAY,EAAEC,MAAML,8BAA8B;IAChF,MAAMM,OAAOR,wBAAwBM,MAAMG,OAAO,CAAC,QAAQ,KAAKC,IAAI;IACpE,OAAOF,KAAKG,MAAM,GAAGJ,MAAM,GAAGC,KAAKI,KAAK,CAAC,GAAGL,MAAM,GAAGM,OAAO,GAAG,CAAC,CAAC,GAAGL;AACtE;AAEA,MAAMM,UAAU,CAACR;QAAkBA;WAAD,EAACA,cAAAA,KAAKS,KAAK,CAAC,ynaAAXT,cAAiC,EAAE,EAAEK,MAAM;;AAE9E,iEAAiE,GACjE,OAAO,SAASK,oBAAoBC,IAA8C;IAChF,OAAOA,KAAKC,MAAM,CAACC,MAAM,CAAC,CAACC,OAAOC,QAAUD,QAASC,CAAAA,MAAMC,IAAI,KAAK,UAAU,IAAIR,QAAQO,MAAMf,IAAI,CAAA,GAAI;AAC1G;AAEA,2FAA2F,GAC3F,MAAMiB,kBAAkB;AAExB;;;;CAIC,GACD,OAAO,SAASC,oBAAoBP,IAAuD;QAE7D,SAACA;QAENA;IAHvB,IAAI,AAACA,CAAAA,KAAKQ,IAAI,CAAChB,OAAO,CAAC,QAAQ,OAAO,GAAE,MAAO,KAAK,OAAO;IAC3D,MAAMiB,MAAMrB,iBAAgB,UAAA,EAACY,cAAAA,KAAKU,KAAK,YAAVV,cAAc,IAAIW,KAAK,CAACL,gBAAgB,CAAC,EAAE,YAA5C,UAAgD;IAC5E,IAAIG,IAAIf,MAAM,IAAI,GAAG,OAAOe;IAC5B,OAAOrB,yBAAgBY,oBAAAA,KAAKC,MAAM,CAACW,IAAI,CAAC,CAACR,QAAUA,MAAMC,IAAI,KAAK,+BAA3CL,kBAAuDX,IAAI,mBAAI;AACxF;AAEA,wCAAwC,GACxC,OAAO,SAASwB,qBAAqBb,IAAoB;IACvD,OAAO;QACLc,KAAKd,KAAKc,GAAG;QACbN,MAAMR,KAAKQ,IAAI;QACfE,OAAOH,oBAAoBP;QAC3Be,SAASf,KAAKe,OAAO;QACrBC,OAAOjB,oBAAoBC;QAC3BiB,QAAQjB,KAAKiB,MAAM,CAACvB,MAAM;IAC5B;AACF;AAEA,8DAA8D,GAC9D,OAAO,SAASwB,qBAAqBlB,IAA+C;IAClF,MAAMmB,OAAO,IAAIC;IACjB,MAAMH,SAA8C,EAAE;IACtD,MAAMI,MAAM,CAACP,KAAaQ;QACxB,IAAI,CAACR,OAAOK,KAAKI,GAAG,CAACT,QAAQG,OAAOvB,MAAM,IAAIV,gCAAgC;QAC9EmC,KAAKE,GAAG,CAACP;QACTG,OAAOO,IAAI,CAAC;YAAEV;YAAKQ,KAAKlC,gBAAgBkC,KAAK;QAAK;IACpD;IACA,KAAK,MAAMlB,SAASJ,KAAKC,MAAM,CAAE,IAAIG,MAAMC,IAAI,KAAK,WAAWD,MAAMqB,MAAM,KAAK,QAAQJ,IAAIjB,MAAMU,GAAG,EAAEV,MAAMkB,GAAG;IAChH,KAAK,MAAMI,SAAS1B,KAAKiB,MAAM,CAAEI,IAAIK,MAAMZ,GAAG,EAAEY,MAAMJ,GAAG;IACzD,OAAOL;AACT;AAEA;;;;;CAKC,GACD,OAAO,SAASU,uBAAuB3B,IAA8C;QAGnD4B;IAFhC,MAAMC,OAAO7B,KAAKC,MAAM,CAAC6B,MAAM,CAAC,CAAC1B,QAAUA,MAAMqB,MAAM,KAAK;IAC5D,MAAMG,WAAWC,KAAKC,MAAM,CAAC,CAAC1B,QAA+DA,MAAMC,IAAI,KAAK;IAC5G,MAAM0B,UAAU3C,gBAAgBwC,EAAAA,iBAAAA,SAAShB,IAAI,CAAC,CAACR,QAAUA,MAAM4B,KAAK,KAAK,uBAAzCJ,eAA6CvC,IAAI,KAAIW,KAAKU,KAAK,IAAI,OAAO;IAC1G,MAAMuB,WAAW;QAACF;KAAQ;IAC1B,MAAMZ,OAAO,IAAIC,IAAI;QAACW,QAAQG,WAAW;KAAG;IAC5C,KAAK,MAAMC,WAAWP,SAAU;QAC9B,IAAIO,QAAQH,KAAK,KAAK,GAAG;QACzB,MAAMI,OAAOhD,gBAAgB+C,QAAQ9C,IAAI;QACzC,IAAI,CAAC+C,QAAQjB,KAAKI,GAAG,CAACa,KAAKF,WAAW,KAAK;QAC3Cf,KAAKE,GAAG,CAACe,KAAKF,WAAW;QACzBD,SAAST,IAAI,CAACY;QACd,IAAIH,SAASvC,MAAM,IAAIR,6BAA6B;IACtD;IACA,IAAI+C,SAASvC,MAAM,KAAK,KAAKK,oBAAoB;QAAEW,OAAO;QAAIT,QAAQ4B;IAAK,KAAK,IAAII,SAAST,IAAI,CAAC;IAClG,OAAOS;AACT;AAEA,qEAAqE,GACrE,OAAO,SAASI,mBAAmBrC,IAA8D;IAC/F,MAAMsC,QAAkB,EAAE;IAC1B,IAAItC,KAAKU,KAAK,EAAE4B,MAAMd,IAAI,CAAC,CAAC,OAAO,EAAExB,KAAKU,KAAK,EAAE;IACjD,IAAIV,KAAKuC,WAAW,EAAED,MAAMd,IAAI,CAAC,CAAC,aAAa,EAAExB,KAAKuC,WAAW,EAAE;IACnE,KAAK,MAAMnC,SAASJ,KAAKC,MAAM,CAAE;QAC/B,MAAMuC,QAAQpC,MAAMqB,MAAM,KAAK,SAAS,KAAK,CAAC,CAAC,EAAErB,MAAMqB,MAAM,CAAC,EAAE,CAAC;QACjE,IAAIrB,MAAMC,IAAI,KAAK,WAAWiC,MAAMd,IAAI,CAAC,GAAG,IAAIiB,MAAM,CAACrC,MAAM4B,KAAK,EAAE,CAAC,EAAE5B,MAAMf,IAAI,EAAE;aAC9E,IAAIe,MAAMC,IAAI,KAAK,SAASiC,MAAMd,IAAI,CAAC,GAAGgB,MAAM,QAAQ,EAAEpC,MAAMkB,GAAG,IAAI,iBAAiB,CAAC,CAAC;aAC1F,IAAIlB,MAAMC,IAAI,KAAK,aAAaiC,MAAMd,IAAI,CAAC,GAAGgB,MAAM,EAAE,EAAEpC,MAAMf,IAAI,EAAE;aACpE,IAAIe,MAAMC,IAAI,KAAK,SAASiC,MAAMd,IAAI,CAAC,GAAGgB,MAAM,EAAE,EAAEpC,MAAMf,IAAI,EAAE;aAChEiD,MAAMd,IAAI,CAAC,GAAGgB,QAAQpC,MAAMf,IAAI,EAAE;IACzC;IACA,MAAMA,OAAOiD,MAAMI,IAAI,CAAC;IACxB,OAAOrD,KAAKK,MAAM,GAAGP,qCAAqCE,KAAKM,KAAK,CAAC,GAAGR,sCAAsCE;AAChH;AAEA,iCAAiC,GACjC,OAAO,SAASsD,mBAAmB3C,IAAoB,EAAE4C,EAAU;IACjE,OAAO;QACLA;QACAlC,OAAOH,oBAAoBP;QAC3BX,MAAMgD,mBAAmBrC;QACzBiC,UAAUN,uBAAuB3B;QACjCiB,QAAQC,qBAAqBlB;IAC/B;AACF;AAEA,uEAAuE,GACvE,OAAO,SAAS6C,wBAAwBC,MAA4B;IAClE,OAAQA;QACN,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF"}

package/src/lib/site-import/site-import-review.d.ts

@@ -0,0 +1,40 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { type AiSiteImportState } from '../model/ai-site-import';
+import type { AiJobItemLedger } from '../model/ai-jobs.types';
+/** The abuse queue's `source` for a site import's row. */
+export declare const SITE_IMPORT_REVIEW_SOURCE = "site-import";
+/** One row per import job. */
+export declare function siteImportReviewId(jobId: string): string;
+/** The reference staff quote: `SI-` and the job's id. */
+export declare function siteImportReviewReference(jobId: string): string;
+export type SiteImportReviewEvent = 'read' | 'wave-started' | 'wave-finished';
+export interface SiteImportReviewInput {
+    orgId: string;
+    hostId: string;
+    jobId: string;
+    state: AiSiteImportState;
+    items: readonly AiJobItemLedger[];
+    event: SiteImportReviewEvent;
+    /** The wave the event is about; absent on `read`. */
+    wave?: number;
+    nowMs: number;
+}
+/** The row's details: what was proved, read, kept, left out and brought in, rebuilt whole each time. */
+export declare function siteImportReviewDetails(input: SiteImportReviewInput): string;
+/** Writes (or rewrites) the import's row in the abuse queue. Never throws. */
+export declare function fileSiteImportReview(firestore: FirebaseFirestore.Firestore, input: SiteImportReviewInput): Promise<string | null>;