@aglyn/plugins-ai 1.0.0-beta.288 → 1.0.0-beta.290

package/src/lib/site-import/site-import-edge.js

@@ -0,0 +1,108 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ /*==========================================
+ * AGLYNBOT THROUGH AGLYN'S OWN EDGE (AGL-4085).
+ *
+ * The tenant deployment's bot protection answers a client that runs no
+ * JavaScript with a 429 challenge page, and AglynBot is such a client. So a
+ * site we host, aglyn.com included, could not be brought in: on 2026-10-20 the
+ * import read 10 of 121 pages, the ten the store-review bypass happens to
+ * admit, and no home page.
+ *
+ * The edge rule "Site import crawler bypass" (declared in
+ * `tools/scripts/lib/firewall-posture.mjs`) lifts that challenge, and nothing
+ * else, for a request carrying BOTH this header with the secret AND a
+ * user agent starting `AglynBot/`. This module is the only place that holds
+ * the secret, and it gives it out for one reason: a request to a hostname this
+ * platform serves (`isPlatformServedDomain`: our host records, then the hosting
+ * platform's live view). A site anywhere else never sees it.
+ *
+ * - The secret is `SITE_IMPORT_EDGE_TOKEN`, a server variable on the console,
+ *   where the crawl runs. Read in the bracket form, so Next never substitutes
+ *   it into a bundle. Unset, or not shaped like a secret, nothing is sent and
+ *   nothing is read: the crawler is challenged as it was before.
+ * - The decision is per hostname and per request: the fetch asks
+ *   `headersFor(url)` again on every redirect hop, so a hop to another host
+ *   carries nothing even where the caller would follow it.
+ * - Rotation: a new value in the console's env and the same value in the
+ *   rule, together. See `apps/docs/docs/operations/firewall-posture.md`.
+ *=========================================*/ /** The header the edge rule matches. Lowercase, as the rule declares it. */ export const SITE_IMPORT_EDGE_HEADER = 'x-aglyn-import';
+/** A secret shorter than this is treated as unset: `openssl rand -hex 32` makes 64. */ export const SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH = 32;
+/** Header-safe characters only, so the value can neither break a header nor fail the rule's match. */ const TOKEN_PATTERN = /^[A-Za-z0-9_-]+$/;
+/** Sends nothing anywhere. */ export const NO_SITE_IMPORT_HEADERS = ()=>({});
+/** The edge secret, or null when it is unset or not shaped like one. Never logged. */ export function siteImportEdgeToken(env) {
+    var _ref;
+    // The direct read is spelled out so the self-host env reference guard sees it.
+    const token = String((_ref = env ? env['SITE_IMPORT_EDGE_TOKEN'] : process.env['SITE_IMPORT_EDGE_TOKEN']) != null ? _ref : '').trim();
+    if (token.length < SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH || token.length > 256) return null;
+    return TOKEN_PATTERN.test(token) ? token : null;
+}
+async function defaultServed(hostname) {
+    // Loaded only once a secret is configured, so a crawl that has none (every
+    // spec, every self-hosted deployment without the rule) never opens Firestore.
+    // By relative path: see `site-import-edge-server.ts`.
+    const { isPlatformServedDomain } = await import("./site-import-edge-server.js");
+    return isPlatformServedDomain(hostname);
+}
+/**
+ * The edge header for requests to one hostname: the secret when this platform
+ * serves it, else nothing. Never throws.
+ */ export async function siteImportEdgeHeaders(hostname, deps = {}) {
+    const token = siteImportEdgeToken(deps.env);
+    if (!token) return {};
+    const name = String(hostname != null ? hostname : '').trim().toLowerCase();
+    if (!name) return {};
+    try {
+        var _deps_served;
+        const served = await ((_deps_served = deps.served) != null ? _deps_served : defaultServed)(name);
+        return served ? {
+            [SITE_IMPORT_EDGE_HEADER]: token
+        } : {};
+    } catch (error) {
+        console.error('[site-import-edge] could not tell whether the platform serves', name, error);
+        return {};
+    }
+}
+/**
+ * Per-request headers for a crawl of these origins: each origin's hostname is
+ * decided once, and a request is given the header only when it is https on
+ * the default port to an origin decided "served". Anything else, a hop to
+ * another host included, gets nothing.
+ */ export async function siteImportEdgeHeadersFor(origins, deps = {}) {
+    if (!siteImportEdgeToken(deps.env)) return NO_SITE_IMPORT_HEADERS;
+    const decided = new Map();
+    for (const raw of origins){
+        let origin;
+        try {
+            origin = new URL(raw);
+        } catch (unused) {
+            continue;
+        }
+        if (!isPlainHttpsOrigin(origin) || decided.has(origin.origin)) continue;
+        const headers = await siteImportEdgeHeaders(origin.hostname, deps);
+        if (Object.keys(headers).length) decided.set(origin.origin, headers);
+    }
+    if (!decided.size) return NO_SITE_IMPORT_HEADERS;
+    return (url)=>{
+        var _decided_get;
+        return isPlainHttpsOrigin(url) ? (_decided_get = decided.get(url.origin)) != null ? _decided_get : {} : {};
+    };
+}
+function isPlainHttpsOrigin(url) {
+    return url.protocol === 'https:' && (!url.port || url.port === '443') && !url.username && !url.password;
+}
+
+//# sourceMappingURL=site-import-edge.js.map

package/src/lib/site-import/site-import-edge.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-edge.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * AGLYNBOT THROUGH AGLYN'S OWN EDGE (AGL-4085).\n *\n * The tenant deployment's bot protection answers a client that runs no\n * JavaScript with a 429 challenge page, and AglynBot is such a client. So a\n * site we host, aglyn.com included, could not be brought in: on 2026-10-20 the\n * import read 10 of 121 pages, the ten the store-review bypass happens to\n * admit, and no home page.\n *\n * The edge rule \"Site import crawler bypass\" (declared in\n * `tools/scripts/lib/firewall-posture.mjs`) lifts that challenge, and nothing\n * else, for a request carrying BOTH this header with the secret AND a\n * user agent starting `AglynBot/`. This module is the only place that holds\n * the secret, and it gives it out for one reason: a request to a hostname this\n * platform serves (`isPlatformServedDomain`: our host records, then the hosting\n * platform's live view). A site anywhere else never sees it.\n *\n * - The secret is `SITE_IMPORT_EDGE_TOKEN`, a server variable on the console,\n *   where the crawl runs. Read in the bracket form, so Next never substitutes\n *   it into a bundle. Unset, or not shaped like a secret, nothing is sent and\n *   nothing is read: the crawler is challenged as it was before.\n * - The decision is per hostname and per request: the fetch asks\n *   `headersFor(url)` again on every redirect hop, so a hop to another host\n *   carries nothing even where the caller would follow it.\n * - Rotation: a new value in the console's env and the same value in the\n *   rule, together. See `apps/docs/docs/operations/firewall-posture.md`.\n *=========================================*/\n\n/** The header the edge rule matches. Lowercase, as the rule declares it. */\nexport const SITE_IMPORT_EDGE_HEADER = 'x-aglyn-import'\n\n/** A secret shorter than this is treated as unset: `openssl rand -hex 32` makes 64. */\nexport const SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH = 32\n\n/** Header-safe characters only, so the value can neither break a header nor fail the rule's match. */\nconst TOKEN_PATTERN = /^[A-Za-z0-9_-]+$/\n\n/** Extra headers for one request, asked per address. */\nexport type SiteImportHeadersFor = (url: URL) => Readonly<Record<string, string>>\n\n/** Sends nothing anywhere. */\nexport const NO_SITE_IMPORT_HEADERS: SiteImportHeadersFor = () => ({})\n\nexport interface SiteImportEdgeDeps {\n  /** Where the secret is read from; `process.env` by default. */\n  env?: Readonly<Record<string, string | undefined>>\n  /** Whether this platform serves a hostname; `isPlatformServedDomain` by default. */\n  served?: (hostname: string) => Promise<boolean>\n}\n\n/** The edge secret, or null when it is unset or not shaped like one. Never logged. */\nexport function siteImportEdgeToken(env?: SiteImportEdgeDeps['env']): string | null {\n  // The direct read is spelled out so the self-host env reference guard sees it.\n  const token = String((env ? env['SITE_IMPORT_EDGE_TOKEN'] : process.env['SITE_IMPORT_EDGE_TOKEN']) ?? '').trim()\n  if (token.length < SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH || token.length > 256) return null\n  return TOKEN_PATTERN.test(token) ? token : null\n}\n\nasync function defaultServed(hostname: string): Promise<boolean> {\n  // Loaded only once a secret is configured, so a crawl that has none (every\n  // spec, every self-hosted deployment without the rule) never opens Firestore.\n  // By relative path: see `site-import-edge-server.ts`.\n  const { isPlatformServedDomain } = await import('./site-import-edge-server')\n  return isPlatformServedDomain(hostname)\n}\n\n/**\n * The edge header for requests to one hostname: the secret when this platform\n * serves it, else nothing. Never throws.\n */\nexport async function siteImportEdgeHeaders(\n  hostname: string,\n  deps: SiteImportEdgeDeps = {},\n): Promise<Readonly<Record<string, string>>> {\n  const token = siteImportEdgeToken(deps.env)\n  if (!token) return {}\n  const name = String(hostname ?? '').trim().toLowerCase()\n  if (!name) return {}\n  try {\n    const served = await (deps.served ?? defaultServed)(name)\n    return served ? { [SITE_IMPORT_EDGE_HEADER]: token } : {}\n  } catch (error) {\n    console.error('[site-import-edge] could not tell whether the platform serves', name, error)\n    return {}\n  }\n}\n\n/**\n * Per-request headers for a crawl of these origins: each origin's hostname is\n * decided once, and a request is given the header only when it is https on\n * the default port to an origin decided \"served\". Anything else, a hop to\n * another host included, gets nothing.\n */\nexport async function siteImportEdgeHeadersFor(\n  origins: readonly string[],\n  deps: SiteImportEdgeDeps = {},\n): Promise<SiteImportHeadersFor> {\n  if (!siteImportEdgeToken(deps.env)) return NO_SITE_IMPORT_HEADERS\n  const decided = new Map<string, Readonly<Record<string, string>>>()\n  for (const raw of origins) {\n    let origin: URL\n    try {\n      origin = new URL(raw)\n    } catch {\n      continue\n    }\n    if (!isPlainHttpsOrigin(origin) || decided.has(origin.origin)) continue\n    const headers = await siteImportEdgeHeaders(origin.hostname, deps)\n    if (Object.keys(headers).length) decided.set(origin.origin, headers)\n  }\n  if (!decided.size) return NO_SITE_IMPORT_HEADERS\n  return (url: URL) => (isPlainHttpsOrigin(url) ? decided.get(url.origin) ?? {} : {})\n}\n\nfunction isPlainHttpsOrigin(url: URL): boolean {\n  return url.protocol === 'https:' && (!url.port || url.port === '443') && !url.username && !url.password\n}\n"],"names":["SITE_IMPORT_EDGE_HEADER","SITE_IMPORT_EDGE_TOKEN_MIN_LENGTH","TOKEN_PATTERN","NO_SITE_IMPORT_HEADERS","siteImportEdgeToken","env","token","String","process","trim","length","test","defaultServed","hostname","isPlatformServedDomain","siteImportEdgeHeaders","deps","name","toLowerCase","served","error","console","siteImportEdgeHeadersFor","origins","decided","Map","raw","origin","URL","isPlainHttpsOrigin","has","headers","Object","keys","set","size","url","get","protocol","port","username","password"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;2CA0B2C,GAE3C,0EAA0E,GAC1E,OAAO,MAAMA,0BAA0B,iBAAgB;AAEvD,qFAAqF,GACrF,OAAO,MAAMC,oCAAoC,GAAE;AAEnD,oGAAoG,GACpG,MAAMC,gBAAgB;AAKtB,4BAA4B,GAC5B,OAAO,MAAMC,yBAA+C,IAAO,CAAA,CAAC,CAAA,EAAE;AAStE,oFAAoF,GACpF,OAAO,SAASC,oBAAoBC,GAA+B;QAE3CA;IADtB,+EAA+E;IAC/E,MAAMC,QAAQC,QAAQF,OAAAA,MAAMA,GAAG,CAAC,yBAAyB,GAAGG,QAAQH,GAAG,CAAC,yBAAyB,YAA3EA,OAAgF,IAAII,IAAI;IAC9G,IAAIH,MAAMI,MAAM,GAAGT,qCAAqCK,MAAMI,MAAM,GAAG,KAAK,OAAO;IACnF,OAAOR,cAAcS,IAAI,CAACL,SAASA,QAAQ;AAC7C;AAEA,eAAeM,cAAcC,QAAgB;IAC3C,2EAA2E;IAC3E,8EAA8E;IAC9E,sDAAsD;IACtD,MAAM,EAAEC,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC;IAChD,OAAOA,uBAAuBD;AAChC;AAEA;;;CAGC,GACD,OAAO,eAAeE,sBACpBF,QAAgB,EAChBG,OAA2B,CAAC,CAAC;IAE7B,MAAMV,QAAQF,oBAAoBY,KAAKX,GAAG;IAC1C,IAAI,CAACC,OAAO,OAAO,CAAC;IACpB,MAAMW,OAAOV,OAAOM,mBAAAA,WAAY,IAAIJ,IAAI,GAAGS,WAAW;IACtD,IAAI,CAACD,MAAM,OAAO,CAAC;IACnB,IAAI;YACoBD;QAAtB,MAAMG,SAAS,MAAM,EAACH,eAAAA,KAAKG,MAAM,YAAXH,eAAeJ,eAAeK;QACpD,OAAOE,SAAS;YAAE,CAACnB,wBAAwB,EAAEM;QAAM,IAAI,CAAC;IAC1D,EAAE,OAAOc,OAAO;QACdC,QAAQD,KAAK,CAAC,iEAAiEH,MAAMG;QACrF,OAAO,CAAC;IACV;AACF;AAEA;;;;;CAKC,GACD,OAAO,eAAeE,yBACpBC,OAA0B,EAC1BP,OAA2B,CAAC,CAAC;IAE7B,IAAI,CAACZ,oBAAoBY,KAAKX,GAAG,GAAG,OAAOF;IAC3C,MAAMqB,UAAU,IAAIC;IACpB,KAAK,MAAMC,OAAOH,QAAS;QACzB,IAAII;QACJ,IAAI;YACFA,SAAS,IAAIC,IAAIF;QACnB,EAAE,eAAM;YACN;QACF;QACA,IAAI,CAACG,mBAAmBF,WAAWH,QAAQM,GAAG,CAACH,OAAOA,MAAM,GAAG;QAC/D,MAAMI,UAAU,MAAMhB,sBAAsBY,OAAOd,QAAQ,EAAEG;QAC7D,IAAIgB,OAAOC,IAAI,CAACF,SAASrB,MAAM,EAAEc,QAAQU,GAAG,CAACP,OAAOA,MAAM,EAAEI;IAC9D;IACA,IAAI,CAACP,QAAQW,IAAI,EAAE,OAAOhC;IAC1B,OAAO,CAACiC;YAAwCZ;eAA1BK,mBAAmBO,QAAOZ,eAAAA,QAAQa,GAAG,CAACD,IAAIT,MAAM,aAAtBH,eAA2B,CAAC,IAAI,CAAC;;AACnF;AAEA,SAASK,mBAAmBO,GAAQ;IAClC,OAAOA,IAAIE,QAAQ,KAAK,YAAa,CAAA,CAACF,IAAIG,IAAI,IAAIH,IAAIG,IAAI,KAAK,KAAI,KAAM,CAACH,IAAII,QAAQ,IAAI,CAACJ,IAAIK,QAAQ;AACzG"}

package/src/lib/site-import/site-import-html.d.ts

@@ -29,6 +29,19 @@ export type SiteImportBlock = {
     url: string;
     alt: string;
     region: SiteImportRegion;
+}
+/**
+ * A link the page shows outside its menus (AGL-4083): `href` is an absolute
+ * http(s) address, a `mailto:` or a `tel:`. `inline` when its words are
+ * already in the block before it (a link inside a sentence); a link on its
+ * own (a button, an email address in a card) is only this block.
+ */
+ | {
+    kind: 'link';
+    text: string;
+    href: string;
+    region: SiteImportRegion;
+    inline?: true;
 };
 /** Where on the page a block sat: the header and footer carry business details. */
 export type SiteImportRegion = 'header' | 'main' | 'footer';
@@ -38,6 +51,46 @@ export interface SiteImportImage {
     alt: string;
     /** On the site's own origin; otherwise a CDN or a third party. The importer decides. */
     sameSite: boolean;
+    /** Where on the page it sat, its menus included (AGL-4083): a header's picture is often the logo. */
+    region: SiteImportRegion;
+}
+/** A `<link rel="icon">` or `apple-touch-icon` of the page's head (AGL-4083). */
+export interface SiteImportIcon {
+    url: string;
+    rel: 'icon' | 'apple-touch-icon';
+    /** The larger side its `sizes` names, or `null` for none or `any`. */
+    size: number | null;
+    /** Its declared type, as written. */
+    type: string | null;
+}
+/**
+ * The link to the home page in a page's header (AGL-4083): what its brand is
+ * drawn with. `logo` is the first picture inside it; `text` its words, which
+ * are the site's name when the header prints one.
+ */
+export interface SiteImportBrandMark {
+    logo: {
+        url: string;
+        alt: string;
+    } | null;
+    text: string;
+    region: SiteImportRegion;
+}
+/**
+ * The page's Organization (or LocalBusiness) structured data (AGL-4083), as
+ * written: the site's own statement of its name, address, phone, email and
+ * profiles. Every field is the empty string, or an empty list, when absent.
+ */
+export interface SiteImportOrganization {
+    name: string;
+    description: string;
+    url: string;
+    logo: string;
+    telephone: string;
+    email: string;
+    /** One line: street, town, region and postal code, country. */
+    address: string;
+    sameAs: string[];
 }
 export interface SiteImportNavLink {
     text: string;
@@ -55,6 +108,13 @@ export interface SiteImportExtract {
     description: string;
     lang: string | null;
     canonical: string | null;
+    /** The share card, `og:image` (or `twitter:image`), as an absolute address (AGL-4083). */
+    ogImage: string | null;
+    /** The site's own name, `og:site_name` (or `application-name`); empty when absent. */
+    siteName: string;
+    icons: SiteImportIcon[];
+    brand: SiteImportBrandMark | null;
+    organization: SiteImportOrganization | null;
     blocks: SiteImportBlock[];
     images: SiteImportImage[];
     navLinks: SiteImportNavLink[];
@@ -69,13 +129,17 @@ export interface SiteImportExtract {
 }
 /** Ceilings that keep one odd page from becoming the whole inventory. */
 export declare const SITE_IMPORT_PAGE_LIMITS: {
-    readonly blocks: 400;
-    readonly blockChars: 2000;
+    /** A pricing page's plan table runs past 400 (AGL-4083); a wave's read keeps fewer, under its own budget. */
+    readonly blocks: 2000;
+    /** A legal page's paragraph runs past 2,000 characters. */
+    readonly blockChars: 10000;
     readonly images: 100;
     readonly links: 300;
-    readonly navLinks: 60;
+    /** A mega-menu's entries and the footer's columns together. */
+    readonly navLinks: 150;
     readonly forms: 10;
     readonly fieldsPerForm: 30;
+    readonly icons: 12;
 };
 /** Character references to characters; an unknown name stays as written. */
 export declare function decodeHtmlEntities(text: string): string;
@@ -85,6 +149,12 @@ export declare function parseAttributes(source: string): Record<string, string>;
 export declare function readMetaValues(html: string, name: string): string[];
 /** `raw` against the page, as an absolute http(s) address without a fragment. */
 export declare function resolveSiteUrl(raw: string | undefined, base: string): URL | null;
+/**
+ * Where a link a page shows goes (AGL-4083): an absolute http(s) address
+ * without its fragment, a `mailto:` with an address in it, or a `tel:` with
+ * a number in it; `null` for anything else (a script, a jump on the page).
+ */
+export declare function resolveSiteLink(raw: string | undefined, base: string): string | null;
 /**
  * The words, structure, pictures and links of one page.
  *