@aglyn/plugins-ai 1.0.0-beta.271 → 1.0.0-beta.272

package/src/lib/site-import/site-import-review.js

@@ -0,0 +1,149 @@
+import { _ as _extends } from "@swc/helpers/_/_extends";
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ /*==========================================
+ * EVERY SITE IMPORT IS LOGGED FOR TRUST & SAFETY (AGL-3976).
+ *
+ * A site import is a one-click copy of a website, which is exactly what a
+ * phishing clone is. Proving ownership is the control; this row is how a
+ * person checks the control held. Every import job files ONE row in the
+ * abuse queue (`abuseReports`), in the shape the outbound screen files a
+ * hold (`fileOutboundHold`): the same fields the admin page and the daily
+ * review already read, `source: 'site-import'`, the proved domain as text
+ * (the queue never renders a reported address as a link), and the proof.
+ *
+ * The row is written when the site is read and rewritten as each wave
+ * starts and finishes, its `reportCount` counting the waves, so the queue
+ * shows one import as one row however long it runs. It holds nothing: the
+ * import goes on while the row is open. Staff look at the old site and the
+ * new one; a workspace bringing in a site that is not its own is locked,
+ * and the row closed with what was done.
+ *
+ * Never throws: a log that cannot be written must not stop a person's
+ * import, and the failure is a log line staff alerts read.
+ *=========================================*/ import { ABUSE_REPORT_COLLECTION, ABUSE_REPORT_MAX_DETAILS } from "@aglyn/aglyn/app-utils/abuse-report";
+import { FieldValue } from "firebase-admin/firestore";
+import { AI_SITE_IMPORT_LEFT_OUT_REASONS, aiSiteImportPagesText } from "../model/ai-site-import.js";
+/** The abuse queue's `source` for a site import's row. */ export const SITE_IMPORT_REVIEW_SOURCE = 'site-import';
+/** One row per import job. */ export function siteImportReviewId(jobId) {
+    return `site-import-${jobId}`;
+}
+/** The reference staff quote: `SI-` and the job's id. */ export function siteImportReviewReference(jobId) {
+    return `SI-${jobId.slice(0, 10).toUpperCase()}`;
+}
+const PROOF_WORDS = {
+    dns: 'a DNS TXT record',
+    meta: 'a meta tag on its home page',
+    email: 'a code emailed to an address at the domain'
+};
+/** The row's details: what was proved, read, kept, left out and brought in, rebuilt whole each time. */ export function siteImportReviewDetails(input) {
+    var _PROOF_WORDS_state_ownership_method, _state_leftOutCount_loginform;
+    const { state, items } = input;
+    const proof = state.ownership.method ? (_PROOF_WORDS_state_ownership_method = PROOF_WORDS[state.ownership.method]) != null ? _PROOF_WORDS_state_ownership_method : state.ownership.method : 'an unknown method';
+    const provenOn = state.ownership.provenAt ? new Date(state.ownership.provenAt).toISOString().slice(0, 10) : 'an unknown day';
+    const leftOut = AI_SITE_IMPORT_LEFT_OUT_REASONS.filter((reason)=>{
+        var _state_leftOutCount_reason;
+        return ((_state_leftOutCount_reason = state.leftOutCount[reason]) != null ? _state_leftOutCount_reason : 0) > 0;
+    }).map((reason)=>`${reason} ${state.leftOutCount[reason]}`).join(', ');
+    const built = items.filter((row)=>row.op === 'page' && (row.status === 'succeeded' || row.status === 'degraded')).length;
+    const started = state.waves.filter((wave)=>wave.status !== 'waiting').length;
+    const photos = state.pages.reduce((total, page)=>{
+        var _ref;
+        var _page_photos;
+        return total + ((_ref = (_page_photos = page.photos) == null ? void 0 : _page_photos.brought) != null ? _ref : 0);
+    }, 0);
+    const lines = [
+        `Site import: ${state.domain} into site ${input.hostId} (AI job ${input.jobId}).`,
+        `Ownership proven by ${proof}${state.ownership.detail ? ` (${state.ownership.detail})` : ''} on ${provenOn}.`,
+        `Read ${aiSiteImportPagesText(state.found)}; bringing in ${aiSiteImportPagesText(state.pages.length)} in ${state.waves.length} ${state.waves.length === 1 ? 'wave' : 'waves'}` + `${state.pageLimit === null ? '' : ` (plan page count ${state.pageLimit}, ${state.existingPages} already on the site)`}.`,
+        ...leftOut ? [
+            `Left out: ${leftOut}.`
+        ] : [],
+        ...((_state_leftOutCount_loginform = state.leftOutCount['login-form']) != null ? _state_leftOutCount_loginform : 0) > 0 ? [
+            `Sign-in pages refused: ${state.leftOutCount['login-form']}. A site with sign-in pages is worth a closer look.`
+        ] : [],
+        `Waves started: ${started} of ${state.waves.length}. Pages brought in so far: ${built}. Photos copied into the media library: ${photos}.`,
+        'Nothing is published by the import: its pages are drafts until a member publishes them.',
+        'Look at the old site and the new one. If this copies a site the workspace does not own, lock the workspace, then close this row with what you did; otherwise dismiss it.'
+    ];
+    return lines.join('\n').slice(0, ABUSE_REPORT_MAX_DETAILS);
+}
+/** Writes (or rewrites) the import's row in the abuse queue. Never throws. */ export async function fileSiteImportReview(firestore, input) {
+    const reviewId = siteImportReviewId(input.jobId);
+    try {
+        var _input_state_leftOutCount_loginform;
+        const ref = firestore.collection(ABUSE_REPORT_COLLECTION).doc(reviewId);
+        const first = !(await ref.get()).exists;
+        const built = input.items.filter((row)=>row.op === 'page' && (row.status === 'succeeded' || row.status === 'degraded')).length;
+        await ref.set(_extends({
+            reference: siteImportReviewReference(input.jobId),
+            // "Impersonation or trademark": the harm a copied site does is
+            // wearing a business it is not. Normal severity: the proof already
+            // held, and this row is the check that it did.
+            category: 'impersonation',
+            severity: 'normal',
+            source: SITE_IMPORT_REVIEW_SOURCE,
+            // The proved site as TEXT: the queue never renders it as a link.
+            url: `https://${input.state.domain}/`,
+            reportedHostname: input.state.domain,
+            hostId: input.hostId,
+            orgId: input.orgId,
+            details: siteImportReviewDetails(input),
+            reporterEmail: null,
+            reporterName: null,
+            dmca: null
+        }, input.event === 'wave-started' ? {
+            reportCount: FieldValue.increment(1)
+        } : first ? {
+            reportCount: 0
+        } : {}, {
+            siteImport: _extends({
+                jobId: input.jobId,
+                domain: input.state.domain,
+                method: input.state.ownership.method,
+                provenAt: input.state.ownership.provenAt,
+                detail: input.state.ownership.detail,
+                found: input.state.found,
+                kept: input.state.pages.length,
+                waves: input.state.waves.length,
+                wavesStarted: input.state.waves.filter((wave)=>wave.status !== 'waiting').length,
+                pagesBrought: built,
+                loginPagesRefused: (_input_state_leftOutCount_loginform = input.state.leftOutCount['login-form']) != null ? _input_state_leftOutCount_loginform : 0,
+                lastEvent: input.event
+            }, input.wave !== undefined ? {
+                lastWave: input.wave
+            } : {}, {
+                atMs: input.nowMs
+            }),
+            updatedAt: FieldValue.serverTimestamp()
+        }, first ? {
+            status: 'open',
+            createdAt: FieldValue.serverTimestamp()
+        } : {}), {
+            merge: true
+        });
+        return reviewId;
+    } catch (error) {
+        console.error('[site-import] trust and safety row failed', {
+            orgId: input.orgId,
+            jobId: input.jobId,
+            error
+        });
+        return null;
+    }
+}
+
+//# sourceMappingURL=site-import-review.js.map

package/src/lib/site-import/site-import-review.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-review.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * EVERY SITE IMPORT IS LOGGED FOR TRUST & SAFETY (AGL-3976).\n *\n * A site import is a one-click copy of a website, which is exactly what a\n * phishing clone is. Proving ownership is the control; this row is how a\n * person checks the control held. Every import job files ONE row in the\n * abuse queue (`abuseReports`), in the shape the outbound screen files a\n * hold (`fileOutboundHold`): the same fields the admin page and the daily\n * review already read, `source: 'site-import'`, the proved domain as text\n * (the queue never renders a reported address as a link), and the proof.\n *\n * The row is written when the site is read and rewritten as each wave\n * starts and finishes, its `reportCount` counting the waves, so the queue\n * shows one import as one row however long it runs. It holds nothing: the\n * import goes on while the row is open. Staff look at the old site and the\n * new one; a workspace bringing in a site that is not its own is locked,\n * and the row closed with what was done.\n *\n * Never throws: a log that cannot be written must not stop a person's\n * import, and the failure is a log line staff alerts read.\n *=========================================*/\n\nimport { ABUSE_REPORT_COLLECTION, ABUSE_REPORT_MAX_DETAILS } from '@aglyn/aglyn/app-utils/abuse-report'\nimport { FieldValue } from 'firebase-admin/firestore'\nimport {\n  AI_SITE_IMPORT_LEFT_OUT_REASONS,\n  aiSiteImportPagesText,\n  type AiSiteImportState,\n} from '../model/ai-site-import'\nimport type { AiJobItemLedger } from '../model/ai-jobs.types'\n\n/** The abuse queue's `source` for a site import's row. */\nexport const SITE_IMPORT_REVIEW_SOURCE = 'site-import'\n\n/** One row per import job. */\nexport function siteImportReviewId(jobId: string): string {\n  return `site-import-${jobId}`\n}\n\n/** The reference staff quote: `SI-` and the job's id. */\nexport function siteImportReviewReference(jobId: string): string {\n  return `SI-${jobId.slice(0, 10).toUpperCase()}`\n}\n\nconst PROOF_WORDS: Record<string, string> = {\n  dns: 'a DNS TXT record',\n  meta: 'a meta tag on its home page',\n  email: 'a code emailed to an address at the domain',\n}\n\nexport type SiteImportReviewEvent = 'read' | 'wave-started' | 'wave-finished'\n\nexport interface SiteImportReviewInput {\n  orgId: string\n  hostId: string\n  jobId: string\n  state: AiSiteImportState\n  items: readonly AiJobItemLedger[]\n  event: SiteImportReviewEvent\n  /** The wave the event is about; absent on `read`. */\n  wave?: number\n  nowMs: number\n}\n\n/** The row's details: what was proved, read, kept, left out and brought in, rebuilt whole each time. */\nexport function siteImportReviewDetails(input: SiteImportReviewInput): string {\n  const { state, items } = input\n  const proof = state.ownership.method ? (PROOF_WORDS[state.ownership.method] ?? state.ownership.method) : 'an unknown method'\n  const provenOn = state.ownership.provenAt ? new Date(state.ownership.provenAt).toISOString().slice(0, 10) : 'an unknown day'\n  const leftOut = AI_SITE_IMPORT_LEFT_OUT_REASONS.filter((reason) => (state.leftOutCount[reason] ?? 0) > 0)\n    .map((reason) => `${reason} ${state.leftOutCount[reason]}`)\n    .join(', ')\n  const built = items.filter((row) => row.op === 'page' && (row.status === 'succeeded' || row.status === 'degraded')).length\n  const started = state.waves.filter((wave) => wave.status !== 'waiting').length\n  const photos = state.pages.reduce((total, page) => total + (page.photos?.brought ?? 0), 0)\n  const lines = [\n    `Site import: ${state.domain} into site ${input.hostId} (AI job ${input.jobId}).`,\n    `Ownership proven by ${proof}${state.ownership.detail ? ` (${state.ownership.detail})` : ''} on ${provenOn}.`,\n    `Read ${aiSiteImportPagesText(state.found)}; bringing in ${aiSiteImportPagesText(state.pages.length)} in ${state.waves.length} ${state.waves.length === 1 ? 'wave' : 'waves'}` +\n      `${state.pageLimit === null ? '' : ` (plan page count ${state.pageLimit}, ${state.existingPages} already on the site)`}.`,\n    ...(leftOut ? [`Left out: ${leftOut}.`] : []),\n    ...((state.leftOutCount['login-form'] ?? 0) > 0\n      ? [`Sign-in pages refused: ${state.leftOutCount['login-form']}. A site with sign-in pages is worth a closer look.`]\n      : []),\n    `Waves started: ${started} of ${state.waves.length}. Pages brought in so far: ${built}. Photos copied into the media library: ${photos}.`,\n    'Nothing is published by the import: its pages are drafts until a member publishes them.',\n    'Look at the old site and the new one. If this copies a site the workspace does not own, lock the workspace, then close this row with what you did; otherwise dismiss it.',\n  ]\n  return lines.join('\\n').slice(0, ABUSE_REPORT_MAX_DETAILS)\n}\n\n/** Writes (or rewrites) the import's row in the abuse queue. Never throws. */\nexport async function fileSiteImportReview(\n  firestore: FirebaseFirestore.Firestore,\n  input: SiteImportReviewInput,\n): Promise<string | null> {\n  const reviewId = siteImportReviewId(input.jobId)\n  try {\n    const ref = firestore.collection(ABUSE_REPORT_COLLECTION).doc(reviewId)\n    const first = !(await ref.get()).exists\n    const built = input.items.filter((row) => row.op === 'page' && (row.status === 'succeeded' || row.status === 'degraded')).length\n    await ref.set(\n      {\n        reference: siteImportReviewReference(input.jobId),\n        // \"Impersonation or trademark\": the harm a copied site does is\n        // wearing a business it is not. Normal severity: the proof already\n        // held, and this row is the check that it did.\n        category: 'impersonation',\n        severity: 'normal',\n        source: SITE_IMPORT_REVIEW_SOURCE,\n        // The proved site as TEXT: the queue never renders it as a link.\n        url: `https://${input.state.domain}/`,\n        reportedHostname: input.state.domain,\n        hostId: input.hostId,\n        orgId: input.orgId,\n        details: siteImportReviewDetails(input),\n        reporterEmail: null,\n        reporterName: null,\n        dmca: null,\n        // One a wave started, so a long import reads as the volume it is.\n        ...(input.event === 'wave-started' ? { reportCount: FieldValue.increment(1) } : first ? { reportCount: 0 } : {}),\n        siteImport: {\n          jobId: input.jobId,\n          domain: input.state.domain,\n          method: input.state.ownership.method,\n          provenAt: input.state.ownership.provenAt,\n          detail: input.state.ownership.detail,\n          found: input.state.found,\n          kept: input.state.pages.length,\n          waves: input.state.waves.length,\n          wavesStarted: input.state.waves.filter((wave) => wave.status !== 'waiting').length,\n          pagesBrought: built,\n          loginPagesRefused: input.state.leftOutCount['login-form'] ?? 0,\n          lastEvent: input.event,\n          ...(input.wave !== undefined ? { lastWave: input.wave } : {}),\n          atMs: input.nowMs,\n        },\n        updatedAt: FieldValue.serverTimestamp(),\n        ...(first ? { status: 'open', createdAt: FieldValue.serverTimestamp() } : {}),\n      },\n      { merge: true },\n    )\n    return reviewId\n  } catch (error) {\n    console.error('[site-import] trust and safety row failed', { orgId: input.orgId, jobId: input.jobId, error })\n    return null\n  }\n}\n"],"names":["ABUSE_REPORT_COLLECTION","ABUSE_REPORT_MAX_DETAILS","FieldValue","AI_SITE_IMPORT_LEFT_OUT_REASONS","aiSiteImportPagesText","SITE_IMPORT_REVIEW_SOURCE","siteImportReviewId","jobId","siteImportReviewReference","slice","toUpperCase","PROOF_WORDS","dns","meta","email","siteImportReviewDetails","input","state","items","proof","ownership","method","provenOn","provenAt","Date","toISOString","leftOut","filter","reason","leftOutCount","map","join","built","row","op","status","length","started","waves","wave","photos","pages","reduce","total","page","brought","lines","domain","hostId","detail","found","pageLimit","existingPages","fileSiteImportReview","firestore","reviewId","ref","collection","doc","first","get","exists","set","reference","category","severity","source","url","reportedHostname","orgId","details","reporterEmail","reporterName","dmca","event","reportCount","increment","siteImport","kept","wavesStarted","pagesBrought","loginPagesRefused","lastEvent","undefined","lastWave","atMs","nowMs","updatedAt","serverTimestamp","createdAt","merge","error","console"],"mappings":";AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;2CAoB2C,GAE3C,SAASA,uBAAuB,EAAEC,wBAAwB,QAAQ,sCAAqC;AACvG,SAASC,UAAU,QAAQ,2BAA0B;AACrD,SACEC,+BAA+B,EAC/BC,qBAAqB,QAEhB,6BAAyB;AAGhC,wDAAwD,GACxD,OAAO,MAAMC,4BAA4B,cAAa;AAEtD,4BAA4B,GAC5B,OAAO,SAASC,mBAAmBC,KAAa;IAC9C,OAAO,CAAC,YAAY,EAAEA,OAAO;AAC/B;AAEA,uDAAuD,GACvD,OAAO,SAASC,0BAA0BD,KAAa;IACrD,OAAO,CAAC,GAAG,EAAEA,MAAME,KAAK,CAAC,GAAG,IAAIC,WAAW,IAAI;AACjD;AAEA,MAAMC,cAAsC;IAC1CC,KAAK;IACLC,MAAM;IACNC,OAAO;AACT;AAgBA,sGAAsG,GACtG,OAAO,SAASC,wBAAwBC,KAA4B;QAE1BL,qCAcjCM;IAfP,MAAM,EAAEA,KAAK,EAAEC,KAAK,EAAE,GAAGF;IACzB,MAAMG,QAAQF,MAAMG,SAAS,CAACC,MAAM,IAAIV,sCAAAA,WAAW,CAACM,MAAMG,SAAS,CAACC,MAAM,CAAC,YAAnCV,sCAAuCM,MAAMG,SAAS,CAACC,MAAM,GAAI;IACzG,MAAMC,WAAWL,MAAMG,SAAS,CAACG,QAAQ,GAAG,IAAIC,KAAKP,MAAMG,SAAS,CAACG,QAAQ,EAAEE,WAAW,GAAGhB,KAAK,CAAC,GAAG,MAAM;IAC5G,MAAMiB,UAAUvB,gCAAgCwB,MAAM,CAAC,CAACC;YAAYX;eAAD,EAACA,6BAAAA,MAAMY,YAAY,CAACD,OAAO,YAA1BX,6BAA8B,KAAK;OACpGa,GAAG,CAAC,CAACF,SAAW,GAAGA,OAAO,CAAC,EAAEX,MAAMY,YAAY,CAACD,OAAO,EAAE,EACzDG,IAAI,CAAC;IACR,MAAMC,QAAQd,MAAMS,MAAM,CAAC,CAACM,MAAQA,IAAIC,EAAE,KAAK,UAAWD,CAAAA,IAAIE,MAAM,KAAK,eAAeF,IAAIE,MAAM,KAAK,UAAS,GAAIC,MAAM;IAC1H,MAAMC,UAAUpB,MAAMqB,KAAK,CAACX,MAAM,CAAC,CAACY,OAASA,KAAKJ,MAAM,KAAK,WAAWC,MAAM;IAC9E,MAAMI,SAASvB,MAAMwB,KAAK,CAACC,MAAM,CAAC,CAACC,OAAOC;;YAAkBA;eAATD,kBAASC,eAAAA,KAAKJ,MAAM,qBAAXI,aAAaC,OAAO,mBAAI;OAAI;IACxF,MAAMC,QAAQ;QACZ,CAAC,aAAa,EAAE7B,MAAM8B,MAAM,CAAC,WAAW,EAAE/B,MAAMgC,MAAM,CAAC,SAAS,EAAEhC,MAAMT,KAAK,CAAC,EAAE,CAAC;QACjF,CAAC,oBAAoB,EAAEY,QAAQF,MAAMG,SAAS,CAAC6B,MAAM,GAAG,CAAC,EAAE,EAAEhC,MAAMG,SAAS,CAAC6B,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,EAAE3B,SAAS,CAAC,CAAC;QAC7G,CAAC,KAAK,EAAElB,sBAAsBa,MAAMiC,KAAK,EAAE,cAAc,EAAE9C,sBAAsBa,MAAMwB,KAAK,CAACL,MAAM,EAAE,IAAI,EAAEnB,MAAMqB,KAAK,CAACF,MAAM,CAAC,CAAC,EAAEnB,MAAMqB,KAAK,CAACF,MAAM,KAAK,IAAI,SAAS,SAAS,GAC5K,GAAGnB,MAAMkC,SAAS,KAAK,OAAO,KAAK,CAAC,kBAAkB,EAAElC,MAAMkC,SAAS,CAAC,EAAE,EAAElC,MAAMmC,aAAa,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;WACvH1B,UAAU;YAAC,CAAC,UAAU,EAAEA,QAAQ,CAAC,CAAC;SAAC,GAAG,EAAE;WACxC,EAACT,gCAAAA,MAAMY,YAAY,CAAC,aAAa,YAAhCZ,gCAAoC,KAAK,IAC1C;YAAC,CAAC,uBAAuB,EAAEA,MAAMY,YAAY,CAAC,aAAa,CAAC,mDAAmD,CAAC;SAAC,GACjH,EAAE;QACN,CAAC,eAAe,EAAEQ,QAAQ,IAAI,EAAEpB,MAAMqB,KAAK,CAACF,MAAM,CAAC,2BAA2B,EAAEJ,MAAM,wCAAwC,EAAEQ,OAAO,CAAC,CAAC;QACzI;QACA;KACD;IACD,OAAOM,MAAMf,IAAI,CAAC,MAAMtB,KAAK,CAAC,GAAGR;AACnC;AAEA,4EAA4E,GAC5E,OAAO,eAAeoD,qBACpBC,SAAsC,EACtCtC,KAA4B;IAE5B,MAAMuC,WAAWjD,mBAAmBU,MAAMT,KAAK;IAC/C,IAAI;YAmCuBS;QAlCzB,MAAMwC,MAAMF,UAAUG,UAAU,CAACzD,yBAAyB0D,GAAG,CAACH;QAC9D,MAAMI,QAAQ,CAAC,AAAC,CAAA,MAAMH,IAAII,GAAG,EAAC,EAAGC,MAAM;QACvC,MAAM7B,QAAQhB,MAAME,KAAK,CAACS,MAAM,CAAC,CAACM,MAAQA,IAAIC,EAAE,KAAK,UAAWD,CAAAA,IAAIE,MAAM,KAAK,eAAeF,IAAIE,MAAM,KAAK,UAAS,GAAIC,MAAM;QAChI,MAAMoB,IAAIM,GAAG,CACX;YACEC,WAAWvD,0BAA0BQ,MAAMT,KAAK;YAChD,+DAA+D;YAC/D,mEAAmE;YACnE,+CAA+C;YAC/CyD,UAAU;YACVC,UAAU;YACVC,QAAQ7D;YACR,iEAAiE;YACjE8D,KAAK,CAAC,QAAQ,EAAEnD,MAAMC,KAAK,CAAC8B,MAAM,CAAC,CAAC,CAAC;YACrCqB,kBAAkBpD,MAAMC,KAAK,CAAC8B,MAAM;YACpCC,QAAQhC,MAAMgC,MAAM;YACpBqB,OAAOrD,MAAMqD,KAAK;YAClBC,SAASvD,wBAAwBC;YACjCuD,eAAe;YACfC,cAAc;YACdC,MAAM;WAEFzD,MAAM0D,KAAK,KAAK,iBAAiB;YAAEC,aAAazE,WAAW0E,SAAS,CAAC;QAAG,IAAIjB,QAAQ;YAAEgB,aAAa;QAAE,IAAI,CAAC;YAC9GE,YAAY;gBACVtE,OAAOS,MAAMT,KAAK;gBAClBwC,QAAQ/B,MAAMC,KAAK,CAAC8B,MAAM;gBAC1B1B,QAAQL,MAAMC,KAAK,CAACG,SAAS,CAACC,MAAM;gBACpCE,UAAUP,MAAMC,KAAK,CAACG,SAAS,CAACG,QAAQ;gBACxC0B,QAAQjC,MAAMC,KAAK,CAACG,SAAS,CAAC6B,MAAM;gBACpCC,OAAOlC,MAAMC,KAAK,CAACiC,KAAK;gBACxB4B,MAAM9D,MAAMC,KAAK,CAACwB,KAAK,CAACL,MAAM;gBAC9BE,OAAOtB,MAAMC,KAAK,CAACqB,KAAK,CAACF,MAAM;gBAC/B2C,cAAc/D,MAAMC,KAAK,CAACqB,KAAK,CAACX,MAAM,CAAC,CAACY,OAASA,KAAKJ,MAAM,KAAK,WAAWC,MAAM;gBAClF4C,cAAchD;gBACdiD,iBAAiB,GAAEjE,sCAAAA,MAAMC,KAAK,CAACY,YAAY,CAAC,aAAa,YAAtCb,sCAA0C;gBAC7DkE,WAAWlE,MAAM0D,KAAK;eAClB1D,MAAMuB,IAAI,KAAK4C,YAAY;gBAAEC,UAAUpE,MAAMuB,IAAI;YAAC,IAAI,CAAC;gBAC3D8C,MAAMrE,MAAMsE,KAAK;;YAEnBC,WAAWrF,WAAWsF,eAAe;WACjC7B,QAAQ;YAAExB,QAAQ;YAAQsD,WAAWvF,WAAWsF,eAAe;QAAG,IAAI,CAAC,IAE7E;YAAEE,OAAO;QAAK;QAEhB,OAAOnC;IACT,EAAE,OAAOoC,OAAO;QACdC,QAAQD,KAAK,CAAC,6CAA6C;YAAEtB,OAAOrD,MAAMqD,KAAK;YAAE9D,OAAOS,MAAMT,KAAK;YAAEoF;QAAM;QAC3G,OAAO;IACT;AACF"}

package/src/lib/site-import/site-import-robots.d.ts

@@ -0,0 +1,52 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** The product token AglynBot matches robots.txt groups by. */
+export declare const AGLYN_BOT_TOKEN = "aglynbot";
+export interface RobotsRule {
+    allow: boolean;
+    /** The path pattern as written, `*` and `$` included. */
+    pattern: string;
+}
+export interface RobotsGroup {
+    /** Lowercased product tokens, version suffixes removed. */
+    agents: string[];
+    rules: RobotsRule[];
+}
+export interface RobotsTxt {
+    groups: RobotsGroup[];
+    /** Every `Sitemap:` address, as written. */
+    sitemaps: string[];
+}
+/** robots.txt as groups and sitemaps; unknown lines are skipped. */
+export declare function parseRobotsTxt(text: string): RobotsTxt;
+/**
+ * The rules that apply to `token`: every group naming it, merged; when none
+ * does, every `*` group, merged; when neither exists, none (all allowed).
+ */
+export declare function robotsRulesFor(robots: RobotsTxt, token?: string): RobotsRule[];
+/**
+ * Whether `pathWithQuery` (`/a/b?c=d`) may be crawled under `rules`.
+ * `/robots.txt` itself is always allowed.
+ */
+export declare function robotsAllows(rules: readonly RobotsRule[], pathWithQuery: string): boolean;
+export interface SitemapEntries {
+    /** True for a `<sitemapindex>`: `locations` are sitemaps, not pages. */
+    index: boolean;
+    locations: string[];
+}
+/** The `<loc>` addresses of a sitemap or a sitemap index, entities decoded. */
+export declare function parseSitemap(xml: string, limit?: number): SitemapEntries;

package/src/lib/site-import/site-import-robots.js

@@ -0,0 +1,126 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ /*==========================================
+ * ROBOTS.TXT AND SITEMAPS, FOR AGLYNBOT (AGL-3974).
+ *
+ * The site-import crawler asks a site's robots.txt before every page, as
+ * RFC 9309 describes it: the group naming `AglynBot` if there is one, else
+ * the `*` group; the longest matching rule decides, and `Allow` wins a tie;
+ * `*` matches any run and a trailing `$` anchors the end. A site that keeps
+ * a path from crawlers keeps it from this one, even though the person asking
+ * owns the site: the file is the site's own statement of what a robot may
+ * read, and the person can change it.
+ *
+ * Sitemaps are read for their `<loc>` addresses, and a sitemap index for the
+ * sitemaps it lists. Gzipped sitemaps are not read.
+ *=========================================*/ /** The product token AglynBot matches robots.txt groups by. */ export const AGLYN_BOT_TOKEN = 'aglynbot';
+/** robots.txt as groups and sitemaps; unknown lines are skipped. */ export function parseRobotsTxt(text) {
+    const groups = [];
+    const sitemaps = [];
+    let group = null;
+    let lastWasAgent = false;
+    for (const rawLine of String(text != null ? text : '').split(/\r\n|\r|\n/)){
+        const line = rawLine.replace(/#.*$/, '').trim();
+        if (!line) continue;
+        const colon = line.indexOf(':');
+        if (colon <= 0) continue;
+        const key = line.slice(0, colon).trim().toLowerCase();
+        const value = line.slice(colon + 1).trim();
+        if (key === 'sitemap') {
+            if (value) sitemaps.push(value);
+            continue;
+        }
+        if (key === 'user-agent') {
+            const agent = value.toLowerCase().split('/')[0].trim();
+            if (!group || !lastWasAgent) {
+                group = {
+                    agents: [],
+                    rules: []
+                };
+                groups.push(group);
+            }
+            if (agent) group.agents.push(agent);
+            lastWasAgent = true;
+            continue;
+        }
+        lastWasAgent = false;
+        if (!group) continue;
+        if (key === 'allow' || key === 'disallow') {
+            // An empty Disallow allows everything, which is what no rule does.
+            if (!value) continue;
+            group.rules.push({
+                allow: key === 'allow',
+                pattern: value
+            });
+        }
+    }
+    return {
+        groups,
+        sitemaps
+    };
+}
+/**
+ * The rules that apply to `token`: every group naming it, merged; when none
+ * does, every `*` group, merged; when neither exists, none (all allowed).
+ */ export function robotsRulesFor(robots, token = AGLYN_BOT_TOKEN) {
+    const own = robots.groups.filter((group)=>group.agents.includes(token));
+    const chosen = own.length ? own : robots.groups.filter((group)=>group.agents.includes('*'));
+    return chosen.flatMap((group)=>group.rules);
+}
+function patternMatchLength(pattern, path) {
+    const anchored = pattern.endsWith('$');
+    const body = anchored ? pattern.slice(0, -1) : pattern;
+    const source = body.split('*').map((part)=>part.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*');
+    const regex = new RegExp(`^${source}${anchored ? '$' : ''}`);
+    return regex.test(path) ? body.length : -1;
+}
+/** A path with a percent-encoding case that compares the way robots.txt writes it. */ function comparablePath(path) {
+    return path.replace(/%[0-9a-f]{2}/gi, (escape)=>escape.toUpperCase());
+}
+/**
+ * Whether `pathWithQuery` (`/a/b?c=d`) may be crawled under `rules`.
+ * `/robots.txt` itself is always allowed.
+ */ export function robotsAllows(rules, pathWithQuery) {
+    const path = comparablePath(pathWithQuery || '/');
+    if (path === '/robots.txt') return true;
+    let best = -1;
+    let allowed = true;
+    for (const rule of rules){
+        const length = patternMatchLength(comparablePath(rule.pattern), path);
+        if (length < 0) continue;
+        if (length > best || length === best && rule.allow) {
+            best = length;
+            allowed = rule.allow;
+        }
+    }
+    return allowed;
+}
+/** The `<loc>` addresses of a sitemap or a sitemap index, entities decoded. */ export function parseSitemap(xml, limit = 5000) {
+    const source = String(xml != null ? xml : '');
+    const index = /<sitemapindex\b/i.test(source);
+    const locations = [];
+    for (const match of source.matchAll(/<loc\b[^>]*>([\s\S]*?)<\/loc\s*>/gi)){
+        const value = match[1].replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&apos;/g, "'").trim();
+        if (value) locations.push(value);
+        if (locations.length >= limit) break;
+    }
+    return {
+        index,
+        locations
+    };
+}
+
+//# sourceMappingURL=site-import-robots.js.map

package/src/lib/site-import/site-import-robots.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../../libs/plugins/ai/src/lib/site-import/site-import-robots.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*==========================================\n * ROBOTS.TXT AND SITEMAPS, FOR AGLYNBOT (AGL-3974).\n *\n * The site-import crawler asks a site's robots.txt before every page, as\n * RFC 9309 describes it: the group naming `AglynBot` if there is one, else\n * the `*` group; the longest matching rule decides, and `Allow` wins a tie;\n * `*` matches any run and a trailing `$` anchors the end. A site that keeps\n * a path from crawlers keeps it from this one, even though the person asking\n * owns the site: the file is the site's own statement of what a robot may\n * read, and the person can change it.\n *\n * Sitemaps are read for their `<loc>` addresses, and a sitemap index for the\n * sitemaps it lists. Gzipped sitemaps are not read.\n *=========================================*/\n\n/** The product token AglynBot matches robots.txt groups by. */\nexport const AGLYN_BOT_TOKEN = 'aglynbot'\n\nexport interface RobotsRule {\n  allow: boolean\n  /** The path pattern as written, `*` and `$` included. */\n  pattern: string\n}\n\nexport interface RobotsGroup {\n  /** Lowercased product tokens, version suffixes removed. */\n  agents: string[]\n  rules: RobotsRule[]\n}\n\nexport interface RobotsTxt {\n  groups: RobotsGroup[]\n  /** Every `Sitemap:` address, as written. */\n  sitemaps: string[]\n}\n\n/** robots.txt as groups and sitemaps; unknown lines are skipped. */\nexport function parseRobotsTxt(text: string): RobotsTxt {\n  const groups: RobotsGroup[] = []\n  const sitemaps: string[] = []\n  let group: RobotsGroup | null = null\n  let lastWasAgent = false\n  for (const rawLine of String(text ?? '').split(/\\r\\n|\\r|\\n/)) {\n    const line = rawLine.replace(/#.*$/, '').trim()\n    if (!line) continue\n    const colon = line.indexOf(':')\n    if (colon <= 0) continue\n    const key = line.slice(0, colon).trim().toLowerCase()\n    const value = line.slice(colon + 1).trim()\n    if (key === 'sitemap') {\n      if (value) sitemaps.push(value)\n      continue\n    }\n    if (key === 'user-agent') {\n      const agent = value.toLowerCase().split('/')[0].trim()\n      if (!group || !lastWasAgent) {\n        group = { agents: [], rules: [] }\n        groups.push(group)\n      }\n      if (agent) group.agents.push(agent)\n      lastWasAgent = true\n      continue\n    }\n    lastWasAgent = false\n    if (!group) continue\n    if (key === 'allow' || key === 'disallow') {\n      // An empty Disallow allows everything, which is what no rule does.\n      if (!value) continue\n      group.rules.push({ allow: key === 'allow', pattern: value })\n    }\n  }\n  return { groups, sitemaps }\n}\n\n/**\n * The rules that apply to `token`: every group naming it, merged; when none\n * does, every `*` group, merged; when neither exists, none (all allowed).\n */\nexport function robotsRulesFor(robots: RobotsTxt, token: string = AGLYN_BOT_TOKEN): RobotsRule[] {\n  const own = robots.groups.filter((group) => group.agents.includes(token))\n  const chosen = own.length ? own : robots.groups.filter((group) => group.agents.includes('*'))\n  return chosen.flatMap((group) => group.rules)\n}\n\nfunction patternMatchLength(pattern: string, path: string): number {\n  const anchored = pattern.endsWith('$')\n  const body = anchored ? pattern.slice(0, -1) : pattern\n  const source = body\n    .split('*')\n    .map((part) => part.replace(/[.+?^${}()|[\\]\\\\]/g, '\\\\$&'))\n    .join('.*')\n  const regex = new RegExp(`^${source}${anchored ? '$' : ''}`)\n  return regex.test(path) ? body.length : -1\n}\n\n/** A path with a percent-encoding case that compares the way robots.txt writes it. */\nfunction comparablePath(path: string): string {\n  return path.replace(/%[0-9a-f]{2}/gi, (escape) => escape.toUpperCase())\n}\n\n/**\n * Whether `pathWithQuery` (`/a/b?c=d`) may be crawled under `rules`.\n * `/robots.txt` itself is always allowed.\n */\nexport function robotsAllows(rules: readonly RobotsRule[], pathWithQuery: string): boolean {\n  const path = comparablePath(pathWithQuery || '/')\n  if (path === '/robots.txt') return true\n  let best = -1\n  let allowed = true\n  for (const rule of rules) {\n    const length = patternMatchLength(comparablePath(rule.pattern), path)\n    if (length < 0) continue\n    if (length > best || (length === best && rule.allow)) {\n      best = length\n      allowed = rule.allow\n    }\n  }\n  return allowed\n}\n\nexport interface SitemapEntries {\n  /** True for a `<sitemapindex>`: `locations` are sitemaps, not pages. */\n  index: boolean\n  locations: string[]\n}\n\n/** The `<loc>` addresses of a sitemap or a sitemap index, entities decoded. */\nexport function parseSitemap(xml: string, limit = 5_000): SitemapEntries {\n  const source = String(xml ?? '')\n  const index = /<sitemapindex\\b/i.test(source)\n  const locations: string[] = []\n  for (const match of source.matchAll(/<loc\\b[^>]*>([\\s\\S]*?)<\\/loc\\s*>/gi)) {\n    const value = match[1]\n      .replace(/<!\\[CDATA\\[([\\s\\S]*?)\\]\\]>/g, '$1')\n      .replace(/&amp;/g, '&')\n      .replace(/&lt;/g, '<')\n      .replace(/&gt;/g, '>')\n      .replace(/&quot;/g, '\"')\n      .replace(/&apos;/g, \"'\")\n      .trim()\n    if (value) locations.push(value)\n    if (locations.length >= limit) break\n  }\n  return { index, locations }\n}\n"],"names":["AGLYN_BOT_TOKEN","parseRobotsTxt","text","groups","sitemaps","group","lastWasAgent","rawLine","String","split","line","replace","trim","colon","indexOf","key","slice","toLowerCase","value","push","agent","agents","rules","allow","pattern","robotsRulesFor","robots","token","own","filter","includes","chosen","length","flatMap","patternMatchLength","path","anchored","endsWith","body","source","map","part","join","regex","RegExp","test","comparablePath","escape","toUpperCase","robotsAllows","pathWithQuery","best","allowed","rule","parseSitemap","xml","limit","index","locations","match","matchAll"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;2CAa2C,GAE3C,6DAA6D,GAC7D,OAAO,MAAMA,kBAAkB,WAAU;AAoBzC,kEAAkE,GAClE,OAAO,SAASC,eAAeC,IAAY;IACzC,MAAMC,SAAwB,EAAE;IAChC,MAAMC,WAAqB,EAAE;IAC7B,IAAIC,QAA4B;IAChC,IAAIC,eAAe;IACnB,KAAK,MAAMC,WAAWC,OAAON,eAAAA,OAAQ,IAAIO,KAAK,CAAC,cAAe;QAC5D,MAAMC,OAAOH,QAAQI,OAAO,CAAC,QAAQ,IAAIC,IAAI;QAC7C,IAAI,CAACF,MAAM;QACX,MAAMG,QAAQH,KAAKI,OAAO,CAAC;QAC3B,IAAID,SAAS,GAAG;QAChB,MAAME,MAAML,KAAKM,KAAK,CAAC,GAAGH,OAAOD,IAAI,GAAGK,WAAW;QACnD,MAAMC,QAAQR,KAAKM,KAAK,CAACH,QAAQ,GAAGD,IAAI;QACxC,IAAIG,QAAQ,WAAW;YACrB,IAAIG,OAAOd,SAASe,IAAI,CAACD;YACzB;QACF;QACA,IAAIH,QAAQ,cAAc;YACxB,MAAMK,QAAQF,MAAMD,WAAW,GAAGR,KAAK,CAAC,IAAI,CAAC,EAAE,CAACG,IAAI;YACpD,IAAI,CAACP,SAAS,CAACC,cAAc;gBAC3BD,QAAQ;oBAAEgB,QAAQ,EAAE;oBAAEC,OAAO,EAAE;gBAAC;gBAChCnB,OAAOgB,IAAI,CAACd;YACd;YACA,IAAIe,OAAOf,MAAMgB,MAAM,CAACF,IAAI,CAACC;YAC7Bd,eAAe;YACf;QACF;QACAA,eAAe;QACf,IAAI,CAACD,OAAO;QACZ,IAAIU,QAAQ,WAAWA,QAAQ,YAAY;YACzC,mEAAmE;YACnE,IAAI,CAACG,OAAO;YACZb,MAAMiB,KAAK,CAACH,IAAI,CAAC;gBAAEI,OAAOR,QAAQ;gBAASS,SAASN;YAAM;QAC5D;IACF;IACA,OAAO;QAAEf;QAAQC;IAAS;AAC5B;AAEA;;;CAGC,GACD,OAAO,SAASqB,eAAeC,MAAiB,EAAEC,QAAgB3B,eAAe;IAC/E,MAAM4B,MAAMF,OAAOvB,MAAM,CAAC0B,MAAM,CAAC,CAACxB,QAAUA,MAAMgB,MAAM,CAACS,QAAQ,CAACH;IAClE,MAAMI,SAASH,IAAII,MAAM,GAAGJ,MAAMF,OAAOvB,MAAM,CAAC0B,MAAM,CAAC,CAACxB,QAAUA,MAAMgB,MAAM,CAACS,QAAQ,CAAC;IACxF,OAAOC,OAAOE,OAAO,CAAC,CAAC5B,QAAUA,MAAMiB,KAAK;AAC9C;AAEA,SAASY,mBAAmBV,OAAe,EAAEW,IAAY;IACvD,MAAMC,WAAWZ,QAAQa,QAAQ,CAAC;IAClC,MAAMC,OAAOF,WAAWZ,QAAQR,KAAK,CAAC,GAAG,CAAC,KAAKQ;IAC/C,MAAMe,SAASD,KACZ7B,KAAK,CAAC,KACN+B,GAAG,CAAC,CAACC,OAASA,KAAK9B,OAAO,CAAC,sBAAsB,SACjD+B,IAAI,CAAC;IACR,MAAMC,QAAQ,IAAIC,OAAO,CAAC,CAAC,EAAEL,SAASH,WAAW,MAAM,IAAI;IAC3D,OAAOO,MAAME,IAAI,CAACV,QAAQG,KAAKN,MAAM,GAAG,CAAC;AAC3C;AAEA,oFAAoF,GACpF,SAASc,eAAeX,IAAY;IAClC,OAAOA,KAAKxB,OAAO,CAAC,kBAAkB,CAACoC,SAAWA,OAAOC,WAAW;AACtE;AAEA;;;CAGC,GACD,OAAO,SAASC,aAAa3B,KAA4B,EAAE4B,aAAqB;IAC9E,MAAMf,OAAOW,eAAeI,iBAAiB;IAC7C,IAAIf,SAAS,eAAe,OAAO;IACnC,IAAIgB,OAAO,CAAC;IACZ,IAAIC,UAAU;IACd,KAAK,MAAMC,QAAQ/B,MAAO;QACxB,MAAMU,SAASE,mBAAmBY,eAAeO,KAAK7B,OAAO,GAAGW;QAChE,IAAIH,SAAS,GAAG;QAChB,IAAIA,SAASmB,QAASnB,WAAWmB,QAAQE,KAAK9B,KAAK,EAAG;YACpD4B,OAAOnB;YACPoB,UAAUC,KAAK9B,KAAK;QACtB;IACF;IACA,OAAO6B;AACT;AAQA,6EAA6E,GAC7E,OAAO,SAASE,aAAaC,GAAW,EAAEC,QAAQ,IAAK;IACrD,MAAMjB,SAAS/B,OAAO+C,cAAAA,MAAO;IAC7B,MAAME,QAAQ,mBAAmBZ,IAAI,CAACN;IACtC,MAAMmB,YAAsB,EAAE;IAC9B,KAAK,MAAMC,SAASpB,OAAOqB,QAAQ,CAAC,sCAAuC;QACzE,MAAM1C,QAAQyC,KAAK,CAAC,EAAE,CACnBhD,OAAO,CAAC,+BAA+B,MACvCA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,SAAS,KACjBA,OAAO,CAAC,SAAS,KACjBA,OAAO,CAAC,WAAW,KACnBA,OAAO,CAAC,WAAW,KACnBC,IAAI;QACP,IAAIM,OAAOwC,UAAUvC,IAAI,CAACD;QAC1B,IAAIwC,UAAU1B,MAAM,IAAIwB,OAAO;IACjC;IACA,OAAO;QAAEC;QAAOC;IAAU;AAC5B"}