@event4u/agent-config 17.3.0 → 17.4.0

package/src/scripts/_lib/confusables.ts

@@ -49,25 +49,6 @@ export const CONFUSABLE_FOREIGN: ReadonlySet<number> = new Set([
     0x03a1 /*Ρ→P*/, 0x03a4 /*Τ→T*/, 0x03a5 /*Υ→Y*/, 0x03a7 /*Χ→X*/, 0x0392 /*Β→B*/,
 ]);
 
-/**
- * The basic-Latin twin of each confusable foreign letter that has one, for a
- * detector that folds a lookalike back before matching. Every key is in
- * `CONFUSABLE_FOREIGN`, which a test holds.
- */
-export const LATIN_TWIN: ReadonlyMap<number, string> = new Map([
-    [0x0430, 'a'], [0x0435, 'e'], [0x043e, 'o'], [0x0440, 'p'], [0x0441, 'c'],
-    [0x0443, 'y'], [0x0445, 'x'], [0x0455, 's'], [0x0456, 'i'], [0x0458, 'j'],
-    [0x04bb, 'h'], [0x0501, 'd'], [0x051b, 'q'],
-    [0x0410, 'A'], [0x0412, 'B'], [0x0415, 'E'], [0x041a, 'K'], [0x041c, 'M'],
-    [0x041d, 'H'], [0x041e, 'O'], [0x0420, 'P'], [0x0421, 'C'], [0x0422, 'T'],
-    [0x0423, 'Y'], [0x0425, 'X'], [0x0405, 'S'], [0x0406, 'I'], [0x0408, 'J'],
-    [0x03b1, 'a'], [0x03b5, 'e'], [0x03bf, 'o'], [0x03bd, 'v'], [0x03c1, 'p'],
-    [0x03c5, 'u'], [0x03c7, 'x'], [0x03ba, 'k'],
-    [0x0391, 'A'], [0x0392, 'B'], [0x0395, 'E'], [0x0396, 'Z'], [0x0397, 'H'],
-    [0x0399, 'I'], [0x039a, 'K'], [0x039c, 'M'], [0x039d, 'N'], [0x039f, 'O'],
-    [0x03a1, 'P'], [0x03a4, 'T'], [0x03a5, 'Y'], [0x03a7, 'X'],
-]);
-
 type ScriptName = 'latin' | 'cyrillic' | 'greek' | 'other';
 
 function _script_of(ch: string): ScriptName {

package/src/scripts/_lib/egress_normalize.ts

@@ -8,9 +8,20 @@
  * back to the original offsets, so a replacement removes the disguised bytes
  * too. Pure: no I/O.
  */
-import { LATIN_TWIN } from './confusables.js';
 import { detect, EGRESS_RULES, type EgressRule, type EgressSpan } from './egress_rules.js';
 
+// Aligned: the n-th code point of FOREIGN, three hex digits each, folds to the
+// n-th letter of LATIN. Kept here, not in confusables.ts, so the engine chunk
+// does not carry that module's script regexes.
+const FOREIGN = '43043543e4404414434454554564584bb50151b41041241541a41c41d41e4204214224234254054064083b13b53bf3bd3c13c53c73ba39139239539639739939a39c39d39f3a13a43a53a7';
+const LATIN = 'aeopcyxsijhdqABEKMHOPCTYXSIJaeovpuxkABEZHIKMNOPTYX';
+
+/**
+ * The basic-Latin twin of each confusable foreign letter that has one. Every
+ * key is in `CONFUSABLE_FOREIGN` (confusables.ts), which a test holds.
+ */
+export const LATIN_TWIN: ReadonlyMap<number, string> = new Map([...LATIN].map((c, i) => [parseInt(FOREIGN.slice(i * 3, i * 3 + 3), 16), c]));
+
 /**
  * Invisible and bidirectional-control characters: soft hyphen, zero-width
  * marks, bidi embeddings and isolates, word joiners, the BOM and the Tag block.
@@ -19,17 +30,15 @@ export const HIDDEN_UNICODE_RE = /[­​-‏‪-‮⁠-⁤⁦-⁩\u{E0000}-\u
 
 export interface Normalized {
     readonly text: string;
-    /** For each UTF-16 unit of `text`, the original offsets of the character it came from. */
-    readonly originStart: readonly number[];
-    readonly originEnd: readonly number[];
+    /** For each UTF-16 unit of `text`, the original offset of the character it came from. */
+    readonly origin: readonly number[];
     readonly hidden: number;
     readonly folded: number;
 }
 
 export function normalizeForDetection(text: string): Normalized {
-    const out: string[] = [];
-    const originStart: number[] = [];
-    const originEnd: number[] = [];
+    let out = '';
+    const origin: number[] = [];
     let hidden = 0;
     let folded = 0;
     let at = 0;
@@ -38,30 +47,51 @@ export function normalizeForDetection(text: string): Normalized {
         if (HIDDEN_UNICODE_RE.test(ch)) {
             hidden += 1;
         } else {
-            const twin = LATIN_TWIN.get(ch.codePointAt(0) as number);
-            if (twin !== undefined) folded += 1;
-            const emit = twin ?? ch;
-            for (let k = 0; k < emit.length; k += 1) {
-                originStart.push(at);
-                originEnd.push(at + ch.length);
-            }
-            out.push(emit);
+            const emit = LATIN_TWIN.get(ch.codePointAt(0) as number) ?? ch;
+            if (emit !== ch) folded += 1;
+            out += emit;
+            for (let k = 0; k < emit.length; k += 1) origin.push(at);
         }
         at += ch.length;
     }
-    return { text: out.join(''), originStart, originEnd, hidden, folded };
+    return { text: out, origin, hidden, folded };
+}
+
+export interface NormalizedDetection {
+    readonly spans: EgressSpan[];
+    /**
+     * An extent reader such as a rule's `through`, applied to the text the spans
+     * were found in, with its end in original offsets. A disguised END marker is
+     * then seen by the reader exactly as the disguised BEGIN was by the rule.
+     */
+    end<S extends EgressSpan>(span: S, read: (text: string, span: S) => number | null): number | null;
 }
 
 /**
  * `detect`, run on the normalised text with spans in original offsets. With no
  * invisible or confusable character the result is `detect(text)` exactly.
  */
-export function detectNormalized(text: string, rules: readonly EgressRule[] = EGRESS_RULES): EgressSpan[] {
+export function detectWithExtent(text: string, rules: readonly EgressRule[] = EGRESS_RULES): NormalizedDetection {
     const n = normalizeForDetection(text);
-    if (n.hidden === 0 && n.folded === 0) return detect(text, rules);
-    return detect(n.text, rules).map((s) => ({
-        ...s,
-        start: n.originStart[s.start] as number,
-        end: n.originEnd[s.end - 1] as number,
-    }));
+    if (n.hidden === 0 && n.folded === 0) return { spans: detect(text, rules), end: (s, read) => read(text, s) };
+    const back = (i: number): number => {
+        const o = n.origin[i - 1] as number;
+        return o + ((text.codePointAt(o) as number) > 0xffff ? 2 : 1);
+    };
+    const fwd = (o: number): number => {
+        let i = 0;
+        while (i < n.origin.length && (n.origin[i] as number) < o) i += 1;
+        return i;
+    };
+    return {
+        spans: detect(n.text, rules).map((s) => ({ ...s, start: n.origin[s.start] as number, end: back(s.end) })),
+        end: (s, read) => {
+            const e = read(n.text, { ...s, start: fwd(s.start), end: fwd(s.end) });
+            return e === null ? null : back(e);
+        },
+    };
+}
+
+export function detectNormalized(text: string, rules: readonly EgressRule[] = EGRESS_RULES): EgressSpan[] {
+    return detectWithExtent(text, rules).spans;
 }

package/src/scripts/_lib/egress_text.ts

@@ -3,7 +3,8 @@
 // (`egress_engine.ts`, D9), so the stage keeps the field walk and the chunk
 // keeps everything that reads a rule.
 
-import { detect, EGRESS_RULES, type EgressClass } from './egress_rules.js';
+import { detectWithExtent } from './egress_normalize.js';
+import { EGRESS_RULES, type EgressClass } from './egress_rules.js';
 import { EgressRefusedError, replaceValues, type TokenScope } from './egress_token.js';
 
 const RULE_BY_ID = new Map(EGRESS_RULES.map((r) => [r.rule, r]));
@@ -19,10 +20,11 @@ export interface TextTally {
 export function tallyText(text: string): TextTally {
     const found: Array<{ class: EgressClass; type: string }> = [];
     const refused: string[] = [];
-    for (const span of detect(text)) {
+    const at = detectWithExtent(text);
+    for (const span of at.spans) {
         const policy = RULE_BY_ID.get(span.rule)?.replacement;
         if (policy !== undefined && policy !== 'detection') {
-            const end = policy.through(text, span);
+            const end = at.end(span, policy.through);
             if (end === null || end < span.end) {
                 refused.push(span.rule);
                 continue;

package/src/scripts/_lib/egress_token.ts

@@ -5,7 +5,8 @@
 
 import { createHmac, randomBytes } from 'node:crypto';
 
-import { detect, EGRESS_RULES, type EgressClass, type EgressSpan, replaceSpans } from './egress_rules.js';
+import { detectWithExtent, type NormalizedDetection } from './egress_normalize.js';
+import { EGRESS_RULES, type EgressClass, type EgressSpan, replaceSpans } from './egress_rules.js';
 
 export const TOKEN_FP_LENGTH = 6;
 
@@ -209,12 +210,12 @@ const RULE_BY_ID = new Map(EGRESS_RULES.map((r) => [r.rule, r]));
 const CLASS_RANK: Readonly<Record<EgressClass, number>> = { credential: 1, identifier: 0 };
 
 /** The bytes each detection obliges a replacement to remove; overlapping extents merge. */
-function replacementSpans(text: string, detected: readonly EgressSpan[]): EgressSpan[] {
+function replacementSpans(detected: readonly EgressSpan[], at: NormalizedDetection): EgressSpan[] {
     const extents = detected.map((s) => {
         const policy = RULE_BY_ID.get(s.rule)?.replacement;
         if (policy === undefined) throw new EgressRefusedError(s.rule);
         if (policy === 'detection') return s;
-        const end = policy.through(text, s);
+        const end = at.end(s, policy.through);
         if (end === null || end < s.end) throw new EgressRefusedError(s.rule);
         return { ...s, end };
     });
@@ -243,8 +244,9 @@ function replacementSpans(text: string, detected: readonly EgressSpan[]): Egress
  */
 export function replaceValues(text: string, scope: TokenScope): string {
     const owned = parseTokens(text).filter((t) => scope.owns(text.slice(t.start, t.end)));
-    const detected = detect(text).filter((s) => !owned.some((t) => t.start <= s.start && s.end <= t.end));
-    const spans = replacementSpans(text, detected);
+    const at = detectWithExtent(text);
+    const detected = at.spans.filter((s) => !owned.some((t) => t.start <= s.start && s.end <= t.end));
+    const spans = replacementSpans(detected, at);
     // Mint in text order first: replaceSpans substitutes right to left, and minting
     // inside its callback would number the last value 1.
     const tokens = spans.map((s) => scope.mint(s.type, s.class, text.slice(s.start, s.end)));