@theoremjs/agents 0.0.0-stage → 0.3.0

package/esm/src/guardrails/egress-stream.js

@@ -0,0 +1,645 @@
+/**
+ * The bundled egress policy, run on a reply as it streams.
+ *
+ * The bundled checks match their patterns on the reply as written and, for
+ * injection phrasing, on
+ * rewrites of it (typo-folded, Unicode-folded, rot13, leet, URL-decoded, read
+ * backwards). The stream keeps each rewrite growing with the reply and, for
+ * every pattern, an automaton accepting every match of it and more
+ * (`egress-automata.ts`). Text is held from the earliest place any match could
+ * still be under way in any rewrite, and nowhere else: a match that completes
+ * later starts at or after that place, so none of it has been released.
+ *
+ * A match whose outcome no later text can change is settled; a settled match
+ * of the pattern itself (not the automaton) is a hit, and the reply is
+ * blocked there. Each character is read a bounded number of times, so a long
+ * reply costs time in proportion to its length.
+ *
+ * @module
+ */
+import { DEFAULT_CHECKS, standardEgressEnforce } from './egress.js';
+import { FORWARD_AUTOMATON, REVERSED_AUTOMATON, REVERSED_INJECTION_PATTERNS, } from './egress-automata.js';
+import { EGRESS_PATTERNS } from './egress-patterns.js';
+import { globalPattern } from './egress-rules.js';
+import { IMAGE_PATTERNS, imageReadings, LINK_PATTERNS, linkReadings, } from './egress-urls.js';
+import { decodeUrlRuns, INJECTION_BLOBS, TYPO_TARGETS, tryLeet, tryRot13, typoNormalize, } from './injection.js';
+import { isEmoji, normalizeCodePoint } from './normalize.js';
+import { EGRESS_RULES } from './rules.js';
+import { cardHit } from './sensitive.js';
+const UNITS = 0x10000;
+function compile(data) {
+    const classCount = data.classStarts.length;
+    const classOf = new Uint16Array(UNITS);
+    for (let k = 0; k < classCount; k++) {
+        classOf.fill(k, data.classStarts[k], data.classStarts[k + 1] ?? UNITS);
+    }
+    const has = data.charsets.map((classes) => {
+        const row = new Uint8Array(classCount);
+        for (const k of classes)
+            row[k] = 1;
+        return row;
+    });
+    const size = data.nodes.length;
+    const pattern = new Int32Array(size);
+    const final = new Uint8Array(size);
+    const edges = [];
+    data.nodes.forEach((node, id) => {
+        pattern[id] = node[0];
+        final[id] = node[1];
+        edges.push(Int32Array.from(node.slice(2)));
+    });
+    return { classOf, has, pattern, final, edges, initials: data.initials, size };
+}
+let forward;
+let backward;
+/** The first edges of the given patterns, per character class. */
+function startEdges(automaton, patterns) {
+    const classCount = automaton.has[0]?.length ?? 0;
+    const byClass = Array.from({ length: classCount }, () => []);
+    for (const p of patterns) {
+        const e = automaton.edges[automaton.initials[p]];
+        for (let i = 0; i < e.length; i += 2) {
+            const set = automaton.has[e[i + 1]];
+            for (let k = 0; k < classCount; k++)
+                if (set[k])
+                    byClass[k]?.push(e[i]);
+        }
+    }
+    return byClass.map((targets) => Int32Array.from(targets));
+}
+function grow(target, piece) {
+    target.text += piece;
+    target.fresh += piece;
+}
+/** Where `fresh` starts in `text`. */
+function freshFrom(source) {
+    return source.text.length - source.fresh.length;
+}
+function unitAt(source, at) {
+    const base = freshFrom(source);
+    return at >= base ? source.fresh.charCodeAt(at - base) : source.text.charCodeAt(at);
+}
+function sliceOf(source, from, to = source.text.length) {
+    const base = freshFrom(source);
+    return from >= base ? source.fresh.slice(from - base, to - base) : source.text.slice(from, to);
+}
+/** The reply itself. */
+function rawView(reply) {
+    const view = {
+        text: '',
+        fresh: '',
+        rawAt: (j) => j,
+        update() {
+            view.text = reply.text;
+            view.fresh = reply.fresh;
+        },
+    };
+    return view;
+}
+/** A rewrite of each UTF-16 unit on its own (rot13, leet): settled as soon as read. */
+function unitView(source, rewrite) {
+    const view = {
+        text: '',
+        fresh: '',
+        rawAt: source.rawAt,
+        update() {
+            view.fresh = '';
+            grow(view, rewrite(sliceOf(source, view.text.length)));
+        },
+    };
+    return view;
+}
+const WORD_UNIT = /\w/;
+const MAX_TARGET = Math.max(...TYPO_TARGETS.map((target) => target.length));
+const TARGET_FIRSTS = new Set(TYPO_TARGETS.map((target) => target[0]));
+/** Whether a word still being written could yet fold to a typo target. */
+function mayFold(run) {
+    return (run.length <= MAX_TARGET &&
+        /^[A-Za-z]*$/.test(run) &&
+        (run.length === 0 || TARGET_FIRSTS.has(run[0]?.toLowerCase())));
+}
+/**
+ * `typoNormalize` of the source. It rewrites whole words (`\w` runs) of the
+ * same length, so a word settles when a non-word character ends it, or as soon
+ * as it can no longer fold to a target (then the rest of it passes as is).
+ */
+function typoView(source) {
+    let read = 0;
+    /** The word being read, while it may still fold. */
+    let word;
+    let inWord = false;
+    const view = {
+        text: '',
+        fresh: '',
+        rawAt: (j) => source.rawAt(Math.min(j, view.text.length)),
+        update() {
+            view.fresh = '';
+            for (const end = source.text.length; read < end; read++) {
+                const unit = String.fromCharCode(unitAt(source, read));
+                const wordUnit = WORD_UNIT.test(unit);
+                if (wordUnit && !inWord)
+                    word = '';
+                inWord = wordUnit;
+                if (!wordUnit) {
+                    if (word !== undefined)
+                        grow(view, typoNormalize(word));
+                    word = undefined;
+                    grow(view, unit);
+                }
+                else if (word === undefined)
+                    grow(view, unit);
+                else
+                    word += unit;
+            }
+            if (word !== undefined && !mayFold(word)) {
+                grow(view, word);
+                word = undefined;
+            }
+        },
+    };
+    return view;
+}
+function mappedView(pendingAt) {
+    const view = {
+        text: '',
+        fresh: '',
+        from: [],
+        rawAt: (j) => (j < view.from.length ? view.from[j] : pendingAt()),
+        update() { },
+    };
+    return view;
+}
+function append(view, text, from) {
+    grow(view, text);
+    for (let i = 0; i < text.length; i++)
+        view.from.push(from);
+}
+const HIGH_LO = 0xd800;
+const HIGH_HI = 0xdbff;
+const ASCII_LETTER = /[A-Za-z]/;
+const LOW_LO = 0xdc00;
+const LOW_HI = 0xdfff;
+/**
+ * `normalizeForDetection` of the reply, in its three passes: each code point
+ * mapped, emoji runs between letters dropped, a backslash before a letter
+ * dropped. Unsettled: a high surrogate whose low half may follow (in the reply
+ * or in the mapped text), an emoji run after a letter until the character
+ * after it, a backslash until the character after it.
+ */
+function normalizedView(reply) {
+    let read = 0;
+    /** Mapped text the emoji pass has not read: at most a trailing high surrogate. */
+    let mapped = '';
+    let mappedFrom = [];
+    /** The emoji run after a letter, waiting on the character after it. */
+    let run = '';
+    let runFrom = [];
+    let afterLetter = false;
+    /** A backslash waiting on the character after it: its reply index, or -1. */
+    let slashFrom = -1;
+    const pendingAt = () => Math.min(read, mappedFrom[0] ?? read, runFrom[0] ?? read, slashFrom < 0 ? read : slashFrom);
+    const view = mappedView(pendingAt);
+    function backslashPass(unit, from) {
+        if (slashFrom >= 0) {
+            if (!ASCII_LETTER.test(unit))
+                append(view, '\\', slashFrom);
+            slashFrom = -1;
+        }
+        if (unit === '\\')
+            slashFrom = from;
+        else
+            append(view, unit, from);
+    }
+    function emojiPass(point, from) {
+        if (isEmoji(point) && (afterLetter || run)) {
+            run += point;
+            for (let i = 0; i < point.length; i++)
+                runFrom.push(from);
+            return;
+        }
+        if (run) {
+            if (!ASCII_LETTER.test(point)) {
+                for (let i = 0; i < run.length; i++)
+                    backslashPass(run[i], runFrom[i]);
+            }
+            run = '';
+            runFrom = [];
+        }
+        for (let i = 0; i < point.length; i++)
+            backslashPass(point[i], from);
+        afterLetter = ASCII_LETTER.test(point) && point.length === 1;
+    }
+    /** Feed the mapped text to the emoji pass by code point, keeping a high surrogate at the end. */
+    function drainMapped() {
+        let i = 0;
+        while (i < mapped.length) {
+            const code = mapped.charCodeAt(i);
+            const high = code >= HIGH_LO && code <= HIGH_HI;
+            if (high && i + 1 >= mapped.length)
+                break;
+            const low = mapped.charCodeAt(i + 1);
+            const width = high && low >= LOW_LO && low <= LOW_HI ? 2 : 1;
+            emojiPass(mapped.slice(i, i + width), mappedFrom[i]);
+            i += width;
+        }
+        mapped = mapped.slice(i);
+        mappedFrom = mappedFrom.slice(i);
+    }
+    view.update = () => {
+        view.fresh = '';
+        const end = reply.text.length;
+        while (read < end) {
+            const code = unitAt(reply, read);
+            const high = code >= HIGH_LO && code <= HIGH_HI;
+            if (high && read + 1 >= end)
+                break;
+            const low = unitAt(reply, read + 1);
+            const width = high && low >= LOW_LO && low <= LOW_HI ? 2 : 1;
+            const out = normalizeCodePoint(sliceOf(reply, read, read + width));
+            mapped += out;
+            for (let i = 0; i < out.length; i++)
+                mappedFrom.push(read);
+            read += width;
+        }
+        drainMapped();
+    };
+    return view;
+}
+const URL_HEX = /[0-9A-Fa-f]/;
+/**
+ * `decodeUrlRuns` of the reply: a run of `%XX` escapes is unsettled until the
+ * character after it shows the run is over. Its decoded text holds from the
+ * run's first `%`.
+ */
+function urlView(reply) {
+    let read = 0;
+    /** Where the escape run being read starts, or -1. */
+    let runStart = -1;
+    const view = mappedView(() => (runStart < 0 ? read : runStart));
+    view.update = () => {
+        view.fresh = '';
+        const end = reply.text.length;
+        while (read < end) {
+            const unit = String.fromCharCode(unitAt(reply, read));
+            if (runStart < 0) {
+                if (unit === '%') {
+                    runStart = read;
+                }
+                else {
+                    append(view, unit, read);
+                    read++;
+                }
+                continue;
+            }
+            // Inside a run: complete escapes end at runStart + 3k.
+            const offset = (read - runStart) % 3;
+            const fits = offset === 0 ? unit === '%' : URL_HEX.test(unit);
+            if (fits) {
+                read++;
+                continue;
+            }
+            endRun(read - offset);
+            // The broken escape's text is literal; `read` rereads from its end.
+            for (let i = read - offset; i < read; i++)
+                append(view, String.fromCharCode(unitAt(reply, i)), i);
+        }
+    };
+    function endRun(end) {
+        if (end > runStart)
+            append(view, decodeUrlRuns(sliceOf(reply, runStart, end)), runStart);
+        runStart = -1;
+    }
+    return view;
+}
+const BLOB_HITS = new Map(INJECTION_BLOBS.map(({ pattern, hit }) => [pattern, hit]));
+/** The bundled rule a pattern kind trips, with `collectEgressHits`' severity. */
+const KIND_RULES = {
+    injection: { rule: EGRESS_RULES.injection, severity: 'medium' },
+    base64: { rule: EGRESS_RULES.injection, severity: 'medium' },
+    hex: { rule: EGRESS_RULES.injection, severity: 'medium' },
+    spaced: { rule: EGRESS_RULES.injection, severity: 'medium' },
+    pipe: { rule: EGRESS_RULES.injection, severity: 'medium' },
+    sensitive: { rule: EGRESS_RULES.sensitive, severity: 'high' },
+    card: { rule: EGRESS_RULES.sensitive, severity: 'high' },
+    boundary: { rule: EGRESS_RULES.boundary, severity: 'medium' },
+    image: { rule: EGRESS_RULES.image, severity: 'high' },
+    link: { rule: EGRESS_RULES.link, severity: 'high' },
+};
+/** How a settled match of `pattern` is read: a test of the match, or a finder in place of the regex. */
+function patternReading(kind, pattern, urls) {
+    if (kind === 'card')
+        return { hit: cardHit };
+    if (kind === 'image' || kind === 'link') {
+        const index = (kind === 'image' ? IMAGE_PATTERNS : LINK_PATTERNS).indexOf(pattern);
+        const reading = urls[kind][index];
+        return 'find' in reading ? { find: reading.find } : { hit: reading.test };
+    }
+    const hit = BLOB_HITS.get(pattern);
+    return hit ? { hit } : {};
+}
+const INJECTION_KINDS = new Set([
+    'injection',
+    'base64',
+    'hex',
+    'spaced',
+    'pipe',
+]);
+/** Whether `checks` runs the check a pattern belongs to. */
+function runs({ kind, group }, checks) {
+    if (INJECTION_KINDS.has(kind))
+        return checks.injection;
+    switch (kind) {
+        case 'sensitive':
+        case 'card':
+            return group !== undefined && checks.sensitive[group];
+        case 'boundary':
+            return checks.boundary;
+        case 'image':
+            return checks.images !== undefined;
+        default:
+            return checks.links !== undefined;
+    }
+}
+/** The patterns `checks` runs on the reply as written. */
+function forwardPatterns(checks, given) {
+    const scope = (check) => ({ ...check, ...(given ? { given } : {}) });
+    const urls = {
+        image: imageReadings(scope(checks.images)),
+        link: linkReadings(scope(checks.links), checks.images !== undefined),
+    };
+    const out = [];
+    EGRESS_PATTERNS.forEach((entry, id) => {
+        if (!runs(entry, checks))
+            return;
+        out.push({
+            ...KIND_RULES[entry.kind],
+            id,
+            regex: new RegExp(entry.pattern.source, entry.pattern.flags),
+            ...patternReading(entry.kind, entry.pattern, urls),
+        });
+    });
+    return out;
+}
+/** The injection patterns, for the rewritten views. */
+function injectionPatterns() {
+    const out = [];
+    EGRESS_PATTERNS.forEach(({ kind, pattern }, id) => {
+        if (kind !== 'injection')
+            return;
+        const hit = BLOB_HITS.get(pattern);
+        out.push({
+            ...KIND_RULES.injection,
+            id,
+            regex: new RegExp(pattern.source, pattern.flags),
+            ...(hit ? { hit } : {}),
+        });
+    });
+    return out;
+}
+function reversedPatterns() {
+    return REVERSED_INJECTION_PATTERNS.map((pattern, id) => ({
+        ...KIND_RULES.injection,
+        id,
+        regex: new RegExp(pattern.source, pattern.flags),
+    }));
+}
+/**
+ * One view scanned for some patterns: the automata, run one character at a
+ * time, give where each pattern could still be matching; the pattern's own
+ * regex, run only where the automaton reached a match, finds the match and
+ * whether it has settled. A pattern whose matches nest (tags inside tags) is
+ * read instead by its finder, once over each stretch that settles.
+ */
+function createScan(view, automaton, patterns) {
+    const starts = startEdges(automaton, patterns.map((p) => p.id));
+    const local = new Int32Array(automaton.initials.length).fill(-1);
+    patterns.forEach((p, i) => {
+        local[p.id] = i;
+    });
+    let nodes = new Int32Array(automaton.size);
+    let from = new Int32Array(automaton.size);
+    let next = new Int32Array(automaton.size);
+    let nextFrom = new Int32Array(automaton.size);
+    let count = 0;
+    const slot = new Int32Array(automaton.size);
+    const mark = new Int32Array(automaton.size);
+    let step = 0;
+    let fed = 0;
+    const reached = new Uint8Array(patterns.length);
+    /** Whether the regex must look again: a match of it was unsettled, or its automaton sat on a final state. */
+    const recheck = new Uint8Array(patterns.length);
+    const resume = new Int32Array(patterns.length);
+    const liveFrom = new Int32Array(patterns.length);
+    const atFinal = new Uint8Array(patterns.length);
+    const finds = Uint8Array.from(patterns, (p) => (p.find ? 1 : 0));
+    /**
+     * Per finder: where its automaton first started a thread since the finder
+     * last read, or -1. A construct starts where a thread does, so a stretch
+     * with none needs no read; skipping it keeps a reply with no constructs from
+     * being reread whole at every step.
+     */
+    const openedFrom = new Int32Array(patterns.length).fill(-1);
+    const lastOpened = new Int32Array(patterns.length).fill(-1);
+    function feed() {
+        const { text, fresh } = view;
+        const base = freshFrom(view);
+        const { classOf, has, edges, final, pattern } = automaton;
+        for (; fed < text.length; fed++) {
+            const code = fed >= base ? fresh.charCodeAt(fed - base) : text.charCodeAt(fed);
+            const cls = classOf[code];
+            step++;
+            let size = 0;
+            const add = (target, start) => {
+                if (mark[target] === step) {
+                    const at = slot[target];
+                    if (start < nextFrom[at])
+                        nextFrom[at] = start;
+                    return;
+                }
+                mark[target] = step;
+                slot[target] = size;
+                next[size] = target;
+                nextFrom[size] = start;
+                size++;
+                if (final[target])
+                    reached[local[pattern[target]]] = 1;
+            };
+            for (let i = 0; i < count; i++) {
+                const e = edges[nodes[i]];
+                const start = from[i];
+                for (let k = 0; k < e.length; k += 2) {
+                    if (has[e[k + 1]][cls])
+                        add(e[k], start);
+                }
+            }
+            const first = starts[cls];
+            for (let k = 0; k < first.length; k++) {
+                const target = first[k];
+                add(target, fed);
+                const p = local[pattern[target]];
+                if (finds[p]) {
+                    lastOpened[p] = fed;
+                    if (openedFrom[p] < 0)
+                        openedFrom[p] = fed;
+                }
+            }
+            [nodes, next] = [next, nodes];
+            [from, nextFrom] = [nextFrom, from];
+            count = size;
+        }
+    }
+    /** Per pattern: the earliest start still under way, and whether it sits on a final state. */
+    function live() {
+        const end = view.text.length;
+        liveFrom.fill(end);
+        atFinal.fill(0);
+        let earliest = end;
+        for (let i = 0; i < count; i++) {
+            const node = nodes[i];
+            const p = local[automaton.pattern[node]];
+            const start = from[i];
+            if (start < liveFrom[p])
+                liveFrom[p] = start;
+            if (automaton.final[node])
+                atFinal[p] = 1;
+            if (start < earliest)
+                earliest = start;
+        }
+        return earliest;
+    }
+    /** A finder's hit up to `settledTo`, run only on a stretch its automaton opened a thread in. */
+    function found(p, settledTo) {
+        const opened = openedFrom[p];
+        const from = resume[p];
+        resume[p] = settledTo;
+        if (opened < 0 || opened >= settledTo)
+            return undefined;
+        openedFrom[p] = lastOpened[p] >= settledTo ? settledTo : -1;
+        const { find, rule, severity } = patterns[p];
+        const at = find?.(view.text, from, settledTo);
+        return at === undefined ? undefined : { rule, severity, start: view.rawAt(at) };
+    }
+    /** A regex pattern's hit up to `settledTo`. */
+    function matched(p, settledTo) {
+        if (!reached[p] && !recheck[p]) {
+            resume[p] = settledTo;
+            return undefined;
+        }
+        reached[p] = 0;
+        const text = view.text;
+        const { regex, hit, rule, severity } = patterns[p];
+        regex.lastIndex = resume[p];
+        for (;;) {
+            const match = regex.exec(text);
+            if (!match || match.index >= settledTo) {
+                resume[p] = settledTo;
+                recheck[p] = match || atFinal[p] ? 1 : 0;
+                return undefined;
+            }
+            const [blob] = match;
+            if (blob && (!hit || hit(blob, text, match.index))) {
+                return { rule, severity, start: view.rawAt(match.index) };
+            }
+            if (!blob)
+                regex.lastIndex++;
+            resume[p] = regex.lastIndex;
+        }
+    }
+    function detect() {
+        for (let p = 0; p < patterns.length; p++) {
+            const settledTo = liveFrom[p];
+            if (settledTo <= resume[p])
+                continue;
+            const hit = patterns[p].find ? found(p, settledTo) : matched(p, settledTo);
+            if (hit)
+                return hit;
+        }
+        return undefined;
+    }
+    return {
+        /** Read what the view settled; a hit, or the reply index to hold from. */
+        run() {
+            feed();
+            const earliest = live();
+            return detect() ?? view.rawAt(earliest);
+        },
+    };
+}
+/** Compiled host automata, kept per table so each is built once. */
+const hostAutomata = new WeakMap();
+function hostPatterns({ rules }) {
+    return rules.map(({ rule, severity, pattern }, id) => ({
+        rule,
+        severity,
+        id,
+        regex: globalPattern(pattern),
+    }));
+}
+/** Scans the bundled egress policy's patterns, and any host rules, as a reply streams. */
+function createEgressStream(options = {}) {
+    const reply = { text: '', fresh: '' };
+    const raw = rawView(reply);
+    const views = [raw];
+    const scan = (view, automaton, patterns) => {
+        if (!views.includes(view))
+            views.push(view);
+        return createScan(view, automaton, patterns);
+    };
+    const scans = [];
+    const checks = options.checks ?? DEFAULT_CHECKS;
+    const forwardScan = forwardPatterns(checks, options.given);
+    if (forwardScan.length > 0) {
+        forward ??= compile(FORWARD_AUTOMATON);
+        scans.push(scan(raw, forward, forwardScan));
+    }
+    if (checks.injection) {
+        forward ??= compile(FORWARD_AUTOMATON);
+        backward ??= compile(REVERSED_AUTOMATON);
+        const normalized = normalizedView(reply);
+        const injection = injectionPatterns();
+        scans.push(scan(raw, backward, reversedPatterns()), scan(typoView(raw), forward, injection), scan(normalized, forward, injection), scan(typoView(normalized), forward, injection), scan(unitView(raw, tryRot13), forward, injection), scan(unitView(raw, (text) => tryLeet(text) ?? text), forward, injection), scan(urlView(reply), forward, injection));
+    }
+    if (options.host) {
+        let automaton = hostAutomata.get(options.host.automaton);
+        if (!automaton) {
+            automaton = compile(options.host.automaton);
+            hostAutomata.set(options.host.automaton, automaton);
+        }
+        scans.push(scan(raw, automaton, hostPatterns(options.host)));
+    }
+    let hold = 0;
+    return {
+        push(chunk) {
+            reply.text += chunk;
+            reply.fresh = chunk;
+            for (const view of views)
+                view.update();
+            let from = reply.text.length;
+            for (const s of scans) {
+                const result = s.run();
+                if (typeof result !== 'number')
+                    return result;
+                from = Math.min(from, result);
+            }
+            hold = from;
+            return undefined;
+        },
+        holdFrom: () => hold,
+    };
+}
+/** Streaming plans for the policies whose checks the stream runs itself. */
+const STREAM_PLANS = new WeakMap([
+    [
+        standardEgressEnforce,
+        (context) => createEgressStream(context.givenUrls ? { given: context.givenUrls } : {}),
+    ],
+]);
+/** The streaming plan for `enforce`, when the stream knows its checks. */
+function streamPlanOf(enforce) {
+    return STREAM_PLANS.get(enforce);
+}
+/** Tell the gate `enforce` runs exactly the checks `plan` streams. */
+function registerStreamPlan(enforce, plan) {
+    STREAM_PLANS.set(enforce, plan);
+}
+export { createEgressStream, normalizedView, registerStreamPlan, streamPlanOf, typoView, urlView };