@abhishekvardanbotta/mask 0.0.0-stage → 0.2.0

package/dist/privacy.js

@@ -0,0 +1,600 @@
+import { deriveKeys, fingerprint, normalizeMasterKey, open, seal } from "./crypto.js";
+import { runRecognizer } from "./detectors/recognizers.js";
+import { asNode, inferFieldType, itemSchema, matchFieldRule, nodeAnnotation, propertySchema, typeFromKey } from "./fields.js";
+import { compilePolicy, deniedReplacement, isGranted, mergePolicyConfig } from "./policy.js";
+import { PrivacyError, } from "./types.js";
+import { InMemoryVaultStore } from "./vault.js";
+import { randomToken, renderToken, TOKEN_MARKER, TOKEN_REFERENCE } from "./tokens.js";
+const TOKEN_ID_LENGTH = 10;
+/** Strict `[TYPE_id]` plus a bare `TYPE_id` (models sometimes drop the brackets); only tokens in this scope's vault are ever resolved. */
+const MAX_AUDIT_EVENTS = 5000;
+const MAX_MEMO_ENTRIES = 2000;
+/** Everything one scope knows: shared by every policy view of that scope. */
+class ScopeState {
+    scope;
+    byToken = new Map();
+    byFingerprint = new Map();
+    /** Exact plaintexts already tokenized in this scope -> token, so later free-text mentions are caught too. */
+    known = new Map();
+    knownVersion = 0;
+    /** Last sequential id issued per type. */
+    sequence = new Map();
+    knownRegexCache;
+    pending = [];
+    audit = [];
+    blocked = 0;
+    restored = 0;
+    denied = 0;
+    lastUsed = Date.now();
+    constructor(scope) {
+        this.scope = scope;
+    }
+    record(event) {
+        if (this.audit.length >= MAX_AUDIT_EVENTS)
+            this.audit.shift();
+        this.audit.push({ at: new Date().toISOString(), ...event });
+    }
+    /** Two alternations over the known plaintexts: case-sensitive, and case-folded for emails. */
+    knownRegexes() {
+        if (this.knownRegexCache?.version === this.knownVersion)
+            return this.knownRegexCache;
+        const exact = [];
+        const folded = [];
+        const foldedMap = new Map();
+        for (const [value, token] of this.known) {
+            if (this.byToken.get(token)?.type === "EMAIL") {
+                folded.push(value);
+                foldedMap.set(value.toLowerCase(), token);
+            }
+            else
+                exact.push(value);
+        }
+        const build = (values, flags) => values.length
+            ? new RegExp(`(?<![\\p{L}\\p{N}_])(?:${values.sort((a, b) => b.length - a.length).map(escapeRegex).join("|")})(?![\\p{L}\\p{N}_])`, flags)
+            : undefined;
+        this.knownRegexCache = { version: this.knownVersion, exact: build(exact, "gu"), folded: build(folded, "giu"), foldedMap };
+        return this.knownRegexCache;
+    }
+}
+function escapeRegex(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function normalizeForType(type, value) {
+    const trimmed = value.trim();
+    switch (type) {
+        case "EMAIL": return trimmed.toLowerCase();
+        case "PHONE": return (trimmed.startsWith("+") ? "+" : "") + trimmed.replace(/\D/g, "");
+        case "CREDIT_CARD":
+        case "US_SSN":
+        case "IN_AADHAAR":
+        case "CARD_SECURITY_CODE": return trimmed.replace(/\D/g, "");
+        case "IBAN":
+        case "IN_PAN": return trimmed.replace(/\s+/g, "").toUpperCase();
+        default: return trimmed.replace(/\s+/g, " ").toLowerCase();
+    }
+}
+/** Plaintexts shorter than this are not used for free-text matching: "Al" or "42" would match everywhere. */
+function knownEligible(value) {
+    if (/^[\d\s().+-]+$/.test(value))
+        return value.replace(/\D/g, "").length >= 4;
+    return value.trim().length >= 3;
+}
+function isPlainObject(value) {
+    if (!value || typeof value !== "object")
+        return false;
+    const proto = Object.getPrototypeOf(value);
+    return proto === Object.prototype || proto === null;
+}
+/**
+ * The privacy engine: holds the keys, the vault store and the per-scope state. Create one per process (or per
+ * tenant, if tenants need separate keys) and open a session per conversation, run or request.
+ */
+export class Privacy {
+    keys;
+    store;
+    retentionMs;
+    tokenStyle;
+    baseConfig;
+    basePolicy;
+    scopes = new Map();
+    hydrating = new Map();
+    constructor(options = {}) {
+        const { key, store, retentionMs, tokenStyle, ...policy } = options;
+        this.keys = deriveKeys(normalizeMasterKey(key));
+        this.store = store ?? new InMemoryVaultStore();
+        this.retentionMs = retentionMs ?? 60 * 60 * 1000;
+        if (tokenStyle !== undefined && tokenStyle !== "random" && tokenStyle !== "sequential" && tokenStyle !== "opaque")
+            throw new PrivacyError(`mask: tokenStyle must be "random", "sequential" or "opaque".`, "config");
+        this.tokenStyle = tokenStyle ?? "random";
+        this.baseConfig = policy;
+        this.basePolicy = compilePolicy(policy);
+    }
+    /** Compiles a policy layered over this engine's own. Compile once and reuse it across sessions. */
+    policy(overrides) {
+        return overrides ? compilePolicy(mergePolicyConfig(this.baseConfig, overrides)) : this.basePolicy;
+    }
+    /**
+     * A session over `scope`, without loading previously persisted records. Tokens are deterministic within a scope
+     * and unlinkable across scopes. Use `open()` instead when the scope may have records from an earlier process.
+     */
+    session(scope, policy) {
+        this.sweep();
+        let state = this.scopes.get(scope);
+        if (!state) {
+            state = new ScopeState(scope);
+            this.scopes.set(scope, state);
+        }
+        return new PrivacySession(this, state, this.resolvePolicy(policy));
+    }
+    /** Like `session()`, but first loads and decrypts the scope's persisted records from the store. */
+    async open(scope, policy) {
+        this.sweep();
+        let state = this.scopes.get(scope);
+        if (!state) {
+            let loading = this.hydrating.get(scope);
+            if (!loading) {
+                loading = this.hydrate(scope).finally(() => this.hydrating.delete(scope));
+                this.hydrating.set(scope, loading);
+            }
+            state = await loading;
+        }
+        return new PrivacySession(this, state, this.resolvePolicy(policy));
+    }
+    /** Forgets a scope: drops its in-memory state and deletes its records from the store. Its tokens become unrestorable. */
+    async destroy(scope) {
+        const state = this.scopes.get(scope);
+        if (state)
+            await Promise.allSettled(state.pending);
+        this.scopes.delete(scope);
+        await this.store.deleteScope(scope);
+    }
+    /** Runs detection only — what would be found in `text`, with confidences — without tokenizing anything. */
+    detect(text, policy) {
+        return detectSync(text, this.resolvePolicy(policy), []).map(({ type, start, end, score }) => ({ type, start, end, score }));
+    }
+    /** @internal */
+    tokenFor(state, type, value, classification, kind) {
+        const fp = fingerprint(this.keys, state.scope, type, normalizeForType(type, value));
+        const existing = state.byFingerprint.get(fp);
+        if (existing) {
+            const entry = state.byToken.get(existing);
+            // The stricter classification wins: once any field marked this value SENSITIVE, it stays SENSITIVE.
+            if (classification === "SENSITIVE" && entry.classification === "PII") {
+                entry.classification = "SENSITIVE";
+                this.persist(state, entry);
+            }
+            this.remember(state, value, existing, type);
+            return existing;
+        }
+        let token;
+        if (this.tokenStyle === "random") {
+            let attempt = 0;
+            token = randomToken(fp, attempt);
+            while (state.byToken.has(token))
+                token = randomToken(fp, ++attempt);
+        }
+        else if (this.tokenStyle === "sequential") {
+            let n = state.sequence.get(type) ?? 0;
+            do {
+                n += 1;
+                token = `${type}_${n}`;
+            } while (state.byToken.has(token));
+            state.sequence.set(type, n);
+        }
+        else {
+            let offset = 0;
+            token = `${type}_${fp.slice(0, TOKEN_ID_LENGTH)}`;
+            while (state.byToken.has(token)) {
+                offset += 1;
+                token = `${type}_${fp.slice(offset, offset + TOKEN_ID_LENGTH)}`;
+            }
+        }
+        const entry = { token, type, classification, value, kind, fp };
+        state.byToken.set(token, entry);
+        state.byFingerprint.set(fp, token);
+        state.record({ action: "tokenize", type, classification, token });
+        this.persist(state, entry);
+        this.remember(state, value, token, type);
+        if (type === "PERSON" && /\s/.test(value.trim())) {
+            // "Jane Doe" also makes "Jane" and "Doe" recognizable later on their own.
+            for (const part of value.trim().split(/\s+/))
+                if (part.length >= 3 && /^\p{Lu}/u.test(part))
+                    this.tokenFor(state, "PERSON", part, classification, "string");
+        }
+        return token;
+    }
+    remember(state, value, token, type) {
+        const key = type === "EMAIL" ? value.trim().toLowerCase() : value.trim();
+        if (!knownEligible(key) || state.known.get(key) === token)
+            return;
+        state.known.set(key, token);
+        state.knownVersion += 1;
+    }
+    persist(state, entry) {
+        const sealed = seal(this.keys, JSON.stringify({ v: entry.value }), `${state.scope}\u0000${entry.token}`);
+        const now = Date.now();
+        const record = { token: entry.token, type: entry.type, classification: entry.classification, kind: entry.kind, fp: entry.fp, createdAt: now, expiresAt: now + this.retentionMs, ...sealed };
+        try {
+            const result = this.store.put(state.scope, record);
+            if (result)
+                state.pending.push(result.catch((error) => { throw new PrivacyError(`mask: vault write failed: ${error?.message ?? error}`, "vault"); }));
+        }
+        catch (error) {
+            state.pending.push(Promise.reject(new PrivacyError(`mask: vault write failed: ${error?.message ?? error}`, "vault")));
+        }
+    }
+    async hydrate(scope) {
+        const state = new ScopeState(scope);
+        for (const record of await this.store.list(scope)) {
+            const { v } = JSON.parse(open(this.keys, record, `${scope}\u0000${record.token}`));
+            const entry = { token: record.token, type: record.type, classification: record.classification, value: v, kind: record.kind, fp: record.fp };
+            state.byToken.set(entry.token, entry);
+            state.byFingerprint.set(entry.fp, entry.token);
+            const seq = /_([1-9]\d{0,5})$/.exec(entry.token);
+            if (seq && entry.token === `${entry.type}_${seq[1]}`)
+                state.sequence.set(entry.type, Math.max(state.sequence.get(entry.type) ?? 0, Number(seq[1])));
+            this.remember(state, v, entry.token, entry.type);
+        }
+        this.scopes.set(scope, state);
+        return state;
+    }
+    resolvePolicy(policy) {
+        if (!policy)
+            return this.basePolicy;
+        return typeof policy.classify === "function" ? policy : this.policy(policy);
+    }
+    /** Drops in-memory scopes idle for longer than the retention period. Their store records expire on the same clock. */
+    sweep() {
+        const cutoff = Date.now() - this.retentionMs;
+        for (const [scope, state] of this.scopes)
+            if (state.lastUsed < cutoff)
+                this.scopes.delete(scope);
+    }
+}
+function detectSync(text, policy, extra, state) {
+    const protectedSpans = [];
+    for (const match of text.matchAll(TOKEN_MARKER))
+        protectedSpans.push([match.index, match.index + match[0].length]);
+    const candidates = [...extra];
+    for (const recognizer of policy.recognizers)
+        candidates.push(...runRecognizer(recognizer, text));
+    for (const detector of policy.detectors) {
+        const result = detector.detect(text);
+        if (Array.isArray(result))
+            candidates.push(...result);
+        // A promise means an asynchronous detector: it ran in protect()'s pre-pass and arrives via `extra`.
+    }
+    if (state?.known.size) {
+        const { exact, folded, foldedMap } = state.knownRegexes();
+        if (exact)
+            for (const m of text.matchAll(exact)) {
+                const token = state.known.get(m[0]);
+                if (token)
+                    candidates.push({ type: state.byToken.get(token).type, start: m.index, end: m.index + m[0].length, score: 1, token });
+            }
+        if (folded)
+            for (const m of text.matchAll(folded)) {
+                const token = foldedMap.get(m[0].toLowerCase());
+                if (token)
+                    candidates.push({ type: "EMAIL", start: m.index, end: m.index + m[0].length, score: 1, token });
+            }
+    }
+    // One occupancy map for existing tokens and chosen spans: overlap checks cost O(span length), not O(detections),
+    // which kept large inputs (thousands of detections) from going quadratic.
+    const occupied = new Uint8Array(text.length);
+    for (const [s, e] of protectedSpans)
+        occupied.fill(1, s, e);
+    const free = (d) => { for (let i = d.start; i < d.end; i++)
+        if (occupied[i])
+            return false; return true; };
+    const eligible = candidates.filter((d) => d.end > d.start && d.end <= text.length && d.score >= policy.threshold &&
+        (d.token !== undefined || policy.classify(d.type) !== "PUBLIC") &&
+        !policy.allowlisted(text.slice(d.start, d.end)));
+    // Highest confidence first, then the longest span; keep each one that overlaps neither a token nor a chosen span.
+    eligible.sort((a, b) => b.score - a.score || (b.end - b.start) - (a.end - a.start) || a.start - b.start);
+    const chosen = [];
+    for (const d of eligible)
+        if (free(d)) {
+            chosen.push(d);
+            occupied.fill(1, d.start, d.end);
+        }
+    return chosen.sort((a, b) => a.start - b.start);
+}
+/**
+ * A view over one scope, under one policy. `protect` replaces personal data with tokens before data leaves your
+ * trust boundary; `restore` puts it back, for exactly the types the principal is granted.
+ */
+export class PrivacySession {
+    engine;
+    state;
+    policy;
+    memo = new Map();
+    /** @internal — use `Privacy.session()` / `Privacy.open()`. */
+    constructor(engine, state, policy) {
+        this.engine = engine;
+        this.state = state;
+        this.policy = policy;
+    }
+    get scope() {
+        return this.state.scope;
+    }
+    /**
+     * Replaces personal data in `value` (a string, or any JSON-shaped value) with vault tokens. Field classifications
+     * come from `options.schema`, the policy's `fields`, and key names; free text runs through every detector,
+     * including asynchronous ones. Idempotent: protecting already-protected data changes nothing.
+     */
+    async protect(value, options = {}) {
+        const asyncDetectors = this.policy.detectors;
+        let extras;
+        if (asyncDetectors.length) {
+            const strings = new Set();
+            collectStrings(value, strings);
+            extras = new Map();
+            await Promise.all([...strings].map(async (text) => {
+                const found = [];
+                for (const detector of asyncDetectors) {
+                    const result = detector.detect(text);
+                    if (!Array.isArray(result))
+                        found.push(...(await result));
+                }
+                if (found.length)
+                    extras.set(text, found);
+            }));
+        }
+        return this.walk(value, [], asNode(options.schema), asNode(options.schema), undefined, undefined, extras);
+    }
+    /** `protect()` without asynchronous detectors — for hot paths such as log lines. Values they found earlier in this scope are still caught. */
+    protectSync(value, options = {}) {
+        return this.walk(value, [], asNode(options.schema), asNode(options.schema), undefined, undefined, undefined);
+    }
+    /**
+     * Replaces tokens in `value` with the original values the principal may see. Unknown tokens (another scope's, or
+     * made up by a model) are left exactly as they are. BLOCKed values were never stored and cannot come back.
+     */
+    restore(value, options = {}) {
+        this.state.lastUsed = Date.now();
+        const seen = new Set();
+        const result = this.restoreValue(value, options, seen);
+        for (const token of seen) {
+            const entry = this.state.byToken.get(token);
+            const allowed = options.trusted === true || isGranted(options.allow, entry.type, entry.classification);
+            if (allowed)
+                this.state.restored += 1;
+            else
+                this.state.denied += 1;
+            this.state.record({ action: allowed ? "restore" : "deny", type: entry.type, classification: entry.classification, principal: options.principal, token });
+        }
+        return result;
+    }
+    /** Whether `text` contains a token issued in this scope. */
+    hasTokens(text) {
+        for (const match of text.matchAll(TOKEN_REFERENCE))
+            if (this.state.byToken.has(match[1]))
+                return true;
+        return false;
+    }
+    /** Restores a streamed response chunk by chunk, holding back a possible partial token at each chunk's end. */
+    streamRestorer(options = {}) {
+        return new StreamRestorer((text) => this.restore(text, options));
+    }
+    /** Waits for vault writes to reach the store; rejects if any failed. */
+    async flush() {
+        const pending = this.state.pending.splice(0);
+        const results = await Promise.allSettled(pending);
+        const failed = results.find((r) => r.status === "rejected");
+        if (failed)
+            throw failed.reason;
+    }
+    summary() {
+        const types = new Set();
+        for (const entry of this.state.byToken.values())
+            types.add(entry.type);
+        for (const event of this.state.audit)
+            if (event.action === "block" && event.type)
+                types.add(event.type);
+        return { tokenized: this.state.byToken.size, blocked: this.state.blocked, restored: this.state.restored, denied: this.state.denied, types: [...types].sort() };
+    }
+    /** What happened in this scope — tokenizations, blocks, restorations, denials. Holds tokens and types, never plaintext. */
+    audit() {
+        return [...this.state.audit];
+    }
+    // ---------------------------------------------------------------------------------------------------------------
+    walk(value, path, root, node, inherited, parentKey, extras) {
+        this.state.lastUsed = Date.now();
+        if (typeof value === "string")
+            return inherited ? this.whole(value, inherited, "string") : this.protectText(value, extras?.get(value));
+        if (typeof value === "number") {
+            if (inherited && Number.isFinite(value))
+                return this.whole(String(value), inherited, "number");
+            if (Number.isInteger(value) && Math.abs(value) >= 1_000_000) {
+                // Long integers can be card numbers, phone numbers or IDs stored numerically: only a detection that covers
+                // the whole number counts.
+                const text = String(value);
+                const hit = detectSync(text, this.policy, [], this.state).find((d) => d.start === 0 && d.end === text.length);
+                if (hit)
+                    return this.whole(text, { classification: this.classificationOf(hit), type: hit.type }, "number");
+            }
+            return value;
+        }
+        if (value === null || typeof value !== "object")
+            return value;
+        if (value instanceof Date)
+            return value;
+        if (Array.isArray(value))
+            return value.map((item, index) => this.walk(item, path, root, root && itemSchema(root, node, index), inherited, parentKey, extras));
+        const source = isPlainObject(value) ? value : (typeof value.toJSON === "function" ? value.toJSON() : { ...value });
+        if (!isPlainObject(source))
+            return this.walk(source, path, root, node, inherited, parentKey, extras);
+        const out = {};
+        for (const [key, child] of Object.entries(source)) {
+            const childPath = [...path, key];
+            const childNode = root ? propertySchema(root, node, key) : undefined;
+            const fieldClass = inherited ?? this.fieldClass(key, childPath, root, childNode, path[path.length - 1] ?? parentKey);
+            const outKey = inherited || fieldClass ? key : this.protectText(key, extras?.get(key));
+            out[outKey] = fieldClass?.classification === "PUBLIC" ? child : this.walk(child, childPath, root, childNode, fieldClass, key, extras);
+        }
+        return out;
+    }
+    fieldClass(key, path, root, node, parentKey) {
+        const inferred = () => inferFieldType(key, parentKey);
+        const rule = matchFieldRule(this.policy.fieldRules, path);
+        if (rule)
+            return { classification: rule.classification, type: rule.type ?? inferred() ?? typeFromKey(key) };
+        const annotation = root ? nodeAnnotation(root, node) : undefined;
+        if (annotation?.classification)
+            return { classification: annotation.classification, type: annotation.type ?? inferred() ?? typeFromKey(key) };
+        if (annotation?.type)
+            return { classification: this.policy.classify(annotation.type), type: annotation.type };
+        if (this.policy.inferFromFieldNames) {
+            const type = inferred();
+            if (type)
+                return { classification: this.policy.classify(type), type };
+        }
+        return undefined;
+    }
+    classificationOf(detection) {
+        return detection.token ? this.state.byToken.get(detection.token).classification : this.policy.classify(detection.type);
+    }
+    /** Tokenizes a field's entire value as one entity. */
+    whole(value, fieldClass, kind) {
+        const type = fieldClass.type ?? "FIELD";
+        if (fieldClass.classification === "PUBLIC" || value.trim() === "")
+            return kind === "number" ? Number(value) : value;
+        if (kind === "string" && isMarker(value))
+            return value;
+        if (this.policy.allowlisted(value))
+            return kind === "number" ? Number(value) : value;
+        if (fieldClass.classification === "BLOCK")
+            return this.block(type);
+        return renderToken(this.engine.tokenFor(this.state, type, value, fieldClass.classification, kind));
+    }
+    block(type) {
+        if (this.policy.onBlock === "throw")
+            throw new PrivacyError(`mask: input contains ${type}, which the policy blocks.`, "blocked");
+        this.state.blocked += 1;
+        this.state.record({ action: "block", type, classification: "BLOCK" });
+        return `[BLOCKED:${type}]`;
+    }
+    protectText(text, extra) {
+        if (text.length < 3)
+            return text;
+        if (!extra) {
+            const cached = this.memo.get(text);
+            if (cached && cached.version === this.state.knownVersion)
+                return cached.output;
+        }
+        const detections = detectSync(text, this.policy, extra ?? [], this.state);
+        let output = "";
+        let cursor = 0;
+        for (const d of detections) {
+            output += text.slice(cursor, d.start);
+            const original = text.slice(d.start, d.end);
+            if (d.token)
+                output += renderToken(d.token);
+            else {
+                const classification = this.policy.classify(d.type);
+                output += classification === "BLOCK" ? this.block(d.type) : renderToken(this.engine.tokenFor(this.state, d.type, original, classification, "string"));
+            }
+            cursor = d.end;
+        }
+        output += text.slice(cursor);
+        if (!extra) {
+            if (this.memo.size >= MAX_MEMO_ENTRIES)
+                this.memo.clear();
+            this.memo.set(text, { version: this.state.knownVersion, output });
+        }
+        return output;
+    }
+    restoreValue(value, options, seen) {
+        if (typeof value === "string")
+            return this.restoreText(value, options, seen);
+        if (value === null || typeof value !== "object" || value instanceof Date)
+            return value;
+        if (Array.isArray(value))
+            return value.map((item) => this.restoreValue(item, options, seen));
+        if (!isPlainObject(value))
+            return value;
+        const out = {};
+        for (const [key, child] of Object.entries(value))
+            out[String(this.restoreText(key, options, seen))] = this.restoreValue(child, options, seen);
+        return out;
+    }
+    restoreText(text, options, seen) {
+        if (!text.includes("_"))
+            return text;
+        const mode = options.onDenied ?? "redact";
+        let wholeNumber;
+        const restored = text.replace(TOKEN_REFERENCE, (match, token) => {
+            const entry = this.state.byToken.get(token);
+            if (!entry)
+                return match;
+            seen.add(token);
+            const allowed = options.trusted === true || isGranted(options.allow, entry.type, entry.classification);
+            if (!allowed)
+                return deniedReplacement(mode, entry.type, entry.value, entry.token);
+            if (entry.kind === "number" && match.length === text.length)
+                wholeNumber = Number(entry.value);
+            return entry.value;
+        });
+        return wholeNumber ?? restored;
+    }
+}
+function isMarker(value) {
+    const match = value.match(TOKEN_MARKER);
+    return match?.length === 1 && match[0] === value;
+}
+function collectStrings(value, into, depth = 0) {
+    if (depth > 64)
+        return;
+    if (typeof value === "string") {
+        if (value.length >= 3)
+            into.add(value);
+        return;
+    }
+    if (!value || typeof value !== "object")
+        return;
+    if (Array.isArray(value)) {
+        for (const item of value)
+            collectStrings(item, into, depth + 1);
+        return;
+    }
+    for (const [key, child] of Object.entries(value)) {
+        into.add(key);
+        collectStrings(child, into, depth + 1);
+    }
+}
+/** Longest possible token reference, `[` + type + `_` + id + `]`, with headroom for long custom type names. */
+const MAX_TOKEN_LENGTH = 72;
+/**
+ * Restores a stream without ever emitting half a token: text after an unclosed `[`, or a trailing run of token
+ * characters, is held back until the next chunk (or `flush()`) decides it.
+ */
+export class StreamRestorer {
+    restoreText;
+    buffer = "";
+    constructor(restoreText) {
+        this.restoreText = restoreText;
+    }
+    push(delta) {
+        this.buffer += delta;
+        let cut = this.buffer.length;
+        const open = this.buffer.lastIndexOf("[");
+        if (open !== -1 && !this.buffer.includes("]", open) && this.buffer.length - open <= MAX_TOKEN_LENGTH)
+            cut = open;
+        const tail = /[A-Za-z0-9_]{1,64}$/.exec(this.buffer.slice(0, cut));
+        if (tail)
+            cut = tail.index;
+        const ready = this.buffer.slice(0, cut);
+        this.buffer = this.buffer.slice(cut);
+        return ready ? this.restoreText(ready) : "";
+    }
+    flush() {
+        const rest = this.buffer;
+        this.buffer = "";
+        return rest ? this.restoreText(rest) : "";
+    }
+}
+/** Creates a privacy engine. See `PrivacyOptions`. */
+export function createPrivacy(options = {}) {
+    return new Privacy(options);
+}

package/dist/tokens.d.ts

@@ -0,0 +1,20 @@
+/**
+ * How tokens look, in one place. Three styles:
+ *   random     (default)  ss1_kqfmzxat         — keyed-HMAC letters, no type shown; written bare
+ *   sequential            [EMAIL_1]             — type plus a per-scope counter; written in brackets
+ *   opaque                [EMAIL_3f9a1c0b7e]    — type plus keyed-HMAC hex; written in brackets
+ */
+export type TokenStyle = "random" | "sequential" | "opaque";
+export declare const RANDOM_PREFIX = "ss1_";
+export declare const RANDOM_ID_LENGTH = 8;
+/** Anything already masked: a token in any style, or a BLOCKED/REDACTED marker. Detection never touches these spans. */
+export declare const TOKEN_MARKER: RegExp;
+/**
+ * A token reference to restore. Brackets are optional on both forms, since models add or drop them; capture group 1 is
+ * the bare token. Only tokens present in the scope's vault are ever resolved, so look-alikes stay untouched.
+ */
+export declare const TOKEN_REFERENCE: RegExp;
+/** How a token is written into text: random tokens bare, typed tokens in brackets. */
+export declare function renderToken(token: string): string;
+/** `ss1_` + 8 letters derived from a hex fingerprint; `attempt` shifts the window to resolve the (rare) collision. */
+export declare function randomToken(fingerprintHex: string, attempt: number): string;

package/dist/tokens.js

@@ -0,0 +1,25 @@
+export const RANDOM_PREFIX = "ss1_";
+export const RANDOM_ID_LENGTH = 8;
+const RANDOM_ALPHABET = "abcdefghijklmnopqrstuvwxyz";
+const TYPED = String.raw `[A-Z][A-Z0-9_]*_(?:[0-9a-f]{10}|[1-9]\d{0,5})`;
+const RANDOM = String.raw `ss1_[a-z]{8}`;
+/** Anything already masked: a token in any style, or a BLOCKED/REDACTED marker. Detection never touches these spans. */
+export const TOKEN_MARKER = new RegExp(String.raw `\[(?:${TYPED}|BLOCKED:[A-Z][A-Z0-9_]*|REDACTED:[A-Z][A-Z0-9_]*)\]|(?<![A-Za-z0-9_])\[?${RANDOM}\]?(?![A-Za-z0-9_])`, "g");
+/**
+ * A token reference to restore. Brackets are optional on both forms, since models add or drop them; capture group 1 is
+ * the bare token. Only tokens present in the scope's vault are ever resolved, so look-alikes stay untouched.
+ */
+export const TOKEN_REFERENCE = new RegExp(String.raw `(?<![A-Za-z0-9_])\[?(${TYPED}|${RANDOM})\]?(?![A-Za-z0-9_])`, "g");
+/** How a token is written into text: random tokens bare, typed tokens in brackets. */
+export function renderToken(token) {
+    return token.startsWith(RANDOM_PREFIX) ? token : `[${token}]`;
+}
+/** `ss1_` + 8 letters derived from a hex fingerprint; `attempt` shifts the window to resolve the (rare) collision. */
+export function randomToken(fingerprintHex, attempt) {
+    let id = "";
+    for (let i = 0; i < RANDOM_ID_LENGTH; i++) {
+        const at = (attempt * 2 + i * 2) % (fingerprintHex.length - 1);
+        id += RANDOM_ALPHABET[parseInt(fingerprintHex.slice(at, at + 2), 16) % RANDOM_ALPHABET.length];
+    }
+    return RANDOM_PREFIX + id;
+}