@doccloak/core 0.8.0

package/dist/session.js

@@ -0,0 +1,153 @@
+export class AnonymizationSession {
+    constructor() {
+        this.forwardMap = new Map();
+        this.reverseMap = new Map();
+        this.entityTypeMap = new Map();
+        this.counter = 0;
+        this.mode = 'labeled';
+    }
+    setMode(mode) {
+        this.mode = mode;
+    }
+    getMode() {
+        return this.mode;
+    }
+    anonymize(original, entityType) {
+        if (this.forwardMap.has(original)) {
+            return this.forwardMap.get(original);
+        }
+        this.counter++;
+        const placeholder = this.mode === 'blanked'
+            ? '________'
+            : `<<REDACTED_${this.counter}>>`;
+        this.forwardMap.set(original, placeholder);
+        // Blanked placeholders all collide on the same string; they are not
+        // reversible, so keep them out of the restore map.
+        if (this.mode !== 'blanked') {
+            this.reverseMap.set(placeholder, original);
+        }
+        this.entityTypeMap.set(original, entityType);
+        return placeholder;
+    }
+    deanonymize(text) {
+        let result = text;
+        // Sort by placeholder length (longest first) to avoid partial replacements
+        const entries = [...this.reverseMap.entries()].sort((a, b) => b[0].length - a[0].length);
+        for (const [placeholder, original] of entries) {
+            // Replacer function keeps '$' sequences in the original value inert
+            result = result.replaceAll(placeholder, () => original);
+        }
+        return result;
+    }
+    anonymizeText(text, entities) {
+        // Sort entities by position (end to start) to preserve indices during replacement
+        const sorted = [...entities].sort((a, b) => b.start - a.start);
+        let result = text;
+        // Clamp overlapping entities (e.g. a manual selection crossing a detected
+        // one) so a later replacement never splices into an already-replaced range.
+        let minStart = text.length;
+        for (const entity of sorted) {
+            const effectiveEnd = Math.min(entity.end, minStart);
+            if (effectiveEnd <= entity.start)
+                continue;
+            const placeholder = this.anonymize(entity.value, entity.type);
+            result = result.slice(0, entity.start) + placeholder + result.slice(effectiveEnd);
+            minStart = entity.start;
+        }
+        return result;
+    }
+    getEntries() {
+        return [...this.forwardMap.entries()].map(([original, replacement]) => {
+            const entityType = this.entityTypeMap.get(original) ?? 'OTHER';
+            return { original, replacement, entityType };
+        });
+    }
+    getForward(original) {
+        return this.forwardMap.get(original);
+    }
+    renameLabel(original, newLabel) {
+        const oldLabel = this.forwardMap.get(original);
+        if (!oldLabel)
+            return;
+        this.forwardMap.set(original, newLabel);
+        this.reverseMap.delete(oldLabel);
+        this.reverseMap.set(newLabel, original);
+    }
+    /**
+     * Serialize the session map to JSON.
+     *
+     * Schema (field-for-field parity with the Python CLI's
+     * DocCloak.Cli/doccloak/core/session.py save_map/load_map):
+     *
+     *   [
+     *     {
+     *       "original": string,      // the original sensitive value
+     *       "replacement": string,   // its placeholder, e.g. "<<REDACTED_1>>" or a renamed label
+     *       "entity_type": string    // EntityType value, e.g. "PERSON" (snake_case key, matching Python)
+     *     },
+     *     ...
+     *   ]
+     *
+     * - Top level is a JSON array; one object per mapping, in insertion order.
+     * - The counter is NOT stored as a field. Like Python's load_map, deserialize
+     *   derives it from the number of entries.
+     * - The mode is NOT stored as a field. Like Python's load_map (which returns
+     *   `cls()` with the default mode), deserialize yields a 'labeled' session.
+     * - Blanked entries are intentionally OMITTED from the output: blanked
+     *   placeholders ('________') are irreversible by design (this class already
+     *   keeps them out of the reverse map), so persisting their originals would
+     *   defeat that. Only reversible entries are serialized.
+     * - Output formatting matches Python's json.dumps(entries, indent=2,
+     *   ensure_ascii=False): 2-space indent, non-ASCII characters kept raw.
+     */
+    serialize() {
+        const entries = this.getEntries()
+            .filter(({ replacement, original }) => this.reverseMap.get(replacement) === original)
+            .map(({ original, replacement, entityType }) => ({
+            original,
+            replacement,
+            entity_type: entityType,
+        }));
+        return JSON.stringify(entries, null, 2);
+    }
+    /**
+     * Rebuild a session from JSON produced by serialize() or by the Python
+     * CLI's save_map. Mirrors Python's load_map: forward/reverse/type maps are
+     * repopulated per entry and the counter is incremented once per entry.
+     * Additionally, the counter is raised to the highest N found among
+     * '<<REDACTED_N>>' placeholders (a map serialized with gaps, e.g. after
+     * blanked entries were omitted, would otherwise reissue a taken number),
+     * so new entities anonymized after restore continue numbering without
+     * colliding with restored placeholders. Blanked entries ('________') that a
+     * Python-produced map may contain are kept in the forward map but excluded
+     * from the reverse map, preserving their irreversibility.
+     */
+    static deserialize(json) {
+        const data = JSON.parse(json);
+        if (!Array.isArray(data)) {
+            throw new Error('Invalid session map JSON: expected a top-level array');
+        }
+        const session = new AnonymizationSession();
+        let maxPlaceholderNumber = 0;
+        for (const entry of data) {
+            session.forwardMap.set(entry.original, entry.replacement);
+            if (entry.replacement !== '________') {
+                session.reverseMap.set(entry.replacement, entry.original);
+            }
+            session.entityTypeMap.set(entry.original, entry.entity_type);
+            session.counter++;
+            const match = /^<<REDACTED_(\d+)>>$/.exec(entry.replacement);
+            if (match) {
+                maxPlaceholderNumber = Math.max(maxPlaceholderNumber, Number(match[1]));
+            }
+        }
+        session.counter = Math.max(session.counter, maxPlaceholderNumber);
+        return session;
+    }
+    clear() {
+        this.forwardMap.clear();
+        this.reverseMap.clear();
+        this.entityTypeMap.clear();
+        this.counter = 0;
+    }
+}

package/dist/types.d.ts

@@ -0,0 +1,42 @@
+export type EntityType = 'PERSON' | 'EMAIL' | 'PHONE' | 'SSN' | 'CREDIT_CARD' | 'DATE' | 'CURRENCY' | 'IP_ADDRESS' | 'IBAN' | 'ADDRESS' | 'COMPANY' | 'OTHER';
+export interface DetectedEntity {
+    type: EntityType;
+    value: string;
+    start: number;
+    end: number;
+    confidence: number;
+    detector: string;
+}
+export type ProgressCallback = (downloaded: number, total: number) => void;
+/**
+ * Interface for detection providers.
+ * Implement this to add a new model/detection backend.
+ * The engine only depends on this interface — swap providers without changing anything else.
+ */
+export interface DetectionProvider {
+    /** Human-readable name for this provider */
+    readonly name: string;
+    /** Initialize/download the model. Called once on app load. */
+    load(): Promise<void>;
+    /** Whether the provider is ready to detect */
+    isLoaded(): boolean;
+    /** Whether the provider is currently loading */
+    isLoading(): boolean;
+    /** Register a callback for download progress updates */
+    onProgress(callback: ProgressCallback): void;
+    /** Run detection on the given text. Optional progress callback (0-1). */
+    detect(text: string, onProgress?: (progress: number) => void): Promise<DetectedEntity[]>;
+    /** Set detection confidence threshold */
+    setThreshold(value: number): void;
+    /** Get current detection confidence threshold */
+    getThreshold(): number;
+    /** Release the ONNX session and free memory. Provider can be re-loaded later. */
+    release(): void;
+}
+export interface ReplacementEntry {
+    original: string;
+    replacement: string;
+    entityType: EntityType;
+}
+export declare const ENTITY_COLORS: Record<EntityType, string>;
+export declare const ENTITY_LABELS: Record<EntityType, string>;

package/dist/types.js

@@ -0,0 +1,28 @@
+export const ENTITY_COLORS = {
+    PERSON: '#7B2FF7',
+    EMAIL: '#2563EB',
+    PHONE: '#0D9488',
+    SSN: '#B45309',
+    CREDIT_CARD: '#CA8A04',
+    DATE: '#6D28D9',
+    CURRENCY: '#EA580C',
+    IP_ADDRESS: '#0284C7',
+    IBAN: '#A16207',
+    ADDRESS: '#DC2626',
+    COMPANY: '#DB2777',
+    OTHER: '#6B7280',
+};
+export const ENTITY_LABELS = {
+    PERSON: 'Name',
+    EMAIL: 'Email',
+    PHONE: 'Phone',
+    SSN: 'SSN',
+    CREDIT_CARD: 'Credit Card',
+    DATE: 'Date',
+    CURRENCY: 'Currency',
+    IP_ADDRESS: 'IP Address',
+    IBAN: 'IBAN',
+    ADDRESS: 'Address',
+    COMPANY: 'Company',
+    OTHER: 'Other',
+};

package/dist/worker-protocol.d.ts

@@ -0,0 +1,119 @@
+/**
+ * @doccloak/core - transport-agnostic worker message protocol (T009).
+ *
+ * Formalizes the message pairs the web app's engine.ts and
+ * detection.worker.ts have always exchanged, keeping the exact message
+ * names and payload fields on the wire:
+ *
+ *   init / switchProvider  -> loaded | loadError   (+ downloadProgress events)
+ *   detect                 -> detected | detectError (+ detectionProgress events)
+ *   releaseModel           -> released
+ *   setThreshold / setCustomLabels / setRegex / setRegexRegion  (fire and forget)
+ *
+ * Correlation keeps the existing requestId scheme; requestId is optional on
+ * the request side so legacy-style senders (and the regression harness)
+ * keep working, while connectEngine always correlates.
+ *
+ * serveEngine() is the host side: it answers protocol messages with a real
+ * engine (web worker entry, extension offscreen document). connectEngine()
+ * is the client side: a DocCloakEngine that forwards over the port (web
+ * main thread, extension side panel / service worker).
+ */
+import type { DetectedEntity } from './types.js';
+import type { DocCloakEngine, EngineSettings, ProviderId } from './engine.js';
+import type { RegexRegionId } from './regex/index.js';
+/** Satisfied by Worker, MessagePort and chrome.runtime.Port wrappers. */
+export interface PortLike {
+    postMessage(msg: unknown): void;
+    /** Register a message callback; returns an unsubscribe function. */
+    onMessage(cb: (msg: unknown) => void | Promise<void>): () => void;
+}
+export type EngineRequest = {
+    type: 'init';
+    requestId?: number;
+    providerId?: ProviderId;
+    customLabels?: string[];
+    regexEnabled?: boolean;
+    regexRegion?: RegexRegionId;
+} | {
+    type: 'detect';
+    requestId: number;
+    text: string;
+} | {
+    type: 'switchProvider';
+    requestId?: number;
+    providerId: ProviderId;
+    customLabels?: string[];
+} | {
+    type: 'setThreshold';
+    value: number;
+} | {
+    type: 'setCustomLabels';
+    labels: string[];
+} | {
+    type: 'setRegex';
+    enabled: boolean;
+    region?: RegexRegionId;
+} | {
+    type: 'setRegexRegion';
+    region: RegexRegionId;
+} | {
+    type: 'releaseModel';
+    requestId?: number;
+};
+export type EngineResponse = {
+    type: 'loaded';
+    requestId?: number;
+    providerId: ProviderId;
+    threshold: number;
+    customLabels: string[];
+} | {
+    type: 'loadError';
+    requestId?: number;
+    error: string;
+} | {
+    type: 'detected';
+    requestId: number;
+    entities: DetectedEntity[];
+} | {
+    type: 'detectError';
+    requestId: number;
+    error: string;
+} | {
+    type: 'detectionProgress';
+    requestId: number;
+    progress: number;
+} | {
+    type: 'downloadProgress';
+    downloaded: number;
+    total: number;
+} | {
+    type: 'released';
+    requestId?: number;
+};
+/**
+ * Serve a real engine over a port. Returns a stop function that
+ * unsubscribes from the port and the engine's progress events.
+ */
+export declare function serveEngine(engine: DocCloakEngine, port: PortLike): () => void;
+/** DocCloakEngine proxy returned by connectEngine. */
+export interface EngineClient extends DocCloakEngine {
+    /**
+     * Switch provider, optionally sending fresh custom labels in the same
+     * message (mirrors the pre-extraction switchProvider payload).
+     */
+    switchProvider(id: ProviderId, customLabels?: string[]): Promise<void>;
+    /**
+     * Tear the connection down: rejects pending detections/loads (resolves
+     * pending releases) and stops listening. Hosts call this when the
+     * underlying worker crashes.
+     */
+    close(reason?: Error): void;
+}
+/**
+ * Connect to a served engine over a port. The client keeps a synchronous
+ * settings mirror (seeded from `initial`, updated from loaded responses)
+ * so getSettings() works without a round trip; persistence stays with
+ * whichever side owns the KV store.
+ */
+export declare function connectEngine(port: PortLike, initial?: Partial<EngineSettings>): EngineClient;

package/dist/worker-protocol.js

@@ -0,0 +1,285 @@
+/**
+ * @doccloak/core - transport-agnostic worker message protocol (T009).
+ *
+ * Formalizes the message pairs the web app's engine.ts and
+ * detection.worker.ts have always exchanged, keeping the exact message
+ * names and payload fields on the wire:
+ *
+ *   init / switchProvider  -> loaded | loadError   (+ downloadProgress events)
+ *   detect                 -> detected | detectError (+ detectionProgress events)
+ *   releaseModel           -> released
+ *   setThreshold / setCustomLabels / setRegex / setRegexRegion  (fire and forget)
+ *
+ * Correlation keeps the existing requestId scheme; requestId is optional on
+ * the request side so legacy-style senders (and the regression harness)
+ * keep working, while connectEngine always correlates.
+ *
+ * serveEngine() is the host side: it answers protocol messages with a real
+ * engine (web worker entry, extension offscreen document). connectEngine()
+ * is the client side: a DocCloakEngine that forwards over the port (web
+ * main thread, extension side panel / service worker).
+ */
+import { clampThreshold, defaultThresholdFor } from "./engine.js";
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+// ── Host side ──────────────────────────────────────────────
+/**
+ * Serve a real engine over a port. Returns a stop function that
+ * unsubscribes from the port and the engine's progress events.
+ */
+export function serveEngine(engine, port) {
+    const unsubscribeDownload = engine.onDownloadProgress(({ loaded, total }) => {
+        const msg = { type: 'downloadProgress', downloaded: loaded, total };
+        port.postMessage(msg);
+    });
+    function post(msg) {
+        port.postMessage(msg);
+    }
+    async function handleLoad(requestId, run) {
+        try {
+            await run();
+            const s = engine.getSettings();
+            post({ type: 'loaded', requestId, providerId: s.providerId, threshold: s.threshold, customLabels: s.customLabels });
+        }
+        catch (err) {
+            post({ type: 'loadError', requestId, error: errorMessage(err) });
+        }
+    }
+    const unsubscribeMessages = port.onMessage(async (raw) => {
+        const msg = raw;
+        switch (msg.type) {
+            case 'init': {
+                await handleLoad(msg.requestId, async () => {
+                    const patch = {};
+                    if (msg.providerId !== undefined)
+                        patch.providerId = msg.providerId;
+                    if (msg.customLabels !== undefined)
+                        patch.customLabels = msg.customLabels;
+                    if (msg.regexEnabled !== undefined)
+                        patch.regexEnabled = msg.regexEnabled;
+                    if (msg.regexRegion !== undefined)
+                        patch.regexRegion = msg.regexRegion;
+                    await engine.updateSettings(patch);
+                    await engine.preload();
+                });
+                break;
+            }
+            case 'switchProvider': {
+                await handleLoad(msg.requestId, async () => {
+                    if (msg.customLabels !== undefined) {
+                        await engine.updateSettings({ customLabels: msg.customLabels });
+                    }
+                    await engine.switchProvider(msg.providerId);
+                });
+                break;
+            }
+            case 'detect': {
+                try {
+                    const entities = await engine.detect(msg.text, undefined, (progress) => {
+                        post({ type: 'detectionProgress', requestId: msg.requestId, progress });
+                    });
+                    post({ type: 'detected', requestId: msg.requestId, entities });
+                }
+                catch (err) {
+                    post({ type: 'detectError', requestId: msg.requestId, error: errorMessage(err) });
+                }
+                break;
+            }
+            case 'setThreshold': {
+                await engine.updateSettings({ threshold: msg.value });
+                break;
+            }
+            case 'setCustomLabels': {
+                await engine.updateSettings({ customLabels: msg.labels });
+                break;
+            }
+            case 'setRegex': {
+                const patch = { regexEnabled: msg.enabled };
+                if (msg.region !== undefined)
+                    patch.regexRegion = msg.region;
+                await engine.updateSettings(patch);
+                break;
+            }
+            case 'setRegexRegion': {
+                await engine.updateSettings({ regexRegion: msg.region });
+                break;
+            }
+            case 'releaseModel': {
+                await engine.release();
+                post({ type: 'released', requestId: msg.requestId });
+                break;
+            }
+        }
+    });
+    return () => {
+        unsubscribeMessages();
+        unsubscribeDownload();
+    };
+}
+/**
+ * Connect to a served engine over a port. The client keeps a synchronous
+ * settings mirror (seeded from `initial`, updated from loaded responses)
+ * so getSettings() works without a round trip; persistence stays with
+ * whichever side owns the KV store.
+ */
+export function connectEngine(port, initial) {
+    const initialProvider = initial?.providerId ?? 'bardsai';
+    const settings = {
+        providerId: initialProvider,
+        threshold: initial?.threshold ?? defaultThresholdFor(initialProvider),
+        regexEnabled: initial?.regexEnabled ?? false,
+        regexRegion: initial?.regexRegion ?? 'all',
+        customLabels: [...(initial?.customLabels ?? [])],
+    };
+    let nextRequestId = 0;
+    let closed = false;
+    const pending = new Map();
+    const downloadListeners = new Set();
+    const detectionListeners = new Set();
+    function takePending(requestId) {
+        if (requestId === undefined)
+            return undefined;
+        const entry = pending.get(requestId);
+        if (entry)
+            pending.delete(requestId);
+        return entry;
+    }
+    const unsubscribe = port.onMessage((raw) => {
+        const msg = raw;
+        switch (msg.type) {
+            case 'downloadProgress': {
+                for (const cb of downloadListeners)
+                    cb({ loaded: msg.downloaded, total: msg.total });
+                break;
+            }
+            case 'detectionProgress': {
+                pending.get(msg.requestId)?.onProgress?.(msg.progress);
+                for (const cb of detectionListeners)
+                    cb(msg.progress);
+                break;
+            }
+            case 'loaded': {
+                settings.providerId = msg.providerId;
+                settings.threshold = msg.threshold;
+                settings.customLabels = [...(msg.customLabels ?? [])];
+                takePending(msg.requestId)?.resolve(undefined);
+                break;
+            }
+            case 'loadError': {
+                takePending(msg.requestId)?.reject(new Error(msg.error));
+                break;
+            }
+            case 'detected': {
+                takePending(msg.requestId)?.resolve(msg.entities ?? []);
+                break;
+            }
+            case 'detectError': {
+                takePending(msg.requestId)?.reject(new Error(msg.error));
+                break;
+            }
+            case 'released': {
+                takePending(msg.requestId)?.resolve(undefined);
+                break;
+            }
+        }
+    });
+    function request(kind, message, onProgress) {
+        if (closed)
+            return Promise.reject(new Error('Engine connection closed'));
+        const requestId = ++nextRequestId;
+        return new Promise((resolve, reject) => {
+            pending.set(requestId, { kind, resolve, reject, onProgress });
+            port.postMessage({ ...message, requestId });
+        });
+    }
+    async function preload() {
+        await request('load', {
+            type: 'init',
+            providerId: settings.providerId,
+            customLabels: settings.customLabels,
+            regexEnabled: settings.regexEnabled,
+            regexRegion: settings.regexRegion,
+        });
+    }
+    async function switchProvider(id, customLabels) {
+        const labels = customLabels ?? settings.customLabels;
+        settings.customLabels = [...labels];
+        await request('load', { type: 'switchProvider', providerId: id, customLabels: labels });
+    }
+    async function updateSettings(patch) {
+        if (patch.regexRegion !== undefined && patch.regexEnabled === undefined) {
+            settings.regexRegion = patch.regexRegion;
+            port.postMessage({ type: 'setRegexRegion', region: patch.regexRegion });
+        }
+        if (patch.regexEnabled !== undefined) {
+            settings.regexEnabled = patch.regexEnabled;
+            if (patch.regexRegion !== undefined)
+                settings.regexRegion = patch.regexRegion;
+            port.postMessage({
+                type: 'setRegex',
+                enabled: patch.regexEnabled,
+                region: settings.regexRegion,
+            });
+        }
+        if (patch.threshold !== undefined) {
+            settings.threshold = clampThreshold(patch.threshold);
+            port.postMessage({ type: 'setThreshold', value: settings.threshold });
+        }
+        if (patch.customLabels !== undefined) {
+            settings.customLabels = patch.customLabels.filter((l) => l.trim().length > 0);
+            port.postMessage({ type: 'setCustomLabels', labels: settings.customLabels });
+        }
+        if (patch.providerId !== undefined && patch.providerId !== settings.providerId) {
+            await switchProvider(patch.providerId);
+        }
+    }
+    function detect(text, signal, onProgress) {
+        if (signal?.aborted)
+            return Promise.reject(new Error('Detection aborted'));
+        const promise = request('detect', { type: 'detect', text }, onProgress);
+        if (signal) {
+            const requestId = nextRequestId;
+            signal.addEventListener('abort', () => {
+                takePending(requestId)?.reject(new Error('Detection aborted'));
+            }, { once: true });
+        }
+        return promise;
+    }
+    function close(reason) {
+        if (closed)
+            return;
+        closed = true;
+        unsubscribe();
+        const err = reason ?? new Error('Engine connection closed');
+        for (const [, entry] of pending) {
+            if (entry.kind === 'release')
+                entry.resolve(undefined);
+            else
+                entry.reject(err);
+        }
+        pending.clear();
+    }
+    return {
+        ready: Promise.resolve(),
+        detect,
+        preload,
+        switchProvider,
+        getSettings() {
+            return { ...settings, customLabels: [...settings.customLabels] };
+        },
+        updateSettings,
+        onDownloadProgress(cb) {
+            downloadListeners.add(cb);
+            return () => downloadListeners.delete(cb);
+        },
+        onDetectionProgress(cb) {
+            detectionListeners.add(cb);
+            return () => detectionListeners.delete(cb);
+        },
+        async release() {
+            await request('release', { type: 'releaseModel' });
+        },
+        close,
+    };
+}