@doccloak/core 0.8.0

package/dist/providers/gliner.js

@@ -0,0 +1,360 @@
+/**
+ * GLiNER PII Edge provider.
+ *
+ * Environment-agnostic: wasm paths, custom-label persistence (KV), model
+ * bytes (blob cache + fetch) and the tokenizer all arrive through the
+ * injected CoreEnv. This module never touches import.meta, localStorage,
+ * caches, navigator or @huggingface/transformers directly.
+ */
+import * as ort from 'onnxruntime-web';
+import { fetchModelBlob, retryAsync } from "../model-loader.js";
+// ── Model config ──────────────────────────────────────────
+const DEFAULT_MODEL_URL = 'https://huggingface.co/knowledgator/gliner-pii-edge-v1.0/resolve/main/onnx/model_quint8.onnx';
+const DEFAULT_TOKENIZER_HF = 'knowledgator/gliner-pii-edge-v1.0';
+const DEFAULT_MODEL_NAME = 'GLiNER PII Edge';
+const CUSTOM_LABELS_STORAGE_KEY = 'doccloak-custom-labels';
+const CLS_TOKEN_ID = 50281;
+const SEP_TOKEN_ID = 50282;
+const ENT_TOKEN_ID = 50368;
+const SEP_PROMPT_TOKEN_ID = 50369; // <<SEP>> (prompt separator, not [SEP])
+const DEFAULT_THRESHOLD = 0.35;
+const MAX_WORDS_PER_CHUNK = 150;
+const CHUNK_OVERLAP = 40;
+const DEFAULT_PII_LABELS = [
+    'person name',
+    'calendar date',
+    'email address', 'phone number', 'ip address',
+    'street address', 'city', 'zip code',
+    'bank account number', 'credit card number', 'iban',
+    'social security number', 'tax id', 'national id number',
+    'passport number', 'driver license number',
+    'money amount',
+    'company name',
+];
+// ── Label → EntityType mapping ────────────────────────────
+function mapLabelToEntityType(label) {
+    const l = label.toLowerCase();
+    if (l === 'person name')
+        return 'PERSON';
+    if (l === 'email address')
+        return 'EMAIL';
+    if (l === 'phone number')
+        return 'PHONE';
+    if (['social security number', 'national id number', 'tax id', 'passport number', 'driver license number'].includes(l))
+        return 'SSN';
+    if (l === 'credit card number')
+        return 'CREDIT_CARD';
+    if (l === 'calendar date')
+        return 'DATE';
+    if (l === 'money amount')
+        return 'CURRENCY';
+    if (l === 'ip address')
+        return 'IP_ADDRESS';
+    if (['iban', 'bank account number'].includes(l))
+        return 'IBAN';
+    if (['street address', 'city', 'zip code'].includes(l))
+        return 'ADDRESS';
+    if (l === 'company name')
+        return 'COMPANY';
+    return 'OTHER';
+}
+// ── Word splitter ─────────────────────────────────────────
+const WORD_PATTERN = /[\p{L}\p{N}]+(?:[-_][\p{L}\p{N}]+)*|\S/gu;
+function splitWords(text) {
+    const words = [];
+    const starts = [];
+    const ends = [];
+    let match;
+    WORD_PATTERN.lastIndex = 0;
+    while ((match = WORD_PATTERN.exec(text)) !== null) {
+        words.push(match[0]);
+        starts.push(match.index);
+        ends.push(WORD_PATTERN.lastIndex);
+    }
+    return { words, starts, ends };
+}
+// ── Sigmoid ───────────────────────────────────────────────
+function sigmoid(x) {
+    return 1 / (1 + Math.exp(-x));
+}
+function greedySelect(spans) {
+    const sorted = spans.slice().sort((a, b) => b[4] - a[4]);
+    const selected = [];
+    for (const span of sorted) {
+        const overlaps = selected.some(s => span[1] < s[2] && span[2] > s[1]);
+        if (!overlaps)
+            selected.push(span);
+    }
+    return selected.sort((a, b) => a[1] - b[1]);
+}
+// ── Provider ──────────────────────────────────────────────
+export class GlinerProvider {
+    get name() { return this._name; }
+    constructor(env) {
+        this._name = DEFAULT_MODEL_NAME;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        this.tokenizer = null;
+        this.session = null;
+        this.loading = false;
+        this.loadError = null;
+        this.loadWaiters = [];
+        this.progressCallback = null;
+        this.threshold = DEFAULT_THRESHOLD;
+        this.customLabels = [];
+        this.env = env;
+    }
+    isLoaded() {
+        return this.session !== null && this.tokenizer !== null;
+    }
+    isLoading() {
+        return this.loading;
+    }
+    onProgress(callback) {
+        this.progressCallback = callback;
+    }
+    setThreshold(value) {
+        this.threshold = Math.max(0.05, Math.min(0.95, value));
+    }
+    getThreshold() {
+        return this.threshold;
+    }
+    /**
+     * Get the combined list of all active labels (built-in + custom).
+     */
+    getActiveLabels() {
+        return [...DEFAULT_PII_LABELS, ...this.customLabels];
+    }
+    /**
+     * Get user-defined custom labels.
+     */
+    getCustomLabels() {
+        return [...this.customLabels];
+    }
+    /**
+     * Set custom labels and persist through the injected KV store. The
+     * in-memory list updates synchronously (before the first await), so
+     * fire-and-forget callers keep the previous behavior.
+     */
+    async setCustomLabels(labels) {
+        this.customLabels = labels.filter((l) => l.trim().length > 0);
+        try {
+            await this.env.kv.set(CUSTOM_LABELS_STORAGE_KEY, JSON.stringify(this.customLabels));
+        }
+        catch { /* KV backend unavailable (e.g. storage-less Web Worker) */ }
+    }
+    /**
+     * Restore custom labels from the injected KV store on startup.
+     */
+    async restoreCustomLabels() {
+        try {
+            const saved = await this.env.kv.get(CUSTOM_LABELS_STORAGE_KEY);
+            if (saved) {
+                this.customLabels = JSON.parse(saved);
+            }
+        }
+        catch { /* KV backend unavailable in Web Worker - labels are passed via message */ }
+    }
+    async load() {
+        if (this.isLoaded())
+            return;
+        if (this.loadError)
+            throw this.loadError;
+        if (this.loading) {
+            return new Promise((resolve, reject) => {
+                this.loadWaiters.push({ resolve, reject });
+            });
+        }
+        this.loading = true;
+        try {
+            ort.env.wasm.wasmPaths = this.env.wasm.paths;
+            // Default to single-threaded WASM. ORT's multi-threaded path hangs
+            // silently in some cross-origin-isolated browser contexts when spawning
+            // pthread workers (model downloads but session.create never resolves).
+            ort.env.wasm.numThreads = this.env.wasm.numThreads ?? 1;
+            const loaderEnv = {
+                cache: this.env.modelCache,
+                fetch: this.env.fetch,
+                persistStorage: this.env.persistStorage,
+            };
+            // Load tokenizer and model in parallel (skip tokenizer if already loaded from a prior session)
+            const tasks = [
+                fetchModelBlob(loaderEnv, DEFAULT_MODEL_URL, (downloaded, total) => this.progressCallback?.(downloaded, total)),
+            ];
+            if (!this.tokenizer) {
+                tasks.push(retryAsync(() => this.env.loadTokenizer(DEFAULT_TOKENIZER_HF), 'Tokenizer download').then((t) => {
+                    this.tokenizer = t;
+                }));
+            }
+            const [modelBlob] = await Promise.all(tasks);
+            const blobUrl = URL.createObjectURL(modelBlob);
+            try {
+                this.session = await ort.InferenceSession.create(blobUrl, {
+                    executionProviders: ['wasm'],
+                });
+            }
+            finally {
+                // ONNX Runtime has already read the data into the WASM heap
+                URL.revokeObjectURL(blobUrl);
+            }
+            console.info(`[DocCloak] Model loaded: ${this._name} | inputs: [${this.session.inputNames.join(', ')}] | outputs: [${this.session.outputNames.join(', ')}]`);
+            this.loading = false;
+            this.loadWaiters.forEach((w) => w.resolve());
+            this.loadWaiters.length = 0;
+        }
+        catch (err) {
+            this.session = null;
+            this.loadError = err instanceof Error ? err : new Error(String(err));
+            this.loading = false;
+            this.loadWaiters.forEach((w) => w.reject(this.loadError));
+            this.loadWaiters.length = 0;
+            throw this.loadError;
+        }
+    }
+    release() {
+        if (this.session) {
+            this.session.release();
+            this.session = null;
+        }
+        this.loading = false;
+        this.loadError = null;
+    }
+    async detect(text, onProgress) {
+        if (!this.isLoaded())
+            await this.load();
+        if (!this.session || !this.tokenizer)
+            return [];
+        const { words, starts, ends } = splitWords(text);
+        if (words.length === 0)
+            return [];
+        // Build prompt tokens once (shared across all chunks)
+        const activeLabels = this.getActiveLabels();
+        const promptTokens = this.buildPromptTokens(activeLabels);
+        const chunkSize = MAX_WORDS_PER_CHUNK;
+        const overlap = CHUNK_OVERLAP;
+        // Split into chunks if text is too long
+        const allSpans = [];
+        onProgress?.(0);
+        if (words.length <= chunkSize) {
+            // Single chunk — no splitting needed
+            const spans = await this.inferChunk(words, starts, ends, text, promptTokens, activeLabels);
+            allSpans.push(...spans);
+            onProgress?.(1);
+        }
+        else {
+            // Count total chunks
+            let totalChunks = 0;
+            for (let cs = 0; cs < words.length; cs += chunkSize - overlap) {
+                totalChunks++;
+                if (Math.min(cs + chunkSize, words.length) >= words.length)
+                    break;
+            }
+            // Multiple overlapping chunks
+            let chunksDone = 0;
+            for (let chunkStart = 0; chunkStart < words.length; chunkStart += chunkSize - overlap) {
+                const chunkEnd = Math.min(chunkStart + chunkSize, words.length);
+                const chunkWords = words.slice(chunkStart, chunkEnd);
+                const chunkStarts = starts.slice(chunkStart, chunkEnd);
+                const chunkEnds = ends.slice(chunkStart, chunkEnd);
+                const spans = await this.inferChunk(chunkWords, chunkStarts, chunkEnds, text, promptTokens, activeLabels);
+                allSpans.push(...spans);
+                chunksDone++;
+                onProgress?.(chunksDone / totalChunks);
+                await new Promise((r) => setTimeout(r, 0));
+                if (chunkEnd >= words.length)
+                    break;
+            }
+        }
+        // Greedy non-overlapping selection across all chunks
+        const selected = greedySelect(allSpans);
+        return selected
+            .filter(([spanText]) => spanText.trim().length >= 2)
+            .map(([spanText, start, end, label, score]) => ({
+            type: mapLabelToEntityType(label),
+            value: spanText,
+            start,
+            end,
+            confidence: score,
+            detector: `gliner:${label}`,
+        }));
+    }
+    buildPromptTokens(labels) {
+        const tokens = [];
+        for (const label of labels) {
+            tokens.push(ENT_TOKEN_ID);
+            for (const part of label.split(' ')) {
+                const subTokens = this.tokenizer.encode(part).slice(1, -1);
+                tokens.push(...subTokens);
+            }
+        }
+        tokens.push(SEP_PROMPT_TOKEN_ID);
+        return tokens;
+    }
+    async inferChunk(words, charStarts, charEnds, fullText, promptTokens, labels) {
+        if (!this.session)
+            return [];
+        // Build token sequence: [CLS] + prompt + text words + [SEP]
+        const inputIds = [CLS_TOKEN_ID, ...promptTokens];
+        const attentionMask = new Array(inputIds.length).fill(1);
+        const wordsMask = new Array(inputIds.length).fill(0);
+        let wordCounter = 1;
+        for (const word of words) {
+            const subTokens = this.tokenizer.encode(word).slice(1, -1);
+            for (let j = 0; j < subTokens.length; j++) {
+                inputIds.push(subTokens[j]);
+                attentionMask.push(1);
+                wordsMask.push(j === 0 ? wordCounter : 0);
+            }
+            wordCounter++;
+        }
+        inputIds.push(SEP_TOKEN_ID);
+        attentionMask.push(1);
+        wordsMask.push(0);
+        const seqLen = inputIds.length;
+        const textLength = words.length;
+        const numEntities = labels.length;
+        const idToClass = {};
+        labels.forEach((label, i) => { idToClass[i + 1] = label; });
+        const feeds = {
+            input_ids: new ort.Tensor('int64', BigInt64Array.from(inputIds.map(BigInt)), [1, seqLen]),
+            attention_mask: new ort.Tensor('int64', BigInt64Array.from(attentionMask.map(BigInt)), [1, seqLen]),
+            words_mask: new ort.Tensor('int64', BigInt64Array.from(wordsMask.map(BigInt)), [1, seqLen]),
+            text_lengths: new ort.Tensor('int64', BigInt64Array.from([BigInt(textLength)]), [1, 1]),
+        };
+        const results = await this.session.run(feeds);
+        const outputName = this.session.outputNames[0] || 'logits';
+        const logits = results[outputName].data;
+        // Token-based: logits shape [1, num_words, num_entities, 3] (start/end/inside)
+        const rawSpans = [];
+        const selectedStarts = [];
+        const selectedEnds = [];
+        const insideScores = Array.from({ length: textLength }, () => Array(numEntities).fill(0));
+        for (let token = 0; token < textLength; token++) {
+            for (let entity = 0; entity < numEntities; entity++) {
+                const base = (token * numEntities + entity) * 3;
+                const startProb = sigmoid(logits[base]);
+                const endProb = sigmoid(logits[base + 1]);
+                const insideProb = sigmoid(logits[base + 2]);
+                if (startProb >= this.threshold)
+                    selectedStarts.push([token, entity]);
+                if (endProb >= this.threshold)
+                    selectedEnds.push([token, entity]);
+                insideScores[token][entity] = insideProb;
+            }
+        }
+        for (const [startTok, startCls] of selectedStarts) {
+            for (const [endTok, endCls] of selectedEnds) {
+                if (endTok < startTok || startCls !== endCls)
+                    continue;
+                const inside = insideScores.slice(startTok, endTok + 1).map(s => s[startCls]);
+                if (inside.some(s => s < this.threshold))
+                    continue;
+                const score = inside.reduce((a, b) => a + b, 0) / inside.length;
+                const charStart = charStarts[startTok];
+                const charEnd = charEnds[endTok];
+                const spanText = fullText.slice(charStart, charEnd);
+                rawSpans.push([spanText, charStart, charEnd, idToClass[startCls + 1], score]);
+            }
+        }
+        return rawSpans;
+    }
+}

package/dist/regex/index.d.ts

@@ -0,0 +1,17 @@
+export type { RegexRule, PiiDomain, RegionCode } from './types.js';
+export type { RegexRuleJson, RegionRulesJson } from './loader.js';
+/**
+ * Region choices for the regex detector: 'all' plus every shipped region
+ * pack. Order is significant - hosts render region pickers straight from
+ * this list (moved verbatim from the web app's engine.ts in T009).
+ */
+export declare const REGEX_REGIONS: readonly ["all", "gb", "us", "pl", "de", "fr", "es", "pt", "se", "no", "it", "nl", "be", "at", "ch", "ie", "dk", "fi", "jp", "cn"];
+export type RegexRegionId = typeof REGEX_REGIONS[number];
+import type { RegexRule } from './types.js';
+/**
+ * All regex rules, compiled from the canonical rules/*.json packs (T018).
+ * The JSON is embedded at authoring time via the generated rules.data.ts;
+ * pack order there matches the pre-T018 hand-written concatenation order.
+ * An unknown validator name in the data fails right here, at module init.
+ */
+export declare const ALL_REGEX_RULES: RegexRule[];

package/dist/regex/index.js

@@ -0,0 +1,18 @@
+/**
+ * Region choices for the regex detector: 'all' plus every shipped region
+ * pack. Order is significant - hosts render region pickers straight from
+ * this list (moved verbatim from the web app's engine.ts in T009).
+ */
+export const REGEX_REGIONS = [
+    'all', 'gb', 'us', 'pl', 'de', 'fr', 'es', 'pt', 'se', 'no',
+    'it', 'nl', 'be', 'at', 'ch', 'ie', 'dk', 'fi', 'jp', 'cn',
+];
+import { loadRegexRules } from "./loader.js";
+import { RULES_DATA } from "./rules.data.js";
+/**
+ * All regex rules, compiled from the canonical rules/*.json packs (T018).
+ * The JSON is embedded at authoring time via the generated rules.data.ts;
+ * pack order there matches the pre-T018 hand-written concatenation order.
+ * An unknown validator name in the data fails right here, at module init.
+ */
+export const ALL_REGEX_RULES = loadRegexRules(RULES_DATA);

package/dist/regex/loader.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Loader for the shared rules format (T018, arch doc section 6).
+ *
+ * Turns the canonical rules/*.json data (embedded at authoring time as
+ * rules.data.ts) into runtime RegexRule objects: compiles each pattern with
+ * its declared flags and resolves `validate` names against the validator
+ * registry. An unknown validator name throws, which surfaces at module init
+ * time because index.ts calls loadRegexRules() at module scope.
+ */
+import type { EntityType } from '../types.js';
+import type { PiiDomain, RegexRule, RegionCode } from './types.js';
+/** JSON shape of a single rule inside a rules/<region>.json pack. */
+export interface RegexRuleJson {
+    /** Stable detector identifier, e.g. 'regex:pl:pesel' */
+    id: string;
+    /** Entity type the rule detects */
+    entityType: EntityType;
+    /** Regex source (ECMAScript/Python common subset) */
+    pattern: string;
+    /** Regex flags; must include 'g' */
+    flags: string;
+    /** Default confidence score (0-1) */
+    confidence: number;
+    /** Domain tags for categorization/filtering */
+    domains: PiiDomain[];
+    /** Human-readable description for developers maintaining rules */
+    description: string;
+    /** Example strings that SHOULD match (used in auto-generated tests) */
+    examples?: string[];
+    /** Known false-positive patterns or notes */
+    falsePositiveNotes?: string;
+    /** Name of a post-match validation function in the validator registry */
+    validate?: string;
+}
+/** JSON shape of a whole rules/<region>.json file. */
+export interface RegionRulesJson {
+    region: RegionCode;
+    rules: readonly RegexRuleJson[];
+}
+/**
+ * Compile rule packs into the flat RegexRule list, preserving pack order and
+ * in-pack rule order (the concatenation order defines ALL_REGEX_RULES).
+ */
+export declare function loadRegexRules(packs: readonly RegionRulesJson[]): RegexRule[];

package/dist/regex/loader.js

@@ -0,0 +1,44 @@
+import { VALIDATORS } from "./validators.js";
+function compileRule(region, json) {
+    let pattern;
+    try {
+        pattern = new RegExp(json.pattern, json.flags);
+    }
+    catch (err) {
+        throw new Error(`Rule ${json.id}: pattern does not compile with flags "${json.flags}": ${String(err)}`);
+    }
+    const rule = {
+        pattern,
+        type: json.entityType,
+        detector: json.id,
+        confidence: json.confidence,
+        region,
+        domains: json.domains,
+        description: json.description,
+    };
+    if (json.examples !== undefined)
+        rule.examples = json.examples;
+    if (json.falsePositiveNotes !== undefined)
+        rule.falsePositiveNotes = json.falsePositiveNotes;
+    if (json.validate !== undefined) {
+        const validate = VALIDATORS[json.validate];
+        if (validate === undefined) {
+            throw new Error(`Rule ${json.id}: unknown validator "${json.validate}" (not in the validators.ts registry)`);
+        }
+        rule.validate = validate;
+    }
+    return rule;
+}
+/**
+ * Compile rule packs into the flat RegexRule list, preserving pack order and
+ * in-pack rule order (the concatenation order defines ALL_REGEX_RULES).
+ */
+export function loadRegexRules(packs) {
+    const rules = [];
+    for (const pack of packs) {
+        for (const json of pack.rules) {
+            rules.push(compileRule(pack.region, json));
+        }
+    }
+    return rules;
+}

package/dist/regex/rules.data.d.ts

@@ -0,0 +1,2 @@
+import type { RegionRulesJson } from './loader.js';
+export declare const RULES_DATA: readonly RegionRulesJson[];