@doccloak/core 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +35 -0
  2. package/LICENSE +202 -0
  3. package/NOTICE +5 -0
  4. package/README.md +230 -0
  5. package/dist/doc.d.ts +13 -0
  6. package/dist/doc.js +683 -0
  7. package/dist/docx.d.ts +31 -0
  8. package/dist/docx.js +26 -0
  9. package/dist/dom/docx.d.ts +90 -0
  10. package/dist/dom/docx.js +472 -0
  11. package/dist/dom/index.d.ts +11 -0
  12. package/dist/dom/index.js +11 -0
  13. package/dist/dom/ocr.d.ts +112 -0
  14. package/dist/dom/ocr.js +224 -0
  15. package/dist/engine.d.ts +81 -0
  16. package/dist/engine.js +274 -0
  17. package/dist/env.d.ts +49 -0
  18. package/dist/env.js +53 -0
  19. package/dist/index.d.ts +20 -0
  20. package/dist/index.js +29 -0
  21. package/dist/model-loader.d.ts +55 -0
  22. package/dist/model-loader.js +253 -0
  23. package/dist/pipeline.d.ts +19 -0
  24. package/dist/pipeline.js +128 -0
  25. package/dist/providers/bardsai.d.ts +38 -0
  26. package/dist/providers/bardsai.js +393 -0
  27. package/dist/providers/gliner.d.ts +52 -0
  28. package/dist/providers/gliner.js +360 -0
  29. package/dist/regex/index.d.ts +17 -0
  30. package/dist/regex/index.js +18 -0
  31. package/dist/regex/loader.d.ts +44 -0
  32. package/dist/regex/loader.js +44 -0
  33. package/dist/regex/rules.data.d.ts +2 -0
  34. package/dist/regex/rules.data.js +2671 -0
  35. package/dist/regex/types.d.ts +27 -0
  36. package/dist/regex/types.js +1 -0
  37. package/dist/regex/validators.d.ts +17 -0
  38. package/dist/regex/validators.js +492 -0
  39. package/dist/session.d.ts +59 -0
  40. package/dist/session.js +153 -0
  41. package/dist/types.d.ts +42 -0
  42. package/dist/types.js +28 -0
  43. package/dist/worker-protocol.d.ts +119 -0
  44. package/dist/worker-protocol.js +285 -0
  45. package/package.json +75 -0
  46. package/rules/at.json +70 -0
  47. package/rules/be.json +70 -0
  48. package/rules/ch.json +66 -0
  49. package/rules/cn.json +155 -0
  50. package/rules/de.json +192 -0
  51. package/rules/dk.json +43 -0
  52. package/rules/es.json +158 -0
  53. package/rules/fi.json +43 -0
  54. package/rules/fr.json +178 -0
  55. package/rules/gb.json +194 -0
  56. package/rules/ie.json +48 -0
  57. package/rules/it.json +116 -0
  58. package/rules/jp.json +136 -0
  59. package/rules/nl.json +99 -0
  60. package/rules/no.json +132 -0
  61. package/rules/pl.json +210 -0
  62. package/rules/pt.json +113 -0
  63. package/rules/schema.json +91 -0
  64. package/rules/se.json +149 -0
  65. package/rules/universal.json +279 -0
  66. package/rules/us.json +218 -0
package/dist/env.d.ts ADDED
@@ -0,0 +1,49 @@
1
+ /**
2
+ * @doccloak/core - environment abstraction.
3
+ *
4
+ * Every host (web app, MV3 extension, tests) supplies a CoreEnv describing how
5
+ * the engine talks to the outside world. Core code never touches localStorage,
6
+ * chrome.storage, caches, navigator or import.meta directly.
7
+ *
8
+ * Interfaces match architecture doc section 4.2 verbatim.
9
+ */
10
+ /** Async key-value store for settings. Web: localStorage wrapper. Extension: chrome.storage.local. */
11
+ export interface KVStore {
12
+ get(key: string): Promise<string | null>;
13
+ set(key: string, value: string): Promise<void>;
14
+ remove(key: string): Promise<void>;
15
+ }
16
+ /** Blob cache for model files. Web + extension: Cache Storage. Tests: in-memory. */
17
+ export interface BlobCache {
18
+ match(url: string): Promise<Blob | undefined>;
19
+ put(url: string, blob: Blob): Promise<boolean>;
20
+ delete(url: string): Promise<void>;
21
+ }
22
+ export interface HardwareHints {
23
+ deviceMemoryGB?: number;
24
+ isMobile?: boolean;
25
+ }
26
+ export interface CoreEnv {
27
+ kv: KVStore;
28
+ modelCache: BlobCache;
29
+ fetch: typeof fetch;
30
+ wasm: {
31
+ paths: string;
32
+ numThreads?: number;
33
+ };
34
+ loadTokenizer(hfModelId: string): Promise<unknown>;
35
+ hardware?: HardwareHints;
36
+ persistStorage?: () => Promise<boolean>;
37
+ }
38
+ /** In-memory KVStore for tests. */
39
+ export declare function memoryKV(): KVStore;
40
+ export interface MemoryBlobCacheOptions {
41
+ /**
42
+ * Simulated storage quota in bytes. A put whose blob would push the total
43
+ * stored size above this limit is refused (resolves false), mirroring the
44
+ * best-effort Cache Storage behavior on quota pressure. Unlimited if omitted.
45
+ */
46
+ maxBytes?: number;
47
+ }
48
+ /** In-memory BlobCache for tests, with an optional simulated quota. */
49
+ export declare function memoryBlobCache(options?: MemoryBlobCacheOptions): BlobCache;
package/dist/env.js ADDED
@@ -0,0 +1,53 @@
1
+ /**
2
+ * @doccloak/core - environment abstraction.
3
+ *
4
+ * Every host (web app, MV3 extension, tests) supplies a CoreEnv describing how
5
+ * the engine talks to the outside world. Core code never touches localStorage,
6
+ * chrome.storage, caches, navigator or import.meta directly.
7
+ *
8
+ * Interfaces match architecture doc section 4.2 verbatim.
9
+ */
10
+ /** In-memory KVStore for tests. */
11
+ export function memoryKV() {
12
+ const store = new Map();
13
+ return {
14
+ async get(key) {
15
+ return store.has(key) ? store.get(key) : null;
16
+ },
17
+ async set(key, value) {
18
+ store.set(key, value);
19
+ },
20
+ async remove(key) {
21
+ store.delete(key);
22
+ },
23
+ };
24
+ }
25
+ /** In-memory BlobCache for tests, with an optional simulated quota. */
26
+ export function memoryBlobCache(options = {}) {
27
+ const { maxBytes } = options;
28
+ const store = new Map();
29
+ function usedBytes() {
30
+ let total = 0;
31
+ for (const blob of store.values())
32
+ total += blob.size;
33
+ return total;
34
+ }
35
+ return {
36
+ async match(url) {
37
+ return store.get(url);
38
+ },
39
+ async put(url, blob) {
40
+ if (maxBytes !== undefined) {
41
+ const existing = store.get(url);
42
+ const sizeAfter = usedBytes() - (existing?.size ?? 0) + blob.size;
43
+ if (sizeAfter > maxBytes)
44
+ return false;
45
+ }
46
+ store.set(url, blob);
47
+ return true;
48
+ },
49
+ async delete(url) {
50
+ store.delete(url);
51
+ },
52
+ };
53
+ }
@@ -0,0 +1,20 @@
1
+ /**
2
+ * @doccloak/core - public API surface.
3
+ */
4
+ export type { KVStore, BlobCache, HardwareHints, CoreEnv, MemoryBlobCacheOptions, } from './env.js';
5
+ export { memoryKV, memoryBlobCache } from './env.js';
6
+ export declare const CORE_PACKAGE_NAME = "@doccloak/core";
7
+ export { fetchModelBlob, evictModelFromCache, retryAsync } from './model-loader.js';
8
+ export type { ModelLoaderEnv, DownloadProgress, FetchModelOptions } from './model-loader.js';
9
+ export { ALL_REGEX_RULES, REGEX_REGIONS } from './regex/index.js';
10
+ export type { RegexRule, PiiDomain, RegionCode, RegexRegionId } from './regex/index.js';
11
+ export * from './types.js';
12
+ export * from './session.js';
13
+ export * from './doc.js';
14
+ export { GlinerProvider } from './providers/gliner.js';
15
+ export { BardsaiProvider } from './providers/bardsai.js';
16
+ export { detectWithRegex, resolveOverlaps, filterFalsePositives, propagateEntities, detectEntities, } from './pipeline.js';
17
+ export { createEngine, PROVIDERS, pickDefaultProvider, defaultThresholdFor, clampThreshold, ENGINE_SETTINGS_KEYS, } from './engine.js';
18
+ export type { ProviderId, ProviderEntry, ProviderFactory, EngineOptions, EngineSettings, DocCloakEngine, } from './engine.js';
19
+ export { serveEngine, connectEngine } from './worker-protocol.js';
20
+ export type { PortLike, EngineClient, EngineRequest, EngineResponse, } from './worker-protocol.js';
package/dist/index.js ADDED
@@ -0,0 +1,29 @@
1
+ /**
2
+ * @doccloak/core - public API surface.
3
+ */
4
+ export { memoryKV, memoryBlobCache } from "./env.js";
5
+ export const CORE_PACKAGE_NAME = '@doccloak/core';
6
+ // T006: model loader
7
+ export { fetchModelBlob, evictModelFromCache, retryAsync } from "./model-loader.js";
8
+ // end T006
9
+ // T004: regex rules (REGEX_REGIONS moved here from the web engine in T009)
10
+ export { ALL_REGEX_RULES, REGEX_REGIONS } from "./regex/index.js";
11
+ // end T004
12
+ // T003: types / session / doc
13
+ export * from "./types.js";
14
+ export * from "./session.js";
15
+ export * from "./doc.js";
16
+ // end T003
17
+ // T007: gliner provider
18
+ export { GlinerProvider } from "./providers/gliner.js";
19
+ // end T007
20
+ // T008: bardsai provider
21
+ export { BardsaiProvider } from "./providers/bardsai.js";
22
+ // end T008
23
+ // T005: pipeline
24
+ export { detectWithRegex, resolveOverlaps, filterFalsePositives, propagateEntities, detectEntities, } from "./pipeline.js";
25
+ // end T005
26
+ // T009: engine assembly + worker protocol
27
+ export { createEngine, PROVIDERS, pickDefaultProvider, defaultThresholdFor, clampThreshold, ENGINE_SETTINGS_KEYS, } from "./engine.js";
28
+ export { serveEngine, connectEngine } from "./worker-protocol.js";
29
+ // end T009
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Shared model download utility used by all detection providers.
3
+ *
4
+ * Designed for large downloads (65-280 MB) on unreliable connections,
5
+ * especially mobile networks:
6
+ * - Resumes interrupted downloads with HTTP Range requests instead of
7
+ * restarting from zero.
8
+ * - Retries transient failures (network errors, stalls, 5xx) with
9
+ * exponential backoff.
10
+ * - Aborts stalled connections via a watchdog so a dead socket surfaces
11
+ * as a retryable error instead of hanging forever.
12
+ * - Throttles progress callbacks so the UI thread is not flooded with
13
+ * thousands of postMessage/setState calls during the download.
14
+ * - Serves the model from the injected BlobCache when available and prefers
15
+ * the cached (typically disk-backed) copy over the in-memory one to reduce
16
+ * peak RAM, which matters on iOS Safari where large tabs get killed.
17
+ *
18
+ * Environment-agnostic: all host capabilities (blob cache, fetch, persistent
19
+ * storage request) are injected via ModelLoaderEnv. This module never touches
20
+ * caches, navigator or global fetch directly.
21
+ */
22
+ import type { BlobCache } from './env.js';
23
+ export type DownloadProgress = (downloaded: number, total: number) => void;
24
+ /**
25
+ * Host capabilities the loader needs. Hosts pass the relevant slice of their
26
+ * CoreEnv (modelCache as cache, fetch, persistStorage).
27
+ */
28
+ export interface ModelLoaderEnv {
29
+ /** Blob cache for downloaded models. Web/extension: Cache Storage adapter. Tests: memoryBlobCache. */
30
+ cache: BlobCache;
31
+ /** Network access. Injectable for tests and proxies. */
32
+ fetch: typeof fetch;
33
+ /** Best-effort persistent-storage request (navigator.storage.persist() on the web). */
34
+ persistStorage?: () => Promise<boolean>;
35
+ }
36
+ export interface FetchModelOptions {
37
+ maxAttempts?: number;
38
+ retryBaseDelayMs?: number;
39
+ }
40
+ /**
41
+ * Download a model with progress reporting, resume and retry.
42
+ * Returns a Blob (served from the injected cache when possible).
43
+ */
44
+ export declare function fetchModelBlob(env: ModelLoaderEnv, url: string, onProgress?: DownloadProgress, options?: FetchModelOptions): Promise<Blob>;
45
+ /**
46
+ * Remove a single model from the cache (e.g. when its URL changes).
47
+ */
48
+ export declare function evictModelFromCache(env: ModelLoaderEnv, url: string): Promise<void>;
49
+ /**
50
+ * Retry an async operation with exponential backoff. Used for the smaller
51
+ * companion downloads (tokenizer files) that go through libraries without
52
+ * their own retry handling - one dropped request on a flaky connection
53
+ * should not abort the whole model load.
54
+ */
55
+ export declare function retryAsync<T>(operation: () => Promise<T>, label: string, options?: FetchModelOptions): Promise<T>;
@@ -0,0 +1,253 @@
1
+ /**
2
+ * Shared model download utility used by all detection providers.
3
+ *
4
+ * Designed for large downloads (65-280 MB) on unreliable connections,
5
+ * especially mobile networks:
6
+ * - Resumes interrupted downloads with HTTP Range requests instead of
7
+ * restarting from zero.
8
+ * - Retries transient failures (network errors, stalls, 5xx) with
9
+ * exponential backoff.
10
+ * - Aborts stalled connections via a watchdog so a dead socket surfaces
11
+ * as a retryable error instead of hanging forever.
12
+ * - Throttles progress callbacks so the UI thread is not flooded with
13
+ * thousands of postMessage/setState calls during the download.
14
+ * - Serves the model from the injected BlobCache when available and prefers
15
+ * the cached (typically disk-backed) copy over the in-memory one to reduce
16
+ * peak RAM, which matters on iOS Safari where large tabs get killed.
17
+ *
18
+ * Environment-agnostic: all host capabilities (blob cache, fetch, persistent
19
+ * storage request) are injected via ModelLoaderEnv. This module never touches
20
+ * caches, navigator or global fetch directly.
21
+ */
22
+ const MAX_ATTEMPTS = 4;
23
+ const STALL_TIMEOUT_MS = 30000;
24
+ const PROGRESS_INTERVAL_MS = 150;
25
+ const RETRY_BASE_DELAY_MS = 1000;
26
+ class HttpError extends Error {
27
+ constructor(status, url) {
28
+ super(`Failed to download model: HTTP ${status} (${url})`);
29
+ this.name = 'HttpError';
30
+ this.status = status;
31
+ }
32
+ }
33
+ function isRetryable(err) {
34
+ if (err instanceof HttpError) {
35
+ return err.status === 408 || err.status === 429 || err.status >= 500;
36
+ }
37
+ // AbortError comes from our stall watchdog; TypeError is fetch's generic
38
+ // network failure (connection reset, DNS, CORS transport error, ...).
39
+ if (err instanceof DOMException && err.name === 'AbortError')
40
+ return true;
41
+ if (err instanceof TypeError)
42
+ return true;
43
+ return false;
44
+ }
45
+ function delay(ms) {
46
+ return new Promise((resolve) => setTimeout(resolve, ms));
47
+ }
48
+ let persistRequested = false;
49
+ /**
50
+ * Best-effort request for persistent storage so the browser does not evict
51
+ * the cached model under storage pressure (common on mobile). Requested at
52
+ * most once per session, matching the previous web-only behavior.
53
+ */
54
+ async function requestPersistentStorage(env) {
55
+ if (persistRequested || !env.persistStorage)
56
+ return;
57
+ persistRequested = true;
58
+ try {
59
+ await env.persistStorage();
60
+ }
61
+ catch { /* not supported or denied - caching still works, just evictable */ }
62
+ }
63
+ async function readFromCache(cache, url) {
64
+ try {
65
+ const cached = await cache.match(url);
66
+ if (cached)
67
+ return cached;
68
+ }
69
+ catch { /* corrupt entry or read failure - fall through to network */ }
70
+ return null;
71
+ }
72
+ /**
73
+ * Best-effort cache put - large models can exceed per-origin quota, which
74
+ * surfaces either as put() resolving false or as a thrown error depending on
75
+ * the adapter. Caching is an optimisation, not a correctness requirement.
76
+ */
77
+ async function tryCachePut(cache, url, blob) {
78
+ try {
79
+ const stored = await cache.put(url, blob);
80
+ if (!stored) {
81
+ console.warn('[DocCloak] Model cache put refused (model still loaded in memory)');
82
+ }
83
+ }
84
+ catch (err) {
85
+ console.warn('[DocCloak] Model cache put failed (model still loaded in memory):', err);
86
+ }
87
+ }
88
+ /**
89
+ * One download attempt. Streams the body through onChunk. A watchdog aborts
90
+ * the request if no bytes arrive for STALL_TIMEOUT_MS, covering both the
91
+ * initial connection and mid-body stalls. onHeaders fires as soon as the
92
+ * response headers are parsed - before the body streams - so callers can
93
+ * report a correct total alongside per-chunk progress.
94
+ */
95
+ async function downloadAttempt(fetchFn, url, offset, onChunk, onHeaders) {
96
+ const controller = new AbortController();
97
+ let watchdog = setTimeout(() => controller.abort(), STALL_TIMEOUT_MS);
98
+ const kickWatchdog = () => {
99
+ clearTimeout(watchdog);
100
+ watchdog = setTimeout(() => controller.abort(), STALL_TIMEOUT_MS);
101
+ };
102
+ try {
103
+ const headers = {};
104
+ if (offset > 0)
105
+ headers['Range'] = `bytes=${offset}-`;
106
+ const response = await fetchFn(url, { headers, signal: controller.signal });
107
+ kickWatchdog();
108
+ if (!response.ok)
109
+ throw new HttpError(response.status, url);
110
+ const resumed = response.status === 206;
111
+ let total = 0;
112
+ if (resumed) {
113
+ // Content-Range: bytes <start>-<end>/<total>
114
+ const contentRange = response.headers.get('content-range');
115
+ const match = contentRange?.match(/\/(\d+)\s*$/);
116
+ if (match)
117
+ total = parseInt(match[1], 10);
118
+ }
119
+ else {
120
+ const contentLength = response.headers.get('content-length');
121
+ if (contentLength)
122
+ total = parseInt(contentLength, 10);
123
+ }
124
+ onHeaders?.({ total, resumed });
125
+ if (!response.body) {
126
+ // No streaming support - read in one shot (no resume possible, but
127
+ // the retry loop still restarts from scratch on failure).
128
+ const buffer = await response.arrayBuffer();
129
+ onChunk(new Uint8Array(buffer));
130
+ return { total, resumed };
131
+ }
132
+ const reader = response.body.getReader();
133
+ while (true) {
134
+ const { done, value } = await reader.read();
135
+ if (done)
136
+ break;
137
+ kickWatchdog();
138
+ onChunk(value);
139
+ }
140
+ return { total, resumed };
141
+ }
142
+ finally {
143
+ clearTimeout(watchdog);
144
+ }
145
+ }
146
+ /**
147
+ * Download a model with progress reporting, resume and retry.
148
+ * Returns a Blob (served from the injected cache when possible).
149
+ */
150
+ export async function fetchModelBlob(env, url, onProgress, options) {
151
+ const maxAttempts = options?.maxAttempts ?? MAX_ATTEMPTS;
152
+ const retryBaseDelayMs = options?.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS;
153
+ const cache = env.cache;
154
+ const cachedBlob = await readFromCache(cache, url);
155
+ if (cachedBlob) {
156
+ onProgress?.(cachedBlob.size, cachedBlob.size);
157
+ return cachedBlob;
158
+ }
159
+ await requestPersistentStorage(env);
160
+ let lastProgressAt = 0;
161
+ const reportProgress = (downloaded, total, force = false) => {
162
+ const now = Date.now();
163
+ if (!force && now - lastProgressAt < PROGRESS_INTERVAL_MS)
164
+ return;
165
+ lastProgressAt = now;
166
+ onProgress?.(downloaded, total);
167
+ };
168
+ let chunks = [];
169
+ let downloaded = 0;
170
+ let total = 0;
171
+ for (let attempt = 1;; attempt++) {
172
+ try {
173
+ const requestedOffset = downloaded;
174
+ const result = await downloadAttempt(env.fetch, url, requestedOffset, (chunk) => {
175
+ chunks.push(chunk);
176
+ downloaded += chunk.length;
177
+ reportProgress(downloaded, total);
178
+ }, (info) => {
179
+ if (info.total > 0)
180
+ total = info.resumed ? info.total : Math.max(total, info.total);
181
+ });
182
+ if (requestedOffset > 0 && !result.resumed) {
183
+ // Server ignored the Range header and sent the file from the start.
184
+ // The freshly received chunks already replace everything, but they
185
+ // were appended after the stale ones - rebuild keeping only bytes
186
+ // from this attempt.
187
+ const freshBytes = downloaded - requestedOffset;
188
+ const fresh = [];
189
+ let need = freshBytes;
190
+ for (let i = chunks.length - 1; i >= 0 && need > 0; i--) {
191
+ fresh.unshift(chunks[i]);
192
+ need -= chunks[i].length;
193
+ }
194
+ chunks = fresh;
195
+ downloaded = freshBytes;
196
+ }
197
+ if (result.total > 0)
198
+ total = result.resumed ? result.total : Math.max(total, result.total);
199
+ if (total > 0 && downloaded < total) {
200
+ // Connection closed early without an error. Treat as retryable.
201
+ throw new DOMException('Download ended before completion', 'AbortError');
202
+ }
203
+ break;
204
+ }
205
+ catch (err) {
206
+ if (!isRetryable(err) || attempt >= maxAttempts) {
207
+ throw err instanceof Error ? err : new Error(String(err));
208
+ }
209
+ console.warn(`[DocCloak] Model download interrupted at ${downloaded} bytes (attempt ${attempt}/${maxAttempts}), retrying...`, err);
210
+ await delay(retryBaseDelayMs * 2 ** (attempt - 1));
211
+ }
212
+ }
213
+ reportProgress(downloaded, total || downloaded, true);
214
+ let blob = new Blob(chunks);
215
+ chunks = [];
216
+ await tryCachePut(cache, url, blob);
217
+ // Prefer the cached (disk-backed) copy so the in-memory chunks can be
218
+ // collected before ONNX Runtime allocates its own copy of the model.
219
+ const diskBlob = await readFromCache(cache, url);
220
+ if (diskBlob && diskBlob.size === blob.size)
221
+ blob = diskBlob;
222
+ return blob;
223
+ }
224
+ /**
225
+ * Remove a single model from the cache (e.g. when its URL changes).
226
+ */
227
+ export async function evictModelFromCache(env, url) {
228
+ try {
229
+ await env.cache.delete(url);
230
+ }
231
+ catch { /* ignore */ }
232
+ }
233
+ /**
234
+ * Retry an async operation with exponential backoff. Used for the smaller
235
+ * companion downloads (tokenizer files) that go through libraries without
236
+ * their own retry handling - one dropped request on a flaky connection
237
+ * should not abort the whole model load.
238
+ */
239
+ export async function retryAsync(operation, label, options) {
240
+ const maxAttempts = options?.maxAttempts ?? MAX_ATTEMPTS;
241
+ const retryBaseDelayMs = options?.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS;
242
+ for (let attempt = 1;; attempt++) {
243
+ try {
244
+ return await operation();
245
+ }
246
+ catch (err) {
247
+ if (attempt >= maxAttempts)
248
+ throw err;
249
+ console.warn(`[DocCloak] ${label} failed (attempt ${attempt}/${maxAttempts}), retrying...`, err);
250
+ await delay(retryBaseDelayMs * 2 ** (attempt - 1));
251
+ }
252
+ }
253
+ }
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Pure entity pipeline (T005).
3
+ *
4
+ * Extracted verbatim from the web app's detection.worker.ts: text and
5
+ * entities in, entities out. No worker messaging, no providers, no
6
+ * environment access. The worker (and any other host) composes these
7
+ * functions around its own ML provider call.
8
+ *
9
+ * Function bodies are byte-identical to the originals by design; only the
10
+ * signatures of detectWithRegex (regexRegion becomes a parameter instead of
11
+ * worker module state) and detectEntities (ML results are passed in instead
12
+ * of fetched from a provider) differ.
13
+ */
14
+ import type { DetectedEntity } from './types.js';
15
+ export declare function detectWithRegex(text: string, regexRegion?: string): DetectedEntity[];
16
+ export declare function resolveOverlaps(entities: DetectedEntity[]): DetectedEntity[];
17
+ export declare function filterFalsePositives(entities: DetectedEntity[]): DetectedEntity[];
18
+ export declare function propagateEntities(text: string, entities: DetectedEntity[]): DetectedEntity[];
19
+ export declare function detectEntities(text: string, mlEntities: DetectedEntity[], regexEntities: DetectedEntity[]): DetectedEntity[];
@@ -0,0 +1,128 @@
1
+ /**
2
+ * Pure entity pipeline (T005).
3
+ *
4
+ * Extracted verbatim from the web app's detection.worker.ts: text and
5
+ * entities in, entities out. No worker messaging, no providers, no
6
+ * environment access. The worker (and any other host) composes these
7
+ * functions around its own ML provider call.
8
+ *
9
+ * Function bodies are byte-identical to the originals by design; only the
10
+ * signatures of detectWithRegex (regexRegion becomes a parameter instead of
11
+ * worker module state) and detectEntities (ML results are passed in instead
12
+ * of fetched from a provider) differ.
13
+ */
14
+ import { ALL_REGEX_RULES } from "./regex/index.js";
15
+ export function detectWithRegex(text, regexRegion = 'all') {
16
+ const entities = [];
17
+ const rules = regexRegion === 'all'
18
+ ? ALL_REGEX_RULES
19
+ : ALL_REGEX_RULES.filter((r) => r.region === 'universal' || r.region === regexRegion);
20
+ for (const rule of rules) {
21
+ rule.pattern.lastIndex = 0;
22
+ let match;
23
+ while ((match = rule.pattern.exec(text)) !== null) {
24
+ if (rule.validate && !rule.validate(match[0]))
25
+ continue;
26
+ entities.push({
27
+ type: rule.type,
28
+ value: match[0],
29
+ start: match.index,
30
+ end: match.index + match[0].length,
31
+ confidence: rule.confidence,
32
+ detector: rule.detector,
33
+ });
34
+ }
35
+ }
36
+ return entities;
37
+ }
38
+ export function resolveOverlaps(entities) {
39
+ const sorted = [...entities].sort((a, b) => {
40
+ if (a.start !== b.start)
41
+ return a.start - b.start;
42
+ const aLen = a.end - a.start;
43
+ const bLen = b.end - b.start;
44
+ if (aLen !== bLen)
45
+ return bLen - aLen;
46
+ return b.confidence - a.confidence;
47
+ });
48
+ const resolved = [];
49
+ for (const entity of sorted) {
50
+ const overlaps = resolved.some((existing) => entity.start < existing.end && entity.end > existing.start);
51
+ if (!overlaps) {
52
+ resolved.push(entity);
53
+ }
54
+ }
55
+ return resolved;
56
+ }
57
+ export function filterFalsePositives(entities) {
58
+ return entities.filter((e) => {
59
+ if (e.value.trim().length < 3)
60
+ return false;
61
+ return true;
62
+ });
63
+ }
64
+ export function propagateEntities(text, entities) {
65
+ const propagated = [];
66
+ const seen = new Set();
67
+ for (const e of entities) {
68
+ seen.add(`${e.start}:${e.end}`);
69
+ }
70
+ const terms = [];
71
+ const addedTerms = new Set();
72
+ for (const e of entities) {
73
+ const val = e.value.trim();
74
+ if (val.length >= 3 && !addedTerms.has(val.toLowerCase())) {
75
+ addedTerms.add(val.toLowerCase());
76
+ terms.push({ term: val, type: e.type });
77
+ }
78
+ if (val.includes(' ')) {
79
+ for (const word of val.split(/\s+/)) {
80
+ const w = word.replace(/[.,;:!?()]+$/, '');
81
+ if (w.length >= 4 && !addedTerms.has(w.toLowerCase())) {
82
+ addedTerms.add(w.toLowerCase());
83
+ terms.push({ term: w, type: e.type });
84
+ }
85
+ }
86
+ }
87
+ }
88
+ for (const { term, type } of terms) {
89
+ let searchFrom = 0;
90
+ const lowerText = text.toLowerCase();
91
+ const lowerTerm = term.toLowerCase();
92
+ while (searchFrom < text.length) {
93
+ const idx = lowerText.indexOf(lowerTerm, searchFrom);
94
+ if (idx === -1)
95
+ break;
96
+ const end = idx + term.length;
97
+ const key = `${idx}:${end}`;
98
+ if (!seen.has(key)) {
99
+ const charBefore = idx > 0 ? text[idx - 1] : ' ';
100
+ const charAfter = end < text.length ? text[end] : ' ';
101
+ const isBoundaryBefore = /[\s.,;:!?()"'„"\-–—/]/.test(charBefore);
102
+ const isBoundaryAfter = /[\s.,;:!?()"'„"\-–—/]/.test(charAfter);
103
+ if (isBoundaryBefore && isBoundaryAfter) {
104
+ seen.add(key);
105
+ propagated.push({
106
+ type,
107
+ value: text.slice(idx, end),
108
+ start: idx,
109
+ end,
110
+ confidence: 0.95,
111
+ detector: 'propagated',
112
+ });
113
+ }
114
+ }
115
+ searchFrom = idx + 1;
116
+ }
117
+ }
118
+ return propagated;
119
+ }
120
+ export function detectEntities(text, mlEntities, regexEntities) {
121
+ if (!text.trim())
122
+ return [];
123
+ const mlResults = filterFalsePositives(mlEntities);
124
+ const regexResults = regexEntities;
125
+ const initial = resolveOverlaps([...mlResults, ...regexResults]);
126
+ const propagated = propagateEntities(text, initial);
127
+ return resolveOverlaps([...initial, ...propagated]);
128
+ }
@@ -0,0 +1,38 @@
1
+ /**
2
+ * BardS.ai EU PII Anonymization Provider
3
+ *
4
+ * XLM-RoBERTa-base fine-tuned for EU PII detection (24 EU languages, natively multilingual).
5
+ * 35 entity types, ~0.3B parameters, quantized ONNX (~279 MB).
6
+ * Model: bardsai/eu-pii-anonimization-multilang
7
+ * Runs in-browser via ONNX Runtime WebAssembly.
8
+ *
9
+ * Environment-agnostic: wasm paths, model bytes (blob cache + fetch) and the
10
+ * tokenizer all arrive through the injected CoreEnv. This module never touches
11
+ * import.meta, caches, navigator or @huggingface/transformers directly.
12
+ */
13
+ import type { DetectedEntity, DetectionProvider, ProgressCallback } from '../types.js';
14
+ import type { CoreEnv } from '../env.js';
15
+ export declare class BardsaiProvider implements DetectionProvider {
16
+ private _name;
17
+ get name(): string;
18
+ private readonly env;
19
+ private tokenizer;
20
+ private session;
21
+ private loading;
22
+ private loadError;
23
+ private loadWaiters;
24
+ private progressCallback;
25
+ private threshold;
26
+ constructor(env: CoreEnv);
27
+ isLoaded(): boolean;
28
+ isLoading(): boolean;
29
+ onProgress(callback: ProgressCallback): void;
30
+ setThreshold(value: number): void;
31
+ getThreshold(): number;
32
+ load(): Promise<void>;
33
+ release(): void;
34
+ detect(text: string, onProgress?: (progress: number) => void): Promise<DetectedEntity[]>;
35
+ private inferChunk;
36
+ private emitEntity;
37
+ private deduplicateSpans;
38
+ }