@doccloak/core 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/LICENSE +202 -0
- package/NOTICE +5 -0
- package/README.md +230 -0
- package/dist/doc.d.ts +13 -0
- package/dist/doc.js +683 -0
- package/dist/docx.d.ts +31 -0
- package/dist/docx.js +26 -0
- package/dist/dom/docx.d.ts +90 -0
- package/dist/dom/docx.js +472 -0
- package/dist/dom/index.d.ts +11 -0
- package/dist/dom/index.js +11 -0
- package/dist/dom/ocr.d.ts +112 -0
- package/dist/dom/ocr.js +224 -0
- package/dist/engine.d.ts +81 -0
- package/dist/engine.js +274 -0
- package/dist/env.d.ts +49 -0
- package/dist/env.js +53 -0
- package/dist/index.d.ts +20 -0
- package/dist/index.js +29 -0
- package/dist/model-loader.d.ts +55 -0
- package/dist/model-loader.js +253 -0
- package/dist/pipeline.d.ts +19 -0
- package/dist/pipeline.js +128 -0
- package/dist/providers/bardsai.d.ts +38 -0
- package/dist/providers/bardsai.js +393 -0
- package/dist/providers/gliner.d.ts +52 -0
- package/dist/providers/gliner.js +360 -0
- package/dist/regex/index.d.ts +17 -0
- package/dist/regex/index.js +18 -0
- package/dist/regex/loader.d.ts +44 -0
- package/dist/regex/loader.js +44 -0
- package/dist/regex/rules.data.d.ts +2 -0
- package/dist/regex/rules.data.js +2671 -0
- package/dist/regex/types.d.ts +27 -0
- package/dist/regex/types.js +1 -0
- package/dist/regex/validators.d.ts +17 -0
- package/dist/regex/validators.js +492 -0
- package/dist/session.d.ts +59 -0
- package/dist/session.js +153 -0
- package/dist/types.d.ts +42 -0
- package/dist/types.js +28 -0
- package/dist/worker-protocol.d.ts +119 -0
- package/dist/worker-protocol.js +285 -0
- package/package.json +75 -0
- package/rules/at.json +70 -0
- package/rules/be.json +70 -0
- package/rules/ch.json +66 -0
- package/rules/cn.json +155 -0
- package/rules/de.json +192 -0
- package/rules/dk.json +43 -0
- package/rules/es.json +158 -0
- package/rules/fi.json +43 -0
- package/rules/fr.json +178 -0
- package/rules/gb.json +194 -0
- package/rules/ie.json +48 -0
- package/rules/it.json +116 -0
- package/rules/jp.json +136 -0
- package/rules/nl.json +99 -0
- package/rules/no.json +132 -0
- package/rules/pl.json +210 -0
- package/rules/pt.json +113 -0
- package/rules/schema.json +91 -0
- package/rules/se.json +149 -0
- package/rules/universal.json +279 -0
- package/rules/us.json +218 -0
package/dist/env.d.ts
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @doccloak/core - environment abstraction.
|
|
3
|
+
*
|
|
4
|
+
* Every host (web app, MV3 extension, tests) supplies a CoreEnv describing how
|
|
5
|
+
* the engine talks to the outside world. Core code never touches localStorage,
|
|
6
|
+
* chrome.storage, caches, navigator or import.meta directly.
|
|
7
|
+
*
|
|
8
|
+
* Interfaces match architecture doc section 4.2 verbatim.
|
|
9
|
+
*/
|
|
10
|
+
/** Async key-value store for settings. Web: localStorage wrapper. Extension: chrome.storage.local. */
|
|
11
|
+
export interface KVStore {
|
|
12
|
+
get(key: string): Promise<string | null>;
|
|
13
|
+
set(key: string, value: string): Promise<void>;
|
|
14
|
+
remove(key: string): Promise<void>;
|
|
15
|
+
}
|
|
16
|
+
/** Blob cache for model files. Web + extension: Cache Storage. Tests: in-memory. */
|
|
17
|
+
export interface BlobCache {
|
|
18
|
+
match(url: string): Promise<Blob | undefined>;
|
|
19
|
+
put(url: string, blob: Blob): Promise<boolean>;
|
|
20
|
+
delete(url: string): Promise<void>;
|
|
21
|
+
}
|
|
22
|
+
export interface HardwareHints {
|
|
23
|
+
deviceMemoryGB?: number;
|
|
24
|
+
isMobile?: boolean;
|
|
25
|
+
}
|
|
26
|
+
export interface CoreEnv {
|
|
27
|
+
kv: KVStore;
|
|
28
|
+
modelCache: BlobCache;
|
|
29
|
+
fetch: typeof fetch;
|
|
30
|
+
wasm: {
|
|
31
|
+
paths: string;
|
|
32
|
+
numThreads?: number;
|
|
33
|
+
};
|
|
34
|
+
loadTokenizer(hfModelId: string): Promise<unknown>;
|
|
35
|
+
hardware?: HardwareHints;
|
|
36
|
+
persistStorage?: () => Promise<boolean>;
|
|
37
|
+
}
|
|
38
|
+
/** In-memory KVStore for tests. */
|
|
39
|
+
export declare function memoryKV(): KVStore;
|
|
40
|
+
export interface MemoryBlobCacheOptions {
|
|
41
|
+
/**
|
|
42
|
+
* Simulated storage quota in bytes. A put whose blob would push the total
|
|
43
|
+
* stored size above this limit is refused (resolves false), mirroring the
|
|
44
|
+
* best-effort Cache Storage behavior on quota pressure. Unlimited if omitted.
|
|
45
|
+
*/
|
|
46
|
+
maxBytes?: number;
|
|
47
|
+
}
|
|
48
|
+
/** In-memory BlobCache for tests, with an optional simulated quota. */
|
|
49
|
+
export declare function memoryBlobCache(options?: MemoryBlobCacheOptions): BlobCache;
|
package/dist/env.js
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @doccloak/core - environment abstraction.
|
|
3
|
+
*
|
|
4
|
+
* Every host (web app, MV3 extension, tests) supplies a CoreEnv describing how
|
|
5
|
+
* the engine talks to the outside world. Core code never touches localStorage,
|
|
6
|
+
* chrome.storage, caches, navigator or import.meta directly.
|
|
7
|
+
*
|
|
8
|
+
* Interfaces match architecture doc section 4.2 verbatim.
|
|
9
|
+
*/
|
|
10
|
+
/** In-memory KVStore for tests. */
|
|
11
|
+
export function memoryKV() {
|
|
12
|
+
const store = new Map();
|
|
13
|
+
return {
|
|
14
|
+
async get(key) {
|
|
15
|
+
return store.has(key) ? store.get(key) : null;
|
|
16
|
+
},
|
|
17
|
+
async set(key, value) {
|
|
18
|
+
store.set(key, value);
|
|
19
|
+
},
|
|
20
|
+
async remove(key) {
|
|
21
|
+
store.delete(key);
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
/** In-memory BlobCache for tests, with an optional simulated quota. */
|
|
26
|
+
export function memoryBlobCache(options = {}) {
|
|
27
|
+
const { maxBytes } = options;
|
|
28
|
+
const store = new Map();
|
|
29
|
+
function usedBytes() {
|
|
30
|
+
let total = 0;
|
|
31
|
+
for (const blob of store.values())
|
|
32
|
+
total += blob.size;
|
|
33
|
+
return total;
|
|
34
|
+
}
|
|
35
|
+
return {
|
|
36
|
+
async match(url) {
|
|
37
|
+
return store.get(url);
|
|
38
|
+
},
|
|
39
|
+
async put(url, blob) {
|
|
40
|
+
if (maxBytes !== undefined) {
|
|
41
|
+
const existing = store.get(url);
|
|
42
|
+
const sizeAfter = usedBytes() - (existing?.size ?? 0) + blob.size;
|
|
43
|
+
if (sizeAfter > maxBytes)
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
store.set(url, blob);
|
|
47
|
+
return true;
|
|
48
|
+
},
|
|
49
|
+
async delete(url) {
|
|
50
|
+
store.delete(url);
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @doccloak/core - public API surface.
|
|
3
|
+
*/
|
|
4
|
+
export type { KVStore, BlobCache, HardwareHints, CoreEnv, MemoryBlobCacheOptions, } from './env.js';
|
|
5
|
+
export { memoryKV, memoryBlobCache } from './env.js';
|
|
6
|
+
export declare const CORE_PACKAGE_NAME = "@doccloak/core";
|
|
7
|
+
export { fetchModelBlob, evictModelFromCache, retryAsync } from './model-loader.js';
|
|
8
|
+
export type { ModelLoaderEnv, DownloadProgress, FetchModelOptions } from './model-loader.js';
|
|
9
|
+
export { ALL_REGEX_RULES, REGEX_REGIONS } from './regex/index.js';
|
|
10
|
+
export type { RegexRule, PiiDomain, RegionCode, RegexRegionId } from './regex/index.js';
|
|
11
|
+
export * from './types.js';
|
|
12
|
+
export * from './session.js';
|
|
13
|
+
export * from './doc.js';
|
|
14
|
+
export { GlinerProvider } from './providers/gliner.js';
|
|
15
|
+
export { BardsaiProvider } from './providers/bardsai.js';
|
|
16
|
+
export { detectWithRegex, resolveOverlaps, filterFalsePositives, propagateEntities, detectEntities, } from './pipeline.js';
|
|
17
|
+
export { createEngine, PROVIDERS, pickDefaultProvider, defaultThresholdFor, clampThreshold, ENGINE_SETTINGS_KEYS, } from './engine.js';
|
|
18
|
+
export type { ProviderId, ProviderEntry, ProviderFactory, EngineOptions, EngineSettings, DocCloakEngine, } from './engine.js';
|
|
19
|
+
export { serveEngine, connectEngine } from './worker-protocol.js';
|
|
20
|
+
export type { PortLike, EngineClient, EngineRequest, EngineResponse, } from './worker-protocol.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @doccloak/core - public API surface.
|
|
3
|
+
*/
|
|
4
|
+
export { memoryKV, memoryBlobCache } from "./env.js";
|
|
5
|
+
export const CORE_PACKAGE_NAME = '@doccloak/core';
|
|
6
|
+
// T006: model loader
|
|
7
|
+
export { fetchModelBlob, evictModelFromCache, retryAsync } from "./model-loader.js";
|
|
8
|
+
// end T006
|
|
9
|
+
// T004: regex rules (REGEX_REGIONS moved here from the web engine in T009)
|
|
10
|
+
export { ALL_REGEX_RULES, REGEX_REGIONS } from "./regex/index.js";
|
|
11
|
+
// end T004
|
|
12
|
+
// T003: types / session / doc
|
|
13
|
+
export * from "./types.js";
|
|
14
|
+
export * from "./session.js";
|
|
15
|
+
export * from "./doc.js";
|
|
16
|
+
// end T003
|
|
17
|
+
// T007: gliner provider
|
|
18
|
+
export { GlinerProvider } from "./providers/gliner.js";
|
|
19
|
+
// end T007
|
|
20
|
+
// T008: bardsai provider
|
|
21
|
+
export { BardsaiProvider } from "./providers/bardsai.js";
|
|
22
|
+
// end T008
|
|
23
|
+
// T005: pipeline
|
|
24
|
+
export { detectWithRegex, resolveOverlaps, filterFalsePositives, propagateEntities, detectEntities, } from "./pipeline.js";
|
|
25
|
+
// end T005
|
|
26
|
+
// T009: engine assembly + worker protocol
|
|
27
|
+
export { createEngine, PROVIDERS, pickDefaultProvider, defaultThresholdFor, clampThreshold, ENGINE_SETTINGS_KEYS, } from "./engine.js";
|
|
28
|
+
export { serveEngine, connectEngine } from "./worker-protocol.js";
|
|
29
|
+
// end T009
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared model download utility used by all detection providers.
|
|
3
|
+
*
|
|
4
|
+
* Designed for large downloads (65-280 MB) on unreliable connections,
|
|
5
|
+
* especially mobile networks:
|
|
6
|
+
* - Resumes interrupted downloads with HTTP Range requests instead of
|
|
7
|
+
* restarting from zero.
|
|
8
|
+
* - Retries transient failures (network errors, stalls, 5xx) with
|
|
9
|
+
* exponential backoff.
|
|
10
|
+
* - Aborts stalled connections via a watchdog so a dead socket surfaces
|
|
11
|
+
* as a retryable error instead of hanging forever.
|
|
12
|
+
* - Throttles progress callbacks so the UI thread is not flooded with
|
|
13
|
+
* thousands of postMessage/setState calls during the download.
|
|
14
|
+
* - Serves the model from the injected BlobCache when available and prefers
|
|
15
|
+
* the cached (typically disk-backed) copy over the in-memory one to reduce
|
|
16
|
+
* peak RAM, which matters on iOS Safari where large tabs get killed.
|
|
17
|
+
*
|
|
18
|
+
* Environment-agnostic: all host capabilities (blob cache, fetch, persistent
|
|
19
|
+
* storage request) are injected via ModelLoaderEnv. This module never touches
|
|
20
|
+
* caches, navigator or global fetch directly.
|
|
21
|
+
*/
|
|
22
|
+
import type { BlobCache } from './env.js';
|
|
23
|
+
export type DownloadProgress = (downloaded: number, total: number) => void;
|
|
24
|
+
/**
|
|
25
|
+
* Host capabilities the loader needs. Hosts pass the relevant slice of their
|
|
26
|
+
* CoreEnv (modelCache as cache, fetch, persistStorage).
|
|
27
|
+
*/
|
|
28
|
+
export interface ModelLoaderEnv {
|
|
29
|
+
/** Blob cache for downloaded models. Web/extension: Cache Storage adapter. Tests: memoryBlobCache. */
|
|
30
|
+
cache: BlobCache;
|
|
31
|
+
/** Network access. Injectable for tests and proxies. */
|
|
32
|
+
fetch: typeof fetch;
|
|
33
|
+
/** Best-effort persistent-storage request (navigator.storage.persist() on the web). */
|
|
34
|
+
persistStorage?: () => Promise<boolean>;
|
|
35
|
+
}
|
|
36
|
+
export interface FetchModelOptions {
|
|
37
|
+
maxAttempts?: number;
|
|
38
|
+
retryBaseDelayMs?: number;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Download a model with progress reporting, resume and retry.
|
|
42
|
+
* Returns a Blob (served from the injected cache when possible).
|
|
43
|
+
*/
|
|
44
|
+
export declare function fetchModelBlob(env: ModelLoaderEnv, url: string, onProgress?: DownloadProgress, options?: FetchModelOptions): Promise<Blob>;
|
|
45
|
+
/**
|
|
46
|
+
* Remove a single model from the cache (e.g. when its URL changes).
|
|
47
|
+
*/
|
|
48
|
+
export declare function evictModelFromCache(env: ModelLoaderEnv, url: string): Promise<void>;
|
|
49
|
+
/**
|
|
50
|
+
* Retry an async operation with exponential backoff. Used for the smaller
|
|
51
|
+
* companion downloads (tokenizer files) that go through libraries without
|
|
52
|
+
* their own retry handling - one dropped request on a flaky connection
|
|
53
|
+
* should not abort the whole model load.
|
|
54
|
+
*/
|
|
55
|
+
export declare function retryAsync<T>(operation: () => Promise<T>, label: string, options?: FetchModelOptions): Promise<T>;
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared model download utility used by all detection providers.
|
|
3
|
+
*
|
|
4
|
+
* Designed for large downloads (65-280 MB) on unreliable connections,
|
|
5
|
+
* especially mobile networks:
|
|
6
|
+
* - Resumes interrupted downloads with HTTP Range requests instead of
|
|
7
|
+
* restarting from zero.
|
|
8
|
+
* - Retries transient failures (network errors, stalls, 5xx) with
|
|
9
|
+
* exponential backoff.
|
|
10
|
+
* - Aborts stalled connections via a watchdog so a dead socket surfaces
|
|
11
|
+
* as a retryable error instead of hanging forever.
|
|
12
|
+
* - Throttles progress callbacks so the UI thread is not flooded with
|
|
13
|
+
* thousands of postMessage/setState calls during the download.
|
|
14
|
+
* - Serves the model from the injected BlobCache when available and prefers
|
|
15
|
+
* the cached (typically disk-backed) copy over the in-memory one to reduce
|
|
16
|
+
* peak RAM, which matters on iOS Safari where large tabs get killed.
|
|
17
|
+
*
|
|
18
|
+
* Environment-agnostic: all host capabilities (blob cache, fetch, persistent
|
|
19
|
+
* storage request) are injected via ModelLoaderEnv. This module never touches
|
|
20
|
+
* caches, navigator or global fetch directly.
|
|
21
|
+
*/
|
|
22
|
+
const MAX_ATTEMPTS = 4;
|
|
23
|
+
const STALL_TIMEOUT_MS = 30000;
|
|
24
|
+
const PROGRESS_INTERVAL_MS = 150;
|
|
25
|
+
const RETRY_BASE_DELAY_MS = 1000;
|
|
26
|
+
class HttpError extends Error {
|
|
27
|
+
constructor(status, url) {
|
|
28
|
+
super(`Failed to download model: HTTP ${status} (${url})`);
|
|
29
|
+
this.name = 'HttpError';
|
|
30
|
+
this.status = status;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function isRetryable(err) {
|
|
34
|
+
if (err instanceof HttpError) {
|
|
35
|
+
return err.status === 408 || err.status === 429 || err.status >= 500;
|
|
36
|
+
}
|
|
37
|
+
// AbortError comes from our stall watchdog; TypeError is fetch's generic
|
|
38
|
+
// network failure (connection reset, DNS, CORS transport error, ...).
|
|
39
|
+
if (err instanceof DOMException && err.name === 'AbortError')
|
|
40
|
+
return true;
|
|
41
|
+
if (err instanceof TypeError)
|
|
42
|
+
return true;
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
function delay(ms) {
|
|
46
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
47
|
+
}
|
|
48
|
+
let persistRequested = false;
|
|
49
|
+
/**
|
|
50
|
+
* Best-effort request for persistent storage so the browser does not evict
|
|
51
|
+
* the cached model under storage pressure (common on mobile). Requested at
|
|
52
|
+
* most once per session, matching the previous web-only behavior.
|
|
53
|
+
*/
|
|
54
|
+
async function requestPersistentStorage(env) {
|
|
55
|
+
if (persistRequested || !env.persistStorage)
|
|
56
|
+
return;
|
|
57
|
+
persistRequested = true;
|
|
58
|
+
try {
|
|
59
|
+
await env.persistStorage();
|
|
60
|
+
}
|
|
61
|
+
catch { /* not supported or denied - caching still works, just evictable */ }
|
|
62
|
+
}
|
|
63
|
+
async function readFromCache(cache, url) {
|
|
64
|
+
try {
|
|
65
|
+
const cached = await cache.match(url);
|
|
66
|
+
if (cached)
|
|
67
|
+
return cached;
|
|
68
|
+
}
|
|
69
|
+
catch { /* corrupt entry or read failure - fall through to network */ }
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Best-effort cache put - large models can exceed per-origin quota, which
|
|
74
|
+
* surfaces either as put() resolving false or as a thrown error depending on
|
|
75
|
+
* the adapter. Caching is an optimisation, not a correctness requirement.
|
|
76
|
+
*/
|
|
77
|
+
async function tryCachePut(cache, url, blob) {
|
|
78
|
+
try {
|
|
79
|
+
const stored = await cache.put(url, blob);
|
|
80
|
+
if (!stored) {
|
|
81
|
+
console.warn('[DocCloak] Model cache put refused (model still loaded in memory)');
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
console.warn('[DocCloak] Model cache put failed (model still loaded in memory):', err);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* One download attempt. Streams the body through onChunk. A watchdog aborts
|
|
90
|
+
* the request if no bytes arrive for STALL_TIMEOUT_MS, covering both the
|
|
91
|
+
* initial connection and mid-body stalls. onHeaders fires as soon as the
|
|
92
|
+
* response headers are parsed - before the body streams - so callers can
|
|
93
|
+
* report a correct total alongside per-chunk progress.
|
|
94
|
+
*/
|
|
95
|
+
async function downloadAttempt(fetchFn, url, offset, onChunk, onHeaders) {
|
|
96
|
+
const controller = new AbortController();
|
|
97
|
+
let watchdog = setTimeout(() => controller.abort(), STALL_TIMEOUT_MS);
|
|
98
|
+
const kickWatchdog = () => {
|
|
99
|
+
clearTimeout(watchdog);
|
|
100
|
+
watchdog = setTimeout(() => controller.abort(), STALL_TIMEOUT_MS);
|
|
101
|
+
};
|
|
102
|
+
try {
|
|
103
|
+
const headers = {};
|
|
104
|
+
if (offset > 0)
|
|
105
|
+
headers['Range'] = `bytes=${offset}-`;
|
|
106
|
+
const response = await fetchFn(url, { headers, signal: controller.signal });
|
|
107
|
+
kickWatchdog();
|
|
108
|
+
if (!response.ok)
|
|
109
|
+
throw new HttpError(response.status, url);
|
|
110
|
+
const resumed = response.status === 206;
|
|
111
|
+
let total = 0;
|
|
112
|
+
if (resumed) {
|
|
113
|
+
// Content-Range: bytes <start>-<end>/<total>
|
|
114
|
+
const contentRange = response.headers.get('content-range');
|
|
115
|
+
const match = contentRange?.match(/\/(\d+)\s*$/);
|
|
116
|
+
if (match)
|
|
117
|
+
total = parseInt(match[1], 10);
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
const contentLength = response.headers.get('content-length');
|
|
121
|
+
if (contentLength)
|
|
122
|
+
total = parseInt(contentLength, 10);
|
|
123
|
+
}
|
|
124
|
+
onHeaders?.({ total, resumed });
|
|
125
|
+
if (!response.body) {
|
|
126
|
+
// No streaming support - read in one shot (no resume possible, but
|
|
127
|
+
// the retry loop still restarts from scratch on failure).
|
|
128
|
+
const buffer = await response.arrayBuffer();
|
|
129
|
+
onChunk(new Uint8Array(buffer));
|
|
130
|
+
return { total, resumed };
|
|
131
|
+
}
|
|
132
|
+
const reader = response.body.getReader();
|
|
133
|
+
while (true) {
|
|
134
|
+
const { done, value } = await reader.read();
|
|
135
|
+
if (done)
|
|
136
|
+
break;
|
|
137
|
+
kickWatchdog();
|
|
138
|
+
onChunk(value);
|
|
139
|
+
}
|
|
140
|
+
return { total, resumed };
|
|
141
|
+
}
|
|
142
|
+
finally {
|
|
143
|
+
clearTimeout(watchdog);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Download a model with progress reporting, resume and retry.
|
|
148
|
+
* Returns a Blob (served from the injected cache when possible).
|
|
149
|
+
*/
|
|
150
|
+
export async function fetchModelBlob(env, url, onProgress, options) {
|
|
151
|
+
const maxAttempts = options?.maxAttempts ?? MAX_ATTEMPTS;
|
|
152
|
+
const retryBaseDelayMs = options?.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS;
|
|
153
|
+
const cache = env.cache;
|
|
154
|
+
const cachedBlob = await readFromCache(cache, url);
|
|
155
|
+
if (cachedBlob) {
|
|
156
|
+
onProgress?.(cachedBlob.size, cachedBlob.size);
|
|
157
|
+
return cachedBlob;
|
|
158
|
+
}
|
|
159
|
+
await requestPersistentStorage(env);
|
|
160
|
+
let lastProgressAt = 0;
|
|
161
|
+
const reportProgress = (downloaded, total, force = false) => {
|
|
162
|
+
const now = Date.now();
|
|
163
|
+
if (!force && now - lastProgressAt < PROGRESS_INTERVAL_MS)
|
|
164
|
+
return;
|
|
165
|
+
lastProgressAt = now;
|
|
166
|
+
onProgress?.(downloaded, total);
|
|
167
|
+
};
|
|
168
|
+
let chunks = [];
|
|
169
|
+
let downloaded = 0;
|
|
170
|
+
let total = 0;
|
|
171
|
+
for (let attempt = 1;; attempt++) {
|
|
172
|
+
try {
|
|
173
|
+
const requestedOffset = downloaded;
|
|
174
|
+
const result = await downloadAttempt(env.fetch, url, requestedOffset, (chunk) => {
|
|
175
|
+
chunks.push(chunk);
|
|
176
|
+
downloaded += chunk.length;
|
|
177
|
+
reportProgress(downloaded, total);
|
|
178
|
+
}, (info) => {
|
|
179
|
+
if (info.total > 0)
|
|
180
|
+
total = info.resumed ? info.total : Math.max(total, info.total);
|
|
181
|
+
});
|
|
182
|
+
if (requestedOffset > 0 && !result.resumed) {
|
|
183
|
+
// Server ignored the Range header and sent the file from the start.
|
|
184
|
+
// The freshly received chunks already replace everything, but they
|
|
185
|
+
// were appended after the stale ones - rebuild keeping only bytes
|
|
186
|
+
// from this attempt.
|
|
187
|
+
const freshBytes = downloaded - requestedOffset;
|
|
188
|
+
const fresh = [];
|
|
189
|
+
let need = freshBytes;
|
|
190
|
+
for (let i = chunks.length - 1; i >= 0 && need > 0; i--) {
|
|
191
|
+
fresh.unshift(chunks[i]);
|
|
192
|
+
need -= chunks[i].length;
|
|
193
|
+
}
|
|
194
|
+
chunks = fresh;
|
|
195
|
+
downloaded = freshBytes;
|
|
196
|
+
}
|
|
197
|
+
if (result.total > 0)
|
|
198
|
+
total = result.resumed ? result.total : Math.max(total, result.total);
|
|
199
|
+
if (total > 0 && downloaded < total) {
|
|
200
|
+
// Connection closed early without an error. Treat as retryable.
|
|
201
|
+
throw new DOMException('Download ended before completion', 'AbortError');
|
|
202
|
+
}
|
|
203
|
+
break;
|
|
204
|
+
}
|
|
205
|
+
catch (err) {
|
|
206
|
+
if (!isRetryable(err) || attempt >= maxAttempts) {
|
|
207
|
+
throw err instanceof Error ? err : new Error(String(err));
|
|
208
|
+
}
|
|
209
|
+
console.warn(`[DocCloak] Model download interrupted at ${downloaded} bytes (attempt ${attempt}/${maxAttempts}), retrying...`, err);
|
|
210
|
+
await delay(retryBaseDelayMs * 2 ** (attempt - 1));
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
reportProgress(downloaded, total || downloaded, true);
|
|
214
|
+
let blob = new Blob(chunks);
|
|
215
|
+
chunks = [];
|
|
216
|
+
await tryCachePut(cache, url, blob);
|
|
217
|
+
// Prefer the cached (disk-backed) copy so the in-memory chunks can be
|
|
218
|
+
// collected before ONNX Runtime allocates its own copy of the model.
|
|
219
|
+
const diskBlob = await readFromCache(cache, url);
|
|
220
|
+
if (diskBlob && diskBlob.size === blob.size)
|
|
221
|
+
blob = diskBlob;
|
|
222
|
+
return blob;
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Remove a single model from the cache (e.g. when its URL changes).
|
|
226
|
+
*/
|
|
227
|
+
export async function evictModelFromCache(env, url) {
|
|
228
|
+
try {
|
|
229
|
+
await env.cache.delete(url);
|
|
230
|
+
}
|
|
231
|
+
catch { /* ignore */ }
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Retry an async operation with exponential backoff. Used for the smaller
|
|
235
|
+
* companion downloads (tokenizer files) that go through libraries without
|
|
236
|
+
* their own retry handling - one dropped request on a flaky connection
|
|
237
|
+
* should not abort the whole model load.
|
|
238
|
+
*/
|
|
239
|
+
export async function retryAsync(operation, label, options) {
|
|
240
|
+
const maxAttempts = options?.maxAttempts ?? MAX_ATTEMPTS;
|
|
241
|
+
const retryBaseDelayMs = options?.retryBaseDelayMs ?? RETRY_BASE_DELAY_MS;
|
|
242
|
+
for (let attempt = 1;; attempt++) {
|
|
243
|
+
try {
|
|
244
|
+
return await operation();
|
|
245
|
+
}
|
|
246
|
+
catch (err) {
|
|
247
|
+
if (attempt >= maxAttempts)
|
|
248
|
+
throw err;
|
|
249
|
+
console.warn(`[DocCloak] ${label} failed (attempt ${attempt}/${maxAttempts}), retrying...`, err);
|
|
250
|
+
await delay(retryBaseDelayMs * 2 ** (attempt - 1));
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure entity pipeline (T005).
|
|
3
|
+
*
|
|
4
|
+
* Extracted verbatim from the web app's detection.worker.ts: text and
|
|
5
|
+
* entities in, entities out. No worker messaging, no providers, no
|
|
6
|
+
* environment access. The worker (and any other host) composes these
|
|
7
|
+
* functions around its own ML provider call.
|
|
8
|
+
*
|
|
9
|
+
* Function bodies are byte-identical to the originals by design; only the
|
|
10
|
+
* signatures of detectWithRegex (regexRegion becomes a parameter instead of
|
|
11
|
+
* worker module state) and detectEntities (ML results are passed in instead
|
|
12
|
+
* of fetched from a provider) differ.
|
|
13
|
+
*/
|
|
14
|
+
import type { DetectedEntity } from './types.js';
|
|
15
|
+
export declare function detectWithRegex(text: string, regexRegion?: string): DetectedEntity[];
|
|
16
|
+
export declare function resolveOverlaps(entities: DetectedEntity[]): DetectedEntity[];
|
|
17
|
+
export declare function filterFalsePositives(entities: DetectedEntity[]): DetectedEntity[];
|
|
18
|
+
export declare function propagateEntities(text: string, entities: DetectedEntity[]): DetectedEntity[];
|
|
19
|
+
export declare function detectEntities(text: string, mlEntities: DetectedEntity[], regexEntities: DetectedEntity[]): DetectedEntity[];
|
package/dist/pipeline.js
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure entity pipeline (T005).
|
|
3
|
+
*
|
|
4
|
+
* Extracted verbatim from the web app's detection.worker.ts: text and
|
|
5
|
+
* entities in, entities out. No worker messaging, no providers, no
|
|
6
|
+
* environment access. The worker (and any other host) composes these
|
|
7
|
+
* functions around its own ML provider call.
|
|
8
|
+
*
|
|
9
|
+
* Function bodies are byte-identical to the originals by design; only the
|
|
10
|
+
* signatures of detectWithRegex (regexRegion becomes a parameter instead of
|
|
11
|
+
* worker module state) and detectEntities (ML results are passed in instead
|
|
12
|
+
* of fetched from a provider) differ.
|
|
13
|
+
*/
|
|
14
|
+
import { ALL_REGEX_RULES } from "./regex/index.js";
|
|
15
|
+
export function detectWithRegex(text, regexRegion = 'all') {
|
|
16
|
+
const entities = [];
|
|
17
|
+
const rules = regexRegion === 'all'
|
|
18
|
+
? ALL_REGEX_RULES
|
|
19
|
+
: ALL_REGEX_RULES.filter((r) => r.region === 'universal' || r.region === regexRegion);
|
|
20
|
+
for (const rule of rules) {
|
|
21
|
+
rule.pattern.lastIndex = 0;
|
|
22
|
+
let match;
|
|
23
|
+
while ((match = rule.pattern.exec(text)) !== null) {
|
|
24
|
+
if (rule.validate && !rule.validate(match[0]))
|
|
25
|
+
continue;
|
|
26
|
+
entities.push({
|
|
27
|
+
type: rule.type,
|
|
28
|
+
value: match[0],
|
|
29
|
+
start: match.index,
|
|
30
|
+
end: match.index + match[0].length,
|
|
31
|
+
confidence: rule.confidence,
|
|
32
|
+
detector: rule.detector,
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return entities;
|
|
37
|
+
}
|
|
38
|
+
export function resolveOverlaps(entities) {
|
|
39
|
+
const sorted = [...entities].sort((a, b) => {
|
|
40
|
+
if (a.start !== b.start)
|
|
41
|
+
return a.start - b.start;
|
|
42
|
+
const aLen = a.end - a.start;
|
|
43
|
+
const bLen = b.end - b.start;
|
|
44
|
+
if (aLen !== bLen)
|
|
45
|
+
return bLen - aLen;
|
|
46
|
+
return b.confidence - a.confidence;
|
|
47
|
+
});
|
|
48
|
+
const resolved = [];
|
|
49
|
+
for (const entity of sorted) {
|
|
50
|
+
const overlaps = resolved.some((existing) => entity.start < existing.end && entity.end > existing.start);
|
|
51
|
+
if (!overlaps) {
|
|
52
|
+
resolved.push(entity);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return resolved;
|
|
56
|
+
}
|
|
57
|
+
export function filterFalsePositives(entities) {
|
|
58
|
+
return entities.filter((e) => {
|
|
59
|
+
if (e.value.trim().length < 3)
|
|
60
|
+
return false;
|
|
61
|
+
return true;
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
export function propagateEntities(text, entities) {
|
|
65
|
+
const propagated = [];
|
|
66
|
+
const seen = new Set();
|
|
67
|
+
for (const e of entities) {
|
|
68
|
+
seen.add(`${e.start}:${e.end}`);
|
|
69
|
+
}
|
|
70
|
+
const terms = [];
|
|
71
|
+
const addedTerms = new Set();
|
|
72
|
+
for (const e of entities) {
|
|
73
|
+
const val = e.value.trim();
|
|
74
|
+
if (val.length >= 3 && !addedTerms.has(val.toLowerCase())) {
|
|
75
|
+
addedTerms.add(val.toLowerCase());
|
|
76
|
+
terms.push({ term: val, type: e.type });
|
|
77
|
+
}
|
|
78
|
+
if (val.includes(' ')) {
|
|
79
|
+
for (const word of val.split(/\s+/)) {
|
|
80
|
+
const w = word.replace(/[.,;:!?()]+$/, '');
|
|
81
|
+
if (w.length >= 4 && !addedTerms.has(w.toLowerCase())) {
|
|
82
|
+
addedTerms.add(w.toLowerCase());
|
|
83
|
+
terms.push({ term: w, type: e.type });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
for (const { term, type } of terms) {
|
|
89
|
+
let searchFrom = 0;
|
|
90
|
+
const lowerText = text.toLowerCase();
|
|
91
|
+
const lowerTerm = term.toLowerCase();
|
|
92
|
+
while (searchFrom < text.length) {
|
|
93
|
+
const idx = lowerText.indexOf(lowerTerm, searchFrom);
|
|
94
|
+
if (idx === -1)
|
|
95
|
+
break;
|
|
96
|
+
const end = idx + term.length;
|
|
97
|
+
const key = `${idx}:${end}`;
|
|
98
|
+
if (!seen.has(key)) {
|
|
99
|
+
const charBefore = idx > 0 ? text[idx - 1] : ' ';
|
|
100
|
+
const charAfter = end < text.length ? text[end] : ' ';
|
|
101
|
+
const isBoundaryBefore = /[\s.,;:!?()"'„"\-–—/]/.test(charBefore);
|
|
102
|
+
const isBoundaryAfter = /[\s.,;:!?()"'„"\-–—/]/.test(charAfter);
|
|
103
|
+
if (isBoundaryBefore && isBoundaryAfter) {
|
|
104
|
+
seen.add(key);
|
|
105
|
+
propagated.push({
|
|
106
|
+
type,
|
|
107
|
+
value: text.slice(idx, end),
|
|
108
|
+
start: idx,
|
|
109
|
+
end,
|
|
110
|
+
confidence: 0.95,
|
|
111
|
+
detector: 'propagated',
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
searchFrom = idx + 1;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return propagated;
|
|
119
|
+
}
|
|
120
|
+
export function detectEntities(text, mlEntities, regexEntities) {
|
|
121
|
+
if (!text.trim())
|
|
122
|
+
return [];
|
|
123
|
+
const mlResults = filterFalsePositives(mlEntities);
|
|
124
|
+
const regexResults = regexEntities;
|
|
125
|
+
const initial = resolveOverlaps([...mlResults, ...regexResults]);
|
|
126
|
+
const propagated = propagateEntities(text, initial);
|
|
127
|
+
return resolveOverlaps([...initial, ...propagated]);
|
|
128
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BardS.ai EU PII Anonymization Provider
|
|
3
|
+
*
|
|
4
|
+
* XLM-RoBERTa-base fine-tuned for EU PII detection (24 EU languages, natively multilingual).
|
|
5
|
+
* 35 entity types, ~0.3B parameters, quantized ONNX (~279 MB).
|
|
6
|
+
* Model: bardsai/eu-pii-anonimization-multilang
|
|
7
|
+
* Runs in-browser via ONNX Runtime WebAssembly.
|
|
8
|
+
*
|
|
9
|
+
* Environment-agnostic: wasm paths, model bytes (blob cache + fetch) and the
|
|
10
|
+
* tokenizer all arrive through the injected CoreEnv. This module never touches
|
|
11
|
+
* import.meta, caches, navigator or @huggingface/transformers directly.
|
|
12
|
+
*/
|
|
13
|
+
import type { DetectedEntity, DetectionProvider, ProgressCallback } from '../types.js';
|
|
14
|
+
import type { CoreEnv } from '../env.js';
|
|
15
|
+
export declare class BardsaiProvider implements DetectionProvider {
|
|
16
|
+
private _name;
|
|
17
|
+
get name(): string;
|
|
18
|
+
private readonly env;
|
|
19
|
+
private tokenizer;
|
|
20
|
+
private session;
|
|
21
|
+
private loading;
|
|
22
|
+
private loadError;
|
|
23
|
+
private loadWaiters;
|
|
24
|
+
private progressCallback;
|
|
25
|
+
private threshold;
|
|
26
|
+
constructor(env: CoreEnv);
|
|
27
|
+
isLoaded(): boolean;
|
|
28
|
+
isLoading(): boolean;
|
|
29
|
+
onProgress(callback: ProgressCallback): void;
|
|
30
|
+
setThreshold(value: number): void;
|
|
31
|
+
getThreshold(): number;
|
|
32
|
+
load(): Promise<void>;
|
|
33
|
+
release(): void;
|
|
34
|
+
detect(text: string, onProgress?: (progress: number) => void): Promise<DetectedEntity[]>;
|
|
35
|
+
private inferChunk;
|
|
36
|
+
private emitEntity;
|
|
37
|
+
private deduplicateSpans;
|
|
38
|
+
}
|