@doccloak/core 0.12.1 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +333 -0
- package/NOTICE +5 -0
- package/README.md +131 -9
- package/dist/dom/errors.d.ts +1 -1
- package/dist/dom/ocr.d.ts +33 -7
- package/dist/dom/ocr.js +98 -4
- package/dist/dom/office.d.ts +20 -6
- package/dist/dom/office.js +44 -11
- package/dist/engine.js +4 -5
- package/dist/env.d.ts +3 -3
- package/dist/pdf/cmap.d.ts +83 -0
- package/dist/pdf/cmap.js +659 -0
- package/dist/pdf/encodings.d.ts +49 -0
- package/dist/pdf/encodings.js +181 -0
- package/dist/pdf/extract.d.ts +106 -0
- package/dist/pdf/extract.js +1064 -0
- package/dist/pdf/fallback-fonts.d.ts +146 -0
- package/dist/pdf/fallback-fonts.js +312 -0
- package/dist/pdf/fit.d.ts +35 -0
- package/dist/pdf/fit.js +60 -0
- package/dist/pdf/fonts.d.ts +82 -0
- package/dist/pdf/fonts.js +465 -0
- package/dist/pdf/glyphlist.d.ts +37 -0
- package/dist/pdf/glyphlist.js +485 -0
- package/dist/pdf/index.d.ts +21 -0
- package/dist/pdf/index.js +7 -0
- package/dist/pdf/lexer.d.ts +61 -0
- package/dist/pdf/lexer.js +554 -0
- package/dist/pdf/objects.d.ts +57 -0
- package/dist/pdf/objects.js +217 -0
- package/dist/pdf/oracle.d.ts +149 -0
- package/dist/pdf/oracle.js +384 -0
- package/dist/pdf/raster.d.ts +45 -0
- package/dist/pdf/raster.js +89 -0
- package/dist/pdf/read.d.ts +36 -0
- package/dist/pdf/read.js +65 -0
- package/dist/pdf/serializer.d.ts +21 -0
- package/dist/pdf/serializer.js +106 -0
- package/dist/pdf/standard-metrics.d.ts +27 -0
- package/dist/pdf/standard-metrics.js +72 -0
- package/dist/pdf/surgery.d.ts +131 -0
- package/dist/pdf/surgery.js +847 -0
- package/dist/pdf/textstate.d.ts +77 -0
- package/dist/pdf/textstate.js +280 -0
- package/dist/pdf/types.d.ts +191 -0
- package/dist/pdf/types.js +11 -0
- package/dist/pdf/verify.d.ts +33 -0
- package/dist/pdf/verify.js +446 -0
- package/dist/pdf/write.d.ts +52 -0
- package/dist/pdf/write.js +644 -0
- package/dist/pipeline.d.ts +15 -0
- package/dist/pipeline.js +123 -8
- package/dist/providers/bardsai.d.ts +1 -1
- package/dist/providers/bardsai.js +5 -1
- package/dist/providers/gliner-base.d.ts +1 -1
- package/dist/providers/gliner-base.js +5 -1
- package/dist/providers/gliner.d.ts +1 -1
- package/dist/providers/gliner.js +5 -1
- package/dist/regex/rules.data.js +13 -8
- package/dist/types.d.ts +17 -2
- package/dist/types.js +17 -0
- package/dist/worker-protocol.d.ts +11 -0
- package/dist/worker-protocol.js +41 -8
- package/fonts/liberation/LICENSE +102 -0
- package/fonts/liberation/LiberationMono-Bold.ttf +0 -0
- package/fonts/liberation/LiberationMono-BoldItalic.ttf +0 -0
- package/fonts/liberation/LiberationMono-Italic.ttf +0 -0
- package/fonts/liberation/LiberationMono-Regular.ttf +0 -0
- package/fonts/liberation/LiberationSans-Bold.ttf +0 -0
- package/fonts/liberation/LiberationSans-BoldItalic.ttf +0 -0
- package/fonts/liberation/LiberationSans-Italic.ttf +0 -0
- package/fonts/liberation/LiberationSans-Regular.ttf +0 -0
- package/fonts/liberation/LiberationSerif-Bold.ttf +0 -0
- package/fonts/liberation/LiberationSerif-BoldItalic.ttf +0 -0
- package/fonts/liberation/LiberationSerif-Italic.ttf +0 -0
- package/fonts/liberation/LiberationSerif-Regular.ttf +0 -0
- package/package.json +11 -2
- package/rules/pl.json +7 -3
- package/rules/universal.json +6 -5
package/dist/pipeline.js
CHANGED
|
@@ -45,13 +45,19 @@ function runRule(rule, segment, out) {
|
|
|
45
45
|
rule.pattern.lastIndex++;
|
|
46
46
|
continue;
|
|
47
47
|
}
|
|
48
|
-
|
|
48
|
+
// A class that admits whitespace (IBAN, phone) can end the match on a
|
|
49
|
+
// space before the next word; the replacement would then swallow it.
|
|
50
|
+
const lead = match[0].length - match[0].trimStart().length;
|
|
51
|
+
const value = match[0].trim();
|
|
52
|
+
if (value.length === 0)
|
|
53
|
+
continue;
|
|
54
|
+
if (rule.validate && !rule.validate(value))
|
|
49
55
|
continue;
|
|
50
56
|
out.push({
|
|
51
57
|
type: rule.type,
|
|
52
|
-
value
|
|
53
|
-
start: segment.offset + match.index,
|
|
54
|
-
end: segment.offset + match.index +
|
|
58
|
+
value,
|
|
59
|
+
start: segment.offset + match.index + lead,
|
|
60
|
+
end: segment.offset + match.index + lead + value.length,
|
|
55
61
|
confidence: rule.confidence,
|
|
56
62
|
detector: rule.detector,
|
|
57
63
|
});
|
|
@@ -99,13 +105,56 @@ export function resolveOverlaps(entities) {
|
|
|
99
105
|
return b.confidence - a.confidence;
|
|
100
106
|
});
|
|
101
107
|
const resolved = [];
|
|
108
|
+
const remainders = [];
|
|
102
109
|
for (const entity of sorted) {
|
|
103
|
-
|
|
104
|
-
|
|
110
|
+
let coveredTo = -1;
|
|
111
|
+
for (const existing of resolved) {
|
|
112
|
+
if (entity.start < existing.end && entity.end > existing.start)
|
|
113
|
+
coveredTo = Math.max(coveredTo, existing.end);
|
|
114
|
+
}
|
|
115
|
+
if (coveredTo < 0) {
|
|
105
116
|
resolved.push(entity);
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
// The entity loses to the spans it overlaps, but what it covers past them may still be a value:
|
|
120
|
+
// keep it (remainderFrom) so it is not left in clear (T233). A PDF exported from a spreadsheet
|
|
121
|
+
// glued a clipped cell to the next one ("31-147 Krakók.wisniewska@firma.pl"): the model took
|
|
122
|
+
// "31-147 Krakók", the e-mail rule the whole token, and dropping the e-mail left
|
|
123
|
+
// ".wisniewska@firma.pl" in the output.
|
|
124
|
+
remainders.push(...remainderFrom(entity, coveredTo));
|
|
125
|
+
}
|
|
126
|
+
// Each remainder is strictly shorter than its entity, so this terminates.
|
|
127
|
+
return remainders.length > 0 ? resolveOverlaps([...resolved, ...remainders]) : resolved;
|
|
128
|
+
}
|
|
129
|
+
const REMAINDER_LEAD = /^[\s.,;:!?\-–—/()[\]{}"'«»“”‘’„]+/u;
|
|
130
|
+
const REMAINDER_TRAIL = /[\s,.;:!?)\]}"'»”’]+$/u;
|
|
131
|
+
const HAS_TEXT = /[\p{L}\p{N}]/u;
|
|
132
|
+
/**
|
|
133
|
+
* What `entity` covers from `from` on, as entities. A regex match only means something whole
|
|
134
|
+
* ("00 before 2025" is a date because "00" reads as a day), so its rule runs again on the rest and
|
|
135
|
+
* only what it matches there survives ("wisniewska@firma.pl" does, "before 2025" does not). A model
|
|
136
|
+
* span has no rule to re-run: its rest is kept without the punctuation at its edges ("ith" of a
|
|
137
|
+
* "Smith" the winner cut at "Sm").
|
|
138
|
+
*/
|
|
139
|
+
function remainderFrom(entity, from) {
|
|
140
|
+
if (from >= entity.end)
|
|
141
|
+
return [];
|
|
142
|
+
const raw = entity.value.slice(from - entity.start);
|
|
143
|
+
if (entity.detector.startsWith('regex:')) {
|
|
144
|
+
const rule = ALL_REGEX_RULES.find((r) => r.detector === entity.detector);
|
|
145
|
+
if (rule) {
|
|
146
|
+
const out = [];
|
|
147
|
+
runRule(rule, { text: raw, offset: from }, out);
|
|
148
|
+
return out;
|
|
106
149
|
}
|
|
107
150
|
}
|
|
108
|
-
|
|
151
|
+
const lead = REMAINDER_LEAD.exec(raw)?.[0].length ?? 0;
|
|
152
|
+
const trail = REMAINDER_TRAIL.exec(raw.slice(lead))?.[0].length ?? 0;
|
|
153
|
+
const value = raw.slice(lead, raw.length - trail);
|
|
154
|
+
if (!HAS_TEXT.test(value))
|
|
155
|
+
return [];
|
|
156
|
+
const start = from + lead;
|
|
157
|
+
return [{ ...entity, value, start, end: start + value.length }];
|
|
109
158
|
}
|
|
110
159
|
// ── trimSpanPunctuation ────────────────────────────────────
|
|
111
160
|
const LEADING_PUNCT = /^[\s(\[{"'«“‘„]+/u;
|
|
@@ -316,12 +365,78 @@ export function propagateEntities(text, entities) {
|
|
|
316
365
|
return propagated;
|
|
317
366
|
}
|
|
318
367
|
// ── detectEntities ─────────────────────────────────────────
|
|
368
|
+
// ── splitCatchAlls ─────────────────────────────────────────
|
|
369
|
+
/**
|
|
370
|
+
* Regex rules at or below this confidence are catch-alls (`universal:
|
|
371
|
+
* long_number`): they exist to cover digit runs no specific rule knows.
|
|
372
|
+
*/
|
|
373
|
+
const CATCH_ALL_CONFIDENCE = 0.5;
|
|
374
|
+
/** Detections (ML or regex) a catch-all yields to when they overlap it. */
|
|
375
|
+
const SPECIFIC_CONFIDENCE = 0.8;
|
|
376
|
+
/**
|
|
377
|
+
* T230: a catch-all regex span yields to the specific detections it
|
|
378
|
+
* overlaps. `long_number` admits whitespace, so a PESEL cell followed by a
|
|
379
|
+
* phone cell ("90050598761 512 987 654") is one 23-character span that
|
|
380
|
+
* resolveOverlaps prefers (same start, longer) over PESEL 0.9 + PHONE 0.8:
|
|
381
|
+
* the row came out as one [SSN] and the phone "was not detected".
|
|
382
|
+
*
|
|
383
|
+
* Every catch-all that overlaps a specific span is replaced by its
|
|
384
|
+
* leftovers: the uncovered slices are re-run through the same rule, so an
|
|
385
|
+
* unknown long number next to a phone is still caught, and the specific
|
|
386
|
+
* spans then win. Catch-alls nothing specific overlaps are untouched, and
|
|
387
|
+
* so is everything else (the leak-averse "earlier start, longer span" order
|
|
388
|
+
* stays for all other overlaps).
|
|
389
|
+
*/
|
|
390
|
+
export function splitCatchAlls(text, entities) {
|
|
391
|
+
const specific = entities.filter((e) => e.confidence >= SPECIFIC_CONFIDENCE);
|
|
392
|
+
if (specific.length === 0)
|
|
393
|
+
return entities;
|
|
394
|
+
const out = [];
|
|
395
|
+
for (const entity of entities) {
|
|
396
|
+
const catchAll = entity.detector.startsWith('regex:') && entity.confidence <= CATCH_ALL_CONFIDENCE;
|
|
397
|
+
if (!catchAll) {
|
|
398
|
+
out.push(entity);
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
const covering = specific
|
|
402
|
+
.filter((s) => s.start < entity.end && s.end > entity.start)
|
|
403
|
+
.sort((a, b) => a.start - b.start);
|
|
404
|
+
if (covering.length === 0) {
|
|
405
|
+
out.push(entity);
|
|
406
|
+
continue;
|
|
407
|
+
}
|
|
408
|
+
const rule = ALL_REGEX_RULES.find((r) => r.detector === entity.detector);
|
|
409
|
+
let cursor = entity.start;
|
|
410
|
+
const gaps = [];
|
|
411
|
+
for (const s of covering) {
|
|
412
|
+
if (s.start > cursor)
|
|
413
|
+
gaps.push([cursor, s.start]);
|
|
414
|
+
cursor = Math.max(cursor, s.end);
|
|
415
|
+
}
|
|
416
|
+
if (cursor < entity.end)
|
|
417
|
+
gaps.push([cursor, entity.end]);
|
|
418
|
+
for (const [start, end] of gaps) {
|
|
419
|
+
if (rule) {
|
|
420
|
+
runRule(rule, { text: text.slice(start, end), offset: start }, out);
|
|
421
|
+
}
|
|
422
|
+
else {
|
|
423
|
+
const slice = text.slice(start, end);
|
|
424
|
+
const lead = slice.length - slice.trimStart().length;
|
|
425
|
+
const value = slice.trim();
|
|
426
|
+
if (value.length > 0) {
|
|
427
|
+
out.push({ ...entity, value, start: start + lead, end: start + lead + value.length });
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
return out;
|
|
433
|
+
}
|
|
319
434
|
export function detectEntities(text, mlEntities, regexEntities) {
|
|
320
435
|
if (!text.trim())
|
|
321
436
|
return [];
|
|
322
437
|
const mlResults = filterFalsePositives(trimSpanPunctuation(mlEntities));
|
|
323
438
|
const regexResults = regexEntities;
|
|
324
|
-
const initial = resolveOverlaps([...mlResults, ...regexResults]);
|
|
439
|
+
const initial = resolveOverlaps(splitCatchAlls(text, [...mlResults, ...regexResults]));
|
|
325
440
|
const propagated = propagateEntities(text, initial);
|
|
326
441
|
return resolveOverlaps([...initial, ...propagated]);
|
|
327
442
|
}
|
|
@@ -50,7 +50,7 @@ export declare class BardsaiProvider implements DetectionProvider {
|
|
|
50
50
|
getThreshold(): number;
|
|
51
51
|
load(): Promise<void>;
|
|
52
52
|
release(): void;
|
|
53
|
-
detect(text: string, onProgress?: (progress: number) => void): Promise<DetectedEntity[]>;
|
|
53
|
+
detect(text: string, onProgress?: (progress: number) => void, signal?: AbortSignal): Promise<DetectedEntity[]>;
|
|
54
54
|
private inferChunk;
|
|
55
55
|
private emitEntity;
|
|
56
56
|
private deduplicateSpans;
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
// wasm fallback, asyncify binary). The bare entry resolves the deprecated JSEP
|
|
15
15
|
// bundle whose wasm exceeds Cloudflare Pages' 25 MiB per-file limit.
|
|
16
16
|
import * as ort from 'onnxruntime-web/webgpu';
|
|
17
|
+
import { throwIfAborted } from "../types.js";
|
|
17
18
|
import { evictModelFromCache, fetchModelBlob, loadPinnedTokenizer } from "../model-loader.js";
|
|
18
19
|
// ── Model config ──────────────────────────────────────────
|
|
19
20
|
// Supply-chain pinning (T116): the model is fetched from an immutable
|
|
@@ -238,7 +239,8 @@ export class BardsaiProvider {
|
|
|
238
239
|
this.loading = false;
|
|
239
240
|
this.loadError = null;
|
|
240
241
|
}
|
|
241
|
-
async detect(text, onProgress) {
|
|
242
|
+
async detect(text, onProgress, signal) {
|
|
243
|
+
throwIfAborted(signal);
|
|
242
244
|
if (!this.isLoaded())
|
|
243
245
|
await this.load();
|
|
244
246
|
if (!this.session || !this.tokenizer)
|
|
@@ -305,6 +307,8 @@ export class BardsaiProvider {
|
|
|
305
307
|
let chunksDone = 0;
|
|
306
308
|
onProgress?.(0.1); // tokenization done
|
|
307
309
|
while (i < words.length) {
|
|
310
|
+
// T222: a cancelled run stops here, before the next inference.
|
|
311
|
+
throwIfAborted(signal);
|
|
308
312
|
let subtokenSum = 0;
|
|
309
313
|
let end = i;
|
|
310
314
|
while (end < words.length && subtokenSum + subtokenCounts[end] <= maxSubtokens) {
|
|
@@ -93,7 +93,7 @@ export declare class GlinerBaseProvider implements DetectionProvider {
|
|
|
93
93
|
restoreCustomLabels(): Promise<void>;
|
|
94
94
|
load(): Promise<void>;
|
|
95
95
|
release(): void;
|
|
96
|
-
detect(text: string, onProgress?: (progress: number) => void): Promise<DetectedEntity[]>;
|
|
96
|
+
detect(text: string, onProgress?: (progress: number) => void, signal?: AbortSignal): Promise<DetectedEntity[]>;
|
|
97
97
|
private buildPromptTokens;
|
|
98
98
|
private inferChunk;
|
|
99
99
|
}
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
// wasm fallback, asyncify binary). The bare entry resolves the deprecated JSEP
|
|
20
20
|
// bundle whose wasm exceeds Cloudflare Pages' 25 MiB per-file limit.
|
|
21
21
|
import * as ort from 'onnxruntime-web/webgpu';
|
|
22
|
+
import { throwIfAborted } from "../types.js";
|
|
22
23
|
import { fetchModelBlob, loadPinnedTokenizer } from "../model-loader.js";
|
|
23
24
|
import { DEFAULT_PII_LABELS, greedySelect, mapLabelToEntityType, sigmoid, splitWords, } from "./gliner.js";
|
|
24
25
|
// ── Model config ──────────────────────────────────────────
|
|
@@ -265,7 +266,8 @@ export class GlinerBaseProvider {
|
|
|
265
266
|
this.loading = false;
|
|
266
267
|
this.loadError = null;
|
|
267
268
|
}
|
|
268
|
-
async detect(text, onProgress) {
|
|
269
|
+
async detect(text, onProgress, signal) {
|
|
270
|
+
throwIfAborted(signal);
|
|
269
271
|
if (!this.isLoaded())
|
|
270
272
|
await this.load();
|
|
271
273
|
if (!this.session || !this.tokenizer)
|
|
@@ -298,6 +300,8 @@ export class GlinerBaseProvider {
|
|
|
298
300
|
// Multiple overlapping chunks
|
|
299
301
|
let chunksDone = 0;
|
|
300
302
|
for (let chunkStart = 0; chunkStart < words.length; chunkStart += chunkSize - overlap) {
|
|
303
|
+
// T222: a cancelled run stops here, before the next inference.
|
|
304
|
+
throwIfAborted(signal);
|
|
301
305
|
const chunkEnd = Math.min(chunkStart + chunkSize, words.length);
|
|
302
306
|
const chunkWords = words.slice(chunkStart, chunkEnd);
|
|
303
307
|
const chunkStarts = starts.slice(chunkStart, chunkEnd);
|
|
@@ -81,7 +81,7 @@ export declare class GlinerProvider implements DetectionProvider {
|
|
|
81
81
|
restoreCustomLabels(): Promise<void>;
|
|
82
82
|
load(): Promise<void>;
|
|
83
83
|
release(): void;
|
|
84
|
-
detect(text: string, onProgress?: (progress: number) => void): Promise<DetectedEntity[]>;
|
|
84
|
+
detect(text: string, onProgress?: (progress: number) => void, signal?: AbortSignal): Promise<DetectedEntity[]>;
|
|
85
85
|
private buildPromptTokens;
|
|
86
86
|
private inferChunk;
|
|
87
87
|
}
|
package/dist/providers/gliner.js
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
// wasm fallback, asyncify binary). The bare entry resolves the deprecated JSEP
|
|
11
11
|
// bundle whose wasm exceeds Cloudflare Pages' 25 MiB per-file limit.
|
|
12
12
|
import * as ort from 'onnxruntime-web/webgpu';
|
|
13
|
+
import { throwIfAborted } from "../types.js";
|
|
13
14
|
import { evictModelFromCache, fetchModelBlob, loadPinnedTokenizer } from "../model-loader.js";
|
|
14
15
|
// ── Model config ──────────────────────────────────────────
|
|
15
16
|
// Supply-chain pinning (T116): the model is fetched from an immutable
|
|
@@ -293,7 +294,8 @@ export class GlinerProvider {
|
|
|
293
294
|
this.loading = false;
|
|
294
295
|
this.loadError = null;
|
|
295
296
|
}
|
|
296
|
-
async detect(text, onProgress) {
|
|
297
|
+
async detect(text, onProgress, signal) {
|
|
298
|
+
throwIfAborted(signal);
|
|
297
299
|
if (!this.isLoaded())
|
|
298
300
|
await this.load();
|
|
299
301
|
if (!this.session || !this.tokenizer)
|
|
@@ -326,6 +328,8 @@ export class GlinerProvider {
|
|
|
326
328
|
// Multiple overlapping chunks
|
|
327
329
|
let chunksDone = 0;
|
|
328
330
|
for (let chunkStart = 0; chunkStart < words.length; chunkStart += chunkSize - overlap) {
|
|
331
|
+
// T222: a cancelled run stops here, before the next inference.
|
|
332
|
+
throwIfAborted(signal);
|
|
329
333
|
const chunkEnd = Math.min(chunkStart + chunkSize, words.length);
|
|
330
334
|
const chunkWords = words.slice(chunkStart, chunkEnd);
|
|
331
335
|
const chunkStarts = starts.slice(chunkStart, chunkEnd);
|
package/dist/regex/rules.data.js
CHANGED
|
@@ -36,7 +36,7 @@ export const RULES_DATA = [
|
|
|
36
36
|
{
|
|
37
37
|
"id": "regex:universal:iban",
|
|
38
38
|
"entityType": "IBAN",
|
|
39
|
-
"pattern": "\\b[A-Z]{2}\\s?\\d{2}
|
|
39
|
+
"pattern": "\\b[A-Z]{2}\\s?\\d{2}\\s?[A-Z\\d\\s]{9,29}[A-Z\\d]\\b",
|
|
40
40
|
"flags": "g",
|
|
41
41
|
"confidence": 0.9,
|
|
42
42
|
"domains": [
|
|
@@ -263,18 +263,19 @@ export const RULES_DATA = [
|
|
|
263
263
|
{
|
|
264
264
|
"id": "regex:universal:postal_city",
|
|
265
265
|
"entityType": "ADDRESS",
|
|
266
|
-
"pattern": "\\
|
|
266
|
+
"pattern": "(?:^|\\s)(?!\\d{4}-(?:\\d{2}|\\d{4})\\s)\\d{2,5}[-\\s]?\\d{2,4}\\s+[\\p{Lu}][\\p{L}]+(?![\\p{L}\\p{N}])",
|
|
267
267
|
"flags": "gu",
|
|
268
268
|
"confidence": 0.65,
|
|
269
269
|
"domains": [
|
|
270
270
|
"contact"
|
|
271
271
|
],
|
|
272
|
-
"description": "Postal/zip code followed by city name (various country formats)",
|
|
272
|
+
"description": "Postal/zip code followed by a capitalised city name (various country formats); the code must not be glued to a preceding digit, slash, dot or hyphen",
|
|
273
273
|
"examples": [
|
|
274
274
|
"00-950 Warszawa",
|
|
275
|
-
"75008 Paris"
|
|
275
|
+
"75008 Paris",
|
|
276
|
+
"1010 Wien"
|
|
276
277
|
],
|
|
277
|
-
"falsePositiveNotes": "May match non-address number+word sequences"
|
|
278
|
+
"falsePositiveNotes": "May match non-address number+word sequences (a number followed by a capitalised word)"
|
|
278
279
|
},
|
|
279
280
|
{
|
|
280
281
|
"id": "regex:universal:aws_access_key",
|
|
@@ -761,17 +762,21 @@ export const RULES_DATA = [
|
|
|
761
762
|
{
|
|
762
763
|
"id": "regex:pl:phone",
|
|
763
764
|
"entityType": "PHONE",
|
|
764
|
-
"pattern": "\\b(?:\\+?48[\\s-]?)?\\d{3}[\\s-]?\\d{3}[\\s-]?\\d{3}\\b",
|
|
765
|
+
"pattern": "(?:\\+?48[\\s-]?)?(?:\\(\\d{2}\\)|\\b\\d{2})[\\s-]?\\d{3}[\\s-]?\\d{2}[\\s-]?\\d{2}\\b|(?:\\+?48[\\s-]?)?\\b\\d{3}[\\s-]?\\d{3}[\\s-]?\\d{3}\\b",
|
|
765
766
|
"flags": "g",
|
|
766
767
|
"confidence": 0.8,
|
|
767
768
|
"domains": [
|
|
768
769
|
"contact"
|
|
769
770
|
],
|
|
770
|
-
"description": "Polish phone number
|
|
771
|
+
"description": "Polish phone number: 9 digits as mobile 3-3-3 or landline 2-3-2-2 (area code optionally in brackets), optionally with +48 prefix",
|
|
771
772
|
"examples": [
|
|
772
773
|
"+48 600 123 456",
|
|
773
774
|
"600-123-456",
|
|
774
|
-
"48 600123456"
|
|
775
|
+
"48 600123456",
|
|
776
|
+
"22 555 01 02",
|
|
777
|
+
"(22) 555-01-02",
|
|
778
|
+
"+48 22 555 01 02",
|
|
779
|
+
"22 5550102"
|
|
775
780
|
]
|
|
776
781
|
},
|
|
777
782
|
{
|
package/dist/types.d.ts
CHANGED
|
@@ -8,6 +8,16 @@ export interface DetectedEntity {
|
|
|
8
8
|
detector: string;
|
|
9
9
|
}
|
|
10
10
|
export type ProgressCallback = (downloaded: number, total: number) => void;
|
|
11
|
+
/**
|
|
12
|
+
* A detect call was cancelled through its AbortSignal (T222). Providers throw
|
|
13
|
+
* it between chunks, the engine before and after the ML pass, and the worker
|
|
14
|
+
* protocol client when the host acknowledges a `cancelDetect`.
|
|
15
|
+
*/
|
|
16
|
+
export declare class DetectionAbortedError extends Error {
|
|
17
|
+
constructor(message?: string);
|
|
18
|
+
}
|
|
19
|
+
/** Throw DetectionAbortedError when `signal` has been aborted. */
|
|
20
|
+
export declare function throwIfAborted(signal: AbortSignal | undefined): void;
|
|
11
21
|
/**
|
|
12
22
|
* Interface for detection providers.
|
|
13
23
|
* Implement this to add a new model/detection backend.
|
|
@@ -24,8 +34,13 @@ export interface DetectionProvider {
|
|
|
24
34
|
isLoading(): boolean;
|
|
25
35
|
/** Register a callback for download progress updates */
|
|
26
36
|
onProgress(callback: ProgressCallback): void;
|
|
27
|
-
/**
|
|
28
|
-
|
|
37
|
+
/**
|
|
38
|
+
* Run detection on the given text. Optional progress callback (0-1).
|
|
39
|
+
* `signal` (T222) is checked between inference chunks: an aborted signal
|
|
40
|
+
* makes the provider stop at the next chunk boundary and reject with
|
|
41
|
+
* DetectionAbortedError (work already done is discarded).
|
|
42
|
+
*/
|
|
43
|
+
detect(text: string, onProgress?: (progress: number) => void, signal?: AbortSignal): Promise<DetectedEntity[]>;
|
|
29
44
|
/** Set detection confidence threshold */
|
|
30
45
|
setThreshold(value: number): void;
|
|
31
46
|
/** Get current detection confidence threshold */
|
package/dist/types.js
CHANGED
|
@@ -1,3 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A detect call was cancelled through its AbortSignal (T222). Providers throw
|
|
3
|
+
* it between chunks, the engine before and after the ML pass, and the worker
|
|
4
|
+
* protocol client when the host acknowledges a `cancelDetect`.
|
|
5
|
+
*/
|
|
6
|
+
export class DetectionAbortedError extends Error {
|
|
7
|
+
constructor(message = 'Detection aborted') {
|
|
8
|
+
super(message);
|
|
9
|
+
this.name = 'DetectionAbortedError';
|
|
10
|
+
Object.setPrototypeOf(this, new.target.prototype);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
/** Throw DetectionAbortedError when `signal` has been aborted. */
|
|
14
|
+
export function throwIfAborted(signal) {
|
|
15
|
+
if (signal?.aborted)
|
|
16
|
+
throw new DetectionAbortedError();
|
|
17
|
+
}
|
|
1
18
|
export const ENTITY_COLORS = {
|
|
2
19
|
PERSON: '#7B2FF7',
|
|
3
20
|
EMAIL: '#2563EB',
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*
|
|
8
8
|
* init / switchProvider -> loaded | loadError (+ downloadProgress events)
|
|
9
9
|
* detect -> detected | detectError (+ detectionProgress events)
|
|
10
|
+
* cancelDetect -> detectError { aborted: true } on the cancelled requestId (T222)
|
|
10
11
|
* releaseModel -> released
|
|
11
12
|
* setThreshold / setCustomLabels / setRegex / setRegexRegion (fire and forget)
|
|
12
13
|
*
|
|
@@ -39,6 +40,15 @@ export type EngineRequest = {
|
|
|
39
40
|
type: 'detect';
|
|
40
41
|
requestId: number;
|
|
41
42
|
text: string;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* T222: stop the detect call with this requestId at its next chunk
|
|
46
|
+
* boundary. Unknown or already finished ids are ignored; the cancelled
|
|
47
|
+
* call answers with detectError { aborted: true }.
|
|
48
|
+
*/
|
|
49
|
+
| {
|
|
50
|
+
type: 'cancelDetect';
|
|
51
|
+
requestId: number;
|
|
42
52
|
} | {
|
|
43
53
|
type: 'switchProvider';
|
|
44
54
|
requestId?: number;
|
|
@@ -79,6 +89,7 @@ export type EngineResponse = {
|
|
|
79
89
|
type: 'detectError';
|
|
80
90
|
requestId: number;
|
|
81
91
|
error: string;
|
|
92
|
+
aborted?: boolean;
|
|
82
93
|
} | {
|
|
83
94
|
type: 'detectionProgress';
|
|
84
95
|
requestId: number;
|
package/dist/worker-protocol.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*
|
|
8
8
|
* init / switchProvider -> loaded | loadError (+ downloadProgress events)
|
|
9
9
|
* detect -> detected | detectError (+ detectionProgress events)
|
|
10
|
+
* cancelDetect -> detectError { aborted: true } on the cancelled requestId (T222)
|
|
10
11
|
* releaseModel -> released
|
|
11
12
|
* setThreshold / setCustomLabels / setRegex / setRegexRegion (fire and forget)
|
|
12
13
|
*
|
|
@@ -19,6 +20,7 @@
|
|
|
19
20
|
* is the client side: a DocCloakEngine that forwards over the port (web
|
|
20
21
|
* main thread, extension side panel / service worker).
|
|
21
22
|
*/
|
|
23
|
+
import { DetectionAbortedError } from "./types.js";
|
|
22
24
|
import { PROVIDERS, clampThreshold, defaultThresholdFor } from "./engine.js";
|
|
23
25
|
import { REGEX_REGIONS } from "./regex/index.js";
|
|
24
26
|
function errorMessage(err) {
|
|
@@ -26,7 +28,7 @@ function errorMessage(err) {
|
|
|
26
28
|
}
|
|
27
29
|
// ── Message validation (R12) ───────────────────────────────
|
|
28
30
|
const REQUEST_TYPES = new Set([
|
|
29
|
-
'init', 'detect', 'switchProvider', 'setThreshold', 'setCustomLabels', 'setRegex', 'setRegexRegion', 'releaseModel',
|
|
31
|
+
'init', 'detect', 'cancelDetect', 'switchProvider', 'setThreshold', 'setCustomLabels', 'setRegex', 'setRegexRegion', 'releaseModel',
|
|
30
32
|
]);
|
|
31
33
|
function isRecord(value) {
|
|
32
34
|
return typeof value === 'object' && value !== null;
|
|
@@ -68,6 +70,10 @@ function requestProblem(msg) {
|
|
|
68
70
|
if (typeof msg.text !== 'string')
|
|
69
71
|
return 'text must be a string';
|
|
70
72
|
return null;
|
|
73
|
+
case 'cancelDetect':
|
|
74
|
+
if (!isFiniteNumber(msg.requestId))
|
|
75
|
+
return 'requestId must be a number';
|
|
76
|
+
return null;
|
|
71
77
|
case 'switchProvider':
|
|
72
78
|
if (!isProviderId(msg.providerId))
|
|
73
79
|
return 'providerId must be a known provider id';
|
|
@@ -109,6 +115,8 @@ export function serveEngine(engine, port) {
|
|
|
109
115
|
function post(msg) {
|
|
110
116
|
port.postMessage(msg);
|
|
111
117
|
}
|
|
118
|
+
/** In-flight detect calls by requestId, so cancelDetect can abort them (T222). */
|
|
119
|
+
const inflight = new Map();
|
|
112
120
|
async function handleLoad(requestId, run) {
|
|
113
121
|
try {
|
|
114
122
|
await run();
|
|
@@ -177,17 +185,30 @@ export function serveEngine(engine, port) {
|
|
|
177
185
|
break;
|
|
178
186
|
}
|
|
179
187
|
case 'detect': {
|
|
188
|
+
// A repeated requestId supersedes the earlier call's controller;
|
|
189
|
+
// the earlier call keeps running to completion on its own signal.
|
|
190
|
+
const controller = new AbortController();
|
|
191
|
+
inflight.set(msg.requestId, controller);
|
|
180
192
|
try {
|
|
181
|
-
const entities = await engine.detect(msg.text,
|
|
193
|
+
const entities = await engine.detect(msg.text, controller.signal, (progress) => {
|
|
182
194
|
post({ type: 'detectionProgress', requestId: msg.requestId, progress });
|
|
183
195
|
});
|
|
184
196
|
post({ type: 'detected', requestId: msg.requestId, entities });
|
|
185
197
|
}
|
|
186
198
|
catch (err) {
|
|
187
|
-
|
|
199
|
+
const aborted = err instanceof DetectionAbortedError || controller.signal.aborted;
|
|
200
|
+
post({ type: 'detectError', requestId: msg.requestId, error: errorMessage(err), aborted });
|
|
201
|
+
}
|
|
202
|
+
finally {
|
|
203
|
+
if (inflight.get(msg.requestId) === controller)
|
|
204
|
+
inflight.delete(msg.requestId);
|
|
188
205
|
}
|
|
189
206
|
break;
|
|
190
207
|
}
|
|
208
|
+
case 'cancelDetect': {
|
|
209
|
+
inflight.get(msg.requestId)?.abort();
|
|
210
|
+
break;
|
|
211
|
+
}
|
|
191
212
|
case 'setThreshold': {
|
|
192
213
|
await handleSetting(() => engine.updateSettings({ threshold: msg.value }));
|
|
193
214
|
break;
|
|
@@ -316,7 +337,7 @@ export function connectEngine(port, initial) {
|
|
|
316
337
|
break;
|
|
317
338
|
}
|
|
318
339
|
case 'detectError': {
|
|
319
|
-
takePending(msg.requestId)?.reject(new Error(msg.error));
|
|
340
|
+
takePending(msg.requestId)?.reject(msg.aborted ? new DetectionAbortedError(msg.error) : new Error(msg.error));
|
|
320
341
|
break;
|
|
321
342
|
}
|
|
322
343
|
case 'released': {
|
|
@@ -386,15 +407,27 @@ export function connectEngine(port, initial) {
|
|
|
386
407
|
await switchProvider(patch.providerId);
|
|
387
408
|
}
|
|
388
409
|
}
|
|
410
|
+
/**
|
|
411
|
+
* Detect over the port. An aborted `signal` sends cancelDetect to the host
|
|
412
|
+
* and the promise rejects with DetectionAbortedError once the host has
|
|
413
|
+
* acknowledged (its detectError { aborted: true }), so a resolved rejection
|
|
414
|
+
* means the worker is idle again; a host that never answers is the
|
|
415
|
+
* caller's watchdog's business (the web app terminates it after a grace
|
|
416
|
+
* period). A signal aborted before the call rejects at once.
|
|
417
|
+
*/
|
|
389
418
|
function detect(text, signal, onProgress) {
|
|
390
419
|
if (signal?.aborted)
|
|
391
|
-
return Promise.reject(new
|
|
420
|
+
return Promise.reject(new DetectionAbortedError());
|
|
392
421
|
const promise = request('detect', { type: 'detect', text }, onProgress);
|
|
393
422
|
if (signal) {
|
|
394
423
|
const requestId = nextRequestId;
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
424
|
+
const onAbort = () => {
|
|
425
|
+
if (closed || !pending.has(requestId))
|
|
426
|
+
return;
|
|
427
|
+
port.postMessage({ type: 'cancelDetect', requestId });
|
|
428
|
+
};
|
|
429
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
430
|
+
promise.finally(() => signal.removeEventListener('abort', onAbort)).catch(() => { });
|
|
398
431
|
}
|
|
399
432
|
return promise;
|
|
400
433
|
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Digitized data copyright (c) 2010 Google Corporation
|
|
2
|
+
with Reserved Font Arimo, Tinos and Cousine.
|
|
3
|
+
Copyright (c) 2012 Red Hat, Inc.
|
|
4
|
+
with Reserved Font Name Liberation.
|
|
5
|
+
|
|
6
|
+
This Font Software is licensed under the SIL Open Font License,
|
|
7
|
+
Version 1.1.
|
|
8
|
+
|
|
9
|
+
This license is copied below, and is also available with a FAQ at:
|
|
10
|
+
http://scripts.sil.org/OFL
|
|
11
|
+
|
|
12
|
+
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
|
13
|
+
|
|
14
|
+
PREAMBLE The goals of the Open Font License (OFL) are to stimulate
|
|
15
|
+
worldwide development of collaborative font projects, to support the font
|
|
16
|
+
creation efforts of academic and linguistic communities, and to provide
|
|
17
|
+
a free and open framework in which fonts may be shared and improved in
|
|
18
|
+
partnership with others.
|
|
19
|
+
|
|
20
|
+
The OFL allows the licensed fonts to be used, studied, modified and
|
|
21
|
+
redistributed freely as long as they are not sold by themselves.
|
|
22
|
+
The fonts, including any derivative works, can be bundled, embedded,
|
|
23
|
+
redistributed and/or sold with any software provided that any reserved
|
|
24
|
+
names are not used by derivative works. The fonts and derivatives,
|
|
25
|
+
however, cannot be released under any other type of license. The
|
|
26
|
+
requirement for fonts to remain under this license does not apply to
|
|
27
|
+
any document created using the fonts or their derivatives.
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
DEFINITIONS
|
|
32
|
+
"Font Software" refers to the set of files released by the Copyright
|
|
33
|
+
Holder(s) under this license and clearly marked as such.
|
|
34
|
+
This may include source files, build scripts and documentation.
|
|
35
|
+
|
|
36
|
+
"Reserved Font Name" refers to any names specified as such after the
|
|
37
|
+
copyright statement(s).
|
|
38
|
+
|
|
39
|
+
"Original Version" refers to the collection of Font Software components
|
|
40
|
+
as distributed by the Copyright Holder(s).
|
|
41
|
+
|
|
42
|
+
"Modified Version" refers to any derivative made by adding to, deleting,
|
|
43
|
+
or substituting ? in part or in whole ?
|
|
44
|
+
any of the components of the Original Version, by changing formats or
|
|
45
|
+
by porting the Font Software to a new environment.
|
|
46
|
+
|
|
47
|
+
"Author" refers to any designer, engineer, programmer, technical writer
|
|
48
|
+
or other person who contributed to the Font Software.
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
PERMISSION & CONDITIONS
|
|
52
|
+
|
|
53
|
+
Permission is hereby granted, free of charge, to any person obtaining a
|
|
54
|
+
copy of the Font Software, to use, study, copy, merge, embed, modify,
|
|
55
|
+
redistribute, and sell modified and unmodified copies of the Font
|
|
56
|
+
Software, subject to the following conditions:
|
|
57
|
+
|
|
58
|
+
1) Neither the Font Software nor any of its individual components,in
|
|
59
|
+
Original or Modified Versions, may be sold by itself.
|
|
60
|
+
|
|
61
|
+
2) Original or Modified Versions of the Font Software may be bundled,
|
|
62
|
+
redistributed and/or sold with any software, provided that each copy
|
|
63
|
+
contains the above copyright notice and this license. These can be
|
|
64
|
+
included either as stand-alone text files, human-readable headers or
|
|
65
|
+
in the appropriate machine-readable metadata fields within text or
|
|
66
|
+
binary files as long as those fields can be easily viewed by the user.
|
|
67
|
+
|
|
68
|
+
3) No Modified Version of the Font Software may use the Reserved Font
|
|
69
|
+
Name(s) unless explicit written permission is granted by the
|
|
70
|
+
corresponding Copyright Holder. This restriction only applies to the
|
|
71
|
+
primary font name as presented to the users.
|
|
72
|
+
|
|
73
|
+
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
|
74
|
+
Software shall not be used to promote, endorse or advertise any
|
|
75
|
+
Modified Version, except to acknowledge the contribution(s) of the
|
|
76
|
+
Copyright Holder(s) and the Author(s) or with their explicit written
|
|
77
|
+
permission.
|
|
78
|
+
|
|
79
|
+
5) The Font Software, modified or unmodified, in part or in whole, must
|
|
80
|
+
be distributed entirely under this license, and must not be distributed
|
|
81
|
+
under any other license. The requirement for fonts to remain under
|
|
82
|
+
this license does not apply to any document created using the Font
|
|
83
|
+
Software.
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
TERMINATION
|
|
88
|
+
This license becomes null and void if any of the above conditions are not met.
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
DISCLAIMER
|
|
93
|
+
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
94
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
|
95
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
|
96
|
+
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
|
97
|
+
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
98
|
+
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
|
99
|
+
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
100
|
+
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER
|
|
101
|
+
DEALINGS IN THE FONT SOFTWARE.
|
|
102
|
+
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|