@stll/anonymize 0.0.9 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index.d.ts → index.d.mts} +17 -37
- package/dist/{index.js → index.mjs} +95 -539
- package/dist/index.mjs.map +1 -0
- package/package.json +7 -7
- package/dist/index.js.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Validator } from "@stll/stdnum";
|
|
2
1
|
import { Match, PatternEntry, TextSearch } from "@stll/text-search";
|
|
2
|
+
import { Validator } from "@stll/stdnum";
|
|
3
3
|
import { Tokenizer } from "@huggingface/tokenizers";
|
|
4
4
|
|
|
5
5
|
//#region src/types.d.ts
|
|
@@ -143,9 +143,7 @@ declare const OPERATOR_TYPES: readonly ["replace", "redact"];
|
|
|
143
143
|
type OperatorType = (typeof OPERATOR_TYPES)[number];
|
|
144
144
|
/** Per-label operator selection. Key is the entity label. */
|
|
145
145
|
type OperatorConfig = {
|
|
146
|
-
/** Operator per label. Missing labels default to "replace". */
|
|
147
|
-
operators: Record<string, OperatorType>;
|
|
148
|
-
/** Custom replacement string for the redact operator. */
|
|
146
|
+
/** Operator per label. Missing labels default to "replace". */operators: Record<string, OperatorType>; /** Custom replacement string for the redact operator. */
|
|
149
147
|
redactString: string;
|
|
150
148
|
};
|
|
151
149
|
/** Whether an operator produces a reversible redaction entry. */
|
|
@@ -168,8 +166,7 @@ type RedactionResult = {
|
|
|
168
166
|
* Maps placeholder to original text. Only populated for
|
|
169
167
|
* reversible operators (replace). Empty for redact.
|
|
170
168
|
*/
|
|
171
|
-
redactionMap: Map<string, string>;
|
|
172
|
-
/** Maps placeholder to the operator that produced it. */
|
|
169
|
+
redactionMap: Map<string, string>; /** Maps placeholder to the operator that produced it. */
|
|
173
170
|
operatorMap: Map<string, OperatorType>;
|
|
174
171
|
entityCount: number;
|
|
175
172
|
};
|
|
@@ -209,8 +206,7 @@ declare const DEFAULT_ENTITY_LABELS: readonly ["person", "organization", "phone
|
|
|
209
206
|
//#region src/detectors/regex.d.ts
|
|
210
207
|
type RegexMeta = {
|
|
211
208
|
label: string;
|
|
212
|
-
score: number;
|
|
213
|
-
/** Post-match stdnum validator for confirmation. */
|
|
209
|
+
score: number; /** Post-match stdnum validator for confirmation. */
|
|
214
210
|
validator?: Validator;
|
|
215
211
|
};
|
|
216
212
|
/** Flat pattern array for text-search. */
|
|
@@ -269,10 +265,8 @@ type DenyListData = {
|
|
|
269
265
|
* appears in multiple dictionaries (e.g., "Denver"
|
|
270
266
|
* is both a person name and a city name).
|
|
271
267
|
*/
|
|
272
|
-
labels: string[][];
|
|
273
|
-
/** Maps pattern index →
|
|
274
|
-
originals: string[];
|
|
275
|
-
/** Maps pattern index → source types (plural). */
|
|
268
|
+
labels: string[][]; /** Maps pattern index → original pattern text. */
|
|
269
|
+
originals: string[]; /** Maps pattern index → source types (plural). */
|
|
276
270
|
sources: PatternSource[][];
|
|
277
271
|
};
|
|
278
272
|
/**
|
|
@@ -317,8 +311,7 @@ type PatternSlice = {
|
|
|
317
311
|
end: number;
|
|
318
312
|
};
|
|
319
313
|
type GazetteerData = {
|
|
320
|
-
/** Maps local pattern index to entry label. */
|
|
321
|
-
labels: string[];
|
|
314
|
+
/** Maps local pattern index to entry label. */labels: string[];
|
|
322
315
|
/**
|
|
323
316
|
* Whether each pattern is fuzzy (distance > 0).
|
|
324
317
|
* Used by the post-processor to assign scores.
|
|
@@ -326,9 +319,7 @@ type GazetteerData = {
|
|
|
326
319
|
isFuzzy: boolean[];
|
|
327
320
|
};
|
|
328
321
|
type UnifiedSearchInstance = {
|
|
329
|
-
/** Regex + triggers + legal-forms. */
|
|
330
|
-
tsRegex: TextSearch;
|
|
331
|
-
/** Deny-list + street-types + gazetteer. */
|
|
322
|
+
/** Regex + triggers + legal-forms. */tsRegex: TextSearch; /** Deny-list + street-types + gazetteer. */
|
|
332
323
|
tsLiterals: TextSearch;
|
|
333
324
|
slices: {
|
|
334
325
|
regex: PatternSlice;
|
|
@@ -369,8 +360,7 @@ type NameCorpusData = {
|
|
|
369
360
|
firstNames: ReadonlySet<string>;
|
|
370
361
|
surnames: ReadonlySet<string>;
|
|
371
362
|
titleTokens: ReadonlySet<string>;
|
|
372
|
-
excludedWords: ReadonlySet<string>;
|
|
373
|
-
/** Raw arrays exposed for deny-list AC integration. */
|
|
363
|
+
excludedWords: ReadonlySet<string>; /** Raw arrays exposed for deny-list AC integration. */
|
|
374
364
|
firstNamesList: readonly string[];
|
|
375
365
|
surnamesList: readonly string[];
|
|
376
366
|
titlesList: readonly string[];
|
|
@@ -397,8 +387,7 @@ type PipelineContext = {
|
|
|
397
387
|
allowList: ReadonlySet<string> | null;
|
|
398
388
|
allowListPromise: Promise<ReadonlySet<string>> | null;
|
|
399
389
|
personStopwords: ReadonlySet<string> | null;
|
|
400
|
-
personStopwordsPromise: Promise<ReadonlySet<string>> | null;
|
|
401
|
-
/** First-name exclusions for stopword filtering. */
|
|
390
|
+
personStopwordsPromise: Promise<ReadonlySet<string>> | null; /** First-name exclusions for stopword filtering. */
|
|
402
391
|
firstNameExclusions: ReadonlySet<string> | null;
|
|
403
392
|
firstNameExclusionCorpusLen: number;
|
|
404
393
|
genericRoles: ReadonlySet<string> | null;
|
|
@@ -425,11 +414,6 @@ type PipelineContext = {
|
|
|
425
414
|
};
|
|
426
415
|
/** Create a fresh, empty pipeline context. */
|
|
427
416
|
declare const createPipelineContext: () => PipelineContext;
|
|
428
|
-
/**
|
|
429
|
-
* Module-level default context. Used when callers
|
|
430
|
-
* don't provide an explicit context, preserving full
|
|
431
|
-
* backward compatibility with the existing API.
|
|
432
|
-
*/
|
|
433
417
|
//#endregion
|
|
434
418
|
//#region src/pipeline.d.ts
|
|
435
419
|
/** Strip leading/trailing whitespace and punctuation. */
|
|
@@ -613,10 +597,8 @@ declare const processGazetteerMatches: (allMatches: Match[], sliceStart: number,
|
|
|
613
597
|
//#region src/detectors/coreference.d.ts
|
|
614
598
|
type DefinedTerm = {
|
|
615
599
|
alias: string;
|
|
616
|
-
label: string;
|
|
617
|
-
|
|
618
|
-
definitionStart: number;
|
|
619
|
-
/** Original entity text the alias refers to */
|
|
600
|
+
label: string; /** Position of the definition in the source text */
|
|
601
|
+
definitionStart: number; /** Original entity text the alias refers to */
|
|
620
602
|
sourceText: string;
|
|
621
603
|
};
|
|
622
604
|
declare const extractDefinedTerms: (fullText: string, entities: Entity[], ctx?: PipelineContext) => Promise<DefinedTerm[]>;
|
|
@@ -647,7 +629,9 @@ declare const propagateOrgNames: (entities: Entity[], fullText: string) => Entit
|
|
|
647
629
|
//#endregion
|
|
648
630
|
//#region src/detectors/names.d.ts
|
|
649
631
|
/**
|
|
650
|
-
* Load name corpus data from
|
|
632
|
+
* Load name corpus data from per-language dictionary
|
|
633
|
+
* files and legacy config files. Merges all sources.
|
|
634
|
+
*
|
|
651
635
|
* Safe to call multiple times; only loads once per
|
|
652
636
|
* context. Must be called before detectNameCorpus or
|
|
653
637
|
* the getNameCorpus*() accessors are used.
|
|
@@ -673,8 +657,7 @@ declare const detectNameCorpus: (fullText: string, ctx?: PipelineContext) => Ent
|
|
|
673
657
|
//#endregion
|
|
674
658
|
//#region src/unified-search.d.ts
|
|
675
659
|
type UnifiedResult = {
|
|
676
|
-
/** All matches from both instances combined. */
|
|
677
|
-
regexMatches: Match[];
|
|
660
|
+
/** All matches from both instances combined. */regexMatches: Match[];
|
|
678
661
|
literalMatches: Match[];
|
|
679
662
|
};
|
|
680
663
|
declare const runUnifiedSearch: (instance: UnifiedSearchInstance, fullText: string) => UnifiedResult;
|
|
@@ -740,7 +723,6 @@ declare const filterFalsePositives: (entities: Entity[], ctx?: PipelineContext)
|
|
|
740
723
|
* entities pass through unchanged.
|
|
741
724
|
*/
|
|
742
725
|
declare const boostNearMissEntities: (entities: Entity[], threshold: number) => Entity[];
|
|
743
|
-
/** Ensure preposition data is loaded. */
|
|
744
726
|
//#endregion
|
|
745
727
|
//#region src/filters/hotword-rules.d.ts
|
|
746
728
|
type HotwordRule = {
|
|
@@ -849,8 +831,6 @@ declare const decodeTokenSpans: (batchSize: number, numWords: number, numEntitie
|
|
|
849
831
|
//#region src/gliner/processor.d.ts
|
|
850
832
|
/** Tokenize text into words with character offsets. */
|
|
851
833
|
declare const tokenizeText: (text: string) => [words: string[], starts: number[], ends: number[]];
|
|
852
|
-
/** Pad a 2D or 3D array to uniform inner length. */
|
|
853
|
-
|
|
854
834
|
/** Prepare a complete batch for ONNX inference. */
|
|
855
835
|
declare const prepareBatch: (tokenizer: Tokenizer, texts: string[], entities: string[], maxWidth: number) => {
|
|
856
836
|
inputsIds: number[][];
|
|
@@ -936,4 +916,4 @@ declare const levenshtein: (rawA: string, rawB: string) => number;
|
|
|
936
916
|
declare const normalizeForSearch: (text: string) => string;
|
|
937
917
|
//#endregion
|
|
938
918
|
export { type AnonymisationOperator, CURRENCY_PATTERN_META, type CountryCode, DATE_PATTERN_META, DEFAULT_ENTITY_LABELS, DEFAULT_OPERATOR_CONFIG, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefinitionPattern, type DenyListCategory, type DenyListData, type DetectionSource, type DocumentZone, type Entity, type EntityResult, type GazetteerData, type GazetteerEntry, type HotwordRule, type NameCorpusData, type NerInferenceFn, OPERATOR_REGISTRY, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, type PipelineContext, type PipelineOptions, REGEX_META, REGEX_PATTERNS, REGIONS, type RawInferenceResult, type RedactionResult, type RegexMeta, type RegionId, type ReviewDecision, type ReviewedEntity, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, type UnifiedResult, type UnifiedSearchInstance, ZONE_SCORE_ADJUSTMENTS, type ZoneSpan, applyHotwordRules, applyZoneAdjustments, boostNearMissEntities, buildDenyList, buildGazetteerPatterns, buildLegalFormPatterns, buildPlaceholderMap, buildStreetTypePatterns, buildTriggerPatterns, buildUnifiedSearch, chunkText, classifyZones, computeChunkOffsets, corefKey, createPipelineContext, deanonymise, decodeSpans, decodeTokenSpans, detectNameCorpus, ensureDenyListData, exportRedactionKey, extractDefinedTerms, filterFalsePositives, findCoreferenceSpans, getCurrencyPatterns, getDatePatterns, initHotwordRules, initNameCorpus, initZoneClassifier, levenshtein, mergeAndDedup, mergeChunkEntities, normalizeForSearch, prepareBatch, processAddressSeeds, processDenyListMatches, processGazetteerMatches, processLegalFormMatches, processRegexMatches, processTriggerMatches, propagateOrgNames, redactText, resolveCountries, resolveOperator, runPipeline, runUnifiedSearch, sanitizeEntities, tokenizeText };
|
|
939
|
-
//# sourceMappingURL=index.d.
|
|
919
|
+
//# sourceMappingURL=index.d.mts.map
|