@stll/anonymize 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +160 -3
- package/dist/capabilities.d.mts +123 -0
- package/dist/capabilities.mjs +22 -0
- package/dist/capabilities.mjs.map +1 -0
- package/dist/constants.d.mts +2 -2
- package/dist/constants.mjs +158 -28
- package/dist/constants.mjs.map +1 -1
- package/dist/constants2.d.mts +123 -6
- package/dist/index.d.mts +4 -4
- package/dist/index.mjs +4 -3
- package/dist/index.mjs.map +1 -1
- package/dist/native-node.d.mts +8 -36
- package/dist/native-node.mjs +2 -2
- package/dist/native-node2.d.mts +2 -2
- package/dist/native-node2.mjs +17 -8
- package/dist/native-node2.mjs.map +1 -1
- package/dist/native.d.mts +219 -88
- package/dist/native.mjs +281 -4
- package/dist/native.mjs.map +1 -1
- package/dist/native2.d.mts +2 -2
- package/package.json +16 -12
package/dist/constants2.d.mts
CHANGED
|
@@ -50,18 +50,135 @@ declare const DETECTOR_PRIORITY: {
|
|
|
50
50
|
* Anonymization operator types. Each operator defines
|
|
51
51
|
* how a confirmed entity is replaced in the output.
|
|
52
52
|
*/
|
|
53
|
-
declare const OPERATOR_TYPES: readonly ["replace", "redact"];
|
|
53
|
+
declare const OPERATOR_TYPES: readonly ["replace", "redact", "keep", "mask"];
|
|
54
54
|
type OperatorType = (typeof OPERATOR_TYPES)[number];
|
|
55
|
+
declare const ENTITY_SELECTIONS: {
|
|
56
|
+
readonly DEFAULT: "default";
|
|
57
|
+
readonly OPT_IN: "opt-in";
|
|
58
|
+
};
|
|
59
|
+
type EntitySelection = (typeof ENTITY_SELECTIONS)[keyof typeof ENTITY_SELECTIONS];
|
|
60
|
+
type EntityCapability = {
|
|
61
|
+
label: string;
|
|
62
|
+
selection: EntitySelection;
|
|
63
|
+
detectionSources: readonly DetectionSource[];
|
|
64
|
+
};
|
|
55
65
|
/**
|
|
56
|
-
* Canonical entity
|
|
57
|
-
*
|
|
58
|
-
*
|
|
66
|
+
* Canonical entity capabilities exposed by the deterministic native pipeline.
|
|
67
|
+
* `selection` describes whether the default package requests the label; opt-in
|
|
68
|
+
* labels have built-in detection rules but must be requested explicitly.
|
|
59
69
|
*
|
|
60
70
|
* These labels are ephemeral: entities are regenerated on
|
|
61
71
|
* every pipeline run and never persisted to the database.
|
|
62
72
|
* Renaming a label here requires no migration.
|
|
63
73
|
*/
|
|
64
|
-
declare const
|
|
74
|
+
declare const ENTITY_CAPABILITIES: readonly [{
|
|
75
|
+
readonly label: "person";
|
|
76
|
+
readonly selection: "default";
|
|
77
|
+
readonly detectionSources: readonly ["trigger", "regex", "deny-list", "coreference"];
|
|
78
|
+
}, {
|
|
79
|
+
readonly label: "organization";
|
|
80
|
+
readonly selection: "default";
|
|
81
|
+
readonly detectionSources: readonly ["trigger", "deny-list", "legal-form", "gazetteer", "coreference"];
|
|
82
|
+
}, {
|
|
83
|
+
readonly label: "phone number";
|
|
84
|
+
readonly selection: "default";
|
|
85
|
+
readonly detectionSources: readonly ["regex", "trigger"];
|
|
86
|
+
}, {
|
|
87
|
+
readonly label: "address";
|
|
88
|
+
readonly selection: "default";
|
|
89
|
+
readonly detectionSources: readonly ["regex", "trigger", "deny-list"];
|
|
90
|
+
}, {
|
|
91
|
+
readonly label: "country";
|
|
92
|
+
readonly selection: "default";
|
|
93
|
+
readonly detectionSources: readonly ["country"];
|
|
94
|
+
}, {
|
|
95
|
+
readonly label: "email address";
|
|
96
|
+
readonly selection: "default";
|
|
97
|
+
readonly detectionSources: readonly ["regex"];
|
|
98
|
+
}, {
|
|
99
|
+
readonly label: "date";
|
|
100
|
+
readonly selection: "default";
|
|
101
|
+
readonly detectionSources: readonly ["regex", "trigger"];
|
|
102
|
+
}, {
|
|
103
|
+
readonly label: "date of birth";
|
|
104
|
+
readonly selection: "default";
|
|
105
|
+
readonly detectionSources: readonly ["trigger"];
|
|
106
|
+
}, {
|
|
107
|
+
readonly label: "bank account number";
|
|
108
|
+
readonly selection: "default";
|
|
109
|
+
readonly detectionSources: readonly ["regex", "trigger"];
|
|
110
|
+
}, {
|
|
111
|
+
readonly label: "iban";
|
|
112
|
+
readonly selection: "default";
|
|
113
|
+
readonly detectionSources: readonly ["regex", "trigger"];
|
|
114
|
+
}, {
|
|
115
|
+
readonly label: "tax identification number";
|
|
116
|
+
readonly selection: "default";
|
|
117
|
+
readonly detectionSources: readonly ["regex", "trigger"];
|
|
118
|
+
}, {
|
|
119
|
+
readonly label: "identity card number";
|
|
120
|
+
readonly selection: "default";
|
|
121
|
+
readonly detectionSources: readonly ["regex", "trigger"];
|
|
122
|
+
}, {
|
|
123
|
+
readonly label: "birth number";
|
|
124
|
+
readonly selection: "default";
|
|
125
|
+
readonly detectionSources: readonly ["regex", "trigger"];
|
|
126
|
+
}, {
|
|
127
|
+
readonly label: "national identification number";
|
|
128
|
+
readonly selection: "default";
|
|
129
|
+
readonly detectionSources: readonly ["regex", "trigger"];
|
|
130
|
+
}, {
|
|
131
|
+
readonly label: "social security number";
|
|
132
|
+
readonly selection: "default";
|
|
133
|
+
readonly detectionSources: readonly ["regex", "trigger"];
|
|
134
|
+
}, {
|
|
135
|
+
readonly label: "registration number";
|
|
136
|
+
readonly selection: "default";
|
|
137
|
+
readonly detectionSources: readonly ["regex", "trigger"];
|
|
138
|
+
}, {
|
|
139
|
+
readonly label: "credit card number";
|
|
140
|
+
readonly selection: "default";
|
|
141
|
+
readonly detectionSources: readonly ["regex"];
|
|
142
|
+
}, {
|
|
143
|
+
readonly label: "passport number";
|
|
144
|
+
readonly selection: "default";
|
|
145
|
+
readonly detectionSources: readonly ["regex"];
|
|
146
|
+
}, {
|
|
147
|
+
readonly label: "crypto";
|
|
148
|
+
readonly selection: "default";
|
|
149
|
+
readonly detectionSources: readonly ["regex"];
|
|
150
|
+
}, {
|
|
151
|
+
readonly label: "monetary amount";
|
|
152
|
+
readonly selection: "default";
|
|
153
|
+
readonly detectionSources: readonly ["regex", "trigger"];
|
|
154
|
+
}, {
|
|
155
|
+
readonly label: "land parcel";
|
|
156
|
+
readonly selection: "default";
|
|
157
|
+
readonly detectionSources: readonly ["trigger"];
|
|
158
|
+
}, {
|
|
159
|
+
readonly label: "misc";
|
|
160
|
+
readonly selection: "default";
|
|
161
|
+
readonly detectionSources: readonly ["regex", "deny-list"];
|
|
162
|
+
}, {
|
|
163
|
+
readonly label: "ip address";
|
|
164
|
+
readonly selection: "opt-in";
|
|
165
|
+
readonly detectionSources: readonly ["regex"];
|
|
166
|
+
}, {
|
|
167
|
+
readonly label: "mac address";
|
|
168
|
+
readonly selection: "opt-in";
|
|
169
|
+
readonly detectionSources: readonly ["regex"];
|
|
170
|
+
}, {
|
|
171
|
+
readonly label: "url";
|
|
172
|
+
readonly selection: "opt-in";
|
|
173
|
+
readonly detectionSources: readonly ["regex"];
|
|
174
|
+
}];
|
|
175
|
+
type KnownEntityCapability = (typeof ENTITY_CAPABILITIES)[number];
|
|
176
|
+
type EntityLabel = KnownEntityCapability["label"];
|
|
177
|
+
declare const ENTITY_LABELS: readonly EntityLabel[];
|
|
178
|
+
type DefaultEntityLabel = Extract<KnownEntityCapability, {
|
|
179
|
+
selection: typeof ENTITY_SELECTIONS.DEFAULT;
|
|
180
|
+
}>["label"];
|
|
181
|
+
declare const DEFAULT_ENTITY_LABELS: readonly DefaultEntityLabel[];
|
|
65
182
|
//#endregion
|
|
66
|
-
export {
|
|
183
|
+
export { DetectionSource as a, ENTITY_SELECTIONS as c, EntitySelection as d, OPERATOR_TYPES as f, DefaultEntityLabel as i, EntityCapability as l, DETECTION_SOURCES as n, ENTITY_CAPABILITIES as o, OperatorType as p, DETECTOR_PRIORITY as r, ENTITY_LABELS as s, DEFAULT_ENTITY_LABELS as t, EntityLabel as u };
|
|
67
184
|
//# sourceMappingURL=constants2.d.mts.map
|
package/dist/index.d.mts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { a as OPERATOR_TYPES, i as
|
|
2
|
-
import {
|
|
1
|
+
import { a as DetectionSource, c as ENTITY_SELECTIONS, d as EntitySelection, f as OPERATOR_TYPES, i as DefaultEntityLabel, l as EntityCapability, n as DETECTION_SOURCES, o as ENTITY_CAPABILITIES, p as OperatorType, r as DETECTOR_PRIORITY, s as ENTITY_LABELS, t as DEFAULT_ENTITY_LABELS, u as EntityLabel } from "./constants2.mjs";
|
|
2
|
+
import { CAPABILITY_MANIFEST, CAPABILITY_MANIFEST_SCHEMA_VERSION, CAPABILITY_RUNTIMES, CapabilityManifest, CapabilityRuntime } from "./capabilities.mjs";
|
|
3
|
+
import { $ as getNativeBindingVersion, A as NativeStaticRedactionResult, B as SharedNativePreparedPackageOptions, C as NativeSessionCallerRedactionInput, Ct as TriggerGroupConfig, D as NativeSessionMetadata, Dt as NativePreparedSearchConfig, E as NativeSessionLifecycle, Et as TriggerValidation, F as PreparedNativeRedactionSession, G as assertNativeBindingVersion, H as SharedNativeRedactTextOptions, I as PreparedNativeSessionRedactionPlan, J as createNativePipelineFromPackage, K as createNativeAnonymizerFromConfig, L as PreparedSearch, M as PreparedAnonymizer, N as PreparedNativeAnonymizer, O as NativeSessionRedactionAtOptions, P as PreparedNativePipeline, Q as encodeNativeSearchConfigInput, R as SharedNativeDiagnosticsJsonOptions, S as NativeSessionBlockRedactionPlan, St as TriggerExtension, T as NativeSessionDeletionSummary, Tt as TriggerStrategy, U as SharedNativeRedactTextStreamJsonOptions, V as SharedNativeRedactTextJsonOptions, W as SharedNativeSearchPackageOptions, Z as encodeNativeSearchConfig, _ as NativePreparedSessionRedactionPlanBinding, _t as OperatorConfig, a as NativeBindingVersionOptions, b as NativeSearchPackageInput, bt as ReviewDecision, c as NativeCreateSessionWithLifecycleOptions, d as NativeOpenSessionArchiveOptions, dt as CustomRegexPattern, f as NativeOperatorConfig, ft as DenyListCategory, g as NativePreparedSearchBinding, gt as GazetteerEntry, h as NativePreparedRedactionSessionBinding, ht as Entity, i as NativeAnonymizerFromPackageOptions, j as NativeTextReplacement, k as NativeSessionStatus, l as NativeDiagnosticsBatchCallback, lt as AnonymisationOperator, m as NativePipelineFromPackageOptions, mt as DictionaryMeta, n as NativeAnonymizeBinding, o as NativeCallerDetection, p as NativePipelineEntity, pt as Dictionaries, q as createNativeAnonymizerFromPackage, r as NativeAnonymizerFromConfigOptions, rt as prepareNativeSearchPackage, s as NativeCallerRedactionOptions, t as CALLER_DETECTION_CONTRACT_VERSION, u as NativeNormalizeOptions, ut as CustomDenyListEntry, v as NativeRedactionResult, vt as PipelineConfig, w as NativeSessionCallerRedactionPlanOptions, wt as TriggerRule, x as NativeSearchPackageOptions, xt as ReviewedEntity, y as NativeResultEventCallback, yt as RedactionResult, z as SharedNativeDiagnosticsStreamJsonOptions } from "./native.mjs";
|
|
3
4
|
import { A as readDefaultNativePipelinePackageFileAsync, B as redact_text_stream_json, C as native_package_version, D as preload_default_native_pipeline, E as preloadDefaultNativePipelineAsync, F as redactDefaultTextJson, G as NativePipelinePackageOptions, H as DEFAULT_NATIVE_PIPELINE_CONFIG, I as redact_default_text, J as createNativePipelineFromConfig, K as NativePipelineUnsupportedFeature, L as redact_default_text_json, M as readNativePipelinePackageFileAsync, N as read_default_native_pipeline_package_file, O as prepare_search_package, P as redactDefaultText, R as redact_text, S as load_prepared_package_file, T as preloadDefaultNativePipeline, U as NativePipelineBuildOptions, V as summary_diagnostics_json, W as NativePipelineCompatibility, X as prepareNativePipelineConfig, Y as getNativePipelineCompatibility, Z as prepareNativePipelinePackage, _ as diagnostics_stream_json, a as LoadNativeBindingOptions, b as loadNativeAnonymizeBinding, c as NativeRequire, d as availableDefaultNativePipelineLanguages, f as available_default_native_pipeline_languages, g as diagnostics_json, h as create_native_pipeline_from_default_package, i as DefaultNativePipelineWarmup, j as readNativePipelinePackageFile, k as readDefaultNativePipelinePackageFile, l as NativeSdkOptions, m as createNativePipelineFromPackageFile, n as DefaultNativePipelinePackageFileOptions, o as NativeLibc, p as createNativePipelineFromDefaultPackage, q as assertNativePipelineSupported, r as DefaultNativePipelinePackageOptions, s as NativePipelinePackageFileOptions, t as DEFAULT_NATIVE_PIPELINE_WARMUPS, u as NativeSdkPackageOptions, v as getDefaultNativePipeline, w as normalize_for_search, x as load_prepared_package, y as get_default_native_pipeline, z as redact_text_json } from "./native-node.mjs";
|
|
4
|
-
|
|
5
5
|
//#region src/redact.d.ts
|
|
6
6
|
/**
|
|
7
7
|
* Serialize the redaction key to JSON for export.
|
|
@@ -15,5 +15,5 @@ declare const exportRedactionKey: (redactionMap: Map<string, string>, operatorMa
|
|
|
15
15
|
*/
|
|
16
16
|
declare const deanonymise: (redactedText: string, redactionMap: Map<string, string>) => string;
|
|
17
17
|
//#endregion
|
|
18
|
-
export { type AnonymisationOperator, type CustomDenyListEntry, type CustomRegexPattern, DEFAULT_ENTITY_LABELS, DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_NATIVE_PIPELINE_WARMUPS, DETECTION_SOURCES, DETECTOR_PRIORITY, DefaultNativePipelinePackageFileOptions, DefaultNativePipelinePackageOptions, DefaultNativePipelineWarmup, type DenyListCategory, type DetectionSource, type Dictionaries, type DictionaryMeta, type Entity, type GazetteerEntry, LoadNativeBindingOptions, NativeAnonymizeBinding, NativeAnonymizerFromConfigOptions, NativeAnonymizerFromPackageOptions, NativeBindingVersionOptions, NativeDiagnosticsBatchCallback, NativeLibc, NativeNormalizeOptions, NativeOperatorConfig, type NativePipelineBuildOptions, type NativePipelineCompatibility, NativePipelineEntity, NativePipelineFromPackageOptions, NativePipelinePackageFileOptions, type NativePipelinePackageOptions, type NativePipelineUnsupportedFeature, NativePreparedSearchBinding, type NativePreparedSearchConfig, NativeRedactionResult, NativeRequire, NativeResultEventCallback, NativeSdkOptions, NativeSdkPackageOptions, NativeSearchPackageInput, NativeSearchPackageOptions, NativeStaticRedactionResult, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedSearch, type RedactionResult, type ReviewDecision, type ReviewedEntity, SharedNativeDiagnosticsJsonOptions, SharedNativeDiagnosticsStreamJsonOptions, SharedNativePreparedPackageOptions, SharedNativeRedactTextJsonOptions, SharedNativeRedactTextOptions, SharedNativeRedactTextStreamJsonOptions, SharedNativeSearchPackageOptions, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, assertNativeBindingVersion, assertNativePipelineSupported, availableDefaultNativePipelineLanguages, available_default_native_pipeline_languages, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromDefaultPackage, createNativePipelineFromPackage, createNativePipelineFromPackageFile, create_native_pipeline_from_default_package, deanonymise, diagnostics_json, diagnostics_stream_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, exportRedactionKey, getDefaultNativePipeline, getNativeBindingVersion, getNativePipelineCompatibility, get_default_native_pipeline, loadNativeAnonymizeBinding, load_prepared_package, load_prepared_package_file, native_package_version, normalize_for_search, preloadDefaultNativePipeline, preloadDefaultNativePipelineAsync, preload_default_native_pipeline, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, prepare_search_package, readDefaultNativePipelinePackageFile, readDefaultNativePipelinePackageFileAsync, readNativePipelinePackageFile, readNativePipelinePackageFileAsync, read_default_native_pipeline_package_file, redactDefaultText, redactDefaultTextJson, redact_default_text, redact_default_text_json, redact_text, redact_text_json, redact_text_stream_json, summary_diagnostics_json };
|
|
18
|
+
export { type AnonymisationOperator, CALLER_DETECTION_CONTRACT_VERSION, CAPABILITY_MANIFEST, CAPABILITY_MANIFEST_SCHEMA_VERSION, CAPABILITY_RUNTIMES, type CapabilityManifest, type CapabilityRuntime, type CustomDenyListEntry, type CustomRegexPattern, DEFAULT_ENTITY_LABELS, DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_NATIVE_PIPELINE_WARMUPS, DETECTION_SOURCES, DETECTOR_PRIORITY, type DefaultEntityLabel, DefaultNativePipelinePackageFileOptions, DefaultNativePipelinePackageOptions, DefaultNativePipelineWarmup, type DenyListCategory, type DetectionSource, type Dictionaries, type DictionaryMeta, ENTITY_CAPABILITIES, ENTITY_LABELS, ENTITY_SELECTIONS, type Entity, type EntityCapability, type EntityLabel, type EntitySelection, type GazetteerEntry, LoadNativeBindingOptions, NativeAnonymizeBinding, NativeAnonymizerFromConfigOptions, NativeAnonymizerFromPackageOptions, NativeBindingVersionOptions, NativeCallerDetection, NativeCallerRedactionOptions, NativeCreateSessionWithLifecycleOptions, NativeDiagnosticsBatchCallback, NativeLibc, NativeNormalizeOptions, NativeOpenSessionArchiveOptions, NativeOperatorConfig, type NativePipelineBuildOptions, type NativePipelineCompatibility, NativePipelineEntity, NativePipelineFromPackageOptions, NativePipelinePackageFileOptions, type NativePipelinePackageOptions, type NativePipelineUnsupportedFeature, NativePreparedRedactionSessionBinding, NativePreparedSearchBinding, type NativePreparedSearchConfig, NativePreparedSessionRedactionPlanBinding, NativeRedactionResult, NativeRequire, NativeResultEventCallback, NativeSdkOptions, NativeSdkPackageOptions, NativeSearchPackageInput, NativeSearchPackageOptions, NativeSessionBlockRedactionPlan, NativeSessionCallerRedactionInput, NativeSessionCallerRedactionPlanOptions, NativeSessionDeletionSummary, NativeSessionLifecycle, NativeSessionMetadata, NativeSessionRedactionAtOptions, NativeSessionStatus, NativeStaticRedactionResult, NativeTextReplacement, OPERATOR_TYPES, type OperatorConfig, type OperatorType, type PipelineConfig, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedNativeRedactionSession, PreparedNativeSessionRedactionPlan, PreparedSearch, type RedactionResult, type ReviewDecision, type ReviewedEntity, SharedNativeDiagnosticsJsonOptions, SharedNativeDiagnosticsStreamJsonOptions, SharedNativePreparedPackageOptions, SharedNativeRedactTextJsonOptions, SharedNativeRedactTextOptions, SharedNativeRedactTextStreamJsonOptions, SharedNativeSearchPackageOptions, type TriggerExtension, type TriggerGroupConfig, type TriggerRule, type TriggerStrategy, type TriggerValidation, assertNativeBindingVersion, assertNativePipelineSupported, availableDefaultNativePipelineLanguages, available_default_native_pipeline_languages, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromDefaultPackage, createNativePipelineFromPackage, createNativePipelineFromPackageFile, create_native_pipeline_from_default_package, deanonymise, diagnostics_json, diagnostics_stream_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, exportRedactionKey, getDefaultNativePipeline, getNativeBindingVersion, getNativePipelineCompatibility, get_default_native_pipeline, loadNativeAnonymizeBinding, load_prepared_package, load_prepared_package_file, native_package_version, normalize_for_search, preloadDefaultNativePipeline, preloadDefaultNativePipelineAsync, preload_default_native_pipeline, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, prepare_search_package, readDefaultNativePipelinePackageFile, readDefaultNativePipelinePackageFileAsync, readNativePipelinePackageFile, readNativePipelinePackageFileAsync, read_default_native_pipeline_package_file, redactDefaultText, redactDefaultTextJson, redact_default_text, redact_default_text_json, redact_text, redact_text_json, redact_text_stream_json, summary_diagnostics_json };
|
|
19
19
|
//# sourceMappingURL=index.d.mts.map
|
package/dist/index.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedSearch, assertNativeBindingVersion, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromPackage, encodeNativeSearchConfig, encodeNativeSearchConfigInput, getNativeBindingVersion, prepareNativeSearchPackage } from "./native.mjs";
|
|
1
|
+
import { CALLER_DETECTION_CONTRACT_VERSION, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedNativeRedactionSession, PreparedNativeSessionRedactionPlan, PreparedSearch, assertNativeBindingVersion, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromPackage, encodeNativeSearchConfig, encodeNativeSearchConfigInput, getNativeBindingVersion, prepareNativeSearchPackage } from "./native.mjs";
|
|
2
2
|
import { A as redact_text_json, C as readNativePipelinePackageFileAsync, D as redact_default_text, E as redactDefaultTextJson, F as createNativePipelineFromConfig, I as getNativePipelineCompatibility, L as prepareNativePipelineConfig, M as summary_diagnostics_json, N as DEFAULT_NATIVE_PIPELINE_CONFIG, O as redact_default_text_json, P as assertNativePipelineSupported, R as prepareNativePipelinePackage, S as readNativePipelinePackageFile, T as redactDefaultText, _ as preloadDefaultNativePipelineAsync, a as createNativePipelineFromPackageFile, b as readDefaultNativePipelinePackageFile, c as diagnostics_stream_json, d as loadNativeAnonymizeBinding, f as load_prepared_package, g as preloadDefaultNativePipeline, h as normalize_for_search, i as createNativePipelineFromDefaultPackage, j as redact_text_stream_json, k as redact_text, l as getDefaultNativePipeline, m as native_package_version, n as availableDefaultNativePipelineLanguages, o as create_native_pipeline_from_default_package, p as load_prepared_package_file, r as available_default_native_pipeline_languages, s as diagnostics_json, t as DEFAULT_NATIVE_PIPELINE_WARMUPS, u as get_default_native_pipeline, v as preload_default_native_pipeline, w as read_default_native_pipeline_package_file, x as readDefaultNativePipelinePackageFileAsync, y as prepare_search_package } from "./native-node2.mjs";
|
|
3
|
-
import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_TYPES } from "./constants.mjs";
|
|
3
|
+
import { DEFAULT_ENTITY_LABELS, DETECTION_SOURCES, DETECTOR_PRIORITY, ENTITY_CAPABILITIES, ENTITY_LABELS, ENTITY_SELECTIONS, OPERATOR_TYPES } from "./constants.mjs";
|
|
4
|
+
import { CAPABILITY_MANIFEST, CAPABILITY_MANIFEST_SCHEMA_VERSION, CAPABILITY_RUNTIMES } from "./capabilities.mjs";
|
|
4
5
|
//#region src/redact.ts
|
|
5
6
|
/**
|
|
6
7
|
* Serialize the redaction key to JSON for export.
|
|
@@ -25,6 +26,6 @@ const deanonymise = (redactedText, redactionMap) => {
|
|
|
25
26
|
return result;
|
|
26
27
|
};
|
|
27
28
|
//#endregion
|
|
28
|
-
export { DEFAULT_ENTITY_LABELS, DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_NATIVE_PIPELINE_WARMUPS, DETECTION_SOURCES, DETECTOR_PRIORITY, OPERATOR_TYPES, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedSearch, assertNativeBindingVersion, assertNativePipelineSupported, availableDefaultNativePipelineLanguages, available_default_native_pipeline_languages, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromDefaultPackage, createNativePipelineFromPackage, createNativePipelineFromPackageFile, create_native_pipeline_from_default_package, deanonymise, diagnostics_json, diagnostics_stream_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, exportRedactionKey, getDefaultNativePipeline, getNativeBindingVersion, getNativePipelineCompatibility, get_default_native_pipeline, loadNativeAnonymizeBinding, load_prepared_package, load_prepared_package_file, native_package_version, normalize_for_search, preloadDefaultNativePipeline, preloadDefaultNativePipelineAsync, preload_default_native_pipeline, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, prepare_search_package, readDefaultNativePipelinePackageFile, readDefaultNativePipelinePackageFileAsync, readNativePipelinePackageFile, readNativePipelinePackageFileAsync, read_default_native_pipeline_package_file, redactDefaultText, redactDefaultTextJson, redact_default_text, redact_default_text_json, redact_text, redact_text_json, redact_text_stream_json, summary_diagnostics_json };
|
|
29
|
+
export { CALLER_DETECTION_CONTRACT_VERSION, CAPABILITY_MANIFEST, CAPABILITY_MANIFEST_SCHEMA_VERSION, CAPABILITY_RUNTIMES, DEFAULT_ENTITY_LABELS, DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_NATIVE_PIPELINE_WARMUPS, DETECTION_SOURCES, DETECTOR_PRIORITY, ENTITY_CAPABILITIES, ENTITY_LABELS, ENTITY_SELECTIONS, OPERATOR_TYPES, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedNativeRedactionSession, PreparedNativeSessionRedactionPlan, PreparedSearch, assertNativeBindingVersion, assertNativePipelineSupported, availableDefaultNativePipelineLanguages, available_default_native_pipeline_languages, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromDefaultPackage, createNativePipelineFromPackage, createNativePipelineFromPackageFile, create_native_pipeline_from_default_package, deanonymise, diagnostics_json, diagnostics_stream_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, exportRedactionKey, getDefaultNativePipeline, getNativeBindingVersion, getNativePipelineCompatibility, get_default_native_pipeline, loadNativeAnonymizeBinding, load_prepared_package, load_prepared_package_file, native_package_version, normalize_for_search, preloadDefaultNativePipeline, preloadDefaultNativePipelineAsync, preload_default_native_pipeline, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, prepare_search_package, readDefaultNativePipelinePackageFile, readDefaultNativePipelinePackageFileAsync, readNativePipelinePackageFile, readNativePipelinePackageFileAsync, read_default_native_pipeline_package_file, redactDefaultText, redactDefaultTextJson, redact_default_text, redact_default_text_json, redact_text, redact_text_json, redact_text_stream_json, summary_diagnostics_json };
|
|
29
30
|
|
|
30
31
|
//# sourceMappingURL=index.mjs.map
|
package/dist/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.mjs","names":[],"sources":["../src/redact.ts"],"sourcesContent":["import {\n DEFAULT_OPERATOR_CONFIG,\n OPERATOR_REGISTRY,\n resolveOperator,\n} from \"./operators\";\nimport type {\n Entity,\n OperatorConfig,\n OperatorType,\n RedactionResult,\n} from \"./types\";\nimport type { PipelineContext } from \"./context\";\nimport { defaultContext } from \"./context\";\n\nconst WHITESPACE_RE = /\\s+/g;\nconst PHONE_NOISE_RE = /[()\\s-]/g;\nconst ETHEREUM_ADDRESS_RE = /0x[0-9A-Fa-f]{40}/;\nconst BECH32_ADDRESS_RE = /\\bbc1[ac-hj-np-z02-9]{11,71}\\b/i;\nconst BASE58_ADDRESS_RE = /\\b[13][a-km-zA-HJ-NP-Z1-9]{25,34}\\b/;\nconst NHS_NUMBER_CUE_RE = /\\b(?:NHS|National\\s+Health\\s+Service)\\b/i;\nconst PLACEHOLDER_TOKEN_RE = /\\[[^\\s[\\]]+_[1-9]\\d*\\]/g;\nconst PASSPORT_IDENTIFIER_RE =\n /\\b(?:[A-Za-z]{1,2}\\d{6,8}|\\d{2}[A-Za-z]{2}\\d{5}|\\d{7,9})\\b/;\n// Strip all separators the ID detectors accept so the\n// same real-world value canonicalises to one placeholder:\n// - whitespace and `-` for IBAN, NIP, REGON, etc.\n// - `/` for birth numbers (\"900101/1234\") and Czech\n// bank accounts (\"123-4567/0100\").\n// - `.` for credit cards (\"4111.1111.1111.1111\") and\n// other dotted IDs.\nconst ID_SEPARATOR_RE = /[\\s\\-/.]/g;\n\nconst nextPlaceholder = (\n labelKey: string,\n counters: Map<string, number>,\n reservedPlaceholders: ReadonlySet<string>,\n): string => {\n let count = counters.get(labelKey) ?? 0;\n\n while (true) {\n count += 1;\n const placeholder = `[${labelKey}_${count}]`;\n if (reservedPlaceholders.has(placeholder)) continue;\n\n counters.set(labelKey, count);\n return placeholder;\n }\n};\n\nconst collectReservedPlaceholders = (\n reservedText: string,\n): ReadonlySet<string> => new Set(reservedText.match(PLACEHOLDER_TOKEN_RE));\n\nconst normalizeCryptoText = (text: string): string => {\n const trimmed = text.trim();\n\n const ethereumAddress = ETHEREUM_ADDRESS_RE.exec(trimmed)?.[0];\n if (ethereumAddress) {\n return ethereumAddress.toLowerCase();\n }\n\n const bech32Address = BECH32_ADDRESS_RE.exec(trimmed)?.[0];\n if (bech32Address) {\n return bech32Address.toLowerCase();\n }\n\n const base58Address = BASE58_ADDRESS_RE.exec(trimmed)?.[0];\n return base58Address ?? trimmed;\n};\n\nconst normalizePassportText = (text: string): string => {\n const passportIdentifier = PASSPORT_IDENTIFIER_RE.exec(text)?.[0] ?? text;\n return passportIdentifier.replace(ID_SEPARATOR_RE, \"\").toUpperCase();\n};\n\n/**\n * Normalize entity text so that surface-form variations\n * of the same real-world value map to a single canonical\n * key. Lowercased emails, stripped phone formatting, etc.\n */\nconst normalizeEntityText = (label: string, text: string): string => {\n const upper = label.toUpperCase().replace(WHITESPACE_RE, \"_\");\n\n if (upper === \"EMAIL_ADDRESS\" || upper === \"EMAIL\") {\n return text.toLowerCase().trim();\n }\n if (upper === \"PHONE_NUMBER\" || upper === \"PHONE\") {\n return text.replace(PHONE_NOISE_RE, \"\");\n }\n if (upper === \"CRYPTO\") {\n return normalizeCryptoText(text);\n }\n if (\n upper === \"NATIONAL_IDENTIFICATION_NUMBER\" &&\n NHS_NUMBER_CUE_RE.test(text)\n ) {\n return text.replace(/\\D/g, \"\");\n }\n if (\n upper === \"IBAN\" ||\n upper === \"BANK_ACCOUNT_NUMBER\" ||\n upper === \"TAX_IDENTIFICATION_NUMBER\" ||\n upper === \"REGISTRATION_NUMBER\" ||\n upper === \"NATIONAL_IDENTIFICATION_NUMBER\" ||\n upper === \"SOCIAL_SECURITY_NUMBER\" ||\n upper === \"BIRTH_NUMBER\" ||\n upper === \"IDENTITY_CARD_NUMBER\" ||\n upper === \"CREDIT_CARD_NUMBER\"\n ) {\n return text.replace(ID_SEPARATOR_RE, \"\").toUpperCase();\n }\n if (upper === \"PASSPORT_NUMBER\") {\n return normalizePassportText(text);\n }\n if (\n upper === \"PERSON\" ||\n upper === \"ORGANIZATION\" ||\n upper === \"ADDRESS\" ||\n upper === \"LAND_PARCEL\" ||\n upper === \"MISC\"\n ) {\n return text.replace(WHITESPACE_RE, \" \").toLowerCase().trim();\n }\n return text.trim();\n};\n\n/**\n * Build a stable mapping from entity text to numbered\n * placeholders. Same real-world value always maps to the\n * same placeholder (e.g., \"Dr. Muller\" and \"Dr. Muller\"\n * share one person placeholder).\n *\n * Placeholder format: [LABEL_N] where LABEL is uppercase.\n * N is allocated per label and skips tokens already present\n * in reserved text.\n *\n * @param _ctx Unused. Kept for signature compatibility;\n * coref alias links now travel on the entities\n * themselves (`corefSourceText`).\n */\ntype PlaceholderMapOptions = {\n reservedText?: string;\n};\n\nexport const buildPlaceholderMap = (\n entities: Entity[],\n _ctx: PipelineContext = defaultContext,\n { reservedText = \"\" }: PlaceholderMapOptions = {},\n): Map<string, string> => {\n const counters = new Map<string, number>();\n const textLabelToPlaceholder = new Map<string, string>();\n const normalizedToPlaceholder = new Map<string, string>();\n const reservedPlaceholders = collectReservedPlaceholders(reservedText);\n\n const sorted = entities.toSorted((a, b) => a.start - b.start);\n\n for (const entity of sorted) {\n const compositeKey = `${entity.label}\\0${entity.text}`;\n if (textLabelToPlaceholder.has(compositeKey)) {\n continue;\n }\n\n const labelKey = entity.label.toUpperCase().replace(WHITESPACE_RE, \"_\");\n\n // If this entity is a coref alias, unify its key\n // with the source entity's key so both get the same\n // number — in either direction: a backward alias\n // joins the source's existing placeholder, and a\n // forward alias (bare mention before the full form)\n // reserves its placeholder under the source key so\n // the source joins it when numbered later. The link\n // is carried on the entity itself, so it cannot be\n // lost between detection and redaction.\n const sourceText =\n entity.source === \"coreference\" ? entity.corefSourceText : undefined;\n const sourceNormalizedKey =\n sourceText === undefined\n ? undefined\n : `${labelKey}\\0${normalizeEntityText(entity.label, sourceText)}`;\n if (sourceNormalizedKey !== undefined) {\n const sourceExisting = normalizedToPlaceholder.get(sourceNormalizedKey);\n if (sourceExisting) {\n textLabelToPlaceholder.set(compositeKey, sourceExisting);\n continue;\n }\n }\n\n const normalized = normalizeEntityText(entity.label, entity.text);\n const normalizedKey = `${labelKey}\\0${normalized}`;\n const existing = normalizedToPlaceholder.get(normalizedKey);\n if (existing) {\n textLabelToPlaceholder.set(compositeKey, existing);\n if (sourceNormalizedKey !== undefined) {\n normalizedToPlaceholder.set(sourceNormalizedKey, existing);\n }\n continue;\n }\n\n const placeholder = nextPlaceholder(\n labelKey,\n counters,\n reservedPlaceholders,\n );\n textLabelToPlaceholder.set(compositeKey, placeholder);\n normalizedToPlaceholder.set(normalizedKey, placeholder);\n if (sourceNormalizedKey !== undefined) {\n normalizedToPlaceholder.set(sourceNormalizedKey, placeholder);\n }\n }\n\n return textLabelToPlaceholder;\n};\n\n/**\n * Apply redactions to the source text, replacing each\n * confirmed entity span using the configured operator.\n *\n * Co-references are consistent: if the same text appears\n * multiple times, all occurrences get the same placeholder.\n *\n * @param ctx Pipeline context. Must be the same instance\n * passed to `runPipeline` (or `findCoreferenceSpans`)\n * so coreference placeholder links are preserved.\n * Defaults to `defaultContext` for single-tenant usage.\n */\nexport const redactText = (\n fullText: string,\n entities: Entity[],\n config: OperatorConfig = DEFAULT_OPERATOR_CONFIG,\n ctx: PipelineContext = defaultContext,\n): RedactionResult => {\n if (entities.length === 0) {\n return {\n redactedText: fullText,\n redactionMap: new Map(),\n operatorMap: new Map(),\n entityCount: 0,\n };\n }\n\n const placeholderMap = buildPlaceholderMap(entities, ctx, {\n reservedText: fullText,\n });\n\n const sorted = entities.toSorted((a, b) => a.start - b.start);\n\n // Remove overlapping spans (keep first occurrence)\n const nonOverlapping: Entity[] = [];\n let lastEnd = 0;\n for (const entity of sorted) {\n if (entity.start >= lastEnd) {\n nonOverlapping.push(entity);\n lastEnd = entity.end;\n }\n }\n\n const parts: string[] = [];\n const redactionMap = new Map<string, string>();\n const operatorMap = new Map<string, OperatorType>();\n let cursor = 0;\n\n for (const entity of nonOverlapping) {\n if (entity.start > cursor) {\n parts.push(fullText.slice(cursor, entity.start));\n }\n\n const placeholder =\n placeholderMap.get(`${entity.label}\\0${entity.text}`) ??\n `[${entity.label.toUpperCase().replace(/\\s+/g, \"_\")}]`;\n\n const opType = resolveOperator(config, entity.label);\n const operator = OPERATOR_REGISTRY[opType];\n\n const replacement = operator.apply(\n entity.text,\n entity.label,\n placeholder,\n config.redactString,\n );\n\n parts.push(replacement);\n // operatorMap is keyed by the conceptual placeholder\n // ([LABEL_N]), not by the replacement text. For \"redact\"\n // operators the placeholder never appears in the output;\n // the map is only consulted via exportRedactionKey which\n // iterates redactionMap (replace entries only).\n operatorMap.set(placeholder, opType);\n\n // Only populate redactionMap for reversible operators.\n // A coref alias contributes its source's full text, so\n // a forward alias (\"Acme\" before \"Acme Corporation\")\n // cannot pin the shortened surface form as the key's\n // canonical value for the shared placeholder.\n if (\n operator.reversibility === \"reversible\" &&\n !redactionMap.has(placeholder)\n ) {\n redactionMap.set(\n placeholder,\n entity.source === \"coreference\" ? entity.corefSourceText : entity.text,\n );\n }\n\n cursor = entity.end;\n }\n\n if (cursor < fullText.length) {\n parts.push(fullText.slice(cursor));\n }\n\n return {\n redactedText: parts.join(\"\"),\n redactionMap,\n operatorMap,\n entityCount: nonOverlapping.length,\n };\n};\n\n/**\n * Serialize the redaction key to JSON for export.\n * Includes operator metadata so the export is self-describing.\n */\nexport const exportRedactionKey = (\n redactionMap: Map<string, string>,\n operatorMap: Map<string, OperatorType>,\n): string => {\n const entries: Record<string, { original: string; operator: OperatorType }> =\n {};\n\n for (const [placeholder, value] of redactionMap) {\n entries[placeholder] = {\n original: value,\n operator: operatorMap.get(placeholder) ?? \"replace\",\n };\n }\n\n return JSON.stringify({ entries }, null, 2);\n};\n\n/**\n * De-anonymise text using a redaction key.\n * Replaces placeholders back with original values.\n * Only works for reversible operators (replace).\n */\nexport const deanonymise = (\n redactedText: string,\n redactionMap: Map<string, string>,\n): string => {\n let result = redactedText;\n\n for (const [placeholder, original] of redactionMap) {\n result = result.replaceAll(placeholder, original);\n }\n\n return result;\n};\n"],"mappings":";;;;;;;;AAkUA,MAAa,sBACX,cACA,gBACW;CACX,MAAM,UACJ,CAAC;CAEH,KAAK,MAAM,CAAC,aAAa,UAAU,cACjC,QAAQ,eAAe;EACrB,UAAU;EACV,UAAU,YAAY,IAAI,WAAW,KAAK;CAC5C;CAGF,OAAO,KAAK,UAAU,EAAE,QAAQ,GAAG,MAAM,CAAC;AAC5C;;;;;;AAOA,MAAa,eACX,cACA,iBACW;CACX,IAAI,SAAS;CAEb,KAAK,MAAM,CAAC,aAAa,aAAa,cACpC,SAAS,OAAO,WAAW,aAAa,QAAQ;CAGlD,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"index.mjs","names":[],"sources":["../src/redact.ts"],"sourcesContent":["import {\n DEFAULT_OPERATOR_CONFIG,\n maskReplacementSpans,\n OPERATOR_REGISTRY,\n operatorType,\n requireMaskSelection,\n resolveOperator,\n} from \"./operators\";\nimport type {\n Entity,\n OperatorConfig,\n OperatorType,\n RedactionResult,\n} from \"./types\";\nimport type { PipelineContext } from \"./context\";\nimport { defaultContext } from \"./context\";\n\nconst WHITESPACE_RE = /\\s+/g;\nconst PHONE_NOISE_RE = /[()\\s-]/g;\nconst ETHEREUM_ADDRESS_RE = /0x[0-9A-Fa-f]{40}/;\nconst BECH32_ADDRESS_RE = /\\bbc1[ac-hj-np-z02-9]{11,71}\\b/i;\nconst BASE58_ADDRESS_RE = /\\b[13][a-km-zA-HJ-NP-Z1-9]{25,34}\\b/;\nconst NHS_NUMBER_CUE_RE = /\\b(?:NHS|National\\s+Health\\s+Service)\\b/i;\nconst PLACEHOLDER_TOKEN_RE = /\\[[^\\s[\\]]+_[1-9]\\d*\\]/g;\nconst PASSPORT_IDENTIFIER_RE =\n /\\b(?:[A-Za-z]{1,2}\\d{6,8}|\\d{2}[A-Za-z]{2}\\d{5}|\\d{7,9})\\b/;\n// Strip all separators the ID detectors accept so the\n// same real-world value canonicalises to one placeholder:\n// - whitespace and `-` for IBAN, NIP, REGON, etc.\n// - `/` for birth numbers (\"900101/1234\") and Czech\n// bank accounts (\"123-4567/0100\").\n// - `.` for credit cards (\"4111.1111.1111.1111\") and\n// other dotted IDs.\nconst ID_SEPARATOR_RE = /[\\s\\-/.]/g;\n\nconst nextPlaceholder = (\n labelKey: string,\n counters: Map<string, number>,\n reservedPlaceholders: ReadonlySet<string>,\n): string => {\n let count = counters.get(labelKey) ?? 0;\n\n while (true) {\n count += 1;\n const placeholder = `[${labelKey}_${count}]`;\n if (reservedPlaceholders.has(placeholder)) continue;\n\n counters.set(labelKey, count);\n return placeholder;\n }\n};\n\nconst collectReservedPlaceholders = (\n reservedText: string,\n): ReadonlySet<string> => new Set(reservedText.match(PLACEHOLDER_TOKEN_RE));\n\nconst normalizeCryptoText = (text: string): string => {\n const trimmed = text.trim();\n\n const ethereumAddress = ETHEREUM_ADDRESS_RE.exec(trimmed)?.[0];\n if (ethereumAddress) {\n return ethereumAddress.toLowerCase();\n }\n\n const bech32Address = BECH32_ADDRESS_RE.exec(trimmed)?.[0];\n if (bech32Address) {\n return bech32Address.toLowerCase();\n }\n\n const base58Address = BASE58_ADDRESS_RE.exec(trimmed)?.[0];\n return base58Address ?? trimmed;\n};\n\nconst normalizePassportText = (text: string): string => {\n const passportIdentifier = PASSPORT_IDENTIFIER_RE.exec(text)?.[0] ?? text;\n return passportIdentifier.replace(ID_SEPARATOR_RE, \"\").toUpperCase();\n};\n\n/**\n * Normalize entity text so that surface-form variations\n * of the same real-world value map to a single canonical\n * key. Lowercased emails, stripped phone formatting, etc.\n */\nconst normalizeEntityText = (label: string, text: string): string => {\n const upper = label.toUpperCase().replace(WHITESPACE_RE, \"_\");\n\n if (upper === \"EMAIL_ADDRESS\" || upper === \"EMAIL\") {\n return text.toLowerCase().trim();\n }\n if (upper === \"PHONE_NUMBER\" || upper === \"PHONE\") {\n return text.replace(PHONE_NOISE_RE, \"\");\n }\n if (upper === \"CRYPTO\") {\n return normalizeCryptoText(text);\n }\n if (\n upper === \"NATIONAL_IDENTIFICATION_NUMBER\" &&\n NHS_NUMBER_CUE_RE.test(text)\n ) {\n return text.replace(/\\D/g, \"\");\n }\n if (\n upper === \"IBAN\" ||\n upper === \"BANK_ACCOUNT_NUMBER\" ||\n upper === \"TAX_IDENTIFICATION_NUMBER\" ||\n upper === \"REGISTRATION_NUMBER\" ||\n upper === \"NATIONAL_IDENTIFICATION_NUMBER\" ||\n upper === \"SOCIAL_SECURITY_NUMBER\" ||\n upper === \"BIRTH_NUMBER\" ||\n upper === \"IDENTITY_CARD_NUMBER\" ||\n upper === \"CREDIT_CARD_NUMBER\"\n ) {\n return text.replace(ID_SEPARATOR_RE, \"\").toUpperCase();\n }\n if (upper === \"PASSPORT_NUMBER\") {\n return normalizePassportText(text);\n }\n if (\n upper === \"PERSON\" ||\n upper === \"ORGANIZATION\" ||\n upper === \"ADDRESS\" ||\n upper === \"LAND_PARCEL\" ||\n upper === \"MISC\"\n ) {\n return text.replace(WHITESPACE_RE, \" \").toLowerCase().trim();\n }\n return text.trim();\n};\n\nconst nonOverlappingEntities = (entities: Entity[]): Entity[] => {\n const result: Entity[] = [];\n let lastEnd = 0;\n for (const entity of entities) {\n if (entity.start < lastEnd) continue;\n result.push(entity);\n lastEnd = entity.end;\n }\n return result;\n};\n\ntype MaskReplacementSpan = {\n start: number;\n end: number;\n replacement: string;\n};\n\nconst removeRedactedMaskOverlaps = (\n replacements: MaskReplacementSpan[],\n redacted: Entity[],\n): MaskReplacementSpan[] => {\n const result: MaskReplacementSpan[] = [];\n let redactedIndex = 0;\n for (const replacement of replacements) {\n while (true) {\n const candidate = redacted.at(redactedIndex);\n if (candidate === undefined || candidate.end > replacement.start) break;\n redactedIndex += 1;\n }\n const redactedEntity = redacted.at(redactedIndex);\n const overlaps =\n redactedEntity !== undefined &&\n redactedEntity.start < replacement.end &&\n replacement.start < redactedEntity.end;\n if (!overlaps) result.push(replacement);\n }\n return result;\n};\n\n/**\n * Build a stable mapping from entity text to numbered\n * placeholders. Same real-world value always maps to the\n * same placeholder (e.g., \"Dr. Muller\" and \"Dr. Muller\"\n * share one person placeholder).\n *\n * Placeholder format: [LABEL_N] where LABEL is uppercase.\n * N is allocated per label and skips tokens already present\n * in reserved text.\n *\n * @param _ctx Unused. Kept for signature compatibility;\n * coref alias links now travel on the entities\n * themselves (`corefSourceText`).\n */\ntype PlaceholderMapOptions = {\n reservedText?: string;\n};\n\nexport const buildPlaceholderMap = (\n entities: Entity[],\n _ctx: PipelineContext = defaultContext,\n { reservedText = \"\" }: PlaceholderMapOptions = {},\n): Map<string, string> => {\n const counters = new Map<string, number>();\n const textLabelToPlaceholder = new Map<string, string>();\n const normalizedToPlaceholder = new Map<string, string>();\n const reservedPlaceholders = collectReservedPlaceholders(reservedText);\n\n const sorted = entities.toSorted((a, b) => a.start - b.start);\n\n for (const entity of sorted) {\n const compositeKey = `${entity.label}\\0${entity.text}`;\n if (textLabelToPlaceholder.has(compositeKey)) {\n continue;\n }\n\n const labelKey = entity.label.toUpperCase().replace(WHITESPACE_RE, \"_\");\n\n // If this entity is a coref alias, unify its key\n // with the source entity's key so both get the same\n // number — in either direction: a backward alias\n // joins the source's existing placeholder, and a\n // forward alias (bare mention before the full form)\n // reserves its placeholder under the source key so\n // the source joins it when numbered later. The link\n // is carried on the entity itself, so it cannot be\n // lost between detection and redaction.\n const sourceText =\n entity.source === \"coreference\" ? entity.corefSourceText : undefined;\n const sourceNormalizedKey =\n sourceText === undefined\n ? undefined\n : `${labelKey}\\0${normalizeEntityText(entity.label, sourceText)}`;\n if (sourceNormalizedKey !== undefined) {\n const sourceExisting = normalizedToPlaceholder.get(sourceNormalizedKey);\n if (sourceExisting) {\n textLabelToPlaceholder.set(compositeKey, sourceExisting);\n continue;\n }\n }\n\n const normalized = normalizeEntityText(entity.label, entity.text);\n const normalizedKey = `${labelKey}\\0${normalized}`;\n const existing = normalizedToPlaceholder.get(normalizedKey);\n if (existing) {\n textLabelToPlaceholder.set(compositeKey, existing);\n if (sourceNormalizedKey !== undefined) {\n normalizedToPlaceholder.set(sourceNormalizedKey, existing);\n }\n continue;\n }\n\n const placeholder = nextPlaceholder(\n labelKey,\n counters,\n reservedPlaceholders,\n );\n textLabelToPlaceholder.set(compositeKey, placeholder);\n normalizedToPlaceholder.set(normalizedKey, placeholder);\n if (sourceNormalizedKey !== undefined) {\n normalizedToPlaceholder.set(sourceNormalizedKey, placeholder);\n }\n }\n\n return textLabelToPlaceholder;\n};\n\n/**\n * Apply redactions to the source text, replacing each\n * confirmed entity span using the configured operator.\n *\n * Co-references are consistent: if the same text appears\n * multiple times, all occurrences get the same placeholder.\n *\n * @param ctx Pipeline context. Must be the same instance\n * passed to `runPipeline` (or `findCoreferenceSpans`)\n * so coreference placeholder links are preserved.\n * Defaults to `defaultContext` for single-tenant usage.\n */\nexport const redactText = (\n fullText: string,\n entities: Entity[],\n config: OperatorConfig = DEFAULT_OPERATOR_CONFIG,\n ctx: PipelineContext = defaultContext,\n): RedactionResult => {\n if (entities.length === 0) {\n return {\n redactedText: fullText,\n redactionMap: new Map(),\n operatorMap: new Map(),\n entityCount: 0,\n };\n }\n\n const placeholderMap = buildPlaceholderMap(entities, ctx, {\n reservedText: fullText,\n });\n\n const sorted = entities.toSorted((a, b) => a.start - b.start);\n\n const kept: Entity[] = [];\n const masked: Entity[] = [];\n const redacted: Entity[] = [];\n for (const entity of sorted) {\n const opType = operatorType(resolveOperator(config, entity.label));\n if (opType === \"keep\") {\n kept.push(entity);\n } else if (opType === \"mask\") {\n masked.push(entity);\n } else {\n redacted.push(entity);\n }\n }\n const selectedKept = nonOverlappingEntities(kept);\n const selectedMasked = nonOverlappingEntities(masked);\n const selectedRedacted = nonOverlappingEntities(redacted);\n\n const maskReplacements: MaskReplacementSpan[] = [];\n for (const entity of selectedMasked) {\n const selection = resolveOperator(config, entity.label);\n const sourceText = fullText.slice(entity.start, entity.end);\n for (const replacement of maskReplacementSpans(\n sourceText,\n requireMaskSelection(selection),\n )) {\n maskReplacements.push({\n start: entity.start + replacement.start,\n end: entity.start + replacement.end,\n replacement: replacement.replacement,\n });\n }\n }\n const visibleMaskReplacements = removeRedactedMaskOverlaps(\n maskReplacements,\n selectedRedacted,\n );\n\n const parts: string[] = [];\n const redactionMap = new Map<string, string>();\n const operatorMap = new Map<string, OperatorType>();\n let cursor = 0;\n\n const placeholderFor = (entity: Entity): string =>\n placeholderMap.get(`${entity.label}\\0${entity.text}`) ??\n `[${entity.label.toUpperCase().replace(/\\s+/g, \"_\")}]`;\n const processed = [\n ...selectedKept,\n ...selectedMasked,\n ...selectedRedacted,\n ].toSorted((a, b) => a.start - b.start);\n for (const entity of processed) {\n operatorMap.set(\n placeholderFor(entity),\n operatorType(resolveOperator(config, entity.label)),\n );\n }\n\n let redactedIndex = 0;\n let maskIndex = 0;\n while (\n redactedIndex < selectedRedacted.length ||\n maskIndex < visibleMaskReplacements.length\n ) {\n const redactedEntity = selectedRedacted.at(redactedIndex);\n const maskReplacement = visibleMaskReplacements.at(maskIndex);\n const useRedacted =\n redactedEntity !== undefined &&\n (maskReplacement === undefined ||\n redactedEntity.start <= maskReplacement.start);\n const start = useRedacted\n ? (redactedEntity?.start ?? cursor)\n : (maskReplacement?.start ?? cursor);\n const end = useRedacted\n ? (redactedEntity?.end ?? start)\n : (maskReplacement?.end ?? start);\n if (start > cursor) parts.push(fullText.slice(cursor, start));\n\n if (!useRedacted && maskReplacement !== undefined) {\n parts.push(maskReplacement.replacement);\n cursor = end;\n maskIndex += 1;\n continue;\n }\n if (redactedEntity === undefined) break;\n\n const entity = redactedEntity;\n const placeholder = placeholderFor(entity);\n\n const selection = resolveOperator(config, entity.label);\n const opType = operatorType(selection);\n const operator = OPERATOR_REGISTRY[opType];\n\n const replacement = operator.apply(\n entity.text,\n entity.label,\n placeholder,\n config.redactString,\n selection,\n );\n\n parts.push(replacement);\n // Only populate redactionMap for reversible operators.\n // A coref alias contributes its source's full text, so\n // a forward alias (\"Acme\" before \"Acme Corporation\")\n // cannot pin the shortened surface form as the key's\n // canonical value for the shared placeholder.\n if (\n operator.reversibility === \"reversible\" &&\n !redactionMap.has(placeholder)\n ) {\n redactionMap.set(\n placeholder,\n entity.source === \"coreference\" ? entity.corefSourceText : entity.text,\n );\n }\n\n cursor = end;\n redactedIndex += 1;\n }\n\n if (cursor < fullText.length) {\n parts.push(fullText.slice(cursor));\n }\n\n return {\n redactedText: parts.join(\"\"),\n redactionMap,\n operatorMap,\n entityCount:\n selectedKept.length + selectedMasked.length + selectedRedacted.length,\n };\n};\n\n/**\n * Serialize the redaction key to JSON for export.\n * Includes operator metadata so the export is self-describing.\n */\nexport const exportRedactionKey = (\n redactionMap: Map<string, string>,\n operatorMap: Map<string, OperatorType>,\n): string => {\n const entries: Record<string, { original: string; operator: OperatorType }> =\n {};\n\n for (const [placeholder, value] of redactionMap) {\n entries[placeholder] = {\n original: value,\n operator: operatorMap.get(placeholder) ?? \"replace\",\n };\n }\n\n return JSON.stringify({ entries }, null, 2);\n};\n\n/**\n * De-anonymise text using a redaction key.\n * Replaces placeholders back with original values.\n * Only works for reversible operators (replace).\n */\nexport const deanonymise = (\n redactedText: string,\n redactionMap: Map<string, string>,\n): string => {\n let result = redactedText;\n\n for (const [placeholder, original] of redactionMap) {\n result = result.replaceAll(placeholder, original);\n }\n\n return result;\n};\n"],"mappings":";;;;;;;;;AAyaA,MAAa,sBACX,cACA,gBACW;CACX,MAAM,UACJ,CAAC;CAEH,KAAK,MAAM,CAAC,aAAa,UAAU,cACjC,QAAQ,eAAe;EACrB,UAAU;EACV,UAAU,YAAY,IAAI,WAAW,KAAK;CAC5C;CAGF,OAAO,KAAK,UAAU,EAAE,QAAQ,GAAG,MAAM,CAAC;AAC5C;;;;;;AAOA,MAAa,eACX,cACA,iBACW;CACX,IAAI,SAAS;CAEb,KAAK,MAAM,CAAC,aAAa,aAAa,cACpC,SAAS,OAAO,WAAW,aAAa,QAAQ;CAGlD,OAAO;AACT"}
|
package/dist/native-node.d.mts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import { A as NativeStaticRedactionResult, Dt as NativePreparedSearchConfig, N as PreparedNativeAnonymizer, P as PreparedNativePipeline, b as NativeSearchPackageInput, f as NativeOperatorConfig, gt as GazetteerEntry, n as NativeAnonymizeBinding, vt as PipelineConfig } from "./native.mjs";
|
|
3
2
|
//#region src/context.d.ts
|
|
4
3
|
/**
|
|
5
4
|
* Cached state for a single pipeline run (or a sequence of runs sharing the
|
|
@@ -33,24 +32,9 @@ type NativePipelinePackageOptions = NativePipelineBuildOptions & {
|
|
|
33
32
|
};
|
|
34
33
|
declare const getNativePipelineCompatibility: (config: PipelineConfig) => NativePipelineCompatibility;
|
|
35
34
|
declare const assertNativePipelineSupported: (config: PipelineConfig) => void;
|
|
36
|
-
declare const prepareNativePipelineConfig: ({
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
gazetteerEntries
|
|
40
|
-
}: Omit<NativePipelineBuildOptions, "context">) => Promise<NativePreparedSearchConfig>;
|
|
41
|
-
declare const prepareNativePipelinePackage: ({
|
|
42
|
-
binding,
|
|
43
|
-
config,
|
|
44
|
-
gazetteerEntries,
|
|
45
|
-
context,
|
|
46
|
-
compressed
|
|
47
|
-
}: NativePipelinePackageOptions) => Promise<Uint8Array>;
|
|
48
|
-
declare const createNativePipelineFromConfig: ({
|
|
49
|
-
binding,
|
|
50
|
-
config,
|
|
51
|
-
gazetteerEntries,
|
|
52
|
-
context
|
|
53
|
-
}: NativePipelineBuildOptions) => Promise<PreparedNativePipeline>;
|
|
35
|
+
declare const prepareNativePipelineConfig: ({ binding, config, gazetteerEntries }: Omit<NativePipelineBuildOptions, "context">) => Promise<NativePreparedSearchConfig>;
|
|
36
|
+
declare const prepareNativePipelinePackage: ({ binding, config, gazetteerEntries, context, compressed }: NativePipelinePackageOptions) => Promise<Uint8Array>;
|
|
37
|
+
declare const createNativePipelineFromConfig: ({ binding, config, gazetteerEntries, context }: NativePipelineBuildOptions) => Promise<PreparedNativePipeline>;
|
|
54
38
|
//#endregion
|
|
55
39
|
//#region src/native-default-config.d.ts
|
|
56
40
|
declare const DEFAULT_NATIVE_PIPELINE_CONFIG: PipelineConfig;
|
|
@@ -95,10 +79,7 @@ declare const readNativePipelinePackageFile: (packagePath: string) => Uint8Array
|
|
|
95
79
|
declare const readNativePipelinePackageFileAsync: (packagePath: string) => Promise<Uint8Array>;
|
|
96
80
|
declare const native_package_version: (options?: NativeSdkOptions) => string;
|
|
97
81
|
declare const normalize_for_search: (text: string, options?: NativeSdkOptions) => string;
|
|
98
|
-
declare const prepare_search_package: (config: NativeSearchPackageInput, {
|
|
99
|
-
compressed,
|
|
100
|
-
...options
|
|
101
|
-
}?: NativeSdkPackageOptions) => Uint8Array;
|
|
82
|
+
declare const prepare_search_package: (config: NativeSearchPackageInput, { compressed, ...options }?: NativeSdkPackageOptions) => Uint8Array;
|
|
102
83
|
declare const load_prepared_package: (packageBytes: Uint8Array, options?: NativeSdkOptions) => PreparedNativeAnonymizer;
|
|
103
84
|
declare const load_prepared_package_file: (packagePath: string, options?: NativeSdkOptions) => PreparedNativeAnonymizer;
|
|
104
85
|
declare const redact_text: (config: NativeSearchPackageInput, fullText: string, operators?: NativeOperatorConfig, options?: NativeSdkOptions) => NativeStaticRedactionResult;
|
|
@@ -107,21 +88,12 @@ declare const redact_text_stream_json: (config: NativeSearchPackageInput, fullTe
|
|
|
107
88
|
declare const diagnostics_json: (config: NativeSearchPackageInput, fullText: string, operators?: NativeOperatorConfig, options?: NativeSdkOptions) => string | null;
|
|
108
89
|
declare const diagnostics_stream_json: (config: NativeSearchPackageInput, fullText: string, onBatch: (diagnosticsJson: string) => void, operators?: NativeOperatorConfig, options?: NativeSdkOptions) => string | null;
|
|
109
90
|
declare const summary_diagnostics_json: (config: NativeSearchPackageInput, fullText: string, operators?: NativeOperatorConfig, options?: NativeSdkOptions) => string | null;
|
|
110
|
-
declare const readDefaultNativePipelinePackageFile: ({
|
|
111
|
-
language
|
|
112
|
-
}?: DefaultNativePipelinePackageFileOptions) => Uint8Array;
|
|
91
|
+
declare const readDefaultNativePipelinePackageFile: ({ language }?: DefaultNativePipelinePackageFileOptions) => Uint8Array;
|
|
113
92
|
declare const read_default_native_pipeline_package_file: (options?: DefaultNativePipelinePackageFileOptions) => Uint8Array;
|
|
114
93
|
declare const availableDefaultNativePipelineLanguages: () => string[];
|
|
115
94
|
declare const available_default_native_pipeline_languages: () => string[];
|
|
116
|
-
declare const readDefaultNativePipelinePackageFileAsync: ({
|
|
117
|
-
|
|
118
|
-
}?: DefaultNativePipelinePackageFileOptions) => Promise<Uint8Array>;
|
|
119
|
-
declare const createNativePipelineFromPackageFile: ({
|
|
120
|
-
binding,
|
|
121
|
-
packagePath,
|
|
122
|
-
expectedVersion,
|
|
123
|
-
...loadOptions
|
|
124
|
-
}: NativePipelinePackageFileOptions) => PreparedNativePipeline;
|
|
95
|
+
declare const readDefaultNativePipelinePackageFileAsync: ({ language }?: DefaultNativePipelinePackageFileOptions) => Promise<Uint8Array>;
|
|
96
|
+
declare const createNativePipelineFromPackageFile: ({ binding, packagePath, expectedVersion, ...loadOptions }: NativePipelinePackageFileOptions) => PreparedNativePipeline;
|
|
125
97
|
declare const createNativePipelineFromDefaultPackage: (options?: DefaultNativePipelinePackageOptions) => PreparedNativePipeline;
|
|
126
98
|
declare const create_native_pipeline_from_default_package: (options?: DefaultNativePipelinePackageOptions) => PreparedNativePipeline;
|
|
127
99
|
declare const getDefaultNativePipeline: (options?: DefaultNativePipelinePackageOptions) => PreparedNativePipeline;
|
package/dist/native-node.mjs
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import { PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedSearch, assertNativeBindingVersion, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromPackage, encodeNativeSearchConfig, encodeNativeSearchConfigInput, getNativeBindingVersion, prepareNativeSearchPackage } from "./native.mjs";
|
|
1
|
+
import { CALLER_DETECTION_CONTRACT_VERSION, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedNativeRedactionSession, PreparedNativeSessionRedactionPlan, PreparedSearch, assertNativeBindingVersion, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromPackage, encodeNativeSearchConfig, encodeNativeSearchConfigInput, getNativeBindingVersion, prepareNativeSearchPackage } from "./native.mjs";
|
|
2
2
|
import { A as redact_text_json, C as readNativePipelinePackageFileAsync, D as redact_default_text, E as redactDefaultTextJson, F as createNativePipelineFromConfig, I as getNativePipelineCompatibility, L as prepareNativePipelineConfig, M as summary_diagnostics_json, N as DEFAULT_NATIVE_PIPELINE_CONFIG, O as redact_default_text_json, P as assertNativePipelineSupported, R as prepareNativePipelinePackage, S as readNativePipelinePackageFile, T as redactDefaultText, _ as preloadDefaultNativePipelineAsync, a as createNativePipelineFromPackageFile, b as readDefaultNativePipelinePackageFile, c as diagnostics_stream_json, d as loadNativeAnonymizeBinding, f as load_prepared_package, g as preloadDefaultNativePipeline, h as normalize_for_search, i as createNativePipelineFromDefaultPackage, j as redact_text_stream_json, k as redact_text, l as getDefaultNativePipeline, m as native_package_version, n as availableDefaultNativePipelineLanguages, o as create_native_pipeline_from_default_package, p as load_prepared_package_file, r as available_default_native_pipeline_languages, s as diagnostics_json, t as DEFAULT_NATIVE_PIPELINE_WARMUPS, u as get_default_native_pipeline, v as preload_default_native_pipeline, w as read_default_native_pipeline_package_file, x as readDefaultNativePipelinePackageFileAsync, y as prepare_search_package } from "./native-node2.mjs";
|
|
3
|
-
export { DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_NATIVE_PIPELINE_WARMUPS, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedSearch, assertNativeBindingVersion, assertNativePipelineSupported, availableDefaultNativePipelineLanguages, available_default_native_pipeline_languages, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromDefaultPackage, createNativePipelineFromPackage, createNativePipelineFromPackageFile, create_native_pipeline_from_default_package, diagnostics_json, diagnostics_stream_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, getDefaultNativePipeline, getNativeBindingVersion, getNativePipelineCompatibility, get_default_native_pipeline, loadNativeAnonymizeBinding, load_prepared_package, load_prepared_package_file, native_package_version, normalize_for_search, preloadDefaultNativePipeline, preloadDefaultNativePipelineAsync, preload_default_native_pipeline, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, prepare_search_package, readDefaultNativePipelinePackageFile, readDefaultNativePipelinePackageFileAsync, readNativePipelinePackageFile, readNativePipelinePackageFileAsync, read_default_native_pipeline_package_file, redactDefaultText, redactDefaultTextJson, redact_default_text, redact_default_text_json, redact_text, redact_text_json, redact_text_stream_json, summary_diagnostics_json };
|
|
3
|
+
export { CALLER_DETECTION_CONTRACT_VERSION, DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_NATIVE_PIPELINE_WARMUPS, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedNativeRedactionSession, PreparedNativeSessionRedactionPlan, PreparedSearch, assertNativeBindingVersion, assertNativePipelineSupported, availableDefaultNativePipelineLanguages, available_default_native_pipeline_languages, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromDefaultPackage, createNativePipelineFromPackage, createNativePipelineFromPackageFile, create_native_pipeline_from_default_package, diagnostics_json, diagnostics_stream_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, getDefaultNativePipeline, getNativeBindingVersion, getNativePipelineCompatibility, get_default_native_pipeline, loadNativeAnonymizeBinding, load_prepared_package, load_prepared_package_file, native_package_version, normalize_for_search, preloadDefaultNativePipeline, preloadDefaultNativePipelineAsync, preload_default_native_pipeline, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, prepare_search_package, readDefaultNativePipelinePackageFile, readDefaultNativePipelinePackageFileAsync, readNativePipelinePackageFile, readNativePipelinePackageFileAsync, read_default_native_pipeline_package_file, redactDefaultText, redactDefaultTextJson, redact_default_text, redact_default_text_json, redact_text, redact_text_json, redact_text_stream_json, summary_diagnostics_json };
|
package/dist/native-node2.d.mts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { $ as getNativeBindingVersion, A as NativeStaticRedactionResult, B as SharedNativePreparedPackageOptions, C as NativeSessionCallerRedactionInput, D as NativeSessionMetadata, Dt as NativePreparedSearchConfig, E as NativeSessionLifecycle, F as PreparedNativeRedactionSession, G as assertNativeBindingVersion, H as SharedNativeRedactTextOptions, I as PreparedNativeSessionRedactionPlan, J as createNativePipelineFromPackage, K as createNativeAnonymizerFromConfig, L as PreparedSearch, M as PreparedAnonymizer, N as PreparedNativeAnonymizer, O as NativeSessionRedactionAtOptions, P as PreparedNativePipeline, Q as encodeNativeSearchConfigInput, R as SharedNativeDiagnosticsJsonOptions, S as NativeSessionBlockRedactionPlan, T as NativeSessionDeletionSummary, U as SharedNativeRedactTextStreamJsonOptions, V as SharedNativeRedactTextJsonOptions, W as SharedNativeSearchPackageOptions, Z as encodeNativeSearchConfig, _ as NativePreparedSessionRedactionPlanBinding, a as NativeBindingVersionOptions, b as NativeSearchPackageInput, c as NativeCreateSessionWithLifecycleOptions, d as NativeOpenSessionArchiveOptions, f as NativeOperatorConfig, g as NativePreparedSearchBinding, h as NativePreparedRedactionSessionBinding, i as NativeAnonymizerFromPackageOptions, j as NativeTextReplacement, k as NativeSessionStatus, l as NativeDiagnosticsBatchCallback, m as NativePipelineFromPackageOptions, n as NativeAnonymizeBinding, o as NativeCallerDetection, p as NativePipelineEntity, q as createNativeAnonymizerFromPackage, r as NativeAnonymizerFromConfigOptions, rt as prepareNativeSearchPackage, s as NativeCallerRedactionOptions, t as CALLER_DETECTION_CONTRACT_VERSION, u as NativeNormalizeOptions, v as NativeRedactionResult, w as NativeSessionCallerRedactionPlanOptions, x as NativeSearchPackageOptions, y as NativeResultEventCallback, z as SharedNativeDiagnosticsStreamJsonOptions } from "./native.mjs";
|
|
2
2
|
import { A as readDefaultNativePipelinePackageFileAsync, B as redact_text_stream_json, C as native_package_version, D as preload_default_native_pipeline, E as preloadDefaultNativePipelineAsync, F as redactDefaultTextJson, G as NativePipelinePackageOptions, H as DEFAULT_NATIVE_PIPELINE_CONFIG, I as redact_default_text, J as createNativePipelineFromConfig, K as NativePipelineUnsupportedFeature, L as redact_default_text_json, M as readNativePipelinePackageFileAsync, N as read_default_native_pipeline_package_file, O as prepare_search_package, P as redactDefaultText, R as redact_text, S as load_prepared_package_file, T as preloadDefaultNativePipeline, U as NativePipelineBuildOptions, V as summary_diagnostics_json, W as NativePipelineCompatibility, X as prepareNativePipelineConfig, Y as getNativePipelineCompatibility, Z as prepareNativePipelinePackage, _ as diagnostics_stream_json, a as LoadNativeBindingOptions, b as loadNativeAnonymizeBinding, c as NativeRequire, d as availableDefaultNativePipelineLanguages, f as available_default_native_pipeline_languages, g as diagnostics_json, h as create_native_pipeline_from_default_package, i as DefaultNativePipelineWarmup, j as readNativePipelinePackageFile, k as readDefaultNativePipelinePackageFile, l as NativeSdkOptions, m as createNativePipelineFromPackageFile, n as DefaultNativePipelinePackageFileOptions, o as NativeLibc, p as createNativePipelineFromDefaultPackage, q as assertNativePipelineSupported, r as DefaultNativePipelinePackageOptions, s as NativePipelinePackageFileOptions, t as DEFAULT_NATIVE_PIPELINE_WARMUPS, u as NativeSdkPackageOptions, v as getDefaultNativePipeline, w as normalize_for_search, x as load_prepared_package, y as get_default_native_pipeline, z as redact_text_json } from "./native-node.mjs";
|
|
3
|
-
export { DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_NATIVE_PIPELINE_WARMUPS, DefaultNativePipelinePackageFileOptions, DefaultNativePipelinePackageOptions, DefaultNativePipelineWarmup, LoadNativeBindingOptions, NativeAnonymizeBinding, NativeAnonymizerFromConfigOptions, NativeAnonymizerFromPackageOptions, NativeBindingVersionOptions, NativeDiagnosticsBatchCallback, NativeLibc, NativeNormalizeOptions, NativeOperatorConfig, type NativePipelineBuildOptions, type NativePipelineCompatibility, NativePipelineEntity, NativePipelineFromPackageOptions, NativePipelinePackageFileOptions, type NativePipelinePackageOptions, type NativePipelineUnsupportedFeature, NativePreparedSearchBinding, type NativePreparedSearchConfig, NativeRedactionResult, NativeRequire, NativeResultEventCallback, NativeSdkOptions, NativeSdkPackageOptions, NativeSearchPackageInput, NativeSearchPackageOptions, NativeStaticRedactionResult, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedSearch, SharedNativeDiagnosticsJsonOptions, SharedNativeDiagnosticsStreamJsonOptions, SharedNativePreparedPackageOptions, SharedNativeRedactTextJsonOptions, SharedNativeRedactTextOptions, SharedNativeRedactTextStreamJsonOptions, SharedNativeSearchPackageOptions, assertNativeBindingVersion, assertNativePipelineSupported, availableDefaultNativePipelineLanguages, available_default_native_pipeline_languages, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromDefaultPackage, createNativePipelineFromPackage, createNativePipelineFromPackageFile, create_native_pipeline_from_default_package, diagnostics_json, diagnostics_stream_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, getDefaultNativePipeline, getNativeBindingVersion, getNativePipelineCompatibility, get_default_native_pipeline, loadNativeAnonymizeBinding, load_prepared_package, load_prepared_package_file, native_package_version, normalize_for_search, preloadDefaultNativePipeline, preloadDefaultNativePipelineAsync, preload_default_native_pipeline, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, prepare_search_package, readDefaultNativePipelinePackageFile, readDefaultNativePipelinePackageFileAsync, readNativePipelinePackageFile, readNativePipelinePackageFileAsync, read_default_native_pipeline_package_file, redactDefaultText, redactDefaultTextJson, redact_default_text, redact_default_text_json, redact_text, redact_text_json, redact_text_stream_json, summary_diagnostics_json };
|
|
3
|
+
export { CALLER_DETECTION_CONTRACT_VERSION, DEFAULT_NATIVE_PIPELINE_CONFIG, DEFAULT_NATIVE_PIPELINE_WARMUPS, DefaultNativePipelinePackageFileOptions, DefaultNativePipelinePackageOptions, DefaultNativePipelineWarmup, LoadNativeBindingOptions, NativeAnonymizeBinding, NativeAnonymizerFromConfigOptions, NativeAnonymizerFromPackageOptions, NativeBindingVersionOptions, NativeCallerDetection, NativeCallerRedactionOptions, NativeCreateSessionWithLifecycleOptions, NativeDiagnosticsBatchCallback, NativeLibc, NativeNormalizeOptions, NativeOpenSessionArchiveOptions, NativeOperatorConfig, type NativePipelineBuildOptions, type NativePipelineCompatibility, NativePipelineEntity, NativePipelineFromPackageOptions, NativePipelinePackageFileOptions, type NativePipelinePackageOptions, type NativePipelineUnsupportedFeature, NativePreparedRedactionSessionBinding, NativePreparedSearchBinding, type NativePreparedSearchConfig, NativePreparedSessionRedactionPlanBinding, NativeRedactionResult, NativeRequire, NativeResultEventCallback, NativeSdkOptions, NativeSdkPackageOptions, NativeSearchPackageInput, NativeSearchPackageOptions, NativeSessionBlockRedactionPlan, NativeSessionCallerRedactionInput, NativeSessionCallerRedactionPlanOptions, NativeSessionDeletionSummary, NativeSessionLifecycle, NativeSessionMetadata, NativeSessionRedactionAtOptions, NativeSessionStatus, NativeStaticRedactionResult, NativeTextReplacement, PreparedAnonymizer, PreparedNativeAnonymizer, PreparedNativePipeline, PreparedNativeRedactionSession, PreparedNativeSessionRedactionPlan, PreparedSearch, SharedNativeDiagnosticsJsonOptions, SharedNativeDiagnosticsStreamJsonOptions, SharedNativePreparedPackageOptions, SharedNativeRedactTextJsonOptions, SharedNativeRedactTextOptions, SharedNativeRedactTextStreamJsonOptions, SharedNativeSearchPackageOptions, assertNativeBindingVersion, assertNativePipelineSupported, availableDefaultNativePipelineLanguages, available_default_native_pipeline_languages, createNativeAnonymizerFromConfig, createNativeAnonymizerFromPackage, createNativePipelineFromConfig, createNativePipelineFromDefaultPackage, createNativePipelineFromPackage, createNativePipelineFromPackageFile, create_native_pipeline_from_default_package, diagnostics_json, diagnostics_stream_json, encodeNativeSearchConfig, encodeNativeSearchConfigInput, getDefaultNativePipeline, getNativeBindingVersion, getNativePipelineCompatibility, get_default_native_pipeline, loadNativeAnonymizeBinding, load_prepared_package, load_prepared_package_file, native_package_version, normalize_for_search, preloadDefaultNativePipeline, preloadDefaultNativePipelineAsync, preload_default_native_pipeline, prepareNativePipelineConfig, prepareNativePipelinePackage, prepareNativeSearchPackage, prepare_search_package, readDefaultNativePipelinePackageFile, readDefaultNativePipelinePackageFileAsync, readNativePipelinePackageFile, readNativePipelinePackageFileAsync, read_default_native_pipeline_package_file, redactDefaultText, redactDefaultTextJson, redact_default_text, redact_default_text_json, redact_text, redact_text_json, redact_text_stream_json, summary_diagnostics_json };
|
package/dist/native-node2.mjs
CHANGED
|
@@ -229,8 +229,11 @@ const encoder = new TextEncoder();
|
|
|
229
229
|
* the separate bundle preferentially, and keeping the (large) dictionaries out
|
|
230
230
|
* of the config JSON avoids serializing them twice.
|
|
231
231
|
*/
|
|
232
|
-
const toAssembleInputs = ({ dictionaries, ...config }, gazetteerEntries) => ({
|
|
233
|
-
pipelineConfigJson: encoder.encode(JSON.stringify(
|
|
232
|
+
const toAssembleInputs = ({ dictionaries, enableNer = false, ...config }, gazetteerEntries) => ({
|
|
233
|
+
pipelineConfigJson: encoder.encode(JSON.stringify({
|
|
234
|
+
...config,
|
|
235
|
+
enableNer
|
|
236
|
+
})),
|
|
234
237
|
dictionariesJson: dictionaries === void 0 ? void 0 : encoder.encode(JSON.stringify(dictionaries)),
|
|
235
238
|
gazetteerJson: gazetteerEntries.length === 0 ? void 0 : encoder.encode(JSON.stringify(gazetteerEntries))
|
|
236
239
|
});
|
|
@@ -365,9 +368,10 @@ const loadNativeAnonymizeBinding = (options = {}) => {
|
|
|
365
368
|
const platform = options.platform ?? process.platform;
|
|
366
369
|
const arch = options.arch ?? process.arch;
|
|
367
370
|
const libc = options.libc ?? detectNativeLibc(platform);
|
|
371
|
+
const env = options.env ?? process.env;
|
|
368
372
|
const specifiers = nativeBindingSpecifiers({
|
|
369
373
|
arch,
|
|
370
|
-
env
|
|
374
|
+
env,
|
|
371
375
|
libc,
|
|
372
376
|
platform
|
|
373
377
|
});
|
|
@@ -515,7 +519,8 @@ const getDefaultNativePipeline = (options = {}) => {
|
|
|
515
519
|
};
|
|
516
520
|
const get_default_native_pipeline = (options = {}) => getDefaultNativePipeline(options);
|
|
517
521
|
const preloadDefaultNativePipeline = (options = {}) => {
|
|
518
|
-
|
|
522
|
+
const pipeline = getDefaultNativePipeline(options);
|
|
523
|
+
return applyDefaultNativePipelineWarmup(pipeline, DEFAULT_NATIVE_PIPELINE_WARMUPS.lazyRegex);
|
|
519
524
|
};
|
|
520
525
|
const preload_default_native_pipeline = (options = {}) => preloadDefaultNativePipeline(options);
|
|
521
526
|
const redactDefaultText = (fullText, operators, options = {}) => getDefaultNativePipeline(options).redactText(fullText, operators);
|
|
@@ -569,10 +574,12 @@ const applyDefaultNativePipelineWarmup = (pipeline, warmup) => {
|
|
|
569
574
|
return pipeline;
|
|
570
575
|
};
|
|
571
576
|
const createNativePipelineFromResolvedDefaultPackage = ({ binding, language, packagePath }) => {
|
|
572
|
-
|
|
577
|
+
const packageBytes = packagePath === void 0 ? readDefaultNativePipelinePackageFile(defaultPackageFileOptions(language)) : readNativePipelinePackageFile(packagePath);
|
|
578
|
+
return createNativePipelineFromTrustedDefaultPackage(binding, packageBytes);
|
|
573
579
|
};
|
|
574
580
|
const createNativePipelineFromResolvedDefaultPackageAsync = async ({ binding, language, packagePath }) => {
|
|
575
|
-
|
|
581
|
+
const packageBytes = packagePath === void 0 ? await readDefaultNativePipelinePackageFileAsync(defaultPackageFileOptions(language)) : await readNativePipelinePackageFileAsync(packagePath);
|
|
582
|
+
return createNativePipelineFromTrustedDefaultPackage(binding, packageBytes);
|
|
576
583
|
};
|
|
577
584
|
const createNativePipelineFromTrustedDefaultPackage = (binding, packageBytes) => new PreparedNativePipeline(new PreparedNativeAnonymizer(binding.NativePreparedSearch.fromTrustedPreparedPackageBytesWithoutCache?.(packageBytes) ?? binding.NativePreparedSearch.fromTrustedPreparedPackageBytes?.(packageBytes) ?? binding.NativePreparedSearch.fromPreparedPackageBytesWithoutCache?.(packageBytes) ?? binding.NativePreparedSearch.fromPreparedPackageBytes(packageBytes)));
|
|
578
585
|
const defaultPackageFileOptions = (language) => language === void 0 ? {} : { language };
|
|
@@ -612,7 +619,8 @@ const defaultPipelineInflightCacheFor = (binding) => {
|
|
|
612
619
|
const defaultPipelineCacheKey = ({ binding, language, packagePath }) => [binding.nativePackageVersion(), packagePath ?? (language === void 0 ? DEFAULT_NATIVE_PIPELINE_PACKAGE_CACHE_KEY : `language:${language}`)].join("\0");
|
|
613
620
|
const defaultNativePipelinePackageUrl = (language) => {
|
|
614
621
|
if (language === void 0) return DEFAULT_NATIVE_PIPELINE_PACKAGE_URL;
|
|
615
|
-
|
|
622
|
+
const normalized = resolveDefaultNativePipelineLanguage(language);
|
|
623
|
+
return defaultNativePipelineLanguagePackageUrl(normalized);
|
|
616
624
|
};
|
|
617
625
|
const defaultNativePipelineLanguagePackageUrl = (language) => new URL(`../native-pipeline.${language}.stlanonpkg`, import.meta.url);
|
|
618
626
|
const resolveDefaultNativePipelineLanguage = (language) => {
|
|
@@ -693,7 +701,8 @@ const detectNativeLibc = (platform) => {
|
|
|
693
701
|
};
|
|
694
702
|
const tryLoadNativeBinding = ({ specifier, requireModule, errors }) => {
|
|
695
703
|
try {
|
|
696
|
-
const
|
|
704
|
+
const loaded = requireModule(specifier);
|
|
705
|
+
const binding = toNativeAnonymizeBinding(loaded);
|
|
697
706
|
if (binding) return binding;
|
|
698
707
|
errors.push(`${specifier}: module does not match native binding shape`);
|
|
699
708
|
} catch (error) {
|