euredact 0.3.9 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -9
- package/dist/cjs/cloud/client.js +219 -0
- package/dist/cjs/cloud/config.js +57 -0
- package/dist/cjs/cloud/errors.js +55 -0
- package/dist/cjs/cloud/index.js +31 -0
- package/dist/cjs/index.js +22 -3
- package/dist/cjs/sdk.js +92 -0
- package/dist/cjs/types.js +45 -1
- package/dist/esm/cloud/client.d.ts +42 -0
- package/dist/esm/cloud/client.js +215 -0
- package/dist/esm/cloud/config.d.ts +43 -0
- package/dist/esm/cloud/config.js +51 -0
- package/dist/esm/cloud/errors.d.ts +32 -0
- package/dist/esm/cloud/errors.js +48 -0
- package/dist/esm/cloud/index.d.ts +19 -0
- package/dist/esm/cloud/index.js +19 -0
- package/dist/esm/index.d.ts +9 -1
- package/dist/esm/index.js +11 -1
- package/dist/esm/sdk.d.ts +7 -0
- package/dist/esm/sdk.js +59 -0
- package/dist/esm/types.d.ts +33 -1
- package/dist/esm/types.js +44 -1
- package/package.json +4 -3
package/dist/esm/index.d.ts
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
|
-
export { EntityType, DetectionSource, LEGACY_TYPE_ALIASES } from "./types.js";
|
|
1
|
+
export { EntityType, DetectionSource, LEGACY_TYPE_ALIASES, canonicalType } from "./types.js";
|
|
2
2
|
export type { Detection, RedactResult, PatternDef, CountryConfig, CountryEvidence } from "./types.js";
|
|
3
3
|
export { DocumentContext } from "./rules/context.js";
|
|
4
4
|
export { EuRedact, type RedactOptions } from "./sdk.js";
|
|
5
5
|
export { COUNTRY_CONFIGS } from "./rules/countries/index.js";
|
|
6
6
|
export { setBicRegistry, getBicRegistry, SEED_BIC6_PREFIXES, type BicRegistryProvider } from "./rules/bicRegistry.js";
|
|
7
|
+
export { configure, getConfig, CloudClient, CloudError, NotConfiguredError, QuotaExceededError, TooLargeError, type CloudConfig, type ConfigureOptions, type CloudRedactOptions, } from "./cloud/index.js";
|
|
7
8
|
import { type RedactOptions } from "./sdk.js";
|
|
8
9
|
import type { RedactResult } from "./types.js";
|
|
9
10
|
export declare function availableCountries(): string[];
|
|
10
11
|
export declare function redact(text: string, options?: RedactOptions): RedactResult;
|
|
12
|
+
/**
|
|
13
|
+
* Redact, awaiting the cloud tier when `mode: "cloud"` is asked for.
|
|
14
|
+
*
|
|
15
|
+
* `mode: "rules"` resolves immediately with exactly what `redact()` returns,
|
|
16
|
+
* so a caller that may or may not use the cloud tier can hold one code path.
|
|
17
|
+
*/
|
|
18
|
+
export declare function redactAsync(text: string, options?: RedactOptions): Promise<RedactResult>;
|
|
11
19
|
export declare function addCustomPattern(name: string, pattern: string): void;
|
|
12
20
|
export declare function clear(): void;
|
|
13
21
|
export declare function redactBatch(texts: string[], options?: RedactOptions): RedactResult[];
|
package/dist/esm/index.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
export { EntityType, DetectionSource, LEGACY_TYPE_ALIASES } from "./types.js";
|
|
1
|
+
export { EntityType, DetectionSource, LEGACY_TYPE_ALIASES, canonicalType } from "./types.js";
|
|
2
2
|
export { DocumentContext } from "./rules/context.js";
|
|
3
3
|
export { EuRedact } from "./sdk.js";
|
|
4
4
|
export { COUNTRY_CONFIGS } from "./rules/countries/index.js";
|
|
5
5
|
export { setBicRegistry, getBicRegistry, SEED_BIC6_PREFIXES } from "./rules/bicRegistry.js";
|
|
6
|
+
export { configure, getConfig, CloudClient, CloudError, NotConfiguredError, QuotaExceededError, TooLargeError, } from "./cloud/index.js";
|
|
6
7
|
import { EuRedact } from "./sdk.js";
|
|
7
8
|
import { COUNTRY_CONFIGS } from "./rules/countries/index.js";
|
|
8
9
|
let _instance = null;
|
|
@@ -17,6 +18,15 @@ export function availableCountries() {
|
|
|
17
18
|
export function redact(text, options) {
|
|
18
19
|
return getInstance().redact(text, options);
|
|
19
20
|
}
|
|
21
|
+
/**
|
|
22
|
+
* Redact, awaiting the cloud tier when `mode: "cloud"` is asked for.
|
|
23
|
+
*
|
|
24
|
+
* `mode: "rules"` resolves immediately with exactly what `redact()` returns,
|
|
25
|
+
* so a caller that may or may not use the cloud tier can hold one code path.
|
|
26
|
+
*/
|
|
27
|
+
export function redactAsync(text, options) {
|
|
28
|
+
return getInstance().redactAsync(text, options);
|
|
29
|
+
}
|
|
20
30
|
export function addCustomPattern(name, pattern) {
|
|
21
31
|
getInstance().addCustomPattern(name, pattern);
|
|
22
32
|
}
|
package/dist/esm/sdk.d.ts
CHANGED
|
@@ -40,5 +40,12 @@ export declare class EuRedact {
|
|
|
40
40
|
* Call this in long-running processes to free PII from memory. */
|
|
41
41
|
clear(): void;
|
|
42
42
|
redact(text: string, options?: RedactOptions): RedactResult;
|
|
43
|
+
/**
|
|
44
|
+
* Redact, awaiting the cloud tier when `mode: "cloud"` is asked for.
|
|
45
|
+
*
|
|
46
|
+
* `mode: "rules"` resolves immediately with exactly what `redact()` returns,
|
|
47
|
+
* so callers that may or may not use the cloud tier can hold one code path.
|
|
48
|
+
*/
|
|
49
|
+
redactAsync(text: string, options?: RedactOptions): Promise<RedactResult>;
|
|
43
50
|
redactBatch(texts: string[], options?: RedactOptions): RedactResult[];
|
|
44
51
|
}
|
package/dist/esm/sdk.js
CHANGED
|
@@ -45,6 +45,40 @@ class ReferentialMapper {
|
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
47
|
const DEFAULT_MAX_INPUT_LENGTH = 10_485_760; // ~10 MB of text
|
|
48
|
+
/**
|
|
49
|
+
* Send a document to the cloud tier.
|
|
50
|
+
*
|
|
51
|
+
* Options the service cannot honour throw rather than being ignored. Silently
|
|
52
|
+
* dropping one would mean returning a result that does not match what was
|
|
53
|
+
* asked for — which, for anything that changes which spans come back, is
|
|
54
|
+
* under-redaction wearing a plausible face.
|
|
55
|
+
*/
|
|
56
|
+
async function redactViaCloud(text, options) {
|
|
57
|
+
const { CloudClient } = await import("./cloud/client.js");
|
|
58
|
+
const countries = options.countries ?? null;
|
|
59
|
+
if (!countries || countries.length !== 1) {
|
|
60
|
+
throw new Error('cloud mode needs exactly one country, e.g. { countries: ["BE"] }. The ' +
|
|
61
|
+
"model is trained and evaluated per country, so a multi-country request " +
|
|
62
|
+
"has no defined behaviour.");
|
|
63
|
+
}
|
|
64
|
+
if (options.countryHint) {
|
|
65
|
+
throw new Error("countryHint is not supported in cloud mode");
|
|
66
|
+
}
|
|
67
|
+
if (options.context || options.chunkOffset) {
|
|
68
|
+
throw new Error("context/chunkOffset are not supported in cloud mode: the model has never " +
|
|
69
|
+
"seen a chunk boundary, so the service rejects oversized input rather " +
|
|
70
|
+
"than splitting it");
|
|
71
|
+
}
|
|
72
|
+
if (options.referentialIntegrity) {
|
|
73
|
+
throw new Error("referentialIntegrity is not supported in cloud mode");
|
|
74
|
+
}
|
|
75
|
+
// detectDates is deliberately NOT forwarded. The service always runs its
|
|
76
|
+
// rules engine with dates on, because that is what the model was trained
|
|
77
|
+
// against; the caller's value cannot change that. It is the one ignored
|
|
78
|
+
// option that is safe to ignore — it can only cause MORE to be detected,
|
|
79
|
+
// never less, so it cannot produce under-redaction.
|
|
80
|
+
return new CloudClient().redact(text, { country: countries[0] });
|
|
81
|
+
}
|
|
48
82
|
/**
|
|
49
83
|
* Reject a bare string where a list of country codes is expected.
|
|
50
84
|
*
|
|
@@ -92,6 +126,19 @@ export class EuRedact {
|
|
|
92
126
|
redact(text, options = {}) {
|
|
93
127
|
checkCountryArg(options.countries, "countries");
|
|
94
128
|
checkCountryArg(options.countryHint, "countryHint");
|
|
129
|
+
const requestedMode = options.mode ?? "rules";
|
|
130
|
+
if (requestedMode === "cloud") {
|
|
131
|
+
// Cannot be served synchronously: the cloud tier is a network call, and
|
|
132
|
+
// returning rules-only output instead would be the silent under-redaction
|
|
133
|
+
// this guard exists to prevent. Point at the async entry point rather
|
|
134
|
+
// than inventing a Promise-or-value union return type.
|
|
135
|
+
throw new Error('mode: "cloud" is asynchronous — use redactAsync(text, { mode: "cloud" }) ' +
|
|
136
|
+
"(or EuRedact#redactAsync). redact() is synchronous and can only serve " +
|
|
137
|
+
'mode: "rules".');
|
|
138
|
+
}
|
|
139
|
+
if (requestedMode !== "rules") {
|
|
140
|
+
throw new Error(`unknown mode ${JSON.stringify(requestedMode)}: expected "rules" or "cloud"`);
|
|
141
|
+
}
|
|
95
142
|
if (text.length > this.maxInputLength) {
|
|
96
143
|
throw new Error(`Input text length (${text.length.toLocaleString()} chars) exceeds the maximum ` +
|
|
97
144
|
`(${this.maxInputLength.toLocaleString()} chars). Split the input or ` +
|
|
@@ -175,6 +222,18 @@ export class EuRedact {
|
|
|
175
222
|
}
|
|
176
223
|
return result;
|
|
177
224
|
}
|
|
225
|
+
/**
|
|
226
|
+
* Redact, awaiting the cloud tier when `mode: "cloud"` is asked for.
|
|
227
|
+
*
|
|
228
|
+
* `mode: "rules"` resolves immediately with exactly what `redact()` returns,
|
|
229
|
+
* so callers that may or may not use the cloud tier can hold one code path.
|
|
230
|
+
*/
|
|
231
|
+
async redactAsync(text, options = {}) {
|
|
232
|
+
const mode = options.mode ?? "rules";
|
|
233
|
+
if (mode !== "cloud")
|
|
234
|
+
return this.redact(text, options);
|
|
235
|
+
return redactViaCloud(text, options);
|
|
236
|
+
}
|
|
178
237
|
redactBatch(texts, options = {}) {
|
|
179
238
|
// Generation is country-blind, so every country's patterns are loaded
|
|
180
239
|
// regardless of what the caller declared.
|
package/dist/esm/types.d.ts
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
export declare enum EntityType {
|
|
2
|
-
|
|
2
|
+
/**
|
|
3
|
+
* Canonical name for a person's name. `NAME` is kept below as a legacy alias
|
|
4
|
+
* with the same value, exactly as `IBAN` aliases `BANK_ACCOUNT`, so
|
|
5
|
+
* `EntityType.NAME === EntityType.PERSON_NAME` and existing code keeps
|
|
6
|
+
* working — but the emitted value, and therefore the `[PERSON_NAME]`
|
|
7
|
+
* placeholder written into redacted text, is canonical.
|
|
8
|
+
*
|
|
9
|
+
* Renamed rather than translated at the cloud boundary: two names for one
|
|
10
|
+
* type is how a whole category goes missing when someone filters on the
|
|
11
|
+
* spelling they happened to know. Nothing could have depended on the old
|
|
12
|
+
* value — the type is cloud-only and the cloud tier was stubbed, so it was
|
|
13
|
+
* never emitted.
|
|
14
|
+
*/
|
|
15
|
+
PERSON_NAME = "PERSON_NAME",
|
|
16
|
+
/** @deprecated legacy alias of {@link EntityType.PERSON_NAME} */
|
|
17
|
+
NAME = "PERSON_NAME",
|
|
3
18
|
ADDRESS = "ADDRESS",
|
|
4
19
|
/**
|
|
5
20
|
* Canonical name for a bank account identifier. `IBAN` is kept below as a
|
|
@@ -47,6 +62,15 @@ export declare enum EntityType {
|
|
|
47
62
|
UUID = "UUID",
|
|
48
63
|
SOCIAL_HANDLE = "SOCIAL_HANDLE",
|
|
49
64
|
SECRET = "SECRET",
|
|
65
|
+
ORGANISATION_NAME = "ORGANISATION_NAME",
|
|
66
|
+
JOB_TITLE = "JOB_TITLE",
|
|
67
|
+
MEDICAL_CONDITION = "MEDICAL_CONDITION",
|
|
68
|
+
SENSITIVE_ATTRIBUTE = "SENSITIVE_ATTRIBUTE",
|
|
69
|
+
BIOMETRIC_REF = "BIOMETRIC_REF",
|
|
70
|
+
FINANCIAL_AMOUNT = "FINANCIAL_AMOUNT",
|
|
71
|
+
QUASI_IDENTIFIER = "QUASI_IDENTIFIER",
|
|
72
|
+
CREDENTIAL = "CREDENTIAL",
|
|
73
|
+
URL = "URL",
|
|
50
74
|
OTHER = "OTHER"
|
|
51
75
|
}
|
|
52
76
|
/**
|
|
@@ -55,6 +79,14 @@ export declare enum EntityType {
|
|
|
55
79
|
* legacy_aliases). The engine never *emits* these names.
|
|
56
80
|
*/
|
|
57
81
|
export declare const LEGACY_TYPE_ALIASES: Record<string, string>;
|
|
82
|
+
/**
|
|
83
|
+
* Resolve a possibly-legacy type name to its canonical form.
|
|
84
|
+
*
|
|
85
|
+
* Mirrors `EntityType._missing_` in the Python SDK. An unrecognised name is
|
|
86
|
+
* returned unchanged rather than dropped: a client one release behind the
|
|
87
|
+
* service must not silently lose a whole category of PII.
|
|
88
|
+
*/
|
|
89
|
+
export declare function canonicalType(name: string): string;
|
|
58
90
|
export declare enum DetectionSource {
|
|
59
91
|
RULES = "rules",
|
|
60
92
|
CLOUD = "cloud"
|
package/dist/esm/types.js
CHANGED
|
@@ -1,6 +1,21 @@
|
|
|
1
1
|
export var EntityType;
|
|
2
2
|
(function (EntityType) {
|
|
3
|
-
|
|
3
|
+
/**
|
|
4
|
+
* Canonical name for a person's name. `NAME` is kept below as a legacy alias
|
|
5
|
+
* with the same value, exactly as `IBAN` aliases `BANK_ACCOUNT`, so
|
|
6
|
+
* `EntityType.NAME === EntityType.PERSON_NAME` and existing code keeps
|
|
7
|
+
* working — but the emitted value, and therefore the `[PERSON_NAME]`
|
|
8
|
+
* placeholder written into redacted text, is canonical.
|
|
9
|
+
*
|
|
10
|
+
* Renamed rather than translated at the cloud boundary: two names for one
|
|
11
|
+
* type is how a whole category goes missing when someone filters on the
|
|
12
|
+
* spelling they happened to know. Nothing could have depended on the old
|
|
13
|
+
* value — the type is cloud-only and the cloud tier was stubbed, so it was
|
|
14
|
+
* never emitted.
|
|
15
|
+
*/
|
|
16
|
+
EntityType["PERSON_NAME"] = "PERSON_NAME";
|
|
17
|
+
/** @deprecated legacy alias of {@link EntityType.PERSON_NAME} */
|
|
18
|
+
EntityType["NAME"] = "PERSON_NAME";
|
|
4
19
|
EntityType["ADDRESS"] = "ADDRESS";
|
|
5
20
|
/**
|
|
6
21
|
* Canonical name for a bank account identifier. `IBAN` is kept below as a
|
|
@@ -48,6 +63,20 @@ export var EntityType;
|
|
|
48
63
|
EntityType["UUID"] = "UUID";
|
|
49
64
|
EntityType["SOCIAL_HANDLE"] = "SOCIAL_HANDLE";
|
|
50
65
|
EntityType["SECRET"] = "SECRET";
|
|
66
|
+
// ── [CLOUD EXTENSION] ────────────────────────────────────────────────────
|
|
67
|
+
// Types only the cloud tier can detect. The rule engine never emits these —
|
|
68
|
+
// there is no shape to match on — which is exactly why they exist: the
|
|
69
|
+
// model's job is what the rules cannot catch. Named to match the detection
|
|
70
|
+
// canon the service is trained and evaluated against.
|
|
71
|
+
EntityType["ORGANISATION_NAME"] = "ORGANISATION_NAME";
|
|
72
|
+
EntityType["JOB_TITLE"] = "JOB_TITLE";
|
|
73
|
+
EntityType["MEDICAL_CONDITION"] = "MEDICAL_CONDITION";
|
|
74
|
+
EntityType["SENSITIVE_ATTRIBUTE"] = "SENSITIVE_ATTRIBUTE";
|
|
75
|
+
EntityType["BIOMETRIC_REF"] = "BIOMETRIC_REF";
|
|
76
|
+
EntityType["FINANCIAL_AMOUNT"] = "FINANCIAL_AMOUNT";
|
|
77
|
+
EntityType["QUASI_IDENTIFIER"] = "QUASI_IDENTIFIER";
|
|
78
|
+
EntityType["CREDENTIAL"] = "CREDENTIAL";
|
|
79
|
+
EntityType["URL"] = "URL";
|
|
51
80
|
EntityType["OTHER"] = "OTHER";
|
|
52
81
|
})(EntityType || (EntityType = {}));
|
|
53
82
|
/**
|
|
@@ -57,7 +86,21 @@ export var EntityType;
|
|
|
57
86
|
*/
|
|
58
87
|
export const LEGACY_TYPE_ALIASES = {
|
|
59
88
|
IBAN: "BANK_ACCOUNT",
|
|
89
|
+
NAME: "PERSON_NAME",
|
|
90
|
+
STREET_ADDRESS: "ADDRESS",
|
|
91
|
+
NATIONALITY_ETHNICITY: "SENSITIVE_ATTRIBUTE",
|
|
60
92
|
};
|
|
93
|
+
/**
|
|
94
|
+
* Resolve a possibly-legacy type name to its canonical form.
|
|
95
|
+
*
|
|
96
|
+
* Mirrors `EntityType._missing_` in the Python SDK. An unrecognised name is
|
|
97
|
+
* returned unchanged rather than dropped: a client one release behind the
|
|
98
|
+
* service must not silently lose a whole category of PII.
|
|
99
|
+
*/
|
|
100
|
+
export function canonicalType(name) {
|
|
101
|
+
const upper = String(name ?? "").trim().toUpperCase();
|
|
102
|
+
return LEGACY_TYPE_ALIASES[upper] ?? upper;
|
|
103
|
+
}
|
|
61
104
|
export var DetectionSource;
|
|
62
105
|
(function (DetectionSource) {
|
|
63
106
|
DetectionSource["RULES"] = "rules";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "euredact",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "European PII redaction SDK -- fast, zero-dependency, rule-based detection and redaction of personal data across 31 European countries",
|
|
5
5
|
"homepage": "https://euredact.dev",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"build:cjs": "tsc -p tsconfig.cjs.json",
|
|
55
55
|
"postbuild:cjs": "echo '{\"type\":\"commonjs\"}' > dist/cjs/package.json",
|
|
56
56
|
"build": "rm -rf dist && npm run build:esm && npm run build:cjs",
|
|
57
|
-
"test": "npm run test:spec && npm run test:inference && npm run test:conformance",
|
|
57
|
+
"test": "npm run test:spec && npm run test:inference && npm run test:cloud && npm run test:conformance",
|
|
58
58
|
"benchmark": "node --import tsx src/__tests__/benchmark.ts",
|
|
59
59
|
"prepublishOnly": "npm run build",
|
|
60
60
|
"test:spec": "node --import tsx src/__tests__/spec.ts",
|
|
@@ -62,7 +62,8 @@
|
|
|
62
62
|
"test:eval:full": "node --max-old-space-size=8192 --import tsx src/__tests__/evalFull.ts",
|
|
63
63
|
"test:conformance": "node --import tsx src/__tests__/conformance.ts",
|
|
64
64
|
"test:inference": "node --import tsx src/__tests__/inference.ts",
|
|
65
|
-
"dump": "node --import tsx src/__tests__/dump.ts"
|
|
65
|
+
"dump": "node --import tsx src/__tests__/dump.ts",
|
|
66
|
+
"test:cloud": "node --import tsx src/__tests__/cloud.ts"
|
|
66
67
|
},
|
|
67
68
|
"devDependencies": {
|
|
68
69
|
"@types/node": "22.10.2",
|