@mlx-node/privacy 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/classifier.d.ts +32 -0
- package/dist/classifier.d.ts.map +1 -0
- package/dist/classifier.js +51 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1 -0
- package/dist/redactor.d.ts +28 -0
- package/dist/redactor.d.ts.map +1 -0
- package/dist/redactor.js +54 -0
- package/dist/train.d.ts +84 -0
- package/dist/train.d.ts.map +1 -0
- package/dist/train.js +48 -0
- package/dist/types.d.ts +110 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +9 -0
- package/package.json +36 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { ClassifyOptions, ClassifyResult, RedactOptions, RedactResult } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* High-level wrapper around the native `PrivacyFilterModel` NAPI class.
|
|
4
|
+
*
|
|
5
|
+
* The native binding's `load`/`classify` methods are synchronous, but the
|
|
6
|
+
* public API here is intentionally `async` so we can later move work to a
|
|
7
|
+
* worker thread without a breaking change to consumers.
|
|
8
|
+
*/
|
|
9
|
+
export declare class PrivacyFilter {
|
|
10
|
+
private readonly native;
|
|
11
|
+
private constructor();
|
|
12
|
+
/**
|
|
13
|
+
* Load a privacy-filter checkpoint from a directory.
|
|
14
|
+
*
|
|
15
|
+
* The directory must contain `config.json`, `model.safetensors`,
|
|
16
|
+
* `tokenizer.json`, and optionally `viterbi_calibration.json` and
|
|
17
|
+
* `tokenizer_config.json`.
|
|
18
|
+
*/
|
|
19
|
+
static load(modelPath: string): Promise<PrivacyFilter>;
|
|
20
|
+
/**
|
|
21
|
+
* Classify `text` and return detected PII entities (and optionally
|
|
22
|
+
* per-token tags when `opts.returnTokens` is `true`).
|
|
23
|
+
*/
|
|
24
|
+
classify(text: string, opts?: ClassifyOptions): Promise<ClassifyResult>;
|
|
25
|
+
/**
|
|
26
|
+
* Classify `text`, then replace each detected entity according to
|
|
27
|
+
* `opts.replacement` (defaults to `[<label>]`). If `opts.labels` is set,
|
|
28
|
+
* only entities whose label is in that list are redacted.
|
|
29
|
+
*/
|
|
30
|
+
redact(text: string, opts?: RedactOptions): Promise<RedactResult>;
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=classifier.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../src/classifier.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAU,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEvG;;;;;;GAMG;AACH,qBAAa,aAAa;IACJ,OAAO,CAAC,QAAQ,CAAC,MAAM;IAA3C,OAAO,eAAkE;IAEzE;;;;;;OAMG;IACH,OAAa,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAK3D;IAED;;;OAGG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAS5E;IAED;;;;OAIG;IACG,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAGtE;CACF"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { PrivacyFilterModel as NativePrivacyFilterModel } from '@mlx-node/core';
|
|
2
|
+
import { redactImpl } from './redactor.js';
|
|
3
|
+
/**
|
|
4
|
+
* High-level wrapper around the native `PrivacyFilterModel` NAPI class.
|
|
5
|
+
*
|
|
6
|
+
* The native binding's `load`/`classify` methods are synchronous, but the
|
|
7
|
+
* public API here is intentionally `async` so we can later move work to a
|
|
8
|
+
* worker thread without a breaking change to consumers.
|
|
9
|
+
*/
|
|
10
|
+
export class PrivacyFilter {
|
|
11
|
+
native;
|
|
12
|
+
constructor(native) {
|
|
13
|
+
this.native = native;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Load a privacy-filter checkpoint from a directory.
|
|
17
|
+
*
|
|
18
|
+
* The directory must contain `config.json`, `model.safetensors`,
|
|
19
|
+
* `tokenizer.json`, and optionally `viterbi_calibration.json` and
|
|
20
|
+
* `tokenizer_config.json`.
|
|
21
|
+
*/
|
|
22
|
+
static async load(modelPath) {
|
|
23
|
+
// Native `load` is synchronous; we keep the public surface async so
|
|
24
|
+
// future implementations (e.g. off-main-thread load) don't break ABI.
|
|
25
|
+
const native = NativePrivacyFilterModel.load(modelPath);
|
|
26
|
+
return new PrivacyFilter(native);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Classify `text` and return detected PII entities (and optionally
|
|
30
|
+
* per-token tags when `opts.returnTokens` is `true`).
|
|
31
|
+
*/
|
|
32
|
+
async classify(text, opts) {
|
|
33
|
+
const result = this.native.classify(text, opts ?? null);
|
|
34
|
+
// The native binding returns `label: string`; at runtime any
|
|
35
|
+
// non-background span is one of the 8 `PrivacyLabel` values, so the
|
|
36
|
+
// cast through `Entity[]` narrows the public surface for callers.
|
|
37
|
+
return {
|
|
38
|
+
entities: result.entities,
|
|
39
|
+
tokens: result.tokens ?? undefined,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Classify `text`, then replace each detected entity according to
|
|
44
|
+
* `opts.replacement` (defaults to `[<label>]`). If `opts.labels` is set,
|
|
45
|
+
* only entities whose label is in that list are redacted.
|
|
46
|
+
*/
|
|
47
|
+
async redact(text, opts) {
|
|
48
|
+
const { entities } = await this.classify(text, opts);
|
|
49
|
+
return redactImpl(text, entities, opts);
|
|
50
|
+
}
|
|
51
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,mBAAmB,YAAY,CAAC;AAEhC,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { PrivacyFilter } from './classifier.js';
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Entity, RedactOptions } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Apply entity redaction to `text` using the entities returned by the
|
|
4
|
+
* classifier.
|
|
5
|
+
*
|
|
6
|
+
* - If `opts.labels` is set, only entities whose label is in that list are
|
|
7
|
+
* redacted (others are left in place verbatim).
|
|
8
|
+
* - Each surviving entity span is replaced via {@link renderReplacement}.
|
|
9
|
+
* - The returned `entities` array is sorted by start offset and only
|
|
10
|
+
* contains the entities that were actually redacted (post-filter).
|
|
11
|
+
*
|
|
12
|
+
* Entities are assumed to be non-overlapping (the privacy-filter Viterbi
|
|
13
|
+
* decoder produces non-overlapping spans), and we walk them left-to-right
|
|
14
|
+
* with a running cursor so the output is built in a single pass.
|
|
15
|
+
*
|
|
16
|
+
* IMPORTANT: `Entity.start`/`Entity.end` are **UTF-8 byte offsets** (the
|
|
17
|
+
* Hugging Face `tokenizers` convention used by the underlying Rust
|
|
18
|
+
* classifier). JavaScript's `String.prototype.slice` indexes UTF-16 code
|
|
19
|
+
* units, so slicing the original string directly corrupts spans whenever
|
|
20
|
+
* any non-ASCII character (emoji, CJK, accented Latin) appears before an
|
|
21
|
+
* entity. We therefore encode `text` as a UTF-8 Buffer once and slice
|
|
22
|
+
* bytes, decoding each segment back to a string.
|
|
23
|
+
*/
|
|
24
|
+
export declare function redactImpl(text: string, entities: Entity[], opts?: RedactOptions): {
|
|
25
|
+
redacted: string;
|
|
26
|
+
entities: Entity[];
|
|
27
|
+
};
|
|
28
|
+
//# sourceMappingURL=redactor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redactor.d.ts","sourceRoot":"","sources":["../src/redactor.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,aAAa,EAAe,MAAM,YAAY,CAAC;AAErE;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAAE,EAClB,IAAI,CAAC,EAAE,aAAa,GACnB;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAsB1C"}
|
package/dist/redactor.js
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
/**
|
|
3
|
+
* Apply entity redaction to `text` using the entities returned by the
|
|
4
|
+
* classifier.
|
|
5
|
+
*
|
|
6
|
+
* - If `opts.labels` is set, only entities whose label is in that list are
|
|
7
|
+
* redacted (others are left in place verbatim).
|
|
8
|
+
* - Each surviving entity span is replaced via {@link renderReplacement}.
|
|
9
|
+
* - The returned `entities` array is sorted by start offset and only
|
|
10
|
+
* contains the entities that were actually redacted (post-filter).
|
|
11
|
+
*
|
|
12
|
+
* Entities are assumed to be non-overlapping (the privacy-filter Viterbi
|
|
13
|
+
* decoder produces non-overlapping spans), and we walk them left-to-right
|
|
14
|
+
* with a running cursor so the output is built in a single pass.
|
|
15
|
+
*
|
|
16
|
+
* IMPORTANT: `Entity.start`/`Entity.end` are **UTF-8 byte offsets** (the
|
|
17
|
+
* Hugging Face `tokenizers` convention used by the underlying Rust
|
|
18
|
+
* classifier). JavaScript's `String.prototype.slice` indexes UTF-16 code
|
|
19
|
+
* units, so slicing the original string directly corrupts spans whenever
|
|
20
|
+
* any non-ASCII character (emoji, CJK, accented Latin) appears before an
|
|
21
|
+
* entity. We therefore encode `text` as a UTF-8 Buffer once and slice
|
|
22
|
+
* bytes, decoding each segment back to a string.
|
|
23
|
+
*/
|
|
24
|
+
export function redactImpl(text, entities, opts) {
|
|
25
|
+
const labelFilter = opts?.labels ? new Set(opts.labels) : null;
|
|
26
|
+
const filtered = labelFilter ? entities.filter((e) => labelFilter.has(e.label)) : entities;
|
|
27
|
+
const sorted = [...filtered].sort((a, b) => a.start - b.start);
|
|
28
|
+
// Fast path: no redactions → return the original text unchanged without
|
|
29
|
+
// a Buffer round-trip.
|
|
30
|
+
if (sorted.length === 0) {
|
|
31
|
+
return { redacted: text, entities: sorted };
|
|
32
|
+
}
|
|
33
|
+
const buf = Buffer.from(text, 'utf8');
|
|
34
|
+
let out = '';
|
|
35
|
+
let cursor = 0;
|
|
36
|
+
for (const e of sorted) {
|
|
37
|
+
out += buf.toString('utf8', cursor, e.start);
|
|
38
|
+
out += renderReplacement(e, opts?.replacement);
|
|
39
|
+
cursor = e.end;
|
|
40
|
+
}
|
|
41
|
+
out += buf.toString('utf8', cursor, buf.length);
|
|
42
|
+
return { redacted: out, entities: sorted };
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Render the replacement string for a single entity. Defaults to
|
|
46
|
+
* `[<label>]` when no replacement is provided.
|
|
47
|
+
*/
|
|
48
|
+
function renderReplacement(entity, replacement) {
|
|
49
|
+
if (typeof replacement === 'function')
|
|
50
|
+
return replacement(entity);
|
|
51
|
+
if (replacement === 'label' || replacement == null)
|
|
52
|
+
return `[${entity.label}]`;
|
|
53
|
+
return replacement;
|
|
54
|
+
}
|
package/dist/train.d.ts
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration for {@link PrivacyLoraTrainer.create}.
|
|
3
|
+
*
|
|
4
|
+
* All tunables are optional; omitting a field falls back to the
|
|
5
|
+
* documented defaults from the native trainer:
|
|
6
|
+
*
|
|
7
|
+
* - `rank` = 16, `alpha` = 32, `dropout` = 0.05
|
|
8
|
+
* - `loraLr` = 1e-4, `classifierLr` = 5e-5
|
|
9
|
+
* - `batchSize` = 2, `maxSeqLen` = 256, `numEpochs` = 3
|
|
10
|
+
* - `gradAccumSteps` = 4, `gradClip` = 1.0
|
|
11
|
+
* - `saveEvery` = 500 (set 0 to disable intermediate checkpoints)
|
|
12
|
+
* - `padTokenId` = 0
|
|
13
|
+
*
|
|
14
|
+
* `modelPath`, `dataPath`, and `outputDir` are required.
|
|
15
|
+
*
|
|
16
|
+
* The field names match the native NAPI casing
|
|
17
|
+
* ({@link import('@mlx-node/core').PrivacyLoraTrainConfigJs}) so the
|
|
18
|
+
* wrapper can pass values straight through without re-keying.
|
|
19
|
+
*/
|
|
20
|
+
export interface PrivacyLoraTrainConfig {
|
|
21
|
+
/** Filesystem path to the base privacy-filter checkpoint directory. */
|
|
22
|
+
modelPath: string;
|
|
23
|
+
/** Path to the training JSONL dataset (pre-tokenized). */
|
|
24
|
+
dataPath: string;
|
|
25
|
+
/**
|
|
26
|
+
* Optional evaluation JSONL dataset path. Currently ignored by the
|
|
27
|
+
* trainer — reserved for future eval-loop integration.
|
|
28
|
+
*/
|
|
29
|
+
evalPath?: string;
|
|
30
|
+
/**
|
|
31
|
+
* Directory where `adapter.safetensors`, `adapter_config.json`,
|
|
32
|
+
* `optimizer_state.safetensors`, and `trainer_state.json` are written.
|
|
33
|
+
*/
|
|
34
|
+
outputDir: string;
|
|
35
|
+
rank?: number;
|
|
36
|
+
alpha?: number;
|
|
37
|
+
dropout?: number;
|
|
38
|
+
loraLr?: number;
|
|
39
|
+
classifierLr?: number;
|
|
40
|
+
batchSize?: number;
|
|
41
|
+
maxSeqLen?: number;
|
|
42
|
+
numEpochs?: number;
|
|
43
|
+
gradAccumSteps?: number;
|
|
44
|
+
gradClip?: number;
|
|
45
|
+
saveEvery?: number;
|
|
46
|
+
resumeFrom?: string;
|
|
47
|
+
padTokenId?: number;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* High-level wrapper around the native `PrivacyLoraTrainerJs` NAPI class.
|
|
51
|
+
*
|
|
52
|
+
* The native binding's `create` factory is synchronous (it loads the base
|
|
53
|
+
* checkpoint, attaches zero-B LoRA adapters, and parses the JSONL dataset
|
|
54
|
+
* inline), but the public API here is intentionally `async` so we can later
|
|
55
|
+
* move work to a worker thread without a breaking change to consumers.
|
|
56
|
+
*
|
|
57
|
+
* `train()` and `saveAdapter()` are already async on the native side — they
|
|
58
|
+
* funnel through `spawn_blocking` so the Node.js event loop stays responsive
|
|
59
|
+
* for the duration of the training run.
|
|
60
|
+
*/
|
|
61
|
+
export declare class PrivacyLoraTrainer {
|
|
62
|
+
private readonly inner;
|
|
63
|
+
private constructor();
|
|
64
|
+
/**
|
|
65
|
+
* Construct a new trainer.
|
|
66
|
+
*
|
|
67
|
+
* Loads the base privacy-filter model from `config.modelPath`, attaches
|
|
68
|
+
* zero-B LoRA adapters on every attention projection, and parses the
|
|
69
|
+
* pre-tokenized JSONL dataset at `config.dataPath`.
|
|
70
|
+
*/
|
|
71
|
+
static create(config: PrivacyLoraTrainConfig): Promise<PrivacyLoraTrainer>;
|
|
72
|
+
/**
|
|
73
|
+
* Run the full training loop. Resolves with the final optimizer step
|
|
74
|
+
* count once `numEpochs` epochs have completed.
|
|
75
|
+
*/
|
|
76
|
+
train(): Promise<number>;
|
|
77
|
+
/**
|
|
78
|
+
* Persist adapter weights, optimizer state, and trainer progress to the
|
|
79
|
+
* configured `outputDir`. Overwrites any previous checkpoint at that
|
|
80
|
+
* path.
|
|
81
|
+
*/
|
|
82
|
+
saveAdapter(): Promise<void>;
|
|
83
|
+
}
|
|
84
|
+
//# sourceMappingURL=train.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"train.d.ts","sourceRoot":"","sources":["../src/train.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,sBAAsB;IACrC,uEAAuE;IACvE,SAAS,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,kBAAkB;IACT,OAAO,CAAC,QAAQ,CAAC,KAAK;IAA1C,OAAO;IAEP;;;;;;OAMG;WACU,MAAM,CAAC,MAAM,EAAE,sBAAsB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAQhF;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAI9B;;;;OAIG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;CAGnC"}
|
package/dist/train.js
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { PrivacyLoraTrainerJs as NativePrivacyLoraTrainerJs, } from '@mlx-node/core';
|
|
2
|
+
/**
|
|
3
|
+
* High-level wrapper around the native `PrivacyLoraTrainerJs` NAPI class.
|
|
4
|
+
*
|
|
5
|
+
* The native binding's `create` factory is synchronous (it loads the base
|
|
6
|
+
* checkpoint, attaches zero-B LoRA adapters, and parses the JSONL dataset
|
|
7
|
+
* inline), but the public API here is intentionally `async` so we can later
|
|
8
|
+
* move work to a worker thread without a breaking change to consumers.
|
|
9
|
+
*
|
|
10
|
+
* `train()` and `saveAdapter()` are already async on the native side — they
|
|
11
|
+
* funnel through `spawn_blocking` so the Node.js event loop stays responsive
|
|
12
|
+
* for the duration of the training run.
|
|
13
|
+
*/
|
|
14
|
+
export class PrivacyLoraTrainer {
|
|
15
|
+
inner;
|
|
16
|
+
constructor(inner) {
|
|
17
|
+
this.inner = inner;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Construct a new trainer.
|
|
21
|
+
*
|
|
22
|
+
* Loads the base privacy-filter model from `config.modelPath`, attaches
|
|
23
|
+
* zero-B LoRA adapters on every attention projection, and parses the
|
|
24
|
+
* pre-tokenized JSONL dataset at `config.dataPath`.
|
|
25
|
+
*/
|
|
26
|
+
static async create(config) {
|
|
27
|
+
// NAPI requires the camelCase keys to exactly match the interface — pass
|
|
28
|
+
// the object through directly so any future field additions are picked
|
|
29
|
+
// up without an extra mapping table.
|
|
30
|
+
const native = NativePrivacyLoraTrainerJs.create(config);
|
|
31
|
+
return new PrivacyLoraTrainer(native);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Run the full training loop. Resolves with the final optimizer step
|
|
35
|
+
* count once `numEpochs` epochs have completed.
|
|
36
|
+
*/
|
|
37
|
+
async train() {
|
|
38
|
+
return this.inner.train();
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Persist adapter weights, optimizer state, and trainer progress to the
|
|
42
|
+
* configured `outputDir`. Overwrites any previous checkpoint at that
|
|
43
|
+
* path.
|
|
44
|
+
*/
|
|
45
|
+
async saveAdapter() {
|
|
46
|
+
return this.inner.saveAdapter();
|
|
47
|
+
}
|
|
48
|
+
}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public types for the `@mlx-node/privacy` package.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the native NAPI surface ({@link import('@mlx-node/core').PrivacyEntity}
|
|
5
|
+
* etc.) but narrows the `label` field to the literal union of the 8 PII
|
|
6
|
+
* classes produced by the privacy-filter checkpoint, so consumers get
|
|
7
|
+
* autocomplete on `e.label === 'private_email'`.
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* The 8 PII categories the privacy-filter checkpoint can emit (without the
|
|
11
|
+
* BIOES prefix). At runtime the native binding returns a plain `string`, but
|
|
12
|
+
* any non-background span produced by the model is guaranteed to be one of
|
|
13
|
+
* these values, so the wrapper narrows it for callers.
|
|
14
|
+
*/
|
|
15
|
+
export type PrivacyLabel = 'account_number' | 'private_address' | 'private_date' | 'private_email' | 'private_person' | 'private_phone' | 'private_url' | 'secret';
|
|
16
|
+
/**
|
|
17
|
+
* A detected PII span. `start`/`end` are byte offsets into the original
|
|
18
|
+
* input string (Hugging Face `tokenizers` convention). `score` is the mean —
|
|
19
|
+
* across the span's tokens — of the softmax probability of the Viterbi-emitted
|
|
20
|
+
* tag at each token.
|
|
21
|
+
*/
|
|
22
|
+
export interface Entity {
|
|
23
|
+
start: number;
|
|
24
|
+
end: number;
|
|
25
|
+
label: PrivacyLabel;
|
|
26
|
+
score: number;
|
|
27
|
+
text: string;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Per-call Viterbi calibration overrides.
|
|
31
|
+
*
|
|
32
|
+
* Field names match the native NAPI binding's camelCase casing
|
|
33
|
+
* ({@link import('@mlx-node/core').PrivacyCalibration}) so the wrapper can
|
|
34
|
+
* pass user options straight through without re-keying.
|
|
35
|
+
*
|
|
36
|
+
* Any omitted field falls back to the model's default calibration
|
|
37
|
+
* (loaded from `viterbi_calibration.json` at load time).
|
|
38
|
+
*/
|
|
39
|
+
export interface ViterbiCalibration {
|
|
40
|
+
transitionBiasBackgroundStay: number;
|
|
41
|
+
transitionBiasBackgroundToStart: number;
|
|
42
|
+
transitionBiasEndToBackground: number;
|
|
43
|
+
transitionBiasEndToStart: number;
|
|
44
|
+
transitionBiasInsideToContinue: number;
|
|
45
|
+
transitionBiasInsideToEnd: number;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Options for {@link PrivacyFilter.classify}.
|
|
49
|
+
*
|
|
50
|
+
* - `threshold` (default `0.5`): minimum mean per-token probability for an
|
|
51
|
+
* extracted span to be returned.
|
|
52
|
+
* - `calibration`: per-call overrides on top of the checkpoint default.
|
|
53
|
+
* - `returnTokens` (default `false`): when `true`, the result includes a
|
|
54
|
+
* `tokens` array with one entry per input token.
|
|
55
|
+
*/
|
|
56
|
+
export interface ClassifyOptions {
|
|
57
|
+
threshold?: number;
|
|
58
|
+
calibration?: Partial<ViterbiCalibration>;
|
|
59
|
+
returnTokens?: boolean;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* A single token with its Viterbi-decoded tag. Emitted by
|
|
63
|
+
* {@link PrivacyFilter.classify} when `returnTokens: true`.
|
|
64
|
+
*
|
|
65
|
+
* `tag` is the full BIOES tag (`'O'` or `'B-...'`/`'I-...'`/`'E-...'`/
|
|
66
|
+
* `'S-...'`) chosen by the Viterbi decoder. `score` is the softmax
|
|
67
|
+
* probability of that emitted tag at this token, so `tag` and `score`
|
|
68
|
+
* always share decoders (at boundary tokens the Viterbi pick can differ
|
|
69
|
+
* from the local argmax).
|
|
70
|
+
*/
|
|
71
|
+
export interface Token {
|
|
72
|
+
text: string;
|
|
73
|
+
tag: string;
|
|
74
|
+
score: number;
|
|
75
|
+
start: number;
|
|
76
|
+
end: number;
|
|
77
|
+
}
|
|
78
|
+
/** Result of {@link PrivacyFilter.classify}. */
|
|
79
|
+
export interface ClassifyResult {
|
|
80
|
+
entities: Entity[];
|
|
81
|
+
tokens?: Token[];
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* How to replace a detected entity in {@link PrivacyFilter.redact}.
|
|
85
|
+
*
|
|
86
|
+
* - The sentinel string `'label'` produces `[<label>]` (this is the
|
|
87
|
+
* default when no `replacement` is provided).
|
|
88
|
+
* - Any other string is inserted verbatim.
|
|
89
|
+
* - A function receives the entity and returns the replacement string.
|
|
90
|
+
*/
|
|
91
|
+
export type Replacement = string | ((entity: Entity) => string);
|
|
92
|
+
/**
|
|
93
|
+
* Options for {@link PrivacyFilter.redact}. Extends {@link ClassifyOptions}
|
|
94
|
+
* so threshold / calibration / returnTokens can be passed through to the
|
|
95
|
+
* underlying classify call.
|
|
96
|
+
*/
|
|
97
|
+
export interface RedactOptions extends ClassifyOptions {
|
|
98
|
+
/** How to replace each entity. Defaults to `'label'`. */
|
|
99
|
+
replacement?: Replacement;
|
|
100
|
+
/** If set, only entities whose label is in this list are redacted. */
|
|
101
|
+
labels?: PrivacyLabel[];
|
|
102
|
+
}
|
|
103
|
+
/** Result of {@link PrivacyFilter.redact}. */
|
|
104
|
+
export interface RedactResult {
|
|
105
|
+
/** The input text with each entity span replaced. */
|
|
106
|
+
redacted: string;
|
|
107
|
+
/** The entities that were actually redacted (after `labels` filter). */
|
|
108
|
+
entities: Entity[];
|
|
109
|
+
}
|
|
110
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH;;;;;GAKG;AACH,MAAM,MAAM,YAAY,GACpB,gBAAgB,GAChB,iBAAiB,GACjB,cAAc,GACd,eAAe,GACf,gBAAgB,GAChB,eAAe,GACf,aAAa,GACb,QAAQ,CAAC;AAEb;;;;;GAKG;AACH,MAAM,WAAW,MAAM;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,kBAAkB;IACjC,4BAA4B,EAAE,MAAM,CAAC;IACrC,+BAA+B,EAAE,MAAM,CAAC;IACxC,6BAA6B,EAAE,MAAM,CAAC;IACtC,wBAAwB,EAAE,MAAM,CAAC;IACjC,8BAA8B,EAAE,MAAM,CAAC;IACvC,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC1C,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED,gDAAgD;AAChD,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;CAClB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC;AAEhE;;;;GAIG;AACH,MAAM,WAAW,aAAc,SAAQ,eAAe;IACpD,yDAAyD;IACzD,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,sEAAsE;IACtE,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public types for the `@mlx-node/privacy` package.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the native NAPI surface ({@link import('@mlx-node/core').PrivacyEntity}
|
|
5
|
+
* etc.) but narrows the `label` field to the literal union of the 8 PII
|
|
6
|
+
* classes produced by the privacy-filter checkpoint, so consumers get
|
|
7
|
+
* autocomplete on `e.label === 'private_email'`.
|
|
8
|
+
*/
|
|
9
|
+
export {};
|
package/package.json
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@mlx-node/privacy",
|
|
3
|
+
"version": "0.0.0",
|
|
4
|
+
"homepage": "https://github.com/mlx-node/mlx-node",
|
|
5
|
+
"bugs": {
|
|
6
|
+
"url": "https://github.com/mlx-node/mlx-node/issues"
|
|
7
|
+
},
|
|
8
|
+
"license": "MIT",
|
|
9
|
+
"repository": {
|
|
10
|
+
"type": "git",
|
|
11
|
+
"url": "https://github.com/mlx-node/mlx-node.git",
|
|
12
|
+
"directory": "packages/privacy"
|
|
13
|
+
},
|
|
14
|
+
"files": [
|
|
15
|
+
"dist"
|
|
16
|
+
],
|
|
17
|
+
"type": "module",
|
|
18
|
+
"main": "./dist/index.js",
|
|
19
|
+
"types": "./dist/index.d.ts",
|
|
20
|
+
"exports": {
|
|
21
|
+
".": {
|
|
22
|
+
"types": "./dist/index.d.ts",
|
|
23
|
+
"import": "./dist/index.js"
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
"scripts": {
|
|
27
|
+
"build": "tsc -b",
|
|
28
|
+
"test": "vite test run"
|
|
29
|
+
},
|
|
30
|
+
"dependencies": {
|
|
31
|
+
"@mlx-node/core": "0.0.7"
|
|
32
|
+
},
|
|
33
|
+
"devDependencies": {
|
|
34
|
+
"@types/node": "^26.0.0"
|
|
35
|
+
}
|
|
36
|
+
}
|