@jinn-network/core 0.1.0 → 0.1.1-canary.7b188db5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/corpus-read/ipfs.d.ts +10 -1
  2. package/dist/corpus-read/ipfs.js +14 -5
  3. package/dist/scrub/apply-dispositions.d.ts +74 -0
  4. package/dist/scrub/apply-dispositions.js +309 -0
  5. package/dist/scrub/build.d.ts +54 -31
  6. package/dist/scrub/build.js +114 -55
  7. package/dist/scrub/checksummed-instruments-detector.d.ts +13 -0
  8. package/dist/scrub/checksummed-instruments-detector.js +119 -0
  9. package/dist/scrub/data/bip39-english.d.ts +2 -0
  10. package/dist/scrub/data/bip39-english.js +2051 -0
  11. package/dist/scrub/data/gitleaks-rules.d.ts +56 -0
  12. package/dist/scrub/data/gitleaks-rules.js +67 -0
  13. package/dist/scrub/emit-scrub.d.ts +13 -2
  14. package/dist/scrub/emit-scrub.js +39 -4
  15. package/dist/scrub/eval/findings-from-scrub.d.ts +18 -0
  16. package/dist/scrub/eval/findings-from-scrub.js +99 -0
  17. package/dist/scrub/eval/fixtures.d.ts +14 -0
  18. package/dist/scrub/eval/fixtures.js +159 -0
  19. package/dist/scrub/eval/index.d.ts +6 -0
  20. package/dist/scrub/eval/index.js +6 -0
  21. package/dist/scrub/eval/local-corpus.d.ts +15 -0
  22. package/dist/scrub/eval/local-corpus.js +29 -0
  23. package/dist/scrub/eval/metrics.d.ts +19 -0
  24. package/dist/scrub/eval/metrics.js +77 -0
  25. package/dist/scrub/eval/run-bench.d.ts +8 -0
  26. package/dist/scrub/eval/run-bench.js +125 -0
  27. package/dist/scrub/eval/types.d.ts +69 -0
  28. package/dist/scrub/eval/types.js +7 -0
  29. package/dist/scrub/finding.d.ts +38 -0
  30. package/dist/scrub/finding.js +7 -0
  31. package/dist/scrub/git-identity-detector.d.ts +16 -0
  32. package/dist/scrub/git-identity-detector.js +116 -0
  33. package/dist/scrub/gitleaks-detector.d.ts +13 -0
  34. package/dist/scrub/gitleaks-detector.js +45 -0
  35. package/dist/scrub/gliner-detector.d.ts +65 -0
  36. package/dist/scrub/gliner-detector.js +129 -0
  37. package/dist/scrub/index.d.ts +15 -1
  38. package/dist/scrub/index.js +15 -1
  39. package/dist/scrub/ip-address-detector.d.ts +12 -0
  40. package/dist/scrub/ip-address-detector.js +81 -0
  41. package/dist/scrub/key-policy.d.ts +18 -8
  42. package/dist/scrub/key-policy.js +49 -19
  43. package/dist/scrub/known-identity-detector.d.ts +66 -0
  44. package/dist/scrub/known-identity-detector.js +284 -0
  45. package/dist/scrub/layer2.d.ts +13 -18
  46. package/dist/scrub/layer2.js +22 -26
  47. package/dist/scrub/ml-pii-stage.d.ts +28 -10
  48. package/dist/scrub/ml-pii-stage.js +100 -23
  49. package/dist/scrub/pii-build.d.ts +17 -8
  50. package/dist/scrub/pii-build.js +9 -9
  51. package/dist/scrub/pipeline.d.ts +63 -24
  52. package/dist/scrub/pipeline.js +180 -38
  53. package/dist/scrub/plain-patterns-stage.d.ts +21 -19
  54. package/dist/scrub/plain-patterns-stage.js +61 -57
  55. package/dist/scrub/policy.d.ts +28 -0
  56. package/dist/scrub/policy.js +156 -0
  57. package/dist/scrub/provenance.d.ts +53 -0
  58. package/dist/scrub/provenance.js +83 -0
  59. package/dist/scrub/reject-classes-detector.d.ts +11 -0
  60. package/dist/scrub/reject-classes-detector.js +195 -0
  61. package/dist/scrub/reject-publish-error.d.ts +30 -0
  62. package/dist/scrub/reject-publish-error.js +47 -0
  63. package/dist/scrub/review-queue.d.ts +90 -0
  64. package/dist/scrub/review-queue.js +259 -0
  65. package/dist/scrub/secretlint-stage.d.ts +11 -26
  66. package/dist/scrub/secretlint-stage.js +45 -30
  67. package/dist/scrub/transformers-detector.d.ts +15 -9
  68. package/dist/scrub/transformers-detector.js +20 -10
  69. package/dist/scrub/types.d.ts +11 -0
  70. package/dist/scrub/url-credentials-detector.d.ts +9 -0
  71. package/dist/scrub/url-credentials-detector.js +66 -0
  72. package/dist/trajectory/schema.d.ts +75 -0
  73. package/dist/trajectory/schema.js +9 -0
  74. package/package.json +8 -3
  75. package/dist/scrub/openredaction-stage.d.ts +0 -196
  76. package/dist/scrub/openredaction-stage.js +0 -270
@@ -0,0 +1,284 @@
1
+ /**
2
+ * Known-identity pack + non-address instance allowlist (#1971 / design §6.4).
3
+ *
4
+ * Locked Q1: stub ALL address-shaped values (no address pass-allowlist). The
5
+ * pack exact-matches operator self-PII only. The allowlist covers loopback /
6
+ * reserved IPs and repo-slug patterns — never wallets.
7
+ */
8
+ import { applyDispositions } from './apply-dispositions.js';
9
+ import { classifyKey } from './key-policy.js';
10
+ const VERSION = '0.1.0';
11
+ const ETH_ADDRESS_RE = /^0x[a-fA-F0-9]{40}$/;
12
+ const IPV4_RE = /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g;
13
+ /** Built-in non-address public values (Q1): loopback + common reserved. */
14
+ export const DEFAULT_INSTANCE_ALLOWLIST = {
15
+ entries: [
16
+ { value: '127.0.0.1', kind: 'loopback-ip', provenance: 'RFC 1122 loopback' },
17
+ { value: '0.0.0.0', kind: 'reserved-ip', provenance: 'RFC 1122 this-host' },
18
+ { value: '::1', kind: 'loopback-ip', provenance: 'IPv6 loopback' },
19
+ ],
20
+ };
21
+ const PACK_FIELDS = [
22
+ { key: 'gitUserName', scrubClass: 'B3', evidence: 'known-identity:git-user-name' },
23
+ { key: 'gitUserEmail', scrubClass: 'B1', evidence: 'known-identity:git-user-email' },
24
+ { key: 'homeUsername', scrubClass: 'B4', evidence: 'known-identity:home-username' },
25
+ { key: 'hostname', scrubClass: 'D3', evidence: 'known-identity:hostname' },
26
+ { key: 'ghLogin', scrubClass: 'B4', evidence: 'known-identity:gh-login' },
27
+ ];
28
+ export function isAddressShaped(value) {
29
+ return ETH_ADDRESS_RE.test(value.trim());
30
+ }
31
+ function normalizeToken(value) {
32
+ if (value === undefined)
33
+ return undefined;
34
+ const trimmed = value.trim();
35
+ if (!trimmed)
36
+ return undefined;
37
+ return trimmed;
38
+ }
39
+ function normalizePack(pack) {
40
+ const out = {};
41
+ for (const field of PACK_FIELDS) {
42
+ const v = normalizeToken(pack[field.key]);
43
+ if (!v)
44
+ continue;
45
+ // Q1: never put addresses into the redact pack.
46
+ if (isAddressShaped(v))
47
+ continue;
48
+ out[field.key] = field.key === 'gitUserEmail' ? v.toLowerCase() : v;
49
+ }
50
+ return out;
51
+ }
52
+ function filterAllowlist(entries) {
53
+ const seen = new Set();
54
+ const out = [];
55
+ for (const entry of entries) {
56
+ const value = normalizeToken(entry.value);
57
+ if (!value)
58
+ continue;
59
+ // Q1: no address pass-allowlist.
60
+ if (isAddressShaped(value))
61
+ continue;
62
+ const key = `${entry.kind}:${value.toLowerCase()}`;
63
+ if (seen.has(key))
64
+ continue;
65
+ seen.add(key);
66
+ out.push({ ...entry, value });
67
+ }
68
+ return out;
69
+ }
70
+ /**
71
+ * Assemble pack + allowlist from injected options, optionally filling gaps from
72
+ * env/homedir. Injectable pack is enough for this PR; daemon wiring stays thin.
73
+ */
74
+ export function assembleKnownIdentity(opts = {}) {
75
+ const env = opts.env ?? {};
76
+ const pack = { ...(opts.pack ?? {}) };
77
+ if (!pack.gitUserName && env.GIT_AUTHOR_NAME) {
78
+ pack.gitUserName = env.GIT_AUTHOR_NAME;
79
+ }
80
+ if (!pack.gitUserEmail && env.GIT_AUTHOR_EMAIL) {
81
+ pack.gitUserEmail = env.GIT_AUTHOR_EMAIL;
82
+ }
83
+ if (!pack.ghLogin && (env.GH_USER || env.GITHUB_USER)) {
84
+ pack.ghLogin = env.GH_USER || env.GITHUB_USER;
85
+ }
86
+ if (!pack.homeUsername && opts.homedir) {
87
+ try {
88
+ const home = opts.homedir();
89
+ const base = home.split(/[/\\]/).filter(Boolean).pop();
90
+ if (base)
91
+ pack.homeUsername = base;
92
+ }
93
+ catch {
94
+ // ignore — injectable pack is the primary path
95
+ }
96
+ }
97
+ if (!pack.hostname && opts.hostname) {
98
+ try {
99
+ pack.hostname = opts.hostname();
100
+ }
101
+ catch {
102
+ // ignore
103
+ }
104
+ }
105
+ const entries = [
106
+ ...DEFAULT_INSTANCE_ALLOWLIST.entries,
107
+ ...(opts.allowlist?.entries ?? []),
108
+ ];
109
+ return {
110
+ pack: normalizePack(pack),
111
+ allowlist: { entries: filterAllowlist(entries) },
112
+ };
113
+ }
114
+ function isLoopbackOrReservedIpv4(ip) {
115
+ const parts = ip.split('.').map((p) => Number(p));
116
+ if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n)))
117
+ return false;
118
+ const [a, b] = parts;
119
+ if (a === 127)
120
+ return true; // loopback
121
+ if (a === 0)
122
+ return true; // this-host
123
+ if (a === 169 && b === 254)
124
+ return true; // link-local
125
+ if (a >= 224)
126
+ return true; // multicast / reserved
127
+ return false;
128
+ }
129
+ function isPrivateIpv4(ip) {
130
+ const parts = ip.split('.').map((p) => Number(p));
131
+ if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n)))
132
+ return false;
133
+ const [a, b] = parts;
134
+ if (a === 10)
135
+ return true;
136
+ if (a === 172 && b >= 16 && b <= 31)
137
+ return true;
138
+ if (a === 192 && b === 168)
139
+ return true;
140
+ return false;
141
+ }
142
+ function escapeRegExp(s) {
143
+ return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
144
+ }
145
+ /**
146
+ * Exact match with word-ish boundaries so `ann` does not hit `canonical`.
147
+ * Multi-word needles match the full phrase.
148
+ */
149
+ function findExactSpans(text, needle) {
150
+ if (needle.length < 2)
151
+ return [];
152
+ const re = new RegExp(`(?<![A-Za-z0-9_])${escapeRegExp(needle)}(?![A-Za-z0-9_])`, 'gi');
153
+ const spans = [];
154
+ let match;
155
+ while ((match = re.exec(text)) !== null) {
156
+ spans.push({ start: match.index, end: match.index + match[0].length });
157
+ }
158
+ return spans;
159
+ }
160
+ function allowlistHit(matched, allowlist) {
161
+ const lower = matched.toLowerCase();
162
+ return allowlist.entries.find((e) => e.value.toLowerCase() === lower);
163
+ }
164
+ function overlaps(a, b) {
165
+ return a.start < b.end && a.end > b.start;
166
+ }
167
+ /**
168
+ * Detector: pack → VERY_HIGH redact findings; allowlist → pass findings with
169
+ * auditable evidence; public IPs → D2 redact; private IPs → D2 flag band.
170
+ */
171
+ export function knownIdentityDetector(policy, opts = {}) {
172
+ const assembled = opts.assembled ??
173
+ assembleKnownIdentity({ pack: opts.pack, allowlist: opts.allowlist });
174
+ const meta = { name: 'known-identity', version: VERSION };
175
+ return {
176
+ ...meta,
177
+ detect(attributes) {
178
+ const findings = [];
179
+ const allowlistSpansByKey = new Map();
180
+ for (const [key, value] of Object.entries(attributes)) {
181
+ if (typeof value !== 'string' || classifyKey(key, policy) !== 'content')
182
+ continue;
183
+ const keyAllowlistSpans = [];
184
+ // --- Instance allowlist exact matches (pass + manifest) ---
185
+ for (const entry of assembled.allowlist.entries) {
186
+ for (const span of findExactSpans(value, entry.value)) {
187
+ keyAllowlistSpans.push(span);
188
+ findings.push({
189
+ class: entry.kind === 'repo-slug' ? 'B4' : 'D2',
190
+ span: { key, start: span.start, end: span.end },
191
+ confidence: 'VERY_LOW',
192
+ evidence: [`allowlist:${entry.kind}`, `provenance:${entry.provenance}`],
193
+ detector: meta,
194
+ });
195
+ }
196
+ }
197
+ // --- IPv4 shapes (D2), classified against allowlist / ranges ---
198
+ const ipRe = new RegExp(IPV4_RE.source, 'g');
199
+ let ipMatch;
200
+ while ((ipMatch = ipRe.exec(value)) !== null) {
201
+ const ip = ipMatch[0];
202
+ const span = { start: ipMatch.index, end: ipMatch.index + ip.length };
203
+ const listed = allowlistHit(ip, assembled.allowlist);
204
+ if (listed || isLoopbackOrReservedIpv4(ip)) {
205
+ if (!keyAllowlistSpans.some((s) => overlaps(s, span))) {
206
+ keyAllowlistSpans.push(span);
207
+ const kind = listed?.kind
208
+ ?? (ip.startsWith('127.') || ip === '0.0.0.0' ? 'loopback-ip' : 'reserved-ip');
209
+ const provenance = listed?.provenance
210
+ ?? (isLoopbackOrReservedIpv4(ip) ? 'RFC reserved/loopback' : 'allowlist');
211
+ findings.push({
212
+ class: 'D2',
213
+ span: { key, start: span.start, end: span.end },
214
+ confidence: 'VERY_LOW',
215
+ evidence: [`allowlist:${kind}`, `provenance:${provenance}`],
216
+ detector: meta,
217
+ });
218
+ }
219
+ continue;
220
+ }
221
+ if (isPrivateIpv4(ip)) {
222
+ findings.push({
223
+ class: 'D2',
224
+ span: { key, start: span.start, end: span.end },
225
+ confidence: 'MEDIUM',
226
+ evidence: ['ip-address', 'private-range'],
227
+ detector: meta,
228
+ });
229
+ continue;
230
+ }
231
+ findings.push({
232
+ class: 'D2',
233
+ span: { key, start: span.start, end: span.end },
234
+ confidence: 'VERY_HIGH',
235
+ evidence: ['ip-address'],
236
+ detector: meta,
237
+ });
238
+ }
239
+ allowlistSpansByKey.set(key, keyAllowlistSpans);
240
+ // --- Known-identity pack exact matches ---
241
+ for (const field of PACK_FIELDS) {
242
+ const needle = assembled.pack[field.key];
243
+ if (!needle)
244
+ continue;
245
+ // Allowlist suppresses pack hits on the same span (e.g. slug == handle).
246
+ for (const span of findExactSpans(value, needle)) {
247
+ if (keyAllowlistSpans.some((s) => overlaps(s, span)))
248
+ continue;
249
+ findings.push({
250
+ class: field.scrubClass,
251
+ span: { key, start: span.start, end: span.end },
252
+ confidence: 'VERY_HIGH',
253
+ evidence: [field.evidence],
254
+ detector: meta,
255
+ });
256
+ }
257
+ }
258
+ }
259
+ // Drop non-allowlist findings that overlap an allowlisted span (same key).
260
+ // Never suppress C1 — addresses are not allowlisted (Q1); this only covers
261
+ // pack/D2 collisions on non-address public values.
262
+ return findings.filter((f) => {
263
+ if (f.evidence.some((e) => e.startsWith('allowlist:')))
264
+ return true;
265
+ if (f.class === 'C1')
266
+ return true;
267
+ const spans = allowlistSpansByKey.get(f.span.key) ?? [];
268
+ return !spans.some((s) => overlaps(s, f.span));
269
+ });
270
+ },
271
+ };
272
+ }
273
+ /** Legacy ScrubStage wrapper for stage-chain tests. */
274
+ export function knownIdentityStage(policy, opts = {}) {
275
+ const detector = knownIdentityDetector(policy, opts);
276
+ return {
277
+ name: detector.name,
278
+ version: detector.version,
279
+ scrub(attributes) {
280
+ const findings = detector.detect(attributes);
281
+ return applyDispositions(attributes, findings);
282
+ },
283
+ };
284
+ }
@@ -1,23 +1,18 @@
1
1
  import { ScrubPipeline } from './pipeline.js';
2
- import { type KeyPolicy } from './key-policy.js';
2
+ import type { KeyPolicy } from './key-policy.js';
3
+ import type { AssembleKnownIdentityOptions, AssembledKnownIdentity } from './known-identity-detector.js';
4
+ export interface BuildLayer2ScrubPipelineOptions {
5
+ policy?: KeyPolicy;
6
+ knownIdentity?: AssembleKnownIdentityOptions | AssembledKnownIdentity;
7
+ }
3
8
  /**
4
- * The layer-2 / public / derived-content scrub altitude
5
- * (spec/2026-07-06-distillation-v1.md §10, D6; reconciled 2026-07-07 with the
6
- * shipped #1409 seed profile).
9
+ * Layer-2 / check-mode preset (#1969 / design §6.5).
7
10
  *
8
- * Secret-only: the structural key policy + deterministic plain-patterns
9
- * (emails, home-dir paths, credential-ID prefixes — #1330/#1415) + the FULL
10
- * secretlint stage (including its Pass-2 entropy secret-shape fallback). It
11
- * deliberately DROPS openredaction and ML-PII — the stages whose PII
12
- * shape-matching and trigger-word behaviour deface ordinary public prose
13
- * (#1409).
11
+ * Same owned detector inventory as the seed preset; checkMode maps any non-pass
12
+ * disposition to reject — one mapping line, not a second pipeline. Entropy
13
+ * fallback stays ON (stricter net; a false positive costs one re-distill, never
14
+ * defaces published content).
14
15
  *
15
- * Mode note vs the seed profile (`buildSeedScrubPipeline`): seeds are
16
- * REDACT-mode (the scrubbed output is published, so prose must survive →
17
- * entropy fallback OFF there). This pipeline serves CHECK-mode consumers —
18
- * `distillClusters` REJECTS a skill on any scrub change rather than publishing
19
- * a redacted body — so a false positive costs one re-distill, never defaces
20
- * published content, and the entropy fallback stays ON as the stricter net for
21
- * a fresh secret surface (DR-2026-07-06 decision 3).
16
+ * @deprecated Compatibility preset over the one inventory + policy table.
22
17
  */
23
- export declare function buildLayer2ScrubPipeline(policy?: KeyPolicy): ScrubPipeline;
18
+ export declare function buildLayer2ScrubPipeline(policyOrOpts?: KeyPolicy | BuildLayer2ScrubPipelineOptions): ScrubPipeline;
@@ -1,32 +1,28 @@
1
1
  import { ScrubPipeline } from './pipeline.js';
2
- import { keyPolicyStage } from './key-policy.js';
3
- import { secretlintStage } from './secretlint-stage.js';
4
- import { plainPatternsStage } from './plain-patterns-stage.js';
5
- import { DEFAULT_KEY_POLICY } from './build.js';
2
+ import { DEFAULT_KEY_POLICY, buildProvenanceExtras, resolveKnownIdentity, sharedDetectorInventory, } from './build.js';
3
+ import { DEFAULT_POLICY } from './policy.js';
6
4
  /**
7
- * The layer-2 / public / derived-content scrub altitude
8
- * (spec/2026-07-06-distillation-v1.md §10, D6; reconciled 2026-07-07 with the
9
- * shipped #1409 seed profile).
5
+ * Layer-2 / check-mode preset (#1969 / design §6.5).
10
6
  *
11
- * Secret-only: the structural key policy + deterministic plain-patterns
12
- * (emails, home-dir paths, credential-ID prefixes — #1330/#1415) + the FULL
13
- * secretlint stage (including its Pass-2 entropy secret-shape fallback). It
14
- * deliberately DROPS openredaction and ML-PII — the stages whose PII
15
- * shape-matching and trigger-word behaviour deface ordinary public prose
16
- * (#1409).
7
+ * Same owned detector inventory as the seed preset; checkMode maps any non-pass
8
+ * disposition to reject — one mapping line, not a second pipeline. Entropy
9
+ * fallback stays ON (stricter net; a false positive costs one re-distill, never
10
+ * defaces published content).
17
11
  *
18
- * Mode note vs the seed profile (`buildSeedScrubPipeline`): seeds are
19
- * REDACT-mode (the scrubbed output is published, so prose must survive →
20
- * entropy fallback OFF there). This pipeline serves CHECK-mode consumers —
21
- * `distillClusters` REJECTS a skill on any scrub change rather than publishing
22
- * a redacted body — so a false positive costs one re-distill, never defaces
23
- * published content, and the entropy fallback stays ON as the stricter net for
24
- * a fresh secret surface (DR-2026-07-06 decision 3).
12
+ * @deprecated Compatibility preset over the one inventory + policy table.
25
13
  */
26
- export function buildLayer2ScrubPipeline(policy = DEFAULT_KEY_POLICY) {
27
- return new ScrubPipeline([
28
- keyPolicyStage(policy),
29
- plainPatternsStage(policy, { credentialIds: true }),
30
- secretlintStage(policy),
31
- ]);
14
+ export function buildLayer2ScrubPipeline(policyOrOpts = DEFAULT_KEY_POLICY) {
15
+ const opts = policyOrOpts && 'safe' in policyOrOpts
16
+ ? { policy: policyOrOpts }
17
+ : policyOrOpts;
18
+ const policy = opts.policy ?? DEFAULT_KEY_POLICY;
19
+ const knownIdentity = resolveKnownIdentity(opts.knownIdentity);
20
+ return new ScrubPipeline(sharedDetectorInventory(policy, {
21
+ entropyFallback: true,
22
+ knownIdentity,
23
+ }), {
24
+ policy: DEFAULT_POLICY,
25
+ checkMode: true,
26
+ provenance: buildProvenanceExtras(knownIdentity),
27
+ });
32
28
  }
@@ -1,24 +1,42 @@
1
+ import type { Band, Detector } from './finding.js';
1
2
  import { type KeyPolicy } from './key-policy.js';
2
3
  import type { ScrubStage } from './types.js';
3
- /** A PII entity detected in text: a label (PER/ORG/LOC/…) and the matched span text. */
4
+ /**
5
+ * A PII entity detected in text. When the detector supplies character offsets
6
+ * (`start`/`end`), those win; otherwise the stage recovers spans via whole-word
7
+ * search (Transformers.js NER fallback). Optional `score` maps to a Band.
8
+ */
4
9
  export interface PiiEntity {
5
10
  label: string;
6
11
  text: string;
12
+ /** Inclusive start offset in the source string, when the detector provides it. */
13
+ start?: number;
14
+ /** Exclusive end offset in the source string, when the detector provides it. */
15
+ end?: number;
16
+ /** Model confidence in [0, 1]. Mapped via {@link scoreToBand}. */
17
+ score?: number;
7
18
  }
8
19
  /**
9
- * ML PII detector seam. The production implementation wraps a Transformers.js
10
- * token-classification (NER) model; tests inject a deterministic fake. Kept
11
- * behind an interface so the stage logic is unit-tested without loading a model.
20
+ * ML PII detector seam. Production uses GLiNER ONNX (`gliner-detector.ts`);
21
+ * TransformersPiiDetector remains for tests. Kept behind an interface so stage
22
+ * logic is unit-tested without loading a model.
12
23
  */
13
24
  export interface PiiDetector {
14
25
  detect(text: string): Promise<PiiEntity[]>;
15
26
  }
16
27
  /**
17
- * ML PII stage. Runs the injected NER detector over each `content`-classified
18
- * string value and replaces whole-word occurrences of each detected entity with
19
- * `[PII:<label>]` (word-boundary matching so substrings aren't corrupted — e.g.
20
- * detecting "London" leaves "Londoner" intact). `safe` and non-string values
21
- * pass through untouched. The detector returns matched text rather than offsets
22
- * because the Transformers.js NER pipeline does not surface character offsets.
28
+ * Map a model score onto a DLP-style Band (#1973 / design §6.1).
29
+ *
30
+ * Thresholds chosen so B3 policy (VERY_HIGH → redact; HIGH/MEDIUM → flag)
31
+ * separates auto-redact names from review-queue candidates.
32
+ */
33
+ export declare function scoreToBand(score: number | undefined): Band;
34
+ /**
35
+ * ML PII detector — emits B3/B5/B6/… findings from NER/GLiNER entities.
36
+ * Prefers detector-supplied offsets; falls back to whole-word search.
37
+ */
38
+ export declare function mlPiiDetector(policy: KeyPolicy, detector: PiiDetector): Detector;
39
+ /**
40
+ * Legacy ScrubStage wrapper around {@link mlPiiDetector}.
23
41
  */
24
42
  export declare function mlPiiStage(policy: KeyPolicy, detector: PiiDetector): ScrubStage;
@@ -1,42 +1,119 @@
1
+ import { applyDispositions } from './apply-dispositions.js';
1
2
  import { classifyKey } from './key-policy.js';
2
- const VERSION = '0.1.0';
3
+ const VERSION = '0.3.0';
4
+ /**
5
+ * Map a model score onto a DLP-style Band (#1973 / design §6.1).
6
+ *
7
+ * Thresholds chosen so B3 policy (VERY_HIGH → redact; HIGH/MEDIUM → flag)
8
+ * separates auto-redact names from review-queue candidates.
9
+ */
10
+ export function scoreToBand(score) {
11
+ if (score === undefined || Number.isNaN(score))
12
+ return 'VERY_HIGH';
13
+ if (score >= 0.85)
14
+ return 'VERY_HIGH';
15
+ if (score >= 0.7)
16
+ return 'HIGH';
17
+ if (score >= 0.55)
18
+ return 'MEDIUM';
19
+ return 'LOW';
20
+ }
3
21
  function escapeRegExp(s) {
4
22
  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
5
23
  }
24
+ function classForLabel(label) {
25
+ const l = label.toUpperCase();
26
+ if (l === 'PER' || l === 'PERSON' || l.includes('PERSON') || l === 'NAME')
27
+ return 'B3';
28
+ if (l.includes('PHONE') || l.includes('TEL'))
29
+ return 'B5';
30
+ if (l.includes('EMAIL'))
31
+ return 'B1';
32
+ if (l.includes('LOC') || l.includes('ADDRESS') || l.includes('STREET'))
33
+ return 'B6';
34
+ if (l.includes('USER') || l.includes('HANDLE') || l.includes('USERNAME'))
35
+ return 'B4';
36
+ if (l.includes('IP'))
37
+ return 'D2';
38
+ if (l.includes('CARD') ||
39
+ l.includes('IBAN') ||
40
+ l.includes('SSN') ||
41
+ l.includes('SOCIAL SECURITY')) {
42
+ return 'B7';
43
+ }
44
+ return 'E1';
45
+ }
46
+ function offsetsValid(text, start, end) {
47
+ if (typeof start !== 'number' ||
48
+ typeof end !== 'number' ||
49
+ start < 0 ||
50
+ end > text.length ||
51
+ start >= end) {
52
+ return null;
53
+ }
54
+ return { start, end };
55
+ }
6
56
  /**
7
- * ML PII stage. Runs the injected NER detector over each `content`-classified
8
- * string value and replaces whole-word occurrences of each detected entity with
9
- * `[PII:<label>]` (word-boundary matching so substrings aren't corrupted — e.g.
10
- * detecting "London" leaves "Londoner" intact). `safe` and non-string values
11
- * pass through untouched. The detector returns matched text rather than offsets
12
- * because the Transformers.js NER pipeline does not surface character offsets.
57
+ * ML PII detector — emits B3/B5/B6/… findings from NER/GLiNER entities.
58
+ * Prefers detector-supplied offsets; falls back to whole-word search.
13
59
  */
14
- export function mlPiiStage(policy, detector) {
60
+ export function mlPiiDetector(policy, detector) {
61
+ const meta = { name: 'ml-pii', version: VERSION };
15
62
  return {
16
- name: 'ml-pii',
17
- version: VERSION,
18
- async scrub(attributes) {
19
- const out = {};
20
- const redactions = [];
63
+ ...meta,
64
+ async detect(attributes) {
65
+ const findings = [];
21
66
  for (const [key, value] of Object.entries(attributes)) {
22
- if (typeof value !== 'string' || classifyKey(key, policy) !== 'content') {
23
- out[key] = value;
67
+ if (typeof value !== 'string' || classifyKey(key, policy) !== 'content')
24
68
  continue;
25
- }
26
- let text = value;
27
69
  for (const ent of await detector.detect(value)) {
28
70
  if (!ent.text)
29
71
  continue;
72
+ const confidence = scoreToBand(ent.score);
73
+ const evidence = [`ml:${ent.label}`];
74
+ if (typeof ent.score === 'number') {
75
+ evidence.push(`score:${ent.score.toFixed(3)}`);
76
+ }
77
+ const offsets = offsetsValid(value, ent.start, ent.end);
78
+ if (offsets) {
79
+ findings.push({
80
+ class: classForLabel(ent.label),
81
+ span: { key, start: offsets.start, end: offsets.end },
82
+ confidence,
83
+ evidence,
84
+ detector: meta,
85
+ });
86
+ continue;
87
+ }
88
+ // Offset-less fallback (Transformers.js NER).
30
89
  const re = new RegExp(`\\b${escapeRegExp(ent.text)}\\b`, 'g');
31
- const replaced = text.replace(re, `[PII:${ent.label}]`);
32
- if (replaced !== text) {
33
- redactions.push({ key, stage: 'ml-pii', kind: 'pii', detail: ent.label });
34
- text = replaced;
90
+ let match;
91
+ while ((match = re.exec(value)) !== null) {
92
+ findings.push({
93
+ class: classForLabel(ent.label),
94
+ span: { key, start: match.index, end: match.index + match[0].length },
95
+ confidence,
96
+ evidence,
97
+ detector: meta,
98
+ });
35
99
  }
36
100
  }
37
- out[key] = text;
38
101
  }
39
- return { attributes: out, redactions };
102
+ return findings;
103
+ },
104
+ };
105
+ }
106
+ /**
107
+ * Legacy ScrubStage wrapper around {@link mlPiiDetector}.
108
+ */
109
+ export function mlPiiStage(policy, detector) {
110
+ const det = mlPiiDetector(policy, detector);
111
+ return {
112
+ name: det.name,
113
+ version: det.version,
114
+ async scrub(attributes) {
115
+ const findings = await det.detect(attributes);
116
+ return applyDispositions(attributes, findings);
40
117
  },
41
118
  };
42
119
  }
@@ -1,32 +1,41 @@
1
- import { type TransformersPiiDetectorOptions } from './transformers-detector.js';
1
+ import { type GlinerPiiDetectorOptions } from './gliner-detector.js';
2
2
  import type { PiiDetector } from './ml-pii-stage.js';
3
3
  export interface PiiDetectionConfig {
4
+ /**
5
+ * When true (default for publish lanes), build and warm the GLiNER detector.
6
+ * Explicit `false` disables the ML tier (deterministic detectors only).
7
+ * When enabled, a model-load failure fails closed at publish altitude.
8
+ */
4
9
  enabled: boolean;
5
- /** Optional Transformers.js NER model id (defaults to Xenova/bert-base-NER). */
10
+ /**
11
+ * Optional GLiNER / ONNX model id. Defaults to
12
+ * {@link DEFAULT_GLINER_MODEL} (`urchade/gliner_multi_pii-v1`).
13
+ * Alternate: `knowledgator/gliner-pii-edge-v1.0` (benchmark before switching).
14
+ */
6
15
  model?: string;
7
16
  }
8
17
  export interface PiiDetectorLoader extends PiiDetector {
9
18
  init(): Promise<void>;
10
19
  }
11
20
  export interface PiiDetectorFactoryPort {
12
- create(options: TransformersPiiDetectorOptions): PiiDetectorLoader;
21
+ create(options: GlinerPiiDetectorOptions): PiiDetectorLoader;
13
22
  }
14
23
  /** Thrown by the fail-closed detector on every scrub call when ML PII was enabled but the model never loaded. */
15
24
  export declare class MlPiiUnavailableError extends Error {
16
25
  constructor(cause: string);
17
26
  }
18
27
  /**
19
- * Builds the ML PII detector (Transformers.js NER, in-process) when enabled,
20
- * warming up the model.
28
+ * Builds the ML PII detector (GLiNER ONNX, in-process) when enabled, warming
29
+ * up the model. Default factory is {@link GlinerPiiDetector}; tests inject a
30
+ * mock factory so CI never hits the network.
21
31
  *
22
32
  * Failure altitude is **publish-time, not boot-time** (per
23
33
  * `spec/2026-06-15-ts-trajectory-scrub-stack.md`, §"Failure posture": "if any
24
34
  * stage errors or the model fails to load, the trajectory is **not**
25
35
  * published"). Three cases:
26
36
  *
27
- * - Disabled (the default): returns `undefined` — the scrub pipeline runs the
28
- * structural key policy + openredaction + secretlint/entropy stages only, with
29
- * no ML PII tier and no error. Exactly the legacy behaviour.
37
+ * - Disabled (`enabled: false`): returns `undefined` — deterministic inventory
38
+ * only, no ML PII tier and no error.
30
39
  * - Enabled and the model loads: returns the real detector.
31
40
  * - Enabled but the model fails to load: does NOT throw at construction. Returns
32
41
  * a fail-closed detector that hard-throws on every scrub call, so each affected