@jinn-network/core 0.1.0-canary.3afae198

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/dist/canonical-json.d.ts +1 -0
  2. package/dist/canonical-json.js +67 -0
  3. package/dist/captured-task.d.ts +140 -0
  4. package/dist/captured-task.js +122 -0
  5. package/dist/contribution-store.d.ts +97 -0
  6. package/dist/contribution-store.js +770 -0
  7. package/dist/corpus-read/acquire.d.ts +39 -0
  8. package/dist/corpus-read/acquire.js +212 -0
  9. package/dist/corpus-read/cache.d.ts +14 -0
  10. package/dist/corpus-read/cache.js +17 -0
  11. package/dist/corpus-read/capture-meta.d.ts +16 -0
  12. package/dist/corpus-read/capture-meta.js +18 -0
  13. package/dist/corpus-read/create-corpus.d.ts +2 -0
  14. package/dist/corpus-read/create-corpus.js +97 -0
  15. package/dist/corpus-read/fetch-artifact.d.ts +21 -0
  16. package/dist/corpus-read/fetch-artifact.js +32 -0
  17. package/dist/corpus-read/fetch.d.ts +12 -0
  18. package/dist/corpus-read/fetch.js +24 -0
  19. package/dist/corpus-read/http-discovery.d.ts +9 -0
  20. package/dist/corpus-read/http-discovery.js +128 -0
  21. package/dist/corpus-read/index.d.ts +10 -0
  22. package/dist/corpus-read/index.js +10 -0
  23. package/dist/corpus-read/ipfs.d.ts +14 -0
  24. package/dist/corpus-read/ipfs.js +81 -0
  25. package/dist/corpus-read/route-resolver.d.ts +16 -0
  26. package/dist/corpus-read/route-resolver.js +19 -0
  27. package/dist/corpus-read/types.d.ts +177 -0
  28. package/dist/corpus-read/types.js +42 -0
  29. package/dist/envelope.d.ts +134 -0
  30. package/dist/envelope.js +162 -0
  31. package/dist/evidence-adapter.d.ts +26 -0
  32. package/dist/evidence-adapter.js +321 -0
  33. package/dist/evidence-filesystem.d.ts +40 -0
  34. package/dist/evidence-filesystem.js +267 -0
  35. package/dist/evidence-index.d.ts +117 -0
  36. package/dist/evidence-index.js +1083 -0
  37. package/dist/evidence-store-lock.d.ts +2 -0
  38. package/dist/evidence-store-lock.js +208 -0
  39. package/dist/execution-envelope.d.ts +3142 -0
  40. package/dist/execution-envelope.js +203 -0
  41. package/dist/index.d.ts +12 -0
  42. package/dist/index.js +12 -0
  43. package/dist/manifest.d.ts +75 -0
  44. package/dist/manifest.js +184 -0
  45. package/dist/paired.d.ts +68 -0
  46. package/dist/paired.js +92 -0
  47. package/dist/scrub/apply-dispositions.d.ts +74 -0
  48. package/dist/scrub/apply-dispositions.js +309 -0
  49. package/dist/scrub/build.d.ts +75 -0
  50. package/dist/scrub/build.js +143 -0
  51. package/dist/scrub/checksummed-instruments-detector.d.ts +13 -0
  52. package/dist/scrub/checksummed-instruments-detector.js +119 -0
  53. package/dist/scrub/data/bip39-english.d.ts +2 -0
  54. package/dist/scrub/data/bip39-english.js +2051 -0
  55. package/dist/scrub/data/gitleaks-rules.d.ts +56 -0
  56. package/dist/scrub/data/gitleaks-rules.js +67 -0
  57. package/dist/scrub/emit-scrub.d.ts +52 -0
  58. package/dist/scrub/emit-scrub.js +122 -0
  59. package/dist/scrub/eval/findings-from-scrub.d.ts +18 -0
  60. package/dist/scrub/eval/findings-from-scrub.js +99 -0
  61. package/dist/scrub/eval/fixtures.d.ts +14 -0
  62. package/dist/scrub/eval/fixtures.js +159 -0
  63. package/dist/scrub/eval/index.d.ts +6 -0
  64. package/dist/scrub/eval/index.js +6 -0
  65. package/dist/scrub/eval/local-corpus.d.ts +15 -0
  66. package/dist/scrub/eval/local-corpus.js +29 -0
  67. package/dist/scrub/eval/metrics.d.ts +19 -0
  68. package/dist/scrub/eval/metrics.js +77 -0
  69. package/dist/scrub/eval/run-bench.d.ts +8 -0
  70. package/dist/scrub/eval/run-bench.js +125 -0
  71. package/dist/scrub/eval/types.d.ts +69 -0
  72. package/dist/scrub/eval/types.js +7 -0
  73. package/dist/scrub/finding.d.ts +38 -0
  74. package/dist/scrub/finding.js +7 -0
  75. package/dist/scrub/git-identity-detector.d.ts +16 -0
  76. package/dist/scrub/git-identity-detector.js +116 -0
  77. package/dist/scrub/gitleaks-detector.d.ts +13 -0
  78. package/dist/scrub/gitleaks-detector.js +45 -0
  79. package/dist/scrub/gliner-detector.d.ts +65 -0
  80. package/dist/scrub/gliner-detector.js +129 -0
  81. package/dist/scrub/index.d.ts +26 -0
  82. package/dist/scrub/index.js +26 -0
  83. package/dist/scrub/ip-address-detector.d.ts +12 -0
  84. package/dist/scrub/ip-address-detector.js +81 -0
  85. package/dist/scrub/key-policy.d.ts +30 -0
  86. package/dist/scrub/key-policy.js +72 -0
  87. package/dist/scrub/known-identity-detector.d.ts +66 -0
  88. package/dist/scrub/known-identity-detector.js +284 -0
  89. package/dist/scrub/layer2.d.ts +18 -0
  90. package/dist/scrub/layer2.js +28 -0
  91. package/dist/scrub/ml-pii-stage.d.ts +42 -0
  92. package/dist/scrub/ml-pii-stage.js +119 -0
  93. package/dist/scrub/pii-build.d.ts +47 -0
  94. package/dist/scrub/pii-build.js +67 -0
  95. package/dist/scrub/pipeline.d.ts +77 -0
  96. package/dist/scrub/pipeline.js +225 -0
  97. package/dist/scrub/plain-patterns-stage.d.ts +33 -0
  98. package/dist/scrub/plain-patterns-stage.js +88 -0
  99. package/dist/scrub/policy.d.ts +28 -0
  100. package/dist/scrub/policy.js +156 -0
  101. package/dist/scrub/provenance.d.ts +53 -0
  102. package/dist/scrub/provenance.js +83 -0
  103. package/dist/scrub/reject-classes-detector.d.ts +11 -0
  104. package/dist/scrub/reject-classes-detector.js +195 -0
  105. package/dist/scrub/reject-publish-error.d.ts +30 -0
  106. package/dist/scrub/reject-publish-error.js +47 -0
  107. package/dist/scrub/review-queue.d.ts +90 -0
  108. package/dist/scrub/review-queue.js +259 -0
  109. package/dist/scrub/secretlint-stage.d.ts +21 -0
  110. package/dist/scrub/secretlint-stage.js +264 -0
  111. package/dist/scrub/transformers-detector.d.ts +41 -0
  112. package/dist/scrub/transformers-detector.js +77 -0
  113. package/dist/scrub/types.d.ts +36 -0
  114. package/dist/scrub/types.js +1 -0
  115. package/dist/scrub/url-credentials-detector.d.ts +9 -0
  116. package/dist/scrub/url-credentials-detector.js +66 -0
  117. package/dist/session-provenance.d.ts +75 -0
  118. package/dist/session-provenance.js +20 -0
  119. package/dist/skill-artifact.d.ts +442 -0
  120. package/dist/skill-artifact.js +136 -0
  121. package/dist/trajectory/hash-chain.d.ts +18 -0
  122. package/dist/trajectory/hash-chain.js +47 -0
  123. package/dist/trajectory/index.d.ts +4 -0
  124. package/dist/trajectory/index.js +4 -0
  125. package/dist/trajectory/schema.d.ts +681 -0
  126. package/dist/trajectory/schema.js +119 -0
  127. package/dist/trajectory/transcript-parsers/aider-history.d.ts +42 -0
  128. package/dist/trajectory/transcript-parsers/aider-history.js +181 -0
  129. package/dist/trajectory/transcript-parsers/claude-code-jsonl.d.ts +37 -0
  130. package/dist/trajectory/transcript-parsers/claude-code-jsonl.js +142 -0
  131. package/dist/trajectory/transcript-parsers/codex-session.d.ts +42 -0
  132. package/dist/trajectory/transcript-parsers/codex-session.js +235 -0
  133. package/dist/trajectory/transcript-parsers/continue-devdata.d.ts +41 -0
  134. package/dist/trajectory/transcript-parsers/continue-devdata.js +190 -0
  135. package/dist/trajectory/transcript-parsers/cursor-sqlite.d.ts +38 -0
  136. package/dist/trajectory/transcript-parsers/cursor-sqlite.js +118 -0
  137. package/dist/trajectory/transcript-parsers/gemini-session.d.ts +35 -0
  138. package/dist/trajectory/transcript-parsers/gemini-session.js +137 -0
  139. package/dist/trajectory/transcript-parsers/index.d.ts +7 -0
  140. package/dist/trajectory/transcript-parsers/index.js +7 -0
  141. package/dist/trajectory/transcript-parsers/types.d.ts +101 -0
  142. package/dist/trajectory/transcript-parsers/types.js +32 -0
  143. package/dist/trajectory/transcript-to-spans/attrs.d.ts +4 -0
  144. package/dist/trajectory/transcript-to-spans/attrs.js +30 -0
  145. package/dist/trajectory/transcript-to-spans/claude-code-stream-json.d.ts +8 -0
  146. package/dist/trajectory/transcript-to-spans/claude-code-stream-json.js +162 -0
  147. package/dist/trajectory/transcript-to-spans/codex-exec-json.d.ts +8 -0
  148. package/dist/trajectory/transcript-to-spans/codex-exec-json.js +253 -0
  149. package/dist/trajectory/transcript-to-spans/index.d.ts +4 -0
  150. package/dist/trajectory/transcript-to-spans/index.js +4 -0
  151. package/dist/trajectory/transcript-to-spans/types.d.ts +18 -0
  152. package/dist/trajectory/transcript-to-spans/types.js +1 -0
  153. package/dist/window.d.ts +12 -0
  154. package/dist/window.js +5 -0
  155. package/package.json +72 -0
@@ -0,0 +1,284 @@
1
+ /**
2
+ * Known-identity pack + non-address instance allowlist (#1971 / design §6.4).
3
+ *
4
+ * Locked Q1: stub ALL address-shaped values (no address pass-allowlist). The
5
+ * pack exact-matches operator self-PII only. The allowlist covers loopback /
6
+ * reserved IPs and repo-slug patterns — never wallets.
7
+ */
8
+ import { applyDispositions } from './apply-dispositions.js';
9
+ import { classifyKey } from './key-policy.js';
10
+ const VERSION = '0.1.0';
11
+ const ETH_ADDRESS_RE = /^0x[a-fA-F0-9]{40}$/;
12
+ const IPV4_RE = /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g;
13
+ /** Built-in non-address public values (Q1): loopback + common reserved. */
14
+ export const DEFAULT_INSTANCE_ALLOWLIST = {
15
+ entries: [
16
+ { value: '127.0.0.1', kind: 'loopback-ip', provenance: 'RFC 1122 loopback' },
17
+ { value: '0.0.0.0', kind: 'reserved-ip', provenance: 'RFC 1122 this-host' },
18
+ { value: '::1', kind: 'loopback-ip', provenance: 'IPv6 loopback' },
19
+ ],
20
+ };
21
+ const PACK_FIELDS = [
22
+ { key: 'gitUserName', scrubClass: 'B3', evidence: 'known-identity:git-user-name' },
23
+ { key: 'gitUserEmail', scrubClass: 'B1', evidence: 'known-identity:git-user-email' },
24
+ { key: 'homeUsername', scrubClass: 'B4', evidence: 'known-identity:home-username' },
25
+ { key: 'hostname', scrubClass: 'D3', evidence: 'known-identity:hostname' },
26
+ { key: 'ghLogin', scrubClass: 'B4', evidence: 'known-identity:gh-login' },
27
+ ];
28
+ export function isAddressShaped(value) {
29
+ return ETH_ADDRESS_RE.test(value.trim());
30
+ }
31
+ function normalizeToken(value) {
32
+ if (value === undefined)
33
+ return undefined;
34
+ const trimmed = value.trim();
35
+ if (!trimmed)
36
+ return undefined;
37
+ return trimmed;
38
+ }
39
+ function normalizePack(pack) {
40
+ const out = {};
41
+ for (const field of PACK_FIELDS) {
42
+ const v = normalizeToken(pack[field.key]);
43
+ if (!v)
44
+ continue;
45
+ // Q1: never put addresses into the redact pack.
46
+ if (isAddressShaped(v))
47
+ continue;
48
+ out[field.key] = field.key === 'gitUserEmail' ? v.toLowerCase() : v;
49
+ }
50
+ return out;
51
+ }
52
+ function filterAllowlist(entries) {
53
+ const seen = new Set();
54
+ const out = [];
55
+ for (const entry of entries) {
56
+ const value = normalizeToken(entry.value);
57
+ if (!value)
58
+ continue;
59
+ // Q1: no address pass-allowlist.
60
+ if (isAddressShaped(value))
61
+ continue;
62
+ const key = `${entry.kind}:${value.toLowerCase()}`;
63
+ if (seen.has(key))
64
+ continue;
65
+ seen.add(key);
66
+ out.push({ ...entry, value });
67
+ }
68
+ return out;
69
+ }
70
+ /**
71
+ * Assemble pack + allowlist from injected options, optionally filling gaps from
72
+ * env/homedir. Injectable pack is enough for this PR; daemon wiring stays thin.
73
+ */
74
+ export function assembleKnownIdentity(opts = {}) {
75
+ const env = opts.env ?? {};
76
+ const pack = { ...(opts.pack ?? {}) };
77
+ if (!pack.gitUserName && env.GIT_AUTHOR_NAME) {
78
+ pack.gitUserName = env.GIT_AUTHOR_NAME;
79
+ }
80
+ if (!pack.gitUserEmail && env.GIT_AUTHOR_EMAIL) {
81
+ pack.gitUserEmail = env.GIT_AUTHOR_EMAIL;
82
+ }
83
+ if (!pack.ghLogin && (env.GH_USER || env.GITHUB_USER)) {
84
+ pack.ghLogin = env.GH_USER || env.GITHUB_USER;
85
+ }
86
+ if (!pack.homeUsername && opts.homedir) {
87
+ try {
88
+ const home = opts.homedir();
89
+ const base = home.split(/[/\\]/).filter(Boolean).pop();
90
+ if (base)
91
+ pack.homeUsername = base;
92
+ }
93
+ catch {
94
+ // ignore — injectable pack is the primary path
95
+ }
96
+ }
97
+ if (!pack.hostname && opts.hostname) {
98
+ try {
99
+ pack.hostname = opts.hostname();
100
+ }
101
+ catch {
102
+ // ignore
103
+ }
104
+ }
105
+ const entries = [
106
+ ...DEFAULT_INSTANCE_ALLOWLIST.entries,
107
+ ...(opts.allowlist?.entries ?? []),
108
+ ];
109
+ return {
110
+ pack: normalizePack(pack),
111
+ allowlist: { entries: filterAllowlist(entries) },
112
+ };
113
+ }
114
+ function isLoopbackOrReservedIpv4(ip) {
115
+ const parts = ip.split('.').map((p) => Number(p));
116
+ if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n)))
117
+ return false;
118
+ const [a, b] = parts;
119
+ if (a === 127)
120
+ return true; // loopback
121
+ if (a === 0)
122
+ return true; // this-host
123
+ if (a === 169 && b === 254)
124
+ return true; // link-local
125
+ if (a >= 224)
126
+ return true; // multicast / reserved
127
+ return false;
128
+ }
129
+ function isPrivateIpv4(ip) {
130
+ const parts = ip.split('.').map((p) => Number(p));
131
+ if (parts.length !== 4 || parts.some((n) => !Number.isFinite(n)))
132
+ return false;
133
+ const [a, b] = parts;
134
+ if (a === 10)
135
+ return true;
136
+ if (a === 172 && b >= 16 && b <= 31)
137
+ return true;
138
+ if (a === 192 && b === 168)
139
+ return true;
140
+ return false;
141
+ }
142
+ function escapeRegExp(s) {
143
+ return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
144
+ }
145
+ /**
146
+ * Exact match with word-ish boundaries so `ann` does not hit `canonical`.
147
+ * Multi-word needles match the full phrase.
148
+ */
149
+ function findExactSpans(text, needle) {
150
+ if (needle.length < 2)
151
+ return [];
152
+ const re = new RegExp(`(?<![A-Za-z0-9_])${escapeRegExp(needle)}(?![A-Za-z0-9_])`, 'gi');
153
+ const spans = [];
154
+ let match;
155
+ while ((match = re.exec(text)) !== null) {
156
+ spans.push({ start: match.index, end: match.index + match[0].length });
157
+ }
158
+ return spans;
159
+ }
160
+ function allowlistHit(matched, allowlist) {
161
+ const lower = matched.toLowerCase();
162
+ return allowlist.entries.find((e) => e.value.toLowerCase() === lower);
163
+ }
164
+ function overlaps(a, b) {
165
+ return a.start < b.end && a.end > b.start;
166
+ }
167
+ /**
168
+ * Detector: pack → VERY_HIGH redact findings; allowlist → pass findings with
169
+ * auditable evidence; public IPs → D2 redact; private IPs → D2 flag band.
170
+ */
171
+ export function knownIdentityDetector(policy, opts = {}) {
172
+ const assembled = opts.assembled ??
173
+ assembleKnownIdentity({ pack: opts.pack, allowlist: opts.allowlist });
174
+ const meta = { name: 'known-identity', version: VERSION };
175
+ return {
176
+ ...meta,
177
+ detect(attributes) {
178
+ const findings = [];
179
+ const allowlistSpansByKey = new Map();
180
+ for (const [key, value] of Object.entries(attributes)) {
181
+ if (typeof value !== 'string' || classifyKey(key, policy) !== 'content')
182
+ continue;
183
+ const keyAllowlistSpans = [];
184
+ // --- Instance allowlist exact matches (pass + manifest) ---
185
+ for (const entry of assembled.allowlist.entries) {
186
+ for (const span of findExactSpans(value, entry.value)) {
187
+ keyAllowlistSpans.push(span);
188
+ findings.push({
189
+ class: entry.kind === 'repo-slug' ? 'B4' : 'D2',
190
+ span: { key, start: span.start, end: span.end },
191
+ confidence: 'VERY_LOW',
192
+ evidence: [`allowlist:${entry.kind}`, `provenance:${entry.provenance}`],
193
+ detector: meta,
194
+ });
195
+ }
196
+ }
197
+ // --- IPv4 shapes (D2), classified against allowlist / ranges ---
198
+ const ipRe = new RegExp(IPV4_RE.source, 'g');
199
+ let ipMatch;
200
+ while ((ipMatch = ipRe.exec(value)) !== null) {
201
+ const ip = ipMatch[0];
202
+ const span = { start: ipMatch.index, end: ipMatch.index + ip.length };
203
+ const listed = allowlistHit(ip, assembled.allowlist);
204
+ if (listed || isLoopbackOrReservedIpv4(ip)) {
205
+ if (!keyAllowlistSpans.some((s) => overlaps(s, span))) {
206
+ keyAllowlistSpans.push(span);
207
+ const kind = listed?.kind
208
+ ?? (ip.startsWith('127.') || ip === '0.0.0.0' ? 'loopback-ip' : 'reserved-ip');
209
+ const provenance = listed?.provenance
210
+ ?? (isLoopbackOrReservedIpv4(ip) ? 'RFC reserved/loopback' : 'allowlist');
211
+ findings.push({
212
+ class: 'D2',
213
+ span: { key, start: span.start, end: span.end },
214
+ confidence: 'VERY_LOW',
215
+ evidence: [`allowlist:${kind}`, `provenance:${provenance}`],
216
+ detector: meta,
217
+ });
218
+ }
219
+ continue;
220
+ }
221
+ if (isPrivateIpv4(ip)) {
222
+ findings.push({
223
+ class: 'D2',
224
+ span: { key, start: span.start, end: span.end },
225
+ confidence: 'MEDIUM',
226
+ evidence: ['ip-address', 'private-range'],
227
+ detector: meta,
228
+ });
229
+ continue;
230
+ }
231
+ findings.push({
232
+ class: 'D2',
233
+ span: { key, start: span.start, end: span.end },
234
+ confidence: 'VERY_HIGH',
235
+ evidence: ['ip-address'],
236
+ detector: meta,
237
+ });
238
+ }
239
+ allowlistSpansByKey.set(key, keyAllowlistSpans);
240
+ // --- Known-identity pack exact matches ---
241
+ for (const field of PACK_FIELDS) {
242
+ const needle = assembled.pack[field.key];
243
+ if (!needle)
244
+ continue;
245
+ // Allowlist suppresses pack hits on the same span (e.g. slug == handle).
246
+ for (const span of findExactSpans(value, needle)) {
247
+ if (keyAllowlistSpans.some((s) => overlaps(s, span)))
248
+ continue;
249
+ findings.push({
250
+ class: field.scrubClass,
251
+ span: { key, start: span.start, end: span.end },
252
+ confidence: 'VERY_HIGH',
253
+ evidence: [field.evidence],
254
+ detector: meta,
255
+ });
256
+ }
257
+ }
258
+ }
259
+ // Drop non-allowlist findings that overlap an allowlisted span (same key).
260
+ // Never suppress C1 — addresses are not allowlisted (Q1); this only covers
261
+ // pack/D2 collisions on non-address public values.
262
+ return findings.filter((f) => {
263
+ if (f.evidence.some((e) => e.startsWith('allowlist:')))
264
+ return true;
265
+ if (f.class === 'C1')
266
+ return true;
267
+ const spans = allowlistSpansByKey.get(f.span.key) ?? [];
268
+ return !spans.some((s) => overlaps(s, f.span));
269
+ });
270
+ },
271
+ };
272
+ }
273
+ /** Legacy ScrubStage wrapper for stage-chain tests. */
274
+ export function knownIdentityStage(policy, opts = {}) {
275
+ const detector = knownIdentityDetector(policy, opts);
276
+ return {
277
+ name: detector.name,
278
+ version: detector.version,
279
+ scrub(attributes) {
280
+ const findings = detector.detect(attributes);
281
+ return applyDispositions(attributes, findings);
282
+ },
283
+ };
284
+ }
@@ -0,0 +1,18 @@
1
+ import { ScrubPipeline } from './pipeline.js';
2
+ import type { KeyPolicy } from './key-policy.js';
3
+ import type { AssembleKnownIdentityOptions, AssembledKnownIdentity } from './known-identity-detector.js';
4
+ export interface BuildLayer2ScrubPipelineOptions {
5
+ policy?: KeyPolicy;
6
+ knownIdentity?: AssembleKnownIdentityOptions | AssembledKnownIdentity;
7
+ }
8
+ /**
9
+ * Layer-2 / check-mode preset (#1969 / design §6.5).
10
+ *
11
+ * Same owned detector inventory as the seed preset; checkMode maps any non-pass
12
+ * disposition to reject — one mapping line, not a second pipeline. Entropy
13
+ * fallback stays ON (stricter net; a false positive costs one re-distill, never
14
+ * defaces published content).
15
+ *
16
+ * @deprecated Compatibility preset over the one inventory + policy table.
17
+ */
18
+ export declare function buildLayer2ScrubPipeline(policyOrOpts?: KeyPolicy | BuildLayer2ScrubPipelineOptions): ScrubPipeline;
@@ -0,0 +1,28 @@
1
+ import { ScrubPipeline } from './pipeline.js';
2
+ import { DEFAULT_KEY_POLICY, buildProvenanceExtras, resolveKnownIdentity, sharedDetectorInventory, } from './build.js';
3
+ import { DEFAULT_POLICY } from './policy.js';
4
+ /**
5
+ * Layer-2 / check-mode preset (#1969 / design §6.5).
6
+ *
7
+ * Same owned detector inventory as the seed preset; checkMode maps any non-pass
8
+ * disposition to reject — one mapping line, not a second pipeline. Entropy
9
+ * fallback stays ON (stricter net; a false positive costs one re-distill, never
10
+ * defaces published content).
11
+ *
12
+ * @deprecated Compatibility preset over the one inventory + policy table.
13
+ */
14
+ export function buildLayer2ScrubPipeline(policyOrOpts = DEFAULT_KEY_POLICY) {
15
+ const opts = policyOrOpts && 'safe' in policyOrOpts
16
+ ? { policy: policyOrOpts }
17
+ : policyOrOpts;
18
+ const policy = opts.policy ?? DEFAULT_KEY_POLICY;
19
+ const knownIdentity = resolveKnownIdentity(opts.knownIdentity);
20
+ return new ScrubPipeline(sharedDetectorInventory(policy, {
21
+ entropyFallback: true,
22
+ knownIdentity,
23
+ }), {
24
+ policy: DEFAULT_POLICY,
25
+ checkMode: true,
26
+ provenance: buildProvenanceExtras(knownIdentity),
27
+ });
28
+ }
@@ -0,0 +1,42 @@
1
+ import type { Band, Detector } from './finding.js';
2
+ import { type KeyPolicy } from './key-policy.js';
3
+ import type { ScrubStage } from './types.js';
4
+ /**
5
+ * A PII entity detected in text. When the detector supplies character offsets
6
+ * (`start`/`end`), those win; otherwise the stage recovers spans via whole-word
7
+ * search (Transformers.js NER fallback). Optional `score` maps to a Band.
8
+ */
9
+ export interface PiiEntity {
10
+ label: string;
11
+ text: string;
12
+ /** Inclusive start offset in the source string, when the detector provides it. */
13
+ start?: number;
14
+ /** Exclusive end offset in the source string, when the detector provides it. */
15
+ end?: number;
16
+ /** Model confidence in [0, 1]. Mapped via {@link scoreToBand}. */
17
+ score?: number;
18
+ }
19
+ /**
20
+ * ML PII detector seam. Production uses GLiNER ONNX (`gliner-detector.ts`);
21
+ * TransformersPiiDetector remains for tests. Kept behind an interface so stage
22
+ * logic is unit-tested without loading a model.
23
+ */
24
+ export interface PiiDetector {
25
+ detect(text: string): Promise<PiiEntity[]>;
26
+ }
27
+ /**
28
+ * Map a model score onto a DLP-style Band (#1973 / design §6.1).
29
+ *
30
+ * Thresholds chosen so B3 policy (VERY_HIGH → redact; HIGH/MEDIUM → flag)
31
+ * separates auto-redact names from review-queue candidates.
32
+ */
33
+ export declare function scoreToBand(score: number | undefined): Band;
34
+ /**
35
+ * ML PII detector — emits B3/B5/B6/… findings from NER/GLiNER entities.
36
+ * Prefers detector-supplied offsets; falls back to whole-word search.
37
+ */
38
+ export declare function mlPiiDetector(policy: KeyPolicy, detector: PiiDetector): Detector;
39
+ /**
40
+ * Legacy ScrubStage wrapper around {@link mlPiiDetector}.
41
+ */
42
+ export declare function mlPiiStage(policy: KeyPolicy, detector: PiiDetector): ScrubStage;
@@ -0,0 +1,119 @@
1
+ import { applyDispositions } from './apply-dispositions.js';
2
+ import { classifyKey } from './key-policy.js';
3
+ const VERSION = '0.3.0';
4
+ /**
5
+ * Map a model score onto a DLP-style Band (#1973 / design §6.1).
6
+ *
7
+ * Thresholds chosen so B3 policy (VERY_HIGH → redact; HIGH/MEDIUM → flag)
8
+ * separates auto-redact names from review-queue candidates.
9
+ */
10
+ export function scoreToBand(score) {
11
+ if (score === undefined || Number.isNaN(score))
12
+ return 'VERY_HIGH';
13
+ if (score >= 0.85)
14
+ return 'VERY_HIGH';
15
+ if (score >= 0.7)
16
+ return 'HIGH';
17
+ if (score >= 0.55)
18
+ return 'MEDIUM';
19
+ return 'LOW';
20
+ }
21
+ function escapeRegExp(s) {
22
+ return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
23
+ }
24
+ function classForLabel(label) {
25
+ const l = label.toUpperCase();
26
+ if (l === 'PER' || l === 'PERSON' || l.includes('PERSON') || l === 'NAME')
27
+ return 'B3';
28
+ if (l.includes('PHONE') || l.includes('TEL'))
29
+ return 'B5';
30
+ if (l.includes('EMAIL'))
31
+ return 'B1';
32
+ if (l.includes('LOC') || l.includes('ADDRESS') || l.includes('STREET'))
33
+ return 'B6';
34
+ if (l.includes('USER') || l.includes('HANDLE') || l.includes('USERNAME'))
35
+ return 'B4';
36
+ if (l.includes('IP'))
37
+ return 'D2';
38
+ if (l.includes('CARD') ||
39
+ l.includes('IBAN') ||
40
+ l.includes('SSN') ||
41
+ l.includes('SOCIAL SECURITY')) {
42
+ return 'B7';
43
+ }
44
+ return 'E1';
45
+ }
46
+ function offsetsValid(text, start, end) {
47
+ if (typeof start !== 'number' ||
48
+ typeof end !== 'number' ||
49
+ start < 0 ||
50
+ end > text.length ||
51
+ start >= end) {
52
+ return null;
53
+ }
54
+ return { start, end };
55
+ }
56
+ /**
57
+ * ML PII detector — emits B3/B5/B6/… findings from NER/GLiNER entities.
58
+ * Prefers detector-supplied offsets; falls back to whole-word search.
59
+ */
60
+ export function mlPiiDetector(policy, detector) {
61
+ const meta = { name: 'ml-pii', version: VERSION };
62
+ return {
63
+ ...meta,
64
+ async detect(attributes) {
65
+ const findings = [];
66
+ for (const [key, value] of Object.entries(attributes)) {
67
+ if (typeof value !== 'string' || classifyKey(key, policy) !== 'content')
68
+ continue;
69
+ for (const ent of await detector.detect(value)) {
70
+ if (!ent.text)
71
+ continue;
72
+ const confidence = scoreToBand(ent.score);
73
+ const evidence = [`ml:${ent.label}`];
74
+ if (typeof ent.score === 'number') {
75
+ evidence.push(`score:${ent.score.toFixed(3)}`);
76
+ }
77
+ const offsets = offsetsValid(value, ent.start, ent.end);
78
+ if (offsets) {
79
+ findings.push({
80
+ class: classForLabel(ent.label),
81
+ span: { key, start: offsets.start, end: offsets.end },
82
+ confidence,
83
+ evidence,
84
+ detector: meta,
85
+ });
86
+ continue;
87
+ }
88
+ // Offset-less fallback (Transformers.js NER).
89
+ const re = new RegExp(`\\b${escapeRegExp(ent.text)}\\b`, 'g');
90
+ let match;
91
+ while ((match = re.exec(value)) !== null) {
92
+ findings.push({
93
+ class: classForLabel(ent.label),
94
+ span: { key, start: match.index, end: match.index + match[0].length },
95
+ confidence,
96
+ evidence,
97
+ detector: meta,
98
+ });
99
+ }
100
+ }
101
+ }
102
+ return findings;
103
+ },
104
+ };
105
+ }
106
+ /**
107
+ * Legacy ScrubStage wrapper around {@link mlPiiDetector}.
108
+ */
109
+ export function mlPiiStage(policy, detector) {
110
+ const det = mlPiiDetector(policy, detector);
111
+ return {
112
+ name: det.name,
113
+ version: det.version,
114
+ async scrub(attributes) {
115
+ const findings = await det.detect(attributes);
116
+ return applyDispositions(attributes, findings);
117
+ },
118
+ };
119
+ }
@@ -0,0 +1,47 @@
1
+ import { type GlinerPiiDetectorOptions } from './gliner-detector.js';
2
+ import type { PiiDetector } from './ml-pii-stage.js';
3
+ export interface PiiDetectionConfig {
4
+ /**
5
+ * When true (default for publish lanes), build and warm the GLiNER detector.
6
+ * Explicit `false` disables the ML tier (deterministic detectors only).
7
+ * When enabled, a model-load failure fails closed at publish altitude.
8
+ */
9
+ enabled: boolean;
10
+ /**
11
+ * Optional GLiNER / ONNX model id. Defaults to
12
+ * {@link DEFAULT_GLINER_MODEL} (`urchade/gliner_multi_pii-v1`).
13
+ * Alternate: `knowledgator/gliner-pii-edge-v1.0` (benchmark before switching).
14
+ */
15
+ model?: string;
16
+ }
17
+ export interface PiiDetectorLoader extends PiiDetector {
18
+ init(): Promise<void>;
19
+ }
20
+ export interface PiiDetectorFactoryPort {
21
+ create(options: GlinerPiiDetectorOptions): PiiDetectorLoader;
22
+ }
23
+ /** Thrown by the fail-closed detector on every scrub call when ML PII was enabled but the model never loaded. */
24
+ export declare class MlPiiUnavailableError extends Error {
25
+ constructor(cause: string);
26
+ }
27
+ /**
28
+ * Builds the ML PII detector (GLiNER ONNX, in-process) when enabled, warming
29
+ * up the model. Default factory is {@link GlinerPiiDetector}; tests inject a
30
+ * mock factory so CI never hits the network.
31
+ *
32
+ * Failure altitude is **publish-time, not boot-time** (per
33
+ * `spec/2026-06-15-ts-trajectory-scrub-stack.md`, §"Failure posture": "if any
34
+ * stage errors or the model fails to load, the trajectory is **not**
35
+ * published"). Three cases:
36
+ *
37
+ * - Disabled (`enabled: false`): returns `undefined` — deterministic inventory
38
+ * only, no ML PII tier and no error.
39
+ * - Enabled and the model loads: returns the real detector.
40
+ * - Enabled but the model fails to load: does NOT throw at construction. Returns
41
+ * a fail-closed detector that hard-throws on every scrub call, so each affected
42
+ * publish aborts (fails closed — never publishes under-redacted/raw) while the
43
+ * rest of the daemon keeps running. A loud one-time boot warning is emitted so
44
+ * the degraded posture is visible in logs; the per-publish throw carries its own
45
+ * clear error so aborted publishes are never silent.
46
+ */
47
+ export declare function maybeBuildPiiDetector(cfg: PiiDetectionConfig, log?: (msg: string) => void, factory?: PiiDetectorFactoryPort): Promise<PiiDetector | undefined>;
@@ -0,0 +1,67 @@
1
+ import { GlinerPiiDetector, DEFAULT_GLINER_MODEL, } from './gliner-detector.js';
2
+ /** Thrown by the fail-closed detector on every scrub call when ML PII was enabled but the model never loaded. */
3
+ export class MlPiiUnavailableError extends Error {
4
+ constructor(cause) {
5
+ super('ML PII detection was enabled but the model failed to load; failing closed — ' +
6
+ `this trajectory is NOT published (disable captures.piiDetection or fix the model to publish): ${cause}`);
7
+ this.name = 'MlPiiUnavailableError';
8
+ }
9
+ }
10
+ /**
11
+ * A detector that hard-throws on every `detect()` call. Used when ML PII is
12
+ * enabled but the model failed to load: the failure is narrowed to **publish
13
+ * altitude** — the throw propagates through the ML scrub stage and aborts the
14
+ * one publish in flight (capture publish aborts; task-trajectory ref drops to
15
+ * null), so no raw trajectory is ever published. The daemon's other loops
16
+ * (earning, claim, engine, balance-topup) are unaffected because the failure
17
+ * is no longer raised at construction/boot.
18
+ */
19
+ function failClosedDetector(cause) {
20
+ return {
21
+ detect() {
22
+ return Promise.reject(new MlPiiUnavailableError(cause));
23
+ },
24
+ };
25
+ }
26
+ /**
27
+ * Builds the ML PII detector (GLiNER ONNX, in-process) when enabled, warming
28
+ * up the model. Default factory is {@link GlinerPiiDetector}; tests inject a
29
+ * mock factory so CI never hits the network.
30
+ *
31
+ * Failure altitude is **publish-time, not boot-time** (per
32
+ * `spec/2026-06-15-ts-trajectory-scrub-stack.md`, §"Failure posture": "if any
33
+ * stage errors or the model fails to load, the trajectory is **not**
34
+ * published"). Three cases:
35
+ *
36
+ * - Disabled (`enabled: false`): returns `undefined` — deterministic inventory
37
+ * only, no ML PII tier and no error.
38
+ * - Enabled and the model loads: returns the real detector.
39
+ * - Enabled but the model fails to load: does NOT throw at construction. Returns
40
+ * a fail-closed detector that hard-throws on every scrub call, so each affected
41
+ * publish aborts (fails closed — never publishes under-redacted/raw) while the
42
+ * rest of the daemon keeps running. A loud one-time boot warning is emitted so
43
+ * the degraded posture is visible in logs; the per-publish throw carries its own
44
+ * clear error so aborted publishes are never silent.
45
+ */
46
+ export async function maybeBuildPiiDetector(cfg, log = (m) => console.warn(m), factory = {
47
+ create(options) {
48
+ return new GlinerPiiDetector(options);
49
+ },
50
+ }) {
51
+ if (!cfg.enabled)
52
+ return undefined;
53
+ try {
54
+ const detector = factory.create(cfg.model ? { model: cfg.model } : { model: DEFAULT_GLINER_MODEL });
55
+ await detector.init();
56
+ log(`[scrub] ML PII detection (GLiNER ONNX, model=${cfg.model ?? DEFAULT_GLINER_MODEL}) enabled.`);
57
+ return detector;
58
+ }
59
+ catch (err) {
60
+ const message = err instanceof Error ? err.message : String(err);
61
+ log('[scrub] WARNING: ML PII detection is enabled but the model failed to load. ' +
62
+ 'Failing closed at publish altitude: trajectory/capture publishing will ABORT ' +
63
+ '(no raw trajectories are published) until this is resolved or captures.piiDetection ' +
64
+ `is disabled. The daemon's other loops continue running. Cause: ${message}`);
65
+ return failClosedDetector(message);
66
+ }
67
+ }