@jinn-network/core 0.1.0-canary.3afae198
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/canonical-json.d.ts +1 -0
- package/dist/canonical-json.js +67 -0
- package/dist/captured-task.d.ts +140 -0
- package/dist/captured-task.js +122 -0
- package/dist/contribution-store.d.ts +97 -0
- package/dist/contribution-store.js +770 -0
- package/dist/corpus-read/acquire.d.ts +39 -0
- package/dist/corpus-read/acquire.js +212 -0
- package/dist/corpus-read/cache.d.ts +14 -0
- package/dist/corpus-read/cache.js +17 -0
- package/dist/corpus-read/capture-meta.d.ts +16 -0
- package/dist/corpus-read/capture-meta.js +18 -0
- package/dist/corpus-read/create-corpus.d.ts +2 -0
- package/dist/corpus-read/create-corpus.js +97 -0
- package/dist/corpus-read/fetch-artifact.d.ts +21 -0
- package/dist/corpus-read/fetch-artifact.js +32 -0
- package/dist/corpus-read/fetch.d.ts +12 -0
- package/dist/corpus-read/fetch.js +24 -0
- package/dist/corpus-read/http-discovery.d.ts +9 -0
- package/dist/corpus-read/http-discovery.js +128 -0
- package/dist/corpus-read/index.d.ts +10 -0
- package/dist/corpus-read/index.js +10 -0
- package/dist/corpus-read/ipfs.d.ts +14 -0
- package/dist/corpus-read/ipfs.js +81 -0
- package/dist/corpus-read/route-resolver.d.ts +16 -0
- package/dist/corpus-read/route-resolver.js +19 -0
- package/dist/corpus-read/types.d.ts +177 -0
- package/dist/corpus-read/types.js +42 -0
- package/dist/envelope.d.ts +134 -0
- package/dist/envelope.js +162 -0
- package/dist/evidence-adapter.d.ts +26 -0
- package/dist/evidence-adapter.js +321 -0
- package/dist/evidence-filesystem.d.ts +40 -0
- package/dist/evidence-filesystem.js +267 -0
- package/dist/evidence-index.d.ts +117 -0
- package/dist/evidence-index.js +1083 -0
- package/dist/evidence-store-lock.d.ts +2 -0
- package/dist/evidence-store-lock.js +208 -0
- package/dist/execution-envelope.d.ts +3142 -0
- package/dist/execution-envelope.js +203 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.js +12 -0
- package/dist/manifest.d.ts +75 -0
- package/dist/manifest.js +184 -0
- package/dist/paired.d.ts +68 -0
- package/dist/paired.js +92 -0
- package/dist/scrub/apply-dispositions.d.ts +74 -0
- package/dist/scrub/apply-dispositions.js +309 -0
- package/dist/scrub/build.d.ts +75 -0
- package/dist/scrub/build.js +143 -0
- package/dist/scrub/checksummed-instruments-detector.d.ts +13 -0
- package/dist/scrub/checksummed-instruments-detector.js +119 -0
- package/dist/scrub/data/bip39-english.d.ts +2 -0
- package/dist/scrub/data/bip39-english.js +2051 -0
- package/dist/scrub/data/gitleaks-rules.d.ts +56 -0
- package/dist/scrub/data/gitleaks-rules.js +67 -0
- package/dist/scrub/emit-scrub.d.ts +52 -0
- package/dist/scrub/emit-scrub.js +122 -0
- package/dist/scrub/eval/findings-from-scrub.d.ts +18 -0
- package/dist/scrub/eval/findings-from-scrub.js +99 -0
- package/dist/scrub/eval/fixtures.d.ts +14 -0
- package/dist/scrub/eval/fixtures.js +159 -0
- package/dist/scrub/eval/index.d.ts +6 -0
- package/dist/scrub/eval/index.js +6 -0
- package/dist/scrub/eval/local-corpus.d.ts +15 -0
- package/dist/scrub/eval/local-corpus.js +29 -0
- package/dist/scrub/eval/metrics.d.ts +19 -0
- package/dist/scrub/eval/metrics.js +77 -0
- package/dist/scrub/eval/run-bench.d.ts +8 -0
- package/dist/scrub/eval/run-bench.js +125 -0
- package/dist/scrub/eval/types.d.ts +69 -0
- package/dist/scrub/eval/types.js +7 -0
- package/dist/scrub/finding.d.ts +38 -0
- package/dist/scrub/finding.js +7 -0
- package/dist/scrub/git-identity-detector.d.ts +16 -0
- package/dist/scrub/git-identity-detector.js +116 -0
- package/dist/scrub/gitleaks-detector.d.ts +13 -0
- package/dist/scrub/gitleaks-detector.js +45 -0
- package/dist/scrub/gliner-detector.d.ts +65 -0
- package/dist/scrub/gliner-detector.js +129 -0
- package/dist/scrub/index.d.ts +26 -0
- package/dist/scrub/index.js +26 -0
- package/dist/scrub/ip-address-detector.d.ts +12 -0
- package/dist/scrub/ip-address-detector.js +81 -0
- package/dist/scrub/key-policy.d.ts +30 -0
- package/dist/scrub/key-policy.js +72 -0
- package/dist/scrub/known-identity-detector.d.ts +66 -0
- package/dist/scrub/known-identity-detector.js +284 -0
- package/dist/scrub/layer2.d.ts +18 -0
- package/dist/scrub/layer2.js +28 -0
- package/dist/scrub/ml-pii-stage.d.ts +42 -0
- package/dist/scrub/ml-pii-stage.js +119 -0
- package/dist/scrub/pii-build.d.ts +47 -0
- package/dist/scrub/pii-build.js +67 -0
- package/dist/scrub/pipeline.d.ts +77 -0
- package/dist/scrub/pipeline.js +225 -0
- package/dist/scrub/plain-patterns-stage.d.ts +33 -0
- package/dist/scrub/plain-patterns-stage.js +88 -0
- package/dist/scrub/policy.d.ts +28 -0
- package/dist/scrub/policy.js +156 -0
- package/dist/scrub/provenance.d.ts +53 -0
- package/dist/scrub/provenance.js +83 -0
- package/dist/scrub/reject-classes-detector.d.ts +11 -0
- package/dist/scrub/reject-classes-detector.js +195 -0
- package/dist/scrub/reject-publish-error.d.ts +30 -0
- package/dist/scrub/reject-publish-error.js +47 -0
- package/dist/scrub/review-queue.d.ts +90 -0
- package/dist/scrub/review-queue.js +259 -0
- package/dist/scrub/secretlint-stage.d.ts +21 -0
- package/dist/scrub/secretlint-stage.js +264 -0
- package/dist/scrub/transformers-detector.d.ts +41 -0
- package/dist/scrub/transformers-detector.js +77 -0
- package/dist/scrub/types.d.ts +36 -0
- package/dist/scrub/types.js +1 -0
- package/dist/scrub/url-credentials-detector.d.ts +9 -0
- package/dist/scrub/url-credentials-detector.js +66 -0
- package/dist/session-provenance.d.ts +75 -0
- package/dist/session-provenance.js +20 -0
- package/dist/skill-artifact.d.ts +442 -0
- package/dist/skill-artifact.js +136 -0
- package/dist/trajectory/hash-chain.d.ts +18 -0
- package/dist/trajectory/hash-chain.js +47 -0
- package/dist/trajectory/index.d.ts +4 -0
- package/dist/trajectory/index.js +4 -0
- package/dist/trajectory/schema.d.ts +681 -0
- package/dist/trajectory/schema.js +119 -0
- package/dist/trajectory/transcript-parsers/aider-history.d.ts +42 -0
- package/dist/trajectory/transcript-parsers/aider-history.js +181 -0
- package/dist/trajectory/transcript-parsers/claude-code-jsonl.d.ts +37 -0
- package/dist/trajectory/transcript-parsers/claude-code-jsonl.js +142 -0
- package/dist/trajectory/transcript-parsers/codex-session.d.ts +42 -0
- package/dist/trajectory/transcript-parsers/codex-session.js +235 -0
- package/dist/trajectory/transcript-parsers/continue-devdata.d.ts +41 -0
- package/dist/trajectory/transcript-parsers/continue-devdata.js +190 -0
- package/dist/trajectory/transcript-parsers/cursor-sqlite.d.ts +38 -0
- package/dist/trajectory/transcript-parsers/cursor-sqlite.js +118 -0
- package/dist/trajectory/transcript-parsers/gemini-session.d.ts +35 -0
- package/dist/trajectory/transcript-parsers/gemini-session.js +137 -0
- package/dist/trajectory/transcript-parsers/index.d.ts +7 -0
- package/dist/trajectory/transcript-parsers/index.js +7 -0
- package/dist/trajectory/transcript-parsers/types.d.ts +101 -0
- package/dist/trajectory/transcript-parsers/types.js +32 -0
- package/dist/trajectory/transcript-to-spans/attrs.d.ts +4 -0
- package/dist/trajectory/transcript-to-spans/attrs.js +30 -0
- package/dist/trajectory/transcript-to-spans/claude-code-stream-json.d.ts +8 -0
- package/dist/trajectory/transcript-to-spans/claude-code-stream-json.js +162 -0
- package/dist/trajectory/transcript-to-spans/codex-exec-json.d.ts +8 -0
- package/dist/trajectory/transcript-to-spans/codex-exec-json.js +253 -0
- package/dist/trajectory/transcript-to-spans/index.d.ts +4 -0
- package/dist/trajectory/transcript-to-spans/index.js +4 -0
- package/dist/trajectory/transcript-to-spans/types.d.ts +18 -0
- package/dist/trajectory/transcript-to-spans/types.js +1 -0
- package/dist/window.d.ts +12 -0
- package/dist/window.js +5 -0
- package/package.json +72 -0
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Operator-local review queue for scrub `flag` dispositions (#1973 / design §6.6).
|
|
3
|
+
*
|
|
4
|
+
* Persistence: `~/.jinn-client/scrub-review/queue.jsonl` (one JSON object per
|
|
5
|
+
* line). Traces never leave the machine — this queue is local-only.
|
|
6
|
+
*
|
|
7
|
+
* Unattended publish lanes fail closed while any matching flag remains
|
|
8
|
+
* `pending` (see {@link assertNoUnresolvedFlags}).
|
|
9
|
+
*/
|
|
10
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
11
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { homedir } from 'node:os';
|
|
13
|
+
import { dirname, join } from 'node:path';
|
|
14
|
+
import { resolveDisposition } from './policy.js';
|
|
15
|
+
import { DEFAULT_POLICY } from './policy.js';
|
|
16
|
+
export const DEFAULT_REVIEW_QUEUE_PATH = join(homedir(), '.jinn-client', 'scrub-review', 'queue.jsonl');
|
|
17
|
+
/** Stable fingerprint for matching a finding across scrub runs. */
|
|
18
|
+
export function findingFingerprint(finding) {
|
|
19
|
+
const payload = [
|
|
20
|
+
finding.class,
|
|
21
|
+
finding.span.key,
|
|
22
|
+
String(finding.span.start),
|
|
23
|
+
String(finding.span.end),
|
|
24
|
+
finding.evidence.join('|'),
|
|
25
|
+
finding.detector.name,
|
|
26
|
+
].join('\0');
|
|
27
|
+
return createHash('sha256').update(payload).digest('hex').slice(0, 24);
|
|
28
|
+
}
|
|
29
|
+
function readAll(path) {
|
|
30
|
+
if (!existsSync(path))
|
|
31
|
+
return [];
|
|
32
|
+
const text = readFileSync(path, 'utf8');
|
|
33
|
+
const items = [];
|
|
34
|
+
for (const line of text.split('\n')) {
|
|
35
|
+
const trimmed = line.trim();
|
|
36
|
+
if (!trimmed)
|
|
37
|
+
continue;
|
|
38
|
+
try {
|
|
39
|
+
items.push(JSON.parse(trimmed));
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
// skip corrupt lines
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return items;
|
|
46
|
+
}
|
|
47
|
+
function writeAll(path, items) {
|
|
48
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
49
|
+
const body = items.map((i) => JSON.stringify(i)).join('\n');
|
|
50
|
+
writeFileSync(path, body.length > 0 ? `${body}\n` : '', 'utf8');
|
|
51
|
+
}
|
|
52
|
+
/** In-memory review queue for unit tests (no disk I/O). */
|
|
53
|
+
export function createMemoryReviewQueueStore(seed = []) {
|
|
54
|
+
let items = [...seed];
|
|
55
|
+
return {
|
|
56
|
+
listFlagged(filter) {
|
|
57
|
+
if (!filter?.status)
|
|
58
|
+
return [...items];
|
|
59
|
+
return items.filter((i) => i.status === filter.status);
|
|
60
|
+
},
|
|
61
|
+
enqueue(rawItems) {
|
|
62
|
+
const pendingFingerprints = new Set(items.filter((i) => i.status === 'pending').map((i) => findingFingerprint(i.finding)));
|
|
63
|
+
const created = [];
|
|
64
|
+
for (const raw of rawItems) {
|
|
65
|
+
const fp = findingFingerprint(raw.finding);
|
|
66
|
+
if (pendingFingerprints.has(fp))
|
|
67
|
+
continue;
|
|
68
|
+
const prior = [...items]
|
|
69
|
+
.reverse()
|
|
70
|
+
.find((i) => findingFingerprint(i.finding) === fp && i.status === 'resolved');
|
|
71
|
+
if (prior)
|
|
72
|
+
continue;
|
|
73
|
+
const item = {
|
|
74
|
+
id: randomUUID(),
|
|
75
|
+
finding: raw.finding,
|
|
76
|
+
context: raw.context,
|
|
77
|
+
createdAt: new Date().toISOString(),
|
|
78
|
+
status: 'pending',
|
|
79
|
+
};
|
|
80
|
+
items.push(item);
|
|
81
|
+
pendingFingerprints.add(fp);
|
|
82
|
+
created.push(item);
|
|
83
|
+
}
|
|
84
|
+
return created;
|
|
85
|
+
},
|
|
86
|
+
resolveFlag(id, decision) {
|
|
87
|
+
const idx = items.findIndex((i) => i.id === id);
|
|
88
|
+
if (idx < 0)
|
|
89
|
+
throw new Error(`review queue: unknown flag id ${id}`);
|
|
90
|
+
const item = items[idx];
|
|
91
|
+
if (item.status === 'resolved') {
|
|
92
|
+
throw new Error(`review queue: flag ${id} is already resolved`);
|
|
93
|
+
}
|
|
94
|
+
const updated = {
|
|
95
|
+
...item,
|
|
96
|
+
status: 'resolved',
|
|
97
|
+
resolution: { decision, resolvedAt: new Date().toISOString() },
|
|
98
|
+
};
|
|
99
|
+
items = items.map((i, iidx) => (iidx === idx ? updated : i));
|
|
100
|
+
return updated;
|
|
101
|
+
},
|
|
102
|
+
resolutionFor(finding) {
|
|
103
|
+
const fp = findingFingerprint(finding);
|
|
104
|
+
const match = [...items]
|
|
105
|
+
.reverse()
|
|
106
|
+
.find((i) => findingFingerprint(i.finding) === fp && i.status === 'resolved');
|
|
107
|
+
return match?.resolution?.decision;
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* File-backed review queue. Latest write wins per id; fingerprints look at the
|
|
113
|
+
* newest resolved entry.
|
|
114
|
+
*/
|
|
115
|
+
export function createReviewQueueStore(path = DEFAULT_REVIEW_QUEUE_PATH) {
|
|
116
|
+
return {
|
|
117
|
+
listFlagged(filter) {
|
|
118
|
+
const all = readAll(path);
|
|
119
|
+
if (!filter?.status)
|
|
120
|
+
return all;
|
|
121
|
+
return all.filter((i) => i.status === filter.status);
|
|
122
|
+
},
|
|
123
|
+
enqueue(rawItems) {
|
|
124
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
125
|
+
const existing = readAll(path);
|
|
126
|
+
const pendingFingerprints = new Set(existing
|
|
127
|
+
.filter((i) => i.status === 'pending')
|
|
128
|
+
.map((i) => findingFingerprint(i.finding)));
|
|
129
|
+
const created = [];
|
|
130
|
+
for (const raw of rawItems) {
|
|
131
|
+
const fp = findingFingerprint(raw.finding);
|
|
132
|
+
if (pendingFingerprints.has(fp))
|
|
133
|
+
continue;
|
|
134
|
+
// Skip if already resolved for this fingerprint.
|
|
135
|
+
const prior = [...existing]
|
|
136
|
+
.reverse()
|
|
137
|
+
.find((i) => findingFingerprint(i.finding) === fp && i.status === 'resolved');
|
|
138
|
+
if (prior)
|
|
139
|
+
continue;
|
|
140
|
+
const item = {
|
|
141
|
+
id: randomUUID(),
|
|
142
|
+
finding: raw.finding,
|
|
143
|
+
context: raw.context,
|
|
144
|
+
createdAt: new Date().toISOString(),
|
|
145
|
+
status: 'pending',
|
|
146
|
+
};
|
|
147
|
+
appendFileSync(path, `${JSON.stringify(item)}\n`, 'utf8');
|
|
148
|
+
pendingFingerprints.add(fp);
|
|
149
|
+
created.push(item);
|
|
150
|
+
}
|
|
151
|
+
return created;
|
|
152
|
+
},
|
|
153
|
+
resolveFlag(id, decision) {
|
|
154
|
+
const all = readAll(path);
|
|
155
|
+
const idx = all.findIndex((i) => i.id === id);
|
|
156
|
+
if (idx < 0) {
|
|
157
|
+
throw new Error(`review queue: unknown flag id ${id}`);
|
|
158
|
+
}
|
|
159
|
+
const item = all[idx];
|
|
160
|
+
if (item.status === 'resolved') {
|
|
161
|
+
throw new Error(`review queue: flag ${id} is already resolved`);
|
|
162
|
+
}
|
|
163
|
+
const updated = {
|
|
164
|
+
...item,
|
|
165
|
+
status: 'resolved',
|
|
166
|
+
resolution: { decision, resolvedAt: new Date().toISOString() },
|
|
167
|
+
};
|
|
168
|
+
all[idx] = updated;
|
|
169
|
+
writeAll(path, all);
|
|
170
|
+
return updated;
|
|
171
|
+
},
|
|
172
|
+
resolutionFor(finding) {
|
|
173
|
+
const fp = findingFingerprint(finding);
|
|
174
|
+
const match = [...readAll(path)]
|
|
175
|
+
.reverse()
|
|
176
|
+
.find((i) => findingFingerprint(i.finding) === fp && i.status === 'resolved');
|
|
177
|
+
return match?.resolution?.decision;
|
|
178
|
+
},
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
/** Convenience exports bound to the default operator-local path. */
|
|
182
|
+
const defaultStore = createReviewQueueStore();
|
|
183
|
+
export function listFlagged(filter) {
|
|
184
|
+
return defaultStore.listFlagged(filter);
|
|
185
|
+
}
|
|
186
|
+
export function resolveFlag(id, decision) {
|
|
187
|
+
return defaultStore.resolveFlag(id, decision);
|
|
188
|
+
}
|
|
189
|
+
export function enqueueFlags(items, store = defaultStore) {
|
|
190
|
+
return store.enqueue(items);
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Thrown when redact-mode scrub leaves unresolved `flag` dispositions.
|
|
194
|
+
* Unattended publish must not proceed over an open flag (design §6.5).
|
|
195
|
+
*/
|
|
196
|
+
export class UnresolvedFlagError extends Error {
|
|
197
|
+
findings;
|
|
198
|
+
queueIds;
|
|
199
|
+
constructor(findings, queueIds = []) {
|
|
200
|
+
const n = findings.length;
|
|
201
|
+
super(`unresolved-flag: ${n} scrub finding(s) require review — publish aborted ` +
|
|
202
|
+
`(run \`jinn scrub review\` to resolve; ids: ${queueIds.slice(0, 5).join(', ') || 'n/a'})`);
|
|
203
|
+
this.name = 'UnresolvedFlagError';
|
|
204
|
+
this.findings = findings;
|
|
205
|
+
this.queueIds = queueIds;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Apply review-queue resolutions to flag findings, enqueue new pending flags,
|
|
210
|
+
* and optionally fail closed.
|
|
211
|
+
*
|
|
212
|
+
* - `approve-instance` / allowlist / identity-pack → treat as pass (no throw)
|
|
213
|
+
* - `redact-instance` → caller should promote to redact (returned in `toRedact`)
|
|
214
|
+
* - unresolved → enqueue + throw when `failClosed`
|
|
215
|
+
*/
|
|
216
|
+
export function processFlagFindings(findings, opts = {}) {
|
|
217
|
+
const policy = opts.policy ?? DEFAULT_POLICY;
|
|
218
|
+
const store = opts.store ?? defaultStore;
|
|
219
|
+
const attributes = opts.attributes ?? {};
|
|
220
|
+
const flagged = findings.filter((f) => resolveDisposition(f.class, f.confidence, policy) === 'flag');
|
|
221
|
+
const unresolved = [];
|
|
222
|
+
const toRedact = [];
|
|
223
|
+
const toPass = [];
|
|
224
|
+
const toEnqueue = [];
|
|
225
|
+
for (const finding of flagged) {
|
|
226
|
+
const decision = store.resolutionFor(finding);
|
|
227
|
+
if (decision === 'redact-instance') {
|
|
228
|
+
toRedact.push(finding);
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
if (decision === 'approve-instance' ||
|
|
232
|
+
decision === 'add-to-allowlist' ||
|
|
233
|
+
decision === 'add-to-identity-pack') {
|
|
234
|
+
toPass.push(finding);
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
237
|
+
unresolved.push(finding);
|
|
238
|
+
const value = attributes[finding.span.key];
|
|
239
|
+
const text = typeof value === 'string' ? value : '';
|
|
240
|
+
const snippet = text.slice(Math.max(0, finding.span.start - 40), Math.min(text.length, finding.span.end + 40));
|
|
241
|
+
toEnqueue.push({
|
|
242
|
+
finding,
|
|
243
|
+
context: { attributeKey: finding.span.key, snippet },
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
const enqueued = toEnqueue.length > 0 ? store.enqueue(toEnqueue) : [];
|
|
247
|
+
return { unresolved, toRedact, toPass, enqueued };
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Fail closed when unresolved flags remain after {@link processFlagFindings}.
|
|
251
|
+
*/
|
|
252
|
+
export function assertNoUnresolvedFlags(findings, opts = {}) {
|
|
253
|
+
if (opts.failClosed === false)
|
|
254
|
+
return;
|
|
255
|
+
const { unresolved, enqueued } = processFlagFindings(findings, opts);
|
|
256
|
+
if (unresolved.length === 0)
|
|
257
|
+
return;
|
|
258
|
+
throw new UnresolvedFlagError(unresolved, enqueued.map((e) => e.id));
|
|
259
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { Detector } from './finding.js';
|
|
2
|
+
import { type KeyPolicy } from './key-policy.js';
|
|
3
|
+
import type { ScrubStage } from './types.js';
|
|
4
|
+
/**
|
|
5
|
+
* Secrets detector. Two passes over each `content`-classified string value:
|
|
6
|
+
* 1. secretlint preset-recommend rules → A1 findings
|
|
7
|
+
* 2. Shannon-entropy + secret-shape fallback → A2 findings
|
|
8
|
+
*
|
|
9
|
+
* Emits findings only; disposition owns stubs (#1969).
|
|
10
|
+
*/
|
|
11
|
+
export interface SecretlintStageOptions {
|
|
12
|
+
/**
|
|
13
|
+
* Gate for the pass-2 Shannon-entropy fallback. Default true. Under the
|
|
14
|
+
* one-inventory redesign (#1969) every builder leaves this on; the option
|
|
15
|
+
* remains for unit tests that pin the fallback in isolation.
|
|
16
|
+
*/
|
|
17
|
+
entropyFallback?: boolean;
|
|
18
|
+
}
|
|
19
|
+
export declare function secretlintDetector(policy: KeyPolicy, opts?: SecretlintStageOptions): Detector;
|
|
20
|
+
/** Legacy ScrubStage wrapper around {@link secretlintDetector}. */
|
|
21
|
+
export declare function secretlintStage(policy: KeyPolicy, opts?: SecretlintStageOptions): ScrubStage;
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
import { lintSource } from '@secretlint/core';
|
|
2
|
+
import { creator } from '@secretlint/secretlint-rule-preset-recommend';
|
|
3
|
+
import { applyDispositions } from './apply-dispositions.js';
|
|
4
|
+
import { classifyKey } from './key-policy.js';
|
|
5
|
+
const VERSION = '0.5.0'; // 0.5.0: emit findings (#1969); disposition owns stubs
|
|
6
|
+
const PRESET_RULE_ID = '@secretlint/secretlint-rule-preset-recommend';
|
|
7
|
+
// Entropy fallback thresholds. Conservative on purpose: a long token whose
|
|
8
|
+
// character distribution is near-random is almost certainly a key/token, not
|
|
9
|
+
// prose. Tunable; raise the bar if legitimate high-entropy content over-redacts.
|
|
10
|
+
//
|
|
11
|
+
// `ENTROPY_MIN_BITS` stays at 4.0: English prose sits ~4.0–4.5 bits/char, so
|
|
12
|
+
// lowering it below 4.0 globally nukes legitimate prose. `ENTROPY_MIN_LEN` is
|
|
13
|
+
// lowered from 20 → 16 so secret-shaped tokens in the 16–19 char band (which
|
|
14
|
+
// the old 20-char floor never even considered) are caught — but only when they
|
|
15
|
+
// ALSO pass the structural shape gate below, so the longer tail of legit
|
|
16
|
+
// identifiers/words in that band is not swept in. Tokens of length ≥ 20 keep
|
|
17
|
+
// the original entropy-only behaviour (the shape gate does not narrow it).
|
|
18
|
+
const ENTROPY_MIN_LEN = 16;
|
|
19
|
+
const ENTROPY_MIN_BITS = 4.0;
|
|
20
|
+
// Above this length the entropy bar alone decides (original behaviour). Between
|
|
21
|
+
// ENTROPY_MIN_LEN and here the structural shape gate is additionally required.
|
|
22
|
+
const ENTROPY_STRICT_LEN = 20;
|
|
23
|
+
// A "secret-shaped" token: a single run of base64 / base64url / hex charset
|
|
24
|
+
// characters only (letters, digits, and the `+/=_-` set), with NO natural-language
|
|
25
|
+
// punctuation or spaces.
|
|
26
|
+
const SECRET_CHARSET = /^[A-Za-z0-9+/=_-]+$/;
|
|
27
|
+
// Tuning note (#1348): a path-shaped token — slug segments joined by `/`,
|
|
28
|
+
// e.g. `obra/superpowers/skills/test-driven-development` — is judged
|
|
29
|
+
// per-SEGMENT, not whole-token, but ONLY when the whole token mixes at most
|
|
30
|
+
// 2 character classes. Concatenating several short human-readable slugs
|
|
31
|
+
// inflates whole-token Shannon entropy past 4.0 bits/char purely by
|
|
32
|
+
// length/variety, which was redacting seed-import task summaries as
|
|
33
|
+
// `[SECRET:high-entropy]`; real slugs are overwhelmingly lowercase (1–2
|
|
34
|
+
// classes), while secrets that merely contain `/` or `.` (AWS-style base64
|
|
35
|
+
// keys, `dir/<blob>.tar.gz` filenames) almost always mix 3 classes and keep
|
|
36
|
+
// the original whole-token entropy behaviour. The segment charset excludes
|
|
37
|
+
// `+` and `=`, so base64 blobs containing either are never path-shaped.
|
|
38
|
+
// Residual exposure: a secret with ≤ 2 whole-token character classes whose
|
|
39
|
+
// every `[/.]`-split segment is under 16 chars (or below the entropy bar)
|
|
40
|
+
// qualifies as a path and escapes the fallback.
|
|
41
|
+
const PATH_SHAPED = /^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)+$/;
|
|
42
|
+
// SWE-rebench instance ids are public corpus identifiers (`owner__repo-PR`),
|
|
43
|
+
// not key material. Some short bridge summaries repeat them and otherwise cross
|
|
44
|
+
// the defacement gate after entropy redaction.
|
|
45
|
+
const SWE_REBENCH_INSTANCE_ID = /^[a-z0-9][a-z0-9._-]*__[a-z0-9][a-z0-9._-]*-\d+$/;
|
|
46
|
+
// Tuning note (#1391): the entropy fallback's threat model is ASCII key
|
|
47
|
+
// material — base64 / base64url / hex runs. Natural-language text outside the
|
|
48
|
+
// Latin script (CJK especially) has char-level Shannon entropy far above 4.0
|
|
49
|
+
// bits/char (most characters in a sentence are unique), so whole-token entropy
|
|
50
|
+
// classified ordinary Chinese skill prose as secrets and shredded it. A token
|
|
51
|
+
// whose non-ASCII share exceeds this threshold skips the entropy fallback
|
|
52
|
+
// entirely; the pass-1 secretlint preset rules still run on everything.
|
|
53
|
+
// Residual exposure: a generic (rule-less) secret glued WITHOUT whitespace
|
|
54
|
+
// into a majority-non-ASCII token escapes the fallback — prefixed keys
|
|
55
|
+
// (ghp_…, AKIA…, sk-…) are still caught by pass 1, and CJK prose separates
|
|
56
|
+
// embedded ASCII material with spacing or punctuation in practice.
|
|
57
|
+
const NON_ASCII_MAX_SHARE = 0.3;
|
|
58
|
+
// Tuning note (#1391): a STRUCTURED token — one containing characters outside
|
|
59
|
+
// SECRET_CHARSET, i.e. URLs (`://`, `?`, `&`, `#`), markdown links (`](`),
|
|
60
|
+
// shell interpolations (`${…}`), serialised code — cannot itself be a
|
|
61
|
+
// contiguous key: real key material never contains brackets, quotes or `?`.
|
|
62
|
+
// A key can only be EMBEDDED in such a token as a delimited segment, so the
|
|
63
|
+
// token is judged per-segment (mirroring the #1348 PATH_SHAPED carve-out)
|
|
64
|
+
// instead of by whole-token entropy, which URLs and markdown links inflate
|
|
65
|
+
// past 4.0 bits/char purely by punctuation/length variety (every seed
|
|
66
|
+
// envelope's attribution URL published as [SECRET:high-entropy]).
|
|
67
|
+
// Segments split on everything outside `[A-Za-z0-9+=_-]`: `/` and `.` split
|
|
68
|
+
// (slug and query-string segments stay short and human-readable — matching
|
|
69
|
+
// the PATH gate's `[/.]` split), while `-`, `_`, `+`, `=` stay inside a
|
|
70
|
+
// segment because base64url alphabets and key=value pairs use them
|
|
71
|
+
// (`?key=<blob>` gates as one segment; a JWT's `.`-separated base64url
|
|
72
|
+
// segments gate individually). Residual exposure, same shape as the #1348
|
|
73
|
+
// note: a secret whose every delimited segment falls under 16 chars (or the
|
|
74
|
+
// entropy bar, or a single character class) escapes — e.g. a base64 blob
|
|
75
|
+
// with `/` every few chars additionally wrapped in punctuation.
|
|
76
|
+
const STRUCTURED_SEGMENT_DELIMITER = /[^A-Za-z0-9+=_-]+/;
|
|
77
|
+
// Tuning note (#1391): a "wordish" segment — two or more purely-alphabetic
|
|
78
|
+
// runs joined by `_` or `-` (`Microsoft_Azure_Capacity`, `backend-refactor`)
|
|
79
|
+
// — is an identifier, not key material, even when its length and character
|
|
80
|
+
// variety clear the entropy bar (the Azure portal URL's
|
|
81
|
+
// `Microsoft_Azure_Capacity` sits at 4.002 bits/char). Key material of
|
|
82
|
+
// segment-gate length (≥ 16 chars) drawn from a base64/base64url alphabet
|
|
83
|
+
// contains a digit with ~94% probability and separator-split all-alphabetic
|
|
84
|
+
// runs almost never; the residual all-alpha, separator-structured key is
|
|
85
|
+
// accepted as out of the fallback's reach.
|
|
86
|
+
const WORDISH_SEGMENT = /^[A-Za-z]+([_-][A-Za-z]+)+$/;
|
|
87
|
+
function shannonEntropy(s) {
|
|
88
|
+
const freq = {};
|
|
89
|
+
for (const ch of s)
|
|
90
|
+
freq[ch] = (freq[ch] ?? 0) + 1;
|
|
91
|
+
let e = 0;
|
|
92
|
+
for (const count of Object.values(freq)) {
|
|
93
|
+
const p = count / s.length;
|
|
94
|
+
e -= p * Math.log2(p);
|
|
95
|
+
}
|
|
96
|
+
return e;
|
|
97
|
+
}
|
|
98
|
+
/** Count distinct character classes present (lowercase / uppercase / digit). */
|
|
99
|
+
function charClassCount(s) {
|
|
100
|
+
let n = 0;
|
|
101
|
+
if (/[a-z]/.test(s))
|
|
102
|
+
n += 1;
|
|
103
|
+
if (/[A-Z]/.test(s))
|
|
104
|
+
n += 1;
|
|
105
|
+
if (/[0-9]/.test(s))
|
|
106
|
+
n += 1;
|
|
107
|
+
return n;
|
|
108
|
+
}
|
|
109
|
+
function isSweRebenchInstanceId(token) {
|
|
110
|
+
return SWE_REBENCH_INSTANCE_ID.test(token);
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Decide whether a single whitespace-delimited token should be redacted as a
|
|
114
|
+
* high-entropy / secret-shaped value. Purely additive over the original rule:
|
|
115
|
+
* - length ≥ ENTROPY_STRICT_LEN (20): entropy bar alone (unchanged behaviour),
|
|
116
|
+
* - ENTROPY_MIN_LEN (16) ≤ length < 20: entropy bar AND the structural shape
|
|
117
|
+
* gate — the token must be a single secret-charset run mixing ≥ 2 character
|
|
118
|
+
* classes (so dictionary words + a trailing suffix like `authentication123`,
|
|
119
|
+
* which sit below 4.0 bits/char anyway, are not swept in).
|
|
120
|
+
* Never fires below ENTROPY_MIN_LEN.
|
|
121
|
+
*
|
|
122
|
+
* Path carve-out (#1348): a PATH_SHAPED token whose WHOLE-token character-
|
|
123
|
+
* class count is ≤ 2 is redacted only when at least one `[/.]`-delimited
|
|
124
|
+
* segment independently passes the strict shape gate (length ≥
|
|
125
|
+
* ENTROPY_MIN_LEN, entropy ≥ ENTROPY_MIN_BITS, secret charset, ≥ 2 character
|
|
126
|
+
* classes) — applied at ALL lengths, so a long path of short slugs never
|
|
127
|
+
* trips the whole-token entropy bar, while a real key embedded as a path
|
|
128
|
+
* segment (`api/v1/ghp_…`) still redacts. A path-shaped token mixing ≥ 3
|
|
129
|
+
* whole-token classes never takes the carve-out — real slugs are 1–2
|
|
130
|
+
* classes, and 3-class "paths" are secrets wearing slashes/dots (#1348
|
|
131
|
+
* review finding) — it falls through to the whole-token logic below.
|
|
132
|
+
* Splitting on dots as well as slashes means a sentence full stop or a file
|
|
133
|
+
* extension yields its own (failing) segment instead of poisoning the
|
|
134
|
+
* secret blob next to it. Non-path tokens are unaffected.
|
|
135
|
+
*
|
|
136
|
+
* #1391 additions, checked in order:
|
|
137
|
+
* - a non-ASCII-dominant token (share > NON_ASCII_MAX_SHARE) is prose in a
|
|
138
|
+
* non-Latin script, not ASCII key material — never entropy-gated;
|
|
139
|
+
* - a structured token (any character outside SECRET_CHARSET: URLs,
|
|
140
|
+
* markdown links, shell interpolations) is judged per delimited segment
|
|
141
|
+
* via the same strict gate as path segments — see the
|
|
142
|
+
* STRUCTURED_SEGMENT_DELIMITER tuning note.
|
|
143
|
+
* Pure SECRET_CHARSET runs keep the original whole-token behaviour.
|
|
144
|
+
*/
|
|
145
|
+
function isSecretShapedToken(token) {
|
|
146
|
+
if (token.length < ENTROPY_MIN_LEN)
|
|
147
|
+
return false;
|
|
148
|
+
if (nonAsciiShare(token) > NON_ASCII_MAX_SHARE)
|
|
149
|
+
return false;
|
|
150
|
+
if (isSweRebenchInstanceId(token))
|
|
151
|
+
return false;
|
|
152
|
+
if (PATH_SHAPED.test(token) && charClassCount(token) <= 2) {
|
|
153
|
+
return token.split(/[/.]/).some(passesStrictSegmentGate);
|
|
154
|
+
}
|
|
155
|
+
if (!SECRET_CHARSET.test(token)) {
|
|
156
|
+
// Structured token (#1391) — URL, markdown link, shell interpolation…
|
|
157
|
+
// A contiguous key can only live inside a delimited segment.
|
|
158
|
+
return token.split(STRUCTURED_SEGMENT_DELIMITER).some(passesStrictSegmentGate);
|
|
159
|
+
}
|
|
160
|
+
if (shannonEntropy(token) < ENTROPY_MIN_BITS)
|
|
161
|
+
return false;
|
|
162
|
+
if (token.length >= ENTROPY_STRICT_LEN)
|
|
163
|
+
return true;
|
|
164
|
+
return charClassCount(token) >= 2;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* The strict per-segment gate shared by the PATH_SHAPED (#1348) and
|
|
168
|
+
* structured-token (#1391) carve-outs: a segment is secret-shaped only when
|
|
169
|
+
* it is ≥ ENTROPY_MIN_LEN chars, clears the entropy bar, is a single
|
|
170
|
+
* secret-charset run, and mixes ≥ 2 character classes.
|
|
171
|
+
*/
|
|
172
|
+
function passesStrictSegmentGate(segment) {
|
|
173
|
+
return (segment.length >= ENTROPY_MIN_LEN &&
|
|
174
|
+
!WORDISH_SEGMENT.test(segment) &&
|
|
175
|
+
shannonEntropy(segment) >= ENTROPY_MIN_BITS &&
|
|
176
|
+
SECRET_CHARSET.test(segment) &&
|
|
177
|
+
charClassCount(segment) >= 2);
|
|
178
|
+
}
|
|
179
|
+
/** Share of characters outside the 7-bit ASCII range (by code point). */
|
|
180
|
+
function nonAsciiShare(s) {
|
|
181
|
+
let n = 0;
|
|
182
|
+
let total = 0;
|
|
183
|
+
for (const ch of s) {
|
|
184
|
+
total += 1;
|
|
185
|
+
if ((ch.codePointAt(0) ?? 0) > 0x7f)
|
|
186
|
+
n += 1;
|
|
187
|
+
}
|
|
188
|
+
return total === 0 ? 0 : n / total;
|
|
189
|
+
}
|
|
190
|
+
/** Short label from a secretlint ruleId, e.g. `@secretlint/secretlint-rule-github` → `github`. */
|
|
191
|
+
function shortRule(ruleId) {
|
|
192
|
+
return ruleId.replace(/^@secretlint\/secretlint-rule-/, '').replace(/^@secretlint\//, '');
|
|
193
|
+
}
|
|
194
|
+
// #1378 cosmetic: a `\S+` token inside serialised JSON drags its wrapping
|
|
195
|
+
// delimiters along (`"…/blob",` is one whitespace-delimited token), so
|
|
196
|
+
// replacing the whole token swallowed the quotes/comma and left malformed
|
|
197
|
+
// JSON (`"resolved_path": [SECRET:high-entropy] "files_modified": …`). Split
|
|
198
|
+
// each token into leading punctuation / core / trailing punctuation, gate on
|
|
199
|
+
// the core, and reattach the punctuation around the placeholder. None of the
|
|
200
|
+
// stripped characters belong to SECRET_CHARSET (`=` deliberately excluded —
|
|
201
|
+
// base64 padding), so no secret material is ever preserved; if anything,
|
|
202
|
+
// stripping the wrapper improves detection (the 16–19 char shape gate
|
|
203
|
+
// previously failed on quoted tokens because the quotes broke the charset).
|
|
204
|
+
const TOKEN_WRAPPING = /^([("'`[{,;:]*)([\s\S]*?)([)"'`\]},;:.]*)$/;
|
|
205
|
+
export function secretlintDetector(policy, opts = {}) {
|
|
206
|
+
const entropyFallback = opts.entropyFallback ?? true;
|
|
207
|
+
const config = { rules: [{ id: PRESET_RULE_ID, rule: creator }] };
|
|
208
|
+
const meta = { name: 'secretlint', version: VERSION };
|
|
209
|
+
return {
|
|
210
|
+
...meta,
|
|
211
|
+
async detect(attributes) {
|
|
212
|
+
const findings = [];
|
|
213
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
214
|
+
if (typeof value !== 'string' || classifyKey(key, policy) !== 'content')
|
|
215
|
+
continue;
|
|
216
|
+
const result = await lintSource({
|
|
217
|
+
source: { filePath: '/span', content: value, ext: '.txt', contentType: 'text' },
|
|
218
|
+
options: { config },
|
|
219
|
+
});
|
|
220
|
+
for (const m of result.messages) {
|
|
221
|
+
const label = shortRule(m.ruleId);
|
|
222
|
+
findings.push({
|
|
223
|
+
class: 'A1',
|
|
224
|
+
span: { key, start: m.range[0], end: m.range[1] },
|
|
225
|
+
confidence: 'VERY_HIGH',
|
|
226
|
+
evidence: [`secret:${label}`],
|
|
227
|
+
detector: meta,
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
if (!entropyFallback)
|
|
231
|
+
continue;
|
|
232
|
+
const tokenRe = /\S+/g;
|
|
233
|
+
let tokenMatch;
|
|
234
|
+
while ((tokenMatch = tokenRe.exec(value)) !== null) {
|
|
235
|
+
const token = tokenMatch[0];
|
|
236
|
+
const [, lead = '', core = token] = TOKEN_WRAPPING.exec(token) ?? [];
|
|
237
|
+
if (!isSecretShapedToken(core))
|
|
238
|
+
continue;
|
|
239
|
+
const coreStart = tokenMatch.index + lead.length;
|
|
240
|
+
findings.push({
|
|
241
|
+
class: 'A2',
|
|
242
|
+
span: { key, start: coreStart, end: coreStart + core.length },
|
|
243
|
+
confidence: 'HIGH',
|
|
244
|
+
evidence: ['secret:high-entropy'],
|
|
245
|
+
detector: meta,
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return findings;
|
|
250
|
+
},
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
/** Legacy ScrubStage wrapper around {@link secretlintDetector}. */
|
|
254
|
+
export function secretlintStage(policy, opts = {}) {
|
|
255
|
+
const detector = secretlintDetector(policy, opts);
|
|
256
|
+
return {
|
|
257
|
+
name: detector.name,
|
|
258
|
+
version: detector.version,
|
|
259
|
+
async scrub(attributes) {
|
|
260
|
+
const findings = await detector.detect(attributes);
|
|
261
|
+
return applyDispositions(attributes, findings);
|
|
262
|
+
},
|
|
263
|
+
};
|
|
264
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { PiiDetector, PiiEntity } from './ml-pii-stage.js';
|
|
2
|
+
/**
|
|
3
|
+
* Legacy Transformers.js NER model. Retired as the default ML path in #1973
|
|
4
|
+
* (replaced by GLiNER via {@link GlinerPiiDetector}). Kept for unit tests and
|
|
5
|
+
* explicit opt-in via a custom {@link PiiDetectorFactoryPort}.
|
|
6
|
+
*/
|
|
7
|
+
export declare const DEFAULT_NER_MODEL = "Xenova/bert-base-NER";
|
|
8
|
+
/**
|
|
9
|
+
* Entity groups treated as PII. Deterministic detectors own email/card/SSN
|
|
10
|
+
* shapes; this legacy NER tier only covers free-text PER/ORG/LOC. MISC is
|
|
11
|
+
* excluded (noisy).
|
|
12
|
+
*/
|
|
13
|
+
export declare const DEFAULT_PII_ENTITY_GROUPS: string[];
|
|
14
|
+
export interface TransformersPiiDetectorOptions {
|
|
15
|
+
model?: string;
|
|
16
|
+
/** Minimum NER confidence to redact (0–1). */
|
|
17
|
+
threshold?: number;
|
|
18
|
+
/** Entity groups to treat as PII. */
|
|
19
|
+
entityGroups?: string[];
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Legacy Transformers.js NER detector (bert-base-NER). Not the default path —
|
|
23
|
+
* production uses {@link GlinerPiiDetector}. Retained for tests and explicit
|
|
24
|
+
* factory injection.
|
|
25
|
+
*/
|
|
26
|
+
export declare class TransformersPiiDetector implements PiiDetector {
|
|
27
|
+
private pipe;
|
|
28
|
+
private initPromise;
|
|
29
|
+
private readonly model;
|
|
30
|
+
private readonly threshold;
|
|
31
|
+
private readonly entityGroups;
|
|
32
|
+
constructor(opts?: TransformersPiiDetectorOptions);
|
|
33
|
+
/** Model id for the policy hash (#1974). */
|
|
34
|
+
get modelId(): string;
|
|
35
|
+
/** Entity-group label set for the policy hash (#1974). */
|
|
36
|
+
get labelSet(): readonly string[];
|
|
37
|
+
/** Eagerly load the model (e.g. at daemon boot) so the first publish isn't slow. */
|
|
38
|
+
init(): Promise<void>;
|
|
39
|
+
private ensureReady;
|
|
40
|
+
detect(text: string): Promise<PiiEntity[]>;
|
|
41
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Legacy Transformers.js NER model. Retired as the default ML path in #1973
|
|
3
|
+
* (replaced by GLiNER via {@link GlinerPiiDetector}). Kept for unit tests and
|
|
4
|
+
* explicit opt-in via a custom {@link PiiDetectorFactoryPort}.
|
|
5
|
+
*/
|
|
6
|
+
export const DEFAULT_NER_MODEL = 'Xenova/bert-base-NER';
|
|
7
|
+
/**
|
|
8
|
+
* Entity groups treated as PII. Deterministic detectors own email/card/SSN
|
|
9
|
+
* shapes; this legacy NER tier only covers free-text PER/ORG/LOC. MISC is
|
|
10
|
+
* excluded (noisy).
|
|
11
|
+
*/
|
|
12
|
+
export const DEFAULT_PII_ENTITY_GROUPS = ['PER', 'ORG', 'LOC'];
|
|
13
|
+
/**
|
|
14
|
+
* Legacy Transformers.js NER detector (bert-base-NER). Not the default path —
|
|
15
|
+
* production uses {@link GlinerPiiDetector}. Retained for tests and explicit
|
|
16
|
+
* factory injection.
|
|
17
|
+
*/
|
|
18
|
+
export class TransformersPiiDetector {
|
|
19
|
+
pipe;
|
|
20
|
+
initPromise;
|
|
21
|
+
model;
|
|
22
|
+
threshold;
|
|
23
|
+
entityGroups;
|
|
24
|
+
constructor(opts = {}) {
|
|
25
|
+
this.model = opts.model ?? DEFAULT_NER_MODEL;
|
|
26
|
+
this.threshold = opts.threshold ?? 0.6;
|
|
27
|
+
this.entityGroups = new Set(opts.entityGroups ?? DEFAULT_PII_ENTITY_GROUPS);
|
|
28
|
+
}
|
|
29
|
+
/** Model id for the policy hash (#1974). */
|
|
30
|
+
get modelId() {
|
|
31
|
+
return this.model;
|
|
32
|
+
}
|
|
33
|
+
/** Entity-group label set for the policy hash (#1974). */
|
|
34
|
+
get labelSet() {
|
|
35
|
+
return [...this.entityGroups].sort((a, b) => a.localeCompare(b));
|
|
36
|
+
}
|
|
37
|
+
/** Eagerly load the model (e.g. at daemon boot) so the first publish isn't slow. */
|
|
38
|
+
async init() {
|
|
39
|
+
await this.ensureReady();
|
|
40
|
+
}
|
|
41
|
+
ensureReady() {
|
|
42
|
+
if (this.pipe)
|
|
43
|
+
return Promise.resolve(this.pipe);
|
|
44
|
+
if (!this.initPromise) {
|
|
45
|
+
this.initPromise = (async () => {
|
|
46
|
+
const { pipeline } = await import('@huggingface/transformers');
|
|
47
|
+
const pipe = (await pipeline('token-classification', this.model));
|
|
48
|
+
this.pipe = pipe;
|
|
49
|
+
return pipe;
|
|
50
|
+
})();
|
|
51
|
+
}
|
|
52
|
+
return this.initPromise;
|
|
53
|
+
}
|
|
54
|
+
async detect(text) {
|
|
55
|
+
if (!text.trim())
|
|
56
|
+
return [];
|
|
57
|
+
const pipe = await this.ensureReady();
|
|
58
|
+
const results = await pipe(text, { aggregation_strategy: 'simple' });
|
|
59
|
+
const out = [];
|
|
60
|
+
const seen = new Set();
|
|
61
|
+
for (const r of results) {
|
|
62
|
+
if (r.score < this.threshold)
|
|
63
|
+
continue;
|
|
64
|
+
if (!this.entityGroups.has(r.entity_group))
|
|
65
|
+
continue;
|
|
66
|
+
const t = r.word.trim();
|
|
67
|
+
if (!t)
|
|
68
|
+
continue;
|
|
69
|
+
const key = `${r.entity_group}:${t}`;
|
|
70
|
+
if (seen.has(key))
|
|
71
|
+
continue;
|
|
72
|
+
seen.add(key);
|
|
73
|
+
out.push({ label: r.entity_group, text: t, score: r.score });
|
|
74
|
+
}
|
|
75
|
+
return out;
|
|
76
|
+
}
|
|
77
|
+
}
|