@jinn-network/core 0.1.0-canary.3afae198

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/dist/canonical-json.d.ts +1 -0
  2. package/dist/canonical-json.js +67 -0
  3. package/dist/captured-task.d.ts +140 -0
  4. package/dist/captured-task.js +122 -0
  5. package/dist/contribution-store.d.ts +97 -0
  6. package/dist/contribution-store.js +770 -0
  7. package/dist/corpus-read/acquire.d.ts +39 -0
  8. package/dist/corpus-read/acquire.js +212 -0
  9. package/dist/corpus-read/cache.d.ts +14 -0
  10. package/dist/corpus-read/cache.js +17 -0
  11. package/dist/corpus-read/capture-meta.d.ts +16 -0
  12. package/dist/corpus-read/capture-meta.js +18 -0
  13. package/dist/corpus-read/create-corpus.d.ts +2 -0
  14. package/dist/corpus-read/create-corpus.js +97 -0
  15. package/dist/corpus-read/fetch-artifact.d.ts +21 -0
  16. package/dist/corpus-read/fetch-artifact.js +32 -0
  17. package/dist/corpus-read/fetch.d.ts +12 -0
  18. package/dist/corpus-read/fetch.js +24 -0
  19. package/dist/corpus-read/http-discovery.d.ts +9 -0
  20. package/dist/corpus-read/http-discovery.js +128 -0
  21. package/dist/corpus-read/index.d.ts +10 -0
  22. package/dist/corpus-read/index.js +10 -0
  23. package/dist/corpus-read/ipfs.d.ts +14 -0
  24. package/dist/corpus-read/ipfs.js +81 -0
  25. package/dist/corpus-read/route-resolver.d.ts +16 -0
  26. package/dist/corpus-read/route-resolver.js +19 -0
  27. package/dist/corpus-read/types.d.ts +177 -0
  28. package/dist/corpus-read/types.js +42 -0
  29. package/dist/envelope.d.ts +134 -0
  30. package/dist/envelope.js +162 -0
  31. package/dist/evidence-adapter.d.ts +26 -0
  32. package/dist/evidence-adapter.js +321 -0
  33. package/dist/evidence-filesystem.d.ts +40 -0
  34. package/dist/evidence-filesystem.js +267 -0
  35. package/dist/evidence-index.d.ts +117 -0
  36. package/dist/evidence-index.js +1083 -0
  37. package/dist/evidence-store-lock.d.ts +2 -0
  38. package/dist/evidence-store-lock.js +208 -0
  39. package/dist/execution-envelope.d.ts +3142 -0
  40. package/dist/execution-envelope.js +203 -0
  41. package/dist/index.d.ts +12 -0
  42. package/dist/index.js +12 -0
  43. package/dist/manifest.d.ts +75 -0
  44. package/dist/manifest.js +184 -0
  45. package/dist/paired.d.ts +68 -0
  46. package/dist/paired.js +92 -0
  47. package/dist/scrub/apply-dispositions.d.ts +74 -0
  48. package/dist/scrub/apply-dispositions.js +309 -0
  49. package/dist/scrub/build.d.ts +75 -0
  50. package/dist/scrub/build.js +143 -0
  51. package/dist/scrub/checksummed-instruments-detector.d.ts +13 -0
  52. package/dist/scrub/checksummed-instruments-detector.js +119 -0
  53. package/dist/scrub/data/bip39-english.d.ts +2 -0
  54. package/dist/scrub/data/bip39-english.js +2051 -0
  55. package/dist/scrub/data/gitleaks-rules.d.ts +56 -0
  56. package/dist/scrub/data/gitleaks-rules.js +67 -0
  57. package/dist/scrub/emit-scrub.d.ts +52 -0
  58. package/dist/scrub/emit-scrub.js +122 -0
  59. package/dist/scrub/eval/findings-from-scrub.d.ts +18 -0
  60. package/dist/scrub/eval/findings-from-scrub.js +99 -0
  61. package/dist/scrub/eval/fixtures.d.ts +14 -0
  62. package/dist/scrub/eval/fixtures.js +159 -0
  63. package/dist/scrub/eval/index.d.ts +6 -0
  64. package/dist/scrub/eval/index.js +6 -0
  65. package/dist/scrub/eval/local-corpus.d.ts +15 -0
  66. package/dist/scrub/eval/local-corpus.js +29 -0
  67. package/dist/scrub/eval/metrics.d.ts +19 -0
  68. package/dist/scrub/eval/metrics.js +77 -0
  69. package/dist/scrub/eval/run-bench.d.ts +8 -0
  70. package/dist/scrub/eval/run-bench.js +125 -0
  71. package/dist/scrub/eval/types.d.ts +69 -0
  72. package/dist/scrub/eval/types.js +7 -0
  73. package/dist/scrub/finding.d.ts +38 -0
  74. package/dist/scrub/finding.js +7 -0
  75. package/dist/scrub/git-identity-detector.d.ts +16 -0
  76. package/dist/scrub/git-identity-detector.js +116 -0
  77. package/dist/scrub/gitleaks-detector.d.ts +13 -0
  78. package/dist/scrub/gitleaks-detector.js +45 -0
  79. package/dist/scrub/gliner-detector.d.ts +65 -0
  80. package/dist/scrub/gliner-detector.js +129 -0
  81. package/dist/scrub/index.d.ts +26 -0
  82. package/dist/scrub/index.js +26 -0
  83. package/dist/scrub/ip-address-detector.d.ts +12 -0
  84. package/dist/scrub/ip-address-detector.js +81 -0
  85. package/dist/scrub/key-policy.d.ts +30 -0
  86. package/dist/scrub/key-policy.js +72 -0
  87. package/dist/scrub/known-identity-detector.d.ts +66 -0
  88. package/dist/scrub/known-identity-detector.js +284 -0
  89. package/dist/scrub/layer2.d.ts +18 -0
  90. package/dist/scrub/layer2.js +28 -0
  91. package/dist/scrub/ml-pii-stage.d.ts +42 -0
  92. package/dist/scrub/ml-pii-stage.js +119 -0
  93. package/dist/scrub/pii-build.d.ts +47 -0
  94. package/dist/scrub/pii-build.js +67 -0
  95. package/dist/scrub/pipeline.d.ts +77 -0
  96. package/dist/scrub/pipeline.js +225 -0
  97. package/dist/scrub/plain-patterns-stage.d.ts +33 -0
  98. package/dist/scrub/plain-patterns-stage.js +88 -0
  99. package/dist/scrub/policy.d.ts +28 -0
  100. package/dist/scrub/policy.js +156 -0
  101. package/dist/scrub/provenance.d.ts +53 -0
  102. package/dist/scrub/provenance.js +83 -0
  103. package/dist/scrub/reject-classes-detector.d.ts +11 -0
  104. package/dist/scrub/reject-classes-detector.js +195 -0
  105. package/dist/scrub/reject-publish-error.d.ts +30 -0
  106. package/dist/scrub/reject-publish-error.js +47 -0
  107. package/dist/scrub/review-queue.d.ts +90 -0
  108. package/dist/scrub/review-queue.js +259 -0
  109. package/dist/scrub/secretlint-stage.d.ts +21 -0
  110. package/dist/scrub/secretlint-stage.js +264 -0
  111. package/dist/scrub/transformers-detector.d.ts +41 -0
  112. package/dist/scrub/transformers-detector.js +77 -0
  113. package/dist/scrub/types.d.ts +36 -0
  114. package/dist/scrub/types.js +1 -0
  115. package/dist/scrub/url-credentials-detector.d.ts +9 -0
  116. package/dist/scrub/url-credentials-detector.js +66 -0
  117. package/dist/session-provenance.d.ts +75 -0
  118. package/dist/session-provenance.js +20 -0
  119. package/dist/skill-artifact.d.ts +442 -0
  120. package/dist/skill-artifact.js +136 -0
  121. package/dist/trajectory/hash-chain.d.ts +18 -0
  122. package/dist/trajectory/hash-chain.js +47 -0
  123. package/dist/trajectory/index.d.ts +4 -0
  124. package/dist/trajectory/index.js +4 -0
  125. package/dist/trajectory/schema.d.ts +681 -0
  126. package/dist/trajectory/schema.js +119 -0
  127. package/dist/trajectory/transcript-parsers/aider-history.d.ts +42 -0
  128. package/dist/trajectory/transcript-parsers/aider-history.js +181 -0
  129. package/dist/trajectory/transcript-parsers/claude-code-jsonl.d.ts +37 -0
  130. package/dist/trajectory/transcript-parsers/claude-code-jsonl.js +142 -0
  131. package/dist/trajectory/transcript-parsers/codex-session.d.ts +42 -0
  132. package/dist/trajectory/transcript-parsers/codex-session.js +235 -0
  133. package/dist/trajectory/transcript-parsers/continue-devdata.d.ts +41 -0
  134. package/dist/trajectory/transcript-parsers/continue-devdata.js +190 -0
  135. package/dist/trajectory/transcript-parsers/cursor-sqlite.d.ts +38 -0
  136. package/dist/trajectory/transcript-parsers/cursor-sqlite.js +118 -0
  137. package/dist/trajectory/transcript-parsers/gemini-session.d.ts +35 -0
  138. package/dist/trajectory/transcript-parsers/gemini-session.js +137 -0
  139. package/dist/trajectory/transcript-parsers/index.d.ts +7 -0
  140. package/dist/trajectory/transcript-parsers/index.js +7 -0
  141. package/dist/trajectory/transcript-parsers/types.d.ts +101 -0
  142. package/dist/trajectory/transcript-parsers/types.js +32 -0
  143. package/dist/trajectory/transcript-to-spans/attrs.d.ts +4 -0
  144. package/dist/trajectory/transcript-to-spans/attrs.js +30 -0
  145. package/dist/trajectory/transcript-to-spans/claude-code-stream-json.d.ts +8 -0
  146. package/dist/trajectory/transcript-to-spans/claude-code-stream-json.js +162 -0
  147. package/dist/trajectory/transcript-to-spans/codex-exec-json.d.ts +8 -0
  148. package/dist/trajectory/transcript-to-spans/codex-exec-json.js +253 -0
  149. package/dist/trajectory/transcript-to-spans/index.d.ts +4 -0
  150. package/dist/trajectory/transcript-to-spans/index.js +4 -0
  151. package/dist/trajectory/transcript-to-spans/types.d.ts +18 -0
  152. package/dist/trajectory/transcript-to-spans/types.js +1 -0
  153. package/dist/window.d.ts +12 -0
  154. package/dist/window.js +5 -0
  155. package/package.json +72 -0
@@ -0,0 +1,259 @@
1
+ /**
2
+ * Operator-local review queue for scrub `flag` dispositions (#1973 / design §6.6).
3
+ *
4
+ * Persistence: `~/.jinn-client/scrub-review/queue.jsonl` (one JSON object per
5
+ * line). Traces never leave the machine — this queue is local-only.
6
+ *
7
+ * Unattended publish lanes fail closed while any matching flag remains
8
+ * `pending` (see {@link assertNoUnresolvedFlags}).
9
+ */
10
+ import { createHash, randomUUID } from 'node:crypto';
11
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
12
+ import { homedir } from 'node:os';
13
+ import { dirname, join } from 'node:path';
14
+ import { resolveDisposition } from './policy.js';
15
+ import { DEFAULT_POLICY } from './policy.js';
16
+ export const DEFAULT_REVIEW_QUEUE_PATH = join(homedir(), '.jinn-client', 'scrub-review', 'queue.jsonl');
17
+ /** Stable fingerprint for matching a finding across scrub runs. */
18
+ export function findingFingerprint(finding) {
19
+ const payload = [
20
+ finding.class,
21
+ finding.span.key,
22
+ String(finding.span.start),
23
+ String(finding.span.end),
24
+ finding.evidence.join('|'),
25
+ finding.detector.name,
26
+ ].join('\0');
27
+ return createHash('sha256').update(payload).digest('hex').slice(0, 24);
28
+ }
29
+ function readAll(path) {
30
+ if (!existsSync(path))
31
+ return [];
32
+ const text = readFileSync(path, 'utf8');
33
+ const items = [];
34
+ for (const line of text.split('\n')) {
35
+ const trimmed = line.trim();
36
+ if (!trimmed)
37
+ continue;
38
+ try {
39
+ items.push(JSON.parse(trimmed));
40
+ }
41
+ catch {
42
+ // skip corrupt lines
43
+ }
44
+ }
45
+ return items;
46
+ }
47
+ function writeAll(path, items) {
48
+ mkdirSync(dirname(path), { recursive: true });
49
+ const body = items.map((i) => JSON.stringify(i)).join('\n');
50
+ writeFileSync(path, body.length > 0 ? `${body}\n` : '', 'utf8');
51
+ }
52
+ /** In-memory review queue for unit tests (no disk I/O). */
53
+ export function createMemoryReviewQueueStore(seed = []) {
54
+ let items = [...seed];
55
+ return {
56
+ listFlagged(filter) {
57
+ if (!filter?.status)
58
+ return [...items];
59
+ return items.filter((i) => i.status === filter.status);
60
+ },
61
+ enqueue(rawItems) {
62
+ const pendingFingerprints = new Set(items.filter((i) => i.status === 'pending').map((i) => findingFingerprint(i.finding)));
63
+ const created = [];
64
+ for (const raw of rawItems) {
65
+ const fp = findingFingerprint(raw.finding);
66
+ if (pendingFingerprints.has(fp))
67
+ continue;
68
+ const prior = [...items]
69
+ .reverse()
70
+ .find((i) => findingFingerprint(i.finding) === fp && i.status === 'resolved');
71
+ if (prior)
72
+ continue;
73
+ const item = {
74
+ id: randomUUID(),
75
+ finding: raw.finding,
76
+ context: raw.context,
77
+ createdAt: new Date().toISOString(),
78
+ status: 'pending',
79
+ };
80
+ items.push(item);
81
+ pendingFingerprints.add(fp);
82
+ created.push(item);
83
+ }
84
+ return created;
85
+ },
86
+ resolveFlag(id, decision) {
87
+ const idx = items.findIndex((i) => i.id === id);
88
+ if (idx < 0)
89
+ throw new Error(`review queue: unknown flag id ${id}`);
90
+ const item = items[idx];
91
+ if (item.status === 'resolved') {
92
+ throw new Error(`review queue: flag ${id} is already resolved`);
93
+ }
94
+ const updated = {
95
+ ...item,
96
+ status: 'resolved',
97
+ resolution: { decision, resolvedAt: new Date().toISOString() },
98
+ };
99
+ items = items.map((i, iidx) => (iidx === idx ? updated : i));
100
+ return updated;
101
+ },
102
+ resolutionFor(finding) {
103
+ const fp = findingFingerprint(finding);
104
+ const match = [...items]
105
+ .reverse()
106
+ .find((i) => findingFingerprint(i.finding) === fp && i.status === 'resolved');
107
+ return match?.resolution?.decision;
108
+ },
109
+ };
110
+ }
111
+ /**
112
+ * File-backed review queue. Latest write wins per id; fingerprints look at the
113
+ * newest resolved entry.
114
+ */
115
+ export function createReviewQueueStore(path = DEFAULT_REVIEW_QUEUE_PATH) {
116
+ return {
117
+ listFlagged(filter) {
118
+ const all = readAll(path);
119
+ if (!filter?.status)
120
+ return all;
121
+ return all.filter((i) => i.status === filter.status);
122
+ },
123
+ enqueue(rawItems) {
124
+ mkdirSync(dirname(path), { recursive: true });
125
+ const existing = readAll(path);
126
+ const pendingFingerprints = new Set(existing
127
+ .filter((i) => i.status === 'pending')
128
+ .map((i) => findingFingerprint(i.finding)));
129
+ const created = [];
130
+ for (const raw of rawItems) {
131
+ const fp = findingFingerprint(raw.finding);
132
+ if (pendingFingerprints.has(fp))
133
+ continue;
134
+ // Skip if already resolved for this fingerprint.
135
+ const prior = [...existing]
136
+ .reverse()
137
+ .find((i) => findingFingerprint(i.finding) === fp && i.status === 'resolved');
138
+ if (prior)
139
+ continue;
140
+ const item = {
141
+ id: randomUUID(),
142
+ finding: raw.finding,
143
+ context: raw.context,
144
+ createdAt: new Date().toISOString(),
145
+ status: 'pending',
146
+ };
147
+ appendFileSync(path, `${JSON.stringify(item)}\n`, 'utf8');
148
+ pendingFingerprints.add(fp);
149
+ created.push(item);
150
+ }
151
+ return created;
152
+ },
153
+ resolveFlag(id, decision) {
154
+ const all = readAll(path);
155
+ const idx = all.findIndex((i) => i.id === id);
156
+ if (idx < 0) {
157
+ throw new Error(`review queue: unknown flag id ${id}`);
158
+ }
159
+ const item = all[idx];
160
+ if (item.status === 'resolved') {
161
+ throw new Error(`review queue: flag ${id} is already resolved`);
162
+ }
163
+ const updated = {
164
+ ...item,
165
+ status: 'resolved',
166
+ resolution: { decision, resolvedAt: new Date().toISOString() },
167
+ };
168
+ all[idx] = updated;
169
+ writeAll(path, all);
170
+ return updated;
171
+ },
172
+ resolutionFor(finding) {
173
+ const fp = findingFingerprint(finding);
174
+ const match = [...readAll(path)]
175
+ .reverse()
176
+ .find((i) => findingFingerprint(i.finding) === fp && i.status === 'resolved');
177
+ return match?.resolution?.decision;
178
+ },
179
+ };
180
+ }
181
+ /** Convenience exports bound to the default operator-local path. */
182
+ const defaultStore = createReviewQueueStore();
183
+ export function listFlagged(filter) {
184
+ return defaultStore.listFlagged(filter);
185
+ }
186
+ export function resolveFlag(id, decision) {
187
+ return defaultStore.resolveFlag(id, decision);
188
+ }
189
+ export function enqueueFlags(items, store = defaultStore) {
190
+ return store.enqueue(items);
191
+ }
192
+ /**
193
+ * Thrown when redact-mode scrub leaves unresolved `flag` dispositions.
194
+ * Unattended publish must not proceed over an open flag (design §6.5).
195
+ */
196
+ export class UnresolvedFlagError extends Error {
197
+ findings;
198
+ queueIds;
199
+ constructor(findings, queueIds = []) {
200
+ const n = findings.length;
201
+ super(`unresolved-flag: ${n} scrub finding(s) require review — publish aborted ` +
202
+ `(run \`jinn scrub review\` to resolve; ids: ${queueIds.slice(0, 5).join(', ') || 'n/a'})`);
203
+ this.name = 'UnresolvedFlagError';
204
+ this.findings = findings;
205
+ this.queueIds = queueIds;
206
+ }
207
+ }
208
+ /**
209
+ * Apply review-queue resolutions to flag findings, enqueue new pending flags,
210
+ * and optionally fail closed.
211
+ *
212
+ * - `approve-instance` / allowlist / identity-pack → treat as pass (no throw)
213
+ * - `redact-instance` → caller should promote to redact (returned in `toRedact`)
214
+ * - unresolved → enqueue + throw when `failClosed`
215
+ */
216
+ export function processFlagFindings(findings, opts = {}) {
217
+ const policy = opts.policy ?? DEFAULT_POLICY;
218
+ const store = opts.store ?? defaultStore;
219
+ const attributes = opts.attributes ?? {};
220
+ const flagged = findings.filter((f) => resolveDisposition(f.class, f.confidence, policy) === 'flag');
221
+ const unresolved = [];
222
+ const toRedact = [];
223
+ const toPass = [];
224
+ const toEnqueue = [];
225
+ for (const finding of flagged) {
226
+ const decision = store.resolutionFor(finding);
227
+ if (decision === 'redact-instance') {
228
+ toRedact.push(finding);
229
+ continue;
230
+ }
231
+ if (decision === 'approve-instance' ||
232
+ decision === 'add-to-allowlist' ||
233
+ decision === 'add-to-identity-pack') {
234
+ toPass.push(finding);
235
+ continue;
236
+ }
237
+ unresolved.push(finding);
238
+ const value = attributes[finding.span.key];
239
+ const text = typeof value === 'string' ? value : '';
240
+ const snippet = text.slice(Math.max(0, finding.span.start - 40), Math.min(text.length, finding.span.end + 40));
241
+ toEnqueue.push({
242
+ finding,
243
+ context: { attributeKey: finding.span.key, snippet },
244
+ });
245
+ }
246
+ const enqueued = toEnqueue.length > 0 ? store.enqueue(toEnqueue) : [];
247
+ return { unresolved, toRedact, toPass, enqueued };
248
+ }
249
+ /**
250
+ * Fail closed when unresolved flags remain after {@link processFlagFindings}.
251
+ */
252
+ export function assertNoUnresolvedFlags(findings, opts = {}) {
253
+ if (opts.failClosed === false)
254
+ return;
255
+ const { unresolved, enqueued } = processFlagFindings(findings, opts);
256
+ if (unresolved.length === 0)
257
+ return;
258
+ throw new UnresolvedFlagError(unresolved, enqueued.map((e) => e.id));
259
+ }
@@ -0,0 +1,21 @@
1
+ import type { Detector } from './finding.js';
2
+ import { type KeyPolicy } from './key-policy.js';
3
+ import type { ScrubStage } from './types.js';
4
+ /**
5
+ * Secrets detector. Two passes over each `content`-classified string value:
6
+ * 1. secretlint preset-recommend rules → A1 findings
7
+ * 2. Shannon-entropy + secret-shape fallback → A2 findings
8
+ *
9
+ * Emits findings only; disposition owns stubs (#1969).
10
+ */
11
+ export interface SecretlintStageOptions {
12
+ /**
13
+ * Gate for the pass-2 Shannon-entropy fallback. Default true. Under the
14
+ * one-inventory redesign (#1969) every builder leaves this on; the option
15
+ * remains for unit tests that pin the fallback in isolation.
16
+ */
17
+ entropyFallback?: boolean;
18
+ }
19
+ export declare function secretlintDetector(policy: KeyPolicy, opts?: SecretlintStageOptions): Detector;
20
+ /** Legacy ScrubStage wrapper around {@link secretlintDetector}. */
21
+ export declare function secretlintStage(policy: KeyPolicy, opts?: SecretlintStageOptions): ScrubStage;
@@ -0,0 +1,264 @@
1
+ import { lintSource } from '@secretlint/core';
2
+ import { creator } from '@secretlint/secretlint-rule-preset-recommend';
3
+ import { applyDispositions } from './apply-dispositions.js';
4
+ import { classifyKey } from './key-policy.js';
5
+ const VERSION = '0.5.0'; // 0.5.0: emit findings (#1969); disposition owns stubs
6
+ const PRESET_RULE_ID = '@secretlint/secretlint-rule-preset-recommend';
7
+ // Entropy fallback thresholds. Conservative on purpose: a long token whose
8
+ // character distribution is near-random is almost certainly a key/token, not
9
+ // prose. Tunable; raise the bar if legitimate high-entropy content over-redacts.
10
+ //
11
+ // `ENTROPY_MIN_BITS` stays at 4.0: English prose sits ~4.0–4.5 bits/char, so
12
+ // lowering it below 4.0 globally nukes legitimate prose. `ENTROPY_MIN_LEN` is
13
+ // lowered from 20 → 16 so secret-shaped tokens in the 16–19 char band (which
14
+ // the old 20-char floor never even considered) are caught — but only when they
15
+ // ALSO pass the structural shape gate below, so the longer tail of legit
16
+ // identifiers/words in that band is not swept in. Tokens of length ≥ 20 keep
17
+ // the original entropy-only behaviour (the shape gate does not narrow it).
18
+ const ENTROPY_MIN_LEN = 16;
19
+ const ENTROPY_MIN_BITS = 4.0;
20
+ // Above this length the entropy bar alone decides (original behaviour). Between
21
+ // ENTROPY_MIN_LEN and here the structural shape gate is additionally required.
22
+ const ENTROPY_STRICT_LEN = 20;
23
+ // A "secret-shaped" token: a single run of base64 / base64url / hex charset
24
+ // characters only (letters, digits, and the `+/=_-` set), with NO natural-language
25
+ // punctuation or spaces.
26
+ const SECRET_CHARSET = /^[A-Za-z0-9+/=_-]+$/;
27
+ // Tuning note (#1348): a path-shaped token — slug segments joined by `/`,
28
+ // e.g. `obra/superpowers/skills/test-driven-development` — is judged
29
+ // per-SEGMENT, not whole-token, but ONLY when the whole token mixes at most
30
+ // 2 character classes. Concatenating several short human-readable slugs
31
+ // inflates whole-token Shannon entropy past 4.0 bits/char purely by
32
+ // length/variety, which was redacting seed-import task summaries as
33
+ // `[SECRET:high-entropy]`; real slugs are overwhelmingly lowercase (1–2
34
+ // classes), while secrets that merely contain `/` or `.` (AWS-style base64
35
+ // keys, `dir/<blob>.tar.gz` filenames) almost always mix 3 classes and keep
36
+ // the original whole-token entropy behaviour. The segment charset excludes
37
+ // `+` and `=`, so base64 blobs containing either are never path-shaped.
38
+ // Residual exposure: a secret with ≤ 2 whole-token character classes whose
39
+ // every `[/.]`-split segment is under 16 chars (or below the entropy bar)
40
+ // qualifies as a path and escapes the fallback.
41
+ const PATH_SHAPED = /^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)+$/;
42
+ // SWE-rebench instance ids are public corpus identifiers (`owner__repo-PR`),
43
+ // not key material. Some short bridge summaries repeat them and otherwise cross
44
+ // the defacement gate after entropy redaction.
45
+ const SWE_REBENCH_INSTANCE_ID = /^[a-z0-9][a-z0-9._-]*__[a-z0-9][a-z0-9._-]*-\d+$/;
46
+ // Tuning note (#1391): the entropy fallback's threat model is ASCII key
47
+ // material — base64 / base64url / hex runs. Natural-language text outside the
48
+ // Latin script (CJK especially) has char-level Shannon entropy far above 4.0
49
+ // bits/char (most characters in a sentence are unique), so whole-token entropy
50
+ // classified ordinary Chinese skill prose as secrets and shredded it. A token
51
+ // whose non-ASCII share exceeds this threshold skips the entropy fallback
52
+ // entirely; the pass-1 secretlint preset rules still run on everything.
53
+ // Residual exposure: a generic (rule-less) secret glued WITHOUT whitespace
54
+ // into a majority-non-ASCII token escapes the fallback — prefixed keys
55
+ // (ghp_…, AKIA…, sk-…) are still caught by pass 1, and CJK prose separates
56
+ // embedded ASCII material with spacing or punctuation in practice.
57
+ const NON_ASCII_MAX_SHARE = 0.3;
58
+ // Tuning note (#1391): a STRUCTURED token — one containing characters outside
59
+ // SECRET_CHARSET, i.e. URLs (`://`, `?`, `&`, `#`), markdown links (`](`),
60
+ // shell interpolations (`${…}`), serialised code — cannot itself be a
61
+ // contiguous key: real key material never contains brackets, quotes or `?`.
62
+ // A key can only be EMBEDDED in such a token as a delimited segment, so the
63
+ // token is judged per-segment (mirroring the #1348 PATH_SHAPED carve-out)
64
+ // instead of by whole-token entropy, which URLs and markdown links inflate
65
+ // past 4.0 bits/char purely by punctuation/length variety (every seed
66
+ // envelope's attribution URL published as [SECRET:high-entropy]).
67
+ // Segments split on everything outside `[A-Za-z0-9+=_-]`: `/` and `.` split
68
+ // (slug and query-string segments stay short and human-readable — matching
69
+ // the PATH gate's `[/.]` split), while `-`, `_`, `+`, `=` stay inside a
70
+ // segment because base64url alphabets and key=value pairs use them
71
+ // (`?key=<blob>` gates as one segment; a JWT's `.`-separated base64url
72
+ // segments gate individually). Residual exposure, same shape as the #1348
73
+ // note: a secret whose every delimited segment falls under 16 chars (or the
74
+ // entropy bar, or a single character class) escapes — e.g. a base64 blob
75
+ // with `/` every few chars additionally wrapped in punctuation.
76
+ const STRUCTURED_SEGMENT_DELIMITER = /[^A-Za-z0-9+=_-]+/;
77
+ // Tuning note (#1391): a "wordish" segment — two or more purely-alphabetic
78
+ // runs joined by `_` or `-` (`Microsoft_Azure_Capacity`, `backend-refactor`)
79
+ // — is an identifier, not key material, even when its length and character
80
+ // variety clear the entropy bar (the Azure portal URL's
81
+ // `Microsoft_Azure_Capacity` sits at 4.002 bits/char). Key material of
82
+ // segment-gate length (≥ 16 chars) drawn from a base64/base64url alphabet
83
+ // contains a digit with ~94% probability and separator-split all-alphabetic
84
+ // runs almost never; the residual all-alpha, separator-structured key is
85
+ // accepted as out of the fallback's reach.
86
+ const WORDISH_SEGMENT = /^[A-Za-z]+([_-][A-Za-z]+)+$/;
87
+ function shannonEntropy(s) {
88
+ const freq = {};
89
+ for (const ch of s)
90
+ freq[ch] = (freq[ch] ?? 0) + 1;
91
+ let e = 0;
92
+ for (const count of Object.values(freq)) {
93
+ const p = count / s.length;
94
+ e -= p * Math.log2(p);
95
+ }
96
+ return e;
97
+ }
98
+ /** Count distinct character classes present (lowercase / uppercase / digit). */
99
+ function charClassCount(s) {
100
+ let n = 0;
101
+ if (/[a-z]/.test(s))
102
+ n += 1;
103
+ if (/[A-Z]/.test(s))
104
+ n += 1;
105
+ if (/[0-9]/.test(s))
106
+ n += 1;
107
+ return n;
108
+ }
109
+ function isSweRebenchInstanceId(token) {
110
+ return SWE_REBENCH_INSTANCE_ID.test(token);
111
+ }
112
+ /**
113
+ * Decide whether a single whitespace-delimited token should be redacted as a
114
+ * high-entropy / secret-shaped value. Purely additive over the original rule:
115
+ * - length ≥ ENTROPY_STRICT_LEN (20): entropy bar alone (unchanged behaviour),
116
+ * - ENTROPY_MIN_LEN (16) ≤ length < 20: entropy bar AND the structural shape
117
+ * gate — the token must be a single secret-charset run mixing ≥ 2 character
118
+ * classes (so dictionary words + a trailing suffix like `authentication123`,
119
+ * which sit below 4.0 bits/char anyway, are not swept in).
120
+ * Never fires below ENTROPY_MIN_LEN.
121
+ *
122
+ * Path carve-out (#1348): a PATH_SHAPED token whose WHOLE-token character-
123
+ * class count is ≤ 2 is redacted only when at least one `[/.]`-delimited
124
+ * segment independently passes the strict shape gate (length ≥
125
+ * ENTROPY_MIN_LEN, entropy ≥ ENTROPY_MIN_BITS, secret charset, ≥ 2 character
126
+ * classes) — applied at ALL lengths, so a long path of short slugs never
127
+ * trips the whole-token entropy bar, while a real key embedded as a path
128
+ * segment (`api/v1/ghp_…`) still redacts. A path-shaped token mixing ≥ 3
129
+ * whole-token classes never takes the carve-out — real slugs are 1–2
130
+ * classes, and 3-class "paths" are secrets wearing slashes/dots (#1348
131
+ * review finding) — it falls through to the whole-token logic below.
132
+ * Splitting on dots as well as slashes means a sentence full stop or a file
133
+ * extension yields its own (failing) segment instead of poisoning the
134
+ * secret blob next to it. Non-path tokens are unaffected.
135
+ *
136
+ * #1391 additions, checked in order:
137
+ * - a non-ASCII-dominant token (share > NON_ASCII_MAX_SHARE) is prose in a
138
+ * non-Latin script, not ASCII key material — never entropy-gated;
139
+ * - a structured token (any character outside SECRET_CHARSET: URLs,
140
+ * markdown links, shell interpolations) is judged per delimited segment
141
+ * via the same strict gate as path segments — see the
142
+ * STRUCTURED_SEGMENT_DELIMITER tuning note.
143
+ * Pure SECRET_CHARSET runs keep the original whole-token behaviour.
144
+ */
145
+ function isSecretShapedToken(token) {
146
+ if (token.length < ENTROPY_MIN_LEN)
147
+ return false;
148
+ if (nonAsciiShare(token) > NON_ASCII_MAX_SHARE)
149
+ return false;
150
+ if (isSweRebenchInstanceId(token))
151
+ return false;
152
+ if (PATH_SHAPED.test(token) && charClassCount(token) <= 2) {
153
+ return token.split(/[/.]/).some(passesStrictSegmentGate);
154
+ }
155
+ if (!SECRET_CHARSET.test(token)) {
156
+ // Structured token (#1391) — URL, markdown link, shell interpolation…
157
+ // A contiguous key can only live inside a delimited segment.
158
+ return token.split(STRUCTURED_SEGMENT_DELIMITER).some(passesStrictSegmentGate);
159
+ }
160
+ if (shannonEntropy(token) < ENTROPY_MIN_BITS)
161
+ return false;
162
+ if (token.length >= ENTROPY_STRICT_LEN)
163
+ return true;
164
+ return charClassCount(token) >= 2;
165
+ }
166
+ /**
167
+ * The strict per-segment gate shared by the PATH_SHAPED (#1348) and
168
+ * structured-token (#1391) carve-outs: a segment is secret-shaped only when
169
+ * it is ≥ ENTROPY_MIN_LEN chars, clears the entropy bar, is a single
170
+ * secret-charset run, and mixes ≥ 2 character classes.
171
+ */
172
+ function passesStrictSegmentGate(segment) {
173
+ return (segment.length >= ENTROPY_MIN_LEN &&
174
+ !WORDISH_SEGMENT.test(segment) &&
175
+ shannonEntropy(segment) >= ENTROPY_MIN_BITS &&
176
+ SECRET_CHARSET.test(segment) &&
177
+ charClassCount(segment) >= 2);
178
+ }
179
+ /** Share of characters outside the 7-bit ASCII range (by code point). */
180
+ function nonAsciiShare(s) {
181
+ let n = 0;
182
+ let total = 0;
183
+ for (const ch of s) {
184
+ total += 1;
185
+ if ((ch.codePointAt(0) ?? 0) > 0x7f)
186
+ n += 1;
187
+ }
188
+ return total === 0 ? 0 : n / total;
189
+ }
190
+ /** Short label from a secretlint ruleId, e.g. `@secretlint/secretlint-rule-github` → `github`. */
191
+ function shortRule(ruleId) {
192
+ return ruleId.replace(/^@secretlint\/secretlint-rule-/, '').replace(/^@secretlint\//, '');
193
+ }
194
+ // #1378 cosmetic: a `\S+` token inside serialised JSON drags its wrapping
195
+ // delimiters along (`"…/blob",` is one whitespace-delimited token), so
196
+ // replacing the whole token swallowed the quotes/comma and left malformed
197
+ // JSON (`"resolved_path": [SECRET:high-entropy] "files_modified": …`). Split
198
+ // each token into leading punctuation / core / trailing punctuation, gate on
199
+ // the core, and reattach the punctuation around the placeholder. None of the
200
+ // stripped characters belong to SECRET_CHARSET (`=` deliberately excluded —
201
+ // base64 padding), so no secret material is ever preserved; if anything,
202
+ // stripping the wrapper improves detection (the 16–19 char shape gate
203
+ // previously failed on quoted tokens because the quotes broke the charset).
204
+ const TOKEN_WRAPPING = /^([("'`[{,;:]*)([\s\S]*?)([)"'`\]},;:.]*)$/;
205
+ export function secretlintDetector(policy, opts = {}) {
206
+ const entropyFallback = opts.entropyFallback ?? true;
207
+ const config = { rules: [{ id: PRESET_RULE_ID, rule: creator }] };
208
+ const meta = { name: 'secretlint', version: VERSION };
209
+ return {
210
+ ...meta,
211
+ async detect(attributes) {
212
+ const findings = [];
213
+ for (const [key, value] of Object.entries(attributes)) {
214
+ if (typeof value !== 'string' || classifyKey(key, policy) !== 'content')
215
+ continue;
216
+ const result = await lintSource({
217
+ source: { filePath: '/span', content: value, ext: '.txt', contentType: 'text' },
218
+ options: { config },
219
+ });
220
+ for (const m of result.messages) {
221
+ const label = shortRule(m.ruleId);
222
+ findings.push({
223
+ class: 'A1',
224
+ span: { key, start: m.range[0], end: m.range[1] },
225
+ confidence: 'VERY_HIGH',
226
+ evidence: [`secret:${label}`],
227
+ detector: meta,
228
+ });
229
+ }
230
+ if (!entropyFallback)
231
+ continue;
232
+ const tokenRe = /\S+/g;
233
+ let tokenMatch;
234
+ while ((tokenMatch = tokenRe.exec(value)) !== null) {
235
+ const token = tokenMatch[0];
236
+ const [, lead = '', core = token] = TOKEN_WRAPPING.exec(token) ?? [];
237
+ if (!isSecretShapedToken(core))
238
+ continue;
239
+ const coreStart = tokenMatch.index + lead.length;
240
+ findings.push({
241
+ class: 'A2',
242
+ span: { key, start: coreStart, end: coreStart + core.length },
243
+ confidence: 'HIGH',
244
+ evidence: ['secret:high-entropy'],
245
+ detector: meta,
246
+ });
247
+ }
248
+ }
249
+ return findings;
250
+ },
251
+ };
252
+ }
253
+ /** Legacy ScrubStage wrapper around {@link secretlintDetector}. */
254
+ export function secretlintStage(policy, opts = {}) {
255
+ const detector = secretlintDetector(policy, opts);
256
+ return {
257
+ name: detector.name,
258
+ version: detector.version,
259
+ async scrub(attributes) {
260
+ const findings = await detector.detect(attributes);
261
+ return applyDispositions(attributes, findings);
262
+ },
263
+ };
264
+ }
@@ -0,0 +1,41 @@
1
+ import type { PiiDetector, PiiEntity } from './ml-pii-stage.js';
2
+ /**
3
+ * Legacy Transformers.js NER model. Retired as the default ML path in #1973
4
+ * (replaced by GLiNER via {@link GlinerPiiDetector}). Kept for unit tests and
5
+ * explicit opt-in via a custom {@link PiiDetectorFactoryPort}.
6
+ */
7
+ export declare const DEFAULT_NER_MODEL = "Xenova/bert-base-NER";
8
+ /**
9
+ * Entity groups treated as PII. Deterministic detectors own email/card/SSN
10
+ * shapes; this legacy NER tier only covers free-text PER/ORG/LOC. MISC is
11
+ * excluded (noisy).
12
+ */
13
+ export declare const DEFAULT_PII_ENTITY_GROUPS: string[];
14
+ export interface TransformersPiiDetectorOptions {
15
+ model?: string;
16
+ /** Minimum NER confidence to redact (0–1). */
17
+ threshold?: number;
18
+ /** Entity groups to treat as PII. */
19
+ entityGroups?: string[];
20
+ }
21
+ /**
22
+ * Legacy Transformers.js NER detector (bert-base-NER). Not the default path —
23
+ * production uses {@link GlinerPiiDetector}. Retained for tests and explicit
24
+ * factory injection.
25
+ */
26
+ export declare class TransformersPiiDetector implements PiiDetector {
27
+ private pipe;
28
+ private initPromise;
29
+ private readonly model;
30
+ private readonly threshold;
31
+ private readonly entityGroups;
32
+ constructor(opts?: TransformersPiiDetectorOptions);
33
+ /** Model id for the policy hash (#1974). */
34
+ get modelId(): string;
35
+ /** Entity-group label set for the policy hash (#1974). */
36
+ get labelSet(): readonly string[];
37
+ /** Eagerly load the model (e.g. at daemon boot) so the first publish isn't slow. */
38
+ init(): Promise<void>;
39
+ private ensureReady;
40
+ detect(text: string): Promise<PiiEntity[]>;
41
+ }
@@ -0,0 +1,77 @@
1
+ /**
2
+ * Legacy Transformers.js NER model. Retired as the default ML path in #1973
3
+ * (replaced by GLiNER via {@link GlinerPiiDetector}). Kept for unit tests and
4
+ * explicit opt-in via a custom {@link PiiDetectorFactoryPort}.
5
+ */
6
+ export const DEFAULT_NER_MODEL = 'Xenova/bert-base-NER';
7
+ /**
8
+ * Entity groups treated as PII. Deterministic detectors own email/card/SSN
9
+ * shapes; this legacy NER tier only covers free-text PER/ORG/LOC. MISC is
10
+ * excluded (noisy).
11
+ */
12
+ export const DEFAULT_PII_ENTITY_GROUPS = ['PER', 'ORG', 'LOC'];
13
+ /**
14
+ * Legacy Transformers.js NER detector (bert-base-NER). Not the default path —
15
+ * production uses {@link GlinerPiiDetector}. Retained for tests and explicit
16
+ * factory injection.
17
+ */
18
+ export class TransformersPiiDetector {
19
+ pipe;
20
+ initPromise;
21
+ model;
22
+ threshold;
23
+ entityGroups;
24
+ constructor(opts = {}) {
25
+ this.model = opts.model ?? DEFAULT_NER_MODEL;
26
+ this.threshold = opts.threshold ?? 0.6;
27
+ this.entityGroups = new Set(opts.entityGroups ?? DEFAULT_PII_ENTITY_GROUPS);
28
+ }
29
+ /** Model id for the policy hash (#1974). */
30
+ get modelId() {
31
+ return this.model;
32
+ }
33
+ /** Entity-group label set for the policy hash (#1974). */
34
+ get labelSet() {
35
+ return [...this.entityGroups].sort((a, b) => a.localeCompare(b));
36
+ }
37
+ /** Eagerly load the model (e.g. at daemon boot) so the first publish isn't slow. */
38
+ async init() {
39
+ await this.ensureReady();
40
+ }
41
+ ensureReady() {
42
+ if (this.pipe)
43
+ return Promise.resolve(this.pipe);
44
+ if (!this.initPromise) {
45
+ this.initPromise = (async () => {
46
+ const { pipeline } = await import('@huggingface/transformers');
47
+ const pipe = (await pipeline('token-classification', this.model));
48
+ this.pipe = pipe;
49
+ return pipe;
50
+ })();
51
+ }
52
+ return this.initPromise;
53
+ }
54
+ async detect(text) {
55
+ if (!text.trim())
56
+ return [];
57
+ const pipe = await this.ensureReady();
58
+ const results = await pipe(text, { aggregation_strategy: 'simple' });
59
+ const out = [];
60
+ const seen = new Set();
61
+ for (const r of results) {
62
+ if (r.score < this.threshold)
63
+ continue;
64
+ if (!this.entityGroups.has(r.entity_group))
65
+ continue;
66
+ const t = r.word.trim();
67
+ if (!t)
68
+ continue;
69
+ const key = `${r.entity_group}:${t}`;
70
+ if (seen.has(key))
71
+ continue;
72
+ seen.add(key);
73
+ out.push({ label: r.entity_group, text: t, score: r.score });
74
+ }
75
+ return out;
76
+ }
77
+ }