@bli-cockpit/telemetry-core 0.1.41 → 0.1.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/collector-heartbeat.d.ts +4 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/memory-install-receipt.d.ts +4 -0
- package/dist/memory-install-receipt.js +24 -0
- package/dist/outbound-guard.d.ts +77 -0
- package/dist/outbound-guard.js +119 -0
- package/dist/secret-guards-content-rules.d.ts +60 -0
- package/dist/secret-guards-content-rules.js +122 -0
- package/dist/secret-guards-masking.d.ts +32 -0
- package/dist/secret-guards-masking.js +240 -0
- package/dist/secret-guards-metadata.d.ts +102 -0
- package/dist/secret-guards-metadata.js +192 -0
- package/dist/secret-guards-path-rules.d.ts +22 -0
- package/dist/secret-guards-path-rules.js +24 -0
- package/dist/secret-guards.d.ts +23 -107
- package/dist/secret-guards.js +24 -489
- package/package.json +1 -1
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import { SECRET_ASSIGNMENT_SOURCE, SECRET_LIKE_CONTENT_PATTERNS, SECRET_NAME_MATCH_SOURCE, } from "./secret-guards-content-rules.js";
|
|
2
|
+
import { RawEvidenceRedactionMetadataSchema } from "./secret-guards-metadata.js";
|
|
3
|
+
// Same fence as the detector (BLI-3116) — the two must agree or a prefixed
|
|
4
|
+
// name would be detected and then left unredacted, which costs whole sessions
|
|
5
|
+
// (see findSecretRedactionMatches).
|
|
6
|
+
const SECRET_ASSIGNMENT_REDACTION_PATTERN = new RegExp(`(${SECRET_NAME_MATCH_SOURCE}${SECRET_ASSIGNMENT_SOURCE})([A-Za-z0-9_./+=-]{12,})`, "gi");
|
|
7
|
+
const SECRET_REDACTION_RULES = [
|
|
8
|
+
{
|
|
9
|
+
ruleId: "credential_assignment",
|
|
10
|
+
pattern: SECRET_ASSIGNMENT_REDACTION_PATTERN,
|
|
11
|
+
secretGroup: 2,
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
// The body stops at `{`, and excludes nothing else (BLI-4168).
|
|
15
|
+
//
|
|
16
|
+
// `[\s\S]*?` used to sit here, which gave the rule no notion of where one
|
|
17
|
+
// record ends. Transcripts are JSONL, so a key whose `-----END-----` never
|
|
18
|
+
// arrived — a truncated write, a killed session — paired with the next
|
|
19
|
+
// `-----END-----` many records later and collapsed everything between into
|
|
20
|
+
// one `[REDACTED:private_key_block]`. That over-masks: nothing leaked, but
|
|
21
|
+
// a session's worth of legitimate records was destroyed silently, because
|
|
22
|
+
// a redaction looks exactly like the guard working.
|
|
23
|
+
//
|
|
24
|
+
// `{` is the JSONL record boundary: it is what opens the next record, and a
|
|
25
|
+
// PEM body is base64, so it cannot appear inside a key. This is JSONL-record
|
|
26
|
+
// scoping specifically, and JSONL is the ONLY shape it fixes. The same
|
|
27
|
+
// function also receives plain text — a raw `.pem`, a git diff, a log tail,
|
|
28
|
+
// YAML, CSV, a markdown note — where `{` is not a boundary at all. There it
|
|
29
|
+
// splits two ways. Where a `{` does happen to fall between the markers it
|
|
30
|
+
// merely ends the match early, leaving a narrow residue; that residue is the
|
|
31
|
+
// price of the scope. Where NO `{` falls between an unterminated BEGIN and a
|
|
32
|
+
// later stray END — the ordinary case for every one of those formats — the
|
|
33
|
+
// match is still unbounded and the original over-masking is FULLY ALIVE.
|
|
34
|
+
// Measured on six shapes, each an unterminated key, then a bystander record,
|
|
35
|
+
// then a stray END: JSONL preserved the bystander and the other five
|
|
36
|
+
// destroyed it. This rule does not fix that case. BLI-4216 carries it.
|
|
37
|
+
//
|
|
38
|
+
// It MUST stay a DENYLIST of that one character. An allowlist of the PEM
|
|
39
|
+
// alphabet was tried here and reverted the same day: one character outside
|
|
40
|
+
// the class anywhere between BEGIN and END — a `.`, a `[info] ` log prefix,
|
|
41
|
+
// an `N | ` gutter, an ANSI reset, a diff hunk header — makes the whole
|
|
42
|
+
// rule fail to match, and the body is then left in PLAINTEXT while
|
|
43
|
+
// `private_key_header` masks only the BEGIN line and
|
|
44
|
+
// `containsSecretLikeContent` reports the residue clean, so
|
|
45
|
+
// `secret_redaction_failed` never fires. That is a leak, and a leak is
|
|
46
|
+
// strictly worse than the over-masking this rule exists to fix. The
|
|
47
|
+
// leak-direction tests in `secret-guards.test.ts` pin it.
|
|
48
|
+
ruleId: "private_key_block",
|
|
49
|
+
pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[^{]*?-----END [A-Z ]*PRIVATE KEY-----/g,
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
ruleId: "private_key_header",
|
|
53
|
+
pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----/g,
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
ruleId: "bearer_token",
|
|
57
|
+
pattern: /\b(Bearer\s+)([A-Za-z0-9._~+/=-]{16,})/gi,
|
|
58
|
+
secretGroup: 2,
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
ruleId: "jwt",
|
|
62
|
+
pattern: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
ruleId: "stripe_token",
|
|
66
|
+
pattern: /\b(?:sk|pk)_(?:live|test)_[A-Za-z0-9]{16,}/g,
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
ruleId: "openai_token",
|
|
70
|
+
pattern: /\bsk-[A-Za-z0-9_-]{16,}/g,
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
ruleId: "github_token",
|
|
74
|
+
pattern: /\b(?:gh[pousr]_[A-Za-z0-9_]{16,}|github_pat_[A-Za-z0-9_]{20,})/g,
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
ruleId: "linear_token",
|
|
78
|
+
pattern: /\blin_api_[A-Za-z0-9]{16,}/g,
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
ruleId: "supabase_secret_token",
|
|
82
|
+
pattern: /\bsb_secret_[A-Za-z0-9_]{16,}/g,
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
ruleId: "slack_token",
|
|
86
|
+
pattern: /\bxox[baprs]-[A-Za-z0-9-]{16,}/g,
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
ruleId: "aws_access_key_id",
|
|
90
|
+
pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
ruleId: "langfuse_token",
|
|
94
|
+
pattern: /\b(?:pk|sk)-lf-[A-Za-z0-9_-]{16,}\b/gi,
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
ruleId: "mem0_token",
|
|
98
|
+
pattern: /\bm0[-_][A-Za-z0-9_-]{16,}\b/gi,
|
|
99
|
+
},
|
|
100
|
+
];
|
|
101
|
+
/**
|
|
102
|
+
* Mask every secret-shaped span and keep the rest of the text, because a
|
|
103
|
+
* session is never dropped for carrying one: this returns the redacted bytes
|
|
104
|
+
* plus the receipt that says which rules fired, where, and how much was
|
|
105
|
+
* replaced.
|
|
106
|
+
*/
|
|
107
|
+
export function redactSecretLikeContent(value, options = {}) {
|
|
108
|
+
const matches = findSecretRedactionMatches(value);
|
|
109
|
+
if (matches.length === 0) {
|
|
110
|
+
return { redacted: false, text: value, metadata: null };
|
|
111
|
+
}
|
|
112
|
+
let text = "";
|
|
113
|
+
let cursor = 0;
|
|
114
|
+
const ruleCounts = new Map();
|
|
115
|
+
for (const match of matches) {
|
|
116
|
+
text += value.slice(cursor, match.start);
|
|
117
|
+
text += `[REDACTED:${match.ruleId}]`;
|
|
118
|
+
cursor = match.end;
|
|
119
|
+
const existing = ruleCounts.get(match.ruleId) ?? {
|
|
120
|
+
rule_id: match.ruleId,
|
|
121
|
+
match_count: 0,
|
|
122
|
+
redacted_char_count: 0,
|
|
123
|
+
};
|
|
124
|
+
existing.match_count += 1;
|
|
125
|
+
existing.redacted_char_count += match.end - match.start;
|
|
126
|
+
ruleCounts.set(match.ruleId, existing);
|
|
127
|
+
}
|
|
128
|
+
text += value.slice(cursor);
|
|
129
|
+
const metadata = RawEvidenceRedactionMetadataSchema.parse({
|
|
130
|
+
schema_version: "raw-evidence-redaction.v1",
|
|
131
|
+
status: "sanitized",
|
|
132
|
+
mode: "deterministic_text_replacement",
|
|
133
|
+
applied_by: [options.appliedBy ?? "local_collector"],
|
|
134
|
+
rule_counts: [...ruleCounts.values()].sort((a, b) => a.rule_id.localeCompare(b.rule_id)),
|
|
135
|
+
secret_like_match_count: matches.length,
|
|
136
|
+
redacted_fields: options.redactedFields ?? [],
|
|
137
|
+
redacted_ranges: matches.slice(0, 200).map((match) => ({
|
|
138
|
+
start: match.start,
|
|
139
|
+
end: match.end,
|
|
140
|
+
rule_id: match.ruleId,
|
|
141
|
+
})),
|
|
142
|
+
});
|
|
143
|
+
return { redacted: true, text, metadata };
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Every region the detector considers a secret, redacted — precisely where a
|
|
147
|
+
* rule can pinpoint the value, coarsely where none can.
|
|
148
|
+
*
|
|
149
|
+
* `containsSecretLikeContent` and `SECRET_REDACTION_RULES` are two lists that
|
|
150
|
+
* have to agree, and nothing used to make them. When the detector matched and
|
|
151
|
+
* no rule produced a range, `redactSecretLikeContent` reported
|
|
152
|
+
* `redacted: false`, and the caller dropped the entire session file
|
|
153
|
+
* (`secret_like_content_guard`). Drift between the lists cost whole sessions:
|
|
154
|
+
* 78 fleet-wide as of 2026-08-14, 3 of Viet's 17 Claude sessions.
|
|
155
|
+
*
|
|
156
|
+
* The two lists can still drift — regexes are like that — but drift now costs
|
|
157
|
+
* precision instead of evidence. A detector hit with no matching rule redacts
|
|
158
|
+
* the whole matched span, so `redactSecretLikeContent` always reports the
|
|
159
|
+
* redaction it performed and the transcript survives with the value removed.
|
|
160
|
+
*/
|
|
161
|
+
function findSecretRedactionMatches(value) {
|
|
162
|
+
const accepted = acceptNonOverlapping(secretRuleCandidates(value));
|
|
163
|
+
const uncovered = detectorFallbackCandidates(value).filter((candidate) => !overlapsAny(candidate, accepted));
|
|
164
|
+
if (uncovered.length === 0)
|
|
165
|
+
return accepted;
|
|
166
|
+
return acceptNonOverlapping([...accepted, ...uncovered]);
|
|
167
|
+
}
|
|
168
|
+
function secretRuleCandidates(value) {
|
|
169
|
+
const candidates = [];
|
|
170
|
+
SECRET_REDACTION_RULES.forEach((rule, priority) => {
|
|
171
|
+
const pattern = cloneGlobalPattern(rule.pattern);
|
|
172
|
+
for (const match of value.matchAll(pattern)) {
|
|
173
|
+
const fullMatch = match[0];
|
|
174
|
+
const matchIndex = match.index;
|
|
175
|
+
if (!fullMatch || matchIndex === undefined)
|
|
176
|
+
continue;
|
|
177
|
+
const secret = rule.secretGroup ? match[rule.secretGroup] : fullMatch;
|
|
178
|
+
if (!secret)
|
|
179
|
+
continue;
|
|
180
|
+
const relativeStart = fullMatch.indexOf(secret);
|
|
181
|
+
if (relativeStart < 0)
|
|
182
|
+
continue;
|
|
183
|
+
candidates.push({
|
|
184
|
+
start: matchIndex + relativeStart,
|
|
185
|
+
end: matchIndex + relativeStart + secret.length,
|
|
186
|
+
ruleId: rule.ruleId,
|
|
187
|
+
priority,
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
return candidates;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Ranked below every real rule, so a rule that can name the credential always
|
|
195
|
+
* wins and the coarse span is only used where nothing else reached.
|
|
196
|
+
*/
|
|
197
|
+
function detectorFallbackCandidates(value) {
|
|
198
|
+
const candidates = [];
|
|
199
|
+
const basePriority = SECRET_REDACTION_RULES.length;
|
|
200
|
+
SECRET_LIKE_CONTENT_PATTERNS.forEach((pattern, offset) => {
|
|
201
|
+
for (const match of value.matchAll(cloneGlobalPattern(pattern))) {
|
|
202
|
+
const fullMatch = match[0];
|
|
203
|
+
const matchIndex = match.index;
|
|
204
|
+
if (!fullMatch || matchIndex === undefined)
|
|
205
|
+
continue;
|
|
206
|
+
candidates.push({
|
|
207
|
+
start: matchIndex,
|
|
208
|
+
end: matchIndex + fullMatch.length,
|
|
209
|
+
ruleId: "unnamed_secret_shape",
|
|
210
|
+
priority: basePriority + offset,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
return candidates;
|
|
215
|
+
}
|
|
216
|
+
function overlapsAny(candidate, accepted) {
|
|
217
|
+
return accepted.some((other) => candidate.start < other.end && other.start < candidate.end);
|
|
218
|
+
}
|
|
219
|
+
function acceptNonOverlapping(candidates) {
|
|
220
|
+
const ordered = [...candidates].sort((a, b) => {
|
|
221
|
+
if (a.start !== b.start)
|
|
222
|
+
return a.start - b.start;
|
|
223
|
+
if (a.priority !== b.priority)
|
|
224
|
+
return a.priority - b.priority;
|
|
225
|
+
return b.end - b.start - (a.end - a.start);
|
|
226
|
+
});
|
|
227
|
+
const accepted = [];
|
|
228
|
+
let lastEnd = -1;
|
|
229
|
+
for (const candidate of ordered) {
|
|
230
|
+
if (candidate.start < lastEnd)
|
|
231
|
+
continue;
|
|
232
|
+
accepted.push(candidate);
|
|
233
|
+
lastEnd = candidate.end;
|
|
234
|
+
}
|
|
235
|
+
return accepted;
|
|
236
|
+
}
|
|
237
|
+
function cloneGlobalPattern(pattern) {
|
|
238
|
+
const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`;
|
|
239
|
+
return new RegExp(pattern.source, flags);
|
|
240
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* What a secret scan RECORDS: the redaction receipt every uploaded object
|
|
4
|
+
* carries, and the one builder of the clean half of it.
|
|
5
|
+
*
|
|
6
|
+
* Owned by `secret-guards.ts` (BLI-3986); the rules that produce these records
|
|
7
|
+
* live in `secret-guards-content-rules.ts` and `secret-guards-masking.ts`.
|
|
8
|
+
* Nothing here decides what a secret IS — it decides what a reader downstream
|
|
9
|
+
* is told about a scan that already ran, which is why the per-status
|
|
10
|
+
* invariants below are the load-bearing part: a record that claims a
|
|
11
|
+
* sanitization that never happened is worse than a missing record.
|
|
12
|
+
*/
|
|
13
|
+
export declare const RawEvidenceRedactionSourceSchema: z.ZodEnum<{
|
|
14
|
+
local_collector: "local_collector";
|
|
15
|
+
server_commit: "server_commit";
|
|
16
|
+
legacy_upload: "legacy_upload";
|
|
17
|
+
}>;
|
|
18
|
+
export type RawEvidenceRedactionSource = z.infer<typeof RawEvidenceRedactionSourceSchema>;
|
|
19
|
+
export declare const RawEvidenceRedactionRuleCountSchema: z.ZodObject<{
|
|
20
|
+
rule_id: z.ZodString;
|
|
21
|
+
match_count: z.ZodNumber;
|
|
22
|
+
redacted_char_count: z.ZodNumber;
|
|
23
|
+
}, z.core.$strict>;
|
|
24
|
+
export type RawEvidenceRedactionRuleCount = z.infer<typeof RawEvidenceRedactionRuleCountSchema>;
|
|
25
|
+
export declare const RawEvidenceRedactionRangeSchema: z.ZodObject<{
|
|
26
|
+
start: z.ZodNumber;
|
|
27
|
+
end: z.ZodNumber;
|
|
28
|
+
rule_id: z.ZodString;
|
|
29
|
+
}, z.core.$strict>;
|
|
30
|
+
export type RawEvidenceRedactionRange = z.infer<typeof RawEvidenceRedactionRangeSchema>;
|
|
31
|
+
/**
|
|
32
|
+
* The verdict of a secret scan — NOT "did the bytes change" (BLI-3277).
|
|
33
|
+
*
|
|
34
|
+
* `sanitized` means the scan matched and the uploaded bytes differ from what was
|
|
35
|
+
* read. `scanned_clean` means the same scan ran over the same bytes and matched
|
|
36
|
+
* nothing, so the uploaded bytes are the original ones. The second value exists
|
|
37
|
+
* because every reader downstream asks "was this scanned?", and for 70% of
|
|
38
|
+
* transcripts — the ones with no secret in them — the honest answer used to be
|
|
39
|
+
* an absent record, which reads identically to "nobody ever looked".
|
|
40
|
+
*/
|
|
41
|
+
export declare const RawEvidenceRedactionStatusSchema: z.ZodEnum<{
|
|
42
|
+
sanitized: "sanitized";
|
|
43
|
+
scanned_clean: "scanned_clean";
|
|
44
|
+
}>;
|
|
45
|
+
export type RawEvidenceRedactionStatus = z.infer<typeof RawEvidenceRedactionStatusSchema>;
|
|
46
|
+
export declare const RawEvidenceRedactionMetadataSchema: z.ZodObject<{
|
|
47
|
+
schema_version: z.ZodLiteral<"raw-evidence-redaction.v1">;
|
|
48
|
+
status: z.ZodEnum<{
|
|
49
|
+
sanitized: "sanitized";
|
|
50
|
+
scanned_clean: "scanned_clean";
|
|
51
|
+
}>;
|
|
52
|
+
mode: z.ZodLiteral<"deterministic_text_replacement">;
|
|
53
|
+
applied_by: z.ZodArray<z.ZodEnum<{
|
|
54
|
+
local_collector: "local_collector";
|
|
55
|
+
server_commit: "server_commit";
|
|
56
|
+
legacy_upload: "legacy_upload";
|
|
57
|
+
}>>;
|
|
58
|
+
rule_counts: z.ZodArray<z.ZodObject<{
|
|
59
|
+
rule_id: z.ZodString;
|
|
60
|
+
match_count: z.ZodNumber;
|
|
61
|
+
redacted_char_count: z.ZodNumber;
|
|
62
|
+
}, z.core.$strict>>;
|
|
63
|
+
secret_like_match_count: z.ZodNumber;
|
|
64
|
+
redacted_fields: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
65
|
+
redacted_ranges: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
66
|
+
start: z.ZodNumber;
|
|
67
|
+
end: z.ZodNumber;
|
|
68
|
+
rule_id: z.ZodString;
|
|
69
|
+
}, z.core.$strict>>>;
|
|
70
|
+
original_content_hash_sha256: z.ZodOptional<z.ZodString>;
|
|
71
|
+
sanitized_content_hash_sha256: z.ZodOptional<z.ZodString>;
|
|
72
|
+
original_byte_size: z.ZodOptional<z.ZodNumber>;
|
|
73
|
+
sanitized_byte_size: z.ZodOptional<z.ZodNumber>;
|
|
74
|
+
applied_at: z.ZodOptional<z.ZodString>;
|
|
75
|
+
}, z.core.$strict>;
|
|
76
|
+
export type RawEvidenceRedactionMetadata = z.infer<typeof RawEvidenceRedactionMetadataSchema>;
|
|
77
|
+
/**
|
|
78
|
+
* The receipt for a file the scan cleared (BLI-3277, moved here by BLI-3280).
|
|
79
|
+
*
|
|
80
|
+
* Same schema, same `mode` — the deterministic ruleset is what ran — with the
|
|
81
|
+
* verdict `scanned_clean` and zero of everything else. The content fields are
|
|
82
|
+
* the point: an evidence ref is only readable downstream when its redaction
|
|
83
|
+
* record hashes the bytes that are actually in the bucket, and for a clean file
|
|
84
|
+
* those are the original bytes, so both halves carry the same digest and size.
|
|
85
|
+
*
|
|
86
|
+
* It lives in telemetry-core rather than in the collector because two callers
|
|
87
|
+
* now build this exact record and they must never drift: the collector writes
|
|
88
|
+
* it at upload time, and the BLI-3280 operator backfill writes it onto refs
|
|
89
|
+
* that were stored before the receipt existed. Both must satisfy the same
|
|
90
|
+
* `sanitized_content_hash_sha256 === content_hash_sha256` /
|
|
91
|
+
* `sanitized_byte_size === byte_size` equality the digest reader checks, and a
|
|
92
|
+
* second copy of this function is a second chance to get that wrong.
|
|
93
|
+
*
|
|
94
|
+
* `appliedBy` is who ran the scan, and it is not decoration: a backfill passes
|
|
95
|
+
* `legacy_upload` so a reader can tell "the collector scanned this before it
|
|
96
|
+
* uploaded" apart from "an operator scanned the stored bytes afterwards".
|
|
97
|
+
*/
|
|
98
|
+
export declare function scannedCleanRedactionMetadata(bytes: Uint8Array, options?: {
|
|
99
|
+
appliedBy?: RawEvidenceRedactionSource;
|
|
100
|
+
/** ISO-8601 with offset. Omitted entirely when absent (BLI-3290). */
|
|
101
|
+
appliedAt?: string;
|
|
102
|
+
}): RawEvidenceRedactionMetadata;
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { IsoDateTimeSchema } from "./common.js";
|
|
4
|
+
/**
|
|
5
|
+
* What a secret scan RECORDS: the redaction receipt every uploaded object
|
|
6
|
+
* carries, and the one builder of the clean half of it.
|
|
7
|
+
*
|
|
8
|
+
* Owned by `secret-guards.ts` (BLI-3986); the rules that produce these records
|
|
9
|
+
* live in `secret-guards-content-rules.ts` and `secret-guards-masking.ts`.
|
|
10
|
+
* Nothing here decides what a secret IS — it decides what a reader downstream
|
|
11
|
+
* is told about a scan that already ran, which is why the per-status
|
|
12
|
+
* invariants below are the load-bearing part: a record that claims a
|
|
13
|
+
* sanitization that never happened is worse than a missing record.
|
|
14
|
+
*/
|
|
15
|
+
export const RawEvidenceRedactionSourceSchema = z.enum([
|
|
16
|
+
"local_collector",
|
|
17
|
+
"server_commit",
|
|
18
|
+
"legacy_upload",
|
|
19
|
+
]);
|
|
20
|
+
export const RawEvidenceRedactionRuleCountSchema = z
|
|
21
|
+
.object({
|
|
22
|
+
rule_id: z.string().trim().min(1).max(80),
|
|
23
|
+
match_count: z.number().int().nonnegative(),
|
|
24
|
+
redacted_char_count: z.number().int().nonnegative(),
|
|
25
|
+
})
|
|
26
|
+
.strict();
|
|
27
|
+
export const RawEvidenceRedactionRangeSchema = z
|
|
28
|
+
.object({
|
|
29
|
+
start: z.number().int().nonnegative(),
|
|
30
|
+
end: z.number().int().positive(),
|
|
31
|
+
rule_id: z.string().trim().min(1).max(80),
|
|
32
|
+
})
|
|
33
|
+
.strict()
|
|
34
|
+
.superRefine((range, context) => {
|
|
35
|
+
if (range.end <= range.start) {
|
|
36
|
+
context.addIssue({
|
|
37
|
+
code: "custom",
|
|
38
|
+
message: "redaction range end must be greater than start",
|
|
39
|
+
path: ["end"],
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
/**
|
|
44
|
+
* The verdict of a secret scan — NOT "did the bytes change" (BLI-3277).
|
|
45
|
+
*
|
|
46
|
+
* `sanitized` means the scan matched and the uploaded bytes differ from what was
|
|
47
|
+
* read. `scanned_clean` means the same scan ran over the same bytes and matched
|
|
48
|
+
* nothing, so the uploaded bytes are the original ones. The second value exists
|
|
49
|
+
* because every reader downstream asks "was this scanned?", and for 70% of
|
|
50
|
+
* transcripts — the ones with no secret in them — the honest answer used to be
|
|
51
|
+
* an absent record, which reads identically to "nobody ever looked".
|
|
52
|
+
*/
|
|
53
|
+
export const RawEvidenceRedactionStatusSchema = z.enum([
|
|
54
|
+
"sanitized",
|
|
55
|
+
"scanned_clean",
|
|
56
|
+
]);
|
|
57
|
+
export const RawEvidenceRedactionMetadataSchema = z
|
|
58
|
+
.object({
|
|
59
|
+
schema_version: z.literal("raw-evidence-redaction.v1"),
|
|
60
|
+
status: RawEvidenceRedactionStatusSchema,
|
|
61
|
+
mode: z.literal("deterministic_text_replacement"),
|
|
62
|
+
applied_by: z.array(RawEvidenceRedactionSourceSchema).min(1),
|
|
63
|
+
rule_counts: z.array(RawEvidenceRedactionRuleCountSchema),
|
|
64
|
+
secret_like_match_count: z.number().int().nonnegative(),
|
|
65
|
+
redacted_fields: z.array(z.string().trim().min(1).max(160)).default([]),
|
|
66
|
+
redacted_ranges: z.array(RawEvidenceRedactionRangeSchema).max(200).default([]),
|
|
67
|
+
original_content_hash_sha256: z.string().regex(/^[a-f0-9]{64}$/).optional(),
|
|
68
|
+
sanitized_content_hash_sha256: z.string().regex(/^[a-f0-9]{64}$/).optional(),
|
|
69
|
+
original_byte_size: z.number().int().nonnegative().optional(),
|
|
70
|
+
sanitized_byte_size: z.number().int().nonnegative().optional(),
|
|
71
|
+
/**
|
|
72
|
+
* When the scan ran, ISO-8601 with an offset (BLI-3290).
|
|
73
|
+
*
|
|
74
|
+
* Optional, and it stays optional: every receipt written before this field
|
|
75
|
+
* existed is still valid, and the collector does not stamp it — at upload
|
|
76
|
+
* time the ref's own `received_at` already answers "when". It exists for
|
|
77
|
+
* receipts written over bytes that were stored long ago, where the ref's
|
|
78
|
+
* timestamps describe the ORIGINAL upload and nothing in the record would
|
|
79
|
+
* otherwise say when an operator re-scanned or re-masked the object.
|
|
80
|
+
*/
|
|
81
|
+
applied_at: IsoDateTimeSchema.optional(),
|
|
82
|
+
})
|
|
83
|
+
.strict()
|
|
84
|
+
// The per-status invariants the old `z.literal("sanitized")` + `.min(1)` +
|
|
85
|
+
// `.positive()` shape enforced structurally. They still hold for a sanitized
|
|
86
|
+
// record; a clean one has to prove the opposite — no rule fired, no range was
|
|
87
|
+
// replaced, no field was masked, and the bytes came out the way they went in.
|
|
88
|
+
.superRefine((metadata, context) => {
|
|
89
|
+
if (metadata.status === "sanitized") {
|
|
90
|
+
if (metadata.rule_counts.length === 0) {
|
|
91
|
+
context.addIssue({
|
|
92
|
+
code: "custom",
|
|
93
|
+
message: "a sanitized redaction must name at least one rule",
|
|
94
|
+
path: ["rule_counts"],
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
if (metadata.secret_like_match_count < 1) {
|
|
98
|
+
context.addIssue({
|
|
99
|
+
code: "custom",
|
|
100
|
+
message: "a sanitized redaction must count at least one match",
|
|
101
|
+
path: ["secret_like_match_count"],
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
if (metadata.rule_counts.length > 0) {
|
|
107
|
+
context.addIssue({
|
|
108
|
+
code: "custom",
|
|
109
|
+
message: "a clean scan cannot name a rule that fired",
|
|
110
|
+
path: ["rule_counts"],
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
if (metadata.secret_like_match_count !== 0) {
|
|
114
|
+
context.addIssue({
|
|
115
|
+
code: "custom",
|
|
116
|
+
message: "a clean scan cannot count a match",
|
|
117
|
+
path: ["secret_like_match_count"],
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
if (metadata.redacted_ranges.length > 0) {
|
|
121
|
+
context.addIssue({
|
|
122
|
+
code: "custom",
|
|
123
|
+
message: "a clean scan cannot redact a range",
|
|
124
|
+
path: ["redacted_ranges"],
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
if (metadata.redacted_fields.length > 0) {
|
|
128
|
+
context.addIssue({
|
|
129
|
+
code: "custom",
|
|
130
|
+
message: "a clean scan cannot redact a field",
|
|
131
|
+
path: ["redacted_fields"],
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
if (metadata.original_content_hash_sha256 !== undefined &&
|
|
135
|
+
metadata.sanitized_content_hash_sha256 !== undefined &&
|
|
136
|
+
metadata.original_content_hash_sha256 !==
|
|
137
|
+
metadata.sanitized_content_hash_sha256) {
|
|
138
|
+
context.addIssue({
|
|
139
|
+
code: "custom",
|
|
140
|
+
message: "a clean scan cannot change the content hash",
|
|
141
|
+
path: ["sanitized_content_hash_sha256"],
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
if (metadata.original_byte_size !== undefined &&
|
|
145
|
+
metadata.sanitized_byte_size !== undefined &&
|
|
146
|
+
metadata.original_byte_size !== metadata.sanitized_byte_size) {
|
|
147
|
+
context.addIssue({
|
|
148
|
+
code: "custom",
|
|
149
|
+
message: "a clean scan cannot change the byte size",
|
|
150
|
+
path: ["sanitized_byte_size"],
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
/**
|
|
155
|
+
* The receipt for a file the scan cleared (BLI-3277, moved here by BLI-3280).
|
|
156
|
+
*
|
|
157
|
+
* Same schema, same `mode` — the deterministic ruleset is what ran — with the
|
|
158
|
+
* verdict `scanned_clean` and zero of everything else. The content fields are
|
|
159
|
+
* the point: an evidence ref is only readable downstream when its redaction
|
|
160
|
+
* record hashes the bytes that are actually in the bucket, and for a clean file
|
|
161
|
+
* those are the original bytes, so both halves carry the same digest and size.
|
|
162
|
+
*
|
|
163
|
+
* It lives in telemetry-core rather than in the collector because two callers
|
|
164
|
+
* now build this exact record and they must never drift: the collector writes
|
|
165
|
+
* it at upload time, and the BLI-3280 operator backfill writes it onto refs
|
|
166
|
+
* that were stored before the receipt existed. Both must satisfy the same
|
|
167
|
+
* `sanitized_content_hash_sha256 === content_hash_sha256` /
|
|
168
|
+
* `sanitized_byte_size === byte_size` equality the digest reader checks, and a
|
|
169
|
+
* second copy of this function is a second chance to get that wrong.
|
|
170
|
+
*
|
|
171
|
+
* `appliedBy` is who ran the scan, and it is not decoration: a backfill passes
|
|
172
|
+
* `legacy_upload` so a reader can tell "the collector scanned this before it
|
|
173
|
+
* uploaded" apart from "an operator scanned the stored bytes afterwards".
|
|
174
|
+
*/
|
|
175
|
+
export function scannedCleanRedactionMetadata(bytes, options = {}) {
|
|
176
|
+
const digest = createHash("sha256").update(bytes).digest("hex");
|
|
177
|
+
return {
|
|
178
|
+
schema_version: "raw-evidence-redaction.v1",
|
|
179
|
+
status: "scanned_clean",
|
|
180
|
+
mode: "deterministic_text_replacement",
|
|
181
|
+
applied_by: [options.appliedBy ?? "local_collector"],
|
|
182
|
+
rule_counts: [],
|
|
183
|
+
secret_like_match_count: 0,
|
|
184
|
+
redacted_fields: [],
|
|
185
|
+
redacted_ranges: [],
|
|
186
|
+
original_content_hash_sha256: digest,
|
|
187
|
+
sanitized_content_hash_sha256: digest,
|
|
188
|
+
original_byte_size: bytes.byteLength,
|
|
189
|
+
sanitized_byte_size: bytes.byteLength,
|
|
190
|
+
...(options.appliedAt === undefined ? {} : { applied_at: options.appliedAt }),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which file NAMES are never opened at all.
|
|
3
|
+
*
|
|
4
|
+
* Owned by `secret-guards.ts` (BLI-3986). This is the cheapest of the guard's
|
|
5
|
+
* questions and the only one answered without reading a byte of content, which
|
|
6
|
+
* is why it stays broad and why it is asked first: the collector's attribution
|
|
7
|
+
* pass, its collection pass and the path walk all share this one definition
|
|
8
|
+
* instead of re-declaring a pattern each.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* File-path / file-name segments that must never be read at all. This is a
|
|
12
|
+
* NAME matcher (it never reads file content), so it stays broad: a file called
|
|
13
|
+
* `service-role.json` or a dotenv file is skipped on its name alone. The
|
|
14
|
+
* collector imports this instead of re-declaring it, so attribution,
|
|
15
|
+
* collection, and the path walk share one definition.
|
|
16
|
+
*/
|
|
17
|
+
export declare const SECRET_FILE_SEGMENT_PATTERN: RegExp;
|
|
18
|
+
/**
|
|
19
|
+
* Does this path segment name a file the collector must not open? One place
|
|
20
|
+
* asks the pattern so every caller skips the same names.
|
|
21
|
+
*/
|
|
22
|
+
export declare function isSecretLikePathSegment(value: string): boolean;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which file NAMES are never opened at all.
|
|
3
|
+
*
|
|
4
|
+
* Owned by `secret-guards.ts` (BLI-3986). This is the cheapest of the guard's
|
|
5
|
+
* questions and the only one answered without reading a byte of content, which
|
|
6
|
+
* is why it stays broad and why it is asked first: the collector's attribution
|
|
7
|
+
* pass, its collection pass and the path walk all share this one definition
|
|
8
|
+
* instead of re-declaring a pattern each.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* File-path / file-name segments that must never be read at all. This is a
|
|
12
|
+
* NAME matcher (it never reads file content), so it stays broad: a file called
|
|
13
|
+
* `service-role.json` or a dotenv file is skipped on its name alone. The
|
|
14
|
+
* collector imports this instead of re-declaring it, so attribution,
|
|
15
|
+
* collection, and the path walk share one definition.
|
|
16
|
+
*/
|
|
17
|
+
export const SECRET_FILE_SEGMENT_PATTERN = /(^|[/\\])(?:\.env(?:\..*)?|.*(?:secret|credential|private[_-]?key|service[_-]?role).*)$/i;
|
|
18
|
+
/**
|
|
19
|
+
* Does this path segment name a file the collector must not open? One place
|
|
20
|
+
* asks the pattern so every caller skips the same names.
|
|
21
|
+
*/
|
|
22
|
+
export function isSecretLikePathSegment(value) {
|
|
23
|
+
return SECRET_FILE_SEGMENT_PATTERN.test(value);
|
|
24
|
+
}
|