@bli-cockpit/telemetry-core 0.1.41 → 0.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/secret-guards-content-rules.d.ts +60 -0
- package/dist/secret-guards-content-rules.js +122 -0
- package/dist/secret-guards-masking.d.ts +32 -0
- package/dist/secret-guards-masking.js +206 -0
- package/dist/secret-guards-metadata.d.ts +102 -0
- package/dist/secret-guards-metadata.js +192 -0
- package/dist/secret-guards-path-rules.d.ts +22 -0
- package/dist/secret-guards-path-rules.js +24 -0
- package/dist/secret-guards.d.ts +23 -107
- package/dist/secret-guards.js +24 -489
- package/package.json +1 -1
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which CONTENT looks like a secret — the detector, in the order it decides.
|
|
3
|
+
*
|
|
4
|
+
* Owned by `secret-guards.ts` (BLI-3986). Two kinds of rule live here and the
|
|
5
|
+
* difference between them is the whole D13 precision argument: a credential
|
|
6
|
+
* NAME is a leak only when it carries an assignment and an opaque value, while
|
|
7
|
+
* a self-identifying VALUE (a PEM block, a JWT, `ghp_…`, `AKIA…`) is a leak
|
|
8
|
+
* wherever it appears. `SECRET_LIKE_CONTENT_PATTERNS` is that order — the
|
|
9
|
+
* name-with-value rule first, then the value shapes as listed — and
|
|
10
|
+
* `secret-guards-masking.ts` reads the same list, so a rule added here without
|
|
11
|
+
* a masking rule beside it still redacts (coarsely) rather than costing a
|
|
12
|
+
* session.
|
|
13
|
+
*
|
|
14
|
+
* `SECRET_NAME_MATCH_SOURCE` and `SECRET_ASSIGNMENT_SOURCE` are exported for
|
|
15
|
+
* that sibling alone and are deliberately NOT re-exported by
|
|
16
|
+
* `secret-guards.ts`: the masking rule has to be built from the same fence as
|
|
17
|
+
* the detector, and a second spelling of it is the BLI-3116 hole again.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Credential names live INSIDE longer identifiers, so `\b` is the wrong fence
|
|
21
|
+
* (BLI-3116).
|
|
22
|
+
*
|
|
23
|
+
* Every name fragment above used to be wrapped in `\b(?:…)\b`. Underscore is a
|
|
24
|
+
* word character, so there is no word boundary between `AWS_` and `SECRET`:
|
|
25
|
+
* `AWS_SECRET_ACCESS_KEY=AKIA…` — the most common real spelling of the most
|
|
26
|
+
* common real leak — never matched and uploaded unmasked from every fleet
|
|
27
|
+
* machine, while the bare `SECRET_ACCESS_KEY=…` matched fine. The same hole hid
|
|
28
|
+
* `AZURE_OPENAI_API_KEY`, `VITE_SUPABASE_ANON_KEY`, `GITHUB_ACCESS_TOKEN`,
|
|
29
|
+
* `MY_APP_CLIENT_SECRET` and every other prefixed spelling: the bug was the
|
|
30
|
+
* fence, not the vocabulary, so the whole list is fenced differently now.
|
|
31
|
+
*
|
|
32
|
+
* `_`, `-` and `.` are identifier JOINERS here, not boundaries:
|
|
33
|
+
*
|
|
34
|
+
* - leading `(?<![A-Za-z0-9])` — the fragment may begin right after a joiner or
|
|
35
|
+
* at a real boundary, but a letter or digit immediately before it still
|
|
36
|
+
* blocks the match (`notapikey=…` stays out, as before).
|
|
37
|
+
* - trailing `[A-Za-z0-9_-]{0,40}` — a bounded identifier tail, so
|
|
38
|
+
* `OPENAI_API_KEY_2=…` and `AWS_SECRET_ACCESS_KEY_ID=…` are seen too.
|
|
39
|
+
*
|
|
40
|
+
* Prose is unaffected because the assignment requirement below is unchanged:
|
|
41
|
+
* "rotate your secret access key" has no `=`/`:` + opaque value and is not a
|
|
42
|
+
* leak. The guard's job is assignments, not vocabulary.
|
|
43
|
+
*/
|
|
44
|
+
export declare const SECRET_NAME_MATCH_SOURCE: string;
|
|
45
|
+
/**
|
|
46
|
+
* The assignment operator, with the closing quote of a JSON/YAML key allowed
|
|
47
|
+
* before it. Found while auditing the fence above: `{"aws_secret_access_key":
|
|
48
|
+
* "…"}` never matched either, because the name was followed by `"` and the
|
|
49
|
+
* pattern demanded `[:=]` immediately. Transcripts are JSONL, so this is the
|
|
50
|
+
* shape a leaked credential most often has on the way in. The sibling redactor
|
|
51
|
+
* in harvest-analysis (`study/prepare.ts`) already allowed it; this one did not.
|
|
52
|
+
*/
|
|
53
|
+
export declare const SECRET_ASSIGNMENT_SOURCE = "[\"']?\\s*[:=]\\s*[\"']?";
|
|
54
|
+
export declare const SECRET_LIKE_CONTENT_PATTERNS: RegExp[];
|
|
55
|
+
/**
|
|
56
|
+
* Does this text carry something the fleet must not upload as it stands? The
|
|
57
|
+
* one question the collector and the ingest door both ask, so a laptop and the
|
|
58
|
+
* server can never disagree about which bytes were secret-shaped.
|
|
59
|
+
*/
|
|
60
|
+
export declare function containsSecretLikeContent(value: string): boolean;
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which CONTENT looks like a secret — the detector, in the order it decides.
|
|
3
|
+
*
|
|
4
|
+
* Owned by `secret-guards.ts` (BLI-3986). Two kinds of rule live here and the
|
|
5
|
+
* difference between them is the whole D13 precision argument: a credential
|
|
6
|
+
* NAME is a leak only when it carries an assignment and an opaque value, while
|
|
7
|
+
* a self-identifying VALUE (a PEM block, a JWT, `ghp_…`, `AKIA…`) is a leak
|
|
8
|
+
* wherever it appears. `SECRET_LIKE_CONTENT_PATTERNS` is that order — the
|
|
9
|
+
* name-with-value rule first, then the value shapes as listed — and
|
|
10
|
+
* `secret-guards-masking.ts` reads the same list, so a rule added here without
|
|
11
|
+
* a masking rule beside it still redacts (coarsely) rather than costing a
|
|
12
|
+
* session.
|
|
13
|
+
*
|
|
14
|
+
* `SECRET_NAME_MATCH_SOURCE` and `SECRET_ASSIGNMENT_SOURCE` are exported for
|
|
15
|
+
* that sibling alone and are deliberately NOT re-exported by
|
|
16
|
+
* `secret-guards.ts`: the masking rule has to be built from the same fence as
|
|
17
|
+
* the detector, and a second spelling of it is the BLI-3116 hole again.
|
|
18
|
+
*/
|
|
19
|
+
// Constructed from parts so the literal privileged key name never appears in
|
|
20
|
+
// this source file (the public-package pack scanner bans it as a content
|
|
21
|
+
// pattern, just as the original guard did).
|
|
22
|
+
const PRIVILEGED_SUPABASE_KEY_NAME = ["SUPABASE", "SERVICE", "ROLE", "KEY"].join("_");
|
|
23
|
+
const SUPABASE_SERVICE_ROLE_NAME = ["SUPABASE", "SERVICE", "ROLE"].join("_");
|
|
24
|
+
const OPENAI_KEY_NAME = ["OPENAI", "API", "KEY"].join("_");
|
|
25
|
+
const MEM0_KEY_NAME = ["MEM0", "API", "KEY"].join("_");
|
|
26
|
+
const SECRET_NAME_PATTERN_SOURCE = [
|
|
27
|
+
"(?:NEXT_PUBLIC_)?SUPABASE_ANON_KEY",
|
|
28
|
+
PRIVILEGED_SUPABASE_KEY_NAME,
|
|
29
|
+
SUPABASE_SERVICE_ROLE_NAME,
|
|
30
|
+
OPENAI_KEY_NAME,
|
|
31
|
+
MEM0_KEY_NAME,
|
|
32
|
+
"SERVICE[_\\s-]?ROLE(?:[_\\s-]?KEY)?",
|
|
33
|
+
"LANGFUSE_(?:PUBLIC|SECRET)_KEY",
|
|
34
|
+
"api[_-]?key",
|
|
35
|
+
"access[_-]?token",
|
|
36
|
+
"refresh[_-]?token",
|
|
37
|
+
"private[_-]?key",
|
|
38
|
+
"client[_-]?secret",
|
|
39
|
+
"secret[_-]?access[_-]?key",
|
|
40
|
+
].join("|");
|
|
41
|
+
/**
|
|
42
|
+
* Credential names live INSIDE longer identifiers, so `\b` is the wrong fence
|
|
43
|
+
* (BLI-3116).
|
|
44
|
+
*
|
|
45
|
+
* Every name fragment above used to be wrapped in `\b(?:…)\b`. Underscore is a
|
|
46
|
+
* word character, so there is no word boundary between `AWS_` and `SECRET`:
|
|
47
|
+
* `AWS_SECRET_ACCESS_KEY=AKIA…` — the most common real spelling of the most
|
|
48
|
+
* common real leak — never matched and uploaded unmasked from every fleet
|
|
49
|
+
* machine, while the bare `SECRET_ACCESS_KEY=…` matched fine. The same hole hid
|
|
50
|
+
* `AZURE_OPENAI_API_KEY`, `VITE_SUPABASE_ANON_KEY`, `GITHUB_ACCESS_TOKEN`,
|
|
51
|
+
* `MY_APP_CLIENT_SECRET` and every other prefixed spelling: the bug was the
|
|
52
|
+
* fence, not the vocabulary, so the whole list is fenced differently now.
|
|
53
|
+
*
|
|
54
|
+
* `_`, `-` and `.` are identifier JOINERS here, not boundaries:
|
|
55
|
+
*
|
|
56
|
+
* - leading `(?<![A-Za-z0-9])` — the fragment may begin right after a joiner or
|
|
57
|
+
* at a real boundary, but a letter or digit immediately before it still
|
|
58
|
+
* blocks the match (`notapikey=…` stays out, as before).
|
|
59
|
+
* - trailing `[A-Za-z0-9_-]{0,40}` — a bounded identifier tail, so
|
|
60
|
+
* `OPENAI_API_KEY_2=…` and `AWS_SECRET_ACCESS_KEY_ID=…` are seen too.
|
|
61
|
+
*
|
|
62
|
+
* Prose is unaffected because the assignment requirement below is unchanged:
|
|
63
|
+
* "rotate your secret access key" has no `=`/`:` + opaque value and is not a
|
|
64
|
+
* leak. The guard's job is assignments, not vocabulary.
|
|
65
|
+
*/
|
|
66
|
+
export const SECRET_NAME_MATCH_SOURCE = `(?<![A-Za-z0-9])(?:${SECRET_NAME_PATTERN_SOURCE})[A-Za-z0-9_-]{0,40}`;
|
|
67
|
+
/**
|
|
68
|
+
* The assignment operator, with the closing quote of a JSON/YAML key allowed
|
|
69
|
+
* before it. Found while auditing the fence above: `{"aws_secret_access_key":
|
|
70
|
+
* "…"}` never matched either, because the name was followed by `"` and the
|
|
71
|
+
* pattern demanded `[:=]` immediately. Transcripts are JSONL, so this is the
|
|
72
|
+
* shape a leaked credential most often has on the way in. The sibling redactor
|
|
73
|
+
* in harvest-analysis (`study/prepare.ts`) already allowed it; this one did not.
|
|
74
|
+
*/
|
|
75
|
+
export const SECRET_ASSIGNMENT_SOURCE = `["']?\\s*[:=]\\s*["']?`;
|
|
76
|
+
/**
|
|
77
|
+
* Credential names that are only a leak when assigned a value (D13). The value
|
|
78
|
+
* shape mirrors the long-standing generic pattern: an assignment operator
|
|
79
|
+
* followed by a 12+ char opaque value. A bare env var reference (name only) and
|
|
80
|
+
* `GRANT ... TO service_role` (no assignment) do not match; the same name with
|
|
81
|
+
* an assigned value does.
|
|
82
|
+
*/
|
|
83
|
+
const SECRET_NAME_WITH_VALUE_PATTERN = new RegExp(`${SECRET_NAME_MATCH_SOURCE}${SECRET_ASSIGNMENT_SOURCE}[A-Za-z0-9_./+=-]{12,}`, "i");
|
|
84
|
+
/**
|
|
85
|
+
* Self-identifying secret values — blocked regardless of context because the
|
|
86
|
+
* token shape itself is the credential, with no benign reading.
|
|
87
|
+
*
|
|
88
|
+
* `\b` is deliberately KEPT here, unlike the name fragments above (BLI-3116).
|
|
89
|
+
* These patterns are whole tokens, not fragments of a longer identifier: a
|
|
90
|
+
* `ghp_`/`sk-`/`AKIA…` run that continues into surrounding alphanumerics is not
|
|
91
|
+
* that provider's token, and unanchoring them would mask ordinary identifiers
|
|
92
|
+
* (`chart_m0_2024_revenue…`) for no safety gain. A real leak of any of these
|
|
93
|
+
* shapes is preceded by a quote, whitespace, `=`, `:` or `/` — all of which
|
|
94
|
+
* `\b` already admits.
|
|
95
|
+
*/
|
|
96
|
+
const SECRET_VALUE_PATTERNS = [
|
|
97
|
+
/-----BEGIN [A-Z ]*PRIVATE KEY-----/,
|
|
98
|
+
/\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/,
|
|
99
|
+
/\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/,
|
|
100
|
+
/\b(?:sk|pk)_(?:live|test)_[A-Za-z0-9]{16,}/,
|
|
101
|
+
/\bsk-[A-Za-z0-9_-]{16,}/,
|
|
102
|
+
/\bgh[pousr]_[A-Za-z0-9_]{16,}/,
|
|
103
|
+
/\bgithub_pat_[A-Za-z0-9_]{20,}/,
|
|
104
|
+
/\blin_api_[A-Za-z0-9]{16,}/,
|
|
105
|
+
/\bsb_secret_[A-Za-z0-9_]{16,}/,
|
|
106
|
+
/\bxox[baprs]-[A-Za-z0-9-]{16,}/,
|
|
107
|
+
/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/,
|
|
108
|
+
/\b(?:pk|sk)-lf-[A-Za-z0-9_-]{16,}\b/i,
|
|
109
|
+
/\bm0[-_][A-Za-z0-9_-]{16,}\b/i,
|
|
110
|
+
];
|
|
111
|
+
export const SECRET_LIKE_CONTENT_PATTERNS = [
|
|
112
|
+
SECRET_NAME_WITH_VALUE_PATTERN,
|
|
113
|
+
...SECRET_VALUE_PATTERNS,
|
|
114
|
+
];
|
|
115
|
+
/**
|
|
116
|
+
* Does this text carry something the fleet must not upload as it stands? The
|
|
117
|
+
* one question the collector and the ingest door both ask, so a laptop and the
|
|
118
|
+
* server can never disagree about which bytes were secret-shaped.
|
|
119
|
+
*/
|
|
120
|
+
export function containsSecretLikeContent(value) {
|
|
121
|
+
return SECRET_LIKE_CONTENT_PATTERNS.some((pattern) => pattern.test(value));
|
|
122
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { RawEvidenceRedactionMetadata, RawEvidenceRedactionSource } from "./secret-guards-metadata.js";
|
|
2
|
+
/**
|
|
3
|
+
* How a match is MASKED, and which rule gets to name it.
|
|
4
|
+
*
|
|
5
|
+
* Owned by `secret-guards.ts` (BLI-3986). `SECRET_REDACTION_RULES` is a table
|
|
6
|
+
* and its INDEX is a priority: the earliest rule that reaches a span wins, so
|
|
7
|
+
* the reason label a dashboard reads (`[REDACTED:openai_token]`,
|
|
8
|
+
* `rule_counts[].rule_id`) is decided by position in that list and by nothing
|
|
9
|
+
* else. Reordering it, or inserting a rule in the middle, relabels live
|
|
10
|
+
* redactions — `secret-guards-lock.test.ts` records the answers so that cannot
|
|
11
|
+
* happen by accident.
|
|
12
|
+
*
|
|
13
|
+
* Below the whole table sits the fallback (`unnamed_secret_shape`), ranked
|
|
14
|
+
* after every real rule: masking is best-effort and NEVER drops a file, so a
|
|
15
|
+
* detector hit no rule can pinpoint is masked coarsely rather than costing the
|
|
16
|
+
* session (the BLI-3116 note on the first rule explains what that cost was).
|
|
17
|
+
*/
|
|
18
|
+
export interface SecretRedactionResult {
|
|
19
|
+
redacted: boolean;
|
|
20
|
+
text: string;
|
|
21
|
+
metadata: RawEvidenceRedactionMetadata | null;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Mask every secret-shaped span and keep the rest of the text, because a
|
|
25
|
+
* session is never dropped for carrying one: this returns the redacted bytes
|
|
26
|
+
* plus the receipt that says which rules fired, where, and how much was
|
|
27
|
+
* replaced.
|
|
28
|
+
*/
|
|
29
|
+
export declare function redactSecretLikeContent(value: string, options?: {
|
|
30
|
+
appliedBy?: RawEvidenceRedactionSource;
|
|
31
|
+
redactedFields?: string[];
|
|
32
|
+
}): SecretRedactionResult;
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { SECRET_ASSIGNMENT_SOURCE, SECRET_LIKE_CONTENT_PATTERNS, SECRET_NAME_MATCH_SOURCE, } from "./secret-guards-content-rules.js";
|
|
2
|
+
import { RawEvidenceRedactionMetadataSchema } from "./secret-guards-metadata.js";
|
|
3
|
+
// Same fence as the detector (BLI-3116) — the two must agree or a prefixed
|
|
4
|
+
// name would be detected and then left unredacted, which costs whole sessions
|
|
5
|
+
// (see findSecretRedactionMatches).
|
|
6
|
+
const SECRET_ASSIGNMENT_REDACTION_PATTERN = new RegExp(`(${SECRET_NAME_MATCH_SOURCE}${SECRET_ASSIGNMENT_SOURCE})([A-Za-z0-9_./+=-]{12,})`, "gi");
|
|
7
|
+
const SECRET_REDACTION_RULES = [
|
|
8
|
+
{
|
|
9
|
+
ruleId: "credential_assignment",
|
|
10
|
+
pattern: SECRET_ASSIGNMENT_REDACTION_PATTERN,
|
|
11
|
+
secretGroup: 2,
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
ruleId: "private_key_block",
|
|
15
|
+
pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
ruleId: "private_key_header",
|
|
19
|
+
pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----/g,
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
ruleId: "bearer_token",
|
|
23
|
+
pattern: /\b(Bearer\s+)([A-Za-z0-9._~+/=-]{16,})/gi,
|
|
24
|
+
secretGroup: 2,
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
ruleId: "jwt",
|
|
28
|
+
pattern: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
ruleId: "stripe_token",
|
|
32
|
+
pattern: /\b(?:sk|pk)_(?:live|test)_[A-Za-z0-9]{16,}/g,
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
ruleId: "openai_token",
|
|
36
|
+
pattern: /\bsk-[A-Za-z0-9_-]{16,}/g,
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
ruleId: "github_token",
|
|
40
|
+
pattern: /\b(?:gh[pousr]_[A-Za-z0-9_]{16,}|github_pat_[A-Za-z0-9_]{20,})/g,
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
ruleId: "linear_token",
|
|
44
|
+
pattern: /\blin_api_[A-Za-z0-9]{16,}/g,
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
ruleId: "supabase_secret_token",
|
|
48
|
+
pattern: /\bsb_secret_[A-Za-z0-9_]{16,}/g,
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
ruleId: "slack_token",
|
|
52
|
+
pattern: /\bxox[baprs]-[A-Za-z0-9-]{16,}/g,
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
ruleId: "aws_access_key_id",
|
|
56
|
+
pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
ruleId: "langfuse_token",
|
|
60
|
+
pattern: /\b(?:pk|sk)-lf-[A-Za-z0-9_-]{16,}\b/gi,
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
ruleId: "mem0_token",
|
|
64
|
+
pattern: /\bm0[-_][A-Za-z0-9_-]{16,}\b/gi,
|
|
65
|
+
},
|
|
66
|
+
];
|
|
67
|
+
/**
|
|
68
|
+
* Mask every secret-shaped span and keep the rest of the text, because a
|
|
69
|
+
* session is never dropped for carrying one: this returns the redacted bytes
|
|
70
|
+
* plus the receipt that says which rules fired, where, and how much was
|
|
71
|
+
* replaced.
|
|
72
|
+
*/
|
|
73
|
+
export function redactSecretLikeContent(value, options = {}) {
|
|
74
|
+
const matches = findSecretRedactionMatches(value);
|
|
75
|
+
if (matches.length === 0) {
|
|
76
|
+
return { redacted: false, text: value, metadata: null };
|
|
77
|
+
}
|
|
78
|
+
let text = "";
|
|
79
|
+
let cursor = 0;
|
|
80
|
+
const ruleCounts = new Map();
|
|
81
|
+
for (const match of matches) {
|
|
82
|
+
text += value.slice(cursor, match.start);
|
|
83
|
+
text += `[REDACTED:${match.ruleId}]`;
|
|
84
|
+
cursor = match.end;
|
|
85
|
+
const existing = ruleCounts.get(match.ruleId) ?? {
|
|
86
|
+
rule_id: match.ruleId,
|
|
87
|
+
match_count: 0,
|
|
88
|
+
redacted_char_count: 0,
|
|
89
|
+
};
|
|
90
|
+
existing.match_count += 1;
|
|
91
|
+
existing.redacted_char_count += match.end - match.start;
|
|
92
|
+
ruleCounts.set(match.ruleId, existing);
|
|
93
|
+
}
|
|
94
|
+
text += value.slice(cursor);
|
|
95
|
+
const metadata = RawEvidenceRedactionMetadataSchema.parse({
|
|
96
|
+
schema_version: "raw-evidence-redaction.v1",
|
|
97
|
+
status: "sanitized",
|
|
98
|
+
mode: "deterministic_text_replacement",
|
|
99
|
+
applied_by: [options.appliedBy ?? "local_collector"],
|
|
100
|
+
rule_counts: [...ruleCounts.values()].sort((a, b) => a.rule_id.localeCompare(b.rule_id)),
|
|
101
|
+
secret_like_match_count: matches.length,
|
|
102
|
+
redacted_fields: options.redactedFields ?? [],
|
|
103
|
+
redacted_ranges: matches.slice(0, 200).map((match) => ({
|
|
104
|
+
start: match.start,
|
|
105
|
+
end: match.end,
|
|
106
|
+
rule_id: match.ruleId,
|
|
107
|
+
})),
|
|
108
|
+
});
|
|
109
|
+
return { redacted: true, text, metadata };
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Every region the detector considers a secret, redacted — precisely where a
|
|
113
|
+
* rule can pinpoint the value, coarsely where none can.
|
|
114
|
+
*
|
|
115
|
+
* `containsSecretLikeContent` and `SECRET_REDACTION_RULES` are two lists that
|
|
116
|
+
* have to agree, and nothing used to make them. When the detector matched and
|
|
117
|
+
* no rule produced a range, `redactSecretLikeContent` reported
|
|
118
|
+
* `redacted: false`, and the caller dropped the entire session file
|
|
119
|
+
* (`secret_like_content_guard`). Drift between the lists cost whole sessions:
|
|
120
|
+
* 78 fleet-wide as of 2026-08-14, 3 of Viet's 17 Claude sessions.
|
|
121
|
+
*
|
|
122
|
+
* The two lists can still drift — regexes are like that — but drift now costs
|
|
123
|
+
* precision instead of evidence. A detector hit with no matching rule redacts
|
|
124
|
+
* the whole matched span, so `redactSecretLikeContent` always reports the
|
|
125
|
+
* redaction it performed and the transcript survives with the value removed.
|
|
126
|
+
*/
|
|
127
|
+
function findSecretRedactionMatches(value) {
|
|
128
|
+
const accepted = acceptNonOverlapping(secretRuleCandidates(value));
|
|
129
|
+
const uncovered = detectorFallbackCandidates(value).filter((candidate) => !overlapsAny(candidate, accepted));
|
|
130
|
+
if (uncovered.length === 0)
|
|
131
|
+
return accepted;
|
|
132
|
+
return acceptNonOverlapping([...accepted, ...uncovered]);
|
|
133
|
+
}
|
|
134
|
+
function secretRuleCandidates(value) {
|
|
135
|
+
const candidates = [];
|
|
136
|
+
SECRET_REDACTION_RULES.forEach((rule, priority) => {
|
|
137
|
+
const pattern = cloneGlobalPattern(rule.pattern);
|
|
138
|
+
for (const match of value.matchAll(pattern)) {
|
|
139
|
+
const fullMatch = match[0];
|
|
140
|
+
const matchIndex = match.index;
|
|
141
|
+
if (!fullMatch || matchIndex === undefined)
|
|
142
|
+
continue;
|
|
143
|
+
const secret = rule.secretGroup ? match[rule.secretGroup] : fullMatch;
|
|
144
|
+
if (!secret)
|
|
145
|
+
continue;
|
|
146
|
+
const relativeStart = fullMatch.indexOf(secret);
|
|
147
|
+
if (relativeStart < 0)
|
|
148
|
+
continue;
|
|
149
|
+
candidates.push({
|
|
150
|
+
start: matchIndex + relativeStart,
|
|
151
|
+
end: matchIndex + relativeStart + secret.length,
|
|
152
|
+
ruleId: rule.ruleId,
|
|
153
|
+
priority,
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
});
|
|
157
|
+
return candidates;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Ranked below every real rule, so a rule that can name the credential always
|
|
161
|
+
* wins and the coarse span is only used where nothing else reached.
|
|
162
|
+
*/
|
|
163
|
+
function detectorFallbackCandidates(value) {
|
|
164
|
+
const candidates = [];
|
|
165
|
+
const basePriority = SECRET_REDACTION_RULES.length;
|
|
166
|
+
SECRET_LIKE_CONTENT_PATTERNS.forEach((pattern, offset) => {
|
|
167
|
+
for (const match of value.matchAll(cloneGlobalPattern(pattern))) {
|
|
168
|
+
const fullMatch = match[0];
|
|
169
|
+
const matchIndex = match.index;
|
|
170
|
+
if (!fullMatch || matchIndex === undefined)
|
|
171
|
+
continue;
|
|
172
|
+
candidates.push({
|
|
173
|
+
start: matchIndex,
|
|
174
|
+
end: matchIndex + fullMatch.length,
|
|
175
|
+
ruleId: "unnamed_secret_shape",
|
|
176
|
+
priority: basePriority + offset,
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
return candidates;
|
|
181
|
+
}
|
|
182
|
+
function overlapsAny(candidate, accepted) {
|
|
183
|
+
return accepted.some((other) => candidate.start < other.end && other.start < candidate.end);
|
|
184
|
+
}
|
|
185
|
+
function acceptNonOverlapping(candidates) {
|
|
186
|
+
const ordered = [...candidates].sort((a, b) => {
|
|
187
|
+
if (a.start !== b.start)
|
|
188
|
+
return a.start - b.start;
|
|
189
|
+
if (a.priority !== b.priority)
|
|
190
|
+
return a.priority - b.priority;
|
|
191
|
+
return b.end - b.start - (a.end - a.start);
|
|
192
|
+
});
|
|
193
|
+
const accepted = [];
|
|
194
|
+
let lastEnd = -1;
|
|
195
|
+
for (const candidate of ordered) {
|
|
196
|
+
if (candidate.start < lastEnd)
|
|
197
|
+
continue;
|
|
198
|
+
accepted.push(candidate);
|
|
199
|
+
lastEnd = candidate.end;
|
|
200
|
+
}
|
|
201
|
+
return accepted;
|
|
202
|
+
}
|
|
203
|
+
function cloneGlobalPattern(pattern) {
|
|
204
|
+
const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`;
|
|
205
|
+
return new RegExp(pattern.source, flags);
|
|
206
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* What a secret scan RECORDS: the redaction receipt every uploaded object
|
|
4
|
+
* carries, and the one builder of the clean half of it.
|
|
5
|
+
*
|
|
6
|
+
* Owned by `secret-guards.ts` (BLI-3986); the rules that produce these records
|
|
7
|
+
* live in `secret-guards-content-rules.ts` and `secret-guards-masking.ts`.
|
|
8
|
+
* Nothing here decides what a secret IS — it decides what a reader downstream
|
|
9
|
+
* is told about a scan that already ran, which is why the per-status
|
|
10
|
+
* invariants below are the load-bearing part: a record that claims a
|
|
11
|
+
* sanitization that never happened is worse than a missing record.
|
|
12
|
+
*/
|
|
13
|
+
export declare const RawEvidenceRedactionSourceSchema: z.ZodEnum<{
|
|
14
|
+
local_collector: "local_collector";
|
|
15
|
+
server_commit: "server_commit";
|
|
16
|
+
legacy_upload: "legacy_upload";
|
|
17
|
+
}>;
|
|
18
|
+
export type RawEvidenceRedactionSource = z.infer<typeof RawEvidenceRedactionSourceSchema>;
|
|
19
|
+
export declare const RawEvidenceRedactionRuleCountSchema: z.ZodObject<{
|
|
20
|
+
rule_id: z.ZodString;
|
|
21
|
+
match_count: z.ZodNumber;
|
|
22
|
+
redacted_char_count: z.ZodNumber;
|
|
23
|
+
}, z.core.$strict>;
|
|
24
|
+
export type RawEvidenceRedactionRuleCount = z.infer<typeof RawEvidenceRedactionRuleCountSchema>;
|
|
25
|
+
export declare const RawEvidenceRedactionRangeSchema: z.ZodObject<{
|
|
26
|
+
start: z.ZodNumber;
|
|
27
|
+
end: z.ZodNumber;
|
|
28
|
+
rule_id: z.ZodString;
|
|
29
|
+
}, z.core.$strict>;
|
|
30
|
+
export type RawEvidenceRedactionRange = z.infer<typeof RawEvidenceRedactionRangeSchema>;
|
|
31
|
+
/**
|
|
32
|
+
* The verdict of a secret scan — NOT "did the bytes change" (BLI-3277).
|
|
33
|
+
*
|
|
34
|
+
* `sanitized` means the scan matched and the uploaded bytes differ from what was
|
|
35
|
+
* read. `scanned_clean` means the same scan ran over the same bytes and matched
|
|
36
|
+
* nothing, so the uploaded bytes are the original ones. The second value exists
|
|
37
|
+
* because every reader downstream asks "was this scanned?", and for 70% of
|
|
38
|
+
* transcripts — the ones with no secret in them — the honest answer used to be
|
|
39
|
+
* an absent record, which reads identically to "nobody ever looked".
|
|
40
|
+
*/
|
|
41
|
+
export declare const RawEvidenceRedactionStatusSchema: z.ZodEnum<{
|
|
42
|
+
sanitized: "sanitized";
|
|
43
|
+
scanned_clean: "scanned_clean";
|
|
44
|
+
}>;
|
|
45
|
+
export type RawEvidenceRedactionStatus = z.infer<typeof RawEvidenceRedactionStatusSchema>;
|
|
46
|
+
export declare const RawEvidenceRedactionMetadataSchema: z.ZodObject<{
|
|
47
|
+
schema_version: z.ZodLiteral<"raw-evidence-redaction.v1">;
|
|
48
|
+
status: z.ZodEnum<{
|
|
49
|
+
sanitized: "sanitized";
|
|
50
|
+
scanned_clean: "scanned_clean";
|
|
51
|
+
}>;
|
|
52
|
+
mode: z.ZodLiteral<"deterministic_text_replacement">;
|
|
53
|
+
applied_by: z.ZodArray<z.ZodEnum<{
|
|
54
|
+
local_collector: "local_collector";
|
|
55
|
+
server_commit: "server_commit";
|
|
56
|
+
legacy_upload: "legacy_upload";
|
|
57
|
+
}>>;
|
|
58
|
+
rule_counts: z.ZodArray<z.ZodObject<{
|
|
59
|
+
rule_id: z.ZodString;
|
|
60
|
+
match_count: z.ZodNumber;
|
|
61
|
+
redacted_char_count: z.ZodNumber;
|
|
62
|
+
}, z.core.$strict>>;
|
|
63
|
+
secret_like_match_count: z.ZodNumber;
|
|
64
|
+
redacted_fields: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
65
|
+
redacted_ranges: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
66
|
+
start: z.ZodNumber;
|
|
67
|
+
end: z.ZodNumber;
|
|
68
|
+
rule_id: z.ZodString;
|
|
69
|
+
}, z.core.$strict>>>;
|
|
70
|
+
original_content_hash_sha256: z.ZodOptional<z.ZodString>;
|
|
71
|
+
sanitized_content_hash_sha256: z.ZodOptional<z.ZodString>;
|
|
72
|
+
original_byte_size: z.ZodOptional<z.ZodNumber>;
|
|
73
|
+
sanitized_byte_size: z.ZodOptional<z.ZodNumber>;
|
|
74
|
+
applied_at: z.ZodOptional<z.ZodString>;
|
|
75
|
+
}, z.core.$strict>;
|
|
76
|
+
export type RawEvidenceRedactionMetadata = z.infer<typeof RawEvidenceRedactionMetadataSchema>;
|
|
77
|
+
/**
|
|
78
|
+
* The receipt for a file the scan cleared (BLI-3277, moved here by BLI-3280).
|
|
79
|
+
*
|
|
80
|
+
* Same schema, same `mode` — the deterministic ruleset is what ran — with the
|
|
81
|
+
* verdict `scanned_clean` and zero of everything else. The content fields are
|
|
82
|
+
* the point: an evidence ref is only readable downstream when its redaction
|
|
83
|
+
* record hashes the bytes that are actually in the bucket, and for a clean file
|
|
84
|
+
* those are the original bytes, so both halves carry the same digest and size.
|
|
85
|
+
*
|
|
86
|
+
* It lives in telemetry-core rather than in the collector because two callers
|
|
87
|
+
* now build this exact record and they must never drift: the collector writes
|
|
88
|
+
* it at upload time, and the BLI-3280 operator backfill writes it onto refs
|
|
89
|
+
* that were stored before the receipt existed. Both must satisfy the same
|
|
90
|
+
* `sanitized_content_hash_sha256 === content_hash_sha256` /
|
|
91
|
+
* `sanitized_byte_size === byte_size` equality the digest reader checks, and a
|
|
92
|
+
* second copy of this function is a second chance to get that wrong.
|
|
93
|
+
*
|
|
94
|
+
* `appliedBy` is who ran the scan, and it is not decoration: a backfill passes
|
|
95
|
+
* `legacy_upload` so a reader can tell "the collector scanned this before it
|
|
96
|
+
* uploaded" apart from "an operator scanned the stored bytes afterwards".
|
|
97
|
+
*/
|
|
98
|
+
export declare function scannedCleanRedactionMetadata(bytes: Uint8Array, options?: {
|
|
99
|
+
appliedBy?: RawEvidenceRedactionSource;
|
|
100
|
+
/** ISO-8601 with offset. Omitted entirely when absent (BLI-3290). */
|
|
101
|
+
appliedAt?: string;
|
|
102
|
+
}): RawEvidenceRedactionMetadata;
|