codiedev 0.9.89 → 0.9.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/captureLog.d.ts +1 -1
- package/dist/codie.js +56 -4
- package/dist/codieLauncher.d.ts +3 -0
- package/dist/codieLauncher.js +16 -1
- package/dist/hook.js +52 -0
- package/dist/launchContext.d.ts +27 -0
- package/dist/launchContext.js +79 -0
- package/dist/promptGuard.d.ts +63 -0
- package/dist/promptGuard.js +261 -0
- package/dist/scrub/promptRules.d.ts +119 -0
- package/dist/scrub/promptRules.js +941 -0
- package/dist/utils.d.ts +4 -0
- package/dist/utils.js +48 -9
- package/package.json +1 -1
|
@@ -0,0 +1,941 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Stable-branch copy of main's scrub/rules.ts (0.9.90 backport of #2610), used
|
|
3
|
+
// only by promptGuard so a stopped prompt matches main exactly. scrub/rules.ts
|
|
4
|
+
// stays an exact copy of this branch's convex/lib/scrubSecrets.ts.
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.SHAPE_RULES = exports.isTestFile = exports.GENERIC_RULES = void 0;
|
|
7
|
+
exports.shannonEntropy = shannonEntropy;
|
|
8
|
+
exports.scrubSecrets = scrubSecrets;
|
|
9
|
+
exports.redactSpans = redactSpans;
|
|
10
|
+
exports.looksLikeTestValue = looksLikeTestValue;
|
|
11
|
+
exports.locateInLine = locateInLine;
|
|
12
|
+
exports.locateMarkers = locateMarkers;
|
|
13
|
+
exports.namesASecret = namesASecret;
|
|
14
|
+
exports.notWorthRotating = notWorthRotating;
|
|
15
|
+
exports.hintOf = hintOf;
|
|
16
|
+
exports.secretFindings = secretFindings;
|
|
17
|
+
exports.possibleSecrets = possibleSecrets;
|
|
18
|
+
exports.redactPossible = redactPossible;
|
|
19
|
+
// Secret/PII scrubber for session transcripts.
|
|
20
|
+
//
|
|
21
|
+
// Runs server-side over the decoded transcript before it is stored at rest
|
|
22
|
+
// (see /api/captureSession) and over LLM-generated summary fields before they
|
|
23
|
+
// are persisted. Design goal: catch secrets (recall) while redacting as little
|
|
24
|
+
// benign content as possible (fidelity). See spec-transcript-scrubber.md.
|
|
25
|
+
//
|
|
26
|
+
// Principles:
|
|
27
|
+
// - Lead with precise, prefix/structured detectors (near-zero false positives).
|
|
28
|
+
// - No free-floating entropy rules — entropy is only considered inside
|
|
29
|
+
// secret-named assignment contexts.
|
|
30
|
+
// - Every redaction is tagged [REDACTED:<rule>] so false positives are auditable.
|
|
31
|
+
const scrubProviderRules_1 = require("./scrubProviderRules");
|
|
32
|
+
/** Shannon entropy in bits per character, as Gitleaks measures it. */
|
|
33
|
+
function shannonEntropy(s) {
|
|
34
|
+
if (!s)
|
|
35
|
+
return 0;
|
|
36
|
+
const counts = new Map();
|
|
37
|
+
for (const ch of s)
|
|
38
|
+
counts.set(ch, (counts.get(ch) ?? 0) + 1);
|
|
39
|
+
let h = 0;
|
|
40
|
+
for (const n of counts.values()) {
|
|
41
|
+
const p = n / s.length;
|
|
42
|
+
h -= p * Math.log2(p);
|
|
43
|
+
}
|
|
44
|
+
return h;
|
|
45
|
+
}
|
|
46
|
+
// A secret-named assignment whose KEY ends in one of these is metadata ABOUT a
|
|
47
|
+
// secret (its vault name, file, ARN, URL…), not the secret itself.
|
|
48
|
+
const SECRET_REF_KEY = /_(?:NAME|ID|ARN|FILE|PATH|URL|URI|HOST|REGION|BUCKET|PREFIX|ENABLED|TYPE|VERSION|COUNT)$/i;
|
|
49
|
+
// Does this assignment value look like an actual secret rather than a label,
|
|
50
|
+
// dictionary word, or placeholder? Real secrets are long and/or mixed
|
|
51
|
+
// alphanumeric; benign values are short words, lowercase kebab, or placeholders.
|
|
52
|
+
function looksSecret(v) {
|
|
53
|
+
// --- reject non-secret shapes (the fidelity lever) ---
|
|
54
|
+
if (v.includes("<") || v.includes(">"))
|
|
55
|
+
return false; // <your-token>
|
|
56
|
+
if (/^(your[-_]|changeme|change-me|placeholder|example|none|null|undefined|redacted|true|false)/i.test(v))
|
|
57
|
+
return false;
|
|
58
|
+
if (/[${}]/.test(v) || v.includes("$("))
|
|
59
|
+
return false; // ${VAR} / $(cmd) refs
|
|
60
|
+
// Code punctuation — real secrets never contain these; source code does.
|
|
61
|
+
// (React hooks `useState(...)`, TS types `string;`, arg lists `a, b)`.)
|
|
62
|
+
if (/[()[\]{};,]/.test(v))
|
|
63
|
+
return false;
|
|
64
|
+
if (/:\/\//.test(v) || /^https?:/i.test(v))
|
|
65
|
+
return false; // URLs
|
|
66
|
+
if (/^[~./]|^[A-Za-z]:\\/.test(v))
|
|
67
|
+
return false; // file path (/, ./, ~/, C:\)
|
|
68
|
+
if (/\.(pem|key|json|crt|cer|txt|log|env|conf|config|ya?ml|pub|p12|pfx|sh|js|ts|py)$/i.test(v))
|
|
69
|
+
return false; // path with a known file extension
|
|
70
|
+
if (/^[A-Za-z_$][\w$]*(\.[A-Za-z_$][\w$]*)+$/.test(v))
|
|
71
|
+
return false; // dotted code identifier
|
|
72
|
+
if (/^[A-Za-z-]+$/.test(v))
|
|
73
|
+
return false; // all letters/hyphens, no digit → identifier / word / type
|
|
74
|
+
// --- require secret entropy ---
|
|
75
|
+
if (/^[0-9a-f]{16,}$/i.test(v))
|
|
76
|
+
return true; // hex token
|
|
77
|
+
if (v.length >= 12 && /[0-9]/.test(v) && /[A-Za-z]/.test(v))
|
|
78
|
+
return true; // mixed alnum w/ a digit
|
|
79
|
+
if (v.length >= 20 && /[+/=]/.test(v))
|
|
80
|
+
return true; // base64 blob
|
|
81
|
+
return false;
|
|
82
|
+
}
|
|
83
|
+
// Documentation/sample values that share a real key's shape (same list as the
|
|
84
|
+
// phiScan audit). Rejected so docs and fixtures are not redacted.
|
|
85
|
+
const PLACEHOLDER = /x{6,}|\.\.\.|your|example|placeholder|changeme|redacted|dummy|fake|<|\$\{|\*{4,}/i;
|
|
86
|
+
const notPlaceholder = (v) => !PLACEHOLDER.test(v);
|
|
87
|
+
const notTestValue = (v) => notPlaceholder(v) && !looksLikeTestValue(v);
|
|
88
|
+
// URL passwords that are clearly not real: `user:password@`, `:pass@`, `:secret@`.
|
|
89
|
+
const URL_PASSWORD_PLACEHOLDER = /^(?:pass(?:word)?|secret|pwd|test|admin|root|guest)$/i;
|
|
90
|
+
// Order matters only for readability; overlap resolution (below) is by span,
|
|
91
|
+
// not by rule order. All regexes are global.
|
|
92
|
+
const RULES = [
|
|
93
|
+
// PEM private-key blocks (any key type), multiline.
|
|
94
|
+
{
|
|
95
|
+
name: "pem-private-key",
|
|
96
|
+
re: /-----BEGIN[A-Z ]*PRIVATE KEY-----[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----/g,
|
|
97
|
+
},
|
|
98
|
+
// JWTs (three base64url segments). Also covers `Bearer eyJ...`.
|
|
99
|
+
{
|
|
100
|
+
name: "jwt",
|
|
101
|
+
re: /eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/g,
|
|
102
|
+
},
|
|
103
|
+
// OpenAI project keys: anchored on the literal sk-proj- prefix, so the
|
|
104
|
+
// permissive body (which may contain -/_) can't match kebab identifiers.
|
|
105
|
+
{ name: "openai-key", re: /\bsk-proj-[A-Za-z0-9_-]{20,}/g },
|
|
106
|
+
// OpenAI classic keys: alphanumeric body only, so `sk-feature-branch-...`
|
|
107
|
+
// (hyphenated kebab) is rejected — the hyphen breaks the run below 20.
|
|
108
|
+
{ name: "openai-key", re: /\bsk-[A-Za-z0-9]{20,}\b/g },
|
|
109
|
+
// Stripe secret/restricted keys.
|
|
110
|
+
{ name: "stripe-key", re: /\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}/g },
|
|
111
|
+
// GitHub tokens: ghp_ gho_ ghu_ ghs_ ghr_ (+ fine-grained github_pat_).
|
|
112
|
+
{ name: "github-token", re: /\bgh[posru]_[A-Za-z0-9]{36,}/g },
|
|
113
|
+
{ name: "github-token", re: /\bgithub_pat_[A-Za-z0-9_]{40,}/g },
|
|
114
|
+
// AWS access key ids.
|
|
115
|
+
{ name: "aws-access-key", re: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g },
|
|
116
|
+
// Google API keys.
|
|
117
|
+
{ name: "google-api-key", re: /\bAIza[0-9A-Za-z_-]{35}\b/g },
|
|
118
|
+
// Slack tokens.
|
|
119
|
+
{ name: "slack-token", re: /\bxox[baprs]-[A-Za-z0-9-]{10,}/g },
|
|
120
|
+
// Formats the phiScan secret audit (convex/lib/secretAudit.ts on
|
|
121
|
+
// feat/phi-count-scan) found surviving in stored transcripts. Same shapes as
|
|
122
|
+
// its SURVIVOR_RULES so the audit and the scrubber agree on what a key is.
|
|
123
|
+
{ name: "gitlab-token", re: /\bgl(?:pat|dt|rt|ptt|cbt|oas|soat|ffct|imt)-[A-Za-z0-9_-]{20,}/g, validate: notPlaceholder },
|
|
124
|
+
// sk-ant-api03-… has hyphens, so the classic openai-key rule never matched it.
|
|
125
|
+
{ name: "anthropic-key", re: /\bsk-ant-(?:api|admin)\d{2}-[A-Za-z0-9_-]{40,}/g, validate: notPlaceholder },
|
|
126
|
+
// Convex deploy keys: `prod:happy-otter-123|<base64>`.
|
|
127
|
+
{ name: "convex-deploy-key", re: /\b(?:prod|dev|preview):[a-z0-9-]+\|[A-Za-z0-9+/=]{40,}/g, validate: notPlaceholder },
|
|
128
|
+
{ name: "linear-key", re: /\blin_(?:api|oauth)_[A-Za-z0-9]{30,}/g, validate: notPlaceholder },
|
|
129
|
+
// Our own keys: cdv_ (CLI), cdvm_ (model gateway), cdvci_, cdvi_, cdvd_,
|
|
130
|
+
// cdvx_, cdvh_, cdvr_, cdva_, cdvs_. Every body is 40+ random alnum/hex
|
|
131
|
+
// (lib/secureTokens.ts, lib/dataApiToken.ts); the 32+ floor keeps display
|
|
132
|
+
// prefixes like `cdvi_01234567…` and seed names like `cdv_seed` intact.
|
|
133
|
+
{ name: "codiedev-key", re: /\bcdv(?:m|ci|i|d|x|h|r|a|s)?_[A-Za-z0-9]{32,}\b/g, validate: notPlaceholder },
|
|
134
|
+
// Browser sign-in tokens (convex/oauth.ts): access cdv_at_, refresh cdv_rt_,
|
|
135
|
+
// device code cdv_dc_, each + base64url of 32 random bytes (43 chars). The
|
|
136
|
+
// public client id cdv_client_ is not a secret.
|
|
137
|
+
{ name: "codiedev-key", re: /\bcdv_(?:at|rt|dc)_[A-Za-z0-9_-]{40,}/g, validate: notPlaceholder },
|
|
138
|
+
{ name: "fireworks-key", re: /\bfw_[A-Za-z0-9]{20,}/g, validate: notPlaceholder },
|
|
139
|
+
{ name: "slack-webhook", re: /hooks\.slack\.com\/services\/T[A-Z0-9]+\/B[A-Z0-9]+\/[A-Za-z0-9]{20,}/g, validate: notPlaceholder },
|
|
140
|
+
{ name: "npm-token", re: /\bnpm_[A-Za-z0-9]{36}\b/g, validate: notPlaceholder },
|
|
141
|
+
{ name: "sendgrid-key", re: /\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b/g, validate: notPlaceholder },
|
|
142
|
+
{ name: "twilio-key", re: /\bSK[0-9a-f]{32}\b/g, validate: notPlaceholder },
|
|
143
|
+
// Azure: redact the key/signature value, keep the connection string readable.
|
|
144
|
+
{ name: "azure-storage-key", re: /AccountKey=([A-Za-z0-9+/=]{40,})/g, group: 1, validate: notPlaceholder },
|
|
145
|
+
{ name: "azure-sas", re: /[?&]sig=([A-Za-z0-9%+/=]{30,})/g, group: 1, validate: notPlaceholder },
|
|
146
|
+
// Passwords inside DB / basic-auth URLs. Only the password is redacted so the
|
|
147
|
+
// host, user and database stay readable for summaries and debugging.
|
|
148
|
+
{
|
|
149
|
+
name: "url-password",
|
|
150
|
+
re: /\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|rediss|mssql|sqlserver|amqps?|https?):\/\/[^:\s/"'@\\]+:([^@\s"'/\\]{6,})@[A-Za-z0-9.-]+/g,
|
|
151
|
+
group: 1,
|
|
152
|
+
validate: (v) => notPlaceholder(v) && !URL_PASSWORD_PLACEHOLDER.test(v),
|
|
153
|
+
},
|
|
154
|
+
// Generic Bearer tokens (non-JWT covered above) — redact only the token.
|
|
155
|
+
{ name: "bearer-token", re: /\bBearer\s+([A-Za-z0-9._-]{20,})/g, group: 1 },
|
|
156
|
+
// Secret-named assignments: redact the value of KEY/SECRET/PASSWORD/etc.
|
|
157
|
+
// Name must look secret-y; value must be >= 8 non-space chars. This is the
|
|
158
|
+
// only entropy-ish rule and it is gated on the assignment name so benign
|
|
159
|
+
// env assignments (NODE_ENV=production, PORT=3000, LOG_LEVEL=debug) are safe.
|
|
160
|
+
{
|
|
161
|
+
name: "secret-assignment",
|
|
162
|
+
// The secret keyword must END the key name (it abuts the : or =), so the
|
|
163
|
+
// secret-word as a SUBSTRING of a larger code identifier does NOT match:
|
|
164
|
+
// `Last_Password_Change=`, `PasswordHash=`, `class ApiKeyController : Base`.
|
|
165
|
+
// Compound forms (SECRET_KEY, CLIENT_SECRET…) are listed before their parts.
|
|
166
|
+
// Quotes around the key or value may be JSON-escaped (\"KEY\": \"value\") —
|
|
167
|
+
// transcripts embed JSON-encoded tool results, so both forms must match;
|
|
168
|
+
// the value run excludes backslash so it stops at an escaped closing quote.
|
|
169
|
+
re: /\b([A-Za-z0-9_-]*?(?:SECRET[_-]?KEY|PRIVATE[_-]?KEY|CLIENT[_-]?SECRET|ACCESS[_-]?TOKEN|AUTH[_-]?TOKEN|REFRESH[_-]?TOKEN|API[_-]?KEY|PASSWORD|PASSWD|SECRET|TOKEN))\\?['"]?\s*[:=]\s*(?:\\?['"])?([^\s'"\\]{8,})(?:\\?['"])?/gi,
|
|
170
|
+
group: 2,
|
|
171
|
+
validate: (v, m) => !SECRET_REF_KEY.test(m[1]) &&
|
|
172
|
+
!/public/i.test(m[1]) && // .NET `PublicKeyToken=`, `PublicKey=` — public, not secret
|
|
173
|
+
// Pagination cursors (Jira nextPageToken, S3 ContinuationToken…) are
|
|
174
|
+
// high-entropy but not secrets — the one confirmed FP of the shadow week.
|
|
175
|
+
!/(?:page|continuation)[_-]?token$/i.test(m[1]) &&
|
|
176
|
+
m.input[m.index - 1] !== ":" && // ARN/path segment `arn:…:secret:<name>` — names a secret
|
|
177
|
+
looksSecret(v),
|
|
178
|
+
},
|
|
179
|
+
// Formats Gitleaks doesn't give a prefix rule for, seen in our sessions.
|
|
180
|
+
{ name: "supabase-token", re: /\bsbp_[a-f0-9]{40}\b/g, validate: notTestValue },
|
|
181
|
+
{ name: "groq-key", re: /\bgsk_[A-Za-z0-9]{48,56}\b/g, validate: notTestValue },
|
|
182
|
+
{ name: "xai-key", re: /\bxai-[A-Za-z0-9]{70,90}\b/g, validate: notTestValue },
|
|
183
|
+
{ name: "replicate-token", re: /\br8_[A-Za-z0-9]{37}\b/g, validate: notTestValue },
|
|
184
|
+
// Provider-prefix formats ported from Gitleaks (scrubProviderRules.ts).
|
|
185
|
+
...scrubProviderRules_1.PROVIDER_RULES.map((r) => ({
|
|
186
|
+
name: r.name,
|
|
187
|
+
re: r.re,
|
|
188
|
+
...(r.group !== undefined ? { group: r.group } : {}),
|
|
189
|
+
...(r.firstGroup ? { firstGroup: true } : {}),
|
|
190
|
+
keywords: r.keywords,
|
|
191
|
+
fallback: true,
|
|
192
|
+
validate: (v) => notTestValue(v) && (r.entropy === undefined || shannonEntropy(v) >= r.entropy),
|
|
193
|
+
})),
|
|
194
|
+
];
|
|
195
|
+
// Every rule keyword, found in one case-insensitive pass over the text
|
|
196
|
+
// instead of one search per keyword.
|
|
197
|
+
const ALL_KEYWORDS = [...new Set(RULES.flatMap((r) => r.keywords ?? []))].sort((a, b) => b.length - a.length);
|
|
198
|
+
const KEYWORD_RE = new RegExp(ALL_KEYWORDS.map((k) => k.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|"), "gi");
|
|
199
|
+
const KEYWORD_PARTS = new Map(ALL_KEYWORDS.map((k) => [k, ALL_KEYWORDS.filter((p) => k.includes(p))]));
|
|
200
|
+
function keywordsIn(input) {
|
|
201
|
+
const found = new Set();
|
|
202
|
+
if (!ALL_KEYWORDS.length)
|
|
203
|
+
return found;
|
|
204
|
+
KEYWORD_RE.lastIndex = 0;
|
|
205
|
+
let m;
|
|
206
|
+
while ((m = KEYWORD_RE.exec(input)) !== null) {
|
|
207
|
+
// A keyword inside a longer one (sk_ in sk_live_) is present too.
|
|
208
|
+
for (const k of KEYWORD_PARTS.get(m[0].toLowerCase()) ?? [])
|
|
209
|
+
found.add(k);
|
|
210
|
+
if (found.size === ALL_KEYWORDS.length)
|
|
211
|
+
break;
|
|
212
|
+
}
|
|
213
|
+
return found;
|
|
214
|
+
}
|
|
215
|
+
function scrubSecrets(input, opts = {}) {
|
|
216
|
+
if (!input)
|
|
217
|
+
return { text: input, redactions: [] };
|
|
218
|
+
// 1) Collect every candidate span across all rules.
|
|
219
|
+
const candidates = [];
|
|
220
|
+
const fallbacks = [];
|
|
221
|
+
const generic = [];
|
|
222
|
+
// A caller's rules are few; they skip the keyword pre-check.
|
|
223
|
+
const extra = (opts.extraRules ?? []).map((r) => ({ name: r.name, re: new RegExp(r.re.source, r.re.flags.includes("g") ? r.re.flags : r.re.flags + "g"), validate: notTestValue }));
|
|
224
|
+
const present = keywordsIn(input);
|
|
225
|
+
for (const rule of extra.length ? [...RULES, ...extra] : RULES) {
|
|
226
|
+
if (rule.keywords?.length && !rule.keywords.some((k) => present.has(k)))
|
|
227
|
+
continue;
|
|
228
|
+
rule.re.lastIndex = 0;
|
|
229
|
+
let m;
|
|
230
|
+
while ((m = rule.re.exec(input)) !== null) {
|
|
231
|
+
let start = m.index;
|
|
232
|
+
let end = m.index + m[0].length;
|
|
233
|
+
const groupIndex = rule.firstGroup ? m.findIndex((g, i) => i > 0 && g) : rule.group;
|
|
234
|
+
if (rule.firstGroup && groupIndex === -1) {
|
|
235
|
+
if (m[0].length === 0)
|
|
236
|
+
rule.re.lastIndex++;
|
|
237
|
+
continue; // no secret group matched: never redact the whole match
|
|
238
|
+
}
|
|
239
|
+
if (groupIndex !== undefined && groupIndex > 0) {
|
|
240
|
+
const gv = m[groupIndex];
|
|
241
|
+
if (gv === undefined || gv === "")
|
|
242
|
+
continue;
|
|
243
|
+
if (rule.validate && !rule.validate(gv, m))
|
|
244
|
+
continue;
|
|
245
|
+
// Locate the group within the match (it appears after the name/prefix).
|
|
246
|
+
const rel = m[0].lastIndexOf(gv);
|
|
247
|
+
if (rel >= 0) {
|
|
248
|
+
start = m.index + rel;
|
|
249
|
+
end = start + gv.length;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
else if (rule.validate && !rule.validate(m[0], m)) {
|
|
253
|
+
if (m[0].length === 0)
|
|
254
|
+
rule.re.lastIndex++;
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
if (end > start)
|
|
258
|
+
(rule.fallback ? fallbacks : exports.GENERIC_RULES.has(rule.name) ? generic : candidates).push({ rule: rule.name, start, end });
|
|
259
|
+
// Guard against zero-width matches looping forever.
|
|
260
|
+
if (m[0].length === 0)
|
|
261
|
+
rule.re.lastIndex++;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
if (candidates.length === 0 && fallbacks.length === 0 && generic.length === 0)
|
|
265
|
+
return { text: input, redactions: [] };
|
|
266
|
+
// 2) Resolve overlaps: earlier start wins; on tie, longer span wins. Most
|
|
267
|
+
// specific first: hand-written provider rules, then ported provider
|
|
268
|
+
// rules, then the generic ones (a secret-named variable, a bearer
|
|
269
|
+
// header, a JWT) only where nothing more specific matched.
|
|
270
|
+
const pick = (list, taken) => {
|
|
271
|
+
list.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
272
|
+
const out = [];
|
|
273
|
+
let cursor = -1;
|
|
274
|
+
let t = 0;
|
|
275
|
+
for (const c of list) {
|
|
276
|
+
if (c.start < cursor)
|
|
277
|
+
continue;
|
|
278
|
+
while (t < taken.length && taken[t].end <= c.start)
|
|
279
|
+
t++;
|
|
280
|
+
if (t < taken.length && taken[t].start < c.end)
|
|
281
|
+
continue;
|
|
282
|
+
out.push(c);
|
|
283
|
+
cursor = c.end;
|
|
284
|
+
}
|
|
285
|
+
return out;
|
|
286
|
+
};
|
|
287
|
+
let chosen = pick(candidates, []);
|
|
288
|
+
for (const tier of [fallbacks, generic]) {
|
|
289
|
+
if (tier.length)
|
|
290
|
+
chosen = [...chosen, ...pick(tier, chosen)].sort((a, b) => a.start - b.start);
|
|
291
|
+
}
|
|
292
|
+
return { text: redactSpans(input, chosen), redactions: chosen };
|
|
293
|
+
}
|
|
294
|
+
/** Rewrite `input` with each span replaced by `[REDACTED:<rule>]`. Spans must
|
|
295
|
+
* be sorted by start and not overlap. One forward pass: repeated
|
|
296
|
+
* slice+concat would copy the whole string per redaction (quadratic). */
|
|
297
|
+
function redactSpans(input, spans) {
|
|
298
|
+
const parts = [];
|
|
299
|
+
let last = 0;
|
|
300
|
+
for (const c of spans) {
|
|
301
|
+
parts.push(input.slice(last, c.start), `[REDACTED:${c.rule}]`);
|
|
302
|
+
last = c.end;
|
|
303
|
+
}
|
|
304
|
+
parts.push(input.slice(last));
|
|
305
|
+
return parts.join("");
|
|
306
|
+
}
|
|
307
|
+
// Rules without a provider-specific format: the only ones that misfire on
|
|
308
|
+
// test values, and the ones whose findings need a person to check.
|
|
309
|
+
exports.GENERIC_RULES = new Set(["secret-assignment", "bearer-token", "jwt"]);
|
|
310
|
+
const SEQUENTIAL = /0123456|1234567|2345678|3456789|abcdefg|bcdefgh|qwertyu/i;
|
|
311
|
+
const WORDISH = /^[A-Za-z]{2,}$/;
|
|
312
|
+
/** Values that share a key's shape but are test or sample values: words
|
|
313
|
+
* joined by _ or - with only digits or one repeated character between
|
|
314
|
+
* (`cdv_local_pilot_token_2026`, `…_token_000000`), a long counting run
|
|
315
|
+
* (`sk-1234567890…`), or a long repeated tail. Still removed from sessions;
|
|
316
|
+
* just not listed as a key to rotate. */
|
|
317
|
+
function looksLikeTestValue(value) {
|
|
318
|
+
if (SEQUENTIAL.test(value))
|
|
319
|
+
return true;
|
|
320
|
+
if (/(.)\1{9,}$/.test(value))
|
|
321
|
+
return true;
|
|
322
|
+
const parts = value.split(/[_-]+/).filter(Boolean);
|
|
323
|
+
if (parts.length < 3)
|
|
324
|
+
return false;
|
|
325
|
+
const words = parts.filter((p) => WORDISH.test(p)).length;
|
|
326
|
+
const filler = parts.filter((p) => /^\d+$/.test(p) || /^(.)\1*$/.test(p)).length;
|
|
327
|
+
return words >= 2 && words + filler === parts.length;
|
|
328
|
+
}
|
|
329
|
+
// `foo.test.ts`, `tests/x.py`, `test_auth.py`, `Cert_Tests.cs`, `fixtures/…`.
|
|
330
|
+
const TEST_FILE = /(?:\.test\.|\.spec\.|__tests__|__mocks__|fixtures?\/|(?:^|[\/_.-])tests?[\/_.-])/i;
|
|
331
|
+
// `CertPairTests.cs`, `LoginTest.java`: case matters, or `latests.ts` would match.
|
|
332
|
+
const TEST_FILE_SUFFIX = /[a-z0-9]Tests?\.\w+$/;
|
|
333
|
+
const isTestFile = (file) => TEST_FILE.test(file) || TEST_FILE_SUFFIX.test(file);
|
|
334
|
+
exports.isTestFile = isTestFile;
|
|
335
|
+
// `NAME=`, `NAME: `, `"NAME": "`, `export NAME=`, `--name=` just before the value.
|
|
336
|
+
const NAME_BEFORE = /([A-Za-z_][A-Za-z0-9_.-]{0,63})\??["']?\s*[:=]\s*\??["']?(?:Bearer\s+)?$/;
|
|
337
|
+
function nameBefore(input, start, lineStart) {
|
|
338
|
+
const window = input.slice(Math.max(lineStart, start - 160), start).replace(/\\[nrt]/g, " ");
|
|
339
|
+
const m = window.match(NAME_BEFORE)?.[1];
|
|
340
|
+
if (!m || /^(?:https?|Bearer|Authorization)$/i.test(m))
|
|
341
|
+
return m === "Authorization" ? m : undefined;
|
|
342
|
+
return m;
|
|
343
|
+
}
|
|
344
|
+
const FILE_PATH_FIELD = /"(?:file_path|filePath|path|target_file)"\s*:\s*"([^"]{1,400})"/g;
|
|
345
|
+
// A shell redirect to a path. Not an arrow (`=>`, `->`), not a diff or quote
|
|
346
|
+
// marker at the start of a line (`> AWS_KEY=…`, also JSON-escaped `\n> …`),
|
|
347
|
+
// and the target has to look like a path (no `=`, `\`, `@`, `{`, `,`).
|
|
348
|
+
const REDIRECT = /(?<![-=>\w]|\\n|\\r)(?<!^)(?:>>?|\btee(?:\s+-a)?)[ \t]*['"]?([\w.~/-]{2,300})/gm;
|
|
349
|
+
const PATH_LIKE = /[./]/;
|
|
350
|
+
const CONFIG_FILE = /((?:[\w.~-]+\/)*\.env(?:\.[\w-]+)?|(?:[\w.~-]+\/)*[\w.-]+\.(?:env|ya?ml|toml|ini|cfg|conf|properties|tfvars|npmrc|netrc|pgpass))(?![\w/])/g;
|
|
351
|
+
const lastTwo = (p) => p.replace(/\\\//g, "/").split("/").filter(Boolean).slice(-2).join("/");
|
|
352
|
+
function lastMatch(re, text) {
|
|
353
|
+
let last;
|
|
354
|
+
for (const m of text.matchAll(re))
|
|
355
|
+
last = m[1];
|
|
356
|
+
return last;
|
|
357
|
+
}
|
|
358
|
+
/** The file this line writes or reads, if it names one. */
|
|
359
|
+
function fileOf(input, start, lineStart) {
|
|
360
|
+
const lineEnd = input.indexOf("\n", start);
|
|
361
|
+
const line = input.slice(lineStart, lineEnd === -1 ? input.length : lineEnd);
|
|
362
|
+
const before = input.slice(Math.max(lineStart, start - 400), start);
|
|
363
|
+
const redirect = lastMatch(REDIRECT, before);
|
|
364
|
+
const hit = lastMatch(FILE_PATH_FIELD, line) ?? (redirect && PATH_LIKE.test(redirect) && !/^[.\d-]+$/.test(redirect) ? redirect : undefined) ?? lastMatch(CONFIG_FILE, before);
|
|
365
|
+
if (!hit || hit.startsWith("/dev/"))
|
|
366
|
+
return undefined;
|
|
367
|
+
const short = lastTwo(hit);
|
|
368
|
+
return short && short.length <= 120 ? short : undefined;
|
|
369
|
+
}
|
|
370
|
+
// Transcript shapes (one JSON object per line):
|
|
371
|
+
// Claude Code: {"type":"user"|"assistant","message":{"role","content":[{type:"tool_use",name,input}|{type:"tool_result"}|text]}}
|
|
372
|
+
// Codex: {"type":"response_item","payload":{"type":"message",role}|{"type":"function_call",name,arguments}|{"type":"function_call_output"}}
|
|
373
|
+
// Others: matched by the same generic markers (role, *tool*call*, *result*/*output*).
|
|
374
|
+
const TOOL_CALL_TYPE = /^(?:tool_use|function_call|custom_tool_call|tool_call|toolCall|tool_request|tool\.execution_start|server_tool_use|mcp_tool_use)$/;
|
|
375
|
+
const TOOL_OUTPUT_TYPE = /^(?:tool_result|function_call_output|custom_tool_call_output|tool_output|toolResult|tool\.execution_complete|mcp_tool_result)$/;
|
|
376
|
+
const SHELL_TOOL = /bash|shell|exec|terminal|run_command|run_terminal/i;
|
|
377
|
+
const GIT_COMMIT = /\bgit\s+(?:[^\n]*\s)?(?:commit|push)\b/;
|
|
378
|
+
// A person's message this long was pasted, not typed: keys in it came along
|
|
379
|
+
// with a log, config file or API response.
|
|
380
|
+
const PASTE_LINES = 8;
|
|
381
|
+
const PASTE_CHARS = 800;
|
|
382
|
+
/** A line of the transcript, parsed once per scan. */
|
|
383
|
+
function lineAt(input, at, cache) {
|
|
384
|
+
const lineStart = input.lastIndexOf("\n", at - 1) + 1;
|
|
385
|
+
let parsed = cache.get(lineStart);
|
|
386
|
+
if (parsed === undefined) {
|
|
387
|
+
const lineEnd = input.indexOf("\n", at);
|
|
388
|
+
try {
|
|
389
|
+
parsed = JSON.parse(input.slice(lineStart, lineEnd === -1 ? input.length : lineEnd));
|
|
390
|
+
}
|
|
391
|
+
catch {
|
|
392
|
+
parsed = null;
|
|
393
|
+
}
|
|
394
|
+
cache.set(lineStart, parsed);
|
|
395
|
+
}
|
|
396
|
+
return parsed;
|
|
397
|
+
}
|
|
398
|
+
/** The tool a result belongs to. Claude Code's tool_result carries only the
|
|
399
|
+
* call's id, so find the tool_use with that id (an earlier line). */
|
|
400
|
+
function toolForResult(input, id, cache) {
|
|
401
|
+
if (!/^[\w-]{6,}$/.test(id))
|
|
402
|
+
return undefined;
|
|
403
|
+
const at = input.indexOf(`"${id}"`);
|
|
404
|
+
if (at === -1)
|
|
405
|
+
return undefined;
|
|
406
|
+
let name;
|
|
407
|
+
const walk = (node) => {
|
|
408
|
+
if (name || !node || typeof node !== "object")
|
|
409
|
+
return;
|
|
410
|
+
if (Array.isArray(node)) {
|
|
411
|
+
node.forEach(walk);
|
|
412
|
+
return;
|
|
413
|
+
}
|
|
414
|
+
const n = node;
|
|
415
|
+
if ((n.id === id || n.call_id === id || n.toolCallId === id) && typeof n.name === "string" && TOOL_CALL_TYPE.test(typeof n.type === "string" ? n.type : "tool_use")) {
|
|
416
|
+
name = n.name;
|
|
417
|
+
return;
|
|
418
|
+
}
|
|
419
|
+
Object.values(n).forEach(walk);
|
|
420
|
+
};
|
|
421
|
+
walk(lineAt(input, at, cache));
|
|
422
|
+
return name;
|
|
423
|
+
}
|
|
424
|
+
/** Classify one redaction by the nearest enclosing tool call, tool result or
|
|
425
|
+
* message role in its transcript line. Non-JSON lines are "unknown". */
|
|
426
|
+
function locate(input, r, cache, exact) {
|
|
427
|
+
const parsed = lineAt(input, r.start, cache);
|
|
428
|
+
if (parsed === null || typeof parsed !== "object")
|
|
429
|
+
return { where: "unknown" };
|
|
430
|
+
// Search for an alphanumeric run from the secret: JSON escaping (\n, \")
|
|
431
|
+
// differs between the raw line and the parsed strings.
|
|
432
|
+
const runs = exact ? null : input.slice(r.start, r.end).match(/[A-Za-z0-9]{8,}/g);
|
|
433
|
+
const needle = exact ?? (runs ? runs.reduce((a, b) => (b.length > a.length ? b : a)) : null);
|
|
434
|
+
if (!needle)
|
|
435
|
+
return { where: "unknown" };
|
|
436
|
+
return classify(parsed, needle, input, cache);
|
|
437
|
+
}
|
|
438
|
+
/** Where in one transcript line (a JSONL record) some text appeared, and the
|
|
439
|
+
* file that line reads or writes, if it names one. For any detector that
|
|
440
|
+
* works line by line (PHI/PII). `text` is the matched text as it reads
|
|
441
|
+
* after JSON unescaping. */
|
|
442
|
+
function locateInLine(line, text) {
|
|
443
|
+
let parsed;
|
|
444
|
+
try {
|
|
445
|
+
parsed = JSON.parse(line);
|
|
446
|
+
}
|
|
447
|
+
catch {
|
|
448
|
+
parsed = null;
|
|
449
|
+
}
|
|
450
|
+
const fileHit = lastMatch(FILE_PATH_FIELD, line);
|
|
451
|
+
const file = fileHit && !fileHit.startsWith("/dev/") ? lastTwo(fileHit).slice(0, 120) : undefined;
|
|
452
|
+
if (parsed === null || typeof parsed !== "object")
|
|
453
|
+
return { where: "unknown", ...(file ? { file } : {}) };
|
|
454
|
+
const runs = text.match(/[A-Za-z0-9]{4,}/g);
|
|
455
|
+
const needle = runs ? runs.reduce((a, b) => (b.length > a.length ? b : a)) : null;
|
|
456
|
+
const loc = needle ? classify(parsed, needle, line, new Map()) : { where: "unknown" };
|
|
457
|
+
return { ...loc, ...(file ? { file } : {}) };
|
|
458
|
+
}
|
|
459
|
+
/** Where `needle` sits in one parsed record. `input` is the whole transcript
|
|
460
|
+
* (or just this line), used to name the tool behind a tool result. */
|
|
461
|
+
function classify(parsed, needle, input, cache) {
|
|
462
|
+
let path = null;
|
|
463
|
+
const walk = (node, trail) => {
|
|
464
|
+
if (path)
|
|
465
|
+
return;
|
|
466
|
+
if (typeof node === "string") {
|
|
467
|
+
if (node.includes(needle))
|
|
468
|
+
path = trail;
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
471
|
+
if (!node || typeof node !== "object")
|
|
472
|
+
return;
|
|
473
|
+
if (Array.isArray(node)) {
|
|
474
|
+
node.forEach((c) => walk(c, [...trail, { node: c }]));
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
for (const [k, c] of Object.entries(node))
|
|
478
|
+
walk(c, [...trail, { node: c, key: k }]);
|
|
479
|
+
};
|
|
480
|
+
walk(parsed, [{ node: parsed }]);
|
|
481
|
+
if (!path)
|
|
482
|
+
return { where: "unknown" };
|
|
483
|
+
const frames = path;
|
|
484
|
+
for (let i = frames.length - 1; i >= 0; i--) {
|
|
485
|
+
const n = frames[i].node;
|
|
486
|
+
if (!n || typeof n !== "object" || Array.isArray(n))
|
|
487
|
+
continue;
|
|
488
|
+
const type = typeof n.type === "string" ? n.type : "";
|
|
489
|
+
const tool = typeof n.name === "string" ? n.name : typeof n.toolName === "string" ? n.toolName : undefined;
|
|
490
|
+
if (TOOL_CALL_TYPE.test(type)) {
|
|
491
|
+
const args = JSON.stringify(n.input ?? n.arguments ?? n.args ?? "");
|
|
492
|
+
if (tool && SHELL_TOOL.test(tool) && GIT_COMMIT.test(args))
|
|
493
|
+
return { where: "commit", tool };
|
|
494
|
+
return tool ? { where: "tool_call", tool } : { where: "tool_call" };
|
|
495
|
+
}
|
|
496
|
+
if (TOOL_OUTPUT_TYPE.test(type)) {
|
|
497
|
+
const id = n.tool_use_id ?? n.call_id ?? n.toolCallId;
|
|
498
|
+
const named = tool ?? (typeof id === "string" ? toolForResult(input, id, cache) : undefined);
|
|
499
|
+
return named ? { where: "tool_output", tool: named } : { where: "tool_output" };
|
|
500
|
+
}
|
|
501
|
+
if (n.role === "user" || (type === "user" && i === 0)) {
|
|
502
|
+
const line = frames[0].node;
|
|
503
|
+
// Claude Code repeats a tool's result beside the message (toolUseResult).
|
|
504
|
+
if (frames.some((f) => f.key === "toolUseResult")) {
|
|
505
|
+
const content = line.message?.content;
|
|
506
|
+
const id = Array.isArray(content) ? content.find((c) => c && typeof c === "object" && "tool_use_id" in c)?.tool_use_id : undefined;
|
|
507
|
+
const named = typeof id === "string" ? toolForResult(input, id, cache) : undefined;
|
|
508
|
+
return named ? { where: "tool_output", tool: named } : { where: "tool_output" };
|
|
509
|
+
}
|
|
510
|
+
if (line.isMeta === true || line.isCompactSummary === true)
|
|
511
|
+
return { where: "attached" };
|
|
512
|
+
const text = frames[frames.length - 1].node;
|
|
513
|
+
if (typeof text === "string" && (text.length >= PASTE_CHARS || text.split("\n").length >= PASTE_LINES))
|
|
514
|
+
return { where: "pasted" };
|
|
515
|
+
return { where: "typed" };
|
|
516
|
+
}
|
|
517
|
+
if (n.role === "assistant" || (type === "assistant" && i === 0))
|
|
518
|
+
return { where: "assistant" };
|
|
519
|
+
}
|
|
520
|
+
return { where: "unknown" };
|
|
521
|
+
}
|
|
522
|
+
/** Where each `[REDACTED:<rule>]` marker sits in an already-scrubbed
|
|
523
|
+
* transcript. Re-labels older findings that predate typed/pasted/attached. */
|
|
524
|
+
function locateMarkers(scrubbed, rule) {
|
|
525
|
+
const marker = `[REDACTED:${rule}]`;
|
|
526
|
+
const cache = new Map();
|
|
527
|
+
const out = [];
|
|
528
|
+
for (let at = scrubbed.indexOf(marker); at !== -1; at = scrubbed.indexOf(marker, at + marker.length)) {
|
|
529
|
+
const loc = locate(scrubbed, { rule, start: at, end: at + marker.length }, cache, marker);
|
|
530
|
+
if (!out.some((l) => l.where === loc.where && l.tool === loc.tool))
|
|
531
|
+
out.push(loc);
|
|
532
|
+
if (out.length >= 16)
|
|
533
|
+
break;
|
|
534
|
+
}
|
|
535
|
+
return out;
|
|
536
|
+
}
|
|
537
|
+
// ─── Is this a key someone should rotate? ───
|
|
538
|
+
//
|
|
539
|
+
// Everything a rule matches is removed from the session. Only a key that is
|
|
540
|
+
// real, still usable and worth a person's time is listed to rotate: the list
|
|
541
|
+
// is read as "do this", so a placeholder on it costs more than a miss.
|
|
542
|
+
/** Rules that match a shape (a secret-named variable, a header, a URL, a PEM
|
|
543
|
+
* block) rather than a provider's own key format. */
|
|
544
|
+
exports.SHAPE_RULES = new Set([
|
|
545
|
+
"secret-assignment", "bearer-token", "jwt", "pem-private-key", "url-password", "curl-auth-header", "curl-auth-user",
|
|
546
|
+
]);
|
|
547
|
+
// Words placeholder passwords and sample tokens are built from.
|
|
548
|
+
const FILLER_WORD = /secret|password|passwd|passw|pass|pwd|token|apikey|key|auth|bearer|basic|api|app|access|refresh|client|user|name|admin|root|guest|test(?:ing)?|dummy|sample|example|demo|default|change(?:me|it)?|service|local(?:host)?|super|your|mine|my|the|new|old|temp|tmp|dev|prod|stag(?:ing)?|value|string|here|foo|bar|baz|qux|private|public|hello|world|welcome|letmein/gi;
|
|
549
|
+
const LEET = { "0": "o", "1": "l", "3": "e", "4": "a", "5": "s", "7": "t", "@": "a", $: "s" };
|
|
550
|
+
/** Built from filler words plus digits or punctuation: `secret1`,
|
|
551
|
+
* `MySecret123`, `pa55w0rd`, `test-api-key`. */
|
|
552
|
+
function madeOfFillerWords(v) {
|
|
553
|
+
const residue = (t) => {
|
|
554
|
+
FILLER_WORD.lastIndex = 0;
|
|
555
|
+
if (!FILLER_WORD.test(t))
|
|
556
|
+
return Infinity;
|
|
557
|
+
return t.replace(FILLER_WORD, "").replace(/[^A-Za-z]/g, "").length;
|
|
558
|
+
};
|
|
559
|
+
return residue(v) <= 3 || residue(v.replace(/[013457@$]/g, (c) => LEET[c])) <= 3;
|
|
560
|
+
}
|
|
561
|
+
/** Two or more three-character counting runs: `abc123`, `abc123def456`.
|
|
562
|
+
* Three in a long value, where two can happen by chance (64 hex digits). */
|
|
563
|
+
function hasCountingRuns(v) {
|
|
564
|
+
const t = v; // same case only: `hIj` in a mixed-case key is chance, not counting
|
|
565
|
+
const need = t.length > 24 ? 3 : 2;
|
|
566
|
+
let runs = 0;
|
|
567
|
+
for (let i = 0; i + 2 < t.length; i++) {
|
|
568
|
+
const a = t.charCodeAt(i);
|
|
569
|
+
if (!/[A-Za-z0-9]/.test(t[i]) || t.charCodeAt(i + 1) !== a + 1 || t.charCodeAt(i + 2) !== a + 2)
|
|
570
|
+
continue;
|
|
571
|
+
if (++runs >= need)
|
|
572
|
+
return true;
|
|
573
|
+
i += 2;
|
|
574
|
+
}
|
|
575
|
+
return false;
|
|
576
|
+
}
|
|
577
|
+
// Letter pairs that turn up inside English words and service names. A run of
|
|
578
|
+
// random letters almost always has a pair that isn't here.
|
|
579
|
+
const WORD_PAIRS = new Set(("ab ac ad ae ag ai ak al am an ao ap ar as at au av aw ax ay az ba be bi bj bl bo br bs bu by ca ce ch ci ck cl co " +
|
|
580
|
+
"cr cs ct cu cy da db de dg di dm do dr ds du dw dy ea eb ec ed ee ef eg ei el em en eo ep er es et eu ev ew ex ey " +
|
|
581
|
+
"fa fe ff fi fl fo fr ft fu ga ge gh gi gl gn go gr gs gu gy ha hc he hi ho hs ht hu ia ib ic id ie if ig il im in " +
|
|
582
|
+
"io ip ir is it iu iv ix iz ja je jo ju ka ke ki kn ko ks ku la ld le lf li lk ll lm lo lp ls lt lu lv ly ma mb me " +
|
|
583
|
+
"mi mm mo mp ms mu my na nc nd ne nf ng ni nk nm nn no ns nt nv ny oa ob oc od oe of og oi ok ol om on oo op or os " +
|
|
584
|
+
"ot ou ov ow ox oz pa pe ph pi pl po pp pr ps pt pu py qu ra rc rd re rg ri rk rl rm rn ro rr rs rt ru ry sa sc se " +
|
|
585
|
+
"sh si sk sl sm sn so sp ss st su sw sy ta te th ti tl to tr ts tt tu tw ty ua ub uc ud ue ug ui ul um un uo up ur " +
|
|
586
|
+
"us ut va ve vi vo vu wa we wh wi wo ws xp xt ya yc ye yl ym yn yo yp ys yt za ze zi zo").split(" "));
|
|
587
|
+
/** A token that reads as a word: three or more letters in one case (or
|
|
588
|
+
* Capitalised), a vowel, and letter pairs words are made of. A long one
|
|
589
|
+
* may have one odd pair (a product name). */
|
|
590
|
+
function readsAsWord(t) {
|
|
591
|
+
if (t.length < 3 || !/^(?:[a-z]+|[A-Z]+|[A-Z][a-z]+)$/.test(t))
|
|
592
|
+
return false;
|
|
593
|
+
const l = t.toLowerCase();
|
|
594
|
+
if (!/[aeiouy]/.test(l))
|
|
595
|
+
return false;
|
|
596
|
+
let odd = 0;
|
|
597
|
+
for (let i = 0; i + 1 < l.length; i++)
|
|
598
|
+
if (!WORD_PAIRS.has(l.slice(i, i + 2)))
|
|
599
|
+
odd++;
|
|
600
|
+
return odd <= (l.length >= 7 ? 1 : 0);
|
|
601
|
+
}
|
|
602
|
+
/** The name or location of a secret, or a value built from words, rather
|
|
603
|
+
* than a random key: an ARN, a secret-store path, or a value whose letters
|
|
604
|
+
* are mostly whole words (`mobile-data/client2`, `bracecomb-hc-service`,
|
|
605
|
+
* `mobileDataClientV2`). A random key does not break into words, whatever
|
|
606
|
+
* separators it has: under 1% of random 16-character keys pass this, and
|
|
607
|
+
* next to none at 24 or more. */
|
|
608
|
+
function namesASecret(v) {
|
|
609
|
+
if (/^arn:[a-z-]+:/i.test(v))
|
|
610
|
+
return true;
|
|
611
|
+
if (/^(?:vault|ssm|secretsmanager|keyvault|op|ref|env|file):/i.test(v))
|
|
612
|
+
return true;
|
|
613
|
+
if (/(?:^|\/)secrets?\//i.test(v))
|
|
614
|
+
return true; // secret/data/app, projects/p/secrets/name
|
|
615
|
+
const tokens = v
|
|
616
|
+
.replace(/([a-z])([A-Z])/g, "$1 $2")
|
|
617
|
+
.replace(/([A-Za-z])(\d)/g, "$1 $2")
|
|
618
|
+
.replace(/(\d)([A-Za-z])/g, "$1 $2")
|
|
619
|
+
.split(/[^A-Za-z0-9]+/)
|
|
620
|
+
.filter(Boolean);
|
|
621
|
+
let total = 0;
|
|
622
|
+
let letters = 0;
|
|
623
|
+
let inWords = 0;
|
|
624
|
+
for (const t of tokens) {
|
|
625
|
+
total += t.length;
|
|
626
|
+
if (/^\d+$/.test(t))
|
|
627
|
+
continue;
|
|
628
|
+
letters += t.length;
|
|
629
|
+
if (readsAsWord(t))
|
|
630
|
+
inWords += t.length;
|
|
631
|
+
}
|
|
632
|
+
return inWords >= 6 && inWords / letters >= 0.7 && letters / total >= 0.5;
|
|
633
|
+
}
|
|
634
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
635
|
+
/** A JWT that is cut short, already expired, or issued for a day or less:
|
|
636
|
+
* nothing to rotate. One with no expiry, or a long one, stays. */
|
|
637
|
+
function jwtNotWorthRotating(v, now) {
|
|
638
|
+
const parts = v.split(".");
|
|
639
|
+
if (parts.length < 3)
|
|
640
|
+
return true; // header only, or cut off
|
|
641
|
+
let payload;
|
|
642
|
+
try {
|
|
643
|
+
payload = JSON.parse(atob(parts[1].replace(/-/g, "+").replace(/_/g, "/")));
|
|
644
|
+
}
|
|
645
|
+
catch {
|
|
646
|
+
return true; // truncated in the transcript: not a usable token
|
|
647
|
+
}
|
|
648
|
+
if (!payload || typeof payload.exp !== "number")
|
|
649
|
+
return false;
|
|
650
|
+
if (payload.exp * 1000 < now + DAY_MS)
|
|
651
|
+
return true;
|
|
652
|
+
return typeof payload.iat === "number" && (payload.exp - payload.iat) * 1000 <= DAY_MS;
|
|
653
|
+
}
|
|
654
|
+
/** The base64 body of a PEM block, without armor headers or (escaped) line breaks. */
|
|
655
|
+
function pemBody(block) {
|
|
656
|
+
return block
|
|
657
|
+
.replace(/-----BEGIN[A-Z ]*PRIVATE KEY-----/, "")
|
|
658
|
+
.split("-----END")[0]
|
|
659
|
+
.split(/\\r|\\n|\r|\n/)
|
|
660
|
+
.filter((l) => !l.includes(":")) // Proc-Type / DEK-Info
|
|
661
|
+
.join("")
|
|
662
|
+
.replace(/\s+/g, "");
|
|
663
|
+
}
|
|
664
|
+
/** True when a redacted value should not be listed as a key to rotate: a
|
|
665
|
+
* placeholder, a sample or test value, a reference to a variable, or a
|
|
666
|
+
* credential that expires by itself. `now` is injectable for tests. */
|
|
667
|
+
function notWorthRotating(k, now = Date.now()) {
|
|
668
|
+
const { rule, name, file } = k;
|
|
669
|
+
const value = k.value.trim();
|
|
670
|
+
if (looksLikeTestValue(value))
|
|
671
|
+
return true;
|
|
672
|
+
if (exports.SHAPE_RULES.has(rule) && file && (0, exports.isTestFile)(file))
|
|
673
|
+
return true;
|
|
674
|
+
if (rule === "pem-private-key") {
|
|
675
|
+
const body = pemBody(value);
|
|
676
|
+
// The smallest real key (Ed25519, PKCS#8) is 64 base64 characters.
|
|
677
|
+
return body.length < 60 || !/^[A-Za-z0-9+/=]+$/.test(body) || shannonEntropy(body) < 4;
|
|
678
|
+
}
|
|
679
|
+
// Documentation keys (AKIAIOSFODNN7EXAMPLE), `<your-token>`, `xxxxxxxx`.
|
|
680
|
+
if (PLACEHOLDER.test(value))
|
|
681
|
+
return true;
|
|
682
|
+
if (hasCountingRuns(value))
|
|
683
|
+
return true;
|
|
684
|
+
// Credentials that expire by themselves: STS keys and session tokens, JWTs.
|
|
685
|
+
if (rule === "aws-access-key" && value.startsWith("ASIA"))
|
|
686
|
+
return true;
|
|
687
|
+
if (name && /(?:session|security)[_-]?token$/i.test(name))
|
|
688
|
+
return true;
|
|
689
|
+
if (/^(?:IQoJ|FwoG|FQoG|AQoD)[A-Za-z0-9+/]{20,}/.test(value))
|
|
690
|
+
return true; // STS session token
|
|
691
|
+
if (value.startsWith("eyJ"))
|
|
692
|
+
return jwtNotWorthRotating(value, now);
|
|
693
|
+
if (!exports.SHAPE_RULES.has(rule))
|
|
694
|
+
return false;
|
|
695
|
+
// `X-Atlassian-Token: no-check` switches off a CSRF check; it isn't a credential.
|
|
696
|
+
if (name && /^x-atlassian-token$/i.test(name))
|
|
697
|
+
return true;
|
|
698
|
+
let secret = value;
|
|
699
|
+
if (rule === "curl-auth-user") {
|
|
700
|
+
const at = value.indexOf(":");
|
|
701
|
+
const user = value.slice(0, at).replace(/["']/g, "");
|
|
702
|
+
secret = value.slice(at + 1).replace(/["']/g, "");
|
|
703
|
+
if (at < 0 || !secret || secret === user)
|
|
704
|
+
return true;
|
|
705
|
+
}
|
|
706
|
+
if (rule === "url-password" && name && secret === name)
|
|
707
|
+
return true; // postgres:postgres@
|
|
708
|
+
if (secret.includes("$"))
|
|
709
|
+
return true; // $TOKEN, ${TOKEN}: a reference, not the key
|
|
710
|
+
if (/^[A-Za-z._-]+$/.test(secret))
|
|
711
|
+
return true; // words or an identifier: `no-check`, `header.payload.signature`
|
|
712
|
+
if (/^[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+$/.test(secret))
|
|
713
|
+
return true; // OAUTH2_CLIENT_SECRET: a variable's name
|
|
714
|
+
if (madeOfFillerWords(secret))
|
|
715
|
+
return true;
|
|
716
|
+
if (rule === "secret-assignment" && namesASecret(secret))
|
|
717
|
+
return true; // points at a secret, isn't one
|
|
718
|
+
return rule !== "url-password" && shannonEntropy(secret) < 3;
|
|
719
|
+
}
|
|
720
|
+
/** Last four letters or digits of a value. A PEM block ends in a fixed
|
|
721
|
+
* footer, and in a transcript its line breaks are a literal `\n`: the hint
|
|
722
|
+
* comes from the key body, without either. */
|
|
723
|
+
function hintOf(rule, value) {
|
|
724
|
+
const body = rule === "pem-private-key" ? value.split("-----END")[0].replace(/\\[nr]/g, "") : value;
|
|
725
|
+
return body.replace(/[^A-Za-z0-9]/g, "").slice(-4);
|
|
726
|
+
}
|
|
727
|
+
const MAX_LOCATIONS = 8;
|
|
728
|
+
/** One finding per distinct (rule, last-4) in a scrub result, with every
|
|
729
|
+
* distinct place it appeared. */
|
|
730
|
+
function secretFindings(input, redactions, opts = {}) {
|
|
731
|
+
const seen = new Map();
|
|
732
|
+
const cache = new Map();
|
|
733
|
+
for (const r of redactions) {
|
|
734
|
+
const value = input.slice(r.start, r.end);
|
|
735
|
+
const hint = hintOf(r.rule, value);
|
|
736
|
+
if (!hint)
|
|
737
|
+
continue;
|
|
738
|
+
const lineStart = input.lastIndexOf("\n", r.start - 1) + 1;
|
|
739
|
+
const name = nameBefore(input, r.start, lineStart);
|
|
740
|
+
const file = fileOf(input, r.start, lineStart);
|
|
741
|
+
if (notWorthRotating({ rule: r.rule, value, ...(name ? { name } : {}), ...(file ? { file } : {}) }, opts.now))
|
|
742
|
+
continue;
|
|
743
|
+
if (opts.skip?.({ rule: r.rule, value, ...(name ? { name } : {}), ...(file ? { file } : {}) }))
|
|
744
|
+
continue;
|
|
745
|
+
const key = `${r.rule}\u0000${hint}`;
|
|
746
|
+
const finding = seen.get(key) ?? { rule: r.rule, hint, locations: [] };
|
|
747
|
+
if (name && !finding.name)
|
|
748
|
+
finding.name = name;
|
|
749
|
+
if (file && !finding.file)
|
|
750
|
+
finding.file = file;
|
|
751
|
+
seen.set(key, finding);
|
|
752
|
+
const loc = locate(input, r, cache);
|
|
753
|
+
if (finding.locations.length < MAX_LOCATIONS && !finding.locations.some((l) => l.where === loc.where && l.tool === loc.tool)) {
|
|
754
|
+
finding.locations.push(loc);
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
return [...seen.values()];
|
|
758
|
+
}
|
|
759
|
+
// Key-ish names the secret-assignment rule doesn't cover (it already handles
|
|
760
|
+
// API_KEY, SECRET, TOKEN, PASSWORD…). The keyword must end the name.
|
|
761
|
+
const KEYISH_ASSIGNMENT = /\b([A-Za-z][A-Za-z0-9_-]*?(?:KEY|CREDENTIALS?|CREDS?|AUTH|SIGNATURE|SIGNING[_-]?KEY|SALT|DSN|PASS|PWD|PASSPHRASE|CONN(?:ECTION)?[_-]?STRING|COOKIE))\\?['"]?\s*[:=]\s*(?:\\?['"])?([A-Za-z0-9+/=_.-]{20,200})(?:\\?['"])?/gi;
|
|
762
|
+
// Names that hold random-looking values that aren't secrets.
|
|
763
|
+
const NOT_SECRET_NAME = /(?:cache|idempoten|partition|sort|primary|foreign|composite|unique|hash|etag|checksum|digest|storage|object|row|item|react|list|map|lookup|index|public|publishable|pub|page|continuation|cursor|nonce|trace|span|request|correlation|commit|sha|fingerprint|kid|key[_-]?id|keyof|monkey|turkey|hotkey|shortcut)/i;
|
|
764
|
+
// Unknown vendor tokens: `acme_live_…`, `xyz_prod_…`, `foo_sk_…`. pk_ is public.
|
|
765
|
+
const PREFIXED_TOKEN = /\b((?!pk_)[a-z][a-z0-9]{1,9}_(?:live|prod|secret|sk|key|token|api)_)([A-Za-z0-9]{24,})\b/g;
|
|
766
|
+
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
767
|
+
function shannon(v) {
|
|
768
|
+
const counts = new Map();
|
|
769
|
+
for (const c of v)
|
|
770
|
+
counts.set(c, (counts.get(c) ?? 0) + 1);
|
|
771
|
+
let h = 0;
|
|
772
|
+
for (const n of counts.values()) {
|
|
773
|
+
const p = n / v.length;
|
|
774
|
+
h -= p * Math.log2(p);
|
|
775
|
+
}
|
|
776
|
+
return h;
|
|
777
|
+
}
|
|
778
|
+
/** Random enough to be a credential, and not a known non-secret shape. */
|
|
779
|
+
function randomLooking(v) {
|
|
780
|
+
if (v.length < 20 || UUID.test(v))
|
|
781
|
+
return false;
|
|
782
|
+
if (/^\d+$/.test(v) || /^[A-Za-z]+$/.test(v))
|
|
783
|
+
return false;
|
|
784
|
+
if (/[._-]{2}|^[._-]|[._-]$/.test(v))
|
|
785
|
+
return false;
|
|
786
|
+
if (/\.[a-z]{2,4}$/i.test(v) || /^[a-z]+(?:[._-][a-z]+)+$/i.test(v))
|
|
787
|
+
return false; // file names, dotted/kebab words
|
|
788
|
+
if (PLACEHOLDER.test(v))
|
|
789
|
+
return false;
|
|
790
|
+
// Slugs and ids: `openai/gpt-5-mini` (base64 never mixes / and -), and
|
|
791
|
+
// lowercase names like `local-team_app-4870`.
|
|
792
|
+
if (v.includes("/") && v.includes("-"))
|
|
793
|
+
return false;
|
|
794
|
+
if (/^[a-z0-9]+(?:[_-][a-z0-9]+){2,}$/.test(v))
|
|
795
|
+
return false;
|
|
796
|
+
if (!/[0-9]/.test(v) || !/[A-Za-z]/.test(v))
|
|
797
|
+
return false;
|
|
798
|
+
return shannon(v) >= 3.5;
|
|
799
|
+
}
|
|
800
|
+
function shapeOf(v) {
|
|
801
|
+
if (/^[0-9a-f]+$/i.test(v))
|
|
802
|
+
return "hex";
|
|
803
|
+
if (/[+/=]/.test(v))
|
|
804
|
+
return "base64";
|
|
805
|
+
return "alnum";
|
|
806
|
+
}
|
|
807
|
+
// Long runs with letters AND digits get masked in snippets; names like
|
|
808
|
+
// `partnerCredentials` stay readable.
|
|
809
|
+
const LONG_RUN = /[A-Za-z0-9+/=_-]{16,}/g;
|
|
810
|
+
const maskRuns = (s) => s.replace(LONG_RUN, (r) => (/[0-9]/.test(r) && /[A-Za-z]/.test(r) ? "••••" : r));
|
|
811
|
+
/** The decoded string (prompt text, command, tool output…) the value sits
|
|
812
|
+
* in, when its transcript line is JSON; else the raw line. */
|
|
813
|
+
function containingText(text, start, end, cache) {
|
|
814
|
+
const lineStart = text.lastIndexOf("\n", start - 1) + 1;
|
|
815
|
+
const lineEndRaw = text.indexOf("\n", end);
|
|
816
|
+
const raw = text.slice(lineStart, lineEndRaw === -1 ? text.length : lineEndRaw);
|
|
817
|
+
let parsed = cache.get(lineStart);
|
|
818
|
+
if (parsed === undefined) {
|
|
819
|
+
try {
|
|
820
|
+
parsed = JSON.parse(raw);
|
|
821
|
+
}
|
|
822
|
+
catch {
|
|
823
|
+
parsed = null;
|
|
824
|
+
}
|
|
825
|
+
cache.set(lineStart, parsed);
|
|
826
|
+
}
|
|
827
|
+
const value = text.slice(start, end);
|
|
828
|
+
let hit = null;
|
|
829
|
+
const walk = (node) => {
|
|
830
|
+
if (hit !== null)
|
|
831
|
+
return;
|
|
832
|
+
if (typeof node === "string") {
|
|
833
|
+
if (!node.includes(value))
|
|
834
|
+
return;
|
|
835
|
+
// Codex function_call arguments are JSON in a string: look inside.
|
|
836
|
+
if (/^[[{]/.test(node.trim())) {
|
|
837
|
+
try {
|
|
838
|
+
walk(JSON.parse(node));
|
|
839
|
+
}
|
|
840
|
+
catch { /* not JSON */ }
|
|
841
|
+
}
|
|
842
|
+
if (hit === null)
|
|
843
|
+
hit = node;
|
|
844
|
+
return;
|
|
845
|
+
}
|
|
846
|
+
if (node && typeof node === "object")
|
|
847
|
+
for (const c of Object.values(node))
|
|
848
|
+
walk(c);
|
|
849
|
+
};
|
|
850
|
+
if (parsed !== null)
|
|
851
|
+
walk(parsed);
|
|
852
|
+
return hit ?? raw;
|
|
853
|
+
}
|
|
854
|
+
/** ~80 characters around a value, the value shown as ••••last4 and every
|
|
855
|
+
* other random-looking run masked, so the snippet can't carry a secret. */
|
|
856
|
+
function snippetAround(text, start, end, hint, cache) {
|
|
857
|
+
const value = text.slice(start, end);
|
|
858
|
+
const host = containingText(text, start, end, cache);
|
|
859
|
+
const at = host.indexOf(value);
|
|
860
|
+
const from = Math.max(0, at - 40);
|
|
861
|
+
const to = Math.min(host.length, at + value.length + 40);
|
|
862
|
+
const before = maskRuns(host.slice(from, at));
|
|
863
|
+
const after = maskRuns(host.slice(at + value.length, to));
|
|
864
|
+
return `${from > 0 ? "…" : ""}${before}••••${hint}${after}${to < host.length ? "…" : ""}`.replace(/\s+/g, " ").trim();
|
|
865
|
+
}
|
|
866
|
+
const MAX_POSSIBLE = 20;
|
|
867
|
+
function possibleSecrets(text, limit = MAX_POSSIBLE) {
|
|
868
|
+
if (!text)
|
|
869
|
+
return [];
|
|
870
|
+
const found = new Map();
|
|
871
|
+
const cache = new Map();
|
|
872
|
+
const add = (label, prefix, value, start) => {
|
|
873
|
+
const end = start + value.length;
|
|
874
|
+
const hint = value.replace(/[^A-Za-z0-9]/g, "").slice(-4);
|
|
875
|
+
const key = `${label.toLowerCase()}\u0000${hint}\u0000${value.length}`;
|
|
876
|
+
let item = found.get(key);
|
|
877
|
+
if (!item) {
|
|
878
|
+
if (found.size >= limit)
|
|
879
|
+
return;
|
|
880
|
+
item = { label, prefix, hint, length: value.length, shape: shapeOf(value), snippet: snippetAround(text, start, end, hint, cache), locations: [], spans: [] };
|
|
881
|
+
found.set(key, item);
|
|
882
|
+
}
|
|
883
|
+
item.spans.push({ start, end });
|
|
884
|
+
const loc = locate(text, { rule: "possible", start, end }, cache);
|
|
885
|
+
if (item.locations.length < MAX_LOCATIONS && !item.locations.some((l) => l.where === loc.where && l.tool === loc.tool))
|
|
886
|
+
item.locations.push(loc);
|
|
887
|
+
};
|
|
888
|
+
KEYISH_ASSIGNMENT.lastIndex = 0;
|
|
889
|
+
let m;
|
|
890
|
+
while ((m = KEYISH_ASSIGNMENT.exec(text)) !== null) {
|
|
891
|
+
const [whole, rawName, value] = m;
|
|
892
|
+
// `\nAZURE_KEY=` in JSON-escaped text: the n belongs to the escape.
|
|
893
|
+
const name = text[m.index - 1] === "\\" ? rawName.replace(/^(?:u00[0-9a-fA-F]{2}|[ntr](?=[A-Z]))/, "") : rawName;
|
|
894
|
+
// A signed URL's signature parameter (`?…&signature=`) expires with the URL.
|
|
895
|
+
const before = text.slice(Math.max(0, m.index - 7), m.index);
|
|
896
|
+
if (/sig(?:nature)?$/i.test(name) && /(?:[?&]|\\u0026|\\u003f)$/i.test(before + rawName.slice(0, rawName.length - name.length)))
|
|
897
|
+
continue;
|
|
898
|
+
if (NOT_SECRET_NAME.test(name) || SECRET_REF_KEY.test(name))
|
|
899
|
+
continue;
|
|
900
|
+
if (text[m.index - 1] === ":" || text[m.index - 1] === ".")
|
|
901
|
+
continue; // arn:…, obj.key
|
|
902
|
+
if (!randomLooking(value))
|
|
903
|
+
continue;
|
|
904
|
+
add(name, undefined, value, m.index + whole.lastIndexOf(value));
|
|
905
|
+
}
|
|
906
|
+
PREFIXED_TOKEN.lastIndex = 0;
|
|
907
|
+
while ((m = PREFIXED_TOKEN.exec(text)) !== null) {
|
|
908
|
+
const [whole, prefix, body] = m;
|
|
909
|
+
if (!randomLooking(body))
|
|
910
|
+
continue;
|
|
911
|
+
const start = m.index;
|
|
912
|
+
// Already found under its name (`X-Acme-Key: acme_live_…`): one item,
|
|
913
|
+
// keep the name and note the prefix.
|
|
914
|
+
const named = [...found.values()].find((f) => f.spans.some((sp) => sp.start < start + whole.length && start < sp.end));
|
|
915
|
+
if (named) {
|
|
916
|
+
named.prefix ??= prefix;
|
|
917
|
+
continue;
|
|
918
|
+
}
|
|
919
|
+
add(prefix.replace(/_$/, ""), prefix, whole, start);
|
|
920
|
+
}
|
|
921
|
+
return [...found.values()];
|
|
922
|
+
}
|
|
923
|
+
/** Redact every occurrence of an already-reviewed possible secret. */
|
|
924
|
+
function redactPossible(text, match, rule = "reviewed-secret") {
|
|
925
|
+
const hits = possibleSecrets(text, Infinity).filter((p) => p.label.toLowerCase() === match.label.toLowerCase() && p.hint === match.hint && p.length === match.length);
|
|
926
|
+
const spans = hits.flatMap((h) => h.spans).sort((a, b) => a.start - b.start);
|
|
927
|
+
if (!spans.length)
|
|
928
|
+
return { text, redactions: [] };
|
|
929
|
+
const parts = [];
|
|
930
|
+
let last = 0;
|
|
931
|
+
const redactions = [];
|
|
932
|
+
for (const s of spans) {
|
|
933
|
+
if (s.start < last)
|
|
934
|
+
continue;
|
|
935
|
+
parts.push(text.slice(last, s.start), `[REDACTED:${rule}]`);
|
|
936
|
+
redactions.push({ rule, start: s.start, end: s.end });
|
|
937
|
+
last = s.end;
|
|
938
|
+
}
|
|
939
|
+
parts.push(text.slice(last));
|
|
940
|
+
return { text: parts.join(""), redactions };
|
|
941
|
+
}
|