@coo-quack/sensitive-canary 0.7.0 ā 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +791 -0
- package/README.md +142 -45
- package/dist/lib/bash-commands.js +405 -0
- package/dist/lib/command-tables.js +462 -0
- package/dist/lib/default-config.json +570 -0
- package/dist/lib/encoding.js +123 -0
- package/dist/lib/fail-closed.js +31 -0
- package/dist/lib/inspector.js +0 -0
- package/dist/lib/rules.js +399 -0
- package/dist/lib/shapes.js +161 -0
- package/dist/lib/shell.js +436 -0
- package/dist/lib/tool-inputs.js +217 -0
- package/dist/lib/transcript.js +115 -0
- package/dist/lib/validators.js +435 -0
- package/dist/pre-tool-use-hook.js +773 -0
- package/dist/user-prompt-submit-hook.js +105 -0
- package/hooks/hooks.json +1 -1
- package/package.json +25 -11
- package/src/lib/bash-commands.ts +455 -0
- package/src/lib/command-tables.ts +518 -0
- package/src/lib/default-config.json +155 -46
- package/src/lib/encoding.ts +135 -0
- package/src/lib/fail-closed.ts +36 -0
- package/src/lib/inspector.ts +0 -0
- package/src/lib/rules.ts +202 -365
- package/src/lib/shapes.ts +175 -0
- package/src/lib/shell.ts +512 -0
- package/src/lib/tool-inputs.ts +235 -0
- package/src/lib/transcript.ts +142 -0
- package/src/lib/validators.ts +435 -0
- package/src/pre-tool-use-hook.ts +774 -198
- package/src/user-prompt-submit-hook.ts +60 -18
- package/src/__tests__/pre-tool-use-hook.test.ts +0 -779
- package/src/__tests__/user-prompt-submit-hook.test.ts +0 -297
- package/src/lib/__tests__/inspector.test.ts +0 -289
- package/src/lib/__tests__/rules.test.ts +0 -1370
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// What both hooks do when the check itself goes wrong.
|
|
2
|
+
//
|
|
3
|
+
// A hook that crashes exits 1, and only exit 2 blocks, so an unforeseen error
|
|
4
|
+
// is a silent pass ā the failure this whole tool exists to prevent. What went
|
|
5
|
+
// wrong is unknown at this point, and "unknown" is not "safe": the check did
|
|
6
|
+
// not finish, so the call is stopped rather than let through. `[allow-all]`
|
|
7
|
+
// gets past it, and the message says the check failed rather than claiming a
|
|
8
|
+
// finding.
|
|
9
|
+
//
|
|
10
|
+
// One copy, because two would be two rules: the wording, the exit code and
|
|
11
|
+
// which events are handled all have to be the same in both hooks, and a fix
|
|
12
|
+
// applied to one of two copies is a hook that fails open on the other.
|
|
13
|
+
export function failClosed(error) {
|
|
14
|
+
try {
|
|
15
|
+
process.stderr.write(`\nš¤ sensitive-canary: the check could not complete ā ${error instanceof Error ? error.message : String(error)}\n\n` +
|
|
16
|
+
" Nothing was scanned, so nothing can be vouched for. Stopping rather\n" +
|
|
17
|
+
" than passing it through. Add [allow-all] to your prompt to proceed\n" +
|
|
18
|
+
" anyway, and please report this.\n");
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
// A closed stderr must not turn the block back into a pass.
|
|
22
|
+
}
|
|
23
|
+
process.exit(2);
|
|
24
|
+
}
|
|
25
|
+
// Registered by both hooks as their first statement. A rejection that nothing
|
|
26
|
+
// awaited reaches the process the same way an exception does, and either one
|
|
27
|
+
// arriving unhandled is the pass this exists to stop.
|
|
28
|
+
export function blockOnUnhandledError() {
|
|
29
|
+
process.on("uncaughtException", failClosed);
|
|
30
|
+
process.on("unhandledRejection", failClosed);
|
|
31
|
+
}
|
|
Binary file
|
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
import { readFileSync, statSync } from "node:fs";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import vm from "node:vm";
|
|
6
|
+
import { entropy, isNotSecretShaped, isPlaceholder, keyDescribesRatherThanHolds, } from "./shapes.js";
|
|
7
|
+
import { getValidator } from "./validators.js";
|
|
8
|
+
const ALL_CATEGORIES = new Set(["secret", "pii"]);
|
|
9
|
+
// Parse the SENSITIVE_CANARY_CATEGORIES env var: a comma-separated list of
|
|
10
|
+
// "secret", "pii", or "all" (e.g. "secret" or "secret,pii"). Unset, empty, or
|
|
11
|
+
// containing no valid token means all categories are enabled.
|
|
12
|
+
export function parseCategories(value) {
|
|
13
|
+
const categories = new Set();
|
|
14
|
+
for (const token of (value ?? "").split(",")) {
|
|
15
|
+
const normalized = token.trim().toLowerCase();
|
|
16
|
+
if (normalized === "all")
|
|
17
|
+
return new Set(ALL_CATEGORIES);
|
|
18
|
+
if (normalized === "secret" || normalized === "pii")
|
|
19
|
+
categories.add(normalized);
|
|
20
|
+
}
|
|
21
|
+
return categories.size > 0 ? categories : new Set(ALL_CATEGORIES);
|
|
22
|
+
}
|
|
23
|
+
// Rule categories enabled for this process, from SENSITIVE_CANARY_CATEGORIES
|
|
24
|
+
// ("secret", "pii", "secret,pii", or "all"; default: all).
|
|
25
|
+
export function enabledCategoriesFromEnv() {
|
|
26
|
+
const { SENSITIVE_CANARY_CATEGORIES } = process.env;
|
|
27
|
+
return parseCategories(SENSITIVE_CANARY_CATEGORIES);
|
|
28
|
+
}
|
|
29
|
+
// āā Context enhancement āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā
|
|
30
|
+
// Set from the default config during module initialisation (see buildRules).
|
|
31
|
+
let effectiveContextWindow = 3;
|
|
32
|
+
export function getDefaultContextWindow() {
|
|
33
|
+
return effectiveContextWindow;
|
|
34
|
+
}
|
|
35
|
+
// Words as they were written, with only the punctuation around them removed.
|
|
36
|
+
// Splitting on punctuation made `extract-zip` supply "zip" and
|
|
37
|
+
// `golang.org/x/mobile` supply "mobile", so a version number beside either read
|
|
38
|
+
// as a postal code or a telephone number ā which is to say lockfiles and
|
|
39
|
+
// `go.sum` could not be read.
|
|
40
|
+
function contextTokens(text) {
|
|
41
|
+
const out = new Set();
|
|
42
|
+
for (const raw of text.split(/\s+/)) {
|
|
43
|
+
const word = raw
|
|
44
|
+
.replace(/^[\p{P}\p{S}]+/gu, "")
|
|
45
|
+
.replace(/[\p{P}\p{S}]+$/gu, "")
|
|
46
|
+
.toLowerCase();
|
|
47
|
+
if (word)
|
|
48
|
+
out.add(word);
|
|
49
|
+
}
|
|
50
|
+
return out;
|
|
51
|
+
}
|
|
52
|
+
function hasNearbyContextWord(text, matchStart, matchEnd, contextWords, windowTokens) {
|
|
53
|
+
if (contextWords.length === 0)
|
|
54
|
+
return true;
|
|
55
|
+
const charWindow = windowTokens * 8;
|
|
56
|
+
const before = text.slice(Math.max(0, matchStart - charWindow), matchStart);
|
|
57
|
+
const after = text.slice(matchEnd, matchEnd + charWindow);
|
|
58
|
+
const window = `${before} ${after}`;
|
|
59
|
+
const nearby = contextTokens(window);
|
|
60
|
+
const lowered = window.toLowerCase();
|
|
61
|
+
return contextWords.some((raw) => {
|
|
62
|
+
const word = raw.toLowerCase();
|
|
63
|
+
// A label in a language that does not put spaces around its words is
|
|
64
|
+
// written against the number, so it is looked for as written.
|
|
65
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: the ASCII range is the test
|
|
66
|
+
if (!/^[\x00-\x7f]+$/.test(word))
|
|
67
|
+
return lowered.includes(word);
|
|
68
|
+
return nearby.has(word);
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
// āā Config loading āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā
|
|
72
|
+
const MODULE_DIR = dirname(fileURLToPath(import.meta.url));
|
|
73
|
+
const DEFAULT_CONFIG_PATH = join(MODULE_DIR, "default-config.json");
|
|
74
|
+
const { SENSITIVE_CANARY_CONFIG: userConfigPath } = process.env;
|
|
75
|
+
const USER_CONFIG_PATH = userConfigPath ??
|
|
76
|
+
join(homedir(), ".config", "sensitive-canary", "config.json");
|
|
77
|
+
function readJsonFile(filePath) {
|
|
78
|
+
return JSON.parse(readFileSync(filePath, "utf-8"));
|
|
79
|
+
}
|
|
80
|
+
// Validate a raw JSON object against the RuleConfig schema. Throws with a
|
|
81
|
+
// descriptive message when a required field is missing, a type is wrong, or a
|
|
82
|
+
// cross-field constraint is violated.
|
|
83
|
+
function validateRuleConfig(rc) {
|
|
84
|
+
if (typeof rc !== "object" || rc === null) {
|
|
85
|
+
throw new Error("rule must be an object");
|
|
86
|
+
}
|
|
87
|
+
const { id, description, regex: source, category, flags, secretGroup, entropyThreshold, validate: validateName, contextWords, excludeContext, requireContext, contextWindow, } = rc;
|
|
88
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
89
|
+
throw new Error('missing or empty "id" field');
|
|
90
|
+
}
|
|
91
|
+
if (typeof description !== "string" || description.length === 0) {
|
|
92
|
+
throw new Error('missing or empty "description" field');
|
|
93
|
+
}
|
|
94
|
+
if (typeof source !== "string" || source.length === 0) {
|
|
95
|
+
throw new Error('missing or empty "regex" field');
|
|
96
|
+
}
|
|
97
|
+
if (category !== "secret" && category !== "pii") {
|
|
98
|
+
throw new Error(`invalid "category" ${JSON.stringify(category)} (must be "secret" or "pii")`);
|
|
99
|
+
}
|
|
100
|
+
if (flags != null && typeof flags !== "string") {
|
|
101
|
+
throw new Error('"flags" must be a string');
|
|
102
|
+
}
|
|
103
|
+
if (secretGroup != null &&
|
|
104
|
+
(typeof secretGroup !== "number" ||
|
|
105
|
+
!Number.isInteger(secretGroup) ||
|
|
106
|
+
secretGroup < 0)) {
|
|
107
|
+
throw new Error('"secretGroup" must be a non-negative integer');
|
|
108
|
+
}
|
|
109
|
+
if (entropyThreshold != null &&
|
|
110
|
+
(typeof entropyThreshold !== "number" || entropyThreshold < 0)) {
|
|
111
|
+
throw new Error('"entropyThreshold" must be a non-negative number');
|
|
112
|
+
}
|
|
113
|
+
if (validateName != null && typeof validateName !== "string") {
|
|
114
|
+
throw new Error('"validate" must be a string');
|
|
115
|
+
}
|
|
116
|
+
if (excludeContext != null) {
|
|
117
|
+
if (!Array.isArray(excludeContext) ||
|
|
118
|
+
excludeContext.some((w) => typeof w !== "string" || w.length === 0)) {
|
|
119
|
+
throw new Error('"excludeContext" must be an array of non-empty strings');
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (contextWords != null) {
|
|
123
|
+
if (!Array.isArray(contextWords) ||
|
|
124
|
+
contextWords.some((w) => typeof w !== "string" || w.length === 0)) {
|
|
125
|
+
throw new Error('"contextWords" must be an array of non-empty strings');
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
if (requireContext != null && typeof requireContext !== "boolean") {
|
|
129
|
+
throw new Error('"requireContext" must be a boolean');
|
|
130
|
+
}
|
|
131
|
+
if (contextWindow != null &&
|
|
132
|
+
(typeof contextWindow !== "number" ||
|
|
133
|
+
!Number.isInteger(contextWindow) ||
|
|
134
|
+
contextWindow < 1)) {
|
|
135
|
+
throw new Error('"contextWindow" must be a positive integer');
|
|
136
|
+
}
|
|
137
|
+
// Cross-field: requireContext is meaningless without contextWords
|
|
138
|
+
if (requireContext === true &&
|
|
139
|
+
(!Array.isArray(contextWords) || contextWords.length === 0)) {
|
|
140
|
+
throw new Error('"requireContext" is true but "contextWords" is empty ā context gating would be disabled and the rule would always fire');
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
// Compile a single RuleConfig (JSON) into a Rule (with compiled RegExp and
|
|
144
|
+
// resolved validator function). Throws on invalid regex or missing required
|
|
145
|
+
// fields so the caller (buildRules) can catch and warn per-rule.
|
|
146
|
+
export function compileRule(rc) {
|
|
147
|
+
validateRuleConfig(rc);
|
|
148
|
+
const { regex: source, flags, validate: validateName, ...rest } = rc;
|
|
149
|
+
// matchAll requires the global flag; ensure it is always present.
|
|
150
|
+
const flagStr = flags ?? "g";
|
|
151
|
+
const withG = flagStr.includes("g") ? flagStr : `${flagStr}g`;
|
|
152
|
+
const rule = {
|
|
153
|
+
...rest,
|
|
154
|
+
regex: new RegExp(source, withG),
|
|
155
|
+
};
|
|
156
|
+
if (validateName) {
|
|
157
|
+
const fn = getValidator(validateName);
|
|
158
|
+
if (fn) {
|
|
159
|
+
rule.validate = fn;
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
process.stderr.write(`sensitive-canary: unknown validator "${validateName}" in rule "${rc.id}" ā validation disabled\n`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return rule;
|
|
166
|
+
}
|
|
167
|
+
// Load and compile the built-in default rules from default-config.json.
|
|
168
|
+
function loadDefaultConfig() {
|
|
169
|
+
return readJsonFile(DEFAULT_CONFIG_PATH);
|
|
170
|
+
}
|
|
171
|
+
// Load user config if it exists. Returns null when the file is absent (the
|
|
172
|
+
// common case). JSON parse errors and permission issues are reported on stderr
|
|
173
|
+
// so that a broken config file is not silently ignored.
|
|
174
|
+
function loadUserConfig() {
|
|
175
|
+
try {
|
|
176
|
+
// A FIFO or a device here would block the read until something wrote to
|
|
177
|
+
// it, and a hook that never returns is killed by the timeout, which does
|
|
178
|
+
// not block. The transcript reader and the file scanner both pay this stat
|
|
179
|
+
// already; this path was the one that did not.
|
|
180
|
+
if (!statSync(USER_CONFIG_PATH).isFile()) {
|
|
181
|
+
process.stderr.write(`sensitive-canary: user config "${USER_CONFIG_PATH}" is not a regular file, ignoring\n`);
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
return readJsonFile(USER_CONFIG_PATH);
|
|
185
|
+
}
|
|
186
|
+
catch (e) {
|
|
187
|
+
if (e.code !== "ENOENT") {
|
|
188
|
+
process.stderr.write(`sensitive-canary: could not read user config "${USER_CONFIG_PATH}": ${e instanceof Error ? e.message : String(e)}\n`);
|
|
189
|
+
}
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
// Build the final rule list: default rules first, then user rules. A user rule
|
|
194
|
+
// with the same id as a built-in rule replaces it; new ids are appended.
|
|
195
|
+
// Invalid user rules (bad regex, etc.) are skipped with a warning so that one
|
|
196
|
+
// bad entry does not break the entire hook.
|
|
197
|
+
function buildRules() {
|
|
198
|
+
const defaultConfig = loadDefaultConfig();
|
|
199
|
+
effectiveContextWindow = defaultConfig.contextWindow ?? 3;
|
|
200
|
+
const defaultRules = [];
|
|
201
|
+
for (const rc of defaultConfig.rules) {
|
|
202
|
+
try {
|
|
203
|
+
defaultRules.push(compileRule(rc));
|
|
204
|
+
}
|
|
205
|
+
catch (e) {
|
|
206
|
+
process.stderr.write(`sensitive-canary: failed to compile built-in rule "${rc?.id ?? "(unknown)"}": ${e instanceof Error ? e.message : String(e)}\n`);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
const userConfig = loadUserConfig();
|
|
210
|
+
if (userConfig) {
|
|
211
|
+
if (typeof userConfig.contextWindow === "number" &&
|
|
212
|
+
Number.isInteger(userConfig.contextWindow) &&
|
|
213
|
+
userConfig.contextWindow >= 1) {
|
|
214
|
+
effectiveContextWindow = userConfig.contextWindow;
|
|
215
|
+
}
|
|
216
|
+
else if (userConfig.contextWindow != null) {
|
|
217
|
+
process.stderr.write(`sensitive-canary: invalid contextWindow in user config, ignoring\n`);
|
|
218
|
+
}
|
|
219
|
+
if (userConfig.rules != null && !Array.isArray(userConfig.rules)) {
|
|
220
|
+
process.stderr.write(`sensitive-canary: "rules" in user config must be an array, ignoring\n`);
|
|
221
|
+
}
|
|
222
|
+
if (Array.isArray(userConfig.rules) && userConfig.rules.length) {
|
|
223
|
+
const userRules = [];
|
|
224
|
+
for (const rc of userConfig.rules) {
|
|
225
|
+
try {
|
|
226
|
+
userRules.push(compileRule(rc));
|
|
227
|
+
}
|
|
228
|
+
catch (e) {
|
|
229
|
+
process.stderr.write(`sensitive-canary: skipping user rule "${rc?.id ?? "(unknown)"}": ${e instanceof Error ? e.message : String(e)}\n`);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
// De-duplicate by id (last definition wins) so duplicate ids in the
|
|
233
|
+
// user config don't produce duplicate rules and duplicate findings.
|
|
234
|
+
const byId = new Map();
|
|
235
|
+
for (const rule of userRules) {
|
|
236
|
+
if (byId.has(rule.id)) {
|
|
237
|
+
process.stderr.write(`sensitive-canary: duplicate user rule id "${rule.id}" ā using the last definition\n`);
|
|
238
|
+
}
|
|
239
|
+
byId.set(rule.id, rule);
|
|
240
|
+
}
|
|
241
|
+
return defaultRules
|
|
242
|
+
.filter((r) => !byId.has(r.id))
|
|
243
|
+
.concat(Array.from(byId.values()));
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
return defaultRules;
|
|
247
|
+
}
|
|
248
|
+
export const RULES = buildRules();
|
|
249
|
+
// Enough of a value to say which one was found, and no more.
|
|
250
|
+
//
|
|
251
|
+
// The block reason is written to stderr, which is where Claude reads it, so
|
|
252
|
+
// whatever is shown here reaches the API that the block exists to keep it from.
|
|
253
|
+
// Four characters at each end returned eight of a nine-character password.
|
|
254
|
+
// A quarter of the value, capped at four per end.
|
|
255
|
+
export function redact(str) {
|
|
256
|
+
// Code points, not code units. Slicing by unit cuts a surrogate pair in half
|
|
257
|
+
// and writes a lone surrogate to the terminal, which is neither the character
|
|
258
|
+
// nor a redaction of it.
|
|
259
|
+
const characters = [...str];
|
|
260
|
+
const shown = Math.min(4, Math.floor(characters.length / 8));
|
|
261
|
+
if (shown === 0)
|
|
262
|
+
return "****";
|
|
263
|
+
const head = characters.slice(0, shown).join("");
|
|
264
|
+
const tail = characters.slice(-shown).join("");
|
|
265
|
+
return `${head}****${tail}`;
|
|
266
|
+
}
|
|
267
|
+
// Longer than any honest scan and far shorter than the hook timeout. A rule
|
|
268
|
+
// that backtracks badly takes minutes on a megabyte, and a hook killed by the
|
|
269
|
+
// timeout does not block, so the damage is silent. The patterns that did that
|
|
270
|
+
// are bounded; this catches the next one of that shape rather than letting it
|
|
271
|
+
// repeat. The check sits between rules because a single `matchAll` cannot be
|
|
272
|
+
// interrupted.
|
|
273
|
+
export const SCAN_BUDGET_MS = 10_000;
|
|
274
|
+
export class ScanBudgetExceeded extends Error {
|
|
275
|
+
constructor(ruleId, elapsed) {
|
|
276
|
+
super(`the scan passed ${SCAN_BUDGET_MS}ms (${elapsed}ms at rule "${ruleId}")`);
|
|
277
|
+
this.name = "ScanBudgetExceeded";
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
// The budget belongs to the hook invocation, not to one `scan()` call. A single
|
|
281
|
+
// call is a small part of the work: `scanEnvironment` scans once per variable,
|
|
282
|
+
// a file is scanned at both ends, and `Object.keys(process.env)` sets the
|
|
283
|
+
// multiplier. Per call, each stays inside the budget while the total runs past
|
|
284
|
+
// the hook timeout ā and a hook killed by the timeout does not block.
|
|
285
|
+
//
|
|
286
|
+
// Set once by each hook entry point. Left unset, every call gets the full
|
|
287
|
+
// budget, which is what the test suite needs.
|
|
288
|
+
let deadline = null;
|
|
289
|
+
// `null` clears it, which is the state a process starts in.
|
|
290
|
+
export function beginScanBudget(totalMs = SCAN_BUDGET_MS) {
|
|
291
|
+
deadline = totalMs === null ? null : Date.now() + totalMs;
|
|
292
|
+
}
|
|
293
|
+
// What is left of the budget, or the whole of it when none was begun.
|
|
294
|
+
function remainingBudget() {
|
|
295
|
+
return deadline === null ? SCAN_BUDGET_MS : deadline - Date.now();
|
|
296
|
+
}
|
|
297
|
+
// The between-rule check below cannot interrupt a single `matchAll`, and one
|
|
298
|
+
// rule from a user config is enough to hang the hook ā which is then killed by
|
|
299
|
+
// the timeout, and a killed hook does not block. A V8-side timeout does
|
|
300
|
+
// interrupt a running match. Measured at 0.06ms per call, against a scan that
|
|
301
|
+
// costs hundreds of times that.
|
|
302
|
+
const SCAN_SLOT = "__sensitiveCanaryScan";
|
|
303
|
+
const HARD_LIMIT_SLACK_MS = 2_000;
|
|
304
|
+
// `limitMs` bounds this call so it cannot overshoot what the invocation has
|
|
305
|
+
// left. Without it a single call could run the full hard limit past a deadline
|
|
306
|
+
// that was already spent.
|
|
307
|
+
function runInterruptibly(work, limitMs) {
|
|
308
|
+
const slots = globalThis;
|
|
309
|
+
slots[SCAN_SLOT] = work;
|
|
310
|
+
try {
|
|
311
|
+
return vm.runInThisContext(`globalThis.${SCAN_SLOT}()`, {
|
|
312
|
+
timeout: limitMs,
|
|
313
|
+
displayErrors: false,
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
catch (error) {
|
|
317
|
+
if (error instanceof Error && error.message.includes("timed out"))
|
|
318
|
+
throw new ScanBudgetExceeded("a single rule", limitMs);
|
|
319
|
+
throw error;
|
|
320
|
+
}
|
|
321
|
+
finally {
|
|
322
|
+
delete slots[SCAN_SLOT];
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
export function scan(text, categories = ALL_CATEGORIES) {
|
|
326
|
+
const remaining = remainingBudget();
|
|
327
|
+
if (remaining <= 0)
|
|
328
|
+
throw new ScanBudgetExceeded("this call's total", SCAN_BUDGET_MS);
|
|
329
|
+
return runInterruptibly(() => scanUninterrupted(text, categories, remaining), remaining + HARD_LIMIT_SLACK_MS);
|
|
330
|
+
}
|
|
331
|
+
function scanUninterrupted(text, categories, budgetMs) {
|
|
332
|
+
const findings = [];
|
|
333
|
+
const startedAt = Date.now();
|
|
334
|
+
for (const rule of RULES) {
|
|
335
|
+
if (!categories.has(rule.category))
|
|
336
|
+
continue;
|
|
337
|
+
const elapsed = Date.now() - startedAt;
|
|
338
|
+
// Thrown rather than returned: a partial result is indistinguishable from a
|
|
339
|
+
// clean one, and the hooks stop the call on an error they cannot explain.
|
|
340
|
+
if (elapsed > budgetMs)
|
|
341
|
+
throw new ScanBudgetExceeded(rule.id, elapsed);
|
|
342
|
+
for (const match of text.matchAll(rule.regex)) {
|
|
343
|
+
const secretValue = rule.secretGroup != null ? match[rule.secretGroup] : match[0];
|
|
344
|
+
if (!secretValue)
|
|
345
|
+
continue;
|
|
346
|
+
// Both the captured value and the whole match: a rule with a
|
|
347
|
+
// `secretGroup` captures only part of what it matched, and the
|
|
348
|
+
// connection-string rule stops at the `@`, so the host ā the one thing
|
|
349
|
+
// that separates `user:password@localhost` from `user:password@` in front
|
|
350
|
+
// of real infrastructure ā is outside the capture.
|
|
351
|
+
const matchStart = match.index ?? 0;
|
|
352
|
+
const matchEnd = matchStart + match[0].length;
|
|
353
|
+
const following = text.slice(matchEnd, matchEnd + 64);
|
|
354
|
+
// The shape test applies only where the rule captured a free-form value.
|
|
355
|
+
// A rule that matches a fixed prefix has already said what the thing is ā
|
|
356
|
+
// a Slack webhook is a URL and a secret, and asking whether it looks like
|
|
357
|
+
// a URL is asking the wrong question.
|
|
358
|
+
const capturesAValue = rule.secretGroup != null;
|
|
359
|
+
if (rule.category === "secret" &&
|
|
360
|
+
(isPlaceholder(secretValue, following) ||
|
|
361
|
+
(capturesAValue &&
|
|
362
|
+
(isNotSecretShaped(secretValue) ||
|
|
363
|
+
isPlaceholder(match[0], following) ||
|
|
364
|
+
isNotSecretShaped(match[0]) ||
|
|
365
|
+
keyDescribesRatherThanHolds(match[0])))))
|
|
366
|
+
continue;
|
|
367
|
+
if (rule.entropyThreshold != null &&
|
|
368
|
+
entropy(secretValue) < rule.entropyThreshold)
|
|
369
|
+
continue;
|
|
370
|
+
if (rule.validate != null && !rule.validate(secretValue))
|
|
371
|
+
continue;
|
|
372
|
+
const hasContext = !rule.contextWords || rule.contextWords.length === 0
|
|
373
|
+
? true
|
|
374
|
+
: hasNearbyContextWord(text, matchStart, matchEnd, rule.contextWords, rule.contextWindow ?? effectiveContextWindow);
|
|
375
|
+
// Rules that require context (e.g. bare postal codes) are dropped when
|
|
376
|
+
// no context label is nearby, to avoid flagging every 5-digit number.
|
|
377
|
+
if (rule.requireContext && !hasContext)
|
|
378
|
+
continue;
|
|
379
|
+
// And the other way: a word nearby that says this is not what the rule is
|
|
380
|
+
// for. `git clone git@github.com:ā¦` and `ssh deploy@host` are addresses by
|
|
381
|
+
// shape, and the command in front of them is what says they are not
|
|
382
|
+
// anyone's mail.
|
|
383
|
+
if (rule.excludeContext &&
|
|
384
|
+
rule.excludeContext.length > 0 &&
|
|
385
|
+
hasNearbyContextWord(text, matchStart, matchEnd, rule.excludeContext, rule.contextWindow ?? effectiveContextWindow)) {
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
findings.push({
|
|
389
|
+
ruleId: rule.id,
|
|
390
|
+
description: rule.description,
|
|
391
|
+
category: rule.category,
|
|
392
|
+
matchRedacted: redact(secretValue),
|
|
393
|
+
secretValue,
|
|
394
|
+
score: hasContext ? 1.0 : 0.4,
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
return findings;
|
|
399
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// Whether a value that matched a rule is a credential after all.
|
|
2
|
+
//
|
|
3
|
+
// A rule finds a shape. These decide what the shape is standing for: a name
|
|
4
|
+
// that points at a secret, a slot waiting to be filled, a reference to a value
|
|
5
|
+
// in code. Each one waves a match through, so each is a way past every rule it
|
|
6
|
+
// runs over, and each is written to be narrower than the shape it answers for.
|
|
7
|
+
// A value whose shape says it is not a credential, whatever its name suggests.
|
|
8
|
+
// `TOKEN_ENDPOINT`, `secret_name`, `VAULT_TOKEN_PATH` and `TOKEN_HEADER_NAME`
|
|
9
|
+
// all assign something that points at a secret rather than being one, and
|
|
10
|
+
// blocking them made Terraform, Kubernetes manifests and OAuth configuration
|
|
11
|
+
// unreadable ā fourteen of the twenty-six wrong blocks in a survey of six
|
|
12
|
+
// hundred real files.
|
|
13
|
+
export function isNotSecretShaped(value) {
|
|
14
|
+
const v = value.trim();
|
|
15
|
+
// A URL or a URN. Credentials embedded in one are the connection-string
|
|
16
|
+
// rule's business, and a URL carrying a token in its query is left alone
|
|
17
|
+
// here so the `?` case still reaches the other rules.
|
|
18
|
+
if (/^[a-z][a-z0-9+.-]*:\/\//i.test(v) &&
|
|
19
|
+
!v.includes("?") &&
|
|
20
|
+
!v.includes("@"))
|
|
21
|
+
return true;
|
|
22
|
+
if (/^urn:/i.test(v))
|
|
23
|
+
return true;
|
|
24
|
+
// A filesystem path.
|
|
25
|
+
if (/^[~.]?\/[^\s]*$/.test(v))
|
|
26
|
+
return true;
|
|
27
|
+
// The name of a variable rather than its value.
|
|
28
|
+
if (/^[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+$/.test(v))
|
|
29
|
+
return true;
|
|
30
|
+
// An HTTP header name.
|
|
31
|
+
if (/^[A-Z][A-Za-z0-9]*(?:-[A-Z][A-Za-z0-9]*)+$/.test(v))
|
|
32
|
+
return true;
|
|
33
|
+
// A number.
|
|
34
|
+
if (/^\d+$/.test(v))
|
|
35
|
+
return true;
|
|
36
|
+
// A dotted lower-case identifier, as a storage key or a setting name.
|
|
37
|
+
if (/^[a-z][a-z0-9]*(?:\.[a-z0-9]+)+$/.test(v))
|
|
38
|
+
return true;
|
|
39
|
+
// A reference to a value in code rather than the value: `process.env.API_KEY`,
|
|
40
|
+
// `user.password_digest`, `self.api_key`, `response.data.accessToken`. Of the
|
|
41
|
+
// distinct values `env-assignment` matched across thirty thousand real files,
|
|
42
|
+
// two in five were one of these.
|
|
43
|
+
if (/^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+$/.test(v) &&
|
|
44
|
+
v.split(".").every(readsAsWords))
|
|
45
|
+
return true;
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
// Whether a name is built out of words rather than out of random characters.
|
|
49
|
+
//
|
|
50
|
+
// Dotted credentials exist ā a JWT, and the dotted forms several vendors issue ā
|
|
51
|
+
// so the shape alone cannot stand for "this is code". What separates them is
|
|
52
|
+
// that a name is words: `connectionString` and `password_digest` run several
|
|
53
|
+
// letters between one boundary and the next, where a random segment changes
|
|
54
|
+
// case or slips in a digit every character or two.
|
|
55
|
+
//
|
|
56
|
+
// Measured over 147,643 dotted identifiers taken from the source on this
|
|
57
|
+
// machine, 0.06% fall below the threshold below, and the ones that do are JWTs.
|
|
58
|
+
export const MIN_MEAN_WORD_LENGTH = 2.5;
|
|
59
|
+
// Short segments are words by default: `env`, `data`, `id`. The statistic needs
|
|
60
|
+
// something to average over before it says anything.
|
|
61
|
+
export const SHORTEST_MEASURABLE_SEGMENT = 8;
|
|
62
|
+
export function readsAsWords(segment) {
|
|
63
|
+
if (segment.length < SHORTEST_MEASURABLE_SEGMENT)
|
|
64
|
+
return true;
|
|
65
|
+
// A leading capital belongs to the lowercase run after it, so `toLowerCase`
|
|
66
|
+
// is three words and not five runs of one case.
|
|
67
|
+
const words = segment.match(/[A-Z]+(?![a-z])|[A-Z]?[a-z]+|\d+/g);
|
|
68
|
+
if (words === null || words.length === 0)
|
|
69
|
+
return false;
|
|
70
|
+
return segment.length / words.length >= MIN_MEAN_WORD_LENGTH;
|
|
71
|
+
}
|
|
72
|
+
// A key that says where a secret lives, or what it is called, rather than what
|
|
73
|
+
// it is. The rule fires on the keyword anywhere in the name, so
|
|
74
|
+
// `SECRET_MANAGER_PROJECT` reads as a secret because of its first word, when its
|
|
75
|
+
// last one says it holds a project.
|
|
76
|
+
const DESCRIBES_A_SECRET = /\b[A-Za-z0-9_]*_(?:PROJECT|NAME|PATH|FILE|DIR|URL|URI|ENDPOINT|HOST|PORT|ID|TYPE|HEADER|PREFIX|SUFFIX|FIELD|COLUMN|TABLE|ENV|REGION|BUCKET|ARN|VERSION|TTL|TIMEOUT|LENGTH|COUNT|ENABLED|ALGORITHM|ISSUER|AUDIENCE|SCOPE|PROVIDER|BACKEND|SOURCE)\b[ \t]*[:=]/i;
|
|
77
|
+
export function keyDescribesRatherThanHolds(matchText) {
|
|
78
|
+
return DESCRIBES_A_SECRET.test(matchText);
|
|
79
|
+
}
|
|
80
|
+
// A value written to be replaced. Half of a realistic `.env.example` was being
|
|
81
|
+
// blocked on its contents, which is the block most likely to get the tool turned
|
|
82
|
+
// off ā the file is meant to be committed and read.
|
|
83
|
+
//
|
|
84
|
+
// Only secret rules consult this. "todo@company.com" is a real address, and
|
|
85
|
+
// AWS's own documented key ends in EXAMPLE and is still a key, so `example` is
|
|
86
|
+
// deliberately absent from the marker list where it would matter.
|
|
87
|
+
// A word that only ever appears in a value nobody typed.
|
|
88
|
+
const PLACEHOLDER_MARKERS = /^(?:changeme|change|me|replace|insert|set|with|real|this|your|my|here|todo|tbd|fixme|dummy|placeholder|insecure|sample|example|test|fake|redacted|value|x{3,})$/i;
|
|
89
|
+
// A word that can make up the rest of such a value, but never marks one alone.
|
|
90
|
+
const PLACEHOLDER_FILLER = /^(?:api|key|keys|token|tokens|secret|secrets|password|passwd|pwd|pass|base|url|uri|host|hostname|name|user|username|id|access|refresh|client|auth|sk|pk|in|production|development|staging|local|dev|the|a|of|for|and|[0-9]+)$/i;
|
|
91
|
+
// The scheme is bounded: an unbounded `\w+` in front of a literal that usually
|
|
92
|
+
// is not there makes the match quadratic in the length of the value, and a
|
|
93
|
+
// value is as long as whoever wrote the text wants. A scheme is a word.
|
|
94
|
+
const GENERIC_CREDENTIALS = /\w{1,32}:\/\/(?:your[_-]?)?(?:user|username)(?:name)?:(?:your[_-]?)?(?:password|passwd|pwd)@/i;
|
|
95
|
+
const GENERIC_HOST = /^(?:localhost|127\.0\.0\.1|0\.0\.0\.0|host|hostname|db|database|example\.(?:com|org|net))\b/i;
|
|
96
|
+
export function isPlaceholder(value, following = "") {
|
|
97
|
+
const v = value.trim();
|
|
98
|
+
if (!v)
|
|
99
|
+
return false;
|
|
100
|
+
// Filler on its own: `xxxxxxxxxxxx`.
|
|
101
|
+
if (/^[Xx]+$/.test(v))
|
|
102
|
+
return true;
|
|
103
|
+
// A slot rather than a value: `<your-token>`, `${TOKEN}`, `{{ token }}`.
|
|
104
|
+
if (/^[<{[]/.test(v) && /[>}\]]$/.test(v))
|
|
105
|
+
return true;
|
|
106
|
+
// A shell or template reference, which holds nothing at all.
|
|
107
|
+
if (/^\$\{?[A-Za-z_][A-Za-z0-9_]*\}?$/.test(v))
|
|
108
|
+
return true;
|
|
109
|
+
// An unexpanded reference anywhere inside a connection string: no character
|
|
110
|
+
// of the credentials has been substituted yet.
|
|
111
|
+
if (/:\/\/[^@\s]*(?:\$\{[A-Za-z_][^}]*\}|\$[A-Za-z_][A-Za-z0-9_]*|\{\})[^@\s]*@/.test(v))
|
|
112
|
+
return true;
|
|
113
|
+
// A user and a password that are the same word, and that word names the
|
|
114
|
+
// service: `postgres:postgres@`, `root:root@`, `guest:guest@` are what a
|
|
115
|
+
// compose file and a quickstart ship with.
|
|
116
|
+
const samePair = v.match(/:\/\/([A-Za-z]{3,12}):([A-Za-z]{3,12})@/);
|
|
117
|
+
if (samePair &&
|
|
118
|
+
samePair[1]?.toLowerCase() === samePair[2]?.toLowerCase() &&
|
|
119
|
+
/^(?:postgres|postgresql|mysql|mariadb|mongo|mongodb|redis|root|guest|admin|user|test|rabbitmq)$/i.test(samePair[1] ?? ""))
|
|
120
|
+
return true;
|
|
121
|
+
// The default the django template generates, which ships in every new project.
|
|
122
|
+
if (/^django-insecure-/i.test(v))
|
|
123
|
+
return true;
|
|
124
|
+
// A connection string where the user, the password and the host are all the
|
|
125
|
+
// words for them. Each of the three matters: `root:secret@` is a password
|
|
126
|
+
// people set, and `user:password@prod.corp.internal` names real infrastructure.
|
|
127
|
+
// The rule that finds these stops at the `@`, so the host arrives as the text
|
|
128
|
+
// that follows rather than as part of the value.
|
|
129
|
+
//
|
|
130
|
+
// Matched once. Asking three times ran the same backtracking three times.
|
|
131
|
+
const credentials = GENERIC_CREDENTIALS.exec(v);
|
|
132
|
+
if (credentials !== null) {
|
|
133
|
+
const afterAt = v.slice(credentials.index + credentials[0].length);
|
|
134
|
+
if (GENERIC_HOST.test(afterAt || following))
|
|
135
|
+
return true;
|
|
136
|
+
}
|
|
137
|
+
// Otherwise every part of the value has to be one of these words, and at
|
|
138
|
+
// least one has to be a marker. Testing whether the value *contains* a marker
|
|
139
|
+
// was a way through: `changeme_` in front of a live key disabled the rule.
|
|
140
|
+
const parts = v.split(/[-_.\s]+/).filter(Boolean);
|
|
141
|
+
if (parts.length === 0)
|
|
142
|
+
return false;
|
|
143
|
+
if (!parts.some((part) => PLACEHOLDER_MARKERS.test(part)))
|
|
144
|
+
return false;
|
|
145
|
+
return parts.every((part) => PLACEHOLDER_MARKERS.test(part) || PLACEHOLDER_FILLER.test(part));
|
|
146
|
+
}
|
|
147
|
+
// Shannon entropy (bits per character; ā0ā8 for byte-sized alphabets)
|
|
148
|
+
export function entropy(str) {
|
|
149
|
+
if (str.length === 0)
|
|
150
|
+
return 0;
|
|
151
|
+
const freq = {};
|
|
152
|
+
for (const ch of str)
|
|
153
|
+
freq[ch] = (freq[ch] ?? 0) + 1;
|
|
154
|
+
let h = 0;
|
|
155
|
+
const n = str.length;
|
|
156
|
+
for (const count of Object.values(freq)) {
|
|
157
|
+
const p = count / n;
|
|
158
|
+
h -= p * Math.log2(p);
|
|
159
|
+
}
|
|
160
|
+
return h;
|
|
161
|
+
}
|