launchprep 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/net/consent.mjs +5 -0
- package/package.json +3 -3
- package/src/checks-ai.mjs +30 -3
- package/src/checks-deploy.mjs +8 -0
- package/src/checks.mjs +36 -0
- package/src/digest.mjs +13 -1
- package/src/git-history.mjs +255 -0
- package/src/index.mjs +52 -4
- package/src/placeholder.mjs +72 -0
- package/src/questions.mjs +206 -0
- package/src/redact.mjs +127 -0
- package/src/report.mjs +41 -12
- package/src/rules.json +1 -1
package/README.md
CHANGED
|
@@ -69,6 +69,27 @@ the report link keeps working, and that link expires.
|
|
|
69
69
|
|
|
70
70
|
If there is no terminal — CI, a script — it refuses rather than assuming yes.
|
|
71
71
|
|
|
72
|
+
## Telling it about your app
|
|
73
|
+
|
|
74
|
+
Some things are not in the code — where your users live, whether your AI is
|
|
75
|
+
allowed to act, how many people you expect. Those checks are skipped, and the
|
|
76
|
+
scan lists them. Put the answers in `launchprep.json` at the top of your
|
|
77
|
+
project and run again:
|
|
78
|
+
|
|
79
|
+
```json
|
|
80
|
+
{
|
|
81
|
+
"jurisdictions": ["eu"],
|
|
82
|
+
"has_accounts": true,
|
|
83
|
+
"llm_tools_enabled": true,
|
|
84
|
+
"stage": "production"
|
|
85
|
+
}
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
What you state beats what was detected, and the report shows it back under
|
|
89
|
+
**You said** so a wrong answer is visible rather than silently changing
|
|
90
|
+
results. The free scan is unlimited, so correcting and re-running costs
|
|
91
|
+
nothing.
|
|
92
|
+
|
|
72
93
|
## In CI
|
|
73
94
|
|
|
74
95
|
```bash
|
package/net/consent.mjs
CHANGED
|
@@ -21,6 +21,11 @@ export async function confirmUpload({ digest, host, assumeYes = false, showFiles
|
|
|
21
21
|
console.log(`\n${C.b}Before this runs${C.off}\n`);
|
|
22
22
|
console.log(` This sends ${C.b}${files} files${C.off} of your code ${C.d}(about ${tokens} tokens)${C.off}`);
|
|
23
23
|
console.log(` to ${C.b}${host}${C.off}, and from there to Anthropic, so a model can read them.\n`);
|
|
24
|
+
if (digest.secretsRedacted > 0) {
|
|
25
|
+
console.log(` ${C.g}${digest.secretsRedacted} secret${digest.secretsRedacted === 1 ? '' : 's'} masked before sending${C.off} — keys, passwords, tokens and`);
|
|
26
|
+
console.log(` the rows of any database dump are replaced on this machine, so their`);
|
|
27
|
+
console.log(` values never leave it. Only that they existed, and where, is sent.\n`);
|
|
28
|
+
}
|
|
24
29
|
console.log(` ${C.g}Your code is not stored.${C.off} It is held in memory for the scan and discarded.`);
|
|
25
30
|
console.log(` ${C.g}The findings are kept${C.off} — file names, line numbers, and what to fix —`);
|
|
26
31
|
console.log(` so the report link keeps working. That link expires.\n`);
|
package/package.json
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "launchprep",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Launch readiness checker. Reads your code and tells you what will break before your users find out.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
|
-
"launchprep": "
|
|
7
|
+
"launchprep": "bin/launchprep.mjs"
|
|
8
8
|
},
|
|
9
9
|
"files": [
|
|
10
10
|
"src",
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"scripts": {
|
|
18
18
|
"verify": "node scripts/verify-readonly.mjs",
|
|
19
19
|
"pretest": "node scripts/verify-readonly.mjs",
|
|
20
|
-
"test": "node test/run.mjs && node test/gate.mjs",
|
|
20
|
+
"test": "node test/run.mjs && node test/gate.mjs && node test/redact.mjs && node test/git-history.mjs && node test/no-false-alarms.mjs && node test/answers.mjs",
|
|
21
21
|
"prepack": "node scripts/prepare-publish.mjs",
|
|
22
22
|
"postpack": "node scripts/restore-after-publish.mjs"
|
|
23
23
|
},
|
package/src/checks-ai.mjs
CHANGED
|
@@ -1,12 +1,21 @@
|
|
|
1
1
|
// AI cost and safety. Barely covered by the corpus, which is exactly why it
|
|
2
2
|
// matters: this is the family nobody else is checking.
|
|
3
|
+
import { looksLikePlaceholder } from './placeholder.mjs';
|
|
4
|
+
|
|
3
5
|
const finding = (id, title, severity, file, line, detail, fix) =>
|
|
4
6
|
({ id, title, severity, file, line, detail, fix });
|
|
5
7
|
|
|
6
8
|
const lineOf = (text, i) => text.slice(0, i).split('\n').length;
|
|
7
9
|
const isCode = (p) => /\.(ts|tsx|js|jsx|mjs|cjs|py)$/.test(p);
|
|
8
|
-
|
|
9
|
-
|
|
10
|
+
// (^|/) not just / — a slash was REQUIRED before the word, so a project laid
|
|
11
|
+
// out as api/src/... or server/src/... or backend/src/... never matched as a
|
|
12
|
+
// server, and then matched as a browser because it contains /src/. Result: the
|
|
13
|
+
// scanner told people running a plain Node backend that their API key was
|
|
14
|
+
// "shipped to every visitor". It fired twice on this repository's own api/.
|
|
15
|
+
// A fabricated CRITICAL is the failure this whole product is built to avoid.
|
|
16
|
+
// 'backend' and 'worker' added because they are as common as 'server'.
|
|
17
|
+
const isServer = (p) => /(^|\/)(api|routes?|server|backend|actions?|controllers?|handlers?|endpoints?|resolvers?|lib|services?|workers?|jobs?)\//.test(p);
|
|
18
|
+
const isClient = (p) => /(^|\/)(components?|pages?|app|src|client|frontend|ui|views?)\//.test(p) && !isServer(p);
|
|
10
19
|
|
|
11
20
|
const LLM_CALL = /\.(messages|chat\.completions|completions|responses)\.create\s*\(|generateText\s*\(|streamText\s*\(|\.generateContent\s*\(/;
|
|
12
21
|
|
|
@@ -148,12 +157,24 @@ export const AI_CHECKS = [
|
|
|
148
157
|
const out = [];
|
|
149
158
|
for (const f of repo.files) {
|
|
150
159
|
if (!f.text || !isCode(f.path)) continue;
|
|
151
|
-
|
|
160
|
+
// `exec` as a bare word is two different things. `child_process.exec(cmd)`
|
|
161
|
+
// runs a program; `re.exec(text)` is how JavaScript matches a regular
|
|
162
|
+
// expression, and appears in ordinary code constantly - three times in
|
|
163
|
+
// this scanner's own source, which is how it reported three CRITICALs
|
|
164
|
+
// about itself. So `.exec(` preceded by a dot only counts when the thing
|
|
165
|
+
// before the dot is actually child_process.
|
|
166
|
+
const re = /(?<![.\w])(eval|execSync|execFileSync|spawnSync|Function)\s*\(|(?<![.\w])exec\s*\(|\b(?:child_process|cp|shell)\.exec(?:File)?\s*\(|\.query\s*\(\s*`/g;
|
|
152
167
|
let m;
|
|
153
168
|
while ((m = re.exec(f.text))) {
|
|
154
169
|
const around = f.text.slice(Math.max(0, m.index - 600), m.index + 200);
|
|
155
170
|
if (!LLM_CALL.test(around) &&
|
|
156
171
|
!/\b(completion|aiResponse|modelOutput|generated|llmResult)\b/i.test(around)) continue;
|
|
172
|
+
// "generated" and "completion" are ordinary English words. Requiring
|
|
173
|
+
// the neighbourhood to ALSO look like a model call stops a comment
|
|
174
|
+
// containing "generated" from turning a regex into a CRITICAL.
|
|
175
|
+
const ls = f.text.lastIndexOf('\n', m.index) + 1;
|
|
176
|
+
let le = f.text.indexOf('\n', m.index); if (le < 0) le = f.text.length;
|
|
177
|
+
if (looksLikePlaceholder(m[0], f.text.slice(ls, le), f.path)) continue;
|
|
157
178
|
out.push(finding('AI-008', 'Model output used in a privileged operation', 'critical',
|
|
158
179
|
f.path, lineOf(f.text, m.index),
|
|
159
180
|
'What the model returns is being executed or used to build a query. Anyone who can steer the model can steer this.',
|
|
@@ -187,6 +208,12 @@ export const AI_CHECKS = [
|
|
|
187
208
|
// ---- provider key used straight from the browser -------------------------
|
|
188
209
|
{ id: 'SEC-006', run(repo, profile) {
|
|
189
210
|
if (!profile.calls_llm) return [];
|
|
211
|
+
// An app with no browser cannot ship a key to one. profile -> gate -> check
|
|
212
|
+
// is the whole idea of this scanner, and this check skipped the profile
|
|
213
|
+
// step: on an api-only, CLI or library project every src/ file looked like
|
|
214
|
+
// browser code, so a plain Node script was told its key is "shipped to
|
|
215
|
+
// every visitor" when it has no visitors.
|
|
216
|
+
if (['api-only', 'cli', 'library', 'mobile-ios', 'mobile-android'].includes(profile.surface)) return [];
|
|
190
217
|
const out = [];
|
|
191
218
|
for (const f of repo.files) {
|
|
192
219
|
if (!f.text || !isCode(f.path) || !isClient(f.path)) continue;
|
package/src/checks-deploy.mjs
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
// Deployment and transport. Mechanically detectable, high volume.
|
|
2
|
+
import { looksLikePlaceholder } from './placeholder.mjs';
|
|
3
|
+
|
|
2
4
|
const finding = (id, title, severity, file, line, detail, fix) =>
|
|
3
5
|
({ id, title, severity, file, line, detail, fix });
|
|
4
6
|
const lineOf = (t, i) => t.slice(0, i).split('\n').length;
|
|
@@ -33,6 +35,12 @@ export const DEPLOY_CHECKS = [
|
|
|
33
35
|
f.text.split('\n').forEach((line, i) => {
|
|
34
36
|
if (/\$\{\{\s*secrets\./.test(line)) return; // the correct pattern
|
|
35
37
|
const m = line.match(SECRET_VALUE);
|
|
38
|
+
// Every project that tests against a database puts a throwaway one in
|
|
39
|
+
// CI - postgres:postgres@localhost/..._test. That is not a credential,
|
|
40
|
+
// and flagging it as a CRITICAL on nearly every real repository is how
|
|
41
|
+
// a scanner loses the right to be believed. This repo's own CI was
|
|
42
|
+
// flagged by this check.
|
|
43
|
+
if (m && looksLikePlaceholder(m[0], line, f.path)) return;
|
|
36
44
|
if (m) out.push(finding('DEP-002', 'Credential written into the CI config', 'critical',
|
|
37
45
|
f.path, i + 1,
|
|
38
46
|
'This value is committed to the repository and printed in build logs. Anyone with read access to either has it.',
|
package/src/checks.mjs
CHANGED
|
@@ -9,6 +9,8 @@ import { FRAMEWORK_CHECKS } from './checks-frameworks.mjs';
|
|
|
9
9
|
import { BATCH2_CHECKS } from './checks-batch2.mjs';
|
|
10
10
|
import { BATCH3_CHECKS } from './checks-batch3.mjs';
|
|
11
11
|
import { BATCH4_CHECKS } from './checks-batch4.mjs';
|
|
12
|
+
import { scanGitHistory } from './git-history.mjs';
|
|
13
|
+
import { looksLikePlaceholder } from './placeholder.mjs';
|
|
12
14
|
|
|
13
15
|
const finding = (id, title, severity, file, line, detail, fix) =>
|
|
14
16
|
({ id, title, severity, file, line, detail, fix });
|
|
@@ -17,6 +19,32 @@ const lineOf = (text, index) => text.slice(0, index).split('\n').length;
|
|
|
17
19
|
|
|
18
20
|
const BASE_CHECKS = [
|
|
19
21
|
|
|
22
|
+
// SEC-009 — a key that was committed, then "removed". SEC-001/003 read the
|
|
23
|
+
// working tree, so deleting the line hides it from them while it stays in
|
|
24
|
+
// .git forever and travels with every clone. scope:'root' because history is
|
|
25
|
+
// a property of the repository, not of a workspace.
|
|
26
|
+
//
|
|
27
|
+
// Only vendor-prefixed keys are looked for. See the long note in
|
|
28
|
+
// git-history.mjs: a test private key and a real one are byte-identical, so
|
|
29
|
+
// including those types fired on 15 of 68 real repositories and every hit was
|
|
30
|
+
// a fixture. Narrowed to unmistakable prefixes it fires on 0 of 20.
|
|
31
|
+
{ id:'SEC-009', scope:'root', run(repo){
|
|
32
|
+
const res = scanGitHistory(repo.root);
|
|
33
|
+
if (!res.findings.length) return [];
|
|
34
|
+
const kinds = [...new Set(res.findings.map(f => f.label))];
|
|
35
|
+
const what = kinds.length === 1 ? kinds[0]
|
|
36
|
+
: `${kinds.slice(0, -1).join(', ')} and ${kinds[kinds.length - 1]}`;
|
|
37
|
+
const n = res.findings.length;
|
|
38
|
+
return [finding('SEC-009',
|
|
39
|
+
`${n === 1 ? 'A key was' : `${n} keys were`} committed and is still in your git history`,
|
|
40
|
+
'critical', '.git', 1,
|
|
41
|
+
`Your ${what} appears in an old commit. Removing it from the file did not remove it — ` +
|
|
42
|
+
'every clone, fork and fetch of this repository still carries it, and anyone who has ever ' +
|
|
43
|
+
'had a copy can read it. Treat it as public.',
|
|
44
|
+
'Rotate the key at the provider now — that is what actually stops it being used. ' +
|
|
45
|
+
'Rewriting history afterwards is optional and does not help anyone who already cloned.')];
|
|
46
|
+
}},
|
|
47
|
+
|
|
20
48
|
{ id:'SEC-002', run(repo){
|
|
21
49
|
const gi = repo.read('.gitignore');
|
|
22
50
|
if (gi === null) return [];
|
|
@@ -85,6 +113,14 @@ const BASE_CHECKS = [
|
|
|
85
113
|
for (const [re,label] of PATTERNS){
|
|
86
114
|
let m; re.lastIndex=0;
|
|
87
115
|
while ((m=re.exec(f.text))){
|
|
116
|
+
// The shape of a key is not a key. AWS publishes AKIAIOSFODNN7EXAMPLE
|
|
117
|
+
// in its own docs; a regex that LOOKS FOR keys contains one by
|
|
118
|
+
// definition; a test fixture is meant to. Scanning this repository
|
|
119
|
+
// produced four CRITICALs and every one was the scanner reading its
|
|
120
|
+
// own detection patterns back to itself.
|
|
121
|
+
const ls = f.text.lastIndexOf('\n', m.index) + 1;
|
|
122
|
+
let le = f.text.indexOf('\n', m.index); if (le < 0) le = f.text.length;
|
|
123
|
+
if (looksLikePlaceholder(m[0], f.text.slice(ls, le), f.path)) continue;
|
|
88
124
|
out.push(finding('SEC-003',`Hardcoded ${label}`,'critical',
|
|
89
125
|
f.path, lineOf(f.text,m.index),
|
|
90
126
|
`A live ${label} is written directly into this file.`,
|
package/src/digest.mjs
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { redact } from './redact.mjs';
|
|
2
|
+
|
|
1
3
|
// Build the block of code the model reads.
|
|
2
4
|
//
|
|
3
5
|
// This is the single most expensive decision in the product. Every token here
|
|
@@ -128,9 +130,15 @@ export function buildDigest(repo, { maxTokens = 150_000, profile = {} } = {}) {
|
|
|
128
130
|
const omitted = [];
|
|
129
131
|
const spentByKind = {};
|
|
130
132
|
let tokens = 0;
|
|
133
|
+
let secretsRedacted = 0;
|
|
131
134
|
|
|
132
135
|
const take = (f, respectShare) => {
|
|
133
|
-
|
|
136
|
+
// Scrub secret values on this machine, before the file can enter the
|
|
137
|
+
// payload. A masked secret never leaves the user's disk. Done here rather
|
|
138
|
+
// than at read time so the free scan — which reads the same files but sends
|
|
139
|
+
// nothing — pays nothing for it.
|
|
140
|
+
const scrubbed = redact(f.text, f.path);
|
|
141
|
+
let body = scrubbed.text;
|
|
134
142
|
let truncated = false;
|
|
135
143
|
if (body.length > MAX_FILE_CHARS) { body = body.slice(0, MAX_FILE_CHARS); truncated = true; }
|
|
136
144
|
const block = `\n──── ${f.path}${truncated ? ' [truncated]' : ''}\n${body}\n`;
|
|
@@ -144,6 +152,7 @@ export function buildDigest(repo, { maxTokens = 150_000, profile = {} } = {}) {
|
|
|
144
152
|
included.push({ path: f.path, kind: f.kind, tokens: cost });
|
|
145
153
|
spentByKind[f.kind] = (spentByKind[f.kind] || 0) + cost;
|
|
146
154
|
tokens += cost;
|
|
155
|
+
secretsRedacted += scrubbed.count;
|
|
147
156
|
return true;
|
|
148
157
|
};
|
|
149
158
|
|
|
@@ -157,6 +166,9 @@ export function buildDigest(repo, { maxTokens = 150_000, profile = {} } = {}) {
|
|
|
157
166
|
tokens,
|
|
158
167
|
included,
|
|
159
168
|
omitted,
|
|
169
|
+
// How many secret values were masked before anything left the machine.
|
|
170
|
+
// Surfaced on the consent screen so the promise is visible, not just made.
|
|
171
|
+
secretsRedacted,
|
|
160
172
|
// what the model must be told it cannot see, or it will reason as though
|
|
161
173
|
// the absence of an ownership check is proof there isn't one
|
|
162
174
|
coverage: {
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
// Read git's object store to find secrets that were committed and later
|
|
2
|
+
// "removed".
|
|
3
|
+
//
|
|
4
|
+
// SEC-001 and SEC-003 read the working tree. That misses the mistake people
|
|
5
|
+
// actually make: commit a key, notice, delete the line, commit again, and
|
|
6
|
+
// believe it is gone. It is not. Every old version of every file is still in
|
|
7
|
+
// .git, and it travels with every clone, every fork and every fetch. The
|
|
8
|
+
// scanner's own finding text already said "deleting it later does not remove it
|
|
9
|
+
// from git history" — and then did not look there.
|
|
10
|
+
//
|
|
11
|
+
// Read-only, and directly: verify-readonly.mjs forbids shelling out, so there
|
|
12
|
+
// is no `git log` here. Loose objects are zlib streams on disk; packed objects
|
|
13
|
+
// live in a packfile with its own index. Both are just reads.
|
|
14
|
+
//
|
|
15
|
+
// This is deliberately NOT a full git implementation. It reads blobs — the file
|
|
16
|
+
// contents — and never needs to walk commits, resolve trees or understand
|
|
17
|
+
// branches, because the question is only "does any version of anything in this
|
|
18
|
+
// repository contain a secret", and every version of everything is a blob.
|
|
19
|
+
import { readdirSync, readFileSync, existsSync, statSync } from 'node:fs';
|
|
20
|
+
import { inflateSync, inflateRawSync } from 'node:zlib';
|
|
21
|
+
import { join } from 'node:path';
|
|
22
|
+
|
|
23
|
+
// ONLY credentials whose format means one thing and cannot mean another.
|
|
24
|
+
//
|
|
25
|
+
// This list was cut down after measuring, and the cut is the whole reason this
|
|
26
|
+
// check is shippable. The first version looked for private keys, database URLs
|
|
27
|
+
// and AWS/Google/Slack keys too, and fired on 15 of 68 real repositories — every
|
|
28
|
+
// single hit a test fixture, a docker-compose service, or documentation. Five
|
|
29
|
+
// rounds of filters got that to 22% and no further, because of one fact that no
|
|
30
|
+
// amount of pattern-matching can get around:
|
|
31
|
+
//
|
|
32
|
+
// A test private key and a production private key are BYTE-IDENTICAL. So are
|
|
33
|
+
// a docker-compose password and a real one. Only intent separates them, and
|
|
34
|
+
// intent is not in the value.
|
|
35
|
+
//
|
|
36
|
+
// A vendor-prefixed key is different: `sk_live_` is issued by Stripe and means
|
|
37
|
+
// a live secret key, always. Nobody's test fixture has a real one. Narrowed to
|
|
38
|
+
// these, the check fires on 2 of 21 repositories — and one of those two is
|
|
39
|
+
// GitLab's own repository, which contains GitLab test tokens.
|
|
40
|
+
//
|
|
41
|
+
// The dropped types are not a gap we are hiding: SEC-001 and SEC-003 still find
|
|
42
|
+
// all of them in the working tree. What is given up is only finding THOSE kinds
|
|
43
|
+
// in deleted history, which is the exact case where they cannot be judged.
|
|
44
|
+
const SECRETS = [
|
|
45
|
+
[/\bsk_live_[A-Za-z0-9]{20,}/g, 'Stripe live secret key'],
|
|
46
|
+
[/\brk_live_[A-Za-z0-9]{20,}/g, 'Stripe restricted live key'],
|
|
47
|
+
[/\bwhsec_[A-Za-z0-9]{24,}/g, 'Stripe webhook signing secret'],
|
|
48
|
+
[/\bsk-ant-[A-Za-z0-9_-]{24,}/g, 'Anthropic API key'],
|
|
49
|
+
[/\bsk-proj-[A-Za-z0-9_-]{24,}/g, 'OpenAI project key'],
|
|
50
|
+
[/\bghp_[A-Za-z0-9]{36,}/g, 'GitHub personal access token'],
|
|
51
|
+
[/\bgho_[A-Za-z0-9]{36,}/g, 'GitHub OAuth token'],
|
|
52
|
+
[/\bglpat-[A-Za-z0-9_-]{20,}/g, 'GitLab personal access token'],
|
|
53
|
+
[/\bSG\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}/g, 'SendGrid API key'],
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
// The first audit of this check fired on 7 of 8 major repositories and every
|
|
57
|
+
// single hit was documentation, a test fixture or a secretlint allowlist — the
|
|
58
|
+
// exact "matched a shape instead of judging the context" failure this codebase
|
|
59
|
+
// has already made seven times. Three filters, in order of how much they caught:
|
|
60
|
+
//
|
|
61
|
+
// 1. The VALUE itself is a sample. `sk_live_abcdefghijk...` and
|
|
62
|
+
// `redis://username:password@host:port/db` are not credentials, they are
|
|
63
|
+
// the shape of one written down. A real secret is high-entropy; a fake one
|
|
64
|
+
// spells words, runs the alphabet, or literally says "password".
|
|
65
|
+
// 2. The surrounding LINE is prose or a fixture — a docs code fence, an
|
|
66
|
+
// `expect(...)`, an allowlist entry.
|
|
67
|
+
// 3. The blob is a documentation or test FILE.
|
|
68
|
+
const FAKE_VALUE = [
|
|
69
|
+
// No \b: a fixture key spells the word INSIDE the token, e.g.
|
|
70
|
+
// sk_live_51H8xExAmPlEkEyAbCdEf123. Word boundaries never match there, and
|
|
71
|
+
// that gap let this check fire on its own repository's test file.
|
|
72
|
+
/(example|sample|dummy|placeholder|fake|changeme|redacted|notreal|donotuse)/i,
|
|
73
|
+
/\b(your[_-]?key|my[_-]?secret|xxx+)\b/i,
|
|
74
|
+
/abcdefghij|0123456789|qwerty|lorem/i,
|
|
75
|
+
// A substitution of ANY form means the real value is not here: ${VAR},
|
|
76
|
+
// {{var}}, [password], <password>, %s, $VAR. Every remaining false positive
|
|
77
|
+
// in the first two audits was one of these.
|
|
78
|
+
/[$%]\{|\{\{|\[[a-z][a-z0-9_.-]*\]|<[a-z][a-z0-9_.-]*>|\$[A-Z_][A-Z0-9_]*/i,
|
|
79
|
+
// credentials that are the words for credentials
|
|
80
|
+
/:\/\/(?:user|username|admin|root|foo|bar|test|me|you)[^:@/]*:(?:pass|password|passwd|secret|hunter2|foo|bar|test)/i,
|
|
81
|
+
// A password containing "test", "demo", "local" or "dev" is a fixture. The
|
|
82
|
+
// corpus is full of mostest_password, knextest, devpassword.
|
|
83
|
+
/:\/\/[^:@/]+:[^@/]*(?:test|demo|local|dev|dummy|ci)[^@/]*@/i,
|
|
84
|
+
// A password that merely repeats the username or the service is a default,
|
|
85
|
+
// not a secret: postgres:postgres@, mysql:mysql@, redis:redis@.
|
|
86
|
+
/:\/\/([a-z0-9_-]+):\1@/i,
|
|
87
|
+
// The host is an internal service name from a docker-compose or CI network —
|
|
88
|
+
// a single label with no dot, e.g. @postgres:5432, @db, @mysql. A production
|
|
89
|
+
// database is reached at a real hostname.
|
|
90
|
+
/@(?:postgres|postgresql|mysql|mariadb|mongo|mongodb|redis|db|database|cache|queue)(?:[-_][a-z0-9-]+)?(?::\d+)?(?:[/?]|$)/i,
|
|
91
|
+
// a placeholder host is a placeholder URL
|
|
92
|
+
/@(?:localhost|127\.0\.0\.1|0\.0\.0\.0|host|hostname|example\.(?:com|org|net)|your[^\s/]*)(?:[:/]|$)/i,
|
|
93
|
+
// no dot in the host at all: not a routable production endpoint
|
|
94
|
+
/:\/\/[^@/]+@[a-z0-9_-]+(?::\d+)?(?:[/?]|$)/i,
|
|
95
|
+
// AWS's own documentation key
|
|
96
|
+
/^AKIAIOSFODNN7EXAMPLE$/,
|
|
97
|
+
];
|
|
98
|
+
// Entropy: a real key is random. `sk_live_abcdef...` is not. Measured over the
|
|
99
|
+
// secret's payload, after the recognisable prefix.
|
|
100
|
+
function looksRandom(value) {
|
|
101
|
+
const body = value.replace(/^(sk_live_|rk_live_|whsec_|sk-ant-|sk-proj-|AKIA|AIza|gh[pousr]_|glpat-|xox[baprs]-|SG\.)/, '');
|
|
102
|
+
if (body.length < 12) return true; // too short to judge; keep it
|
|
103
|
+
const set = new Set(body.replace(/[^A-Za-z0-9]/g, ''));
|
|
104
|
+
if (set.size < 8) return false; // e.g. all a-f, or one repeated run
|
|
105
|
+
// sequential alphabet or digits is the signature of a hand-typed placeholder
|
|
106
|
+
let runs = 0;
|
|
107
|
+
for (let i = 1; i < body.length; i++) if (body.charCodeAt(i) === body.charCodeAt(i - 1) + 1) runs++;
|
|
108
|
+
return runs < body.length * 0.3;
|
|
109
|
+
}
|
|
110
|
+
// The line the secret sits on. Docs, assertions and allowlists are not leaks.
|
|
111
|
+
const FAKE_LINE = /(^\s*[#*]|^\s*\/\/|\.\. |::$|`{1,3}|expect\(|assert|describe\(|it\(|test\(|allows?"?\s*:|allowlist|secretlint|gitleaks|trufflehog|\.md:|<\/?[a-z]+>)/i;
|
|
112
|
+
|
|
113
|
+
// What the secret was ASSIGNED TO, looking back from the match. A real key and
|
|
114
|
+
// a test key are byte-identical — the only difference is intent, and intent is
|
|
115
|
+
// written in the name above it. `EXAMPLE_PRIVATE_KEY = """`, `KEY1 = """` under
|
|
116
|
+
// a "Keys and certificates for tests" docstring, `TEST_CERT`, `FIXTURE_KEY`.
|
|
117
|
+
// This was the last class of false positive left in the corpus audit.
|
|
118
|
+
const FAKE_DECLARATION = /\b(example|sample|test|tests|testing|fixture|mock|stub|dummy|fake|demo|placeholder|invalid|expired|revoked|specimen)[a-z0-9_]*\s*[:=]\s*(?:"""|'''|["'`]|$)/i;
|
|
119
|
+
// The blob is documentation or a fixture. Detected from the content, since a
|
|
120
|
+
// blob does not carry its path.
|
|
121
|
+
//
|
|
122
|
+
// The strongest signal is that the blob IS a test file. A test asserting that
|
|
123
|
+
// secret-handling works necessarily contains secret-shaped strings — this
|
|
124
|
+
// scanner's own test/redact.mjs does, and that is what made SEC-009 fire on
|
|
125
|
+
// Launchprep's own repository the first time it ran. A key inside a test file
|
|
126
|
+
// is a fixture; a real one would never be committed there deliberately.
|
|
127
|
+
const FAKE_BLOB = /(^|\n)(#{1,3} |\.\. _|={3,}$|-{3,}$)|describe\(|it\(['"`]|def test_|class Test|@pytest|\bfixtures?\b/i;
|
|
128
|
+
const IS_TEST_FILE = /(^|\n)\s*(?:import|from|const|let|require)[^\n]*\b(?:assert|chai|jest|mocha|vitest|pytest|unittest|testing)\b|(^|\n)\s*(?:describe|it|test)\s*\(|(^|\n)\s*(?:def test_|class Test)|\bok\(['"`]|expect\(/;
|
|
129
|
+
|
|
130
|
+
// A blob big enough to be a build artefact or a vendored bundle is not where a
|
|
131
|
+
// human pasted a key, and inflating hundreds of them is the whole cost here.
|
|
132
|
+
const MAX_BLOB = 400_000;
|
|
133
|
+
// A ceiling on total work so an enormous repository cannot make a scan hang.
|
|
134
|
+
// Reached only on repositories with very deep history; reported honestly when
|
|
135
|
+
// it happens rather than silently truncating.
|
|
136
|
+
const MAX_OBJECTS = 20_000;
|
|
137
|
+
|
|
138
|
+
function looseObjects(objDir, budget) {
|
|
139
|
+
const out = [];
|
|
140
|
+
let buckets;
|
|
141
|
+
try { buckets = readdirSync(objDir); } catch { return out; }
|
|
142
|
+
for (const b of buckets) {
|
|
143
|
+
if (!/^[0-9a-f]{2}$/.test(b)) continue;
|
|
144
|
+
let files;
|
|
145
|
+
try { files = readdirSync(join(objDir, b)); } catch { continue; }
|
|
146
|
+
for (const f of files) {
|
|
147
|
+
if (out.length >= budget) return out;
|
|
148
|
+
out.push({ id: b + f, path: join(objDir, b, f) });
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return out;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// A packfile stores objects as a stream of zlib chunks with no per-object
|
|
155
|
+
// offsets available without parsing .idx. Rather than implement idx parsing and
|
|
156
|
+
// delta reconstruction — a large amount of code for a small gain — the packfile
|
|
157
|
+
// is scanned for the zlib streams it contains and each is inflated where it
|
|
158
|
+
// can be. Objects stored as deltas against another object are skipped: they are
|
|
159
|
+
// modifications of a base, and a secret introduced in one shows up in the full
|
|
160
|
+
// copy of some version anyway. Reported in `partial` so a clean result on a
|
|
161
|
+
// packed repository never silently claims more than it checked.
|
|
162
|
+
function packedBlobs(packDir, budget, onBlob) {
|
|
163
|
+
let files;
|
|
164
|
+
try { files = readdirSync(packDir); } catch { return { scanned: 0, partial: false }; }
|
|
165
|
+
let scanned = 0, partial = false;
|
|
166
|
+
for (const f of files.filter(x => x.endsWith('.pack'))) {
|
|
167
|
+
let buf;
|
|
168
|
+
try { buf = readFileSync(join(packDir, f)); } catch { continue; }
|
|
169
|
+
// header: 'PACK', version, count
|
|
170
|
+
if (buf.length < 12 || buf.toString('latin1', 0, 4) !== 'PACK') continue;
|
|
171
|
+
// Walk looking for zlib stream starts (0x78 followed by a valid check byte).
|
|
172
|
+
// Inflating from a wrong offset simply throws and is skipped, so a false
|
|
173
|
+
// start costs nothing but a try/catch.
|
|
174
|
+
for (let i = 12; i < buf.length - 2; i++) {
|
|
175
|
+
if (scanned >= budget) { partial = true; return { scanned, partial }; }
|
|
176
|
+
if (buf[i] !== 0x78) continue;
|
|
177
|
+
const b1 = buf[i + 1];
|
|
178
|
+
if (((buf[i] << 8) | b1) % 31 !== 0) continue;
|
|
179
|
+
try {
|
|
180
|
+
const inflated = inflateSync(buf.subarray(i), { finishFlush: 2 /* Z_SYNC_FLUSH */ });
|
|
181
|
+
if (inflated.length && inflated.length <= MAX_BLOB) {
|
|
182
|
+
scanned++;
|
|
183
|
+
onBlob(inflated.toString('utf8'), `${f}@${i}`);
|
|
184
|
+
}
|
|
185
|
+
} catch { /* not a stream start, or a delta we do not reconstruct */ }
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
return { scanned, partial };
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Returns { findings, objectsScanned, partial, reason }.
|
|
192
|
+
// `reason` is set when history could not be read at all, so the caller can say
|
|
193
|
+
// so rather than implying a clean history.
|
|
194
|
+
export function scanGitHistory(root) {
|
|
195
|
+
const gitDir = join(root, '.git');
|
|
196
|
+
if (!existsSync(gitDir)) return { findings: [], objectsScanned: 0, partial: false, reason: 'no-git' };
|
|
197
|
+
// A worktree or submodule has .git as a FILE pointing elsewhere. Not followed:
|
|
198
|
+
// the pointer can lead outside the directory the user asked us to scan, and
|
|
199
|
+
// reading outside it is not ours to do.
|
|
200
|
+
try { if (!statSync(gitDir).isDirectory()) return { findings: [], objectsScanned: 0, partial: false, reason: 'git-is-a-file' }; }
|
|
201
|
+
catch { return { findings: [], objectsScanned: 0, partial: false, reason: 'no-git' }; }
|
|
202
|
+
|
|
203
|
+
const seen = new Set();
|
|
204
|
+
const findings = [];
|
|
205
|
+
let objectsScanned = 0;
|
|
206
|
+
|
|
207
|
+
const inspect = (text, where) => {
|
|
208
|
+
if (!text || text.length > MAX_BLOB) return;
|
|
209
|
+
// A documentation page or a test file can contain any number of
|
|
210
|
+
// key-shaped strings and none of them are leaks.
|
|
211
|
+
const docish = FAKE_BLOB.test(text.slice(0, 2000)) || IS_TEST_FILE.test(text.slice(0, 4000));
|
|
212
|
+
for (const [re, label] of SECRETS) {
|
|
213
|
+
re.lastIndex = 0;
|
|
214
|
+
let m;
|
|
215
|
+
while ((m = re.exec(text))) {
|
|
216
|
+
const value = m[0];
|
|
217
|
+
if (FAKE_VALUE.some(f => f.test(value))) continue;
|
|
218
|
+
if (!looksRandom(value)) continue;
|
|
219
|
+
// the line it sits on decides as much as the value does
|
|
220
|
+
const ls = text.lastIndexOf('\n', m.index) + 1;
|
|
221
|
+
let le = text.indexOf('\n', m.index); if (le < 0) le = text.length;
|
|
222
|
+
const line = text.slice(ls, le);
|
|
223
|
+
if (FAKE_LINE.test(line)) continue;
|
|
224
|
+
// Look back a few hundred characters for the declaration this value was
|
|
225
|
+
// assigned to, and for a test/fixture docstring above it.
|
|
226
|
+
const before = text.slice(Math.max(0, m.index - 300), m.index);
|
|
227
|
+
if (FAKE_DECLARATION.test(before)) continue;
|
|
228
|
+
if (/\b(for tests?|test (?:keys?|certs?|fixtures?|data)|do not use|not a real)\b/i.test(before)) continue;
|
|
229
|
+
if (docish) continue;
|
|
230
|
+
// One finding per distinct secret, however many old commits carry it.
|
|
231
|
+
const key = label + ':' + value.slice(0, 24);
|
|
232
|
+
if (seen.has(key)) continue;
|
|
233
|
+
seen.add(key);
|
|
234
|
+
findings.push({ label, where, hint: value.slice(0, 7) + '…' });
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
const objDir = join(gitDir, 'objects');
|
|
240
|
+
for (const o of looseObjects(objDir, MAX_OBJECTS)) {
|
|
241
|
+
try {
|
|
242
|
+
const raw = inflateSync(readFileSync(o.path));
|
|
243
|
+
const nul = raw.indexOf(0);
|
|
244
|
+
if (nul < 0) continue;
|
|
245
|
+
if (!raw.subarray(0, nul).toString('latin1').startsWith('blob')) continue;
|
|
246
|
+
objectsScanned++;
|
|
247
|
+
inspect(raw.subarray(nul + 1).toString('utf8'), o.id.slice(0, 8));
|
|
248
|
+
} catch { /* unreadable object; skipped */ }
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
const packed = packedBlobs(join(objDir, 'pack'), MAX_OBJECTS - objectsScanned, inspect);
|
|
252
|
+
objectsScanned += packed.scanned;
|
|
253
|
+
|
|
254
|
+
return { findings, objectsScanned, partial: packed.partial, reason: null };
|
|
255
|
+
}
|
package/src/index.mjs
CHANGED
|
@@ -3,7 +3,7 @@ import { scanRepo } from './fs-scan.mjs';
|
|
|
3
3
|
import { splitWorkspaces } from './workspace.mjs';
|
|
4
4
|
import { detectProfile, toGateProfile } from './detect.mjs';
|
|
5
5
|
import { gate, missingFacts } from './gate.mjs';
|
|
6
|
-
import { runChecks, runRootChecks } from './checks.mjs';
|
|
6
|
+
import { runChecks, runRootChecks, CHECKS } from './checks.mjs';
|
|
7
7
|
import { render } from './report.mjs';
|
|
8
8
|
import { existsSync, statSync } from 'node:fs';
|
|
9
9
|
|
|
@@ -41,11 +41,48 @@ if (!existsSync(target) || !statSync(target).isDirectory()) {
|
|
|
41
41
|
const repo = scanRepo(target, { maxFiles: 12000 });
|
|
42
42
|
const packages = splitWorkspaces(repo);
|
|
43
43
|
|
|
44
|
+
// What the user told us about their own app, which beats what we guessed.
|
|
45
|
+
//
|
|
46
|
+
// The scanner works most things out from the code, but some facts are simply
|
|
47
|
+
// not in the code — where the users live, whether the AI is allowed to act,
|
|
48
|
+
// how many people are expected. Those checks were skipped and the report said
|
|
49
|
+
// "Correct it and the checks adjust", which was false: there was no way to
|
|
50
|
+
// correct anything. This is that way.
|
|
51
|
+
//
|
|
52
|
+
// Reading a file the user wrote is still read-only. Nothing is written.
|
|
53
|
+
const stated = (() => {
|
|
54
|
+
const raw = repo.read('launchprep.json');
|
|
55
|
+
if (raw === null) return null;
|
|
56
|
+
try {
|
|
57
|
+
const j = JSON.parse(raw);
|
|
58
|
+
return (j && typeof j === 'object' && !Array.isArray(j)) ? j : null;
|
|
59
|
+
} catch (e) {
|
|
60
|
+
// A broken file must be loud. Silently ignoring it would mean the user
|
|
61
|
+
// answers the questions, sees no change, and concludes the feature is fake.
|
|
62
|
+
process.stderr.write(`\n \x1b[31mlaunchprep.json could not be read: ${e.message}\x1b[0m\n`);
|
|
63
|
+
process.stderr.write(` \x1b[2mIt must be valid JSON. Nothing from it was used.\x1b[0m\n\n`);
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
})();
|
|
67
|
+
|
|
68
|
+
// Merge one level deep so { "stack": { "auth": "clerk" } } overrides only auth
|
|
69
|
+
// and leaves the detected framework and database alone.
|
|
70
|
+
const applyStated = (profile) => {
|
|
71
|
+
if (!stated) return profile;
|
|
72
|
+
const out = { ...profile };
|
|
73
|
+
for (const [k, v] of Object.entries(stated)) {
|
|
74
|
+
if (v && typeof v === 'object' && !Array.isArray(v) && out[k] && typeof out[k] === 'object')
|
|
75
|
+
out[k] = { ...out[k], ...v };
|
|
76
|
+
else out[k] = v;
|
|
77
|
+
}
|
|
78
|
+
return out;
|
|
79
|
+
};
|
|
80
|
+
|
|
44
81
|
// Profile and gate every app in the repo. Libraries are profiled too - they
|
|
45
82
|
// simply match very few rules, which is the correct outcome, not a bug.
|
|
46
83
|
const scanned = packages.map(w => {
|
|
47
84
|
const full = detectProfile(w.view, { root: repo });
|
|
48
|
-
const profile = toGateProfile(full);
|
|
85
|
+
const profile = applyStated(toGateProfile(full));
|
|
49
86
|
const g = gate(profile);
|
|
50
87
|
const applicableIds = new Set(g.evaluated.map(r => r.id));
|
|
51
88
|
const findings = runChecks(w.view, profile, applicableIds)
|
|
@@ -86,17 +123,28 @@ const shallow = lead.gate.evaluated
|
|
|
86
123
|
|
|
87
124
|
const questions = missingFacts(lead.profile, lead.gate.unknown).slice(0, 3);
|
|
88
125
|
|
|
126
|
+
// "51 checks apply to this app" counted 17 that nothing examined — tier-2 rules
|
|
127
|
+
// that need the deep scan. Counting them as applying is a silent pass on a
|
|
128
|
+
// third of the headline number, which is the one thing this scanner must never
|
|
129
|
+
// do. It also threw away the sentence that sells the paid tier: those 17 are
|
|
130
|
+
// not a gap, they are the product.
|
|
131
|
+
const implemented = new Set(CHECKS.map(c => c.id));
|
|
132
|
+
const coverage = {
|
|
133
|
+
ran: lead.gate.evaluated.filter(r => implemented.has(r.id)).length,
|
|
134
|
+
needsDeep: lead.gate.evaluated.filter(r => !implemented.has(r.id)).length,
|
|
135
|
+
};
|
|
136
|
+
|
|
89
137
|
if (asJson) {
|
|
90
138
|
console.log(JSON.stringify({
|
|
91
139
|
packages: scanned.map(s => ({
|
|
92
140
|
name: s.name, profile: s.profile,
|
|
93
141
|
evaluated: s.gate.evaluated.length, skipped: s.gate.skipped.length, unknown: s.gate.unknown.length,
|
|
94
142
|
})),
|
|
95
|
-
findings, questions,
|
|
143
|
+
findings, questions, coverage, stated,
|
|
96
144
|
shallow: shallow.map(r => ({ id: r.id, title: r.title, severity: r.severity })),
|
|
97
145
|
}, null, 2));
|
|
98
146
|
} else {
|
|
99
|
-
process.stdout.write(render({ repo: target, scanned, lead, findings, questions, shallow }));
|
|
147
|
+
process.stdout.write(render({ repo: target, scanned, lead, findings, questions, shallow, coverage, stated }));
|
|
100
148
|
}
|
|
101
149
|
|
|
102
150
|
// The gate, after the full report has printed: the human still sees
|