launchprep 0.4.0 → 0.5.1

package/src/fs-scan.mjs

@@ -105,8 +105,19 @@ export function buildIgnore(rootAbs, sources) {
   };
 }
 
-// Every .gitignore from the git root down to the scan root, plus .git/info/exclude.
-export function ignoreChain(root) {
+// Every .gitignore from the git root down to the scan root, plus
+// .git/info/exclude, plus any .gitignore found INSIDE the tree.
+//
+// The ancestors alone were not enough. A .gitignore written in a subdirectory -
+// config/.gitignore, apps/web/.gitignore - was never read, so a .env it
+// correctly excluded was reported by SEC-001 as a leaked credential. Nesting
+// one is ordinary practice and it is the natural place for it in a monorepo.
+//
+// `inner` is appended AFTER the ancestors on purpose: buildIgnore is
+// last-match-wins and scopes each rule to the directory it was written in, so
+// appending gives the deeper file precedence, which is what git does. A nested
+// `!keep.env` therefore beats a root `.env*`.
+export function ignoreChain(root, inner = []) {
   const rootAbs = resolve(root);
   const gr = gitRoot(rootAbs);
   const dirs = [];
@@ -123,11 +134,37 @@ export function ignoreChain(root) {
     const t = readAt(gr, join('.git', 'info', 'exclude'));
     if (t !== null) sources.push({ dir: gr, text: t });
   }
-  return buildIgnore(rootAbs, sources);
+  return buildIgnore(rootAbs, [...sources, ...inner]);
+}
+
+// The ONE repo-view constructor. Every "repo" object a check receives - the
+// whole-repo scan, a workspace slice, the tests-excluded view - is built here,
+// so all three expose the SAME capabilities. Three hand-rolled versions is
+// exactly how isIgnored went missing from the workspace view and SEC-001 cried
+// wolf on every monorepo. Add a capability here and it exists in every view, or
+// the omission is a one-line default rather than a silent gap.
+//
+// has/find/grep always read the `files` array. The file accessors
+// (exists/read/isIgnored) are injected because they genuinely differ: the root
+// scan answers from disk, a slice answers from its own files. Injecting them
+// keeps that difference explicit instead of accidentally absent.
+const GREP_CODE = /\.(ts|tsx|js|jsx|mjs|cjs|py|rb|sql|prisma)$/;
+export function makeView({ files, root, isIgnored, read, exists, grepDefault = GREP_CODE }) {
+  return {
+    root, files,
+    has:  (re) => files.some(f => re.test(f.path)),
+    find: (re) => files.filter(f => re.test(f.path)),
+    grep: (re, pathRe = grepDefault) =>
+      files.filter(f => f.text && pathRe.test(f.path) && re.test(f.text)),
+    exists: exists   || ((rel) => files.some(f => f.path === rel)),
+    read:   read     || ((rel) => files.find(f => f.path === rel)?.text ?? null),
+    isIgnored: isIgnored || (() => false),
+  };
 }
 
 export function scanRepo(root, { maxFiles = 6000 } = {}) {
   const files = [];
+  const nested = [];          // .gitignore files found inside the tree
   const rootAbs = resolve(root);
   const walk = (dir) => {
     if (files.length >= maxFiles) return;
@@ -135,6 +172,15 @@ export function scanRepo(root, { maxFiles = 6000 } = {}) {
     try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
     for (const e of entries) {
       if (files.length >= maxFiles) return;
+      // Read before the dotfile skip below, and deliberately NOT added to
+      // `files` - this is input to the ignore matcher, not part of the scan.
+      // The one at the scan root is already covered by the ancestor chain.
+      if (e.name === '.gitignore') {
+        if (dir !== rootAbs) {
+          try { nested.push({ dir, text: readFileSync(join(dir, e.name), 'utf8') }); } catch {}
+        }
+        continue;
+      }
       if (e.name.startsWith('.') && !e.name.startsWith('.env') && e.name !== '.github') continue;
       const full = join(dir, e.name);
 
@@ -157,17 +203,16 @@ export function scanRepo(root, { maxFiles = 6000 } = {}) {
     }
   };
   walk(root);
-  return {
-    root,
-    files,
-    has: (re) => files.some(f => re.test(f.path)),
-    find: (re) => files.filter(f => re.test(f.path)),
-    grep: (re, pathRe = /\.(ts|tsx|js|jsx|mjs|cjs|py|rb|php|erb|sql|prisma|ya?ml)$|(^|\/)(Gemfile|manage\.py|artisan)$/) =>
-      files.filter(f => f.text && pathRe.test(f.path) && re.test(f.text)),
+  // The root scan answers exists/read from disk (so a check can ask about a file
+  // that was not walked into `files`, e.g. a large or binary one), greps a wider
+  // set of file types than a slice does, and carries the full .gitignore chain.
+  return makeView({
+    files, root,
+    grepDefault: /\.(ts|tsx|js|jsx|mjs|cjs|py|rb|php|erb|sql|prisma|ya?ml)$|(^|\/)(Gemfile|manage\.py|artisan)$/,
     exists: (rel) => existsSync(join(root, rel)),
-    read: (rel) => { try { return readFileSync(join(root, rel), 'utf8'); } catch { return null; } },
-    isIgnored: ignoreChain(root),
-  };
+    read:   (rel) => { try { return readFileSync(join(root, rel), 'utf8'); } catch { return null; } },
+    isIgnored: ignoreChain(root, nested),
+  });
 }
 
 export function readPackageJson(repo) {

package/src/git-history.mjs

@@ -0,0 +1,255 @@
+// Read git's object store to find secrets that were committed and later
+// "removed".
+//
+// SEC-001 and SEC-003 read the working tree. That misses the mistake people
+// actually make: commit a key, notice, delete the line, commit again, and
+// believe it is gone. It is not. Every old version of every file is still in
+// .git, and it travels with every clone, every fork and every fetch. The
+// scanner's own finding text already said "deleting it later does not remove it
+// from git history" — and then did not look there.
+//
+// Read-only, and directly: verify-readonly.mjs forbids shelling out, so there
+// is no `git log` here. Loose objects are zlib streams on disk; packed objects
+// live in a packfile with its own index. Both are just reads.
+//
+// This is deliberately NOT a full git implementation. It reads blobs — the file
+// contents — and never needs to walk commits, resolve trees or understand
+// branches, because the question is only "does any version of anything in this
+// repository contain a secret", and every version of everything is a blob.
+import { readdirSync, readFileSync, existsSync, statSync } from 'node:fs';
+import { inflateSync, inflateRawSync } from 'node:zlib';
+import { join } from 'node:path';
+
+// ONLY credentials whose format means one thing and cannot mean another.
+//
+// This list was cut down after measuring, and the cut is the whole reason this
+// check is shippable. The first version looked for private keys, database URLs
+// and AWS/Google/Slack keys too, and fired on 15 of 68 real repositories — every
+// single hit a test fixture, a docker-compose service, or documentation. Five
+// rounds of filters got that to 22% and no further, because of one fact that no
+// amount of pattern-matching can get around:
+//
+//   A test private key and a production private key are BYTE-IDENTICAL. So are
+//   a docker-compose password and a real one. Only intent separates them, and
+//   intent is not in the value.
+//
+// A vendor-prefixed key is different: `sk_live_` is issued by Stripe and means
+// a live secret key, always. Nobody's test fixture has a real one. Narrowed to
+// these, the check fires on 2 of 21 repositories — and one of those two is
+// GitLab's own repository, which contains GitLab test tokens.
+//
+// The dropped types are not a gap we are hiding: SEC-001 and SEC-003 still find
+// all of them in the working tree. What is given up is only finding THOSE kinds
+// in deleted history, which is the exact case where they cannot be judged.
+const SECRETS = [
+  [/\bsk_live_[A-Za-z0-9]{20,}/g,            'Stripe live secret key'],
+  [/\brk_live_[A-Za-z0-9]{20,}/g,            'Stripe restricted live key'],
+  [/\bwhsec_[A-Za-z0-9]{24,}/g,              'Stripe webhook signing secret'],
+  [/\bsk-ant-[A-Za-z0-9_-]{24,}/g,           'Anthropic API key'],
+  [/\bsk-proj-[A-Za-z0-9_-]{24,}/g,          'OpenAI project key'],
+  [/\bghp_[A-Za-z0-9]{36,}/g,                'GitHub personal access token'],
+  [/\bgho_[A-Za-z0-9]{36,}/g,                'GitHub OAuth token'],
+  [/\bglpat-[A-Za-z0-9_-]{20,}/g,            'GitLab personal access token'],
+  [/\bSG\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}/g, 'SendGrid API key'],
+];
+
+// The first audit of this check fired on 7 of 8 major repositories and every
+// single hit was documentation, a test fixture or a secretlint allowlist — the
+// exact "matched a shape instead of judging the context" failure this codebase
+// has already made seven times. Three filters, in order of how much they caught:
+//
+//  1. The VALUE itself is a sample. `sk_live_abcdefghijk...` and
+//     `redis://username:password@host:port/db` are not credentials, they are
+//     the shape of one written down. A real secret is high-entropy; a fake one
+//     spells words, runs the alphabet, or literally says "password".
+//  2. The surrounding LINE is prose or a fixture — a docs code fence, an
+//     `expect(...)`, an allowlist entry.
+//  3. The blob is a documentation or test FILE.
+const FAKE_VALUE = [
+  // No \b: a fixture key spells the word INSIDE the token, e.g.
+  // sk_live_51H8xExAmPlEkEyAbCdEf123. Word boundaries never match there, and
+  // that gap let this check fire on its own repository's test file.
+  /(example|sample|dummy|placeholder|fake|changeme|redacted|notreal|donotuse)/i,
+  /\b(your[_-]?key|my[_-]?secret|xxx+)\b/i,
+  /abcdefghij|0123456789|qwerty|lorem/i,
+  // A substitution of ANY form means the real value is not here: ${VAR},
+  // {{var}}, [password], <password>, %s, $VAR. Every remaining false positive
+  // in the first two audits was one of these.
+  /[$%]\{|\{\{|\[[a-z][a-z0-9_.-]*\]|<[a-z][a-z0-9_.-]*>|\$[A-Z_][A-Z0-9_]*/i,
+  // credentials that are the words for credentials
+  /:\/\/(?:user|username|admin|root|foo|bar|test|me|you)[^:@/]*:(?:pass|password|passwd|secret|hunter2|foo|bar|test)/i,
+  // A password containing "test", "demo", "local" or "dev" is a fixture. The
+  // corpus is full of mostest_password, knextest, devpassword.
+  /:\/\/[^:@/]+:[^@/]*(?:test|demo|local|dev|dummy|ci)[^@/]*@/i,
+  // A password that merely repeats the username or the service is a default,
+  // not a secret: postgres:postgres@, mysql:mysql@, redis:redis@.
+  /:\/\/([a-z0-9_-]+):\1@/i,
+  // The host is an internal service name from a docker-compose or CI network —
+  // a single label with no dot, e.g. @postgres:5432, @db, @mysql. A production
+  // database is reached at a real hostname.
+  /@(?:postgres|postgresql|mysql|mariadb|mongo|mongodb|redis|db|database|cache|queue)(?:[-_][a-z0-9-]+)?(?::\d+)?(?:[/?]|$)/i,
+  // a placeholder host is a placeholder URL
+  /@(?:localhost|127\.0\.0\.1|0\.0\.0\.0|host|hostname|example\.(?:com|org|net)|your[^\s/]*)(?:[:/]|$)/i,
+  // no dot in the host at all: not a routable production endpoint
+  /:\/\/[^@/]+@[a-z0-9_-]+(?::\d+)?(?:[/?]|$)/i,
+  // AWS's own documentation key
+  /^AKIAIOSFODNN7EXAMPLE$/,
+];
+// Entropy: a real key is random. `sk_live_abcdef...` is not. Measured over the
+// secret's payload, after the recognisable prefix.
+function looksRandom(value) {
+  const body = value.replace(/^(sk_live_|rk_live_|whsec_|sk-ant-|sk-proj-|AKIA|AIza|gh[pousr]_|glpat-|xox[baprs]-|SG\.)/, '');
+  if (body.length < 12) return true;                 // too short to judge; keep it
+  const set = new Set(body.replace(/[^A-Za-z0-9]/g, ''));
+  if (set.size < 8) return false;                    // e.g. all a-f, or one repeated run
+  // sequential alphabet or digits is the signature of a hand-typed placeholder
+  let runs = 0;
+  for (let i = 1; i < body.length; i++) if (body.charCodeAt(i) === body.charCodeAt(i - 1) + 1) runs++;
+  return runs < body.length * 0.3;
+}
+// The line the secret sits on. Docs, assertions and allowlists are not leaks.
+const FAKE_LINE = /(^\s*[#*]|^\s*\/\/|\.\. |::$|`{1,3}|expect\(|assert|describe\(|it\(|test\(|allows?"?\s*:|allowlist|secretlint|gitleaks|trufflehog|\.md:|<\/?[a-z]+>)/i;
+
+// What the secret was ASSIGNED TO, looking back from the match. A real key and
+// a test key are byte-identical — the only difference is intent, and intent is
+// written in the name above it. `EXAMPLE_PRIVATE_KEY = """`, `KEY1 = """` under
+// a "Keys and certificates for tests" docstring, `TEST_CERT`, `FIXTURE_KEY`.
+// This was the last class of false positive left in the corpus audit.
+const FAKE_DECLARATION = /\b(example|sample|test|tests|testing|fixture|mock|stub|dummy|fake|demo|placeholder|invalid|expired|revoked|specimen)[a-z0-9_]*\s*[:=]\s*(?:"""|'''|["'`]|$)/i;
+// The blob is documentation or a fixture. Detected from the content, since a
+// blob does not carry its path.
+//
+// The strongest signal is that the blob IS a test file. A test asserting that
+// secret-handling works necessarily contains secret-shaped strings — this
+// scanner's own test/redact.mjs does, and that is what made SEC-009 fire on
+// Launchprep's own repository the first time it ran. A key inside a test file
+// is a fixture; a real one would never be committed there deliberately.
+const FAKE_BLOB = /(^|\n)(#{1,3} |\.\. _|={3,}$|-{3,}$)|describe\(|it\(['"`]|def test_|class Test|@pytest|\bfixtures?\b/i;
+const IS_TEST_FILE = /(^|\n)\s*(?:import|from|const|let|require)[^\n]*\b(?:assert|chai|jest|mocha|vitest|pytest|unittest|testing)\b|(^|\n)\s*(?:describe|it|test)\s*\(|(^|\n)\s*(?:def test_|class Test)|\bok\(['"`]|expect\(/;
+
+// A blob big enough to be a build artefact or a vendored bundle is not where a
+// human pasted a key, and inflating hundreds of them is the whole cost here.
+const MAX_BLOB = 400_000;
+// A ceiling on total work so an enormous repository cannot make a scan hang.
+// Reached only on repositories with very deep history; reported honestly when
+// it happens rather than silently truncating.
+const MAX_OBJECTS = 20_000;
+
+function looseObjects(objDir, budget) {
+  const out = [];
+  let buckets;
+  try { buckets = readdirSync(objDir); } catch { return out; }
+  for (const b of buckets) {
+    if (!/^[0-9a-f]{2}$/.test(b)) continue;
+    let files;
+    try { files = readdirSync(join(objDir, b)); } catch { continue; }
+    for (const f of files) {
+      if (out.length >= budget) return out;
+      out.push({ id: b + f, path: join(objDir, b, f) });
+    }
+  }
+  return out;
+}
+
+// A packfile stores objects as a stream of zlib chunks with no per-object
+// offsets available without parsing .idx. Rather than implement idx parsing and
+// delta reconstruction — a large amount of code for a small gain — the packfile
+// is scanned for the zlib streams it contains and each is inflated where it
+// can be. Objects stored as deltas against another object are skipped: they are
+// modifications of a base, and a secret introduced in one shows up in the full
+// copy of some version anyway. Reported in `partial` so a clean result on a
+// packed repository never silently claims more than it checked.
+function packedBlobs(packDir, budget, onBlob) {
+  let files;
+  try { files = readdirSync(packDir); } catch { return { scanned: 0, partial: false }; }
+  let scanned = 0, partial = false;
+  for (const f of files.filter(x => x.endsWith('.pack'))) {
+    let buf;
+    try { buf = readFileSync(join(packDir, f)); } catch { continue; }
+    // header: 'PACK', version, count
+    if (buf.length < 12 || buf.toString('latin1', 0, 4) !== 'PACK') continue;
+    // Walk looking for zlib stream starts (0x78 followed by a valid check byte).
+    // Inflating from a wrong offset simply throws and is skipped, so a false
+    // start costs nothing but a try/catch.
+    for (let i = 12; i < buf.length - 2; i++) {
+      if (scanned >= budget) { partial = true; return { scanned, partial }; }
+      if (buf[i] !== 0x78) continue;
+      const b1 = buf[i + 1];
+      if (((buf[i] << 8) | b1) % 31 !== 0) continue;
+      try {
+        const inflated = inflateSync(buf.subarray(i), { finishFlush: 2 /* Z_SYNC_FLUSH */ });
+        if (inflated.length && inflated.length <= MAX_BLOB) {
+          scanned++;
+          onBlob(inflated.toString('utf8'), `${f}@${i}`);
+        }
+      } catch { /* not a stream start, or a delta we do not reconstruct */ }
+    }
+  }
+  return { scanned, partial };
+}
+
+// Returns { findings, objectsScanned, partial, reason }.
+// `reason` is set when history could not be read at all, so the caller can say
+// so rather than implying a clean history.
+export function scanGitHistory(root) {
+  const gitDir = join(root, '.git');
+  if (!existsSync(gitDir)) return { findings: [], objectsScanned: 0, partial: false, reason: 'no-git' };
+  // A worktree or submodule has .git as a FILE pointing elsewhere. Not followed:
+  // the pointer can lead outside the directory the user asked us to scan, and
+  // reading outside it is not ours to do.
+  try { if (!statSync(gitDir).isDirectory()) return { findings: [], objectsScanned: 0, partial: false, reason: 'git-is-a-file' }; }
+  catch { return { findings: [], objectsScanned: 0, partial: false, reason: 'no-git' }; }
+
+  const seen = new Set();
+  const findings = [];
+  let objectsScanned = 0;
+
+  const inspect = (text, where) => {
+    if (!text || text.length > MAX_BLOB) return;
+    // A documentation page or a test file can contain any number of
+    // key-shaped strings and none of them are leaks.
+    const docish = FAKE_BLOB.test(text.slice(0, 2000)) || IS_TEST_FILE.test(text.slice(0, 4000));
+    for (const [re, label] of SECRETS) {
+      re.lastIndex = 0;
+      let m;
+      while ((m = re.exec(text))) {
+        const value = m[0];
+        if (FAKE_VALUE.some(f => f.test(value))) continue;
+        if (!looksRandom(value)) continue;
+        // the line it sits on decides as much as the value does
+        const ls = text.lastIndexOf('\n', m.index) + 1;
+        let le = text.indexOf('\n', m.index); if (le < 0) le = text.length;
+        const line = text.slice(ls, le);
+        if (FAKE_LINE.test(line)) continue;
+        // Look back a few hundred characters for the declaration this value was
+        // assigned to, and for a test/fixture docstring above it.
+        const before = text.slice(Math.max(0, m.index - 300), m.index);
+        if (FAKE_DECLARATION.test(before)) continue;
+        if (/\b(for tests?|test (?:keys?|certs?|fixtures?|data)|do not use|not a real)\b/i.test(before)) continue;
+        if (docish) continue;
+        // One finding per distinct secret, however many old commits carry it.
+        const key = label + ':' + value.slice(0, 24);
+        if (seen.has(key)) continue;
+        seen.add(key);
+        findings.push({ label, where, hint: value.slice(0, 7) + '…' });
+      }
+    }
+  };
+
+  const objDir = join(gitDir, 'objects');
+  for (const o of looseObjects(objDir, MAX_OBJECTS)) {
+    try {
+      const raw = inflateSync(readFileSync(o.path));
+      const nul = raw.indexOf(0);
+      if (nul < 0) continue;
+      if (!raw.subarray(0, nul).toString('latin1').startsWith('blob')) continue;
+      objectsScanned++;
+      inspect(raw.subarray(nul + 1).toString('utf8'), o.id.slice(0, 8));
+    } catch { /* unreadable object; skipped */ }
+  }
+
+  const packed = packedBlobs(join(objDir, 'pack'), MAX_OBJECTS - objectsScanned, inspect);
+  objectsScanned += packed.scanned;
+
+  return { findings, objectsScanned, partial: packed.partial, reason: null };
+}

package/src/index.mjs

@@ -3,7 +3,7 @@ import { scanRepo } from './fs-scan.mjs';
 import { splitWorkspaces } from './workspace.mjs';
 import { detectProfile, toGateProfile } from './detect.mjs';
 import { gate, missingFacts } from './gate.mjs';
-import { runChecks, runRootChecks } from './checks.mjs';
+import { runChecks, runRootChecks, CHECKS } from './checks.mjs';
 import { render } from './report.mjs';
 import { existsSync, statSync } from 'node:fs';
 
@@ -41,11 +41,48 @@ if (!existsSync(target) || !statSync(target).isDirectory()) {
 const repo = scanRepo(target, { maxFiles: 12000 });
 const packages = splitWorkspaces(repo);
 
+// What the user told us about their own app, which beats what we guessed.
+//
+// The scanner works most things out from the code, but some facts are simply
+// not in the code — where the users live, whether the AI is allowed to act,
+// how many people are expected. Those checks were skipped and the report said
+// "Correct it and the checks adjust", which was false: there was no way to
+// correct anything. This is that way.
+//
+// Reading a file the user wrote is still read-only. Nothing is written.
+const stated = (() => {
+  const raw = repo.read('launchprep.json');
+  if (raw === null) return null;
+  try {
+    const j = JSON.parse(raw);
+    return (j && typeof j === 'object' && !Array.isArray(j)) ? j : null;
+  } catch (e) {
+    // A broken file must be loud. Silently ignoring it would mean the user
+    // answers the questions, sees no change, and concludes the feature is fake.
+    process.stderr.write(`\n  \x1b[31mlaunchprep.json could not be read: ${e.message}\x1b[0m\n`);
+    process.stderr.write(`  \x1b[2mIt must be valid JSON. Nothing from it was used.\x1b[0m\n\n`);
+    return null;
+  }
+})();
+
+// Merge one level deep so { "stack": { "auth": "clerk" } } overrides only auth
+// and leaves the detected framework and database alone.
+const applyStated = (profile) => {
+  if (!stated) return profile;
+  const out = { ...profile };
+  for (const [k, v] of Object.entries(stated)) {
+    if (v && typeof v === 'object' && !Array.isArray(v) && out[k] && typeof out[k] === 'object')
+      out[k] = { ...out[k], ...v };
+    else out[k] = v;
+  }
+  return out;
+};
+
 // Profile and gate every app in the repo. Libraries are profiled too - they
 // simply match very few rules, which is the correct outcome, not a bug.
 const scanned = packages.map(w => {
   const full = detectProfile(w.view, { root: repo });
-  const profile = toGateProfile(full);
+  const profile = applyStated(toGateProfile(full));
   const g = gate(profile);
   const applicableIds = new Set(g.evaluated.map(r => r.id));
   const findings = runChecks(w.view, profile, applicableIds)
@@ -86,17 +123,28 @@ const shallow = lead.gate.evaluated
 
 const questions = missingFacts(lead.profile, lead.gate.unknown).slice(0, 3);
 
+// "51 checks apply to this app" counted 17 that nothing examined — tier-2 rules
+// that need the deep scan. Counting them as applying is a silent pass on a
+// third of the headline number, which is the one thing this scanner must never
+// do. It also threw away the sentence that sells the paid tier: those 17 are
+// not a gap, they are the product.
+const implemented = new Set(CHECKS.map(c => c.id));
+const coverage = {
+  ran:       lead.gate.evaluated.filter(r => implemented.has(r.id)).length,
+  needsDeep: lead.gate.evaluated.filter(r => !implemented.has(r.id)).length,
+};
+
 if (asJson) {
   console.log(JSON.stringify({
     packages: scanned.map(s => ({
       name: s.name, profile: s.profile,
       evaluated: s.gate.evaluated.length, skipped: s.gate.skipped.length, unknown: s.gate.unknown.length,
     })),
-    findings, questions,
+    findings, questions, coverage, stated,
     shallow: shallow.map(r => ({ id: r.id, title: r.title, severity: r.severity })),
   }, null, 2));
 } else {
-  process.stdout.write(render({ repo: target, scanned, lead, findings, questions, shallow }));
+  process.stdout.write(render({ repo: target, scanned, lead, findings, questions, shallow, coverage, stated }));
 }
 
 // The gate, after the full report has printed: the human still sees

package/src/placeholder.mjs

@@ -0,0 +1,72 @@
+// Is this value a real credential, or something that only looks like one?
+//
+// This is the seventh-and-eighth instance of the mistake this codebase keeps
+// making: matching a SHAPE and calling it a secret. Scanning its own repository
+// produced five CRITICALs and every one was this —
+//
+//   .github/workflows/ci.yml   postgresql://postgres:postgres@localhost/..._test
+//   cli/src/git-history.mjs    /^AKIAIOSFODNN7EXAMPLE$/     (a detection pattern)
+//   cli/src/checks-*.mjs       the regexes that look for model output
+//
+// None is a credential. The first is the throwaway database every project with
+// CI tests spins up; the second is AWS's own published example; the third is
+// source code that describes a pattern rather than containing a secret.
+//
+// The discipline is the one written at the top of the project guide: read the
+// VALUE, and the line it sits on, not the shape. A leaked key is high-entropy,
+// points at a real host, and is not sitting inside a regex.
+//
+// The redaction pass in redact.mjs learned the same lesson separately. This is
+// the shared version so a fix in one place is a fix in both.
+
+// The value spells out that it is not real.
+const FAKE_WORDS = /(example|sample|dummy|placeholder|fake|changeme|redacted|notreal|donotuse|yourkey|your[_-]key|xxx+|test[_-]?key|foobar)/i;
+
+// Sequences a human types when inventing a key.
+const TYPED_BY_HAND = /abcdefghij|0123456789|qwerty|lorem|(.)\1{7,}/i;
+
+// A substitution means the real value is somewhere else: ${VAR}, {{var}},
+// [password], <password>, $VAR, %s.
+const SUBSTITUTION = /[$%]\{|\{\{|\[[a-z][a-z0-9_.-]*\]|<[a-z][a-z0-9_.-]*>|\$[A-Z_][A-Z0-9_]*/i;
+
+// A connection string that points nowhere real.
+const LOCAL_OR_SERVICE_HOST =
+  /@(?:localhost|127\.0\.0\.1|0\.0\.0\.0|host|hostname|example\.(?:com|org|net)|postgres|postgresql|mysql|mariadb|mongo|mongodb|redis|db|database)(?:[-_][a-z0-9-]+)?(?::\d+)?(?:[/?]|$)/i;
+// user and password are the same word, or are the words for themselves.
+const DEFAULT_CREDENTIALS =
+  /:\/\/([a-z0-9_-]+):\1@|:\/\/(?:user|username|admin|root|foo|bar|test|me)[^:@/]*:(?:pass|password|passwd|secret|hunter2|foo|bar|test)/i;
+// a password that announces it is for testing
+const TEST_CREDENTIALS = /:\/\/[^:@/]+:[^@/]*(?:test|demo|local|dev|dummy|ci|example)[^@/]*@/i;
+
+// AWS publishes these two in its own documentation. They are in a thousand
+// READMEs and are not anyone's key.
+const AWS_DOC_KEYS = /^(AKIAIOSFODNN7EXAMPLE|ASIAIOSFODNN7EXAMPLE)$/;
+
+/**
+ * @param value the matched string
+ * @param line  the whole line it appeared on (optional, but much better with it)
+ * @param path  the file path (optional)
+ */
+export function looksLikePlaceholder(value, line = '', path = '') {
+  const v = String(value);
+
+  if (FAKE_WORDS.test(v) || TYPED_BY_HAND.test(v) || SUBSTITUTION.test(v)) return true;
+  if (AWS_DOC_KEYS.test(v)) return true;
+  if (LOCAL_OR_SERVICE_HOST.test(v) || DEFAULT_CREDENTIALS.test(v) || TEST_CREDENTIALS.test(v)) return true;
+
+  const l = String(line);
+  // The line DEFINES a pattern rather than holding a secret: a regex literal,
+  // a list of shapes to look for. This is what made the scanner flag itself.
+  if (/\/\^|\^.*\$\/|\\b|\\d|\\w|\[A-Za-z0-9|\[0-9A-Z|RegExp\(|\.test\(|\.match\(/.test(l)) return true;
+  // A comment, or documentation showing the format.
+  if (/^\s*(?:[#*]|\/\/|--)/.test(l)) return true;
+  // An assertion or fixture.
+  if (/expect\(|assert|describe\(|it\(['"`]|toBe\(|toEqual\(/.test(l)) return true;
+
+  const p = String(path);
+  // A test file or a fixture directory is not where real credentials live.
+  if (/(^|\/)(test|tests|__tests__|spec|e2e|cypress|fixtures?|mocks?|__mocks__|examples?)\//.test(p)) return true;
+  if (/\.(test|spec)\.[a-z]+$/.test(p)) return true;
+
+  return false;
+}