hullproof 0.0.0-stage → 0.5.0

package/dist/scanners/common.js

@@ -0,0 +1,162 @@
+// Helpers shared by the adapters: running a tool, reading the git index without git, hashing, masking, path rules.
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { existsSync, lstatSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { isAbsolute, join, relative, resolve, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { cmp } from '../util.js';
+export const ENGINE_ROOT = fileURLToPath(new URL('../../', import.meta.url));
+export const RULES_DIR = join(ENGINE_ROOT, 'rules');
+const MAX_BUFFER = 512 * 1024 * 1024;
+// An address nothing listens on. Go, Python and git clients send all HTTP and HTTPS traffic to it and fail,
+// so a tool that tries the network while network is not allowed gets an error instead of a connection.
+const BLACKHOLE = 'http://127.0.0.1:9';
+export function toolEnv(base, allowNetwork) {
+    const env = {
+        ...base,
+        GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null', GIT_CONFIG_NOSYSTEM: '1',
+        GIT_OPTIONAL_LOCKS: '0', GIT_TERMINAL_PROMPT: '0', NO_COLOR: '1',
+        SEMGREP_SEND_METRICS: 'off', SEMGREP_ENABLE_VERSION_CHECK: '0',
+        // Set by the container entry point of the GitHub Action only (see src/commands/git.ts).
+        ...(process.env.HULLPROOF_TRUST_REPO_OWNER === '1' ? { GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'safe.directory', GIT_CONFIG_VALUE_0: '*' } : {}),
+    };
+    if (!allowNetwork) {
+        for (const k of ['HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'])
+            env[k] = BLACKHOLE;
+        env.NO_PROXY = '';
+        env.no_proxy = '';
+    }
+    return env;
+}
+export function exec(argv, cwd, env) {
+    const r = spawnSync(argv[0], argv.slice(1), { cwd, env, encoding: 'utf8', maxBuffer: MAX_BUFFER, shell: false });
+    const error = r.error ? (r.error.code ?? r.error.message) : undefined;
+    return { argv, stdout: r.stdout ?? '', stderr: r.stderr ?? '', status: r.status, error };
+}
+// First x.y.z in the version command's output. null when the tool is missing.
+export function detectVersion(argv, env) {
+    const r = exec(argv, process.cwd(), env);
+    if (r.error)
+        return null;
+    return /(\d+\.\d+\.\d+)/.exec(r.stdout + '\n' + r.stderr)?.[1] ?? null;
+}
+// "1.176.x" style ranges: the installed version must have the same major and minor.
+export function versionTested(version, range) {
+    if (!version)
+        return false;
+    const [maj, min] = range.split('.');
+    const [vmaj, vmin] = version.split('.');
+    return maj === vmaj && min === vmin;
+}
+export const sha256 = (data) => createHash('sha256').update(data).digest('hex');
+export const fileSha256 = (p) => sha256(readFileSync(p));
+// The scan.sh method for a folder: sha256 over the per file sha256 lines, files in sorted path order.
+export function dirSha256(dir) {
+    const files = [];
+    const walk = (d) => {
+        for (const e of readdirSync(d, { withFileTypes: true })) {
+            const p = join(d, e.name);
+            if (e.isDirectory())
+                walk(p);
+            else if (e.isFile())
+                files.push(relative(dir, p).split(sep).join('/'));
+        }
+    };
+    walk(dir);
+    return sha256(files.sort(cmp).map((f) => fileSha256(join(dir, f)) + '\n').join(''));
+}
+// Repository relative POSIX path. Absolute paths inside the repository lose the prefix; "./" is dropped.
+export function relPath(p, repo) {
+    let out = p.replace(/\\/g, '/');
+    if (isAbsolute(p)) {
+        const r = relative(resolve(repo), p);
+        out = r.startsWith('..') ? out : r.split(sep).join('/');
+    }
+    return out.replace(/^(\.\/)+/, '');
+}
+// Replaces long token shaped runs (keys, hashes, encoded blobs: 20 or more key characters with both a letter and a
+// digit) so evidence text does not carry a secret value. Plain identifiers such as AWS_SECRET_ACCESS_KEY stay readable.
+// ponytail: a secret with no digit passes; evidence for secret rules is the Gitleaks redacted form or null anyway.
+const TOKEN = /[A-Za-z0-9+/_-]{20,}={0,2}/g;
+export function mask(text, max = 160) {
+    const one = text.replace(/\s+/g, ' ').trim().replace(TOKEN, (m) => (/\d/.test(m) && /[A-Za-z]/.test(m) ? '***' : m));
+    return one.length > max ? one.slice(0, max - 3) + '...' : one;
+}
+// One masked source line read from the repository, or null when the file or line is not there.
+export function sourceLine(repo, file, line) {
+    if (line < 1)
+        return null;
+    try {
+        const p = join(repo, file);
+        if (!lstatSync(p).isFile() || statSync(p).size > 4 * 1024 * 1024)
+            return null;
+        const text = readFileSync(p, 'utf8').split('\n')[line - 1];
+        return text === undefined ? null : mask(text);
+    }
+    catch {
+        return null;
+    }
+}
+// Paths tracked by git, read from .git/index (versions 2, 3 and 4) without running git. Stage 0 to 3 entries are one
+// path; submodule entries (mode 160000) are left out. null when there is no index.
+export function readTracked(repo) {
+    let gitDir = join(repo, '.git');
+    if (!existsSync(gitDir))
+        return null;
+    if (statSync(gitDir).isFile()) {
+        const m = /^gitdir:\s*(.+)$/m.exec(readFileSync(gitDir, 'utf8'));
+        if (!m)
+            return null;
+        gitDir = resolve(repo, m[1].trim());
+    }
+    const idx = join(gitDir, 'index');
+    if (!existsSync(idx))
+        return null;
+    return parseIndex(readFileSync(idx));
+}
+export function parseIndex(buf) {
+    if (buf.toString('latin1', 0, 4) !== 'DIRC')
+        throw new Error('not a git index');
+    const version = buf.readUInt32BE(4);
+    if (version < 2 || version > 4)
+        throw new Error(`git index version ${version} is not supported`);
+    const count = buf.readUInt32BE(8);
+    const out = new Set();
+    let off = 12;
+    let prev = Buffer.alloc(0);
+    for (let i = 0; i < count; i++) {
+        const start = off;
+        const mode = buf.readUInt32BE(off + 24);
+        const flags = buf.readUInt16BE(off + 60);
+        off += 62;
+        if (version >= 3 && flags & 0x4000)
+            off += 2;
+        let name;
+        if (version === 4) {
+            let c = buf[off++];
+            let strip = c & 127;
+            while (c & 128) {
+                c = buf[off++];
+                strip = ((strip + 1) << 7) | (c & 127);
+            }
+            const end = buf.indexOf(0, off);
+            name = Buffer.concat([prev.subarray(0, prev.length - strip), buf.subarray(off, end)]);
+            off = end + 1;
+        }
+        else {
+            const end = buf.indexOf(0, off);
+            name = buf.subarray(off, end);
+            off = start + Math.floor((end - start + 8) / 8) * 8; // NUL padding to a multiple of 8
+        }
+        prev = name;
+        if ((mode >>> 12) !== 0o16)
+            out.add(name.toString('utf8')); // 0o160000 is a submodule
+    }
+    return out;
+}
+export const isRegularFile = (p) => { try {
+    return lstatSync(p).isFile();
+}
+catch {
+    return false;
+} };

package/dist/scanners/gitleaks.js

@@ -0,0 +1,90 @@
+// Gitleaks adapter. `gitleaks dir` over the working tree always; `gitleaks git` over history only when the caller says
+// the git configuration was checked (opts.history), because it runs git. Both use the kit config in rules/gitleaks.toml,
+// --redact, --ignore-gitleaks-allow and an empty ignore folder, so the scanned repository cannot hide its own findings.
+// Kept per hit: rule id, file, line, the redacted match (masked again) and the Gitleaks fingerprint. Never the secret,
+// the entropy, the author or the date.
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { secretNoise } from './secret-noise.js';
+import { sortUnique } from '../util.js';
+import { fileSha256, mask, relPath } from './common.js';
+import { inEdition, loadControls } from '../mapping/controls.js';
+import { mapHit, tableSecIds } from '../mapping/map.js';
+const flags = (config, ignoreDir) => ['--redact', '--no-banner', '--no-color', '--config', config,
+    '--ignore-gitleaks-allow', '--gitleaks-ignore-path', ignoreDir, '--report-format', 'json', '--report-path', '-', '--exit-code', '0'];
+export const adapter = {
+    id: 'gitleaks',
+    tool: 'gitleaks',
+    versionCommand: ['gitleaks', 'version'],
+    testedVersions: '8.30.x',
+    needsNetwork: false,
+    applies: () => true,
+    notApplicableReason: () => null,
+    configFiles: (ctx) => [join(ctx.rulesDir ?? '', 'gitleaks.toml')],
+    command: (ctx) => ['gitleaks', 'dir', '.', ...flags(join(ctx.rulesDir ?? '', 'gitleaks.toml'), '<empty-dir>')],
+    parse: (stdout, ctx) => parseGitleaks(stdout, ctx, 'dir', null, '').hits,
+    possibleSecIds: (o) => sortUnique(inEdition(tableSecIds('gitleaks'), o.edition, loadControls(o.controlsPath))),
+    execute(o, env) {
+        const config = join(env.rulesDir, 'gitleaks.toml');
+        const configSha256 = fileSha256(config);
+        const ruleVersion = `gitleaks ${env.version}, config sha256:${configSha256.slice(0, 16)}`;
+        const show = (argv) => argv.map((a) => (a === config ? '<engine>/rules/gitleaks.toml' : a === env.scratch ? '<empty-dir>' : a));
+        const runOne = (mode) => {
+            // Trace logging is the only place Gitleaks names each file it read. The log also prints skipped candidate values,
+            // so stderr is read here for the file names only, never stored and never printed.
+            const argv = ['gitleaks', mode, '.', ...flags(config, env.scratch), ...(mode === 'dir' ? ['--log-level', 'trace'] : [])];
+            const r = env.run(argv);
+            if (r.status !== 0)
+                throw new Error(`gitleaks ${mode} exited with ${r.status ?? r.error}`);
+            return r;
+        };
+        const dir = runOne('dir');
+        // The engine's own output folder is left out: it is written by this tool, so reading it would change the next run.
+        const own = (f) => f === '.hullproof' || f.startsWith('.hullproof/');
+        const scanned = sortUnique([...dir.stderr.matchAll(/scanning path path=(.+)$/gm)].map((m) => relPath(m[1].trim(), o.repo))).filter((f) => !own(f));
+        const hits = parseGitleaks(dir.stdout, o, 'dir', env.tracked, ruleVersion).hits.filter((h) => !own(h.file));
+        const commands = [show(dir.argv)];
+        if (o.history) {
+            const git = runOne('git');
+            hits.push(...parseGitleaks(git.stdout, o, 'git', env.tracked, ruleVersion).hits);
+            commands.push(show(git.argv));
+        }
+        return { hits, commands, configSha256, scanned, exitCode: dir.status, toolErrors: [] };
+    },
+};
+export function parseGitleaks(stdout, o, source, tracked, ruleVersion) {
+    const controls = loadControls(o.controlsPath);
+    const list = (stdout.trim() ? JSON.parse(stdout) : []);
+    // The source line is the only place a placeholder can be told from a key, because the scanner redacts the value. History hits
+    // cannot be matched to the checked out files, so they get the path rules only.
+    const lines = new Map();
+    const lineOf = (file, n) => {
+        if (source === 'git')
+            return null;
+        if (!lines.has(file)) {
+            try {
+                lines.set(file, readFileSync(join(o.repo, file), 'utf8').split('\n'));
+            }
+            catch {
+                lines.set(file, null);
+            }
+        }
+        return lines.get(file)?.[n - 1] ?? null;
+    };
+    return {
+        hits: list.map((f) => {
+            const file = relPath(f.File, o.repo);
+            const isTracked = source === 'git' || !tracked || tracked.has(file);
+            const m = mapHit({ tool: 'gitleaks', rule_id: f.RuleID, file, tracked: isTracked }, o.edition, controls);
+            const redacted = mask(f.Match);
+            const noise = secretNoise(f.RuleID, file, lineOf(file, f.StartLine));
+            return {
+                tool: 'gitleaks', rule_id: f.RuleID, file, line: f.StartLine, end_line: f.EndLine, sec_ids: m.sec_ids, severity_hint: null,
+                ...(noise ? { noise } : {}), message_key: `gitleaks:${f.RuleID}`, rule_version: ruleVersion, evidence: redacted, mapped_by: m.mapped_by, unmapped: !m.sec_ids.length,
+                secret: { redacted, fingerprint: relFingerprint(f.Fingerprint, o.repo), source, commit: f.Commit || null, tracked: isTracked },
+            };
+        }),
+    };
+}
+// The fingerprint is [commit:]file:rule:line; make its file part repository relative too.
+const relFingerprint = (fp, repo) => fp.includes(repo) ? fp.split(repo.replace(/\/$/, '') + '/').join('') : fp;

package/dist/scanners/index.js

@@ -0,0 +1,113 @@
+// Runs the scanner adapters and writes, per tool, scan/<tool>.json (HitsFile), scan/<tool>.meta.json (ToolMeta) and
+// scan/<tool>.detail.json (DetailFile). Nothing is installed. A missing tool, a tool that needs the network when the
+// network is not allowed, a tool with nothing to read, and a tool that fails are each recorded with the reason.
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { cmp, stableJson } from '../util.js';
+import { readHeadCommit } from '../discover/git.js';
+import { loadControls, notInControls } from '../mapping/controls.js';
+import { detectVersion, exec, readTracked, RULES_DIR, toolEnv, versionTested } from './common.js';
+import { adapter as semgrep } from './semgrep.js';
+import { adapter as gitleaks } from './gitleaks.js';
+import { adapter as osv } from './osv-scanner.js';
+import { adapter as trivy } from './trivy.js';
+export const ADAPTERS = [semgrep, gitleaks, osv, trivy];
+const hitKey = (h) => [h.file, String(h.line).padStart(9, '0'), h.tool, h.rule_id];
+const byKey = (a, b) => { for (let i = 0; i < a.length; i++) {
+    const c = cmp(a[i], b[i]);
+    if (c)
+        return c;
+} return 0; };
+export function toContractHits(details) {
+    const seen = new Map();
+    for (const d of details) {
+        const h = { tool: d.tool, rule_id: d.rule_id, file: d.file, line: d.line, sec_ids: d.sec_ids, severity_hint: d.severity_hint, message_key: d.message_key, ...(d.noise ? { noise: d.noise } : {}) };
+        const k = stableJson(h);
+        if (!seen.has(k))
+            seen.set(k, h);
+    }
+    return [...seen.values()].sort((a, b) => byKey(hitKey(a), hitKey(b)) || cmp(stableJson(a), stableJson(b)));
+}
+const sortDetails = (xs) => xs.map((d) => [d, stableJson(d)]).sort((a, b) => byKey(hitKey(a[0]), hitKey(b[0])) || cmp(a[1], b[1])).map((x) => x[0]);
+export function runScanner(a, opts) {
+    const started = Date.now();
+    const o = { ...opts, rulesDir: opts.rulesDir ?? RULES_DIR };
+    const env0 = toolEnv(o.env ?? process.env, o.allowNetwork);
+    const controls = loadControls(o.controlsPath);
+    const tracked = readTracked(o.repo);
+    const version = detectVersion(a.versionCommand, env0);
+    let status = 'ran';
+    let reason = null;
+    let res = null;
+    if (!version) {
+        status = 'skipped: not installed';
+        reason = `${a.tool} is not installed on this machine (not found on PATH). The engine does not install tools.`;
+    }
+    else if (a.needsNetwork && !o.allowNetwork) {
+        status = 'skipped: network not allowed';
+        reason = `${a.tool} sends package names and versions to osv.dev and needs its advisory data. Run with --allow-network to include it.`;
+    }
+    else if (!a.applies(o.stack)) {
+        status = 'skipped: not applicable';
+        reason = a.notApplicableReason(o);
+    }
+    else {
+        const scratch = mkdtempSync(join(tmpdir(), 'hullproof-scan-'));
+        const env = {
+            version, tracked, rulesDir: o.rulesDir, scratch,
+            run: (argv) => exec(argv, o.repo, env0),
+        };
+        try {
+            res = a.execute(o, env);
+            if (res && o.skipFiles?.size)
+                res = { ...res, hits: res.hits.filter((h) => !o.skipFiles.has(h.file)) };
+        }
+        catch (e) {
+            status = 'failed';
+            reason = `${a.tool} did not finish: ${e.message}`;
+        }
+        finally {
+            rmSync(scratch, { recursive: true, force: true });
+        }
+    }
+    const details = sortDetails(res?.hits ?? []);
+    const scanned = res?.scanned ?? null;
+    const scannedSet = new Set(scanned ?? []);
+    const files = {
+        scanned: scanned ? scanned.length : null,
+        tracked: tracked ? tracked.size : null,
+        scanned_not_tracked: scanned && tracked ? scanned.filter((f) => !tracked.has(f)).length : null,
+        tracked_not_scanned: scanned && tracked ? [...tracked].filter((f) => !scannedSet.has(f)).length : null,
+    };
+    const ran = status === 'ran';
+    const meta = {
+        schema_version: '1.0.0', tool: a.id, status, version, tested_version_range: a.testedVersions,
+        version_tested: versionTested(version, a.testedVersions), argv: res?.commands[0] ?? [], config_sha256: res?.configSha256 ?? null,
+        commit: readHeadCommit(o.repo), files_scanned: files.scanned, files_tracked: files.tracked, exit_code: res?.exitCode ?? null,
+        db_date: res?.dbDate ?? null, not_assessed_sec_ids: ran ? [] : a.possibleSecIds(o),
+    };
+    const detail = {
+        schema_version: '0.1.0', tool: a.id, status, not_run_reason: reason, tool_version: version, commands: res?.commands ?? [],
+        files, as_of: res?.asOf ?? null, policy: res?.policy ?? null,
+        sec_ids_not_in_controls: [...new Set(notInControls(details.flatMap((d) => [...d.sec_ids, ...(d.advisory?.mapped_sec_ids ?? [])]), controls))].sort(cmp),
+        tool_errors: (res?.toolErrors ?? []).sort((x, y) => cmp(stableJson(x), stableJson(y))),
+        hits: details,
+    };
+    return { hits: { schema_version: '1.0.0', tool: a.id, hits: toContractHits(details) }, meta, detail, seconds: (Date.now() - started) / 1000 };
+}
+export function writeScan(outDir, s) {
+    const dir = join(outDir, 'scan');
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, `${s.meta.tool}.json`), stableJson(s.hits));
+    writeFileSync(join(dir, `${s.meta.tool}.meta.json`), stableJson(s.meta));
+    writeFileSync(join(dir, `${s.meta.tool}.detail.json`), stableJson(s.detail));
+}
+// Runs every adapter (or the ones named) and writes their files. Run times go to scan/run.json, outside the contract.
+export function runScanners(opts, tools) {
+    const out = ADAPTERS.filter((a) => !tools || tools.includes(a.id)).map((a) => runScanner(a, opts));
+    for (const s of out)
+        writeScan(opts.out, s);
+    writeFileSync(join(opts.out, 'scan', 'run.json'), JSON.stringify({ seconds: Object.fromEntries(out.map((s) => [s.meta.tool, s.seconds])) }, null, 2) + '\n');
+    return out;
+}

package/dist/scanners/osv-scanner.js

@@ -0,0 +1,77 @@
+// OSV Scanner adapter. It sends package names and versions to osv.dev, so it runs only with --allow-network; without it
+// the tool is NOT RUN with that reason. One hit per advisory group per package, with the advisory id, package, version,
+// severity class, fixed versions and the dependency policy decision. The result is reproducible only for the same
+// advisory data date, recorded as db_date.
+import { cmp, sortUnique } from '../util.js';
+import { relPath } from './common.js';
+import { inEdition, loadControls } from '../mapping/controls.js';
+import { mapHit, tableSecIds } from '../mapping/map.js';
+import { classFromScore, classFromWord, decide, DEFAULT_POLICY } from '../mapping/dependency-policy.js';
+const RANK = { UNKNOWN: 0, LOW: 1, MEDIUM: 2, HIGH: 3, CRITICAL: 4 };
+export const today = () => new Date().toISOString().slice(0, 10);
+export const adapter = {
+    id: 'osv-scanner',
+    tool: 'osv-scanner',
+    versionCommand: ['osv-scanner', '--version'],
+    testedVersions: '2.6.x',
+    needsNetwork: true,
+    applies: (stack) => stack.package_managers.length > 0,
+    notApplicableReason: (o) => (adapter.applies(o.stack) ? null : 'No package manifest or lockfile was found, so there are no dependencies to check.'),
+    configFiles: () => [],
+    command: () => ['osv-scanner', 'scan', 'source', '--format', 'json', '--recursive', '.'],
+    parse: (stdout, ctx) => parseOsv(stdout, ctx, ctx.asOf ?? today()).hits,
+    possibleSecIds: (o) => sortUnique(inEdition(tableSecIds('osv-scanner'), o.edition, loadControls(o.controlsPath))),
+    execute(o, env) {
+        const asOf = o.asOf ?? today();
+        const argv = adapter.command(o);
+        const r = env.run(argv);
+        // 0: no advisories, 1: advisories found, 128: no packages found. Anything else is a failure.
+        if (r.status !== 0 && r.status !== 1 && r.status !== 128)
+            throw new Error(`osv-scanner exited with ${r.status ?? r.error}`);
+        const out = r.status === 128 ? { hits: [], scanned: [] } : parseOsv(r.stdout, o, asOf);
+        return { hits: out.hits, commands: [argv], configSha256: null, scanned: out.scanned, exitCode: r.status, toolErrors: [], dbDate: asOf, asOf, policy: o.policy ?? DEFAULT_POLICY };
+    },
+};
+export function parseOsv(stdout, o, asOf) {
+    const controls = loadControls(o.controlsPath);
+    const policy = o.policy ?? DEFAULT_POLICY;
+    const j = JSON.parse(stdout);
+    const hits = [];
+    const scanned = new Set();
+    for (const res of j.results ?? []) {
+        const file = relPath(res.source.path, o.repo);
+        scanned.add(file);
+        for (const p of res.packages ?? []) {
+            for (const g of p.groups ?? []) {
+                const ids = [...g.ids].sort(cmp);
+                const id = ids.find((x) => x.startsWith('GHSA-')) ?? ids[0];
+                const vulns = (p.vulnerabilities ?? []).filter((v) => ids.includes(v.id));
+                const score = Number.parseFloat(g.max_severity ?? '');
+                const cls = score > 0 ? classFromScore(score)
+                    : vulns.map((v) => classFromWord(v.database_specific?.severity)).reduce((a, b) => (RANK[b] > RANK[a] ? b : a), 'UNKNOWN');
+                const dates = vulns.map((v) => v.published?.slice(0, 10)).filter((d) => !!d).sort(cmp);
+                const published = dates[0] ?? null;
+                const fixed = sortUnique(vulns.flatMap((v) => (v.affected ?? [])
+                    .filter((a) => a.package?.name === p.package.name)
+                    .flatMap((a) => (a.ranges ?? []).flatMap((rg) => (rg.events ?? []).map((e) => e.fixed).filter((x) => !!x)))));
+                const d = decide(cls, published, asOf, policy);
+                const version = p.package.version || p.package.commit || '';
+                const mapped = mapHit({ tool: 'osv-scanner', rule_id: id, file }, o.edition, controls);
+                const fails = d.decision === 'fail';
+                const modified = vulns.map((v) => v.modified ?? '').sort(cmp).pop() || 'unknown';
+                hits.push({
+                    tool: 'osv-scanner', rule_id: id, file, line: 0, end_line: 0, sec_ids: fails ? mapped.sec_ids : [],
+                    severity_hint: cls, message_key: `osv-scanner:${id}`, rule_version: `advisory modified ${modified}`,
+                    evidence: `${p.package.name}@${version} (${p.package.ecosystem})`,
+                    mapped_by: fails ? mapped.mapped_by : 'policy report only', unmapped: fails && !mapped.sec_ids.length,
+                    advisory: {
+                        id, aliases: sortUnique([...(g.aliases ?? []), ...ids].filter((x) => x !== id)), package: p.package.name,
+                        ecosystem: p.package.ecosystem, version, severity_class: cls, cvss_score: score > 0 ? score : null,
+                        fixed_versions: fixed, published, ...d, mapped_sec_ids: mapped.sec_ids,
+                    },
+                });
+            }
+        }
+    }
+    return { hits, scanned: [...scanned].sort(cmp) };
+}

package/dist/scanners/run-targets.js

@@ -0,0 +1,34 @@
+// Development tool. Runs every adapter twice on each target folder, writing into two
+// temporary output folders (never into the target), and prints per tool: status, version, hits, mapped and unmapped
+// hits, files scanned and tracked, run time of each run, and whether both runs gave the same bytes.
+// Usage: node src/scanners/run-targets.ts [--allow-network] [--history] DIR...
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { basename, join, resolve } from 'node:path';
+import { discover } from '../discover/discover.js';
+import { sha256 } from './common.js';
+import { runScanners } from './index.js';
+const args = process.argv.slice(2);
+const allowNetwork = args.includes('--allow-network');
+const history = args.includes('--history');
+const rows = [];
+for (const target of args.filter((a) => !a.startsWith('--')).map((a) => resolve(a))) {
+    const stack = discover(target);
+    const outs = [0, 1].map(() => mkdtempSync(join(tmpdir(), 'hullproof-runs-')));
+    const runs = outs.map((out) => runScanners({ repo: target, out, stack, allowNetwork, edition: 'pro', history }));
+    for (const [i, s] of runs[0].entries()) {
+        const files = ['json', 'meta.json', 'detail.json'].map((ext) => `scan/${s.meta.tool}.${ext}`);
+        const same = files.every((f) => sha256(readFileSync(join(outs[0], f))) === sha256(readFileSync(join(outs[1], f))));
+        const hits = s.detail.hits;
+        rows.push({
+            target: basename(target), tool: s.meta.tool, status: s.meta.status, version: s.meta.version, reason: s.detail.not_run_reason,
+            hits: s.hits.hits.length, detail_hits: hits.length, mapped: hits.filter((h) => h.sec_ids.length).length,
+            unmapped: hits.filter((h) => h.unmapped).length, report_only: hits.filter((h) => h.mapped_by === 'policy report only').length,
+            files: s.detail.files, tool_errors: s.detail.tool_errors.length, seconds: [runs[0][i].seconds, runs[1][i].seconds],
+            identical: same, sha256: sha256(readFileSync(join(outs[0], files[0]))).slice(0, 12),
+        });
+    }
+    for (const o of outs)
+        rmSync(o, { recursive: true, force: true });
+}
+console.log(JSON.stringify(rows, null, 2));

package/dist/scanners/secret-noise.js

@@ -0,0 +1,89 @@
+// Secret hits that are almost surely not a live secret: a placeholder or a published example value, or a shapeless rule
+// (a generic key, a curl header, a bare JWT) matching in documentation, fixture data, an example env file, a lockfile,
+// vendored or generated code, or a migration. The hit is kept and reported as an example value; it does not hold the gate.
+//
+// What stays held, on purpose: a rule with a vendor shape (AWS, GitHub, Stripe, a private key block and so on) anywhere
+// except where its value is a placeholder, and every rule in source and config files and in test source files.
+// Rules with no vendor shape: they match on a variable name or a loose pattern, so a path says more than the value does.
+export const WEAK_RULES = new Set(['generic-api-key', 'curl-auth-header', 'curl-auth-user', 'jwt', 'hullproof-url-with-password']);
+const FOLDERS = (s) => new Set(s.split(' '));
+const DOC_FOLDERS = FOLDERS('docs doc documentation api-docs apidocs');
+const DOC_EXT = /\.(?:md|mdx|rst|adoc|txt|html?|ya?ml|json)$/i;
+const DOC_FILE = /\.(?:md|mdx|rst|adoc)$|^(?:readme|changelog|changes|history|contributing|license|notice)(?:\.\w+)?$/i;
+const FIXTURE_FOLDERS = FOLDERS('fixtures fixture __fixtures__ testdata test-data test_data testfixtures __snapshots__ cassettes');
+const TEST_FOLDERS = FOLDERS('test tests __tests__ spec specs e2e cypress');
+const DATA_EXT = /\.(?:json|ya?ml|txt|csv|xml|toml|snap|har|md)$/i;
+const MIGRATION_FOLDERS = FOLDERS('migrations migrate alembic');
+const VENDOR_FOLDERS = FOLDERS('vendor vendors third_party third-party node_modules bower_components site-packages pods plugins webjars');
+const LOCKFILES = /^(?:package-lock\.json|npm-shrinkwrap\.json|yarn\.lock|pnpm-lock\.yaml|bun\.lockb?|cargo\.lock|composer\.lock|poetry\.lock|pipfile\.lock|gemfile\.lock|go\.sum|packages\.lock\.json|gradle\.lockfile|mix\.lock|pubspec\.lock|podfile\.lock|flake\.lock)$/i;
+const EXAMPLE_FILE = /(?:^|\/)\.env\.(?:example|sample|template|dist|tpl)$|\.(?:example|sample|template|dist|tpl)$|\.(?:example|sample|template)\.\w+$/i;
+const GENERATED = /(?:\.min\.(?:js|css)|\.pb\.go|_pb2\.py|\.g\.dart|\.generated\.\w+|_generated\.\w+|\.snap)$/i;
+// Why a repository relative path makes a shapeless rule's hit an example value, or null.
+export function pathNoise(file) {
+    const parts = file.split('/');
+    const name = parts.pop() ?? '';
+    const dirs = parts.map((p) => p.toLowerCase());
+    const has = (set) => parts.find((p) => set.has(p.toLowerCase()));
+    if (LOCKFILES.test(name))
+        return 'lockfile';
+    if (DOC_FILE.test(name))
+        return 'documentation file';
+    if (EXAMPLE_FILE.test(file))
+        return 'example or sample file';
+    const doc = has(DOC_FOLDERS);
+    if (doc && DOC_EXT.test(name))
+        return `documentation folder ${doc}/`;
+    const fx = has(FIXTURE_FOLDERS);
+    if (fx)
+        return `fixture data folder ${fx}/`;
+    if (/\.(?:fixtures?)\.\w+$/i.test(name))
+        return 'fixture file';
+    const t = has(TEST_FOLDERS);
+    if (t && DATA_EXT.test(name))
+        return `test data in ${t}/`;
+    const mg = has(MIGRATION_FOLDERS) ?? (dirs.includes('db') && dirs.includes('migrate') ? 'migrate' : undefined);
+    if (mg)
+        return `migration folder ${mg}/`;
+    const vd = has(VENDOR_FOLDERS);
+    if (vd)
+        return `vendored code in ${vd}/`;
+    if (GENERATED.test(name) || dirs.includes('__generated__'))
+        return 'generated file';
+    return null;
+}
+// Values that are published examples, never a credential.
+const KNOWN_EXAMPLES = [
+    'AKIAIOSFODNN7EXAMPLE', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', 'ASIAIOSFODNN7EXAMPLE',
+    // The jwt.io sample token (John Doe, iat 1516239022).
+    'eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ',
+];
+const PLACEHOLDER = /^(?:(.)\1{5,}|(?:0123456789|1234567890|12345678|abcdef|abcdefgh)[\w-]*|.*(?:example|your[-_.]|changeme|change[-_]me|placeholder|dummy|sample|redacted|fake|insert|replace|todo)[\w-]*|x{4,}[\w-]*|\*{4,})$/i;
+const GUID_SEQ = /^(?:12345678-1234-1234-1234-1234567890ab|12345678-1234-1234-1234-123456789abc|00000000-0000-0000-0000-000000000000)$/i;
+// The values on a line: the text inside quotes (after a "name:" or "name=" prefix if it has one), or else what follows a : or =.
+// A line with no such value is never judged a placeholder, so a short real password beside a variable named EXAMPLE_PASSWORD still holds.
+function valuesOf(line) {
+    const out = [];
+    for (const m of line.matchAll(/(["'`])((?:(?!\1).){4,})\1/g))
+        out.push(m[2].replace(/^.*?[:=]\s*(?=\S)/, (pre) => (/^[\w.-]+\s*[:=]\s*$/.test(pre) ? '' : pre)).trim());
+    if (!out.length)
+        for (const m of line.matchAll(/[:=]\s*([^\s"'`,;]{4,})/g))
+            out.push(m[1]);
+    return out;
+}
+// A line holds a placeholder when it has values and every one is placeholder shaped or a known example. The secret itself is
+// redacted by the scanner, so the line is the only place the value can be judged.
+export function valueNoise(line) {
+    if (KNOWN_EXAMPLES.some((k) => line.includes(k)))
+        return 'published example value';
+    const vs = valuesOf(line);
+    if (!vs.length)
+        return null;
+    return vs.every((v) => PLACEHOLDER.test(v) || GUID_SEQ.test(v) || /^(?:<[\w\s.-]+>|\$\{[^}]+\}|\{\{[^}]+\}\}|\$\w+)$/.test(v)) ? 'placeholder value' : null;
+}
+// The reason a Gitleaks hit is an example value, or null when it must hold. `line` is the source line, or null when it is not known.
+export function secretNoise(rule, file, line) {
+    const v = line === null ? null : valueNoise(line);
+    if (v)
+        return v;
+    return WEAK_RULES.has(rule) ? pathNoise(file) : null;
+}