hullproof 0.0.0-stage → 0.5.0

package/src/scanners/index.ts

@@ -0,0 +1,103 @@
+// Runs the scanner adapters and writes, per tool, scan/<tool>.json (HitsFile), scan/<tool>.meta.json (ToolMeta) and
+// scan/<tool>.detail.json (DetailFile). Nothing is installed. A missing tool, a tool that needs the network when the
+// network is not allowed, a tool with nothing to read, and a tool that fails are each recorded with the reason.
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { Hit, HitsFile, ScannerId, ToolMeta } from '../types/index.ts';
+import { cmp, stableJson } from '../util.ts';
+import { readHeadCommit } from '../discover/git.ts';
+import { loadControls, notInControls } from '../mapping/controls.ts';
+import { detectVersion, exec, readTracked, RULES_DIR, toolEnv, versionTested } from './common.ts';
+import { adapter as semgrep } from './semgrep.ts';
+import { adapter as gitleaks } from './gitleaks.ts';
+import { adapter as osv } from './osv-scanner.ts';
+import { adapter as trivy } from './trivy.ts';
+import type { Adapter, AdapterResult, DetailFile, DetailHit, Env, ScanOptions } from './types.ts';
+
+export const ADAPTERS: Adapter[] = [semgrep, gitleaks, osv, trivy];
+
+export interface ScanOutput { hits: HitsFile; meta: ToolMeta; detail: DetailFile; seconds: number }
+
+const hitKey = (h: Hit): string[] => [h.file, String(h.line).padStart(9, '0'), h.tool, h.rule_id];
+const byKey = (a: string[], b: string[]): number => { for (let i = 0; i < a.length; i++) { const c = cmp(a[i], b[i]); if (c) return c; } return 0; };
+
+export function toContractHits(details: DetailHit[]): Hit[] {
+  const seen = new Map<string, Hit>();
+  for (const d of details) {
+    const h: Hit = { tool: d.tool, rule_id: d.rule_id, file: d.file, line: d.line, sec_ids: d.sec_ids, severity_hint: d.severity_hint, message_key: d.message_key, ...(d.noise ? { noise: d.noise } : {}) };
+    const k = stableJson(h);
+    if (!seen.has(k)) seen.set(k, h);
+  }
+  return [...seen.values()].sort((a, b) => byKey(hitKey(a), hitKey(b)) || cmp(stableJson(a), stableJson(b)));
+}
+
+const sortDetails = (xs: DetailHit[]): DetailHit[] =>
+  xs.map((d) => [d, stableJson(d)] as const).sort((a, b) => byKey(hitKey(a[0]), hitKey(b[0])) || cmp(a[1], b[1])).map((x) => x[0]);
+
+export function runScanner(a: Adapter, opts: ScanOptions): ScanOutput {
+  const started = Date.now();
+  const o: ScanOptions = { ...opts, rulesDir: opts.rulesDir ?? RULES_DIR };
+  const env0 = toolEnv(o.env ?? process.env, o.allowNetwork);
+  const controls = loadControls(o.controlsPath);
+  const tracked = readTracked(o.repo);
+  const version = detectVersion(a.versionCommand, env0);
+  let status: ToolMeta['status'] = 'ran';
+  let reason: string | null = null;
+  let res: AdapterResult | null = null;
+
+  if (!version) { status = 'skipped: not installed'; reason = `${a.tool} is not installed on this machine (not found on PATH). The engine does not install tools.`; }
+  else if (a.needsNetwork && !o.allowNetwork) { status = 'skipped: network not allowed'; reason = `${a.tool} sends package names and versions to osv.dev and needs its advisory data. Run with --allow-network to include it.`; }
+  else if (!a.applies(o.stack)) { status = 'skipped: not applicable'; reason = a.notApplicableReason(o); }
+  else {
+    const scratch = mkdtempSync(join(tmpdir(), 'hullproof-scan-'));
+    const env: Env = {
+      version, tracked, rulesDir: o.rulesDir!, scratch,
+      run: (argv) => exec(argv, o.repo, env0),
+    };
+    try { res = a.execute(o, env); if (res && o.skipFiles?.size) res = { ...res, hits: res.hits.filter((h) => !o.skipFiles!.has(h.file)) }; }
+    catch (e) { status = 'failed'; reason = `${a.tool} did not finish: ${(e as Error).message}`; }
+    finally { rmSync(scratch, { recursive: true, force: true }); }
+  }
+
+  const details = sortDetails(res?.hits ?? []);
+  const scanned = res?.scanned ?? null;
+  const scannedSet = new Set(scanned ?? []);
+  const files: DetailFile['files'] = {
+    scanned: scanned ? scanned.length : null,
+    tracked: tracked ? tracked.size : null,
+    scanned_not_tracked: scanned && tracked ? scanned.filter((f) => !tracked.has(f)).length : null,
+    tracked_not_scanned: scanned && tracked ? [...tracked].filter((f) => !scannedSet.has(f)).length : null,
+  };
+  const ran = status === 'ran';
+  const meta: ToolMeta = {
+    schema_version: '1.0.0', tool: a.id, status, version, tested_version_range: a.testedVersions,
+    version_tested: versionTested(version, a.testedVersions), argv: res?.commands[0] ?? [], config_sha256: res?.configSha256 ?? null,
+    commit: readHeadCommit(o.repo), files_scanned: files.scanned, files_tracked: files.tracked, exit_code: res?.exitCode ?? null,
+    db_date: res?.dbDate ?? null, not_assessed_sec_ids: ran ? [] : a.possibleSecIds(o),
+  };
+  const detail: DetailFile = {
+    schema_version: '0.1.0', tool: a.id, status, not_run_reason: reason, tool_version: version, commands: res?.commands ?? [],
+    files, as_of: res?.asOf ?? null, policy: res?.policy ?? null,
+    sec_ids_not_in_controls: [...new Set(notInControls(details.flatMap((d) => [...d.sec_ids, ...(d.advisory?.mapped_sec_ids ?? [])]), controls))].sort(cmp),
+    tool_errors: (res?.toolErrors ?? []).sort((x, y) => cmp(stableJson(x), stableJson(y))),
+    hits: details,
+  };
+  return { hits: { schema_version: '1.0.0', tool: a.id, hits: toContractHits(details) }, meta, detail, seconds: (Date.now() - started) / 1000 };
+}
+
+export function writeScan(outDir: string, s: ScanOutput): void {
+  const dir = join(outDir, 'scan');
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(join(dir, `${s.meta.tool}.json`), stableJson(s.hits));
+  writeFileSync(join(dir, `${s.meta.tool}.meta.json`), stableJson(s.meta));
+  writeFileSync(join(dir, `${s.meta.tool}.detail.json`), stableJson(s.detail));
+}
+
+// Runs every adapter (or the ones named) and writes their files. Run times go to scan/run.json, outside the contract.
+export function runScanners(opts: ScanOptions, tools?: ScannerId[]): ScanOutput[] {
+  const out = ADAPTERS.filter((a) => !tools || tools.includes(a.id)).map((a) => runScanner(a, opts));
+  for (const s of out) writeScan(opts.out, s);
+  writeFileSync(join(opts.out, 'scan', 'run.json'), JSON.stringify({ seconds: Object.fromEntries(out.map((s) => [s.meta.tool, s.seconds])) }, null, 2) + '\n');
+  return out;
+}

package/src/scanners/osv-scanner.ts

@@ -0,0 +1,90 @@
+// OSV Scanner adapter. It sends package names and versions to osv.dev, so it runs only with --allow-network; without it
+// the tool is NOT RUN with that reason. One hit per advisory group per package, with the advisory id, package, version,
+// severity class, fixed versions and the dependency policy decision. The result is reproducible only for the same
+// advisory data date, recorded as db_date.
+import { cmp, sortUnique } from '../util.ts';
+import { relPath } from './common.ts';
+import { inEdition, loadControls } from '../mapping/controls.ts';
+import { mapHit, tableSecIds } from '../mapping/map.ts';
+import { classFromScore, classFromWord, decide, DEFAULT_POLICY, type SeverityClass } from '../mapping/dependency-policy.ts';
+import type { Adapter, DetailHit, ScanOptions } from './types.ts';
+
+interface Osv {
+  id: string; published?: string; modified?: string;
+  database_specific?: { severity?: string };
+  affected?: { package?: { name?: string; ecosystem?: string }; ranges?: { events?: { fixed?: string }[] }[] }[];
+}
+interface OsvJson {
+  results?: { source: { path: string }; packages?: { package: { name: string; version?: string; commit?: string; ecosystem: string }; vulnerabilities?: Osv[]; groups?: { ids: string[]; aliases?: string[]; max_severity?: string }[] }[] }[];
+}
+
+const RANK: Record<SeverityClass, number> = { UNKNOWN: 0, LOW: 1, MEDIUM: 2, HIGH: 3, CRITICAL: 4 };
+export const today = (): string => new Date().toISOString().slice(0, 10);
+
+export const adapter: Adapter = {
+  id: 'osv-scanner',
+  tool: 'osv-scanner',
+  versionCommand: ['osv-scanner', '--version'],
+  testedVersions: '2.6.x',
+  needsNetwork: true,
+  applies: (stack) => stack.package_managers.length > 0,
+  notApplicableReason: (o) => (adapter.applies(o.stack) ? null : 'No package manifest or lockfile was found, so there are no dependencies to check.'),
+  configFiles: () => [],
+  command: () => ['osv-scanner', 'scan', 'source', '--format', 'json', '--recursive', '.'],
+  parse: (stdout, ctx) => parseOsv(stdout, ctx as ScanOptions, (ctx as ScanOptions).asOf ?? today()).hits,
+  possibleSecIds: (o) => sortUnique(inEdition(tableSecIds('osv-scanner'), o.edition, loadControls(o.controlsPath))),
+
+  execute(o, env) {
+    const asOf = o.asOf ?? today();
+    const argv = adapter.command(o);
+    const r = env.run(argv);
+    // 0: no advisories, 1: advisories found, 128: no packages found. Anything else is a failure.
+    if (r.status !== 0 && r.status !== 1 && r.status !== 128) throw new Error(`osv-scanner exited with ${r.status ?? r.error}`);
+    const out = r.status === 128 ? { hits: [], scanned: [] } : parseOsv(r.stdout, o, asOf);
+    return { hits: out.hits, commands: [argv], configSha256: null, scanned: out.scanned, exitCode: r.status, toolErrors: [], dbDate: asOf, asOf, policy: o.policy ?? DEFAULT_POLICY };
+  },
+};
+
+export function parseOsv(stdout: string, o: ScanOptions, asOf: string): { hits: DetailHit[]; scanned: string[] } {
+  const controls = loadControls(o.controlsPath);
+  const policy = o.policy ?? DEFAULT_POLICY;
+  const j = JSON.parse(stdout) as OsvJson;
+  const hits: DetailHit[] = [];
+  const scanned = new Set<string>();
+  for (const res of j.results ?? []) {
+    const file = relPath(res.source.path, o.repo);
+    scanned.add(file);
+    for (const p of res.packages ?? []) {
+      for (const g of p.groups ?? []) {
+        const ids = [...g.ids].sort(cmp);
+        const id = ids.find((x) => x.startsWith('GHSA-')) ?? ids[0];
+        const vulns = (p.vulnerabilities ?? []).filter((v) => ids.includes(v.id));
+        const score = Number.parseFloat(g.max_severity ?? '');
+        const cls: SeverityClass = score > 0 ? classFromScore(score)
+          : vulns.map((v) => classFromWord(v.database_specific?.severity)).reduce((a, b) => (RANK[b] > RANK[a] ? b : a), 'UNKNOWN');
+        const dates = vulns.map((v) => v.published?.slice(0, 10)).filter((d): d is string => !!d).sort(cmp);
+        const published = dates[0] ?? null;
+        const fixed = sortUnique(vulns.flatMap((v) => (v.affected ?? [])
+          .filter((a) => a.package?.name === p.package.name)
+          .flatMap((a) => (a.ranges ?? []).flatMap((rg) => (rg.events ?? []).map((e) => e.fixed).filter((x): x is string => !!x)))));
+        const d = decide(cls, published, asOf, policy);
+        const version = p.package.version || p.package.commit || '';
+        const mapped = mapHit({ tool: 'osv-scanner', rule_id: id, file }, o.edition, controls);
+        const fails = d.decision === 'fail';
+        const modified = vulns.map((v) => v.modified ?? '').sort(cmp).pop() || 'unknown';
+        hits.push({
+          tool: 'osv-scanner', rule_id: id, file, line: 0, end_line: 0, sec_ids: fails ? mapped.sec_ids : [],
+          severity_hint: cls, message_key: `osv-scanner:${id}`, rule_version: `advisory modified ${modified}`,
+          evidence: `${p.package.name}@${version} (${p.package.ecosystem})`,
+          mapped_by: fails ? mapped.mapped_by : 'policy report only', unmapped: fails && !mapped.sec_ids.length,
+          advisory: {
+            id, aliases: sortUnique([...(g.aliases ?? []), ...ids].filter((x) => x !== id)), package: p.package.name,
+            ecosystem: p.package.ecosystem, version, severity_class: cls, cvss_score: score > 0 ? score : null,
+            fixed_versions: fixed, published, ...d, mapped_sec_ids: mapped.sec_ids,
+          },
+        });
+      }
+    }
+  }
+  return { hits, scanned: [...scanned].sort(cmp) };
+}

package/src/scanners/run-targets.ts

@@ -0,0 +1,35 @@
+// Development tool. Runs every adapter twice on each target folder, writing into two
+// temporary output folders (never into the target), and prints per tool: status, version, hits, mapped and unmapped
+// hits, files scanned and tracked, run time of each run, and whether both runs gave the same bytes.
+// Usage: node src/scanners/run-targets.ts [--allow-network] [--history] DIR...
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { basename, join, resolve } from 'node:path';
+import { discover } from '../discover/discover.ts';
+import { sha256 } from './common.ts';
+import { runScanners } from './index.ts';
+
+const args = process.argv.slice(2);
+const allowNetwork = args.includes('--allow-network');
+const history = args.includes('--history');
+const rows: Record<string, unknown>[] = [];
+
+for (const target of args.filter((a) => !a.startsWith('--')).map((a) => resolve(a))) {
+  const stack = discover(target);
+  const outs = [0, 1].map(() => mkdtempSync(join(tmpdir(), 'hullproof-runs-')));
+  const runs = outs.map((out) => runScanners({ repo: target, out, stack, allowNetwork, edition: 'pro', history }));
+  for (const [i, s] of runs[0].entries()) {
+    const files = ['json', 'meta.json', 'detail.json'].map((ext) => `scan/${s.meta.tool}.${ext}`);
+    const same = files.every((f) => sha256(readFileSync(join(outs[0], f))) === sha256(readFileSync(join(outs[1], f))));
+    const hits = s.detail.hits;
+    rows.push({
+      target: basename(target), tool: s.meta.tool, status: s.meta.status, version: s.meta.version, reason: s.detail.not_run_reason,
+      hits: s.hits.hits.length, detail_hits: hits.length, mapped: hits.filter((h) => h.sec_ids.length).length,
+      unmapped: hits.filter((h) => h.unmapped).length, report_only: hits.filter((h) => h.mapped_by === 'policy report only').length,
+      files: s.detail.files, tool_errors: s.detail.tool_errors.length, seconds: [runs[0][i].seconds, runs[1][i].seconds],
+      identical: same, sha256: sha256(readFileSync(join(outs[0], files[0]))).slice(0, 12),
+    });
+  }
+  for (const o of outs) rmSync(o, { recursive: true, force: true });
+}
+console.log(JSON.stringify(rows, null, 2));

package/src/scanners/secret-noise.ts

@@ -0,0 +1,80 @@
+// Secret hits that are almost surely not a live secret: a placeholder or a published example value, or a shapeless rule
+// (a generic key, a curl header, a bare JWT) matching in documentation, fixture data, an example env file, a lockfile,
+// vendored or generated code, or a migration. The hit is kept and reported as an example value; it does not hold the gate.
+//
+// What stays held, on purpose: a rule with a vendor shape (AWS, GitHub, Stripe, a private key block and so on) anywhere
+// except where its value is a placeholder, and every rule in source and config files and in test source files.
+
+// Rules with no vendor shape: they match on a variable name or a loose pattern, so a path says more than the value does.
+export const WEAK_RULES = new Set(['generic-api-key', 'curl-auth-header', 'curl-auth-user', 'jwt', 'hullproof-url-with-password']);
+
+const FOLDERS = (s: string): Set<string> => new Set(s.split(' '));
+const DOC_FOLDERS = FOLDERS('docs doc documentation api-docs apidocs');
+const DOC_EXT = /\.(?:md|mdx|rst|adoc|txt|html?|ya?ml|json)$/i;
+const DOC_FILE = /\.(?:md|mdx|rst|adoc)$|^(?:readme|changelog|changes|history|contributing|license|notice)(?:\.\w+)?$/i;
+const FIXTURE_FOLDERS = FOLDERS('fixtures fixture __fixtures__ testdata test-data test_data testfixtures __snapshots__ cassettes');
+const TEST_FOLDERS = FOLDERS('test tests __tests__ spec specs e2e cypress');
+const DATA_EXT = /\.(?:json|ya?ml|txt|csv|xml|toml|snap|har|md)$/i;
+const MIGRATION_FOLDERS = FOLDERS('migrations migrate alembic');
+const VENDOR_FOLDERS = FOLDERS('vendor vendors third_party third-party node_modules bower_components site-packages pods plugins webjars');
+const LOCKFILES = /^(?:package-lock\.json|npm-shrinkwrap\.json|yarn\.lock|pnpm-lock\.yaml|bun\.lockb?|cargo\.lock|composer\.lock|poetry\.lock|pipfile\.lock|gemfile\.lock|go\.sum|packages\.lock\.json|gradle\.lockfile|mix\.lock|pubspec\.lock|podfile\.lock|flake\.lock)$/i;
+const EXAMPLE_FILE = /(?:^|\/)\.env\.(?:example|sample|template|dist|tpl)$|\.(?:example|sample|template|dist|tpl)$|\.(?:example|sample|template)\.\w+$/i;
+const GENERATED = /(?:\.min\.(?:js|css)|\.pb\.go|_pb2\.py|\.g\.dart|\.generated\.\w+|_generated\.\w+|\.snap)$/i;
+
+// Why a repository relative path makes a shapeless rule's hit an example value, or null.
+export function pathNoise(file: string): string | null {
+  const parts = file.split('/');
+  const name = parts.pop() ?? '';
+  const dirs = parts.map((p) => p.toLowerCase());
+  const has = (set: Set<string>): string | undefined => parts.find((p) => set.has(p.toLowerCase()));
+  if (LOCKFILES.test(name)) return 'lockfile';
+  if (DOC_FILE.test(name)) return 'documentation file';
+  if (EXAMPLE_FILE.test(file)) return 'example or sample file';
+  const doc = has(DOC_FOLDERS);
+  if (doc && DOC_EXT.test(name)) return `documentation folder ${doc}/`;
+  const fx = has(FIXTURE_FOLDERS);
+  if (fx) return `fixture data folder ${fx}/`;
+  if (/\.(?:fixtures?)\.\w+$/i.test(name)) return 'fixture file';
+  const t = has(TEST_FOLDERS);
+  if (t && DATA_EXT.test(name)) return `test data in ${t}/`;
+  const mg = has(MIGRATION_FOLDERS) ?? (dirs.includes('db') && dirs.includes('migrate') ? 'migrate' : undefined);
+  if (mg) return `migration folder ${mg}/`;
+  const vd = has(VENDOR_FOLDERS);
+  if (vd) return `vendored code in ${vd}/`;
+  if (GENERATED.test(name) || dirs.includes('__generated__')) return 'generated file';
+  return null;
+}
+
+// Values that are published examples, never a credential.
+const KNOWN_EXAMPLES = [
+  'AKIAIOSFODNN7EXAMPLE', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', 'ASIAIOSFODNN7EXAMPLE',
+  // The jwt.io sample token (John Doe, iat 1516239022).
+  'eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ',
+];
+const PLACEHOLDER = /^(?:(.)\1{5,}|(?:0123456789|1234567890|12345678|abcdef|abcdefgh)[\w-]*|.*(?:example|your[-_.]|changeme|change[-_]me|placeholder|dummy|sample|redacted|fake|insert|replace|todo)[\w-]*|x{4,}[\w-]*|\*{4,})$/i;
+const GUID_SEQ = /^(?:12345678-1234-1234-1234-1234567890ab|12345678-1234-1234-1234-123456789abc|00000000-0000-0000-0000-000000000000)$/i;
+
+// The values on a line: the text inside quotes (after a "name:" or "name=" prefix if it has one), or else what follows a : or =.
+// A line with no such value is never judged a placeholder, so a short real password beside a variable named EXAMPLE_PASSWORD still holds.
+function valuesOf(line: string): string[] {
+  const out: string[] = [];
+  for (const m of line.matchAll(/(["'`])((?:(?!\1).){4,})\1/g)) out.push(m[2].replace(/^.*?[:=]\s*(?=\S)/, (pre) => (/^[\w.-]+\s*[:=]\s*$/.test(pre) ? '' : pre)).trim());
+  if (!out.length) for (const m of line.matchAll(/[:=]\s*([^\s"'`,;]{4,})/g)) out.push(m[1]);
+  return out;
+}
+
+// A line holds a placeholder when it has values and every one is placeholder shaped or a known example. The secret itself is
+// redacted by the scanner, so the line is the only place the value can be judged.
+export function valueNoise(line: string): string | null {
+  if (KNOWN_EXAMPLES.some((k) => line.includes(k))) return 'published example value';
+  const vs = valuesOf(line);
+  if (!vs.length) return null;
+  return vs.every((v) => PLACEHOLDER.test(v) || GUID_SEQ.test(v) || /^(?:<[\w\s.-]+>|\$\{[^}]+\}|\{\{[^}]+\}\}|\$\w+)$/.test(v)) ? 'placeholder value' : null;
+}
+
+// The reason a Gitleaks hit is an example value, or null when it must hold. `line` is the source line, or null when it is not known.
+export function secretNoise(rule: string, file: string, line: string | null): string | null {
+  const v = line === null ? null : valueNoise(line);
+  if (v) return v;
+  return WEAK_RULES.has(rule) ? pathNoise(file) : null;
+}

package/src/scanners/semgrep.ts

@@ -0,0 +1,139 @@
+// Semgrep adapter. Runs the Hullproof rules in rules/semgrep/, plus an optional external rule pack folder (for example a rule pack of your own),
+// on the files git tracks, named one by one, with --no-git-ignore so Semgrep itself never calls git and the repository's
+// own ignore files cannot hide a tracked file.
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync } from 'node:fs';
+import { basename, join } from 'node:path';
+import { cmp, sortUnique } from '../util.ts';
+import { dirSha256, fileSha256, isRegularFile, relPath, sha256, sourceLine } from './common.ts';
+import { inEdition, loadControls } from '../mapping/controls.ts';
+import { mapHit } from '../mapping/map.ts';
+import type { Adapter, DetailHit, ScanOptions, ToolError } from './types.ts';
+
+export interface RuleInfo { id: string; file: string; sec_ids: string[]; version: string }
+
+const yamlIn = (dir: string): string[] => readdirSync(dir).filter((n) => /\.ya?ml$/.test(n)).sort(cmp);
+
+// The rule files Semgrep runs, by file name: the engine's folder, then the pack. A pack file with the same name as an
+// engine file replaces it, so a pack may be a whole kit rules folder (identical files change nothing) or only the
+// files it adds. Throws when the pack folder is missing or holds no rule file.
+export function ruleFiles(dir: string, pack: string | null = null): string[] {
+  const byName = new Map<string, string>(yamlIn(dir).map((n) => [n, join(dir, n)]));
+  if (pack) {
+    if (!existsSync(pack)) throw new Error(`the rule pack folder ${basename(pack)} does not exist`);
+    const names = yamlIn(pack);
+    if (!names.length) throw new Error(`the rule pack folder ${basename(pack)} holds no .yaml rule file`);
+    for (const n of names) byName.set(n, join(pack, n));
+  }
+  return [...byName.keys()].sort(cmp).map((n) => byName.get(n)!);
+}
+
+// Reads rule ids and metadata.sec_ids from the rule files. The kit writes `- id: x` and `sec_ids: [A, B]` on one line
+// each; test/scanners.test.ts checks every rule has both.
+export function readRules(dir: string, pack: string | null = null): RuleInfo[] {
+  const out: RuleInfo[] = [];
+  for (const path of ruleFiles(dir, pack)) {
+    const f = basename(path);
+    const version = 'sha256:' + fileSha256(path).slice(0, 16);
+    let cur: RuleInfo | null = null;
+    for (const line of readFileSync(path, 'utf8').split('\n')) {
+      const id = /^\s*-\s+id:\s*([\w.-]+)\s*$/.exec(line)?.[1];
+      if (id) { cur = { id, file: f, sec_ids: [], version }; out.push(cur); continue; }
+      const ids = /^\s*sec_ids:\s*\[(.*)\]\s*$/.exec(line)?.[1];
+      if (ids !== undefined && cur) cur.sec_ids = sortUnique(ids.split(',').map((s) => s.trim()).filter(Boolean));
+    }
+  }
+  return out;
+}
+
+const TIMEOUT_SECONDS = '30';
+// Minified or generated files (a line longer than this) make Semgrep time out at random, which breaks byte identical
+// output. They are left out by this fixed rule and listed as tool errors of type "skipped: minified".
+const MAX_LINE = 1000;
+const minified = (p: string): boolean => {
+  try { return readFileSync(p, 'utf8').split('\n').some((l) => l.length > MAX_LINE); } catch { return false; }
+};
+const MAX_TARGET_BYTES = '1000000';
+
+interface SemgrepJson {
+  version: string;
+  results: { check_id: string; path: string; start: { line: number }; end: { line: number }; extra: { severity?: string; metadata?: { sec_ids?: string[] } } }[];
+  errors: { type?: unknown; path?: string; rule_id?: string }[];
+  paths: { scanned: string[] };
+}
+
+export const adapter: Adapter = {
+  id: 'semgrep',
+  tool: 'semgrep',
+  versionCommand: ['semgrep', '--version'],
+  testedVersions: '1.176.x',
+  needsNetwork: false,
+  applies: (stack) => stack.languages.length > 0 || stack.ci.length > 0,
+  notApplicableReason: (o) => (adapter.applies(o.stack) ? null : 'No source language or CI workflow was found, so there is nothing for the Semgrep rules to read.'),
+  configFiles: (ctx) => [join((ctx as ScanOptions).rulesDir ?? '', 'semgrep')],
+  command: (ctx) => ['semgrep', 'scan', '--config', join((ctx as ScanOptions).rulesDir ?? '', 'semgrep'), '--json', '--quiet',
+    '--metrics=off', '--disable-version-check', '--no-git-ignore', '--timeout', TIMEOUT_SECONDS, '--timeout-threshold', '3',
+    '--max-target-bytes', MAX_TARGET_BYTES],
+  parse: (stdout, ctx) => parseSemgrep(stdout, ctx as ScanOptions, readRules(join((ctx as ScanOptions).rulesDir ?? '', 'semgrep'), (ctx as ScanOptions).rulePack ?? null)).hits,
+  possibleSecIds: (o) => sortUnique(inEdition(readRules(join(o.rulesDir!, 'semgrep'), o.rulePack ?? null).flatMap((r) => r.sec_ids), o.edition, loadControls(o.controlsPath))),
+
+  execute(o, env) {
+    const engineRules = join(env.rulesDir, 'semgrep');
+    const pack = o.rulePack ?? null;
+    // With a pack, the merged rule files go into one folder in the scratch space, so Semgrep reads one --config folder
+    // and the config sha256 covers exactly the rules that ran.
+    let rulesDir = engineRules;
+    if (pack) {
+      const rules = readRules(engineRules, pack);
+      const twice = rules.map((r) => r.id).filter((id, i, a) => a.indexOf(id) !== i);
+      if (twice.length) throw new Error(`rule id ${twice[0]} is in two rule files once the pack is added`);
+      rulesDir = join(env.scratch, 'rules');
+      mkdirSync(rulesDir);
+      for (const f of ruleFiles(engineRules, pack)) copyFileSync(f, join(rulesDir, basename(f)));
+    }
+    // ponytail: every tracked file goes on the command line; split into batches if a repository ever passes ARG_MAX.
+    const listed = o.targets ? new Set(o.targets) : env.tracked;
+    const files = listed ? [...listed].filter((f) => isRegularFile(join(o.repo, f))).sort(cmp) : [];
+    const skipped = files.filter((f) => /\.(js|mjs|cjs|ts|jsx|tsx|json|ya?ml|html|py)$/.test(f) && minified(join(o.repo, f)));
+    const targets = listed ? files.filter((f) => !skipped.includes(f)) : ['.'];
+    const base = adapter.command({ ...o, rulesDir: env.rulesDir } as ScanOptions);
+    const shown = [...base.map((a) => (a === join(env.rulesDir, 'semgrep') ? (pack ? '<engine>/rules/semgrep + <rule-pack>' : '<engine>/rules/semgrep') : a)),
+      listed ? `<${targets.length} ${o.targets ? 'listed' : 'tracked'} files, list sha256 ${sha256(targets.join('\n')).slice(0, 16)}>` : '.'];
+    // No file to read: Semgrep with no target would read the whole folder, so it is not called.
+    if (listed && targets.length === 0) return { hits: [], toolErrors: [], commands: [shown], configSha256: dirSha256(rulesDir), scanned: [], exitCode: 0 };
+    const r = env.run([...base.map((a) => (a === join(env.rulesDir, 'semgrep') ? rulesDir : a)), ...targets]);
+    let parsed: SemgrepJson;
+    try { parsed = JSON.parse(r.stdout) as SemgrepJson; } catch { throw new Error(`semgrep printed no JSON (exit ${r.status})`); }
+    const out = parseSemgrep(r.stdout, o, readRules(rulesDir));
+    return {
+      hits: out.hits, toolErrors: [...out.errors, ...skipped.map((file) => ({ type: 'skipped: minified', file, rule_id: '' }))], commands: [shown], configSha256: dirSha256(rulesDir),
+      scanned: sortUnique(parsed.paths.scanned.map((p) => relPath(p, o.repo))), exitCode: r.status,
+    };
+  },
+};
+
+export function parseSemgrep(stdout: string, o: ScanOptions, rules: RuleInfo[]): { hits: DetailHit[]; errors: ToolError[] } {
+  const j = JSON.parse(stdout) as SemgrepJson;
+  const controls = loadControls(o.controlsPath);
+  // check_id is prefixed with the rules folder written as a dotted path; the rule id is the known id it ends with.
+  const ruleOf = (checkId: string): RuleInfo | undefined =>
+    rules.find((r) => checkId === r.id || checkId.endsWith('.' + r.id));
+  const hits = j.results.map((x): DetailHit => {
+    const rule = ruleOf(x.check_id);
+    const rule_id = rule?.id ?? x.check_id.split('.').pop()!;
+    const file = relPath(x.path, o.repo);
+    const meta = x.extra.metadata?.sec_ids ?? rule?.sec_ids ?? [];
+    const m = mapHit({ tool: 'semgrep', rule_id, file, metadataSecIds: meta }, o.edition, controls);
+    const secretRule = meta.some((id) => id.startsWith('SEC-SECRETS-'));
+    return {
+      tool: 'semgrep', rule_id, file, line: x.start.line, end_line: x.end.line, sec_ids: m.sec_ids,
+      severity_hint: x.extra.severity ?? null, message_key: `semgrep:${rule_id}`, rule_version: rule?.version ?? 'unknown',
+      evidence: secretRule ? null : sourceLine(o.repo, file, x.start.line), mapped_by: m.mapped_by, unmapped: m.sec_ids.length === 0,
+    };
+  });
+  const errors = j.errors.map((e): ToolError => ({
+    type: typeof e.type === 'string' ? e.type : Array.isArray(e.type) ? String(e.type[0]) : 'unknown',
+    file: e.path ? relPath(e.path, o.repo) : '',
+    rule_id: e.rule_id ? (ruleOf(e.rule_id)?.id ?? e.rule_id.split('.').pop()!) : '',
+  }));
+  return { hits, errors };
+}

package/src/scanners/trivy.ts

@@ -0,0 +1,66 @@
+// Trivy adapter: `trivy config` only, so misconfiguration checks on IaC, Kubernetes, Helm and Dockerfiles. No image is
+// pulled. The checks are the ones built into the Trivy binary (--skip-check-update, an empty cache folder), so the rule
+// set is fixed by the Trivy version, and network is blocked unless allowed (Terraform remote modules then do not load).
+import { join } from 'node:path';
+import { cmp, sortUnique } from '../util.ts';
+import { isRegularFile, relPath, sourceLine } from './common.ts';
+import { inEdition, loadControls } from '../mapping/controls.ts';
+import { mapHit, tableSecIds } from '../mapping/map.ts';
+import type { Adapter, DetailHit, ScanOptions } from './types.ts';
+
+// Pinned so a Trivy update cannot add a scanner silently. The list is Trivy 0.75.0's default.
+const MISCONFIG_SCANNERS = 'azure-arm,cloudformation,dockerfile,helm,kubernetes,terraform,terraformplan-json,terraformplan-snapshot,ansible';
+
+interface Misconf { ID: string; Severity: string; Status: string; CauseMetadata?: { StartLine?: number; EndLine?: number } }
+interface TrivyJson { Results?: { Target: string; Class: string; Misconfigurations?: Misconf[] }[] }
+
+export const adapter: Adapter = {
+  id: 'trivy',
+  tool: 'trivy',
+  versionCommand: ['trivy', '--version'],
+  testedVersions: '0.75.x',
+  needsNetwork: false,
+  applies: (stack) => stack.iac.length > 0 || stack.containers.length > 0,
+  notApplicableReason: (o) => (adapter.applies(o.stack) ? null : 'No infrastructure as code, Kubernetes, Helm or container file was found, so Trivy has nothing to check.'),
+  configFiles: () => [],
+  command: () => ['trivy', 'config', '--format', 'json', '--quiet', '--skip-check-update', '--skip-version-check',
+    '--disable-telemetry', '--include-non-failures', '--misconfig-scanners', MISCONFIG_SCANNERS, '--cache-dir', '<empty-dir>', '.'],
+  parse: (stdout, ctx) => parseTrivy(stdout, ctx as ScanOptions, 'unknown').hits,
+  possibleSecIds: (o) => sortUnique(inEdition(tableSecIds('trivy'), o.edition, loadControls(o.controlsPath))),
+
+  execute(o, env) {
+    const argv = adapter.command(o).map((a) => (a === '<empty-dir>' ? env.scratch : a));
+    const shown = adapter.command(o);
+    const r = env.run(argv);
+    if (r.status !== 0) throw new Error(`trivy exited with ${r.status ?? r.error}`);
+    const out = parseTrivy(r.stdout, o, `trivy ${env.version} built in checks`, env.tracked);
+    return { hits: out.hits, commands: [shown], configSha256: null, scanned: out.scanned, exitCode: r.status, toolErrors: [] };
+  },
+};
+
+// Terraform results come per module folder as well as per file; a folder target stands for the tracked .tf files in it.
+export function parseTrivy(stdout: string, o: ScanOptions, ruleVersion: string, tracked: Set<string> | null = null): { hits: DetailHit[]; scanned: string[] } {
+  const controls = loadControls(o.controlsPath);
+  const j = JSON.parse(stdout) as TrivyJson;
+  const hits: DetailHit[] = [];
+  const scanned = new Set<string>();
+  for (const res of j.Results ?? []) {
+    if (res.Class !== 'config') continue;
+    const target = relPath(res.Target, o.repo);
+    if (tracked && !isRegularFile(join(o.repo, target))) {
+      const prefix = target === '.' ? '' : target + '/';
+      for (const f of tracked) if (f.startsWith(prefix) && !f.slice(prefix.length).includes('/') && /\.tf(\.json)?$/.test(f)) scanned.add(f);
+    } else scanned.add(target);
+    for (const m of res.Misconfigurations ?? []) {
+      if (m.Status !== 'FAIL') continue;
+      const line = m.CauseMetadata?.StartLine ?? 0;
+      const mapped = mapHit({ tool: 'trivy', rule_id: m.ID, file: target }, o.edition, controls);
+      hits.push({
+        tool: 'trivy', rule_id: m.ID, file: target, line, end_line: m.CauseMetadata?.EndLine ?? line, sec_ids: mapped.sec_ids,
+        severity_hint: m.Severity ?? null, message_key: `trivy:${m.ID}`, rule_version: ruleVersion,
+        evidence: sourceLine(o.repo, target, line), mapped_by: mapped.mapped_by, unmapped: !mapped.sec_ids.length,
+      });
+    }
+  }
+  return { hits, scanned: [...scanned].sort(cmp) };
+}

package/src/scanners/types.ts

@@ -0,0 +1,98 @@
+// Types local to the scanner adapters. The contract types (Hit, HitsFile, ToolMeta, RunContext, ScannerAdapter) live in
+// src/types/scan.ts and are not changed here. Fields the contract has no room for yet (rule version, evidence, advisory
+// data, the reason a tool did not run) go into scan/<tool>.detail.json, described by DetailFile below.
+import type { Hit, RunContext, ScannerAdapter, ScannerId, ToolMeta } from '../types/index.ts';
+import type { DependencyPolicy, SeverityClass } from '../mapping/dependency-policy.ts';
+
+export interface ScanOptions extends RunContext {
+  rulesDir?: string;            // default: <engine>/rules
+  rulePack?: string | null;     // optional external Semgrep rule pack folder (for example a rule pack of your own); see src/scanners/semgrep.ts
+  controlsPath?: string;        // default: see src/mapping/controls.ts
+  asOf?: string;                // YYYY-MM-DD, the date OSV advisories are judged against. Default: today (UTC).
+  policy?: DependencyPolicy;
+  env?: NodeJS.ProcessEnv;      // base environment (tests use it to hide a tool from PATH)
+  skipFiles?: ReadonlySet<string>; // unchanged kit files in the project (src/discover/kit-files.ts): hits in them are dropped
+  targets?: string[];           // Semgrep only: read just these repository relative files (the pre-commit hook passes the staged files)
+}
+
+export interface SecretDetail {
+  redacted: string;             // the match as Gitleaks printed it with --redact, masked again by us
+  fingerprint: string;          // Gitleaks fingerprint: [commit:]file:rule:line. Holds no secret.
+  source: 'dir' | 'git';
+  commit: string | null;
+  tracked: boolean;             // false: the file is on disk but not tracked by git
+}
+
+export interface AdvisoryDetail {
+  id: string;
+  aliases: string[];
+  package: string;
+  ecosystem: string;
+  version: string;
+  severity_class: SeverityClass;
+  cvss_score: number | null;
+  fixed_versions: string[];
+  published: string | null;     // YYYY-MM-DD
+  deadline_days: number;
+  past_deadline: boolean;
+  decision: 'fail' | 'report';
+  mapped_sec_ids: string[];     // what the hit maps to when it fails; sec_ids is [] when the decision is report
+}
+
+export interface DetailHit extends Hit {
+  rule_version: string;
+  end_line: number;
+  evidence: string | null;      // short, masked; null when it could expose a secret
+  mapped_by: 'rule metadata' | 'controls tests' | 'map rule_id' | 'map default' | 'policy report only' | 'none';
+  unmapped: boolean;
+  secret?: SecretDetail;
+  advisory?: AdvisoryDetail;
+}
+
+export interface ToolError { type: string; file: string; rule_id: string }
+
+export interface DetailFile {
+  schema_version: '0.1.0';
+  tool: ScannerId;
+  status: ToolMeta['status'];
+  not_run_reason: string | null;
+  tool_version: string | null;
+  commands: string[][];         // every command run, absolute paths replaced by <placeholders>
+  files: { scanned: number | null; tracked: number | null; scanned_not_tracked: number | null; tracked_not_scanned: number | null };
+  as_of: string | null;         // OSV only
+  policy: DependencyPolicy | null; // OSV only
+  sec_ids_not_in_controls: string[];
+  tool_errors: ToolError[];
+  hits: DetailHit[];
+}
+
+export interface Exec { argv: string[]; stdout: string; stderr: string; status: number | null; error?: string }
+
+// What an adapter needs on top of the shared ScannerAdapter interface.
+export interface Adapter extends ScannerAdapter {
+  tool: string;                 // executable name
+  // Runs the tool (one or more commands) and returns detail hits. Throws on a tool failure.
+  execute(opts: ScanOptions, env: Env): AdapterResult;
+  possibleSecIds(opts: ScanOptions): string[];
+  notApplicableReason(opts: ScanOptions): string | null;
+}
+
+export interface Env {
+  version: string;
+  tracked: Set<string> | null;
+  rulesDir: string;
+  scratch: string;              // empty temp folder, removed after the run
+  run(argv: string[]): Exec;
+}
+
+export interface AdapterResult {
+  hits: DetailHit[];
+  commands: string[][];
+  configSha256: string | null;
+  scanned: string[] | null;     // repository relative files the tool read, when it says
+  exitCode: number | null;
+  toolErrors: ToolError[];
+  dbDate?: string | null;
+  asOf?: string | null;
+  policy?: DependencyPolicy | null;
+}