hullproof 0.0.0-stage → 0.5.0

package/src/confirm/query.ts

@@ -0,0 +1,126 @@
+// A small declarative query language over codemap.json, and its runner. A model proposes a query; the engine runs it. It has
+// no regular expressions and no code, so a query cannot loop or run anything. See docs/INTERFACES.md, section 13.
+import type { CodeMap, Cond, EntryPoint, Json, LeadStep, Query } from '../types/index.ts';
+
+export const MAX_DEPTH = 8;
+export const MAX_NODES = 60;
+export const WITNESS_SLACK = 2;       // a fact counts when its line is within this many lines of a cited line
+export const TOO_BROAD_MIN = 3;       // a query that matches every entry point of a map this large proves nothing
+const OPS = new Set(['eq', 'ne', 'in', 'contains', 'startsWith', 'exists', 'gt', 'lt']);
+const FIELD = /^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$/;
+const BAD_SEG = new Set(['__proto__', 'constructor', 'prototype']);
+const isObj = (v: unknown): v is Record<string, unknown> => !!v && typeof v === 'object' && !Array.isArray(v);
+const goodField = (f: unknown): f is string => typeof f === 'string' && FIELD.test(f) && !f.split('.').some((s) => BAD_SEG.has(s));
+
+export function parseQuery(v: unknown): { ok: true; query: Query } | { ok: false; reason: string } {
+  if (!isObj(v) || !('where' in v)) return { ok: false, reason: 'a query is {"where": condition}' };
+  let nodes = 0;
+  const cond = (c: unknown, depth: number): Cond | string => {
+    if (depth > MAX_DEPTH) return `nested deeper than ${MAX_DEPTH}`;
+    if (++nodes > MAX_NODES) return `more than ${MAX_NODES} conditions`;
+    if (!isObj(c)) return 'a condition must be an object';
+    for (const k of ['all', 'any'] as const) {
+      if (k in c) {
+        if (!Array.isArray(c[k]) || c[k].length === 0) return `${k} needs a non empty list`;
+        const out: Cond[] = [];
+        for (const x of c[k] as unknown[]) { const r = cond(x, depth + 1); if (typeof r === 'string') return r; out.push(r); }
+        return k === 'all' ? { all: out } : { any: out };
+      }
+    }
+    if ('not' in c) { const r = cond(c.not, depth + 1); return typeof r === 'string' ? r : { not: r }; }
+    for (const k of ['some', 'none'] as const) {
+      if (k in c) {
+        if (!goodField(c[k])) return `${k} needs a field name`;
+        let where: Cond | undefined;
+        if (c.where !== undefined) { const r = cond(c.where, depth + 1); if (typeof r === 'string') return r; where = r; }
+        return k === 'some' ? { some: c.some as string, ...(where ? { where } : {}) } : { none: c.none as string, ...(where ? { where } : {}) };
+      }
+    }
+    if ('field' in c) {
+      if (!goodField(c.field)) return 'field is not a plain dotted name';
+      if (typeof c.op !== 'string' || !OPS.has(c.op)) return `op is one of ${[...OPS].join(', ')}`;
+      const val = c.value as Json | undefined;
+      const scalar = (x: unknown): boolean => x === null || ['string', 'number', 'boolean'].includes(typeof x);
+      if (c.op === 'in' ? !Array.isArray(val) || !val.every(scalar) || val.length > 20 : c.op === 'exists' ? val !== undefined && typeof val !== 'boolean' : !scalar(val)) return `value does not fit op ${c.op}`;
+      if ((c.op === 'gt' || c.op === 'lt') && typeof val !== 'number') return `${c.op} needs a number`;
+      if ((c.op === 'startsWith') && typeof val !== 'string') return 'startsWith needs a string';
+      return { field: c.field, op: c.op as 'eq', ...(val !== undefined ? { value: val } : {}) };
+    }
+    return 'a condition is all, any, not, some, none or field';
+  };
+  const r = cond(v.where, 1);
+  return typeof r === 'string' ? { ok: false, reason: r } : { ok: true, query: { where: r } };
+}
+
+export interface Witness { file: string; line: number }
+interface Ev { ok: boolean; w: Witness[] }
+const NO: Ev = { ok: false, w: [] };
+
+function pick(node: unknown, path: string): unknown {
+  let cur = node;
+  for (const seg of path.split('.')) {
+    if (!isObj(cur) || !Object.hasOwn(cur, seg)) return undefined;
+    cur = cur[seg];
+  }
+  return cur;
+}
+
+function compare(c: Extract<Cond, { field: string }>, node: unknown): boolean {
+  const got = pick(node, c.field);
+  const want = c.value;
+  switch (c.op) {
+    case 'exists': return (got !== undefined && got !== null) === (want !== false);
+    case 'eq': return got === want;
+    case 'ne': return got !== want;
+    case 'in': return Array.isArray(want) && (want as unknown[]).includes(got);
+    case 'contains': return Array.isArray(got) ? got.includes(want) : typeof got === 'string' && typeof want === 'string' && got.includes(want);
+    case 'startsWith': return typeof got === 'string' && typeof want === 'string' && got.startsWith(want);
+    case 'gt': return typeof got === 'number' && typeof want === 'number' && got > want;
+    case 'lt': return typeof got === 'number' && typeof want === 'number' && got < want;
+  }
+}
+
+function run(c: Cond, node: unknown, file: string): Ev {
+  if ('all' in c) {
+    const rs = c.all.map((x) => run(x, node, file));
+    return rs.every((r) => r.ok) ? { ok: true, w: rs.flatMap((r) => r.w) } : NO;
+  }
+  if ('any' in c) {
+    const rs = c.any.map((x) => run(x, node, file)).filter((r) => r.ok);
+    return rs.length ? { ok: true, w: rs.flatMap((r) => r.w) } : NO;
+  }
+  if ('not' in c) return run(c.not, node, file).ok ? NO : { ok: true, w: [] };
+  if ('some' in c || 'none' in c) {
+    const f = 'some' in c ? c.some : c.none;
+    const arr = pick(node, f);
+    const items = Array.isArray(arr) ? arr : [];
+    const hits = items.filter((it) => !c.where || run(c.where, it, isObj(it) && typeof it.file === 'string' ? it.file : file).ok);
+    if ('none' in c) return hits.length === 0 ? { ok: true, w: [] } : NO;
+    if (!hits.length) return NO;
+    const w: Witness[] = [];
+    for (const it of hits) if (isObj(it) && typeof it.line === 'number') w.push({ file: typeof it.file === 'string' ? it.file : file, line: it.line });
+    return { ok: true, w };
+  }
+  return compare(c, node) ? { ok: true, w: [] } : NO;
+}
+
+export interface QueryOutcome { confirmed: boolean; detail: string; witnesses: Witness[] }
+
+// Runs the query on the lead's entry point. The lead is confirmed only when the entry point matches and at least one fact the
+// query matched (a data touch, a protection, a read or an outbound call) sits on or beside a line the lead cites. A query that
+// matches every entry point of a map of TOO_BROAD_MIN or more proves nothing and is not accepted.
+export function runQuery(map: CodeMap, q: Query, entryId: string, cited: LeadStep[]): QueryOutcome {
+  const ep = map.entry_points.find((e) => e.id === entryId);
+  if (!ep) return { confirmed: false, detail: 'the entry point is not in the code map', witnesses: [] };
+  const at = (e: EntryPoint): Ev => run(q.where, e, e.handler?.file ?? e.file);
+  const mine = at(ep);
+  if (!mine.ok) return { confirmed: false, detail: 'the query does not match this entry point', witnesses: [] };
+  const matched = map.entry_points.filter((e) => at(e).ok).length;
+  if (map.entry_points.length >= TOO_BROAD_MIN && matched === map.entry_points.length) {
+    return { confirmed: false, detail: `the query matches all ${matched} entry points, so it proves nothing about this one`, witnesses: [] };
+  }
+  const near = mine.w.filter((w) => cited.some((s) => s.file === w.file && Math.abs(s.line - w.line) <= WITNESS_SLACK));
+  if (!near.length) return { confirmed: false, detail: 'no fact the query matched sits on a line the lead cites', witnesses: [] };
+  const uniq = [...new Map(near.map((w) => [`${w.file}:${w.line}`, w])).values()].sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : a.line - b.line));
+  return { confirmed: true, detail: `the query matches this entry point and ${matched - 1} other of ${map.entry_points.length}; matched facts at ${uniq.map((w) => `${w.file}:${w.line}`).join(', ')}`, witnesses: uniq };
+}

package/src/confirm/rule.ts

@@ -0,0 +1,76 @@
+// Confirmation by a generated Semgrep rule. The rule must match the lines the lead cites in the real file, and it must not match a
+// negative control: a safe sample of the same kind of code that the model supplies. Both runs must really scan their file, so a rule
+// for another language cannot pass by matching nothing. Semgrep runs offline with metrics off. If Semgrep is not installed the
+// outcome is "not run" and the lead stays a lead.
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import type { Lead } from '../types/index.ts';
+
+export interface SemgrepRun { status: number | null; stdout: string; missing?: boolean }
+export type SemgrepRunner = (args: string[]) => SemgrepRun;
+const TIMEOUT_MS = 120_000;
+const SLACK = 1;
+
+export const spawnSemgrep: SemgrepRunner = (args) => {
+  const r = spawnSync('semgrep', args, { encoding: 'utf8', timeout: TIMEOUT_MS, maxBuffer: 32 * 1024 * 1024, env: { PATH: process.env.PATH ?? '', HOME: process.env.HOME ?? '', SEMGREP_SEND_METRICS: 'off', SEMGREP_ENABLE_VERSION_CHECK: '0' } });
+  if ((r.error as NodeJS.ErrnoException | undefined)?.code === 'ENOENT') return { status: null, stdout: '', missing: true };
+  return { status: r.status, stdout: r.stdout ?? '' };
+};
+
+export type RuleOutcome =
+  | { status: 'confirmed'; detail: string; rule_id: string; tool_version: string; config_sha256: string }
+  | { status: 'not_confirmed'; detail: string }
+  | { status: 'not_run'; detail: string };
+
+interface Out { results: { start: { line: number }; end: { line: number } }[]; errors: unknown[]; paths?: { scanned?: string[] } }
+const BANNED = /pattern-where-python|pattern-sources:\s*\n\s*-\s*exact|\binclude:|r2c-internal/;
+
+function scanFile(run: SemgrepRunner, rule: string, target: string): { out: Out } | { fail: string } | { missing: true } {
+  const r = run(['scan', '--config', rule, '--json', '--quiet', '--metrics=off', '--disable-version-check', '--no-git-ignore', '--timeout', '10', target]);
+  if (r.missing) return { missing: true };
+  let out: Out;
+  try { out = JSON.parse(r.stdout) as Out; } catch { return { fail: 'semgrep printed no JSON for this rule' }; }
+  if (!Array.isArray(out.results)) return { fail: 'semgrep printed no results list' };
+  if (Array.isArray(out.errors) && out.errors.length) return { fail: 'semgrep reported an error for the rule or the file' };
+  if (!(out.paths?.scanned ?? []).some((p) => resolve(p) === resolve(target))) return { fail: 'semgrep did not scan the file (the rule does not cover its language)' };
+  return { out };
+}
+
+export function confirmRule(lead: Lead, repo: string, run: SemgrepRunner = spawnSemgrep): RuleOutcome {
+  const p = lead.evidence_proposal;
+  if (p.kind !== 'semgrep_rule') return { status: 'not_confirmed', detail: 'no rule was proposed' };
+  const ids = [...p.rule_yaml.matchAll(/^\s*-\s*id:\s*([^\s#]+)/gm)].map((m) => m[1]);
+  if (!/^\s*rules:/m.test(p.rule_yaml) || ids.length !== 1) return { status: 'not_confirmed', detail: 'the rule file must hold exactly one rule' };
+  if (!/^[A-Za-z0-9._-]{1,100}$/.test(ids[0])) return { status: 'not_confirmed', detail: 'the rule id has characters outside letters, digits, dot, underscore and hyphen' };
+  if (BANNED.test(p.rule_yaml)) return { status: 'not_confirmed', detail: 'the rule uses a feature the engine does not accept' };
+  const norm = (s: string): string => s.replace(/\s+/g, ' ').trim();
+  if (norm(p.control_code).includes(norm(lead.quoted_code))) return { status: 'not_confirmed', detail: 'the negative control still holds the quoted code, so it is not a control' };
+
+  const sink = lead.path[lead.path.length - 1];
+  const dir = mkdtempSync(join(tmpdir(), 'hp-rule-'));
+  try {
+    const ruleFile = join(dir, 'rule.yaml');
+    const controlFile = join(dir, `control.${p.control_extension}`);
+    writeFileSync(ruleFile, p.rule_yaml);
+    writeFileSync(controlFile, p.control_code);
+    const ver = run(['--version']);
+    if (ver.missing) return { status: 'not_run', detail: 'Semgrep is not installed' };
+    const hit = scanFile(run, ruleFile, join(resolve(repo), sink.file));
+    if ('missing' in hit) return { status: 'not_run', detail: 'Semgrep is not installed' };
+    if ('fail' in hit) return { status: 'not_confirmed', detail: hit.fail };
+    const onSink = hit.out.results.some((r) => r.end.line >= sink.line - SLACK && r.start.line <= sink.line + SLACK);
+    if (!onSink) return { status: 'not_confirmed', detail: `the rule does not match the cited lines (${sink.file}:${sink.line})` };
+    const ctl = scanFile(run, ruleFile, controlFile);
+    if ('missing' in ctl) return { status: 'not_run', detail: 'Semgrep is not installed' };
+    if ('fail' in ctl) return { status: 'not_confirmed', detail: `negative control: ${ctl.fail}` };
+    if (ctl.out.results.length) return { status: 'not_confirmed', detail: 'the rule also matches the negative control, so it does not tell the flaw from safe code' };
+    return {
+      status: 'confirmed', detail: `the rule matches ${sink.file}:${sink.line} and does not match the negative control`, rule_id: `hunt.${ids[0]}`.slice(0, 190),
+      tool_version: (/^\d+\.\d+[\w.+-]*/m.exec(ver.stdout)?.[0] ?? 'unknown').replace(/[^A-Za-z0-9._+-]/g, '-').slice(0, 59) || 'unknown',
+      config_sha256: createHash('sha256').update(p.rule_yaml).digest('hex'),
+    };
+  } finally { rmSync(dir, { recursive: true, force: true }); }
+}

package/src/confirm/store.ts

@@ -0,0 +1,42 @@
+// Where hunt files live and how they are read and written: <out>/<commit>/hunt/. Every answer is written the moment it arrives,
+// through a temporary file and a rename, so an interrupted run leaves whole files only and can resume.
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { HuntAnswer, HuntPlan, HuntResult, HuntSynthesis } from '../types/index.ts';
+import { cmp, stableJson } from '../util.ts';
+
+export const huntDir = (scanDir: string): string => join(scanDir, 'hunt');
+const slug = (s: string): string => s.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'entry';
+export const answerFile = (hunt: string, run: number, entryId: string): string =>
+  join(hunt, `run-${run}`, `${slug(entryId)}-${createHash('sha256').update(entryId).digest('hex').slice(0, 8)}.json`);
+
+function writeAtomic(path: string, text: string): void {
+  mkdirSync(join(path, '..'), { recursive: true });
+  const tmp = `${path}.tmp`;
+  writeFileSync(tmp, text);
+  renameSync(tmp, path);
+}
+const readJson = <T>(p: string): T => { try { return JSON.parse(readFileSync(p, 'utf8')) as T; } catch (e) { throw new Error(`${p} cannot be read: ${(e as Error).message}`); } };
+
+export const writeAnswer = (hunt: string, a: HuntAnswer): void => writeAtomic(answerFile(hunt, a.run, a.entry_id), stableJson(a));
+export const hasAnswer = (hunt: string, run: number, entryId: string): boolean => existsSync(answerFile(hunt, run, entryId));
+
+export function readAnswers(hunt: string): HuntAnswer[] {
+  if (!existsSync(hunt)) return [];
+  const out: HuntAnswer[] = [];
+  for (const d of readdirSync(hunt).filter((n) => /^run-\d+$/.test(n)).sort(cmp)) {
+    for (const f of readdirSync(join(hunt, d)).filter((n) => n.endsWith('.json')).sort(cmp)) out.push(readJson<HuntAnswer>(join(hunt, d, f)));
+  }
+  return out.sort((a, b) => a.run - b.run || cmp(a.entry_id, b.entry_id));
+}
+
+export const writePlan = (hunt: string, p: HuntPlan): void => writeAtomic(join(hunt, 'plan.json'), stableJson(p));
+export const readPlan = (hunt: string): HuntPlan | null => (existsSync(join(hunt, 'plan.json')) ? readJson<HuntPlan>(join(hunt, 'plan.json')) : null);
+export const writeResult = (hunt: string, r: HuntResult): void => writeAtomic(join(hunt, 'result.json'), stableJson(r));
+export const readResult = (hunt: string): HuntResult | null => (existsSync(join(hunt, 'result.json')) ? readJson<HuntResult>(join(hunt, 'result.json')) : null);
+export const writeSynthesis = (hunt: string, s: HuntSynthesis): void => writeAtomic(join(hunt, 'synthesis.json'), stableJson(s));
+export const readSynthesis = (hunt: string): HuntSynthesis | null => (existsSync(join(hunt, 'synthesis.json')) ? readJson<HuntSynthesis>(join(hunt, 'synthesis.json')) : null);
+export const writeSkipped = (hunt: string, reason: string): void => writeAtomic(join(hunt, 'skipped.json'), stableJson({ schema_version: '1.0.0', reason }));
+export const readSkipped = (hunt: string): string | null => (existsSync(join(hunt, 'skipped.json')) ? readJson<{ reason: string }>(join(hunt, 'skipped.json')).reason : null);
+export const hasHuntFiles = (hunt: string): boolean => existsSync(hunt) && readdirSync(hunt).some((n) => /^run-\d+$/.test(n) || n === 'plan.json');

package/src/confirm/summary.ts

@@ -0,0 +1,9 @@
+// The two lines `hullproof hunt` prints about a result.
+import type { HuntResult } from '../types/index.ts';
+
+export function summary(r: HuntResult): string[] {
+  return [
+    `hullproof hunt: ${r.confirmed.length} confirmed, ${r.leads.length} leads not confirmed, ${r.hallucinations.count} hallucinations dropped, ${r.malformed.count} malformed`,
+    `hullproof hunt: calls ${r.calls.complete} complete, ${r.calls.incomplete} incomplete, ${r.calls.missing} not saved; tokens ${r.usage.input_tokens} in, ${r.usage.output_tokens} out (reported, not limited)`,
+  ];
+}

package/src/confirm/synthesis.ts

@@ -0,0 +1,33 @@
+// The guard around the synthesis step. The synthesis call may explain the findings and leads that are already in result.json. It
+// cannot add one, remove one or change one: this function keeps only a summary and one explanation per known key, and nothing
+// else from the model's reply reaches any file the gate reads.
+import type { HuntResult, HuntSynthesis } from '../types/index.ts';
+import { cmp } from '../util.ts';
+import { plain } from './lead.ts';
+
+export const SYNTHESIS_LIMITS = { summary: 4000, explanation: 800, explanations: 60 } as const;
+
+export function guardSynthesis(result: HuntResult, reply: unknown, meta: { provider: string; model: string; usage: { input_tokens: number; output_tokens: number } }): HuntSynthesis {
+  const known = new Set([...result.confirmed, ...result.leads].map((l) => l.key));
+  const o = reply && typeof reply === 'object' && !Array.isArray(reply) ? (reply as Record<string, unknown>) : {};
+  const summary = typeof o.summary === 'string' ? plain(o.summary, SYNTHESIS_LIMITS.summary) : '';
+  const seen = new Set<string>();
+  const explanations: HuntSynthesis['explanations'] = [];
+  if (Array.isArray(o.explanations)) {
+    for (const x of o.explanations) {
+      if (!x || typeof x !== 'object') continue;
+      const { key, text } = x as { key?: unknown; text?: unknown };
+      if (typeof key !== 'string' || typeof text !== 'string' || !known.has(key) || seen.has(key) || explanations.length >= SYNTHESIS_LIMITS.explanations) continue;
+      seen.add(key);
+      explanations.push({ key, text: plain(text, SYNTHESIS_LIMITS.explanation) });
+    }
+  }
+  explanations.sort((a, b) => cmp(a.key, b.key));
+  return { schema_version: '1.0.0', status: 'complete', stop: null, detail: null, provider: meta.provider, model: meta.model, summary, explanations, usage: meta.usage };
+}
+
+// A saved synthesis is applied again on replay against the result just computed: explanations for keys that no longer exist drop out.
+export function reguard(result: HuntResult, s: HuntSynthesis): HuntSynthesis {
+  const g = guardSynthesis(result, { summary: s.summary, explanations: s.explanations }, { provider: s.provider, model: s.model, usage: s.usage });
+  return { ...g, status: s.status, stop: s.stop, detail: s.detail };
+}

package/src/discover/detectors.ts

@@ -0,0 +1,71 @@
+// Loads the detector rule files (detectors/*.json). Detectors are data; this file only reads and checks them.
+// The format is documented in docs/INTERFACES.md, section "Detector rule files".
+import { createHash } from 'node:crypto';
+import { readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { STACK_CATEGORIES } from '../types/stack.ts';
+import type { Confidence, StackCategory } from '../types/stack.ts';
+import { cmp } from '../util.ts';
+
+export interface Signal {
+  files: string[];                 // globs
+  regex?: string;                  // when absent, the file being there is the signal
+  all?: string[];                  // further patterns that must also match somewhere in the same file
+  ignore_case?: boolean;
+  confidence?: Confidence;         // default: high for a file signal, medium for a content signal
+  attrs?: Record<string, string>;  // list fields to add to the item; "$file" is replaced by the file path
+}
+
+export interface DetectorEntry {
+  id: string;
+  attrs?: Record<string, string>;  // scalar fields of the item, for example {"kind": "cloud"}
+  extensions?: string[];           // languages only
+  deps?: Record<string, string[]>; // ecosystem to package names; "x*" is a prefix
+  deps_confidence?: Confidence | Record<string, Confidence>; // default high; an object sets it per ecosystem, "*" for the rest
+  imports?: Record<string, string[]>; // import language key to module names; "x*" is a prefix
+  imports_confidence?: Confidence; // default medium
+  signals?: Signal[];
+}
+
+export interface DetectorFile { category: StackCategory; entries: DetectorEntry[] }
+
+export const DETECTORS_DIR = fileURLToPath(new URL('../../detectors/', import.meta.url));
+
+const CONF = new Set(['high', 'medium', 'low']);
+
+function check(name: string, f: DetectorFile): void {
+  const bad = (msg: string): never => { throw new Error(`detectors/${name}: ${msg}`); };
+  if (!STACK_CATEGORIES.includes(f.category)) bad(`unknown category ${f.category}`);
+  const ids = new Set<string>();
+  for (const e of f.entries ?? bad('no entries')) {
+    if (!/^[a-z0-9][a-z0-9.-]*$/.test(e.id ?? '')) bad(`bad id ${e.id}`);
+    if (ids.has(e.id)) bad(`duplicate id ${e.id}`);
+    ids.add(e.id);
+    if (!e.deps && !e.imports && !e.signals && !e.extensions) bad(`${e.id} has no signal`);
+    const dc = typeof e.deps_confidence === 'object' ? Object.values(e.deps_confidence) : [e.deps_confidence];
+    for (const c of [...dc, e.imports_confidence, ...(e.signals ?? []).map((s) => s.confidence)]) {
+      if (c !== undefined && !CONF.has(c)) bad(`${e.id} bad confidence ${c}`);
+    }
+    for (const s of e.signals ?? []) {
+      if (!Array.isArray(s.files) || s.files.length === 0) bad(`${e.id} signal without files`);
+      for (const r of [s.regex, ...(s.all ?? [])]) if (r !== undefined) new RegExp(r); // throws on a bad pattern
+    }
+  }
+}
+
+export interface LoadedDetectors { files: DetectorFile[]; sha256: string }
+
+export function loadDetectors(dir: string = DETECTORS_DIR): LoadedDetectors {
+  const names = readdirSync(dir).filter((n) => n.endsWith('.json')).sort(cmp);
+  const sums: string[] = [];
+  const files: DetectorFile[] = [];
+  for (const n of names) {
+    const raw = readFileSync(join(dir, n));
+    sums.push(`${createHash('sha256').update(raw).digest('hex')}  ${n}\n`);
+    const f = JSON.parse(raw.toString('utf8')) as DetectorFile;
+    check(n, f);
+    files.push(f);
+  }
+  return { files, sha256: createHash('sha256').update(sums.join('')).digest('hex') };
+}

package/src/discover/discover.ts

@@ -0,0 +1,253 @@
+// Stack discovery: deterministic, no model, no network, runs no program. Lists files, extracts facts, applies the detector
+// rules, and builds stack.json. Same files and same detector files give the same bytes.
+import { closeSync, openSync, readSync, fstatSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import type { Confidence, Evidence, Stack, StackCategory, StackItem, Unknown } from '../types/stack.ts';
+import { STACK_CATEGORIES } from '../types/stack.ts';
+import { cmp, globToRegExp, sortUnique } from '../util.ts';
+import { ENGINE_VERSION } from '../version.ts';
+import { loadDetectors } from './detectors.ts';
+import type { DetectorEntry, LoadedDetectors, Signal } from './detectors.ts';
+import { IMPORT_LANGS, MANIFEST_NAMES, extractDeps, extractImports, importLangOf, nameMatches } from './extract.ts';
+import type { DepFact, ImportFact } from './extract.ts';
+import { readHeadCommit } from './git.ts';
+import { walk } from './walk.ts';
+
+export const LIMITS = { maxFiles: 200_000, maxBytesPerFile: 512 * 1024, maxImportFiles: 50_000, evidencePerItem: 5, languageSamples: 3 };
+
+// Separator for prefix matching of package names per ecosystem; '' means exact names only (use "x*" for a prefix).
+const DEP_SEP: Record<string, string> = {
+  npm: '/', composer: '/', go: '/', maven: ':', nuget: '.', cocoapods: '/', swiftpm: '/',
+  pypi: '', cargo: '', rubygems: '', pub: '', hex: '', terraform: '',
+};
+
+// Extensions that are data, docs, media or build artefacts: neither a language nor worth listing as unknown.
+const NOT_CODE = new Set(('md markdown mdx txt rst adoc json jsonc json5 yaml yml toml xml csv tsv ini cfg conf lock log ' +
+  'png jpg jpeg gif svg ico webp bmp tif tiff avif heic mp3 mp4 m4a wav ogg webm mov avi pdf zip gz tgz tar bz2 xz 7z rar jar war ' +
+  'woff woff2 ttf otf eot map snap example sample template tpl j2 properties plist pbxproj xcconfig xcscheme xcworkspacedata ' +
+  'storyboard xib entitlements strings pem crt cer key pub gitkeep keep sqlite sqlite3 db bin dat pkl pickle h5 npy ' +
+  'sum mod tfvars tfstate hcl lockb editorconfig env patch diff po pot mo xlsx xls docx doc pptx odt license gradle bat cmd ' +
+  'iml http rest bak orig tmp swp dockerignore gitignore gitattributes npmrc nvmrc prettierrc eslintrc babelrc ipynb').split(/\s+/));
+
+const LOCKFILE = /(^|\/)(package-lock\.json|npm-shrinkwrap\.json|yarn\.lock|pnpm-lock\.yaml|composer\.lock|packages\.lock\.json|[^/]+\.lock)$/;
+// Configuration files are looked for at the repository root and one level inside dot folders (.github, .do, .gitlab).
+const CONFIG_EXT = /\.(ya?ml|toml|json|jsonc|ini|cfg|conf|hcl|properties)$/;
+// Common tool configuration with no security meaning for discovery; not listed as unknown.
+const KNOWN_TOOLING = /(^|\/)(\.?(eslint|prettier|stylelint|babel|jest|nyc|mocha|lint-staged|commitlint|markdownlint|yamllint|remark|cspell|renovate|codecov|release-it|semantic-release|browserslist|swcrc|postcss|tailwind|editorconfig|husky|c8|ava|vitest|karma|protractor|typedoc|tslint|lerna|nx|turbo|crowdin|codeclimate|mailmap|devcontainer|dependabot|pre-commit-config|gitpod|vscode|readthedocs|mkdocs|coveralls|sonar-project|cypress|playwright|stryker|snyk|zap|ncurc|npmpackagejsonlintrc|jsdoc|jsconfig|typos)(rc)?(\.[^/]*)?|tsconfig[^/]*|package-lock\.json|composer\.lock|\.?github\/[^/]*\.ya?ml|\.?github\/FUNDING\.yml|bower\.json|manifest\.json)$/i;
+const TF_UTILITY = new Set(['random', 'null', 'local', 'tls', 'time', 'archive', 'template', 'external', 'http', 'terraform', 'cloudinit']);
+
+const depConf = (e: DetectorEntry, eco: string): Confidence => {
+  const c = e.deps_confidence;
+  return (typeof c === 'object' ? c[eco] ?? c['*'] : c) ?? 'high';
+};
+
+const RANK: Record<Confidence, number> = { high: 0, medium: 1, low: 2 };
+
+interface Acc { evidence: Evidence[]; versions: Set<string>; lists: Map<string, Set<string>>; scalars: Record<string, string>; fileCount?: number }
+
+function readHead(abs: string, max: number): { text: string; truncated: boolean } {
+  const fd = openSync(abs, 'r');
+  try {
+    const size = fstatSync(fd).size;
+    const buf = Buffer.alloc(Math.min(size, max));
+    const n = readSync(fd, buf, 0, buf.length, 0);
+    return { text: buf.subarray(0, n).toString('utf8'), truncated: size > max };
+  } finally { closeSync(fd); }
+}
+
+const lineAt = (text: string, index: number): number => {
+  let n = 1;
+  for (let i = 0; i < index; i++) if (text.charCodeAt(i) === 10) n++;
+  return n;
+};
+
+const extOf = (f: string): string => {
+  const b = f.slice(f.lastIndexOf('/') + 1);
+  const d = b.lastIndexOf('.');
+  return d > 0 ? b.slice(d + 1).toLowerCase() : '';
+};
+
+export interface DiscoverOptions { detectors?: LoadedDetectors; skipFiles?: ReadonlySet<string> } // skipFiles: unchanged kit files (kit-files.ts)
+
+export function discover(repo: string, opts: DiscoverOptions = {}): Stack {
+  const root = resolve(repo);
+  const det = opts.detectors ?? loadDetectors();
+  const listing = walk(root, LIMITS.maxFiles, opts.skipFiles);
+  const files = listing.files;
+
+  const cache = new Map<string, string>();
+  const truncated = new Set<string>();
+  const text = (f: string): string => {
+    let t = cache.get(f);
+    if (t === undefined) {
+      try {
+        const r = readHead(join(root, f), LIMITS.maxBytesPerFile);
+        t = r.text;
+        if (r.truncated) truncated.add(f);
+      } catch { t = ''; }
+      cache.set(f, t);
+    }
+    return t;
+  };
+
+  // Facts.
+  const deps: DepFact[] = [];
+  for (const f of files) if (MANIFEST_NAMES.test(f)) deps.push(...extractDeps(f, text(f)));
+  const importFiles = files.filter((f) => importLangOf(f) !== null);
+  const scanned = importFiles.slice(0, LIMITS.maxImportFiles);
+  const imports: ImportFact[] = [];
+  for (const f of scanned) {
+    imports.push(...extractImports(f, text(f)));
+    cache.delete(f); // source text is not needed again unless a content signal asks for it
+  }
+  const depsByEco = Map.groupBy(deps, (d) => d.eco);
+  const importsByLang = Map.groupBy(imports, (i) => i.lang);
+
+  // Items.
+  const items = new Map<string, Acc>();
+  const usedFiles = new Set<string>();
+  const matchedTf = new Set<string>();
+  const acc = (cat: StackCategory, id: string): Acc => {
+    const k = `${cat}\u0000${id}`;
+    let a = items.get(k);
+    if (!a) { a = { evidence: [], versions: new Set(), lists: new Map(), scalars: {} }; items.set(k, a); }
+    return a;
+  };
+  const add = (a: Acc, ev: Evidence): void => { a.evidence.push(ev); usedFiles.add(ev.file); };
+  const addList = (a: Acc, attrs: Record<string, string> | undefined, file: string): void => {
+    for (const [k, v] of Object.entries(attrs ?? {})) {
+      if (!a.lists.has(k)) a.lists.set(k, new Set());
+      a.lists.get(k)!.add(v === '$file' ? file : v);
+    }
+  };
+  const globCache = new Map<string, string[]>();
+  const filesFor = (globs: string[]): string[] => {
+    const key = globs.join('\n');
+    let r = globCache.get(key);
+    if (!r) { const res = globs.map(globToRegExp); r = files.filter((f) => res.some((re) => re.test(f))); globCache.set(key, r); }
+    return r;
+  };
+
+  const runSignal = (cat: StackCategory, e: DetectorEntry, s: Signal, i: number): void => {
+    const kind = s.regex ? 'content' : 'file';
+    const rule = `${cat}.${e.id}.${kind}.${i + 1}`;
+    const conf: Confidence = s.confidence ?? (s.regex ? 'medium' : 'high');
+    const flags = 'm' + (s.ignore_case ? 'i' : '');
+    const re = s.regex ? new RegExp(s.regex, flags) : null;
+    const all = (s.all ?? []).map((r) => new RegExp(r, flags));
+    for (const f of filesFor(s.files)) {
+      if (re && LOCKFILE.test(f)) continue; // lockfiles are never templates or config; reading them only costs time
+      if (!re) { const a = acc(cat, e.id); add(a, { file: f, key: '(present)', rule, confidence: conf }); addList(a, s.attrs, f); continue; }
+      const t = text(f);
+      const m = re.exec(t);
+      if (!m || !all.every((r) => r.test(t))) continue;
+      const a = acc(cat, e.id);
+      add(a, { file: f, line: lineAt(t, m.index), rule, confidence: conf });
+      addList(a, s.attrs, f);
+    }
+  };
+
+  for (const df of det.files) {
+    const cat = df.category;
+    for (const e of df.entries) {
+      for (const [eco, names] of Object.entries(e.deps ?? {})) {
+        for (const d of depsByEco.get(eco) ?? []) {
+          if (!names.some((n) => nameMatches(d.name, n, DEP_SEP[eco] ?? ''))) continue;
+          if (eco === 'terraform') matchedTf.add(d.name);
+          const a = acc(cat, e.id);
+          add(a, { file: d.file, ...(d.line !== undefined ? { line: d.line } : { key: d.key }), rule: `${cat}.${e.id}.dep.${eco}`, confidence: depConf(e, eco) });
+          if (d.version) a.versions.add(`${d.name}@${d.version}`);
+        }
+      }
+      for (const [lang, mods] of Object.entries(e.imports ?? {})) {
+        const sep = IMPORT_LANGS[lang]?.sep ?? '';
+        for (const im of importsByLang.get(lang) ?? []) {
+          if (!mods.some((m) => nameMatches(im.module, m, sep))) continue;
+          add(acc(cat, e.id), { file: im.file, line: im.line, rule: `${cat}.${e.id}.import.${lang}`, confidence: e.imports_confidence ?? 'medium' });
+        }
+      }
+      (e.signals ?? []).forEach((s, i) => runSignal(cat, e, s, i));
+      if (e.extensions) {
+        const exts = new Set(e.extensions);
+        const hits = files.filter((f) => exts.has(extOf(f)));
+        if (hits.length) {
+          const a = acc(cat, e.id);
+          a.fileCount = hits.length;
+          for (const f of hits) a.evidence.push({ file: f, key: '(extension)', rule: `${cat}.${e.id}.extension`, confidence: 'high' });
+        }
+      }
+      const a = items.get(`${cat}\u0000${e.id}`);
+      if (a) Object.assign(a.scalars, e.attrs ?? {});
+    }
+  }
+
+  // Unknowns: never guessed, only listed.
+  const unknown: Unknown[] = [];
+  const langExts = new Set(det.files.filter((f) => f.category === 'languages').flatMap((f) => f.entries.flatMap((e) => e.extensions ?? [])));
+  const byExt = new Map<string, string[]>();
+  for (const f of files) {
+    const x = extOf(f);
+    if (!x || langExts.has(x) || NOT_CODE.has(x)) continue;
+    if (!byExt.has(x)) byExt.set(x, []);
+    byExt.get(x)!.push(f);
+  }
+  for (const [x, fs] of byExt) unknown.push({ file: fs[0], note: `no detector for the file extension .${x} (${fs.length} file${fs.length === 1 ? '' : 's'})` });
+  for (const f of files) {
+    const parts = f.split('/');
+    const configPlace = parts.length === 1 || (parts.length === 2 && parts[0].startsWith('.'));
+    if (!configPlace || !CONFIG_EXT.test(f) || usedFiles.has(f) || KNOWN_TOOLING.test(f)) continue;
+    unknown.push({ file: f, note: 'configuration file with no detector' });
+  }
+  const tfSeen = new Set<string>();
+  for (const d of depsByEco.get('terraform') ?? []) {
+    if (matchedTf.has(d.name) || TF_UTILITY.has(d.name) || tfSeen.has(d.name)) continue;
+    tfSeen.add(d.name);
+    unknown.push({ file: d.file, note: `Terraform provider "${d.name}" has no detector` });
+  }
+  const LOCKS = ['package-lock.json', 'npm-shrinkwrap.json', 'yarn.lock', 'pnpm-lock.yaml', 'bun.lockb', 'bun.lock'];
+  const fileSet = new Set(files);
+  for (const f of files.filter((x) => x === 'package.json' || x.endsWith('/package.json'))) {
+    const dir = f.slice(0, -'package.json'.length);
+    if (LOCKS.some((l) => fileSet.has(dir + l) || fileSet.has(l))) continue;
+    unknown.push({ file: f, note: 'package.json with no lockfile in its folder or the repository root: package manager not determined' });
+  }
+
+  // Assemble, sorted.
+  const out = Object.fromEntries(STACK_CATEGORIES.map((c) => [c, [] as StackItem[]])) as Record<StackCategory, StackItem[]>;
+  for (const [k, a] of items) {
+    const [cat, id] = k.split('\u0000') as [StackCategory, string];
+    const seen = new Set<string>();
+    const ev = a.evidence
+      .sort((x, y) => RANK[x.confidence] - RANK[y.confidence] || cmp(x.file, y.file) || (x.line ?? 0) - (y.line ?? 0) || cmp(x.key ?? '', y.key ?? '') || cmp(x.rule, y.rule))
+      .filter((x) => { const s = `${x.file}|${x.line}|${x.key}|${x.rule}`; return seen.has(s) ? false : (seen.add(s), true); });
+    const shown = cat === 'languages' && a.fileCount ? LIMITS.languageSamples : LIMITS.evidencePerItem;
+    // The sample shows the best entry of each file first, so it spans files instead of repeating one file.
+    const fileSeen = new Set<string>();
+    const firsts = ev.filter((x) => !fileSeen.has(x.file) && !!fileSeen.add(x.file));
+    const firstSet = new Set(firsts);
+    const sample = [...firsts, ...ev.filter((x) => !firstSet.has(x))].slice(0, shown);
+    const item: StackItem = { id, confidence: ev[0].confidence, evidence: sample, evidence_count: ev.length, ...a.scalars };
+    if (a.versions.size) item.versions = sortUnique(a.versions);
+    if (a.fileCount) item.file_count = a.fileCount;
+    for (const [lk, set] of a.lists) (item as unknown as Record<string, string[]>)[lk] = sortUnique(set);
+    out[cat].push(item);
+  }
+  for (const c of STACK_CATEGORIES) out[c].sort((x, y) => cmp(x.id, y.id));
+
+  return {
+    schema_version: '1.0.0',
+    engine_version: ENGINE_VERSION,
+    detectors_sha256: det.sha256,
+    commit: readHeadCommit(root),
+    confirmed_by_user: false,
+    files: { listed: files.length, scanned_for_imports: scanned.length },
+    limits: {
+      max_files: LIMITS.maxFiles, max_bytes_per_file: LIMITS.maxBytesPerFile,
+      file_cap_reached: listing.capReached || importFiles.length > scanned.length,
+      truncated_files: sortUnique(truncated),
+    },
+    skipped: listing.skipped.slice().sort((x, y) => cmp(x.path, y.path)),
+    ...out,
+    unknown: unknown.sort((x, y) => cmp(x.file, y.file) || cmp(x.note, y.note)),
+  };
+}