hullproof 0.0.0-stage → 0.5.0

package/dist/confirm/query.js

@@ -0,0 +1,145 @@
+export const MAX_DEPTH = 8;
+export const MAX_NODES = 60;
+export const WITNESS_SLACK = 2; // a fact counts when its line is within this many lines of a cited line
+export const TOO_BROAD_MIN = 3; // a query that matches every entry point of a map this large proves nothing
+const OPS = new Set(['eq', 'ne', 'in', 'contains', 'startsWith', 'exists', 'gt', 'lt']);
+const FIELD = /^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$/;
+const BAD_SEG = new Set(['__proto__', 'constructor', 'prototype']);
+const isObj = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
+const goodField = (f) => typeof f === 'string' && FIELD.test(f) && !f.split('.').some((s) => BAD_SEG.has(s));
+export function parseQuery(v) {
+    if (!isObj(v) || !('where' in v))
+        return { ok: false, reason: 'a query is {"where": condition}' };
+    let nodes = 0;
+    const cond = (c, depth) => {
+        if (depth > MAX_DEPTH)
+            return `nested deeper than ${MAX_DEPTH}`;
+        if (++nodes > MAX_NODES)
+            return `more than ${MAX_NODES} conditions`;
+        if (!isObj(c))
+            return 'a condition must be an object';
+        for (const k of ['all', 'any']) {
+            if (k in c) {
+                if (!Array.isArray(c[k]) || c[k].length === 0)
+                    return `${k} needs a non empty list`;
+                const out = [];
+                for (const x of c[k]) {
+                    const r = cond(x, depth + 1);
+                    if (typeof r === 'string')
+                        return r;
+                    out.push(r);
+                }
+                return k === 'all' ? { all: out } : { any: out };
+            }
+        }
+        if ('not' in c) {
+            const r = cond(c.not, depth + 1);
+            return typeof r === 'string' ? r : { not: r };
+        }
+        for (const k of ['some', 'none']) {
+            if (k in c) {
+                if (!goodField(c[k]))
+                    return `${k} needs a field name`;
+                let where;
+                if (c.where !== undefined) {
+                    const r = cond(c.where, depth + 1);
+                    if (typeof r === 'string')
+                        return r;
+                    where = r;
+                }
+                return k === 'some' ? { some: c.some, ...(where ? { where } : {}) } : { none: c.none, ...(where ? { where } : {}) };
+            }
+        }
+        if ('field' in c) {
+            if (!goodField(c.field))
+                return 'field is not a plain dotted name';
+            if (typeof c.op !== 'string' || !OPS.has(c.op))
+                return `op is one of ${[...OPS].join(', ')}`;
+            const val = c.value;
+            const scalar = (x) => x === null || ['string', 'number', 'boolean'].includes(typeof x);
+            if (c.op === 'in' ? !Array.isArray(val) || !val.every(scalar) || val.length > 20 : c.op === 'exists' ? val !== undefined && typeof val !== 'boolean' : !scalar(val))
+                return `value does not fit op ${c.op}`;
+            if ((c.op === 'gt' || c.op === 'lt') && typeof val !== 'number')
+                return `${c.op} needs a number`;
+            if ((c.op === 'startsWith') && typeof val !== 'string')
+                return 'startsWith needs a string';
+            return { field: c.field, op: c.op, ...(val !== undefined ? { value: val } : {}) };
+        }
+        return 'a condition is all, any, not, some, none or field';
+    };
+    const r = cond(v.where, 1);
+    return typeof r === 'string' ? { ok: false, reason: r } : { ok: true, query: { where: r } };
+}
+const NO = { ok: false, w: [] };
+function pick(node, path) {
+    let cur = node;
+    for (const seg of path.split('.')) {
+        if (!isObj(cur) || !Object.hasOwn(cur, seg))
+            return undefined;
+        cur = cur[seg];
+    }
+    return cur;
+}
+function compare(c, node) {
+    const got = pick(node, c.field);
+    const want = c.value;
+    switch (c.op) {
+        case 'exists': return (got !== undefined && got !== null) === (want !== false);
+        case 'eq': return got === want;
+        case 'ne': return got !== want;
+        case 'in': return Array.isArray(want) && want.includes(got);
+        case 'contains': return Array.isArray(got) ? got.includes(want) : typeof got === 'string' && typeof want === 'string' && got.includes(want);
+        case 'startsWith': return typeof got === 'string' && typeof want === 'string' && got.startsWith(want);
+        case 'gt': return typeof got === 'number' && typeof want === 'number' && got > want;
+        case 'lt': return typeof got === 'number' && typeof want === 'number' && got < want;
+    }
+}
+function run(c, node, file) {
+    if ('all' in c) {
+        const rs = c.all.map((x) => run(x, node, file));
+        return rs.every((r) => r.ok) ? { ok: true, w: rs.flatMap((r) => r.w) } : NO;
+    }
+    if ('any' in c) {
+        const rs = c.any.map((x) => run(x, node, file)).filter((r) => r.ok);
+        return rs.length ? { ok: true, w: rs.flatMap((r) => r.w) } : NO;
+    }
+    if ('not' in c)
+        return run(c.not, node, file).ok ? NO : { ok: true, w: [] };
+    if ('some' in c || 'none' in c) {
+        const f = 'some' in c ? c.some : c.none;
+        const arr = pick(node, f);
+        const items = Array.isArray(arr) ? arr : [];
+        const hits = items.filter((it) => !c.where || run(c.where, it, isObj(it) && typeof it.file === 'string' ? it.file : file).ok);
+        if ('none' in c)
+            return hits.length === 0 ? { ok: true, w: [] } : NO;
+        if (!hits.length)
+            return NO;
+        const w = [];
+        for (const it of hits)
+            if (isObj(it) && typeof it.line === 'number')
+                w.push({ file: typeof it.file === 'string' ? it.file : file, line: it.line });
+        return { ok: true, w };
+    }
+    return compare(c, node) ? { ok: true, w: [] } : NO;
+}
+// Runs the query on the lead's entry point. The lead is confirmed only when the entry point matches and at least one fact the
+// query matched (a data touch, a protection, a read or an outbound call) sits on or beside a line the lead cites. A query that
+// matches every entry point of a map of TOO_BROAD_MIN or more proves nothing and is not accepted.
+export function runQuery(map, q, entryId, cited) {
+    const ep = map.entry_points.find((e) => e.id === entryId);
+    if (!ep)
+        return { confirmed: false, detail: 'the entry point is not in the code map', witnesses: [] };
+    const at = (e) => run(q.where, e, e.handler?.file ?? e.file);
+    const mine = at(ep);
+    if (!mine.ok)
+        return { confirmed: false, detail: 'the query does not match this entry point', witnesses: [] };
+    const matched = map.entry_points.filter((e) => at(e).ok).length;
+    if (map.entry_points.length >= TOO_BROAD_MIN && matched === map.entry_points.length) {
+        return { confirmed: false, detail: `the query matches all ${matched} entry points, so it proves nothing about this one`, witnesses: [] };
+    }
+    const near = mine.w.filter((w) => cited.some((s) => s.file === w.file && Math.abs(s.line - w.line) <= WITNESS_SLACK));
+    if (!near.length)
+        return { confirmed: false, detail: 'no fact the query matched sits on a line the lead cites', witnesses: [] };
+    const uniq = [...new Map(near.map((w) => [`${w.file}:${w.line}`, w])).values()].sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : a.line - b.line));
+    return { confirmed: true, detail: `the query matches this entry point and ${matched - 1} other of ${map.entry_points.length}; matched facts at ${uniq.map((w) => `${w.file}:${w.line}`).join(', ')}`, witnesses: uniq };
+}

package/dist/confirm/rule.js

@@ -0,0 +1,86 @@
+// Confirmation by a generated Semgrep rule. The rule must match the lines the lead cites in the real file, and it must not match a
+// negative control: a safe sample of the same kind of code that the model supplies. Both runs must really scan their file, so a rule
+// for another language cannot pass by matching nothing. Semgrep runs offline with metrics off. If Semgrep is not installed the
+// outcome is "not run" and the lead stays a lead.
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+const TIMEOUT_MS = 120_000;
+const SLACK = 1;
+export const spawnSemgrep = (args) => {
+    const r = spawnSync('semgrep', args, { encoding: 'utf8', timeout: TIMEOUT_MS, maxBuffer: 32 * 1024 * 1024, env: { PATH: process.env.PATH ?? '', HOME: process.env.HOME ?? '', SEMGREP_SEND_METRICS: 'off', SEMGREP_ENABLE_VERSION_CHECK: '0' } });
+    if (r.error?.code === 'ENOENT')
+        return { status: null, stdout: '', missing: true };
+    return { status: r.status, stdout: r.stdout ?? '' };
+};
+const BANNED = /pattern-where-python|pattern-sources:\s*\n\s*-\s*exact|\binclude:|r2c-internal/;
+function scanFile(run, rule, target) {
+    const r = run(['scan', '--config', rule, '--json', '--quiet', '--metrics=off', '--disable-version-check', '--no-git-ignore', '--timeout', '10', target]);
+    if (r.missing)
+        return { missing: true };
+    let out;
+    try {
+        out = JSON.parse(r.stdout);
+    }
+    catch {
+        return { fail: 'semgrep printed no JSON for this rule' };
+    }
+    if (!Array.isArray(out.results))
+        return { fail: 'semgrep printed no results list' };
+    if (Array.isArray(out.errors) && out.errors.length)
+        return { fail: 'semgrep reported an error for the rule or the file' };
+    if (!(out.paths?.scanned ?? []).some((p) => resolve(p) === resolve(target)))
+        return { fail: 'semgrep did not scan the file (the rule does not cover its language)' };
+    return { out };
+}
+export function confirmRule(lead, repo, run = spawnSemgrep) {
+    const p = lead.evidence_proposal;
+    if (p.kind !== 'semgrep_rule')
+        return { status: 'not_confirmed', detail: 'no rule was proposed' };
+    const ids = [...p.rule_yaml.matchAll(/^\s*-\s*id:\s*([^\s#]+)/gm)].map((m) => m[1]);
+    if (!/^\s*rules:/m.test(p.rule_yaml) || ids.length !== 1)
+        return { status: 'not_confirmed', detail: 'the rule file must hold exactly one rule' };
+    if (!/^[A-Za-z0-9._-]{1,100}$/.test(ids[0]))
+        return { status: 'not_confirmed', detail: 'the rule id has characters outside letters, digits, dot, underscore and hyphen' };
+    if (BANNED.test(p.rule_yaml))
+        return { status: 'not_confirmed', detail: 'the rule uses a feature the engine does not accept' };
+    const norm = (s) => s.replace(/\s+/g, ' ').trim();
+    if (norm(p.control_code).includes(norm(lead.quoted_code)))
+        return { status: 'not_confirmed', detail: 'the negative control still holds the quoted code, so it is not a control' };
+    const sink = lead.path[lead.path.length - 1];
+    const dir = mkdtempSync(join(tmpdir(), 'hp-rule-'));
+    try {
+        const ruleFile = join(dir, 'rule.yaml');
+        const controlFile = join(dir, `control.${p.control_extension}`);
+        writeFileSync(ruleFile, p.rule_yaml);
+        writeFileSync(controlFile, p.control_code);
+        const ver = run(['--version']);
+        if (ver.missing)
+            return { status: 'not_run', detail: 'Semgrep is not installed' };
+        const hit = scanFile(run, ruleFile, join(resolve(repo), sink.file));
+        if ('missing' in hit)
+            return { status: 'not_run', detail: 'Semgrep is not installed' };
+        if ('fail' in hit)
+            return { status: 'not_confirmed', detail: hit.fail };
+        const onSink = hit.out.results.some((r) => r.end.line >= sink.line - SLACK && r.start.line <= sink.line + SLACK);
+        if (!onSink)
+            return { status: 'not_confirmed', detail: `the rule does not match the cited lines (${sink.file}:${sink.line})` };
+        const ctl = scanFile(run, ruleFile, controlFile);
+        if ('missing' in ctl)
+            return { status: 'not_run', detail: 'Semgrep is not installed' };
+        if ('fail' in ctl)
+            return { status: 'not_confirmed', detail: `negative control: ${ctl.fail}` };
+        if (ctl.out.results.length)
+            return { status: 'not_confirmed', detail: 'the rule also matches the negative control, so it does not tell the flaw from safe code' };
+        return {
+            status: 'confirmed', detail: `the rule matches ${sink.file}:${sink.line} and does not match the negative control`, rule_id: `hunt.${ids[0]}`.slice(0, 190),
+            tool_version: (/^\d+\.\d+[\w.+-]*/m.exec(ver.stdout)?.[0] ?? 'unknown').replace(/[^A-Za-z0-9._+-]/g, '-').slice(0, 59) || 'unknown',
+            config_sha256: createHash('sha256').update(p.rule_yaml).digest('hex'),
+        };
+    }
+    finally {
+        rmSync(dir, { recursive: true, force: true });
+    }
+}

package/dist/confirm/store.js

@@ -0,0 +1,42 @@
+// Where hunt files live and how they are read and written: <out>/<commit>/hunt/. Every answer is written the moment it arrives,
+// through a temporary file and a rename, so an interrupted run leaves whole files only and can resume.
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { cmp, stableJson } from '../util.js';
+export const huntDir = (scanDir) => join(scanDir, 'hunt');
+const slug = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'entry';
+export const answerFile = (hunt, run, entryId) => join(hunt, `run-${run}`, `${slug(entryId)}-${createHash('sha256').update(entryId).digest('hex').slice(0, 8)}.json`);
+function writeAtomic(path, text) {
+    mkdirSync(join(path, '..'), { recursive: true });
+    const tmp = `${path}.tmp`;
+    writeFileSync(tmp, text);
+    renameSync(tmp, path);
+}
+const readJson = (p) => { try {
+    return JSON.parse(readFileSync(p, 'utf8'));
+}
+catch (e) {
+    throw new Error(`${p} cannot be read: ${e.message}`);
+} };
+export const writeAnswer = (hunt, a) => writeAtomic(answerFile(hunt, a.run, a.entry_id), stableJson(a));
+export const hasAnswer = (hunt, run, entryId) => existsSync(answerFile(hunt, run, entryId));
+export function readAnswers(hunt) {
+    if (!existsSync(hunt))
+        return [];
+    const out = [];
+    for (const d of readdirSync(hunt).filter((n) => /^run-\d+$/.test(n)).sort(cmp)) {
+        for (const f of readdirSync(join(hunt, d)).filter((n) => n.endsWith('.json')).sort(cmp))
+            out.push(readJson(join(hunt, d, f)));
+    }
+    return out.sort((a, b) => a.run - b.run || cmp(a.entry_id, b.entry_id));
+}
+export const writePlan = (hunt, p) => writeAtomic(join(hunt, 'plan.json'), stableJson(p));
+export const readPlan = (hunt) => (existsSync(join(hunt, 'plan.json')) ? readJson(join(hunt, 'plan.json')) : null);
+export const writeResult = (hunt, r) => writeAtomic(join(hunt, 'result.json'), stableJson(r));
+export const readResult = (hunt) => (existsSync(join(hunt, 'result.json')) ? readJson(join(hunt, 'result.json')) : null);
+export const writeSynthesis = (hunt, s) => writeAtomic(join(hunt, 'synthesis.json'), stableJson(s));
+export const readSynthesis = (hunt) => (existsSync(join(hunt, 'synthesis.json')) ? readJson(join(hunt, 'synthesis.json')) : null);
+export const writeSkipped = (hunt, reason) => writeAtomic(join(hunt, 'skipped.json'), stableJson({ schema_version: '1.0.0', reason }));
+export const readSkipped = (hunt) => (existsSync(join(hunt, 'skipped.json')) ? readJson(join(hunt, 'skipped.json')).reason : null);
+export const hasHuntFiles = (hunt) => existsSync(hunt) && readdirSync(hunt).some((n) => /^run-\d+$/.test(n) || n === 'plan.json');

package/dist/confirm/summary.js

@@ -0,0 +1,6 @@
+export function summary(r) {
+    return [
+        `hullproof hunt: ${r.confirmed.length} confirmed, ${r.leads.length} leads not confirmed, ${r.hallucinations.count} hallucinations dropped, ${r.malformed.count} malformed`,
+        `hullproof hunt: calls ${r.calls.complete} complete, ${r.calls.incomplete} incomplete, ${r.calls.missing} not saved; tokens ${r.usage.input_tokens} in, ${r.usage.output_tokens} out (reported, not limited)`,
+    ];
+}

package/dist/confirm/synthesis.js

@@ -0,0 +1,28 @@
+import { cmp } from '../util.js';
+import { plain } from './lead.js';
+export const SYNTHESIS_LIMITS = { summary: 4000, explanation: 800, explanations: 60 };
+export function guardSynthesis(result, reply, meta) {
+    const known = new Set([...result.confirmed, ...result.leads].map((l) => l.key));
+    const o = reply && typeof reply === 'object' && !Array.isArray(reply) ? reply : {};
+    const summary = typeof o.summary === 'string' ? plain(o.summary, SYNTHESIS_LIMITS.summary) : '';
+    const seen = new Set();
+    const explanations = [];
+    if (Array.isArray(o.explanations)) {
+        for (const x of o.explanations) {
+            if (!x || typeof x !== 'object')
+                continue;
+            const { key, text } = x;
+            if (typeof key !== 'string' || typeof text !== 'string' || !known.has(key) || seen.has(key) || explanations.length >= SYNTHESIS_LIMITS.explanations)
+                continue;
+            seen.add(key);
+            explanations.push({ key, text: plain(text, SYNTHESIS_LIMITS.explanation) });
+        }
+    }
+    explanations.sort((a, b) => cmp(a.key, b.key));
+    return { schema_version: '1.0.0', status: 'complete', stop: null, detail: null, provider: meta.provider, model: meta.model, summary, explanations, usage: meta.usage };
+}
+// A saved synthesis is applied again on replay against the result just computed: explanations for keys that no longer exist drop out.
+export function reguard(result, s) {
+    const g = guardSynthesis(result, { summary: s.summary, explanations: s.explanations }, { provider: s.provider, model: s.model, usage: s.usage });
+    return { ...g, status: s.status, stop: s.stop, detail: s.detail };
+}

package/dist/discover/detectors.js

@@ -0,0 +1,50 @@
+// Loads the detector rule files (detectors/*.json). Detectors are data; this file only reads and checks them.
+// The format is documented in docs/INTERFACES.md, section "Detector rule files".
+import { createHash } from 'node:crypto';
+import { readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { STACK_CATEGORIES } from '../types/stack.js';
+import { cmp } from '../util.js';
+export const DETECTORS_DIR = fileURLToPath(new URL('../../detectors/', import.meta.url));
+const CONF = new Set(['high', 'medium', 'low']);
+function check(name, f) {
+    const bad = (msg) => { throw new Error(`detectors/${name}: ${msg}`); };
+    if (!STACK_CATEGORIES.includes(f.category))
+        bad(`unknown category ${f.category}`);
+    const ids = new Set();
+    for (const e of f.entries ?? bad('no entries')) {
+        if (!/^[a-z0-9][a-z0-9.-]*$/.test(e.id ?? ''))
+            bad(`bad id ${e.id}`);
+        if (ids.has(e.id))
+            bad(`duplicate id ${e.id}`);
+        ids.add(e.id);
+        if (!e.deps && !e.imports && !e.signals && !e.extensions)
+            bad(`${e.id} has no signal`);
+        const dc = typeof e.deps_confidence === 'object' ? Object.values(e.deps_confidence) : [e.deps_confidence];
+        for (const c of [...dc, e.imports_confidence, ...(e.signals ?? []).map((s) => s.confidence)]) {
+            if (c !== undefined && !CONF.has(c))
+                bad(`${e.id} bad confidence ${c}`);
+        }
+        for (const s of e.signals ?? []) {
+            if (!Array.isArray(s.files) || s.files.length === 0)
+                bad(`${e.id} signal without files`);
+            for (const r of [s.regex, ...(s.all ?? [])])
+                if (r !== undefined)
+                    new RegExp(r); // throws on a bad pattern
+        }
+    }
+}
+export function loadDetectors(dir = DETECTORS_DIR) {
+    const names = readdirSync(dir).filter((n) => n.endsWith('.json')).sort(cmp);
+    const sums = [];
+    const files = [];
+    for (const n of names) {
+        const raw = readFileSync(join(dir, n));
+        sums.push(`${createHash('sha256').update(raw).digest('hex')}  ${n}\n`);
+        const f = JSON.parse(raw.toString('utf8'));
+        check(n, f);
+        files.push(f);
+    }
+    return { files, sha256: createHash('sha256').update(sums.join('')).digest('hex') };
+}

package/dist/discover/discover.js

@@ -0,0 +1,270 @@
+// Stack discovery: deterministic, no model, no network, runs no program. Lists files, extracts facts, applies the detector
+// rules, and builds stack.json. Same files and same detector files give the same bytes.
+import { closeSync, openSync, readSync, fstatSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { STACK_CATEGORIES } from '../types/stack.js';
+import { cmp, globToRegExp, sortUnique } from '../util.js';
+import { ENGINE_VERSION } from '../version.js';
+import { loadDetectors } from './detectors.js';
+import { IMPORT_LANGS, MANIFEST_NAMES, extractDeps, extractImports, importLangOf, nameMatches } from './extract.js';
+import { readHeadCommit } from './git.js';
+import { walk } from './walk.js';
+export const LIMITS = { maxFiles: 200_000, maxBytesPerFile: 512 * 1024, maxImportFiles: 50_000, evidencePerItem: 5, languageSamples: 3 };
+// Separator for prefix matching of package names per ecosystem; '' means exact names only (use "x*" for a prefix).
+const DEP_SEP = {
+    npm: '/', composer: '/', go: '/', maven: ':', nuget: '.', cocoapods: '/', swiftpm: '/',
+    pypi: '', cargo: '', rubygems: '', pub: '', hex: '', terraform: '',
+};
+// Extensions that are data, docs, media or build artefacts: neither a language nor worth listing as unknown.
+const NOT_CODE = new Set(('md markdown mdx txt rst adoc json jsonc json5 yaml yml toml xml csv tsv ini cfg conf lock log ' +
+    'png jpg jpeg gif svg ico webp bmp tif tiff avif heic mp3 mp4 m4a wav ogg webm mov avi pdf zip gz tgz tar bz2 xz 7z rar jar war ' +
+    'woff woff2 ttf otf eot map snap example sample template tpl j2 properties plist pbxproj xcconfig xcscheme xcworkspacedata ' +
+    'storyboard xib entitlements strings pem crt cer key pub gitkeep keep sqlite sqlite3 db bin dat pkl pickle h5 npy ' +
+    'sum mod tfvars tfstate hcl lockb editorconfig env patch diff po pot mo xlsx xls docx doc pptx odt license gradle bat cmd ' +
+    'iml http rest bak orig tmp swp dockerignore gitignore gitattributes npmrc nvmrc prettierrc eslintrc babelrc ipynb').split(/\s+/));
+const LOCKFILE = /(^|\/)(package-lock\.json|npm-shrinkwrap\.json|yarn\.lock|pnpm-lock\.yaml|composer\.lock|packages\.lock\.json|[^/]+\.lock)$/;
+// Configuration files are looked for at the repository root and one level inside dot folders (.github, .do, .gitlab).
+const CONFIG_EXT = /\.(ya?ml|toml|json|jsonc|ini|cfg|conf|hcl|properties)$/;
+// Common tool configuration with no security meaning for discovery; not listed as unknown.
+const KNOWN_TOOLING = /(^|\/)(\.?(eslint|prettier|stylelint|babel|jest|nyc|mocha|lint-staged|commitlint|markdownlint|yamllint|remark|cspell|renovate|codecov|release-it|semantic-release|browserslist|swcrc|postcss|tailwind|editorconfig|husky|c8|ava|vitest|karma|protractor|typedoc|tslint|lerna|nx|turbo|crowdin|codeclimate|mailmap|devcontainer|dependabot|pre-commit-config|gitpod|vscode|readthedocs|mkdocs|coveralls|sonar-project|cypress|playwright|stryker|snyk|zap|ncurc|npmpackagejsonlintrc|jsdoc|jsconfig|typos)(rc)?(\.[^/]*)?|tsconfig[^/]*|package-lock\.json|composer\.lock|\.?github\/[^/]*\.ya?ml|\.?github\/FUNDING\.yml|bower\.json|manifest\.json)$/i;
+const TF_UTILITY = new Set(['random', 'null', 'local', 'tls', 'time', 'archive', 'template', 'external', 'http', 'terraform', 'cloudinit']);
+const depConf = (e, eco) => {
+    const c = e.deps_confidence;
+    return (typeof c === 'object' ? c[eco] ?? c['*'] : c) ?? 'high';
+};
+const RANK = { high: 0, medium: 1, low: 2 };
+function readHead(abs, max) {
+    const fd = openSync(abs, 'r');
+    try {
+        const size = fstatSync(fd).size;
+        const buf = Buffer.alloc(Math.min(size, max));
+        const n = readSync(fd, buf, 0, buf.length, 0);
+        return { text: buf.subarray(0, n).toString('utf8'), truncated: size > max };
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+const lineAt = (text, index) => {
+    let n = 1;
+    for (let i = 0; i < index; i++)
+        if (text.charCodeAt(i) === 10)
+            n++;
+    return n;
+};
+const extOf = (f) => {
+    const b = f.slice(f.lastIndexOf('/') + 1);
+    const d = b.lastIndexOf('.');
+    return d > 0 ? b.slice(d + 1).toLowerCase() : '';
+};
+export function discover(repo, opts = {}) {
+    const root = resolve(repo);
+    const det = opts.detectors ?? loadDetectors();
+    const listing = walk(root, LIMITS.maxFiles, opts.skipFiles);
+    const files = listing.files;
+    const cache = new Map();
+    const truncated = new Set();
+    const text = (f) => {
+        let t = cache.get(f);
+        if (t === undefined) {
+            try {
+                const r = readHead(join(root, f), LIMITS.maxBytesPerFile);
+                t = r.text;
+                if (r.truncated)
+                    truncated.add(f);
+            }
+            catch {
+                t = '';
+            }
+            cache.set(f, t);
+        }
+        return t;
+    };
+    // Facts.
+    const deps = [];
+    for (const f of files)
+        if (MANIFEST_NAMES.test(f))
+            deps.push(...extractDeps(f, text(f)));
+    const importFiles = files.filter((f) => importLangOf(f) !== null);
+    const scanned = importFiles.slice(0, LIMITS.maxImportFiles);
+    const imports = [];
+    for (const f of scanned) {
+        imports.push(...extractImports(f, text(f)));
+        cache.delete(f); // source text is not needed again unless a content signal asks for it
+    }
+    const depsByEco = Map.groupBy(deps, (d) => d.eco);
+    const importsByLang = Map.groupBy(imports, (i) => i.lang);
+    // Items.
+    const items = new Map();
+    const usedFiles = new Set();
+    const matchedTf = new Set();
+    const acc = (cat, id) => {
+        const k = `${cat}\u0000${id}`;
+        let a = items.get(k);
+        if (!a) {
+            a = { evidence: [], versions: new Set(), lists: new Map(), scalars: {} };
+            items.set(k, a);
+        }
+        return a;
+    };
+    const add = (a, ev) => { a.evidence.push(ev); usedFiles.add(ev.file); };
+    const addList = (a, attrs, file) => {
+        for (const [k, v] of Object.entries(attrs ?? {})) {
+            if (!a.lists.has(k))
+                a.lists.set(k, new Set());
+            a.lists.get(k).add(v === '$file' ? file : v);
+        }
+    };
+    const globCache = new Map();
+    const filesFor = (globs) => {
+        const key = globs.join('\n');
+        let r = globCache.get(key);
+        if (!r) {
+            const res = globs.map(globToRegExp);
+            r = files.filter((f) => res.some((re) => re.test(f)));
+            globCache.set(key, r);
+        }
+        return r;
+    };
+    const runSignal = (cat, e, s, i) => {
+        const kind = s.regex ? 'content' : 'file';
+        const rule = `${cat}.${e.id}.${kind}.${i + 1}`;
+        const conf = s.confidence ?? (s.regex ? 'medium' : 'high');
+        const flags = 'm' + (s.ignore_case ? 'i' : '');
+        const re = s.regex ? new RegExp(s.regex, flags) : null;
+        const all = (s.all ?? []).map((r) => new RegExp(r, flags));
+        for (const f of filesFor(s.files)) {
+            if (re && LOCKFILE.test(f))
+                continue; // lockfiles are never templates or config; reading them only costs time
+            if (!re) {
+                const a = acc(cat, e.id);
+                add(a, { file: f, key: '(present)', rule, confidence: conf });
+                addList(a, s.attrs, f);
+                continue;
+            }
+            const t = text(f);
+            const m = re.exec(t);
+            if (!m || !all.every((r) => r.test(t)))
+                continue;
+            const a = acc(cat, e.id);
+            add(a, { file: f, line: lineAt(t, m.index), rule, confidence: conf });
+            addList(a, s.attrs, f);
+        }
+    };
+    for (const df of det.files) {
+        const cat = df.category;
+        for (const e of df.entries) {
+            for (const [eco, names] of Object.entries(e.deps ?? {})) {
+                for (const d of depsByEco.get(eco) ?? []) {
+                    if (!names.some((n) => nameMatches(d.name, n, DEP_SEP[eco] ?? '')))
+                        continue;
+                    if (eco === 'terraform')
+                        matchedTf.add(d.name);
+                    const a = acc(cat, e.id);
+                    add(a, { file: d.file, ...(d.line !== undefined ? { line: d.line } : { key: d.key }), rule: `${cat}.${e.id}.dep.${eco}`, confidence: depConf(e, eco) });
+                    if (d.version)
+                        a.versions.add(`${d.name}@${d.version}`);
+                }
+            }
+            for (const [lang, mods] of Object.entries(e.imports ?? {})) {
+                const sep = IMPORT_LANGS[lang]?.sep ?? '';
+                for (const im of importsByLang.get(lang) ?? []) {
+                    if (!mods.some((m) => nameMatches(im.module, m, sep)))
+                        continue;
+                    add(acc(cat, e.id), { file: im.file, line: im.line, rule: `${cat}.${e.id}.import.${lang}`, confidence: e.imports_confidence ?? 'medium' });
+                }
+            }
+            (e.signals ?? []).forEach((s, i) => runSignal(cat, e, s, i));
+            if (e.extensions) {
+                const exts = new Set(e.extensions);
+                const hits = files.filter((f) => exts.has(extOf(f)));
+                if (hits.length) {
+                    const a = acc(cat, e.id);
+                    a.fileCount = hits.length;
+                    for (const f of hits)
+                        a.evidence.push({ file: f, key: '(extension)', rule: `${cat}.${e.id}.extension`, confidence: 'high' });
+                }
+            }
+            const a = items.get(`${cat}\u0000${e.id}`);
+            if (a)
+                Object.assign(a.scalars, e.attrs ?? {});
+        }
+    }
+    // Unknowns: never guessed, only listed.
+    const unknown = [];
+    const langExts = new Set(det.files.filter((f) => f.category === 'languages').flatMap((f) => f.entries.flatMap((e) => e.extensions ?? [])));
+    const byExt = new Map();
+    for (const f of files) {
+        const x = extOf(f);
+        if (!x || langExts.has(x) || NOT_CODE.has(x))
+            continue;
+        if (!byExt.has(x))
+            byExt.set(x, []);
+        byExt.get(x).push(f);
+    }
+    for (const [x, fs] of byExt)
+        unknown.push({ file: fs[0], note: `no detector for the file extension .${x} (${fs.length} file${fs.length === 1 ? '' : 's'})` });
+    for (const f of files) {
+        const parts = f.split('/');
+        const configPlace = parts.length === 1 || (parts.length === 2 && parts[0].startsWith('.'));
+        if (!configPlace || !CONFIG_EXT.test(f) || usedFiles.has(f) || KNOWN_TOOLING.test(f))
+            continue;
+        unknown.push({ file: f, note: 'configuration file with no detector' });
+    }
+    const tfSeen = new Set();
+    for (const d of depsByEco.get('terraform') ?? []) {
+        if (matchedTf.has(d.name) || TF_UTILITY.has(d.name) || tfSeen.has(d.name))
+            continue;
+        tfSeen.add(d.name);
+        unknown.push({ file: d.file, note: `Terraform provider "${d.name}" has no detector` });
+    }
+    const LOCKS = ['package-lock.json', 'npm-shrinkwrap.json', 'yarn.lock', 'pnpm-lock.yaml', 'bun.lockb', 'bun.lock'];
+    const fileSet = new Set(files);
+    for (const f of files.filter((x) => x === 'package.json' || x.endsWith('/package.json'))) {
+        const dir = f.slice(0, -'package.json'.length);
+        if (LOCKS.some((l) => fileSet.has(dir + l) || fileSet.has(l)))
+            continue;
+        unknown.push({ file: f, note: 'package.json with no lockfile in its folder or the repository root: package manager not determined' });
+    }
+    // Assemble, sorted.
+    const out = Object.fromEntries(STACK_CATEGORIES.map((c) => [c, []]));
+    for (const [k, a] of items) {
+        const [cat, id] = k.split('\u0000');
+        const seen = new Set();
+        const ev = a.evidence
+            .sort((x, y) => RANK[x.confidence] - RANK[y.confidence] || cmp(x.file, y.file) || (x.line ?? 0) - (y.line ?? 0) || cmp(x.key ?? '', y.key ?? '') || cmp(x.rule, y.rule))
+            .filter((x) => { const s = `${x.file}|${x.line}|${x.key}|${x.rule}`; return seen.has(s) ? false : (seen.add(s), true); });
+        const shown = cat === 'languages' && a.fileCount ? LIMITS.languageSamples : LIMITS.evidencePerItem;
+        // The sample shows the best entry of each file first, so it spans files instead of repeating one file.
+        const fileSeen = new Set();
+        const firsts = ev.filter((x) => !fileSeen.has(x.file) && !!fileSeen.add(x.file));
+        const firstSet = new Set(firsts);
+        const sample = [...firsts, ...ev.filter((x) => !firstSet.has(x))].slice(0, shown);
+        const item = { id, confidence: ev[0].confidence, evidence: sample, evidence_count: ev.length, ...a.scalars };
+        if (a.versions.size)
+            item.versions = sortUnique(a.versions);
+        if (a.fileCount)
+            item.file_count = a.fileCount;
+        for (const [lk, set] of a.lists)
+            item[lk] = sortUnique(set);
+        out[cat].push(item);
+    }
+    for (const c of STACK_CATEGORIES)
+        out[c].sort((x, y) => cmp(x.id, y.id));
+    return {
+        schema_version: '1.0.0',
+        engine_version: ENGINE_VERSION,
+        detectors_sha256: det.sha256,
+        commit: readHeadCommit(root),
+        confirmed_by_user: false,
+        files: { listed: files.length, scanned_for_imports: scanned.length },
+        limits: {
+            max_files: LIMITS.maxFiles, max_bytes_per_file: LIMITS.maxBytesPerFile,
+            file_cap_reached: listing.capReached || importFiles.length > scanned.length,
+            truncated_files: sortUnique(truncated),
+        },
+        skipped: listing.skipped.slice().sort((x, y) => cmp(x.path, y.path)),
+        ...out,
+        unknown: unknown.sort((x, y) => cmp(x.file, y.file) || cmp(x.note, y.note)),
+    };
+}