@holmes-lab/holmes-kit 0.4.0 → 0.4.1

package/dist/holmes/rtm/taint-benchmark.js

@@ -1,6 +1,41 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TAINT_CASES = exports.TAINT_CONFIG = void 0;
+exports.runTaintBenchmarkAsync = runTaintBenchmarkAsync;
+exports.compareEngines = compareEngines;
 exports.runTaintBenchmark = runTaintBenchmark;
 exports.taintOnRealCorpus = taintOnRealCorpus;
 // @implements A-SPEC-354
@@ -78,7 +113,67 @@ exports.TAINT_CASES = [
         code: `function h(req) { const c = req.body.cmd; return c.length; }`,
     },
 ];
-/** Run the current engine over the labelled cases and score it. */
+// @implements A-SPEC-512.1
+/**
+ * The flow-SENSITIVE verdict for one case, judged on reaching definitions (the CPG foundation's
+ * first consumer). Async because the wasm substrate parses; `runTaintBenchmarkAsync` is the entry
+ * point that can use it, while the synchronous `runTaintBenchmark` keeps the original engine.
+ */
+async function flowSensitiveVerdict(code) {
+    const { parseAst } = await Promise.resolve().then(() => __importStar(require('../cpg/foundation/ast-store')));
+    const { cfgOf, functionsIn } = await Promise.resolve().then(() => __importStar(require('../cpg/foundation/cfg')));
+    const { ddgOf } = await Promise.resolve().then(() => __importStar(require('../cpg/foundation/ddg')));
+    const { flowSensitiveTaint } = await Promise.resolve().then(() => __importStar(require('./flow-sensitive-taint')));
+    const ast = await parseAst(code, 'case.ts');
+    if (!ast)
+        return false;
+    for (const fn of functionsIn(ast)) {
+        const cfg = cfgOf(ast, fn, code);
+        if ('unsupported' in cfg)
+            continue;
+        const ddg = ddgOf(ast, cfg, fn, code);
+        if ('unsupported' in ddg)
+            continue;
+        if (flowSensitiveTaint({ ast, cfg, ddg, source: code, fnName: 'case', config: exports.TAINT_CONFIG }).findings.length > 0) {
+            return true;
+        }
+    }
+    return false;
+}
+// @implements A-SPEC-512.1
+/** Score the corpus with EITHER engine — an improvement is reported as a contrast, never alone. */
+async function runTaintBenchmarkAsync(cases = exports.TAINT_CASES, engine = 'flow-insensitive') {
+    if (engine === 'flow-insensitive')
+        return runTaintBenchmark(cases);
+    const outcomes = [];
+    for (const c of cases) {
+        const actual = await flowSensitiveVerdict(c.code);
+        const cell = c.vulnerable ? (actual ? 'tp' : 'fn') : (actual ? 'fp' : 'tn');
+        outcomes.push({ name: c.name, difficulty: c.difficulty, expected: c.vulnerable, actual, cell });
+    }
+    return scoreOutcomes(outcomes);
+}
+// @implements A-SPEC-512.1
+/** before/after in ONE call — the single truth both the report and its test consume. */
+async function compareEngines(cases = exports.TAINT_CASES) {
+    const before = await runTaintBenchmarkAsync(cases, 'flow-insensitive');
+    const after = await runTaintBenchmarkAsync(cases, 'flow-sensitive');
+    const moved = after.outcomes
+        .map((a, i) => ({ name: a.name, from: before.outcomes[i].cell, to: a.cell }))
+        .filter((m) => m.from !== m.to);
+    return { before, after, moved };
+}
+/** Cells → metrics. Shared so both engines are scored by exactly the same arithmetic. */
+function scoreOutcomes(outcomes) {
+    const count = (k) => outcomes.filter((o) => o.cell === k).length;
+    const tp = count('tp'), tn = count('tn'), fp = count('fp'), fn = count('fn');
+    return {
+        outcomes, tp, tn, fp, fn,
+        precision: tp + fp === 0 ? null : tp / (tp + fp),
+        recall: tp + fn === 0 ? null : tp / (tp + fn),
+        controlFlowAttributable: outcomes.filter((o) => (o.cell === 'fp' || o.cell === 'fn') && o.difficulty !== 'flow-free').length,
+    };
+}
 function runTaintBenchmark(cases = exports.TAINT_CASES) {
     const parser = new language_parser_1.TreeSitterTsParser();
     const outcomes = [];
@@ -94,14 +189,7 @@ function runTaintBenchmark(cases = exports.TAINT_CASES) {
             : (actual ? 'fp' : 'tn');
         outcomes.push({ name: c.name, difficulty: c.difficulty, expected: c.vulnerable, actual, cell });
     }
-    const count = (k) => outcomes.filter((o) => o.cell === k).length;
-    const tp = count('tp'), tn = count('tn'), fp = count('fp'), fn = count('fn');
-    return {
-        outcomes, tp, tn, fp, fn,
-        precision: tp + fp === 0 ? null : tp / (tp + fp),
-        recall: tp + fn === 0 ? null : tp / (tp + fn),
-        controlFlowAttributable: outcomes.filter((o) => (o.cell === 'fp' || o.cell === 'fn') && o.difficulty !== 'flow-free').length,
-    };
+    return scoreOutcomes(outcomes);
 }
 /**
  * How often the taint analysis fires on REAL code.

package/dist/holmes/rtm/taint-vocabulary.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Taint vocabulary, per language — because "we found nothing" and "we do not speak this language"
+ * are different sentences and the tool was saying the first while meaning the second.
+ *
+ * Measured on jarvis (27,232 Python files) 2026-09-02: the shipped list matched `subprocess.Popen`
+ * and `exec(`, and missed `subprocess.run` (7 files), `os.environ` (5) and `input(` (4) — the three
+ * most common Python spellings. The engine already handled Python (its CFG/DDG landed in S-510.7);
+ * it simply had no Python words to look for.
+ *
+ * The lists are DATA and the judgement is one function: adding a language grows this table, not the
+ * code. And the vocabulary is SELECTED by file language, never unioned — a union would let Python's
+ * `input(` taint an ordinary TypeScript `input` and each language would inherit the other's false
+ * positives.
+ */
+import type { DataFlowTaintConfig } from './dataflow-taint';
+export declare const TAINT_VOCABULARY: Record<'typescript' | 'python', DataFlowTaintConfig>;
+/**
+ * The vocabulary this file should be judged with, or null when the file is outside the analysable
+ * set. Null is deliberate: degrading an unknown language to the TypeScript list would produce
+ * findings in a language whose control flow we cannot even build.
+ */
+export declare function vocabularyFor(relPath: string): DataFlowTaintConfig | null;
+/** Languages this vocabulary table covers — the matrix derives its taint row from this. */
+export declare const TAINT_LANGUAGES: ReadonlySet<string>;

package/dist/holmes/rtm/taint-vocabulary.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TAINT_LANGUAGES = exports.TAINT_VOCABULARY = void 0;
+exports.vocabularyFor = vocabularyFor;
+/**
+ * TypeScript / JavaScript — value-identical to the shipped list, which is what makes "no regression
+ * for TS" checkable rather than asserted (a test compares the two).
+ */
+const TYPESCRIPT_VOCABULARY = {
+    sources: ['getenv', 'argv', 'req.body', 'request', 'stdin', 'readfile', 'readinput', 'query.get'],
+    sinks: ['exec', 'eval', 'system', 'query', 'spawn', 'popen', 'deserialize'],
+    sanitizers: ['escape', 'sanitize', 'encodeuricomponent'],
+};
+/**
+ * Python. Sinks are matched on the callee's LAST segment, which is why `subprocess.run` appears as
+ * `run` and `pickle.loads` as `loads`. Sources are matched against expression TEXT, so they carry
+ * their qualifier (`os.environ`) — that qualifier is what keeps a local variable named `environ`
+ * from becoming a source.
+ *
+ * `input` is listed with its opening parenthesis on purpose: bare `input` is an extremely common
+ * identifier (a parameter, a field, a React prop in mixed repos), so only the CALL is a source.
+ * That decision closes H-SPEC-513's open question, and it is the same "the act, not the name" rule
+ * the sink matcher follows.
+ */
+const PYTHON_VOCABULARY = {
+    sources: [
+        'os.environ', 'os.getenv', 'sys.argv', 'sys.stdin', 'input(',
+        'request.args', 'request.form', 'request.json', 'request.data', 'request.values',
+    ],
+    sinks: [
+        'run', 'call', 'check_output', 'check_call', // subprocess.*
+        'Popen', 'popen', 'system', // subprocess.Popen / os.popen / os.system
+        'exec', 'eval', 'compile', // built-in code execution
+        'loads', 'load', // pickle / yaml deserialization
+    ],
+    sanitizers: ['quote', 'escape', 'shlex.quote', 're.escape'],
+};
+exports.TAINT_VOCABULARY = {
+    typescript: TYPESCRIPT_VOCABULARY,
+    python: PYTHON_VOCABULARY,
+};
+const TS_EXT = /\.(ts|mts|cts|tsx|js|mjs|cjs|jsx)$/i;
+const PY_EXT = /\.py$/i;
+// @implements A-SPEC-513.1
+/**
+ * The vocabulary this file should be judged with, or null when the file is outside the analysable
+ * set. Null is deliberate: degrading an unknown language to the TypeScript list would produce
+ * findings in a language whose control flow we cannot even build.
+ */
+function vocabularyFor(relPath) {
+    if (TS_EXT.test(relPath))
+        return exports.TAINT_VOCABULARY.typescript;
+    if (PY_EXT.test(relPath))
+        return exports.TAINT_VOCABULARY.python;
+    return null;
+}
+/** Languages this vocabulary table covers — the matrix derives its taint row from this. */
+exports.TAINT_LANGUAGES = new Set(Object.keys(exports.TAINT_VOCABULARY));

package/package.json

@@ -1,7 +1,7 @@
 {
   "//": "@implements A-SPEC-209",
   "name": "@holmes-lab/holmes-kit",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "Holmes-Kit — deterministic Agentic Software Engineering (ASE) harness with causal traceability (spec chain + D-CPG + RTM + phase guardrail)",
   "main": "dist/holmes/mcp/server.js",
   "types": "dist/holmes/mcp/server.d.ts",