@holmes-lab/holmes-kit 0.1.0

package/dist/holmes/cpg/cpg-scanner.js

@@ -0,0 +1,213 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CpgScanner = exports.isTestFile = exports.TEST_FILE_PATTERNS = exports.SCANNABLE_EXTENSIONS = exports.SKIP = void 0;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const language_parser_1 = require("./language-parser");
+const source_path_1 = require("./source-path");
+/**
+ * Directories that never contain hand-written source for the project under analysis.
+ *
+ * The original set (`node_modules`, `dist`, `.git`) was JS-shaped, and that silently made the
+ * scanner unusable on other ecosystems: measured on a real brownfield Python target, a scan
+ * ingested **20,631 files instead of 315** — 98.5% of it third-party code inside `.venv`. That is
+ * not merely slow; the RTM graph fills with library symbols, clustering becomes meaningless, and
+ * reverse-engineered specs would describe someone else's code.
+ *
+ * Entries here must be *unambiguous* vendor or cache directories. Names that are conventional but
+ * genuinely ambiguous as source roots — `build`, `out`, `bin`, `obj`, `target`, `lib` — are
+ * deliberately EXCLUDED: silently dropping real source is the worse failure, and `isVendorDir`
+ * below catches the case that actually matters through evidence rather than by name.
+ */
+exports.SKIP = new Set([
+    // JS/TS
+    'node_modules', 'dist', '.next', '.nuxt', '.turbo',
+    // Python
+    '.venv', 'venv', 'site-packages', '__pycache__', '.tox', '.nox',
+    '.mypy_cache', '.pytest_cache', '.ruff_cache', '.eggs',
+    // Go / Rust / JVM / .NET tool caches
+    'vendor', '.gradle', '.cargo',
+    // VCS + generic tool output
+    '.git', '.hg', '.svn', 'coverage', '.idea', '.vscode',
+]);
+/**
+ * Evidence-based virtualenv detection. A Python virtualenv is defined by containing `pyvenv.cfg`,
+ * whatever the user named the directory (`env`, `.env311`, `myproject-venv`, …), so name matching
+ * alone misses them. Checking for the marker file is precise: no real source tree has one at its
+ * root. Same idea for a vendored `site-packages` nested under an unconventionally named env.
+ */
+function isVendorDir(dir) {
+    try {
+        return fs.existsSync(path.join(dir, 'pyvenv.cfg'));
+    }
+    catch {
+        return false;
+    }
+}
+const IMPL = /@implements\s+(A-SPEC-\d{3,}(?:\.\d+)?)/g;
+// Well-formed governed-but-non-A-SPEC ids used (wrongly) as an @implements
+// anchor. Deliberately requires the full "KIND-NNN" shape (kind + '-' +
+// 3+ digits) so prose/comments like `@implements A-SPEC refs` (no id number)
+// or a bare word never match — only a genuine wrong-kind spec-id does.
+const WRONG_KIND_IMPL = /@implements\s+((?:REQ|H-SPEC|C-SPEC|T-SPEC)-\d{3,}(?:\.\d+)?)/g;
+/** Single source of truth for which file extensions CpgScanner ingests (REQ-124 gate 2). */
+exports.SCANNABLE_EXTENSIONS = ['.ts', '.mts', '.cts', '.tsx', '.jsx', '.js', '.mjs', '.cjs', '.py', '.cs', '.java', '.go', '.rs', '.cpp', '.cc', '.cxx', '.hpp', '.hh', '.h'];
+const SCANNABLE_RE = /\.(ts|mts|cts|tsx|jsx|js|mjs|cjs|py|cs|java|go|rs|cpp|cc|cxx|hpp|hh|h)$/;
+// The test-file table moved to ./test-files, so the PreToolUse phase gate can read the SAME rules
+// without pulling the scanner (and tree-sitter) into a latency-critical path. Re-exported here
+// because this module was its published home and remains the natural place to look for it.
+var test_files_1 = require("./test-files");
+Object.defineProperty(exports, "TEST_FILE_PATTERNS", { enumerable: true, get: function () { return test_files_1.TEST_FILE_PATTERNS; } });
+Object.defineProperty(exports, "isTestFile", { enumerable: true, get: function () { return test_files_1.isTestFile; } });
+const test_files_2 = require("./test-files");
+// .ts/.mts/.cts can contain TS type-assertion syntax (`<T>x`, arrow-generics)
+// that the 'tsx' grammar misreads as JSX, so they must stay on 'typescript'.
+// Everything else in SCANNABLE_EXTENSIONS (.tsx/.jsx/.js/.mjs/.cjs) never uses
+// that syntax, and JSX-in-.js is common (React "escape hatch", older
+// Next.js pages) — the 'tsx' grammar parses plain JS/TS byte-identically to
+// 'typescript' for our walk while also correctly handling JSX, so route the
+// whole JS/JSX family there.
+const TSX_GRAMMAR_RE = /\.(tsx|jsx|js|mjs|cjs)$/;
+// @implements A-SPEC-121.1
+class CpgScanner {
+    parser;
+    cache;
+    opts;
+    // Optional N7 scan cache: when supplied, unchanged files (matching content hash) skip the
+    // tree-sitter parse entirely and reuse their cached symbols — file-key incremental scanning.
+    constructor(parser = new language_parser_1.TreeSitterTsParser(), cache, 
+    // @implements A-SPEC-140.1 — see ScannedFile.dataflow for why this is opt-in.
+    opts = {}) {
+        this.parser = parser;
+        this.cache = cache;
+        this.opts = opts;
+    }
+    // @implements A-SPEC-131
+    // The recorded-skip channel: per-file failures used to vanish in the catch below, which is how
+    // handlers.ts (37KB, over the parse cliff) stayed outside the graph for the project's entire
+    // history. The report is per-scan (reset on entry), path-sorted, and a second channel — scan()'s
+    // signature and every existing consumer are untouched.
+    skipped = [];
+    /** `{file, reason}` for every file the LAST scan claimed but could not ingest. Empty on a clean tree. */
+    scanSkipped() {
+        return [...this.skipped].sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0));
+    }
+    scan(rootDir, repoRoot = rootDir) {
+        this.skipped = [];
+        const out = [];
+        const walk = (d) => {
+            let entries;
+            try {
+                entries = fs.readdirSync(d, { withFileTypes: true });
+            }
+            catch {
+                return;
+            }
+            for (const e of entries) {
+                if (exports.SKIP.has(e.name))
+                    continue;
+                const p = path.join(d, e.name);
+                if (e.isDirectory()) {
+                    if (isVendorDir(p))
+                        continue;
+                    walk(p);
+                }
+                else if (SCANNABLE_RE.test(e.name) && !(0, test_files_2.isTestFile)(path.relative(rootDir, p).split(path.sep).join('/'))) {
+                    // Test detection needs the path, not the basename: a Rust integration test is `tests/x.rs`
+                    // and a JVM one is `src/test/java/X.java` — nothing in the filename says "test". Relative
+                    // to the SCAN ROOT, so a repository that merely lives under some `/…/test/` directory on
+                    // disk is not mistaken for one big test tree.
+                    try {
+                        const code = fs.readFileSync(p, 'utf8');
+                        // @implements A-SPEC-121.5
+                        // @implements A-SPEC-121.6
+                        // .h is routed to the C++ grammar (not a separate C parser): the
+                        // C++ grammar parses plain C headers acceptably for our lexical
+                        // symbol walk, and disambiguating a bare `.h` between C and C++
+                        // is out of scope for this slice.
+                        const lang = /\.py$/.test(e.name) ? 'python' : (/\.cs$/.test(e.name) ? 'csharp' : (/\.java$/.test(e.name) ? 'java' : (/\.go$/.test(e.name) ? 'go' : (/\.rs$/.test(e.name) ? 'rust' : (/\.(cpp|cc|cxx|hpp|hh|h)$/.test(e.name) ? 'cpp' : (TSX_GRAMMAR_RE.test(e.name) ? 'tsx' : 'typescript'))))));
+                        const implementsSpecs = [...code.matchAll(IMPL)].map(m => m[1]);
+                        const unanchoredImplements = [...code.matchAll(WRONG_KIND_IMPL)].map(m => m[1]);
+                        const sourcePath = (0, source_path_1.toSourcePath)(repoRoot, path.resolve(p));
+                        // @implements A-SPEC-140.1
+                        // The third parse happens only when asked for. `undefined` from the parser (a language
+                        // with no walk) is carried through as `undefined`, not coerced to an empty fact set.
+                        const dataflow = () => (this.opts.dataflow ? this.parser.extractDataFlow(code, lang) : undefined);
+                        // N7 incremental: on a content-hash hit, reuse cached symbols and skip the parse.
+                        if (this.cache) {
+                            const { ScanFileCache } = require('./scan-cache');
+                            const h = ScanFileCache.hashOf(code);
+                            const hit = this.cache.get(sourcePath, h);
+                            // @implements A-SPEC-140.1
+                            // A hit written by a scan that did NOT extract data-flow cannot satisfy one that does:
+                            // serving it would hand the taint pass zero facts, which is indistinguishable from
+                            // "no tainted flows". Correctness over hit rate — re-extract and re-store instead.
+                            // The language check keeps this from re-parsing every Go file on every scan forever.
+                            const usable = hit && !(this.opts.dataflow && hit.dataflow === undefined && (0, language_parser_1.hasDataFlowWalk)(lang));
+                            if (usable) {
+                                out.push({ path: p, ...hit });
+                                continue;
+                            }
+                            const fresh = { sourcePath, symbols: this.parser.extractSymbols(code, lang), edges: this.parser.extractEdges(code, lang), dataflow: dataflow(), implementsSpecs, unanchoredImplements };
+                            this.cache.put(sourcePath, h, fresh);
+                            out.push({ path: p, ...fresh });
+                            continue;
+                        }
+                        out.push({ path: p, sourcePath, symbols: this.parser.extractSymbols(code, lang), edges: this.parser.extractEdges(code, lang), dataflow: dataflow(), implementsSpecs, unanchoredImplements });
+                    }
+                    catch (err) {
+                        // A single unreadable/unparseable file must not abort a whole-tree scan — but it must
+                        // not VANISH either (REQ-131: the old bare skip is how a >32K file stayed outside the
+                        // graph, anchors and all, for the project's entire history). Infrastructure failures
+                        // still propagate: under Jest the worker path can throw TreeSitterInfraError
+                        // (timeout/crash/desync), and a dead worker is not a property of the file.
+                        if (err instanceof language_parser_1.TreeSitterInfraError)
+                            throw err;
+                        // repoRoot-relative, matching sourcePath's path-key convention — a skip entry must be
+                        // comparable with scan results and change sets even when rootDir is a subtree.
+                        const rel = path.relative(repoRoot, p).split(path.sep).join('/');
+                        const reason = (err instanceof Error ? err.message : String(err)).slice(0, 200);
+                        this.skipped.push({ file: rel, reason });
+                    }
+                }
+            }
+        };
+        walk(rootDir);
+        this.cache?.save(); // batched persist + mark-and-sweep prune, once per scan
+        return out;
+    }
+}
+exports.CpgScanner = CpgScanner;

package/dist/holmes/cpg/hash-cache.js

@@ -0,0 +1,86 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HashCache = void 0;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const node_crypto_1 = require("node:crypto");
+// @implements A-SPEC-113.1
+/**
+ * Content-hash skip cache used by the reindex path to avoid re-scanning
+ * files whose content hasn't changed since the last recorded hash. Keyed by
+ * repo-relative path, persisted as a single JSON file under `dir` (callers
+ * pass `.ax/cpg_cache`, which is gitignored — cache files are derived, never
+ * committed).
+ *
+ * Deterministic: SHA-256 of the exact content string, no timestamps or other
+ * nondeterministic inputs recorded.
+ */
+class HashCache {
+    dir;
+    file;
+    hashes;
+    constructor(dir) {
+        this.dir = dir;
+        this.file = path.join(dir, 'hashes.json');
+        this.hashes = this.load();
+    }
+    load() {
+        try {
+            const raw = fs.readFileSync(this.file, 'utf8');
+            return JSON.parse(raw);
+        }
+        catch {
+            // Missing dir/file, or unreadable/corrupt JSON: treat as no prior hash
+            // rather than throwing — a cold or damaged cache must never block a
+            // reindex, only make it fall back to a full re-scan.
+            return {};
+        }
+    }
+    static hashOf(content) {
+        return (0, node_crypto_1.createHash)('sha256').update(content, 'utf8').digest('hex');
+    }
+    /** True iff a hash was previously stored for relPath and it matches content's hash. */
+    unchanged(relPath, content) {
+        const stored = this.hashes[relPath];
+        return stored !== undefined && stored === HashCache.hashOf(content);
+    }
+    put(relPath, content) {
+        this.hashes[relPath] = HashCache.hashOf(content);
+        fs.mkdirSync(this.dir, { recursive: true });
+        fs.writeFileSync(this.file, JSON.stringify(this.hashes, null, 2), 'utf8');
+    }
+}
+exports.HashCache = HashCache;