knodin 0.7.6 → 0.8.2

package/dist/src/engine/perf.js

@@ -0,0 +1,115 @@
+/** Opt-in phase timing used by the R43 benchmark harness. */
+export const PERF_PHASES = [
+    "db_open_migration",
+    "file_collection",
+    "parser_wasm_init",
+    "read",
+    "parse_extract",
+    "sqlite_write",
+    "model_load",
+    "embedding_inference_write",
+    "search_row_fetch_decode",
+    "similarity",
+    "fts",
+    "map_community",
+    "fusion",
+    "freshness_probe",
+    "graph_analytics",
+    "traversal_snapshot",
+    "architecture_facets",
+    "architecture_edge_materialization",
+    "context_composition",
+    "status_audit",
+    "pack_walk",
+    "pack_serialize",
+    "artifact_read",
+    "artifact_grep",
+    "review_diff_discovery",
+    "review_churn",
+    "review_parsing",
+    "review_flow_lookup",
+    "review_serialization",
+];
+let active = null;
+function emptyPhases() {
+    return Object.fromEntries(PERF_PHASES.map((phase) => [phase, 0]));
+}
+function peakRss() {
+    try {
+        const kilobytes = process.resourceUsage().maxRSS;
+        return Number.isFinite(kilobytes) && kilobytes >= 0 ? kilobytes * 1024 : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Start a measurement explicitly. Starting while active resets that measurement. */
+export function beginPerfSession() {
+    active = { startedAt: performance.now(), phases: emptyPhases(), peakRssBytes: peakRss() };
+}
+/** Finish the active measurement, or return null when instrumentation is disabled. */
+export function endPerfSession() {
+    const session = active;
+    if (!session)
+        return null;
+    active = null;
+    const currentRss = peakRss();
+    if (currentRss !== undefined) {
+        session.peakRssBytes = Math.max(session.peakRssBytes ?? 0, currentRss);
+    }
+    return {
+        phases: { ...session.phases },
+        totalMs: Math.max(0, performance.now() - session.startedAt),
+        ...(session.peakRssBytes === undefined ? {} : { peakRssBytes: session.peakRssBytes }),
+    };
+}
+function record(phase, startedAt) {
+    const session = active;
+    if (!session)
+        return;
+    session.phases[phase] += Math.max(0, performance.now() - startedAt);
+    const currentRss = peakRss();
+    if (currentRss !== undefined)
+        session.peakRssBytes = Math.max(session.peakRssBytes ?? 0, currentRss);
+}
+/** Start a phase that spans an existing try/finally block. The returned closer is idempotent. */
+export function beginPerfPhase(phase) {
+    if (!active)
+        return () => undefined;
+    const startedAt = performance.now();
+    let closed = false;
+    return () => {
+        if (closed)
+            return;
+        closed = true;
+        record(phase, startedAt);
+    };
+}
+/** Time an async phase. Nested phases are retained independently and errors still close timers. */
+export async function measurePerfPhase(phase, work) {
+    if (!active)
+        return work();
+    const startedAt = performance.now();
+    try {
+        return await work();
+    }
+    finally {
+        record(phase, startedAt);
+    }
+}
+/** Synchronous counterpart for filesystem, parsing, and SQLite calls. */
+export function measurePerfPhaseSync(phase, work) {
+    if (!active)
+        return work();
+    const startedAt = performance.now();
+    try {
+        return work();
+    }
+    finally {
+        record(phase, startedAt);
+    }
+}
+/** Test-only state reset; deliberately does not enable timing. */
+export function resetPerfForTests() {
+    active = null;
+}

package/dist/src/engine/prune.js

@@ -0,0 +1,112 @@
+import path from "node:path";
+/**
+ * Single source of truth for "which directories does knodin ignore" (R14).
+ *
+ * Before this module, the same intent — skip node_modules, .git, .knodin,
+ * build output, caches, VCS internals — was encoded THREE separate times:
+ *   1. src/__tests__/setup.ts's `PRUNE_DIRS` regex (glob-shim exclude filter)
+ *   2. the indexer's own directory filter (`EXCLUDED_DIR_SEGMENTS` / `isIndexablePath`)
+ *   3. the chokidar watcher's `makeWatchIgnorePredicate` (added by R13)
+ *
+ * They drifted. R13's bug was exactly this shape: the watcher's copy silently
+ * stopped matching anything for an entire chokidar major version while the
+ * indexer's copy kept working, so indexing looked correct and only the
+ * watcher melted (it registered ~12.6k files and starved the event loop under
+ * Bun). This module is the ONE list; every consumer below is a thin adapter
+ * over it, so a future drift is structurally impossible rather than merely
+ * discouraged.
+ */
+/**
+ * Directory segments never worth indexing or watching: build output, deps,
+ * VCS/tool state, caches. Matched by whole path SEGMENT, never substring, so
+ * a real file/dir like `distributor.ts` or `build-graph.ts` is never wrongly
+ * pruned.
+ */
+export const PRUNE_DIR_NAMES = new Set([
+    "node_modules",
+    ".git",
+    ".knodin",
+    // Local state maintained by other code-intelligence tools. It is neither
+    // repository source nor stable project configuration, and indexing it makes
+    // a healthy checkout look perpetually stale as those tools update databases
+    // and caches in the background.
+    ".gitnexus",
+    ".codegraph",
+    ".code-review-graph",
+    ".aider.tags.cache.v4",
+    "dist",
+    "build",
+    "coverage",
+    ".next",
+    ".turbo",
+    ".svelte-kit",
+    "vendor",
+    "__pycache__",
+]);
+/** True when a single path segment (one directory name) is a pruned directory. */
+export function isPrunedSegment(segment) {
+    return PRUNE_DIR_NAMES.has(segment);
+}
+/**
+ * Agent worktrees are checkouts of this same repo living under
+ * `.claude/worktrees/*`; indexing/watching them would double-count every
+ * symbol. `segments` is the full split relative path; `i` is the segment
+ * index currently being tested against the two-segment `.claude/worktrees`
+ * pattern.
+ */
+function isClaudeWorktreeSegment(segments, i) {
+    return segments[i] === ".claude" && segments[i + 1] === "worktrees";
+}
+/**
+ * True when ANY segment of an already-split repo-relative path trips the
+ * ignore policy (a pruned directory name, or the `.claude/worktrees` root).
+ * This is the one walk both `isIndexablePath` (indexer) and
+ * `makeWatchIgnorePredicate` (watcher) run — the code that used to be
+ * duplicated between them.
+ */
+export function pathSegmentsAreIgnored(segments) {
+    for (let i = 0; i < segments.length; i++) {
+        if (isPrunedSegment(segments[i]))
+            return true;
+        if (isClaudeWorktreeSegment(segments, i))
+            return true;
+    }
+    return false;
+}
+/**
+ * Indexer adapter: true when a repo-relative path (either OS separator)
+ * should be indexed.
+ */
+export function isIndexablePath(relPath) {
+    return !pathSegmentsAreIgnored(relPath.split(/[/\\]/));
+}
+/**
+ * Watcher adapter: builds chokidar's `ignored` predicate for a given repo
+ * root.
+ *
+ * chokidar v4 removed glob support; v5 accepts only a function, a regex, or a
+ * literal path. Matching here is done on the path RELATIVE to the repo, so a
+ * repo that happens to live under a directory named e.g. `build/` does not
+ * prune its own root (the R13 fix this preserves).
+ */
+export function makeWatchIgnorePredicate(repoPath) {
+    return (targetPath) => {
+        const relative = path.relative(repoPath, targetPath);
+        // The watch root itself ("") and anything outside it are never pruned.
+        if (relative === "" || relative.startsWith(".."))
+            return false;
+        return pathSegmentsAreIgnored(relative.split(path.sep));
+    };
+}
+/**
+ * Policy-test adapter: a regex equivalent to the segment-based policy above.
+ * Anchored on `/` boundaries so it only matches whole segments — same guarantee
+ * as `isPrunedSegment` — never a substring inside an unrelated name.
+ */
+export function makePruneDirsRegex() {
+    const escaped = [...PRUNE_DIR_NAMES].map((n) => n.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
+    // `.claude/worktrees` is a two-segment pattern, expressed directly rather
+    // than through the single-segment name list.
+    escaped.push("\\.claude/worktrees");
+    return new RegExp(`(?:^|/)(?:${escaped.join("|")})(?:/|$)`);
+}

package/dist/src/engine/sarif-import.js

@@ -0,0 +1,341 @@
+/**
+ * Bounded SARIF import.
+ *
+ * SARIF (OASIS "Static Analysis Results Interchange Format") is what most
+ * analyzers can emit — Salesforce Code Analyzer, ESLint, Semgrep, CodeQL, PMD,
+ * Trivy. Importing the format rather than any one tool's native output means one
+ * reader covers all of them.
+ *
+ * Findings are NOT graph facts. knodin resolves references from source; a
+ * finding is somebody else's judgement about a location. It is stored in its own
+ * table, tagged with the tool that produced it, and never merged into symbols or
+ * references. What the graph adds is the join: a finding on a hub matters more
+ * than the same finding on a leaf, and only the graph knows which is which.
+ *
+ * Everything here is bounded and fail-closed, matching the SCIP importer: a file
+ * over the byte ceiling, a malformed document, or a finding count over the cap
+ * is refused rather than partially applied.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { compareBytes } from "../compare.js";
+// Calibrated against real output rather than guessed: one Salesforce Code
+// Analyzer run over a mid-sized org (pmd + regex + retire-js) produced a 97 MB
+// SARIF log with ~75k findings. A 64 MB ceiling would have refused it.
+export const SARIF_DEFAULT_LIMITS = {
+    maxBytes: 256 * 1024 * 1024,
+    maxFindings: 250_000,
+    timeoutMs: 60_000,
+};
+/**
+ * The ceiling knodin cannot raise. `JSON.parse` needs the document as a single
+ * string and V8 caps string length at 2^29 - 24 bytes on 64-bit, so a log past
+ * this size cannot be read whole regardless of the configured limit. Kept
+ * slightly under the hard maximum to leave room for decoding overhead.
+ */
+const MAX_PARSEABLE_BYTES = 512 * 1024 * 1024;
+/** Byte counts appear in operator-facing errors, so render them readably. */
+function formatBytes(value) {
+    if (value < 1024)
+        return `${value} bytes`;
+    const units = ["KiB", "MiB", "GiB"];
+    let scaled = value / 1024;
+    let unit = 0;
+    while (scaled >= 1024 && unit < units.length - 1) {
+        scaled /= 1024;
+        unit++;
+    }
+    return `${scaled.toFixed(1)} ${units[unit]}`;
+}
+/** SARIF level is optional; analyzers often carry severity on the rule instead. */
+function normalizeLevel(value) {
+    if (value === "error" || value === "warning" || value === "note" || value === "none")
+        return value;
+    // SARIF's own default when level is absent and no rule default applies.
+    return "warning";
+}
+/**
+ * Resolves a SARIF artifact location to a repository-relative POSIX path.
+ *
+ * SARIF permits `file:///abs/path`, a bare absolute path, or a path relative to
+ * some base URI. Anything landing outside the repository is rejected: a findings
+ * file is external input, and a `../` escape must not attach a finding to a path
+ * the caller never indexed.
+ */
+function resolveArtifactPath(repoRoot, uri, uriBase) {
+    if (!uri)
+        return null;
+    let raw = uri;
+    if (raw.startsWith("file://")) {
+        try {
+            raw = decodeURIComponent(new URL(raw).pathname);
+        }
+        catch {
+            return null;
+        }
+    }
+    else {
+        try {
+            raw = decodeURIComponent(raw);
+        }
+        catch {
+            // Not percent-encoded; use as-is.
+        }
+    }
+    // A relative uri may be anchored by an originalUriBaseId value.
+    const anchor = uriBase && path.isAbsolute(uriBase) ? uriBase : repoRoot;
+    const candidate = path.isAbsolute(raw) ? raw : path.resolve(anchor, raw);
+    // Compare real paths. The analyzer may have recorded a route through a
+    // symlink — /var vs /private/var on macOS, or a symlinked checkout — and a
+    // textual prefix check would reject those valid locations. Resolving also
+    // means a symlink pointing outside the repository is still caught, because
+    // the real path is what gets compared.
+    let resolved = path.resolve(candidate);
+    try {
+        resolved = fs.realpathSync(resolved);
+    }
+    catch {
+        // Not on disk: the analyzer may report a file since deleted. Keep the
+        // lexically resolved path so containment is still enforced below.
+    }
+    if (resolved !== repoRoot && !resolved.startsWith(`${repoRoot}${path.sep}`))
+        return null;
+    const relative = path.relative(repoRoot, resolved);
+    if (!relative || relative.startsWith(".."))
+        return null;
+    return relative.split(path.sep).join("/");
+}
+/** Collects `originalUriBaseId` -> absolute path, used to anchor relative uris. */
+function readUriBases(run) {
+    const bases = new Map();
+    const raw = run.originalUriBaseIds;
+    if (!raw || typeof raw !== "object")
+        return bases;
+    for (const [id, entry] of Object.entries(raw)) {
+        const uri = entry?.uri;
+        if (typeof uri !== "string")
+            continue;
+        let value = uri;
+        if (value.startsWith("file://")) {
+            try {
+                value = decodeURIComponent(new URL(value).pathname);
+            }
+            catch {
+                continue;
+            }
+        }
+        bases.set(id, value);
+    }
+    return bases;
+}
+/** Rule metadata by id, so a finding can inherit a helpUri it does not carry. */
+function readRuleHelp(run) {
+    const help = new Map();
+    const driver = run.tool?.driver;
+    const rules = driver?.rules;
+    if (!Array.isArray(rules))
+        return help;
+    for (const rule of rules) {
+        const id = rule.id;
+        const uri = rule.helpUri;
+        if (typeof id === "string" && typeof uri === "string")
+            help.set(id, uri);
+    }
+    return help;
+}
+/** Stable order so repeated imports of an unchanged log agree exactly. */
+function compareFindings(left, right) {
+    if (left.filePath !== right.filePath)
+        return compareBytes(left.filePath, right.filePath);
+    if (left.startLine !== right.startLine)
+        return left.startLine - right.startLine;
+    return compareBytes(left.ruleId, right.ruleId);
+}
+/**
+ * Builds one finding from a SARIF result, or null when the result cannot be
+ * placed inside the repository. The caller counts those rather than guessing
+ * at a path.
+ */
+function readResult(result, context) {
+    const location = Array.isArray(result.locations) ? result.locations[0] : undefined;
+    const physical = location
+        ?.physicalLocation;
+    const artifact = physical?.artifactLocation;
+    const uri = typeof artifact?.uri === "string" ? artifact.uri : "";
+    const baseId = typeof artifact?.uriBaseId === "string" ? artifact.uriBaseId : undefined;
+    // Spec says uriBaseId is a key into originalUriBaseIds. Salesforce Code
+    // Analyzer instead puts the absolute workspace path there directly and
+    // omits originalUriBaseIds entirely, so accept either form.
+    const base = baseId ? (context.uriBases.get(baseId) ?? baseId) : undefined;
+    const filePath = resolveArtifactPath(context.repoRoot, uri, base);
+    if (!filePath)
+        return null;
+    const region = (physical?.region ?? {});
+    const startLine = Number.isInteger(region.startLine) ? region.startLine : 1;
+    const startColumn = Number.isInteger(region.startColumn) ? region.startColumn : 1;
+    const ruleIdRaw = result.ruleId;
+    const ruleId = typeof ruleIdRaw === "string" && ruleIdRaw ? ruleIdRaw : "unknown-rule";
+    const messageText = result.message?.text;
+    const helpUri = context.ruleHelp.get(ruleId);
+    return {
+        ruleId,
+        tool: context.tool,
+        level: normalizeLevel(result.level),
+        message: typeof messageText === "string" ? messageText : "",
+        filePath,
+        startLine,
+        startColumn,
+        // SARIF omits end positions for single-point results; collapse to start.
+        endLine: Number.isInteger(region.endLine) ? region.endLine : startLine,
+        endColumn: Number.isInteger(region.endColumn) ? region.endColumn : startColumn,
+        ...(helpUri ? { helpUri } : {}),
+    };
+}
+/**
+ * Validates the input against the size ceilings and parses it, returning the
+ * SARIF runs. Every refusal names the flag and a value that would admit the
+ * input, because a ceiling the operator cannot move is just a failure.
+ */
+function readRuns(displayPath, inputPath, size, maxBytes) {
+    if (size > maxBytes)
+        throw new Error(`knodin index --sarif: ${displayPath} is ${formatBytes(size)}, over the ` +
+            `${formatBytes(maxBytes)} ceiling. Raise it with ` +
+            `--sarif-max-bytes ${size}, or narrow the analysis that produced the log.`);
+    // A ceiling knodin cannot raise. JSON.parse needs the whole document as one
+    // string, and V8 refuses strings beyond roughly 512 MiB, so a larger log
+    // fails inside readFileSync no matter what --sarif-max-bytes says. Saying so
+    // is more useful than letting "Invalid string length" surface.
+    if (size > MAX_PARSEABLE_BYTES)
+        throw new Error(`knodin index --sarif: ${displayPath} is ${formatBytes(size)}, beyond the ` +
+            `${formatBytes(MAX_PARSEABLE_BYTES)} limit this runtime can parse as one document. ` +
+            `Raising --sarif-max-bytes will not help. Split the analysis by rule selector or ` +
+            `workspace and import each log, which is additive across tools.`);
+    let document;
+    try {
+        document = JSON.parse(fs.readFileSync(inputPath, "utf8"));
+    }
+    catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        // A resource failure is not a malformed document, and reporting it as one
+        // sends the reader to inspect a file that is probably fine.
+        if (error instanceof RangeError || /heap|memory|string length/i.test(detail))
+            throw new Error(`knodin index --sarif: ran out of memory reading ${displayPath} ` +
+                `(${formatBytes(size)}). The document is held whole while parsing, so this is a ` +
+                `runtime limit rather than a knodin one. Retry with a larger heap, for example ` +
+                `NODE_OPTIONS=--max-old-space-size=8192, or split the analysis into several logs.`);
+        throw new Error(`knodin index --sarif: ${displayPath} is not valid JSON (${detail})`);
+    }
+    const runs = document?.runs;
+    if (!Array.isArray(runs))
+        throw new Error("knodin index --sarif: input has no runs array; not a SARIF log");
+    return runs;
+}
+/** Applies the defaults for any ceiling the caller left unset. */
+function resolveLimits(options) {
+    return {
+        maxBytes: options.maxBytes ?? SARIF_DEFAULT_LIMITS.maxBytes,
+        maxFindings: options.maxFindings ?? SARIF_DEFAULT_LIMITS.maxFindings,
+        timeoutMs: options.timeoutMs ?? SARIF_DEFAULT_LIMITS.timeoutMs,
+    };
+}
+/**
+ * Reads one SARIF run into findings, or null when the value is not a run that
+ * carries results at all.
+ *
+ * `skipped` counts results whose location could not be placed inside the
+ * repository; the caller reports that total rather than discarding it.
+ *
+ * `budget.carried` is how many findings earlier runs already contributed, so
+ * the ceiling can be enforced against the running total from inside the loop.
+ * It is checked on every finding rather than once per run because the ceiling
+ * exists to stop the work, not to report on it afterwards: a SARIF log commonly
+ * holds exactly one run, so a per-run check would only fire once the entire log
+ * was already accumulated — the case the ceiling is there to prevent.
+ */
+function readRun(rawRun, repoRoot, budget) {
+    if (!rawRun || typeof rawRun !== "object")
+        return null;
+    const run = rawRun;
+    const results = run.results;
+    if (!Array.isArray(results))
+        return null;
+    const driverName = run.tool?.driver?.name;
+    const tool = typeof driverName === "string" && driverName ? driverName : "unknown";
+    const context = {
+        repoRoot,
+        tool,
+        uriBases: readUriBases(run),
+        ruleHelp: readRuleHelp(run),
+    };
+    const findings = [];
+    let skipped = 0;
+    for (const rawResult of results) {
+        if (!rawResult || typeof rawResult !== "object")
+            continue;
+        const finding = readResult(rawResult, context);
+        // A finding we cannot place inside the repository is counted, not guessed at.
+        if (!finding) {
+            skipped++;
+            continue;
+        }
+        findings.push(finding);
+        if (budget.carried + findings.length > budget.maxFindings)
+            throw new Error(`knodin index --sarif: more than ${budget.maxFindings.toLocaleString()} findings. ` +
+                `Raise it with --sarif-max-findings, or narrow the analysis that produced the log.`);
+    }
+    return { tool, findings, skipped };
+}
+/**
+ * Reads and validates a SARIF log, returning findings resolved against the
+ * repository. Throws rather than returning partial facts.
+ */
+export function readSarifLog(repoPath, options) {
+    const started = Date.now();
+    const limits = resolveLimits(options);
+    const repoRoot = fs.realpathSync(path.resolve(repoPath));
+    const inputPath = path.resolve(options.path);
+    const stat = fs.statSync(inputPath, { throwIfNoEntry: false });
+    if (!stat?.isFile())
+        throw new Error(`knodin index --sarif: ${options.path} is not a file`);
+    const runs = readRuns(options.path, inputPath, stat.size, limits.maxBytes);
+    const findings = [];
+    const tools = new Set();
+    const rules = new Set();
+    const files = new Set();
+    let skippedUnresolved = 0;
+    for (const rawRun of runs) {
+        if (Date.now() - started > limits.timeoutMs)
+            throw new Error(`knodin index --sarif: exceeded the ${limits.timeoutMs}ms read budget. ` +
+                `Raise it with --sarif-timeout-ms.`);
+        // readRun enforces the findings ceiling against this running total as it
+        // reads, so it throws on the offending finding rather than after the run.
+        const run = readRun(rawRun, repoRoot, {
+            carried: findings.length,
+            maxFindings: limits.maxFindings,
+        });
+        if (!run)
+            continue;
+        skippedUnresolved += run.skipped;
+        // A tool is only recorded once it has placed a finding, so a run that
+        // resolved nothing does not advertise coverage it did not deliver.
+        if (run.findings.length === 0)
+            continue;
+        tools.add(run.tool);
+        for (const finding of run.findings) {
+            findings.push(finding);
+            rules.add(finding.ruleId);
+            files.add(finding.filePath);
+        }
+    }
+    findings.sort(compareFindings);
+    return {
+        inputPath,
+        bytes: stat.size,
+        tools: [...tools].sort(compareBytes),
+        rules: rules.size,
+        files: files.size,
+        findings,
+        skippedUnresolved,
+        elapsedMs: Date.now() - started,
+    };
+}