brainclaw 1.9.1 → 1.10.0

package/dist/core/code-map/finalizer.js

@@ -0,0 +1,191 @@
+/**
+ * Code Map P1a — CORE finalizer (spec §3, §6; dec#108 #1, dec#109 P0 #4 / P1 #5/#6/#7).
+ *
+ * The finalizer is the ONE AND ONLY identity authority. Providers hand it typed,
+ * id-free {@link ExtractionDraft}s; this module reproduces TODAY'S exact legacy
+ * `extractor.ts` output: byte-identical node/edge IDs (via `ids.ts`), spans,
+ * confidence, the `exported` flag, and — critically — the LEGACY SOURCE-APPEND
+ * ORDER. Nothing here sorts node/edge content: draft items carry an `ordinal`
+ * (their traversal position) and the finalizer replays them in that order.
+ *
+ * Emission contract (mirrors the legacy `addSymbol` / `addModule` / `markOrAddExport`):
+ *  - File node FIRST.
+ *  - Per definition (ascending ordinal): symbol node, `contains` edge, `defines` edge.
+ *  - Per import / re-export source: `module` node, `imports` edge.
+ *  - Per export clause / default-id (`markOrAddExport`): if the name matches an
+ *    already-emitted symbol, set that node `exported=true` + emit ONE `exports`
+ *    edge to it (no new node); otherwise fabricate an `export`-subtype symbol
+ *    (node + `contains` + `defines`) then emit the `exports` edge to it.
+ *
+ * The `exported` FLAG is NOT an `exports` EDGE: in-place exported declarations set
+ * only `node.exported=true`; only export clauses / default-identifier exports emit
+ * an `exports` edge (and they ALSO flip the referenced node's flag, matching legacy).
+ *
+ * Output nodes/edges are validated against the `types.ts` zod schemas before return.
+ */
+import { edgeId, fileNodeId, nodeId } from './ids.js';
+import { EdgeSchema, NodeSchema } from './types.js';
+/** Compute the legacy `sym:<hash>` node id (mirrors `extractor.ts:symId`). */
+function symNodeId(projectId, path, lang, subtype, name, span) {
+    return `sym:${nodeId({
+        projectId,
+        path,
+        lang,
+        kind: 'symbol',
+        subtype,
+        name,
+        startLine: span.start_line,
+        startCol: span.start_col,
+    })}`;
+}
+/** Compute the legacy `module:<hash>` node id (mirrors `extractor.ts:addModule`). */
+function moduleNodeId(projectId, path, lang, source, span) {
+    return `module:${nodeId({
+        projectId,
+        path,
+        lang,
+        kind: 'module',
+        subtype: null,
+        name: source,
+        startLine: span.start_line,
+        startCol: span.start_col,
+    })}`;
+}
+/**
+ * Turn a provider draft into the final {@link ExtractResult}. The ONLY identity
+ * authority — reproduces the legacy extractor output exactly (see file header).
+ *
+ * `input` supplies `projectId`/`path`/`lang` (the id inputs). `parseStatus`
+ * defaults to `'parsed'`; provider diagnostics ride on `draft.facts`.
+ */
+export function finalize(draft, input) {
+    const { projectId, path, lang } = input;
+    const fileNode = fileNodeId(projectId, path, lang);
+    const nodes = [
+        {
+            id: fileNode,
+            kind: 'file',
+            subtype: null,
+            lang,
+            name: path,
+            path,
+            span: null,
+            exported: false,
+            confidence: 1.0,
+            related_memory_ids: [],
+            imported_names: [],
+        },
+    ];
+    const edges = [];
+    // symbol name -> node id, mirroring the legacy `ctx.byName` (used by export
+    // clauses to mark-or-add). Last writer per name wins, exactly like legacy.
+    const byName = new Map();
+    // node id -> index in `nodes`, so an export clause can flip `exported` in place.
+    const nodeIndexById = new Map();
+    const pushSymbol = (subtype, name, span, exported, confidence) => {
+        const id = symNodeId(projectId, path, lang, subtype, name, span);
+        nodeIndexById.set(id, nodes.length);
+        nodes.push({
+            id,
+            kind: 'symbol',
+            subtype,
+            lang,
+            name,
+            path,
+            span,
+            exported,
+            confidence,
+            related_memory_ids: [],
+            imported_names: [],
+        });
+        byName.set(name, id);
+        edges.push({
+            id: edgeId({ projectId, from: fileNode, to: id, kind: 'contains' }),
+            from: fileNode,
+            to: id,
+            kind: 'contains',
+            confidence: 1.0,
+            source: { path, line: span.start_line },
+        });
+        edges.push({
+            id: edgeId({ projectId, from: fileNode, to: id, kind: 'defines' }),
+            from: fileNode,
+            to: id,
+            kind: 'defines',
+            confidence: 1.0,
+            source: { path, line: span.start_line },
+        });
+        return id;
+    };
+    // Build a single ordinal-ordered stream across all draft kinds so the finalizer
+    // replays the legacy source-append order without ever sorting node/edge content.
+    const items = [
+        ...draft.definitions.map((ref) => ({ kind: 'def', ordinal: ref.ordinal, ref })),
+        ...draft.imports.map((ref) => ({ kind: 'import', ordinal: ref.ordinal, ref })),
+        ...draft.exports.map((ref) => ({ kind: 'export', ordinal: ref.ordinal, ref })),
+    ].sort((a, b) => a.ordinal - b.ordinal);
+    for (const item of items) {
+        if (item.kind === 'def') {
+            const d = item.ref;
+            pushSymbol(d.subtype, d.name, d.span, d.exported === true, d.confidence ?? 1.0);
+        }
+        else if (item.kind === 'import') {
+            const im = item.ref;
+            const id = moduleNodeId(projectId, path, lang, im.source, im.span);
+            nodeIndexById.set(id, nodes.length);
+            nodes.push({
+                id,
+                kind: 'module',
+                subtype: null,
+                lang,
+                name: im.source,
+                path,
+                span: im.span,
+                exported: false,
+                confidence: im.confidence ?? 1.0,
+                related_memory_ids: [],
+                imported_names: [...im.importedNames],
+            });
+            edges.push({
+                id: edgeId({ projectId, from: fileNode, to: id, kind: 'imports' }),
+                from: fileNode,
+                to: id,
+                kind: 'imports',
+                confidence: 1.0,
+                source: { path, line: im.span.start_line },
+            });
+        }
+        else {
+            // export clause / default-identifier — legacy `markOrAddExport`.
+            const ex = item.ref;
+            const existing = byName.get(ex.name);
+            let target;
+            if (existing) {
+                const idx = nodeIndexById.get(existing);
+                if (idx !== undefined)
+                    nodes[idx] = { ...nodes[idx], exported: true };
+                target = existing;
+            }
+            else {
+                target = pushSymbol('export', ex.name, ex.span, true, ex.confidence ?? 1.0);
+            }
+            edges.push({
+                id: edgeId({ projectId, from: fileNode, to: target, kind: 'exports' }),
+                from: fileNode,
+                to: target,
+                kind: 'exports',
+                confidence: 1.0,
+                source: { path, line: ex.span.start_line },
+            });
+        }
+    }
+    const parseStatus = draft.attributes?.parseStatus ?? 'parsed';
+    const diagnostics = draft.facts.map((f) => ({ ...f }));
+    // Validate the finalized output against the durable schemas (spec §6).
+    for (const n of nodes)
+        NodeSchema.parse(n);
+    for (const e of edges)
+        EdgeSchema.parse(e);
+    return { parseStatus, nodes, edges, diagnostics };
+}
+//# sourceMappingURL=finalizer.js.map

package/dist/core/code-map/freshness.js

@@ -0,0 +1,108 @@
+/**
+ * Write-side freshness hashing + per-shard freshness classification
+ * (spec §5.1, §6.2, §12.4).
+ *
+ * The READ-path lazy freshness check (§6.1) is Sprint 3 — NOT implemented here.
+ * This module owns only:
+ *  - `computeExtractorConfigHash` — sha256 of a stable serialization of
+ *    extractor_config + the active language set + (P1a) the registry's
+ *    `configHashInputs()` (provider versions + every query-asset hash). Changing
+ *    ignore rules, size caps, supported extensions, query budget, active langs, a
+ *    provider version, OR a tags/imports `.scm` => stale_extractor.
+ *    NOTE: grammar/engine hashes are deliberately NOT folded in (spec §6.2):
+ *    stale_grammar (changed parse binary) is kept separable from stale_extractor.
+ *  - `shardFreshnessStatus` — classify a stored shard against the current
+ *    extractor_config_hash + per-language grammar hashes.
+ */
+import crypto from 'node:crypto';
+/** Stable serialization: sort object keys recursively so hashing is order-independent. */
+function stableStringify(value) {
+    if (value === null || typeof value !== 'object')
+        return JSON.stringify(value);
+    if (Array.isArray(value))
+        return `[${value.map(stableStringify).join(',')}]`;
+    const obj = value;
+    const keys = Object.keys(obj).sort();
+    return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`).join(',')}}`;
+}
+/**
+ * spec §5.1 / §9 — sha256 of (extractor_config + active language set + the
+ * registry's provider/query-asset fingerprint). The active language set is the
+ * sorted list of enabled languages, so enabling/disabling a language invalidates
+ * affected shards as stale_extractor. `registryInputs` (from
+ * `registry.configHashInputs()`) folds in provider `version` + every tags/imports
+ * `.scm` hash, so editing a query asset flips affected shards to stale_extractor
+ * (dec#109 P0#3). Optional + omitted-vs-undefined hash the same so legacy callers
+ * (config-only) keep a stable hash for that input combination.
+ */
+export function computeExtractorConfigHash(config, activeLanguages, registryInputs) {
+    const payload = {
+        extractor_config: config,
+        active_languages: [...activeLanguages].sort(),
+        registry: registryInputs ?? null,
+    };
+    return `sha256:${crypto.createHash('sha256').update(stableStringify(payload)).digest('hex')}`;
+}
+/**
+ * spec §12.4 — classify a stored shard:
+ *  - extractor_config_hash mismatch => stale_extractor
+ *  - tree_sitter_grammar_hash mismatch => stale_grammar
+ *  - otherwise fresh (content/path drift is the §6.1 read-path concern, Sprint 3)
+ *
+ * Precedence: extractor first, then grammar — both are "the binary/logic that
+ * produced this shard changed", and the badge only needs to surface one reason;
+ * extractor-config drift is the cheaper, more common cause.
+ */
+export function shardFreshnessStatus(input) {
+    const { shard } = input;
+    if (shard.extractor_config_hash !== input.currentExtractorConfigHash) {
+        return 'stale_extractor';
+    }
+    const expectedGrammar = input.grammarHashFor(shard.lang);
+    if (expectedGrammar !== undefined &&
+        shard.tree_sitter_grammar_hash != null &&
+        shard.tree_sitter_grammar_hash !== expectedGrammar) {
+        return 'stale_grammar';
+    }
+    return 'fresh';
+}
+/**
+ * Roll per-shard freshness up into a manifest-level freshness summary
+ * (spec §5.1). `missing_index` when nothing parsed; otherwise the dominant
+ * stale reason, else fresh.
+ */
+export function summarizeFreshness(shards) {
+    if (shards.length === 0) {
+        return { status: 'missing_index', stale_file_count: 0, partial_reason: null };
+    }
+    let staleExtractor = 0;
+    let staleGrammar = 0;
+    let staleChanged = 0;
+    for (const s of shards) {
+        switch (s.freshness.status) {
+            case 'stale_extractor':
+                staleExtractor++;
+                break;
+            case 'stale_grammar':
+                staleGrammar++;
+                break;
+            case 'stale_changed_files':
+                staleChanged++;
+                break;
+            default:
+                break;
+        }
+    }
+    const staleTotal = staleExtractor + staleGrammar + staleChanged;
+    if (staleTotal === 0) {
+        return { status: 'fresh', stale_file_count: 0, partial_reason: null };
+    }
+    // Surface the dominant reason for the manifest badge.
+    let status = 'stale_changed_files';
+    if (staleExtractor >= staleGrammar && staleExtractor >= staleChanged)
+        status = 'stale_extractor';
+    else if (staleGrammar >= staleChanged)
+        status = 'stale_grammar';
+    return { status, stale_file_count: staleTotal, partial_reason: null };
+}
+//# sourceMappingURL=freshness.js.map

package/dist/core/code-map/importable.js

@@ -0,0 +1,35 @@
+import { defaultImportableSymbol } from './lang/provider.js';
+/**
+ * Build `name -> importable symbol candidates` for one target file. `nodes` is the
+ * target shard's node list; only `kind: 'symbol'` nodes are considered, and the full
+ * symbol set is passed to the provider hook (Python needs it for top-level span
+ * containment).
+ */
+export function buildImportableIndex(nodes, provider) {
+    const symbols = nodes.filter((n) => n.kind === 'symbol');
+    const predicate = provider?.isImportableSymbol
+        ? (n) => provider.isImportableSymbol(n, symbols)
+        : defaultImportableSymbol;
+    const byName = new Map();
+    for (const n of symbols) {
+        if (!predicate(n))
+            continue;
+        const arr = byName.get(n.name);
+        if (arr)
+            arr.push(n);
+        else
+            byName.set(n.name, [n]);
+    }
+    return byName;
+}
+/**
+ * Resolve one imported name to its UNAMBIGUOUS importable symbol, or null when the
+ * name is absent OR matches more than one importable candidate (ambiguous → skip).
+ */
+export function lookupImportable(index, name) {
+    const cands = index.get(name);
+    if (!cands || cands.length !== 1)
+        return null;
+    return cands[0];
+}
+//# sourceMappingURL=importable.js.map

package/dist/core/code-map/indexes.js

@@ -0,0 +1,197 @@
+/**
+ * Code Map index builders (spec §5.6, §5.7).
+ *
+ * Both indexes are derived purely from `files/**` shards and written atomically
+ * (store.ts uses writeFileAtomic). Queries answer from these + shards alone;
+ * materialized JSONL is never required.
+ *
+ * Ordering is deterministic so two refreshes over identical inputs produce
+ * byte-identical indexes (concurrency rule 5 spirit; helps "no JSONL committed"
+ * diffs stay clean).
+ */
+import { CODE_MAP_SCHEMA_VERSION, } from './types.js';
+import { fileNodeId } from './ids.js';
+/** Lowercase token normalization (spec §5.6 keys). */
+function tokenize(name) {
+    const lower = name.toLowerCase();
+    const tokens = new Set();
+    tokens.add(lower);
+    // split camelCase / snake / kebab boundaries into sub-tokens for partial recall
+    for (const part of name.split(/[^A-Za-z0-9]+/)) {
+        if (!part)
+            continue;
+        // camelCase split
+        for (const sub of part.replace(/([a-z0-9])([A-Z])/g, '$1 $2').split(/\s+/)) {
+            if (sub)
+                tokens.add(sub.toLowerCase());
+        }
+    }
+    return [...tokens];
+}
+export function buildSymbolsIndex(projectId, shards, extractorVersion) {
+    // Null-proto map: symbol-name tokens can collide with Object.prototype members
+    // (e.g. a method named `constructor`, or a `__proto__` key), which would make
+    // `entries[token] ??= []` see the inherited function and crash on .push, or make
+    // `entries['__proto__'] = …` mutate the prototype instead of adding a key.
+    const entries = Object.create(null);
+    // Deterministic shard order by path.
+    const ordered = [...shards].sort((a, b) => a.path.localeCompare(b.path));
+    for (const shard of ordered) {
+        for (const node of shard.nodes) {
+            if (node.kind !== 'symbol')
+                continue;
+            const entry = {
+                node_id: node.id,
+                name: node.name,
+                kind: node.kind,
+                subtype: node.subtype ?? null,
+                path: node.path,
+                file_id: shard.file_id,
+                score_hint: node.exported ? 1.0 : 0.8,
+            };
+            for (const token of tokenize(node.name)) {
+                (entries[token] ??= []).push(entry);
+            }
+        }
+    }
+    // Deterministic ordering within each token bucket.
+    for (const token of Object.keys(entries)) {
+        entries[token].sort((a, b) => a.path.localeCompare(b.path) ||
+            a.name.localeCompare(b.name) ||
+            a.node_id.localeCompare(b.node_id));
+    }
+    // Sort keys for byte-stable output.
+    const sortedEntries = Object.create(null);
+    for (const key of Object.keys(entries).sort())
+        sortedEntries[key] = entries[key];
+    return {
+        schema_version: CODE_MAP_SCHEMA_VERSION,
+        project_id: projectId,
+        updated_at: new Date().toISOString(),
+        extractor_version: extractorVersion,
+        entries: sortedEntries,
+    };
+}
+export function buildImportsIndex(projectId, shards) {
+    // module specifier -> (path -> entry)
+    const byModule = new Map();
+    const ordered = [...shards].sort((a, b) => a.path.localeCompare(b.path));
+    for (const shard of ordered) {
+        for (const node of shard.nodes) {
+            if (node.kind !== 'module')
+                continue;
+            const module = node.name;
+            const perPath = byModule.get(module) ?? new Map();
+            const entry = perPath.get(shard.path) ?? {
+                path: shard.path,
+                file_id: shard.file_id,
+                imported: [],
+            };
+            // Merge imported bindings across multiple imports of the same module in one
+            // file, deduped + sorted for byte-stable output (spec §5.7 imported[]).
+            const merged = new Set(entry.imported);
+            for (const name of node.imported_names ?? [])
+                merged.add(name);
+            entry.imported = [...merged].sort();
+            perPath.set(shard.path, entry);
+            byModule.set(module, perPath);
+        }
+    }
+    const entries = Object.create(null);
+    for (const key of [...byModule.keys()].sort()) {
+        const list = [...byModule.get(key).values()].sort((a, b) => a.path.localeCompare(b.path));
+        entries[key] = list;
+    }
+    return {
+        schema_version: CODE_MAP_SCHEMA_VERSION,
+        project_id: projectId,
+        updated_at: new Date().toISOString(),
+        entries,
+    };
+}
+/**
+ * P1d reverse-dependency index: "who imports this target". Derived from the P1c
+ * resolution edges that live on each IMPORTER's shard:
+ *  - `resolves_to` (module → target FILE node) → `dependents_by_file[targetPath]`
+ *  - `imports_symbol` (module → target SYMBOL node) → `dependents_by_symbol[nodeId]`
+ *
+ * `resolves_to.to` is a file NODE ID (not a path), so we invert fileNodeId→path by
+ * computing the id for every indexed shard (Codex review). One entry per (target,
+ * importer file): multiple module nodes in one importer that hit the same target are
+ * merged (imported names unioned, strongest confidence, lexicographically-smallest
+ * specifier) for byte-stable output. Deterministic key + array ordering.
+ */
+export function buildResolutionIndex(projectId, shards) {
+    // Invert file-node id → path so reverse resolves_to can be keyed by target path.
+    const fileNodeIdToPath = new Map();
+    for (const shard of shards) {
+        fileNodeIdToPath.set(fileNodeId(projectId, shard.path, shard.lang), shard.path);
+    }
+    // target key -> (importer path -> merged entry)
+    const byFile = new Map();
+    const bySymbol = new Map();
+    const addDependent = (bucket, targetKey, importerPath, importerFileId, module, imported, confidence) => {
+        const perImporter = bucket.get(targetKey) ?? new Map();
+        const prev = perImporter.get(importerPath);
+        if (!prev) {
+            perImporter.set(importerPath, {
+                path: importerPath,
+                file_id: importerFileId,
+                module,
+                imported: [...new Set(imported)].sort(),
+                confidence,
+            });
+        }
+        else {
+            // Merge a second edge from the same importer to the same target.
+            const mergedNames = new Set(prev.imported);
+            for (const n of imported)
+                mergedNames.add(n);
+            prev.imported = [...mergedNames].sort();
+            if (module && (!prev.module || module < prev.module))
+                prev.module = module; // smallest spec, stable
+            if (typeof confidence === 'number') {
+                prev.confidence = typeof prev.confidence === 'number' ? Math.max(prev.confidence, confidence) : confidence;
+            }
+        }
+        bucket.set(targetKey, perImporter);
+    };
+    const ordered = [...shards].sort((a, b) => a.path.localeCompare(b.path));
+    for (const shard of ordered) {
+        // module node id -> its specifier + imported names (for reason metadata).
+        const moduleById = new Map();
+        for (const n of shard.nodes) {
+            if (n.kind === 'module')
+                moduleById.set(n.id, { name: n.name, imported: n.imported_names ?? [] });
+        }
+        for (const e of shard.edges) {
+            if (e.kind !== 'resolves_to' && e.kind !== 'imports_symbol')
+                continue;
+            const mod = moduleById.get(e.from);
+            if (e.kind === 'resolves_to') {
+                const targetPath = fileNodeIdToPath.get(e.to);
+                if (!targetPath)
+                    continue; // target id not an indexed file (defensive)
+                addDependent(byFile, targetPath, shard.path, shard.file_id, mod?.name, mod?.imported ?? [], e.confidence);
+            }
+            else {
+                addDependent(bySymbol, e.to, shard.path, shard.file_id, mod?.name, mod?.imported ?? [], e.confidence);
+            }
+        }
+    }
+    const finalize = (bucket) => {
+        const out = Object.create(null);
+        for (const key of [...bucket.keys()].sort()) {
+            out[key] = [...bucket.get(key).values()].sort((a, b) => a.path.localeCompare(b.path));
+        }
+        return out;
+    };
+    return {
+        schema_version: CODE_MAP_SCHEMA_VERSION,
+        project_id: projectId,
+        updated_at: new Date().toISOString(),
+        dependents_by_file: finalize(byFile),
+        dependents_by_symbol: finalize(bySymbol),
+    };
+}
+//# sourceMappingURL=indexes.js.map

package/dist/core/code-map/lang/java/imports.scm

@@ -0,0 +1,17 @@
+; Code Map — Java imports (imports.scm). Provider #4 (langs#3-4).
+;
+; enclosingStatementNodeTypes = [import_declaration] (provider declaration); that
+; statement is the import span/ordinal anchor. Java import shapes (Codex R1):
+;   import a.b.C;            -> module a.b.C
+;   import a.b.*;            -> module a.b   + imported name '*'
+;   import static a.b.C.m;   -> module a.b.C + imported name 'm'   (split type/member)
+;   import static a.b.C.*;   -> module a.b.C + imported name '*'
+;
+; A single capture takes the scoped_identifier path verbatim (e.g. "a.b.C" or, for a
+; wildcard import, the package "a.b" — the grammar puts the `*` in a sibling
+; `asterisk` node, NOT inside scoped_identifier). The provider's refine() inspects
+; each import_declaration for the `static` keyword and the `asterisk` sibling and
+; rewrites source/imported-names accordingly (the static-split and wildcard rules
+; can't be expressed structurally without overlapping matches).
+
+(import_declaration [(scoped_identifier) (identifier)] @import.source)