@esneiderbravo/speclaw 0.4.0 → 1.0.0

package/dist/modules/compass/embedder.js

@@ -1,4 +1,5 @@
 import { createHash } from "node:crypto";
+import { EMBED_INPUT_VERSION } from "./embed-input.js";
 /**
  * Split identifiers into lowercase subtokens.
  *
@@ -23,7 +24,8 @@ export function tokenize(text) {
  */
 export class LexicalEmbedder {
     dim;
-    id = "lexical-hash-v1";
+    /** Identity includes embed-input recipe version so cache invalidates on bump. */
+    id = `lexical-hash-v1+${EMBED_INPUT_VERSION}`;
     constructor(dim = 256) {
         this.dim = dim;
     }

package/dist/modules/compass/explore-rich.js

@@ -1,4 +1,4 @@
-import { explore, impact, trace, search, recall } from "./query.js";
+import { explore, impact, trace } from "./query.js";
 import { affectedTests } from "./affected.js";
 import { hotspots } from "./hotspots.js";
 import { summarizeImpact } from "./impact-summary.js";
@@ -117,10 +117,15 @@ export async function exploreRich(query) {
     return out;
 }
 /** Merge lexical and semantic search behind one surface. */
-export async function findSymbols(projectPath, query, mode, limit) {
-    if (mode === "exact")
-        return search(projectPath, query, limit ?? 25);
-    return recall(projectPath, query, limit ?? 15);
+export async function findSymbols(projectPath, query, mode, limit, opts) {
+    const { hybridSearch } = await import("./hybrid.js");
+    const result = await hybridSearch(projectPath, query, {
+        mode,
+        focus: opts?.focus,
+        maxTokens: opts?.maxTokens,
+        seedLimit: limit ?? 50,
+    });
+    return result;
 }
 /** Serialize explore-rich with output budget applied. */
 export function formatExploreRich(result, mode = "brief") {

package/dist/modules/compass/extract.js

@@ -1,5 +1,6 @@
 import { parse } from "./parser.js";
 import { rawHash, structuralHash } from "./hash.js";
+import { tokenize } from "./embedder.js";
 const COMMENT_TYPES = new Set(["comment", "line_comment", "block_comment"]);
 /** `Covers:` / `Needs:` / `@covers` at the start of a comment line. */
 const RE_DIRECTIVE = /(?:^|\s|\*)\s*(?:@)?(covers|needs)\s*:?\s+([^\n*]+)/i;
@@ -37,6 +38,89 @@ function calleeName(node, lang) {
 function signatureOf(node) {
     return node.text.split("\n")[0].trim().slice(0, 200);
 }
+/** Strip comment delimiters from a block/line comment. */
+function stripCommentText(raw) {
+    return raw
+        .replace(/^\/\*\*?/, "")
+        .replace(/\*\/$/, "")
+        .replace(/^\/\//, "")
+        .replace(/^\s*\*/gm, "")
+        .trim()
+        .slice(0, 2000);
+}
+/**
+ * Docstring for a definition: prior block/JSDoc comment (TS/JS) or first string
+ * literal in the body (Python).
+ *
+ * @param node - Definition AST node.
+ * @param lang - Language config (`id` selects strategy).
+ */
+export function docstringOf(node, lang) {
+    if (lang.id === "python") {
+        const body = node.childForFieldName("body");
+        if (body) {
+            for (let i = 0; i < body.childCount; i++) {
+                const child = body.child(i);
+                if (!child)
+                    continue;
+                if (child.type === "expression_statement") {
+                    const inner = child.child(0);
+                    if (inner && (inner.type === "string" || inner.type === "concatenated_string")) {
+                        return inner.text
+                            .replace(/^['"]{1,3}|['"]{1,3}$/g, "")
+                            .trim()
+                            .slice(0, 2000);
+                    }
+                }
+                if (COMMENT_TYPES.has(child.type))
+                    continue;
+                break;
+            }
+        }
+        return "";
+    }
+    let prev = node.previousSibling;
+    while (prev) {
+        if (COMMENT_TYPES.has(prev.type)) {
+            const t = prev.text.trim();
+            if (t.startsWith("/**") || t.startsWith("/*") || t.startsWith("//")) {
+                return stripCommentText(t);
+            }
+            prev = prev.previousSibling;
+            continue;
+        }
+        if (prev.type === "decorator" || prev.type === "decorator_list") {
+            prev = prev.previousSibling;
+            continue;
+        }
+        break;
+    }
+    // JSDoc often sits before `export function` / `export class` (parent statement).
+    const parent = node.parent;
+    if (parent && (parent.type === "export_statement" || parent.type === "lexical_declaration")) {
+        let p = parent.previousSibling;
+        while (p) {
+            if (COMMENT_TYPES.has(p.type)) {
+                const t = p.text.trim();
+                if (t.startsWith("/**") || t.startsWith("/*") || t.startsWith("//")) {
+                    return stripCommentText(t);
+                }
+                p = p.previousSibling;
+                continue;
+            }
+            break;
+        }
+    }
+    return "";
+}
+/**
+ * Space-separated lowercase subtokens of a symbol name for FTS.
+ *
+ * @param name - Identifier.
+ */
+export function nameSubtokens(name) {
+    return tokenize(name).join(" ");
+}
 const BOOL_OPS = new Set(["&&", "||", "and", "or"]);
 /**
  * Compute LOC / max nesting / branch counts for a definition subtree.
@@ -155,6 +239,8 @@ export async function extract(source, lang) {
                     endByte: node.endIndex,
                     parentIndex: ownerIndex,
                     signature: signatureOf(node),
+                    docstring: docstringOf(node, lang),
+                    subtokens: nameSubtokens(name),
                     bodyHash: rawHash(source, node.startIndex, node.endIndex),
                     normHash: structuralHash(node),
                     loc: health.loc,

package/dist/modules/compass/hybrid.js

@@ -0,0 +1,318 @@
+/**
+ * Hybrid retrieval pipeline: BM25 + KNN + name → RRF → ego expand →
+ * personalized PageRank → structural rerank → token budget.
+ */
+import { openDb, indexExists, ftsAvailable } from "./db.js";
+import { getEmbedder, fromBlob, cosine } from "./embedder.js";
+import { isGitRepo, worktreeChangedFiles } from "../../shared/git.js";
+import { escapeFtsQuery, nameBoost, routeWeights, rrfFuse, structuralScore, MAX_DEGREE, isSymbolQuery, } from "./rank.js";
+import { personalizedPageRank, edgeWeightMul } from "./pagerank.js";
+import { defaultBudget, fitToBudget } from "./budget.js";
+function requireIndex(projectPath) {
+    if (!indexExists(projectPath)) {
+        throw new Error("No index found. Build it first with the index_build tool (creates .speclaw/index.db).");
+    }
+}
+/**
+ * Resolve focus paths: explicit list, else worktree changes, else empty.
+ *
+ * @param projectPath - Project root.
+ * @param focus - Optional explicit paths.
+ */
+export function resolveFocus(projectPath, focus) {
+    if (focus && focus.length > 0)
+        return [...new Set(focus)];
+    if (!isGitRepo(projectPath))
+        return [];
+    return worktreeChangedFiles(projectPath);
+}
+/**
+ * Hybrid code search over the local Compass index.
+ *
+ * @param projectPath - Absolute project root.
+ * @param query - Identifier or prose.
+ * @param opts - Focus, budget, mode.
+ */
+export async function hybridSearch(projectPath, query, opts = {}) {
+    requireIndex(projectPath);
+    const degraded = [];
+    const q = query.trim();
+    const focus = resolveFocus(projectPath, opts.focus);
+    const focusSet = new Set(focus);
+    const mode = opts.mode;
+    const route = mode === "exact"
+        ? "symbol"
+        : mode === "concept"
+            ? "prose"
+            : isSymbolQuery(q)
+                ? "symbol"
+                : "prose";
+    const weights = route === "symbol" ? { bm25: 1.0, knn: 0.3, name: 1.0 } : routeWeights(q); // concept / prose
+    if (mode === "concept") {
+        weights.bm25 = 0.7;
+        weights.knn = 1.0;
+        weights.name = 0.5;
+    }
+    const budget = opts.maxTokens ?? defaultBudget(focus.length > 0);
+    const seedLimit = opts.seedLimit ?? 50;
+    const db = openDb(projectPath);
+    try {
+        const hasFts = ftsAvailable(db);
+        if (!hasFts)
+            degraded.push("fts5-unavailable");
+        const bm25Ids = [];
+        const bm25Rank = new Map();
+        if (hasFts && q) {
+            const match = escapeFtsQuery(q);
+            if (match) {
+                try {
+                    const rows = db
+                        .prepare(`SELECT f.rowid AS node_id
+               FROM nodes_fts f
+               WHERE nodes_fts MATCH ?
+               ORDER BY bm25(nodes_fts, 10.0, 4.0, 2.0, 1.0) ASC
+               LIMIT ?`)
+                        .all(match, seedLimit);
+                    rows.forEach((r, i) => {
+                        bm25Ids.push(r.node_id);
+                        bm25Rank.set(r.node_id, i + 1);
+                    });
+                }
+                catch {
+                    degraded.push("fts5-query-error");
+                }
+            }
+        }
+        const knnIds = [];
+        const knnRank = new Map();
+        const embedder = getEmbedder();
+        const embRows = db
+            .prepare(`SELECT n.id AS node_id, e.vec
+         FROM node_embeddings e
+         JOIN nodes n ON n.id = e.node_id
+         WHERE e.dim = ?`)
+            .all(embedder.dim);
+        if (embRows.length === 0) {
+            degraded.push("no-embeddings");
+        }
+        else if (q) {
+            const qvec = await embedder.embed(q);
+            const scored = embRows.map((r) => ({
+                id: r.node_id,
+                score: cosine(qvec, fromBlob(r.vec)),
+            }));
+            scored.sort((a, b) => b.score - a.score);
+            scored.slice(0, seedLimit).forEach((r, i) => {
+                knnIds.push(r.id);
+                knnRank.set(r.id, i + 1);
+            });
+        }
+        const nameIds = [];
+        const nameRank = new Map();
+        if (q) {
+            const likeRows = db
+                .prepare(`SELECT n.id
+           FROM nodes n
+           WHERE n.name LIKE ?
+           ORDER BY (n.name = ?) DESC, length(n.name) ASC
+           LIMIT 20`)
+                .all(`%${q}%`, q);
+            likeRows.forEach((r, i) => {
+                nameIds.push(r.id);
+                nameRank.set(r.id, i + 1);
+            });
+        }
+        const fused = rrfFuse({ bm25: bm25Ids, knn: knnIds, name: nameIds }, { bm25: weights.bm25, knn: weights.knn, name: weights.name });
+        // Apply name boost using node names.
+        const idMeta = new Map();
+        const loadMeta = db.prepare(`SELECT n.id, n.name, n.kind, f.path AS file, n.start_line AS line, n.signature
+       FROM nodes n JOIN files f ON f.id = n.file_id WHERE n.id = ?`);
+        for (const id of fused.keys()) {
+            const row = loadMeta.get(id);
+            if (row) {
+                idMeta.set(id, {
+                    name: row.name,
+                    kind: row.kind,
+                    file: row.file,
+                    line: row.line,
+                    signature: row.signature,
+                });
+                const boosted = (fused.get(id) ?? 0) * nameBoost(row.name, q);
+                fused.set(id, boosted);
+            }
+        }
+        let seeds = [...fused.entries()].sort((a, b) => b[1] - a[1]).slice(0, seedLimit);
+        // Empty / stopword query: top by global pagerank in focus (or global).
+        if (!q || seeds.length === 0) {
+            const rows = focus.length > 0
+                ? db
+                    .prepare(`SELECT n.id, pr.score FROM pagerank pr
+                 JOIN nodes n ON n.id = pr.node_id
+                 JOIN files f ON f.id = n.file_id
+                 WHERE f.path IN (${focus.map(() => "?").join(",")})
+                 ORDER BY pr.score DESC LIMIT ?`)
+                    .all(...focus, seedLimit)
+                : db
+                    .prepare(`SELECT n.id, pr.score FROM pagerank pr
+                 JOIN nodes n ON n.id = pr.node_id
+                 ORDER BY pr.score DESC LIMIT ?`)
+                    .all(seedLimit);
+            seeds = rows.map((r) => [r.id, r.score]);
+            for (const [id] of seeds) {
+                if (!idMeta.has(id)) {
+                    const row = loadMeta.get(id);
+                    if (row)
+                        idMeta.set(id, row);
+                }
+            }
+        }
+        // Ego-graph expand (1-hop, degree-capped).
+        const expanded = new Set(seeds.map(([id]) => id));
+        const hopOf = new Map();
+        for (const [id] of seeds)
+            hopOf.set(id, 0);
+        const neigh = db.prepare(`SELECT dst_node_id AS id FROM edges
+       WHERE src_node_id = ? AND kind = 'call' AND dst_node_id IS NOT NULL
+       UNION
+       SELECT src_node_id AS id FROM edges
+       WHERE dst_node_id = ? AND kind = 'call' AND src_node_id IS NOT NULL`);
+        for (const [id] of seeds) {
+            const rows = neigh.all(id, id);
+            const capped = rows.slice(0, MAX_DEGREE);
+            for (const r of capped) {
+                if (!expanded.has(r.id)) {
+                    expanded.add(r.id);
+                    hopOf.set(r.id, 1);
+                    if (!idMeta.has(r.id)) {
+                        const row = loadMeta.get(r.id);
+                        if (row)
+                            idMeta.set(r.id, row);
+                    }
+                }
+            }
+        }
+        // Personalized PageRank on ego subgraph (symbols only for score table).
+        const files = db.prepare("SELECT id, path FROM files").all();
+        const pathByFileId = new Map(files.map((f) => [f.id, f.path]));
+        const fileIdByPath = new Map(files.map((f) => [f.path, f.id]));
+        const fileNodeId = (fid) => -fid;
+        const subgraphNodes = [...expanded];
+        const fileIdsNeeded = new Set();
+        for (const id of subgraphNodes) {
+            const meta = idMeta.get(id);
+            if (meta) {
+                const fid = fileIdByPath.get(meta.file);
+                if (fid !== undefined)
+                    fileIdsNeeded.add(fid);
+            }
+        }
+        const prNodeIds = [...subgraphNodes, ...[...fileIdsNeeded].map((fid) => fileNodeId(fid))];
+        const defCount = new Map();
+        const allNames = db.prepare("SELECT name FROM nodes").all();
+        for (const r of allNames)
+            defCount.set(r.name, (defCount.get(r.name) ?? 0) + 1);
+        const refCount = new Map();
+        const mentioned = new Set(q.split(/[^A-Za-z0-9_$]+/).filter((t) => t.length > 1));
+        const prEdges = [];
+        for (const fid of fileIdsNeeded) {
+            const kids = db.prepare("SELECT id, name FROM nodes WHERE file_id = ?").all(fid);
+            for (const k of kids) {
+                if (!expanded.has(k.id))
+                    continue;
+                prEdges.push({ from: fileNodeId(fid), to: k.id, weight: 1 });
+            }
+        }
+        const edgeRows = subgraphNodes.length === 0
+            ? []
+            : db
+                .prepare(`SELECT e.src_node_id, e.dst_node_id, e.dst_name, e.src_file_id
+               FROM edges e
+               WHERE e.kind = 'call'
+                 AND e.src_node_id IS NOT NULL AND e.dst_node_id IS NOT NULL
+                 AND e.src_node_id IN (${subgraphNodes.map(() => "?").join(",")})
+                 AND e.dst_node_id IN (${subgraphNodes.map(() => "?").join(",")})`)
+                .all(...subgraphNodes, ...subgraphNodes);
+        for (const e of edgeRows) {
+            refCount.set(e.dst_name, (refCount.get(e.dst_name) ?? 0) + 1);
+            const srcPath = pathByFileId.get(e.src_file_id) ?? "";
+            const w = edgeWeightMul(e.dst_name, srcPath, {
+                mentionedIdents: mentioned,
+                focusFiles: focusSet,
+                defCount,
+                refCount,
+            });
+            prEdges.push({ from: e.src_node_id, to: e.dst_node_id, weight: w });
+        }
+        const personalize = focus
+            .map((p) => fileIdByPath.get(p))
+            .filter((id) => id !== undefined)
+            .map(fileNodeId);
+        const prScores = subgraphNodes.length > 0
+            ? personalizedPageRank(prNodeIds, prEdges, personalize)
+            : new Map();
+        // Fallback to global pagerank table when PR missing.
+        const globalPr = db.prepare("SELECT score FROM pagerank WHERE node_id = ?");
+        // Churn: cheap proxy — skip full git history; use 0 when unavailable.
+        const commits30 = 0;
+        const seedMap = new Map(seeds);
+        const candidates = [...expanded].filter((id) => idMeta.has(id));
+        const ranked = candidates
+            .map((id) => {
+            const meta = idMeta.get(id);
+            const seed = seedMap.get(id) ?? (prScores.get(id) ?? 0) * 0.01;
+            const pagerank = prScores.get(id) ?? globalPr.get(id)?.score ?? 1e-9;
+            const hops = hopOf.get(id) ?? 2;
+            const isDef = /^(function|class|method|type|interface|enum)$/.test(meta.kind);
+            const score = structuralScore(seed, {
+                pagerank,
+                commits30d: commits30,
+                hopsToFocus: focus.length === 0 ? 0 : hops,
+                isDefinition: isDef,
+            });
+            return {
+                nodeId: id,
+                name: meta.name,
+                kind: meta.kind,
+                file: meta.file,
+                line: meta.line,
+                signature: meta.signature,
+                signals: {
+                    bm25Rank: bm25Rank.get(id),
+                    knnRank: knnRank.get(id),
+                    nameRank: nameRank.get(id),
+                    pagerank,
+                    hops,
+                    score,
+                },
+            };
+        })
+            .sort((a, b) => b.signals.score - a.signals.score);
+        if (focus.length > 0) {
+            const known = new Set(files.map((f) => f.path));
+            if (focus.every((p) => !known.has(p))) {
+                degraded.push("focus-unindexed");
+            }
+        }
+        const budgetHits = ranked.map((h) => ({
+            name: h.name,
+            kind: h.kind,
+            file: h.file,
+            line: h.line,
+            signature: h.signature,
+        }));
+        const fitted = fitToBudget(budgetHits, budget);
+        const finalHits = ranked.slice(0, fitted.hitCount);
+        return {
+            rendered: fitted.rendered,
+            tokens: fitted.tokens,
+            budget: fitted.budget,
+            route,
+            focus,
+            hits: finalHits,
+            degraded,
+        };
+    }
+    finally {
+        db.close();
+    }
+}