@esneiderbravo/speclaw 0.4.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +91 -72
  2. package/dist/cli/commands/index-build.js +12 -3
  3. package/dist/cli/commands/lawbook.js +1 -0
  4. package/dist/cli/commands/laws.js +149 -8
  5. package/dist/cli/commands/owners.js +44 -0
  6. package/dist/cli/commands/query.js +32 -10
  7. package/dist/cli/commands/update.js +39 -0
  8. package/dist/cli/commands/verify.js +8 -0
  9. package/dist/cli/index.js +13 -4
  10. package/dist/modules/compass/budget.js +128 -0
  11. package/dist/modules/compass/db.js +290 -30
  12. package/dist/modules/compass/embed-input.js +28 -0
  13. package/dist/modules/compass/embedder.js +3 -1
  14. package/dist/modules/compass/explore-rich.js +10 -5
  15. package/dist/modules/compass/extract.js +86 -0
  16. package/dist/modules/compass/hybrid.js +318 -0
  17. package/dist/modules/compass/indexer.js +204 -33
  18. package/dist/modules/compass/merkle.js +76 -0
  19. package/dist/modules/compass/pagerank.js +122 -0
  20. package/dist/modules/compass/rank.js +95 -0
  21. package/dist/modules/compass/register.js +8 -4
  22. package/dist/modules/foundation/assets/laws/laws-manifest.json +16 -7
  23. package/dist/modules/foundation/check.js +4 -2
  24. package/dist/modules/foundation/compile-laws.js +210 -0
  25. package/dist/modules/foundation/dialects/agentsmd.js +95 -0
  26. package/dist/modules/foundation/dialects/claude-cursor.js +45 -0
  27. package/dist/modules/foundation/dialects/coderabbit.js +27 -0
  28. package/dist/modules/foundation/dialects/copilot.js +35 -0
  29. package/dist/modules/foundation/dialects/index.js +5 -0
  30. package/dist/modules/foundation/dialects/types.js +58 -0
  31. package/dist/modules/foundation/doctor.js +220 -14
  32. package/dist/modules/foundation/graph.js +7 -3
  33. package/dist/modules/foundation/import-rules.js +67 -0
  34. package/dist/modules/foundation/integrity.js +307 -0
  35. package/dist/modules/foundation/laws-parse.js +131 -0
  36. package/dist/modules/foundation/laws.js +35 -32
  37. package/dist/modules/foundation/lock.js +283 -0
  38. package/dist/modules/foundation/ownership.js +4 -0
  39. package/dist/modules/foundation/scaffold.js +34 -6
  40. package/dist/modules/foundation/scan.js +227 -0
  41. package/dist/modules/foundation/seed-laws.js +263 -0
  42. package/dist/modules/foundation/verify.js +11 -3
  43. package/dist/modules/lawbook/coverage.js +45 -6
  44. package/dist/modules/lawbook/ears.js +417 -0
  45. package/dist/modules/lawbook/engine.js +29 -0
  46. package/dist/modules/lawbook/spec-items.js +4 -1
  47. package/dist/modules/team/owners.js +464 -0
  48. package/package.json +4 -3
@@ -145,6 +145,45 @@ const MIGRATIONS = [
145
145
  "and `law_verify` are CLI-only. Minimal profile omits setup/check/investigate/index.\n" +
146
146
  "- Preserve all project-specific wording; only apply these speclaw-authored changes.",
147
147
  },
148
+ {
149
+ version: "0.4.1",
150
+ describe: "Rule integrity — speclaw.lock digests + injection scan",
151
+ agentPrompt: "- Mention committed `speclaw.lock` at the repo root (never under `.speclaw/`), " +
152
+ "`speclaw laws lock` / `accept` / `scan`, and that digest acceptance is interactive TTY only " +
153
+ "(never via MCP). `speclaw verify` folds integrity findings with deps/graph. " +
154
+ "Strict paths include AGENTS.md / CLAUDE.md / compiled rules; standards docs are advisory.\n" +
155
+ "- Preserve all project-specific wording; only apply these speclaw-authored changes.",
156
+ },
157
+ {
158
+ version: "0.4.2",
159
+ describe: "Spec ownership — team.owners → CODEOWNERS",
160
+ agentPrompt: '- Optional `team.owners` in `lawbook/config.yaml` maps capability names (and `"*"`) to ' +
161
+ "`@user` / `@org/team` / email owners. Run `speclaw owners --write` to compile a managed " +
162
+ "block at the **end** of `.github/CODEOWNERS` (GitHub: last match wins). `speclaw doctor` " +
163
+ "errors if content appears after the end marker. No new MCP tool — CLI only. " +
164
+ "deriveFromTraceability is not enabled in this release.\n" +
165
+ "- Preserve all project-specific wording; only apply these speclaw-authored changes.",
166
+ },
167
+ {
168
+ version: "1.0.0",
169
+ describe: "speclaw 1.0 — official release (enforcement + graph + lawbook + owners)",
170
+ agentPrompt: "- speclaw **1.0** is the official release: Foundation (hooks + `speclaw.lock` integrity), " +
171
+ "Compass (schema 10, eight canonical MCP tools), Lawbook (ceremony 0–3, coverage, drift, " +
172
+ "bugfix), and Team (`team.owners` → `speclaw owners --write`). Install remains " +
173
+ "`npx @esneiderbravo/speclaw@latest init`. CI consumers use `esneiderbravo/speclaw@v1`.\n" +
174
+ "- Preserve all project-specific wording; only apply these speclaw-authored changes.",
175
+ },
176
+ {
177
+ version: "1.0.1",
178
+ describe: "Seed laws match the target repository layout",
179
+ agentPrompt: "- After this update, speclaw rewrites `.speclaw/laws-manifest.json` from the " +
180
+ "target tree: dogfood laws (compass/foundation, ATTRIBUTION.md, local-first, " +
181
+ "protect-templates, shared-stays-inner) are dropped when those paths do not exist; " +
182
+ "the cycle law is scoped to detected source roots (`apps/*/src`, `packages/*/src`, " +
183
+ "`src/`, `lib/`) excluding test files, and considers import edges only. Run " +
184
+ "`speclaw laws compile` if agent rule files look stale, then `speclaw index`.\n" +
185
+ "- Preserve all project-specific wording; only apply these speclaw-authored changes.",
186
+ },
148
187
  ];
149
188
  /**
150
189
  * Update speclaw and bring the current project up to date without a full re-init:
@@ -8,6 +8,7 @@ import { toMarkdown } from "../../modules/foundation/report-md.js";
8
8
  import { toSarif } from "../../modules/foundation/sarif.js";
9
9
  import { loadManifestForVerify } from "../../modules/foundation/laws.js";
10
10
  import { verifyLaws } from "../../modules/foundation/verify.js";
11
+ import { foldIntegrityIntoReport, verifyIntegrity } from "../../modules/foundation/integrity.js";
11
12
  import { driftFindingsForVerify } from "../../modules/lawbook/drift.js";
12
13
  const FORMATS = new Set(["text", "json", "sarif", "markdown"]);
13
14
  /**
@@ -46,6 +47,13 @@ export async function runVerify(flags) {
46
47
  engines: engines.length ? engines : undefined,
47
48
  lawIds: list(flags.law).length ? list(flags.law) : undefined,
48
49
  });
50
+ // Rule-file integrity (speclaw.lock digests + injection scan). Soft when no lock.
51
+ // Covers: req~integrity-verify~1
52
+ const integrity = verifyIntegrity({ projectPath: cwd });
53
+ foldIntegrityIntoReport(report, integrity);
54
+ if (integrity.guidance && format === "text" && flags.json !== true) {
55
+ ui.info(integrity.guidance);
56
+ }
49
57
  // Structural spec↔code drift (when anchors exist) contributes semantic/deleted
50
58
  // findings into the same report stream used by SARIF / exit codes.
51
59
  for (const f of driftFindingsForVerify(cwd)) {
package/dist/cli/index.js CHANGED
@@ -18,11 +18,11 @@ Setup
18
18
  agent add <id> Configure another agent later (symlinks + MCP)
19
19
 
20
20
  Compass (code intelligence — the same surface agents use via MCP)
21
- index (Re)build the local code graph, with progress
21
+ index (Re)build the local code graph (--force / --prune)
22
22
  watch Keep the index fresh on file changes
23
23
  explore <node> A node's source + callers/callees
24
- search <query> Find nodes by name/keyword
25
- recall "<query>" Find code by meaning (semantic)
24
+ search <query> Hybrid find (BM25+vector+name); --focus --max-tokens --explain
25
+ recall "<query>" Hybrid find with concept weights; same flags as search
26
26
  impact <node> Blast radius (grouped by module; --flat / --json)
27
27
  affected-tests Tests affected by a change (--file / --from-diff / --json)
28
28
  diff-context Change context for a diff (--file / --rev / --worktree / --json)
@@ -47,10 +47,16 @@ Other
47
47
  budget Measure always-on context cost (tools, skills, instructions)
48
48
  coverage Requirement → impl → test coverage (--json, --tap, --adopt, --write)
49
49
  drift Spec↔code drift (--json, --reseal, --reverse, --fail-on)
50
+ owners Compile team.owners → .github/CODEOWNERS (--write / --check / --diff)
50
51
  telemetry status Confirm speclaw ships no telemetry
51
52
  check Evaluate an action against the laws (hooks call this; --dry-run to preview)
52
53
  laws verify Verify the deterministic dependency/graph laws against the index
53
- verify Verify laws for CI: exit codes, --sarif, --json, --strict-engines
54
+ laws compile Compile laws into agent rule dialects (AGENTS / Claude / Cursor / …)
55
+ laws import Import third-party rules as draft laws (--from rulesync)
56
+ laws lock Create/refresh committed speclaw.lock digests for rule files
57
+ laws accept <path> Interactively accept a changed rule-file digest (TTY only)
58
+ laws scan Scan rule/skill files for prompt-injection patterns
59
+ verify Verify laws + integrity for CI: exit codes, --sarif, --json, --strict-engines
54
60
  mcp Start the MCP server (used by your agent's config)
55
61
  help Show this help
56
62
  --version Print the installed speclaw version
@@ -74,6 +80,7 @@ const HEADER_COMMANDS = new Set([
74
80
  "coverage",
75
81
  "drift",
76
82
  "telemetry",
83
+ "owners",
77
84
  "index",
78
85
  "watch",
79
86
  "lawbook",
@@ -165,6 +172,8 @@ async function dispatch(cmd, flags) {
165
172
  return (await import("./commands/drift.js")).runDrift(flags);
166
173
  case "telemetry":
167
174
  return (await import("./commands/telemetry.js")).runTelemetry(flags);
175
+ case "owners":
176
+ return (await import("./commands/owners.js")).runOwners(flags);
168
177
  case "check":
169
178
  return (await import("./commands/check.js")).runCheck(flags);
170
179
  case "laws":
@@ -0,0 +1,128 @@
1
+ /**
2
+ * Fit ranked retrieval hits into a token budget via binary search, and render
3
+ * a compact TreeContext with elision markers.
4
+ */
5
+ /**
6
+ * Rough token estimate: ~4 chars/token, sampling every 100th line on large text.
7
+ *
8
+ * @param s - Text to estimate.
9
+ */
10
+ export function estimateTokens(s) {
11
+ if (s.length < 4000)
12
+ return Math.ceil(s.length / 4);
13
+ const lines = s.split("\n");
14
+ if (lines.length < 200)
15
+ return Math.ceil(s.length / 4);
16
+ let sampled = 0;
17
+ let count = 0;
18
+ for (let i = 0; i < lines.length; i += 100) {
19
+ sampled += lines[i].length + 1;
20
+ count++;
21
+ }
22
+ const avg = sampled / Math.max(count, 1);
23
+ return Math.ceil((avg * lines.length) / 4);
24
+ }
25
+ /**
26
+ * Render hits as a TreeContext block with `⋮` elision between non-adjacent lines.
27
+ *
28
+ * @param hits - Ordered hits to include.
29
+ */
30
+ export function renderTreeContext(hits) {
31
+ if (hits.length === 0)
32
+ return "";
33
+ const byFile = new Map();
34
+ for (const h of hits) {
35
+ const list = byFile.get(h.file) ?? [];
36
+ list.push(h);
37
+ byFile.set(h.file, list);
38
+ }
39
+ const parts = [];
40
+ for (const [file, list] of byFile) {
41
+ parts.push(`# ${file}`);
42
+ list.sort((a, b) => a.line - b.line);
43
+ let lastLine = -Infinity;
44
+ for (const h of list) {
45
+ if (h.line - lastLine > 3 && lastLine !== -Infinity)
46
+ parts.push("⋮");
47
+ const sig = h.signature ?? `${h.kind} ${h.name}`;
48
+ parts.push(`${h.line}| ${sig}`);
49
+ if (h.excerpt) {
50
+ const lines = h.excerpt.split("\n");
51
+ if (lines.length > 6) {
52
+ parts.push(...lines.slice(0, 3).map((l) => ` ${l}`));
53
+ parts.push(" ⋮");
54
+ parts.push(...lines.slice(-2).map((l) => ` ${l}`));
55
+ }
56
+ else {
57
+ parts.push(...lines.map((l) => ` ${l}`));
58
+ }
59
+ }
60
+ lastLine = h.line;
61
+ }
62
+ }
63
+ return parts.join("\n");
64
+ }
65
+ /**
66
+ * Binary-search how many leading hits fit `maxTokens` within 15% tolerance.
67
+ * Never returns empty when hits is non-empty — truncates the first hit instead.
68
+ *
69
+ * @param ranked - Hits in final rank order.
70
+ * @param maxTokens - Token budget.
71
+ */
72
+ export function fitToBudget(ranked, maxTokens) {
73
+ const budget = Math.max(1, maxTokens);
74
+ if (ranked.length === 0) {
75
+ return { rendered: "", tokens: 0, budget, hitCount: 0 };
76
+ }
77
+ let lower = 1;
78
+ let upper = ranked.length;
79
+ let best = renderTreeContext(ranked.slice(0, 1));
80
+ let bestCount = 1;
81
+ while (lower <= upper) {
82
+ const mid = (lower + upper) >> 1;
83
+ const tree = renderTreeContext(ranked.slice(0, mid));
84
+ const n = estimateTokens(tree);
85
+ if (Math.abs(n - budget) / budget < 0.15) {
86
+ return { rendered: tree, tokens: n, budget, hitCount: mid };
87
+ }
88
+ if (n <= budget) {
89
+ best = tree;
90
+ bestCount = mid;
91
+ lower = mid + 1;
92
+ }
93
+ else {
94
+ upper = mid - 1;
95
+ }
96
+ }
97
+ // Single oversized hit: truncate excerpt.
98
+ if (bestCount === 1 && estimateTokens(best) > budget) {
99
+ const h = { ...ranked[0], excerpt: truncateExcerpt(ranked[0].excerpt, budget) };
100
+ const rendered = renderTreeContext([h]);
101
+ return { rendered, tokens: estimateTokens(rendered), budget, hitCount: 1 };
102
+ }
103
+ return {
104
+ rendered: best,
105
+ tokens: estimateTokens(best),
106
+ budget,
107
+ hitCount: bestCount,
108
+ };
109
+ }
110
+ function truncateExcerpt(excerpt, budget) {
111
+ if (!excerpt)
112
+ return "";
113
+ const maxChars = Math.max(32, budget * 3);
114
+ if (excerpt.length <= maxChars)
115
+ return excerpt;
116
+ return `${excerpt.slice(0, maxChars)}\n⋮`;
117
+ }
118
+ /**
119
+ * Default token budget inspired by aider: clamp between 1024 and 4096, ×8 when
120
+ * there is no focus set.
121
+ *
122
+ * @param hasFocus - Whether a non-empty focus file set is in use.
123
+ * @param maxInput - Optional model input window hint.
124
+ */
125
+ export function defaultBudget(hasFocus, maxInput = 32_000) {
126
+ const base = Math.max(1024, Math.min(Math.floor(maxInput / 8), 4096));
127
+ return hasFocus ? base : base * 8;
128
+ }
@@ -1,6 +1,7 @@
1
1
  import { DatabaseSync } from "node:sqlite";
2
2
  import fs from "node:fs";
3
3
  import path from "node:path";
4
+ import { contentHashFor, defaultEmbedText } from "./embed-input.js";
4
5
  const SCHEMA = `
5
6
  CREATE TABLE IF NOT EXISTS meta (
6
7
  key TEXT PRIMARY KEY,
@@ -12,7 +13,9 @@ CREATE TABLE IF NOT EXISTS files (
12
13
  hash TEXT NOT NULL,
13
14
  lang TEXT NOT NULL,
14
15
  is_test INTEGER NOT NULL DEFAULT 0,
15
- module TEXT NOT NULL DEFAULT ''
16
+ module TEXT NOT NULL DEFAULT '',
17
+ mtime_ms INTEGER,
18
+ size INTEGER
16
19
  );
17
20
  CREATE INDEX IF NOT EXISTS idx_files_is_test ON files(is_test);
18
21
  -- nodes: the definitions in the codebase (functions, classes, methods, types).
@@ -28,11 +31,13 @@ CREATE TABLE IF NOT EXISTS nodes (
28
31
  parent_id INTEGER,
29
32
  signature TEXT,
30
33
  body_hash TEXT,
31
- norm_hash TEXT
34
+ norm_hash TEXT,
35
+ content_hash TEXT
32
36
  );
33
37
  CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name);
34
38
  CREATE INDEX IF NOT EXISTS idx_nodes_file ON nodes(file_id);
35
39
  CREATE INDEX IF NOT EXISTS idx_nodes_norm_hash ON nodes(norm_hash);
40
+ CREATE INDEX IF NOT EXISTS idx_nodes_content_hash ON nodes(content_hash);
36
41
  -- node_metrics: AST health frames (LOC / nesting / branches) per definition.
37
42
  CREATE TABLE IF NOT EXISTS node_metrics (
38
43
  node_id INTEGER PRIMARY KEY REFERENCES nodes(id) ON DELETE CASCADE,
@@ -53,12 +58,23 @@ CREATE TABLE IF NOT EXISTS edges (
53
58
  CREATE INDEX IF NOT EXISTS idx_edges_dst ON edges(dst_name);
54
59
  CREATE INDEX IF NOT EXISTS idx_edges_src ON edges(src_node_id);
55
60
  CREATE INDEX IF NOT EXISTS idx_edges_dstid ON edges(dst_node_id);
56
- -- node_embeddings: the local vector store (one embedding per node).
57
- CREATE TABLE IF NOT EXISTS node_embeddings (
58
- node_id INTEGER PRIMARY KEY REFERENCES nodes(id) ON DELETE CASCADE,
59
- dim INTEGER NOT NULL,
61
+ -- embedding_cache: vectors keyed by embedder-input content hash (survives reindex).
62
+ CREATE TABLE IF NOT EXISTS embedding_cache (
63
+ content_hash TEXT NOT NULL,
60
64
  model TEXT NOT NULL,
61
- vec BLOB NOT NULL
65
+ dim INTEGER NOT NULL,
66
+ vec BLOB NOT NULL,
67
+ created_at INTEGER NOT NULL,
68
+ last_seen_at INTEGER NOT NULL,
69
+ PRIMARY KEY (content_hash, model)
70
+ );
71
+ CREATE INDEX IF NOT EXISTS idx_embedding_cache_seen ON embedding_cache(last_seen_at);
72
+ -- dir_hashes: Merkle tree of indexed directories ("" = project root).
73
+ CREATE TABLE IF NOT EXISTS dir_hashes (
74
+ path TEXT PRIMARY KEY,
75
+ hash TEXT NOT NULL,
76
+ n_files INTEGER NOT NULL,
77
+ updated_at INTEGER NOT NULL
62
78
  );
63
79
  -- git_history_cache: memoized results of the expensive git-history scans
64
80
  -- (churn, co-change), keyed by query and invalidated when HEAD moves.
@@ -109,9 +125,104 @@ CREATE TABLE IF NOT EXISTS spec_anchors (
109
125
  CREATE INDEX IF NOT EXISTS idx_anchors_capability ON spec_anchors(capability);
110
126
  CREATE INDEX IF NOT EXISTS idx_anchors_symbol ON spec_anchors(symbol_name);
111
127
  CREATE INDEX IF NOT EXISTS idx_anchors_node ON spec_anchors(node_id);
128
+ -- node_text: searchable name/subtokens/signature/doc (FTS content source).
129
+ CREATE TABLE IF NOT EXISTS node_text (
130
+ node_id INTEGER PRIMARY KEY REFERENCES nodes(id) ON DELETE CASCADE,
131
+ name TEXT NOT NULL,
132
+ subtokens TEXT NOT NULL DEFAULT '',
133
+ signature TEXT NOT NULL DEFAULT '',
134
+ doc TEXT NOT NULL DEFAULT ''
135
+ );
136
+ -- pagerank: global (non-personalized) scores recomputed at index time.
137
+ CREATE TABLE IF NOT EXISTS pagerank (
138
+ node_id INTEGER PRIMARY KEY REFERENCES nodes(id) ON DELETE CASCADE,
139
+ score REAL NOT NULL
140
+ );
141
+ CREATE INDEX IF NOT EXISTS idx_pagerank_score ON pagerank(score DESC);
112
142
  `;
143
+ const FTS_DDL = `
144
+ CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
145
+ name, subtokens, signature, doc,
146
+ content='node_text', content_rowid='node_id',
147
+ tokenize="unicode61 remove_diacritics 0 tokenchars '_$'",
148
+ prefix='2 3'
149
+ );
150
+ CREATE TRIGGER IF NOT EXISTS node_text_ai AFTER INSERT ON node_text BEGIN
151
+ INSERT INTO nodes_fts(rowid, name, subtokens, signature, doc)
152
+ VALUES (new.node_id, new.name, new.subtokens, new.signature, new.doc);
153
+ END;
154
+ CREATE TRIGGER IF NOT EXISTS node_text_ad AFTER DELETE ON node_text BEGIN
155
+ INSERT INTO nodes_fts(nodes_fts, rowid, name, subtokens, signature, doc)
156
+ VALUES ('delete', old.node_id, old.name, old.subtokens, old.signature, old.doc);
157
+ END;
158
+ CREATE TRIGGER IF NOT EXISTS node_text_au AFTER UPDATE ON node_text BEGIN
159
+ INSERT INTO nodes_fts(nodes_fts, rowid, name, subtokens, signature, doc)
160
+ VALUES ('delete', old.node_id, old.name, old.subtokens, old.signature, old.doc);
161
+ INSERT INTO nodes_fts(rowid, name, subtokens, signature, doc)
162
+ VALUES (new.node_id, new.name, new.subtokens, new.signature, new.doc);
163
+ END;
164
+ `;
165
+ /** Ensure node_embeddings is a VIEW over embedding_cache (idempotent). */
166
+ function ensureEmbeddingsView(db) {
167
+ const row = db
168
+ .prepare("SELECT type FROM sqlite_master WHERE name = 'node_embeddings' LIMIT 1")
169
+ .get();
170
+ if (row?.type === "view")
171
+ return;
172
+ if (row?.type === "table") {
173
+ db.exec("DROP TABLE node_embeddings");
174
+ }
175
+ db.exec(`
176
+ CREATE VIEW node_embeddings AS
177
+ SELECT n.id AS node_id, ec.dim AS dim, ec.model AS model, ec.vec AS vec
178
+ FROM nodes n
179
+ JOIN embedding_cache ec ON ec.content_hash = n.content_hash
180
+ `);
181
+ }
182
+ /**
183
+ * Try to create the FTS5 virtual table + sync triggers. Soft-degrades when the
184
+ * Node SQLite build lacks FTS5 (pre-22.16).
185
+ *
186
+ * @returns Whether FTS5 is usable after this call.
187
+ */
188
+ export function ensureFts(db) {
189
+ const existing = db.prepare("SELECT 1 FROM sqlite_master WHERE name = 'nodes_fts' LIMIT 1").get();
190
+ if (existing) {
191
+ db.prepare("INSERT INTO meta(key, value) VALUES ('fts5', '1') ON CONFLICT(key) DO UPDATE SET value = excluded.value").run();
192
+ return true;
193
+ }
194
+ try {
195
+ db.exec(FTS_DDL);
196
+ db.prepare("INSERT INTO meta(key, value) VALUES ('fts5', '1') ON CONFLICT(key) DO UPDATE SET value = excluded.value").run();
197
+ return true;
198
+ }
199
+ catch {
200
+ db.prepare("INSERT INTO meta(key, value) VALUES ('fts5', '0') ON CONFLICT(key) DO UPDATE SET value = excluded.value").run();
201
+ return false;
202
+ }
203
+ }
204
+ /** Whether the open database has a usable FTS5 index. */
205
+ export function ftsAvailable(db) {
206
+ const row = db.prepare("SELECT value FROM meta WHERE key = 'fts5'").get();
207
+ if (row?.value === "0")
208
+ return false;
209
+ const tbl = db.prepare("SELECT 1 FROM sqlite_master WHERE name = 'nodes_fts' LIMIT 1").get();
210
+ return Boolean(tbl);
211
+ }
212
+ /** Probe whether this Node build can create an FTS5 virtual table. */
213
+ export function probeFts5Support() {
214
+ try {
215
+ const mem = new DatabaseSync(":memory:");
216
+ mem.exec("CREATE VIRTUAL TABLE t USING fts5(x)");
217
+ mem.close();
218
+ return true;
219
+ }
220
+ catch {
221
+ return false;
222
+ }
223
+ }
113
224
  /** Schema version stamped into the `meta` table on first creation. */
114
- export const SCHEMA_VERSION = "8";
225
+ export const SCHEMA_VERSION = "10";
115
226
  /** The stamped schema version, or null if the db predates versioning / has no meta table. */
116
227
  function readSchemaVersion(db) {
117
228
  try {
@@ -124,10 +235,7 @@ function readSchemaVersion(db) {
124
235
  }
125
236
  /**
126
237
  * Decide whether an existing database is from an incompatible schema and must be
127
- * rebuilt. A fresh database (no tables) needs no reset the schema will create
128
- * them. An existing one is stale if its stamped version differs from the current
129
- * one, or if the `edges` table is missing a column the current code writes to
130
- * (guards against past schema changes that weren't version-bumped).
238
+ * rebuilt. Schema 8→9 and 9→10 are handled by migrators instead of a wipe.
131
239
  */
132
240
  function isStale(db) {
133
241
  const hasEdges = db
@@ -135,7 +243,10 @@ function isStale(db) {
135
243
  .get();
136
244
  if (!hasEdges)
137
245
  return false;
138
- if (readSchemaVersion(db) !== SCHEMA_VERSION)
246
+ const ver = readSchemaVersion(db);
247
+ if (ver === "8" || ver === "9")
248
+ return false; // migrate in openDb
249
+ if (ver !== SCHEMA_VERSION)
139
250
  return true;
140
251
  const edgeCols = db.prepare("PRAGMA table_info(edges)").all().map((c) => c.name);
141
252
  if (!edgeCols.includes("src_node_id") || !edgeCols.includes("dst_node_id"))
@@ -151,10 +262,18 @@ function isStale(db) {
151
262
  /** Drop every table (children first) so the current schema can be recreated cleanly. */
152
263
  function resetSchema(db) {
153
264
  db.exec(`
265
+ DROP VIEW IF EXISTS node_embeddings;
266
+ DROP TRIGGER IF EXISTS node_text_ai;
267
+ DROP TRIGGER IF EXISTS node_text_ad;
268
+ DROP TRIGGER IF EXISTS node_text_au;
269
+ DROP TABLE IF EXISTS nodes_fts;
270
+ DROP TABLE IF EXISTS pagerank;
271
+ DROP TABLE IF EXISTS node_text;
154
272
  DROP TABLE IF EXISTS spec_anchors;
155
273
  DROP TABLE IF EXISTS coverage_links;
156
274
  DROP TABLE IF EXISTS git_history_cache;
157
- DROP TABLE IF EXISTS node_embeddings;
275
+ DROP TABLE IF EXISTS embedding_cache;
276
+ DROP TABLE IF EXISTS dir_hashes;
158
277
  DROP TABLE IF EXISTS edges;
159
278
  DROP TABLE IF EXISTS node_metrics;
160
279
  DROP TABLE IF EXISTS nodes;
@@ -162,14 +281,131 @@ function resetSchema(db) {
162
281
  DROP TABLE IF EXISTS meta;
163
282
  `);
164
283
  }
284
+ /**
285
+ * Migrate schema 8 → 9: embedding_cache, dir_hashes, mtime/size, content_hash,
286
+ * preserve vectors into the cache, replace node_embeddings table with a view.
287
+ *
288
+ * @param db - Open connection already at schema 8.
289
+ * @param projectPath - Project root for backfilling content hashes from disk.
290
+ */
291
+ export function migrate8to9(db, projectPath) {
292
+ db.exec("BEGIN IMMEDIATE");
293
+ try {
294
+ db.exec(`
295
+ CREATE TABLE IF NOT EXISTS embedding_cache (
296
+ content_hash TEXT NOT NULL,
297
+ model TEXT NOT NULL,
298
+ dim INTEGER NOT NULL,
299
+ vec BLOB NOT NULL,
300
+ created_at INTEGER NOT NULL,
301
+ last_seen_at INTEGER NOT NULL,
302
+ PRIMARY KEY (content_hash, model)
303
+ );
304
+ CREATE INDEX IF NOT EXISTS idx_embedding_cache_seen ON embedding_cache(last_seen_at);
305
+ CREATE TABLE IF NOT EXISTS dir_hashes (
306
+ path TEXT PRIMARY KEY,
307
+ hash TEXT NOT NULL,
308
+ n_files INTEGER NOT NULL,
309
+ updated_at INTEGER NOT NULL
310
+ );
311
+ `);
312
+ const fileCols = db.prepare("PRAGMA table_info(files)").all().map((c) => c.name);
313
+ if (!fileCols.includes("mtime_ms"))
314
+ db.exec("ALTER TABLE files ADD COLUMN mtime_ms INTEGER");
315
+ if (!fileCols.includes("size"))
316
+ db.exec("ALTER TABLE files ADD COLUMN size INTEGER");
317
+ const nodeCols = db.prepare("PRAGMA table_info(nodes)").all().map((c) => c.name);
318
+ if (!nodeCols.includes("content_hash")) {
319
+ db.exec("ALTER TABLE nodes ADD COLUMN content_hash TEXT");
320
+ db.exec("CREATE INDEX IF NOT EXISTS idx_nodes_content_hash ON nodes(content_hash)");
321
+ }
322
+ const upd = db.prepare("UPDATE nodes SET content_hash = ? WHERE id = ?");
323
+ const rows = db
324
+ .prepare(`SELECT n.id, n.kind, n.name, n.signature, f.lang
325
+ FROM nodes n JOIN files f ON f.id = n.file_id`)
326
+ .all();
327
+ for (const r of rows) {
328
+ const embedText = defaultEmbedText(r.kind, r.name, r.signature);
329
+ upd.run(contentHashFor({
330
+ lang: r.lang,
331
+ kind: r.kind,
332
+ name: r.name,
333
+ signature: r.signature,
334
+ embedText,
335
+ }), r.id);
336
+ }
337
+ void projectPath;
338
+ const now = Date.now();
339
+ const embType = db
340
+ .prepare("SELECT type FROM sqlite_master WHERE name = 'node_embeddings' LIMIT 1")
341
+ .get();
342
+ if (embType?.type === "table") {
343
+ db.prepare(`INSERT OR IGNORE INTO embedding_cache(content_hash, model, dim, vec, created_at, last_seen_at)
344
+ SELECT n.content_hash, ne.model, ne.dim, ne.vec, ?, ?
345
+ FROM node_embeddings ne
346
+ JOIN nodes n ON n.id = ne.node_id
347
+ WHERE n.content_hash IS NOT NULL`).run(now, now);
348
+ db.exec("DROP TABLE node_embeddings");
349
+ }
350
+ ensureEmbeddingsView(db);
351
+ db.prepare("INSERT INTO meta(key, value) VALUES ('schema_version', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run("9");
352
+ db.exec("COMMIT");
353
+ }
354
+ catch (err) {
355
+ try {
356
+ db.exec("ROLLBACK");
357
+ }
358
+ catch {
359
+ /* already rolled back */
360
+ }
361
+ throw new Error(`schema 8→9 migration failed — delete .speclaw/index.db to rebuild: ${err.message}`, { cause: err });
362
+ }
363
+ }
364
+ /**
365
+ * Migrate schema 9 → 10: `node_text`, optional FTS5, `pagerank`. Embedding cache
366
+ * is left intact; a reindex is required to populate text rows.
367
+ *
368
+ * @param db - Open connection already at schema 9.
369
+ */
370
+ export function migrate9to10(db) {
371
+ db.exec("BEGIN IMMEDIATE");
372
+ try {
373
+ db.exec(`
374
+ CREATE TABLE IF NOT EXISTS node_text (
375
+ node_id INTEGER PRIMARY KEY REFERENCES nodes(id) ON DELETE CASCADE,
376
+ name TEXT NOT NULL,
377
+ subtokens TEXT NOT NULL DEFAULT '',
378
+ signature TEXT NOT NULL DEFAULT '',
379
+ doc TEXT NOT NULL DEFAULT ''
380
+ );
381
+ CREATE TABLE IF NOT EXISTS pagerank (
382
+ node_id INTEGER PRIMARY KEY REFERENCES nodes(id) ON DELETE CASCADE,
383
+ score REAL NOT NULL
384
+ );
385
+ CREATE INDEX IF NOT EXISTS idx_pagerank_score ON pagerank(score DESC);
386
+ `);
387
+ ensureFts(db);
388
+ db.prepare("INSERT INTO meta(key, value) VALUES ('schema_version', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run("10");
389
+ db.prepare("INSERT INTO meta(key, value) VALUES ('needs_reindex', '1') ON CONFLICT(key) DO UPDATE SET value = excluded.value").run();
390
+ db.prepare("INSERT INTO meta(key, value) VALUES ('reindex_reason', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run("schema 10 adds full-text index (names, subtokens, signatures, docs); reindex required");
391
+ db.exec("COMMIT");
392
+ }
393
+ catch (err) {
394
+ try {
395
+ db.exec("ROLLBACK");
396
+ }
397
+ catch {
398
+ /* already rolled back */
399
+ }
400
+ throw new Error(`schema 9→10 migration failed — delete .speclaw/index.db to rebuild: ${err.message}`, { cause: err });
401
+ }
402
+ }
165
403
  /**
166
404
  * Open (creating if needed) the index database at `<projectPath>/.speclaw/index.db`.
167
405
  *
168
406
  * Ensures the `.speclaw` directory exists, enables WAL journaling and foreign
169
- * keys, and applies the schema. If an existing database is from an incompatible
170
- * schema (e.g. after a speclaw upgrade), it is dropped and rebuilt — `.speclaw`
171
- * is fully regenerable, so the next index just repopulates it. The schema
172
- * version is stamped on a fresh (or freshly reset) database.
407
+ * keys, and applies the schema. Schema 8→9 and 9→10 migrate in place
408
+ * (embeddings preserved). Other incompatible schemas are wiped and rebuilt.
173
409
  *
174
410
  * @param projectPath - Absolute path to the project root.
175
411
  * @returns An open connection to the index database.
@@ -178,19 +414,43 @@ export function openDb(projectPath) {
178
414
  const dir = path.join(projectPath, ".speclaw");
179
415
  fs.mkdirSync(dir, { recursive: true });
180
416
  const db = new DatabaseSync(path.join(dir, "index.db"));
181
- db.exec("PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON;");
182
- const wiped = isStale(db);
183
- if (wiped)
184
- resetSchema(db);
185
- db.exec(SCHEMA);
186
- const row = db.prepare("SELECT value FROM meta WHERE key = 'schema_version'").get();
187
- if (!row) {
188
- db.prepare("INSERT INTO meta(key, value) VALUES ('schema_version', ?)").run(SCHEMA_VERSION);
189
- }
190
- if (wiped) {
191
- db.prepare("INSERT INTO meta(key, value) VALUES ('needs_reindex', '1') ON CONFLICT(key) DO UPDATE SET value = excluded.value").run();
417
+ db.exec("PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;");
418
+ const ver = (() => {
419
+ try {
420
+ return readSchemaVersion(db);
421
+ }
422
+ catch {
423
+ return null;
424
+ }
425
+ })();
426
+ if (ver === "8") {
427
+ migrate8to9(db, projectPath);
428
+ migrate9to10(db);
429
+ db.exec(SCHEMA);
430
+ ensureEmbeddingsView(db);
431
+ ensureFts(db);
432
+ }
433
+ else if (ver === "9") {
434
+ migrate9to10(db);
435
+ db.exec(SCHEMA);
436
+ ensureEmbeddingsView(db);
437
+ ensureFts(db);
438
+ }
439
+ else {
440
+ const wiped = isStale(db);
441
+ if (wiped)
442
+ resetSchema(db);
443
+ db.exec(SCHEMA);
444
+ ensureEmbeddingsView(db);
445
+ ensureFts(db);
446
+ const row = db.prepare("SELECT value FROM meta WHERE key = 'schema_version'").get();
447
+ if (!row) {
448
+ db.prepare("INSERT INTO meta(key, value) VALUES ('schema_version', ?)").run(SCHEMA_VERSION);
449
+ }
450
+ if (wiped) {
451
+ db.prepare("INSERT INTO meta(key, value) VALUES ('needs_reindex', '1') ON CONFLICT(key) DO UPDATE SET value = excluded.value").run();
452
+ }
192
453
  }
193
- // Projection from committed JSON — safe even when nodes are empty (node_id null).
194
454
  rehydrateAnchors(db, projectPath);
195
455
  return db;
196
456
  }
@@ -0,0 +1,28 @@
1
+ import { createHash } from "node:crypto";
2
+ /**
3
+ * Bump when the text (or metadata) fed to the embedder changes shape.
4
+ * Combined into LexicalEmbedder.id so stale cache rows never match.
5
+ */
6
+ export const EMBED_INPUT_VERSION = "in2";
7
+ /**
8
+ * Content-addressable key for `embedding_cache`: hash of the embedder recipe,
9
+ * not the file path and not solely `body_hash` (drift uses a different recipe).
10
+ *
11
+ * @param input - Language, kind, name, optional signature, and embedder text.
12
+ */
13
+ export function contentHashFor(input) {
14
+ return createHash("sha256")
15
+ .update([
16
+ EMBED_INPUT_VERSION,
17
+ input.lang,
18
+ input.kind,
19
+ input.name,
20
+ input.signature ?? "",
21
+ input.embedText,
22
+ ].join("\0"))
23
+ .digest("hex");
24
+ }
25
+ /** Default embedder text for a symbol (matches historical indexer behaviour). */
26
+ export function defaultEmbedText(kind, name, signature) {
27
+ return `${kind} ${name} ${signature ?? ""}`;
28
+ }