@yurtsever/capsa 0.1.0-alpha.0 → 0.1.0-alpha.3

package/dist/index.js

@@ -10053,6 +10053,7 @@ function loadSqlite() {
 }
 var DB_DIR = ".capsa";
 var DB_FILE = "index.db";
+var SCHEMA_VERSION = 3;
 function dbPath(projectRoot) {
   return join6(projectRoot, DB_DIR, DB_FILE);
 }
@@ -10069,6 +10070,27 @@ var Store = class {
     this.db.exec("PRAGMA journal_mode = WAL");
     this.migrate();
   }
+  /** True when this index was written by an older schema and must be rebuilt. */
+  needsRebuild() {
+    const row = this.db.prepare("SELECT value FROM meta WHERE key = 'schema'").get();
+    return row?.value !== String(SCHEMA_VERSION);
+  }
+  /**
+    * Drop all indexed content (not the log, not decisions) and mark the schema
+    * current. The virtual tables are dropped and recreated, not emptied:
+    * `CREATE VIRTUAL TABLE IF NOT EXISTS` would keep an old tokenizer alive
+    * through a rebuild — which once left FTS without stemming after v2.
+    */
+  reset() {
+    this.db.exec(`
+      DROP TABLE IF EXISTS chunks_fts;
+      DROP TABLE IF EXISTS chunks_vec;
+      DELETE FROM chunks;
+      DELETE FROM items;
+    `);
+    this.migrate();
+    this.db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES ('schema', ?)").run(String(SCHEMA_VERSION));
+  }
   migrate() {
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
@@ -10097,7 +10119,10 @@ var Store = class {
       );
       CREATE INDEX IF NOT EXISTS chunks_item ON chunks(item_id);
 
-      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(text, tokenize = 'unicode61');
+      -- porter: "enforcement" must find "Enforce". The indexed text is the
+      -- chunk plus its provenance header (title, heading trail) so a section
+      -- called "Acceptance criteria" still knows which ticket it belongs to.
+      CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(text, tokenize = 'porter unicode61');
 
       CREATE VIRTUAL TABLE IF NOT EXISTS chunks_vec USING vec0(
         chunk_id INTEGER PRIMARY KEY,
@@ -10151,10 +10176,14 @@ var Store = class {
            status = excluded.status, updated_at = excluded.updated_at, tags = excluded.tags,
            indexed_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`).run(row.id, row.format, row.kind, row.name, row.path, row.rel_path, row.content_hash, row.status, row.updated_at, row.tags);
   }
-  insertChunk(itemId, idx, headingPath, text, embedding) {
+  /**
+   * `text` is what the user sees; `searchText` (chunk + provenance header)
+   * is what FTS and the embedder see. Falls back to `text` when omitted.
+   */
+  insertChunk(itemId, idx, headingPath, text, embedding, searchText = text) {
     const res = this.db.prepare("INSERT INTO chunks (item_id, idx, heading_path, text) VALUES (?, ?, ?, ?)").run(itemId, idx, JSON.stringify(headingPath), text);
     const id = Number(res.lastInsertRowid);
-    this.db.prepare("INSERT INTO chunks_fts (rowid, text) VALUES (?, ?)").run(id, text);
+    this.db.prepare("INSERT INTO chunks_fts (rowid, text) VALUES (?, ?)").run(id, searchText);
     this.db.prepare("INSERT INTO chunks_vec (chunk_id, embedding) VALUES (?, ?)").run(BigInt(id), new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength));
     return id;
   }
@@ -10221,7 +10250,7 @@ import { relative as relative5 } from "path";
 import { readFile as readFile5, stat as stat3 } from "fs/promises";
 import { basename as basename4, dirname as dirname7, relative as relative3, sep as sep4 } from "path";
 var DOC_DIRS = /* @__PURE__ */ new Set(["docs", "doc", "runbooks", "runbook", "wiki", "adr", "decisions"]);
-var EXCLUDED_DIRS = /* @__PURE__ */ new Set(["tickets", "issues", "plans", "node_modules"]);
+var EXCLUDED_DIRS = /* @__PURE__ */ new Set(["tickets", "issues", "todo", "backlog", "plans", "plan", "roadmap", "milestones", "milestone", "node_modules"]);
 function segments(relPath) {
   return relPath.split(/[\\/]/).filter(Boolean);
 }
@@ -10294,7 +10323,7 @@ function firstHeading(raw2) {
 import { readFile as readFile6, stat as stat4 } from "fs/promises";
 import { basename as basename5, dirname as dirname8, relative as relative4, sep as sep5 } from "path";
 var TICKET_DIRS = /* @__PURE__ */ new Set(["tickets", "issues", "todo", "backlog"]);
-var PLAN_DIRS = /* @__PURE__ */ new Set(["plans", "plan", "roadmap"]);
+var PLAN_DIRS = /* @__PURE__ */ new Set(["plans", "plan", "roadmap", "milestones", "milestone"]);
 var CHECKBOX = /^\s*[-*+]\s+\[( |x|X)\]\s+/gm;
 function segments2(relPath) {
   return relPath.split(/[\\/]/).filter(Boolean);
@@ -10414,6 +10443,10 @@ async function indexProject(projectRoot, embedder, onProgress) {
   const started = Date.now();
   const store = openStore(projectRoot, embedder);
   try {
+    if (store.needsRebuild()) {
+      onProgress?.({ phase: "rebuild" });
+      store.reset();
+    }
     onProgress?.({ phase: "scan" });
     const result = await scan(knowledgeAdapters(), { roots: [projectRoot] });
     const items = result.items.filter((i) => i.scope === "project" && isInside(projectRoot, i.path));
@@ -10431,8 +10464,8 @@ async function indexProject(projectRoot, embedder, onProgress) {
       if (existing)
         store.deleteItem(existing.id);
       const chunks = chunkMarkdown(item.content);
-      const texts = chunks.map((c) => withContext(item, c.headingPath, c.text));
-      const vectors = texts.length ? await embedder.embed(texts) : [];
+      const searchTexts = chunks.map((c) => withContext(item, c.headingPath, c.text));
+      const vectors = searchTexts.length ? await embedder.embed(searchTexts) : [];
       const k = item.knowledge;
       store.upsertItem({
         id: item.id,
@@ -10446,7 +10479,7 @@ async function indexProject(projectRoot, embedder, onProgress) {
         updated_at: k?.updatedAt ?? null,
         tags: k?.tags ? JSON.stringify(k.tags) : null
       });
-      chunks.forEach((c, i) => store.insertChunk(item.id, c.index, c.headingPath, c.text, vectors[i]));
+      chunks.forEach((c, i) => store.insertChunk(item.id, c.index, c.headingPath, c.text, vectors[i], searchTexts[i]));
       indexed++;
       onProgress?.({ phase: "embed", scanned: items.length, indexed, skipped });
     }
@@ -10503,6 +10536,7 @@ var DEFAULT_WEIGHTS = {
     plan: 1.05,
     ticket: 1,
     runbook: 1,
+    doc: 0.95,
     wiki: 0.95,
     commit: 0.85
   }
@@ -10518,7 +10552,9 @@ function recencyFactor(updatedAt, w, now = Date.now()) {
   return w.recencyFloor + (1 - w.recencyFloor) * decay;
 }
 function signalFactor(s, w = DEFAULT_WEIGHTS, now = Date.now()) {
-  return recencyFactor(s.updatedAt, w, now) * w.status[s.status ?? "unknown"] * w.kind[s.kind];
+  const status = w.status[s.status ?? "unknown"] ?? 1;
+  const kind = w.kind[s.kind] ?? 1;
+  return recencyFactor(s.updatedAt, w, now) * status * kind;
 }
 function rrf(lists, k = DEFAULT_WEIGHTS.rrfK) {
   const scores = /* @__PURE__ */ new Map();
@@ -10536,27 +10572,54 @@ async function retrieve(store, embedder, query, options = {}) {
   const maxTokens = options.maxTokens ?? 1500;
   const candidates = options.candidates ?? 30;
   const weights = options.weights ?? DEFAULT_WEIGHTS;
+  const minRelative = options.minRelativeScore ?? 0.5;
   const [qv] = await embedder.embed([query]);
   const vec = qv ? store.vectorSearch(qv, candidates).map((r) => r.chunkId) : [];
   const fts = store.textSearch(query, candidates).map((r) => r.chunkId);
   const fused = rrf([vec, fts], weights.rrfK);
   const rows = store.chunksWithItems([...fused.keys()]);
   const now = Date.now();
-  const scored = rows.map((r) => ({
-    row: r,
-    score: (fused.get(r.id) ?? 0) * signalFactor({ kind: r.item.kind, status: r.item.status, updatedAt: r.item.updated_at }, weights, now)
-  })).sort((a, b) => b.score - a.score);
+  const scored = rows.map((r) => {
+    const fusedScore = fused.get(r.id) ?? 0;
+    const factor = signalFactor({ kind: r.item.kind, status: r.item.status, updatedAt: r.item.updated_at }, weights, now);
+    return { row: r, fusedScore, factor, score: fusedScore * factor };
+  }).sort((a, b) => b.score - a.score);
   const perItem = /* @__PURE__ */ new Map();
   const chunks = [];
+  const debug = [];
   let tokens = 0;
-  for (const { row, score } of scored) {
-    const used = perItem.get(row.item_id) ?? 0;
-    if (used >= 2)
+  const best = scored[0]?.score ?? 0;
+  let cut = false;
+  for (const { row, fusedScore, factor, score } of scored) {
+    let why = "kept";
+    if (cut || chunks.length > 0 && score < best * minRelative) {
+      cut = true;
+      why = "cutoff";
+    } else if ((perItem.get(row.item_id) ?? 0) >= 2) {
+      why = "per-item cap";
+    } else if (tokens + estimateTokens(row.text) > maxTokens && chunks.length > 0) {
+      why = "budget";
+    }
+    if (options.debug) {
+      debug.push({
+        chunkId: row.id,
+        relPath: row.item.rel_path,
+        headingPath: JSON.parse(row.heading_path),
+        kind: row.item.kind,
+        status: row.item.status,
+        vecRank: rankIn(vec, row.id),
+        ftsRank: rankIn(fts, row.id),
+        fused: fusedScore,
+        factor,
+        score,
+        kept: why === "kept",
+        why
+      });
+    }
+    if (why !== "kept")
       continue;
     const t = estimateTokens(row.text);
-    if (tokens + t > maxTokens && chunks.length > 0)
-      continue;
-    perItem.set(row.item_id, used + 1);
+    perItem.set(row.item_id, (perItem.get(row.item_id) ?? 0) + 1);
     tokens += t;
     chunks.push({
       chunkId: row.id,
@@ -10570,7 +10633,7 @@ async function retrieve(store, embedder, query, options = {}) {
       tokens: t
     });
     if (tokens >= maxTokens)
-      break;
+      cut = true;
   }
   const durationMs = Date.now() - started;
   store.logContext({
@@ -10580,7 +10643,28 @@ async function retrieve(store, embedder, query, options = {}) {
     tokensEst: tokens,
     durationMs
   });
-  return { query, chunks, tokensEst: tokens, durationMs };
+  return { query, chunks, tokensEst: tokens, durationMs, ...options.debug ? { debug } : {} };
+}
+function rankIn(list, id) {
+  const i = list.indexOf(id);
+  return i === -1 ? null : i + 1;
+}
+function formatDebug(result) {
+  if (!result.debug)
+    return "";
+  const lines = ["  vec  fts  factor  score    keep  where"];
+  for (const d of result.debug) {
+    const where = [d.relPath, ...d.headingPath].join(" \u203A ").slice(0, 70);
+    lines.push([
+      String(d.vecRank ?? "-").padStart(5),
+      String(d.ftsRank ?? "-").padStart(4),
+      d.factor.toFixed(2).padStart(7),
+      d.score.toFixed(4).padStart(8),
+      (d.kept ? "yes" : d.why).padEnd(12),
+      where
+    ].join(" "));
+  }
+  return lines.join("\n");
 }
 function formatContext(result) {
   if (result.chunks.length === 0)
@@ -10602,7 +10686,8 @@ function projectState(store) {
   for (const i of items)
     counts[i.kind] = (counts[i.kind] ?? 0) + 1;
   const live = (i) => (i.kind === "ticket" || i.kind === "plan") && (i.status === "open" || i.status === "in-progress");
-  const open2 = items.filter(live).sort((a, b) => (b.updated_at ?? "").localeCompare(a.updated_at ?? "")).map((i) => ({ relPath: i.rel_path, name: i.name, status: i.status ?? "unknown", updatedAt: i.updated_at }));
+  const rank = (i) => i.status === "in-progress" ? 0 : 1;
+  const open2 = items.filter(live).sort((a, b) => rank(a) - rank(b) || a.rel_path.localeCompare(b.rel_path)).map((i) => ({ relPath: i.rel_path, name: i.name, status: i.status ?? "unknown", updatedAt: i.updated_at }));
   const recentlyChanged = [...items].sort((a, b) => (b.updated_at ?? "").localeCompare(a.updated_at ?? "")).slice(0, 10).map((i) => ({ relPath: i.rel_path, kind: i.kind, updatedAt: i.updated_at }));
   const decisions = store.db.prepare("SELECT id, created_at, title FROM decisions ORDER BY id DESC LIMIT 10").all().map((d) => ({ id: d.id, createdAt: d.created_at, title: d.title }));
   return { counts, open: open2, recentlyChanged, decisions };
@@ -39971,19 +40056,40 @@ async function guarded(run) {
     process.exit(1);
   }
 }
-function rootFrom(args) {
-  const positional = args.filter((a) => !a.startsWith("--"));
-  return resolve3(positional[0] ?? process.cwd());
+var VALUE_FLAGS = /* @__PURE__ */ new Set(["--model", "--tokens"]);
+function parseArgs(args) {
+  const positional = [];
+  const flags = /* @__PURE__ */ new Map();
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (VALUE_FLAGS.has(arg)) {
+      const value = args[++i];
+      if (value === void 0 || value.startsWith("--")) {
+        throw new Error(`${arg} expects a value`);
+      }
+      flags.set(arg, value);
+    } else if (arg.startsWith("--")) {
+      flags.set(arg, "true");
+    } else {
+      positional.push(arg);
+    }
+  }
+  return { positional, flags };
 }
-function flag(args, name) {
-  const i = args.indexOf(name);
-  return i >= 0 ? args[i + 1] : void 0;
+function tokensFlag(flags) {
+  const raw2 = flags.get("--tokens");
+  if (raw2 === void 0) return void 0;
+  const n = Number(raw2);
+  if (!Number.isInteger(n) || n < 200) throw new Error(`--tokens expects an integer \u2265 200, got "${raw2}"`);
+  return n;
 }
 async function runIndexCommand(args) {
-  const root = rootFrom(args);
-  const embedder = ollamaEmbedder({ model: flag(args, "--model") });
+  const { positional, flags } = parseArgs(args);
+  const root = resolve3(positional[0] ?? process.cwd());
+  const embedder = ollamaEmbedder({ model: flags.get("--model") });
   console.error(`capsa: indexing ${root} with ${embedder.model} \u2026`);
   const res = await indexProject(root, embedder, (p) => {
+    if (p.phase === "rebuild") console.error("capsa: index schema changed, rebuilding from scratch \u2026");
     if (p.phase === "embed") process.stderr.write(`\r  embedded ${p.indexed} / ${p.scanned} (skipped ${p.skipped})   `);
   });
   process.stderr.write("\n");
@@ -39993,19 +40099,25 @@ async function runIndexCommand(args) {
   for (const i of res.issues.slice(0, 10)) console.error(`  ! ${i.path}: ${i.message}`);
 }
 async function runSearchCommand(args) {
-  const positional = args.filter((a) => !a.startsWith("--"));
+  const { positional, flags } = parseArgs(args);
   const query = positional[0];
   if (!query) {
-    console.error("capsa search <query> [path]");
+    console.error("capsa search <query> [path] [--tokens n] [--model name] [--debug]");
     process.exit(1);
   }
   const root = resolve3(positional[1] ?? process.cwd());
-  const embedder = ollamaEmbedder({ model: flag(args, "--model") });
-  const maxTokens = flag(args, "--tokens") ? Number(flag(args, "--tokens")) : void 0;
+  const embedder = ollamaEmbedder({ model: flags.get("--model") });
+  const maxTokens = tokensFlag(flags);
   const store = openStore(root, embedder);
   try {
-    const res = await retrieve(store, embedder, query, { maxTokens, source: "cli:search" });
-    console.log(formatContext(res));
+    const debug = flags.get("--debug") === "true";
+    const res = await retrieve(store, embedder, query, { maxTokens, source: "cli:search", debug });
+    if (debug) {
+      console.log(formatDebug(res));
+      console.log("");
+    } else {
+      console.log(formatContext(res));
+    }
     console.error(`
 ${res.chunks.length} chunks, ~${res.tokensEst} tokens, ${res.durationMs} ms`);
   } finally {
@@ -40013,7 +40125,7 @@ ${res.chunks.length} chunks, ~${res.tokensEst} tokens, ${res.durationMs} ms`);
   }
 }
 async function runStateCommand(args) {
-  const root = rootFrom(args);
+  const root = resolve3(parseArgs(args).positional[0] ?? process.cwd());
   const store = openStore(root, ollamaEmbedder());
   try {
     console.log(formatState(projectState(store)));
@@ -40022,7 +40134,11 @@ async function runStateCommand(args) {
   }
 }
 async function runMcpCommand(args) {
-  await serveStdio({ projectRoot: rootFrom(args) });
+  const { positional, flags } = parseArgs(args);
+  await serveStdio({
+    projectRoot: resolve3(positional[0] ?? process.cwd()),
+    embedder: ollamaEmbedder({ model: flags.get("--model") })
+  });
 }
 
 // src/index.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yurtsever/capsa",
-  "version": "0.1.0-alpha.0",
+  "version": "0.1.0-alpha.3",
   "description": "Your project's knowledge in one capsule. Local-first project memory and cockpit for AI coding agents: indexes runbooks, tickets, plans and instruction files, serves the smallest relevant context over MCP.",
   "type": "module",
   "license": "MIT",