karajan-code 4.62.0 → 4.63.0

package/src/rag/indexer.js

@@ -14,6 +14,9 @@ import { chunkMarkdown, chunkPlan, chunkSource } from "./chunker.js";
 import { insertChunk, deleteChunksBySource, findChunkByHash, getEmbeddingsByIds } from "./vec-store.js";
 import { detectAdaptersForProject, getAllCodeExtensions } from "../lang/registry.js";
 import { indexableReason, listProjectFiles } from "./indexable.js";
+import { EMBED_SCHEMA, embedText } from "./embed-header.js";
+import { writeEmbedSchema } from "./embed-schema.js";
+import { untrackedFiles } from "./worktree-files.js";
 
 // KJC-PCS-0052 PR-A — antes vivían dos consts hard-coded para JS aquí
 // (CODE_EXT_RE + SKIP_SEGMENTS). Ahora vienen del registry de adapters,
@@ -59,7 +62,7 @@ export const sourceKind = () => "code";
  * fresh batch is written, so a second call against the same file
  * leaves the store with exactly the latest chunks (no duplicates).
  */
-export async function indexFile(path, { db, embedder, logger = console, project = null, kind: declaredKind = null } = {}) {
+export async function indexFile(path, { db, embedder, logger = console, project = null, kind: declaredKind = null, root = process.cwd() } = {}) {
   if (!existsSync(path)) { logger.warn?.(`[rag-indexer] file not found: ${path}`); return { indexed: 0, failed: 0 }; }
   const kind = declaredKind ?? detectKind(path);
   if (!kind) { logger.warn?.(`[rag-indexer] unknown kind: ${path}`); return { indexed: 0, failed: 0 }; }
@@ -75,11 +78,17 @@ export async function indexFile(path, { db, embedder, logger = console, project
     // the stored vector: a file whose every chunk duplicates another (CLAUDE.md
     // next to the AGENTS.md kj harden writes) was left with no rows at all, and
     // the coverage check reported it missing after every reindex.
-    const contentHash = createHash("sha256").update(ch.text).digest("hex");
+    // KJC-TSK-1171 (RAG-B): the embedder reads the metadata header before the body, so a
+    // query by file, module or symbol name lands; the stored text stays the body alone.
+    // The dedup key is what the embedder read, header included, under the current schema: a
+    // vector is reused only when the same body sits under the same path, symbol and lines,
+    // never one embedded under another header or an older schema.
+    const embeddable = embedText(ch, { root });
+    const contentHash = createHash("sha256").update(`${EMBED_SCHEMA}\n${embeddable}`).digest("hex");
     try {
       const dup = findChunkByHash(db, contentHash, project);
       const stored = dup ? getEmbeddingsByIds(db, [dup.id]).get(Number(dup.id)) : null;
-      const embedding = stored ?? await embedder.embed(ch.text);
+      const embedding = stored ?? await embedder.embed(embeddable);
       insertChunk(db, { source: path, kind: ch.metadata.kind || kind, text: ch.text, metadata: ch.metadata, embedding, project, contentHash });
       if (stored) skipped += 1; else indexed += 1;
     } catch (err) {
@@ -145,13 +154,13 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
   if (existsSync(planRoot)) {
     const plans = await listFiles(planRoot, (p) => /plan-.*\.json$/.test(p));
     for (const p of plans) {
-      const r = await indexFile(p, { db, embedder, logger, project: slug });
+      const r = await indexFile(p, { db, embedder, logger, project: slug, root: projectDir });
       totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
     }
   }
   const onboarding = join(karajanHome, ONBOARDING_DIR, `${slug}.md`);
   if (existsSync(onboarding)) {
-    const r = await indexFile(onboarding, { db, embedder, logger, project: slug });
+    const r = await indexFile(onboarding, { db, embedder, logger, project: slug, root: projectDir });
     totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
   }
   if (withSources) {
@@ -173,9 +182,13 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
       if (source.startsWith(prefix) && outOf(source) !== null) deleteChunksBySource(db, source);
     }
     for (const s of sources) {
-      const r = await indexFile(s, { db, embedder, logger, project: slug, kind: sourceKind() });
+      const r = await indexFile(s, { db, embedder, logger, project: slug, kind: sourceKind(), root: projectDir });
       totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
     }
+    // KJC-TSK-1171: only a COMPLETE index of the sources carries the embed schema; a file the
+    // embedder failed on leaves the marker as it was, so the rebuild is asked for again.
+    if (totals.failed === 0) writeEmbedSchema(projectDir);
+    else logger.warn?.(`[rag-indexer] ${totals.failed} chunk(s) failed: the embed schema is not recorded, the index will be rebuilt again`);
   }
   return totals;
 }
@@ -187,21 +200,30 @@ export async function indexProject(projectDir, { db, embedder, karajanHome, logg
  * `head` lets callers persist via setLastIndexedCommit. Throws on diff failure
  * (shallow clone, unknown ref) so the CLI can fall back to a full reindex.
  */
-export async function indexProjectDelta(projectDir, { db, embedder, since, logger = console, exclude = [] } = {}) {
+export async function indexProjectDelta(projectDir, { db, embedder, since, logger = console, exclude = [], worktree = false, reconcile = [] } = {}) {
   const totals = { indexed: 0, failed: 0, files: 0, deleted: 0, head: null };
   const slug = projectDir.split("/").pop()?.replace(/[^a-zA-Z0-9._-]/g, "-").toLowerCase() || "project";
   const { stdout: head } = await execa("git", ["-C", projectDir, "rev-parse", "HEAD"]);
   totals.head = head.trim();
-  if (totals.head === since) return totals;
-  const { stdout } = await execa("git", ["-C", projectDir, "diff", "--name-status", "-z", since, "HEAD"]);
+  if (totals.head === since && !worktree) return totals;
+  // KJC-TSK-1174 (RAG-A1): with `worktree`, the diff ends at the files on disk instead of HEAD,
+  // and untracked files join it, so what is being written is retrievable before it is committed.
+  const { stdout } = await execa("git", ["-C", projectDir, "diff", "--name-status", "-z", since, ...(worktree ? [] : ["HEAD"])]);
   const parts = stdout.split("\0").filter(Boolean);
   const ops = []; // { kind: 'del'|'idx', p }
-  for (let i = 0; i < parts.length; i += 1) {
+  let i = 0;
+  while (i < parts.length) {
     const s = parts[i];
-    if (s.startsWith("R")) { ops.push({ kind: "del", p: parts[++i] }, { kind: "idx", p: parts[++i] }); }
-    else if (s === "D") { ops.push({ kind: "del", p: parts[++i] }); }
-    else { ops.push({ kind: "idx", p: parts[++i] }); }
+    if (s.startsWith("R")) { ops.push({ kind: "del", p: parts[i + 1] }, { kind: "idx", p: parts[i + 2] }); i += 3; }
+    else if (s.startsWith("C")) { ops.push({ kind: "idx", p: parts[i + 2] }); i += 3; } // a copy keeps its source
+    else if (s === "D") { ops.push({ kind: "del", p: parts[i + 1] }); i += 2; }
+    else { ops.push({ kind: "idx", p: parts[i + 1] }); i += 2; }
   }
+  if (worktree) for (const p of await untrackedFiles(projectDir)) ops.push({ kind: "idx", p });
+  // Paths a previous refresh took from a dirty tree: on disk they are reindexed as they are now
+  // (reverted or still dirty), gone they are dropped. Each path is handled once.
+  const seen = new Set(ops.map((o) => o.p));
+  for (const p of reconcile) if (!seen.has(p)) ops.push({ kind: existsSync(join(projectDir, p)) ? "idx" : "del", p });
   const adapters = detectAdaptersForProject(projectDir);
   await prepareAdapters(adapters, { logger });
   for (const { kind, p } of ops) {
@@ -211,7 +233,7 @@ export async function indexProjectDelta(projectDir, { db, embedder, since, logge
       logger.info?.(`[rag-indexer] delta delete ${relative(process.cwd(), abs)}`);
     } else if (indexableReason(relative(projectDir, abs), abs, { exclude, projectDir }) === null) {
       // KJC-TSK-0891: the same criterion as the full walk, not a list of extensions.
-      const r = await indexFile(abs, { db, embedder, logger, project: slug, kind: sourceKind() });
+      const r = await indexFile(abs, { db, embedder, logger, project: slug, kind: sourceKind(), root: projectDir });
       totals.indexed += r.indexed; totals.failed += r.failed; totals.files += 1;
     }
   }

package/src/rag/worktree-files.js

@@ -0,0 +1,26 @@
+// KJC-TSK-1174 (RAG-A1): what the working tree holds beyond HEAD, for the pre-query refresh.
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { execa } from "execa";
+import { projectDbPath } from "./project-store.js";
+
+/** Files git does not track yet and does not ignore: the ones a session has just created. */
+export async function untrackedFiles(projectDir) {
+  const { stdout } = await execa("git", ["-C", projectDir, "ls-files", "--others", "--exclude-standard", "-z"]);
+  return stdout.split("\0").filter(Boolean);
+}
+
+// The dirty paths the last refresh indexed, next to the store. Reverted or deleted since, git no
+// longer lists them, yet the index holds their discarded content: the next refresh reconciles them.
+const notePath = (projectDir) => `${projectDbPath(projectDir)}.worktree-paths`;
+
+export function readRefreshedPaths(projectDir) {
+  const file = notePath(projectDir);
+  return existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : [];
+}
+
+export function writeRefreshedPaths(projectDir, paths) {
+  const file = notePath(projectDir);
+  mkdirSync(dirname(file), { recursive: true });
+  writeFileSync(file, JSON.stringify(paths), "utf8");
+}