@nanhara/hara 0.122.0 → 0.122.2

package/dist/search/semindex.js

@@ -2,20 +2,72 @@
 // code-asset / repo / knowledge-base scale (hundreds–low-thousands of chunks); the optional zvec adapter is
 // the scale-up path later. Markdown/code stays the SSOT; this index is a derived, rebuildable, gitignored
 // artifact. The embedder is injected (see embed.ts) so the store + chunking are testable without a model.
-import { readFileSync, writeFileSync, mkdirSync, existsSync, statSync } from "node:fs";
+import { existsSync, lstatSync, realpathSync, statSync } from "node:fs";
+import { createHash } from "node:crypto";
 import { homedir } from "node:os";
-import { join, dirname } from "node:path";
+import { basename, join, isAbsolute, resolve } from "node:path";
 import { findProjectRoot } from "../context/agents-md.js";
-import { listProjectFiles, walkFiles, isProbablyBinary, fileSize } from "../fs-walk.js";
-import { zvecBuild, zvecQueryIds } from "./zvec-store.js";
+import { listProjectFiles, walkFiles } from "../fs-walk.js";
+import { zvecBuild, zvecQueryIds, zvecRemove } from "./zvec-store.js";
+import { isSensitiveFilePath } from "../security/sensitive-files.js";
+import { readModelContextFileSync } from "../fs-read.js";
+import { ensurePrivateStateSubdirectory, readPrivateStateFileSnapshot, removePrivateStateFile, } from "../security/private-state.js";
+import { atomicWriteText, bindHaraPrivateStateWritePath, } from "../fs-write.js";
 // Same code/text extensions codebase_search ranks lexically — keep the two walks in sync.
 const CODE_RE = /\.(ts|tsx|js|jsx|mjs|cjs|py|go|rs|java|kt|rb|php|c|h|cc|cpp|hpp|cs|swift|scala|sh|bash|sql|md|mdx|json|ya?ml|toml|html|css|scss|less|vue|svelte|astro|tf|proto|graphql|gql|gradle|txt)$/i;
+// Version 2 is the first cache format built exclusively through the protected-file identity boundary and
+// keyed by source content rather than mtime alone. Versionless/older caches may contain historical aliases
+// of secret files and are never queried or reused.
+const INDEX_FORMAT = 2;
 /** Index location — repo index lives in the project (gitignore it); the rest are global. Derived/rebuildable. */
 export function indexPath(name, cwd) {
     if (name === "repo")
         return join(findProjectRoot(cwd), ".hara", "index", "repo.json");
+    if (!/^[a-z0-9][a-z0-9_-]{0,63}$/i.test(name))
+        throw new Error(`invalid semantic index name '${name}'`);
     return join(homedir(), ".hara", "index", `${name}.json`);
 }
+function indexBase(name, cwd) {
+    return name === "repo" ? findProjectRoot(cwd) : homedir();
+}
+async function ensurePrivateIndexState(name, cwd) {
+    const requested = indexPath(name, cwd); // validates non-repo names before touching the filesystem
+    // Project `.hara` also contains explicitly user-authored memory/roles and keeps its existing sharing
+    // mode; only the private `index` child is repaired. The global ~/.hara control plane is private in full.
+    const dir = ensurePrivateStateSubdirectory(indexBase(name, cwd), [".hara", "index"], name === "repo" ? 1 : 0);
+    const path = join(dir.path, basename(requested));
+    const ignore = join(dir.path, ".gitignore");
+    const indexSnapshot = await readPrivateStateFileSnapshot(path);
+    const ignoreSnapshot = await readPrivateStateFileSnapshot(ignore);
+    return {
+        dir,
+        path,
+        ignore,
+        indexSnapshot,
+        ignoreSnapshot,
+        indexBoundary: bindHaraPrivateStateWritePath(path, dir.path, "write semantic index"),
+        ignoreBoundary: bindHaraPrivateStateWritePath(ignore, dir.path, "write semantic index ignore rule"),
+    };
+}
+function existingPrivateIndexPath(name, cwd) {
+    try {
+        const requested = indexPath(name, cwd);
+        const base = realpathSync.native(indexBase(name, cwd));
+        const hara = join(base, ".hara");
+        const dir = join(hara, "index");
+        for (const component of [hara, dir]) {
+            const info = lstatSync(component);
+            if (!info.isDirectory() || info.isSymbolicLink() || realpathSync.native(component) !== component)
+                return null;
+        }
+        const path = join(dir, basename(requested));
+        const info = lstatSync(path);
+        return info.isFile() && !info.isSymbolicLink() && info.nlink === 1 ? path : null;
+    }
+    catch {
+        return null;
+    }
+}
 function statMtime(p) {
     try {
         return statSync(p).mtimeMs;
@@ -58,30 +110,52 @@ function cosine(a, b) {
     }
     return na && nb ? dot / (Math.sqrt(na) * Math.sqrt(nb)) : 0;
 }
-/** Build/refresh the index. **Incremental**: files whose mtime is unchanged since the last build keep their
- *  existing vectors (no re-embed); only new/changed files are embedded, and deleted files drop out. A changed
+function contentHash(chunks) {
+    const hash = createHash("sha256");
+    for (const chunk of chunks) {
+        const encoded = JSON.stringify([chunk.file, chunk.source, chunk.text]);
+        hash.update(String(Buffer.byteLength(encoded, "utf8"))).update(":").update(encoded);
+    }
+    return hash.digest("hex");
+}
+async function rotateLegacyIndex(name, cwd, state) {
+    if (state.indexSnapshot)
+        removePrivateStateFile(state.path, state.indexSnapshot, state.dir);
+    await zvecRemove(name, cwd);
+}
+/** Build/refresh the index. **Incremental**: files whose content hash is unchanged keep their existing
+ *  vectors (no re-embed); only new/changed files are embedded, and deleted files drop out. A changed
  *  embedding model forces a full rebuild (old vectors aren't comparable). Returns counts. */
 export async function buildIndex(name, chunks, embed, cwd, model = "embed") {
-    const p = indexPath(name, cwd);
+    const state = await ensurePrivateIndexState(name, cwd);
+    const p = state.path;
     // Load the previous index → reuse vectors for unchanged files.
     const prevByFile = new Map();
     let prevModel = "";
-    if (existsSync(p)) {
+    let legacy = false;
+    if (state.indexSnapshot) {
         try {
-            const old = JSON.parse(readFileSync(p, "utf8"));
-            prevModel = old.model;
-            for (const it of old.items ?? []) {
-                const arr = prevByFile.get(it.file);
-                if (arr)
-                    arr.push(it);
-                else
-                    prevByFile.set(it.file, [it]);
+            const old = JSON.parse(state.indexSnapshot.text);
+            if (old.format !== INDEX_FORMAT) {
+                legacy = true;
+            }
+            else {
+                prevModel = old.model;
+                for (const it of old.items ?? []) {
+                    const arr = prevByFile.get(it.file);
+                    if (arr)
+                        arr.push(it);
+                    else
+                        prevByFile.set(it.file, [it]);
+                }
             }
         }
         catch {
-            /* corrupt index → full rebuild */
+            legacy = true; // corrupt/unknown cache → full rebuild and ANN rotation
         }
     }
+    if (legacy)
+        await zvecRemove(name, cwd);
     const sameModel = prevModel === model;
     const byFile = new Map();
     for (const c of chunks) {
@@ -95,14 +169,16 @@ export async function buildIndex(name, chunks, embed, cwd, model = "embed") {
     const toEmbed = [];
     let reused = 0;
     for (const [file, fchunks] of byFile) {
-        const mtime = fchunks[0].mtime ?? 0;
+        const currentHash = contentHash(fchunks);
         const prev = prevByFile.get(file);
-        if (sameModel && prev?.length && mtime > 0 && prev.every((it) => it.mtime === mtime)) {
+        if (sameModel
+            && prev?.length === fchunks.length
+            && prev.every((it) => it.contentHash === currentHash)) {
             items.push(...prev); // file unchanged → keep its vectors
             reused += prev.length;
         }
         else {
-            toEmbed.push(...fchunks);
+            toEmbed.push(...fchunks.map((chunk) => ({ ...chunk, contentHash: currentHash })));
         }
     }
     const B = 64;
@@ -111,18 +187,27 @@ export async function buildIndex(name, chunks, embed, cwd, model = "embed") {
         const vecs = await embed(batch.map((c) => c.text));
         batch.forEach((c, j) => vecs[j] && items.push({ ...c, vec: vecs[j] }));
     }
-    const dir = dirname(p);
-    mkdirSync(dir, { recursive: true });
     // The index is derived + rebuildable (and may embed file contents) — never let it be committed.
-    if (!existsSync(join(dir, ".gitignore")))
-        writeFileSync(join(dir, ".gitignore"), "*\n", "utf8");
-    writeFileSync(p, JSON.stringify({ model, items }), "utf8");
+    if (!state.ignoreSnapshot || state.ignoreSnapshot.text !== "*\n") {
+        await atomicWriteText(state.ignoreBoundary.target, "*\n", {
+            expected: state.ignoreSnapshot?.text ?? null,
+            expectedIdentity: state.ignoreSnapshot ?? undefined,
+            mode: 0o600,
+            boundary: state.ignoreBoundary,
+        });
+    }
+    await atomicWriteText(p, JSON.stringify({ format: INDEX_FORMAT, model, items }), {
+        expected: state.indexSnapshot?.text ?? null,
+        expectedIdentity: state.indexSnapshot ?? undefined,
+        mode: 0o600,
+        boundary: state.indexBoundary,
+    });
     // Build a zvec ANN index alongside the JSON cache (best-effort; queryIndex prefers it for retrieval).
     await zvecBuild(name, items.map((it) => ({ id: it.id, vec: it.vec })), cwd);
     return { total: items.length, embedded: toEmbed.length, reused };
 }
 export function indexExists(name, cwd) {
-    return existsSync(indexPath(name, cwd));
+    return existingPrivateIndexPath(name, cwd) !== null;
 }
 // Never embed (and POST to an embedding provider, then persist in the index) a secret-bearing file —
 // the asset/skill/memory dirs aren't .gitignore-filtered, so a stray credentials.json/secrets.yaml/.env
@@ -141,18 +226,14 @@ export function collectDirChunks(dir, source) {
         if (!CODE_RE.test(rel) || looksSecret(rel))
             continue;
         const abs = join(dir, rel);
-        if (fileSize(abs) > 200_000)
-            continue;
-        let buf;
+        let text;
         try {
-            buf = readFileSync(abs);
+            text = readModelContextFileSync(abs, 200_000);
         }
         catch {
             continue;
         }
-        if (isProbablyBinary(buf))
-            continue;
-        chunks.push(...chunkText(buf.toString("utf8"), abs, source, statMtime(abs)));
+        chunks.push(...chunkText(text, abs, source, statMtime(abs)));
     }
     return chunks;
 }
@@ -163,55 +244,103 @@ export function collectRepoChunks(root) {
         if (!CODE_RE.test(rel) || looksSecret(rel))
             continue;
         const abs = join(root, rel);
-        if (fileSize(abs) > 200_000)
-            continue;
-        let buf;
+        let text;
         try {
-            buf = readFileSync(abs);
+            text = readModelContextFileSync(abs, 200_000);
         }
         catch {
             continue;
         }
-        if (isProbablyBinary(buf))
-            continue;
-        chunks.push(...chunkText(buf.toString("utf8"), rel, "repo", statMtime(abs)));
+        chunks.push(...chunkText(text, rel, "repo", statMtime(abs)));
     }
     return chunks;
 }
 /** Cosine-rank the index against the query embedding. Returns top-k hits (empty if no index). */
 export async function queryIndex(name, query, embed, cwd, k = 6) {
-    const p = indexPath(name, cwd);
-    if (!existsSync(p))
+    if (!existingPrivateIndexPath(name, cwd))
+        return [];
+    let state;
+    try {
+        state = await ensurePrivateIndexState(name, cwd);
+    }
+    catch {
+        return [];
+    }
+    const p = state.path;
+    if (!state.indexSnapshot)
         return [];
     let idx;
     try {
-        idx = JSON.parse(readFileSync(p, "utf8"));
+        idx = JSON.parse(state.indexSnapshot.text);
     }
     catch {
         return [];
     }
+    if (idx.format !== INDEX_FORMAT) {
+        await rotateLegacyIndex(name, cwd, state);
+        return [];
+    }
     if (!idx.items?.length)
         return [];
+    // Old indexes may predate the sensitive-file boundary. Filter at query time as well as build time so a
+    // stale JSON/zvec cache can never resurrect protected content into model context.
+    const root = findProjectRoot(cwd);
+    const safeItems = idx.items.filter((item) => {
+        const path = isAbsolute(item.file) ? item.file : resolve(root, item.file);
+        return !looksSecret(item.file) && !isSensitiveFilePath(path);
+    });
+    if (!safeItems.length)
+        return [];
     const [qv] = await embed([query]);
     if (!qv)
         return [];
+    // A format marker proves which writer created the cache, not that its source still has the same bytes.
+    // Validate only ranked candidate files (cached per path) before returning their historical text. Missing,
+    // replaced, hard-linked, protected, oversized, or changed sources fail closed without making every query
+    // reread the entire corpus.
+    const freshness = new Map();
+    const isFresh = (item) => {
+        const path = isAbsolute(item.file) ? item.file : resolve(root, item.file);
+        const cached = freshness.get(path);
+        if (cached !== undefined)
+            return cached;
+        let fresh = false;
+        try {
+            const text = readModelContextFileSync(path, 200_000);
+            fresh = contentHash(chunkText(text, item.file, item.source)) === item.contentHash;
+        }
+        catch {
+            fresh = false;
+        }
+        freshness.set(path, fresh);
+        return fresh;
+    };
+    const rankFresh = (candidates) => {
+        const ranked = candidates
+            .map((it) => ({ item: it, score: cosine(qv, it.vec) }))
+            .sort((a, b) => b.score - a.score);
+        const hits = [];
+        for (const { item, score } of ranked) {
+            if (!isFresh(item))
+                continue;
+            hits.push({ file: item.file, source: item.source, text: item.text, score });
+            if (hits.length >= k)
+                break;
+        }
+        return hits;
+    };
     // Prefer the zvec ANN index for candidate retrieval; re-rank candidates by EXACT cosine from the JSON
     // store (identical score semantics to the brute-force path). Fall back to full brute-force if zvec is
     // unavailable / has no index / errors.
     const ids = await zvecQueryIds(name, qv, cwd, k);
     if (ids?.length) {
-        const byId = new Map(idx.items.map((it) => [it.id, it]));
-        const hits = ids
+        const byId = new Map(safeItems.map((it) => [it.id, it]));
+        const candidates = ids
             .map((id) => byId.get(id))
-            .filter((it) => Boolean(it))
-            .map((it) => ({ file: it.file, source: it.source, text: it.text, score: cosine(qv, it.vec) }))
-            .sort((a, b) => b.score - a.score)
-            .slice(0, k);
+            .filter((it) => Boolean(it));
+        const hits = rankFresh(candidates);
         if (hits.length)
             return hits;
     }
-    return idx.items
-        .map((it) => ({ file: it.file, source: it.source, text: it.text, score: cosine(qv, it.vec) }))
-        .sort((a, b) => b.score - a.score)
-        .slice(0, k);
+    return rankFresh(safeItems);
 }

package/dist/search/zvec-store.js

@@ -3,28 +3,23 @@
 // null/false and the caller falls back to the JSON brute-force store in semindex.ts — so installs
 // without the native binding keep working (lexical/JSON floor preserved).
 //
-// Design: zvec stores only vector→id (a pure ANN index). The JSON store stays the durable embedding
-// cache + SSOT for hit text/score. So zvec is used for fast candidate retrieval, but the SCORE is
-// recomputed from the JSON vectors (in semindex.queryIndex) — identical score semantics, zero risk
+// Design: zvec stores vector→internal-id plus the original semantic id (a retrieval-only ANN index).
+// The JSON store stays the durable embedding cache + SSOT for hit text/score. So zvec is used for fast
+// candidate retrieval, but the SCORE is recomputed from the JSON vectors (in semindex.queryIndex) —
+// identical score semantics, zero risk
 // to hybrid.ts's thresholds.
 import { homedir } from "node:os";
 import { join } from "node:path";
 import { rmSync, existsSync } from "node:fs";
+import { createHash } from "node:crypto";
 import { findProjectRoot } from "../context/agents-md.js";
 /* eslint-disable @typescript-eslint/no-explicit-any */
-let cached = null;
-let attempted = false;
+let loadPromise = null;
 async function zvec() {
-    if (attempted)
-        return cached;
-    attempted = true;
-    try {
-        cached = await import("@zvec/zvec");
-    }
-    catch {
-        cached = null; // optional dep / no native prebuilt → callers fall back to brute-force
-    }
-    return cached;
+    // Share the in-flight import as well as its result. Returning a still-null cache to a concurrent first
+    // caller would make availability depend on timing and unnecessarily drop that operation to JSON search.
+    loadPromise ??= import("@zvec/zvec").catch(() => null);
+    return loadPromise;
 }
 export async function zvecAvailable() {
     return (await zvec()) !== null;
@@ -34,44 +29,128 @@ function zvecDir(name, cwd) {
         return join(findProjectRoot(cwd), ".hara", "index", "repo.zvec");
     return join(homedir(), ".hara", "index", `${name}.zvec`);
 }
+/** Remove a derived ANN cache without loading the optional native binding. Used when the JSON format is
+ * rotated so an old candidate store can never outlive the policy/version that produced it. */
+export async function zvecRemove(name, cwd) {
+    const path = zvecDir(name, cwd);
+    await withPathLock(path, () => {
+        rmSync(path, { recursive: true, force: true });
+    });
+}
 const VEC = "v";
-/** (Re)build a fresh zvec collection from already-embedded items. Best-effort — false on any failure. */
-export async function zvecBuild(name, items, cwd) {
-    const z = await zvec();
-    if (!z || items.length === 0)
-        return false;
+const SOURCE_ID = "source_id";
+// zvec document ids are limited to a small ASCII character set and 64 bytes, while semantic-index ids are
+// intentionally path-like (and may be absolute or Unicode). Keep the original id in a scalar field and use
+// a fixed-width digest only as zvec's internal primary key.
+function zvecId(id) {
+    return createHash("sha256").update(id, "utf8").digest("hex");
+}
+// A collection owns native RocksDB resources. closeSync() is the documented @zvec/zvec 0.5 API; making
+// cleanup explicit and unconditional prevents a failed insert/query from retaining the process-local LOCK.
+function closeCollection(col) {
+    if (!col)
+        return true;
     try {
-        const dim = items[0].vec.length;
-        const path = zvecDir(name, cwd);
-        rmSync(path, { recursive: true, force: true }); // rebuild from the JSON cache (vectors already computed)
-        const schema = new z.ZVecCollectionSchema({
-            name,
-            vectors: [{ name: VEC, dataType: z.ZVecDataType.VECTOR_FP32, dimension: dim, indexParams: { indexType: z.ZVecIndexType.FLAT, metricType: z.ZVecMetricType.COSINE } }],
-        });
-        const col = z.ZVecCreateAndOpen(path, schema);
-        col.insertSync(items.map((it) => ({ id: it.id, vectors: { [VEC]: it.vec } })));
-        col.closeSync?.(); // closeSync flushes + releases the LOCK so a later (read-only) open succeeds
+        if (typeof col.closeSync !== "function")
+            return false;
+        col.closeSync();
         return true;
     }
     catch {
         return false;
     }
 }
+// zvec permits concurrent readers but only one writer. More importantly, rebuilding removes and recreates
+// the collection directory, so every operation on one path must be ordered in this process (readers too).
+const pathTails = new Map();
+async function withPathLock(path, operation) {
+    const previous = pathTails.get(path) ?? Promise.resolve();
+    let release;
+    const gate = new Promise((resolve) => {
+        release = resolve;
+    });
+    const tail = previous.then(() => gate);
+    pathTails.set(path, tail);
+    await previous;
+    try {
+        return await operation();
+    }
+    finally {
+        release();
+        if (pathTails.get(path) === tail)
+            pathTails.delete(path);
+    }
+}
+/** (Re)build a fresh zvec collection from already-embedded items. Best-effort — false on any failure. */
+export async function zvecBuild(name, items, cwd) {
+    const z = await zvec();
+    if (!z || items.length === 0)
+        return false;
+    const path = zvecDir(name, cwd);
+    return withPathLock(path, () => {
+        let col = null;
+        let success = false;
+        try {
+            const dim = items[0].vec.length;
+            rmSync(path, { recursive: true, force: true }); // rebuild from the JSON cache (vectors already computed)
+            const schema = new z.ZVecCollectionSchema({
+                name,
+                fields: [{ name: SOURCE_ID, dataType: z.ZVecDataType.STRING }],
+                vectors: [{ name: VEC, dataType: z.ZVecDataType.VECTOR_FP32, dimension: dim, indexParams: { indexType: z.ZVecIndexType.FLAT, metricType: z.ZVecMetricType.COSINE } }],
+            });
+            col = z.ZVecCreateAndOpen(path, schema);
+            col.insertSync(items.map((it) => ({ id: zvecId(it.id), fields: { [SOURCE_ID]: it.id }, vectors: { [VEC]: it.vec } })));
+            success = true;
+        }
+        catch {
+            success = false;
+        }
+        finally {
+            if (!closeCollection(col))
+                success = false;
+        }
+        // Never leave a partially inserted ANN index visible: semindex treats any non-empty candidate set as
+        // authoritative for candidate selection, so an incomplete native store would be worse than JSON fallback.
+        if (!success) {
+            try {
+                rmSync(path, { recursive: true, force: true });
+            }
+            catch {
+                /* best-effort optional index cleanup */
+            }
+        }
+        return success;
+    });
+}
 /** ANN candidate ids (best first) from the zvec index. null if unavailable/error → caller brute-forces. */
 export async function zvecQueryIds(name, qv, cwd, k) {
     const z = await zvec();
     if (!z)
         return null;
-    if (!existsSync(zvecDir(name, cwd)))
-        return null;
-    try {
-        const col = z.ZVecOpen(zvecDir(name, cwd), { readOnly: true }); // read-only → no write-lock contention
-        // over-fetch, then semindex re-ranks the candidates by exact cosine from the JSON store
-        const docs = await col.query({ fieldName: VEC, vector: qv, topk: Math.max(k * 4, 24) });
-        col.closeSync?.();
-        return docs.map((d) => d.id);
-    }
-    catch {
-        return null;
-    }
+    const path = zvecDir(name, cwd);
+    return withPathLock(path, async () => {
+        if (!existsSync(path))
+            return null;
+        let col = null;
+        let docs = null;
+        let closed = false;
+        try {
+            col = z.ZVecOpen(path, { readOnly: true });
+            // over-fetch, then semindex re-ranks the candidates by exact cosine from the JSON store
+            // Omitting outputFields returns every scalar field (only source_id in current indexes) and also lets
+            // pre-source_id collections answer with their original native id during a rolling upgrade.
+            docs = await col.query({ fieldName: VEC, vector: qv, topk: Math.max(k * 4, 24) });
+        }
+        catch {
+            docs = null;
+        }
+        finally {
+            closed = closeCollection(col);
+        }
+        if (!docs || !closed)
+            return null;
+        // Old collections did not have source_id and used their JSON id directly; retain that best-effort read
+        // compatibility even though normal index builds replace them with the current schema.
+        return docs.map((doc) => (typeof doc.fields?.[SOURCE_ID] === "string" ? doc.fields[SOURCE_ID] : doc.id));
+    });
 }