@dotdotgod/cli 0.6.6 → 0.6.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dotdotgod/cli",
-  "version": "0.6.6",
+  "version": "0.6.7",
   "description": "CLI for dotdotgod validation, hybrid document/code-outline search, and graph indexing.",
   "type": "module",
   "license": "Elastic-2.0",

package/src/commands/query.mjs

@@ -1,10 +1,11 @@
 import { resolve } from 'node:path';
+import { vectorCacheSession } from '../query/cache-session.mjs';
 import { usage } from '../cli/usage.mjs';
 import { readMemoryConfig } from '../memory/config.mjs';
 import { collectDocumentationChunks } from '../query/chunks.mjs';
 import { resolveEmbedder } from '../query/embedder.mjs';
 import { resolveEmbeddingProfile } from '../query/embedding-config.mjs';
-import { isValidNormalizedVectorValues, profileFingerprint, readVectorCache, writeVectorCache } from '../query/store.mjs';
+import { isValidNormalizedVectorValues, profileFingerprint, writeVectorCache } from '../query/store.mjs';
 
 import { projectCorpus, keywordCandidates, fuseCandidates, attachRelationships } from '../query/hybrid.mjs';
 
@@ -69,7 +70,12 @@ export async function buildVectorIndex(root, embedOrOptions = {}) {
   const config = readMemoryConfig(root);
   const exclude = config.load?.documentationSummary?.exclude ?? ['docs/plan', 'docs/archive'];
   const chunks = options.chunks ?? collectDocumentationChunks(root, exclude, config.documentation?.root ?? 'docs');
-  const cached = readVectorCache(root, resolved.identity);
+  return vectorCacheSession(root, resolved.identity, async (state, save, diagnostics) => {
+  const selectedFingerprints = new Set(chunks.map((chunk) => chunk.fingerprint));
+  state.pending = state.pending.filter((row) => selectedFingerprints.has(row.chunk.fingerprint));
+  const recovered = new Set(state.pending.map((row) => row.chunk.fingerprint));
+  const cacheRows = [...state.docs, ...state.code, ...state.pending];
+  const cached = state.dimensions ? { chunks: cacheRows.map((row) => row.chunk), vectors: Float32Array.from(cacheRows.flatMap((row) => row.vector)), manifest: { dimensions: state.dimensions } } : null;
   const cachedOffsets = new Map((cached?.chunks ?? []).map((chunk, index) => [chunk.fingerprint, index]));
   const rows = new Array(chunks.length);
   const missing = []; let reused = 0;
@@ -77,16 +83,38 @@ export async function buildVectorIndex(root, embedOrOptions = {}) {
   for (let offset = 0; offset < missing.length; offset += EMBED_BATCH_SIZE) {
     const batch = missing.slice(offset, offset + EMBED_BATCH_SIZE);
     const embedded = await resolved.embed(batch.map(({ chunk }) => chunk.passage), 'document');
-    embedded.forEach((vector, index) => { rows[batch[index].index] = vector; });
+    if (!Array.isArray(embedded) || embedded.length !== batch.length) throw new Error('Embedder returned an unexpected vector count.');
+    embedded.forEach((vector, index) => {
+      const dimensions = state.dimensions ?? vector.length;
+      assertVector(vector, dimensions, 'Passage vector');
+      state.dimensions = dimensions;
+      rows[batch[index].index] = vector;
+      const { passage, ...chunk } = batch[index].chunk;
+      state.pending.push({ chunk, vector: Array.from(vector) });
+    });
+    save();
   }
   const dimensions = rows[0]?.length ?? cached?.manifest?.dimensions;
-  if (!dimensions) throw new Error('No documentation chunks were available to establish vector dimensions.');
+  if (!dimensions && chunks.length) throw new Error('No documentation chunks were available to establish vector dimensions.');
   if (rows.some((row) => !row || row.length !== dimensions)) throw new Error('Embedder returned inconsistent passage-vector dimensions.');
   const vectors = new Float32Array(chunks.length * dimensions);
   rows.forEach((row, index) => { assertVector(row, dimensions, 'Passage vector'); vectors.set(row, index * dimensions); });
   const storedChunks = chunks.map(({ passage, ...chunk }) => chunk);
-  const manifest = writeVectorCache(root, storedChunks, vectors, { provider: resolved.profile.provider, model: resolved.profile.model, dimensions, profileFingerprint: profileFingerprint(resolved.identity), excluded: exclude, embedded: missing.length, reused });
+  const refreshed = options.corpora ?? (options.chunks ? [...new Set(chunks.map((chunk) => chunk.kind === 'symbol' ? 'code' : 'docs'))] : ['docs']);
+  const counters = {};
+  for (const corpus of refreshed) {
+    const positions = chunks.map((chunk, index) => ({ chunk, index })).filter(({ chunk }) => (chunk.kind === 'symbol' ? 'code' : 'docs') === corpus);
+    state[corpus] = positions.map(({ index }) => ({ chunk: storedChunks[index], vector: Array.from(rows[index]) }));
+    counters[corpus] = { chunks: positions.length, embedded: missing.filter(({ chunk }) => (chunk.kind === 'symbol' ? 'code' : 'docs') === corpus).length, recovered: positions.filter(({ chunk }) => recovered.has(chunk.fingerprint)).length };
+    counters[corpus].reused = positions.length - counters[corpus].embedded;
+  }
+  state.pending = [];
+  save();
+  const metadata = { provider: resolved.profile.provider, model: resolved.profile.model, dimensions, profileFingerprint: profileFingerprint(resolved.identity), excluded: exclude, embedded: missing.length, reused, diagnostics: { ...diagnostics, corpora: counters } };
+  // Keep the legacy export readable; authoritative snapshots remain atomic above.
+  const manifest = dimensions ? writeVectorCache(root, storedChunks, vectors, metadata) : { chunks: 0, ...metadata };
   return { chunks: storedChunks, vectors, manifest, embedding: resolved };
+  });
 }
 
 export async function queryDocumentation(root, query, options = {}) {
@@ -113,10 +141,10 @@ export async function queryProject(root, query, options = {}) {
     catch (error) { if (search === 'vector') throw error; vectorError = error.message; }
     if (!configured && !vectorError && search === 'vector') throw new Error('Embedding is not configured. Set embedding in project or global configuration to use vector search.');
   }
-  if (search !== 'keyword' && configured && corpus.chunks.length) {
+  if (search !== 'keyword' && configured) {
     try {
-      index = await buildVectorIndex(root, { ...options, chunks: corpus.chunks });
-      const [queryVector] = await index.embedding.embed([query], 'query');
+      index = await buildVectorIndex(root, { ...options, chunks: corpus.chunks, corpora: scope === 'all' ? ['docs', 'code'] : [scope] });
+      const [queryVector] = index.chunks.length ? await index.embedding.embed([query], 'query') : [];
       vector = index.chunks.map((chunk, position) => ({ ...chunk, vectorScore: cosineScore(queryVector, index.vectors, position * index.manifest.dimensions, index.manifest.dimensions) }))
         .sort((a, b) => b.vectorScore - a.vectorScore || a.id.localeCompare(b.id));
     } catch (error) {

package/src/query/cache-session.mjs

@@ -0,0 +1,82 @@
+import { randomUUID } from 'node:crypto';
+import { mkdirSync, openSync, closeSync, readFileSync, writeFileSync, renameSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { isValidNormalizedVectorValues, readVectorCache, profileFingerprint } from './store.mjs';
+
+// One atomic state file keeps completed snapshots and unfinished batches consistent.
+export async function vectorCacheSession(root, identity, action) {
+  const directory = join(root, '.dotdotgod', 'vectors');
+  mkdirSync(directory, { recursive: true });
+  const lock = join(directory, 'build.lock');
+  const token = randomUUID();
+  const started = Date.now();
+  let waited = false;
+  while (true) {
+    try {
+      const fd = openSync(lock, 'wx');
+      try { writeFileSync(fd, JSON.stringify({ pid: process.pid, token })); } finally { closeSync(fd); }
+      break;
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+      waited = true;
+      if (Date.now() - started >= 30000) throw new Error('Vector cache lock timed out; another build may be running or the lock needs inspection.');
+      try {
+        const owner = JSON.parse(readFileSync(lock, 'utf8'));
+        if (!Number.isInteger(owner.pid) || owner.pid < 1) throw new Error('Invalid lock owner.');
+        try { process.kill(owner.pid, 0); }
+        catch (probe) {
+          if (probe.code !== 'ESRCH') throw probe;
+          const recovery = `${lock}.recovery`;
+          try { mkdirSync(recovery); }
+          catch (claim) { if (claim.code === 'EEXIST') { await sleep(50); continue; } throw claim; }
+          try {
+            const current = JSON.parse(readFileSync(lock, 'utf8'));
+            if (current.token === owner.token && current.pid === owner.pid) rmSync(lock);
+          } finally { rmSync(recovery, { recursive: true, force: true }); }
+          continue;
+        }
+      } catch (probe) { if (probe.code === 'ENOENT') continue; }
+      await sleep(50);
+    }
+  }
+  try {
+    const path = join(directory, 'snapshots.json');
+    const fingerprint = profileFingerprint(identity);
+    let state; let reason = 'missing';
+    try {
+      state = JSON.parse(readFileSync(path, 'utf8'));
+      if (state.schema !== 1) { reason = 'schema-mismatch'; state = null; }
+      else if (state.profile !== fingerprint) { reason = 'profile-mismatch'; state = null; }
+      else {
+        if (state.dimensions !== null && (!Number.isInteger(state.dimensions) || state.dimensions < 1)) throw new Error('Invalid cache dimensions.');
+        for (const rows of [state.docs, state.code, state.pending]) {
+          if (!Array.isArray(rows) || rows.some((row) => typeof row.chunk?.fingerprint !== 'string' || !Array.isArray(row.vector) || row.vector.length !== state.dimensions || !isValidNormalizedVectorValues(row.vector, state.dimensions))) throw new Error('Invalid cache rows.');
+        }
+        reason = 'valid';
+      }
+    } catch (error) { state = null; reason = error.code === 'ENOENT' ? 'missing' : 'corrupt'; }
+    if (!state) {
+      state = { schema: 1, profile: fingerprint, dimensions: null, docs: [], code: [], pending: [] };
+      if (reason === 'missing') {
+        const legacyDiagnostics = {};
+        const legacy = readVectorCache(root, identity, legacyDiagnostics);
+        reason = legacyDiagnostics.reason;
+        if (legacy) {
+          state.dimensions = legacy.manifest.dimensions;
+          legacy.chunks.forEach((chunk, i) => state[chunk.kind === 'symbol' ? 'code' : 'docs'].push({ chunk, vector: Array.from(legacy.vectors.subarray(i * state.dimensions, (i + 1) * state.dimensions)) }));
+          reason = 'legacy-migrated';
+        }
+      }
+    }
+    const save = () => {
+      const temporary = `${path}.tmp-${token}`;
+      try { writeFileSync(temporary, JSON.stringify(state)); renameSync(temporary, path); }
+      finally { rmSync(temporary, { force: true }); }
+    };
+    return await action(state, save, { reason, lockWaitMs: waited ? Date.now() - started : 0 });
+  } finally {
+    try { if (JSON.parse(readFileSync(lock, 'utf8')).token === token) rmSync(lock); }
+    catch (error) { if (error.code !== 'ENOENT') throw error; }
+  }
+}

package/src/query/store.mjs

@@ -29,20 +29,25 @@ export function vectorCachePaths(root) {
   return { directory, manifest: join(directory, 'manifest.json'), chunks: join(directory, 'chunks.jsonl'), embeddings: join(directory, 'embeddings.f32') };
 }
 
-export function readVectorCache(root, identity = null) {
+export function readVectorCache(root, identity = null, diagnostics = {}) {
+  diagnostics.reason = 'missing';
   const paths = vectorCachePaths(root);
   if (!existsSync(paths.manifest) || !existsSync(paths.chunks) || !existsSync(paths.embeddings)) return null;
   try {
     const manifest = JSON.parse(readFileSync(paths.manifest, 'utf8'));
+    diagnostics.reason = 'schema-mismatch';
     if (manifest.schemaVersion !== VECTOR_SCHEMA_VERSION || !Number.isInteger(manifest.dimensions) || manifest.dimensions < 1) return null;
+    diagnostics.reason = 'profile-mismatch';
     if (identity && manifest.profileFingerprint !== profileFingerprint(identity)) return null;
+    diagnostics.reason = 'corrupt';
     const chunks = readFileSync(paths.chunks, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line));
     const buffer = readFileSync(paths.embeddings);
     if (buffer.byteLength !== chunks.length * manifest.dimensions * 4) return null;
     const vectors = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
     if (!isValidNormalizedVectorValues(vectors, manifest.dimensions)) return null;
+    diagnostics.reason = 'valid';
     return { manifest, chunks, vectors: new Float32Array(vectors) };
-  } catch { return null; }
+  } catch { diagnostics.reason = 'corrupt'; return null; }
 }
 
 function atomicWrite(path, data) {