@descix/cli 1.0.13 → 1.0.15

package/bin/descix.js

@@ -2229,14 +2229,13 @@ corpusCommand
   });
 
 // ============ KB Doctor (M3, 2026-04-20) ============
-// Compare local sync-state vs Pinecone vectorCount and scan recent sync
-// logs for per-file 0-chunk warnings. Exits non-zero on drift > threshold.
+// Compare the live corpus files against the last sync's walk (orphans / missing) and scan recent
+// sync logs for per-file 0-chunk warnings. Exits 1 on orphans or missing files.
 kbCommand
   .command('doctor')
-  .description('Detect drift between local sync-state and live Pinecone vector count')
+  .description('Check a KB against its last sync by FILE IDENTITY: live corpus files that were not walked (orphans) and walked files that are not live (missing)')
   .requiredOption('-a, --app <id>', 'App ID')
   .requiredOption('-k, --kb <name>', 'KB name')
-  .option('-t, --threshold <ratio>', 'Drift threshold as a fraction (default: 0.05)', parseFloat)
   .option('--live', 'Compute vectorCount from the TRUE live Pinecone scope (bypass the cached counter, which lies after an interrupted op)')
   .option('--reconcile', 'Compute the live count AND write it back to rag_vector_count so the cached read is truthful again (implies --live)')
   .action(async (options) => {

package/lib/commands/corpus.js

@@ -156,6 +156,21 @@ async function chunkCorpusFile(fileEntry, context) {
   }).filter(Boolean);
 }
 
+
+/**
+ * The walked files that produce NO chunks, for this sync's state. Returns null (UNKNOWN) when a
+ * file carried over unchanged comes from a state that never recorded the field: a partial list
+ * would make `kb doctor` report a file as lost from the store when it is merely empty.
+ */
+export function zeroChunkBlobShas(previousState, localBlobShas, newOrChangedFiles, zeroThisRun) {
+  const changed = new Set(newOrChangedFiles.map((f) => f.blob_sha));
+  const carriedUnchanged = [...localBlobShas].filter((sha) => !changed.has(sha));
+  if (carriedUnchanged.length > 0 && !Array.isArray(previousState?.zero_chunk_blob_shas)) return null;
+  const previousZero = new Set(previousState?.zero_chunk_blob_shas || []);
+  const carried = carriedUnchanged.filter((sha) => previousZero.has(sha));
+  return [...new Set([...carried, ...zeroThisRun])].sort();
+}
+
 /**
  * Read the previously-synced chunk total out of sync state, for REPORTING only.
  *
@@ -660,26 +675,41 @@ export async function runCorpusSync(apiClient, options) {
       const localBlobShas = new Set(files.map(f => f.blob_sha));
       const newOrChangedFiles = files.filter(f => !previousBlobShas.has(f.blob_sha));
 
-      // Stale blob_shas: anything we synced previously that is no longer in the local
-      // walk. This covers BOTH cases:
-      //   (a) File deleted from the source corpus entirely.
-      //   (b) File modified in place — same source_path, new blob_sha → old blob_sha
-      //       is no longer in localBlobShas.
-      // Each stale blob_sha maps to a file_id of `corpus:{blob_sha}` in Pinecone metadata
-      // and is purged via the deleteStaleChunksByFileId primitive (multi-tenancy filter).
-      const deletedBlobShas = [...previousBlobShas].filter(sha => !localBlobShas.has(sha));
-      const fileIdsToDelete = deletedBlobShas.map(sha => `corpus:${sha}`);
+      // STALE = what is LIVE in this KB's corpus scheme and absent from the walk. ONE
+      // derivation, read from the store itself — never from local history.
+      //
+      // It used to be "previously synced blob_shas minus the walk", which is only as good as
+      // the history. Measured 2026-09-18 (GODSWORLD-DEV, PROD): the first origin-keyed sync
+      // had NO history for its origin, so it purged nothing, and a superseded INDEX.md stayed
+      // retrievable beside its replacement indefinitely — no later incremental sync could
+      // ever name it. Lost state, a repointed origin and a hand edit all fail the same way.
+      // A `corpus:` file_id is written only by this manifest's sync, so a live corpus id the
+      // walk does not produce is stale by definition. Both cases are covered:
+      //   (a) file deleted from the corpus;  (b) file modified in place (new blob_sha).
+      // --rebuild purges the whole scope instead, so it computes no per-file list.
+      const localFileIds = new Set(files.map(f => `corpus:${f.blob_sha}`));
+      let fileIdsToDelete = [];
+      if (!options.rebuild) {
+        const remote = await listRemoteFileIds(apiClient, appId, kbName);
+        fileIdsToDelete = remote.file_ids
+          .filter(id => typeof id === 'string' && id.startsWith('corpus:') && !localFileIds.has(id))
+          .sort();
+      }
 
       const unchangedCount = files.length - newOrChangedFiles.length;
 
       if (options.verbose) {
         console.log(chalk.gray(`  New/changed files: ${newOrChangedFiles.length}`));
         console.log(chalk.gray(`  Unchanged files: ${unchangedCount}`));
-        console.log(chalk.gray(`  Stale blob SHAs (delete): ${deletedBlobShas.length}`));
+        console.log(chalk.gray(`  Stale corpus file_ids live in the KB (delete): ${fileIdsToDelete.length}`));
       }
 
       // 3d. Chunk new/changed files
       const allChunks = [];
+      // Files this run chunked to NOTHING — recorded so `kb doctor` can tell "empty by design"
+      // from "lost from the store". A file that FAILED to chunk is not added: it is missing
+      // content, and the doctor should say so.
+      const zeroChunkShasThisRun = [];
       if (newOrChangedFiles.length > 0) {
         const chunkSpinner = ora(`  Chunking ${newOrChangedFiles.length} file(s)...`).start();
 
@@ -689,6 +719,7 @@ export async function runCorpusSync(apiClient, options) {
             allChunks.push(...chunks);
 
             if (chunks.length === 0) {
+              zeroChunkShasThisRun.push(fileEntry.blob_sha);
               // M1 (2026-04-20): always surface 0-chunk files even without -v.
               // These are the silent failures that caused the 77% EGPT gap.
               // 'kb doctor' scans sync logs for this exact string.
@@ -713,6 +744,21 @@ export async function runCorpusSync(apiClient, options) {
         console.log(chalk.cyan(`\n  [dry-run] Plan for ${kbName}:`));
         console.log(chalk.white(`    Would upsert: ${allChunks.length} chunks from ${newOrChangedFiles.length} file(s)`));
         console.log(chalk.white(`    Would delete (stale-file): ${fileIdsToDelete.length} blob SHA(s)`));
+        // NAME them, so the operator sees WHICH documents a real run removes, not just how many.
+        // A corpus file_id is a git blob SHA, so the corpus's own repositories can say where it lived.
+        if (fileIdsToDelete.length > 0) {
+          const { describeStaleBlobs } = await import('../core/staleBlobNames.js');
+          const { resolveOwningRepo } = await import('../core/ManifestLoader.js');
+          const repoRoots = [...new Set(files
+            .map((f) => resolveOwningRepo(f.absolute_path)?.repoRoot)
+            .filter(Boolean))];
+          for (const s of describeStaleBlobs(fileIdsToDelete, repoRoots)) {
+            const where = s.path
+              ? `${s.repo.split('/').pop()}:${s.path} (last seen in ${s.commit.substring(0, 8)})`
+              : 'not in any local history of this corpus — cannot be named';
+            console.log(chalk.gray(`      - ${s.file_id.substring(0, 16)}…  ${where}`));
+          }
+        }
         if (options.rebuild) {
           console.log(chalk.white(
             `    Would purge (rebuild): FULL scope ${communityId}/${appId}/${kbName} ` +
@@ -938,6 +984,11 @@ export async function runCorpusSync(apiClient, options) {
         chunks_upserted: isReportedCount(upserted) ? upserted : null,
         chunks_deleted: isReportedCount(deleted) ? deleted : null,
         synced_blob_shas: allSyncedBlobShas,
+        // Walked files that produced no chunks. KNOWN only when every file's outcome is known:
+        // chunked this run, or carried from a previous state that recorded the field. A state
+        // written before this field existed leaves unchanged files unaccounted for, so the
+        // field is null (UNKNOWN) rather than a partial list the doctor would read as complete.
+        zero_chunk_blob_shas: zeroChunkBlobShas(previousState, localBlobShas, newOrChangedFiles, zeroChunkShasThisRun),
         // The retrieval canary's verdict — read this, not `total_chunks`, to know whether the
         // last sync's content was CONFIRMED searchable or only confirmed present.
         retrieval_canary: retrievalCanary

package/lib/commands/kb.js

@@ -27,7 +27,7 @@ import { requireInteractive } from '../interactive.js';
 import { WorkspaceConfig } from '../workspace-config.js';
 import { hydrateKb, pushStaging, checkStagingFiles } from '../core/Hydrator.js';
 import { processKb } from '../core/Chunker.js';
-import { syncKb, getSyncStatus } from '../core/Syncer.js';
+import { syncKb, getSyncStatus, listRemoteFileIds } from '../core/Syncer.js';
 import * as driveADC from '../google-storage-adc.js';
 // The one canonical KB-sync surface, from its owner. Never spell it as a literal here: these
 // commands' next steps are read by a developer as instructions, so they must name the live verb
@@ -252,32 +252,30 @@ export async function runKbPush(apiClient, options) {
 
 // ============ M3: `descix kb doctor` — drift detector (2026-04-20) ============
 /**
- * Compare local sync-state vs live Pinecone vectorCount, and scan the most
- * recent verbose sync log (if any) for per-file 0-chunk warnings. Reports
- * drift direction and exits non-zero when drift exceeds the threshold.
+ * Check a KB against its last corpus sync by FILE IDENTITY, and scan the most recent verbose
+ * sync log (if any) for per-file 0-chunk warnings.
  *
- * WHY THIS EXISTS
- * ---------------
- * `descix kb corpus sync` trusts local sync-state/<KB>.json for delta
- * detection. When the Pinecone index is reset (e.g. dev-env churn), the
- * local state still lists all blob SHAs as "synced", so the next sync
- * skips re-upload and orphan/loss goes undetected until a content
- * retrieval test fails. This command surfaces the drift directly.
+ * WHY IDENTITY, NOT COUNT
+ * -----------------------
+ * The first version compared Pinecone's vectorCount with the sync-state's total_chunks. But the
+ * sync writes total_chunks FROM the reconciled live count, so the comparison was the store
+ * against a copy of itself: drift 0 by construction, "HEALTHY" over a superseded document that
+ * was still live and retrievable (measured 2026-09-18, GODSWORLD-DEV, PROD). vectorCount also
+ * lags deletes. What matters is WHICH files are live, so that is what this checks.
  *
  * BEHAVIOUR
  * ---------
- *   descix kb doctor -a <app> -k <kb>
+ *   descix kb doctor -a <app> -k <kb> [--live | --reconcile]
  *
- *   (a) Queries Pinecone via get_kb_rag_status for vectorCount.
- *   (b) Reads {appRoot}/.descix/sync-state/{kb}.json for total_chunks.
- *   (c) Reports diff and direction:
- *          Pinecone < sync-state  → LOST (chunks missing from Pinecone)
- *          Pinecone > sync-state  → ORPHANS (Pinecone has extra vectors)
- *          |drift| / local ≤ threshold  → HEALTHY
- *   (d) Scans the most recent file matching logs/kb-sync-*.log for
- *       '0-chunk' / 'skipped' warnings (these are the signal M1 wired
- *       into corpus.js).
- *   (e) Exits non-zero on drift above threshold (default: 5%).
+ *   (a) get_kb_rag_status for vectorCount — reported as information only. --live / --reconcile
+ *       still check the server's CACHED counter against the live store, a separate question.
+ *   (b) Reads this origin's sync-state: the walked blob SHAs and the zero-chunk ones.
+ *   (c) Lists the live corpus file_ids and reports:
+ *          ORPHANS — live but not walked: retrievable stale content. Fails.
+ *          MISSING — walked, has content, not live. Fails. UNVERIFIABLE (not failing) when the
+ *                    state predates the zero-chunk record, rather than guessing.
+ *   (d) Scans the most recent logs/kb-sync-*.log for '0-chunk' / 'skipped' warnings.
+ *   (e) Exits 1 on orphans or missing files, 2 when the state cannot be read.
  */
 export async function runKbDoctor(apiClient, options) {
   const { WorkspaceConfig } = await import('../workspace-config.js');
@@ -293,10 +291,6 @@ export async function runKbDoctor(apiClient, options) {
   const appRoot = appMeta.absolutePath;
   const communityId = appMeta.communityId || appMeta.community_id || null;
 
-  // Drift threshold (fraction). CEO guidance 2026-04-20: ~5% is reasonable.
-  const DRIFT_THRESHOLD = Number.isFinite(Number(options.threshold))
-    ? Number(options.threshold) : 0.05;
-
   const scope = communityId ? `${communityId}/${appId}/${kbName}` : `${appId}/${kbName}`;
   console.log(chalk.cyan(`\n🩺 kb doctor — ${scope}\n`));
 
@@ -362,23 +356,33 @@ export async function runKbDoctor(apiClient, options) {
     console.log(chalk.gray(`    Run 'descix kb corpus sync -a ${appId} -k ${kbName}' against this origin first.`));
     process.exit(2);
   }
-  const localChunks = syncStateRaw.total_chunks;
-  if (typeof localChunks !== 'number') {
-    throw new Error(`sync-state has no total_chunks: ${statePathForOrigin}`);
+  // (c) IDENTITY, not count. Which corpus files are LIVE, against which files the last sync
+  // WALKED. The count comparison this replaced read the store against `total_chunks`, which the
+  // sync itself writes FROM the reconciled live count — so drift was zero by construction and
+  // the check could not fail (measured 2026-09-18, GODSWORLD-DEV: "HEALTHY" printed over a
+  // superseded INDEX.md that was still live and retrievable). Identity catches exactly that.
+  if (!Array.isArray(syncStateRaw.synced_blob_shas)) {
+    console.log(chalk.yellow(`  ⚠ sync-state records no synced_blob_shas at ${statePathForOrigin} — the walked set is UNKNOWN.`));
+    process.exit(2);
   }
-
-  // (c) Report
-  const delta = vectorCount - localChunks;
-  const ratio = localChunks === 0 ? Infinity : Math.abs(delta) / localChunks;
-  const direction = delta === 0 ? 'EXACT'
-    : delta < 0 ? 'LOST (Pinecone missing chunks)'
-    : 'ORPHANS (Pinecone has extra vectors)';
-  const healthy = Math.abs(delta) / Math.max(localChunks, 1) <= DRIFT_THRESHOLD;
-
-  console.log(`  Pinecone vectorCount : ${chalk.white(vectorCount)}`);
-  console.log(`  Local total_chunks   : ${chalk.white(localChunks)}`);
-  console.log(`  Drift                : ${chalk.white((delta >= 0 ? '+' : '') + delta)} (${(ratio * 100).toFixed(1)}%)`);
-  console.log(`  Direction            : ${delta === 0 ? chalk.green(direction) : (healthy ? chalk.yellow(direction) : chalk.red(direction))}`);
+  const report = diagnoseCorpusIdentity({
+    liveFileIds: (await listRemoteFileIds(apiClient, appId, kbName)).file_ids,
+    walkedBlobShas: syncStateRaw.synced_blob_shas,
+    zeroChunkBlobShas: syncStateRaw.zero_chunk_blob_shas,
+  });
+
+  console.log(`  Walked by last sync  : ${chalk.white(report.walked)} file(s)` +
+    (report.zeroKnown ? chalk.gray(` (${report.zeroChunk} empty by design)`) : ''));
+  console.log(`  Live corpus files    : ${chalk.white(report.live)}`);
+  console.log(`  Orphans (live, not walked) : ${report.orphans.length ? chalk.red(report.orphans.length) : chalk.green(0)}`);
+  report.orphans.slice(0, 10).forEach((id) => console.log(chalk.red(`    ${id}`)));
+  if (report.zeroKnown) {
+    console.log(`  Missing (walked, not live) : ${report.missing.length ? chalk.red(report.missing.length) : chalk.green(0)}`);
+    report.missing.slice(0, 10).forEach((id) => console.log(chalk.red(`    ${id}`)));
+  } else {
+    console.log(chalk.yellow('  Missing (walked, not live) : UNVERIFIABLE — this state predates the record of empty files; the next sync writes it.'));
+  }
+  console.log(chalk.gray(`  Pinecone vectorCount : ${vectorCount} (informational — it lags deletes and is not a health signal)`));
   console.log(`  Index                : ${ragStatus.indexName || '(unknown)'}`);
   console.log(`  Last sync (server)   : ${ragStatus.lastSync || '(unknown)'}`);
   console.log(`  Last sync (local)    : ${syncStateRaw.timestamp || '(unknown)'}`);
@@ -420,19 +424,37 @@ export async function runKbDoctor(apiClient, options) {
 
   // (e) Exit
   console.log();
-  if (!healthy) {
-    console.log(chalk.red(`  ✗ DRIFT detected: ${(ratio * 100).toFixed(1)}% exceeds ${(DRIFT_THRESHOLD * 100).toFixed(1)}% threshold.\n`));
-    if (delta < 0) {
-      console.log(chalk.gray(`    Recommended: invalidate sync-state and re-sync:`));
-      console.log(chalk.gray(`      rm "${syncStatePath}"`));
-      console.log(chalk.gray(`      descix kb corpus sync -a ${appId} -k ${kbName}\n`));
-    } else {
-      console.log(chalk.gray(`    Recommended: clear stale vectors and re-sync to rebuild Pinecone from local state.\n`));
-    }
-    process.exit(1);
+  if (report.orphans.length > 0) {
+    console.log(chalk.red(`  ✗ ORPHANS: ${report.orphans.length} corpus file(s) are live and retrievable but were not in the last walk.`));
+    console.log(chalk.gray(`    A plain sync purges them (it reconciles against the live KB). Preview it first:`));
+    console.log(chalk.gray(`      descix kb corpus sync -a ${appId} -k ${kbName} --dry-run\n`));
   }
+  if (report.missing.length > 0) {
+    console.log(chalk.red(`  ✗ MISSING: ${report.missing.length} walked file(s) hold chunks but are not live.`));
+    console.log(chalk.gray(`    A plain sync skips files its state already records, so rebuild:`));
+    console.log(chalk.gray(`      descix kb corpus sync -a ${appId} -k ${kbName} --rebuild --dry-run\n`));
+  }
+  if (report.orphans.length > 0 || report.missing.length > 0) process.exit(1);
 
-  console.log(chalk.green(`  ✓ HEALTHY (drift within ${(DRIFT_THRESHOLD * 100).toFixed(1)}% threshold)\n`));
+  if (report.zeroKnown) {
+    console.log(chalk.green(`  ✓ HEALTHY — every live corpus file was walked, and every walked file with content is live.\n`));
+  } else {
+    console.log(chalk.green(`  ✓ No orphans.`) + chalk.yellow(` Missing-content check pending the next sync.\n`));
+  }
+}
+
+/**
+ * Compare the live corpus file set against the last walk. Pure, so the verdict is testable
+ * without a store. Only `corpus:` ids are this manifest's to judge; other schemes are ignored.
+ */
+export function diagnoseCorpusIdentity({ liveFileIds, walkedBlobShas, zeroChunkBlobShas }) {
+  const live = new Set((liveFileIds || []).filter((id) => typeof id === 'string' && id.startsWith('corpus:')));
+  const walked = new Set((walkedBlobShas || []).map((sha) => `corpus:${sha}`));
+  const zeroKnown = Array.isArray(zeroChunkBlobShas);
+  const zero = new Set((zeroChunkBlobShas || []).map((sha) => `corpus:${sha}`));
+  const orphans = [...live].filter((id) => !walked.has(id)).sort();
+  const missing = zeroKnown ? [...walked].filter((id) => !zero.has(id) && !live.has(id)).sort() : [];
+  return { live: live.size, walked: walked.size, zeroKnown, zeroChunk: zero.size, orphans, missing };
 }
 
 export default {

package/lib/core/staleBlobNames.js

@@ -0,0 +1,53 @@
+/**
+ * Name the stale corpus files a sync would delete, so an operator sees WHICH documents go
+ * before a real run — not just how many. (GODSWORLD-DEV, 2026-09-18: a dry run reported
+ * "Would delete (stale-file): 3" on a KB where the three might matter, with no way to tell
+ * which three.)
+ *
+ * A `corpus:` file_id IS a git blob SHA, so the corpus's own repositories can say which path
+ * held that blob, and in which commit. No store round-trip and no server change. A blob that
+ * no local history holds is reported as such rather than guessed at.
+ */
+
+import { execFileSync } from 'child_process';
+
+/**
+ * Parse `git log --all --no-abbrev --format=%h --raw --find-object=<sha>` output: the most
+ * recent commit that added or removed that blob, and the path it lived at.
+ * @returns {{commit:string, path:string} | null}
+ */
+export function parseFindObjectRaw(output, sha) {
+    let commit = null;
+    for (const line of String(output || '').split('\n')) {
+        if (/^[0-9a-f]{7,40}$/.test(line.trim())) { commit = line.trim(); continue; }
+        const m = line.match(/^:\d{6} \d{6} ([0-9a-f]{40}) ([0-9a-f]{40}) \S+\t(.+)$/);
+        if (m && commit && (m[1] === sha || m[2] === sha)) {
+            return { commit, path: m[3].split('\t').pop() };
+        }
+    }
+    return null;
+}
+
+/**
+ * Resolve each stale `corpus:<sha>` id to where that blob lived, searching each repo root.
+ * @param {string[]} fileIds - `corpus:<sha>` ids
+ * @param {string[]} repoRoots - the repositories that own this corpus's sources
+ * @returns {Array<{file_id:string, repo:string|null, commit:string|null, path:string|null}>}
+ */
+export function describeStaleBlobs(fileIds, repoRoots) {
+    return fileIds.map((file_id) => {
+        const sha = file_id.replace(/^corpus:/, '');
+        for (const repo of repoRoots) {
+            let out = '';
+            try {
+                out = execFileSync('git', ['log', '--all', '--no-abbrev', '--format=%h', '--raw', `--find-object=${sha}`],
+                    { cwd: repo, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 16 * 1024 * 1024 });
+            } catch {
+                continue;
+            }
+            const hit = parseFindObjectRaw(out, sha);
+            if (hit) return { file_id, repo, commit: hit.commit, path: hit.path };
+        }
+        return { file_id, repo: null, commit: null, path: null };
+    });
+}

package/lib/verb-invokes.generated.json

@@ -143,7 +143,8 @@
     "delete_knowledge_base"
   ],
   "kb doctor": [
-    "get_kb_rag_status"
+    "get_kb_rag_status",
+    "kb_list_file_ids"
   ],
   "kb list": [
     "list_knowledge_bases"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@descix/cli",
-  "version": "1.0.13",
+  "version": "1.0.15",
   "description": "DeSciX CLI and MCP Server - Unified command-line interface for DeSciX platform",
   "repository": {
     "type": "git",