@descix/cli 1.0.12 → 1.0.14

package/bin/descix.js

@@ -2229,14 +2229,13 @@ corpusCommand
   });
 
 // ============ KB Doctor (M3, 2026-04-20) ============
-// Compare local sync-state vs Pinecone vectorCount and scan recent sync
-// logs for per-file 0-chunk warnings. Exits non-zero on drift > threshold.
+// Compare the live corpus files against the last sync's walk (orphans / missing) and scan recent
+// sync logs for per-file 0-chunk warnings. Exits 1 on orphans or missing files.
 kbCommand
   .command('doctor')
-  .description('Detect drift between local sync-state and live Pinecone vector count')
+  .description('Check a KB against its last sync by FILE IDENTITY: live corpus files that were not walked (orphans) and walked files that are not live (missing)')
   .requiredOption('-a, --app <id>', 'App ID')
   .requiredOption('-k, --kb <name>', 'KB name')
-  .option('-t, --threshold <ratio>', 'Drift threshold as a fraction (default: 0.05)', parseFloat)
   .option('--live', 'Compute vectorCount from the TRUE live Pinecone scope (bypass the cached counter, which lies after an interrupted op)')
   .option('--reconcile', 'Compute the live count AND write it back to rag_vector_count so the cached read is truthful again (implies --live)')
   .action(async (options) => {
@@ -2537,6 +2536,11 @@ siteCommand
         return;
       }
 
+      // 1b. Refuse to publish backend source, an agent's persona, credentials or workspace
+      // state — BEFORE any token is requested, so a dry run refuses too (SiteDenyLint).
+      const { assertSitePublishable } = await import('../lib/core/SiteDenyLint.js');
+      assertSitePublishable(fileList.map((f) => f.path));
+
       // 2. Request deploy token with file list
       const tokenResponse = await apiClient.invoke('get_site_deploy_token', {
         community_id: communityId,

package/lib/commands/corpus.js

@@ -156,6 +156,21 @@ async function chunkCorpusFile(fileEntry, context) {
   }).filter(Boolean);
 }
 
+
+/**
+ * The walked files that produce NO chunks, for this sync's state. Returns null (UNKNOWN) when a
+ * file carried over unchanged comes from a state that never recorded the field: a partial list
+ * would make `kb doctor` report a file as lost from the store when it is merely empty.
+ */
+export function zeroChunkBlobShas(previousState, localBlobShas, newOrChangedFiles, zeroThisRun) {
+  const changed = new Set(newOrChangedFiles.map((f) => f.blob_sha));
+  const carriedUnchanged = [...localBlobShas].filter((sha) => !changed.has(sha));
+  if (carriedUnchanged.length > 0 && !Array.isArray(previousState?.zero_chunk_blob_shas)) return null;
+  const previousZero = new Set(previousState?.zero_chunk_blob_shas || []);
+  const carried = carriedUnchanged.filter((sha) => previousZero.has(sha));
+  return [...new Set([...carried, ...zeroThisRun])].sort();
+}
+
 /**
  * Read the previously-synced chunk total out of sync state, for REPORTING only.
  *
@@ -660,26 +675,41 @@ export async function runCorpusSync(apiClient, options) {
       const localBlobShas = new Set(files.map(f => f.blob_sha));
       const newOrChangedFiles = files.filter(f => !previousBlobShas.has(f.blob_sha));
 
-      // Stale blob_shas: anything we synced previously that is no longer in the local
-      // walk. This covers BOTH cases:
-      //   (a) File deleted from the source corpus entirely.
-      //   (b) File modified in place — same source_path, new blob_sha → old blob_sha
-      //       is no longer in localBlobShas.
-      // Each stale blob_sha maps to a file_id of `corpus:{blob_sha}` in Pinecone metadata
-      // and is purged via the deleteStaleChunksByFileId primitive (multi-tenancy filter).
-      const deletedBlobShas = [...previousBlobShas].filter(sha => !localBlobShas.has(sha));
-      const fileIdsToDelete = deletedBlobShas.map(sha => `corpus:${sha}`);
+      // STALE = what is LIVE in this KB's corpus scheme and absent from the walk. ONE
+      // derivation, read from the store itself — never from local history.
+      //
+      // It used to be "previously synced blob_shas minus the walk", which is only as good as
+      // the history. Measured 2026-09-18 (GODSWORLD-DEV, PROD): the first origin-keyed sync
+      // had NO history for its origin, so it purged nothing, and a superseded INDEX.md stayed
+      // retrievable beside its replacement indefinitely — no later incremental sync could
+      // ever name it. Lost state, a repointed origin and a hand edit all fail the same way.
+      // A `corpus:` file_id is written only by this manifest's sync, so a live corpus id the
+      // walk does not produce is stale by definition. Both cases are covered:
+      //   (a) file deleted from the corpus;  (b) file modified in place (new blob_sha).
+      // --rebuild purges the whole scope instead, so it computes no per-file list.
+      const localFileIds = new Set(files.map(f => `corpus:${f.blob_sha}`));
+      let fileIdsToDelete = [];
+      if (!options.rebuild) {
+        const remote = await listRemoteFileIds(apiClient, appId, kbName);
+        fileIdsToDelete = remote.file_ids
+          .filter(id => typeof id === 'string' && id.startsWith('corpus:') && !localFileIds.has(id))
+          .sort();
+      }
 
       const unchangedCount = files.length - newOrChangedFiles.length;
 
       if (options.verbose) {
         console.log(chalk.gray(`  New/changed files: ${newOrChangedFiles.length}`));
         console.log(chalk.gray(`  Unchanged files: ${unchangedCount}`));
-        console.log(chalk.gray(`  Stale blob SHAs (delete): ${deletedBlobShas.length}`));
+        console.log(chalk.gray(`  Stale corpus file_ids live in the KB (delete): ${fileIdsToDelete.length}`));
       }
 
       // 3d. Chunk new/changed files
       const allChunks = [];
+      // Files this run chunked to NOTHING — recorded so `kb doctor` can tell "empty by design"
+      // from "lost from the store". A file that FAILED to chunk is not added: it is missing
+      // content, and the doctor should say so.
+      const zeroChunkShasThisRun = [];
       if (newOrChangedFiles.length > 0) {
         const chunkSpinner = ora(`  Chunking ${newOrChangedFiles.length} file(s)...`).start();
 
@@ -689,6 +719,7 @@ export async function runCorpusSync(apiClient, options) {
             allChunks.push(...chunks);
 
             if (chunks.length === 0) {
+              zeroChunkShasThisRun.push(fileEntry.blob_sha);
               // M1 (2026-04-20): always surface 0-chunk files even without -v.
               // These are the silent failures that caused the 77% EGPT gap.
               // 'kb doctor' scans sync logs for this exact string.
@@ -938,6 +969,11 @@ export async function runCorpusSync(apiClient, options) {
         chunks_upserted: isReportedCount(upserted) ? upserted : null,
         chunks_deleted: isReportedCount(deleted) ? deleted : null,
         synced_blob_shas: allSyncedBlobShas,
+        // Walked files that produced no chunks. KNOWN only when every file's outcome is known:
+        // chunked this run, or carried from a previous state that recorded the field. A state
+        // written before this field existed leaves unchanged files unaccounted for, so the
+        // field is null (UNKNOWN) rather than a partial list the doctor would read as complete.
+        zero_chunk_blob_shas: zeroChunkBlobShas(previousState, localBlobShas, newOrChangedFiles, zeroChunkShasThisRun),
         // The retrieval canary's verdict — read this, not `total_chunks`, to know whether the
         // last sync's content was CONFIRMED searchable or only confirmed present.
         retrieval_canary: retrievalCanary

package/lib/commands/kb.js

@@ -27,7 +27,7 @@ import { requireInteractive } from '../interactive.js';
 import { WorkspaceConfig } from '../workspace-config.js';
 import { hydrateKb, pushStaging, checkStagingFiles } from '../core/Hydrator.js';
 import { processKb } from '../core/Chunker.js';
-import { syncKb, getSyncStatus } from '../core/Syncer.js';
+import { syncKb, getSyncStatus, listRemoteFileIds } from '../core/Syncer.js';
 import * as driveADC from '../google-storage-adc.js';
 // The one canonical KB-sync surface, from its owner. Never spell it as a literal here: these
 // commands' next steps are read by a developer as instructions, so they must name the live verb
@@ -252,32 +252,30 @@ export async function runKbPush(apiClient, options) {
 
 // ============ M3: `descix kb doctor` — drift detector (2026-04-20) ============
 /**
- * Compare local sync-state vs live Pinecone vectorCount, and scan the most
- * recent verbose sync log (if any) for per-file 0-chunk warnings. Reports
- * drift direction and exits non-zero when drift exceeds the threshold.
+ * Check a KB against its last corpus sync by FILE IDENTITY, and scan the most recent verbose
+ * sync log (if any) for per-file 0-chunk warnings.
  *
- * WHY THIS EXISTS
- * ---------------
- * `descix kb corpus sync` trusts local sync-state/<KB>.json for delta
- * detection. When the Pinecone index is reset (e.g. dev-env churn), the
- * local state still lists all blob SHAs as "synced", so the next sync
- * skips re-upload and orphan/loss goes undetected until a content
- * retrieval test fails. This command surfaces the drift directly.
+ * WHY IDENTITY, NOT COUNT
+ * -----------------------
+ * The first version compared Pinecone's vectorCount with the sync-state's total_chunks. But the
+ * sync writes total_chunks FROM the reconciled live count, so the comparison was the store
+ * against a copy of itself: drift 0 by construction, "HEALTHY" over a superseded document that
+ * was still live and retrievable (measured 2026-09-18, GODSWORLD-DEV, PROD). vectorCount also
+ * lags deletes. What matters is WHICH files are live, so that is what this checks.
  *
  * BEHAVIOUR
  * ---------
- *   descix kb doctor -a <app> -k <kb>
+ *   descix kb doctor -a <app> -k <kb> [--live | --reconcile]
  *
- *   (a) Queries Pinecone via get_kb_rag_status for vectorCount.
- *   (b) Reads {appRoot}/.descix/sync-state/{kb}.json for total_chunks.
- *   (c) Reports diff and direction:
- *          Pinecone < sync-state  → LOST (chunks missing from Pinecone)
- *          Pinecone > sync-state  → ORPHANS (Pinecone has extra vectors)
- *          |drift| / local ≤ threshold  → HEALTHY
- *   (d) Scans the most recent file matching logs/kb-sync-*.log for
- *       '0-chunk' / 'skipped' warnings (these are the signal M1 wired
- *       into corpus.js).
- *   (e) Exits non-zero on drift above threshold (default: 5%).
+ *   (a) get_kb_rag_status for vectorCount — reported as information only. --live / --reconcile
+ *       still check the server's CACHED counter against the live store, a separate question.
+ *   (b) Reads this origin's sync-state: the walked blob SHAs and the zero-chunk ones.
+ *   (c) Lists the live corpus file_ids and reports:
+ *          ORPHANS — live but not walked: retrievable stale content. Fails.
+ *          MISSING — walked, has content, not live. Fails. UNVERIFIABLE (not failing) when the
+ *                    state predates the zero-chunk record, rather than guessing.
+ *   (d) Scans the most recent logs/kb-sync-*.log for '0-chunk' / 'skipped' warnings.
+ *   (e) Exits 1 on orphans or missing files, 2 when the state cannot be read.
  */
 export async function runKbDoctor(apiClient, options) {
   const { WorkspaceConfig } = await import('../workspace-config.js');
@@ -293,10 +291,6 @@ export async function runKbDoctor(apiClient, options) {
   const appRoot = appMeta.absolutePath;
   const communityId = appMeta.communityId || appMeta.community_id || null;
 
-  // Drift threshold (fraction). CEO guidance 2026-04-20: ~5% is reasonable.
-  const DRIFT_THRESHOLD = Number.isFinite(Number(options.threshold))
-    ? Number(options.threshold) : 0.05;
-
   const scope = communityId ? `${communityId}/${appId}/${kbName}` : `${appId}/${kbName}`;
   console.log(chalk.cyan(`\n🩺 kb doctor — ${scope}\n`));
 
@@ -362,23 +356,33 @@ export async function runKbDoctor(apiClient, options) {
     console.log(chalk.gray(`    Run 'descix kb corpus sync -a ${appId} -k ${kbName}' against this origin first.`));
     process.exit(2);
   }
-  const localChunks = syncStateRaw.total_chunks;
-  if (typeof localChunks !== 'number') {
-    throw new Error(`sync-state has no total_chunks: ${statePathForOrigin}`);
+  // (c) IDENTITY, not count. Which corpus files are LIVE, against which files the last sync
+  // WALKED. The count comparison this replaced read the store against `total_chunks`, which the
+  // sync itself writes FROM the reconciled live count — so drift was zero by construction and
+  // the check could not fail (measured 2026-09-18, GODSWORLD-DEV: "HEALTHY" printed over a
+  // superseded INDEX.md that was still live and retrievable). Identity catches exactly that.
+  if (!Array.isArray(syncStateRaw.synced_blob_shas)) {
+    console.log(chalk.yellow(`  ⚠ sync-state records no synced_blob_shas at ${statePathForOrigin} — the walked set is UNKNOWN.`));
+    process.exit(2);
   }
-
-  // (c) Report
-  const delta = vectorCount - localChunks;
-  const ratio = localChunks === 0 ? Infinity : Math.abs(delta) / localChunks;
-  const direction = delta === 0 ? 'EXACT'
-    : delta < 0 ? 'LOST (Pinecone missing chunks)'
-    : 'ORPHANS (Pinecone has extra vectors)';
-  const healthy = Math.abs(delta) / Math.max(localChunks, 1) <= DRIFT_THRESHOLD;
-
-  console.log(`  Pinecone vectorCount : ${chalk.white(vectorCount)}`);
-  console.log(`  Local total_chunks   : ${chalk.white(localChunks)}`);
-  console.log(`  Drift                : ${chalk.white((delta >= 0 ? '+' : '') + delta)} (${(ratio * 100).toFixed(1)}%)`);
-  console.log(`  Direction            : ${delta === 0 ? chalk.green(direction) : (healthy ? chalk.yellow(direction) : chalk.red(direction))}`);
+  const report = diagnoseCorpusIdentity({
+    liveFileIds: (await listRemoteFileIds(apiClient, appId, kbName)).file_ids,
+    walkedBlobShas: syncStateRaw.synced_blob_shas,
+    zeroChunkBlobShas: syncStateRaw.zero_chunk_blob_shas,
+  });
+
+  console.log(`  Walked by last sync  : ${chalk.white(report.walked)} file(s)` +
+    (report.zeroKnown ? chalk.gray(` (${report.zeroChunk} empty by design)`) : ''));
+  console.log(`  Live corpus files    : ${chalk.white(report.live)}`);
+  console.log(`  Orphans (live, not walked) : ${report.orphans.length ? chalk.red(report.orphans.length) : chalk.green(0)}`);
+  report.orphans.slice(0, 10).forEach((id) => console.log(chalk.red(`    ${id}`)));
+  if (report.zeroKnown) {
+    console.log(`  Missing (walked, not live) : ${report.missing.length ? chalk.red(report.missing.length) : chalk.green(0)}`);
+    report.missing.slice(0, 10).forEach((id) => console.log(chalk.red(`    ${id}`)));
+  } else {
+    console.log(chalk.yellow('  Missing (walked, not live) : UNVERIFIABLE — this state predates the record of empty files; the next sync writes it.'));
+  }
+  console.log(chalk.gray(`  Pinecone vectorCount : ${vectorCount} (informational — it lags deletes and is not a health signal)`));
   console.log(`  Index                : ${ragStatus.indexName || '(unknown)'}`);
   console.log(`  Last sync (server)   : ${ragStatus.lastSync || '(unknown)'}`);
   console.log(`  Last sync (local)    : ${syncStateRaw.timestamp || '(unknown)'}`);
@@ -420,19 +424,37 @@ export async function runKbDoctor(apiClient, options) {
 
   // (e) Exit
   console.log();
-  if (!healthy) {
-    console.log(chalk.red(`  ✗ DRIFT detected: ${(ratio * 100).toFixed(1)}% exceeds ${(DRIFT_THRESHOLD * 100).toFixed(1)}% threshold.\n`));
-    if (delta < 0) {
-      console.log(chalk.gray(`    Recommended: invalidate sync-state and re-sync:`));
-      console.log(chalk.gray(`      rm "${syncStatePath}"`));
-      console.log(chalk.gray(`      descix kb corpus sync -a ${appId} -k ${kbName}\n`));
-    } else {
-      console.log(chalk.gray(`    Recommended: clear stale vectors and re-sync to rebuild Pinecone from local state.\n`));
-    }
-    process.exit(1);
+  if (report.orphans.length > 0) {
+    console.log(chalk.red(`  ✗ ORPHANS: ${report.orphans.length} corpus file(s) are live and retrievable but were not in the last walk.`));
+    console.log(chalk.gray(`    A plain sync purges them (it reconciles against the live KB). Preview it first:`));
+    console.log(chalk.gray(`      descix kb corpus sync -a ${appId} -k ${kbName} --dry-run\n`));
   }
+  if (report.missing.length > 0) {
+    console.log(chalk.red(`  ✗ MISSING: ${report.missing.length} walked file(s) hold chunks but are not live.`));
+    console.log(chalk.gray(`    A plain sync skips files its state already records, so rebuild:`));
+    console.log(chalk.gray(`      descix kb corpus sync -a ${appId} -k ${kbName} --rebuild --dry-run\n`));
+  }
+  if (report.orphans.length > 0 || report.missing.length > 0) process.exit(1);
 
-  console.log(chalk.green(`  ✓ HEALTHY (drift within ${(DRIFT_THRESHOLD * 100).toFixed(1)}% threshold)\n`));
+  if (report.zeroKnown) {
+    console.log(chalk.green(`  ✓ HEALTHY — every live corpus file was walked, and every walked file with content is live.\n`));
+  } else {
+    console.log(chalk.green(`  ✓ No orphans.`) + chalk.yellow(` Missing-content check pending the next sync.\n`));
+  }
+}
+
+/**
+ * Compare the live corpus file set against the last walk. Pure, so the verdict is testable
+ * without a store. Only `corpus:` ids are this manifest's to judge; other schemes are ignored.
+ */
+export function diagnoseCorpusIdentity({ liveFileIds, walkedBlobShas, zeroChunkBlobShas }) {
+  const live = new Set((liveFileIds || []).filter((id) => typeof id === 'string' && id.startsWith('corpus:')));
+  const walked = new Set((walkedBlobShas || []).map((sha) => `corpus:${sha}`));
+  const zeroKnown = Array.isArray(zeroChunkBlobShas);
+  const zero = new Set((zeroChunkBlobShas || []).map((sha) => `corpus:${sha}`));
+  const orphans = [...live].filter((id) => !walked.has(id)).sort();
+  const missing = zeroKnown ? [...walked].filter((id) => !zero.has(id) && !live.has(id)).sort() : [];
+  return { live: live.size, walked: walked.size, zeroKnown, zeroChunk: zero.size, orphans, missing };
 }
 
 export default {

package/lib/core/SiteDenyLint.js

@@ -0,0 +1,69 @@
+/**
+ * What a site upload must never publish. Hard deny — the upload fails loud before any file
+ * leaves the machine, dry-run included.
+ *
+ * WHY (measured, 2026-09-18): egpt-godsworld's DEV site had been publicly serving, as static
+ * files, the app's persona (`assets/system_instructions.md`) and its whole backend source tree
+ * (`microservice/app.js`, `Dockerfile`, `services/*`, config examples). An earlier upload used a
+ * broad manifest, and `site upload` publishes whatever its manifest names. The KB sync already
+ * refuses private content (CorpusDenyLint); the site upload had no equivalent.
+ *
+ * A DIFFERENT PATTERN SET from the corpus, on purpose: the corpus lint is about editorial
+ * publishability (drafts, patents, test dumps), and a site legitimately ships some of those. This
+ * set is about what is private at RUNTIME — backend source, an agent's persona, credentials,
+ * workspace and tooling state. The MATCHER is shared (CorpusDenyLint.matchPathPattern), so there
+ * is one definition of how a pattern meets a path.
+ *
+ * No exemption flag: every class here is repaired by not naming the file in the manifest, which
+ * is always possible. An escape hatch would be the one path the next leak takes.
+ */
+
+import { matchPathPattern } from './CorpusDenyLint.js';
+
+export const SITE_DENY_PATH_PATTERNS = [
+    { id: 1, class: 'app-source', patterns: ['**/microservice/**', 'microservice'],
+      reason: 'backend source — an app\'s microservice is deployed as a service, never served as files' },
+    { id: 2, class: 'agent-persona', patterns: ['system_instructions.md'],
+      reason: 'the agent\'s persona / system instructions — its prompt is not a public asset' },
+    { id: 3, class: 'credential', caseFold: true,
+      patterns: ['wallet*.json', '.env', '.env.*', 'dev-overrides*.json', 'secret*.json', '*.pem', '*.key',
+                 'service-account*.json', 'credentials*.json', 'id_rsa*', '*.p12'],
+      reason: 'credential or credential-shaped config (a placeholder copy teaches the real layout)' },
+    { id: 4, class: 'workspace-state', patterns: ['**/.descix/**', '.descix'],
+      reason: 'DeSciX workspace / session state' },
+    { id: 5, class: 'tooling-state', patterns: ['**/.claude/**', '**/.git/**', '.claude', '.git'],
+      reason: 'agent or VCS internals' },
+];
+
+/**
+ * Every published path that matches a deny class.
+ * @param {string[]} publishedPaths - paths relative to the site root, as they would be served
+ * @returns {Array<{path:string, class:string, glob:string, reason:string}>}
+ */
+export function lintSiteFiles(publishedPaths) {
+    const violations = [];
+    for (const p of publishedPaths) {
+        for (const pattern of SITE_DENY_PATH_PATTERNS) {
+            const glob = matchPathPattern(p, pattern);
+            if (glob) {
+                violations.push({ path: p, class: pattern.class, glob, reason: pattern.reason });
+                break;
+            }
+        }
+    }
+    return violations;
+}
+
+/** Throw, naming every offending file and the repair, when any path is denied. */
+export function assertSitePublishable(publishedPaths) {
+    const violations = lintSiteFiles(publishedPaths);
+    if (violations.length === 0) return;
+    const lines = violations.map((v) => `  - ${v.path}  [${v.class}] ${v.reason}`);
+    const err = new Error(
+        `site upload REFUSED: ${violations.length} file(s) must never be publicly served:\n${lines.join('\n')}\n` +
+        'Nothing was uploaded. Remove these from the site manifest\'s sources (.descix/manifests/site.json) ' +
+        'or from the build output, then upload again.');
+    err.code = 'SITE_DENY_PATH';
+    err.violations = violations;
+    throw err;
+}

package/lib/verb-invokes.generated.json

@@ -143,7 +143,8 @@
     "delete_knowledge_base"
   ],
   "kb doctor": [
-    "get_kb_rag_status"
+    "get_kb_rag_status",
+    "kb_list_file_ids"
   ],
   "kb list": [
     "list_knowledge_bases"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@descix/cli",
-  "version": "1.0.12",
+  "version": "1.0.14",
   "description": "DeSciX CLI and MCP Server - Unified command-line interface for DeSciX platform",
   "repository": {
     "type": "git",