@descix/cli 1.0.11 → 1.0.13

package/bin/descix.js

@@ -2537,6 +2537,11 @@ siteCommand
         return;
       }
 
+      // 1b. Refuse to publish backend source, an agent's persona, credentials or workspace
+      // state — BEFORE any token is requested, so a dry run refuses too (SiteDenyLint).
+      const { assertSitePublishable } = await import('../lib/core/SiteDenyLint.js');
+      assertSitePublishable(fileList.map((f) => f.path));
+
       // 2. Request deploy token with file list
       const tokenResponse = await apiClient.invoke('get_site_deploy_token', {
         community_id: communityId,

package/lib/commands/corpus.js

@@ -33,6 +33,7 @@ import {
 } from '../core/Syncer.js';
 import { lintManifestForDenyClasses, assertNoViolations } from '../core/CorpusDenyLint.js';
 import { pickCanaryChunk, runRetrievalCanary, DEFAULT_CANARY_DELAYS_MS } from '../core/RetrievalCanary.js';
+import { loadSyncState, saveSyncState, syncStatePath, syncFailureLogPath } from '../core/syncState.js';
 
 /**
  * Resolve community_id for corpus sync — same precedence as site upload / update kb:
@@ -207,35 +208,12 @@ export function renderCount(value, unknownText = 'an unreported number of') {
 }
 
 /**
- * Save sync state after a successful corpus sync.
- *
- * @param {string} appRoot - App root directory
- * @param {string} kbName - Knowledge base name
- * @param {Object} state - { last_sync_commit, sources_provenance, synced_files_count, total_chunks, timestamp }
+ * Sync state is owned by lib/core/syncState.js and keyed by the ORIGIN the sync talked to.
+ * The unscoped local helpers that used to live here are DELETED, not wrapped: they were the
+ * defect (one file per KB name, no environment anywhere in it), and a compat path that still
+ * resolved `.descix/sync-state/<KB>.json` would keep producing the silent under-sync this
+ * module exists to end. See that file's header for the measurement.
  */
-async function saveSyncState(appRoot, kbName, state) {
-  const stateDir = path.join(appRoot, '.descix', 'sync-state');
-  await fs.mkdir(stateDir, { recursive: true });
-  const statePath = path.join(stateDir, `${kbName}.json`);
-  await fs.writeFile(statePath, JSON.stringify(state, null, 2));
-}
-
-/**
- * Load sync state from a previous corpus sync.
- *
- * @param {string} appRoot - App root directory
- * @param {string} kbName - Knowledge base name
- * @returns {Promise<Object|null>} Sync state or null if no previous sync
- */
-async function loadSyncState(appRoot, kbName) {
-  const statePath = path.join(appRoot, '.descix', 'sync-state', `${kbName}.json`);
-  try {
-    const raw = await fs.readFile(statePath, 'utf-8');
-    return JSON.parse(raw);
-  } catch {
-    return null;
-  }
-}
 
 /**
  * Detect the current git branch in workspaceRoot.
@@ -398,6 +376,13 @@ export async function runCorpusSync(apiClient, options) {
 
     spinner.succeed(`Found ${manifests.length} manifest(s)`);
 
+    // THE ORIGIN THIS SYNC IS TALKING TO — resolved once, from the client that will actually
+    // carry the upserts, and used to key every sync-state read and write below. Taken from the
+    // client rather than re-derived from the workspace: a re-derivation could name a different
+    // origin than the one receiving the chunks, which is precisely the lie that state file
+    // told for months. Awaited because baseUrl is loaded lazily.
+    const syncOrigin = await apiClient.ensureBaseUrl();
+
     // Deliverable B: branch-mismatch advisory. We only emit this when:
     //   - no --ref override was supplied, AND
     //   - every loaded manifest's every source uses the default "main", AND
@@ -649,7 +634,16 @@ export async function runCorpusSync(apiClient, options) {
       // Deliverable A note: in dry-run + rebuild we ALSO null previousState so
       // the count below reflects what a real rebuild would upsert (all files),
       // not what an incremental sync would upsert.
-      let previousState = await loadSyncState(appRoot, kbName);
+      // State is per-ORIGIN. The blob-SHA skip below is only sound against the origin that
+      // actually received those SHAs — see lib/core/syncState.js.
+      const loadedState = await loadSyncState(appRoot, kbName, syncOrigin);
+      let previousState = loadedState.state;
+      if (loadedState.reason === 'unkeyed-legacy' || loadedState.reason === 'origin-mismatch') {
+        // LOUD, because the consequence of not saying it is a full re-walk the operator did not
+        // ask for, and silence here is what made the original defect invisible for months.
+        console.log(chalk.yellow(`  ⚠ No sync state for ${syncOrigin} — walking the FULL corpus.`));
+        console.log(chalk.gray(`    ${loadedState.detail}`));
+      }
       if (options.rebuild) {
         previousState = null;
       }
@@ -657,9 +651,9 @@ export async function runCorpusSync(apiClient, options) {
       const previousChunkCount = readPreviousChunkCount(previousState);
 
       if (previousBlobShas.size > 0) {
-        console.log(chalk.gray(`  Previous sync: ${previousBlobShas.size} files, ${previousChunkCount ?? 'an unknown number of'} chunks`));
+        console.log(chalk.gray(`  Previous sync to ${syncOrigin}: ${previousBlobShas.size} files, ${previousChunkCount ?? 'an unknown number of'} chunks`));
       } else {
-        console.log(chalk.gray('  First sync (no previous state)'));
+        console.log(chalk.gray(`  First sync to ${syncOrigin} (no previous state for this origin)`));
       }
 
       // 3c. Compute delta — compare by blob SHA against local sync state
@@ -822,10 +816,10 @@ export async function runCorpusSync(apiClient, options) {
 
         // Write failure log to .descix if any batches failed
         if (syncFailures.length > 0) {
-          const failLogDir = path.join(appRoot, '.descix', 'sync-state');
-          await fs.mkdir(failLogDir, { recursive: true });
-          const failLogPath = path.join(failLogDir, `${kbName}-failures.json`);
+          const failLogPath = syncFailureLogPath(appRoot, kbName, syncOrigin);
+          await fs.mkdir(path.dirname(failLogPath), { recursive: true });
           await fs.writeFile(failLogPath, JSON.stringify({
+            origin: syncOrigin,
             timestamp: new Date().toISOString(),
             commit: commitSha,
             failures: syncFailures,
@@ -928,7 +922,7 @@ export async function runCorpusSync(apiClient, options) {
 
       // 3g. Save sync state with blob SHAs for future delta computation
       const allSyncedBlobShas = [...localBlobShas]; // All current files
-      await saveSyncState(appRoot, kbName, {
+      await saveSyncState(appRoot, kbName, syncOrigin, {
         last_sync_commit: commitSha,
         // Per-source provenance: WHICH repo, at WHICH ref, and the RESOLVED COMMIT SHA actually
         // synced. `ref` may be a mutable branch, so the ref alone is not a record of what was
@@ -1048,6 +1042,19 @@ export async function runCorpusStatus(apiClient, options) {
 
     const appRoot = appConfig.absolutePath;
 
+    // Sync state is per-ORIGIN, so status cannot report it without knowing which origin is
+    // being asked about. Resolved from the same client the sync would use — never re-derived
+    // here, which would let status describe one origin while sync wrote another.
+    if (!apiClient) {
+      spinner.fail('Origin required');
+      throw new Error(
+        'corpus status needs an API client to know WHICH origin\'s sync state to report. '
+        + 'Sync state is keyed by origin (see lib/core/syncState.js), so there is no '
+        + 'origin-independent status to print.',
+      );
+    }
+    const statusOrigin = await apiClient.ensureBaseUrl();
+
     // Load manifests
     const manifests = await loadManifests(appRoot, workspaceRoot, options.kb || null);
 
@@ -1064,11 +1071,20 @@ export async function runCorpusStatus(apiClient, options) {
       const refResolution = resolveRef(rawManifest, options.ref || null);
       const manifest = refResolution.manifest;
       const kbName = manifest.kb_name;
-      const syncState = await loadSyncState(appRoot, kbName);
+      // Status reports state for ONE origin — the one this invocation is pointed at. A status
+      // that merged or guessed across origins would be the same unscoped claim the state file
+      // used to make.
+      const loadedState = await loadSyncState(appRoot, kbName, statusOrigin);
+      const syncState = loadedState.state;
 
       console.log(chalk.cyan(`\nCorpus Status: ${appId} / ${kbName}`));
       console.log(chalk.gray('─'.repeat(50)));
+      console.log(chalk.white(`  Origin:          ${statusOrigin}`));
       console.log(chalk.white(`  Resolved ref:    ${refResolution.resolvedRef} (source=${refResolution.source})`));
+      if (loadedState.reason === 'unkeyed-legacy' || loadedState.reason === 'origin-mismatch') {
+        console.log(chalk.yellow(`  ⚠ No sync state for this origin.`));
+        console.log(chalk.gray(`    ${loadedState.detail}`));
+      }
 
       // File count from a fresh walk so the operator sees the LIVE file count
       // at the resolved ref, not the count from the last sync. This is the

package/lib/commands/kb.js

@@ -33,6 +33,7 @@ import * as driveADC from '../google-storage-adc.js';
 // commands' next steps are read by a developer as instructions, so they must name the live verb
 // and nothing else.
 import { CANONICAL_KB_SYNC } from './retired-kb-sync.js';
+import { loadSyncState, syncStatePath } from '../core/syncState.js';
 
 // ============ Pull Command ============
 
@@ -346,24 +347,24 @@ export async function runKbDoctor(apiClient, options) {
     }
   }
 
-  // (b) Local sync-state total_chunks
-  const syncStatePath = path.join(appRoot, '.descix', 'sync-state', `${kbName}.json`);
-  let localChunks = null;
-  let syncStateRaw = null;
-  try {
-    const raw = await fs.readFile(syncStatePath, 'utf-8');
-    syncStateRaw = JSON.parse(raw);
-    localChunks = syncStateRaw?.total_chunks;
-    if (typeof localChunks !== 'number') {
-      throw new Error(`sync-state has no total_chunks: ${syncStatePath}`);
-    }
-  } catch (err) {
-    if (err.code === 'ENOENT') {
-      console.log(chalk.yellow(`  ⚠ No local sync-state at ${syncStatePath}`));
-      console.log(chalk.gray(`    Run 'descix kb corpus sync -a ${appId} -k ${kbName}' first.`));
-      process.exit(2);
-    }
-    throw err;
+  // (b) Local sync-state total_chunks, for THIS ORIGIN.
+  //
+  // The path is resolved by its owner (lib/core/syncState.js), never rebuilt here: doctor
+  // compares a LOCAL record against a LIVE store, so reading one origin's state while querying
+  // another's store would manufacture the very drift this command exists to detect.
+  const doctorOrigin = await apiClient.ensureBaseUrl();
+  const statePathForOrigin = syncStatePath(appRoot, kbName, doctorOrigin);
+  const loadedState = await loadSyncState(appRoot, kbName, doctorOrigin);
+  const syncStateRaw = loadedState.state;
+  if (!syncStateRaw) {
+    console.log(chalk.yellow(`  ⚠ No local sync-state for ${doctorOrigin} at ${statePathForOrigin}`));
+    if (loadedState.detail) console.log(chalk.gray(`    ${loadedState.detail}`));
+    console.log(chalk.gray(`    Run 'descix kb corpus sync -a ${appId} -k ${kbName}' against this origin first.`));
+    process.exit(2);
+  }
+  const localChunks = syncStateRaw.total_chunks;
+  if (typeof localChunks !== 'number') {
+    throw new Error(`sync-state has no total_chunks: ${statePathForOrigin}`);
   }
 
   // (c) Report

package/lib/core/SiteDenyLint.js

@@ -0,0 +1,69 @@
+/**
+ * What a site upload must never publish. Hard deny — the upload fails loud before any file
+ * leaves the machine, dry-run included.
+ *
+ * WHY (measured, 2026-09-18): egpt-godsworld's DEV site had been publicly serving, as static
+ * files, the app's persona (`assets/system_instructions.md`) and its whole backend source tree
+ * (`microservice/app.js`, `Dockerfile`, `services/*`, config examples). An earlier upload used a
+ * broad manifest, and `site upload` publishes whatever its manifest names. The KB sync already
+ * refuses private content (CorpusDenyLint); the site upload had no equivalent.
+ *
+ * A DIFFERENT PATTERN SET from the corpus, on purpose: the corpus lint is about editorial
+ * publishability (drafts, patents, test dumps), and a site legitimately ships some of those. This
+ * set is about what is private at RUNTIME — backend source, an agent's persona, credentials,
+ * workspace and tooling state. The MATCHER is shared (CorpusDenyLint.matchPathPattern), so there
+ * is one definition of how a pattern meets a path.
+ *
+ * No exemption flag: every class here is repaired by not naming the file in the manifest, which
+ * is always possible. An escape hatch would be the one path the next leak takes.
+ */
+
+import { matchPathPattern } from './CorpusDenyLint.js';
+
+export const SITE_DENY_PATH_PATTERNS = [
+    { id: 1, class: 'app-source', patterns: ['**/microservice/**', 'microservice'],
+      reason: 'backend source — an app\'s microservice is deployed as a service, never served as files' },
+    { id: 2, class: 'agent-persona', patterns: ['system_instructions.md'],
+      reason: 'the agent\'s persona / system instructions — its prompt is not a public asset' },
+    { id: 3, class: 'credential', caseFold: true,
+      patterns: ['wallet*.json', '.env', '.env.*', 'dev-overrides*.json', 'secret*.json', '*.pem', '*.key',
+                 'service-account*.json', 'credentials*.json', 'id_rsa*', '*.p12'],
+      reason: 'credential or credential-shaped config (a placeholder copy teaches the real layout)' },
+    { id: 4, class: 'workspace-state', patterns: ['**/.descix/**', '.descix'],
+      reason: 'DeSciX workspace / session state' },
+    { id: 5, class: 'tooling-state', patterns: ['**/.claude/**', '**/.git/**', '.claude', '.git'],
+      reason: 'agent or VCS internals' },
+];
+
+/**
+ * Every published path that matches a deny class.
+ * @param {string[]} publishedPaths - paths relative to the site root, as they would be served
+ * @returns {Array<{path:string, class:string, glob:string, reason:string}>}
+ */
+export function lintSiteFiles(publishedPaths) {
+    const violations = [];
+    for (const p of publishedPaths) {
+        for (const pattern of SITE_DENY_PATH_PATTERNS) {
+            const glob = matchPathPattern(p, pattern);
+            if (glob) {
+                violations.push({ path: p, class: pattern.class, glob, reason: pattern.reason });
+                break;
+            }
+        }
+    }
+    return violations;
+}
+
+/** Throw, naming every offending file and the repair, when any path is denied. */
+export function assertSitePublishable(publishedPaths) {
+    const violations = lintSiteFiles(publishedPaths);
+    if (violations.length === 0) return;
+    const lines = violations.map((v) => `  - ${v.path}  [${v.class}] ${v.reason}`);
+    const err = new Error(
+        `site upload REFUSED: ${violations.length} file(s) must never be publicly served:\n${lines.join('\n')}\n` +
+        'Nothing was uploaded. Remove these from the site manifest\'s sources (.descix/manifests/site.json) ' +
+        'or from the build output, then upload again.');
+    err.code = 'SITE_DENY_PATH';
+    err.violations = violations;
+    throw err;
+}

package/lib/core/syncState.js

@@ -0,0 +1,144 @@
+/**
+ * syncState — THE owner of where a KB's local sync state lives, and which ORIGIN it describes.
+ *
+ * ── The defect this closes (measured 2026-09-17, two apps) ───────────────────
+ * State used to live at `.descix/sync-state/<KB>.json` — one file per KB NAME, with no
+ * environment and no origin anywhere in the path or the payload. `corpus sync` skips a file
+ * when its blob SHA appears in that state, and nothing asked WHICH ORIGIN had received those
+ * SHAs. Syncing one KB to two environments therefore produced two failures from one cause:
+ *
+ *   UNDER-SYNC  the second origin skips files it never received, as "unchanged"
+ *   BLINDING    the first origin's drift becomes invisible, because the second origin's
+ *               commit was written over the shared state — a stale KB reporting healthy
+ *
+ * Measured: GodsWorld on DEV took 2 of 8 files and then reported PROD clean ("Drift: NONE")
+ * over a PROD KB two files behind. Independently, daita/General's one state file described
+ * DEV (886 chunks, matching DEV's 886 vectors) while PROD sat on a different corpus (2545
+ * vectors, 2026-05-29); a PROD dry-run against it declared 43 files "Unchanged" that PROD had
+ * never received at those SHAs.
+ *
+ * ── The fix ─────────────────────────────────────────────────────────────────
+ * State is keyed by the ORIGIN the sync actually talked to — the same fact `lib/origin.js`
+ * owns and that every MCP result is stamped with. The origin is used because it is the fact
+ * that is ALWAYS available: an env NAME is not (a workspace `env.apiUrl` or a DESCIX_API_URL
+ * carries no label), and keying on a name would also miss a repointed workspace — same label,
+ * different origin — which is the same lie one level up. The full origin is recorded INSIDE
+ * the file as well, so a slug collision is caught rather than trusted.
+ *
+ * ── Migration: an unkeyed file belongs to NOBODY ─────────────────────────────
+ * A legacy `.descix/sync-state/<KB>.json` records a sync to an origin nobody wrote down.
+ * Adopting it for whichever origin asks first would hard-code today's accident as tomorrow's
+ * truth — exactly the bug, re-created by the fix. So it is reported as UNKNOWN-ORIGIN and
+ * forces one full walk, for every origin, and is never read as state and never deleted: it is
+ * the only surviving record of what was synced before keying existed.
+ */
+
+import path from 'path';
+import fs from 'fs/promises';
+
+/** Directory holding every origin's state for an app. */
+export function syncStateRoot(appRoot) {
+  return path.join(appRoot, '.descix', 'sync-state');
+}
+
+/**
+ * A filename-safe token for an origin. ONE derivation, used for the directory name; the exact
+ * origin is recorded in the file, so this only has to be stable and readable, not injective.
+ *
+ * `https://dev.descix.net` -> `dev.descix.net`
+ * `https://localhost:4000` -> `localhost-4000`
+ *
+ * @param {string} origin an absolute http(s) origin, already normalized by lib/origin.js
+ * @returns {string}
+ * @throws {Error} on an empty or non-string origin — a state file keyed by "undefined" is
+ *   exactly the unscoped file this module exists to abolish, so it must never be writable.
+ */
+export function originKey(origin) {
+  if (typeof origin !== 'string' || origin.trim() === '') {
+    throw new Error(
+      'syncState: refusing to key sync state by an empty origin. The caller must pass the '
+      + 'origin the sync actually talked to (apiClient.baseUrl) — an unkeyed state file is the '
+      + 'defect this module exists to prevent.',
+    );
+  }
+  return origin
+    .trim()
+    .replace(/^https?:\/\//, '')
+    .replace(/\/+$/, '')
+    .replace(/[^A-Za-z0-9._-]/g, '-');
+}
+
+/** Where this origin's state for this KB lives. */
+export function syncStatePath(appRoot, kbName, origin) {
+  return path.join(syncStateRoot(appRoot), originKey(origin), `${kbName}.json`);
+}
+
+/**
+ * Where a failed-batch log for this origin lives. Keyed for the same reason the state is: a
+ * failure log named only by KB let a DEV run overwrite the record of which PROD batches failed,
+ * so the surviving file described an environment nobody could identify from it.
+ */
+export function syncFailureLogPath(appRoot, kbName, origin) {
+  return path.join(syncStateRoot(appRoot), originKey(origin), `${kbName}-failures.json`);
+}
+
+/** Where an unkeyed (pre-origin-keying) file would be. Read only to DETECT it. */
+export function legacySyncStatePath(appRoot, kbName) {
+  return path.join(syncStateRoot(appRoot), `${kbName}.json`);
+}
+
+/**
+ * Load the state for THIS origin, or report why there is none.
+ *
+ * @returns {Promise<{state: Object|null, reason: 'loaded'|'first-sync'|'unkeyed-legacy'|'origin-mismatch', detail?: string}>}
+ *   `state` is null for every reason but 'loaded'. A null state means a FULL walk, which is
+ *   always safe: it re-upserts rather than purging, and it is self-correcting.
+ */
+export async function loadSyncState(appRoot, kbName, origin) {
+  const statePath = syncStatePath(appRoot, kbName, origin);
+  try {
+    const parsed = JSON.parse(await fs.readFile(statePath, 'utf-8'));
+    // Defensive: the directory is a slug, the payload is exact. They can only disagree through
+    // a slug collision, and trusting the slug in that case would resurrect the original bug.
+    if (parsed.origin && parsed.origin !== origin) {
+      return {
+        state: null,
+        reason: 'origin-mismatch',
+        detail: `${statePath} records origin ${parsed.origin}, but this sync is talking to ${origin}. `
+          + 'Treating it as no state and walking the full corpus.',
+      };
+    }
+    return { state: parsed, reason: 'loaded' };
+  } catch {
+    // No state for this origin. Is there an unkeyed file from before keying existed?
+    const legacy = legacySyncStatePath(appRoot, kbName);
+    try {
+      await fs.access(legacy);
+      return {
+        state: null,
+        reason: 'unkeyed-legacy',
+        detail: `${legacy} was written before sync state was keyed by origin, so it does not say `
+          + `WHICH environment it describes. It is not being read — this sync walks the full corpus `
+          + `once for ${origin}, and writes its own state at ${statePath}. The old file is left `
+          + `alone as the only record of what came before.`,
+      };
+    } catch {
+      return { state: null, reason: 'first-sync' };
+    }
+  }
+}
+
+/**
+ * Persist state for THIS origin. The origin is written into the payload as well as the path.
+ *
+ * @param {string} appRoot
+ * @param {string} kbName
+ * @param {string} origin
+ * @param {Object} state the same shape as before, minus provenance this function adds
+ */
+export async function saveSyncState(appRoot, kbName, origin, state) {
+  const statePath = syncStatePath(appRoot, kbName, origin);
+  await fs.mkdir(path.dirname(statePath), { recursive: true });
+  await fs.writeFile(statePath, JSON.stringify({ origin, ...state }, null, 2));
+  return statePath;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@descix/cli",
-  "version": "1.0.11",
+  "version": "1.0.13",
   "description": "DeSciX CLI and MCP Server - Unified command-line interface for DeSciX platform",
   "repository": {
     "type": "git",