@descix/cli 1.0.0 → 1.0.3

package/lib/core/SiteWalker.js

@@ -0,0 +1,218 @@
+/**
+ * SiteWalker - Manifest-driven file walker for site deployment
+ *
+ * Given a site manifest (.descix/manifests/site.json), walks source entries,
+ * applies include/exclude glob patterns, and computes file hashes for
+ * delta-based deployment to GCS.
+ *
+ * Architecture:
+ * - Files stay in place — no copying or staging
+ * - File hashes (SHA-256) enable delta uploads (skip unchanged files)
+ * - Multiple source entries are flattened into a deploy tree preserving
+ *   relative paths from each source root
+ * - Unlike CorpusWalker, accepts any file type (HTML, CSS, JS, images, fonts, etc.)
+ *   and uses include/exclude globs instead of syncignore + extension filtering
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { createHash } from 'crypto';
+import { minimatch } from 'minimatch';
+
+/**
+ * Check if a relative path matches any of the given glob patterns.
+ *
+ * @param {string} relativePath - Path relative to the source entry root
+ * @param {string[]} patterns - Array of glob patterns
+ * @returns {boolean} True if the path matches at least one pattern
+ */
+function matchesAny(relativePath, patterns) {
+  for (const pattern of patterns) {
+    if (minimatch(relativePath, pattern, { dot: true })) {
+      return true;
+    }
+    // Also check just the filename for patterns like "*.css"
+    const basename = path.basename(relativePath);
+    if (minimatch(basename, pattern, { dot: true })) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Check if a directory name matches any exclude patterns that look like directory names.
+ * Patterns ending with "/" are directory excludes (e.g., "node_modules/").
+ *
+ * @param {string} dirName - Directory name (not full path)
+ * @param {string[]} excludePatterns - Exclude patterns
+ * @returns {boolean} True if the directory should be skipped entirely
+ */
+function isDirExcluded(dirName, excludePatterns) {
+  for (const pattern of excludePatterns) {
+    // Pattern ending with / targets directories
+    if (pattern.endsWith('/') && dirName === pattern.slice(0, -1)) {
+      return true;
+    }
+    // Also match directory-style patterns without trailing slash
+    if (dirName === pattern) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Compute SHA-256 hash of a file's contents.
+ *
+ * @param {string} absolutePath - Absolute path to the file
+ * @returns {Promise<string>} Hex-encoded SHA-256 hash
+ */
+async function computeFileHash(absolutePath) {
+  const content = await fs.readFile(absolutePath);
+  return createHash('sha256').update(content).digest('hex');
+}
+
+/**
+ * Get file size in bytes.
+ *
+ * @param {string} absolutePath - Absolute path to the file
+ * @returns {Promise<number>} File size
+ */
+async function getFileSize(absolutePath) {
+  const stat = await fs.stat(absolutePath);
+  return stat.size;
+}
+
+/**
+ * Recursively walk a directory and collect file paths,
+ * applying include and exclude glob filters.
+ *
+ * @param {string} dir - Directory to walk
+ * @param {string} baseDir - Base directory for computing relative paths
+ * @param {string[]} include - Include glob patterns (empty = include all)
+ * @param {string[]} exclude - Exclude glob patterns
+ * @returns {Promise<string[]>} Array of absolute file paths that pass filters
+ */
+async function walkDir(dir, baseDir, include, exclude) {
+  const results = [];
+
+  let entries;
+  try {
+    entries = await fs.readdir(dir, { withFileTypes: true });
+  } catch {
+    return results; // Directory doesn't exist or unreadable
+  }
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    const relativePath = path.relative(baseDir, fullPath);
+
+    if (entry.isDirectory()) {
+      // Skip excluded directories early (avoids walking into node_modules, etc.)
+      if (isDirExcluded(entry.name, exclude)) {
+        continue;
+      }
+      // Also check full relative path against exclude patterns
+      if (matchesAny(relativePath + '/', exclude) || matchesAny(relativePath, exclude)) {
+        continue;
+      }
+      const subFiles = await walkDir(fullPath, baseDir, include, exclude);
+      results.push(...subFiles);
+    } else if (entry.isFile()) {
+      // Check exclude first
+      if (exclude.length > 0 && matchesAny(relativePath, exclude)) {
+        continue;
+      }
+      // Check include (empty include = match all)
+      if (include.length === 0 || matchesAny(relativePath, include)) {
+        results.push(fullPath);
+      }
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Walk a site manifest and return file entries with hashes for deployment.
+ *
+ * For each source entry in the manifest:
+ * 1. Resolve the local path (relative to app root / workspace root)
+ * 2. Walk the directory, applying include/exclude globs
+ * 3. Compute SHA-256 hash for each file
+ * 4. Return file list with deploy paths (relative to deploy root)
+ *
+ * @param {Object} manifest - Validated site manifest with _resolvedSources
+ * @param {string} appRoot - Absolute path to the app root directory
+ * @returns {Promise<Object>} { files: Array<SiteFileEntry>, totalSize: number }
+ *
+ * SiteFileEntry: {
+ *   absolutePath: string,       // Local filesystem path
+ *   deployPath: string,         // Path in the deploy tree (relative)
+ *   hash: string,               // SHA-256 hex hash
+ *   size: number,               // File size in bytes
+ *   sourceEntry: string         // Source manifest path for traceability
+ * }
+ */
+export async function walkSite(manifest, appRoot) {
+  const files = [];
+  let totalSize = 0;
+
+  for (const source of manifest._resolvedSources) {
+    const { absolutePath, include, exclude, sourcePath } = source;
+
+    // Check if source is a single file
+    let stat;
+    try {
+      stat = await fs.stat(absolutePath);
+    } catch (err) {
+      if (err.code === 'ENOENT') {
+        // Source doesn't exist — skip with warning (caller can log)
+        continue;
+      }
+      throw err;
+    }
+
+    if (stat.isFile()) {
+      // Single file source entry — deploy at the filename level
+      const hash = await computeFileHash(absolutePath);
+      const size = stat.size;
+      totalSize += size;
+
+      files.push({
+        absolutePath,
+        deployPath: path.basename(absolutePath),
+        hash,
+        size,
+        sourceEntry: sourcePath
+      });
+    } else if (stat.isDirectory()) {
+      // Directory: walk with include/exclude
+      const filePaths = await walkDir(absolutePath, absolutePath, include, exclude);
+
+      for (const filePath of filePaths) {
+        const hash = await computeFileHash(filePath);
+        const size = await getFileSize(filePath);
+        totalSize += size;
+
+        // Deploy path is relative to the source directory root (not the source entry label).
+        // e.g. source.path = "site/dist" → dist/index.html under {env}/{app}/site/
+        const relativeToSource = path.relative(absolutePath, filePath);
+        const deployPath = relativeToSource.split(path.sep).join('/');
+
+        files.push({
+          absolutePath: filePath,
+          deployPath,
+          hash,
+          size,
+          sourceEntry: sourcePath
+        });
+      }
+    }
+  }
+
+  return { files, totalSize };
+}
+
+export default { walkSite };

package/lib/core/Syncer.js

@@ -89,7 +89,9 @@ export async function getRemoteChunkMetadata(apiClient, communityId, appId, kbId
       app_id: appId,
       kb_id: kbId
     });
-    return result.chunks || [];
+    // apiClient.invoke returns { status, message: { chunks, count, ... } }
+    const data = result.message || result;
+    return data.chunks || [];
   } catch (error) {
     // Fall back to IDs-only endpoint
     try {
@@ -97,8 +99,9 @@ export async function getRemoteChunkMetadata(apiClient, communityId, appId, kbId
         app_id: appId,
         kb_id: kbId
       });
-      // Return as array of IDs (old format) for backward compatibility
-      return result.chunk_ids || [];
+      // apiClient.invoke returns { status, message: { chunk_ids, count, ... } }
+      const data = result.message || result;
+      return data.chunk_ids || [];
     } catch {
       return [];
     }
@@ -151,8 +154,11 @@ export async function upsertChunks(apiClient, communityId, appId, kbId, chunks)
 }
 
 /**
- * Delete stale chunks from Pinecone
- * 
+ * Delete stale chunks from Pinecone by explicit chunk IDs.
+ *
+ * Use this when you have the exact chunk_ids to purge (e.g., shrinking file
+ * where chunks N..M no longer exist locally but linger in Pinecone).
+ *
  * @param {Object} apiClient - DeSciXApiClient instance
  * @param {string} communityId - Community ID
  * @param {string} appId - App ID
@@ -164,14 +170,127 @@ export async function deleteStaleChunks(apiClient, communityId, appId, kbId, chu
   if (chunkIds.length === 0) {
     return { deleted: 0 };
   }
-  
-  await apiClient.invoke('kb_delete_chunks', {
+
+  const result = await apiClient.invoke('kb_delete_chunks', {
+    community_id: communityId,
     app_id: appId,
     kb_id: kbId,
     chunk_ids: chunkIds
   });
-  
-  return { deleted: chunkIds.length };
+
+  const data = result?.message || result;
+  return { deleted: data.deleted_count ?? chunkIds.length };
+}
+
+/**
+ * Delete stale chunks from Pinecone by file_id (metadata filter).
+ *
+ * This is the canonical stale-purge path for `kb corpus sync`: when a file's
+ * blob_sha changes, the prior blob_sha is a stale file_id. Passing it here
+ * deletes every chunk where metadata.file_id matches, scoped by the full
+ * multi-tenancy filter (community_id, app_id, knowledgebase_name) on the
+ * server side via pineconeService.deleteVectorsByFileId.
+ *
+ * Bulk: pass multiple file_ids in one call to amortize the round-trip.
+ *
+ * @param {Object} apiClient - DeSciXApiClient instance
+ * @param {string} communityId - Community ID
+ * @param {string} appId - App ID
+ * @param {string} kbId - Knowledge base ID
+ * @param {Array<string>} fileIds - file_id values to delete (e.g., ['corpus:abc...', 'local:def...'])
+ * @returns {Promise<{deleted: number, deleted_by_file_id: number}>}
+ */
+export async function deleteStaleChunksByFileId(apiClient, communityId, appId, kbId, fileIds) {
+  if (!Array.isArray(fileIds) || fileIds.length === 0) {
+    return { deleted: 0, deleted_by_file_id: 0 };
+  }
+
+  // Filter out null/undefined/empty entries — pineconeService treats empty fileId as 0-count no-op
+  const validFileIds = fileIds.filter(Boolean);
+  if (validFileIds.length === 0) {
+    return { deleted: 0, deleted_by_file_id: 0 };
+  }
+
+  const result = await apiClient.invoke('kb_delete_chunks', {
+    community_id: communityId,
+    app_id: appId,
+    kb_id: kbId,
+    file_ids: validFileIds
+  });
+
+  const data = result?.message || result;
+  return {
+    deleted: data.deleted_count ?? validFileIds.length,
+    deleted_by_file_id: data.deleted_by_file_id ?? validFileIds.length
+  };
+}
+
+/**
+ * Purge EVERY vector in a KB's {community_id, app_id, knowledgebase_name} scope.
+ *
+ * This is the FULL metadata-scoped purge — it clears all vectors regardless of
+ * their file_id scheme, including legacy/orphan vectors that carry no `corpus:`
+ * file_id and are therefore INVISIBLE to listRemoteFileIds (the corpus-scheme
+ * enumeration). It is the only primitive that fully clears accumulated orphan
+ * pollution before a `--rebuild` re-upsert.
+ *
+ * Server-side it routes to kb_delete_chunks(purge_scope: true), which delegates
+ * to the canonical KnowledgeBase.deleteRAG(). As of WS-KB-CORPUS-SCOPEPURGE-SCALE
+ * that purge is ROBUST AT SCALE: it enumerates the KB's vectors by their canonical
+ * id-prefix `${app_id}:${kb_id}:` (deterministic listPaginated) and deletes them in
+ * bounded batches of 1000, so an arbitrarily large scope (~20k+ vectors) is cleared
+ * without exceeding the Pinecone client's max-retries ceiling (the prior single
+ * giant metadata deleteMany failed at that size). It still clears ALL orphans
+ * (they share the prefix) AND resets the KB doc's rag fields (rag_vector_count→0)
+ * so `kb doctor` reports clean drift after the subsequent re-sync. The rag-field
+ * reset happens ONLY after the live purge completes. Touches Pinecone vectors +
+ * the KB doc's rag fields ONLY — no Firestore/GCS/Products cascade. On a partial
+ * batch failure it FAILS LOUD (recoverable by re-running — idempotent).
+ *
+ * @param {Object} apiClient - DeSciXApiClient instance
+ * @param {string} communityId - Community ID (required server-side; full scope)
+ * @param {string} appId - App ID
+ * @param {string} kbId - Knowledge base ID
+ * @returns {Promise<{ deleted: number, purged_scope: boolean }>}
+ *          `deleted` is the actual count of vectors purged (the batched path always
+ *          reports it); -1 only if a future primitive cannot report a count.
+ */
+export async function purgeKbScope(apiClient, communityId, appId, kbId) {
+  const result = await apiClient.invoke('kb_delete_chunks', {
+    community_id: communityId,
+    app_id: appId,
+    kb_id: kbId,
+    purge_scope: true
+  });
+  const data = result?.message || result;
+  return {
+    deleted: typeof data?.deleted_count === 'number' ? data.deleted_count : -1,
+    purged_scope: data?.purged_scope === true
+  };
+}
+
+/**
+ * Enumerate all file_id values currently present in a KB's Pinecone namespace.
+ *
+ * Used by `descix kb corpus sync --rebuild` to compute drift:
+ *   stale = pinecone_file_ids − current_local_file_ids
+ *
+ * @param {Object} apiClient - DeSciXApiClient instance
+ * @param {string} appId - App ID
+ * @param {string} kbId - Knowledge base ID
+ * @returns {Promise<{ file_ids: string[], unique_count: number, total_chunks: number }>}
+ */
+export async function listRemoteFileIds(apiClient, appId, kbId) {
+  const result = await apiClient.invoke('kb_list_file_ids', {
+    app_id: appId,
+    kb_id: kbId
+  });
+  const data = result?.message || result;
+  return {
+    file_ids: data.file_ids || [],
+    unique_count: data.unique_count || 0,
+    total_chunks: data.total_chunks || 0
+  };
 }
 
 /**
@@ -255,10 +374,40 @@ export async function syncKb(apiClient, config, options = {}) {
   } else {
     if (onProgress) onProgress('All chunks already in sync');
   }
-  
+
+  // 6. File-level stale purge (belt-and-suspenders).
+  // The chunk_id-based delete in step 4 handles the common shrinking/deletion case
+  // but only sees chunk_ids that were returned by getRemoteChunkMetadata. For
+  // robustness — and to close DN-2 at the lower-level kb sync surface too — we
+  // also enumerate remote file_ids and delete any whose file_id is not present
+  // in the current local chunk set. This catches file deletions and modifications
+  // that produced new chunk_id shapes (e.g., content-hash-based ids).
+  let deletedByFileId = 0;
+  try {
+    const localFileIds = new Set(localChunks.map(c => c.file_id).filter(Boolean));
+    const remoteFileIdsResp = await apiClient.invoke('kb_list_file_ids', {
+      app_id: appId,
+      kb_id: kbId
+    });
+    const remoteData = remoteFileIdsResp?.message || remoteFileIdsResp;
+    const remoteFileIds = remoteData?.file_ids || [];
+    const staleFileIds = remoteFileIds.filter(fid => !localFileIds.has(fid));
+    if (staleFileIds.length > 0) {
+      if (onProgress) onProgress(`Purging ${staleFileIds.length} stale file_id(s)...`);
+      const purgeResult = await deleteStaleChunksByFileId(apiClient, communityId, appId, kbId, staleFileIds);
+      deletedByFileId = purgeResult.deleted;
+    }
+  } catch (purgeErr) {
+    // Non-fatal: surface via verbose log. The chunk_id-based delete above is the primary
+    // path; this is a safety net. We do NOT silently swallow — caller sees a warning.
+    if (verbose) console.log(`  File-id purge skipped: ${purgeErr.message}`);
+  }
+
   return {
     synced,
-    deleted,
+    deleted: deleted + deletedByFileId,
+    deleted_chunk_ids: deleted,
+    deleted_by_file_id: deletedByFileId,
     unchanged
   };
 }
@@ -306,5 +455,8 @@ export default {
   getRemoteChunkMetadata,
   upsertChunks,
   deleteStaleChunks,
+  deleteStaleChunksByFileId,
+  purgeKbScope,
+  listRemoteFileIds,
   getSyncStatus
 };

package/lib/diagnostic-log.js

@@ -0,0 +1,41 @@
+/**
+ * THE ONE OWNER of every write the CLI makes ABOUT ITS OWN EXECUTION.
+ *
+ * WHY THIS EXISTS (measured 2026-08-30). `bin/mcp-server.js` speaks JSON-RPC over stdio, so its
+ * stdout IS the protocol stream. Driving one `tools/call` put 130 bytes of non-JSON-RPC text into
+ * that stream, from `console.log` calls in `lib/api-client.js`, on the credential-refresh path —
+ * which fires exactly when a token expires, the moment a long-running MCP session is least able
+ * to survive a corrupt frame.
+ *
+ * THE SCOPE IS THE MEASURED IMPORT GRAPH, NOT A GREP. Walking the real import graph from
+ * `bin/mcp-server.js` (static and dynamic), 12 files are reachable carrying 74 `console.log`
+ * sites — `lib/api-client.js` 9 and `lib/commands/auth.js` 65. The package-wide count is 1108;
+ * that number is not the target and never was. Everything on the reachable graph writes through
+ * here.
+ *
+ * WHY A WRITER AND NOT A WRAPPER. There WAS already a fix, and its shape is the lesson:
+ * `mcpListTools()` monkey-patched `console.log = console.error` for the duration of ONE call and
+ * restored it in a `finally`. A call-site workaround for a write-site defect — it protected the
+ * one path its author was looking at and left every other path through `invoke()`/`invokeRaw()`
+ * corrupting the stream. It is DELETED in the same change that makes it unnecessary; a fix that
+ * does not sweep the workarounds it invalidates leaves a time bomb primed toward silence.
+ *
+ * STDERR, ALWAYS, WITH NO MODE SWITCH. A stream decision that can be configured is one that will
+ * eventually be configured wrongly, and the caller who gets it wrong is a protocol transport that
+ * cannot report its own corruption. On a terminal stderr is still visible to the human, so the
+ * interactive login flow reads exactly as before; what changes is that `descix login > file` no
+ * longer captures the prompts, because prompts were never the command's OUTPUT.
+ *
+ * WHAT DOES NOT BELONG HERE: anything the user ASKED FOR — a command's JSON document, a report
+ * like `config show`, a value meant to be piped. Those are OUTPUT and they belong on stdout.
+ * `bin/descix.js::progress()` owns the narrower question of where human progress text goes when
+ * `--json` makes stdout a data channel; this owns the general one.
+ */
+
+/**
+ * Write a diagnostic / human-progress line. Same call shape as console.log; always stderr.
+ * @param {...any} args
+ */
+export function diag(...args) {
+    console.error(...args);
+}

package/lib/environment-report.js

@@ -0,0 +1,100 @@
+/**
+ * THE ONE OWNER OF "which environment is this origin, and how does the CLI SAY SO".
+ *
+ * WHY THIS FILE EXISTS: contract I1 (A', rev 2) requires that every network-bound command
+ * PRINT the resolved environment, origin and source — always, not only when it landed on the
+ * default. That print needs two facts: the env NAME for an origin, and one spelling of the
+ * line. Both were about to be derived in a second place.
+ *
+ * `wallet-environment.js` already derived origin -> env name privately, for the wallet stamp.
+ * Two derivations of one fact is the mirror-drift bug class this contract exists to end, so
+ * the derivation moved HERE and the wallet stamp became a consumer of it. There is exactly one
+ * answer to "what environment is https://dev.descix.net", and both surfaces read it from here.
+ *
+ * The origin TABLE is still owned upstream (`@descix/app-sdk/dev` ENV_ORIGINS, surfaced as
+ * `WorkspaceConfig.ENV_MAP`). This module holds no copy of it.
+ */
+import { WorkspaceConfig } from './workspace-config.js';
+import { normalizeOrigin } from './origin.js';
+
+/**
+ * The environment NAME for an origin.
+ *
+ * @param {string} origin
+ * @returns {string} the ENV_MAP name whose url matches (`dev` | `demo` | `prod`), or `'custom'`
+ *          when the origin is a real one that simply is not in the map — a self-hosted or
+ *          port-forwarded gateway. 'custom' is an HONEST answer ("a real origin, not one of the
+ *          three named ones"), not a fallback standing in for a missing fact: the origin itself
+ *          is always printed beside it.
+ * @throws when called with no origin. There is no default: guessing which environment an
+ *         origin belongs to is exactly the misreport this contract exists to end.
+ */
+export function environmentNameFor(origin) {
+    if (!origin || String(origin).trim() === '') {
+        throw new Error(
+            'environmentNameFor: refusing to name the environment of an empty origin — ' +
+            'it cannot be guessed.',
+        );
+    }
+    const normalized = normalizeOrigin(origin);
+    const match = Object.entries(WorkspaceConfig.ENV_MAP)
+        .find(([, entry]) => entry && normalizeOrigin(entry.url) === normalized);
+    return match ? match[0] : 'custom';
+}
+
+/**
+ * The ONE spelling of the env line every network-bound command prints.
+ *
+ * Shape is the contract's own (I1, rev 2): `env: <name> (<source>) <origin>`. The SOURCE is the
+ * whole point — it is the fact whose absence made "the developer chose production" and "the
+ * developer chose nothing" one observable state on 1.0.1.
+ *
+ * @param {{origin: string, source: string}} resolved - as returned by `resolveOrigin`
+ * @returns {string} the line, WITHOUT a trailing newline
+ */
+export function formatEnvLine({ origin, source }) {
+    return `env: ${environmentNameFor(origin)} (${source}) ${origin}`;
+}
+
+/**
+ * The SOURCE label for an origin handed in by the caller rather than resolved from config —
+ * `--api-url`, `descix auth login -u`, a microservice constructing a client against a known
+ * origin. Exported so the label has one spelling.
+ */
+export const EXPLICIT_ORIGIN_SOURCE = 'explicit --api-url / caller-supplied baseUrl';
+
+/**
+ * Distinct (origin, source) pairs already reported in this process.
+ *
+ * WHY DEDUPE AND NOT A ONE-SHOT FLAG: a single command constructs many clients (87 sites), so
+ * a per-client print would emit the same line a dozen times and train the reader to skim past
+ * it. But a one-shot flag would HIDE a second, DIFFERENT origin being contacted in the same
+ * invocation — which is precisely the kind of thing this contract exists to surface. Keying on
+ * the pair prints each distinct target exactly once.
+ */
+const reported = new Set();
+
+/**
+ * PRINT the resolved environment for a network-bound invocation. Always — not only when the
+ * origin came from the declared default.
+ *
+ * STDERR, DELIBERATELY. stdout is a DATA CHANNEL under `--json` and is the protocol channel for
+ * the MCP stdio server; a status line on stdout would corrupt both. stderr is where a human
+ * piping stdout still sees it.
+ *
+ * @param {{origin: string, source: string}} resolved
+ * @returns {string|null} the line printed, or null when this pair was already reported
+ */
+export function reportEnvironment({ origin, source }) {
+    const key = `${origin}\u0000${source}`;
+    if (reported.has(key)) return null;
+    reported.add(key);
+    const line = formatEnvLine({ origin, source });
+    process.stderr.write(line + '\n');
+    return line;
+}
+
+/** TEST SEAM ONLY: forget what has been reported, so a test can observe the print again. */
+export function _resetEnvironmentReportForTests() {
+    reported.clear();
+}

package/lib/global-config.js

@@ -20,7 +20,10 @@ export class GlobalConfig {
   constructor(config = {}) {
     this.user_base_folder_id = config.user_base_folder_id || null;
     this.user_base_folder_name = config.user_base_folder_name || null;
-    this.api_url = config.api_url || 'https://descix.net';
+    // NO default. An absent api_url means the global config names no origin, which is a fact
+    // the origin owner knows how to report; substituting the production literal here made
+    // ~/.descix/config.json claim an origin the user never set.
+    this.api_url = config.api_url || null;
     this.environment = config.environment || 'production';
     this.registered_at = config.registered_at || null;
   }