@descix/cli 1.0.0

package/lib/core/PathContext.js

@@ -0,0 +1,335 @@
+/**
+ * PathContext - Unified filesystem context for DeSciX CLI
+ * 
+ * Design principle: No "finding" after setup.
+ * - Setup time: Detect IDE markers, persist absolute paths to workspace.json
+ * - Runtime: Load workspace.json, all paths are simple lookups
+ * 
+ * The workspace.json is the single source of truth for all paths.
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+/**
+ * Find IDE workspace root by searching for .cursor/ or .vscode/ markers
+ * This is the only "searching" allowed - IDE markers are guaranteed in Cursor/VSCode
+ * 
+ * @param {string} startDir - Directory to start searching from
+ * @returns {Promise<string|null>} Workspace root path, or null if not in IDE workspace
+ */
+export async function findIDEMarker(startDir = process.cwd()) {
+  let currentDir = path.resolve(startDir);
+  const root = path.parse(currentDir).root;
+  
+  // Only look for IDE markers (not .descix - that's created by setup)
+  const ideMarkers = ['.cursor', '.vscode'];
+  
+  while (currentDir !== root) {
+    for (const marker of ideMarkers) {
+      const markerDir = path.join(currentDir, marker);
+      try {
+        const stat = await fs.stat(markerDir);
+        if (stat.isDirectory()) {
+          return currentDir;
+        }
+      } catch {
+        // Marker not found, continue
+      }
+    }
+    
+    const parentDir = path.dirname(currentDir);
+    if (parentDir === currentDir) break;
+    currentDir = parentDir;
+  }
+  
+  return null;
+}
+
+/**
+ * PathContext - Unified path resolution from workspace.json
+ * 
+ * All paths come from workspace.json - no runtime "finding" of app contexts.
+ */
+export class PathContext {
+  /**
+   * @param {string} workspaceRoot - Absolute path to workspace root
+   * @param {Object} config - Parsed workspace.json contents
+   */
+  constructor(workspaceRoot, config) {
+    this.workspaceRoot = workspaceRoot;
+    this.config = config;
+  }
+
+  /**
+   * Load PathContext from workspace.json
+   * 
+   * @param {string} startDir - Directory to start searching from (default: cwd)
+   * @returns {Promise<PathContext>} Loaded context
+   * @throws {Error} If not in IDE workspace or workspace not configured
+   */
+  static async load(startDir = process.cwd()) {
+    // Find IDE marker (only .cursor/ or .vscode/)
+    const wsRoot = await findIDEMarker(startDir);
+    if (!wsRoot) {
+      throw new Error(
+        'Not in a Cursor/VSCode workspace.\n' +
+        'Open this folder in Cursor or VSCode first.'
+      );
+    }
+    
+    // Load workspace.json
+    const configPath = path.join(wsRoot, '.descix', 'workspace.json');
+    let configData;
+    try {
+      const content = await fs.readFile(configPath, 'utf-8');
+      configData = JSON.parse(content);
+    } catch (error) {
+      throw new Error(
+        'Workspace not configured.\n' +
+        'Run "npx descix init" first to configure your workspace.'
+      );
+    }
+    
+    return new PathContext(wsRoot, configData);
+  }
+
+  /**
+   * Try to load PathContext, return null if not configured
+   * Useful for commands that need to check if workspace exists
+   * 
+   * @param {string} startDir - Directory to start searching from
+   * @returns {Promise<PathContext|null>}
+   */
+  static async tryLoad(startDir = process.cwd()) {
+    try {
+      return await PathContext.load(startDir);
+    } catch {
+      return null;
+    }
+  }
+
+  // ============ Path Getters (all from workspace.json) ============
+
+  /**
+   * Get workspace root path
+   * @returns {string} Absolute path to workspace root
+   */
+  getWorkspaceRoot() {
+    return this.config.workspaceRoot || this.workspaceRoot;
+  }
+
+  /**
+   * Get community path
+   * @param {string} communityId - Community identifier
+   * @returns {string|null} Absolute path to community folder, or null if not found
+   */
+  getCommunityPath(communityId) {
+    const community = this.config.communities?.[communityId];
+    if (!community) return null;
+    
+    // Use absolutePath if available, otherwise compute from localPath
+    if (community.absolutePath) {
+      return community.absolutePath;
+    }
+    if (community.localPath) {
+      return path.join(this.getWorkspaceRoot(), community.localPath);
+    }
+    return null;
+  }
+
+  /**
+   * Get app path
+   * @param {string} communityId - Community identifier
+   * @param {string} appId - App identifier
+   * @returns {string|null} Absolute path to app folder, or null if not found
+   */
+  getAppPath(communityId, appId) {
+    const app = this.config.communities?.[communityId]?.apps?.[appId];
+    if (!app) return null;
+    
+    // Use absolutePath if available, otherwise compute from localPath
+    if (app.absolutePath) {
+      return app.absolutePath;
+    }
+    if (app.localPath) {
+      return path.join(this.getWorkspaceRoot(), app.localPath);
+    }
+    return null;
+  }
+
+  /**
+   * Get KB path
+   * @param {string} communityId - Community identifier
+   * @param {string} appId - App identifier
+   * @param {string} kbId - KB identifier (default: 'General')
+   * @returns {string|null} Absolute path to KB folder, or null if app not found
+   */
+  getKbPath(communityId, appId, kbId = 'General') {
+    const appPath = this.getAppPath(communityId, appId);
+    return appPath ? path.join(appPath, 'kb', kbId) : null;
+  }
+
+  /**
+   * Get site path
+   * @param {string} communityId - Community identifier
+   * @param {string} appId - App identifier
+   * @returns {string|null} Absolute path to site folder, or null if app not found
+   */
+  getSitePath(communityId, appId) {
+    const appPath = this.getAppPath(communityId, appId);
+    return appPath ? path.join(appPath, 'site') : null;
+  }
+
+  /**
+   * Get microservice path
+   * @param {string} communityId - Community identifier
+   * @param {string} appId - App identifier
+   * @returns {string|null} Absolute path to microservice folder, or null if app not found
+   */
+  getMicroservicePath(communityId, appId) {
+    const appPath = this.getAppPath(communityId, appId);
+    return appPath ? path.join(appPath, 'microservice') : null;
+  }
+
+  // ============ Context Detection (match cwd against known paths) ============
+
+  /**
+   * Detect community/app context from current working directory
+   * Matches cwd against known absolutePaths in workspace.json
+   * 
+   * @returns {{ communityId: string, appId: string, kbId: string }|null}
+   */
+  detectContext() {
+    const cwd = process.cwd();
+    
+    for (const [commId, comm] of Object.entries(this.config.communities || {})) {
+      for (const [appId, app] of Object.entries(comm.apps || {})) {
+        const appPath = app.absolutePath || 
+          (app.localPath ? path.join(this.getWorkspaceRoot(), app.localPath) : null);
+        
+        if (appPath && cwd.startsWith(appPath)) {
+          return {
+            communityId: commId,
+            appId,
+            kbId: app.kbId || 'General'
+          };
+        }
+      }
+    }
+    
+    return null;
+  }
+
+  /**
+   * Resolve context by merging CLI options with detected context
+   * CLI options take priority over detected context
+   * 
+   * @param {Object} options - CLI options { community, app, kb }
+   * @returns {{ communityId: string|null, appId: string|null, kbId: string }}
+   */
+  resolveContext(options = {}) {
+    const detected = this.detectContext();
+    
+    const resolved = {
+      communityId: options.community || detected?.communityId || null,
+      appId: options.app || detected?.appId || null,
+      kbId: options.kb || detected?.kbId || 'General'
+    };
+    
+    return resolved;
+  }
+
+  /**
+   * Resolve context and throw if community/app not determined
+   * 
+   * @param {Object} options - CLI options { community, app, kb }
+   * @returns {{ communityId: string, appId: string, kbId: string }}
+   * @throws {Error} If community or app cannot be determined
+   */
+  requireContext(options = {}) {
+    const ctx = this.resolveContext(options);
+    
+    if (!ctx.communityId || !ctx.appId) {
+      throw new Error(
+        'Could not determine app context.\n\n' +
+        'Options:\n' +
+        '  1. cd into an app directory\n' +
+        '  2. Use flags: -c <community> -a <app>\n\n' +
+        'Example: npx descix kb build -c descix -a appsdk'
+      );
+    }
+    
+    return ctx;
+  }
+
+  // ============ Config Access ============
+
+  /**
+   * Get raw workspace.json config
+   * @returns {Object}
+   */
+  getWorkspaceConfig() {
+    return this.config;
+  }
+
+  /**
+   * Get Drive configuration
+   * @returns {{ base_folder_id: string }|null}
+   */
+  getDriveConfig() {
+    return this.config.driveConfig || null;
+  }
+
+  /**
+   * Get API URL
+   * Derives from env.platform.microservice.port in DEV mode (v2.1 workspace format).
+   * @returns {string}
+   */
+  getApiUrl() {
+    if (this.config.apiUrl) return this.config.apiUrl;
+    const env = this.config.env || {};
+    const platformPort = env.platform?.microservice?.port;
+    if (platformPort && env.environment === 'DEV') return `https://localhost:${platformPort}`;
+    return 'https://descix.net';
+  }
+
+  /**
+   * Get environment
+   * @returns {string}
+   */
+  getEnvironment() {
+    return this.config.environment || 'production';
+  }
+
+  // ============ Listing ============
+
+  /**
+   * List all community IDs
+   * @returns {string[]}
+   */
+  listCommunities() {
+    return Object.keys(this.config.communities || {});
+  }
+
+  /**
+   * List all app IDs (optionally filtered by community)
+   * @param {string} communityId - Optional community filter
+   * @returns {Array<{ communityId: string, appId: string }>}
+   */
+  listApps(communityId = null) {
+    const apps = [];
+    
+    for (const [commId, comm] of Object.entries(this.config.communities || {})) {
+      if (communityId && commId !== communityId) continue;
+      
+      for (const appId of Object.keys(comm.apps || {})) {
+        apps.push({ communityId: commId, appId });
+      }
+    }
+    
+    return apps;
+  }
+}
+
+export default PathContext;

package/lib/core/Syncer.js

@@ -0,0 +1,310 @@
+/**
+ * Syncer - Canonical module for upstream sync to Pinecone
+ * 
+ * Responsibilities:
+ * - Sync local chunks to Pinecone via backend API
+ * - Compute delta between local and remote chunks using content_hash
+ * - Manage chunk lifecycle (upsert/delete)
+ * 
+ * Architecture:
+ * - Chunks are processed locally by Chunker (with content_hash)
+ * - Syncer compares hashes to determine what needs upserting
+ * - Only changed/new chunks are sent to backend
+ * - Backend handles Pinecone authentication and embedding
+ * - Pinecone performs integrated embeddings (server-side)
+ * 
+ * Security:
+ * - Pinecone API keys never leave the backend
+ * - CLI only sends text + metadata; backend handles embedding
+ * 
+ * Chunk ID Format (v2.1):
+ * - Composite: ${community_id}:${app_id}:${kb_id}:${file_id}:${chunk_idx}
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { loadChunks } from './Chunker.js';
+
+/**
+ * Compute delta between local chunks and remote Pinecone records
+ * Uses content_hash for efficient change detection when available.
+ * 
+ * @param {Array} localChunks - Chunk records from local JSON files (with content_hash)
+ * @param {Array<Object>} remoteChunks - Remote chunk metadata { id, content_hash } or just IDs
+ * @returns {Object} { toUpsert: Array, toDelete: Array<string>, unchanged: number }
+ */
+export function computeChunkDelta(localChunks, remoteChunks) {
+  // Handle both old format (array of IDs) and new format (array of {id, content_hash})
+  const isNewFormat = remoteChunks.length > 0 && typeof remoteChunks[0] === 'object';
+  
+  // Build maps for efficient lookup
+  const localById = new Map(localChunks.map(c => [c.id, c]));
+  const remoteById = isNewFormat 
+    ? new Map(remoteChunks.map(c => [c.id, c.content_hash]))
+    : new Map(remoteChunks.map(id => [id, null]));  // No hash available
+  
+  const toUpsert = [];
+  let unchanged = 0;
+  
+  for (const chunk of localChunks) {
+    const remoteHash = remoteById.get(chunk.id);
+    
+    if (remoteHash === undefined) {
+      // Chunk doesn't exist in remote - need to upsert
+      toUpsert.push(chunk);
+    } else if (remoteHash === null) {
+      // No hash comparison available (old backend) - always upsert
+      toUpsert.push(chunk);
+    } else if (chunk.content_hash !== remoteHash) {
+      // Hash differs - content changed, need to upsert
+      toUpsert.push(chunk);
+    } else {
+      // Hash matches - skip (already in sync)
+      unchanged++;
+    }
+  }
+  
+  // Find chunks to delete (exist in remote but not locally)
+  const localIds = new Set(localChunks.map(c => c.id));
+  const remoteIds = isNewFormat ? remoteChunks.map(c => c.id) : remoteChunks;
+  const toDelete = remoteIds.filter(id => !localIds.has(id));
+  
+  return { toUpsert, toDelete, unchanged };
+}
+
+/**
+ * Get existing chunk metadata from Pinecone (with content_hash for delta)
+ * Falls back to IDs-only if metadata endpoint not available.
+ * 
+ * @param {Object} apiClient - DeSciXApiClient instance
+ * @param {string} communityId - Community ID
+ * @param {string} appId - App ID
+ * @param {string} kbId - Knowledge base ID
+ * @returns {Promise<Array<{id: string, content_hash: string}>>} Chunk metadata
+ */
+export async function getRemoteChunkMetadata(apiClient, communityId, appId, kbId) {
+  try {
+    // Try new metadata endpoint first
+    const result = await apiClient.invoke('kb_get_chunk_metadata', {
+      app_id: appId,
+      kb_id: kbId
+    });
+    return result.chunks || [];
+  } catch (error) {
+    // Fall back to IDs-only endpoint
+    try {
+      const result = await apiClient.invoke('kb_get_chunk_ids', {
+        app_id: appId,
+        kb_id: kbId
+      });
+      // Return as array of IDs (old format) for backward compatibility
+      return result.chunk_ids || [];
+    } catch {
+      return [];
+    }
+  }
+}
+
+/**
+ * @deprecated Use getRemoteChunkMetadata instead
+ * Get existing chunk IDs from Pinecone
+ */
+export async function getRemoteChunkIds(apiClient, communityId, appId, kbId) {
+  const metadata = await getRemoteChunkMetadata(apiClient, communityId, appId, kbId);
+  // Handle both formats
+  if (metadata.length > 0 && typeof metadata[0] === 'object') {
+    return metadata.map(c => c.id);
+  }
+  return metadata;
+}
+
+/**
+ * Upsert chunks to Pinecone via backend API
+ * 
+ * @param {Object} apiClient - DeSciXApiClient instance
+ * @param {string} communityId - Community ID
+ * @param {string} appId - App ID
+ * @param {string} kbId - Knowledge base ID
+ * @param {Array} chunks - Chunk records to upsert
+ * @returns {Promise<{upserted: number}>}
+ */
+export async function upsertChunks(apiClient, communityId, appId, kbId, chunks) {
+  if (chunks.length === 0) {
+    return { upserted: 0 };
+  }
+
+  // Pinecone limits upserts to 96 vectors per batch
+  const BATCH_SIZE = 90;
+  let totalUpserted = 0;
+
+  for (let i = 0; i < chunks.length; i += BATCH_SIZE) {
+    const batch = chunks.slice(i, i + BATCH_SIZE);
+    const result = await apiClient.invoke('kb_sync_chunks', {
+      app_id: appId,
+      kb_id: kbId,
+      chunks: batch
+    });
+    totalUpserted += result.upserted_count || batch.length;
+  }
+
+  return { upserted: totalUpserted };
+}
+
+/**
+ * Delete stale chunks from Pinecone
+ * 
+ * @param {Object} apiClient - DeSciXApiClient instance
+ * @param {string} communityId - Community ID
+ * @param {string} appId - App ID
+ * @param {string} kbId - Knowledge base ID
+ * @param {Array<string>} chunkIds - Chunk IDs to delete
+ * @returns {Promise<{deleted: number}>}
+ */
+export async function deleteStaleChunks(apiClient, communityId, appId, kbId, chunkIds) {
+  if (chunkIds.length === 0) {
+    return { deleted: 0 };
+  }
+  
+  await apiClient.invoke('kb_delete_chunks', {
+    app_id: appId,
+    kb_id: kbId,
+    chunk_ids: chunkIds
+  });
+  
+  return { deleted: chunkIds.length };
+}
+
+/**
+ * Sync a KB to Pinecone - full workflow with content-hash based delta
+ * 
+ * Workflow:
+ * 1. Load local chunks from kb/chunks/ (with content_hash)
+ * 2. Get existing chunk metadata from Pinecone (with content_hash)
+ * 3. Compute delta using hash comparison (only upsert changed/new)
+ * 4. Delete stale chunks (exist in Pinecone but not locally)
+ * 5. Upsert new/modified chunks
+ * 
+ * @param {Object} apiClient - DeSciXApiClient instance (required)
+ * @param {Object} config - { workspaceRoot, communityId, appId, kbId, localPath }
+ * @param {Object} options - { verbose, onProgress }
+ * @returns {Promise<{synced: number, deleted: number, unchanged: number}>}
+ */
+export async function syncKb(apiClient, config, options = {}) {
+  const { workspaceRoot, communityId, appId, kbId = 'General', localPath } = config;
+  const { verbose = false, onProgress } = options;
+  
+  if (!apiClient) {
+    throw new Error('Authentication required. Run "descix login" first.');
+  }
+  
+  // 1. Load local chunks
+  const localChunks = await loadChunks({
+    workspaceRoot,
+    communityId,
+    appId,
+    kbId,
+    localPath
+  });
+  
+  if (localChunks.length === 0) {
+    if (verbose) console.log('No chunks to sync');
+    return { synced: 0, deleted: 0, unchanged: 0 };
+  }
+  
+  if (onProgress) onProgress(`Found ${localChunks.length} local chunks`);
+  
+  // 2. Get existing chunk metadata from Pinecone (with content_hash if available)
+  if (onProgress) onProgress('Fetching existing chunks from Pinecone...');
+  const remoteChunks = await getRemoteChunkMetadata(apiClient, communityId, appId, kbId);
+  
+  const remoteCount = Array.isArray(remoteChunks) 
+    ? remoteChunks.length 
+    : 0;
+  
+  if (verbose) {
+    console.log(`  Local chunks: ${localChunks.length}`);
+    console.log(`  Remote chunks: ${remoteCount}`);
+  }
+  
+  // 3. Compute delta using content_hash
+  const { toUpsert, toDelete, unchanged } = computeChunkDelta(localChunks, remoteChunks);
+  
+  if (verbose && unchanged > 0) {
+    console.log(`  Unchanged (skipped): ${unchanged}`);
+  }
+  
+  // 4. Delete stale chunks
+  let deleted = 0;
+  if (toDelete.length > 0) {
+    if (onProgress) onProgress(`Deleting ${toDelete.length} stale chunks...`);
+    try {
+      const deleteResult = await deleteStaleChunks(apiClient, communityId, appId, kbId, toDelete);
+      deleted = deleteResult.deleted;
+      if (verbose) console.log(`  Deleted ${deleted} stale chunks`);
+    } catch (error) {
+      if (verbose) console.log(`  Delete failed: ${error.message}`);
+    }
+  }
+  
+  // 5. Upsert only changed/new chunks
+  let synced = 0;
+  if (toUpsert.length > 0) {
+    if (onProgress) onProgress(`Upserting ${toUpsert.length} chunks...`);
+    const upsertResult = await upsertChunks(apiClient, communityId, appId, kbId, toUpsert);
+    synced = upsertResult.upserted;
+  } else {
+    if (onProgress) onProgress('All chunks already in sync');
+  }
+  
+  return {
+    synced,
+    deleted,
+    unchanged
+  };
+}
+
+/**
+ * Get sync status - compare local chunks with Pinecone using content_hash
+ * 
+ * @param {Object} apiClient - DeSciXApiClient instance
+ * @param {Object} config - { workspaceRoot, communityId, appId, kbId, localPath }
+ * @returns {Promise<{local: number, remote: number, toUpsert: number, toDelete: number, unchanged: number, inSync: boolean}>}
+ */
+export async function getSyncStatus(apiClient, config) {
+  const { workspaceRoot, communityId, appId, kbId = 'General', localPath } = config;
+  
+  // Load local chunks
+  const localChunks = await loadChunks({
+    workspaceRoot,
+    communityId,
+    appId,
+    kbId,
+    localPath
+  });
+  
+  // Get remote chunk metadata
+  const remoteChunks = await getRemoteChunkMetadata(apiClient, communityId, appId, kbId);
+  const remoteCount = Array.isArray(remoteChunks) ? remoteChunks.length : 0;
+  
+  // Compute delta using content_hash
+  const { toUpsert, toDelete, unchanged } = computeChunkDelta(localChunks, remoteChunks);
+  
+  return {
+    local: localChunks.length,
+    remote: remoteCount,
+    toUpsert: toUpsert.length,
+    toDelete: toDelete.length,
+    unchanged,
+    inSync: toUpsert.length === 0 && toDelete.length === 0
+  };
+}
+
+export default {
+  syncKb,
+  computeChunkDelta,
+  getRemoteChunkIds,
+  getRemoteChunkMetadata,
+  upsertChunks,
+  deleteStaleChunks,
+  getSyncStatus
+};

package/lib/core/index.js

@@ -0,0 +1,32 @@
+/**
+ * Core Library - Canonical modules for SDK operations
+ * 
+ * This barrel export provides the three core modules:
+ * 
+ * - Hydrator: Drive sync operations (pull/push/convert)
+ * - Chunker: Document chunking for RAG
+ * - Syncer: Upstream sync to Pinecone
+ * 
+ * Architecture:
+ * - All CLI commands delegate to these core modules
+ * - Ensures DRY - no duplicate code paths
+ * - Consistent error handling and progress reporting
+ * 
+ * Usage:
+ *   import { Hydrator, Chunker, Syncer } from '../core/index.js';
+ *   
+ *   // Or individual imports:
+ *   import { hydrateKb, pushStaging } from '../core/Hydrator.js';
+ *   import { processKb, loadChunks } from '../core/Chunker.js';
+ *   import { syncKb, getSyncStatus } from '../core/Syncer.js';
+ */
+
+// Re-export all named exports from each module
+export * from './Hydrator.js';
+export * from './Chunker.js';
+export * from './Syncer.js';
+
+// Re-export default exports as named modules
+export { default as Hydrator } from './Hydrator.js';
+export { default as Chunker } from './Chunker.js';
+export { default as Syncer } from './Syncer.js';