@tyvm/knowhow 0.0.108 → 0.0.109-dev.38b1faa

package/src/cloudWorker.ts

@@ -4,12 +4,18 @@ import { KnowhowSimpleClient, KNOWHOW_API_URL } from "./services/KnowhowClient";
 import { loadJwt } from "./login";
 import { getConfig, updateConfig, getLanguageConfig } from "./config";
 import { services } from "./services";
-import { Language, Config } from "./types";
-import { S3Service } from "./services/S3";
+import { Language, Config, McpConfig } from "./types";
+import { uploadFile, uploadDirectory } from "./fileSync";
+
+export interface CloudWorkerPullOptions {
+  id: string;
+  apiUrl?: string;
+}
 
 export interface CloudWorkerOptions {
   create?: boolean;
   push?: string; // uid of existing cloud worker
+  init?: boolean; // initialize config.files entries (mutates config)
   name?: string; // optional name for create
   apiUrl?: string;
   dryRun?: boolean;
@@ -25,25 +31,6 @@ interface FileToSync {
   isDirectory?: boolean; // true if this represents a whole directory
 }
 
-/**
- * Recursively list all files in a local directory, returning relative paths
- */
-function listFilesRecursively(dir: string): string[] {
-  const results: string[] = [];
-  if (!fs.existsSync(dir)) return results;
-  const entries = fs.readdirSync(dir, { withFileTypes: true });
-  for (const entry of entries) {
-    if (entry.isDirectory()) {
-      listFilesRecursively(path.join(dir, entry.name)).forEach((f) =>
-        results.push(entry.name + "/" + f)
-      );
-    } else {
-      results.push(entry.name);
-    }
-  }
-  return results;
-}
-
 /**
  * Build the worker config JSON from the local knowhow config
  */
@@ -66,21 +53,19 @@ function buildWorkerConfigJson(config: Config, files: { remotePath: string; loca
 }
 
 /**
- * Collect all files from the .knowhow directory that should be synced
- * Uses directory-level entries where possible so the worker config stays compact
- * and the folder upload/download feature handles individual files automatically.
+ * Collect all files from the .knowhow directory that should be synced.
+ * Only includes files/directories that currently exist locally.
+ * Used by --init to populate config.files.
  */
 async function collectFilesToSync(projectName: string): Promise<FileToSync[]> {
   const filesToSync: FileToSync[] = [];
 
-  // Helper to add file if it exists
   const addIfExists = (localPath: string, remotePath: string) => {
     if (fs.existsSync(localPath)) {
       filesToSync.push({ localPath, remotePath });
     }
   };
 
-  // Helper to add a directory entry if it exists (trailing slash = directory mode)
   const addDirIfExists = (localPath: string, remotePath: string) => {
     if (fs.existsSync(localPath)) {
       filesToSync.push({ localPath: localPath + "/", remotePath: remotePath + "/", isDirectory: true });
@@ -103,7 +88,9 @@ async function collectFilesToSync(projectName: string): Promise<FileToSync[]> {
 }
 
 /**
- * Collect files referenced in language.json sources
+ * Collect files referenced in language.json sources.
+ * These are always re-collected on both --init and --push so that new
+ * language term sources are picked up automatically.
  */
 async function collectLanguageReferencedFiles(
   language: Language,
@@ -119,17 +106,14 @@ async function collectLanguageReferencedFiles(
       if (source.kind !== "file" || !source.data) continue;
 
       for (const filePath of source.data) {
-        // Normalize the path (strip leading ./)
         const normalizedPath = filePath.replace(/^\.\//, "");
 
         // Skip the main knowhow config — it should not be synced to the language folder
-        // as it would overwrite the worker's own config
         if (normalizedPath === ".knowhow/knowhow.json") continue;
 
         if (fs.existsSync(normalizedPath)) {
           const basename = path.basename(normalizedPath);
           const remotePath = `${projectName}/.knowhow/language/${basename}`;
-          // localPath is the original path so the worker downloads it to the right place
           filesToSync.push({ localPath: normalizedPath, remotePath, downloadLocalPath: normalizedPath });
         }
       }
@@ -140,37 +124,83 @@ async function collectLanguageReferencedFiles(
 }
 
 /**
- * Upload a single file to the cloud worker's file storage
+ * Collect language-referenced files if language.json is present in the
+ * given config.files entries. Returns empty array if language.json is not
+ * configured for sync.
+ */
+async function collectLanguageFilesIfConfigured(
+  configFiles: { remotePath: string; localPath: string }[],
+  projectName: string
+): Promise<FileToSync[]> {
+  const syncingLanguage = configFiles.some(
+    (f) => !f.remotePath.endsWith("/") && f.remotePath.endsWith("language.json")
+  );
+  if (!syncingLanguage) return [];
+
+  const language = await getLanguageConfig();
+  return collectLanguageReferencedFiles(language, projectName);
+}
+
+/**
+ * Initialize the local config.files entries based on what exists in .knowhow/.
+ * This is the --init step — mutates config. Run once to set up sync entries.
+ * language-referenced files are also collected if language.json is present.
  */
-async function uploadSingleFile(
-  client: KnowhowSimpleClient,
-  s3Service: S3Service,
-  localPath: string,
-  remotePath: string,
-  dryRun: boolean
-): Promise<void> {
-  console.log(`  ⬆️  Uploading ${localPath} → ${remotePath}`);
-
-  if (dryRun) {
-    console.log(`     [DRY RUN] Would upload from ${localPath}`);
-    return;
+export async function initCloudWorker(options: { apiUrl?: string; dryRun?: boolean } = {}) {
+  const { dryRun = false } = options;
+
+  const config = await getConfig();
+  if (!config || Object.keys(config).length === 0) {
+    console.error("❌ No knowhow config found. Please run 'knowhow init' first.");
+    process.exit(1);
   }
 
-  if (!fs.existsSync(localPath)) {
-    console.warn(`     ⚠️  Local file not found, skipping: ${localPath}`);
-    return;
+  const projectName = path.basename(process.cwd());
+  console.log(`📁 Project name: ${projectName}`);
+
+  console.log("\n📂 Collecting files to sync...");
+  const mainFiles = await collectFilesToSync(projectName);
+  const languageFiles = await collectLanguageFilesIfConfigured(mainFiles, projectName);
+
+  if (languageFiles.length === 0 && !mainFiles.some((f) => f.remotePath.endsWith("language.json"))) {
+    console.log("   ℹ️  Skipping language-referenced files (language.json not found locally)");
   }
 
-  const presignedUrl = await client.getOrgFilePresignedUploadUrl(remotePath);
-  await s3Service.uploadToPresignedUrl(presignedUrl, localPath);
-  await client.markOrgFileUploadComplete(remotePath);
+  // Deduplicate by remotePath
+  const allFilesMap = new Map<string, FileToSync>();
+  for (const f of [...mainFiles, ...languageFiles]) {
+    allFilesMap.set(f.remotePath, f);
+  }
+  const allFiles = Array.from(allFilesMap.values());
+
+  console.log(`   Found ${allFiles.length} files to register`);
 
-  const stats = fs.statSync(localPath);
-  console.log(`     ✓ Uploaded ${stats.size} bytes`);
+  const configFilesEntries = allFiles.map((f) => ({
+    remotePath: f.remotePath,
+    localPath: f.downloadLocalPath ?? f.localPath,
+    direction: "download" as const,
+  }));
+
+  console.log("\n💾 Updating config.files with sync entries...");
+  if (!dryRun) {
+    const existingFiles = config.files || [];
+    const newRemotePaths = new Set(configFilesEntries.map((e) => e.remotePath));
+    const preserved = existingFiles.filter((e) => !newRemotePaths.has(e.remotePath));
+    config.files = [...preserved, ...configFilesEntries];
+    await updateConfig(config);
+    console.log(`   ✓ Updated config with ${config.files.length} file entries`);
+  } else {
+    console.log(`   [DRY RUN] Would update config with ${configFilesEntries.length} file entries`);
+    for (const f of allFiles) {
+      console.log(`   ${f.localPath} → ${f.remotePath}`);
+    }
+  }
 }
 
 /**
- * Main cloudWorker command handler
+ * Main cloudWorker command handler — push/create only.
+ * Reads config.files (set up by --init) and also re-collects any language-referenced
+ * files so new language term sources are always included without requiring --init again.
  */
 export async function cloudWorker(options: CloudWorkerOptions) {
   const {
@@ -200,10 +230,6 @@ export async function cloudWorker(options: CloudWorkerOptions) {
     process.exit(1);
   }
 
-  // Load language config
-  const language = await getLanguageConfig();
-
-  // Get project name from current directory
   const projectName = path.basename(process.cwd());
   console.log(`📁 Project name: ${projectName}`);
 
@@ -213,86 +239,63 @@ export async function cloudWorker(options: CloudWorkerOptions) {
   // Get S3 service
   const { AwsS3 } = services();
 
-  // Step 1: Collect all files to sync
-  console.log("\n📂 Collecting files to sync...");
-  const mainFiles = await collectFilesToSync(projectName);
-  const languageFiles = await collectLanguageReferencedFiles(language, projectName);
-
-  // Deduplicate by remotePath
-  const allFilesMap = new Map<string, FileToSync>();
-  for (const f of [...mainFiles, ...languageFiles]) {
-    allFilesMap.set(f.remotePath, f);
+  // Start with config.files (set up via --init)
+  const configFiles = config.files || [];
+  if (configFiles.length === 0) {
+    console.warn("⚠️  No files configured. Run 'knowhow cloudworker --init' first to set up file sync entries.");
   }
-  const allFiles = Array.from(allFilesMap.values());
 
-  console.log(`   Found ${allFiles.length} files to sync`);
-
-  if (dryRun) {
-    console.log("\n📋 Files that would be synced:");
-    for (const f of allFiles) {
-      console.log(`   ${f.localPath} → ${f.remotePath}`);
-    }
+  // Re-collect language-referenced files on every push (if language.json is in config.files)
+  // so that new language term sources are picked up without needing --init again.
+  const languageFiles = await collectLanguageFilesIfConfigured(configFiles, projectName);
+  if (languageFiles.length > 0) {
+    console.log(`   + ${languageFiles.length} language-referenced file(s) to sync`);
   }
 
-  // Step 2: Build the config.files array for all synced files
-  const configFilesEntries = allFiles.map((f) => ({
-    remotePath: f.remotePath,
-    localPath: f.downloadLocalPath ?? f.localPath,
-    direction: "download" as const,
-  }));
-
-  // Step 3: Update config.files and save
-  console.log("\n💾 Updating config.files with sync entries...");
-  if (!dryRun) {
-    // Preserve any existing files entries not in our set
-    const existingFiles = config.files || [];
-    const newRemotePaths = new Set(configFilesEntries.map((e) => e.remotePath));
-
-    // Keep entries that don't overlap with new ones
-    const preserved = existingFiles.filter(
-      (e) => !newRemotePaths.has(e.remotePath)
-    );
+  // Merge language files into the upload list (deduplicate by remotePath)
+  const allFilesMap = new Map<string, { remotePath: string; localPath: string }>();
+  for (const f of configFiles) {
+    allFilesMap.set(f.remotePath, f);
+  }
+  for (const f of languageFiles) {
+    const entry = { remotePath: f.remotePath, localPath: f.downloadLocalPath ?? f.localPath };
+    allFilesMap.set(f.remotePath, entry);
+  }
+  const allFiles = Array.from(allFilesMap.values());
 
-    config.files = [...preserved, ...configFilesEntries];
+  // If new language files were found, update config.files so they persist
+  if (languageFiles.length > 0 && !dryRun) {
+    config.files = allFiles.map((f) => ({ ...f, direction: "download" as const }));
     await updateConfig(config);
-    console.log(`   ✓ Updated config with ${config.files.length} file entries`);
-  } else {
-    console.log(`   [DRY RUN] Would update config with ${configFilesEntries.length} file entries`);
   }
 
-  // Step 4: Build workerConfigJson
-  const workerConfigJson = buildWorkerConfigJson(config, configFilesEntries);
+  // Build the workerConfigJson using the full file list
+  const workerConfigJson = buildWorkerConfigJson(config, allFiles.map((f) => ({ ...f, direction: "download" })));
 
-  // Step 5: Upload all files
-  console.log(`\n🚀 Uploading ${allFiles.length} files...`);
+  // Upload all files
+  console.log(`\n🚀 Uploading ${allFiles.length} configured files...`);
   let successCount = 0;
   let failCount = 0;
 
-  for (const file of allFiles) {
+  for (const mount of allFiles) {
+    const { remotePath, localPath } = mount;
     try {
-      if (file.isDirectory) {
-        // Upload all files recursively in the local directory
-        const localDir = file.localPath.endsWith("/") ? file.localPath : file.localPath + "/";
-        const remoteDir = file.remotePath.endsWith("/") ? file.remotePath : file.remotePath + "/";
-        const relFiles = listFilesRecursively(localDir);
-        console.log(`  📁 Uploading directory ${localDir} → ${remoteDir} (${relFiles.length} files)`);
-        for (const relFile of relFiles) {
-          await uploadSingleFile(client, AwsS3, localDir + relFile, remoteDir + relFile, dryRun);
-          successCount++;
-        }
+      if (remotePath.endsWith("/") || localPath.endsWith("/")) {
+        const count = await uploadDirectory(client, AwsS3, remotePath, localPath, dryRun);
+        successCount += count;
       } else {
-        await uploadSingleFile(client, AwsS3, file.localPath, file.remotePath, dryRun);
+        await uploadFile(client, AwsS3, remotePath, localPath, dryRun);
         successCount++;
       }
     } catch (error) {
-      console.error(`  ❌ Failed to upload ${file.localPath}: ${error.message}`);
+      console.error(`  ❌ Failed to upload ${localPath}: ${error.message}`);
       failCount++;
     }
   }
 
   console.log(`\n   ✓ Upload complete: ${successCount} succeeded, ${failCount} failed`);
 
-  // Step 6: Create or update cloud worker
+  // Create or update cloud worker
   if (create) {
     const workerName = name || `${projectName}-worker`;
     console.log(`\n🌩️  Creating cloud worker "${workerName}"...`);
@@ -330,3 +333,62 @@ export async function cloudWorker(options: CloudWorkerOptions) {
     console.log(`\n✅ Cloud worker sync complete!`);
   }
 }
+
+/**
+ * Pull the latest workerConfigJson from the cloud worker API and update the
+ * local knowhow.json config to match.
+ */
+export async function pullCloudWorkerConfig(options: CloudWorkerPullOptions) {
+  const { id, apiUrl = KNOWHOW_API_URL } = options;
+
+  // Load JWT
+  const jwt = await loadJwt();
+  if (!jwt) {
+    console.error("❌ No JWT token found. Please run 'knowhow login' first.");
+    process.exit(1);
+  }
+
+  const client = new KnowhowSimpleClient(apiUrl, jwt);
+
+  console.log(`🔄 Pulling config for cloud worker ${id}...`);
+
+  const resp = await client.getCloudWorker(id);
+  const remoteWorker = resp.data;
+
+  if (!remoteWorker) {
+    console.error(`❌ Cloud worker ${id} not found.`);
+    process.exit(1);
+  }
+
+  const remoteConfig = (remoteWorker.workerConfigJson ?? {}) as {
+    mcps?: McpConfig[];
+    modules?: string[];
+    plugins?: Config["plugins"];
+    agents?: Config["agents"];
+  };
+
+  // Load current local config
+  const localConfig = await getConfig();
+
+  // Merge remote fields into local config
+  if (remoteConfig.mcps !== undefined) {
+    localConfig.mcps = remoteConfig.mcps;
+  }
+  if (remoteConfig.modules !== undefined) {
+    localConfig.modules = remoteConfig.modules;
+  }
+  if (remoteConfig.plugins !== undefined) {
+    localConfig.plugins = remoteConfig.plugins;
+  }
+  if (remoteConfig.agents !== undefined) {
+    localConfig.agents = remoteConfig.agents;
+  }
+
+  await updateConfig(localConfig);
+
+  const mcpCount = remoteConfig.mcps?.length ?? 0;
+  console.log(`✅ Config pulled! ${mcpCount} MCP(s) now configured locally.`);
+  console.log(`   Run 'knowhow worker' or trigger reloadConfig to apply changes.`);
+
+  return { mcps: remoteConfig.mcps, modules: remoteConfig.modules };
+}

package/src/commands/agent.ts

@@ -0,0 +1,246 @@
+import { Command } from "commander";
+import { readPromptFile } from "../ai";
+import { AgentModule } from "../chat/modules/AgentModule";
+import { AskModule } from "../chat/modules/AskModule";
+import { SearchModule } from "../chat/modules/SearchModule";
+import { SessionsModule } from "../chat/modules/SessionsModule";
+import { SetupModule } from "../chat/modules/SetupModule";
+
+async function readStdin(): Promise<string> {
+  return new Promise((resolve) => {
+    let data = "";
+    process.stdin.setEncoding("utf8");
+
+    if (process.stdin.isTTY) {
+      resolve("");
+      return;
+    }
+
+    process.stdin.on("readable", () => {
+      const chunk = process.stdin.read();
+      if (chunk !== null) data += chunk;
+    });
+
+    process.stdin.on("end", () => resolve(data.trim()));
+  });
+}
+
+export function addAgentCommand(program: Command, getChatService: () => any): void {
+  program
+    .command("agent")
+    .description("Spin up agents directly from CLI")
+    .option(
+      "--provider <provider>",
+      "AI provider (openai, anthropic, google, xai)"
+    )
+    .option("--model <model>", "Specific model for the provider")
+    .option("--agent-name <name>", "Which agent to use", "Patcher")
+    .option(
+      "--max-time-limit <minutes>",
+      "Time limit for agent execution (minutes)",
+      "30"
+    )
+    .option(
+      "--max-spend-limit <dollars>",
+      "Cost limit for agent execution (dollars)",
+      "10"
+    )
+    .option("--message-id <messageId>", "Knowhow message ID for task tracking")
+    .option("--sync-fs", "Enable filesystem-based synchronization")
+    .option(
+      "--task-id <taskId>",
+      "Pre-generated task ID (used with --sync-fs for predictable agent directory path)"
+    )
+    .option("--prompt-file <path>", "Custom prompt template file with {text}")
+    .option("--input <text>", "Task input (fallback to stdin if not provided)")
+    .option(
+      "--resume",
+      "Resume a previously started task using the --task-id (local FS or remote)"
+    )
+    .action(async (options) => {
+      try {
+        const { setupServices } = await import("./services");
+        await setupServices();
+        const chatService = getChatService();
+        const agentModule = new AgentModule();
+
+        if (options.resume) {
+          const threads = await agentModule.loadThreadsForTask(
+            options.taskId,
+            options.messageId
+          );
+          const resumeInput =
+            options.input || "Please continue from where you left off.";
+
+          await agentModule.initialize(chatService);
+          const { taskCompleted: resumed } =
+            await agentModule.resumeFromMessages({
+              agentName: options.agentName || "Patcher",
+              input: resumeInput,
+              threads,
+              messageId: options.messageId,
+              taskId: options.taskId,
+            });
+          await resumed;
+          return;
+        }
+
+        let input = options.input;
+
+        if (!input && !options.promptFile) {
+          input = await readStdin();
+        }
+
+        input = readPromptFile(options.promptFile, input);
+
+        if (!input) {
+          console.error(
+            "Error: No input provided. Use --input flag, pipe input via stdin, or provide --prompt-file."
+          );
+          process.exit(1);
+        }
+
+        await agentModule.initialize(chatService);
+        const { taskCompleted } = await agentModule.setupAgent({
+          ...options,
+          input,
+          maxTimeLimit: parseInt(options.maxTimeLimit, 10),
+          maxSpendLimit: parseFloat(options.maxSpendLimit),
+          run: true,
+        });
+        await taskCompleted;
+      } catch (error) {
+        console.error("Error running agent:", error);
+        process.exit(1);
+      }
+    });
+}
+
+export function addAskCommand(program: Command, getChatService: () => any, getConfig: () => any): void {
+  program
+    .command("ask")
+    .description("Direct AI questioning without agent overhead")
+    .option("--provider <provider>", "AI provider to use")
+    .option("--model <model>", "Specific model")
+    .option("--input <text>", "Question (fallback to stdin if not provided)")
+    .option("--prompt-file <path>", "Custom prompt template file")
+    .action(async (options) => {
+      try {
+        const { setupServices } = await import("./services");
+        await setupServices();
+        const chatService = getChatService();
+        const config = getConfig();
+        let input = options.input;
+
+        if (!input && !options.promptFile) {
+          input = await readStdin();
+        }
+
+        input = readPromptFile(options.promptFile, input);
+
+        if (!input) {
+          console.error(
+            "Error: No question provided. Use --input flag, pipe input via stdin, or provide --prompt-file."
+          );
+          process.exit(1);
+        }
+
+        const askModule = new AskModule();
+        await askModule.initialize(chatService);
+        await askModule.processAIQuery(input, {
+          plugins: config.plugins.enabled,
+          currentModel: options.model,
+          currentProvider: options.provider,
+          chatHistory: [],
+        });
+      } catch (error) {
+        console.error("Error asking AI:", error);
+        process.exit(1);
+      }
+    });
+}
+
+export function addSetupCommand(program: Command, getChatService: () => any): void {
+  program
+    .command("setup")
+    .description("Ask the agent to configure knowhow")
+    .action(async () => {
+      try {
+        const { setupServices } = await import("./services");
+        await setupServices();
+        const chatService = getChatService();
+        const agentModule = new AgentModule();
+        await agentModule.initialize(chatService);
+        const setupModule = new SetupModule(agentModule);
+        await setupModule.initialize(chatService);
+        await setupModule.handleSetupCommand([]);
+      } catch (error) {
+        console.error("Error running agent:", error);
+        process.exit(1);
+      }
+    });
+}
+
+export function addSearchCommand(program: Command): void {
+  program
+    .command("search")
+    .description("Search embeddings directly from CLI")
+    .option(
+      "--input <text>",
+      "Search query (fallback to stdin if not provided)"
+    )
+    .option(
+      "-e, --embedding <path>",
+      "Specific embedding path (default: all)",
+      "all"
+    )
+    .action(async (options) => {
+      try {
+        const { setupServices } = await import("./services");
+        await setupServices();
+        let input = options.input;
+        if (!input) {
+          input = await readStdin();
+          if (!input) {
+            console.error(
+              "Error: No search query provided. Use --input flag or pipe input via stdin."
+            );
+            process.exit(1);
+          }
+        }
+
+        await new SearchModule().searchEmbeddingsCLI(input, options.embedding);
+      } catch (error) {
+        console.error("Error searching embeddings:", error);
+        process.exit(1);
+      }
+    });
+}
+
+export function addSessionsCommand(program: Command, getChatService: () => any): void {
+  program
+    .command("sessions")
+    .description("Manage agent sessions from CLI")
+    .option(
+      "--all",
+      "Show all historical sessions (default: current process only)"
+    )
+    .option("--csv", "Output sessions as CSV")
+    .action(async (options) => {
+      try {
+        const chatService = getChatService();
+        const agentModule = new AgentModule();
+        await agentModule.initialize(chatService);
+        const sessionsModule = new SessionsModule(agentModule);
+        await sessionsModule.initialize(chatService);
+        await sessionsModule.logSessionTable(
+          options.all || false,
+          options.csv || false,
+          true
+        );
+      } catch (error) {
+        console.error("Error listing sessions:", error);
+        process.exit(1);
+      }
+    });
+}