@tyvm/knowhow 0.0.108 → 0.0.109-dev.05fe5a0

package/src/clients/anthropic.ts

@@ -40,11 +40,11 @@ export class GenericAnthropicClient implements GenericClient {
     });
   }
 
-  handleToolCaching(tools: Anthropic.Tool[]) {
+  handleToolCaching(tools: Anthropic.Tool[], longTtl = false) {
     const lastTool = tools[tools.length - 1];
 
     if (lastTool) {
-      lastTool.cache_control = { type: "ephemeral" };
+      lastTool.cache_control = longTtl ? { type: "ephemeral", ttl: "1h" } as any : { type: "ephemeral" };
     }
   }
 
@@ -94,7 +94,7 @@ export class GenericAnthropicClient implements GenericClient {
     return cleaned;
   }
 
-  transformTools(tools?: Tool[]): Anthropic.Tool[] {
+  transformTools(tools?: Tool[], longTtl = false): Anthropic.Tool[] {
     if (!tools) {
       return [];
     }
@@ -104,7 +104,7 @@ export class GenericAnthropicClient implements GenericClient {
       input_schema: this.cleanSchemaForAnthropic(tool.function.parameters) as any,
     }));
 
-    this.handleToolCaching(transformed);
+    this.handleToolCaching(transformed, longTtl);
 
     return transformed;
   }
@@ -153,16 +153,16 @@ export class GenericAnthropicClient implements GenericClient {
     return messages;
   }
 
-  cacheLastContent(message: MessageParam) {
+  cacheLastContent(message: MessageParam, longTtl = false) {
     if (Array.isArray(message.content)) {
       const lastMessage = message.content[message.content.length - 1];
       if (
         lastMessage.type !== "thinking" &&
         lastMessage.type !== "redacted_thinking"
       ) {
-        lastMessage.cache_control = {
-          type: "ephemeral",
-        };
+        lastMessage.cache_control = longTtl
+          ? ({ type: "ephemeral", ttl: "1h" } as any)
+          : { type: "ephemeral" };
       }
     }
   }
@@ -179,7 +179,7 @@ export class GenericAnthropicClient implements GenericClient {
     }
   }
 
-  handleMessageCaching(groupedMessages: MessageParam[]) {
+  handleMessageCaching(groupedMessages: MessageParam[], longTtl = false) {
     this.handleClearingCache(groupedMessages);
 
     // find the last two messages and mark them as ephemeral
@@ -189,7 +189,7 @@ export class GenericAnthropicClient implements GenericClient {
 
     for (const m of lastTwoUserMessages) {
       if (Array.isArray(m.content)) {
-        this.cacheLastContent(m);
+        this.cacheLastContent(m, longTtl);
       }
     }
   }
@@ -203,7 +203,7 @@ export class GenericAnthropicClient implements GenericClient {
     }
   }
 
-  transformMessages(messages: Message[]): MessageParam[] {
+  transformMessages(messages: Message[], longTtl = false): MessageParam[] {
     const toolCalls = messages.flatMap((msg) => msg.tool_calls || []);
     const claudeMessages: MessageParam[] = messages
       .filter((msg) => msg.role !== "system")
@@ -302,7 +302,7 @@ export class GenericAnthropicClient implements GenericClient {
 
     const groupedMessages = this.combineMessages(claudeMessages);
 
-    this.handleMessageCaching(groupedMessages);
+    this.handleMessageCaching(groupedMessages, longTtl);
 
     return groupedMessages;
   }
@@ -349,14 +349,15 @@ export class GenericAnthropicClient implements GenericClient {
   async createChatCompletion(
     options: CompletionOptions
   ): Promise<CompletionResponse> {
+    const longTtl = !!options.long_ttl_cache;
     const systemMessage = options.messages
       .filter((msg) => msg.role === "system")
       .map((msg) => msg.content || "")
       .join("\n");
 
-    const claudeMessages = this.transformMessages(options.messages);
+    const claudeMessages = this.transformMessages(options.messages, longTtl);
 
-    const tools = this.transformTools(options.tools);
+    const tools = this.transformTools(options.tools, longTtl);
     try {
       const response = await this.client.messages.create({
         model: options.model,
@@ -365,7 +366,7 @@ export class GenericAnthropicClient implements GenericClient {
           ? [
               {
                 text: systemMessage,
-                cache_control: { type: "ephemeral" },
+                cache_control: longTtl ? ({ type: "ephemeral", ttl: "1h" } as any) : { type: "ephemeral" },
                 type: "text",
               },
             ]
@@ -424,7 +425,7 @@ export class GenericAnthropicClient implements GenericClient {
         usd_cost: this.calculateCost(options.model, response.usage),
       };
     } catch (err) {
-      if ("headers" in err && err.headers["x-should-retry"] === "true") {
+      if ("headers" in err && err.headers?.["x-should-retry"] === "true") {
         console.warn("Retrying failed request", err);
         await wait(2500);
         return this.createChatCompletion(options);

package/src/clients/index.ts

@@ -577,11 +577,12 @@ export class AIClient {
    * @param modelQuery - the model name to search for (can be partial/normalized)
    * @param provider   - optional provider to restrict search to
    */
-  findModelFuzzy(modelQuery: string, provider?: string): { provider: string; model: string } | undefined {
+  findModelFuzzy(
+    modelQuery: string,
+    provider?: string
+  ): { provider: string; model: string } | undefined {
     const queryNorm = AIClient.normalizeModelId(modelQuery);
-    const providers = provider
-      ? [provider]
-      : Object.keys(this.clientModels);
+    const providers = provider ? [provider] : Object.keys(this.clientModels);
 
     for (const p of providers) {
       const models = (this.clientModels[p] as string[]) ?? [];
@@ -835,7 +836,7 @@ export class AIClient {
         const splitModel = m.id.split("/");
 
         if (splitModel.length < 2) {
-          console.error(`Cannot parse model format: ${m.id}`);
+          console.warn(`Cannot parse model format: ${m.id}`);
         }
 
         const provider = splitModel.length > 1 ? splitModel[0] : "";

package/src/clients/types.ts

@@ -1,4 +1,10 @@
-export type ModelModality = "completion" | "embedding" | "image" | "audio" | "video" | "transcription";
+export type ModelModality =
+  | "completion"
+  | "embedding"
+  | "image"
+  | "audio"
+  | "video"
+  | "transcription";
 
 export type MessageContent =
   | { type: "text"; text: string }
@@ -8,7 +14,7 @@ export type MessageContent =
 
 export interface Message {
   role: "system" | "user" | "assistant" | "tool";
-  content?: string | MessageContent[];
+  content?: string | MessageContent[] | null;
 
   name?: string;
   tool_call_id?: string;
@@ -16,7 +22,7 @@ export interface Message {
 }
 
 export interface OutputMessage extends Message {
-  content: string;
+  content?: string | null;
 }
 
 export interface ToolProp {
@@ -61,6 +67,13 @@ export interface CompletionOptions {
    *  Maps to: OpenAI reasoning_effort, xAI reasoning.effort, Gemini thinkingLevel/thinkingBudget, Anthropic thinking budget.
    *  "low" = minimal thinking, "medium" = balanced, "high" = maximum reasoning */
   reasoning_effort?: "low" | "medium" | "high";
+  /**
+   * When true, hints to the client that this task is long-running and it should
+   * use a long-TTL cache where available.
+   * - Anthropic: enables the `extended-cache-ttl-2025-02-19` beta and sets
+   *   `cache_control.ttl` to 3600 (1 hour) instead of the default 5-minute ephemeral cache.
+   */
+  long_ttl_cache?: boolean;
 }
 
 /**
@@ -294,7 +307,9 @@ export interface GenericClient {
    * When modality is provided, return only models for that modality (static list).
    * When omitted, return ALL models (backward compat — may do a live API call).
    */
-  getModels(modality?: ModelModality): Promise<{ id: string; modality?: ModelModality[] }[]>;
+  getModels(
+    modality?: ModelModality
+  ): Promise<{ id: string; modality?: ModelModality[] }[]>;
   /**
    * Returns the context window limit and compression threshold for a given model,
    * or undefined if the model is not known to this client.

package/src/cloudWorker.ts

@@ -4,12 +4,18 @@ import { KnowhowSimpleClient, KNOWHOW_API_URL } from "./services/KnowhowClient";
 import { loadJwt } from "./login";
 import { getConfig, updateConfig, getLanguageConfig } from "./config";
 import { services } from "./services";
-import { Language, Config } from "./types";
-import { S3Service } from "./services/S3";
+import { Language, Config, McpConfig } from "./types";
+import { uploadFile, uploadDirectory } from "./fileSync";
+
+export interface CloudWorkerPullOptions {
+  id: string;
+  apiUrl?: string;
+}
 
 export interface CloudWorkerOptions {
   create?: boolean;
   push?: string; // uid of existing cloud worker
+  init?: boolean; // initialize config.files entries (mutates config)
   name?: string; // optional name for create
   apiUrl?: string;
   dryRun?: boolean;
@@ -25,25 +31,6 @@ interface FileToSync {
   isDirectory?: boolean; // true if this represents a whole directory
 }
 
-/**
- * Recursively list all files in a local directory, returning relative paths
- */
-function listFilesRecursively(dir: string): string[] {
-  const results: string[] = [];
-  if (!fs.existsSync(dir)) return results;
-  const entries = fs.readdirSync(dir, { withFileTypes: true });
-  for (const entry of entries) {
-    if (entry.isDirectory()) {
-      listFilesRecursively(path.join(dir, entry.name)).forEach((f) =>
-        results.push(entry.name + "/" + f)
-      );
-    } else {
-      results.push(entry.name);
-    }
-  }
-  return results;
-}
-
 /**
  * Build the worker config JSON from the local knowhow config
  */
@@ -66,21 +53,19 @@ function buildWorkerConfigJson(config: Config, files: { remotePath: string; loca
 }
 
 /**
- * Collect all files from the .knowhow directory that should be synced
- * Uses directory-level entries where possible so the worker config stays compact
- * and the folder upload/download feature handles individual files automatically.
+ * Collect all files from the .knowhow directory that should be synced.
+ * Only includes files/directories that currently exist locally.
+ * Used by --init to populate config.files.
  */
 async function collectFilesToSync(projectName: string): Promise<FileToSync[]> {
   const filesToSync: FileToSync[] = [];
 
-  // Helper to add file if it exists
   const addIfExists = (localPath: string, remotePath: string) => {
     if (fs.existsSync(localPath)) {
       filesToSync.push({ localPath, remotePath });
     }
   };
 
-  // Helper to add a directory entry if it exists (trailing slash = directory mode)
   const addDirIfExists = (localPath: string, remotePath: string) => {
     if (fs.existsSync(localPath)) {
       filesToSync.push({ localPath: localPath + "/", remotePath: remotePath + "/", isDirectory: true });
@@ -103,7 +88,9 @@ async function collectFilesToSync(projectName: string): Promise<FileToSync[]> {
 }
 
 /**
- * Collect files referenced in language.json sources
+ * Collect files referenced in language.json sources.
+ * These are always re-collected on both --init and --push so that new
+ * language term sources are picked up automatically.
  */
 async function collectLanguageReferencedFiles(
   language: Language,
@@ -119,17 +106,14 @@ async function collectLanguageReferencedFiles(
       if (source.kind !== "file" || !source.data) continue;
 
       for (const filePath of source.data) {
-        // Normalize the path (strip leading ./)
         const normalizedPath = filePath.replace(/^\.\//, "");
 
         // Skip the main knowhow config — it should not be synced to the language folder
-        // as it would overwrite the worker's own config
         if (normalizedPath === ".knowhow/knowhow.json") continue;
 
         if (fs.existsSync(normalizedPath)) {
           const basename = path.basename(normalizedPath);
           const remotePath = `${projectName}/.knowhow/language/${basename}`;
-          // localPath is the original path so the worker downloads it to the right place
           filesToSync.push({ localPath: normalizedPath, remotePath, downloadLocalPath: normalizedPath });
         }
       }
@@ -140,37 +124,83 @@ async function collectLanguageReferencedFiles(
 }
 
 /**
- * Upload a single file to the cloud worker's file storage
+ * Collect language-referenced files if language.json is present in the
+ * given config.files entries. Returns empty array if language.json is not
+ * configured for sync.
+ */
+async function collectLanguageFilesIfConfigured(
+  configFiles: { remotePath: string; localPath: string }[],
+  projectName: string
+): Promise<FileToSync[]> {
+  const syncingLanguage = configFiles.some(
+    (f) => !f.remotePath.endsWith("/") && f.remotePath.endsWith("language.json")
+  );
+  if (!syncingLanguage) return [];
+
+  const language = await getLanguageConfig();
+  return collectLanguageReferencedFiles(language, projectName);
+}
+
+/**
+ * Initialize the local config.files entries based on what exists in .knowhow/.
+ * This is the --init step — mutates config. Run once to set up sync entries.
+ * language-referenced files are also collected if language.json is present.
  */
-async function uploadSingleFile(
-  client: KnowhowSimpleClient,
-  s3Service: S3Service,
-  localPath: string,
-  remotePath: string,
-  dryRun: boolean
-): Promise<void> {
-  console.log(`  ⬆️  Uploading ${localPath} → ${remotePath}`);
-
-  if (dryRun) {
-    console.log(`     [DRY RUN] Would upload from ${localPath}`);
-    return;
+export async function initCloudWorker(options: { apiUrl?: string; dryRun?: boolean } = {}) {
+  const { dryRun = false } = options;
+
+  const config = await getConfig();
+  if (!config || Object.keys(config).length === 0) {
+    console.error("❌ No knowhow config found. Please run 'knowhow init' first.");
+    process.exit(1);
   }
 
-  if (!fs.existsSync(localPath)) {
-    console.warn(`     ⚠️  Local file not found, skipping: ${localPath}`);
-    return;
+  const projectName = path.basename(process.cwd());
+  console.log(`📁 Project name: ${projectName}`);
+
+  console.log("\n📂 Collecting files to sync...");
+  const mainFiles = await collectFilesToSync(projectName);
+  const languageFiles = await collectLanguageFilesIfConfigured(mainFiles, projectName);
+
+  if (languageFiles.length === 0 && !mainFiles.some((f) => f.remotePath.endsWith("language.json"))) {
+    console.log("   ℹ️  Skipping language-referenced files (language.json not found locally)");
   }
 
-  const presignedUrl = await client.getOrgFilePresignedUploadUrl(remotePath);
-  await s3Service.uploadToPresignedUrl(presignedUrl, localPath);
-  await client.markOrgFileUploadComplete(remotePath);
+  // Deduplicate by remotePath
+  const allFilesMap = new Map<string, FileToSync>();
+  for (const f of [...mainFiles, ...languageFiles]) {
+    allFilesMap.set(f.remotePath, f);
+  }
+  const allFiles = Array.from(allFilesMap.values());
+
+  console.log(`   Found ${allFiles.length} files to register`);
 
-  const stats = fs.statSync(localPath);
-  console.log(`     ✓ Uploaded ${stats.size} bytes`);
+  const configFilesEntries = allFiles.map((f) => ({
+    remotePath: f.remotePath,
+    localPath: f.downloadLocalPath ?? f.localPath,
+    direction: "download" as const,
+  }));
+
+  console.log("\n💾 Updating config.files with sync entries...");
+  if (!dryRun) {
+    const existingFiles = config.files || [];
+    const newRemotePaths = new Set(configFilesEntries.map((e) => e.remotePath));
+    const preserved = existingFiles.filter((e) => !newRemotePaths.has(e.remotePath));
+    config.files = [...preserved, ...configFilesEntries];
+    await updateConfig(config);
+    console.log(`   ✓ Updated config with ${config.files.length} file entries`);
+  } else {
+    console.log(`   [DRY RUN] Would update config with ${configFilesEntries.length} file entries`);
+    for (const f of allFiles) {
+      console.log(`   ${f.localPath} → ${f.remotePath}`);
+    }
+  }
 }
 
 /**
- * Main cloudWorker command handler
+ * Main cloudWorker command handler — push/create only.
+ * Reads config.files (set up by --init) and also re-collects any language-referenced
+ * files so new language term sources are always included without requiring --init again.
  */
 export async function cloudWorker(options: CloudWorkerOptions) {
   const {
@@ -200,10 +230,6 @@ export async function cloudWorker(options: CloudWorkerOptions) {
     process.exit(1);
   }
 
-  // Load language config
-  const language = await getLanguageConfig();
-
-  // Get project name from current directory
   const projectName = path.basename(process.cwd());
   console.log(`📁 Project name: ${projectName}`);
 
@@ -213,86 +239,63 @@ export async function cloudWorker(options: CloudWorkerOptions) {
   // Get S3 service
   const { AwsS3 } = services();
 
-  // Step 1: Collect all files to sync
-  console.log("\n📂 Collecting files to sync...");
-  const mainFiles = await collectFilesToSync(projectName);
-  const languageFiles = await collectLanguageReferencedFiles(language, projectName);
-
-  // Deduplicate by remotePath
-  const allFilesMap = new Map<string, FileToSync>();
-  for (const f of [...mainFiles, ...languageFiles]) {
-    allFilesMap.set(f.remotePath, f);
+  // Start with config.files (set up via --init)
+  const configFiles = config.files || [];
+  if (configFiles.length === 0) {
+    console.warn("⚠️  No files configured. Run 'knowhow cloudworker --init' first to set up file sync entries.");
   }
-  const allFiles = Array.from(allFilesMap.values());
 
-  console.log(`   Found ${allFiles.length} files to sync`);
-
-  if (dryRun) {
-    console.log("\n📋 Files that would be synced:");
-    for (const f of allFiles) {
-      console.log(`   ${f.localPath} → ${f.remotePath}`);
-    }
+  // Re-collect language-referenced files on every push (if language.json is in config.files)
+  // so that new language term sources are picked up without needing --init again.
+  const languageFiles = await collectLanguageFilesIfConfigured(configFiles, projectName);
+  if (languageFiles.length > 0) {
+    console.log(`   + ${languageFiles.length} language-referenced file(s) to sync`);
   }
 
-  // Step 2: Build the config.files array for all synced files
-  const configFilesEntries = allFiles.map((f) => ({
-    remotePath: f.remotePath,
-    localPath: f.downloadLocalPath ?? f.localPath,
-    direction: "download" as const,
-  }));
-
-  // Step 3: Update config.files and save
-  console.log("\n💾 Updating config.files with sync entries...");
-  if (!dryRun) {
-    // Preserve any existing files entries not in our set
-    const existingFiles = config.files || [];
-    const newRemotePaths = new Set(configFilesEntries.map((e) => e.remotePath));
-
-    // Keep entries that don't overlap with new ones
-    const preserved = existingFiles.filter(
-      (e) => !newRemotePaths.has(e.remotePath)
-    );
+  // Merge language files into the upload list (deduplicate by remotePath)
+  const allFilesMap = new Map<string, { remotePath: string; localPath: string }>();
+  for (const f of configFiles) {
+    allFilesMap.set(f.remotePath, f);
+  }
+  for (const f of languageFiles) {
+    const entry = { remotePath: f.remotePath, localPath: f.downloadLocalPath ?? f.localPath };
+    allFilesMap.set(f.remotePath, entry);
+  }
+  const allFiles = Array.from(allFilesMap.values());
 
-    config.files = [...preserved, ...configFilesEntries];
+  // If new language files were found, update config.files so they persist
+  if (languageFiles.length > 0 && !dryRun) {
+    config.files = allFiles.map((f) => ({ ...f, direction: "download" as const }));
     await updateConfig(config);
-    console.log(`   ✓ Updated config with ${config.files.length} file entries`);
-  } else {
-    console.log(`   [DRY RUN] Would update config with ${configFilesEntries.length} file entries`);
   }
 
-  // Step 4: Build workerConfigJson
-  const workerConfigJson = buildWorkerConfigJson(config, configFilesEntries);
+  // Build the workerConfigJson using the full file list
+  const workerConfigJson = buildWorkerConfigJson(config, allFiles.map((f) => ({ ...f, direction: "download" })));
 
-  // Step 5: Upload all files
-  console.log(`\n🚀 Uploading ${allFiles.length} files...`);
+  // Upload all files
+  console.log(`\n🚀 Uploading ${allFiles.length} configured files...`);
   let successCount = 0;
   let failCount = 0;
 
-  for (const file of allFiles) {
+  for (const mount of allFiles) {
+    const { remotePath, localPath } = mount;
     try {
-      if (file.isDirectory) {
-        // Upload all files recursively in the local directory
-        const localDir = file.localPath.endsWith("/") ? file.localPath : file.localPath + "/";
-        const remoteDir = file.remotePath.endsWith("/") ? file.remotePath : file.remotePath + "/";
-        const relFiles = listFilesRecursively(localDir);
-        console.log(`  📁 Uploading directory ${localDir} → ${remoteDir} (${relFiles.length} files)`);
-        for (const relFile of relFiles) {
-          await uploadSingleFile(client, AwsS3, localDir + relFile, remoteDir + relFile, dryRun);
-          successCount++;
-        }
+      if (remotePath.endsWith("/") || localPath.endsWith("/")) {
+        const count = await uploadDirectory(client, AwsS3, remotePath, localPath, dryRun);
+        successCount += count;
       } else {
-        await uploadSingleFile(client, AwsS3, file.localPath, file.remotePath, dryRun);
+        await uploadFile(client, AwsS3, remotePath, localPath, dryRun);
         successCount++;
       }
     } catch (error) {
-      console.error(`  ❌ Failed to upload ${file.localPath}: ${error.message}`);
+      console.error(`  ❌ Failed to upload ${localPath}: ${error.message}`);
       failCount++;
     }
   }
 
   console.log(`\n   ✓ Upload complete: ${successCount} succeeded, ${failCount} failed`);
 
-  // Step 6: Create or update cloud worker
+  // Create or update cloud worker
   if (create) {
     const workerName = name || `${projectName}-worker`;
     console.log(`\n🌩️  Creating cloud worker "${workerName}"...`);
@@ -330,3 +333,62 @@ export async function cloudWorker(options: CloudWorkerOptions) {
     console.log(`\n✅ Cloud worker sync complete!`);
   }
 }
+
+/**
+ * Pull the latest workerConfigJson from the cloud worker API and update the
+ * local knowhow.json config to match.
+ */
+export async function pullCloudWorkerConfig(options: CloudWorkerPullOptions) {
+  const { id, apiUrl = KNOWHOW_API_URL } = options;
+
+  // Load JWT
+  const jwt = await loadJwt();
+  if (!jwt) {
+    console.error("❌ No JWT token found. Please run 'knowhow login' first.");
+    process.exit(1);
+  }
+
+  const client = new KnowhowSimpleClient(apiUrl, jwt);
+
+  console.log(`🔄 Pulling config for cloud worker ${id}...`);
+
+  const resp = await client.getCloudWorker(id);
+  const remoteWorker = resp.data;
+
+  if (!remoteWorker) {
+    console.error(`❌ Cloud worker ${id} not found.`);
+    process.exit(1);
+  }
+
+  const remoteConfig = (remoteWorker.workerConfigJson ?? {}) as {
+    mcps?: McpConfig[];
+    modules?: string[];
+    plugins?: Config["plugins"];
+    agents?: Config["agents"];
+  };
+
+  // Load current local config
+  const localConfig = await getConfig();
+
+  // Merge remote fields into local config
+  if (remoteConfig.mcps !== undefined) {
+    localConfig.mcps = remoteConfig.mcps;
+  }
+  if (remoteConfig.modules !== undefined) {
+    localConfig.modules = remoteConfig.modules;
+  }
+  if (remoteConfig.plugins !== undefined) {
+    localConfig.plugins = remoteConfig.plugins;
+  }
+  if (remoteConfig.agents !== undefined) {
+    localConfig.agents = remoteConfig.agents;
+  }
+
+  await updateConfig(localConfig);
+
+  const mcpCount = remoteConfig.mcps?.length ?? 0;
+  console.log(`✅ Config pulled! ${mcpCount} MCP(s) now configured locally.`);
+  console.log(`   Run 'knowhow worker' or trigger reloadConfig to apply changes.`);
+
+  return { mcps: remoteConfig.mcps, modules: remoteConfig.modules };
+}