@tyvm/knowhow 0.0.101 → 0.0.102

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tyvm/knowhow",
-  "version": "0.0.101",
+  "version": "0.0.102",
   "description": "ai cli with plugins and agents",
   "main": "ts_build/src/index.js",
   "bin": {

package/src/agents/base/base.ts

@@ -64,7 +64,13 @@ export abstract class BaseAgent implements IAgent {
   protected compressMinMessages = 30;
 
   protected threads = [] as Message[][];
+
+  // Message from users
   protected pendingUserMessages = [] as Message[];
+
+  // Internal messages
+  protected pendingMessages = [] as Message[];
+
   protected taskBreakdown = "";
   protected summaries = [] as string[];
   protected currentTaskId: string | null = null;
@@ -538,10 +544,14 @@ export abstract class BaseAgent implements IAgent {
 
   async kill() {
     this.log("Killing agent");
+    if (this.status === this.eventTypes.kill || this.status === this.eventTypes.done) {
+      this.log("Agent is already being killed or done, ignoring duplicate kill()", "warn");
+      return;
+    }
     this.agentEvents.emit(this.eventTypes.kill, this);
     this.status = this.eventTypes.kill;
 
-    this.addPendingUserMessage({
+    this.addPendingMessage({
       role: "user",
       content: `<Workflow>The user has requested the task to end, please call ${this.requiredToolNames} with a report of your ending state</Workflow>`,
     } as Message);
@@ -599,6 +609,11 @@ export abstract class BaseAgent implements IAgent {
         this.pendingUserMessages = [];
       }
 
+      if (this.pendingMessages.length) {
+        messages.push(...this.pendingMessages);
+        this.pendingMessages = [];
+      }
+
       messages = this.formatInputMessages(messages);
       this.updateCurrentThread(messages);
       const isMissingTool = this.isRequiredToolMissing();
@@ -673,7 +688,7 @@ export abstract class BaseAgent implements IAgent {
           this.updateCurrentThread(messages);
 
           for (const toolCall of toolCalls) {
-            if(this.status === this.eventTypes.pause) {
+            if (this.status === this.eventTypes.pause) {
               this.log(
                 "Agent was paused before tool call, waiting before processing tool calls"
               );
@@ -716,7 +731,6 @@ export abstract class BaseAgent implements IAgent {
               });
               const doneMsg = finalMessage.content || "Done";
 
-
               this.agentEvents.emit(this.eventTypes.done, doneMsg);
               this.status = this.eventTypes.done;
               return doneMsg;
@@ -908,21 +922,32 @@ export abstract class BaseAgent implements IAgent {
     });
   }
 
+  // A new message from system, non blocking
   addPendingMessage(message: Message) {
     if (this.status === this.eventTypes.done) {
       this.log("Agent is done, cannot take more messages", "warn");
     } else {
-      const pendingMessages = this.pendingUserMessages.map((m) => m.content);
+      const pendingMessages = this.pendingMessages.map((m) => m.content);
       if (pendingMessages.includes(message.content)) {
         // Ignore messages we already have queue'd up
         return;
       }
-      this.pendingUserMessages.push(message);
+      this.pendingMessages.push(message);
     }
   }
 
+  // A new message from users, blocks completion
   addPendingUserMessage(message: Message) {
-    this.addPendingMessage(message);
+    if (this.status === this.eventTypes.done) {
+      this.log("Agent is done, cannot take more messages", "warn");
+    } else {
+      const pendingMessages = this.pendingUserMessages.map((m) => m.content);
+      if (pendingMessages.includes(message.content)) {
+        // Ignore messages we already have queue'd up
+        return;
+      }
+      this.pendingUserMessages.push(message);
+    }
     this.events.emit(this.eventTypes.userSay, message.content);
   }
 

package/src/cloudWorker.ts

@@ -1,6 +1,5 @@
 import * as fs from "fs";
 import * as path from "path";
-import { glob } from "glob";
 import { KnowhowSimpleClient, KNOWHOW_API_URL } from "./services/KnowhowClient";
 import { loadJwt } from "./login";
 import { getConfig, updateConfig, getLanguageConfig } from "./config";
@@ -23,6 +22,26 @@ interface FileToSync {
   localPath: string;
   remotePath: string;
   downloadLocalPath?: string; // override localPath used when worker downloads the file
+  isDirectory?: boolean; // true if this represents a whole directory
+}
+
+/**
+ * Recursively list all files in a local directory, returning relative paths
+ */
+function listFilesRecursively(dir: string): string[] {
+  const results: string[] = [];
+  if (!fs.existsSync(dir)) return results;
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      listFilesRecursively(path.join(dir, entry.name)).forEach((f) =>
+        results.push(entry.name + "/" + f)
+      );
+    } else {
+      results.push(entry.name);
+    }
+  }
+  return results;
 }
 
 /**
@@ -48,6 +67,8 @@ function buildWorkerConfigJson(config: Config, files: { remotePath: string; loca
 
 /**
  * Collect all files from the .knowhow directory that should be synced
+ * Uses directory-level entries where possible so the worker config stays compact
+ * and the folder upload/download feature handles individual files automatically.
  */
 async function collectFilesToSync(projectName: string): Promise<FileToSync[]> {
   const filesToSync: FileToSync[] = [];
@@ -59,39 +80,24 @@ async function collectFilesToSync(projectName: string): Promise<FileToSync[]> {
     }
   };
 
+  // Helper to add a directory entry if it exists (trailing slash = directory mode)
+  const addDirIfExists = (localPath: string, remotePath: string) => {
+    if (fs.existsSync(localPath)) {
+      filesToSync.push({ localPath: localPath + "/", remotePath: remotePath + "/", isDirectory: true });
+    }
+  };
+
   // .knowhow/language.json
   addIfExists(".knowhow/language.json", `${projectName}/.knowhow/language.json`);
 
   // .knowhow/hashes.json
   addIfExists(".knowhow/hashes.json", `${projectName}/.knowhow/hashes.json`);
 
-  // .knowhow/prompts/**/*
-  const promptFiles = await glob(".knowhow/prompts/**/*", { nodir: true });
-  for (const filePath of promptFiles) {
-    const relativeToDotKnowhow = filePath.replace(/^\.knowhow\//, "");
-    const remotePath = `${projectName}/.knowhow/${relativeToDotKnowhow}`;
-    filesToSync.push({ localPath: filePath, remotePath });
-  }
-
-  // .knowhow/scripts/**/* (if exists)
-  if (fs.existsSync(".knowhow/scripts")) {
-    const scriptFiles = await glob(".knowhow/scripts/**/*", { nodir: true });
-    for (const filePath of scriptFiles) {
-      const relativeToDotKnowhow = filePath.replace(/^\.knowhow\//, "");
-      const remotePath = `${projectName}/.knowhow/${relativeToDotKnowhow}`;
-      filesToSync.push({ localPath: filePath, remotePath });
-    }
-  }
-
-  // .knowhow/skills/**/* (if exists)
-  if (fs.existsSync(".knowhow/skills")) {
-    const skillFiles = await glob(".knowhow/skills/**/*", { nodir: true });
-    for (const filePath of skillFiles) {
-      const relativeToDotKnowhow = filePath.replace(/^\.knowhow\//, "");
-      const remotePath = `${projectName}/.knowhow/${relativeToDotKnowhow}`;
-      filesToSync.push({ localPath: filePath, remotePath });
-    }
-  }
+  // Directories — use trailing-slash entries so folder upload/download handles them
+  addDirIfExists(".knowhow/prompts", `${projectName}/.knowhow/prompts`);
+  addDirIfExists(".knowhow/scripts", `${projectName}/.knowhow/scripts`);
+  addDirIfExists(".knowhow/skills", `${projectName}/.knowhow/skills`);
+  addDirIfExists(".knowhow/tasks", `${projectName}/.knowhow/tasks`);
 
   return filesToSync;
 }
@@ -264,8 +270,20 @@ export async function cloudWorker(options: CloudWorkerOptions) {
 
   for (const file of allFiles) {
     try {
-      await uploadSingleFile(client, AwsS3, file.localPath, file.remotePath, dryRun);
-      successCount++;
+      if (file.isDirectory) {
+        // Upload all files recursively in the local directory
+        const localDir = file.localPath.endsWith("/") ? file.localPath : file.localPath + "/";
+        const remoteDir = file.remotePath.endsWith("/") ? file.remotePath : file.remotePath + "/";
+        const relFiles = listFilesRecursively(localDir);
+        console.log(`  📁 Uploading directory ${localDir} → ${remoteDir} (${relFiles.length} files)`);
+        for (const relFile of relFiles) {
+          await uploadSingleFile(client, AwsS3, localDir + relFile, remoteDir + relFile, dryRun);
+          successCount++;
+        }
+      } else {
+        await uploadSingleFile(client, AwsS3, file.localPath, file.remotePath, dryRun);
+        successCount++;
+      }
     } catch (error) {
       console.error(`  ❌ Failed to upload ${file.localPath}: ${error.message}`);
       failCount++;

package/src/fileSync.ts

@@ -5,6 +5,7 @@ import { loadJwt } from "./login";
 import { getConfig } from "./config";
 import { services } from "./services";
 import { S3Service } from "./services/S3";
+import { getHashes, hasFileChangedSinceUpload, saveUploadHash, isLocalFileMatchingRemote } from "./hashes";
 
 export interface FileSyncOptions {
   upload?: boolean;
@@ -14,6 +15,33 @@ export interface FileSyncOptions {
   dryRun?: boolean;
 }
 
+/**
+ * Returns true if the path looks like a directory (ends with /)
+ */
+function isDirectoryPath(p: string): boolean {
+  return p.endsWith("/");
+}
+
+/**
+ * Recursively list all files in a local directory, returning relative paths
+ */
+function listFilesRecursively(dir: string): string[] {
+  const results: string[] = [];
+  if (!fs.existsSync(dir)) return results;
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      const subFiles = listFilesRecursively(path.join(dir, entry.name));
+      for (const f of subFiles) {
+        results.push(entry.name + "/" + f);
+      }
+    } else {
+      results.push(entry.name);
+    }
+  }
+  return results;
+}
+
 /**
  * Sync files between local filesystem and Knowhow FS
  */
@@ -67,11 +95,21 @@ export async function fileSync(options: FileSyncOptions = {}) {
 
     try {
       if (actualDirection === "download") {
-        await downloadFile(client, AwsS3, remotePath, localPath, dryRun);
-        successCount++;
+        if (isDirectoryPath(remotePath) || isDirectoryPath(localPath)) {
+          const count = await downloadDirectory(client, AwsS3, remotePath, localPath, dryRun);
+          successCount += count;
+        } else {
+          await downloadFile(client, AwsS3, remotePath, localPath, dryRun);
+          successCount++;
+        }
       } else if (actualDirection === "upload") {
-        await uploadFile(client, AwsS3, remotePath, localPath, dryRun);
-        successCount++;
+        if (isDirectoryPath(remotePath) || isDirectoryPath(localPath)) {
+          const count = await uploadDirectory(client, AwsS3, remotePath, localPath, dryRun);
+          successCount += count;
+        } else {
+          await uploadFile(client, AwsS3, remotePath, localPath, dryRun);
+          successCount++;
+        }
       }
     } catch (error) {
       console.error(`❌ Failed to sync ${remotePath}: ${error.message}`);
@@ -88,6 +126,7 @@ export async function fileSync(options: FileSyncOptions = {}) {
   }
 }
 
+
 /**
  * Download a file from Knowhow FS to local filesystem
  */
@@ -106,10 +145,14 @@ async function downloadFile(
   }
 
   try {
-    // Get presigned download URL
-    const presignedUrl = await client.getOrgFilePresignedDownloadUrl(
-      remotePath
-    );
+    // Get presigned download URL + remote checksum
+    const { downloadUrl, checksumSHA256 } = await client.getOrgFilePresignedDownloadUrl(remotePath);
+
+    // Skip if local file matches remote checksum
+    if (isLocalFileMatchingRemote(localPath, checksumSHA256)) {
+      console.log(`   ✓ Skipping ${localPath} (matches remote checksum)`);
+      return;
+    }
 
     // Ensure parent directory exists
     const dir = path.dirname(localPath);
@@ -118,7 +161,7 @@ async function downloadFile(
     }
 
     // Download file using presigned URL
-    await s3Service.downloadFromPresignedUrl(presignedUrl, localPath);
+    await s3Service.downloadFromPresignedUrl(downloadUrl, localPath);
 
     // Get file size for logging
     const stats = fs.statSync(localPath);
@@ -151,6 +194,14 @@ async function uploadFile(
     return;
   }
 
+  // Skip upload if file hasn't changed since last upload
+  const hashes = await getHashes();
+  const changed = await hasFileChangedSinceUpload(localPath, hashes);
+  if (!changed) {
+    console.log(`   ✓ Skipping ${localPath} (unchanged since last upload)`);
+    return;
+  }
+
   // Get presigned upload URL
   const presignedUrl = await client.getOrgFilePresignedUploadUrl(remotePath);
 
@@ -160,6 +211,99 @@ async function uploadFile(
   // Notify backend that upload is complete to update the updatedAt timestamp
   await client.markOrgFileUploadComplete(remotePath);
 
+  // Save upload hash so we can skip unchanged files next time
+  await saveUploadHash(localPath);
+
   const stats = fs.statSync(localPath);
   console.log(`   ✓ Uploaded ${stats.size} bytes`);
 }
+
+/**
+ * Upload all files from a local directory to a remote directory path
+ */
+export async function uploadDirectory(
+  client: KnowhowSimpleClient,
+  s3Service: S3Service,
+  remotePath: string,
+  localPath: string,
+  dryRun: boolean
+): Promise<number> {
+  // Normalize paths to end with /
+  const remoteDir = remotePath.endsWith("/") ? remotePath : remotePath + "/";
+  const localDir = localPath.endsWith("/") ? localPath : localPath + "/";
+
+  console.log(`⬆️  Uploading directory ${localDir} → ${remoteDir}`);
+
+  if (!fs.existsSync(localDir)) {
+    console.warn(`   ⚠️  Local directory not found: ${localDir}`);
+    return 0;
+  }
+
+  // Find all files recursively in the local directory
+  const localFiles = listFilesRecursively(localDir);
+
+  if (localFiles.length === 0) {
+    console.log(`   ⚠️  No local files found under ${localDir}`);
+    return 0;
+  }
+
+  console.log(`   Found ${localFiles.length} local file(s)`);
+
+  let count = 0;
+  for (const relFile of localFiles) {
+    const localFilePath = localDir + relFile;
+    const remoteFilePath = remoteDir + relFile;
+    await uploadFile(client, s3Service, remoteFilePath, localFilePath, dryRun);
+    count++;
+  }
+  return count;
+}
+
+/**
+ * Download all files from a remote directory path to a local directory
+ */
+async function downloadDirectory(
+  client: KnowhowSimpleClient,
+  s3Service: S3Service,
+  remotePath: string,
+  localPath: string,
+  dryRun: boolean
+): Promise<number> {
+  // Normalize paths to end with /
+  const remoteDir = remotePath.endsWith("/") ? remotePath : remotePath + "/";
+  const localDir = localPath.endsWith("/") ? localPath : localPath + "/";
+
+  console.log(`⬇️  Downloading directory ${remoteDir} → ${localDir}`);
+
+  // List all org files and find those in the remote directory
+  const response = await client.listOrgFiles();
+  const allFiles = response.data;
+
+  // Find files where the full path starts with remoteDir
+  const matchingFiles = allFiles.filter((f) => {
+    const fullPath = f.folderPath.endsWith("/")
+      ? f.folderPath + f.fileName
+      : f.folderPath + "/" + f.fileName;
+    return fullPath.startsWith(remoteDir);
+  });
+
+  if (matchingFiles.length === 0) {
+    console.log(`   ⚠️  No remote files found under ${remoteDir}`);
+    return 0;
+  }
+
+  console.log(`   Found ${matchingFiles.length} remote file(s)`);
+
+  let count = 0;
+  for (const f of matchingFiles) {
+    const fullRemotePath = f.folderPath.endsWith("/")
+      ? f.folderPath + f.fileName
+      : f.folderPath + "/" + f.fileName;
+    // Strip the base remote dir prefix to get relative path
+    const relativePath = fullRemotePath.slice(remoteDir.length);
+    const localFilePath = localDir + relativePath;
+    await downloadFile(client, s3Service, fullRemotePath, localFilePath, dryRun);
+    count++;
+  }
+  return count;
+}

package/src/hashes.ts

@@ -70,3 +70,55 @@ export async function saveAllFileHashes(files: string[], promptHash: string) {
 
   await saveHashes(hashes);
 }
+
+const UPLOAD_KEY = "upload";
+
+/**
+ * Returns true if the file has changed since the last successful upload
+ * (or if it has never been uploaded before)
+ */
+export async function hasFileChangedSinceUpload(
+  localPath: string,
+  hashes: any
+): Promise<boolean> {
+  if (!fs.existsSync(localPath)) return true;
+  const content = fs.readFileSync(localPath);
+  const currentHash = crypto.createHash("md5").update(content).digest("hex");
+  return hashes[localPath]?.[UPLOAD_KEY] !== currentHash;
+}
+
+/**
+ * Saves the hash of the file at the time of a successful upload
+ */
+export async function saveUploadHash(localPath: string) {
+  const hashes = await getHashes();
+  const content = fs.readFileSync(localPath);
+  const currentHash = crypto.createHash("md5").update(content).digest("hex");
+  if (!hashes[localPath]) {
+    hashes[localPath] = { fileHash: currentHash, promptHash: "" };
+  }
+  hashes[localPath][UPLOAD_KEY] = currentHash;
+  await saveHashes(hashes);
+}
+
+/**
+ * Compute SHA-256 of a local file, returned as base64 (matches S3 encoding)
+ */
+export function computeSHA256Base64(filePath: string): string {
+  const content = fs.readFileSync(filePath);
+  return crypto.createHash("sha256").update(content).digest("base64");
+}
+
+/**
+ * Returns true if the local file's SHA-256 matches the remote checksum,
+ * meaning the file is up-to-date and download can be skipped.
+ */
+export function isLocalFileMatchingRemote(
+  localPath: string,
+  remoteChecksumSHA256: string | null
+): boolean {
+  if (!remoteChecksumSHA256) return false;
+  if (!fs.existsSync(localPath)) return false;
+  const localChecksum = computeSHA256Base64(localPath);
+  return localChecksum === remoteChecksumSHA256;
+}

package/src/services/KnowhowClient.ts

@@ -575,7 +575,9 @@ export class KnowhowSimpleClient {
    * Get presigned S3 URL for downloading a file from Knowhow FS.
    * First finds or creates the file by path, then gets its download URL.
    */
-  async getOrgFilePresignedDownloadUrl(filePath: string): Promise<string> {
+  async getOrgFilePresignedDownloadUrl(
+    filePath: string
+  ): Promise<{ downloadUrl: string; checksumSHA256: string | null }> {
     await this.checkJwt();
 
     // Find the file by path
@@ -585,12 +587,15 @@ export class KnowhowSimpleClient {
     }
 
     // Get download URL using the file ID
-    const response = await http.post<{ downloadUrl: string }>(
+    const response = await http.post<{ downloadUrl: string; checksumSHA256: string | null }>(
       `${this.baseUrl}/api/org-files/download/${file.id}`,
       {},
       { headers: this.headers }
     );
-    return response.data.downloadUrl;
+    return {
+      downloadUrl: response.data.downloadUrl,
+      checksumSHA256: response.data.checksumSHA256 ?? null,
+    };
   }
 
   /**

package/src/services/S3.ts

@@ -1,4 +1,5 @@
 import * as fs from "fs";
+import * as crypto from "crypto";
 import { createWriteStream, createReadStream } from "fs";
 import { pipeline, Readable } from "stream";
 import * as util from "util";
@@ -11,19 +12,28 @@ export class S3Service {
     filePath: string
   ): Promise<void> {
     try {
-      const fileStream = createReadStream(filePath);
+      const fileContent = fs.readFileSync(filePath);
       const fileStats = await fs.promises.stat(filePath);
+      const sha256Base64 = crypto
+        .createHash("sha256")
+        .update(fileContent)
+        .digest("base64");
 
       const response = await fetch(presignedUrl, {
         method: "PUT",
-        headers: { "Content-Length": String(fileStats.size) },
-        body: fileStream as any,
-        // @ts-ignore - Node 18+ supports ReadableStream body with duplex
+        headers: {
+          "Content-Length": String(fileStats.size),
+          "x-amz-checksum-sha256": sha256Base64,
+          "x-amz-sdk-checksum-algorithm": "SHA256",
+        },
+        body: fileContent,
+        // @ts-ignore
         duplex: "half",
       });
 
       if (!response.ok) {
-        throw new Error(`Upload failed with status code: ${response.status}`);
+        const text = await response.text();
+        throw new Error(`Upload failed with status code: ${response.status} - ${text}`);
       }
 
       console.log("File uploaded successfully to pre-signed URL");

package/ts_build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@tyvm/knowhow",
-    "version": "0.0.101",
+    "version": "0.0.102",
     "description": "ai cli with plugins and agents",
     "main": "ts_build/src/index.js",
     "bin": {

package/ts_build/src/agents/base/base.d.ts

@@ -42,6 +42,7 @@ export declare abstract class BaseAgent implements IAgent {
     protected compressMinMessages: number;
     protected threads: Message[][];
     protected pendingUserMessages: Message[];
+    protected pendingMessages: Message[];
     protected taskBreakdown: string;
     protected summaries: string[];
     protected currentTaskId: string | null;

package/ts_build/src/agents/base/base.js

@@ -28,6 +28,7 @@ class BaseAgent {
     compressMinMessages = 30;
     threads = [];
     pendingUserMessages = [];
+    pendingMessages = [];
     taskBreakdown = "";
     summaries = [];
     currentTaskId = null;
@@ -372,9 +373,13 @@ class BaseAgent {
     }
     async kill() {
         this.log("Killing agent");
+        if (this.status === this.eventTypes.kill || this.status === this.eventTypes.done) {
+            this.log("Agent is already being killed or done, ignoring duplicate kill()", "warn");
+            return;
+        }
         this.agentEvents.emit(this.eventTypes.kill, this);
         this.status = this.eventTypes.kill;
-        this.addPendingUserMessage({
+        this.addPendingMessage({
             role: "user",
             content: `<Workflow>The user has requested the task to end, please call ${this.requiredToolNames} with a report of your ending state</Workflow>`,
         });
@@ -407,6 +412,10 @@ class BaseAgent {
                 messages.push(...this.pendingUserMessages);
                 this.pendingUserMessages = [];
             }
+            if (this.pendingMessages.length) {
+                messages.push(...this.pendingMessages);
+                this.pendingMessages = [];
+            }
             messages = this.formatInputMessages(messages);
             this.updateCurrentThread(messages);
             const isMissingTool = this.isRequiredToolMissing();
@@ -601,15 +610,24 @@ class BaseAgent {
             this.log("Agent is done, cannot take more messages", "warn");
         }
         else {
-            const pendingMessages = this.pendingUserMessages.map((m) => m.content);
+            const pendingMessages = this.pendingMessages.map((m) => m.content);
             if (pendingMessages.includes(message.content)) {
                 return;
             }
-            this.pendingUserMessages.push(message);
+            this.pendingMessages.push(message);
         }
     }
     addPendingUserMessage(message) {
-        this.addPendingMessage(message);
+        if (this.status === this.eventTypes.done) {
+            this.log("Agent is done, cannot take more messages", "warn");
+        }
+        else {
+            const pendingMessages = this.pendingUserMessages.map((m) => m.content);
+            if (pendingMessages.includes(message.content)) {
+                return;
+            }
+            this.pendingUserMessages.push(message);
+        }
         this.events.emit(this.eventTypes.userSay, message.content);
     }
     getMessagesLength(messages) {