@tyvm/knowhow 0.0.108 → 0.0.109-dev.38b1faa

package/src/commands/workers.ts

@@ -0,0 +1,168 @@
+import { Command } from "commander";
+import { worker } from "../worker";
+import { TUNNEL_MINIMAL_TOOLS } from "../tunnel";
+import { fileSync } from "../fileSync";
+import {
+  startAllWorkers,
+  listWorkerPaths,
+  unregisterWorkerPath,
+  clearWorkerRegistry,
+} from "../workerRegistry";
+
+export function addWorkerCommand(program: Command): void {
+  program
+    .command("worker")
+    .description(
+      "Start worker process and optionally register current directory"
+    )
+    .option("--register", "Register current directory as a worker path")
+    .option(
+      "--share",
+      "Share this worker with your organization (allows other users to use it)"
+    )
+    .option("--unshare", "Make this worker private (only you can use it)")
+    .option("--sandbox", "Run worker in a Docker container for isolation")
+    .option(
+      "--no-sandbox",
+      "Run worker directly on host (disable sandbox mode)"
+    )
+    .option("--passkey", "Set up passkey authentication for this worker")
+    .option("--passkey-reset", "Remove passkey authentication requirement")
+    .action(async (options) => {
+      const { setupServices } = await import("./services");
+      await setupServices();
+      await worker(options);
+    });
+}
+
+export function addWorkersCommand(program: Command): void {
+  program
+    .command("workers")
+    .description("Manage and start all registered workers")
+    .option("--list", "List all registered worker paths")
+    .option("--unregister <path>", "Unregister a worker path")
+    .option("--clear", "Clear all registered worker paths")
+    .action(async (options) => {
+      try {
+        if (options.list) {
+          const workers = await listWorkerPaths();
+          if (workers.length === 0) {
+            console.log("No workers registered.");
+            console.log(
+              "\nTo register a worker, run 'knowhow worker --register' from the worker directory."
+            );
+          } else {
+            console.log(`Registered workers (${workers.length}):`);
+            workers.forEach((workerPath, index) => {
+              console.log(`  ${index + 1}. ${workerPath}`);
+            });
+          }
+          return;
+        }
+
+        if (options.unregister) {
+          await unregisterWorkerPath(options.unregister);
+          return;
+        }
+
+        if (options.clear) {
+          await clearWorkerRegistry();
+          return;
+        }
+
+        // Default action: start all workers
+        const { setupServices } = await import("./services");
+        await setupServices();
+        await startAllWorkers();
+      } catch (error) {
+        console.error("Error managing workers:", error);
+        process.exit(1);
+      }
+    });
+}
+
+export function addTunnelCommand(program: Command): void {
+  program
+    .command("tunnel")
+    .description(
+      "Start a minimal worker with tunnel enabled: exposes local ports to the cloud. " +
+      "Registers essential tools (unlock, lock, listAllowedPorts) so the backend is aware of the worker and ports. " +
+      "If passkey auth is configured, the tunnel is locked until unlocked via tool call or WebSocket auth protocol."
+    )
+    .option(
+      "--share",
+      "Share this tunnel with your organization (allows other users to use it)"
+    )
+    .option("--unshare", "Make this tunnel private (only you can use it)")
+    .action(async (options) => {
+      console.log("🌐 Starting tunnel (minimal worker) mode...");
+      console.log(`   Tools: ${TUNNEL_MINIMAL_TOOLS.join(", ")}`);
+      await worker({
+        ...options,
+        allowedTools: TUNNEL_MINIMAL_TOOLS,
+      });
+    });
+}
+
+export function addFilesCommand(program: Command): void {
+  program
+    .command("files")
+    .description(
+      "Sync files between local filesystem and Knowhow FS (uses fileMounts config)"
+    )
+    .option("--upload", "Force upload direction for all mounts")
+    .option("--download", "Force download direction for all mounts")
+    .option("--config <path>", "Path to knowhow.json", "./knowhow.json")
+    .option("--dry-run", "Print what would be synced without doing it")
+    .action(async (options) => {
+      try {
+        await fileSync(options);
+      } catch (error) {
+        console.error("Error syncing files:", error);
+        process.exit(1);
+      }
+    });
+}
+
+export function addCloudWorkerCommand(program: Command): void {
+  program
+    .command("cloudworker")
+    .description("Create or sync a cloud worker with your local knowhow config")
+    .option(
+      "--init",
+      "Initialize config.files entries based on what exists in .knowhow/ (run once before --push)"
+    )
+    .option(
+      "--create",
+      "Create a new cloud worker with synced config and files"
+    )
+    .option(
+      "--push <uid>",
+      "Push/sync local config and files to an existing cloud worker"
+    )
+    .option(
+      "--pull <id>",
+      "Pull the latest workerConfigJson from a cloud worker and update local config"
+    )
+    .option("--name <name>", "Name for the cloud worker (used with --create)")
+    .option("--dry-run", "Print what would be synced without doing it")
+    .action(async (options) => {
+      try {
+        const { cloudWorker, pullCloudWorkerConfig, initCloudWorker } = await import(
+          "../cloudWorker"
+        );
+        if (options.init) {
+          await initCloudWorker({ dryRun: options.dryRun });
+          return;
+        }
+        if (options.pull) {
+          await pullCloudWorkerConfig({ id: options.pull });
+        } else {
+          await cloudWorker(options);
+        }
+      } catch (error) {
+        console.error("Error running cloudworker:", error);
+        process.exit(1);
+      }
+    });
+}

package/src/config.ts

@@ -74,7 +74,7 @@ const defaultConfig = {
       description:
         "You can define agents in the config. They will have access to all tools.",
       instructions: "Reply to the user saying 'Hello, world!'",
-      model: "gpt-4o-2024-08-06",
+      model: "gpt-5.4-nano",
       provider: "openai",
     },
   ],
@@ -168,6 +168,27 @@ export async function init() {
   console.log("Initializing global knowhow config at ~/.knowhow");
   const globalConfigDir = await ensureGlobalConfigDir();
 
+  // Ensure the script module is registered in the global config so that
+  // `knowhow script` and the `executeScript` tool are available everywhere.
+  const SCRIPT_MODULE = "@tyvm/knowhow-module-script";
+  const globalConfigPath = path.join(globalConfigDir, "knowhow.json");
+  try {
+    const rawGlobal = fs.existsSync(globalConfigPath)
+      ? fs.readFileSync(globalConfigPath, "utf8")
+      : JSON.stringify(defaultConfig, null, 2);
+    const globalConf = JSON.parse(rawGlobal) as Config;
+    if (!globalConf.modules) {
+      globalConf.modules = [];
+    }
+    if (!globalConf.modules.includes(SCRIPT_MODULE)) {
+      globalConf.modules.push(SCRIPT_MODULE);
+      fs.writeFileSync(globalConfigPath, JSON.stringify(globalConf, null, 2));
+      console.log(`✅ Added ${SCRIPT_MODULE} to ~/.knowhow/knowhow.json modules`);
+    }
+  } catch (e) {
+    console.warn(`⚠ Could not update global config to add script module:`, e);
+  }
+
   // create the folder structure
   console.log("Initializing local knowhow config at ./.knowhow");
   await mkdir(".knowhow", { recursive: true });
@@ -287,6 +308,22 @@ export async function getGlobalConfig(): Promise<Config> {
   }
 }
 
+export async function updateGlobalConfig(config: Config) {
+  if (!config || typeof config !== "object") {
+    throw new Error("Invalid config object");
+  }
+
+  const globalConfigDir = getGlobalConfigDir();
+  await mkdir(globalConfigDir, { recursive: true });
+  const globalConfigPath = path.join(globalConfigDir, "knowhow.json");
+
+  if (fs.existsSync(globalConfigPath)) {
+    await fs.promises.copyFile(globalConfigPath, globalConfigPath + ".bak");
+  }
+
+  await writeFile(globalConfigPath, JSON.stringify(config, null, 2));
+}
+
 export async function migrateConfig() {
   // Apply migrations, used to keep config structure up to date.
   if (!fs.existsSync(".knowhow/knowhow.json")) {

package/src/fileSync.ts

@@ -1,11 +1,14 @@
 import * as fs from "fs";
 import * as path from "path";
+import * as os from "os";
 import { KnowhowSimpleClient, KNOWHOW_API_URL } from "./services/KnowhowClient";
 import { loadJwt } from "./login";
 import { getConfig } from "./config";
 import { services } from "./services";
 import { S3Service } from "./services/S3";
-import { getHashes, hasFileChangedSinceUpload, saveUploadHash, isLocalFileMatchingRemote, isLocalFileMatchingDownloadHash, saveDownloadHash } from "./hashes";
+import { getHashes, saveHashes, hasFileChangedSinceUpload, saveUploadHash, isLocalFileMatchingRemote, isLocalFileMatchingDownloadHash, saveDownloadHash } from "./hashes";
+
+export const DEFAULT_BATCH_SIZE = 5;
 
 export interface FileSyncOptions {
   upload?: boolean;
@@ -15,6 +18,33 @@ export interface FileSyncOptions {
   dryRun?: boolean;
 }
 
+/**
+ * Run an array of async tasks in batches of `batchSize` at a time.
+ * Returns results in the same order as the input tasks.
+ */
+export async function batchRun<T>(
+  tasks: (() => Promise<T>)[],
+  batchSize: number = DEFAULT_BATCH_SIZE
+): Promise<T[]> {
+  const results: T[] = [];
+  for (let i = 0; i < tasks.length; i += batchSize) {
+    const batch = tasks.slice(i, i + batchSize);
+    const batchResults = await Promise.all(batch.map((t) => t()));
+    results.push(...batchResults);
+  }
+  return results;
+}
+
+/**
+ * Expands a leading ~ to the user's home directory
+ */
+function expandHome(p: string): string {
+  if (p === "~" || p.startsWith("~/") || p.startsWith("~\\")) {
+    return path.join(os.homedir(), p.slice(1));
+  }
+  return p;
+}
+
 /**
  * Returns true if the path looks like a directory (ends with /)
  */
@@ -25,7 +55,7 @@ function isDirectoryPath(p: string): boolean {
 /**
  * Recursively list all files in a local directory, returning relative paths
  */
-function listFilesRecursively(dir: string): string[] {
+export function listFilesRecursively(dir: string): string[] {
   const results: string[] = [];
   if (!fs.existsSync(dir)) return results;
   const entries = fs.readdirSync(dir, { withFileTypes: true });
@@ -83,7 +113,8 @@ export async function fileSync(options: FileSyncOptions = {}) {
 
   // Process each file mount
   for (const mount of config.files) {
-    const { remotePath, localPath, direction = "download" } = mount;
+    const { remotePath, localPath: rawLocalPath, direction = "download" } = mount;
+    const localPath = expandHome(rawLocalPath);
 
     // Determine actual direction based on flags and config
     let actualDirection = direction;
@@ -126,16 +157,16 @@ export async function fileSync(options: FileSyncOptions = {}) {
   }
 }
 
-
 /**
  * Download a file from Knowhow FS to local filesystem
  */
-async function downloadFile(
+export async function downloadFile(
   client: KnowhowSimpleClient,
   s3Service: S3Service,
   remotePath: string,
   localPath: string,
-  dryRun: boolean
+  dryRun: boolean,
+  hashes?: any
 ): Promise<void> {
   console.log(`⬇️  Downloading ${remotePath} → ${localPath}`);
 
@@ -146,8 +177,7 @@ async function downloadFile(
 
   try {
     // Fast-path: check stored download hash before hitting the API
-    const hashes = await getHashes();
-    if (await isLocalFileMatchingDownloadHash(localPath, hashes)) {
+    if (hashes && await isLocalFileMatchingDownloadHash(localPath, hashes)) {
       console.log(`   ✓ Skipping ${localPath} (matches stored download hash)`);
       return;
     }
@@ -159,7 +189,7 @@ async function downloadFile(
     if (isLocalFileMatchingRemote(localPath, checksumSHA256)) {
       console.log(`   ✓ Skipping ${localPath} (matches remote checksum)`);
       // Store the hash so future syncs can skip without hitting the API
-      await saveDownloadHash(localPath);
+      await saveDownloadHash(localPath, hashes);
       return;
     }
 
@@ -173,7 +203,7 @@ async function downloadFile(
     await s3Service.downloadFromPresignedUrl(downloadUrl, localPath);
 
     // Save download hash so we can skip unchanged files next time
-    await saveDownloadHash(localPath);
+    await saveDownloadHash(localPath, hashes);
 
     // Get file size for logging
     const stats = fs.statSync(localPath);
@@ -186,12 +216,13 @@ async function downloadFile(
 /**
  * Upload a file from local filesystem to Knowhow FS
  */
-async function uploadFile(
+export async function uploadFile(
   client: KnowhowSimpleClient,
   s3Service: S3Service,
   remotePath: string,
   localPath: string,
-  dryRun: boolean
+  dryRun: boolean,
+  hashes?: any
 ): Promise<void> {
   console.log(`⬆️  Uploading ${localPath} → ${remotePath}`);
 
@@ -207,15 +238,14 @@ async function uploadFile(
   }
 
   // Skip upload if file hasn't changed since last upload
-  const hashes = await getHashes();
-  const changed = await hasFileChangedSinceUpload(localPath, hashes);
+  const changed = hashes ? await hasFileChangedSinceUpload(localPath, hashes) : true;
   if (!changed) {
     console.log(`   ✓ Skipping ${localPath} (unchanged since last upload)`);
     return;
   }
 
   // Get presigned upload URL
-  const presignedUrl = await client.getOrgFilePresignedUploadUrl(remotePath);
+  const presignedUrl = await client.getOrgFilePresignedUploadUrl(remotePath, localPath);
 
   // Upload file using presigned URL
   await s3Service.uploadToPresignedUrl(presignedUrl, localPath);
@@ -224,7 +254,7 @@ async function uploadFile(
   await client.markOrgFileUploadComplete(remotePath);
 
   // Save upload hash so we can skip unchanged files next time
-  await saveUploadHash(localPath);
+  await saveUploadHash(localPath, hashes);
 
   const stats = fs.statSync(localPath);
   console.log(`   ✓ Uploaded ${stats.size} bytes`);
@@ -246,6 +276,8 @@ export async function uploadDirectory(
 
   console.log(`⬆️  Uploading directory ${localDir} → ${remoteDir}`);
 
+  const hashes = await getHashes();
+
   if (!fs.existsSync(localDir)) {
     console.warn(`   ⚠️  Local directory not found: ${localDir}`);
     return 0;
@@ -261,26 +293,30 @@ export async function uploadDirectory(
 
   console.log(`   Found ${localFiles.length} local file(s)`);
 
-  let count = 0;
-  for (const relFile of localFiles) {
+  const tasks = localFiles.map((relFile) => async () => {
     const localFilePath = localDir + relFile;
     const remoteFilePath = remoteDir + relFile;
     try {
-      await uploadFile(client, s3Service, remoteFilePath, localFilePath, dryRun);
-      count++;
+      await uploadFile(client, s3Service, remoteFilePath, localFilePath, dryRun, hashes);
+      return 1;
     } catch (error) {
       console.error(
         `   ❌ Failed to upload ${localFilePath}, skipping: ${error.message}`
       );
+      return 0;
     }
-  }
-  return count;
+  });
+
+  const counts = await batchRun(tasks);
+  await saveHashes(hashes);
+
+  return counts.reduce((sum, n) => sum + n, 0);
 }
 
 /**
  * Download all files from a remote directory path to a local directory
  */
-async function downloadDirectory(
+export async function downloadDirectory(
   client: KnowhowSimpleClient,
   s3Service: S3Service,
   remotePath: string,
@@ -293,6 +329,8 @@ async function downloadDirectory(
 
   console.log(`⬇️  Downloading directory ${remoteDir} → ${localDir}`);
 
+  const hashes = await getHashes();
+
   // List all org files and find those in the remote directory
   const response = await client.listOrgFiles();
   const allFiles = response.data;
@@ -313,16 +351,19 @@ async function downloadDirectory(
 
   console.log(`   Found ${matchingFiles.length} remote file(s)`);
 
-  let count = 0;
-  for (const f of matchingFiles) {
+  const tasks = matchingFiles.map((f) => async () => {
     const fullRemotePath = f.folderPath.endsWith("/")
       ? f.folderPath + f.fileName
       : f.folderPath + "/" + f.fileName;
     // Strip the base remote dir prefix to get relative path
     const relativePath = fullRemotePath.slice(remoteDir.length);
     const localFilePath = localDir + relativePath;
-    await downloadFile(client, s3Service, fullRemotePath, localFilePath, dryRun);
-    count++;
-  }
-  return count;
+    await downloadFile(client, s3Service, fullRemotePath, localFilePath, dryRun, hashes);
+    return 1;
+  });
+
+  const counts = await batchRun(tasks);
+  await saveHashes(hashes);
+
+  return counts.reduce((sum, n) => sum + n, 0);
 }

package/src/hashes.ts

@@ -1,16 +1,35 @@
 import fs from "fs";
 import * as crypto from "crypto";
 import { Hashes } from "./types";
-import { readFile, writeFile } from "./utils";
+import { readFile } from "./utils";
 import { convertToText } from "./conversion";
 
 export async function getHashes() {
-  const hashes = JSON.parse(await readFile(".knowhow/.hashes.json", "utf8"));
-  return hashes as Hashes;
+  try {
+    const hashes = JSON.parse(await readFile(".knowhow/.hashes.json", "utf8"));
+    return hashes as Hashes;
+  } catch (err: any) {
+    if (err.code === "ENOENT") {
+      return {} as Hashes;
+    }
+    throw err;
+  }
 }
 
+/**
+ * Atomically save hashes to disk — writes to a temp file then renames,
+ * preventing concurrent writes from producing corrupted/truncated JSON.
+ */
 export async function saveHashes(hashes: any) {
-  await writeFile(".knowhow/.hashes.json", JSON.stringify(hashes, null, 2));
+  const target = ".knowhow/.hashes.json";
+  const tmp = `${target}.tmp.${process.pid}`;
+  try {
+    fs.writeFileSync(tmp, JSON.stringify(hashes, null, 2));
+    fs.renameSync(tmp, target);
+  } catch (err) {
+    try { fs.unlinkSync(tmp); } catch (_) {}
+    throw err;
+  }
 }
 
 export async function md5Hash(str: string) {
@@ -90,17 +109,19 @@ export async function hasFileChangedSinceUpload(
 }
 
 /**
- * Saves the hash of the file at the time of a successful upload
+ * Mutates the provided hashes object with the upload hash for localPath.
+ * If no hashes object is provided, loads, mutates, and saves independently.
  */
-export async function saveUploadHash(localPath: string) {
-  const hashes = await getHashes();
+export async function saveUploadHash(localPath: string, hashes?: any) {
+  const standalone = !hashes;
+  if (standalone) hashes = await getHashes();
   const content = fs.readFileSync(localPath);
   const currentHash = crypto.createHash("md5").update(content).digest("hex");
   if (!hashes[localPath]) {
     hashes[localPath] = { fileHash: currentHash, promptHash: "" };
   }
   hashes[localPath][UPLOAD_KEY] = currentHash;
-  await saveHashes(hashes);
+  if (standalone) await saveHashes(hashes);
 }
 
 /**
@@ -120,18 +141,19 @@ export async function isLocalFileMatchingDownloadHash(
 }
 
 /**
- * Saves the SHA-256 hash of the file after a successful download so we can
- * skip unchanged files on the next sync.
+ * Mutates the provided hashes object with the download hash for localPath.
+ * If no hashes object is provided, loads, mutates, and saves independently.
  */
-export async function saveDownloadHash(localPath: string) {
-  const hashes = await getHashes();
+export async function saveDownloadHash(localPath: string, hashes?: any) {
+  const standalone = !hashes;
+  if (standalone) hashes = await getHashes();
   const content = fs.readFileSync(localPath);
   const currentHash = crypto.createHash("sha256").update(content).digest("base64");
   if (!hashes[localPath]) {
     hashes[localPath] = { fileHash: currentHash, promptHash: "" };
   }
   hashes[localPath][DOWNLOAD_KEY] = currentHash;
-  await saveHashes(hashes);
+  if (standalone) await saveHashes(hashes);
 }
 
 /**

package/src/index.ts

@@ -53,6 +53,7 @@ export * as ai from "./ai";
 
 // Export module system types for external modules
 export * from "./services/modules/types";
+export { ModulesService } from "./services/modules";
 // Export plugin types for external plugins
 export { PluginBase } from "./plugins/PluginBase";
 export { PluginMeta, Plugin, PluginContext } from "./plugins/types";
@@ -138,6 +139,23 @@ export async function upload() {
       if (!source.remoteId) {
         throw new Error("remoteId is required for knowhow uploads");
       }
+      // Warn if the local embeddingModel differs from the one stored on the backend
+      try {
+        const remoteEmbedding = await knowhowApiClient.getOrgEmbedding(source.remoteId);
+        const localModel = config.embeddingModel || EmbeddingModels.openai.EmbeddingAda2;
+        const remoteModel = remoteEmbedding?.modelName;
+        if (remoteModel && remoteModel !== localModel) {
+          console.warn(
+            `⚠️  WARNING: Embedding model mismatch for "${remoteEmbedding.name}" (remoteId: ${source.remoteId}).\n` +
+            `   Local config.embeddingModel:  ${localModel}\n` +
+            `   Backend embedding modelName:  ${remoteModel}\n` +
+            `   Vectors generated with different models are not comparable — search results will be incorrect.\n` +
+            `   Update your config.embeddingModel to "${remoteModel}" or update the backend embedding to "${localModel}".`
+          );
+        }
+      } catch (e) {
+        // Non-fatal — don't block upload if metadata fetch fails
+      }
       const url = await knowhowApiClient.getPresignedUploadUrl(source);
       console.log("Uploading to", url);
       await AwsS3.uploadToPresignedUrl(url, source.output);