@tyvm/knowhow 0.0.108-dev.4a8ba55 → 0.0.108-dev.501f36f

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tyvm/knowhow",
-  "version": "0.0.108-dev.4a8ba55",
+  "version": "0.0.108-dev.501f36f",
   "description": "ai cli with plugins and agents",
   "main": "ts_build/src/index.js",
   "bin": {

package/src/chat/CliChatService.ts

@@ -19,6 +19,7 @@ import editor from "@inquirer/editor";
 import fs from "fs";
 import path from "path";
 import { services } from "../services";
+import { logger } from "../logger";
 
 export class CliChatService implements ChatService {
   private context: ChatContext;
@@ -267,10 +268,12 @@ export class CliChatService implements ChatService {
     } else if (this.context.multilineMode) {
       const renderer = this.context.renderer;
       if (renderer) renderer.pause();
+      logger.silence();
       try {
         value = await editor({ message: prompt });
       } finally {
         if (renderer) renderer.resume();
+        logger.unsilence();
       }
       this.context.multilineMode = false; // Disable after use like original
     } else {

package/src/cli.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node --no-node-snapshot
 import { Command } from "commander";
 import { version } from "../package.json";
+import { logger } from "./logger";
 import { migrateConfig } from "./config";
 import { getConfig, getGlobalConfig } from "./config";
 import { getEnabledPlugins } from "./types";
@@ -48,6 +49,19 @@ process.on("unhandledRejection", (reason: unknown) => {
 
 async function main() {
   const program = new Command();
+
+  // Install console overload early so ALL output (including third-party modules)
+  // goes through our logger closure — respects silence() for clean-stdout commands.
+  logger.installConsoleOverload();
+
+  // Silence immediately if this is a clean-stdout command (e.g. git credential helpers).
+  // Module loading happens before parseAsync, so we must silence before that point.
+  const rawArgs = process.argv.slice(2);
+  const SILENT_COMMANDS = ["github-credentials"];
+  if (rawArgs.some((a) => SILENT_COMMANDS.includes(a))) {
+    logger.silence();
+  }
+
   await migrateConfig();
   const config = await getConfig();
   const chatService = new CliChatService(getEnabledPlugins(config.plugins));

package/src/clients/index.ts

@@ -577,11 +577,12 @@ export class AIClient {
    * @param modelQuery - the model name to search for (can be partial/normalized)
    * @param provider   - optional provider to restrict search to
    */
-  findModelFuzzy(modelQuery: string, provider?: string): { provider: string; model: string } | undefined {
+  findModelFuzzy(
+    modelQuery: string,
+    provider?: string
+  ): { provider: string; model: string } | undefined {
     const queryNorm = AIClient.normalizeModelId(modelQuery);
-    const providers = provider
-      ? [provider]
-      : Object.keys(this.clientModels);
+    const providers = provider ? [provider] : Object.keys(this.clientModels);
 
     for (const p of providers) {
       const models = (this.clientModels[p] as string[]) ?? [];
@@ -835,7 +836,7 @@ export class AIClient {
         const splitModel = m.id.split("/");
 
         if (splitModel.length < 2) {
-          console.error(`Cannot parse model format: ${m.id}`);
+          console.warn(`Cannot parse model format: ${m.id}`);
         }
 
         const provider = splitModel.length > 1 ? splitModel[0] : "";

package/src/cloudWorker.ts

@@ -5,7 +5,7 @@ import { loadJwt } from "./login";
 import { getConfig, updateConfig, getLanguageConfig } from "./config";
 import { services } from "./services";
 import { Language, Config, McpConfig } from "./types";
-import { S3Service } from "./services/S3";
+import { uploadFile, uploadDirectory } from "./fileSync";
 
 export interface CloudWorkerPullOptions {
   id: string;
@@ -15,6 +15,7 @@ export interface CloudWorkerPullOptions {
 export interface CloudWorkerOptions {
   create?: boolean;
   push?: string; // uid of existing cloud worker
+  init?: boolean; // initialize config.files entries (mutates config)
   name?: string; // optional name for create
   apiUrl?: string;
   dryRun?: boolean;
@@ -30,25 +31,6 @@ interface FileToSync {
   isDirectory?: boolean; // true if this represents a whole directory
 }
 
-/**
- * Recursively list all files in a local directory, returning relative paths
- */
-function listFilesRecursively(dir: string): string[] {
-  const results: string[] = [];
-  if (!fs.existsSync(dir)) return results;
-  const entries = fs.readdirSync(dir, { withFileTypes: true });
-  for (const entry of entries) {
-    if (entry.isDirectory()) {
-      listFilesRecursively(path.join(dir, entry.name)).forEach((f) =>
-        results.push(entry.name + "/" + f)
-      );
-    } else {
-      results.push(entry.name);
-    }
-  }
-  return results;
-}
-
 /**
  * Build the worker config JSON from the local knowhow config
  */
@@ -71,21 +53,19 @@ function buildWorkerConfigJson(config: Config, files: { remotePath: string; loca
 }
 
 /**
- * Collect all files from the .knowhow directory that should be synced
- * Uses directory-level entries where possible so the worker config stays compact
- * and the folder upload/download feature handles individual files automatically.
+ * Collect all files from the .knowhow directory that should be synced.
+ * Only includes files/directories that currently exist locally.
+ * Used by --init to populate config.files.
  */
 async function collectFilesToSync(projectName: string): Promise<FileToSync[]> {
   const filesToSync: FileToSync[] = [];
 
-  // Helper to add file if it exists
   const addIfExists = (localPath: string, remotePath: string) => {
     if (fs.existsSync(localPath)) {
       filesToSync.push({ localPath, remotePath });
     }
   };
 
-  // Helper to add a directory entry if it exists (trailing slash = directory mode)
   const addDirIfExists = (localPath: string, remotePath: string) => {
     if (fs.existsSync(localPath)) {
       filesToSync.push({ localPath: localPath + "/", remotePath: remotePath + "/", isDirectory: true });
@@ -108,7 +88,9 @@ async function collectFilesToSync(projectName: string): Promise<FileToSync[]> {
 }
 
 /**
- * Collect files referenced in language.json sources
+ * Collect files referenced in language.json sources.
+ * These are always re-collected on both --init and --push so that new
+ * language term sources are picked up automatically.
  */
 async function collectLanguageReferencedFiles(
   language: Language,
@@ -124,17 +106,14 @@ async function collectLanguageReferencedFiles(
       if (source.kind !== "file" || !source.data) continue;
 
       for (const filePath of source.data) {
-        // Normalize the path (strip leading ./)
         const normalizedPath = filePath.replace(/^\.\//, "");
 
         // Skip the main knowhow config — it should not be synced to the language folder
-        // as it would overwrite the worker's own config
         if (normalizedPath === ".knowhow/knowhow.json") continue;
 
         if (fs.existsSync(normalizedPath)) {
           const basename = path.basename(normalizedPath);
           const remotePath = `${projectName}/.knowhow/language/${basename}`;
-          // localPath is the original path so the worker downloads it to the right place
           filesToSync.push({ localPath: normalizedPath, remotePath, downloadLocalPath: normalizedPath });
         }
       }
@@ -145,37 +124,83 @@ async function collectLanguageReferencedFiles(
 }
 
 /**
- * Upload a single file to the cloud worker's file storage
+ * Collect language-referenced files if language.json is present in the
+ * given config.files entries. Returns empty array if language.json is not
+ * configured for sync.
+ */
+async function collectLanguageFilesIfConfigured(
+  configFiles: { remotePath: string; localPath: string }[],
+  projectName: string
+): Promise<FileToSync[]> {
+  const syncingLanguage = configFiles.some(
+    (f) => !f.remotePath.endsWith("/") && f.remotePath.endsWith("language.json")
+  );
+  if (!syncingLanguage) return [];
+
+  const language = await getLanguageConfig();
+  return collectLanguageReferencedFiles(language, projectName);
+}
+
+/**
+ * Initialize the local config.files entries based on what exists in .knowhow/.
+ * This is the --init step — mutates config. Run once to set up sync entries.
+ * language-referenced files are also collected if language.json is present.
  */
-async function uploadSingleFile(
-  client: KnowhowSimpleClient,
-  s3Service: S3Service,
-  localPath: string,
-  remotePath: string,
-  dryRun: boolean
-): Promise<void> {
-  console.log(`  ⬆️  Uploading ${localPath} → ${remotePath}`);
-
-  if (dryRun) {
-    console.log(`     [DRY RUN] Would upload from ${localPath}`);
-    return;
+export async function initCloudWorker(options: { apiUrl?: string; dryRun?: boolean } = {}) {
+  const { dryRun = false } = options;
+
+  const config = await getConfig();
+  if (!config || Object.keys(config).length === 0) {
+    console.error("❌ No knowhow config found. Please run 'knowhow init' first.");
+    process.exit(1);
   }
 
-  if (!fs.existsSync(localPath)) {
-    console.warn(`     ⚠️  Local file not found, skipping: ${localPath}`);
-    return;
+  const projectName = path.basename(process.cwd());
+  console.log(`📁 Project name: ${projectName}`);
+
+  console.log("\n📂 Collecting files to sync...");
+  const mainFiles = await collectFilesToSync(projectName);
+  const languageFiles = await collectLanguageFilesIfConfigured(mainFiles, projectName);
+
+  if (languageFiles.length === 0 && !mainFiles.some((f) => f.remotePath.endsWith("language.json"))) {
+    console.log("   ℹ️  Skipping language-referenced files (language.json not found locally)");
   }
 
-  const presignedUrl = await client.getOrgFilePresignedUploadUrl(remotePath);
-  await s3Service.uploadToPresignedUrl(presignedUrl, localPath);
-  await client.markOrgFileUploadComplete(remotePath);
+  // Deduplicate by remotePath
+  const allFilesMap = new Map<string, FileToSync>();
+  for (const f of [...mainFiles, ...languageFiles]) {
+    allFilesMap.set(f.remotePath, f);
+  }
+  const allFiles = Array.from(allFilesMap.values());
+
+  console.log(`   Found ${allFiles.length} files to register`);
 
-  const stats = fs.statSync(localPath);
-  console.log(`     ✓ Uploaded ${stats.size} bytes`);
+  const configFilesEntries = allFiles.map((f) => ({
+    remotePath: f.remotePath,
+    localPath: f.downloadLocalPath ?? f.localPath,
+    direction: "download" as const,
+  }));
+
+  console.log("\n💾 Updating config.files with sync entries...");
+  if (!dryRun) {
+    const existingFiles = config.files || [];
+    const newRemotePaths = new Set(configFilesEntries.map((e) => e.remotePath));
+    const preserved = existingFiles.filter((e) => !newRemotePaths.has(e.remotePath));
+    config.files = [...preserved, ...configFilesEntries];
+    await updateConfig(config);
+    console.log(`   ✓ Updated config with ${config.files.length} file entries`);
+  } else {
+    console.log(`   [DRY RUN] Would update config with ${configFilesEntries.length} file entries`);
+    for (const f of allFiles) {
+      console.log(`   ${f.localPath} → ${f.remotePath}`);
+    }
+  }
 }
 
 /**
- * Main cloudWorker command handler
+ * Main cloudWorker command handler — push/create only.
+ * Reads config.files (set up by --init) and also re-collects any language-referenced
+ * files so new language term sources are always included without requiring --init again.
  */
 export async function cloudWorker(options: CloudWorkerOptions) {
   const {
@@ -205,10 +230,6 @@ export async function cloudWorker(options: CloudWorkerOptions) {
     process.exit(1);
   }
 
-  // Load language config
-  const language = await getLanguageConfig();
-
-  // Get project name from current directory
   const projectName = path.basename(process.cwd());
   console.log(`📁 Project name: ${projectName}`);
 
@@ -218,86 +239,63 @@ export async function cloudWorker(options: CloudWorkerOptions) {
   // Get S3 service
   const { AwsS3 } = services();
 
-  // Step 1: Collect all files to sync
-  console.log("\n📂 Collecting files to sync...");
-  const mainFiles = await collectFilesToSync(projectName);
-  const languageFiles = await collectLanguageReferencedFiles(language, projectName);
-
-  // Deduplicate by remotePath
-  const allFilesMap = new Map<string, FileToSync>();
-  for (const f of [...mainFiles, ...languageFiles]) {
-    allFilesMap.set(f.remotePath, f);
+  // Start with config.files (set up via --init)
+  const configFiles = config.files || [];
+  if (configFiles.length === 0) {
+    console.warn("⚠️  No files configured. Run 'knowhow cloudworker --init' first to set up file sync entries.");
   }
-  const allFiles = Array.from(allFilesMap.values());
-
-  console.log(`   Found ${allFiles.length} files to sync`);
 
-  if (dryRun) {
-    console.log("\n📋 Files that would be synced:");
-    for (const f of allFiles) {
-      console.log(`   ${f.localPath} → ${f.remotePath}`);
-    }
+  // Re-collect language-referenced files on every push (if language.json is in config.files)
+  // so that new language term sources are picked up without needing --init again.
+  const languageFiles = await collectLanguageFilesIfConfigured(configFiles, projectName);
+  if (languageFiles.length > 0) {
+    console.log(`   + ${languageFiles.length} language-referenced file(s) to sync`);
   }
 
-  // Step 2: Build the config.files array for all synced files
-  const configFilesEntries = allFiles.map((f) => ({
-    remotePath: f.remotePath,
-    localPath: f.downloadLocalPath ?? f.localPath,
-    direction: "download" as const,
-  }));
-
-  // Step 3: Update config.files and save
-  console.log("\n💾 Updating config.files with sync entries...");
-  if (!dryRun) {
-    // Preserve any existing files entries not in our set
-    const existingFiles = config.files || [];
-    const newRemotePaths = new Set(configFilesEntries.map((e) => e.remotePath));
-
-    // Keep entries that don't overlap with new ones
-    const preserved = existingFiles.filter(
-      (e) => !newRemotePaths.has(e.remotePath)
-    );
+  // Merge language files into the upload list (deduplicate by remotePath)
+  const allFilesMap = new Map<string, { remotePath: string; localPath: string }>();
+  for (const f of configFiles) {
+    allFilesMap.set(f.remotePath, f);
+  }
+  for (const f of languageFiles) {
+    const entry = { remotePath: f.remotePath, localPath: f.downloadLocalPath ?? f.localPath };
+    allFilesMap.set(f.remotePath, entry);
+  }
+  const allFiles = Array.from(allFilesMap.values());
 
-    config.files = [...preserved, ...configFilesEntries];
+  // If new language files were found, update config.files so they persist
+  if (languageFiles.length > 0 && !dryRun) {
+    config.files = allFiles.map((f) => ({ ...f, direction: "download" as const }));
     await updateConfig(config);
-    console.log(`   ✓ Updated config with ${config.files.length} file entries`);
-  } else {
-    console.log(`   [DRY RUN] Would update config with ${configFilesEntries.length} file entries`);
   }
 
-  // Step 4: Build workerConfigJson
-  const workerConfigJson = buildWorkerConfigJson(config, configFilesEntries);
+  // Build the workerConfigJson using the full file list
+  const workerConfigJson = buildWorkerConfigJson(config, allFiles.map((f) => ({ ...f, direction: "download" })));
 
-  // Step 5: Upload all files
-  console.log(`\n🚀 Uploading ${allFiles.length} files...`);
+  // Upload all files
+  console.log(`\n🚀 Uploading ${allFiles.length} configured files...`);
   let successCount = 0;
   let failCount = 0;
 
-  for (const file of allFiles) {
+  for (const mount of allFiles) {
+    const { remotePath, localPath } = mount;
     try {
-      if (file.isDirectory) {
-        // Upload all files recursively in the local directory
-        const localDir = file.localPath.endsWith("/") ? file.localPath : file.localPath + "/";
-        const remoteDir = file.remotePath.endsWith("/") ? file.remotePath : file.remotePath + "/";
-        const relFiles = listFilesRecursively(localDir);
-        console.log(`  📁 Uploading directory ${localDir} → ${remoteDir} (${relFiles.length} files)`);
-        for (const relFile of relFiles) {
-          await uploadSingleFile(client, AwsS3, localDir + relFile, remoteDir + relFile, dryRun);
-          successCount++;
-        }
+      if (remotePath.endsWith("/") || localPath.endsWith("/")) {
+        const count = await uploadDirectory(client, AwsS3, remotePath, localPath, dryRun);
+        successCount += count;
       } else {
-        await uploadSingleFile(client, AwsS3, file.localPath, file.remotePath, dryRun);
+        await uploadFile(client, AwsS3, remotePath, localPath, dryRun);
         successCount++;
       }
     } catch (error) {
-      console.error(`  ❌ Failed to upload ${file.localPath}: ${error.message}`);
+      console.error(`  ❌ Failed to upload ${localPath}: ${error.message}`);
       failCount++;
     }
   }
 
   console.log(`\n   ✓ Upload complete: ${successCount} succeeded, ${failCount} failed`);
 
-  // Step 6: Create or update cloud worker
+  // Create or update cloud worker
   if (create) {
     const workerName = name || `${projectName}-worker`;
     console.log(`\n🌩️  Creating cloud worker "${workerName}"...`);
@@ -339,16 +337,6 @@ export async function cloudWorker(options: CloudWorkerOptions) {
 /**
  * Pull the latest workerConfigJson from the cloud worker API and update the
  * local knowhow.json config to match.
- *
- * This is the "pull" half of the config sync cycle.  After running this,
- * you can reload the worker's MCPs (in-process) via the reloadConfig
- * WebSocket message or by calling `knowhow worker` again.
- *
- * Merged fields from workerConfigJson:
- *   - mcps        → overwrites config.mcps
- *   - modules     → overwrites config.modules  (optional, only if present)
- *   - plugins     → overwrites config.plugins  (optional, only if present)
- *   - agents      → overwrites config.agents   (optional, only if present)
  */
 export async function pullCloudWorkerConfig(options: CloudWorkerPullOptions) {
   const { id, apiUrl = KNOWHOW_API_URL } = options;

package/src/commands/misc.ts

@@ -1,6 +1,7 @@
 import { Command } from "commander";
 import { execSync } from "child_process";
 import { version } from "../../package.json";
+import { logger } from "../logger";
 import { generate, embed, upload, download, purge } from "../index";
 import { init } from "../config";
 import { login } from "../login";
@@ -118,6 +119,10 @@ export function addGithubCredentialsCommand(program: Command): void {
       "Repository in owner/repo format (e.g. myorg/myrepo)"
     )
     .action(async (action: string | undefined, options: { repo?: string }) => {
+      // Silence ALL output immediately — git credential helpers must produce
+      // only the protocol=.../host=.../username=.../password=... lines on stdout.
+      logger.silence();
+
       const client = new KnowhowSimpleClient();
 
       let repo = options.repo;

package/src/commands/services.ts

@@ -15,8 +15,12 @@ export async function setupServices() {
     Tools: AllTools,
     Embeddings,
     Plugins,
+    Events,
     MediaProcessor,
   } = services();
+
+
+  // cli uses LazyTools to keep context slim
   const Tools = new LazyToolsService();
 
   Tools.setContext({
@@ -66,6 +70,7 @@ export async function setupServices() {
     Clients,
     Tools,
     MediaProcessor,
+    Events
   });
 
   return { Tools, Clients };

package/src/commands/workers.ts

@@ -128,6 +128,10 @@ export function addCloudWorkerCommand(program: Command): void {
   program
     .command("cloudworker")
     .description("Create or sync a cloud worker with your local knowhow config")
+    .option(
+      "--init",
+      "Initialize config.files entries based on what exists in .knowhow/ (run once before --push)"
+    )
     .option(
       "--create",
       "Create a new cloud worker with synced config and files"
@@ -144,9 +148,13 @@ export function addCloudWorkerCommand(program: Command): void {
     .option("--dry-run", "Print what would be synced without doing it")
     .action(async (options) => {
       try {
-        const { cloudWorker, pullCloudWorkerConfig } = await import(
+        const { cloudWorker, pullCloudWorkerConfig, initCloudWorker } = await import(
           "../cloudWorker"
         );
+        if (options.init) {
+          await initCloudWorker({ dryRun: options.dryRun });
+          return;
+        }
         if (options.pull) {
           await pullCloudWorkerConfig({ id: options.pull });
         } else {

package/src/fileSync.ts

@@ -1,5 +1,6 @@
 import * as fs from "fs";
 import * as path from "path";
+import * as os from "os";
 import { KnowhowSimpleClient, KNOWHOW_API_URL } from "./services/KnowhowClient";
 import { loadJwt } from "./login";
 import { getConfig } from "./config";
@@ -7,6 +8,8 @@ import { services } from "./services";
 import { S3Service } from "./services/S3";
 import { getHashes, hasFileChangedSinceUpload, saveUploadHash, isLocalFileMatchingRemote, isLocalFileMatchingDownloadHash, saveDownloadHash } from "./hashes";
 
+export const DEFAULT_BATCH_SIZE = 5;
+
 export interface FileSyncOptions {
   upload?: boolean;
   download?: boolean;
@@ -15,6 +18,33 @@ export interface FileSyncOptions {
   dryRun?: boolean;
 }
 
+/**
+ * Run an array of async tasks in batches of `batchSize` at a time.
+ * Returns results in the same order as the input tasks.
+ */
+export async function batchRun<T>(
+  tasks: (() => Promise<T>)[],
+  batchSize: number = DEFAULT_BATCH_SIZE
+): Promise<T[]> {
+  const results: T[] = [];
+  for (let i = 0; i < tasks.length; i += batchSize) {
+    const batch = tasks.slice(i, i + batchSize);
+    const batchResults = await Promise.all(batch.map((t) => t()));
+    results.push(...batchResults);
+  }
+  return results;
+}
+
+/**
+ * Expands a leading ~ to the user's home directory
+ */
+function expandHome(p: string): string {
+  if (p === "~" || p.startsWith("~/") || p.startsWith("~\\")) {
+    return path.join(os.homedir(), p.slice(1));
+  }
+  return p;
+}
+
 /**
  * Returns true if the path looks like a directory (ends with /)
  */
@@ -25,7 +55,7 @@ function isDirectoryPath(p: string): boolean {
 /**
  * Recursively list all files in a local directory, returning relative paths
  */
-function listFilesRecursively(dir: string): string[] {
+export function listFilesRecursively(dir: string): string[] {
   const results: string[] = [];
   if (!fs.existsSync(dir)) return results;
   const entries = fs.readdirSync(dir, { withFileTypes: true });
@@ -83,7 +113,8 @@ export async function fileSync(options: FileSyncOptions = {}) {
 
   // Process each file mount
   for (const mount of config.files) {
-    const { remotePath, localPath, direction = "download" } = mount;
+    const { remotePath, localPath: rawLocalPath, direction = "download" } = mount;
+    const localPath = expandHome(rawLocalPath);
 
     // Determine actual direction based on flags and config
     let actualDirection = direction;
@@ -126,11 +157,10 @@ export async function fileSync(options: FileSyncOptions = {}) {
   }
 }
 
-
 /**
  * Download a file from Knowhow FS to local filesystem
  */
-async function downloadFile(
+export async function downloadFile(
   client: KnowhowSimpleClient,
   s3Service: S3Service,
   remotePath: string,
@@ -186,7 +216,7 @@ async function downloadFile(
 /**
  * Upload a file from local filesystem to Knowhow FS
  */
-async function uploadFile(
+export async function uploadFile(
   client: KnowhowSimpleClient,
   s3Service: S3Service,
   remotePath: string,
@@ -215,7 +245,7 @@ async function uploadFile(
   }
 
   // Get presigned upload URL
-  const presignedUrl = await client.getOrgFilePresignedUploadUrl(remotePath);
+  const presignedUrl = await client.getOrgFilePresignedUploadUrl(remotePath, localPath);
 
   // Upload file using presigned URL
   await s3Service.uploadToPresignedUrl(presignedUrl, localPath);
@@ -261,26 +291,28 @@ export async function uploadDirectory(
 
   console.log(`   Found ${localFiles.length} local file(s)`);
 
-  let count = 0;
-  for (const relFile of localFiles) {
+  const tasks = localFiles.map((relFile) => async () => {
     const localFilePath = localDir + relFile;
     const remoteFilePath = remoteDir + relFile;
     try {
       await uploadFile(client, s3Service, remoteFilePath, localFilePath, dryRun);
-      count++;
+      return 1;
     } catch (error) {
       console.error(
         `   ❌ Failed to upload ${localFilePath}, skipping: ${error.message}`
       );
+      return 0;
     }
-  }
-  return count;
+  });
+
+  const counts = await batchRun(tasks);
+  return counts.reduce((sum, n) => sum + n, 0);
 }
 
 /**
  * Download all files from a remote directory path to a local directory
  */
-async function downloadDirectory(
+export async function downloadDirectory(
   client: KnowhowSimpleClient,
   s3Service: S3Service,
   remotePath: string,
@@ -313,8 +345,7 @@ async function downloadDirectory(
 
   console.log(`   Found ${matchingFiles.length} remote file(s)`);
 
-  let count = 0;
-  for (const f of matchingFiles) {
+  const tasks = matchingFiles.map((f) => async () => {
     const fullRemotePath = f.folderPath.endsWith("/")
       ? f.folderPath + f.fileName
       : f.folderPath + "/" + f.fileName;
@@ -322,7 +353,9 @@ async function downloadDirectory(
     const relativePath = fullRemotePath.slice(remoteDir.length);
     const localFilePath = localDir + relativePath;
     await downloadFile(client, s3Service, fullRemotePath, localFilePath, dryRun);
-    count++;
-  }
-  return count;
+    return 1;
+  });
+
+  const counts = await batchRun(tasks);
+  return counts.reduce((sum, n) => sum + n, 0);
 }