@tyvm/knowhow 0.0.108-dev.126b29e → 0.0.108-dev.4a8ba55

package/src/commands/services.ts

@@ -0,0 +1,72 @@
+import { includedTools } from "../agents/tools/list";
+import * as allTools from "../agents/tools";
+import { LazyToolsService, services } from "../services";
+import { agents } from "../agents";
+import { ModulesService } from "../services/modules";
+
+/**
+ * Shared service setup used by commands that need full services (chat, agent, worker, etc.)
+ */
+export async function setupServices() {
+  const {
+    Agents,
+    Mcp,
+    Clients,
+    Tools: AllTools,
+    Embeddings,
+    Plugins,
+    MediaProcessor,
+  } = services();
+  const Tools = new LazyToolsService();
+
+  Tools.setContext({
+    ...AllTools.getContext(),
+  });
+
+  const agentContext: import("../agents/base/base").AgentContext = {
+    ...services(),
+    Tools,
+  };
+
+  const { Researcher, Developer, Patcher, Setup } = agents({
+    ...agentContext,
+  });
+
+  Agents.registerAgent(Researcher);
+  Agents.registerAgent(Patcher);
+  Agents.registerAgent(Developer);
+  Agents.registerAgent(Setup);
+  Agents.loadAgentsFromConfig(agentContext);
+
+  Tools.defineTools(includedTools, allTools);
+
+  Tools.addContext("Mcp", Mcp);
+
+  Agents.setAgentContext(agentContext);
+
+  console.log("🔌 Connecting to MCP...");
+  try {
+    await Mcp.connectToConfigured(Tools);
+  } catch (mcpError) {
+    const msg = mcpError instanceof Error ? mcpError.message : String(mcpError);
+    console.warn(
+      `⚠ Some MCP servers failed to connect (continuing without them): ${msg}`
+    );
+  }
+  console.log("Connecting to clients...");
+  await Clients.registerConfiguredModels();
+  console.log("✓ Services are set up and ready to go!");
+
+  console.log("📦 Loading modules from config...");
+  const modulesService = new ModulesService();
+  await modulesService.loadModulesFromConfig({
+    Agents,
+    Embeddings,
+    Plugins,
+    Clients,
+    Tools,
+    MediaProcessor,
+  });
+
+  return { Tools, Clients };
+}

package/src/commands/workers.ts

@@ -0,0 +1,160 @@
+import { Command } from "commander";
+import { worker } from "../worker";
+import { TUNNEL_MINIMAL_TOOLS } from "../tunnel";
+import { fileSync } from "../fileSync";
+import {
+  startAllWorkers,
+  listWorkerPaths,
+  unregisterWorkerPath,
+  clearWorkerRegistry,
+} from "../workerRegistry";
+
+export function addWorkerCommand(program: Command): void {
+  program
+    .command("worker")
+    .description(
+      "Start worker process and optionally register current directory"
+    )
+    .option("--register", "Register current directory as a worker path")
+    .option(
+      "--share",
+      "Share this worker with your organization (allows other users to use it)"
+    )
+    .option("--unshare", "Make this worker private (only you can use it)")
+    .option("--sandbox", "Run worker in a Docker container for isolation")
+    .option(
+      "--no-sandbox",
+      "Run worker directly on host (disable sandbox mode)"
+    )
+    .option("--passkey", "Set up passkey authentication for this worker")
+    .option("--passkey-reset", "Remove passkey authentication requirement")
+    .action(async (options) => {
+      const { setupServices } = await import("./services");
+      await setupServices();
+      await worker(options);
+    });
+}
+
+export function addWorkersCommand(program: Command): void {
+  program
+    .command("workers")
+    .description("Manage and start all registered workers")
+    .option("--list", "List all registered worker paths")
+    .option("--unregister <path>", "Unregister a worker path")
+    .option("--clear", "Clear all registered worker paths")
+    .action(async (options) => {
+      try {
+        if (options.list) {
+          const workers = await listWorkerPaths();
+          if (workers.length === 0) {
+            console.log("No workers registered.");
+            console.log(
+              "\nTo register a worker, run 'knowhow worker --register' from the worker directory."
+            );
+          } else {
+            console.log(`Registered workers (${workers.length}):`);
+            workers.forEach((workerPath, index) => {
+              console.log(`  ${index + 1}. ${workerPath}`);
+            });
+          }
+          return;
+        }
+
+        if (options.unregister) {
+          await unregisterWorkerPath(options.unregister);
+          return;
+        }
+
+        if (options.clear) {
+          await clearWorkerRegistry();
+          return;
+        }
+
+        // Default action: start all workers
+        const { setupServices } = await import("./services");
+        await setupServices();
+        await startAllWorkers();
+      } catch (error) {
+        console.error("Error managing workers:", error);
+        process.exit(1);
+      }
+    });
+}
+
+export function addTunnelCommand(program: Command): void {
+  program
+    .command("tunnel")
+    .description(
+      "Start a minimal worker with tunnel enabled: exposes local ports to the cloud. " +
+      "Registers essential tools (unlock, lock, listAllowedPorts) so the backend is aware of the worker and ports. " +
+      "If passkey auth is configured, the tunnel is locked until unlocked via tool call or WebSocket auth protocol."
+    )
+    .option(
+      "--share",
+      "Share this tunnel with your organization (allows other users to use it)"
+    )
+    .option("--unshare", "Make this tunnel private (only you can use it)")
+    .action(async (options) => {
+      console.log("🌐 Starting tunnel (minimal worker) mode...");
+      console.log(`   Tools: ${TUNNEL_MINIMAL_TOOLS.join(", ")}`);
+      await worker({
+        ...options,
+        allowedTools: TUNNEL_MINIMAL_TOOLS,
+      });
+    });
+}
+
+export function addFilesCommand(program: Command): void {
+  program
+    .command("files")
+    .description(
+      "Sync files between local filesystem and Knowhow FS (uses fileMounts config)"
+    )
+    .option("--upload", "Force upload direction for all mounts")
+    .option("--download", "Force download direction for all mounts")
+    .option("--config <path>", "Path to knowhow.json", "./knowhow.json")
+    .option("--dry-run", "Print what would be synced without doing it")
+    .action(async (options) => {
+      try {
+        await fileSync(options);
+      } catch (error) {
+        console.error("Error syncing files:", error);
+        process.exit(1);
+      }
+    });
+}
+
+export function addCloudWorkerCommand(program: Command): void {
+  program
+    .command("cloudworker")
+    .description("Create or sync a cloud worker with your local knowhow config")
+    .option(
+      "--create",
+      "Create a new cloud worker with synced config and files"
+    )
+    .option(
+      "--push <uid>",
+      "Push/sync local config and files to an existing cloud worker"
+    )
+    .option(
+      "--pull <id>",
+      "Pull the latest workerConfigJson from a cloud worker and update local config"
+    )
+    .option("--name <name>", "Name for the cloud worker (used with --create)")
+    .option("--dry-run", "Print what would be synced without doing it")
+    .action(async (options) => {
+      try {
+        const { cloudWorker, pullCloudWorkerConfig } = await import(
+          "../cloudWorker"
+        );
+        if (options.pull) {
+          await pullCloudWorkerConfig({ id: options.pull });
+        } else {
+          await cloudWorker(options);
+        }
+      } catch (error) {
+        console.error("Error running cloudworker:", error);
+        process.exit(1);
+      }
+    });
+}

package/src/config.ts

@@ -168,6 +168,27 @@ export async function init() {
   console.log("Initializing global knowhow config at ~/.knowhow");
   const globalConfigDir = await ensureGlobalConfigDir();
 
+  // Ensure the script module is registered in the global config so that
+  // `knowhow script` and the `executeScript` tool are available everywhere.
+  const SCRIPT_MODULE = "@tyvm/knowhow-module-script";
+  const globalConfigPath = path.join(globalConfigDir, "knowhow.json");
+  try {
+    const rawGlobal = fs.existsSync(globalConfigPath)
+      ? fs.readFileSync(globalConfigPath, "utf8")
+      : JSON.stringify(defaultConfig, null, 2);
+    const globalConf = JSON.parse(rawGlobal) as Config;
+    if (!globalConf.modules) {
+      globalConf.modules = [];
+    }
+    if (!globalConf.modules.includes(SCRIPT_MODULE)) {
+      globalConf.modules.push(SCRIPT_MODULE);
+      fs.writeFileSync(globalConfigPath, JSON.stringify(globalConf, null, 2));
+      console.log(`✅ Added ${SCRIPT_MODULE} to ~/.knowhow/knowhow.json modules`);
+    }
+  } catch (e) {
+    console.warn(`⚠ Could not update global config to add script module:`, e);
+  }
+
   // create the folder structure
   console.log("Initializing local knowhow config at ./.knowhow");
   await mkdir(".knowhow", { recursive: true });
@@ -287,6 +308,22 @@ export async function getGlobalConfig(): Promise<Config> {
   }
 }
 
+export async function updateGlobalConfig(config: Config) {
+  if (!config || typeof config !== "object") {
+    throw new Error("Invalid config object");
+  }
+
+  const globalConfigDir = getGlobalConfigDir();
+  await mkdir(globalConfigDir, { recursive: true });
+  const globalConfigPath = path.join(globalConfigDir, "knowhow.json");
+
+  if (fs.existsSync(globalConfigPath)) {
+    await fs.promises.copyFile(globalConfigPath, globalConfigPath + ".bak");
+  }
+
+  await writeFile(globalConfigPath, JSON.stringify(config, null, 2));
+}
+
 export async function migrateConfig() {
   // Apply migrations, used to keep config structure up to date.
   if (!fs.existsSync(".knowhow/knowhow.json")) {

package/src/index.ts

@@ -53,6 +53,7 @@ export * as ai from "./ai";
 
 // Export module system types for external modules
 export * from "./services/modules/types";
+export { ModulesService } from "./services/modules";
 // Export plugin types for external plugins
 export { PluginBase } from "./plugins/PluginBase";
 export { PluginMeta, Plugin, PluginContext } from "./plugins/types";

package/src/plugins/plugins.ts

@@ -1,5 +1,4 @@
 import { Plugin, PluginContext } from "./types";
-import { Config } from "../types";
 import { VimPlugin } from "./vim";
 import { LinterPlugin } from "./LinterPlugin";
 import { LanguagePlugin } from "./language";
@@ -49,26 +48,6 @@ export class PluginService {
     return instance.meta.key;
   }
 
-  /**
-   * Load plugins from config's pluginPackages map.
-   * Each entry maps a plugin key to an npm package name or file path.
-   * Errors are caught and logged as warnings without crashing.
-   */
-  async loadPluginsFromConfig(config: Config): Promise<void> {
-    const pluginPackages = config.pluginPackages || {};
-    for (const [key, spec] of Object.entries(pluginPackages)) {
-      try {
-        await this.loadPlugin(spec);
-      } catch (error) {
-        this.events?.log(
-          "PluginService",
-          `Failed to load plugin "${key}" from "${spec}": ${error instanceof Error ? error.message : error}`,
-          "warn"
-        );
-      }
-    }
-  }
-
   /** Disable a plugin by its key; returns `true` if found. */
   disablePlugin(key: string): boolean {
     const p = this.pluginMap.get(key);

package/src/processors/JsonCompressor.ts

@@ -382,7 +382,7 @@ export class JsonCompressor {
             i + currentChunk.length - 1
           }]\nPreview: ${chunkString.substring(0, 100)}...\n[Use ${
             this.toolName
-          } tool with key "${key}" to retrieve this chunk]`;
+          } tool with key "${key}" to retrieve this chunk]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
           finalArray.unshift(stub); // Add stub to the start of our final result.
 
           currentChunk = [];
@@ -415,11 +415,11 @@ export class JsonCompressor {
       
       // Store objects on FIRST occurrence so second occurrence can reference it
       // We increment seenCount above, so after increment:
-      // seenCount=1: first occurrence (just incremented from 0 to 1), store it
-      // seenCount>=2: we already stored it on first occurrence, should be in dedup map
+      // seenCount=0: first occurrence (before increment), store it
+      // seenCount>=1: we already stored it on first occurrence, should be in dedup map
       // Note: This means we store proactively - first occurrence gets stored AND returned in full
       // Second+ occurrences will find it in the dedup map and return a reference
-      const isFirstOccurrence = seenCount === 1;
+      const isFirstOccurrence = seenCount === 0;
 
       // Process the object - apply low-signal detection
       const objWithLowSignalCompressed = this.compressObjectWithLowSignalDetection(obj, path);
@@ -453,7 +453,7 @@ export class JsonCompressor {
           result
         ).join(", ")}\nPreview: ${objectAsString.substring(0, 200)}...\n[Use ${
           this.toolName
-        } tool with key "${key}" to retrieve full content]`;
+        } tool with key "${key}" to retrieve full content]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
       }
       return result;
     }
@@ -486,7 +486,7 @@ export class JsonCompressor {
           200
         )}...\n[Use ${
           this.toolName
-        } tool with key "${key}" to retrieve full content]`;
+        } tool with key "${key}" to retrieve full content]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
       }
       return obj;
     }

package/src/services/modules/index.ts

@@ -1,38 +1,28 @@
 import { getConfig, getGlobalConfig } from "../../config";
 import { KnowhowModule, ModuleContext } from "./types";
-import { ToolsService } from "../Tools";
 import { services } from "../";
 import * as path from "path";
 
 export class ModulesService {
-  async loadModulesFromConfig(context?: ModuleContext) {
-    const config = await getConfig();
+  async getDefaultContext() {
+    return { ...services() };
+  }
 
+  async overrideDefaultContext(overrides: Partial<ModuleContext>) {
+    const defaultContext = await this.getDefaultContext();
+    return { ...defaultContext, ...overrides };
+  }
+
+  async loadModulesFrom(
+    config: { modules: string[] } & any,
+    context?: Partial<ModuleContext>
+  ) {
     // If no context provided, fall back to global singletons
     if (!context) {
-      const { Clients, Plugins, Agents, Tools, Embeddings, MediaProcessor } = services();
-      context = {
-        Agents,
-        Embeddings,
-        Plugins,
-        Clients,
-        Tools,
-        MediaProcessor,
-      };
+      context = { ...(await this.getDefaultContext()) };
     }
 
-    // Use the toolsService from context
-    const toolsService = context.Tools;
-    const agentService = context.Agents;
-    const pluginService = context.Plugins;
-    const clients = context.Clients;
-
-    // Load from global config (~/.knowhow/knowhow.json) first, then local config
-    const globalConfig = await getGlobalConfig();
-    const allModulePaths = [
-      ...(globalConfig.modules || []),
-      ...(config.modules || []),
-    ];
+    const allModulePaths = config.modules;
 
     for (const modulePath of allModulePaths) {
       // Resolve relative paths relative to process.cwd() so that paths like
@@ -43,40 +33,59 @@ export class ModulesService {
         : modulePath;
       const rawModule = require(resolvedPath);
       const importedModule = (rawModule.default || rawModule) as KnowhowModule;
-      console.log(`🔌 Loading module: ${modulePath} (resolved: ${resolvedPath})`);
-      await importedModule.init({ config, cwd: process.cwd(), context });
-      console.log(`✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`);
+      console.log(
+        `🔌 Loading module: ${modulePath} (resolved: ${resolvedPath})`
+      );
+      await importedModule.init({ config, cwd: process.cwd(), context: context as ModuleContext });
+      console.log(
+        `✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`
+      );
 
-      for (const agent of importedModule.agents) {
-        agentService.registerAgent(agent);
+      // Only register tools/agents/plugins/clients if the relevant services
+      // are available in context (they may not be during early CLI command registration)
+      if (context.Agents) {
+        for (const agent of importedModule.agents) {
+          context.Agents.registerAgent(agent);
+        }
       }
 
-      for (const tool of importedModule.tools) {
-        toolsService.addTool(tool.definition);
-        toolsService.setFunction(tool.definition.function.name, tool.handler);
+      if (context.Tools) {
+        for (const tool of importedModule.tools) {
+          context.Tools.addTool(tool.definition);
+          context.Tools.setFunction(tool.definition.function.name, tool.handler);
+        }
       }
 
-      for (const plugin of importedModule.plugins) {
-        const pluginContext = {
-          Agents: agentService,
-          Clients: clients,
-          Tools: toolsService,
-          Plugins: pluginService,
-          ...(context.MediaProcessor ? { MediaProcessor: context.MediaProcessor } : {}),
-        };
-        pluginService.registerPlugin(plugin.name, new plugin.plugin(pluginContext as any));
+      if (context.Plugins) {
+        for (const plugin of importedModule.plugins) {
+          context.Plugins.registerPlugin(
+            plugin.name,
+            new plugin.plugin(context as any)
+          );
+        }
       }
 
-      for (const client of importedModule.clients) {
-        clients.registerClient(client.provider, client.client);
-        clients.registerModels(client.provider, client.models);
+      if (context.Clients) {
+        for (const client of importedModule.clients) {
+          context.Clients.registerClient(client.provider, client.client);
+          context.Clients.registerModels(client.provider, client.models);
+        }
       }
     }
+  }
 
-    // Also load plugins directly from config's pluginPackages map
-    if (pluginService) {
-      await pluginService.loadPluginsFromConfig(config);
-      await pluginService.loadPluginsFromConfig(globalConfig);
-    }
+  async loadModulesFromConfig(context?: ModuleContext) {
+    const config = await getConfig();
+
+    const globalConfig = await getGlobalConfig();
+    const allModulePaths = [
+      ...(globalConfig.modules || []),
+      ...(config.modules || []),
+    ];
+
+    return this.loadModulesFrom(
+      { ...config, modules: allModulePaths },
+      context
+    );
   }
 }

package/src/services/modules/types.ts

@@ -1,4 +1,5 @@
 import { Plugin, PluginContext } from "../../plugins/types";
+import { Command } from "commander";
 import { IAgent } from "../../agents/interface";
 import { Tool } from "../../clients/types";
 import { Config } from "../../types";
@@ -55,6 +56,7 @@ export interface ModuleContext {
   Tools: ToolsService;
   MediaProcessor?: MediaProcessorService;
   Tunnel?: TunnelHandler;
+  Program?: Command;
 }
 
 export interface KnowhowModule {