@tyvm/knowhow 0.0.107 → 0.0.108-dev.4a8ba55

package/src/services/S3.ts

@@ -1,5 +1,6 @@
 import * as fs from "fs";
 import { createWriteStream, createReadStream } from "fs";
+import * as crypto from "crypto";
 import { pipeline, Readable } from "stream";
 import * as util from "util";
 
@@ -14,10 +15,19 @@ export class S3Service {
       const fileContent = fs.readFileSync(filePath);
       const fileStats = await fs.promises.stat(filePath);
 
+      // Compute SHA-256 checksum (base64) — required when presigned URL was
+      // generated with ChecksumAlgorithm: SHA256
+      const sha256Base64 = crypto
+        .createHash("sha256")
+        .update(fileContent)
+        .digest("base64");
+
       const response = await fetch(presignedUrl, {
         method: "PUT",
         headers: {
           "Content-Length": String(fileStats.size),
+          "x-amz-checksum-sha256": sha256Base64,
+          "x-amz-sdk-checksum-algorithm": "SHA256",
         },
         body: fileContent,
         // @ts-ignore

package/src/services/modules/index.ts

@@ -1,38 +1,28 @@
 import { getConfig, getGlobalConfig } from "../../config";
 import { KnowhowModule, ModuleContext } from "./types";
-import { ToolsService } from "../Tools";
 import { services } from "../";
 import * as path from "path";
 
 export class ModulesService {
-  async loadModulesFromConfig(context?: ModuleContext) {
-    const config = await getConfig();
+  async getDefaultContext() {
+    return { ...services() };
+  }
 
+  async overrideDefaultContext(overrides: Partial<ModuleContext>) {
+    const defaultContext = await this.getDefaultContext();
+    return { ...defaultContext, ...overrides };
+  }
+
+  async loadModulesFrom(
+    config: { modules: string[] } & any,
+    context?: Partial<ModuleContext>
+  ) {
     // If no context provided, fall back to global singletons
     if (!context) {
-      const { Clients, Plugins, Agents, Tools, Embeddings, MediaProcessor } = services();
-      context = {
-        Agents,
-        Embeddings,
-        Plugins,
-        Clients,
-        Tools,
-        MediaProcessor,
-      };
+      context = { ...(await this.getDefaultContext()) };
     }
 
-    // Use the toolsService from context
-    const toolsService = context.Tools;
-    const agentService = context.Agents;
-    const pluginService = context.Plugins;
-    const clients = context.Clients;
-
-    // Load from global config (~/.knowhow/knowhow.json) first, then local config
-    const globalConfig = await getGlobalConfig();
-    const allModulePaths = [
-      ...(globalConfig.modules || []),
-      ...(config.modules || []),
-    ];
+    const allModulePaths = config.modules;
 
     for (const modulePath of allModulePaths) {
       // Resolve relative paths relative to process.cwd() so that paths like
@@ -43,40 +33,59 @@ export class ModulesService {
         : modulePath;
       const rawModule = require(resolvedPath);
       const importedModule = (rawModule.default || rawModule) as KnowhowModule;
-      console.log(`🔌 Loading module: ${modulePath} (resolved: ${resolvedPath})`);
-      await importedModule.init({ config, cwd: process.cwd(), context });
-      console.log(`✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`);
+      console.log(
+        `🔌 Loading module: ${modulePath} (resolved: ${resolvedPath})`
+      );
+      await importedModule.init({ config, cwd: process.cwd(), context: context as ModuleContext });
+      console.log(
+        `✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`
+      );
 
-      for (const agent of importedModule.agents) {
-        agentService.registerAgent(agent);
+      // Only register tools/agents/plugins/clients if the relevant services
+      // are available in context (they may not be during early CLI command registration)
+      if (context.Agents) {
+        for (const agent of importedModule.agents) {
+          context.Agents.registerAgent(agent);
+        }
       }
 
-      for (const tool of importedModule.tools) {
-        toolsService.addTool(tool.definition);
-        toolsService.setFunction(tool.definition.function.name, tool.handler);
+      if (context.Tools) {
+        for (const tool of importedModule.tools) {
+          context.Tools.addTool(tool.definition);
+          context.Tools.setFunction(tool.definition.function.name, tool.handler);
+        }
       }
 
-      for (const plugin of importedModule.plugins) {
-        const pluginContext = {
-          Agents: agentService,
-          Clients: clients,
-          Tools: toolsService,
-          Plugins: pluginService,
-          ...(context.MediaProcessor ? { MediaProcessor: context.MediaProcessor } : {}),
-        };
-        pluginService.registerPlugin(plugin.name, new plugin.plugin(pluginContext as any));
+      if (context.Plugins) {
+        for (const plugin of importedModule.plugins) {
+          context.Plugins.registerPlugin(
+            plugin.name,
+            new plugin.plugin(context as any)
+          );
+        }
       }
 
-      for (const client of importedModule.clients) {
-        clients.registerClient(client.provider, client.client);
-        clients.registerModels(client.provider, client.models);
+      if (context.Clients) {
+        for (const client of importedModule.clients) {
+          context.Clients.registerClient(client.provider, client.client);
+          context.Clients.registerModels(client.provider, client.models);
+        }
       }
     }
+  }
 
-    // Also load plugins directly from config's pluginPackages map
-    if (pluginService) {
-      await pluginService.loadPluginsFromConfig(config);
-      await pluginService.loadPluginsFromConfig(globalConfig);
-    }
+  async loadModulesFromConfig(context?: ModuleContext) {
+    const config = await getConfig();
+
+    const globalConfig = await getGlobalConfig();
+    const allModulePaths = [
+      ...(globalConfig.modules || []),
+      ...(config.modules || []),
+    ];
+
+    return this.loadModulesFrom(
+      { ...config, modules: allModulePaths },
+      context
+    );
   }
 }

package/src/services/modules/types.ts

@@ -1,4 +1,5 @@
 import { Plugin, PluginContext } from "../../plugins/types";
+import { Command } from "commander";
 import { IAgent } from "../../agents/interface";
 import { Tool } from "../../clients/types";
 import { Config } from "../../types";
@@ -9,6 +10,7 @@ import { PluginService } from "../../plugins/plugins";
 import { AIClient } from "../../clients";
 import { ToolsService } from "../Tools";
 import { MediaProcessorService } from "../MediaProcessorService";
+import { TunnelHandler } from "@tyvm/knowhow-tunnel";
 
 /*
  *
@@ -53,6 +55,8 @@ export interface ModuleContext {
   Clients: AIClient;
   Tools: ToolsService;
   MediaProcessor?: MediaProcessorService;
+  Tunnel?: TunnelHandler;
+  Program?: Command;
 }
 
 export interface KnowhowModule {

package/src/tunnel.ts

@@ -0,0 +1,216 @@
+import os from "os";
+import { WebSocket } from "ws";
+import { createTunnelHandler, TunnelHandler } from "@tyvm/knowhow-tunnel";
+import { loadJwt } from "./login";
+import { wait } from "./utils";
+import { getConfig } from "./config";
+import { KNOWHOW_API_URL } from "./services/KnowhowClient";
+import { ModulesService } from "./services/modules";
+import { WorkerPasskeyAuthService } from "./workers/auth/WorkerPasskeyAuth";
+import { WsMiddlewareStack } from "./workers/auth/WsMiddleware";
+import { makeAuthMiddleware } from "./workers/auth/authMiddleware";
+
+/**
+ * Extract the tunnel domain and protocol from the API URL.
+ * e.g., "https://api.knowhow.tyvm.ai" -> { domain: "worker.knowhow.tyvm.ai", useHttps: true }
+ * e.g., "http://localhost:4000" -> { domain: "worker.localhost:4000", useHttps: false }
+ */
+export function extractTunnelDomain(apiUrl: string): {
+  domain: string;
+  useHttps: boolean;
+} {
+  try {
+    const url = new URL(apiUrl);
+    const useHttps = url.protocol === "https:";
+
+    // For localhost, include port; for production, just use hostname
+    if (url.hostname === "localhost" || url.hostname === "127.0.0.1") {
+      return {
+        domain: `worker.${url.hostname}:${url.port || "80"}`,
+        useHttps,
+      };
+    }
+    return { domain: `worker.${url.hostname}`, useHttps };
+  } catch (err) {
+    console.error("Failed to parse API_URL for tunnel domain:", err);
+    return { domain: "worker.localhost:4000", useHttps: false }; // fallback
+  }
+}
+
+/**
+ * Initialize a tunnel handler and load tunnel modules.
+ */
+export async function initTunnelHandler(
+  tunnelConnection: WebSocket,
+  tunnelConfig: Parameters<typeof createTunnelHandler>[1]
+): Promise<TunnelHandler> {
+  const handler = createTunnelHandler(tunnelConnection, tunnelConfig);
+  console.log("🌐 Tunnel handler initialized");
+  console.log(tunnelConfig);
+
+  const tunnelModuleService = new ModulesService();
+  const tunnelContext = await tunnelModuleService.overrideDefaultContext({
+    Tunnel: handler,
+  });
+  tunnelModuleService.loadModulesFromConfig(tunnelContext).catch((err) => {
+    console.error("Failed to load tunnel modules:", err);
+  });
+
+  return handler;
+}
+
+/**
+ * Resolve tunnel local host, log port mapping, and return shared tunnel setup values.
+ * Extracted to avoid duplication between worker() and tunnel().
+ */
+export function resolveTunnelConfig(
+  config: Awaited<ReturnType<typeof getConfig>>,
+  isInsideDocker: boolean
+): { tunnelLocalHost: string; portMapping: Record<string, number> } {
+  // Determine localHost based on environment
+  let tunnelLocalHost = config.worker?.tunnel?.localHost;
+  if (!tunnelLocalHost) {
+    if (isInsideDocker) {
+      tunnelLocalHost = "host.docker.internal";
+      console.log(
+        "🐳 Docker detected: tunnel will use host.docker.internal to reach host services"
+      );
+    } else {
+      tunnelLocalHost = "127.0.0.1";
+    }
+  }
+
+  // Check for port mapping configuration
+  const portMapping = (config.worker?.tunnel?.portMapping || {}) as Record<string, number>;
+  if (Object.keys(portMapping).length > 0) {
+    console.log("🔀 Port mapping configured:");
+    for (const [containerPort, hostPort] of Object.entries(portMapping)) {
+      console.log(`   Container port ${containerPort} → Host port ${hostPort}`);
+    }
+  }
+
+  return { tunnelLocalHost, portMapping };
+}
+
+/**
+ * Options for connectTunnelWebSocket helper.
+ */
+export interface TunnelWebSocketOptions {
+  /** Already-resolved tunnel domain (hostname only, no protocol) */
+  tunnelDomain: string;
+  /** Whether the tunnel should use HTTPS */
+  tunnelUseHttps: boolean;
+  /** Local host to forward tunnel traffic to */
+  tunnelLocalHost: string;
+  /** Port mapping configuration */
+  portMapping: Record<string, number>;
+  /** Worker config (for tunnel sub-config) */
+  config: Awaited<ReturnType<typeof getConfig>>;
+  /** HTTP headers to attach to the WebSocket upgrade request */
+  headers: Record<string, string>;
+  /** Callback invoked with the TunnelHandler once the connection opens */
+  onOpen?: (handler: TunnelHandler) => void;
+  /** Called when the connection closes; receives code + reason string */
+  onClose?: (code: number, reason: string) => void;
+  /** Called on error */
+  onError?: (error: Error) => void;
+  /** Optional passkey auth service — if provided, applies WS middleware to gate tunnel traffic */
+  authService?: WorkerPasskeyAuthService | null;
+}
+
+/**
+ * Create a tunnel WebSocket connection, build the tunnelConfig, and
+ * initialize the tunnel handler.  Returns the WebSocket.
+ *
+ * The caller is responsible for storing a reference to the returned TunnelHandler
+ * (via onOpen) and performing any outer-state cleanup (via onClose / onError).
+ */
+export function connectTunnelWebSocket(
+  options: TunnelWebSocketOptions
+): WebSocket {
+  const {
+    tunnelDomain,
+    tunnelUseHttps,
+    tunnelLocalHost,
+    portMapping,
+    config,
+    headers,
+    onOpen,
+    onClose,
+    onError,
+    authService,
+  } = options;
+
+  const tunnelConnection = new WebSocket(`${KNOWHOW_API_URL}/ws/tunnel`, { headers });
+
+  tunnelConnection.on("open", async () => {
+    console.log("Tunnel WebSocket connected");
+
+    // Apply passkey auth middleware FIRST, before tunnel handler registers its
+    // "message" listener. Node.js EventEmitter fires listeners in registration
+    // order, so our middleware runs first. wrapSocket() also redirects future
+    // ws.on("message", ...) calls to an inner emitter, ensuring the tunnel
+    // handler only receives messages that passed the middleware.
+    if (authService) {
+      const stack = new WsMiddlewareStack();
+      stack.use(makeAuthMiddleware(authService));
+      stack.wrapSocket(tunnelConnection);
+    }
+
+    const allowedPorts = config.worker?.tunnel?.allowedPorts || [];
+
+    // Create URL rewriter callback that returns the hostname (without protocol).
+    // The tunnel package will add the protocol based on the useHttps config.
+    const urlRewriter = (port: number, metadata?: any) => {
+      const workerId = metadata?.workerId;
+      const secret = metadata?.secret;
+      // Examples: secret-p3000.worker.example.com  /  workerId-p3000.worker.example.com
+      const subdomain = secret
+        ? `${secret}-p${port}`
+        : `${workerId}-p${port}`;
+      return `${subdomain}.${tunnelDomain}`;
+    };
+
+    const tunnelConfig = {
+      allowedPorts,
+      maxConcurrentStreams: config.worker?.tunnel?.maxConcurrentStreams || 50,
+      tunnelUseHttps,
+      localHost: tunnelLocalHost,
+      urlRewriter,
+      enableUrlRewriting: config.worker?.tunnel?.enableUrlRewriting !== false,
+      portMapping,
+      logLevel: "debug" as const,
+    };
+
+    const handler = await initTunnelHandler(tunnelConnection, tunnelConfig);
+    onOpen?.(handler);
+  });
+
+  tunnelConnection.on("close", (code, reason) => {
+    console.log(
+      `Tunnel WebSocket closed. Code: ${code}, Reason: ${reason.toString()}`
+    );
+    onClose?.(code, reason.toString());
+  });
+
+  tunnelConnection.on("error", (error) => {
+    console.error("Tunnel WebSocket error:", error);
+    onError?.(error);
+  });
+
+  return tunnelConnection;
+}
+
+/**
+ * The minimal set of tool names that are always registered when running in
+ * tunnel mode. These are the tools the backend and frontend need to interact
+ * with the tunnel worker (port discovery, passkey auth).
+ *
+ * Additional tools can be added here in the future without changing the CLI.
+ */
+export const TUNNEL_MINIMAL_TOOLS = [
+  "listAllowedPorts",
+  "unlock",
+  "lock",
+  "reloadConfig",
+];

package/src/types.ts

@@ -52,7 +52,6 @@ export type Config = {
     modules?: string[];
   };
   modules: string[];
-  pluginPackages?: Record<string, string>;
   agents: Assistant[];
   mcps: McpConfig[];
   modelProviders: ModelProvider[];