@tyvm/knowhow 0.0.108 → 0.0.109

package/src/processors/JsonCompressor.ts

@@ -382,7 +382,7 @@ export class JsonCompressor {
             i + currentChunk.length - 1
           }]\nPreview: ${chunkString.substring(0, 100)}...\n[Use ${
             this.toolName
-          } tool with key "${key}" to retrieve this chunk]`;
+          } tool with key "${key}" to retrieve this chunk]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
           finalArray.unshift(stub); // Add stub to the start of our final result.
 
           currentChunk = [];
@@ -415,11 +415,11 @@ export class JsonCompressor {
       
       // Store objects on FIRST occurrence so second occurrence can reference it
       // We increment seenCount above, so after increment:
-      // seenCount=1: first occurrence (just incremented from 0 to 1), store it
-      // seenCount>=2: we already stored it on first occurrence, should be in dedup map
+      // seenCount=0: first occurrence (before increment), store it
+      // seenCount>=1: we already stored it on first occurrence, should be in dedup map
       // Note: This means we store proactively - first occurrence gets stored AND returned in full
       // Second+ occurrences will find it in the dedup map and return a reference
-      const isFirstOccurrence = seenCount === 1;
+      const isFirstOccurrence = seenCount === 0;
 
       // Process the object - apply low-signal detection
       const objWithLowSignalCompressed = this.compressObjectWithLowSignalDetection(obj, path);
@@ -453,7 +453,7 @@ export class JsonCompressor {
           result
         ).join(", ")}\nPreview: ${objectAsString.substring(0, 200)}...\n[Use ${
           this.toolName
-        } tool with key "${key}" to retrieve full content]`;
+        } tool with key "${key}" to retrieve full content]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
       }
       return result;
     }
@@ -486,7 +486,7 @@ export class JsonCompressor {
           200
         )}...\n[Use ${
           this.toolName
-        } tool with key "${key}" to retrieve full content]`;
+        } tool with key "${key}" to retrieve full content]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
       }
       return obj;
     }

package/src/services/EventService.ts

@@ -1,6 +1,8 @@
 import { EventEmitter } from "events";
 import { IAgent } from "../agents/interface";
 
+export type LogLevel = "info" | "warn" | "error";
+
 export type EventHandlerFn = (...args: any[]) => any;
 
 export interface EventHandler {
@@ -31,9 +33,38 @@ type ManagedListenerRecord = {
   blocking: boolean;
 };
 
+/**
+ * Default console handler for plugin:log events.
+ * Active when no renderer has taken over (e.g. worker mode, CLI before chat starts).
+ * Can be suppressed by calling suppressDefaultLogger() when a renderer is active.
+ *
+ * IMPORTANT: Uses process.stdout/stderr directly to avoid infinite recursion
+ * with logger.installConsoleOverload() which overrides console.log/warn.
+ */
+function defaultConsoleLogHandler(event: {
+  source: string;
+  message: string;
+  level: LogLevel;
+}): void {
+  const prefix = event.source ? `[${event.source}] ` : "";
+  const line = `${prefix}${event.message}\n`;
+  switch (event.level) {
+    case "warn":
+      process.stderr.write(line);
+      break;
+    case "error":
+      process.stderr.write(line);
+      break;
+    default:
+      process.stdout.write(line);
+  }
+}
+
 export class EventService extends EventEmitter {
   private blockingHandlers: Map<string, EventHandler[]> = new Map();
   private managedListeners: Map<string, ManagedListenerRecord> = new Map();
+  private defaultLoggerActive = true;
+  private boundDefaultLogHandler = defaultConsoleLogHandler;
 
   eventTypes = {
     agentMsg: "agent:msg",
@@ -45,6 +76,35 @@ export class EventService extends EventEmitter {
   constructor() {
     super();
     this.setMaxListeners(100);
+    // Register the default console logger so Events.log() always produces output
+    // even before a renderer is attached (worker mode, module loading, etc.)
+    this.on(this.eventTypes.pluginLog, this.boundDefaultLogHandler);
+  }
+
+  /**
+   * Suppress the default console logger.
+   * Call this when a renderer has taken over and will handle plugin:log events.
+   * This prevents double-printing when both the renderer and the default handler fire.
+   */
+  suppressDefaultLogger(): void {
+    if (this.defaultLoggerActive) {
+      this.removeListener(
+        this.eventTypes.pluginLog,
+        this.boundDefaultLogHandler
+      );
+      this.defaultLoggerActive = false;
+    }
+  }
+
+  /**
+   * Restore the default console logger.
+   * Call this when the renderer is torn down.
+   */
+  restoreDefaultLogger(): void {
+    if (!this.defaultLoggerActive) {
+      this.on(this.eventTypes.pluginLog, this.boundDefaultLogHandler);
+      this.defaultLoggerActive = true;
+    }
   }
 
   /**
@@ -232,7 +292,7 @@ export class EventService extends EventEmitter {
   log(
     source: string,
     message: string,
-    level: "info" | "warn" | "error" = "info"
+    level: LogLevel = "info"
   ): void {
     this.emit(this.eventTypes.pluginLog, {
       source,

package/src/services/KnowhowClient.ts

@@ -1,3 +1,4 @@
+import { createHash } from "crypto";
 import http from "../utils/http";
 import fs from "fs";
 import { Message } from "../clients/types";
@@ -171,11 +172,9 @@ export class KnowhowSimpleClient {
       try {
         this.jwtValidated = true;
         const response = await this.me();
-
         const user = response.data.user;
         const orgs = user.orgs;
         const orgId = response.data.orgId;
-
         const currentOrg = orgs.find((org) => {
           return org.organizationId === orgId;
         });
@@ -227,6 +226,17 @@ export class KnowhowSimpleClient {
     return presignedUrl;
   }
 
+  async getOrgEmbedding(id: string) {
+    await this.checkJwt();
+    const resp = await http.get(
+      `${this.baseUrl}/api/org-embeddings/${id}`,
+      {
+        headers: this.headers,
+      }
+    );
+    return resp.data as { id: string; modelName: string; name: string; [key: string]: unknown };
+  }
+
   async updateEmbeddingMetadata(
     id: string,
     data: {
@@ -654,8 +664,10 @@ export class KnowhowSimpleClient {
   /**
    * Get presigned S3 URL for uploading a file to Knowhow FS.
    * First finds or creates the file by path, then gets its upload URL.
+   * Computes SHA256 hash of the file content and stores it as S3 metadata
+   * so any client can determine if they already have this version without downloading.
    */
-  async getOrgFilePresignedUploadUrl(filePath: string): Promise<string> {
+  async getOrgFilePresignedUploadUrl(filePath: string, localFilePath?: string): Promise<string> {
     await this.checkJwt();
 
     // Find or create the file by path
@@ -666,10 +678,17 @@ export class KnowhowSimpleClient {
     const fileName =
       lastSlash >= 0 ? filePath.substring(lastSlash + 1) : filePath;
 
+    // Compute SHA256 hash if we have the local file path, so S3 stores it as metadata
+    let sha256Hash: string | undefined;
+    if (localFilePath) {
+      const fileContent = fs.readFileSync(localFilePath);
+      sha256Hash = createHash("sha256").update(fileContent).digest("base64");
+    }
+
     // Get upload URL using the file ID
     const response = await http.post<{ uploadUrl: string }>(
       `${this.baseUrl}/api/org-files/upload/${file.id}`,
-      { fileName },
+      { fileName, sha256Hash },
       { headers: this.headers }
     );
     return response.data.uploadUrl;
@@ -718,6 +737,17 @@ export class KnowhowSimpleClient {
     );
   }
 
+  /**
+   * Get a single cloud worker by ID
+   */
+  async getCloudWorker(id: string) {
+    await this.checkJwt();
+    return http.get<{ id: string; name: string; status: string; workerConfigJson?: Record<string, unknown> }>(
+      `${this.baseUrl}/api/cloud-workers/${id}`,
+      { headers: this.headers }
+    );
+  }
+
   /**
    * Update an existing cloud worker
    */

package/src/services/modules/index.ts

@@ -1,38 +1,30 @@
+import * as path from "path";
+
 import { getConfig, getGlobalConfig } from "../../config";
 import { KnowhowModule, ModuleContext } from "./types";
-import { ToolsService } from "../Tools";
 import { services } from "../";
-import * as path from "path";
+import { toUniqueArray } from "../../utils";
 
 export class ModulesService {
-  async loadModulesFromConfig(context?: ModuleContext) {
-    const config = await getConfig();
+  async getDefaultContext() {
+    return { ...services() };
+  }
 
+  async overrideDefaultContext(overrides: Partial<ModuleContext>) {
+    const defaultContext = await this.getDefaultContext();
+    return { ...defaultContext, ...overrides };
+  }
+
+  async loadModulesFrom(
+    config: { modules: string[] } & any,
+    context?: Partial<ModuleContext>
+  ) {
     // If no context provided, fall back to global singletons
     if (!context) {
-      const { Clients, Plugins, Agents, Tools, Embeddings, MediaProcessor } = services();
-      context = {
-        Agents,
-        Embeddings,
-        Plugins,
-        Clients,
-        Tools,
-        MediaProcessor,
-      };
+      context = { ...(await this.getDefaultContext()) };
     }
 
-    // Use the toolsService from context
-    const toolsService = context.Tools;
-    const agentService = context.Agents;
-    const pluginService = context.Plugins;
-    const clients = context.Clients;
-
-    // Load from global config (~/.knowhow/knowhow.json) first, then local config
-    const globalConfig = await getGlobalConfig();
-    const allModulePaths = [
-      ...(globalConfig.modules || []),
-      ...(config.modules || []),
-    ];
+    const allModulePaths = config.modules;
 
     for (const modulePath of allModulePaths) {
       // Resolve relative paths relative to process.cwd() so that paths like
@@ -43,40 +35,68 @@ export class ModulesService {
         : modulePath;
       const rawModule = require(resolvedPath);
       const importedModule = (rawModule.default || rawModule) as KnowhowModule;
-      console.log(`🔌 Loading module: ${modulePath} (resolved: ${resolvedPath})`);
-      await importedModule.init({ config, cwd: process.cwd(), context });
-      console.log(`✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`);
+      context.Events?.log(
+        "ModulesService",
+        `🔌 Loading module: ${modulePath} (resolved: ${resolvedPath})`
+      );
+      await importedModule.init({
+        config,
+        cwd: process.cwd(),
+        context: context as ModuleContext,
+      });
+      context.Events?.log(
+        "ModulesService",
+        `✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`
+      );
 
-      for (const agent of importedModule.agents) {
-        agentService.registerAgent(agent);
+      // Only register tools/agents/plugins/clients if the relevant services
+      // are available in context (they may not be during early CLI command registration)
+      if (context.Agents) {
+        for (const agent of importedModule.agents) {
+          context.Agents.registerAgent(agent);
+        }
       }
 
-      for (const tool of importedModule.tools) {
-        toolsService.addTool(tool.definition);
-        toolsService.setFunction(tool.definition.function.name, tool.handler);
+      if (context.Tools) {
+        for (const tool of importedModule.tools) {
+          context.Tools.addTool(tool.definition);
+          context.Tools.setFunction(
+            tool.definition.function.name,
+            tool.handler
+          );
+        }
       }
 
-      for (const plugin of importedModule.plugins) {
-        const pluginContext = {
-          Agents: agentService,
-          Clients: clients,
-          Tools: toolsService,
-          Plugins: pluginService,
-          ...(context.MediaProcessor ? { MediaProcessor: context.MediaProcessor } : {}),
-        };
-        pluginService.registerPlugin(plugin.name, new plugin.plugin(pluginContext as any));
+      if (context.Plugins) {
+        for (const plugin of importedModule.plugins) {
+          context.Plugins.registerPlugin(
+            plugin.name,
+            new plugin.plugin(context as any)
+          );
+        }
       }
 
-      for (const client of importedModule.clients) {
-        clients.registerClient(client.provider, client.client);
-        clients.registerModels(client.provider, client.models);
+      if (context.Clients) {
+        for (const client of importedModule.clients) {
+          context.Clients.registerClient(client.provider, client.client);
+          context.Clients.registerModels(client.provider, client.models);
+        }
       }
     }
+  }
 
-    // Also load plugins directly from config's pluginPackages map
-    if (pluginService) {
-      await pluginService.loadPluginsFromConfig(config);
-      await pluginService.loadPluginsFromConfig(globalConfig);
-    }
+  async loadModulesFromConfig(context?: ModuleContext) {
+    const config = await getConfig();
+
+    const globalConfig = await getGlobalConfig();
+    const allModulePaths = [
+      ...(globalConfig.modules || []),
+      ...(config.modules || []),
+    ];
+
+    return this.loadModulesFrom(
+      { ...config, modules: toUniqueArray(allModulePaths) },
+      context
+    );
   }
 }

package/src/services/modules/types.ts

@@ -1,4 +1,5 @@
 import { Plugin, PluginContext } from "../../plugins/types";
+import { Command } from "commander";
 import { IAgent } from "../../agents/interface";
 import { Tool } from "../../clients/types";
 import { Config } from "../../types";
@@ -9,6 +10,8 @@ import { PluginService } from "../../plugins/plugins";
 import { AIClient } from "../../clients";
 import { ToolsService } from "../Tools";
 import { MediaProcessorService } from "../MediaProcessorService";
+import { TunnelHandler } from "@tyvm/knowhow-tunnel";
+import { EventService } from "../EventService";
 
 /*
  *
@@ -52,7 +55,10 @@ export interface ModuleContext {
   Plugins: PluginService;
   Clients: AIClient;
   Tools: ToolsService;
+  Events: EventService;
   MediaProcessor?: MediaProcessorService;
+  Tunnel?: TunnelHandler;
+  Program?: Command;
 }
 
 export interface KnowhowModule {

package/src/tunnel.ts

@@ -0,0 +1,216 @@
+import os from "os";
+import { WebSocket } from "ws";
+import { createTunnelHandler, TunnelHandler } from "@tyvm/knowhow-tunnel";
+import { loadJwt } from "./login";
+import { wait } from "./utils";
+import { getConfig } from "./config";
+import { KNOWHOW_API_URL } from "./services/KnowhowClient";
+import { ModulesService } from "./services/modules";
+import { WorkerPasskeyAuthService } from "./workers/auth/WorkerPasskeyAuth";
+import { WsMiddlewareStack } from "./workers/auth/WsMiddleware";
+import { makeAuthMiddleware } from "./workers/auth/authMiddleware";
+
+/**
+ * Extract the tunnel domain and protocol from the API URL.
+ * e.g., "https://api.knowhow.tyvm.ai" -> { domain: "worker.knowhow.tyvm.ai", useHttps: true }
+ * e.g., "http://localhost:4000" -> { domain: "worker.localhost:4000", useHttps: false }
+ */
+export function extractTunnelDomain(apiUrl: string): {
+  domain: string;
+  useHttps: boolean;
+} {
+  try {
+    const url = new URL(apiUrl);
+    const useHttps = url.protocol === "https:";
+
+    // For localhost, include port; for production, just use hostname
+    if (url.hostname === "localhost" || url.hostname === "127.0.0.1") {
+      return {
+        domain: `worker.${url.hostname}:${url.port || "80"}`,
+        useHttps,
+      };
+    }
+    return { domain: `worker.${url.hostname}`, useHttps };
+  } catch (err) {
+    console.error("Failed to parse API_URL for tunnel domain:", err);
+    return { domain: "worker.localhost:4000", useHttps: false }; // fallback
+  }
+}
+
+/**
+ * Initialize a tunnel handler and load tunnel modules.
+ */
+export async function initTunnelHandler(
+  tunnelConnection: WebSocket,
+  tunnelConfig: Parameters<typeof createTunnelHandler>[1]
+): Promise<TunnelHandler> {
+  const handler = createTunnelHandler(tunnelConnection, tunnelConfig);
+  console.log("🌐 Tunnel handler initialized");
+  console.log(tunnelConfig);
+
+  const tunnelModuleService = new ModulesService();
+  const tunnelContext = await tunnelModuleService.overrideDefaultContext({
+    Tunnel: handler,
+  });
+  tunnelModuleService.loadModulesFromConfig(tunnelContext).catch((err) => {
+    console.error("Failed to load tunnel modules:", err);
+  });
+
+  return handler;
+}
+
+/**
+ * Resolve tunnel local host, log port mapping, and return shared tunnel setup values.
+ * Extracted to avoid duplication between worker() and tunnel().
+ */
+export function resolveTunnelConfig(
+  config: Awaited<ReturnType<typeof getConfig>>,
+  isInsideDocker: boolean
+): { tunnelLocalHost: string; portMapping: Record<string, number> } {
+  // Determine localHost based on environment
+  let tunnelLocalHost = config.worker?.tunnel?.localHost;
+  if (!tunnelLocalHost) {
+    if (isInsideDocker) {
+      tunnelLocalHost = "host.docker.internal";
+      console.log(
+        "🐳 Docker detected: tunnel will use host.docker.internal to reach host services"
+      );
+    } else {
+      tunnelLocalHost = "127.0.0.1";
+    }
+  }
+
+  // Check for port mapping configuration
+  const portMapping = (config.worker?.tunnel?.portMapping || {}) as Record<string, number>;
+  if (Object.keys(portMapping).length > 0) {
+    console.log("🔀 Port mapping configured:");
+    for (const [containerPort, hostPort] of Object.entries(portMapping)) {
+      console.log(`   Container port ${containerPort} → Host port ${hostPort}`);
+    }
+  }
+
+  return { tunnelLocalHost, portMapping };
+}
+
+/**
+ * Options for connectTunnelWebSocket helper.
+ */
+export interface TunnelWebSocketOptions {
+  /** Already-resolved tunnel domain (hostname only, no protocol) */
+  tunnelDomain: string;
+  /** Whether the tunnel should use HTTPS */
+  tunnelUseHttps: boolean;
+  /** Local host to forward tunnel traffic to */
+  tunnelLocalHost: string;
+  /** Port mapping configuration */
+  portMapping: Record<string, number>;
+  /** Worker config (for tunnel sub-config) */
+  config: Awaited<ReturnType<typeof getConfig>>;
+  /** HTTP headers to attach to the WebSocket upgrade request */
+  headers: Record<string, string>;
+  /** Callback invoked with the TunnelHandler once the connection opens */
+  onOpen?: (handler: TunnelHandler) => void;
+  /** Called when the connection closes; receives code + reason string */
+  onClose?: (code: number, reason: string) => void;
+  /** Called on error */
+  onError?: (error: Error) => void;
+  /** Optional passkey auth service — if provided, applies WS middleware to gate tunnel traffic */
+  authService?: WorkerPasskeyAuthService | null;
+}
+
+/**
+ * Create a tunnel WebSocket connection, build the tunnelConfig, and
+ * initialize the tunnel handler.  Returns the WebSocket.
+ *
+ * The caller is responsible for storing a reference to the returned TunnelHandler
+ * (via onOpen) and performing any outer-state cleanup (via onClose / onError).
+ */
+export function connectTunnelWebSocket(
+  options: TunnelWebSocketOptions
+): WebSocket {
+  const {
+    tunnelDomain,
+    tunnelUseHttps,
+    tunnelLocalHost,
+    portMapping,
+    config,
+    headers,
+    onOpen,
+    onClose,
+    onError,
+    authService,
+  } = options;
+
+  const tunnelConnection = new WebSocket(`${KNOWHOW_API_URL}/ws/tunnel`, { headers });
+
+  tunnelConnection.on("open", async () => {
+    console.log("Tunnel WebSocket connected");
+
+    // Apply passkey auth middleware FIRST, before tunnel handler registers its
+    // "message" listener. Node.js EventEmitter fires listeners in registration
+    // order, so our middleware runs first. wrapSocket() also redirects future
+    // ws.on("message", ...) calls to an inner emitter, ensuring the tunnel
+    // handler only receives messages that passed the middleware.
+    if (authService) {
+      const stack = new WsMiddlewareStack();
+      stack.use(makeAuthMiddleware(authService));
+      stack.wrapSocket(tunnelConnection);
+    }
+
+    const allowedPorts = config.worker?.tunnel?.allowedPorts || [];
+
+    // Create URL rewriter callback that returns the hostname (without protocol).
+    // The tunnel package will add the protocol based on the useHttps config.
+    const urlRewriter = (port: number, metadata?: any) => {
+      const workerId = metadata?.workerId;
+      const secret = metadata?.secret;
+      // Examples: secret-p3000.worker.example.com  /  workerId-p3000.worker.example.com
+      const subdomain = secret
+        ? `${secret}-p${port}`
+        : `${workerId}-p${port}`;
+      return `${subdomain}.${tunnelDomain}`;
+    };
+
+    const tunnelConfig = {
+      allowedPorts,
+      maxConcurrentStreams: config.worker?.tunnel?.maxConcurrentStreams || 50,
+      tunnelUseHttps,
+      localHost: tunnelLocalHost,
+      urlRewriter,
+      enableUrlRewriting: config.worker?.tunnel?.enableUrlRewriting !== false,
+      portMapping,
+      logLevel: "debug" as const,
+    };
+
+    const handler = await initTunnelHandler(tunnelConnection, tunnelConfig);
+    onOpen?.(handler);
+  });
+
+  tunnelConnection.on("close", (code, reason) => {
+    console.log(
+      `Tunnel WebSocket closed. Code: ${code}, Reason: ${reason.toString()}`
+    );
+    onClose?.(code, reason.toString());
+  });
+
+  tunnelConnection.on("error", (error) => {
+    console.error("Tunnel WebSocket error:", error);
+    onError?.(error);
+  });
+
+  return tunnelConnection;
+}
+
+/**
+ * The minimal set of tool names that are always registered when running in
+ * tunnel mode. These are the tools the backend and frontend need to interact
+ * with the tunnel worker (port discovery, passkey auth).
+ *
+ * Additional tools can be added here in the future without changing the CLI.
+ */
+export const TUNNEL_MINIMAL_TOOLS = [
+  "listAllowedPorts",
+  "unlock",
+  "lock",
+  "reloadConfig",
+];

package/src/types.ts

@@ -52,7 +52,6 @@ export type Config = {
     modules?: string[];
   };
   modules: string[];
-  pluginPackages?: Record<string, string>;
   agents: Assistant[];
   mcps: McpConfig[];
   modelProviders: ModelProvider[];