@tyvm/knowhow 0.0.108 → 0.0.109-dev.e88af1e

package/src/logger.ts

@@ -0,0 +1,197 @@
+import type { LogLevel } from "./services/EventService";
+
+/**
+ * App-wide logger utility.
+ *
+ * Features:
+ * 1. `logger.info/warn/error(source, message)` — routes through EventService
+ * 2. `Logger.of("ClassName")` — creates a bound logger so you don't repeat the source
+ * 3. `logger.installConsoleOverload()` — replaces console.log/warn/error/info with
+ *    our closure, so ALL output (including third-party modules) goes through us
+ * 4. `logger.silence()` / `logger.unsilence()` — suppress all output, useful for
+ *    commands that need clean stdout (e.g. github-credentials)
+ *
+ * Usage (module-level):
+ *   import { logger } from "../logger";
+ *   logger.info("MyService", "Something happened");
+ *
+ * Usage (class-level):
+ *   import { Logger } from "../logger";
+ *   class MyClass {
+ *     private logger = Logger.of("MyClass");
+ *     doThing() { this.logger.info("Something happened"); }
+ *   }
+ *
+ * Silence mode (for clean-stdout commands):
+ *   logger.silence();   // suppress everything
+ *   // ... do work that must produce clean stdout ...
+ *   logger.unsilence(); // restore
+ */
+
+// ---- Internal state ---------------------------------------------------------
+
+let silenced = false;
+
+// Original console methods — saved before any overload is installed
+const _originalConsole = {
+  log: console.log.bind(console),
+  warn: console.warn.bind(console),
+  error: console.error.bind(console),
+  info: console.info.bind(console),
+};
+
+let consoleOverloadInstalled = false;
+
+// ---- EventService lazy accessor ---------------------------------------------
+
+function getEvents() {
+  try {
+    const { services } = require("./services") as typeof import("./services");
+    return services().Events;
+  } catch {
+    return null;
+  }
+}
+
+// ---- Core emit logic --------------------------------------------------------
+
+function emit(source: string, message: string, level: LogLevel): void {
+  if (silenced) return;
+
+  try {
+    const events = getEvents();
+    if (events) {
+      events.log(source, message, level);
+      return;
+    }
+  } catch {
+    // fall through to direct console output
+  }
+
+  // Fallback: use original console methods (bypasses any overload we installed)
+  const prefix = source ? `[${source}] ` : "";
+  if (level === "warn") _originalConsole.warn(`${prefix}${message}`);
+  else if (level === "error") _originalConsole.error(`${prefix}${message}`);
+  else _originalConsole.log(`${prefix}${message}`);
+}
+
+// ---- Bound logger (returned by Logger.of) -----------------------------------
+
+export interface BoundLogger {
+  log(message: string, level?: LogLevel): void;
+  info(message: string): void;
+  warn(message: string): void;
+  error(message: string): void;
+}
+
+function makeBoundLogger(source: string): BoundLogger {
+  return {
+    log(message: string, level: LogLevel = "info"): void {
+      emit(source, message, level);
+    },
+    info(message: string): void {
+      emit(source, message, "info");
+    },
+    warn(message: string): void {
+      emit(source, message, "warn");
+    },
+    error(message: string): void {
+      emit(source, message, "error");
+    },
+  };
+}
+
+// ---- Public API -------------------------------------------------------------
+
+export const logger = {
+  log(source: string, message: string, level: LogLevel = "info"): void {
+    emit(source, message, level);
+  },
+
+  info(source: string, message: string): void {
+    emit(source, message, "info");
+  },
+
+  warn(source: string, message: string): void {
+    emit(source, message, "warn");
+  },
+
+  error(source: string, message: string): void {
+    emit(source, message, "error");
+  },
+
+  /**
+   * Suppress all log output. Useful for commands that need clean stdout
+   * (e.g. git credential helpers). All logger.* calls and overloaded
+   * console.* calls become no-ops until unsilence() is called.
+   */
+  silence(): void {
+    silenced = true;
+  },
+
+  /**
+   * Restore log output after a silence() call.
+   */
+  unsilence(): void {
+    silenced = false;
+  },
+
+  /**
+   * Returns true if the logger is currently silenced.
+   */
+  isSilenced(): boolean {
+    return silenced;
+  },
+
+  /**
+   * Install console overload. After this call, console.log/warn/error/info
+   * all route through our closure (respecting silence mode).
+   * Safe to call multiple times — only installs once.
+   *
+   * Call this early in CLI startup (before any modules are loaded) to ensure
+   * third-party module logs don't bypass the silence mechanism.
+   */
+  installConsoleOverload(): void {
+    if (consoleOverloadInstalled) return;
+    consoleOverloadInstalled = true;
+
+    const route = (originalFn: (...args: any[]) => void, args: any[]) => {
+      if (silenced) return;
+      originalFn(...args);
+    };
+
+    console.log = (...args: any[]) => route(_originalConsole.log, args);
+    console.info = (...args: any[]) => route(_originalConsole.info, args);
+    console.warn = (...args: any[]) => route(_originalConsole.warn, args);
+    // Note: console.error is intentionally NOT overloaded — real errors (stack
+    // traces, crash reports) should always be visible. Only suppress via silence().
+    // If you want to suppress errors too, call logger.silence() which checks the flag
+    // before the overloaded console.warn/log routes reach here anyway.
+  },
+
+  /**
+   * Remove the console overload and restore original console methods.
+   */
+  uninstallConsoleOverload(): void {
+    if (!consoleOverloadInstalled) return;
+    console.log = _originalConsole.log;
+    console.info = _originalConsole.info;
+    console.warn = _originalConsole.warn;
+    consoleOverloadInstalled = false;
+  },
+};
+
+/**
+ * Factory for creating a bound logger with a fixed source name.
+ * Ideal for class-level loggers:
+ *
+ *   class MyClass {
+ *     private logger = Logger.of("MyClass");
+ *     doThing() { this.logger.info("hello"); }
+ *   }
+ */
+export const Logger = {
+  of(source: string): BoundLogger {
+    return makeBoundLogger(source);
+  },
+};

package/src/plugins/embedding.ts

@@ -20,9 +20,12 @@ export class EmbeddingPlugin extends PluginBase {
 
   constructor(context) {
     super(context);
-    
+
     // Subscribe to file:post-edit events
-    this.context.Events.on("file:post-edit", this.handleFilePostEdit.bind(this));
+    this.context.Events.on(
+      "file:post-edit",
+      this.handleFilePostEdit.bind(this)
+    );
   }
 
   async embed() {
@@ -68,10 +71,12 @@ export class EmbeddingPlugin extends PluginBase {
       this.log(`Reading entry ${entry.id}`);
     }
 
-    const contextLength = JSON.stringify(context).split(" ").length;
+    const ids = context.map((entry) => entry.id);
+
+    const contextLength = JSON.stringify(ids).split(" ").length;
     this.log(`Found ${context.length} entries. Loading ${contextLength} words`);
 
-    return `EMBEDDING PLUGIN: Our knowledgebase contains this information which can be used to answer the question:
-    ${JSON.stringify(context)}`;
+    return `EMBEDDING PLUGIN: Our knowledgebase indicates these embedding entries may be related to the question:
+    ${JSON.stringify(ids)}`;
   }
-}
+}

package/src/plugins/plugins.ts

@@ -1,5 +1,4 @@
 import { Plugin, PluginContext } from "./types";
-import { Config } from "../types";
 import { VimPlugin } from "./vim";
 import { LinterPlugin } from "./LinterPlugin";
 import { LanguagePlugin } from "./language";
@@ -49,26 +48,6 @@ export class PluginService {
     return instance.meta.key;
   }
 
-  /**
-   * Load plugins from config's pluginPackages map.
-   * Each entry maps a plugin key to an npm package name or file path.
-   * Errors are caught and logged as warnings without crashing.
-   */
-  async loadPluginsFromConfig(config: Config): Promise<void> {
-    const pluginPackages = config.pluginPackages || {};
-    for (const [key, spec] of Object.entries(pluginPackages)) {
-      try {
-        await this.loadPlugin(spec);
-      } catch (error) {
-        this.events?.log(
-          "PluginService",
-          `Failed to load plugin "${key}" from "${spec}": ${error instanceof Error ? error.message : error}`,
-          "warn"
-        );
-      }
-    }
-  }
-
   /** Disable a plugin by its key; returns `true` if found. */
   disablePlugin(key: string): boolean {
     const p = this.pluginMap.get(key);

package/src/plugins/vim.ts

@@ -73,22 +73,11 @@ export class VimPlugin extends PluginBase {
 
   async call() {
     const vimFiles = await this.getVimFiles();
-    const fileContents = await Promise.all(
-      vimFiles.map(async (f) => {
-        const loaded = await this.getFileContents(f);
-
-        const preview =
-          loaded.content.length > 1000
-            ? loaded.content.slice(0, 1000) +
-              "... file trimmed, read file for full content"
-            : loaded.content;
-
-        return {
-          sourceFile: loaded.filePath,
-          content: loaded.content.slice(0, 1000),
-        };
-      })
-    );
+    const fileContents = vimFiles.map((f) => {
+      return {
+        sourceFile: f,
+      };
+    });
     if (fileContents.length === 0) {
       return "VIM PLUGIN: No files open in vim";
     }

package/src/processors/JsonCompressor.ts

@@ -382,7 +382,7 @@ export class JsonCompressor {
             i + currentChunk.length - 1
           }]\nPreview: ${chunkString.substring(0, 100)}...\n[Use ${
             this.toolName
-          } tool with key "${key}" to retrieve this chunk]`;
+          } tool with key "${key}" to retrieve this chunk]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
           finalArray.unshift(stub); // Add stub to the start of our final result.
 
           currentChunk = [];
@@ -415,11 +415,11 @@ export class JsonCompressor {
       
       // Store objects on FIRST occurrence so second occurrence can reference it
       // We increment seenCount above, so after increment:
-      // seenCount=1: first occurrence (just incremented from 0 to 1), store it
-      // seenCount>=2: we already stored it on first occurrence, should be in dedup map
+      // seenCount=0: first occurrence (before increment), store it
+      // seenCount>=1: we already stored it on first occurrence, should be in dedup map
       // Note: This means we store proactively - first occurrence gets stored AND returned in full
       // Second+ occurrences will find it in the dedup map and return a reference
-      const isFirstOccurrence = seenCount === 1;
+      const isFirstOccurrence = seenCount === 0;
 
       // Process the object - apply low-signal detection
       const objWithLowSignalCompressed = this.compressObjectWithLowSignalDetection(obj, path);
@@ -453,7 +453,7 @@ export class JsonCompressor {
           result
         ).join(", ")}\nPreview: ${objectAsString.substring(0, 200)}...\n[Use ${
           this.toolName
-        } tool with key "${key}" to retrieve full content]`;
+        } tool with key "${key}" to retrieve full content]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
       }
       return result;
     }
@@ -486,7 +486,7 @@ export class JsonCompressor {
           200
         )}...\n[Use ${
           this.toolName
-        } tool with key "${key}" to retrieve full content]`;
+        } tool with key "${key}" to retrieve full content]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
       }
       return obj;
     }

package/src/services/EventService.ts

@@ -1,6 +1,8 @@
 import { EventEmitter } from "events";
 import { IAgent } from "../agents/interface";
 
+export type LogLevel = "info" | "warn" | "error";
+
 export type EventHandlerFn = (...args: any[]) => any;
 
 export interface EventHandler {
@@ -31,9 +33,38 @@ type ManagedListenerRecord = {
   blocking: boolean;
 };
 
+/**
+ * Default console handler for plugin:log events.
+ * Active when no renderer has taken over (e.g. worker mode, CLI before chat starts).
+ * Can be suppressed by calling suppressDefaultLogger() when a renderer is active.
+ *
+ * IMPORTANT: Uses process.stdout/stderr directly to avoid infinite recursion
+ * with logger.installConsoleOverload() which overrides console.log/warn.
+ */
+function defaultConsoleLogHandler(event: {
+  source: string;
+  message: string;
+  level: LogLevel;
+}): void {
+  const prefix = event.source ? `[${event.source}] ` : "";
+  const line = `${prefix}${event.message}\n`;
+  switch (event.level) {
+    case "warn":
+      process.stderr.write(line);
+      break;
+    case "error":
+      process.stderr.write(line);
+      break;
+    default:
+      process.stdout.write(line);
+  }
+}
+
 export class EventService extends EventEmitter {
   private blockingHandlers: Map<string, EventHandler[]> = new Map();
   private managedListeners: Map<string, ManagedListenerRecord> = new Map();
+  private defaultLoggerActive = true;
+  private boundDefaultLogHandler = defaultConsoleLogHandler;
 
   eventTypes = {
     agentMsg: "agent:msg",
@@ -45,6 +76,35 @@ export class EventService extends EventEmitter {
   constructor() {
     super();
     this.setMaxListeners(100);
+    // Register the default console logger so Events.log() always produces output
+    // even before a renderer is attached (worker mode, module loading, etc.)
+    this.on(this.eventTypes.pluginLog, this.boundDefaultLogHandler);
+  }
+
+  /**
+   * Suppress the default console logger.
+   * Call this when a renderer has taken over and will handle plugin:log events.
+   * This prevents double-printing when both the renderer and the default handler fire.
+   */
+  suppressDefaultLogger(): void {
+    if (this.defaultLoggerActive) {
+      this.removeListener(
+        this.eventTypes.pluginLog,
+        this.boundDefaultLogHandler
+      );
+      this.defaultLoggerActive = false;
+    }
+  }
+
+  /**
+   * Restore the default console logger.
+   * Call this when the renderer is torn down.
+   */
+  restoreDefaultLogger(): void {
+    if (!this.defaultLoggerActive) {
+      this.on(this.eventTypes.pluginLog, this.boundDefaultLogHandler);
+      this.defaultLoggerActive = true;
+    }
   }
 
   /**
@@ -232,7 +292,7 @@ export class EventService extends EventEmitter {
   log(
     source: string,
     message: string,
-    level: "info" | "warn" | "error" = "info"
+    level: LogLevel = "info"
   ): void {
     this.emit(this.eventTypes.pluginLog, {
       source,

package/src/services/KnowhowClient.ts

@@ -1,3 +1,4 @@
+import { createHash } from "crypto";
 import http from "../utils/http";
 import fs from "fs";
 import { Message } from "../clients/types";
@@ -171,11 +172,9 @@ export class KnowhowSimpleClient {
       try {
         this.jwtValidated = true;
         const response = await this.me();
-
         const user = response.data.user;
         const orgs = user.orgs;
         const orgId = response.data.orgId;
-
         const currentOrg = orgs.find((org) => {
           return org.organizationId === orgId;
         });
@@ -227,6 +226,17 @@ export class KnowhowSimpleClient {
     return presignedUrl;
   }
 
+  async getOrgEmbedding(id: string) {
+    await this.checkJwt();
+    const resp = await http.get(
+      `${this.baseUrl}/api/org-embeddings/${id}`,
+      {
+        headers: this.headers,
+      }
+    );
+    return resp.data as { id: string; modelName: string; name: string; [key: string]: unknown };
+  }
+
   async updateEmbeddingMetadata(
     id: string,
     data: {
@@ -654,8 +664,10 @@ export class KnowhowSimpleClient {
   /**
    * Get presigned S3 URL for uploading a file to Knowhow FS.
    * First finds or creates the file by path, then gets its upload URL.
+   * Computes SHA256 hash of the file content and stores it as S3 metadata
+   * so any client can determine if they already have this version without downloading.
    */
-  async getOrgFilePresignedUploadUrl(filePath: string): Promise<string> {
+  async getOrgFilePresignedUploadUrl(filePath: string, localFilePath?: string): Promise<string> {
     await this.checkJwt();
 
     // Find or create the file by path
@@ -666,10 +678,17 @@ export class KnowhowSimpleClient {
     const fileName =
       lastSlash >= 0 ? filePath.substring(lastSlash + 1) : filePath;
 
+    // Compute SHA256 hash if we have the local file path, so S3 stores it as metadata
+    let sha256Hash: string | undefined;
+    if (localFilePath) {
+      const fileContent = fs.readFileSync(localFilePath);
+      sha256Hash = createHash("sha256").update(fileContent).digest("base64");
+    }
+
     // Get upload URL using the file ID
     const response = await http.post<{ uploadUrl: string }>(
       `${this.baseUrl}/api/org-files/upload/${file.id}`,
-      { fileName },
+      { fileName, sha256Hash },
       { headers: this.headers }
     );
     return response.data.uploadUrl;
@@ -718,6 +737,17 @@ export class KnowhowSimpleClient {
     );
   }
 
+  /**
+   * Get a single cloud worker by ID
+   */
+  async getCloudWorker(id: string) {
+    await this.checkJwt();
+    return http.get<{ id: string; name: string; status: string; workerConfigJson?: Record<string, unknown> }>(
+      `${this.baseUrl}/api/cloud-workers/${id}`,
+      { headers: this.headers }
+    );
+  }
+
   /**
    * Update an existing cloud worker
    */

package/src/services/modules/index.ts

@@ -1,82 +1,126 @@
+import * as path from "path";
+import * as os from "os";
+
 import { getConfig, getGlobalConfig } from "../../config";
 import { KnowhowModule, ModuleContext } from "./types";
-import { ToolsService } from "../Tools";
 import { services } from "../";
-import * as path from "path";
+import { toUniqueArray } from "../../utils";
 
 export class ModulesService {
-  async loadModulesFromConfig(context?: ModuleContext) {
-    const config = await getConfig();
+  async getDefaultContext() {
+    return { ...services() };
+  }
 
+  async overrideDefaultContext(overrides: Partial<ModuleContext>) {
+    const defaultContext = await this.getDefaultContext();
+    return { ...defaultContext, ...overrides };
+  }
+
+  async loadModulesFrom(
+    config: { modules: string[] } & any,
+    context?: Partial<ModuleContext>
+  ) {
     // If no context provided, fall back to global singletons
     if (!context) {
-      const { Clients, Plugins, Agents, Tools, Embeddings, MediaProcessor } = services();
-      context = {
-        Agents,
-        Embeddings,
-        Plugins,
-        Clients,
-        Tools,
-        MediaProcessor,
-      };
+      context = { ...(await this.getDefaultContext()) };
     }
 
-    // Use the toolsService from context
-    const toolsService = context.Tools;
-    const agentService = context.Agents;
-    const pluginService = context.Plugins;
-    const clients = context.Clients;
+    const allModulePaths = config.modules;
 
-    // Load from global config (~/.knowhow/knowhow.json) first, then local config
-    const globalConfig = await getGlobalConfig();
-    const allModulePaths = [
-      ...(globalConfig.modules || []),
-      ...(config.modules || []),
+    // Search paths: .knowhow/node_modules first (where `knowhow modules install`
+    // puts packages), then cwd node_modules, then global node_modules.
+    // This allows modules installed via `knowhow modules install` to be found
+    // even when knowhow itself is installed globally.
+    const cwdPaths = (require as any).resolve
+      ? require.resolve.paths?.("") || []
+      : [];
+    const resolvePaths = [
+      path.join(process.cwd(), ".knowhow", "node_modules"),
+      path.join(os.homedir(), ".knowhow", "node_modules"),
+      path.join(process.cwd(), "node_modules"),
+      ...cwdPaths,
     ];
 
     for (const modulePath of allModulePaths) {
       // Resolve relative paths relative to process.cwd() so that paths like
       // "../../packages/knowhow-module-load-webpage" in knowhow.json work
       // regardless of where the compiled output lives.
-      const resolvedPath = modulePath.startsWith(".")
-        ? path.resolve(process.cwd(), modulePath)
-        : modulePath;
+      let resolvedPath: string;
+      if (modulePath.startsWith(".")) {
+        resolvedPath = path.resolve(process.cwd(), modulePath);
+      } else {
+        // For npm package names, try resolving from cwd first so locally-installed
+        // modules are found even when knowhow is installed globally.
+        try {
+          resolvedPath = require.resolve(modulePath, { paths: resolvePaths });
+        } catch {
+          resolvedPath = modulePath; // fall back to normal require resolution
+        }
+      }
       const rawModule = require(resolvedPath);
       const importedModule = (rawModule.default || rawModule) as KnowhowModule;
-      console.log(`🔌 Loading module: ${modulePath} (resolved: ${resolvedPath})`);
-      await importedModule.init({ config, cwd: process.cwd(), context });
-      console.log(`✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`);
+      context.Events?.log(
+        "ModulesService",
+        `🔌 Loading module: ${modulePath} (resolved: ${resolvedPath})`
+      );
+      await importedModule.init({
+        config,
+        cwd: process.cwd(),
+        context: context as ModuleContext,
+      });
+      context.Events?.log(
+        "ModulesService",
+        `✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`
+      );
 
-      for (const agent of importedModule.agents) {
-        agentService.registerAgent(agent);
+      // Only register tools/agents/plugins/clients if the relevant services
+      // are available in context (they may not be during early CLI command registration)
+      if (context.Agents) {
+        for (const agent of importedModule.agents) {
+          context.Agents.registerAgent(agent);
+        }
       }
 
-      for (const tool of importedModule.tools) {
-        toolsService.addTool(tool.definition);
-        toolsService.setFunction(tool.definition.function.name, tool.handler);
+      if (context.Tools) {
+        for (const tool of importedModule.tools) {
+          context.Tools.addTool(tool.definition);
+          context.Tools.setFunction(
+            tool.definition.function.name,
+            tool.handler
+          );
+        }
       }
 
-      for (const plugin of importedModule.plugins) {
-        const pluginContext = {
-          Agents: agentService,
-          Clients: clients,
-          Tools: toolsService,
-          Plugins: pluginService,
-          ...(context.MediaProcessor ? { MediaProcessor: context.MediaProcessor } : {}),
-        };
-        pluginService.registerPlugin(plugin.name, new plugin.plugin(pluginContext as any));
+      if (context.Plugins) {
+        for (const plugin of importedModule.plugins) {
+          context.Plugins.registerPlugin(
+            plugin.name,
+            new plugin.plugin(context as any)
+          );
+        }
       }
 
-      for (const client of importedModule.clients) {
-        clients.registerClient(client.provider, client.client);
-        clients.registerModels(client.provider, client.models);
+      if (context.Clients) {
+        for (const client of importedModule.clients) {
+          context.Clients.registerClient(client.provider, client.client);
+          context.Clients.registerModels(client.provider, client.models);
+        }
       }
     }
+  }
 
-    // Also load plugins directly from config's pluginPackages map
-    if (pluginService) {
-      await pluginService.loadPluginsFromConfig(config);
-      await pluginService.loadPluginsFromConfig(globalConfig);
-    }
+  async loadModulesFromConfig(context?: ModuleContext) {
+    const config = await getConfig();
+
+    const globalConfig = await getGlobalConfig();
+    const allModulePaths = [
+      ...(globalConfig.modules || []),
+      ...(config.modules || []),
+    ];
+
+    return this.loadModulesFrom(
+      { ...config, modules: toUniqueArray(allModulePaths) },
+      context
+    );
   }
 }