@tyvm/knowhow 0.0.108 → 0.0.109-dev.38b1faa

package/src/logger.ts

@@ -0,0 +1,197 @@
+import type { LogLevel } from "./services/EventService";
+
+/**
+ * App-wide logger utility.
+ *
+ * Features:
+ * 1. `logger.info/warn/error(source, message)` — routes through EventService
+ * 2. `Logger.of("ClassName")` — creates a bound logger so you don't repeat the source
+ * 3. `logger.installConsoleOverload()` — replaces console.log/warn/error/info with
+ *    our closure, so ALL output (including third-party modules) goes through us
+ * 4. `logger.silence()` / `logger.unsilence()` — suppress all output, useful for
+ *    commands that need clean stdout (e.g. github-credentials)
+ *
+ * Usage (module-level):
+ *   import { logger } from "../logger";
+ *   logger.info("MyService", "Something happened");
+ *
+ * Usage (class-level):
+ *   import { Logger } from "../logger";
+ *   class MyClass {
+ *     private logger = Logger.of("MyClass");
+ *     doThing() { this.logger.info("Something happened"); }
+ *   }
+ *
+ * Silence mode (for clean-stdout commands):
+ *   logger.silence();   // suppress everything
+ *   // ... do work that must produce clean stdout ...
+ *   logger.unsilence(); // restore
+ */
+
+// ---- Internal state ---------------------------------------------------------
+
+let silenced = false;
+
+// Original console methods — saved before any overload is installed
+const _originalConsole = {
+  log: console.log.bind(console),
+  warn: console.warn.bind(console),
+  error: console.error.bind(console),
+  info: console.info.bind(console),
+};
+
+let consoleOverloadInstalled = false;
+
+// ---- EventService lazy accessor ---------------------------------------------
+
+function getEvents() {
+  try {
+    const { services } = require("./services") as typeof import("./services");
+    return services().Events;
+  } catch {
+    return null;
+  }
+}
+
+// ---- Core emit logic --------------------------------------------------------
+
+function emit(source: string, message: string, level: LogLevel): void {
+  if (silenced) return;
+
+  try {
+    const events = getEvents();
+    if (events) {
+      events.log(source, message, level);
+      return;
+    }
+  } catch {
+    // fall through to direct console output
+  }
+
+  // Fallback: use original console methods (bypasses any overload we installed)
+  const prefix = source ? `[${source}] ` : "";
+  if (level === "warn") _originalConsole.warn(`${prefix}${message}`);
+  else if (level === "error") _originalConsole.error(`${prefix}${message}`);
+  else _originalConsole.log(`${prefix}${message}`);
+}
+
+// ---- Bound logger (returned by Logger.of) -----------------------------------
+
+export interface BoundLogger {
+  log(message: string, level?: LogLevel): void;
+  info(message: string): void;
+  warn(message: string): void;
+  error(message: string): void;
+}
+
+function makeBoundLogger(source: string): BoundLogger {
+  return {
+    log(message: string, level: LogLevel = "info"): void {
+      emit(source, message, level);
+    },
+    info(message: string): void {
+      emit(source, message, "info");
+    },
+    warn(message: string): void {
+      emit(source, message, "warn");
+    },
+    error(message: string): void {
+      emit(source, message, "error");
+    },
+  };
+}
+
+// ---- Public API -------------------------------------------------------------
+
+export const logger = {
+  log(source: string, message: string, level: LogLevel = "info"): void {
+    emit(source, message, level);
+  },
+
+  info(source: string, message: string): void {
+    emit(source, message, "info");
+  },
+
+  warn(source: string, message: string): void {
+    emit(source, message, "warn");
+  },
+
+  error(source: string, message: string): void {
+    emit(source, message, "error");
+  },
+
+  /**
+   * Suppress all log output. Useful for commands that need clean stdout
+   * (e.g. git credential helpers). All logger.* calls and overloaded
+   * console.* calls become no-ops until unsilence() is called.
+   */
+  silence(): void {
+    silenced = true;
+  },
+
+  /**
+   * Restore log output after a silence() call.
+   */
+  unsilence(): void {
+    silenced = false;
+  },
+
+  /**
+   * Returns true if the logger is currently silenced.
+   */
+  isSilenced(): boolean {
+    return silenced;
+  },
+
+  /**
+   * Install console overload. After this call, console.log/warn/error/info
+   * all route through our closure (respecting silence mode).
+   * Safe to call multiple times — only installs once.
+   *
+   * Call this early in CLI startup (before any modules are loaded) to ensure
+   * third-party module logs don't bypass the silence mechanism.
+   */
+  installConsoleOverload(): void {
+    if (consoleOverloadInstalled) return;
+    consoleOverloadInstalled = true;
+
+    const route = (originalFn: (...args: any[]) => void, args: any[]) => {
+      if (silenced) return;
+      originalFn(...args);
+    };
+
+    console.log = (...args: any[]) => route(_originalConsole.log, args);
+    console.info = (...args: any[]) => route(_originalConsole.info, args);
+    console.warn = (...args: any[]) => route(_originalConsole.warn, args);
+    // Note: console.error is intentionally NOT overloaded — real errors (stack
+    // traces, crash reports) should always be visible. Only suppress via silence().
+    // If you want to suppress errors too, call logger.silence() which checks the flag
+    // before the overloaded console.warn/log routes reach here anyway.
+  },
+
+  /**
+   * Remove the console overload and restore original console methods.
+   */
+  uninstallConsoleOverload(): void {
+    if (!consoleOverloadInstalled) return;
+    console.log = _originalConsole.log;
+    console.info = _originalConsole.info;
+    console.warn = _originalConsole.warn;
+    consoleOverloadInstalled = false;
+  },
+};
+
+/**
+ * Factory for creating a bound logger with a fixed source name.
+ * Ideal for class-level loggers:
+ *
+ *   class MyClass {
+ *     private logger = Logger.of("MyClass");
+ *     doThing() { this.logger.info("hello"); }
+ *   }
+ */
+export const Logger = {
+  of(source: string): BoundLogger {
+    return makeBoundLogger(source);
+  },
+};

package/src/plugins/embedding.ts

@@ -20,9 +20,12 @@ export class EmbeddingPlugin extends PluginBase {
 
   constructor(context) {
     super(context);
-    
+
     // Subscribe to file:post-edit events
-    this.context.Events.on("file:post-edit", this.handleFilePostEdit.bind(this));
+    this.context.Events.on(
+      "file:post-edit",
+      this.handleFilePostEdit.bind(this)
+    );
   }
 
   async embed() {
@@ -68,10 +71,12 @@ export class EmbeddingPlugin extends PluginBase {
       this.log(`Reading entry ${entry.id}`);
     }
 
-    const contextLength = JSON.stringify(context).split(" ").length;
+    const ids = context.map((entry) => entry.id);
+
+    const contextLength = JSON.stringify(ids).split(" ").length;
     this.log(`Found ${context.length} entries. Loading ${contextLength} words`);
 
-    return `EMBEDDING PLUGIN: Our knowledgebase contains this information which can be used to answer the question:
-    ${JSON.stringify(context)}`;
+    return `EMBEDDING PLUGIN: Our knowledgebase indicates these embedding entries may be related to the question:
+    ${JSON.stringify(ids)}`;
   }
-}
+}

package/src/plugins/plugins.ts

@@ -1,5 +1,4 @@
 import { Plugin, PluginContext } from "./types";
-import { Config } from "../types";
 import { VimPlugin } from "./vim";
 import { LinterPlugin } from "./LinterPlugin";
 import { LanguagePlugin } from "./language";
@@ -49,26 +48,6 @@ export class PluginService {
     return instance.meta.key;
   }
 
-  /**
-   * Load plugins from config's pluginPackages map.
-   * Each entry maps a plugin key to an npm package name or file path.
-   * Errors are caught and logged as warnings without crashing.
-   */
-  async loadPluginsFromConfig(config: Config): Promise<void> {
-    const pluginPackages = config.pluginPackages || {};
-    for (const [key, spec] of Object.entries(pluginPackages)) {
-      try {
-        await this.loadPlugin(spec);
-      } catch (error) {
-        this.events?.log(
-          "PluginService",
-          `Failed to load plugin "${key}" from "${spec}": ${error instanceof Error ? error.message : error}`,
-          "warn"
-        );
-      }
-    }
-  }
-
   /** Disable a plugin by its key; returns `true` if found. */
   disablePlugin(key: string): boolean {
     const p = this.pluginMap.get(key);

package/src/plugins/vim.ts

@@ -73,22 +73,11 @@ export class VimPlugin extends PluginBase {
 
   async call() {
     const vimFiles = await this.getVimFiles();
-    const fileContents = await Promise.all(
-      vimFiles.map(async (f) => {
-        const loaded = await this.getFileContents(f);
-
-        const preview =
-          loaded.content.length > 1000
-            ? loaded.content.slice(0, 1000) +
-              "... file trimmed, read file for full content"
-            : loaded.content;
-
-        return {
-          sourceFile: loaded.filePath,
-          content: loaded.content.slice(0, 1000),
-        };
-      })
-    );
+    const fileContents = vimFiles.map((f) => {
+      return {
+        sourceFile: f,
+      };
+    });
     if (fileContents.length === 0) {
       return "VIM PLUGIN: No files open in vim";
     }

package/src/processors/JsonCompressor.ts

@@ -382,7 +382,7 @@ export class JsonCompressor {
             i + currentChunk.length - 1
           }]\nPreview: ${chunkString.substring(0, 100)}...\n[Use ${
             this.toolName
-          } tool with key "${key}" to retrieve this chunk]`;
+          } tool with key "${key}" to retrieve this chunk]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
           finalArray.unshift(stub); // Add stub to the start of our final result.
 
           currentChunk = [];
@@ -415,11 +415,11 @@ export class JsonCompressor {
       
       // Store objects on FIRST occurrence so second occurrence can reference it
       // We increment seenCount above, so after increment:
-      // seenCount=1: first occurrence (just incremented from 0 to 1), store it
-      // seenCount>=2: we already stored it on first occurrence, should be in dedup map
+      // seenCount=0: first occurrence (before increment), store it
+      // seenCount>=1: we already stored it on first occurrence, should be in dedup map
       // Note: This means we store proactively - first occurrence gets stored AND returned in full
       // Second+ occurrences will find it in the dedup map and return a reference
-      const isFirstOccurrence = seenCount === 1;
+      const isFirstOccurrence = seenCount === 0;
 
       // Process the object - apply low-signal detection
       const objWithLowSignalCompressed = this.compressObjectWithLowSignalDetection(obj, path);
@@ -453,7 +453,7 @@ export class JsonCompressor {
           result
         ).join(", ")}\nPreview: ${objectAsString.substring(0, 200)}...\n[Use ${
           this.toolName
-        } tool with key "${key}" to retrieve full content]`;
+        } tool with key "${key}" to retrieve full content]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
       }
       return result;
     }
@@ -486,7 +486,7 @@ export class JsonCompressor {
           200
         )}...\n[Use ${
           this.toolName
-        } tool with key "${key}" to retrieve full content]`;
+        } tool with key "${key}" to retrieve full content]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map without repeated ${this.toolName} calls - especially useful for JSON data]`;
       }
       return obj;
     }

package/src/services/EventService.ts

@@ -1,6 +1,8 @@
 import { EventEmitter } from "events";
 import { IAgent } from "../agents/interface";
 
+export type LogLevel = "info" | "warn" | "error";
+
 export type EventHandlerFn = (...args: any[]) => any;
 
 export interface EventHandler {
@@ -31,9 +33,38 @@ type ManagedListenerRecord = {
   blocking: boolean;
 };
 
+/**
+ * Default console handler for plugin:log events.
+ * Active when no renderer has taken over (e.g. worker mode, CLI before chat starts).
+ * Can be suppressed by calling suppressDefaultLogger() when a renderer is active.
+ *
+ * IMPORTANT: Uses process.stdout/stderr directly to avoid infinite recursion
+ * with logger.installConsoleOverload() which overrides console.log/warn.
+ */
+function defaultConsoleLogHandler(event: {
+  source: string;
+  message: string;
+  level: LogLevel;
+}): void {
+  const prefix = event.source ? `[${event.source}] ` : "";
+  const line = `${prefix}${event.message}\n`;
+  switch (event.level) {
+    case "warn":
+      process.stderr.write(line);
+      break;
+    case "error":
+      process.stderr.write(line);
+      break;
+    default:
+      process.stdout.write(line);
+  }
+}
+
 export class EventService extends EventEmitter {
   private blockingHandlers: Map<string, EventHandler[]> = new Map();
   private managedListeners: Map<string, ManagedListenerRecord> = new Map();
+  private defaultLoggerActive = true;
+  private boundDefaultLogHandler = defaultConsoleLogHandler;
 
   eventTypes = {
     agentMsg: "agent:msg",
@@ -45,6 +76,35 @@ export class EventService extends EventEmitter {
   constructor() {
     super();
     this.setMaxListeners(100);
+    // Register the default console logger so Events.log() always produces output
+    // even before a renderer is attached (worker mode, module loading, etc.)
+    this.on(this.eventTypes.pluginLog, this.boundDefaultLogHandler);
+  }
+
+  /**
+   * Suppress the default console logger.
+   * Call this when a renderer has taken over and will handle plugin:log events.
+   * This prevents double-printing when both the renderer and the default handler fire.
+   */
+  suppressDefaultLogger(): void {
+    if (this.defaultLoggerActive) {
+      this.removeListener(
+        this.eventTypes.pluginLog,
+        this.boundDefaultLogHandler
+      );
+      this.defaultLoggerActive = false;
+    }
+  }
+
+  /**
+   * Restore the default console logger.
+   * Call this when the renderer is torn down.
+   */
+  restoreDefaultLogger(): void {
+    if (!this.defaultLoggerActive) {
+      this.on(this.eventTypes.pluginLog, this.boundDefaultLogHandler);
+      this.defaultLoggerActive = true;
+    }
   }
 
   /**
@@ -232,7 +292,7 @@ export class EventService extends EventEmitter {
   log(
     source: string,
     message: string,
-    level: "info" | "warn" | "error" = "info"
+    level: LogLevel = "info"
   ): void {
     this.emit(this.eventTypes.pluginLog, {
       source,

package/src/services/KnowhowClient.ts

@@ -1,3 +1,4 @@
+import { createHash } from "crypto";
 import http from "../utils/http";
 import fs from "fs";
 import { Message } from "../clients/types";
@@ -171,11 +172,9 @@ export class KnowhowSimpleClient {
       try {
         this.jwtValidated = true;
         const response = await this.me();
-
         const user = response.data.user;
         const orgs = user.orgs;
         const orgId = response.data.orgId;
-
         const currentOrg = orgs.find((org) => {
           return org.organizationId === orgId;
         });
@@ -227,6 +226,17 @@ export class KnowhowSimpleClient {
     return presignedUrl;
   }
 
+  async getOrgEmbedding(id: string) {
+    await this.checkJwt();
+    const resp = await http.get(
+      `${this.baseUrl}/api/org-embeddings/${id}`,
+      {
+        headers: this.headers,
+      }
+    );
+    return resp.data as { id: string; modelName: string; name: string; [key: string]: unknown };
+  }
+
   async updateEmbeddingMetadata(
     id: string,
     data: {
@@ -654,8 +664,10 @@ export class KnowhowSimpleClient {
   /**
    * Get presigned S3 URL for uploading a file to Knowhow FS.
    * First finds or creates the file by path, then gets its upload URL.
+   * Computes SHA256 hash of the file content and stores it as S3 metadata
+   * so any client can determine if they already have this version without downloading.
    */
-  async getOrgFilePresignedUploadUrl(filePath: string): Promise<string> {
+  async getOrgFilePresignedUploadUrl(filePath: string, localFilePath?: string): Promise<string> {
     await this.checkJwt();
 
     // Find or create the file by path
@@ -666,10 +678,17 @@ export class KnowhowSimpleClient {
     const fileName =
       lastSlash >= 0 ? filePath.substring(lastSlash + 1) : filePath;
 
+    // Compute SHA256 hash if we have the local file path, so S3 stores it as metadata
+    let sha256Hash: string | undefined;
+    if (localFilePath) {
+      const fileContent = fs.readFileSync(localFilePath);
+      sha256Hash = createHash("sha256").update(fileContent).digest("base64");
+    }
+
     // Get upload URL using the file ID
     const response = await http.post<{ uploadUrl: string }>(
       `${this.baseUrl}/api/org-files/upload/${file.id}`,
-      { fileName },
+      { fileName, sha256Hash },
       { headers: this.headers }
     );
     return response.data.uploadUrl;
@@ -718,6 +737,17 @@ export class KnowhowSimpleClient {
     );
   }
 
+  /**
+   * Get a single cloud worker by ID
+   */
+  async getCloudWorker(id: string) {
+    await this.checkJwt();
+    return http.get<{ id: string; name: string; status: string; workerConfigJson?: Record<string, unknown> }>(
+      `${this.baseUrl}/api/cloud-workers/${id}`,
+      { headers: this.headers }
+    );
+  }
+
   /**
    * Update an existing cloud worker
    */

package/src/services/MediaProcessorService.ts

@@ -4,6 +4,7 @@ import { exec } from "child_process";
 import { promisify } from "util";
 import { fileExists, readFile, mkdir } from "../utils";
 import { AIClient } from "../clients";
+import { Models } from "../types";
 
 const execPromise = promisify(exec);
 
@@ -36,7 +37,7 @@ export interface KeyframeInfo {
  * audio/video processing steps after downloading with ytdl.
  */
 export class MediaProcessorService {
-  constructor(private clients: any) {}
+  constructor(private clients: AIClient) {}
 
   /**
    * Split an audio/video file into fixed-length mp3 chunks using ffmpeg.
@@ -298,8 +299,9 @@ export class MediaProcessorService {
     });
     const image = `data:image/jpeg;base64,${base64}`;
     return this.clients.createCompletion("openai", {
-      model: "gpt-4o",
+      model: Models.openai.GPT_54_Nano,
       max_tokens: 2500,
+      timeout: 20000,
       messages: [
         {
           role: "user",