@tyvm/knowhow 0.0.47 → 0.0.49

package/src/processors/CustomVariables.ts

@@ -101,18 +101,24 @@ export class CustomVariables {
    */
   private listVariables(): string {
     const variableNames = Object.keys(this.variables);
-    
+
     if (variableNames.length === 0) {
       return "No variables are currently stored.";
     }
 
-    const variableList = variableNames.map(name => {
-      const value = this.variables[name];
-      const preview = typeof value === "string" 
-        ? (value.length > 50 ? value.substring(0, 50) + "..." : value)
-        : JSON.stringify(value).substring(0, 50) + (JSON.stringify(value).length > 50 ? "..." : "");
-      return `- ${name}: ${preview}`;
-    }).join("\n");
+    const variableList = variableNames
+      .map((name) => {
+        const value = this.variables[name];
+        const preview =
+          typeof value === "string"
+            ? value.length > 50
+              ? value.substring(0, 50) + "..."
+              : value
+            : JSON.stringify(value).substring(0, 50) +
+              (JSON.stringify(value).length > 50 ? "..." : "");
+        return `- ${name}: ${preview}`;
+      })
+      .join("\n");
 
     return `Currently stored variables (${variableNames.length}):\n${variableList}`;
   }
@@ -136,14 +142,28 @@ export class CustomVariables {
   }
 
   /**
-   * Recursively processes a value to substitute {{variable}} patterns
+   * Processes a value to substitute {{variableName}} patterns
    */
   private substituteVariables(
     value: any,
     processedVars: Set<string> = new Set()
   ): any {
     if (typeof value === "string") {
-      // Find all {{variable}} patterns
+      // Check if ALL variables are undefined - if so, return just the error message for the first one
+      const variableMatches = value.match(/\{\{([a-zA-Z0-9_]+)\}\}/g);
+      if (variableMatches) {
+        const allUndefined = variableMatches.every((match) => {
+          const varName = match.replace(/[{}]/g, "");
+          return !(varName in this.variables);
+        });
+
+        if (allUndefined && variableMatches.length > 0) {
+          const firstUndefinedVar = variableMatches[0].replace(/[{}]/g, "");
+          return `{{ERROR: Variable "${firstUndefinedVar}" is not defined}}`;
+        }
+      }
+
+      // Otherwise, proceed with partial substitution
       return value.replace(/\{\{([a-zA-Z0-9_]+)\}\}/g, (match, varName) => {
         // Prevent infinite recursion
         if (processedVars.has(varName)) {
@@ -156,11 +176,9 @@ export class CustomVariables {
 
         const varValue = this.variables[varName];
 
-        // If the variable value is a string, recursively process it
+        // For nested variables, return the raw value without further processing
         if (typeof varValue === "string") {
-          const newProcessedVars = new Set(processedVars);
-          newProcessedVars.add(varName);
-          return this.substituteVariables(varValue, newProcessedVars);
+          return varValue;
         }
 
         // For non-string values, convert to JSON
@@ -239,62 +257,34 @@ export class CustomVariables {
    */
   private registerTools(toolsService: ToolsService): void {
     // Register setVariable tool
-    if (!toolsService.getTool(this.setVariableToolName)) {
-      toolsService.addTool(setVariableToolDefinition);
-      toolsService.addFunctions({
-        [this.setVariableToolName]: (name: string, contents: any) => {
-          return this.setVariable(name, contents);
-        },
-      });
-    }
-
-    // Register getVariable tool
-    if (!toolsService.getTool(this.getVariableToolName)) {
-      toolsService.addTool(getVariableToolDefinition);
-      toolsService.addFunctions({
-        [this.getVariableToolName]: (varName: string) => {
-          return this.getVariable(varName);
-        },
-      });
-    }
-
-    // Register storeToolCallToVariable tool
-    if (!toolsService.getTool(this.storeToolCallToolName)) {
-      toolsService.addTool(storeToolCallToVariableDefinition);
-      toolsService.addFunctions({
-        [this.storeToolCallToolName]: async (
-          varName: string,
-          toolName: string,
-          toolArgs: string
-        ) => {
-          return await this.storeToolCallToVariable(
-            varName,
-            toolName,
-            toolArgs
-          );
-        },
-      });
-    }
-
-    // Register listVariables tool
-    if (!toolsService.getTool(this.listVariablesToolName)) {
-      toolsService.addTool(listVariablesToolDefinition);
-      toolsService.addFunctions({
-        [this.listVariablesToolName]: () => {
-          return this.listVariables();
-        },
-      });
-    }
-
-    // Register deleteVariable tool
-    if (!toolsService.getTool(this.deleteVariableToolName)) {
-      toolsService.addTool(deleteVariableToolDefinition);
-      toolsService.addFunctions({
-        [this.deleteVariableToolName]: (varName: string) => {
-          return this.deleteVariable(varName);
-        },
-      });
-    }
+    toolsService.addTools([
+      setVariableToolDefinition,
+      getVariableToolDefinition,
+      storeToolCallToVariableDefinition,
+      listVariablesToolDefinition,
+      deleteVariableToolDefinition,
+    ]);
+    toolsService.addFunctions({
+      [this.setVariableToolName]: (name: string, contents: any) => {
+        return this.setVariable(name, contents);
+      },
+      [this.getVariableToolName]: (varName: string) => {
+        return this.getVariable(varName);
+      },
+      [this.storeToolCallToolName]: async (
+        varName: string,
+        toolName: string,
+        toolArgs: string
+      ) => {
+        return await this.storeToolCallToVariable(varName, toolName, toolArgs);
+      },
+      [this.listVariablesToolName]: () => {
+        return this.listVariables();
+      },
+      [this.deleteVariableToolName]: (varName: string) => {
+        return this.deleteVariable(varName);
+      },
+    });
   }
 
   /**

package/src/processors/TokenCompressor.ts

@@ -54,10 +54,6 @@ export class TokenCompressor {
    * Compresses a string into chunks from the end, creating a chain of references
    */
   public compressStringInChunks(content: string, path: string = ""): string {
-    if (path === "" && this.estimateTokens(content) <= this.maxTokens) {
-      return content;
-    }
-
     if (content.length <= this.characterLimit) {
       return content;
     }
@@ -80,10 +76,14 @@ export class TokenCompressor {
     }
 
     // Store chunks and create chain of references
-    for (let i = chunks.length - 1; i >= 0; i--) {
+    // First, generate all keys
+    for (let i = 0; i < chunks.length; i++) {
       const key = this.generateKey();
-      chunkKeys.unshift(key);
+      chunkKeys.push(key);
+    }
 
+    // Then store chunks with proper linking
+    for (let i = 0; i < chunks.length; i++) {
       let chunkContent = chunks[i];
 
       // Add reference to next chunk if it exists
@@ -92,7 +92,7 @@ export class TokenCompressor {
         chunkContent += `\n\n[NEXT_CHUNK_KEY: ${nextKey}]`;
       }
 
-      this.storeString(key, chunkContent);
+      this.storeString(chunkKeys[i], chunkContent);
     }
 
     // Return reference to the first chunk
@@ -114,11 +114,12 @@ export class TokenCompressor {
   public compressContent(content: string, path: string = ""): string {
     const tokens = this.estimateTokens(content);
 
-    if (path === "" && tokens <= this.maxTokens) {
-      return content;
-    }
+    // For nested properties (path !== ""), use maxTokens to avoid recompressing stored data
+    // For top-level content (path === ""), use compressionThreshold to determine compression
+    const thresholdToUse =
+      path === "" ? this.compressionThreshold : this.maxTokens;
 
-    if (tokens <= this.compressionThreshold) {
+    if (tokens <= thresholdToUse) {
       return content;
     }
 
@@ -239,7 +240,7 @@ export class TokenCompressor {
 
       // If not JSON or compression wasn't effective, handle as regular string
       const tokens = this.estimateTokens(obj);
-      if (tokens > this.characterLimit * 4) {
+      if (tokens > this.compressionThreshold) {
         const key = this.generateKey();
         this.storeString(key, obj);
 
@@ -320,8 +321,8 @@ export class TokenCompressor {
   }
 
   registerTool(toolsService?: ToolsService): void {
-    if (toolsService && !toolsService.getTool(this.toolName)) {
-      toolsService.addTool(expandTokensDefinition);
+    if (toolsService) {
+      toolsService.addTools([expandTokensDefinition]);
       toolsService.addFunctions({
         [this.toolName]: (key: string) => {
           const data = this.retrieveString(key);

package/src/processors/ToolResponseCache.ts

@@ -41,7 +41,7 @@ export class ToolResponseCache {
   /**
    * Recursively searches for JSON strings within an object and parses them
    */
-  private parseNestedJsonStrings(obj: any): any {
+  public parseNestedJsonStrings(obj: any): any {
     if (typeof obj === "string") {
       const parsed = this.tryParseJson(obj);
       if (parsed) {
@@ -68,7 +68,7 @@ export class ToolResponseCache {
   /**
    * Stores a tool response for later manipulation
    */
-  private storeToolResponse(content: string, toolCallId: string): void {
+  public storeToolResponse(content: string, toolCallId: string): void {
     // Always store the original content for later JQ manipulation
     this.storage[toolCallId] = content;
 
@@ -130,17 +130,25 @@ export class ToolResponseCache {
     }
 
     try {
-      // Parse the data as JSON (handles nested JSON strings)
-      const parsedData = this.parseNestedJsonStrings(data);
+      // First parse the stored string as JSON, then handle nested JSON strings
+      const jsonData = this.tryParseJson(data);
+      if (!jsonData) {
+        return `Error: Tool response data is not valid JSON for toolCallId "${toolCallId}"`;
+      }
+      const parsedData = this.parseNestedJsonStrings(jsonData);
 
       // Execute JQ query
       const result = await jq.run(jqQuery, parsedData, { input: "json" });
 
-      // Return the result as a string
+      // Handle the result based on its type
       if (typeof result === "string") {
         return result;
+      } else if (typeof result === "number" || typeof result === "boolean") {
+        return String(result);
+      } else if (result === null) {
+        return "null";
       } else {
-        return JSON.stringify(result, null, 2);
+        return JSON.stringify(result);
       }
     } catch (error: any) {
       // If JQ fails, try to provide helpful error message
@@ -198,14 +206,12 @@ export class ToolResponseCache {
    * Registers the jqToolResponse tool with the ToolsService
    */
   registerTool(toolsService: ToolsService): void {
-    if (!toolsService.getTool(this.toolName)) {
-      toolsService.addTool(jqToolResponseDefinition);
-      toolsService.addFunctions({
-        [this.toolName]: async (toolCallId: string, jqQuery: string) => {
-          return await this.queryToolResponse(toolCallId, jqQuery);
-        },
-      });
-    }
+    toolsService.addTools([jqToolResponseDefinition]);
+    toolsService.addFunctions({
+      [this.toolName]: async (toolCallId: string, jqQuery: string) => {
+        return await this.queryToolResponse(toolCallId, jqQuery);
+      },
+    });
   }
 }
 

package/src/services/EmbeddingService.ts

@@ -1,3 +1,6 @@
+import * as path from "path";
+import { globSync } from "glob";
+
 import {
   Config,
   Embeddable,
@@ -13,10 +16,8 @@ import {
   cosineSimilarity,
 } from "../utils";
 import { summarizeTexts, chunkText } from "../ai";
-import { Plugins } from "../plugins/plugins";
 import { Clients, GenericClient } from "../clients";
-import * as path from "path";
-import glob from "glob";
+import { PluginService } from "..//plugins/plugins";
 
 export type CreateEmbeddingOptions = {
   provider?: string;
@@ -31,13 +32,21 @@ export type EmbedOptions = {
   minLength?: number;
 };
 
+interface EmbeddingServiceContext {
+  config: Config;
+  embeddingClient?: GenericClient;
+  plugins: PluginService;
+}
+
 export class EmbeddingService {
   private config: Config;
   private embeddingClient: GenericClient;
+  private plugins: PluginService;
 
-  constructor(config: Config, embeddingClient: GenericClient) {
-    this.config = config;
-    this.embeddingClient = embeddingClient;
+  constructor(protected context: EmbeddingServiceContext) {
+    this.config = context.config;
+    this.embeddingClient = context.embeddingClient;
+    this.plugins = context.plugins;
   }
 
   setEmbeddingClient(client: GenericClient) {
@@ -102,7 +111,7 @@ export class EmbeddingService {
     }
 
     console.log("Embedding", source.input, "to", source.output);
-    let files = await glob.sync(source.input, { ignore: ignorePattern });
+    let files = await globSync(source.input, { ignore: ignorePattern });
 
     if (source.kind && files.length === 0) {
       files = [source.input];
@@ -193,9 +202,9 @@ export class EmbeddingService {
     const contents = "";
     const ids = [];
 
-    if (Plugins.isPlugin(kind)) {
+    if (this.plugins.isPlugin(kind)) {
       console.log("Embedding with plugin", kind);
-      return Plugins.embed(kind, input);
+      return this.plugins.embed(kind, input);
     }
     switch (kind) {
       case "text":

package/src/services/EventService.ts

@@ -1,11 +1,91 @@
 import { EventEmitter } from "events";
 import { IAgent } from "../agents/interface";
 
+export interface EventHandler {
+  handler: (...args: any[]) => any;
+}
+
 export class EventService extends EventEmitter {
+  private blockingHandlers: Map<string, EventHandler[]> = new Map();
+
   constructor() {
     super();
   }
 
+  /**
+   * Register an event handler as blocking or non-blocking
+   * @param event The event name
+   * @param handler The event handler function
+   * @param isBlocking Whether this handler should be blocking
+   */
+  onBlocking(event: string, handler: (...args: any[]) => any): void {
+    if (!this.blockingHandlers.has(event)) {
+      this.blockingHandlers.set(event, []);
+    }
+
+    this.blockingHandlers.get(event)!.push({ handler });
+  }
+
+  /**
+   * Emit a blocking event - if any blocking handler throws, execution stops
+   * @param event The event name
+   * @param args Arguments to pass to handlers
+   * @returns Promise that resolves with array of handler results when all handlers complete, or rejects if any blocking handler throws
+   */
+  async emitBlocking(event: string, ...args: any[]): Promise<any[]> {
+    const results: any[] = [];
+
+    const handlers = this.blockingHandlers.get(event) || [];
+
+    for (const { handler } of handlers) {
+      try {
+        const result = handler(...args);
+        if (result instanceof Promise) {
+          const awaitedResult = await result;
+          results.push(awaitedResult);
+        } else {
+          results.push(result);
+        }
+      } catch (error) {
+        // If this is a blocking handler and it throws, stop execution
+        throw error;
+      }
+    }
+
+    this.emit(event, ...args);
+    return results.filter((r) => Boolean(r));
+  }
+
+  /**
+   * Emit a non-blocking event - all handlers run, errors are logged but don't stop execution
+   * @param event The event name
+   * @param args Arguments to pass to handlers
+   * @returns Promise that resolves with array of handler results when all handlers complete
+   */
+  async emitNonBlocking(event: string, ...args: any[]): Promise<any[]> {
+    const handlers = this.blockingHandlers.get(event) || [];
+    const results: any[] = [];
+    for (const { handler } of handlers) {
+      try {
+        const result = handler(...args);
+        if (result instanceof Promise) {
+          const awaitedResult = await result;
+          results.push(awaitedResult);
+        } else {
+          results.push(result);
+        }
+      } catch (error) {
+        console.error(
+          `Non-blocking handler error for event '${event}':`,
+          error
+        );
+      }
+    }
+
+    this.emit(event, ...args);
+    return results.filter((r) => Boolean(r));
+  }
+
   registerAgent(agent: IAgent): void {
     this.emit("agents:register", { name: agent.name, agent });
   }

package/src/services/Mcp.ts

@@ -2,6 +2,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"
 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import Anthropic from "@anthropic-ai/sdk";
 
+import fs from "fs";
 import { McpConfig } from "../types";
 import { Tool } from "../clients";
 import { getConfig } from "../config";
@@ -79,6 +80,10 @@ export class McpService {
       }
       if (mcp.url) {
         // TODO: also support refresh tokens
+        if (mcp.authorization_token_file) {
+          const token = fs.readFileSync(mcp.authorization_token_file, "utf-8");
+          mcp.authorization_token = token.trim();
+        }
         return new StreamableHTTPClientTransport(new URL(mcp.url), {
           requestInit: {
             headers: {

package/src/services/S3.ts

@@ -8,7 +8,8 @@ import {
   PutObjectCommandInput,
 } from "@aws-sdk/client-s3";
 
-import fs from "fs/promises";
+import * as fs from "fs";
+const fsPromises = fs.promises;
 import { createWriteStream } from "fs";
 import * as path from "path";
 import { pipeline } from "stream";
@@ -30,7 +31,7 @@ export class S3Service {
     bucketName: string,
     key: string
   ): Promise<void> {
-    const fileContent = await fs.readFile(filePath);
+    const fileContent = await fsPromises.readFile(filePath);
 
     const params = {
       Bucket: bucketName,
@@ -80,7 +81,7 @@ export class S3Service {
   ): Promise<void> {
     try {
       const fileStream = createReadStream(filePath);
-      const fileStats = await fs.stat(filePath);
+      const fileStats = await fsPromises.stat(filePath);
 
       const response = await axios.put(presignedUrl, fileStream, {
         headers: {