@tyvm/knowhow 0.0.56 → 0.0.59

package/src/processors/tools/grepToolResponse.ts

@@ -0,0 +1,99 @@
+import { Tool } from "../../clients";
+
+export interface GrepOptions {
+  ignoreCase?: boolean;
+  invertMatch?: boolean;
+  contextBefore?: number;
+  contextAfter?: number;
+  maxResults?: number;
+}
+
+/**
+ * Grep through tool response data to find matching lines
+ */
+export async function executeGrep(
+  data: string,
+  toolCallId: string,
+  pattern: string,
+  availableIds: string[],
+  options?: GrepOptions
+): Promise<string> {
+  if (!data) {
+    return `Error: No tool response found for toolCallId "${toolCallId}". Available IDs: ${availableIds.join(
+      ", "
+    )}`;
+  }
+
+  try {
+    const lines = data.split("\n");
+    const matchedResults: string[] = [];
+    const ignoreCase = options?.ignoreCase || false;
+    const invertMatch = options?.invertMatch || false;
+    const contextBefore = options?.contextBefore || 0;
+    const contextAfter = options?.contextAfter || 0;
+    const maxResults = options?.maxResults || 1000;
+
+    // Create regex from pattern
+    const flags = ignoreCase ? "i" : "";
+    const regex = new RegExp(pattern, flags);
+
+    for (let i = 0; i < lines.length && matchedResults.length < maxResults; i++) {
+      const line = lines[i];
+      const matches = regex.test(line);
+      const shouldInclude = invertMatch ? !matches : matches;
+
+      if (shouldInclude) {
+        const startIdx = Math.max(0, i - contextBefore);
+        const endIdx = Math.min(lines.length - 1, i + contextAfter);
+        
+        const contextLines = [];
+        for (let j = startIdx; j <= endIdx; j++) {
+          const prefix = j === i ? "> " : "  ";
+          contextLines.push(`${prefix}${j + 1}: ${lines[j]}`);
+        }
+        
+        matchedResults.push(contextLines.join("\n"));
+        
+        // Skip ahead to avoid overlapping context
+        i += contextAfter;
+      }
+    }
+
+    if (matchedResults.length === 0) {
+      return `No matches found for pattern "${pattern}" in toolCallId "${toolCallId}"`;
+    }
+
+    return matchedResults.join("\n---\n");
+  } catch (error: any) {
+    return `Grep Error: ${error.message}`;
+  }
+}
+
+export const grepToolResponseDefinition: Tool = {
+  type: "function",
+  function: {
+    name: "grepToolResponse",
+    description:
+      "Search through a stored tool response using grep-like pattern matching. Useful when a tool response is too large and you need to find specific lines or content without re-running the tool. Returns matching lines with optional context.",
+    parameters: {
+      type: "object",
+      positional: true,
+      properties: {
+        toolCallId: {
+          type: "string",
+          description: "The toolCallId of the stored tool response",
+        },
+        pattern: {
+          type: "string",
+          description: "Regular expression pattern to search for in the tool response",
+        },
+        options: {
+          type: "object",
+          description: "Optional grep settings: ignoreCase (boolean), invertMatch (boolean), contextBefore (number), contextAfter (number), maxResults (number, default: 1000)",
+          properties: {},
+        },
+      },
+      required: ["toolCallId", "pattern"],
+    },
+  },
+};

package/src/processors/tools/index.ts

@@ -0,0 +1,21 @@
+export {
+  jqToolResponseDefinition,
+  executeJqQuery,
+} from "./jqToolResponse";
+
+export {
+  grepToolResponseDefinition,
+  executeGrep,
+  GrepOptions,
+} from "./grepToolResponse";
+
+export {
+  tailToolResponseDefinition,
+  executeTail,
+  TailOptions,
+} from "./tailToolResponse";
+
+export {
+  listStoredToolResponsesDefinition,
+  executeListStoredToolResponses,
+} from "./listStoredToolResponses";

package/src/processors/tools/jqToolResponse.ts

@@ -0,0 +1,124 @@
+import { Tool } from "../../clients";
+import * as jq from "node-jq";
+
+/**
+ * Attempts to parse content as JSON and returns parsed object if successful
+ */
+function tryParseJson(content: string): any | null {
+  try {
+    return JSON.parse(content);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Recursively searches for JSON strings within an object and parses them
+ */
+function parseNestedJsonStrings(obj: any): any {
+  if (typeof obj === "string") {
+    const parsed = tryParseJson(obj);
+    if (parsed) {
+      return parseNestedJsonStrings(parsed);
+    }
+    return obj;
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map((item) => parseNestedJsonStrings(item));
+  }
+
+  if (obj && typeof obj === "object") {
+    const result: any = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = parseNestedJsonStrings(value);
+    }
+    return result;
+  }
+
+  return obj;
+}
+
+/**
+ * Retrieves and processes tool response data with JQ query
+ */
+export async function executeJqQuery(
+  data: string,
+  toolCallId: string,
+  jqQuery: string,
+  availableIds: string[]
+): Promise<string> {
+  if (!data) {
+    return `Error: No tool response found for toolCallId "${toolCallId}". Available IDs: ${availableIds.join(
+      ", "
+    )}`;
+  }
+
+  try {
+    // First parse the stored string as JSON, then handle nested JSON strings
+    const jsonData = tryParseJson(data);
+    if (!jsonData) {
+      return `Error: Tool response data is not valid JSON for toolCallId "${toolCallId}"`;
+    }
+    const parsedData = parseNestedJsonStrings(jsonData);
+
+    // Execute JQ query
+    const result = await jq.run(jqQuery, parsedData, { input: "json" });
+
+    // Handle the result based on its type
+    if (typeof result === "string") {
+      return result;
+    } else if (typeof result === "number" || typeof result === "boolean") {
+      return String(result);
+    } else if (result === null) {
+      return "null";
+    } else {
+      return JSON.stringify(result);
+    }
+  } catch (error: any) {
+    // If JQ fails, try to provide helpful error message
+    let errorMessage = `JQ Query Error: ${error.message}`;
+
+    // Try to parse as JSON to see if it's valid
+    const jsonObj = tryParseJson(data);
+    if (!jsonObj) {
+      errorMessage += `\nNote: The tool response data is not valid JSON. Raw data preview:\n${data.substring(
+        0,
+        300
+      )}...`;
+    } else {
+      errorMessage += `\nData structure preview:\n${JSON.stringify(
+        jsonObj,
+        null,
+        2
+      ).substring(0, 500)}...`;
+    }
+
+    return errorMessage;
+  }
+}
+
+export const jqToolResponseDefinition: Tool = {
+  type: "function",
+  function: {
+    name: "jqToolResponse",
+    description:
+      "Execute a JQ query on a stored tool response to extract specific data. Use this when you need to extract specific information from any tool response that has been stored. Many MCP tool responses store data in nested structures like .content[0].text where the actual data array is located.",
+    parameters: {
+      type: "object",
+      positional: true,
+      properties: {
+        toolCallId: {
+          type: "string",
+          description: "The toolCallId of the stored tool response",
+        },
+        jqQuery: {
+          type: "string",
+          description:
+            "The JQ query to execute on the tool response data. Examples: '.content[0].text | map(.title)' (extract titles from MCP array), '.content[0].text | map(select(.createdAt > \"2025-01-01\"))' (filter MCP items by date) ",
+        },
+      },
+      required: ["toolCallId", "jqQuery"],
+    },
+  },
+};

package/src/processors/tools/listStoredToolResponses.ts

@@ -0,0 +1,83 @@
+import { Tool } from "../../clients";
+
+export interface ToolResponseInfo {
+  toolCallId: string;
+  toolName: string;
+  size: number;
+  storedAt: number;
+  preview: string;
+}
+
+/**
+ * List all stored tool responses with metadata
+ */
+export async function executeListStoredToolResponses(
+  storage: { [toolCallId: string]: string },
+  metadataStorage: {
+    [toolCallId: string]: {
+      toolCallId: string;
+      originalLength: number;
+      storedAt: number;
+      toolName?: string;
+    };
+  },
+  toolNameMap: { [toolCallId: string]: string }
+): Promise<string> {
+  const toolCallIds = Object.keys(storage);
+
+  if (toolCallIds.length === 0) {
+    return "No tool responses have been stored yet.";
+  }
+
+  const responses: ToolResponseInfo[] = toolCallIds.map((toolCallId) => {
+    const data = storage[toolCallId];
+    const metadata = metadataStorage[toolCallId];
+    const toolName = toolNameMap[toolCallId] || "unknown";
+    
+    // Create a preview (first 100 characters)
+    const preview =
+      data.length > 100 ? data.substring(0, 100) + "..." : data;
+
+    return {
+      toolCallId,
+      toolName,
+      size: metadata?.originalLength || data.length,
+      storedAt: metadata?.storedAt || 0,
+      preview,
+    };
+  });
+
+  // Sort by most recent first
+  responses.sort((a, b) => b.storedAt - a.storedAt);
+
+  // Format the output in a readable way
+  const output = responses
+    .map((resp) => {
+      const date = new Date(resp.storedAt).toISOString();
+      return `
+Tool Call ID: ${resp.toolCallId}
+Tool Name: ${resp.toolName}
+Size: ${resp.size} characters
+Stored At: ${date}
+Preview: ${resp.preview}
+---`;
+    })
+    .join("\n");
+
+  return `Found ${responses.length} stored tool response(s):\n${output}`;
+}
+
+export const listStoredToolResponsesDefinition: Tool = {
+  type: "function",
+  function: {
+    name: "listStoredToolResponses",
+    description:
+      "List all stored tool responses with metadata including tool call ID, tool name, size, timestamp, and a preview of the content. Use this to discover which tool responses are available for querying with jqToolResponse or grepToolResponse.",
+    parameters: {
+      type: "object",
+      positional: false,
+      properties: {},
+      required: [],
+    },
+  },
+};

package/src/processors/tools/tailToolResponse.ts

@@ -0,0 +1,75 @@
+import { Tool } from "../../clients";
+
+export interface TailOptions {
+  lines?: number;
+}
+
+/**
+ * Get the last n lines from a tool response
+ */
+export async function executeTail(
+  data: string,
+  toolCallId: string,
+  availableIds: string[],
+  options?: TailOptions
+): Promise<string> {
+  if (data === null || data === undefined) {
+    return `Error: No tool response found for toolCallId "${toolCallId}". Available IDs: ${availableIds.join(
+      ", "
+    )}`;
+  }
+
+  try {
+    const lines = data.split("\n");
+    const numLines = options?.lines ?? 10;
+
+    // Handle edge case: 0 lines requested
+    if (numLines <= 0) {
+      return "";
+    }
+
+    // Get the last n lines
+    const startIdx = Math.max(0, lines.length - numLines);
+    const tailLines = lines.slice(startIdx);
+
+    // Format with line numbers
+    const formatted = tailLines.map((line, idx) => {
+      const lineNum = startIdx + idx + 1;
+      return `${lineNum}: ${line}`;
+    });
+
+    return formatted.join("\n");
+  } catch (error: any) {
+    return `Tail Error: ${error.message}`;
+  }
+}
+
+export const tailToolResponseDefinition: Tool = {
+  type: "function",
+  function: {
+    name: "tailToolResponse",
+    description:
+      "Get the last n lines from a stored tool response. Similar to the Unix 'tail' command, useful when a tool response is too large and you only need to see the end. Returns the last n lines with line numbers.",
+    parameters: {
+      type: "object",
+      positional: true,
+      properties: {
+        toolCallId: {
+          type: "string",
+          description: "The toolCallId of the stored tool response",
+        },
+        options: {
+          type: "object",
+          description: "Optional tail settings: lines (number, default: 10) - number of lines to return from the end",
+          properties: {
+            lines: {
+              type: "number",
+              description: "Number of lines to return from the end (default: 10)",
+            },
+          },
+        },
+      },
+      required: ["toolCallId"],
+    },
+  },
+};

package/src/services/AgentService.ts

@@ -3,7 +3,7 @@ import { IAgent } from "../agents/interface";
 import { EventService } from "./EventService";
 import { ToolsService } from "./Tools";
 import { ConfigAgent } from "../agents/configurable/ConfigAgent";
-import { AgentContext } from "src/agents/base/base";
+import { AgentContext } from "../agents/base/base";
 
 export class AgentService {
   private agents: Map<string, IAgent> = new Map();

package/src/services/AgentSynchronization.ts

@@ -0,0 +1,291 @@
+/**
+ * Agent Synchronization Service - Handles synchronization with Knowhow API
+ * including task creation, updates, and message processing
+ */
+import {
+  KnowhowSimpleClient,
+  KNOWHOW_API_URL,
+  TaskDetailsResponse,
+  PendingMessage,
+} from "./KnowhowClient";
+import { BaseAgent } from "../agents/base/base";
+import { wait } from "../utils";
+
+export interface SyncOptions {
+  messageId?: string;
+  existingKnowhowTaskId?: string;
+  prompt: string;
+}
+
+export interface TaskSyncState {
+  knowhowTaskId?: string;
+  client: KnowhowSimpleClient;
+  messageId?: string;
+}
+
+/**
+ * AgentSynchronization handles all communication with the Knowhow API
+ * for task creation, updates, status polling, and message synchronization
+ */
+export class AgentSynchronization {
+  private client: KnowhowSimpleClient;
+  private baseUrl: string;
+  private knowhowTaskId: string | undefined;
+  private eventHandlersSetup: boolean = false;
+
+  constructor(baseUrl: string = KNOWHOW_API_URL) {
+    this.baseUrl = baseUrl;
+    this.client = new KnowhowSimpleClient(baseUrl);
+  }
+
+  /**
+   * Create a new chat task in Knowhow
+   */
+  async createChatTask(options: SyncOptions): Promise<string | undefined> {
+    if (!options.messageId || !this.baseUrl) {
+      return undefined;
+    }
+
+    try {
+      console.log(
+        `Base URL for Knowhow API: ${this.baseUrl}, Message ID: ${options.messageId}`
+      );
+      const response = await this.client.createChatTask({
+        messageId: options.messageId,
+        prompt: options.prompt,
+      });
+      const knowhowTaskId = response.data.id;
+      console.log(`✅ Created Knowhow chat task: ${knowhowTaskId}`);
+      return knowhowTaskId;
+    } catch (error) {
+      console.error(`❌ Failed to create Knowhow chat task:`, error);
+      return undefined;
+    }
+  }
+
+  /**
+   * Update a chat task with current agent state
+   */
+  async updateChatTask(
+    knowhowTaskId: string,
+    agent: BaseAgent,
+    inProgress: boolean,
+    result?: string
+  ): Promise<void> {
+    if (!knowhowTaskId || !this.baseUrl) {
+      return;
+    }
+
+    try {
+      await this.client.updateChatTask(knowhowTaskId, {
+        threads: agent.getThreads(),
+        totalCostUsd: agent.getTotalCostUsd(),
+        inProgress,
+        ...(result ? { result } : {}),
+      });
+      console.log(`✅ Updated Knowhow chat task: ${knowhowTaskId}`);
+    } catch (error) {
+      console.error(`❌ Failed to update Knowhow chat task:`, error);
+    }
+  }
+
+  /**
+   * Check for pending messages and process them, also handle pause/kill status
+   */
+  async checkAndProcessPendingMessages(
+    agent: BaseAgent,
+    knowhowTaskId: string
+  ): Promise<void> {
+    if (!knowhowTaskId || !this.baseUrl) {
+      return;
+    }
+
+    try {
+      // Fetch task details to check status
+      const taskDetailsResponse = await this.client.getTaskDetails(
+        knowhowTaskId
+      );
+      const taskDetails: TaskDetailsResponse = taskDetailsResponse.data;
+
+      // Handle killed status
+      if (taskDetails.status === "killed") {
+        console.log(`🛑 Agent task ${knowhowTaskId} was killed via API`);
+        await agent.kill();
+        return;
+      }
+
+      // Handle paused status
+      if (taskDetails.status === "paused") {
+        console.log(
+          `⏸️ Agent task ${knowhowTaskId} is paused, waiting for resume...`
+        );
+        await agent.pause();
+        await this.waitForResume(agent, knowhowTaskId);
+        return; // After resume, we'll process messages on the next threadUpdate
+      }
+
+      // Fetch pending messages
+      const pendingResponse = await this.client.getPendingMessages(
+        knowhowTaskId
+      );
+      const pendingMessages: PendingMessage[] = pendingResponse.data || [];
+
+      if (pendingMessages.length === 0) {
+        return; // No pending messages to process
+      }
+
+      console.log(
+        `📬 Processing ${pendingMessages.length} pending message(s) for task ${knowhowTaskId}`
+      );
+
+      // Inject pending messages into the agent
+      const messageIds: string[] = [];
+      for (const msg of pendingMessages) {
+        agent.addPendingUserMessage({
+          role: msg.role as "user" | "assistant",
+          content: msg.message,
+        });
+        messageIds.push(msg.id);
+      }
+
+      // Mark messages as processed
+      await this.client.markMessagesAsProcessed(knowhowTaskId, messageIds);
+      console.log(`✅ Marked ${messageIds.length} message(s) as processed`);
+    } catch (error) {
+      console.error(`❌ Error checking/processing pending messages:`, error);
+      // Continue execution even if synchronization fails
+    }
+  }
+
+  /**
+   * Wait for the agent to be resumed or killed via API
+   * Polls the API every 2 seconds, with a 1 hour timeout
+   */
+  private async waitForResume(
+    agent: BaseAgent,
+    knowhowTaskId: string
+  ): Promise<void> {
+    const POLL_INTERVAL_MS = 2000;
+    const MAX_WAIT_MS = 60 * 60 * 1000; // 1 hour
+    const startTime = Date.now();
+
+    while (Date.now() - startTime < MAX_WAIT_MS) {
+      await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
+
+      try {
+        const taskDetailsResponse = await this.client.getTaskDetails(
+          knowhowTaskId
+        );
+        const taskDetails: TaskDetailsResponse = taskDetailsResponse.data;
+
+        if (taskDetails.status === "killed") {
+          console.log(`🛑 Agent task ${knowhowTaskId} was killed while paused`);
+          await agent.kill();
+          return;
+        }
+
+        if (
+          taskDetails.status === "running" ||
+          taskDetails.status === "completed"
+        ) {
+          console.log(`▶️ Agent task ${knowhowTaskId} resumed`);
+          await agent.unpause();
+          return;
+        }
+        // Still paused, continue waiting
+      } catch (error) {
+        console.error(`❌ Error polling task status:`, error);
+        // Continue polling even on errors
+      }
+    }
+
+    console.warn(`⚠️ Timeout waiting for resume on task ${knowhowTaskId}`);
+  }
+
+  /**
+   * Set up event-based synchronization for an agent task
+   * This sets up event listeners that automatically sync on threadUpdate and completion
+   */
+  async setupAgentSync(
+    agent: BaseAgent,
+    knowhowTaskId?: string
+  ): Promise<void> {
+    if (!knowhowTaskId || !this.baseUrl) {
+      return;
+    }
+
+    this.knowhowTaskId = knowhowTaskId;
+
+    // Set up event listeners for automatic synchronization
+    if (!this.eventHandlersSetup) {
+      this.setupEventHandlers(agent);
+      this.eventHandlersSetup = true;
+    }
+  }
+
+  /**
+   * Set up event handlers for automatic synchronization
+   */
+  private setupEventHandlers(agent: BaseAgent): void {
+    // Listen to thread updates to sync state and check for pending messages
+    agent.agentEvents.on(agent.eventTypes.threadUpdate, async () => {
+      if (!this.knowhowTaskId || !this.baseUrl) {
+        return;
+      }
+
+      try {
+        // Update task with current state
+        await this.updateChatTask(this.knowhowTaskId, agent, true);
+
+        // Check for pending messages, pause, or kill status
+        await this.checkAndProcessPendingMessages(agent, this.knowhowTaskId);
+      } catch (error) {
+        console.error(`❌ Error during threadUpdate sync:`, error);
+        // Continue execution even if synchronization fails
+      }
+    });
+
+    // Listen to completion event to finalize task
+    agent.agentEvents.on(agent.eventTypes.done, async (result: string) => {
+      if (!this.knowhowTaskId || !this.baseUrl) {
+        return;
+      }
+
+      try {
+        await wait(200);
+        console.log(
+          `Updating Knowhow chat task on completion..., ${this.knowhowTaskId}`
+        );
+        await this.updateChatTask(this.knowhowTaskId, agent, false, result);
+        console.log(`✅ Completed Knowhow chat task: ${this.knowhowTaskId}`);
+      } catch (error) {
+        console.error(`❌ Error finalizing task:`, error);
+      }
+    });
+  }
+
+  /**
+   * Reset synchronization state (useful for reusing the service)
+   */
+  reset(): void {
+    this.knowhowTaskId = undefined;
+    this.eventHandlersSetup = false;
+  }
+
+  /**
+   * Finalize an agent task on completion
+   */
+  async finalizeTask(
+    agent: BaseAgent,
+    knowhowTaskId: string | undefined,
+    result: string
+  ): Promise<void> {
+    if (knowhowTaskId && this.baseUrl) {
+      console.log(
+        `Updating Knowhow chat task on completion..., ${knowhowTaskId}`
+      );
+      await this.updateChatTask(knowhowTaskId, agent, false, result);
+      console.log(`✅ Completed Knowhow chat task: ${knowhowTaskId}`);
+    }
+  }
+}