@tyvm/knowhow 0.0.145 → 0.0.146

package/src/processors/ToolResponseCache.ts

@@ -5,6 +5,7 @@ import { JsonCompressor } from "./JsonCompressor";
 import {
   jqToolResponseDefinition,
   executeJqQuery,
+  JqOptions,
   grepToolResponseDefinition,
   executeGrep,
   GrepOptions,
@@ -13,6 +14,7 @@ import {
   TailOptions,
   listStoredToolResponsesDefinition,
   executeListStoredToolResponses,
+  ListStoredToolResponsesOptions,
 } from "./tools";
 
 interface ToolResponseStorage {
@@ -196,11 +198,12 @@ export class ToolResponseCache {
    */
   async queryToolResponse(
     toolCallId: string,
-    jqQuery: string
+    jqQuery: string,
+    options?: JqOptions
   ): Promise<string> {
     const data = this.storage[toolCallId];
     const availableIds = Object.keys(this.storage);
-    return executeJqQuery(data, toolCallId, jqQuery, availableIds, this.toolNameMap);
+    return executeJqQuery(data, toolCallId, jqQuery, availableIds, this.toolNameMap, options);
   }
 
   /**
@@ -231,11 +234,14 @@ export class ToolResponseCache {
   /**
    * List all stored tool responses with metadata
    */
-  async listStoredToolResponses(): Promise<string> {
+  async listStoredToolResponses(
+    options?: ListStoredToolResponsesOptions
+  ): Promise<string> {
     return executeListStoredToolResponses(
       this.storage,
       this.metadataStorage,
-      this.toolNameMap
+      this.toolNameMap,
+      options
     );
   }
 
@@ -281,9 +287,10 @@ export class ToolResponseCache {
     toolsService.addFunctions({
       [jqToolResponseDefinition.function.name]: async (
         toolCallId: string,
-        jqQuery: string
+        jqQuery: string,
+        options?: JqOptions
       ) => {
-        return await this.queryToolResponse(toolCallId, jqQuery);
+        return await this.queryToolResponse(toolCallId, jqQuery, options);
       },
       [grepToolResponseDefinition.function.name]: async (
         toolCallId: string,
@@ -298,8 +305,10 @@ export class ToolResponseCache {
       ) => {
         return await this.tailToolResponse(toolCallId, options);
       },
-      [listStoredToolResponsesDefinition.function.name]: async () => {
-        return await this.listStoredToolResponses();
+      [listStoredToolResponsesDefinition.function.name]: async (
+        options?: ListStoredToolResponsesOptions
+      ) => {
+        return await this.listStoredToolResponses(options);
       },
     });
   }

package/src/processors/tools/boundedToolResponse.ts

@@ -0,0 +1,49 @@
+export const DEFAULT_TOOL_RESPONSE_CHARACTERS = 20_000;
+export const MAX_TOOL_RESPONSE_CHARACTERS = 50_000;
+
+export function boundedInteger(
+  value: number | undefined,
+  fallback: number,
+  min: number,
+  max: number
+): number {
+  return Math.min(
+    max,
+    Math.max(min, Number.isFinite(value) ? Math.floor(value as number) : fallback)
+  );
+}
+
+/** Return a bounded character page with instructions for retrieving the next page. */
+export function paginateToolResponse(
+  value: string,
+  options?: { characterOffset?: number; maxCharacters?: number },
+  label = "response"
+): string {
+  const characterOffset = boundedInteger(
+    options?.characterOffset,
+    0,
+    0,
+    Number.MAX_SAFE_INTEGER
+  );
+  const maxCharacters = boundedInteger(
+    options?.maxCharacters,
+    DEFAULT_TOOL_RESPONSE_CHARACTERS,
+    1_000,
+    MAX_TOOL_RESPONSE_CHARACTERS
+  );
+  const footerReserve = 300;
+  const contentLimit = Math.max(1, maxCharacters - footerReserve);
+  const page = value.slice(characterOffset, characterOffset + contentLimit);
+  const end = characterOffset + page.length;
+  if (characterOffset === 0 && end >= value.length) {
+    return page;
+  }
+
+  const safeLabel = label.slice(0, 200);
+  const footer =
+    end < value.length
+      ? `\n\n[${safeLabel} characters ${characterOffset}-${Math.max(characterOffset, end - 1)} of ${value.length}. Repeat with characterOffset=${end} to continue; maxCharacters=${maxCharacters}.]`
+      : `\n\n[End of ${safeLabel}; characters ${characterOffset}-${Math.max(characterOffset, end - 1)} of ${value.length}.]`;
+
+  return (page + footer).slice(0, maxCharacters);
+}

package/src/processors/tools/grepToolResponse.ts

@@ -6,11 +6,33 @@ export interface GrepOptions {
   contextBefore?: number;
   contextAfter?: number;
   maxResults?: number;
+  resultOffset?: number;
+  maxCharacters?: number;
+  lineCharacterOffset?: number;
+  maxLineCharacters?: number;
 }
 
-/**
- * Grep through tool response data to find matching lines
- */
+const DEFAULT_MAX_RESULTS = 100;
+const MAX_RESULTS = 1000;
+const DEFAULT_MAX_CHARACTERS = 20_000;
+const MAX_CHARACTERS = 50_000;
+const DEFAULT_MAX_LINE_CHARACTERS = 4_000;
+const MAX_LINE_CHARACTERS = 10_000;
+const FOOTER_RESERVE = 700;
+
+function boundedInteger(
+  value: number | undefined,
+  fallback: number,
+  min: number,
+  max: number
+): number {
+  return Math.min(
+    max,
+    Math.max(min, Number.isFinite(value) ? Math.floor(value as number) : fallback)
+  );
+}
+
+/** Search stored tool-response text without ever returning an unbounded result. */
 export async function executeGrep(
   data: string,
   toolCallId: string,
@@ -26,51 +48,145 @@ export async function executeGrep(
         return name ? `${id} (${name})` : id;
       })
       .join("\n  - ");
-    return `Error: No tool response found for toolCallId "${toolCallId}". Call listStoredToolResponses to see all available responses with their tool names.\n\nAvailable toolCallIds:\n  - ${idList || "(none)"}`;
+    const maxCharacters = boundedInteger(
+      options?.maxCharacters,
+      DEFAULT_MAX_CHARACTERS,
+      1000,
+      MAX_CHARACTERS
+    );
+    return `Error: No tool response found for toolCallId "${toolCallId}". Call listStoredToolResponses to see all available responses with their tool names.\n\nAvailable toolCallIds:\n  - ${idList || "(none)"}`.slice(
+      0,
+      maxCharacters
+    );
   }
 
   try {
     const lines = data.split("\n");
     const matchedResults: string[] = [];
-    const ignoreCase = options?.ignoreCase || false;
-    const invertMatch = options?.invertMatch || false;
-    const contextBefore = options?.contextBefore || 0;
-    const contextAfter = options?.contextAfter || 0;
-    const maxResults = options?.maxResults || 1000;
-
-    // Create regex from pattern
-    const flags = ignoreCase ? "i" : "";
-    const regex = new RegExp(pattern, flags);
-
-    for (let i = 0; i < lines.length && matchedResults.length < maxResults; i++) {
-      const line = lines[i];
-      const matches = regex.test(line);
-      const shouldInclude = invertMatch ? !matches : matches;
-
-      if (shouldInclude) {
-        const startIdx = Math.max(0, i - contextBefore);
-        const endIdx = Math.min(lines.length - 1, i + contextAfter);
-        
-        const contextLines = [];
+    const ignoreCase = options?.ignoreCase ?? false;
+    const invertMatch = options?.invertMatch ?? false;
+    const contextBefore = boundedInteger(options?.contextBefore, 0, 0, 1000);
+    const contextAfter = boundedInteger(options?.contextAfter, 0, 0, 1000);
+    const maxResults = boundedInteger(options?.maxResults, DEFAULT_MAX_RESULTS, 1, MAX_RESULTS);
+    const resultOffset = boundedInteger(options?.resultOffset, 0, 0, Number.MAX_SAFE_INTEGER);
+    const maxCharacters = boundedInteger(
+      options?.maxCharacters,
+      DEFAULT_MAX_CHARACTERS,
+      1000,
+      MAX_CHARACTERS
+    );
+    const lineCharacterOffset = boundedInteger(
+      options?.lineCharacterOffset,
+      0,
+      0,
+      Number.MAX_SAFE_INTEGER
+    );
+    const maxLineCharacters = boundedInteger(
+      options?.maxLineCharacters,
+      DEFAULT_MAX_LINE_CHARACTERS,
+      100,
+      MAX_LINE_CHARACTERS
+    );
+    const contentBudget = Math.max(1, maxCharacters - FOOTER_RESERVE);
+    const regex = new RegExp(pattern, ignoreCase ? "i" : "");
+
+    let totalMatches = 0;
+    let outputCharacters = 0;
+    let outputTruncated = false;
+    const pagedLineNumbers = new Set<number>();
+
+    for (let i = 0; i < lines.length; i++) {
+      const matches = regex.test(lines[i]);
+      if (invertMatch ? matches : !matches) continue;
+
+      const resultIndex = totalMatches++;
+      const startIdx = Math.max(0, i - contextBefore);
+      const endIdx = Math.min(lines.length - 1, i + contextAfter);
+
+      if (
+        resultIndex >= resultOffset &&
+        matchedResults.length < maxResults &&
+        !outputTruncated
+      ) {
+        const contextLines: string[] = [];
         for (let j = startIdx; j <= endIdx; j++) {
+          const sourceLine = lines[j];
+          const visibleLine = sourceLine.slice(
+            lineCharacterOffset,
+            lineCharacterOffset + maxLineCharacters
+          );
           const prefix = j === i ? "> " : "  ";
-          contextLines.push(`${prefix}${j + 1}: ${lines[j]}`);
+          let rendered = `${prefix}${j + 1}: ${visibleLine}`;
+
+          if (lineCharacterOffset >= sourceLine.length && sourceLine.length > 0) {
+            rendered += ` [no characters at offset ${lineCharacterOffset}; line length=${sourceLine.length}]`;
+          } else if (lineCharacterOffset + maxLineCharacters < sourceLine.length) {
+            pagedLineNumbers.add(j + 1);
+            rendered += `\n  [line ${j + 1} truncated; characters ${lineCharacterOffset}-${lineCharacterOffset + visibleLine.length - 1} of ${sourceLine.length}]`;
+          }
+          contextLines.push(rendered);
+        }
+
+        const separatorLength = matchedResults.length > 0 ? 5 : 0;
+        const result = contextLines.join("\n");
+        const remaining = contentBudget - outputCharacters - separatorLength;
+        if (result.length > remaining) {
+          if (matchedResults.length === 0 && remaining > 80) {
+            matchedResults.push(
+              `${result.slice(0, remaining - 48)}\n[result page truncated by maxCharacters]`
+            );
+            outputCharacters = contentBudget;
+          }
+          outputTruncated = true;
+        } else {
+          matchedResults.push(result);
+          outputCharacters += separatorLength + result.length;
         }
-        
-        matchedResults.push(contextLines.join("\n"));
-        
-        // Skip ahead to avoid overlapping context
-        i += contextAfter;
       }
+
+      // Avoid duplicate result blocks when their after-context overlaps a match.
+      i += contextAfter;
     }
 
-    if (matchedResults.length === 0) {
-      return `No matches found for pattern "${pattern}" in toolCallId "${toolCallId}"`;
+    if (totalMatches === 0) {
+      const preview = pattern.length > 200 ? `${pattern.slice(0, 200)}…` : pattern;
+      return `No matches found for pattern "${preview}" in toolCallId "${toolCallId}"`.slice(
+        0,
+        maxCharacters
+      );
+    }
+
+    const consumedResults = matchedResults.length;
+    const nextResultOffset = resultOffset + consumedResults;
+    const hasMoreResults = nextResultOffset < totalMatches;
+    const range = consumedResults
+      ? `${resultOffset + 1}-${nextResultOffset}`
+      : `none (offset ${resultOffset} is past the final result)`;
+    const footerLines = [
+      "",
+      `[grep page: results ${range} of ${totalMatches}; ${consumedResults} returned; maxCharacters=${maxCharacters}]`,
+    ];
+
+    if (pagedLineNumbers.size > 0) {
+      footerLines.push(
+        `[Long source line(s) ${Array.from(pagedLineNumbers).join(", ")} were sliced. Repeat with options.lineCharacterOffset=${lineCharacterOffset + maxLineCharacters} to read their next characters.]`
+      );
+    }
+    if (hasMoreResults) {
+      footerLines.push(
+        `[More matches available. Repeat with options.resultOffset=${nextResultOffset}.]`
+      );
+    } else {
+      footerLines.push("[End of matches.]");
     }
 
-    return matchedResults.join("\n---\n");
-  } catch (error: any) {
-    return `Grep Error: ${error.message}`;
+    return (matchedResults.join("\n---\n") + footerLines.join("\n")).slice(
+      0,
+      maxCharacters
+    );
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : String(error);
+    return `Grep Error: ${message.slice(0, 500)}`;
   }
 }
 
@@ -79,7 +195,7 @@ export const grepToolResponseDefinition: Tool = {
   function: {
     name: "grepToolResponse",
     description:
-      "Search through a stored tool response using grep-like pattern matching. Useful when a tool response is too large and you need to find specific lines or content without re-running the tool. Returns matching lines with optional context.",
+      "Search a stored tool response using a regular expression. Results and individual source lines are bounded and paginated. Follow the returned resultOffset or lineCharacterOffset instruction to retrieve more.",
     parameters: {
       type: "object",
       positional: true,
@@ -94,8 +210,18 @@ export const grepToolResponseDefinition: Tool = {
         },
         options: {
           type: "object",
-          description: "Optional grep settings: ignoreCase (boolean), invertMatch (boolean), contextBefore (number), contextAfter (number), maxResults (number, default: 1000)",
-          properties: {},
+          description: "Optional grep, output-bound, and pagination settings.",
+          properties: {
+            ignoreCase: { type: "boolean", description: "Case-insensitive matching." },
+            invertMatch: { type: "boolean", description: "Return non-matching lines." },
+            contextBefore: { type: "number", description: "Context lines before a match (maximum 1000)." },
+            contextAfter: { type: "number", description: "Context lines after a match (maximum 1000)." },
+            maxResults: { type: "number", description: "Matches per page (default 100, maximum 1000)." },
+            resultOffset: { type: "number", description: "Zero-based match offset for the next result page." },
+            maxCharacters: { type: "number", description: "Response size (default 20000, hard maximum 50000)." },
+            lineCharacterOffset: { type: "number", description: "Character offset within every source line, for paging very long lines." },
+            maxLineCharacters: { type: "number", description: "Characters shown per source line (default 4000, maximum 10000)." },
+          },
         },
       },
       required: ["toolCallId", "pattern"],

package/src/processors/tools/index.ts

@@ -1,6 +1,7 @@
 export {
   jqToolResponseDefinition,
   executeJqQuery,
+  JqOptions,
 } from "./jqToolResponse";
 
 export {
@@ -18,4 +19,5 @@ export {
 export {
   listStoredToolResponsesDefinition,
   executeListStoredToolResponses,
+  ListStoredToolResponsesOptions,
 } from "./listStoredToolResponses";

package/src/processors/tools/jqToolResponse.ts

@@ -1,5 +1,11 @@
 import { Tool } from "../../clients";
 import * as jq from "node-jq";
+import { paginateToolResponse } from "./boundedToolResponse";
+
+export interface JqOptions {
+  characterOffset?: number;
+  maxCharacters?: number;
+}
 
 /**
  * Attempts to parse content as JSON and returns parsed object if successful
@@ -47,7 +53,8 @@ export async function executeJqQuery(
   toolCallId: string,
   jqQuery: string,
   availableIds: string[],
-  toolNameMap?: { [toolCallId: string]: string }
+  toolNameMap?: { [toolCallId: string]: string },
+  options?: JqOptions
 ): Promise<string> {
   if (!data) {
     const idList = availableIds
@@ -56,30 +63,39 @@ export async function executeJqQuery(
         return name ? `${id} (${name})` : id;
       })
       .join("\n  - ");
-    return `Error: No tool response found for toolCallId "${toolCallId}". Call listStoredToolResponses to see all available responses with their tool names.\n\nAvailable toolCallIds:\n  - ${idList || "(none)"}`;
+    return paginateToolResponse(
+      `Error: No tool response found for toolCallId "${toolCallId}". Call listStoredToolResponses to see all available responses with their tool names.\n\nAvailable toolCallIds:\n  - ${idList || "(none)"}`,
+      options,
+      "available toolCallId list"
+    );
   }
 
   try {
     // First parse the stored string as JSON, then handle nested JSON strings
     const jsonData = tryParseJson(data);
     if (!jsonData) {
-      return `Error: Tool response data is not valid JSON for toolCallId "${toolCallId}"`;
+      return paginateToolResponse(
+        `Error: Tool response data is not valid JSON for toolCallId "${toolCallId}"`,
+        options,
+        "jq input error"
+      );
     }
     const parsedData = parseNestedJsonStrings(jsonData);
 
     // Execute JQ query
     const result = await jq.run(jqQuery, parsedData, { input: "json" });
 
-    // Handle the result based on its type
+    let rendered: string;
     if (typeof result === "string") {
-      return result;
+      rendered = result;
     } else if (typeof result === "number" || typeof result === "boolean") {
-      return String(result);
+      rendered = String(result);
     } else if (result === null) {
-      return "null";
+      rendered = "null";
     } else {
-      return JSON.stringify(result);
+      rendered = JSON.stringify(result);
     }
+    return paginateToolResponse(rendered, options, "jq result");
   } catch (error: any) {
     // If JQ fails, try to provide helpful error message
     let errorMessage = `JQ Query Error: ${error.message}`;
@@ -99,7 +115,7 @@ export async function executeJqQuery(
       ).substring(0, 500)}...`;
     }
 
-    return errorMessage;
+    return paginateToolResponse(errorMessage, options, "jq error");
   }
 }
 
@@ -108,7 +124,7 @@ export const jqToolResponseDefinition: Tool = {
   function: {
     name: "jqToolResponse",
     description:
-      "Execute a JQ query on a stored tool response to extract specific data. Use this when you need to extract specific information from any tool response that has been stored. This is the preferred way to search or filter compressed JSON tool responses — it parses the data automatically without requiring repeated expandTokens calls. IMPORTANT: You do NOT know the toolCallId at the time you call a tool — you must call listStoredToolResponses first to discover the correct toolCallId. The listStoredToolResponses output shows each response's tool name so you can identify which response belongs to which call. How to determine the correct JQ query: (1) For mcp_* tool responses (external MCP tools like mcp_1_*): the response is stored as a raw JSON object — use '.' to access the root directly, e.g. '.children | map(.name)' or '.state'. Do NOT use .content[0].text | fromjson for these. (2) For compressed MCP tool responses (._mcp_format === true): use '._data' e.g. '._data.children | map(.name)'. (3) For standard built-in tool responses: data may be nested under '.content[0].text | fromjson'. Use jqToolResponse instead of expandTokens whenever the stored data is JSON.",
+      "Execute a JQ query on a stored tool response. Output is bounded to 20,000 characters by default; use options.characterOffset to page through a large result. Prefer a selective JQ query over paging a broad query. Call listStoredToolResponses first to discover the toolCallId. MCP responses are generally rooted at '.' (or '._data' when _mcp_format is true); standard built-in responses may be nested under '.content[0].text | fromjson'.",
     parameters: {
       type: "object",
       positional: true,
@@ -122,6 +138,20 @@ export const jqToolResponseDefinition: Tool = {
           description:
             "The JQ query to execute on the tool response data. For mcp_* tool responses (raw JSON object): '.children | map({id: .id, name: .name})' (extract fields from children array), '.children | map(select(.state == \"PENDING\")) | length' (count pending children), '.name' (get a top-level field). For compressed responses (._mcp_format true): '._data.children | map(.name)' or '._data | map(select(.state == \"PENDING\")) | length'. For standard built-in tool responses: '.content[0].text | fromjson | map(.title)' (extract titles from standard MCP array), '.content[0].text | fromjson | map(select(.createdAt > \"2025-01-01\"))' (filter by date).",
         },
+        options: {
+          type: "object",
+          description: "Optional output pagination settings.",
+          properties: {
+            characterOffset: {
+              type: "number",
+              description: "Zero-based character offset into the rendered JQ result.",
+            },
+            maxCharacters: {
+              type: "number",
+              description: "Response size (default 20000, hard maximum 50000).",
+            },
+          },
+        },
       },
       required: ["toolCallId", "jqQuery"],
     },

package/src/processors/tools/listStoredToolResponses.ts

@@ -1,4 +1,12 @@
 import { Tool } from "../../clients";
+import { boundedInteger, paginateToolResponse } from "./boundedToolResponse";
+
+export interface ListStoredToolResponsesOptions {
+  resultOffset?: number;
+  maxResults?: number;
+  characterOffset?: number;
+  maxCharacters?: number;
+}
 
 export interface ToolResponseInfo {
   toolCallId: string;
@@ -21,7 +29,8 @@ export async function executeListStoredToolResponses(
       toolName?: string;
     };
   },
-  toolNameMap: { [toolCallId: string]: string }
+  toolNameMap: { [toolCallId: string]: string },
+  options?: ListStoredToolResponsesOptions
 ): Promise<string> {
   const toolCallIds = Object.keys(storage);
 
@@ -50,13 +59,15 @@ export async function executeListStoredToolResponses(
   // Sort by most recent first
   responses.sort((a, b) => b.storedAt - a.storedAt);
 
-  // Format the output in a readable way
-  const output = responses
+  const resultOffset = boundedInteger(options?.resultOffset, 0, 0, responses.length);
+  const maxResults = boundedInteger(options?.maxResults, 100, 1, 1000);
+  const page = responses.slice(resultOffset, resultOffset + maxResults);
+  const output = page
     .map((resp) => {
       const date = new Date(resp.storedAt).toISOString();
       return `
-Tool Call ID: ${resp.toolCallId}
-Tool Name: ${resp.toolName}
+Tool Call ID: ${resp.toolCallId.slice(0, 500)}
+Tool Name: ${resp.toolName.slice(0, 500)}
 Size: ${resp.size} characters
 Stored At: ${date}
 Preview: ${resp.preview}
@@ -64,7 +75,15 @@ Preview: ${resp.preview}
     })
     .join("\n");
 
-  return `Found ${responses.length} stored tool response(s):\n${output}`;
+  const nextOffset = resultOffset + page.length;
+  const footer = nextOffset < responses.length
+    ? `\n[More stored responses available. Repeat with options.resultOffset=${nextOffset}.]`
+    : "\n[End of stored responses.]";
+  return paginateToolResponse(
+    `Found ${responses.length} stored tool response(s); showing ${resultOffset + 1}-${nextOffset}:\n${output}${footer}`,
+    options,
+    "stored-response list"
+  );
 }
 
 export const listStoredToolResponsesDefinition: Tool = {
@@ -72,12 +91,34 @@ export const listStoredToolResponsesDefinition: Tool = {
   function: {
     name: "listStoredToolResponses",
     description:
-      "List all stored tool responses with metadata including tool call ID, tool name, size, timestamp, and a preview of the content. ALWAYS call this before using jqToolResponse, grepToolResponse, or tailToolResponse — you need the toolCallId from this list to query a specific response. The tool name shown here corresponds to the tool that produced the response (e.g. 'mcp_1_knowhow-web_GetOrgUserTask'), making it easy to identify which response you want.",
+      "List a bounded page of stored tool responses with IDs, tool names, sizes, timestamps, and previews. Call this before jqToolResponse, grepToolResponse, or tailToolResponse to discover a toolCallId.",
     parameters: {
       type: "object",
-      positional: false,
-      properties: {},
-      required: [],
+      positional: true,
+      properties: {
+        options: {
+          type: "object",
+          description: "Optional result and output pagination settings.",
+          properties: {
+            resultOffset: {
+              type: "number",
+              description: "Zero-based response offset (default 0).",
+            },
+            maxResults: {
+              type: "number",
+              description: "Responses per page (default 100, maximum 1000).",
+            },
+            characterOffset: {
+              type: "number",
+              description: "Character offset within the rendered page.",
+            },
+            maxCharacters: {
+              type: "number",
+              description: "Response size (default 20000, hard maximum 50000).",
+            },
+          },
+        },
+      },
     },
   },
 };