@tyvm/knowhow 0.0.89 → 0.0.91

package/src/index.ts

@@ -5,7 +5,6 @@ import {
   getHashes,
   checkNoFilesChanged,
 } from "./hashes";
-import axios from "axios";
 import * as fs from "fs";
 import * as path from "path";
 import * as crypto from "crypto";
@@ -52,6 +51,14 @@ export * as types from "./types";
 export * as processors from "./processors";
 export * as ai from "./ai";
 
+// Export module system types for external modules
+export * from "./services/modules/types";
+// Export plugin types for external plugins
+export { PluginBase } from "./plugins/PluginBase";
+export { PluginMeta, Plugin, PluginContext } from "./plugins/types";
+// Export embedding types
+export { MinimalEmbedding, Embeddable } from "./types";
+
 export async function embed() {
   // load config
   const config = await getConfig();
@@ -154,12 +161,32 @@ export async function upload() {
   }
 }
 
+/**
+ * Normalizes an input pattern to a valid glob pattern.
+ * Supports:
+ *   - Standard glob patterns (e.g. "src/**\/*.ts")
+ *   - Brace expansion (e.g. "{src/a.ts,src/b.ts}")
+ *   - Comma-separated file paths (e.g. "src/a.ts,src/b.ts") — auto-converted to brace expansion
+ */
+function normalizeInputPattern(input: string): string {
+  // If it already has braces or glob chars other than comma, use as-is
+  if (input.includes("{") || input.includes("*") || input.includes("?")) {
+    return input;
+  }
+  // If it contains commas, treat as comma-separated list and wrap in braces
+  if (input.includes(",")) {
+    const parts = input.split(",").map((p) => p.trim());
+    return `{${parts.join(",")}}`;
+  }
+  return input;
+}
+
 export async function generate(): Promise<void> {
   const config = await getConfig();
   for (const source of config.sources) {
     console.log("Generating", source.input, "to", source.output);
     if (source.kind === "file" || !source.kind) {
-      const files = globSync(source.input);
+      const files = globSync(normalizeInputPattern(source.input));
       const prompt = await loadPrompt(source.prompt);
 
       if (source.output.endsWith("/")) {
@@ -205,7 +232,7 @@ async function handleAllKindsGeneration(source: GenerationSource) {
 
 async function handleFileKindGeneration(source: GenerationSource) {
   const prompt = await loadPrompt(source.prompt);
-  const files = globSync(source.input);
+  const files = globSync(normalizeInputPattern(source.input));
   console.log("Analyzing files: ", files);
 
   if (source.output.endsWith("/")) {

package/src/login.ts

@@ -1,4 +1,4 @@
-import axios from "axios";
+import http from "./utils/http";
 import fs from "fs";
 import path from "path";
 import { chmod } from "fs/promises";
@@ -68,11 +68,10 @@ export async function login(jwtFlag?: boolean): Promise<void> {
       await updateConfig(config);
     }
   } catch (error) {
-    if (axios.isAxiosError(error) && error.response) {
+    if (http.isHttpError(error) && error.response) {
+      const errData = await error.response.json().catch(() => ({ message: "Unknown error" }));
       throw new Error(
-        `Error: ${error.response.status} - ${
-          error.response.data.message || "Unknown error"
-        }`
+        `Error: ${error.status} - ${errData.message || "Unknown error"}`
       );
     }
     console.log(
@@ -98,7 +97,7 @@ export async function loadJwt(): Promise<string> {
 }
 
 export async function checkJwt(storedJwt: string) {
-  const response = await axios.get(`${KNOWHOW_API_URL}/api/users/me`, {
+  const response = await http.get(`${KNOWHOW_API_URL}/api/users/me`, {
     headers: {
       Authorization: `Bearer ${storedJwt}`,
     },

package/src/plugins/language.ts

@@ -6,10 +6,6 @@ import { getConfig, getLanguageConfig } from "../config";
 import { getEnabledPlugins } from "../types";
 import { PluginBase, PluginMeta } from "./PluginBase";
 import { Plugin, PluginContext } from "./types";
-import { GitHubPlugin } from "./github";
-import { AsanaPlugin } from "./asana";
-import { JiraPlugin } from "./jira";
-import { LinearPlugin } from "./linear";
 import { PluginService } from "./plugins";
 
 /**

package/src/plugins/plugins.ts

@@ -4,13 +4,6 @@ import { VimPlugin } from "./vim";
 import { LinterPlugin } from "./LinterPlugin";
 import { LanguagePlugin } from "./language";
 import { EmbeddingPlugin } from "./embedding";
-import { GitHubPlugin } from "./github";
-import { AsanaPlugin } from "./asana";
-import { LinearPlugin } from "./linear";
-import { JiraPlugin } from "./jira";
-import { NotionPlugin } from "./notion";
-import { DownloaderPlugin } from "./downloader/plugin";
-import { FigmaPlugin } from "./figma";
 import { UrlPlugin } from "./url";
 import { GitPlugin } from "./GitPlugin";
 import { TmuxPlugin } from "./tmux";
@@ -33,13 +26,6 @@ export class PluginService {
     this.pluginMap.set("embeddings", new EmbeddingPlugin(context));
     this.pluginMap.set("vim", new VimPlugin(context));
     this.pluginMap.set("linter", new LinterPlugin(context));
-    this.pluginMap.set("github", new GitHubPlugin(context));
-    this.pluginMap.set("asana", new AsanaPlugin(context));
-    this.pluginMap.set("linear", new LinearPlugin(context));
-    this.pluginMap.set("jira", new JiraPlugin(context));
-    this.pluginMap.set("notion", new NotionPlugin(context));
-    this.pluginMap.set("download", new DownloaderPlugin(context));
-    this.pluginMap.set("figma", new FigmaPlugin(context));
     this.pluginMap.set("language", new LanguagePlugin(context));
     this.pluginMap.set("url", new UrlPlugin(context));
     this.pluginMap.set("git", new GitPlugin(context));

package/src/plugins/url.ts

@@ -1,9 +1,6 @@
 import { PluginBase, PluginMeta } from "./PluginBase";
 import { Plugin, PluginContext } from "./types";
 import { MinimalEmbedding } from "../types";
-import axios from "axios";
-import * as cheerio from "cheerio";
-import loadWebpage from "../agents/tools/loadWebpage";
 
 export class UrlPlugin extends PluginBase implements Plugin {
   static readonly meta: PluginMeta = {
@@ -20,22 +17,47 @@ export class UrlPlugin extends PluginBase implements Plugin {
 
   async embed(userPrompt: string): Promise<MinimalEmbedding[]> {
     const urls = this.extractUrls(userPrompt);
-    const embeddings = await Promise.all(urls.map(this.fetchAndParseUrl));
+    const embeddings = await Promise.all(urls.map((url) => this.fetchAndParseUrl(url)));
     return embeddings.filter((e): e is MinimalEmbedding => e !== null);
   }
 
   extractUrls(userPrompt: string): string[] {
     const urlRegex = /(https?:\/\/[^\s]+)/g;
     const urls = userPrompt.match(urlRegex) || [];
-
     return Array.from(new Set(urls));
   }
 
   async fetchAndParseUrl(url: string): Promise<MinimalEmbedding | null> {
     try {
-      const text = await loadWebpage(url);
+      const response = await fetch(url, {
+        headers: {
+          "User-Agent": "Mozilla/5.0 (compatible; KnowhowBot/1.0)",
+          "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        },
+      });
+
+      if (!response.ok) {
+        this.log(`URL PLUGIN: Failed to fetch ${url}: ${response.status}`, "warn");
+        return null;
+      }
+
+      const contentType = response.headers.get("content-type") || "";
+      let text = "";
+
+      if (contentType.includes("text/html") || contentType.includes("text/plain")) {
+        const html = await response.text();
+        // Simple HTML to text: strip tags
+        text = html
+          .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
+          .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
+          .replace(/<[^>]+>/g, " ")
+          .replace(/\s+/g, " ")
+          .trim();
+      } else {
+        text = await response.text();
+      }
 
-      this.log(`URL PLUGIN: Fetched content from ${url}: ${text}`);
+      this.log(`URL PLUGIN: Fetched content from ${url}: ${text.substring(0, 100)}`);
 
       return {
         id: url + "-url",
@@ -58,7 +80,7 @@ export class UrlPlugin extends PluginBase implements Plugin {
       return "URL PLUGIN: Too many URLs detected. Skipping like unintentional bulk browse.";
     }
 
-    const results = await Promise.all(urls.map(this.fetchAndParseUrl));
+    const results = await Promise.all(urls.map((url) => this.fetchAndParseUrl(url)));
     const validResults = results.filter(
       (r): r is MinimalEmbedding => r !== null
     );
@@ -70,10 +92,7 @@ export class UrlPlugin extends PluginBase implements Plugin {
     const formattedResults = validResults
       .map(
         (result) =>
-          `URL: ${result.metadata.url}\n\nContent:\n${result.text.substring(
-            0,
-            500
-          )}...`
+          `URL: ${result.metadata.url}\n\nContent:\n${result.text.substring(0, 500)}...`
       )
       .join("\n\n---\n\n");
 

package/src/processors/TokenCompressor.ts

@@ -131,7 +131,7 @@ export class TokenCompressor implements JsonCompressorStorage {
       200
     )}...\n[Use ${
       this.toolName
-    } tool with key "${firstKey}" to retrieve content. Follow NEXT_CHUNK_KEY references for complete content]`;
+    } tool with key "${firstKey}" to retrieve content. Follow NEXT_CHUNK_KEY references for complete content]\n[TIP: If this is JSON data, prefer using the jqToolResponse tool with the toolCallId instead — it parses ._data automatically and lets you filter/search without repeated expandTokens calls]`;
   }
 
   /**
@@ -413,7 +413,7 @@ export const expandTokensDefinition: Tool = {
   function: {
     name: "expandTokens",
     description:
-      "Retrieve a chunk of compressed data that was stored during message processing. The returned content may contain a `NEXT_CHUNK_KEY` to retrieve subsequent chunks.",
+      "Retrieve a chunk of compressed data that was stored during message processing. The returned content may contain a `NEXT_CHUNK_KEY` to retrieve subsequent chunks. NOTE: If the compressed data is JSON (e.g. a tool response), prefer using jqToolResponse instead — it lets you query the data directly using JQ without repeatedly calling expandTokens. Use expandTokens only for plain text/string data or when you need the raw content.",
     parameters: {
       type: "object",
       positional: true,

package/src/processors/ToolResponseCache.ts

@@ -200,7 +200,7 @@ export class ToolResponseCache {
   ): Promise<string> {
     const data = this.storage[toolCallId];
     const availableIds = Object.keys(this.storage);
-    return executeJqQuery(data, toolCallId, jqQuery, availableIds);
+    return executeJqQuery(data, toolCallId, jqQuery, availableIds, this.toolNameMap);
   }
 
   /**
@@ -213,7 +213,7 @@ export class ToolResponseCache {
   ): Promise<string> {
     const data = this.storage[toolCallId];
     const availableIds = Object.keys(this.storage);
-    return executeGrep(data, toolCallId, pattern, availableIds, options);
+    return executeGrep(data, toolCallId, pattern, availableIds, options, this.toolNameMap);
   }
 
   /**
@@ -225,7 +225,7 @@ export class ToolResponseCache {
   ): Promise<string> {
     const data = this.storage[toolCallId];
     const availableIds = Object.keys(this.storage);
-    return executeTail(data, toolCallId, availableIds, options);
+    return executeTail(data, toolCallId, availableIds, options, this.toolNameMap);
   }
 
   /**

package/src/processors/tools/grepToolResponse.ts

@@ -16,12 +16,17 @@ export async function executeGrep(
   toolCallId: string,
   pattern: string,
   availableIds: string[],
-  options?: GrepOptions
+  options?: GrepOptions,
+  toolNameMap?: { [toolCallId: string]: string }
 ): Promise<string> {
   if (!data) {
-    return `Error: No tool response found for toolCallId "${toolCallId}". Available IDs: ${availableIds.join(
-      ", "
-    )}`;
+    const idList = availableIds
+      .map((id) => {
+        const name = toolNameMap?.[id];
+        return name ? `${id} (${name})` : id;
+      })
+      .join("\n  - ");
+    return `Error: No tool response found for toolCallId "${toolCallId}". Call listStoredToolResponses to see all available responses with their tool names.\n\nAvailable toolCallIds:\n  - ${idList || "(none)"}`;
   }
 
   try {

package/src/processors/tools/jqToolResponse.ts

@@ -46,12 +46,17 @@ export async function executeJqQuery(
   data: string,
   toolCallId: string,
   jqQuery: string,
-  availableIds: string[]
+  availableIds: string[],
+  toolNameMap?: { [toolCallId: string]: string }
 ): Promise<string> {
   if (!data) {
-    return `Error: No tool response found for toolCallId "${toolCallId}". Available IDs: ${availableIds.join(
-      ", "
-    )}`;
+    const idList = availableIds
+      .map((id) => {
+        const name = toolNameMap?.[id];
+        return name ? `${id} (${name})` : id;
+      })
+      .join("\n  - ");
+    return `Error: No tool response found for toolCallId "${toolCallId}". Call listStoredToolResponses to see all available responses with their tool names.\n\nAvailable toolCallIds:\n  - ${idList || "(none)"}`;
   }
 
   try {
@@ -103,7 +108,7 @@ export const jqToolResponseDefinition: Tool = {
   function: {
     name: "jqToolResponse",
     description:
-      "Execute a JQ query on a stored tool response to extract specific data. Use this when you need to extract specific information from any tool response that has been stored. Many MCP tool responses store data in nested structures like .content[0].text where the actual data array is located.",
+      "Execute a JQ query on a stored tool response to extract specific data. Use this when you need to extract specific information from any tool response that has been stored. This is the preferred way to search or filter compressed JSON tool responses — it parses the data automatically without requiring repeated expandTokens calls. IMPORTANT: You do NOT know the toolCallId at the time you call a tool — you must call listStoredToolResponses first to discover the correct toolCallId. The listStoredToolResponses output shows each response's tool name so you can identify which response belongs to which call. How to determine the correct JQ query: (1) For mcp_* tool responses (external MCP tools like mcp_1_*): the response is stored as a raw JSON object — use '.' to access the root directly, e.g. '.children | map(.name)' or '.state'. Do NOT use .content[0].text | fromjson for these. (2) For compressed MCP tool responses (._mcp_format === true): use '._data' e.g. '._data.children | map(.name)'. (3) For standard built-in tool responses: data may be nested under '.content[0].text | fromjson'. Use jqToolResponse instead of expandTokens whenever the stored data is JSON.",
     parameters: {
       type: "object",
       positional: true,
@@ -115,7 +120,7 @@ export const jqToolResponseDefinition: Tool = {
         jqQuery: {
           type: "string",
           description:
-            "The JQ query to execute on the tool response data. Examples: '.content[0].text | map(.title)' (extract titles from MCP array), '.content[0].text | map(select(.createdAt > \"2025-01-01\"))' (filter MCP items by date) ",
+            "The JQ query to execute on the tool response data. For mcp_* tool responses (raw JSON object): '.children | map({id: .id, name: .name})' (extract fields from children array), '.children | map(select(.state == \"PENDING\")) | length' (count pending children), '.name' (get a top-level field). For compressed responses (._mcp_format true): '._data.children | map(.name)' or '._data | map(select(.state == \"PENDING\")) | length'. For standard built-in tool responses: '.content[0].text | fromjson | map(.title)' (extract titles from standard MCP array), '.content[0].text | fromjson | map(select(.createdAt > \"2025-01-01\"))' (filter by date).",
         },
       },
       required: ["toolCallId", "jqQuery"],

package/src/processors/tools/listStoredToolResponses.ts

@@ -72,7 +72,7 @@ export const listStoredToolResponsesDefinition: Tool = {
   function: {
     name: "listStoredToolResponses",
     description:
-      "List all stored tool responses with metadata including tool call ID, tool name, size, timestamp, and a preview of the content. Use this to discover which tool responses are available for querying with jqToolResponse or grepToolResponse.",
+      "List all stored tool responses with metadata including tool call ID, tool name, size, timestamp, and a preview of the content. ALWAYS call this before using jqToolResponse, grepToolResponse, or tailToolResponse — you need the toolCallId from this list to query a specific response. The tool name shown here corresponds to the tool that produced the response (e.g. 'mcp_1_knowhow-web_GetOrgUserTask'), making it easy to identify which response you want.",
     parameters: {
       type: "object",
       positional: false,

package/src/processors/tools/tailToolResponse.ts

@@ -11,12 +11,17 @@ export async function executeTail(
   data: string,
   toolCallId: string,
   availableIds: string[],
-  options?: TailOptions
+  options?: TailOptions,
+  toolNameMap?: { [toolCallId: string]: string }
 ): Promise<string> {
   if (data === null || data === undefined) {
-    return `Error: No tool response found for toolCallId "${toolCallId}". Available IDs: ${availableIds.join(
-      ", "
-    )}`;
+    const idList = availableIds
+      .map((id) => {
+        const name = toolNameMap?.[id];
+        return name ? `${id} (${name})` : id;
+      })
+      .join("\n  - ");
+    return `Error: No tool response found for toolCallId "${toolCallId}". Call listStoredToolResponses to see all available responses with their tool names.\n\nAvailable toolCallIds:\n  - ${idList || "(none)"}`;
   }
 
   try {

package/src/services/GitHub.ts

@@ -1,5 +1,5 @@
 import { Octokit } from "@octokit/rest";
-import axios from "axios";
+import http from "../utils/http";
 
 export class GitHubService {
   octokit: Octokit;
@@ -22,7 +22,7 @@ export class GitHubService {
   }
 
   private async getLfsContent(downloadUrl: string): Promise<string> {
-    const response = await axios.get(downloadUrl);
+    const response = await http.get(downloadUrl);
     return JSON.stringify(response.data);
   }