@tyvm/knowhow 0.0.144 → 0.0.145

package/bin/knowhow.js

@@ -1,23 +1,25 @@
-#!/usr/bin/env node
+#!/usr/bin/env node --no-node-snapshot
 
-const { spawnSync } = require("node:child_process");
-const path = require("node:path");
-
-// If the current Node process wasn't started with --no-node-snapshot,
-// re-exec Node with the flag and forward the original arguments.
-if (!process.execArgv.includes("--no-node-snapshot")) {
-  const cliEntrypoint = path.join(__dirname, "../ts_build/src/cli.js");
+// knowhow CLI entry point.
+//
+// --no-node-snapshot is required for isolated-vm (used by executeScript).
+// It is declared in the shebang above so that Node starts with it directly —
+// no re-exec needed. This means the process stays as a direct child of the
+// terminal (e.g. Ghostty) and inherits its macOS TCC (Screen Recording /
+// Accessibility) permissions without needing to grant them to node itself.
 
-  const result = spawnSync(
-    process.execPath,
-    ["--no-node-snapshot", "--enable-source-maps", cliEntrypoint, ...process.argv.slice(2)],
-    {
-      stdio: "inherit",
-    }
-  );
-
-  process.exit(result.status ?? 1);
-}
+const path = require("node:path");
+const cliJs = path.join(__dirname, "../ts_build/src/cli.js");
 
-// Already running with the flag → just load the CLI
-require("../ts_build/src/cli.js");
+// cli.js only auto-runs main() when it is the entry module (require.main ===
+// module). Since we're requiring it here, we call main() ourselves so the
+// CLI actually runs and exits cleanly.
+const cli = require(cliJs);
+cli.main()
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  })
+  .then(() => {
+    process.exit(0);
+  });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tyvm/knowhow",
-  "version": "0.0.144",
+  "version": "0.0.145",
   "description": "ai cli with plugins and agents",
   "main": "ts_build/src/index.js",
   "bin": {

package/scripts/migrate-usage-json.js

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+/**
+ * Migration script: Compress usage.json files in .knowhow/processes/agents/
+ *
+ * The old format writes the full message thread with every usage entry, causing
+ * exponential file growth. This script applies the same compression logic as the
+ * new appendUsageEntry code: if entry[N].messages is a prefix of entry[N+1].messages,
+ * replace entry[N].messages with [{ PREV_CACHE_HIT: true }].
+ *
+ * Usage:
+ *   node scripts/migrate-usage-json.js            # dry run (shows space savings)
+ *   node scripts/migrate-usage-json.js --migrate  # actually migrates files
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const AGENTS_DIR = path.join(
+  __dirname,
+  "../.knowhow/processes/agents"
+);
+
+const DRY_RUN = !process.argv.includes("--migrate");
+
+if (DRY_RUN) {
+  console.log("=== DRY RUN (pass --migrate to actually migrate) ===\n");
+} else {
+  console.log("=== MIGRATING FILES ===\n");
+}
+
+function compressEntries(entries) {
+  let compressed = 0;
+  for (let i = 0; i < entries.length - 1; i++) {
+    const curr = entries[i];
+    const next = entries[i + 1];
+    const prevMessages = curr?.messages;
+    const nextMessages = next?.messages;
+
+    // Skip if already compressed
+    if (
+      Array.isArray(prevMessages) &&
+      prevMessages.length === 1 &&
+      prevMessages[0]?.PREV_CACHE_HIT
+    ) {
+      continue;
+    }
+
+    if (
+      Array.isArray(prevMessages) &&
+      Array.isArray(nextMessages) &&
+      prevMessages.length > 0 &&
+      nextMessages.length >= prevMessages.length &&
+      JSON.stringify(nextMessages.slice(0, prevMessages.length)) ===
+        JSON.stringify(prevMessages)
+    ) {
+      curr.messages = [{ PREV_CACHE_HIT: true }];
+      compressed++;
+    }
+  }
+  return compressed;
+}
+
+let totalOriginalBytes = 0;
+let totalNewBytes = 0;
+let totalFilesProcessed = 0;
+let totalFilesChanged = 0;
+let totalEntriesCompressed = 0;
+let errors = 0;
+
+const agentDirs = fs.readdirSync(AGENTS_DIR);
+
+for (const dir of agentDirs) {
+  const usagePath = path.join(AGENTS_DIR, dir, "usage.json");
+
+  if (!fs.existsSync(usagePath)) continue;
+
+  try {
+    const originalContent = fs.readFileSync(usagePath, "utf8");
+    const originalBytes = Buffer.byteLength(originalContent, "utf8");
+    totalOriginalBytes += originalBytes;
+    totalFilesProcessed++;
+
+    let entries;
+    try {
+      entries = JSON.parse(originalContent);
+    } catch (e) {
+      console.warn(`  [SKIP] ${dir}/usage.json - invalid JSON: ${e.message}`);
+      errors++;
+      continue;
+    }
+
+    if (!Array.isArray(entries) || entries.length === 0) {
+      totalNewBytes += originalBytes;
+      continue;
+    }
+
+    const compressed = compressEntries(entries);
+    totalEntriesCompressed += compressed;
+
+    const newContent = JSON.stringify(entries, null, 2);
+    const newBytes = Buffer.byteLength(newContent, "utf8");
+    totalNewBytes += newBytes;
+
+    const savedBytes = originalBytes - newBytes;
+    const savedPct = ((savedBytes / originalBytes) * 100).toFixed(1);
+
+    if (compressed > 0) {
+      totalFilesChanged++;
+      console.log(
+        `  ${dir}/usage.json: ${compressed} entries compressed, ` +
+          `${(originalBytes / 1024).toFixed(1)}KB -> ${(newBytes / 1024).toFixed(1)}KB ` +
+          `(-${(savedBytes / 1024).toFixed(1)}KB, ${savedPct}%)`
+      );
+
+      if (!DRY_RUN) {
+        fs.writeFileSync(usagePath, newContent, "utf8");
+      }
+    }
+  } catch (e) {
+    console.error(`  [ERROR] ${dir}: ${e.message}`);
+    errors++;
+  }
+}
+
+const totalSaved = totalOriginalBytes - totalNewBytes;
+const totalSavedPct = ((totalSaved / totalOriginalBytes) * 100).toFixed(1);
+
+console.log("\n=== SUMMARY ===");
+console.log(`Files found with usage.json:  ${totalFilesProcessed}`);
+console.log(`Files with compressible data: ${totalFilesChanged}`);
+console.log(`Total entries compressed:     ${totalEntriesCompressed}`);
+console.log(
+  `Total size before: ${(totalOriginalBytes / 1024 / 1024).toFixed(2)} MB`
+);
+console.log(
+  `Total size after:  ${(totalNewBytes / 1024 / 1024).toFixed(2)} MB`
+);
+console.log(
+  `Space saved:       ${(totalSaved / 1024 / 1024).toFixed(2)} MB (${totalSavedPct}%)`
+);
+if (errors > 0) console.log(`Errors: ${errors}`);
+
+if (DRY_RUN && totalFilesChanged > 0) {
+  console.log(
+    "\nRun with --migrate to apply changes (backups written as .bak files)."
+  );
+}

package/src/agents/base/base.ts

@@ -43,6 +43,11 @@ export interface ToolCallEvent {
   functionResp: any;
 }
 
+type ChatCompletion = Parameters<GenericClient["createChatCompletion"]>[0];
+type CachedChatCompletion = { messages: ChatCompletion["messages"] } & Partial<
+  Omit<ChatCompletion, "messages">
+>;
+
 /**
  * A source is anything that can (a) start producing data, (b) hand each datum to
  * a callback, and (c) be torn down. This is the unified contract behind
@@ -145,6 +150,12 @@ export abstract class BaseAgent implements IAgent {
   protected taskBreakdown = "";
   protected summaries = [] as string[];
   protected currentTaskId: string | null = null;
+  /**
+   * The taskId of the parent agent that spawned this task, if any.
+   * Used by self-referential tools (e.g. replyToParent) to know who to
+   * report back to without relying solely on reading metadata.json off disk.
+   */
+  public parentTaskId: string | null = null;
 
   public agentEvents = new EventService();
   public eventTypes = {
@@ -349,6 +360,14 @@ export abstract class BaseAgent implements IAgent {
     return this.provider;
   }
 
+  getParentTaskId(): string | null {
+    return this.parentTaskId;
+  }
+
+  setParentTaskId(value: string | null) {
+    this.parentTaskId = value ?? null;
+  }
+
   setProvider(value: keyof typeof Clients.clients) {
     this.provider = value;
     this.client = null; // Reset client to force re-fetch
@@ -595,14 +614,31 @@ export abstract class BaseAgent implements IAgent {
    * no-op).
    */
   protected async createAgentCompletion(
-    params: Parameters<GenericClient["createChatCompletion"]>[0],
+    params: CachedChatCompletion,
     options: {
       interruptValue?: CompletionResponse;
     } = {}
   ): Promise<CompletionResponse> {
     const { interruptValue } = options;
 
-    const callPromise = this.getClient().createChatCompletion(params);
+    // If you change the tools, or any of this, you bust cache
+    // only safe to change messages
+    const defaultedParams: ChatCompletion = {
+      model: this.getModel(),
+      tools: this.getEnabledTools(),
+      tool_choice: "auto",
+      long_ttl_cache: this.runTime() > 300_000,
+      ...(this.reasoningEffort !== undefined && {
+        reasoning_effort: this.reasoningEffort,
+      }),
+      ...(this.summarizeReasoning !== undefined && {
+        reasoning_summary: this.summarizeReasoning,
+      }),
+
+      ...params,
+    };
+
+    const callPromise = this.getClient().createChatCompletion(defaultedParams);
     const response = interruptValue
       ? await this.makeInterruptible(callPromise, interruptValue)
       : await callPromise;
@@ -663,6 +699,7 @@ export abstract class BaseAgent implements IAgent {
       this.tools.callTool(toolCall, this.getEnabledToolNames(), {
         caller: this,
         taskId: this.currentTaskId,
+        parentTaskId: this.parentTaskId ?? undefined,
       }),
       interruptResult
     );
@@ -677,6 +714,14 @@ export abstract class BaseAgent implements IAgent {
 
   logMessages(messages: Message[]) {
     for (const message of messages) {
+      // Surface the model's reasoning summary (what it's "thinking") when the
+      // provider exposes one, so it appears alongside the assistant's output.
+      const reasoningSummary = (message as any).reasoning_summary;
+      if (message.role === "assistant" && reasoningSummary) {
+        this.agentEvents.emit(this.eventTypes.agentSay, {
+          message: `💭 ${reasoningSummary}`,
+        });
+      }
       if (message.role === "assistant" && message.content) {
         this.agentEvents.emit(this.eventTypes.agentSay, {
           message: message.content,
@@ -901,10 +946,7 @@ export abstract class BaseAgent implements IAgent {
       // would pre-empt the next tool call and drop its result. Any message
       // supplied with the poke was already added as a pending user message
       // above, so the agent will still see it on its next step.
-      this.log(
-        "No active interruptible operation to interrupt",
-        "warn"
-      );
+      this.log("No active interruptible operation to interrupt", "warn");
     }
   }
 
@@ -1435,12 +1477,18 @@ export abstract class BaseAgent implements IAgent {
     };
 
     const teardownPromise = Promise.resolve(source.start(emit)).catch((e) => {
-      this.log(`observe(${source.label}) start failed: ${e?.message ?? e}`, "warn");
+      this.log(
+        `observe(${source.label}) start failed: ${e?.message ?? e}`,
+        "warn"
+      );
       return undefined;
     });
 
     const expiry = maxDurationMs
-      ? setTimeout(() => this.stopObserving(id, "max duration reached"), maxDurationMs)
+      ? setTimeout(
+          () => this.stopObserving(id, "max duration reached"),
+          maxDurationMs
+        )
       : undefined;
     if (expiry && typeof (expiry as any).unref === "function") {
       (expiry as any).unref();
@@ -1601,7 +1649,15 @@ export abstract class BaseAgent implements IAgent {
     const response = await this.createAgentCompletion({
       model,
       messages: taskBreakdownMessages,
-      max_tokens: 2000,
+      tools: this.getEnabledTools(),
+      tool_choice: "auto",
+      long_ttl_cache: this.runTime() > 300_000,
+      ...(this.reasoningEffort !== undefined && {
+        reasoning_effort: this.reasoningEffort,
+      }),
+      ...(this.summarizeReasoning !== undefined && {
+        reasoning_summary: this.summarizeReasoning,
+      }),
     });
 
     this.taskBreakdown = response.choices[0].message.content;
@@ -1644,7 +1700,6 @@ export abstract class BaseAgent implements IAgent {
     ] as Message[];
 
     const response = await this.createAgentCompletion({
-      model,
       messages: compressMessagesPayload,
     });
 

package/src/agents/tools/patch.ts

@@ -907,6 +907,46 @@ export function fixPatch(originalContent: string, patch: string): string {
   return finalPatch;
 }
 
+/**
+ * Verify that auto-correction did not broaden or change the requested edit.
+ * Line locations and hunk headers may change, but the exact multiset of added
+ * and removed lines must remain the same.
+ */
+export function autoCorrectionPreservesChanges(
+  originalContent: string,
+  updatedContent: string,
+  requestedPatch: string
+): boolean {
+  const changedLines = (patch: string, prefix: "+" | "-"): string[] =>
+    parseHunks(patch)
+      .flatMap((hunk) =>
+        hunk.lines
+          .filter((line) => line.startsWith(prefix))
+          .map((line) => line.slice(1))
+      )
+      .sort();
+
+  const actualPatch = createTwoFilesPatch(
+    "original",
+    "updated",
+    originalContent,
+    updatedContent,
+    "",
+    "",
+    { context: 0 }
+  );
+
+  const requestedAdditions = changedLines(requestedPatch, "+");
+  const requestedRemovals = changedLines(requestedPatch, "-");
+  const actualAdditions = changedLines(actualPatch, "+");
+  const actualRemovals = changedLines(actualPatch, "-");
+
+  return (
+    JSON.stringify(actualAdditions) === JSON.stringify(requestedAdditions) &&
+    JSON.stringify(actualRemovals) === JSON.stringify(requestedRemovals)
+  );
+}
+
 // --- Existing Application/Utility Code (Keep as is, ensure imports/exports are correct) ---
 
 export async function savePatchError(
@@ -976,9 +1016,32 @@ export async function patchFile(
       originalContent = await readFile(filePath, "utf8"); // Use async read
     }
 
-    let updatedContent = applyPatch(originalContent, patch);
-    let appliedPatch = patch; // Keep track of which patch succeeded
     const patchHunks = parseHunks(patch);
+    // `diff.applyPatch` can report success for malformed hunks whose declared
+    // line counts disagree with their bodies. In that case it consumes the
+    // wrong number of lines and can move additions across scope boundaries.
+    // Send those hunks through our re-anchoring logic instead of trusting the
+    // library's successful-but-corrupt result.
+    const hasInvalidHunkCounts = patchHunks.some((hunk) => {
+      const originalBodyCount = hunk.lines.filter(
+        (line) =>
+          !line.startsWith("+") &&
+          line !== "\"
+      ).length;
+      const newBodyCount = hunk.lines.filter(
+        (line) =>
+          !line.startsWith("-") &&
+          line !== "\"
+      ).length;
+      return (
+        originalBodyCount !== hunk.originalLineCount ||
+        newBodyCount !== hunk.newLineCount
+      );
+    });
+    let updatedContent = hasInvalidHunkCounts
+      ? false
+      : applyPatch(originalContent, patch);
+    let appliedPatch = patch; // Keep track of which patch succeeded
     const patchHasAdditions = patchHunks.some((h) => h.additions.length > 0);
     const patchHasDeletions = patchHunks.some((h) => h.subtractions.length > 0);
 
@@ -1113,6 +1176,25 @@ export async function patchFile(
 
     const wasFixed = appliedPatch !== patch;
 
+    // Auto-correction may re-anchor hunks, but it must never invent removals,
+    // drop requested edits, or move lines by representing them as extra edits.
+    // Reject before events or disk writes if its actual delta differs from the
+    // submitted patch.
+    if (
+      wasFixed &&
+      !autoCorrectionPreservesChanges(
+        originalContent,
+        updatedContent as string,
+        patch
+      )
+    ) {
+      return [
+        `❌ Patch failed: auto-correction changed the requested edit, so no changes were written.`,
+        `File: ${filePath} (${originalContent.split("\n").length} lines)`,
+        `Tip: Re-read the file and submit smaller hunks with unique surrounding context.`,
+      ].join("\n");
+    }
+
     const eventResults: any[] = [];
     // Emit pre-edit blocking event
     if (context.Events) {

package/src/agents/tools/textSearch.ts

@@ -1,10 +1,85 @@
 import { getConfiguredEmbeddings } from "../../embeddings";
 import { execAsync } from "./execCommand";
 
+// Max number of matching lines (with full text) we show per file before we
+// summarize the rest as a list of line numbers.
+const PER_FILE_LINE_LIMIT = 5;
+
+interface FileMatches {
+  file: string;
+  // [lineNumber, lineText]
+  lines: Array<[number, string]>;
+}
+
+/**
+ * Parse `ag --group --numbers` style output into per-file match groups.
+ * With --group, ag prints the file path on its own line, followed by
+ * `<lineNumber>:<text>` lines, then a blank line between files.
+ */
+function parseAgGroupedOutput(stdout: string): FileMatches[] {
+  const results: FileMatches[] = [];
+  let current: FileMatches | null = null;
+
+  for (const rawLine of stdout.split("\n")) {
+    if (rawLine === "") {
+      // Blank line separates file groups
+      current = null;
+      continue;
+    }
+
+    const numberMatch = rawLine.match(/^(\d+):([\s\S]*)$/);
+    if (numberMatch && current) {
+      current.lines.push([Number(numberMatch[1]), numberMatch[2]]);
+    } else {
+      // Treat as a file header line
+      current = { file: rawLine, lines: [] };
+      results.push(current);
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Format the parsed matches into a helpful, readable report. Instead of ag's
+ * "Too many matches ... Skipping" error, we report the total match count, the
+ * line numbers, and cap the displayed lines at PER_FILE_LINE_LIMIT.
+ */
+function formatMatches(files: FileMatches[]): string {
+  if (files.length === 0) return "No matches found.";
+
+  const sections: string[] = [];
+
+  for (const { file, lines } of files) {
+    const total = lines.length;
+    const shown = lines.slice(0, PER_FILE_LINE_LIMIT);
+
+    const header = `${file} (${total} match${total === 1 ? "" : "es"})`;
+    const body = shown
+      .map(([num, text]) => `${num}:${text}`)
+      .join("\n");
+
+    let section = `${header}\n${body}`;
+
+    if (total > PER_FILE_LINE_LIMIT) {
+      const remaining = lines.slice(PER_FILE_LINE_LIMIT);
+      const remainingLineNumbers = remaining.map(([num]) => num);
+      section +=
+        `\n... ${remaining.length} more match${
+          remaining.length === 1 ? "" : "es"
+        } (showing first ${PER_FILE_LINE_LIMIT} lines). ` +
+        `Remaining matches on lines: [${remainingLineNumbers.join(", ")}]`;
+    }
+
+    sections.push(section);
+  }
+
+  return sections.join("\n\n");
+}
+
 export async function textSearch(searchTerm) {
   try {
     // Escape the search term for safe shell usage
-    // Replace single quotes with '\'' which closes quote, adds escaped quote, reopens quote
     // 1) Normalize whitespace (turn newlines/tabs into spaces)
     const normalized = String(searchTerm)
       .replace(/\r\n?/g, "\n") // normalize CRLF/CR → LF
@@ -16,8 +91,28 @@ export async function textSearch(searchTerm) {
     // 2) Escape single quotes for safe single-quoted shell arg
     const escapedTerm = normalized.replace(/'/g, "'\\''");
 
-    const command = `ag -m 3 -Q '${escapedTerm}'`;
-    return await execAsync(command);
+    // Use --group + --numbers so we can parse per-file matches ourselves and
+    // produce a richer summary (counts + line numbers) instead of ag's
+    // unhelpful "Too many matches" error. We intentionally do NOT pass -m here.
+    const command = `ag --group --numbers -Q '${escapedTerm}'`;
+    const { stdout, stderr } = await execAsync(command);
+
+    const files = parseAgGroupedOutput(stdout || "");
+    const formatted = formatMatches(files);
+
+    // Preserve any non-"too many matches" stderr (real warnings/errors), but
+    // drop ag's per-file "Too many matches ... Skipping" noise since we now
+    // handle match limits ourselves.
+    const filteredStderr = (stderr || "")
+      .split("\n")
+      .filter((line) => !/Too many matches/i.test(line))
+      .join("\n")
+      .trim();
+
+    return {
+      stdout: formatted,
+      stderr: filteredStderr,
+    };
   } catch (err) {
     console.log(
       "Falling back to embeddings text search since ag was not available"