@tyvm/knowhow 0.0.137 → 0.0.140

package/autodoc/chat-guide.md

@@ -79,6 +79,7 @@ The following commands are available from the CLI modules shown in the code. (Ad
 | `/pause` | Pause the attached agent | `agent:attached` |
 | `/unpause` | Unpause the attached agent | `agent:attached` |
 | `/kill` | Terminate the attached agent | `agent:attached` |
+| `/compact` | Force the attached agent to summarize/compact its conversation now, dropping back to a much cheaper context size | `agent:attached` |
 | `/detach` | Detach from the attached agent task (leaves agent running) | `agent:attached` |
 | `/done` | Exit the current agent interaction (detaches) | `agent:attached` |
 

package/package.json

@@ -1,11 +1,14 @@
 {
   "name": "@tyvm/knowhow",
-  "version": "0.0.137",
+  "version": "0.0.140",
   "description": "ai cli with plugins and agents",
   "main": "ts_build/src/index.js",
   "bin": {
     "knowhow": "bin/knowhow.js"
   },
+  "engines": {
+    "node": ">=20"
+  },
   "scripts": {
     "test": "jest --testTimeout 300000",
     "test:debug": "node --inspect-brk ../../node_modules/jest/bin/jest.js --detectOpenHandles --forceExit --testTimeout 300000",
@@ -39,7 +42,7 @@
     "@types/express": "^4.17.13",
     "@types/jest": "^29.5.13",
     "@types/mocha": "^10.0.8",
-    "@types/node": "^20.6.3",
+    "@types/node": "^24.0.0",
     "@types/ws": "^8.18.1",
     "jest": "^29.1.1",
     "prettier": "2.6.2",

package/scripts/build-for-node.sh

@@ -5,7 +5,7 @@
 # Example: npm run node:build 24
 #
 # This script:
-#   1. Compiles TypeScript with Node 20 (required for workspace deps)
+#   1. Compiles TypeScript with Node 24 (falls back to current node if unavailable)
 #   2. Creates /tmp/knowhow-node-<major> with the compiled output
 #   3. Symlinks the package globally for ALL installed nvm versions matching the target
 #
@@ -31,9 +31,9 @@ PACKAGE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
 
 echo "📦 Package dir: $PACKAGE_DIR"
 
-# --- Find Node 20 for compiling TypeScript ---
+# --- Find Node 24 for compiling TypeScript ---
 NODE20_BIN=""
-for dir in "$HOME/.nvm/versions/node"/v20.*/bin; do
+for dir in "$HOME/.nvm/versions/node"/v24.*/bin; do
   if [ -f "$dir/node" ]; then
     NODE20_BIN="$dir/node"
     break
@@ -41,7 +41,7 @@ for dir in "$HOME/.nvm/versions/node"/v20.*/bin; do
 done
 
 if [ -z "$NODE20_BIN" ]; then
-  echo "⚠️  Node 20 not found via nvm, falling back to current node for TS compile"
+  echo "⚠️  Node 24 not found via nvm, falling back to current node for TS compile"
   NODE20_BIN="$(which node)"
 fi
 

package/src/agents/base/base.ts

@@ -43,6 +43,41 @@ export interface ToolCallEvent {
   functionResp: any;
 }
 
+/**
+ * A source is anything that can (a) start producing data, (b) hand each datum to
+ * a callback, and (c) be torn down. This is the unified contract behind
+ * `agent.observe()` — tool polls, events, streams, promises, peer agent threads,
+ * file tails, etc. all become adapters that satisfy this shape.
+ */
+export interface ObservationSource {
+  /** Human-friendly label for logs / delivered messages. */
+  label: string;
+  /**
+   * Begin producing. Call `emit(datum)` for each piece of data. Return a
+   * teardown function (or void). May be sync or async.
+   */
+  start(
+    emit: (datum: unknown) => void
+  ): void | (() => void) | Promise<void | (() => void)>;
+}
+
+export interface ObserveOpts {
+  /** Only deliver when the datum changed since the last emit. Default true. */
+  onlyOnChange?: boolean;
+  /** Max number of updates before auto-stopping. Default 20. */
+  maxUpdates?: number;
+  /** Max total duration in ms before auto-stopping. Default 10 min. */
+  maxDurationMs?: number;
+}
+
+interface ActiveObservation {
+  id: string;
+  label: string;
+  updates: number;
+  teardownPromise: Promise<void | (() => void)>;
+  expiry?: NodeJS.Timeout;
+}
+
 export abstract class BaseAgent implements IAgent {
   abstract name: string;
   abstract description: string;
@@ -63,25 +98,35 @@ export abstract class BaseAgent implements IAgent {
   protected turnCount = 0;
   protected totalCostUsd = 0;
   protected currentThread = 0;
-  protected reasoningEffort: CompletionOptions["reasoning_effort"] | undefined = undefined;
+  protected reasoningEffort: CompletionOptions["reasoning_effort"] | undefined =
+    undefined;
   protected summarizeReasoning: boolean | undefined = undefined;
   protected totalInputTokens = 0;
   protected totalOutputTokens = 0;
   protected totalCacheReadTokens = 0;
   protected totalCacheWriteTokens = 0;
 
+  // The real prompt/input token count reported by the most recent completion
+  // response (input + cache read + cache write). This reflects exactly how many
+  // tokens the model consumed for the last request and is far more accurate than
+  // the whitespace-based estimate in getMessagesLength(). Used to decide when to
+  // compact. 0 means "no real usage seen yet" (fall back to the estimate).
+  protected lastPromptTokens = 0;
+
   protected compressThreshold = 30000;
   protected compressMinMessages = 30;
 
+  // Set when the user manually requests compaction (e.g. via /compact while
+  // attached). Forces the next loop iteration to compress the conversation
+  // regardless of whether the token threshold has been reached.
+  protected _forceCompact = false;
+
   // Interrupt support: resolves the currently awaited tool call or completion
   private _interruptResolve: (() => void) | null = null;
   // A monotonically-increasing token identifying the active interruptible window.
   // Used to ensure a stale (slow) operation completing in the background cannot
   // clobber a newer interruptible window's resolver.
   private _interruptToken = 0;
-  // Set when interrupt() is called with no active window, so the next
-  // makeInterruptible fires immediately instead of dropping the interrupt.
-  private _pendingInterrupt = false;
 
   protected threads = [] as Message[][];
 
@@ -91,6 +136,12 @@ export abstract class BaseAgent implements IAgent {
   // Internal messages
   protected pendingMessages = [] as Message[];
 
+  // Active observations (subscriptions) owned by this agent instance. Kept
+  // per-agent (not module scope) so stopObserving() on one agent can't clobber
+  // another agent's observers, and so newTask() can tear them all down.
+  protected _observations = new Map<string, ActiveObservation>();
+  private _observationSeq = 0;
+
   protected taskBreakdown = "";
   protected summaries = [] as string[];
   protected currentTaskId: string | null = null;
@@ -178,11 +229,24 @@ export abstract class BaseAgent implements IAgent {
     this.compressThreshold = threshold;
   }
 
+  /**
+   * Request that the conversation be compacted at the next loop iteration,
+   * regardless of whether the token threshold has been reached. Combined with
+   * an interrupt(), this lets a user force compaction on an attached agent
+   * mid-task (e.g. via the `/compact` chat command) to drop back to a much
+   * cheaper per-interaction context size.
+   */
+  requestCompact() {
+    this._forceCompact = true;
+  }
+
   /**
    * Returns the effective compress threshold for the current model.
    * If the user has manually set a custom threshold (different from the default 30k),
    * that value is used as-is. Otherwise, the threshold is dynamically computed as
-   * 85% of the model's context window limit, falling back to DEFAULT_CONTEXT_LIMIT.
+   * 70% of the model's context window limit (or tiered-pricing threshold), falling
+   * back to DEFAULT_CONTEXT_LIMIT. The 70% factor leaves headroom so token-count
+   * lag doesn't push a request past the model/tier limit before compression fires.
    */
   getCompressThreshold(): number {
     if (this.compressThreshold !== DEFAULT_CONTEXT_LIMIT) {
@@ -194,7 +258,7 @@ export abstract class BaseAgent implements IAgent {
     );
     const contextLimit = result?.contextLimit ?? DEFAULT_CONTEXT_LIMIT;
     const threshold = result?.threshold ?? contextLimit;
-    return Math.floor(threshold * 0.85);
+    return Math.floor(threshold * 0.7);
   }
 
   setMaxTurns(maxTurns: number | null) {
@@ -219,6 +283,7 @@ export abstract class BaseAgent implements IAgent {
     this.totalOutputTokens = 0;
     this.totalCacheReadTokens = 0;
     this.totalCacheWriteTokens = 0;
+    this.lastPromptTokens = 0;
     this.status = this.eventTypes.inProgress;
     this.turnCount = 0;
     this.startTimeMs = Date.now();
@@ -227,7 +292,11 @@ export abstract class BaseAgent implements IAgent {
     // Reset interrupt state so a queued/stale interrupt from a prior task
     // can't fire against the new task.
     this._interruptResolve = null;
-    this._pendingInterrupt = false;
+    this._forceCompact = false;
+
+    // Tear down any observers from a prior task so they can't leak or inject
+    // into the new task.
+    this.stopObserving(undefined, "new task started");
 
     // Emit event for plugin integration
     const id = taskId || this.startTimeMs.toString();
@@ -473,6 +542,18 @@ export abstract class BaseAgent implements IAgent {
     const cacheWriteTokens =
       usage.cache_creation_input_tokens ?? usage.cache_write_tokens ?? 0;
 
+    // Record the real prompt token count for this single completion (not the
+    // running total). This represents the full context the model processed for
+    // the request: fresh input + cache-read + cache-write tokens. It is used by
+    // the compaction check as an accurate replacement for the whitespace-based
+    // getMessagesLength() estimate. Only track it for "prompt-shaped" usage
+    // (i.e. when there are input tokens) so a usage-less interrupt stub doesn't
+    // reset it to 0.
+    const promptTokens = inputTokens + cacheReadTokens + cacheWriteTokens;
+    if (promptTokens > 0) {
+      this.lastPromptTokens = promptTokens;
+    }
+
     this.totalInputTokens += inputTokens;
     this.totalOutputTokens += outputTokens;
     this.totalCacheReadTokens += cacheReadTokens;
@@ -575,7 +656,14 @@ export abstract class BaseAgent implements IAgent {
     };
 
     const { functionResp, toolMessages } = await this.makeInterruptible(
-      this.tools.callTool(toolCall, this.getEnabledToolNames()),
+      // Pass this agent in as the per-call context (`_ctx`) so tools can be
+      // self-referential: they can read `_ctx.caller` (this agent) and
+      // `_ctx.taskId` to, e.g., look up the current task, send pending
+      // messages, or subscribe to the caller's events.
+      this.tools.callTool(toolCall, this.getEnabledToolNames(), {
+        caller: this,
+        taskId: this.currentTaskId,
+      }),
       interruptResult
     );
 
@@ -771,13 +859,6 @@ export abstract class BaseAgent implements IAgent {
 
       this._interruptResolve = () => finish(() => resolve(interruptValue));
 
-      // If an interrupt arrived while there was no active window, honor it now.
-      if (this._pendingInterrupt) {
-        this._pendingInterrupt = false;
-        finish(() => resolve(interruptValue));
-        return;
-      }
-
       promise
         .then((result) => {
           // If this window was already interrupted/settled, drop the stale
@@ -815,12 +896,13 @@ export abstract class BaseAgent implements IAgent {
       this._interruptResolve();
     } else {
       // No active interruptible window right now (we're between the AI
-      // completion and the tool call, or in message processing). Queue the
-      // interrupt so the next makeInterruptible window fires immediately
-      // instead of silently dropping it.
-      this._pendingInterrupt = true;
+      // completion and the tool call, or in message processing). There is
+      // nothing to interrupt, so we do NOT queue it — a queued interrupt
+      // would pre-empt the next tool call and drop its result. Any message
+      // supplied with the poke was already added as a pending user message
+      // above, so the agent will still see it on its next step.
       this.log(
-        "No active interruptible operation — queued interrupt for next step",
+        "No active interruptible operation to interrupt",
         "warn"
       );
     }
@@ -917,8 +999,12 @@ export abstract class BaseAgent implements IAgent {
           tools: this.getEnabledTools(),
           tool_choice: "auto",
           long_ttl_cache: this.runTime() > 300_000,
-          ...(this.reasoningEffort !== undefined && { reasoning_effort: this.reasoningEffort }),
-          ...(this.summarizeReasoning !== undefined && { reasoning_summary: this.summarizeReasoning }),
+          ...(this.reasoningEffort !== undefined && {
+            reasoning_effort: this.reasoningEffort,
+          }),
+          ...(this.summarizeReasoning !== undefined && {
+            reasoning_summary: this.summarizeReasoning,
+          }),
         },
         { interruptValue: interruptResponse }
       );
@@ -1109,18 +1195,37 @@ export abstract class BaseAgent implements IAgent {
         return firstMessage.content;
       }
 
-      if (
-        this.getMessagesLength(messages) > this.getCompressThreshold() &&
-        messages.length > this.compressMinMessages
-      ) {
+      // Compress when either the token threshold is exceeded OR the user has
+      // manually requested compaction (e.g. via /compact on an attached agent).
+      // A manual request bypasses the token threshold but still respects the
+      // minimum-message guard so we never try to compress a tiny conversation.
+      //
+      // The size check uses getContextTokenCount(), which prefers the REAL
+      // prompt token count from the last completion response over the
+      // whitespace-based estimate — so it compares real tokens against the
+      // real (token-based) compress threshold.
+      const contextTokens = this.getContextTokenCount(messages);
+      const overThreshold =
+        contextTokens > this.getCompressThreshold() &&
+        messages.length > this.compressMinMessages;
+
+      if (overThreshold || this._forceCompact) {
         const taskBreakdown = await this.getTaskBreakdown(messages);
         this.log(
-          `Compressing messages: ${this.getMessagesLength(
-            messages
-          )} exceeds ${this.getCompressThreshold()}`
+          this._forceCompact
+            ? `Compacting messages (manual /compact request): ${contextTokens} tokens`
+            : `Compressing messages: ${contextTokens} tokens exceeds ${this.getCompressThreshold()}`
         );
+        this._forceCompact = false;
         messages = await this.compressMessages(messages, startIndex, endIndex);
         this.startNewThread(messages);
+        // Reset the real-token tracker. compressMessages() ran a completion over
+        // the FULL (pre-compression) thread, so lastPromptTokens now reflects
+        // that large context — not the small compressed thread. Zeroing it makes
+        // getContextTokenCount() fall back to the whitespace estimate of the new
+        // (small) thread until the next real completion reports fresh usage,
+        // preventing an immediate false re-compaction on the following iteration.
+        this.lastPromptTokens = 0;
       }
 
       if (["assistant", "tool"].includes(messages[messages.length - 1].role)) {
@@ -1284,10 +1389,130 @@ export abstract class BaseAgent implements IAgent {
     this.events.emit(this.eventTypes.userSay, message.content);
   }
 
+  /**
+   * Subscribe to any data-yielding source and stream its data into this agent's
+   * loop as non-blocking pending messages. This is the unified subscription
+   * primitive behind the `observe` tool — a tool poll, an event, a stream, a
+   * peer agent's thread, a file tail, etc. all become `ObservationSource`
+   * adapters feeding the same sink (`addPendingMessage`).
+   *
+   * Returns an observation id that can be passed to `stopObserving`. Auto-expires
+   * after `maxUpdates` or `maxDurationMs` so a forgotten observer never leaks.
+   */
+  observe(source: ObservationSource, opts: ObserveOpts = {}): string {
+    const {
+      onlyOnChange = true,
+      maxUpdates = 20,
+      maxDurationMs = 10 * 60 * 1000,
+    } = opts;
+
+    this._observationSeq += 1;
+    // Keep the id free of characters that would break id-extraction regexes
+    // (labels can contain parens, spaces, JSON, etc).
+    const safeLabel = source.label.replace(/[^A-Za-z0-9]+/g, "_").slice(0, 24);
+    const id = `obs_${safeLabel}_${Date.now()}_${this._observationSeq}`;
+    let last: string | undefined;
+
+    const emit = (datum: unknown) => {
+      // status gate: don't inject once the agent is done
+      if (this.status === this.eventTypes.done) {
+        this.stopObserving(id);
+        return;
+      }
+      const str = typeof datum === "string" ? datum : JSON.stringify(datum);
+      if (onlyOnChange && str === last) return; // dedupe noise
+      last = str;
+      const obs = this._observations.get(id);
+      if (!obs) return;
+      obs.updates += 1;
+      this.addPendingMessage({
+        role: "user",
+        content: `[observe:${id}] ${source.label} (update ${obs.updates}/${maxUpdates}):\n${str}`,
+      });
+      if (obs.updates >= maxUpdates) {
+        this.stopObserving(id, "max updates reached");
+      }
+    };
+
+    const teardownPromise = Promise.resolve(source.start(emit)).catch((e) => {
+      this.log(`observe(${source.label}) start failed: ${e?.message ?? e}`, "warn");
+      return undefined;
+    });
+
+    const expiry = maxDurationMs
+      ? setTimeout(() => this.stopObserving(id, "max duration reached"), maxDurationMs)
+      : undefined;
+    if (expiry && typeof (expiry as any).unref === "function") {
+      (expiry as any).unref();
+    }
+
+    this._observations.set(id, {
+      id,
+      label: source.label,
+      updates: 0,
+      teardownPromise,
+      expiry,
+    });
+    return id;
+  }
+
+  /**
+   * Cancel one observation (by id) or all of them (no id). Companion to
+   * `observe`. Runs each source's teardown function if it returned one.
+   */
+  stopObserving(id?: string, reason = "stopped"): string {
+    const ids = id ? [id] : [...this._observations.keys()];
+    let stopped = 0;
+    for (const oid of ids) {
+      const obs = this._observations.get(oid);
+      if (!obs) continue;
+      if (obs.expiry) clearTimeout(obs.expiry);
+      Promise.resolve(obs.teardownPromise)
+        .then((fn) => {
+          if (typeof fn === "function") fn();
+        })
+        .catch(() => {});
+      this._observations.delete(oid);
+      stopped += 1;
+    }
+    return id
+      ? stopped
+        ? `Stopped observer ${id} (${reason}).`
+        : `No active observer found with id: ${id}.`
+      : `Stopped ${stopped} active observer(s) (${reason}).`;
+  }
+
   getMessagesLength(messages: Message[]) {
     return JSON.stringify(messages).split(" ").length;
   }
 
+  /**
+   * Returns the best available estimate of how many tokens the current
+   * conversation occupies in the model's context window, for the purpose of
+   * deciding when to compact.
+   *
+   * Prefers the REAL prompt token count reported by the most recent completion
+   * response (lastPromptTokens) — this is exactly how many tokens the model
+   * consumed for the last request (fresh input + cache read + cache write), so
+   * it is far more accurate than the whitespace-based getMessagesLength()
+   * heuristic (which counts space-separated chunks of the JSON-stringified
+   * messages and can be off by a large factor).
+   *
+   * Falls back to the whitespace estimate when no real usage has been observed
+   * yet (e.g. before the first completion, or when a provider omits usage).
+   *
+   * Note: lastPromptTokens reflects the context size at the START of the last
+   * request, so it lags by whatever the model generated + tool results appended
+   * since then. That lag is intentionally covered by the 70% headroom baked
+   * into getCompressThreshold().
+   */
+  getContextTokenCount(messages: Message[]): number {
+    if (this.lastPromptTokens > 0) {
+      return this.lastPromptTokens;
+    }
+    return this.getMessagesLength(messages);
+  }
+
   /**
    * Detects whether tool call arguments appear truncated due to hitting the output token limit.
    * Two signals are checked:

package/src/agents/patcher/patcher.ts

@@ -80,7 +80,7 @@ export class PatchingAgent extends BaseAgent {
 
     this.setModelPreferences([
       { model: Models.anthropic.Sonnet4_6, provider: "anthropic" },
-      { model: Models.openai.GPT_53_Codex, provider: "openai" },
+      { model: Models.openai.GPT_56_Sol, provider: "openai" },
       { model: Models.xai.GrokCodeFast, provider: "xai" },
     ]);
   }

package/src/agents/researcher/researcher.ts

@@ -13,9 +13,9 @@ export class ResearcherAgent extends BaseAgent {
     // fallbacks, an unregistered "google" provider causes the health-check loop
     // to immediately throw "We have exhausted all model preferences."
     this.setModelPreferences([
-      { model: Models.google.Gemini_3_Flash_Preview, provider: "google" },
+      { model: Models.openai.GPT_56_Terra, provider: "openai" },
       { model: Models.anthropic.Sonnet4_6, provider: "anthropic" },
-      { model: Models.openai.GPT_53_Codex, provider: "openai" },
+      { model: Models.google.Gemini_35_Flash_Lite, provider: "google" },
     ]);
     this.disableTool("patchFile");
     this.disableTool("writeFile");

package/src/agents/tools/agentCall.ts

@@ -2,6 +2,12 @@ import { getConfig } from "../../config";
 import { services, ToolsService } from "../../services";
 import { getEnabledPlugins } from "../../types";
 
+/**
+ * Ask another registered agent a question. Resolves with a structured
+ * `{ answer, costUsd }` object (via `AgentService.callAgent`) so callers — most
+ * importantly the script sandbox — can account the subagent's spend against a
+ * budget. `answer` is the target agent's final response.
+ */
 export async function agentCall(agentName: string, userInput: string) {
   return new Promise(async (resolve, reject) => {
     const config = await getConfig();