clementine-agent 1.18.167 → 1.18.169

package/dist/agent/run-agent.d.ts

@@ -31,6 +31,7 @@ export declare function invalidateMcpStatusEntry(name: string): {
     cleared: boolean;
     updatedAt: string;
 };
+import { type ToolOutputGuardConfig } from './tool-output-guard.js';
 import type { AgentProfile } from '../types.js';
 import type { AgentManager } from './agent-manager.js';
 import type { MemoryStore } from '../memory/store.js';
@@ -110,6 +111,12 @@ export interface RunAgentOptions {
      *  team-task) keep the prompt small. When unset, falls back to
      *  profile.systemPromptBody (legacy single-source behavior). */
     systemPromptAppend?: string;
+    /** Per-run override for the tool-output-guard config (1.18.169).
+     *  Defaults come from src/config.ts TOOL_OUTPUT_GUARD (env +
+     *  clementine.json). Pass null to disable the guard for this run
+     *  (rarely needed — almost always a sign that perTool overrides
+     *  would be safer). */
+    toolOutputGuard?: Partial<ToolOutputGuardConfig> | null;
 }
 export interface RunAgentResult {
     /** Final text response from the agent. */

package/dist/agent/run-agent.js

@@ -84,7 +84,8 @@ export function invalidateMcpStatusEntry(name) {
     };
     return { cleared: _lastMcpStatusSnapshot.servers.length < before, updatedAt: _lastMcpStatusSnapshot.updatedAt };
 }
-import { BASE_DIR, PKG_DIR, CLAUDE_CODE_OAUTH_TOKEN, ANTHROPIC_API_KEY as CONFIG_ANTHROPIC_API_KEY, normalizeClaudeSdkOptionsForOneMillionContext, } from '../config.js';
+import { BASE_DIR, PKG_DIR, CLAUDE_CODE_OAUTH_TOKEN, ANTHROPIC_API_KEY as CONFIG_ANTHROPIC_API_KEY, normalizeClaudeSdkOptionsForOneMillionContext, TOOL_OUTPUT_GUARD, } from '../config.js';
+import { buildGuardHooks } from './tool-output-guard.js';
 import { buildAgentMap } from './agent-definitions.js';
 import { buildExecutionToolPolicy, } from './execution-policy.js';
 const MCP_SERVER_SCRIPT = path.join(PKG_DIR, 'dist', 'tools', 'mcp-server.js');
@@ -277,6 +278,69 @@ export async function runAgent(prompt, opts) {
             opts.abortSignal.addEventListener('abort', () => sdkAbortController.abort(), { once: true });
         }
     }
+    // PRD §6 / 1.18.85: stable run id created before sdkOptions so the
+    // tool-output guard (1.18.169) can namespace its on-disk archive by
+    // runId. EventLog writers below also reference this id.
+    const runId = randomUUID();
+    const eventLog = new EventLog();
+    let eventSeq = 0;
+    const writeEvent = (e) => {
+        try {
+            eventLog.append({
+                ...e,
+                runId,
+                seq: eventSeq++,
+            });
+        }
+        catch { /* never block */ }
+    };
+    // ── Tool-output guard hooks (1.18.169) ─────────────────────────────
+    // Bounds the per-tool-call output that reaches the model so SDK
+    // auto-compaction never thrashes on a runaway MCP result. The hook
+    // ALSO mirrors compression events into the run's EventLog so the Run
+    // detail page can show "[guard] outlook_inbox: 412KB → 28KB" badges.
+    // Per-run config merges over the static TOOL_OUTPUT_GUARD defaults;
+    // pass opts.toolOutputGuard = null to opt out entirely.
+    //
+    // `mostRecentUsageTokens` is updated from each assistant message's
+    // usage block (input + cache_read + cache_creation tokens). The
+    // window estimate is a conservative 180K — even 1M-context runs
+    // benefit from staying near 200K because compaction kicks in
+    // earlier and tools.outputs amplify thrash regardless of window.
+    let mostRecentUsageTokens = 0;
+    const usageWindowEstimate = 180_000; // tokens, conservative
+    const guardConfig = opts.toolOutputGuard === null
+        ? null
+        : {
+            softLimitBytes: opts.toolOutputGuard?.softLimitBytes ?? TOOL_OUTPUT_GUARD.softLimitBytes,
+            hardLimitBytes: opts.toolOutputGuard?.hardLimitBytes ?? TOOL_OUTPUT_GUARD.hardLimitBytes,
+            adaptive: opts.toolOutputGuard?.adaptive ?? TOOL_OUTPUT_GUARD.adaptive,
+            perTool: { ...TOOL_OUTPUT_GUARD.perTool, ...(opts.toolOutputGuard?.perTool ?? {}) },
+        };
+    const guard = guardConfig
+        ? buildGuardHooks({
+            runId,
+            config: guardConfig,
+            usageRatio: () => mostRecentUsageTokens / usageWindowEstimate,
+            onCompress: (info) => {
+                writeEvent({
+                    kind: 'tool_result',
+                    ts: new Date().toISOString(),
+                    sessionId,
+                    toolUseId: info.toolUseId,
+                    toolResult: {
+                        _clementine_guard: true,
+                        tool: info.toolName,
+                        originalBytes: info.originalBytes,
+                        capBytes: info.capBytes,
+                        bytesShed: info.bytesShed,
+                        ceilingHit: info.ceilingHit,
+                        ...(info.archivePath ? { archivePath: info.archivePath } : {}),
+                    },
+                });
+            },
+        })
+        : { hooks: {}, stats: { inspected: 0, compressed: 0, ceilingHits: 0, bytesShed: 0, compactions: 0 } };
     // Apply 1M-context env normalization (existing infra)
     const sdkOptionsRaw = {
         systemPrompt: profileAppend
@@ -316,6 +380,10 @@ export async function runAgent(prompt, opts) {
         ...(opts.additionalDirectories && opts.additionalDirectories.length > 0
             ? { additionalDirectories: opts.additionalDirectories }
             : {}),
+        // 1.18.169 — install the tool-output guard hooks. SDK types accept
+        // `hooks` keyed by HookEvent; the empty object is a no-op when the
+        // guard is disabled.
+        ...(Object.keys(guard.hooks).length > 0 ? { hooks: guard.hooks } : {}),
     };
     const sdkOptions = normalizeClaudeSdkOptionsForOneMillionContext(sdkOptionsRaw);
     logger.info({
@@ -332,23 +400,9 @@ export async function runAgent(prompt, opts) {
         permissionMode: toolPolicy.permissionMode,
         mcpServerCount: Object.keys(mcpServers).length,
     }, 'runAgent: starting query');
-    // PRD §6 / 1.18.85: path A in-process tap. One Event row per significant
-    // SDK message, written to ~/.clementine/events/<runId>.jsonl. The Run
-    // detail page (Phase 4b) reads from this file. EventLog.append never
-    // throws back to the caller — telemetry is best-effort.
-    const runId = randomUUID();
-    const eventLog = new EventLog();
-    let eventSeq = 0;
-    const writeEvent = (e) => {
-        try {
-            eventLog.append({
-                ...e,
-                runId,
-                seq: eventSeq++,
-            });
-        }
-        catch { /* never block */ }
-    };
+    // PRD §6 / 1.18.85: path A in-process tap. runId / eventLog / writeEvent
+    // are declared earlier (above sdkOptionsRaw) because the tool-output
+    // guard's onCompress callback needs them at hook-registration time.
     let finalText = '';
     let sessionId = '';
     let totalCostUsd = 0;
@@ -389,6 +443,21 @@ export async function runAgent(prompt, opts) {
             }
             if (message.type === 'assistant') {
                 const am = message;
+                // 1.18.169 — capture this turn's usage so the tool-output guard can
+                // adaptively tighten its cap as cumulative context climbs. We sum
+                // input + cache_read + cache_creation because all three count
+                // against the model's window for the NEXT turn. Output_tokens isn't
+                // included — it's not retained in context after the model response
+                // is processed.
+                const tokenUsage = am.message?.usage;
+                if (tokenUsage) {
+                    const recent = (tokenUsage.input_tokens ?? 0)
+                        + (tokenUsage.cache_read_input_tokens ?? 0)
+                        + (tokenUsage.cache_creation_input_tokens ?? 0);
+                    if (Number.isFinite(recent) && recent > 0) {
+                        mostRecentUsageTokens = recent;
+                    }
+                }
                 // SDK content blocks include text, tool_use, and (when extended-thinking
                 // is enabled) thinking. We tap each kind into the Event store.
                 const blocks = (am.message?.content ?? []);
@@ -562,6 +631,15 @@ export async function runAgent(prompt, opts) {
         totalCostUsd: Number(totalCostUsd.toFixed(4)),
         durationMs: Date.now() - startedAt,
         finalTextChars: finalText.length,
+        // 1.18.169 — tool-output guard summary, surfaced for observability.
+        // Non-zero `compressed` means the guard kept the SDK from thrashing.
+        guard: guard.stats.inspected > 0 ? {
+            inspected: guard.stats.inspected,
+            compressed: guard.stats.compressed,
+            bytesShed: guard.stats.bytesShed,
+            compactions: guard.stats.compactions,
+            ceilingHits: guard.stats.ceilingHits,
+        } : undefined,
     }, 'runAgent: query complete');
     // PRD §6 Phase 4e: subagent transcript backfill (Path C). The SDK persists
     // every subagent's full message stream to ~/.claude/projects/<encoded-cwd>/

package/dist/agent/tool-output-guard.d.ts

@@ -0,0 +1,165 @@
+/**
+ * tool-output-guard — PostToolUse hook that bounds per-call tool output size
+ * so the SDK's auto-compactor can never thrash on a runaway MCP result.
+ *
+ * Why this exists
+ * ───────────────
+ * Anthropic's Claude Agent SDK auto-compacts when context approaches the
+ * model's window. When a *single* tool result is larger than the room left
+ * after compaction, the next turn refills the window immediately. After 3
+ * consecutive compactions that don't help, the SDK throws:
+ *
+ *   "Autocompact is thrashing: the context refilled to the limit within 3
+ *    turns of the previous compact, 3 times in a row."
+ *
+ * This used to happen on outlook-email-triage and imessage-triage when their
+ * MCP tools (`mcp__claude_ai_Microsoft_365__outlook_inbox`,
+ * `imessage_read`) returned tens-to-hundreds of KB per call. Our own
+ * Clementine MCP tools cap output at 30KB (`capOutput` in src/tools/shared.ts)
+ * but third-party MCPs (Composio, claude.ai, iMessage) ignore that.
+ *
+ * The fix is the canonical Anthropic primitive: a `PostToolUse` hook that
+ * returns `hookSpecificOutput.updatedToolOutput` to replace the result
+ * before it reaches the model. From sdk.d.ts:1979 — "Replaces the tool
+ * output before it is sent to the model."
+ *
+ * Design properties
+ * ─────────────────
+ * 1. Operates at the SDK boundary, once. Every runAgent caller (chat, cron,
+ *    runSkill, heartbeat, team-task, hired agents) is protected for free.
+ * 2. Transparent: full payload is archived to disk so the agent can
+ *    `Read <path>` if it really needs the rest. Nothing is silently lost.
+ * 3. Structure-aware: arrays of objects keep the first N + last 2 items
+ *    plus a summary of the rest; emails/messages drop verbose body fields
+ *    when they alone exceed the cap.
+ * 4. Per-tool overrides via clementine.json `toolOutputGuard.perTool`.
+ * 5. Adaptive: when cumulative cache-creation tokens climb >50% of the
+ *    model's window, the soft cap shrinks ×0.5. Stops thrashing in the
+ *    pathological "many medium-sized calls" case the static cap misses.
+ * 6. Hard ceiling (default 200KB) always enforced — three back-to-back
+ *    400KB outputs would thrash even at 1M context.
+ *
+ * Failure mode: this module never throws. A bad input, an archive write
+ * failure, anything — falls back to returning the original output. The
+ * caller (runAgent → SDK) is unblocked. The guard logs the issue and
+ * continues. Telemetry must never break execution.
+ */
+import type { HookCallbackMatcher, HookEvent } from '@anthropic-ai/claude-agent-sdk';
+export interface ToolOutputGuardConfig {
+    /** Bytes — typical soft cap before compression kicks in. */
+    softLimitBytes: number;
+    /** Bytes — feasibility ceiling. Never exceeded regardless of context. */
+    hardLimitBytes: number;
+    /** When true, the soft cap shrinks as cumulative context fills up. */
+    adaptive: boolean;
+    /** Tool-name → bytes overrides. Keys match the SDK `tool_name` field
+     *  (MCP tools: `mcp__<server>__<tool>`; native tools: `Read`, `Bash`, …). */
+    perTool: Record<string, number>;
+}
+/** Default config from src/config.ts; callers can override per-run via
+ *  RunAgentOptions if a particular run truly needs more space. */
+export declare function defaultGuardConfig(): ToolOutputGuardConfig;
+export interface GuardRunStats {
+    /** Tool calls inspected by the guard. */
+    inspected: number;
+    /** Tool calls that exceeded the soft cap and were compressed. */
+    compressed: number;
+    /** Tool calls that exceeded the hard ceiling. */
+    ceilingHits: number;
+    /** Bytes of payload deferred to the archive (i.e. not sent to the model). */
+    bytesShed: number;
+    /** Number of SDK auto-compactions observed for this run. */
+    compactions: number;
+}
+/**
+ * Approximate the byte size of a tool_response as it will appear in the
+ * model's context. The SDK's tool_response may be a primitive, an object,
+ * a content-block array, or already-truncated string. We JSON-stringify
+ * because that's how the SDK ships it onwards, then byte-length the result.
+ */
+export declare function estimateBytes(value: unknown): number;
+interface CompressionContext {
+    toolName: string;
+    archivePath: string | null;
+    cap: number;
+}
+/**
+ * Tighten the soft cap when cumulative context usage is already high.
+ *
+ * Inputs: `recentUsageRatio` is the most-recent (cache_read + input) /
+ * window-size estimate from the SDK's usage block — see runAgent for how
+ * it's plumbed. We don't know the exact window (200K vs 1M depends on
+ * model + plan + 1m flag) so we pass a conservative 180K when nothing
+ * else is known.
+ *
+ *   usage <50% of window:  full softLimit
+ *   usage 50–75%:          softLimit × 0.6
+ *   usage ≥75%:            softLimit × 0.35
+ *
+ * The hard ceiling is never reduced — it's already a feasibility cap.
+ */
+export declare function adaptiveSoftCap(baseSoftLimit: number, recentUsageRatio: number, adaptive: boolean): number;
+/** Resolve the effective cap for a given tool call. */
+export declare function resolveCap(toolName: string, config: ToolOutputGuardConfig, usageRatio: number): {
+    softCap: number;
+    hardCap: number;
+};
+export interface CompressOutcome {
+    /** What goes back to the model. Same shape contract as the SDK
+     *  tool_response — string OR an object — preserving caller intent. */
+    output: unknown;
+    /** Bytes of payload that didn't reach the model. */
+    bytesShed: number;
+    /** Did we trip the hard ceiling? */
+    ceilingHit: boolean;
+    /** Did the input fit under the cap (no compression done)? */
+    passthrough: boolean;
+}
+/**
+ * Compress a tool result if it exceeds the cap. Pure function — does not
+ * write to disk. The caller handles archive + telemetry.
+ */
+export declare function compressToolOutput(_toolName: string, rawOutput: unknown, ctx: CompressionContext): CompressOutcome;
+export interface GuardHookOptions {
+    /** Stable run identifier — used to namespace the on-disk archive. */
+    runId: string;
+    /** Static config (env + clementine.json). */
+    config?: ToolOutputGuardConfig;
+    /** Optional callback fired on every compression. Used by runAgent to
+     *  record an Event row for the Run detail page. */
+    onCompress?: (info: {
+        toolName: string;
+        toolUseId: string;
+        originalBytes: number;
+        capBytes: number;
+        bytesShed: number;
+        ceilingHit: boolean;
+        archivePath: string | null;
+    }) => void;
+    /** Optional source of the current cumulative context-usage ratio
+     *  (cache_read + input) / window. Returns a number in [0,1]. The
+     *  guard calls this once per tool result to adapt the cap. When
+     *  absent, ratio is assumed 0 (full soft cap is always used). */
+    usageRatio?: () => number;
+    /** Optional archive root override for tests. Defaults to Clementine home. */
+    archiveBaseDir?: string;
+}
+export interface GuardHookHandles {
+    /** Hook map suitable for SDK `query({ options: { hooks } })`. */
+    hooks: Partial<Record<HookEvent, HookCallbackMatcher[]>>;
+    /** Aggregated telemetry — read after the run completes. */
+    stats: GuardRunStats;
+}
+/**
+ * Build the hook handles that runAgent will hand to the SDK.
+ *
+ * Returns one PostToolUse hook (does the work), one PreCompact hook
+ * (logs the compaction request), and one PostCompact hook (logs the
+ * summary length so we can correlate with the next turn's behavior).
+ *
+ * If `TOOL_OUTPUT_GUARD.enabled` is false, returns empty handles —
+ * a noop merge with whatever the caller already had.
+ */
+export declare function buildGuardHooks(opts: GuardHookOptions): GuardHookHandles;
+export {};
+//# sourceMappingURL=tool-output-guard.d.ts.map