@bluecopa/harness 0.1.0-snapshot.61 → 0.1.0-snapshot.62

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bluecopa/harness",
-  "version": "0.1.0-snapshot.61",
+  "version": "0.1.0-snapshot.62",
   "description": "Provider-agnostic TypeScript agent framework",
   "license": "UNLICENSED",
   "scripts": {

package/src/agent/create-agent.ts

@@ -7,12 +7,14 @@ import type { HarnessTelemetry } from '../observability/otel';
 import { HookRunner } from '../hooks/hook-runner';
 import { PermissionManager } from '../permissions/permission-manager';
 import { VercelAgentLoop } from '../loop/vercel-agent-loop';
+export type { SystemPromptBlock, VercelAgentLoopConfig } from '../loop/vercel-agent-loop';
+export type { PrepareStepContext, PrepareStepResult } from './types';
 import { SkillManager } from '../skills/skill-manager';
 import { SkillRouter } from '../skills/skill-router';
 import type { SkillSummary } from '../skills/skill-types';
 import { SingleFlightStepExecutor } from './step-executor';
-import type { AgentAction, AgentLoop, AgentMessage, AgentRunResult, AgentStreamEvent, ToolCallAction, ToolBatchAction, ToolCallInfo, ToolResultInfo } from './types';
-export type { AgentAction, AgentLoop, AgentMessage, AgentRunResult, AgentStreamEvent, ToolCallAction, ToolBatchAction, ToolCallInfo, ToolResultInfo };
+import type { AgentAction, AgentLoop, AgentMessage, AgentRunResult, AgentStreamEvent, StepUsage, ToolCallAction, ToolBatchAction, ToolCallInfo, ToolResultInfo } from './types';
+export type { AgentAction, AgentLoop, AgentMessage, AgentRunResult, AgentStreamEvent, StepUsage, ToolCallAction, ToolBatchAction, ToolCallInfo, ToolResultInfo };
 export { HookRunner } from '../hooks/hook-runner';
 export { PermissionManager } from '../permissions/permission-manager';
 export type { PermissionMode, PermissionResolver, PermissionRequest } from '../permissions/permission-manager';
@@ -220,9 +222,21 @@ function toStreamResult(r: ToolResult): { success: boolean; output: string; erro
   return base;
 }
 
-/** Format a display-friendly content string for tool results (used in content field). */
+/** Build the text the LLM sees for a tool result.
+ *  Success: prefer modelOutput (compact) over raw output.
+ *  Failure: prefer modelOutput (structured fix guidance) → error → output → generic fallback.
+ *  This ensures custom tools can feed actionable error feedback to the model via modelOutput
+ *  so the agent can self-correct instead of stopping with "unknown failure". */
+function resultTextForLLM(result: ToolResult): string {
+  if (result.success) return result.modelOutput ?? result.output;
+  return result.modelOutput ?? result.error ?? result.output ?? 'unknown failure';
+}
+
+/** Format content string for LLM context. Uses modelOutput (compact summary) when available. */
 function formatToolResultContent(call: ToolCallAction, result: ToolResult): string {
-  const content = result.success ? result.output : `ERROR: ${result.error ?? 'unknown failure'}`;
+  const content = result.success
+    ? resultTextForLLM(result)
+    : `ERROR: ${resultTextForLLM(result)}`;
   switch (call.name) {
     case 'Write':
       return `Write(${call.args.path}): ${result.success ? 'ok' : content}`;
@@ -517,6 +531,11 @@ export function createAgent(runtime: AgentRuntime) {
       ? { nextAction: runtime.nextAction }
       : new VercelAgentLoop());
 
+  /** Read lastUsage from the loop if it's a VercelAgentLoop. */
+  function getLoopUsage(): StepUsage | undefined {
+    return loop instanceof VercelAgentLoop ? loop.lastUsage : undefined;
+  }
+
   async function resolveSkillContext(prompt: string): Promise<string> {
     if (!skillManager || !skillIndexPath) return '';
 
@@ -609,7 +628,7 @@ export function createAgent(runtime: AgentRuntime) {
                 if (!r.success) {
                   recordAgentError(runtime.telemetry);
                 }
-                const resultText = r.success ? r.output : `ERROR: ${r.error ?? 'unknown failure'}`;
+                const resultText = r.success ? resultTextForLLM(r) : `ERROR: ${resultTextForLLM(r)}`;
                 messages.push({
                   role: 'tool',
                   content: formatToolResultContent(call, r),
@@ -684,7 +703,7 @@ export function createAgent(runtime: AgentRuntime) {
           if (!result.success) {
             recordAgentError(runtime.telemetry);
           }
-          const singleResultText = result.success ? result.output : `ERROR: ${result.error ?? 'unknown failure'}`;
+          const singleResultText = result.success ? resultTextForLLM(result) : `ERROR: ${resultTextForLLM(result)}`;
           messages.push({
             role: 'tool',
             content: formatToolResultContent(action, result),
@@ -746,7 +765,7 @@ export function createAgent(runtime: AgentRuntime) {
           // If no tools → final response
           if (pendingTools.length === 0) {
             messages.push({ role: 'assistant', content: finalText });
-            yield { type: 'step_end', step };
+            { const u = getLoopUsage(); yield u ? { type: 'step_end' as const, step, usage: u } : { type: 'step_end' as const, step }; }
             yield { type: 'done', output: finalText, steps: step };
             return;
           }
@@ -772,7 +791,7 @@ export function createAgent(runtime: AgentRuntime) {
             if (action.type === 'final') {
               yield { type: 'text_delta', text: action.content };
               messages.push({ role: 'assistant', content: action.content });
-              yield { type: 'step_end', step };
+              { const u = getLoopUsage(); yield u ? { type: 'step_end' as const, step, usage: u } : { type: 'step_end' as const, step }; }
               yield { type: 'done', output: action.content, steps: step };
               return;
             }
@@ -784,7 +803,7 @@ export function createAgent(runtime: AgentRuntime) {
                 try {
                   const r = await executeTool(runtime.toolProvider, call, runtime);
                   yield { type: 'tool_end', name: call.name, result: toStreamResult(r) };
-                  const rText = r.success ? r.output : `ERROR: ${r.error ?? 'unknown failure'}`;
+                  const rText = r.success ? resultTextForLLM(r) : `ERROR: ${resultTextForLLM(r)}`;
                   messages.push({
                     role: 'tool',
                     content: formatToolResultContent(call, r),
@@ -806,7 +825,7 @@ export function createAgent(runtime: AgentRuntime) {
               try {
                 const r = await executeTool(runtime.toolProvider, action, runtime);
                 yield { type: 'tool_end', name: action.name, result: toStreamResult(r) };
-                const rText = r.success ? r.output : `ERROR: ${r.error ?? 'unknown failure'}`;
+                const rText = r.success ? resultTextForLLM(r) : `ERROR: ${resultTextForLLM(r)}`;
                 messages.push({
                   role: 'tool',
                   content: formatToolResultContent(action, r),
@@ -822,7 +841,7 @@ export function createAgent(runtime: AgentRuntime) {
                 });
               }
             }
-            yield { type: 'step_end', step };
+            { const u = getLoopUsage(); yield u ? { type: 'step_end' as const, step, usage: u } : { type: 'step_end' as const, step }; }
             continue;
           }
 
@@ -832,7 +851,7 @@ export function createAgent(runtime: AgentRuntime) {
             const call = pendingTools[i]!;
             const r = results[i]!;
             yield { type: 'tool_end', name: call.name, result: toStreamResult(r) };
-            const rText = r.success ? r.output : `ERROR: ${r.error ?? 'unknown failure'}`;
+            const rText = r.success ? resultTextForLLM(r) : `ERROR: ${resultTextForLLM(r)}`;
             messages.push({
               role: 'tool',
               content: formatToolResultContent(call, r),
@@ -850,7 +869,7 @@ export function createAgent(runtime: AgentRuntime) {
 
           if (action.type === 'final') {
             messages.push({ role: 'assistant', content: action.content });
-            yield { type: 'step_end', step };
+            { const u = getLoopUsage(); yield u ? { type: 'step_end' as const, step, usage: u } : { type: 'step_end' as const, step }; }
             yield { type: 'done', output: action.content, steps: step };
             return;
           }
@@ -874,7 +893,7 @@ export function createAgent(runtime: AgentRuntime) {
             const call = calls[i]!;
             const r = results[i]!;
             yield { type: 'tool_end', name: call.name, result: toStreamResult(r) };
-            const rText = r.success ? r.output : `ERROR: ${r.error ?? 'unknown failure'}`;
+            const rText = r.success ? resultTextForLLM(r) : `ERROR: ${resultTextForLLM(r)}`;
             messages.push({
               role: 'tool',
               content: formatToolResultContent(call, r),
@@ -888,7 +907,7 @@ export function createAgent(runtime: AgentRuntime) {
           }
         }
 
-        yield { type: 'step_end', step };
+        { const u = getLoopUsage(); yield u ? { type: 'step_end' as const, step, usage: u } : { type: 'step_end' as const, step }; }
       }
 
       yield { type: 'done', output: 'ERROR: max steps exceeded', steps: maxSteps };

package/src/agent/types.ts

@@ -60,15 +60,36 @@ export interface AgentRunResult {
   steps: number;
 }
 
+/** Token usage breakdown for a single LLM step. */
+export interface StepUsage {
+  inputTokens?: number;
+  outputTokens?: number;
+  cacheReadTokens?: number;
+  cacheWriteTokens?: number;
+  reasoningTokens?: number;
+}
+
 export type AgentStreamEvent =
   | { type: 'text_delta'; text: string }
   | { type: 'tool_start'; name: string; args: Record<string, unknown>; toolCallId?: string }
   | { type: 'tool_end'; name: string; result: { success: boolean; output: string; error?: string; [key: string]: unknown } }
   | { type: 'step_start'; step: number }
-  | { type: 'step_end'; step: number }
+  | { type: 'step_end'; step: number; usage?: StepUsage }
   | { type: 'done'; output: string; steps: number };
 
 export interface AgentLoop {
   nextAction(messages: AgentMessage[]): Promise<AgentAction>;
   streamAction?(messages: AgentMessage[]): AsyncIterable<AgentStreamEvent>;
 }
+
+/** Context passed to `prepareStep` before each LLM call. */
+export interface PrepareStepContext {
+  stepNumber: number;
+  toolCallHistory: string[];
+}
+
+/** Overrides returned by `prepareStep`. All fields optional — omit to keep defaults. */
+export interface PrepareStepResult {
+  model?: string;
+  activeTools?: string[];
+}

package/src/interfaces/tool-provider.ts

@@ -1,6 +1,8 @@
 export interface ToolResult {
   success: boolean;
   output: string;
+  /** Compact summary for LLM context. When present, sent to the model instead of `output`. */
+  modelOutput?: string | undefined;
   error?: string | undefined;
   metadata?: Record<string, unknown> | undefined;
 }

package/src/loop/vercel-agent-loop.ts

@@ -4,8 +4,7 @@ import type { ModelFactory, ToolChoiceConfig } from '../arc/types';
 import { resolveToolChoice } from '../arc/types';
 import { z } from 'zod';
 
-import type { AgentAction, AgentMessage, AgentLoop, AgentStreamEvent, ToolCallAction, ToolBatchAction } from '../agent/types';
-import { getTextContent } from '../agent/types';
+import type { AgentAction, AgentMessage, AgentLoop, AgentStreamEvent, StepUsage, ToolCallAction, ToolBatchAction, PrepareStepContext, PrepareStepResult } from '../agent/types';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 type AnyTool = Tool<any, any>;
@@ -161,15 +160,25 @@ function toModelMessages(messages: AgentMessage[]): ModelMessage[] {
   return out;
 }
 
+/** A system prompt block with optional Anthropic cache control. */
+export interface SystemPromptBlock {
+  text: string;
+  cacheControl?: { type: 'ephemeral' };
+}
+
 export interface VercelAgentLoopConfig {
   model?: string;
+  /** System prompt — string or structured blocks with cache control markers. */
+  systemPrompt?: string | SystemPromptBlock[];
   createModel?: ModelFactory;
-  systemPrompt?: string;
-  apiKey?: string;
   /** Custom tool definitions. If provided, replaces built-in agentTools for LLM calls. */
   tools?: Record<string, AnyTool>;
   /** Tool choice for LLM calls. Supports per-turn callbacks. Default: 'auto'. */
   toolChoice?: ToolChoiceConfig;
+  /** Provider options passed to generateText/streamText (e.g. anthropic thinking config). */
+  providerOptions?: Record<string, unknown>;
+  /** Per-step callback to override model and active tools before each LLM call. */
+  prepareStep?: (context: PrepareStepContext) => PrepareStepResult | void;
 }
 
 export class VercelAgentLoop implements AgentLoop {
@@ -181,15 +190,25 @@ export class VercelAgentLoop implements AgentLoop {
   private readonly tools: Record<string, AnyTool>;
   private readonly validToolNames: Set<string>;
   private readonly toolChoiceConfig?: ToolChoiceConfig;
+  private readonly providerOptions: Record<string, unknown> | undefined;
+  private readonly prepareStep: VercelAgentLoopConfig['prepareStep'];
+  /** Track tool names called across steps for prepareStep context. */
+  private toolCallHistory: string[] = [];
   private step = 0;
 
+  /** Last step's token usage — read after nextAction/streamAction completes. */
+  public lastUsage: StepUsage | undefined;
+
   constructor(config: VercelAgentLoopConfig = {}) {
     this.toolChoiceConfig = config.toolChoice;
     this.model = config.model ?? process.env.HARNESS_MODEL ?? 'claude-sonnet-4-5';
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
     this.createModel = config.createModel ?? defaultAnthropicProvider;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     this.tools = config.tools ?? builtinTools as any;
     this.validToolNames = new Set(Object.keys(this.tools));
+    this.providerOptions = config.providerOptions;
+    this.prepareStep = config.prepareStep;
     this.systemPrompt =
       config.systemPrompt ??
       [
@@ -211,9 +230,63 @@ export class VercelAgentLoop implements AgentLoop {
     }
   }
 
+  /** Build the `system` parameter for generateText/streamText. */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private buildSystemParam(): any {
+    if (typeof this.systemPrompt === 'string') return this.systemPrompt;
+    // Structured blocks → AI SDK v6 SystemModelMessage format
+    return this.systemPrompt.map(block => ({
+      role: 'system' as const,
+      content: block.text,
+      ...(block.cacheControl
+        ? { providerOptions: { anthropic: { cacheControl: block.cacheControl } } }
+        : {}),
+    }));
+  }
+
+  /** Resolve model + tools for this step via prepareStep callback. */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private resolveStep(): { model: string; tools: Record<string, any>; validNames: Set<string> } {
+    this.stepNumber++;
+    if (!this.prepareStep) {
+      return { model: this.model, tools: this.tools, validNames: this.validToolNames };
+    }
+    const overrides = this.prepareStep({ stepNumber: this.stepNumber, toolCallHistory: this.toolCallHistory });
+    if (!overrides) {
+      return { model: this.model, tools: this.tools, validNames: this.validToolNames };
+    }
+    const model = overrides.model ?? this.model;
+    let tools: Record<string, AnyTool> = this.tools;
+    let validNames = this.validToolNames;
+    if (overrides.activeTools) {
+      const allowed = new Set(overrides.activeTools);
+      tools = Object.fromEntries(Object.entries(this.tools).filter(([k]) => allowed.has(k)));
+      validNames = new Set(Object.keys(tools));
+    }
+    return { model, tools, validNames };
+  }
+
+  /** Extract StepUsage from AI SDK usage object. */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private static extractUsage(usage: any): StepUsage | undefined {
+    if (!usage) return undefined;
+    const u: StepUsage = {};
+    if (usage.inputTokens != null) u.inputTokens = usage.inputTokens;
+    if (usage.outputTokens != null) u.outputTokens = usage.outputTokens;
+    // AI SDK v6 nests cache/reasoning under inputTokenDetails/outputTokenDetails
+    const inputDetails = usage.inputTokenDetails ?? usage;
+    const outputDetails = usage.outputTokenDetails ?? usage;
+    if (inputDetails.cacheReadTokens != null) u.cacheReadTokens = inputDetails.cacheReadTokens;
+    if (inputDetails.cacheWriteTokens != null) u.cacheWriteTokens = inputDetails.cacheWriteTokens;
+    if (outputDetails.reasoningTokens != null) u.reasoningTokens = outputDetails.reasoningTokens;
+    return Object.keys(u).length > 0 ? u : undefined;
+  }
+
   async nextAction(messages: AgentMessage[]): Promise<AgentAction> {
     const currentStep = this.step++;
 
+    const { model, tools, validNames } = this.resolveStep();
+
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const result = await (generateText as any)({
       model: this.createModel(this.model),
@@ -222,14 +295,18 @@ export class VercelAgentLoop implements AgentLoop {
       system: this.cachedSystem,
       messages: toModelMessages(messages),
       stopWhen: stepCountIs(1),
+      ...(this.providerOptions ? { providerOptions: this.providerOptions } : {}),
     });
 
+    // Capture usage
+    this.lastUsage = VercelAgentLoop.extractUsage(result.usage);
+
     // If the model made tool calls, extract them
     if (result.toolCalls && result.toolCalls.length > 0) {
       const validCalls: ToolCallAction[] = [];
       for (const call of result.toolCalls) {
         const name = call.toolName;
-        if (this.validToolNames.has(name)) {
+        if (validNames.has(name)) {
           const toolCallId = (call as { toolCallId?: string }).toolCallId;
           validCalls.push({
             type: 'tool',
@@ -237,6 +314,7 @@ export class VercelAgentLoop implements AgentLoop {
             args: (call as { input: Record<string, unknown> }).input,
             ...(toolCallId != null ? { toolCallId } : {}),
           });
+          this.toolCallHistory.push(name);
         }
       }
 
@@ -261,6 +339,8 @@ export class VercelAgentLoop implements AgentLoop {
   async *streamAction(messages: AgentMessage[]): AsyncGenerator<AgentStreamEvent> {
     const currentStep = this.step++;
 
+    const { model, tools, validNames } = this.resolveStep();
+
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const result = (streamText as any)({
       model: this.createModel(this.model),
@@ -269,6 +349,7 @@ export class VercelAgentLoop implements AgentLoop {
       system: this.cachedSystem,
       messages: toModelMessages(messages),
       stopWhen: stepCountIs(1),
+      ...(this.providerOptions ? { providerOptions: this.providerOptions } : {}),
     });
 
     const toolArgs = new Map<string, string>();
@@ -285,14 +366,23 @@ export class VercelAgentLoop implements AgentLoop {
       }
       if (part.type === 'tool-call') {
         const name = part.toolName;
-        if (this.validToolNames.has(name)) {
+        if (validNames.has(name)) {
           // eslint-disable-next-line @typescript-eslint/no-explicit-any
           const p = part as any;
           const args: Record<string, unknown> = p.args ?? p.input ?? {};
           const toolCallId: string | undefined = p.toolCallId;
           yield { type: 'tool_start', name, args, ...(toolCallId != null ? { toolCallId } : {}) };
+          this.toolCallHistory.push(name);
         }
       }
     }
+
+    // Capture usage after stream completes
+    try {
+      const usage = await result.usage;
+      this.lastUsage = VercelAgentLoop.extractUsage(usage);
+    } catch {
+      this.lastUsage = undefined;
+    }
   }
 }