@inflexa-ai/harness 0.39.2 → 0.41.0

package/dist/loop/metrics.js

@@ -1,25 +1,29 @@
 /**
  * Agent-loop OTel metrics, exported through the global MeterProvider.
  *
- * Every `runAgent` completion records, keyed by `agent_id`:
+ * Each LLM call records its token counters when it completes. A durable step
+ * that a recovery replays returns its stored reply and skips the body, so a
+ * replay never double-counts a call.
+ *
+ * Cache-write tokens are billed at a premium and only pay off once something
+ * reads them. Each counter carries `agent_id`, plus `model` and `provider`
+ * when the response reports them. Session provenance is not a label: agents
+ * that share one root provenance also share one series. Providers do not
+ * agree on whether `outputTokens` includes the reasoning tokens, so the
+ * reasoning counter keeps a `provider` label.
+ *
+ * Each `runAgent` completion also records, with the `agent_id` label only:
  *   - cortex.harness.agent.iterations — histogram of LLM iterations per run.
  *     A creeping distribution is a leading indicator of prompt drift.
- *   - cortex.harness.agent.cap_hits   — counter, incremented once per run
- *     that exhausted `maxIterations` and took the forced wrap-up path.
- *   - cortex.harness.agent.input_tokens / .output_tokens — counters, summed
- *     over every LLM call the run made (the wrap-up included).
- *   - cortex.harness.agent.cache_read_tokens  — counter, prefix tokens served
- *     from the prompt cache.
- *   - cortex.harness.agent.cache_write_tokens — counter, prefix tokens written
- *     into it (billed at a premium; only pays off once something reads it).
+ *   - cortex.harness.agent.cap_hits — counter, incremented once per run that
+ *     exhausted `maxIterations` and took the forced wrap-up path.
  *
  * The two cache counters are what make prompt caching *observable*: the hit
  * rate for an agent type is `cache_read_tokens / input_tokens` (the harness's
  * `inputTokens` is the total billed prefix, cache reads included). A flat-zero
  * `cache_read_tokens` against a non-zero `cache_write_tokens` means every write
  * is being thrown away — either a cache defeater is shifting the prefix (see
- * `providers/prompt-cache.ts`) or the endpoint ignores cache directives
- * outright, as the Claude Max OAuth path does.
+ * `providers/prompt-cache.ts`) or the endpoint ignores cache directives.
  *
  * Token counters are recorded only from what a provider actually reports; a
  * provider that reports no usage contributes nothing rather than zero.
@@ -43,11 +47,11 @@ function getInstruments() {
                 description: "runAgent completions that exhausted maxIterations and took the " + "forced wrap-up path",
             }),
             inputTokens: meter.createCounter("cortex.harness.agent.input_tokens", {
-                description: "Input tokens billed across every LLM call in a runAgent completion, cache reads included",
+                description: "Input tokens billed for each completed LLM call, cache reads included",
                 unit: "{token}",
             }),
             outputTokens: meter.createCounter("cortex.harness.agent.output_tokens", {
-                description: "Output tokens generated across every LLM call in a runAgent completion",
+                description: "Output tokens generated by each completed LLM call",
                 unit: "{token}",
             }),
             cacheReadTokens: meter.createCounter("cortex.harness.agent.cache_read_tokens", {
@@ -58,6 +62,10 @@ function getInstruments() {
                 description: "Prompt-prefix tokens written into the provider's prompt cache",
                 unit: "{token}",
             }),
+            reasoningTokens: meter.createCounter("cortex.harness.agent.reasoning_tokens", {
+                description: "Reasoning tokens of each completed LLM call, as the provider reports them",
+                unit: "{token}",
+            }),
         };
     }
     return instruments;
@@ -88,16 +96,27 @@ export function hasReportedUsage(usage) {
         return false;
     return Object.values(usage).some((value) => value !== undefined);
 }
-/** Record one completed `runAgent` invocation. */
-export function recordAgentRun(run) {
-    const { iterations, capHits, inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens } = getInstruments();
-    const attributes = { agent_id: run.agentId };
-    iterations.record(run.iterations, attributes);
-    if (run.cappedOut)
-        capHits.add(1, attributes);
-    const usage = run.usage;
+/**
+ * Maps a step's response to grow its token counters. Use inside the step
+ * body, `provider.chat(...).map(countChatTokens(agentId))`. A call that gives
+ * `err` counts nothing.
+ */
+export function countChatTokens(agentId) {
+    return (response) => {
+        recordChatCall({ agentId, response });
+        return response;
+    };
+}
+export function recordChatCall(call) {
+    const { usage, servedModelId, provider } = call.response;
     if (usage === undefined)
         return;
+    const { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens } = getInstruments();
+    const attributes = {
+        agent_id: call.agentId,
+        ...(servedModelId === undefined ? {} : { model: servedModelId }),
+        ...(provider === undefined ? {} : { provider }),
+    };
     if (usage.inputTokens !== undefined)
         inputTokens.add(usage.inputTokens, attributes);
     if (usage.outputTokens !== undefined)
@@ -106,9 +125,19 @@ export function recordAgentRun(run) {
         cacheReadTokens.add(usage.cacheReadInputTokens, attributes);
     if (usage.cacheCreationInputTokens !== undefined)
         cacheWriteTokens.add(usage.cacheCreationInputTokens, attributes);
+    if (usage.reasoningTokens !== undefined)
+        reasoningTokens.add(usage.reasoningTokens, attributes);
+}
+/** Record one completed `runAgent` invocation. */
+export function recordAgentRun(run) {
+    const { iterations, capHits } = getInstruments();
+    const attributes = { agent_id: run.agentId };
+    iterations.record(run.iterations, attributes);
+    if (run.cappedOut)
+        capHits.add(1, attributes);
 }
 /**
- * Drop the memoized instruments so the next `recordAgentRun` rebinds to a
+ * Drop the memoized instruments so the next record rebinds to a
  * freshly-registered `MeterProvider`. Test-only — production registers its
  * provider once at startup.
  */

package/dist/loop/run-agent.d.ts

@@ -2,9 +2,12 @@ import { type FinishReason } from "ai";
 import type { AgentSession } from "../auth/types.js";
 import type { UsageRecorder } from "../billing/usage-recorder.js";
 import type { Logger } from "../lib/logger.js";
-import type { AgentChat, ChatRequest, PromptCachePolicy, ReasoningPolicy } from "../providers/types.js";
+import { type CompactionPolicy } from "./compaction.js";
+import type { AgentChat, ChatRequest, ChatResponse, PromptCachePolicy, ReasoningPolicy } from "../providers/types.js";
 import { type AskApproval, type AskRequest } from "../tools/approval/contract.js";
 import { type AgentRunUsage } from "./metrics.js";
+import { type ToolBudget, type ToolMask } from "./tool-mask.js";
+import { type ToolOutputStore } from "./tool-output.js";
 import type { AgentDefinition, EmitFn, LoopMessage, RunStep } from "./types.js";
 export interface AgentFinish {
     readonly reason: FinishReason | "aborted" | "max_iterations" | "denied";
@@ -31,6 +34,19 @@ export interface RunAgentResult {
     readonly messages: LoopMessage[];
     readonly finish: AgentFinish;
 }
+/** A request that the provider refused for its content: the HTTP status and the text of the provider error. */
+export interface RequestRefusal {
+    readonly status: number;
+    readonly message: string;
+}
+/** The result of a segment. A segment that ends on a refusal of its request carries the refusal. */
+export interface SegmentResult extends RunAgentResult {
+    readonly refusal?: RequestRefusal;
+}
+/** The messages that the loop appended after the last call of the round sink. */
+export interface AgentRound {
+    readonly messages: readonly LoopMessage[];
+}
 export interface StepNameFormatter {
     llm(iteration: number): string;
     tool(toolName: string, toolUseId: string): string;
@@ -40,6 +56,17 @@ export interface RunAgentOptions {
     readonly provider: AgentChat;
     readonly signal: AbortSignal;
     readonly emit: EmitFn;
+    /**
+     * The round sink. The loop awaits it before each model request, at each exit, and before a throw.
+     * A durable loop passes no sink, because DBOS replays the loop body.
+     */
+    readonly onRound?: (round: AgentRound) => Promise<void>;
+    /**
+     * The compaction policy. Each request then sends the view of the transcript, and the loop compacts
+     * before a task request whose view passes the budget. A durable loop passes no policy, because
+     * `recordsAfter` reads the database outside a step.
+     */
+    readonly compaction?: CompactionPolicy;
     /**
      * Per-turn user-approval seam threaded into every tool's `ToolContext` as
      * `ctx.ask`. A conversation tool calls it to pause for an explicit user
@@ -53,8 +80,10 @@ export interface RunAgentOptions {
     readonly runStep: RunStep;
     readonly formatStepName?: StepNameFormatter;
     readonly isFatalLoopError?: (err: unknown) => boolean;
-    /** Tool-selection policy for in-loop model calls. The cap wrap-up forbids a
-     * tool call regardless. */
+    /**
+     * Tool choice for every request of this run: part of the prefix that the
+     * prompt cache and a signed thinking block bind to.
+     */
     readonly toolChoice?: ChatRequest["toolChoice"];
     /**
      * Optional outcome predicate for loop-driving agents whose result is
@@ -62,6 +91,13 @@ export interface RunAgentOptions {
      * dispatch round, once every sibling tool result has been appended.
      */
     readonly resolved?: () => boolean;
+    /**
+     * Tools allowed to run per request. The full tool set stays declared either
+     * way, since it is part of the cache/thinking-block prefix.
+     */
+    readonly toolMask?: ToolMask;
+    /** Max calls per tool for this run; a call in the same round counts against the budget too. */
+    readonly toolBudget?: ToolBudget;
     /**
      * Prompt-cache policy for every LLM call this run makes. Defaults to
      * `DEFAULT_PROMPT_CACHE` (5m) — an agent loop always re-sends its prefix, so
@@ -75,16 +111,9 @@ export interface RunAgentOptions {
      */
     readonly promptCache?: PromptCachePolicy;
     /**
-     * Reasoning policy for every LLM call this run makes. Defaults to
-     * `DEFAULT_REASONING` (`xhigh`) — an agent loop drives tools over many
-     * iterations, and a shallow turn there costs more in wasted calls than the
-     * deeper turn costs in tokens. The provider package resolves the name for
-     * the model that it is bound to.
-     *
-     * The policy lives here, on the run, rather than on the provider, for the
-     * same reason that the cache policy does: a one-shot LLM call elsewhere has
-     * its own depth needs and must not inherit the depth of a loop. A host on a
-     * model with no reasoning support passes `"provider-default"`.
+     * The reasoning effort for each LLM call of this run, the forced wrap-up
+     * included. Anthropic discards its message cache when the top-level
+     * effort changes, so the run must send the same value, or none, throughout.
      */
     readonly reasoning?: ReasoningPolicy;
     /**
@@ -104,6 +133,8 @@ export interface RunAgentOptions {
      * and it logs the reason of an `err` when the result arrives.
      */
     readonly usageRecorder?: UsageRecorder;
+    /** Keeps the text of each cut tool result. A run with no store cuts, and it keeps nothing. */
+    readonly toolOutputStore?: ToolOutputStore;
     /**
      * The turn's usage accumulator, supplied by whatever ran this loop as part
      * of a larger turn (a sub-agent-running tool passes `ctx.turnUsage`). A run
@@ -121,5 +152,72 @@ export interface RunAgentOptions {
      */
     readonly invocationId?: string;
 }
+/** A run that receives no turn accumulator is the turn's root. See `RunAgentOptions.turnUsage`. */
+export declare const isTurnRoot: (opts: RunAgentOptions) => boolean;
 export declare function runAgent(agent: AgentDefinition, initial: readonly LoopMessage[], session: AgentSession, opts: RunAgentOptions): Promise<RunAgentResult>;
+/** The harness request of the wrap-up at the iteration cap. */
+export declare const WRAP_UP_REQUEST = "You reached the limit of tool rounds for this run, thus no tool can run now. Answer in text with the results that you have.";
+/** The cap of wrap-up requests: one answer, and one more request after a refused call. */
+export declare const WRAP_UP_MAX_REQUESTS = 2;
+/** One stretch of model requests of a conversation, under one mask. */
+export interface LoopSegment {
+    /** The tools that can run for each request of the segment. Absent lets each declared tool run. */
+    readonly mask: ToolMask | undefined;
+    readonly maxRequests: number;
+    readonly stepNames: StepNameFormatter;
+    /** The `index` of the first request of the segment in its `iteration` events. */
+    readonly firstIndex: number;
+    /** A harness request that the segment appends, as a synthetic user message, before its first request. */
+    readonly requestText?: string;
+    /**
+     * The segment runs after the iteration cap: its requests do not count as
+     * iterations, and the last one always ends the run, whatever its reply.
+     */
+    readonly closesCappedRun?: boolean;
+    /** The loop compacts before a request of the segment whose view passes the budget of the policy. */
+    readonly checksBudget?: boolean;
+    /** A refusal of a request, a `provider` error with the status `400` or `413`, ends the segment with no new message. */
+    readonly endsOnRequestRefusal?: boolean;
+}
+/**
+ * One conversation, open for its segments. Each segment extends the same
+ * prefix (system prompt, tools, tool choice) as the one before it.
+ */
+export interface OpenLoop {
+    /** Run one segment: the result of the run when a reply ends it, or `"capped"` when the segment used its cap. */
+    runSegment(segment: LoopSegment): Promise<SegmentResult | "capped">;
+    /** End a run after its last segment used its cap. */
+    endCapped(): Promise<RunAgentResult>;
+}
+/**
+ * Open the loop over one conversation. `metricAgentId` is the id that usage
+ * and run metrics count under, which is not always `agent.id`.
+ */
+export declare function openLoop(agent: AgentDefinition, initial: readonly LoopMessage[], session: AgentSession, opts: RunAgentOptions, metricAgentId: string): OpenLoop;
+/** What one completed LLM call is accounted under. */
+export interface ChatCallAccounting {
+    readonly session: AgentSession;
+    readonly agentId: string;
+    readonly callPath: readonly string[];
+    /**
+     * The deterministic name inside the record key, for example `llm-{n}` for
+     * a loop call or `adhoc-route` for a direct call.
+     */
+    readonly stepName: string;
+    /** The id of the tool call that makes the call, when the call runs inside a tool dispatch. */
+    readonly invocationId?: string;
+    readonly usageRecorder: UsageRecorder;
+    /** The sink that the notice helper logs the reason of a recorder `err` to. */
+    readonly logger: Logger;
+    /** The accumulators that the usage of the call folds into, for example the run total and the turn total. */
+    readonly rollups: readonly AgentRunUsage[];
+}
+/**
+ * Called at the fold point, before any run-ending branch, so an aborted or
+ * dead run is still accounted. Token counters grow inside the call's step
+ * body, not here, because a replay re-delivers the stored reply to this
+ * function too; the record re-sends under the same idempotency key, which an
+ * upserting sink counts once.
+ */
+export declare function accountForChatCall(reply: ChatResponse, call: ChatCallAccounting): void;
 export declare function finalText(messages: readonly LoopMessage[]): string;