ai-sdk-harness-sessions 0.2.1 → 0.3.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # ai-sdk-harness-sessions
 
+## 0.3.0
+
+### Minor Changes
+
+- 02b3ad7: `onTurn`, a new option of `createSessionManager()`, is called once each turn is over and the session written, with a `TurnReport`: the turn's tokens, cache reads and writes included, the model, tokens and tool calls of each step, and the raw events of its harness — Claude Code's `rate_limit_event`, for instance. An application can account for its sessions, and read the rate limits of a subscription. `SessionStreamResult` may give `steps` and `fullStream`, as a `StreamTextResult` does; an agent whose results give neither reports its tokens only.
+
 ## 0.2.1
 
 ### Patch Changes

package/README.md

@@ -371,6 +371,33 @@ A resume state holds secrets: the token of the harness bridge, the placeholders
 
 Keep one manager per store, unless the store gives leases (below): at its start, a manager takes the sessions the store holds as live for those of a process that crashed, and marks them `interrupted`.
 
+### What a turn cost
+
+`onTurn` is called once each turn is over and the session written, with a `TurnReport`: the turn's tokens, cache reads and writes included, each of its steps — the model that answered, its tokens and the tools it called, with their input — and the raw events of its harness, in order. Keep them to account for your sessions, or to read the rate limits of a subscription, which Claude Code reports as `rate_limit_event`:
+
+```ts
+const sessions = createSessionManager<Metadata>({
+  // …
+  onTurn: async ({ sessionId, usage, steps, rawEvents }) => {
+    await db.turns.insert({
+      sessionId,
+      inputTokens: usage.inputTokens,
+      cacheReadTokens: usage.inputTokenDetails.cacheReadTokens,
+      outputTokens: usage.outputTokens,
+      models: [...new Set(steps.map(({ modelId }) => modelId))],
+    });
+    const limits = rawEvents.filter(
+      (event) => (event as { type?: string }).type === 'rate_limit_event',
+    );
+    if (limits.length > 0) await db.rateLimits.insert(sessionId, limits.at(-1));
+  },
+});
+```
+
+A `rate_limit_event` of Claude Code (signed in with a subscription) gives every window in `rate_limit_info.unifiedWindows`, such as `five_hour` and `seven_day`, each with its `utilization` (a fraction) and its `resetsAt` (in seconds). The harness names its tools in lower case: `bash`, not `Bash`.
+
+The steps and the raw events come from the agent's result, `steps` and `fullStream`: a `HarnessAgent` gives both; an agent of your own that gives neither reports its tokens only. What `onTurn` throws goes to `onError`; the turn is settled either way.
+
 ### Several instances
 
 An application that runs on several instances — behind a load balancer, scaled out, or a new version deployed beside the old one — shares one store between their managers. The store tells them apart with leases: three more methods, on the same database, and the managers share it.
@@ -433,6 +460,7 @@ The lease's expiry is read on the database's clock, the one clock all instances
 | `approve`               | none                     | Answers an approval before anyone sees it: `{ approved, reason }`, or `undefined` to ask a person   |
 | `generateId`            | `crypto.randomUUID`      | The ids of new sessions and of the messages of their turns                                          |
 | `errorMessage`          | `getHarnessErrorMessage` | What a client is told of an error in a turn; the session's `error` keeps the full message           |
+| `onTurn`                | none                     | Called once each turn is over, with its `TurnReport`: tokens in detail, steps, raw harness events   |
 | `onError`               | `console.error`          | Called with what fails in the background: a start, a suspension, a write to the store               |
 
 | Method                     | What it does                                                                                               |

package/dist/definitions/agent.d.ts

@@ -3,8 +3,12 @@ import type { HarnessAgentResumeSessionState, HarnessAgentSession } from '@ai-sd
 import type { StreamTextResult, ToolApprovalResponse, ToolResultPart, ToolSet } from 'ai';
 /** A turn's result, whatever the agent's runtime context and output. */
 type AnyStreamTextResult = StreamTextResult<ToolSet, any, any>;
-/** What the manager reads of a turn: a `StreamTextResult` is one. */
-export type SessionStreamResult = Pick<AnyStreamTextResult, 'totalUsage' | 'toUIMessageStream'>;
+/**
+ * What the manager reads of a turn: a `StreamTextResult` is one. Its `steps` and `fullStream`, when
+ * it has them, give `onTurn` the model, the tokens and the tools of each step, and the raw events of
+ * the harness.
+ */
+export type SessionStreamResult = Partial<Pick<AnyStreamTextResult, 'fullStream' | 'steps'>> & Pick<AnyStreamTextResult, 'totalUsage' | 'toUIMessageStream'>;
 /**
  * What the manager needs of an agent: a `HarnessAgent` is one, whatever its harness, tools and
  * settings.

package/dist/definitions/manager.d.ts

@@ -1,7 +1,7 @@
 import type { ToolApprovalResponse, ToolResultPart, UIMessage, UIMessageChunk } from 'ai';
 import type { SessionAgent } from './agent.js';
 import type { SessionSandboxes } from './sandboxes.js';
-import type { ApprovalVerdict, PendingApproval, SessionRecord, SessionSummary } from './session.js';
+import type { ApprovalVerdict, PendingApproval, SessionRecord, SessionSummary, TurnReport } from './session.js';
 import type { SessionStore } from './store.js';
 export interface SessionManagerOptions<METADATA> {
     /**
@@ -68,6 +68,15 @@ export interface SessionManagerOptions<METADATA> {
      * `@ai-sdk/harness`. The session's `error` keeps the full message.
      */
     errorMessage?: (error: unknown) => string;
+    /**
+     * Called once each turn is over and the session written, with what the turn cost: its tokens,
+     * cache reads and writes included, the model, tokens and tool calls of each step, and the raw
+     * events of its harness — what an application needs to account for its sessions, or to read the
+     * rate limits of a subscription. What it throws goes to `onError`; the turn is settled either way.
+     */
+    onTurn?: (report: TurnReport, context: {
+        session: SessionSummary<METADATA>;
+    }) => PromiseLike<void> | void;
     /**
      * Called with what fails in the background: a start, a suspension, a write to the store. Default:
      * `console.error`.

package/dist/definitions/session.d.ts

@@ -1,4 +1,4 @@
-import type { UIMessage } from 'ai';
+import type { LanguageModelUsage, UIMessage } from 'ai';
 /**
  * Where a session stands in its lifecycle.
  *
@@ -29,6 +29,36 @@ export interface SessionUsage {
     outputTokens: number;
     totalTokens: number;
 }
+/** A step of a turn: the model that answered, the tokens it used, and the tools it called. */
+export interface TurnStepReport {
+    /** The model, as the harness names it: `claude-sonnet-4-5`, for instance. */
+    modelId: string | undefined;
+    usage: LanguageModelUsage;
+    /** The tools it called, with their input: the skill it loaded, the subagent it started… */
+    toolCalls: {
+        toolCallId: string;
+        toolName: string;
+        input: unknown;
+    }[];
+}
+/**
+ * A turn once it is over, for accounting: its tokens, cache reads and writes included, each of its
+ * steps, and the raw events its harness sent — rate limits, for instance, which Claude Code reports
+ * as `rate_limit_event`.
+ */
+export interface TurnReport {
+    sessionId: string;
+    startedAt: Date;
+    endedAt: Date;
+    /** What went wrong, when the turn failed or was cut short. */
+    failure: string | undefined;
+    /** The tokens of the whole turn, over all its streams. */
+    usage: LanguageModelUsage;
+    /** Its steps, in order; empty when the agent's results do not give them. */
+    steps: TurnStepReport[];
+    /** The raw events of the harness, in order; empty when the agent's results do not stream them. */
+    rawEvents: unknown[];
+}
 /** An answer to an approval request, as `ToolApprovalResponse` carries it. */
 export interface ApprovalVerdict {
     approved: boolean;

package/dist/index.d.ts

@@ -2,7 +2,7 @@ export type { SessionAgent, SessionStreamResult } from './definitions/agent.js';
 export type { ContinueOptions, SendOptions, SessionEvent, SessionManager, SessionManagerOptions, SessionTurn, } from './definitions/manager.js';
 export type { SessionSandboxes, SessionSandboxLease } from './definitions/sandboxes.js';
 export { LIVE_STATUSES, SESSION_STATUSES } from './definitions/session.js';
-export type { PendingApproval, PendingInput, PendingToolCall, SessionRecord, SessionStatus, SessionSummary, SessionUsage, } from './definitions/session.js';
+export type { PendingApproval, PendingInput, PendingToolCall, SessionRecord, SessionStatus, SessionSummary, SessionUsage, TurnReport, TurnStepReport, } from './definitions/session.js';
 export type { SessionClaim, SessionStore } from './definitions/store.js';
 export { SessionCapacityError } from './errors/session-capacity-error.js';
 export { SessionConflictError } from './errors/session-conflict-error.js';

package/dist/manager/session-manager.js

@@ -294,6 +294,18 @@ class HarnessSessionManager {
         await this.options.sandboxes.shutdown?.().catch(this.report('shutdown'));
         await this.journal.flush();
     };
+    /** Hands a turn that is over to `onTurn`: what it throws goes to `onError`, the turn stays settled. */
+    async reportTurn(session, turn) {
+        if (this.options.onTurn === undefined)
+            return;
+        const report = { sessionId: session.record.id, ...turn, endedAt: new Date() };
+        try {
+            await this.options.onTurn(report, { session: summaryOf(session.record) });
+        }
+        catch (error) {
+            this.report('report turn', session.record.id)(error);
+        }
+    }
     report = (action, sessionId) => (error) => this.settings.onError(error, { action, ...(sessionId !== undefined && { sessionId }) });
     async ready() {
         if (this.stopped)
@@ -579,6 +591,7 @@ class HarnessSessionManager {
             : undefined;
         let finish;
         session.turn = { abort, done: new Promise((resolve) => (finish = resolve)) };
+        const startedAt = new Date();
         await this.transition(session, 'busy');
         const settle = async (failure) => {
             clearTimeout(timer);
@@ -604,7 +617,7 @@ class HarnessSessionManager {
             generateMessageId: generateId,
             errorMessage,
             next: (messages) => this.answerPause(session, messages, answered),
-            onSettled: async ({ messages, failure, usage }) => {
+            onSettled: async ({ messages, failure, usage, report }) => {
                 if (messages !== undefined)
                     record.messages = messages;
                 record.turns += 1;
@@ -612,6 +625,7 @@ class HarnessSessionManager {
                 record.usage.outputTokens += usage.outputTokens ?? 0;
                 record.usage.totalTokens += usage.totalTokens ?? 0;
                 await settle(failure);
+                await this.reportTurn(session, { startedAt, failure, ...report });
             },
         });
         // A turn cut short by a close or a shutdown is over once they wrote the session.

package/dist/turns/turn-stream.d.ts

@@ -1,5 +1,6 @@
 import type { LanguageModelUsage, UIMessage, UIMessageChunk } from 'ai';
 import type { SessionStreamResult } from '../definitions/agent.js';
+import type { TurnStepReport } from '../definitions/session.js';
 /** The tokens a turn used, added up over its streams. */
 export type TurnUsage = Pick<LanguageModelUsage, 'inputTokens' | 'outputTokens' | 'totalTokens'>;
 /** How a turn ended. */
@@ -9,6 +10,12 @@ export interface TurnOutcome {
     /** What went wrong, when the turn failed or was cut short. */
     failure: string | undefined;
     usage: TurnUsage;
+    /** The turn's tokens in detail, its steps and the raw events of its harness, for `onTurn`. */
+    report: {
+        usage: LanguageModelUsage;
+        steps: TurnStepReport[];
+        rawEvents: unknown[];
+    };
 }
 /**
  * What a turn does once one of its streams ends: answers given in between, sent to the client and

package/dist/turns/turn-stream.js

@@ -40,6 +40,68 @@ async function readSegment(result, originalMessages, { first, send, options, })
     }
     return outcome;
 }
+/** Tokens of nothing yet: what the turn's tokens are added to. */
+function emptyUsage() {
+    return {
+        inputTokens: 0,
+        inputTokenDetails: { noCacheTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 },
+        outputTokens: 0,
+        outputTokenDetails: { textTokens: 0, reasoningTokens: 0 },
+        totalTokens: 0,
+    };
+}
+const add = (a, b) => a === undefined && b === undefined ? undefined : (a ?? 0) + (b ?? 0);
+/** `total` plus `spent`, detail by detail: a count neither of them knows stays unknown. */
+function addDetails(total, spent) {
+    if (spent === undefined)
+        return total;
+    return {
+        inputTokens: add(total.inputTokens, spent.inputTokens),
+        inputTokenDetails: {
+            noCacheTokens: add(total.inputTokenDetails.noCacheTokens, spent.inputTokenDetails?.noCacheTokens),
+            cacheReadTokens: add(total.inputTokenDetails.cacheReadTokens, spent.inputTokenDetails?.cacheReadTokens),
+            cacheWriteTokens: add(total.inputTokenDetails.cacheWriteTokens, spent.inputTokenDetails?.cacheWriteTokens),
+        },
+        outputTokens: add(total.outputTokens, spent.outputTokens),
+        outputTokenDetails: {
+            textTokens: add(total.outputTokenDetails.textTokens, spent.outputTokenDetails?.textTokens),
+            reasoningTokens: add(total.outputTokenDetails.reasoningTokens, spent.outputTokenDetails?.reasoningTokens),
+        },
+        totalTokens: add(total.totalTokens, spent.totalTokens),
+    };
+}
+/** The steps of a stream of the turn: their model, tokens and tool calls. */
+async function stepsOf(result) {
+    const steps = await Promise.resolve(result.steps).catch(() => undefined);
+    return (steps ?? []).map((step) => ({
+        modelId: step.response?.modelId,
+        usage: step.usage,
+        toolCalls: (step.toolCalls ?? []).map(({ toolCallId, toolName, input }) => ({
+            toolCallId,
+            toolName,
+            input: input,
+        })),
+    }));
+}
+/**
+ * The raw events a stream of the turn carries, read from its own copy of the stream, beside the
+ * client's: what the harness sends that no other part says.
+ */
+async function rawEventsOf(result) {
+    const events = [];
+    if (result.fullStream === undefined)
+        return events;
+    try {
+        for await (const part of result.fullStream) {
+            if (part.type === 'raw')
+                events.push(part.rawValue);
+        }
+    }
+    catch {
+        // The turn's failure is told by the client's stream: the events read so far are kept.
+    }
+    return events;
+}
 /** Adds the tokens a stream of the turn used, once it is over. */
 async function addUsage(usage, result) {
     const spent = await Promise.resolve(result.totalUsage).catch(() => undefined);
@@ -52,6 +114,9 @@ class TurnRun {
     options;
     send;
     usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
+    details = emptyUsage();
+    steps = [];
+    rawEvents = [];
     messages;
     failure;
     finish;
@@ -63,12 +128,19 @@ class TurnRun {
         let result = first;
         let originalMessages = this.options.originalMessages;
         for (let index = 0; result !== undefined; index += 1) {
+            const raw = rawEventsOf(result);
             const outcome = await readSegment(result, originalMessages, {
                 first: index === 0,
                 send: this.send,
                 options: this.options,
             });
             await addUsage(this.usage, result);
+            const spent = await Promise.resolve(result.totalUsage).catch(() => undefined);
+            this.details = addDetails(this.details, spent);
+            const steps = await stepsOf(result);
+            const events = await raw;
+            this.steps.push(...steps);
+            this.rawEvents.push(...events);
             this.failure ??= outcome.failure;
             this.finish = outcome.finish;
             this.messages = outcome.messages ?? this.messages;
@@ -79,7 +151,12 @@ class TurnRun {
         }
         if (this.finish !== undefined)
             this.send(this.finish);
-        return { messages: this.messages, failure: this.failure, usage: this.usage };
+        return {
+            messages: this.messages,
+            failure: this.failure,
+            usage: this.usage,
+            report: { usage: this.details, steps: this.steps, rawEvents: this.rawEvents },
+        };
     }
     /**
      * What follows a stream that ended well, its approval requests sent first, with the answers

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-sdk-harness-sessions",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Sessions for AI SDK harness agents: the lifecycle of conversations with Claude Code, Codex and friends — one turn at a time, suspended when idle, resumed on the next message and after a restart, several of them sharing one sandbox.",
   "keywords": [
     "ai",
@@ -55,12 +55,12 @@
     "ai": "^7.0.136",
     "typescript": "^6.0.3",
     "vitest": "^5.0.3",
-    "@repo/build": "0.0.0",
-    "@repo/prettier-config": "0.0.0",
     "@repo/eslint-config": "0.0.0",
+    "@repo/prettier-config": "0.0.0",
+    "@repo/build": "0.0.0",
     "@repo/typescript-config": "0.0.0",
     "@repo/vitest-config": "0.0.0",
-    "ai-sdk-sandbox-sbx": "0.1.2"
+    "ai-sdk-sandbox-sbx": "0.1.3"
   },
   "author": {
     "name": "Fabien Pasquet",