@pikku/core 0.12.80 → 0.12.83

package/src/wirings/ai-agent/ai-agent-stream.ts

@@ -1,6 +1,7 @@
 import type {
   AIStreamChannel,
   AIStreamEvent,
+  AIAgentStep,
   AIMessage,
   AIToolCall,
   AIToolResult,
@@ -9,6 +10,10 @@ import type {
   CoreAIAgent,
   AIAgentMemoryConfig,
 } from './ai-agent.types.js'
+import {
+  finalizeAgentRun,
+  lastUserMessageText,
+} from './ai-agent-finalize.js'
 import { pikkuState, getSingletonServices } from '../../pikku-state.js'
 import { applyInputMiddleware } from './ai-agent-turn.js'
 import { AIProviderNotConfiguredError } from '../../errors/errors.js'
@@ -68,6 +73,8 @@ type PersistingChannel = AIStreamChannel & {
   fullText: string
   flush: (opts?: { interrupted?: boolean }) => Promise<void>
   totalUsage: { inputTokens: number; outputTokens: number; model?: string }
+  /** Every tool the run called, kept for the whole run rather than per step. */
+  runToolCalls: NonNullable<AIAgentStep['toolCalls']>
 }
 
 function createPersistingChannel(
@@ -89,6 +96,9 @@ function createPersistingChannel(
     inputTokens: 0,
     outputTokens: 0,
   }
+  // Survives the per-step flush below, which clears its own buffers: the run
+  // record needs every call the run made, not just the last step's.
+  const runToolCalls: NonNullable<AIAgentStep['toolCalls']> = []
 
   const flushStep = async (opts?: { interrupted?: boolean }) => {
     if (!storage) return
@@ -137,6 +147,8 @@ function createPersistingChannel(
     })
   }
 
+  const runToolCallIndex = new Map<string, number>()
+
   const channel: PersistingChannel = {
     channelId: parent.channelId,
     openingData: parent.openingData,
@@ -149,6 +161,9 @@ function createPersistingChannel(
     get totalUsage() {
       return totalUsage
     },
+    get runToolCalls() {
+      return runToolCalls
+    },
     flush: flushStep,
     close: () => parent.close(),
     sendBinary: (data) => parent.sendBinary(data),
@@ -157,6 +172,26 @@ function createPersistingChannel(
       // the client was streamed, and an interrupted run has to be able to
       // report the fragment it got through even with persistence turned off.
       if (event.type === 'text-delta') fullText += event.text
+      if (event.type === 'tool-call') {
+        runToolCallIndex.set(event.toolCallId, runToolCalls.length)
+        runToolCalls.push({
+          name: event.toolName,
+          args: event.args as Record<string, unknown>,
+          result: '',
+        })
+      }
+      if (event.type === 'tool-result') {
+        const index = runToolCallIndex.get(event.toolCallId)
+        const result =
+          typeof event.result === 'string'
+            ? event.result
+            : JSON.stringify(event.result)
+        const call = index === undefined ? undefined : runToolCalls[index]
+        if (call) {
+          call.result = result
+          if (event.error) call.error = event.error
+        }
+      }
       if (storage) {
         switch (event.type) {
           case 'text-delta':
@@ -177,6 +212,7 @@ function createPersistingChannel(
                 typeof event.result === 'string'
                   ? event.result
                   : JSON.stringify(event.result),
+              ...(event.error ? { error: event.error } : {}),
             })
             break
           case 'generative-ui':
@@ -203,44 +239,67 @@ function createPersistingChannel(
   return channel
 }
 
+/**
+ * Agents already warned about, so a per-request hook does not become a
+ * per-request log line.
+ */
+const warnedUnstreamedOutputHooks = new Set<string>()
+
+/**
+ * `modifyOutput` does not run on a streamed run at all. Nothing here could act
+ * on what it returns — the text has already reached the client, and
+ * `createPersistingChannel` flushes each step to storage as it goes, so by the
+ * time the run ends the transcript is already written.
+ *
+ * Rewriting on this path belongs to `modifyOutputStream`, which genuinely
+ * works: the stream middleware wraps the persisting channel, so what is stored
+ * and accumulated is already what the client was sent. A middleware that
+ * rewrites in `modifyOutput` only — a redaction hook, typically — is therefore
+ * silently ineffective when the agent is streamed, and is told so once.
+ */
+const warnUnstreamedOutputHooks = (
+  agentName: string,
+  aiMiddlewares: PikkuAIMiddlewareHooks[],
+  logger?: { warn: (...args: any[]) => void }
+) => {
+  if (warnedUnstreamedOutputHooks.has(agentName)) return
+  const unstreamed = aiMiddlewares.some(
+    (mw) => mw.modifyOutput && !mw.modifyOutputStream
+  )
+  if (!unstreamed) return
+  warnedUnstreamedOutputHooks.add(agentName)
+  logger?.warn(
+    `Agent '${agentName}' has AI middleware with modifyOutput but no modifyOutputStream — modifyOutput does not apply to streamed runs. Implement modifyOutputStream to affect a streamed reply.`
+  )
+}
+
 async function postStreamCleanup(
   persistingChannel: PersistingChannel,
-  aiMiddlewares: PikkuAIMiddlewareHooks[],
-  singletonServices: any,
-  messages: AIMessage[],
   aiRunState: AIRunStateService,
-  runId: string
-): Promise<void> {
-  const usage = persistingChannel.totalUsage
-  let outputText = persistingChannel.fullText
-  let outputMessages = messages
-  for (let i = aiMiddlewares.length - 1; i >= 0; i--) {
-    const mw = aiMiddlewares[i]
-    if (mw.modifyOutput) {
-      const result = await mw.modifyOutput(singletonServices, {
-        text: outputText,
-        messages: outputMessages,
-        usage: {
-          inputTokens: usage.inputTokens,
-          outputTokens: usage.outputTokens,
-        },
-      })
-      outputText = result.text
-      outputMessages = result.messages
-    }
+  runId: string,
+  run: {
+    agentName: string
+    threadId: string
+    resourceId?: string
+    input: string
   }
-
-  await aiRunState.updateRun(runId, {
-    status: 'completed',
-    ...(usage.model
-      ? {
-          usage: {
-            inputTokens: usage.inputTokens,
-            outputTokens: usage.outputTokens,
-            model: usage.model,
-          },
-        }
-      : {}),
+): Promise<void> {
+  await finalizeAgentRun(aiRunState, {
+    runId,
+    agentName: run.agentName,
+    threadId: run.threadId,
+    resourceId: run.resourceId,
+    input: run.input,
+    // Already what the client received: the stream middleware wraps the
+    // persisting channel, so both were accumulated post-rewrite.
+    text: persistingChannel.fullText,
+    steps: [
+      {
+        usage: persistingChannel.totalUsage,
+        toolCalls: persistingChannel.runToolCalls,
+      },
+    ],
+    usage: persistingChannel.totalUsage,
   })
 }
 
@@ -723,6 +782,8 @@ export async function streamAIAgent(
     await storage.saveMessages(threadId, [persistedUserMessage])
   }
 
+  warnUnstreamedOutputHooks(agentName, aiMiddlewares, singletonServices.logger)
+
   const streamMiddleware = aiMiddlewares
     .filter((mw) => mw.modifyOutputStream)
     .map((mw) => {
@@ -849,14 +910,12 @@ export async function streamAIAgent(
       return persistingChannel.fullText
     }
 
-    await postStreamCleanup(
-      persistingChannel,
-      aiMiddlewares,
-      singletonServices,
-      runnerParams.messages,
-      aiRunState,
-      runId
-    )
+    await postStreamCleanup(persistingChannel, aiRunState, runId, {
+      agentName,
+      threadId,
+      resourceId: input.resourceId,
+      input: lastUserMessageText(runnerParams.messages),
+    })
 
     // knowledge: decisions/internals/the-agent-done-event-goes-through-the-middleware-and-is-awaited.md
     await outputChannel.send({ type: 'done' })
@@ -1184,16 +1243,17 @@ export async function resumeAIAgent(
       typeof pending.args === 'string' ? JSON.parse(pending.args) : pending.args
 
     let toolResult: unknown
-    let isError = false
+    let toolError: string | undefined
     try {
       toolResult = await matchingTool.execute(toolArgs)
     } catch (execErr: any) {
       if (execErr?.payload?.error === 'missing_credential') {
         toolResult = execErr.payload
+        toolError = 'missing_credential'
       } else {
-        toolResult = `Error: ${execErr instanceof Error ? execErr.message : String(execErr)}`
+        toolError = execErr instanceof Error ? execErr.message : String(execErr)
+        toolResult = `Error: ${toolError}`
       }
-      isError = true
     }
 
     const resultStr =
@@ -1220,7 +1280,7 @@ export async function resumeAIAgent(
       toolCallId: input.toolCallId,
       toolName: pending.toolName,
       result: toolResult,
-      ...(isError ? { isError: true } : {}),
+      ...(toolError ? { error: toolError } : {}),
     })
   }
 
@@ -1313,6 +1373,12 @@ async function continueAfterToolResult(
   // knowledge: decisions/internals/a-resumed-agent-turn-is-as-interruptible-as-the-first.md
   const interruptHandle = registerInterruptibleRun(run.runId)
 
+  warnUnstreamedOutputHooks(
+    run.agentName,
+    aiMiddlewares,
+    singletonServices.logger
+  )
+
   const streamMiddleware = aiMiddlewares
     .filter((mw) => mw.modifyOutputStream)
     .map((mw) => {
@@ -1434,14 +1500,10 @@ async function continueAfterToolResult(
       return
     }
 
-    await postStreamCleanup(
-      persistingChannel,
-      aiMiddlewares,
-      singletonServices,
-      runnerParams.messages,
-      aiRunState,
-      run.runId
-    )
+    await postStreamCleanup(persistingChannel, aiRunState, run.runId, {
+      ...run,
+      input: lastUserMessageText(runnerParams.messages),
+    })
 
     // knowledge: decisions/internals/the-agent-done-event-goes-through-the-middleware-and-is-awaited.md
     await wrappedChannel.send({ type: 'done' })

package/src/wirings/ai-agent/ai-agent-turn.test.ts

@@ -0,0 +1,67 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { toAccumulatedStep } from './ai-agent-turn.js'
+import type { AIAgentStepResult } from '../../services/ai-agent-runner-service.js'
+
+const stepResult = (
+  overrides?: Partial<AIAgentStepResult>
+): AIAgentStepResult => ({
+  text: '',
+  toolCalls: [],
+  toolResults: [],
+  usage: { inputTokens: 0, outputTokens: 0 },
+  finishReason: 'stop',
+  ...overrides,
+})
+
+describe('toAccumulatedStep', () => {
+  test('carries a tool failure as its own field, not only as rendered text', () => {
+    const step = toAccumulatedStep(
+      stepResult({
+        toolCalls: [
+          { toolCallId: 'call-1', toolName: 'lookupOrder', args: { id: 7 } },
+        ],
+        toolResults: [
+          {
+            toolCallId: 'call-1',
+            toolName: 'lookupOrder',
+            result: 'Error: order service unreachable',
+            error: 'order service unreachable',
+          },
+        ],
+      })
+    )
+
+    assert.deepEqual(step.toolCalls, [
+      {
+        name: 'lookupOrder',
+        args: { id: 7 },
+        result: 'Error: order service unreachable',
+        error: 'order service unreachable',
+      },
+    ])
+  })
+
+  test('leaves error unset on a tool that returned normally, even if it says Error', () => {
+    // A tool is allowed to return the word "Error" — which is exactly why
+    // "did this fail" cannot be answered by matching on the result text.
+    const step = toAccumulatedStep(
+      stepResult({
+        toolCalls: [
+          { toolCallId: 'call-1', toolName: 'searchLogs', args: { q: 'x' } },
+        ],
+        toolResults: [
+          {
+            toolCallId: 'call-1',
+            toolName: 'searchLogs',
+            result: 'Error: connection refused (1 match)',
+          },
+        ],
+      })
+    )
+
+    assert.equal(step.toolCalls[0].error, undefined)
+    assert.ok(!('error' in step.toolCalls[0]))
+  })
+})

package/src/wirings/ai-agent/ai-agent-turn.ts

@@ -83,6 +83,7 @@ export const toAccumulatedStep = (stepResult: StepResult) => ({
         typeof tr?.result === 'string'
           ? tr.result
           : JSON.stringify(tr?.result ?? ''),
+      ...(tr?.error ? { error: tr.error } : {}),
     }
   }),
 })

package/src/wirings/ai-agent/ai-agent.types.ts

@@ -51,6 +51,13 @@ export interface AIToolResult {
   id: string
   name: string
   result: string
+  /**
+   * Set when the tool threw rather than returned. Carried separately from
+   * `result`, which is a rendered string by the time it is persisted — a tool
+   * may legitimately return text beginning `Error:`, so the prefix cannot be
+   * read as a failure signal.
+   */
+  error?: string
 }
 
 export interface AIMessage {
@@ -82,7 +89,13 @@ export interface AIMessage {
 
 export interface AIAgentStep {
   usage: { inputTokens: number; outputTokens: number }
-  toolCalls?: { name: string; args: Record<string, unknown>; result: string }[]
+  toolCalls?: {
+    name: string
+    args: Record<string, unknown>
+    result: string
+    /** The failure message, when the tool threw rather than returned. */
+    error?: string
+  }[]
 }
 
 export interface AIAgentInputAttachment {
@@ -233,16 +246,40 @@ export interface PikkuAIMiddlewareHooks<
     | AIStreamEvent[]
     | null
 
+  /**
+   * The last chance to rewrite what the run produced, before it is persisted
+   * and returned.
+   *
+   * It does **not** run on a streamed run: there the text has already reached
+   * the client and each step is flushed to storage as it goes, so nothing could
+   * act on what this returned. Use {@link modifyOutputStream} to rewrite a
+   * streamed reply — a middleware that implements only this one is warned about
+   * when an agent it is attached to streams.
+   *
+   * `toolCalls` is here so a redaction pass covers the whole run record rather
+   * than just the visible answer: the tool arguments and results are persisted
+   * and handed to anything that grades the run, and scrubbing the reply alone
+   * leaves them untouched.
+   */
   modifyOutput?: (
     services: Services,
     ctx: {
       text: string
       messages: AIMessage[]
       usage: { inputTokens: number; outputTokens: number }
+      toolCalls: NonNullable<AIAgentStep['toolCalls']>
     }
   ) =>
-    | Promise<{ text: string; messages: AIMessage[] }>
-    | { text: string; messages: AIMessage[] }
+    | Promise<{
+        text: string
+        messages: AIMessage[]
+        toolCalls?: NonNullable<AIAgentStep['toolCalls']>
+      }>
+    | {
+        text: string
+        messages: AIMessage[]
+        toolCalls?: NonNullable<AIAgentStep['toolCalls']>
+      }
 
   beforeToolCall?: (
     services: Services,
@@ -273,7 +310,13 @@ export interface PikkuAIMiddlewareHooks<
       stepNumber: number
       text: string
       toolCalls: { toolCallId: string; toolName: string; args: unknown }[]
-      toolResults: { toolCallId: string; toolName: string; result: unknown }[]
+      toolResults: {
+        toolCallId: string
+        toolName: string
+        result: unknown
+        /** Set when the tool threw rather than returned. */
+        error?: string
+      }[]
       usage: { inputTokens: number; outputTokens: number }
       finishReason: string
     }
@@ -301,6 +344,7 @@ export type CoreAIAgent<
   PikkuPermission = CorePikkuPermission<any, any>,
   PikkuMiddleware = CorePikkuMiddleware<any>,
   Scope extends string = string,
+  Scorer extends string = string,
 > = {
   name: string
   description: string
@@ -327,6 +371,16 @@ export type CoreAIAgent<
   tools?: unknown[]
   agents?: unknown[]
   workflows?: unknown[]
+  /**
+   * Grades this agent's finished runs on live traffic, named by the generated
+   * `ScorerName` union rather than by `ref()` — a scorer is not a function, so
+   * there is nothing in the function map for a ref to resolve against.
+   *
+   * A reference-based judge listed here is never sampled: live traffic has no
+   * answer key. Scenarios name scorers directly and may grade with scorers an
+   * agent does not ship with.
+   */
+  scorers?: Scorer[]
   agentMode?: 'delegate' | 'supervise'
   memory?: AIAgentMemoryConfig
   maxSteps?: number
@@ -392,6 +446,12 @@ export type AIStreamEvent =
       toolCallId: string
       toolName: string
       result: unknown
+      /**
+       * The failure message, set when the tool threw rather than returned.
+       * Carried explicitly because a tool may legitimately return text that
+       * reads like an error, so `result` cannot be matched on to tell.
+       */
+      error?: string
       agent?: string
       session?: string
     }

package/src/wirings/ai-agent/index.ts

@@ -5,8 +5,9 @@ export {
   agentApprove,
   agentInterrupt,
 } from './ai-agent-helpers.js'
-export { wrapChannelWithAGUI, type AGUIEvent } from './ai-agent-agui.js'
+export { wrapChannelWithAGUI } from './ai-agent-agui.js'
 export { runAIAgent, resumeAIAgentSync } from './ai-agent-runner.js'
+export { resolveModelAlias } from './ai-agent-model-config.js'
 export {
   streamAIAgent,
   resumeAIAgent,
@@ -14,7 +15,6 @@ export {
 } from './ai-agent-stream.js'
 export {
   voiceInput,
-  readsAsNonSpeech,
   NoSpeechDetectedError,
   SPOKEN_TURN,
   SPOKEN_TRANSCRIPT,
@@ -27,21 +27,12 @@ export {
 } from './voice-output.js'
 export {
   AgentInterruptedError,
-  awaitPendingInterruptNote,
-  getInFlightTools,
-  isAbortError,
-  isRunInterruptible,
-  persistOrphanedToolResults,
-  registerInterruptibleRun,
   signalRunInterrupt,
-  trackInterruptNote,
-  trackToolExecution,
 } from './ai-agent-interrupt.js'
 export type {
   AgentInterruption,
   AgentInterruptResult,
   InterruptibleRunHandle,
-  OrphanedToolResult,
 } from './ai-agent-interrupt.js'
 export {
   type RunAIAgentParams,
@@ -50,18 +41,13 @@ export {
   ToolCredentialRequired,
   canAccessThread,
   isOwnedByPrincipal,
-  sessionPrincipals,
   threadOwnerConstraint,
 } from './ai-agent-prepare.js'
 export {
   addAIAgent,
-  approveAIAgent,
-  getAIAgents,
-  getAIAgentsMeta,
 } from './ai-agent-registry.js'
 export type {
   AIAgentInput,
-  AIAgentInputAttachment,
   AIAgentMeta,
   AIAgentMemoryConfig,
   AIAgentStep,

package/src/wirings/ai-scorer/ai-scorer-grade.test.ts

@@ -0,0 +1,106 @@
+import { beforeEach, describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { pikkuState, resetPikkuState } from '../../pikku-state.js'
+import { gradeRun } from './ai-scorer-grade.js'
+import { pikkuAIJudge, pikkuAIScorer } from './ai-scorer.js'
+import type { ScoreJob } from './ai-scorer.types.js'
+
+const job = (overrides: Partial<ScoreJob> = {}): ScoreJob => ({
+  scorerName: 'correctness',
+  runId: 'run-1',
+  agentName: 'assistant',
+  input: 'what is the capital of France?',
+  output: 'Paris',
+  toolCalls: [],
+  usage: { inputTokens: 10, outputTokens: 5 },
+  ...overrides,
+})
+
+describe('gradeRun', () => {
+  beforeEach(() => resetPikkuState())
+
+  test('a scenario grade is returned and not written to the live record', async () => {
+    pikkuState(null, 'agent', 'scorers').set(
+      'correctness',
+      pikkuAIScorer({
+        name: 'correctness',
+        description: 'Matches the answer key',
+        requiresReference: true,
+        score: (input) => ({
+          score: input.output === input.reference ? 1 : 0,
+        }),
+      })
+    )
+    let saves = 0
+
+    const result = await gradeRun(
+      job({ reference: 'Paris' }),
+      {
+        aiRunState: {
+          saveScore: async () => {
+            saves++
+          },
+        },
+      },
+      { persist: false }
+    )
+
+    assert.deepEqual(result, { score: 1 })
+    assert.equal(saves, 0)
+  })
+
+  test('a reference-based scorer sees the answer key it was given', async () => {
+    pikkuState(null, 'agent', 'scorers').set(
+      'correctness',
+      pikkuAIScorer({
+        name: 'correctness',
+        description: 'Matches the answer key',
+        requiresReference: true,
+        score: (input) => ({
+          score: input.output === input.reference ? 1 : 0,
+        }),
+      })
+    )
+
+    const result = await gradeRun(
+      job({ reference: 'Lyon' }),
+      {},
+      {
+        persist: false,
+      }
+    )
+
+    assert.equal(result.score, 0)
+  })
+
+  test('a judge grades without a score function, and reports the model it used', async () => {
+    pikkuState(null, 'agent', 'scorers').set(
+      'helpfulness',
+      pikkuAIJudge({
+        name: 'helpfulness',
+        description: 'Is the answer useful',
+        model: 'claude-opus-5',
+        goal: 'Grade helpfulness.',
+      })
+    )
+
+    const result = await gradeRun(
+      job({ scorerName: 'helpfulness' }),
+      {
+        aiAgentRunner: {
+          run: async () => ({
+            object: { score: 0.75, reason: 'Correct but terse.' },
+            usage: { inputTokens: 80, outputTokens: 20 },
+          }),
+        },
+      },
+      { persist: false }
+    )
+
+    assert.equal(result.score, 0.75)
+    assert.deepEqual(result.metadata, {
+      judgeModel: 'claude-opus-5',
+      judgeTokens: 100,
+    })
+  })
+})

package/src/wirings/ai-scorer/ai-scorer-grade.ts

@@ -0,0 +1,55 @@
+import { runJudge } from './ai-scorer-judge.js'
+import { resolveAIScorer } from './ai-scorer-registry.js'
+import type { ScoreJob, ScorerOutput } from './ai-scorer.types.js'
+
+/**
+ * Grade one run with one scorer.
+ *
+ * The single path both callers take — the lane worker on live traffic and the
+ * scenario grading RPC — so a scenario's grade is the same computation the
+ * production sampler would have made, not an approximation of it.
+ *
+ * Persisting is optional because the two callers differ on it: a live grade is
+ * only useful once recorded, while a scenario asserts on the returned value and
+ * runs against servers that may have no run-state adapter at all.
+ */
+export const gradeRun = async (
+  job: ScoreJob,
+  services: {
+    aiAgentRunner?: unknown
+    aiRunState?: {
+      saveScore: (score: {
+        runId: string
+        scorerName: string
+        score: number
+        reason?: string
+        metadata?: Record<string, unknown>
+      }) => Promise<void>
+    }
+  },
+  options: { persist: boolean }
+): Promise<ScorerOutput> => {
+  const { scorerName, ...input } = job
+  const scorer = resolveAIScorer(scorerName)
+
+  const result = scorer.score
+    ? await scorer.score(input, services)
+    : await runJudge(scorer, input, services.aiAgentRunner as never)
+
+  if (options.persist) {
+    if (!services.aiRunState) {
+      throw new Error(
+        `AI run state service not initialized: cannot record the '${scorerName}' grade of run ${job.runId}`
+      )
+    }
+    await services.aiRunState.saveScore({
+      runId: job.runId,
+      scorerName,
+      score: result.score,
+      ...(result.reason !== undefined ? { reason: result.reason } : {}),
+      ...(result.metadata !== undefined ? { metadata: result.metadata } : {}),
+    })
+  }
+
+  return result
+}