@pikku/core 0.12.79 → 0.12.82

package/src/wirings/ai-agent/ai-agent.types.ts

@@ -51,6 +51,13 @@ export interface AIToolResult {
   id: string
   name: string
   result: string
+  /**
+   * Set when the tool threw rather than returned. Carried separately from
+   * `result`, which is a rendered string by the time it is persisted — a tool
+   * may legitimately return text beginning `Error:`, so the prefix cannot be
+   * read as a failure signal.
+   */
+  error?: string
 }
 
 export interface AIMessage {
@@ -82,7 +89,13 @@ export interface AIMessage {
 
 export interface AIAgentStep {
   usage: { inputTokens: number; outputTokens: number }
-  toolCalls?: { name: string; args: Record<string, unknown>; result: string }[]
+  toolCalls?: {
+    name: string
+    args: Record<string, unknown>
+    result: string
+    /** The failure message, when the tool threw rather than returned. */
+    error?: string
+  }[]
 }
 
 export interface AIAgentInputAttachment {
@@ -233,16 +246,40 @@ export interface PikkuAIMiddlewareHooks<
     | AIStreamEvent[]
     | null
 
+  /**
+   * The last chance to rewrite what the run produced, before it is persisted
+   * and returned.
+   *
+   * It does **not** run on a streamed run: there the text has already reached
+   * the client and each step is flushed to storage as it goes, so nothing could
+   * act on what this returned. Use {@link modifyOutputStream} to rewrite a
+   * streamed reply — a middleware that implements only this one is warned about
+   * when an agent it is attached to streams.
+   *
+   * `toolCalls` is here so a redaction pass covers the whole run record rather
+   * than just the visible answer: the tool arguments and results are persisted
+   * and handed to anything that grades the run, and scrubbing the reply alone
+   * leaves them untouched.
+   */
   modifyOutput?: (
     services: Services,
     ctx: {
       text: string
       messages: AIMessage[]
       usage: { inputTokens: number; outputTokens: number }
+      toolCalls: NonNullable<AIAgentStep['toolCalls']>
     }
   ) =>
-    | Promise<{ text: string; messages: AIMessage[] }>
-    | { text: string; messages: AIMessage[] }
+    | Promise<{
+        text: string
+        messages: AIMessage[]
+        toolCalls?: NonNullable<AIAgentStep['toolCalls']>
+      }>
+    | {
+        text: string
+        messages: AIMessage[]
+        toolCalls?: NonNullable<AIAgentStep['toolCalls']>
+      }
 
   beforeToolCall?: (
     services: Services,
@@ -273,7 +310,13 @@ export interface PikkuAIMiddlewareHooks<
       stepNumber: number
       text: string
       toolCalls: { toolCallId: string; toolName: string; args: unknown }[]
-      toolResults: { toolCallId: string; toolName: string; result: unknown }[]
+      toolResults: {
+        toolCallId: string
+        toolName: string
+        result: unknown
+        /** Set when the tool threw rather than returned. */
+        error?: string
+      }[]
       usage: { inputTokens: number; outputTokens: number }
       finishReason: string
     }
@@ -301,6 +344,7 @@ export type CoreAIAgent<
   PikkuPermission = CorePikkuPermission<any, any>,
   PikkuMiddleware = CorePikkuMiddleware<any>,
   Scope extends string = string,
+  Scorer extends string = string,
 > = {
   name: string
   description: string
@@ -327,6 +371,16 @@ export type CoreAIAgent<
   tools?: unknown[]
   agents?: unknown[]
   workflows?: unknown[]
+  /**
+   * Grades this agent's finished runs on live traffic, named by the generated
+   * `ScorerName` union rather than by `ref()` — a scorer is not a function, so
+   * there is nothing in the function map for a ref to resolve against.
+   *
+   * A reference-based judge listed here is never sampled: live traffic has no
+   * answer key. Scenarios name scorers directly and may grade with scorers an
+   * agent does not ship with.
+   */
+  scorers?: Scorer[]
   agentMode?: 'delegate' | 'supervise'
   memory?: AIAgentMemoryConfig
   maxSteps?: number
@@ -392,6 +446,12 @@ export type AIStreamEvent =
       toolCallId: string
       toolName: string
       result: unknown
+      /**
+       * The failure message, set when the tool threw rather than returned.
+       * Carried explicitly because a tool may legitimately return text that
+       * reads like an error, so `result` cannot be matched on to tell.
+       */
+      error?: string
       agent?: string
       session?: string
     }
@@ -413,7 +473,7 @@ export type AIStreamEvent =
       toolName: string
       args: unknown
       reason?: string
-      runId?: string
+      runId: string
       agent?: string
       session?: string
     }

package/src/wirings/ai-agent/index.ts

@@ -5,8 +5,9 @@ export {
   agentApprove,
   agentInterrupt,
 } from './ai-agent-helpers.js'
-export { wrapChannelWithAGUI, type AGUIEvent } from './ai-agent-agui.js'
+export { wrapChannelWithAGUI } from './ai-agent-agui.js'
 export { runAIAgent, resumeAIAgentSync } from './ai-agent-runner.js'
+export { resolveModelAlias } from './ai-agent-model-config.js'
 export {
   streamAIAgent,
   resumeAIAgent,
@@ -14,7 +15,6 @@ export {
 } from './ai-agent-stream.js'
 export {
   voiceInput,
-  readsAsNonSpeech,
   NoSpeechDetectedError,
   SPOKEN_TURN,
   SPOKEN_TRANSCRIPT,
@@ -27,21 +27,12 @@ export {
 } from './voice-output.js'
 export {
   AgentInterruptedError,
-  awaitPendingInterruptNote,
-  getInFlightTools,
-  isAbortError,
-  isRunInterruptible,
-  persistOrphanedToolResults,
-  registerInterruptibleRun,
   signalRunInterrupt,
-  trackInterruptNote,
-  trackToolExecution,
 } from './ai-agent-interrupt.js'
 export type {
   AgentInterruption,
   AgentInterruptResult,
   InterruptibleRunHandle,
-  OrphanedToolResult,
 } from './ai-agent-interrupt.js'
 export {
   type RunAIAgentParams,
@@ -50,18 +41,13 @@ export {
   ToolCredentialRequired,
   canAccessThread,
   isOwnedByPrincipal,
-  sessionPrincipals,
   threadOwnerConstraint,
 } from './ai-agent-prepare.js'
 export {
   addAIAgent,
-  approveAIAgent,
-  getAIAgents,
-  getAIAgentsMeta,
 } from './ai-agent-registry.js'
 export type {
   AIAgentInput,
-  AIAgentInputAttachment,
   AIAgentMeta,
   AIAgentMemoryConfig,
   AIAgentStep,

package/src/wirings/ai-agent/voice-input.ts

@@ -164,12 +164,7 @@ export const voiceInput = (config?: {
       // to send, and a message with no content is not a question.
       if (updatedContent.length === 0) throw new NoSpeechDetectedError()
 
-      // Only when something was actually heard. A turn can carry audio that all
-      // reads as non-speech and still have content — an image with a silent
-      // caption clip — which leaves nothing above to throw. Recording `''` here
-      // would send a transcript event saying the user said nothing, and a client
-      // that tells "not transcribed yet" from "transcribed" by the key being
-      // absent would render the turn as an empty bubble rather than a pending one.
+      // knowledge: decisions/internals/an-empty-transcript-is-not-recorded.md
       if (heard.length > 0) {
         shared[SPOKEN_TRANSCRIPT] = heard.join(' ')
       }

package/src/wirings/ai-agent/voice-output.ts

@@ -206,11 +206,7 @@ export const voiceOutput = (config?: {
       }
       if (!aiAgentRunner?.generateSpeech) return event
 
-      // Only an explicit `false` silences the reply. `voiceInput` sets this on
-      // every turn it handles, so `false` means a real user really typed; the
-      // key being absent means nothing reported either way — no voice input is
-      // wired — and that caller's replies are spoken exactly as they were before
-      // this option existed.
+      // knowledge: decisions/internals/voice-output-speaks-unless-voice-input-explicitly-says-otherwise.md
       if (!config?.always && shared[SPOKEN_TURN] === false) return event
 
       /**
@@ -228,13 +224,7 @@ export const voiceOutput = (config?: {
           )
         }
 
-        // Checked per sentence but announced once per reply: a bilingual answer
-        // should still speak the half it can, and repeating the notice for
-        // every sentence of a long one would bury the reply itself.
-        //
-        // Per sentence is also what makes the voice right. A reply that answers
-        // in English and then quotes a Chinese title is two sentences in two
-        // scripts, and each is synthesized in the voice its own script needs.
+        // knowledge: decisions/internals/speech-synthesis-picks-a-voice-per-sentence-and-warns-once.md
         let voice = config.voice
         if (config.speakableScripts) {
           const unspeakable = unspeakableScripts(text, config.speakableScripts)

package/src/wirings/ai-scorer/ai-scorer-grade.test.ts

@@ -0,0 +1,106 @@
+import { beforeEach, describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { pikkuState, resetPikkuState } from '../../pikku-state.js'
+import { gradeRun } from './ai-scorer-grade.js'
+import { pikkuAIJudge, pikkuAIScorer } from './ai-scorer.js'
+import type { ScoreJob } from './ai-scorer.types.js'
+
+const job = (overrides: Partial<ScoreJob> = {}): ScoreJob => ({
+  scorerName: 'correctness',
+  runId: 'run-1',
+  agentName: 'assistant',
+  input: 'what is the capital of France?',
+  output: 'Paris',
+  toolCalls: [],
+  usage: { inputTokens: 10, outputTokens: 5 },
+  ...overrides,
+})
+
+describe('gradeRun', () => {
+  beforeEach(() => resetPikkuState())
+
+  test('a scenario grade is returned and not written to the live record', async () => {
+    pikkuState(null, 'agent', 'scorers').set(
+      'correctness',
+      pikkuAIScorer({
+        name: 'correctness',
+        description: 'Matches the answer key',
+        requiresReference: true,
+        score: (input) => ({
+          score: input.output === input.reference ? 1 : 0,
+        }),
+      })
+    )
+    let saves = 0
+
+    const result = await gradeRun(
+      job({ reference: 'Paris' }),
+      {
+        aiRunState: {
+          saveScore: async () => {
+            saves++
+          },
+        },
+      },
+      { persist: false }
+    )
+
+    assert.deepEqual(result, { score: 1 })
+    assert.equal(saves, 0)
+  })
+
+  test('a reference-based scorer sees the answer key it was given', async () => {
+    pikkuState(null, 'agent', 'scorers').set(
+      'correctness',
+      pikkuAIScorer({
+        name: 'correctness',
+        description: 'Matches the answer key',
+        requiresReference: true,
+        score: (input) => ({
+          score: input.output === input.reference ? 1 : 0,
+        }),
+      })
+    )
+
+    const result = await gradeRun(
+      job({ reference: 'Lyon' }),
+      {},
+      {
+        persist: false,
+      }
+    )
+
+    assert.equal(result.score, 0)
+  })
+
+  test('a judge grades without a score function, and reports the model it used', async () => {
+    pikkuState(null, 'agent', 'scorers').set(
+      'helpfulness',
+      pikkuAIJudge({
+        name: 'helpfulness',
+        description: 'Is the answer useful',
+        model: 'claude-opus-5',
+        goal: 'Grade helpfulness.',
+      })
+    )
+
+    const result = await gradeRun(
+      job({ scorerName: 'helpfulness' }),
+      {
+        aiAgentRunner: {
+          run: async () => ({
+            object: { score: 0.75, reason: 'Correct but terse.' },
+            usage: { inputTokens: 80, outputTokens: 20 },
+          }),
+        },
+      },
+      { persist: false }
+    )
+
+    assert.equal(result.score, 0.75)
+    assert.deepEqual(result.metadata, {
+      judgeModel: 'claude-opus-5',
+      judgeTokens: 100,
+    })
+  })
+})

package/src/wirings/ai-scorer/ai-scorer-grade.ts

@@ -0,0 +1,55 @@
+import { runJudge } from './ai-scorer-judge.js'
+import { resolveAIScorer } from './ai-scorer-registry.js'
+import type { ScoreJob, ScorerOutput } from './ai-scorer.types.js'
+
+/**
+ * Grade one run with one scorer.
+ *
+ * The single path both callers take — the lane worker on live traffic and the
+ * scenario grading RPC — so a scenario's grade is the same computation the
+ * production sampler would have made, not an approximation of it.
+ *
+ * Persisting is optional because the two callers differ on it: a live grade is
+ * only useful once recorded, while a scenario asserts on the returned value and
+ * runs against servers that may have no run-state adapter at all.
+ */
+export const gradeRun = async (
+  job: ScoreJob,
+  services: {
+    aiAgentRunner?: unknown
+    aiRunState?: {
+      saveScore: (score: {
+        runId: string
+        scorerName: string
+        score: number
+        reason?: string
+        metadata?: Record<string, unknown>
+      }) => Promise<void>
+    }
+  },
+  options: { persist: boolean }
+): Promise<ScorerOutput> => {
+  const { scorerName, ...input } = job
+  const scorer = resolveAIScorer(scorerName)
+
+  const result = scorer.score
+    ? await scorer.score(input, services)
+    : await runJudge(scorer, input, services.aiAgentRunner as never)
+
+  if (options.persist) {
+    if (!services.aiRunState) {
+      throw new Error(
+        `AI run state service not initialized: cannot record the '${scorerName}' grade of run ${job.runId}`
+      )
+    }
+    await services.aiRunState.saveScore({
+      runId: job.runId,
+      scorerName,
+      score: result.score,
+      ...(result.reason !== undefined ? { reason: result.reason } : {}),
+      ...(result.metadata !== undefined ? { metadata: result.metadata } : {}),
+    })
+  }
+
+  return result
+}

package/src/wirings/ai-scorer/ai-scorer-judge.test.ts

@@ -0,0 +1,143 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { buildJudgePrompt, runJudge } from './ai-scorer-judge.js'
+import { pikkuAIJudge } from './ai-scorer.js'
+import type { ScorerInput } from './ai-scorer.types.js'
+import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
+
+const input = (overrides: Partial<ScorerInput> = {}): ScorerInput => ({
+  runId: 'run-1',
+  agentName: 'assistant',
+  input: 'what is the capital of France?',
+  output: 'Paris',
+  toolCalls: [],
+  usage: { inputTokens: 10, outputTokens: 5 },
+  ...overrides,
+})
+
+const runner = (
+  object: unknown,
+  seen?: { params: any }
+): AIAgentRunnerService =>
+  ({
+    run: async (params: any) => {
+      if (seen) seen.params = params
+      return { object, usage: { inputTokens: 100, outputTokens: 20 } }
+    },
+  }) as unknown as AIAgentRunnerService
+
+describe('buildJudgePrompt', () => {
+  test('shows the answer key to a reference-based judge', () => {
+    const scorer = pikkuAIJudge({
+      name: 'correctness',
+      description: 'Is the answer right',
+      model: 'claude-opus-5',
+      goal: 'Grade correctness.',
+      requiresReference: true,
+    })
+
+    const prompt = buildJudgePrompt(
+      scorer.judge!,
+      input({ reference: 'Paris' })
+    )
+
+    assert.match(prompt, /Reference answer:\nParis/)
+  })
+
+  test('withholds a reference section when there is no answer key', () => {
+    const scorer = pikkuAIJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+    })
+
+    const prompt = buildJudgePrompt(scorer.judge!, input())
+
+    assert.doesNotMatch(prompt, /Reference answer/)
+    assert.match(prompt, /Grade helpfulness\./)
+  })
+
+  test('a scorer that supplies its own prompt replaces the framing entirely', () => {
+    const scorer = pikkuAIJudge({
+      name: 'custom',
+      description: 'Custom framing',
+      model: 'claude-opus-5',
+      goal: 'ignored',
+      prompt: (run) => `Only this: ${run.output}`,
+    })
+
+    assert.equal(buildJudgePrompt(scorer.judge!, input()), 'Only this: Paris')
+  })
+})
+
+describe('runJudge', () => {
+  const helpfulness = pikkuAIJudge({
+    name: 'helpfulness',
+    description: 'Is the answer useful',
+    model: 'claude-opus-5',
+    goal: 'Grade helpfulness.',
+  })
+
+  test('records the score, the reason and what the judgement cost', async () => {
+    const result = await runJudge(
+      helpfulness,
+      input(),
+      runner({ score: 0.8, reason: 'Direct and correct.' })
+    )
+
+    assert.equal(result.score, 0.8)
+    assert.equal(result.reason, 'Direct and correct.')
+    assert.deepEqual(result.metadata, {
+      judgeModel: 'claude-opus-5',
+      judgeTokens: 120,
+    })
+  })
+
+  test('clamps an out-of-range score rather than losing the judgement', async () => {
+    const high = await runJudge(helpfulness, input(), runner({ score: 1.4 }))
+    const low = await runJudge(helpfulness, input(), runner({ score: -2 }))
+
+    assert.equal(high.score, 1)
+    assert.equal(low.score, 0)
+  })
+
+  test('rejects a non-numeric score, which is not a judgement at all', async () => {
+    await assert.rejects(
+      () => runJudge(helpfulness, input(), runner({ score: 'great' })),
+      /non-numeric score/
+    )
+  })
+
+  test('gives the judge no tools, so its structured output is actually honoured', async () => {
+    const seen = { params: undefined as any }
+    await runJudge(helpfulness, input(), runner({ score: 1 }, seen))
+
+    assert.deepEqual(seen.params.tools, [])
+    assert.equal(seen.params.toolChoice, 'none')
+    assert.equal(seen.params.maxSteps, 1)
+    assert.equal(seen.params.model, 'claude-opus-5')
+  })
+
+  test('names the missing provider rather than failing as an undefined call', async () => {
+    await assert.rejects(
+      () => runJudge(helpfulness, input(), undefined),
+      /needs an AI provider/
+    )
+  })
+
+  test('refuses to grade a reference-based judge with no answer key', async () => {
+    const correctness = pikkuAIJudge({
+      name: 'correctness',
+      description: 'Is the answer right',
+      model: 'claude-opus-5',
+      goal: 'Grade correctness.',
+      requiresReference: true,
+    })
+
+    await assert.rejects(
+      () => runJudge(correctness, input(), runner({ score: 1 })),
+      /grades against a reference answer/
+    )
+  })
+})

package/src/wirings/ai-scorer/ai-scorer-judge.ts

@@ -0,0 +1,120 @@
+import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
+import type { AIMessage } from '../ai-agent/ai-agent.types.js'
+import { randomUUID } from '../ai-agent/ai-agent-utils.js'
+import type {
+  PikkuAIScorer,
+  ScorerInput,
+  ScorerJudgeConfig,
+  ScorerOutput,
+} from './ai-scorer.types.js'
+
+const JUDGE_OUTPUT_SCHEMA = {
+  type: 'object',
+  properties: {
+    score: {
+      type: 'number',
+      minimum: 0,
+      maximum: 1,
+      description: 'How well the answer meets the rubric. 1 is best.',
+    },
+    reason: {
+      type: 'string',
+      description: 'One or two sentences explaining the score.',
+    },
+  },
+  required: ['score', 'reason'],
+  additionalProperties: false,
+}
+
+/**
+ * The message a judge grades when its scorer supplies no `prompt`.
+ *
+ * A reference-based judge is shown the answer key; a reference-free one is not,
+ * and grades the answer on its own merits.
+ */
+export const buildJudgePrompt = (
+  judge: ScorerJudgeConfig,
+  input: ScorerInput
+): string => {
+  if (judge.prompt) return judge.prompt(input)
+
+  const sections = [`User asked:\n${input.input}`]
+  if (input.reference !== undefined) {
+    sections.push(`Reference answer:\n${input.reference}`)
+  }
+  sections.push(`Assistant answered:\n${input.output}`)
+  sections.push(judge.goal)
+  sections.push('Respond with a score 0..1 and a brief reason.')
+  return sections.join('\n\n')
+}
+
+/**
+ * Clamp rather than reject: a model that answers 1.2 has still made a legible
+ * judgement, and failing the job would lose it. A non-number has not.
+ */
+const normalizeScore = (scorerName: string, value: unknown): number => {
+  if (typeof value !== 'number' || Number.isNaN(value)) {
+    throw new Error(
+      `Judge '${scorerName}' returned a non-numeric score: ${JSON.stringify(value)}`
+    )
+  }
+  return Math.min(1, Math.max(0, value))
+}
+
+/**
+ * Run an LLM judge and force the `{ score, reason }` contract, so a scorer
+ * author writes a rubric rather than a parser.
+ *
+ * A judge has no tools, which is the one case where an output schema is
+ * reliably honoured.
+ */
+export const runJudge = async (
+  scorer: PikkuAIScorer<any>,
+  input: ScorerInput,
+  aiAgentRunner: AIAgentRunnerService | undefined
+): Promise<ScorerOutput> => {
+  const judge = scorer.judge
+  if (!judge) {
+    throw new Error(`Scorer '${scorer.name}' is not a judge`)
+  }
+  if (!aiAgentRunner) {
+    throw new Error(
+      `Judge '${scorer.name}' needs an AI provider, but no aiAgentRunner is registered in this process. ` +
+        `A worker deployed apart from the API has to register one to run the slow scoring lane.`
+    )
+  }
+  if (scorer.requiresReference && input.reference === undefined) {
+    throw new Error(
+      `Judge '${scorer.name}' grades against a reference answer, but none was supplied`
+    )
+  }
+
+  const messages: AIMessage[] = [
+    {
+      id: randomUUID(),
+      role: 'user',
+      content: buildJudgePrompt(judge, input),
+      createdAt: new Date(),
+    },
+  ]
+
+  const result = await aiAgentRunner.run({
+    model: judge.model,
+    instructions: judge.goal,
+    messages,
+    tools: [],
+    maxSteps: 1,
+    toolChoice: 'none',
+    outputSchema: JUDGE_OUTPUT_SCHEMA,
+  })
+
+  const object = (result.object ?? {}) as { score?: unknown; reason?: unknown }
+  return {
+    score: normalizeScore(scorer.name, object.score),
+    ...(typeof object.reason === 'string' ? { reason: object.reason } : {}),
+    metadata: {
+      judgeModel: judge.model,
+      judgeTokens: result.usage.inputTokens + result.usage.outputTokens,
+    },
+  }
+}