@pikku/core 0.12.83 → 0.12.86

package/src/wirings/agent-scorer/agent-scorer-judge.test.ts

@@ -0,0 +1,285 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { buildJudgePrompt, runJudge } from './agent-scorer-judge.js'
+import { pikkuAgentJudge } from './agent-scorer.js'
+import type { ScorerInput } from './agent-scorer.types.js'
+import type { AgentRunnerService } from '../../services/agent-runner-service.js'
+
+const input = (overrides: Partial<ScorerInput> = {}): ScorerInput => ({
+  runId: 'run-1',
+  agentName: 'assistant',
+  input: 'what is the capital of France?',
+  output: 'Paris',
+  toolCalls: [],
+  usage: { inputTokens: 10, outputTokens: 5 },
+  ...overrides,
+})
+
+const runner = (object: unknown, seen?: { params: any }): AgentRunnerService =>
+  ({
+    run: async (params: any) => {
+      if (seen) seen.params = params
+      return { object, usage: { inputTokens: 100, outputTokens: 20 } }
+    },
+  }) as unknown as AgentRunnerService
+
+describe('buildJudgePrompt', () => {
+  test('shows the answer key to a reference-based judge', () => {
+    const scorer = pikkuAgentJudge({
+      name: 'correctness',
+      description: 'Is the answer right',
+      model: 'claude-opus-5',
+      goal: 'Grade correctness.',
+      requiresReference: true,
+    })
+
+    const prompt = buildJudgePrompt(
+      scorer.judge!,
+      input({ reference: 'Paris' })
+    )
+
+    assert.match(prompt, /Reference answer:\nParis/)
+  })
+
+  test('withholds a reference section when there is no answer key', () => {
+    const scorer = pikkuAgentJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+    })
+
+    const prompt = buildJudgePrompt(scorer.judge!, input())
+
+    assert.doesNotMatch(prompt, /Reference answer/)
+    assert.match(prompt, /Grade helpfulness\./)
+  })
+
+  test('shows which tools ran, so an answer is distinguishable from a guess', () => {
+    const scorer = pikkuAgentJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+    })
+
+    const prompt = buildJudgePrompt(
+      scorer.judge!,
+      input({
+        toolCalls: [
+          { name: 'listTodos', args: {}, result: { todos: ['Buy groceries'] } },
+        ],
+      })
+    )
+
+    assert.match(prompt, /Tools the assistant ran:\n- listTodos/)
+  })
+
+  test('the default disclosure sends the judge no arguments and no results', () => {
+    const scorer = pikkuAgentJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+    })
+
+    const prompt = buildJudgePrompt(
+      scorer.judge!,
+      input({
+        toolCalls: [
+          {
+            name: 'lookupCustomer',
+            args: { email: 'ada@example.com' },
+            result: { address: '12 Mill Lane' },
+          },
+        ],
+      })
+    )
+
+    assert.doesNotMatch(prompt, /ada@example\.com/)
+    assert.doesNotMatch(prompt, /Mill Lane/)
+  })
+
+  test('a run that called nothing gets no tool section rather than an empty one', () => {
+    const scorer = pikkuAgentJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+    })
+
+    assert.doesNotMatch(buildJudgePrompt(scorer.judge!, input()), /Tools the/)
+  })
+
+  test('a failed call is shown as failed, which is not the same as no call', () => {
+    const scorer = pikkuAgentJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+    })
+
+    const prompt = buildJudgePrompt(
+      scorer.judge!,
+      input({
+        toolCalls: [
+          {
+            name: 'listTodos',
+            args: {},
+            error: 'no such user ada@example.com',
+          },
+        ],
+      })
+    )
+
+    assert.match(prompt, /- listTodos \(failed\)/)
+    assert.doesNotMatch(prompt, /ada@example\.com/)
+  })
+
+  test("'full' shows the arguments and results, for a judge that grades against them", () => {
+    const scorer = pikkuAgentJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+      toolCalls: 'full',
+    })
+
+    const prompt = buildJudgePrompt(
+      scorer.judge!,
+      input({
+        toolCalls: [
+          { name: 'listTodos', args: {}, result: { todos: ['Buy groceries'] } },
+          { name: 'save', args: {}, error: 'store unreachable' },
+        ],
+      })
+    )
+
+    assert.match(
+      prompt,
+      /listTodos\(\{\}\) returned \{"todos":\["Buy groceries"\]\}/
+    )
+    assert.match(prompt, /save\(\{\}\) failed: store unreachable/)
+  })
+
+  test("'off' withholds the trajectory entirely, even from a run that used tools", () => {
+    const scorer = pikkuAgentJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+      toolCalls: 'off',
+    })
+
+    const prompt = buildJudgePrompt(
+      scorer.judge!,
+      input({ toolCalls: [{ name: 'listTodos', args: {}, result: {} }] })
+    )
+
+    assert.doesNotMatch(prompt, /Tools the/)
+    assert.doesNotMatch(prompt, /listTodos/)
+  })
+
+  test('a large tool result is truncated rather than crowding out the answer', () => {
+    const scorer = pikkuAgentJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+      toolCalls: 'full',
+    })
+
+    const prompt = buildJudgePrompt(
+      scorer.judge!,
+      input({
+        toolCalls: [{ name: 'search', args: {}, result: 'x'.repeat(5000) }],
+      })
+    )
+
+    assert.match(prompt, /… \(truncated\)/)
+    assert.ok(prompt.length < 1500, `prompt was ${prompt.length} characters`)
+  })
+
+  test('a scorer that supplies its own prompt replaces the framing entirely', () => {
+    const scorer = pikkuAgentJudge({
+      name: 'custom',
+      description: 'Custom framing',
+      model: 'claude-opus-5',
+      goal: 'ignored',
+      prompt: (run) => `Only this: ${run.output}`,
+    })
+
+    assert.equal(buildJudgePrompt(scorer.judge!, input()), 'Only this: Paris')
+  })
+})
+
+describe('runJudge', () => {
+  const helpfulness = pikkuAgentJudge({
+    name: 'helpfulness',
+    description: 'Is the answer useful',
+    model: 'claude-opus-5',
+    goal: 'Grade helpfulness.',
+  })
+
+  test('records the score, the reason and what the judgement cost', async () => {
+    const result = await runJudge(
+      helpfulness,
+      input(),
+      runner({ score: 0.8, reason: 'Direct and correct.' })
+    )
+
+    assert.equal(result.score, 0.8)
+    assert.equal(result.reason, 'Direct and correct.')
+    assert.deepEqual(result.metadata, {
+      judgeModel: 'claude-opus-5',
+      judgeTokens: 120,
+    })
+  })
+
+  test('clamps an out-of-range score rather than losing the judgement', async () => {
+    const high = await runJudge(helpfulness, input(), runner({ score: 1.4 }))
+    const low = await runJudge(helpfulness, input(), runner({ score: -2 }))
+
+    assert.equal(high.score, 1)
+    assert.equal(low.score, 0)
+  })
+
+  test('rejects a non-numeric score, which is not a judgement at all', async () => {
+    await assert.rejects(
+      () => runJudge(helpfulness, input(), runner({ score: 'great' })),
+      /non-numeric score/
+    )
+  })
+
+  test('gives the judge no tools, so its structured output is actually honoured', async () => {
+    const seen = { params: undefined as any }
+    await runJudge(helpfulness, input(), runner({ score: 1 }, seen))
+
+    assert.deepEqual(seen.params.tools, [])
+    assert.equal(seen.params.toolChoice, 'none')
+    assert.equal(seen.params.maxSteps, 1)
+    assert.equal(seen.params.model, 'claude-opus-5')
+  })
+
+  test('names the missing provider rather than failing as an undefined call', async () => {
+    await assert.rejects(
+      () => runJudge(helpfulness, input(), undefined),
+      /needs an AI provider/
+    )
+  })
+
+  test('refuses to grade a reference-based judge with no answer key', async () => {
+    const correctness = pikkuAgentJudge({
+      name: 'correctness',
+      description: 'Is the answer right',
+      model: 'claude-opus-5',
+      goal: 'Grade correctness.',
+      requiresReference: true,
+    })
+
+    await assert.rejects(
+      () => runJudge(correctness, input(), runner({ score: 1 })),
+      /grades against a reference answer/
+    )
+  })
+})

package/src/wirings/agent-scorer/agent-scorer-judge.ts

@@ -0,0 +1,180 @@
+import type { AgentRunnerService } from '../../services/agent-runner-service.js'
+import type { AgentMessage } from '../agent/agent.types.js'
+import { randomUUID } from '../agent/agent-utils.js'
+import type {
+  PikkuAgentScorer,
+  ScorerInput,
+  ScorerJudgeConfig,
+  ScorerOutput,
+} from './agent-scorer.types.js'
+
+const JUDGE_OUTPUT_SCHEMA = {
+  type: 'object',
+  properties: {
+    score: {
+      type: 'number',
+      minimum: 0,
+      maximum: 1,
+      description: 'How well the answer meets the rubric. 1 is best.',
+    },
+    reason: {
+      type: 'string',
+      description: 'One or two sentences explaining the score.',
+    },
+  },
+  required: ['score', 'reason'],
+  additionalProperties: false,
+}
+
+/**
+ * The default disclosure: which tools ran and which failed, and nothing about
+ * what went in or came back. A judge that can see a run reached `listTodos`
+ * can tell a retrieved list from an invented one, which is the whole of what
+ * the trajectory is needed for — the rows themselves it does not need, and an
+ * error string is a tool's own words about the caller's data, so it is
+ * summarised rather than quoted.
+ */
+const describeToolCallName = (call: ScorerInput['toolCalls'][number]): string =>
+  `- ${call.name}${call.error ? ' (failed)' : ''}`
+
+/** Everything about a call, for a judge that grades against what came back. */
+const describeToolCall = (call: ScorerInput['toolCalls'][number]): string => {
+  const outcome = call.error
+    ? `failed: ${call.error}`
+    : `returned ${formatToolValue(call.result)}`
+  return `- ${call.name}(${formatToolValue(call.args)}) ${outcome}`
+}
+
+/**
+ * A tool's arguments and result are arbitrary JSON, and some of it is large —
+ * a listing, a document, a page of search results. Truncating keeps a single
+ * fat call from crowding the answer out of the judge's context; what survives
+ * is the head, which is where the shape of a value is.
+ */
+const TOOL_VALUE_LIMIT = 500
+
+const formatToolValue = (value: unknown): string => {
+  if (value === undefined) {
+    return 'nothing'
+  }
+  const text = typeof value === 'string' ? value : JSON.stringify(value)
+  if (text === undefined) {
+    return 'nothing'
+  }
+  return text.length > TOOL_VALUE_LIMIT
+    ? `${text.slice(0, TOOL_VALUE_LIMIT)}… (truncated)`
+    : text
+}
+
+/**
+ * The message a judge grades when its scorer supplies no `prompt`.
+ *
+ * A reference-based judge is shown the answer key; a reference-free one is not,
+ * and grades the answer on its own merits.
+ *
+ * The tool calls are shown because without them a judge cannot tell a fact from
+ * a fabrication. Grading a todo agent that had just listed the user's todos
+ * correctly, a judge given only the answer called it "a plausible-looking list"
+ * that "offers no real access to your actual list, making it effectively a
+ * guess", and scored 0.2 what it scored 1 on other runs — right by its rubric,
+ * on the evidence it was given, and wrong about the run. An agent that answers
+ * from a tool and one that invents the same sentence are indistinguishable in
+ * the output alone; they differ only in what the run did to get there.
+ *
+ * How much of that to show is `judge.toolCalls`, and the default stops at the
+ * names: the 0.2 was the judge doubting the run had any access, and a name
+ * settles that without handing a third-party model the rows.
+ */
+export const buildJudgePrompt = (
+  judge: ScorerJudgeConfig,
+  input: ScorerInput
+): string => {
+  if (judge.prompt) return judge.prompt(input)
+
+  const sections = [`User asked:\n${input.input}`]
+  if (input.reference !== undefined) {
+    sections.push(`Reference answer:\n${input.reference}`)
+  }
+  sections.push(`Assistant answered:\n${input.output}`)
+  if (judge.toolCalls !== 'off' && input.toolCalls.length > 0) {
+    const describe =
+      judge.toolCalls === 'full' ? describeToolCall : describeToolCallName
+    sections.push(
+      `Tools the assistant ran:\n${input.toolCalls.map(describe).join('\n')}`
+    )
+  }
+  sections.push(judge.goal)
+  sections.push('Respond with a score 0..1 and a brief reason.')
+  return sections.join('\n\n')
+}
+
+/**
+ * Clamp rather than reject: a model that answers 1.2 has still made a legible
+ * judgement, and failing the job would lose it. A non-number has not.
+ */
+const normalizeScore = (scorerName: string, value: unknown): number => {
+  if (typeof value !== 'number' || Number.isNaN(value)) {
+    throw new Error(
+      `Judge '${scorerName}' returned a non-numeric score: ${JSON.stringify(value)}`
+    )
+  }
+  return Math.min(1, Math.max(0, value))
+}
+
+/**
+ * Run an LLM judge and force the `{ score, reason }` contract, so a scorer
+ * author writes a rubric rather than a parser.
+ *
+ * A judge has no tools, which is the one case where an output schema is
+ * reliably honoured.
+ */
+export const runJudge = async (
+  scorer: PikkuAgentScorer<any>,
+  input: ScorerInput,
+  agentRunner: AgentRunnerService | undefined
+): Promise<ScorerOutput> => {
+  const judge = scorer.judge
+  if (!judge) {
+    throw new Error(`Scorer '${scorer.name}' is not a judge`)
+  }
+  if (!agentRunner) {
+    throw new Error(
+      `Judge '${scorer.name}' needs an AI provider, but no agentRunner is registered in this process. ` +
+        `A worker deployed apart from the API has to register one to run the slow scoring lane.`
+    )
+  }
+  if (scorer.requiresReference && input.reference === undefined) {
+    throw new Error(
+      `Judge '${scorer.name}' grades against a reference answer, but none was supplied`
+    )
+  }
+
+  const messages: AgentMessage[] = [
+    {
+      id: randomUUID(),
+      role: 'user',
+      content: buildJudgePrompt(judge, input),
+      createdAt: new Date(),
+    },
+  ]
+
+  const result = await agentRunner.run({
+    model: judge.model,
+    instructions: judge.goal,
+    messages,
+    tools: [],
+    maxSteps: 1,
+    toolChoice: 'none',
+    outputSchema: JUDGE_OUTPUT_SCHEMA,
+  })
+
+  const object = (result.object ?? {}) as { score?: unknown; reason?: unknown }
+  return {
+    score: normalizeScore(scorer.name, object.score),
+    ...(typeof object.reason === 'string' ? { reason: object.reason } : {}),
+    metadata: {
+      judgeModel: judge.model,
+      judgeTokens: result.usage.inputTokens + result.usage.outputTokens,
+    },
+  }
+}

package/src/wirings/{ai-scorer/ai-scorer-live.test.ts → agent-scorer/agent-scorer-live.test.ts}

@@ -1,11 +1,11 @@
 import { beforeEach, describe, test } from 'node:test'
 import assert from 'node:assert/strict'
 import { pikkuState, resetPikkuState } from '../../pikku-state.js'
-import { scoreFinishedRun } from './ai-scorer-live.js'
-import { pikkuAIJudge, pikkuAIScorer } from './ai-scorer.js'
-import type { PikkuAIScorer, ScorerInput } from './ai-scorer.types.js'
+import { scoreFinishedRun } from './agent-scorer-live.js'
+import { pikkuAgentJudge, pikkuAgentScorer } from './agent-scorer.js'
+import type { PikkuAgentScorer, ScorerInput } from './agent-scorer.types.js'
 
-const register = (agentName: string, scorers: PikkuAIScorer[]) => {
+const register = (agentName: string, scorers: PikkuAgentScorer[]) => {
   pikkuState(null, 'agent', 'agents').set(agentName, {
     name: agentName,
     scorers: scorers.map((scorer) => scorer.name),
@@ -49,7 +49,7 @@ const collectingServices = () => {
   }
 }
 
-const alwaysOne = pikkuAIScorer({
+const alwaysOne = pikkuAgentScorer({
   name: 'brevity',
   description: 'Shorter is better',
   score: () => ({ score: 1 }),
@@ -61,7 +61,7 @@ describe('scoreFinishedRun', () => {
   })
 
   test('sends one message per scorer, each to its own lane', async () => {
-    const judge = pikkuAIJudge({
+    const judge = pikkuAgentJudge({
       name: 'helpfulness',
       description: 'Is the answer useful',
       model: 'claude-opus-5',
@@ -75,14 +75,14 @@ describe('scoreFinishedRun', () => {
     assert.deepEqual(
       added.map((entry) => [entry.queueName, entry.data.scorerName]),
       [
-        ['ai-score-fast', 'brevity'],
-        ['ai-score-slow', 'helpfulness'],
+        ['agent-score-fast', 'brevity'],
+        ['agent-score-slow', 'helpfulness'],
       ]
     )
   })
 
   test('never grades a reference-based judge, because live traffic has no answer key', async () => {
-    const correctness = pikkuAIJudge({
+    const correctness = pikkuAgentJudge({
       name: 'correctness',
       description: 'Is the answer right',
       model: 'claude-opus-5',
@@ -98,7 +98,7 @@ describe('scoreFinishedRun', () => {
   })
 
   test('honours the sample rate rather than grading every run', async () => {
-    const sampled = pikkuAIScorer({
+    const sampled = pikkuAgentScorer({
       name: 'brevity',
       description: 'Shorter is better',
       sampleRate: 0,
@@ -138,7 +138,7 @@ describe('scoreFinishedRun', () => {
   })
 
   test('one scorer that cannot be enqueued does not stop the others', async () => {
-    const judge = pikkuAIJudge({
+    const judge = pikkuAgentJudge({
       name: 'helpfulness',
       description: 'Is the answer useful',
       model: 'claude-opus-5',

package/src/wirings/{ai-scorer/ai-scorer-live.ts → agent-scorer/agent-scorer-live.ts}

@@ -1,7 +1,7 @@
 import type { CoreSingletonServices } from '../../types/core.types.js'
-import { scorersForAgent } from './ai-scorer-registry.js'
-import { isSampled } from './ai-scorer-sampling.js'
-import { SCORER_LANE_QUEUES, type ScorerInput } from './ai-scorer.types.js'
+import { scorersForAgent } from './agent-scorer-registry.js'
+import { isSampled } from './agent-scorer-sampling.js'
+import { SCORER_LANE_QUEUES, type ScorerInput } from './agent-scorer.types.js'
 
 /**
  * Grade one finished run on live traffic.

package/src/wirings/{ai-scorer/ai-scorer-registry.ts → agent-scorer/agent-scorer-registry.ts}

@@ -1,9 +1,9 @@
 import { pikkuState } from '../../pikku-state.js'
-import type { PikkuAIScorer } from './ai-scorer.types.js'
+import type { PikkuAgentScorer } from './agent-scorer.types.js'
 
-export const addAIScorer = (
+export const addAgentScorer = (
   scorerName: string,
-  scorer: PikkuAIScorer<any>,
+  scorer: PikkuAgentScorer<any>,
   packageName: string | null = null
 ) => {
   const scorersMeta = pikkuState(packageName, 'agent', 'scorersMeta')
@@ -20,15 +20,18 @@ export const addAIScorer = (
   scorers.set(scorerName, scorer)
 }
 
-export const getAIScorers = () => pikkuState(null, 'agent', 'scorers')
+export const getAgentScorers = () => pikkuState(null, 'agent', 'scorers')
 
-export const getAIScorersMeta = () => pikkuState(null, 'agent', 'scorersMeta')
+export const getAgentScorersMeta = () =>
+  pikkuState(null, 'agent', 'scorersMeta')
 
 /**
  * Resolve a scorer by name across every registered package, the way an agent's
  * tools are resolved: a scorer declared in an addon is nameable by an app agent.
  */
-export const resolveAIScorer = (scorerName: string): PikkuAIScorer<any> => {
+export const resolveAgentScorer = (
+  scorerName: string
+): PikkuAgentScorer<any> => {
   const scorer = pikkuState(null, 'agent', 'scorers').get(scorerName)
   if (!scorer) {
     throw new Error(`AI scorer not found: ${scorerName}`)
@@ -45,10 +48,10 @@ export const resolveAIScorer = (scorerName: string): PikkuAIScorer<any> => {
 export const scorersForAgent = (
   agentName: string,
   logger?: { warn: (message: string) => void }
-): PikkuAIScorer<any>[] => {
+): PikkuAgentScorer<any>[] => {
   const agent = pikkuState(null, 'agent', 'agents').get(agentName)
   const names = agent?.scorers ?? []
-  const scorers: PikkuAIScorer<any>[] = []
+  const scorers: PikkuAgentScorer<any>[] = []
   for (const name of names) {
     const scorer = pikkuState(null, 'agent', 'scorers').get(name)
     if (!scorer) {

package/src/wirings/{ai-scorer/ai-scorer-sampling.test.ts → agent-scorer/agent-scorer-sampling.test.ts}

@@ -1,6 +1,6 @@
 import { describe, test } from 'node:test'
 import assert from 'node:assert/strict'
-import { isSampled } from './ai-scorer-sampling.js'
+import { isSampled } from './agent-scorer-sampling.js'
 
 describe('isSampled', () => {
   test('grades every run at a rate of 1 and none at 0', () => {

package/src/wirings/{ai-scorer/ai-scorer-snapshots.test.ts → agent-scorer/agent-scorer-snapshots.test.ts}

@@ -6,8 +6,8 @@ import {
   getScoreSnapshot,
   recordScoreSnapshot,
   resetScoreSnapshots,
-} from './ai-scorer-snapshots.js'
-import type { ScorerInput } from './ai-scorer.types.js'
+} from './agent-scorer-snapshots.js'
+import type { ScorerInput } from './agent-scorer.types.js'
 
 const run = (runId: string): ScorerInput => ({
   runId,

package/src/wirings/{ai-scorer/ai-scorer-snapshots.ts → agent-scorer/agent-scorer-snapshots.ts}

@@ -1,4 +1,4 @@
-import type { ScorerInput } from './ai-scorer.types.js'
+import type { ScorerInput } from './agent-scorer.types.js'
 
 /**
  * The last few finished runs, held so a scenario can grade one.