@pikku/core 0.12.107 → 0.12.112

package/src/wirings/agent-scorer/agent-scorer-judge.test.ts

@@ -1,285 +0,0 @@
-import { describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-import { buildJudgePrompt, runJudge } from './agent-scorer-judge.js'
-import { pikkuAgentJudge } from './agent-scorer.js'
-import type { ScorerInput } from './agent-scorer.types.js'
-import type { AgentRunnerService } from '../../services/agent-runner-service.js'
-
-const input = (overrides: Partial<ScorerInput> = {}): ScorerInput => ({
-  runId: 'run-1',
-  agentName: 'assistant',
-  input: 'what is the capital of France?',
-  output: 'Paris',
-  toolCalls: [],
-  usage: { inputTokens: 10, outputTokens: 5 },
-  ...overrides,
-})
-
-const runner = (object: unknown, seen?: { params: any }): AgentRunnerService =>
-  ({
-    run: async (params: any) => {
-      if (seen) seen.params = params
-      return { object, usage: { inputTokens: 100, outputTokens: 20 } }
-    },
-  }) as unknown as AgentRunnerService
-
-describe('buildJudgePrompt', () => {
-  test('shows the answer key to a reference-based judge', () => {
-    const scorer = pikkuAgentJudge({
-      name: 'correctness',
-      description: 'Is the answer right',
-      model: 'claude-opus-5',
-      goal: 'Grade correctness.',
-      requiresReference: true,
-    })
-
-    const prompt = buildJudgePrompt(
-      scorer.judge!,
-      input({ reference: 'Paris' })
-    )
-
-    assert.match(prompt, /Reference answer:\nParis/)
-  })
-
-  test('withholds a reference section when there is no answer key', () => {
-    const scorer = pikkuAgentJudge({
-      name: 'helpfulness',
-      description: 'Is the answer useful',
-      model: 'claude-opus-5',
-      goal: 'Grade helpfulness.',
-    })
-
-    const prompt = buildJudgePrompt(scorer.judge!, input())
-
-    assert.doesNotMatch(prompt, /Reference answer/)
-    assert.match(prompt, /Grade helpfulness\./)
-  })
-
-  test('shows which tools ran, so an answer is distinguishable from a guess', () => {
-    const scorer = pikkuAgentJudge({
-      name: 'helpfulness',
-      description: 'Is the answer useful',
-      model: 'claude-opus-5',
-      goal: 'Grade helpfulness.',
-    })
-
-    const prompt = buildJudgePrompt(
-      scorer.judge!,
-      input({
-        toolCalls: [
-          { name: 'listTodos', args: {}, result: { todos: ['Buy groceries'] } },
-        ],
-      })
-    )
-
-    assert.match(prompt, /Tools the assistant ran:\n- listTodos/)
-  })
-
-  test('the default disclosure sends the judge no arguments and no results', () => {
-    const scorer = pikkuAgentJudge({
-      name: 'helpfulness',
-      description: 'Is the answer useful',
-      model: 'claude-opus-5',
-      goal: 'Grade helpfulness.',
-    })
-
-    const prompt = buildJudgePrompt(
-      scorer.judge!,
-      input({
-        toolCalls: [
-          {
-            name: 'lookupCustomer',
-            args: { email: 'ada@example.com' },
-            result: { address: '12 Mill Lane' },
-          },
-        ],
-      })
-    )
-
-    assert.doesNotMatch(prompt, /ada@example\.com/)
-    assert.doesNotMatch(prompt, /Mill Lane/)
-  })
-
-  test('a run that called nothing gets no tool section rather than an empty one', () => {
-    const scorer = pikkuAgentJudge({
-      name: 'helpfulness',
-      description: 'Is the answer useful',
-      model: 'claude-opus-5',
-      goal: 'Grade helpfulness.',
-    })
-
-    assert.doesNotMatch(buildJudgePrompt(scorer.judge!, input()), /Tools the/)
-  })
-
-  test('a failed call is shown as failed, which is not the same as no call', () => {
-    const scorer = pikkuAgentJudge({
-      name: 'helpfulness',
-      description: 'Is the answer useful',
-      model: 'claude-opus-5',
-      goal: 'Grade helpfulness.',
-    })
-
-    const prompt = buildJudgePrompt(
-      scorer.judge!,
-      input({
-        toolCalls: [
-          {
-            name: 'listTodos',
-            args: {},
-            error: 'no such user ada@example.com',
-          },
-        ],
-      })
-    )
-
-    assert.match(prompt, /- listTodos \(failed\)/)
-    assert.doesNotMatch(prompt, /ada@example\.com/)
-  })
-
-  test("'full' shows the arguments and results, for a judge that grades against them", () => {
-    const scorer = pikkuAgentJudge({
-      name: 'helpfulness',
-      description: 'Is the answer useful',
-      model: 'claude-opus-5',
-      goal: 'Grade helpfulness.',
-      toolCalls: 'full',
-    })
-
-    const prompt = buildJudgePrompt(
-      scorer.judge!,
-      input({
-        toolCalls: [
-          { name: 'listTodos', args: {}, result: { todos: ['Buy groceries'] } },
-          { name: 'save', args: {}, error: 'store unreachable' },
-        ],
-      })
-    )
-
-    assert.match(
-      prompt,
-      /listTodos\(\{\}\) returned \{"todos":\["Buy groceries"\]\}/
-    )
-    assert.match(prompt, /save\(\{\}\) failed: store unreachable/)
-  })
-
-  test("'off' withholds the trajectory entirely, even from a run that used tools", () => {
-    const scorer = pikkuAgentJudge({
-      name: 'helpfulness',
-      description: 'Is the answer useful',
-      model: 'claude-opus-5',
-      goal: 'Grade helpfulness.',
-      toolCalls: 'off',
-    })
-
-    const prompt = buildJudgePrompt(
-      scorer.judge!,
-      input({ toolCalls: [{ name: 'listTodos', args: {}, result: {} }] })
-    )
-
-    assert.doesNotMatch(prompt, /Tools the/)
-    assert.doesNotMatch(prompt, /listTodos/)
-  })
-
-  test('a large tool result is truncated rather than crowding out the answer', () => {
-    const scorer = pikkuAgentJudge({
-      name: 'helpfulness',
-      description: 'Is the answer useful',
-      model: 'claude-opus-5',
-      goal: 'Grade helpfulness.',
-      toolCalls: 'full',
-    })
-
-    const prompt = buildJudgePrompt(
-      scorer.judge!,
-      input({
-        toolCalls: [{ name: 'search', args: {}, result: 'x'.repeat(5000) }],
-      })
-    )
-
-    assert.match(prompt, /… \(truncated\)/)
-    assert.ok(prompt.length < 1500, `prompt was ${prompt.length} characters`)
-  })
-
-  test('a scorer that supplies its own prompt replaces the framing entirely', () => {
-    const scorer = pikkuAgentJudge({
-      name: 'custom',
-      description: 'Custom framing',
-      model: 'claude-opus-5',
-      goal: 'ignored',
-      prompt: (run) => `Only this: ${run.output}`,
-    })
-
-    assert.equal(buildJudgePrompt(scorer.judge!, input()), 'Only this: Paris')
-  })
-})
-
-describe('runJudge', () => {
-  const helpfulness = pikkuAgentJudge({
-    name: 'helpfulness',
-    description: 'Is the answer useful',
-    model: 'claude-opus-5',
-    goal: 'Grade helpfulness.',
-  })
-
-  test('records the score, the reason and what the judgement cost', async () => {
-    const result = await runJudge(
-      helpfulness,
-      input(),
-      runner({ score: 0.8, reason: 'Direct and correct.' })
-    )
-
-    assert.equal(result.score, 0.8)
-    assert.equal(result.reason, 'Direct and correct.')
-    assert.deepEqual(result.metadata, {
-      judgeModel: 'claude-opus-5',
-      judgeTokens: 120,
-    })
-  })
-
-  test('clamps an out-of-range score rather than losing the judgement', async () => {
-    const high = await runJudge(helpfulness, input(), runner({ score: 1.4 }))
-    const low = await runJudge(helpfulness, input(), runner({ score: -2 }))
-
-    assert.equal(high.score, 1)
-    assert.equal(low.score, 0)
-  })
-
-  test('rejects a non-numeric score, which is not a judgement at all', async () => {
-    await assert.rejects(
-      () => runJudge(helpfulness, input(), runner({ score: 'great' })),
-      /non-numeric score/
-    )
-  })
-
-  test('gives the judge no tools, so its structured output is actually honoured', async () => {
-    const seen = { params: undefined as any }
-    await runJudge(helpfulness, input(), runner({ score: 1 }, seen))
-
-    assert.deepEqual(seen.params.tools, [])
-    assert.equal(seen.params.toolChoice, 'none')
-    assert.equal(seen.params.maxSteps, 1)
-    assert.equal(seen.params.model, 'claude-opus-5')
-  })
-
-  test('names the missing provider rather than failing as an undefined call', async () => {
-    await assert.rejects(
-      () => runJudge(helpfulness, input(), undefined),
-      /needs an AI provider/
-    )
-  })
-
-  test('refuses to grade a reference-based judge with no answer key', async () => {
-    const correctness = pikkuAgentJudge({
-      name: 'correctness',
-      description: 'Is the answer right',
-      model: 'claude-opus-5',
-      goal: 'Grade correctness.',
-      requiresReference: true,
-    })
-
-    await assert.rejects(
-      () => runJudge(correctness, input(), runner({ score: 1 })),
-      /grades against a reference answer/
-    )
-  })
-})

package/src/wirings/agent-scorer/agent-scorer-judge.ts

@@ -1,180 +0,0 @@
-import type { AgentRunnerService } from '../../services/agent-runner-service.js'
-import type { AgentMessage } from '../agent/agent.types.js'
-import { randomUUID } from '../agent/agent-utils.js'
-import type {
-  PikkuAgentScorer,
-  ScorerInput,
-  ScorerJudgeConfig,
-  ScorerOutput,
-} from './agent-scorer.types.js'
-
-const JUDGE_OUTPUT_SCHEMA = {
-  type: 'object',
-  properties: {
-    score: {
-      type: 'number',
-      minimum: 0,
-      maximum: 1,
-      description: 'How well the answer meets the rubric. 1 is best.',
-    },
-    reason: {
-      type: 'string',
-      description: 'One or two sentences explaining the score.',
-    },
-  },
-  required: ['score', 'reason'],
-  additionalProperties: false,
-}
-
-/**
- * The default disclosure: which tools ran and which failed, and nothing about
- * what went in or came back. A judge that can see a run reached `listTodos`
- * can tell a retrieved list from an invented one, which is the whole of what
- * the trajectory is needed for — the rows themselves it does not need, and an
- * error string is a tool's own words about the caller's data, so it is
- * summarised rather than quoted.
- */
-const describeToolCallName = (call: ScorerInput['toolCalls'][number]): string =>
-  `- ${call.name}${call.error ? ' (failed)' : ''}`
-
-/** Everything about a call, for a judge that grades against what came back. */
-const describeToolCall = (call: ScorerInput['toolCalls'][number]): string => {
-  const outcome = call.error
-    ? `failed: ${call.error}`
-    : `returned ${formatToolValue(call.result)}`
-  return `- ${call.name}(${formatToolValue(call.args)}) ${outcome}`
-}
-
-/**
- * A tool's arguments and result are arbitrary JSON, and some of it is large —
- * a listing, a document, a page of search results. Truncating keeps a single
- * fat call from crowding the answer out of the judge's context; what survives
- * is the head, which is where the shape of a value is.
- */
-const TOOL_VALUE_LIMIT = 500
-
-const formatToolValue = (value: unknown): string => {
-  if (value === undefined) {
-    return 'nothing'
-  }
-  const text = typeof value === 'string' ? value : JSON.stringify(value)
-  if (text === undefined) {
-    return 'nothing'
-  }
-  return text.length > TOOL_VALUE_LIMIT
-    ? `${text.slice(0, TOOL_VALUE_LIMIT)}… (truncated)`
-    : text
-}
-
-/**
- * The message a judge grades when its scorer supplies no `prompt`.
- *
- * A reference-based judge is shown the answer key; a reference-free one is not,
- * and grades the answer on its own merits.
- *
- * The tool calls are shown because without them a judge cannot tell a fact from
- * a fabrication. Grading a todo agent that had just listed the user's todos
- * correctly, a judge given only the answer called it "a plausible-looking list"
- * that "offers no real access to your actual list, making it effectively a
- * guess", and scored 0.2 what it scored 1 on other runs — right by its rubric,
- * on the evidence it was given, and wrong about the run. An agent that answers
- * from a tool and one that invents the same sentence are indistinguishable in
- * the output alone; they differ only in what the run did to get there.
- *
- * How much of that to show is `judge.toolCalls`, and the default stops at the
- * names: the 0.2 was the judge doubting the run had any access, and a name
- * settles that without handing a third-party model the rows.
- */
-export const buildJudgePrompt = (
-  judge: ScorerJudgeConfig,
-  input: ScorerInput
-): string => {
-  if (judge.prompt) return judge.prompt(input)
-
-  const sections = [`User asked:\n${input.input}`]
-  if (input.reference !== undefined) {
-    sections.push(`Reference answer:\n${input.reference}`)
-  }
-  sections.push(`Assistant answered:\n${input.output}`)
-  if (judge.toolCalls !== 'off' && input.toolCalls.length > 0) {
-    const describe =
-      judge.toolCalls === 'full' ? describeToolCall : describeToolCallName
-    sections.push(
-      `Tools the assistant ran:\n${input.toolCalls.map(describe).join('\n')}`
-    )
-  }
-  sections.push(judge.goal)
-  sections.push('Respond with a score 0..1 and a brief reason.')
-  return sections.join('\n\n')
-}
-
-/**
- * Clamp rather than reject: a model that answers 1.2 has still made a legible
- * judgement, and failing the job would lose it. A non-number has not.
- */
-const normalizeScore = (scorerName: string, value: unknown): number => {
-  if (typeof value !== 'number' || Number.isNaN(value)) {
-    throw new Error(
-      `Judge '${scorerName}' returned a non-numeric score: ${JSON.stringify(value)}`
-    )
-  }
-  return Math.min(1, Math.max(0, value))
-}
-
-/**
- * Run an LLM judge and force the `{ score, reason }` contract, so a scorer
- * author writes a rubric rather than a parser.
- *
- * A judge has no tools, which is the one case where an output schema is
- * reliably honoured.
- */
-export const runJudge = async (
-  scorer: PikkuAgentScorer<any>,
-  input: ScorerInput,
-  agentRunner: AgentRunnerService | undefined
-): Promise<ScorerOutput> => {
-  const judge = scorer.judge
-  if (!judge) {
-    throw new Error(`Scorer '${scorer.name}' is not a judge`)
-  }
-  if (!agentRunner) {
-    throw new Error(
-      `Judge '${scorer.name}' needs an AI provider, but no agentRunner is registered in this process. ` +
-        `A worker deployed apart from the API has to register one to run the slow scoring lane.`
-    )
-  }
-  if (scorer.requiresReference && input.reference === undefined) {
-    throw new Error(
-      `Judge '${scorer.name}' grades against a reference answer, but none was supplied`
-    )
-  }
-
-  const messages: AgentMessage[] = [
-    {
-      id: randomUUID(),
-      role: 'user',
-      content: buildJudgePrompt(judge, input),
-      createdAt: new Date(),
-    },
-  ]
-
-  const result = await agentRunner.run({
-    model: judge.model,
-    instructions: judge.goal,
-    messages,
-    tools: [],
-    maxSteps: 1,
-    toolChoice: 'none',
-    outputSchema: JUDGE_OUTPUT_SCHEMA,
-  })
-
-  const object = (result.object ?? {}) as { score?: unknown; reason?: unknown }
-  return {
-    score: normalizeScore(scorer.name, object.score),
-    ...(typeof object.reason === 'string' ? { reason: object.reason } : {}),
-    metadata: {
-      judgeModel: judge.model,
-      judgeTokens: result.usage.inputTokens + result.usage.outputTokens,
-    },
-  }
-}

package/src/wirings/agent-scorer/agent-scorer-live.test.ts

@@ -1,174 +0,0 @@
-import { beforeEach, describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-import { pikkuState, resetPikkuState } from '../../pikku-state.js'
-import { scoreFinishedRun } from './agent-scorer-live.js'
-import { pikkuAgentJudge, pikkuAgentScorer } from './agent-scorer.js'
-import type { PikkuAgentScorer, ScorerInput } from './agent-scorer.types.js'
-
-const register = (agentName: string, scorers: PikkuAgentScorer[]) => {
-  pikkuState(null, 'agent', 'agents').set(agentName, {
-    name: agentName,
-    scorers: scorers.map((scorer) => scorer.name),
-  } as never)
-  for (const scorer of scorers) {
-    pikkuState(null, 'agent', 'scorers').set(scorer.name, scorer)
-  }
-}
-
-const run = (overrides: Partial<ScorerInput> = {}): ScorerInput => ({
-  runId: 'run-1',
-  agentName: 'assistant',
-  threadId: 'thread-1',
-  input: 'what is the capital of France?',
-  output: 'Paris',
-  toolCalls: [],
-  usage: { inputTokens: 10, outputTokens: 5 },
-  ...overrides,
-})
-
-const collectingServices = () => {
-  const added: { queueName: string; data: any }[] = []
-  const warnings: string[] = []
-  const errors: string[] = []
-  return {
-    added,
-    warnings,
-    errors,
-    services: {
-      logger: {
-        warn: (message: string) => warnings.push(message),
-        error: (message: string) => errors.push(message),
-      },
-      queueService: {
-        add: async (queueName: string, data: unknown) => {
-          added.push({ queueName, data })
-          return undefined
-        },
-      },
-    } as any,
-  }
-}
-
-const alwaysOne = pikkuAgentScorer({
-  name: 'brevity',
-  description: 'Shorter is better',
-  score: () => ({ score: 1 }),
-})
-
-describe('scoreFinishedRun', () => {
-  beforeEach(() => {
-    resetPikkuState()
-  })
-
-  test('sends one message per scorer, each to its own lane', async () => {
-    const judge = pikkuAgentJudge({
-      name: 'helpfulness',
-      description: 'Is the answer useful',
-      model: 'claude-opus-5',
-      goal: 'Grade helpfulness.',
-    })
-    register('assistant', [alwaysOne, judge])
-    const { added, services } = collectingServices()
-
-    await scoreFinishedRun(run(), services)
-
-    assert.deepEqual(
-      added.map((entry) => [entry.queueName, entry.data.scorerName]),
-      [
-        ['agent-score-fast', 'brevity'],
-        ['agent-score-slow', 'helpfulness'],
-      ]
-    )
-  })
-
-  test('never grades a reference-based judge, because live traffic has no answer key', async () => {
-    const correctness = pikkuAgentJudge({
-      name: 'correctness',
-      description: 'Is the answer right',
-      model: 'claude-opus-5',
-      goal: 'Grade correctness.',
-      requiresReference: true,
-    })
-    register('assistant', [correctness])
-    const { added, services } = collectingServices()
-
-    await scoreFinishedRun(run(), services)
-
-    assert.deepEqual(added, [])
-  })
-
-  test('honours the sample rate rather than grading every run', async () => {
-    const sampled = pikkuAgentScorer({
-      name: 'brevity',
-      description: 'Shorter is better',
-      sampleRate: 0,
-      score: () => ({ score: 1 }),
-    })
-    register('assistant', [sampled])
-    const { added, services } = collectingServices()
-
-    await scoreFinishedRun(run(), services)
-
-    assert.deepEqual(added, [])
-  })
-
-  test('warns about a scorer the agent named but nothing registered, rather than failing the run', async () => {
-    pikkuState(null, 'agent', 'agents').set('assistant', {
-      name: 'assistant',
-      scorers: ['nonexistent'],
-    } as never)
-    const { added, warnings, services } = collectingServices()
-
-    await scoreFinishedRun(run(), services)
-
-    assert.deepEqual(added, [])
-    assert.equal(warnings.length, 1)
-    assert.match(warnings[0]!, /nonexistent/)
-  })
-
-  test('says why nothing was graded when there is no queue to grade on', async () => {
-    register('assistant', [alwaysOne])
-    const { warnings, services } = collectingServices()
-    delete services.queueService
-
-    await scoreFinishedRun(run(), services)
-
-    assert.equal(warnings.length, 1)
-    assert.match(warnings[0]!, /no queue service is registered/)
-  })
-
-  test('one scorer that cannot be enqueued does not stop the others', async () => {
-    const judge = pikkuAgentJudge({
-      name: 'helpfulness',
-      description: 'Is the answer useful',
-      model: 'claude-opus-5',
-      goal: 'Grade helpfulness.',
-    })
-    register('assistant', [alwaysOne, judge])
-    const { added, errors, services } = collectingServices()
-    const add = services.queueService.add
-    services.queueService.add = async (queueName: string, data: any) => {
-      if (data.scorerName === 'brevity') throw new Error('queue is down')
-      return add(queueName, data)
-    }
-
-    await scoreFinishedRun(run(), services)
-
-    assert.deepEqual(
-      added.map((entry) => entry.data.scorerName),
-      ['helpfulness']
-    )
-    assert.equal(errors.length, 1)
-  })
-
-  test('an agent that names no scorers touches the queue at all', async () => {
-    pikkuState(null, 'agent', 'agents').set('assistant', {
-      name: 'assistant',
-    } as never)
-    const { added, services } = collectingServices()
-
-    await scoreFinishedRun(run(), services)
-
-    assert.deepEqual(added, [])
-  })
-})

package/src/wirings/agent-scorer/agent-scorer-live.ts

@@ -1,56 +0,0 @@
-import type { CoreSingletonServices } from '../../types/core.types.js'
-import { scorersForAgent } from './agent-scorer-registry.js'
-import { isSampled } from './agent-scorer-sampling.js'
-import { SCORER_LANE_QUEUES, type ScorerInput } from './agent-scorer.types.js'
-
-/**
- * Grade one finished run on live traffic.
- *
- * Called as the terminal step of `finalizeAgentRun`, so it is structurally last
- * — a developer cannot register anything after it, and cannot reorder it. It is
- * also strictly best-effort: the client already has its answer, so nothing here
- * may fail the run.
- */
-export const scoreFinishedRun = async (
-  run: ScorerInput,
-  services: Pick<CoreSingletonServices, 'logger'> & {
-    queueService?: { add: (queueName: string, data: unknown) => Promise<any> }
-  }
-): Promise<void> => {
-  const scorers = scorersForAgent(run.agentName, services.logger).filter(
-    // A reference-based judge grades against an answer key, and live traffic
-    // has none.
-    (scorer) => !scorer.requiresReference
-  )
-  if (scorers.length === 0) return
-
-  if (!services.queueService) {
-    services.logger?.warn(
-      `[pikku] Agent '${run.agentName}' declares scorers but no queue service is registered — skipping live scoring`
-    )
-    return
-  }
-
-  const sampled = scorers.filter((scorer) =>
-    isSampled(run.runId, scorer.name, scorer.sampleRate)
-  )
-
-  // One message per scorer, so each gets its own retry, isolation and lane.
-  await Promise.all(
-    sampled.map(async (scorer) => {
-      try {
-        await services.queueService!.add(SCORER_LANE_QUEUES[scorer.lane], {
-          ...run,
-          scorerName: scorer.name,
-          // No reference: a live run has no answer key.
-          reference: undefined,
-        })
-      } catch (error) {
-        services.logger?.error(
-          `[pikku] Failed to enqueue the '${scorer.name}' grade of run ${run.runId}`,
-          { error }
-        )
-      }
-    })
-  )
-}