@pikku/core 0.12.80 → 0.12.83

package/src/wirings/ai-scorer/ai-scorer-judge.test.ts

@@ -0,0 +1,143 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { buildJudgePrompt, runJudge } from './ai-scorer-judge.js'
+import { pikkuAIJudge } from './ai-scorer.js'
+import type { ScorerInput } from './ai-scorer.types.js'
+import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
+
+const input = (overrides: Partial<ScorerInput> = {}): ScorerInput => ({
+  runId: 'run-1',
+  agentName: 'assistant',
+  input: 'what is the capital of France?',
+  output: 'Paris',
+  toolCalls: [],
+  usage: { inputTokens: 10, outputTokens: 5 },
+  ...overrides,
+})
+
+const runner = (
+  object: unknown,
+  seen?: { params: any }
+): AIAgentRunnerService =>
+  ({
+    run: async (params: any) => {
+      if (seen) seen.params = params
+      return { object, usage: { inputTokens: 100, outputTokens: 20 } }
+    },
+  }) as unknown as AIAgentRunnerService
+
+describe('buildJudgePrompt', () => {
+  test('shows the answer key to a reference-based judge', () => {
+    const scorer = pikkuAIJudge({
+      name: 'correctness',
+      description: 'Is the answer right',
+      model: 'claude-opus-5',
+      goal: 'Grade correctness.',
+      requiresReference: true,
+    })
+
+    const prompt = buildJudgePrompt(
+      scorer.judge!,
+      input({ reference: 'Paris' })
+    )
+
+    assert.match(prompt, /Reference answer:\nParis/)
+  })
+
+  test('withholds a reference section when there is no answer key', () => {
+    const scorer = pikkuAIJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+    })
+
+    const prompt = buildJudgePrompt(scorer.judge!, input())
+
+    assert.doesNotMatch(prompt, /Reference answer/)
+    assert.match(prompt, /Grade helpfulness\./)
+  })
+
+  test('a scorer that supplies its own prompt replaces the framing entirely', () => {
+    const scorer = pikkuAIJudge({
+      name: 'custom',
+      description: 'Custom framing',
+      model: 'claude-opus-5',
+      goal: 'ignored',
+      prompt: (run) => `Only this: ${run.output}`,
+    })
+
+    assert.equal(buildJudgePrompt(scorer.judge!, input()), 'Only this: Paris')
+  })
+})
+
+describe('runJudge', () => {
+  const helpfulness = pikkuAIJudge({
+    name: 'helpfulness',
+    description: 'Is the answer useful',
+    model: 'claude-opus-5',
+    goal: 'Grade helpfulness.',
+  })
+
+  test('records the score, the reason and what the judgement cost', async () => {
+    const result = await runJudge(
+      helpfulness,
+      input(),
+      runner({ score: 0.8, reason: 'Direct and correct.' })
+    )
+
+    assert.equal(result.score, 0.8)
+    assert.equal(result.reason, 'Direct and correct.')
+    assert.deepEqual(result.metadata, {
+      judgeModel: 'claude-opus-5',
+      judgeTokens: 120,
+    })
+  })
+
+  test('clamps an out-of-range score rather than losing the judgement', async () => {
+    const high = await runJudge(helpfulness, input(), runner({ score: 1.4 }))
+    const low = await runJudge(helpfulness, input(), runner({ score: -2 }))
+
+    assert.equal(high.score, 1)
+    assert.equal(low.score, 0)
+  })
+
+  test('rejects a non-numeric score, which is not a judgement at all', async () => {
+    await assert.rejects(
+      () => runJudge(helpfulness, input(), runner({ score: 'great' })),
+      /non-numeric score/
+    )
+  })
+
+  test('gives the judge no tools, so its structured output is actually honoured', async () => {
+    const seen = { params: undefined as any }
+    await runJudge(helpfulness, input(), runner({ score: 1 }, seen))
+
+    assert.deepEqual(seen.params.tools, [])
+    assert.equal(seen.params.toolChoice, 'none')
+    assert.equal(seen.params.maxSteps, 1)
+    assert.equal(seen.params.model, 'claude-opus-5')
+  })
+
+  test('names the missing provider rather than failing as an undefined call', async () => {
+    await assert.rejects(
+      () => runJudge(helpfulness, input(), undefined),
+      /needs an AI provider/
+    )
+  })
+
+  test('refuses to grade a reference-based judge with no answer key', async () => {
+    const correctness = pikkuAIJudge({
+      name: 'correctness',
+      description: 'Is the answer right',
+      model: 'claude-opus-5',
+      goal: 'Grade correctness.',
+      requiresReference: true,
+    })
+
+    await assert.rejects(
+      () => runJudge(correctness, input(), runner({ score: 1 })),
+      /grades against a reference answer/
+    )
+  })
+})

package/src/wirings/ai-scorer/ai-scorer-judge.ts

@@ -0,0 +1,120 @@
+import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
+import type { AIMessage } from '../ai-agent/ai-agent.types.js'
+import { randomUUID } from '../ai-agent/ai-agent-utils.js'
+import type {
+  PikkuAIScorer,
+  ScorerInput,
+  ScorerJudgeConfig,
+  ScorerOutput,
+} from './ai-scorer.types.js'
+
+const JUDGE_OUTPUT_SCHEMA = {
+  type: 'object',
+  properties: {
+    score: {
+      type: 'number',
+      minimum: 0,
+      maximum: 1,
+      description: 'How well the answer meets the rubric. 1 is best.',
+    },
+    reason: {
+      type: 'string',
+      description: 'One or two sentences explaining the score.',
+    },
+  },
+  required: ['score', 'reason'],
+  additionalProperties: false,
+}
+
+/**
+ * The message a judge grades when its scorer supplies no `prompt`.
+ *
+ * A reference-based judge is shown the answer key; a reference-free one is not,
+ * and grades the answer on its own merits.
+ */
+export const buildJudgePrompt = (
+  judge: ScorerJudgeConfig,
+  input: ScorerInput
+): string => {
+  if (judge.prompt) return judge.prompt(input)
+
+  const sections = [`User asked:\n${input.input}`]
+  if (input.reference !== undefined) {
+    sections.push(`Reference answer:\n${input.reference}`)
+  }
+  sections.push(`Assistant answered:\n${input.output}`)
+  sections.push(judge.goal)
+  sections.push('Respond with a score 0..1 and a brief reason.')
+  return sections.join('\n\n')
+}
+
+/**
+ * Clamp rather than reject: a model that answers 1.2 has still made a legible
+ * judgement, and failing the job would lose it. A non-number has not.
+ */
+const normalizeScore = (scorerName: string, value: unknown): number => {
+  if (typeof value !== 'number' || Number.isNaN(value)) {
+    throw new Error(
+      `Judge '${scorerName}' returned a non-numeric score: ${JSON.stringify(value)}`
+    )
+  }
+  return Math.min(1, Math.max(0, value))
+}
+
+/**
+ * Run an LLM judge and force the `{ score, reason }` contract, so a scorer
+ * author writes a rubric rather than a parser.
+ *
+ * A judge has no tools, which is the one case where an output schema is
+ * reliably honoured.
+ */
+export const runJudge = async (
+  scorer: PikkuAIScorer<any>,
+  input: ScorerInput,
+  aiAgentRunner: AIAgentRunnerService | undefined
+): Promise<ScorerOutput> => {
+  const judge = scorer.judge
+  if (!judge) {
+    throw new Error(`Scorer '${scorer.name}' is not a judge`)
+  }
+  if (!aiAgentRunner) {
+    throw new Error(
+      `Judge '${scorer.name}' needs an AI provider, but no aiAgentRunner is registered in this process. ` +
+        `A worker deployed apart from the API has to register one to run the slow scoring lane.`
+    )
+  }
+  if (scorer.requiresReference && input.reference === undefined) {
+    throw new Error(
+      `Judge '${scorer.name}' grades against a reference answer, but none was supplied`
+    )
+  }
+
+  const messages: AIMessage[] = [
+    {
+      id: randomUUID(),
+      role: 'user',
+      content: buildJudgePrompt(judge, input),
+      createdAt: new Date(),
+    },
+  ]
+
+  const result = await aiAgentRunner.run({
+    model: judge.model,
+    instructions: judge.goal,
+    messages,
+    tools: [],
+    maxSteps: 1,
+    toolChoice: 'none',
+    outputSchema: JUDGE_OUTPUT_SCHEMA,
+  })
+
+  const object = (result.object ?? {}) as { score?: unknown; reason?: unknown }
+  return {
+    score: normalizeScore(scorer.name, object.score),
+    ...(typeof object.reason === 'string' ? { reason: object.reason } : {}),
+    metadata: {
+      judgeModel: judge.model,
+      judgeTokens: result.usage.inputTokens + result.usage.outputTokens,
+    },
+  }
+}

package/src/wirings/ai-scorer/ai-scorer-live.test.ts

@@ -0,0 +1,174 @@
+import { beforeEach, describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { pikkuState, resetPikkuState } from '../../pikku-state.js'
+import { scoreFinishedRun } from './ai-scorer-live.js'
+import { pikkuAIJudge, pikkuAIScorer } from './ai-scorer.js'
+import type { PikkuAIScorer, ScorerInput } from './ai-scorer.types.js'
+
+const register = (agentName: string, scorers: PikkuAIScorer[]) => {
+  pikkuState(null, 'agent', 'agents').set(agentName, {
+    name: agentName,
+    scorers: scorers.map((scorer) => scorer.name),
+  } as never)
+  for (const scorer of scorers) {
+    pikkuState(null, 'agent', 'scorers').set(scorer.name, scorer)
+  }
+}
+
+const run = (overrides: Partial<ScorerInput> = {}): ScorerInput => ({
+  runId: 'run-1',
+  agentName: 'assistant',
+  threadId: 'thread-1',
+  input: 'what is the capital of France?',
+  output: 'Paris',
+  toolCalls: [],
+  usage: { inputTokens: 10, outputTokens: 5 },
+  ...overrides,
+})
+
+const collectingServices = () => {
+  const added: { queueName: string; data: any }[] = []
+  const warnings: string[] = []
+  const errors: string[] = []
+  return {
+    added,
+    warnings,
+    errors,
+    services: {
+      logger: {
+        warn: (message: string) => warnings.push(message),
+        error: (message: string) => errors.push(message),
+      },
+      queueService: {
+        add: async (queueName: string, data: unknown) => {
+          added.push({ queueName, data })
+          return undefined
+        },
+      },
+    } as any,
+  }
+}
+
+const alwaysOne = pikkuAIScorer({
+  name: 'brevity',
+  description: 'Shorter is better',
+  score: () => ({ score: 1 }),
+})
+
+describe('scoreFinishedRun', () => {
+  beforeEach(() => {
+    resetPikkuState()
+  })
+
+  test('sends one message per scorer, each to its own lane', async () => {
+    const judge = pikkuAIJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+    })
+    register('assistant', [alwaysOne, judge])
+    const { added, services } = collectingServices()
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(
+      added.map((entry) => [entry.queueName, entry.data.scorerName]),
+      [
+        ['ai-score-fast', 'brevity'],
+        ['ai-score-slow', 'helpfulness'],
+      ]
+    )
+  })
+
+  test('never grades a reference-based judge, because live traffic has no answer key', async () => {
+    const correctness = pikkuAIJudge({
+      name: 'correctness',
+      description: 'Is the answer right',
+      model: 'claude-opus-5',
+      goal: 'Grade correctness.',
+      requiresReference: true,
+    })
+    register('assistant', [correctness])
+    const { added, services } = collectingServices()
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(added, [])
+  })
+
+  test('honours the sample rate rather than grading every run', async () => {
+    const sampled = pikkuAIScorer({
+      name: 'brevity',
+      description: 'Shorter is better',
+      sampleRate: 0,
+      score: () => ({ score: 1 }),
+    })
+    register('assistant', [sampled])
+    const { added, services } = collectingServices()
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(added, [])
+  })
+
+  test('warns about a scorer the agent named but nothing registered, rather than failing the run', async () => {
+    pikkuState(null, 'agent', 'agents').set('assistant', {
+      name: 'assistant',
+      scorers: ['nonexistent'],
+    } as never)
+    const { added, warnings, services } = collectingServices()
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(added, [])
+    assert.equal(warnings.length, 1)
+    assert.match(warnings[0]!, /nonexistent/)
+  })
+
+  test('says why nothing was graded when there is no queue to grade on', async () => {
+    register('assistant', [alwaysOne])
+    const { warnings, services } = collectingServices()
+    delete services.queueService
+
+    await scoreFinishedRun(run(), services)
+
+    assert.equal(warnings.length, 1)
+    assert.match(warnings[0]!, /no queue service is registered/)
+  })
+
+  test('one scorer that cannot be enqueued does not stop the others', async () => {
+    const judge = pikkuAIJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+    })
+    register('assistant', [alwaysOne, judge])
+    const { added, errors, services } = collectingServices()
+    const add = services.queueService.add
+    services.queueService.add = async (queueName: string, data: any) => {
+      if (data.scorerName === 'brevity') throw new Error('queue is down')
+      return add(queueName, data)
+    }
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(
+      added.map((entry) => entry.data.scorerName),
+      ['helpfulness']
+    )
+    assert.equal(errors.length, 1)
+  })
+
+  test('an agent that names no scorers touches the queue at all', async () => {
+    pikkuState(null, 'agent', 'agents').set('assistant', {
+      name: 'assistant',
+    } as never)
+    const { added, services } = collectingServices()
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(added, [])
+  })
+})

package/src/wirings/ai-scorer/ai-scorer-live.ts

@@ -0,0 +1,56 @@
+import type { CoreSingletonServices } from '../../types/core.types.js'
+import { scorersForAgent } from './ai-scorer-registry.js'
+import { isSampled } from './ai-scorer-sampling.js'
+import { SCORER_LANE_QUEUES, type ScorerInput } from './ai-scorer.types.js'
+
+/**
+ * Grade one finished run on live traffic.
+ *
+ * Called as the terminal step of `finalizeAgentRun`, so it is structurally last
+ * — a developer cannot register anything after it, and cannot reorder it. It is
+ * also strictly best-effort: the client already has its answer, so nothing here
+ * may fail the run.
+ */
+export const scoreFinishedRun = async (
+  run: ScorerInput,
+  services: Pick<CoreSingletonServices, 'logger'> & {
+    queueService?: { add: (queueName: string, data: unknown) => Promise<any> }
+  }
+): Promise<void> => {
+  const scorers = scorersForAgent(run.agentName, services.logger).filter(
+    // A reference-based judge grades against an answer key, and live traffic
+    // has none.
+    (scorer) => !scorer.requiresReference
+  )
+  if (scorers.length === 0) return
+
+  if (!services.queueService) {
+    services.logger?.warn(
+      `[pikku] Agent '${run.agentName}' declares scorers but no queue service is registered — skipping live scoring`
+    )
+    return
+  }
+
+  const sampled = scorers.filter((scorer) =>
+    isSampled(run.runId, scorer.name, scorer.sampleRate)
+  )
+
+  // One message per scorer, so each gets its own retry, isolation and lane.
+  await Promise.all(
+    sampled.map(async (scorer) => {
+      try {
+        await services.queueService!.add(SCORER_LANE_QUEUES[scorer.lane], {
+          ...run,
+          scorerName: scorer.name,
+          // No reference: a live run has no answer key.
+          reference: undefined,
+        })
+      } catch (error) {
+        services.logger?.error(
+          `[pikku] Failed to enqueue the '${scorer.name}' grade of run ${run.runId}`,
+          { error }
+        )
+      }
+    })
+  )
+}

package/src/wirings/ai-scorer/ai-scorer-registry.ts

@@ -0,0 +1,63 @@
+import { pikkuState } from '../../pikku-state.js'
+import type { PikkuAIScorer } from './ai-scorer.types.js'
+
+export const addAIScorer = (
+  scorerName: string,
+  scorer: PikkuAIScorer<any>,
+  packageName: string | null = null
+) => {
+  const scorersMeta = pikkuState(packageName, 'agent', 'scorersMeta')
+  if (!scorersMeta[scorerName]) {
+    console.warn(
+      `[pikku] Skipping AI scorer '${scorerName}' — metadata not found. Scorers must be declared in a *.scorer.ts file.`
+    )
+    return
+  }
+  const scorers = pikkuState(packageName, 'agent', 'scorers')
+  if (scorers.has(scorerName)) {
+    throw new Error(`AI scorer already exists: ${scorerName}`)
+  }
+  scorers.set(scorerName, scorer)
+}
+
+export const getAIScorers = () => pikkuState(null, 'agent', 'scorers')
+
+export const getAIScorersMeta = () => pikkuState(null, 'agent', 'scorersMeta')
+
+/**
+ * Resolve a scorer by name across every registered package, the way an agent's
+ * tools are resolved: a scorer declared in an addon is nameable by an app agent.
+ */
+export const resolveAIScorer = (scorerName: string): PikkuAIScorer<any> => {
+  const scorer = pikkuState(null, 'agent', 'scorers').get(scorerName)
+  if (!scorer) {
+    throw new Error(`AI scorer not found: ${scorerName}`)
+  }
+  return scorer
+}
+
+/**
+ * The scorers an agent asked to be graded by.
+ *
+ * A name that resolves to nothing is warned about rather than thrown: a missing
+ * scorer must not take down a run that has already answered the user.
+ */
+export const scorersForAgent = (
+  agentName: string,
+  logger?: { warn: (message: string) => void }
+): PikkuAIScorer<any>[] => {
+  const agent = pikkuState(null, 'agent', 'agents').get(agentName)
+  const names = agent?.scorers ?? []
+  const scorers: PikkuAIScorer<any>[] = []
+  for (const name of names) {
+    const scorer = pikkuState(null, 'agent', 'scorers').get(name)
+    if (!scorer) {
+      logger?.warn(
+        `[pikku] Agent '${agentName}' names scorer '${name}', which is not registered — skipping it`
+      )
+      continue
+    }
+    scorers.push(scorer)
+  }
+  return scorers
+}

package/src/wirings/ai-scorer/ai-scorer-sampling.test.ts

@@ -0,0 +1,34 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { isSampled } from './ai-scorer-sampling.js'
+
+describe('isSampled', () => {
+  test('grades every run at a rate of 1 and none at 0', () => {
+    assert.equal(isSampled('run-1', 'relevance', 1), true)
+    assert.equal(isSampled('run-1', 'relevance', 0), false)
+  })
+
+  test('lands the same way every time it is asked, so a retried job is not regraded by chance', () => {
+    const first = isSampled('run-42', 'relevance', 0.5)
+    for (let i = 0; i < 20; i++) {
+      assert.equal(isSampled('run-42', 'relevance', 0.5), first)
+    }
+  })
+
+  test('two scorers at the same rate do not sample the same set of runs', () => {
+    const runs = Array.from({ length: 200 }, (_, i) => `run-${i}`)
+    const a = runs.filter((run) => isSampled(run, 'relevance', 0.5))
+    const b = runs.filter((run) => isSampled(run, 'toxicity', 0.5))
+    assert.notDeepEqual(a, b)
+  })
+
+  test('samples roughly the requested fraction over many runs', () => {
+    const runs = Array.from({ length: 2000 }, (_, i) => `run-${i}`)
+    const sampled = runs.filter((run) => isSampled(run, 'relevance', 0.25))
+    const rate = sampled.length / runs.length
+    assert.ok(
+      rate > 0.2 && rate < 0.3,
+      `expected roughly 0.25 of runs to be sampled, got ${rate}`
+    )
+  })
+})

package/src/wirings/ai-scorer/ai-scorer-sampling.ts

@@ -0,0 +1,36 @@
+/**
+ * Decide whether one run is in a scorer's sample.
+ *
+ * Deterministic on `(runId, scorerName)` rather than random, so re-running the
+ * decision — a retried job, a replayed run, a test — always lands the same way,
+ * and so two scorers at the same rate do not sample the same runs.
+ */
+export const isSampled = (
+  runId: string,
+  scorerName: string,
+  sampleRate: number
+): boolean => {
+  if (sampleRate >= 1) return true
+  if (sampleRate <= 0) return false
+
+  const key = `${scorerName}:${runId}`
+  // FNV-1a: no crypto dependency, and stable across processes and platforms,
+  // which a language-level string hash is not guaranteed to be.
+  let hash = 0x811c9dc5
+  for (let i = 0; i < key.length; i++) {
+    hash ^= key.charCodeAt(i)
+    hash = Math.imul(hash, 0x01000193) >>> 0
+  }
+
+  // FNV-1a alone leaves its high bits barely moved between keys that differ
+  // only in their last characters — sequential run ids being exactly that —
+  // which biases the fraction actually sampled well away from the rate asked
+  // for. MurmurHash3's finalizer spreads the low bits back over the whole word.
+  hash ^= hash >>> 16
+  hash = Math.imul(hash, 0x85ebca6b) >>> 0
+  hash ^= hash >>> 13
+  hash = Math.imul(hash, 0xc2b2ae35) >>> 0
+  hash ^= hash >>> 16
+
+  return (hash >>> 0) / 0x100000000 < sampleRate
+}

package/src/wirings/ai-scorer/ai-scorer-snapshots.test.ts

@@ -0,0 +1,49 @@
+import { describe, test, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
+
+import {
+  enableScoreSnapshots,
+  getScoreSnapshot,
+  recordScoreSnapshot,
+  resetScoreSnapshots,
+} from './ai-scorer-snapshots.js'
+import type { ScorerInput } from './ai-scorer.types.js'
+
+const run = (runId: string): ScorerInput => ({
+  runId,
+  agentName: 'assistant',
+  input: 'Where is Paris?',
+  output: 'France.',
+  toolCalls: [],
+  usage: { inputTokens: 5, outputTokens: 2 },
+})
+
+describe('score snapshots', () => {
+  beforeEach(() => resetScoreSnapshots())
+
+  test('a process that was never asked to retain runs holds none', () => {
+    recordScoreSnapshot(run('run-1'))
+
+    assert.equal(getScoreSnapshot('run-1'), undefined)
+  })
+
+  test('a retained run is handed back whole, so a grade sees what live scoring saw', () => {
+    enableScoreSnapshots()
+
+    recordScoreSnapshot(run('run-1'))
+
+    assert.deepEqual(getScoreSnapshot('run-1'), run('run-1'))
+  })
+
+  test('the buffer is bounded, so a long-lived dev server cannot accumulate run content', () => {
+    enableScoreSnapshots(2)
+
+    recordScoreSnapshot(run('run-1'))
+    recordScoreSnapshot(run('run-2'))
+    recordScoreSnapshot(run('run-3'))
+
+    assert.equal(getScoreSnapshot('run-1'), undefined)
+    assert.equal(getScoreSnapshot('run-2')?.runId, 'run-2')
+    assert.equal(getScoreSnapshot('run-3')?.runId, 'run-3')
+  })
+})