@pikku/core 0.12.79 → 0.12.82

package/src/wirings/ai-scorer/ai-scorer-live.test.ts

@@ -0,0 +1,174 @@
+import { beforeEach, describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { pikkuState, resetPikkuState } from '../../pikku-state.js'
+import { scoreFinishedRun } from './ai-scorer-live.js'
+import { pikkuAIJudge, pikkuAIScorer } from './ai-scorer.js'
+import type { PikkuAIScorer, ScorerInput } from './ai-scorer.types.js'
+
+const register = (agentName: string, scorers: PikkuAIScorer[]) => {
+  pikkuState(null, 'agent', 'agents').set(agentName, {
+    name: agentName,
+    scorers: scorers.map((scorer) => scorer.name),
+  } as never)
+  for (const scorer of scorers) {
+    pikkuState(null, 'agent', 'scorers').set(scorer.name, scorer)
+  }
+}
+
+const run = (overrides: Partial<ScorerInput> = {}): ScorerInput => ({
+  runId: 'run-1',
+  agentName: 'assistant',
+  threadId: 'thread-1',
+  input: 'what is the capital of France?',
+  output: 'Paris',
+  toolCalls: [],
+  usage: { inputTokens: 10, outputTokens: 5 },
+  ...overrides,
+})
+
+const collectingServices = () => {
+  const added: { queueName: string; data: any }[] = []
+  const warnings: string[] = []
+  const errors: string[] = []
+  return {
+    added,
+    warnings,
+    errors,
+    services: {
+      logger: {
+        warn: (message: string) => warnings.push(message),
+        error: (message: string) => errors.push(message),
+      },
+      queueService: {
+        add: async (queueName: string, data: unknown) => {
+          added.push({ queueName, data })
+          return undefined
+        },
+      },
+    } as any,
+  }
+}
+
+const alwaysOne = pikkuAIScorer({
+  name: 'brevity',
+  description: 'Shorter is better',
+  score: () => ({ score: 1 }),
+})
+
+describe('scoreFinishedRun', () => {
+  beforeEach(() => {
+    resetPikkuState()
+  })
+
+  test('sends one message per scorer, each to its own lane', async () => {
+    const judge = pikkuAIJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+    })
+    register('assistant', [alwaysOne, judge])
+    const { added, services } = collectingServices()
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(
+      added.map((entry) => [entry.queueName, entry.data.scorerName]),
+      [
+        ['ai-score-fast', 'brevity'],
+        ['ai-score-slow', 'helpfulness'],
+      ]
+    )
+  })
+
+  test('never grades a reference-based judge, because live traffic has no answer key', async () => {
+    const correctness = pikkuAIJudge({
+      name: 'correctness',
+      description: 'Is the answer right',
+      model: 'claude-opus-5',
+      goal: 'Grade correctness.',
+      requiresReference: true,
+    })
+    register('assistant', [correctness])
+    const { added, services } = collectingServices()
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(added, [])
+  })
+
+  test('honours the sample rate rather than grading every run', async () => {
+    const sampled = pikkuAIScorer({
+      name: 'brevity',
+      description: 'Shorter is better',
+      sampleRate: 0,
+      score: () => ({ score: 1 }),
+    })
+    register('assistant', [sampled])
+    const { added, services } = collectingServices()
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(added, [])
+  })
+
+  test('warns about a scorer the agent named but nothing registered, rather than failing the run', async () => {
+    pikkuState(null, 'agent', 'agents').set('assistant', {
+      name: 'assistant',
+      scorers: ['nonexistent'],
+    } as never)
+    const { added, warnings, services } = collectingServices()
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(added, [])
+    assert.equal(warnings.length, 1)
+    assert.match(warnings[0]!, /nonexistent/)
+  })
+
+  test('says why nothing was graded when there is no queue to grade on', async () => {
+    register('assistant', [alwaysOne])
+    const { warnings, services } = collectingServices()
+    delete services.queueService
+
+    await scoreFinishedRun(run(), services)
+
+    assert.equal(warnings.length, 1)
+    assert.match(warnings[0]!, /no queue service is registered/)
+  })
+
+  test('one scorer that cannot be enqueued does not stop the others', async () => {
+    const judge = pikkuAIJudge({
+      name: 'helpfulness',
+      description: 'Is the answer useful',
+      model: 'claude-opus-5',
+      goal: 'Grade helpfulness.',
+    })
+    register('assistant', [alwaysOne, judge])
+    const { added, errors, services } = collectingServices()
+    const add = services.queueService.add
+    services.queueService.add = async (queueName: string, data: any) => {
+      if (data.scorerName === 'brevity') throw new Error('queue is down')
+      return add(queueName, data)
+    }
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(
+      added.map((entry) => entry.data.scorerName),
+      ['helpfulness']
+    )
+    assert.equal(errors.length, 1)
+  })
+
+  test('an agent that names no scorers touches the queue at all', async () => {
+    pikkuState(null, 'agent', 'agents').set('assistant', {
+      name: 'assistant',
+    } as never)
+    const { added, services } = collectingServices()
+
+    await scoreFinishedRun(run(), services)
+
+    assert.deepEqual(added, [])
+  })
+})

package/src/wirings/ai-scorer/ai-scorer-live.ts

@@ -0,0 +1,56 @@
+import type { CoreSingletonServices } from '../../types/core.types.js'
+import { scorersForAgent } from './ai-scorer-registry.js'
+import { isSampled } from './ai-scorer-sampling.js'
+import { SCORER_LANE_QUEUES, type ScorerInput } from './ai-scorer.types.js'
+
+/**
+ * Grade one finished run on live traffic.
+ *
+ * Called as the terminal step of `finalizeAgentRun`, so it is structurally last
+ * — a developer cannot register anything after it, and cannot reorder it. It is
+ * also strictly best-effort: the client already has its answer, so nothing here
+ * may fail the run.
+ */
+export const scoreFinishedRun = async (
+  run: ScorerInput,
+  services: Pick<CoreSingletonServices, 'logger'> & {
+    queueService?: { add: (queueName: string, data: unknown) => Promise<any> }
+  }
+): Promise<void> => {
+  const scorers = scorersForAgent(run.agentName, services.logger).filter(
+    // A reference-based judge grades against an answer key, and live traffic
+    // has none.
+    (scorer) => !scorer.requiresReference
+  )
+  if (scorers.length === 0) return
+
+  if (!services.queueService) {
+    services.logger?.warn(
+      `[pikku] Agent '${run.agentName}' declares scorers but no queue service is registered — skipping live scoring`
+    )
+    return
+  }
+
+  const sampled = scorers.filter((scorer) =>
+    isSampled(run.runId, scorer.name, scorer.sampleRate)
+  )
+
+  // One message per scorer, so each gets its own retry, isolation and lane.
+  await Promise.all(
+    sampled.map(async (scorer) => {
+      try {
+        await services.queueService!.add(SCORER_LANE_QUEUES[scorer.lane], {
+          ...run,
+          scorerName: scorer.name,
+          // No reference: a live run has no answer key.
+          reference: undefined,
+        })
+      } catch (error) {
+        services.logger?.error(
+          `[pikku] Failed to enqueue the '${scorer.name}' grade of run ${run.runId}`,
+          { error }
+        )
+      }
+    })
+  )
+}

package/src/wirings/ai-scorer/ai-scorer-registry.ts

@@ -0,0 +1,63 @@
+import { pikkuState } from '../../pikku-state.js'
+import type { PikkuAIScorer } from './ai-scorer.types.js'
+
+export const addAIScorer = (
+  scorerName: string,
+  scorer: PikkuAIScorer<any>,
+  packageName: string | null = null
+) => {
+  const scorersMeta = pikkuState(packageName, 'agent', 'scorersMeta')
+  if (!scorersMeta[scorerName]) {
+    console.warn(
+      `[pikku] Skipping AI scorer '${scorerName}' — metadata not found. Scorers must be declared in a *.scorer.ts file.`
+    )
+    return
+  }
+  const scorers = pikkuState(packageName, 'agent', 'scorers')
+  if (scorers.has(scorerName)) {
+    throw new Error(`AI scorer already exists: ${scorerName}`)
+  }
+  scorers.set(scorerName, scorer)
+}
+
+export const getAIScorers = () => pikkuState(null, 'agent', 'scorers')
+
+export const getAIScorersMeta = () => pikkuState(null, 'agent', 'scorersMeta')
+
+/**
+ * Resolve a scorer by name across every registered package, the way an agent's
+ * tools are resolved: a scorer declared in an addon is nameable by an app agent.
+ */
+export const resolveAIScorer = (scorerName: string): PikkuAIScorer<any> => {
+  const scorer = pikkuState(null, 'agent', 'scorers').get(scorerName)
+  if (!scorer) {
+    throw new Error(`AI scorer not found: ${scorerName}`)
+  }
+  return scorer
+}
+
+/**
+ * The scorers an agent asked to be graded by.
+ *
+ * A name that resolves to nothing is warned about rather than thrown: a missing
+ * scorer must not take down a run that has already answered the user.
+ */
+export const scorersForAgent = (
+  agentName: string,
+  logger?: { warn: (message: string) => void }
+): PikkuAIScorer<any>[] => {
+  const agent = pikkuState(null, 'agent', 'agents').get(agentName)
+  const names = agent?.scorers ?? []
+  const scorers: PikkuAIScorer<any>[] = []
+  for (const name of names) {
+    const scorer = pikkuState(null, 'agent', 'scorers').get(name)
+    if (!scorer) {
+      logger?.warn(
+        `[pikku] Agent '${agentName}' names scorer '${name}', which is not registered — skipping it`
+      )
+      continue
+    }
+    scorers.push(scorer)
+  }
+  return scorers
+}

package/src/wirings/ai-scorer/ai-scorer-sampling.test.ts

@@ -0,0 +1,34 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { isSampled } from './ai-scorer-sampling.js'
+
+describe('isSampled', () => {
+  test('grades every run at a rate of 1 and none at 0', () => {
+    assert.equal(isSampled('run-1', 'relevance', 1), true)
+    assert.equal(isSampled('run-1', 'relevance', 0), false)
+  })
+
+  test('lands the same way every time it is asked, so a retried job is not regraded by chance', () => {
+    const first = isSampled('run-42', 'relevance', 0.5)
+    for (let i = 0; i < 20; i++) {
+      assert.equal(isSampled('run-42', 'relevance', 0.5), first)
+    }
+  })
+
+  test('two scorers at the same rate do not sample the same set of runs', () => {
+    const runs = Array.from({ length: 200 }, (_, i) => `run-${i}`)
+    const a = runs.filter((run) => isSampled(run, 'relevance', 0.5))
+    const b = runs.filter((run) => isSampled(run, 'toxicity', 0.5))
+    assert.notDeepEqual(a, b)
+  })
+
+  test('samples roughly the requested fraction over many runs', () => {
+    const runs = Array.from({ length: 2000 }, (_, i) => `run-${i}`)
+    const sampled = runs.filter((run) => isSampled(run, 'relevance', 0.25))
+    const rate = sampled.length / runs.length
+    assert.ok(
+      rate > 0.2 && rate < 0.3,
+      `expected roughly 0.25 of runs to be sampled, got ${rate}`
+    )
+  })
+})

package/src/wirings/ai-scorer/ai-scorer-sampling.ts

@@ -0,0 +1,36 @@
+/**
+ * Decide whether one run is in a scorer's sample.
+ *
+ * Deterministic on `(runId, scorerName)` rather than random, so re-running the
+ * decision — a retried job, a replayed run, a test — always lands the same way,
+ * and so two scorers at the same rate do not sample the same runs.
+ */
+export const isSampled = (
+  runId: string,
+  scorerName: string,
+  sampleRate: number
+): boolean => {
+  if (sampleRate >= 1) return true
+  if (sampleRate <= 0) return false
+
+  const key = `${scorerName}:${runId}`
+  // FNV-1a: no crypto dependency, and stable across processes and platforms,
+  // which a language-level string hash is not guaranteed to be.
+  let hash = 0x811c9dc5
+  for (let i = 0; i < key.length; i++) {
+    hash ^= key.charCodeAt(i)
+    hash = Math.imul(hash, 0x01000193) >>> 0
+  }
+
+  // FNV-1a alone leaves its high bits barely moved between keys that differ
+  // only in their last characters — sequential run ids being exactly that —
+  // which biases the fraction actually sampled well away from the rate asked
+  // for. MurmurHash3's finalizer spreads the low bits back over the whole word.
+  hash ^= hash >>> 16
+  hash = Math.imul(hash, 0x85ebca6b) >>> 0
+  hash ^= hash >>> 13
+  hash = Math.imul(hash, 0xc2b2ae35) >>> 0
+  hash ^= hash >>> 16
+
+  return (hash >>> 0) / 0x100000000 < sampleRate
+}

package/src/wirings/ai-scorer/ai-scorer-snapshots.test.ts

@@ -0,0 +1,49 @@
+import { describe, test, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
+
+import {
+  enableScoreSnapshots,
+  getScoreSnapshot,
+  recordScoreSnapshot,
+  resetScoreSnapshots,
+} from './ai-scorer-snapshots.js'
+import type { ScorerInput } from './ai-scorer.types.js'
+
+const run = (runId: string): ScorerInput => ({
+  runId,
+  agentName: 'assistant',
+  input: 'Where is Paris?',
+  output: 'France.',
+  toolCalls: [],
+  usage: { inputTokens: 5, outputTokens: 2 },
+})
+
+describe('score snapshots', () => {
+  beforeEach(() => resetScoreSnapshots())
+
+  test('a process that was never asked to retain runs holds none', () => {
+    recordScoreSnapshot(run('run-1'))
+
+    assert.equal(getScoreSnapshot('run-1'), undefined)
+  })
+
+  test('a retained run is handed back whole, so a grade sees what live scoring saw', () => {
+    enableScoreSnapshots()
+
+    recordScoreSnapshot(run('run-1'))
+
+    assert.deepEqual(getScoreSnapshot('run-1'), run('run-1'))
+  })
+
+  test('the buffer is bounded, so a long-lived dev server cannot accumulate run content', () => {
+    enableScoreSnapshots(2)
+
+    recordScoreSnapshot(run('run-1'))
+    recordScoreSnapshot(run('run-2'))
+    recordScoreSnapshot(run('run-3'))
+
+    assert.equal(getScoreSnapshot('run-1'), undefined)
+    assert.equal(getScoreSnapshot('run-2')?.runId, 'run-2')
+    assert.equal(getScoreSnapshot('run-3')?.runId, 'run-3')
+  })
+})

package/src/wirings/ai-scorer/ai-scorer-snapshots.ts

@@ -0,0 +1,46 @@
+import type { ScorerInput } from './ai-scorer.types.js'
+
+/**
+ * The last few finished runs, held so a scenario can grade one.
+ *
+ * A scenario cannot rebuild a `ScorerInput` from storage: the run record holds
+ * status and usage, and the prompt, answer and tool calls are spread across the
+ * thread's messages, where the boundary of a single run is not recoverable. So
+ * the runtime keeps the snapshot it already took at finalize — which is also
+ * what makes a scenario's grade comparable to a live one, since both grade the
+ * identical object, redactions included.
+ *
+ * Off unless something turns it on, and bounded when it is. A process that
+ * grades no scenarios holds nothing, and one that does cannot accumulate run
+ * content without limit.
+ */
+const snapshots = new Map<string, ScorerInput>()
+
+let limit = 0
+
+/**
+ * Start retaining snapshots. Called where the scenario grading RPC is
+ * registered, so the buffer exists in exactly the processes that can read it —
+ * a development server — and never in a deployed bundle.
+ */
+export const enableScoreSnapshots = (maxRuns = 50): void => {
+  limit = maxRuns
+}
+
+export const recordScoreSnapshot = (run: ScorerInput): void => {
+  if (limit === 0) return
+  snapshots.set(run.runId, run)
+  while (snapshots.size > limit) {
+    const oldest = snapshots.keys().next().value
+    if (oldest === undefined) break
+    snapshots.delete(oldest)
+  }
+}
+
+export const getScoreSnapshot = (runId: string): ScorerInput | undefined =>
+  snapshots.get(runId)
+
+export const resetScoreSnapshots = (): void => {
+  snapshots.clear()
+  limit = 0
+}

package/src/wirings/ai-scorer/ai-scorer-worker.test.ts

@@ -0,0 +1,122 @@
+import { beforeEach, describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { pikkuState, resetPikkuState } from '../../pikku-state.js'
+import { pikkuAIScoreWorkerFunc } from './ai-scorer-worker.js'
+import { pikkuAIJudge, pikkuAIScorer } from './ai-scorer.js'
+import type { PikkuAIScorer, ScoreJob } from './ai-scorer.types.js'
+
+const job = (overrides: Partial<ScoreJob> = {}): ScoreJob => ({
+  scorerName: 'brevity',
+  runId: 'run-1',
+  agentName: 'assistant',
+  input: 'what is the capital of France?',
+  output: 'Paris',
+  toolCalls: [],
+  usage: { inputTokens: 10, outputTokens: 5 },
+  ...overrides,
+})
+
+const install = (scorer: PikkuAIScorer, aiAgentRunner?: unknown) => {
+  pikkuState(null, 'agent', 'scorers').set(scorer.name, scorer)
+  const saved: unknown[] = []
+  pikkuState(null, 'package', 'singletonServices', {
+    aiRunState: {
+      saveScore: async (score: unknown) => {
+        saved.push(score)
+      },
+    },
+    aiAgentRunner,
+  } as never)
+  return saved
+}
+
+describe('pikkuAIScoreWorkerFunc', () => {
+  beforeEach(() => {
+    resetPikkuState()
+  })
+
+  test('records what a heuristic scorer graded, against the run it graded', async () => {
+    const saved = install(
+      pikkuAIScorer({
+        name: 'brevity',
+        description: 'Shorter is better',
+        score: (input) => ({
+          score: input.output.length < 20 ? 1 : 0,
+          reason: 'Answered in one word.',
+        }),
+      })
+    )
+
+    await pikkuAIScoreWorkerFunc({}, job())
+
+    assert.deepEqual(saved, [
+      {
+        runId: 'run-1',
+        scorerName: 'brevity',
+        score: 1,
+        reason: 'Answered in one word.',
+      },
+    ])
+  })
+
+  test('hands a judge to the model rather than calling a score function it has not got', async () => {
+    const saved = install(
+      pikkuAIJudge({
+        name: 'helpfulness',
+        description: 'Is the answer useful',
+        model: 'claude-opus-5',
+        goal: 'Grade helpfulness.',
+      }),
+      {
+        run: async () => ({
+          object: { score: 0.5, reason: 'Terse.' },
+          usage: { inputTokens: 80, outputTokens: 20 },
+        }),
+      }
+    )
+
+    await pikkuAIScoreWorkerFunc({}, job({ scorerName: 'helpfulness' }))
+
+    assert.deepEqual(saved, [
+      {
+        runId: 'run-1',
+        scorerName: 'helpfulness',
+        score: 0.5,
+        reason: 'Terse.',
+        metadata: { judgeModel: 'claude-opus-5', judgeTokens: 100 },
+      },
+    ])
+  })
+
+  test('fails the job when there is nowhere to record the grade, so it is retried rather than lost', async () => {
+    pikkuState(null, 'agent', 'scorers').set(
+      'brevity',
+      pikkuAIScorer({
+        name: 'brevity',
+        description: 'Shorter is better',
+        score: () => ({ score: 1 }),
+      })
+    )
+    pikkuState(null, 'package', 'singletonServices', {} as never)
+
+    await assert.rejects(
+      () => pikkuAIScoreWorkerFunc({}, job()),
+      /AI run state service not initialized/
+    )
+  })
+
+  test('fails on a scorer name that resolves to nothing', async () => {
+    install(
+      pikkuAIScorer({
+        name: 'brevity',
+        description: 'Shorter is better',
+        score: () => ({ score: 1 }),
+      })
+    )
+
+    await assert.rejects(
+      () => pikkuAIScoreWorkerFunc({}, job({ scorerName: 'gone' })),
+      /AI scorer not found: gone/
+    )
+  })
+})

package/src/wirings/ai-scorer/ai-scorer-worker.ts

@@ -0,0 +1,69 @@
+import { pikkuState } from '../../pikku-state.js'
+import { addFunction } from '../../function/function-runner.js'
+import { wireQueueWorker } from '../queue/queue-runner.js'
+import { gradeRun } from './ai-scorer-grade.js'
+import { SCORER_LANE_QUEUES, type ScoreJob } from './ai-scorer.types.js'
+
+/**
+ * The one worker behind both lanes: it resolves the scorer by name and grades.
+ * The lane a job arrived on decides nothing except how long it may take.
+ */
+export async function pikkuAIScoreWorkerFunc(
+  _services: Record<string, unknown>,
+  job: ScoreJob
+): Promise<void> {
+  const services = pikkuState(null, 'package', 'singletonServices')
+  if (!services) {
+    throw new Error(
+      `Singleton services not initialized: cannot run the '${job.scorerName}' grade of run ${job.runId}`
+    )
+  }
+
+  await gradeRun(job, services, { persist: true })
+}
+
+const SCORE_WORKER_FUNC_ID = 'pikkuAIScoreWorker'
+
+/**
+ * Bind the two lane queues to the shared worker.
+ *
+ * Registered programmatically rather than emitted into the user's bootstrap:
+ * `wireQueueWorker` warns and returns for a name codegen produced no metadata
+ * for, so the metadata is synthesised here alongside the registration. This
+ * mirrors how workflows wire their own queues.
+ */
+export const wireAIScorerQueueWorkers = (): void => {
+  // No scorers means no lanes: a deployment that grades nothing should not be
+  // left holding two queues nothing ever writes to.
+  if (pikkuState(null, 'agent', 'scorers').size === 0) return
+
+  const functions = pikkuState(null, 'function', 'functions')
+  const functionsMeta = pikkuState(null, 'function', 'meta')
+  const queueMeta = pikkuState(null, 'queue', 'meta')
+
+  if (!functions.has(SCORE_WORKER_FUNC_ID)) {
+    addFunction(SCORE_WORKER_FUNC_ID, { func: pikkuAIScoreWorkerFunc } as never)
+  }
+  if (!functionsMeta[SCORE_WORKER_FUNC_ID]) {
+    functionsMeta[SCORE_WORKER_FUNC_ID] = {
+      pikkuFuncId: SCORE_WORKER_FUNC_ID,
+      sessionless: true,
+      functionType: 'helper',
+      inputSchemaName: null,
+      outputSchemaName: null,
+    }
+  }
+
+  for (const queueName of Object.values(SCORER_LANE_QUEUES)) {
+    if (!queueMeta[queueName]) {
+      queueMeta[queueName] = {
+        pikkuFuncId: SCORE_WORKER_FUNC_ID,
+        name: queueName,
+      }
+    }
+    wireQueueWorker({
+      name: queueName,
+      func: { func: pikkuAIScoreWorkerFunc },
+    } as never)
+  }
+}