@pikku/core 0.12.107 → 0.12.112

package/src/wirings/agent-scorer/agent-scorer-registry.ts

@@ -1,66 +0,0 @@
-import { pikkuState } from '../../pikku-state.js'
-import type { PikkuAgentScorer } from './agent-scorer.types.js'
-
-export const addAgentScorer = (
-  scorerName: string,
-  scorer: PikkuAgentScorer<any>,
-  packageName: string | null = null
-) => {
-  const scorersMeta = pikkuState(packageName, 'agent', 'scorersMeta')
-  if (!scorersMeta[scorerName]) {
-    console.warn(
-      `[pikku] Skipping AI scorer '${scorerName}' — metadata not found. Scorers must be declared in a *.scorer.ts file.`
-    )
-    return
-  }
-  const scorers = pikkuState(packageName, 'agent', 'scorers')
-  if (scorers.has(scorerName)) {
-    throw new Error(`AI scorer already exists: ${scorerName}`)
-  }
-  scorers.set(scorerName, scorer)
-}
-
-export const getAgentScorers = () => pikkuState(null, 'agent', 'scorers')
-
-export const getAgentScorersMeta = () =>
-  pikkuState(null, 'agent', 'scorersMeta')
-
-/**
- * Resolve a scorer by name across every registered package, the way an agent's
- * tools are resolved: a scorer declared in an addon is nameable by an app agent.
- */
-export const resolveAgentScorer = (
-  scorerName: string
-): PikkuAgentScorer<any> => {
-  const scorer = pikkuState(null, 'agent', 'scorers').get(scorerName)
-  if (!scorer) {
-    throw new Error(`AI scorer not found: ${scorerName}`)
-  }
-  return scorer
-}
-
-/**
- * The scorers an agent asked to be graded by.
- *
- * A name that resolves to nothing is warned about rather than thrown: a missing
- * scorer must not take down a run that has already answered the user.
- */
-export const scorersForAgent = (
-  agentName: string,
-  logger?: { warn: (message: string) => void }
-): PikkuAgentScorer<any>[] => {
-  const agent = pikkuState(null, 'agent', 'agents').get(agentName)
-  const names = agent?.scorers ?? []
-  const scorers: PikkuAgentScorer<any>[] = []
-  for (const name of names) {
-    const scorer = pikkuState(null, 'agent', 'scorers').get(name)
-    if (!scorer) {
-      logger?.warn(
-        `[pikku] Agent '${agentName}' names scorer '${name}', which is not registered — skipping it`
-      )
-      continue
-    }
-    scorers.push(scorer)
-  }
-  return scorers
-}

package/src/wirings/agent-scorer/agent-scorer-sampling.test.ts

@@ -1,34 +0,0 @@
-import { describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-import { isSampled } from './agent-scorer-sampling.js'
-
-describe('isSampled', () => {
-  test('grades every run at a rate of 1 and none at 0', () => {
-    assert.equal(isSampled('run-1', 'relevance', 1), true)
-    assert.equal(isSampled('run-1', 'relevance', 0), false)
-  })
-
-  test('lands the same way every time it is asked, so a retried job is not regraded by chance', () => {
-    const first = isSampled('run-42', 'relevance', 0.5)
-    for (let i = 0; i < 20; i++) {
-      assert.equal(isSampled('run-42', 'relevance', 0.5), first)
-    }
-  })
-
-  test('two scorers at the same rate do not sample the same set of runs', () => {
-    const runs = Array.from({ length: 200 }, (_, i) => `run-${i}`)
-    const a = runs.filter((run) => isSampled(run, 'relevance', 0.5))
-    const b = runs.filter((run) => isSampled(run, 'toxicity', 0.5))
-    assert.notDeepEqual(a, b)
-  })
-
-  test('samples roughly the requested fraction over many runs', () => {
-    const runs = Array.from({ length: 2000 }, (_, i) => `run-${i}`)
-    const sampled = runs.filter((run) => isSampled(run, 'relevance', 0.25))
-    const rate = sampled.length / runs.length
-    assert.ok(
-      rate > 0.2 && rate < 0.3,
-      `expected roughly 0.25 of runs to be sampled, got ${rate}`
-    )
-  })
-})

package/src/wirings/agent-scorer/agent-scorer-sampling.ts

@@ -1,36 +0,0 @@
-/**
- * Decide whether one run is in a scorer's sample.
- *
- * Deterministic on `(runId, scorerName)` rather than random, so re-running the
- * decision — a retried job, a replayed run, a test — always lands the same way,
- * and so two scorers at the same rate do not sample the same runs.
- */
-export const isSampled = (
-  runId: string,
-  scorerName: string,
-  sampleRate: number
-): boolean => {
-  if (sampleRate >= 1) return true
-  if (sampleRate <= 0) return false
-
-  const key = `${scorerName}:${runId}`
-  // FNV-1a: no crypto dependency, and stable across processes and platforms,
-  // which a language-level string hash is not guaranteed to be.
-  let hash = 0x811c9dc5
-  for (let i = 0; i < key.length; i++) {
-    hash ^= key.charCodeAt(i)
-    hash = Math.imul(hash, 0x01000193) >>> 0
-  }
-
-  // FNV-1a alone leaves its high bits barely moved between keys that differ
-  // only in their last characters — sequential run ids being exactly that —
-  // which biases the fraction actually sampled well away from the rate asked
-  // for. MurmurHash3's finalizer spreads the low bits back over the whole word.
-  hash ^= hash >>> 16
-  hash = Math.imul(hash, 0x85ebca6b) >>> 0
-  hash ^= hash >>> 13
-  hash = Math.imul(hash, 0xc2b2ae35) >>> 0
-  hash ^= hash >>> 16
-
-  return (hash >>> 0) / 0x100000000 < sampleRate
-}

package/src/wirings/agent-scorer/agent-scorer-snapshots.test.ts

@@ -1,49 +0,0 @@
-import { describe, test, beforeEach } from 'node:test'
-import assert from 'node:assert/strict'
-
-import {
-  enableScoreSnapshots,
-  getScoreSnapshot,
-  recordScoreSnapshot,
-  resetScoreSnapshots,
-} from './agent-scorer-snapshots.js'
-import type { ScorerInput } from './agent-scorer.types.js'
-
-const run = (runId: string): ScorerInput => ({
-  runId,
-  agentName: 'assistant',
-  input: 'Where is Paris?',
-  output: 'France.',
-  toolCalls: [],
-  usage: { inputTokens: 5, outputTokens: 2 },
-})
-
-describe('score snapshots', () => {
-  beforeEach(() => resetScoreSnapshots())
-
-  test('a process that was never asked to retain runs holds none', () => {
-    recordScoreSnapshot(run('run-1'))
-
-    assert.equal(getScoreSnapshot('run-1'), undefined)
-  })
-
-  test('a retained run is handed back whole, so a grade sees what live scoring saw', () => {
-    enableScoreSnapshots()
-
-    recordScoreSnapshot(run('run-1'))
-
-    assert.deepEqual(getScoreSnapshot('run-1'), run('run-1'))
-  })
-
-  test('the buffer is bounded, so a long-lived dev server cannot accumulate run content', () => {
-    enableScoreSnapshots(2)
-
-    recordScoreSnapshot(run('run-1'))
-    recordScoreSnapshot(run('run-2'))
-    recordScoreSnapshot(run('run-3'))
-
-    assert.equal(getScoreSnapshot('run-1'), undefined)
-    assert.equal(getScoreSnapshot('run-2')?.runId, 'run-2')
-    assert.equal(getScoreSnapshot('run-3')?.runId, 'run-3')
-  })
-})

package/src/wirings/agent-scorer/agent-scorer-snapshots.ts

@@ -1,46 +0,0 @@
-import type { ScorerInput } from './agent-scorer.types.js'
-
-/**
- * The last few finished runs, held so a scenario can grade one.
- *
- * A scenario cannot rebuild a `ScorerInput` from storage: the run record holds
- * status and usage, and the prompt, answer and tool calls are spread across the
- * thread's messages, where the boundary of a single run is not recoverable. So
- * the runtime keeps the snapshot it already took at finalize — which is also
- * what makes a scenario's grade comparable to a live one, since both grade the
- * identical object, redactions included.
- *
- * Off unless something turns it on, and bounded when it is. A process that
- * grades no scenarios holds nothing, and one that does cannot accumulate run
- * content without limit.
- */
-const snapshots = new Map<string, ScorerInput>()
-
-let limit = 0
-
-/**
- * Start retaining snapshots. Called where the scenario grading RPC is
- * registered, so the buffer exists in exactly the processes that can read it —
- * a development server — and never in a deployed bundle.
- */
-export const enableScoreSnapshots = (maxRuns = 50): void => {
-  limit = maxRuns
-}
-
-export const recordScoreSnapshot = (run: ScorerInput): void => {
-  if (limit === 0) return
-  snapshots.set(run.runId, run)
-  while (snapshots.size > limit) {
-    const oldest = snapshots.keys().next().value
-    if (oldest === undefined) break
-    snapshots.delete(oldest)
-  }
-}
-
-export const getScoreSnapshot = (runId: string): ScorerInput | undefined =>
-  snapshots.get(runId)
-
-export const resetScoreSnapshots = (): void => {
-  snapshots.clear()
-  limit = 0
-}

package/src/wirings/agent-scorer/agent-scorer-worker.test.ts

@@ -1,122 +0,0 @@
-import { beforeEach, describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-import { pikkuState, resetPikkuState } from '../../pikku-state.js'
-import { pikkuAgentScoreWorkerFunc } from './agent-scorer-worker.js'
-import { pikkuAgentJudge, pikkuAgentScorer } from './agent-scorer.js'
-import type { PikkuAgentScorer, ScoreJob } from './agent-scorer.types.js'
-
-const job = (overrides: Partial<ScoreJob> = {}): ScoreJob => ({
-  scorerName: 'brevity',
-  runId: 'run-1',
-  agentName: 'assistant',
-  input: 'what is the capital of France?',
-  output: 'Paris',
-  toolCalls: [],
-  usage: { inputTokens: 10, outputTokens: 5 },
-  ...overrides,
-})
-
-const install = (scorer: PikkuAgentScorer, agentRunner?: unknown) => {
-  pikkuState(null, 'agent', 'scorers').set(scorer.name, scorer)
-  const saved: unknown[] = []
-  pikkuState(null, 'package', 'singletonServices', {
-    agentRunState: {
-      saveScore: async (score: unknown) => {
-        saved.push(score)
-      },
-    },
-    agentRunner,
-  } as never)
-  return saved
-}
-
-describe('pikkuAgentScoreWorkerFunc', () => {
-  beforeEach(() => {
-    resetPikkuState()
-  })
-
-  test('records what a heuristic scorer graded, against the run it graded', async () => {
-    const saved = install(
-      pikkuAgentScorer({
-        name: 'brevity',
-        description: 'Shorter is better',
-        score: (input) => ({
-          score: input.output.length < 20 ? 1 : 0,
-          reason: 'Answered in one word.',
-        }),
-      })
-    )
-
-    await pikkuAgentScoreWorkerFunc({}, job())
-
-    assert.deepEqual(saved, [
-      {
-        runId: 'run-1',
-        scorerName: 'brevity',
-        score: 1,
-        reason: 'Answered in one word.',
-      },
-    ])
-  })
-
-  test('hands a judge to the model rather than calling a score function it has not got', async () => {
-    const saved = install(
-      pikkuAgentJudge({
-        name: 'helpfulness',
-        description: 'Is the answer useful',
-        model: 'claude-opus-5',
-        goal: 'Grade helpfulness.',
-      }),
-      {
-        run: async () => ({
-          object: { score: 0.5, reason: 'Terse.' },
-          usage: { inputTokens: 80, outputTokens: 20 },
-        }),
-      }
-    )
-
-    await pikkuAgentScoreWorkerFunc({}, job({ scorerName: 'helpfulness' }))
-
-    assert.deepEqual(saved, [
-      {
-        runId: 'run-1',
-        scorerName: 'helpfulness',
-        score: 0.5,
-        reason: 'Terse.',
-        metadata: { judgeModel: 'claude-opus-5', judgeTokens: 100 },
-      },
-    ])
-  })
-
-  test('fails the job when there is nowhere to record the grade, so it is retried rather than lost', async () => {
-    pikkuState(null, 'agent', 'scorers').set(
-      'brevity',
-      pikkuAgentScorer({
-        name: 'brevity',
-        description: 'Shorter is better',
-        score: () => ({ score: 1 }),
-      })
-    )
-    pikkuState(null, 'package', 'singletonServices', {} as never)
-
-    await assert.rejects(
-      () => pikkuAgentScoreWorkerFunc({}, job()),
-      /AI run state service not initialized/
-    )
-  })
-
-  test('fails on a scorer name that resolves to nothing', async () => {
-    install(
-      pikkuAgentScorer({
-        name: 'brevity',
-        description: 'Shorter is better',
-        score: () => ({ score: 1 }),
-      })
-    )
-
-    await assert.rejects(
-      () => pikkuAgentScoreWorkerFunc({}, job({ scorerName: 'gone' })),
-      /AI scorer not found: gone/
-    )
-  })
-})

package/src/wirings/agent-scorer/agent-scorer-worker.ts

@@ -1,71 +0,0 @@
-import { pikkuState } from '../../pikku-state.js'
-import { addFunction } from '../../function/function-runner.js'
-import { wireQueueWorker } from '../queue/queue-runner.js'
-import { gradeRun } from './agent-scorer-grade.js'
-import { SCORER_LANE_QUEUES, type ScoreJob } from './agent-scorer.types.js'
-
-/**
- * The one worker behind both lanes: it resolves the scorer by name and grades.
- * The lane a job arrived on decides nothing except how long it may take.
- */
-export async function pikkuAgentScoreWorkerFunc(
-  _services: Record<string, unknown>,
-  job: ScoreJob
-): Promise<void> {
-  const services = pikkuState(null, 'package', 'singletonServices')
-  if (!services) {
-    throw new Error(
-      `Singleton services not initialized: cannot run the '${job.scorerName}' grade of run ${job.runId}`
-    )
-  }
-
-  await gradeRun(job, services, { persist: true })
-}
-
-const SCORE_WORKER_FUNC_ID = 'pikkuAgentScoreWorker'
-
-/**
- * Bind the two lane queues to the shared worker.
- *
- * Registered programmatically rather than emitted into the user's bootstrap:
- * `wireQueueWorker` warns and returns for a name codegen produced no metadata
- * for, so the metadata is synthesised here alongside the registration. This
- * mirrors how workflows wire their own queues.
- */
-export const wireAgentScorerQueueWorkers = (): void => {
-  // No scorers means no lanes: a deployment that grades nothing should not be
-  // left holding two queues nothing ever writes to.
-  if (pikkuState(null, 'agent', 'scorers').size === 0) return
-
-  const functions = pikkuState(null, 'function', 'functions')
-  const functionsMeta = pikkuState(null, 'function', 'meta')
-  const queueMeta = pikkuState(null, 'queue', 'meta')
-
-  if (!functions.has(SCORE_WORKER_FUNC_ID)) {
-    addFunction(SCORE_WORKER_FUNC_ID, {
-      func: pikkuAgentScoreWorkerFunc,
-    } as never)
-  }
-  if (!functionsMeta[SCORE_WORKER_FUNC_ID]) {
-    functionsMeta[SCORE_WORKER_FUNC_ID] = {
-      pikkuFuncId: SCORE_WORKER_FUNC_ID,
-      sessionless: true,
-      functionType: 'helper',
-      inputSchemaName: null,
-      outputSchemaName: null,
-    }
-  }
-
-  for (const queueName of Object.values(SCORER_LANE_QUEUES)) {
-    if (!queueMeta[queueName]) {
-      queueMeta[queueName] = {
-        pikkuFuncId: SCORE_WORKER_FUNC_ID,
-        name: queueName,
-      }
-    }
-    wireQueueWorker({
-      name: queueName,
-      func: { func: pikkuAgentScoreWorkerFunc },
-    } as never)
-  }
-}

package/src/wirings/agent-scorer/agent-scorer.ts

@@ -1,102 +0,0 @@
-import type {
-  JudgeToolCallDisclosure,
-  PikkuAgentScorer,
-  ScorerInput,
-  ScorerOutput,
-} from './agent-scorer.types.js'
-
-const assertSampleRate = (name: string, sampleRate: number | undefined) => {
-  if (sampleRate === undefined) return 1
-  if (sampleRate < 0 || sampleRate > 1) {
-    throw new Error(
-      `Scorer '${name}' has a sampleRate of ${sampleRate} — it is a fraction of live runs to grade, so it must be between 0 and 1`
-    )
-  }
-  return sampleRate
-}
-
-/**
- * A heuristic scorer: pure code over the finished run, no model call, so it
- * grades on the fast lane.
- *
- * @example snippet: agentScorer
- */
-export const pikkuAgentScorer = <Services = any>(config: {
-  /** Identifies the scorer in results and in the Console. Unique per project. */
-  name: string
-  /** What this scorer grades, in one line, for whoever reads the score later. */
-  description: string
-  /** 0..1 fraction of live runs to grade. Defaults to all of them. */
-  sampleRate?: number
-  /**
-   * Grades against a known-correct answer. Such a scorer is test-only — live
-   * traffic has no answer key, so the runtime never samples it.
-   */
-  requiresReference?: boolean
-  /**
-   * The grade itself: read the finished run and return `{ score, reason }`.
-   * Runs in-process, so it may use your own services.
-   */
-  score: (
-    input: ScorerInput,
-    services: Services
-  ) => ScorerOutput | Promise<ScorerOutput>
-}): PikkuAgentScorer<Services> => ({
-  name: config.name,
-  description: config.description,
-  lane: 'fast',
-  sampleRate: assertSampleRate(config.name, config.sampleRate),
-  requiresReference: config.requiresReference ?? false,
-  score: config.score,
-})
-
-/**
- * An LLM-judge scorer: the runtime makes the model call and forces a structured
- * `{ score, reason }`, so a judge is a rubric rather than a prompt to parse.
- *
- * The rubric field is `goal`, matching `pikkuAgent`'s prompt vocabulary — a
- * judge is a degenerate agent, and should use the same word for the same thing.
- * `prompt` is the escape hatch for non-standard framing.
- *
- * @example snippet: agentJudge
- */
-export const pikkuAgentJudge = <Services = any>(config: {
-  /** Identifies the judge in results and in the Console. Unique per project. */
-  name: string
-  /** What this judge grades, in one line, for whoever reads the score later. */
-  description: string
-  /** 0..1 fraction of live runs to grade. Defaults to all of them. */
-  sampleRate?: number
-  /**
-   * Grades against a known-correct answer. Such a judge is test-only — live
-   * traffic has no answer key, so the runtime never samples it.
-   */
-  requiresReference?: boolean
-  /** The model that grades, e.g. `'claude-sonnet-4-5'`. Not the model under test. */
-  model: string
-  /** The rubric: what a good answer looks like, phrased as the goal it should meet. */
-  goal: string
-  /**
-   * How much of the run's trajectory to disclose to the judge. Defaults to
-   * `names`: enough to tell a tool-backed answer from an invented one, without
-   * sending a third-party model the rows the tools returned.
-   */
-  toolCalls?: JudgeToolCallDisclosure
-  /**
-   * Replaces the generated rubric prompt outright, for framing `goal` cannot
-   * express. The `{ score, reason }` response is still forced.
-   */
-  prompt?: (input: ScorerInput) => string
-}): PikkuAgentScorer<Services> => ({
-  name: config.name,
-  description: config.description,
-  lane: 'slow',
-  sampleRate: assertSampleRate(config.name, config.sampleRate),
-  requiresReference: config.requiresReference ?? false,
-  judge: {
-    model: config.model,
-    goal: config.goal,
-    toolCalls: config.toolCalls ?? 'names',
-    ...(config.prompt ? { prompt: config.prompt } : {}),
-  },
-})

package/src/wirings/agent-scorer/agent-scorer.types.ts

@@ -1,128 +0,0 @@
-/**
- * What a scorer is shown about a finished run.
- *
- * One snapshot, taken after the output middleware has resolved, so a scorer and
- * the persisted run record see the same thing — including the redactions.
- */
-export interface ScorerInput {
-  runId: string
-  agentName: string
-  threadId?: string
-  resourceId?: string
-  /** The user prompt the run answered. */
-  input: string
-  /** The agent's final text answer. */
-  output: string
-  /**
-   * The known-correct answer. Supplied by a scenario for a reference-based
-   * judge; never set on live traffic, which has no answer key.
-   */
-  reference?: string
-  toolCalls: {
-    name: string
-    args: unknown
-    result?: unknown
-    error?: string
-  }[]
-  usage: {
-    inputTokens: number
-    outputTokens: number
-    model?: string
-  }
-}
-
-export interface ScorerOutput {
-  /** 0..1, so grades are comparable across scorers. */
-  score: number
-  reason?: string
-  metadata?: Record<string, unknown>
-}
-
-/**
- * The queue a scorer's jobs go to. Two lanes exist only so a flood of slow
- * LLM-judge jobs cannot starve the cheap heuristic ones; within a lane it is
- * plain FIFO.
- */
-export type ScorerLane = 'fast' | 'slow'
-
-export const SCORER_LANE_QUEUES: Record<ScorerLane, string> = {
-  fast: 'agent-score-fast',
-  slow: 'agent-score-slow',
-}
-
-/**
- * How much of the run's trajectory the default judge prompt discloses.
- *
- * A judge is a third-party model, and a tool's arguments and results are the
- * most sensitive thing a run touches — the row it read, the address it looked
- * up, the message it sent. Output middleware has already had its pass by the
- * time a scorer sees any of this, so what a judge is shown is what was stored;
- * this is the second gate, for a judge that should be told less than the
- * database was.
- *
- * - `names` — which tools ran, and whether each failed. No arguments, no
- *   results. Enough to tell an answer backed by a tool from an invented one,
- *   which is the failure this exists to catch, while disclosing no row data.
- * - `full` — names, arguments and results, truncated. For a judge that must
- *   check the answer *against* what the tool returned.
- * - `off` — no trajectory at all. The judge grades the answer alone.
- */
-export type JudgeToolCallDisclosure = 'off' | 'names' | 'full'
-
-/**
- * A judge's model call. Present only on a scorer built with `pikkuAgentJudge`;
- * `score` is present only on one built with `pikkuAgentScorer`. Exactly one of the
- * two is set, which is what the two constructors exist to guarantee.
- */
-export type ScorerJudgeConfig = {
-  model: string
-  goal: string
-  /** Always set: `pikkuAgentJudge` resolves the default. */
-  toolCalls: JudgeToolCallDisclosure
-  prompt?: (input: ScorerInput) => string
-}
-
-export type PikkuAgentScorer<Services = any> = {
-  name: string
-  description: string
-  lane: ScorerLane
-  /** 0..1 fraction of live runs to grade. */
-  sampleRate: number
-  /**
-   * Grades against a known-correct answer, so it is test-only: live traffic has
-   * no answer key and the runtime never samples it.
-   */
-  requiresReference: boolean
-  score?: (
-    input: ScorerInput,
-    services: Services
-  ) => ScorerOutput | Promise<ScorerOutput>
-  judge?: ScorerJudgeConfig
-}
-
-export type ScorerMeta = Record<
-  string,
-  {
-    name: string
-    description: string
-    lane: ScorerLane
-    sampleRate: number
-    requiresReference: boolean
-    sourceFile?: string
-    exportedName?: string
-  }
->
-
-/** A single scorer's job on a lane queue. */
-export type ScoreJob = ScorerInput & {
-  scorerName: string
-}
-
-export type AgentRunScore = {
-  runId: string
-  scorerName: string
-  score: number
-  reason?: string
-  metadata?: Record<string, unknown>
-  createdAt: Date
-}

package/src/wirings/agent-scorer/index.ts

@@ -1,23 +0,0 @@
-export { pikkuAgentScorer, pikkuAgentJudge } from './agent-scorer.js'
-export {
-  addAgentScorer,
-  getAgentScorers,
-  getAgentScorersMeta,
-} from './agent-scorer-registry.js'
-export { gradeRun } from './agent-scorer-grade.js'
-export {
-  enableScoreSnapshots,
-  getScoreSnapshot,
-} from './agent-scorer-snapshots.js'
-export { wireAgentScorerQueueWorkers } from './agent-scorer-worker.js'
-export type {
-  AgentRunScore,
-  JudgeToolCallDisclosure,
-  PikkuAgentScorer,
-  ScoreJob,
-  ScorerInput,
-  ScorerJudgeConfig,
-  ScorerLane,
-  ScorerMeta,
-  ScorerOutput,
-} from './agent-scorer.types.js'