@pikku/core 0.12.80 → 0.12.83

package/src/wirings/ai-scorer/ai-scorer-snapshots.ts

@@ -0,0 +1,46 @@
+import type { ScorerInput } from './ai-scorer.types.js'
+
+/**
+ * The last few finished runs, held so a scenario can grade one.
+ *
+ * A scenario cannot rebuild a `ScorerInput` from storage: the run record holds
+ * status and usage, and the prompt, answer and tool calls are spread across the
+ * thread's messages, where the boundary of a single run is not recoverable. So
+ * the runtime keeps the snapshot it already took at finalize — which is also
+ * what makes a scenario's grade comparable to a live one, since both grade the
+ * identical object, redactions included.
+ *
+ * Off unless something turns it on, and bounded when it is. A process that
+ * grades no scenarios holds nothing, and one that does cannot accumulate run
+ * content without limit.
+ */
+const snapshots = new Map<string, ScorerInput>()
+
+let limit = 0
+
+/**
+ * Start retaining snapshots. Called where the scenario grading RPC is
+ * registered, so the buffer exists in exactly the processes that can read it —
+ * a development server — and never in a deployed bundle.
+ */
+export const enableScoreSnapshots = (maxRuns = 50): void => {
+  limit = maxRuns
+}
+
+export const recordScoreSnapshot = (run: ScorerInput): void => {
+  if (limit === 0) return
+  snapshots.set(run.runId, run)
+  while (snapshots.size > limit) {
+    const oldest = snapshots.keys().next().value
+    if (oldest === undefined) break
+    snapshots.delete(oldest)
+  }
+}
+
+export const getScoreSnapshot = (runId: string): ScorerInput | undefined =>
+  snapshots.get(runId)
+
+export const resetScoreSnapshots = (): void => {
+  snapshots.clear()
+  limit = 0
+}

package/src/wirings/ai-scorer/ai-scorer-worker.test.ts

@@ -0,0 +1,122 @@
+import { beforeEach, describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import { pikkuState, resetPikkuState } from '../../pikku-state.js'
+import { pikkuAIScoreWorkerFunc } from './ai-scorer-worker.js'
+import { pikkuAIJudge, pikkuAIScorer } from './ai-scorer.js'
+import type { PikkuAIScorer, ScoreJob } from './ai-scorer.types.js'
+
+const job = (overrides: Partial<ScoreJob> = {}): ScoreJob => ({
+  scorerName: 'brevity',
+  runId: 'run-1',
+  agentName: 'assistant',
+  input: 'what is the capital of France?',
+  output: 'Paris',
+  toolCalls: [],
+  usage: { inputTokens: 10, outputTokens: 5 },
+  ...overrides,
+})
+
+const install = (scorer: PikkuAIScorer, aiAgentRunner?: unknown) => {
+  pikkuState(null, 'agent', 'scorers').set(scorer.name, scorer)
+  const saved: unknown[] = []
+  pikkuState(null, 'package', 'singletonServices', {
+    aiRunState: {
+      saveScore: async (score: unknown) => {
+        saved.push(score)
+      },
+    },
+    aiAgentRunner,
+  } as never)
+  return saved
+}
+
+describe('pikkuAIScoreWorkerFunc', () => {
+  beforeEach(() => {
+    resetPikkuState()
+  })
+
+  test('records what a heuristic scorer graded, against the run it graded', async () => {
+    const saved = install(
+      pikkuAIScorer({
+        name: 'brevity',
+        description: 'Shorter is better',
+        score: (input) => ({
+          score: input.output.length < 20 ? 1 : 0,
+          reason: 'Answered in one word.',
+        }),
+      })
+    )
+
+    await pikkuAIScoreWorkerFunc({}, job())
+
+    assert.deepEqual(saved, [
+      {
+        runId: 'run-1',
+        scorerName: 'brevity',
+        score: 1,
+        reason: 'Answered in one word.',
+      },
+    ])
+  })
+
+  test('hands a judge to the model rather than calling a score function it has not got', async () => {
+    const saved = install(
+      pikkuAIJudge({
+        name: 'helpfulness',
+        description: 'Is the answer useful',
+        model: 'claude-opus-5',
+        goal: 'Grade helpfulness.',
+      }),
+      {
+        run: async () => ({
+          object: { score: 0.5, reason: 'Terse.' },
+          usage: { inputTokens: 80, outputTokens: 20 },
+        }),
+      }
+    )
+
+    await pikkuAIScoreWorkerFunc({}, job({ scorerName: 'helpfulness' }))
+
+    assert.deepEqual(saved, [
+      {
+        runId: 'run-1',
+        scorerName: 'helpfulness',
+        score: 0.5,
+        reason: 'Terse.',
+        metadata: { judgeModel: 'claude-opus-5', judgeTokens: 100 },
+      },
+    ])
+  })
+
+  test('fails the job when there is nowhere to record the grade, so it is retried rather than lost', async () => {
+    pikkuState(null, 'agent', 'scorers').set(
+      'brevity',
+      pikkuAIScorer({
+        name: 'brevity',
+        description: 'Shorter is better',
+        score: () => ({ score: 1 }),
+      })
+    )
+    pikkuState(null, 'package', 'singletonServices', {} as never)
+
+    await assert.rejects(
+      () => pikkuAIScoreWorkerFunc({}, job()),
+      /AI run state service not initialized/
+    )
+  })
+
+  test('fails on a scorer name that resolves to nothing', async () => {
+    install(
+      pikkuAIScorer({
+        name: 'brevity',
+        description: 'Shorter is better',
+        score: () => ({ score: 1 }),
+      })
+    )
+
+    await assert.rejects(
+      () => pikkuAIScoreWorkerFunc({}, job({ scorerName: 'gone' })),
+      /AI scorer not found: gone/
+    )
+  })
+})

package/src/wirings/ai-scorer/ai-scorer-worker.ts

@@ -0,0 +1,69 @@
+import { pikkuState } from '../../pikku-state.js'
+import { addFunction } from '../../function/function-runner.js'
+import { wireQueueWorker } from '../queue/queue-runner.js'
+import { gradeRun } from './ai-scorer-grade.js'
+import { SCORER_LANE_QUEUES, type ScoreJob } from './ai-scorer.types.js'
+
+/**
+ * The one worker behind both lanes: it resolves the scorer by name and grades.
+ * The lane a job arrived on decides nothing except how long it may take.
+ */
+export async function pikkuAIScoreWorkerFunc(
+  _services: Record<string, unknown>,
+  job: ScoreJob
+): Promise<void> {
+  const services = pikkuState(null, 'package', 'singletonServices')
+  if (!services) {
+    throw new Error(
+      `Singleton services not initialized: cannot run the '${job.scorerName}' grade of run ${job.runId}`
+    )
+  }
+
+  await gradeRun(job, services, { persist: true })
+}
+
+const SCORE_WORKER_FUNC_ID = 'pikkuAIScoreWorker'
+
+/**
+ * Bind the two lane queues to the shared worker.
+ *
+ * Registered programmatically rather than emitted into the user's bootstrap:
+ * `wireQueueWorker` warns and returns for a name codegen produced no metadata
+ * for, so the metadata is synthesised here alongside the registration. This
+ * mirrors how workflows wire their own queues.
+ */
+export const wireAIScorerQueueWorkers = (): void => {
+  // No scorers means no lanes: a deployment that grades nothing should not be
+  // left holding two queues nothing ever writes to.
+  if (pikkuState(null, 'agent', 'scorers').size === 0) return
+
+  const functions = pikkuState(null, 'function', 'functions')
+  const functionsMeta = pikkuState(null, 'function', 'meta')
+  const queueMeta = pikkuState(null, 'queue', 'meta')
+
+  if (!functions.has(SCORE_WORKER_FUNC_ID)) {
+    addFunction(SCORE_WORKER_FUNC_ID, { func: pikkuAIScoreWorkerFunc } as never)
+  }
+  if (!functionsMeta[SCORE_WORKER_FUNC_ID]) {
+    functionsMeta[SCORE_WORKER_FUNC_ID] = {
+      pikkuFuncId: SCORE_WORKER_FUNC_ID,
+      sessionless: true,
+      functionType: 'helper',
+      inputSchemaName: null,
+      outputSchemaName: null,
+    }
+  }
+
+  for (const queueName of Object.values(SCORER_LANE_QUEUES)) {
+    if (!queueMeta[queueName]) {
+      queueMeta[queueName] = {
+        pikkuFuncId: SCORE_WORKER_FUNC_ID,
+        name: queueName,
+      }
+    }
+    wireQueueWorker({
+      name: queueName,
+      func: { func: pikkuAIScoreWorkerFunc },
+    } as never)
+  }
+}

package/src/wirings/ai-scorer/ai-scorer.ts

@@ -0,0 +1,76 @@
+import type {
+  PikkuAIScorer,
+  ScorerInput,
+  ScorerOutput,
+} from './ai-scorer.types.js'
+
+const assertSampleRate = (name: string, sampleRate: number | undefined) => {
+  if (sampleRate === undefined) return 1
+  if (sampleRate < 0 || sampleRate > 1) {
+    throw new Error(
+      `Scorer '${name}' has a sampleRate of ${sampleRate} — it is a fraction of live runs to grade, so it must be between 0 and 1`
+    )
+  }
+  return sampleRate
+}
+
+/**
+ * A heuristic scorer: pure code over the finished run, no model call, so it
+ * grades on the fast lane.
+ */
+export const pikkuAIScorer = <Services = any>(config: {
+  name: string
+  description: string
+  /** 0..1 fraction of live runs to grade. Defaults to all of them. */
+  sampleRate?: number
+  /**
+   * Grades against a known-correct answer. Such a scorer is test-only — live
+   * traffic has no answer key, so the runtime never samples it.
+   */
+  requiresReference?: boolean
+  score: (
+    input: ScorerInput,
+    services: Services
+  ) => ScorerOutput | Promise<ScorerOutput>
+}): PikkuAIScorer<Services> => ({
+  name: config.name,
+  description: config.description,
+  lane: 'fast',
+  sampleRate: assertSampleRate(config.name, config.sampleRate),
+  requiresReference: config.requiresReference ?? false,
+  score: config.score,
+})
+
+/**
+ * An LLM-judge scorer: the runtime makes the model call and forces a structured
+ * `{ score, reason }`, so a judge is a rubric rather than a prompt to parse.
+ *
+ * The rubric field is `goal`, matching `pikkuAIAgent`'s prompt vocabulary — a
+ * judge is a degenerate agent, and should use the same word for the same thing.
+ * `prompt` is the escape hatch for non-standard framing.
+ */
+export const pikkuAIJudge = <Services = any>(config: {
+  name: string
+  description: string
+  /** 0..1 fraction of live runs to grade. Defaults to all of them. */
+  sampleRate?: number
+  /**
+   * Grades against a known-correct answer. Such a judge is test-only — live
+   * traffic has no answer key, so the runtime never samples it.
+   */
+  requiresReference?: boolean
+  model: string
+  goal: string
+  prompt?: (input: ScorerInput) => string
+}): PikkuAIScorer<Services> => ({
+  name: config.name,
+  description: config.description,
+  lane: 'slow',
+  sampleRate: assertSampleRate(config.name, config.sampleRate),
+  requiresReference: config.requiresReference ?? false,
+  judge: {
+    model: config.model,
+    goal: config.goal,
+    ...(config.prompt ? { prompt: config.prompt } : {}),
+  },
+})

package/src/wirings/ai-scorer/ai-scorer.types.ts

@@ -0,0 +1,107 @@
+/**
+ * What a scorer is shown about a finished run.
+ *
+ * One snapshot, taken after the output middleware has resolved, so a scorer and
+ * the persisted run record see the same thing — including the redactions.
+ */
+export interface ScorerInput {
+  runId: string
+  agentName: string
+  threadId?: string
+  resourceId?: string
+  /** The user prompt the run answered. */
+  input: string
+  /** The agent's final text answer. */
+  output: string
+  /**
+   * The known-correct answer. Supplied by a scenario for a reference-based
+   * judge; never set on live traffic, which has no answer key.
+   */
+  reference?: string
+  toolCalls: {
+    name: string
+    args: unknown
+    result?: unknown
+    error?: string
+  }[]
+  usage: {
+    inputTokens: number
+    outputTokens: number
+    model?: string
+  }
+}
+
+export interface ScorerOutput {
+  /** 0..1, so grades are comparable across scorers. */
+  score: number
+  reason?: string
+  metadata?: Record<string, unknown>
+}
+
+/**
+ * The queue a scorer's jobs go to. Two lanes exist only so a flood of slow
+ * LLM-judge jobs cannot starve the cheap heuristic ones; within a lane it is
+ * plain FIFO.
+ */
+export type ScorerLane = 'fast' | 'slow'
+
+export const SCORER_LANE_QUEUES: Record<ScorerLane, string> = {
+  fast: 'ai-score-fast',
+  slow: 'ai-score-slow',
+}
+
+/**
+ * A judge's model call. Present only on a scorer built with `pikkuAIJudge`;
+ * `score` is present only on one built with `pikkuAIScorer`. Exactly one of the
+ * two is set, which is what the two constructors exist to guarantee.
+ */
+export type ScorerJudgeConfig = {
+  model: string
+  goal: string
+  prompt?: (input: ScorerInput) => string
+}
+
+export type PikkuAIScorer<Services = any> = {
+  name: string
+  description: string
+  lane: ScorerLane
+  /** 0..1 fraction of live runs to grade. */
+  sampleRate: number
+  /**
+   * Grades against a known-correct answer, so it is test-only: live traffic has
+   * no answer key and the runtime never samples it.
+   */
+  requiresReference: boolean
+  score?: (
+    input: ScorerInput,
+    services: Services
+  ) => ScorerOutput | Promise<ScorerOutput>
+  judge?: ScorerJudgeConfig
+}
+
+export type ScorerMeta = Record<
+  string,
+  {
+    name: string
+    description: string
+    lane: ScorerLane
+    sampleRate: number
+    requiresReference: boolean
+    sourceFile?: string
+    exportedName?: string
+  }
+>
+
+/** A single scorer's job on a lane queue. */
+export type ScoreJob = ScorerInput & {
+  scorerName: string
+}
+
+export type AIRunScore = {
+  runId: string
+  scorerName: string
+  score: number
+  reason?: string
+  metadata?: Record<string, unknown>
+  createdAt: Date
+}

package/src/wirings/ai-scorer/index.ts

@@ -0,0 +1,24 @@
+export { pikkuAIScorer, pikkuAIJudge } from './ai-scorer.js'
+export {
+  addAIScorer,
+  getAIScorers,
+  getAIScorersMeta,
+} from './ai-scorer-registry.js'
+export { gradeRun } from './ai-scorer-grade.js'
+export {
+  enableScoreSnapshots,
+  getScoreSnapshot,
+} from './ai-scorer-snapshots.js'
+export {
+  wireAIScorerQueueWorkers,
+} from './ai-scorer-worker.js'
+export type {
+  AIRunScore,
+  PikkuAIScorer,
+  ScoreJob,
+  ScorerInput,
+  ScorerJudgeConfig,
+  ScorerLane,
+  ScorerMeta,
+  ScorerOutput,
+} from './ai-scorer.types.js'

package/src/wirings/channel/index.ts

@@ -1,8 +1,7 @@
-export { wireChannel, openChannel } from './channel-runner.js'
+export { wireChannel } from './channel-runner.js'
 export {
   addChannelMiddleware,
   combineChannelMiddleware,
-  wrapChannelWithMiddleware,
 } from './channel-middleware-runner.js'
 export { logChannels } from './log-channels.js'
 export { PikkuAbstractChannelHandler } from './pikku-abstract-channel-handler.js'
@@ -18,25 +17,14 @@ export type {
   CorePikkuChannelMiddlewareFactory,
   ChannelMessageMeta,
   ChannelMeta,
-  ChannelRemote,
   PikkuChannel,
   PikkuChannelHandlerFactory,
 } from './channel.types.js'
 export { defineChannelRoutes } from './define-channel-routes.js'
 export {
-  CHANNEL_RPC_PENDING,
-  CHANNEL_RPC_REQUEST,
-  CHANNEL_RPC_RESPONSE,
   ChannelDeploymentService,
   ChannelRPCError,
   ChannelRPCRegistry,
-  createChannelRPCResponder,
-  createChannelRPCInputValidator,
-  createChannelRPCResultValidator,
-  isChannelRPCPending,
-  isChannelRPCRequest,
-  isChannelRPCResponse,
-  resolveCapability,
   unsupportedChannelRemote,
 } from './channel-rpc.js'
 export type {
@@ -44,16 +32,9 @@ export type {
   ApprovalRequester,
   Capabilities,
   Capability,
-  CapabilityDef,
   CapabilityHandler,
   ChannelRPCPending,
   ChannelRPCRequest,
   ChannelRPCResponse,
   ChannelRPCValidator,
 } from './channel-rpc.js'
-export {
-  channelRemote,
-  getChannelHostRPC,
-  handleChannelRPCResponse,
-  releaseChannelHostRPC,
-} from './channel-host-rpc.js'

package/src/wirings/channel/local/local-channel-runner.test.ts

@@ -3,6 +3,7 @@ import * as assert from 'node:assert/strict'
 import { runLocalChannel } from './local-channel-runner.js'
 import { pikkuState, resetPikkuState } from '../../../pikku-state.js'
 import { wireChannel } from '../channel-runner.js'
+import { addFunction } from '../../../function/function-runner.js'
 import { addHTTPMiddleware } from '../../http/http-runner.js'
 import type {
   HTTPMethod,
@@ -270,3 +271,70 @@ test('runChannel should run HTTP middleware on websocket upgrade and establish s
   await result.message(JSON.stringify({ action: 'ping' }))
   assert.deepEqual(sent, { action: 'ping' })
 })
+
+test('a message handler that returns nothing does not attempt to send', async () => {
+  // The connect path already guards this; the message path did not, so a handler
+  // with nothing to say produced `send requires a non-empty message` on every
+  // message. Gateway websockets hit it every time — their generated message
+  // handler returns undefined by design — which is why a chat gateway could
+  // accept a connection and never deliver anything to its handler.
+  resetPikkuState()
+  pikkuState(null, 'package', 'singletonServices', mockSingletonServices as any)
+  pikkuState(null, 'package', 'factories', {
+    createWireServices: mockCreateWireServices,
+  } as any)
+
+  const sent: unknown[] = []
+  let handled = 0
+
+  pikkuState(null, 'channel', 'meta', {
+    quiet: {
+      name: 'quiet',
+      route: '/quiet-channel',
+      message: { pikkuFuncId: 'quietMessage' },
+      messageWirings: {},
+    },
+  } as any)
+  pikkuState(null, 'function', 'meta')['quietMessage'] = {
+    pikkuFuncId: 'quietMessage',
+    inputSchemaName: null,
+    outputSchemaName: null,
+    sessionless: true,
+  } as any
+  const quietMessage = {
+    auth: false,
+    func: async () => {
+      handled++
+      return undefined
+    },
+  }
+  wireChannel({
+    name: 'quiet',
+    route: '/quiet-channel',
+    auth: false,
+    onMessage: quietMessage as any,
+  } as any)
+
+  httpRouter.initialize()
+
+  const handler = await runLocalChannel({
+    channelId: 'quiet-channel-id',
+    request: new PikkuMockRequest('/quiet-channel', 'get'),
+    response: new PikkuMockResponse(),
+    route: '/quiet-channel',
+  })
+  assert.ok(handler)
+  handler.registerOnSend(async (message: unknown) => {
+    sent.push(message)
+  })
+  handler.open()
+
+  await handler.message(JSON.stringify({ text: 'hello' }))
+
+  assert.equal(handled, 1, 'the handler should have run')
+  assert.deepEqual(
+    sent,
+    [],
+    'nothing should have been sent for an empty result'
+  )
+})

package/src/wirings/channel/local/local-channel-runner.ts

@@ -213,7 +213,14 @@ export const runLocalChannel = async ({
       channelHandler.registerOnMessage(async (data) => {
         try {
           const result = await onMessage(data)
-          await channel.send(result)
+          // Guarded exactly as the connect path above is. A handler with nothing
+          // to say is normal — a gateway websocket's generated message handler
+          // always returns undefined — and sending that raised `send requires a
+          // non-empty message`, which surfaced as a channel that accepted a
+          // connection and then silently delivered nothing.
+          if (result !== undefined) {
+            await channel.send(result)
+          }
         } catch (e: any) {
           singletonServices.logger.error(e)
           const errorResponse = getErrorResponse(e)

package/src/wirings/cli/channel/cli-raw-channel-runner.test.ts

@@ -103,6 +103,29 @@ describe('handleRawCLI', () => {
     assert.match(result.error!, /deploy plan is red/)
   })
 
+  test('masks the raw error message to the client in production', async () => {
+    const prior = process.env.NODE_ENV
+    process.env.NODE_ENV = 'production'
+    try {
+      wireTestCLI(() => {
+        throw new Error('connection to internal-db:5432 failed: bad password')
+      })
+
+      const result = await handleRawCLI({
+        programName: 'fabric',
+        args: ['deploy'],
+        singletonServices,
+      })
+
+      assert.equal(result.exitCode, 1)
+      assert.equal(result.error, 'Command failed')
+      assert.doesNotMatch(result.error!, /internal-db|password/)
+    } finally {
+      if (prior === undefined) delete process.env.NODE_ENV
+      else process.env.NODE_ENV = prior
+    }
+  })
+
   test('exits 0 on help and 1 on an unknown command', async () => {
     wireTestCLI(() => ({}))
 

package/src/wirings/cli/channel/cli-raw-channel-runner.ts

@@ -1,4 +1,5 @@
 import { pikkuState } from '../../../pikku-state.js'
+import { isProduction } from '../../../env.js'
 import type { CLIMeta } from '../cli.types.js'
 import { parseCLIArguments, generateCommandHelp } from '../command-parser.js'
 import { runCLICommand } from '../cli-runner.js'
@@ -132,6 +133,16 @@ export async function handleRawCLI({
     // A command can throw anything; `.message` off a string is undefined, and
     // the run would then exit 1 saying nothing at all.
     const message = e instanceof Error ? e.message : String(e)
-    return { error: message || 'Command failed', exitCode: 1, commandId }
+    // The raw message can carry internals (a stack, a DB error, a path). This
+    // runs over a channel that may be remote, so return a generic error in
+    // production and keep the detail server-side. Dev keeps the message inline.
+    singletonServices.logger.error?.(
+      `CLI channel command failed: ${message}`,
+      e
+    )
+    const clientError = isProduction()
+      ? 'Command failed'
+      : message || 'Command failed'
+    return { error: clientError, exitCode: 1, commandId }
   }
 }

package/src/wirings/cli/channel/index.ts

@@ -3,13 +3,6 @@ export { handleRawCLI } from './cli-raw-channel-runner.js'
 export type { RawCLIFrame, RawCLIResult } from './cli-raw-channel-runner.js'
 export { executeRawCLIViaChannel } from './cli-raw-client-runner.js'
 export type {
-  ClientCLIRenderServices,
   CorePikkuCLIClientRender,
 } from './cli-raw-client-runner.js'
-export {
-  APPROVAL_FLAGS,
-  approverForMode,
-  createTerminalApprover,
-  takeApprovalFlags,
-} from './cli-approval.js'
 export type { ApprovalMode } from './cli-approval.js'