@pikku/core 0.12.79 → 0.12.82

package/src/wirings/ai-scorer/ai-scorer.ts

@@ -0,0 +1,76 @@
+import type {
+  PikkuAIScorer,
+  ScorerInput,
+  ScorerOutput,
+} from './ai-scorer.types.js'
+
+const assertSampleRate = (name: string, sampleRate: number | undefined) => {
+  if (sampleRate === undefined) return 1
+  if (sampleRate < 0 || sampleRate > 1) {
+    throw new Error(
+      `Scorer '${name}' has a sampleRate of ${sampleRate} — it is a fraction of live runs to grade, so it must be between 0 and 1`
+    )
+  }
+  return sampleRate
+}
+
+/**
+ * A heuristic scorer: pure code over the finished run, no model call, so it
+ * grades on the fast lane.
+ */
+export const pikkuAIScorer = <Services = any>(config: {
+  name: string
+  description: string
+  /** 0..1 fraction of live runs to grade. Defaults to all of them. */
+  sampleRate?: number
+  /**
+   * Grades against a known-correct answer. Such a scorer is test-only — live
+   * traffic has no answer key, so the runtime never samples it.
+   */
+  requiresReference?: boolean
+  score: (
+    input: ScorerInput,
+    services: Services
+  ) => ScorerOutput | Promise<ScorerOutput>
+}): PikkuAIScorer<Services> => ({
+  name: config.name,
+  description: config.description,
+  lane: 'fast',
+  sampleRate: assertSampleRate(config.name, config.sampleRate),
+  requiresReference: config.requiresReference ?? false,
+  score: config.score,
+})
+
+/**
+ * An LLM-judge scorer: the runtime makes the model call and forces a structured
+ * `{ score, reason }`, so a judge is a rubric rather than a prompt to parse.
+ *
+ * The rubric field is `goal`, matching `pikkuAIAgent`'s prompt vocabulary — a
+ * judge is a degenerate agent, and should use the same word for the same thing.
+ * `prompt` is the escape hatch for non-standard framing.
+ */
+export const pikkuAIJudge = <Services = any>(config: {
+  name: string
+  description: string
+  /** 0..1 fraction of live runs to grade. Defaults to all of them. */
+  sampleRate?: number
+  /**
+   * Grades against a known-correct answer. Such a judge is test-only — live
+   * traffic has no answer key, so the runtime never samples it.
+   */
+  requiresReference?: boolean
+  model: string
+  goal: string
+  prompt?: (input: ScorerInput) => string
+}): PikkuAIScorer<Services> => ({
+  name: config.name,
+  description: config.description,
+  lane: 'slow',
+  sampleRate: assertSampleRate(config.name, config.sampleRate),
+  requiresReference: config.requiresReference ?? false,
+  judge: {
+    model: config.model,
+    goal: config.goal,
+    ...(config.prompt ? { prompt: config.prompt } : {}),
+  },
+})

package/src/wirings/ai-scorer/ai-scorer.types.ts

@@ -0,0 +1,107 @@
+/**
+ * What a scorer is shown about a finished run.
+ *
+ * One snapshot, taken after the output middleware has resolved, so a scorer and
+ * the persisted run record see the same thing — including the redactions.
+ */
+export interface ScorerInput {
+  runId: string
+  agentName: string
+  threadId?: string
+  resourceId?: string
+  /** The user prompt the run answered. */
+  input: string
+  /** The agent's final text answer. */
+  output: string
+  /**
+   * The known-correct answer. Supplied by a scenario for a reference-based
+   * judge; never set on live traffic, which has no answer key.
+   */
+  reference?: string
+  toolCalls: {
+    name: string
+    args: unknown
+    result?: unknown
+    error?: string
+  }[]
+  usage: {
+    inputTokens: number
+    outputTokens: number
+    model?: string
+  }
+}
+
+export interface ScorerOutput {
+  /** 0..1, so grades are comparable across scorers. */
+  score: number
+  reason?: string
+  metadata?: Record<string, unknown>
+}
+
+/**
+ * The queue a scorer's jobs go to. Two lanes exist only so a flood of slow
+ * LLM-judge jobs cannot starve the cheap heuristic ones; within a lane it is
+ * plain FIFO.
+ */
+export type ScorerLane = 'fast' | 'slow'
+
+export const SCORER_LANE_QUEUES: Record<ScorerLane, string> = {
+  fast: 'ai-score-fast',
+  slow: 'ai-score-slow',
+}
+
+/**
+ * A judge's model call. Present only on a scorer built with `pikkuAIJudge`;
+ * `score` is present only on one built with `pikkuAIScorer`. Exactly one of the
+ * two is set, which is what the two constructors exist to guarantee.
+ */
+export type ScorerJudgeConfig = {
+  model: string
+  goal: string
+  prompt?: (input: ScorerInput) => string
+}
+
+export type PikkuAIScorer<Services = any> = {
+  name: string
+  description: string
+  lane: ScorerLane
+  /** 0..1 fraction of live runs to grade. */
+  sampleRate: number
+  /**
+   * Grades against a known-correct answer, so it is test-only: live traffic has
+   * no answer key and the runtime never samples it.
+   */
+  requiresReference: boolean
+  score?: (
+    input: ScorerInput,
+    services: Services
+  ) => ScorerOutput | Promise<ScorerOutput>
+  judge?: ScorerJudgeConfig
+}
+
+export type ScorerMeta = Record<
+  string,
+  {
+    name: string
+    description: string
+    lane: ScorerLane
+    sampleRate: number
+    requiresReference: boolean
+    sourceFile?: string
+    exportedName?: string
+  }
+>
+
+/** A single scorer's job on a lane queue. */
+export type ScoreJob = ScorerInput & {
+  scorerName: string
+}
+
+export type AIRunScore = {
+  runId: string
+  scorerName: string
+  score: number
+  reason?: string
+  metadata?: Record<string, unknown>
+  createdAt: Date
+}

package/src/wirings/ai-scorer/index.ts

@@ -0,0 +1,24 @@
+export { pikkuAIScorer, pikkuAIJudge } from './ai-scorer.js'
+export {
+  addAIScorer,
+  getAIScorers,
+  getAIScorersMeta,
+} from './ai-scorer-registry.js'
+export { gradeRun } from './ai-scorer-grade.js'
+export {
+  enableScoreSnapshots,
+  getScoreSnapshot,
+} from './ai-scorer-snapshots.js'
+export {
+  wireAIScorerQueueWorkers,
+} from './ai-scorer-worker.js'
+export type {
+  AIRunScore,
+  PikkuAIScorer,
+  ScoreJob,
+  ScorerInput,
+  ScorerJudgeConfig,
+  ScorerLane,
+  ScorerMeta,
+  ScorerOutput,
+} from './ai-scorer.types.js'

package/src/wirings/channel/channel-common.ts

@@ -1,3 +1,4 @@
+import type { CorePikkuChannelMiddleware } from './channel.types.js'
 import type {
   CoreSingletonServices,
   CorePikkuMiddleware,
@@ -61,7 +62,9 @@ export const runChannelLifecycleWithMiddleware = async ({
     `${channelConfig.name}:${lifecycleType}:cm`,
     {
       wireInheritedChannelMiddleware: channelMiddlewareMeta,
-      wireChannelMiddleware: channelConfig.channelMiddleware as any,
+      // knowledge: questions/channel-middleware-accepts-bare-factories-that-nothing-resolves.md
+      wireChannelMiddleware:
+        channelConfig.channelMiddleware as CorePikkuChannelMiddleware[],
     }
   )
 
@@ -76,7 +79,7 @@ export const runChannelLifecycleWithMiddleware = async ({
   const runLifecycle = async () => {
     return await runPikkuFunc('channel', channelConfig.name, meta.pikkuFuncId, {
       singletonServices: services,
-      data: () => data as any,
+      data: () => data,
       wire,
       sessionService: userSession,
       tags: meta.tags ?? [],

package/src/wirings/channel/channel-handler-shapes.test.ts

@@ -0,0 +1,72 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { pikkuState, resetPikkuState } from '../../pikku-state.js'
+import { wireChannel } from './channel-runner.js'
+
+const registeredFunc = (funcId: string) =>
+  pikkuState(null, 'function', 'functions').get(funcId)
+
+const wire = (channel: any) => {
+  resetPikkuState()
+  pikkuState(null, 'channel', 'meta', {
+    shapes: {
+      name: 'shapes',
+      route: '/shapes',
+      input: null,
+      connect: { pikkuFuncId: 'shapes__connect' },
+      disconnect: { pikkuFuncId: 'shapes__disconnect' },
+      message: { pikkuFuncId: 'shapes__message' },
+      messageWirings: {},
+    },
+  } as any)
+  wireChannel({ name: 'shapes', route: '/shapes', ...channel })
+}
+
+/**
+ * knowledge: decisions/internals/channel-message-handlers-accept-three-config-shapes.md
+ */
+describe('every channel handler shape registers a callable function config', () => {
+  const handler = async () => undefined
+
+  test('a direct function config registers as-is', () => {
+    wire({ onConnect: { func: handler } })
+
+    assert.equal(typeof registeredFunc('shapes__connect')?.func, 'function')
+  })
+
+  test('a wrapper around a function config registers the inner config', () => {
+    wire({ onConnect: { func: { func: handler }, middleware: [] } })
+
+    // Registering the wrapper itself leaves `.func` an object, and the function
+    // runner calls `.func(...)` — that is a TypeError at connect time.
+    assert.equal(
+      typeof registeredFunc('shapes__connect')?.func,
+      'function',
+      'the wrapper was registered instead of the config it wraps'
+    )
+  })
+
+  test('onDisconnect unwraps the same way', () => {
+    wire({ onDisconnect: { func: { func: handler }, middleware: [] } })
+
+    assert.equal(typeof registeredFunc('shapes__disconnect')?.func, 'function')
+  })
+
+  test('onMessage unwraps the same way', () => {
+    wire({ onMessage: { func: { func: handler } } })
+
+    assert.equal(typeof registeredFunc('shapes__message')?.func, 'function')
+  })
+
+  test('a simple wrapper — a function plus sibling middleware — stays whole', () => {
+    wire({ onConnect: { func: handler, middleware: [] } })
+
+    const registered = registeredFunc('shapes__connect')
+    assert.equal(typeof registered?.func, 'function')
+    assert.ok(
+      'middleware' in (registered as object),
+      'the sibling middleware must survive registration'
+    )
+  })
+})

package/src/wirings/channel/channel-handler.ts

@@ -4,6 +4,7 @@ import type {
   CoreUserSession,
 } from '../../types/core.types.js'
 import type {
+  CorePikkuChannelMiddleware,
   BinaryData,
   ChannelMessageMeta,
   CoreChannel,
@@ -94,7 +95,7 @@ export const processMessageHandlers = (
       logger.error(
         `Channel ${channelConfig.name} with id ${channelHandler.getChannel().channelId} requires a session for ${routeMessage}. No session is attached to this websocket connection. Ensure auth middleware establishes a session during websocket upgrade, and configure sessionStore when the runtime needs persisted sessions.`
       )
-      // TODO: Send error message back breaks typescript, but should be implemented somehow
+      // knowledge: questions/unauthorized-channel-replies-escape-the-declared-out-type.md
       channelHandler.getChannel().send(`Unauthorized for ${routeMessage}`)
       return
     }
@@ -111,10 +112,7 @@ export const processMessageHandlers = (
     // Get wire middleware: channel-level middleware + message-specific middleware
     const channelWireMiddleware = channelConfig.middleware || []
 
-    // Check if onMessage is a wrapper object vs direct function config:
-    // - Direct config: onMessage.func is a plain Function
-    // - Wrapper: onMessage.func is a CorePikkuFunctionConfig (has onMessage.func.func)
-    // - Simple wrapper: onMessage has both func (plain Function) and middleware properties
+    // knowledge: decisions/internals/channel-message-handlers-accept-three-config-shapes.md
     const isWrapper =
       onMessage &&
       typeof onMessage === 'object' &&
@@ -145,7 +143,9 @@ export const processMessageHandlers = (
       inheritedMiddleware,
       wireMiddleware,
       inheritedChannelMiddleware: channelMeta?.channelMiddleware,
-      wireChannelMiddleware: wireChannelMiddleware as any,
+      // knowledge: questions/channel-middleware-accepts-bare-factories-that-nothing-resolves.md
+      wireChannelMiddleware:
+        wireChannelMiddleware as CorePikkuChannelMiddleware[],
       coerceDataFromSchema: true,
       tags: channelConfig.tags,
       sessionService: userSession,
@@ -191,7 +191,7 @@ export const processMessageHandlers = (
 
             processed = true
             const routeResult = await processMessage(
-              data as any,
+              data,
               routes[routerValue],
               routingProperty,
               routerValue

package/src/wirings/channel/channel-rpc-service.ts

@@ -10,20 +10,6 @@ import {
   type ChannelRPCValidator,
 } from './channel-rpc.types.js'
 
-/**
- * `remote` for channels that only ever flow one way — a server-sent stream, an
- * agent's output, a local CLI. Nothing is listening for a request on them, so
- * the call is refused rather than waiting out a timeout.
- */
-export const unsupportedChannelRemote = async (
-  funcName: string
-): Promise<never> => {
-  throw new ChannelRPCError(
-    `Cannot call "${funcName}" remotely: this channel has no peer that answers`,
-    'unsupported'
-  )
-}
-
 /**
  * A `DeploymentService` whose transport is an open channel rather than an
  * address, so a server-side function can call back into a peer that has no

package/src/wirings/channel/channel-rpc.test.ts

@@ -109,21 +109,42 @@ describe('ChannelRPCRegistry', () => {
     })
   })
 
-  test('times out instead of hanging forever', async () => {
+  /**
+   * The registry unrefs its timeout timer so an in-flight call can never hold
+   * a process open. That makes the timeout unawaitable in a test: when the
+   * pending call is the only thing left, the loop drains before the timer
+   * fires, and node:test cancels the file — taking every later test with it.
+   *
+   * Driving the clock by hand removes the dependency on real time entirely.
+   * `t.mock.timers` is scoped to the test and resets itself, so the tests that
+   * do want a real clock are unaffected.
+   *
+   * knowledge: decisions/internals/an-unref-d-timer-cannot-be-awaited-under-node-test.md
+   */
+  test('times out instead of hanging forever', async (t) => {
+    t.mock.timers.enable({ apis: ['setTimeout'] })
+
     const registry = new ChannelRPCRegistry(10)
     const call = registry.register()
 
-    await assert.rejects(call.promise, (e: ChannelRPCError) => {
+    const rejected = assert.rejects(call.promise, (e: ChannelRPCError) => {
       assert.equal(e.reason, 'timeout')
       return true
     })
+    t.mock.timers.tick(10)
+    await rejected
+
     assert.equal(registry.inFlight, 0, 'a timed out call is not leaked')
   })
 
-  test('drops a late response for an already-timed-out call', async () => {
+  test('drops a late response for an already-timed-out call', async (t) => {
+    t.mock.timers.enable({ apis: ['setTimeout'] })
+
     const registry = new ChannelRPCRegistry(10)
     const call = registry.register()
-    await assert.rejects(call.promise)
+    const rejected = assert.rejects(call.promise)
+    t.mock.timers.tick(10)
+    await rejected
 
     const settled = registry.settle({
       action: CHANNEL_RPC_RESPONSE,

package/src/wirings/channel/channel-rpc.types.ts

@@ -148,3 +148,17 @@ export type ChannelRPCValidator = (
   funcName: string,
   value: unknown
 ) => Promise<void> | void
+
+/**
+ * `remote` for wires with no peer that answers — a server-sent stream, an
+ * agent's output, a local CLI. Nothing is listening for a request on them, so
+ * the call is refused rather than waiting out a timeout.
+ */
+export const unsupportedChannelRemote = async (
+  funcName: string
+): Promise<never> => {
+  throw new ChannelRPCError(
+    `Cannot call "${funcName}" remotely: this channel has no peer that answers`,
+    'unsupported'
+  )
+}

package/src/wirings/channel/channel-runner.ts

@@ -1,10 +1,9 @@
 import { NotFoundError } from '../../errors/errors.js'
 import { addFunction } from '../../function/function-runner.js'
-import type { CorePikkuPermission } from '../../function/functions.types.js'
+import type { CorePikkuFunctionConfig } from '../../function/functions.types.js'
 import { pikkuState, getSingletonServices } from '../../pikku-state.js'
 import { coerceTopLevelDataFromSchema, validateSchema } from '../../schema.js'
 import type { SessionService } from '../../services/user-session-service.js'
-import type { CorePikkuMiddleware } from '../../types/core.types.js'
 import { httpRouter } from '../http/routers/http-router.js'
 import type {
   ChannelMeta,
@@ -13,19 +12,34 @@ import type {
   RunChannelParams,
 } from './channel.types.js'
 
+type HandlerConfig = CorePikkuFunctionConfig<any, any>
+
+/**
+ * The config to register for a channel handler: the handler itself when it is a
+ * direct function config, or its inner `func` when it is a wrapper.
+ *
+ * knowledge: decisions/internals/channel-message-handlers-accept-three-config-shapes.md
+ */
+const registeredHandler = (handler: unknown): HandlerConfig => {
+  const candidate = handler as { func?: unknown }
+  return typeof candidate.func === 'function'
+    ? (handler as HandlerConfig)
+    : (candidate.func as HandlerConfig)
+}
+
 export const wireChannel = <
   In,
   Channel extends string,
-  PikkuPermission extends CorePikkuPermission<In>,
-  PikkuMiddleware extends CorePikkuMiddleware,
-  ChannelFunction,
+  ChannelConnect,
+  ChannelDisconnect,
+  ChannelFunctionMessage,
 >(
   channel: CoreChannel<
     In,
     Channel,
-    PikkuPermission,
-    PikkuMiddleware,
-    ChannelFunction
+    ChannelConnect,
+    ChannelDisconnect,
+    ChannelFunctionMessage
   >
 ) => {
   const channelsMeta = pikkuState(null, 'channel', 'meta')
@@ -38,19 +52,23 @@ export const wireChannel = <
   }
 
   if (channel.onConnect && channelMeta.connect) {
-    addFunction(channelMeta.connect.pikkuFuncId, channel.onConnect as any)
+    addFunction(
+      channelMeta.connect.pikkuFuncId,
+      registeredHandler(channel.onConnect)
+    )
   }
 
   if (channel.onDisconnect && channelMeta.disconnect) {
-    addFunction(channelMeta.disconnect.pikkuFuncId, channel.onDisconnect as any)
+    addFunction(
+      channelMeta.disconnect.pikkuFuncId,
+      registeredHandler(channel.onDisconnect)
+    )
   }
 
   if (channel.onMessage && channelMeta.message?.pikkuFuncId) {
     addFunction(
       channelMeta.message.pikkuFuncId,
-      (channel.onMessage as any).func instanceof Function
-        ? channel.onMessage
-        : (channel.onMessage as any).func
+      registeredHandler(channel.onMessage)
     )
   }
 
@@ -63,17 +81,12 @@ export const wireChannel = <
         const wiringMeta = channelWirings[wiringKey]
         if (!wiringMeta) return
 
-        addFunction(
-          wiringMeta.pikkuFuncId,
-          (handler as any).func instanceof Function
-            ? handler
-            : (handler as any).func
-        )
+        addFunction(wiringMeta.pikkuFuncId, registeredHandler(handler))
       })
     })
   }
 
-  pikkuState(null, 'channel', 'channels').set(channel.name, channel as any)
+  pikkuState(null, 'channel', 'channels').set(channel.name, channel)
 }
 
 const getMatchingChannelConfig = (path: string) => {

package/src/wirings/channel/channel.types.ts

@@ -64,6 +64,8 @@ export interface ChannelMeta {
   message: ChannelMessageMeta | null
   messageWirings: Record<string, Record<string, ChannelMessageMeta>>
   binary?: boolean | null
+  /** Set when the channel was wired by a gateway rather than declared directly. */
+  gateway?: boolean
   summary?: string
   description?: string
   errors?: string[]

package/src/wirings/channel/index.ts

@@ -1,8 +1,7 @@
-export { wireChannel, openChannel } from './channel-runner.js'
+export { wireChannel } from './channel-runner.js'
 export {
   addChannelMiddleware,
   combineChannelMiddleware,
-  wrapChannelWithMiddleware,
 } from './channel-middleware-runner.js'
 export { logChannels } from './log-channels.js'
 export { PikkuAbstractChannelHandler } from './pikku-abstract-channel-handler.js'
@@ -18,25 +17,14 @@ export type {
   CorePikkuChannelMiddlewareFactory,
   ChannelMessageMeta,
   ChannelMeta,
-  ChannelRemote,
   PikkuChannel,
   PikkuChannelHandlerFactory,
 } from './channel.types.js'
 export { defineChannelRoutes } from './define-channel-routes.js'
 export {
-  CHANNEL_RPC_PENDING,
-  CHANNEL_RPC_REQUEST,
-  CHANNEL_RPC_RESPONSE,
   ChannelDeploymentService,
   ChannelRPCError,
   ChannelRPCRegistry,
-  createChannelRPCResponder,
-  createChannelRPCInputValidator,
-  createChannelRPCResultValidator,
-  isChannelRPCPending,
-  isChannelRPCRequest,
-  isChannelRPCResponse,
-  resolveCapability,
   unsupportedChannelRemote,
 } from './channel-rpc.js'
 export type {
@@ -44,16 +32,9 @@ export type {
   ApprovalRequester,
   Capabilities,
   Capability,
-  CapabilityDef,
   CapabilityHandler,
   ChannelRPCPending,
   ChannelRPCRequest,
   ChannelRPCResponse,
   ChannelRPCValidator,
 } from './channel-rpc.js'
-export {
-  channelRemote,
-  getChannelHostRPC,
-  handleChannelRPCResponse,
-  releaseChannelHostRPC,
-} from './channel-host-rpc.js'

package/src/wirings/channel/local/local-channel-runner.test.ts

@@ -3,6 +3,7 @@ import * as assert from 'node:assert/strict'
 import { runLocalChannel } from './local-channel-runner.js'
 import { pikkuState, resetPikkuState } from '../../../pikku-state.js'
 import { wireChannel } from '../channel-runner.js'
+import { addFunction } from '../../../function/function-runner.js'
 import { addHTTPMiddleware } from '../../http/http-runner.js'
 import type {
   HTTPMethod,
@@ -270,3 +271,70 @@ test('runChannel should run HTTP middleware on websocket upgrade and establish s
   await result.message(JSON.stringify({ action: 'ping' }))
   assert.deepEqual(sent, { action: 'ping' })
 })
+
+test('a message handler that returns nothing does not attempt to send', async () => {
+  // The connect path already guards this; the message path did not, so a handler
+  // with nothing to say produced `send requires a non-empty message` on every
+  // message. Gateway websockets hit it every time — their generated message
+  // handler returns undefined by design — which is why a chat gateway could
+  // accept a connection and never deliver anything to its handler.
+  resetPikkuState()
+  pikkuState(null, 'package', 'singletonServices', mockSingletonServices as any)
+  pikkuState(null, 'package', 'factories', {
+    createWireServices: mockCreateWireServices,
+  } as any)
+
+  const sent: unknown[] = []
+  let handled = 0
+
+  pikkuState(null, 'channel', 'meta', {
+    quiet: {
+      name: 'quiet',
+      route: '/quiet-channel',
+      message: { pikkuFuncId: 'quietMessage' },
+      messageWirings: {},
+    },
+  } as any)
+  pikkuState(null, 'function', 'meta')['quietMessage'] = {
+    pikkuFuncId: 'quietMessage',
+    inputSchemaName: null,
+    outputSchemaName: null,
+    sessionless: true,
+  } as any
+  const quietMessage = {
+    auth: false,
+    func: async () => {
+      handled++
+      return undefined
+    },
+  }
+  wireChannel({
+    name: 'quiet',
+    route: '/quiet-channel',
+    auth: false,
+    onMessage: quietMessage as any,
+  } as any)
+
+  httpRouter.initialize()
+
+  const handler = await runLocalChannel({
+    channelId: 'quiet-channel-id',
+    request: new PikkuMockRequest('/quiet-channel', 'get'),
+    response: new PikkuMockResponse(),
+    route: '/quiet-channel',
+  })
+  assert.ok(handler)
+  handler.registerOnSend(async (message: unknown) => {
+    sent.push(message)
+  })
+  handler.open()
+
+  await handler.message(JSON.stringify({ text: 'hello' }))
+
+  assert.equal(handled, 1, 'the handler should have run')
+  assert.deepEqual(
+    sent,
+    [],
+    'nothing should have been sent for an empty result'
+  )
+})