@pikku/core 0.12.83 → 0.12.86

package/src/wirings/{ai-scorer/ai-scorer-worker.test.ts → agent-scorer/agent-scorer-worker.test.ts}

@@ -1,9 +1,9 @@
 import { beforeEach, describe, test } from 'node:test'
 import assert from 'node:assert/strict'
 import { pikkuState, resetPikkuState } from '../../pikku-state.js'
-import { pikkuAIScoreWorkerFunc } from './ai-scorer-worker.js'
-import { pikkuAIJudge, pikkuAIScorer } from './ai-scorer.js'
-import type { PikkuAIScorer, ScoreJob } from './ai-scorer.types.js'
+import { pikkuAgentScoreWorkerFunc } from './agent-scorer-worker.js'
+import { pikkuAgentJudge, pikkuAgentScorer } from './agent-scorer.js'
+import type { PikkuAgentScorer, ScoreJob } from './agent-scorer.types.js'
 
 const job = (overrides: Partial<ScoreJob> = {}): ScoreJob => ({
   scorerName: 'brevity',
@@ -16,28 +16,28 @@ const job = (overrides: Partial<ScoreJob> = {}): ScoreJob => ({
   ...overrides,
 })
 
-const install = (scorer: PikkuAIScorer, aiAgentRunner?: unknown) => {
+const install = (scorer: PikkuAgentScorer, agentRunner?: unknown) => {
   pikkuState(null, 'agent', 'scorers').set(scorer.name, scorer)
   const saved: unknown[] = []
   pikkuState(null, 'package', 'singletonServices', {
-    aiRunState: {
+    agentRunState: {
       saveScore: async (score: unknown) => {
         saved.push(score)
       },
     },
-    aiAgentRunner,
+    agentRunner,
   } as never)
   return saved
 }
 
-describe('pikkuAIScoreWorkerFunc', () => {
+describe('pikkuAgentScoreWorkerFunc', () => {
   beforeEach(() => {
     resetPikkuState()
   })
 
   test('records what a heuristic scorer graded, against the run it graded', async () => {
     const saved = install(
-      pikkuAIScorer({
+      pikkuAgentScorer({
         name: 'brevity',
         description: 'Shorter is better',
         score: (input) => ({
@@ -47,7 +47,7 @@ describe('pikkuAIScoreWorkerFunc', () => {
       })
     )
 
-    await pikkuAIScoreWorkerFunc({}, job())
+    await pikkuAgentScoreWorkerFunc({}, job())
 
     assert.deepEqual(saved, [
       {
@@ -61,7 +61,7 @@ describe('pikkuAIScoreWorkerFunc', () => {
 
   test('hands a judge to the model rather than calling a score function it has not got', async () => {
     const saved = install(
-      pikkuAIJudge({
+      pikkuAgentJudge({
         name: 'helpfulness',
         description: 'Is the answer useful',
         model: 'claude-opus-5',
@@ -75,7 +75,7 @@ describe('pikkuAIScoreWorkerFunc', () => {
       }
     )
 
-    await pikkuAIScoreWorkerFunc({}, job({ scorerName: 'helpfulness' }))
+    await pikkuAgentScoreWorkerFunc({}, job({ scorerName: 'helpfulness' }))
 
     assert.deepEqual(saved, [
       {
@@ -91,7 +91,7 @@ describe('pikkuAIScoreWorkerFunc', () => {
   test('fails the job when there is nowhere to record the grade, so it is retried rather than lost', async () => {
     pikkuState(null, 'agent', 'scorers').set(
       'brevity',
-      pikkuAIScorer({
+      pikkuAgentScorer({
         name: 'brevity',
         description: 'Shorter is better',
         score: () => ({ score: 1 }),
@@ -100,14 +100,14 @@ describe('pikkuAIScoreWorkerFunc', () => {
     pikkuState(null, 'package', 'singletonServices', {} as never)
 
     await assert.rejects(
-      () => pikkuAIScoreWorkerFunc({}, job()),
+      () => pikkuAgentScoreWorkerFunc({}, job()),
       /AI run state service not initialized/
     )
   })
 
   test('fails on a scorer name that resolves to nothing', async () => {
     install(
-      pikkuAIScorer({
+      pikkuAgentScorer({
         name: 'brevity',
         description: 'Shorter is better',
         score: () => ({ score: 1 }),
@@ -115,7 +115,7 @@ describe('pikkuAIScoreWorkerFunc', () => {
     )
 
     await assert.rejects(
-      () => pikkuAIScoreWorkerFunc({}, job({ scorerName: 'gone' })),
+      () => pikkuAgentScoreWorkerFunc({}, job({ scorerName: 'gone' })),
       /AI scorer not found: gone/
     )
   })

package/src/wirings/{ai-scorer/ai-scorer-worker.ts → agent-scorer/agent-scorer-worker.ts}

@@ -1,14 +1,14 @@
 import { pikkuState } from '../../pikku-state.js'
 import { addFunction } from '../../function/function-runner.js'
 import { wireQueueWorker } from '../queue/queue-runner.js'
-import { gradeRun } from './ai-scorer-grade.js'
-import { SCORER_LANE_QUEUES, type ScoreJob } from './ai-scorer.types.js'
+import { gradeRun } from './agent-scorer-grade.js'
+import { SCORER_LANE_QUEUES, type ScoreJob } from './agent-scorer.types.js'
 
 /**
  * The one worker behind both lanes: it resolves the scorer by name and grades.
  * The lane a job arrived on decides nothing except how long it may take.
  */
-export async function pikkuAIScoreWorkerFunc(
+export async function pikkuAgentScoreWorkerFunc(
   _services: Record<string, unknown>,
   job: ScoreJob
 ): Promise<void> {
@@ -22,7 +22,7 @@ export async function pikkuAIScoreWorkerFunc(
   await gradeRun(job, services, { persist: true })
 }
 
-const SCORE_WORKER_FUNC_ID = 'pikkuAIScoreWorker'
+const SCORE_WORKER_FUNC_ID = 'pikkuAgentScoreWorker'
 
 /**
  * Bind the two lane queues to the shared worker.
@@ -32,7 +32,7 @@ const SCORE_WORKER_FUNC_ID = 'pikkuAIScoreWorker'
  * for, so the metadata is synthesised here alongside the registration. This
  * mirrors how workflows wire their own queues.
  */
-export const wireAIScorerQueueWorkers = (): void => {
+export const wireAgentScorerQueueWorkers = (): void => {
   // No scorers means no lanes: a deployment that grades nothing should not be
   // left holding two queues nothing ever writes to.
   if (pikkuState(null, 'agent', 'scorers').size === 0) return
@@ -42,7 +42,9 @@ export const wireAIScorerQueueWorkers = (): void => {
   const queueMeta = pikkuState(null, 'queue', 'meta')
 
   if (!functions.has(SCORE_WORKER_FUNC_ID)) {
-    addFunction(SCORE_WORKER_FUNC_ID, { func: pikkuAIScoreWorkerFunc } as never)
+    addFunction(SCORE_WORKER_FUNC_ID, {
+      func: pikkuAgentScoreWorkerFunc,
+    } as never)
   }
   if (!functionsMeta[SCORE_WORKER_FUNC_ID]) {
     functionsMeta[SCORE_WORKER_FUNC_ID] = {
@@ -63,7 +65,7 @@ export const wireAIScorerQueueWorkers = (): void => {
     }
     wireQueueWorker({
       name: queueName,
-      func: { func: pikkuAIScoreWorkerFunc },
+      func: { func: pikkuAgentScoreWorkerFunc },
     } as never)
   }
 }

package/src/wirings/{ai-scorer/ai-scorer.ts → agent-scorer/agent-scorer.ts}

@@ -1,8 +1,9 @@
 import type {
-  PikkuAIScorer,
+  JudgeToolCallDisclosure,
+  PikkuAgentScorer,
   ScorerInput,
   ScorerOutput,
-} from './ai-scorer.types.js'
+} from './agent-scorer.types.js'
 
 const assertSampleRate = (name: string, sampleRate: number | undefined) => {
   if (sampleRate === undefined) return 1
@@ -18,7 +19,7 @@ const assertSampleRate = (name: string, sampleRate: number | undefined) => {
  * A heuristic scorer: pure code over the finished run, no model call, so it
  * grades on the fast lane.
  */
-export const pikkuAIScorer = <Services = any>(config: {
+export const pikkuAgentScorer = <Services = any>(config: {
   name: string
   description: string
   /** 0..1 fraction of live runs to grade. Defaults to all of them. */
@@ -32,7 +33,7 @@ export const pikkuAIScorer = <Services = any>(config: {
     input: ScorerInput,
     services: Services
   ) => ScorerOutput | Promise<ScorerOutput>
-}): PikkuAIScorer<Services> => ({
+}): PikkuAgentScorer<Services> => ({
   name: config.name,
   description: config.description,
   lane: 'fast',
@@ -45,11 +46,11 @@ export const pikkuAIScorer = <Services = any>(config: {
  * An LLM-judge scorer: the runtime makes the model call and forces a structured
  * `{ score, reason }`, so a judge is a rubric rather than a prompt to parse.
  *
- * The rubric field is `goal`, matching `pikkuAIAgent`'s prompt vocabulary — a
+ * The rubric field is `goal`, matching `pikkuAgent`'s prompt vocabulary — a
  * judge is a degenerate agent, and should use the same word for the same thing.
  * `prompt` is the escape hatch for non-standard framing.
  */
-export const pikkuAIJudge = <Services = any>(config: {
+export const pikkuAgentJudge = <Services = any>(config: {
   name: string
   description: string
   /** 0..1 fraction of live runs to grade. Defaults to all of them. */
@@ -61,8 +62,14 @@ export const pikkuAIJudge = <Services = any>(config: {
   requiresReference?: boolean
   model: string
   goal: string
+  /**
+   * How much of the run's trajectory to disclose to the judge. Defaults to
+   * `names`: enough to tell a tool-backed answer from an invented one, without
+   * sending a third-party model the rows the tools returned.
+   */
+  toolCalls?: JudgeToolCallDisclosure
   prompt?: (input: ScorerInput) => string
-}): PikkuAIScorer<Services> => ({
+}): PikkuAgentScorer<Services> => ({
   name: config.name,
   description: config.description,
   lane: 'slow',
@@ -71,6 +78,7 @@ export const pikkuAIJudge = <Services = any>(config: {
   judge: {
     model: config.model,
     goal: config.goal,
+    toolCalls: config.toolCalls ?? 'names',
     ...(config.prompt ? { prompt: config.prompt } : {}),
   },
 })

package/src/wirings/{ai-scorer/ai-scorer.types.ts → agent-scorer/agent-scorer.types.ts}

@@ -46,22 +46,43 @@ export interface ScorerOutput {
 export type ScorerLane = 'fast' | 'slow'
 
 export const SCORER_LANE_QUEUES: Record<ScorerLane, string> = {
-  fast: 'ai-score-fast',
-  slow: 'ai-score-slow',
+  fast: 'agent-score-fast',
+  slow: 'agent-score-slow',
 }
 
 /**
- * A judge's model call. Present only on a scorer built with `pikkuAIJudge`;
- * `score` is present only on one built with `pikkuAIScorer`. Exactly one of the
+ * How much of the run's trajectory the default judge prompt discloses.
+ *
+ * A judge is a third-party model, and a tool's arguments and results are the
+ * most sensitive thing a run touches — the row it read, the address it looked
+ * up, the message it sent. Output middleware has already had its pass by the
+ * time a scorer sees any of this, so what a judge is shown is what was stored;
+ * this is the second gate, for a judge that should be told less than the
+ * database was.
+ *
+ * - `names` — which tools ran, and whether each failed. No arguments, no
+ *   results. Enough to tell an answer backed by a tool from an invented one,
+ *   which is the failure this exists to catch, while disclosing no row data.
+ * - `full` — names, arguments and results, truncated. For a judge that must
+ *   check the answer *against* what the tool returned.
+ * - `off` — no trajectory at all. The judge grades the answer alone.
+ */
+export type JudgeToolCallDisclosure = 'off' | 'names' | 'full'
+
+/**
+ * A judge's model call. Present only on a scorer built with `pikkuAgentJudge`;
+ * `score` is present only on one built with `pikkuAgentScorer`. Exactly one of the
  * two is set, which is what the two constructors exist to guarantee.
  */
 export type ScorerJudgeConfig = {
   model: string
   goal: string
+  /** Always set: `pikkuAgentJudge` resolves the default. */
+  toolCalls: JudgeToolCallDisclosure
   prompt?: (input: ScorerInput) => string
 }
 
-export type PikkuAIScorer<Services = any> = {
+export type PikkuAgentScorer<Services = any> = {
   name: string
   description: string
   lane: ScorerLane
@@ -97,7 +118,7 @@ export type ScoreJob = ScorerInput & {
   scorerName: string
 }
 
-export type AIRunScore = {
+export type AgentRunScore = {
   runId: string
   scorerName: string
   score: number

package/src/wirings/agent-scorer/index.ts

@@ -0,0 +1,23 @@
+export { pikkuAgentScorer, pikkuAgentJudge } from './agent-scorer.js'
+export {
+  addAgentScorer,
+  getAgentScorers,
+  getAgentScorersMeta,
+} from './agent-scorer-registry.js'
+export { gradeRun } from './agent-scorer-grade.js'
+export {
+  enableScoreSnapshots,
+  getScoreSnapshot,
+} from './agent-scorer-snapshots.js'
+export { wireAgentScorerQueueWorkers } from './agent-scorer-worker.js'
+export type {
+  AgentRunScore,
+  JudgeToolCallDisclosure,
+  PikkuAgentScorer,
+  ScoreJob,
+  ScorerInput,
+  ScorerJudgeConfig,
+  ScorerLane,
+  ScorerMeta,
+  ScorerOutput,
+} from './agent-scorer.types.js'

package/src/wirings/channel/channel-common.ts

@@ -1,11 +1,13 @@
 import type { CorePikkuChannelMiddleware } from './channel.types.js'
 import type {
   CoreSingletonServices,
-  CorePikkuMiddleware,
   PikkuRawWire,
   PikkuWire,
-  MiddlewareMetadata,
 } from '../../types/core.types.js'
+import type {
+  CorePikkuMiddleware,
+  MiddlewareMetadata,
+} from '../../middleware/middleware.types.js'
 import type { CoreChannel, ChannelMessageMeta } from './channel.types.js'
 import { combineMiddleware, runMiddleware } from '../../middleware-runner.js'
 import { runPikkuFunc } from '../../function/function-runner.js'

package/src/wirings/channel/channel-handler.ts

@@ -1,8 +1,5 @@
-import type {
-  CoreServices,
-  JSONValue,
-  CoreUserSession,
-} from '../../types/core.types.js'
+import type { CoreServices, CoreUserSession } from '../../types/core.types.js'
+import type { JSONValue } from '../../utils.js'
 import type {
   CorePikkuChannelMiddleware,
   BinaryData,

package/src/wirings/channel/channel-middleware-runner.ts

@@ -1,8 +1,8 @@
 import type {
   CoreSingletonServices,
-  MiddlewareMetadata,
   PikkuRawWire,
 } from '../../types/core.types.js'
+import type { MiddlewareMetadata } from '../../middleware/middleware.types.js'
 import type { CorePikkuChannelMiddleware } from './channel.types.js'
 import { pikkuState } from '../../pikku-state.js'
 import { freezeDedupe, getTagGroups } from '../../utils.js'

package/src/wirings/channel/channel-rpc.types.ts

@@ -77,11 +77,7 @@ export class ChannelRPCError extends Error {
   constructor(
     message: string,
     public readonly reason:
-      | 'timeout'
-      | 'closed'
-      | 'remote'
-      | 'invalid'
-      | 'unsupported'
+      'timeout' | 'closed' | 'remote' | 'invalid' | 'unsupported'
   ) {
     super(message)
     this.name = 'ChannelRPCError'
@@ -92,7 +88,7 @@ export class ChannelRPCError extends Error {
  * Whether a callable exposed to a remote decider needs a human to agree before
  * it runs, and how to describe the invocation while asking.
  *
- * Shared with `AIAgentToolDef`: both are an allowlist of named callables
+ * Shared with `AgentToolDef`: both are an allowlist of named callables
  * invoked by something other than the code that wrote them. The runtime around
  * them is not shared — an agent suspends and resumes, a reverse call is a live
  * await.
@@ -105,7 +101,7 @@ export interface ApprovalPolicy {
 export type CapabilityHandler = (data: any) => Promise<unknown> | unknown
 
 /**
- * `needsApproval` is required here on purpose. On `AIAgentToolDef` the same
+ * `needsApproval` is required here on purpose. On `AgentToolDef` the same
  * field is optional and absence means "do not ask"; here absence means the
  * opposite, so it must not be expressible — a bare function is the
  * unclassified form.

package/src/wirings/channel/channel.types.ts

@@ -6,8 +6,7 @@ import type {
 import type {
   CorePikkuMiddleware,
   MiddlewareMetadata,
-} from '../../types/core.types.js'
-
+} from '../../middleware/middleware.types.js'
 export type BinaryData = ArrayBuffer | Uint8Array
 
 export type CorePikkuChannelMiddleware<Services = any, Event = unknown> = (

package/src/wirings/channel/local/local-channel-runner.test.ts

@@ -13,8 +13,7 @@ import type {
 } from '../../http/http.types.js'
 import type { SerializeOptions } from 'cookie'
 import { httpRouter } from '../../http/routers/http-router.js'
-import { pikkuMiddleware } from '../../../types/core.types.js'
-
+import { pikkuMiddleware } from '../../../middleware/middleware-factories.js'
 const mockLogger = {
   info: () => {},
   error: () => {},

package/src/wirings/channel/serverless/index.ts

@@ -3,3 +3,4 @@ export {
   runChannelDisconnect,
   runChannelMessage,
 } from './serverless-channel-runner.js'
+export type { RunServerlessChannelParams } from './serverless-channel-runner.js'

package/src/wirings/cli/channel/cli-raw-channel-runner.ts

@@ -99,8 +99,8 @@ export async function handleRawCLI({
   }
 
   let current:
-    | { pikkuFuncId?: string; subcommands?: Record<string, any> }
-    | undefined = programMeta.commands[parsed.commandPath[0]]
+    { pikkuFuncId?: string; subcommands?: Record<string, any> } | undefined =
+    programMeta.commands[parsed.commandPath[0]]
   for (let i = 1; i < parsed.commandPath.length; i++) {
     current = current?.subcommands?.[parsed.commandPath[i]]
   }

package/src/wirings/cli/channel/index.ts

@@ -2,7 +2,5 @@ export { executeCLIViaChannel } from './cli-channel-runner.js'
 export { handleRawCLI } from './cli-raw-channel-runner.js'
 export type { RawCLIFrame, RawCLIResult } from './cli-raw-channel-runner.js'
 export { executeRawCLIViaChannel } from './cli-raw-client-runner.js'
-export type {
-  CorePikkuCLIClientRender,
-} from './cli-raw-client-runner.js'
+export type { CorePikkuCLIClientRender } from './cli-raw-client-runner.js'
 export type { ApprovalMode } from './cli-approval.js'

package/src/wirings/cli/cli-runner.test.ts

@@ -1,7 +1,7 @@
 import { test, describe, beforeEach, afterEach } from 'node:test'
 import * as assert from 'assert'
 import { NotFoundError } from '../../errors/errors.js'
-import type { CorePikkuMiddleware } from '../../types/core.types.js'
+import type { CorePikkuMiddleware } from '../../middleware/middleware.types.js'
 import {
   CLIError,
   executeCLI,

package/src/wirings/cli/cli-runner.ts

@@ -2,10 +2,8 @@ import { NotFoundError } from '../../errors/errors.js'
 import { isExpectedError } from '../../errors/error-handler.js'
 import { addFunction, runPikkuFunc } from '../../function/function-runner.js'
 import { pikkuState } from '../../pikku-state.js'
-import type {
-  CorePikkuMiddleware,
-  CoreUserSession,
-} from '../../types/core.types.js'
+import type { CoreUserSession } from '../../types/core.types.js'
+import type { CorePikkuMiddleware } from '../../middleware/middleware.types.js'
 import type {
   CoreCLI,
   CLICommandMeta,
@@ -449,8 +447,7 @@ export async function executeCLI({
 
   try {
     const allCLIMeta = pikkuState(null, 'cli', 'meta') as unknown as
-      | CLIMeta
-      | undefined
+      CLIMeta | undefined
     if (!allCLIMeta) {
       throw new Error(
         '[PKU342] CLI metadata not found. No CLI wirings were registered. See https://pikku.dev/docs/pikku-cli/errors/pku342 for more information.'
@@ -509,8 +506,15 @@ export async function executeCLI({
 
     const data = { ...parsed.positionals, ...parsed.options }
 
+    // The command path is passed through so a config factory can tell which
+    // command is about to run. Some commands are meant to work outside a
+    // project, and cannot be gated behind loading that project's config.
     const config = createConfig
-      ? await createConfig(new LocalVariablesService(), data)
+      ? await createConfig(
+          new LocalVariablesService(),
+          data,
+          parsed.commandPath
+        )
       : ({} as CoreConfig)
 
     const singletonServices = await createSingletonServices(config)

package/src/wirings/cli/cli.types.ts

@@ -1,10 +1,12 @@
 import type {
-  CorePikkuMiddleware,
   CoreSingletonServices,
   CoreUserSession,
   CoreServices,
-  MiddlewareMetadata,
 } from '../../types/core.types.js'
+import type {
+  CorePikkuMiddleware,
+  MiddlewareMetadata,
+} from '../../middleware/middleware.types.js'
 import type {
   CorePikkuFunctionConfig,
   CorePikkuPermission,
@@ -13,15 +15,27 @@ import type {
 } from '../../function/functions.types.js'
 import type { ChannelRemote, PikkuChannel } from '../channel/channel.types.js'
 
+/**
+ * The shape an option's value is parsed into. `'string[]'` splits a single
+ * value on commas; `'boolean'` is a flag that consumes the next token only
+ * when it is an explicit literal (`true`/`false`/`1`/`0`/`yes`/`no`); every
+ * other type consumes one token verbatim, so a value may start with `-`. Left
+ * unset — which is the usual case — it is read from the command function's
+ * input schema, then from `default`, and otherwise falls back to `'string'`.
+ * Declare it only where neither can say: an option belonging to no function
+ * input, such as one the config factory reads straight off the CLI data.
+ */
+export type CLIOptionType = 'string' | 'number' | 'boolean' | 'string[]'
+
 /**
  * CLI option definition
  */
 export interface CLIOption<T = any> {
   description: string
   short?: string
+  type?: CLIOptionType
   default?: T
   choices?: T[]
-  array?: boolean
   required?: boolean
 }
 
@@ -222,7 +236,14 @@ export type CoreCLICommandConfig<
     [K in keyof ExtractFunctionInput<FuncConfig>]?: {
       description?: string
       short?: string
+      type?: CLIOptionType
       default?: ExtractFunctionInput<FuncConfig>[K]
+      /**
+       * The values this option accepts. Rejected by the parser and listed in
+       * `--help`, so a flag with a fixed vocabulary says so once rather than
+       * repeating itself in its description.
+       */
+      choices?: ReadonlyArray<ExtractFunctionInput<FuncConfig>[K]>
     }
   }
   middleware?: PikkuMiddleware[]