@pikku/core 0.12.84 → 0.12.88

package/src/wirings/agent-scorer/agent-scorer-judge.ts

@@ -26,11 +26,64 @@ const JUDGE_OUTPUT_SCHEMA = {
   additionalProperties: false,
 }
 
+/**
+ * The default disclosure: which tools ran and which failed, and nothing about
+ * what went in or came back. A judge that can see a run reached `listTodos`
+ * can tell a retrieved list from an invented one, which is the whole of what
+ * the trajectory is needed for — the rows themselves it does not need, and an
+ * error string is a tool's own words about the caller's data, so it is
+ * summarised rather than quoted.
+ */
+const describeToolCallName = (call: ScorerInput['toolCalls'][number]): string =>
+  `- ${call.name}${call.error ? ' (failed)' : ''}`
+
+/** Everything about a call, for a judge that grades against what came back. */
+const describeToolCall = (call: ScorerInput['toolCalls'][number]): string => {
+  const outcome = call.error
+    ? `failed: ${call.error}`
+    : `returned ${formatToolValue(call.result)}`
+  return `- ${call.name}(${formatToolValue(call.args)}) ${outcome}`
+}
+
+/**
+ * A tool's arguments and result are arbitrary JSON, and some of it is large —
+ * a listing, a document, a page of search results. Truncating keeps a single
+ * fat call from crowding the answer out of the judge's context; what survives
+ * is the head, which is where the shape of a value is.
+ */
+const TOOL_VALUE_LIMIT = 500
+
+const formatToolValue = (value: unknown): string => {
+  if (value === undefined) {
+    return 'nothing'
+  }
+  const text = typeof value === 'string' ? value : JSON.stringify(value)
+  if (text === undefined) {
+    return 'nothing'
+  }
+  return text.length > TOOL_VALUE_LIMIT
+    ? `${text.slice(0, TOOL_VALUE_LIMIT)}… (truncated)`
+    : text
+}
+
 /**
  * The message a judge grades when its scorer supplies no `prompt`.
  *
  * A reference-based judge is shown the answer key; a reference-free one is not,
  * and grades the answer on its own merits.
+ *
+ * The tool calls are shown because without them a judge cannot tell a fact from
+ * a fabrication. Grading a todo agent that had just listed the user's todos
+ * correctly, a judge given only the answer called it "a plausible-looking list"
+ * that "offers no real access to your actual list, making it effectively a
+ * guess", and scored 0.2 what it scored 1 on other runs — right by its rubric,
+ * on the evidence it was given, and wrong about the run. An agent that answers
+ * from a tool and one that invents the same sentence are indistinguishable in
+ * the output alone; they differ only in what the run did to get there.
+ *
+ * How much of that to show is `judge.toolCalls`, and the default stops at the
+ * names: the 0.2 was the judge doubting the run had any access, and a name
+ * settles that without handing a third-party model the rows.
  */
 export const buildJudgePrompt = (
   judge: ScorerJudgeConfig,
@@ -43,6 +96,13 @@ export const buildJudgePrompt = (
     sections.push(`Reference answer:\n${input.reference}`)
   }
   sections.push(`Assistant answered:\n${input.output}`)
+  if (judge.toolCalls !== 'off' && input.toolCalls.length > 0) {
+    const describe =
+      judge.toolCalls === 'full' ? describeToolCall : describeToolCallName
+    sections.push(
+      `Tools the assistant ran:\n${input.toolCalls.map(describe).join('\n')}`
+    )
+  }
   sections.push(judge.goal)
   sections.push('Respond with a score 0..1 and a brief reason.')
   return sections.join('\n\n')

package/src/wirings/agent-scorer/agent-scorer.ts

@@ -1,4 +1,5 @@
 import type {
+  JudgeToolCallDisclosure,
   PikkuAgentScorer,
   ScorerInput,
   ScorerOutput,
@@ -61,6 +62,12 @@ export const pikkuAgentJudge = <Services = any>(config: {
   requiresReference?: boolean
   model: string
   goal: string
+  /**
+   * How much of the run's trajectory to disclose to the judge. Defaults to
+   * `names`: enough to tell a tool-backed answer from an invented one, without
+   * sending a third-party model the rows the tools returned.
+   */
+  toolCalls?: JudgeToolCallDisclosure
   prompt?: (input: ScorerInput) => string
 }): PikkuAgentScorer<Services> => ({
   name: config.name,
@@ -71,6 +78,7 @@ export const pikkuAgentJudge = <Services = any>(config: {
   judge: {
     model: config.model,
     goal: config.goal,
+    toolCalls: config.toolCalls ?? 'names',
     ...(config.prompt ? { prompt: config.prompt } : {}),
   },
 })

package/src/wirings/agent-scorer/agent-scorer.types.ts

@@ -50,6 +50,25 @@ export const SCORER_LANE_QUEUES: Record<ScorerLane, string> = {
   slow: 'agent-score-slow',
 }
 
+/**
+ * How much of the run's trajectory the default judge prompt discloses.
+ *
+ * A judge is a third-party model, and a tool's arguments and results are the
+ * most sensitive thing a run touches — the row it read, the address it looked
+ * up, the message it sent. Output middleware has already had its pass by the
+ * time a scorer sees any of this, so what a judge is shown is what was stored;
+ * this is the second gate, for a judge that should be told less than the
+ * database was.
+ *
+ * - `names` — which tools ran, and whether each failed. No arguments, no
+ *   results. Enough to tell an answer backed by a tool from an invented one,
+ *   which is the failure this exists to catch, while disclosing no row data.
+ * - `full` — names, arguments and results, truncated. For a judge that must
+ *   check the answer *against* what the tool returned.
+ * - `off` — no trajectory at all. The judge grades the answer alone.
+ */
+export type JudgeToolCallDisclosure = 'off' | 'names' | 'full'
+
 /**
  * A judge's model call. Present only on a scorer built with `pikkuAgentJudge`;
  * `score` is present only on one built with `pikkuAgentScorer`. Exactly one of the
@@ -58,6 +77,8 @@ export const SCORER_LANE_QUEUES: Record<ScorerLane, string> = {
 export type ScorerJudgeConfig = {
   model: string
   goal: string
+  /** Always set: `pikkuAgentJudge` resolves the default. */
+  toolCalls: JudgeToolCallDisclosure
   prompt?: (input: ScorerInput) => string
 }
 

package/src/wirings/agent-scorer/index.ts

@@ -12,6 +12,7 @@ export {
 export { wireAgentScorerQueueWorkers } from './agent-scorer-worker.js'
 export type {
   AgentRunScore,
+  JudgeToolCallDisclosure,
   PikkuAgentScorer,
   ScoreJob,
   ScorerInput,

package/src/wirings/channel/channel-common.ts

@@ -1,11 +1,13 @@
 import type { CorePikkuChannelMiddleware } from './channel.types.js'
 import type {
   CoreSingletonServices,
-  CorePikkuMiddleware,
   PikkuRawWire,
   PikkuWire,
-  MiddlewareMetadata,
 } from '../../types/core.types.js'
+import type {
+  CorePikkuMiddleware,
+  MiddlewareMetadata,
+} from '../../middleware/middleware.types.js'
 import type { CoreChannel, ChannelMessageMeta } from './channel.types.js'
 import { combineMiddleware, runMiddleware } from '../../middleware-runner.js'
 import { runPikkuFunc } from '../../function/function-runner.js'

package/src/wirings/channel/channel-handler.ts

@@ -1,8 +1,5 @@
-import type {
-  CoreServices,
-  JSONValue,
-  CoreUserSession,
-} from '../../types/core.types.js'
+import type { CoreServices, CoreUserSession } from '../../types/core.types.js'
+import type { JSONValue } from '../../utils.js'
 import type {
   CorePikkuChannelMiddleware,
   BinaryData,

package/src/wirings/channel/channel-middleware-runner.ts

@@ -1,8 +1,8 @@
 import type {
   CoreSingletonServices,
-  MiddlewareMetadata,
   PikkuRawWire,
 } from '../../types/core.types.js'
+import type { MiddlewareMetadata } from '../../middleware/middleware.types.js'
 import type { CorePikkuChannelMiddleware } from './channel.types.js'
 import { pikkuState } from '../../pikku-state.js'
 import { freezeDedupe, getTagGroups } from '../../utils.js'

package/src/wirings/channel/channel-rpc.types.ts

@@ -77,11 +77,7 @@ export class ChannelRPCError extends Error {
   constructor(
     message: string,
     public readonly reason:
-      | 'timeout'
-      | 'closed'
-      | 'remote'
-      | 'invalid'
-      | 'unsupported'
+      'timeout' | 'closed' | 'remote' | 'invalid' | 'unsupported'
   ) {
     super(message)
     this.name = 'ChannelRPCError'

package/src/wirings/channel/channel.types.ts

@@ -6,8 +6,7 @@ import type {
 import type {
   CorePikkuMiddleware,
   MiddlewareMetadata,
-} from '../../types/core.types.js'
-
+} from '../../middleware/middleware.types.js'
 export type BinaryData = ArrayBuffer | Uint8Array
 
 export type CorePikkuChannelMiddleware<Services = any, Event = unknown> = (

package/src/wirings/channel/local/local-channel-runner.test.ts

@@ -13,8 +13,7 @@ import type {
 } from '../../http/http.types.js'
 import type { SerializeOptions } from 'cookie'
 import { httpRouter } from '../../http/routers/http-router.js'
-import { pikkuMiddleware } from '../../../types/core.types.js'
-
+import { pikkuMiddleware } from '../../../middleware/middleware-factories.js'
 const mockLogger = {
   info: () => {},
   error: () => {},

package/src/wirings/channel/serverless/index.ts

@@ -3,3 +3,4 @@ export {
   runChannelDisconnect,
   runChannelMessage,
 } from './serverless-channel-runner.js'
+export type { RunServerlessChannelParams } from './serverless-channel-runner.js'

package/src/wirings/cli/channel/cli-raw-channel-runner.ts

@@ -99,8 +99,8 @@ export async function handleRawCLI({
   }
 
   let current:
-    | { pikkuFuncId?: string; subcommands?: Record<string, any> }
-    | undefined = programMeta.commands[parsed.commandPath[0]]
+    { pikkuFuncId?: string; subcommands?: Record<string, any> } | undefined =
+    programMeta.commands[parsed.commandPath[0]]
   for (let i = 1; i < parsed.commandPath.length; i++) {
     current = current?.subcommands?.[parsed.commandPath[i]]
   }

package/src/wirings/cli/channel/index.ts

@@ -2,7 +2,5 @@ export { executeCLIViaChannel } from './cli-channel-runner.js'
 export { handleRawCLI } from './cli-raw-channel-runner.js'
 export type { RawCLIFrame, RawCLIResult } from './cli-raw-channel-runner.js'
 export { executeRawCLIViaChannel } from './cli-raw-client-runner.js'
-export type {
-  CorePikkuCLIClientRender,
-} from './cli-raw-client-runner.js'
+export type { CorePikkuCLIClientRender } from './cli-raw-client-runner.js'
 export type { ApprovalMode } from './cli-approval.js'

package/src/wirings/cli/cli-runner.test.ts

@@ -1,7 +1,7 @@
 import { test, describe, beforeEach, afterEach } from 'node:test'
 import * as assert from 'assert'
 import { NotFoundError } from '../../errors/errors.js'
-import type { CorePikkuMiddleware } from '../../types/core.types.js'
+import type { CorePikkuMiddleware } from '../../middleware/middleware.types.js'
 import {
   CLIError,
   executeCLI,

package/src/wirings/cli/cli-runner.ts

@@ -2,10 +2,8 @@ import { NotFoundError } from '../../errors/errors.js'
 import { isExpectedError } from '../../errors/error-handler.js'
 import { addFunction, runPikkuFunc } from '../../function/function-runner.js'
 import { pikkuState } from '../../pikku-state.js'
-import type {
-  CorePikkuMiddleware,
-  CoreUserSession,
-} from '../../types/core.types.js'
+import type { CoreUserSession } from '../../types/core.types.js'
+import type { CorePikkuMiddleware } from '../../middleware/middleware.types.js'
 import type {
   CoreCLI,
   CLICommandMeta,
@@ -449,8 +447,7 @@ export async function executeCLI({
 
   try {
     const allCLIMeta = pikkuState(null, 'cli', 'meta') as unknown as
-      | CLIMeta
-      | undefined
+      CLIMeta | undefined
     if (!allCLIMeta) {
       throw new Error(
         '[PKU342] CLI metadata not found. No CLI wirings were registered. See https://pikku.dev/docs/pikku-cli/errors/pku342 for more information.'
@@ -509,8 +506,15 @@ export async function executeCLI({
 
     const data = { ...parsed.positionals, ...parsed.options }
 
+    // The command path is passed through so a config factory can tell which
+    // command is about to run. Some commands are meant to work outside a
+    // project, and cannot be gated behind loading that project's config.
     const config = createConfig
-      ? await createConfig(new LocalVariablesService(), data)
+      ? await createConfig(
+          new LocalVariablesService(),
+          data,
+          parsed.commandPath
+        )
       : ({} as CoreConfig)
 
     const singletonServices = await createSingletonServices(config)

package/src/wirings/cli/cli.types.ts

@@ -1,10 +1,12 @@
 import type {
-  CorePikkuMiddleware,
   CoreSingletonServices,
   CoreUserSession,
   CoreServices,
-  MiddlewareMetadata,
 } from '../../types/core.types.js'
+import type {
+  CorePikkuMiddleware,
+  MiddlewareMetadata,
+} from '../../middleware/middleware.types.js'
 import type {
   CorePikkuFunctionConfig,
   CorePikkuPermission,
@@ -13,15 +15,27 @@ import type {
 } from '../../function/functions.types.js'
 import type { ChannelRemote, PikkuChannel } from '../channel/channel.types.js'
 
+/**
+ * The shape an option's value is parsed into. `'string[]'` splits a single
+ * value on commas; `'boolean'` is a flag that consumes the next token only
+ * when it is an explicit literal (`true`/`false`/`1`/`0`/`yes`/`no`); every
+ * other type consumes one token verbatim, so a value may start with `-`. Left
+ * unset — which is the usual case — it is read from the command function's
+ * input schema, then from `default`, and otherwise falls back to `'string'`.
+ * Declare it only where neither can say: an option belonging to no function
+ * input, such as one the config factory reads straight off the CLI data.
+ */
+export type CLIOptionType = 'string' | 'number' | 'boolean' | 'string[]'
+
 /**
  * CLI option definition
  */
 export interface CLIOption<T = any> {
   description: string
   short?: string
+  type?: CLIOptionType
   default?: T
   choices?: T[]
-  array?: boolean
   required?: boolean
 }
 
@@ -222,7 +236,14 @@ export type CoreCLICommandConfig<
     [K in keyof ExtractFunctionInput<FuncConfig>]?: {
       description?: string
       short?: string
+      type?: CLIOptionType
       default?: ExtractFunctionInput<FuncConfig>[K]
+      /**
+       * The values this option accepts. Rejected by the parser and listed in
+       * `--help`, so a flag with a fixed vocabulary says so once rather than
+       * repeating itself in its description.
+       */
+      choices?: ReadonlyArray<ExtractFunctionInput<FuncConfig>[K]>
     }
   }
   middleware?: PikkuMiddleware[]