@pikku/core 0.12.107 → 0.12.111

package/src/wirings/agent/agent-finalize.test.ts

@@ -1,187 +0,0 @@
-import { describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-import {
-  applyOutputMiddleware,
-  finalizeAgentRun,
-  type FinalizedRun,
-} from './agent-finalize.js'
-import type { AgentStep, PikkuAgentMiddlewareHooks } from './agent.types.js'
-import type { AgentRunStateService } from '../../services/agent-run-state-service.js'
-
-const step = (toolCalls: NonNullable<AgentStep['toolCalls']>): AgentStep => ({
-  usage: { inputTokens: 0, outputTokens: 0 },
-  toolCalls,
-})
-
-const call = (name: string, result: string) => ({
-  name,
-  args: {} as Record<string, unknown>,
-  result,
-})
-
-describe('applyOutputMiddleware', () => {
-  test('hands the middleware every tool call the run made, flattened across steps', async () => {
-    const seen: string[][] = []
-    const middleware: PikkuAgentMiddlewareHooks[] = [
-      {
-        modifyOutput: (_services, ctx) => {
-          seen.push(ctx.toolCalls.map((c) => c.name))
-          return { text: ctx.text, messages: ctx.messages }
-        },
-      },
-    ]
-
-    await applyOutputMiddleware(
-      middleware,
-      {},
-      {
-        text: 'done',
-        messages: [],
-        steps: [step([call('search', 'a')]), step([call('fetch', 'b')])],
-        usage: { inputTokens: 1, outputTokens: 2 },
-      }
-    )
-
-    assert.deepEqual(seen, [['search', 'fetch']])
-  })
-
-  test('a rewritten tool call is redistributed back onto the step it came from', async () => {
-    const middleware: PikkuAgentMiddlewareHooks[] = [
-      {
-        modifyOutput: (_services, ctx) => ({
-          text: ctx.text,
-          messages: ctx.messages,
-          toolCalls: ctx.toolCalls.map((c) => ({ ...c, result: '[redacted]' })),
-        }),
-      },
-    ]
-
-    const { steps } = await applyOutputMiddleware(
-      middleware,
-      {},
-      {
-        text: 'done',
-        messages: [],
-        steps: [
-          step([call('search', 'secret')]),
-          step([call('fetch', 'token')]),
-        ],
-        usage: { inputTokens: 0, outputTokens: 0 },
-      }
-    )
-
-    assert.deepEqual(
-      steps.map((s) => s.toolCalls?.map((c) => [c.name, c.result])),
-      [[['search', '[redacted]']], [['fetch', '[redacted]']]]
-    )
-  })
-
-  test('dropping a tool call collapses the calls into the last step rather than mis-attributing them', async () => {
-    const middleware: PikkuAgentMiddlewareHooks[] = [
-      {
-        modifyOutput: (_services, ctx) => ({
-          text: ctx.text,
-          messages: ctx.messages,
-          toolCalls: ctx.toolCalls.filter((c) => c.name !== 'search'),
-        }),
-      },
-    ]
-
-    const { steps } = await applyOutputMiddleware(
-      middleware,
-      {},
-      {
-        text: 'done',
-        messages: [],
-        steps: [step([call('search', 'a')]), step([call('fetch', 'b')])],
-        usage: { inputTokens: 0, outputTokens: 0 },
-      }
-    )
-
-    assert.deepEqual(
-      steps.map((s) => s.toolCalls?.map((c) => c.name)),
-      [[], ['fetch']]
-    )
-  })
-
-  test('runs the chain in reverse registration order, each hook seeing the previous rewrite', async () => {
-    const order: string[] = []
-    const middleware: PikkuAgentMiddlewareHooks[] = [
-      {
-        modifyOutput: (_services, ctx) => {
-          order.push(`first saw ${ctx.text}`)
-          return { text: `${ctx.text}+first`, messages: ctx.messages }
-        },
-      },
-      {
-        modifyOutput: (_services, ctx) => {
-          order.push(`second saw ${ctx.text}`)
-          return { text: `${ctx.text}+second`, messages: ctx.messages }
-        },
-      },
-    ]
-
-    const { text } = await applyOutputMiddleware(
-      middleware,
-      {},
-      {
-        text: 'raw',
-        messages: [],
-        steps: [],
-        usage: { inputTokens: 0, outputTokens: 0 },
-      }
-    )
-
-    assert.deepEqual(order, ['second saw raw', 'first saw raw+second'])
-    assert.equal(text, 'raw+second+first')
-  })
-})
-
-describe('finalizeAgentRun', () => {
-  const run = (usage: FinalizedRun['usage']): FinalizedRun => ({
-    runId: 'run-1',
-    agentName: 'assistant',
-    threadId: 'thread-1',
-    input: 'what is the capital of France?',
-    text: 'done',
-    steps: [],
-    usage,
-  })
-
-  test('completes the run and records its usage', async () => {
-    const updates: unknown[] = []
-    const agentRunState = {
-      updateRun: async (_runId: string, update: unknown) => {
-        updates.push(update)
-      },
-    } as unknown as AgentRunStateService
-
-    await finalizeAgentRun(
-      agentRunState,
-      run({ inputTokens: 10, outputTokens: 20, model: 'gpt-4o' })
-    )
-
-    assert.deepEqual(updates, [
-      {
-        status: 'completed',
-        usage: { inputTokens: 10, outputTokens: 20, model: 'gpt-4o' },
-      },
-    ])
-  })
-
-  test('omits usage entirely when no model reported one', async () => {
-    const updates: any[] = []
-    const agentRunState = {
-      updateRun: async (_runId: string, update: unknown) => {
-        updates.push(update)
-      },
-    } as unknown as AgentRunStateService
-
-    await finalizeAgentRun(
-      agentRunState,
-      run({ inputTokens: 0, outputTokens: 0 })
-    )
-
-    assert.deepEqual(updates, [{ status: 'completed' }])
-  })
-})

package/src/wirings/agent/agent-finalize.ts

@@ -1,197 +0,0 @@
-import type {
-  AgentStep,
-  AgentMessage,
-  PikkuAgentMiddlewareHooks,
-} from './agent.types.js'
-import type { AgentRunStateService } from '../../services/agent-run-state-service.js'
-import { pikkuState } from '../../pikku-state.js'
-import { scoreFinishedRun } from '../agent-scorer/agent-scorer-live.js'
-import { recordScoreSnapshot } from '../agent-scorer/agent-scorer-snapshots.js'
-
-export type RunUsage = {
-  inputTokens: number
-  outputTokens: number
-  model?: string
-}
-
-/**
- * Everything a finished run produced, after all middleware has had its say.
- *
- * One snapshot feeds persistence and anything that grades the run, so that what
- * a scorer is shown is exactly what was stored — including the redactions.
- */
-export type FinalizedRun = {
-  runId: string
-  agentName: string
-  threadId: string
-  resourceId?: string
-  /** The prompt the run answered — what a scorer grades the answer against. */
-  input: string
-  text: string
-  steps: AgentStep[]
-  usage: RunUsage
-}
-
-/**
- * The prompt a run answered: the most recent user turn, which is what the model
- * was last asked. On a resumed run the earlier turns are context, not the ask.
- */
-export const lastUserMessageText = (messages: AgentMessage[]): string => {
-  for (let i = messages.length - 1; i >= 0; i--) {
-    const message = messages[i]
-    if (message?.role !== 'user') continue
-    if (typeof message.content === 'string') return message.content
-    if (Array.isArray(message.content)) {
-      return message.content
-        .filter((part) => part.type === 'text')
-        .map((part) => (part as { text: string }).text)
-        .join('\n')
-    }
-  }
-  return ''
-}
-
-const flattenToolCalls = (
-  steps: AgentStep[]
-): NonNullable<AgentStep['toolCalls']> =>
-  steps.flatMap((step) => step.toolCalls ?? [])
-
-/**
- * Push a rewritten flat tool-call list back onto the steps it came from,
- * preserving the step boundaries. A hook that adds or drops calls would make
- * the boundaries meaningless, so a length change collapses them into the last
- * step rather than silently mis-attributing calls to the wrong step.
- */
-const distributeToolCalls = (
-  steps: AgentStep[],
-  toolCalls: NonNullable<AgentStep['toolCalls']>
-): AgentStep[] => {
-  if (toolCalls.length !== flattenToolCalls(steps).length) {
-    return steps.map((step, index) => ({
-      ...step,
-      toolCalls: index === steps.length - 1 ? toolCalls : [],
-    }))
-  }
-
-  let cursor = 0
-  return steps.map((step) => {
-    if (!step.toolCalls) return step
-    const next = toolCalls.slice(cursor, cursor + step.toolCalls.length)
-    cursor += step.toolCalls.length
-    return { ...step, toolCalls: next }
-  })
-}
-
-/**
- * Run the `modifyOutput` chain over a finished non-streaming run.
- *
- * Reverse order, matching the input chain: the middleware registered first
- * wraps the others, so it sees the output last.
- */
-export const applyOutputMiddleware = async (
-  agentMiddlewares: PikkuAgentMiddlewareHooks[],
-  singletonServices: any,
-  input: {
-    text: string
-    messages: AgentMessage[]
-    steps: AgentStep[]
-    usage: { inputTokens: number; outputTokens: number }
-  }
-): Promise<{ text: string; messages: AgentMessage[]; steps: AgentStep[] }> => {
-  let text = input.text
-  let messages = input.messages
-  let steps = input.steps
-  let toolCalls = flattenToolCalls(steps)
-
-  for (let i = agentMiddlewares.length - 1; i >= 0; i--) {
-    const mw = agentMiddlewares[i]
-    if (!mw.modifyOutput) continue
-    const result = await mw.modifyOutput(singletonServices, {
-      text,
-      messages,
-      usage: {
-        inputTokens: input.usage.inputTokens,
-        outputTokens: input.usage.outputTokens,
-      },
-      toolCalls,
-    })
-    text = result.text
-    messages = result.messages
-    if (result.toolCalls) {
-      toolCalls = result.toolCalls
-      steps = distributeToolCalls(steps, toolCalls)
-    }
-  }
-
-  return { text, messages, steps }
-}
-
-/**
- * The one place a run ends successfully.
- *
- * Every path that completes a run — streamed, non-streamed, and resumed after a
- * tool approval — goes through here, so that anything terminal is reachable
- * from all of them and cannot be reordered by a middleware author. Nothing here
- * may rewrite the run: by this point the output middleware has resolved and, on
- * the streaming path, the client already has the reply.
- */
-export const finalizeAgentRun = async (
-  agentRunState: AgentRunStateService,
-  run: FinalizedRun
-): Promise<void> => {
-  await agentRunState.updateRun(run.runId, {
-    status: 'completed',
-    ...(run.usage.model
-      ? {
-          usage: {
-            inputTokens: run.usage.inputTokens,
-            outputTokens: run.usage.outputTokens,
-            model: run.usage.model,
-          },
-        }
-      : {}),
-  })
-
-  // Read rather than `getSingletonServices()`: a process that never registered
-  // them grades nothing, which is not an error at the point a run has already
-  // succeeded.
-  const services = pikkuState(null, 'package', 'singletonServices')
-  if (!services) return
-
-  const snapshot = {
-    runId: run.runId,
-    agentName: run.agentName,
-    threadId: run.threadId,
-    ...(run.resourceId !== undefined ? { resourceId: run.resourceId } : {}),
-    input: run.input,
-    output: run.text,
-    toolCalls: run.steps.flatMap((step) =>
-      (step.toolCalls ?? []).map((call) => ({
-        name: call.name,
-        args: call.args,
-        result: call.result,
-        ...(call.error !== undefined ? { error: call.error } : {}),
-      }))
-    ),
-    usage: {
-      inputTokens: run.usage.inputTokens,
-      outputTokens: run.usage.outputTokens,
-      ...(run.usage.model !== undefined ? { model: run.usage.model } : {}),
-    },
-  }
-
-  // The same object a scenario grades, so an asserted score and a sampled one
-  // are the same measurement. A no-op unless a dev server turned retention on.
-  recordScoreSnapshot(snapshot)
-
-  // Best-effort and last: the client already has its answer, so a grading
-  // failure must not surface as a failed run.
-  try {
-    await scoreFinishedRun(snapshot, services)
-  } catch (error) {
-    services.logger?.error(
-      `[pikku] Live scoring failed for run ${run.runId}`,
-      error
-    )
-  }
-}

package/src/wirings/agent/agent-helpers.test.ts

@@ -1,215 +0,0 @@
-import { describe, test } from 'node:test'
-import assert from 'node:assert'
-import {
-  agent,
-  agentStream,
-  agentResume,
-  agentApprove,
-} from './agent-helpers.js'
-import { agentCallOptions } from './agent-prepare.js'
-
-describe('agent helpers', () => {
-  describe('agent', () => {
-    test('should return an object with func', () => {
-      const a = agent('myAgent')
-      assert.strictEqual(typeof a.func, 'function')
-    })
-
-    test('func should call rpc.agent.run', async () => {
-      const a = agent('myAgent')
-      let calledWith: any
-      const rpc = {
-        agent: {
-          run: async (...args: any[]) => {
-            calledWith = args
-            return {
-              runId: 'run-1',
-              result: 'done',
-              usage: { inputTokens: 10, outputTokens: 20 },
-            }
-          },
-        },
-      }
-      const input = { message: 'hello', threadId: 't1', resourceId: 'r1' }
-      const result = await a.func({}, input, { rpc })
-      assert.strictEqual(calledWith[0], 'myAgent')
-      assert.deepStrictEqual(calledWith[1], input)
-      assert.strictEqual(result.runId, 'run-1')
-    })
-  })
-
-  describe('agentStream', () => {
-    test('should return an object with func', () => {
-      const a = agentStream('myAgent')
-      assert.strictEqual(typeof a.func, 'function')
-    })
-
-    test('func should call rpc.agent.stream with bound name', async () => {
-      const a = agentStream('myAgent')
-      let calledWith: any
-      const rpc = {
-        agent: {
-          stream: async (...args: any[]) => {
-            calledWith = args
-          },
-        },
-      }
-      const input = { message: 'hello', threadId: 't1', resourceId: 'r1' }
-      await a.func({}, input, { rpc })
-      assert.strictEqual(calledWith[0], 'myAgent')
-      assert.deepStrictEqual(calledWith[1], input)
-    })
-
-    test('func should extract agentName from data when no name bound', async () => {
-      const a = agentStream()
-      let calledWith: any
-      const rpc = {
-        agent: {
-          stream: async (...args: any[]) => {
-            calledWith = args
-          },
-        },
-      }
-      const input = {
-        agentName: 'dynamicAgent',
-        message: 'hello',
-        threadId: 't1',
-        resourceId: 'r1',
-      }
-      await a.func({}, input, { rpc })
-      assert.strictEqual(calledWith[0], 'dynamicAgent')
-      assert.deepStrictEqual(calledWith[1], {
-        message: 'hello',
-        threadId: 't1',
-        resourceId: 'r1',
-      })
-    })
-
-    test('func should throw when no name bound and no agentName in data', async () => {
-      const a = agentStream()
-      const rpc = { agent: { stream: async () => {} } }
-      await assert.rejects(() => a.func({}, { message: 'hello' }, { rpc }), {
-        message:
-          'agentStream requires an agentName either as a parameter or in the input data',
-      })
-    })
-  })
-
-  describe('agentResume', () => {
-    test('should return an object with func', () => {
-      const a = agentResume()
-      assert.strictEqual(typeof a.func, 'function')
-    })
-
-    test('func should call rpc.agent.resume', async () => {
-      const a = agentResume()
-      let calledWith: any
-      const rpc = {
-        agent: {
-          resume: async (...args: any[]) => {
-            calledWith = args
-          },
-        },
-      }
-      const data = { runId: 'run-1', toolCallId: 'tc1', approved: true }
-      await a.func({}, data, { rpc })
-      assert.strictEqual(calledWith[0], 'run-1')
-      assert.deepStrictEqual(calledWith[1], {
-        toolCallId: 'tc1',
-        approved: true,
-      })
-    })
-  })
-
-  describe('agentApprove', () => {
-    test('should return an object with func', () => {
-      const a = agentApprove('myAgent')
-      assert.strictEqual(typeof a.func, 'function')
-    })
-
-    test('func should call rpc.agent.approve', async () => {
-      const a = agentApprove('myAgent')
-      let calledWith: any
-      const rpc = {
-        agent: {
-          approve: async (...args: any[]) => {
-            calledWith = args
-            return { approved: true }
-          },
-        },
-      }
-      const data = {
-        runId: 'run-1',
-        approvals: [{ toolCallId: 'tc1', approved: true }],
-      }
-      const result = await a.func({}, data, { rpc })
-      assert.strictEqual(calledWith[0], 'run-1')
-      assert.deepStrictEqual(calledWith[1], [
-        { toolCallId: 'tc1', approved: true },
-      ])
-      assert.deepStrictEqual(result, { approved: true })
-    })
-  })
-})
-
-describe('agentCallOptions', () => {
-  // An explicit `undefined` overrides the agent's own declared default with
-  // nothing, so a request that names no model would silently unset the one the
-  // agent declares.
-  test('a field nobody supplied is left out rather than sent as undefined', () => {
-    const options = agentCallOptions({
-      message: 'hello',
-      threadId: 't1',
-      resourceId: 'r1',
-    })
-    assert.equal('model' in options, false)
-    assert.equal('temperature' in options, false)
-    assert.equal('context' in options, false)
-    assert.equal('attachments' in options, false)
-  })
-
-  test('what was supplied is carried through unchanged', () => {
-    const options = agentCallOptions({
-      message: 'hello',
-      threadId: 't1',
-      resourceId: 'user-1',
-      model: 'a-model',
-      temperature: 0.2,
-      context: 'some context',
-      attachments: [{ type: 'image' as const, url: 'a.png' }],
-    })
-    assert.deepEqual(options, {
-      message: 'hello',
-      threadId: 't1',
-      resourceId: 'user-1',
-      attachments: [{ type: 'image' as const, url: 'a.png' }],
-      model: 'a-model',
-      temperature: 0.2,
-      context: 'some context',
-    })
-  })
-
-  // Zero is a temperature a caller can mean, and the falsy check every other
-  // field uses would drop it.
-  test('a temperature of zero survives, unlike an empty string', () => {
-    assert.equal(
-      agentCallOptions({
-        message: 'x',
-        threadId: 't',
-        resourceId: 'r',
-        temperature: 0,
-      }).temperature,
-      0
-    )
-    assert.equal(
-      'context' in
-        agentCallOptions({
-          message: 'x',
-          threadId: 't',
-          resourceId: 'r',
-          context: '',
-        }),
-      false
-    )
-  })
-})

package/src/wirings/agent/agent-helpers.ts

@@ -1,96 +0,0 @@
-import type { AgentInterruptResult } from './agent-interrupt.js'
-
-export function agent<TAgentMap extends Record<string, { output: any }>>(
-  agentName: string & keyof TAgentMap
-): { func: (services: any, data: any, wire: any) => Promise<any> } {
-  return {
-    func: async (_services: any, data: any, { rpc }: any) => {
-      return rpc.agent.run(agentName, data)
-    },
-  }
-}
-
-export function agentStream<TAgentMap extends Record<string, { output: any }>>(
-  agentName?: string & keyof TAgentMap
-): { func: (services: any, data: any, wire: any) => Promise<void> } {
-  return {
-    func: async (_services: any, data: any, { rpc }: any) => {
-      if (agentName) {
-        await rpc.agent.stream(agentName, data)
-      } else {
-        const { agentName: name, ...rest } = data
-        if (!name) {
-          throw new Error(
-            'agentStream requires an agentName either as a parameter or in the input data'
-          )
-        }
-        await rpc.agent.stream(name, rest)
-      }
-    },
-  }
-}
-
-export function agentResume(): {
-  func: (
-    services: any,
-    data: { runId: string; toolCallId: string; approved: boolean },
-    wire: any
-  ) => Promise<void>
-} {
-  return {
-    func: async (
-      _services: any,
-      data: { runId: string; toolCallId: string; approved: boolean },
-      { rpc }: any
-    ) => {
-      await rpc.agent.resume(data.runId, {
-        toolCallId: data.toolCallId,
-        approved: data.approved,
-      })
-    },
-  }
-}
-
-export function agentInterrupt(): {
-  func: (
-    services: any,
-    data: { runId: string; reason?: 'speech' | 'user' | 'timeout' },
-    wire: any
-  ) => Promise<AgentInterruptResult>
-} {
-  return {
-    func: async (
-      _services: any,
-      data: { runId: string; reason?: 'speech' | 'user' | 'timeout' },
-      { rpc }: any
-    ) => {
-      return rpc.agent.interrupt(data.runId, data.reason)
-    },
-  }
-}
-
-export function agentApprove<TAgentMap extends Record<string, { output: any }>>(
-  _agentName: string & keyof TAgentMap
-): {
-  func: (
-    services: any,
-    data: {
-      runId: string
-      approvals: { toolCallId: string; approved: boolean }[]
-    },
-    wire: any
-  ) => Promise<any>
-} {
-  return {
-    func: async (
-      _services: any,
-      data: {
-        runId: string
-        approvals: { toolCallId: string; approved: boolean }[]
-      },
-      { rpc }: any
-    ) => {
-      return rpc.agent.approve(data.runId, data.approvals)
-    },
-  }
-}