@pikku/core 0.12.80 → 0.12.82

package/src/services/scoped-credential-service.ts

@@ -0,0 +1,63 @@
+import type { CredentialService } from './credential-service.js'
+
+/**
+ * A `CredentialService` narrowed to the credentials a package declared.
+ *
+ * Unlike `ScopedSecretService` this stays writable — an addon refreshing its
+ * own OAuth token is the normal case. What it cannot do is reach a credential
+ * it never declared, or enumerate the app's users.
+ */
+export class ScopedCredentialService implements CredentialService {
+  constructor(
+    private credentials: CredentialService,
+    private allowedNames: Set<string>
+  ) {}
+
+  private assertAllowed(name: string): void {
+    if (!this.allowedNames.has(name)) {
+      throw new Error(`Access denied to credential: ${name}`)
+    }
+  }
+
+  async get<T = unknown>(name: string, userId?: string): Promise<T | null> {
+    this.assertAllowed(name)
+    return this.credentials.get<T>(name, userId)
+  }
+
+  async set(name: string, value: unknown, userId?: string): Promise<void> {
+    this.assertAllowed(name)
+    return this.credentials.set(name, value, userId)
+  }
+
+  async delete(name: string, userId?: string): Promise<void> {
+    this.assertAllowed(name)
+    return this.credentials.delete(name, userId)
+  }
+
+  async has(name: string, userId?: string): Promise<boolean> {
+    this.assertAllowed(name)
+    return this.credentials.has(name, userId)
+  }
+
+  async getAll(userId: string): Promise<Record<string, unknown>> {
+    const all = await this.credentials.getAll(userId)
+    const scoped: Record<string, unknown> = {}
+    for (const name of this.allowedNames) {
+      if (name in all) {
+        scoped[name] = all[name]
+      }
+    }
+    return scoped
+  }
+
+  async getUsersWithCredential(name: string): Promise<string[]> {
+    this.assertAllowed(name)
+    return this.credentials.getUsersWithCredential(name)
+  }
+
+  async getAllUsers(): Promise<string[]> {
+    throw new Error(
+      'Access denied: enumerating users is not allowed in a scoped credential service'
+    )
+  }
+}

package/src/testing/service-tests/ai-storage-service-tests.ts

@@ -216,6 +216,99 @@ export const defineAiStorageServiceTests = (
       assert.ok(runs.every((r) => r.threadId === thread.id))
     })
 
+    test('saveScore and getScores', async () => {
+      const thread = await storage.createThread('resource-score')
+      const now = new Date()
+
+      const runId = await storage.createRun({
+        agentName: 'scored-agent',
+        threadId: thread.id,
+        resourceId: 'resource-score',
+        status: 'completed',
+        usage: { inputTokens: 10, outputTokens: 5, model: 'test' },
+        createdAt: now,
+        updatedAt: now,
+      })
+
+      await storage.saveScore({
+        runId,
+        scorerName: 'tool-error-rate',
+        score: 1,
+      })
+      await storage.saveScore({
+        runId,
+        scorerName: 'helpfulness',
+        score: 0.75,
+        reason: 'answered the question but skipped the fee',
+        metadata: { subScores: { accuracy: 0.9 }, judgeTokens: 412 },
+      })
+
+      const scores = await storage.getScores(runId)
+      assert.equal(scores.length, 2)
+
+      const byName = new Map(scores.map((s) => [s.scorerName, s]))
+
+      const clean = byName.get('tool-error-rate')
+      assert.ok(clean)
+      assert.equal(clean.score, 1)
+      assert.equal(clean.reason, undefined)
+      assert.equal(clean.metadata, undefined)
+
+      const helpful = byName.get('helpfulness')
+      assert.ok(helpful)
+      // A fractional score has to survive the round trip: an integer column
+      // would quietly turn every grade into 0 or 1.
+      assert.equal(helpful.score, 0.75)
+      assert.equal(helpful.reason, 'answered the question but skipped the fee')
+      assert.deepEqual(helpful.metadata, {
+        subScores: { accuracy: 0.9 },
+        judgeTokens: 412,
+      })
+      assert.ok(helpful.createdAt instanceof Date)
+    })
+
+    test('a re-grade appends rather than replacing the grade that was acted on', async () => {
+      const thread = await storage.createThread('resource-regrade')
+      const now = new Date()
+
+      const runId = await storage.createRun({
+        agentName: 'regraded-agent',
+        threadId: thread.id,
+        resourceId: 'resource-regrade',
+        status: 'completed',
+        usage: { inputTokens: 1, outputTokens: 1, model: 'test' },
+        createdAt: now,
+        updatedAt: now,
+      })
+
+      await storage.saveScore({ runId, scorerName: 'helpfulness', score: 0.2 })
+      await storage.saveScore({ runId, scorerName: 'helpfulness', score: 0.8 })
+
+      const scores = await storage.getScores(runId)
+      assert.equal(scores.length, 2)
+      assert.deepEqual(
+        scores.map((s) => s.score),
+        [0.2, 0.8]
+      )
+    })
+
+    test('getScores is empty for a run nothing graded', async () => {
+      const thread = await storage.createThread('resource-ungraded')
+      const now = new Date()
+
+      const runId = await storage.createRun({
+        agentName: 'ungraded-agent',
+        threadId: thread.id,
+        resourceId: 'resource-ungraded',
+        status: 'completed',
+        usage: { inputTokens: 1, outputTokens: 1, model: 'test' },
+        createdAt: now,
+        updatedAt: now,
+      })
+
+      assert.deepEqual(await storage.getScores(runId), [])
+    })
+
     test('resolveApproval', async () => {
       const thread = await storage.createThread('resource-8')
       const now = new Date()

package/src/types/core.types.ts

@@ -188,8 +188,6 @@ export type FunctionRuntimeMeta = {
   audit?: {
     durability: AuditDurability
   }
-  /** Keeps the full `SecretService`. Set by the inspector, read by the runner. */
-  secretBroker?: boolean
   version?: number
   approvalRequired?: boolean
   approvalDescription?: string
@@ -383,10 +381,9 @@ export type PikkuWire<
   MCPTools extends string | never = never,
   TypedWorkflow extends PikkuWorkflowWire | never = PikkuWorkflowWire,
   TriggerOutput = unknown,
-  // Defaulted to `any` rather than to `Out`: the emitted `TypedScenario<Out>`
-  // supplies the real context, while `PikkuWire`'s own defaults are what other
-  // generics constrain against — and a constraint that pinned the context to
-  // `Out` would reject every wire whose scenario output is anything else.
+  // `any`, not `Out`: the emitted `TypedScenario<Out>` supplies the real
+  // context, and this default is what other generics constrain against. See
+  // `ScenarioContext`.
   TypedScenario extends PikkuScenarioWire<any> | never = PikkuScenarioWire<any>,
   TypedActors extends ScenarioPersonas = ScenarioPersonas,
 > = {

package/src/types/state.types.ts

@@ -29,6 +29,10 @@ import type {
   CoreAIAgent,
   AIAgentMeta,
 } from '../wirings/ai-agent/ai-agent.types.js'
+import type {
+  PikkuAIScorer,
+  ScorerMeta,
+} from '../wirings/ai-scorer/ai-scorer.types.js'
 import type {
   CoreGateway,
   GatewaysMeta,
@@ -98,6 +102,10 @@ export interface PikkuPackageState {
         variableOverrides?: Record<string, string>
         /** Per-instance name-aliases: logical name the addon reads -> actual project credential name */
         credentialOverrides?: Record<string, string>
+        /** Why this instance gets the whole `SecretService` rather than one scoped to its declared secrets */
+        globalSecrets?: string
+        /** Why this instance gets the whole `CredentialService` rather than one scoped to its declared credentials */
+        globalCredentials?: string
         /** Set by `wireRemoteAddon`: this namespace is consumed remotely over HTTP, not bundled */
         remote?: boolean
         /** Remote host base URL (wireRemoteAddon) — string or resolver over singleton services */
@@ -153,6 +161,10 @@ export interface PikkuPackageState {
   agent: {
     agents: Map<string, CoreAIAgent>
     agentsMeta: AIAgentMeta
+    scorers: Map<string, PikkuAIScorer>
+    scorersMeta: ScorerMeta
+    /** Alias -> `provider/model`, from the `models` table in pikku.config.json. */
+    modelAliases: Record<string, string>
   }
   gateway: {
     gateways: Map<string, CoreGateway>
@@ -204,5 +216,9 @@ export interface PikkuPackageState {
       { name: string; displayName: string; type: string; oauth2?: boolean }
     > | null
     requiredParentServices: string[] | null
+    /** The secret keys this package is allowed to read, emitted by the CLI from
+     *  what the package declares and is seen to read. `null` when the package
+     *  was never analysed, which scopes it to nothing. */
+    declaredSecrets: string[] | null
   }
 }

package/src/wirings/actor-flow/index.ts

@@ -1,9 +1,6 @@
 export type {
-  ActorFlowApprovalPolicy,
   ActorFlowVerdict,
   ConverseOptions,
-  TargetAgentReply,
-  TargetPendingApproval,
   TargetAgentDriver,
 } from './actor-flow.types.js'
 export {

package/src/wirings/ai-agent/ai-agent-finalize.test.ts

@@ -0,0 +1,186 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import {
+  applyOutputMiddleware,
+  finalizeAgentRun,
+  type FinalizedRun,
+} from './ai-agent-finalize.js'
+import type { AIAgentStep, PikkuAIMiddlewareHooks } from './ai-agent.types.js'
+import type { AIRunStateService } from '../../services/ai-run-state-service.js'
+
+const step = (
+  toolCalls: NonNullable<AIAgentStep['toolCalls']>
+): AIAgentStep => ({
+  usage: { inputTokens: 0, outputTokens: 0 },
+  toolCalls,
+})
+
+const call = (name: string, result: string) => ({
+  name,
+  args: {} as Record<string, unknown>,
+  result,
+})
+
+describe('applyOutputMiddleware', () => {
+  test('hands the middleware every tool call the run made, flattened across steps', async () => {
+    const seen: string[][] = []
+    const middleware: PikkuAIMiddlewareHooks[] = [
+      {
+        modifyOutput: (_services, ctx) => {
+          seen.push(ctx.toolCalls.map((c) => c.name))
+          return { text: ctx.text, messages: ctx.messages }
+        },
+      },
+    ]
+
+    await applyOutputMiddleware(
+      middleware,
+      {},
+      {
+        text: 'done',
+        messages: [],
+        steps: [step([call('search', 'a')]), step([call('fetch', 'b')])],
+        usage: { inputTokens: 1, outputTokens: 2 },
+      }
+    )
+
+    assert.deepEqual(seen, [['search', 'fetch']])
+  })
+
+  test('a rewritten tool call is redistributed back onto the step it came from', async () => {
+    const middleware: PikkuAIMiddlewareHooks[] = [
+      {
+        modifyOutput: (_services, ctx) => ({
+          text: ctx.text,
+          messages: ctx.messages,
+          toolCalls: ctx.toolCalls.map((c) => ({ ...c, result: '[redacted]' })),
+        }),
+      },
+    ]
+
+    const { steps } = await applyOutputMiddleware(
+      middleware,
+      {},
+      {
+        text: 'done',
+        messages: [],
+        steps: [
+          step([call('search', 'secret')]),
+          step([call('fetch', 'token')]),
+        ],
+        usage: { inputTokens: 0, outputTokens: 0 },
+      }
+    )
+
+    assert.deepEqual(
+      steps.map((s) => s.toolCalls?.map((c) => [c.name, c.result])),
+      [[['search', '[redacted]']], [['fetch', '[redacted]']]]
+    )
+  })
+
+  test('dropping a tool call collapses the calls into the last step rather than mis-attributing them', async () => {
+    const middleware: PikkuAIMiddlewareHooks[] = [
+      {
+        modifyOutput: (_services, ctx) => ({
+          text: ctx.text,
+          messages: ctx.messages,
+          toolCalls: ctx.toolCalls.filter((c) => c.name !== 'search'),
+        }),
+      },
+    ]
+
+    const { steps } = await applyOutputMiddleware(
+      middleware,
+      {},
+      {
+        text: 'done',
+        messages: [],
+        steps: [step([call('search', 'a')]), step([call('fetch', 'b')])],
+        usage: { inputTokens: 0, outputTokens: 0 },
+      }
+    )
+
+    assert.deepEqual(
+      steps.map((s) => s.toolCalls?.map((c) => c.name)),
+      [[], ['fetch']]
+    )
+  })
+
+  test('runs the chain in reverse registration order, each hook seeing the previous rewrite', async () => {
+    const order: string[] = []
+    const middleware: PikkuAIMiddlewareHooks[] = [
+      {
+        modifyOutput: (_services, ctx) => {
+          order.push(`first saw ${ctx.text}`)
+          return { text: `${ctx.text}+first`, messages: ctx.messages }
+        },
+      },
+      {
+        modifyOutput: (_services, ctx) => {
+          order.push(`second saw ${ctx.text}`)
+          return { text: `${ctx.text}+second`, messages: ctx.messages }
+        },
+      },
+    ]
+
+    const { text } = await applyOutputMiddleware(
+      middleware,
+      {},
+      {
+        text: 'raw',
+        messages: [],
+        steps: [],
+        usage: { inputTokens: 0, outputTokens: 0 },
+      }
+    )
+
+    assert.deepEqual(order, ['second saw raw', 'first saw raw+second'])
+    assert.equal(text, 'raw+second+first')
+  })
+})
+
+describe('finalizeAgentRun', () => {
+  const run = (usage: FinalizedRun['usage']): FinalizedRun => ({
+    runId: 'run-1',
+    agentName: 'assistant',
+    threadId: 'thread-1',
+    input: 'what is the capital of France?',
+    text: 'done',
+    steps: [],
+    usage,
+  })
+
+  test('completes the run and records its usage', async () => {
+    const updates: unknown[] = []
+    const aiRunState = {
+      updateRun: async (_runId: string, update: unknown) => {
+        updates.push(update)
+      },
+    } as unknown as AIRunStateService
+
+    await finalizeAgentRun(
+      aiRunState,
+      run({ inputTokens: 10, outputTokens: 20, model: 'gpt-4o' })
+    )
+
+    assert.deepEqual(updates, [
+      {
+        status: 'completed',
+        usage: { inputTokens: 10, outputTokens: 20, model: 'gpt-4o' },
+      },
+    ])
+  })
+
+  test('omits usage entirely when no model reported one', async () => {
+    const updates: any[] = []
+    const aiRunState = {
+      updateRun: async (_runId: string, update: unknown) => {
+        updates.push(update)
+      },
+    } as unknown as AIRunStateService
+
+    await finalizeAgentRun(aiRunState, run({ inputTokens: 0, outputTokens: 0 }))
+
+    assert.deepEqual(updates, [{ status: 'completed' }])
+  })
+})

package/src/wirings/ai-agent/ai-agent-finalize.ts

@@ -0,0 +1,197 @@
+import type {
+  AIAgentStep,
+  AIMessage,
+  PikkuAIMiddlewareHooks,
+} from './ai-agent.types.js'
+import type { AIRunStateService } from '../../services/ai-run-state-service.js'
+import { pikkuState } from '../../pikku-state.js'
+import { scoreFinishedRun } from '../ai-scorer/ai-scorer-live.js'
+import { recordScoreSnapshot } from '../ai-scorer/ai-scorer-snapshots.js'
+
+export type RunUsage = {
+  inputTokens: number
+  outputTokens: number
+  model?: string
+}
+
+/**
+ * Everything a finished run produced, after all middleware has had its say.
+ *
+ * One snapshot feeds persistence and anything that grades the run, so that what
+ * a scorer is shown is exactly what was stored — including the redactions.
+ */
+export type FinalizedRun = {
+  runId: string
+  agentName: string
+  threadId: string
+  resourceId?: string
+  /** The prompt the run answered — what a scorer grades the answer against. */
+  input: string
+  text: string
+  steps: AIAgentStep[]
+  usage: RunUsage
+}
+
+/**
+ * The prompt a run answered: the most recent user turn, which is what the model
+ * was last asked. On a resumed run the earlier turns are context, not the ask.
+ */
+export const lastUserMessageText = (messages: AIMessage[]): string => {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const message = messages[i]
+    if (message?.role !== 'user') continue
+    if (typeof message.content === 'string') return message.content
+    if (Array.isArray(message.content)) {
+      return message.content
+        .filter((part) => part.type === 'text')
+        .map((part) => (part as { text: string }).text)
+        .join('\n')
+    }
+  }
+  return ''
+}
+
+const flattenToolCalls = (
+  steps: AIAgentStep[]
+): NonNullable<AIAgentStep['toolCalls']> =>
+  steps.flatMap((step) => step.toolCalls ?? [])
+
+/**
+ * Push a rewritten flat tool-call list back onto the steps it came from,
+ * preserving the step boundaries. A hook that adds or drops calls would make
+ * the boundaries meaningless, so a length change collapses them into the last
+ * step rather than silently mis-attributing calls to the wrong step.
+ */
+const distributeToolCalls = (
+  steps: AIAgentStep[],
+  toolCalls: NonNullable<AIAgentStep['toolCalls']>
+): AIAgentStep[] => {
+  if (toolCalls.length !== flattenToolCalls(steps).length) {
+    return steps.map((step, index) => ({
+      ...step,
+      toolCalls: index === steps.length - 1 ? toolCalls : [],
+    }))
+  }
+
+  let cursor = 0
+  return steps.map((step) => {
+    if (!step.toolCalls) return step
+    const next = toolCalls.slice(cursor, cursor + step.toolCalls.length)
+    cursor += step.toolCalls.length
+    return { ...step, toolCalls: next }
+  })
+}
+
+/**
+ * Run the `modifyOutput` chain over a finished non-streaming run.
+ *
+ * Reverse order, matching the input chain: the middleware registered first
+ * wraps the others, so it sees the output last.
+ */
+export const applyOutputMiddleware = async (
+  aiMiddlewares: PikkuAIMiddlewareHooks[],
+  singletonServices: any,
+  input: {
+    text: string
+    messages: AIMessage[]
+    steps: AIAgentStep[]
+    usage: { inputTokens: number; outputTokens: number }
+  }
+): Promise<{ text: string; messages: AIMessage[]; steps: AIAgentStep[] }> => {
+  let text = input.text
+  let messages = input.messages
+  let steps = input.steps
+  let toolCalls = flattenToolCalls(steps)
+
+  for (let i = aiMiddlewares.length - 1; i >= 0; i--) {
+    const mw = aiMiddlewares[i]
+    if (!mw.modifyOutput) continue
+    const result = await mw.modifyOutput(singletonServices, {
+      text,
+      messages,
+      usage: {
+        inputTokens: input.usage.inputTokens,
+        outputTokens: input.usage.outputTokens,
+      },
+      toolCalls,
+    })
+    text = result.text
+    messages = result.messages
+    if (result.toolCalls) {
+      toolCalls = result.toolCalls
+      steps = distributeToolCalls(steps, toolCalls)
+    }
+  }
+
+  return { text, messages, steps }
+}
+
+/**
+ * The one place a run ends successfully.
+ *
+ * Every path that completes a run — streamed, non-streamed, and resumed after a
+ * tool approval — goes through here, so that anything terminal is reachable
+ * from all of them and cannot be reordered by a middleware author. Nothing here
+ * may rewrite the run: by this point the output middleware has resolved and, on
+ * the streaming path, the client already has the reply.
+ */
+export const finalizeAgentRun = async (
+  aiRunState: AIRunStateService,
+  run: FinalizedRun
+): Promise<void> => {
+  await aiRunState.updateRun(run.runId, {
+    status: 'completed',
+    ...(run.usage.model
+      ? {
+          usage: {
+            inputTokens: run.usage.inputTokens,
+            outputTokens: run.usage.outputTokens,
+            model: run.usage.model,
+          },
+        }
+      : {}),
+  })
+
+  // Read rather than `getSingletonServices()`: a process that never registered
+  // them grades nothing, which is not an error at the point a run has already
+  // succeeded.
+  const services = pikkuState(null, 'package', 'singletonServices')
+  if (!services) return
+
+  const snapshot = {
+    runId: run.runId,
+    agentName: run.agentName,
+    threadId: run.threadId,
+    ...(run.resourceId !== undefined ? { resourceId: run.resourceId } : {}),
+    input: run.input,
+    output: run.text,
+    toolCalls: run.steps.flatMap((step) =>
+      (step.toolCalls ?? []).map((call) => ({
+        name: call.name,
+        args: call.args,
+        result: call.result,
+        ...(call.error !== undefined ? { error: call.error } : {}),
+      }))
+    ),
+    usage: {
+      inputTokens: run.usage.inputTokens,
+      outputTokens: run.usage.outputTokens,
+      ...(run.usage.model !== undefined ? { model: run.usage.model } : {}),
+    },
+  }
+
+  // The same object a scenario grades, so an asserted score and a sampled one
+  // are the same measurement. A no-op unless a dev server turned retention on.
+  recordScoreSnapshot(snapshot)
+
+  // Best-effort and last: the client already has its answer, so a grading
+  // failure must not surface as a failed run.
+  try {
+    await scoreFinishedRun(snapshot, services)
+  } catch (error) {
+    services.logger?.error(
+      `[pikku] Live scoring failed for run ${run.runId}`,
+      error
+    )
+  }
+}

package/src/wirings/ai-agent/ai-agent-interrupt.ts

@@ -381,6 +381,7 @@ export const persistOrphanedToolResults = async (
           : typeof entry.result === 'string'
             ? entry.result
             : JSON.stringify(entry.result),
+        ...(entry.error ? { error: entry.error } : {}),
       })),
       undelivered: true,
       createdAt: new Date(),