@pikku/core 0.12.80 → 0.12.83

package/src/services/scoped-credential-service.test.ts

@@ -0,0 +1,86 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { ScopedCredentialService } from './scoped-credential-service.js'
+import type { CredentialService } from './credential-service.js'
+
+const createCredentialService = (): CredentialService => {
+  const store: Record<string, Record<string, unknown>> = {
+    'user-1': { slack: 'slack-token', stripe: 'stripe-key' },
+    'user-2': { slack: 'other-slack-token' },
+  }
+  return {
+    get: async (name, userId = 'user-1') =>
+      (store[userId]?.[name] ?? null) as never,
+    set: async (name, value, userId = 'user-1') => {
+      store[userId] = { ...store[userId], [name]: value }
+    },
+    delete: async (name, userId = 'user-1') => {
+      delete store[userId]?.[name]
+    },
+    has: async (name, userId = 'user-1') => name in (store[userId] ?? {}),
+    getAll: async (userId) => store[userId] ?? {},
+    getUsersWithCredential: async (name) =>
+      Object.keys(store).filter((userId) => name in store[userId]!),
+    getAllUsers: async () => Object.keys(store),
+  }
+}
+
+const scoped = (names: string[]) =>
+  new ScopedCredentialService(createCredentialService(), new Set(names))
+
+describe('ScopedCredentialService', () => {
+  test('a declared credential is readable and writable', async () => {
+    const credentials = scoped(['slack'])
+
+    assert.equal(await credentials.get('slack'), 'slack-token')
+    assert.equal(await credentials.has('slack'), true)
+    await credentials.set('slack', 'refreshed')
+    assert.equal(await credentials.get('slack'), 'refreshed')
+    await credentials.delete('slack')
+    assert.equal(await credentials.get('slack'), null)
+  })
+
+  test('an undeclared credential is denied on every accessor', async () => {
+    const credentials = scoped(['slack'])
+
+    await assert.rejects(() => credentials.get('stripe'), /denied/i)
+    await assert.rejects(() => credentials.has('stripe'), /denied/i)
+    await assert.rejects(() => credentials.set('stripe', 'x'), /denied/i)
+    await assert.rejects(() => credentials.delete('stripe'), /denied/i)
+    await assert.rejects(
+      () => credentials.getUsersWithCredential('stripe'),
+      /denied/i
+    )
+  })
+
+  test('getAll returns only the declared credentials, not the whole user', async () => {
+    const credentials = scoped(['slack'])
+
+    assert.deepEqual(await credentials.getAll('user-1'), {
+      slack: 'slack-token',
+    })
+  })
+
+  test('enumerating every user of the app is denied outright', async () => {
+    const credentials = scoped(['slack'])
+
+    await assert.rejects(() => credentials.getAllUsers(), /denied/i)
+  })
+
+  test('the users of a declared credential are still visible', async () => {
+    const credentials = scoped(['slack'])
+
+    assert.deepEqual(await credentials.getUsersWithCredential('slack'), [
+      'user-1',
+      'user-2',
+    ])
+  })
+
+  test('an addon that declares nothing reaches nothing', async () => {
+    const credentials = scoped([])
+
+    await assert.rejects(() => credentials.get('slack'), /denied/i)
+    assert.deepEqual(await credentials.getAll('user-1'), {})
+  })
+})

package/src/services/scoped-credential-service.ts

@@ -0,0 +1,63 @@
+import type { CredentialService } from './credential-service.js'
+
+/**
+ * A `CredentialService` narrowed to the credentials a package declared.
+ *
+ * Unlike `ScopedSecretService` this stays writable — an addon refreshing its
+ * own OAuth token is the normal case. What it cannot do is reach a credential
+ * it never declared, or enumerate the app's users.
+ */
+export class ScopedCredentialService implements CredentialService {
+  constructor(
+    private credentials: CredentialService,
+    private allowedNames: Set<string>
+  ) {}
+
+  private assertAllowed(name: string): void {
+    if (!this.allowedNames.has(name)) {
+      throw new Error(`Access denied to credential: ${name}`)
+    }
+  }
+
+  async get<T = unknown>(name: string, userId?: string): Promise<T | null> {
+    this.assertAllowed(name)
+    return this.credentials.get<T>(name, userId)
+  }
+
+  async set(name: string, value: unknown, userId?: string): Promise<void> {
+    this.assertAllowed(name)
+    return this.credentials.set(name, value, userId)
+  }
+
+  async delete(name: string, userId?: string): Promise<void> {
+    this.assertAllowed(name)
+    return this.credentials.delete(name, userId)
+  }
+
+  async has(name: string, userId?: string): Promise<boolean> {
+    this.assertAllowed(name)
+    return this.credentials.has(name, userId)
+  }
+
+  async getAll(userId: string): Promise<Record<string, unknown>> {
+    const all = await this.credentials.getAll(userId)
+    const scoped: Record<string, unknown> = {}
+    for (const name of this.allowedNames) {
+      if (name in all) {
+        scoped[name] = all[name]
+      }
+    }
+    return scoped
+  }
+
+  async getUsersWithCredential(name: string): Promise<string[]> {
+    this.assertAllowed(name)
+    return this.credentials.getUsersWithCredential(name)
+  }
+
+  async getAllUsers(): Promise<string[]> {
+    throw new Error(
+      'Access denied: enumerating users is not allowed in a scoped credential service'
+    )
+  }
+}

package/src/testing/service-tests/ai-storage-service-tests.ts

@@ -216,6 +216,99 @@ export const defineAiStorageServiceTests = (
       assert.ok(runs.every((r) => r.threadId === thread.id))
     })
 
+    test('saveScore and getScores', async () => {
+      const thread = await storage.createThread('resource-score')
+      const now = new Date()
+
+      const runId = await storage.createRun({
+        agentName: 'scored-agent',
+        threadId: thread.id,
+        resourceId: 'resource-score',
+        status: 'completed',
+        usage: { inputTokens: 10, outputTokens: 5, model: 'test' },
+        createdAt: now,
+        updatedAt: now,
+      })
+
+      await storage.saveScore({
+        runId,
+        scorerName: 'tool-error-rate',
+        score: 1,
+      })
+      await storage.saveScore({
+        runId,
+        scorerName: 'helpfulness',
+        score: 0.75,
+        reason: 'answered the question but skipped the fee',
+        metadata: { subScores: { accuracy: 0.9 }, judgeTokens: 412 },
+      })
+
+      const scores = await storage.getScores(runId)
+      assert.equal(scores.length, 2)
+
+      const byName = new Map(scores.map((s) => [s.scorerName, s]))
+
+      const clean = byName.get('tool-error-rate')
+      assert.ok(clean)
+      assert.equal(clean.score, 1)
+      assert.equal(clean.reason, undefined)
+      assert.equal(clean.metadata, undefined)
+
+      const helpful = byName.get('helpfulness')
+      assert.ok(helpful)
+      // A fractional score has to survive the round trip: an integer column
+      // would quietly turn every grade into 0 or 1.
+      assert.equal(helpful.score, 0.75)
+      assert.equal(helpful.reason, 'answered the question but skipped the fee')
+      assert.deepEqual(helpful.metadata, {
+        subScores: { accuracy: 0.9 },
+        judgeTokens: 412,
+      })
+      assert.ok(helpful.createdAt instanceof Date)
+    })
+
+    test('a re-grade appends rather than replacing the grade that was acted on', async () => {
+      const thread = await storage.createThread('resource-regrade')
+      const now = new Date()
+
+      const runId = await storage.createRun({
+        agentName: 'regraded-agent',
+        threadId: thread.id,
+        resourceId: 'resource-regrade',
+        status: 'completed',
+        usage: { inputTokens: 1, outputTokens: 1, model: 'test' },
+        createdAt: now,
+        updatedAt: now,
+      })
+
+      await storage.saveScore({ runId, scorerName: 'helpfulness', score: 0.2 })
+      await storage.saveScore({ runId, scorerName: 'helpfulness', score: 0.8 })
+
+      const scores = await storage.getScores(runId)
+      assert.equal(scores.length, 2)
+      assert.deepEqual(
+        scores.map((s) => s.score),
+        [0.2, 0.8]
+      )
+    })
+
+    test('getScores is empty for a run nothing graded', async () => {
+      const thread = await storage.createThread('resource-ungraded')
+      const now = new Date()
+
+      const runId = await storage.createRun({
+        agentName: 'ungraded-agent',
+        threadId: thread.id,
+        resourceId: 'resource-ungraded',
+        status: 'completed',
+        usage: { inputTokens: 1, outputTokens: 1, model: 'test' },
+        createdAt: now,
+        updatedAt: now,
+      })
+
+      assert.deepEqual(await storage.getScores(runId), [])
+    })
+
     test('resolveApproval', async () => {
       const thread = await storage.createThread('resource-8')
       const now = new Date()

package/src/types/core.types.ts

@@ -17,11 +17,11 @@ import type { PikkuQueue, QueueService } from '../wirings/queue/queue.types.js'
 import type { PikkuCLI } from '../wirings/cli/cli.types.js'
 import type {
   PikkuWorkflowWire,
-  PikkuScenarioWire,
   WorkflowService,
   WorkflowServiceConfig,
   WorkflowStepWire,
 } from '../wirings/workflow/workflow.types.js'
+import type { PikkuScenarioWire } from '../wirings/workflow/scenario.types.js'
 import type {
   PikkuBrowserWire,
   PikkuScenarioStepWire,
@@ -188,8 +188,6 @@ export type FunctionRuntimeMeta = {
   audit?: {
     durability: AuditDurability
   }
-  /** Keeps the full `SecretService`. Set by the inspector, read by the runner. */
-  secretBroker?: boolean
   version?: number
   approvalRequired?: boolean
   approvalDescription?: string
@@ -383,10 +381,9 @@ export type PikkuWire<
   MCPTools extends string | never = never,
   TypedWorkflow extends PikkuWorkflowWire | never = PikkuWorkflowWire,
   TriggerOutput = unknown,
-  // Defaulted to `any` rather than to `Out`: the emitted `TypedScenario<Out>`
-  // supplies the real context, while `PikkuWire`'s own defaults are what other
-  // generics constrain against — and a constraint that pinned the context to
-  // `Out` would reject every wire whose scenario output is anything else.
+  // `any`, not `Out`: the emitted `TypedScenario<Out>` supplies the real
+  // context, and this default is what other generics constrain against. See
+  // `ScenarioContext`.
   TypedScenario extends PikkuScenarioWire<any> | never = PikkuScenarioWire<any>,
   TypedActors extends ScenarioPersonas = ScenarioPersonas,
 > = {

package/src/types/state.types.ts

@@ -29,6 +29,10 @@ import type {
   CoreAIAgent,
   AIAgentMeta,
 } from '../wirings/ai-agent/ai-agent.types.js'
+import type {
+  PikkuAIScorer,
+  ScorerMeta,
+} from '../wirings/ai-scorer/ai-scorer.types.js'
 import type {
   CoreGateway,
   GatewaysMeta,
@@ -43,9 +47,9 @@ import type {
 } from '../wirings/scheduler/scheduler.types.js'
 import type {
   CoreWorkflow,
-  CoreFeature,
   WorkflowsRuntimeMeta,
 } from '../wirings/workflow/workflow.types.js'
+import type { CoreFeature } from '../wirings/workflow/scenario.types.js'
 import type {
   CoreTrigger,
   CoreTriggerSource,
@@ -98,6 +102,14 @@ export interface PikkuPackageState {
         variableOverrides?: Record<string, string>
         /** Per-instance name-aliases: logical name the addon reads -> actual project credential name */
         credentialOverrides?: Record<string, string>
+        /** Secrets the host lends this instance, named as the addon reads them */
+        secretGrants?: string[]
+        /** Credentials the host lends this instance, named as the addon reads them */
+        credentialGrants?: string[]
+        /** Why this instance gets the whole `SecretService` rather than one scoped to its declared secrets */
+        globalSecrets?: string
+        /** Why this instance gets the whole `CredentialService` rather than one scoped to its declared credentials */
+        globalCredentials?: string
         /** Set by `wireRemoteAddon`: this namespace is consumed remotely over HTTP, not bundled */
         remote?: boolean
         /** Remote host base URL (wireRemoteAddon) — string or resolver over singleton services */
@@ -153,6 +165,10 @@ export interface PikkuPackageState {
   agent: {
     agents: Map<string, CoreAIAgent>
     agentsMeta: AIAgentMeta
+    scorers: Map<string, PikkuAIScorer>
+    scorersMeta: ScorerMeta
+    /** Alias -> `provider/model`, from the `models` table in pikku.config.json. */
+    modelAliases: Record<string, string>
   }
   gateway: {
     gateways: Map<string, CoreGateway>
@@ -204,5 +220,9 @@ export interface PikkuPackageState {
       { name: string; displayName: string; type: string; oauth2?: boolean }
     > | null
     requiredParentServices: string[] | null
+    /** The secret keys this package is allowed to read, emitted by the CLI from
+     *  what the package declares and is seen to read. `null` when the package
+     *  was never analysed, which scopes it to nothing. */
+    declaredSecrets: string[] | null
   }
 }

package/src/wirings/actor-flow/index.ts

@@ -1,9 +1,6 @@
 export type {
-  ActorFlowApprovalPolicy,
   ActorFlowVerdict,
   ConverseOptions,
-  TargetAgentReply,
-  TargetPendingApproval,
   TargetAgentDriver,
 } from './actor-flow.types.js'
 export {

package/src/wirings/ai-agent/ai-agent-finalize.test.ts

@@ -0,0 +1,186 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import {
+  applyOutputMiddleware,
+  finalizeAgentRun,
+  type FinalizedRun,
+} from './ai-agent-finalize.js'
+import type { AIAgentStep, PikkuAIMiddlewareHooks } from './ai-agent.types.js'
+import type { AIRunStateService } from '../../services/ai-run-state-service.js'
+
+const step = (
+  toolCalls: NonNullable<AIAgentStep['toolCalls']>
+): AIAgentStep => ({
+  usage: { inputTokens: 0, outputTokens: 0 },
+  toolCalls,
+})
+
+const call = (name: string, result: string) => ({
+  name,
+  args: {} as Record<string, unknown>,
+  result,
+})
+
+describe('applyOutputMiddleware', () => {
+  test('hands the middleware every tool call the run made, flattened across steps', async () => {
+    const seen: string[][] = []
+    const middleware: PikkuAIMiddlewareHooks[] = [
+      {
+        modifyOutput: (_services, ctx) => {
+          seen.push(ctx.toolCalls.map((c) => c.name))
+          return { text: ctx.text, messages: ctx.messages }
+        },
+      },
+    ]
+
+    await applyOutputMiddleware(
+      middleware,
+      {},
+      {
+        text: 'done',
+        messages: [],
+        steps: [step([call('search', 'a')]), step([call('fetch', 'b')])],
+        usage: { inputTokens: 1, outputTokens: 2 },
+      }
+    )
+
+    assert.deepEqual(seen, [['search', 'fetch']])
+  })
+
+  test('a rewritten tool call is redistributed back onto the step it came from', async () => {
+    const middleware: PikkuAIMiddlewareHooks[] = [
+      {
+        modifyOutput: (_services, ctx) => ({
+          text: ctx.text,
+          messages: ctx.messages,
+          toolCalls: ctx.toolCalls.map((c) => ({ ...c, result: '[redacted]' })),
+        }),
+      },
+    ]
+
+    const { steps } = await applyOutputMiddleware(
+      middleware,
+      {},
+      {
+        text: 'done',
+        messages: [],
+        steps: [
+          step([call('search', 'secret')]),
+          step([call('fetch', 'token')]),
+        ],
+        usage: { inputTokens: 0, outputTokens: 0 },
+      }
+    )
+
+    assert.deepEqual(
+      steps.map((s) => s.toolCalls?.map((c) => [c.name, c.result])),
+      [[['search', '[redacted]']], [['fetch', '[redacted]']]]
+    )
+  })
+
+  test('dropping a tool call collapses the calls into the last step rather than mis-attributing them', async () => {
+    const middleware: PikkuAIMiddlewareHooks[] = [
+      {
+        modifyOutput: (_services, ctx) => ({
+          text: ctx.text,
+          messages: ctx.messages,
+          toolCalls: ctx.toolCalls.filter((c) => c.name !== 'search'),
+        }),
+      },
+    ]
+
+    const { steps } = await applyOutputMiddleware(
+      middleware,
+      {},
+      {
+        text: 'done',
+        messages: [],
+        steps: [step([call('search', 'a')]), step([call('fetch', 'b')])],
+        usage: { inputTokens: 0, outputTokens: 0 },
+      }
+    )
+
+    assert.deepEqual(
+      steps.map((s) => s.toolCalls?.map((c) => c.name)),
+      [[], ['fetch']]
+    )
+  })
+
+  test('runs the chain in reverse registration order, each hook seeing the previous rewrite', async () => {
+    const order: string[] = []
+    const middleware: PikkuAIMiddlewareHooks[] = [
+      {
+        modifyOutput: (_services, ctx) => {
+          order.push(`first saw ${ctx.text}`)
+          return { text: `${ctx.text}+first`, messages: ctx.messages }
+        },
+      },
+      {
+        modifyOutput: (_services, ctx) => {
+          order.push(`second saw ${ctx.text}`)
+          return { text: `${ctx.text}+second`, messages: ctx.messages }
+        },
+      },
+    ]
+
+    const { text } = await applyOutputMiddleware(
+      middleware,
+      {},
+      {
+        text: 'raw',
+        messages: [],
+        steps: [],
+        usage: { inputTokens: 0, outputTokens: 0 },
+      }
+    )
+
+    assert.deepEqual(order, ['second saw raw', 'first saw raw+second'])
+    assert.equal(text, 'raw+second+first')
+  })
+})
+
+describe('finalizeAgentRun', () => {
+  const run = (usage: FinalizedRun['usage']): FinalizedRun => ({
+    runId: 'run-1',
+    agentName: 'assistant',
+    threadId: 'thread-1',
+    input: 'what is the capital of France?',
+    text: 'done',
+    steps: [],
+    usage,
+  })
+
+  test('completes the run and records its usage', async () => {
+    const updates: unknown[] = []
+    const aiRunState = {
+      updateRun: async (_runId: string, update: unknown) => {
+        updates.push(update)
+      },
+    } as unknown as AIRunStateService
+
+    await finalizeAgentRun(
+      aiRunState,
+      run({ inputTokens: 10, outputTokens: 20, model: 'gpt-4o' })
+    )
+
+    assert.deepEqual(updates, [
+      {
+        status: 'completed',
+        usage: { inputTokens: 10, outputTokens: 20, model: 'gpt-4o' },
+      },
+    ])
+  })
+
+  test('omits usage entirely when no model reported one', async () => {
+    const updates: any[] = []
+    const aiRunState = {
+      updateRun: async (_runId: string, update: unknown) => {
+        updates.push(update)
+      },
+    } as unknown as AIRunStateService
+
+    await finalizeAgentRun(aiRunState, run({ inputTokens: 0, outputTokens: 0 }))
+
+    assert.deepEqual(updates, [{ status: 'completed' }])
+  })
+})