@pikku/core 0.12.72 → 0.12.74

package/src/wirings/ai-agent/ai-agent-thread-ownership.test.ts

@@ -0,0 +1,301 @@
+import { beforeEach, describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { resetPikkuState, pikkuState } from '../../pikku-state.js'
+import { runAIAgent, resumeAIAgentSync } from './ai-agent-runner.js'
+import { ForbiddenError } from '../../errors/errors.js'
+import type { AIMessage, CoreAIAgent } from './ai-agent.types.js'
+import type { AIAgentRunnerParams } from '../../services/ai-agent-runner-service.js'
+
+const logger = {
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as any
+
+const sessionService = (session?: Record<string, unknown>) =>
+  ({ get: () => session, setInitial: () => {}, sessionChanged: false }) as any
+
+const addTestAgent = (agentName: string) => {
+  const agent: CoreAIAgent = {
+    name: agentName,
+    description: 'test agent',
+    instructions: 'be helpful',
+    model: 'test/test-model',
+  } as CoreAIAgent
+
+  pikkuState(null, 'agent', 'agentsMeta')[agentName] = {
+    ...agent,
+    inputSchema: null,
+    outputSchema: null,
+    workingMemorySchema: null,
+  } as any
+  pikkuState(null, 'agent', 'agents').set(agentName, agent)
+}
+
+type Harness = {
+  threads: Map<string, { id: string; resourceId: string }>
+  messages: Map<string, AIMessage[]>
+  created: Array<{ threadId: string; resourceId: string }>
+  modelSawMessages: AIMessage[][]
+}
+
+const createHarness = (
+  seedThreads: Array<{ threadId: string; resourceId: string }> = [],
+  seedMessages: Record<string, string[]> = {}
+): Harness => {
+  const threads = new Map(
+    seedThreads.map((t) => [t.threadId, { id: t.threadId, ...t }])
+  )
+  const messages = new Map<string, AIMessage[]>(
+    Object.entries(seedMessages).map(([threadId, texts]) => [
+      threadId,
+      texts.map((text, index) => ({
+        id: `${threadId}-${index}`,
+        role: 'user' as const,
+        content: text,
+        createdAt: new Date(),
+      })),
+    ])
+  )
+  const created: Harness['created'] = []
+  const modelSawMessages: AIMessage[][] = []
+
+  const storage = {
+    getThread: async (threadId: string) => {
+      const thread = threads.get(threadId)
+      if (!thread) throw new Error(`No thread ${threadId}`)
+      return thread
+    },
+    createThread: async (resourceId: string, options: { threadId: string }) => {
+      const thread = { id: options.threadId, resourceId }
+      threads.set(options.threadId, thread)
+      created.push({ threadId: options.threadId, resourceId })
+      return thread
+    },
+    getMessages: async (threadId: string) => messages.get(threadId) ?? [],
+    saveMessages: async (threadId: string, newMessages: AIMessage[]) => {
+      messages.set(threadId, [
+        ...(messages.get(threadId) ?? []),
+        ...newMessages,
+      ])
+    },
+    getWorkingMemory: async () => null,
+    saveWorkingMemory: async () => {},
+  }
+
+  pikkuState(null, 'package', 'singletonServices', {
+    logger,
+    aiStorage: storage,
+    aiAgentRunner: {
+      run: async (params: AIAgentRunnerParams) => {
+        modelSawMessages.push(params.messages)
+        return {
+          text: 'ok',
+          toolCalls: [],
+          toolResults: [],
+          usage: { inputTokens: 0, outputTokens: 0 },
+          finishReason: 'stop',
+        }
+      },
+    },
+    aiRunState: {
+      createRun: async () => 'run-1',
+      updateRun: async () => {},
+      getRun: async () => ({
+        id: 'run-1',
+        agentName: 'ownership-agent',
+        resourceId: 'victim',
+        threadId: 'thread-victim',
+        status: 'suspended',
+        pendingApprovals: [
+          { toolCallId: 'tc-1', toolName: 'refund', args: {}, runId: 'run-1' },
+        ],
+        messages: [],
+      }),
+      resolveApproval: async () => {},
+    },
+  } as any)
+
+  return { threads, messages, created, modelSawMessages }
+}
+
+beforeEach(() => {
+  resetPikkuState()
+})
+
+describe('AI agent thread ownership without a session', () => {
+  test('a sessionless caller cannot reach a thread by naming its resourceId', async () => {
+    addTestAgent('ownership-agent')
+    const harness = createHarness(
+      [{ threadId: 'thread-victim', resourceId: 'victim' }],
+      { 'thread-victim': ['the victim bank details are 1234'] }
+    )
+
+    await assert.rejects(
+      () =>
+        runAIAgent(
+          'ownership-agent',
+          {
+            message: 'what did we discuss?',
+            threadId: 'thread-victim',
+            resourceId: 'victim',
+          },
+          {}
+        ),
+      (error: unknown) => error instanceof ForbiddenError
+    )
+
+    assert.deepEqual(
+      harness.modelSawMessages,
+      [],
+      "the victim's history must never reach the model"
+    )
+  })
+
+  test('a sessionless caller cannot resume another party run', async () => {
+    addTestAgent('ownership-agent')
+    createHarness()
+
+    await assert.rejects(
+      () =>
+        resumeAIAgentSync(
+          'run-1',
+          [{ toolCallId: 'tc-1', approved: true }],
+          {}
+        ),
+      (error: unknown) => error instanceof ForbiddenError
+    )
+  })
+
+  test('a sessionless one-shot conversation still runs', async () => {
+    addTestAgent('ownership-agent')
+    const harness = createHarness()
+
+    const result = await runAIAgent(
+      'ownership-agent',
+      { message: 'hello', threadId: 'thread-new', resourceId: 'whatever' },
+      {}
+    )
+
+    assert.equal(result.text, 'ok')
+    assert.equal(harness.created.length, 1)
+    assert.equal(harness.created[0]!.threadId, 'thread-new')
+    assert.notEqual(
+      harness.created[0]!.resourceId,
+      'whatever',
+      'a client-supplied resourceId must never become the ownership key'
+    )
+    assert.deepEqual(
+      harness.modelSawMessages[0]?.map((m) => m.content),
+      ['hello']
+    )
+  })
+
+  test('a sessionless run keeps its own thread across nested agent calls in one request', async () => {
+    addTestAgent('ownership-agent')
+    const harness = createHarness()
+    const requestParams = {}
+
+    await runAIAgent(
+      'ownership-agent',
+      { message: 'first', threadId: 'thread-sub', resourceId: 'ignored' },
+      requestParams
+    )
+    await runAIAgent(
+      'ownership-agent',
+      { message: 'second', threadId: 'thread-sub', resourceId: 'ignored' },
+      requestParams
+    )
+
+    assert.equal(harness.created.length, 1)
+    assert.equal(harness.modelSawMessages.length, 2)
+  })
+
+  test('two sessionless one-shot runs do not share an owner', async () => {
+    addTestAgent('ownership-agent')
+    const harness = createHarness()
+
+    await runAIAgent(
+      'ownership-agent',
+      { message: 'hello', threadId: 'thread-a', resourceId: 'shared' },
+      {}
+    )
+    await runAIAgent(
+      'ownership-agent',
+      { message: 'hello', threadId: 'thread-b', resourceId: 'shared' },
+      {}
+    )
+
+    assert.notEqual(
+      harness.created[0]!.resourceId,
+      harness.created[1]!.resourceId
+    )
+  })
+})
+
+describe('AI agent thread ownership with a session', () => {
+  test('an authenticated caller still reaches its own thread', async () => {
+    addTestAgent('ownership-agent')
+    const harness = createHarness(
+      [{ threadId: 'thread-alice', resourceId: 'alice:default' }],
+      { 'thread-alice': ['earlier alice turn'] }
+    )
+
+    const result = await runAIAgent(
+      'ownership-agent',
+      {
+        message: 'and then?',
+        threadId: 'thread-alice',
+        resourceId: 'default',
+      },
+      { sessionService: sessionService({ userId: 'alice' }) }
+    )
+
+    assert.equal(result.text, 'ok')
+    assert.deepEqual(
+      harness.modelSawMessages[0]?.map((m) => m.content),
+      ['earlier alice turn', 'and then?']
+    )
+    assert.equal(harness.created.length, 0)
+  })
+
+  test('an authenticated caller creates threads under its own principal', async () => {
+    addTestAgent('ownership-agent')
+    const harness = createHarness()
+
+    await runAIAgent(
+      'ownership-agent',
+      { message: 'hello', threadId: 'thread-new', resourceId: 'project-1' },
+      { sessionService: sessionService({ userId: 'alice' }) }
+    )
+
+    assert.deepEqual(harness.created, [
+      { threadId: 'thread-new', resourceId: 'alice:project-1' },
+    ])
+  })
+
+  test("an authenticated caller cannot reach another user's thread", async () => {
+    addTestAgent('ownership-agent')
+    const harness = createHarness(
+      [{ threadId: 'thread-bob', resourceId: 'bob:default' }],
+      { 'thread-bob': ['bob secret'] }
+    )
+
+    await assert.rejects(
+      () =>
+        runAIAgent(
+          'ownership-agent',
+          {
+            message: 'what did we discuss?',
+            threadId: 'thread-bob',
+            resourceId: 'bob:default',
+          },
+          { sessionService: sessionService({ userId: 'alice' }) }
+        ),
+      (error: unknown) => error instanceof ForbiddenError
+    )
+    assert.deepEqual(harness.modelSawMessages, [])
+  })
+})

package/src/wirings/ai-agent/ai-agent.types.ts

@@ -7,8 +7,10 @@ import type {
   MiddlewareMetadata,
   PermissionMetadata,
 } from '../../types/core.types.js'
+import type { AIProviderOptions } from '../../services/ai-agent-runner-service.js'
 import type { PikkuChannel } from '../channel/channel.types.js'
 import type { CorePikkuChannelMiddleware } from '../channel/channel.types.js'
+import type { ApprovalPolicy } from '../channel/channel-rpc.js'
 
 export interface AIThread {
   id: string
@@ -58,6 +60,23 @@ export interface AIMessage {
   toolCalls?: AIToolCall[]
   toolResults?: AIToolResult[]
   reasoningContent?: string
+  /**
+   * Set on an assistant message whose generation was cut short by
+   * an interrupt — the text is a mid-sentence fragment, not a reply.
+   * Carried into the next turn's context so the model can see it said only
+   * this much before being talked over, and decide for itself whether to
+   * resume, restate or move on. Storage that drops unknown fields loses the
+   * marker but keeps the fragment, which degrades to a truncated reply.
+   */
+  interrupted?: boolean
+  /**
+   * Set on a tool message whose result arrived after the run was interrupted,
+   * so the model never got to describe it. The work happened; only the reply
+   * about it was lost. It is ordinary thread context on the next turn — the
+   * model raises it unprompted ("that deploy did go through") rather than the
+   * result being silently dropped or replayed as a fresh tool call.
+   */
+  undelivered?: boolean
   createdAt: Date
 }
 
@@ -118,13 +137,24 @@ export interface AIAgentOutput {
  */
 export type SessionScope = 'user' | 'org'
 
-export interface AIAgentToolDef {
+/**
+ * `ApprovalPolicy` is shared with channel capabilities, but `Partial` here:
+ * absent `needsApproval` means "do not ask", which is safe because a tool is
+ * written by whoever runs the server it executes on. A capability runs on
+ * someone else's machine, so it requires the field instead.
+ */
+export interface AIAgentToolDef extends Partial<ApprovalPolicy> {
   name: string
   description: string
   inputSchema: Record<string, unknown>
   execute: (input: unknown) => Promise<unknown>
-  needsApproval?: boolean
-  approvalDescriptionFn?: (input: unknown) => Promise<string>
+  /**
+   * Mirrors the pikku function's `readonly` flag. A read that gets interrupted
+   * is discarded rather than reported: nothing changed, and by the next turn
+   * the data may be stale anyway — re-reading is cheaper than explaining a
+   * result the user never asked to hear about.
+   */
+  readonly?: boolean
   /**
    * Set only by the framework on sub-agent delegating tools. Such a tool may
    * legitimately return an `__approvalRequired` marker to forward a nested
@@ -152,6 +182,27 @@ export interface PikkuAIMiddlewareHooks<
       event: AIStreamEvent
       allEvents: readonly AIStreamEvent[]
       state: State
+      /**
+       * Push an event into the stream *after* this call has returned.
+       *
+       * Returning events blocks the stream until the hook resolves, which is
+       * right for anything derived from the event and wrong for anything slow.
+       * Speech is the motivating case: awaiting a text-to-speech call before
+       * returning the text delta stalls the reader at every full stop and
+       * serialises synthesis behind playback. Return the text immediately,
+       * start the slow work, and `emit` its result when it lands.
+       *
+       * Ordering among emitted events is the caller's problem — chain them if
+       * it matters — and a hook that emits must make sure anything still
+       * outstanding has landed before it lets `done` through.
+       */
+      emit: (event: AIStreamEvent) => Promise<void>
+      /**
+       * Aborts when the run is interrupted. Pass it to any provider call the
+       * hook makes: work started for a reply the user has already talked over
+       * is billed but never heard.
+       */
+      signal?: AbortSignal
     }
   ) =>
     | Promise<AIStreamEvent | AIStreamEvent[] | null>
@@ -232,8 +283,19 @@ export type CoreAIAgent<
   description: string
   summary?: string
   errors?: string[]
+  /**
+   * The three fields below are the system prompt. `buildInstructions` joins
+   * whichever are set with a blank line, always in this order — role, then
+   * personality, then goal — and appends the tool-usage rules if the agent has
+   * tools. Nothing checks them against each other: the split is there to keep a
+   * prompt legible, and text put in the wrong one still reaches the model.
+   *
+   * Who the agent is. 'You are a support engineer triaging inbound bugs.'
+   */
   role?: string
+  /** How it should sound: tone, vocabulary, how much it says at a time. */
   personality?: string
+  /** What it is for — the only one of the three that is required. */
   goal: string
   model: string
   temperature?: number
@@ -246,6 +308,16 @@ export type CoreAIAgent<
   memory?: AIAgentMemoryConfig
   maxSteps?: number
   toolChoice?: 'auto' | 'required' | 'none'
+  /**
+   * Per-provider model settings, keyed by provider id and passed through
+   * untouched — for anything only one vendor offers, which the fields above
+   * deliberately do not try to unify.
+   *
+   * `{ openai: { reasoningEffort: 'minimal' } }` is the one that matters for
+   * voice: on gpt-5-mini it measured 0.9s to first token against 2.5s at the
+   * default, and a spoken reply is waited through rather than skimmed.
+   */
+  providerOptions?: AIProviderOptions
   input?: unknown
   output?: unknown
   tags?: string[]
@@ -368,6 +440,15 @@ export type AIStreamEvent =
       reason: 'rpc-missing'
       missingRpcs: string[]
     }
+  | {
+      type: 'interrupted'
+      runId: string
+      /** The truncated assistant text at the moment generation stopped. */
+      text: string
+      reason: 'speech' | 'user' | 'timeout'
+      agent?: string
+      session?: string
+    }
   | { type: 'done' }
 
 export interface AIStreamChannel extends PikkuChannel<unknown, AIStreamEvent> {}
@@ -402,7 +483,7 @@ export interface AgentRunState {
   agentName: string
   threadId: string
   resourceId: string
-  status: 'running' | 'suspended' | 'completed' | 'failed'
+  status: 'running' | 'suspended' | 'completed' | 'failed' | 'interrupted'
   errorMessage?: string
   suspendReason?: 'approval' | 'credential' | 'rpc-missing'
   missingRpcs?: string[]

package/src/wirings/ai-agent/index.ts

@@ -3,12 +3,44 @@ export {
   agentStream,
   agentResume,
   agentApprove,
+  agentInterrupt,
 } from './ai-agent-helpers.js'
 export { wrapChannelWithAGUI, type AGUIEvent } from './ai-agent-agui.js'
 export { runAIAgent, resumeAIAgentSync } from './ai-agent-runner.js'
-export { streamAIAgent, resumeAIAgent } from './ai-agent-stream.js'
-export { voiceInput } from './voice-input.js'
-export { voiceOutput } from './voice-output.js'
+export {
+  streamAIAgent,
+  resumeAIAgent,
+  interruptAIAgent,
+} from './ai-agent-stream.js'
+export {
+  voiceInput,
+  readsAsNonSpeech,
+  NoSpeechDetectedError,
+} from './voice-input.js'
+export {
+  voiceOutput,
+  unspeakableScripts,
+  voiceForText,
+  type SpeakableScripts,
+} from './voice-output.js'
+export {
+  AgentInterruptedError,
+  awaitPendingInterruptNote,
+  getInFlightTools,
+  isAbortError,
+  isRunInterruptible,
+  persistOrphanedToolResults,
+  registerInterruptibleRun,
+  signalRunInterrupt,
+  trackInterruptNote,
+  trackToolExecution,
+} from './ai-agent-interrupt.js'
+export type {
+  AgentInterruption,
+  AgentInterruptResult,
+  InterruptibleRunHandle,
+  OrphanedToolResult,
+} from './ai-agent-interrupt.js'
 export {
   type RunAIAgentParams,
   type StreamAIAgentOptions,

package/src/wirings/ai-agent/voice-input.test.ts

@@ -1,15 +1,13 @@
 import { describe, test } from 'node:test'
 import assert from 'node:assert/strict'
 
-import { voiceInput } from './voice-input.js'
+import {
+  voiceInput,
+  readsAsNonSpeech,
+  NoSpeechDetectedError,
+} from './voice-input.js'
 import type { AIContentPart, AIMessage } from './ai-agent.types.js'
 
-/**
- * A runner whose `transcribe` reads instance state through `this`, mirroring the
- * real `VercelAIAgentRunner` where `transcribe` calls `this.getModel(...)`. If
- * the middleware grabs the method as a bare reference the receiver is lost and
- * `this` is undefined — which is exactly the regression this guards.
- */
 class ThisDependentRunner {
   private readonly transcript = 'the transcribed spoken words'
 
@@ -87,4 +85,71 @@ describe('voiceInput', () => {
       /voiceInput requires a transcription model/
     )
   })
+
+  test('a turn with no speech in it never reaches the model', async () => {
+    // The failure this prevents: the user says nothing, the model answers
+    // anyway, and what it answers is a guess about audio it could not hear.
+    // An ASR that returns '' on non-speech is what makes this checkable — see
+    // readsAsNonSpeech for why nothing subtler than '' survived contact.
+    const silent = {
+      async transcribe() {
+        return { text: '', segments: [], warnings: [] }
+      },
+    }
+    const mw = voiceInput({
+      model: 'deepinfra/nvidia/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b',
+    })
+
+    await assert.rejects(
+      () =>
+        mw.modifyInput!({ aiAgentRunner: silent } as any, {
+          messages: [audioMessage()],
+          instructions: 'sys',
+        }),
+      NoSpeechDetectedError
+    )
+  })
+
+  test('a transcript is passed through exactly as the provider wrote it', async () => {
+    // No reassembly, no trimming, no filtering — whatever the model heard is
+    // what the agent is asked about.
+    const spaced = {
+      async transcribe() {
+        return {
+          text: '  spacing   the provider chose  ',
+          segments: [],
+          warnings: [],
+        }
+      },
+    }
+    const mw = voiceInput({ model: 'mock/asr' })
+
+    const result = await mw.modifyInput!({ aiAgentRunner: spaced } as any, {
+      messages: [audioMessage()],
+      instructions: 'sys',
+    })
+
+    const parts = result.messages.at(-1)!.content as AIContentPart[]
+    assert.equal(
+      (parts[0] as { text: string }).text,
+      '  spacing   the provider chose  '
+    )
+  })
+})
+
+describe('readsAsNonSpeech', () => {
+  test('only an empty transcript counts as nothing said', () => {
+    assert.equal(readsAsNonSpeech({ text: '' }), true)
+    assert.equal(readsAsNonSpeech({ text: '   ' }), true)
+    assert.equal(readsAsNonSpeech({}), true)
+    assert.equal(readsAsNonSpeech({ text: 'add milk to the list' }), false)
+  })
+
+  test('a short or low-confidence-looking transcript is still speech', () => {
+    // The gate that used to live here would have been tempted by these. "Yes"
+    // is the single most important word in an approval flow, and dropping it
+    // because it is brief or quiet is far worse than keeping a stray one.
+    assert.equal(readsAsNonSpeech({ text: 'yes' }), false)
+    assert.equal(readsAsNonSpeech({ text: 'Thank you.' }), false)
+  })
 })

package/src/wirings/ai-agent/voice-input.ts

@@ -14,6 +14,46 @@ function base64ToUint8Array(base64: string): Uint8Array {
 
 const MAX_AUDIO_SIZE = 50 * 1024 * 1024
 
+/**
+ * Raised when every audio part in a turn turned out to be non-speech.
+ *
+ * Distinct from a transcription failure, because the response is different: a
+ * failure is worth reporting, whereas this is the microphone doing its job on a
+ * pause. A voice loop should catch it and go back to listening without running
+ * the agent — answering a hallucinated sentence is worse than answering nothing.
+ */
+export class NoSpeechDetectedError extends Error {
+  constructor() {
+    super('No speech was detected in the audio.')
+    this.name = 'NoSpeechDetectedError'
+  }
+}
+
+/**
+ * Whether nothing usable was said — an empty transcript, and nothing cleverer
+ * than that.
+ *
+ * There was a confidence gate here once, and it is worth recording why it is
+ * gone. Whisper is trained on subtitles, so audio it cannot parse comes back
+ * not as nothing but as stock filler appended to real speech: "Hello, my name
+ * is Yasir. Thank you." where only the first sentence was spoken. The obvious
+ * defence is to drop low-confidence segments, and it does not work. Measured on
+ * a real turn, the invented "Thank you." scored `avg_logprob` -0.52 against
+ * -0.30 for the genuine sentence beside it, and `no_speech_prob` read 0.000 for
+ * both. Whisper is *confident* when it invents, because the text it invents is
+ * high-probability text. No threshold separates those two numbers, and one set
+ * low enough to try would take real quiet speech with it.
+ *
+ * The fix belongs at the model, not behind it: an ASR that is not a
+ * subtitle-trained autoregressive decoder returns an empty string on non-speech
+ * and there is nothing left to filter. Hence this reduces to asking whether
+ * anything came back at all — which is a property every provider reports
+ * honestly, unlike per-segment confidence (Nemotron, for one, hardcodes
+ * `avg_logprob` to 0.0).
+ */
+export const readsAsNonSpeech = (result: { text?: string }): boolean =>
+  !(result.text ?? '').trim()
+
 async function fetchAsUint8Array(
   url: string,
   allowedAudioHosts?: string[]
@@ -57,9 +97,6 @@ export const voiceInput = (config?: {
       )
       if (!hasAudio) return { messages, instructions }
 
-      // Process parts in order, replacing each audio part with its transcription
-      // in place. Sequential, so no unbounded parallel downloads/transcriptions
-      // and original content ordering is preserved.
       const updatedContent: AIContentPart[] = []
       for (const p of parts) {
         if (!(p.type === 'file' && p.mediaType?.startsWith('audio/'))) {
@@ -87,9 +124,17 @@ export const voiceInput = (config?: {
               }
             : {}),
         })
+        // Dropped when nothing was said: an empty text part is still a turn the
+        // model will answer, and what it answers is a guess about what it could
+        // not hear.
+        if (readsAsNonSpeech(result)) continue
         updatedContent.push({ type: 'text' as const, text: result.text })
       }
 
+      // Every part was audio, and none of it was speech. There is nothing left
+      // to send, and a message with no content is not a question.
+      if (updatedContent.length === 0) throw new NoSpeechDetectedError()
+
       return {
         messages: [
           ...messages.slice(0, -1),