@pikku/core 0.12.1 → 0.12.3

package/src/wirings/ai-agent/ai-agent-stream.ts

@@ -39,10 +39,6 @@ import {
   type StreamAIAgentOptions,
   type StreamContext,
 } from './ai-agent-prepare.js'
-import {
-  createAssistantUIChannel,
-  parseAssistantUIInput,
-} from './ai-agent-assistant-ui.js'
 import { resolveModelConfig } from './ai-agent-model-config.js'
 import type { AIRunStateService } from '../../services/ai-run-state-service.js'
 import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
@@ -114,6 +110,7 @@ function createPersistingChannel(
     },
     flush: flushStep,
     close: () => parent.close(),
+    sendBinary: (data) => parent.sendBinary(data),
     send: (event: AIStreamEvent) => {
       if (storage) {
         switch (event.type) {
@@ -237,7 +234,7 @@ type StepLoopParams = {
 
 type StepLoopResult =
   | { outcome: 'done' }
-  | { outcome: 'approval'; approval: ToolApprovalRequired }
+  | { outcome: 'approval'; approvals: ToolApprovalRequired[] }
 
 async function runStreamStepLoop(
   params: StepLoopParams
@@ -290,13 +287,26 @@ async function runStreamStepLoop(
 
     if (stepResult.toolCalls.length === 0) break
 
-    const approvalNeeded = checkForApproval(
+    const approvalsNeeded = checkForApprovals(
       stepResult,
       runnerParams.tools,
       runId
     )
-    if (approvalNeeded) {
-      return { outcome: 'approval', approval: approvalNeeded }
+    if (approvalsNeeded.length > 0) {
+      // For each approval, call approvalDescriptionFn if available
+      for (const approval of approvalsNeeded) {
+        const toolDef = runnerParams.tools.find(
+          (t) => t.name === approval.toolName
+        )
+        if (toolDef?.approvalDescriptionFn && !approval.reason) {
+          try {
+            approval.reason = await toolDef.approvalDescriptionFn(approval.args)
+          } catch {
+            // If description generation fails, continue without it
+          }
+        }
+      }
+      return { outcome: 'approval', approvals: approvalsNeeded }
     }
 
     appendStepMessages(runnerParams, stepResult)
@@ -305,16 +315,20 @@ async function runStreamStepLoop(
   return { outcome: 'done' }
 }
 
-function checkForApproval(
+export function checkForApprovals(
   stepResult: AIAgentStepResult,
   tools: AIAgentRunnerParams['tools'],
   runId: string
-): ToolApprovalRequired | null {
+): ToolApprovalRequired[] {
+  const approvals: ToolApprovalRequired[] = []
   for (const tc of stepResult.toolCalls) {
     const toolDef = tools.find((t) => t.name === tc.toolName)
 
     if (toolDef?.needsApproval) {
-      return new ToolApprovalRequired(tc.toolCallId, tc.toolName, tc.args)
+      approvals.push(
+        new ToolApprovalRequired(tc.toolCallId, tc.toolName, tc.args)
+      )
+      continue
     }
 
     const tr = stepResult.toolResults.find(
@@ -332,22 +346,46 @@ function checkForApproval(
         displayToolName?: string
         displayArgs?: unknown
         agentRunId?: string
+        subApprovals?: Array<{
+          toolCallId: string
+          toolName: string
+          args: unknown
+          runId: string
+        }>
+      }
+      if (r.subApprovals?.length) {
+        for (const sub of r.subApprovals) {
+          approvals.push(
+            new ToolApprovalRequired(
+              sub.toolCallId,
+              r.toolName,
+              r.args,
+              undefined,
+              sub.toolName,
+              sub.args,
+              r.agentRunId
+            )
+          )
+        }
+      } else {
+        approvals.push(
+          new ToolApprovalRequired(
+            tc.toolCallId,
+            r.toolName,
+            r.args,
+            r.reason,
+            r.displayToolName,
+            r.displayArgs,
+            r.agentRunId
+          )
+        )
       }
-      return new ToolApprovalRequired(
-        tc.toolCallId,
-        r.toolName,
-        r.args,
-        r.reason,
-        r.displayToolName,
-        r.displayArgs,
-        r.agentRunId
-      )
     }
   }
-  return null
+  return approvals
 }
 
-function appendStepMessages(
+export function appendStepMessages(
   runnerParams: AIAgentRunnerParams,
   stepResult: AIAgentStepResult
 ): void {
@@ -383,8 +421,8 @@ function appendStepMessages(
   }
 }
 
-function handleApproval(
-  err: ToolApprovalRequired,
+function handleApprovals(
+  approvals: ToolApprovalRequired[],
   runId: string,
   channel: AIStreamChannel,
   aiRunState: AIRunStateService,
@@ -393,37 +431,41 @@ function handleApproval(
   return (async () => {
     await persistingChannel.flush()
 
-    const pendingApproval = err.agentRunId
-      ? {
-          type: 'agent-call' as const,
-          toolCallId: err.toolCallId,
-          agentName: err.toolName,
-          agentRunId: err.agentRunId,
-          displayToolName: err.displayToolName ?? err.toolName,
-          displayArgs: err.displayArgs ?? err.args,
-        }
-      : {
-          type: 'tool-call' as const,
-          toolCallId: err.toolCallId,
-          toolName: err.toolName,
-          args: err.args,
-        }
+    const pendingApprovals = approvals.map((err) =>
+      err.agentRunId
+        ? {
+            type: 'agent-call' as const,
+            toolCallId: err.toolCallId,
+            agentName: err.toolName,
+            agentRunId: err.agentRunId,
+            displayToolName: err.displayToolName ?? err.toolName,
+            displayArgs: err.displayArgs ?? err.args,
+          }
+        : {
+            type: 'tool-call' as const,
+            toolCallId: err.toolCallId,
+            toolName: err.toolName,
+            args: err.args,
+          }
+    )
 
     await aiRunState.updateRun(runId, {
       status: 'suspended',
       suspendReason: 'approval',
-      pendingApprovals: [pendingApproval],
+      pendingApprovals,
     })
 
-    const approvalEvent = {
-      type: 'approval-request' as const,
-      toolCallId: err.toolCallId,
-      toolName: err.displayToolName ?? err.toolName,
-      args: err.displayArgs ?? err.args,
-      reason: err.reason,
-      runId,
+    for (const err of approvals) {
+      const approvalEvent = {
+        type: 'approval-request' as const,
+        toolCallId: err.toolCallId,
+        toolName: err.displayToolName ?? err.toolName,
+        args: err.displayArgs ?? err.args,
+        reason: err.reason,
+        runId,
+      }
+      channel.send(approvalEvent as any)
     }
-    channel.send(approvalEvent as any)
     channel.send({ type: 'done' })
     channel.close()
   })()
@@ -431,34 +473,24 @@ function handleApproval(
 
 export async function streamAIAgent(
   agentName: string,
-  input:
-    | { message: string; threadId: string; resourceId: string }
-    | Record<string, unknown>,
+  input: {
+    message: string
+    threadId: string
+    resourceId: string
+    model?: string
+    temperature?: number
+  },
   channel: AIStreamChannel,
   params: RunAIAgentParams,
   agentSessionMap?: Map<string, string>,
   options?: StreamAIAgentOptions
-): Promise<void> {
+): Promise<string> {
   const sessionMap = agentSessionMap ?? new Map<string, string>()
 
-  const { agent: resolvedAgentForProtocol } = resolveAgent(agentName)
-  const inputRecord = input as Record<string, unknown>
-  const useUIMessageStream =
-    resolvedAgentForProtocol.protocol === 'ui-message-stream' ||
-    Array.isArray(inputRecord.messages)
-  let normalizedInput: { message: string; threadId: string; resourceId: string }
-  if (useUIMessageStream) {
-    normalizedInput = parseAssistantUIInput(inputRecord)
-    channel = createAssistantUIChannel(channel)
-  } else {
-    normalizedInput = input as {
-      message: string
-      threadId: string
-      resourceId: string
-    }
-  }
+  const normalizedInput = input
 
   const streamContext: StreamContext = { channel, options }
+  // delegateState is attached after prepareAgentRun resolves the agent config
 
   const {
     agent,
@@ -501,7 +533,7 @@ export async function streamAIAgent(
     })
     channel.send({ type: 'suspended', reason: 'rpc-missing', missingRpcs })
     channel.send({ type: 'done' })
-    return
+    return ''
   }
 
   const aiMiddlewares: PikkuAIMiddlewareHooks[] = agent.aiMiddleware ?? []
@@ -547,7 +579,12 @@ export async function streamAIAgent(
           allEvents,
           state,
         })
-        if (result != null) await next(result)
+        if (result == null) return
+        if (Array.isArray(result)) {
+          for (const r of result) await next(r)
+        } else {
+          await next(result)
+        }
       }
     })
 
@@ -574,8 +611,32 @@ export async function streamAIAgent(
         ).channel as AIStreamChannel)
       : channel
 
+  // In delegate mode (default), suppress parent's text from reaching the client
+  // AFTER a sub-agent has been called. If the parent responds directly (no delegation),
+  // its text goes through normally. Sub-agent text bypasses this path entirely
+  // (goes through subChannel → channel directly).
+  const isDelegateMode = agent.agentMode !== 'supervise' && meta.agents?.length
+  const delegateState = { delegated: false }
+  if (isDelegateMode) {
+    streamContext.delegateState = delegateState
+  }
+  const outputChannel = isDelegateMode
+    ? {
+        ...wrappedChannel,
+        send: (event: AIStreamEvent) => {
+          if (
+            delegateState.delegated &&
+            (event.type === 'text-delta' || event.type === 'reasoning-delta')
+          )
+            return
+          wrappedChannel.send(event)
+        },
+        delegateState,
+      }
+    : wrappedChannel
+
   const persistingChannel = createPersistingChannel(
-    wrappedChannel,
+    outputChannel,
     storage,
     threadId
   )
@@ -593,14 +654,14 @@ export async function streamAIAgent(
     })
 
     if (loopResult.outcome === 'approval') {
-      await handleApproval(
-        loopResult.approval,
+      await handleApprovals(
+        loopResult.approvals,
         runId,
         channel,
         aiRunState,
         persistingChannel
       )
-      return
+      return persistingChannel.fullText
     }
 
     await postStreamCleanup(
@@ -618,6 +679,7 @@ export async function streamAIAgent(
 
     channel.send({ type: 'done' })
     channel.close()
+    return persistingChannel.fullText
   } catch (err) {
     for (const mw of aiMiddlewares) {
       if (mw.onError) {
@@ -632,13 +694,17 @@ export async function streamAIAgent(
         }
       }
     }
-    await aiRunState.updateRun(runId, { status: 'failed' })
+    await aiRunState.updateRun(runId, {
+      status: 'failed',
+      errorMessage: err instanceof Error ? err.message : String(err),
+    })
     channel.send({
       type: 'error',
       message: err instanceof Error ? err.message : String(err),
     })
     channel.send({ type: 'done' })
     channel.close()
+    return persistingChannel.fullText
   }
 }
 
@@ -690,7 +756,8 @@ export async function resumeAIAgent(
       await aiRunState.updateRun(pending.agentRunId, { status: 'failed' })
     }
 
-    const denialResult = 'Tool call was not approved by the user'
+    const denialResult =
+      'The user explicitly declined this action. Inform them that it was declined and do not retry.'
 
     if (storage) {
       await storage.saveMessages(run.threadId, [
@@ -712,6 +779,17 @@ export async function resumeAIAgent(
       ])
     }
 
+    // Check remaining pending approvals
+    const updatedRun = await aiRunState.getRun(run.runId)
+    const remaining = updatedRun?.pendingApprovals ?? []
+
+    if (remaining.length > 0) {
+      // Still waiting for more approvals - don't continue step loop
+      channel.send({ type: 'done' })
+      channel.close()
+      return
+    }
+
     await aiRunState.updateRun(run.runId, { status: 'running' })
 
     await continueAfterToolResult(
@@ -735,7 +813,9 @@ export async function resumeAIAgent(
     if (!subRun) {
       throw new Error(`Sub-agent run not found: ${pending.agentRunId}`)
     }
-    const subPending = subRun.pendingApprovals?.[0]
+    const subPending =
+      subRun.pendingApprovals?.find((p) => p.toolCallId === input.toolCallId) ??
+      subRun.pendingApprovals?.[0]
     if (!subPending) {
       throw new Error(
         `No pending approval on sub-agent run ${pending.agentRunId}`
@@ -791,7 +871,7 @@ export async function resumeAIAgent(
     }
     const aiMiddlewaresForResume: PikkuAIMiddlewareHooks[] =
       agent.aiMiddleware ?? []
-    const { tools } = buildToolDefs(
+    const { tools } = await buildToolDefs(
       params,
       new Map<string, string>(),
       run.resourceId,
@@ -812,10 +892,12 @@ export async function resumeAIAgent(
       typeof pending.args === 'string' ? JSON.parse(pending.args) : pending.args
 
     let toolResult: unknown
+    let isError = false
     try {
       toolResult = await matchingTool.execute(toolArgs)
     } catch (execErr) {
       toolResult = `Error: ${execErr instanceof Error ? execErr.message : String(execErr)}`
+      isError = true
     }
 
     const resultStr =
@@ -842,9 +924,21 @@ export async function resumeAIAgent(
       toolCallId: input.toolCallId,
       toolName: pending.toolName,
       result: toolResult,
+      ...(isError ? { isError: true } : {}),
     })
   }
 
+  // Check remaining pending approvals after processing this one
+  const updatedRun = await aiRunState.getRun(run.runId)
+  const remaining = updatedRun?.pendingApprovals ?? []
+
+  if (remaining.length > 0) {
+    // Still waiting for more approvals - don't continue step loop
+    channel.send({ type: 'done' })
+    channel.close()
+    return
+  }
+
   await aiRunState.updateRun(run.runId, { status: 'running' })
 
   await continueAfterToolResult(
@@ -900,7 +994,7 @@ async function continueAfterToolResult(
   const allMessages = [...contextMessages, ...messages]
   const trimmedMessages = trimMessages(allMessages)
 
-  const instructions = buildInstructions(resolvedName, packageName)
+  const instructions = await buildInstructions(resolvedName, packageName)
 
   const aiMiddlewares: PikkuAIMiddlewareHooks[] = agent.aiMiddleware ?? []
   let modifiedMessages = trimmedMessages
@@ -928,7 +1022,12 @@ async function continueAfterToolResult(
           allEvents,
           state,
         })
-        if (result != null) await next(result)
+        if (result == null) return
+        if (Array.isArray(result)) {
+          for (const r of result) await next(r)
+        } else {
+          await next(result)
+        }
       }
     })
 
@@ -960,14 +1059,16 @@ async function continueAfterToolResult(
   )
 
   const streamContext: StreamContext = { channel, options }
-  const resumeTools = buildToolDefs(
-    params,
-    new Map<string, string>(),
-    run.resourceId,
-    resolvedName,
-    packageName,
-    streamContext,
-    aiMiddlewares
+  const resumeTools = (
+    await buildToolDefs(
+      params,
+      new Map<string, string>(),
+      run.resourceId,
+      resolvedName,
+      packageName,
+      streamContext,
+      aiMiddlewares
+    )
   ).tools
 
   const resolved = resolveModelConfig(resolvedName, agent)
@@ -999,8 +1100,8 @@ async function continueAfterToolResult(
     })
 
     if (loopResult.outcome === 'approval') {
-      await handleApproval(
-        loopResult.approval,
+      await handleApprovals(
+        loopResult.approvals,
         run.runId,
         channel,
         aiRunState,
@@ -1038,7 +1139,10 @@ async function continueAfterToolResult(
         }
       }
     }
-    await aiRunState.updateRun(run.runId, { status: 'failed' })
+    await aiRunState.updateRun(run.runId, {
+      status: 'failed',
+      errorMessage: err instanceof Error ? err.message : String(err),
+    })
     channel.send({
       type: 'error',
       message: err instanceof Error ? err.message : String(err),

package/src/wirings/ai-agent/ai-agent.types.ts

@@ -19,6 +19,17 @@ export interface AIThread {
   updatedAt: Date
 }
 
+export type AIContentPart =
+  | { type: 'text'; text: string }
+  | { type: 'image'; data?: string; url?: string; mediaType?: string }
+  | {
+      type: 'file'
+      data?: string
+      url?: string
+      mediaType: string
+      filename?: string
+    }
+
 export interface AIToolCall {
   id: string
   name: string
@@ -34,7 +45,7 @@ export interface AIToolResult {
 export interface AIMessage {
   id: string
   role: 'system' | 'user' | 'assistant' | 'tool'
-  content?: string
+  content?: string | AIContentPart[]
   toolCalls?: AIToolCall[]
   toolResults?: AIToolResult[]
   createdAt: Date
@@ -45,10 +56,21 @@ export interface AIAgentStep {
   toolCalls?: { name: string; args: Record<string, unknown>; result: string }[]
 }
 
+export interface AIAgentInputAttachment {
+  type: 'image' | 'file'
+  data?: string
+  url?: string
+  mediaType?: string
+  filename?: string
+}
+
 export interface AIAgentInput {
   message: string
   threadId: string
   resourceId: string
+  attachments?: AIAgentInputAttachment[]
+  model?: string
+  temperature?: number
 }
 
 export interface AIAgentOutput {
@@ -58,6 +80,14 @@ export interface AIAgentOutput {
   threadId: string
   steps: AIAgentStep[]
   usage: { inputTokens: number; outputTokens: number }
+  status?: 'completed' | 'suspended'
+  pendingApprovals?: Array<{
+    toolCallId: string
+    toolName: string
+    args: unknown
+    reason?: string
+    runId: string
+  }>
 }
 
 export interface AIAgentToolDef {
@@ -66,6 +96,7 @@ export interface AIAgentToolDef {
   inputSchema: Record<string, unknown>
   execute: (input: unknown) => Promise<unknown>
   needsApproval?: boolean
+  approvalDescriptionFn?: (input: unknown) => Promise<string>
 }
 
 export interface PikkuAIMiddlewareHooks<
@@ -86,7 +117,11 @@ export interface PikkuAIMiddlewareHooks<
       allEvents: readonly AIStreamEvent[]
       state: State
     }
-  ) => Promise<AIStreamEvent | null> | AIStreamEvent | null
+  ) =>
+    | Promise<AIStreamEvent | AIStreamEvent[] | null>
+    | AIStreamEvent
+    | AIStreamEvent[]
+    | null
 
   modifyOutput?: (
     services: Services,
@@ -165,13 +200,14 @@ export type CoreAIAgent<
   temperature?: number
   tools?: unknown[]
   agents?: unknown[]
+  agentMode?: 'delegate' | 'supervise'
   memory?: AIAgentMemoryConfig
   maxSteps?: number
   toolChoice?: 'auto' | 'required' | 'none'
+  dynamicWorkflows?: 'read' | 'always' | 'ask'
   input?: unknown
   output?: unknown
   tags?: string[]
-  protocol?: 'ui-message-stream'
   prepareStep?: (ctx: {
     stepNumber: number
     messages: AIMessage[]
@@ -235,6 +271,21 @@ export type AIStreamEvent =
       session?: string
     }
   | { type: 'error'; message: string; agent?: string; session?: string }
+  | {
+      type: 'audio-delta'
+      data: string
+      format: string
+      agent?: string
+      session?: string
+    }
+  | { type: 'audio-done'; agent?: string; session?: string }
+  | {
+      type: 'workflow-created'
+      workflowName: string
+      graph: any
+      agent?: string
+      session?: string
+    }
   | {
       type: 'suspended'
       reason: 'rpc-missing'
@@ -266,6 +317,7 @@ export interface AgentRunState {
   threadId: string
   resourceId: string
   status: 'running' | 'suspended' | 'completed' | 'failed'
+  errorMessage?: string
   suspendReason?: 'approval' | 'rpc-missing'
   missingRpcs?: string[]
   pendingApprovals?: PendingApproval[]
@@ -280,6 +332,7 @@ export interface AgentRunRow {
   threadId: string
   resourceId: string
   status: string
+  errorMessage?: string
   suspendReason?: string
   missingRpcs?: string[]
   usageInputTokens: number
@@ -324,5 +377,6 @@ export type AIAgentMeta = Record<
     channelMiddleware?: MiddlewareMetadata[]
     aiMiddleware?: MiddlewareMetadata[]
     permissions?: PermissionMetadata[]
+    dynamicWorkflows?: 'read' | 'always' | 'ask'
   }
 >

package/src/wirings/ai-agent/index.ts

@@ -4,12 +4,8 @@ export {
   agentResume,
   agentApprove,
 } from './ai-agent-helpers.js'
-export { runAIAgent } from './ai-agent-runner.js'
+export { runAIAgent, resumeAIAgentSync } from './ai-agent-runner.js'
 export { streamAIAgent, resumeAIAgent } from './ai-agent-stream.js'
-export {
-  createAssistantUIChannel,
-  parseAssistantUIInput,
-} from './ai-agent-assistant-ui.js'
 export {
   type RunAIAgentParams,
   type StreamAIAgentOptions,
@@ -23,9 +19,11 @@ export {
 } from './ai-agent-registry.js'
 export type {
   AIAgentInput,
+  AIAgentInputAttachment,
   AIAgentMeta,
   AIAgentMemoryConfig,
   AIAgentStep,
+  AIContentPart,
   AgentRunRow,
   AgentRunService,
   AgentRunState,