@pikku/core 0.12.1 → 0.12.2

package/src/services/local-gateway-service.ts

@@ -0,0 +1,62 @@
+import type { GatewayService } from './gateway-service.js'
+import type { GatewayAdapter } from '../wirings/gateway/gateway.types.js'
+import { pikkuState, getSingletonServices } from '../pikku-state.js'
+import { createListenerMessageHandler } from '../wirings/gateway/gateway-runner.js'
+
+/**
+ * Local GatewayService implementation.
+ *
+ * Starts all registered listener gateways unconditionally (single-process).
+ * For distributed deployments, implement `GatewayService` with leader
+ * election or similar coordination.
+ *
+ * @example
+ * ```typescript
+ * const gatewayService = new LocalGatewayService()
+ * await gatewayService.start()
+ *
+ * // On shutdown
+ * await gatewayService.stop()
+ * ```
+ */
+export class LocalGatewayService implements GatewayService {
+  private activeAdapters = new Map<string, GatewayAdapter>()
+
+  async start(): Promise<void> {
+    const singletonServices = getSingletonServices()
+    const gateways = pikkuState(null, 'gateway', 'gateways')
+
+    for (const [name, config] of gateways) {
+      if (config.type !== 'listener') continue
+      if (this.activeAdapters.has(name)) continue
+
+      const handleMessage = createListenerMessageHandler(
+        name,
+        config,
+        singletonServices
+      )
+
+      await config.adapter.init(handleMessage)
+      this.activeAdapters.set(name, config.adapter)
+      singletonServices.logger.info(`Started listener gateway: ${name}`)
+    }
+  }
+
+  async stop(): Promise<void> {
+    for (const [name, adapter] of this.activeAdapters) {
+      try {
+        await adapter.close()
+      } catch (e: any) {
+        try {
+          getSingletonServices().logger.error(
+            `Error closing listener gateway '${name}':`,
+            e
+          )
+        } catch {
+          // logger may not be available during shutdown
+        }
+      }
+    }
+    this.activeAdapters.clear()
+  }
+}

package/src/types/core.types.ts

@@ -22,6 +22,7 @@ import type {
 } from '../wirings/workflow/workflow.types.js'
 import type { PikkuGraphWire } from '../wirings/workflow/graph/workflow-graph.types.js'
 import type { PikkuTrigger } from '../wirings/trigger/trigger.types.js'
+import type { PikkuGateway } from '../wirings/gateway/gateway.types.js'
 import type { SchedulerService } from '../services/scheduler-service.js'
 import type { DeploymentService } from '../services/deployment-service.js'
 import type { AIStorageService } from '../services/ai-storage-service.js'
@@ -42,6 +43,7 @@ export type PikkuWiringTypes =
   | 'cli'
   | 'workflow'
   | 'agent'
+  | 'gateway'
 
 export interface FunctionServicesMeta {
   optimized: boolean
@@ -249,6 +251,7 @@ export type PikkuWire<
   workflowStep: WorkflowStepWire
   graph: PikkuGraphWire
   trigger: PikkuTrigger<TriggerOutput>
+  gateway: PikkuGateway
   session: HasInitialSession extends true
     ? UserSession
     : UserSession | undefined

package/src/types/state.types.ts

@@ -26,6 +26,10 @@ import type {
   CoreAIAgent,
   AIAgentMeta,
 } from '../wirings/ai-agent/ai-agent.types.js'
+import type {
+  CoreGateway,
+  GatewaysMeta,
+} from '../wirings/gateway/gateway.types.js'
 import type {
   CoreQueueWorker,
   QueueWorkersMeta,
@@ -121,6 +125,10 @@ export interface PikkuPackageState {
     agents: Map<string, CoreAIAgent>
     agentsMeta: AIAgentMeta
   }
+  gateway: {
+    gateways: Map<string, CoreGateway>
+    meta: GatewaysMeta
+  }
   cli: {
     meta: CLIMeta | Record<string, any> // Backward compatible with old published CLI format
     programs: Record<string, CLIProgramState>

package/src/utils.ts

@@ -103,7 +103,7 @@ export const stopSingletonServices = async (): Promise<void> => {
   const singletonServices = getSingletonServices()
   const logger = singletonServices.logger
 
-  // First, stop all addon package singleton services
+  // Stop all addon package singleton services
   const stateMap = getAllPackageStates()
   if (stateMap.size > 0) {
     for (const [packageName, packageState] of stateMap) {

package/src/wirings/ai-agent/ai-agent-assistant-ui.ts

@@ -75,6 +75,7 @@ export function createAssistantUIChannel(
       return parent.state
     },
     close: () => parent.close(),
+    sendBinary: (data) => parent.sendBinary(data),
     send: (event: AIStreamEvent) => {
       switch (event.type) {
         case 'text-delta':
@@ -87,6 +88,16 @@ export function createAssistantUIChannel(
           sendChunk({ type: 'reasoning-delta', delta: event.text })
           break
 
+        case 'audio-delta':
+          ensureStarted()
+          sendChunk({ type: 'audio-delta', data: event } as any)
+          break
+
+        case 'audio-done':
+          ensureStarted()
+          sendChunk({ type: 'audio-done', data: event } as any)
+          break
+
         case 'tool-call':
           ensureStarted()
           hasToolCalls = true
@@ -190,18 +201,31 @@ export function parseAssistantUIInput(
   }
 
   let messageText: string
+  let attachments: AIAgentInput['attachments']
+
+  const extractFromParts = (parts: Array<Record<string, unknown>>) => {
+    const textParts = parts.filter((p) => p.type === 'text')
+    messageText = textParts.map((p) => String(p.text)).join(' ') || ''
+    const nonTextParts = parts.filter(
+      (p) => p.type === 'image' || p.type === 'file'
+    )
+    if (nonTextParts.length > 0) {
+      attachments = nonTextParts.map((p) => ({
+        type: p.type as 'image' | 'file',
+        data: p.data as string | undefined,
+        url: p.url as string | undefined,
+        mediaType: p.mediaType as string | undefined,
+        filename: p.filename as string | undefined,
+      }))
+    }
+  }
+
   if (typeof lastUserMessage.content === 'string') {
     messageText = lastUserMessage.content
   } else if (Array.isArray(lastUserMessage.parts)) {
-    const textPart = (
-      lastUserMessage.parts as Array<Record<string, unknown>>
-    ).find((p) => p.type === 'text')
-    messageText = textPart ? String(textPart.text) : ''
+    extractFromParts(lastUserMessage.parts as Array<Record<string, unknown>>)
   } else if (Array.isArray(lastUserMessage.content)) {
-    const textPart = (
-      lastUserMessage.content as Array<Record<string, unknown>>
-    ).find((p) => p.type === 'text')
-    messageText = textPart ? String(textPart.text) : ''
+    extractFromParts(lastUserMessage.content as Array<Record<string, unknown>>)
   } else {
     messageText = ''
   }
@@ -210,5 +234,5 @@ export function parseAssistantUIInput(
     (input.threadId as string) ?? (input.id as string) ?? randomUUID()
   const resourceId = defaults?.resourceId ?? 'default'
 
-  return { message: messageText, threadId, resourceId }
+  return { message: messageText!, threadId, resourceId, attachments }
 }

package/src/wirings/ai-agent/ai-agent-memory.ts

@@ -277,7 +277,16 @@ function sanitizeToolMessages(messages: AIMessage[]): AIMessage[] {
 
 function estimateTokens(msg: AIMessage): number {
   let chars = 0
-  if (msg.content) chars += msg.content.length
+  if (msg.content) {
+    if (typeof msg.content === 'string') {
+      chars += msg.content.length
+    } else {
+      for (const part of msg.content) {
+        if (part.type === 'text') chars += part.text.length
+        else chars += 1000
+      }
+    }
+  }
   if (msg.toolCalls) chars += JSON.stringify(msg.toolCalls).length
   if (msg.toolResults) chars += JSON.stringify(msg.toolResults).length
   return Math.ceil(chars / 4)

package/src/wirings/ai-agent/ai-agent-prepare.ts

@@ -3,6 +3,7 @@ import type {
   CoreAIAgent,
   AIAgentInput,
   AIAgentToolDef,
+  AIContentPart,
   AIMessage,
   AIStreamChannel,
   AIStreamEvent,
@@ -144,6 +145,7 @@ export function createScopedChannel(
       return capturedApproval
     },
     close: () => {},
+    sendBinary: (data) => parent.sendBinary(data),
     send: (event: AIStreamEvent) => {
       if (event.type === 'done') return
       if (event.type === 'approval-request') {
@@ -448,10 +450,26 @@ export async function prepareAgentRun(
     workingMemoryJsonSchema
   )
 
+  const userContent: AIMessage['content'] = input.attachments?.length
+    ? [
+        { type: 'text' as const, text: input.message },
+        ...input.attachments.map(
+          (a) =>
+            ({
+              type: a.type,
+              data: a.data,
+              url: a.url,
+              mediaType: a.mediaType,
+              ...(a.filename ? { filename: a.filename } : {}),
+            }) as AIContentPart
+        ),
+      ]
+    : input.message
+
   const userMessage: AIMessage = {
     id: randomUUID(),
     role: 'user',
-    content: input.message,
+    content: userContent,
     createdAt: new Date(),
   }
 

package/src/wirings/ai-agent/ai-agent-stream.ts

@@ -114,6 +114,7 @@ function createPersistingChannel(
     },
     flush: flushStep,
     close: () => parent.close(),
+    sendBinary: (data) => parent.sendBinary(data),
     send: (event: AIStreamEvent) => {
       if (storage) {
         switch (event.type) {
@@ -547,7 +548,12 @@ export async function streamAIAgent(
           allEvents,
           state,
         })
-        if (result != null) await next(result)
+        if (result == null) return
+        if (Array.isArray(result)) {
+          for (const r of result) await next(r)
+        } else {
+          await next(result)
+        }
       }
     })
 
@@ -928,7 +934,12 @@ async function continueAfterToolResult(
           allEvents,
           state,
         })
-        if (result != null) await next(result)
+        if (result == null) return
+        if (Array.isArray(result)) {
+          for (const r of result) await next(r)
+        } else {
+          await next(result)
+        }
       }
     })
 

package/src/wirings/ai-agent/ai-agent.types.ts

@@ -19,6 +19,17 @@ export interface AIThread {
   updatedAt: Date
 }
 
+export type AIContentPart =
+  | { type: 'text'; text: string }
+  | { type: 'image'; data?: string; url?: string; mediaType?: string }
+  | {
+      type: 'file'
+      data?: string
+      url?: string
+      mediaType: string
+      filename?: string
+    }
+
 export interface AIToolCall {
   id: string
   name: string
@@ -34,7 +45,7 @@ export interface AIToolResult {
 export interface AIMessage {
   id: string
   role: 'system' | 'user' | 'assistant' | 'tool'
-  content?: string
+  content?: string | AIContentPart[]
   toolCalls?: AIToolCall[]
   toolResults?: AIToolResult[]
   createdAt: Date
@@ -45,10 +56,19 @@ export interface AIAgentStep {
   toolCalls?: { name: string; args: Record<string, unknown>; result: string }[]
 }
 
+export interface AIAgentInputAttachment {
+  type: 'image' | 'file'
+  data?: string
+  url?: string
+  mediaType?: string
+  filename?: string
+}
+
 export interface AIAgentInput {
   message: string
   threadId: string
   resourceId: string
+  attachments?: AIAgentInputAttachment[]
 }
 
 export interface AIAgentOutput {
@@ -86,7 +106,11 @@ export interface PikkuAIMiddlewareHooks<
       allEvents: readonly AIStreamEvent[]
       state: State
     }
-  ) => Promise<AIStreamEvent | null> | AIStreamEvent | null
+  ) =>
+    | Promise<AIStreamEvent | AIStreamEvent[] | null>
+    | AIStreamEvent
+    | AIStreamEvent[]
+    | null
 
   modifyOutput?: (
     services: Services,
@@ -235,6 +259,14 @@ export type AIStreamEvent =
       session?: string
     }
   | { type: 'error'; message: string; agent?: string; session?: string }
+  | {
+      type: 'audio-delta'
+      data: string
+      format: string
+      agent?: string
+      session?: string
+    }
+  | { type: 'audio-done'; agent?: string; session?: string }
   | {
       type: 'suspended'
       reason: 'rpc-missing'

package/src/wirings/ai-agent/index.ts

@@ -23,9 +23,11 @@ export {
 } from './ai-agent-registry.js'
 export type {
   AIAgentInput,
+  AIAgentInputAttachment,
   AIAgentMeta,
   AIAgentMemoryConfig,
   AIAgentStep,
+  AIContentPart,
   AgentRunRow,
   AgentRunService,
   AgentRunState,

package/src/wirings/channel/channel-handler.ts

@@ -4,6 +4,7 @@ import type {
   CoreUserSession,
 } from '../../types/core.types.js'
 import type {
+  BinaryData,
   ChannelMessageMeta,
   CoreChannel,
   PikkuChannelHandler,
@@ -154,7 +155,7 @@ export const processMessageHandlers = (
     })
   }
 
-  return async (rawData): Promise<unknown> => {
+  const onMessage = async (rawData: unknown): Promise<unknown> => {
     let result: unknown
     let processed = false
 
@@ -198,7 +199,10 @@ export const processMessageHandlers = (
     // Default handler if no routes matched and json data wasn't parsed
     if (!processed && channelConfig.onMessage) {
       processed = true
-      result = await processMessage(rawData, channelConfig.onMessage)
+      result = await processMessage(
+        rawData as JSONValue,
+        channelConfig.onMessage
+      )
     }
 
     if (!processed) {
@@ -210,4 +214,28 @@ export const processMessageHandlers = (
 
     return result
   }
+
+  const onBinaryMessage = channelConfig.onBinaryMessage
+    ? async (data: BinaryData): Promise<BinaryData | void> => {
+        if (
+          !validateAuth(
+            requiresSession,
+            userSession?.get() as CoreUserSession | undefined,
+            channelConfig
+          )
+        ) {
+          logger.error(
+            `Channel ${channelConfig.name} with id ${channelHandler.getChannel().channelId} requires a session for binary message`
+          )
+          return
+        }
+        return channelConfig.onBinaryMessage!(
+          services,
+          data,
+          channelHandler.getChannel()
+        )
+      }
+    : undefined
+
+  return { onMessage, onBinaryMessage }
 }

package/src/wirings/channel/channel-middleware-runner.ts

@@ -114,6 +114,7 @@ export function wrapChannelWithMiddleware<Out>(
     channel: {
       ...channel,
       send: ((data: Out) => dispatch(0, data)) as typeof channel.send,
+      sendBinary: channel.sendBinary,
     },
   }
 }

package/src/wirings/channel/channel.types.ts

@@ -10,6 +10,8 @@ import type {
   PermissionMetadata,
 } from '../../types/core.types.js'
 
+export type BinaryData = ArrayBuffer | Uint8Array
+
 export type CorePikkuChannelMiddleware<Services = any, Event = unknown> = (
   services: Services,
   event: Event,
@@ -64,6 +66,7 @@ export interface ChannelMeta {
   disconnect: ChannelMessageMeta | null
   message: ChannelMessageMeta | null
   messageWirings: Record<string, Record<string, ChannelMessageMeta>>
+  binary?: boolean | null
   summary?: string
   description?: string
   errors?: string[]
@@ -132,6 +135,12 @@ export type CoreChannel<
   >
   permissions?: CorePermissionGroup<PikkuPermission>
   auth?: boolean
+  binary?: boolean | null
+  onBinaryMessage?: (
+    services: any,
+    data: BinaryData,
+    channel: PikkuChannel<ChannelData, any>
+  ) => Promise<BinaryData | void> | BinaryData | void
   docs?: Partial<{
     description: string
     response: string
@@ -149,6 +158,7 @@ export interface PikkuChannel<OpeningData, out Out> {
   openingData: OpeningData
   // The data to send. This will fail is the stream has been closed.
   send(data: Out, isBinary?: boolean): Promise<void> | void
+  sendBinary(data: BinaryData): Promise<void> | void
   // This will close the channel.
   close(): Promise<void> | void
   // The current state of the channel
@@ -157,6 +167,7 @@ export interface PikkuChannel<OpeningData, out Out> {
 
 export interface PikkuChannelHandler<OpeningData = unknown, Out = unknown> {
   send(message: Out, isBinary?: boolean): Promise<void> | void
+  sendBinary(data: BinaryData): Promise<void> | void
   getChannel(): PikkuChannel<OpeningData, Out>
 }
 

package/src/wirings/channel/index.ts

@@ -11,6 +11,7 @@ export { ChannelStore } from './channel-store.js'
 export type { Channel } from './channel-store.js'
 export { EventHubStore } from './eventhub-store.js'
 export type {
+  BinaryData,
   ChannelsMeta,
   CoreChannel,
   CorePikkuChannelMiddleware,

package/src/wirings/channel/local/local-channel-handler.ts

@@ -1,3 +1,4 @@
+import type { BinaryData } from '../channel.types.js'
 import { PikkuAbstractChannelHandler } from '../pikku-abstract-channel-handler.js'
 
 export class PikkuLocalChannelHandler<
@@ -5,9 +6,11 @@ export class PikkuLocalChannelHandler<
   Out = unknown,
 > extends PikkuAbstractChannelHandler<OpeningData, Out> {
   private onMessageCallback?: (message: unknown) => void
+  private onBinaryMessageCallback?: (data: BinaryData) => void
   private openCallBack?: () => void
   private closeCallback?: () => void
   private sendCallback?: (message: Out, isBinary?: boolean) => void
+  private sendBinaryCallback?: (data: BinaryData) => void
 
   public registerOnOpen(callback: () => void): void {
     this.openCallBack = callback
@@ -28,6 +31,18 @@ export class PikkuLocalChannelHandler<
     return this.onMessageCallback?.(data)
   }
 
+  public registerOnBinaryMessage(
+    callback: (
+      data: BinaryData
+    ) => Promise<BinaryData | void> | BinaryData | void
+  ): void {
+    this.onBinaryMessageCallback = callback
+  }
+
+  public async binaryMessage(data: BinaryData): Promise<BinaryData | void> {
+    return this.onBinaryMessageCallback?.(data)
+  }
+
   public registerOnClose(callback: () => void): void {
     this.closeCallback = callback
   }
@@ -50,4 +65,15 @@ export class PikkuLocalChannelHandler<
     }
     return this.sendCallback?.(message, isBinary)
   }
+
+  public registerOnSendBinary(send: (data: BinaryData) => void) {
+    this.sendBinaryCallback = send
+  }
+
+  public sendBinary(data: BinaryData): void {
+    if (!this.sendBinaryCallback) {
+      throw new Error('No sendBinary callback registered')
+    }
+    this.sendBinaryCallback(data)
+  }
 }

package/src/wirings/channel/local/local-channel-runner.ts

@@ -147,7 +147,7 @@ export const runLocalChannel = async ({
         await closeServices()
       })
 
-      const onMessage = processMessageHandlers(
+      const { onMessage, onBinaryMessage } = processMessageHandlers(
         services,
         channelConfig as any,
         channelHandler,
@@ -162,6 +162,20 @@ export const runLocalChannel = async ({
           channel.send({ error: e.message || 'Unknown error' })
         }
       })
+      if (onBinaryMessage) {
+        channelHandler.registerOnBinaryMessage(async (data) => {
+          try {
+            const result = await onBinaryMessage(data)
+            if (result) {
+              await channel.sendBinary(result)
+            }
+          } catch (e) {
+            singletonServices.logger.error(
+              `Error handling binary message: ${e.message}`
+            )
+          }
+        })
+      }
     } catch (e: any) {
       handleHTTPError(
         e,

package/src/wirings/channel/pikku-abstract-channel-handler.test.ts

@@ -10,6 +10,8 @@ class TestChannelHandler extends PikkuAbstractChannelHandler<
   public async send(message: { msg: string }, isBinary = false): Promise<void> {
     // Mock send implementation
   }
+
+  public sendBinary(_data: ArrayBuffer | Uint8Array): void {}
 }
 
 let handler: TestChannelHandler

package/src/wirings/channel/pikku-abstract-channel-handler.ts

@@ -1,4 +1,8 @@
-import type { PikkuChannel, PikkuChannelHandler } from './channel.types.js'
+import type {
+  BinaryData,
+  PikkuChannel,
+  PikkuChannelHandler,
+} from './channel.types.js'
 
 export abstract class PikkuAbstractChannelHandler<
   OpeningData = unknown,
@@ -15,12 +19,15 @@ export abstract class PikkuAbstractChannelHandler<
 
   public abstract send(message: Out, isBinary?: boolean): Promise<void> | void
 
+  public abstract sendBinary(data: BinaryData): Promise<void> | void
+
   public getChannel(): PikkuChannel<OpeningData, Out> {
     if (!this.channel) {
       this.channel = {
         channelId: this.channelId,
         openingData: this.openingData,
         send: this.send.bind(this),
+        sendBinary: this.sendBinary.bind(this),
         close: this.close.bind(this),
         state: 'initial',
       }

package/src/wirings/channel/serverless/serverless-channel-runner.ts

@@ -265,7 +265,7 @@ export const runChannelMessage = async (
 
   let response: unknown
   try {
-    const onMessage = processMessageHandlers(
+    const { onMessage } = processMessageHandlers(
       services,
       channelConfig,
       channelHandler,

package/src/wirings/cli/cli-runner.ts

@@ -321,6 +321,9 @@ export async function runCLICommand({
         await Promise.resolve(renderer(singletonServices, data, undefined))
       }
     },
+    sendBinary: () => {
+      throw new Error('Binary data is not supported on CLI channels')
+    },
     close: () => {
       if (channel) {
         channel.state = 'closed'