kernl 0.8.3 → 0.9.0

package/src/realtime/agent.ts

@@ -0,0 +1,24 @@
+import type { RealtimeModel } from "@kernl-sdk/protocol";
+
+import type { UnknownContext } from "@/context";
+import { BaseAgent } from "@/agent/base";
+
+import type { RealtimeAgentConfig, RealtimeAgentVoiceConfig } from "./types";
+
+/**
+ * A realtime agent definition.
+ *
+ * Stateless configuration that describes what a realtime voice agent does.
+ * Create sessions with `new RealtimeSession(agent, options)`.
+ */
+export class RealtimeAgent<TContext = UnknownContext> extends BaseAgent<TContext> {
+  readonly kind = "realtime";
+  readonly model: RealtimeModel;
+  readonly voice?: RealtimeAgentVoiceConfig;
+
+  constructor(config: RealtimeAgentConfig<TContext>) {
+    super(config);
+    this.model = config.model;
+    this.voice = config.voice;
+  }
+}

package/src/realtime/channel.ts

@@ -0,0 +1,32 @@
+import { EventEmitter } from "node:events";
+
+/**
+ * Base interface for audio I/O channels.
+ *
+ * Channels bridge between audio sources (browser mic, Twilio, Discord)
+ * and the realtime session. They handle audio capture/playback and emit
+ * events that the session listens to.
+ *
+ * Events emitted:
+ * - 'audio' (audio: string) - Raw audio chunk (base64)
+ * - 'commit' () - User finished speaking (VAD or manual)
+ * - 'interrupt' () - User started speaking mid-response
+ */
+export interface RealtimeChannel extends EventEmitter {
+  /**
+   * Send audio to be played/transmitted by the channel.
+   * Called by session when audio is received from the model.
+   */
+  sendAudio(audio: string): void;
+
+  /**
+   * Interrupt current audio playback.
+   * Called by session when response is cancelled.
+   */
+  interrupt(): void;
+
+  /**
+   * Clean up resources and close the channel.
+   */
+  close(): void;
+}

package/src/realtime/index.ts

@@ -0,0 +1,4 @@
+export * from "./types";
+export * from "./channel";
+export * from "./agent";
+export * from "./session";

package/src/realtime/session.ts

@@ -0,0 +1,259 @@
+import { EventEmitter } from "node:events";
+
+import {
+  RealtimeModel,
+  RealtimeConnection,
+  RealtimeServerEvent,
+  RealtimeSessionConfig,
+  ToolCallEvent,
+  message,
+} from "@kernl-sdk/protocol";
+
+import { Context, UnknownContext } from "@/context";
+import { MisconfiguredError } from "@/lib/error";
+
+import { RealtimeAgent } from "./agent";
+import type { RealtimeChannel } from "./channel";
+import type { RealtimeSessionOptions } from "./types";
+
+/**
+ * A realtime session manages the connection to a realtime model.
+ *
+ * Handles the bidirectional communication between an agent and a model,
+ * including audio I/O (via channels), tool execution, and event routing.
+ */
+export class RealtimeSession<TContext = UnknownContext> extends EventEmitter {
+  /**
+   * Session ID. Null until connected.
+   */
+  id: string | null = null;
+
+  /**
+   * The agent definition.
+   */
+  readonly agent: RealtimeAgent<TContext>;
+
+  /**
+   * The realtime model.
+   */
+  readonly model: RealtimeModel;
+
+  /**
+   * The audio I/O channel (if any).
+   */
+  readonly channel: RealtimeChannel | null;
+
+  /**
+   * The session context.
+   */
+  readonly context: Context<TContext>;
+
+  /**
+   * The active connection. Null until connected.
+   */
+  private connection: RealtimeConnection | null = null;
+
+  /**
+   * Session options.
+   */
+  private options: RealtimeSessionOptions<TContext>;
+
+  constructor(
+    agent: RealtimeAgent<TContext>,
+    options: RealtimeSessionOptions<TContext> = {},
+  ) {
+    super();
+
+    if (options.transport?.handlesAudio && options.channel) {
+      throw new MisconfiguredError(
+        "Cannot use channel with WebRTC transport - audio is handled by transport",
+      );
+    }
+
+    this.agent = agent;
+    this.model = options.model ?? agent.model;
+    this.channel = options.channel ?? null;
+    this.context = options.context ?? new Context("kernl", {} as TContext);
+    this.options = options;
+  }
+
+  /**
+   * Connect to the realtime model.
+   */
+  async connect(): Promise<void> {
+    const sessionConfig = await this.buildSessionConfig();
+    const options = {
+      ...this.options.connectOptions,
+      sessionConfig,
+    };
+
+    this.connection = this.options.transport
+      ? await this.options.transport.connect(this.model, options)
+      : await this.model.connect(options);
+
+    this.connection.on("event", this.onEvent.bind(this));
+    this.connection.on("error", (e) => this.emit("error", e));
+    this.connection.on("status", (s) => this.emit("status", s));
+
+    this.init();
+  }
+
+  /**
+   * Initialize event listeners and send session configuration.
+   */
+  private async init(): Promise<void> {
+    if (this.channel) {
+      this.channel.on("audio", (audio: string) => this.sendAudio(audio));
+      this.channel.on("commit", () => this.commit());
+      this.channel.on("interrupt", () => this.interrupt());
+    }
+
+    this.connection?.send({
+      kind: "session.update",
+      config: await this.buildSessionConfig(),
+    });
+  }
+
+  /**
+   * Build session configuration from agent.
+   */
+  private async buildSessionConfig(): Promise<RealtimeSessionConfig> {
+    const tools = await this.agent.tools(this.context);
+
+    return {
+      instructions: await this.agent.instructions(this.context),
+      tools: tools.map((t) => t.serialize()),
+      voice: this.agent.voice,
+    };
+  }
+
+  /**
+   * Handle incoming events from the connection.
+   *
+   * Maps protocol events to simplified user-facing events:
+   * - 'audio' - audio output from assistant
+   * - 'transcript' - speech transcriptions (user or assistant)
+   * - 'text' - text output from assistant
+   * - 'error' - errors
+   */
+  private onEvent(event: RealtimeServerEvent): void {
+    switch (event.kind) {
+      // Audio output → 'audio'
+      case "audio.output.delta":
+        this.channel?.sendAudio(event.audio);
+        this.emit("audio", event);
+        break;
+      case "audio.output.done":
+        this.emit("audio", event);
+        break;
+
+      // Speech transcriptions → 'transcript'
+      case "transcript.input":
+      case "transcript.output":
+        this.emit("transcript", event);
+        break;
+
+      // Text output → 'text'
+      case "text.output":
+        this.emit("text", event);
+        break;
+
+      // Errors → 'error'
+      case "session.error":
+        this.emit("error", event.error);
+        break;
+
+      // Tool calls - handled internally
+      case "tool.call":
+        this.performActions(event);
+        break;
+
+      // Session lifecycle - internal state
+      case "session.created":
+        this.id = event.session.id;
+        break;
+    }
+  }
+
+  /**
+   * Execute tool calls from the model.
+   */
+  private async performActions(event: ToolCallEvent): Promise<void> {
+    const tool = this.agent.tool(event.toolId);
+    if (!tool || tool.type !== "function") {
+      this.connection?.send({
+        kind: "tool.result",
+        callId: event.callId,
+        error: `Unknown tool: ${event.toolId}`,
+      });
+      return;
+    }
+
+    const result = await tool.invoke(
+      this.context,
+      event.arguments,
+      event.callId,
+    );
+
+    this.connection?.send({
+      kind: "tool.result",
+      callId: event.callId,
+      result: result.state === "completed" ? String(result.result) : undefined,
+      error: result.error ?? undefined,
+    });
+  }
+
+  /**
+   * Send audio to the model.
+   */
+  sendAudio(audio: string): void {
+    this.connection?.send({ kind: "audio.input.append", audio });
+  }
+
+  /**
+   * Commit the audio buffer (signal end of speech).
+   */
+  commit(): void {
+    this.connection?.send({ kind: "audio.input.commit" });
+  }
+
+  /**
+   * Send a text message to the model.
+   */
+  sendMessage(text: string): void {
+    this.connection?.send({
+      kind: "item.create",
+      item: message({ role: "user", text }),
+    });
+  }
+
+  /**
+   * Interrupt the current response.
+   */
+  interrupt(): void {
+    this.connection?.send({ kind: "response.cancel" });
+    this.channel?.interrupt();
+  }
+
+  /**
+   * Mute audio input.
+   */
+  mute(): void {
+    this.connection?.mute();
+  }
+
+  /**
+   * Unmute audio input.
+   */
+  unmute(): void {
+    this.connection?.unmute();
+  }
+
+  /**
+   * Close the session and release resources.
+   */
+  close(): void {
+    this.channel?.close();
+    this.connection?.close();
+  }
+}

package/src/realtime/types.ts

@@ -0,0 +1,73 @@
+import {
+  RealtimeModel,
+  RealtimeTransport,
+  RealtimeConnectOptions,
+} from "@kernl-sdk/protocol";
+
+import { Context, UnknownContext } from "@/context";
+import type { BaseAgentConfig } from "@/agent/base";
+
+import type { RealtimeChannel } from "./channel";
+
+/**
+ * Configuration for a realtime agent.
+ */
+export interface RealtimeAgentConfig<TContext = UnknownContext>
+  extends BaseAgentConfig<TContext> {
+  /**
+   * The realtime model to use for this agent.
+   */
+  model: RealtimeModel;
+
+  /**
+   * Voice configuration for the agent.
+   */
+  voice?: RealtimeAgentVoiceConfig;
+}
+
+/**
+ * Voice configuration for a realtime agent.
+ */
+export interface RealtimeAgentVoiceConfig {
+  /**
+   * Voice ID to use for audio output.
+   */
+  voiceId: string;
+
+  /**
+   * Playback speed multiplier.
+   */
+  speed?: number;
+}
+
+/**
+ * Options for creating a realtime session.
+ */
+export interface RealtimeSessionOptions<TContext = UnknownContext> {
+  /**
+   * Override the agent's default model for this session.
+   */
+  model?: RealtimeModel;
+
+  /**
+   * Audio I/O channel (e.g., BrowserChannel, TwilioChannel).
+   * Not used with WebRTC transport.
+   */
+  channel?: RealtimeChannel;
+
+  /**
+   * Custom transport (e.g., WebRTCTransport).
+   * If not provided, model.connect() creates the default transport.
+   */
+  transport?: RealtimeTransport;
+
+  /**
+   * Context for this session.
+   */
+  context?: Context<TContext>;
+
+  /**
+   * Options passed to model.connect() or transport.connect().
+   */
+  connectOptions?: RealtimeConnectOptions;
+}

package/src/storage/in-memory.ts

@@ -6,6 +6,7 @@
  * Defined here so that it can be used as default and for testing.
  */
 
+import { Agent } from "@/agent";
 import { Thread } from "@/thread";
 import { Context } from "@/context";
 import { STOPPED } from "@kernl-sdk/protocol";
@@ -241,8 +242,15 @@ export class InMemoryThreadStore implements ThreadStore {
       );
     }
 
+    // safety: threads only exist for llm agents
+    if (agent.kind !== "llm") {
+      throw new Error(
+        `Thread ${data.tid} references non-llm agent ${data.agentId} (kind: ${agent.kind})`,
+      );
+    }
+
     return new Thread({
-      agent,
+      agent: agent as Agent,
       tid: data.tid,
       context: new Context(data.namespace, data.context),
       model,

package/src/tool/__tests__/toolkit.test.ts

@@ -167,10 +167,10 @@ describe("FunctionToolkit", () => {
       const serialized = (await toolkit.list()).map((tool: any) => tool.serialize());
       expect(serialized).toHaveLength(1);
       expect(serialized[0]).toEqual({
-        type: "hosted-tool",
+        kind: "provider-defined",
         id: anotherHostedTool.id,
         name: anotherHostedTool.name,
-        providerData: undefined,
+        args: {},
       });
     });
   });

package/src/tool/tool.ts

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import { Context, UnknownContext } from "@/context";
+import type { BaseAgent } from "@/agent/base";
 
 import { ModelBehaviorError } from "@/lib/error";
 import { logger } from "@/lib/logger";
@@ -61,7 +62,7 @@ export abstract class BaseTool<TContext = UnknownContext> {
   /**
    * Determines whether the tool should be exposed to the model for the current run.
    */
-  abstract isEnabled(context: Context<TContext>, agent: any): Promise<boolean>;
+  abstract isEnabled(context: Context<TContext>, agent: BaseAgent<TContext>): Promise<boolean>;
 
   /**
    * Serialize this tool for sending to the model

package/src/tool/toolkit.ts

@@ -1,4 +1,4 @@
-import type { Agent } from "@/agent";
+import type { BaseAgent } from "@/agent/base";
 import type { Context, UnknownContext } from "@/context";
 
 import { MCPServer } from "@/mcp/base";
@@ -32,13 +32,13 @@ export abstract class BaseToolkit<TContext = UnknownContext> {
   /**
    * The agent this toolkit is bound to (if any)
    */
-  protected agent?: Agent<TContext, any>;
+  protected agent?: BaseAgent<TContext>;
 
   /**
    * Bind this toolkit to an agent.
-   * Called by Agent constructor.
+   * Called by agent constructor.
    */
-  bind(agent: Agent<TContext, any>): void {
+  bind(agent: BaseAgent<TContext>): void {
     this.agent = agent;
   }
 
@@ -216,7 +216,8 @@ export class MCPToolkit<
       const mcpTools = await this.server.listTools();
 
       for (const mcpTool of mcpTools) {
-        const tool = mcpToFunctionTool(this.server, mcpTool);
+        // safety: MCP tools are context-agnostic (external servers)
+        const tool = mcpToFunctionTool(this.server, mcpTool) as Tool<TContext>;
         this.cache.set(tool.id, tool);
       }
 

package/src/tool/types.ts

@@ -1,6 +1,6 @@
 import { z, type ZodType } from "zod";
 
-import { Agent } from "@/agent";
+import type { BaseAgent } from "@/agent/base";
 import { Context, UnknownContext } from "@/context";
 import { MCPServer } from "@/mcp/base";
 import type { ToolCallState } from "@kernl-sdk/protocol";
@@ -79,7 +79,7 @@ export type ToolConfig<
  */
 export interface ToolkitFilterContext<TContext = UnknownContext> {
   context: Context<TContext>;
-  agent: Agent<TContext, any>;
+  agent: BaseAgent<TContext>;
   toolkitId: string;
 }
 
@@ -227,12 +227,12 @@ export type ToolApprovalFunction<TParameters extends ToolInputParameters> = (
 
 export type ToolEnabledFunction<TContext = UnknownContext> = (
   context: Context<TContext>,
-  agent: Agent<any, any>, // (TODO): why would we need to take an agent here?
+  agent: BaseAgent<TContext>,
 ) => Promise<boolean>;
 
 export type ToolEnabledPredicate<TContext = UnknownContext> = (args: {
   context: Context<TContext>;
-  agent: Agent<any, any>; // (TODO): why take an agent here? other options?
+  agent: BaseAgent<TContext>;
 }) => boolean | Promise<boolean>;
 
 type ToolEnabledOption<Context = UnknownContext> =