@alexkroman1/aai 1.4.5 → 1.5.1

package/host/runtime.ts

@@ -15,6 +15,8 @@ import { DEFAULT_SHUTDOWN_TIMEOUT_MS } from "../sdk/constants.ts";
 import type { Kv } from "../sdk/kv.ts";
 import type { ClientSink } from "../sdk/protocol.ts";
 import { buildReadyConfig, type ReadyConfig } from "../sdk/protocol.ts";
+import { DEEPGRAM_KIND } from "../sdk/providers/stt/deepgram.ts";
+import { RIME_KIND } from "../sdk/providers/tts/rime.ts";
 import {
   assertProviderTriple,
   type LlmProvider,
@@ -23,19 +25,68 @@ import {
   type TtsOpener,
   type TtsProvider,
 } from "../sdk/providers.ts";
+import { buildSystemPrompt } from "../sdk/system-prompt.ts";
 import type { AgentDef } from "../sdk/types.ts";
 import { toolError } from "../sdk/utils.ts";
 import { resolveAllBuiltins } from "./builtin-tools.ts";
-import { createPipelineSession } from "./pipeline-session.ts";
 import { resolveApiKey, resolveLlm, resolveStt, resolveTts } from "./providers/resolve.ts";
 import type { Logger, S2SConfig } from "./runtime-config.ts";
 import { consoleLogger, DEFAULT_S2S_CONFIG } from "./runtime-config.ts";
 import type { CreateS2sWebSocket } from "./s2s.ts";
-import { createS2sSession, type Session } from "./session.ts";
+import { createSessionCore, type SessionCore } from "./session-core.ts";
 import { type ExecuteTool, executeToolCall } from "./tool-executor.ts";
+import { createPipelineTransport } from "./transports/pipeline-transport.ts";
+import { createS2sTransport } from "./transports/s2s-transport.ts";
+import type { Transport, TransportCallbacks } from "./transports/types.ts";
 import { createUnstorageKv } from "./unstorage-kv.ts";
 import { type SessionWebSocket, wireSessionSocket } from "./ws-handler.ts";
 
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+
+/**
+ * Resolve the API key env-var for the configured STT provider.
+ *
+ * Each STT provider uses its own env var (e.g. `ASSEMBLYAI_API_KEY`,
+ * `DEEPGRAM_API_KEY`). We read the kind from the descriptor if it is one;
+ * pre-resolved openers have no kind field so we fall back to AssemblyAI for
+ * backward compatibility (openers supply their own key at open-time anyway).
+ */
+function resolveSttApiKey(
+  stt: SttProvider | SttOpener | undefined,
+  env: Record<string, string>,
+): string {
+  // SttProvider descriptors carry a `kind` field; SttOpener does not.
+  const kind =
+    stt != null && "kind" in stt && typeof (stt as SttProvider).kind === "string"
+      ? (stt as SttProvider).kind
+      : undefined;
+  if (kind === DEEPGRAM_KIND) return resolveApiKey("DEEPGRAM_API_KEY", env);
+  // Default: ASSEMBLYAI_KIND or pre-resolved opener (backward compat).
+  return resolveApiKey("ASSEMBLYAI_API_KEY", env);
+}
+
+/**
+ * Resolve the API key env-var for the configured TTS provider.
+ *
+ * Each TTS provider uses its own env var (e.g. `CARTESIA_API_KEY`,
+ * `RIME_API_KEY`). We read the kind from the descriptor if it is one;
+ * pre-resolved openers have no kind field so we fall back to Cartesia for
+ * backward compatibility (openers supply their own key at open-time anyway).
+ */
+function resolveTtsApiKey(
+  tts: TtsProvider | TtsOpener | undefined,
+  env: Record<string, string>,
+): string {
+  // TtsProvider descriptors carry a `kind` field; TtsOpener does not.
+  const kind =
+    tts != null && "kind" in tts && typeof (tts as TtsProvider).kind === "string"
+      ? (tts as TtsProvider).kind
+      : undefined;
+  if (kind === RIME_KIND) return resolveApiKey("RIME_API_KEY", env);
+  // Default: CARTESIA_KIND or pre-resolved opener (backward compat).
+  return resolveApiKey("CARTESIA_API_KEY", env);
+}
+
 // ─── Runtime adapter (formerly adapter.ts) ──────────────────────────────────
 
 /** Per-session options passed to {@link AgentRuntime.startSession}. */
@@ -189,7 +240,7 @@ export type Runtime = AgentRuntime & {
     client: ClientSink;
     skipGreeting?: boolean;
     resumeFrom?: string;
-  }): Session;
+  }): SessionCore;
 };
 
 /**
@@ -217,7 +268,7 @@ export function createRuntime(opts: RuntimeOptions): Runtime {
   } = opts;
   const mode = assertProviderTriple(opts.stt, opts.llm, opts.tts);
   const agentConfig = toAgentConfig(agent);
-  const sessions = new Map<string, Session>();
+  const sessions = new Map<string, SessionCore>();
   const sinkMap = new Map<string, ClientSink>();
   const readyConfig: ReadyConfig = buildReadyConfig(s2sConfig);
 
@@ -310,44 +361,101 @@ export function createRuntime(opts: RuntimeOptions): Runtime {
     client: ClientSink;
     skipGreeting?: boolean;
     resumeFrom?: string;
-  }): Session {
+  }): SessionCore {
     sinkMap.set(sessionOpts.id, sessionOpts.client);
+
+    const isPipeline = Boolean(pipelineProviders);
+    const hasTools = toolSchemas.length > 0 || (agentConfig.builtinTools?.length ?? 0) > 0;
+    const systemPrompt = buildSystemPrompt(agentConfig, {
+      hasTools,
+      voice: true,
+      toolGuidance,
+    });
+
+    // Late-bound reference: callbacks are constructed before SessionCore exists,
+    // so we capture a reference and fill it in below.
+    let core: SessionCore | null = null;
+    function bindCore(): SessionCore {
+      if (!core) throw new Error("SessionCore not yet created");
+      return core;
+    }
+
+    const callbacks: TransportCallbacks = {
+      onReplyStarted: (replyId) => bindCore().onReplyStarted(replyId),
+      onReplyDone: () => bindCore().onReplyDone(),
+      onCancelled: () => bindCore().onCancelled(),
+      onAudioChunk: (bytes) => bindCore().onAudioChunk(bytes),
+      onAudioDone: () => bindCore().onAudioDone(),
+      onUserTranscript: (text) => bindCore().onUserTranscript(text),
+      onAgentTranscript: (text, interrupted) => bindCore().onAgentTranscript(text, interrupted),
+      // Pipeline: tools execute inside streamText; forward the call to the
+      // client sink for UI observability only. Going through SessionCore.onToolCall
+      // would re-execute the tool and leave pendingTools non-empty, hanging the turn.
+      onToolCall: isPipeline
+        ? (id, name, args) =>
+            sessionOpts.client.event({ type: "tool_call", toolCallId: id, toolName: name, args })
+        : (id, name, args) => bindCore().onToolCall(id, name, args),
+      onError: (code, message) => bindCore().onError(code, message),
+      onSpeechStarted: () => bindCore().onSpeechStarted(),
+      onSpeechStopped: () => bindCore().onSpeechStopped(),
+    };
+
+    let transport: Transport;
     if (pipelineProviders) {
-      return createPipelineSession({
-        id: sessionOpts.id,
+      transport = createPipelineTransport({
+        sid: sessionOpts.id,
         agent: sessionOpts.agent,
-        client: sessionOpts.client,
-        agentConfig,
-        toolSchemas,
-        toolGuidance,
-        executeTool,
         stt: pipelineProviders.stt,
         llm: pipelineProviders.llm,
         tts: pipelineProviders.tts,
-        sttApiKey: resolveApiKey("ASSEMBLYAI_API_KEY", env),
-        ttsApiKey: resolveApiKey("CARTESIA_API_KEY", env),
+        callbacks,
+        sessionConfig: {
+          systemPrompt,
+          greeting: agentConfig.greeting,
+          tools: toolSchemas,
+        },
+        toolSchemas,
+        executeTool,
+        providerKeys: {
+          stt: resolveSttApiKey(opts.stt, env),
+          tts: resolveTtsApiKey(opts.tts, env),
+        },
         sttSampleRate: s2sConfig.inputSampleRate,
         ttsSampleRate: s2sConfig.outputSampleRate,
+        maxSteps: agentConfig.maxSteps,
+        toolChoice: agentConfig.toolChoice,
         skipGreeting: sessionOpts.skipGreeting ?? false,
         logger,
       });
+    } else {
+      transport = createS2sTransport({
+        apiKey: env.ASSEMBLYAI_API_KEY ?? "",
+        s2sConfig,
+        sessionConfig: {
+          systemPrompt,
+          tools: toolSchemas as import("./s2s.ts").S2sToolSchema[],
+          ...(agentConfig.greeting !== undefined ? { greeting: agentConfig.greeting } : {}),
+        },
+        toolSchemas: toolSchemas as import("./s2s.ts").S2sToolSchema[],
+        callbacks,
+        sid: sessionOpts.id,
+        agent: sessionOpts.agent,
+        ...(createWebSocket ? { createWebSocket } : {}),
+        logger,
+      });
     }
-    const apiKey = env.ASSEMBLYAI_API_KEY ?? "";
-    return createS2sSession({
+
+    core = createSessionCore({
       id: sessionOpts.id,
       agent: sessionOpts.agent,
       client: sessionOpts.client,
       agentConfig,
-      toolSchemas,
-      toolGuidance,
-      apiKey,
-      s2sConfig,
       executeTool,
-      ...(createWebSocket ? { createWebSocket } : {}),
-      skipGreeting: sessionOpts.skipGreeting ?? false,
+      transport,
       logger,
-      ...(sessionOpts.resumeFrom ? { resumeFrom: sessionOpts.resumeFrom } : {}),
     });
+
+    return core;
   }
 
   // ── AgentRuntime methods ──────────────────────────────────────────────