@effect-uai/core 0.14.0 → 0.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effect-uai/core",
-  "version": "0.14.0",
+  "version": "0.16.0",
   "description": "Low-level primitives (loop, conversation, items, tools, streaming codecs) for building AI agents with Effect.",
   "keywords": [
     "agents",
@@ -90,6 +90,14 @@
       "types": "./dist/domain/Music.d.mts",
       "import": "./dist/domain/Music.mjs"
     },
+    "./Realtime": {
+      "types": "./dist/domain/Realtime.d.mts",
+      "import": "./dist/domain/Realtime.mjs"
+    },
+    "./RealtimeSession": {
+      "types": "./dist/realtime/RealtimeSession.d.mts",
+      "import": "./dist/realtime/RealtimeSession.mjs"
+    },
     "./Transcript": {
       "types": "./dist/domain/Transcript.d.mts",
       "import": "./dist/domain/Transcript.mjs"
@@ -114,6 +122,18 @@
       "types": "./dist/language-model/LanguageModel.d.mts",
       "import": "./dist/language-model/LanguageModel.mjs"
     },
+    "./Messenger": {
+      "types": "./dist/messenger/Messenger.d.mts",
+      "import": "./dist/messenger/Messenger.mjs"
+    },
+    "./MessengerAdapter": {
+      "types": "./dist/messenger/MessengerAdapter.d.mts",
+      "import": "./dist/messenger/MessengerAdapter.mjs"
+    },
+    "./MessengerError": {
+      "types": "./dist/messenger/MessengerError.d.mts",
+      "import": "./dist/messenger/MessengerError.mjs"
+    },
     "./MusicGenerator": {
       "types": "./dist/music-generator/MusicGenerator.d.mts",
       "import": "./dist/music-generator/MusicGenerator.mjs"
@@ -166,6 +186,14 @@
       "types": "./dist/web-search/WebSearchTool.d.mts",
       "import": "./dist/web-search/WebSearchTool.mjs"
     },
+    "./WebSocketSession": {
+      "types": "./dist/streaming/WebSocketSession.d.mts",
+      "import": "./dist/streaming/WebSocketSession.mjs"
+    },
+    "./Settle": {
+      "types": "./dist/streaming/Settle.d.mts",
+      "import": "./dist/streaming/Settle.mjs"
+    },
     "./Loop": {
       "types": "./dist/loop/Loop.d.mts",
       "import": "./dist/loop/Loop.mjs"
@@ -238,6 +266,10 @@
       "types": "./dist/testing/MockProvider.d.mts",
       "import": "./dist/testing/MockProvider.mjs"
     },
+    "./testing/MockMessenger": {
+      "types": "./dist/testing/MockMessenger.d.mts",
+      "import": "./dist/testing/MockMessenger.mjs"
+    },
     "./testing/MockMusicGenerator": {
       "types": "./dist/testing/MockMusicGenerator.d.mts",
       "import": "./dist/testing/MockMusicGenerator.mjs"
@@ -250,6 +282,14 @@
       "types": "./dist/testing/MockSandbox.d.mts",
       "import": "./dist/testing/MockSandbox.mjs"
     },
+    "./testing/FakeWebSocket": {
+      "types": "./dist/testing/FakeWebSocket.d.mts",
+      "import": "./dist/testing/FakeWebSocket.mjs"
+    },
+    "./testing/MockRealtimeSession": {
+      "types": "./dist/testing/MockRealtimeSession.d.mts",
+      "import": "./dist/testing/MockRealtimeSession.mjs"
+    },
     "./testing/MockTranscriber": {
       "types": "./dist/testing/MockTranscriber.d.mts",
       "import": "./dist/testing/MockTranscriber.mjs"

package/src/domain/AiError.ts

@@ -100,6 +100,15 @@ export class Unsupported extends Data.TaggedError("Unsupported")<{
   reason?: string
 }> {}
 
+/**
+ * A long-lived session hit the provider's lifetime cap. Distinct from
+ * `Unavailable`: reconnecting works, retrying the same session does not.
+ */
+export class SessionExpired extends Data.TaggedError("SessionExpired")<{
+  provider: string
+  raw?: unknown
+}> {}
+
 export type AiError =
   | RateLimited
   | Unavailable
@@ -112,6 +121,7 @@ export type AiError =
   | IncompleteTurn
   | GenerationFailed
   | Unsupported
+  | SessionExpired
 
 const withReason = (base: string, reason: string | undefined): string =>
   reason === undefined ? base : `${base}: ${reason}`
@@ -136,5 +146,6 @@ export const describe: (e: AiError) => string = Match.type<AiError>().pipe(
     IncompleteTurn: () => "the provider stream ended without completing",
     GenerationFailed: (e) => e.message ?? `${e.provider} failed mid-generation`,
     Unsupported: (e) => withReason(`${e.provider} does not support ${e.capability}`, e.reason),
+    SessionExpired: (e) => `the ${e.provider} session reached its lifetime limit`,
   }),
 )

package/src/domain/Realtime.ts

@@ -0,0 +1,101 @@
+/**
+ * Types for a duplex speech-to-speech session: one long-lived connection the
+ * server drives, rather than a caller-driven turn.
+ *
+ * Only what both OpenAI Realtime and Gemini Live do natively lives here. VAD
+ * knobs, truncation, resumption, thinking, grounding and noise reduction are
+ * provider-typed on each adapter's request.
+ */
+import { Data, type Duration } from "effect"
+import type { AudioFormat } from "./Audio.js"
+import type { ImageSource } from "./Image.js"
+import type { HistoryItem, ToolCall, ToolCallOutput, Usage } from "./Items.js"
+import type { ToolDescriptor } from "../tool/Tool.js"
+
+/**
+ * What the caller pushes into a live session.
+ *
+ * - `Audio`: encoded per `request.inputFormat`. The adapter neither paces nor
+ *   buffers; send at real time.
+ * - `Text`: role `system` maps to a system item where the provider has one and
+ *   fails `Unsupported` where it does not.
+ * - `Interrupt`: cancel the response in flight.
+ * - `VideoFrame`: gated by the `RealtimeVideoInput` marker.
+ * - `ActivityStart` / `ActivityEnd`: manual turn detection only; `ActivityEnd`
+ *   also asks for a response. Dropped with a warning on a `"server"` session.
+ * - `PlaybackPosition`: how much of a response the user actually heard, so the
+ *   adapter can trim the rest from context. A no-op where the wire has no
+ *   truncate.
+ */
+export type RealtimeInput = Data.TaggedEnum<{
+  Audio: { readonly bytes: Uint8Array }
+  Text: { readonly text: string; readonly role?: "user" | "system" }
+  ToolResult: { readonly output: ToolCallOutput }
+  Interrupt: {}
+  VideoFrame: { readonly frame: ImageSource }
+  ActivityStart: {}
+  ActivityEnd: {}
+  PlaybackPosition: { readonly responseId: string; readonly playedMs: number }
+}>
+
+export const RealtimeInput = Data.taggedEnum<RealtimeInput>()
+
+/**
+ * What the session emits. Every `AudioDelta` and `OutputTranscriptDelta` is
+ * bracketed by `ResponseStarted` and `ResponseDone` carrying the same
+ * `responseId`.
+ *
+ * - `InputTranscript`: transcription of the user's own speech.
+ * - `SpeechStarted` / `SpeechStopped`: sparse. OpenAI emits them, Gemini
+ *   does not.
+ * - `Interrupted`: stop and flush playback now. Always precedes the
+ *   `ResponseDone { reason: "interrupted" }` for that response.
+ * - `ToolCallCancelled`: calls of an interrupted response that will never be
+ *   answered.
+ * - `ResumptionHandle`: Gemini only; pass it back as `request.resume`.
+ * - `SessionEnding`: the connection is about to close on the server's terms.
+ * - `Error`: non-fatal. A fatal error fails the stream instead.
+ */
+export type RealtimeEvent = Data.TaggedEnum<{
+  ResponseStarted: { readonly responseId: string }
+  AudioDelta: { readonly responseId: string; readonly bytes: Uint8Array }
+  OutputTranscriptDelta: { readonly responseId: string; readonly text: string }
+  InputTranscript: { readonly text: string; readonly final: boolean }
+  SpeechStarted: {}
+  SpeechStopped: {}
+  ToolCall: { readonly responseId: string; readonly call: ToolCall }
+  ToolCallCancelled: { readonly callIds: ReadonlyArray<string> }
+  Interrupted: { readonly responseId: string }
+  ResponseDone: {
+    readonly responseId: string
+    readonly reason: "complete" | "interrupted" | "cancelled" | "error"
+    readonly usage?: Usage
+  }
+  ResumptionHandle: { readonly handle: string }
+  SessionEnding: { readonly timeLeft?: Duration.Duration }
+  Error: { readonly code?: string; readonly message: string }
+}>
+
+export const RealtimeEvent = Data.taggedEnum<RealtimeEvent>()
+
+/**
+ * Cross-provider session setup. Each adapter extends this, narrowing `model`
+ * and adding its own knobs.
+ */
+export type CommonSessionRequest = {
+  readonly model: string
+  readonly instructions?: string
+  readonly voiceId?: string
+  /** `Toolkit.descriptors(toolkit)`. Immutable for the connection on Gemini. */
+  readonly tools?: ReadonlyArray<ToolDescriptor>
+  readonly inputFormat: AudioFormat
+  readonly outputFormat: AudioFormat
+  /** Who decides a turn ended. Defaults to `"server"`. */
+  readonly turnDetection?: "server" | "manual"
+  /** Transcribe the user's own audio. Defaults to `true`. */
+  readonly transcribeInput?: boolean
+  /** Text-only conversation seed. */
+  readonly history?: ReadonlyArray<HistoryItem>
+  /** A `ResumptionHandle` from a previous session. `Unsupported` off Gemini. */
+  readonly resume?: string
+}

package/src/domain/Transcript.test.ts

@@ -0,0 +1,98 @@
+import { describe, it } from "@effect/vitest"
+import { Cause, Effect, Queue, Ref, Stream } from "effect"
+import { TestClock } from "effect/testing"
+import { expect } from "vitest"
+import { accumulatePartials, type TranscriptEvent } from "./Transcript.js"
+
+const collecting = (stream: Stream.Stream<TranscriptEvent, never>) =>
+  Effect.gen(function* () {
+    const seen = yield* Ref.make<ReadonlyArray<TranscriptEvent>>([])
+    yield* Effect.forkChild(Stream.runForEach(stream, (e) => Ref.update(seen, (xs) => [...xs, e])))
+    return seen
+  })
+
+/** Deltas arriving faster than any settle window. */
+const speak = (
+  queue: Queue.Queue<TranscriptEvent, Cause.Done>,
+  tokens: ReadonlyArray<string>,
+): Effect.Effect<void> =>
+  Effect.forEach(
+    tokens,
+    (text) =>
+      Effect.andThen(Queue.offer(queue, { _tag: "partial", text }), TestClock.adjust("100 millis")),
+    { discard: true },
+  )
+
+const opened = Effect.gen(function* () {
+  const queue = yield* Queue.unbounded<TranscriptEvent, Cause.Done>()
+  const seen = yield* collecting(
+    Stream.fromQueue(queue).pipe(accumulatePartials({ silence: "1 second" })),
+  )
+  return { queue, seen }
+})
+
+describe("accumulatePartials", () => {
+  it.effect("joins deltas into the running hypothesis, forwarding each as it arrives", () =>
+    Effect.gen(function* () {
+      const { queue, seen } = yield* opened
+
+      yield* speak(queue, [" Hi", ",", " how"])
+
+      // Three in, three out, each the whole sentence so far, none delayed.
+      expect(yield* Ref.get(seen)).toEqual([
+        { _tag: "partial", text: "Hi" },
+        { _tag: "partial", text: "Hi," },
+        { _tag: "partial", text: "Hi, how" },
+      ])
+    }),
+  )
+
+  it.effect("commits on silence and starts the next utterance from empty", () =>
+    Effect.gen(function* () {
+      const { queue, seen } = yield* opened
+
+      yield* speak(queue, [" Hi", " there"])
+      yield* TestClock.adjust("3 seconds")
+      yield* speak(queue, [" Bye"])
+      yield* TestClock.adjust("3 seconds")
+
+      expect(yield* Ref.get(seen)).toEqual([
+        { _tag: "partial", text: "Hi" },
+        { _tag: "partial", text: "Hi there" },
+        { _tag: "final", text: "Hi there" },
+        { _tag: "partial", text: "Bye" },
+        { _tag: "final", text: "Bye" },
+      ])
+    }),
+  )
+
+  it.effect("a provider's own final resets the accumulator, so silence adds no duplicate", () =>
+    Effect.gen(function* () {
+      const { queue, seen } = yield* opened
+
+      yield* speak(queue, [" Hi"])
+      yield* Queue.offer(queue, { _tag: "final", text: "Hi." })
+      yield* TestClock.adjust("3 seconds")
+
+      expect(yield* Ref.get(seen)).toEqual([
+        { _tag: "partial", text: "Hi" },
+        { _tag: "final", text: "Hi." },
+      ])
+    }),
+  )
+
+  it.effect("commits the pending text when the stream ends mid-utterance", () =>
+    Effect.gen(function* () {
+      const { queue, seen } = yield* opened
+
+      yield* speak(queue, [" Half a sen"])
+      yield* Queue.end(queue)
+      yield* TestClock.adjust("10 millis")
+
+      expect(yield* Ref.get(seen)).toEqual([
+        { _tag: "partial", text: "Half a sen" },
+        { _tag: "final", text: "Half a sen" },
+      ])
+    }),
+  )
+})

package/src/domain/Transcript.ts

@@ -1,4 +1,5 @@
-import type { Duration } from "effect"
+import { type Duration, Effect, Match, Option, Ref, Stream } from "effect"
+import * as Settle from "../streaming/Settle.js"
 
 /**
  * Per-word timing + metadata. `confidence` and `speakerId` are optional
@@ -87,3 +88,54 @@ export const isMetadata = (
 ): e is Extract<TranscriptEvent, { _tag: "metadata" }> => e._tag === "metadata"
 export const isError = (e: TranscriptEvent): e is Extract<TranscriptEvent, { _tag: "error" }> =>
   e._tag === "error"
+
+export type AccumulatePartialsOptions = {
+  /** Silence after the last `partial` before the text so far is committed. */
+  readonly silence?: Duration.Input
+}
+
+const DEFAULT_SILENCE = "700 millis"
+
+const finalOf = (text: string): Option.Option<TranscriptEvent> =>
+  text.trim().length === 0 ? Option.none() : Option.some({ _tag: "final", text: text.trim() })
+
+/**
+ * Join fragment `partial`s into the running hypothesis and commit it as a
+ * `final` after `silence`. For providers that stream token-sized deltas
+ * (OpenAI Realtime) or never segment; piping one that already accumulates
+ * repeats its text.
+ *
+ * Partials are forwarded as they arrive; only the synthetic `final` waits.
+ * A real `final` resets the accumulator, and the stream end flushes it.
+ */
+export const accumulatePartials =
+  (options?: AccumulatePartialsOptions) =>
+  <E, R>(self: Stream.Stream<TranscriptEvent, E, R>): Stream.Stream<TranscriptEvent, E, R> =>
+    Stream.unwrap(
+      Effect.gen(function* () {
+        const acc = yield* Ref.make("")
+        const flush = Effect.map(Ref.getAndSet(acc, ""), finalOf)
+
+        const rewrite = (event: TranscriptEvent): Effect.Effect<TranscriptEvent> =>
+          Match.value(event).pipe(
+            Match.when({ _tag: "partial" }, (e) =>
+              Effect.map(
+                Ref.updateAndGet(acc, (text) => text + e.text),
+                (text): TranscriptEvent => ({ ...e, text: text.trim() }),
+              ),
+            ),
+            Match.when({ _tag: "final" }, (e) => Effect.as(Ref.set(acc, ""), e as TranscriptEvent)),
+            Match.orElse((e) => Effect.succeed(e as TranscriptEvent)),
+          )
+
+        return self.pipe(
+          Stream.mapEffect(rewrite),
+          Settle.onQuiet(options?.silence ?? DEFAULT_SILENCE, flush),
+          Stream.concat(
+            Stream.unwrap(
+              Effect.map(flush, Option.match({ onNone: () => Stream.empty, onSome: Stream.make })),
+            ),
+          ),
+        )
+      }),
+    )

package/src/index.ts

@@ -13,12 +13,17 @@ export * as Job from "./job/Job.js"
 export * as Media from "./domain/Media.js"
 export * as Music from "./domain/Music.js"
 export * as Research from "./domain/Research.js"
+export * as Realtime from "./domain/Realtime.js"
+export * as RealtimeSession from "./realtime/RealtimeSession.js"
 export * as Transcript from "./domain/Transcript.js"
 export * as Turn from "./domain/Turn.js"
 export * as Embedding from "./embedding-model/Embedding.js"
 export * as EmbeddingModel from "./embedding-model/EmbeddingModel.js"
 export * as ImageGenerator from "./image-generator/ImageGenerator.js"
 export * as LanguageModel from "./language-model/LanguageModel.js"
+export * as Messenger from "./messenger/Messenger.js"
+export * as MessengerAdapter from "./messenger/MessengerAdapter.js"
+export * as MessengerError from "./messenger/MessengerError.js"
 export * as MusicGenerator from "./music-generator/MusicGenerator.js"
 export * as Reranker from "./reranker/Reranker.js"
 export * as Memory from "./sandbox/Memory.js"
@@ -34,6 +39,8 @@ export * as WebSearch from "./web-search/WebSearch.js"
 export * as WebSearchTool from "./web-search/WebSearchTool.js"
 export * as Vector from "./math/Vector.js"
 export * as Tokenizer from "./tokenizer/Tokenizer.js"
+export * as Settle from "./streaming/Settle.js"
+export * as WebSocketSession from "./streaming/WebSocketSession.js"
 export * as Loop from "./loop/Loop.js"
 export * as Retry from "./retry/Retry.js"
 export * as Tool from "./tool/Tool.js"