@effect-uai/inworld 0.17.0 → 0.19.0-dev-20261109183959

package/src/codec.ts

@@ -1,4 +1,5 @@
-import { Effect, Encoding, Match, Redacted, Result } from "effect"
+import { Effect, Match, Redacted, Result } from "effect"
+import { Base64 } from "effect/encoding"
 import * as AiError from "@effect-uai/core/AiError"
 import type { AudioFormat } from "@effect-uai/core/Audio"
 import type { InworldAudioEncoding } from "./models.js"
@@ -82,7 +83,7 @@ export const defaultFormat: AudioFormat = {
 // ---------------------------------------------------------------------------
 
 export const decodeAudioContent = (b64: string): Effect.Effect<Uint8Array, AiError.AiError> =>
-  Result.match(Encoding.decodeBase64(b64), {
+  Result.match(Base64.decode(b64), {
     onSuccess: Effect.succeed,
     onFailure: (cause) =>
       Effect.fail(

package/src/realtimeStt.test.ts

@@ -0,0 +1,82 @@
+import { describe, it } from "@effect/vitest"
+import { Effect, Exit, Fiber, Redacted, Stream } from "effect"
+import { expect } from "vitest"
+import * as FakeWebSocket from "@effect-uai/core/testing/FakeWebSocket"
+import { streamTranscription } from "./realtimeStt.js"
+
+const pcm = { container: "raw", encoding: "pcm_s16le", sampleRate: 16000, channels: 1 } as const
+
+const request = { model: "inworld/inworld-stt-1", inputFormat: pcm }
+
+const transcribing = FakeWebSocket.make({
+  greeting: [
+    { result: { speechStarted: { startTimeMs: 1500 } } },
+    { result: { transcription: { transcript: "hel", isFinal: false } } },
+    { result: { transcription: { transcript: "hello", isFinal: true } } },
+  ],
+})
+
+/** Run a transcription against `server`, closing it, if asked, once the frames are in. */
+const transcribe = <E>(
+  server: FakeWebSocket.FakeWebSocketServer,
+  audioIn: Stream.Stream<Uint8Array, E>,
+  close?: { readonly code: number; readonly afterSent?: number },
+) =>
+  Effect.gen(function* () {
+    const run = streamTranscription({
+      apiKey: Redacted.make("test-key"),
+      webSocket: server.connect,
+    })
+    const collector = yield* Effect.forkChild(Effect.exit(Stream.runCollect(run(audioIn, request))))
+    if (close !== undefined) {
+      yield* server.greeted
+      yield* server.awaitSent(close.afterSent ?? 0)
+      yield* server.close(close.code)
+    }
+    return yield* Fiber.join(collector)
+  })
+
+describe("Inworld Realtime STT socket", () => {
+  it.live("sends the config before any audio, and ends on a clean close", () =>
+    Effect.gen(function* () {
+      const server = yield* transcribing
+      const exit = yield* transcribe(server, Stream.make(new Uint8Array([1, 2])), {
+        code: 1000,
+        afterSent: 4,
+      })
+
+      expect(exit).toEqual(
+        Exit.succeed([
+          { _tag: "speech-started", atSeconds: 1.5 },
+          { _tag: "partial", text: "hel" },
+          { _tag: "final", text: "hello" },
+        ]),
+      )
+      const sent = (yield* server.sent) as ReadonlyArray<Record<string, unknown>>
+      expect(sent.map((f) => Object.keys(f)[0])).toEqual([
+        "transcribeConfig",
+        "audioChunk",
+        "endTurn",
+        "closeStream",
+      ])
+    }).pipe(Effect.scoped),
+  )
+
+  it.live("fails rather than ending when the connection drops", () =>
+    Effect.gen(function* () {
+      const server = yield* transcribing
+      const exit = yield* transcribe(server, Stream.never, { code: 1011 })
+
+      expect(Exit.isFailure(exit) && JSON.stringify(exit.cause)).toContain("Unavailable")
+    }).pipe(Effect.scoped),
+  )
+
+  it.live("fails with the audio input's own error", () =>
+    Effect.gen(function* () {
+      const server = yield* transcribing
+      const exit = yield* transcribe(server, Stream.fail("microphone unplugged"))
+
+      expect(Exit.isFailure(exit) && JSON.stringify(exit.cause)).toContain("microphone unplugged")
+    }).pipe(Effect.scoped),
+  )
+})

package/src/realtimeStt.ts

@@ -18,17 +18,27 @@
  * Inworld's STT WS sends audio as base64 inside JSON (NOT binary frames),
  * matching the rest of the Inworld API style.
  */
-import { Cause, Effect, Encoding, Match, Queue, Redacted, Schema, Stream } from "effect"
-import * as Socket from "effect/unstable/socket/Socket"
+import { Effect, Match, Option, Predicate, Redacted, Schema, Stream } from "effect"
+import { Base64 } from "effect/encoding"
+import * as Socket from "effect/socket/Socket"
 import * as AiError from "@effect-uai/core/AiError"
 import type { AudioFormat } from "@effect-uai/core/Audio"
 import * as Capabilities from "@effect-uai/core/Capabilities"
 import * as JSONL from "@effect-uai/core/JSONL"
 import type { TranscriptEvent, WordTimestamp } from "@effect-uai/core/Transcript"
 import type { CommonStreamTranscribeRequest } from "@effect-uai/core/Transcriber"
+import * as WebSocketSession from "@effect-uai/core/WebSocketSession"
 import { authedWsConstructor } from "./wsAuth.js"
 
-export type Config = { readonly apiKey: Redacted.Redacted; readonly baseUrl?: string }
+export type Config = {
+  readonly apiKey: Redacted.Redacted
+  readonly baseUrl?: string
+  /**
+   * Build the socket yourself, for a proxy or an in-memory transport. The
+   * default attaches the Basic auth header through the `ws` package.
+   */
+  readonly webSocket?: Socket.WebSocketConstructor["Service"]
+}
 
 // ---------------------------------------------------------------------------
 // AudioFormat → `audioEncoding` slug for Inworld STT
@@ -85,7 +95,7 @@ const configFrame = (encoding: WireEncoding, request: CommonStreamTranscribeRequ
   })
 
 const audioChunkFrame = (bytes: Uint8Array) =>
-  JSON.stringify({ audioChunk: { content: Encoding.encodeBase64(bytes) } })
+  JSON.stringify({ audioChunk: { content: Base64.encode(bytes) } })
 
 const endTurnFrame = JSON.stringify({ endTurn: {} })
 const closeStreamFrame = JSON.stringify({ closeStream: {} })
@@ -177,14 +187,13 @@ export const wireToEvent = (frame: typeof ServerFrame.Type): TranscriptEvent | u
   return undefined
 }
 
-const handleServerMessage = (queue: Queue.Queue<TranscriptEvent, Cause.Done>) => (raw: string) =>
+/** One raw text frame to at most one event; unknown or malformed frames yield `undefined`. */
+const frameToEvent = (raw: string): Effect.Effect<TranscriptEvent | undefined> =>
   Effect.gen(function* () {
     const json = yield* JSONL.parseSafe(raw)
-    if (json === undefined) return
+    if (json === undefined) return undefined
     const decoded = yield* decodeServerFrame(json).pipe(Effect.option)
-    if (decoded._tag === "None") return
-    const event = wireToEvent(decoded.value)
-    if (event !== undefined) yield* Queue.offer(queue, event)
+    return Option.match(decoded, { onNone: () => undefined, onSome: wireToEvent })
   })
 
 // ---------------------------------------------------------------------------
@@ -206,32 +215,33 @@ export const streamTranscription =
           reason: "Inworld STT has no free-form prompt field; bias via `biasingTerms`.",
         })
         const encoding = yield* inputFormatToWire(request.inputFormat)
-        const socket = yield* Socket.makeWebSocket(buildWsUrl(cfg), {
-          // Effect's Socket treats all close codes as errors by default —
-          // whitelist standard clean-close codes (1000 / 1001 / 1005).
-          closeCodeIsError: (code) => code !== 1000 && code !== 1001 && code !== 1005,
-        }).pipe(Effect.provideService(Socket.WebSocketConstructor, authedWsConstructor(cfg.apiKey)))
-        const queue = yield* Queue.bounded<TranscriptEvent, Cause.Done>(64)
-        const write = yield* socket.writer
-
-        // Writer: config → drain audio → endTurn → closeStream. Inworld
-        // emits `result.transcription` partials in real-time via VAD;
-        // `endTurn` is sent on input-stream end to flush any tail audio
-        // into a final transcript, then `closeStream` for graceful close.
-        yield* Effect.gen(function* () {
-          yield* write(configFrame(encoding, request))
-          yield* Stream.runForEach(audioIn, (bytes) => write(audioChunkFrame(bytes)))
-          yield* write(endTurnFrame)
-          yield* write(closeStreamFrame)
-        }).pipe(Effect.ignore, Effect.forkScoped)
-
-        // `Queue.end` flushes pending events then fails the next take with
-        // `Done` — clean stream end. `Queue.shutdown` would CLEAR queued
-        // items and interrupt pending takes (wrong for graceful teardown).
-        yield* socket
-          .runString(handleServerMessage(queue))
-          .pipe(Effect.ensuring(Queue.end(queue)), Effect.forkScoped)
-
-        return Stream.fromQueue(queue)
+        const socket = yield* Socket.makeWebSocket(buildWsUrl(cfg)).pipe(
+          Effect.provideService(
+            Socket.WebSocketConstructor,
+            cfg.webSocket ?? authedWsConstructor(cfg.apiKey),
+          ),
+        )
+        const { write } = yield* socket.writer
+        const asAiError = WebSocketSession.toAiError("inworld")
+
+        const events = Stream.fromPull(Socket.readerString(socket)).pipe(
+          Stream.scoped,
+          Stream.catchIf(WebSocketSession.isCleanClose, () => Stream.empty),
+          Stream.mapError(asAiError),
+          Stream.mapEffect(frameToEvent),
+          Stream.filter(Predicate.isNotUndefined),
+        )
+
+        // `endTurn` flushes tail audio into a final transcript before `closeStream`.
+        const outgoing = Stream.concat(
+          Stream.make(configFrame(encoding, request)),
+          Stream.concat(
+            Stream.map(audioIn, audioChunkFrame),
+            Stream.make(endTurnFrame, closeStreamFrame),
+          ),
+        ).pipe(Stream.mapEffect((frame) => Effect.mapError(write(frame), asAiError)))
+
+        // The server's close ends the transcript; the audio running out does not.
+        return Stream.merge(events, Stream.drain(outgoing), { haltStrategy: "left" })
       }),
     )

package/src/realtimeTts.test.ts

@@ -0,0 +1,74 @@
+import { describe, it } from "@effect/vitest"
+import { Effect, Exit, Fiber, Redacted, Stream } from "effect"
+import { Base64 } from "effect/encoding"
+import { expect } from "vitest"
+import * as FakeWebSocket from "@effect-uai/core/testing/FakeWebSocket"
+import { streamSynthesis } from "./realtimeTts.js"
+
+const request = { model: "inworld-tts-2", voiceId: "Ashley" } as const
+
+const synthesizing = FakeWebSocket.make({
+  greeting: [
+    { result: { contextCreated: {} } },
+    { result: { audioChunk: { audioContent: Base64.encode(new Uint8Array([1, 2, 3])) } } },
+    { error: { message: "transient" } },
+    { result: { audioChunk: { audioContent: "" } } },
+    { result: { contextClosed: {} } },
+  ],
+})
+
+/** Run a synthesis against `server`, closing it, if asked, once the frames are in. */
+const synthesize = <E>(
+  server: FakeWebSocket.FakeWebSocketServer,
+  textIn: Stream.Stream<string, E>,
+  close?: { readonly code: number; readonly afterSent?: number },
+) =>
+  Effect.gen(function* () {
+    const run = streamSynthesis({ apiKey: Redacted.make("test-key"), webSocket: server.connect })
+    const collector = yield* Effect.forkChild(Effect.exit(Stream.runCollect(run(textIn, request))))
+    if (close !== undefined) {
+      yield* server.greeted
+      yield* server.awaitSent(close.afterSent ?? 0)
+      yield* server.close(close.code)
+    }
+    return yield* Fiber.join(collector)
+  })
+
+describe("Inworld Realtime TTS socket", () => {
+  it.live("creates the context before any text, and ends on a clean close", () =>
+    Effect.gen(function* () {
+      const server = yield* synthesizing
+      const exit = yield* synthesize(server, Stream.make("Hello", "", " world"), {
+        code: 1000,
+        afterSent: 4,
+      })
+
+      expect(exit).toEqual(Exit.succeed([{ bytes: new Uint8Array([1, 2, 3]) }]))
+      const sent = (yield* server.sent) as ReadonlyArray<Record<string, unknown>>
+      expect(sent.map((f) => Object.keys(f)[0])).toEqual([
+        "create",
+        "send_text",
+        "send_text",
+        "close_context",
+      ])
+    }).pipe(Effect.scoped),
+  )
+
+  it.live("fails rather than ending when the connection drops", () =>
+    Effect.gen(function* () {
+      const server = yield* synthesizing
+      const exit = yield* synthesize(server, Stream.never, { code: 1011 })
+
+      expect(Exit.isFailure(exit) && JSON.stringify(exit.cause)).toContain("Unavailable")
+    }).pipe(Effect.scoped),
+  )
+
+  it.live("fails with the text input's own error", () =>
+    Effect.gen(function* () {
+      const server = yield* synthesizing
+      const exit = yield* synthesize(server, Stream.fail("upstream llm failed"))
+
+      expect(Exit.isFailure(exit) && JSON.stringify(exit.cause)).toContain("upstream llm failed")
+    }).pipe(Effect.scoped),
+  )
+})

package/src/realtimeTts.ts

@@ -19,17 +19,26 @@
  * Multi-context (`contextId`) is not surfaced here — one logical
  * utterance per call.
  */
-import { Cause, Effect, Queue, Redacted, Schema, Stream } from "effect"
-import * as Socket from "effect/unstable/socket/Socket"
+import { Effect, Option, Predicate, Redacted, Schema, Stream } from "effect"
+import * as Socket from "effect/socket/Socket"
 import * as AiError from "@effect-uai/core/AiError"
 import type { AudioChunk, AudioFormat } from "@effect-uai/core/Audio"
 import * as JSONL from "@effect-uai/core/JSONL"
 import type { CustomPronunciation } from "@effect-uai/core/SpeechSynthesizer"
+import * as WebSocketSession from "@effect-uai/core/WebSocketSession"
 import { audioConfigFor, decodeAudioContent, defaultFormat } from "./codec.js"
 import type { InworldDeliveryMode, InworldTtsModel, InworldVoiceId } from "./models.js"
 import { authedWsConstructor } from "./wsAuth.js"
 
-export type Config = { readonly apiKey: Redacted.Redacted; readonly baseUrl?: string }
+export type Config = {
+  readonly apiKey: Redacted.Redacted
+  readonly baseUrl?: string
+  /**
+   * Build the socket yourself, for a proxy or an in-memory transport. The
+   * default attaches the Basic auth header through the `ws` package.
+   */
+  readonly webSocket?: Socket.WebSocketConstructor["Service"]
+}
 
 /**
  * Incremental-text-in request. Mirrors the sync request minus `text` (which
@@ -110,21 +119,25 @@ const ServerFrame = Schema.Struct({
 })
 const decodeServerFrame = Schema.decodeUnknownEffect(ServerFrame)
 
-const handleServerFrame = (queue: Queue.Queue<AudioChunk, Cause.Done>) => (raw: string) =>
+/** One raw text frame to at most one audio chunk; error frames are logged, not raised. */
+const frameToChunk = (raw: string): Effect.Effect<AudioChunk | undefined> =>
   Effect.gen(function* () {
     const json = yield* JSONL.parseSafe(raw)
-    if (json === undefined) return
+    if (json === undefined) return undefined
     const decoded = yield* decodeServerFrame(json).pipe(Effect.option)
-    if (decoded._tag === "None") return
+    if (Option.isNone(decoded)) return undefined
     const frame = decoded.value
     if (frame.error !== undefined) {
       yield* Effect.logWarning("[inworld-tts] server error frame", { error: frame.error })
-      return
+      return undefined
     }
     const audio = frame.result?.audioChunk?.audioContent
-    if (audio === undefined || audio === "") return
+    if (audio === undefined || audio === "") return undefined
     const bytes = yield* decodeAudioContent(audio).pipe(Effect.option)
-    if (bytes._tag === "Some") yield* Queue.offer(queue, { bytes: bytes.value })
+    return Option.match(bytes, {
+      onNone: () => undefined,
+      onSome: (b): AudioChunk => ({ bytes: b }),
+    })
   })
 
 // ---------------------------------------------------------------------------
@@ -153,32 +166,35 @@ export const streamSynthesis =
           )
         }
         const create = yield* createFrame(request)
-        const socket = yield* Socket.makeWebSocket(buildWsUrl(cfg), {
-          // Effect's Socket treats all close codes as errors by default —
-          // whitelist standard clean-close codes (1000 / 1001 / 1005).
-          closeCodeIsError: (code) => code !== 1000 && code !== 1001 && code !== 1005,
-        }).pipe(Effect.provideService(Socket.WebSocketConstructor, authedWsConstructor(cfg.apiKey)))
-        const queue = yield* Queue.bounded<AudioChunk, Cause.Done>(64)
-        const write = yield* socket.writer
-
-        // Writer: BOS `create` → drain text as `send_text` → `close_context`.
-        // The reader fiber surfaces the remaining audio drained by the server
-        // post-close and shuts the queue when the WS upstream closes.
-        yield* Effect.gen(function* () {
-          yield* write(create)
-          yield* Stream.runForEach(textIn, (text) =>
-            text.length === 0 ? Effect.void : write(sendTextFrame(text)),
-          )
-          yield* write(closeContextFrame)
-        }).pipe(Effect.ignore, Effect.forkScoped)
-
-        // `Queue.end` flushes pending chunks then fails the next take with
-        // `Done` — clean stream end. `Queue.shutdown` would CLEAR queued items
-        // and interrupt pending takes (wrong for graceful teardown).
-        yield* socket
-          .runString(handleServerFrame(queue))
-          .pipe(Effect.ensuring(Queue.end(queue)), Effect.forkScoped)
-
-        return Stream.fromQueue(queue)
+        const socket = yield* Socket.makeWebSocket(buildWsUrl(cfg)).pipe(
+          Effect.provideService(
+            Socket.WebSocketConstructor,
+            cfg.webSocket ?? authedWsConstructor(cfg.apiKey),
+          ),
+        )
+        const { write } = yield* socket.writer
+        const asAiError = WebSocketSession.toAiError("inworld")
+
+        const chunks = Stream.fromPull(Socket.readerString(socket)).pipe(
+          Stream.scoped,
+          Stream.catchIf(WebSocketSession.isCleanClose, () => Stream.empty),
+          Stream.mapError(asAiError),
+          Stream.mapEffect(frameToChunk),
+          Stream.filter(Predicate.isNotUndefined),
+        )
+
+        const outgoing = Stream.concat(
+          Stream.make(create),
+          Stream.concat(
+            textIn.pipe(
+              Stream.filter((text) => text.length > 0),
+              Stream.map(sendTextFrame),
+            ),
+            Stream.make(closeContextFrame),
+          ),
+        ).pipe(Stream.mapEffect((frame) => Effect.mapError(write(frame), asAiError)))
+
+        // The server drains remaining audio after `close_context`, then closes.
+        return Stream.merge(chunks, Stream.drain(outgoing), { haltStrategy: "left" })
       }),
     )

package/src/wsAuth.ts

@@ -9,7 +9,7 @@
  * we use the `ws` peer dep (Node/Bun only).
  */
 import { Redacted } from "effect"
-import type * as Socket from "effect/unstable/socket/Socket"
+import type * as Socket from "effect/socket/Socket"
 import { WebSocket as WSWebSocket } from "ws"
 import { authHeader } from "./codec.js"