@effect-uai/elevenlabs 0.16.0 → 0.18.0

package/src/realtimeStt.test.ts

@@ -0,0 +1,92 @@
+import { describe, it } from "@effect/vitest"
+import { Deferred, Effect, Exit, Fiber, Layer, Redacted, Stream } from "effect"
+import { HttpClient, HttpClientResponse } from "effect/http"
+import { expect } from "vitest"
+import * as FakeWebSocket from "@effect-uai/core/testing/FakeWebSocket"
+import { streamTranscription } from "./realtimeStt.js"
+
+const pcm = { container: "raw", encoding: "pcm_s16le", sampleRate: 16000, channels: 1 } as const
+
+const request = { model: "scribe_v2_realtime", inputFormat: pcm }
+
+/** Hands out a token and signals `issued`, so the test closes only after the connect starts. */
+const tokenClient = (issued: Deferred.Deferred<void>) =>
+  Layer.succeed(
+    HttpClient.HttpClient,
+    HttpClient.make((req) =>
+      Deferred.succeed(issued, undefined).pipe(
+        Effect.as(HttpClientResponse.fromWeb(req, new Response(JSON.stringify({ token: "t" })))),
+      ),
+    ),
+  )
+
+const transcribing = FakeWebSocket.make({
+  greeting: [
+    { message_type: "session_started", session_id: "s1" },
+    { message_type: "partial_transcript", text: "hel" },
+    { message_type: "committed_transcript", text: "hello" },
+  ],
+})
+
+/** Run a transcription against `server`, closing it, if asked, once the frames are in. */
+const transcribe = <E>(
+  server: FakeWebSocket.FakeWebSocketServer,
+  audioIn: Stream.Stream<Uint8Array, E>,
+  close?: { readonly code: number; readonly afterSent?: number },
+) =>
+  Effect.gen(function* () {
+    const issued = yield* Deferred.make<void>()
+    const run = streamTranscription({ apiKey: Redacted.make("test-key") })(audioIn, request).pipe(
+      Stream.provide(Layer.merge(server.layer, tokenClient(issued))),
+    )
+    const collector = yield* Effect.forkChild(Effect.exit(Stream.runCollect(run)))
+    yield* Deferred.await(issued)
+    if (close !== undefined) {
+      yield* server.greeted
+      yield* server.awaitSent(close.afterSent ?? 0)
+      yield* server.close(close.code)
+    }
+    return yield* Fiber.join(collector)
+  })
+
+describe("ElevenLabs realtime STT socket", () => {
+  it.live("streams audio chunks, decodes transcripts, and ends on a clean close", () =>
+    Effect.gen(function* () {
+      const server = yield* transcribing
+      const exit = yield* transcribe(server, Stream.make(new Uint8Array([1, 2])), {
+        code: 1000,
+        afterSent: 1,
+      })
+
+      expect(exit).toEqual(
+        Exit.succeed([
+          { _tag: "partial", text: "hel" },
+          { _tag: "final", text: "hello" },
+        ]),
+      )
+      const sent = (yield* server.sent) as ReadonlyArray<{
+        message_type: string
+        sample_rate: number
+      }>
+      expect(sent).toMatchObject([{ message_type: "input_audio_chunk", sample_rate: 16000 }])
+    }).pipe(Effect.scoped),
+  )
+
+  it.live("fails rather than ending when the connection drops", () =>
+    Effect.gen(function* () {
+      const server = yield* transcribing
+      const exit = yield* transcribe(server, Stream.never, { code: 1011 })
+
+      expect(Exit.isFailure(exit) && JSON.stringify(exit.cause)).toContain("Unavailable")
+    }).pipe(Effect.scoped),
+  )
+
+  it.live("fails with the audio input's own error", () =>
+    Effect.gen(function* () {
+      const server = yield* transcribing
+      const exit = yield* transcribe(server, Stream.fail("microphone unplugged"))
+
+      expect(Exit.isFailure(exit) && JSON.stringify(exit.cause)).toContain("microphone unplugged")
+    }).pipe(Effect.scoped),
+  )
+})

package/src/realtimeStt.ts

@@ -1,12 +1,14 @@
-import { Cause, Effect, Encoding, Match, Queue, Redacted, Schema, Stream } from "effect"
-import { HttpClient, HttpClientRequest } from "effect/unstable/http"
-import * as Socket from "effect/unstable/socket/Socket"
+import { Effect, Match, Option, Predicate, Redacted, Schema, Stream } from "effect"
+import { Base64 } from "effect/encoding"
+import { HttpClient, HttpClientRequest } from "effect/http"
+import * as Socket from "effect/socket/Socket"
 import * as AiError from "@effect-uai/core/AiError"
 import type { AudioFormat } from "@effect-uai/core/Audio"
 import * as Capabilities from "@effect-uai/core/Capabilities"
 import * as JSONL from "@effect-uai/core/JSONL"
 import type { TranscriptEvent, WordTimestamp } from "@effect-uai/core/Transcript"
 import type { CommonStreamTranscribeRequest } from "@effect-uai/core/Transcriber"
+import * as WebSocketSession from "@effect-uai/core/WebSocketSession"
 import { httpStatusError, transportFailure } from "./codec.js"
 import { type ElevenLabsRegion, resolveHost } from "./region.js"
 
@@ -200,29 +202,19 @@ export const buildWsUrl = (
 export const encodeAudioFrame = (bytes: Uint8Array, sampleRate: number) =>
   JSON.stringify({
     message_type: "input_audio_chunk",
-    audio_base_64: Encoding.encodeBase64(bytes),
+    audio_base_64: Base64.encode(bytes),
     sample_rate: sampleRate,
   })
 
-// ---------------------------------------------------------------------------
-// Parse one server message and emit to the queue if it maps to a
-// TranscriptEvent. Invalid JSON / unknown shapes are silently dropped —
-// the connection stays open and we keep streaming subsequent messages.
-// ---------------------------------------------------------------------------
-
-const handleServerMessage =
-  (
-    queue: Queue.Queue<TranscriptEvent, Cause.Done>,
-    mapEvent: (msg: typeof ServerMessage.Type) => TranscriptEvent | undefined,
-  ) =>
-  (raw: string) =>
+/** One raw text frame to at most one event; unknown or malformed frames yield `undefined`. */
+const frameToEvent =
+  (mapEvent: (msg: typeof ServerMessage.Type) => TranscriptEvent | undefined) =>
+  (raw: string): Effect.Effect<TranscriptEvent | undefined> =>
     Effect.gen(function* () {
       const json = yield* JSONL.parseSafe(raw)
-      if (json === undefined) return
+      if (json === undefined) return undefined
       const decoded = yield* decodeServerMessage(json).pipe(Effect.option)
-      if (decoded._tag === "None") return
-      const event = mapEvent(decoded.value)
-      if (event !== undefined) yield* Queue.offer(queue, event)
+      return Option.match(decoded, { onNone: () => undefined, onSome: mapEvent })
     })
 
 // ---------------------------------------------------------------------------
@@ -261,27 +253,25 @@ export const streamTranscription =
           ...(request.biasingTerms !== undefined && { keyterms: request.biasingTerms }),
           includeTimestamps,
         })
-        const socket = yield* Socket.makeWebSocket(url, {
-          // Effect's Socket treats all close codes as errors by default — that
-          // surfaces a clean server-side close as a stream failure. Whitelist
-          // standard clean-close codes (1000 / 1001 / 1005).
-          closeCodeIsError: (code) => code !== 1000 && code !== 1001 && code !== 1005,
-        })
-        const queue = yield* Queue.bounded<TranscriptEvent, Cause.Done>(64)
+        const socket = yield* Socket.makeWebSocket(url)
         const sampleRate = request.inputFormat.sampleRate
+        const { write } = yield* socket.writer
+        const asAiError = WebSocketSession.toAiError("elevenlabs")
 
-        const write = yield* socket.writer
-        yield* Stream.runForEach(audioIn, (bytes) =>
-          write(encodeAudioFrame(bytes, sampleRate)),
-        ).pipe(Effect.ignore, Effect.forkScoped)
+        const events = Stream.fromPull(Socket.readerString(socket)).pipe(
+          Stream.scoped,
+          Stream.catchIf(WebSocketSession.isCleanClose, () => Stream.empty),
+          Stream.mapError(asAiError),
+          Stream.mapEffect(frameToEvent(wireToEvent(includeTimestamps))),
+          Stream.filter(Predicate.isNotUndefined),
+        )
 
-        // `Queue.end` flushes pending events then fails the next take with
-        // `Done` — clean stream end. `Queue.shutdown` would CLEAR queued
-        // items and interrupt pending takes (wrong for graceful teardown).
-        yield* socket
-          .runString(handleServerMessage(queue, wireToEvent(includeTimestamps)))
-          .pipe(Effect.ensuring(Queue.end(queue)), Effect.forkScoped)
+        // Writes wait for the reader to connect.
+        const outgoing = Stream.map(audioIn, (bytes) => encodeAudioFrame(bytes, sampleRate)).pipe(
+          Stream.mapEffect((frame) => Effect.mapError(write(frame), asAiError)),
+        )
 
-        return Stream.fromQueue(queue)
+        // The server's close ends the transcript; the audio running out does not.
+        return Stream.merge(events, Stream.drain(outgoing), { haltStrategy: "left" })
       }),
     )

package/src/realtimeTts.test.ts

@@ -0,0 +1,72 @@
+import { describe, it } from "@effect/vitest"
+import { Effect, Exit, Fiber, Redacted, Stream } from "effect"
+import { Base64 } from "effect/encoding"
+import { expect } from "vitest"
+import * as FakeWebSocket from "@effect-uai/core/testing/FakeWebSocket"
+import { streamSynthesis } from "./realtimeTts.js"
+
+const request = { voiceId: "voice-1" }
+
+const synthesizing = FakeWebSocket.make({
+  greeting: [
+    { audio: Base64.encode(new Uint8Array([1, 2])), isFinal: null },
+    { audio: null, isFinal: true },
+  ],
+})
+
+/** Run a synthesis against `server`, closing it, if asked, once the frames are in. */
+const synthesize = <E>(
+  server: FakeWebSocket.FakeWebSocketServer,
+  textIn: Stream.Stream<string, E>,
+  close?: { readonly code: number; readonly afterSent?: number },
+) =>
+  Effect.gen(function* () {
+    const run = streamSynthesis({ apiKey: Redacted.make("test-key") })(textIn, request).pipe(
+      Stream.provide(server.layer),
+    )
+    const collector = yield* Effect.forkChild(Effect.exit(Stream.runCollect(run)))
+    if (close !== undefined) {
+      yield* server.greeted
+      yield* server.awaitSent(close.afterSent ?? 0)
+      yield* server.close(close.code)
+    }
+    return yield* Fiber.join(collector)
+  })
+
+describe("ElevenLabs realtime TTS socket", () => {
+  it.live("sends BOS before any text and EOS after it, and ends on a clean close", () =>
+    Effect.gen(function* () {
+      const server = yield* synthesizing
+      const exit = yield* synthesize(server, Stream.make("Hello", "", "world "), {
+        code: 1000,
+        afterSent: 4,
+      })
+
+      expect(exit).toEqual(Exit.succeed([{ bytes: new Uint8Array([1, 2]) }]))
+      expect(yield* server.sent).toEqual([
+        { text: " ", "xi-api-key": "test-key" },
+        { text: "Hello " },
+        { text: "world " },
+        { text: "" },
+      ])
+    }).pipe(Effect.scoped),
+  )
+
+  it.live("fails rather than ending when the connection drops", () =>
+    Effect.gen(function* () {
+      const server = yield* synthesizing
+      const exit = yield* synthesize(server, Stream.never, { code: 1011 })
+
+      expect(Exit.isFailure(exit) && JSON.stringify(exit.cause)).toContain("Unavailable")
+    }).pipe(Effect.scoped),
+  )
+
+  it.live("fails with the text input's own error", () =>
+    Effect.gen(function* () {
+      const server = yield* synthesizing
+      const exit = yield* synthesize(server, Stream.fail("llm stream broke"))
+
+      expect(Exit.isFailure(exit) && JSON.stringify(exit.cause)).toContain("llm stream broke")
+    }).pipe(Effect.scoped),
+  )
+})

package/src/realtimeTts.ts

@@ -1,9 +1,11 @@
-import { Cause, Effect, Encoding, Queue, Redacted, Result, Schema, Stream } from "effect"
-import * as Socket from "effect/unstable/socket/Socket"
+import { Effect, Option, Predicate, Redacted, Result, Schema, Stream } from "effect"
+import { Base64 } from "effect/encoding"
+import * as Socket from "effect/socket/Socket"
 import * as AiError from "@effect-uai/core/AiError"
 import type { AudioChunk, AudioFormat } from "@effect-uai/core/Audio"
 import * as JSONL from "@effect-uai/core/JSONL"
 import type { CustomPronunciation } from "@effect-uai/core/SpeechSynthesizer"
+import * as WebSocketSession from "@effect-uai/core/WebSocketSession"
 import {
   defaultFormat,
   formatToOutputSlug,
@@ -85,7 +87,7 @@ const ServerFrame = Schema.Struct({
 const decodeServerFrame = Schema.decodeUnknownEffect(ServerFrame)
 
 const decodeAudio = (b64: string): Effect.Effect<Uint8Array, AiError.AiError> =>
-  Result.match(Encoding.decodeBase64(b64), {
+  Result.match(Base64.decode(b64), {
     onSuccess: Effect.succeed,
     onFailure: (cause) =>
       Effect.fail(
@@ -96,23 +98,24 @@ const decodeAudio = (b64: string): Effect.Effect<Uint8Array, AiError.AiError> =>
       ),
   })
 
-const handleServerFrame = (queue: Queue.Queue<AudioChunk, Cause.Done>) => (raw: string) =>
+/** One raw text frame to at most one chunk; error frames are logged and dropped. */
+const frameToChunk = (raw: string): Effect.Effect<AudioChunk | undefined> =>
   Effect.gen(function* () {
     const json = yield* JSONL.parseSafe(raw)
-    if (json === undefined) return
+    if (json === undefined) return undefined
     const decoded = yield* decodeServerFrame(json).pipe(Effect.option)
-    if (decoded._tag === "None") return
+    if (decoded._tag === "None") return undefined
     const frame = decoded.value
     if (frame.error !== undefined) {
       yield* Effect.logWarning("[elevenlabs-tts] server error frame", {
         error: frame.error,
         message: frame.message,
       })
-      return
+      return undefined
     }
-    if (frame.audio == null || frame.audio === "") return
+    if (frame.audio == null || frame.audio === "") return undefined
     const bytes = yield* decodeAudio(frame.audio).pipe(Effect.option)
-    if (bytes._tag === "Some") yield* Queue.offer(queue, { bytes: bytes.value })
+    return Option.match(bytes, { onNone: () => undefined, onSome: (b) => ({ bytes: b }) })
   })
 
 // ---------------------------------------------------------------------------
@@ -129,35 +132,31 @@ export const streamSynthesis =
       Effect.gen(function* () {
         yield* rejectInlinePronunciations(request.pronunciations)
         const slug = yield* formatToOutputSlug(request.outputFormat ?? defaultFormat)
-        // ElevenLabs closes `/stream-input` with code 1000 after delivering the
-        // final audio chunk. Effect's default treats all close codes as errors,
-        // which would surface as a stream failure right after the last audio
-        // arrives. Whitelist standard clean-close codes.
-        const socket = yield* Socket.makeWebSocket(buildWsUrl(cfg, request, slug), {
-          closeCodeIsError: (code) => code !== 1000 && code !== 1001 && code !== 1005,
-        })
-        const queue = yield* Queue.bounded<AudioChunk, Cause.Done>(64)
-        const write = yield* socket.writer
-
-        // Writer fiber: BOS → drain text stream → EOS. Socket-side failures
-        // end this fiber; the reader still surfaces a clean stream end via
-        // `Queue.end` (in the ensuring below) when the upstream WS closes.
-        yield* Effect.gen(function* () {
-          yield* write(bosFrame(cfg, request))
-          yield* Stream.runForEach(textIn, (text) =>
-            text.length === 0 ? Effect.void : write(textFrame(text)),
-          )
-          yield* write(eosFrame)
-        }).pipe(Effect.ignore, Effect.forkScoped)
-
-        // Reader fiber. `ensuring(Queue.end)` flushes pending chunks then
-        // fails the next take with `Done`, which `Stream.fromQueue` treats
-        // as a clean end. (`Queue.shutdown` would CLEAR queued items and
-        // interrupt pending takes — wrong for graceful teardown.)
-        yield* socket
-          .runString(handleServerFrame(queue))
-          .pipe(Effect.ensuring(Queue.end(queue)), Effect.forkScoped)
-
-        return Stream.fromQueue(queue)
+        const socket = yield* Socket.makeWebSocket(buildWsUrl(cfg, request, slug))
+        const { write } = yield* socket.writer
+        const asAiError = WebSocketSession.toAiError("elevenlabs")
+
+        // ElevenLabs closes `/stream-input` with 1000 after the final audio chunk.
+        const chunks = Stream.fromPull(Socket.readerString(socket)).pipe(
+          Stream.scoped,
+          Stream.catchIf(WebSocketSession.isCleanClose, () => Stream.empty),
+          Stream.mapError(asAiError),
+          Stream.mapEffect(frameToChunk),
+          Stream.filter(Predicate.isNotUndefined),
+        )
+
+        // BOS, then the text, then EOS. Writes wait for the reader to connect.
+        const outgoing = Stream.concat(
+          Stream.make(bosFrame(cfg, request)),
+          Stream.concat(
+            textIn.pipe(
+              Stream.filter((text) => text.length > 0),
+              Stream.map(textFrame),
+            ),
+            Stream.make(eosFrame),
+          ),
+        ).pipe(Stream.mapEffect((frame) => Effect.mapError(write(frame), asAiError)))
+
+        return Stream.merge(chunks, Stream.drain(outgoing), { haltStrategy: "left" })
       }),
     )

package/dist/dist-C0AvqZTC.mjs

@@ -1,145 +0,0 @@
-import { t as __commonJSMin } from "./rolldown-runtime-EAsCxrDo.mjs";
-import "./test.DNmyFkvJ-N3QqMRwZ.mjs";
-//#region ../../../node_modules/.pnpm/vite@7.3.2_@types+node@26.2.0_jiti@2.6.1_lightningcss@1.32.0_tsx@4.23.12_yaml@2.9.0/node_modules/vite/dist/node/module-runner.js
-let SOURCEMAPPING_URL = "sourceMa";
-SOURCEMAPPING_URL += "ppingURL";
-typeof process < "u" && process.platform;
-(async function() {}).constructor;
-var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-var intToChar = /* @__PURE__ */ new Uint8Array(64);
-var charToInt = /* @__PURE__ */ new Uint8Array(128);
-for (let i = 0; i < chars.length; i++) {
-	let c = chars.charCodeAt(i);
-	intToChar[i] = c, charToInt[c] = i;
-}
-`${SOURCEMAPPING_URL}`;
-Error.prepareStackTrace;
-const customizationHookNamespace = "vite-module-runner:import-meta-resolve/v1/";
-`${JSON.stringify(customizationHookNamespace)}${JSON.stringify(customizationHookNamespace)}`;
-new Proxy({}, { get(_, p) {
-	throw Error(`[module runner] Dynamic access of "import.meta.env" is not supported. Please, use "import.meta.env.${String(p)}" instead.`);
-} });
-//#endregion
-//#region ../../../node_modules/.pnpm/expect-type@1.3.0/node_modules/expect-type/dist/branding.js
-var require_branding = /* @__PURE__ */ __commonJSMin(((exports) => {
-	Object.defineProperty(exports, "__esModule", { value: true });
-}));
-//#endregion
-//#region ../../../node_modules/.pnpm/expect-type@1.3.0/node_modules/expect-type/dist/messages.js
-var require_messages = /* @__PURE__ */ __commonJSMin(((exports) => {
-	Object.defineProperty(exports, "__esModule", { value: true });
-}));
-//#endregion
-//#region ../../../node_modules/.pnpm/expect-type@1.3.0/node_modules/expect-type/dist/overloads.js
-var require_overloads = /* @__PURE__ */ __commonJSMin(((exports) => {
-	Object.defineProperty(exports, "__esModule", { value: true });
-}));
-//#endregion
-//#region ../../../node_modules/.pnpm/expect-type@1.3.0/node_modules/expect-type/dist/utils.js
-var require_utils = /* @__PURE__ */ __commonJSMin(((exports) => {
-	Object.defineProperty(exports, "__esModule", { value: true });
-}));
-//#endregion
-//#region ../../../node_modules/.pnpm/vitest@4.1.11_@types+node@26.2.0_vite@7.3.2_@types+node@26.2.0_jiti@2.6.1_lightningcss@1.32.0_tsx@4.23.12_yaml@2.9.0_/node_modules/vitest/dist/index.js
-var import_dist = (/* @__PURE__ */ __commonJSMin(((exports) => {
-	var __createBinding = exports && exports.__createBinding || (Object.create ? (function(o, m, k, k2) {
-		if (k2 === void 0) k2 = k;
-		var desc = Object.getOwnPropertyDescriptor(m, k);
-		if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) desc = {
-			enumerable: true,
-			get: function() {
-				return m[k];
-			}
-		};
-		Object.defineProperty(o, k2, desc);
-	}) : (function(o, m, k, k2) {
-		if (k2 === void 0) k2 = k;
-		o[k2] = m[k];
-	}));
-	var __exportStar = exports && exports.__exportStar || function(m, exports$1) {
-		for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports$1, p)) __createBinding(exports$1, m, p);
-	};
-	Object.defineProperty(exports, "__esModule", { value: true });
-	exports.expectTypeOf = void 0;
-	__exportStar(require_branding(), exports);
-	__exportStar(require_messages(), exports);
-	__exportStar(require_overloads(), exports);
-	__exportStar(require_utils(), exports);
-	const fn = () => true;
-	/**
-	* Similar to Jest's `expect`, but with type-awareness.
-	* Gives you access to a number of type-matchers that let you make assertions about the
-	* form of a reference or generic type parameter.
-	*
-	* @example
-	* ```ts
-	* import { foo, bar } from '../foo'
-	* import { expectTypeOf } from 'expect-type'
-	*
-	* test('foo types', () => {
-	*   // make sure `foo` has type { a: number }
-	*   expectTypeOf(foo).toMatchTypeOf({ a: 1 })
-	*   expectTypeOf(foo).toHaveProperty('a').toBeNumber()
-	*
-	*   // make sure `bar` is a function taking a string:
-	*   expectTypeOf(bar).parameter(0).toBeString()
-	*   expectTypeOf(bar).returns.not.toBeAny()
-	* })
-	* ```
-	*
-	* @description
-	* See the [full docs](https://npmjs.com/package/expect-type#documentation) for lots more examples.
-	*/
-	const expectTypeOf = (_actual) => {
-		const nonFunctionProperties = [
-			"parameters",
-			"returns",
-			"resolves",
-			"not",
-			"items",
-			"constructorParameters",
-			"thisParameter",
-			"instance",
-			"guards",
-			"asserts",
-			"branded"
-		];
-		const obj = {
-			toBeAny: fn,
-			toBeUnknown: fn,
-			toBeNever: fn,
-			toBeFunction: fn,
-			toBeObject: fn,
-			toBeArray: fn,
-			toBeString: fn,
-			toBeNumber: fn,
-			toBeBoolean: fn,
-			toBeVoid: fn,
-			toBeSymbol: fn,
-			toBeNull: fn,
-			toBeUndefined: fn,
-			toBeNullable: fn,
-			toBeBigInt: fn,
-			toMatchTypeOf: fn,
-			toEqualTypeOf: fn,
-			toBeConstructibleWith: fn,
-			toMatchObjectType: fn,
-			toExtend: fn,
-			map: exports.expectTypeOf,
-			toBeCallableWith: exports.expectTypeOf,
-			extract: exports.expectTypeOf,
-			exclude: exports.expectTypeOf,
-			pick: exports.expectTypeOf,
-			omit: exports.expectTypeOf,
-			toHaveProperty: exports.expectTypeOf,
-			parameter: exports.expectTypeOf
-		};
-		nonFunctionProperties.forEach((prop) => Object.defineProperty(obj, prop, { get: () => (0, exports.expectTypeOf)({}) }));
-		return obj;
-	};
-	exports.expectTypeOf = expectTypeOf;
-})))();
-//#endregion
-export { import_dist as t };
-
-//# sourceMappingURL=dist-C0AvqZTC.mjs.map