@flamai/finesse 0.1.1 → 0.2.0

package/CHANGELOG.md

@@ -1,6 +1,23 @@
 # Changelog
 
-## 0.1.1 (unreleased)
+## 0.2.0
+
+- Direct realtime sessions: `realtime.sessions.create({ voice_ids, budget_chars })` and
+  `realtime.connect({ session })` / `{ getSession }` run the socket straight against the
+  synthesis GPU in the account's server region, for lower latency. `RealtimeSession.directSession`
+  reports its region and budget.
+- Realtime: on a direct session, close code `4401` is `RealtimeError` with
+  `code === "unauthorized"` (the token was invalid, already used or past `connect_by`), not
+  `service_auth_failed`. Reusing a spent session throws `session_spent`; one past `connect_by`
+  throws `session_expired` without dialing.
+
+## 0.1.2
+
+- `output_format: "mulaw"` (raw G.711 µ-law) for telephony, with `sample_rate: 8000`, on
+  `speech` and realtime contexts. 8 kHz is now produced as asked; the README shows Twilio Media
+  Streams.
+
+## 0.1.1
 
 - Published on npm as `@flamai/finesse` (was private, installed from GitHub).
 - Token clients (`accessToken`) retry only `429` and connections that never opened. The

package/README.md

@@ -5,10 +5,12 @@
 </picture>
 
 <p>
+  <a href="https://www.npmjs.com/package/@flamai/finesse"><img alt="npm version" src="https://img.shields.io/npm/v/@flamai/finesse?style=flat&labelColor=14151A&color=2547F0"></a>
   <img alt="Node 18+" src="https://img.shields.io/badge/node-18%2B-14151A?style=flat&labelColor=14151A&color=2547F0">
   <img alt="Bun and Deno" src="https://img.shields.io/badge/also-Bun%20%C2%B7%20Deno-14151A?style=flat&labelColor=14151A&color=2547F0">
   <img alt="TypeScript types included" src="https://img.shields.io/badge/types-included-14151A?style=flat&labelColor=14151A&color=2547F0">
   <img alt="Status: beta" src="https://img.shields.io/badge/status-beta-14151A?style=flat&labelColor=14151A&color=63656A">
+  <a href="./LICENSE"><img alt="License: Apache-2.0" src="https://img.shields.io/badge/license-Apache--2.0-14151A?style=flat&labelColor=14151A&color=63656A"></a>
 </p>
 
 # Finesse TypeScript SDK
@@ -37,6 +39,7 @@ and stream it in realtime to voice agents, with a few lines of code.
 - [Text to speech](#text-to-speech)
 - [Streaming speech](#streaming-speech)
 - [Realtime](#realtime)
+- [Direct sessions (lowest latency)](#direct-sessions-lowest-latency)
 - [Browsers and short-lived tokens](#browsers-and-short-lived-tokens)
 - [Which method should I use?](#which-method-should-i-use)
 - [Audio formats](#audio-formats)
@@ -47,6 +50,11 @@ and stream it in realtime to voice agents, with a few lines of code.
 - [Request IDs](#request-ids)
 - [Advanced](#advanced)
 - [Runtimes](#runtimes)
+- [Requirements](#requirements)
+- [Versioning](#versioning)
+- [Getting help](#getting-help)
+- [Security](#security)
+- [Contributing](#contributing)
 
 ## Installation
 
@@ -56,8 +64,14 @@ Realtime in Node also needs the `ws` package:
 npm install @flamai/finesse ws
 # or
 pnpm add @flamai/finesse ws
+# or
+yarn add @flamai/finesse ws
+# or
+bun add @flamai/finesse
 ```
 
+`ws` is only needed for realtime in Node. Bun, Deno and browsers have a built-in `WebSocket`.
+
 ## Get an API key
 
 1. Sign in to the [Finesse dashboard](https://finesse.flamapp.ai/dashboard).
@@ -181,7 +195,7 @@ const audio = await client.speech.create({
   voice_id: voice.id,
   text: "The quick brown fox jumps over the lazy dog.",
   language: "en", // optional; defaults to the voice's language
-  output_format: "wav", // default; or raw PCM: "s16le", "s24le", "s32le", "f32le", "f64le"
+  output_format: "wav", // default; or raw PCM: "s16le", "s24le", "s32le", "f32le", "f64le", or "mulaw"
   num_step: 32, // optional; 1-32, default 32; fewer steps is faster
 });
 
@@ -284,6 +298,52 @@ A turn that fails throws `RealtimeError` (for example `code === "voice_not_found
 - Text is billed per character as it is sent. A session reserves credits in blocks of up to
   1,000 characters and returns what it doesn't use when it ends.
 
+## Direct sessions (lowest latency)
+
+A direct session runs the realtime socket **straight against the synthesis GPU** in your
+account's server region, so audio never passes through the API and each sentence starts sooner.
+Contexts, events, cancel and limits work exactly as above.
+
+```ts
+// On a server, with your API key: a new session for the first connection and every reconnect.
+const rt = await client.realtime.connect({
+  getSession: () => client.realtime.sessions.create({ voice_ids: [voiceId] }),
+});
+for await (const event of rt.say({ voice_id: voiceId, text: "Hello, straight from the GPU." })) {
+  if (event.type === "audio") play(event.pcm);
+}
+console.log(rt.directSession?.region); // "ap-south" or "us-east"
+await rt.close();
+```
+
+In a browser, create the session on your server and return it (it holds no API key), then
+connect with it:
+
+```ts
+// On your server
+const session = await client.realtime.sessions.create({ voice_ids: [voiceId] });
+return Response.json(session);
+
+// In the browser
+import { connectDirect } from "@flamai/finesse";
+
+const getSession = async () => (await fetch("/api/finesse-session", { method: "POST" })).json();
+const rt = await connectDirect({ getSession });
+```
+
+How it differs from a session through the API:
+
+- **Voices:** only the `voice_ids` you list (1-16). A context for any other voice fails with
+  `voice_not_found`; the session keeps running.
+- **Credits:** `budget_chars` (default 20,000) is reserved when the session is created and the
+  rest is returned when it ends. Going past it ends the session with `insufficient_credits`.
+- **Lifetime:** connect within 60 s of creating it (`connect_by`), and a session runs at most an
+  hour (`ttl_seconds`), then ends with `session_ended`.
+- **Single use:** the session's token works for one connection. With `{ session }` alone, a
+  reconnect throws `RealtimeError` with `code === "session_spent"`; pass `getSession` so each
+  reconnect gets a new session. A refused token throws `code === "unauthorized"`.
+- **Region:** set by the account that owns the API key, not per request.
+
 ## Browsers and short-lived tokens
 
 Never ship an API key to a browser. Your server mints a short-lived, single-use token with a key
@@ -331,11 +391,26 @@ outside `voice_ids` throws `PermissionDeniedError` with `code === "voice_not_all
 | `s16le`          | Raw PCM, 16-bit signed little-endian | `speech.stream`, realtime |
 | `s24le`, `s32le` | Raw PCM, 24- or 32-bit signed        |                           |
 | `f32le`, `f64le` | Raw PCM, 32- or 64-bit float         |                           |
+| `mulaw`          | Raw G.711 µ-law, for telephony       | needs `sample_rate: 8000` |
 | `opus`           | Opus                                 | realtime only             |
 
-All audio is mono. The engine's native rate is 24 kHz. You can ask for a higher `sample_rate`,
-but lower ones still come back at 24 kHz, so always read the actual rate from
-`audio.sampleRate`, `stream.sampleRate` or the `task_started` event.
+All audio is mono. The engine's native rate is 24 kHz. You can ask for a higher `sample_rate`, or
+for 8000 (telephony); other lower rates still come back at 24 kHz, so always read the actual rate
+from `audio.sampleRate`, `stream.sampleRate` or the `task_started` event.
+
+### Phone calls (Twilio)
+
+Twilio Media Streams play 8 kHz µ-law. Ask for exactly that and send each chunk as the media
+payload, base64-encoded; there is no WAV header to strip and nothing to resample:
+
+```ts
+const ctx = rt.context({ voice_id: voiceId, output_format: "mulaw", sample_rate: 8000 });
+
+for await (const audio of ctx.audio()) {
+  const payload = Buffer.from(audio).toString("base64");
+  twilioWs.send(JSON.stringify({ event: "media", streamSid, media: { payload } }));
+}
+```
 
 ## Handling errors
 
@@ -486,22 +561,12 @@ const quick = client.withOptions({ timeout: 10_000, maxRetries: 0 });
 await quick.voices.get(voiceId);
 ```
 
-### Environments
-
-The client talks to production by default. Point it elsewhere with `baseURL` or
-`FINESSE_BASE_URL`:
-
-```ts
-const client = new Finesse({ baseURL: "https://finesse-proxy.example.com" });
-```
-
 ### Configuration
 
 | Option                    | Environment variable | Default                               |
 | ------------------------- | -------------------- | ------------------------------------- |
 | `apiKey`                  | `FINESSE_API_KEY`    | required, unless `accessToken`        |
 | `accessToken`             |                      | none (a short-lived `fin_tok_` token) |
-| `baseURL`                 | `FINESSE_BASE_URL`   | `https://fi.production.flamapis.com`  |
 | `timeout`                 |                      | 120 000 ms                            |
 | `maxRetries`              |                      | 2                                     |
 | `defaultHeaders`          |                      | none                                  |
@@ -510,6 +575,8 @@ const client = new Finesse({ baseURL: "https://finesse-proxy.example.com" });
 | `logLevel`                | `FINESSE_LOG`        | `"off"`                               |
 | `logger`                  |                      | `console`                             |
 
+An option passed to `new Finesse({...})` always wins over its environment variable.
+
 ### Custom fetch
 
 Pass your own `fetch` for proxies, logging or tests:
@@ -542,11 +609,28 @@ Browsers can't keep an API key secret, so a client holding a key refuses to run
 deprecated URL parameter, because browsers can't set WebSocket headers (as does Deno before
 2.5), and logs a warning once.
 
+## TypeScript types
+
+Every request and response type is exported, so you can type your own code against the API:
+
+```ts
+import type { Voice } from "@flamai/finesse";
+
+function label(voice: Voice) {
+  return `${voice.name} (${voice.id})`;
+}
+```
+
 ## Examples
 
 Runnable scripts are in [`examples/`](./examples): [`speak.ts`](./examples/speak.ts) and
 [`realtime.ts`](./examples/realtime.ts).
 
+## Requirements
+
+- Node 18 or later, Bun, Deno, or a modern browser (see [Runtimes](#runtimes)).
+- TypeScript 4.7 or later, if you use TypeScript.
+
 ## Versioning
 
 The SDK follows [semantic versioning](https://semver.org). Before 1.0, minor versions may
@@ -556,6 +640,18 @@ include breaking changes; they are listed in the [changelog](./CHANGELOG.md).
 import { VERSION } from "@flamai/finesse";
 ```
 
+## Getting help
+
+- [Documentation](https://docs.finesse.flamapp.ai) for guides and the API reference.
+- [GitHub issues](https://github.com/homingos/finesse-node/issues) for bugs and feature requests
+  in this SDK. Include the `requestId` from the error (see [Request IDs](#request-ids)) when
+  a request failed.
+
+## Security
+
+Please don't report security problems in a public issue. See [SECURITY.md](./SECURITY.md) for
+how to report one privately.
+
 ## Contributing
 
 ```sh

package/dist/index.cjs

@@ -55,6 +55,7 @@ __export(index_exports, {
   VERSION: () => VERSION,
   VoicePage: () => VoicePage,
   VoicePagePromise: () => VoicePagePromise,
+  connectDirect: () => connectDirect,
   fileFromPath: () => fileFromPath,
   isReady: () => isReady
 });
@@ -225,7 +226,7 @@ function numberField(body2, key) {
 }
 
 // src/version.ts
-var VERSION = "0.1.1";
+var VERSION = "0.2.0";
 
 // src/core.ts
 var DEFAULT_BASE_URL = "https://fi.production.flamapis.com";
@@ -588,7 +589,8 @@ var RealtimeError = class extends FinesseError {
    * voices), `invalid_request` (a parameter out of range), a `task_failed` code. Whole session:
    * `insufficient_credits`, `upstream_unavailable` (the engine could not be reached),
    * `upstream_busy` (it is at capacity), `upstream_error`, `upstream_closed`,
-   * `connection_lost`, `token_spent`, ... */
+   * `connection_lost`, `token_spent`, `session_spent`, `session_expired`, `unauthorized` (a
+   * direct session's token was refused), ... */
   code;
   /** The context it applies to, or `null` for the whole session. */
   contextId;
@@ -605,8 +607,15 @@ var RealtimeError = class extends FinesseError {
     return RETRYABLE_CODES.has(this.code) || this.closeCode !== null && RETRYABLE_CLOSES.has(this.closeCode);
   }
 };
-function closeError(code, reason) {
+function closeError(code, reason, direct = false) {
   const c = { closeCode: code };
+  if (direct && code === 4401) {
+    return new RealtimeError(
+      "unauthorized",
+      "The session token was refused (invalid, already used, or past connect_by). Create a new session.",
+      c
+    );
+  }
   if (code === 1e3) return null;
   if (code === 1008) return new RealtimeError(reason || "policy_violation", reason, c);
   if (code === 1009) return new RealtimeError("message_too_big", "A message was too large.", c);
@@ -822,7 +831,13 @@ var RealtimeSession = class {
   constructor(core, options = {}) {
     this.core = core;
     this.options = options;
-    this.staticToken = options.accessToken ?? core.accessToken;
+    this.directMode = Boolean(options.session || options.getSession);
+    if (this.directMode && (options.accessToken || options.getAccessToken)) {
+      throw new FinesseError(
+        "Pass { session } / { getSession } or { accessToken } / { getAccessToken }, not both."
+      );
+    }
+    this.staticToken = this.directMode ? null : options.accessToken ?? core.accessToken;
   }
   core;
   options;
@@ -837,9 +852,19 @@ var RealtimeSession = class {
   closedWaiters = [];
   staticToken;
   tokenSpent = false;
+  directMode;
+  staticSessionSpent = false;
+  current = null;
+  /** Where the socket connects: the API's `/stream`, or a direct session's GPU URL. */
   get url() {
+    if (this.directMode) return this.current?.url ?? this.options.session?.url ?? "";
     return `${this.core.wsBaseURL}${API_PREFIX}/stream`;
   }
+  /** The direct session the socket runs on (its region and budget), or `null` when the session
+   * goes through the API. */
+  get directSession() {
+    return this.current;
+  }
   get isOpen() {
     return this.ws !== null && this.ws.readyState === OPEN;
   }
@@ -973,6 +998,31 @@ var RealtimeSession = class {
     }
     return null;
   }
+  /** The direct session for the next handshake. Like a token, a static session is used first;
+   * once a handshake has spent it, `getSession` supplies one per reconnect. */
+  async nextSession() {
+    const fixed = this.options.session;
+    if (fixed && !this.staticSessionSpent) {
+      if (Date.parse(fixed.connect_by) <= Date.now()) {
+        throw new RealtimeError(
+          "session_expired",
+          "This direct session's connect_by has passed. Create a new session."
+        );
+      }
+      return fixed;
+    }
+    if (this.options.getSession) {
+      const s = await this.options.getSession();
+      if (!s || typeof s.url !== "string" || !s.url || typeof s.token !== "string" || !s.token) {
+        throw new FinesseError("getSession must return a session with a url and a token.");
+      }
+      return s;
+    }
+    throw new RealtimeError(
+      "session_spent",
+      "This direct session was used by an earlier connection. Pass { getSession } so a reconnect can create a new one."
+    );
+  }
   async dial() {
     const timeout = this.options.openTimeout ?? DEFAULT_OPEN_TIMEOUT_MS;
     const custom = this.options.WebSocket;
@@ -980,11 +1030,21 @@ var RealtimeSession = class {
       headers: !isBrowser(),
       open: (url2, h, t, protocols2) => h ? new custom(url2, protocols2, { headers: h, handshakeTimeout: t }) : new custom(url2, protocols2)
     } : await defaultDialer();
-    const token = await this.nextToken();
-    const protocols = token ? [SUBPROTOCOL, token] : void 0;
+    const direct = this.directMode ? await this.nextSession() : null;
+    const token = direct ? null : await this.nextToken();
+    let protocols = token ? [SUBPROTOCOL, token] : void 0;
     let url = this.url;
     let headers = null;
-    if (token) {
+    if (direct) {
+      protocols = [direct.protocol || SUBPROTOCOL, direct.token];
+      url = direct.url;
+      if (dialer.headers) {
+        headers = { ...this.core.headers };
+        delete headers.Accept;
+        delete headers["X-API-Key"];
+        delete headers.Authorization;
+      }
+    } else if (token) {
       if (dialer.headers) {
         headers = { ...this.core.headers };
         delete headers.Accept;
@@ -1043,6 +1103,10 @@ var RealtimeSession = class {
       throw new RealtimeError("session_closed", "This session was closed.");
     }
     if (token && token === this.staticToken) this.tokenSpent = true;
+    if (direct) {
+      if (direct === this.options.session) this.staticSessionSpent = true;
+      this.current = direct;
+    }
     this.ws = ws;
     this.pendingAudio = null;
     this.fatal = null;
@@ -1061,14 +1125,14 @@ var RealtimeSession = class {
   }
   onClose(ws, code, reason) {
     if (this.ws !== null && this.ws !== ws) {
-      const err = closeError(code, reason ?? "") ?? new RealtimeError("connection_closed");
+      const err = closeError(code, reason ?? "", this.directMode) ?? new RealtimeError("connection_closed");
       for (const ctx of [...this.contexts.values()]) if (ctx.socket === ws) ctx._fail(err);
       return;
     }
     this.ws = null;
     let error = this.fatal;
     if (error) error.closeCode ??= code;
-    else error = closeError(code, reason ?? "");
+    else error = closeError(code, reason ?? "", this.directMode);
     if (!error && this.contexts.size) {
       error = new RealtimeError("connection_closed", "The session ended.", { closeCode: code });
     }
@@ -1135,11 +1199,35 @@ var RealtimeSession = class {
     );
   }
 };
+var DirectSessions = class {
+  constructor(core) {
+    this.core = core;
+  }
+  core;
+  /** Create a direct session. Call this on your server with an API key that has the `tts`
+   * scope; it reserves `budget_chars` from your balance now. Connect within 60 s
+   * (`connect_by`), with `realtime.connect({ session })`, or hand `url`, `protocol` and `token`
+   * to a browser:
+   *
+   * ```ts
+   * const session = await client.realtime.sessions.create({ voice_ids: [voiceId] });
+   * const rt = await client.realtime.connect({ session });
+   * ```
+   */
+  async create(params, options = {}) {
+    const json = { voice_ids: params.voice_ids };
+    if (params.budget_chars !== void 0) json.budget_chars = params.budget_chars;
+    return this.core.json("POST", "/realtime/sessions", { ...options, json });
+  }
+};
 var Realtime = class {
   constructor(core) {
     this.core = core;
+    this.sessions = new DirectSessions(core);
   }
   core;
+  /** Direct sessions: lower latency, straight to the GPU. */
+  sessions;
   /** Open a session. */
   async connect(options = {}) {
     const session = new RealtimeSession(this.core, options);
@@ -1151,6 +1239,15 @@ var Realtime = class {
     return new RealtimeSession(this.core, options);
   }
 };
+async function connectDirect(options) {
+  if (!options.session && !options.getSession) {
+    throw new FinesseError("connectDirect needs { session } or { getSession }.");
+  }
+  const core = new Core({ accessToken: "direct-session-only" });
+  const session = new RealtimeSession(core, options);
+  await session.open();
+  return session;
+}
 var warnedLoggers = /* @__PURE__ */ new WeakSet();
 function warnQueryKey(logger) {
   if (warnedLoggers.has(logger)) return;
@@ -1779,6 +1876,7 @@ var Finesse = class _Finesse {
   VERSION,
   VoicePage,
   VoicePagePromise,
+  connectDirect,
   fileFromPath,
   isReady
 });