kugelaudio 0.7.0 → 0.9.0

package/src/livekit/wire.test.ts

@@ -0,0 +1,112 @@
+import { describe, it, expect } from 'vitest';
+import {
+  appendSdkQuery,
+  buildClosePayload,
+  buildMultiWsUrl,
+  buildTextPayload,
+  wordTimestampsToTimed,
+  type WireOptions,
+} from './wire';
+
+const opts: WireOptions = {
+  model: 'kugel-3',
+  voiceId: 1071,
+  sampleRate: 24000,
+  cfgScale: 2.0,
+  maxNewTokens: 2048,
+  wordTimestamps: false,
+  normalize: true,
+};
+
+describe('buildMultiWsUrl', () => {
+  it('rewrites https to wss and targets /ws/tts/multi with the api key', () => {
+    const url = buildMultiWsUrl('https://api.kugelaudio.com', 'secret');
+    expect(url).toContain('wss://api.kugelaudio.com/ws/tts/multi');
+    expect(url).toContain('api_key=secret');
+  });
+
+  it('rewrites http to ws for local dev', () => {
+    expect(buildMultiWsUrl('http://localhost:8000', 'k')).toContain('ws://localhost:8000/ws/tts/multi');
+  });
+
+  it('appends sdk telemetry params', () => {
+    const url = buildMultiWsUrl('https://api.kugelaudio.com', 'k');
+    expect(url).toContain('sdk=js');
+    expect(url).toContain('sdk_version=');
+  });
+});
+
+describe('appendSdkQuery', () => {
+  it('uses ? for the first param and & thereafter', () => {
+    expect(appendSdkQuery('https://x/y')).toContain('?sdk=js');
+    expect(appendSdkQuery('https://x/y?a=b')).toContain('&sdk=js');
+  });
+});
+
+describe('buildTextPayload', () => {
+  it('sends a bare text frame when config is not requested', () => {
+    const msg = buildTextPayload('ctx1', 'hello', opts);
+    expect(msg).toEqual({ text: 'hello', context_id: 'ctx1' });
+  });
+
+  it('marks flush when requested', () => {
+    expect(buildTextPayload('ctx1', 'hi', opts, { flush: true })).toMatchObject({ flush: true });
+  });
+
+  it('attaches session + voice config on the first frame', () => {
+    const msg = buildTextPayload('ctx1', 'hi', opts, { includeConfig: true });
+    expect(msg).toMatchObject({
+      model_id: 'kugel-3',
+      sample_rate: 24000,
+      word_timestamps: false,
+      normalize: true,
+      voice_settings: { voice_id: 1071 },
+    });
+  });
+
+  it('omits default generation params from voice_settings', () => {
+    const msg = buildTextPayload('ctx1', 'hi', opts, { includeConfig: true });
+    expect(msg.voice_settings).toEqual({ voice_id: 1071 });
+  });
+
+  it('includes non-default cfg_scale and max_new_tokens', () => {
+    const msg = buildTextPayload('ctx1', 'hi', { ...opts, cfgScale: 1.5, maxNewTokens: 4096 }, {
+      includeConfig: true,
+    });
+    expect(msg.voice_settings).toMatchObject({ cfg_scale: 1.5, max_new_tokens: 4096 });
+  });
+
+  it('omits voice_settings entirely when the voice is the server default', () => {
+    const msg = buildTextPayload('ctx1', 'hi', { ...opts, voiceId: null }, { includeConfig: true });
+    expect(msg).not.toHaveProperty('voice_settings');
+  });
+
+  it('includes language only when set', () => {
+    expect(buildTextPayload('c', 'x', opts, { includeConfig: true })).not.toHaveProperty('language');
+    const withLang = buildTextPayload('c', 'x', { ...opts, language: 'de' }, { includeConfig: true });
+    expect(withLang).toMatchObject({ language: 'de' });
+  });
+});
+
+describe('buildClosePayload', () => {
+  it('drains by default', () => {
+    expect(buildClosePayload('ctx1')).toEqual({ close_context: true, context_id: 'ctx1' });
+  });
+
+  it('sets immediate for barge-in', () => {
+    expect(buildClosePayload('ctx1', true)).toMatchObject({ immediate: true });
+  });
+});
+
+describe('wordTimestampsToTimed', () => {
+  it('converts server milliseconds to seconds', () => {
+    const timed = wordTimestampsToTimed([
+      { word: 'Hallo', start_ms: 0, end_ms: 500 },
+      { word: 'Welt', start_ms: 500, end_ms: 1200 },
+    ]);
+    expect(timed).toEqual([
+      { text: 'Hallo', startTime: 0, endTime: 0.5 },
+      { text: 'Welt', startTime: 0.5, endTime: 1.2 },
+    ]);
+  });
+});

package/src/livekit/wire.ts

@@ -0,0 +1,126 @@
+/**
+ * Pure wire-protocol helpers for the KugelAudio `/ws/tts/multi` endpoint used
+ * by the LiveKit plugin.
+ *
+ * Kept free of any `@livekit/agents` import so the message-shaping logic can be
+ * unit-tested without the LiveKit runtime. Mirrors the send loop of
+ * `kugelaudio.livekit.tts._Connection` in the Python SDK.
+ */
+
+import packageJson from '../../package.json';
+
+const SDK_NAME = 'js';
+const SDK_VERSION = packageJson.version;
+
+/** Generation parameters carried on the first message of each context. */
+export interface WireOptions {
+  model: string;
+  voiceId: number | null;
+  sampleRate: number;
+  cfgScale: number;
+  maxNewTokens: number;
+  wordTimestamps: boolean;
+  normalize: boolean;
+  language?: string;
+}
+
+/** Append the `sdk` / `sdk_version` query params used for server-side telemetry. */
+export function appendSdkQuery(url: string): string {
+  const separator = url.includes('?') ? '&' : '?';
+  return (
+    `${url}${separator}sdk=${encodeURIComponent(SDK_NAME)}` +
+    `&sdk_version=${encodeURIComponent(SDK_VERSION)}`
+  );
+}
+
+/**
+ * Build the `wss://.../ws/tts/multi` URL from an `https://`/`http://` base URL.
+ * The API key is passed as a query parameter, matching the Python plugin.
+ */
+export function buildMultiWsUrl(baseUrl: string, apiKey: string): string {
+  const wsBase = baseUrl.replace('https://', 'wss://').replace('http://', 'ws://');
+  return appendSdkQuery(`${wsBase}/ws/tts/multi?api_key=${apiKey}`);
+}
+
+/**
+ * Build the JSON payload for a text message to a context.
+ *
+ * When `includeConfig` is true (the first message sent for a context), the
+ * session + voice configuration is attached, exactly as the Python plugin does
+ * on the first frame of a new `context_id`.
+ */
+export function buildTextPayload(
+  contextId: string,
+  text: string,
+  opts: WireOptions,
+  { flush = false, includeConfig = false }: { flush?: boolean; includeConfig?: boolean } = {},
+): Record<string, unknown> {
+  const msg: Record<string, unknown> = { text, context_id: contextId };
+  if (flush) msg.flush = true;
+
+  if (includeConfig) {
+    msg.model_id = opts.model;
+    msg.sample_rate = opts.sampleRate;
+    msg.word_timestamps = opts.wordTimestamps;
+    msg.normalize = opts.normalize;
+    if (opts.language !== undefined) msg.language = opts.language;
+
+    const voiceSettings: Record<string, unknown> = {};
+    if (opts.voiceId !== null && opts.voiceId !== undefined) {
+      voiceSettings.voice_id = opts.voiceId;
+    }
+    // Only send non-default generation params, mirroring the Python plugin.
+    if (opts.cfgScale !== 2.0) voiceSettings.cfg_scale = opts.cfgScale;
+    if (opts.maxNewTokens !== 2048) voiceSettings.max_new_tokens = opts.maxNewTokens;
+    if (Object.keys(voiceSettings).length > 0) msg.voice_settings = voiceSettings;
+  }
+
+  return msg;
+}
+
+/**
+ * Build the JSON payload that closes a context.
+ *
+ * `immediate` cancels in-flight generation (barge-in); omitting it drains any
+ * queued audio first.
+ */
+export function buildClosePayload(
+  contextId: string,
+  immediate = false,
+): Record<string, unknown> {
+  const msg: Record<string, unknown> = {
+    close_context: true,
+    context_id: contextId,
+  };
+  if (immediate) msg.immediate = true;
+  return msg;
+}
+
+/** A single word-timing entry as delivered by the server (`start_ms`/`end_ms`). */
+export interface RawWordTimestamp {
+  word: string;
+  start_ms: number;
+  end_ms: number;
+}
+
+/** A framework-agnostic word timing in seconds. */
+export interface TimedWord {
+  text: string;
+  startTime: number;
+  endTime: number;
+}
+
+/**
+ * Convert the server's `word_timestamps` payload (integer milliseconds) into
+ * seconds-based {@link TimedWord}s, matching the Python plugin's
+ * `_word_timestamps_to_timed`.
+ */
+export function wordTimestampsToTimed(
+  timestamps: RawWordTimestamp[],
+): TimedWord[] {
+  return timestamps.map((ts) => ({
+    text: ts.word,
+    startTime: ts.start_ms / 1000,
+    endTime: ts.end_ms / 1000,
+  }));
+}

package/src/types.ts

@@ -39,6 +39,7 @@ export interface Voice {
   category?: VoiceCategory;
   sex?: VoiceSex;
   age?: VoiceAge;
+  quality?: string;
   supportedLanguages: string[];
   sampleText?: string;
   avatarUrl?: string;
@@ -244,7 +245,7 @@ export interface GenerateOptions {
   modelId?: string;
   /** Voice ID to use */
   voiceId?: number;
-  /** CFG scale for generation (default: 2.0) */
+  /** Classifier-free guidance scale. Clamped to [1.2, 2.5] (default: 2.0). */
   cfgScale?: number;
   /**
    * Sampling variance. Range [0.0, 1.0]. 0 = most stable (near-greedy),
@@ -258,7 +259,13 @@ export interface GenerateOptions {
   maxNewTokens?: number;
   /** Output sample rate (default: 24000) */
   sampleRate?: number;
-  /** 
+  /**
+   * Combined codec+rate token, e.g. 'ulaw_8000' / 'alaw_8000' / 'pcm_8000'.
+   * Opt-in; when set it is authoritative and must not contradict sampleRate.
+   * Absent ⇒ legacy PCM16 at sampleRate.
+   */
+  outputFormat?: string;
+  /**
    * Enable text normalization (converts numbers, dates, etc. to spoken words).
    * When true, text will be normalized before TTS generation.
    * Default: true
@@ -286,8 +293,9 @@ export interface GenerateOptions {
   /**
    * Playback speed multiplier (0.8 = slower, 1.0 = normal, 1.2 = faster).
    *
-   * Uses pitch-preserving time-stretching (WSOLA). Inline `<prosody rate="...">` tags
-   * can also be used for per-segment speed control.
+   * Uses pitch-preserving time-stretching (WSOLA); applies to the whole
+   * request. Wrap text in `<prosody rate="slow|medium|fast|0.8-1.2">` to
+   * override the rate for a span (the span rate wins inside the span).
    * Range: [0.8, 1.2]. Default: 1.0.
    */
   speed?: number;
@@ -298,6 +306,14 @@ export interface GenerateOptions {
    * server treats the value as trusted once received.
    */
   projectId?: number;
+  /**
+   * Per-request dictionary selection. Omit for the default behavior (all
+   * active dictionaries of the project apply, filtered by language). An
+   * empty array disables dictionaries for this request. A list of
+   * dictionary IDs applies exactly those dictionaries — including
+   * inactive ones — bypassing the language filter.
+   */
+  dictionaryIds?: number[];
 }
 
 /**
@@ -322,7 +338,7 @@ export interface StreamConfig {
   voiceId?: number;
   /** Model ID. Default: 'kugel-3'. Legacy ids still accepted; they alias to kugel-3 server-side. */
   modelId?: string;
-  /** CFG scale for generation */
+  /** Classifier-free guidance scale. Clamped to [1.2, 2.5] (default: 2.0). */
   cfgScale?: number;
   /**
    * Sampling variance. Range [0.0, 1.0]. 0 = most stable, 1 = most variance.
@@ -333,6 +349,8 @@ export interface StreamConfig {
   maxNewTokens?: number;
   /** Output sample rate */
   sampleRate?: number;
+  /** Combined codec+rate token (e.g. 'ulaw_8000'); opt-in, set-once per session. */
+  outputFormat?: string;
   /** Auto-flush timeout in milliseconds */
   flushTimeoutMs?: number;
   /** Maximum buffer length */
@@ -377,11 +395,57 @@ export interface StreamConfig {
   /**
    * Playback speed multiplier (0.8 = slower, 1.0 = normal, 1.2 = faster).
    *
-   * Uses pitch-preserving time-stretching (WSOLA). Inline `<prosody rate="...">` tags
-   * can also be used for per-segment speed control.
+   * Uses pitch-preserving time-stretching (WSOLA); applies to the whole
+   * request. Wrap text in `<prosody rate="slow|medium|fast|0.8-1.2">` to
+   * override the rate for a span (the span rate wins inside the span).
    * Range: [0.8, 1.2]. Default: 1.0.
    */
   speed?: number;
+  /**
+   * Per-request dictionary selection. Omit for the default behavior (all
+   * active dictionaries of the project apply, filtered by language). An
+   * empty array disables dictionaries for this request. A list of
+   * dictionary IDs applies exactly those dictionaries — including
+   * inactive ones — bypassing the language filter.
+   */
+  dictionaryIds?: number[];
+}
+
+/**
+ * Generation parameters changeable mid-connection via
+ * {@link StreamingSession.updateSettings} / {@link MultiContextSession.updateSettings}
+ * (KUG-1166). Every field is optional; an update changes only the fields it
+ * carries. Identity / audio-format fields (`voiceId`, `modelId`, `sampleRate`,
+ * `outputFormat`, `dictionaryIds`) are NOT here — they are fixed for the
+ * connection's lifetime and the server rejects them in an update.
+ */
+export interface SettingsUpdate {
+  /** Classifier-free guidance scale (0.0–10.0). */
+  cfgScale?: number;
+  /** Sampling variance (0.0–1.0). */
+  temperature?: number;
+  /** Playback speed multiplier (0.8–1.2). */
+  speed?: number;
+  /** Maximum tokens per generation (1–8192). */
+  maxNewTokens?: number;
+  /** Language code for normalization (e.g. `'de'`). */
+  language?: string;
+  /** Enable text normalization. */
+  normalize?: boolean;
+}
+
+/**
+ * The generation parameters in effect after an
+ * {@link StreamingSession.updateSettings} call — the server's echo. Fields the
+ * session never set come back `null` (e.g. `temperature`, `language`).
+ */
+export interface EffectiveSettings {
+  cfgScale?: number;
+  temperature?: number | null;
+  speed?: number;
+  maxNewTokens?: number;
+  language?: string | null;
+  normalize?: boolean;
 }
 
 /**
@@ -399,9 +463,18 @@ export interface StreamingSessionCallbacks {
    * Carries the segment index, total audio duration, and generation time.
    */
   onChunkComplete?: (chunkId: number, audioSeconds: number, genMs: number) => void;
+  /**
+   * Called when the server marks the end of a turn's audio
+   * (`{"final": true, ...}` — sent after the last audio frame of every
+   * gracefully completed turn, right before `session_closed`). The
+   * ElevenLabs `isFinal` equivalent: once this fires, no further audio
+   * for the turn will arrive. Not fired on a barge-in cancel — that
+   * path fires {@link onInterrupted} instead.
+   */
+  onFinal?: (totalAudioSeconds: number, totalTextChunks: number, totalAudioChunks: number) => void;
   /**
    * Called when the session is fully closed (after `session.close()`).
-   * Equivalent to `onFinal` on the one-shot endpoint.
+   * Fires right after {@link onFinal} and additionally carries usage.
    */
   onSessionClosed?: (totalAudioSeconds: number, totalTextChunks: number, totalAudioChunks: number) => void;
   /** Called when the server begins generating audio for a text segment. */
@@ -419,14 +492,71 @@ export interface StreamingSessionCallbacks {
   onError?: (error: Error) => void;
 }
 
+/**
+ * Per-session usage reported in the `session_closed` frame (KUG-1192).
+ *
+ * Lets you bill your own customers per conversation. `costCents` is the
+ * actual amount charged in **EUR cents**. When the charge could not be
+ * determined at session end (e.g. a transient billing error) `costCents` is
+ * `null` and `costAvailable` is `false` — never a misleading `0`.
+ * `audioSeconds` is always reported. On `/ws/tts/multi` usage is reported per
+ * context (per conversation) on each `context_closed` frame, not aggregated
+ * across contexts.
+ */
+export interface SessionUsage {
+  /** Total audio generated this session, in seconds (the unit we bill on). */
+  audioSeconds: number;
+  /** Actual amount charged in EUR cents, or `null` if undetermined. */
+  costCents: number | null;
+  /** Currency of `costCents` (`"eur"`); present only when `costCents` is set. */
+  currency?: string;
+  /** Total input characters submitted this session, if reported. */
+  characters?: number;
+  /** Model that produced the audio, if reported. */
+  modelId?: string;
+  /** `true` when an authoritative charge was returned for this session. */
+  costAvailable: boolean;
+}
+
+/**
+ * Parse the raw `usage` object (or a legacy `session_closed` payload without
+ * one) into a typed {@link SessionUsage}. Returns `null` when no usage info
+ * is present.
+ */
+export function parseSessionUsage(
+  data: Record<string, unknown>,
+): SessionUsage | null {
+  const raw = data.usage as Record<string, unknown> | undefined;
+  const source = raw && typeof raw === 'object' ? raw : data;
+  const audioSeconds =
+    typeof source.audio_seconds === 'number'
+      ? source.audio_seconds
+      : typeof data.total_audio_seconds === 'number'
+        ? data.total_audio_seconds
+        : undefined;
+  if (audioSeconds === undefined) return null;
+  const costCents =
+    typeof source.cost_cents === 'number' ? source.cost_cents : null;
+  return {
+    audioSeconds,
+    costCents,
+    currency:
+      typeof source.currency === 'string' ? source.currency : undefined,
+    characters:
+      typeof source.characters === 'number' ? source.characters : undefined,
+    modelId: typeof source.model_id === 'string' ? source.model_id : undefined,
+    costAvailable: costCents !== null,
+  };
+}
+
 /**
  * Audio chunk from streaming TTS.
  */
 export interface AudioChunk {
   /** Raw PCM16 audio as base64 */
   audio: string;
-  /** Encoding format */
-  encoding: 'pcm_s16le';
+  /** Encoding format. 'mulaw' / 'alaw' only when output_format requested G.711. */
+  encoding: 'pcm_s16le' | 'mulaw' | 'alaw';
   /** Chunk index */
   index: number;
   /** Sample rate */
@@ -453,6 +583,12 @@ export interface GenerationStats {
   rtf: number;
   /** Error message if any */
   error?: string;
+  /**
+   * Per-request usage (audio time + amount charged), for billing your own
+   * customers. Undefined when the server reports no usage. See
+   * {@link SessionUsage}.
+   */
+  usage?: SessionUsage;
 }
 
 /**
@@ -546,7 +682,9 @@ export interface MultiContextConfig {
   defaultVoiceId?: number;
   /** Output sample rate (default: 24000) */
   sampleRate?: number;
-  /** CFG scale for generation (default: 2.0) */
+  /** Combined codec+rate token (e.g. 'ulaw_8000'); opt-in, set-once per context. */
+  outputFormat?: string;
+  /** Classifier-free guidance scale. Clamped to [1.2, 2.5] (default: 2.0). */
   cfgScale?: number;
   /**
    * Sampling variance. Range [0.0, 1.0]. 0 = most stable, 1 = most variance.
@@ -563,6 +701,14 @@ export interface MultiContextConfig {
    * the language, which adds ~60-150ms to time-to-first-audio.
    */
   language?: string;
+  /**
+   * Per-request dictionary selection. Omit for the default behavior (all
+   * active dictionaries of the project apply, filtered by language). An
+   * empty array disables dictionaries for this request. A list of
+   * dictionary IDs applies exactly those dictionaries — including
+   * inactive ones — bypassing the language filter.
+   */
+  dictionaryIds?: number[];
   /** Seconds before context auto-closes (default: 20.0) */
   inactivityTimeout?: number;
 }
@@ -601,8 +747,20 @@ export interface MultiContextCallbacks {
   onContextCreated?: (contextId: string) => void;
   /** Called when an audio chunk is received */
   onChunk?: (chunk: MultiContextAudioChunk) => void;
-  /** Called when a context is closed */
-  onContextClosed?: (contextId: string) => void;
+  /**
+   * Called when all audio admitted before a `{flush: true}` has been
+   * delivered for a context (`{"final": true, "context_id": ...}`), and
+   * once more before {@link onContextClosed} on a graceful close. The
+   * ElevenLabs multi-context `is_final` equivalent. Not fired on an
+   * immediate (barge-in) close.
+   */
+  onFinal?: (contextId: string) => void;
+  /**
+   * Called when a context is closed (terminal). `usage` carries this
+   * conversation's audio time + amount charged (undefined if not reported).
+   * See {@link SessionUsage}.
+   */
+  onContextClosed?: (contextId: string, usage?: SessionUsage) => void;
   /** Called when a context times out */
   onContextTimeout?: (contextId: string) => void;
   /** Called when session is closed */

package/src/utils.ts

@@ -2,6 +2,22 @@
  * Utility functions for KugelAudio SDK.
  */
 
+/**
+ * Accepted classifier-free guidance band. Values outside [MIN, MAX] are
+ * clamped into the band (both client-side and by the server).
+ */
+export const MIN_CFG_SCALE = 1.2;
+export const MAX_CFG_SCALE = 2.5;
+
+/**
+ * Clamp `cfgScale` into [1.2, 2.5]. `undefined` passes through unchanged
+ * (the server default is applied).
+ */
+export function clampCfgScale(cfgScale: number | undefined): number | undefined {
+  if (cfgScale === undefined) return undefined;
+  return Math.min(MAX_CFG_SCALE, Math.max(MIN_CFG_SCALE, cfgScale));
+}
+
 /**
  * Decode base64 string to ArrayBuffer.
  */