@alia.onl/sdk 7.2.8 → 8.0.0

package/src/lib/speech-synthesis.ts

@@ -0,0 +1,122 @@
+/**
+ * Speech out: ask `POST /v1/audio/speech` for a clip, and play one.
+ *
+ * Shared by read-aloud (`useTTS`) and voice mode (`useVoiceRoom`) so both speak
+ * through the one route Alia has for it — Alia → Oxy → Kaana, billed by Oxy per
+ * call — with the same request body and the same error copy. Voice names are
+ * the PRODUCT's (`male` / `female`); the route maps them, and no upstream voice
+ * name ever appears on this side.
+ */
+
+import { createAudioLevelMeter } from './audio-level';
+
+export type ProductVoice = 'male' | 'female';
+
+export interface SpeechClipRequest {
+  readonly apiUrl: string;
+  readonly token: string;
+  /** The speech routing profile; the route accepts only its own. */
+  readonly model: string;
+  readonly input: string;
+  readonly voice: ProductVoice;
+  readonly speed?: number;
+  /** Both or neither: the route stores the clip on that message when given. */
+  readonly conversationId?: string;
+  readonly messageId?: string;
+  readonly signal: AbortSignal;
+}
+
+/** Synthesize `input` and return the private link the route answers with. */
+export async function requestSpeechClip(request: SpeechClipRequest): Promise<string> {
+  const response = await fetch(`${request.apiUrl}/v1/audio/speech`, {
+    method: 'POST',
+    signal: request.signal,
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${request.token}`,
+    },
+    body: JSON.stringify({
+      model: request.model,
+      input: request.input,
+      voice: request.voice,
+      speed: request.speed,
+      conversationId: request.conversationId,
+      messageId: request.messageId,
+    }),
+  });
+
+  if (!response.ok) {
+    if (response.status === 504) {
+      throw new Error('Request timed out — please try again');
+    }
+    const errData = await response.json().catch(() => ({}));
+    throw new Error(errData.error?.message || errData.error || 'TTS failed');
+  }
+
+  const data = (await response.json()) as { audioUrl?: unknown };
+  request.signal.throwIfAborted();
+  if (typeof data.audioUrl !== 'string' || data.audioUrl === '') throw new Error('TTS failed');
+  return data.audioUrl;
+}
+
+export interface PlayClipOptions {
+  readonly signal: AbortSignal;
+  /** The clip's live level, 0..1, where the platform lets playback be sampled. */
+  readonly onLevel?: (level: number) => void;
+}
+
+/**
+ * Play one clip to its end.
+ *
+ * Resolves when it finishes or when `signal` aborts (the player is released
+ * either way — an interrupted answer stops at once), and rejects if the player
+ * reports an error. `crossOrigin` and sampling are set as `useTTS` sets them;
+ * its comments explain why.
+ */
+export async function playSpeechClip(uri: string, options: PlayClipOptions): Promise<void> {
+  const { signal, onLevel } = options;
+  if (signal.aborted) return;
+  const { createAudioPlayer } = await import('expo-audio');
+  if (signal.aborted) return;
+
+  const player = createAudioPlayer({ uri }, { crossOrigin: 'anonymous' });
+  const release = (): void => {
+    try {
+      player.remove();
+    } catch {
+      // Already removed.
+    }
+  };
+
+  try {
+    await new Promise<void>((resolve, reject) => {
+      const onAbort = (): void => resolve();
+      signal.addEventListener('abort', onAbort, { once: true });
+      const settle = (settleWith: () => void): void => {
+        signal.removeEventListener('abort', onAbort);
+        settleWith();
+      };
+
+      if (onLevel !== undefined) {
+        player.setAudioSamplingEnabled(true);
+        const meter = createAudioLevelMeter();
+        player.addListener('audioSampleUpdate', (sample) => {
+          const frames = sample.channels[0]?.frames;
+          if (frames === undefined) return;
+          onLevel(meter.push(frames, Date.now()));
+        });
+      }
+      player.addListener('playbackStatusUpdate', (status) => {
+        if (status.error) {
+          settle(() => reject(new Error(String(status.error))));
+          return;
+        }
+        if (status.didJustFinish) settle(resolve);
+      });
+      player.play();
+    });
+  } finally {
+    release();
+    onLevel?.(0);
+  }
+}

package/src/lib/voice-levels.ts

@@ -0,0 +1,60 @@
+import { clampLevel } from './speech-recognition-types';
+
+/**
+ * The two live levels of a voice call — what the microphone hears and what the
+ * call is saying — for whatever draws them.
+ *
+ * It replaced the LiveKit `Room` that `useAudioLevelMonitor` used to take and
+ * put analysers on. The voice loop writes here from the recognizer's level
+ * events and the clip player's samples; the monitor subscribes and hands React
+ * a throttled copy. Writes are cheap and never render anything by themselves.
+ */
+export interface VoiceLevelSource {
+  readonly capture: number;
+  readonly playback: number;
+  subscribe(listener: () => void): () => void;
+}
+
+export class VoiceLevelChannel implements VoiceLevelSource {
+  private captureLevel = 0;
+  private playbackLevel = 0;
+  private readonly listeners = new Set<() => void>();
+
+  get capture(): number {
+    return this.captureLevel;
+  }
+
+  get playback(): number {
+    return this.playbackLevel;
+  }
+
+  setCapture(level: number): void {
+    const next = clampLevel(level);
+    if (next === this.captureLevel) return;
+    this.captureLevel = next;
+    this.emit();
+  }
+
+  setPlayback(level: number): void {
+    const next = clampLevel(level);
+    if (next === this.playbackLevel) return;
+    this.playbackLevel = next;
+    this.emit();
+  }
+
+  reset(): void {
+    this.setCapture(0);
+    this.setPlayback(0);
+  }
+
+  subscribe(listener: () => void): () => void {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  }
+
+  private emit(): void {
+    for (const listener of this.listeners) listener();
+  }
+}

package/src/lib/voice-text.ts

@@ -0,0 +1,192 @@
+/**
+ * Text in and out of a voice call, as pure functions: what of an answer is
+ * ready to be spoken, how it is made speakable, and whether what the
+ * microphone just heard was the person or the call's own voice.
+ *
+ * Imports nothing, so every rule here is asserted directly
+ * (`tests/lib/voice-text.test.ts`) rather than through a hook.
+ */
+
+// ============== TITLE TAGS ==============
+
+const TAG = String.raw`ALIA_TITLE|TITLE|TÍTULO|TITRE|TITOLO|TITEL|ЗАГОЛОВОК`;
+
+const TITLE_STRIP_RE = new RegExp(
+  String.raw`\[(${TAG})\].*?\[\/\1\]|<(${TAG})>.*?<\/\2>`, 'gi',
+);
+
+const TITLE_PARTIAL_RE = new RegExp(
+  String.raw`\[(${TAG})\].*?(\[\/\1\])?$|<(${TAG})>.*?(<\/\3>)?$`, 'si',
+);
+
+/**
+ * A settled answer without the conversation-title tag a model may append.
+ *
+ * `trim: false` keeps surrounding whitespace, which the speech chunker needs:
+ * a trailing space is how it knows a period ended a sentence.
+ */
+export function stripTitleTags(content: string, { trim = true }: { trim?: boolean } = {}): string {
+  const stripped = content.replace(TITLE_STRIP_RE, '');
+  return trim ? stripped.trim() : stripped;
+}
+
+/** The same, for an answer still streaming: an opened tag hides the rest. */
+export function stripTitleTagsPartial(content: string, { trim = true }: { trim?: boolean } = {}): string {
+  const stripped = content.replace(TITLE_STRIP_RE, '').replace(TITLE_PARTIAL_RE, '');
+  return trim ? stripped.trim() : stripped;
+}
+
+// ============== SPEAKABLE TEXT ==============
+
+/**
+ * Markdown and links read aloud are noise ("asterisk asterisk", a URL spelled
+ * out). The voice prompt asks for none, but a model does not always listen, and
+ * an agent's own prompt may not ask at all — so the formatting is taken off
+ * here, at the last moment, and the transcript on screen keeps it.
+ */
+export function toSpeakableText(text: string): string {
+  return text
+    .replace(/```[\s\S]*?(```|$)/g, ' ')
+    .replace(/`([^`]*)`/g, '$1')
+    .replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1')
+    .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
+    .replace(/https?:\/\/\S+/g, ' ')
+    .replace(/^\s{0,3}#{1,6}\s+/gm, '')
+    .replace(/^\s*[-*+•]\s+/gm, '')
+    .replace(/^\s*>\s?/gm, '')
+    .replace(/\|/g, ' ')
+    .replace(/(\*\*|__|\*|_|~~)(?=\S)([\s\S]*?\S)\1/g, '$2')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+/**
+ * A sentence ends at terminal punctuation followed by whitespace, or at a line
+ * break. Requiring the whitespace is what keeps `3.5` and a half-streamed
+ * `e.g` whole: until the next character arrives, the period might not be one.
+ */
+const SENTENCE_END = /[.!?…。！？]+["'”’)\]]*\s+|\n+/g;
+
+/** Past this, a run with no sentence end is cut at a pause anyway. */
+const MAX_RUN_CHARS = 280;
+/** After the first, chunks gather sentences until this long: fewer, fuller requests. */
+const MIN_CHUNK_CHARS = 80;
+
+export interface SpeechChunks {
+  /** Ready to synthesize, in order, already speakable. */
+  readonly chunks: string[];
+  /** Not yet ready: the caller keeps it and passes it again with more text. */
+  readonly rest: string;
+}
+
+/**
+ * Split what has streamed so far into what can be spoken now and what must
+ * wait.
+ *
+ * The FIRST chunk of an answer is a single sentence, however short: time to
+ * first sound is the latency a person feels. Later chunks gather sentences up
+ * to `MIN_CHUNK_CHARS`, because each chunk is a synthesis request and playback
+ * of the one before covers the wait for the next. `final` flushes everything.
+ */
+export function takeSpeechChunks(
+  pending: string,
+  { final, isFirst }: { final: boolean; isFirst: boolean },
+): SpeechChunks {
+  const sentences: string[] = [];
+  let cursor = 0;
+  for (const match of pending.matchAll(SENTENCE_END)) {
+    const end = (match.index ?? 0) + match[0].length;
+    sentences.push(pending.slice(cursor, end));
+    cursor = end;
+  }
+  let rest = pending.slice(cursor);
+
+  if (final) {
+    if (rest.trim() !== '') sentences.push(rest);
+    rest = '';
+  } else if (rest.length > MAX_RUN_CHARS) {
+    const window = rest.slice(0, MAX_RUN_CHARS);
+    const pause = Math.max(window.lastIndexOf(', '), window.lastIndexOf('; '), window.lastIndexOf(': '));
+    const cut = pause > 0 ? pause + 2 : Math.max(window.lastIndexOf(' '), 1);
+    sentences.push(rest.slice(0, cut));
+    rest = rest.slice(cut);
+  }
+
+  const chunks: string[] = [];
+  let gathering = '';
+  for (const sentence of sentences) {
+    gathering += sentence;
+    const threshold = chunks.length === 0 && isFirst ? 1 : MIN_CHUNK_CHARS;
+    if (gathering.trim().length >= threshold) {
+      chunks.push(gathering);
+      gathering = '';
+    }
+  }
+  if (gathering !== '') {
+    if (final) chunks.push(gathering);
+    else rest = gathering + rest;
+  }
+
+  return {
+    chunks: chunks.map(toSpeakableText).filter((chunk) => chunk !== ''),
+    rest,
+  };
+}
+
+// ============== BARGE-IN ==============
+
+function words(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s']/gu, ' ')
+    .split(/\s+/)
+    .filter((word) => word !== '');
+}
+
+export function wordCount(text: string): number {
+  return words(text).length;
+}
+
+/** How many of the most recent words decide whether the person is talking. */
+const INTERRUPTION_WINDOW = 6;
+
+/**
+ * Whether what the microphone heard is the person talking over the answer,
+ * rather than the answer coming back through it.
+ *
+ * Voice mode keeps listening while it speaks so a person can interrupt, and on
+ * a laptop speaker the recognizer can hear the answer. Where the platform
+ * cancels echo that never reaches here; where it does not, the words give it
+ * away. `spoken` is everything the call has said this turn, and the transcript
+ * of one session keeps the echo it already heard, so only the latest words
+ * are judged, from the first one the call did not say: at least `minWords` of
+ * them — and at least half — must be new. A single misheard word of echo is
+ * not the person; two new words in a row are.
+ */
+export function isInterruption(heard: string, spoken: string, minWords: number): boolean {
+  const said = new Set(words(spoken));
+  const tail = words(heard).slice(-INTERRUPTION_WINDOW);
+  // Judge from the first word the call did not say: echo before it is echo.
+  const first = tail.findIndex((word) => !said.has(word));
+  if (first < 0) return false;
+  const candidate = tail.slice(first);
+  const novel = candidate.filter((word) => !said.has(word)).length;
+  return novel >= minWords && novel * 2 >= candidate.length;
+}
+
+/**
+ * The person's words without the echo the same session heard before them:
+ * leading words the call itself had just said are dropped.
+ */
+export function dropEchoPrefix(heard: string, spoken: string): string {
+  const said = new Set(words(spoken));
+  if (said.size === 0) return heard.trim();
+  const tokens = heard.trim().split(/\s+/);
+  let start = 0;
+  while (start < tokens.length) {
+    const normalized = words(tokens[start] ?? '');
+    if (normalized.length === 0 || normalized.every((word) => said.has(word))) start += 1;
+    else break;
+  }
+  return tokens.slice(start).join(' ');
+}

package/src/lib/voice-turn.ts

@@ -0,0 +1,87 @@
+/**
+ * How a voice call gets its answers: through the ordinary chat path.
+ *
+ * A spoken turn is a chat turn. There is no voice session endpoint and no
+ * realtime model behind this — the person's words, recognized on the device,
+ * go to Alia's chat handler exactly as typed words do (conversation, memory,
+ * agents, tools and all), and the answer comes back as the same SSE stream.
+ * The only thing that marks it as spoken is `responseMode: 'voice'`, which asks
+ * for an answer meant to be heard.
+ *
+ * A consumer that already owns a chat — the Alia app, whose conversation screen
+ * persists every turn and shows it in the thread — passes its own sender, so
+ * the call's turns ARE that conversation's turns. `createAliaVoiceTurnSender`
+ * is the default for everyone else: the same request `useAliaChat` makes.
+ */
+
+import { streamAliaChat, type AuthenticatedResponseClient } from './chat-transport';
+import { resolveModelId } from './catalogue';
+import { PREFERRED_CHAT_MODEL_ID } from './config';
+
+export interface VoiceTurnMessage {
+  readonly role: 'user' | 'assistant';
+  readonly content: string;
+}
+
+export interface VoiceTurn {
+  /** What the person said, as recognized. */
+  readonly text: string;
+  /** Earlier turns of this call, oldest first, not including `text`. */
+  readonly history: readonly VoiceTurnMessage[];
+  /** Aborted when the person interrupts or the call ends. */
+  readonly signal: AbortSignal;
+  /** Report the answer as it streams — the WHOLE answer so far, each time. */
+  readonly onText: (answerSoFar: string) => void;
+}
+
+/**
+ * Send one spoken turn and stream its answer through `onText`.
+ *
+ * Resolves when the answer is complete; rejects when the turn could not be
+ * sent at all. An answer that is empty is not a rejection — the voice loop
+ * reports it and keeps listening.
+ */
+export type VoiceTurnSender = (turn: VoiceTurn) => Promise<void>;
+
+interface LinkedClientFactory {
+  createLinkedClient(config: { baseURL: string }): { client: AuthenticatedResponseClient; dispose(): void };
+}
+
+export interface AliaVoiceTurnSenderOptions {
+  readonly oxyServices: LinkedClientFactory;
+  readonly apiUrl: string;
+  /** Chat routing profile, checked against `GET /catalogue` like `useAliaChat`'s. */
+  readonly model?: string;
+  readonly agentId?: string;
+}
+
+/** The default sender: `POST /v1/chat/completions` as `useAliaChat` sends it, marked as voice. */
+export function createAliaVoiceTurnSender(options: AliaVoiceTurnSenderOptions): VoiceTurnSender {
+  const { oxyServices, apiUrl, model = PREFERRED_CHAT_MODEL_ID, agentId } = options;
+  return async ({ text, history, signal, onText }) => {
+    const linked = oxyServices.createLinkedClient({ baseURL: apiUrl });
+    try {
+      const effectiveModel = await resolveModelId(apiUrl, model, undefined, PREFERRED_CHAT_MODEL_ID);
+      signal.throwIfAborted();
+      let answer = '';
+      await streamAliaChat(
+        linked.client,
+        {
+          url: '/v1/chat/completions',
+          model: effectiveModel,
+          messages: [...history, { role: 'user', content: text }],
+          responseMode: 'voice',
+          ...(agentId === undefined ? {} : { agentId }),
+        },
+        signal,
+        (event) => {
+          if (event.kind !== 'content' && event.kind !== 'agent_answer') return;
+          answer += event.content;
+          onText(answer);
+        },
+      );
+    } finally {
+      linked.dispose();
+    }
+  };
+}

package/src/types.ts

@@ -56,8 +56,8 @@ export interface ChatMessage {
   pendingPlan?: PendingPlan;
   /** Message origin: text chat or voice */
   source?: 'text' | 'voice';
-  /** Speaker identity in cohost voice mode */
-  speaker?: 'primary' | 'cohost';
+  /** Speaker identity in voice mode */
+  speaker?: 'primary';
   /** Whether a voice transcript is still streaming */
   isStreaming?: boolean;
   /** Cached TTS audio URL for read-aloud */
@@ -90,7 +90,7 @@ export interface VoiceToolInvocation {
 export interface VoiceMessage {
   id: string;
   role: 'user' | 'assistant';
-  speaker?: 'primary' | 'cohost';
+  speaker?: 'primary';
   content: string;
   timestamp: number;
   isStreaming: boolean;

package/src/voice.ts

@@ -1,7 +1,7 @@
 // @alia.onl/sdk/voice — Alia voice surface
-// LiveKit-backed voice room, audio visualisation, ambient wave, and sound effects.
-// Split out of the root entry so text-chat consumers never pull livekit-client
-// into their module graph.
+// The on-device voice loop (listen → chat → speak), audio visualisation,
+// ambient wave, and sound effects. Split out of the root entry so text-chat
+// consumers never compile the call.
 
 // ── Voice components ──
 export { VoiceSession } from './components/voice/VoiceSession';
@@ -11,6 +11,10 @@ export { VoiceControls } from './components/voice/VoiceControls';
 
 // ── Voice hooks ──
 export { useVoiceRoom } from './hooks/useVoiceRoom';
+export type { UseVoiceRoomOptions } from './hooks/useVoiceRoom';
+export { createAliaVoiceTurnSender } from './lib/voice-turn';
+export type { VoiceTurn, VoiceTurnMessage, VoiceTurnSender, AliaVoiceTurnSenderOptions } from './lib/voice-turn';
+export type { VoiceLevelSource } from './lib/voice-levels';
 export { useAudioLevelMonitor } from './hooks/useAudioLevelMonitor';
 export { useAudioLevels } from './hooks/useAudioLevels';
 export { useAmbientWave } from './hooks/useAmbientWave';