@alia.onl/sdk 13.1.0 → 15.2.1

package/src/lib/speech-synthesis.ts

@@ -20,14 +20,112 @@ export interface SpeechClipRequest {
   readonly input: string;
   readonly voice: ProductVoice;
   readonly speed?: number;
-  /** Both or neither: the route stores the clip on that message when given. */
+  /**
+   * Both or neither: the saved message being read aloud. The route answers 404
+   * for one this person does not hold, and files the clip under that
+   * conversation, so deleting the conversation deletes it.
+   */
   readonly conversationId?: string;
   readonly messageId?: string;
   readonly signal: AbortSignal;
 }
 
-/** Synthesize `input` and return the private link the route answers with. */
+/**
+ * The codes `/v1/audio/speech` answers a failure with, plus the two this side
+ * knows on its own: `NOT_AUTHENTICATED` (no session token, or the route said
+ * 401) and `INVALID_REQUEST` (the route refused the body).
+ */
+export type SpeechClipErrorCode =
+  | 'SPEECH_UNAVAILABLE'
+  | 'SPEECH_REJECTED'
+  | 'SPEECH_LIMIT_REACHED'
+  | 'SPEECH_PROVIDER_FAILED'
+  | 'SPEECH_GENERATION_FAILED'
+  | 'INVALID_REQUEST'
+  | 'NOT_AUTHENTICATED';
+
+const SERVER_CODES: ReadonlySet<string> = new Set<SpeechClipErrorCode>([
+  'SPEECH_UNAVAILABLE',
+  'SPEECH_REJECTED',
+  'SPEECH_LIMIT_REACHED',
+  'SPEECH_PROVIDER_FAILED',
+  'SPEECH_GENERATION_FAILED',
+  'INVALID_REQUEST',
+]);
+
+/**
+ * A clip that could not be made, with what the route said about it: the HTTP
+ * status (`null` when no request was answered) and its typed `code` (`null`
+ * when the route named none this SDK knows — a network failure, an older or
+ * newer server).
+ */
+export class SpeechClipError extends Error {
+  readonly status: number | null;
+  readonly code: SpeechClipErrorCode | null;
+  readonly retryable: boolean;
+  /** Oxy's request id, for support, when the route gave one. */
+  readonly requestId: string | null;
+
+  constructor(
+    message: string,
+    details: {
+      status: number | null;
+      code: SpeechClipErrorCode | null;
+      retryable?: boolean;
+      requestId?: string | null;
+    },
+  ) {
+    super(message);
+    this.name = 'SpeechClipError';
+    this.status = details.status;
+    this.code = details.code;
+    this.retryable = details.retryable ?? false;
+    this.requestId = details.requestId ?? null;
+  }
+}
+
+function clipErrorFrom(status: number, body: unknown): SpeechClipError {
+  const envelope =
+    body !== null && typeof body === 'object' ? (body as { error?: unknown }).error : undefined;
+  const error =
+    envelope !== null && typeof envelope === 'object'
+      ? (envelope as {
+          code?: unknown;
+          message?: unknown;
+          retryable?: unknown;
+          requestId?: unknown;
+        })
+      : {};
+  const named =
+    typeof error.code === 'string' && SERVER_CODES.has(error.code)
+      ? (error.code as SpeechClipErrorCode)
+      : null;
+  const code = named ?? (status === 401 ? 'NOT_AUTHENTICATED' : null);
+  const message =
+    status === 504
+      ? 'Request timed out — please try again'
+      : typeof error.message === 'string' && error.message !== ''
+        ? error.message
+        : typeof envelope === 'string' && envelope !== ''
+          ? envelope
+          : 'TTS failed';
+  return new SpeechClipError(message, {
+    status,
+    code,
+    retryable: error.retryable === true,
+    requestId: typeof error.requestId === 'string' ? error.requestId : null,
+  });
+}
+
+/**
+ * Synthesize `input` and return the private link the route answers with.
+ *
+ * @throws {SpeechClipError} when the route refuses or answers no link; a
+ *   network failure or an abort is thrown as `fetch` threw it.
+ */
 export async function requestSpeechClip(request: SpeechClipRequest): Promise<string> {
+  if (request.token === '')
+    throw new SpeechClipError('Not authenticated', { status: null, code: 'NOT_AUTHENTICATED' });
   const response = await fetch(`${request.apiUrl}/v1/audio/speech`, {
     method: 'POST',
     signal: request.signal,
@@ -46,16 +144,13 @@ export async function requestSpeechClip(request: SpeechClipRequest): Promise<str
   });
 
   if (!response.ok) {
-    if (response.status === 504) {
-      throw new Error('Request timed out — please try again');
-    }
-    const errData = await response.json().catch(() => ({}));
-    throw new Error(errData.error?.message || errData.error || 'TTS failed');
+    throw clipErrorFrom(response.status, await response.json().catch(() => null));
   }
 
-  const data = (await response.json()) as { audioUrl?: unknown };
+  const data = (await response.json().catch(() => ({}))) as { audioUrl?: unknown };
   request.signal.throwIfAborted();
-  if (typeof data.audioUrl !== 'string' || data.audioUrl === '') throw new Error('TTS failed');
+  if (typeof data.audioUrl !== 'string' || data.audioUrl === '')
+    throw new SpeechClipError('TTS failed', { status: response.status, code: null });
   return data.audioUrl;
 }
 

package/src/lib/turn-state.ts

@@ -0,0 +1,15 @@
+import type { ChatMessage } from '../types';
+import type { AliaAvatarState } from '../components/AliaAvatar';
+
+/** What a chat turn is doing, from its messages: the one reading the header mark and every button share. */
+export function turnActivityState(
+  messages: ReadonlyArray<ChatMessage>,
+  isStreaming: boolean,
+): AliaAvatarState {
+  if (!isStreaming) return 'idle';
+  const last = messages[messages.length - 1];
+  if (last?.role === 'assistant' && last.toolInvocations?.some((t) => t.state === 'call'))
+    return 'working';
+  if (last?.thinking) return 'thinking';
+  return 'writing';
+}

package/src/lib/voice-audio-engine-types.ts

@@ -0,0 +1,59 @@
+/**
+ * The seam between a voice call and the device's audio (ADR 0017 §4).
+ *
+ * One engine captures the microphone and plays the answer, because echo
+ * cancellation only cancels what the SAME audio stack plays:
+ *
+ * - web (`voice-audio-engine.ts`): `getUserMedia` with echo cancellation, noise
+ *   suppression and gain control, and two `AudioWorklet`s in one `AudioContext`;
+ * - iOS / Android (`voice-audio-engine.native.ts`): `react-native-audio-api`.
+ *
+ * Audio crosses the seam as PCM16 little-endian, 24 kHz, mono — the format the
+ * call carries in both directions — so the call never resamples or converts.
+ * Tests drive a call through a fake engine that implements this interface.
+ */
+
+/** Why an engine could not start, or stopped working mid-call. */
+export type VoiceAudioEngineErrorCode = 'mic-denied' | 'mic-unavailable' | 'voice-unavailable';
+
+export class VoiceAudioEngineError extends Error {
+  readonly code: VoiceAudioEngineErrorCode;
+
+  constructor(code: VoiceAudioEngineErrorCode, message?: string) {
+    super(message ?? code);
+    this.name = 'VoiceAudioEngineError';
+    this.code = code;
+  }
+}
+
+export interface VoiceAudioEngineHandlers {
+  /** One captured frame (~40 ms), PCM16 24 kHz mono. */
+  onCaptureFrame(pcm16: Uint8Array): void;
+  /** The microphone's level, 0..1, from the real captured buffers. */
+  onCaptureLevel(level: number): void;
+  /** The first sample of `itemId` is audible now — not received, PLAYING. */
+  onPlaybackStarted(itemId: string): void;
+  /** The level of what is playing, 0..1, from the real played buffers. */
+  onPlaybackLevel(level: number): void;
+  /** Everything queued has been played. */
+  onPlaybackIdle(): void;
+  /** The device failed during the call (the microphone went away). */
+  onError(error: VoiceAudioEngineError): void;
+}
+
+export interface VoiceAudioEngine {
+  /**
+   * Ask for the microphone and start capturing and the player. Rejects with a
+   * {@link VoiceAudioEngineError}.
+   */
+  start(handlers: VoiceAudioEngineHandlers): Promise<void>;
+  /** Queue a chunk of `itemId`'s audio behind what is already queued. */
+  enqueue(itemId: string, pcm16: Uint8Array): void;
+  /**
+   * Drop everything queued and stop playing at once. Resolves with how much of
+   * the item that was playing was actually heard, or `null` when nothing was.
+   */
+  flush(): Promise<{ itemId: string; playedMs: number } | null>;
+  /** Release the microphone and the player. Safe to call more than once. */
+  stop(): void;
+}

package/src/lib/voice-audio-engine.native.ts

@@ -0,0 +1,232 @@
+/**
+ * The iOS / Android voice-call engine, on `react-native-audio-api` (Software
+ * Mansion) — capture and playback in ONE native audio stack, because the
+ * platform echo canceller only removes what that same stack plays (ADR 0017
+ * §4). `expo-audio` cannot do this: its stream captures without echo
+ * cancellation and it cannot play a PCM stream.
+ *
+ * - **iOS:** the audio session is `playAndRecord` in mode `voiceChat`, which is
+ *   Apple's voice-processing path (echo cancellation, noise suppression, gain
+ *   control), with the loudspeaker as default output and Bluetooth headsets
+ *   allowed.
+ * - **Android:** `react-native-audio-api` 0.13.6 opens its recorder through
+ *   Oboe with the default input preset and exposes no `voiceCommunication`
+ *   preset, so the platform echo canceller is NOT engaged by this engine. That
+ *   is the Android AEC gate of ADR 0017 §4, to be measured on the Pixel 8a; if
+ *   it fails, the ADR's fallback is a small Expo module of Alia's own around
+ *   `AudioRecord(VOICE_COMMUNICATION)` behind this same interface.
+ *
+ * A new native build is required: the library's config plugin is in
+ * `packages/app/app.json`.
+ */
+
+import {
+  type AudioBufferQueueSourceNode,
+  AudioContext,
+  AudioManager,
+  AudioRecorder,
+} from 'react-native-audio-api';
+import { floatToPcm16, levelOfPcm16, pcm16ToFloat } from './pcm';
+import { pcmBytesToMs, VOICE_FRAME_SAMPLES, VOICE_SAMPLE_RATE } from './voice-protocol';
+import {
+  VoiceAudioEngineError,
+  type VoiceAudioEngine,
+  type VoiceAudioEngineHandlers,
+} from './voice-audio-engine-types';
+
+export * from './voice-audio-engine-types';
+
+/** Held before the first buffer of a burst, so network jitter does not stutter it. */
+const PREBUFFER_MS = 80;
+/** A burst shorter than that still plays once nothing more arrives for this long. */
+const PREBUFFER_IDLE_MS = 100;
+
+interface Queued {
+  readonly id: string;
+  readonly itemId: string;
+  readonly ms: number;
+  readonly level: number;
+}
+
+export function createVoiceAudioEngine(): VoiceAudioEngine {
+  let context: AudioContext | null = null;
+  let recorder: AudioRecorder | null = null;
+  let player: AudioBufferQueueSourceNode | null = null;
+  let handlers: VoiceAudioEngineHandlers | null = null;
+  let stopped = false;
+
+  /** Buffers handed to the player, oldest (playing) first. */
+  let queued: Queued[] = [];
+  /** Chunks held back until the prebuffer is full. */
+  let pending: Array<{ itemId: string; pcm: Uint8Array }> = [];
+  let pendingMs = 0;
+  let prebufferTimer: ReturnType<typeof setTimeout> | null = null;
+  /** The item playing now, how much of it has finished, and when its current buffer began. */
+  let current: { itemId: string; playedMs: number; bufferStartedAt: number } | null = null;
+
+  const beginBuffer = (entry: Queued): void => {
+    if (context === null || handlers === null) return;
+    if (current === null || current.itemId !== entry.itemId) {
+      current = { itemId: entry.itemId, playedMs: 0, bufferStartedAt: context.currentTime };
+      handlers.onPlaybackStarted(entry.itemId);
+    } else {
+      current.bufferStartedAt = context.currentTime;
+    }
+    handlers.onPlaybackLevel(entry.level);
+  };
+
+  const releasePlayer = (): void => {
+    if (player === null) return;
+    const old = player;
+    player = null;
+    old.onBufferEnded = null;
+    try {
+      old.clearBuffers();
+      old.stop();
+    } catch {
+      // Already stopped.
+    }
+    old.disconnect();
+  };
+
+  /** A fresh player per burst: a queue source that ran dry is not relied on to resume. */
+  const ensurePlayer = (): AudioBufferQueueSourceNode | null => {
+    if (player !== null || context === null) return player;
+    const node = context.createBufferQueueSource();
+    node.connect(context.destination);
+    node.onBufferEnded = ({ bufferId }) => {
+      if (stopped || node !== player) return;
+      const index = queued.findIndex((entry) => entry.id === bufferId);
+      if (index === -1) return;
+      const [ended] = queued.splice(index, 1);
+      if (ended && current?.itemId === ended.itemId) current.playedMs += ended.ms;
+      const next = queued[0];
+      if (next !== undefined) {
+        beginBuffer(next);
+        return;
+      }
+      releasePlayer();
+      handlers?.onPlaybackLevel(0);
+      handlers?.onPlaybackIdle();
+    };
+    node.start();
+    player = node;
+    return node;
+  };
+
+  const push = (itemId: string, pcm: Uint8Array): void => {
+    const node = ensurePlayer();
+    if (node === null || context === null) return;
+    const samples = pcm16ToFloat(pcm);
+    if (samples.length === 0) return;
+    const buffer = context.createBuffer(1, samples.length, VOICE_SAMPLE_RATE);
+    buffer.copyToChannel(samples as Float32Array<ArrayBuffer>, 0);
+    const id = node.enqueueBuffer(buffer);
+    const entry: Queued = {
+      id,
+      itemId,
+      ms: pcmBytesToMs(pcm.byteLength),
+      level: levelOfPcm16(pcm),
+    };
+    queued.push(entry);
+    if (queued.length === 1) beginBuffer(entry);
+  };
+
+  const releasePending = (): void => {
+    if (prebufferTimer !== null) clearTimeout(prebufferTimer);
+    prebufferTimer = null;
+    const chunks = pending;
+    pending = [];
+    pendingMs = 0;
+    for (const chunk of chunks) push(chunk.itemId, chunk.pcm);
+  };
+
+  return {
+    async start(nextHandlers: VoiceAudioEngineHandlers): Promise<void> {
+      handlers = nextHandlers;
+      AudioManager.setAudioSessionOptions({
+        iosCategory: 'playAndRecord',
+        iosMode: 'voiceChat',
+        iosOptions: ['defaultToSpeaker', 'allowBluetoothHFP'],
+      });
+      const permission = await AudioManager.requestRecordingPermissions().catch(() => 'Denied');
+      if (permission !== 'Granted') throw new VoiceAudioEngineError('mic-denied');
+      try {
+        await AudioManager.setAudioSessionActivity(true);
+      } catch {
+        throw new VoiceAudioEngineError('mic-unavailable');
+      }
+      if (stopped) return;
+
+      context = new AudioContext({ sampleRate: VOICE_SAMPLE_RATE });
+      recorder = new AudioRecorder();
+      const ready = recorder.onAudioReady(
+        { sampleRate: VOICE_SAMPLE_RATE, bufferLength: VOICE_FRAME_SAMPLES, channelCount: 1 },
+        ({ buffer }) => {
+          if (stopped) return;
+          const pcm = floatToPcm16(buffer.getChannelData(0));
+          handlers?.onCaptureLevel(levelOfPcm16(pcm));
+          handlers?.onCaptureFrame(pcm);
+        },
+      );
+      if (ready.status === 'error')
+        throw new VoiceAudioEngineError('mic-unavailable', ready.message);
+      const started = await recorder.start();
+      if (started.status === 'error')
+        throw new VoiceAudioEngineError('mic-unavailable', started.message);
+    },
+
+    enqueue(itemId: string, pcm16: Uint8Array): void {
+      if (stopped || context === null) return;
+      if (player !== null) {
+        push(itemId, pcm16);
+        return;
+      }
+      pending.push({ itemId, pcm: pcm16 });
+      pendingMs += pcmBytesToMs(pcm16.byteLength);
+      if (prebufferTimer !== null) clearTimeout(prebufferTimer);
+      if (pendingMs >= PREBUFFER_MS) releasePending();
+      else prebufferTimer = setTimeout(releasePending, PREBUFFER_IDLE_MS);
+    },
+
+    async flush(): Promise<{ itemId: string; playedMs: number } | null> {
+      if (prebufferTimer !== null) clearTimeout(prebufferTimer);
+      prebufferTimer = null;
+      pending = [];
+      pendingMs = 0;
+      const playing = current;
+      const head = queued[0];
+      let heard: { itemId: string; playedMs: number } | null = null;
+      if (playing !== null && context !== null) {
+        const intoBuffer =
+          head !== undefined && head.itemId === playing.itemId
+            ? Math.min(head.ms, Math.max(0, (context.currentTime - playing.bufferStartedAt) * 1000))
+            : 0;
+        heard = { itemId: playing.itemId, playedMs: Math.round(playing.playedMs + intoBuffer) };
+      }
+      queued = [];
+      current = null;
+      releasePlayer();
+      handlers?.onPlaybackLevel(0);
+      return heard;
+    },
+
+    stop(): void {
+      if (stopped) return;
+      stopped = true;
+      if (prebufferTimer !== null) clearTimeout(prebufferTimer);
+      releasePlayer();
+      if (recorder !== null) {
+        recorder.clearOnAudioReady();
+        void recorder.stop().catch(() => undefined);
+      }
+      void context?.close().catch(() => undefined);
+      void AudioManager.setAudioSessionActivity(false).catch(() => undefined);
+      recorder = null;
+      context = null;
+      handlers = null;
+      queued = [];
+      pending = [];
+    },
+  };
+}