@alia.onl/sdk 13.2.0 → 15.2.1

package/src/lib/labels.tsx

@@ -42,6 +42,8 @@ export interface AliaChatLabels {
   dropUnavailable: string;
   /** Why dictation failed, in the host's language. */
   voiceError: (code: VoiceErrorCode) => string;
+  /** Over the composer in the last 30 s of a dictation's five minutes. */
+  dictationRemaining: (seconds: number) => string;
   // Welcome
   /** The empty chat's greeting for a signed-out visitor. */
   greeting: string;
@@ -99,6 +101,7 @@ export const DEFAULT_LABELS: AliaChatLabels = {
   dropHint: 'Drop files here to attach them',
   dropUnavailable: "Files can't be attached right now",
   voiceError: (code) => VOICE_ERROR_MESSAGES[code],
+  dictationRemaining: (seconds) => `${seconds}s of dictation left`,
   greeting: "Hi, what's on your mind?",
   greetingNamed: (name) => `Hi ${name}, what's on your mind?`,
   copy: 'Copy',

package/src/lib/pcm.ts

@@ -0,0 +1,57 @@
+/**
+ * PCM16 in and out of the -1..1 floats audio engines work in, and the level a
+ * buffer reads at. No imports, so every engine (and the tests) shares one
+ * definition of the conversion and of the meter.
+ */
+
+import { levelFromFrames } from './audio-level';
+
+/** Floats (-1..1) to PCM16 little-endian bytes. */
+export function floatToPcm16(samples: ArrayLike<number>): Uint8Array {
+  const bytes = new Uint8Array(samples.length * 2);
+  const view = new DataView(bytes.buffer);
+  for (let index = 0; index < samples.length; index += 1) {
+    const clamped = Math.max(-1, Math.min(1, samples[index] ?? 0));
+    view.setInt16(index * 2, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
+  }
+  return bytes;
+}
+
+/** PCM16 little-endian bytes to floats (-1..1). A trailing odd byte is dropped. */
+export function pcm16ToFloat(bytes: Uint8Array): Float32Array {
+  const count = Math.floor(bytes.byteLength / 2);
+  const view = new DataView(bytes.buffer, bytes.byteOffset, count * 2);
+  const samples = new Float32Array(count);
+  for (let index = 0; index < count; index += 1) {
+    samples[index] = view.getInt16(index * 2, true) / 0x8000;
+  }
+  return samples;
+}
+
+/**
+ * Linear resampling. Good enough for speech in a call: the model hears 24 kHz
+ * and the device plays at its own rate; neither is a recording kept.
+ */
+export function resampleLinear(
+  samples: Float32Array,
+  fromRate: number,
+  toRate: number,
+): Float32Array {
+  if (fromRate === toRate || samples.length === 0) return samples;
+  const ratio = fromRate / toRate;
+  const length = Math.max(1, Math.floor(samples.length / ratio));
+  const out = new Float32Array(length);
+  for (let index = 0; index < length; index += 1) {
+    const position = index * ratio;
+    const left = Math.floor(position);
+    const right = Math.min(left + 1, samples.length - 1);
+    const fraction = position - left;
+    out[index] = (samples[left] ?? 0) * (1 - fraction) + (samples[right] ?? 0) * fraction;
+  }
+  return out;
+}
+
+/** The field's 0..1 level of a PCM16 buffer, on the dictation meter's dBFS scale. */
+export function levelOfPcm16(bytes: Uint8Array): number {
+  return levelFromFrames(Array.from(pcm16ToFloat(bytes)));
+}

package/src/lib/speech-language.ts

@@ -1,5 +1,5 @@
 /**
- * The language to recognize when the caller names none.
+ * The language spoken, when the caller names none (dictation and calls).
  *
  * The browser's own language on web; the runtime's resolved locale on native
  * (Hermes implements `Intl`); `en-US` when neither answers. An app should pass

package/src/lib/speech-messages.ts

@@ -1,108 +1,54 @@
-import type { SpeechRecognitionFailure } from './speech-recognition-types';
-import type { SpeechClipError } from './speech-synthesis';
+import type { VoiceServerErrorCode } from './voice-protocol';
 
 /**
  * Every way dictation or a voice call can fail that a person is told about,
- * as a stable code.
+ * as a stable code (ADR 0017 §3, §7).
  *
- * The hooks expose the code beside the English sentence (`errorCode` /
- * `turnErrorCode` next to `error` / `turnError`) so an app can say it in its
- * own language; the sentence stays for a consumer that has no translations.
+ * - **Alia's**, sent by `/v1/voice/session` when it ends a call:
+ *   `VoiceServerErrorCode` (`plan-required`, `credits-exhausted`,
+ *   `realtime-*`, …).
+ * - **The device's:** `mic-denied`, `mic-unavailable`, `connection-lost`, and
+ *   `voice-unavailable` where this platform has no audio engine for a call.
+ * - **Dictation's** (`POST /v1/audio/transcriptions`): `too-long` and
+ *   `transcription-*`, beside the shared `mic-*` and `not-authenticated`.
+ *
+ * The hooks expose the code beside the English sentence so an app can say it in
+ * its own language; the sentence stays for a consumer with no translations.
+ * Nothing here is a recognizer's: speech is recognized by Kaana, never by the
+ * browser or the operating system.
  */
 export type VoiceErrorCode =
-  | 'speech-unsupported'
-  | 'microphone-denied'
-  | 'microphone-busy'
-  | 'microphone-missing'
-  | 'speech-network'
-  | 'speech-language'
-  | 'speech-failed'
+  | VoiceServerErrorCode
+  | 'mic-denied'
+  | 'mic-unavailable'
+  | 'connection-lost'
   | 'voice-unavailable'
-  | 'not-authenticated'
-  | 'turn-failed'
-  | 'no-answer'
-  | 'not-played'
-  | 'synthesis-unavailable'
-  | 'synthesis-limited'
-  | 'synthesis-rejected'
-  | 'synthesis-failed';
+  | 'too-long'
+  | 'transcription-unavailable'
+  | 'transcription-limited'
+  | 'transcription-rejected'
+  | 'transcription-failed';
 
-/**
- * The English copy for each code — the same sentences dictation and voice mode
- * used when they recorded through `expo-audio` and LiveKit, so moving
- * recognition onto the device did not change what an error says.
- */
+/** The English copy for each code. */
 export const VOICE_ERROR_MESSAGES: Readonly<Record<VoiceErrorCode, string>> = {
-  'speech-unsupported': 'Speech recognition is not available on this device or browser',
-  'microphone-denied': 'Microphone permission required',
-  'microphone-busy': 'Microphone is in use by another application',
-  'microphone-missing': 'No microphone found — check your input devices',
-  'speech-network': 'Speech recognition needs a network connection',
-  'speech-language': 'Speech recognition does not support this language',
-  'speech-failed': 'Speech recognition failed',
-  'voice-unavailable': 'Voice is not available on this device',
   'not-authenticated': 'Not authenticated',
-  'turn-failed': 'Voice request failed',
-  'no-answer': 'No answer came back — try saying it again',
-  'not-played': 'The answer could not be played aloud — it is in the chat',
-  'synthesis-unavailable': "Alia's voice is not available right now — the answer is in the chat",
-  'synthesis-limited': "Alia's voice has reached its limit for now — the answer is in the chat",
-  'synthesis-rejected': "Alia's voice could not read this answer — it is in the chat",
-  'synthesis-failed': "Alia's voice failed — the answer is in the chat",
+  'credential-not-supported': 'Voice is only available when you are signed in to Alia',
+  'plan-required': 'Voice mode needs a paid plan',
+  'credits-exhausted': "You've run out of credits for this call",
+  'agent-unavailable': 'This agent is not available for a call',
+  'invalid-request': 'The call could not be started',
+  'resume-unavailable': 'The call could not be picked up again — start a new one',
+  'realtime-unavailable': 'Voice is not available right now — try again later',
+  'realtime-limited': 'Voice has reached its limit for now — try again later',
+  'realtime-rejected': 'The call could not be started with these settings',
+  'realtime-failed': 'The call failed — try again',
+  'mic-denied': 'Microphone permission required',
+  'mic-unavailable': 'The microphone could not be used — check your input devices',
+  'connection-lost': 'The connection was lost',
+  'voice-unavailable': 'Voice is not available on this device or browser',
+  'too-long': 'That dictation was too long — five minutes at most',
+  'transcription-unavailable': 'Dictation is not available right now',
+  'transcription-limited': 'Dictation has reached its limit for now — try again later',
+  'transcription-rejected': 'That recording could not be transcribed',
+  'transcription-failed': 'Dictation failed — try again',
 };
-
-/**
- * The code for a clip `/v1/audio/speech` could not make, from the typed code
- * the route answered with. A failure the route did not name (a network
- * failure, an unknown code) is `synthesis-failed`.
- */
-export function synthesisFailureCode(failure: SpeechClipError): VoiceErrorCode {
-  switch (failure.code) {
-    case 'NOT_AUTHENTICATED':
-      return 'not-authenticated';
-    case 'SPEECH_UNAVAILABLE':
-      return 'synthesis-unavailable';
-    case 'SPEECH_LIMIT_REACHED':
-      return 'synthesis-limited';
-    case 'SPEECH_REJECTED':
-    case 'INVALID_REQUEST':
-      return 'synthesis-rejected';
-    case 'SPEECH_PROVIDER_FAILED':
-    case 'SPEECH_GENERATION_FAILED':
-    case null:
-      return 'synthesis-failed';
-  }
-}
-
-/**
- * The code for a recognition failure. `null` for the failures that are not
- * errors: `aborted` is a stop the caller asked for, and `no-speech` is a
- * silence the caller recovers from.
- */
-export function speechFailureCode(failure: SpeechRecognitionFailure): VoiceErrorCode | null {
-  switch (failure.code) {
-    case 'aborted':
-    case 'no-speech':
-      return null;
-    case 'unsupported':
-      return 'speech-unsupported';
-    case 'not-allowed':
-      return 'microphone-denied';
-    case 'audio-capture':
-      return failure.detail === 'NotReadableError' || failure.detail === 'TrackStartError'
-        ? 'microphone-busy'
-        : 'microphone-missing';
-    case 'network':
-      return 'speech-network';
-    case 'language-not-supported':
-      return 'speech-language';
-    case 'other':
-      return 'speech-failed';
-  }
-}
-
-/** The English copy a person sees when listening fails; `null` as for `speechFailureCode`. */
-export function speechFailureMessage(failure: SpeechRecognitionFailure): string | null {
-  const code = speechFailureCode(failure);
-  return code === null ? null : VOICE_ERROR_MESSAGES[code];
-}

package/src/lib/speech-synthesis.ts

@@ -20,7 +20,11 @@ export interface SpeechClipRequest {
   readonly input: string;
   readonly voice: ProductVoice;
   readonly speed?: number;
-  /** Both or neither: the route stores the clip on that message when given. */
+  /**
+   * Both or neither: the saved message being read aloud. The route answers 404
+   * for one this person does not hold, and files the clip under that
+   * conversation, so deleting the conversation deletes it.
+   */
   readonly conversationId?: string;
   readonly messageId?: string;
   readonly signal: AbortSignal;

package/src/lib/turn-state.ts

@@ -0,0 +1,15 @@
+import type { ChatMessage } from '../types';
+import type { AliaAvatarState } from '../components/AliaAvatar';
+
+/** What a chat turn is doing, from its messages: the one reading the header mark and every button share. */
+export function turnActivityState(
+  messages: ReadonlyArray<ChatMessage>,
+  isStreaming: boolean,
+): AliaAvatarState {
+  if (!isStreaming) return 'idle';
+  const last = messages[messages.length - 1];
+  if (last?.role === 'assistant' && last.toolInvocations?.some((t) => t.state === 'call'))
+    return 'working';
+  if (last?.thinking) return 'thinking';
+  return 'writing';
+}

package/src/lib/voice-audio-engine-types.ts

@@ -0,0 +1,59 @@
+/**
+ * The seam between a voice call and the device's audio (ADR 0017 §4).
+ *
+ * One engine captures the microphone and plays the answer, because echo
+ * cancellation only cancels what the SAME audio stack plays:
+ *
+ * - web (`voice-audio-engine.ts`): `getUserMedia` with echo cancellation, noise
+ *   suppression and gain control, and two `AudioWorklet`s in one `AudioContext`;
+ * - iOS / Android (`voice-audio-engine.native.ts`): `react-native-audio-api`.
+ *
+ * Audio crosses the seam as PCM16 little-endian, 24 kHz, mono — the format the
+ * call carries in both directions — so the call never resamples or converts.
+ * Tests drive a call through a fake engine that implements this interface.
+ */
+
+/** Why an engine could not start, or stopped working mid-call. */
+export type VoiceAudioEngineErrorCode = 'mic-denied' | 'mic-unavailable' | 'voice-unavailable';
+
+export class VoiceAudioEngineError extends Error {
+  readonly code: VoiceAudioEngineErrorCode;
+
+  constructor(code: VoiceAudioEngineErrorCode, message?: string) {
+    super(message ?? code);
+    this.name = 'VoiceAudioEngineError';
+    this.code = code;
+  }
+}
+
+export interface VoiceAudioEngineHandlers {
+  /** One captured frame (~40 ms), PCM16 24 kHz mono. */
+  onCaptureFrame(pcm16: Uint8Array): void;
+  /** The microphone's level, 0..1, from the real captured buffers. */
+  onCaptureLevel(level: number): void;
+  /** The first sample of `itemId` is audible now — not received, PLAYING. */
+  onPlaybackStarted(itemId: string): void;
+  /** The level of what is playing, 0..1, from the real played buffers. */
+  onPlaybackLevel(level: number): void;
+  /** Everything queued has been played. */
+  onPlaybackIdle(): void;
+  /** The device failed during the call (the microphone went away). */
+  onError(error: VoiceAudioEngineError): void;
+}
+
+export interface VoiceAudioEngine {
+  /**
+   * Ask for the microphone and start capturing and the player. Rejects with a
+   * {@link VoiceAudioEngineError}.
+   */
+  start(handlers: VoiceAudioEngineHandlers): Promise<void>;
+  /** Queue a chunk of `itemId`'s audio behind what is already queued. */
+  enqueue(itemId: string, pcm16: Uint8Array): void;
+  /**
+   * Drop everything queued and stop playing at once. Resolves with how much of
+   * the item that was playing was actually heard, or `null` when nothing was.
+   */
+  flush(): Promise<{ itemId: string; playedMs: number } | null>;
+  /** Release the microphone and the player. Safe to call more than once. */
+  stop(): void;
+}

package/src/lib/voice-audio-engine.native.ts

@@ -0,0 +1,232 @@
+/**
+ * The iOS / Android voice-call engine, on `react-native-audio-api` (Software
+ * Mansion) — capture and playback in ONE native audio stack, because the
+ * platform echo canceller only removes what that same stack plays (ADR 0017
+ * §4). `expo-audio` cannot do this: its stream captures without echo
+ * cancellation and it cannot play a PCM stream.
+ *
+ * - **iOS:** the audio session is `playAndRecord` in mode `voiceChat`, which is
+ *   Apple's voice-processing path (echo cancellation, noise suppression, gain
+ *   control), with the loudspeaker as default output and Bluetooth headsets
+ *   allowed.
+ * - **Android:** `react-native-audio-api` 0.13.6 opens its recorder through
+ *   Oboe with the default input preset and exposes no `voiceCommunication`
+ *   preset, so the platform echo canceller is NOT engaged by this engine. That
+ *   is the Android AEC gate of ADR 0017 §4, to be measured on the Pixel 8a; if
+ *   it fails, the ADR's fallback is a small Expo module of Alia's own around
+ *   `AudioRecord(VOICE_COMMUNICATION)` behind this same interface.
+ *
+ * A new native build is required: the library's config plugin is in
+ * `packages/app/app.json`.
+ */
+
+import {
+  type AudioBufferQueueSourceNode,
+  AudioContext,
+  AudioManager,
+  AudioRecorder,
+} from 'react-native-audio-api';
+import { floatToPcm16, levelOfPcm16, pcm16ToFloat } from './pcm';
+import { pcmBytesToMs, VOICE_FRAME_SAMPLES, VOICE_SAMPLE_RATE } from './voice-protocol';
+import {
+  VoiceAudioEngineError,
+  type VoiceAudioEngine,
+  type VoiceAudioEngineHandlers,
+} from './voice-audio-engine-types';
+
+export * from './voice-audio-engine-types';
+
+/** Held before the first buffer of a burst, so network jitter does not stutter it. */
+const PREBUFFER_MS = 80;
+/** A burst shorter than that still plays once nothing more arrives for this long. */
+const PREBUFFER_IDLE_MS = 100;
+
+interface Queued {
+  readonly id: string;
+  readonly itemId: string;
+  readonly ms: number;
+  readonly level: number;
+}
+
+export function createVoiceAudioEngine(): VoiceAudioEngine {
+  let context: AudioContext | null = null;
+  let recorder: AudioRecorder | null = null;
+  let player: AudioBufferQueueSourceNode | null = null;
+  let handlers: VoiceAudioEngineHandlers | null = null;
+  let stopped = false;
+
+  /** Buffers handed to the player, oldest (playing) first. */
+  let queued: Queued[] = [];
+  /** Chunks held back until the prebuffer is full. */
+  let pending: Array<{ itemId: string; pcm: Uint8Array }> = [];
+  let pendingMs = 0;
+  let prebufferTimer: ReturnType<typeof setTimeout> | null = null;
+  /** The item playing now, how much of it has finished, and when its current buffer began. */
+  let current: { itemId: string; playedMs: number; bufferStartedAt: number } | null = null;
+
+  const beginBuffer = (entry: Queued): void => {
+    if (context === null || handlers === null) return;
+    if (current === null || current.itemId !== entry.itemId) {
+      current = { itemId: entry.itemId, playedMs: 0, bufferStartedAt: context.currentTime };
+      handlers.onPlaybackStarted(entry.itemId);
+    } else {
+      current.bufferStartedAt = context.currentTime;
+    }
+    handlers.onPlaybackLevel(entry.level);
+  };
+
+  const releasePlayer = (): void => {
+    if (player === null) return;
+    const old = player;
+    player = null;
+    old.onBufferEnded = null;
+    try {
+      old.clearBuffers();
+      old.stop();
+    } catch {
+      // Already stopped.
+    }
+    old.disconnect();
+  };
+
+  /** A fresh player per burst: a queue source that ran dry is not relied on to resume. */
+  const ensurePlayer = (): AudioBufferQueueSourceNode | null => {
+    if (player !== null || context === null) return player;
+    const node = context.createBufferQueueSource();
+    node.connect(context.destination);
+    node.onBufferEnded = ({ bufferId }) => {
+      if (stopped || node !== player) return;
+      const index = queued.findIndex((entry) => entry.id === bufferId);
+      if (index === -1) return;
+      const [ended] = queued.splice(index, 1);
+      if (ended && current?.itemId === ended.itemId) current.playedMs += ended.ms;
+      const next = queued[0];
+      if (next !== undefined) {
+        beginBuffer(next);
+        return;
+      }
+      releasePlayer();
+      handlers?.onPlaybackLevel(0);
+      handlers?.onPlaybackIdle();
+    };
+    node.start();
+    player = node;
+    return node;
+  };
+
+  const push = (itemId: string, pcm: Uint8Array): void => {
+    const node = ensurePlayer();
+    if (node === null || context === null) return;
+    const samples = pcm16ToFloat(pcm);
+    if (samples.length === 0) return;
+    const buffer = context.createBuffer(1, samples.length, VOICE_SAMPLE_RATE);
+    buffer.copyToChannel(samples as Float32Array<ArrayBuffer>, 0);
+    const id = node.enqueueBuffer(buffer);
+    const entry: Queued = {
+      id,
+      itemId,
+      ms: pcmBytesToMs(pcm.byteLength),
+      level: levelOfPcm16(pcm),
+    };
+    queued.push(entry);
+    if (queued.length === 1) beginBuffer(entry);
+  };
+
+  const releasePending = (): void => {
+    if (prebufferTimer !== null) clearTimeout(prebufferTimer);
+    prebufferTimer = null;
+    const chunks = pending;
+    pending = [];
+    pendingMs = 0;
+    for (const chunk of chunks) push(chunk.itemId, chunk.pcm);
+  };
+
+  return {
+    async start(nextHandlers: VoiceAudioEngineHandlers): Promise<void> {
+      handlers = nextHandlers;
+      AudioManager.setAudioSessionOptions({
+        iosCategory: 'playAndRecord',
+        iosMode: 'voiceChat',
+        iosOptions: ['defaultToSpeaker', 'allowBluetoothHFP'],
+      });
+      const permission = await AudioManager.requestRecordingPermissions().catch(() => 'Denied');
+      if (permission !== 'Granted') throw new VoiceAudioEngineError('mic-denied');
+      try {
+        await AudioManager.setAudioSessionActivity(true);
+      } catch {
+        throw new VoiceAudioEngineError('mic-unavailable');
+      }
+      if (stopped) return;
+
+      context = new AudioContext({ sampleRate: VOICE_SAMPLE_RATE });
+      recorder = new AudioRecorder();
+      const ready = recorder.onAudioReady(
+        { sampleRate: VOICE_SAMPLE_RATE, bufferLength: VOICE_FRAME_SAMPLES, channelCount: 1 },
+        ({ buffer }) => {
+          if (stopped) return;
+          const pcm = floatToPcm16(buffer.getChannelData(0));
+          handlers?.onCaptureLevel(levelOfPcm16(pcm));
+          handlers?.onCaptureFrame(pcm);
+        },
+      );
+      if (ready.status === 'error')
+        throw new VoiceAudioEngineError('mic-unavailable', ready.message);
+      const started = await recorder.start();
+      if (started.status === 'error')
+        throw new VoiceAudioEngineError('mic-unavailable', started.message);
+    },
+
+    enqueue(itemId: string, pcm16: Uint8Array): void {
+      if (stopped || context === null) return;
+      if (player !== null) {
+        push(itemId, pcm16);
+        return;
+      }
+      pending.push({ itemId, pcm: pcm16 });
+      pendingMs += pcmBytesToMs(pcm16.byteLength);
+      if (prebufferTimer !== null) clearTimeout(prebufferTimer);
+      if (pendingMs >= PREBUFFER_MS) releasePending();
+      else prebufferTimer = setTimeout(releasePending, PREBUFFER_IDLE_MS);
+    },
+
+    async flush(): Promise<{ itemId: string; playedMs: number } | null> {
+      if (prebufferTimer !== null) clearTimeout(prebufferTimer);
+      prebufferTimer = null;
+      pending = [];
+      pendingMs = 0;
+      const playing = current;
+      const head = queued[0];
+      let heard: { itemId: string; playedMs: number } | null = null;
+      if (playing !== null && context !== null) {
+        const intoBuffer =
+          head !== undefined && head.itemId === playing.itemId
+            ? Math.min(head.ms, Math.max(0, (context.currentTime - playing.bufferStartedAt) * 1000))
+            : 0;
+        heard = { itemId: playing.itemId, playedMs: Math.round(playing.playedMs + intoBuffer) };
+      }
+      queued = [];
+      current = null;
+      releasePlayer();
+      handlers?.onPlaybackLevel(0);
+      return heard;
+    },
+
+    stop(): void {
+      if (stopped) return;
+      stopped = true;
+      if (prebufferTimer !== null) clearTimeout(prebufferTimer);
+      releasePlayer();
+      if (recorder !== null) {
+        recorder.clearOnAudioReady();
+        void recorder.stop().catch(() => undefined);
+      }
+      void context?.close().catch(() => undefined);
+      void AudioManager.setAudioSessionActivity(false).catch(() => undefined);
+      recorder = null;
+      context = null;
+      handlers = null;
+      queued = [];
+      pending = [];
+    },
+  };
+}