@alia.onl/sdk 13.1.0 → 15.2.1

package/src/lib/dictation-recorder.ts

@@ -0,0 +1,131 @@
+/**
+ * Dictation's web recorder: `MediaRecorder` over `getUserMedia`, with an
+ * `AnalyserNode` on the same stream for the level meter. See
+ * `dictation-recorder-types.ts`.
+ */
+
+import { useMemo } from 'react';
+import { levelFromFrames } from './audio-level';
+import { normalizeAudioMediaType, type DictationClip } from './dictation';
+import { DictationRecorderError, type DictationRecorder } from './dictation-recorder-types';
+
+export * from './dictation-recorder-types';
+
+const LEVEL_POLL_MS = 100;
+
+/** Opus in WebM where the browser records it (Chromium, Firefox), MP4/AAC on Safari. */
+function recordingMimeType(): string | null {
+  const Recorder = globalThis.MediaRecorder;
+  if (!Recorder) return null;
+  for (const type of ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4']) {
+    if (Recorder.isTypeSupported?.(type)) return type;
+  }
+  return null;
+}
+
+export function createWebDictationRecorder(): DictationRecorder {
+  let stream: MediaStream | null = null;
+  let recorder: MediaRecorder | null = null;
+  let context: AudioContext | null = null;
+  let levelTimer: ReturnType<typeof setInterval> | null = null;
+  let chunks: Blob[] = [];
+  let startedAt = 0;
+
+  const release = (): void => {
+    if (levelTimer !== null) clearInterval(levelTimer);
+    levelTimer = null;
+    for (const track of stream?.getTracks() ?? []) track.stop();
+    void context?.close().catch(() => undefined);
+    stream = null;
+    context = null;
+  };
+
+  return {
+    async start(onLevel) {
+      const mimeType = recordingMimeType();
+      const media = globalThis.navigator?.mediaDevices;
+      if (mimeType === null || !media?.getUserMedia)
+        throw new DictationRecorderError('mic-unavailable');
+      try {
+        stream = await media.getUserMedia({
+          audio: { noiseSuppression: true, echoCancellation: true, autoGainControl: true },
+        });
+      } catch (caught: unknown) {
+        const name = (caught as { name?: unknown } | null)?.name;
+        throw new DictationRecorderError(
+          name === 'NotAllowedError' || name === 'SecurityError' ? 'mic-denied' : 'mic-unavailable',
+        );
+      }
+
+      const AudioContextCtor = globalThis.AudioContext;
+      if (AudioContextCtor) {
+        context = new AudioContextCtor();
+        const analyser = context.createAnalyser();
+        analyser.fftSize = 1024;
+        context.createMediaStreamSource(stream).connect(analyser);
+        const frames = new Float32Array(analyser.fftSize);
+        levelTimer = setInterval(() => {
+          analyser.getFloatTimeDomainData(frames);
+          onLevel(levelFromFrames(Array.from(frames)));
+        }, LEVEL_POLL_MS);
+      }
+
+      chunks = [];
+      const active = new MediaRecorder(stream, { mimeType });
+      recorder = active;
+      active.ondataavailable = (event) => {
+        if (event.data.size > 0) chunks.push(event.data);
+      };
+      await new Promise<void>((resolve, reject) => {
+        active.onstart = () => {
+          startedAt = Date.now();
+          resolve();
+        };
+        active.onerror = () => reject(new DictationRecorderError('mic-unavailable'));
+        try {
+          active.start();
+        } catch {
+          reject(new DictationRecorderError('mic-unavailable'));
+        }
+      }).catch((error: unknown) => {
+        release();
+        throw error;
+      });
+    },
+
+    stop(): Promise<DictationClip | null> {
+      const active = recorder;
+      recorder = null;
+      if (active === null || active.state === 'inactive') {
+        release();
+        return Promise.resolve(null);
+      }
+      return new Promise((resolve) => {
+        active.onstop = () => {
+          const durationMs = Date.now() - startedAt;
+          const mediaType = normalizeAudioMediaType(active.mimeType || 'audio/webm');
+          const blob = new Blob(chunks, { type: mediaType });
+          chunks = [];
+          release();
+          resolve(blob.size === 0 ? null : { file: blob, mediaType, durationMs });
+        };
+        active.stop();
+      });
+    },
+
+    cancel(): void {
+      const active = recorder;
+      recorder = null;
+      chunks = [];
+      if (active !== null && active.state !== 'inactive') {
+        active.onstop = null;
+        active.stop();
+      }
+      release();
+    },
+  };
+}
+
+export function useDictationRecorder(): DictationRecorder {
+  return useMemo(() => createWebDictationRecorder(), []);
+}

package/src/lib/dictation.ts

@@ -0,0 +1,208 @@
+/**
+ * Dictation's network half: a recorded clip to `POST /v1/audio/transcriptions`
+ * (ADR 0017 §7), which Alia answers with a Kaana transcription model computed
+ * from the catalogue. Free on every plan; at most five minutes per clip.
+ *
+ * Batch, not streaming: the text arrives once the person stops. Nothing is
+ * recognized on the device.
+ */
+
+import type { VoiceErrorCode } from './speech-messages';
+
+/** The longest clip one dictation may be. */
+export const MAX_DICTATION_MS = 5 * 60_000;
+
+/** A clip as a recorder hands it over: a web `Blob`, or a React Native file. */
+export interface DictationClip {
+  readonly file: Blob | { readonly uri: string; readonly name: string; readonly type: string };
+  /** Without parameters: `audio/webm`, `audio/mp4`, … */
+  readonly mediaType: string;
+  /** Measured by the recorder. */
+  readonly durationMs: number;
+}
+
+export interface TranscriptionResult {
+  readonly text: string;
+  readonly language?: string;
+  readonly durationMs?: number;
+  /** The model that transcribed it (its name may be shown). */
+  readonly model?: string;
+}
+
+/** What the route answers a failure with, plus the two this side knows on its own. */
+export type DictationErrorCode =
+  | 'TRANSCRIPTION_UNAVAILABLE'
+  | 'TRANSCRIPTION_LIMITED'
+  | 'TRANSCRIPTION_REJECTED'
+  | 'TRANSCRIPTION_FAILED'
+  | 'CLIP_TOO_LONG'
+  | 'INVALID_REQUEST'
+  | 'NOT_AUTHENTICATED'
+  | 'CREDENTIAL_NOT_SUPPORTED';
+
+const SERVER_CODES: ReadonlySet<string> = new Set<DictationErrorCode>([
+  'TRANSCRIPTION_UNAVAILABLE',
+  'TRANSCRIPTION_LIMITED',
+  'TRANSCRIPTION_REJECTED',
+  'TRANSCRIPTION_FAILED',
+  'CLIP_TOO_LONG',
+  'INVALID_REQUEST',
+  'NOT_AUTHENTICATED',
+  'CREDENTIAL_NOT_SUPPORTED',
+]);
+
+export class DictationError extends Error {
+  /** `null` when no request was answered (network) or the code is unknown. */
+  readonly code: DictationErrorCode | null;
+  readonly status: number | null;
+  readonly retryable: boolean;
+
+  constructor(
+    message: string,
+    details: { code: DictationErrorCode | null; status: number | null; retryable?: boolean },
+  ) {
+    super(message);
+    this.name = 'DictationError';
+    this.code = details.code;
+    this.status = details.status;
+    this.retryable = details.retryable ?? false;
+  }
+}
+
+/** The `VoiceErrorCode` an app says a failed transcription with. */
+export function dictationFailureCode(error: unknown): VoiceErrorCode {
+  if (!(error instanceof DictationError)) return 'transcription-failed';
+  switch (error.code) {
+    case 'TRANSCRIPTION_UNAVAILABLE':
+      return 'transcription-unavailable';
+    case 'TRANSCRIPTION_LIMITED':
+      return 'transcription-limited';
+    case 'TRANSCRIPTION_REJECTED':
+    case 'INVALID_REQUEST':
+      return 'transcription-rejected';
+    case 'CLIP_TOO_LONG':
+      return 'too-long';
+    case 'NOT_AUTHENTICATED':
+      return 'not-authenticated';
+    case 'CREDENTIAL_NOT_SUPPORTED':
+      return 'credential-not-supported';
+    case 'TRANSCRIPTION_FAILED':
+    case null:
+      return 'transcription-failed';
+  }
+}
+
+/** `audio/webm;codecs=opus` → `audio/webm`; iOS's m4a names → `audio/mp4`. */
+export function normalizeAudioMediaType(type: string): string {
+  const bare = type.split(';')[0]!.trim().toLowerCase();
+  if (bare === 'audio/x-m4a' || bare === 'audio/m4a' || bare === 'audio/aac-mp4')
+    return 'audio/mp4';
+  return bare;
+}
+
+function fileNameFor(mediaType: string): string {
+  switch (mediaType) {
+    case 'audio/webm':
+      return 'dictation.webm';
+    case 'audio/ogg':
+      return 'dictation.ogg';
+    case 'audio/mpeg':
+      return 'dictation.mp3';
+    case 'audio/wav':
+      return 'dictation.wav';
+    default:
+      return 'dictation.m4a';
+  }
+}
+
+export interface TranscriptionRequest {
+  readonly apiUrl: string;
+  readonly token: string;
+  readonly clip: DictationClip;
+  /** BCP-47. */
+  readonly language: string;
+  readonly signal?: AbortSignal;
+}
+
+/**
+ * Transcribe one clip.
+ *
+ * @throws {DictationError} for every refusal; an abort is thrown as `fetch` threw it.
+ */
+export async function requestTranscription(
+  request: TranscriptionRequest,
+): Promise<TranscriptionResult> {
+  if (request.token === '')
+    throw new DictationError('Not authenticated', { code: 'NOT_AUTHENTICATED', status: null });
+  const mediaType = normalizeAudioMediaType(request.clip.mediaType);
+  const form = new FormData();
+  const { file } = request.clip;
+  if (typeof Blob !== 'undefined' && file instanceof Blob) {
+    form.append('file', file, fileNameFor(mediaType));
+  } else {
+    // React Native's FormData takes a file by `{ uri, name, type }`.
+    form.append('file', file as unknown as Blob);
+  }
+  form.append('language', request.language);
+  form.append('durationMs', String(Math.max(0, Math.round(request.clip.durationMs))));
+
+  let response: Response;
+  try {
+    response = await fetch(`${request.apiUrl}/v1/audio/transcriptions`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${request.token}` },
+      body: form,
+      ...(request.signal ? { signal: request.signal } : {}),
+    });
+  } catch (caught: unknown) {
+    if ((caught as { name?: unknown } | null)?.name === 'AbortError') throw caught;
+    throw new DictationError('The transcription request did not reach Alia', {
+      code: null,
+      status: null,
+      retryable: true,
+    });
+  }
+
+  const body: unknown = await response.json().catch(() => null);
+  if (!response.ok) {
+    const envelope =
+      body !== null && typeof body === 'object' ? (body as { error?: unknown }).error : undefined;
+    const error =
+      envelope !== null && typeof envelope === 'object'
+        ? (envelope as { code?: unknown; message?: unknown; retryable?: unknown })
+        : {};
+    const named =
+      typeof error.code === 'string' && SERVER_CODES.has(error.code)
+        ? (error.code as DictationErrorCode)
+        : null;
+    throw new DictationError(
+      typeof error.message === 'string' && error.message !== ''
+        ? error.message
+        : 'Transcription failed',
+      {
+        code:
+          named ??
+          (response.status === 401
+            ? 'NOT_AUTHENTICATED'
+            : response.status === 413
+              ? 'CLIP_TOO_LONG'
+              : null),
+        status: response.status,
+        retryable: error.retryable === true,
+      },
+    );
+  }
+  const result = body !== null && typeof body === 'object' ? (body as Record<string, unknown>) : {};
+  if (typeof result.text !== 'string') {
+    throw new DictationError('Alia answered no transcription', {
+      code: 'TRANSCRIPTION_FAILED',
+      status: response.status,
+    });
+  }
+  return {
+    text: result.text,
+    ...(typeof result.language === 'string' ? { language: result.language } : {}),
+    ...(typeof result.durationMs === 'number' ? { durationMs: result.durationMs } : {}),
+    ...(typeof result.model === 'string' ? { model: result.model } : {}),
+  };
+}

package/src/lib/labels.tsx

@@ -1,4 +1,5 @@
 import { createContext, type ReactNode, useContext, useMemo } from 'react';
+import { VOICE_ERROR_MESSAGES, type VoiceErrorCode } from './speech-messages';
 import { englishWorkLog, type Translate } from './work-log-copy';
 
 /** Every string the chat UI draws itself. Bloom's own copy follows `BloomProvider locale`. */
@@ -15,6 +16,34 @@ export interface AliaChatLabels {
   addPhotos: string;
   startVoice: string;
   voiceFailed: string;
+  /** The field's accessible name. */
+  composerMessage: string;
+  /** The add button's accessible name. */
+  composerAddMenu: string;
+  /** The mode selector's accessible name (the host's power levels). */
+  composerModes: string;
+  /** The dictation button's accessible name. */
+  composerVoice: string;
+  composerSend: string;
+  composerStop: string;
+  /** A tile's remove button. */
+  composerRemove: string;
+  /** Bloom names a retry button "<this> <file name>". */
+  composerRetry: string;
+  /** What a tile without a file name is called. */
+  composerUntitled: string;
+  fileReadFailed: (name: string) => string;
+  fileTooLarge: (name: string, limit: string) => string;
+  fileEmpty: (name: string) => string;
+  /** A file of a kind the chat cannot send (a document, where only photos go). */
+  fileUnsupported: (name: string) => string;
+  /** Over the composer while a file is dragged onto it. */
+  dropHint: string;
+  dropUnavailable: string;
+  /** Why dictation failed, in the host's language. */
+  voiceError: (code: VoiceErrorCode) => string;
+  /** Over the composer in the last 30 s of a dictation's five minutes. */
+  dictationRemaining: (seconds: number) => string;
   // Welcome
   /** The empty chat's greeting for a signed-out visitor. */
   greeting: string;
@@ -56,6 +85,23 @@ export const DEFAULT_LABELS: AliaChatLabels = {
   addPhotos: 'Photos',
   startVoice: 'Start voice',
   voiceFailed: "Couldn't start the voice call",
+  composerMessage: 'Message',
+  composerAddMenu: 'Add to chat',
+  composerModes: 'Power level',
+  composerVoice: 'Dictate',
+  composerSend: 'Send message',
+  composerStop: 'Stop generating',
+  composerRemove: 'Remove',
+  composerRetry: 'Retry',
+  composerUntitled: 'untitled',
+  fileReadFailed: (name) => `${name} could not be read — select to try again`,
+  fileTooLarge: (name, limit) => `${name} is larger than ${limit} and was not attached`,
+  fileEmpty: (name) => `${name} is empty — folders can't be attached`,
+  fileUnsupported: (name) => `${name} can't be sent here — only photos can`,
+  dropHint: 'Drop files here to attach them',
+  dropUnavailable: "Files can't be attached right now",
+  voiceError: (code) => VOICE_ERROR_MESSAGES[code],
+  dictationRemaining: (seconds) => `${seconds}s of dictation left`,
   greeting: "Hi, what's on your mind?",
   greetingNamed: (name) => `Hi ${name}, what's on your mind?`,
   copy: 'Copy',
@@ -87,7 +133,10 @@ export function AliaChatLabelsProvider({
   labels?: Partial<AliaChatLabels>;
   children: ReactNode;
 }) {
-  const value = useMemo(() => ({ ...DEFAULT_LABELS, ...labels }), [labels]);
+  // Over whatever labels are already in reach, so a part that takes its own `labels` (the
+  // composer) refines the chat's copy instead of resetting it to English.
+  const inherited = useContext(LabelsContext);
+  const value = useMemo(() => ({ ...inherited, ...labels }), [inherited, labels]);
   return <LabelsContext.Provider value={value}>{children}</LabelsContext.Provider>;
 }
 

package/src/lib/pcm.ts

@@ -0,0 +1,57 @@
+/**
+ * PCM16 in and out of the -1..1 floats audio engines work in, and the level a
+ * buffer reads at. No imports, so every engine (and the tests) shares one
+ * definition of the conversion and of the meter.
+ */
+
+import { levelFromFrames } from './audio-level';
+
+/** Floats (-1..1) to PCM16 little-endian bytes. */
+export function floatToPcm16(samples: ArrayLike<number>): Uint8Array {
+  const bytes = new Uint8Array(samples.length * 2);
+  const view = new DataView(bytes.buffer);
+  for (let index = 0; index < samples.length; index += 1) {
+    const clamped = Math.max(-1, Math.min(1, samples[index] ?? 0));
+    view.setInt16(index * 2, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
+  }
+  return bytes;
+}
+
+/** PCM16 little-endian bytes to floats (-1..1). A trailing odd byte is dropped. */
+export function pcm16ToFloat(bytes: Uint8Array): Float32Array {
+  const count = Math.floor(bytes.byteLength / 2);
+  const view = new DataView(bytes.buffer, bytes.byteOffset, count * 2);
+  const samples = new Float32Array(count);
+  for (let index = 0; index < count; index += 1) {
+    samples[index] = view.getInt16(index * 2, true) / 0x8000;
+  }
+  return samples;
+}
+
+/**
+ * Linear resampling. Good enough for speech in a call: the model hears 24 kHz
+ * and the device plays at its own rate; neither is a recording kept.
+ */
+export function resampleLinear(
+  samples: Float32Array,
+  fromRate: number,
+  toRate: number,
+): Float32Array {
+  if (fromRate === toRate || samples.length === 0) return samples;
+  const ratio = fromRate / toRate;
+  const length = Math.max(1, Math.floor(samples.length / ratio));
+  const out = new Float32Array(length);
+  for (let index = 0; index < length; index += 1) {
+    const position = index * ratio;
+    const left = Math.floor(position);
+    const right = Math.min(left + 1, samples.length - 1);
+    const fraction = position - left;
+    out[index] = (samples[left] ?? 0) * (1 - fraction) + (samples[right] ?? 0) * fraction;
+  }
+  return out;
+}
+
+/** The field's 0..1 level of a PCM16 buffer, on the dictation meter's dBFS scale. */
+export function levelOfPcm16(bytes: Uint8Array): number {
+  return levelFromFrames(Array.from(pcm16ToFloat(bytes)));
+}

package/src/lib/speech-language.ts

@@ -1,5 +1,5 @@
 /**
- * The language to recognize when the caller names none.
+ * The language spoken, when the caller names none (dictation and calls).
  *
  * The browser's own language on web; the runtime's resolved locale on native
  * (Hermes implements `Intl`); `en-US` when neither answers. An app should pass

package/src/lib/speech-messages.ts

@@ -1,76 +1,54 @@
-import type { SpeechRecognitionFailure } from './speech-recognition-types';
+import type { VoiceServerErrorCode } from './voice-protocol';
 
 /**
  * Every way dictation or a voice call can fail that a person is told about,
- * as a stable code.
+ * as a stable code (ADR 0017 §3, §7).
  *
- * The hooks expose the code beside the English sentence (`errorCode` /
- * `turnErrorCode` next to `error` / `turnError`) so an app can say it in its
- * own language; the sentence stays for a consumer that has no translations.
+ * - **Alia's**, sent by `/v1/voice/session` when it ends a call:
+ *   `VoiceServerErrorCode` (`plan-required`, `credits-exhausted`,
+ *   `realtime-*`, …).
+ * - **The device's:** `mic-denied`, `mic-unavailable`, `connection-lost`, and
+ *   `voice-unavailable` where this platform has no audio engine for a call.
+ * - **Dictation's** (`POST /v1/audio/transcriptions`): `too-long` and
+ *   `transcription-*`, beside the shared `mic-*` and `not-authenticated`.
+ *
+ * The hooks expose the code beside the English sentence so an app can say it in
+ * its own language; the sentence stays for a consumer with no translations.
+ * Nothing here is a recognizer's: speech is recognized by Kaana, never by the
+ * browser or the operating system.
  */
 export type VoiceErrorCode =
-  | 'speech-unsupported'
-  | 'microphone-denied'
-  | 'microphone-busy'
-  | 'microphone-missing'
-  | 'speech-network'
-  | 'speech-language'
-  | 'speech-failed'
+  | VoiceServerErrorCode
+  | 'mic-denied'
+  | 'mic-unavailable'
+  | 'connection-lost'
   | 'voice-unavailable'
-  | 'not-authenticated'
-  | 'turn-failed'
-  | 'no-answer'
-  | 'not-played';
+  | 'too-long'
+  | 'transcription-unavailable'
+  | 'transcription-limited'
+  | 'transcription-rejected'
+  | 'transcription-failed';
 
-/**
- * The English copy for each code — the same sentences dictation and voice mode
- * used when they recorded through `expo-audio` and LiveKit, so moving
- * recognition onto the device did not change what an error says.
- */
+/** The English copy for each code. */
 export const VOICE_ERROR_MESSAGES: Readonly<Record<VoiceErrorCode, string>> = {
-  'speech-unsupported': 'Speech recognition is not available on this device or browser',
-  'microphone-denied': 'Microphone permission required',
-  'microphone-busy': 'Microphone is in use by another application',
-  'microphone-missing': 'No microphone found — check your input devices',
-  'speech-network': 'Speech recognition needs a network connection',
-  'speech-language': 'Speech recognition does not support this language',
-  'speech-failed': 'Speech recognition failed',
-  'voice-unavailable': 'Voice is not available on this device',
   'not-authenticated': 'Not authenticated',
-  'turn-failed': 'Voice request failed',
-  'no-answer': 'No answer came back — try saying it again',
-  'not-played': 'The answer could not be played aloud',
+  'credential-not-supported': 'Voice is only available when you are signed in to Alia',
+  'plan-required': 'Voice mode needs a paid plan',
+  'credits-exhausted': "You've run out of credits for this call",
+  'agent-unavailable': 'This agent is not available for a call',
+  'invalid-request': 'The call could not be started',
+  'resume-unavailable': 'The call could not be picked up again — start a new one',
+  'realtime-unavailable': 'Voice is not available right now — try again later',
+  'realtime-limited': 'Voice has reached its limit for now — try again later',
+  'realtime-rejected': 'The call could not be started with these settings',
+  'realtime-failed': 'The call failed — try again',
+  'mic-denied': 'Microphone permission required',
+  'mic-unavailable': 'The microphone could not be used — check your input devices',
+  'connection-lost': 'The connection was lost',
+  'voice-unavailable': 'Voice is not available on this device or browser',
+  'too-long': 'That dictation was too long — five minutes at most',
+  'transcription-unavailable': 'Dictation is not available right now',
+  'transcription-limited': 'Dictation has reached its limit for now — try again later',
+  'transcription-rejected': 'That recording could not be transcribed',
+  'transcription-failed': 'Dictation failed — try again',
 };
-
-/**
- * The code for a recognition failure. `null` for the failures that are not
- * errors: `aborted` is a stop the caller asked for, and `no-speech` is a
- * silence the caller recovers from.
- */
-export function speechFailureCode(failure: SpeechRecognitionFailure): VoiceErrorCode | null {
-  switch (failure.code) {
-    case 'aborted':
-    case 'no-speech':
-      return null;
-    case 'unsupported':
-      return 'speech-unsupported';
-    case 'not-allowed':
-      return 'microphone-denied';
-    case 'audio-capture':
-      return failure.detail === 'NotReadableError' || failure.detail === 'TrackStartError'
-        ? 'microphone-busy'
-        : 'microphone-missing';
-    case 'network':
-      return 'speech-network';
-    case 'language-not-supported':
-      return 'speech-language';
-    case 'other':
-      return 'speech-failed';
-  }
-}
-
-/** The English copy a person sees when listening fails; `null` as for `speechFailureCode`. */
-export function speechFailureMessage(failure: SpeechRecognitionFailure): string | null {
-  const code = speechFailureCode(failure);
-  return code === null ? null : VOICE_ERROR_MESSAGES[code];
-}