@alia.onl/sdk 13.1.0 → 15.2.1

package/src/lib/speech-recognition.ts

@@ -1,308 +0,0 @@
-/**
- * On-device speech recognition for the web: the browser's Web Speech API.
- *
- * Metro resolves `speech-recognition.native.ts` on iOS and Android, so this
- * file is what a web bundle — and every test — gets. See
- * `speech-recognition-types.ts` for the contract both implement.
- *
- * ## Why a microphone stream is opened beside the recognizer
- *
- * The Web Speech API reports words, not sound: there is no level event, and a
- * refused or missing microphone surfaces as a bare `not-allowed` or
- * `audio-capture` with nothing to tell "blocked" from "busy". Opening the
- * microphone first with `getUserMedia` answers both. It is where the browser
- * asks for permission (once — the recognizer shares the origin's grant), its
- * DOMException names say exactly what is wrong, and an `AnalyserNode` on the
- * stream is the honest level the dictation trace and the voice field move to.
- * The stream is never recorded or sent anywhere.
- *
- * ## Support
- *
- * Chromium browsers and Safari implement it (Chrome as `webkitSpeechRecognition`
- * and through a Google speech service; Safari through Apple's). Firefox does
- * not, and `isSpeechRecognitionAvailable()` says so before anything asks for a
- * microphone.
- */
-
-import { levelFromDbfs } from './audio-level';
-import type {
-  SpeechRecognitionFailure,
-  SpeechRecognitionFailureCode,
-  SpeechRecognitionHandlers,
-  SpeechRecognitionOptions,
-  SpeechRecognitionSession,
-  SpeechRecognizerChoice,
-} from './speech-recognition-types';
-
-// ── The slice of the Web Speech API this uses ──
-// Declared here rather than taken from lib.dom: TypeScript's DOM library does
-// not ship the prefixed constructor, and the unprefixed one is not everywhere.
-
-interface WebSpeechAlternative {
-  readonly transcript: string;
-}
-interface WebSpeechResult {
-  readonly isFinal: boolean;
-  readonly length: number;
-  readonly [index: number]: WebSpeechAlternative;
-}
-interface WebSpeechResultList {
-  readonly length: number;
-  readonly [index: number]: WebSpeechResult;
-}
-interface WebSpeechResultEvent {
-  readonly results: WebSpeechResultList;
-}
-interface WebSpeechErrorEvent {
-  readonly error: string;
-  readonly message?: string;
-}
-
-interface WebSpeechRecognition {
-  lang: string;
-  continuous: boolean;
-  interimResults: boolean;
-  maxAlternatives: number;
-  onresult: ((event: WebSpeechResultEvent) => void) | null;
-  onerror: ((event: WebSpeechErrorEvent) => void) | null;
-  onend: (() => void) | null;
-  start(): void;
-  stop(): void;
-  abort(): void;
-}
-
-type WebSpeechRecognitionConstructor = new () => WebSpeechRecognition;
-
-function recognitionConstructor(): WebSpeechRecognitionConstructor | null {
-  const scope = globalThis as unknown as {
-    SpeechRecognition?: WebSpeechRecognitionConstructor;
-    webkitSpeechRecognition?: WebSpeechRecognitionConstructor;
-  };
-  return scope.SpeechRecognition ?? scope.webkitSpeechRecognition ?? null;
-}
-
-/** Whether this browser can recognize speech at all. Firefox cannot. */
-export function isSpeechRecognitionAvailable(): boolean {
-  return recognitionConstructor() !== null;
-}
-
-/**
- * Ask for the microphone before a session needs it.
- *
- * The browser asks once per origin; `startSpeechRecognition` opens the stream
- * itself, so this is only for callers that want the answer up front (voice mode
- * does, so a refusal ends the call before it starts rather than on the first
- * word).
- */
-export async function requestSpeechRecognitionPermission(): Promise<SpeechRecognitionFailure | null> {
-  if (!isSpeechRecognitionAvailable()) return { code: 'unsupported' };
-  const media = globalThis.navigator?.mediaDevices;
-  if (media?.getUserMedia === undefined) return null;
-  try {
-    const stream = await media.getUserMedia({ audio: true });
-    for (const track of stream.getTracks()) track.stop();
-    return null;
-  } catch (error: unknown) {
-    return mediaFailure(error);
-  }
-}
-
-function domErrorName(error: unknown): string {
-  // A DOMException is not always `instanceof Error`, so read `.name` off the object.
-  if (
-    typeof error === 'object' &&
-    error !== null &&
-    'name' in error &&
-    typeof error.name === 'string'
-  ) {
-    return error.name;
-  }
-  return '';
-}
-
-function mediaFailure(error: unknown): SpeechRecognitionFailure {
-  const name = domErrorName(error);
-  switch (name) {
-    case 'NotAllowedError':
-    case 'PermissionDeniedError':
-    case 'SecurityError':
-      return { code: 'not-allowed', detail: name };
-    case 'NotFoundError':
-    case 'DevicesNotFoundError':
-    case 'OverconstrainedError':
-    case 'NotReadableError':
-    case 'TrackStartError':
-      return { code: 'audio-capture', detail: name };
-    default:
-      return { code: 'other', detail: name || 'getUserMedia failed' };
-  }
-}
-
-function recognitionFailure(event: WebSpeechErrorEvent): SpeechRecognitionFailure {
-  const map: Record<string, SpeechRecognitionFailureCode> = {
-    'not-allowed': 'not-allowed',
-    'service-not-allowed': 'not-allowed',
-    'audio-capture': 'audio-capture',
-    network: 'network',
-    'language-not-supported': 'language-not-supported',
-    'no-speech': 'no-speech',
-    aborted: 'aborted',
-  };
-  return { code: map[event.error] ?? 'other', detail: event.error };
-}
-
-/** Everything heard so far, as one string: Chrome segments carry their own leading spaces. */
-function transcriptOf(results: WebSpeechResultList): { transcript: string; isFinal: boolean } {
-  let transcript = '';
-  let isFinal = results.length > 0;
-  for (let index = 0; index < results.length; index += 1) {
-    const result = results[index];
-    if (result === undefined) continue;
-    transcript += result[0]?.transcript ?? '';
-    if (!result.isFinal) isFinal = false;
-  }
-  return { transcript: transcript.replace(/\s+/g, ' ').trim(), isFinal };
-}
-
-const LEVEL_INTERVAL_MS = 100;
-
-/** An analyser on the microphone, polled on a clock. Returns its teardown. */
-function meterStream(stream: MediaStream, onLevel: (level: number) => void): () => void {
-  const AudioContextCtor = (globalThis as unknown as { AudioContext?: typeof AudioContext })
-    .AudioContext;
-  if (AudioContextCtor === undefined) return () => undefined;
-  let context: AudioContext;
-  try {
-    context = new AudioContextCtor();
-  } catch {
-    return () => undefined;
-  }
-  if (context.state === 'suspended') context.resume().catch(() => undefined);
-  const analyser = context.createAnalyser();
-  analyser.fftSize = 512;
-  context.createMediaStreamSource(stream).connect(analyser);
-  const buffer = new Float32Array(analyser.fftSize);
-  const timer = setInterval(() => {
-    analyser.getFloatTimeDomainData(buffer);
-    let sum = 0;
-    for (let index = 0; index < buffer.length; index += 1) sum += buffer[index] * buffer[index];
-    const rms = Math.sqrt(sum / buffer.length);
-    onLevel(rms > 0 ? levelFromDbfs(20 * Math.log10(rms)) : 0);
-  }, LEVEL_INTERVAL_MS);
-  return () => {
-    clearInterval(timer);
-    context.close().catch(() => undefined);
-  };
-}
-
-/**
- * Listen until stopped.
- *
- * Continuous with interim results: the caller decides when an utterance is
- * over (voice mode on a silence timer, dictation on the stop button), because
- * the browser's own end-of-speech detection cuts people off mid-thought.
- */
-/**
- * The browser has one recognizer and no way to ask it which languages it
- * knows: the language goes as given (see the native file for Android's).
- */
-export async function chooseSpeechRecognizer(lang: string): Promise<SpeechRecognizerChoice> {
-  return { lang, silent: false };
-}
-
-export function startSpeechRecognition(
-  options: SpeechRecognitionOptions,
-  handlers: SpeechRecognitionHandlers,
-): SpeechRecognitionSession {
-  let recognition: WebSpeechRecognition | null = null;
-  let stream: MediaStream | null = null;
-  let stopMeter: (() => void) | null = null;
-  let ended = false;
-  let stopRequested: 'stop' | 'abort' | null = null;
-
-  const release = (): void => {
-    stopMeter?.();
-    stopMeter = null;
-    if (stream !== null) for (const track of stream.getTracks()) track.stop();
-    stream = null;
-  };
-
-  const finish = (): void => {
-    if (ended) return;
-    ended = true;
-    release();
-    handlers.onLevel?.(0);
-    handlers.onEnd();
-  };
-
-  const fail = (failure: SpeechRecognitionFailure): void => {
-    if (ended) return;
-    handlers.onError(failure);
-    finish();
-  };
-
-  void (async () => {
-    const Recognition = recognitionConstructor();
-    if (Recognition === null) {
-      fail({ code: 'unsupported' });
-      return;
-    }
-
-    const media = globalThis.navigator?.mediaDevices;
-    if (media?.getUserMedia !== undefined) {
-      try {
-        stream = await media.getUserMedia({
-          audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
-        });
-      } catch (error: unknown) {
-        fail(mediaFailure(error));
-        return;
-      }
-      if (stopRequested !== null) {
-        finish();
-        return;
-      }
-      if (handlers.onLevel !== undefined) stopMeter = meterStream(stream, handlers.onLevel);
-    }
-
-    const instance = new Recognition();
-    recognition = instance;
-    instance.lang = options.lang;
-    instance.continuous = true;
-    instance.interimResults = true;
-    instance.maxAlternatives = 1;
-    instance.onresult = (event) => {
-      if (ended) return;
-      handlers.onResult(transcriptOf(event.results));
-    };
-    instance.onerror = (event) => {
-      if (ended) return;
-      const failure = recognitionFailure(event);
-      if (failure.code === 'aborted') return;
-      handlers.onError(failure);
-    };
-    instance.onend = finish;
-    try {
-      instance.start();
-    } catch (error: unknown) {
-      fail({ code: 'other', detail: domErrorName(error) || 'start failed' });
-      return;
-    }
-    if (stopRequested === 'stop') instance.stop();
-    if (stopRequested === 'abort') instance.abort();
-  })();
-
-  return {
-    stop() {
-      if (ended) return;
-      stopRequested = 'stop';
-      recognition?.stop();
-    },
-    abort() {
-      if (ended) return;
-      stopRequested = 'abort';
-      if (recognition === null) return;
-      recognition.abort();
-    },
-  };
-}

package/src/lib/voice-text.ts

@@ -1,198 +0,0 @@
-/**
- * Text in and out of a voice call, as pure functions: what of an answer is
- * ready to be spoken, how it is made speakable, and whether what the
- * microphone just heard was the person or the call's own voice.
- *
- * Imports nothing, so every rule here is asserted directly
- * (`tests/lib/voice-text.test.ts`) rather than through a hook.
- */
-
-// ============== TITLE TAGS ==============
-
-const TAG = String.raw`ALIA_TITLE|TITLE|TÍTULO|TITRE|TITOLO|TITEL|ЗАГОЛОВОК`;
-
-const TITLE_STRIP_RE = new RegExp(String.raw`\[(${TAG})\].*?\[\/\1\]|<(${TAG})>.*?<\/\2>`, 'gi');
-
-const TITLE_PARTIAL_RE = new RegExp(
-  String.raw`\[(${TAG})\].*?(\[\/\1\])?$|<(${TAG})>.*?(<\/\3>)?$`,
-  'si',
-);
-
-/**
- * A settled answer without the conversation-title tag a model may append.
- *
- * `trim: false` keeps surrounding whitespace, which the speech chunker needs:
- * a trailing space is how it knows a period ended a sentence.
- */
-export function stripTitleTags(content: string, { trim = true }: { trim?: boolean } = {}): string {
-  const stripped = content.replace(TITLE_STRIP_RE, '');
-  return trim ? stripped.trim() : stripped;
-}
-
-/** The same, for an answer still streaming: an opened tag hides the rest. */
-export function stripTitleTagsPartial(
-  content: string,
-  { trim = true }: { trim?: boolean } = {},
-): string {
-  const stripped = content.replace(TITLE_STRIP_RE, '').replace(TITLE_PARTIAL_RE, '');
-  return trim ? stripped.trim() : stripped;
-}
-
-// ============== SPEAKABLE TEXT ==============
-
-/**
- * Markdown and links read aloud are noise ("asterisk asterisk", a URL spelled
- * out). The voice prompt asks for none, but a model does not always listen, and
- * an agent's own prompt may not ask at all — so the formatting is taken off
- * here, at the last moment, and the transcript on screen keeps it.
- */
-export function toSpeakableText(text: string): string {
-  return text
-    .replace(/```[\s\S]*?(```|$)/g, ' ')
-    .replace(/`([^`]*)`/g, '$1')
-    .replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1')
-    .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
-    .replace(/https?:\/\/\S+/g, ' ')
-    .replace(/^\s{0,3}#{1,6}\s+/gm, '')
-    .replace(/^\s*[-*+•]\s+/gm, '')
-    .replace(/^\s*>\s?/gm, '')
-    .replace(/\|/g, ' ')
-    .replace(/(\*\*|__|\*|_|~~)(?=\S)([\s\S]*?\S)\1/g, '$2')
-    .replace(/\s+/g, ' ')
-    .trim();
-}
-
-/**
- * A sentence ends at terminal punctuation followed by whitespace, or at a line
- * break. Requiring the whitespace is what keeps `3.5` and a half-streamed
- * `e.g` whole: until the next character arrives, the period might not be one.
- */
-const SENTENCE_END = /[.!?…。！？]+["'”’)\]]*\s+|\n+/g;
-
-/** Past this, a run with no sentence end is cut at a pause anyway. */
-const MAX_RUN_CHARS = 280;
-/** After the first, chunks gather sentences until this long: fewer, fuller requests. */
-const MIN_CHUNK_CHARS = 80;
-
-export interface SpeechChunks {
-  /** Ready to synthesize, in order, already speakable. */
-  readonly chunks: string[];
-  /** Not yet ready: the caller keeps it and passes it again with more text. */
-  readonly rest: string;
-}
-
-/**
- * Split what has streamed so far into what can be spoken now and what must
- * wait.
- *
- * The FIRST chunk of an answer is a single sentence, however short: time to
- * first sound is the latency a person feels. Later chunks gather sentences up
- * to `MIN_CHUNK_CHARS`, because each chunk is a synthesis request and playback
- * of the one before covers the wait for the next. `final` flushes everything.
- */
-export function takeSpeechChunks(
-  pending: string,
-  { final, isFirst }: { final: boolean; isFirst: boolean },
-): SpeechChunks {
-  const sentences: string[] = [];
-  let cursor = 0;
-  for (const match of pending.matchAll(SENTENCE_END)) {
-    const end = (match.index ?? 0) + match[0].length;
-    sentences.push(pending.slice(cursor, end));
-    cursor = end;
-  }
-  let rest = pending.slice(cursor);
-
-  if (final) {
-    if (rest.trim() !== '') sentences.push(rest);
-    rest = '';
-  } else if (rest.length > MAX_RUN_CHARS) {
-    const window = rest.slice(0, MAX_RUN_CHARS);
-    const pause = Math.max(
-      window.lastIndexOf(', '),
-      window.lastIndexOf('; '),
-      window.lastIndexOf(': '),
-    );
-    const cut = pause > 0 ? pause + 2 : Math.max(window.lastIndexOf(' '), 1);
-    sentences.push(rest.slice(0, cut));
-    rest = rest.slice(cut);
-  }
-
-  const chunks: string[] = [];
-  let gathering = '';
-  for (const sentence of sentences) {
-    gathering += sentence;
-    const threshold = chunks.length === 0 && isFirst ? 1 : MIN_CHUNK_CHARS;
-    if (gathering.trim().length >= threshold) {
-      chunks.push(gathering);
-      gathering = '';
-    }
-  }
-  if (gathering !== '') {
-    if (final) chunks.push(gathering);
-    else rest = gathering + rest;
-  }
-
-  return {
-    chunks: chunks.map(toSpeakableText).filter((chunk) => chunk !== ''),
-    rest,
-  };
-}
-
-// ============== BARGE-IN ==============
-
-function words(text: string): string[] {
-  return text
-    .toLowerCase()
-    .replace(/[^\p{L}\p{N}\s']/gu, ' ')
-    .split(/\s+/)
-    .filter((word) => word !== '');
-}
-
-export function wordCount(text: string): number {
-  return words(text).length;
-}
-
-/** How many of the most recent words decide whether the person is talking. */
-const INTERRUPTION_WINDOW = 6;
-
-/**
- * Whether what the microphone heard is the person talking over the answer,
- * rather than the answer coming back through it.
- *
- * Voice mode keeps listening while it speaks so a person can interrupt, and on
- * a laptop speaker the recognizer can hear the answer. Where the platform
- * cancels echo that never reaches here; where it does not, the words give it
- * away. `spoken` is everything the call has said this turn, and the transcript
- * of one session keeps the echo it already heard, so only the latest words
- * are judged, from the first one the call did not say: at least `minWords` of
- * them — and at least half — must be new. A single misheard word of echo is
- * not the person; two new words in a row are.
- */
-export function isInterruption(heard: string, spoken: string, minWords: number): boolean {
-  const said = new Set(words(spoken));
-  const tail = words(heard).slice(-INTERRUPTION_WINDOW);
-  // Judge from the first word the call did not say: echo before it is echo.
-  const first = tail.findIndex((word) => !said.has(word));
-  if (first < 0) return false;
-  const candidate = tail.slice(first);
-  const novel = candidate.filter((word) => !said.has(word)).length;
-  return novel >= minWords && novel * 2 >= candidate.length;
-}
-
-/**
- * The person's words without the echo the same session heard before them:
- * leading words the call itself had just said are dropped.
- */
-export function dropEchoPrefix(heard: string, spoken: string): string {
-  const said = new Set(words(spoken));
-  if (said.size === 0) return heard.trim();
-  const tokens = heard.trim().split(/\s+/);
-  let start = 0;
-  while (start < tokens.length) {
-    const normalized = words(tokens[start] ?? '');
-    if (normalized.length === 0 || normalized.every((word) => said.has(word))) start += 1;
-    else break;
-  }
-  return tokens.slice(start).join(' ');
-}

package/src/lib/voice-turn.ts

@@ -1,89 +0,0 @@
-/**
- * How a voice call gets its answers: through the ordinary chat path.
- *
- * A spoken turn is a chat turn. There is no voice session endpoint and no
- * realtime model behind this — the person's words, recognized on the device,
- * go to Alia's chat handler exactly as typed words do (conversation, memory,
- * agents, tools and all), and the answer comes back as the same SSE stream.
- * The only thing that marks it as spoken is `responseMode: 'voice'`, which asks
- * for an answer meant to be heard.
- *
- * A consumer that already owns a chat — the Alia app, whose conversation screen
- * persists every turn and shows it in the thread — passes its own sender, so
- * the call's turns ARE that conversation's turns. `createAliaVoiceTurnSender`
- * is the default for everyone else: the same request `useAliaChat` makes.
- */
-
-import { streamAliaChat, type AuthenticatedResponseClient } from './chat-transport';
-import { resolveModelId } from './catalogue';
-
-export interface VoiceTurnMessage {
-  readonly role: 'user' | 'assistant';
-  readonly content: string;
-}
-
-export interface VoiceTurn {
-  /** What the person said, as recognized. */
-  readonly text: string;
-  /** Earlier turns of this call, oldest first, not including `text`. */
-  readonly history: readonly VoiceTurnMessage[];
-  /** Aborted when the person interrupts or the call ends. */
-  readonly signal: AbortSignal;
-  /** Report the answer as it streams — the WHOLE answer so far, each time. */
-  readonly onText: (answerSoFar: string) => void;
-}
-
-/**
- * Send one spoken turn and stream its answer through `onText`.
- *
- * Resolves when the answer is complete; rejects when the turn could not be
- * sent at all. An answer that is empty is not a rejection — the voice loop
- * reports it and keeps listening.
- */
-export type VoiceTurnSender = (turn: VoiceTurn) => Promise<void>;
-
-interface LinkedClientFactory {
-  createLinkedClient(config: { baseURL: string }): {
-    client: AuthenticatedResponseClient;
-    dispose(): void;
-  };
-}
-
-export interface AliaVoiceTurnSenderOptions {
-  readonly oxyServices: LinkedClientFactory;
-  readonly apiUrl: string;
-  /** `publisher/model`, checked against `GET /catalogue` like `useAliaChat`'s; omitted, the server's default. */
-  readonly model?: string;
-  readonly agentId?: string;
-}
-
-/** The default sender: `POST /v1/chat/completions` as `useAliaChat` sends it, marked as voice. */
-export function createAliaVoiceTurnSender(options: AliaVoiceTurnSenderOptions): VoiceTurnSender {
-  const { oxyServices, apiUrl, model, agentId } = options;
-  return async ({ text, history, signal, onText }) => {
-    const linked = oxyServices.createLinkedClient({ baseURL: apiUrl });
-    try {
-      const effectiveModel = await resolveModelId(apiUrl, model);
-      signal.throwIfAborted();
-      let answer = '';
-      await streamAliaChat(
-        linked.client,
-        {
-          url: '/v1/chat/completions',
-          model: effectiveModel,
-          messages: [...history, { role: 'user', content: text }],
-          responseMode: 'voice',
-          ...(agentId === undefined ? {} : { agentId }),
-        },
-        signal,
-        (event) => {
-          if (event.kind !== 'content' && event.kind !== 'agent_answer') return;
-          answer += event.content;
-          onText(answer);
-        },
-      );
-    } finally {
-      linked.dispose();
-    }
-  };
-}