@alia.onl/sdk 22.7.0 → 22.8.0

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @alia.onl/sdk
 
+## 22.8.0
+
+- **Streaming dictation (ADR 0024).** `@alia.onl/sdk/dictation` gains
+  `openDictationStream({ apiUrl, getToken | token, language?, onEvent, signal?, openSocket?,
+  endTimeoutMs? })`: the caller's PCM16 LE mono 24 kHz frames go to
+  `GET /v1/audio/transcriptions/stream`, and Alia's events come back while the person speaks:
+  `dictation.ready`, `transcript.interim` / `transcript.delta` / `transcript.final` per numbered
+  `segment`, `error { code, retryable }` and one `dictation.closed { reason }`. A segment's
+  `transcript.final` is the only text to keep. It returns `{ append, commit, end, close, closed }`;
+  `end()` resolves once the finals have arrived. The protocol module (`DICTATION_PROTOCOL_VERSION`
+  1, `parseDictationServerEvent`, `dictationStreamUrl`) is exported beside it. It reaches no React
+  or native module (`check:entries`). `useDictation` still records a clip for batch
+  transcription; adopting the stream there is a follow-up.
+- The API that serves the stream ships with this release; against an older API the socket's
+  path answers 404 and the stream reports `TRANSCRIPTION_FAILED`, so batch
+  `requestTranscription` stays the fallback.
+
 ## 22.7.0
 
 - **Image results for awaited host tools.** An awaited tool's `execute` may answer

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alia.onl/sdk",
-  "version": "22.7.0",
+  "version": "22.8.0",
   "description": "Alia AI chat SDK — the conversation runtime, chat UI, voice calls and dictation for Oxy apps",
   "main": "src/index.ts",
   "react-native": "src/index.ts",

package/src/dictation.ts

@@ -1,7 +1,8 @@
-// @alia.onl/sdk/dictation — dictation's request, without a component, a hook or a recorder
-// `POST /v1/audio/transcriptions` (ADR 0017 §7) as plain functions: an app that records audio its
-// own way (Astro's browser shell) sends its clips from here and reaches no React, no native module
-// and no Oxy client. `useDictation` and `AliaComposer` use the same request.
+// @alia.onl/sdk/dictation — dictation's network half, without a component, a hook or a recorder
+// `POST /v1/audio/transcriptions` (ADR 0017 §7) and the streaming dictation socket
+// `GET /v1/audio/transcriptions/stream` (ADR 0024) as plain functions: an app that records audio
+// its own way (Astro's browser shell) dictates from here and reaches no React, no native module
+// and no Oxy client. `useDictation` and `AliaComposer` use the same batch request.
 
 export {
   DictationError,
@@ -18,3 +19,22 @@ export type {
 } from './lib/dictation';
 export type { AudioRequestAuthentication } from './lib/audio-request-auth';
 export type { VoiceErrorCode } from './lib/speech-messages';
+export {
+  DICTATION_PROTOCOL_VERSION,
+  DICTATION_SAMPLE_RATE,
+  DICTATION_STREAM_PATH,
+  dictationStreamUrl,
+  parseDictationServerEvent,
+} from './lib/dictation-protocol';
+export type {
+  DictationCloseReason,
+  DictationServerEvent,
+  DictationStreamErrorCode,
+} from './lib/dictation-protocol';
+export { openDictationStream } from './lib/dictation-stream';
+export type {
+  DictationSocket,
+  DictationStream,
+  DictationStreamAuthentication,
+  DictationStreamOptions,
+} from './lib/dictation-stream';

package/src/lib/dictation-protocol.ts

@@ -0,0 +1,142 @@
+/**
+ * The client half of Alia's streaming dictation protocol,
+ * `GET /v1/audio/transcriptions/stream` (ADR 0024).
+ *
+ * The server's copy is `packages/api/src/lib/dictation/protocol.ts`; the SDK
+ * does not depend on the API package, so the shapes are restated here and
+ * `DICTATION_PROTOCOL_VERSION` is what turns a mismatch into a refusal instead
+ * of a misunderstanding. The API's tests hold the two copies equal.
+ *
+ * - Binary, client → Alia: microphone PCM16 LE, 24 kHz, mono.
+ * - Text: one JSON event per frame, both ways.
+ *
+ * Segments are Alia's numbers, from 0, in the order the person spoke them. A
+ * segment's `transcript.final` is the only text a client keeps:
+ * `transcript.interim` (the segment's provisional text, whole, replacing the
+ * last) and `transcript.delta` (locked text appended) are display state the
+ * final supersedes.
+ */
+
+export const DICTATION_PROTOCOL_VERSION = 1;
+
+export const DICTATION_STREAM_PATH = '/v1/audio/transcriptions/stream';
+
+/** PCM16 mono at this rate: the one encoding the stream takes. */
+export const DICTATION_SAMPLE_RATE = 24_000;
+
+/** What a dictation ends with that is not a failure. */
+export type DictationCloseReason =
+  | 'ended'
+  | 'max-duration'
+  | 'idle-timeout'
+  | 'error'
+  | 'server-shutdown';
+
+/** Why a dictation failed, as Alia names it. */
+export type DictationStreamErrorCode =
+  | 'TRANSCRIPTION_UNAVAILABLE'
+  | 'TRANSCRIPTION_LIMITED'
+  | 'TRANSCRIPTION_REJECTED'
+  | 'TRANSCRIPTION_FAILED'
+  | 'CLIP_TOO_LONG'
+  | 'INVALID_REQUEST'
+  | 'NOT_AUTHENTICATED'
+  | 'CREDENTIAL_NOT_SUPPORTED';
+
+export type DictationServerEvent =
+  | { readonly type: 'dictation.ready'; readonly maxDurationMs: number }
+  | { readonly type: 'transcript.interim'; readonly segment: number; readonly text: string }
+  | { readonly type: 'transcript.delta'; readonly segment: number; readonly text: string }
+  | { readonly type: 'transcript.final'; readonly segment: number; readonly text: string }
+  | {
+      readonly type: 'error';
+      readonly code: DictationStreamErrorCode;
+      readonly message: string;
+      readonly retryable: boolean;
+    }
+  | { readonly type: 'dictation.closed'; readonly reason: DictationCloseReason };
+
+export type DictationClientEvent =
+  | { readonly type: 'auth'; readonly token: string }
+  | {
+      readonly type: 'dictation.start';
+      readonly protocol: number;
+      readonly language?: string;
+      readonly sampleRate: number;
+    }
+  | { readonly type: 'dictation.commit' }
+  | { readonly type: 'dictation.end' };
+
+const CLOSE_REASONS: ReadonlySet<string> = new Set<DictationCloseReason>([
+  'ended',
+  'max-duration',
+  'idle-timeout',
+  'error',
+  'server-shutdown',
+]);
+
+const ERROR_CODES: ReadonlySet<string> = new Set<DictationStreamErrorCode>([
+  'TRANSCRIPTION_UNAVAILABLE',
+  'TRANSCRIPTION_LIMITED',
+  'TRANSCRIPTION_REJECTED',
+  'TRANSCRIPTION_FAILED',
+  'CLIP_TOO_LONG',
+  'INVALID_REQUEST',
+  'NOT_AUTHENTICATED',
+  'CREDENTIAL_NOT_SUPPORTED',
+]);
+
+const isSegment = (value: unknown): value is number =>
+  typeof value === 'number' && Number.isInteger(value) && value >= 0;
+
+/** One server text frame, or `null` when it is not one of protocol 1's events. */
+export function parseDictationServerEvent(text: string): DictationServerEvent | null {
+  let value: unknown;
+  try {
+    value = JSON.parse(text);
+  } catch {
+    return null;
+  }
+  if (value === null || typeof value !== 'object') return null;
+  const event = value as Record<string, unknown>;
+  switch (event.type) {
+    case 'dictation.ready':
+      return typeof event.maxDurationMs === 'number' && event.maxDurationMs > 0
+        ? { type: 'dictation.ready', maxDurationMs: event.maxDurationMs }
+        : null;
+    case 'transcript.interim':
+    case 'transcript.delta':
+    case 'transcript.final':
+      return isSegment(event.segment) && typeof event.text === 'string'
+        ? { type: event.type, segment: event.segment, text: event.text }
+        : null;
+    case 'error':
+      return typeof event.code === 'string' &&
+        ERROR_CODES.has(event.code) &&
+        typeof event.message === 'string'
+        ? {
+            type: 'error',
+            code: event.code as DictationStreamErrorCode,
+            message: event.message,
+            retryable: event.retryable === true,
+          }
+        : null;
+    case 'dictation.closed':
+      return typeof event.reason === 'string' && CLOSE_REASONS.has(event.reason)
+        ? { type: 'dictation.closed', reason: event.reason as DictationCloseReason }
+        : null;
+    default:
+      return null;
+  }
+}
+
+/** `wss://…/v1/audio/transcriptions/stream` for an Alia API origin. */
+export function dictationStreamUrl(apiUrl: string): string {
+  const base = apiUrl.replace(/\/+$/, '');
+  const ws = base.startsWith('https://')
+    ? `wss://${base.slice('https://'.length)}`
+    : base.startsWith('http://')
+      ? `ws://${base.slice('http://'.length)}`
+      : base;
+  return `${ws}${DICTATION_STREAM_PATH}`;
+}

package/src/lib/dictation-stream.ts

@@ -0,0 +1,238 @@
+/**
+ * Streaming dictation's network half (ADR 0024): the caller's microphone PCM to
+ * `GET /v1/audio/transcriptions/stream`, and Alia's segments back while the
+ * person speaks.
+ *
+ * It records nothing and draws nothing: the caller brings PCM16 LE mono at
+ * 24 kHz from its own recorder (an AudioWorklet, a native module) and shows
+ * the segments its own way. A segment's `transcript.final` is the only text to
+ * keep; `transcript.interim` and `transcript.delta` are display state the
+ * final supersedes. When the stream cannot open (`TRANSCRIPTION_UNAVAILABLE`),
+ * batch `requestTranscription` is the fallback.
+ */
+
+import {
+  type DictationClientEvent,
+  type DictationCloseReason,
+  type DictationServerEvent,
+  DICTATION_PROTOCOL_VERSION,
+  DICTATION_SAMPLE_RATE,
+  dictationStreamUrl,
+  parseDictationServerEvent,
+} from './dictation-protocol';
+
+/** The WebSocket the stream drives: a browser's, React Native's or a test's. */
+export interface DictationSocket {
+  binaryType: string;
+  readonly readyState: number;
+  onopen: ((event: unknown) => void) | null;
+  onmessage: ((event: { data: unknown }) => void) | null;
+  onclose: ((event: unknown) => void) | null;
+  onerror: ((event: unknown) => void) | null;
+  send(data: string | ArrayBufferLike | ArrayBufferView): void;
+  close(code?: number, reason?: string): void;
+}
+
+/** How the stream proves the person: their Oxy session token, read once at open. */
+export type DictationStreamAuthentication =
+  | {
+      readonly getToken: (
+        target: string,
+        options?: { signal?: AbortSignal },
+      ) => string | null | Promise<string | null>;
+      readonly token?: never;
+    }
+  | { readonly token: string; readonly getToken?: never };
+
+export type DictationStreamOptions = DictationStreamAuthentication & {
+  /** Alia's API origin, `https://api.alia.onl`. */
+  readonly apiUrl: string;
+  /** BCP 47; omitted lets the model detect the language. */
+  readonly language?: string;
+  /**
+   * Every event, in order. It always ends with one `dictation.closed`; a failure
+   * comes as `error` just before it. A connection Alia never answered on is
+   * reported as `TRANSCRIPTION_FAILED`, retryable.
+   */
+  readonly onEvent: (event: DictationServerEvent) => void;
+  /** Aborting it is `close()`. */
+  readonly signal?: AbortSignal;
+  /** The socket factory; the platform's `WebSocket` by default. */
+  readonly openSocket?: (url: string) => DictationSocket;
+  /** How long `end()` waits for Alia's `dictation.closed` before closing the socket. */
+  readonly endTimeoutMs?: number;
+};
+
+export interface DictationStream {
+  /** One microphone frame: PCM16 LE, mono, {@link DICTATION_SAMPLE_RATE} Hz. */
+  append(pcm16: Uint8Array): void;
+  /** End the current segment now (push-to-talk). */
+  commit(): void;
+  /** No more audio: resolves with how the dictation closed, once the finals have arrived. */
+  end(): Promise<DictationCloseReason>;
+  /** Drop the dictation now; nothing more is reported. */
+  close(): void;
+  /** Settles with how the dictation closed, whatever closed it. */
+  readonly closed: Promise<DictationCloseReason>;
+}
+
+const OPEN = 1;
+const DEFAULT_END_TIMEOUT_MS = 8_000;
+/** Audio held until the socket is open and the start sent: ten seconds. */
+const MAX_HELD_BYTES = 10 * DICTATION_SAMPLE_RATE * 2;
+
+function defaultSocket(url: string): DictationSocket {
+  const WebSocketClass = (globalThis as { WebSocket?: new (url: string) => DictationSocket })
+    .WebSocket;
+  if (!WebSocketClass) throw new Error('This platform has no WebSocket');
+  return new WebSocketClass(url);
+}
+
+/** Open one streaming dictation. */
+export function openDictationStream(options: DictationStreamOptions): DictationStream {
+  const url = dictationStreamUrl(options.apiUrl);
+  const credentialAbort = new AbortController();
+  let socket: DictationSocket | null = null;
+  let started = false;
+  let finished = false;
+  let held: Uint8Array[] = [];
+  let heldBytes = 0;
+  let endRequested = false;
+  let endTimer: ReturnType<typeof setTimeout> | null = null;
+  let resolveClosed!: (reason: DictationCloseReason) => void;
+  const closed = new Promise<DictationCloseReason>((resolve) => {
+    resolveClosed = resolve;
+  });
+
+  const sendEvent = (event: DictationClientEvent) => {
+    if (socket?.readyState === OPEN) socket.send(JSON.stringify(event));
+  };
+
+  const finish = (reason: DictationCloseReason, report: boolean) => {
+    if (finished) return;
+    finished = true;
+    if (endTimer !== null) clearTimeout(endTimer);
+    credentialAbort.abort();
+    held = [];
+    const current = socket;
+    socket = null;
+    if (current) {
+      current.onmessage = null;
+      current.onclose = null;
+      current.onerror = null;
+      if (current.readyState === OPEN || current.readyState === 0) current.close(1000);
+    }
+    if (report) options.onEvent({ type: 'dictation.closed', reason });
+    resolveClosed(reason);
+  };
+
+  const fail = () => {
+    if (finished) return;
+    options.onEvent({
+      type: 'error',
+      code: 'TRANSCRIPTION_FAILED',
+      message: 'The dictation lost its connection to Alia.',
+      retryable: true,
+    });
+    finish('error', true);
+  };
+
+  const readToken = async (): Promise<string | null> => {
+    if (options.token !== undefined) return options.token || null;
+    try {
+      const token = await options.getToken(url, { signal: credentialAbort.signal });
+      return token || null;
+    } catch {
+      return null;
+    }
+  };
+
+  const begin = async (opened: DictationSocket) => {
+    const token = await readToken();
+    if (finished || socket !== opened) return;
+    if (!token) {
+      options.onEvent({
+        type: 'error',
+        code: 'NOT_AUTHENTICATED',
+        message: 'Sign in to Alia to dictate.',
+        retryable: false,
+      });
+      finish('error', true);
+      return;
+    }
+    sendEvent({ type: 'auth', token });
+    sendEvent({
+      type: 'dictation.start',
+      protocol: DICTATION_PROTOCOL_VERSION,
+      sampleRate: DICTATION_SAMPLE_RATE,
+      ...(options.language === undefined ? {} : { language: options.language }),
+    });
+    started = true;
+    for (const frame of held) opened.send(frame);
+    held = [];
+    heldBytes = 0;
+    if (endRequested) sendEvent({ type: 'dictation.end' });
+  };
+
+  try {
+    socket = (options.openSocket ?? defaultSocket)(url);
+  } catch {
+    queueMicrotask(fail);
+  }
+  if (socket) {
+    const opened = socket;
+    opened.binaryType = 'arraybuffer';
+    opened.onopen = () => {
+      void begin(opened);
+    };
+    opened.onmessage = (message) => {
+      if (typeof message.data !== 'string') return;
+      const event = parseDictationServerEvent(message.data);
+      if (event === null || finished) return;
+      if (event.type === 'dictation.closed') {
+        options.onEvent(event);
+        finish(event.reason, false);
+        return;
+      }
+      options.onEvent(event);
+    };
+    opened.onclose = () => fail();
+    opened.onerror = () => undefined;
+  }
+
+  const close = () => finish('ended', false);
+  if (options.signal) {
+    if (options.signal.aborted) queueMicrotask(close);
+    else options.signal.addEventListener('abort', close, { once: true });
+  }
+
+  return {
+    append(pcm16) {
+      if (finished || endRequested || pcm16.byteLength === 0) return;
+      if (started && socket?.readyState === OPEN) {
+        socket.send(pcm16);
+        return;
+      }
+      if (heldBytes + pcm16.byteLength > MAX_HELD_BYTES) return;
+      held.push(pcm16.slice());
+      heldBytes += pcm16.byteLength;
+    },
+    commit() {
+      if (!finished && started) sendEvent({ type: 'dictation.commit' });
+    },
+    end() {
+      if (finished) return closed;
+      if (!endRequested) {
+        endRequested = true;
+        if (started) sendEvent({ type: 'dictation.end' });
+        endTimer = setTimeout(
+          () => finish('ended', true),
+          options.endTimeoutMs ?? DEFAULT_END_TIMEOUT_MS,
+        );
+      }
+      return closed;
+    },
+    close,
+    closed,
+  };
+}