@alia.onl/sdk 22.6.0 → 22.8.0

package/CHANGELOG.md

@@ -1,5 +1,36 @@
 # @alia.onl/sdk
 
+## 22.8.0
+
+- **Streaming dictation (ADR 0024).** `@alia.onl/sdk/dictation` gains
+  `openDictationStream({ apiUrl, getToken | token, language?, onEvent, signal?, openSocket?,
+  endTimeoutMs? })`: the caller's PCM16 LE mono 24 kHz frames go to
+  `GET /v1/audio/transcriptions/stream`, and Alia's events come back while the person speaks:
+  `dictation.ready`, `transcript.interim` / `transcript.delta` / `transcript.final` per numbered
+  `segment`, `error { code, retryable }` and one `dictation.closed { reason }`. A segment's
+  `transcript.final` is the only text to keep. It returns `{ append, commit, end, close, closed }`;
+  `end()` resolves once the finals have arrived. The protocol module (`DICTATION_PROTOCOL_VERSION`
+  1, `parseDictationServerEvent`, `dictationStreamUrl`) is exported beside it. It reaches no React
+  or native module (`check:entries`). `useDictation` still records a clip for batch
+  transcription; adopting the stream there is a follow-up.
+- The API that serves the stream ships with this release; against an older API the socket's
+  path answers 404 and the stream reports `TRANSCRIPTION_FAILED`, so batch
+  `requestTranscription` stays the fallback.
+
+## 22.7.0
+
+- **Image results for awaited host tools.** An awaited tool's `execute` may answer
+  `result: { type: "image", mediaType, data, text? }` (`AliaHostImageResult`: `image/jpeg`,
+  `image/png` or `image/webp`, `data` in standard base64 with no `data:` prefix, an optional
+  caption of at most `HOST_IMAGE_TEXT_MAX_CHARS`, 4,000). The model sees the image itself in that
+  turn; the stored conversation and the `alia.tool_result` frame keep
+  `{ type, mediaType, text?, omitted: true }` instead. The whole result still counts toward
+  `HOST_RESULT_MAX_CHARS` (100,000 characters as JSON), so size a screenshot to fit.
+- A result that claims `type: "image"` but is malformed is answered `failed` with the reason
+  (`hostImageResultProblem`), instead of being sent for the API to refuse while the model waits.
+- Needs the Alia API that ships with this release (`docs/agent-embedding.md` §2); an older API
+  passes an image result to the model as JSON text.
+
 ## 22.6.0
 
 - **`@alia.onl/sdk/dictation`: dictation's request on its own.** `requestTranscription`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alia.onl/sdk",
-  "version": "22.6.0",
+  "version": "22.8.0",
   "description": "Alia AI chat SDK — the conversation runtime, chat UI, voice calls and dictation for Oxy apps",
   "main": "src/index.ts",
   "react-native": "src/index.ts",

package/src/dictation.ts

@@ -1,7 +1,8 @@
-// @alia.onl/sdk/dictation — dictation's request, without a component, a hook or a recorder
-// `POST /v1/audio/transcriptions` (ADR 0017 §7) as plain functions: an app that records audio its
-// own way (Astro's browser shell) sends its clips from here and reaches no React, no native module
-// and no Oxy client. `useDictation` and `AliaComposer` use the same request.
+// @alia.onl/sdk/dictation — dictation's network half, without a component, a hook or a recorder
+// `POST /v1/audio/transcriptions` (ADR 0017 §7) and the streaming dictation socket
+// `GET /v1/audio/transcriptions/stream` (ADR 0024) as plain functions: an app that records audio
+// its own way (Astro's browser shell) dictates from here and reaches no React, no native module
+// and no Oxy client. `useDictation` and `AliaComposer` use the same batch request.
 
 export {
   DictationError,
@@ -18,3 +19,22 @@ export type {
 } from './lib/dictation';
 export type { AudioRequestAuthentication } from './lib/audio-request-auth';
 export type { VoiceErrorCode } from './lib/speech-messages';
+export {
+  DICTATION_PROTOCOL_VERSION,
+  DICTATION_SAMPLE_RATE,
+  DICTATION_STREAM_PATH,
+  dictationStreamUrl,
+  parseDictationServerEvent,
+} from './lib/dictation-protocol';
+export type {
+  DictationCloseReason,
+  DictationServerEvent,
+  DictationStreamErrorCode,
+} from './lib/dictation-protocol';
+export { openDictationStream } from './lib/dictation-stream';
+export type {
+  DictationSocket,
+  DictationStream,
+  DictationStreamAuthentication,
+  DictationStreamOptions,
+} from './lib/dictation-stream';

package/src/lib/dictation-protocol.ts

@@ -0,0 +1,142 @@
+/**
+ * The client half of Alia's streaming dictation protocol,
+ * `GET /v1/audio/transcriptions/stream` (ADR 0024).
+ *
+ * The server's copy is `packages/api/src/lib/dictation/protocol.ts`; the SDK
+ * does not depend on the API package, so the shapes are restated here and
+ * `DICTATION_PROTOCOL_VERSION` is what turns a mismatch into a refusal instead
+ * of a misunderstanding. The API's tests hold the two copies equal.
+ *
+ * - Binary, client → Alia: microphone PCM16 LE, 24 kHz, mono.
+ * - Text: one JSON event per frame, both ways.
+ *
+ * Segments are Alia's numbers, from 0, in the order the person spoke them. A
+ * segment's `transcript.final` is the only text a client keeps:
+ * `transcript.interim` (the segment's provisional text, whole, replacing the
+ * last) and `transcript.delta` (locked text appended) are display state the
+ * final supersedes.
+ */
+
+export const DICTATION_PROTOCOL_VERSION = 1;
+
+export const DICTATION_STREAM_PATH = '/v1/audio/transcriptions/stream';
+
+/** PCM16 mono at this rate: the one encoding the stream takes. */
+export const DICTATION_SAMPLE_RATE = 24_000;
+
+/** What a dictation ends with that is not a failure. */
+export type DictationCloseReason =
+  | 'ended'
+  | 'max-duration'
+  | 'idle-timeout'
+  | 'error'
+  | 'server-shutdown';
+
+/** Why a dictation failed, as Alia names it. */
+export type DictationStreamErrorCode =
+  | 'TRANSCRIPTION_UNAVAILABLE'
+  | 'TRANSCRIPTION_LIMITED'
+  | 'TRANSCRIPTION_REJECTED'
+  | 'TRANSCRIPTION_FAILED'
+  | 'CLIP_TOO_LONG'
+  | 'INVALID_REQUEST'
+  | 'NOT_AUTHENTICATED'
+  | 'CREDENTIAL_NOT_SUPPORTED';
+
+export type DictationServerEvent =
+  | { readonly type: 'dictation.ready'; readonly maxDurationMs: number }
+  | { readonly type: 'transcript.interim'; readonly segment: number; readonly text: string }
+  | { readonly type: 'transcript.delta'; readonly segment: number; readonly text: string }
+  | { readonly type: 'transcript.final'; readonly segment: number; readonly text: string }
+  | {
+      readonly type: 'error';
+      readonly code: DictationStreamErrorCode;
+      readonly message: string;
+      readonly retryable: boolean;
+    }
+  | { readonly type: 'dictation.closed'; readonly reason: DictationCloseReason };
+
+export type DictationClientEvent =
+  | { readonly type: 'auth'; readonly token: string }
+  | {
+      readonly type: 'dictation.start';
+      readonly protocol: number;
+      readonly language?: string;
+      readonly sampleRate: number;
+    }
+  | { readonly type: 'dictation.commit' }
+  | { readonly type: 'dictation.end' };
+
+const CLOSE_REASONS: ReadonlySet<string> = new Set<DictationCloseReason>([
+  'ended',
+  'max-duration',
+  'idle-timeout',
+  'error',
+  'server-shutdown',
+]);
+
+const ERROR_CODES: ReadonlySet<string> = new Set<DictationStreamErrorCode>([
+  'TRANSCRIPTION_UNAVAILABLE',
+  'TRANSCRIPTION_LIMITED',
+  'TRANSCRIPTION_REJECTED',
+  'TRANSCRIPTION_FAILED',
+  'CLIP_TOO_LONG',
+  'INVALID_REQUEST',
+  'NOT_AUTHENTICATED',
+  'CREDENTIAL_NOT_SUPPORTED',
+]);
+
+const isSegment = (value: unknown): value is number =>
+  typeof value === 'number' && Number.isInteger(value) && value >= 0;
+
+/** One server text frame, or `null` when it is not one of protocol 1's events. */
+export function parseDictationServerEvent(text: string): DictationServerEvent | null {
+  let value: unknown;
+  try {
+    value = JSON.parse(text);
+  } catch {
+    return null;
+  }
+  if (value === null || typeof value !== 'object') return null;
+  const event = value as Record<string, unknown>;
+  switch (event.type) {
+    case 'dictation.ready':
+      return typeof event.maxDurationMs === 'number' && event.maxDurationMs > 0
+        ? { type: 'dictation.ready', maxDurationMs: event.maxDurationMs }
+        : null;
+    case 'transcript.interim':
+    case 'transcript.delta':
+    case 'transcript.final':
+      return isSegment(event.segment) && typeof event.text === 'string'
+        ? { type: event.type, segment: event.segment, text: event.text }
+        : null;
+    case 'error':
+      return typeof event.code === 'string' &&
+        ERROR_CODES.has(event.code) &&
+        typeof event.message === 'string'
+        ? {
+            type: 'error',
+            code: event.code as DictationStreamErrorCode,
+            message: event.message,
+            retryable: event.retryable === true,
+          }
+        : null;
+    case 'dictation.closed':
+      return typeof event.reason === 'string' && CLOSE_REASONS.has(event.reason)
+        ? { type: 'dictation.closed', reason: event.reason as DictationCloseReason }
+        : null;
+    default:
+      return null;
+  }
+}
+
+/** `wss://…/v1/audio/transcriptions/stream` for an Alia API origin. */
+export function dictationStreamUrl(apiUrl: string): string {
+  const base = apiUrl.replace(/\/+$/, '');
+  const ws = base.startsWith('https://')
+    ? `wss://${base.slice('https://'.length)}`
+    : base.startsWith('http://')
+      ? `ws://${base.slice('http://'.length)}`
+      : base;
+  return `${ws}${DICTATION_STREAM_PATH}`;
+}

package/src/lib/dictation-stream.ts

@@ -0,0 +1,238 @@
+/**
+ * Streaming dictation's network half (ADR 0024): the caller's microphone PCM to
+ * `GET /v1/audio/transcriptions/stream`, and Alia's segments back while the
+ * person speaks.
+ *
+ * It records nothing and draws nothing: the caller brings PCM16 LE mono at
+ * 24 kHz from its own recorder (an AudioWorklet, a native module) and shows
+ * the segments its own way. A segment's `transcript.final` is the only text to
+ * keep; `transcript.interim` and `transcript.delta` are display state the
+ * final supersedes. When the stream cannot open (`TRANSCRIPTION_UNAVAILABLE`),
+ * batch `requestTranscription` is the fallback.
+ */
+
+import {
+  type DictationClientEvent,
+  type DictationCloseReason,
+  type DictationServerEvent,
+  DICTATION_PROTOCOL_VERSION,
+  DICTATION_SAMPLE_RATE,
+  dictationStreamUrl,
+  parseDictationServerEvent,
+} from './dictation-protocol';
+
+/** The WebSocket the stream drives: a browser's, React Native's or a test's. */
+export interface DictationSocket {
+  binaryType: string;
+  readonly readyState: number;
+  onopen: ((event: unknown) => void) | null;
+  onmessage: ((event: { data: unknown }) => void) | null;
+  onclose: ((event: unknown) => void) | null;
+  onerror: ((event: unknown) => void) | null;
+  send(data: string | ArrayBufferLike | ArrayBufferView): void;
+  close(code?: number, reason?: string): void;
+}
+
+/** How the stream proves the person: their Oxy session token, read once at open. */
+export type DictationStreamAuthentication =
+  | {
+      readonly getToken: (
+        target: string,
+        options?: { signal?: AbortSignal },
+      ) => string | null | Promise<string | null>;
+      readonly token?: never;
+    }
+  | { readonly token: string; readonly getToken?: never };
+
+export type DictationStreamOptions = DictationStreamAuthentication & {
+  /** Alia's API origin, `https://api.alia.onl`. */
+  readonly apiUrl: string;
+  /** BCP 47; omitted lets the model detect the language. */
+  readonly language?: string;
+  /**
+   * Every event, in order. It always ends with one `dictation.closed`; a failure
+   * comes as `error` just before it. A connection Alia never answered on is
+   * reported as `TRANSCRIPTION_FAILED`, retryable.
+   */
+  readonly onEvent: (event: DictationServerEvent) => void;
+  /** Aborting it is `close()`. */
+  readonly signal?: AbortSignal;
+  /** The socket factory; the platform's `WebSocket` by default. */
+  readonly openSocket?: (url: string) => DictationSocket;
+  /** How long `end()` waits for Alia's `dictation.closed` before closing the socket. */
+  readonly endTimeoutMs?: number;
+};
+
+export interface DictationStream {
+  /** One microphone frame: PCM16 LE, mono, {@link DICTATION_SAMPLE_RATE} Hz. */
+  append(pcm16: Uint8Array): void;
+  /** End the current segment now (push-to-talk). */
+  commit(): void;
+  /** No more audio: resolves with how the dictation closed, once the finals have arrived. */
+  end(): Promise<DictationCloseReason>;
+  /** Drop the dictation now; nothing more is reported. */
+  close(): void;
+  /** Settles with how the dictation closed, whatever closed it. */
+  readonly closed: Promise<DictationCloseReason>;
+}
+
+const OPEN = 1;
+const DEFAULT_END_TIMEOUT_MS = 8_000;
+/** Audio held until the socket is open and the start sent: ten seconds. */
+const MAX_HELD_BYTES = 10 * DICTATION_SAMPLE_RATE * 2;
+
+function defaultSocket(url: string): DictationSocket {
+  const WebSocketClass = (globalThis as { WebSocket?: new (url: string) => DictationSocket })
+    .WebSocket;
+  if (!WebSocketClass) throw new Error('This platform has no WebSocket');
+  return new WebSocketClass(url);
+}
+
+/** Open one streaming dictation. */
+export function openDictationStream(options: DictationStreamOptions): DictationStream {
+  const url = dictationStreamUrl(options.apiUrl);
+  const credentialAbort = new AbortController();
+  let socket: DictationSocket | null = null;
+  let started = false;
+  let finished = false;
+  let held: Uint8Array[] = [];
+  let heldBytes = 0;
+  let endRequested = false;
+  let endTimer: ReturnType<typeof setTimeout> | null = null;
+  let resolveClosed!: (reason: DictationCloseReason) => void;
+  const closed = new Promise<DictationCloseReason>((resolve) => {
+    resolveClosed = resolve;
+  });
+
+  const sendEvent = (event: DictationClientEvent) => {
+    if (socket?.readyState === OPEN) socket.send(JSON.stringify(event));
+  };
+
+  const finish = (reason: DictationCloseReason, report: boolean) => {
+    if (finished) return;
+    finished = true;
+    if (endTimer !== null) clearTimeout(endTimer);
+    credentialAbort.abort();
+    held = [];
+    const current = socket;
+    socket = null;
+    if (current) {
+      current.onmessage = null;
+      current.onclose = null;
+      current.onerror = null;
+      if (current.readyState === OPEN || current.readyState === 0) current.close(1000);
+    }
+    if (report) options.onEvent({ type: 'dictation.closed', reason });
+    resolveClosed(reason);
+  };
+
+  const fail = () => {
+    if (finished) return;
+    options.onEvent({
+      type: 'error',
+      code: 'TRANSCRIPTION_FAILED',
+      message: 'The dictation lost its connection to Alia.',
+      retryable: true,
+    });
+    finish('error', true);
+  };
+
+  const readToken = async (): Promise<string | null> => {
+    if (options.token !== undefined) return options.token || null;
+    try {
+      const token = await options.getToken(url, { signal: credentialAbort.signal });
+      return token || null;
+    } catch {
+      return null;
+    }
+  };
+
+  const begin = async (opened: DictationSocket) => {
+    const token = await readToken();
+    if (finished || socket !== opened) return;
+    if (!token) {
+      options.onEvent({
+        type: 'error',
+        code: 'NOT_AUTHENTICATED',
+        message: 'Sign in to Alia to dictate.',
+        retryable: false,
+      });
+      finish('error', true);
+      return;
+    }
+    sendEvent({ type: 'auth', token });
+    sendEvent({
+      type: 'dictation.start',
+      protocol: DICTATION_PROTOCOL_VERSION,
+      sampleRate: DICTATION_SAMPLE_RATE,
+      ...(options.language === undefined ? {} : { language: options.language }),
+    });
+    started = true;
+    for (const frame of held) opened.send(frame);
+    held = [];
+    heldBytes = 0;
+    if (endRequested) sendEvent({ type: 'dictation.end' });
+  };
+
+  try {
+    socket = (options.openSocket ?? defaultSocket)(url);
+  } catch {
+    queueMicrotask(fail);
+  }
+  if (socket) {
+    const opened = socket;
+    opened.binaryType = 'arraybuffer';
+    opened.onopen = () => {
+      void begin(opened);
+    };
+    opened.onmessage = (message) => {
+      if (typeof message.data !== 'string') return;
+      const event = parseDictationServerEvent(message.data);
+      if (event === null || finished) return;
+      if (event.type === 'dictation.closed') {
+        options.onEvent(event);
+        finish(event.reason, false);
+        return;
+      }
+      options.onEvent(event);
+    };
+    opened.onclose = () => fail();
+    opened.onerror = () => undefined;
+  }
+
+  const close = () => finish('ended', false);
+  if (options.signal) {
+    if (options.signal.aborted) queueMicrotask(close);
+    else options.signal.addEventListener('abort', close, { once: true });
+  }
+
+  return {
+    append(pcm16) {
+      if (finished || endRequested || pcm16.byteLength === 0) return;
+      if (started && socket?.readyState === OPEN) {
+        socket.send(pcm16);
+        return;
+      }
+      if (heldBytes + pcm16.byteLength > MAX_HELD_BYTES) return;
+      held.push(pcm16.slice());
+      heldBytes += pcm16.byteLength;
+    },
+    commit() {
+      if (!finished && started) sendEvent({ type: 'dictation.commit' });
+    },
+    end() {
+      if (finished) return closed;
+      if (!endRequested) {
+        endRequested = true;
+        if (started) sendEvent({ type: 'dictation.end' });
+        endTimer = setTimeout(
+          () => finish('ended', true),
+          options.endTimeoutMs ?? DEFAULT_END_TIMEOUT_MS,
+        );
+      }
+      return closed;
+    },
+    close,
+    closed,
+  };
+}

package/src/runtime/host-tools.ts

@@ -9,7 +9,8 @@
  *
  * A tool declared with `awaitsResult: true` is read in the SAME turn instead ("this tab's text",
  * "the open tabs"): the server waits, and this client answers it at once on the turn's
- * `host-results` route with the outcome and the `result` its `execute` returned.
+ * `host-results` route with the outcome and the `result` its `execute` returned — text, JSON or
+ * an image ({@link AliaHostImageResult}) the model sees as one.
  *
  * Three rules hold here, in a queue that knows nothing of React:
  *  - an action is performed at most once, whatever the stream repeats;
@@ -45,11 +46,61 @@ export interface AliaHostOutcome {
   detail?: string;
   /**
    * What an awaited tool read, for the model: a string or any JSON value, at most
-   * {@link HOST_RESULT_MAX_CHARS} characters as JSON. Ignored for a tool that does not await.
+   * {@link HOST_RESULT_MAX_CHARS} characters as JSON, or an {@link AliaHostImageResult} the model
+   * sees as an image. Ignored for a tool that does not await.
    */
   result?: unknown;
 }
 
+export const HOST_IMAGE_MEDIA_TYPES = ['image/jpeg', 'image/png', 'image/webp'] as const;
+export type AliaHostImageMediaType = (typeof HOST_IMAGE_MEDIA_TYPES)[number];
+
+/** The longest caption an image result may carry. */
+export const HOST_IMAGE_TEXT_MAX_CHARS = 4_000;
+
+/**
+ * An awaited tool's image (a tab's screenshot): the model sees the image itself in that turn, and
+ * the stored conversation keeps `{ type, mediaType, text?, omitted: true }` in its place.
+ * `data` is the image's bytes in standard base64, with no `data:` prefix; the whole result counts
+ * toward {@link HOST_RESULT_MAX_CHARS}, so size the image to fit (a JPEG of the viewport).
+ */
+export interface AliaHostImageResult {
+  type: 'image';
+  mediaType: AliaHostImageMediaType;
+  data: string;
+  /** A caption for the model (the tab's URL and title), at most {@link HOST_IMAGE_TEXT_MAX_CHARS}. */
+  text?: string;
+}
+
+const BASE64 = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;
+const IMAGE_KEYS = ['type', 'mediaType', 'data', 'text'];
+
+/**
+ * Why a result that claims to be an image (`type: "image"`) is not a valid one, or `null`. The
+ * server refuses such an answer, which would leave the model waiting until the action goes stale;
+ * the runtime answers `failed` with this reason instead.
+ */
+export function hostImageResultProblem(result: unknown): string | null {
+  if (typeof result !== 'object' || result === null || Array.isArray(result)) return null;
+  const image = result as Record<string, unknown>;
+  if (image.type !== 'image') return null;
+  const extra = Object.keys(image).find((key) => !IMAGE_KEYS.includes(key));
+  if (extra !== undefined) return `An image result has no "${extra}".`;
+  if (!HOST_IMAGE_MEDIA_TYPES.includes(image.mediaType as AliaHostImageMediaType)) {
+    return `An image result's mediaType is one of ${HOST_IMAGE_MEDIA_TYPES.join(', ')}.`;
+  }
+  if (typeof image.data !== 'string' || image.data === '' || !BASE64.test(image.data)) {
+    return "An image result's data is standard base64 with no data: prefix.";
+  }
+  if (
+    image.text !== undefined &&
+    (typeof image.text !== 'string' || image.text.length > HOST_IMAGE_TEXT_MAX_CHARS)
+  ) {
+    return `An image result's text is a string of at most ${HOST_IMAGE_TEXT_MAX_CHARS} characters.`;
+  }
+  return null;
+}
+
 /**
  * The host tools a surface declares (`hostTools` on the chat). They are the tools of the chat's
  * `app`, which a surface with host tools must name: the app is said once, on the chat, and is what
@@ -96,8 +147,10 @@ const DETAIL_LIMIT = 500;
 /** The server's bound on an awaited action's `result`, as JSON. */
 export const HOST_RESULT_MAX_CHARS = 100_000;
 
-function resultOf(result: unknown): { result?: unknown } | 'too_large' {
+function resultOf(result: unknown): { result?: unknown } | 'too_large' | { invalid: string } {
   if (result === undefined || result === null) return {};
+  const problem = hostImageResultProblem(result);
+  if (problem !== null) return { invalid: problem };
   let text: string | undefined;
   try {
     text = JSON.stringify(result);
@@ -158,7 +211,9 @@ export class HostOutcomeQueue {
             outcome: 'failed',
             detail: 'The result was too large to send.',
           })
-        : { ...record(action.toolCallId, outcome), ...result };
+        : 'invalid' in result
+          ? record(action.toolCallId, { outcome: 'failed', detail: result.invalid })
+          : { ...record(action.toolCallId, outcome), ...result };
     try {
       await send(answer);
     } catch (error) {

package/src/runtime/index.ts

@@ -33,9 +33,19 @@ export {
   useAgentProfile,
 } from './agents';
 export type { AliaAgentFeature, AliaAgentProfile, AliaAgentStarter, FeatureOffer } from './agents';
-export { HOST_OUTCOMES, HOST_RESULT_MAX_CHARS, HostOutcomeQueue, hostRequest } from './host-tools';
+export {
+  HOST_IMAGE_MEDIA_TYPES,
+  HOST_IMAGE_TEXT_MAX_CHARS,
+  HOST_OUTCOMES,
+  HOST_RESULT_MAX_CHARS,
+  HostOutcomeQueue,
+  hostImageResultProblem,
+  hostRequest,
+} from './host-tools';
 export type {
   AliaHostAction,
+  AliaHostImageMediaType,
+  AliaHostImageResult,
   AliaHostOutcome,
   AliaHostOutcomeKind,
   AliaHostToolDefinition,