@alia.onl/sdk 15.7.0 → 19.0.0

package/src/lib/voice-call-request.ts

@@ -0,0 +1,49 @@
+import { create } from 'zustand';
+
+/** A call somebody asked for, waiting for the screen that shows its conversation. */
+export interface VoiceCallRequest {
+  readonly conversationId: string;
+  readonly accountId: string | null;
+  readonly at: number;
+}
+
+/** How long a request waits for its screen. A navigation that never arrives must not open a call later. */
+export const CALL_REQUEST_TTL_MS = 15_000;
+
+interface VoiceCallRequestStore {
+  request: VoiceCallRequest | null;
+  ask: (conversationId: string, accountId: string | null) => void;
+  clear: () => void;
+}
+
+/**
+ * Where a call is asked for, apart from where it opens.
+ *
+ * Starting a call and showing it are the same thing when the conversation is already on screen,
+ * and two screens apart when it is not: a new-chat route makes the conversation and the host
+ * navigates to it. Both go the same way. The launcher AND the screen of the conversation meet
+ * here — it asks, and whichever launcher is mounted for that conversation (that account, in time)
+ * takes the request and opens the call. The sheet, the screen and the app share this one path.
+ */
+export const useVoiceCallRequests = create<VoiceCallRequestStore>((set) => ({
+  request: null,
+  ask: (conversationId, accountId) =>
+    set({ request: { conversationId, accountId, at: Date.now() } }),
+  clear: () => set({ request: null }),
+}));
+
+/** Whether `request` is for this conversation and account, and still worth opening. */
+export function isRequestFor(
+  request: VoiceCallRequest | null,
+  conversationId: string | undefined,
+  accountId: string | null,
+  now = Date.now(),
+): boolean {
+  return (
+    request !== null &&
+    conversationId !== undefined &&
+    request.conversationId === conversationId &&
+    request.accountId === accountId &&
+    now - request.at <= CALL_REQUEST_TTL_MS
+  );
+}

package/src/lib/voice-call.ts

@@ -88,8 +88,6 @@ export interface VoiceCallOptions {
   readonly onChange: (snapshot: VoiceCallSnapshot) => void;
   /** Alia stored messages of the call in the conversation. */
   readonly onMessagesSaved?: (conversationId: string, messageIds: readonly string[]) => void;
-  /** A tool filed an action for the person's OK; the card is in the conversation. */
-  readonly onApprovalRequested?: (toolName: string) => void;
   readonly tokenRefreshMs?: number;
   readonly resumeRetryMs?: number;
 }
@@ -113,6 +111,13 @@ export class VoiceCall {
   private readonly options: VoiceCallOptions;
   private state: VoiceCallState = { kind: 'idle' };
   private messages: VoiceMessage[] = [];
+  /**
+   * Each message's text by the item it was spoken in, in the order the items
+   * began: an assistant turn that called a tool is one message of two items.
+   */
+  private readonly itemTexts = new Map<string, Map<string, string>>();
+  /** The message each item belongs to. */
+  private readonly messageOfItem = new Map<string, string>();
   private muted = false;
   private endingSoonMs: number | null = null;
   private model: VoiceCallSnapshot['model'] = null;
@@ -362,10 +367,10 @@ export class VoiceCall {
           this.setState({ kind: 'thinking' });
         return;
       case 'transcript.delta':
-        this.upsertMessage(event.role, event.itemId, (previous) => previous + event.text, true);
+        this.upsertMessage(event, (previous) => previous + event.text, true);
         return;
       case 'transcript.done':
-        this.upsertMessage(event.role, event.itemId, () => event.text, false);
+        this.upsertMessage(event, () => event.text, false);
         return;
       case 'assistant.audio_done':
         return;
@@ -379,11 +384,13 @@ export class VoiceCall {
         if (this.state.kind === 'thinking' && this.playingItem === null) this.afterSpeech();
         return;
       case 'tool.started':
+        this.recordTool(event, 'call');
         this.activeTools.set(event.callId, event.name);
         this.toolRan = true;
         if (LIVE.has(this.state.kind)) this.setState({ kind: 'using-tool', tool: event.name });
         return;
       case 'tool.finished': {
+        this.recordTool(event, 'result');
         this.activeTools.delete(event.callId);
         if (this.state.kind !== 'using-tool') return;
         const still = [...this.activeTools.values()].at(-1);
@@ -392,9 +399,6 @@ export class VoiceCall {
         );
         return;
       }
-      case 'approval.requested':
-        this.options.onApprovalRequested?.(event.toolName);
-        return;
       case 'messages.saved':
         this.options.onMessagesSaved?.(event.conversationId, event.messageIds);
         return;
@@ -455,41 +459,79 @@ export class VoiceCall {
     this.finish({ kind: 'ended', reason });
   }
 
+  /**
+   * One message per `messageId`, the id the server stores and announces it
+   * under, so the stored copy that reaches the thread is this message, not a
+   * second one. Its text is its items' texts in order.
+   */
   private upsertMessage(
-    role: 'user' | 'assistant',
-    itemId: string,
-    content: (previous: string) => string,
+    event: { role: 'user' | 'assistant'; messageId: string; itemId: string },
+    text: (previous: string) => string,
     isStreaming: boolean,
   ): void {
-    const id = `${role}-${itemId}`;
-    const index = this.messages.findIndex((message) => message.id === id);
-    if (index === -1) {
-      this.messages = [
-        ...this.messages,
-        {
-          id,
-          role,
-          ...(role === 'assistant' ? { speaker: 'primary' as const } : {}),
-          content: content(''),
-          timestamp: Date.now(),
-          isStreaming,
-        },
-      ];
-    } else {
-      const previous = this.messages[index]!;
-      const next = [...this.messages];
-      next[index] = { ...previous, content: content(previous.content), isStreaming };
-      this.messages = next;
-    }
+    const { messageId, itemId } = event;
+    const items = this.itemTexts.get(messageId) ?? new Map<string, string>();
+    this.itemTexts.set(messageId, items);
+    this.messageOfItem.set(itemId, messageId);
+    items.set(itemId, text(items.get(itemId) ?? ''));
+    const content = [...items.values()].filter((part) => part !== '').join(' ');
+    this.writeMessage(messageId, event.role, (message) => ({ ...message, content, isStreaming }));
+  }
+
+  /**
+   * A tool the answer ran, inside the answer's message, by name and state: the
+   * device never receives its arguments or result, and the stored message that
+   * carries them replaces this one once it is saved.
+   */
+  private recordTool(
+    event: { messageId: string; callId: string; name: string },
+    state: 'call' | 'result',
+  ): void {
+    this.writeMessage(event.messageId, 'assistant', (message) => {
+      const invocation = { toolCallId: event.callId, toolName: event.name, state };
+      const tools = message.toolInvocations ?? [];
+      const index = tools.findIndex((tool) => tool.toolCallId === event.callId);
+      return {
+        ...message,
+        toolInvocations:
+          index === -1
+            ? [...tools, invocation]
+            : tools.map((tool, at) => (at === index ? invocation : tool)),
+      };
+    });
+  }
+
+  private writeMessage(
+    messageId: string,
+    role: 'user' | 'assistant',
+    change: (message: VoiceMessage) => VoiceMessage,
+  ): void {
+    const index = this.messages.findIndex((message) => message.id === messageId);
+    const current: VoiceMessage =
+      index === -1
+        ? {
+            id: messageId,
+            role,
+            ...(role === 'assistant' ? { speaker: 'primary' as const } : {}),
+            content: '',
+            timestamp: Date.now(),
+            isStreaming: false,
+          }
+        : this.messages[index]!;
+    const next = change(current);
+    this.messages =
+      index === -1
+        ? [...this.messages, next]
+        : this.messages.map((message, at) => (at === index ? next : message));
     this.emit();
   }
 
   private settleStreaming(itemId: string | null): void {
-    if (itemId === null) return;
-    const id = `assistant-${itemId}`;
-    if (!this.messages.some((message) => message.id === id && message.isStreaming)) return;
+    const messageId = itemId === null ? undefined : this.messageOfItem.get(itemId);
+    if (messageId === undefined) return;
+    if (!this.messages.some((message) => message.id === messageId && message.isStreaming)) return;
     this.messages = this.messages.map((message) =>
-      message.id === id ? { ...message, isStreaming: false } : message,
+      message.id === messageId ? { ...message, isStreaming: false } : message,
     );
     this.emit();
   }

package/src/lib/voice-controls-labels.ts

@@ -0,0 +1,31 @@
+// The call bar's words, apart from the component so the chat's labels can hold them without
+// reaching React Native.
+
+/** Every word the bar shows or announces. An app in another language passes its own. */
+export interface VoiceControlsLabels {
+  connecting: string;
+  reconnecting: string;
+  connected: string;
+  listening: string;
+  muted: string;
+  thinking: string;
+  usingTool: string;
+  speaking: string;
+  mute: string;
+  unmute: string;
+  end: string;
+}
+
+export const VOICE_CONTROLS_LABELS: VoiceControlsLabels = {
+  connecting: 'Connecting...',
+  reconnecting: 'Reconnecting...',
+  connected: 'Connected',
+  listening: 'Listening...',
+  muted: 'Muted',
+  thinking: 'Thinking...',
+  usingTool: 'Working on it...',
+  speaking: 'Speaking...',
+  mute: 'Mute',
+  unmute: 'Unmute',
+  end: 'End',
+};

package/src/lib/voice-protocol.ts

@@ -13,7 +13,11 @@
  *   `session.accepted` carries `seq`; a resume replays from `afterSeq`.
  */
 
-export const VOICE_PROTOCOL_VERSION = 1;
+/**
+ * 2: every transcript and tool event names `messageId`, the message it belongs
+ * to as it is stored.
+ */
+export const VOICE_PROTOCOL_VERSION = 2;
 
 export const VOICE_SESSION_PATH = '/v1/voice/session';
 
@@ -107,17 +111,20 @@ export type VoiceServerEvent =
   | { type: 'input.ready' }
   | { type: 'user.speech_started'; itemId: string }
   | { type: 'user.speech_stopped'; itemId: string }
-  | { type: 'transcript.delta'; role: VoiceRole; itemId: string; text: string }
-  | { type: 'transcript.done'; role: VoiceRole; itemId: string; text: string }
+  /**
+   * `messageId` is the message the text belongs to, the id it is stored and
+   * announced under: an assistant turn's items are one message.
+   */
+  | { type: 'transcript.delta'; role: VoiceRole; messageId: string; itemId: string; text: string }
+  | { type: 'transcript.done'; role: VoiceRole; messageId: string; itemId: string; text: string }
   | { type: 'assistant.audio_done'; itemId: string }
   | {
       type: 'assistant.done';
       itemId: string | null;
       status: 'completed' | 'cancelled' | 'incomplete' | 'failed';
     }
-  | { type: 'tool.started'; callId: string; name: string }
-  | { type: 'tool.finished'; callId: string; name: string; ok: boolean }
-  | { type: 'approval.requested'; toolName: string }
+  | { type: 'tool.started'; messageId: string; callId: string; name: string }
+  | { type: 'tool.finished'; messageId: string; callId: string; name: string; ok: boolean }
   | { type: 'messages.saved'; conversationId: string; messageIds: string[] }
   | { type: 'session.warning'; code: 'ending-soon'; remainingMs: number }
   | { type: 'session.reconnecting' }
@@ -196,9 +203,17 @@ export function parseVoiceServerEvent(raw: string): ReceivedVoiceEvent | null {
     case 'transcript.delta':
     case 'transcript.done':
       return (event.role === 'user' || event.role === 'assistant') &&
+        isString(event.messageId) &&
         isString(event.itemId) &&
         isString(event.text)
-        ? { type: event.type, role: event.role, itemId: event.itemId, text: event.text, ...seq }
+        ? {
+            type: event.type,
+            role: event.role,
+            messageId: event.messageId,
+            itemId: event.itemId,
+            text: event.text,
+            ...seq,
+          }
         : null;
     case 'assistant.done':
       return event.itemId === null || isString(event.itemId)
@@ -215,23 +230,26 @@ export function parseVoiceServerEvent(raw: string): ReceivedVoiceEvent | null {
           }
         : null;
     case 'tool.started':
-      return isString(event.callId) && isString(event.name)
-        ? { type: 'tool.started', callId: event.callId, name: event.name, ...seq }
+      return isString(event.messageId) && isString(event.callId) && isString(event.name)
+        ? {
+            type: 'tool.started',
+            messageId: event.messageId,
+            callId: event.callId,
+            name: event.name,
+            ...seq,
+          }
         : null;
     case 'tool.finished':
-      return isString(event.callId) && isString(event.name)
+      return isString(event.messageId) && isString(event.callId) && isString(event.name)
         ? {
             type: 'tool.finished',
+            messageId: event.messageId,
             callId: event.callId,
             name: event.name,
             ok: event.ok === true,
             ...seq,
           }
         : null;
-    case 'approval.requested':
-      return isString(event.toolName)
-        ? { type: 'approval.requested', toolName: event.toolName, ...seq }
-        : null;
     case 'messages.saved':
       return isString(event.conversationId) && Array.isArray(event.messageIds)
         ? {

package/src/lib/voice-worklets.ts

@@ -0,0 +1,19 @@
+import { ALIA_WEB_ORIGIN } from './origins';
+
+/**
+ * The voice call's AudioWorklet processors (`worklets/voice-processors.js`), as the web engine
+ * loads them.
+ *
+ * Not a `blob:` URL: a worklet module is a script, so a blob would need `script-src blob:` in
+ * every host's CSP, which would let any blob the page makes run as script. Alia's web app serves
+ * the file instead, under `script-src https://alia.onl` that the avatar already needs
+ * (`@alia.onl/sdk/csp`), with CORS open for the cross-origin `addModule`.
+ *
+ * The name is the file's content hash, so an SDK embedded in another app (on its own release
+ * cycle) always loads the bytes it was written against: `packages/app/public/sdk/voice-worklets/`
+ * keeps every revision ever published, and a test fails if this revision is not the hash of the
+ * file or is not served.
+ */
+export const VOICE_WORKLETS_REVISION = '543d4a8e42937e99';
+
+export const VOICE_WORKLETS_URL = `${ALIA_WEB_ORIGIN}/sdk/voice-worklets/${VOICE_WORKLETS_REVISION}.js`;

package/src/lib/work-log.ts

@@ -8,12 +8,12 @@ import {
 } from '@oxy.so/bloom/icons';
 import type { TaskListChip, TaskListStep, TaskListTask } from '@oxy.so/bloom/task-list';
 import type { WebSearchBrand, WebSearchSource, WebSearchStep } from '@oxy.so/bloom/web-search';
-import type { ToolInvocation } from '../types';
+import type { ToolInvocation } from '../runtime/types';
 import { extractCitationSources } from './citations';
 import { domainOf } from './domain';
 import { getToolDoneLabel, getToolPillLabel } from './task-labels';
 import { toolCallStatus } from './tool-call-status';
-import { englishWorkLog, type Translate } from './work-log-copy';
+import { englishCopy, type Translate } from './copy';
 
 /**
  * A turn's tool calls, as the two Bloom logs draw them.
@@ -32,7 +32,7 @@ import { englishWorkLog, type Translate } from './work-log-copy';
 
 /** What a host can swap: the words, and where a site's icon is served from. */
 export interface WorkLogOptions {
-  /** Translator for the log's words (see `work-log-copy.ts`). Default: English. */
+  /** Translator for the log's words (see `copy.ts`). Default: English. */
   t?: Translate;
   /** The URL of a site's cached icon, or nothing for Bloom's quiet dot. */
   faviconUrl?: (url: string) => string | null | undefined;
@@ -154,7 +154,7 @@ function webStep(
   live: LiveResearch | undefined,
   options: WorkLogOptions,
 ): MappedStep {
-  const t = options.t ?? englishWorkLog;
+  const t = options.t ?? englishCopy;
   const arrived = inv.state !== 'partial-call';
   const ok = succeeded(inv);
   const res = resultOf(inv);
@@ -256,7 +256,7 @@ export function webSearchLog(
   research: LiveResearch | undefined,
   options: WorkLogOptions = {},
 ): WebSearchLog {
-  const t = options.t ?? englishWorkLog;
+  const t = options.t ?? englishCopy;
   const web = (invocations ?? []).filter(isWebInvocation);
   const mapped = web.map((inv) => webStep(inv, research, options));
 
@@ -333,7 +333,7 @@ export function taskListLog(
   live: boolean,
   options: WorkLogOptions = {},
 ): TaskListLog {
-  const t = options.t ?? englishWorkLog;
+  const t = options.t ?? englishCopy;
   let revealed = 0;
   let open = true;
   const tasks = (invocations ?? [])

package/src/runtime/agents.ts

@@ -0,0 +1,151 @@
+import { useOxy } from '@oxy.so/services/ui/client';
+import { useQuery } from '@tanstack/react-query';
+import { useAliaApi } from './api';
+import { aliaQueryKeys } from './query-keys';
+import { API_ROUTES } from './routes';
+
+/**
+ * The agent a chat is opened on, as `GET /v1/agents/:id/profile` answers it
+ * (`docs/agent-embedding.md` §1). Alia herself has no profile: she is the lead when no agent is
+ * named.
+ */
+export interface AliaAgentProfile {
+  id: string;
+  name: string;
+  handle?: string;
+  /** Its look, as Oxy stores a character (JSON); drawn through `readAgentCharacter`. */
+  character?: unknown;
+  color?: string;
+  description?: string;
+  /** What the empty chat says; omitted, the chat's own greeting. */
+  greeting?: string;
+  /** The lines offered over the composer of an empty chat. */
+  starters: AliaAgentStarter[];
+  access?: 'private' | 'public';
+  /** Which of Alia's features this agent gates behind an app's own check. */
+  features: Array<{ key: string; gated: boolean }>;
+}
+
+export interface AliaAgentStarter {
+  title: string;
+  prompt: string;
+}
+
+/** One of Alia's features, for one agent and the person present (`GET /v1/agents/:id/features`). */
+export interface AliaAgentFeature {
+  key: string;
+  state: 'available' | 'locked';
+  /** How many uses are left, when the publishing app counts them. */
+  remaining?: number;
+}
+
+/** The feature keys the chat itself reads: the attach menu's two rows. */
+export const ALIA_FEATURES = {
+  documents: 'attachments.documents',
+  images: 'attachments.images',
+} as const;
+
+const FIVE_MINUTES = 1000 * 60 * 5;
+
+/** A string, or nothing: the profile is read field by field rather than cast. */
+const text = (value: unknown): string | undefined =>
+  typeof value === 'string' && value !== '' ? value : undefined;
+
+/** The profile the server sent, kept only where it has the shape the contract names. */
+export function readAgentProfile(value: unknown): AliaAgentProfile | null {
+  if (value === null || typeof value !== 'object') return null;
+  const raw = value as Record<string, unknown>;
+  const id = text(raw.id);
+  const name = text(raw.name);
+  if (id === undefined || name === undefined) return null;
+  const starters = Array.isArray(raw.starters)
+    ? raw.starters.flatMap((starter): AliaAgentStarter[] => {
+        if (starter === null || typeof starter !== 'object') return [];
+        const { title, prompt } = starter as Record<string, unknown>;
+        const line = text(prompt);
+        return line === undefined ? [] : [{ title: text(title) ?? line, prompt: line }];
+      })
+    : [];
+  const features = Array.isArray(raw.features)
+    ? raw.features.flatMap((feature): Array<{ key: string; gated: boolean }> => {
+        if (feature === null || typeof feature !== 'object') return [];
+        const { key, gated } = feature as Record<string, unknown>;
+        return typeof key === 'string' ? [{ key, gated: gated === true }] : [];
+      })
+    : [];
+  return {
+    id,
+    name,
+    handle: text(raw.handle),
+    character: raw.character ?? undefined,
+    color: text(raw.color),
+    description: text(raw.description),
+    greeting: text(raw.greeting),
+    starters,
+    access: raw.access === 'public' || raw.access === 'private' ? raw.access : undefined,
+    features,
+  };
+}
+
+/** The feature states the server sent; an entry of any other shape is not a state. */
+export function readAgentFeatures(value: unknown): AliaAgentFeature[] {
+  const list =
+    value !== null && typeof value === 'object'
+      ? (value as { features?: unknown }).features
+      : undefined;
+  if (!Array.isArray(list)) return [];
+  return list.flatMap((feature): AliaAgentFeature[] => {
+    if (feature === null || typeof feature !== 'object') return [];
+    const { key, state, remaining } = feature as Record<string, unknown>;
+    if (typeof key !== 'string' || (state !== 'available' && state !== 'locked')) return [];
+    return [{ key, state, ...(typeof remaining === 'number' ? { remaining } : {}) }];
+  });
+}
+
+/** The profile of the agent `agentId`, or nothing for Alia (no id) and while it loads. */
+export function useAgentProfile(agentId: string | undefined) {
+  const { isAuthenticated } = useOxy();
+  const api = useAliaApi();
+  return useQuery({
+    queryKey: aliaQueryKeys.agents.profile(agentId ?? ''),
+    queryFn: async () =>
+      readAgentProfile((await api.get<unknown>(API_ROUTES.agents.profile(agentId ?? ''))).data),
+    enabled: agentId !== undefined && isAuthenticated,
+    staleTime: FIVE_MINUTES,
+    retry: 1,
+  });
+}
+
+/**
+ * What of Alia's features the agent `agentId` offers the person present. Alia (no id) gates
+ * nothing, so she has no answer to wait for; see `featureRows` for what the chat makes of it.
+ */
+export function useAgentFeatures(agentId: string | undefined) {
+  const { isAuthenticated } = useOxy();
+  const api = useAliaApi();
+  return useQuery({
+    queryKey: aliaQueryKeys.agents.features(agentId ?? ''),
+    queryFn: async () =>
+      readAgentFeatures((await api.get<unknown>(API_ROUTES.agents.features(agentId ?? ''))).data),
+    enabled: agentId !== undefined && isAuthenticated,
+    // A gate's state moves with the person's plan in another app: read it each time the chat opens.
+    staleTime: 0,
+    retry: 1,
+  });
+}
+
+/** How an attachment feature is offered: attached, an upsell, or not at all. */
+export type FeatureOffer = 'available' | 'locked' | 'absent';
+
+/**
+ * What the attach menu offers for `key`. Alia's own chat (`features === undefined`) offers every
+ * attachment; an agent offers what its feature answer lists — `locked` is still offered, as the
+ * host's upsell — and nothing it does not list.
+ */
+export function featureOffer(
+  features: readonly AliaAgentFeature[] | undefined,
+  key: string,
+): FeatureOffer {
+  if (features === undefined) return 'available';
+  return features.find((feature) => feature.key === key)?.state ?? 'absent';
+}