@alia.onl/sdk 15.8.0 → 19.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/CHANGELOG.md +242 -13
  2. package/README.md +223 -151
  3. package/package.json +15 -5
  4. package/src/animations.ts +0 -1
  5. package/src/components/AliaAskProvider.tsx +127 -0
  6. package/src/components/AliaChatContent.tsx +604 -392
  7. package/src/components/AliaChatHeader.tsx +14 -38
  8. package/src/components/AliaChatScreen.tsx +56 -31
  9. package/src/components/AliaChatSheet.tsx +110 -36
  10. package/src/components/AliaComposer.tsx +1 -1
  11. package/src/components/AliaParticipants.tsx +202 -0
  12. package/src/components/AliaParticipantsHeader.tsx +60 -0
  13. package/src/components/AliaSharedConversation.tsx +86 -0
  14. package/src/components/AliaWelcomeMessage.tsx +18 -5
  15. package/src/components/Image.tsx +12 -0
  16. package/src/components/Markdown.tsx +498 -14
  17. package/src/components/PlanPreviewCard.tsx +7 -49
  18. package/src/components/ReplyMarkdown.tsx +359 -0
  19. package/src/components/RichBlocks.tsx +195 -0
  20. package/src/components/canvas/CanvasComponent.tsx +59 -0
  21. package/src/components/canvas/ChartRenderer.tsx +73 -0
  22. package/src/components/canvas/CodeRenderer.tsx +24 -0
  23. package/src/components/canvas/FormRenderer.tsx +101 -0
  24. package/src/components/canvas/MarkdownRenderer.tsx +14 -0
  25. package/src/components/canvas/TableRenderer.tsx +47 -0
  26. package/src/components/thought/FilesAndSources.tsx +120 -0
  27. package/src/components/thought/ThoughtPanel.tsx +417 -0
  28. package/src/components/thought/ToolStep.tsx +141 -0
  29. package/src/components/thread/AgentResultCard.tsx +125 -0
  30. package/src/components/thread/AgentTaskCard.tsx +127 -0
  31. package/src/components/thread/AliaThread.tsx +401 -0
  32. package/src/components/thread/MessageBlockBoundary.tsx +78 -0
  33. package/src/components/thread/NewConversationOffer.tsx +54 -0
  34. package/src/components/thread/ThreadRow.tsx +458 -0
  35. package/src/components/thread/ThreadSearch.tsx +129 -0
  36. package/src/components/thread/ToolResultCard.tsx +254 -0
  37. package/src/components/thread/TurnMenu.tsx +54 -0
  38. package/src/components/thread/TurnStatusLine.tsx +34 -0
  39. package/src/components/voice/AliaVoiceCall.tsx +21 -0
  40. package/src/components/voice/VoiceControls.tsx +2 -28
  41. package/src/components/voice/VoiceSession.tsx +39 -11
  42. package/src/hooks/useAliaActivity.ts +2 -2
  43. package/src/hooks/useAtBottom.ts +51 -0
  44. package/src/hooks/useAttachMenu.ts +94 -0
  45. package/src/hooks/useChatComposer.ts +299 -0
  46. package/src/hooks/useChatVoice.ts +129 -0
  47. package/src/hooks/useDocumentPicker.ts +3 -29
  48. package/src/hooks/useFailedTurnToast.ts +43 -0
  49. package/src/hooks/useMessageVote.ts +50 -0
  50. package/src/hooks/useOpenThought.ts +79 -0
  51. package/src/hooks/usePendingQuestions.ts +2 -2
  52. package/src/hooks/useReadAloud.ts +92 -0
  53. package/src/hooks/useShareHandler.ts +24 -0
  54. package/src/hooks/useSoundEffects.ts +1 -1
  55. package/src/hooks/useThoughtScope.ts +59 -0
  56. package/src/hooks/useThreadRows.ts +145 -0
  57. package/src/hooks/useThreadView.ts +68 -0
  58. package/src/hooks/useTimelineWindow.ts +256 -0
  59. package/src/hooks/useVoiceLauncher.ts +180 -0
  60. package/src/hooks/useVoiceRoom.ts +0 -3
  61. package/src/index.ts +43 -29
  62. package/src/lib/activity-store.ts +1 -1
  63. package/src/lib/add-menu.ts +7 -0
  64. package/src/lib/call-thread.ts +33 -0
  65. package/src/lib/citations.ts +79 -2
  66. package/src/lib/clipboard.ts +23 -0
  67. package/src/lib/content-utils.ts +10 -0
  68. package/src/lib/copy-en.ts +284 -0
  69. package/src/lib/copy.ts +32 -0
  70. package/src/lib/document-picker.ts +31 -0
  71. package/src/lib/flights.ts +22 -13
  72. package/src/lib/focus-return.ts +66 -0
  73. package/src/lib/keyboard.native.tsx +19 -0
  74. package/src/lib/keyboard.tsx +9 -0
  75. package/src/lib/labels.tsx +69 -17
  76. package/src/lib/participants.ts +86 -0
  77. package/src/lib/power-level-options.ts +66 -0
  78. package/src/lib/questions.ts +2 -2
  79. package/src/lib/task-labels.ts +4 -4
  80. package/src/lib/tool-call-status.ts +1 -1
  81. package/src/lib/tool-icons.ts +36 -0
  82. package/src/lib/tool-renderers.tsx +48 -0
  83. package/src/lib/turn-state.ts +2 -2
  84. package/src/lib/voice-call-request.ts +49 -0
  85. package/src/lib/voice-call.ts +76 -34
  86. package/src/lib/voice-controls-labels.ts +31 -0
  87. package/src/lib/voice-protocol.ts +32 -14
  88. package/src/lib/work-log.ts +6 -6
  89. package/src/runtime/agents.ts +151 -0
  90. package/src/runtime/api.ts +123 -0
  91. package/src/runtime/attachment-utils.ts +96 -0
  92. package/src/runtime/composer-draft-store.ts +270 -0
  93. package/src/runtime/config.tsx +186 -0
  94. package/src/runtime/conversation-store.ts +49 -0
  95. package/src/runtime/conversations.ts +557 -0
  96. package/src/runtime/error-utils.ts +104 -0
  97. package/src/runtime/host-tools.ts +140 -0
  98. package/src/runtime/index.ts +113 -0
  99. package/src/runtime/message-history.ts +124 -0
  100. package/src/runtime/model.ts +39 -0
  101. package/src/runtime/oxy-files.ts +37 -0
  102. package/src/runtime/power-levels.ts +66 -0
  103. package/src/runtime/query-keys.ts +34 -0
  104. package/src/runtime/random-uuid.ts +25 -0
  105. package/src/runtime/routes.ts +51 -0
  106. package/src/runtime/sharing.ts +165 -0
  107. package/src/runtime/sse-frame-reader.ts +88 -0
  108. package/src/runtime/stream-events.ts +146 -0
  109. package/src/runtime/stream-outcome.ts +21 -0
  110. package/src/runtime/thought-store.ts +103 -0
  111. package/src/runtime/thread/agent-activity.ts +261 -0
  112. package/src/runtime/thread/automation-labels.ts +79 -0
  113. package/src/runtime/thread/conversation-export.ts +135 -0
  114. package/src/runtime/thread/message-days.ts +120 -0
  115. package/src/runtime/thread/message-processor.ts +93 -0
  116. package/src/runtime/thread/task-utils.ts +11 -0
  117. package/src/runtime/thread/thought-utils.ts +752 -0
  118. package/src/runtime/thread/thread-history.ts +139 -0
  119. package/src/runtime/thread/timeline.ts +202 -0
  120. package/src/runtime/thread/title-tags.ts +8 -0
  121. package/src/runtime/thread/tool-cards.ts +383 -0
  122. package/src/runtime/turn-failure.ts +108 -0
  123. package/src/runtime/types.ts +148 -0
  124. package/src/runtime/usage-limit-error.ts +44 -0
  125. package/src/runtime/use-agent-activity.ts +80 -0
  126. package/src/runtime/use-conversation-stream.ts +1427 -0
  127. package/src/runtime/use-conversation-thread.ts +556 -0
  128. package/src/runtime/use-start-conversation.ts +107 -0
  129. package/src/runtime/use-thread-history.ts +64 -0
  130. package/src/runtime/use-thread-search.ts +95 -0
  131. package/src/runtime/use-turn-edit.ts +99 -0
  132. package/src/tools.ts +2 -2
  133. package/src/types.ts +18 -47
  134. package/src/components/AliaChatMessageList.tsx +0 -532
  135. package/src/components/ThinkingBlock.tsx +0 -43
  136. package/src/hooks/useAliaChat.ts +0 -573
  137. package/src/lib/catalogue.ts +0 -236
  138. package/src/lib/chat-stream.ts +0 -655
  139. package/src/lib/chat-transport.ts +0 -98
  140. package/src/lib/delegation.ts +0 -34
  141. package/src/lib/work-log-copy.ts +0 -61
@@ -0,0 +1,86 @@
1
+ import type { AvatarCharacter } from '@oxy.so/bloom/agent-avatar';
2
+ import type { AliaAgentProfile } from '../runtime/agents';
3
+ import type { Message } from '../runtime/types';
4
+ import { readAgentCharacter } from './agent-character';
5
+ import { ALIA_AVATAR } from './alia-character';
6
+
7
+ /** Somebody in the conversation who answers: its lead, or an agent the lead consulted. */
8
+ export interface AliaParticipant {
9
+ id: string;
10
+ name: string;
11
+ handle?: string;
12
+ /** The 3D character it is drawn as. */
13
+ character: AvatarCharacter;
14
+ color?: string;
15
+ }
16
+
17
+ /** The conversation's lead: who greets, whose name is on the composer, who the header shows first. */
18
+ export interface AliaLead extends AliaParticipant {
19
+ /** Alia herself, rather than an agent the chat was opened on. */
20
+ isAlia: boolean;
21
+ /** The agent's own greeting; Alia greets with the chat's labels. */
22
+ greeting?: string;
23
+ /** The agent's own starters, offered over the composer of an empty chat. */
24
+ starters: Array<{ title: string; prompt: string }>;
25
+ }
26
+
27
+ /** Who is in the conversation, as the header draws it: the lead, then whoever has joined. */
28
+ export interface AliaChatParticipants {
29
+ lead: AliaLead;
30
+ /** The consulted agents that have joined the header, in the order they joined. */
31
+ joined: AliaParticipant[];
32
+ }
33
+
34
+ /** Alia as a lead. */
35
+ export const ALIA_LEAD: AliaLead = {
36
+ id: 'alia',
37
+ name: 'Alia',
38
+ handle: 'alia',
39
+ character: ALIA_AVATAR,
40
+ isAlia: true,
41
+ starters: [],
42
+ };
43
+
44
+ /** The lead of a chat: the agent it was opened on (`agent={{ id }}`), or Alia. */
45
+ export function leadOf(profile: AliaAgentProfile | null | undefined): AliaLead {
46
+ if (!profile) return ALIA_LEAD;
47
+ return {
48
+ id: profile.id,
49
+ name: profile.name,
50
+ handle: profile.handle,
51
+ character: readAgentCharacter(profile.character),
52
+ color: profile.color,
53
+ isAlia: false,
54
+ greeting: profile.greeting,
55
+ starters: profile.starters,
56
+ };
57
+ }
58
+
59
+ /** A consulted agent as a message carries it (`agentInfo`). */
60
+ export function participantOf(agentInfo: NonNullable<Message['agentInfo']>): AliaParticipant {
61
+ return {
62
+ id: agentInfo.id,
63
+ name: agentInfo.name,
64
+ handle: agentInfo.handle,
65
+ character: readAgentCharacter(agentInfo.character),
66
+ color: agentInfo.color,
67
+ };
68
+ }
69
+
70
+ /**
71
+ * The agents that answered in `messages`, besides the lead, in the order they first did — each
72
+ * with whether its first answer is the one being streamed now (it is joining) or one already
73
+ * written (it was there when the chat opened).
74
+ */
75
+ export function consultedAgents(
76
+ messages: readonly Message[],
77
+ leadId: string,
78
+ ): Array<{ participant: AliaParticipant; live: boolean }> {
79
+ const seen = new Map<string, { participant: AliaParticipant; live: boolean }>();
80
+ for (const message of messages) {
81
+ const info = message.agentInfo;
82
+ if (!info || info.id === leadId || seen.has(info.id)) continue;
83
+ seen.set(info.id, { participant: participantOf(info), live: message.isStreaming === true });
84
+ }
85
+ return [...seen.values()];
86
+ }
@@ -0,0 +1,66 @@
1
+ import type { ComposerPanelPermissionOption } from '@oxy.so/bloom/composer-panel';
2
+ import { RiComputerLine } from '@oxy.so/bloom/icons/RiComputerLine';
3
+ import { RiFireLine } from '@oxy.so/bloom/icons/RiFireLine';
4
+ import { RiFlashlightLine } from '@oxy.so/bloom/icons/RiFlashlightLine';
5
+ import { RiLightbulbLine } from '@oxy.so/bloom/icons/RiLightbulbLine';
6
+ import { RiRocket2Line } from '@oxy.so/bloom/icons/RiRocket2Line';
7
+ import { RiSparkling2Line } from '@oxy.so/bloom/icons/RiSparkling2Line';
8
+ import { RiSpeedUpLine } from '@oxy.so/bloom/icons/RiSpeedUpLine';
9
+ import { RiVipCrownLine } from '@oxy.so/bloom/icons/RiVipCrownLine';
10
+ import { POWER_LEVELS, type PowerLevel } from '../runtime/power-levels';
11
+ import type { Translate } from './copy';
12
+
13
+ /**
14
+ * The power-level selector, in the shape Bloom's composer mode selector takes.
15
+ *
16
+ * Alia's users choose how much power a turn gets, never which hosted model answers it: Oxy picks
17
+ * the model of the chosen level for every request. The selector is `ComposerPill`'s mode picker —
18
+ * Bloom's own radio menu of a label, a short description and an icon per mode — so the seven
19
+ * levels read the same way every Bloom mode selector does. Models running on the person's OWN
20
+ * devices follow the levels: that is their machine, not a model Alia offers, and which devices
21
+ * are connected is the host's to say.
22
+ */
23
+
24
+ type IconComponent = ComposerPanelPermissionOption['icon'];
25
+
26
+ /** One glyph per level, rising with the power it stands for. */
27
+ export const POWER_LEVEL_ICONS: Record<PowerLevel, IconComponent> = {
28
+ auto: RiSparkling2Line,
29
+ instant: RiFlashlightLine,
30
+ medium: RiSpeedUpLine,
31
+ high: RiLightbulbLine,
32
+ xhigh: RiFireLine,
33
+ pro: RiRocket2Line,
34
+ ultra: RiVipCrownLine,
35
+ };
36
+
37
+ /** A model running on one of the person's own devices, offered after the levels. */
38
+ export interface DeviceModel {
39
+ /** What a request carries as `model` (`local/<runtime>/<model>`). */
40
+ id: string;
41
+ /** The tag the person knows it by, e.g. `llama3.1:8b`. */
42
+ name: string;
43
+ /** Which of their devices is running it. */
44
+ deviceLabel: string;
45
+ }
46
+
47
+ /** The selector's options: the seven levels, then the person's device models. */
48
+ export function buildPowerLevelOptions(
49
+ t: Translate,
50
+ deviceModels: readonly DeviceModel[],
51
+ ): ComposerPanelPermissionOption[] {
52
+ return [
53
+ ...POWER_LEVELS.map((level) => ({
54
+ id: level,
55
+ label: t(`powerLevels.${level}.label`),
56
+ description: t(`powerLevels.${level}.description`),
57
+ icon: POWER_LEVEL_ICONS[level],
58
+ })),
59
+ ...deviceModels.map((model) => ({
60
+ id: model.id,
61
+ label: model.name,
62
+ description: t('powerLevels.onDevice', { device: model.deviceLabel }),
63
+ icon: RiComputerLine,
64
+ })),
65
+ ];
66
+ }
@@ -4,7 +4,7 @@
4
4
  // call: it is stored with the message, survives a reload, and is answered by the person's
5
5
  // next message. Pure functions, no React Native, so a host's models and tests can use them.
6
6
 
7
- import type { ChatMessage, ToolInvocation } from '../types';
7
+ import type { Message, ToolInvocation } from '../runtime/types';
8
8
 
9
9
  /** The tool's name on the wire. */
10
10
  export const ASK_USER_QUESTION_TOOL = 'askUserQuestion';
@@ -83,7 +83,7 @@ export function formatAnswers(questions: AskedQuestion[], answers: QuestionAnswe
83
83
  * is the answer, so nothing is pending then.
84
84
  */
85
85
  export function pendingQuestions(
86
- messages: readonly Pick<ChatMessage, 'role' | 'toolInvocations'>[],
86
+ messages: readonly Pick<Message, 'role' | 'toolInvocations'>[],
87
87
  isStreaming: boolean,
88
88
  ): AskedQuestion[] {
89
89
  const last = messages.at(-1);
@@ -1,13 +1,13 @@
1
- import { englishWorkLog, type Translate } from './work-log-copy';
1
+ import { englishCopy, type Translate } from './copy';
2
2
 
3
3
  /** Whether the catalogue names this tool itself; otherwise the label is made from its id. */
4
4
  function isNamed(kind: 'pill' | 'done', toolName: string): boolean {
5
5
  const key = `tasks.tool.${kind}.${toolName}`;
6
- return englishWorkLog(key) !== key;
6
+ return englishCopy(key) !== key;
7
7
  }
8
8
 
9
9
  /** A tool's short label, in the reader's language when a translator is given. */
10
- export function getToolPillLabel(toolName: string, t: Translate = englishWorkLog): string {
10
+ export function getToolPillLabel(toolName: string, t: Translate = englishCopy): string {
11
11
  if (isNamed('pill', toolName)) return t(`tasks.tool.pill.${toolName}`);
12
12
 
13
13
  // Oxy service tools: oxy_serviceName__toolName → "ServiceName"
@@ -28,7 +28,7 @@ export function getToolPillLabel(toolName: string, t: Translate = englishWorkLog
28
28
  * `runningTitle` for this once the task's steps land). Labels that are already names
29
29
  * ("Telegram", "Email") read the same either way, so they fall back to the running label.
30
30
  */
31
- export function getToolDoneLabel(toolName: string, t: Translate = englishWorkLog): string {
31
+ export function getToolDoneLabel(toolName: string, t: Translate = englishCopy): string {
32
32
  return isNamed('done', toolName)
33
33
  ? t(`tasks.tool.done.${toolName}`)
34
34
  : getToolPillLabel(toolName, t);
@@ -1,4 +1,4 @@
1
- import type { ToolInvocation } from '../types';
1
+ import type { ToolInvocation } from '../runtime/types';
2
2
 
3
3
  /**
4
4
  * Where a tool call stands:
@@ -0,0 +1,36 @@
1
+ import type { BloomIconComponent } from '@oxy.so/bloom/icons';
2
+ import { RiCalendarLine } from '@oxy.so/bloom/icons/RiCalendarLine';
3
+ import { RiChat3Line } from '@oxy.so/bloom/icons/RiChat3Line';
4
+ import { RiDatabase2Line } from '@oxy.so/bloom/icons/RiDatabase2Line';
5
+ import { RiFileTextLine } from '@oxy.so/bloom/icons/RiFileTextLine';
6
+ import { RiGlobalLine } from '@oxy.so/bloom/icons/RiGlobalLine';
7
+ import { RiLightbulbFlashLine } from '@oxy.so/bloom/icons/RiLightbulbFlashLine';
8
+ import { RiLink } from '@oxy.so/bloom/icons/RiLink';
9
+ import { RiSearchLine } from '@oxy.so/bloom/icons/RiSearchLine';
10
+ import { RiSendPlaneLine } from '@oxy.so/bloom/icons/RiSendPlaneLine';
11
+ import { RiSettings3Line } from '@oxy.so/bloom/icons/RiSettings3Line';
12
+ import { RiUserLine } from '@oxy.so/bloom/icons/RiUserLine';
13
+
14
+ /** The glyph a tool's step carries in the thought panel, by tool name. */
15
+ const TOOL_ICONS: ReadonlyMap<string, BloomIconComponent> = new Map([
16
+ ['webSearch', RiSearchLine],
17
+ ['scrapeURL', RiLink],
18
+ ['getTimeline', RiCalendarLine],
19
+ ['searchKnowledgeBase', RiDatabase2Line],
20
+ ['webScraper', RiGlobalLine],
21
+ ['browse', RiGlobalLine],
22
+ ['sendWhatsAppMessage', RiChat3Line],
23
+ ['getWhatsAppChats', RiChat3Line],
24
+ ['getWhatsAppMessages', RiChat3Line],
25
+ ['sendTelegramMessage', RiSendPlaneLine],
26
+ ['getCurrentDate', RiCalendarLine],
27
+ ['generateFile', RiFileTextLine],
28
+ ['saveUserMemory', RiLightbulbFlashLine],
29
+ ['updateUserPreferences', RiSettings3Line],
30
+ ['updateUserContext', RiUserLine],
31
+ ]);
32
+
33
+ /** A tool's glyph; a tool with none of its own reads as the web it reached. */
34
+ export function getToolIcon(toolName: string): BloomIconComponent {
35
+ return TOOL_ICONS.get(toolName) ?? RiGlobalLine;
36
+ }
@@ -0,0 +1,48 @@
1
+ import { createContext, type ReactNode, useContext } from 'react';
2
+
3
+ /**
4
+ * A tool's result as a host's renderer receives it (`docs/agent-embedding.md` §3): the app and the
5
+ * tool its canonical name names, what it was called with and what it returned.
6
+ */
7
+ export interface AliaToolResult {
8
+ /** The app of `oxy_<app>__<tool>` / `self_<app>__<tool>`. */
9
+ app: string;
10
+ tool: string;
11
+ toolCallId: string;
12
+ input?: Record<string, unknown>;
13
+ output: unknown;
14
+ /** The consulted agent that called it; absent for the lead's own calls. */
15
+ agentId?: string;
16
+ }
17
+
18
+ /**
19
+ * The host's own drawings of catalogue tools' results, keyed `<app>:<tool>` — `homiio:searchListings`
20
+ * for `oxy_homiio__searchListings`. Drawn before the generic tool card, for the lead's calls and a
21
+ * consulted agent's alike.
22
+ */
23
+ export type AliaToolRenderers = Readonly<Record<string, (result: AliaToolResult) => ReactNode>>;
24
+
25
+ /**
26
+ * `oxy_<app>__<tool>` or `self_<app>__<tool>`: the only names a renderer is keyed from. The app
27
+ * part is an Oxy app slug; the tool part is whatever follows the separator.
28
+ */
29
+ const CANONICAL_TOOL = /^(?:oxy|self)_([a-z][a-z0-9-]{0,63})__([A-Za-z0-9_-]+)$/;
30
+
31
+ /** The app and tool of a canonical catalogue tool name, or `null` for any other name. */
32
+ export function parseCatalogueTool(toolName: string): { app: string; tool: string } | null {
33
+ const match = CANONICAL_TOOL.exec(toolName);
34
+ return match ? { app: match[1], tool: match[2] } : null;
35
+ }
36
+
37
+ /** The renderer key of a tool name (`<app>:<tool>`), or `null` when it is not a catalogue tool. */
38
+ export function toolRendererKey(toolName: string): string | null {
39
+ const parsed = parseCatalogueTool(toolName);
40
+ return parsed ? `${parsed.app}:${parsed.tool}` : null;
41
+ }
42
+
43
+ const RenderersContext = createContext<AliaToolRenderers>({});
44
+
45
+ export const AliaToolRenderersProvider = RenderersContext.Provider;
46
+
47
+ /** The host's renderers, `{}` when it brought none. */
48
+ export const useToolRenderers = () => useContext(RenderersContext);
@@ -1,9 +1,9 @@
1
- import type { ChatMessage } from '../types';
1
+ import type { Message } from '../runtime/types';
2
2
  import type { AliaAvatarState } from '../components/AliaAvatar';
3
3
 
4
4
  /** What a chat turn is doing, from its messages: the one reading the header mark and every button share. */
5
5
  export function turnActivityState(
6
- messages: ReadonlyArray<ChatMessage>,
6
+ messages: ReadonlyArray<Message>,
7
7
  isStreaming: boolean,
8
8
  ): AliaAvatarState {
9
9
  if (!isStreaming) return 'idle';
@@ -0,0 +1,49 @@
1
+ import { create } from 'zustand';
2
+
3
+ /** A call somebody asked for, waiting for the screen that shows its conversation. */
4
+ export interface VoiceCallRequest {
5
+ readonly conversationId: string;
6
+ readonly accountId: string | null;
7
+ readonly at: number;
8
+ }
9
+
10
+ /** How long a request waits for its screen. A navigation that never arrives must not open a call later. */
11
+ export const CALL_REQUEST_TTL_MS = 15_000;
12
+
13
+ interface VoiceCallRequestStore {
14
+ request: VoiceCallRequest | null;
15
+ ask: (conversationId: string, accountId: string | null) => void;
16
+ clear: () => void;
17
+ }
18
+
19
+ /**
20
+ * Where a call is asked for, apart from where it opens.
21
+ *
22
+ * Starting a call and showing it are the same thing when the conversation is already on screen,
23
+ * and two screens apart when it is not: a new-chat route makes the conversation and the host
24
+ * navigates to it. Both go the same way. The launcher AND the screen of the conversation meet
25
+ * here — it asks, and whichever launcher is mounted for that conversation (that account, in time)
26
+ * takes the request and opens the call. The sheet, the screen and the app share this one path.
27
+ */
28
+ export const useVoiceCallRequests = create<VoiceCallRequestStore>((set) => ({
29
+ request: null,
30
+ ask: (conversationId, accountId) =>
31
+ set({ request: { conversationId, accountId, at: Date.now() } }),
32
+ clear: () => set({ request: null }),
33
+ }));
34
+
35
+ /** Whether `request` is for this conversation and account, and still worth opening. */
36
+ export function isRequestFor(
37
+ request: VoiceCallRequest | null,
38
+ conversationId: string | undefined,
39
+ accountId: string | null,
40
+ now = Date.now(),
41
+ ): boolean {
42
+ return (
43
+ request !== null &&
44
+ conversationId !== undefined &&
45
+ request.conversationId === conversationId &&
46
+ request.accountId === accountId &&
47
+ now - request.at <= CALL_REQUEST_TTL_MS
48
+ );
49
+ }
@@ -88,8 +88,6 @@ export interface VoiceCallOptions {
88
88
  readonly onChange: (snapshot: VoiceCallSnapshot) => void;
89
89
  /** Alia stored messages of the call in the conversation. */
90
90
  readonly onMessagesSaved?: (conversationId: string, messageIds: readonly string[]) => void;
91
- /** A tool filed an action for the person's OK; the card is in the conversation. */
92
- readonly onApprovalRequested?: (toolName: string) => void;
93
91
  readonly tokenRefreshMs?: number;
94
92
  readonly resumeRetryMs?: number;
95
93
  }
@@ -113,6 +111,13 @@ export class VoiceCall {
113
111
  private readonly options: VoiceCallOptions;
114
112
  private state: VoiceCallState = { kind: 'idle' };
115
113
  private messages: VoiceMessage[] = [];
114
+ /**
115
+ * Each message's text by the item it was spoken in, in the order the items
116
+ * began: an assistant turn that called a tool is one message of two items.
117
+ */
118
+ private readonly itemTexts = new Map<string, Map<string, string>>();
119
+ /** The message each item belongs to. */
120
+ private readonly messageOfItem = new Map<string, string>();
116
121
  private muted = false;
117
122
  private endingSoonMs: number | null = null;
118
123
  private model: VoiceCallSnapshot['model'] = null;
@@ -362,10 +367,10 @@ export class VoiceCall {
362
367
  this.setState({ kind: 'thinking' });
363
368
  return;
364
369
  case 'transcript.delta':
365
- this.upsertMessage(event.role, event.itemId, (previous) => previous + event.text, true);
370
+ this.upsertMessage(event, (previous) => previous + event.text, true);
366
371
  return;
367
372
  case 'transcript.done':
368
- this.upsertMessage(event.role, event.itemId, () => event.text, false);
373
+ this.upsertMessage(event, () => event.text, false);
369
374
  return;
370
375
  case 'assistant.audio_done':
371
376
  return;
@@ -379,11 +384,13 @@ export class VoiceCall {
379
384
  if (this.state.kind === 'thinking' && this.playingItem === null) this.afterSpeech();
380
385
  return;
381
386
  case 'tool.started':
387
+ this.recordTool(event, 'call');
382
388
  this.activeTools.set(event.callId, event.name);
383
389
  this.toolRan = true;
384
390
  if (LIVE.has(this.state.kind)) this.setState({ kind: 'using-tool', tool: event.name });
385
391
  return;
386
392
  case 'tool.finished': {
393
+ this.recordTool(event, 'result');
387
394
  this.activeTools.delete(event.callId);
388
395
  if (this.state.kind !== 'using-tool') return;
389
396
  const still = [...this.activeTools.values()].at(-1);
@@ -392,9 +399,6 @@ export class VoiceCall {
392
399
  );
393
400
  return;
394
401
  }
395
- case 'approval.requested':
396
- this.options.onApprovalRequested?.(event.toolName);
397
- return;
398
402
  case 'messages.saved':
399
403
  this.options.onMessagesSaved?.(event.conversationId, event.messageIds);
400
404
  return;
@@ -455,41 +459,79 @@ export class VoiceCall {
455
459
  this.finish({ kind: 'ended', reason });
456
460
  }
457
461
 
462
+ /**
463
+ * One message per `messageId`, the id the server stores and announces it
464
+ * under, so the stored copy that reaches the thread is this message, not a
465
+ * second one. Its text is its items' texts in order.
466
+ */
458
467
  private upsertMessage(
459
- role: 'user' | 'assistant',
460
- itemId: string,
461
- content: (previous: string) => string,
468
+ event: { role: 'user' | 'assistant'; messageId: string; itemId: string },
469
+ text: (previous: string) => string,
462
470
  isStreaming: boolean,
463
471
  ): void {
464
- const id = `${role}-${itemId}`;
465
- const index = this.messages.findIndex((message) => message.id === id);
466
- if (index === -1) {
467
- this.messages = [
468
- ...this.messages,
469
- {
470
- id,
471
- role,
472
- ...(role === 'assistant' ? { speaker: 'primary' as const } : {}),
473
- content: content(''),
474
- timestamp: Date.now(),
475
- isStreaming,
476
- },
477
- ];
478
- } else {
479
- const previous = this.messages[index]!;
480
- const next = [...this.messages];
481
- next[index] = { ...previous, content: content(previous.content), isStreaming };
482
- this.messages = next;
483
- }
472
+ const { messageId, itemId } = event;
473
+ const items = this.itemTexts.get(messageId) ?? new Map<string, string>();
474
+ this.itemTexts.set(messageId, items);
475
+ this.messageOfItem.set(itemId, messageId);
476
+ items.set(itemId, text(items.get(itemId) ?? ''));
477
+ const content = [...items.values()].filter((part) => part !== '').join(' ');
478
+ this.writeMessage(messageId, event.role, (message) => ({ ...message, content, isStreaming }));
479
+ }
480
+
481
+ /**
482
+ * A tool the answer ran, inside the answer's message, by name and state: the
483
+ * device never receives its arguments or result, and the stored message that
484
+ * carries them replaces this one once it is saved.
485
+ */
486
+ private recordTool(
487
+ event: { messageId: string; callId: string; name: string },
488
+ state: 'call' | 'result',
489
+ ): void {
490
+ this.writeMessage(event.messageId, 'assistant', (message) => {
491
+ const invocation = { toolCallId: event.callId, toolName: event.name, state };
492
+ const tools = message.toolInvocations ?? [];
493
+ const index = tools.findIndex((tool) => tool.toolCallId === event.callId);
494
+ return {
495
+ ...message,
496
+ toolInvocations:
497
+ index === -1
498
+ ? [...tools, invocation]
499
+ : tools.map((tool, at) => (at === index ? invocation : tool)),
500
+ };
501
+ });
502
+ }
503
+
504
+ private writeMessage(
505
+ messageId: string,
506
+ role: 'user' | 'assistant',
507
+ change: (message: VoiceMessage) => VoiceMessage,
508
+ ): void {
509
+ const index = this.messages.findIndex((message) => message.id === messageId);
510
+ const current: VoiceMessage =
511
+ index === -1
512
+ ? {
513
+ id: messageId,
514
+ role,
515
+ ...(role === 'assistant' ? { speaker: 'primary' as const } : {}),
516
+ content: '',
517
+ timestamp: Date.now(),
518
+ isStreaming: false,
519
+ }
520
+ : this.messages[index]!;
521
+ const next = change(current);
522
+ this.messages =
523
+ index === -1
524
+ ? [...this.messages, next]
525
+ : this.messages.map((message, at) => (at === index ? next : message));
484
526
  this.emit();
485
527
  }
486
528
 
487
529
  private settleStreaming(itemId: string | null): void {
488
- if (itemId === null) return;
489
- const id = `assistant-${itemId}`;
490
- if (!this.messages.some((message) => message.id === id && message.isStreaming)) return;
530
+ const messageId = itemId === null ? undefined : this.messageOfItem.get(itemId);
531
+ if (messageId === undefined) return;
532
+ if (!this.messages.some((message) => message.id === messageId && message.isStreaming)) return;
491
533
  this.messages = this.messages.map((message) =>
492
- message.id === id ? { ...message, isStreaming: false } : message,
534
+ message.id === messageId ? { ...message, isStreaming: false } : message,
493
535
  );
494
536
  this.emit();
495
537
  }
@@ -0,0 +1,31 @@
1
+ // The call bar's words, apart from the component so the chat's labels can hold them without
2
+ // reaching React Native.
3
+
4
+ /** Every word the bar shows or announces. An app in another language passes its own. */
5
+ export interface VoiceControlsLabels {
6
+ connecting: string;
7
+ reconnecting: string;
8
+ connected: string;
9
+ listening: string;
10
+ muted: string;
11
+ thinking: string;
12
+ usingTool: string;
13
+ speaking: string;
14
+ mute: string;
15
+ unmute: string;
16
+ end: string;
17
+ }
18
+
19
+ export const VOICE_CONTROLS_LABELS: VoiceControlsLabels = {
20
+ connecting: 'Connecting...',
21
+ reconnecting: 'Reconnecting...',
22
+ connected: 'Connected',
23
+ listening: 'Listening...',
24
+ muted: 'Muted',
25
+ thinking: 'Thinking...',
26
+ usingTool: 'Working on it...',
27
+ speaking: 'Speaking...',
28
+ mute: 'Mute',
29
+ unmute: 'Unmute',
30
+ end: 'End',
31
+ };
@@ -13,7 +13,11 @@
13
13
  * `session.accepted` carries `seq`; a resume replays from `afterSeq`.
14
14
  */
15
15
 
16
- export const VOICE_PROTOCOL_VERSION = 1;
16
+ /**
17
+ * 2: every transcript and tool event names `messageId`, the message it belongs
18
+ * to as it is stored.
19
+ */
20
+ export const VOICE_PROTOCOL_VERSION = 2;
17
21
 
18
22
  export const VOICE_SESSION_PATH = '/v1/voice/session';
19
23
 
@@ -107,17 +111,20 @@ export type VoiceServerEvent =
107
111
  | { type: 'input.ready' }
108
112
  | { type: 'user.speech_started'; itemId: string }
109
113
  | { type: 'user.speech_stopped'; itemId: string }
110
- | { type: 'transcript.delta'; role: VoiceRole; itemId: string; text: string }
111
- | { type: 'transcript.done'; role: VoiceRole; itemId: string; text: string }
114
+ /**
115
+ * `messageId` is the message the text belongs to, the id it is stored and
116
+ * announced under: an assistant turn's items are one message.
117
+ */
118
+ | { type: 'transcript.delta'; role: VoiceRole; messageId: string; itemId: string; text: string }
119
+ | { type: 'transcript.done'; role: VoiceRole; messageId: string; itemId: string; text: string }
112
120
  | { type: 'assistant.audio_done'; itemId: string }
113
121
  | {
114
122
  type: 'assistant.done';
115
123
  itemId: string | null;
116
124
  status: 'completed' | 'cancelled' | 'incomplete' | 'failed';
117
125
  }
118
- | { type: 'tool.started'; callId: string; name: string }
119
- | { type: 'tool.finished'; callId: string; name: string; ok: boolean }
120
- | { type: 'approval.requested'; toolName: string }
126
+ | { type: 'tool.started'; messageId: string; callId: string; name: string }
127
+ | { type: 'tool.finished'; messageId: string; callId: string; name: string; ok: boolean }
121
128
  | { type: 'messages.saved'; conversationId: string; messageIds: string[] }
122
129
  | { type: 'session.warning'; code: 'ending-soon'; remainingMs: number }
123
130
  | { type: 'session.reconnecting' }
@@ -196,9 +203,17 @@ export function parseVoiceServerEvent(raw: string): ReceivedVoiceEvent | null {
196
203
  case 'transcript.delta':
197
204
  case 'transcript.done':
198
205
  return (event.role === 'user' || event.role === 'assistant') &&
206
+ isString(event.messageId) &&
199
207
  isString(event.itemId) &&
200
208
  isString(event.text)
201
- ? { type: event.type, role: event.role, itemId: event.itemId, text: event.text, ...seq }
209
+ ? {
210
+ type: event.type,
211
+ role: event.role,
212
+ messageId: event.messageId,
213
+ itemId: event.itemId,
214
+ text: event.text,
215
+ ...seq,
216
+ }
202
217
  : null;
203
218
  case 'assistant.done':
204
219
  return event.itemId === null || isString(event.itemId)
@@ -215,23 +230,26 @@ export function parseVoiceServerEvent(raw: string): ReceivedVoiceEvent | null {
215
230
  }
216
231
  : null;
217
232
  case 'tool.started':
218
- return isString(event.callId) && isString(event.name)
219
- ? { type: 'tool.started', callId: event.callId, name: event.name, ...seq }
233
+ return isString(event.messageId) && isString(event.callId) && isString(event.name)
234
+ ? {
235
+ type: 'tool.started',
236
+ messageId: event.messageId,
237
+ callId: event.callId,
238
+ name: event.name,
239
+ ...seq,
240
+ }
220
241
  : null;
221
242
  case 'tool.finished':
222
- return isString(event.callId) && isString(event.name)
243
+ return isString(event.messageId) && isString(event.callId) && isString(event.name)
223
244
  ? {
224
245
  type: 'tool.finished',
246
+ messageId: event.messageId,
225
247
  callId: event.callId,
226
248
  name: event.name,
227
249
  ok: event.ok === true,
228
250
  ...seq,
229
251
  }
230
252
  : null;
231
- case 'approval.requested':
232
- return isString(event.toolName)
233
- ? { type: 'approval.requested', toolName: event.toolName, ...seq }
234
- : null;
235
253
  case 'messages.saved':
236
254
  return isString(event.conversationId) && Array.isArray(event.messageIds)
237
255
  ? {