@alia.onl/sdk 15.8.0 → 19.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +242 -13
- package/README.md +223 -151
- package/package.json +15 -5
- package/src/animations.ts +0 -1
- package/src/components/AliaAskProvider.tsx +127 -0
- package/src/components/AliaChatContent.tsx +604 -392
- package/src/components/AliaChatHeader.tsx +14 -38
- package/src/components/AliaChatScreen.tsx +56 -31
- package/src/components/AliaChatSheet.tsx +110 -36
- package/src/components/AliaComposer.tsx +1 -1
- package/src/components/AliaParticipants.tsx +202 -0
- package/src/components/AliaParticipantsHeader.tsx +60 -0
- package/src/components/AliaSharedConversation.tsx +86 -0
- package/src/components/AliaWelcomeMessage.tsx +18 -5
- package/src/components/Image.tsx +12 -0
- package/src/components/Markdown.tsx +498 -14
- package/src/components/PlanPreviewCard.tsx +7 -49
- package/src/components/ReplyMarkdown.tsx +359 -0
- package/src/components/RichBlocks.tsx +195 -0
- package/src/components/canvas/CanvasComponent.tsx +59 -0
- package/src/components/canvas/ChartRenderer.tsx +73 -0
- package/src/components/canvas/CodeRenderer.tsx +24 -0
- package/src/components/canvas/FormRenderer.tsx +101 -0
- package/src/components/canvas/MarkdownRenderer.tsx +14 -0
- package/src/components/canvas/TableRenderer.tsx +47 -0
- package/src/components/thought/FilesAndSources.tsx +120 -0
- package/src/components/thought/ThoughtPanel.tsx +417 -0
- package/src/components/thought/ToolStep.tsx +141 -0
- package/src/components/thread/AgentResultCard.tsx +125 -0
- package/src/components/thread/AgentTaskCard.tsx +127 -0
- package/src/components/thread/AliaThread.tsx +401 -0
- package/src/components/thread/MessageBlockBoundary.tsx +78 -0
- package/src/components/thread/NewConversationOffer.tsx +54 -0
- package/src/components/thread/ThreadRow.tsx +458 -0
- package/src/components/thread/ThreadSearch.tsx +129 -0
- package/src/components/thread/ToolResultCard.tsx +254 -0
- package/src/components/thread/TurnMenu.tsx +54 -0
- package/src/components/thread/TurnStatusLine.tsx +34 -0
- package/src/components/voice/AliaVoiceCall.tsx +21 -0
- package/src/components/voice/VoiceControls.tsx +2 -28
- package/src/components/voice/VoiceSession.tsx +39 -11
- package/src/hooks/useAliaActivity.ts +2 -2
- package/src/hooks/useAtBottom.ts +51 -0
- package/src/hooks/useAttachMenu.ts +94 -0
- package/src/hooks/useChatComposer.ts +299 -0
- package/src/hooks/useChatVoice.ts +129 -0
- package/src/hooks/useDocumentPicker.ts +3 -29
- package/src/hooks/useFailedTurnToast.ts +43 -0
- package/src/hooks/useMessageVote.ts +50 -0
- package/src/hooks/useOpenThought.ts +79 -0
- package/src/hooks/usePendingQuestions.ts +2 -2
- package/src/hooks/useReadAloud.ts +92 -0
- package/src/hooks/useShareHandler.ts +24 -0
- package/src/hooks/useSoundEffects.ts +1 -1
- package/src/hooks/useThoughtScope.ts +59 -0
- package/src/hooks/useThreadRows.ts +145 -0
- package/src/hooks/useThreadView.ts +68 -0
- package/src/hooks/useTimelineWindow.ts +256 -0
- package/src/hooks/useVoiceLauncher.ts +180 -0
- package/src/hooks/useVoiceRoom.ts +0 -3
- package/src/index.ts +43 -29
- package/src/lib/activity-store.ts +1 -1
- package/src/lib/add-menu.ts +7 -0
- package/src/lib/call-thread.ts +33 -0
- package/src/lib/citations.ts +79 -2
- package/src/lib/clipboard.ts +23 -0
- package/src/lib/content-utils.ts +10 -0
- package/src/lib/copy-en.ts +284 -0
- package/src/lib/copy.ts +32 -0
- package/src/lib/document-picker.ts +31 -0
- package/src/lib/flights.ts +22 -13
- package/src/lib/focus-return.ts +66 -0
- package/src/lib/keyboard.native.tsx +19 -0
- package/src/lib/keyboard.tsx +9 -0
- package/src/lib/labels.tsx +69 -17
- package/src/lib/participants.ts +86 -0
- package/src/lib/power-level-options.ts +66 -0
- package/src/lib/questions.ts +2 -2
- package/src/lib/task-labels.ts +4 -4
- package/src/lib/tool-call-status.ts +1 -1
- package/src/lib/tool-icons.ts +36 -0
- package/src/lib/tool-renderers.tsx +48 -0
- package/src/lib/turn-state.ts +2 -2
- package/src/lib/voice-call-request.ts +49 -0
- package/src/lib/voice-call.ts +76 -34
- package/src/lib/voice-controls-labels.ts +31 -0
- package/src/lib/voice-protocol.ts +32 -14
- package/src/lib/work-log.ts +6 -6
- package/src/runtime/agents.ts +151 -0
- package/src/runtime/api.ts +123 -0
- package/src/runtime/attachment-utils.ts +96 -0
- package/src/runtime/composer-draft-store.ts +270 -0
- package/src/runtime/config.tsx +186 -0
- package/src/runtime/conversation-store.ts +49 -0
- package/src/runtime/conversations.ts +557 -0
- package/src/runtime/error-utils.ts +104 -0
- package/src/runtime/host-tools.ts +140 -0
- package/src/runtime/index.ts +113 -0
- package/src/runtime/message-history.ts +124 -0
- package/src/runtime/model.ts +39 -0
- package/src/runtime/oxy-files.ts +37 -0
- package/src/runtime/power-levels.ts +66 -0
- package/src/runtime/query-keys.ts +34 -0
- package/src/runtime/random-uuid.ts +25 -0
- package/src/runtime/routes.ts +51 -0
- package/src/runtime/sharing.ts +165 -0
- package/src/runtime/sse-frame-reader.ts +88 -0
- package/src/runtime/stream-events.ts +146 -0
- package/src/runtime/stream-outcome.ts +21 -0
- package/src/runtime/thought-store.ts +103 -0
- package/src/runtime/thread/agent-activity.ts +261 -0
- package/src/runtime/thread/automation-labels.ts +79 -0
- package/src/runtime/thread/conversation-export.ts +135 -0
- package/src/runtime/thread/message-days.ts +120 -0
- package/src/runtime/thread/message-processor.ts +93 -0
- package/src/runtime/thread/task-utils.ts +11 -0
- package/src/runtime/thread/thought-utils.ts +752 -0
- package/src/runtime/thread/thread-history.ts +139 -0
- package/src/runtime/thread/timeline.ts +202 -0
- package/src/runtime/thread/title-tags.ts +8 -0
- package/src/runtime/thread/tool-cards.ts +383 -0
- package/src/runtime/turn-failure.ts +108 -0
- package/src/runtime/types.ts +148 -0
- package/src/runtime/usage-limit-error.ts +44 -0
- package/src/runtime/use-agent-activity.ts +80 -0
- package/src/runtime/use-conversation-stream.ts +1427 -0
- package/src/runtime/use-conversation-thread.ts +556 -0
- package/src/runtime/use-start-conversation.ts +107 -0
- package/src/runtime/use-thread-history.ts +64 -0
- package/src/runtime/use-thread-search.ts +95 -0
- package/src/runtime/use-turn-edit.ts +99 -0
- package/src/tools.ts +2 -2
- package/src/types.ts +18 -47
- package/src/components/AliaChatMessageList.tsx +0 -532
- package/src/components/ThinkingBlock.tsx +0 -43
- package/src/hooks/useAliaChat.ts +0 -573
- package/src/lib/catalogue.ts +0 -236
- package/src/lib/chat-stream.ts +0 -655
- package/src/lib/chat-transport.ts +0 -98
- package/src/lib/delegation.ts +0 -34
- package/src/lib/work-log-copy.ts +0 -61
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { AvatarCharacter } from '@oxy.so/bloom/agent-avatar';
|
|
2
|
+
import type { AliaAgentProfile } from '../runtime/agents';
|
|
3
|
+
import type { Message } from '../runtime/types';
|
|
4
|
+
import { readAgentCharacter } from './agent-character';
|
|
5
|
+
import { ALIA_AVATAR } from './alia-character';
|
|
6
|
+
|
|
7
|
+
/** Somebody in the conversation who answers: its lead, or an agent the lead consulted. */
|
|
8
|
+
export interface AliaParticipant {
|
|
9
|
+
id: string;
|
|
10
|
+
name: string;
|
|
11
|
+
handle?: string;
|
|
12
|
+
/** The 3D character it is drawn as. */
|
|
13
|
+
character: AvatarCharacter;
|
|
14
|
+
color?: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** The conversation's lead: who greets, whose name is on the composer, who the header shows first. */
|
|
18
|
+
export interface AliaLead extends AliaParticipant {
|
|
19
|
+
/** Alia herself, rather than an agent the chat was opened on. */
|
|
20
|
+
isAlia: boolean;
|
|
21
|
+
/** The agent's own greeting; Alia greets with the chat's labels. */
|
|
22
|
+
greeting?: string;
|
|
23
|
+
/** The agent's own starters, offered over the composer of an empty chat. */
|
|
24
|
+
starters: Array<{ title: string; prompt: string }>;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Who is in the conversation, as the header draws it: the lead, then whoever has joined. */
|
|
28
|
+
export interface AliaChatParticipants {
|
|
29
|
+
lead: AliaLead;
|
|
30
|
+
/** The consulted agents that have joined the header, in the order they joined. */
|
|
31
|
+
joined: AliaParticipant[];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Alia as a lead. */
|
|
35
|
+
export const ALIA_LEAD: AliaLead = {
|
|
36
|
+
id: 'alia',
|
|
37
|
+
name: 'Alia',
|
|
38
|
+
handle: 'alia',
|
|
39
|
+
character: ALIA_AVATAR,
|
|
40
|
+
isAlia: true,
|
|
41
|
+
starters: [],
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/** The lead of a chat: the agent it was opened on (`agent={{ id }}`), or Alia. */
|
|
45
|
+
export function leadOf(profile: AliaAgentProfile | null | undefined): AliaLead {
|
|
46
|
+
if (!profile) return ALIA_LEAD;
|
|
47
|
+
return {
|
|
48
|
+
id: profile.id,
|
|
49
|
+
name: profile.name,
|
|
50
|
+
handle: profile.handle,
|
|
51
|
+
character: readAgentCharacter(profile.character),
|
|
52
|
+
color: profile.color,
|
|
53
|
+
isAlia: false,
|
|
54
|
+
greeting: profile.greeting,
|
|
55
|
+
starters: profile.starters,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** A consulted agent as a message carries it (`agentInfo`). */
|
|
60
|
+
export function participantOf(agentInfo: NonNullable<Message['agentInfo']>): AliaParticipant {
|
|
61
|
+
return {
|
|
62
|
+
id: agentInfo.id,
|
|
63
|
+
name: agentInfo.name,
|
|
64
|
+
handle: agentInfo.handle,
|
|
65
|
+
character: readAgentCharacter(agentInfo.character),
|
|
66
|
+
color: agentInfo.color,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The agents that answered in `messages`, besides the lead, in the order they first did — each
|
|
72
|
+
* with whether its first answer is the one being streamed now (it is joining) or one already
|
|
73
|
+
* written (it was there when the chat opened).
|
|
74
|
+
*/
|
|
75
|
+
export function consultedAgents(
|
|
76
|
+
messages: readonly Message[],
|
|
77
|
+
leadId: string,
|
|
78
|
+
): Array<{ participant: AliaParticipant; live: boolean }> {
|
|
79
|
+
const seen = new Map<string, { participant: AliaParticipant; live: boolean }>();
|
|
80
|
+
for (const message of messages) {
|
|
81
|
+
const info = message.agentInfo;
|
|
82
|
+
if (!info || info.id === leadId || seen.has(info.id)) continue;
|
|
83
|
+
seen.set(info.id, { participant: participantOf(info), live: message.isStreaming === true });
|
|
84
|
+
}
|
|
85
|
+
return [...seen.values()];
|
|
86
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import type { ComposerPanelPermissionOption } from '@oxy.so/bloom/composer-panel';
|
|
2
|
+
import { RiComputerLine } from '@oxy.so/bloom/icons/RiComputerLine';
|
|
3
|
+
import { RiFireLine } from '@oxy.so/bloom/icons/RiFireLine';
|
|
4
|
+
import { RiFlashlightLine } from '@oxy.so/bloom/icons/RiFlashlightLine';
|
|
5
|
+
import { RiLightbulbLine } from '@oxy.so/bloom/icons/RiLightbulbLine';
|
|
6
|
+
import { RiRocket2Line } from '@oxy.so/bloom/icons/RiRocket2Line';
|
|
7
|
+
import { RiSparkling2Line } from '@oxy.so/bloom/icons/RiSparkling2Line';
|
|
8
|
+
import { RiSpeedUpLine } from '@oxy.so/bloom/icons/RiSpeedUpLine';
|
|
9
|
+
import { RiVipCrownLine } from '@oxy.so/bloom/icons/RiVipCrownLine';
|
|
10
|
+
import { POWER_LEVELS, type PowerLevel } from '../runtime/power-levels';
|
|
11
|
+
import type { Translate } from './copy';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The power-level selector, in the shape Bloom's composer mode selector takes.
|
|
15
|
+
*
|
|
16
|
+
* Alia's users choose how much power a turn gets, never which hosted model answers it: Oxy picks
|
|
17
|
+
* the model of the chosen level for every request. The selector is `ComposerPill`'s mode picker —
|
|
18
|
+
* Bloom's own radio menu of a label, a short description and an icon per mode — so the seven
|
|
19
|
+
* levels read the same way every Bloom mode selector does. Models running on the person's OWN
|
|
20
|
+
* devices follow the levels: that is their machine, not a model Alia offers, and which devices
|
|
21
|
+
* are connected is the host's to say.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
type IconComponent = ComposerPanelPermissionOption['icon'];
|
|
25
|
+
|
|
26
|
+
/** One glyph per level, rising with the power it stands for. */
|
|
27
|
+
export const POWER_LEVEL_ICONS: Record<PowerLevel, IconComponent> = {
|
|
28
|
+
auto: RiSparkling2Line,
|
|
29
|
+
instant: RiFlashlightLine,
|
|
30
|
+
medium: RiSpeedUpLine,
|
|
31
|
+
high: RiLightbulbLine,
|
|
32
|
+
xhigh: RiFireLine,
|
|
33
|
+
pro: RiRocket2Line,
|
|
34
|
+
ultra: RiVipCrownLine,
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
/** A model running on one of the person's own devices, offered after the levels. */
|
|
38
|
+
export interface DeviceModel {
|
|
39
|
+
/** What a request carries as `model` (`local/<runtime>/<model>`). */
|
|
40
|
+
id: string;
|
|
41
|
+
/** The tag the person knows it by, e.g. `llama3.1:8b`. */
|
|
42
|
+
name: string;
|
|
43
|
+
/** Which of their devices is running it. */
|
|
44
|
+
deviceLabel: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The selector's options: the seven levels, then the person's device models. */
|
|
48
|
+
export function buildPowerLevelOptions(
|
|
49
|
+
t: Translate,
|
|
50
|
+
deviceModels: readonly DeviceModel[],
|
|
51
|
+
): ComposerPanelPermissionOption[] {
|
|
52
|
+
return [
|
|
53
|
+
...POWER_LEVELS.map((level) => ({
|
|
54
|
+
id: level,
|
|
55
|
+
label: t(`powerLevels.${level}.label`),
|
|
56
|
+
description: t(`powerLevels.${level}.description`),
|
|
57
|
+
icon: POWER_LEVEL_ICONS[level],
|
|
58
|
+
})),
|
|
59
|
+
...deviceModels.map((model) => ({
|
|
60
|
+
id: model.id,
|
|
61
|
+
label: model.name,
|
|
62
|
+
description: t('powerLevels.onDevice', { device: model.deviceLabel }),
|
|
63
|
+
icon: RiComputerLine,
|
|
64
|
+
})),
|
|
65
|
+
];
|
|
66
|
+
}
|
package/src/lib/questions.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// call: it is stored with the message, survives a reload, and is answered by the person's
|
|
5
5
|
// next message. Pure functions, no React Native, so a host's models and tests can use them.
|
|
6
6
|
|
|
7
|
-
import type {
|
|
7
|
+
import type { Message, ToolInvocation } from '../runtime/types';
|
|
8
8
|
|
|
9
9
|
/** The tool's name on the wire. */
|
|
10
10
|
export const ASK_USER_QUESTION_TOOL = 'askUserQuestion';
|
|
@@ -83,7 +83,7 @@ export function formatAnswers(questions: AskedQuestion[], answers: QuestionAnswe
|
|
|
83
83
|
* is the answer, so nothing is pending then.
|
|
84
84
|
*/
|
|
85
85
|
export function pendingQuestions(
|
|
86
|
-
messages: readonly Pick<
|
|
86
|
+
messages: readonly Pick<Message, 'role' | 'toolInvocations'>[],
|
|
87
87
|
isStreaming: boolean,
|
|
88
88
|
): AskedQuestion[] {
|
|
89
89
|
const last = messages.at(-1);
|
package/src/lib/task-labels.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { englishCopy, type Translate } from './copy';
|
|
2
2
|
|
|
3
3
|
/** Whether the catalogue names this tool itself; otherwise the label is made from its id. */
|
|
4
4
|
function isNamed(kind: 'pill' | 'done', toolName: string): boolean {
|
|
5
5
|
const key = `tasks.tool.${kind}.${toolName}`;
|
|
6
|
-
return
|
|
6
|
+
return englishCopy(key) !== key;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
/** A tool's short label, in the reader's language when a translator is given. */
|
|
10
|
-
export function getToolPillLabel(toolName: string, t: Translate =
|
|
10
|
+
export function getToolPillLabel(toolName: string, t: Translate = englishCopy): string {
|
|
11
11
|
if (isNamed('pill', toolName)) return t(`tasks.tool.pill.${toolName}`);
|
|
12
12
|
|
|
13
13
|
// Oxy service tools: oxy_serviceName__toolName → "ServiceName"
|
|
@@ -28,7 +28,7 @@ export function getToolPillLabel(toolName: string, t: Translate = englishWorkLog
|
|
|
28
28
|
* `runningTitle` for this once the task's steps land). Labels that are already names
|
|
29
29
|
* ("Telegram", "Email") read the same either way, so they fall back to the running label.
|
|
30
30
|
*/
|
|
31
|
-
export function getToolDoneLabel(toolName: string, t: Translate =
|
|
31
|
+
export function getToolDoneLabel(toolName: string, t: Translate = englishCopy): string {
|
|
32
32
|
return isNamed('done', toolName)
|
|
33
33
|
? t(`tasks.tool.done.${toolName}`)
|
|
34
34
|
: getToolPillLabel(toolName, t);
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { BloomIconComponent } from '@oxy.so/bloom/icons';
|
|
2
|
+
import { RiCalendarLine } from '@oxy.so/bloom/icons/RiCalendarLine';
|
|
3
|
+
import { RiChat3Line } from '@oxy.so/bloom/icons/RiChat3Line';
|
|
4
|
+
import { RiDatabase2Line } from '@oxy.so/bloom/icons/RiDatabase2Line';
|
|
5
|
+
import { RiFileTextLine } from '@oxy.so/bloom/icons/RiFileTextLine';
|
|
6
|
+
import { RiGlobalLine } from '@oxy.so/bloom/icons/RiGlobalLine';
|
|
7
|
+
import { RiLightbulbFlashLine } from '@oxy.so/bloom/icons/RiLightbulbFlashLine';
|
|
8
|
+
import { RiLink } from '@oxy.so/bloom/icons/RiLink';
|
|
9
|
+
import { RiSearchLine } from '@oxy.so/bloom/icons/RiSearchLine';
|
|
10
|
+
import { RiSendPlaneLine } from '@oxy.so/bloom/icons/RiSendPlaneLine';
|
|
11
|
+
import { RiSettings3Line } from '@oxy.so/bloom/icons/RiSettings3Line';
|
|
12
|
+
import { RiUserLine } from '@oxy.so/bloom/icons/RiUserLine';
|
|
13
|
+
|
|
14
|
+
/** The glyph a tool's step carries in the thought panel, by tool name. */
|
|
15
|
+
const TOOL_ICONS: ReadonlyMap<string, BloomIconComponent> = new Map([
|
|
16
|
+
['webSearch', RiSearchLine],
|
|
17
|
+
['scrapeURL', RiLink],
|
|
18
|
+
['getTimeline', RiCalendarLine],
|
|
19
|
+
['searchKnowledgeBase', RiDatabase2Line],
|
|
20
|
+
['webScraper', RiGlobalLine],
|
|
21
|
+
['browse', RiGlobalLine],
|
|
22
|
+
['sendWhatsAppMessage', RiChat3Line],
|
|
23
|
+
['getWhatsAppChats', RiChat3Line],
|
|
24
|
+
['getWhatsAppMessages', RiChat3Line],
|
|
25
|
+
['sendTelegramMessage', RiSendPlaneLine],
|
|
26
|
+
['getCurrentDate', RiCalendarLine],
|
|
27
|
+
['generateFile', RiFileTextLine],
|
|
28
|
+
['saveUserMemory', RiLightbulbFlashLine],
|
|
29
|
+
['updateUserPreferences', RiSettings3Line],
|
|
30
|
+
['updateUserContext', RiUserLine],
|
|
31
|
+
]);
|
|
32
|
+
|
|
33
|
+
/** A tool's glyph; a tool with none of its own reads as the web it reached. */
|
|
34
|
+
export function getToolIcon(toolName: string): BloomIconComponent {
|
|
35
|
+
return TOOL_ICONS.get(toolName) ?? RiGlobalLine;
|
|
36
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { createContext, type ReactNode, useContext } from 'react';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A tool's result as a host's renderer receives it (`docs/agent-embedding.md` §3): the app and the
|
|
5
|
+
* tool its canonical name names, what it was called with and what it returned.
|
|
6
|
+
*/
|
|
7
|
+
export interface AliaToolResult {
|
|
8
|
+
/** The app of `oxy_<app>__<tool>` / `self_<app>__<tool>`. */
|
|
9
|
+
app: string;
|
|
10
|
+
tool: string;
|
|
11
|
+
toolCallId: string;
|
|
12
|
+
input?: Record<string, unknown>;
|
|
13
|
+
output: unknown;
|
|
14
|
+
/** The consulted agent that called it; absent for the lead's own calls. */
|
|
15
|
+
agentId?: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The host's own drawings of catalogue tools' results, keyed `<app>:<tool>` — `homiio:searchListings`
|
|
20
|
+
* for `oxy_homiio__searchListings`. Drawn before the generic tool card, for the lead's calls and a
|
|
21
|
+
* consulted agent's alike.
|
|
22
|
+
*/
|
|
23
|
+
export type AliaToolRenderers = Readonly<Record<string, (result: AliaToolResult) => ReactNode>>;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* `oxy_<app>__<tool>` or `self_<app>__<tool>`: the only names a renderer is keyed from. The app
|
|
27
|
+
* part is an Oxy app slug; the tool part is whatever follows the separator.
|
|
28
|
+
*/
|
|
29
|
+
const CANONICAL_TOOL = /^(?:oxy|self)_([a-z][a-z0-9-]{0,63})__([A-Za-z0-9_-]+)$/;
|
|
30
|
+
|
|
31
|
+
/** The app and tool of a canonical catalogue tool name, or `null` for any other name. */
|
|
32
|
+
export function parseCatalogueTool(toolName: string): { app: string; tool: string } | null {
|
|
33
|
+
const match = CANONICAL_TOOL.exec(toolName);
|
|
34
|
+
return match ? { app: match[1], tool: match[2] } : null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** The renderer key of a tool name (`<app>:<tool>`), or `null` when it is not a catalogue tool. */
|
|
38
|
+
export function toolRendererKey(toolName: string): string | null {
|
|
39
|
+
const parsed = parseCatalogueTool(toolName);
|
|
40
|
+
return parsed ? `${parsed.app}:${parsed.tool}` : null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const RenderersContext = createContext<AliaToolRenderers>({});
|
|
44
|
+
|
|
45
|
+
export const AliaToolRenderersProvider = RenderersContext.Provider;
|
|
46
|
+
|
|
47
|
+
/** The host's renderers, `{}` when it brought none. */
|
|
48
|
+
export const useToolRenderers = () => useContext(RenderersContext);
|
package/src/lib/turn-state.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Message } from '../runtime/types';
|
|
2
2
|
import type { AliaAvatarState } from '../components/AliaAvatar';
|
|
3
3
|
|
|
4
4
|
/** What a chat turn is doing, from its messages: the one reading the header mark and every button share. */
|
|
5
5
|
export function turnActivityState(
|
|
6
|
-
messages: ReadonlyArray<
|
|
6
|
+
messages: ReadonlyArray<Message>,
|
|
7
7
|
isStreaming: boolean,
|
|
8
8
|
): AliaAvatarState {
|
|
9
9
|
if (!isStreaming) return 'idle';
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { create } from 'zustand';
|
|
2
|
+
|
|
3
|
+
/** A call somebody asked for, waiting for the screen that shows its conversation. */
|
|
4
|
+
export interface VoiceCallRequest {
|
|
5
|
+
readonly conversationId: string;
|
|
6
|
+
readonly accountId: string | null;
|
|
7
|
+
readonly at: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/** How long a request waits for its screen. A navigation that never arrives must not open a call later. */
|
|
11
|
+
export const CALL_REQUEST_TTL_MS = 15_000;
|
|
12
|
+
|
|
13
|
+
interface VoiceCallRequestStore {
|
|
14
|
+
request: VoiceCallRequest | null;
|
|
15
|
+
ask: (conversationId: string, accountId: string | null) => void;
|
|
16
|
+
clear: () => void;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Where a call is asked for, apart from where it opens.
|
|
21
|
+
*
|
|
22
|
+
* Starting a call and showing it are the same thing when the conversation is already on screen,
|
|
23
|
+
* and two screens apart when it is not: a new-chat route makes the conversation and the host
|
|
24
|
+
* navigates to it. Both go the same way. The launcher AND the screen of the conversation meet
|
|
25
|
+
* here — it asks, and whichever launcher is mounted for that conversation (that account, in time)
|
|
26
|
+
* takes the request and opens the call. The sheet, the screen and the app share this one path.
|
|
27
|
+
*/
|
|
28
|
+
export const useVoiceCallRequests = create<VoiceCallRequestStore>((set) => ({
|
|
29
|
+
request: null,
|
|
30
|
+
ask: (conversationId, accountId) =>
|
|
31
|
+
set({ request: { conversationId, accountId, at: Date.now() } }),
|
|
32
|
+
clear: () => set({ request: null }),
|
|
33
|
+
}));
|
|
34
|
+
|
|
35
|
+
/** Whether `request` is for this conversation and account, and still worth opening. */
|
|
36
|
+
export function isRequestFor(
|
|
37
|
+
request: VoiceCallRequest | null,
|
|
38
|
+
conversationId: string | undefined,
|
|
39
|
+
accountId: string | null,
|
|
40
|
+
now = Date.now(),
|
|
41
|
+
): boolean {
|
|
42
|
+
return (
|
|
43
|
+
request !== null &&
|
|
44
|
+
conversationId !== undefined &&
|
|
45
|
+
request.conversationId === conversationId &&
|
|
46
|
+
request.accountId === accountId &&
|
|
47
|
+
now - request.at <= CALL_REQUEST_TTL_MS
|
|
48
|
+
);
|
|
49
|
+
}
|
package/src/lib/voice-call.ts
CHANGED
|
@@ -88,8 +88,6 @@ export interface VoiceCallOptions {
|
|
|
88
88
|
readonly onChange: (snapshot: VoiceCallSnapshot) => void;
|
|
89
89
|
/** Alia stored messages of the call in the conversation. */
|
|
90
90
|
readonly onMessagesSaved?: (conversationId: string, messageIds: readonly string[]) => void;
|
|
91
|
-
/** A tool filed an action for the person's OK; the card is in the conversation. */
|
|
92
|
-
readonly onApprovalRequested?: (toolName: string) => void;
|
|
93
91
|
readonly tokenRefreshMs?: number;
|
|
94
92
|
readonly resumeRetryMs?: number;
|
|
95
93
|
}
|
|
@@ -113,6 +111,13 @@ export class VoiceCall {
|
|
|
113
111
|
private readonly options: VoiceCallOptions;
|
|
114
112
|
private state: VoiceCallState = { kind: 'idle' };
|
|
115
113
|
private messages: VoiceMessage[] = [];
|
|
114
|
+
/**
|
|
115
|
+
* Each message's text by the item it was spoken in, in the order the items
|
|
116
|
+
* began: an assistant turn that called a tool is one message of two items.
|
|
117
|
+
*/
|
|
118
|
+
private readonly itemTexts = new Map<string, Map<string, string>>();
|
|
119
|
+
/** The message each item belongs to. */
|
|
120
|
+
private readonly messageOfItem = new Map<string, string>();
|
|
116
121
|
private muted = false;
|
|
117
122
|
private endingSoonMs: number | null = null;
|
|
118
123
|
private model: VoiceCallSnapshot['model'] = null;
|
|
@@ -362,10 +367,10 @@ export class VoiceCall {
|
|
|
362
367
|
this.setState({ kind: 'thinking' });
|
|
363
368
|
return;
|
|
364
369
|
case 'transcript.delta':
|
|
365
|
-
this.upsertMessage(event
|
|
370
|
+
this.upsertMessage(event, (previous) => previous + event.text, true);
|
|
366
371
|
return;
|
|
367
372
|
case 'transcript.done':
|
|
368
|
-
this.upsertMessage(event
|
|
373
|
+
this.upsertMessage(event, () => event.text, false);
|
|
369
374
|
return;
|
|
370
375
|
case 'assistant.audio_done':
|
|
371
376
|
return;
|
|
@@ -379,11 +384,13 @@ export class VoiceCall {
|
|
|
379
384
|
if (this.state.kind === 'thinking' && this.playingItem === null) this.afterSpeech();
|
|
380
385
|
return;
|
|
381
386
|
case 'tool.started':
|
|
387
|
+
this.recordTool(event, 'call');
|
|
382
388
|
this.activeTools.set(event.callId, event.name);
|
|
383
389
|
this.toolRan = true;
|
|
384
390
|
if (LIVE.has(this.state.kind)) this.setState({ kind: 'using-tool', tool: event.name });
|
|
385
391
|
return;
|
|
386
392
|
case 'tool.finished': {
|
|
393
|
+
this.recordTool(event, 'result');
|
|
387
394
|
this.activeTools.delete(event.callId);
|
|
388
395
|
if (this.state.kind !== 'using-tool') return;
|
|
389
396
|
const still = [...this.activeTools.values()].at(-1);
|
|
@@ -392,9 +399,6 @@ export class VoiceCall {
|
|
|
392
399
|
);
|
|
393
400
|
return;
|
|
394
401
|
}
|
|
395
|
-
case 'approval.requested':
|
|
396
|
-
this.options.onApprovalRequested?.(event.toolName);
|
|
397
|
-
return;
|
|
398
402
|
case 'messages.saved':
|
|
399
403
|
this.options.onMessagesSaved?.(event.conversationId, event.messageIds);
|
|
400
404
|
return;
|
|
@@ -455,41 +459,79 @@ export class VoiceCall {
|
|
|
455
459
|
this.finish({ kind: 'ended', reason });
|
|
456
460
|
}
|
|
457
461
|
|
|
462
|
+
/**
|
|
463
|
+
* One message per `messageId`, the id the server stores and announces it
|
|
464
|
+
* under, so the stored copy that reaches the thread is this message, not a
|
|
465
|
+
* second one. Its text is its items' texts in order.
|
|
466
|
+
*/
|
|
458
467
|
private upsertMessage(
|
|
459
|
-
role: 'user' | 'assistant',
|
|
460
|
-
|
|
461
|
-
content: (previous: string) => string,
|
|
468
|
+
event: { role: 'user' | 'assistant'; messageId: string; itemId: string },
|
|
469
|
+
text: (previous: string) => string,
|
|
462
470
|
isStreaming: boolean,
|
|
463
471
|
): void {
|
|
464
|
-
const
|
|
465
|
-
const
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
472
|
+
const { messageId, itemId } = event;
|
|
473
|
+
const items = this.itemTexts.get(messageId) ?? new Map<string, string>();
|
|
474
|
+
this.itemTexts.set(messageId, items);
|
|
475
|
+
this.messageOfItem.set(itemId, messageId);
|
|
476
|
+
items.set(itemId, text(items.get(itemId) ?? ''));
|
|
477
|
+
const content = [...items.values()].filter((part) => part !== '').join(' ');
|
|
478
|
+
this.writeMessage(messageId, event.role, (message) => ({ ...message, content, isStreaming }));
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* A tool the answer ran, inside the answer's message, by name and state: the
|
|
483
|
+
* device never receives its arguments or result, and the stored message that
|
|
484
|
+
* carries them replaces this one once it is saved.
|
|
485
|
+
*/
|
|
486
|
+
private recordTool(
|
|
487
|
+
event: { messageId: string; callId: string; name: string },
|
|
488
|
+
state: 'call' | 'result',
|
|
489
|
+
): void {
|
|
490
|
+
this.writeMessage(event.messageId, 'assistant', (message) => {
|
|
491
|
+
const invocation = { toolCallId: event.callId, toolName: event.name, state };
|
|
492
|
+
const tools = message.toolInvocations ?? [];
|
|
493
|
+
const index = tools.findIndex((tool) => tool.toolCallId === event.callId);
|
|
494
|
+
return {
|
|
495
|
+
...message,
|
|
496
|
+
toolInvocations:
|
|
497
|
+
index === -1
|
|
498
|
+
? [...tools, invocation]
|
|
499
|
+
: tools.map((tool, at) => (at === index ? invocation : tool)),
|
|
500
|
+
};
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
private writeMessage(
|
|
505
|
+
messageId: string,
|
|
506
|
+
role: 'user' | 'assistant',
|
|
507
|
+
change: (message: VoiceMessage) => VoiceMessage,
|
|
508
|
+
): void {
|
|
509
|
+
const index = this.messages.findIndex((message) => message.id === messageId);
|
|
510
|
+
const current: VoiceMessage =
|
|
511
|
+
index === -1
|
|
512
|
+
? {
|
|
513
|
+
id: messageId,
|
|
514
|
+
role,
|
|
515
|
+
...(role === 'assistant' ? { speaker: 'primary' as const } : {}),
|
|
516
|
+
content: '',
|
|
517
|
+
timestamp: Date.now(),
|
|
518
|
+
isStreaming: false,
|
|
519
|
+
}
|
|
520
|
+
: this.messages[index]!;
|
|
521
|
+
const next = change(current);
|
|
522
|
+
this.messages =
|
|
523
|
+
index === -1
|
|
524
|
+
? [...this.messages, next]
|
|
525
|
+
: this.messages.map((message, at) => (at === index ? next : message));
|
|
484
526
|
this.emit();
|
|
485
527
|
}
|
|
486
528
|
|
|
487
529
|
private settleStreaming(itemId: string | null): void {
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
if (!this.messages.some((message) => message.id ===
|
|
530
|
+
const messageId = itemId === null ? undefined : this.messageOfItem.get(itemId);
|
|
531
|
+
if (messageId === undefined) return;
|
|
532
|
+
if (!this.messages.some((message) => message.id === messageId && message.isStreaming)) return;
|
|
491
533
|
this.messages = this.messages.map((message) =>
|
|
492
|
-
message.id ===
|
|
534
|
+
message.id === messageId ? { ...message, isStreaming: false } : message,
|
|
493
535
|
);
|
|
494
536
|
this.emit();
|
|
495
537
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// The call bar's words, apart from the component so the chat's labels can hold them without
|
|
2
|
+
// reaching React Native.
|
|
3
|
+
|
|
4
|
+
/** Every word the bar shows or announces. An app in another language passes its own. */
|
|
5
|
+
export interface VoiceControlsLabels {
|
|
6
|
+
connecting: string;
|
|
7
|
+
reconnecting: string;
|
|
8
|
+
connected: string;
|
|
9
|
+
listening: string;
|
|
10
|
+
muted: string;
|
|
11
|
+
thinking: string;
|
|
12
|
+
usingTool: string;
|
|
13
|
+
speaking: string;
|
|
14
|
+
mute: string;
|
|
15
|
+
unmute: string;
|
|
16
|
+
end: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export const VOICE_CONTROLS_LABELS: VoiceControlsLabels = {
|
|
20
|
+
connecting: 'Connecting...',
|
|
21
|
+
reconnecting: 'Reconnecting...',
|
|
22
|
+
connected: 'Connected',
|
|
23
|
+
listening: 'Listening...',
|
|
24
|
+
muted: 'Muted',
|
|
25
|
+
thinking: 'Thinking...',
|
|
26
|
+
usingTool: 'Working on it...',
|
|
27
|
+
speaking: 'Speaking...',
|
|
28
|
+
mute: 'Mute',
|
|
29
|
+
unmute: 'Unmute',
|
|
30
|
+
end: 'End',
|
|
31
|
+
};
|
|
@@ -13,7 +13,11 @@
|
|
|
13
13
|
* `session.accepted` carries `seq`; a resume replays from `afterSeq`.
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
/**
|
|
17
|
+
* 2: every transcript and tool event names `messageId`, the message it belongs
|
|
18
|
+
* to as it is stored.
|
|
19
|
+
*/
|
|
20
|
+
export const VOICE_PROTOCOL_VERSION = 2;
|
|
17
21
|
|
|
18
22
|
export const VOICE_SESSION_PATH = '/v1/voice/session';
|
|
19
23
|
|
|
@@ -107,17 +111,20 @@ export type VoiceServerEvent =
|
|
|
107
111
|
| { type: 'input.ready' }
|
|
108
112
|
| { type: 'user.speech_started'; itemId: string }
|
|
109
113
|
| { type: 'user.speech_stopped'; itemId: string }
|
|
110
|
-
|
|
111
|
-
|
|
114
|
+
/**
|
|
115
|
+
* `messageId` is the message the text belongs to, the id it is stored and
|
|
116
|
+
* announced under: an assistant turn's items are one message.
|
|
117
|
+
*/
|
|
118
|
+
| { type: 'transcript.delta'; role: VoiceRole; messageId: string; itemId: string; text: string }
|
|
119
|
+
| { type: 'transcript.done'; role: VoiceRole; messageId: string; itemId: string; text: string }
|
|
112
120
|
| { type: 'assistant.audio_done'; itemId: string }
|
|
113
121
|
| {
|
|
114
122
|
type: 'assistant.done';
|
|
115
123
|
itemId: string | null;
|
|
116
124
|
status: 'completed' | 'cancelled' | 'incomplete' | 'failed';
|
|
117
125
|
}
|
|
118
|
-
| { type: 'tool.started'; callId: string; name: string }
|
|
119
|
-
| { type: 'tool.finished'; callId: string; name: string; ok: boolean }
|
|
120
|
-
| { type: 'approval.requested'; toolName: string }
|
|
126
|
+
| { type: 'tool.started'; messageId: string; callId: string; name: string }
|
|
127
|
+
| { type: 'tool.finished'; messageId: string; callId: string; name: string; ok: boolean }
|
|
121
128
|
| { type: 'messages.saved'; conversationId: string; messageIds: string[] }
|
|
122
129
|
| { type: 'session.warning'; code: 'ending-soon'; remainingMs: number }
|
|
123
130
|
| { type: 'session.reconnecting' }
|
|
@@ -196,9 +203,17 @@ export function parseVoiceServerEvent(raw: string): ReceivedVoiceEvent | null {
|
|
|
196
203
|
case 'transcript.delta':
|
|
197
204
|
case 'transcript.done':
|
|
198
205
|
return (event.role === 'user' || event.role === 'assistant') &&
|
|
206
|
+
isString(event.messageId) &&
|
|
199
207
|
isString(event.itemId) &&
|
|
200
208
|
isString(event.text)
|
|
201
|
-
? {
|
|
209
|
+
? {
|
|
210
|
+
type: event.type,
|
|
211
|
+
role: event.role,
|
|
212
|
+
messageId: event.messageId,
|
|
213
|
+
itemId: event.itemId,
|
|
214
|
+
text: event.text,
|
|
215
|
+
...seq,
|
|
216
|
+
}
|
|
202
217
|
: null;
|
|
203
218
|
case 'assistant.done':
|
|
204
219
|
return event.itemId === null || isString(event.itemId)
|
|
@@ -215,23 +230,26 @@ export function parseVoiceServerEvent(raw: string): ReceivedVoiceEvent | null {
|
|
|
215
230
|
}
|
|
216
231
|
: null;
|
|
217
232
|
case 'tool.started':
|
|
218
|
-
return isString(event.callId) && isString(event.name)
|
|
219
|
-
? {
|
|
233
|
+
return isString(event.messageId) && isString(event.callId) && isString(event.name)
|
|
234
|
+
? {
|
|
235
|
+
type: 'tool.started',
|
|
236
|
+
messageId: event.messageId,
|
|
237
|
+
callId: event.callId,
|
|
238
|
+
name: event.name,
|
|
239
|
+
...seq,
|
|
240
|
+
}
|
|
220
241
|
: null;
|
|
221
242
|
case 'tool.finished':
|
|
222
|
-
return isString(event.callId) && isString(event.name)
|
|
243
|
+
return isString(event.messageId) && isString(event.callId) && isString(event.name)
|
|
223
244
|
? {
|
|
224
245
|
type: 'tool.finished',
|
|
246
|
+
messageId: event.messageId,
|
|
225
247
|
callId: event.callId,
|
|
226
248
|
name: event.name,
|
|
227
249
|
ok: event.ok === true,
|
|
228
250
|
...seq,
|
|
229
251
|
}
|
|
230
252
|
: null;
|
|
231
|
-
case 'approval.requested':
|
|
232
|
-
return isString(event.toolName)
|
|
233
|
-
? { type: 'approval.requested', toolName: event.toolName, ...seq }
|
|
234
|
-
: null;
|
|
235
253
|
case 'messages.saved':
|
|
236
254
|
return isString(event.conversationId) && Array.isArray(event.messageIds)
|
|
237
255
|
? {
|