@agent-native/toolkit 0.9.1 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent-native.eject.json +71 -0
- package/dist/chat-history/ChatHistoryRail.d.ts +15 -1
- package/dist/chat-history/ChatHistoryRail.d.ts.map +1 -1
- package/dist/chat-history/ChatHistoryRail.js +26 -13
- package/dist/chat-history/ChatHistoryRail.js.map +1 -1
- package/dist/chat-history/ChatHistoryRail.spec.js +77 -1
- package/dist/chat-history/ChatHistoryRail.spec.js.map +1 -1
- package/dist/chat-history/index.d.ts +2 -1
- package/dist/chat-history/index.d.ts.map +1 -1
- package/dist/chat-history/index.js +2 -1
- package/dist/chat-history/index.js.map +1 -1
- package/dist/chat-history/useChatHistoryRailController.d.ts +26 -0
- package/dist/chat-history/useChatHistoryRailController.d.ts.map +1 -0
- package/dist/chat-history/useChatHistoryRailController.js +36 -0
- package/dist/chat-history/useChatHistoryRailController.js.map +1 -0
- package/dist/chat-history/useChatHistoryRailController.spec.d.ts +2 -0
- package/dist/chat-history/useChatHistoryRailController.spec.d.ts.map +1 -0
- package/dist/chat-history/useChatHistoryRailController.spec.js +101 -0
- package/dist/chat-history/useChatHistoryRailController.spec.js.map +1 -0
- package/dist/chat-history.css +2 -0
- package/dist/composer/ComposerPlusMenu.js +1 -1
- package/dist/composer/ComposerPlusMenu.js.map +1 -1
- package/dist/composer/PromptComposer.d.ts +1 -0
- package/dist/composer/PromptComposer.d.ts.map +1 -1
- package/dist/composer/PromptComposer.js +26 -5
- package/dist/composer/PromptComposer.js.map +1 -1
- package/dist/composer/PromptComposer.spec.js +8 -1
- package/dist/composer/PromptComposer.spec.js.map +1 -1
- package/dist/composer/RealtimeVoiceMode.d.ts.map +1 -1
- package/dist/composer/RealtimeVoiceMode.js +56 -11
- package/dist/composer/RealtimeVoiceMode.js.map +1 -1
- package/dist/composer/RealtimeVoiceMode.spec.js +59 -7
- package/dist/composer/RealtimeVoiceMode.spec.js.map +1 -1
- package/dist/composer/useRealtimeVoiceMode.d.ts +7 -0
- package/dist/composer/useRealtimeVoiceMode.d.ts.map +1 -1
- package/dist/composer/useRealtimeVoiceMode.js +110 -6
- package/dist/composer/useRealtimeVoiceMode.js.map +1 -1
- package/dist/composer/useRealtimeVoiceMode.spec.js +74 -2
- package/dist/composer/useRealtimeVoiceMode.spec.js.map +1 -1
- package/dist/conformance/__fixtures__/css-in-js-adapter.d.ts +7 -0
- package/dist/conformance/__fixtures__/css-in-js-adapter.d.ts.map +1 -0
- package/dist/conformance/__fixtures__/css-in-js-adapter.js +84 -0
- package/dist/conformance/__fixtures__/css-in-js-adapter.js.map +1 -0
- package/dist/conformance/conformance.spec.d.ts +2 -0
- package/dist/conformance/conformance.spec.d.ts.map +1 -0
- package/dist/conformance/conformance.spec.js +87 -0
- package/dist/conformance/conformance.spec.js.map +1 -0
- package/dist/conformance/index.d.ts +4 -0
- package/dist/conformance/index.d.ts.map +1 -0
- package/dist/conformance/index.js +4 -0
- package/dist/conformance/index.js.map +1 -0
- package/dist/conformance/runner.d.ts +12 -0
- package/dist/conformance/runner.d.ts.map +1 -0
- package/dist/conformance/runner.js +515 -0
- package/dist/conformance/runner.js.map +1 -0
- package/dist/conformance/types.d.ts +26 -0
- package/dist/conformance/types.d.ts.map +1 -0
- package/dist/conformance/types.js +13 -0
- package/dist/conformance/types.js.map +1 -0
- package/dist/conformance/version.d.ts +6 -0
- package/dist/conformance/version.d.ts.map +1 -0
- package/dist/conformance/version.js +14 -0
- package/dist/conformance/version.js.map +1 -0
- package/dist/design-system/components.d.ts +64 -0
- package/dist/design-system/components.d.ts.map +1 -0
- package/dist/design-system/components.js +50 -0
- package/dist/design-system/components.js.map +1 -0
- package/dist/design-system/context.d.ts +11 -0
- package/dist/design-system/context.d.ts.map +1 -0
- package/dist/design-system/context.js +17 -0
- package/dist/design-system/context.js.map +1 -0
- package/dist/design-system/default-adapter.d.ts +3 -0
- package/dist/design-system/default-adapter.d.ts.map +1 -0
- package/dist/design-system/default-adapter.js +166 -0
- package/dist/design-system/default-adapter.js.map +1 -0
- package/dist/design-system/default-adapter.spec.d.ts +2 -0
- package/dist/design-system/default-adapter.spec.d.ts.map +1 -0
- package/dist/design-system/default-adapter.spec.js +96 -0
- package/dist/design-system/default-adapter.spec.js.map +1 -0
- package/dist/design-system/definition.d.ts +9 -0
- package/dist/design-system/definition.d.ts.map +1 -0
- package/dist/design-system/definition.js +4 -0
- package/dist/design-system/definition.js.map +1 -0
- package/dist/design-system/design-system.spec.d.ts +2 -0
- package/dist/design-system/design-system.spec.d.ts.map +1 -0
- package/dist/design-system/design-system.spec.js +121 -0
- package/dist/design-system/design-system.spec.js.map +1 -0
- package/dist/design-system/error-boundary.d.ts +18 -0
- package/dist/design-system/error-boundary.d.ts.map +1 -0
- package/dist/design-system/error-boundary.js +16 -0
- package/dist/design-system/error-boundary.js.map +1 -0
- package/dist/design-system/index.d.ts +8 -0
- package/dist/design-system/index.d.ts.map +1 -0
- package/dist/design-system/index.js +8 -0
- package/dist/design-system/index.js.map +1 -0
- package/dist/design-system/theme.d.ts +30 -0
- package/dist/design-system/theme.d.ts.map +1 -0
- package/dist/design-system/theme.js +95 -0
- package/dist/design-system/theme.js.map +1 -0
- package/dist/design-system/theme.spec.d.ts +2 -0
- package/dist/design-system/theme.spec.d.ts.map +1 -0
- package/dist/design-system/theme.spec.js +43 -0
- package/dist/design-system/theme.spec.js.map +1 -0
- package/dist/design-system/types.d.ts +288 -0
- package/dist/design-system/types.d.ts.map +1 -0
- package/dist/design-system/types.js +2 -0
- package/dist/design-system/types.js.map +1 -0
- package/dist/editor/SharedRichEditor.d.ts.map +1 -1
- package/dist/editor/SharedRichEditor.js +3 -2
- package/dist/editor/SharedRichEditor.js.map +1 -1
- package/dist/editor/SlashCommandMenu.d.ts +8 -0
- package/dist/editor/SlashCommandMenu.d.ts.map +1 -1
- package/dist/editor/SlashCommandMenu.js +6 -0
- package/dist/editor/SlashCommandMenu.js.map +1 -1
- package/dist/editor/SlashCommandMenu.spec.d.ts +2 -0
- package/dist/editor/SlashCommandMenu.spec.d.ts.map +1 -0
- package/dist/editor/SlashCommandMenu.spec.js +20 -0
- package/dist/editor/SlashCommandMenu.spec.js.map +1 -0
- package/dist/editor/index.d.ts +1 -1
- package/dist/editor/index.d.ts.map +1 -1
- package/dist/editor/index.js +1 -1
- package/dist/editor/index.js.map +1 -1
- package/dist/provider.d.ts +4 -3
- package/dist/provider.d.ts.map +1 -1
- package/dist/provider.js +46 -4
- package/dist/provider.js.map +1 -1
- package/dist/styles.css +52 -23
- package/dist/ui/button.d.ts +5 -0
- package/dist/ui/button.d.ts.map +1 -1
- package/dist/ui/button.js +25 -4
- package/dist/ui/button.js.map +1 -1
- package/dist/ui/dialog.d.ts +4 -0
- package/dist/ui/dialog.d.ts.map +1 -1
- package/dist/ui/dialog.js +2 -2
- package/dist/ui/dialog.js.map +1 -1
- package/dist/ui/popover.d.ts +1 -0
- package/dist/ui/popover.d.ts.map +1 -1
- package/dist/ui/popover.js +2 -2
- package/dist/ui/popover.js.map +1 -1
- package/dist/ui/select.d.ts +3 -1
- package/dist/ui/select.d.ts.map +1 -1
- package/dist/ui/select.js +1 -1
- package/dist/ui/select.js.map +1 -1
- package/dist/ui/sidebar.d.ts +6 -2
- package/dist/ui/sidebar.d.ts.map +1 -1
- package/dist/ui/sidebar.js +12 -15
- package/dist/ui/sidebar.js.map +1 -1
- package/dist/ui/tooltip.d.ts +3 -1
- package/dist/ui/tooltip.d.ts.map +1 -1
- package/dist/ui/tooltip.js +2 -2
- package/dist/ui/tooltip.js.map +1 -1
- package/package.json +31 -1
- package/src/chat-history/ChatHistoryRail.spec.tsx +174 -0
- package/src/chat-history/ChatHistoryRail.tsx +100 -39
- package/src/chat-history/index.ts +9 -0
- package/src/chat-history/useChatHistoryRailController.spec.tsx +131 -0
- package/src/chat-history/useChatHistoryRailController.ts +76 -0
- package/src/chat-history.css +2 -0
- package/src/composer/ComposerPlusMenu.tsx +11 -8
- package/src/composer/PromptComposer.spec.ts +12 -1
- package/src/composer/PromptComposer.tsx +32 -7
- package/src/composer/RealtimeVoiceMode.spec.tsx +117 -9
- package/src/composer/RealtimeVoiceMode.tsx +119 -31
- package/src/composer/useRealtimeVoiceMode.spec.ts +101 -1
- package/src/composer/useRealtimeVoiceMode.tsx +124 -9
- package/src/conformance/__fixtures__/css-in-js-adapter.tsx +401 -0
- package/src/conformance/conformance.spec.tsx +143 -0
- package/src/conformance/index.ts +3 -0
- package/src/conformance/runner.tsx +862 -0
- package/src/conformance/types.ts +45 -0
- package/src/conformance/version.ts +22 -0
- package/src/design-system/components.tsx +146 -0
- package/src/design-system/context.tsx +31 -0
- package/src/design-system/default-adapter.spec.tsx +143 -0
- package/src/design-system/default-adapter.tsx +915 -0
- package/src/design-system/definition.ts +14 -0
- package/src/design-system/design-system.spec.tsx +183 -0
- package/src/design-system/error-boundary.tsx +37 -0
- package/src/design-system/index.ts +7 -0
- package/src/design-system/theme.spec.ts +61 -0
- package/src/design-system/theme.ts +145 -0
- package/src/design-system/types.ts +363 -0
- package/src/editor/SharedRichEditor.tsx +13 -2
- package/src/editor/SlashCommandMenu.spec.ts +33 -0
- package/src/editor/SlashCommandMenu.tsx +20 -0
- package/src/editor/index.ts +2 -0
- package/src/provider.tsx +98 -5
- package/src/styles.css +52 -23
- package/src/ui/button.tsx +95 -6
- package/src/ui/dialog.tsx +48 -23
- package/src/ui/popover.tsx +16 -2
- package/src/ui/select.tsx +7 -3
- package/src/ui/sidebar.tsx +47 -48
- package/src/ui/tooltip.tsx +7 -3
|
@@ -490,14 +490,17 @@ function ComposerPlusMenuFull({
|
|
|
490
490
|
<Popover open={open} onOpenChange={setOpen}>
|
|
491
491
|
<Tooltip>
|
|
492
492
|
<TooltipTrigger asChild>
|
|
493
|
-
<
|
|
494
|
-
<
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
493
|
+
<span className="inline-flex shrink-0">
|
|
494
|
+
<PopoverTrigger asChild>
|
|
495
|
+
<button
|
|
496
|
+
type="button"
|
|
497
|
+
aria-label="Add..."
|
|
498
|
+
className="shrink-0 flex h-7 w-7 items-center justify-center rounded-md text-muted-foreground hover:text-foreground hover:bg-accent/50 disabled:opacity-30 disabled:cursor-not-allowed"
|
|
499
|
+
>
|
|
500
|
+
<IconPlus className="h-4 w-4" />
|
|
501
|
+
</button>
|
|
502
|
+
</PopoverTrigger>
|
|
503
|
+
</span>
|
|
501
504
|
</TooltipTrigger>
|
|
502
505
|
<TooltipContent>Add...</TooltipContent>
|
|
503
506
|
</Tooltip>
|
|
@@ -2,7 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
import { describe, expect, it } from "vitest";
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
buildPromptComposerSubmission,
|
|
7
|
+
canSubmitWithAgentEngineState,
|
|
8
|
+
} from "./PromptComposer.js";
|
|
9
|
+
|
|
10
|
+
describe("canSubmitWithAgentEngineState", () => {
|
|
11
|
+
it("allows configured engines and blocks unresolved states", () => {
|
|
12
|
+
expect(canSubmitWithAgentEngineState("configured")).toBe(true);
|
|
13
|
+
expect(canSubmitWithAgentEngineState("missing")).toBe(false);
|
|
14
|
+
expect(canSubmitWithAgentEngineState("unavailable")).toBe(false);
|
|
15
|
+
});
|
|
16
|
+
});
|
|
6
17
|
|
|
7
18
|
describe("buildPromptComposerSubmission", () => {
|
|
8
19
|
it("passes images through files only — never inlines base64 into prompt text", async () => {
|
|
@@ -61,6 +61,10 @@ import type {
|
|
|
61
61
|
const MAX_INLINE_TEXT_FILE_CHARS = 60_000;
|
|
62
62
|
const SUBMIT_ENGINE_STATUS_TIMEOUT_MS = 1000;
|
|
63
63
|
|
|
64
|
+
export function canSubmitWithAgentEngineState(state: string): boolean {
|
|
65
|
+
return state === "configured";
|
|
66
|
+
}
|
|
67
|
+
|
|
64
68
|
/**
|
|
65
69
|
* Files the user attached via the "+" button in PromptComposer. The host owns
|
|
66
70
|
* what to do with them — typically POST to a per-app upload endpoint and pass
|
|
@@ -519,6 +523,7 @@ function PromptComposerInner({
|
|
|
519
523
|
resolvedModelStatusChecksEnabled,
|
|
520
524
|
);
|
|
521
525
|
const missingApiKey = agentEngineConfigured.missing;
|
|
526
|
+
const agentEngineUnavailable = agentEngineConfigured.state === "unavailable";
|
|
522
527
|
const [missingKeyBouncePulse, setMissingKeyBouncePulse] = useState(0);
|
|
523
528
|
const bounceMissingKeySetup = useCallback(() => {
|
|
524
529
|
setMissingKeyBouncePulse((pulse) => pulse + 1);
|
|
@@ -542,15 +547,25 @@ function PromptComposerInner({
|
|
|
542
547
|
timeoutMs: SUBMIT_ENGINE_STATUS_TIMEOUT_MS,
|
|
543
548
|
},
|
|
544
549
|
);
|
|
545
|
-
if (state
|
|
546
|
-
|
|
547
|
-
|
|
550
|
+
if (state === "missing") {
|
|
551
|
+
bounceMissingKeySetup();
|
|
552
|
+
return false;
|
|
553
|
+
}
|
|
554
|
+
if (typeof window !== "undefined") {
|
|
555
|
+
window.dispatchEvent(new Event("agent-engine:configured-changed"));
|
|
556
|
+
}
|
|
557
|
+
return canSubmitWithAgentEngineState(state);
|
|
548
558
|
}, [
|
|
549
559
|
agentEngineConfigured.state,
|
|
550
560
|
bounceMissingKeySetup,
|
|
551
561
|
modelsAdapter,
|
|
552
562
|
resolvedModelStatusChecksEnabled,
|
|
553
563
|
]);
|
|
564
|
+
const retryAgentEngineStatus = useCallback(() => {
|
|
565
|
+
if (typeof window !== "undefined") {
|
|
566
|
+
window.dispatchEvent(new Event("agent-engine:configured-changed"));
|
|
567
|
+
}
|
|
568
|
+
}, []);
|
|
554
569
|
|
|
555
570
|
useEffect(() => {
|
|
556
571
|
if (!autoFocus) return;
|
|
@@ -613,21 +628,31 @@ function PromptComposerInner({
|
|
|
613
628
|
<AgentComposerFrame
|
|
614
629
|
className={cn(
|
|
615
630
|
"text-start",
|
|
616
|
-
missingApiKey && "cursor-pointer",
|
|
631
|
+
(missingApiKey || agentEngineUnavailable) && "cursor-pointer",
|
|
617
632
|
className,
|
|
618
633
|
)}
|
|
619
634
|
rootClassName={rootClassName}
|
|
620
635
|
style={style}
|
|
621
636
|
rootStyle={rootStyle}
|
|
622
637
|
layoutVariant={layoutVariant}
|
|
623
|
-
onClick={
|
|
638
|
+
onClick={
|
|
639
|
+
missingApiKey
|
|
640
|
+
? bounceMissingKeySetup
|
|
641
|
+
: agentEngineUnavailable
|
|
642
|
+
? retryAgentEngineStatus
|
|
643
|
+
: undefined
|
|
644
|
+
}
|
|
624
645
|
>
|
|
625
646
|
<PromptAttachmentStrip />
|
|
626
647
|
<TiptapComposer
|
|
627
648
|
focusRef={handleRef}
|
|
628
|
-
disabled={disabled || missingApiKey}
|
|
649
|
+
disabled={disabled || missingApiKey || agentEngineUnavailable}
|
|
629
650
|
placeholder={
|
|
630
|
-
missingApiKey
|
|
651
|
+
missingApiKey
|
|
652
|
+
? "Connect AI above to continue..."
|
|
653
|
+
: agentEngineUnavailable
|
|
654
|
+
? "Unable to check AI connection. Click to retry."
|
|
655
|
+
: placeholder
|
|
631
656
|
}
|
|
632
657
|
initialText={initialText}
|
|
633
658
|
initialTextKey={initialTextKey}
|
|
@@ -89,9 +89,28 @@ const copy: RealtimeVoiceModeCopy = {
|
|
|
89
89
|
describe("RealtimeVoiceMode", () => {
|
|
90
90
|
let container: HTMLDivElement;
|
|
91
91
|
let root: Root;
|
|
92
|
+
let animate: ReturnType<typeof vi.fn>;
|
|
93
|
+
let cancelGlowAnimation: ReturnType<typeof vi.fn>;
|
|
94
|
+
let updateGlowPlaybackRate: ReturnType<typeof vi.fn>;
|
|
95
|
+
let originalAnimate: typeof HTMLElement.prototype.animate | undefined;
|
|
92
96
|
|
|
93
97
|
beforeEach(() => {
|
|
94
98
|
vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
|
|
99
|
+
originalAnimate = HTMLElement.prototype.animate;
|
|
100
|
+
cancelGlowAnimation = vi.fn();
|
|
101
|
+
updateGlowPlaybackRate = vi.fn();
|
|
102
|
+
animate = vi.fn(
|
|
103
|
+
() =>
|
|
104
|
+
({
|
|
105
|
+
cancel: cancelGlowAnimation,
|
|
106
|
+
playbackRate: 1,
|
|
107
|
+
updatePlaybackRate: updateGlowPlaybackRate,
|
|
108
|
+
}) as unknown as Animation,
|
|
109
|
+
);
|
|
110
|
+
Object.defineProperty(HTMLElement.prototype, "animate", {
|
|
111
|
+
configurable: true,
|
|
112
|
+
value: animate,
|
|
113
|
+
});
|
|
95
114
|
container = document.createElement("div");
|
|
96
115
|
document.body.appendChild(container);
|
|
97
116
|
root = createRoot(container);
|
|
@@ -100,6 +119,14 @@ describe("RealtimeVoiceMode", () => {
|
|
|
100
119
|
afterEach(() => {
|
|
101
120
|
act(() => root.unmount());
|
|
102
121
|
container.remove();
|
|
122
|
+
if (originalAnimate) {
|
|
123
|
+
Object.defineProperty(HTMLElement.prototype, "animate", {
|
|
124
|
+
configurable: true,
|
|
125
|
+
value: originalAnimate,
|
|
126
|
+
});
|
|
127
|
+
} else {
|
|
128
|
+
Reflect.deleteProperty(HTMLElement.prototype, "animate");
|
|
129
|
+
}
|
|
103
130
|
vi.unstubAllGlobals();
|
|
104
131
|
});
|
|
105
132
|
|
|
@@ -291,22 +318,21 @@ describe("RealtimeVoiceMode", () => {
|
|
|
291
318
|
expect(document.body.textContent).toContain("Set up voice mode");
|
|
292
319
|
const prompt = document.querySelector<HTMLElement>('[role="dialog"]');
|
|
293
320
|
expect(prompt?.className).toContain(
|
|
294
|
-
"w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,
|
|
321
|
+
"w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,24rem),24rem)]",
|
|
295
322
|
);
|
|
296
323
|
expect(prompt?.dataset.collisionBoundary).toBe("agent-panel");
|
|
297
324
|
const actions = Array.from(prompt?.querySelectorAll("div") ?? []).find(
|
|
298
|
-
(element) => element.className.includes("
|
|
325
|
+
(element) => element.className.includes("bg-muted/30"),
|
|
299
326
|
);
|
|
300
327
|
const actionClasses = actions?.className.split(/\s+/) ?? [];
|
|
301
|
-
expect(actionClasses).toContain("
|
|
302
|
-
expect(actionClasses).
|
|
328
|
+
expect(actionClasses).toContain("grid");
|
|
329
|
+
expect(actionClasses).toContain("rounded-lg");
|
|
303
330
|
const buttons = Array.from(
|
|
304
331
|
document.querySelectorAll<HTMLButtonElement>("button"),
|
|
305
332
|
);
|
|
306
333
|
expect(
|
|
307
|
-
buttons.find((button) => button.textContent === "Connect Builder.io")
|
|
308
|
-
|
|
309
|
-
).toContain("whitespace-nowrap");
|
|
334
|
+
buttons.find((button) => button.textContent === "Connect Builder.io"),
|
|
335
|
+
).toBeDefined();
|
|
310
336
|
act(() =>
|
|
311
337
|
buttons
|
|
312
338
|
.find((button) => button.textContent === "Connect Builder.io")
|
|
@@ -766,7 +792,28 @@ describe("RealtimeVoiceMode", () => {
|
|
|
766
792
|
},
|
|
767
793
|
);
|
|
768
794
|
|
|
769
|
-
it("
|
|
795
|
+
it.each(["listening", "speaking"] as const)(
|
|
796
|
+
"keeps a soft session glow while %s",
|
|
797
|
+
(state) => {
|
|
798
|
+
render(
|
|
799
|
+
<RealtimeVoiceModeDock
|
|
800
|
+
state={state}
|
|
801
|
+
copy={copy}
|
|
802
|
+
chatVisible={false}
|
|
803
|
+
onToggleChat={vi.fn()}
|
|
804
|
+
onEndVoiceMode={vi.fn()}
|
|
805
|
+
/>,
|
|
806
|
+
);
|
|
807
|
+
expect(
|
|
808
|
+
document.querySelector(".agent-realtime-voice-glow"),
|
|
809
|
+
).not.toBeNull();
|
|
810
|
+
expect(
|
|
811
|
+
document.querySelector(".agent-realtime-voice-working"),
|
|
812
|
+
).toBeNull();
|
|
813
|
+
},
|
|
814
|
+
);
|
|
815
|
+
|
|
816
|
+
it("brightens the live glow while working", () => {
|
|
770
817
|
render(
|
|
771
818
|
<RealtimeVoiceModeDock
|
|
772
819
|
state="working"
|
|
@@ -776,10 +823,14 @@ describe("RealtimeVoiceMode", () => {
|
|
|
776
823
|
onEndVoiceMode={vi.fn()}
|
|
777
824
|
/>,
|
|
778
825
|
);
|
|
826
|
+
expect(document.querySelector(".agent-realtime-voice-glow")).not.toBeNull();
|
|
779
827
|
expect(
|
|
780
828
|
document.querySelector(".agent-realtime-voice-working"),
|
|
781
829
|
).not.toBeNull();
|
|
830
|
+
});
|
|
782
831
|
|
|
832
|
+
it("changes glow speed without restarting or jumping its rotation", () => {
|
|
833
|
+
vi.useFakeTimers();
|
|
783
834
|
render(
|
|
784
835
|
<RealtimeVoiceModeDock
|
|
785
836
|
state="listening"
|
|
@@ -789,9 +840,66 @@ describe("RealtimeVoiceMode", () => {
|
|
|
789
840
|
onEndVoiceMode={vi.fn()}
|
|
790
841
|
/>,
|
|
791
842
|
);
|
|
792
|
-
|
|
843
|
+
const glow = document.querySelector('[data-realtime-voice-glow="true"]');
|
|
844
|
+
|
|
845
|
+
expect(animate).toHaveBeenCalledTimes(1);
|
|
846
|
+
expect(updateGlowPlaybackRate).toHaveBeenLastCalledWith(1);
|
|
847
|
+
|
|
848
|
+
render(
|
|
849
|
+
<RealtimeVoiceModeDock
|
|
850
|
+
state="working"
|
|
851
|
+
copy={copy}
|
|
852
|
+
chatVisible={false}
|
|
853
|
+
onToggleChat={vi.fn()}
|
|
854
|
+
onEndVoiceMode={vi.fn()}
|
|
855
|
+
/>,
|
|
856
|
+
);
|
|
857
|
+
|
|
858
|
+
expect(document.querySelector('[data-realtime-voice-glow="true"]')).toBe(
|
|
859
|
+
glow,
|
|
860
|
+
);
|
|
861
|
+
expect(animate).toHaveBeenCalledTimes(1);
|
|
862
|
+
expect(cancelGlowAnimation).not.toHaveBeenCalled();
|
|
863
|
+
expect(updateGlowPlaybackRate).toHaveBeenLastCalledWith(2.4);
|
|
864
|
+
|
|
865
|
+
render(
|
|
866
|
+
<RealtimeVoiceModeDock
|
|
867
|
+
state="speaking"
|
|
868
|
+
copy={copy}
|
|
869
|
+
chatVisible={false}
|
|
870
|
+
onToggleChat={vi.fn()}
|
|
871
|
+
onEndVoiceMode={vi.fn()}
|
|
872
|
+
/>,
|
|
873
|
+
);
|
|
874
|
+
|
|
875
|
+
expect(animate).toHaveBeenCalledTimes(1);
|
|
876
|
+
expect(cancelGlowAnimation).not.toHaveBeenCalled();
|
|
877
|
+
expect(updateGlowPlaybackRate).toHaveBeenLastCalledWith(2.4);
|
|
878
|
+
|
|
879
|
+
act(() => vi.advanceTimersByTime(800));
|
|
880
|
+
expect(updateGlowPlaybackRate).toHaveBeenLastCalledWith(1);
|
|
881
|
+
vi.useRealTimers();
|
|
793
882
|
});
|
|
794
883
|
|
|
884
|
+
it.each(["connecting", "error", "ending"] as const)(
|
|
885
|
+
"does not show the live glow while %s",
|
|
886
|
+
(state) => {
|
|
887
|
+
render(
|
|
888
|
+
<RealtimeVoiceModeDock
|
|
889
|
+
state={state}
|
|
890
|
+
copy={copy}
|
|
891
|
+
chatVisible={false}
|
|
892
|
+
onToggleChat={vi.fn()}
|
|
893
|
+
onEndVoiceMode={vi.fn()}
|
|
894
|
+
/>,
|
|
895
|
+
);
|
|
896
|
+
expect(document.querySelector(".agent-realtime-voice-glow")).toBeNull();
|
|
897
|
+
expect(
|
|
898
|
+
document.querySelector(".agent-realtime-voice-working"),
|
|
899
|
+
).toBeNull();
|
|
900
|
+
},
|
|
901
|
+
);
|
|
902
|
+
|
|
795
903
|
it("shows an unmistakable loader and ignores early audio until connected", () => {
|
|
796
904
|
const audioLevels = createRealtimeVoiceAudioLevelStore();
|
|
797
905
|
audioLevels.set({ input: 0.8, output: 0 });
|
|
@@ -10,10 +10,12 @@ import {
|
|
|
10
10
|
import {
|
|
11
11
|
IconAlertTriangle,
|
|
12
12
|
IconBrain,
|
|
13
|
+
IconKey,
|
|
13
14
|
IconLanguage,
|
|
14
15
|
IconLoader2,
|
|
15
16
|
IconMicrophone,
|
|
16
17
|
IconPhoneOff,
|
|
18
|
+
IconPlugConnected,
|
|
17
19
|
IconSettings,
|
|
18
20
|
IconVolume,
|
|
19
21
|
} from "@tabler/icons-react";
|
|
@@ -225,72 +227,90 @@ export function RealtimeVoiceModeEntry({
|
|
|
225
227
|
collisionPadding={16}
|
|
226
228
|
data-collision-boundary={collisionBoundary ? "agent-panel" : "viewport"}
|
|
227
229
|
className={cn(
|
|
228
|
-
setupRequired ? "p-
|
|
230
|
+
setupRequired ? "p-3.5" : "p-3",
|
|
229
231
|
setupRequired
|
|
230
|
-
? "w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,
|
|
232
|
+
? "w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,24rem),24rem)]"
|
|
231
233
|
: "w-[min(calc(100vw-2rem),var(--radix-popover-content-available-width,18rem),18rem)]",
|
|
232
234
|
)}
|
|
233
235
|
aria-labelledby={titleId}
|
|
234
236
|
aria-describedby={setupRequired ? descriptionId : undefined}
|
|
235
237
|
>
|
|
236
|
-
<div className={cn("grid", setupRequired ? "gap-3" : "gap-2.5")}>
|
|
237
|
-
<div
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
238
|
+
<div className={cn("grid", setupRequired ? "gap-3.5" : "gap-2.5")}>
|
|
239
|
+
<div
|
|
240
|
+
className={cn(
|
|
241
|
+
setupRequired ? "flex items-start gap-3" : "grid gap-1",
|
|
242
|
+
)}
|
|
243
|
+
>
|
|
241
244
|
{setupRequired ? (
|
|
242
|
-
<
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
>
|
|
246
|
-
{copy.setupDescription}
|
|
247
|
-
</p>
|
|
245
|
+
<div className="flex size-9 shrink-0 items-center justify-center rounded-lg bg-primary/10 text-primary">
|
|
246
|
+
<IconMicrophone aria-hidden="true" />
|
|
247
|
+
</div>
|
|
248
248
|
) : null}
|
|
249
|
+
<div className="grid gap-1">
|
|
250
|
+
<h2
|
|
251
|
+
id={titleId}
|
|
252
|
+
className="text-sm font-semibold leading-5 text-foreground"
|
|
253
|
+
>
|
|
254
|
+
{setupRequired ? copy.setupTitle : copy.promptTitle}
|
|
255
|
+
</h2>
|
|
256
|
+
{setupRequired ? (
|
|
257
|
+
<p
|
|
258
|
+
id={descriptionId}
|
|
259
|
+
className="text-xs leading-5 text-muted-foreground"
|
|
260
|
+
>
|
|
261
|
+
{copy.setupDescription}
|
|
262
|
+
</p>
|
|
263
|
+
) : null}
|
|
264
|
+
</div>
|
|
249
265
|
</div>
|
|
250
266
|
|
|
251
267
|
<div
|
|
252
268
|
className={cn(
|
|
253
|
-
"gap-2",
|
|
254
|
-
setupRequired
|
|
255
|
-
? "flex flex-col-reverse sm:flex-row sm:flex-wrap sm:justify-end"
|
|
256
|
-
: "grid",
|
|
269
|
+
"grid gap-2",
|
|
270
|
+
setupRequired ? "rounded-lg bg-muted/30 p-1" : null,
|
|
257
271
|
)}
|
|
258
272
|
>
|
|
259
273
|
{setupRequired ? (
|
|
260
274
|
<>
|
|
261
275
|
<Button
|
|
262
276
|
type="button"
|
|
263
|
-
variant="ghost"
|
|
264
277
|
size="sm"
|
|
265
|
-
|
|
278
|
+
className="w-full justify-start px-3"
|
|
279
|
+
disabled={connectingBuilder}
|
|
280
|
+
onClick={() =>
|
|
281
|
+
choose("realtime", onConnectBuilder ?? onStartVoiceMode)
|
|
282
|
+
}
|
|
266
283
|
>
|
|
267
|
-
{
|
|
284
|
+
{connectingBuilder ? (
|
|
285
|
+
<IconLoader2 className="animate-spin" />
|
|
286
|
+
) : (
|
|
287
|
+
<IconPlugConnected aria-hidden="true" />
|
|
288
|
+
)}
|
|
289
|
+
{copy.connectBuilder}
|
|
268
290
|
</Button>
|
|
269
291
|
<Button
|
|
270
292
|
type="button"
|
|
271
293
|
variant="outline"
|
|
272
294
|
size="sm"
|
|
295
|
+
className="w-full justify-start bg-background px-3"
|
|
273
296
|
onClick={() =>
|
|
274
297
|
choose("realtime", onUseOpenAiKey ?? onStartVoiceMode)
|
|
275
298
|
}
|
|
276
299
|
>
|
|
300
|
+
<IconKey aria-hidden="true" />
|
|
277
301
|
{openAiConfigured
|
|
278
302
|
? copy.startWithOpenAiKey
|
|
279
303
|
: copy.useOpenAiKey}
|
|
280
304
|
</Button>
|
|
281
305
|
<Button
|
|
282
306
|
type="button"
|
|
307
|
+
variant="ghost"
|
|
283
308
|
size="sm"
|
|
284
|
-
className="
|
|
285
|
-
|
|
286
|
-
onClick={() =>
|
|
287
|
-
choose("realtime", onConnectBuilder ?? onStartVoiceMode)
|
|
288
|
-
}
|
|
309
|
+
className="w-full justify-start px-3 text-muted-foreground"
|
|
310
|
+
onClick={() => choose("dictation", onKeepDictating)}
|
|
289
311
|
>
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
) : null}
|
|
293
|
-
{copy.connectBuilder}
|
|
312
|
+
<IconMicrophone aria-hidden="true" />
|
|
313
|
+
{copy.keepDictating}
|
|
294
314
|
</Button>
|
|
295
315
|
</>
|
|
296
316
|
) : (
|
|
@@ -382,9 +402,14 @@ export interface RealtimeVoiceModeInlineSettings {
|
|
|
382
402
|
const SILENT_AUDIO_LEVELS = createRealtimeVoiceAudioLevelStore();
|
|
383
403
|
const WAVEFORM_WEIGHTS = [0.55, 0.82, 1, 0.82, 0.55];
|
|
384
404
|
const AUDIO_ACTIVITY_THRESHOLD = 0.1;
|
|
405
|
+
const WORKING_GLOW_HOLD_MS = 800;
|
|
385
406
|
|
|
386
407
|
function usePrefersReducedMotion(): boolean {
|
|
387
|
-
const [reduced, setReduced] = useState(
|
|
408
|
+
const [reduced, setReduced] = useState(
|
|
409
|
+
() =>
|
|
410
|
+
typeof window !== "undefined" &&
|
|
411
|
+
Boolean(window.matchMedia?.("(prefers-reduced-motion: reduce)").matches),
|
|
412
|
+
);
|
|
388
413
|
useEffect(() => {
|
|
389
414
|
if (typeof window === "undefined" || !window.matchMedia) return;
|
|
390
415
|
const query = window.matchMedia("(prefers-reduced-motion: reduce)");
|
|
@@ -697,6 +722,9 @@ export function RealtimeVoiceModeDock({
|
|
|
697
722
|
const controlsId = useId();
|
|
698
723
|
const [controlsOpen, setControlsOpen] = useState(false);
|
|
699
724
|
const [settingsOpen, setSettingsOpen] = useState(false);
|
|
725
|
+
const [workingGlowHeld, setWorkingGlowHeld] = useState(state === "working");
|
|
726
|
+
const glowRef = useRef<HTMLSpanElement>(null);
|
|
727
|
+
const glowAnimationRef = useRef<Animation | null>(null);
|
|
700
728
|
const selectInteractionRef = useRef(false);
|
|
701
729
|
const selectInteractionFrameRef = useRef<number | null>(null);
|
|
702
730
|
const levels = useSyncExternalStore(
|
|
@@ -724,6 +752,56 @@ export function RealtimeVoiceModeDock({
|
|
|
724
752
|
const errorDetailVisible = state === "error" && Boolean(errorMessage);
|
|
725
753
|
const controlsVisible = controlsOpen || settingsOpen;
|
|
726
754
|
const chatPanelTranslation = useChatPanelTranslation(chatVisible);
|
|
755
|
+
const workingGlowActive = state === "working" || workingGlowHeld;
|
|
756
|
+
|
|
757
|
+
useEffect(() => {
|
|
758
|
+
if (state === "working") {
|
|
759
|
+
setWorkingGlowHeld(true);
|
|
760
|
+
return;
|
|
761
|
+
}
|
|
762
|
+
if (!workingGlowHeld) return;
|
|
763
|
+
const timer = window.setTimeout(
|
|
764
|
+
() => setWorkingGlowHeld(false),
|
|
765
|
+
WORKING_GLOW_HOLD_MS,
|
|
766
|
+
);
|
|
767
|
+
return () => window.clearTimeout(timer);
|
|
768
|
+
}, [state, workingGlowHeld]);
|
|
769
|
+
|
|
770
|
+
useEffect(() => {
|
|
771
|
+
if (!connected || reducedMotion || !glowRef.current) {
|
|
772
|
+
glowAnimationRef.current?.cancel();
|
|
773
|
+
glowAnimationRef.current = null;
|
|
774
|
+
return;
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
const animation = glowRef.current.animate(
|
|
778
|
+
[{ transform: "rotate(0turn)" }, { transform: "rotate(1turn)" }],
|
|
779
|
+
{
|
|
780
|
+
duration: 12_000,
|
|
781
|
+
easing: "linear",
|
|
782
|
+
iterations: Number.POSITIVE_INFINITY,
|
|
783
|
+
},
|
|
784
|
+
);
|
|
785
|
+
glowAnimationRef.current = animation;
|
|
786
|
+
|
|
787
|
+
return () => {
|
|
788
|
+
animation.cancel();
|
|
789
|
+
if (glowAnimationRef.current === animation) {
|
|
790
|
+
glowAnimationRef.current = null;
|
|
791
|
+
}
|
|
792
|
+
};
|
|
793
|
+
}, [connected, reducedMotion]);
|
|
794
|
+
|
|
795
|
+
useEffect(() => {
|
|
796
|
+
const animation = glowAnimationRef.current;
|
|
797
|
+
if (!animation) return;
|
|
798
|
+
const playbackRate = workingGlowActive ? 2.4 : 1;
|
|
799
|
+
if (typeof animation.updatePlaybackRate === "function") {
|
|
800
|
+
animation.updatePlaybackRate(playbackRate);
|
|
801
|
+
} else {
|
|
802
|
+
animation.playbackRate = playbackRate;
|
|
803
|
+
}
|
|
804
|
+
}, [workingGlowActive]);
|
|
727
805
|
|
|
728
806
|
const handleSelectOpenChange = useCallback((open: boolean) => {
|
|
729
807
|
if (selectInteractionFrameRef.current !== null) {
|
|
@@ -900,9 +978,19 @@ export function RealtimeVoiceModeDock({
|
|
|
900
978
|
className={cn(
|
|
901
979
|
"relative isolate size-16 overflow-visible rounded-full ring-1 backdrop-blur-xl transition-transform duration-150 ease-out focus-visible:ring-offset-2 active:scale-[0.97] motion-reduce:transition-none",
|
|
902
980
|
ORB_STATE_CLASSES[state],
|
|
903
|
-
state === "working" && "agent-realtime-voice-working",
|
|
904
981
|
)}
|
|
905
982
|
>
|
|
983
|
+
{connected ? (
|
|
984
|
+
<span
|
|
985
|
+
ref={glowRef}
|
|
986
|
+
aria-hidden="true"
|
|
987
|
+
data-realtime-voice-glow="true"
|
|
988
|
+
className={cn(
|
|
989
|
+
"agent-realtime-voice-glow",
|
|
990
|
+
workingGlowActive && "agent-realtime-voice-working",
|
|
991
|
+
)}
|
|
992
|
+
/>
|
|
993
|
+
) : null}
|
|
906
994
|
<span className="relative z-10 flex items-center justify-center">
|
|
907
995
|
{state === "connecting" ? (
|
|
908
996
|
<VoiceConnectingIndicator />
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
createRealtimeVoiceGreetingEvent,
|
|
7
7
|
createRealtimeVoiceGreetingStarter,
|
|
8
8
|
createRealtimeVoicePreferenceUpdate,
|
|
9
|
+
createRealtimeVoiceResponseCoordinator,
|
|
9
10
|
createRealtimeVoiceSession,
|
|
10
11
|
createRealtimeVoiceSessionWithCapability,
|
|
11
12
|
createRealtimeVoiceTranscriptSequencer,
|
|
@@ -521,7 +522,7 @@ describe("Realtime voice dynamic tool manifests", () => {
|
|
|
521
522
|
});
|
|
522
523
|
|
|
523
524
|
describe("extractRealtimeVoiceFunctionCalls", () => {
|
|
524
|
-
it("
|
|
525
|
+
it("returns a function call as soon as its final arguments arrive", () => {
|
|
525
526
|
expect(
|
|
526
527
|
extractRealtimeVoiceFunctionCalls({
|
|
527
528
|
type: "response.function_call_arguments.done",
|
|
@@ -543,6 +544,7 @@ describe("extractRealtimeVoiceFunctionCalls", () => {
|
|
|
543
544
|
extractRealtimeVoiceFunctionCalls({
|
|
544
545
|
type: "response.done",
|
|
545
546
|
response: {
|
|
547
|
+
status: "completed",
|
|
546
548
|
output: [
|
|
547
549
|
{ type: "message", role: "assistant" },
|
|
548
550
|
{
|
|
@@ -562,6 +564,49 @@ describe("extractRealtimeVoiceFunctionCalls", () => {
|
|
|
562
564
|
},
|
|
563
565
|
]);
|
|
564
566
|
});
|
|
567
|
+
|
|
568
|
+
it("accepts a completed function output item as a provider fallback", () => {
|
|
569
|
+
expect(
|
|
570
|
+
extractRealtimeVoiceFunctionCalls({
|
|
571
|
+
type: "response.output_item.done",
|
|
572
|
+
item: {
|
|
573
|
+
type: "function_call",
|
|
574
|
+
name: "navigate",
|
|
575
|
+
call_id: "call-output-item",
|
|
576
|
+
arguments: '{"path":"/todos"}',
|
|
577
|
+
},
|
|
578
|
+
}),
|
|
579
|
+
).toEqual([
|
|
580
|
+
{
|
|
581
|
+
name: "navigate",
|
|
582
|
+
callId: "call-output-item",
|
|
583
|
+
argumentsText: '{"path":"/todos"}',
|
|
584
|
+
},
|
|
585
|
+
]);
|
|
586
|
+
});
|
|
587
|
+
|
|
588
|
+
it("ignores function items from failed or cancelled responses", () => {
|
|
589
|
+
const output = [
|
|
590
|
+
{
|
|
591
|
+
type: "function_call",
|
|
592
|
+
name: "navigate",
|
|
593
|
+
call_id: "call-3",
|
|
594
|
+
arguments: "{}",
|
|
595
|
+
},
|
|
596
|
+
];
|
|
597
|
+
expect(
|
|
598
|
+
extractRealtimeVoiceFunctionCalls({
|
|
599
|
+
type: "response.done",
|
|
600
|
+
response: { status: "failed", output },
|
|
601
|
+
}),
|
|
602
|
+
).toEqual([]);
|
|
603
|
+
expect(
|
|
604
|
+
extractRealtimeVoiceFunctionCalls({
|
|
605
|
+
type: "response.done",
|
|
606
|
+
response: { status: "cancelled", output },
|
|
607
|
+
}),
|
|
608
|
+
).toEqual([]);
|
|
609
|
+
});
|
|
565
610
|
});
|
|
566
611
|
|
|
567
612
|
describe("extractCompletedRealtimeVoiceTranscript", () => {
|
|
@@ -632,6 +677,61 @@ describe("Realtime voice startup and transcript ordering", () => {
|
|
|
632
677
|
expect(send).toHaveBeenCalledTimes(2);
|
|
633
678
|
});
|
|
634
679
|
|
|
680
|
+
it("queues response requests behind the active response and retries a rejected request", () => {
|
|
681
|
+
const sent: Record<string, unknown>[] = [];
|
|
682
|
+
const coordinator = createRealtimeVoiceResponseCoordinator((event) =>
|
|
683
|
+
sent.push(event),
|
|
684
|
+
);
|
|
685
|
+
|
|
686
|
+
coordinator.request();
|
|
687
|
+
coordinator.handleEvent({
|
|
688
|
+
type: "response.created",
|
|
689
|
+
response: { id: "resp-active" },
|
|
690
|
+
});
|
|
691
|
+
coordinator.request();
|
|
692
|
+
expect(sent).toHaveLength(1);
|
|
693
|
+
|
|
694
|
+
expect(
|
|
695
|
+
coordinator.handleError(
|
|
696
|
+
"Conversation already has an active response in progress: resp-active",
|
|
697
|
+
),
|
|
698
|
+
).toBe(true);
|
|
699
|
+
coordinator.handleEvent({
|
|
700
|
+
type: "response.done",
|
|
701
|
+
response: { id: "resp-active", status: "completed" },
|
|
702
|
+
});
|
|
703
|
+
|
|
704
|
+
expect(sent).toHaveLength(2);
|
|
705
|
+
expect(sent[1]).toEqual({ type: "response.create" });
|
|
706
|
+
});
|
|
707
|
+
|
|
708
|
+
it("drops queued work after a failed response so the next turn can start", () => {
|
|
709
|
+
const sent: Record<string, unknown>[] = [];
|
|
710
|
+
const coordinator = createRealtimeVoiceResponseCoordinator((event) =>
|
|
711
|
+
sent.push(event),
|
|
712
|
+
);
|
|
713
|
+
|
|
714
|
+
coordinator.request();
|
|
715
|
+
coordinator.handleEvent({ type: "response.created" });
|
|
716
|
+
coordinator.request();
|
|
717
|
+
coordinator.handleEvent({
|
|
718
|
+
type: "response.done",
|
|
719
|
+
response: { status: "failed" },
|
|
720
|
+
});
|
|
721
|
+
|
|
722
|
+
expect(sent).toHaveLength(1);
|
|
723
|
+
coordinator.request();
|
|
724
|
+
expect(sent).toHaveLength(2);
|
|
725
|
+
});
|
|
726
|
+
|
|
727
|
+
it("ignores unrelated realtime errors", () => {
|
|
728
|
+
const coordinator = createRealtimeVoiceResponseCoordinator(vi.fn());
|
|
729
|
+
|
|
730
|
+
expect(coordinator.handleError("The microphone was disconnected.")).toBe(
|
|
731
|
+
false,
|
|
732
|
+
);
|
|
733
|
+
});
|
|
734
|
+
|
|
635
735
|
it("publishes in conversation order when user ASR finishes after the assistant", () => {
|
|
636
736
|
const published: Array<{ role: string; text: string }> = [];
|
|
637
737
|
const sequencer = createRealtimeVoiceTranscriptSequencer((transcript) => {
|