@assistant-ui/mcp-docs-server 0.1.38 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.docs/organized/code-examples/waterfall.md +11 -12
- package/.docs/organized/code-examples/with-a2a.md +18 -13
- package/.docs/organized/code-examples/with-ag-ui.md +19 -14
- package/.docs/organized/code-examples/{with-ai-sdk-v6.md → with-ai-sdk-v7.md} +34 -23
- package/.docs/organized/code-examples/with-artifacts.md +473 -141
- package/.docs/organized/code-examples/with-assistant-transport.md +17 -10
- package/.docs/organized/code-examples/with-browser-extension.md +17 -10
- package/.docs/organized/code-examples/with-chain-of-thought.md +21 -14
- package/.docs/organized/code-examples/with-cloud-standalone.md +12 -11
- package/.docs/organized/code-examples/with-cloud.md +20 -15
- package/.docs/organized/code-examples/with-custom-thread-list.md +19 -12
- package/.docs/organized/code-examples/with-elevenlabs-conversational.md +22 -15
- package/.docs/organized/code-examples/with-elevenlabs-scribe.md +22 -15
- package/.docs/organized/code-examples/with-eve.md +18 -11
- package/.docs/organized/code-examples/with-expo.md +35 -52
- package/.docs/organized/code-examples/with-external-store.md +18 -13
- package/.docs/organized/code-examples/with-ffmpeg.md +20 -15
- package/.docs/organized/code-examples/with-generative-ui.md +22 -17
- package/.docs/organized/code-examples/with-google-adk.md +18 -11
- package/.docs/organized/code-examples/with-heat-graph.md +10 -11
- package/.docs/organized/code-examples/with-image-generation.md +19 -12
- package/.docs/organized/code-examples/with-interactables.md +21 -17
- package/.docs/organized/code-examples/with-langchain.md +19 -12
- package/.docs/organized/code-examples/with-langgraph.md +19 -12
- package/.docs/organized/code-examples/with-livekit.md +23 -16
- package/.docs/organized/code-examples/with-mcp.md +46 -28
- package/.docs/organized/code-examples/with-opencode.md +25 -23
- package/.docs/organized/code-examples/with-pi.md +54 -19
- package/.docs/organized/code-examples/with-react-hook-form.md +21 -16
- package/.docs/organized/code-examples/with-react-ink-web.md +9 -9
- package/.docs/organized/code-examples/with-react-ink.md +4 -4
- package/.docs/organized/code-examples/with-react-router.md +22 -17
- package/.docs/organized/code-examples/with-resumable-stream.md +21 -14
- package/.docs/organized/code-examples/with-store.md +10 -11
- package/.docs/organized/code-examples/with-tanstack.md +19 -13
- package/.docs/organized/code-examples/with-tap-runtime.md +18 -13
- package/.docs/organized/code-examples/with-virtualized-thread.md +19 -14
- package/.docs/raw/docs/(docs)/base-ui.mdx +39 -0
- package/.docs/raw/docs/(docs)/cli.mdx +19 -1
- package/.docs/raw/docs/(docs)/devtools.mdx +7 -2
- package/.docs/raw/docs/(docs)/installation.mdx +15 -1
- package/.docs/raw/docs/(docs)/rtl.mdx +2 -4
- package/.docs/raw/docs/(reference)/api-reference/generative-ui/a2ui.mdx +40 -0
- package/.docs/raw/docs/(reference)/api-reference/generative-ui/actions.mdx +56 -0
- package/.docs/raw/docs/(reference)/api-reference/generative-ui/components.mdx +86 -0
- package/.docs/raw/docs/(reference)/api-reference/generative-ui/index.mdx +22 -1
- package/.docs/raw/docs/(reference)/api-reference/generative-ui/json-generative-ui.mdx +42 -0
- package/.docs/raw/docs/(reference)/api-reference/generative-ui/rendering.mdx +53 -2
- package/.docs/raw/docs/(reference)/api-reference/generative-ui/slack.mdx +81 -0
- package/.docs/raw/docs/(reference)/api-reference/generative-ui/teams.mdx +86 -0
- package/.docs/raw/docs/(reference)/api-reference/generative-ui/tokens.mdx +62 -0
- package/.docs/raw/docs/(reference)/api-reference/hooks/primitives.mdx +19 -420
- package/.docs/raw/docs/(reference)/api-reference/hooks/state.mdx +4 -1
- package/.docs/raw/docs/(reference)/api-reference/integrations/index.mdx +4 -1
- package/.docs/raw/docs/(reference)/api-reference/integrations/react-ai-sdk.mdx +25 -2
- package/.docs/raw/docs/(reference)/api-reference/integrations/react-data-stream.mdx +37 -0
- package/.docs/raw/docs/(reference)/api-reference/model-context/context.mdx +1 -9
- package/.docs/raw/docs/(reference)/api-reference/overview.mdx +1 -0
- package/.docs/raw/docs/(reference)/api-reference/primitives/assistant-if.mdx +14 -31
- package/.docs/raw/docs/(reference)/api-reference/primitives/composition.mdx +1 -0
- package/.docs/raw/docs/(reference)/api-reference/primitives/index.mdx +3 -0
- package/.docs/raw/docs/(reference)/api-reference/primitives/selection-toolbar.mdx +2 -0
- package/.docs/raw/docs/(reference)/api-reference/tools/status.mdx +0 -2
- package/.docs/raw/docs/(reference)/api-reference/tools/toolkits.mdx +12 -9
- package/.docs/raw/docs/(reference)/api-reference/utilities/miscellaneous.mdx +4 -0
- package/.docs/raw/docs/cloud/ai-sdk-assistant-ui.mdx +7 -1
- package/.docs/raw/docs/cloud/langgraph.mdx +4 -2
- package/.docs/raw/docs/copilots/model-context.mdx +1 -1
- package/.docs/raw/docs/copilots/motivation.mdx +1 -1
- package/.docs/raw/docs/guides/attachments.mdx +3 -3
- package/.docs/raw/docs/guides/branching.mdx +2 -2
- package/.docs/raw/docs/guides/chatgpt-subscription.mdx +108 -0
- package/.docs/raw/docs/guides/context-api.mdx +89 -111
- package/.docs/raw/docs/guides/dictation.mdx +185 -257
- package/.docs/raw/docs/guides/editing.mdx +5 -5
- package/.docs/raw/docs/guides/electron.mdx +369 -0
- package/.docs/raw/docs/guides/index.mdx +20 -0
- package/.docs/raw/docs/guides/mentions.mdx +31 -3
- package/.docs/raw/docs/guides/quoting.mdx +3 -3
- package/.docs/raw/docs/guides/resumable-stream-deployment.mdx +2 -2
- package/.docs/raw/docs/guides/resumable-streams.mdx +12 -1
- package/.docs/raw/docs/guides/speech.mdx +47 -29
- package/.docs/raw/docs/guides/suggestions.mdx +70 -1
- package/.docs/raw/docs/guides/voice.mdx +197 -267
- package/.docs/raw/docs/ink/hooks.mdx +3 -3
- package/.docs/raw/docs/ink/primitives.mdx +49 -8
- package/.docs/raw/docs/integrations/auth/better-auth.mdx +2 -2
- package/.docs/raw/docs/integrations/auth/clerk.mdx +2 -2
- package/.docs/raw/docs/integrations/auth/next-auth.mdx +3 -3
- package/.docs/raw/docs/integrations/frameworks/ai-sdk.mdx +10 -4
- package/.docs/raw/docs/integrations/frameworks/cloudflare-agents/overview.mdx +3 -3
- package/.docs/raw/docs/integrations/frameworks/mastra/full-stack.mdx +1 -1
- package/.docs/raw/docs/integrations/frameworks/mastra/overview.mdx +3 -3
- package/.docs/raw/docs/integrations/frameworks/mastra/separate-server.mdx +1 -1
- package/.docs/raw/docs/integrations/gateways/index.mdx +2 -2
- package/.docs/raw/docs/integrations/observability/helicone.mdx +1 -1
- package/.docs/raw/docs/integrations/observability/langfuse.mdx +1 -1
- package/.docs/raw/docs/integrations/observability/langsmith.mdx +2 -2
- package/.docs/raw/docs/integrations/persistence/custom-adapter.mdx +36 -9
- package/.docs/raw/docs/migrations/index.mdx +50 -0
- package/.docs/raw/docs/migrations/toolkit-tools.mdx +4 -2
- package/.docs/raw/docs/migrations/v0-15.mdx +156 -0
- package/.docs/raw/docs/primitives/chain-of-thought.mdx +6 -1
- package/.docs/raw/docs/primitives/composer.mdx +17 -1
- package/.docs/raw/docs/primitives/selection-toolbar.mdx +25 -0
- package/.docs/raw/docs/primitives/thread-list.mdx +2 -2
- package/.docs/raw/docs/react-native/hooks.mdx +3 -3
- package/.docs/raw/docs/react-native/index.mdx +2 -2
- package/.docs/raw/docs/react-native/primitives.mdx +62 -5
- package/.docs/raw/docs/runtimes/ag-ui/agent-state.mdx +124 -0
- package/.docs/raw/docs/runtimes/ag-ui/runtime-options.mdx +10 -1
- package/.docs/raw/docs/runtimes/ai-sdk/overview.mdx +14 -5
- package/.docs/raw/docs/runtimes/ai-sdk/v4-legacy.mdx +15 -15
- package/.docs/raw/docs/runtimes/ai-sdk/v5-legacy.mdx +8 -8
- package/.docs/raw/docs/runtimes/ai-sdk/{v6.mdx → v6-legacy.mdx} +11 -9
- package/.docs/raw/docs/runtimes/ai-sdk/v7.mdx +717 -0
- package/.docs/raw/docs/runtimes/concepts/adapters.mdx +1 -1
- package/.docs/raw/docs/runtimes/concepts/architecture.mdx +1 -1
- package/.docs/raw/docs/runtimes/concepts/threads.mdx +10 -10
- package/.docs/raw/docs/runtimes/custom/assistant-transport.mdx +2 -2
- package/.docs/raw/docs/runtimes/custom/data-stream.mdx +10 -19
- package/.docs/raw/docs/runtimes/custom/external-store.mdx +2 -2
- package/.docs/raw/docs/runtimes/custom/local-runtime.mdx +5 -3
- package/.docs/raw/docs/runtimes/eve/overview.mdx +1 -1
- package/.docs/raw/docs/runtimes/langchain.mdx +2 -2
- package/.docs/raw/docs/runtimes/langgraph/agent-state.mdx +181 -0
- package/.docs/raw/docs/runtimes/langgraph/overview.mdx +1 -1
- package/.docs/raw/docs/runtimes/opencode/hooks.mdx +3 -1
- package/.docs/raw/docs/tools/a2ui.mdx +107 -0
- package/.docs/raw/docs/tools/backend.mdx +6 -3
- package/.docs/raw/docs/tools/defining-tools.mdx +7 -1
- package/.docs/raw/docs/tools/generative-ui.mdx +60 -2
- package/.docs/raw/docs/tools/interactables-legacy.mdx +4 -4
- package/.docs/raw/docs/tools/interactables.mdx +3 -3
- package/.docs/raw/docs/tools/mcp-apps.mdx +90 -13
- package/.docs/raw/docs/tools/mcp.mdx +100 -3
- package/.docs/raw/docs/tools/tool-ui.mdx +6 -4
- package/.docs/raw/docs/tools/user-managed-mcp.mdx +77 -9
- package/.docs/raw/docs/ui/accordion.mdx +16 -10
- package/.docs/raw/docs/ui/assistant-modal.mdx +8 -4
- package/.docs/raw/docs/ui/attachment.mdx +5 -1
- package/.docs/raw/docs/ui/badge.mdx +23 -12
- package/.docs/raw/docs/ui/follow-up-suggestions.mdx +4 -2
- package/.docs/raw/docs/ui/model-selector.mdx +33 -3
- package/.docs/raw/docs/ui/part-grouping.mdx +0 -4
- package/.docs/raw/docs/ui/reasoning.mdx +1 -1
- package/.docs/raw/docs/ui/select.mdx +22 -14
- package/.docs/raw/docs/ui/sources.mdx +1 -1
- package/.docs/raw/docs/ui/tabs.mdx +25 -14
- package/.docs/raw/docs/utilities/heat-graph.mdx +2 -2
- package/dist/constants.d.ts.map +1 -1
- package/dist/index.d.ts +1 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +41 -2
- package/dist/index.js.map +1 -1
- package/dist/prepare-docs/code-examples.d.ts.map +1 -1
- package/dist/prepare-docs/code-examples.js.map +1 -1
- package/dist/prepare-docs/copy-raw.d.ts.map +1 -1
- package/dist/prepare-docs/prepare.d.ts +1 -1
- package/dist/prompts/xulux-playground.d.ts +12 -0
- package/dist/prompts/xulux-playground.d.ts.map +1 -0
- package/dist/prompts/xulux-playground.js +33 -0
- package/dist/prompts/xulux-playground.js.map +1 -0
- package/dist/stdio.d.ts +1 -1
- package/dist/tools/docs.d.ts +8 -14
- package/dist/tools/docs.d.ts.map +1 -1
- package/dist/tools/docs.js +26 -10
- package/dist/tools/docs.js.map +1 -1
- package/dist/tools/examples.d.ts +6 -12
- package/dist/tools/examples.d.ts.map +1 -1
- package/dist/tools/examples.js +11 -8
- package/dist/tools/examples.js.map +1 -1
- package/dist/tools/resources.d.ts +1 -2
- package/dist/tools/resources.d.ts.map +1 -1
- package/dist/tools/resources.js +1 -1
- package/dist/tools/resources.js.map +1 -1
- package/dist/tools/search.d.ts +6 -15
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +2 -2
- package/dist/tools/search.js.map +1 -1
- package/dist/tools/tests/mcp-test-client.d.ts +15 -0
- package/dist/tools/tests/mcp-test-client.d.ts.map +1 -0
- package/dist/tools/tests/mcp-test-client.js +68 -0
- package/dist/tools/tests/mcp-test-client.js.map +1 -0
- package/dist/tools/tests/test-setup.d.ts.map +1 -1
- package/dist/tools/tests/test-setup.js +5 -1
- package/dist/tools/tests/test-setup.js.map +1 -1
- package/dist/tools/xulux-templates.d.ts +58 -0
- package/dist/tools/xulux-templates.d.ts.map +1 -0
- package/dist/tools/xulux-templates.js +82 -0
- package/dist/tools/xulux-templates.js.map +1 -0
- package/dist/utils/cache.d.ts +5 -0
- package/dist/utils/cache.d.ts.map +1 -0
- package/dist/utils/cache.js +18 -0
- package/dist/utils/cache.js.map +1 -0
- package/dist/utils/logger.d.ts.map +1 -1
- package/dist/utils/mcp-format.d.ts +1 -0
- package/dist/utils/mcp-format.d.ts.map +1 -1
- package/dist/utils/mcp-format.js +7 -4
- package/dist/utils/mcp-format.js.map +1 -1
- package/dist/utils/mdx.d.ts.map +1 -1
- package/dist/utils/paths.d.ts +1 -1
- package/dist/utils/paths.d.ts.map +1 -1
- package/dist/utils/paths.js +3 -1
- package/dist/utils/paths.js.map +1 -1
- package/dist/utils/search.d.ts.map +1 -1
- package/dist/utils/security.d.ts.map +1 -1
- package/dist/utils/security.js.map +1 -1
- package/dist/xulux/catalog-client.d.ts +14 -0
- package/dist/xulux/catalog-client.d.ts.map +1 -0
- package/dist/xulux/catalog-client.js +67 -0
- package/dist/xulux/catalog-client.js.map +1 -0
- package/dist/xulux/fallback-catalog.d.ts +7 -0
- package/dist/xulux/fallback-catalog.d.ts.map +1 -0
- package/dist/xulux/fallback-catalog.js +47 -0
- package/dist/xulux/fallback-catalog.js.map +1 -0
- package/dist/xulux/fetch-sandbox.d.ts +5 -0
- package/dist/xulux/fetch-sandbox.d.ts.map +1 -0
- package/dist/xulux/fetch-sandbox.js +40 -0
- package/dist/xulux/fetch-sandbox.js.map +1 -0
- package/dist/xulux/template-service.d.ts +84 -0
- package/dist/xulux/template-service.d.ts.map +1 -0
- package/dist/xulux/template-service.js +223 -0
- package/dist/xulux/template-service.js.map +1 -0
- package/dist/xulux/types.d.ts +55 -0
- package/dist/xulux/types.d.ts.map +1 -0
- package/dist/xulux/types.js +6 -0
- package/dist/xulux/types.js.map +1 -0
- package/package.json +7 -6
- package/src/index.ts +55 -2
- package/src/prompts/xulux-playground.ts +36 -0
- package/src/tools/docs.ts +27 -5
- package/src/tools/examples.ts +17 -12
- package/src/tools/resources.ts +1 -4
- package/src/tools/search.ts +2 -2
- package/src/tools/tests/completions.test.ts +40 -26
- package/src/tools/tests/docs.test.ts +20 -0
- package/src/tools/tests/examples.test.ts +5 -5
- package/src/tools/tests/integration.test.ts +3 -4
- package/src/tools/tests/listings-cache.test.ts +19 -0
- package/src/tools/tests/mcp-protocol.test.ts +173 -108
- package/src/tools/tests/mcp-test-client.ts +111 -0
- package/src/tools/tests/resources.test.ts +97 -66
- package/src/tools/tests/test-setup.ts +8 -0
- package/src/tools/tests/xulux-templates.test.ts +262 -0
- package/src/tools/xulux-templates.ts +141 -0
- package/src/utils/cache.ts +20 -0
- package/src/utils/mcp-format.ts +8 -6
- package/src/utils/paths.ts +4 -1
- package/src/utils/tests/cache.test.ts +51 -0
- package/src/utils/tests/mcp-format.test.ts +22 -0
- package/src/utils/tests/security.test.ts +1 -1
- package/src/xulux/catalog-client.ts +105 -0
- package/src/xulux/fallback-catalog.ts +63 -0
- package/src/xulux/fetch-sandbox.ts +56 -0
- package/src/xulux/template-service.ts +406 -0
- package/src/xulux/types.ts +60 -0
- package/.docs/raw/docs/(reference)/api-reference/hooks/utilities.mdx +0 -464
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Electron
|
|
3
|
+
description: Run assistant-ui in an Electron renderer with a hosted backend or a secure, streaming preload and IPC bridge.
|
|
4
|
+
platforms: ["react"]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
assistant-ui works in Electron through `@assistant-ui/react`. The renderer is a React DOM environment, so there is no separate `@assistant-ui/electron` package to install. The Electron-specific decision is where model requests run and how the renderer reaches them.
|
|
8
|
+
|
|
9
|
+
<Callout type="warn">
|
|
10
|
+
Never put a provider API key in renderer code, a `VITE_*` variable, or a preload script. Bundled values are readable by anyone with the app. Keep remote credentials on your backend, or keep local credentials in the main process and provision them with an OS-backed secret store.
|
|
11
|
+
</Callout>
|
|
12
|
+
|
|
13
|
+
## Choose a connection pattern
|
|
14
|
+
|
|
15
|
+
| Pattern | Use it when | Runtime |
|
|
16
|
+
| --- | --- | --- |
|
|
17
|
+
| Hosted backend | You already have an AI SDK endpoint, need server auth or persistence, or ship the app to other people | `useChatRuntime` with an absolute HTTPS URL |
|
|
18
|
+
| Local main process | The desktop app owns the provider or agent process and must work without your web backend | `useLocalRuntime` with a narrow preload/IPC bridge |
|
|
19
|
+
|
|
20
|
+
```mermaid
|
|
21
|
+
flowchart LR
|
|
22
|
+
Renderer["Electron renderer<br/>assistant-ui React"] -->|"absolute HTTPS"| Backend["Hosted chat backend"]
|
|
23
|
+
Renderer -->|"window.assistantAI"| Preload["context-isolated preload"]
|
|
24
|
+
Preload -->|"transferred MessagePort"| Main["Electron main process"]
|
|
25
|
+
Backend --> Provider["Model or agent"]
|
|
26
|
+
Main --> Provider
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Do not send an SDK client, assistant-ui runtime, callback, `AbortSignal`, or `File` through IPC. Electron IPC uses structured clone semantics; define a small data-only protocol instead.
|
|
30
|
+
|
|
31
|
+
## Pattern 1: hosted backend
|
|
32
|
+
|
|
33
|
+
This is the smallest integration. Keep your existing AI SDK chat route and point `AssistantChatTransport` at its public URL.
|
|
34
|
+
|
|
35
|
+
```tsx title="renderer/assistant-runtime.tsx"
|
|
36
|
+
import type { ReactNode } from "react";
|
|
37
|
+
import { AssistantRuntimeProvider } from "@assistant-ui/react";
|
|
38
|
+
import {
|
|
39
|
+
AssistantChatTransport,
|
|
40
|
+
useChatRuntime,
|
|
41
|
+
} from "@assistant-ui/react-ai-sdk";
|
|
42
|
+
|
|
43
|
+
const transport = new AssistantChatTransport({
|
|
44
|
+
api: "https://api.example.com/chat",
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
export function ElectronRuntimeProvider({ children }: { children: ReactNode }) {
|
|
48
|
+
const runtime = useChatRuntime({ transport });
|
|
49
|
+
|
|
50
|
+
return (
|
|
51
|
+
<AssistantRuntimeProvider runtime={runtime}>
|
|
52
|
+
{children}
|
|
53
|
+
</AssistantRuntimeProvider>
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The URL must be absolute in a packaged app. A relative value such as `/api/chat` targets the renderer's `file://` or custom-protocol origin, not your deployed backend. Configure the backend's CORS policy for the packaged origin, allow the endpoint in `connect-src`, and authenticate requests as you would from any desktop client.
|
|
59
|
+
|
|
60
|
+
The endpoint must return the AI SDK UI message stream consumed by `AssistantChatTransport`. See the [AI SDK runtime guide](/docs/runtimes/ai-sdk/v7) for the backend route and tool-calling setup.
|
|
61
|
+
|
|
62
|
+
## Pattern 2: local main process
|
|
63
|
+
|
|
64
|
+
Use this pattern when the Electron main process calls the model or runs a local agent. Keep `contextIsolation: true`, `sandbox: true`, and `nodeIntegration: false` on the `BrowserWindow`. Register `registerAssistantIpc(mainWindow)` after creating the trusted window.
|
|
65
|
+
|
|
66
|
+
The following example is intentionally text-only. It streams one request over a dedicated `MessagePort`; closing that port propagates assistant-ui's Stop action to an `AbortController` in the main process.
|
|
67
|
+
|
|
68
|
+
### 1. Define a data-only protocol
|
|
69
|
+
|
|
70
|
+
Place the shared types somewhere all three Electron bundles can import.
|
|
71
|
+
|
|
72
|
+
```ts title="shared/assistant-ipc.ts"
|
|
73
|
+
export const ASSISTANT_STREAM_CHANNEL = "assistant:stream";
|
|
74
|
+
|
|
75
|
+
export type ChatMessage =
|
|
76
|
+
| { role: "user"; content: string }
|
|
77
|
+
| { role: "assistant"; content: string };
|
|
78
|
+
|
|
79
|
+
export type ChatRequest = {
|
|
80
|
+
system?: string;
|
|
81
|
+
messages: ChatMessage[];
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
export type ChatEvent =
|
|
85
|
+
| { type: "delta"; text: string }
|
|
86
|
+
| { type: "done" }
|
|
87
|
+
| { type: "error"; message: string };
|
|
88
|
+
|
|
89
|
+
export type AssistantAI = {
|
|
90
|
+
streamChat(
|
|
91
|
+
request: ChatRequest,
|
|
92
|
+
onEvent: (event: ChatEvent) => void,
|
|
93
|
+
): () => void;
|
|
94
|
+
};
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### 2. Expose one preload capability
|
|
98
|
+
|
|
99
|
+
Expose the smallest API the renderer needs, not `ipcRenderer` itself. The callback receives only validated event data, never Electron's privileged IPC event object.
|
|
100
|
+
|
|
101
|
+
```ts title="preload/assistant.ts"
|
|
102
|
+
import { contextBridge, ipcRenderer } from "electron";
|
|
103
|
+
import {
|
|
104
|
+
ASSISTANT_STREAM_CHANNEL,
|
|
105
|
+
type AssistantAI,
|
|
106
|
+
type ChatEvent,
|
|
107
|
+
} from "./shared";
|
|
108
|
+
|
|
109
|
+
const assistantAI: AssistantAI = {
|
|
110
|
+
streamChat(request, onEvent) {
|
|
111
|
+
const { port1, port2 } = new MessageChannel();
|
|
112
|
+
const onMessage = (event: MessageEvent<ChatEvent>) => onEvent(event.data);
|
|
113
|
+
|
|
114
|
+
port1.addEventListener("message", onMessage);
|
|
115
|
+
port1.start();
|
|
116
|
+
ipcRenderer.postMessage(ASSISTANT_STREAM_CHANNEL, request, [port2]);
|
|
117
|
+
|
|
118
|
+
let stopped = false;
|
|
119
|
+
return () => {
|
|
120
|
+
if (stopped) return;
|
|
121
|
+
stopped = true;
|
|
122
|
+
port1.removeEventListener("message", onMessage);
|
|
123
|
+
port1.close();
|
|
124
|
+
};
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
contextBridge.exposeInMainWorld("assistantAI", assistantAI);
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
Declare the context-bridge API for renderer TypeScript:
|
|
132
|
+
|
|
133
|
+
```ts title="renderer/electron.d.ts"
|
|
134
|
+
import type { AssistantAI } from "./shared";
|
|
135
|
+
|
|
136
|
+
declare global {
|
|
137
|
+
interface Window {
|
|
138
|
+
assistantAI: AssistantAI;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export {};
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
### 3. Stream from the main process
|
|
146
|
+
|
|
147
|
+
Install the provider packages in the Electron main-process bundle:
|
|
148
|
+
|
|
149
|
+
```sh
|
|
150
|
+
pnpm add ai @ai-sdk/openai
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
The handler checks both the sending `WebContents` and its main frame, validates the untrusted payload and bounds its size before calling the model. Set `OPENAI_API_KEY` only in the main-process environment.
|
|
154
|
+
|
|
155
|
+
```ts title="main/assistant-ipc.ts"
|
|
156
|
+
import { openai } from "@ai-sdk/openai";
|
|
157
|
+
import { streamText } from "ai";
|
|
158
|
+
import { ipcMain, type BrowserWindow, type IpcMainEvent } from "electron";
|
|
159
|
+
import {
|
|
160
|
+
ASSISTANT_STREAM_CHANNEL,
|
|
161
|
+
type ChatEvent,
|
|
162
|
+
type ChatRequest,
|
|
163
|
+
} from "./shared";
|
|
164
|
+
|
|
165
|
+
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
166
|
+
typeof value === "object" && value !== null;
|
|
167
|
+
|
|
168
|
+
const isChatRequest = (value: unknown): value is ChatRequest => {
|
|
169
|
+
if (!isRecord(value) || !Array.isArray(value.messages)) return false;
|
|
170
|
+
if (value.system !== undefined && typeof value.system !== "string") {
|
|
171
|
+
return false;
|
|
172
|
+
}
|
|
173
|
+
if (value.messages.length > 200) return false;
|
|
174
|
+
|
|
175
|
+
let totalLength = typeof value.system === "string" ? value.system.length : 0;
|
|
176
|
+
for (const message of value.messages) {
|
|
177
|
+
if (!isRecord(message)) return false;
|
|
178
|
+
if (message.role !== "user" && message.role !== "assistant") return false;
|
|
179
|
+
if (typeof message.content !== "string") return false;
|
|
180
|
+
totalLength += message.content.length;
|
|
181
|
+
if (totalLength > 1_000_000) return false;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
return true;
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
export function registerAssistantIpc(mainWindow: BrowserWindow) {
|
|
188
|
+
const handleStream = (event: IpcMainEvent, request: unknown) => {
|
|
189
|
+
const [port] = event.ports;
|
|
190
|
+
if (!port) return;
|
|
191
|
+
|
|
192
|
+
if (
|
|
193
|
+
event.sender !== mainWindow.webContents ||
|
|
194
|
+
event.senderFrame !== mainWindow.webContents.mainFrame
|
|
195
|
+
) {
|
|
196
|
+
port.close();
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
port.start();
|
|
201
|
+
if (!isChatRequest(request)) {
|
|
202
|
+
port.postMessage({
|
|
203
|
+
type: "error",
|
|
204
|
+
message: "Invalid chat request.",
|
|
205
|
+
} satisfies ChatEvent);
|
|
206
|
+
port.close();
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const abortController = new AbortController();
|
|
211
|
+
port.once("close", () => abortController.abort());
|
|
212
|
+
const send = (message: ChatEvent) => port.postMessage(message);
|
|
213
|
+
|
|
214
|
+
void (async () => {
|
|
215
|
+
try {
|
|
216
|
+
const result = streamText({
|
|
217
|
+
model: openai("gpt-5.4-mini"),
|
|
218
|
+
messages: request.messages,
|
|
219
|
+
...(request.system ? { system: request.system } : {}),
|
|
220
|
+
abortSignal: abortController.signal,
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
for await (const text of result.textStream) {
|
|
224
|
+
send({ type: "delta", text });
|
|
225
|
+
}
|
|
226
|
+
send({ type: "done" });
|
|
227
|
+
} catch (error) {
|
|
228
|
+
if (abortController.signal.aborted) return;
|
|
229
|
+
console.error("Assistant stream failed", error);
|
|
230
|
+
send({ type: "error", message: "The model request failed." });
|
|
231
|
+
}
|
|
232
|
+
})();
|
|
233
|
+
};
|
|
234
|
+
|
|
235
|
+
ipcMain.on(ASSISTANT_STREAM_CHANNEL, handleStream);
|
|
236
|
+
mainWindow.once("closed", () => {
|
|
237
|
+
ipcMain.removeListener(ASSISTANT_STREAM_CHANNEL, handleStream);
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
If your app can open more than one trusted assistant window, register a unique channel per window or route requests through one application-level registry. A global `ipcMain.on` listener should not be re-registered under the same channel for every window.
|
|
243
|
+
|
|
244
|
+
### 4. Adapt IPC to assistant-ui
|
|
245
|
+
|
|
246
|
+
`ChatModelAdapter` yields complete snapshots, so the renderer accumulates each IPC delta before yielding it. Its cleanup function closes the port on Stop, unmount, or completion.
|
|
247
|
+
|
|
248
|
+
```tsx title="renderer/assistant-runtime.tsx"
|
|
249
|
+
import type { ReactNode } from "react";
|
|
250
|
+
import {
|
|
251
|
+
AssistantRuntimeProvider,
|
|
252
|
+
useLocalRuntime,
|
|
253
|
+
type ChatModelAdapter,
|
|
254
|
+
} from "@assistant-ui/react";
|
|
255
|
+
import type { ChatMessage } from "./shared";
|
|
256
|
+
|
|
257
|
+
const ipcChatModel: ChatModelAdapter = {
|
|
258
|
+
async *run({ messages, context, abortSignal }) {
|
|
259
|
+
abortSignal.throwIfAborted();
|
|
260
|
+
|
|
261
|
+
const system = [context.system];
|
|
262
|
+
const serializedMessages: ChatMessage[] = [];
|
|
263
|
+
|
|
264
|
+
for (const message of messages) {
|
|
265
|
+
const text = message.content
|
|
266
|
+
.flatMap((part) => (part.type === "text" ? [part.text] : []))
|
|
267
|
+
.join("\n");
|
|
268
|
+
if (!text) continue;
|
|
269
|
+
|
|
270
|
+
if (message.role === "system") system.push(text);
|
|
271
|
+
if (message.role === "user") {
|
|
272
|
+
serializedMessages.push({ role: "user", content: text });
|
|
273
|
+
}
|
|
274
|
+
if (message.role === "assistant") {
|
|
275
|
+
serializedMessages.push({ role: "assistant", content: text });
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const systemText = system.filter(Boolean).join("\n\n");
|
|
280
|
+
let stop: (() => void) | undefined;
|
|
281
|
+
let removeAbortListener: (() => void) | undefined;
|
|
282
|
+
const deltas = new ReadableStream<string>({
|
|
283
|
+
start(controller) {
|
|
284
|
+
let settled = false;
|
|
285
|
+
const close = () => {
|
|
286
|
+
if (settled) return;
|
|
287
|
+
settled = true;
|
|
288
|
+
controller.close();
|
|
289
|
+
};
|
|
290
|
+
const fail = (error: unknown) => {
|
|
291
|
+
if (settled) return;
|
|
292
|
+
settled = true;
|
|
293
|
+
controller.error(error);
|
|
294
|
+
};
|
|
295
|
+
|
|
296
|
+
stop = window.assistantAI.streamChat(
|
|
297
|
+
{
|
|
298
|
+
...(systemText ? { system: systemText } : {}),
|
|
299
|
+
messages: serializedMessages,
|
|
300
|
+
},
|
|
301
|
+
(event) => {
|
|
302
|
+
if (event.type === "delta") controller.enqueue(event.text);
|
|
303
|
+
if (event.type === "done") close();
|
|
304
|
+
if (event.type === "error") fail(new Error(event.message));
|
|
305
|
+
},
|
|
306
|
+
);
|
|
307
|
+
|
|
308
|
+
const onAbort = () => {
|
|
309
|
+
stop?.();
|
|
310
|
+
fail(abortSignal.reason);
|
|
311
|
+
};
|
|
312
|
+
abortSignal.addEventListener("abort", onAbort, { once: true });
|
|
313
|
+
removeAbortListener = () =>
|
|
314
|
+
abortSignal.removeEventListener("abort", onAbort);
|
|
315
|
+
if (abortSignal.aborted) onAbort();
|
|
316
|
+
},
|
|
317
|
+
cancel() {
|
|
318
|
+
stop?.();
|
|
319
|
+
},
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
const reader = deltas.getReader();
|
|
323
|
+
let fullText = "";
|
|
324
|
+
try {
|
|
325
|
+
while (true) {
|
|
326
|
+
const { done, value } = await reader.read();
|
|
327
|
+
if (done) return;
|
|
328
|
+
fullText += value;
|
|
329
|
+
yield { content: [{ type: "text", text: fullText }] };
|
|
330
|
+
}
|
|
331
|
+
} finally {
|
|
332
|
+
removeAbortListener?.();
|
|
333
|
+
stop?.();
|
|
334
|
+
reader.releaseLock();
|
|
335
|
+
}
|
|
336
|
+
},
|
|
337
|
+
};
|
|
338
|
+
|
|
339
|
+
export function ElectronRuntimeProvider({ children }: { children: ReactNode }) {
|
|
340
|
+
const runtime = useLocalRuntime(ipcChatModel);
|
|
341
|
+
return (
|
|
342
|
+
<AssistantRuntimeProvider runtime={runtime}>
|
|
343
|
+
{children}
|
|
344
|
+
</AssistantRuntimeProvider>
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
Wrap your existing assistant-ui thread with `ElectronRuntimeProvider`. Primitives and installed UI components work exactly as they do in a browser React app.
|
|
350
|
+
|
|
351
|
+
## Extend the protocol deliberately
|
|
352
|
+
|
|
353
|
+
The local example drops every non-text content part. Add explicit structured-clone-safe fields before claiming support for more features:
|
|
354
|
+
|
|
355
|
+
- Attachments: pass bounded `ArrayBuffer` data or an app-owned file reference after validating type, size, and path in the main process. Do not pass browser `File` objects.
|
|
356
|
+
- Tools: define serializable tool-call and tool-result events, and validate every tool invocation in the privileged process. Never expose a generic shell or filesystem IPC method.
|
|
357
|
+
- Reasoning and metadata: add distinct event variants and map them to the corresponding assistant-ui content parts.
|
|
358
|
+
- Thread persistence: store serializable thread data in the main process or use a remote thread-list adapter; do not try to transfer a runtime object.
|
|
359
|
+
|
|
360
|
+
## Packaged-app checklist
|
|
361
|
+
|
|
362
|
+
- Use an absolute HTTPS endpoint, a privileged custom protocol, or the preload bridge. Relative `/api/*` routes that worked against a development server will not follow your backend after packaging.
|
|
363
|
+
- Serve local content through a custom protocol instead of `file://` when possible, and define a restrictive Content Security Policy. Do not disable `webSecurity` to work around origin errors.
|
|
364
|
+
- Validate every IPC sender and every payload in the main process. Treat renderer data as untrusted even when the renderer is local.
|
|
365
|
+
- Intercept new windows and external links with `webContents.setWindowOpenHandler`; do not let model-generated links create unrestricted Electron windows.
|
|
366
|
+
- If you render remote HTML with `SafeContentFrame`, allow its documented host in `frame-src`. Ordinary text and Markdown rendering need no Electron-specific change.
|
|
367
|
+
- Test the packaged build, not only the Vite or Webpack development server. The scheme, origin, preload path, CSP, and environment loading can all differ.
|
|
368
|
+
|
|
369
|
+
For the underlying platform constraints, see Electron's official guides to [IPC and MessagePorts](https://www.electronjs.org/docs/latest/tutorial/message-ports), [context isolation](https://www.electronjs.org/docs/latest/tutorial/context-isolation), and the [security checklist](https://www.electronjs.org/docs/latest/tutorial/security).
|
|
@@ -95,3 +95,23 @@ Read and drive runtime state from your own code.
|
|
|
95
95
|
Access threads, messages, composer, and tool state via `useAui` and the runtime scope tree.
|
|
96
96
|
</Card>
|
|
97
97
|
</Cards>
|
|
98
|
+
|
|
99
|
+
## Providers
|
|
100
|
+
|
|
101
|
+
Alternative ways to connect a model.
|
|
102
|
+
|
|
103
|
+
<Cards>
|
|
104
|
+
<Card title="ChatGPT Subscription" href="/docs/guides/chatgpt-subscription">
|
|
105
|
+
Run your app locally on a ChatGPT Plus or Pro plan via Codex OAuth, no API key required.
|
|
106
|
+
</Card>
|
|
107
|
+
</Cards>
|
|
108
|
+
|
|
109
|
+
## Environments
|
|
110
|
+
|
|
111
|
+
Run assistant-ui outside a conventional browser deployment.
|
|
112
|
+
|
|
113
|
+
<Cards>
|
|
114
|
+
<Card title="Electron" href="/docs/guides/electron">
|
|
115
|
+
Connect an Electron renderer to a hosted backend or a secure, streaming preload/IPC bridge.
|
|
116
|
+
</Card>
|
|
117
|
+
</Cards>
|
|
@@ -25,9 +25,7 @@ Under the hood, mentions are one kind of [trigger popover](/docs/guides/slash-co
|
|
|
25
25
|
|
|
26
26
|
The fastest path is the pre-built [Mention UI components](/docs/ui/composer-trigger-popover), which wire everything together with two shadcn components — the popover picker and the message-side chip renderer:
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
npx shadcn@latest add "https://r.assistant-ui.com/composer-trigger-popover" "https://r.assistant-ui.com/directive-text"
|
|
30
|
-
```
|
|
28
|
+
<InstallCommand shadcn={["composer-trigger-popover", "directive-text"]} />
|
|
31
29
|
|
|
32
30
|
See the [Composer Trigger Popover](/docs/ui/composer-trigger-popover) and [Directive Text](/docs/ui/directive-text) guides for setup steps.
|
|
33
31
|
|
|
@@ -414,6 +412,36 @@ import { LexicalComposerInput } from "@assistant-ui/react-lexical";
|
|
|
414
412
|
|
|
415
413
|
`LexicalComposerInput` automatically discovers every `Directive` trigger registered under `TriggerPopoverRoot` and renders their selections as inline chips.
|
|
416
414
|
|
|
415
|
+
### Custom Lexical Plugins
|
|
416
|
+
|
|
417
|
+
Children of `LexicalComposerInput` render inside the `LexicalComposer` context after the built-in plugins, so standard Lexical plugin components built on `useLexicalComposerContext` work for editor concerns the mention system does not cover, such as paste normalization or length limits. Custom plugins import Lexical APIs directly, so install `lexical` and `@lexical/react` as direct dependencies of your app. The example below registers an update listener that flags messages over a maximum length.
|
|
418
|
+
|
|
419
|
+
```tsx
|
|
420
|
+
import { useLexicalComposerContext } from "@lexical/react/LexicalComposerContext";
|
|
421
|
+
import { useEffect } from "react";
|
|
422
|
+
import { $getRoot } from "lexical";
|
|
423
|
+
|
|
424
|
+
function MaxLengthPlugin({ maxLength }: { maxLength: number }) {
|
|
425
|
+
const [editor] = useLexicalComposerContext();
|
|
426
|
+
|
|
427
|
+
useEffect(() => {
|
|
428
|
+
return editor.registerUpdateListener(({ editorState }) => {
|
|
429
|
+
editorState.read(() => {
|
|
430
|
+
if ($getRoot().getTextContent().length > maxLength) {
|
|
431
|
+
console.warn(`Message exceeds ${maxLength} characters`);
|
|
432
|
+
}
|
|
433
|
+
});
|
|
434
|
+
});
|
|
435
|
+
}, [editor, maxLength]);
|
|
436
|
+
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
<LexicalComposerInput placeholder="Ask anything...">
|
|
441
|
+
<MaxLengthPlugin maxLength={2000} />
|
|
442
|
+
</LexicalComposerInput>
|
|
443
|
+
```
|
|
444
|
+
|
|
417
445
|
## Rendering Mentions in Messages
|
|
418
446
|
|
|
419
447
|
Use `DirectiveText` as the `Text` component for user messages so directives render as inline chips instead of raw syntax. See the [Directive Text](/docs/ui/directive-text) guide for setup and customization.
|
|
@@ -127,7 +127,7 @@ function CustomQuoteDisplay() {
|
|
|
127
127
|
|
|
128
128
|
## Programmatic API
|
|
129
129
|
|
|
130
|
-
Set or clear quotes via `useAui` from `@assistant-ui/react`. Call `aui.thread
|
|
130
|
+
Set or clear quotes via `useAui` from `@assistant-ui/react`. Call `aui.thread.composer().setQuote()` when your component is rendered outside of a specific thread context, or `aui.composer.setQuote()` when it is rendered inside a thread:
|
|
131
131
|
|
|
132
132
|
```tsx
|
|
133
133
|
import { useAui } from "@assistant-ui/react";
|
|
@@ -136,14 +136,14 @@ function MyComponent() {
|
|
|
136
136
|
const aui = useAui();
|
|
137
137
|
|
|
138
138
|
const quoteText = () => {
|
|
139
|
-
aui.thread
|
|
139
|
+
aui.thread.composer().setQuote({
|
|
140
140
|
text: "The text to quote",
|
|
141
141
|
messageId: "msg-123",
|
|
142
142
|
});
|
|
143
143
|
};
|
|
144
144
|
|
|
145
145
|
const clearQuote = () => {
|
|
146
|
-
aui.thread
|
|
146
|
+
aui.thread.composer().setQuote(undefined);
|
|
147
147
|
};
|
|
148
148
|
|
|
149
149
|
return (
|
|
@@ -143,7 +143,7 @@ Per-tenant prefixes also make incident response cheaper. A `SCAN MATCH aui:app:t
|
|
|
143
143
|
|
|
144
144
|
## Observability hooks
|
|
145
145
|
|
|
146
|
-
`ResumableStreamContextOptions` exposes lifecycle hooks for structured logging, metrics, and tracing.
|
|
146
|
+
`ResumableStreamContextOptions` exposes lifecycle hooks for structured logging, metrics, and tracing. Hook exceptions and rejected promises are reported through `console.error` without affecting stream output or stored lifecycle status.
|
|
147
147
|
|
|
148
148
|
```ts title="/lib/resumable-context.ts"
|
|
149
149
|
import { createResumableStreamContext } from "assistant-stream/resumable";
|
|
@@ -170,7 +170,7 @@ export const resumableContext = createResumableStreamContext({
|
|
|
170
170
|
});
|
|
171
171
|
```
|
|
172
172
|
|
|
173
|
-
Keep hook
|
|
173
|
+
Keep synchronous hook work cheap. `onAcquire` runs on the `run()` request path for producers and consumers alike, while the remaining hooks run on the producer's streaming hot path. Returned promises are observed for failures but are not awaited.
|
|
174
174
|
|
|
175
175
|
## Resource limits
|
|
176
176
|
|
|
@@ -112,7 +112,12 @@ export default function Page() {
|
|
|
112
112
|
}),
|
|
113
113
|
[],
|
|
114
114
|
);
|
|
115
|
-
const runtime = useChatRuntime({
|
|
115
|
+
const runtime = useChatRuntime({
|
|
116
|
+
transport,
|
|
117
|
+
onResumeError: (error) => {
|
|
118
|
+
console.error("Could not resume the previous response", error);
|
|
119
|
+
},
|
|
120
|
+
});
|
|
116
121
|
|
|
117
122
|
return (
|
|
118
123
|
<AssistantRuntimeProvider runtime={runtime}>
|
|
@@ -122,6 +127,8 @@ export default function Page() {
|
|
|
122
127
|
}
|
|
123
128
|
```
|
|
124
129
|
|
|
130
|
+
`onResumeError` runs when the client finds a stored stream id but the reconnect attempt fails. Use it to show a toast, report telemetry, or mark the thread as needing retry; assistant-ui still clears the stale stream id after the callback runs.
|
|
131
|
+
|
|
125
132
|
`createResumableSessionStorage` returns a `ResumableClientStorage` backed by `window.sessionStorage`. Pass `{ key }` to namespace per route or per chat surface, or supply your own implementation of the three methods (`getStreamId`, `setStreamId`, `clear`). If you are running on a transport that already wraps `fetch` or `prepareReconnectToStreamRequest`, the `resumable` option composes with your existing handlers.
|
|
126
133
|
|
|
127
134
|
The default finish detector scans the SSE body for the AI SDK `"type":"finish"` marker. Override `isFinishEvent` on the `resumable` option when you ship a custom encoder.
|
|
@@ -208,3 +215,7 @@ Both helpers default to the data-stream encoder; pass `encoder: () => new Assist
|
|
|
208
215
|
```sh
|
|
209
216
|
npx assistant-ui create my-app -e with-resumable-stream
|
|
210
217
|
```
|
|
218
|
+
|
|
219
|
+
To drop the resumable chat and resume routes into an existing project (in-memory store only; upgrade via [Storage choices](#storage-choices)):
|
|
220
|
+
|
|
221
|
+
<InstallCommand shadcn={["ai-sdk-backend-resumable"]} />
|
|
@@ -8,11 +8,13 @@ import { SpeechSample } from "@/components/docs/samples/speech";
|
|
|
8
8
|
|
|
9
9
|
assistant-ui supports text-to-speech via the `SpeechSynthesisAdapter` interface. When a speech adapter is configured, users can trigger playback for any assistant message.
|
|
10
10
|
|
|
11
|
+
Speech is the read-aloud mode: one message at a time, text to audio. For a live duplex conversation, see [Realtime Voice](/docs/guides/voice). For push-to-talk input into the composer, see [Dictation](/docs/guides/dictation).
|
|
12
|
+
|
|
11
13
|
<SpeechSample />
|
|
12
14
|
|
|
13
15
|
## SpeechSynthesisAdapter
|
|
14
16
|
|
|
15
|
-
The
|
|
17
|
+
The adapter has a single method. The type lives in `@assistant-ui/core` and is re-exported from `@assistant-ui/react`:
|
|
16
18
|
|
|
17
19
|
```tsx
|
|
18
20
|
import type { SpeechSynthesisAdapter } from "@assistant-ui/react";
|
|
@@ -22,26 +24,30 @@ type SpeechSynthesisAdapter = {
|
|
|
22
24
|
};
|
|
23
25
|
```
|
|
24
26
|
|
|
25
|
-
`speak`
|
|
27
|
+
`speak` receives the plain text of an assistant message and returns an `Utterance`:
|
|
26
28
|
|
|
27
29
|
```tsx
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
30
|
+
namespace SpeechSynthesisAdapter {
|
|
31
|
+
type Status =
|
|
32
|
+
| { type: "starting" | "running" }
|
|
33
|
+
| {
|
|
34
|
+
type: "ended";
|
|
35
|
+
reason: "finished" | "cancelled" | "error";
|
|
36
|
+
error?: unknown;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
type Utterance = {
|
|
40
|
+
status: Status;
|
|
41
|
+
cancel: () => void;
|
|
42
|
+
subscribe: (callback: () => void) => Unsubscribe;
|
|
43
|
+
};
|
|
44
|
+
}
|
|
37
45
|
```
|
|
38
46
|
|
|
39
|
-
Currently the following built-in adapter is available:
|
|
40
|
-
|
|
41
|
-
- `WebSpeechSynthesisAdapter`: uses the browser's `Web Speech API` (`SpeechSynthesis`)
|
|
42
|
-
|
|
43
47
|
## WebSpeechSynthesisAdapter
|
|
44
48
|
|
|
49
|
+
The built-in adapter uses the browser's Web Speech API (`SpeechSynthesis` / `SpeechSynthesisUtterance`):
|
|
50
|
+
|
|
45
51
|
```tsx
|
|
46
52
|
import { WebSpeechSynthesisAdapter } from "@assistant-ui/react";
|
|
47
53
|
|
|
@@ -52,7 +58,9 @@ const runtime = useChatRuntime({
|
|
|
52
58
|
});
|
|
53
59
|
```
|
|
54
60
|
|
|
55
|
-
|
|
61
|
+
When a speech adapter is provided, `capabilities.speech` is set to `true` automatically.
|
|
62
|
+
|
|
63
|
+
## UI: ActionBarPrimitive.Speak
|
|
56
64
|
|
|
57
65
|
The default action bar does not include a speech button. Add `ActionBarPrimitive.Speak` and `ActionBarPrimitive.StopSpeaking` to your assistant message action bar:
|
|
58
66
|
|
|
@@ -79,21 +87,17 @@ const AssistantActionBar = () => {
|
|
|
79
87
|
};
|
|
80
88
|
```
|
|
81
89
|
|
|
82
|
-
`ActionBarPrimitive.Speak` is
|
|
90
|
+
`ActionBarPrimitive.Speak` is disabled when no speech adapter is configured. While playback is active, `message.speech` holds the current speech state so the UI can switch to `StopSpeaking`.
|
|
83
91
|
|
|
84
|
-
## Custom
|
|
92
|
+
## Custom adapters
|
|
85
93
|
|
|
86
|
-
Implement `SpeechSynthesisAdapter` to call any external TTS API
|
|
94
|
+
Implement `SpeechSynthesisAdapter` to call any external TTS provider. Fetch audio from your API, play it with `HTMLAudioElement`, and drive utterance status through `subscribe`:
|
|
87
95
|
|
|
88
96
|
```tsx title="lib/custom-tts-adapter.ts"
|
|
89
97
|
import type { SpeechSynthesisAdapter } from "@assistant-ui/react";
|
|
90
98
|
|
|
91
99
|
export class CustomTTSAdapter implements SpeechSynthesisAdapter {
|
|
92
|
-
private apiUrl: string
|
|
93
|
-
|
|
94
|
-
constructor(options: { apiUrl: string }) {
|
|
95
|
-
this.apiUrl = options.apiUrl;
|
|
96
|
-
}
|
|
100
|
+
constructor(private apiUrl: string) {}
|
|
97
101
|
|
|
98
102
|
speak(text: string): SpeechSynthesisAdapter.Utterance {
|
|
99
103
|
const subscribers = new Set<() => void>();
|
|
@@ -104,7 +108,10 @@ export class CustomTTSAdapter implements SpeechSynthesisAdapter {
|
|
|
104
108
|
for (const cb of subscribers) cb();
|
|
105
109
|
};
|
|
106
110
|
|
|
107
|
-
const finish = (
|
|
111
|
+
const finish = (
|
|
112
|
+
reason: "finished" | "cancelled" | "error",
|
|
113
|
+
error?: unknown,
|
|
114
|
+
) => {
|
|
108
115
|
if (status.type === "ended") return;
|
|
109
116
|
status = { type: "ended", reason, error };
|
|
110
117
|
notify();
|
|
@@ -117,17 +124,20 @@ export class CustomTTSAdapter implements SpeechSynthesisAdapter {
|
|
|
117
124
|
})
|
|
118
125
|
.then((res) => res.blob())
|
|
119
126
|
.then((blob) => {
|
|
127
|
+
if (status.type === "ended") return;
|
|
120
128
|
audio = new Audio(URL.createObjectURL(blob));
|
|
121
129
|
status = { type: "running" };
|
|
122
130
|
notify();
|
|
123
131
|
audio.onended = () => finish("finished");
|
|
124
132
|
audio.onerror = (e) => finish("error", e);
|
|
125
|
-
audio.play();
|
|
133
|
+
audio.play().catch((err) => finish("error", err));
|
|
126
134
|
})
|
|
127
135
|
.catch((err) => finish("error", err));
|
|
128
136
|
|
|
129
137
|
return {
|
|
130
|
-
get status() {
|
|
138
|
+
get status() {
|
|
139
|
+
return status;
|
|
140
|
+
},
|
|
131
141
|
cancel: () => {
|
|
132
142
|
audio?.pause();
|
|
133
143
|
finish("cancelled");
|
|
@@ -141,14 +151,22 @@ export class CustomTTSAdapter implements SpeechSynthesisAdapter {
|
|
|
141
151
|
}
|
|
142
152
|
```
|
|
143
153
|
|
|
144
|
-
Wire it
|
|
154
|
+
Wire it on the same `adapters.speech` slot as the built-in adapter:
|
|
145
155
|
|
|
146
156
|
```tsx
|
|
147
157
|
import { CustomTTSAdapter } from "@/lib/custom-tts-adapter";
|
|
148
158
|
|
|
149
159
|
const runtime = useChatRuntime({
|
|
150
160
|
adapters: {
|
|
151
|
-
speech: new CustomTTSAdapter(
|
|
161
|
+
speech: new CustomTTSAdapter("/api/tts"),
|
|
152
162
|
},
|
|
153
163
|
});
|
|
154
164
|
```
|
|
165
|
+
|
|
166
|
+
Use this shape for any provider TTS (OpenAI, ElevenLabs, cloud speech APIs, and so on). Keep the server route responsible for API keys; the adapter only needs a URL that returns audio bytes.
|
|
167
|
+
|
|
168
|
+
## Related guides
|
|
169
|
+
|
|
170
|
+
- [Realtime Voice](/docs/guides/voice): duplex voice sessions with `RealtimeVoiceAdapter`, `createVoiceSession`, and the voice UI component.
|
|
171
|
+
- [Dictation](/docs/guides/dictation): speech-to-text into the composer with `DictationAdapter` and `ComposerPrimitive.Dictate`.
|
|
172
|
+
- [Speech and Dictation API reference](/docs/api-reference/voice/speech-dictation): generated type docs.
|