@tanstack/ai-octane 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +154 -0
- package/package.json +73 -0
- package/src/index.ts +95 -0
- package/src/realtime-types.ts +146 -0
- package/src/types.ts +305 -0
- package/src/use-audio-recorder.tsrx +118 -0
- package/src/use-audio-recorder.tsrx.d.ts +54 -0
- package/src/use-chat.tsrx +587 -0
- package/src/use-chat.tsrx.d.ts +19 -0
- package/src/use-generate-audio.tsrx +125 -0
- package/src/use-generate-audio.tsrx.d.ts +99 -0
- package/src/use-generate-image.tsrx +127 -0
- package/src/use-generate-image.tsrx.d.ts +101 -0
- package/src/use-generate-speech.tsrx +121 -0
- package/src/use-generate-speech.tsrx.d.ts +95 -0
- package/src/use-generate-video.tsrx +245 -0
- package/src/use-generate-video.tsrx.d.ts +108 -0
- package/src/use-generation.tsrx +229 -0
- package/src/use-generation.tsrx.d.ts +103 -0
- package/src/use-mcp-app-bridge.tsrx +60 -0
- package/src/use-mcp-app-bridge.tsrx.d.ts +31 -0
- package/src/use-realtime-chat.tsrx +307 -0
- package/src/use-realtime-chat.tsrx.d.ts +48 -0
- package/src/use-summarize.tsrx +124 -0
- package/src/use-summarize.tsrx.d.ts +98 -0
- package/src/use-transcription.tsrx +131 -0
- package/src/use-transcription.tsrx.d.ts +104 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Tanner Linsley
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# @tanstack/ai-octane
|
|
2
|
+
|
|
3
|
+
[TanStack AI](https://tanstack.com/ai) bindings for the
|
|
4
|
+
[Octane](https://github.com/octanejs/octane) UI framework.
|
|
5
|
+
|
|
6
|
+
This package ports the `@tanstack/ai-react` hook surface onto Octane while
|
|
7
|
+
reusing `@tanstack/ai` and `@tanstack/ai-client` unchanged. The runtime export
|
|
8
|
+
surface matches the React adapter, so migration starts by changing the package
|
|
9
|
+
import:
|
|
10
|
+
|
|
11
|
+
```ts
|
|
12
|
+
// before
|
|
13
|
+
import { useChat } from '@tanstack/ai-react'
|
|
14
|
+
|
|
15
|
+
// after
|
|
16
|
+
import { useChat } from '@tanstack/ai-octane'
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
The renderer-bearing hook modules are authored as `.tsrx` and compiled by
|
|
20
|
+
Octane. Matching `.tsrx.d.ts` companions are checked declaration emits of those
|
|
21
|
+
implementations, preserving the complete generic surface for TypeScript
|
|
22
|
+
consumers.
|
|
23
|
+
|
|
24
|
+
Like Svelte packages shipping `.svelte`, this package publishes **uncompiled
|
|
25
|
+
source**: your Octane plugin (`octane/compiler/vite`, or the rspack / rspeedy
|
|
26
|
+
equivalents) compiles the `.tsrx` modules as part of your build. There is no
|
|
27
|
+
`dist`.
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pnpm add @tanstack/ai-octane @tanstack/ai @tanstack/ai-client octane
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Usage
|
|
36
|
+
|
|
37
|
+
```tsx
|
|
38
|
+
import { useState } from 'octane'
|
|
39
|
+
import { useChat } from '@tanstack/ai-octane'
|
|
40
|
+
|
|
41
|
+
export function Chat() @{
|
|
42
|
+
const [input, setInput] = useState('')
|
|
43
|
+
const chat = useChat({
|
|
44
|
+
fetcher: myFetcher,
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
<div>
|
|
48
|
+
@for (const message of chat.messages; key message.id) {
|
|
49
|
+
<p>
|
|
50
|
+
{message.role}:
|
|
51
|
+
{message.parts
|
|
52
|
+
.filter((part) => part.type === 'text')
|
|
53
|
+
.map((part) => part.content)
|
|
54
|
+
.join('')}
|
|
55
|
+
</p>
|
|
56
|
+
}
|
|
57
|
+
<input
|
|
58
|
+
value={input}
|
|
59
|
+
onInput={(event) => setInput(event.currentTarget.value)}
|
|
60
|
+
/>
|
|
61
|
+
<button
|
|
62
|
+
onClick={() => {
|
|
63
|
+
void chat.sendMessage(input)
|
|
64
|
+
setInput('')
|
|
65
|
+
}}
|
|
66
|
+
>
|
|
67
|
+
Send
|
|
68
|
+
</button>
|
|
69
|
+
</div>
|
|
70
|
+
}
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
`useChat` has no input state of its own — hold the text box value in a local
|
|
74
|
+
`useState` and pass it to `sendMessage`. Note the `onInput` handler: Octane
|
|
75
|
+
drives text controls per keystroke through the native `input` event, not a
|
|
76
|
+
synthetic `onChange`.
|
|
77
|
+
|
|
78
|
+
## API
|
|
79
|
+
|
|
80
|
+
The adapter includes `useChat`, `useRealtimeChat`, `useMcpAppBridge`,
|
|
81
|
+
`useGeneration`, `useGenerateImage`, `useGenerateAudio`, `useGenerateSpeech`,
|
|
82
|
+
`useGenerateVideo`, `useTranscription`, `useSummarize`, and
|
|
83
|
+
`useAudioRecorder`. It also re-exports all 30 `@tanstack/ai-client`
|
|
84
|
+
convenience helpers and types (`fetchServerSentEvents`, `fetchHttpStream`,
|
|
85
|
+
`xhrServerSentEvents`, `xhrHttpStream`, `stream`, `rpcStream`,
|
|
86
|
+
`createChatClientOptions`, `createMcpAppBridge`, and their associated types)
|
|
87
|
+
unchanged, mirroring the `@tanstack/ai-react` index.
|
|
88
|
+
|
|
89
|
+
Server rendering through `octane/server` is supported. `useChat` renders its
|
|
90
|
+
initial message snapshot without browser-only setup.
|
|
91
|
+
|
|
92
|
+
## Divergences from `@tanstack/ai-react`
|
|
93
|
+
|
|
94
|
+
- The `./mcp-apps` subpath and its `MCPAppResource` component are not ported:
|
|
95
|
+
they render `AppRenderer` from the React-only `@mcp-ui/client`, which has no
|
|
96
|
+
Octane equivalent. The framework-agnostic `useMcpAppBridge` hook is ported
|
|
97
|
+
and available on the main entry.
|
|
98
|
+
- Octane uses native events: text/file/recorder inputs drive updates via
|
|
99
|
+
`onInput`; there is no synthetic `onChange` layer.
|
|
100
|
+
- Octane has no StrictMode double-invoke and always provides `useId`, so no
|
|
101
|
+
random-id fallback is needed.
|
|
102
|
+
- The devtools bridge is tagged `framework: 'octane'` (upstream sends
|
|
103
|
+
`'react'`), so the devtools identify this binding correctly.
|
|
104
|
+
- Realtime reconnects and token refreshes use the latest `getToken` and adapter
|
|
105
|
+
supplied to the hook; upstream captures the first render's callbacks.
|
|
106
|
+
- The declared realtime `onStatusChange` callback is invoked alongside the
|
|
107
|
+
hook's state update; upstream `@tanstack/ai-react` currently drops the
|
|
108
|
+
external callback.
|
|
109
|
+
- Changing `useChat`'s connection or fetcher updates the active `ChatClient` in
|
|
110
|
+
place and preserves conversation state.
|
|
111
|
+
|
|
112
|
+
### Fixed here, still present upstream
|
|
113
|
+
|
|
114
|
+
Three defects were found during review of the port and fixed rather than
|
|
115
|
+
mirrored. All are documented in [`status.json`](./status.json) and covered by
|
|
116
|
+
tests, and each is tracked upstream so the other adapters can catch up.
|
|
117
|
+
|
|
118
|
+
- `useAudioRecorder`'s transforming overload requires `onComplete`. Upstream,
|
|
119
|
+
passing any unrelated option (`useAudioRecorder({ onError })`) matched the
|
|
120
|
+
transforming overload, inferred `TOnComplete` as `unknown`, and silently
|
|
121
|
+
collapsed `recording` and `stop()` to `unknown`.
|
|
122
|
+
([#1001](https://github.com/TanStack/ai/issues/1001))
|
|
123
|
+
- `useGeneration` spreads caller `devtools` metadata _before_ the hardcoded
|
|
124
|
+
`framework`/`hookName`, so a caller can't misattribute the binding in the
|
|
125
|
+
devtools. `ai-react` spreads it after; `ai-vue` and `ai-solid` already order it
|
|
126
|
+
this way. ([#1002](https://github.com/TanStack/ai/issues/1002))
|
|
127
|
+
- `UseGenerationReturn<TInput, TOutput>` types `generate` as `(input: TInput)`.
|
|
128
|
+
Upstream declares `UseGenerationReturn<TOutput>` and widens `generate` to
|
|
129
|
+
`(input: Record<string, any>)`, so narrow or required input fields go
|
|
130
|
+
unchecked. **This is the one place the public type surface differs in shape
|
|
131
|
+
from `@tanstack/ai-react`** — the runtime surface is unchanged, so the
|
|
132
|
+
"change the import" migration still holds.
|
|
133
|
+
([#1003](https://github.com/TanStack/ai/issues/1003))
|
|
134
|
+
|
|
135
|
+
## Status
|
|
136
|
+
|
|
137
|
+
This binding was developed as `@octanejs/tanstack-ai` in the
|
|
138
|
+
[octanejs/octane](https://github.com/octanejs/octane) repo as a temporary
|
|
139
|
+
stopgap, and moved here — apart from the rename, the only changes are the three
|
|
140
|
+
fixes listed above and the test-helper hardening noted in `status.json`.
|
|
141
|
+
|
|
142
|
+
Current scope, divergences, and verification state are tracked in
|
|
143
|
+
[`status.json`](./status.json). `useChat` matches the current ChatClient shape
|
|
144
|
+
(`threadId` identity, queue, interrupts, `attach`/`detach`).
|
|
145
|
+
|
|
146
|
+
The port runs TanStack AI's React adapter tests against Octane across all eleven
|
|
147
|
+
hooks, with no skipped, todo, or expected-failure cases (except the untestable
|
|
148
|
+
auto-resume case noted in `status.json`). An SSR fixture and the upstream
|
|
149
|
+
compile-time type tests are also included.
|
|
150
|
+
|
|
151
|
+
## License
|
|
152
|
+
|
|
153
|
+
MIT — contains source derived from
|
|
154
|
+
[TanStack AI](https://github.com/TanStack/ai) (MIT), adapted for Octane.
|
package/package.json
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@tanstack/ai-octane",
|
|
3
|
+
"version": "0.0.0",
|
|
4
|
+
"description": "Octane bindings for TanStack AI streaming chat, structured outputs, and media generation.",
|
|
5
|
+
"author": "Dominic Gannaway",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"homepage": "https://tanstack.com/ai",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/TanStack/ai.git",
|
|
11
|
+
"directory": "packages/ai-octane"
|
|
12
|
+
},
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/TanStack/ai/issues"
|
|
15
|
+
},
|
|
16
|
+
"funding": {
|
|
17
|
+
"type": "github",
|
|
18
|
+
"url": "https://github.com/sponsors/tannerlinsley"
|
|
19
|
+
},
|
|
20
|
+
"type": "module",
|
|
21
|
+
"engines": {
|
|
22
|
+
"node": ">=22"
|
|
23
|
+
},
|
|
24
|
+
"keywords": [
|
|
25
|
+
"ai",
|
|
26
|
+
"ai-sdk",
|
|
27
|
+
"typescript",
|
|
28
|
+
"tanstack",
|
|
29
|
+
"octane",
|
|
30
|
+
"chat",
|
|
31
|
+
"streaming",
|
|
32
|
+
"tool-calling",
|
|
33
|
+
"structured-outputs",
|
|
34
|
+
"media-generation"
|
|
35
|
+
],
|
|
36
|
+
"//": "Like Svelte packages shipping .svelte, this package publishes uncompiled source: the hook modules are .tsrx and are compiled by the consumer's Octane plugin. There is therefore no build/dist and no publint test:build target. The .tsrx.d.ts companions are checked declaration emits, so `tsc` still type-checks the full public surface.",
|
|
37
|
+
"main": "./src/index.ts",
|
|
38
|
+
"module": "./src/index.ts",
|
|
39
|
+
"types": "./src/index.ts",
|
|
40
|
+
"exports": {
|
|
41
|
+
".": "./src/index.ts"
|
|
42
|
+
},
|
|
43
|
+
"files": [
|
|
44
|
+
"src",
|
|
45
|
+
"README.md"
|
|
46
|
+
],
|
|
47
|
+
"//lint": "The .tsrx.d.ts companions are generated declaration emits of the .tsrx implementations, so they are not hand-formatted and are excluded from lint rather than edited in place (a regeneration would undo any fix).",
|
|
48
|
+
"dependencies": {
|
|
49
|
+
"@tanstack/ai-client": "0.25.1"
|
|
50
|
+
},
|
|
51
|
+
"peerDependencies": {
|
|
52
|
+
"octane": "^0.1.17",
|
|
53
|
+
"@tanstack/ai": "^0.47.2"
|
|
54
|
+
},
|
|
55
|
+
"devDependencies": {
|
|
56
|
+
"@octanejs/testing-library": "0.1.14",
|
|
57
|
+
"@standard-schema/spec": "^1.1.0",
|
|
58
|
+
"@types/node": "^24.10.1",
|
|
59
|
+
"@vitest/coverage-v8": "4.1.10",
|
|
60
|
+
"happy-dom": "^20.11.2",
|
|
61
|
+
"octane": "0.1.17",
|
|
62
|
+
"vite": "^8.2.1",
|
|
63
|
+
"zod": "^4.2.0",
|
|
64
|
+
"@tanstack/ai": "0.47.2"
|
|
65
|
+
},
|
|
66
|
+
"scripts": {
|
|
67
|
+
"lint:fix": "oxlint src --type-aware --ignore-pattern '**/*.tsrx.d.ts' --fix",
|
|
68
|
+
"test:oxlint": "oxlint src --type-aware --ignore-pattern '**/*.tsrx.d.ts'",
|
|
69
|
+
"test:lib": "vitest run",
|
|
70
|
+
"test:lib:dev": "pnpm test:lib --watch",
|
|
71
|
+
"test:types": "tsc && tsc -p typetests/tsconfig.json"
|
|
72
|
+
}
|
|
73
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
export { useChat } from './use-chat.tsrx'
|
|
2
|
+
export { useRealtimeChat } from './use-realtime-chat.tsrx'
|
|
3
|
+
export { useMcpAppBridge } from './use-mcp-app-bridge.tsrx'
|
|
4
|
+
export type { UseMcpAppBridgeOptions } from './use-mcp-app-bridge.tsrx'
|
|
5
|
+
export type {
|
|
6
|
+
DeepPartial,
|
|
7
|
+
UseChatOptions,
|
|
8
|
+
UseChatReturn,
|
|
9
|
+
UIMessage,
|
|
10
|
+
ChatRequestBody,
|
|
11
|
+
QueuedMessage,
|
|
12
|
+
SendMessageOptions,
|
|
13
|
+
WhenBusy,
|
|
14
|
+
QueueConfig,
|
|
15
|
+
QueueStrategy,
|
|
16
|
+
QueueOption,
|
|
17
|
+
} from './types'
|
|
18
|
+
export type {
|
|
19
|
+
UseRealtimeChatOptions,
|
|
20
|
+
UseRealtimeChatReturn,
|
|
21
|
+
} from './realtime-types'
|
|
22
|
+
|
|
23
|
+
export { useGeneration } from './use-generation.tsrx'
|
|
24
|
+
export type {
|
|
25
|
+
UseGenerationOptions,
|
|
26
|
+
UseGenerationReturn,
|
|
27
|
+
} from './use-generation.tsrx'
|
|
28
|
+
export { useGenerateImage } from './use-generate-image.tsrx'
|
|
29
|
+
export type {
|
|
30
|
+
UseGenerateImageOptions,
|
|
31
|
+
UseGenerateImageReturn,
|
|
32
|
+
} from './use-generate-image.tsrx'
|
|
33
|
+
export { useGenerateAudio } from './use-generate-audio.tsrx'
|
|
34
|
+
export type {
|
|
35
|
+
UseGenerateAudioOptions,
|
|
36
|
+
UseGenerateAudioReturn,
|
|
37
|
+
} from './use-generate-audio.tsrx'
|
|
38
|
+
export { useGenerateSpeech } from './use-generate-speech.tsrx'
|
|
39
|
+
export type {
|
|
40
|
+
UseGenerateSpeechOptions,
|
|
41
|
+
UseGenerateSpeechReturn,
|
|
42
|
+
} from './use-generate-speech.tsrx'
|
|
43
|
+
export { useTranscription } from './use-transcription.tsrx'
|
|
44
|
+
export type {
|
|
45
|
+
UseTranscriptionOptions,
|
|
46
|
+
UseTranscriptionReturn,
|
|
47
|
+
} from './use-transcription.tsrx'
|
|
48
|
+
export { useSummarize } from './use-summarize.tsrx'
|
|
49
|
+
export type {
|
|
50
|
+
UseSummarizeOptions,
|
|
51
|
+
UseSummarizeReturn,
|
|
52
|
+
} from './use-summarize.tsrx'
|
|
53
|
+
export { useGenerateVideo } from './use-generate-video.tsrx'
|
|
54
|
+
export type {
|
|
55
|
+
UseGenerateVideoOptions,
|
|
56
|
+
UseGenerateVideoReturn,
|
|
57
|
+
} from './use-generate-video.tsrx'
|
|
58
|
+
export { useAudioRecorder } from './use-audio-recorder.tsrx'
|
|
59
|
+
export type {
|
|
60
|
+
UseAudioRecorderOptions,
|
|
61
|
+
UseAudioRecorderReturn,
|
|
62
|
+
} from './use-audio-recorder.tsrx'
|
|
63
|
+
|
|
64
|
+
// Re-export from ai-client for convenience (mirror upstream index.ts)
|
|
65
|
+
export {
|
|
66
|
+
fetchServerSentEvents,
|
|
67
|
+
fetchHttpStream,
|
|
68
|
+
xhrServerSentEvents,
|
|
69
|
+
xhrHttpStream,
|
|
70
|
+
stream,
|
|
71
|
+
rpcStream,
|
|
72
|
+
createChatClientOptions,
|
|
73
|
+
createMcpAppBridge,
|
|
74
|
+
type McpAppBridge,
|
|
75
|
+
type CreateMcpAppBridgeOptions,
|
|
76
|
+
type ChatFetcher,
|
|
77
|
+
type ChatFetcherInput,
|
|
78
|
+
type ChatFetcherOptions,
|
|
79
|
+
type ConnectionAdapter,
|
|
80
|
+
type ConnectConnectionAdapter,
|
|
81
|
+
type SubscribeConnectionAdapter,
|
|
82
|
+
type RunAgentInputContext,
|
|
83
|
+
type FetchConnectionOptions,
|
|
84
|
+
type XhrConnectionOptions,
|
|
85
|
+
type InferChatMessages,
|
|
86
|
+
type GenerationClientState,
|
|
87
|
+
type ImageGenerateInput,
|
|
88
|
+
type AudioGenerateInput,
|
|
89
|
+
type SpeechGenerateInput,
|
|
90
|
+
type TranscriptionGenerateInput,
|
|
91
|
+
type SummarizeGenerateInput,
|
|
92
|
+
type VideoGenerateInput,
|
|
93
|
+
type VideoGenerateResult,
|
|
94
|
+
type VideoStatusInfo,
|
|
95
|
+
} from '@tanstack/ai-client'
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
AnyClientTool,
|
|
3
|
+
RealtimeMessage,
|
|
4
|
+
RealtimeMode,
|
|
5
|
+
RealtimeSessionConfig,
|
|
6
|
+
RealtimeStatus,
|
|
7
|
+
RealtimeToken,
|
|
8
|
+
UsageInfo,
|
|
9
|
+
} from '@tanstack/ai'
|
|
10
|
+
import type { RealtimeAdapter } from '@tanstack/ai-client'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Options for the useRealtimeChat hook.
|
|
14
|
+
*/
|
|
15
|
+
export interface UseRealtimeChatOptions {
|
|
16
|
+
/**
|
|
17
|
+
* Function to fetch a realtime token from the server.
|
|
18
|
+
* Called on connect and when token needs refresh.
|
|
19
|
+
*/
|
|
20
|
+
getToken: () => Promise<RealtimeToken>
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The realtime adapter to use (e.g., openaiRealtime())
|
|
24
|
+
*/
|
|
25
|
+
adapter: RealtimeAdapter
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Client-side tools with execution logic
|
|
29
|
+
*/
|
|
30
|
+
tools?: ReadonlyArray<AnyClientTool>
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Auto-play assistant audio (default: true)
|
|
34
|
+
*/
|
|
35
|
+
autoPlayback?: boolean
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Request microphone access on connect (default: true)
|
|
39
|
+
*/
|
|
40
|
+
autoCapture?: boolean
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* System instructions for the assistant
|
|
44
|
+
*/
|
|
45
|
+
instructions?: string
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Voice to use for audio output
|
|
49
|
+
*/
|
|
50
|
+
voice?: string
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Voice activity detection mode (default: 'server')
|
|
54
|
+
*/
|
|
55
|
+
vadMode?: 'server' | 'semantic' | 'manual'
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Output modalities for responses (e.g., ['audio', 'text'])
|
|
59
|
+
*/
|
|
60
|
+
outputModalities?: Array<'audio' | 'text'>
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Temperature for generation (provider-specific range)
|
|
64
|
+
*/
|
|
65
|
+
temperature?: number
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Maximum number of tokens in a response
|
|
69
|
+
*/
|
|
70
|
+
maxOutputTokens?: number | 'inf'
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Eagerness level for semantic VAD ('low', 'medium', 'high')
|
|
74
|
+
*/
|
|
75
|
+
semanticEagerness?: 'low' | 'medium' | 'high'
|
|
76
|
+
|
|
77
|
+
// Callbacks
|
|
78
|
+
onConnect?: () => void
|
|
79
|
+
onDisconnect?: () => void
|
|
80
|
+
onError?: (error: Error) => void
|
|
81
|
+
onMessage?: (message: RealtimeMessage) => void
|
|
82
|
+
onModeChange?: (mode: RealtimeMode) => void
|
|
83
|
+
onInterrupted?: () => void
|
|
84
|
+
onUsage?: (usage: UsageInfo) => void
|
|
85
|
+
onGoAway?: (timeLeft?: string) => void
|
|
86
|
+
onStatusChange?: (status: RealtimeStatus) => void
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Return type for the useRealtimeChat hook.
|
|
91
|
+
*/
|
|
92
|
+
export interface UseRealtimeChatReturn {
|
|
93
|
+
// Connection state
|
|
94
|
+
/** Current connection status */
|
|
95
|
+
status: RealtimeStatus
|
|
96
|
+
/** Current error, if any */
|
|
97
|
+
error: Error | null
|
|
98
|
+
/** Connect to the realtime session */
|
|
99
|
+
connect: () => Promise<void>
|
|
100
|
+
/** Disconnect from the realtime session */
|
|
101
|
+
disconnect: () => Promise<void>
|
|
102
|
+
|
|
103
|
+
// Conversation state
|
|
104
|
+
/** Current mode (idle, listening, thinking, speaking) */
|
|
105
|
+
mode: RealtimeMode
|
|
106
|
+
/** Conversation messages */
|
|
107
|
+
messages: Array<RealtimeMessage>
|
|
108
|
+
/** User transcript while speaking (before finalized) */
|
|
109
|
+
pendingUserTranscript: string | null
|
|
110
|
+
/** Assistant transcript while speaking (before finalized) */
|
|
111
|
+
pendingAssistantTranscript: string | null
|
|
112
|
+
|
|
113
|
+
// Voice control
|
|
114
|
+
/** Start listening for voice input (manual VAD mode) */
|
|
115
|
+
startListening: () => void
|
|
116
|
+
/** Stop listening for voice input (manual VAD mode) */
|
|
117
|
+
stopListening: () => void
|
|
118
|
+
/** Interrupt the current assistant response */
|
|
119
|
+
interrupt: () => void
|
|
120
|
+
|
|
121
|
+
// Text input
|
|
122
|
+
/** Send a text message instead of voice */
|
|
123
|
+
sendText: (text: string) => void
|
|
124
|
+
|
|
125
|
+
// Image input
|
|
126
|
+
/** Send an image to the conversation */
|
|
127
|
+
sendImage: (imageData: string, mimeType: string) => void
|
|
128
|
+
|
|
129
|
+
// Audio visualization (0-1 normalized)
|
|
130
|
+
/** Current input (microphone) volume level */
|
|
131
|
+
inputLevel: number
|
|
132
|
+
/** Current output (speaker) volume level */
|
|
133
|
+
outputLevel: number
|
|
134
|
+
/** Get frequency data for input audio visualization */
|
|
135
|
+
getInputFrequencyData: () => Uint8Array
|
|
136
|
+
/** Get frequency data for output audio visualization */
|
|
137
|
+
getOutputFrequencyData: () => Uint8Array
|
|
138
|
+
/** Get time domain data for input waveform */
|
|
139
|
+
getInputTimeDomainData: () => Uint8Array
|
|
140
|
+
/** Get time domain data for output waveform */
|
|
141
|
+
getOutputTimeDomainData: () => Uint8Array
|
|
142
|
+
|
|
143
|
+
// Session control
|
|
144
|
+
/** Update the active session and persist the configuration for reconnects. */
|
|
145
|
+
updateSession: (config: RealtimeSessionConfig) => void
|
|
146
|
+
}
|