@jskit-ai/agent-docs 0.1.194 → 0.1.196
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/guide/agent/app-extras/assistant-conversation.md +2608 -72
- package/guide/agent/app-extras/assistant-voice.md +244 -0
- package/guide/agent/app-extras/assistant.md +189 -20
- package/guide/agent/app-extras/realtime.md +6 -0
- package/guide/agent/index.md +1 -0
- package/package.json +1 -1
- package/patterns/feature-package/example/booking-engine/package.json +1 -1
- package/patterns/minimal-foundation/example/package.json +4 -4
- package/patterns/shell-foundation/example/package.json +5 -5
- package/patterns/shell-foundation/example/packages/main/package.json +1 -1
- package/reference/autogen/PATTERN_INDEX.md +35 -35
- package/reference/autogen/README.md +2 -0
- package/reference/autogen/packages/assistant-core.md +974 -9
- package/reference/autogen/packages/assistant-runtime.md +126 -28
- package/reference/autogen/packages/assistant-voice-sherpa.md +82 -0
- package/reference/autogen/packages/assistant-voice.md +227 -0
- package/reference/autogen/packages/kernel.md +10 -0
- package/reference/autogen/packages/realtime.md +5 -2
- package/skills/jskit/references/pattern-index.md +35 -35
- package/skills/jskit/references/patterns/app/minimal-foundation/example/package.json +4 -4
- package/skills/jskit/references/patterns/app/shell-foundation/example/package.json +5 -5
- package/skills/jskit/references/patterns/app/shell-foundation/example/packages/main/package.json +1 -1
- package/skills/jskit/references/patterns/auth/supabase-auth/example/package.json +1 -1
- package/skills/jskit/references/patterns/connectors/calendar-cli/example/package.json +4 -4
- package/skills/jskit/references/patterns/crud/json-api-resource-package/example/packages/books/package.json +2 -2
- package/skills/jskit/references/patterns/database/mysql-application/example/package.json +1 -1
- package/skills/jskit/references/patterns/database/postgres-application/example/package.json +1 -1
- package/skills/jskit/references/patterns/realtime/realtime-application/example/package.json +2 -2
- package/skills/jskit/references/patterns/server/feature-package/example/booking-engine/package.json +1 -1
- package/skills/jskit/references/patterns/users/user-administration-server/example/packages/users/package.json +2 -2
- package/skills/jskit/references/patterns/users/user-administration-server/example/packages/users-workspace/package.json +3 -3
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
<!-- Generated by `npm run agent-docs:build` from `packages/agent-docs/site/guide/app-extras/assistant-voice.md`. Do not edit manually. -->
|
|
2
|
+
|
|
3
|
+
# Voice conversations
|
|
4
|
+
|
|
5
|
+
`@jskit-ai/assistant-voice` adds push-to-talk, hands-free conversation, captions,
|
|
6
|
+
transcript review, streamed playback and interruption to an application's existing
|
|
7
|
+
conversations. One controller and one modal live above the routed screens. The
|
|
8
|
+
application keeps its agent, history, message admission and authorization.
|
|
9
|
+
|
|
10
|
+
The optional `@jskit-ai/assistant-voice-sherpa` package supplies native speech
|
|
11
|
+
recognition, synthesis and explicit model preparation. Installing the browser
|
|
12
|
+
package does not install native inference or download models.
|
|
13
|
+
|
|
14
|
+
## Application integration
|
|
15
|
+
|
|
16
|
+
Install `@jskit-ai/assistant-voice` in a Vue 3/Vuetify application. Mount
|
|
17
|
+
`VoiceConversationHost` once inside the root `v-app`, and create its controller
|
|
18
|
+
in that root's setup scope:
|
|
19
|
+
|
|
20
|
+
```js
|
|
21
|
+
import { onScopeDispose } from "vue";
|
|
22
|
+
import { createVoiceConversationController, VoiceConversationHost }
|
|
23
|
+
from "@jskit-ai/assistant-voice/client";
|
|
24
|
+
|
|
25
|
+
const voice = createVoiceConversationController({
|
|
26
|
+
connectSpeech: binding => `/api/conversations/${encodeURIComponent(binding.id)}/voice`
|
|
27
|
+
});
|
|
28
|
+
onScopeDispose(() => { void voice.dispose(); });
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
```vue
|
|
32
|
+
<VoiceConversationHost :controller="voice" />
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
For the ordinary setup, use the [standard Assistant integration](./assistant.md).
|
|
36
|
+
`AssistantConversationClientElement` supplies the same retained `runtime` to its
|
|
37
|
+
`composer-tools` slot. The five-chat example in
|
|
38
|
+
`@jskit-ai/assistant-voice/examples/conversation` retains that runtime for voice,
|
|
39
|
+
projects its canonical turns with `projectConversationVoiceState`, and forwards
|
|
40
|
+
text and Stop through `runtime.send()` and `runtime.cancel()`. The application
|
|
41
|
+
supplies the speech URL, target label and text-view navigation; it does not build
|
|
42
|
+
another transcript, delivery ledger or realtime subscription.
|
|
43
|
+
|
|
44
|
+
### Custom conversation binding
|
|
45
|
+
|
|
46
|
+
Applications with an existing authorized conversation reader can instead supply
|
|
47
|
+
this binding contract to the same voice controller:
|
|
48
|
+
|
|
49
|
+
```js
|
|
50
|
+
await voice.open({ conversation: {
|
|
51
|
+
id: conversationId,
|
|
52
|
+
label: "Planning",
|
|
53
|
+
get state() { return store.conversations[conversationId]; },
|
|
54
|
+
get available() { return store.canUse(conversationId); },
|
|
55
|
+
captureContext: () => ({ conversationId }),
|
|
56
|
+
submitText: (text, { messageId, context }) =>
|
|
57
|
+
store.send(context.conversationId, { text, messageId }),
|
|
58
|
+
cancelWork: () => store.stop(conversationId)
|
|
59
|
+
} });
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
The store and its operations are application-owned. `state` exposes `messages`,
|
|
63
|
+
an optional `streamingReply`, and `status: "working"` while the agent is running.
|
|
64
|
+
Each message has a stable `id`, `role` and `text`. A streaming answer and its saved
|
|
65
|
+
replacement use the same ID. Supply user and assistant content; exclude tool
|
|
66
|
+
output, reasoning and private operational details.
|
|
67
|
+
|
|
68
|
+
`submitText` resolves when the application's normal server operation accepts the
|
|
69
|
+
message. Reject failures and deduplicate retries by `messageId` in that operation.
|
|
70
|
+
Voice preserves the captured destination and message identity; it does not create
|
|
71
|
+
another history or infer whether an uncertain server request was accepted.
|
|
72
|
+
|
|
73
|
+
Optional `openText()` opens the application's text conversation. It adds Talk and
|
|
74
|
+
Text controls at the top of the modal. Selecting Text calls that operation and
|
|
75
|
+
minimizes voice while capture and playback retain their current state. An
|
|
76
|
+
application can open this modal as its default conversation view; opening never
|
|
77
|
+
starts the microphone.
|
|
78
|
+
|
|
79
|
+
Optional synchronous `retain()` and `release()` keep a conversation reader alive
|
|
80
|
+
outside its screen. A binding's `available` value must reflect access loss.
|
|
81
|
+
Include the current identity scope in the binding ID when IDs can overlap between
|
|
82
|
+
people; dispose the controller on identity replacement. Server authorization is
|
|
83
|
+
still required for every message and speech connection.
|
|
84
|
+
|
|
85
|
+
## Interaction ownership
|
|
86
|
+
|
|
87
|
+
Opening voice shows its controls without acquiring the microphone. The single
|
|
88
|
+
Talk button connects automatically: tap for hands-free, hold for push-to-talk,
|
|
89
|
+
and release to send. Tap Pause to pause hands-free capture. Hands-free can listen
|
|
90
|
+
while answers play. The browser
|
|
91
|
+
requires microphone permission and a secure origin, including recognized localhost
|
|
92
|
+
origins for development.
|
|
93
|
+
|
|
94
|
+
Opening the same binding ID reveals the existing modal. Minimizing or changing
|
|
95
|
+
the visible screen preserves its destination. Opening another target first
|
|
96
|
+
releases the old audio. Unfinished words must be sent or explicitly discarded
|
|
97
|
+
before switching. A send in progress cannot be discarded to switch targets.
|
|
98
|
+
|
|
99
|
+
Stop speaking cancels playback and queued speech. Stop agent work calls the
|
|
100
|
+
binding's separate `cancelWork` operation. `end()` releases voice after pending
|
|
101
|
+
words are resolved; `dispose()` always releases it. Reconnection does not replay
|
|
102
|
+
messages, resend captured audio or automatically restart the microphone.
|
|
103
|
+
|
|
104
|
+
Bindings may supply `defaults.mode` (`push-to-talk` or `hands-free`),
|
|
105
|
+
`defaults.reviewBeforeSend`, and `defaults.voiceId`. Recordings started through
|
|
106
|
+
`startHeldRecording()` require review; the modal's push-to-talk gesture uses
|
|
107
|
+
`reviewBeforeSend`. The Talk/Pause microphone icon and speaker toggle show their states. The
|
|
108
|
+
Voice settings cog loads the service's installed voices without recording.
|
|
109
|
+
Pause finalizes the current utterance before muting, including buffered audio,
|
|
110
|
+
and applies the recording's review preference. Resume starts a fresh recording.
|
|
111
|
+
The review panel stays hidden during automatic admission; failed admission keeps
|
|
112
|
+
the words and message identity available for review and retry.
|
|
113
|
+
The portrait and controls stay fixed while captions scroll. A voice selection affects the next reply; streamed
|
|
114
|
+
phrases of a reply retain the original speaker. Voice does not change the agent's
|
|
115
|
+
reasoning model.
|
|
116
|
+
|
|
117
|
+
`ConversationDialog` is the shared Vue frame for voice and application Text views.
|
|
118
|
+
It fixes header/tab geometry, uses full screen below 600 pixels, and switches
|
|
119
|
+
without content or backdrop transitions. It accepts `title`, `mode`, `showModes`, `minimizable` and
|
|
120
|
+
`closeLabel`; `update:modelValue`, `update:mode` and `minimize` report actions.
|
|
121
|
+
The host's X calls `end({ discard: true })`; minus preserves capture/playback.
|
|
122
|
+
Both `VoiceConversationHost` and `ConversationDialog` accept an optional
|
|
123
|
+
`activator` DOM element. A click over that launcher while the dialog is open
|
|
124
|
+
minimizes it; the backdrop cannot turn that gesture into an audio shutdown.
|
|
125
|
+
An application can replace the host's default floating reopen control through
|
|
126
|
+
the `reopen` slot and use `controller.reveal()` from its own header indicator.
|
|
127
|
+
|
|
128
|
+
The host's `avatar` slot receives `state`, `mouthLevel`, `mouthPose`, `avatar` and
|
|
129
|
+
`binding`. `VoiceAvatar` and `useVoiceAvatar` supply generic presentation; an
|
|
130
|
+
application may provide artwork without acquiring audio ownership.
|
|
131
|
+
|
|
132
|
+
## Authorized speech connection
|
|
133
|
+
|
|
134
|
+
Register the WebSocket plugin on the application's Fastify server, then expose a
|
|
135
|
+
proxy route through `@jskit-ai/assistant-voice/server`:
|
|
136
|
+
|
|
137
|
+
```js
|
|
138
|
+
registerVoiceProxyRoute(fastify, {
|
|
139
|
+
route: "/api/conversations/:id/voice",
|
|
140
|
+
proxyConfig: resolveVoiceProxyConfig({ endpoint, accessTokenFile }),
|
|
141
|
+
async authorize(request) {
|
|
142
|
+
await checkOrigin(request);
|
|
143
|
+
await requireConversationAccess(request.user, request.params.id);
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Import `registerVoiceProxyRoute` and `resolveVoiceProxyConfig` from that server
|
|
149
|
+
entrypoint. The endpoint and protected token file are operator inputs. The
|
|
150
|
+
application supplies the illustrated origin and conversation checks; authorization
|
|
151
|
+
runs before connecting upstream. No service key or token belongs in browser code.
|
|
152
|
+
A speech grant does not authorize access to a conversation or execution of tools.
|
|
153
|
+
|
|
154
|
+
Register the WebSocket plugin before starting the supplied realtime provider.
|
|
155
|
+
The existing realtime attachment hands only `/socket.io` WebSocket upgrades to
|
|
156
|
+
Socket.IO, so Fastify does not close that connection as an unknown speech route.
|
|
157
|
+
Other upgrades remain with their application routes and authorization. The
|
|
158
|
+
application does not need a separate upgrade handler.
|
|
159
|
+
|
|
160
|
+
`createVoiceDaemon({ engine, accessKey, ... })` supplies the bounded service.
|
|
161
|
+
The engine owns recognition and asynchronous synthesis. A host may supply its
|
|
162
|
+
own `authorize(token)` and `handleRequest(request, response)` grant policy.
|
|
163
|
+
Principals contain `tenant` and optional `parentTenant`; connection limits apply
|
|
164
|
+
to both. `revokeTenant(id)` terminates existing sessions and queued work. The
|
|
165
|
+
host's authorizer must also reject subsequent use of a revoked grant.
|
|
166
|
+
|
|
167
|
+
## Native service and configurable voices
|
|
168
|
+
|
|
169
|
+
Install `@jskit-ai/assistant-voice-sherpa` where the service will run. Model
|
|
170
|
+
preparation requires Bash, tar with bzip2, sha256sum, Node and curl unless the
|
|
171
|
+
archives are cached. Models have separate upstream licenses and are not bundled
|
|
172
|
+
in npm. Preparation verifies pinned archives and installed files.
|
|
173
|
+
|
|
174
|
+
```sh
|
|
175
|
+
npx jskit-assistant-voice prepare --models-root /srv/speech/models --pack cori
|
|
176
|
+
npx jskit-assistant-voice verify --models-root /srv/speech/models
|
|
177
|
+
npx jskit-assistant-voice serve --models-root /srv/speech/models --key-file /run/secrets/speech-key
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
The signing key is an operator-created random secret of at least 32 bytes in a
|
|
181
|
+
protected file. `jskit-assistant-voice token --key-file PATH --tenant NAME` emits
|
|
182
|
+
an application token; redirect it into a separate protected file. The standalone
|
|
183
|
+
token command does not implement a revocation store. Service defaults are
|
|
184
|
+
`127.0.0.1:3092`, `/v1/voice` and readiness at `/health`.
|
|
185
|
+
|
|
186
|
+
The `cori` preparation preset supplies one English voice. `piper` adds Alba
|
|
187
|
+
(female, Scottish English), Joe and Bryce (male, American English) and
|
|
188
|
+
Northern English (male, British English), retaining
|
|
189
|
+
Cori (female, British English) as the default. These are separate Piper medium
|
|
190
|
+
models; one synthesis worker retains only the selected model. Changing models
|
|
191
|
+
releases the previous worker and its native caches before loading the next, while
|
|
192
|
+
recognition and the connection stay running. The first reply after a change
|
|
193
|
+
includes loading time. Extra choices require disk space, not resident model RAM.
|
|
194
|
+
`kokoro` supplies Heart, Bella,
|
|
195
|
+
Emma, Michael, Adam, George and Daniel. All presets use the English streaming
|
|
196
|
+
Zipformer recognizer. Operators can supply
|
|
197
|
+
`prepare --sources-file FILE` for pinned model archives and `serve --config FILE`
|
|
198
|
+
for native recognition/synthesis settings, voice IDs, labels and speaker IDs.
|
|
199
|
+
`recognitionVocabulary` maps literal phrases to application spellings, for example
|
|
200
|
+
`{ "acme labs": "Acme Labs" }`. It applies when normalizing uppercase recognition;
|
|
201
|
+
mixed-case input remains intact. CLI hosts can supply the same option as a
|
|
202
|
+
default, overridden by an explicit speech configuration vocabulary.
|
|
203
|
+
A voice's optional `speed` sets its synthesis pace from `0.5` to `2`, default `1`.
|
|
204
|
+
Multiple native models use a `synthesizers` map and a `modelId` on each voice;
|
|
205
|
+
their output sample rates must match unless `outputSampleRate` configures a shared
|
|
206
|
+
rate. The worker then uses Sherpa's resampler before sending audio.
|
|
207
|
+
`kitten` provides Kitten Micro 0.8 Bella, Luna, Rosie and Kiki (female), and
|
|
208
|
+
Jasper, Bruno, Hugo and Leo (male). All eight share one model. Bella uses
|
|
209
|
+
`speed: 1.15`; the others use normal speed. Their upstream catalogue does not
|
|
210
|
+
identify British or Australian accents.
|
|
211
|
+
`piper-kitten` lists Kitten before the five Piper voices, defaults to Cori and
|
|
212
|
+
normalizes output to 22,050 Hz. One synthesis model remains resident across
|
|
213
|
+
engine changes. The settings cog lists installed voices.
|
|
214
|
+
The package README documents the exact source manifest and configuration fields.
|
|
215
|
+
Stop the service before replacing its prepared pack, verify it, then restart.
|
|
216
|
+
|
|
217
|
+
Measure first-audio latency and sustained synthesis on the intended host. Voice
|
|
218
|
+
quality, language support and CPU cost depend on the configured models. Browser
|
|
219
|
+
echo cancellation and text filtering need physical microphone/speaker testing;
|
|
220
|
+
generated audio cannot prove acoustic isolation. The recognizer's configurable
|
|
221
|
+
onset RMS floor prevents silence from starting an utterance, but is not a
|
|
222
|
+
speech-versus-noise classifier.
|
|
223
|
+
|
|
224
|
+
The published package's `examples/conversation/` directory contains an independent
|
|
225
|
+
five-conversation Vue/Fastify application using the supplied file storage and
|
|
226
|
+
retained binding, plus a separately addressed headless task. Model credentials
|
|
227
|
+
and an optional speech endpoint are explicit setup inputs. Use it to check
|
|
228
|
+
installation, retained targets, interruption and recovery.
|
|
229
|
+
|
|
230
|
+
### Application-owned voice preferences
|
|
231
|
+
|
|
232
|
+
`VoiceConversationHost` has an optional `settings` slot for an application-owned
|
|
233
|
+
selector. Persist the voice choice in your application, then supply its current
|
|
234
|
+
value through reactive `binding.defaults.voiceId`. Changes apply to subsequent
|
|
235
|
+
replies. An unavailable ID uses the server default without rewriting the saved
|
|
236
|
+
choice; reconnecting can restore it if the voice returns. An empty string selects
|
|
237
|
+
the server default. The built-in selector remains local to the voice session.
|
|
238
|
+
|
|
239
|
+
`readVoiceCatalogue(proxyConfig)` from `/server` retrieves the daemon's authorized
|
|
240
|
+
`GET /voices` metadata without reserving an audio connection. Expose it only
|
|
241
|
+
through an application-authorized route; credentials stay on the server. The
|
|
242
|
+
result contains `voices` (IDs, labels and optional languages) and `defaultVoice`,
|
|
243
|
+
matching the WebSocket greeting. Listing voices does not synthesize or load
|
|
244
|
+
additional voice models.
|
|
@@ -2,22 +2,137 @@
|
|
|
2
2
|
|
|
3
3
|
# Assistant
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
JSKIT provides one conversation runtime and a supplied browser integration.
|
|
6
|
+
`@jskit-ai/assistant-core` owns conversation execution and storage contracts;
|
|
7
|
+
`@jskit-ai/assistant-runtime` exposes that runtime through authorized HTTP
|
|
8
|
+
operations, shared realtime subscriptions and a Vue/Vuetify conversation component.
|
|
9
|
+
File storage requires no database, Knex or database migrations. File and
|
|
10
|
+
application-owned adapters retain their own persisted-state upgrade policy.
|
|
11
|
+
|
|
12
|
+
## The ordinary API
|
|
13
|
+
|
|
14
|
+
The standard setup uses five entrypoints from three modules. Native engines use
|
|
15
|
+
this same setup; they do not add engine-specific imports.
|
|
16
|
+
|
|
17
|
+
| Entrypoint | Module | Use |
|
|
18
|
+
| --- | --- | --- |
|
|
19
|
+
| `createConversationRuntime` | `@jskit-ai/assistant-core/server/conversation` | Create the authorized conversation owner. Headless tasks call its handles directly. |
|
|
20
|
+
| `createFileConversationStorage` | `@jskit-ai/assistant-core/server/conversation` | Use the supplied single-writer file store, or supply the application's transactional store. |
|
|
21
|
+
| `registerFastifyConversations` | `@jskit-ai/assistant-runtime/server` | Host the supplied transport on plain Fastify. An existing JSKIT host instead supplies `assistant.conversations` to its discovered `AssistantFeature`. |
|
|
22
|
+
| `AssistantConversationClientElement` | `@jskit-ai/assistant-runtime/client` | Render the standard conversation with its retained draft, delivery and subscription owner. |
|
|
23
|
+
| `useAssistantConversation` | `@jskit-ai/assistant-runtime/client` | Optional custom layout or retained text/voice access to that same owner. The standard element already calls it. |
|
|
24
|
+
|
|
25
|
+
Start with those entrypoints. [Optional UI primitives](./assistant-conversation.md#client-contract)
|
|
26
|
+
serve custom presentation; [advanced host facilities](./assistant-conversation.md#advanced-host-facilities)
|
|
27
|
+
serve an application that already owns execution, credential homes or native
|
|
28
|
+
history preservation. [Testing entries](./assistant-conversation.md#testing-entrypoints)
|
|
29
|
+
exercise isolated storage and native-host contracts. These are separate uses,
|
|
30
|
+
not extra setup steps for an ordinary conversation.
|
|
31
|
+
|
|
32
|
+
## Standard integration
|
|
33
|
+
|
|
34
|
+
1. Create `createConversationRuntime` with your conversation storage, instructions,
|
|
35
|
+
selected engine/connections and application authorization. Initialize each
|
|
36
|
+
conversation with its own ID and server-owned configuration. The application
|
|
37
|
+
controls which authenticated actors may read or operate each conversation.
|
|
38
|
+
2. Supply that runtime as the `assistant.conversations` capability on your JSKIT
|
|
39
|
+
host. Package discovery registers `AssistantFeature` once; it uses the host's
|
|
40
|
+
existing HTTP, action and realtime services. Multiple conversations share the
|
|
41
|
+
connection, with separate authorization, subscriptions and controls.
|
|
42
|
+
3. Render `AssistantConversationClientElement` for each conversation. The supplied
|
|
43
|
+
binding handles its draft, delivery receipts, streaming, reconnect and controls.
|
|
44
|
+
Supply presentation props and any optional capabilities the application needs.
|
|
45
|
+
|
|
46
|
+
```vue
|
|
47
|
+
<script setup>
|
|
48
|
+
import { AssistantConversationClientElement } from "@jskit-ai/assistant-runtime/client";
|
|
49
|
+
|
|
50
|
+
defineProps({ conversationId: { type: String, required: true } });
|
|
51
|
+
</script>
|
|
52
|
+
|
|
53
|
+
<template>
|
|
54
|
+
<AssistantConversationClientElement :conversation-id="conversationId" />
|
|
55
|
+
</template>
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The current configured surface determines the endpoint and placement. An app with
|
|
59
|
+
one surface can use `defineAssistantSurface("home")` from
|
|
60
|
+
`@jskit-ai/assistant-runtime/shared` in its application configuration instead of
|
|
61
|
+
writing both surface maps. This explicitly declares that surface; authentication
|
|
62
|
+
and conversation access remain application policy. `surface-id`, `endpoint`,
|
|
63
|
+
`hostSurfaceId` and `workspaceSlug` are explicit overrides for an existing host's
|
|
64
|
+
mapping. An authenticated request supplies the actor; browser fields never grant
|
|
65
|
+
access. Opening the same conversation in another view does not create another
|
|
66
|
+
server runtime or transcript.
|
|
67
|
+
|
|
68
|
+
The standard binding uses the host's authenticated placement identity and HTTP
|
|
69
|
+
client. An application with its own bootstrap subject or request policy can call
|
|
70
|
+
`configureAssistantConversations(app, { actorKey, request })` from the same client
|
|
71
|
+
entry once, before mounting. `actorKey` may be a reactive value or getter; changing
|
|
72
|
+
or clearing it retires the previous actor's browser state. Standard cards inherit
|
|
73
|
+
those settings while retaining explicit per-card overrides. See
|
|
74
|
+
[client defaults](./assistant-conversation.md#supplied-client-integration) for the
|
|
75
|
+
complete example. This does not change server authentication or create another
|
|
76
|
+
realtime connection.
|
|
77
|
+
|
|
78
|
+
For a plain Fastify app, call `registerFastifyConversations(app, { runtime, config,
|
|
79
|
+
authenticate, bootstrap, env })` once. This is configurable shorthand for that
|
|
80
|
+
same hosting assembly. Runtime/storage selection, trusted authentication,
|
|
81
|
+
application configuration and shutdown remain explicit; custom hosts retain the
|
|
82
|
+
lower-level registration facilities. See the complete
|
|
83
|
+
[server integration and Fastify example](./assistant-conversation.md#supplied-server-integration).
|
|
84
|
+
|
|
85
|
+
Use the [supplied client binding](./assistant-conversation.md#supplied-client-integration)
|
|
86
|
+
for custom layouts or retained text/voice readers. It produces the element's
|
|
87
|
+
adapter; an ordinary application does not implement its own generic adapter,
|
|
88
|
+
routes or reconnect loop. A lost acknowledgement offers delivery inspection;
|
|
89
|
+
reconnect does not silently resend uncertain work.
|
|
90
|
+
|
|
91
|
+
For an executable setup, use the five-chat example in
|
|
92
|
+
`@jskit-ai/assistant-voice/examples/conversation`. It uses file storage, the same
|
|
93
|
+
supplied integration and one shared realtime connection. Its separately addressed
|
|
94
|
+
headless task calls the runtime directly. Speech is optional; model credentials,
|
|
95
|
+
authentication and speech setup remain explicit application configuration.
|
|
96
|
+
|
|
97
|
+
## Configuration and customization
|
|
98
|
+
|
|
99
|
+
- [Runtime and storage](./assistant-conversation.md#application-conversation-operations):
|
|
100
|
+
configure an API or native engine and use file, memory or application-owned
|
|
101
|
+
storage. Preserve one transcript owner for each conversation.
|
|
102
|
+
- [Optional controls](./assistant-conversation.md#optional-shared-capabilities):
|
|
103
|
+
enable attachments, model choices, suggestions, questions and goals only when
|
|
104
|
+
supported. Applications keep upload storage, model-selection policy and product
|
|
105
|
+
actions; unused adapter areas need no implementation.
|
|
106
|
+
- [Application tools](./assistant-conversation.md#application-tools): supply actions
|
|
107
|
+
and tool policy to the runtime. The common runtime owns the inference/tool loop;
|
|
108
|
+
the application authorizes each action and owns its effects.
|
|
109
|
+
- [Voice](./assistant-voice.md): add push-to-talk and hands-free interaction to the
|
|
110
|
+
same retained conversation binding. Mounting or closing a text view does not
|
|
111
|
+
change the active voice target.
|
|
112
|
+
|
|
113
|
+
The operation and host-facility reference follows the standard setup in
|
|
114
|
+
[Conversation integration](./assistant-conversation.md). Native engine internals
|
|
115
|
+
are not required for ordinary application setup.
|
|
116
|
+
|
|
117
|
+
## Optional database-backed surface
|
|
118
|
+
|
|
119
|
+
The existing SQL integration is an optional storage and application surface over
|
|
120
|
+
the same common runtime. It retains database conversation selection, settings,
|
|
121
|
+
actions and its existing request-streaming contract. Choose it when those SQL
|
|
122
|
+
contracts are required; custom or file storage uses the standard integration
|
|
123
|
+
above. This section documents that existing surface's behavior and migration
|
|
124
|
+
constraints, not a second engine/tool implementation.
|
|
8
125
|
|
|
9
126
|
```bash
|
|
10
127
|
npm install @jskit-ai/assistant-runtime
|
|
11
128
|
npm run db:migrate
|
|
12
129
|
```
|
|
13
130
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
[embeddable conversation element and backend contracts](./assistant-conversation.md).
|
|
18
|
-
The package includes a standalone application template.
|
|
131
|
+
These migration instructions apply only to an application selecting the database
|
|
132
|
+
integration. Its `assistant/assistant-surface` pattern composes the SQL surface;
|
|
133
|
+
there is no assistant generator.
|
|
19
134
|
|
|
20
|
-
|
|
135
|
+
### Database surface decisions
|
|
21
136
|
|
|
22
137
|
Choose:
|
|
23
138
|
|
|
@@ -31,18 +146,54 @@ The application selects an AI integration ID in server configuration. Credential
|
|
|
31
146
|
come from that integration's authorized shared or per-user account. Existing
|
|
32
147
|
environment-prefix configurations remain supported.
|
|
33
148
|
|
|
34
|
-
|
|
149
|
+
### Database surface composition
|
|
35
150
|
|
|
36
151
|
Use `AssistantSurfaceClientElement` and
|
|
37
152
|
`AssistantSettingsClientElement` from `@jskit-ai/assistant-runtime/client`.
|
|
38
153
|
Configure public surface behavior and server settings in ordinary app-owned
|
|
39
154
|
config, then register routes and placements like any other feature.
|
|
40
155
|
|
|
156
|
+
For an application-supplied `assistant.conversations` common runtime, use
|
|
157
|
+
`AssistantConversationClientElement` or `useAssistantConversation` from that
|
|
158
|
+
same client entry. This path shares the existing HTTP/action and realtime owners
|
|
159
|
+
and accepts file/custom storage without database repositories or migrations.
|
|
160
|
+
See [supplied client integration](./assistant-conversation.md#supplied-client-integration)
|
|
161
|
+
for captured conversation identity, optional controls and retained readers.
|
|
162
|
+
|
|
41
163
|
Workspace scope is valid only when both the runtime and its settings surface
|
|
42
164
|
are workspace-aware. Requests must retain the selected workspace through the
|
|
43
165
|
server action boundary.
|
|
44
166
|
|
|
45
|
-
|
|
167
|
+
### Existing SQL conversations
|
|
168
|
+
|
|
169
|
+
The optional database integration uses the common conversation runtime for
|
|
170
|
+
inference, cancellation and application tools. Its existing settings, actions,
|
|
171
|
+
NDJSON routes, conversation IDs, message rows and request-claim table remain the
|
|
172
|
+
integration boundary. A tool reservation is committed to its original
|
|
173
|
+
`tool_call` row before execution; the matching `tool_result` row retains the
|
|
174
|
+
verified result. No second transcript is stored in conversation metadata.
|
|
175
|
+
|
|
176
|
+
The surface's bounded, client-selected history remains request context. It is
|
|
177
|
+
authorized for attachments and sent to the API model without being inserted into
|
|
178
|
+
saved conversation history. Existing environment clients and the surface's
|
|
179
|
+
cached tool exclusions retain their existing owners.
|
|
180
|
+
|
|
181
|
+
Before adopting historical SQL rows, the storage owner checks their turn and
|
|
182
|
+
tool associations without rewriting them. Unambiguous history stays readable in
|
|
183
|
+
place. If historical output has no clear authored owner, or a tool call has no
|
|
184
|
+
confirmed result, Send fails with `assistant_history_inspection_required` and
|
|
185
|
+
row IDs in the error details. Previous history remains available through
|
|
186
|
+
**Conversations**. Use **Start new conversation** to continue separately, or ask
|
|
187
|
+
the application operator to inspect the reported rows and any affected action
|
|
188
|
+
target. Do not delete a request claim or infer that an action with no result did
|
|
189
|
+
nothing. Repairs require the application's offline upgrade procedure and a
|
|
190
|
+
database backup; requests and startup never backfill these rows.
|
|
191
|
+
|
|
192
|
+
New runtime metadata uses the existing JSON fields, so this ownership move adds
|
|
193
|
+
no database schema migration. Stop old assistant requests before switching
|
|
194
|
+
server versions; keep applying the package's existing migrations normally.
|
|
195
|
+
|
|
196
|
+
### Database surface AI integration
|
|
46
197
|
|
|
47
198
|
1. Configure an `ai` integration using JSKIT's existing integration editor or
|
|
48
199
|
portable `integrations.json`. Its provider/model, account mode and credential
|
|
@@ -52,7 +203,9 @@ server action boundary.
|
|
|
52
203
|
from `@jskit-ai/connectors-catalog/server/ai`. `authorize` derives the trusted
|
|
53
204
|
`{ applicationId, subjectId }` from the authenticated request. Use the normal
|
|
54
205
|
environment reference resolver for shared keys and your account resolver for
|
|
55
|
-
per-user keys.
|
|
206
|
+
per-user keys. The resolved connection also supplies `modelLimits` from the
|
|
207
|
+
catalogue for native engines that need an explicit model definition. Never
|
|
208
|
+
return credentials to the browser.
|
|
56
209
|
3. Select it in server config:
|
|
57
210
|
|
|
58
211
|
```js
|
|
@@ -121,7 +274,16 @@ event. It retains failed uploads and permits typing while uploads are pending.
|
|
|
121
274
|
Supply the controller with an upload scope that changes with user, workspace and
|
|
122
275
|
conversation ownership. The server must enforce those boundaries independently.
|
|
123
276
|
|
|
124
|
-
|
|
277
|
+
The supplied canonical client accepts `useAssistantAttachments()` and
|
|
278
|
+
`useAssistantSuggestions()` results directly. It clears only captured attachment
|
|
279
|
+
IDs after HTTP, canonical history or explicit **Check delivery** confirms
|
|
280
|
+
acceptance; newer uploads remain in the composer. Its suggestion owner binds to
|
|
281
|
+
the canonical draft and complete target identity, while the host supplies a
|
|
282
|
+
separate authorized generator and its configuration. Selecting a suggestion
|
|
283
|
+
edits the draft and never sends it. These controls reuse the same shared
|
|
284
|
+
[attachment and suggestion contracts](./assistant-conversation.md#optional-shared-capabilities).
|
|
285
|
+
|
|
286
|
+
### Database surface action tools
|
|
125
287
|
|
|
126
288
|
The assistant reads automation-capable actions from `runtime.actions`. It keeps
|
|
127
289
|
the typing user's actor, permissions, target surface, and workspace context,
|
|
@@ -274,12 +436,13 @@ the three discovery tools. Permission, surface, channel, actor, and trusted
|
|
|
274
436
|
workspace enforcement remain unchanged. Other actions still require contract
|
|
275
437
|
lookup before discovery-mode execution.
|
|
276
438
|
|
|
277
|
-
|
|
439
|
+
### Database surface conversation lifecycle
|
|
278
440
|
|
|
279
441
|
The client receives answer text incrementally and shows working status before
|
|
280
442
|
text arrives. Tool calls produce timeline events. Internal reasoning tags and raw
|
|
281
443
|
tool arguments do not become visible answer text. A tool/recovery round replaces
|
|
282
|
-
its provisional answer;
|
|
444
|
+
its provisional answer; completed answers occupy the existing chat rows.
|
|
445
|
+
Common runtime commentary is retained separately from those chat rows. Progress-only
|
|
283
446
|
responses such as “Let me query…” may appear provisionally while streaming but
|
|
284
447
|
are retried internally and never stored or replayed as final chat history.
|
|
285
448
|
|
|
@@ -333,6 +496,11 @@ a request automatically: a tool may already have changed application data.
|
|
|
333
496
|
This protects against duplicate execution; it cannot guarantee completion after
|
|
334
497
|
a process crash. The browser does not automatically resubmit ambiguous work.
|
|
335
498
|
|
|
499
|
+
An application action whose outcome cannot be verified stops inference and
|
|
500
|
+
keeps its durable reservation. Inspect the action's target before requesting a
|
|
501
|
+
new execution; ordinary model retry and failure recovery cannot turn an unknown
|
|
502
|
+
outcome into permission to repeat the action.
|
|
503
|
+
|
|
336
504
|
Request records contain the submitted text/history and the replay response.
|
|
337
505
|
They survive conversation removal so an old request cannot become executable
|
|
338
506
|
again, and are deleted when their actor is deleted. Application retention or
|
|
@@ -340,13 +508,14 @@ privacy jobs must account for this table and must not remove active request
|
|
|
340
508
|
identities. Custom transports using only `assistant-core` still own equivalent
|
|
341
509
|
server-side protection; the UI controller alone cannot prevent duplicate work.
|
|
342
510
|
|
|
343
|
-
|
|
511
|
+
### Database surface client behavior
|
|
344
512
|
|
|
345
513
|
`AssistantSurfaceClientElement` connects `useAssistantRuntime` to
|
|
346
514
|
`AssistantConversationElement`. There is one maintained conversation renderer
|
|
347
|
-
and composer. Applications
|
|
348
|
-
|
|
349
|
-
|
|
515
|
+
and composer. Applications supplying a conversation runtime use
|
|
516
|
+
`AssistantConversationClientElement` and its supplied binding, described above.
|
|
517
|
+
A hand-written adapter is an advanced presentation integration, not a prerequisite
|
|
518
|
+
for custom storage or native engines.
|
|
350
519
|
|
|
351
520
|
The runtime owns the draft, requests, saved conversations, history selection,
|
|
352
521
|
provider settings, authorization scope and action tools. Its adapter maps
|
|
@@ -428,7 +597,7 @@ classes or pass the retired renderer's `variant`, `features`, `ui`, or `copy`
|
|
|
428
597
|
objects. `AssistantClientElement` and its Markdown/keyboard helpers have been
|
|
429
598
|
removed; there is no forwarding alias.
|
|
430
599
|
|
|
431
|
-
|
|
600
|
+
### Database surface verification
|
|
432
601
|
|
|
433
602
|
Run migrations, load assistant and settings pages through normal navigation,
|
|
434
603
|
test missing credentials without exposing values, exercise one successful and
|
|
@@ -26,6 +26,12 @@ generic “something changed” escape hatch.
|
|
|
26
26
|
Client features register listeners through `@jskit-ai/realtime`. Keep query
|
|
27
27
|
invalidation close to the resource that owns the query keys.
|
|
28
28
|
|
|
29
|
+
`useRealtimeEvent()` follows its Vue effect scope. Ordinary component listeners
|
|
30
|
+
detach when that component unmounts. A listener created inside an application-owned
|
|
31
|
+
detached scope remains active after its original screen unmounts and detaches when
|
|
32
|
+
that scope is stopped. Retain and release that scope with the resource's actual
|
|
33
|
+
owners; a hidden screen alone must not dispose a conversation still used by voice.
|
|
34
|
+
|
|
29
35
|
When the entire realtime surface is authenticated, the selected `auth.service`
|
|
30
36
|
can expose `realtime.requireAuthentication: true`. The realtime server then
|
|
31
37
|
rejects unauthenticated handshakes before a socket joins any broadcast room.
|
package/guide/agent/index.md
CHANGED
|
@@ -32,6 +32,7 @@ apply or adapt. It does not prescribe an agent host or project orchestrator.
|
|
|
32
32
|
- [Mobile Capacitor](/guide/app-extras/mobile-capacitor)
|
|
33
33
|
- [Realtime](/guide/app-extras/realtime)
|
|
34
34
|
- [Assistant](/guide/app-extras/assistant)
|
|
35
|
+
- [Voice conversations](/guide/app-extras/assistant-voice)
|
|
35
36
|
|
|
36
37
|
## How to use the guide
|
|
37
38
|
|
package/package.json
CHANGED
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
"dependencies": {
|
|
35
35
|
"@local/main": "0.1.0",
|
|
36
36
|
"@fastify/static": "^10.1.3",
|
|
37
|
-
"@jskit-ai/kernel": "0.1.
|
|
37
|
+
"@jskit-ai/kernel": "0.1.224",
|
|
38
38
|
"@tanstack/vue-query": "^5.101.0",
|
|
39
39
|
"fastify": "^5.8.5",
|
|
40
40
|
"json-rest-schema": "^1.0.17",
|
|
@@ -42,13 +42,13 @@
|
|
|
42
42
|
"vue": "^3.5.43",
|
|
43
43
|
"vue-router": "^5.3.1",
|
|
44
44
|
"vuetify": "^4.1.2",
|
|
45
|
-
"@jskit-ai/http-runtime": "0.1.
|
|
45
|
+
"@jskit-ai/http-runtime": "0.1.222",
|
|
46
46
|
"@fastify/fast-json-stringify-compiler": "^5.1.0",
|
|
47
47
|
"@vue/devtools-api": "^8.2.1"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@jskit-ai/config-eslint": "0.1.
|
|
51
|
-
"@jskit-ai/jskit-catalog": "0.1.
|
|
50
|
+
"@jskit-ai/config-eslint": "0.1.221",
|
|
51
|
+
"@jskit-ai/jskit-catalog": "0.1.249",
|
|
52
52
|
"@playwright/test": "1.61.1",
|
|
53
53
|
"@vitejs/plugin-vue": "^6.0.7",
|
|
54
54
|
"eslint": "^10.8.0",
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
"dependencies": {
|
|
35
35
|
"@local/main": "0.1.0",
|
|
36
36
|
"@fastify/static": "^10.1.3",
|
|
37
|
-
"@jskit-ai/kernel": "0.1.
|
|
37
|
+
"@jskit-ai/kernel": "0.1.224",
|
|
38
38
|
"@tanstack/vue-query": "^5.101.0",
|
|
39
39
|
"fastify": "^5.8.5",
|
|
40
40
|
"json-rest-schema": "^1.0.17",
|
|
@@ -42,15 +42,15 @@
|
|
|
42
42
|
"vue": "^3.5.43",
|
|
43
43
|
"vue-router": "^5.3.1",
|
|
44
44
|
"vuetify": "^4.1.2",
|
|
45
|
-
"@jskit-ai/http-runtime": "0.1.
|
|
45
|
+
"@jskit-ai/http-runtime": "0.1.222",
|
|
46
46
|
"@mdi/js": "^7.4.47",
|
|
47
|
-
"@jskit-ai/shell-web": "0.1.
|
|
47
|
+
"@jskit-ai/shell-web": "0.1.228",
|
|
48
48
|
"@fastify/fast-json-stringify-compiler": "^5.1.0",
|
|
49
49
|
"@vue/devtools-api": "^8.2.1"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
|
-
"@jskit-ai/config-eslint": "0.1.
|
|
53
|
-
"@jskit-ai/jskit-catalog": "0.1.
|
|
52
|
+
"@jskit-ai/config-eslint": "0.1.221",
|
|
53
|
+
"@jskit-ai/jskit-catalog": "0.1.249",
|
|
54
54
|
"@playwright/test": "1.61.1",
|
|
55
55
|
"@vitejs/plugin-vue": "^6.0.7",
|
|
56
56
|
"eslint": "^10.8.0",
|