@jskit-ai/assistant-voice 0.0.0-stage → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +169 -2
- package/examples/conversation/App.vue +94 -0
- package/examples/conversation/Conversation.vue +28 -0
- package/examples/conversation/README.md +181 -0
- package/examples/conversation/config.js +15 -0
- package/examples/conversation/index.html +1 -0
- package/examples/conversation/integrations.example.json +16 -0
- package/examples/conversation/main.js +58 -0
- package/examples/conversation/package.json +32 -0
- package/examples/conversation/server.js +105 -0
- package/examples/conversation/src/error.js +1 -0
- package/examples/conversation/src/placement.js +1 -0
- package/examples/conversation/src/placementTopology.js +1 -0
- package/examples/conversation/vite.config.js +9 -0
- package/package.json +67 -4
- package/src/client/ConversationDialog.vue +67 -0
- package/src/client/VoiceAvatar.vue +18 -0
- package/src/client/VoiceConversation.vue +117 -0
- package/src/client/VoiceConversationHost.vue +78 -0
- package/src/client/audioCapture.js +198 -0
- package/src/client/conversationNarration.js +108 -0
- package/src/client/conversationVoiceState.js +18 -0
- package/src/client/index.js +9 -0
- package/src/client/voiceAvatar.js +234 -0
- package/src/client/voiceController.js +124 -0
- package/src/client/voiceConversation.js +557 -0
- package/src/client/voiceLauncher.js +63 -0
- package/src/client/voicePlayback.js +58 -0
- package/src/client/voiceTransport.js +890 -0
- package/src/server/index.js +5 -0
- package/src/server/voiceAccessToken.js +79 -0
- package/src/server/voiceDaemon.js +599 -0
- package/src/server/voiceProxy.js +199 -0
- package/src/server/voiceProxyConfig.js +70 -0
- package/src/server/voiceSocketClose.js +31 -0
- package/src/shared/protocol.js +439 -0
- package/src/shared/visemes.js +130 -0
package/README.md
CHANGED
|
@@ -1,3 +1,170 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Assistant voice
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Optional duplex voice for Vue 3/Vuetify applications. One shell controller owns
|
|
4
|
+
microphone, playback, active destination and modal. Applications retain their
|
|
5
|
+
conversations, permissions, message admission and history. No Vibe64 or Genesis
|
|
6
|
+
runtime is required.
|
|
7
|
+
|
|
8
|
+
Install `@jskit-ai/assistant-voice` with Vue and Vuetify. Run your own speech
|
|
9
|
+
service, or use `@jskit-ai/assistant-voice-sherpa` for local native inference.
|
|
10
|
+
The native engine and model downloads are not dependencies of this package.
|
|
11
|
+
|
|
12
|
+
## Supplied conversation integration
|
|
13
|
+
|
|
14
|
+
For ordinary conversations, use `AssistantConversationClientElement` from
|
|
15
|
+
`@jskit-ai/assistant-runtime/client` with the common conversation runtime. Its
|
|
16
|
+
`composer-tools` slot supplies the retained runtime used by the five-chat
|
|
17
|
+
`examples/conversation/` starter. That example forwards voice text and Stop through
|
|
18
|
+
`runtime.send()` and `runtime.cancel()`, retains the same reader when text closes,
|
|
19
|
+
and uses `projectConversationVoiceState` for canonical speech presentation.
|
|
20
|
+
Application code supplies the speech endpoint, label and navigation. It needs no
|
|
21
|
+
engine-specific imports, separate transcript or conversation connection.
|
|
22
|
+
|
|
23
|
+
### Custom conversation bindings
|
|
24
|
+
|
|
25
|
+
Applications with an existing authorized reader can supply the following binding.
|
|
26
|
+
Create one controller above routed screens, then mount `VoiceConversationHost`
|
|
27
|
+
once within your application's Vuetify app:
|
|
28
|
+
|
|
29
|
+
```js
|
|
30
|
+
import { createVoiceConversationController } from '@jskit-ai/assistant-voice/client';
|
|
31
|
+
|
|
32
|
+
const controller = createVoiceConversationController({
|
|
33
|
+
connectSpeech: conversation => `/api/conversations/${conversation.id}/voice`
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
await controller.open({ conversation: {
|
|
37
|
+
id: conversationId,
|
|
38
|
+
label: 'Planning',
|
|
39
|
+
get state() { return store.conversations[conversationId]; },
|
|
40
|
+
get available() { return store.canUse(conversationId); },
|
|
41
|
+
captureContext: () => ({ conversationId }),
|
|
42
|
+
submitText: (text, { messageId, context }) =>
|
|
43
|
+
store.send(context.conversationId, { text, messageId }),
|
|
44
|
+
cancelWork: () => store.stop(conversationId)
|
|
45
|
+
} });
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
```vue
|
|
49
|
+
<VoiceConversationHost :controller="controller" />
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`state` contains `messages`, optional `streamingReply`, and `status` (`working`
|
|
53
|
+
while the agent runs). Messages have stable `id`, `role` and `text`. A streamed
|
|
54
|
+
reply and its saved replacement must share the same ID. Only user/assistant
|
|
55
|
+
content belongs here; exclude reasoning, tools and private operational output.
|
|
56
|
+
|
|
57
|
+
`submitText` resolves on canonical acceptance and rejects failures. Preserve its
|
|
58
|
+
message ID on uncertain retries and deduplicate in your existing server admission.
|
|
59
|
+
The recording's captured context remains fixed across navigation. Update state
|
|
60
|
+
from your ordinary reader; voice never owns or reconstructs saved history.
|
|
61
|
+
|
|
62
|
+
Optional `openText()` opens the application's text conversation. When supplied,
|
|
63
|
+
the shared modal shows Talk and Text controls at the top. Selecting Text calls
|
|
64
|
+
that operation and minimizes voice; capture and playback keep their current state.
|
|
65
|
+
Applications may open voice as their default conversation view without starting
|
|
66
|
+
the microphone.
|
|
67
|
+
|
|
68
|
+
Optional synchronous `retain()` / `release()` keep the conversation reader alive
|
|
69
|
+
outside its screen. `available = false` releases the active voice session. Dispose
|
|
70
|
+
the controller on app teardown or identity replacement. IDs must include the
|
|
71
|
+
identity scope when different people can use the same conversation ID.
|
|
72
|
+
|
|
73
|
+
Optional defaults: `mode` (`push-to-talk` or `hands-free`), `talkMode` (`tap` or
|
|
74
|
+
`hold` for one-off recording), `reviewBeforeSend`, and an installed `voiceId`.
|
|
75
|
+
One-off recordings started through `startHeldRecording()` require review; the
|
|
76
|
+
modal's push-to-talk gesture uses `reviewBeforeSend`. `avatar` and `onVisual`/`onTranscript`
|
|
77
|
+
allow presentation in the surrounding app without adding another audio owner.
|
|
78
|
+
The host's `avatar` slot receives state, mouthLevel, mouthPose, avatar and binding.
|
|
79
|
+
`useVoiceAvatar` supplies shared animation cues; applications may supply artwork.
|
|
80
|
+
|
|
81
|
+
## Lifecycle
|
|
82
|
+
|
|
83
|
+
Opening a target does not start capture. Tap Talk for hands-free, or hold it for
|
|
84
|
+
push-to-talk and release to send; connection setup is automatic.
|
|
85
|
+
Pause finishes the current utterance, flushing the audio tail before muting.
|
|
86
|
+
Completed words follow the captured review policy; resume opens a fresh recording.
|
|
87
|
+
Automatic sends keep the call layout fixed. A failed send exposes the retained
|
|
88
|
+
transcript for review and retry with the same message identity. Captions,
|
|
89
|
+
review, explicit microphone/sound controls and bounded reconnection share one
|
|
90
|
+
session. Opening the same ID reveals it. Switching targets releases the previous
|
|
91
|
+
audio first; unfinished words require completion or explicit discard. A pending
|
|
92
|
+
send cannot be switched away by the discard action. `minimize()` preserves the
|
|
93
|
+
session; `end()` releases it after pending words are resolved. `dispose()` releases
|
|
94
|
+
it unconditionally. The controller serializes these operations. The host's X calls `end({ discard: true })`,
|
|
95
|
+
so closing stops voice and discards unsent speech. Minus minimizes instead.
|
|
96
|
+
|
|
97
|
+
`ConversationDialog` provides the same fixed header, icon tabs and bounds for
|
|
98
|
+
application Text views, with full screen below 600 pixels and no mode transition.
|
|
99
|
+
The backdrop also stays constant during a mode change. The host and shared dialog
|
|
100
|
+
accept an optional `activator` DOM element: clicking that covered launcher
|
|
101
|
+
minimizes an open conversation instead of closing it.
|
|
102
|
+
The host's `reopen` slot lets an application replace the floating reopen control
|
|
103
|
+
with its own header indicator and `controller.reveal()` action.
|
|
104
|
+
|
|
105
|
+
Stop speaking cancels playback and queued speech. Stop agent work calls the app's
|
|
106
|
+
separate cancellation operation. Reconnect does not resend audio or messages,
|
|
107
|
+
replay history or automatically restart capture. If a service advertises several
|
|
108
|
+
voices, the Voice settings cog offers Speaking voice for the next reply's speaker; an in-flight reply keeps
|
|
109
|
+
its original speaker through all streamed phrases.
|
|
110
|
+
|
|
111
|
+
## Server connection
|
|
112
|
+
|
|
113
|
+
Register an authenticated WebSocket route with Fastify's WebSocket plugin:
|
|
114
|
+
|
|
115
|
+
```js
|
|
116
|
+
import { registerVoiceProxyRoute, resolveVoiceProxyConfig }
|
|
117
|
+
from '@jskit-ai/assistant-voice/server';
|
|
118
|
+
|
|
119
|
+
registerVoiceProxyRoute(fastify, {
|
|
120
|
+
route: '/api/conversations/:id/voice',
|
|
121
|
+
proxyConfig: resolveVoiceProxyConfig({ endpoint, accessTokenFile }),
|
|
122
|
+
async authorize(request) {
|
|
123
|
+
await checkOrigin(request);
|
|
124
|
+
await requireConversationAccess(request.user, request.params.id);
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
The route's authorization is mandatory and runs before upstream connection.
|
|
130
|
+
Keep endpoint and token file server-side. The token grants speech-service use,
|
|
131
|
+
not access to a conversation or permission to run an agent. Browsers need a secure
|
|
132
|
+
origin (HTTPS or a browser-recognized localhost origin) and microphone permission.
|
|
133
|
+
|
|
134
|
+
`createVoiceDaemon({ engine, accessKey, ... })` runs the bounded protocol service.
|
|
135
|
+
The engine supplies a recognizer, asynchronous synthesis and optionally a public
|
|
136
|
+
voice catalogue. A host may supply synchronous `authorize(token)` and
|
|
137
|
+
`handleRequest(request, response)` for its own grant policy. An authorized
|
|
138
|
+
principal has `tenant` and optional `parentTenant`; both connection quotas apply.
|
|
139
|
+
`revokeTenant(id)` ends current child or parent sessions as well as pending work.
|
|
140
|
+
The host must reject future requests for revoked grants in its authorizer.
|
|
141
|
+
|
|
142
|
+
## Example and testing
|
|
143
|
+
|
|
144
|
+
`examples/conversation/` composes the supplied conversation server integration,
|
|
145
|
+
client binding and one shared realtime connection for five file-backed chats.
|
|
146
|
+
Text and voice retain the same conversation; an optional headless task uses the
|
|
147
|
+
same runtime directly. Its README covers packed-package setup, launch and the
|
|
148
|
+
local origin policy. No Vibe64, Genesis or database runtime is required.
|
|
149
|
+
|
|
150
|
+
Focused package checks run one file at a time, for example:
|
|
151
|
+
`npm test --workspace packages/assistant-voice -- test/voiceController.test.js`.
|
|
152
|
+
Native speech and physical microphone/speaker testing are separate. Browser echo
|
|
153
|
+
cancellation plus local text filters cannot guarantee acoustic isolation on every
|
|
154
|
+
device; test hands-free interruption with your actual hardware.
|
|
155
|
+
|
|
156
|
+
### Application-owned voice preferences
|
|
157
|
+
|
|
158
|
+
`VoiceConversationHost` has an optional `settings` slot for an application-owned
|
|
159
|
+
selector. Persist the voice choice in your application, then supply its current
|
|
160
|
+
value through reactive `binding.defaults.voiceId`. Changes apply to subsequent
|
|
161
|
+
replies. An unavailable ID uses the server default without rewriting the saved
|
|
162
|
+
choice; reconnecting can restore it if the voice returns. An empty string selects
|
|
163
|
+
the server default. The built-in selector remains local to the voice session.
|
|
164
|
+
|
|
165
|
+
`readVoiceCatalogue(proxyConfig)` from `/server` retrieves the daemon's authorized
|
|
166
|
+
`GET /voices` metadata without reserving an audio connection. Expose it only
|
|
167
|
+
through an application-authorized route; credentials stay on the server. The
|
|
168
|
+
result contains `voices` (IDs, labels and optional languages) and `defaultVoice`,
|
|
169
|
+
matching the WebSocket greeting. Listing voices does not synthesize or load
|
|
170
|
+
additional voice models.
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
<script setup>
|
|
2
|
+
import { inject, nextTick, onScopeDispose, ref } from "vue";
|
|
3
|
+
import { createVoiceConversationController, projectConversationVoiceState, VoiceConversationHost } from "@jskit-ai/assistant-voice/client";
|
|
4
|
+
import { ShellErrorHost } from "@jskit-ai/shell-web/client";
|
|
5
|
+
import Conversation from "./Conversation.vue";
|
|
6
|
+
|
|
7
|
+
const example = inject("example.application");
|
|
8
|
+
const closed = ref([]);
|
|
9
|
+
const views = new Map();
|
|
10
|
+
const error = ref("");
|
|
11
|
+
const controller = createVoiceConversationController({ connectSpeech: binding => binding.socketUrl });
|
|
12
|
+
async function openText(id) {
|
|
13
|
+
closed.value = closed.value.filter(value => value !== id);
|
|
14
|
+
await nextTick();
|
|
15
|
+
views.get(id)?.focus();
|
|
16
|
+
}
|
|
17
|
+
function voiceBinding(runtime, label) {
|
|
18
|
+
const identity = runtime.identity;
|
|
19
|
+
let retained;
|
|
20
|
+
return {
|
|
21
|
+
id: JSON.stringify([identity.actorKey, identity.conversationId]), label,
|
|
22
|
+
socketUrl: `/api/conversations/${encodeURIComponent(identity.conversationId)}/voice`,
|
|
23
|
+
get state() { return projectConversationVoiceState({ turns: runtime.turns.value, status: runtime.snapshot.value?.status }); },
|
|
24
|
+
get available() { return runtime.available.value; },
|
|
25
|
+
captureContext: () => ({ ...identity }),
|
|
26
|
+
retain() { retained = runtime.retain(); runtime = retained.runtime; },
|
|
27
|
+
release() { retained?.release(); retained = null; },
|
|
28
|
+
submitText(text, { messageId, context } = {}) {
|
|
29
|
+
if (context?.actorKey !== identity.actorKey || context?.conversationId !== identity.conversationId) {
|
|
30
|
+
throw new Error("This recording belongs to another conversation.");
|
|
31
|
+
}
|
|
32
|
+
return runtime.send({ message: text }, { messageId });
|
|
33
|
+
},
|
|
34
|
+
cancelWork: () => runtime.cancel(),
|
|
35
|
+
openText: () => openText(identity.conversationId)
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
async function openVoice(runtime, label) {
|
|
39
|
+
try { await controller.open({ conversation: voiceBinding(runtime, label) }); error.value = ""; }
|
|
40
|
+
catch (cause) { error.value = cause.message; }
|
|
41
|
+
}
|
|
42
|
+
onScopeDispose(() => { void controller.dispose(); });
|
|
43
|
+
</script>
|
|
44
|
+
|
|
45
|
+
<template>
|
|
46
|
+
<v-app>
|
|
47
|
+
<main class="example">
|
|
48
|
+
<h1>Voice conversations</h1>
|
|
49
|
+
<p>{{ example.provider }}</p>
|
|
50
|
+
<p>Each chat has its own history and controls. Voice keeps its original target when you close or focus another text view.</p>
|
|
51
|
+
<nav aria-label="Open a text conversation" class="example__navigation">
|
|
52
|
+
<v-btn
|
|
53
|
+
v-for="conversation in example.conversations" :key="conversation.id" variant="tonal"
|
|
54
|
+
@click="openText(conversation.id)"
|
|
55
|
+
>
|
|
56
|
+
{{ conversation.label }}
|
|
57
|
+
</v-btn>
|
|
58
|
+
</nav>
|
|
59
|
+
<p v-if="error" role="alert">{{ error }}</p>
|
|
60
|
+
<div class="example__grid">
|
|
61
|
+
<section
|
|
62
|
+
v-for="conversation in example.conversations.filter(value => !closed.includes(value.id))"
|
|
63
|
+
:key="conversation.id" :aria-label="`${conversation.label} chat`" class="example__card"
|
|
64
|
+
>
|
|
65
|
+
<header>
|
|
66
|
+
<h2>{{ conversation.label }}</h2>
|
|
67
|
+
<v-btn
|
|
68
|
+
size="small" variant="text" :aria-label="`Close ${conversation.label} text view`"
|
|
69
|
+
@click="closed.push(conversation.id)"
|
|
70
|
+
>
|
|
71
|
+
Close text
|
|
72
|
+
</v-btn>
|
|
73
|
+
</header>
|
|
74
|
+
<Conversation
|
|
75
|
+
:ref="value => value ? views.set(conversation.id, value) : views.delete(conversation.id)"
|
|
76
|
+
:id="conversation.id" :label="conversation.label" @voice="runtime => openVoice(runtime, conversation.label)"
|
|
77
|
+
/>
|
|
78
|
+
</section>
|
|
79
|
+
</div>
|
|
80
|
+
</main>
|
|
81
|
+
<VoiceConversationHost :controller="controller" />
|
|
82
|
+
<ShellErrorHost />
|
|
83
|
+
</v-app>
|
|
84
|
+
</template>
|
|
85
|
+
|
|
86
|
+
<style scoped>
|
|
87
|
+
.example { width: min(100% - 32px, 1500px); margin: 24px auto; }
|
|
88
|
+
p { margin-block: 12px; }
|
|
89
|
+
.example__navigation { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 16px; }
|
|
90
|
+
.example__grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 400px), 1fr)); gap: 16px; }
|
|
91
|
+
.example__card { display: flex; flex-direction: column; min-width: 0; height: min(720px, 85dvh); padding: 12px; border: 1px solid rgb(var(--v-theme-outline)); border-radius: 12px; }
|
|
92
|
+
header { display: flex; align-items: center; justify-content: space-between; gap: 8px; margin-bottom: 8px; }
|
|
93
|
+
h2 { font-size: 1.1rem; }
|
|
94
|
+
</style>
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
<script setup>
|
|
2
|
+
import { ref } from "vue";
|
|
3
|
+
import { AssistantConversationClientElement } from "@jskit-ai/assistant-runtime/client";
|
|
4
|
+
|
|
5
|
+
defineProps({ id: { type: String, required: true }, label: { type: String, required: true } });
|
|
6
|
+
const emit = defineEmits(["voice"]);
|
|
7
|
+
const view = ref(null);
|
|
8
|
+
defineExpose({ focus: () => view.value?.focus() });
|
|
9
|
+
</script>
|
|
10
|
+
|
|
11
|
+
<template>
|
|
12
|
+
<AssistantConversationClientElement
|
|
13
|
+
ref="view" :conversation-id="id" :assistant-label="label" class="example-conversation"
|
|
14
|
+
>
|
|
15
|
+
<template #composer-tools="{ runtime }">
|
|
16
|
+
<v-btn
|
|
17
|
+
size="small" variant="text" :disabled="!runtime?.available.value"
|
|
18
|
+
@click="emit('voice', runtime)"
|
|
19
|
+
>
|
|
20
|
+
Voice chat with {{ label }}
|
|
21
|
+
</v-btn>
|
|
22
|
+
</template>
|
|
23
|
+
</AssistantConversationClientElement>
|
|
24
|
+
</template>
|
|
25
|
+
|
|
26
|
+
<style scoped>
|
|
27
|
+
.example-conversation { height: 100%; min-height: 0; }
|
|
28
|
+
</style>
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# Independent voice conversation example
|
|
2
|
+
|
|
3
|
+
This Vue/Vuetify and Fastify application uses the supplied JSKIT conversation
|
|
4
|
+
server integration, client binding and shared realtime connection. Planning,
|
|
5
|
+
Notes, Research, Writing and Review have separate file-backed histories. One
|
|
6
|
+
root voice modal retains its target independently of mounted text cards.
|
|
7
|
+
Vibe64, Genesis and a database are not required.
|
|
8
|
+
|
|
9
|
+
## Install and start
|
|
10
|
+
|
|
11
|
+
Use a coordinated set of packed packages containing the conversation integration
|
|
12
|
+
changes. The migration source and existing package numbers alone are not a fresh
|
|
13
|
+
installation proof; candidate verification records the exact archives and pins.
|
|
14
|
+
The application imports public package entrypoints only, with no source aliases.
|
|
15
|
+
Its Vite configuration uses JSKIT's supplied client plugin for installed-package
|
|
16
|
+
dependency scanning and shared client dependencies.
|
|
17
|
+
|
|
18
|
+
1. Run `npm install` in this directory using that candidate's package versions.
|
|
19
|
+
2. Copy `integrations.example.json` to your own configuration file. Choose a model
|
|
20
|
+
key from the existing JSKIT AI catalogue. Set the referenced `ASSISTANT_API_KEY`
|
|
21
|
+
in the server environment using your secret manager or protected shell session.
|
|
22
|
+
Keep the key out of JSON configuration and browser environment variables.
|
|
23
|
+
3. For voice, prepare a speech service using the assistant-voice-sherpa README,
|
|
24
|
+
or obtain a speech endpoint and application token from its operator. Supply
|
|
25
|
+
its endpoint and protected tenant token file to this example's server.
|
|
26
|
+
4. Start the server with your protected configuration paths:
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
ASSISTANT_INTEGRATIONS=/path/to/integrations.json \
|
|
30
|
+
ASSISTANT_STORAGE_DIRECTORY=/path/to/private/conversations \
|
|
31
|
+
SPEECH_ENDPOINT=ws://127.0.0.1:3092/v1/voice \
|
|
32
|
+
SPEECH_TOKEN_FILE=/run/secrets/example-speech-token npm run server
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
5. In another terminal run `npm run dev` and open `http://127.0.0.1:5176`.
|
|
36
|
+
Each text card uses `AssistantConversationClientElement`. Send a short request in two
|
|
37
|
+
cards to see independent output and Stop controls on the one connection.
|
|
38
|
+
6. Open **Voice chat with Planning**. Tap Talk for hands-free listening or hold
|
|
39
|
+
it for push-to-talk. The speech connection starts automatically when needed.
|
|
40
|
+
Connect a microphone and speakers and grant browser permission yourself.
|
|
41
|
+
7. Minimize voice, then close Planning's text card or focus Notes. Voice retains
|
|
42
|
+
Planning. Use another card's Voice chat button to switch explicitly; unfinished
|
|
43
|
+
words require completion or discard. The voice modal's Text control reopens
|
|
44
|
+
its original text card. Closing voice stops its audio; minimizing preserves it.
|
|
45
|
+
|
|
46
|
+
The speech WebSocket is separate from the shared conversation realtime channel.
|
|
47
|
+
It carries audio, not a second conversation subscription or transcript store.
|
|
48
|
+
Leave speech configuration unset to use text without running a speech service.
|
|
49
|
+
|
|
50
|
+
## Configuration and storage
|
|
51
|
+
|
|
52
|
+
`ASSISTANT_INTEGRATIONS` uses the existing JSKIT integration configuration and
|
|
53
|
+
credential resolver for every engine. `ASSISTANT_INTEGRATION_ID` selects its
|
|
54
|
+
connection; supplying an integration file without an ID selects `assistant`.
|
|
55
|
+
The server passes one configuration shape: `systemPrompt`, optional
|
|
56
|
+
`integrationId`, `model` and `effort`. The runtime normalizes new configuration
|
|
57
|
+
inputs and uses the example's explicit `defaultIntegrationId: "assistant"` for
|
|
58
|
+
API/OpenCode; Claude/Codex retain native login when no file or ID is supplied.
|
|
59
|
+
Opening existing history does not rewrite its saved configuration.
|
|
60
|
+
The existing `ASSISTANT_NATIVE_MODEL` and `ASSISTANT_NATIVE_EFFORT` variables fill
|
|
61
|
+
the common model and effort fields for any engine. Model IDs stay exact and each
|
|
62
|
+
engine checks its supported choices; setting a model cannot override the
|
|
63
|
+
authorized connection's model.
|
|
64
|
+
|
|
65
|
+
The API starter can open and display empty conversations before a connection is
|
|
66
|
+
configured. Missing connections or credentials fail on Send without selecting
|
|
67
|
+
another model. An explicitly supplied invalid integration file fails setup.
|
|
68
|
+
`config.js` owns the five IDs, labels and home surface. Existing Planning and
|
|
69
|
+
Notes IDs and history paths are preserved.
|
|
70
|
+
|
|
71
|
+
History and configuration live in `ASSISTANT_STORAGE_DIRECTORY` (default
|
|
72
|
+
`.assistant/conversations`). Keep it private and retain it across restarts.
|
|
73
|
+
The supplied file adapter supports one writer process. Multi-process hosts must
|
|
74
|
+
supply their transactional store. Text drafts remain in the retained browser
|
|
75
|
+
conversation while any text or voice reader retains that target; unfinished
|
|
76
|
+
recordings remain voice-owned. A full page reload does not persist either draft.
|
|
77
|
+
|
|
78
|
+
For Claude or Codex, install its CLI and log in as the server's operating-system
|
|
79
|
+
identity. Use `claude auth login` and `ASSISTANT_ENGINE=claude`, or `codex login`
|
|
80
|
+
and `ASSISTANT_ENGINE=codex`. With no integration selected, omitted model and
|
|
81
|
+
effort use the CLI defaults. Alternatively supply the same integration file and
|
|
82
|
+
ID as API/OpenCode, using a provider/model that the native engine supports.
|
|
83
|
+
Changing the engine does not require a different configuration object.
|
|
84
|
+
Codex requires paginated history support; version 0.159.3 was verified earlier.
|
|
85
|
+
Codex conversations share the supplied account/runtime app-server while keeping
|
|
86
|
+
separate threads and Stop controls.
|
|
87
|
+
|
|
88
|
+
For OpenCode, install version 1.18.31 and set `ASSISTANT_ENGINE=opencode` with the
|
|
89
|
+
same integration configuration as the API path. JSKIT owns its loopback server,
|
|
90
|
+
private credential home, instruction hook and history handling. Preserve
|
|
91
|
+
`.assistant/native/opencode` beneath `ASSISTANT_WORKDIR` with application storage.
|
|
92
|
+
|
|
93
|
+
Native engines require a POSIX host. `ASSISTANT_WORKDIR` defaults to this directory
|
|
94
|
+
and must stay stable across restarts. Use a separate storage directory when
|
|
95
|
+
trying a different engine: saved conversations retain their engine and identity.
|
|
96
|
+
The example leaves native coding tools disabled. Goal controls appear only when
|
|
97
|
+
the runtime advertises them. Authorized model selection and native replacement
|
|
98
|
+
remain application policy; this example does not add a browser engine selector.
|
|
99
|
+
|
|
100
|
+
## Delivery, realtime and voice ownership
|
|
101
|
+
|
|
102
|
+
`server.js` calls `registerFastifyConversations` with its runtime, `appConfig`,
|
|
103
|
+
environment, explicit request authentication and bootstrap labels. This is
|
|
104
|
+
configurable shorthand for the existing JSKIT feature and HTTP/action/realtime
|
|
105
|
+
providers. The starter does not assemble those providers or wrap their router.
|
|
106
|
+
It still creates its file storage and runtime, initializes its five identities,
|
|
107
|
+
chooses models and checks the local Origin itself. It closes its runtime when
|
|
108
|
+
Fastify closes; the helper closes its hosting first.
|
|
109
|
+
|
|
110
|
+
No database runtime, Knex or migrations are used. Conversation
|
|
111
|
+
read/send/cancel/inspection/goal routes and socket subscriptions belong to the
|
|
112
|
+
supplied feature. Applications already hosted by JSKIT register `AssistantFeature`
|
|
113
|
+
on their existing host and shared connection instead of using the standalone
|
|
114
|
+
helper. Lower-level route/action/subscription facilities remain available for
|
|
115
|
+
custom hosts. The only example-specific data connection is its authorized speech
|
|
116
|
+
proxy.
|
|
117
|
+
|
|
118
|
+
`main.js` uses `bootClientModules`, `ShellWebClientProvider` and
|
|
119
|
+
`RealtimeClientProvider` for real placement, bootstrap and connection recovery.
|
|
120
|
+
`main.js` also creates one application HTTP/API client using the local server's
|
|
121
|
+
Origin policy. `Conversation.vue` renders `AssistantConversationClientElement`
|
|
122
|
+
with that shared client, the subject key from bootstrap and
|
|
123
|
+
`clear-draft-on="accepted"`. Its `composer-tools` slot receives the same retained
|
|
124
|
+
`runtime` used by the element and passes it to the voice button. The component
|
|
125
|
+
owns loading, errors, draft and delivery state; the example has no polling,
|
|
126
|
+
event parser, receipt map or adapter implementation.
|
|
127
|
+
|
|
128
|
+
`App.vue` binds the retained runtime to the existing
|
|
129
|
+
voice controller and `projectConversationVoiceState`. Stable spoken turn IDs
|
|
130
|
+
survive transient-to-saved output; voice uses the same send/cancel operations.
|
|
131
|
+
The standard component also exposes `focus()` to reopen the original text card.
|
|
132
|
+
Use `useAssistantConversation()` with `AssistantConversationElement` only when
|
|
133
|
+
custom presentation needs the lower-level adapter; the starter needs only the
|
|
134
|
+
standard component and its slot.
|
|
135
|
+
|
|
136
|
+
Closing one text card releases its observer without disconnecting its peers or
|
|
137
|
+
stopping server work. Reopening reads current state. Reconnect subscribes and
|
|
138
|
+
reads; it never resends a message. Lost delivery remains **Check delivery** until
|
|
139
|
+
canonical history or explicit inspection proves acceptance. Inspection uses the
|
|
140
|
+
supplied POST `/api/assistant/home/conversations/:conversationId/deliveries/:messageId/inspect`
|
|
141
|
+
and cannot dispatch another prompt. Provider failures remain errors, not saved
|
|
142
|
+
assistant answers. Interrupted or uncertain native work is not silently replayed.
|
|
143
|
+
A failed process cleanup blocks further work; restore its execution host and
|
|
144
|
+
retry Stop. An unknown process after a hard crash needs host-level cleanup.
|
|
145
|
+
|
|
146
|
+
This is a loopback, single-user example, not a hosted authentication system.
|
|
147
|
+
Writes and socket/voice handshakes require `APP_ORIGIN` (default
|
|
148
|
+
`http://127.0.0.1:5176`). Ordinary GETs may omit Origin; a supplied foreign Origin
|
|
149
|
+
is rejected. The helper applies this same host guard to canonical routes, actions
|
|
150
|
+
and conversation subscriptions. Server-only context identity permits
|
|
151
|
+
only the five browser targets; no actor, scope or room comes from request bodies.
|
|
152
|
+
Runtime authorization rechecks that host context for operations and observations.
|
|
153
|
+
The client's `local-user` scope comes from application bootstrap and is not a
|
|
154
|
+
fabricated authenticated actor. Local events carry no authenticated actor.
|
|
155
|
+
A hosted application supplies its authenticated request/action policy instead.
|
|
156
|
+
|
|
157
|
+
## A headless task in the same process
|
|
158
|
+
|
|
159
|
+
Optionally start the server with a direct task:
|
|
160
|
+
|
|
161
|
+
```sh
|
|
162
|
+
npm run server -- --headless "Give me three short planning questions."
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Use the same environment configuration as above. The server opens the separate
|
|
166
|
+
`headless` target through the same runtime and file store, calls `send()` and
|
|
167
|
+
`wait()` directly, and prints its result. No internal HTTP request, browser
|
|
168
|
+
subscription or extra store is involved. Browser access to this target is denied.
|
|
169
|
+
The task runs only when explicitly requested on that server invocation; startup
|
|
170
|
+
does not retry previous unfinished work. Leave the flag off on ordinary restart.
|
|
171
|
+
Do not start another server against the same file directory.
|
|
172
|
+
|
|
173
|
+
## Verification
|
|
174
|
+
|
|
175
|
+
`npm run build` checks the client build. Focused integration checks must exercise
|
|
176
|
+
the actual server and browser with fresh packed dependencies, including five
|
|
177
|
+
subscriptions on one socket, independent close/Stop, origin and target denial,
|
|
178
|
+
reconnect and inspection without replay, retained voice/text targeting, and the
|
|
179
|
+
headless path. Source migration is not evidence that those candidate checks passed.
|
|
180
|
+
Physical microphone, speaker and mobile hands-free behavior need real-device
|
|
181
|
+
checks; browser fixture responses do not establish real model or speech behavior.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { defineAssistantSurface } from "@jskit-ai/assistant-runtime/shared";
|
|
2
|
+
|
|
3
|
+
// Application-owned identities, labels and the existing home surface.
|
|
4
|
+
export const conversations = Object.freeze([
|
|
5
|
+
{ id: "planning", label: "Planning" },
|
|
6
|
+
{ id: "notes", label: "Notes" },
|
|
7
|
+
{ id: "research", label: "Research" },
|
|
8
|
+
{ id: "writing", label: "Writing" },
|
|
9
|
+
{ id: "review", label: "Review" }
|
|
10
|
+
]);
|
|
11
|
+
|
|
12
|
+
export const appConfig = Object.freeze({
|
|
13
|
+
...defineAssistantSurface("home"),
|
|
14
|
+
realtimeClient: { options: { transports: ["websocket"] } }
|
|
15
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<!doctype html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Voice conversations</title></head><body><div id="app"></div><script type="module" src="/main.js"></script></body></html>
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"registrations": {},
|
|
4
|
+
"integrations": {
|
|
5
|
+
"assistant": {
|
|
6
|
+
"provider": "ai",
|
|
7
|
+
"accountMode": "shared",
|
|
8
|
+
"scopes": [],
|
|
9
|
+
"authentication": {
|
|
10
|
+
"method": "api-key",
|
|
11
|
+
"secretRef": "env:ASSISTANT_API_KEY"
|
|
12
|
+
},
|
|
13
|
+
"settings": { "model": "deepseek/deepseek-flash" }
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { createApp, shallowRef } from "vue";
|
|
2
|
+
import { createPinia } from "pinia";
|
|
3
|
+
import { QueryClient, VueQueryPlugin } from "@tanstack/vue-query";
|
|
4
|
+
import { createRouter, createWebHistory } from "vue-router";
|
|
5
|
+
import { createVuetify } from "vuetify";
|
|
6
|
+
import * as components from "vuetify/components";
|
|
7
|
+
import * as directives from "vuetify/directives";
|
|
8
|
+
import { aliases, mdi } from "vuetify/iconsets/mdi-svg";
|
|
9
|
+
import { bootClientModules } from "@jskit-ai/kernel/client/moduleBootstrap";
|
|
10
|
+
import { defineProvider } from "@jskit-ai/kernel/shared/capabilities";
|
|
11
|
+
import { createSurfaceRuntime } from "@jskit-ai/kernel/shared/surface/runtime";
|
|
12
|
+
import { ShellWebClientProvider } from "@jskit-ai/shell-web/client";
|
|
13
|
+
import { RealtimeClientProvider } from "@jskit-ai/realtime/client";
|
|
14
|
+
import { configureAssistantConversations } from "@jskit-ai/assistant-runtime/client";
|
|
15
|
+
import { createHttpClient } from "@jskit-ai/http-runtime/client";
|
|
16
|
+
import "vuetify/styles";
|
|
17
|
+
import App from "./App.vue";
|
|
18
|
+
import { appConfig } from "./config.js";
|
|
19
|
+
|
|
20
|
+
globalThis.__JSKIT_CLIENT_APP_CONFIG__ = appConfig;
|
|
21
|
+
const example = shallowRef({ subjectId: "", conversations: [], provider: "Loading application configuration…" });
|
|
22
|
+
// This local app uses its existing Origin guard, rather than a hosted CSRF session.
|
|
23
|
+
const http = createHttpClient({ credentials: "include", csrf: { enabled: false } });
|
|
24
|
+
const ExampleClientProvider = defineProvider({
|
|
25
|
+
id: "example.client",
|
|
26
|
+
requires: { shell: "client.shell", vueApp: "client.vue" },
|
|
27
|
+
setup({ shell, vueApp }) {
|
|
28
|
+
shell.bootstrapHandlers.register({ handlerId: "example.application", applyBootstrapPayload({ payload }) {
|
|
29
|
+
example.value = payload.example;
|
|
30
|
+
} });
|
|
31
|
+
vueApp.provide("example.application", example);
|
|
32
|
+
return {};
|
|
33
|
+
}
|
|
34
|
+
});
|
|
35
|
+
const app = createApp(App);
|
|
36
|
+
configureAssistantConversations(app, {
|
|
37
|
+
actorKey: () => example.value.subjectId, request: http.request, clearDraftOn: "accepted"
|
|
38
|
+
});
|
|
39
|
+
const pinia = createPinia();
|
|
40
|
+
const queryClient = new QueryClient();
|
|
41
|
+
const router = createRouter({ history: createWebHistory(), routes: [{ path: "/:pathMatch(.*)*", component: App }] });
|
|
42
|
+
app.use(pinia).use(VueQueryPlugin, { queryClient }).use(router)
|
|
43
|
+
.use(createVuetify({ components, directives, icons: { defaultSet: "mdi", aliases, sets: { mdi } } }));
|
|
44
|
+
const modules = await bootClientModules({
|
|
45
|
+
app, pinia, queryClient, router, env: import.meta.env, surfaceMode: "home",
|
|
46
|
+
surfaceRuntime: createSurfaceRuntime({ defaultSurfaceId: "home", surfaces: appConfig.surfaceDefinitions }),
|
|
47
|
+
clientModules: [
|
|
48
|
+
{ packageId: "@jskit-ai/shell-web", module: { ShellWebClientProvider },
|
|
49
|
+
packageMetadataClientProviders: [{ export: "ShellWebClientProvider" }] },
|
|
50
|
+
{ packageId: "@jskit-ai/realtime", module: { RealtimeClientProvider },
|
|
51
|
+
packageMetadataClientProviders: [{ export: "RealtimeClientProvider" }] },
|
|
52
|
+
{ packageId: "example", module: { ExampleClientProvider },
|
|
53
|
+
packageMetadataClientProviders: [{ export: "ExampleClientProvider" }] }
|
|
54
|
+
]
|
|
55
|
+
});
|
|
56
|
+
await router.isReady();
|
|
57
|
+
app.mount("#app");
|
|
58
|
+
if (import.meta.hot) import.meta.hot.dispose(() => { app.unmount(); void modules.runtime.shutdown(); });
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "assistant-voice-example",
|
|
3
|
+
"private": true,
|
|
4
|
+
"type": "module",
|
|
5
|
+
"scripts": {
|
|
6
|
+
"dev": "vite --host=127.0.0.1 --port=5176 --strictPort",
|
|
7
|
+
"server": "node server.js",
|
|
8
|
+
"build": "vite build"
|
|
9
|
+
},
|
|
10
|
+
"dependencies": {
|
|
11
|
+
"@fastify/websocket": "^11.2.0",
|
|
12
|
+
"@jskit-ai/assistant-core": "0.1.202",
|
|
13
|
+
"@jskit-ai/assistant-runtime": "0.1.195",
|
|
14
|
+
"@jskit-ai/assistant-voice": "0.1.2",
|
|
15
|
+
"@jskit-ai/connectors-catalog": "0.1.41",
|
|
16
|
+
"@jskit-ai/http-runtime": "0.1.222",
|
|
17
|
+
"@jskit-ai/kernel": "0.1.224",
|
|
18
|
+
"@jskit-ai/realtime": "0.1.221",
|
|
19
|
+
"@jskit-ai/shell-web": "0.1.228",
|
|
20
|
+
"@mdi/js": "^7.4.47",
|
|
21
|
+
"@tanstack/vue-query": "^5.90.5",
|
|
22
|
+
"fastify": "^5.8.0",
|
|
23
|
+
"pinia": "^4.0.3",
|
|
24
|
+
"vue": "^3.5.43",
|
|
25
|
+
"vue-router": "^5.3.1",
|
|
26
|
+
"vuetify": "^4.0.0"
|
|
27
|
+
},
|
|
28
|
+
"devDependencies": {
|
|
29
|
+
"@vitejs/plugin-vue": "^6.0.7",
|
|
30
|
+
"vite": "^8.2.1"
|
|
31
|
+
}
|
|
32
|
+
}
|