@bojackduy/opencode-voice 0.10.1 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +121 -6
- package/index.js +40 -86
- package/lib/boot.js +106 -0
- package/lib/conversation.js +20 -7
- package/lib/stt.js +92 -13
- package/lib/tts.js +1 -1
- package/lib/v2-compat.js +561 -0
- package/lib/v2-preview.js +74 -0
- package/package.json +2 -1
- package/tui.js +14 -0
package/README.md
CHANGED
|
@@ -12,6 +12,112 @@ spoken aloud via Piper TTS. Both directions use an LLM to normalize text for
|
|
|
12
12
|
natural speech (fixing homophones, splitting camelCase identifiers, summarizing
|
|
13
13
|
code-heavy responses, etc.).
|
|
14
14
|
|
|
15
|
+
## OpenCode v1 and v2
|
|
16
|
+
|
|
17
|
+
One package runs on both OpenCode 1.x and 2.x. The plugin detects the host
|
|
18
|
+
by capability (never by version string) and adapts; the v1 path is unchanged.
|
|
19
|
+
|
|
20
|
+
> [!IMPORTANT]
|
|
21
|
+
> **Submit-only mode on v2.** OpenCode v2.0.26 exposes no plugin API for
|
|
22
|
+
> appending text to the composer for editing, so the v1 STT loop of
|
|
23
|
+
> transcribe → append → edit → submit is unimplementable there. On v2 every
|
|
24
|
+
> STT turn **transcribes and submits immediately** via `session.prompt` —
|
|
25
|
+
> both `/stt-record` and `/stt-submit` submit — and the turn toast says
|
|
26
|
+
> submitted, never "added to prompt". A warning toast on first load says the
|
|
27
|
+
> same. There is deliberately no edit step; nothing is faked.
|
|
28
|
+
|
|
29
|
+
### v1 (OpenCode 1.x)
|
|
30
|
+
|
|
31
|
+
Use the `tui.json` snippet under [Install](#install) — unchanged.
|
|
32
|
+
|
|
33
|
+
### v2 (OpenCode 2.x)
|
|
34
|
+
|
|
35
|
+
Add the plugin to `cli.json` (create at `~/.config/opencode/cli.json` if it
|
|
36
|
+
doesn't exist):
|
|
37
|
+
|
|
38
|
+
```json
|
|
39
|
+
{
|
|
40
|
+
"$schema": "https://opencode.ai/v2/cli.json",
|
|
41
|
+
"plugins": [
|
|
42
|
+
{
|
|
43
|
+
"package": "@bojackduy/opencode-voice",
|
|
44
|
+
"options": {
|
|
45
|
+
"endpoint": "https://api.anthropic.com/v1",
|
|
46
|
+
"model": "claude-haiku-4-5",
|
|
47
|
+
"apiKeyEnv": "ANTHROPIC_API_KEY"
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
]
|
|
51
|
+
}
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Every plugin option documented below (`sttLanguage`, `ttsEngine`,
|
|
55
|
+
`keybinds`, …) works the same on v2 — options ride `context.options`, and
|
|
56
|
+
the `keybinds` command ids are stable across hosts. To test unpublished
|
|
57
|
+
changes, point `package` at the local repo path instead of the npm name.
|
|
58
|
+
|
|
59
|
+
To verify the plugin loaded: `opencode plugin list` shows it as a `local`
|
|
60
|
+
package (its TUI entrypoint is a literal `tui.js` file — v2 resolves local
|
|
61
|
+
packages by file probe, not by the `exports` map, so that file must exist).
|
|
62
|
+
Then press `Ctrl+P` or type `/` in the composer: all 19 commands are there
|
|
63
|
+
under the `opencode-voice` group with their slash names (`/stt-record`,
|
|
64
|
+
`/tts-speak`, `/voice-cancel`, …), verified live on 2.0.26.
|
|
65
|
+
|
|
66
|
+
> [!NOTE]
|
|
67
|
+
> **Default keybinds on v2.** The same leader shortcuts work on both hosts:
|
|
68
|
+
> press `Ctrl+X`, then `[` to record/stop, `]` to read the reply, `;` to stop
|
|
69
|
+
> playback, `v` for conversation, or `.` to cancel. The v2 keymap resolves
|
|
70
|
+
> `<leader>` itself, including a customized host leader. Record/stop and cancel
|
|
71
|
+
> dispatch were verified on v2.0.26 on both home and session screens. Plugin
|
|
72
|
+
> `keybinds` overrides and `"none"` still apply; palette and `/slash` remain
|
|
73
|
+
> available.
|
|
74
|
+
|
|
75
|
+
`engines` status: the package declares no `engines.opencode` field, so both
|
|
76
|
+
v1 and v2 load it unconstrained. A version pin was considered and skipped on
|
|
77
|
+
purpose: it could refuse to load on untested hosts for no benefit, since
|
|
78
|
+
capability detection already picks the right path per host.
|
|
79
|
+
|
|
80
|
+
### Per-feature v2 caveats
|
|
81
|
+
|
|
82
|
+
- **Commands:** all 19 register with the same ids, titles, palette group,
|
|
83
|
+
slash names, and keybinds as v1. On 2.0.26 the `/slash` completion lists
|
|
84
|
+
every one reliably; the `Ctrl+P` palette shows them for matching queries
|
|
85
|
+
but its search occasionally reports no results for queries that should
|
|
86
|
+
match — when in doubt, use `/slash`.
|
|
87
|
+
- **STT (batch):** submits immediately, no edit step (see above). Recording
|
|
88
|
+
from home creates a session only when final text is ready, opens it, and
|
|
89
|
+
submits once. The selected model is carried into the new session; existing
|
|
90
|
+
sessions retain their agent/model. On home, v2's public plugin API exposes
|
|
91
|
+
the selected model but no selected-agent getter, so session creation uses
|
|
92
|
+
the host's default agent. Create/submit failures show the full transcript in
|
|
93
|
+
a recovery dialog. Cancelling before submission drops the captured text.
|
|
94
|
+
- **STT (streaming dictation):** enable with `"sttMode": "streaming"` in plugin
|
|
95
|
+
options. Local whisper streaming displays a bounded live transcript above
|
|
96
|
+
the prompt footer on home and session screens. Stable words are plain;
|
|
97
|
+
the tentative tail is bracketed and may change. Press the record key again
|
|
98
|
+
to finalize and submit once, or cancel to clear the preview without sending.
|
|
99
|
+
No composer edit step is promised on v2. Batch remains the default and
|
|
100
|
+
produces text after recording stops; the hosted `sttEndpoint` option is for
|
|
101
|
+
batch transcription, not this local streaming preview.
|
|
102
|
+
- **TTS / conversation / live notes / cancel:** same semantics as v1 (auto
|
|
103
|
+
TTS on idle/permission/question gates, conversation barge-in, live-notes
|
|
104
|
+
background saves, one-key cancel).
|
|
105
|
+
- **Reply-streaming triggers:** `message.part.delta` is remapped to v2's
|
|
106
|
+
`session.text.delta`; `session.idle`, `session.status`, and
|
|
107
|
+
`permission.asked` pass through unchanged. `question.asked` has no v2
|
|
108
|
+
equivalent and is mapped best-effort to `form.created`: with several
|
|
109
|
+
sessions open, a form in one session can pause speech in another.
|
|
110
|
+
Single-session use is unaffected.
|
|
111
|
+
- **Pickers** (`/stt-model`, `/stt-language`, `/stt-mic`, `/tts-voice`,
|
|
112
|
+
`/voice-model`): same options through promise dialogs, including the
|
|
113
|
+
nested provider → model pick.
|
|
114
|
+
- **Settings persistence:** stored via v2 storage (survives restarts, syncs
|
|
115
|
+
across windows). If storage is unavailable the plugin keeps working with
|
|
116
|
+
memory-only settings for the session.
|
|
117
|
+
- **Logging:** v2 has no `client.app.log`, so the debug/warn/error
|
|
118
|
+
diagnostics the plugin writes to the OpenCode log on v1 are silent on v2.
|
|
119
|
+
User-facing warnings still arrive as toasts.
|
|
120
|
+
|
|
15
121
|
## Install
|
|
16
122
|
|
|
17
123
|
Add to your `tui.json` (create at `~/.config/opencode/tui.json` if it doesn't
|
|
@@ -641,7 +747,8 @@ Idle invocations toast `Nothing to cancel`. Keybind override:
|
|
|
641
747
|
|
|
642
748
|
Live local dictation for short prompts: the mic stays on, partial
|
|
643
749
|
transcriptions stream in as you speak, and one press of `/stt-record`
|
|
644
|
-
finalizes the text into the prompt box
|
|
750
|
+
finalizes the text into the prompt box on v1, or submits it once on v2.
|
|
751
|
+
Set `"sttMode": "streaming"` in the
|
|
645
752
|
plugin options (default `"batch"` keeps the record-then-transcribe flow
|
|
646
753
|
exactly as before).
|
|
647
754
|
|
|
@@ -655,7 +762,11 @@ What you see: a sticky status toast stays up for the whole session so
|
|
|
655
762
|
there is never a silent gap — `Loading speech model…` during cold start,
|
|
656
763
|
`● Streaming dictation — listening` with an elapsed timer while live
|
|
657
764
|
(plus a `(listening…)` note when inference runs slow), and `Finalizing…`
|
|
658
|
-
between the second keypress and
|
|
765
|
+
between the second keypress and finalization. On v2, a bounded transcript
|
|
766
|
+
above the prompt footer updates during recording: stable words are plain,
|
|
767
|
+
tentative words are bracketed. Finalization submits directly to the active
|
|
768
|
+
session (or creates one from home); cancel clears the preview without sending.
|
|
769
|
+
On v1, the OpenCode renderer only
|
|
659
770
|
exposes `insertText`/`submit`, with no live range-replacement API, so
|
|
660
771
|
partials render as preview toasts (`🎙 ...`) alongside (not instead of)
|
|
661
772
|
the sticky status, and the full text is inserted EXACTLY ONCE at
|
|
@@ -685,8 +796,9 @@ Options:
|
|
|
685
796
|
- `sttMode` _(optional, default `"batch"`)_ - `"streaming"` selects this mode
|
|
686
797
|
- `sttStreamWindowMs` _(optional, default `10000`)_ - rolling audio window
|
|
687
798
|
- `sttStreamStepMs` _(optional, default `1000`)_ - live transcription cadence
|
|
688
|
-
- `sttAutoSubmit` applies after streaming finalize (submit via the
|
|
689
|
-
field only, as above)
|
|
799
|
+
- `sttAutoSubmit` applies after streaming finalize on v1 (submit via the
|
|
800
|
+
dictated field only, as above). v2 always submits final text because there
|
|
801
|
+
is no composer append/edit API.
|
|
690
802
|
|
|
691
803
|
Measured latencies (Apple M3 Pro, 36 GB, whisper.cpp 1.9.4 with Metal,
|
|
692
804
|
`ggml-large-v3-turbo-q5_0.bin`, scripted speech through the real
|
|
@@ -707,11 +819,14 @@ at roughly that rate, not every cadence tick — the scheduler coalesces
|
|
|
707
819
|
while inference is busy, so partials are flicker-free but not instant.
|
|
708
820
|
Only one smaller-model data point exists (none: no smaller model file is
|
|
709
821
|
installed locally, and nothing was downloaded for measurement). E2E
|
|
710
|
-
coverage is mic-less
|
|
822
|
+
coverage is mic-less: v1 session flows are proven
|
|
711
823
|
through the real command paths with faked capture/renderer
|
|
712
824
|
(`test/streaming-e2e.test.js`, `test/streaming-gaps.test.js`), while the
|
|
713
825
|
numbers above come from the real server path with scripted audio — no
|
|
714
|
-
live-mic session was measured.
|
|
826
|
+
live-mic session was measured. v2 preview rendering was additionally verified
|
|
827
|
+
in the real v2.0.26 TUI using injected stable/tentative text on home and
|
|
828
|
+
session screens; final-once/cancellation flows use scripted capture and the
|
|
829
|
+
real streaming controller in `test/v2-preview.test.js`.
|
|
715
830
|
|
|
716
831
|
## How it works
|
|
717
832
|
|
package/index.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
// opencode-voice: Speech-to-text and text-to-speech for OpenCode.
|
|
2
2
|
//
|
|
3
3
|
// STT: Record voice via sox, transcribe with whisper-cpp, normalize with
|
|
4
|
-
// an OpenAI-compatible LLM, append to the TUI prompt
|
|
4
|
+
// an OpenAI-compatible LLM, append to the TUI prompt (v1) or submit
|
|
5
|
+
// immediately via session.prompt (v2 submit-only mode, see README
|
|
6
|
+
// "OpenCode v1 and v2").
|
|
5
7
|
//
|
|
6
8
|
// TTS: Auto-speak assistant responses (or read on demand) via Piper,
|
|
7
9
|
// with LLM normalization for natural speech.
|
|
@@ -38,96 +40,48 @@
|
|
|
38
40
|
// Weird keys [ ] ; were chosen because opencode doesn't use them and shift variants were ignored in terminals.
|
|
39
41
|
// <leader>v is free in opencode defaults (c/e/s/m/a/y/u/r/h etc. are taken) and mnemonic for voice.
|
|
40
42
|
|
|
41
|
-
import
|
|
42
|
-
import
|
|
43
|
-
import { registerSTT } from "./lib/stt.js";
|
|
44
|
-
import { registerTTS } from "./lib/tts.js";
|
|
45
|
-
import { registerConversation } from "./lib/conversation.js";
|
|
46
|
-
import { registerLiveNotes } from "./lib/live-notes.js";
|
|
47
|
-
import { registerVoiceCancel } from "./lib/cancel.js";
|
|
48
|
-
import { registerVoiceModel, resolveVoiceProviderModel } from "./lib/voice-model.js";
|
|
49
|
-
import { createClient } from "./lib/llm-client.js";
|
|
50
|
-
import { createLogger } from "./lib/logger.js";
|
|
51
|
-
|
|
52
|
-
function loadPromptFile(filePath, logger, name) {
|
|
53
|
-
if (!filePath) return null;
|
|
54
|
-
const resolved = filePath.replace(/^~(?=\/|$)/, os.homedir());
|
|
55
|
-
try {
|
|
56
|
-
const prompt = fs.readFileSync(resolved, "utf-8").trim() || null;
|
|
57
|
-
logger?.log(
|
|
58
|
-
"plugin",
|
|
59
|
-
prompt ? `Loaded ${name} prompt: ${resolved}` : `Ignored empty ${name} prompt: ${resolved}`,
|
|
60
|
-
"debug",
|
|
61
|
-
);
|
|
62
|
-
return prompt;
|
|
63
|
-
} catch (err) {
|
|
64
|
-
logger?.log("Plugin", `Failed to load ${name} prompt ${resolved}: ${err.message}`, "warn");
|
|
65
|
-
return null;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
43
|
+
import { bootVoice } from "./lib/boot.js";
|
|
44
|
+
import { adaptV2Context } from "./lib/v2-compat.js";
|
|
68
45
|
|
|
46
|
+
// Dual-compat export: v1's loader reads `tui`, v2's reads `setup`
|
|
47
|
+
// (`Plugin.define` is an identity wrapper, so a plain object carries both).
|
|
48
|
+
// Each host ignores the other's key. The core (`bootVoice`) is shared; v2
|
|
49
|
+
// runs it over the v2-compat adapted api in submit-only mode.
|
|
69
50
|
export default {
|
|
70
51
|
id: "opencode-voice",
|
|
71
52
|
tui: async (api, options) => {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
53
|
+
await bootVoice(api, options);
|
|
54
|
+
},
|
|
55
|
+
setup: async (ctx) => {
|
|
56
|
+
const adapted = adaptV2Context(ctx);
|
|
57
|
+
await bootVoice(adapted.api, ctx.options, {
|
|
58
|
+
submitOnly: true,
|
|
59
|
+
onReady: async ({ logger }) => {
|
|
60
|
+
// First-run submit-only notice (required UX disclosure): v2 has no
|
|
61
|
+
// append-to-composer API, so every STT turn submits immediately.
|
|
62
|
+
const notice =
|
|
63
|
+
"Voice on OpenCode v2: transcripts submit immediately as-is " +
|
|
64
|
+
"(submit-only mode — append-for-edit is unavailable on v2). " +
|
|
65
|
+
"For live preview, set sttMode to streaming.";
|
|
66
|
+
adapted.api.ui.toast({ message: notice, variant: "warning", duration: 8000 });
|
|
67
|
+
logger?.log?.("plugin", `v2 submit-only mode: ${notice}`, "warn");
|
|
68
|
+
// v2 has no client.app.log, so the logger above is a no-op: compat
|
|
69
|
+
// warnings must surface as a toast or they are lost entirely.
|
|
70
|
+
if (adapted.warnMessages.length > 0) {
|
|
71
|
+
const shown = adapted.warnMessages.slice(0, 3).join(" | ");
|
|
72
|
+
const rest =
|
|
73
|
+
adapted.warnMessages.length > 3 ? ` (+${adapted.warnMessages.length - 3} more)` : "";
|
|
74
|
+
adapted.api.ui.toast({
|
|
75
|
+
message: `Voice v2 compat: ${shown}${rest}`,
|
|
76
|
+
variant: "warning",
|
|
77
|
+
duration: 8000,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
for (const message of adapted.warnMessages) {
|
|
81
|
+
logger?.log?.("plugin", `v2-compat: ${message}`, "warn");
|
|
82
|
+
}
|
|
85
83
|
},
|
|
86
|
-
() => resolveVoiceProviderModel(api, kv),
|
|
87
|
-
);
|
|
88
|
-
|
|
89
|
-
const prompts = {
|
|
90
|
-
stt: loadPromptFile(options?.sttPrompt, logger, "STT"),
|
|
91
|
-
ttsAuto: loadPromptFile(options?.ttsAutoPrompt, logger, "TTS auto"),
|
|
92
|
-
ttsManual: loadPromptFile(options?.ttsManualPrompt, logger, "TTS manual"),
|
|
93
|
-
};
|
|
94
|
-
|
|
95
|
-
const shared = {};
|
|
96
|
-
const stt = registerSTT(api, kv, complete, prompts, options, logger, {
|
|
97
|
-
isConversationActive: () => shared.conversation?.isActive() === true,
|
|
98
|
-
onConversationKey: (source) => shared.conversation?.onKey(source),
|
|
99
|
-
isLiveNotesActive: () => shared.liveNotes?.isActive() === true,
|
|
100
|
-
});
|
|
101
|
-
const tts = registerTTS(api, kv, complete, prompts, options, logger, {
|
|
102
|
-
isConversationActive: () => shared.conversation?.isActive() === true,
|
|
103
|
-
onConversationStopKey: () => shared.conversation?.onTtsStop() === true,
|
|
104
84
|
});
|
|
105
|
-
|
|
106
|
-
stt: stt.controller,
|
|
107
|
-
tts: tts.controller,
|
|
108
|
-
isLiveNotesActive: () => shared.liveNotes?.isActive() === true,
|
|
109
|
-
});
|
|
110
|
-
shared.conversation = conversation.controller;
|
|
111
|
-
const liveNotes = registerLiveNotes(api, kv, complete, options, logger, {
|
|
112
|
-
tts: tts.controller,
|
|
113
|
-
isConversationActive: () => shared.conversation?.isActive() === true,
|
|
114
|
-
});
|
|
115
|
-
shared.liveNotes = liveNotes.controller;
|
|
116
|
-
const voiceModel = registerVoiceModel(api, kv, options, logger);
|
|
117
|
-
const cancel = registerVoiceCancel(api, options, logger, {
|
|
118
|
-
stt: stt.controller,
|
|
119
|
-
tts: tts.controller,
|
|
120
|
-
conversation: shared.conversation,
|
|
121
|
-
liveNotes: shared.liveNotes,
|
|
122
|
-
});
|
|
123
|
-
|
|
124
|
-
api.command.register(() => [
|
|
125
|
-
...stt.commands,
|
|
126
|
-
...tts.commands,
|
|
127
|
-
...conversation.commands,
|
|
128
|
-
...liveNotes.commands,
|
|
129
|
-
...voiceModel.commands,
|
|
130
|
-
...cancel.commands,
|
|
131
|
-
]);
|
|
85
|
+
return adapted.cleanup;
|
|
132
86
|
},
|
|
133
87
|
};
|
package/lib/boot.js
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// Shared plugin boot for OpenCode v1 (`tui`) and v2 (`setup`).
|
|
2
|
+
//
|
|
3
|
+
// WHY this file exists: v1 and v2 entrypoints differ only in how they obtain
|
|
4
|
+
// an `api` object (raw v1 `TuiPluginApi` vs the v2-compat adapted `Context`).
|
|
5
|
+
// Everything below runs identically on both, so there is exactly one core to
|
|
6
|
+
// maintain. The v1 `tui` path calls `bootVoice(api, options)` with no extras
|
|
7
|
+
// and behaves exactly as before; the v2 `setup` path passes
|
|
8
|
+
// `{ submitOnly: true }` for the first-run notice. No STT/TTS/routing/
|
|
9
|
+
// keybind behavior branches live here.
|
|
10
|
+
|
|
11
|
+
import fs from "node:fs";
|
|
12
|
+
import os from "node:os";
|
|
13
|
+
import { registerSTT } from "./stt.js";
|
|
14
|
+
import { registerTTS } from "./tts.js";
|
|
15
|
+
import { registerConversation } from "./conversation.js";
|
|
16
|
+
import { registerLiveNotes } from "./live-notes.js";
|
|
17
|
+
import { registerVoiceCancel } from "./cancel.js";
|
|
18
|
+
import { registerVoiceModel, resolveVoiceProviderModel } from "./voice-model.js";
|
|
19
|
+
import { createClient } from "./llm-client.js";
|
|
20
|
+
import { createLogger } from "./logger.js";
|
|
21
|
+
|
|
22
|
+
function loadPromptFile(filePath, logger, name) {
|
|
23
|
+
if (!filePath) return null;
|
|
24
|
+
const resolved = filePath.replace(/^~(?=\/|$)/, os.homedir());
|
|
25
|
+
try {
|
|
26
|
+
const prompt = fs.readFileSync(resolved, "utf-8").trim() || null;
|
|
27
|
+
logger?.log(
|
|
28
|
+
"plugin",
|
|
29
|
+
prompt ? `Loaded ${name} prompt: ${resolved}` : `Ignored empty ${name} prompt: ${resolved}`,
|
|
30
|
+
"debug",
|
|
31
|
+
);
|
|
32
|
+
return prompt;
|
|
33
|
+
} catch (err) {
|
|
34
|
+
logger?.log("Plugin", `Failed to load ${name} prompt ${resolved}: ${err.message}`, "warn");
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export async function bootVoice(api, options, extras = {}) {
|
|
40
|
+
const { kv } = api;
|
|
41
|
+
const logger = createLogger(api.client);
|
|
42
|
+
logger.log("plugin", "Initializing", "debug");
|
|
43
|
+
// Session-scoped gateways (opencode.ai/zen/go) require x-opencode-session.
|
|
44
|
+
// Read the live route at call time so normalize works in any session.
|
|
45
|
+
// The voice-selected provider (/voice-model) fills endpoint/model only
|
|
46
|
+
// when the explicit options leave them out.
|
|
47
|
+
const { complete } = createClient(
|
|
48
|
+
options,
|
|
49
|
+
logger,
|
|
50
|
+
() => {
|
|
51
|
+
const route = api?.route?.current;
|
|
52
|
+
return route?.name === "session" ? route?.params?.sessionID : undefined;
|
|
53
|
+
},
|
|
54
|
+
() => resolveVoiceProviderModel(api, kv),
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
const prompts = {
|
|
58
|
+
stt: loadPromptFile(options?.sttPrompt, logger, "STT"),
|
|
59
|
+
ttsAuto: loadPromptFile(options?.ttsAutoPrompt, logger, "TTS auto"),
|
|
60
|
+
ttsManual: loadPromptFile(options?.ttsManualPrompt, logger, "TTS manual"),
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
const shared = {};
|
|
64
|
+
const stt = registerSTT(api, kv, complete, prompts, options, logger, {
|
|
65
|
+
isConversationActive: () => shared.conversation?.isActive() === true,
|
|
66
|
+
onConversationKey: (source) => shared.conversation?.onKey(source),
|
|
67
|
+
isLiveNotesActive: () => shared.liveNotes?.isActive() === true,
|
|
68
|
+
});
|
|
69
|
+
const tts = registerTTS(api, kv, complete, prompts, options, logger, {
|
|
70
|
+
isConversationActive: () => shared.conversation?.isActive() === true,
|
|
71
|
+
onConversationStopKey: () => shared.conversation?.onTtsStop() === true,
|
|
72
|
+
});
|
|
73
|
+
const conversation = registerConversation(api, options, logger, {
|
|
74
|
+
stt: stt.controller,
|
|
75
|
+
tts: tts.controller,
|
|
76
|
+
isLiveNotesActive: () => shared.liveNotes?.isActive() === true,
|
|
77
|
+
});
|
|
78
|
+
shared.conversation = conversation.controller;
|
|
79
|
+
const liveNotes = registerLiveNotes(api, kv, complete, options, logger, {
|
|
80
|
+
tts: tts.controller,
|
|
81
|
+
isConversationActive: () => shared.conversation?.isActive() === true,
|
|
82
|
+
});
|
|
83
|
+
shared.liveNotes = liveNotes.controller;
|
|
84
|
+
const voiceModel = registerVoiceModel(api, kv, options, logger);
|
|
85
|
+
const cancel = registerVoiceCancel(api, options, logger, {
|
|
86
|
+
stt: stt.controller,
|
|
87
|
+
tts: tts.controller,
|
|
88
|
+
conversation: shared.conversation,
|
|
89
|
+
liveNotes: shared.liveNotes,
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
api.command.register(() => [
|
|
93
|
+
...stt.commands,
|
|
94
|
+
...tts.commands,
|
|
95
|
+
...conversation.commands,
|
|
96
|
+
...liveNotes.commands,
|
|
97
|
+
...voiceModel.commands,
|
|
98
|
+
...cancel.commands,
|
|
99
|
+
]);
|
|
100
|
+
|
|
101
|
+
if (extras?.submitOnly && typeof extras?.onReady === "function") {
|
|
102
|
+
await extras.onReady({ logger, stt, tts, conversation, liveNotes, voiceModel, cancel });
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return { stt, tts, conversation, liveNotes, voiceModel, cancel, logger };
|
|
106
|
+
}
|
package/lib/conversation.js
CHANGED
|
@@ -235,10 +235,12 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
235
235
|
try {
|
|
236
236
|
if (!active || phase !== "waiting") return;
|
|
237
237
|
const props = event.properties || {};
|
|
238
|
-
|
|
238
|
+
const expectedID = typeof sessionID === "function" ? sessionID() : sessionID;
|
|
239
|
+
if (typeof sessionID === "function" && !expectedID) return;
|
|
240
|
+
if (expectedID && props.sessionID && props.sessionID !== expectedID) return;
|
|
239
241
|
if (typeof props.delta !== "string" || !props.delta) return;
|
|
240
242
|
if (props.field && props.field !== "text") return;
|
|
241
|
-
const lookupID = props.sessionID ||
|
|
243
|
+
const lookupID = props.sessionID || expectedID;
|
|
242
244
|
const messages = api?.state?.session?.messages?.(lookupID) || [];
|
|
243
245
|
const msg = messages.find((m) => m?.id === props.messageID);
|
|
244
246
|
if (!msg || msg.role !== "assistant") return;
|
|
@@ -302,8 +304,11 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
302
304
|
cancelFn = (outcome) => finish(outcome || "stopped");
|
|
303
305
|
waitCancel = cancelFn;
|
|
304
306
|
|
|
305
|
-
const forSession = (props) =>
|
|
306
|
-
|
|
307
|
+
const forSession = (props) => {
|
|
308
|
+
const expectedID = typeof sessionID === "function" ? sessionID() : sessionID;
|
|
309
|
+
if (typeof sessionID === "function" && !expectedID) return false;
|
|
310
|
+
return !expectedID || !props?.sessionID || props.sessionID === expectedID;
|
|
311
|
+
};
|
|
307
312
|
|
|
308
313
|
unsubs = [
|
|
309
314
|
api.event.on("session.idle", (event) => {
|
|
@@ -364,6 +369,15 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
364
369
|
return;
|
|
365
370
|
}
|
|
366
371
|
|
|
372
|
+
const turnStart = Date.now();
|
|
373
|
+
// v2 prompt dispatch may emit the entire reply before its promise returns.
|
|
374
|
+
// Home creation navigates before prompting, so resolve the ID at event time.
|
|
375
|
+
let earlyWait = null;
|
|
376
|
+
if (api.client?.tui?.submitOnly) {
|
|
377
|
+
phase = "waiting";
|
|
378
|
+
startReplyStream(currentSessionID, turnStart);
|
|
379
|
+
earlyWait = waitForReply(currentSessionID);
|
|
380
|
+
}
|
|
367
381
|
const submitted = await stt.submitTurnText(res.text);
|
|
368
382
|
if (!active || myGen !== generation) return;
|
|
369
383
|
if (submitted.error) {
|
|
@@ -373,11 +387,10 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
373
387
|
|
|
374
388
|
phase = "waiting";
|
|
375
389
|
const sessionID = currentSessionID();
|
|
376
|
-
const turnStart = Date.now();
|
|
377
390
|
showProcessingToast(`Working - press ${keyLabel()} to talk`);
|
|
378
|
-
startReplyStream(sessionID, turnStart);
|
|
391
|
+
if (!earlyWait) startReplyStream(sessionID, turnStart);
|
|
379
392
|
const tWait = Date.now();
|
|
380
|
-
const outcome = await waitForReply(sessionID);
|
|
393
|
+
const outcome = await (earlyWait || waitForReply(sessionID));
|
|
381
394
|
const waitMs = Date.now() - tWait;
|
|
382
395
|
endReplyStream();
|
|
383
396
|
logger?.log(
|