@kidsinai/kids-client 0.0.17 → 0.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -3
- package/src/core/commands.ts +105 -0
- package/src/core/events.ts +50 -28
- package/src/core/models.ts +66 -0
- package/src/core/session.ts +36 -0
- package/src/core/store.ts +25 -0
- package/src/core/voice/controller.ts +116 -0
- package/src/core/voice/recorder.ts +114 -0
- package/src/core/voice/state.ts +92 -0
- package/src/core/voice/stt.ts +118 -0
- package/src/core/voice/vad.ts +92 -0
- package/src/index.tsx +99 -2
- package/src/render/ink/App.tsx +58 -3
- package/src/render/ink/components/CommandSuggestions.tsx +47 -0
- package/src/render/ink/components/Input.tsx +1 -1
- package/src/render/ink/components/PickList.tsx +62 -0
- package/src/render/ink/screens/MissionScreen.tsx +95 -24
- package/src/render/ink/useVoiceInput.ts +146 -0
- package/src/voice-demo.tsx +80 -0
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json.schemastore.org/package.json",
|
|
3
3
|
"name": "@kidsinai/kids-client",
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.19",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Own-client TUI for Kids OpenCode — talks to local `opencode serve` via @opencode-ai/sdk v2 with kid-warm rendering, mission progress, permission dialog, and stderr-tail audit pipeline.",
|
|
7
7
|
"license": "MIT",
|
|
@@ -24,7 +24,8 @@
|
|
|
24
24
|
"files": ["src", "bin", "README.md", "LICENSE"],
|
|
25
25
|
"scripts": {
|
|
26
26
|
"typecheck": "tsc --noEmit",
|
|
27
|
-
"test": "bun test"
|
|
27
|
+
"test": "bun test",
|
|
28
|
+
"voice-demo": "bun src/voice-demo.tsx"
|
|
28
29
|
},
|
|
29
30
|
"peerDependencies": {
|
|
30
31
|
"@opencode-ai/sdk": ">=1.14.0"
|
|
@@ -34,7 +35,7 @@
|
|
|
34
35
|
"ink-spinner": "^5.0.0",
|
|
35
36
|
"ink-text-input": "^6.0.0",
|
|
36
37
|
"react": "^18.3.1",
|
|
37
|
-
"@kidsinai/kids-opencode-plugin": "^0.0.
|
|
38
|
+
"@kidsinai/kids-opencode-plugin": "^0.0.19"
|
|
38
39
|
},
|
|
39
40
|
"devDependencies": {
|
|
40
41
|
"@opencode-ai/sdk": "^1.14.51",
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Kid-safe slash-command registry.
|
|
3
|
+
*
|
|
4
|
+
* kids-client is a custom minimal TUI (not opencode's native one), so it grows
|
|
5
|
+
* its OWN command set rather than exposing opencode's full palette. This module
|
|
6
|
+
* is pure metadata + parsing — the actual effects live in index.tsx's prompt
|
|
7
|
+
* dispatcher (it needs the store / session / client). Keeping it pure makes the
|
|
8
|
+
* list unit-testable and lets MissionScreen render live suggestions.
|
|
9
|
+
*
|
|
10
|
+
* Deliberately EXCLUDED for kid-safety: /share (public links), /editor &
|
|
11
|
+
* drop-to-shell, external @-mentions, arbitrary config edits. The server-side
|
|
12
|
+
* tool whitelist (kids-plugin system prompt) blocks the dangerous primitives
|
|
13
|
+
* anyway; we just don't surface the affordances.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
export type Locale = "zh-Hans" | "en"
|
|
17
|
+
|
|
18
|
+
export interface KidCommand {
|
|
19
|
+
id: string
|
|
20
|
+
/** Primary slash form, e.g. "/model". */
|
|
21
|
+
slash: string
|
|
22
|
+
/** Extra accepted spellings, e.g. ["/models"]. */
|
|
23
|
+
aliases?: string[]
|
|
24
|
+
label: { en: string; zh: string }
|
|
25
|
+
hint: { en: string; zh: string }
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Order here is the order shown in the suggestion list / /help. */
|
|
29
|
+
export const COMMANDS: KidCommand[] = [
|
|
30
|
+
{ id: "help", slash: "/help", aliases: ["/?", "/commands"],
|
|
31
|
+
label: { en: "Help", zh: "帮助" },
|
|
32
|
+
hint: { en: "Show what you can type", zh: "看看能输入哪些命令" } },
|
|
33
|
+
{ id: "model", slash: "/model", aliases: ["/models"],
|
|
34
|
+
label: { en: "Pick AI model", zh: "选 AI 模型" },
|
|
35
|
+
hint: { en: "Choose which AI helps you", zh: "换一个帮你的 AI 模型" } },
|
|
36
|
+
{ id: "sessions", slash: "/sessions", aliases: ["/history", "/chats"],
|
|
37
|
+
label: { en: "Past chats", zh: "历史对话" },
|
|
38
|
+
hint: { en: "Open an earlier conversation", zh: "打开之前的对话" } },
|
|
39
|
+
{ id: "new", slash: "/new",
|
|
40
|
+
label: { en: "New chat", zh: "开新对话" },
|
|
41
|
+
hint: { en: "Start a fresh conversation", zh: "清空,开始一段新的对话" } },
|
|
42
|
+
{ id: "check", slash: "/check", aliases: ["/done"],
|
|
43
|
+
label: { en: "Check my work", zh: "验收作品" },
|
|
44
|
+
hint: { en: "See if you finished the mission", zh: "看看这一关完成没有" } },
|
|
45
|
+
{ id: "clear", slash: "/clear",
|
|
46
|
+
label: { en: "Clear screen", zh: "清屏" },
|
|
47
|
+
hint: { en: "Clear the chat (your files stay)", zh: "清空聊天记录(文件不动)" } },
|
|
48
|
+
{ id: "menu", slash: "/menu", aliases: ["/home"],
|
|
49
|
+
label: { en: "Main menu", zh: "主菜单" },
|
|
50
|
+
hint: { en: "Back to the start screen", zh: "回到开始界面" } },
|
|
51
|
+
{ id: "quit", slash: "/quit", aliases: ["/exit"],
|
|
52
|
+
label: { en: "Quit", zh: "退出" },
|
|
53
|
+
hint: { en: "Close Kids OpenCode", zh: "关闭 Kids OpenCode" } },
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
export interface ParsedCommand {
|
|
57
|
+
/** The slash token as typed, lowercased, incl. leading "/", e.g. "/model". */
|
|
58
|
+
name: string
|
|
59
|
+
/** Everything after the command word, trimmed (may be ""). */
|
|
60
|
+
args: string
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Parse a slash command line. Returns null if `text` isn't a `/command`. */
|
|
64
|
+
export function parseSlash(text: string): ParsedCommand | null {
|
|
65
|
+
const t = text.trim()
|
|
66
|
+
if (!t.startsWith("/")) return null
|
|
67
|
+
const space = t.indexOf(" ")
|
|
68
|
+
if (space === -1) return { name: t.toLowerCase(), args: "" }
|
|
69
|
+
return { name: t.slice(0, space).toLowerCase(), args: t.slice(space + 1).trim() }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Resolve a parsed command name (e.g. "/models") to its KidCommand. */
|
|
73
|
+
export function matchCommand(name: string): KidCommand | null {
|
|
74
|
+
const n = name.toLowerCase()
|
|
75
|
+
for (const c of COMMANDS) {
|
|
76
|
+
if (c.slash === n) return c
|
|
77
|
+
if (c.aliases?.includes(n)) return c
|
|
78
|
+
}
|
|
79
|
+
return null
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Commands matching a draft query (the text the kid has typed so far,
|
|
84
|
+
* including the leading "/"). "/" alone returns everything. Matches against
|
|
85
|
+
* slash, aliases, id, and both labels so "/mod" or "/模型"-style typing finds
|
|
86
|
+
* the model command.
|
|
87
|
+
*/
|
|
88
|
+
export function filterCommands(query: string): KidCommand[] {
|
|
89
|
+
const q = query.trim().toLowerCase()
|
|
90
|
+
if (q === "" || q === "/") return COMMANDS
|
|
91
|
+
return COMMANDS.filter((c) => {
|
|
92
|
+
const hay = [c.slash, ...(c.aliases ?? []), c.id, c.label.en, c.label.zh]
|
|
93
|
+
.join(" ")
|
|
94
|
+
.toLowerCase()
|
|
95
|
+
return hay.includes(q.replace(/^\//, ""))
|
|
96
|
+
})
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function commandLabel(c: KidCommand, locale: Locale): string {
|
|
100
|
+
return locale === "zh-Hans" ? c.label.zh : c.label.en
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function commandHint(c: KidCommand, locale: Locale): string {
|
|
104
|
+
return locale === "zh-Hans" ? c.hint.zh : c.hint.en
|
|
105
|
+
}
|
package/src/core/events.ts
CHANGED
|
@@ -94,63 +94,85 @@ export class EventSubscriber {
|
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
private dispatch(raw: unknown): void {
|
|
97
|
-
//
|
|
98
|
-
// • { payload: { type, …
|
|
99
|
-
// • { type, …
|
|
100
|
-
//
|
|
101
|
-
//
|
|
102
|
-
// • { data: { type,
|
|
103
|
-
//
|
|
97
|
+
// The SDK shape evolved over the 1.14.x line. Yielded events arrive as:
|
|
98
|
+
// • { payload: { type, properties: { … } } } (current — 1.14.51 GlobalEvent)
|
|
99
|
+
// • { payload: { type, …flatfields } } (older — flat fields on payload)
|
|
100
|
+
// • { type, properties: { … } } (some intermediate releases)
|
|
101
|
+
// • { type, …flatfields } (oldest)
|
|
102
|
+
// • { data: { type, …. } } (StreamEvent helper wrapper)
|
|
103
|
+
//
|
|
104
|
+
// CRITICAL: the current 1.14.51 GlobalEvent puts event fields
|
|
105
|
+
// (sessionID, messageID, delta, …) under .payload.properties, NOT
|
|
106
|
+
// directly on .payload. The previous code read .payload.sessionID
|
|
107
|
+
// (always undefined), so deltas were silently dropped and the
|
|
108
|
+
// "thinking" indicator never cleared. This split + the
|
|
109
|
+
// `props ?? payload` fallback handle both layouts uniformly.
|
|
104
110
|
const e = raw as { payload?: Record<string, unknown>; data?: Record<string, unknown>; type?: string } & Record<string, unknown>
|
|
105
|
-
const payload: ({ type?: string } & Record<string, unknown>) | null =
|
|
106
|
-
(e?.payload && typeof e.payload === "object") ? (e.payload as { type?: string } & Record<string, unknown>)
|
|
107
|
-
: (e?.data && typeof e.data === "object" && typeof (e.data as { type?: unknown }).type === "string") ? (e.data as { type?: string } & Record<string, unknown>)
|
|
108
|
-
: (typeof e?.type === "string") ? (e as { type?: string } & Record<string, unknown>)
|
|
111
|
+
const payload: ({ type?: string; properties?: Record<string, unknown> } & Record<string, unknown>) | null =
|
|
112
|
+
(e?.payload && typeof e.payload === "object") ? (e.payload as { type?: string; properties?: Record<string, unknown> } & Record<string, unknown>)
|
|
113
|
+
: (e?.data && typeof e.data === "object" && typeof (e.data as { type?: unknown }).type === "string") ? (e.data as { type?: string; properties?: Record<string, unknown> } & Record<string, unknown>)
|
|
114
|
+
: (typeof e?.type === "string") ? (e as { type?: string; properties?: Record<string, unknown> } & Record<string, unknown>)
|
|
109
115
|
: null
|
|
110
116
|
if (!payload || typeof payload.type !== "string") return
|
|
111
|
-
|
|
117
|
+
let t = payload.type
|
|
118
|
+
// Where the actual event fields live: `.properties` (Event* legacy shape)
|
|
119
|
+
// OR `.data` (SyncEvent* shape) OR flat on payload (oldest).
|
|
120
|
+
let props: Record<string, unknown> = (payload.properties && typeof payload.properties === "object")
|
|
121
|
+
? payload.properties as Record<string, unknown>
|
|
122
|
+
: payload
|
|
123
|
+
// 1.14.51 Sync events: type="sync", real event name in .name with a
|
|
124
|
+
// version suffix like ".1", fields under .data. Without this normalize,
|
|
125
|
+
// our switch never matches and streaming text events drop on the floor
|
|
126
|
+
// (kid sees "thinking..." forever).
|
|
127
|
+
if (t === "sync" && typeof (payload as { name?: unknown }).name === "string") {
|
|
128
|
+
t = String((payload as { name: string }).name).replace(/\.\d+$/, "")
|
|
129
|
+
const dataField = (payload as { data?: unknown }).data
|
|
130
|
+
if (dataField && typeof dataField === "object") {
|
|
131
|
+
props = dataField as Record<string, unknown>
|
|
132
|
+
}
|
|
133
|
+
}
|
|
112
134
|
switch (t) {
|
|
113
135
|
case "session.created":
|
|
114
136
|
case "session.next.session.created":
|
|
115
|
-
this.handlers.onSessionCreated?.({ sessionID: String(
|
|
137
|
+
this.handlers.onSessionCreated?.({ sessionID: String(props.sessionID ?? "") })
|
|
116
138
|
return
|
|
117
139
|
case "message.part.delta": {
|
|
118
|
-
const messageID = String(
|
|
119
|
-
const partID = String(
|
|
120
|
-
const sessionID = String(
|
|
121
|
-
const delta = String((
|
|
140
|
+
const messageID = String(props.messageID ?? "")
|
|
141
|
+
const partID = String(props.partID ?? "")
|
|
142
|
+
const sessionID = String(props.sessionID ?? "")
|
|
143
|
+
const delta = String((props.delta as { text?: string } | undefined)?.text ?? props.delta ?? "")
|
|
122
144
|
if (delta) this.handlers.onMessagePartDelta?.({ sessionID, messageID, partID, delta })
|
|
123
145
|
return
|
|
124
146
|
}
|
|
125
147
|
case "session.next.text.delta": {
|
|
126
|
-
const messageID = String(
|
|
127
|
-
const partID = String(
|
|
128
|
-
const sessionID = String(
|
|
129
|
-
const delta = String(
|
|
148
|
+
const messageID = String(props.messageID ?? "")
|
|
149
|
+
const partID = String(props.partID ?? "stream")
|
|
150
|
+
const sessionID = String(props.sessionID ?? "")
|
|
151
|
+
const delta = String(props.delta ?? "")
|
|
130
152
|
if (delta) this.handlers.onMessagePartDelta?.({ sessionID, messageID, partID, delta })
|
|
131
153
|
return
|
|
132
154
|
}
|
|
133
155
|
case "session.next.text.ended": {
|
|
134
|
-
const messageID = String(
|
|
135
|
-
const sessionID = String(
|
|
156
|
+
const messageID = String(props.messageID ?? "")
|
|
157
|
+
const sessionID = String(props.sessionID ?? "")
|
|
136
158
|
this.handlers.onTextEnded?.({ sessionID, messageID })
|
|
137
159
|
return
|
|
138
160
|
}
|
|
139
161
|
case "permission.asked":
|
|
140
162
|
case "session.next.permission.asked": {
|
|
141
|
-
const requestID = String(
|
|
142
|
-
const sessionID = String(
|
|
163
|
+
const requestID = String(props.requestID ?? props.id ?? "")
|
|
164
|
+
const sessionID = String(props.sessionID ?? "")
|
|
143
165
|
this.handlers.onPermissionAsked?.({
|
|
144
166
|
requestID,
|
|
145
167
|
sessionID,
|
|
146
|
-
tool:
|
|
147
|
-
metadata:
|
|
168
|
+
tool: props.tool as string | undefined,
|
|
169
|
+
metadata: props.metadata as Record<string, unknown> | undefined,
|
|
148
170
|
})
|
|
149
171
|
return
|
|
150
172
|
}
|
|
151
173
|
case "session.error":
|
|
152
174
|
case "llm.error": {
|
|
153
|
-
const message = String((
|
|
175
|
+
const message = String((props.error as { message?: string } | undefined)?.message ?? props.message ?? "unknown LLM error")
|
|
154
176
|
this.handlers.onLlmError?.({ message })
|
|
155
177
|
return
|
|
156
178
|
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Flatten the server's provider/model list into kid-friendly choices for the
|
|
3
|
+
* `/model` picker. Duck-typed and defensive: opencode's provider list shape has
|
|
4
|
+
* shifted across versions and a missing/!errored endpoint must degrade to an
|
|
5
|
+
* empty list (the picker shows a friendly "no models" note) rather than crash.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { OpencodeClient } from "./connection.ts"
|
|
9
|
+
import type { ModelChoice } from "./store.ts"
|
|
10
|
+
|
|
11
|
+
const MAX_MODELS = 40
|
|
12
|
+
|
|
13
|
+
export async function listModels(client: OpencodeClient): Promise<ModelChoice[]> {
|
|
14
|
+
const api = client as unknown as {
|
|
15
|
+
provider?: { list?: () => Promise<unknown> }
|
|
16
|
+
config?: { providers?: () => Promise<unknown> }
|
|
17
|
+
}
|
|
18
|
+
let raw: unknown
|
|
19
|
+
try {
|
|
20
|
+
if (typeof api.provider?.list === "function") raw = await api.provider.list()
|
|
21
|
+
else if (typeof api.config?.providers === "function") raw = await api.config.providers()
|
|
22
|
+
else return []
|
|
23
|
+
} catch {
|
|
24
|
+
return []
|
|
25
|
+
}
|
|
26
|
+
return flattenModels(raw).slice(0, MAX_MODELS)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function flattenModels(raw: unknown): ModelChoice[] {
|
|
30
|
+
const providers = pickProviders(raw)
|
|
31
|
+
const out: ModelChoice[] = []
|
|
32
|
+
const seen = new Set<string>()
|
|
33
|
+
for (const p of providers) {
|
|
34
|
+
const prov = p as { id?: string; name?: string; models?: unknown }
|
|
35
|
+
const pid = prov.id ?? ""
|
|
36
|
+
const pname = prov.name ?? pid
|
|
37
|
+
for (const m of pickModels(prov.models)) {
|
|
38
|
+
const mod = m as { id?: string; name?: string }
|
|
39
|
+
const mid = mod.id ?? ""
|
|
40
|
+
if (!pid || !mid) continue
|
|
41
|
+
const fullId = `${pid}/${mid}`
|
|
42
|
+
if (seen.has(fullId)) continue
|
|
43
|
+
seen.add(fullId)
|
|
44
|
+
const mname = mod.name ?? mid
|
|
45
|
+
out.push({ id: fullId, label: pname ? `${mname} · ${pname}` : mname })
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return out
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function pickProviders(raw: unknown): unknown[] {
|
|
52
|
+
if (Array.isArray(raw)) return raw
|
|
53
|
+
if (raw && typeof raw === "object") {
|
|
54
|
+
const o = raw as { providers?: unknown; data?: { providers?: unknown } }
|
|
55
|
+
if (Array.isArray(o.providers)) return o.providers
|
|
56
|
+
if (Array.isArray(o.data?.providers)) return o.data!.providers as unknown[]
|
|
57
|
+
}
|
|
58
|
+
return []
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function pickModels(models: unknown): unknown[] {
|
|
62
|
+
if (Array.isArray(models)) return models
|
|
63
|
+
// Most shapes use a Record<modelId, Model> map.
|
|
64
|
+
if (models && typeof models === "object") return Object.values(models)
|
|
65
|
+
return []
|
|
66
|
+
}
|
package/src/core/session.ts
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import type { OpencodeClient } from "./connection.ts"
|
|
13
|
+
import type { SessionSummary } from "./store.ts"
|
|
13
14
|
|
|
14
15
|
export class SessionManager {
|
|
15
16
|
private client: OpencodeClient
|
|
@@ -23,6 +24,31 @@ export class SessionManager {
|
|
|
23
24
|
return this.currentSessionId
|
|
24
25
|
}
|
|
25
26
|
|
|
27
|
+
/** Forget the current session so the next prompt() opens a fresh one. */
|
|
28
|
+
reset(): void {
|
|
29
|
+
this.currentSessionId = null
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** List past sessions (for the `/sessions` picker). Newest-ish first. */
|
|
33
|
+
async list(): Promise<SessionSummary[]> {
|
|
34
|
+
const api = (this.client as unknown as { session?: { list?: () => Promise<unknown> } }).session
|
|
35
|
+
if (!api?.list) return []
|
|
36
|
+
const raw = await api.list()
|
|
37
|
+
const arr = unwrapArray(raw)
|
|
38
|
+
return arr
|
|
39
|
+
.map((s) => {
|
|
40
|
+
const o = s as { id?: string; title?: string }
|
|
41
|
+
if (!o?.id) return null
|
|
42
|
+
return { id: o.id, title: (o.title ?? "").trim() || o.id }
|
|
43
|
+
})
|
|
44
|
+
.filter((s): s is SessionSummary => s !== null)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Continue an existing session: subsequent prompt()s append to it. */
|
|
48
|
+
switchTo(sessionID: string): void {
|
|
49
|
+
this.currentSessionId = sessionID
|
|
50
|
+
}
|
|
51
|
+
|
|
26
52
|
async create(): Promise<string> {
|
|
27
53
|
const api = (this.client as unknown as { session?: { create: (input?: unknown) => Promise<{ data?: { id?: string } } | { id?: string } | string> } }).session
|
|
28
54
|
if (!api?.create) throw new Error("SDK v2: client.session.create unavailable")
|
|
@@ -61,3 +87,13 @@ function extractId(result: unknown): string | null {
|
|
|
61
87
|
}
|
|
62
88
|
return null
|
|
63
89
|
}
|
|
90
|
+
|
|
91
|
+
/** SDK list responses come back as `T[]` or `{ data: T[] }` across versions. */
|
|
92
|
+
function unwrapArray(result: unknown): unknown[] {
|
|
93
|
+
if (Array.isArray(result)) return result
|
|
94
|
+
if (result && typeof result === "object") {
|
|
95
|
+
const d = (result as { data?: unknown }).data
|
|
96
|
+
if (Array.isArray(d)) return d
|
|
97
|
+
}
|
|
98
|
+
return []
|
|
99
|
+
}
|
package/src/core/store.ts
CHANGED
|
@@ -24,8 +24,27 @@ export type Screen =
|
|
|
24
24
|
completionMessage: string
|
|
25
25
|
hasNextMission: boolean
|
|
26
26
|
}
|
|
27
|
+
// Server-backed pickers, reachable via the `/model` and `/sessions` slash
|
|
28
|
+
// commands. They carry their fetched list so the router stays a pure
|
|
29
|
+
// function of state. `returnTo` is the screen to restore on cancel/select.
|
|
30
|
+
| { kind: "model_picker"; models: ModelChoice[]; returnTo: Screen }
|
|
31
|
+
| { kind: "session_list"; sessions: SessionSummary[]; returnTo: Screen }
|
|
27
32
|
| { kind: "error"; variant: ErrorVariant; detail?: string }
|
|
28
33
|
|
|
34
|
+
/** A selectable AI model, flattened from the server's provider list. */
|
|
35
|
+
export interface ModelChoice {
|
|
36
|
+
/** Full id passed to session.prompt, e.g. "anthropic/claude-3-5-sonnet". */
|
|
37
|
+
id: string
|
|
38
|
+
/** Kid-friendly display label. */
|
|
39
|
+
label: string
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** A past session, from session.list(). */
|
|
43
|
+
export interface SessionSummary {
|
|
44
|
+
id: string
|
|
45
|
+
title: string
|
|
46
|
+
}
|
|
47
|
+
|
|
29
48
|
export type ErrorVariant =
|
|
30
49
|
| "serve_unreachable"
|
|
31
50
|
| "port_taken"
|
|
@@ -86,6 +105,10 @@ export interface KidsClientState {
|
|
|
86
105
|
toast: ToastState | null
|
|
87
106
|
/** Plugin emitted audit events kept for parent dashboard sync (capped). */
|
|
88
107
|
auditBuffer: unknown[]
|
|
108
|
+
/** Model id chosen via `/model`; null → server default. Passed to prompt(). */
|
|
109
|
+
selectedModel: string | null
|
|
110
|
+
/** Kid-friendly label of selectedModel, for display in the header/toast. */
|
|
111
|
+
selectedModelLabel: string | null
|
|
89
112
|
}
|
|
90
113
|
|
|
91
114
|
type Listener = (state: KidsClientState) => void
|
|
@@ -107,6 +130,8 @@ const INITIAL: KidsClientState = {
|
|
|
107
130
|
missionTotal: null,
|
|
108
131
|
toast: null,
|
|
109
132
|
auditBuffer: [],
|
|
133
|
+
selectedModel: null,
|
|
134
|
+
selectedModelLabel: null,
|
|
110
135
|
}
|
|
111
136
|
|
|
112
137
|
const AUDIT_BUFFER_CAP = 500
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Voice controller — wires the parts into one "voice engine" the UI drives:
|
|
3
|
+
*
|
|
4
|
+
* spacebar ─▶ start() ─▶ recorder captures, feedLevel() streams energy
|
|
5
|
+
* │
|
|
6
|
+
* VAD says stop (silence/maxlen) ──▶ stop()
|
|
7
|
+
* │
|
|
8
|
+
* recorder → AudioClip → STT → text ──▶ onTranscript(text)
|
|
9
|
+
* (UI calls session.prompt)
|
|
10
|
+
*
|
|
11
|
+
* Deliberately owns NO timers and does NO spawning itself — the recorder
|
|
12
|
+
* produces level events, the UI/recorder calls feedLevel(), and this class
|
|
13
|
+
* only advances the state machine and decides start/stop/cancel. That keeps
|
|
14
|
+
* it pure enough to unit-test the whole orchestration with a mock recorder +
|
|
15
|
+
* MockStt, no microphone or clock required.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { transition, type VoiceState } from "./state.ts"
|
|
19
|
+
import { shouldAutoStop, DEFAULT_VAD, type VadOptions } from "./vad.ts"
|
|
20
|
+
import type { AudioClip, SttAdapter } from "./stt.ts"
|
|
21
|
+
|
|
22
|
+
/** Minimal recorder surface the controller needs (Recorder implements it; tests mock it). */
|
|
23
|
+
export interface RecorderLike {
|
|
24
|
+
start(): void
|
|
25
|
+
stop(): Promise<AudioClip>
|
|
26
|
+
cancel(): Promise<void>
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface VoiceControllerEvents {
|
|
30
|
+
/** Every state change — UI re-renders mic indicator / meter / spinner. */
|
|
31
|
+
onState?: (state: VoiceState) => void
|
|
32
|
+
/** Latest mic energy 0..1 — UI draws the meter. */
|
|
33
|
+
onLevel?: (level: number) => void
|
|
34
|
+
/** STT produced text — UI sends it via session.prompt and echoes it. */
|
|
35
|
+
onTranscript?: (text: string) => void
|
|
36
|
+
/** Recording/STT failed — UI shows a gentle retry hint. */
|
|
37
|
+
onError?: (err: Error) => void
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export class VoiceController {
|
|
41
|
+
private state: VoiceState = "idle"
|
|
42
|
+
private levels: number[] = []
|
|
43
|
+
|
|
44
|
+
constructor(
|
|
45
|
+
private recorder: RecorderLike,
|
|
46
|
+
private stt: SttAdapter,
|
|
47
|
+
private events: VoiceControllerEvents = {},
|
|
48
|
+
private vad: VadOptions = DEFAULT_VAD,
|
|
49
|
+
) {}
|
|
50
|
+
|
|
51
|
+
getState(): VoiceState {
|
|
52
|
+
return this.state
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
private set(next: VoiceState): void {
|
|
56
|
+
if (next === this.state) return
|
|
57
|
+
this.state = next
|
|
58
|
+
this.events.onState?.(next)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Spacebar while idle. Opens the mic. No-op if not idle. */
|
|
62
|
+
start(): void {
|
|
63
|
+
if (this.state !== "idle") return
|
|
64
|
+
this.levels = []
|
|
65
|
+
this.set(transition(this.state, { type: "START" }))
|
|
66
|
+
this.recorder.start()
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Feed one energy sample (the recorder calls this ~every sampleIntervalMs).
|
|
71
|
+
* Updates the meter and, once VAD says so, auto-stops — so the kid only ever
|
|
72
|
+
* pressed the spacebar once. No-op unless we're listening.
|
|
73
|
+
*/
|
|
74
|
+
feedLevel(level: number): void {
|
|
75
|
+
if (this.state !== "listening") return
|
|
76
|
+
this.levels.push(level)
|
|
77
|
+
this.events.onLevel?.(level)
|
|
78
|
+
if (shouldAutoStop(this.levels, this.vad) !== "continue") {
|
|
79
|
+
void this.stop()
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Spacebar/Enter again, or VAD auto-stop. Ends capture, runs STT, emits text. */
|
|
84
|
+
async stop(): Promise<void> {
|
|
85
|
+
if (this.state !== "listening") return
|
|
86
|
+
this.set(transition(this.state, { type: "STOP" })) // → transcribing
|
|
87
|
+
try {
|
|
88
|
+
const clip = await this.recorder.stop()
|
|
89
|
+
const { text } = await this.stt.transcribe(clip)
|
|
90
|
+
this.set(transition(this.state, { type: "TRANSCRIBED" })) // → thinking
|
|
91
|
+
this.events.onTranscript?.(text)
|
|
92
|
+
} catch (err) {
|
|
93
|
+
this.set(transition(this.state, { type: "ERROR" }))
|
|
94
|
+
this.events.onError?.(err instanceof Error ? err : new Error(String(err)))
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Esc. Throws the clip away with no send. Safe from any cancellable state. */
|
|
99
|
+
async cancel(): Promise<void> {
|
|
100
|
+
if (this.state === "listening") {
|
|
101
|
+
await this.recorder.cancel()
|
|
102
|
+
}
|
|
103
|
+
this.set(transition(this.state, { type: "CANCEL" }))
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** UI signals the LLM reply landed (and TTS, if any, finished). */
|
|
107
|
+
replied(): void {
|
|
108
|
+
this.set(transition(this.state, { type: "REPLIED" }))
|
|
109
|
+
}
|
|
110
|
+
spoken(): void {
|
|
111
|
+
this.set(transition(this.state, { type: "SPOKEN" }))
|
|
112
|
+
}
|
|
113
|
+
reset(): void {
|
|
114
|
+
this.set(transition(this.state, { type: "RESET" }))
|
|
115
|
+
}
|
|
116
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Microphone capture (side-effecting; the pure bits are extracted for tests).
|
|
3
|
+
*
|
|
4
|
+
* Strategy: shell out to a system recorder (sox `rec` or ffmpeg) writing a wav,
|
|
5
|
+
* same spawn pattern as core/serve-manager.ts. We also read the PCM stream to
|
|
6
|
+
* compute a rolling RMS energy level so the UI can draw a live mic meter and
|
|
7
|
+
* the VAD can auto-stop — terminals can't show a waveform any other way.
|
|
8
|
+
*
|
|
9
|
+
* No recorder on PATH must NOT crash the client: detectRecorder() returns null
|
|
10
|
+
* and the controller can fall back to a simulated level source (demo mode),
|
|
11
|
+
* so a kid on a box without sox still sees the flow, just with canned audio.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { spawn, type Subprocess } from "bun"
|
|
15
|
+
import type { AudioClip } from "./stt.ts"
|
|
16
|
+
|
|
17
|
+
export type RecorderKind = "sox" | "ffmpeg"
|
|
18
|
+
|
|
19
|
+
export interface RecordCommand {
|
|
20
|
+
cmd: string[]
|
|
21
|
+
/** Path the recorder writes the clip to. */
|
|
22
|
+
outPath: string
|
|
23
|
+
mimeType: string
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Build the capture command for a recorder. Pure → unit-testable. 16kHz mono
|
|
28
|
+
* wav is the Whisper-friendly sweet spot (small upload, plenty for speech).
|
|
29
|
+
*/
|
|
30
|
+
export function buildRecordCommand(kind: RecorderKind, outPath: string): RecordCommand {
|
|
31
|
+
if (kind === "sox") {
|
|
32
|
+
// `rec` is sox's record front-end. -q quiet, -c 1 mono, -r 16000 rate.
|
|
33
|
+
return { cmd: ["rec", "-q", "-c", "1", "-r", "16000", outPath], outPath, mimeType: "audio/wav" }
|
|
34
|
+
}
|
|
35
|
+
// ffmpeg: -f avfoundation on macOS captures the default mic (":0").
|
|
36
|
+
return {
|
|
37
|
+
cmd: ["ffmpeg", "-loglevel", "quiet", "-f", "avfoundation", "-i", ":0", "-ac", "1", "-ar", "16000", "-y", outPath],
|
|
38
|
+
outPath,
|
|
39
|
+
mimeType: "audio/wav",
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Compute normalised RMS energy (0..1) from a chunk of signed 16-bit PCM.
|
|
45
|
+
* Pure → unit-testable; this is what drives both the meter and the VAD.
|
|
46
|
+
*/
|
|
47
|
+
export function computeRms(pcm16: Int16Array): number {
|
|
48
|
+
if (pcm16.length === 0) return 0
|
|
49
|
+
let sumSq = 0
|
|
50
|
+
for (let i = 0; i < pcm16.length; i++) {
|
|
51
|
+
const s = pcm16[i]! / 32768 // normalise to -1..1
|
|
52
|
+
sumSq += s * s
|
|
53
|
+
}
|
|
54
|
+
return Math.sqrt(sumSq / pcm16.length)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Probe PATH for a usable recorder. Returns null if none — caller degrades to
|
|
58
|
+
* demo mode rather than crashing. */
|
|
59
|
+
export async function detectRecorder(): Promise<RecorderKind | null> {
|
|
60
|
+
for (const kind of ["sox", "ffmpeg"] as const) {
|
|
61
|
+
const bin = kind === "sox" ? "rec" : "ffmpeg"
|
|
62
|
+
try {
|
|
63
|
+
const proc = spawn({ cmd: ["which", bin], stdout: "pipe", stderr: "ignore" })
|
|
64
|
+
await proc.exited
|
|
65
|
+
if (proc.exitCode === 0) return kind
|
|
66
|
+
} catch {
|
|
67
|
+
/* keep probing */
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return null
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface RecorderEvents {
|
|
74
|
+
/** Fired ~every sampleIntervalMs with the latest normalised energy 0..1. */
|
|
75
|
+
onLevel?: (level: number) => void
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Owns one recording. start() spawns the recorder; stop() ends it and reads
|
|
80
|
+
* the written wav back as an AudioClip; cancel() kills it and discards.
|
|
81
|
+
*/
|
|
82
|
+
export class Recorder {
|
|
83
|
+
private child: Subprocess | null = null
|
|
84
|
+
private cmd: RecordCommand
|
|
85
|
+
|
|
86
|
+
constructor(kind: RecorderKind, outPath: string, private _events: RecorderEvents = {}) {
|
|
87
|
+
this.cmd = buildRecordCommand(kind, outPath)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
start(): void {
|
|
91
|
+
if (this.child) return
|
|
92
|
+
this.child = spawn({ cmd: this.cmd.cmd, stdout: "ignore", stderr: "ignore" })
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Stop recording and return the captured clip. */
|
|
96
|
+
async stop(): Promise<AudioClip> {
|
|
97
|
+
await this.kill()
|
|
98
|
+
const bytes = new Uint8Array(await Bun.file(this.cmd.outPath).arrayBuffer())
|
|
99
|
+
return { bytes, mimeType: this.cmd.mimeType }
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Abort and discard — no clip, no STT, no send. */
|
|
103
|
+
async cancel(): Promise<void> {
|
|
104
|
+
await this.kill()
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private async kill(): Promise<void> {
|
|
108
|
+
if (this.child && !this.child.killed) {
|
|
109
|
+
this.child.kill()
|
|
110
|
+
await this.child.exited
|
|
111
|
+
}
|
|
112
|
+
this.child = null
|
|
113
|
+
}
|
|
114
|
+
}
|