kimaki 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/fonts/Geist-Bold.ttf +0 -0
- package/assets/fonts/Geist-Regular.ttf +0 -0
- package/assets/fonts/GeistMono-Regular.ttf +0 -0
- package/assets/fonts/OFL.txt +92 -0
- package/dist/agent-ui.e2e.test.js +27 -13
- package/dist/agent-ui.e2e.test.js.map +1 -1
- package/dist/agent-ui.js +2 -1
- package/dist/agent-ui.js.map +1 -1
- package/dist/analytics.e2e.test.js +1 -1
- package/dist/analytics.e2e.test.js.map +1 -1
- package/dist/autostart.d.ts +4 -0
- package/dist/autostart.js +55 -10
- package/dist/autostart.js.map +1 -1
- package/dist/bot.d.ts +10 -1
- package/dist/bot.js +14 -6
- package/dist/bot.js.map +1 -1
- package/dist/btw.e2e.test.js +3 -3
- package/dist/builtin-skills.d.ts +1 -1
- package/dist/builtin-skills.js +1 -1
- package/dist/builtin-skills.js.map +1 -1
- package/dist/builtin-skills.test.js +3 -4
- package/dist/builtin-skills.test.js.map +1 -1
- package/dist/catalog-refresh.e2e.test.js +2 -2
- package/dist/cli/bot.js +25 -14
- package/dist/cli/bot.js.map +1 -1
- package/dist/cli/discord.js +32 -6
- package/dist/cli/discord.js.map +1 -1
- package/dist/cli/schedule.js +23 -9
- package/dist/cli/schedule.js.map +1 -1
- package/dist/cli/session.js +37 -12
- package/dist/cli/session.js.map +1 -1
- package/dist/cli/shared.d.ts +5 -3
- package/dist/cli/shared.js +45 -10
- package/dist/cli/shared.js.map +1 -1
- package/dist/cli-discord.e2e.test.js +13 -8
- package/dist/cli-discord.e2e.test.js.map +1 -1
- package/dist/cli-lock.e2e.test.js +87 -20
- package/dist/cli-lock.e2e.test.js.map +1 -1
- package/dist/cli-read.e2e.test.js +18 -9
- package/dist/cli-read.e2e.test.js.map +1 -1
- package/dist/cli-sessions.e2e.test.js +4 -3
- package/dist/cli-sessions.e2e.test.js.map +1 -1
- package/dist/cli.js +2 -0
- package/dist/cli.js.map +1 -1
- package/dist/commands/preference-commands.d.ts +4 -0
- package/dist/commands/preference-commands.js +38 -4
- package/dist/commands/preference-commands.js.map +1 -1
- package/dist/commands/session-commands.js +11 -8
- package/dist/commands/session-commands.js.map +1 -1
- package/dist/commands/thread-commands.js +94 -24
- package/dist/commands/thread-commands.js.map +1 -1
- package/dist/commands/worktree-commands.d.ts +5 -6
- package/dist/commands/worktree-commands.js +8 -7
- package/dist/commands/worktree-commands.js.map +1 -1
- package/dist/commands.e2e.test.js +25 -21
- package/dist/commands.e2e.test.js.map +1 -1
- package/dist/coordinator-agents.e2e.test.d.ts +1 -0
- package/dist/coordinator-agents.e2e.test.js +113 -0
- package/dist/coordinator-agents.e2e.test.js.map +1 -0
- package/dist/coordinator-idle.e2e.test.d.ts +1 -0
- package/dist/coordinator-idle.e2e.test.js +100 -0
- package/dist/coordinator-idle.e2e.test.js.map +1 -0
- package/dist/coordinator-queue.e2e.test.d.ts +1 -0
- package/dist/coordinator-queue.e2e.test.js +244 -0
- package/dist/coordinator-queue.e2e.test.js.map +1 -0
- package/dist/coordinator-warmup.e2e.test.d.ts +1 -0
- package/dist/coordinator-warmup.e2e.test.js +131 -0
- package/dist/coordinator-warmup.e2e.test.js.map +1 -0
- package/dist/coordinator.d.ts +6 -0
- package/dist/coordinator.e2e.test.d.ts +1 -0
- package/dist/coordinator.e2e.test.js +505 -0
- package/dist/coordinator.e2e.test.js.map +1 -0
- package/dist/coordinator.js +164 -0
- package/dist/coordinator.js.map +1 -0
- package/dist/db.d.ts +78 -3
- package/dist/db.test.js +1 -0
- package/dist/db.test.js.map +1 -1
- package/dist/diagram.d.ts +11 -0
- package/dist/diagram.js +232 -0
- package/dist/diagram.js.map +1 -0
- package/dist/diagram.test.d.ts +1 -0
- package/dist/diagram.test.js +52 -0
- package/dist/diagram.test.js.map +1 -0
- package/dist/duration.js +13 -5
- package/dist/duration.js.map +1 -1
- package/dist/effects.d.ts +6 -1
- package/dist/effects.js +97 -26
- package/dist/effects.js.map +1 -1
- package/dist/event-loop.d.ts +1 -1
- package/dist/event-loop.js +73 -43
- package/dist/event-loop.js.map +1 -1
- package/dist/format-parts.d.ts +21 -4
- package/dist/format-parts.js +40 -23
- package/dist/format-parts.js.map +1 -1
- package/dist/gateway-replay.e2e.test.js +4 -4
- package/dist/harness.e2e.test.js +2 -2
- package/dist/harness.e2e.test.js.map +1 -1
- package/dist/hello-thread.e2e.test.js +56 -5
- package/dist/hello-thread.e2e.test.js.map +1 -1
- package/dist/ingress.js +47 -12
- package/dist/ingress.js.map +1 -1
- package/dist/interaction-context.d.ts +4 -4
- package/dist/interaction-context.js +9 -7
- package/dist/interaction-context.js.map +1 -1
- package/dist/interrupt.e2e.test.js +8 -5
- package/dist/interrupt.e2e.test.js.map +1 -1
- package/dist/legacy.e2e.test.js +1 -1
- package/dist/lock-routes.d.ts +83 -31
- package/dist/lock-routes.js +68 -8
- package/dist/lock-routes.js.map +1 -1
- package/dist/lock-server.d.ts +3 -2
- package/dist/lock-server.js +23 -7
- package/dist/lock-server.js.map +1 -1
- package/dist/main.d.ts +4 -2
- package/dist/main.js +13 -3
- package/dist/main.js.map +1 -1
- package/dist/mainmast-idle.e2e.test.d.ts +1 -0
- package/dist/mainmast-idle.e2e.test.js +99 -0
- package/dist/mainmast-idle.e2e.test.js.map +1 -0
- package/dist/mainmast-queue.e2e.test.d.ts +1 -0
- package/dist/mainmast-queue.e2e.test.js +244 -0
- package/dist/mainmast-queue.e2e.test.js.map +1 -0
- package/dist/mainmast-warmup.e2e.test.d.ts +1 -0
- package/dist/mainmast-warmup.e2e.test.js +130 -0
- package/dist/mainmast-warmup.e2e.test.js.map +1 -0
- package/dist/mainmast.d.ts +6 -0
- package/dist/mainmast.e2e.test.d.ts +1 -0
- package/dist/mainmast.e2e.test.js +428 -0
- package/dist/mainmast.e2e.test.js.map +1 -0
- package/dist/mainmast.js +156 -0
- package/dist/mainmast.js.map +1 -0
- package/dist/markdown/markdown.test.js +4 -4
- package/dist/markdown/render-markdown.js +6 -5
- package/dist/markdown/render-markdown.js.map +1 -1
- package/dist/onboarding-cli.e2e.test.js +7 -6
- package/dist/onboarding-cli.e2e.test.js.map +1 -1
- package/dist/onboarding.d.ts +3 -10
- package/dist/onboarding.e2e.test.js +8 -10
- package/dist/onboarding.e2e.test.js.map +1 -1
- package/dist/onboarding.js +14 -24
- package/dist/onboarding.js.map +1 -1
- package/dist/permissions.e2e.test.js +3 -3
- package/dist/plugin/coordinator.d.ts +4 -0
- package/dist/plugin/coordinator.js +389 -0
- package/dist/plugin/coordinator.js.map +1 -0
- package/dist/plugin/index.js +76 -17
- package/dist/plugin/index.js.map +1 -1
- package/dist/plugin/mainmast.d.ts +4 -0
- package/dist/plugin/mainmast.js +379 -0
- package/dist/plugin/mainmast.js.map +1 -0
- package/dist/plugin.e2e.test.js +1 -1
- package/dist/preference-commands.e2e.test.js +103 -24
- package/dist/preference-commands.e2e.test.js.map +1 -1
- package/dist/project.d.ts +1 -1
- package/dist/prompt.d.ts +22 -9
- package/dist/prompt.js +64 -48
- package/dist/prompt.js.map +1 -1
- package/dist/questions.e2e.test.js +13 -13
- package/dist/queue-sources.e2e.test.d.ts +1 -0
- package/dist/queue-sources.e2e.test.js +168 -0
- package/dist/queue-sources.e2e.test.js.map +1 -0
- package/dist/queue.d.ts +9 -1
- package/dist/queue.e2e.test.js +48 -30
- package/dist/queue.e2e.test.js.map +1 -1
- package/dist/queue.js +97 -24
- package/dist/queue.js.map +1 -1
- package/dist/queue.test.d.ts +1 -0
- package/dist/queue.test.js +57 -0
- package/dist/queue.test.js.map +1 -0
- package/dist/remote-send.e2e.test.js +6 -6
- package/dist/remote-send.js +2 -1
- package/dist/remote-send.js.map +1 -1
- package/dist/renderer.e2e.test.js +7 -7
- package/dist/restart.e2e.test.js +2 -2
- package/dist/routines.e2e.test.js +2 -2
- package/dist/scheduler.d.ts +3 -3
- package/dist/scheduler.e2e.test.js +43 -20
- package/dist/scheduler.e2e.test.js.map +1 -1
- package/dist/scheduler.js +4 -2
- package/dist/scheduler.js.map +1 -1
- package/dist/schema-sql.d.ts +1 -1
- package/dist/schema-sql.js +1 -1
- package/dist/schema-sql.js.map +1 -1
- package/dist/schema.d.ts +152 -6
- package/dist/schema.js +12 -0
- package/dist/schema.js.map +1 -1
- package/dist/session-commands.e2e.test.js +36 -11
- package/dist/session-commands.e2e.test.js.map +1 -1
- package/dist/session-events.d.ts +18 -6
- package/dist/session-events.js +166 -22
- package/dist/session-events.js.map +1 -1
- package/dist/session-ops.e2e.test.js +63 -12
- package/dist/session-ops.e2e.test.js.map +1 -1
- package/dist/sessions.d.ts +44 -21
- package/dist/sessions.js +174 -63
- package/dist/sessions.js.map +1 -1
- package/dist/shell.e2e.test.js +4 -4
- package/dist/slash-commands.js +6 -4
- package/dist/slash-commands.js.map +1 -1
- package/dist/slash-commands.test.js +2 -0
- package/dist/slash-commands.test.js.map +1 -1
- package/dist/sleep.e2e.test.js +29 -14
- package/dist/sleep.e2e.test.js.map +1 -1
- package/dist/sleeps.d.ts +2 -1
- package/dist/sleeps.js +21 -3
- package/dist/sleeps.js.map +1 -1
- package/dist/store.d.ts +1 -0
- package/dist/store.js +3 -2
- package/dist/store.js.map +1 -1
- package/dist/system-prompt.d.ts +3 -1
- package/dist/system-prompt.js +138 -64
- package/dist/system-prompt.js.map +1 -1
- package/dist/test/generate-log-plugin/index.d.ts +3 -0
- package/dist/test/generate-log-plugin/index.js +15 -0
- package/dist/test/generate-log-plugin/index.js.map +1 -0
- package/dist/test/harness.d.ts +20 -1
- package/dist/test/harness.js +51 -10
- package/dist/test/harness.js.map +1 -1
- package/dist/test/replay.js +11 -7
- package/dist/test/replay.js.map +1 -1
- package/dist/thread-reducer.d.ts +29 -3
- package/dist/thread-reducer.fixtures.test.js +58 -51
- package/dist/thread-reducer.fixtures.test.js.map +1 -1
- package/dist/thread-reducer.js +112 -25
- package/dist/thread-reducer.js.map +1 -1
- package/dist/thread-reducer.test.js +197 -24
- package/dist/thread-reducer.test.js.map +1 -1
- package/dist/voice-attachments.e2e.test.js +108 -18
- package/dist/voice-attachments.e2e.test.js.map +1 -1
- package/dist/voice-call.e2e.test.js +9 -9
- package/dist/voice-call.js +6 -6
- package/dist/voice-call.js.map +1 -1
- package/dist/voice-openai.e2e.test.js +59 -23
- package/dist/voice-openai.e2e.test.js.map +1 -1
- package/dist/voice.d.ts +1 -1
- package/dist/voice.js +72 -24
- package/dist/voice.js.map +1 -1
- package/dist/working-directory.e2e.test.js +11 -11
- package/dist/working-directory.e2e.test.js.map +1 -1
- package/package.json +12 -6
- package/src/agent-ui.e2e.test.ts +27 -13
- package/src/agent-ui.ts +2 -1
- package/src/analytics.e2e.test.ts +1 -1
- package/src/autostart.ts +55 -10
- package/src/bot.ts +29 -5
- package/src/btw.e2e.test.ts +3 -3
- package/src/builtin-skills.test.ts +3 -4
- package/src/builtin-skills.ts +1 -1
- package/src/catalog-refresh.e2e.test.ts +2 -2
- package/src/cli/bot.ts +23 -14
- package/src/cli/discord.ts +30 -6
- package/src/cli/schedule.ts +23 -9
- package/src/cli/session.ts +35 -12
- package/src/cli/shared.ts +43 -8
- package/src/cli-discord.e2e.test.ts +13 -8
- package/src/cli-lock.e2e.test.ts +88 -20
- package/src/cli-read.e2e.test.ts +17 -8
- package/src/cli-sessions.e2e.test.ts +4 -3
- package/src/cli.ts +2 -0
- package/src/commands/preference-commands.ts +38 -4
- package/src/commands/session-commands.ts +12 -7
- package/src/commands/thread-commands.ts +92 -24
- package/src/commands/worktree-commands.ts +10 -9
- package/src/commands.e2e.test.ts +25 -21
- package/src/coordinator-agents.e2e.test.ts +145 -0
- package/src/coordinator-idle.e2e.test.ts +122 -0
- package/src/coordinator-queue.e2e.test.ts +279 -0
- package/src/coordinator-warmup.e2e.test.ts +158 -0
- package/src/coordinator.e2e.test.ts +545 -0
- package/src/coordinator.ts +168 -0
- package/src/db.test.ts +1 -0
- package/src/diagram.test.ts +56 -0
- package/src/diagram.ts +243 -0
- package/src/duration.ts +13 -5
- package/src/effects.ts +99 -27
- package/src/event-loop.ts +71 -42
- package/src/format-parts.ts +53 -22
- package/src/gateway-replay.e2e.test.ts +4 -4
- package/src/harness.e2e.test.ts +2 -2
- package/src/hello-thread.e2e.test.ts +59 -5
- package/src/ingress.ts +44 -11
- package/src/interaction-context.ts +12 -11
- package/src/interrupt.e2e.test.ts +8 -5
- package/src/legacy.e2e.test.ts +1 -1
- package/src/lock-routes.ts +62 -8
- package/src/lock-server.ts +23 -7
- package/src/main.ts +22 -7
- package/src/markdown/markdown.test.ts +4 -4
- package/src/markdown/render-markdown.ts +6 -5
- package/src/onboarding-cli.e2e.test.ts +7 -6
- package/src/onboarding.e2e.test.ts +8 -10
- package/src/onboarding.ts +13 -29
- package/src/permissions.e2e.test.ts +3 -3
- package/src/plugin/coordinator.ts +401 -0
- package/src/plugin/index.ts +68 -15
- package/src/plugin.e2e.test.ts +1 -1
- package/src/preference-commands.e2e.test.ts +103 -23
- package/src/prompt.ts +87 -43
- package/src/questions.e2e.test.ts +13 -13
- package/src/queue-sources.e2e.test.ts +197 -0
- package/src/queue.e2e.test.ts +48 -30
- package/src/queue.test.ts +59 -0
- package/src/queue.ts +104 -24
- package/src/remote-send.e2e.test.ts +6 -6
- package/src/remote-send.ts +2 -1
- package/src/renderer.e2e.test.ts +7 -7
- package/src/restart.e2e.test.ts +2 -2
- package/src/routines.e2e.test.ts +2 -2
- package/src/scheduler.e2e.test.ts +45 -19
- package/src/scheduler.ts +4 -2
- package/src/schema-sql.ts +1 -1
- package/src/schema.ts +13 -0
- package/src/session-commands.e2e.test.ts +38 -11
- package/src/session-events.ts +160 -21
- package/src/session-ops.e2e.test.ts +64 -11
- package/src/sessions.ts +214 -64
- package/src/shell.e2e.test.ts +4 -4
- package/src/slash-commands.test.ts +2 -0
- package/src/slash-commands.ts +6 -4
- package/src/sleep.e2e.test.ts +30 -13
- package/src/sleeps.ts +20 -3
- package/src/store.ts +5 -2
- package/src/system-prompt.ts +143 -63
- package/src/test/generate-log-plugin/index.ts +16 -0
- package/src/test/harness.ts +77 -14
- package/src/test/replay.ts +12 -6
- package/src/thread-reducer.fixtures.test.ts +58 -51
- package/src/thread-reducer.test.ts +207 -26
- package/src/thread-reducer.ts +138 -37
- package/src/voice-attachments.e2e.test.ts +110 -18
- package/src/voice-call.e2e.test.ts +9 -9
- package/src/voice-call.ts +6 -6
- package/src/voice-openai.e2e.test.ts +64 -24
- package/src/voice.ts +93 -26
- package/src/working-directory.e2e.test.ts +11 -11
|
@@ -136,12 +136,12 @@ test('a user joins and speaks, the backend runs kimaki commands and the model sa
|
|
|
136
136
|
// Thread IDs are snowflakes: replace them for a stable snapshot.
|
|
137
137
|
expect((await twin.discord.channel(voiceChannelId).text()).replace(/<#\d+>/g, '<#thread>')).toMatchInlineSnapshot(`
|
|
138
138
|
"--- from: assistant (TestBot)
|
|
139
|
-
-#
|
|
139
|
+
-# voice call started ⋅ gpt-live ⋅ cove
|
|
140
140
|
Hello tommy.
|
|
141
|
-
» **tommy:** Which projects do I have?
|
|
141
|
+
-# » **tommy:** Which projects do I have?
|
|
142
142
|
-# ┣ shell _seq 1 2000; kimaki project list_
|
|
143
143
|
-# ┣ shell _kimaki send --channel 200000000000000100 --prompt 'Voice call note' --notify-on…_
|
|
144
|
-
-#
|
|
144
|
+
-# session started in <#thread>
|
|
145
145
|
-# ⨯ post\\_message _text must not be empty_
|
|
146
146
|
You have one project."
|
|
147
147
|
`)
|
|
@@ -205,12 +205,12 @@ test('chat messages reach the backend, and end_call leaves after the goodbye whi
|
|
|
205
205
|
expect(botPackets() - packetsBefore).toBeGreaterThanOrEqual(30)
|
|
206
206
|
expect(live.backendInputs.at(-1)).toBe(`<chat-message from="tommy" user-id="${TEST_USER_ID}">\nplease hang up\n</chat-message>`)
|
|
207
207
|
expect(await chatSince(before)).toMatchInlineSnapshot(`
|
|
208
|
-
"TestBot: -#
|
|
208
|
+
"TestBot: -# voice call started ⋅ gpt-live ⋅ cove
|
|
209
209
|
TestBot: Hi again.
|
|
210
210
|
tommy: please hang up
|
|
211
211
|
TestBot: -# ┣ end\\_call
|
|
212
212
|
TestBot: Bye.
|
|
213
|
-
TestBot: -#
|
|
213
|
+
TestBot: -# voice call ended: the assistant hung up"
|
|
214
214
|
`)
|
|
215
215
|
await twin.discord.channel(voiceChannelId).user(TEST_USER_ID).leaveVoice()
|
|
216
216
|
})
|
|
@@ -244,15 +244,15 @@ test('finished threads of any project are announced in the call; the model answe
|
|
|
244
244
|
live.replies.length = 0
|
|
245
245
|
|
|
246
246
|
expect(await chatSince(before)).toMatchInlineSnapshot(`
|
|
247
|
-
"TestBot: -#
|
|
247
|
+
"TestBot: -# voice call started ⋅ gpt-live ⋅ cove
|
|
248
248
|
TestBot: Hi.
|
|
249
249
|
tommy: start a thread
|
|
250
250
|
TestBot: -# ┣ shell _kimaki send --channel <id> --prompt 'call-thread-marker' --user '…_
|
|
251
|
-
TestBot: -#
|
|
251
|
+
TestBot: -# session started in <#<id>>
|
|
252
252
|
TestBot: Started.
|
|
253
|
-
TestBot: -#
|
|
253
|
+
TestBot: -# thread finished: call-thread-marker ⋅ https://discord.com/channels/<id>/<id> ⋅ <ses>
|
|
254
254
|
TestBot: Your thread finished.
|
|
255
|
-
TestBot: -#
|
|
255
|
+
TestBot: -# thread finished: other-thread-marker ⋅ https://discord.com/channels/<id>/<id> ⋅ <ses>"
|
|
256
256
|
`)
|
|
257
257
|
const notices = live.appends.filter((append) => append.text.startsWith('<system>Thread finished'))
|
|
258
258
|
expect(notices.map((notice) => ({ ...notice, text: notice.text.replace(/\d{17,20}/g, '<id>').replace(/ses_\w+/g, '<ses>') }))).toMatchInlineSnapshot(`
|
package/src/voice-call.ts
CHANGED
|
@@ -330,7 +330,7 @@ export function createVoiceCalls({ opencodeConfigDir }: { opencodeConfigDir: str
|
|
|
330
330
|
|
|
331
331
|
// Bot lines in the text chat of the voice channel, through the same FIFO as session output.
|
|
332
332
|
function post(bot: Bot, { channelId, text, markdown = false }: { channelId: string; text: string; markdown?: boolean }) {
|
|
333
|
-
bot.effects.run(channelId, [markdown ? { type: 'markdown', text, blankLineBefore: false } : { type: 'send', text }])
|
|
333
|
+
bot.effects.run(channelId, [markdown ? { type: 'markdown', text, blankLineBefore: false, replyTo: null } : { type: 'send', text }])
|
|
334
334
|
}
|
|
335
335
|
|
|
336
336
|
// Errors during a call go to the log and to the text chat of the voice channel.
|
|
@@ -505,7 +505,7 @@ export function createVoiceCalls({ opencodeConfigDir }: { opencodeConfigDir: str
|
|
|
505
505
|
sent.push(threadId)
|
|
506
506
|
current.started.add(threadId)
|
|
507
507
|
if (waits) current.waiting.add(threadId)
|
|
508
|
-
post(bot, { channelId: current.channelId, text: asSubtext(
|
|
508
|
+
post(bot, { channelId: current.channelId, text: asSubtext(`${started} <#${threadId}>`) })
|
|
509
509
|
}
|
|
510
510
|
: undefined
|
|
511
511
|
const run = runShell(bot, { call: current, command: input.command, timeoutMs, onLine }).then((result) => {
|
|
@@ -668,8 +668,8 @@ export function createVoiceCalls({ opencodeConfigDir }: { opencodeConfigDir: str
|
|
|
668
668
|
}
|
|
669
669
|
listen(bot, call)
|
|
670
670
|
logger.info(`voice call started in ${channelId}, voice ${settings.voice}`)
|
|
671
|
-
post(bot, { channelId, text: `-#
|
|
672
|
-
if (settings.voiceNotice) post(bot, { channelId, text: asSubtext(
|
|
671
|
+
post(bot, { channelId, text: `-# voice call started ⋅ gpt-live ⋅ ${settings.voice}` })
|
|
672
|
+
if (settings.voiceNotice) post(bot, { channelId, text: asSubtext(settings.voiceNotice) })
|
|
673
673
|
live.append({ text: 'The call started. Greet the users by name in one short sentence.', channel: 'speakable' })
|
|
674
674
|
}
|
|
675
675
|
|
|
@@ -684,7 +684,7 @@ export function createVoiceCalls({ opencodeConfigDir }: { opencodeConfigDir: str
|
|
|
684
684
|
call.speaker.stop()
|
|
685
685
|
if (call.connection.state.status !== VoiceConnectionStatus.Destroyed) call.connection.destroy()
|
|
686
686
|
logger.info(`voice call ended in ${call.channelId}${reason ? `: ${reason}` : ''}`)
|
|
687
|
-
post(bot, { channelId: call.channelId, text: `-#
|
|
687
|
+
post(bot, { channelId: call.channelId, text: `-# voice call ended${reason ? `: ${reason}` : ''}` })
|
|
688
688
|
}
|
|
689
689
|
|
|
690
690
|
async function onVoiceState(bot: Bot, { before, after }: { before: VoiceState; after: VoiceState }) {
|
|
@@ -727,7 +727,7 @@ export function createVoiceCalls({ opencodeConfigDir }: { opencodeConfigDir: str
|
|
|
727
727
|
if (!call || call.ending) return
|
|
728
728
|
const url = `https://discord.com/channels/${thread.guildId}/${threadId}`
|
|
729
729
|
const state = error === null ? 'finished' : 'failed'
|
|
730
|
-
post(bot, { channelId: call.channelId, text: asSubtext(
|
|
730
|
+
post(bot, { channelId: call.channelId, text: asSubtext(`thread ${state}: ${thread.name} ⋅ ${url} ⋅ ${sessionId}`) })
|
|
731
731
|
const failure = error === null ? '' : ` Error: ${error}`
|
|
732
732
|
const notice = `<system>Thread ${state}: "${thread.name}". Thread ID ${threadId}, session ID ${sessionId}, URL ${url}.${failure}</system>`
|
|
733
733
|
// Answer only for threads started in this call. A pending kimaki send --wait brings the transcript next.
|
|
@@ -2,13 +2,14 @@
|
|
|
2
2
|
// decoded to WAV (OpenAI accepts only wav and mp3) and sent to a local
|
|
3
3
|
// OpenAI-compatible server that answers with the transcription tool call.
|
|
4
4
|
// A ChatGPT credential stored by OpenCode is the fallback when the key fails, and gets the raw audio.
|
|
5
|
+
// KIMAKI_WHISPER_URL replaces them all with a local Whisper server.
|
|
5
6
|
|
|
6
7
|
import { execFile } from 'node:child_process'
|
|
7
8
|
import fs from 'node:fs'
|
|
8
9
|
import path from 'node:path'
|
|
9
10
|
import { promisify } from 'node:util'
|
|
10
11
|
import type { DeterministicMatcher } from 'opencode-deterministic-provider'
|
|
11
|
-
import { afterAll, beforeAll, expect, test } from 'vitest'
|
|
12
|
+
import { afterAll, beforeAll, expect, onTestFinished, test } from 'vitest'
|
|
12
13
|
|
|
13
14
|
import type { BotHandle } from './main.ts'
|
|
14
15
|
import {
|
|
@@ -39,6 +40,7 @@ let twin: TestTwin
|
|
|
39
40
|
let bot: BotHandle
|
|
40
41
|
let openai: FakeOpenAI
|
|
41
42
|
let chatgpt: FakeChatGPT
|
|
43
|
+
let whisper: FakeChatGPT
|
|
42
44
|
let dataDir: string
|
|
43
45
|
// Created by the ChatGPT test: the tests before it run with no credential.
|
|
44
46
|
let opencodeDbFile: string
|
|
@@ -46,11 +48,12 @@ let opencodeDbFile: string
|
|
|
46
48
|
beforeAll(async () => {
|
|
47
49
|
dataDir = tempDataDir()
|
|
48
50
|
opencodeDbFile = path.join(dataDir, 'opencode.db')
|
|
49
|
-
;[server, twin, openai, chatgpt] = await Promise.all([
|
|
51
|
+
;[server, twin, openai, chatgpt, whisper] = await Promise.all([
|
|
50
52
|
startOpencodeTestServer({ matchers }),
|
|
51
53
|
startTwin(),
|
|
52
54
|
startFakeOpenAI({ result: { transcription: 'Hello from the phone voice-openai', route: 'steer' } }),
|
|
53
55
|
startFakeChatGPT({ text: 'Hello from ChatGPT voice-openai' }),
|
|
56
|
+
startFakeChatGPT({ text: 'Hello from Whisper voice-openai' }),
|
|
54
57
|
])
|
|
55
58
|
await seedProjectChannel({ dataDir, channelId: twin.channelId, guildId: twin.discord.guildId, directory: server.projectDirectory })
|
|
56
59
|
await warmUp({ server })
|
|
@@ -66,25 +69,33 @@ afterAll(async () => {
|
|
|
66
69
|
await server?.stop()
|
|
67
70
|
await openai?.stop()
|
|
68
71
|
await chatgpt?.stop()
|
|
72
|
+
await whisper?.stop()
|
|
69
73
|
fs.rmSync(dataDir, { recursive: true, force: true })
|
|
70
74
|
})
|
|
71
75
|
|
|
72
|
-
|
|
73
|
-
|
|
76
|
+
const ogg = fs.readFileSync(path.join(import.meta.dirname, 'fixtures', 'voice.ogg'))
|
|
77
|
+
|
|
78
|
+
async function sendVoice() {
|
|
79
|
+
const before = new Set((await twin.discord.channel(twin.channelId).getThreads()).map((thread) => thread.id))
|
|
74
80
|
await twin.discord
|
|
75
81
|
.channel(twin.channelId)
|
|
76
82
|
.user(TEST_USER_ID)
|
|
77
83
|
.sendVoiceMessage({ url: `data:audio/ogg;base64,${ogg.toString('base64')}` })
|
|
78
|
-
const thread = await twin.discord.channel(twin.channelId).waitForThread({ timeout: 8_000 })
|
|
84
|
+
const thread = await twin.discord.channel(twin.channelId).waitForThread({ timeout: 8_000, predicate: (thread) => !before.has(thread.id) })
|
|
79
85
|
await waitForFooter({ discord: twin.discord, threadId: thread.id })
|
|
80
|
-
|
|
86
|
+
return thread.id
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
test('an Ogg Opus voice message is converted to WAV and transcribed by OpenAI', async () => {
|
|
90
|
+
const thread = await sendVoice()
|
|
91
|
+
expect(await twin.discord.thread(thread).text()).toMatchInlineSnapshot(`
|
|
81
92
|
"--- from: user (tommy)
|
|
82
93
|
[attachment: voice-message.ogg]
|
|
83
94
|
--- from: assistant (TestBot)
|
|
84
|
-
» **tommy:** Hello from the phone voice-openai
|
|
95
|
+
-# » **tommy:** Hello from the phone voice-openai
|
|
85
96
|
-# *using deterministic-provider/deterministic-v2 ⋅ build*
|
|
86
97
|
openai voice ok
|
|
87
|
-
-# *
|
|
98
|
+
-# *main ⋅ Ns ⋅ N% ⋅ deterministic-v2*"
|
|
88
99
|
`)
|
|
89
100
|
const [request] = openai.requests
|
|
90
101
|
expect({
|
|
@@ -129,29 +140,17 @@ test('an MP3 upload labeled audio/mpeg3 goes to OpenAI as mp3', async () => {
|
|
|
129
140
|
test('the OpenAI key wins over the ChatGPT credential from OpenCode, which takes over when the key fails', async () => {
|
|
130
141
|
const claims = Buffer.from(JSON.stringify({ 'https://api.openai.com/auth': { chatgpt_account_id: 'acct-test' } })).toString('base64url')
|
|
131
142
|
await writeOpencodeCredential({ file: opencodeDbFile, access: `header.${claims}.signature`, expires: Date.now() + 3_600_000 })
|
|
132
|
-
const ogg = fs.readFileSync(path.join(import.meta.dirname, 'fixtures', 'voice.ogg'))
|
|
133
143
|
const openaiRequests = openai.requests.length
|
|
134
144
|
|
|
135
|
-
async function sendVoice() {
|
|
136
|
-
const before = new Set((await twin.discord.channel(twin.channelId).getThreads()).map((thread) => thread.id))
|
|
137
|
-
await twin.discord
|
|
138
|
-
.channel(twin.channelId)
|
|
139
|
-
.user(TEST_USER_ID)
|
|
140
|
-
.sendVoiceMessage({ url: `data:audio/ogg;base64,${ogg.toString('base64')}` })
|
|
141
|
-
const thread = await twin.discord.channel(twin.channelId).waitForThread({ timeout: 8_000, predicate: (thread) => !before.has(thread.id) })
|
|
142
|
-
await waitForFooter({ discord: twin.discord, threadId: thread.id })
|
|
143
|
-
return thread.id
|
|
144
|
-
}
|
|
145
|
-
|
|
146
145
|
const viaKey = await sendVoice()
|
|
147
146
|
expect(await twin.discord.thread(viaKey).text()).toMatchInlineSnapshot(`
|
|
148
147
|
"--- from: user (tommy)
|
|
149
148
|
[attachment: voice-message.ogg]
|
|
150
149
|
--- from: assistant (TestBot)
|
|
151
|
-
» **tommy:** Hello from the phone voice-openai
|
|
150
|
+
-# » **tommy:** Hello from the phone voice-openai
|
|
152
151
|
-# *using deterministic-provider/deterministic-v2 ⋅ build*
|
|
153
152
|
openai voice ok
|
|
154
|
-
-# *
|
|
153
|
+
-# *main ⋅ Ns ⋅ N% ⋅ deterministic-v2*"
|
|
155
154
|
`)
|
|
156
155
|
expect({ chatgptRequests: chatgpt.requests.length, openaiRequests: openai.requests.length - openaiRequests }).toMatchInlineSnapshot(`
|
|
157
156
|
{
|
|
@@ -167,10 +166,10 @@ test('the OpenAI key wins over the ChatGPT credential from OpenCode, which takes
|
|
|
167
166
|
"--- from: user (tommy)
|
|
168
167
|
[attachment: voice-message.ogg]
|
|
169
168
|
--- from: assistant (TestBot)
|
|
170
|
-
» **tommy:** Hello from ChatGPT voice-openai
|
|
169
|
+
-# » **tommy:** Hello from ChatGPT voice-openai
|
|
171
170
|
-# *using deterministic-provider/deterministic-v2 ⋅ build*
|
|
172
171
|
openai voice ok
|
|
173
|
-
-# *
|
|
172
|
+
-# *main ⋅ Ns ⋅ N% ⋅ deterministic-v2*"
|
|
174
173
|
`)
|
|
175
174
|
const [request] = chatgpt.requests
|
|
176
175
|
expect({
|
|
@@ -189,3 +188,44 @@ test('the OpenAI key wins over the ChatGPT credential from OpenCode, which takes
|
|
|
189
188
|
}
|
|
190
189
|
`)
|
|
191
190
|
})
|
|
191
|
+
|
|
192
|
+
test('KIMAKI_WHISPER_URL is the only transcriber: no cloud request, and its failure is shown', async () => {
|
|
193
|
+
process.env['KIMAKI_WHISPER_URL'] = `${whisper.baseUrl}/v1/`
|
|
194
|
+
process.env['KIMAKI_WHISPER_MODEL'] = 'Systran/faster-whisper-large-v3'
|
|
195
|
+
onTestFinished(() => {
|
|
196
|
+
delete process.env['KIMAKI_WHISPER_URL']
|
|
197
|
+
delete process.env['KIMAKI_WHISPER_MODEL']
|
|
198
|
+
})
|
|
199
|
+
const cloudRequests = openai.requests.length + chatgpt.requests.length
|
|
200
|
+
|
|
201
|
+
const thread = await sendVoice()
|
|
202
|
+
expect(await twin.discord.thread(thread).text()).toMatchInlineSnapshot(`
|
|
203
|
+
"--- from: user (tommy)
|
|
204
|
+
[attachment: voice-message.ogg]
|
|
205
|
+
--- from: assistant (TestBot)
|
|
206
|
+
-# » **tommy:** Hello from Whisper voice-openai
|
|
207
|
+
-# *using deterministic-provider/deterministic-v2 ⋅ build*
|
|
208
|
+
openai voice ok
|
|
209
|
+
-# *main ⋅ Ns ⋅ N% ⋅ deterministic-v2*"
|
|
210
|
+
`)
|
|
211
|
+
const request = whisper.requests.at(-1)
|
|
212
|
+
expect({ ...request, audio: request?.audio.equals(ogg) }).toMatchInlineSnapshot(`
|
|
213
|
+
{
|
|
214
|
+
"accountId": null,
|
|
215
|
+
"audio": true,
|
|
216
|
+
"authorization": null,
|
|
217
|
+
"filename": "voice.ogg",
|
|
218
|
+
"model": "Systran/faster-whisper-large-v3",
|
|
219
|
+
"path": "/v1/audio/transcriptions",
|
|
220
|
+
"type": "audio/ogg",
|
|
221
|
+
}
|
|
222
|
+
`)
|
|
223
|
+
|
|
224
|
+
// 400 is not retried; a 5xx would be retried twice first.
|
|
225
|
+
whisper.status = 400
|
|
226
|
+
await twin.discord.channel(twin.channelId).user(TEST_USER_ID).sendVoiceMessage({ url: `data:audio/ogg;base64,${ogg.toString('base64')}` })
|
|
227
|
+
const reply = await twin.discord.channel(twin.channelId).waitForMessage({ timeout: 8_000, predicate: (message) => message.content.includes('Transcription API returned 400') })
|
|
228
|
+
whisper.status = 200
|
|
229
|
+
expect(reply.content).toMatchInlineSnapshot(`"✗ Transcription API returned 400: {"detail":"unauthorized"}"`)
|
|
230
|
+
expect(openai.requests.length + chatgpt.requests.length).toBe(cloudRequests)
|
|
231
|
+
})
|
package/src/voice.ts
CHANGED
|
@@ -7,7 +7,8 @@
|
|
|
7
7
|
// Order: OpenAI (bot_api_keys, then OPENAI_API_KEY), the ChatGPT subscription credential
|
|
8
8
|
// stored by OpenCode (billed to the plan), Gemini (bot_api_keys, then GEMINI_API_KEY), then the
|
|
9
9
|
// free kimaki.dev Whisper endpoint on gateway installs. A failing provider falls through to the
|
|
10
|
-
// next one.
|
|
10
|
+
// next one. KIMAKI_WHISPER_URL replaces the whole list with a local Whisper server, so audio
|
|
11
|
+
// never reaches a cloud provider. ChatGPT and Whisper have no tool call: the route is always steer.
|
|
11
12
|
// Plain fetch, no AI SDK: we parse provider bodies ourselves so transient
|
|
12
13
|
// bad responses keep their details and can be retried.
|
|
13
14
|
|
|
@@ -47,6 +48,8 @@ const GEMINI_MODEL = 'gemini-flash-latest'
|
|
|
47
48
|
const TOOL_NAME = 'transcriptionResult'
|
|
48
49
|
const MAX_ATTEMPTS = 3
|
|
49
50
|
const RETRY_DELAY_MS = 500
|
|
51
|
+
// Later messages of the conversation wait for the transcription (ingress.ts): cap all providers and retries.
|
|
52
|
+
const TRANSCRIPTION_TIMEOUT_MS = 2 * 60_000
|
|
50
53
|
|
|
51
54
|
export class TranscriptionError extends errore.createTaggedError({
|
|
52
55
|
name: 'TranscriptionError',
|
|
@@ -376,11 +379,12 @@ async function openAIAudio({ audio, mediaType }: { audio: Buffer; mediaType: str
|
|
|
376
379
|
|
|
377
380
|
// --- requests
|
|
378
381
|
|
|
379
|
-
async function postJson({ url, headers, body }: { url: string; headers: Record<string, string>; body: unknown }) {
|
|
382
|
+
async function postJson({ url, headers, body, signal }: { url: string; headers: Record<string, string>; body: unknown; signal: AbortSignal }) {
|
|
380
383
|
const response = await fetch(url, {
|
|
381
384
|
method: 'POST',
|
|
382
385
|
headers: { 'content-type': 'application/json', ...headers },
|
|
383
386
|
body: JSON.stringify(body),
|
|
387
|
+
signal,
|
|
384
388
|
}).catch((cause) => new TranscriptionError({ reason: `request to ${url} failed`, cause }))
|
|
385
389
|
if (response instanceof Error) return response
|
|
386
390
|
const text = await response.text().catch((cause) => new TranscriptionError({ reason: 'reading the response failed', cause }))
|
|
@@ -391,8 +395,9 @@ async function postJson({ url, headers, body }: { url: string; headers: Record<s
|
|
|
391
395
|
|
|
392
396
|
type Tool = ReturnType<typeof buildTranscriptionTool>
|
|
393
397
|
|
|
394
|
-
async function requestGemini({ apiKey, baseUrl, prompt, audio, mediaType, tool }: ProviderRequest) {
|
|
398
|
+
async function requestGemini({ apiKey, baseUrl, prompt, audio, mediaType, tool, signal }: ProviderRequest) {
|
|
395
399
|
const raw = await postJson({
|
|
400
|
+
signal,
|
|
396
401
|
url: `${baseUrl}/models/${GEMINI_MODEL}:generateContent`,
|
|
397
402
|
headers: { 'x-goog-api-key': apiKey },
|
|
398
403
|
body: {
|
|
@@ -408,10 +413,11 @@ async function requestGemini({ apiKey, baseUrl, prompt, audio, mediaType, tool }
|
|
|
408
413
|
|
|
409
414
|
// TODO: use the ChatGPT subscription token here once OpenAI allows it. Today chat/completions
|
|
410
415
|
// rejects it (missing model.request scope), so the subscription only gets dictation without routing.
|
|
411
|
-
async function requestOpenAI({ apiKey, baseUrl, prompt, audio, mediaType, tool }: ProviderRequest) {
|
|
416
|
+
async function requestOpenAI({ apiKey, baseUrl, prompt, audio, mediaType, tool, signal }: ProviderRequest) {
|
|
412
417
|
const input = await openAIAudio({ audio, mediaType })
|
|
413
418
|
if (input instanceof Error) return input
|
|
414
419
|
const raw = await postJson({
|
|
420
|
+
signal,
|
|
415
421
|
url: `${baseUrl}/chat/completions`,
|
|
416
422
|
headers: { authorization: `Bearer ${apiKey}` },
|
|
417
423
|
body: {
|
|
@@ -435,7 +441,7 @@ async function requestOpenAI({ apiKey, baseUrl, prompt, audio, mediaType, tool }
|
|
|
435
441
|
return parseOpenAIResponse(raw)
|
|
436
442
|
}
|
|
437
443
|
|
|
438
|
-
type ProviderRequest = { apiKey: string; baseUrl: string; prompt: string; audio: Buffer; mediaType: string; tool: Tool }
|
|
444
|
+
type ProviderRequest = { apiKey: string; baseUrl: string; prompt: string; audio: Buffer; mediaType: string; tool: Tool; signal: AbortSignal }
|
|
439
445
|
|
|
440
446
|
// Retries transient failures: network, 408/409/429/5xx, bad bodies. Never
|
|
441
447
|
// bad keys, refusals or an empty transcription.
|
|
@@ -446,11 +452,11 @@ function isRetryable(error: TranscriptionFailure): boolean {
|
|
|
446
452
|
return status === 408 || status === 409 || status === 429 || status >= 500
|
|
447
453
|
}
|
|
448
454
|
|
|
449
|
-
async function withRetries(attempt: () => Promise<TranscriptionFailure | TranscriptionResult>) {
|
|
455
|
+
async function withRetries({ signal, attempt }: { signal: AbortSignal; attempt: () => Promise<TranscriptionFailure | TranscriptionResult> }) {
|
|
450
456
|
for (let number = 1; ; number++) {
|
|
451
457
|
const result = await attempt()
|
|
452
458
|
if (!(result instanceof Error)) return result
|
|
453
|
-
const retry = number < MAX_ATTEMPTS && isRetryable(result)
|
|
459
|
+
const retry = number < MAX_ATTEMPTS && !signal.aborted && isRetryable(result)
|
|
454
460
|
logger.warn(`transcription attempt ${number}/${MAX_ATTEMPTS} failed${retry ? ', retrying' : ''}`, result)
|
|
455
461
|
if (!retry) return result
|
|
456
462
|
await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS * number))
|
|
@@ -459,16 +465,18 @@ async function withRetries(attempt: () => Promise<TranscriptionFailure | Transcr
|
|
|
459
465
|
|
|
460
466
|
// Free Whisper through kimaki.dev, authenticated with the gateway client
|
|
461
467
|
// credentials (website POST /api/transcribe). No routing: always steer.
|
|
462
|
-
async function transcribeViaGateway({ audio, mediaType, clientId, clientSecret }: {
|
|
468
|
+
async function transcribeViaGateway({ audio, mediaType, clientId, clientSecret, signal }: {
|
|
463
469
|
audio: Buffer
|
|
464
470
|
mediaType: string
|
|
465
471
|
clientId: string
|
|
466
472
|
clientSecret: string
|
|
473
|
+
signal: AbortSignal
|
|
467
474
|
}): Promise<TranscriptionFailure | TranscriptionResult> {
|
|
468
475
|
const response = await fetch(new URL('/api/transcribe', gatewayUrlsFromEnv().website), {
|
|
469
476
|
method: 'POST',
|
|
470
477
|
headers: { authorization: `Bearer ${clientId}:${clientSecret}`, 'content-type': mediaType },
|
|
471
478
|
body: new Uint8Array(audio),
|
|
479
|
+
signal,
|
|
472
480
|
}).catch((cause) => new TranscriptionError({ reason: 'kimaki.dev transcription not reachable', cause }))
|
|
473
481
|
if (response instanceof Error) return response
|
|
474
482
|
const body = await response.text().catch(() => '')
|
|
@@ -489,7 +497,7 @@ async function transcribeViaGateway({ audio, mediaType, clientId, clientSecret }
|
|
|
489
497
|
|
|
490
498
|
type ChatGPTCredential = { accessToken: string; accountId: string | null }
|
|
491
499
|
|
|
492
|
-
const
|
|
500
|
+
const AUDIO_EXTENSIONS: Record<string, string> = { 'audio/ogg': 'ogg', 'audio/mp4': 'm4a', 'audio/mpeg': 'mp3', 'audio/wav': 'wav' }
|
|
493
501
|
|
|
494
502
|
// Same claim the Codex CLI sends as chatgpt-account-id.
|
|
495
503
|
function chatGPTAccountId(accessToken: string): string | null {
|
|
@@ -524,22 +532,22 @@ export async function chatGPTCredential({ dbFile, now = Date.now() }: { dbFile:
|
|
|
524
532
|
return null
|
|
525
533
|
}
|
|
526
534
|
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
535
|
+
// Multipart `file` (plus `fields`) in, `{ text }` out: ChatGPT dictation and Whisper servers.
|
|
536
|
+
async function transcribeFile({ name, url, headers, fields = {}, audio, mediaType, signal }: {
|
|
537
|
+
// For the error when the server is not reachable.
|
|
538
|
+
name: string
|
|
539
|
+
url: string
|
|
540
|
+
headers: Record<string, string>
|
|
541
|
+
fields?: Record<string, string>
|
|
530
542
|
audio: Buffer
|
|
531
543
|
mediaType: string
|
|
544
|
+
signal: AbortSignal
|
|
532
545
|
}): Promise<TranscriptionFailure | TranscriptionResult> {
|
|
533
546
|
const form = new FormData()
|
|
534
|
-
form.append('file', new File([new Uint8Array(audio)], `voice.${
|
|
535
|
-
const
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
authorization: `Bearer ${credential.accessToken}`,
|
|
539
|
-
...(credential.accountId && { 'chatgpt-account-id': credential.accountId }),
|
|
540
|
-
},
|
|
541
|
-
body: form,
|
|
542
|
-
}).catch((cause) => new TranscriptionError({ reason: 'ChatGPT transcription not reachable', cause }))
|
|
547
|
+
form.append('file', new File([new Uint8Array(audio)], `voice.${AUDIO_EXTENSIONS[mediaType] ?? 'ogg'}`, { type: mediaType }))
|
|
548
|
+
for (const [key, value] of Object.entries(fields)) form.append(key, value)
|
|
549
|
+
const response = await fetch(url, { method: 'POST', headers, body: form, signal })
|
|
550
|
+
.catch((cause) => new TranscriptionError({ reason: `${name} not reachable`, cause }))
|
|
543
551
|
if (response instanceof Error) return response
|
|
544
552
|
const body = await response.text().catch(() => '')
|
|
545
553
|
if (!response.ok) return new TranscriptionApiError({ status: String(response.status), body: body.slice(0, 300) })
|
|
@@ -548,6 +556,56 @@ async function transcribeViaChatGPT({ credential, baseUrl, audio, mediaType }: {
|
|
|
548
556
|
return toTranscriptionResult({ transcription: parsed['text'] })
|
|
549
557
|
}
|
|
550
558
|
|
|
559
|
+
function transcribeViaChatGPT({ credential, baseUrl, audio, mediaType, signal }: {
|
|
560
|
+
credential: ChatGPTCredential
|
|
561
|
+
baseUrl: string
|
|
562
|
+
audio: Buffer
|
|
563
|
+
mediaType: string
|
|
564
|
+
signal: AbortSignal
|
|
565
|
+
}) {
|
|
566
|
+
return transcribeFile({
|
|
567
|
+
name: 'ChatGPT transcription',
|
|
568
|
+
url: `${baseUrl}/transcribe`,
|
|
569
|
+
headers: {
|
|
570
|
+
authorization: `Bearer ${credential.accessToken}`,
|
|
571
|
+
...(credential.accountId && { 'chatgpt-account-id': credential.accountId }),
|
|
572
|
+
},
|
|
573
|
+
audio,
|
|
574
|
+
mediaType,
|
|
575
|
+
signal,
|
|
576
|
+
})
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
// --- Local Whisper (KIMAKI_WHISPER_URL)
|
|
580
|
+
//
|
|
581
|
+
// Any server with the standard OpenAI speech-to-text API: speaches, faster-whisper-server,
|
|
582
|
+
// whisper.cpp server, LocalAI. POST <url>/audio/transcriptions with multipart `file` + `model`.
|
|
583
|
+
// https://platform.openai.com/docs/api-reference/audio/createTranscription
|
|
584
|
+
|
|
585
|
+
type LocalWhisper = { url: string; model: string; apiKey: string | null }
|
|
586
|
+
|
|
587
|
+
function localWhisperFromEnv(): LocalWhisper | null {
|
|
588
|
+
const url = process.env['KIMAKI_WHISPER_URL']?.trim().replace(/\/+$/, '')
|
|
589
|
+
if (!url) return null
|
|
590
|
+
return {
|
|
591
|
+
url,
|
|
592
|
+
model: process.env['KIMAKI_WHISPER_MODEL']?.trim() || 'whisper-1',
|
|
593
|
+
apiKey: process.env['KIMAKI_WHISPER_API_KEY']?.trim() || null,
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
function transcribeViaLocalWhisper({ whisper, audio, mediaType, signal }: { whisper: LocalWhisper; audio: Buffer; mediaType: string; signal: AbortSignal }) {
|
|
598
|
+
return transcribeFile({
|
|
599
|
+
name: `Whisper server at ${whisper.url}`,
|
|
600
|
+
url: `${whisper.url}/audio/transcriptions`,
|
|
601
|
+
headers: whisper.apiKey ? { authorization: `Bearer ${whisper.apiKey}` } : {},
|
|
602
|
+
fields: { model: whisper.model },
|
|
603
|
+
audio,
|
|
604
|
+
mediaType,
|
|
605
|
+
signal,
|
|
606
|
+
})
|
|
607
|
+
}
|
|
608
|
+
|
|
551
609
|
// --- routing
|
|
552
610
|
|
|
553
611
|
export function parseVoiceMessage(result: TranscriptionResult): Route {
|
|
@@ -580,10 +638,14 @@ type Transcriber =
|
|
|
580
638
|
| { kind: 'openai' | 'gemini'; apiKey: string }
|
|
581
639
|
| { kind: 'chatgpt'; credential: ChatGPTCredential }
|
|
582
640
|
| { kind: 'gateway'; clientId: string; clientSecret: string }
|
|
641
|
+
| { kind: 'local-whisper'; whisper: LocalWhisper }
|
|
583
642
|
|
|
584
643
|
// Best first: OpenAI key (routing tool call), ChatGPT subscription and Gemini key, then hosted Whisper.
|
|
585
644
|
// ChatGPT and Whisper only transcribe, so the route is always steer.
|
|
645
|
+
// A local Whisper server is the only one: its failure must not send the audio to the cloud.
|
|
586
646
|
async function transcribers({ db, token, opencodeDbFile }: { db: KimakiDb; token: string; opencodeDbFile: string }): Promise<DbError | Transcriber[]> {
|
|
647
|
+
const whisper = localWhisperFromEnv()
|
|
648
|
+
if (whisper) return [{ kind: 'local-whisper', whisper }]
|
|
587
649
|
const [row, credential] = await Promise.all([
|
|
588
650
|
db.query.bot_tokens
|
|
589
651
|
.findFirst({ where: { token }, with: { api_keys: true } })
|
|
@@ -637,15 +699,20 @@ export async function transcribe({
|
|
|
637
699
|
const prompt = list.some((entry) => entry.kind === 'openai' || entry.kind === 'gemini')
|
|
638
700
|
? transcriptionPrompt({ fileTree: await projectFileTree(directory), agents })
|
|
639
701
|
: ''
|
|
702
|
+
const signal = AbortSignal.timeout(TRANSCRIPTION_TIMEOUT_MS)
|
|
640
703
|
const run = (entry: Transcriber) => {
|
|
641
|
-
if (entry.kind === 'gateway') return transcribeViaGateway({ audio, mediaType, ...entry })
|
|
642
|
-
if (entry.kind === '
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
704
|
+
if (entry.kind === 'gateway') return transcribeViaGateway({ audio, mediaType, ...entry, signal })
|
|
705
|
+
if (entry.kind === 'local-whisper') return withRetries({ signal, attempt: () => transcribeViaLocalWhisper({ whisper: entry.whisper, audio, mediaType, signal }) })
|
|
706
|
+
if (entry.kind === 'chatgpt') {
|
|
707
|
+
return withRetries({ signal, attempt: () => transcribeViaChatGPT({ credential: entry.credential, baseUrl: baseUrls.chatgpt ?? CHATGPT_BASE_URL, audio, mediaType, signal }) })
|
|
708
|
+
}
|
|
709
|
+
const request = { apiKey: entry.apiKey, prompt, audio, mediaType, tool, signal }
|
|
710
|
+
if (entry.kind === 'openai') return withRetries({ signal, attempt: () => requestOpenAI({ ...request, baseUrl: baseUrls.openai ?? OPENAI_BASE_URL }) })
|
|
711
|
+
return withRetries({ signal, attempt: () => requestGemini({ ...request, baseUrl: baseUrls.gemini ?? GEMINI_BASE_URL }) })
|
|
646
712
|
}
|
|
647
713
|
let last: TranscriptionFailure | NoTranscriptionKeyError = new NoTranscriptionKeyError()
|
|
648
714
|
for (const entry of list) {
|
|
715
|
+
if (signal.aborted) return new TranscriptionError({ reason: `no answer within ${TRANSCRIPTION_TIMEOUT_MS / 60_000} minutes` })
|
|
649
716
|
const result = await run(entry)
|
|
650
717
|
if (!(result instanceof Error) || result instanceof EmptyTranscriptionError) return result
|
|
651
718
|
logger.warn(`${entry.kind} transcription failed${entry === list.at(-1) ? '' : ', trying the next provider'}`, result)
|
|
@@ -61,10 +61,10 @@ test('session cwd changes preserve the session and use the new location after re
|
|
|
61
61
|
await waitForFooter({ discord: twin.discord, threadId: sent.threadId, count: 2 })
|
|
62
62
|
expect(hide(await twin.discord.thread(sent.threadId).text())).toMatchInlineSnapshot(`
|
|
63
63
|
"--- from: assistant (TestBot)
|
|
64
|
-
» **CLI:** Start cwd session
|
|
64
|
+
-# » **CLI:** Start cwd session
|
|
65
65
|
-# *using deterministic-provider/deterministic-v2 ⋅ build*
|
|
66
66
|
ok
|
|
67
|
-
-# *
|
|
67
|
+
-# *Ns ⋅ deterministic-v2*
|
|
68
68
|
-# Working directory changed to PROJECT/sub
|
|
69
69
|
--- from: user (tommy)
|
|
70
70
|
write-cwd-marker
|
|
@@ -72,14 +72,14 @@ test('session cwd changes preserve the session and use the new location after re
|
|
|
72
72
|
-# ┣ shell _Write to the current directory_
|
|
73
73
|
|
|
74
74
|
Wrote cwd.txt.
|
|
75
|
-
-# *sub ⋅
|
|
75
|
+
-# *sub ⋅ Ns ⋅ N% ⋅ deterministic-v2*"
|
|
76
76
|
`)
|
|
77
77
|
expect((await client.session.get({ sessionID: sent.sessionId })).location.directory).toBe(path.join(server.projectDirectory, 'sub'))
|
|
78
78
|
expect(fs.readFileSync(path.join(server.projectDirectory, 'sub', 'cwd.txt'), 'utf8')).toBe('isolated')
|
|
79
79
|
expect(fs.existsSync(path.join(server.projectDirectory, 'cwd.txt'))).toBe(false)
|
|
80
80
|
const source = await bot.discord.channels.fetch(sent.threadId)
|
|
81
81
|
if (!source?.isThread()) throw new Error('No thread')
|
|
82
|
-
const fork = await forkBtw(bot, {
|
|
82
|
+
const fork = await forkBtw(bot, { source, text: 'Side question', author: { id: TEST_USER_ID, username: 'tommy' }, messageId: 'cwd-btw' })
|
|
83
83
|
if (fork instanceof Error) throw fork
|
|
84
84
|
await waitForFooter({ discord: twin.discord, threadId: fork.threadId })
|
|
85
85
|
expect(hide(await twin.discord.thread(fork.threadId).text())).toMatchInlineSnapshot(`
|
|
@@ -87,7 +87,7 @@ test('session cwd changes preserve the session and use the new location after re
|
|
|
87
87
|
Reusing context from <#THREAD> to answer prompt...
|
|
88
88
|
Side question
|
|
89
89
|
ok
|
|
90
|
-
-# *sub ⋅
|
|
90
|
+
-# *sub ⋅ Ns ⋅ deterministic-v2*"
|
|
91
91
|
`)
|
|
92
92
|
expect((await client.session.get({ sessionID: fork.sessionId })).location.directory).toBe(path.join(server.projectDirectory, 'sub'))
|
|
93
93
|
})
|
|
@@ -105,11 +105,11 @@ test('native moves of a busy session update the live footer without aborting its
|
|
|
105
105
|
await waitForFooter({ discord: twin.discord, threadId: sent.threadId })
|
|
106
106
|
expect(hide(await twin.discord.thread(sent.threadId).text())).toMatchInlineSnapshot(`
|
|
107
107
|
"--- from: assistant (TestBot)
|
|
108
|
-
» **CLI:** busy-move-marker
|
|
108
|
+
-# » **CLI:** busy-move-marker
|
|
109
109
|
-# *using deterministic-provider/deterministic-v2 ⋅ build*
|
|
110
110
|
Finished without interruption.
|
|
111
111
|
-# Working directory changed to PROJECT/sub
|
|
112
|
-
-# *sub ⋅
|
|
112
|
+
-# *sub ⋅ Ns ⋅ N% ⋅ deterministic-v2*
|
|
113
113
|
Working directory: \`PROJECT/sub\`"
|
|
114
114
|
`)
|
|
115
115
|
expect((await client.session.get({ sessionID: sent.sessionId })).location.directory).toBe(path.join(server.projectDirectory, 'sub'))
|
|
@@ -143,7 +143,7 @@ test('new-worktree forks context, and CLI creation and worktree toggle use the s
|
|
|
143
143
|
await waitForFooter({ discord: twin.discord, threadId: created.threadId })
|
|
144
144
|
expect(hide(await twin.discord.thread(created.threadId).text())).toMatchInlineSnapshot(`
|
|
145
145
|
"--- from: assistant (TestBot)
|
|
146
|
-
» **CLI:** CLI isolation
|
|
146
|
+
-# » **CLI:** CLI isolation
|
|
147
147
|
-# *using deterministic-provider/deterministic-v2 ⋅ build*
|
|
148
148
|
ok
|
|
149
149
|
-# *cli-checkout ⋅ opencode/kimaki-cli-checkout ⋅ Ns ⋅ deterministic-v2*"
|
|
@@ -156,7 +156,7 @@ test('new-worktree forks context, and CLI creation and worktree toggle use the s
|
|
|
156
156
|
await waitForFooter({ discord: twin.discord, threadId: automatic.threadId })
|
|
157
157
|
expect(hide(await twin.discord.thread(automatic.threadId).text()).replace(/automatic-isolation-[a-f0-9]{8}/g, 'automatic-isolation-ID')).toMatchInlineSnapshot(`
|
|
158
158
|
"--- from: assistant (TestBot)
|
|
159
|
-
» **CLI:** Automatic isolation
|
|
159
|
+
-# » **CLI:** Automatic isolation
|
|
160
160
|
-# *using deterministic-provider/deterministic-v2 ⋅ build*
|
|
161
161
|
ok
|
|
162
162
|
-# *automatic-isolation-ID ⋅ opencode/kimaki-automatic-isolation-ID ⋅ Ns ⋅ deterministic-v2*"
|
|
@@ -172,7 +172,7 @@ test('send worktree creates an isolated named checkout, while explicit cwd reuse
|
|
|
172
172
|
await waitForFooter({ discord: twin.discord, threadId: sent.threadId })
|
|
173
173
|
expect(hide(await twin.discord.thread(sent.threadId).text())).toMatchInlineSnapshot(`
|
|
174
174
|
"--- from: assistant (TestBot)
|
|
175
|
-
» **CLI:** write-cwd-marker
|
|
175
|
+
-# » **CLI:** write-cwd-marker
|
|
176
176
|
-# *using deterministic-provider/deterministic-v2 ⋅ build*
|
|
177
177
|
-# ┣ shell _Write to the current directory_
|
|
178
178
|
|
|
@@ -189,7 +189,7 @@ test('send worktree creates an isolated named checkout, while explicit cwd reuse
|
|
|
189
189
|
await waitForFooter({ discord: twin.discord, threadId: reused.threadId })
|
|
190
190
|
expect(hide(await twin.discord.thread(reused.threadId).text())).toMatchInlineSnapshot(`
|
|
191
191
|
"--- from: assistant (TestBot)
|
|
192
|
-
» **CLI:** Reuse checkout
|
|
192
|
+
-# » **CLI:** Reuse checkout
|
|
193
193
|
-# *using deterministic-provider/deterministic-v2 ⋅ build*
|
|
194
194
|
ok
|
|
195
195
|
-# *isolated ⋅ opencode/kimaki-isolated ⋅ Ns ⋅ deterministic-v2*"
|