@markusylisiurunen/tau 0.2.102 → 0.2.104
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -39
- package/dist/core/async/server_config.js +0 -1
- package/dist/core/async/server_config.js.map +1 -1
- package/dist/core/async/session_manager.js +0 -3
- package/dist/core/async/session_manager.js.map +1 -1
- package/dist/core/async/telegram.js +1 -1
- package/dist/core/async/telegram.js.map +1 -1
- package/dist/core/cli.js +5 -9
- package/dist/core/cli.js.map +1 -1
- package/dist/core/commands/registry.js +17 -2
- package/dist/core/commands/registry.js.map +1 -1
- package/dist/core/config/index.js.map +1 -1
- package/dist/core/config/schema.js +21 -61
- package/dist/core/config/schema.js.map +1 -1
- package/dist/core/debug.js +4 -12
- package/dist/core/debug.js.map +1 -1
- package/dist/core/diff_review/review_thread.js +4 -7
- package/dist/core/diff_review/review_thread.js.map +1 -1
- package/dist/core/index.js +2 -1
- package/dist/core/index.js.map +1 -1
- package/dist/core/personas.js +44 -1
- package/dist/core/personas.js.map +1 -1
- package/dist/core/runtime/chat_runtime.js +0 -10
- package/dist/core/runtime/chat_runtime.js.map +1 -1
- package/dist/core/runtime/runtime_bootstrap.js +5 -67
- package/dist/core/runtime/runtime_bootstrap.js.map +1 -1
- package/dist/core/runtime/session_prompt_composer.js +2 -7
- package/dist/core/runtime/session_prompt_composer.js.map +1 -1
- package/dist/core/session/core_session.js.map +1 -1
- package/dist/core/session/session_engine.js +1 -13
- package/dist/core/session/session_engine.js.map +1 -1
- package/dist/core/static/prompts/diff-review-wrapper.md +4 -3
- package/dist/core/tool/cli.js +37 -0
- package/dist/core/tool/cli.js.map +1 -0
- package/dist/core/tool/errors.js +9 -0
- package/dist/core/tool/errors.js.map +1 -0
- package/dist/core/tool/index.js +3 -0
- package/dist/core/tool/index.js.map +1 -0
- package/dist/core/tool/pdf_unpack.js +406 -0
- package/dist/core/tool/pdf_unpack.js.map +1 -0
- package/dist/core/tools/bash.js +1 -1
- package/dist/core/tools/bash.js.map +1 -1
- package/dist/core/tools/edit.js +2 -2
- package/dist/core/tools/edit.js.map +1 -1
- package/dist/core/tools/emit_output.js +1 -1
- package/dist/core/tools/emit_output.js.map +1 -1
- package/dist/core/tools/execution_backend.js +21 -251
- package/dist/core/tools/execution_backend.js.map +1 -1
- package/dist/core/tools/grep.js +1 -1
- package/dist/core/tools/grep.js.map +1 -1
- package/dist/core/tools/list.js +1 -1
- package/dist/core/tools/list.js.map +1 -1
- package/dist/core/tools/read.js +1 -1
- package/dist/core/tools/read.js.map +1 -1
- package/dist/core/tools/registry.js.map +1 -1
- package/dist/core/tools/send_input_to_agent.js +1 -1
- package/dist/core/tools/send_input_to_agent.js.map +1 -1
- package/dist/core/tools/spawn_agent.js +7 -39
- package/dist/core/tools/spawn_agent.js.map +1 -1
- package/dist/core/tools/terminate_agent.js +1 -1
- package/dist/core/tools/terminate_agent.js.map +1 -1
- package/dist/core/tools/view_image.js +1 -1
- package/dist/core/tools/view_image.js.map +1 -1
- package/dist/core/tools/wait_for_agent.js +1 -1
- package/dist/core/tools/wait_for_agent.js.map +1 -1
- package/dist/core/tools/web_fetch.js +1 -1
- package/dist/core/tools/web_fetch.js.map +1 -1
- package/dist/core/tools/web_search.js +1 -1
- package/dist/core/tools/web_search.js.map +1 -1
- package/dist/core/tools/write.js +1 -1
- package/dist/core/tools/write.js.map +1 -1
- package/dist/core/utils/context.js +1 -1
- package/dist/core/utils/context.js.map +1 -1
- package/dist/core/utils/context_builder.js +1 -11
- package/dist/core/utils/context_builder.js.map +1 -1
- package/dist/core/utils/gemini_speech.js +367 -0
- package/dist/core/utils/gemini_speech.js.map +1 -0
- package/dist/core/utils/git.js +2 -7
- package/dist/core/utils/git.js.map +1 -1
- package/dist/core/utils/mistral_document_ocr.js +186 -0
- package/dist/core/utils/mistral_document_ocr.js.map +1 -0
- package/dist/core/version.js +1 -1
- package/dist/diff_tool/app/dist/assets/{index-AomQ5Wco.js → index-D1e3Qk8w.js} +26 -26
- package/dist/diff_tool/app/dist/index.html +1 -1
- package/dist/main.js +91 -112
- package/dist/main.js.map +1 -1
- package/dist/sdk/client.js +0 -1
- package/dist/sdk/client.js.map +1 -1
- package/dist/sdk/types.d.ts +0 -1
- package/dist/tui/app.js.map +1 -1
- package/dist/tui/chat_controller.js +231 -151
- package/dist/tui/chat_controller.js.map +1 -1
- package/dist/tui/chat_view.js +0 -1
- package/dist/tui/chat_view.js.map +1 -1
- package/dist/tui/terminal.js +8 -0
- package/dist/tui/terminal.js.map +1 -1
- package/dist/tui/ui/footer.js +6 -7
- package/dist/tui/ui/footer.js.map +1 -1
- package/package.json +5 -5
- package/dist/core/tools/sandbox/docker_sandbox.js +0 -183
- package/dist/core/tools/sandbox/docker_sandbox.js.map +0 -1
- package/dist/core/utils/agent_environment.js +0 -48
- package/dist/core/utils/agent_environment.js.map +0 -1
- package/dist/core/utils/sandbox_paths.js +0 -29
- package/dist/core/utils/sandbox_paths.js.map +0 -1
- package/dist/core/utils/sandbox_prompt_paths.js +0 -86
- package/dist/core/utils/sandbox_prompt_paths.js.map +0 -1
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import {
|
|
2
|
+
import { statSync } from "node:fs";
|
|
3
3
|
import { mkdtemp, readFile, unlink, writeFile } from "node:fs/promises";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
|
-
import { dirname, join,
|
|
5
|
+
import { dirname, join, resolve } from "node:path";
|
|
6
6
|
import { z } from "zod";
|
|
7
7
|
import { formatCodexAuthError } from "../core/auth/auth_messages.js";
|
|
8
8
|
import { getAuthPath } from "../core/auth/auth_paths.js";
|
|
@@ -19,11 +19,10 @@ import { buildBashUiText, formatBashUserMessageText, getBashOutputPolicy, prepar
|
|
|
19
19
|
import { ToolCatalog } from "../core/tools/catalog.js";
|
|
20
20
|
import { createLocalToolExecutionBackend } from "../core/tools/execution_backend.js";
|
|
21
21
|
import { REASONING_LEVELS, } from "../core/types.js";
|
|
22
|
-
import { resolveAgentCwd, resolveSandboxPath } from "../core/utils/agent_environment.js";
|
|
23
22
|
import { findAgentsFilesFromCwdToHome } from "../core/utils/agents_files.js";
|
|
24
23
|
import { formatCwdChangeNotice, formatProjectContextChangeNotice, formatRiskLevelChangeNotice, } from "../core/utils/context.js";
|
|
25
24
|
import { formatAdaptiveNumber, formatCwd, formatPathForDisplay, formatTokenWindow, } from "../core/utils/format.js";
|
|
26
|
-
import {
|
|
25
|
+
import { generateGeminiSpeech } from "../core/utils/gemini_speech.js";
|
|
27
26
|
import { extractAllFencedCodeBlocks, extractAssistantText } from "../core/utils/messages.js";
|
|
28
27
|
import { transcribeMistralAudio } from "../core/utils/mistral_transcription.js";
|
|
29
28
|
import { streamModel } from "../core/utils/model_stream.js";
|
|
@@ -55,9 +54,11 @@ const RELOAD_PLANS = {
|
|
|
55
54
|
},
|
|
56
55
|
};
|
|
57
56
|
const ALLOWED_RISK_LEVELS = ["read-only", "read-write"];
|
|
57
|
+
const LISTEN_TEMP_FILE_TEMPLATE = "/tmp/tau-listen.XXXXXX";
|
|
58
58
|
const SPEAK_TEMP_FILE_TEMPLATE = "/tmp/tau-speak.XXXXXX";
|
|
59
|
-
const
|
|
60
|
-
const
|
|
59
|
+
const LISTEN_RECORDING_MIN_BYTES = 1024;
|
|
60
|
+
const LISTEN_RECORDING_MAX_DURATION_MS = 5 * 60 * 1000;
|
|
61
|
+
const SPEAK_PLAYBACK_RATE = 1.4;
|
|
61
62
|
const CAFFEINATE_COMMAND = "/usr/bin/caffeinate";
|
|
62
63
|
const SmartPruneResponseSchema = z.object({
|
|
63
64
|
prune: z.array(z.string().trim().min(1)),
|
|
@@ -76,9 +77,7 @@ export class ChatController {
|
|
|
76
77
|
activeThemeId;
|
|
77
78
|
credentialResolver;
|
|
78
79
|
authPath;
|
|
79
|
-
sandboxEnabled;
|
|
80
80
|
caffeinated;
|
|
81
|
-
sandboxRootReal;
|
|
82
81
|
agentCwd;
|
|
83
82
|
includeAgentContext;
|
|
84
83
|
runtime;
|
|
@@ -86,7 +85,6 @@ export class ChatController {
|
|
|
86
85
|
commandRegistry;
|
|
87
86
|
commandHandlers;
|
|
88
87
|
toolBackend;
|
|
89
|
-
toolBackendDispose;
|
|
90
88
|
deps;
|
|
91
89
|
eventUnsubscribe;
|
|
92
90
|
isStreaming = false;
|
|
@@ -100,6 +98,7 @@ export class ChatController {
|
|
|
100
98
|
showThinking = false;
|
|
101
99
|
compactToolUi = true;
|
|
102
100
|
commandHint;
|
|
101
|
+
speechStatusHint;
|
|
103
102
|
riskLevel = "read-only";
|
|
104
103
|
projectContextBlock;
|
|
105
104
|
projectFiles = [];
|
|
@@ -118,9 +117,10 @@ export class ChatController {
|
|
|
118
117
|
lastTurnDurationMs = 0;
|
|
119
118
|
turnTimer;
|
|
120
119
|
lastEmptySubmitAt;
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
120
|
+
listenRecording;
|
|
121
|
+
isTranscribingListen = false;
|
|
122
|
+
listenTransition;
|
|
123
|
+
speakTask;
|
|
124
124
|
turnCaffeinate;
|
|
125
125
|
disableCaffeinateForSession = false;
|
|
126
126
|
constructor(options) {
|
|
@@ -139,9 +139,7 @@ export class ChatController {
|
|
|
139
139
|
this.initialUserMessage = options.initialUserMessage;
|
|
140
140
|
this.config = options.config ?? {};
|
|
141
141
|
this.defaultDiffTool = options.defaultDiffTool;
|
|
142
|
-
this.sandboxEnabled = options.sandboxEnabled;
|
|
143
142
|
this.caffeinated = options.caffeinated ?? false;
|
|
144
|
-
this.sandboxRootReal = this.sandboxEnabled ? this.resolveSandboxRoot(cwd) : undefined;
|
|
145
143
|
this.activeThemeId = this.config.defaultTheme;
|
|
146
144
|
this.authPath = getAuthPath(this.deps.env.home());
|
|
147
145
|
const authStorage = new AuthStorage(this.authPath);
|
|
@@ -153,7 +151,6 @@ export class ChatController {
|
|
|
153
151
|
const queuedUserMessages = options.queuedUserMessages ?? [];
|
|
154
152
|
this.queuedMessageBuffer = new QueuedUserMessages(queuedUserMessages);
|
|
155
153
|
this.interruptLifecycle = new InterruptLifecycle();
|
|
156
|
-
this.toolBackendDispose = options.toolBackendDispose;
|
|
157
154
|
this.includeAgentContext = !options.noAgentContextFiles;
|
|
158
155
|
const initialPersona = (options.initialPersonaId &&
|
|
159
156
|
this.personas.find((p) => p.id.toLowerCase() === options.initialPersonaId.toLowerCase())) ||
|
|
@@ -175,10 +172,6 @@ export class ChatController {
|
|
|
175
172
|
cwd,
|
|
176
173
|
home,
|
|
177
174
|
includeAgentContext: this.includeAgentContext,
|
|
178
|
-
sandboxEnabled: this.sandboxEnabled,
|
|
179
|
-
sandboxConfig: this.config.sandbox,
|
|
180
|
-
sandboxHostRoot: this.sandboxRootReal,
|
|
181
|
-
sandboxEnvironmentInfo: this.config.sandbox?.environmentInfo,
|
|
182
175
|
readFile: this.deps.fs.readFile,
|
|
183
176
|
});
|
|
184
177
|
this.agentCwd = startupBootstrap.promptContext.cwd;
|
|
@@ -194,9 +187,6 @@ export class ChatController {
|
|
|
194
187
|
spawn: this.deps.spawn,
|
|
195
188
|
env: this.deps.env,
|
|
196
189
|
});
|
|
197
|
-
if (this.sandboxEnabled && this.toolBackend.kind !== "sandbox") {
|
|
198
|
-
throw new Error("sandbox enabled but tool backend is not sandboxed.");
|
|
199
|
-
}
|
|
200
190
|
const toolRegistry = ToolCatalog.createRegistry(this.toolBackend);
|
|
201
191
|
this.runtime = ChatRuntime.create({
|
|
202
192
|
persona: this.currentPersona,
|
|
@@ -249,7 +239,8 @@ export class ChatController {
|
|
|
249
239
|
pruneLargest: (extra) => this.pruneToolResults("largest", extra),
|
|
250
240
|
pruneSmart: (extra) => this.pruneToolResultsSmart(extra),
|
|
251
241
|
reload: () => this.reloadContent(),
|
|
252
|
-
|
|
242
|
+
listen: () => this.startListenCaptureFromCommand(),
|
|
243
|
+
speak: () => this.speakLastAssistantMessage(),
|
|
253
244
|
risk: (level) => this.setRiskLevel(level),
|
|
254
245
|
persona: (id) => this.switchPersona(id),
|
|
255
246
|
prompt: (id) => this.insertPrompt(id),
|
|
@@ -296,7 +287,7 @@ export class ChatController {
|
|
|
296
287
|
onCtrlR: () => this.cycleRiskLevel(),
|
|
297
288
|
onCtrlP: () => this.cyclePersonality(),
|
|
298
289
|
onCtrlS: () => void this.stashEditorToClipboard(),
|
|
299
|
-
onCtrlY: () => void this.
|
|
290
|
+
onCtrlY: () => void this.toggleListenCapture(),
|
|
300
291
|
onEscape: () => this.onInterrupt(),
|
|
301
292
|
onCtrlF: () => {
|
|
302
293
|
this.expandFileMentions().catch((err) => {
|
|
@@ -318,23 +309,31 @@ export class ChatController {
|
|
|
318
309
|
}
|
|
319
310
|
async dispose() {
|
|
320
311
|
this.eventUnsubscribe?.();
|
|
321
|
-
if (this.
|
|
322
|
-
await this.
|
|
312
|
+
if (this.listenTransition) {
|
|
313
|
+
await this.listenTransition;
|
|
314
|
+
}
|
|
315
|
+
if (this.speakTask) {
|
|
316
|
+
this.speakTask.abortController.abort();
|
|
317
|
+
await this.speakTask.completion;
|
|
323
318
|
}
|
|
324
319
|
await this.cancelDiffReview();
|
|
325
|
-
await this.
|
|
320
|
+
await this.cancelListenCapture();
|
|
326
321
|
await this.stopTurnCaffeinate();
|
|
327
|
-
if (!this.toolBackendDispose)
|
|
328
|
-
return;
|
|
329
|
-
await this.toolBackendDispose();
|
|
330
322
|
}
|
|
331
323
|
// Mode Adapter ---------------------------------------------------------------------------------
|
|
332
324
|
async onUserInput(text) {
|
|
333
325
|
await this.handleSubmit(text);
|
|
334
326
|
}
|
|
335
327
|
onInterrupt() {
|
|
336
|
-
if (this.
|
|
337
|
-
void this.
|
|
328
|
+
if (this.listenRecording) {
|
|
329
|
+
void this.runListenTransition(() => this.stopListenCapture());
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
if (this.speakTask) {
|
|
333
|
+
if (!this.speakTask.abortController.signal.aborted) {
|
|
334
|
+
this.speakTask.abortController.abort();
|
|
335
|
+
this.view.addSystemMessage("interrupted", "error");
|
|
336
|
+
}
|
|
338
337
|
return;
|
|
339
338
|
}
|
|
340
339
|
this.interruptActiveTask();
|
|
@@ -491,7 +490,6 @@ export class ChatController {
|
|
|
491
490
|
sessionCost,
|
|
492
491
|
duration,
|
|
493
492
|
riskLevel: this.riskLevel,
|
|
494
|
-
sandboxed: this.sandboxEnabled,
|
|
495
493
|
commandHint: this.getActiveCommandHint(),
|
|
496
494
|
},
|
|
497
495
|
editor: {
|
|
@@ -504,7 +502,7 @@ export class ChatController {
|
|
|
504
502
|
});
|
|
505
503
|
}
|
|
506
504
|
getInputMode() {
|
|
507
|
-
if (this.
|
|
505
|
+
if (this.listenRecording)
|
|
508
506
|
return "recording";
|
|
509
507
|
if (this.isBashIncognito)
|
|
510
508
|
return "bash_incognito";
|
|
@@ -515,7 +513,7 @@ export class ChatController {
|
|
|
515
513
|
return "normal";
|
|
516
514
|
}
|
|
517
515
|
getActiveCommandHint() {
|
|
518
|
-
return this.diffReviewService.getCommandHint(this.commandHint);
|
|
516
|
+
return this.diffReviewService.getCommandHint(this.speechStatusHint ?? this.commandHint);
|
|
519
517
|
}
|
|
520
518
|
// Context & Cost Tracking -----------------------------------------------------------------------
|
|
521
519
|
getContextUsageString() {
|
|
@@ -674,10 +672,6 @@ export class ChatController {
|
|
|
674
672
|
const resolved = resolvePersonaSkillsForPromptContext({
|
|
675
673
|
persona,
|
|
676
674
|
discoveredSkills: this.skills,
|
|
677
|
-
cwd: this.deps.env.cwd(),
|
|
678
|
-
sandboxEnabled: this.sandboxEnabled,
|
|
679
|
-
sandboxConfig: this.config.sandbox,
|
|
680
|
-
sandboxHostRoot: this.sandboxRootReal,
|
|
681
675
|
});
|
|
682
676
|
return { skillsBlock: resolved.skillsBlock, unknown: resolved.unknown };
|
|
683
677
|
}
|
|
@@ -724,10 +718,6 @@ export class ChatController {
|
|
|
724
718
|
this.view.addSystemMessage(`not a directory: ${normalized}`, "error");
|
|
725
719
|
return;
|
|
726
720
|
}
|
|
727
|
-
if (this.sandboxEnabled && !this.isPathWithinSandboxRoot(resolved)) {
|
|
728
|
-
this.view.addSystemMessage(`directory is outside the sandbox mount: ${normalized}`, "error");
|
|
729
|
-
return;
|
|
730
|
-
}
|
|
731
721
|
try {
|
|
732
722
|
process.chdir(resolved);
|
|
733
723
|
}
|
|
@@ -738,18 +728,7 @@ export class ChatController {
|
|
|
738
728
|
const nextCwd = this.deps.env.cwd();
|
|
739
729
|
const previousAgentCwd = this.agentCwd;
|
|
740
730
|
const previousProjectContextBlock = this.projectContextBlock;
|
|
741
|
-
this.agentCwd =
|
|
742
|
-
this.sandboxEnabled && this.sandboxRootReal
|
|
743
|
-
? resolveSandboxPath({
|
|
744
|
-
hostPath: nextCwd,
|
|
745
|
-
cwd: this.sandboxRootReal,
|
|
746
|
-
sandboxConfig: this.config.sandbox,
|
|
747
|
-
})
|
|
748
|
-
: resolveAgentCwd({
|
|
749
|
-
cwd: nextCwd,
|
|
750
|
-
sandboxEnabled: this.sandboxEnabled,
|
|
751
|
-
sandboxConfig: this.config.sandbox,
|
|
752
|
-
});
|
|
731
|
+
this.agentCwd = nextCwd;
|
|
753
732
|
this.refreshProjectContext(nextCwd);
|
|
754
733
|
this.updatePendingProjectContextChange(previousProjectContextBlock, this.projectContextBlock);
|
|
755
734
|
this.projectFiles = [];
|
|
@@ -784,36 +763,11 @@ export class ChatController {
|
|
|
784
763
|
return join(home, input.slice(2));
|
|
785
764
|
return resolve(cwd, input);
|
|
786
765
|
}
|
|
787
|
-
resolveSandboxRoot(cwd) {
|
|
788
|
-
try {
|
|
789
|
-
const root = getGitRoot(cwd) ?? cwd;
|
|
790
|
-
return realpathSync(root);
|
|
791
|
-
}
|
|
792
|
-
catch {
|
|
793
|
-
return undefined;
|
|
794
|
-
}
|
|
795
|
-
}
|
|
796
|
-
isPathWithinSandboxRoot(targetPath) {
|
|
797
|
-
if (!this.sandboxRootReal)
|
|
798
|
-
return true;
|
|
799
|
-
let resolved = targetPath;
|
|
800
|
-
try {
|
|
801
|
-
resolved = realpathSync(targetPath);
|
|
802
|
-
}
|
|
803
|
-
catch {
|
|
804
|
-
// fall back to provided path
|
|
805
|
-
}
|
|
806
|
-
const rel = relative(this.sandboxRootReal, resolved);
|
|
807
|
-
return !(rel === ".." || rel.startsWith(`..${sep}`));
|
|
808
|
-
}
|
|
809
766
|
refreshProjectContext(cwd) {
|
|
810
767
|
const projectContext = resolveProjectContextForPromptContext({
|
|
811
768
|
cwd,
|
|
812
769
|
home: this.deps.env.home(),
|
|
813
770
|
includeAgentContext: this.includeAgentContext,
|
|
814
|
-
sandboxEnabled: this.sandboxEnabled,
|
|
815
|
-
sandboxConfig: this.config.sandbox,
|
|
816
|
-
sandboxHostRoot: this.sandboxRootReal,
|
|
817
771
|
readFile: this.deps.fs.readFile,
|
|
818
772
|
});
|
|
819
773
|
this.agentsFiles = projectContext.agentsFiles;
|
|
@@ -880,7 +834,7 @@ export class ChatController {
|
|
|
880
834
|
}
|
|
881
835
|
// Input Handling --------------------------------------------------------------------------------
|
|
882
836
|
beforeSubmit(text) {
|
|
883
|
-
if (this.
|
|
837
|
+
if (this.listenRecording)
|
|
884
838
|
return false;
|
|
885
839
|
if (!this.isStreaming)
|
|
886
840
|
return true;
|
|
@@ -970,8 +924,10 @@ export class ChatController {
|
|
|
970
924
|
return "prune smart-selected tool results and compact edit calls, optional fraction and guidance";
|
|
971
925
|
case "reload":
|
|
972
926
|
return "reload prompts, skills, themes, bash commands, and AGENTS.md";
|
|
927
|
+
case "listen":
|
|
928
|
+
return "start microphone recording and transcribe to editor (macOS only)";
|
|
973
929
|
case "speak":
|
|
974
|
-
return "
|
|
930
|
+
return "speak the last assistant message aloud (macOS only)";
|
|
975
931
|
case "risk":
|
|
976
932
|
return "set risk level: /risk:read-only or /risk:read-write";
|
|
977
933
|
case "bash":
|
|
@@ -1212,16 +1168,27 @@ export class ChatController {
|
|
|
1212
1168
|
this.view.addMessage({ type: "user", text: trimmed }, historyEntryId);
|
|
1213
1169
|
await this.runAssistantTurn();
|
|
1214
1170
|
}
|
|
1215
|
-
async
|
|
1216
|
-
if (this.
|
|
1171
|
+
async toggleListenCapture() {
|
|
1172
|
+
if (this.listenTransition) {
|
|
1173
|
+
this.view.addSystemMessage("speech recording state change already in progress", "warn");
|
|
1174
|
+
return;
|
|
1175
|
+
}
|
|
1176
|
+
if (this.listenRecording) {
|
|
1177
|
+
await this.runListenTransition(() => this.stopListenCapture());
|
|
1178
|
+
return;
|
|
1179
|
+
}
|
|
1180
|
+
await this.startListenCaptureFromCommand();
|
|
1181
|
+
}
|
|
1182
|
+
async startListenCaptureFromCommand() {
|
|
1183
|
+
if (this.listenTransition) {
|
|
1217
1184
|
this.view.addSystemMessage("speech recording state change already in progress", "warn");
|
|
1218
1185
|
return;
|
|
1219
1186
|
}
|
|
1220
|
-
if (this.
|
|
1221
|
-
|
|
1187
|
+
if (this.listenRecording) {
|
|
1188
|
+
this.view.addSystemMessage("speech recording already in progress", "warn");
|
|
1222
1189
|
return;
|
|
1223
1190
|
}
|
|
1224
|
-
if (this.
|
|
1191
|
+
if (this.isTranscribingListen) {
|
|
1225
1192
|
this.view.addSystemMessage("speech transcription already in progress", "warn");
|
|
1226
1193
|
return;
|
|
1227
1194
|
}
|
|
@@ -1229,36 +1196,36 @@ export class ChatController {
|
|
|
1229
1196
|
this.view.addSystemMessage("wait for the assistant to finish before recording", "warn");
|
|
1230
1197
|
return;
|
|
1231
1198
|
}
|
|
1232
|
-
await this.
|
|
1199
|
+
await this.runListenTransition(() => this.startListenCapture());
|
|
1233
1200
|
}
|
|
1234
|
-
async
|
|
1235
|
-
if (this.
|
|
1201
|
+
async runListenTransition(task) {
|
|
1202
|
+
if (this.listenTransition) {
|
|
1236
1203
|
return;
|
|
1237
1204
|
}
|
|
1238
1205
|
const transition = task();
|
|
1239
|
-
this.
|
|
1206
|
+
this.listenTransition = transition;
|
|
1240
1207
|
try {
|
|
1241
1208
|
await transition;
|
|
1242
1209
|
}
|
|
1243
1210
|
finally {
|
|
1244
|
-
if (this.
|
|
1245
|
-
this.
|
|
1211
|
+
if (this.listenTransition === transition) {
|
|
1212
|
+
this.listenTransition = undefined;
|
|
1246
1213
|
}
|
|
1247
1214
|
}
|
|
1248
1215
|
}
|
|
1249
|
-
async
|
|
1216
|
+
async startListenCapture() {
|
|
1250
1217
|
if (this.deps.env.platform() !== "darwin") {
|
|
1251
|
-
this.view.addSystemMessage("/
|
|
1218
|
+
this.view.addSystemMessage("/listen is currently supported only on macOS.", "warn");
|
|
1252
1219
|
return;
|
|
1253
1220
|
}
|
|
1254
1221
|
const apiKey = getMistralApiKey(this.config, this.deps.env.env());
|
|
1255
1222
|
if (!apiKey) {
|
|
1256
|
-
this.view.addSystemMessage("set MISTRAL_API_KEY or apiKeys.mistral to use /
|
|
1223
|
+
this.view.addSystemMessage("set MISTRAL_API_KEY or apiKeys.mistral to use /listen", "error");
|
|
1257
1224
|
return;
|
|
1258
1225
|
}
|
|
1259
1226
|
let audioPath;
|
|
1260
1227
|
try {
|
|
1261
|
-
audioPath = await this.
|
|
1228
|
+
audioPath = await this.createTempFilePath(LISTEN_TEMP_FILE_TEMPLATE);
|
|
1262
1229
|
const abortController = new AbortController();
|
|
1263
1230
|
const completion = this.deps.spawn("ffmpeg", [
|
|
1264
1231
|
"-hide_banner",
|
|
@@ -1292,29 +1259,29 @@ export class ChatController {
|
|
|
1292
1259
|
completion,
|
|
1293
1260
|
};
|
|
1294
1261
|
recording.maxDurationTimeout = setTimeout(() => {
|
|
1295
|
-
if (this.
|
|
1262
|
+
if (this.listenRecording !== recording || this.listenTransition)
|
|
1296
1263
|
return;
|
|
1297
|
-
void this.
|
|
1298
|
-
},
|
|
1299
|
-
this.
|
|
1264
|
+
void this.runListenTransition(() => this.stopListenCapture());
|
|
1265
|
+
}, LISTEN_RECORDING_MAX_DURATION_MS);
|
|
1266
|
+
this.listenRecording = recording;
|
|
1300
1267
|
this.view.setEditorInputEnabled(false);
|
|
1301
1268
|
this.refreshStatus();
|
|
1302
|
-
void this.
|
|
1269
|
+
void this.watchListenRecording(recording);
|
|
1303
1270
|
}
|
|
1304
1271
|
catch (err) {
|
|
1305
1272
|
if (audioPath) {
|
|
1306
|
-
await this.
|
|
1273
|
+
await this.cleanupTempFile(audioPath);
|
|
1307
1274
|
}
|
|
1308
1275
|
this.view.addSystemMessage(`failed to start recording: ${err.message}`, "error");
|
|
1309
1276
|
}
|
|
1310
1277
|
}
|
|
1311
|
-
async
|
|
1312
|
-
const recording = this.
|
|
1278
|
+
async stopListenCapture() {
|
|
1279
|
+
const recording = this.listenRecording;
|
|
1313
1280
|
if (!recording)
|
|
1314
1281
|
return;
|
|
1315
1282
|
recording.stopRequested = true;
|
|
1316
|
-
this.
|
|
1317
|
-
this.
|
|
1283
|
+
this.clearListenRecordingMaxDurationTimeout(recording);
|
|
1284
|
+
this.listenRecording = undefined;
|
|
1318
1285
|
this.view.setEditorInputEnabled(true);
|
|
1319
1286
|
this.refreshStatus();
|
|
1320
1287
|
recording.abortController.abort();
|
|
@@ -1323,17 +1290,17 @@ export class ChatController {
|
|
|
1323
1290
|
}
|
|
1324
1291
|
catch (err) {
|
|
1325
1292
|
this.view.addSystemMessage(`recording failed: ${err.message}`, "error");
|
|
1326
|
-
await this.
|
|
1293
|
+
await this.cleanupTempFile(recording.audioPath);
|
|
1327
1294
|
return;
|
|
1328
1295
|
}
|
|
1329
|
-
this.
|
|
1296
|
+
this.isTranscribingListen = true;
|
|
1330
1297
|
try {
|
|
1331
1298
|
const audio = await readFile(recording.audioPath);
|
|
1332
|
-
if (audio.byteLength <
|
|
1299
|
+
if (audio.byteLength < LISTEN_RECORDING_MIN_BYTES) {
|
|
1333
1300
|
this.view.addSystemMessage("recording too short, try again", "warn");
|
|
1334
1301
|
return;
|
|
1335
1302
|
}
|
|
1336
|
-
const transcript = await this.
|
|
1303
|
+
const transcript = await this.transcribeListenAudio(audio);
|
|
1337
1304
|
const text = transcript.trim();
|
|
1338
1305
|
if (!text) {
|
|
1339
1306
|
return;
|
|
@@ -1344,17 +1311,17 @@ export class ChatController {
|
|
|
1344
1311
|
this.view.addSystemMessage(`speech transcription failed: ${err.message}`, "error");
|
|
1345
1312
|
}
|
|
1346
1313
|
finally {
|
|
1347
|
-
this.
|
|
1348
|
-
await this.
|
|
1314
|
+
this.isTranscribingListen = false;
|
|
1315
|
+
await this.cleanupTempFile(recording.audioPath);
|
|
1349
1316
|
}
|
|
1350
1317
|
}
|
|
1351
|
-
async
|
|
1352
|
-
const recording = this.
|
|
1318
|
+
async cancelListenCapture() {
|
|
1319
|
+
const recording = this.listenRecording;
|
|
1353
1320
|
if (!recording)
|
|
1354
1321
|
return;
|
|
1355
1322
|
recording.stopRequested = true;
|
|
1356
|
-
this.
|
|
1357
|
-
this.
|
|
1323
|
+
this.clearListenRecordingMaxDurationTimeout(recording);
|
|
1324
|
+
this.listenRecording = undefined;
|
|
1358
1325
|
this.view.setEditorInputEnabled(true);
|
|
1359
1326
|
this.refreshStatus();
|
|
1360
1327
|
recording.abortController.abort();
|
|
@@ -1364,15 +1331,15 @@ export class ChatController {
|
|
|
1364
1331
|
catch {
|
|
1365
1332
|
// ignore disposal errors
|
|
1366
1333
|
}
|
|
1367
|
-
await this.
|
|
1334
|
+
await this.cleanupTempFile(recording.audioPath);
|
|
1368
1335
|
}
|
|
1369
|
-
async
|
|
1336
|
+
async watchListenRecording(recording) {
|
|
1370
1337
|
try {
|
|
1371
1338
|
const result = await recording.completion;
|
|
1372
|
-
this.
|
|
1373
|
-
if (this.
|
|
1339
|
+
this.clearListenRecordingMaxDurationTimeout(recording);
|
|
1340
|
+
if (this.listenRecording !== recording || recording.stopRequested)
|
|
1374
1341
|
return;
|
|
1375
|
-
this.
|
|
1342
|
+
this.listenRecording = undefined;
|
|
1376
1343
|
this.view.setEditorInputEnabled(true);
|
|
1377
1344
|
this.refreshStatus();
|
|
1378
1345
|
const detail = result.exitCode !== null
|
|
@@ -1381,13 +1348,13 @@ export class ChatController {
|
|
|
1381
1348
|
? `ffmpeg terminated by signal ${result.closeSignal}`
|
|
1382
1349
|
: "ffmpeg exited";
|
|
1383
1350
|
this.view.addSystemMessage(`recording stopped unexpectedly (${detail})`, "error");
|
|
1384
|
-
await this.
|
|
1351
|
+
await this.cleanupTempFile(recording.audioPath);
|
|
1385
1352
|
}
|
|
1386
1353
|
catch (err) {
|
|
1387
|
-
this.
|
|
1388
|
-
if (this.
|
|
1354
|
+
this.clearListenRecordingMaxDurationTimeout(recording);
|
|
1355
|
+
if (this.listenRecording !== recording || recording.stopRequested)
|
|
1389
1356
|
return;
|
|
1390
|
-
this.
|
|
1357
|
+
this.listenRecording = undefined;
|
|
1391
1358
|
this.view.setEditorInputEnabled(true);
|
|
1392
1359
|
this.refreshStatus();
|
|
1393
1360
|
const error = err;
|
|
@@ -1397,17 +1364,17 @@ export class ChatController {
|
|
|
1397
1364
|
else {
|
|
1398
1365
|
this.view.addSystemMessage(`recording failed: ${error.message}`, "error");
|
|
1399
1366
|
}
|
|
1400
|
-
await this.
|
|
1367
|
+
await this.cleanupTempFile(recording.audioPath);
|
|
1401
1368
|
}
|
|
1402
1369
|
}
|
|
1403
|
-
|
|
1370
|
+
clearListenRecordingMaxDurationTimeout(recording) {
|
|
1404
1371
|
if (!recording.maxDurationTimeout)
|
|
1405
1372
|
return;
|
|
1406
1373
|
clearTimeout(recording.maxDurationTimeout);
|
|
1407
1374
|
recording.maxDurationTimeout = undefined;
|
|
1408
1375
|
}
|
|
1409
|
-
async
|
|
1410
|
-
const result = await this.deps.spawn("mktemp", [
|
|
1376
|
+
async createTempFilePath(template) {
|
|
1377
|
+
const result = await this.deps.spawn("mktemp", [template]);
|
|
1411
1378
|
if (result.exitCode !== 0) {
|
|
1412
1379
|
const message = result.stderr.trim() || result.stdout.trim() || "mktemp failed";
|
|
1413
1380
|
throw new Error(message);
|
|
@@ -1418,7 +1385,7 @@ export class ChatController {
|
|
|
1418
1385
|
}
|
|
1419
1386
|
return path;
|
|
1420
1387
|
}
|
|
1421
|
-
async
|
|
1388
|
+
async transcribeListenAudio(audio) {
|
|
1422
1389
|
const apiKey = getMistralApiKey(this.config, this.deps.env.env());
|
|
1423
1390
|
if (!apiKey) {
|
|
1424
1391
|
throw new Error("missing MISTRAL_API_KEY or apiKeys.mistral");
|
|
@@ -1431,7 +1398,7 @@ export class ChatController {
|
|
|
1431
1398
|
language: "en",
|
|
1432
1399
|
});
|
|
1433
1400
|
}
|
|
1434
|
-
async
|
|
1401
|
+
async cleanupTempFile(path) {
|
|
1435
1402
|
try {
|
|
1436
1403
|
await unlink(path);
|
|
1437
1404
|
}
|
|
@@ -1439,18 +1406,138 @@ export class ChatController {
|
|
|
1439
1406
|
// best-effort cleanup
|
|
1440
1407
|
}
|
|
1441
1408
|
}
|
|
1409
|
+
async speakLastAssistantMessage() {
|
|
1410
|
+
if (this.speakTask) {
|
|
1411
|
+
this.view.addSystemMessage("speech playback already in progress", "warn");
|
|
1412
|
+
return;
|
|
1413
|
+
}
|
|
1414
|
+
if (this.isStreaming) {
|
|
1415
|
+
this.view.addSystemMessage("wait for the assistant to finish before speaking", "warn");
|
|
1416
|
+
return;
|
|
1417
|
+
}
|
|
1418
|
+
if (this.deps.env.platform() !== "darwin") {
|
|
1419
|
+
this.view.addSystemMessage("/speak is currently supported only on macOS.", "warn");
|
|
1420
|
+
return;
|
|
1421
|
+
}
|
|
1422
|
+
const lastAssistant = this.getLastAssistantMessage();
|
|
1423
|
+
if (!lastAssistant) {
|
|
1424
|
+
this.view.addSystemMessage("no assistant message to speak yet.", "warn");
|
|
1425
|
+
return;
|
|
1426
|
+
}
|
|
1427
|
+
const sourceText = extractAssistantText(lastAssistant).trim();
|
|
1428
|
+
if (!sourceText) {
|
|
1429
|
+
this.view.addSystemMessage("last assistant message was empty.", "warn");
|
|
1430
|
+
return;
|
|
1431
|
+
}
|
|
1432
|
+
const apiKey = await this.credentialResolver.getApiKey("google");
|
|
1433
|
+
if (!apiKey) {
|
|
1434
|
+
this.view.addSystemMessage("set GEMINI_API_KEY or apiKeys.google to use /speak", "error");
|
|
1435
|
+
return;
|
|
1436
|
+
}
|
|
1437
|
+
this.isStreaming = true;
|
|
1438
|
+
this.speechStatusHint = "rewriting for speech...";
|
|
1439
|
+
this.refreshStatus();
|
|
1440
|
+
this.view.startWorkingIcon();
|
|
1441
|
+
const abortController = new AbortController();
|
|
1442
|
+
const completion = this.runSpeakTask({
|
|
1443
|
+
apiKey,
|
|
1444
|
+
sourceText,
|
|
1445
|
+
signal: abortController.signal,
|
|
1446
|
+
});
|
|
1447
|
+
const task = { abortController, completion };
|
|
1448
|
+
this.speakTask = task;
|
|
1449
|
+
try {
|
|
1450
|
+
await completion;
|
|
1451
|
+
}
|
|
1452
|
+
catch (err) {
|
|
1453
|
+
if (abortController.signal.aborted) {
|
|
1454
|
+
return;
|
|
1455
|
+
}
|
|
1456
|
+
this.view.addSystemMessage(`speech synthesis failed: ${err.message}`, "error");
|
|
1457
|
+
}
|
|
1458
|
+
finally {
|
|
1459
|
+
const wasAborted = abortController.signal.aborted;
|
|
1460
|
+
if (this.speakTask === task) {
|
|
1461
|
+
this.speakTask = undefined;
|
|
1462
|
+
}
|
|
1463
|
+
this.speechStatusHint = undefined;
|
|
1464
|
+
this.view.stopWorkingIcon();
|
|
1465
|
+
this.isStreaming = false;
|
|
1466
|
+
this.refreshStatus();
|
|
1467
|
+
this.view.requestRender();
|
|
1468
|
+
if (wasAborted) {
|
|
1469
|
+
this.dequeueQueuedUserMessagesIntoEditor();
|
|
1470
|
+
}
|
|
1471
|
+
else {
|
|
1472
|
+
void this.drainQueuedUserMessages();
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
async runSpeakTask(args) {
|
|
1477
|
+
let audioPath;
|
|
1478
|
+
try {
|
|
1479
|
+
const result = await generateGeminiSpeech({
|
|
1480
|
+
apiKey: args.apiKey,
|
|
1481
|
+
sourceText: args.sourceText,
|
|
1482
|
+
signal: args.signal,
|
|
1483
|
+
onStageChange: () => { },
|
|
1484
|
+
onChunkProgress: ({ ready, total }) => {
|
|
1485
|
+
this.speechStatusHint = this.formatSpeechChunkProgressMessage(ready, total);
|
|
1486
|
+
this.refreshStatus();
|
|
1487
|
+
},
|
|
1488
|
+
});
|
|
1489
|
+
if (args.signal.aborted) {
|
|
1490
|
+
return;
|
|
1491
|
+
}
|
|
1492
|
+
audioPath = await this.createTempFilePath(SPEAK_TEMP_FILE_TEMPLATE);
|
|
1493
|
+
await writeFile(audioPath, result.audio);
|
|
1494
|
+
this.speechStatusHint = "playing speech...";
|
|
1495
|
+
this.refreshStatus();
|
|
1496
|
+
const playback = await this.deps.spawn("afplay", ["-r", String(SPEAK_PLAYBACK_RATE), audioPath], {
|
|
1497
|
+
detached: true,
|
|
1498
|
+
killProcessGroup: true,
|
|
1499
|
+
signal: args.signal,
|
|
1500
|
+
stdio: ["ignore", "ignore", "ignore"],
|
|
1501
|
+
});
|
|
1502
|
+
if (args.signal.aborted || playback.aborted) {
|
|
1503
|
+
return;
|
|
1504
|
+
}
|
|
1505
|
+
if (playback.exitCode !== 0) {
|
|
1506
|
+
const detail = playback.exitCode !== null
|
|
1507
|
+
? `afplay exited with code ${playback.exitCode}`
|
|
1508
|
+
: playback.closeSignal
|
|
1509
|
+
? `afplay terminated by signal ${playback.closeSignal}`
|
|
1510
|
+
: "afplay exited";
|
|
1511
|
+
throw new Error(detail);
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
catch (err) {
|
|
1515
|
+
if (args.signal.aborted) {
|
|
1516
|
+
return;
|
|
1517
|
+
}
|
|
1518
|
+
const error = err;
|
|
1519
|
+
if (error.name === "AbortError") {
|
|
1520
|
+
return;
|
|
1521
|
+
}
|
|
1522
|
+
if (error.code === "ENOENT") {
|
|
1523
|
+
throw new Error("afplay not found.");
|
|
1524
|
+
}
|
|
1525
|
+
throw err;
|
|
1526
|
+
}
|
|
1527
|
+
finally {
|
|
1528
|
+
if (audioPath) {
|
|
1529
|
+
await this.cleanupTempFile(audioPath);
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
formatSpeechChunkProgressMessage(ready, total) {
|
|
1534
|
+
return `generating speech chunks (${ready} out of ${total} ready)...`;
|
|
1535
|
+
}
|
|
1442
1536
|
getMemoryModeFilePath() {
|
|
1443
1537
|
const cwd = this.deps.env.cwd();
|
|
1444
1538
|
const home = this.deps.env.home();
|
|
1445
1539
|
const nearestAgentsFiles = findAgentsFilesFromCwdToHome(cwd, home);
|
|
1446
1540
|
const hostPath = nearestAgentsFiles.length > 0 ? nearestAgentsFiles[0] : resolve(join(cwd, "AGENTS.md"));
|
|
1447
|
-
if (this.sandboxEnabled) {
|
|
1448
|
-
return resolveSandboxPath({
|
|
1449
|
-
hostPath,
|
|
1450
|
-
cwd,
|
|
1451
|
-
sandboxConfig: this.config.sandbox,
|
|
1452
|
-
});
|
|
1453
|
-
}
|
|
1454
1541
|
return hostPath;
|
|
1455
1542
|
}
|
|
1456
1543
|
formatMemoryModeUserMessage(agentsFilePath, request) {
|
|
@@ -1629,9 +1716,10 @@ export class ChatController {
|
|
|
1629
1716
|
}
|
|
1630
1717
|
isDiffReviewIdle() {
|
|
1631
1718
|
return (!this.isStreaming &&
|
|
1632
|
-
!this.
|
|
1633
|
-
!this.
|
|
1634
|
-
!this.
|
|
1719
|
+
!this.listenRecording &&
|
|
1720
|
+
!this.listenTransition &&
|
|
1721
|
+
!this.isTranscribingListen &&
|
|
1722
|
+
!this.speakTask &&
|
|
1635
1723
|
!this.diffReviewService.isActive());
|
|
1636
1724
|
}
|
|
1637
1725
|
resolveDiffToolConfig() {
|
|
@@ -1742,12 +1830,9 @@ export class ChatController {
|
|
|
1742
1830
|
syncRuntimePromptContext() {
|
|
1743
1831
|
this.runtime.updatePromptContext({
|
|
1744
1832
|
cwd: this.agentCwd,
|
|
1745
|
-
hostCwd: this.deps.env.cwd(),
|
|
1746
1833
|
home: this.deps.env.home(),
|
|
1747
1834
|
includeAgentContext: this.includeAgentContext,
|
|
1748
1835
|
projectContextBlock: this.projectContextBlock,
|
|
1749
|
-
sandboxEnabled: this.sandboxEnabled,
|
|
1750
|
-
sandboxEnvironmentInfo: this.config.sandbox?.environmentInfo,
|
|
1751
1836
|
});
|
|
1752
1837
|
}
|
|
1753
1838
|
rebuildSystemPrompt(skillsBlock) {
|
|
@@ -1933,12 +2018,7 @@ export class ChatController {
|
|
|
1933
2018
|
this.view.addSystemMessage(`unknown bash command '${id}'.`, "error");
|
|
1934
2019
|
return;
|
|
1935
2020
|
}
|
|
1936
|
-
const
|
|
1937
|
-
const cwd = resolveAgentCwd({
|
|
1938
|
-
cwd: baseCwd,
|
|
1939
|
-
sandboxEnabled: this.sandboxEnabled,
|
|
1940
|
-
sandboxConfig: this.config.sandbox,
|
|
1941
|
-
});
|
|
2021
|
+
const cwd = saved.cwd ?? this.deps.env.cwd();
|
|
1942
2022
|
await this.runBashCommand(saved.cmd, { cwd });
|
|
1943
2023
|
}
|
|
1944
2024
|
async refreshReloadableContent(scope) {
|