@markusylisiurunen/tau 0.2.102 → 0.2.104

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +26 -39
  2. package/dist/core/async/server_config.js +0 -1
  3. package/dist/core/async/server_config.js.map +1 -1
  4. package/dist/core/async/session_manager.js +0 -3
  5. package/dist/core/async/session_manager.js.map +1 -1
  6. package/dist/core/async/telegram.js +1 -1
  7. package/dist/core/async/telegram.js.map +1 -1
  8. package/dist/core/cli.js +5 -9
  9. package/dist/core/cli.js.map +1 -1
  10. package/dist/core/commands/registry.js +17 -2
  11. package/dist/core/commands/registry.js.map +1 -1
  12. package/dist/core/config/index.js.map +1 -1
  13. package/dist/core/config/schema.js +21 -61
  14. package/dist/core/config/schema.js.map +1 -1
  15. package/dist/core/debug.js +4 -12
  16. package/dist/core/debug.js.map +1 -1
  17. package/dist/core/diff_review/review_thread.js +4 -7
  18. package/dist/core/diff_review/review_thread.js.map +1 -1
  19. package/dist/core/index.js +2 -1
  20. package/dist/core/index.js.map +1 -1
  21. package/dist/core/personas.js +44 -1
  22. package/dist/core/personas.js.map +1 -1
  23. package/dist/core/runtime/chat_runtime.js +0 -10
  24. package/dist/core/runtime/chat_runtime.js.map +1 -1
  25. package/dist/core/runtime/runtime_bootstrap.js +5 -67
  26. package/dist/core/runtime/runtime_bootstrap.js.map +1 -1
  27. package/dist/core/runtime/session_prompt_composer.js +2 -7
  28. package/dist/core/runtime/session_prompt_composer.js.map +1 -1
  29. package/dist/core/session/core_session.js.map +1 -1
  30. package/dist/core/session/session_engine.js +1 -13
  31. package/dist/core/session/session_engine.js.map +1 -1
  32. package/dist/core/static/prompts/diff-review-wrapper.md +4 -3
  33. package/dist/core/tool/cli.js +37 -0
  34. package/dist/core/tool/cli.js.map +1 -0
  35. package/dist/core/tool/errors.js +9 -0
  36. package/dist/core/tool/errors.js.map +1 -0
  37. package/dist/core/tool/index.js +3 -0
  38. package/dist/core/tool/index.js.map +1 -0
  39. package/dist/core/tool/pdf_unpack.js +406 -0
  40. package/dist/core/tool/pdf_unpack.js.map +1 -0
  41. package/dist/core/tools/bash.js +1 -1
  42. package/dist/core/tools/bash.js.map +1 -1
  43. package/dist/core/tools/edit.js +2 -2
  44. package/dist/core/tools/edit.js.map +1 -1
  45. package/dist/core/tools/emit_output.js +1 -1
  46. package/dist/core/tools/emit_output.js.map +1 -1
  47. package/dist/core/tools/execution_backend.js +21 -251
  48. package/dist/core/tools/execution_backend.js.map +1 -1
  49. package/dist/core/tools/grep.js +1 -1
  50. package/dist/core/tools/grep.js.map +1 -1
  51. package/dist/core/tools/list.js +1 -1
  52. package/dist/core/tools/list.js.map +1 -1
  53. package/dist/core/tools/read.js +1 -1
  54. package/dist/core/tools/read.js.map +1 -1
  55. package/dist/core/tools/registry.js.map +1 -1
  56. package/dist/core/tools/send_input_to_agent.js +1 -1
  57. package/dist/core/tools/send_input_to_agent.js.map +1 -1
  58. package/dist/core/tools/spawn_agent.js +7 -39
  59. package/dist/core/tools/spawn_agent.js.map +1 -1
  60. package/dist/core/tools/terminate_agent.js +1 -1
  61. package/dist/core/tools/terminate_agent.js.map +1 -1
  62. package/dist/core/tools/view_image.js +1 -1
  63. package/dist/core/tools/view_image.js.map +1 -1
  64. package/dist/core/tools/wait_for_agent.js +1 -1
  65. package/dist/core/tools/wait_for_agent.js.map +1 -1
  66. package/dist/core/tools/web_fetch.js +1 -1
  67. package/dist/core/tools/web_fetch.js.map +1 -1
  68. package/dist/core/tools/web_search.js +1 -1
  69. package/dist/core/tools/web_search.js.map +1 -1
  70. package/dist/core/tools/write.js +1 -1
  71. package/dist/core/tools/write.js.map +1 -1
  72. package/dist/core/utils/context.js +1 -1
  73. package/dist/core/utils/context.js.map +1 -1
  74. package/dist/core/utils/context_builder.js +1 -11
  75. package/dist/core/utils/context_builder.js.map +1 -1
  76. package/dist/core/utils/gemini_speech.js +367 -0
  77. package/dist/core/utils/gemini_speech.js.map +1 -0
  78. package/dist/core/utils/git.js +2 -7
  79. package/dist/core/utils/git.js.map +1 -1
  80. package/dist/core/utils/mistral_document_ocr.js +186 -0
  81. package/dist/core/utils/mistral_document_ocr.js.map +1 -0
  82. package/dist/core/version.js +1 -1
  83. package/dist/diff_tool/app/dist/assets/{index-AomQ5Wco.js → index-D1e3Qk8w.js} +26 -26
  84. package/dist/diff_tool/app/dist/index.html +1 -1
  85. package/dist/main.js +91 -112
  86. package/dist/main.js.map +1 -1
  87. package/dist/sdk/client.js +0 -1
  88. package/dist/sdk/client.js.map +1 -1
  89. package/dist/sdk/types.d.ts +0 -1
  90. package/dist/tui/app.js.map +1 -1
  91. package/dist/tui/chat_controller.js +231 -151
  92. package/dist/tui/chat_controller.js.map +1 -1
  93. package/dist/tui/chat_view.js +0 -1
  94. package/dist/tui/chat_view.js.map +1 -1
  95. package/dist/tui/terminal.js +8 -0
  96. package/dist/tui/terminal.js.map +1 -1
  97. package/dist/tui/ui/footer.js +6 -7
  98. package/dist/tui/ui/footer.js.map +1 -1
  99. package/package.json +5 -5
  100. package/dist/core/tools/sandbox/docker_sandbox.js +0 -183
  101. package/dist/core/tools/sandbox/docker_sandbox.js.map +0 -1
  102. package/dist/core/utils/agent_environment.js +0 -48
  103. package/dist/core/utils/agent_environment.js.map +0 -1
  104. package/dist/core/utils/sandbox_paths.js +0 -29
  105. package/dist/core/utils/sandbox_paths.js.map +0 -1
  106. package/dist/core/utils/sandbox_prompt_paths.js +0 -86
  107. package/dist/core/utils/sandbox_prompt_paths.js.map +0 -1
@@ -1,8 +1,8 @@
1
1
  import { randomUUID } from "node:crypto";
2
- import { realpathSync, statSync } from "node:fs";
2
+ import { statSync } from "node:fs";
3
3
  import { mkdtemp, readFile, unlink, writeFile } from "node:fs/promises";
4
4
  import { tmpdir } from "node:os";
5
- import { dirname, join, relative, resolve, sep } from "node:path";
5
+ import { dirname, join, resolve } from "node:path";
6
6
  import { z } from "zod";
7
7
  import { formatCodexAuthError } from "../core/auth/auth_messages.js";
8
8
  import { getAuthPath } from "../core/auth/auth_paths.js";
@@ -19,11 +19,10 @@ import { buildBashUiText, formatBashUserMessageText, getBashOutputPolicy, prepar
19
19
  import { ToolCatalog } from "../core/tools/catalog.js";
20
20
  import { createLocalToolExecutionBackend } from "../core/tools/execution_backend.js";
21
21
  import { REASONING_LEVELS, } from "../core/types.js";
22
- import { resolveAgentCwd, resolveSandboxPath } from "../core/utils/agent_environment.js";
23
22
  import { findAgentsFilesFromCwdToHome } from "../core/utils/agents_files.js";
24
23
  import { formatCwdChangeNotice, formatProjectContextChangeNotice, formatRiskLevelChangeNotice, } from "../core/utils/context.js";
25
24
  import { formatAdaptiveNumber, formatCwd, formatPathForDisplay, formatTokenWindow, } from "../core/utils/format.js";
26
- import { getGitRoot } from "../core/utils/git.js";
25
+ import { generateGeminiSpeech } from "../core/utils/gemini_speech.js";
27
26
  import { extractAllFencedCodeBlocks, extractAssistantText } from "../core/utils/messages.js";
28
27
  import { transcribeMistralAudio } from "../core/utils/mistral_transcription.js";
29
28
  import { streamModel } from "../core/utils/model_stream.js";
@@ -55,9 +54,11 @@ const RELOAD_PLANS = {
55
54
  },
56
55
  };
57
56
  const ALLOWED_RISK_LEVELS = ["read-only", "read-write"];
57
+ const LISTEN_TEMP_FILE_TEMPLATE = "/tmp/tau-listen.XXXXXX";
58
58
  const SPEAK_TEMP_FILE_TEMPLATE = "/tmp/tau-speak.XXXXXX";
59
- const SPEAK_RECORDING_MIN_BYTES = 1024;
60
- const SPEAK_RECORDING_MAX_DURATION_MS = 5 * 60 * 1000;
59
+ const LISTEN_RECORDING_MIN_BYTES = 1024;
60
+ const LISTEN_RECORDING_MAX_DURATION_MS = 5 * 60 * 1000;
61
+ const SPEAK_PLAYBACK_RATE = 1.4;
61
62
  const CAFFEINATE_COMMAND = "/usr/bin/caffeinate";
62
63
  const SmartPruneResponseSchema = z.object({
63
64
  prune: z.array(z.string().trim().min(1)),
@@ -76,9 +77,7 @@ export class ChatController {
76
77
  activeThemeId;
77
78
  credentialResolver;
78
79
  authPath;
79
- sandboxEnabled;
80
80
  caffeinated;
81
- sandboxRootReal;
82
81
  agentCwd;
83
82
  includeAgentContext;
84
83
  runtime;
@@ -86,7 +85,6 @@ export class ChatController {
86
85
  commandRegistry;
87
86
  commandHandlers;
88
87
  toolBackend;
89
- toolBackendDispose;
90
88
  deps;
91
89
  eventUnsubscribe;
92
90
  isStreaming = false;
@@ -100,6 +98,7 @@ export class ChatController {
100
98
  showThinking = false;
101
99
  compactToolUi = true;
102
100
  commandHint;
101
+ speechStatusHint;
103
102
  riskLevel = "read-only";
104
103
  projectContextBlock;
105
104
  projectFiles = [];
@@ -118,9 +117,10 @@ export class ChatController {
118
117
  lastTurnDurationMs = 0;
119
118
  turnTimer;
120
119
  lastEmptySubmitAt;
121
- speakRecording;
122
- isTranscribingSpeak = false;
123
- speakTransition;
120
+ listenRecording;
121
+ isTranscribingListen = false;
122
+ listenTransition;
123
+ speakTask;
124
124
  turnCaffeinate;
125
125
  disableCaffeinateForSession = false;
126
126
  constructor(options) {
@@ -139,9 +139,7 @@ export class ChatController {
139
139
  this.initialUserMessage = options.initialUserMessage;
140
140
  this.config = options.config ?? {};
141
141
  this.defaultDiffTool = options.defaultDiffTool;
142
- this.sandboxEnabled = options.sandboxEnabled;
143
142
  this.caffeinated = options.caffeinated ?? false;
144
- this.sandboxRootReal = this.sandboxEnabled ? this.resolveSandboxRoot(cwd) : undefined;
145
143
  this.activeThemeId = this.config.defaultTheme;
146
144
  this.authPath = getAuthPath(this.deps.env.home());
147
145
  const authStorage = new AuthStorage(this.authPath);
@@ -153,7 +151,6 @@ export class ChatController {
153
151
  const queuedUserMessages = options.queuedUserMessages ?? [];
154
152
  this.queuedMessageBuffer = new QueuedUserMessages(queuedUserMessages);
155
153
  this.interruptLifecycle = new InterruptLifecycle();
156
- this.toolBackendDispose = options.toolBackendDispose;
157
154
  this.includeAgentContext = !options.noAgentContextFiles;
158
155
  const initialPersona = (options.initialPersonaId &&
159
156
  this.personas.find((p) => p.id.toLowerCase() === options.initialPersonaId.toLowerCase())) ||
@@ -175,10 +172,6 @@ export class ChatController {
175
172
  cwd,
176
173
  home,
177
174
  includeAgentContext: this.includeAgentContext,
178
- sandboxEnabled: this.sandboxEnabled,
179
- sandboxConfig: this.config.sandbox,
180
- sandboxHostRoot: this.sandboxRootReal,
181
- sandboxEnvironmentInfo: this.config.sandbox?.environmentInfo,
182
175
  readFile: this.deps.fs.readFile,
183
176
  });
184
177
  this.agentCwd = startupBootstrap.promptContext.cwd;
@@ -194,9 +187,6 @@ export class ChatController {
194
187
  spawn: this.deps.spawn,
195
188
  env: this.deps.env,
196
189
  });
197
- if (this.sandboxEnabled && this.toolBackend.kind !== "sandbox") {
198
- throw new Error("sandbox enabled but tool backend is not sandboxed.");
199
- }
200
190
  const toolRegistry = ToolCatalog.createRegistry(this.toolBackend);
201
191
  this.runtime = ChatRuntime.create({
202
192
  persona: this.currentPersona,
@@ -249,7 +239,8 @@ export class ChatController {
249
239
  pruneLargest: (extra) => this.pruneToolResults("largest", extra),
250
240
  pruneSmart: (extra) => this.pruneToolResultsSmart(extra),
251
241
  reload: () => this.reloadContent(),
252
- speak: () => this.toggleSpeakCapture(),
242
+ listen: () => this.startListenCaptureFromCommand(),
243
+ speak: () => this.speakLastAssistantMessage(),
253
244
  risk: (level) => this.setRiskLevel(level),
254
245
  persona: (id) => this.switchPersona(id),
255
246
  prompt: (id) => this.insertPrompt(id),
@@ -296,7 +287,7 @@ export class ChatController {
296
287
  onCtrlR: () => this.cycleRiskLevel(),
297
288
  onCtrlP: () => this.cyclePersonality(),
298
289
  onCtrlS: () => void this.stashEditorToClipboard(),
299
- onCtrlY: () => void this.toggleSpeakCapture(),
290
+ onCtrlY: () => void this.toggleListenCapture(),
300
291
  onEscape: () => this.onInterrupt(),
301
292
  onCtrlF: () => {
302
293
  this.expandFileMentions().catch((err) => {
@@ -318,23 +309,31 @@ export class ChatController {
318
309
  }
319
310
  async dispose() {
320
311
  this.eventUnsubscribe?.();
321
- if (this.speakTransition) {
322
- await this.speakTransition;
312
+ if (this.listenTransition) {
313
+ await this.listenTransition;
314
+ }
315
+ if (this.speakTask) {
316
+ this.speakTask.abortController.abort();
317
+ await this.speakTask.completion;
323
318
  }
324
319
  await this.cancelDiffReview();
325
- await this.cancelSpeakCapture();
320
+ await this.cancelListenCapture();
326
321
  await this.stopTurnCaffeinate();
327
- if (!this.toolBackendDispose)
328
- return;
329
- await this.toolBackendDispose();
330
322
  }
331
323
  // Mode Adapter ---------------------------------------------------------------------------------
332
324
  async onUserInput(text) {
333
325
  await this.handleSubmit(text);
334
326
  }
335
327
  onInterrupt() {
336
- if (this.speakRecording) {
337
- void this.runSpeakTransition(() => this.stopSpeakCapture());
328
+ if (this.listenRecording) {
329
+ void this.runListenTransition(() => this.stopListenCapture());
330
+ return;
331
+ }
332
+ if (this.speakTask) {
333
+ if (!this.speakTask.abortController.signal.aborted) {
334
+ this.speakTask.abortController.abort();
335
+ this.view.addSystemMessage("interrupted", "error");
336
+ }
338
337
  return;
339
338
  }
340
339
  this.interruptActiveTask();
@@ -491,7 +490,6 @@ export class ChatController {
491
490
  sessionCost,
492
491
  duration,
493
492
  riskLevel: this.riskLevel,
494
- sandboxed: this.sandboxEnabled,
495
493
  commandHint: this.getActiveCommandHint(),
496
494
  },
497
495
  editor: {
@@ -504,7 +502,7 @@ export class ChatController {
504
502
  });
505
503
  }
506
504
  getInputMode() {
507
- if (this.speakRecording)
505
+ if (this.listenRecording)
508
506
  return "recording";
509
507
  if (this.isBashIncognito)
510
508
  return "bash_incognito";
@@ -515,7 +513,7 @@ export class ChatController {
515
513
  return "normal";
516
514
  }
517
515
  getActiveCommandHint() {
518
- return this.diffReviewService.getCommandHint(this.commandHint);
516
+ return this.diffReviewService.getCommandHint(this.speechStatusHint ?? this.commandHint);
519
517
  }
520
518
  // Context & Cost Tracking -----------------------------------------------------------------------
521
519
  getContextUsageString() {
@@ -674,10 +672,6 @@ export class ChatController {
674
672
  const resolved = resolvePersonaSkillsForPromptContext({
675
673
  persona,
676
674
  discoveredSkills: this.skills,
677
- cwd: this.deps.env.cwd(),
678
- sandboxEnabled: this.sandboxEnabled,
679
- sandboxConfig: this.config.sandbox,
680
- sandboxHostRoot: this.sandboxRootReal,
681
675
  });
682
676
  return { skillsBlock: resolved.skillsBlock, unknown: resolved.unknown };
683
677
  }
@@ -724,10 +718,6 @@ export class ChatController {
724
718
  this.view.addSystemMessage(`not a directory: ${normalized}`, "error");
725
719
  return;
726
720
  }
727
- if (this.sandboxEnabled && !this.isPathWithinSandboxRoot(resolved)) {
728
- this.view.addSystemMessage(`directory is outside the sandbox mount: ${normalized}`, "error");
729
- return;
730
- }
731
721
  try {
732
722
  process.chdir(resolved);
733
723
  }
@@ -738,18 +728,7 @@ export class ChatController {
738
728
  const nextCwd = this.deps.env.cwd();
739
729
  const previousAgentCwd = this.agentCwd;
740
730
  const previousProjectContextBlock = this.projectContextBlock;
741
- this.agentCwd =
742
- this.sandboxEnabled && this.sandboxRootReal
743
- ? resolveSandboxPath({
744
- hostPath: nextCwd,
745
- cwd: this.sandboxRootReal,
746
- sandboxConfig: this.config.sandbox,
747
- })
748
- : resolveAgentCwd({
749
- cwd: nextCwd,
750
- sandboxEnabled: this.sandboxEnabled,
751
- sandboxConfig: this.config.sandbox,
752
- });
731
+ this.agentCwd = nextCwd;
753
732
  this.refreshProjectContext(nextCwd);
754
733
  this.updatePendingProjectContextChange(previousProjectContextBlock, this.projectContextBlock);
755
734
  this.projectFiles = [];
@@ -784,36 +763,11 @@ export class ChatController {
784
763
  return join(home, input.slice(2));
785
764
  return resolve(cwd, input);
786
765
  }
787
- resolveSandboxRoot(cwd) {
788
- try {
789
- const root = getGitRoot(cwd) ?? cwd;
790
- return realpathSync(root);
791
- }
792
- catch {
793
- return undefined;
794
- }
795
- }
796
- isPathWithinSandboxRoot(targetPath) {
797
- if (!this.sandboxRootReal)
798
- return true;
799
- let resolved = targetPath;
800
- try {
801
- resolved = realpathSync(targetPath);
802
- }
803
- catch {
804
- // fall back to provided path
805
- }
806
- const rel = relative(this.sandboxRootReal, resolved);
807
- return !(rel === ".." || rel.startsWith(`..${sep}`));
808
- }
809
766
  refreshProjectContext(cwd) {
810
767
  const projectContext = resolveProjectContextForPromptContext({
811
768
  cwd,
812
769
  home: this.deps.env.home(),
813
770
  includeAgentContext: this.includeAgentContext,
814
- sandboxEnabled: this.sandboxEnabled,
815
- sandboxConfig: this.config.sandbox,
816
- sandboxHostRoot: this.sandboxRootReal,
817
771
  readFile: this.deps.fs.readFile,
818
772
  });
819
773
  this.agentsFiles = projectContext.agentsFiles;
@@ -880,7 +834,7 @@ export class ChatController {
880
834
  }
881
835
  // Input Handling --------------------------------------------------------------------------------
882
836
  beforeSubmit(text) {
883
- if (this.speakRecording)
837
+ if (this.listenRecording)
884
838
  return false;
885
839
  if (!this.isStreaming)
886
840
  return true;
@@ -970,8 +924,10 @@ export class ChatController {
970
924
  return "prune smart-selected tool results and compact edit calls, optional fraction and guidance";
971
925
  case "reload":
972
926
  return "reload prompts, skills, themes, bash commands, and AGENTS.md";
927
+ case "listen":
928
+ return "start microphone recording and transcribe to editor (macOS only)";
973
929
  case "speak":
974
- return "toggle microphone recording and transcribe to editor (macOS only)";
930
+ return "speak the last assistant message aloud (macOS only)";
975
931
  case "risk":
976
932
  return "set risk level: /risk:read-only or /risk:read-write";
977
933
  case "bash":
@@ -1212,16 +1168,27 @@ export class ChatController {
1212
1168
  this.view.addMessage({ type: "user", text: trimmed }, historyEntryId);
1213
1169
  await this.runAssistantTurn();
1214
1170
  }
1215
- async toggleSpeakCapture() {
1216
- if (this.speakTransition) {
1171
+ async toggleListenCapture() {
1172
+ if (this.listenTransition) {
1173
+ this.view.addSystemMessage("speech recording state change already in progress", "warn");
1174
+ return;
1175
+ }
1176
+ if (this.listenRecording) {
1177
+ await this.runListenTransition(() => this.stopListenCapture());
1178
+ return;
1179
+ }
1180
+ await this.startListenCaptureFromCommand();
1181
+ }
1182
+ async startListenCaptureFromCommand() {
1183
+ if (this.listenTransition) {
1217
1184
  this.view.addSystemMessage("speech recording state change already in progress", "warn");
1218
1185
  return;
1219
1186
  }
1220
- if (this.speakRecording) {
1221
- await this.runSpeakTransition(() => this.stopSpeakCapture());
1187
+ if (this.listenRecording) {
1188
+ this.view.addSystemMessage("speech recording already in progress", "warn");
1222
1189
  return;
1223
1190
  }
1224
- if (this.isTranscribingSpeak) {
1191
+ if (this.isTranscribingListen) {
1225
1192
  this.view.addSystemMessage("speech transcription already in progress", "warn");
1226
1193
  return;
1227
1194
  }
@@ -1229,36 +1196,36 @@ export class ChatController {
1229
1196
  this.view.addSystemMessage("wait for the assistant to finish before recording", "warn");
1230
1197
  return;
1231
1198
  }
1232
- await this.runSpeakTransition(() => this.startSpeakCapture());
1199
+ await this.runListenTransition(() => this.startListenCapture());
1233
1200
  }
1234
- async runSpeakTransition(task) {
1235
- if (this.speakTransition) {
1201
+ async runListenTransition(task) {
1202
+ if (this.listenTransition) {
1236
1203
  return;
1237
1204
  }
1238
1205
  const transition = task();
1239
- this.speakTransition = transition;
1206
+ this.listenTransition = transition;
1240
1207
  try {
1241
1208
  await transition;
1242
1209
  }
1243
1210
  finally {
1244
- if (this.speakTransition === transition) {
1245
- this.speakTransition = undefined;
1211
+ if (this.listenTransition === transition) {
1212
+ this.listenTransition = undefined;
1246
1213
  }
1247
1214
  }
1248
1215
  }
1249
- async startSpeakCapture() {
1216
+ async startListenCapture() {
1250
1217
  if (this.deps.env.platform() !== "darwin") {
1251
- this.view.addSystemMessage("/speak is currently supported only on macOS.", "warn");
1218
+ this.view.addSystemMessage("/listen is currently supported only on macOS.", "warn");
1252
1219
  return;
1253
1220
  }
1254
1221
  const apiKey = getMistralApiKey(this.config, this.deps.env.env());
1255
1222
  if (!apiKey) {
1256
- this.view.addSystemMessage("set MISTRAL_API_KEY or apiKeys.mistral to use /speak", "error");
1223
+ this.view.addSystemMessage("set MISTRAL_API_KEY or apiKeys.mistral to use /listen", "error");
1257
1224
  return;
1258
1225
  }
1259
1226
  let audioPath;
1260
1227
  try {
1261
- audioPath = await this.createSpeakTempFilePath();
1228
+ audioPath = await this.createTempFilePath(LISTEN_TEMP_FILE_TEMPLATE);
1262
1229
  const abortController = new AbortController();
1263
1230
  const completion = this.deps.spawn("ffmpeg", [
1264
1231
  "-hide_banner",
@@ -1292,29 +1259,29 @@ export class ChatController {
1292
1259
  completion,
1293
1260
  };
1294
1261
  recording.maxDurationTimeout = setTimeout(() => {
1295
- if (this.speakRecording !== recording || this.speakTransition)
1262
+ if (this.listenRecording !== recording || this.listenTransition)
1296
1263
  return;
1297
- void this.runSpeakTransition(() => this.stopSpeakCapture());
1298
- }, SPEAK_RECORDING_MAX_DURATION_MS);
1299
- this.speakRecording = recording;
1264
+ void this.runListenTransition(() => this.stopListenCapture());
1265
+ }, LISTEN_RECORDING_MAX_DURATION_MS);
1266
+ this.listenRecording = recording;
1300
1267
  this.view.setEditorInputEnabled(false);
1301
1268
  this.refreshStatus();
1302
- void this.watchSpeakRecording(recording);
1269
+ void this.watchListenRecording(recording);
1303
1270
  }
1304
1271
  catch (err) {
1305
1272
  if (audioPath) {
1306
- await this.cleanupSpeakTempFile(audioPath);
1273
+ await this.cleanupTempFile(audioPath);
1307
1274
  }
1308
1275
  this.view.addSystemMessage(`failed to start recording: ${err.message}`, "error");
1309
1276
  }
1310
1277
  }
1311
- async stopSpeakCapture() {
1312
- const recording = this.speakRecording;
1278
+ async stopListenCapture() {
1279
+ const recording = this.listenRecording;
1313
1280
  if (!recording)
1314
1281
  return;
1315
1282
  recording.stopRequested = true;
1316
- this.clearSpeakRecordingMaxDurationTimeout(recording);
1317
- this.speakRecording = undefined;
1283
+ this.clearListenRecordingMaxDurationTimeout(recording);
1284
+ this.listenRecording = undefined;
1318
1285
  this.view.setEditorInputEnabled(true);
1319
1286
  this.refreshStatus();
1320
1287
  recording.abortController.abort();
@@ -1323,17 +1290,17 @@ export class ChatController {
1323
1290
  }
1324
1291
  catch (err) {
1325
1292
  this.view.addSystemMessage(`recording failed: ${err.message}`, "error");
1326
- await this.cleanupSpeakTempFile(recording.audioPath);
1293
+ await this.cleanupTempFile(recording.audioPath);
1327
1294
  return;
1328
1295
  }
1329
- this.isTranscribingSpeak = true;
1296
+ this.isTranscribingListen = true;
1330
1297
  try {
1331
1298
  const audio = await readFile(recording.audioPath);
1332
- if (audio.byteLength < SPEAK_RECORDING_MIN_BYTES) {
1299
+ if (audio.byteLength < LISTEN_RECORDING_MIN_BYTES) {
1333
1300
  this.view.addSystemMessage("recording too short, try again", "warn");
1334
1301
  return;
1335
1302
  }
1336
- const transcript = await this.transcribeSpeakAudio(audio);
1303
+ const transcript = await this.transcribeListenAudio(audio);
1337
1304
  const text = transcript.trim();
1338
1305
  if (!text) {
1339
1306
  return;
@@ -1344,17 +1311,17 @@ export class ChatController {
1344
1311
  this.view.addSystemMessage(`speech transcription failed: ${err.message}`, "error");
1345
1312
  }
1346
1313
  finally {
1347
- this.isTranscribingSpeak = false;
1348
- await this.cleanupSpeakTempFile(recording.audioPath);
1314
+ this.isTranscribingListen = false;
1315
+ await this.cleanupTempFile(recording.audioPath);
1349
1316
  }
1350
1317
  }
1351
- async cancelSpeakCapture() {
1352
- const recording = this.speakRecording;
1318
+ async cancelListenCapture() {
1319
+ const recording = this.listenRecording;
1353
1320
  if (!recording)
1354
1321
  return;
1355
1322
  recording.stopRequested = true;
1356
- this.clearSpeakRecordingMaxDurationTimeout(recording);
1357
- this.speakRecording = undefined;
1323
+ this.clearListenRecordingMaxDurationTimeout(recording);
1324
+ this.listenRecording = undefined;
1358
1325
  this.view.setEditorInputEnabled(true);
1359
1326
  this.refreshStatus();
1360
1327
  recording.abortController.abort();
@@ -1364,15 +1331,15 @@ export class ChatController {
1364
1331
  catch {
1365
1332
  // ignore disposal errors
1366
1333
  }
1367
- await this.cleanupSpeakTempFile(recording.audioPath);
1334
+ await this.cleanupTempFile(recording.audioPath);
1368
1335
  }
1369
- async watchSpeakRecording(recording) {
1336
+ async watchListenRecording(recording) {
1370
1337
  try {
1371
1338
  const result = await recording.completion;
1372
- this.clearSpeakRecordingMaxDurationTimeout(recording);
1373
- if (this.speakRecording !== recording || recording.stopRequested)
1339
+ this.clearListenRecordingMaxDurationTimeout(recording);
1340
+ if (this.listenRecording !== recording || recording.stopRequested)
1374
1341
  return;
1375
- this.speakRecording = undefined;
1342
+ this.listenRecording = undefined;
1376
1343
  this.view.setEditorInputEnabled(true);
1377
1344
  this.refreshStatus();
1378
1345
  const detail = result.exitCode !== null
@@ -1381,13 +1348,13 @@ export class ChatController {
1381
1348
  ? `ffmpeg terminated by signal ${result.closeSignal}`
1382
1349
  : "ffmpeg exited";
1383
1350
  this.view.addSystemMessage(`recording stopped unexpectedly (${detail})`, "error");
1384
- await this.cleanupSpeakTempFile(recording.audioPath);
1351
+ await this.cleanupTempFile(recording.audioPath);
1385
1352
  }
1386
1353
  catch (err) {
1387
- this.clearSpeakRecordingMaxDurationTimeout(recording);
1388
- if (this.speakRecording !== recording || recording.stopRequested)
1354
+ this.clearListenRecordingMaxDurationTimeout(recording);
1355
+ if (this.listenRecording !== recording || recording.stopRequested)
1389
1356
  return;
1390
- this.speakRecording = undefined;
1357
+ this.listenRecording = undefined;
1391
1358
  this.view.setEditorInputEnabled(true);
1392
1359
  this.refreshStatus();
1393
1360
  const error = err;
@@ -1397,17 +1364,17 @@ export class ChatController {
1397
1364
  else {
1398
1365
  this.view.addSystemMessage(`recording failed: ${error.message}`, "error");
1399
1366
  }
1400
- await this.cleanupSpeakTempFile(recording.audioPath);
1367
+ await this.cleanupTempFile(recording.audioPath);
1401
1368
  }
1402
1369
  }
1403
- clearSpeakRecordingMaxDurationTimeout(recording) {
1370
+ clearListenRecordingMaxDurationTimeout(recording) {
1404
1371
  if (!recording.maxDurationTimeout)
1405
1372
  return;
1406
1373
  clearTimeout(recording.maxDurationTimeout);
1407
1374
  recording.maxDurationTimeout = undefined;
1408
1375
  }
1409
- async createSpeakTempFilePath() {
1410
- const result = await this.deps.spawn("mktemp", [SPEAK_TEMP_FILE_TEMPLATE]);
1376
+ async createTempFilePath(template) {
1377
+ const result = await this.deps.spawn("mktemp", [template]);
1411
1378
  if (result.exitCode !== 0) {
1412
1379
  const message = result.stderr.trim() || result.stdout.trim() || "mktemp failed";
1413
1380
  throw new Error(message);
@@ -1418,7 +1385,7 @@ export class ChatController {
1418
1385
  }
1419
1386
  return path;
1420
1387
  }
1421
- async transcribeSpeakAudio(audio) {
1388
+ async transcribeListenAudio(audio) {
1422
1389
  const apiKey = getMistralApiKey(this.config, this.deps.env.env());
1423
1390
  if (!apiKey) {
1424
1391
  throw new Error("missing MISTRAL_API_KEY or apiKeys.mistral");
@@ -1431,7 +1398,7 @@ export class ChatController {
1431
1398
  language: "en",
1432
1399
  });
1433
1400
  }
1434
- async cleanupSpeakTempFile(path) {
1401
+ async cleanupTempFile(path) {
1435
1402
  try {
1436
1403
  await unlink(path);
1437
1404
  }
@@ -1439,18 +1406,138 @@ export class ChatController {
1439
1406
  // best-effort cleanup
1440
1407
  }
1441
1408
  }
1409
+ async speakLastAssistantMessage() {
1410
+ if (this.speakTask) {
1411
+ this.view.addSystemMessage("speech playback already in progress", "warn");
1412
+ return;
1413
+ }
1414
+ if (this.isStreaming) {
1415
+ this.view.addSystemMessage("wait for the assistant to finish before speaking", "warn");
1416
+ return;
1417
+ }
1418
+ if (this.deps.env.platform() !== "darwin") {
1419
+ this.view.addSystemMessage("/speak is currently supported only on macOS.", "warn");
1420
+ return;
1421
+ }
1422
+ const lastAssistant = this.getLastAssistantMessage();
1423
+ if (!lastAssistant) {
1424
+ this.view.addSystemMessage("no assistant message to speak yet.", "warn");
1425
+ return;
1426
+ }
1427
+ const sourceText = extractAssistantText(lastAssistant).trim();
1428
+ if (!sourceText) {
1429
+ this.view.addSystemMessage("last assistant message was empty.", "warn");
1430
+ return;
1431
+ }
1432
+ const apiKey = await this.credentialResolver.getApiKey("google");
1433
+ if (!apiKey) {
1434
+ this.view.addSystemMessage("set GEMINI_API_KEY or apiKeys.google to use /speak", "error");
1435
+ return;
1436
+ }
1437
+ this.isStreaming = true;
1438
+ this.speechStatusHint = "rewriting for speech...";
1439
+ this.refreshStatus();
1440
+ this.view.startWorkingIcon();
1441
+ const abortController = new AbortController();
1442
+ const completion = this.runSpeakTask({
1443
+ apiKey,
1444
+ sourceText,
1445
+ signal: abortController.signal,
1446
+ });
1447
+ const task = { abortController, completion };
1448
+ this.speakTask = task;
1449
+ try {
1450
+ await completion;
1451
+ }
1452
+ catch (err) {
1453
+ if (abortController.signal.aborted) {
1454
+ return;
1455
+ }
1456
+ this.view.addSystemMessage(`speech synthesis failed: ${err.message}`, "error");
1457
+ }
1458
+ finally {
1459
+ const wasAborted = abortController.signal.aborted;
1460
+ if (this.speakTask === task) {
1461
+ this.speakTask = undefined;
1462
+ }
1463
+ this.speechStatusHint = undefined;
1464
+ this.view.stopWorkingIcon();
1465
+ this.isStreaming = false;
1466
+ this.refreshStatus();
1467
+ this.view.requestRender();
1468
+ if (wasAborted) {
1469
+ this.dequeueQueuedUserMessagesIntoEditor();
1470
+ }
1471
+ else {
1472
+ void this.drainQueuedUserMessages();
1473
+ }
1474
+ }
1475
+ }
1476
+ async runSpeakTask(args) {
1477
+ let audioPath;
1478
+ try {
1479
+ const result = await generateGeminiSpeech({
1480
+ apiKey: args.apiKey,
1481
+ sourceText: args.sourceText,
1482
+ signal: args.signal,
1483
+ onStageChange: () => { },
1484
+ onChunkProgress: ({ ready, total }) => {
1485
+ this.speechStatusHint = this.formatSpeechChunkProgressMessage(ready, total);
1486
+ this.refreshStatus();
1487
+ },
1488
+ });
1489
+ if (args.signal.aborted) {
1490
+ return;
1491
+ }
1492
+ audioPath = await this.createTempFilePath(SPEAK_TEMP_FILE_TEMPLATE);
1493
+ await writeFile(audioPath, result.audio);
1494
+ this.speechStatusHint = "playing speech...";
1495
+ this.refreshStatus();
1496
+ const playback = await this.deps.spawn("afplay", ["-r", String(SPEAK_PLAYBACK_RATE), audioPath], {
1497
+ detached: true,
1498
+ killProcessGroup: true,
1499
+ signal: args.signal,
1500
+ stdio: ["ignore", "ignore", "ignore"],
1501
+ });
1502
+ if (args.signal.aborted || playback.aborted) {
1503
+ return;
1504
+ }
1505
+ if (playback.exitCode !== 0) {
1506
+ const detail = playback.exitCode !== null
1507
+ ? `afplay exited with code ${playback.exitCode}`
1508
+ : playback.closeSignal
1509
+ ? `afplay terminated by signal ${playback.closeSignal}`
1510
+ : "afplay exited";
1511
+ throw new Error(detail);
1512
+ }
1513
+ }
1514
+ catch (err) {
1515
+ if (args.signal.aborted) {
1516
+ return;
1517
+ }
1518
+ const error = err;
1519
+ if (error.name === "AbortError") {
1520
+ return;
1521
+ }
1522
+ if (error.code === "ENOENT") {
1523
+ throw new Error("afplay not found.");
1524
+ }
1525
+ throw err;
1526
+ }
1527
+ finally {
1528
+ if (audioPath) {
1529
+ await this.cleanupTempFile(audioPath);
1530
+ }
1531
+ }
1532
+ }
1533
+ formatSpeechChunkProgressMessage(ready, total) {
1534
+ return `generating speech chunks (${ready} out of ${total} ready)...`;
1535
+ }
1442
1536
  getMemoryModeFilePath() {
1443
1537
  const cwd = this.deps.env.cwd();
1444
1538
  const home = this.deps.env.home();
1445
1539
  const nearestAgentsFiles = findAgentsFilesFromCwdToHome(cwd, home);
1446
1540
  const hostPath = nearestAgentsFiles.length > 0 ? nearestAgentsFiles[0] : resolve(join(cwd, "AGENTS.md"));
1447
- if (this.sandboxEnabled) {
1448
- return resolveSandboxPath({
1449
- hostPath,
1450
- cwd,
1451
- sandboxConfig: this.config.sandbox,
1452
- });
1453
- }
1454
1541
  return hostPath;
1455
1542
  }
1456
1543
  formatMemoryModeUserMessage(agentsFilePath, request) {
@@ -1629,9 +1716,10 @@ export class ChatController {
1629
1716
  }
1630
1717
  isDiffReviewIdle() {
1631
1718
  return (!this.isStreaming &&
1632
- !this.speakRecording &&
1633
- !this.speakTransition &&
1634
- !this.isTranscribingSpeak &&
1719
+ !this.listenRecording &&
1720
+ !this.listenTransition &&
1721
+ !this.isTranscribingListen &&
1722
+ !this.speakTask &&
1635
1723
  !this.diffReviewService.isActive());
1636
1724
  }
1637
1725
  resolveDiffToolConfig() {
@@ -1742,12 +1830,9 @@ export class ChatController {
1742
1830
  syncRuntimePromptContext() {
1743
1831
  this.runtime.updatePromptContext({
1744
1832
  cwd: this.agentCwd,
1745
- hostCwd: this.deps.env.cwd(),
1746
1833
  home: this.deps.env.home(),
1747
1834
  includeAgentContext: this.includeAgentContext,
1748
1835
  projectContextBlock: this.projectContextBlock,
1749
- sandboxEnabled: this.sandboxEnabled,
1750
- sandboxEnvironmentInfo: this.config.sandbox?.environmentInfo,
1751
1836
  });
1752
1837
  }
1753
1838
  rebuildSystemPrompt(skillsBlock) {
@@ -1933,12 +2018,7 @@ export class ChatController {
1933
2018
  this.view.addSystemMessage(`unknown bash command '${id}'.`, "error");
1934
2019
  return;
1935
2020
  }
1936
- const baseCwd = saved.cwd ?? this.deps.env.cwd();
1937
- const cwd = resolveAgentCwd({
1938
- cwd: baseCwd,
1939
- sandboxEnabled: this.sandboxEnabled,
1940
- sandboxConfig: this.config.sandbox,
1941
- });
2021
+ const cwd = saved.cwd ?? this.deps.env.cwd();
1942
2022
  await this.runBashCommand(saved.cmd, { cwd });
1943
2023
  }
1944
2024
  async refreshReloadableContent(scope) {