@oh-my-pi/pi-coding-agent 17.1.0 → 17.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/CHANGELOG.md +24 -0
  2. package/dist/cli.js +3686 -4013
  3. package/dist/types/config/settings-schema.d.ts +58 -0
  4. package/dist/types/extensibility/legacy-pi-coding-agent-shim.d.ts +3 -0
  5. package/dist/types/live/attestation.d.ts +2 -0
  6. package/dist/types/live/controller.d.ts +10 -2
  7. package/dist/types/live/protocol.d.ts +1 -1
  8. package/dist/types/live/transport.d.ts +6 -19
  9. package/dist/types/live/visualizer.d.ts +8 -11
  10. package/dist/types/modes/components/assistant-message.d.ts +1 -0
  11. package/dist/types/modes/components/custom-message.d.ts +1 -1
  12. package/dist/types/modes/components/message-frame.d.ts +8 -4
  13. package/dist/types/modes/components/session-account-selector.d.ts +11 -0
  14. package/dist/types/modes/controllers/selector-controller.d.ts +1 -0
  15. package/dist/types/modes/interactive-mode.d.ts +1 -0
  16. package/dist/types/modes/types.d.ts +1 -0
  17. package/dist/types/session/agent-session-types.d.ts +8 -1
  18. package/dist/types/session/agent-session.d.ts +20 -1
  19. package/dist/types/session/auth-storage.d.ts +1 -1
  20. package/dist/types/session/eval-runner.d.ts +2 -0
  21. package/dist/types/session/messages.d.ts +2 -0
  22. package/dist/types/session/session-tools.d.ts +15 -0
  23. package/dist/types/session/streaming-output.d.ts +8 -0
  24. package/dist/types/slash-commands/helpers/session-pin.d.ts +9 -0
  25. package/dist/types/stt/index.d.ts +0 -2
  26. package/dist/types/stt/stt-controller.d.ts +7 -0
  27. package/dist/types/tiny/title-client.d.ts +10 -0
  28. package/dist/types/tools/builtin-names.d.ts +1 -1
  29. package/dist/types/tools/computer/protocol.d.ts +43 -0
  30. package/dist/types/tools/computer/supervisor.d.ts +32 -0
  31. package/dist/types/tools/computer/worker-entry.d.ts +1 -0
  32. package/dist/types/tools/computer/worker.d.ts +15 -0
  33. package/dist/types/tools/computer-renderer.d.ts +22 -0
  34. package/dist/types/tools/computer.d.ts +71 -0
  35. package/dist/types/tools/context.d.ts +2 -0
  36. package/dist/types/tools/default-renderer.d.ts +21 -0
  37. package/dist/types/tools/index.d.ts +2 -0
  38. package/dist/types/tts/streaming-player.d.ts +10 -43
  39. package/dist/types/utils/tools-manager.d.ts +1 -2
  40. package/package.json +12 -12
  41. package/src/cli/args.ts +1 -0
  42. package/src/cli/setup-cli.ts +2 -14
  43. package/src/cli.ts +8 -0
  44. package/src/config/model-registry.ts +6 -0
  45. package/src/config/settings-schema.ts +61 -0
  46. package/src/eval/executor-base.ts +1 -0
  47. package/src/eval/js/executor.ts +2 -0
  48. package/src/exec/bash-executor.ts +1 -0
  49. package/src/extensibility/extensions/wrapper.ts +68 -12
  50. package/src/extensibility/legacy-pi-coding-agent-shim.ts +7 -1
  51. package/src/live/attestation.ts +91 -0
  52. package/src/live/controller.ts +76 -23
  53. package/src/live/protocol.test.ts +3 -3
  54. package/src/live/protocol.ts +1 -1
  55. package/src/live/transport.ts +72 -140
  56. package/src/live/visualizer.ts +114 -134
  57. package/src/modes/components/assistant-message.ts +7 -2
  58. package/src/modes/components/custom-message.ts +4 -1
  59. package/src/modes/components/message-frame.ts +14 -8
  60. package/src/modes/components/session-account-selector.ts +62 -0
  61. package/src/modes/components/tool-execution.ts +17 -110
  62. package/src/modes/controllers/input-controller.ts +47 -39
  63. package/src/modes/controllers/live-command-controller.ts +82 -5
  64. package/src/modes/controllers/selector-controller.ts +62 -0
  65. package/src/modes/interactive-mode.ts +19 -0
  66. package/src/modes/types.ts +1 -0
  67. package/src/prompts/system/computer-safety.md +14 -0
  68. package/src/prompts/tools/computer.md +26 -0
  69. package/src/sdk.ts +5 -0
  70. package/src/session/agent-session-types.ts +9 -0
  71. package/src/session/agent-session.ts +56 -0
  72. package/src/session/auth-storage.ts +1 -0
  73. package/src/session/eval-runner.ts +5 -0
  74. package/src/session/messages.ts +3 -0
  75. package/src/session/session-tools.ts +37 -0
  76. package/src/session/streaming-output.ts +52 -5
  77. package/src/slash-commands/builtin-registry.ts +165 -9
  78. package/src/slash-commands/helpers/session-pin.ts +44 -0
  79. package/src/stt/downloader.ts +0 -2
  80. package/src/stt/index.ts +0 -2
  81. package/src/stt/stt-controller.ts +57 -146
  82. package/src/system-prompt.ts +4 -0
  83. package/src/tiny/title-client.ts +22 -0
  84. package/src/tools/bash-interactive.ts +90 -86
  85. package/src/tools/builtin-names.ts +1 -0
  86. package/src/tools/computer/protocol.ts +28 -0
  87. package/src/tools/computer/supervisor.ts +258 -0
  88. package/src/tools/computer/worker-entry.ts +25 -0
  89. package/src/tools/computer/worker.ts +135 -0
  90. package/src/tools/computer-renderer.ts +108 -0
  91. package/src/tools/computer.ts +433 -0
  92. package/src/tools/context.ts +2 -0
  93. package/src/tools/default-renderer.ts +139 -0
  94. package/src/tools/essential-tools.ts +1 -0
  95. package/src/tools/index.ts +5 -0
  96. package/src/tools/renderers.ts +2 -0
  97. package/src/tools/xdev.ts +54 -26
  98. package/src/tts/streaming-player.ts +81 -340
  99. package/src/utils/clipboard.ts +1 -30
  100. package/src/utils/mac-file-urls.applescript +37 -0
  101. package/src/utils/tool-choice.ts +14 -0
  102. package/src/utils/tools-manager.ts +1 -19
  103. package/dist/types/stt/recorder.d.ts +0 -30
  104. package/dist/types/stt/transcriber.d.ts +0 -14
  105. package/dist/types/stt/wav.d.ts +0 -29
  106. package/dist/types/tts/player.d.ts +0 -32
  107. package/src/live/audio-worklet.txt +0 -59
  108. package/src/live/browser-runtime.txt +0 -221
  109. package/src/stt/recorder.ts +0 -551
  110. package/src/stt/transcriber.ts +0 -60
  111. package/src/stt/wav.ts +0 -173
  112. package/src/tts/player.ts +0 -137
package/src/tools/xdev.ts CHANGED
@@ -30,10 +30,11 @@ import { parseStreamingJson } from "@oh-my-pi/pi-utils";
30
30
  import type { RenderResultOptions } from "../extensibility/custom-tools/types";
31
31
  import { XD_URL_PREFIX } from "../internal-urls/xd-protocol";
32
32
  import type { Theme } from "../modes/theme/theme";
33
+ import { renderDefaultToolExecution } from "./default-renderer";
33
34
  import type { Tool } from "./index";
34
35
  import { replaceTabs } from "./render-utils";
35
36
  import type { ToolRenderer } from "./renderers";
36
- import { ToolError } from "./tool-errors";
37
+ import { renderError, ToolAbortError, ToolError } from "./tool-errors";
37
38
 
38
39
  /**
39
40
  * Discoverable built-ins that must stay top-level even when xdev mounting is
@@ -389,28 +390,45 @@ export class XdevRegistry {
389
390
  onUpdate?: AgentToolUpdateCallback,
390
391
  context?: AgentToolContext,
391
392
  ): Promise<{ result: AgentToolResult<unknown>; xdev: XdevDispatch }> {
392
- const inst = this.#resolve(name);
393
+ let xdev: XdevDispatch = { tool: name, mode: "execute" };
394
+ try {
395
+ const inst = this.#resolve(name);
396
+
397
+ if (HELP_CONTENT_RE.test(content)) {
398
+ return {
399
+ result: { content: [{ type: "text", text: renderDocs(inst) }] },
400
+ xdev: { tool: name, mode: "help" },
401
+ };
402
+ }
393
403
 
394
- if (HELP_CONTENT_RE.test(content)) {
404
+ const validated = parseDeviceArgs(inst as AiTool, content, toolCallId, () => renderDocs(inst));
405
+ xdev = { ...xdev, args: validated };
406
+ const innerOnUpdate: AgentToolUpdateCallback | undefined = onUpdate
407
+ ? partial =>
408
+ onUpdate({
409
+ content: partial.content,
410
+ details: { xdev: { ...xdev, inner: partial.details } },
411
+ isError: partial.isError,
412
+ })
413
+ : undefined;
414
+ const result = await inst.execute(toolCallId, validated as never, signal, innerOnUpdate, context);
415
+ return { result, xdev: { ...xdev, inner: result.details } };
416
+ } catch (error) {
417
+ if (
418
+ error instanceof ToolAbortError ||
419
+ signal?.aborted ||
420
+ (error instanceof Error && error.name === "AbortError")
421
+ ) {
422
+ throw error;
423
+ }
395
424
  return {
396
- result: { content: [{ type: "text", text: renderDocs(inst) }] },
397
- xdev: { tool: name, mode: "help" },
425
+ result: {
426
+ content: [{ type: "text", text: renderError(error) }],
427
+ isError: true,
428
+ },
429
+ xdev,
398
430
  };
399
431
  }
400
-
401
- const validated = parseDeviceArgs(inst as AiTool, content, toolCallId, () => renderDocs(inst));
402
-
403
- const xdevBase: XdevDispatch = { tool: name, mode: "execute", args: validated };
404
- const innerOnUpdate: AgentToolUpdateCallback | undefined = onUpdate
405
- ? partial =>
406
- onUpdate({
407
- content: partial.content,
408
- details: { xdev: { ...xdevBase, inner: partial.details } },
409
- isError: partial.isError,
410
- })
411
- : undefined;
412
- const result = await inst.execute(toolCallId, validated as never, signal, innerOnUpdate, context);
413
- return { result, xdev: { ...xdevBase, inner: result.details } };
414
432
  }
415
433
  }
416
434
 
@@ -423,9 +441,8 @@ export class XdevRegistry {
423
441
  * map keyed by name. */
424
442
  function resolveDeviceRenderer(
425
443
  name: string,
426
- resolveMounted: ((name: string) => Tool | undefined) | undefined,
444
+ mounted: Tool | undefined,
427
445
  ): Pick<ToolRenderer, "renderCall" | "renderResult" | "mergeCallAndResult"> | undefined {
428
- const mounted = resolveMounted?.(name);
429
446
  if (mounted && (mounted.renderCall || mounted.renderResult)) {
430
447
  // A mounted AgentTool exposes the same renderCall/renderResult/mergeCallAndResult
431
448
  // surface as a static ToolRenderer; only the parameter generics differ, so unify
@@ -447,11 +464,13 @@ export function renderXdevCall(
447
464
  theme: Theme,
448
465
  resolveMounted?: (name: string) => Tool | undefined,
449
466
  ): Component | undefined {
450
- const renderer = resolveDeviceRenderer(name, resolveMounted);
467
+ const mounted = resolveMounted?.(name);
468
+ const renderer = resolveDeviceRenderer(name, mounted);
469
+ const args = decodeInnerArgs(content);
451
470
  if (renderer?.renderCall) {
452
- return renderer.renderCall(decodeInnerArgs(content), options, theme);
471
+ return renderer.renderCall(args, options, theme);
453
472
  }
454
- return new Text(theme.fg("toolTitle", theme.bold(`${XD_URL_PREFIX}${name}`)), 0, 0);
473
+ return renderDefaultToolExecution({ label: mounted?.label ?? name, args, options }, theme);
455
474
  }
456
475
 
457
476
  /** Forward an `xd://` dispatch result to the mounted tool's renderer. */
@@ -469,7 +488,8 @@ export function renderXdevResult(
469
488
  if (dispatch.mode === "help") {
470
489
  return text ? new Text(theme.fg("toolOutput", replaceTabs(text)), 0, 0) : undefined;
471
490
  }
472
- const renderer = resolveDeviceRenderer(dispatch.tool, resolveMounted);
491
+ const mounted = resolveMounted?.(dispatch.tool);
492
+ const renderer = resolveDeviceRenderer(dispatch.tool, mounted);
473
493
  const innerResult = { content: result.content, details: dispatch.inner, isError: result.isError };
474
494
  if (renderer?.renderResult) {
475
495
  const parts: Component[] = [];
@@ -488,5 +508,13 @@ export function renderXdevResult(
488
508
  return box;
489
509
  }
490
510
  }
491
- return text ? new Text(theme.fg("toolOutput", replaceTabs(text)), 0, 0) : undefined;
511
+ return renderDefaultToolExecution(
512
+ {
513
+ label: mounted?.label ?? dispatch.tool,
514
+ args: dispatch.args ?? {},
515
+ result: { output: text, isError: result.isError },
516
+ options,
517
+ },
518
+ theme,
519
+ );
492
520
  }
@@ -1,379 +1,120 @@
1
- /**
2
- * Gapless streaming audio output for assistant speech.
3
- *
4
- * Replaces the spawn-`afplay`-per-sentence approach (a fresh process per chunk
5
- * meant audible gaps, per-spawn latency, and no way to interrupt a clip mid-play)
6
- * with a single persistent player process fed raw 32-bit-float mono PCM over
7
- * stdin. Chunks are queued and drained by one writer so segments play back to
8
- * back; writes are paced to stay only {@link LEAD_SECONDS} ahead of realtime so
9
- * ducking and stop take effect promptly instead of after seconds of buffered
10
- * audio. {@link StreamingAudioPlayer.stop} kills the process for instant silence.
11
- *
12
- * Where no streaming backend exists (Windows, or a host without ffmpeg/sox), it
13
- * degrades to the per-file {@link playAudioFile} path so speech still works —
14
- * just without gapless playback or mid-clip interruption. A backend that spawns
15
- * but dies early (e.g. an ffmpeg built without its platform audio device) is
16
- * detected via its exit and the session downgrades to per-file playback without
17
- * dropping the chunk being played.
18
- */
19
- import * as fs from "node:fs/promises";
20
- import * as os from "node:os";
21
- import * as path from "node:path";
22
- import { $which, logger, Snowflake } from "@oh-my-pi/pi-utils";
23
- import type { FileSink, Subprocess } from "bun";
24
- import { getToolPath } from "../utils/tools-manager";
25
- import { type PlayerCommand, playAudioFile } from "./player";
26
- import { encodeWav } from "./wav";
1
+ import { AudioPlayback } from "@oh-my-pi/pi-natives";
27
2
 
28
- /** Kokoro emits 24 kHz mono; used when a chunk does not declare a rate. */
3
+ /** Kokoro emits 24 kHz mono PCM when a chunk does not declare a rate. */
29
4
  const DEFAULT_SAMPLE_RATE = 24_000;
30
- /** Cap how far ahead of realtime we buffer into the player so duck/stop are responsive. */
31
- const LEAD_SECONDS = 0.6;
32
- /** Output gain applied while ducked (the user is speaking over the assistant). */
33
- export const DUCK_GAIN = 0.25;
34
- /**
35
- * Cap on streamed PCM retained for the nonzero-exit replay. Past this the
36
- * buffer is dropped: the failure being recovered is a short clip that fits the
37
- * OS pipe buffer before a broken backend dies, while a backend that consumed
38
- * minutes of realtime-paced audio was playing it — replaying a whole long
39
- * utterance would duplicate audio, and unbounded retention (~5.8 MB/min at
40
- * 24 kHz mono f32) would defeat streaming for long input.
41
- */
42
- const REPLAY_RETENTION_SECONDS = 60;
43
-
44
- /** Injection seam for {@link streamingPlayerCommandsFor} — defaults to real PATH/tools lookups. */
45
- export interface StreamingPlayerLookup {
46
- which?: (bin: string) => string | null;
47
- ffmpeg?: () => string | null;
48
- }
49
-
50
- /**
51
- * Ordered candidate commands for a persistent raw-PCM player on `platform`: each
52
- * reads 32-bit-float little-endian mono PCM at `sampleRate` from stdin (`pipe:0`)
53
- * and plays it to the default output device. An empty list means no streaming
54
- * backend is available and the caller should fall back to per-file playback.
55
- *
56
- * - darwin: `ffmpeg` (AudioToolbox output device) → sox's `play` (coreaudio).
57
- * - linux/other POSIX: `ffmpeg` (`-f pulse` then `-f alsa`) → `paplay`/`aplay`
58
- * raw fallbacks.
59
- * - win32: none (PowerShell `SoundPlayer` is file-only).
60
- */
61
- export function streamingPlayerCommandsFor(
62
- platform: NodeJS.Platform,
63
- sampleRate: number,
64
- lookup: StreamingPlayerLookup = {},
65
- ): PlayerCommand[] {
66
- const which = lookup.which ?? $which;
67
- const ffmpeg = lookup.ffmpeg ?? ((): string | null => getToolPath("ffmpeg"));
68
- const rate = String(sampleRate > 0 ? sampleRate : DEFAULT_SAMPLE_RATE);
69
- const input = ["-loglevel", "error", "-nostdin", "-f", "f32le", "-ar", rate, "-ac", "1", "-i", "pipe:0"];
70
-
71
- if (platform === "darwin") {
72
- const commands: PlayerCommand[] = [];
73
- const ffmpegBin = ffmpeg();
74
- if (ffmpegBin) commands.push({ cmd: ffmpegBin, args: [...input, "-f", "audiotoolbox", "default"] });
75
- const play = which("play");
76
- if (play) {
77
- commands.push({
78
- cmd: play,
79
- args: ["-q", "-t", "raw", "-e", "floating-point", "-b", "32", "-r", rate, "-c", "1", "-"],
80
- });
81
- }
82
- return commands;
83
- }
84
- if (platform === "win32") {
85
- return [];
86
- }
87
5
 
88
- const commands: PlayerCommand[] = [];
89
- const ffmpegBin = ffmpeg();
90
- if (ffmpegBin) {
91
- commands.push({ cmd: ffmpegBin, args: [...input, "-f", "pulse", "default"] });
92
- commands.push({ cmd: ffmpegBin, args: [...input, "-f", "alsa", "default"] });
93
- }
94
- const paplay = which("paplay");
95
- if (paplay) commands.push({ cmd: paplay, args: ["--raw", `--rate=${rate}`, "--format=float32le", "--channels=1"] });
96
- const aplay = which("aplay");
97
- if (aplay) commands.push({ cmd: aplay, args: ["-q", "-f", "FLOAT_LE", "-r", rate, "-c", "1", "-"] });
98
- return commands;
99
- }
6
+ /** Output gain applied while the user speaks over assistant audio. */
7
+ export const DUCK_GAIN = 0.25;
100
8
 
101
- /**
102
- * Test seams for {@link StreamingAudioPlayer}: override backend discovery and
103
- * the per-file fallback so playback logic can be exercised without a real audio
104
- * device. Both default to the platform lookup and {@link playAudioFile}.
105
- */
106
- export interface StreamingPlayerOptions {
107
- /** Ordered backend commands for a sample rate; defaults to {@link streamingPlayerCommandsFor}. */
108
- commandsFor?: (sampleRate: number) => PlayerCommand[];
109
- /** Per-file fallback playback; defaults to {@link playAudioFile}. */
110
- playAudio?: (wavPath: string, signal: AbortSignal) => Promise<void>;
111
- /** Max seconds of streamed PCM retained for the nonzero-exit replay; defaults to {@link REPLAY_RETENTION_SECONDS}. */
112
- replayRetentionSeconds?: number;
9
+ function errorFrom(cause: unknown): Error {
10
+ return cause instanceof Error ? cause : new Error(String(cause));
113
11
  }
114
12
 
115
13
  /**
116
- * Single-session gapless player. Lifecycle: {@link start} once, {@link write}
117
- * chunks in order, then {@link end} to drain or {@link stop} to abort. Not
118
- * reusable after stop/end — create a new instance per utterance.
14
+ * One native gapless playback session. Call {@link start}, enqueue mono `f32`
15
+ * chunks with {@link write}, then {@link end} to drain or {@link stop} to abort.
119
16
  */
120
17
  export class StreamingAudioPlayer {
121
- #queue: Float32Array[] = [];
18
+ #native: AudioPlayback | null = null;
122
19
  #sampleRate = DEFAULT_SAMPLE_RATE;
123
20
  #gain = 1;
124
- #mode: "stream" | "file" = "file";
125
- #proc: Subprocess<"pipe", "ignore", "ignore"> | null = null;
126
- #sink: FileSink | null = null;
127
- /** Streaming backends not yet tried; consumed head-first by {@link #spawnStream}. */
128
- #candidates: PlayerCommand[] | null = null;
129
- #writtenSec = 0;
130
- #startedAt = 0;
131
- #started = false;
21
+ #error: Error | null = null;
22
+ #ending: Promise<void> | null = null;
132
23
  #inputClosed = false;
133
24
  #stopped = false;
134
- #abortController = new AbortController();
135
- #wake: (() => void) | null = null;
136
- #drain: Promise<void> = Promise.resolve();
137
- readonly #commandsFor: (sampleRate: number) => PlayerCommand[];
138
- readonly #playAudio: (wavPath: string, signal: AbortSignal) => Promise<void>;
139
- readonly #replayRetentionSec: number;
140
- /** Streamed PCM retained for this utterance so a failed backend can be replayed via file playback. */
141
- #played: Float32Array[] = [];
142
- #playedSec = 0;
143
-
144
- constructor(options: StreamingPlayerOptions = {}) {
145
- this.#commandsFor = options.commandsFor ?? (rate => streamingPlayerCommandsFor(process.platform, rate));
146
- this.#playAudio = options.playAudio ?? ((wavPath, signal) => playAudioFile(wavPath, { signal }));
147
- this.#replayRetentionSec = options.replayRetentionSeconds ?? REPLAY_RETENTION_SECONDS;
25
+ #failNative(native: AudioPlayback, cause: unknown): void {
26
+ this.#error = errorFrom(cause);
27
+ if (this.#native === native) this.#native = null;
28
+ try {
29
+ native.stop();
30
+ } catch {
31
+ // Preserve the original playback failure.
32
+ }
148
33
  }
149
34
 
150
- /** Pick a backend and begin draining. Idempotent; the first call's rate wins. */
151
- start(sampleRate: number): void {
152
- if (this.#started || this.#stopped) return;
153
- this.#started = true;
35
+ /** Opens the default speaker at the stream's logical sample rate. */
36
+ start(sampleRate = DEFAULT_SAMPLE_RATE): void {
37
+ if (this.#native || this.#error || this.#inputClosed || this.#stopped) return;
154
38
  this.#sampleRate = sampleRate > 0 ? sampleRate : DEFAULT_SAMPLE_RATE;
155
- this.#mode = this.#spawnStream() ? "stream" : "file";
156
- this.#startedAt = performance.now();
157
- this.#drain = this.#drainLoop();
39
+ let native: AudioPlayback | undefined;
40
+ try {
41
+ native = new AudioPlayback(this.#sampleRate);
42
+ native.setGain(this.#gain);
43
+ this.#native = native;
44
+ } catch (cause) {
45
+ if (native) this.#failNative(native, cause);
46
+ else this.#error = errorFrom(cause);
47
+ }
158
48
  }
159
49
 
160
- /** Queue a mono float32 PCM chunk for playback in arrival order. */
50
+ /** Queues one mono `f32` PCM chunk without copying it in TypeScript. */
161
51
  write(pcm: Float32Array): void {
162
- if (this.#stopped) return;
163
- this.#queue.push(pcm);
164
- this.#signal();
165
- }
166
-
167
- /** Scale subsequent output (1 = normal, <1 = ducked). Applies within {@link LEAD_SECONDS}. */
168
- setGain(gain: number): void {
169
- this.#gain = gain < 0 ? 0 : gain;
170
- }
171
-
172
- /** Close the input; resolves once all queued audio has finished playing. */
173
- async end(): Promise<void> {
174
- this.#inputClosed = true;
175
- this.#signal();
176
- await this.#drain;
177
- }
178
-
179
- /** Stop immediately: kill the player, drop everything still queued. */
180
- stop(): void {
181
- if (this.#stopped) return;
182
- this.#stopped = true;
183
- this.#queue.length = 0;
184
- this.#played.length = 0;
185
- this.#abortController.abort();
186
- this.#signal();
52
+ if (this.#inputClosed || this.#stopped || pcm.length === 0) return;
53
+ if (!this.#native && !this.#error) this.start(this.#sampleRate);
54
+ const native = this.#native;
55
+ if (!native) return;
187
56
  try {
188
- // end() flushes asynchronously; the SIGKILL below races it, so a broken
189
- // pipe here is expected — swallow the rejection (it otherwise surfaces
190
- // as an unhandled EPIPE right as speech ends).
191
- void Promise.resolve(this.#sink?.end()).catch(() => {});
192
- } catch {}
193
- try {
194
- this.#proc?.kill("SIGKILL");
195
- } catch {}
196
- }
197
-
198
- /**
199
- * Spawn the next untried streaming backend; false once the list is
200
- * exhausted. A backend that spawns but dies early (e.g. an ffmpeg built
201
- * without this platform's audio output device) would otherwise swallow PCM
202
- * into a dead pipe, so its exit advances to the next candidate — or to
203
- * per-file playback — and #writeStream's failure path replays the
204
- * in-flight chunk.
205
- */
206
- #spawnStream(): boolean {
207
- this.#candidates ??= this.#commandsFor(this.#sampleRate);
208
- for (let command = this.#candidates.shift(); command; command = this.#candidates.shift()) {
209
- const { cmd, args } = command;
210
- try {
211
- const proc = Bun.spawn([cmd, ...args], {
212
- stdin: "pipe",
213
- stdout: "ignore",
214
- stderr: "ignore",
215
- });
216
- this.#proc = proc;
217
- this.#sink = proc.stdin;
218
- void proc.exited.then(code => {
219
- if (this.#proc !== proc || this.#stopped || this.#inputClosed) return;
220
- logger.debug("tts: streaming backend exited early; trying next backend", { cmd, code });
221
- this.#proc = null;
222
- this.#sink = null;
223
- this.#mode = this.#spawnStream() ? "stream" : "file";
224
- });
225
- return true;
226
- } catch (error) {
227
- logger.debug("tts: streaming player spawn failed", {
228
- cmd,
229
- error: error instanceof Error ? error.message : String(error),
230
- });
231
- }
57
+ native.write(pcm);
58
+ } catch (cause) {
59
+ this.#failNative(native, cause);
232
60
  }
233
- return false;
234
61
  }
235
62
 
236
- #signal(): void {
237
- const wake = this.#wake;
238
- this.#wake = null;
239
- wake?.();
240
- }
241
-
242
- async #drainLoop(): Promise<void> {
63
+ /** Applies gain at render time, including to samples already queued natively. */
64
+ setGain(gain: number): void {
65
+ if (!Number.isFinite(gain) || gain < 0) throw new RangeError("Audio gain must be a finite non-negative number");
66
+ this.#gain = gain;
67
+ const native = this.#native;
68
+ if (!native) return;
243
69
  try {
244
- while (!this.#stopped) {
245
- const chunk = this.#queue.shift();
246
- if (!chunk) {
247
- if (this.#inputClosed) break;
248
- await this.#waitForWork();
249
- continue;
250
- }
251
- if (this.#mode === "stream") {
252
- if (this.#playedSec <= this.#replayRetentionSec) {
253
- this.#played.push(chunk);
254
- this.#playedSec += chunk.length / this.#sampleRate;
255
- // Over the cap: drop retention for the rest of the utterance.
256
- if (this.#playedSec > this.#replayRetentionSec) this.#played.length = 0;
257
- }
258
- // Pace writes so the player buffers ~LEAD_SECONDS, no more, keeping
259
- // ducking and stop responsive instead of locked behind buffered audio.
260
- const ahead = this.#writtenSec - (performance.now() - this.#startedAt) / 1000;
261
- if (ahead > LEAD_SECONDS) {
262
- await Bun.sleep((ahead - LEAD_SECONDS) * 1000);
263
- if (this.#stopped) return;
264
- }
265
- if (await this.#writeStream(chunk)) {
266
- this.#writtenSec += chunk.length / this.#sampleRate;
267
- continue;
268
- }
269
- // Backend died mid-write: move to the next streaming candidate
270
- // (or the file path) and replay this exact chunk so nothing is
271
- // dropped.
272
- this.#mode = this.#spawnStream() ? "stream" : "file";
273
- if (this.#mode === "stream" && (await this.#writeStream(chunk))) {
274
- this.#writtenSec += chunk.length / this.#sampleRate;
275
- } else {
276
- this.#mode = "file";
277
- await this.#playFile(chunk);
278
- }
279
- } else {
280
- await this.#playFile(chunk);
281
- }
282
- }
283
- if (!this.#stopped && this.#mode === "stream") {
284
- try {
285
- await this.#sink?.end();
286
- } catch {}
287
- const proc = this.#proc;
288
- let exitCode: number | null = null;
289
- if (proc) {
290
- try {
291
- exitCode = await proc.exited;
292
- } catch {}
293
- }
294
- // A streaming backend that exits nonzero never opened its audio
295
- // device (e.g. the bundled ffmpeg built without pulse/alsa output).
296
- // For a short single-segment clip the pipe write succeeds before
297
- // that death and #inputClosed is already set, so neither the
298
- // broken-pipe replay nor the early-exit handler advances backends.
299
- // Replay the buffered utterance through per-file playback so it
300
- // still reaches the speakers.
301
- if (!this.#stopped && proc && exitCode !== 0) {
302
- this.#mode = "file";
303
- for (const chunk of this.#played) {
304
- if (this.#stopped) break;
305
- await this.#playFile(chunk);
306
- }
307
- }
308
- }
309
- } catch (error) {
310
- logger.debug("tts: streaming player drain failed", {
311
- error: error instanceof Error ? error.message : String(error),
312
- });
70
+ native.setGain(gain);
71
+ } catch (cause) {
72
+ this.#failNative(native, cause);
313
73
  }
314
74
  }
315
75
 
316
- /** Block until a chunk is queued, the input closes, or stop is called. */
317
- #waitForWork(): Promise<void> {
318
- const { promise, resolve } = Promise.withResolvers<void>();
319
- this.#wake = resolve;
320
- // Re-check after arming to close the gap between the empty shift and here.
321
- if (this.#queue.length > 0 || this.#inputClosed || this.#stopped) {
322
- this.#wake = null;
323
- resolve();
76
+ /** Closes input and resolves after every queued sample reaches the speaker. */
77
+ end(): Promise<void> {
78
+ if (this.#ending) return this.#ending;
79
+ if (this.#stopped) return Promise.resolve();
80
+ this.#inputClosed = true;
81
+ if (this.#error) {
82
+ this.#stopped = true;
83
+ return Promise.reject(this.#error);
324
84
  }
325
- return promise;
326
- }
327
-
328
- /**
329
- * Write one chunk into the backend's stdin and await the flush — a broken
330
- * pipe rejects here (not as an unhandled rejection later), so the caller
331
- * can replay the exact chunk on the next backend.
332
- */
333
- async #writeStream(pcm: Float32Array): Promise<boolean> {
334
- const sink = this.#sink;
335
- if (!sink) return false;
336
- try {
337
- sink.write(this.#bytes(pcm));
338
- await sink.flush();
339
- return true;
340
- } catch (error) {
341
- logger.debug("tts: streaming write failed", {
342
- error: error instanceof Error ? error.message : String(error),
343
- });
344
- return false;
85
+ const native = this.#native;
86
+ if (!native) {
87
+ this.#stopped = true;
88
+ return Promise.resolve();
345
89
  }
90
+ this.#ending = native
91
+ .end()
92
+ .catch(cause => {
93
+ throw errorFrom(cause);
94
+ })
95
+ .finally(() => {
96
+ if (this.#native === native) this.#native = null;
97
+ this.#stopped = true;
98
+ });
99
+ return this.#ending;
346
100
  }
347
101
 
348
- async #playFile(pcm: Float32Array): Promise<void> {
349
- const wavPath = path.join(os.tmpdir(), `omp-speech-${Snowflake.next()}.wav`);
102
+ /** Stops immediately and discards queued audio. Safe to call repeatedly. */
103
+ stop(): void {
104
+ this.#inputClosed = true;
105
+ this.#stopped = true;
106
+ const native = this.#native;
107
+ this.#native = null;
108
+ if (!native) return;
350
109
  try {
351
- await fs.writeFile(wavPath, encodeWav(this.#scaled(pcm), this.#sampleRate));
352
- if (!this.#stopped) await this.#playAudio(wavPath, this.#abortController.signal);
353
- } catch (error) {
354
- logger.debug("tts: file playback failed", {
355
- error: error instanceof Error ? error.message : String(error),
356
- });
357
- } finally {
358
- await fs.unlink(wavPath).catch(() => {});
110
+ native.stop();
111
+ } catch {
112
+ // Best-effort abort during session teardown.
359
113
  }
360
114
  }
361
-
362
- /** Raw f32le bytes for the stream sink, applying gain only when ducked (avoids a copy at unity). */
363
- #bytes(pcm: Float32Array): Uint8Array {
364
- if (this.#gain === 1) return new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
365
- return new Uint8Array(this.#scaled(pcm).buffer);
366
- }
367
-
368
- #scaled(pcm: Float32Array): Float32Array {
369
- if (this.#gain === 1) return pcm;
370
- const out = new Float32Array(pcm.length);
371
- for (let i = 0; i < pcm.length; i++) out[i] = (pcm[i] ?? 0) * this.#gain;
372
- return out;
373
- }
374
115
  }
375
116
 
376
- /** Factory the vocalizer calls; a function so tests can stub it without spawning a player. */
117
+ /** Creates the single-use player used by the speech vocalizer. */
377
118
  export function createStreamingPlayer(): StreamingAudioPlayer {
378
119
  return new StreamingAudioPlayer();
379
120
  }
@@ -1,6 +1,7 @@
1
1
  import type { ClipboardImage } from "@oh-my-pi/pi-natives";
2
2
  import * as native from "@oh-my-pi/pi-natives";
3
3
  import { logger } from "@oh-my-pi/pi-utils";
4
+ import MAC_FILE_URL_SCRIPT from "./mac-file-urls.applescript" with { type: "text" };
4
5
 
5
6
  /**
6
7
  * Run a subprocess and capture its stdout without blocking the event loop.
@@ -52,36 +53,6 @@ function isWsl(): boolean {
52
53
  return process.platform === "linux" && Boolean(process.env.WSL_DISTRO_NAME || process.env.WSL_INTEROP);
53
54
  }
54
55
 
55
- // AppleScript that returns the POSIX paths of every file URL currently on the
56
- // macOS pasteboard, one path per line. `pbpaste(1)` only surfaces plain text,
57
- // EPS, or RTF, so a Finder `Cmd+C` (which puts only a `public.file-url`
58
- // representation on the pasteboard) makes `pbpaste` empty. AppleScript's
59
- // `«class furl»` coercion reaches the file-URL representation directly and
60
- // works for both single-file and multi-file selections. The `try` blocks
61
- // suppress the `-1700` "can't make … into type" error AppleScript raises when
62
- // the clipboard holds no file URLs, so the script's exit status only reflects
63
- // `osascript` itself.
64
- const MAC_FILE_URL_SCRIPT = [
65
- "on run",
66
- '\tset output to ""',
67
- "\ttry",
68
- "\t\tset theClip to the clipboard as «class furl»",
69
- "\t\tif class of theClip is list then",
70
- "\t\t\trepeat with anItem in theClip",
71
- "\t\t\t\ttry",
72
- "\t\t\t\t\tset output to output & POSIX path of anItem & linefeed",
73
- "\t\t\t\tend try",
74
- "\t\t\tend repeat",
75
- "\t\telse",
76
- "\t\t\ttry",
77
- "\t\t\t\tset output to POSIX path of theClip & linefeed",
78
- "\t\t\tend try",
79
- "\t\tend if",
80
- "\tend try",
81
- "\treturn output",
82
- "end run",
83
- ].join("\n");
84
-
85
56
  /**
86
57
  * Read file paths from the macOS pasteboard's `public.file-url` representation.
87
58
  *
@@ -0,0 +1,37 @@
1
+ -- Returns the POSIX paths of every file URL currently on the macOS
2
+ -- pasteboard, one path per line. `pbpaste(1)` only surfaces plain text,
3
+ -- EPS, or RTF, so a Finder Cmd+C (which puts only a `public.file-url`
4
+ -- representation on the pasteboard) makes `pbpaste` empty. The
5
+ -- `«class furl»` coercion reaches the file-URL representation directly and
6
+ -- works for both single-file and multi-file selections.
7
+ --
8
+ -- The coercion is guarded by `clipboard info for «class furl»`: AppleScript
9
+ -- happily coerces plain *text* into a file URL by treating it as an HFS
10
+ -- path, so an unguarded cast turns a copied `https://i.can.ac/x.png` string
11
+ -- into the mangled `/https/::i.can.ac:x.png` (HFS `:`↔`/` swap) and the
12
+ -- paste dead-ends with "Image not found" instead of falling back to pasting
13
+ -- the text. `clipboard info for` inspects the actual pasteboard types, so
14
+ -- it is `{}` unless a real `public.file-url` representation is present.
15
+ --
16
+ -- The `try` blocks suppress the `-1700` "can't make … into type" error
17
+ -- AppleScript raises when the clipboard holds no file URLs, so the script's
18
+ -- exit status only reflects `osascript` itself.
19
+ on run
20
+ set output to ""
21
+ try
22
+ if (clipboard info for «class furl») is {} then return output
23
+ set theClip to the clipboard as «class furl»
24
+ if class of theClip is list then
25
+ repeat with anItem in theClip
26
+ try
27
+ set output to output & POSIX path of anItem & linefeed
28
+ end try
29
+ end repeat
30
+ else
31
+ try
32
+ set output to POSIX path of theClip & linefeed
33
+ end try
34
+ end if
35
+ end try
36
+ return output
37
+ end run