@oh-my-pi/pi-coding-agent 17.1.0 → 17.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/CHANGELOG.md +55 -0
  2. package/dist/cli.js +6325 -4420
  3. package/dist/types/cli/bench-cli.d.ts +1 -0
  4. package/dist/types/config/model-resolver.d.ts +6 -3
  5. package/dist/types/config/settings-schema.d.ts +62 -0
  6. package/dist/types/extensibility/legacy-pi-coding-agent-shim.d.ts +3 -0
  7. package/dist/types/launch/broker-list-order.test.d.ts +1 -0
  8. package/dist/types/live/attestation.d.ts +2 -0
  9. package/dist/types/live/controller.d.ts +10 -2
  10. package/dist/types/live/protocol.d.ts +1 -1
  11. package/dist/types/live/transport.d.ts +6 -19
  12. package/dist/types/live/visualizer.d.ts +8 -11
  13. package/dist/types/modes/components/assistant-message.d.ts +1 -0
  14. package/dist/types/modes/components/custom-message.d.ts +1 -1
  15. package/dist/types/modes/components/message-frame.d.ts +8 -4
  16. package/dist/types/modes/components/session-account-selector.d.ts +11 -0
  17. package/dist/types/modes/controllers/selector-controller.d.ts +1 -0
  18. package/dist/types/modes/interactive-mode.d.ts +3 -6
  19. package/dist/types/modes/types.d.ts +9 -5
  20. package/dist/types/modes/utils/ui-helpers.d.ts +1 -6
  21. package/dist/types/session/agent-session-types.d.ts +8 -1
  22. package/dist/types/session/agent-session.d.ts +20 -1
  23. package/dist/types/session/auth-storage.d.ts +1 -1
  24. package/dist/types/session/eval-runner.d.ts +2 -0
  25. package/dist/types/session/messages.d.ts +2 -0
  26. package/dist/types/session/session-tools.d.ts +15 -0
  27. package/dist/types/session/streaming-output.d.ts +8 -0
  28. package/dist/types/slash-commands/helpers/session-pin.d.ts +9 -0
  29. package/dist/types/stt/index.d.ts +0 -2
  30. package/dist/types/stt/stt-controller.d.ts +7 -0
  31. package/dist/types/task/executor.d.ts +3 -1
  32. package/dist/types/task/structured-subagent.d.ts +3 -0
  33. package/dist/types/task/types.d.ts +11 -11
  34. package/dist/types/thinking.d.ts +13 -0
  35. package/dist/types/tiny/title-client.d.ts +10 -0
  36. package/dist/types/tools/builtin-names.d.ts +1 -1
  37. package/dist/types/tools/computer/protocol.d.ts +43 -0
  38. package/dist/types/tools/computer/supervisor.d.ts +32 -0
  39. package/dist/types/tools/computer/worker-entry.d.ts +1 -0
  40. package/dist/types/tools/computer/worker.d.ts +15 -0
  41. package/dist/types/tools/computer-renderer.d.ts +22 -0
  42. package/dist/types/tools/computer.d.ts +71 -0
  43. package/dist/types/tools/context.d.ts +2 -0
  44. package/dist/types/tools/default-renderer.d.ts +21 -0
  45. package/dist/types/tools/eval-format/index.d.ts +7 -0
  46. package/dist/types/tools/eval-format/javascript.d.ts +2 -0
  47. package/dist/types/tools/eval-format/julia.d.ts +2 -0
  48. package/dist/types/tools/eval-format/python.d.ts +2 -0
  49. package/dist/types/tools/eval-format/ruby.d.ts +2 -0
  50. package/dist/types/tools/index.d.ts +2 -0
  51. package/dist/types/tools/resolve.d.ts +7 -0
  52. package/dist/types/tools/todo.d.ts +4 -1
  53. package/dist/types/tts/streaming-player.d.ts +10 -43
  54. package/dist/types/utils/tools-manager.d.ts +1 -2
  55. package/dist/types/web/search/providers/base.d.ts +16 -0
  56. package/dist/types/web/search/providers/brave.d.ts +2 -0
  57. package/dist/types/web/search/providers/firecrawl.d.ts +2 -0
  58. package/dist/types/web/search/providers/gemini.d.ts +3 -0
  59. package/dist/types/web/search/providers/jina.d.ts +2 -0
  60. package/dist/types/web/search/providers/kagi.d.ts +2 -0
  61. package/dist/types/web/search/providers/kimi.d.ts +2 -0
  62. package/dist/types/web/search/providers/parallel.d.ts +2 -0
  63. package/dist/types/web/search/providers/perplexity.d.ts +3 -0
  64. package/dist/types/web/search/providers/searxng.d.ts +10 -0
  65. package/dist/types/web/search/providers/tavily.d.ts +8 -0
  66. package/dist/types/web/search/query.d.ts +190 -0
  67. package/package.json +12 -12
  68. package/src/cli/args.ts +1 -0
  69. package/src/cli/bench-cli.ts +12 -1
  70. package/src/cli/setup-cli.ts +2 -14
  71. package/src/cli/web-search-cli.ts +7 -0
  72. package/src/cli.ts +8 -0
  73. package/src/config/model-registry.ts +6 -0
  74. package/src/config/model-resolver.ts +17 -6
  75. package/src/config/settings-schema.ts +66 -0
  76. package/src/debug/raw-sse-buffer.ts +157 -30
  77. package/src/eval/executor-base.ts +1 -0
  78. package/src/eval/js/executor.ts +2 -0
  79. package/src/exec/bash-executor.ts +1 -0
  80. package/src/export/share.ts +4 -3
  81. package/src/extensibility/extensions/wrapper.ts +68 -12
  82. package/src/extensibility/legacy-pi-coding-agent-shim.ts +7 -1
  83. package/src/launch/broker-list-order.test.ts +89 -0
  84. package/src/launch/broker.ts +49 -8
  85. package/src/live/attestation.ts +91 -0
  86. package/src/live/controller.ts +76 -23
  87. package/src/live/protocol.test.ts +3 -3
  88. package/src/live/protocol.ts +1 -1
  89. package/src/live/transport.ts +72 -140
  90. package/src/live/visualizer.ts +114 -134
  91. package/src/modes/components/assistant-message.ts +7 -2
  92. package/src/modes/components/custom-message.ts +4 -1
  93. package/src/modes/components/message-frame.ts +14 -8
  94. package/src/modes/components/session-account-selector.ts +62 -0
  95. package/src/modes/components/tool-execution.ts +17 -110
  96. package/src/modes/controllers/input-controller.ts +55 -47
  97. package/src/modes/controllers/live-command-controller.ts +82 -5
  98. package/src/modes/controllers/selector-controller.ts +62 -0
  99. package/src/modes/interactive-mode.ts +79 -5
  100. package/src/modes/types.ts +10 -4
  101. package/src/modes/utils/ui-helpers.ts +7 -8
  102. package/src/prompts/system/computer-safety.md +14 -0
  103. package/src/prompts/system/resolve-device-reminder.md +1 -1
  104. package/src/prompts/system/workflow-notice.md +1 -1
  105. package/src/prompts/tools/ast-edit.md +1 -0
  106. package/src/prompts/tools/bash.md +1 -0
  107. package/src/prompts/tools/computer.md +26 -0
  108. package/src/prompts/tools/eval.md +2 -2
  109. package/src/prompts/tools/task.md +5 -2
  110. package/src/prompts/tools/web-search.md +2 -0
  111. package/src/sdk.ts +13 -4
  112. package/src/session/agent-session-types.ts +9 -0
  113. package/src/session/agent-session.ts +70 -0
  114. package/src/session/auth-storage.ts +1 -0
  115. package/src/session/eval-runner.ts +5 -0
  116. package/src/session/messages.ts +3 -0
  117. package/src/session/queued-messages.ts +7 -1
  118. package/src/session/session-tools.ts +37 -0
  119. package/src/session/streaming-output.ts +52 -5
  120. package/src/slash-commands/builtin-registry.ts +165 -9
  121. package/src/slash-commands/helpers/session-pin.ts +44 -0
  122. package/src/stt/downloader.ts +0 -2
  123. package/src/stt/index.ts +0 -2
  124. package/src/stt/stt-controller.ts +57 -146
  125. package/src/system-prompt.ts +4 -0
  126. package/src/task/executor.ts +17 -9
  127. package/src/task/index.ts +14 -34
  128. package/src/task/structured-subagent.ts +4 -0
  129. package/src/task/types.ts +15 -13
  130. package/src/thinking.ts +27 -0
  131. package/src/tiny/title-client.ts +22 -0
  132. package/src/tools/ast-edit.ts +4 -1
  133. package/src/tools/bash-interactive.ts +90 -86
  134. package/src/tools/bash.ts +13 -0
  135. package/src/tools/browser/render.ts +2 -1
  136. package/src/tools/builtin-names.ts +1 -0
  137. package/src/tools/computer/protocol.ts +28 -0
  138. package/src/tools/computer/supervisor.ts +258 -0
  139. package/src/tools/computer/worker-entry.ts +25 -0
  140. package/src/tools/computer/worker.ts +135 -0
  141. package/src/tools/computer-renderer.ts +108 -0
  142. package/src/tools/computer.ts +433 -0
  143. package/src/tools/context.ts +2 -0
  144. package/src/tools/default-renderer.ts +139 -0
  145. package/src/tools/essential-tools.ts +1 -0
  146. package/src/tools/eval-format/index.ts +24 -0
  147. package/src/tools/eval-format/javascript.ts +952 -0
  148. package/src/tools/eval-format/julia.ts +446 -0
  149. package/src/tools/eval-format/python.ts +544 -0
  150. package/src/tools/eval-format/ruby.ts +380 -0
  151. package/src/tools/eval-render.ts +12 -6
  152. package/src/tools/index.ts +5 -0
  153. package/src/tools/renderers.ts +2 -0
  154. package/src/tools/resolve.ts +10 -1
  155. package/src/tools/todo.ts +58 -6
  156. package/src/tools/xdev.ts +54 -26
  157. package/src/tts/streaming-player.ts +81 -340
  158. package/src/utils/clipboard.ts +1 -30
  159. package/src/utils/mac-file-urls.applescript +37 -0
  160. package/src/utils/tool-choice.ts +14 -0
  161. package/src/utils/tools-manager.ts +1 -19
  162. package/src/web/search/index.ts +28 -5
  163. package/src/web/search/providers/anthropic.ts +62 -4
  164. package/src/web/search/providers/base.ts +16 -0
  165. package/src/web/search/providers/brave.ts +30 -3
  166. package/src/web/search/providers/codex.ts +14 -1
  167. package/src/web/search/providers/duckduckgo.ts +23 -1
  168. package/src/web/search/providers/ecosia.ts +5 -1
  169. package/src/web/search/providers/exa.ts +28 -1
  170. package/src/web/search/providers/firecrawl.ts +37 -3
  171. package/src/web/search/providers/gemini.ts +12 -2
  172. package/src/web/search/providers/google.ts +2 -1
  173. package/src/web/search/providers/jina.ts +31 -8
  174. package/src/web/search/providers/kagi.ts +10 -1
  175. package/src/web/search/providers/kimi.ts +16 -1
  176. package/src/web/search/providers/mojeek.ts +14 -1
  177. package/src/web/search/providers/parallel.ts +48 -2
  178. package/src/web/search/providers/perplexity.ts +94 -6
  179. package/src/web/search/providers/searxng.ts +145 -9
  180. package/src/web/search/providers/startpage.ts +8 -2
  181. package/src/web/search/providers/synthetic.ts +7 -1
  182. package/src/web/search/providers/tavily.ts +52 -3
  183. package/src/web/search/providers/tinyfish.ts +6 -1
  184. package/src/web/search/providers/xai.ts +49 -2
  185. package/src/web/search/providers/zai.ts +19 -1
  186. package/src/web/search/query.ts +850 -0
  187. package/dist/types/stt/recorder.d.ts +0 -30
  188. package/dist/types/stt/transcriber.d.ts +0 -14
  189. package/dist/types/stt/wav.d.ts +0 -29
  190. package/dist/types/tts/player.d.ts +0 -32
  191. package/src/live/audio-worklet.txt +0 -59
  192. package/src/live/browser-runtime.txt +0 -221
  193. package/src/stt/recorder.ts +0 -551
  194. package/src/stt/transcriber.ts +0 -60
  195. package/src/stt/wav.ts +0 -173
  196. package/src/tts/player.ts +0 -137
@@ -0,0 +1,91 @@
1
+ import { deviceCheckGenerateToken } from "@oh-my-pi/pi-natives";
2
+
3
+ const CHATGPT_BUNDLE_ID = "com.openai.codex";
4
+ const APP_SESSION_ID = crypto.randomUUID();
5
+
6
+ type DeviceCheckResult = {
7
+ supported: boolean;
8
+ tokenBase64?: string | null;
9
+ latencyMs?: number | null;
10
+ };
11
+
12
+ function cborHeader(major: number, value: number): Buffer {
13
+ if (!Number.isSafeInteger(value) || value < 0) throw new Error(`Invalid CBOR length: ${value}`);
14
+ if (value < 24) return Buffer.from([major + value]);
15
+ if (value <= 0xff) return Buffer.from([major + 24, value]);
16
+ if (value <= 0xffff) {
17
+ const output = Buffer.allocUnsafe(3);
18
+ output[0] = major + 25;
19
+ output.writeUInt16BE(value, 1);
20
+ return output;
21
+ }
22
+ if (value <= 0xffff_ffff) {
23
+ const output = Buffer.allocUnsafe(5);
24
+ output[0] = major + 26;
25
+ output.writeUInt32BE(value, 1);
26
+ return output;
27
+ }
28
+ throw new Error(`CBOR length is too large: ${value}`);
29
+ }
30
+
31
+ function cborUnsigned(value: number): Buffer {
32
+ return cborHeader(0, value);
33
+ }
34
+
35
+ function cborText(value: string): Buffer {
36
+ const text = Buffer.from(value, "utf8");
37
+ return Buffer.concat([cborHeader(96, text.length), text]);
38
+ }
39
+
40
+ function cborMap(entries: ReadonlyArray<readonly [Buffer, Buffer]>): Buffer {
41
+ const values: Buffer[] = [cborHeader(160, entries.length)];
42
+ for (const [key, value] of entries) values.push(key, value);
43
+ return Buffer.concat(values);
44
+ }
45
+
46
+ function attestationSignals(): Buffer {
47
+ const resolved = Intl.DateTimeFormat().resolvedOptions();
48
+ const locale = (resolved.locale || "unknown").slice(0, 64);
49
+ const timezone = (resolved.timeZone || "unknown").slice(0, 64);
50
+ const preferredLanguages = Buffer.concat([cborHeader(128, 1), cborText(locale)]);
51
+ return cborMap([
52
+ [cborUnsigned(0), cborUnsigned(1)],
53
+ [cborUnsigned(1), preferredLanguages],
54
+ [cborUnsigned(2), cborText(locale)],
55
+ [cborUnsigned(3), cborText(timezone)],
56
+ [cborUnsigned(4), cborUnsigned(0)],
57
+ [cborUnsigned(5), cborUnsigned(1)],
58
+ [cborUnsigned(6), cborText(APP_SESSION_ID.slice(0, 128))],
59
+ ]);
60
+ }
61
+
62
+ function buildClientAttestation(result: DeviceCheckResult): string {
63
+ const entries: Array<readonly [Buffer, Buffer]> = [];
64
+ if (result.supported && result.tokenBase64) {
65
+ entries.push([cborText("token"), cborText(result.tokenBase64)]);
66
+ } else {
67
+ entries.push([cborText("error_code"), cborUnsigned(result.supported ? 4 : 3)]);
68
+ }
69
+ entries.push([cborText("bundle_id"), cborText(CHATGPT_BUNDLE_ID)]);
70
+ const signals = attestationSignals();
71
+ entries.push([cborText("f"), Buffer.concat([cborHeader(64, signals.length), signals])]);
72
+ if (result.latencyMs !== undefined && result.latencyMs !== null) {
73
+ const latency = Buffer.allocUnsafe(9);
74
+ latency[0] = 0xfb;
75
+ latency.writeDoubleBE(result.latencyMs, 1);
76
+ entries.push([cborText("t"), latency]);
77
+ }
78
+ return `v1.${cborMap(entries).toString("base64url")}`;
79
+ }
80
+
81
+ /** Generates the Codex DeviceCheck attestation envelope sent as `x-oai-attestation` on ChatGPT-OAuth requests. */
82
+ export async function generateCodexAttestation(): Promise<string | undefined> {
83
+ if (process.platform !== "darwin" || process.arch !== "arm64") return undefined;
84
+ let result: DeviceCheckResult;
85
+ try {
86
+ result = await deviceCheckGenerateToken();
87
+ } catch {
88
+ return undefined;
89
+ }
90
+ return JSON.stringify({ v: 1, s: 0, t: buildClientAttestation(result) });
91
+ }
@@ -1,10 +1,11 @@
1
1
  import * as os from "node:os";
2
2
  import type { AgentMessage } from "@oh-my-pi/pi-agent-core";
3
3
  import type { AssistantMessage } from "@oh-my-pi/pi-ai";
4
+ import { AudioCapture } from "@oh-my-pi/pi-natives";
4
5
  import { prompt } from "@oh-my-pi/pi-utils";
5
6
  import type { AgentSession } from "../session/agent-session";
6
7
  import type { AgentSessionEvent } from "../session/agent-session-events";
7
- import { type StreamingRecordingHandle, startStreamingRecording } from "../stt/recorder";
8
+ import { LIVE_DELEGATION_MESSAGE_TYPE } from "../session/messages";
8
9
  import agentFinalMessageTemplate from "./prompts/agent-final-message.md" with { type: "text" };
9
10
  import liveInstructionsTemplate from "./prompts/live-instructions.md" with { type: "text" };
10
11
  import {
@@ -15,13 +16,22 @@ import {
15
16
  type LiveServerEvent,
16
17
  } from "./protocol";
17
18
  import { CodexLiveTransport } from "./transport";
18
- import type { LivePhase, LiveTranscript } from "./visualizer";
19
+ import type { LivePhase } from "./visualizer";
19
20
 
20
- const DEFAULT_VOICE = "marin";
21
+ const DEFAULT_VOICE = "sol";
21
22
  const OUTPUT_ACTIVE_LEVEL = 0.015;
22
23
  const MIN_BARGE_IN_LEVEL = 0.04;
23
24
  const OUTPUT_ECHO_RATIO = 0.65;
24
25
 
26
+ /** Incremental or final transcript for one realtime conversational turn. */
27
+ export interface LiveTranscript {
28
+ role: "user" | "assistant";
29
+ text: string;
30
+ /** Monotonic role-local turn number used to coalesce streaming updates. */
31
+ turn: number;
32
+ final: boolean;
33
+ }
34
+
25
35
  /** UI notifications emitted during a live session. */
26
36
  export interface LiveSessionCallbacks {
27
37
  /** Reports connection and activity phase changes. */
@@ -42,7 +52,7 @@ export interface LiveSessionControllerOptions {
42
52
  callbacks: LiveSessionCallbacks;
43
53
  /** Extracts visible assistant text using the caller's normal UI rules. */
44
54
  extractAssistantText(message: AssistantMessage): string;
45
- /** Realtime output voice, defaulting to marin. */
55
+ /** Realtime output voice, defaulting to sol. */
46
56
  voice?: string;
47
57
  }
48
58
 
@@ -85,7 +95,7 @@ export class LiveSessionController {
85
95
  readonly #voice: string;
86
96
 
87
97
  #transport: CodexLiveTransport | undefined;
88
- #recorder: StreamingRecordingHandle | undefined;
98
+ #recorder: AudioCapture | undefined;
89
99
  #unsubscribeSession: (() => void) | undefined;
90
100
  #sendChain: Promise<void> = Promise.resolve();
91
101
  #stopPromise: Promise<void> | undefined;
@@ -102,6 +112,8 @@ export class LiveSessionController {
102
112
  #assistantTranscript = "";
103
113
  #userTranscriptFinal = false;
104
114
  #assistantTranscriptFinal = false;
115
+ #userTranscriptTurn = 0;
116
+ #assistantTranscriptTurn = 0;
105
117
  #lastTranscript: LiveTranscript | undefined;
106
118
 
107
119
  constructor(options: LiveSessionControllerOptions) {
@@ -161,20 +173,21 @@ export class LiveSessionController {
161
173
  if (this.#stopped) {
162
174
  throw this.#failure ?? new Error("The live session stopped before recording began.");
163
175
  }
164
- const recorder = await startStreamingRecording(samples => this.#handleMicrophoneAudio(samples));
176
+ const recorder = new AudioCapture(16_000, (error, samples) => {
177
+ if (error) {
178
+ this.#reportFailure(error);
179
+ return;
180
+ }
181
+ this.#handleMicrophoneAudio(samples);
182
+ });
165
183
  if (this.#stopped) {
166
- if (recorder) {
167
- try {
168
- await recorder.stop();
169
- } catch {
170
- // Preserve the failure that stopped startup.
171
- }
184
+ try {
185
+ recorder.stop();
186
+ } catch {
187
+ // Preserve the failure that stopped startup.
172
188
  }
173
189
  throw this.#failure ?? new Error("The live session stopped while recording began.");
174
190
  }
175
- if (!recorder) {
176
- throw new Error("Live mode needs a streaming audio recorder; run `omp setup speech` and try again.");
177
- }
178
191
  this.#recorder = recorder;
179
192
  this.#refreshAudioPhase();
180
193
  } catch (cause) {
@@ -216,7 +229,7 @@ export class LiveSessionController {
216
229
  this.#recorder = undefined;
217
230
  if (recorder) {
218
231
  try {
219
- await recorder.stop();
232
+ recorder.stop();
220
233
  } catch (cause) {
221
234
  cleanupError = errorFrom(cause);
222
235
  }
@@ -288,7 +301,17 @@ export class LiveSessionController {
288
301
  if (!request) return;
289
302
  this.#activeDelegationId = event.item.id;
290
303
  this.#emitPhase("working");
291
- void this.#session.sendUserMessage(request).catch(cause => this.#reportFailure(errorFrom(cause)));
304
+ void this.#session
305
+ .sendCustomMessage(
306
+ {
307
+ customType: LIVE_DELEGATION_MESSAGE_TYPE,
308
+ content: request,
309
+ display: true,
310
+ attribution: "agent",
311
+ },
312
+ { triggerTurn: true },
313
+ )
314
+ .catch(cause => this.#reportFailure(errorFrom(cause)));
292
315
  }
293
316
 
294
317
  #handleSessionEvent(event: AgentSessionEvent): void {
@@ -354,7 +377,12 @@ export class LiveSessionController {
354
377
  const current = role === "user" ? this.#userTranscript : this.#assistantTranscript;
355
378
  const wasFinal = role === "user" ? this.#userTranscriptFinal : this.#assistantTranscriptFinal;
356
379
  let next: string;
357
- if (wasFinal && current !== text && !text.startsWith(current) && !current.endsWith(text)) {
380
+ if (!current) {
381
+ this.#startTranscriptTurn(role);
382
+ next = text;
383
+ } else if (wasFinal) {
384
+ if (text === current || current.endsWith(text)) return;
385
+ this.#startTranscriptTurn(role);
358
386
  next = text;
359
387
  } else if (text.startsWith(current)) {
360
388
  next = text;
@@ -369,20 +397,45 @@ export class LiveSessionController {
369
397
  #finishTranscript(role: LiveTranscript["role"], text: string): void {
370
398
  if (!text) return;
371
399
  const current = role === "user" ? this.#userTranscript : this.#assistantTranscript;
372
- const next = current.startsWith(text) && current.length > text.length ? current : text;
400
+ const wasFinal = role === "user" ? this.#userTranscriptFinal : this.#assistantTranscriptFinal;
401
+ if (!current) {
402
+ this.#startTranscriptTurn(role);
403
+ } else if (wasFinal) {
404
+ if (text === current) return;
405
+ this.#startTranscriptTurn(role);
406
+ }
407
+ const next = !wasFinal && current.startsWith(text) && current.length > text.length ? current : text;
373
408
  this.#storeTranscript(role, next, true);
374
409
  }
375
410
 
411
+ #startTranscriptTurn(role: LiveTranscript["role"]): void {
412
+ if (role === "user") {
413
+ this.#userTranscriptTurn += 1;
414
+ } else {
415
+ this.#assistantTranscriptTurn += 1;
416
+ }
417
+ }
418
+
376
419
  #storeTranscript(role: LiveTranscript["role"], text: string, final: boolean): void {
420
+ const normalized = text.trim();
421
+ if (!normalized) return;
422
+ const turn = role === "user" ? this.#userTranscriptTurn : this.#assistantTranscriptTurn;
377
423
  if (role === "user") {
378
- this.#userTranscript = text;
424
+ this.#userTranscript = normalized;
379
425
  this.#userTranscriptFinal = final;
380
426
  } else {
381
- this.#assistantTranscript = text;
427
+ this.#assistantTranscript = normalized;
382
428
  this.#assistantTranscriptFinal = final;
383
429
  }
384
- if (this.#lastTranscript?.role === role && this.#lastTranscript.text === text) return;
385
- this.#emitTranscript({ role, text });
430
+ if (
431
+ this.#lastTranscript?.role === role &&
432
+ this.#lastTranscript.turn === turn &&
433
+ this.#lastTranscript.text === normalized &&
434
+ this.#lastTranscript.final === final
435
+ ) {
436
+ return;
437
+ }
438
+ this.#emitTranscript({ role, turn, text: normalized, final });
386
439
  }
387
440
 
388
441
  #queueSend(message: LiveClientMessage): void {
@@ -95,12 +95,12 @@ describe("Frameless Bidi server events", () => {
95
95
  });
96
96
 
97
97
  describe("Frameless Bidi client payloads", () => {
98
- test("builds the exact multipart call session JSON", () => {
98
+ test("builds the exact live call session JSON", () => {
99
99
  const payload = buildLiveSessionPayload("Be concise.", "marin");
100
100
 
101
- expect(LIVE_MODEL).toBe("gpt-live-1-boulder-alpha");
101
+ expect(LIVE_MODEL).toBe("gpt-live-1-codex");
102
102
  expect(JSON.stringify(payload)).toBe(
103
- '{"model":"gpt-live-1-boulder-alpha","instructions":"Be concise.","audio":{"output":{"voice":"marin"}},"delegation":{"type":"client"}}',
103
+ '{"model":"gpt-live-1-codex","instructions":"Be concise.","audio":{"output":{"voice":"marin"}},"delegation":{"type":"client"}}',
104
104
  );
105
105
  });
106
106
 
@@ -1,5 +1,5 @@
1
1
  /** Frameless Bidi model used by Codex Desktop live calls. */
2
- export const LIVE_MODEL: "gpt-live-1-boulder-alpha" = "gpt-live-1-boulder-alpha";
2
+ export const LIVE_MODEL: "gpt-live-1-codex" = "gpt-live-1-codex";
3
3
 
4
4
  /** Maximum UTF-8 payload size accepted by each context append. */
5
5
  export const CONTEXT_CHUNK_BYTES = 500;
@@ -4,13 +4,10 @@ import {
4
4
  CODEX_BASE_URL,
5
5
  CODEX_CLIENT_VERSION,
6
6
  getCodexAccountId,
7
- OPENAI_HEADER_VALUES,
8
7
  OPENAI_HEADERS,
9
8
  } from "@oh-my-pi/pi-catalog/wire/codex";
10
- import type { Browser, HTTPRequest, Page } from "puppeteer-core";
11
- import { launchHeadlessBrowser } from "../tools/browser/launch";
12
- import audioWorkletSource from "./audio-worklet.txt" with { type: "text" };
13
- import browserRuntimeSource from "./browser-runtime.txt" with { type: "text" };
9
+ import { LiveWebRtcPeer } from "@oh-my-pi/pi-natives";
10
+ import { generateCodexAttestation } from "./attestation";
14
11
  import {
15
12
  buildLiveSessionPayload,
16
13
  type LiveClientMessage,
@@ -20,37 +17,19 @@ import {
20
17
 
21
18
  const SIGNALING_URL = `${CODEX_BASE_URL}/codex/realtime/calls?intent=quicksilver&architecture=avas`;
22
19
  const MAX_ERROR_BODY_LENGTH = 2_048;
23
- const MAX_HOST_AUDIO_SAMPLES = 32_000;
24
20
  const SIDEBAND_CONNECT_ATTEMPTS = 5;
25
21
  const SIDEBAND_CONNECT_TIMEOUT_MS = 15_000;
26
22
  const LIVE_PROVIDER = "openai-codex";
27
- const LIVE_CALL_ID_PATTERN = /^rtc_[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$/i;
23
+ const LIVE_ORIGINATOR = "Codex Desktop";
24
+ const LIVE_CALL_ID_PATTERN = /^rtc_[\w-]+$/;
28
25
 
29
26
  type Lifecycle = "idle" | "connecting" | "connected" | "closing" | "closed";
30
27
 
31
- type BrowserLiveApi = {
32
- start(workletSource: string): Promise<string>;
33
- acceptAnswer(sdp: string): Promise<void>;
34
- waitForOpen(): Promise<void>;
35
- send(payload: string): void;
36
- pushAudio(payload: string): void;
37
- setMuted(muted: boolean): void;
38
- close(): Promise<void>;
39
- };
40
-
41
- declare global {
42
- var ompCodexLive: BrowserLiveApi;
43
- }
44
-
45
- interface QueuedAudio {
46
- payload: string;
47
- sampleCount: number;
48
- }
49
-
50
28
  interface LiveSignalingResult {
51
29
  answer: string;
52
30
  callId: string;
53
31
  access: OAuthAccess;
32
+ attestation: string | undefined;
54
33
  }
55
34
 
56
35
  class LiveSignalingError extends Error {
@@ -81,7 +60,7 @@ export interface LiveTransportOptions {
81
60
  signal?: AbortSignal;
82
61
  }
83
62
 
84
- /** Extracts the server-assigned `rtc_<uuid>` call ID from a signaling Location header. */
63
+ /** Extracts the server-assigned `rtc_*` call ID from a signaling Location header. */
85
64
  export function parseLiveCallId(location: string | null): string | undefined {
86
65
  if (!location) return undefined;
87
66
  return location
@@ -92,23 +71,30 @@ export function parseLiveCallId(location: string | null): string | undefined {
92
71
 
93
72
  /** Builds the Frameless Bidi sideband WebSocket URL for an accepted Codex call. */
94
73
  export function buildLiveSidebandUrl(callId: string): string {
95
- const url = new URL(`${CODEX_BASE_URL}/codex/${encodeURIComponent(callId)}`);
96
- url.protocol = url.protocol === "http:" ? "ws:" : "wss:";
74
+ const url = new URL(`https://api.openai.com/v1/live/${encodeURIComponent(callId)}`);
75
+ url.protocol = "wss:";
97
76
  return url.toString();
98
77
  }
99
78
 
100
- function liveSessionHeaders(access: OAuthAccess, sessionId: string): Record<string, string> {
79
+ function liveSessionHeaders(
80
+ access: OAuthAccess,
81
+ sessionId: string,
82
+ realtimeSessionId: string,
83
+ attestation: string | undefined,
84
+ ): Record<string, string> {
101
85
  const headers: Record<string, string> = {
102
86
  Authorization: `Bearer ${access.accessToken}`,
103
87
  "OpenAI-Alpha": "quicksilver=v2",
104
- "x-session-id": sessionId,
105
- [OPENAI_HEADERS.ORIGINATOR]: OPENAI_HEADER_VALUES.ORIGINATOR_CODEX,
88
+ "User-Agent": `Codex Desktop/${CODEX_CLIENT_VERSION}`,
89
+ "x-session-id": realtimeSessionId,
90
+ [OPENAI_HEADERS.ORIGINATOR]: LIVE_ORIGINATOR,
106
91
  [OPENAI_HEADERS.VERSION]: CODEX_CLIENT_VERSION,
107
92
  [OPENAI_HEADERS.SCOPED_SESSION_ID]: sessionId,
108
93
  [OPENAI_HEADERS.THREAD_ID]: sessionId,
109
94
  };
110
95
  const accountId = access.accountId ?? getCodexAccountId(access.accessToken);
111
96
  if (accountId) headers[OPENAI_HEADERS.ACCOUNT_ID] = accountId;
97
+ if (attestation) headers["x-oai-attestation"] = attestation;
112
98
  return headers;
113
99
  }
114
100
 
@@ -128,23 +114,16 @@ function abortReason(signal: AbortSignal | undefined): Error {
128
114
  return new DOMException("Live connection aborted", "AbortError");
129
115
  }
130
116
 
131
- function encodePcm(samples: Float32Array): string {
132
- return Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength).toString("base64");
133
- }
134
-
135
- /** Headless-Chromium WebRTC transport for a Codex Frameless Bidi live session. */
117
+ /** Native WebRTC transport for a Codex Frameless Bidi live session. */
136
118
  export class CodexLiveTransport {
137
119
  readonly #options: LiveTransportOptions;
138
- #browser: Browser | undefined;
139
- #page: Page | undefined;
120
+ #peer: LiveWebRtcPeer | undefined;
121
+ readonly #realtimeSessionId = crypto.randomUUID();
140
122
  #sideband: Bun.WebSocket | undefined;
141
123
  #state: Lifecycle = "idle";
142
124
  #connectPromise: Promise<void> | undefined;
143
125
  #closePromise: Promise<void> | undefined;
144
126
  #sendTail: Promise<void> = Promise.resolve();
145
- #audioQueue: QueuedAudio[] = [];
146
- #queuedAudioSamples = 0;
147
- #audioPumpRunning = false;
148
127
  #muted = false;
149
128
  #unexpectedFailureReported = false;
150
129
  readonly #abortListener: () => void;
@@ -157,7 +136,7 @@ export class CodexLiveTransport {
157
136
  if (!options.signal?.aborted) options.signal?.addEventListener("abort", this.#abortListener, { once: true });
158
137
  }
159
138
 
160
- /** Establish the browser peer, perform Codex signaling, and wait for the data channel. */
139
+ /** Establish the native peer, perform Codex signaling, and wait for the data channel. */
161
140
  connect(): Promise<void> {
162
141
  if (this.#state === "connected") return Promise.resolve();
163
142
  if (this.#connectPromise) return this.#connectPromise;
@@ -174,49 +153,42 @@ export class CodexLiveTransport {
174
153
  }
175
154
 
176
155
  async #connect(): Promise<void> {
177
- const browser = await launchHeadlessBrowser({
178
- headless: true,
179
- args: ["--autoplay-policy=no-user-gesture-required"],
180
- ignoreDefaultArgs: ["--mute-audio"],
181
- });
182
- this.#browser = browser;
183
- if (this.#state !== "connecting") throw abortReason(this.#options.signal);
184
- const page = await browser.newPage();
185
- this.#page = page;
186
- await page.setRequestInterception(true);
187
- const serveBlankPage = (request: HTTPRequest): void => {
188
- void request
189
- .respond({ status: 200, contentType: "text/html", body: "<!doctype html><title>Codex Live</title>" })
190
- .catch(() => {});
191
- };
192
- page.on("request", serveBlankPage);
193
- try {
194
- await page.goto("http://localhost/", { waitUntil: "domcontentloaded" });
195
- } finally {
196
- page.off("request", serveBlankPage);
197
- await page.setRequestInterception(false);
198
- }
199
- await page.exposeFunction("__ompLiveServerEvent", (payload: string) => this.#handleServerEvent(payload));
200
- await page.exposeFunction("__ompLiveOutputLevel", (level: number) => this.#handleOutputLevel(level));
201
- await page.exposeFunction("__ompLiveFailure", (message: string) => this.#handleBrowserFailure(message));
202
- await page.evaluate(source => Function(source)(), browserRuntimeSource);
203
- const offer = await page.evaluate(source => globalThis.ompCodexLive.start(source), audioWorkletSource);
156
+ const peer = new LiveWebRtcPeer(
157
+ (error, payload) => {
158
+ if (error) {
159
+ this.#handlePeerFailure(error.message);
160
+ } else {
161
+ this.#handleServerEvent(payload);
162
+ }
163
+ },
164
+ (error, level) => {
165
+ if (error) {
166
+ this.#handlePeerFailure(error.message);
167
+ } else {
168
+ this.#handleOutputLevel(level);
169
+ }
170
+ },
171
+ (error, message) => this.#handlePeerFailure(error?.message ?? message),
172
+ );
173
+ this.#peer = peer;
174
+ const offer = await peer.createOffer();
204
175
  if (this.#state !== "connecting") throw abortReason(this.#options.signal);
205
176
  const signaling = await this.#signal(offer);
206
- await page.evaluate(sdp => globalThis.ompCodexLive.acceptAnswer(sdp), signaling.answer);
207
- await page.evaluate(muted => globalThis.ompCodexLive.setMuted(muted), this.#muted);
208
- await page.evaluate(() => globalThis.ompCodexLive.waitForOpen());
177
+ await peer.acceptAnswer(signaling.answer);
178
+ peer.setMuted(this.#muted);
179
+ await peer.waitForOpen();
209
180
  if (this.#state !== "connecting") throw abortReason(this.#options.signal);
210
- await this.#connectSideband(signaling.callId, signaling.access);
181
+ await this.#connectSideband(signaling.callId, signaling.access, signaling.attestation);
211
182
  if (this.#state !== "connecting") throw abortReason(this.#options.signal);
212
183
  this.#state = "connected";
213
184
  }
214
185
 
215
186
  async #signal(offer: string): Promise<LiveSignalingResult> {
187
+ const attestation = await generateCodexAttestation();
216
188
  return await withOAuthAccess(
217
189
  this.#options.authStorage,
218
190
  LIVE_PROVIDER,
219
- access => this.#signalWithAccess(offer, access),
191
+ access => this.#signalWithAccess(offer, access, attestation),
220
192
  {
221
193
  sessionId: this.#options.sessionId,
222
194
  signal: this.#options.signal,
@@ -226,10 +198,14 @@ export class CodexLiveTransport {
226
198
  );
227
199
  }
228
200
 
229
- async #signalWithAccess(offer: string, access: OAuthAccess): Promise<LiveSignalingResult> {
201
+ async #signalWithAccess(
202
+ offer: string,
203
+ access: OAuthAccess,
204
+ attestation: string | undefined,
205
+ ): Promise<LiveSignalingResult> {
230
206
  const headers = new Headers({
231
- ...liveSessionHeaders(access, this.#options.sessionId),
232
- Accept: "application/sdp",
207
+ ...liveSessionHeaders(access, this.#options.sessionId, this.#realtimeSessionId, attestation),
208
+ Accept: "*/*",
233
209
  "Content-Type": "application/json",
234
210
  });
235
211
  const fetchImpl = wrapFetchForProxy(fetch, LIVE_PROVIDER);
@@ -247,20 +223,21 @@ export class CodexLiveTransport {
247
223
  const detail = boundedErrorBody(responseBody, response.statusText);
248
224
  throw new LiveSignalingError(response.status, `Codex live signaling failed (${response.status}): ${detail}`);
249
225
  }
250
- const answer = responseBody.trim();
251
- if (!answer) throw new LiveSignalingError(response.status, "Codex live signaling returned an empty SDP answer");
226
+ const answer = responseBody;
227
+ if (!answer.trim())
228
+ throw new LiveSignalingError(response.status, "Codex live signaling returned an empty SDP answer");
252
229
  const callId = parseLiveCallId(response.headers.get("location"));
253
230
  if (!callId) {
254
231
  throw new LiveSignalingError(response.status, "Codex live signaling returned no valid call ID");
255
232
  }
256
- return { answer, callId, access };
233
+ return { answer, callId, access, attestation };
257
234
  }
258
235
 
259
- async #connectSideband(callId: string, access: OAuthAccess): Promise<void> {
236
+ async #connectSideband(callId: string, access: OAuthAccess, attestation: string | undefined): Promise<void> {
260
237
  let failure = new Error("Codex live sideband connection failed");
261
238
  for (let attempt = 0; attempt < SIDEBAND_CONNECT_ATTEMPTS; attempt++) {
262
239
  try {
263
- await this.#openSideband(callId, access);
240
+ await this.#openSideband(callId, access, attestation);
264
241
  return;
265
242
  } catch (cause) {
266
243
  failure = cause instanceof Error ? cause : new Error(String(cause));
@@ -271,10 +248,10 @@ export class CodexLiveTransport {
271
248
  throw failure;
272
249
  }
273
250
 
274
- async #openSideband(callId: string, access: OAuthAccess): Promise<void> {
251
+ async #openSideband(callId: string, access: OAuthAccess, attestation: string | undefined): Promise<void> {
275
252
  const url = buildLiveSidebandUrl(callId);
276
253
  const options = {
277
- headers: liveSessionHeaders(access, this.#options.sessionId),
254
+ headers: liveSessionHeaders(access, this.#options.sessionId, this.#realtimeSessionId, attestation),
278
255
  proxy: getProxyForProvider(LIVE_PROVIDER),
279
256
  } satisfies Bun.WebSocketOptions;
280
257
  const socket: Bun.WebSocket = Reflect.construct(WebSocket, [url, options]);
@@ -377,7 +354,7 @@ export class CodexLiveTransport {
377
354
  } catch {}
378
355
  }
379
356
 
380
- #handleBrowserFailure(message: string): void {
357
+ #handlePeerFailure(message: string): void {
381
358
  this.#reportFailure(message);
382
359
  }
383
360
 
@@ -405,52 +382,19 @@ export class CodexLiveTransport {
405
382
  return operation;
406
383
  }
407
384
 
408
- /** Queue 16 kHz mono Float32 PCM for continuous browser-side resampling and playback. */
385
+ /** Queue 16 kHz mono Float32 PCM for native Opus transmission. */
409
386
  pushAudio(samples: Float32Array): void {
410
387
  if (this.#state !== "connected" || this.#muted || samples.length === 0) return;
411
- const retained =
412
- samples.length > MAX_HOST_AUDIO_SAMPLES ? samples.subarray(samples.length - MAX_HOST_AUDIO_SAMPLES) : samples;
413
- const queued = { payload: encodePcm(retained), sampleCount: retained.length };
414
- this.#audioQueue.push(queued);
415
- this.#queuedAudioSamples += queued.sampleCount;
416
- while (this.#queuedAudioSamples > MAX_HOST_AUDIO_SAMPLES && this.#audioQueue.length > 1) {
417
- const stale = this.#audioQueue.shift();
418
- if (stale) this.#queuedAudioSamples -= stale.sampleCount;
419
- }
420
- if (!this.#audioPumpRunning) void this.#pumpAudio();
388
+ this.#peer?.pushAudio(samples);
421
389
  }
422
390
 
423
- async #pumpAudio(): Promise<void> {
424
- this.#audioPumpRunning = true;
425
- try {
426
- while (this.#state === "connected" && !this.#muted) {
427
- const queued = this.#audioQueue.shift();
428
- if (!queued) break;
429
- this.#queuedAudioSamples -= queued.sampleCount;
430
- const page = this.#page;
431
- if (!page) break;
432
- await page.evaluate(payload => globalThis.ompCodexLive.pushAudio(payload), queued.payload);
433
- }
434
- } catch {
435
- this.#audioQueue.length = 0;
436
- this.#queuedAudioSamples = 0;
437
- } finally {
438
- this.#audioPumpRunning = false;
439
- if (this.#audioQueue.length > 0 && this.#state === "connected" && !this.#muted) void this.#pumpAudio();
440
- }
441
- }
442
-
443
- /** Enable or disable the browser audio source and discard queued input when muted. */
391
+ /** Enable or disable the native audio source and discard partial input when muted. */
444
392
  async setMuted(muted: boolean): Promise<void> {
445
393
  this.#muted = muted;
446
- this.#audioQueue.length = 0;
447
- this.#queuedAudioSamples = 0;
448
- const page = this.#page;
449
- if (!page || this.#state !== "connected") return;
450
- await page.evaluate(value => globalThis.ompCodexLive.setMuted(value), muted);
394
+ if (this.#state === "connected") this.#peer?.setMuted(muted);
451
395
  }
452
396
 
453
- /** Stop audio, WebRTC, the page, and Chromium. Safe to call repeatedly. */
397
+ /** Stop sideband signaling and the native WebRTC media peer. Safe to call repeatedly. */
454
398
  close(): Promise<void> {
455
399
  if (this.#closePromise) return this.#closePromise;
456
400
  this.#state = "closing";
@@ -461,28 +405,16 @@ export class CodexLiveTransport {
461
405
 
462
406
  async #close(): Promise<void> {
463
407
  this.#options.signal?.removeEventListener("abort", this.#abortListener);
464
- this.#audioQueue.length = 0;
465
- this.#queuedAudioSamples = 0;
466
408
  const sideband = this.#sideband;
467
- const page = this.#page;
468
- const browser = this.#browser;
409
+ const peer = this.#peer;
469
410
  this.#sideband = undefined;
470
- this.#page = undefined;
471
- this.#browser = undefined;
411
+ this.#peer = undefined;
472
412
  if (sideband && (sideband.readyState === WebSocket.OPEN || sideband.readyState === WebSocket.CONNECTING)) {
473
413
  sideband.close(1000, "done");
474
414
  }
475
- if (page) {
476
- try {
477
- await page.evaluate(() => globalThis.ompCodexLive?.close());
478
- } catch {}
479
- try {
480
- await page.close();
481
- } catch {}
482
- }
483
- if (browser) {
415
+ if (peer) {
484
416
  try {
485
- await browser.close();
417
+ await peer.close();
486
418
  } catch {}
487
419
  }
488
420
  this.#state = "closed";