@oh-my-pi/pi-coding-agent 17.1.0 → 17.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/CHANGELOG.md +24 -0
  2. package/dist/cli.js +3686 -4013
  3. package/dist/types/config/settings-schema.d.ts +58 -0
  4. package/dist/types/extensibility/legacy-pi-coding-agent-shim.d.ts +3 -0
  5. package/dist/types/live/attestation.d.ts +2 -0
  6. package/dist/types/live/controller.d.ts +10 -2
  7. package/dist/types/live/protocol.d.ts +1 -1
  8. package/dist/types/live/transport.d.ts +6 -19
  9. package/dist/types/live/visualizer.d.ts +8 -11
  10. package/dist/types/modes/components/assistant-message.d.ts +1 -0
  11. package/dist/types/modes/components/custom-message.d.ts +1 -1
  12. package/dist/types/modes/components/message-frame.d.ts +8 -4
  13. package/dist/types/modes/components/session-account-selector.d.ts +11 -0
  14. package/dist/types/modes/controllers/selector-controller.d.ts +1 -0
  15. package/dist/types/modes/interactive-mode.d.ts +1 -0
  16. package/dist/types/modes/types.d.ts +1 -0
  17. package/dist/types/session/agent-session-types.d.ts +8 -1
  18. package/dist/types/session/agent-session.d.ts +20 -1
  19. package/dist/types/session/auth-storage.d.ts +1 -1
  20. package/dist/types/session/eval-runner.d.ts +2 -0
  21. package/dist/types/session/messages.d.ts +2 -0
  22. package/dist/types/session/session-tools.d.ts +15 -0
  23. package/dist/types/session/streaming-output.d.ts +8 -0
  24. package/dist/types/slash-commands/helpers/session-pin.d.ts +9 -0
  25. package/dist/types/stt/index.d.ts +0 -2
  26. package/dist/types/stt/stt-controller.d.ts +7 -0
  27. package/dist/types/tiny/title-client.d.ts +10 -0
  28. package/dist/types/tools/builtin-names.d.ts +1 -1
  29. package/dist/types/tools/computer/protocol.d.ts +43 -0
  30. package/dist/types/tools/computer/supervisor.d.ts +32 -0
  31. package/dist/types/tools/computer/worker-entry.d.ts +1 -0
  32. package/dist/types/tools/computer/worker.d.ts +15 -0
  33. package/dist/types/tools/computer-renderer.d.ts +22 -0
  34. package/dist/types/tools/computer.d.ts +71 -0
  35. package/dist/types/tools/context.d.ts +2 -0
  36. package/dist/types/tools/default-renderer.d.ts +21 -0
  37. package/dist/types/tools/index.d.ts +2 -0
  38. package/dist/types/tts/streaming-player.d.ts +10 -43
  39. package/dist/types/utils/tools-manager.d.ts +1 -2
  40. package/package.json +12 -12
  41. package/src/cli/args.ts +1 -0
  42. package/src/cli/setup-cli.ts +2 -14
  43. package/src/cli.ts +8 -0
  44. package/src/config/model-registry.ts +6 -0
  45. package/src/config/settings-schema.ts +61 -0
  46. package/src/eval/executor-base.ts +1 -0
  47. package/src/eval/js/executor.ts +2 -0
  48. package/src/exec/bash-executor.ts +1 -0
  49. package/src/extensibility/extensions/wrapper.ts +68 -12
  50. package/src/extensibility/legacy-pi-coding-agent-shim.ts +7 -1
  51. package/src/live/attestation.ts +91 -0
  52. package/src/live/controller.ts +76 -23
  53. package/src/live/protocol.test.ts +3 -3
  54. package/src/live/protocol.ts +1 -1
  55. package/src/live/transport.ts +72 -140
  56. package/src/live/visualizer.ts +114 -134
  57. package/src/modes/components/assistant-message.ts +7 -2
  58. package/src/modes/components/custom-message.ts +4 -1
  59. package/src/modes/components/message-frame.ts +14 -8
  60. package/src/modes/components/session-account-selector.ts +62 -0
  61. package/src/modes/components/tool-execution.ts +17 -110
  62. package/src/modes/controllers/input-controller.ts +47 -39
  63. package/src/modes/controllers/live-command-controller.ts +82 -5
  64. package/src/modes/controllers/selector-controller.ts +62 -0
  65. package/src/modes/interactive-mode.ts +19 -0
  66. package/src/modes/types.ts +1 -0
  67. package/src/prompts/system/computer-safety.md +14 -0
  68. package/src/prompts/tools/computer.md +26 -0
  69. package/src/sdk.ts +5 -0
  70. package/src/session/agent-session-types.ts +9 -0
  71. package/src/session/agent-session.ts +56 -0
  72. package/src/session/auth-storage.ts +1 -0
  73. package/src/session/eval-runner.ts +5 -0
  74. package/src/session/messages.ts +3 -0
  75. package/src/session/session-tools.ts +37 -0
  76. package/src/session/streaming-output.ts +52 -5
  77. package/src/slash-commands/builtin-registry.ts +165 -9
  78. package/src/slash-commands/helpers/session-pin.ts +44 -0
  79. package/src/stt/downloader.ts +0 -2
  80. package/src/stt/index.ts +0 -2
  81. package/src/stt/stt-controller.ts +57 -146
  82. package/src/system-prompt.ts +4 -0
  83. package/src/tiny/title-client.ts +22 -0
  84. package/src/tools/bash-interactive.ts +90 -86
  85. package/src/tools/builtin-names.ts +1 -0
  86. package/src/tools/computer/protocol.ts +28 -0
  87. package/src/tools/computer/supervisor.ts +258 -0
  88. package/src/tools/computer/worker-entry.ts +25 -0
  89. package/src/tools/computer/worker.ts +135 -0
  90. package/src/tools/computer-renderer.ts +108 -0
  91. package/src/tools/computer.ts +433 -0
  92. package/src/tools/context.ts +2 -0
  93. package/src/tools/default-renderer.ts +139 -0
  94. package/src/tools/essential-tools.ts +1 -0
  95. package/src/tools/index.ts +5 -0
  96. package/src/tools/renderers.ts +2 -0
  97. package/src/tools/xdev.ts +54 -26
  98. package/src/tts/streaming-player.ts +81 -340
  99. package/src/utils/clipboard.ts +1 -30
  100. package/src/utils/mac-file-urls.applescript +37 -0
  101. package/src/utils/tool-choice.ts +14 -0
  102. package/src/utils/tools-manager.ts +1 -19
  103. package/dist/types/stt/recorder.d.ts +0 -30
  104. package/dist/types/stt/transcriber.d.ts +0 -14
  105. package/dist/types/stt/wav.d.ts +0 -29
  106. package/dist/types/tts/player.d.ts +0 -32
  107. package/src/live/audio-worklet.txt +0 -59
  108. package/src/live/browser-runtime.txt +0 -221
  109. package/src/stt/recorder.ts +0 -551
  110. package/src/stt/transcriber.ts +0 -60
  111. package/src/stt/wav.ts +0 -173
  112. package/src/tts/player.ts +0 -137
@@ -0,0 +1,71 @@
1
+ import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback, ToolApprovalDecision } from "@oh-my-pi/pi-agent-core";
2
+ import type { ComputerAction } from "@oh-my-pi/pi-ai";
3
+ import type { DesktopCapabilities, DesktopCapture, DesktopDisplay, DesktopSessionOptions } from "@oh-my-pi/pi-natives";
4
+ import { type ComputerController } from "./computer/supervisor.js";
5
+ import type { ToolSession } from "./index.js";
6
+ declare const computerSchema: import("arktype/internal/variants/object.ts").ObjectType<{
7
+ actions?: {
8
+ type: "click" | "double_click" | "drag" | "keypress" | "move" | "screenshot" | "scroll" | "type" | "wait";
9
+ x?: number | undefined;
10
+ y?: number | undefined;
11
+ button?: "back" | "forward" | "left" | "right" | "wheel" | undefined;
12
+ path?: {
13
+ x: number;
14
+ y: number;
15
+ }[] | undefined;
16
+ keys?: string[] | null | undefined;
17
+ scroll_x?: number | undefined;
18
+ scroll_y?: number | undefined;
19
+ text?: string | undefined;
20
+ }[] | undefined;
21
+ }, {}>;
22
+ export type ComputerParams = typeof computerSchema.infer;
23
+ export interface ComputerToolDetails {
24
+ width: number;
25
+ height: number;
26
+ backend: DesktopCapture["backend"];
27
+ displayServer?: string;
28
+ capturePermission: string;
29
+ inputPermission: string;
30
+ displays: DesktopDisplay[];
31
+ capabilities?: DesktopCapabilities;
32
+ actions: ComputerAction["type"][];
33
+ }
34
+ export type ComputerControllerFactory = (options: DesktopSessionOptions) => ComputerController;
35
+ export declare function computerApproval(args: unknown): ToolApprovalDecision;
36
+ export declare class ComputerTool implements AgentTool<typeof computerSchema, ComputerToolDetails> {
37
+ #private;
38
+ readonly session: ToolSession;
39
+ readonly name = "computer";
40
+ readonly native: {
41
+ readonly type: "computer";
42
+ };
43
+ readonly label = "Computer";
44
+ readonly loadMode: "essential";
45
+ readonly concurrency: "exclusive";
46
+ readonly summary = "Capture and control the host desktop through native OS APIs";
47
+ readonly parameters: import("arktype/internal/variants/object.ts").ObjectType<{
48
+ actions?: {
49
+ type: "click" | "double_click" | "drag" | "keypress" | "move" | "screenshot" | "scroll" | "type" | "wait";
50
+ x?: number | undefined;
51
+ y?: number | undefined;
52
+ button?: "back" | "forward" | "left" | "right" | "wheel" | undefined;
53
+ path?: {
54
+ x: number;
55
+ y: number;
56
+ }[] | undefined;
57
+ keys?: string[] | null | undefined;
58
+ scroll_x?: number | undefined;
59
+ scroll_y?: number | undefined;
60
+ text?: string | undefined;
61
+ }[] | undefined;
62
+ }, {}>;
63
+ readonly strict = false;
64
+ readonly approval: typeof computerApproval;
65
+ readonly formatApprovalDetails: (args: unknown) => string[];
66
+ constructor(session: ToolSession, createController?: ComputerControllerFactory);
67
+ get description(): string;
68
+ execute(_toolCallId: string, params: ComputerParams, signal?: AbortSignal, _onUpdate?: AgentToolUpdateCallback<ComputerToolDetails>, context?: AgentToolContext): Promise<AgentToolResult<ComputerToolDetails>>;
69
+ close(): Promise<void>;
70
+ }
71
+ export {};
@@ -12,6 +12,8 @@ declare module "@oh-my-pi/pi-agent-core" {
12
12
  * wrapper must not re-prompt for the same action (explicit per-tool
13
13
  * policies and overrides still apply). */
14
14
  xdevApproved?: boolean;
15
+ /** Set only after an interactive prompt approves provider computer safety checks. */
16
+ providerSafetyApproved?: boolean;
15
17
  }
16
18
  }
17
19
  export declare class ToolContextStore {
@@ -0,0 +1,21 @@
1
+ import type { Component } from "@oh-my-pi/pi-tui";
2
+ import type { RenderResultOptions } from "../extensibility/custom-tools/types.js";
3
+ import type { Theme } from "../modes/theme/theme.js";
4
+ /** Inputs rendered by the fallback card used when a tool has no bespoke renderer. */
5
+ export interface DefaultToolRenderInput {
6
+ /** Human-readable tool label. */
7
+ label: string;
8
+ /** Tool arguments, shown inline when collapsed and as a tree when expanded. */
9
+ args: unknown;
10
+ /** Settled or streaming result; omitted while only the call is available. */
11
+ result?: {
12
+ output: string;
13
+ isError?: boolean;
14
+ };
15
+ /** Current expansion and lifecycle state. */
16
+ options: RenderResultOptions;
17
+ }
18
+ /** Format one generic tool call/result card at the available content width. */
19
+ export declare function formatDefaultToolExecution(input: DefaultToolRenderInput, contentWidth: number, uiTheme: Theme): string;
20
+ /** Render the generic fallback as the state-tinted card used by direct custom tools. */
21
+ export declare function renderDefaultToolExecution(input: DefaultToolRenderInput, uiTheme: Theme): Component;
@@ -42,6 +42,8 @@ export * from "./ast-grep.js";
42
42
  export * from "./bash.js";
43
43
  export * from "./browser.js";
44
44
  export * from "./checkpoint.js";
45
+ export * from "./computer.js";
46
+ export * from "./computer/supervisor.js";
45
47
  export * from "./debug.js";
46
48
  export * from "./essential-tools.js";
47
49
  export * from "./eval.js";
@@ -1,54 +1,21 @@
1
- import { type PlayerCommand } from "./player.js";
2
- /** Output gain applied while ducked (the user is speaking over the assistant). */
1
+ /** Output gain applied while the user speaks over assistant audio. */
3
2
  export declare const DUCK_GAIN = 0.25;
4
- /** Injection seam for {@link streamingPlayerCommandsFor} — defaults to real PATH/tools lookups. */
5
- export interface StreamingPlayerLookup {
6
- which?: (bin: string) => string | null;
7
- ffmpeg?: () => string | null;
8
- }
9
- /**
10
- * Ordered candidate commands for a persistent raw-PCM player on `platform`: each
11
- * reads 32-bit-float little-endian mono PCM at `sampleRate` from stdin (`pipe:0`)
12
- * and plays it to the default output device. An empty list means no streaming
13
- * backend is available and the caller should fall back to per-file playback.
14
- *
15
- * - darwin: `ffmpeg` (AudioToolbox output device) → sox's `play` (coreaudio).
16
- * - linux/other POSIX: `ffmpeg` (`-f pulse` then `-f alsa`) → `paplay`/`aplay`
17
- * raw fallbacks.
18
- * - win32: none (PowerShell `SoundPlayer` is file-only).
19
- */
20
- export declare function streamingPlayerCommandsFor(platform: NodeJS.Platform, sampleRate: number, lookup?: StreamingPlayerLookup): PlayerCommand[];
21
- /**
22
- * Test seams for {@link StreamingAudioPlayer}: override backend discovery and
23
- * the per-file fallback so playback logic can be exercised without a real audio
24
- * device. Both default to the platform lookup and {@link playAudioFile}.
25
- */
26
- export interface StreamingPlayerOptions {
27
- /** Ordered backend commands for a sample rate; defaults to {@link streamingPlayerCommandsFor}. */
28
- commandsFor?: (sampleRate: number) => PlayerCommand[];
29
- /** Per-file fallback playback; defaults to {@link playAudioFile}. */
30
- playAudio?: (wavPath: string, signal: AbortSignal) => Promise<void>;
31
- /** Max seconds of streamed PCM retained for the nonzero-exit replay; defaults to {@link REPLAY_RETENTION_SECONDS}. */
32
- replayRetentionSeconds?: number;
33
- }
34
3
  /**
35
- * Single-session gapless player. Lifecycle: {@link start} once, {@link write}
36
- * chunks in order, then {@link end} to drain or {@link stop} to abort. Not
37
- * reusable after stop/end — create a new instance per utterance.
4
+ * One native gapless playback session. Call {@link start}, enqueue mono `f32`
5
+ * chunks with {@link write}, then {@link end} to drain or {@link stop} to abort.
38
6
  */
39
7
  export declare class StreamingAudioPlayer {
40
8
  #private;
41
- constructor(options?: StreamingPlayerOptions);
42
- /** Pick a backend and begin draining. Idempotent; the first call's rate wins. */
43
- start(sampleRate: number): void;
44
- /** Queue a mono float32 PCM chunk for playback in arrival order. */
9
+ /** Opens the default speaker at the stream's logical sample rate. */
10
+ start(sampleRate?: number): void;
11
+ /** Queues one mono `f32` PCM chunk without copying it in TypeScript. */
45
12
  write(pcm: Float32Array): void;
46
- /** Scale subsequent output (1 = normal, <1 = ducked). Applies within {@link LEAD_SECONDS}. */
13
+ /** Applies gain at render time, including to samples already queued natively. */
47
14
  setGain(gain: number): void;
48
- /** Close the input; resolves once all queued audio has finished playing. */
15
+ /** Closes input and resolves after every queued sample reaches the speaker. */
49
16
  end(): Promise<void>;
50
- /** Stop immediately: kill the player, drop everything still queued. */
17
+ /** Stops immediately and discards queued audio. Safe to call repeatedly. */
51
18
  stop(): void;
52
19
  }
53
- /** Factory the vocalizer calls; a function so tests can stub it without spawning a player. */
20
+ /** Creates the single-use player used by the speech vocalizer. */
54
21
  export declare function createStreamingPlayer(): StreamingAudioPlayer;
@@ -1,5 +1,4 @@
1
- export declare function ffmpegAssetName(_version: string, plat: string, architecture: string): string | null;
2
- export type ToolName = "sd" | "sg" | "yt-dlp" | "trafilatura" | "ffmpeg";
1
+ export type ToolName = "sd" | "sg" | "yt-dlp" | "trafilatura";
3
2
  export declare function getToolPath(tool: ToolName): string | null;
4
3
  /** Download a tool asset without handing the streaming Response to Bun.write. */
5
4
  export declare function downloadFile(url: string, dest: string, signal?: AbortSignal): Promise<void>;
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "type": "module",
3
3
  "name": "@oh-my-pi/pi-coding-agent",
4
- "version": "17.1.0",
4
+ "version": "17.1.1",
5
5
  "description": "Coding agent CLI with read, bash, edit, write tools and session management",
6
6
  "homepage": "https://omp.sh",
7
7
  "author": "Can Boluk",
@@ -53,17 +53,17 @@
53
53
  "@babel/parser": "^7.29.7",
54
54
  "@babel/traverse": "^7.29.7",
55
55
  "@mozilla/readability": "^0.6.0",
56
- "@oh-my-pi/hashline": "17.1.0",
57
- "@oh-my-pi/omp-stats": "17.1.0",
58
- "@oh-my-pi/pi-agent-core": "17.1.0",
59
- "@oh-my-pi/pi-ai": "17.1.0",
60
- "@oh-my-pi/pi-catalog": "17.1.0",
61
- "@oh-my-pi/pi-mnemopi": "17.1.0",
62
- "@oh-my-pi/pi-natives": "17.1.0",
63
- "@oh-my-pi/pi-tui": "17.1.0",
64
- "@oh-my-pi/pi-utils": "17.1.0",
65
- "@oh-my-pi/pi-wire": "17.1.0",
66
- "@oh-my-pi/snapcompact": "17.1.0",
56
+ "@oh-my-pi/hashline": "17.1.1",
57
+ "@oh-my-pi/omp-stats": "17.1.1",
58
+ "@oh-my-pi/pi-agent-core": "17.1.1",
59
+ "@oh-my-pi/pi-ai": "17.1.1",
60
+ "@oh-my-pi/pi-catalog": "17.1.1",
61
+ "@oh-my-pi/pi-mnemopi": "17.1.1",
62
+ "@oh-my-pi/pi-natives": "17.1.1",
63
+ "@oh-my-pi/pi-tui": "17.1.1",
64
+ "@oh-my-pi/pi-utils": "17.1.1",
65
+ "@oh-my-pi/pi-wire": "17.1.1",
66
+ "@oh-my-pi/snapcompact": "17.1.1",
67
67
  "@opentelemetry/api": "^1.9.1",
68
68
  "@opentelemetry/api-logs": "^0.220.0",
69
69
  "@opentelemetry/context-async-hooks": "^2.9.0",
package/src/cli/args.ts CHANGED
@@ -396,6 +396,7 @@ ${chalk.bold("Available Tools (default-enabled unless noted):")}
396
396
  notebook - Edit Jupyter notebooks
397
397
  inspect_image - Analyze images with a vision model
398
398
  browser - Browser automation (Puppeteer)
399
+ computer - Native host desktop capture and input (disabled by default)
399
400
  task - Launch sub-agents for parallel tasks
400
401
  todo - Manage todo/task lists
401
402
  web_search - Search the web
@@ -11,7 +11,6 @@ import { Settings, settings } from "../config/settings";
11
11
  import { theme } from "../modes/theme/theme";
12
12
  import { downloadSttModel, isSttModelCached } from "../stt/downloader";
13
13
  import { isSttModelKey, STT_MODEL_OPTIONS } from "../stt/models";
14
- import { detectRecorder, ensureRecorder } from "../stt/recorder";
15
14
  import { downloadTtsModel, isTtsLocalModelKey, isTtsModelCached, TTS_LOCAL_MODEL_OPTIONS } from "../tts";
16
15
  import { selectSetupModel } from "./setup-model-picker";
17
16
 
@@ -170,17 +169,6 @@ interface SpeechComponent {
170
169
 
171
170
  function buildSpeechComponents(): SpeechComponent[] {
172
171
  return [
173
- {
174
- name: "Recorder",
175
- isReady: async () => detectRecorder() !== null,
176
- status: async () => {
177
- const recorder = detectRecorder();
178
- return recorder ? `${recorder.tool} (${recorder.bin})` : "none — ffmpeg will be downloaded";
179
- },
180
- ensure: async onProgress => {
181
- await ensureRecorder(onProgress);
182
- },
183
- },
184
172
  {
185
173
  name: "Speech-to-Text model",
186
174
  isReady: () => isSttModelCached(settings.get("stt.modelName")),
@@ -316,7 +304,7 @@ ${chalk.bold("Usage:")}
316
304
 
317
305
  ${chalk.bold("Components:")}
318
306
  python Verify a Python 3 interpreter is reachable for code execution
319
- speech Pick + download the speech-to-text and text-to-speech models and an audio recorder
307
+ speech Pick and download speech-to-text and text-to-speech models
320
308
 
321
309
  ${chalk.bold("Options:")}
322
310
  -c, --check Check if dependencies are installed without installing
@@ -325,7 +313,7 @@ ${chalk.bold("Options:")}
325
313
  ${chalk.bold("Examples:")}
326
314
  ${APP_NAME} setup Run the onboarding wizard
327
315
  ${APP_NAME} setup python Check Python execution dependencies
328
- ${APP_NAME} setup speech Set up speech (pick STT + TTS models, install a recorder)
316
+ ${APP_NAME} setup speech Pick and download the STT and TTS models
329
317
  ${APP_NAME} setup speech --check Check if speech dependencies are available
330
318
  ${APP_NAME} setup python --check Check if Python execution is available
331
319
  `);
package/src/cli.ts CHANGED
@@ -79,6 +79,7 @@ async function runSmokeTest(): Promise<void> {
79
79
  const { smokeTestTtsWorker } = await import("./tts/tts-client");
80
80
  const { smokeTestMnemopiEmbedWorker } = await import("./mnemopi/embed-client");
81
81
  const { smokeTestJsEvalWorker } = await import("./eval/js/context-manager");
82
+ const { smokeTestComputerWorker } = await import("./tools/computer/supervisor");
82
83
  // Smoke dependencies stay lazy so normal CLI startup does not load worker clients.
83
84
  const { smokeTestDaemonBroker } = await import("./launch/client");
84
85
  await smokeTestSyncWorker();
@@ -98,6 +99,7 @@ async function runSmokeTest(): Promise<void> {
98
99
  await smokeTestTinyTitleWorker();
99
100
  await smokeTestSttWorker();
100
101
  await smokeTestJsEvalWorker();
102
+ await smokeTestComputerWorker();
101
103
  await smokeTestTtsWorker();
102
104
  await smokeTestMnemopiEmbedWorker();
103
105
  await smokeTestDaemonBroker();
@@ -107,6 +109,7 @@ async function runSmokeTest(): Promise<void> {
107
109
  const TINY_WORKER_ARG = "__omp_worker_tiny_inference";
108
110
  const STATS_SYNC_WORKER_ARG = "__omp_worker_stats_sync";
109
111
  const TAB_WORKER_ARG = "__omp_worker_tab";
112
+ const COMPUTER_WORKER_ARG = "__omp_worker_computer";
110
113
  const JS_EVAL_WORKER_ARG = "__omp_worker_js_eval";
111
114
  const JS_EVAL_PROCESS_ARG = "__omp_worker_js_eval_process";
112
115
  const STT_WORKER_ARG = "__omp_worker_stt";
@@ -152,6 +155,11 @@ async function runWorkerEntrypoint(arg: string | undefined): Promise<boolean> {
152
155
  await import("./tools/browser/tab-worker-entry");
153
156
  return true;
154
157
  }
158
+ if (arg === COMPUTER_WORKER_ARG) {
159
+ if (parentPort) installWorkerInbox(parentPort);
160
+ await import("./tools/computer/worker-entry");
161
+ return true;
162
+ }
155
163
  if (arg === JS_EVAL_WORKER_ARG) {
156
164
  if (parentPort) installWorkerInbox(parentPort);
157
165
  await import("./eval/js/worker-entry");
@@ -65,6 +65,7 @@ const BUILT_IN_DISCOVERY_NON_AUTHORITATIVE_RETRY_MS = 5 * 60 * 1000;
65
65
  import type { ApiKeyResolver, FetchImpl } from "@oh-my-pi/pi-ai";
66
66
  import { registerOAuthProvider, unregisterOAuthProviders } from "@oh-my-pi/pi-ai/oauth";
67
67
  import type { OAuthCredentials, OAuthLoginCallbacks } from "@oh-my-pi/pi-ai/oauth/types";
68
+ import { setCodexAttestationProvider } from "@oh-my-pi/pi-ai/providers/openai-codex-responses";
68
69
  import {
69
70
  getBundledModelReferenceIndex,
70
71
  inheritReferenceThinking,
@@ -72,6 +73,7 @@ import {
72
73
  } from "@oh-my-pi/pi-catalog/identity";
73
74
  import { isBunTestRuntime, isRecord, logger, wrapFetchForExtraCa } from "@oh-my-pi/pi-utils";
74
75
  import { parseModelString, resolveProviderModelReference } from "../config/model-resolver";
76
+ import { generateCodexAttestation } from "../live/attestation";
75
77
  import type { AuthStorage, OAuthCredential } from "../session/auth-storage";
76
78
  import { type ApiKeyResolverModel, type ApiKeyResolverOptions, createApiKeyResolver } from "./api-key-resolver";
77
79
  import type { ConfigError, ConfigFile } from "./config-file";
@@ -90,6 +92,10 @@ import { ModelsConfigFile, type ProviderValidationModel, validateProviderConfigu
90
92
  import type { ModelOverride, ModelsConfig, ProviderAuthMode } from "./models-config-schema";
91
93
  import { settings } from "./settings";
92
94
 
95
+ // DeviceCheck attestation (`x-oai-attestation`) for ChatGPT-OAuth Codex
96
+ // requests; the pi-ai provider resolves it just-in-time per request.
97
+ setCodexAttestationProvider(generateCodexAttestation);
98
+
93
99
  export const kNoAuth = "N/A";
94
100
 
95
101
  export function isAuthenticated(apiKey: string | undefined | null): apiKey is string {
@@ -141,6 +141,7 @@ export const TAB_GROUPS: Record<SettingTab, readonly string[]> = {
141
141
  "Available Tools",
142
142
  "Todos",
143
143
  "Grep & Browser",
144
+ "Computer",
144
145
  "GitHub",
145
146
  "Output Limits",
146
147
  "Execution",
@@ -3830,6 +3831,66 @@ export const SETTINGS_SCHEMA = {
3830
3831
  },
3831
3832
  },
3832
3833
 
3834
+ "computer.enabled": {
3835
+ type: "boolean",
3836
+ default: false,
3837
+ ui: {
3838
+ tab: "tools",
3839
+ group: "Available Tools",
3840
+ label: "Computer",
3841
+ description: "Enable native host-desktop screenshots and input for OpenAI computer use",
3842
+ },
3843
+ },
3844
+
3845
+ "computer.backend": {
3846
+ type: "enum",
3847
+ values: ["auto", "native"] as const,
3848
+ default: "auto",
3849
+ ui: {
3850
+ tab: "tools",
3851
+ group: "Computer",
3852
+ label: "Computer Backend",
3853
+ description: "Select automatic or explicit platform-native desktop capture and input",
3854
+ options: [
3855
+ { value: "auto", label: "Auto" },
3856
+ { value: "native", label: "Native" },
3857
+ ],
3858
+ },
3859
+ },
3860
+
3861
+ "computer.display": {
3862
+ type: "string",
3863
+ default: "all",
3864
+ ui: {
3865
+ tab: "tools",
3866
+ group: "Computer",
3867
+ label: "Computer Display",
3868
+ description: "Composite all displays or select a native display id",
3869
+ },
3870
+ },
3871
+
3872
+ "computer.maxWidth": {
3873
+ type: "number",
3874
+ default: 1920,
3875
+ ui: {
3876
+ tab: "tools",
3877
+ group: "Computer",
3878
+ label: "Computer Screenshot Width",
3879
+ description: "Maximum composite screenshot width in pixels",
3880
+ },
3881
+ },
3882
+
3883
+ "computer.maxHeight": {
3884
+ type: "number",
3885
+ default: 1200,
3886
+ ui: {
3887
+ tab: "tools",
3888
+ group: "Computer",
3889
+ label: "Computer Screenshot Height",
3890
+ description: "Maximum composite screenshot height in pixels",
3891
+ },
3892
+ },
3893
+
3833
3894
  "checkpoint.enabled": {
3834
3895
  type: "boolean",
3835
3896
  default: false,
@@ -497,6 +497,7 @@ export async function executeWithKernelBase<
497
497
  logger.error(`${errorLogLabel} execution failed`, { error: error.message });
498
498
  throw error;
499
499
  } finally {
500
+ await sink.dispose();
500
501
  unregisterBridge?.();
501
502
  abortShield.dispose?.();
502
503
  }
@@ -169,5 +169,7 @@ export async function executeJs(code: string, options: JsExecutorOptions): Promi
169
169
  outputBytes: summary.outputBytes,
170
170
  displayOutputs,
171
171
  };
172
+ } finally {
173
+ await outputSink.dispose();
172
174
  }
173
175
  }
@@ -581,6 +581,7 @@ export async function executeBash(command: string, options?: BashExecutorOptions
581
581
  resetSession = true;
582
582
  throw err;
583
583
  } finally {
584
+ await sink.dispose();
584
585
  if (timeoutTimer) {
585
586
  clearTimeout(timeoutTimer);
586
587
  }
@@ -8,10 +8,11 @@ import type {
8
8
  AgentToolUpdateCallback,
9
9
  ToolLoadMode,
10
10
  } from "@oh-my-pi/pi-agent-core";
11
- import type { ImageContent, Static, TextContent, TSchema } from "@oh-my-pi/pi-ai";
11
+ import type { ComputerSafetyCheck, ImageContent, Static, TextContent, TSchema } from "@oh-my-pi/pi-ai";
12
+ import { sanitizeText } from "@oh-my-pi/pi-utils";
12
13
  import type { Settings } from "../../config/settings";
13
14
  import type { Theme } from "../../modes/theme/theme";
14
- import { type ApprovalMode, formatApprovalPrompt, resolveApproval } from "../../tools/approval";
15
+ import { type ApprovalMode, formatApprovalPrompt, resolveApproval, truncateForPrompt } from "../../tools/approval";
15
16
  import { defaultLoadModeForToolName } from "../../tools/essential-tools";
16
17
  import { normalizeToolEventInput, resolveToolEventInput } from "../tool-event-input";
17
18
  import { applyToolProxy } from "../tool-proxy";
@@ -83,6 +84,42 @@ export function wrapRegisteredTools(registeredTools: RegisteredTool[], runner: E
83
84
  return registeredTools.map(rt => wrapRegisteredTool(rt, runner));
84
85
  }
85
86
 
87
+ function computerSafetyChecks(context: AgentToolContext | undefined): ComputerSafetyCheck[] {
88
+ const metadata = context?.toolCall?.providerMetadata;
89
+ return metadata?.type === "computer" ? metadata.pendingSafetyChecks : [];
90
+ }
91
+
92
+ function approvalArgs(params: unknown, context: AgentToolContext | undefined): unknown {
93
+ const metadata = context?.toolCall?.providerMetadata;
94
+ return metadata?.type === "computer" ? { actions: metadata.actions } : params;
95
+ }
96
+
97
+ function toolEventArgs(params: unknown, context: AgentToolContext | undefined): Record<string, unknown> {
98
+ const metadata = context?.toolCall?.providerMetadata;
99
+ if (metadata?.type === "computer") {
100
+ return {
101
+ actions: metadata.actions,
102
+ pendingSafetyChecks: metadata.pendingSafetyChecks,
103
+ };
104
+ }
105
+ return params as Record<string, unknown>;
106
+ }
107
+
108
+ function approvalData(value: string): string {
109
+ const sanitized = sanitizeText(value)
110
+ .replace(/[\r\n\t]+/g, " ")
111
+ .trim();
112
+ const truncated = truncateForPrompt(sanitized, 500);
113
+ return truncated.replace(/([\\`*_{}[\]()<>#+\-.!|])/g, "\\$1");
114
+ }
115
+
116
+ function safetyCheckLines(checks: readonly ComputerSafetyCheck[]): string[] {
117
+ return checks.map((check, index) => {
118
+ const value = check.message || check.code || check.id;
119
+ return `${index + 1}. ${approvalData(value)}`;
120
+ });
121
+ }
122
+
86
123
  /**
87
124
  * Wraps a tool with extension callbacks for interception.
88
125
  * - Emits tool_call event before execution (can block)
@@ -128,19 +165,25 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
128
165
  const configuredMode = (settings?.get("tools.approvalMode") ?? "yolo") as ApprovalMode;
129
166
  const approvalMode: ApprovalMode = cliAutoApprove ? "yolo" : configuredMode;
130
167
  const userPolicies = (settings?.get("tools.approval") ?? {}) as Record<string, unknown>;
131
- const resolved = resolveApproval(this.tool, params, approvalMode, userPolicies);
168
+ const resolvedArgs = approvalArgs(params, context);
169
+ const resolved = resolveApproval(this.tool, resolvedArgs, approvalMode, userPolicies);
132
170
  if (resolved.policy === "deny") {
133
171
  throw new Error(
134
172
  `Tool "${this.tool.name}" is blocked by user policy.\n` +
135
173
  `To allow: remove "tools.approval.${this.tool.name}: deny" from config.`,
136
174
  );
137
175
  }
176
+ const pendingSafetyChecks = computerSafetyChecks(context);
138
177
  // An xd:// device dispatch already cleared the write tool's outer gate at
139
178
  // this tool's tier — re-prompting would double-ask for one action. Explicit
140
179
  // per-tool "prompt" policies and tool-demanded overrides still prompt.
180
+ // Provider safety checks are stronger: yolo, per-tool allow, and xdev approval
181
+ // never acknowledge them on the user's behalf.
141
182
  const explicitPrompt = resolved.override || Object.hasOwn(userPolicies, this.tool.name);
142
183
  const approvalCheck = {
143
- required: resolved.policy === "prompt" && (explicitPrompt || context?.xdevApproved !== true),
184
+ required:
185
+ pendingSafetyChecks.length > 0 ||
186
+ (resolved.policy === "prompt" && (explicitPrompt || context?.xdevApproved !== true)),
144
187
  reason: resolved.reason,
145
188
  };
146
189
 
@@ -171,10 +214,16 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
171
214
  });
172
215
  };
173
216
 
174
- // Check if UI is available
217
+ // Provider safety checks fail closed without an interactive prompt. Unlike
218
+ // ordinary tier approval, no setting or yolo mode may bypass this gate.
175
219
  if (!this.runner.hasUI()) {
176
220
  const reason = "no interactive UI available";
177
221
  await resolveApproval(false, reason);
222
+ if (pendingSafetyChecks.length > 0) {
223
+ throw new Error(
224
+ `Tool "${this.tool.name}" has pending provider safety checks but no interactive UI is available.`,
225
+ );
226
+ }
178
227
  throw new Error(
179
228
  `Tool "${this.tool.name}" requires approval but no interactive UI available.\n` +
180
229
  `Options:\n` +
@@ -185,12 +234,14 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
185
234
  }
186
235
 
187
236
  const uiContext = this.runner.getUIContext();
237
+ const basePrompt = formatApprovalPrompt(this.tool, resolvedArgs, approvalCheck.reason);
238
+ const safetyPrompt =
239
+ pendingSafetyChecks.length > 0
240
+ ? `${basePrompt}\nProvider safety checks:\n${safetyCheckLines(pendingSafetyChecks).join("\n")}`
241
+ : basePrompt;
188
242
  let choice: string | undefined;
189
243
  try {
190
- choice = await uiContext.select(formatApprovalPrompt(this.tool, params, approvalCheck.reason), [
191
- "Approve",
192
- "Deny",
193
- ]);
244
+ choice = await uiContext.select(safetyPrompt, ["Approve", "Deny"]);
194
245
  } catch (err) {
195
246
  await resolveApproval(false, err instanceof Error ? err.message : "approval aborted");
196
247
  throw err;
@@ -200,6 +251,10 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
200
251
  if (!approved) {
201
252
  throw new Error(`Tool call denied by user: ${this.tool.name}`);
202
253
  }
254
+ if (pendingSafetyChecks.length > 0) {
255
+ if (!context) throw new Error("Provider safety approval context is unavailable");
256
+ context.providerSafetyApproved = true;
257
+ }
203
258
  }
204
259
 
205
260
  // 2. Emit tool_call event - extensions can block execution
@@ -211,7 +266,7 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
211
266
  toolCallId,
212
267
  input: normalizeToolEventInput(
213
268
  this.tool.name,
214
- resolveToolEventInput(this.tool, params as Record<string, unknown>),
269
+ resolveToolEventInput(this.tool, toolEventArgs(params, context)),
215
270
  ),
216
271
  })) as ToolCallEventResult | undefined;
217
272
 
@@ -228,7 +283,7 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
228
283
  }
229
284
 
230
285
  // Execute the actual tool
231
- let result: { content: any; details?: TDetails };
286
+ let result: AgentToolResult<TDetails, TParameters>;
232
287
  let executionError: Error | undefined;
233
288
 
234
289
  try {
@@ -249,7 +304,7 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
249
304
  toolCallId,
250
305
  input: normalizeToolEventInput(
251
306
  this.tool.name,
252
- resolveToolEventInput(this.tool, params as Record<string, unknown>),
307
+ resolveToolEventInput(this.tool, toolEventArgs(params, context)),
253
308
  ),
254
309
  content: result.content,
255
310
  details: result.details,
@@ -275,6 +330,7 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
275
330
  return {
276
331
  content: modifiedContent,
277
332
  details: modifiedDetails,
333
+ providerMetadata: result.providerMetadata,
278
334
  ...(effectiveError ? { isError: true } : {}),
279
335
  };
280
336
  }
@@ -17,7 +17,7 @@ import * as fs from "node:fs";
17
17
  import * as path from "node:path";
18
18
  import type { AgentToolResult, AgentToolUpdateCallback } from "@oh-my-pi/pi-agent-core";
19
19
  import { type AuthCredential, SqliteAuthCredentialStore, type TSchema } from "@oh-my-pi/pi-ai";
20
- import { Text } from "@oh-my-pi/pi-tui";
20
+ import { getKeybindings, type Keybinding, Text } from "@oh-my-pi/pi-tui";
21
21
  import {
22
22
  getAgentDbPath,
23
23
  getAgentDir,
@@ -26,6 +26,7 @@ import {
26
26
  parseFrontmatter as parseOmpFrontmatter,
27
27
  } from "@oh-my-pi/pi-utils";
28
28
  import { getPackageDir as getOmpPackageDir } from "../config";
29
+ import { formatKeyHints } from "../config/keybindings";
29
30
  import type { PromptTemplate } from "../config/prompt-templates";
30
31
  import { type SettingPath, Settings } from "../config/settings";
31
32
  import { EditTool } from "../edit";
@@ -373,6 +374,11 @@ async function executeLegacyBashOperations(
373
374
  }
374
375
  }
375
376
 
377
+ /** Format the active shortcut for legacy extensions that render keybinding hints. */
378
+ export function keyText(action: Keybinding): string {
379
+ return formatKeyHints(getKeybindings().getKeys(action));
380
+ }
381
+
376
382
  /** Parse frontmatter using the historical Pi package-root helper. */
377
383
  export interface ParsedFrontmatter<T extends Record<string, unknown> = Record<string, unknown>> {
378
384
  frontmatter: T;