@oh-my-pi/pi-coding-agent 17.1.0 → 17.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/cli.js +3686 -4013
- package/dist/types/config/settings-schema.d.ts +58 -0
- package/dist/types/extensibility/legacy-pi-coding-agent-shim.d.ts +3 -0
- package/dist/types/live/attestation.d.ts +2 -0
- package/dist/types/live/controller.d.ts +10 -2
- package/dist/types/live/protocol.d.ts +1 -1
- package/dist/types/live/transport.d.ts +6 -19
- package/dist/types/live/visualizer.d.ts +8 -11
- package/dist/types/modes/components/assistant-message.d.ts +1 -0
- package/dist/types/modes/components/custom-message.d.ts +1 -1
- package/dist/types/modes/components/message-frame.d.ts +8 -4
- package/dist/types/modes/components/session-account-selector.d.ts +11 -0
- package/dist/types/modes/controllers/selector-controller.d.ts +1 -0
- package/dist/types/modes/interactive-mode.d.ts +1 -0
- package/dist/types/modes/types.d.ts +1 -0
- package/dist/types/session/agent-session-types.d.ts +8 -1
- package/dist/types/session/agent-session.d.ts +20 -1
- package/dist/types/session/auth-storage.d.ts +1 -1
- package/dist/types/session/eval-runner.d.ts +2 -0
- package/dist/types/session/messages.d.ts +2 -0
- package/dist/types/session/session-tools.d.ts +15 -0
- package/dist/types/session/streaming-output.d.ts +8 -0
- package/dist/types/slash-commands/helpers/session-pin.d.ts +9 -0
- package/dist/types/stt/index.d.ts +0 -2
- package/dist/types/stt/stt-controller.d.ts +7 -0
- package/dist/types/tiny/title-client.d.ts +10 -0
- package/dist/types/tools/builtin-names.d.ts +1 -1
- package/dist/types/tools/computer/protocol.d.ts +43 -0
- package/dist/types/tools/computer/supervisor.d.ts +32 -0
- package/dist/types/tools/computer/worker-entry.d.ts +1 -0
- package/dist/types/tools/computer/worker.d.ts +15 -0
- package/dist/types/tools/computer-renderer.d.ts +22 -0
- package/dist/types/tools/computer.d.ts +71 -0
- package/dist/types/tools/context.d.ts +2 -0
- package/dist/types/tools/default-renderer.d.ts +21 -0
- package/dist/types/tools/index.d.ts +2 -0
- package/dist/types/tts/streaming-player.d.ts +10 -43
- package/dist/types/utils/tools-manager.d.ts +1 -2
- package/package.json +12 -12
- package/src/cli/args.ts +1 -0
- package/src/cli/setup-cli.ts +2 -14
- package/src/cli.ts +8 -0
- package/src/config/model-registry.ts +6 -0
- package/src/config/settings-schema.ts +61 -0
- package/src/eval/executor-base.ts +1 -0
- package/src/eval/js/executor.ts +2 -0
- package/src/exec/bash-executor.ts +1 -0
- package/src/extensibility/extensions/wrapper.ts +68 -12
- package/src/extensibility/legacy-pi-coding-agent-shim.ts +7 -1
- package/src/live/attestation.ts +91 -0
- package/src/live/controller.ts +76 -23
- package/src/live/protocol.test.ts +3 -3
- package/src/live/protocol.ts +1 -1
- package/src/live/transport.ts +72 -140
- package/src/live/visualizer.ts +114 -134
- package/src/modes/components/assistant-message.ts +7 -2
- package/src/modes/components/custom-message.ts +4 -1
- package/src/modes/components/message-frame.ts +14 -8
- package/src/modes/components/session-account-selector.ts +62 -0
- package/src/modes/components/tool-execution.ts +17 -110
- package/src/modes/controllers/input-controller.ts +47 -39
- package/src/modes/controllers/live-command-controller.ts +82 -5
- package/src/modes/controllers/selector-controller.ts +62 -0
- package/src/modes/interactive-mode.ts +19 -0
- package/src/modes/types.ts +1 -0
- package/src/prompts/system/computer-safety.md +14 -0
- package/src/prompts/tools/computer.md +26 -0
- package/src/sdk.ts +5 -0
- package/src/session/agent-session-types.ts +9 -0
- package/src/session/agent-session.ts +56 -0
- package/src/session/auth-storage.ts +1 -0
- package/src/session/eval-runner.ts +5 -0
- package/src/session/messages.ts +3 -0
- package/src/session/session-tools.ts +37 -0
- package/src/session/streaming-output.ts +52 -5
- package/src/slash-commands/builtin-registry.ts +165 -9
- package/src/slash-commands/helpers/session-pin.ts +44 -0
- package/src/stt/downloader.ts +0 -2
- package/src/stt/index.ts +0 -2
- package/src/stt/stt-controller.ts +57 -146
- package/src/system-prompt.ts +4 -0
- package/src/tiny/title-client.ts +22 -0
- package/src/tools/bash-interactive.ts +90 -86
- package/src/tools/builtin-names.ts +1 -0
- package/src/tools/computer/protocol.ts +28 -0
- package/src/tools/computer/supervisor.ts +258 -0
- package/src/tools/computer/worker-entry.ts +25 -0
- package/src/tools/computer/worker.ts +135 -0
- package/src/tools/computer-renderer.ts +108 -0
- package/src/tools/computer.ts +433 -0
- package/src/tools/context.ts +2 -0
- package/src/tools/default-renderer.ts +139 -0
- package/src/tools/essential-tools.ts +1 -0
- package/src/tools/index.ts +5 -0
- package/src/tools/renderers.ts +2 -0
- package/src/tools/xdev.ts +54 -26
- package/src/tts/streaming-player.ts +81 -340
- package/src/utils/clipboard.ts +1 -30
- package/src/utils/mac-file-urls.applescript +37 -0
- package/src/utils/tool-choice.ts +14 -0
- package/src/utils/tools-manager.ts +1 -19
- package/dist/types/stt/recorder.d.ts +0 -30
- package/dist/types/stt/transcriber.d.ts +0 -14
- package/dist/types/stt/wav.d.ts +0 -29
- package/dist/types/tts/player.d.ts +0 -32
- package/src/live/audio-worklet.txt +0 -59
- package/src/live/browser-runtime.txt +0 -221
- package/src/stt/recorder.ts +0 -551
- package/src/stt/transcriber.ts +0 -60
- package/src/stt/wav.ts +0 -173
- package/src/tts/player.ts +0 -137
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback, ToolApprovalDecision } from "@oh-my-pi/pi-agent-core";
|
|
2
|
+
import type { ComputerAction } from "@oh-my-pi/pi-ai";
|
|
3
|
+
import type { DesktopCapabilities, DesktopCapture, DesktopDisplay, DesktopSessionOptions } from "@oh-my-pi/pi-natives";
|
|
4
|
+
import { type ComputerController } from "./computer/supervisor.js";
|
|
5
|
+
import type { ToolSession } from "./index.js";
|
|
6
|
+
declare const computerSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
|
7
|
+
actions?: {
|
|
8
|
+
type: "click" | "double_click" | "drag" | "keypress" | "move" | "screenshot" | "scroll" | "type" | "wait";
|
|
9
|
+
x?: number | undefined;
|
|
10
|
+
y?: number | undefined;
|
|
11
|
+
button?: "back" | "forward" | "left" | "right" | "wheel" | undefined;
|
|
12
|
+
path?: {
|
|
13
|
+
x: number;
|
|
14
|
+
y: number;
|
|
15
|
+
}[] | undefined;
|
|
16
|
+
keys?: string[] | null | undefined;
|
|
17
|
+
scroll_x?: number | undefined;
|
|
18
|
+
scroll_y?: number | undefined;
|
|
19
|
+
text?: string | undefined;
|
|
20
|
+
}[] | undefined;
|
|
21
|
+
}, {}>;
|
|
22
|
+
export type ComputerParams = typeof computerSchema.infer;
|
|
23
|
+
export interface ComputerToolDetails {
|
|
24
|
+
width: number;
|
|
25
|
+
height: number;
|
|
26
|
+
backend: DesktopCapture["backend"];
|
|
27
|
+
displayServer?: string;
|
|
28
|
+
capturePermission: string;
|
|
29
|
+
inputPermission: string;
|
|
30
|
+
displays: DesktopDisplay[];
|
|
31
|
+
capabilities?: DesktopCapabilities;
|
|
32
|
+
actions: ComputerAction["type"][];
|
|
33
|
+
}
|
|
34
|
+
export type ComputerControllerFactory = (options: DesktopSessionOptions) => ComputerController;
|
|
35
|
+
export declare function computerApproval(args: unknown): ToolApprovalDecision;
|
|
36
|
+
export declare class ComputerTool implements AgentTool<typeof computerSchema, ComputerToolDetails> {
|
|
37
|
+
#private;
|
|
38
|
+
readonly session: ToolSession;
|
|
39
|
+
readonly name = "computer";
|
|
40
|
+
readonly native: {
|
|
41
|
+
readonly type: "computer";
|
|
42
|
+
};
|
|
43
|
+
readonly label = "Computer";
|
|
44
|
+
readonly loadMode: "essential";
|
|
45
|
+
readonly concurrency: "exclusive";
|
|
46
|
+
readonly summary = "Capture and control the host desktop through native OS APIs";
|
|
47
|
+
readonly parameters: import("arktype/internal/variants/object.ts").ObjectType<{
|
|
48
|
+
actions?: {
|
|
49
|
+
type: "click" | "double_click" | "drag" | "keypress" | "move" | "screenshot" | "scroll" | "type" | "wait";
|
|
50
|
+
x?: number | undefined;
|
|
51
|
+
y?: number | undefined;
|
|
52
|
+
button?: "back" | "forward" | "left" | "right" | "wheel" | undefined;
|
|
53
|
+
path?: {
|
|
54
|
+
x: number;
|
|
55
|
+
y: number;
|
|
56
|
+
}[] | undefined;
|
|
57
|
+
keys?: string[] | null | undefined;
|
|
58
|
+
scroll_x?: number | undefined;
|
|
59
|
+
scroll_y?: number | undefined;
|
|
60
|
+
text?: string | undefined;
|
|
61
|
+
}[] | undefined;
|
|
62
|
+
}, {}>;
|
|
63
|
+
readonly strict = false;
|
|
64
|
+
readonly approval: typeof computerApproval;
|
|
65
|
+
readonly formatApprovalDetails: (args: unknown) => string[];
|
|
66
|
+
constructor(session: ToolSession, createController?: ComputerControllerFactory);
|
|
67
|
+
get description(): string;
|
|
68
|
+
execute(_toolCallId: string, params: ComputerParams, signal?: AbortSignal, _onUpdate?: AgentToolUpdateCallback<ComputerToolDetails>, context?: AgentToolContext): Promise<AgentToolResult<ComputerToolDetails>>;
|
|
69
|
+
close(): Promise<void>;
|
|
70
|
+
}
|
|
71
|
+
export {};
|
|
@@ -12,6 +12,8 @@ declare module "@oh-my-pi/pi-agent-core" {
|
|
|
12
12
|
* wrapper must not re-prompt for the same action (explicit per-tool
|
|
13
13
|
* policies and overrides still apply). */
|
|
14
14
|
xdevApproved?: boolean;
|
|
15
|
+
/** Set only after an interactive prompt approves provider computer safety checks. */
|
|
16
|
+
providerSafetyApproved?: boolean;
|
|
15
17
|
}
|
|
16
18
|
}
|
|
17
19
|
export declare class ToolContextStore {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { Component } from "@oh-my-pi/pi-tui";
|
|
2
|
+
import type { RenderResultOptions } from "../extensibility/custom-tools/types.js";
|
|
3
|
+
import type { Theme } from "../modes/theme/theme.js";
|
|
4
|
+
/** Inputs rendered by the fallback card used when a tool has no bespoke renderer. */
|
|
5
|
+
export interface DefaultToolRenderInput {
|
|
6
|
+
/** Human-readable tool label. */
|
|
7
|
+
label: string;
|
|
8
|
+
/** Tool arguments, shown inline when collapsed and as a tree when expanded. */
|
|
9
|
+
args: unknown;
|
|
10
|
+
/** Settled or streaming result; omitted while only the call is available. */
|
|
11
|
+
result?: {
|
|
12
|
+
output: string;
|
|
13
|
+
isError?: boolean;
|
|
14
|
+
};
|
|
15
|
+
/** Current expansion and lifecycle state. */
|
|
16
|
+
options: RenderResultOptions;
|
|
17
|
+
}
|
|
18
|
+
/** Format one generic tool call/result card at the available content width. */
|
|
19
|
+
export declare function formatDefaultToolExecution(input: DefaultToolRenderInput, contentWidth: number, uiTheme: Theme): string;
|
|
20
|
+
/** Render the generic fallback as the state-tinted card used by direct custom tools. */
|
|
21
|
+
export declare function renderDefaultToolExecution(input: DefaultToolRenderInput, uiTheme: Theme): Component;
|
|
@@ -42,6 +42,8 @@ export * from "./ast-grep.js";
|
|
|
42
42
|
export * from "./bash.js";
|
|
43
43
|
export * from "./browser.js";
|
|
44
44
|
export * from "./checkpoint.js";
|
|
45
|
+
export * from "./computer.js";
|
|
46
|
+
export * from "./computer/supervisor.js";
|
|
45
47
|
export * from "./debug.js";
|
|
46
48
|
export * from "./essential-tools.js";
|
|
47
49
|
export * from "./eval.js";
|
|
@@ -1,54 +1,21 @@
|
|
|
1
|
-
|
|
2
|
-
/** Output gain applied while ducked (the user is speaking over the assistant). */
|
|
1
|
+
/** Output gain applied while the user speaks over assistant audio. */
|
|
3
2
|
export declare const DUCK_GAIN = 0.25;
|
|
4
|
-
/** Injection seam for {@link streamingPlayerCommandsFor} — defaults to real PATH/tools lookups. */
|
|
5
|
-
export interface StreamingPlayerLookup {
|
|
6
|
-
which?: (bin: string) => string | null;
|
|
7
|
-
ffmpeg?: () => string | null;
|
|
8
|
-
}
|
|
9
|
-
/**
|
|
10
|
-
* Ordered candidate commands for a persistent raw-PCM player on `platform`: each
|
|
11
|
-
* reads 32-bit-float little-endian mono PCM at `sampleRate` from stdin (`pipe:0`)
|
|
12
|
-
* and plays it to the default output device. An empty list means no streaming
|
|
13
|
-
* backend is available and the caller should fall back to per-file playback.
|
|
14
|
-
*
|
|
15
|
-
* - darwin: `ffmpeg` (AudioToolbox output device) → sox's `play` (coreaudio).
|
|
16
|
-
* - linux/other POSIX: `ffmpeg` (`-f pulse` then `-f alsa`) → `paplay`/`aplay`
|
|
17
|
-
* raw fallbacks.
|
|
18
|
-
* - win32: none (PowerShell `SoundPlayer` is file-only).
|
|
19
|
-
*/
|
|
20
|
-
export declare function streamingPlayerCommandsFor(platform: NodeJS.Platform, sampleRate: number, lookup?: StreamingPlayerLookup): PlayerCommand[];
|
|
21
|
-
/**
|
|
22
|
-
* Test seams for {@link StreamingAudioPlayer}: override backend discovery and
|
|
23
|
-
* the per-file fallback so playback logic can be exercised without a real audio
|
|
24
|
-
* device. Both default to the platform lookup and {@link playAudioFile}.
|
|
25
|
-
*/
|
|
26
|
-
export interface StreamingPlayerOptions {
|
|
27
|
-
/** Ordered backend commands for a sample rate; defaults to {@link streamingPlayerCommandsFor}. */
|
|
28
|
-
commandsFor?: (sampleRate: number) => PlayerCommand[];
|
|
29
|
-
/** Per-file fallback playback; defaults to {@link playAudioFile}. */
|
|
30
|
-
playAudio?: (wavPath: string, signal: AbortSignal) => Promise<void>;
|
|
31
|
-
/** Max seconds of streamed PCM retained for the nonzero-exit replay; defaults to {@link REPLAY_RETENTION_SECONDS}. */
|
|
32
|
-
replayRetentionSeconds?: number;
|
|
33
|
-
}
|
|
34
3
|
/**
|
|
35
|
-
*
|
|
36
|
-
* chunks
|
|
37
|
-
* reusable after stop/end — create a new instance per utterance.
|
|
4
|
+
* One native gapless playback session. Call {@link start}, enqueue mono `f32`
|
|
5
|
+
* chunks with {@link write}, then {@link end} to drain or {@link stop} to abort.
|
|
38
6
|
*/
|
|
39
7
|
export declare class StreamingAudioPlayer {
|
|
40
8
|
#private;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
/** Queue a mono float32 PCM chunk for playback in arrival order. */
|
|
9
|
+
/** Opens the default speaker at the stream's logical sample rate. */
|
|
10
|
+
start(sampleRate?: number): void;
|
|
11
|
+
/** Queues one mono `f32` PCM chunk without copying it in TypeScript. */
|
|
45
12
|
write(pcm: Float32Array): void;
|
|
46
|
-
/**
|
|
13
|
+
/** Applies gain at render time, including to samples already queued natively. */
|
|
47
14
|
setGain(gain: number): void;
|
|
48
|
-
/**
|
|
15
|
+
/** Closes input and resolves after every queued sample reaches the speaker. */
|
|
49
16
|
end(): Promise<void>;
|
|
50
|
-
/**
|
|
17
|
+
/** Stops immediately and discards queued audio. Safe to call repeatedly. */
|
|
51
18
|
stop(): void;
|
|
52
19
|
}
|
|
53
|
-
/**
|
|
20
|
+
/** Creates the single-use player used by the speech vocalizer. */
|
|
54
21
|
export declare function createStreamingPlayer(): StreamingAudioPlayer;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
export
|
|
2
|
-
export type ToolName = "sd" | "sg" | "yt-dlp" | "trafilatura" | "ffmpeg";
|
|
1
|
+
export type ToolName = "sd" | "sg" | "yt-dlp" | "trafilatura";
|
|
3
2
|
export declare function getToolPath(tool: ToolName): string | null;
|
|
4
3
|
/** Download a tool asset without handing the streaming Response to Bun.write. */
|
|
5
4
|
export declare function downloadFile(url: string, dest: string, signal?: AbortSignal): Promise<void>;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@oh-my-pi/pi-coding-agent",
|
|
4
|
-
"version": "17.1.
|
|
4
|
+
"version": "17.1.1",
|
|
5
5
|
"description": "Coding agent CLI with read, bash, edit, write tools and session management",
|
|
6
6
|
"homepage": "https://omp.sh",
|
|
7
7
|
"author": "Can Boluk",
|
|
@@ -53,17 +53,17 @@
|
|
|
53
53
|
"@babel/parser": "^7.29.7",
|
|
54
54
|
"@babel/traverse": "^7.29.7",
|
|
55
55
|
"@mozilla/readability": "^0.6.0",
|
|
56
|
-
"@oh-my-pi/hashline": "17.1.
|
|
57
|
-
"@oh-my-pi/omp-stats": "17.1.
|
|
58
|
-
"@oh-my-pi/pi-agent-core": "17.1.
|
|
59
|
-
"@oh-my-pi/pi-ai": "17.1.
|
|
60
|
-
"@oh-my-pi/pi-catalog": "17.1.
|
|
61
|
-
"@oh-my-pi/pi-mnemopi": "17.1.
|
|
62
|
-
"@oh-my-pi/pi-natives": "17.1.
|
|
63
|
-
"@oh-my-pi/pi-tui": "17.1.
|
|
64
|
-
"@oh-my-pi/pi-utils": "17.1.
|
|
65
|
-
"@oh-my-pi/pi-wire": "17.1.
|
|
66
|
-
"@oh-my-pi/snapcompact": "17.1.
|
|
56
|
+
"@oh-my-pi/hashline": "17.1.1",
|
|
57
|
+
"@oh-my-pi/omp-stats": "17.1.1",
|
|
58
|
+
"@oh-my-pi/pi-agent-core": "17.1.1",
|
|
59
|
+
"@oh-my-pi/pi-ai": "17.1.1",
|
|
60
|
+
"@oh-my-pi/pi-catalog": "17.1.1",
|
|
61
|
+
"@oh-my-pi/pi-mnemopi": "17.1.1",
|
|
62
|
+
"@oh-my-pi/pi-natives": "17.1.1",
|
|
63
|
+
"@oh-my-pi/pi-tui": "17.1.1",
|
|
64
|
+
"@oh-my-pi/pi-utils": "17.1.1",
|
|
65
|
+
"@oh-my-pi/pi-wire": "17.1.1",
|
|
66
|
+
"@oh-my-pi/snapcompact": "17.1.1",
|
|
67
67
|
"@opentelemetry/api": "^1.9.1",
|
|
68
68
|
"@opentelemetry/api-logs": "^0.220.0",
|
|
69
69
|
"@opentelemetry/context-async-hooks": "^2.9.0",
|
package/src/cli/args.ts
CHANGED
|
@@ -396,6 +396,7 @@ ${chalk.bold("Available Tools (default-enabled unless noted):")}
|
|
|
396
396
|
notebook - Edit Jupyter notebooks
|
|
397
397
|
inspect_image - Analyze images with a vision model
|
|
398
398
|
browser - Browser automation (Puppeteer)
|
|
399
|
+
computer - Native host desktop capture and input (disabled by default)
|
|
399
400
|
task - Launch sub-agents for parallel tasks
|
|
400
401
|
todo - Manage todo/task lists
|
|
401
402
|
web_search - Search the web
|
package/src/cli/setup-cli.ts
CHANGED
|
@@ -11,7 +11,6 @@ import { Settings, settings } from "../config/settings";
|
|
|
11
11
|
import { theme } from "../modes/theme/theme";
|
|
12
12
|
import { downloadSttModel, isSttModelCached } from "../stt/downloader";
|
|
13
13
|
import { isSttModelKey, STT_MODEL_OPTIONS } from "../stt/models";
|
|
14
|
-
import { detectRecorder, ensureRecorder } from "../stt/recorder";
|
|
15
14
|
import { downloadTtsModel, isTtsLocalModelKey, isTtsModelCached, TTS_LOCAL_MODEL_OPTIONS } from "../tts";
|
|
16
15
|
import { selectSetupModel } from "./setup-model-picker";
|
|
17
16
|
|
|
@@ -170,17 +169,6 @@ interface SpeechComponent {
|
|
|
170
169
|
|
|
171
170
|
function buildSpeechComponents(): SpeechComponent[] {
|
|
172
171
|
return [
|
|
173
|
-
{
|
|
174
|
-
name: "Recorder",
|
|
175
|
-
isReady: async () => detectRecorder() !== null,
|
|
176
|
-
status: async () => {
|
|
177
|
-
const recorder = detectRecorder();
|
|
178
|
-
return recorder ? `${recorder.tool} (${recorder.bin})` : "none — ffmpeg will be downloaded";
|
|
179
|
-
},
|
|
180
|
-
ensure: async onProgress => {
|
|
181
|
-
await ensureRecorder(onProgress);
|
|
182
|
-
},
|
|
183
|
-
},
|
|
184
172
|
{
|
|
185
173
|
name: "Speech-to-Text model",
|
|
186
174
|
isReady: () => isSttModelCached(settings.get("stt.modelName")),
|
|
@@ -316,7 +304,7 @@ ${chalk.bold("Usage:")}
|
|
|
316
304
|
|
|
317
305
|
${chalk.bold("Components:")}
|
|
318
306
|
python Verify a Python 3 interpreter is reachable for code execution
|
|
319
|
-
speech Pick
|
|
307
|
+
speech Pick and download speech-to-text and text-to-speech models
|
|
320
308
|
|
|
321
309
|
${chalk.bold("Options:")}
|
|
322
310
|
-c, --check Check if dependencies are installed without installing
|
|
@@ -325,7 +313,7 @@ ${chalk.bold("Options:")}
|
|
|
325
313
|
${chalk.bold("Examples:")}
|
|
326
314
|
${APP_NAME} setup Run the onboarding wizard
|
|
327
315
|
${APP_NAME} setup python Check Python execution dependencies
|
|
328
|
-
${APP_NAME} setup speech
|
|
316
|
+
${APP_NAME} setup speech Pick and download the STT and TTS models
|
|
329
317
|
${APP_NAME} setup speech --check Check if speech dependencies are available
|
|
330
318
|
${APP_NAME} setup python --check Check if Python execution is available
|
|
331
319
|
`);
|
package/src/cli.ts
CHANGED
|
@@ -79,6 +79,7 @@ async function runSmokeTest(): Promise<void> {
|
|
|
79
79
|
const { smokeTestTtsWorker } = await import("./tts/tts-client");
|
|
80
80
|
const { smokeTestMnemopiEmbedWorker } = await import("./mnemopi/embed-client");
|
|
81
81
|
const { smokeTestJsEvalWorker } = await import("./eval/js/context-manager");
|
|
82
|
+
const { smokeTestComputerWorker } = await import("./tools/computer/supervisor");
|
|
82
83
|
// Smoke dependencies stay lazy so normal CLI startup does not load worker clients.
|
|
83
84
|
const { smokeTestDaemonBroker } = await import("./launch/client");
|
|
84
85
|
await smokeTestSyncWorker();
|
|
@@ -98,6 +99,7 @@ async function runSmokeTest(): Promise<void> {
|
|
|
98
99
|
await smokeTestTinyTitleWorker();
|
|
99
100
|
await smokeTestSttWorker();
|
|
100
101
|
await smokeTestJsEvalWorker();
|
|
102
|
+
await smokeTestComputerWorker();
|
|
101
103
|
await smokeTestTtsWorker();
|
|
102
104
|
await smokeTestMnemopiEmbedWorker();
|
|
103
105
|
await smokeTestDaemonBroker();
|
|
@@ -107,6 +109,7 @@ async function runSmokeTest(): Promise<void> {
|
|
|
107
109
|
const TINY_WORKER_ARG = "__omp_worker_tiny_inference";
|
|
108
110
|
const STATS_SYNC_WORKER_ARG = "__omp_worker_stats_sync";
|
|
109
111
|
const TAB_WORKER_ARG = "__omp_worker_tab";
|
|
112
|
+
const COMPUTER_WORKER_ARG = "__omp_worker_computer";
|
|
110
113
|
const JS_EVAL_WORKER_ARG = "__omp_worker_js_eval";
|
|
111
114
|
const JS_EVAL_PROCESS_ARG = "__omp_worker_js_eval_process";
|
|
112
115
|
const STT_WORKER_ARG = "__omp_worker_stt";
|
|
@@ -152,6 +155,11 @@ async function runWorkerEntrypoint(arg: string | undefined): Promise<boolean> {
|
|
|
152
155
|
await import("./tools/browser/tab-worker-entry");
|
|
153
156
|
return true;
|
|
154
157
|
}
|
|
158
|
+
if (arg === COMPUTER_WORKER_ARG) {
|
|
159
|
+
if (parentPort) installWorkerInbox(parentPort);
|
|
160
|
+
await import("./tools/computer/worker-entry");
|
|
161
|
+
return true;
|
|
162
|
+
}
|
|
155
163
|
if (arg === JS_EVAL_WORKER_ARG) {
|
|
156
164
|
if (parentPort) installWorkerInbox(parentPort);
|
|
157
165
|
await import("./eval/js/worker-entry");
|
|
@@ -65,6 +65,7 @@ const BUILT_IN_DISCOVERY_NON_AUTHORITATIVE_RETRY_MS = 5 * 60 * 1000;
|
|
|
65
65
|
import type { ApiKeyResolver, FetchImpl } from "@oh-my-pi/pi-ai";
|
|
66
66
|
import { registerOAuthProvider, unregisterOAuthProviders } from "@oh-my-pi/pi-ai/oauth";
|
|
67
67
|
import type { OAuthCredentials, OAuthLoginCallbacks } from "@oh-my-pi/pi-ai/oauth/types";
|
|
68
|
+
import { setCodexAttestationProvider } from "@oh-my-pi/pi-ai/providers/openai-codex-responses";
|
|
68
69
|
import {
|
|
69
70
|
getBundledModelReferenceIndex,
|
|
70
71
|
inheritReferenceThinking,
|
|
@@ -72,6 +73,7 @@ import {
|
|
|
72
73
|
} from "@oh-my-pi/pi-catalog/identity";
|
|
73
74
|
import { isBunTestRuntime, isRecord, logger, wrapFetchForExtraCa } from "@oh-my-pi/pi-utils";
|
|
74
75
|
import { parseModelString, resolveProviderModelReference } from "../config/model-resolver";
|
|
76
|
+
import { generateCodexAttestation } from "../live/attestation";
|
|
75
77
|
import type { AuthStorage, OAuthCredential } from "../session/auth-storage";
|
|
76
78
|
import { type ApiKeyResolverModel, type ApiKeyResolverOptions, createApiKeyResolver } from "./api-key-resolver";
|
|
77
79
|
import type { ConfigError, ConfigFile } from "./config-file";
|
|
@@ -90,6 +92,10 @@ import { ModelsConfigFile, type ProviderValidationModel, validateProviderConfigu
|
|
|
90
92
|
import type { ModelOverride, ModelsConfig, ProviderAuthMode } from "./models-config-schema";
|
|
91
93
|
import { settings } from "./settings";
|
|
92
94
|
|
|
95
|
+
// DeviceCheck attestation (`x-oai-attestation`) for ChatGPT-OAuth Codex
|
|
96
|
+
// requests; the pi-ai provider resolves it just-in-time per request.
|
|
97
|
+
setCodexAttestationProvider(generateCodexAttestation);
|
|
98
|
+
|
|
93
99
|
export const kNoAuth = "N/A";
|
|
94
100
|
|
|
95
101
|
export function isAuthenticated(apiKey: string | undefined | null): apiKey is string {
|
|
@@ -141,6 +141,7 @@ export const TAB_GROUPS: Record<SettingTab, readonly string[]> = {
|
|
|
141
141
|
"Available Tools",
|
|
142
142
|
"Todos",
|
|
143
143
|
"Grep & Browser",
|
|
144
|
+
"Computer",
|
|
144
145
|
"GitHub",
|
|
145
146
|
"Output Limits",
|
|
146
147
|
"Execution",
|
|
@@ -3830,6 +3831,66 @@ export const SETTINGS_SCHEMA = {
|
|
|
3830
3831
|
},
|
|
3831
3832
|
},
|
|
3832
3833
|
|
|
3834
|
+
"computer.enabled": {
|
|
3835
|
+
type: "boolean",
|
|
3836
|
+
default: false,
|
|
3837
|
+
ui: {
|
|
3838
|
+
tab: "tools",
|
|
3839
|
+
group: "Available Tools",
|
|
3840
|
+
label: "Computer",
|
|
3841
|
+
description: "Enable native host-desktop screenshots and input for OpenAI computer use",
|
|
3842
|
+
},
|
|
3843
|
+
},
|
|
3844
|
+
|
|
3845
|
+
"computer.backend": {
|
|
3846
|
+
type: "enum",
|
|
3847
|
+
values: ["auto", "native"] as const,
|
|
3848
|
+
default: "auto",
|
|
3849
|
+
ui: {
|
|
3850
|
+
tab: "tools",
|
|
3851
|
+
group: "Computer",
|
|
3852
|
+
label: "Computer Backend",
|
|
3853
|
+
description: "Select automatic or explicit platform-native desktop capture and input",
|
|
3854
|
+
options: [
|
|
3855
|
+
{ value: "auto", label: "Auto" },
|
|
3856
|
+
{ value: "native", label: "Native" },
|
|
3857
|
+
],
|
|
3858
|
+
},
|
|
3859
|
+
},
|
|
3860
|
+
|
|
3861
|
+
"computer.display": {
|
|
3862
|
+
type: "string",
|
|
3863
|
+
default: "all",
|
|
3864
|
+
ui: {
|
|
3865
|
+
tab: "tools",
|
|
3866
|
+
group: "Computer",
|
|
3867
|
+
label: "Computer Display",
|
|
3868
|
+
description: "Composite all displays or select a native display id",
|
|
3869
|
+
},
|
|
3870
|
+
},
|
|
3871
|
+
|
|
3872
|
+
"computer.maxWidth": {
|
|
3873
|
+
type: "number",
|
|
3874
|
+
default: 1920,
|
|
3875
|
+
ui: {
|
|
3876
|
+
tab: "tools",
|
|
3877
|
+
group: "Computer",
|
|
3878
|
+
label: "Computer Screenshot Width",
|
|
3879
|
+
description: "Maximum composite screenshot width in pixels",
|
|
3880
|
+
},
|
|
3881
|
+
},
|
|
3882
|
+
|
|
3883
|
+
"computer.maxHeight": {
|
|
3884
|
+
type: "number",
|
|
3885
|
+
default: 1200,
|
|
3886
|
+
ui: {
|
|
3887
|
+
tab: "tools",
|
|
3888
|
+
group: "Computer",
|
|
3889
|
+
label: "Computer Screenshot Height",
|
|
3890
|
+
description: "Maximum composite screenshot height in pixels",
|
|
3891
|
+
},
|
|
3892
|
+
},
|
|
3893
|
+
|
|
3833
3894
|
"checkpoint.enabled": {
|
|
3834
3895
|
type: "boolean",
|
|
3835
3896
|
default: false,
|
package/src/eval/js/executor.ts
CHANGED
|
@@ -8,10 +8,11 @@ import type {
|
|
|
8
8
|
AgentToolUpdateCallback,
|
|
9
9
|
ToolLoadMode,
|
|
10
10
|
} from "@oh-my-pi/pi-agent-core";
|
|
11
|
-
import type { ImageContent, Static, TextContent, TSchema } from "@oh-my-pi/pi-ai";
|
|
11
|
+
import type { ComputerSafetyCheck, ImageContent, Static, TextContent, TSchema } from "@oh-my-pi/pi-ai";
|
|
12
|
+
import { sanitizeText } from "@oh-my-pi/pi-utils";
|
|
12
13
|
import type { Settings } from "../../config/settings";
|
|
13
14
|
import type { Theme } from "../../modes/theme/theme";
|
|
14
|
-
import { type ApprovalMode, formatApprovalPrompt, resolveApproval } from "../../tools/approval";
|
|
15
|
+
import { type ApprovalMode, formatApprovalPrompt, resolveApproval, truncateForPrompt } from "../../tools/approval";
|
|
15
16
|
import { defaultLoadModeForToolName } from "../../tools/essential-tools";
|
|
16
17
|
import { normalizeToolEventInput, resolveToolEventInput } from "../tool-event-input";
|
|
17
18
|
import { applyToolProxy } from "../tool-proxy";
|
|
@@ -83,6 +84,42 @@ export function wrapRegisteredTools(registeredTools: RegisteredTool[], runner: E
|
|
|
83
84
|
return registeredTools.map(rt => wrapRegisteredTool(rt, runner));
|
|
84
85
|
}
|
|
85
86
|
|
|
87
|
+
function computerSafetyChecks(context: AgentToolContext | undefined): ComputerSafetyCheck[] {
|
|
88
|
+
const metadata = context?.toolCall?.providerMetadata;
|
|
89
|
+
return metadata?.type === "computer" ? metadata.pendingSafetyChecks : [];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function approvalArgs(params: unknown, context: AgentToolContext | undefined): unknown {
|
|
93
|
+
const metadata = context?.toolCall?.providerMetadata;
|
|
94
|
+
return metadata?.type === "computer" ? { actions: metadata.actions } : params;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function toolEventArgs(params: unknown, context: AgentToolContext | undefined): Record<string, unknown> {
|
|
98
|
+
const metadata = context?.toolCall?.providerMetadata;
|
|
99
|
+
if (metadata?.type === "computer") {
|
|
100
|
+
return {
|
|
101
|
+
actions: metadata.actions,
|
|
102
|
+
pendingSafetyChecks: metadata.pendingSafetyChecks,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
return params as Record<string, unknown>;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function approvalData(value: string): string {
|
|
109
|
+
const sanitized = sanitizeText(value)
|
|
110
|
+
.replace(/[\r\n\t]+/g, " ")
|
|
111
|
+
.trim();
|
|
112
|
+
const truncated = truncateForPrompt(sanitized, 500);
|
|
113
|
+
return truncated.replace(/([\\`*_{}[\]()<>#+\-.!|])/g, "\\$1");
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function safetyCheckLines(checks: readonly ComputerSafetyCheck[]): string[] {
|
|
117
|
+
return checks.map((check, index) => {
|
|
118
|
+
const value = check.message || check.code || check.id;
|
|
119
|
+
return `${index + 1}. ${approvalData(value)}`;
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
|
|
86
123
|
/**
|
|
87
124
|
* Wraps a tool with extension callbacks for interception.
|
|
88
125
|
* - Emits tool_call event before execution (can block)
|
|
@@ -128,19 +165,25 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
|
|
|
128
165
|
const configuredMode = (settings?.get("tools.approvalMode") ?? "yolo") as ApprovalMode;
|
|
129
166
|
const approvalMode: ApprovalMode = cliAutoApprove ? "yolo" : configuredMode;
|
|
130
167
|
const userPolicies = (settings?.get("tools.approval") ?? {}) as Record<string, unknown>;
|
|
131
|
-
const
|
|
168
|
+
const resolvedArgs = approvalArgs(params, context);
|
|
169
|
+
const resolved = resolveApproval(this.tool, resolvedArgs, approvalMode, userPolicies);
|
|
132
170
|
if (resolved.policy === "deny") {
|
|
133
171
|
throw new Error(
|
|
134
172
|
`Tool "${this.tool.name}" is blocked by user policy.\n` +
|
|
135
173
|
`To allow: remove "tools.approval.${this.tool.name}: deny" from config.`,
|
|
136
174
|
);
|
|
137
175
|
}
|
|
176
|
+
const pendingSafetyChecks = computerSafetyChecks(context);
|
|
138
177
|
// An xd:// device dispatch already cleared the write tool's outer gate at
|
|
139
178
|
// this tool's tier — re-prompting would double-ask for one action. Explicit
|
|
140
179
|
// per-tool "prompt" policies and tool-demanded overrides still prompt.
|
|
180
|
+
// Provider safety checks are stronger: yolo, per-tool allow, and xdev approval
|
|
181
|
+
// never acknowledge them on the user's behalf.
|
|
141
182
|
const explicitPrompt = resolved.override || Object.hasOwn(userPolicies, this.tool.name);
|
|
142
183
|
const approvalCheck = {
|
|
143
|
-
required:
|
|
184
|
+
required:
|
|
185
|
+
pendingSafetyChecks.length > 0 ||
|
|
186
|
+
(resolved.policy === "prompt" && (explicitPrompt || context?.xdevApproved !== true)),
|
|
144
187
|
reason: resolved.reason,
|
|
145
188
|
};
|
|
146
189
|
|
|
@@ -171,10 +214,16 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
|
|
|
171
214
|
});
|
|
172
215
|
};
|
|
173
216
|
|
|
174
|
-
//
|
|
217
|
+
// Provider safety checks fail closed without an interactive prompt. Unlike
|
|
218
|
+
// ordinary tier approval, no setting or yolo mode may bypass this gate.
|
|
175
219
|
if (!this.runner.hasUI()) {
|
|
176
220
|
const reason = "no interactive UI available";
|
|
177
221
|
await resolveApproval(false, reason);
|
|
222
|
+
if (pendingSafetyChecks.length > 0) {
|
|
223
|
+
throw new Error(
|
|
224
|
+
`Tool "${this.tool.name}" has pending provider safety checks but no interactive UI is available.`,
|
|
225
|
+
);
|
|
226
|
+
}
|
|
178
227
|
throw new Error(
|
|
179
228
|
`Tool "${this.tool.name}" requires approval but no interactive UI available.\n` +
|
|
180
229
|
`Options:\n` +
|
|
@@ -185,12 +234,14 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
|
|
|
185
234
|
}
|
|
186
235
|
|
|
187
236
|
const uiContext = this.runner.getUIContext();
|
|
237
|
+
const basePrompt = formatApprovalPrompt(this.tool, resolvedArgs, approvalCheck.reason);
|
|
238
|
+
const safetyPrompt =
|
|
239
|
+
pendingSafetyChecks.length > 0
|
|
240
|
+
? `${basePrompt}\nProvider safety checks:\n${safetyCheckLines(pendingSafetyChecks).join("\n")}`
|
|
241
|
+
: basePrompt;
|
|
188
242
|
let choice: string | undefined;
|
|
189
243
|
try {
|
|
190
|
-
choice = await uiContext.select(
|
|
191
|
-
"Approve",
|
|
192
|
-
"Deny",
|
|
193
|
-
]);
|
|
244
|
+
choice = await uiContext.select(safetyPrompt, ["Approve", "Deny"]);
|
|
194
245
|
} catch (err) {
|
|
195
246
|
await resolveApproval(false, err instanceof Error ? err.message : "approval aborted");
|
|
196
247
|
throw err;
|
|
@@ -200,6 +251,10 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
|
|
|
200
251
|
if (!approved) {
|
|
201
252
|
throw new Error(`Tool call denied by user: ${this.tool.name}`);
|
|
202
253
|
}
|
|
254
|
+
if (pendingSafetyChecks.length > 0) {
|
|
255
|
+
if (!context) throw new Error("Provider safety approval context is unavailable");
|
|
256
|
+
context.providerSafetyApproved = true;
|
|
257
|
+
}
|
|
203
258
|
}
|
|
204
259
|
|
|
205
260
|
// 2. Emit tool_call event - extensions can block execution
|
|
@@ -211,7 +266,7 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
|
|
|
211
266
|
toolCallId,
|
|
212
267
|
input: normalizeToolEventInput(
|
|
213
268
|
this.tool.name,
|
|
214
|
-
resolveToolEventInput(this.tool, params
|
|
269
|
+
resolveToolEventInput(this.tool, toolEventArgs(params, context)),
|
|
215
270
|
),
|
|
216
271
|
})) as ToolCallEventResult | undefined;
|
|
217
272
|
|
|
@@ -228,7 +283,7 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
|
|
|
228
283
|
}
|
|
229
284
|
|
|
230
285
|
// Execute the actual tool
|
|
231
|
-
let result:
|
|
286
|
+
let result: AgentToolResult<TDetails, TParameters>;
|
|
232
287
|
let executionError: Error | undefined;
|
|
233
288
|
|
|
234
289
|
try {
|
|
@@ -249,7 +304,7 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
|
|
|
249
304
|
toolCallId,
|
|
250
305
|
input: normalizeToolEventInput(
|
|
251
306
|
this.tool.name,
|
|
252
|
-
resolveToolEventInput(this.tool, params
|
|
307
|
+
resolveToolEventInput(this.tool, toolEventArgs(params, context)),
|
|
253
308
|
),
|
|
254
309
|
content: result.content,
|
|
255
310
|
details: result.details,
|
|
@@ -275,6 +330,7 @@ export class ExtensionToolWrapper<TParameters extends TSchema = TSchema, TDetail
|
|
|
275
330
|
return {
|
|
276
331
|
content: modifiedContent,
|
|
277
332
|
details: modifiedDetails,
|
|
333
|
+
providerMetadata: result.providerMetadata,
|
|
278
334
|
...(effectiveError ? { isError: true } : {}),
|
|
279
335
|
};
|
|
280
336
|
}
|
|
@@ -17,7 +17,7 @@ import * as fs from "node:fs";
|
|
|
17
17
|
import * as path from "node:path";
|
|
18
18
|
import type { AgentToolResult, AgentToolUpdateCallback } from "@oh-my-pi/pi-agent-core";
|
|
19
19
|
import { type AuthCredential, SqliteAuthCredentialStore, type TSchema } from "@oh-my-pi/pi-ai";
|
|
20
|
-
import { Text } from "@oh-my-pi/pi-tui";
|
|
20
|
+
import { getKeybindings, type Keybinding, Text } from "@oh-my-pi/pi-tui";
|
|
21
21
|
import {
|
|
22
22
|
getAgentDbPath,
|
|
23
23
|
getAgentDir,
|
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
parseFrontmatter as parseOmpFrontmatter,
|
|
27
27
|
} from "@oh-my-pi/pi-utils";
|
|
28
28
|
import { getPackageDir as getOmpPackageDir } from "../config";
|
|
29
|
+
import { formatKeyHints } from "../config/keybindings";
|
|
29
30
|
import type { PromptTemplate } from "../config/prompt-templates";
|
|
30
31
|
import { type SettingPath, Settings } from "../config/settings";
|
|
31
32
|
import { EditTool } from "../edit";
|
|
@@ -373,6 +374,11 @@ async function executeLegacyBashOperations(
|
|
|
373
374
|
}
|
|
374
375
|
}
|
|
375
376
|
|
|
377
|
+
/** Format the active shortcut for legacy extensions that render keybinding hints. */
|
|
378
|
+
export function keyText(action: Keybinding): string {
|
|
379
|
+
return formatKeyHints(getKeybindings().getKeys(action));
|
|
380
|
+
}
|
|
381
|
+
|
|
376
382
|
/** Parse frontmatter using the historical Pi package-root helper. */
|
|
377
383
|
export interface ParsedFrontmatter<T extends Record<string, unknown> = Record<string, unknown>> {
|
|
378
384
|
frontmatter: T;
|