codex-computer-use-mcp 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +18 -2
- package/MIGRATION.md +22 -5
- package/PROOF.md +31 -4
- package/README.md +16 -12
- package/SECURITY.md +10 -8
- package/dist/audit.d.ts +1 -1
- package/dist/direct-broker.d.ts +17 -3
- package/dist/direct-broker.js +30 -16
- package/dist/direct-broker.js.map +1 -1
- package/dist/direct-service.d.ts +3 -1
- package/dist/direct-service.js +11 -6
- package/dist/direct-service.js.map +1 -1
- package/dist/mcp-elicitation.d.ts +8 -0
- package/dist/mcp-elicitation.js +47 -0
- package/dist/mcp-elicitation.js.map +1 -0
- package/dist/mcp-server.js +44 -42
- package/dist/mcp-server.js.map +1 -1
- package/dist/tools.d.ts +12 -6
- package/dist/tools.js +93 -23
- package/dist/tools.js.map +1 -1
- package/docs/adr/0001-standalone-code-mode-mcp.md +45 -0
- package/integrations/pi/README.md +16 -6
- package/integrations/pi/index.ts +156 -31
- package/integrations/pi/mcp.json.example +1 -1
- package/npm-shrinkwrap.json +24 -27
- package/package.json +5 -4
package/dist/tools.js
CHANGED
|
@@ -1,50 +1,120 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
const app = z.string().
|
|
3
|
-
const
|
|
4
|
-
const coordinate = z.number().finite().min(0).max(1_000_000);
|
|
2
|
+
const app = z.string().describe("App name, full app path, or unambiguous bundle identifier");
|
|
3
|
+
const selectTextApp = z.string().describe("App name or bundle identifier");
|
|
5
4
|
export const DIRECT_TOOL_SCHEMAS = {
|
|
6
5
|
list_apps: z.object({}).strict(),
|
|
7
6
|
get_app_state: z.object({ app }).strict(),
|
|
8
7
|
click: z
|
|
9
8
|
.object({
|
|
10
9
|
app,
|
|
11
|
-
click_count: z.number().
|
|
12
|
-
element_index:
|
|
13
|
-
mouse_button: z.enum(["left", "right", "middle"]).optional(),
|
|
14
|
-
x: coordinate.optional(),
|
|
15
|
-
y: coordinate.optional(),
|
|
10
|
+
click_count: z.number().refine(Number.isInteger).describe("Number of clicks. Defaults to 1").optional(),
|
|
11
|
+
element_index: z.string().describe("Element index to click").optional(),
|
|
12
|
+
mouse_button: z.enum(["left", "right", "middle"]).describe("Mouse button to click. Defaults to left.").optional(),
|
|
13
|
+
x: z.number().describe("X coordinate in screenshot pixel coordinates").optional(),
|
|
14
|
+
y: z.number().describe("Y coordinate in screenshot pixel coordinates").optional(),
|
|
16
15
|
})
|
|
17
16
|
.strict(),
|
|
18
17
|
perform_secondary_action: z
|
|
19
|
-
.object({
|
|
18
|
+
.object({
|
|
19
|
+
app,
|
|
20
|
+
element_index: z.string().describe("Element identifier"),
|
|
21
|
+
action: z.string().describe("Secondary accessibility action name"),
|
|
22
|
+
})
|
|
20
23
|
.strict(),
|
|
21
|
-
set_value: z
|
|
22
|
-
select_text: z
|
|
24
|
+
set_value: z
|
|
23
25
|
.object({
|
|
24
26
|
app,
|
|
25
|
-
element_index:
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
27
|
+
element_index: z.string().describe("Element identifier"),
|
|
28
|
+
value: z.string().describe("Value to assign"),
|
|
29
|
+
})
|
|
30
|
+
.strict(),
|
|
31
|
+
select_text: z
|
|
32
|
+
.object({
|
|
33
|
+
app: selectTextApp,
|
|
34
|
+
element_index: z.string().describe("Text element identifier"),
|
|
35
|
+
text: z.string().describe("Target text as shown in the accessibility tree"),
|
|
36
|
+
prefix: z.string().describe("Optional text immediately before the target, used to disambiguate repeated matches").optional(),
|
|
37
|
+
selection: z.enum(["text", "cursor_before", "cursor_after"]).describe("Whether to select the text or place the cursor before or after it. Defaults to text.").optional(),
|
|
38
|
+
suffix: z.string().describe("Optional text immediately after the target, used to disambiguate repeated matches").optional(),
|
|
30
39
|
})
|
|
31
40
|
.strict(),
|
|
32
41
|
scroll: z
|
|
33
42
|
.object({
|
|
34
43
|
app,
|
|
35
|
-
element_index:
|
|
36
|
-
direction: z.
|
|
37
|
-
pages: z.number().
|
|
44
|
+
element_index: z.string().describe("Element identifier"),
|
|
45
|
+
direction: z.string().describe("Scroll direction: up, down, left, or right"),
|
|
46
|
+
pages: z.number().describe("Number of pages to scroll. Fractional values are supported. Defaults to 1").optional(),
|
|
38
47
|
})
|
|
39
48
|
.strict(),
|
|
40
49
|
drag: z
|
|
41
|
-
.object({
|
|
50
|
+
.object({
|
|
51
|
+
app,
|
|
52
|
+
from_x: z.number().describe("Start X coordinate"),
|
|
53
|
+
from_y: z.number().describe("Start Y coordinate"),
|
|
54
|
+
to_x: z.number().describe("End X coordinate"),
|
|
55
|
+
to_y: z.number().describe("End Y coordinate"),
|
|
56
|
+
})
|
|
42
57
|
.strict(),
|
|
43
|
-
press_key: z.object({ app, key: z.string().
|
|
44
|
-
type_text: z.object({ app, text: z.string().
|
|
58
|
+
press_key: z.object({ app, key: z.string().describe("Key or key combination to press") }).strict(),
|
|
59
|
+
type_text: z.object({ app, text: z.string().describe("Literal text to type") }).strict(),
|
|
45
60
|
};
|
|
46
61
|
export const OFFICIAL_METHODS = Object.freeze(Object.keys(DIRECT_TOOL_SCHEMAS));
|
|
47
|
-
|
|
62
|
+
const readAnnotations = Object.freeze({
|
|
63
|
+
destructiveHint: false,
|
|
64
|
+
idempotentHint: true,
|
|
65
|
+
openWorldHint: false,
|
|
66
|
+
readOnlyHint: true,
|
|
67
|
+
});
|
|
68
|
+
const actionAnnotations = Object.freeze({
|
|
69
|
+
destructiveHint: false,
|
|
70
|
+
idempotentHint: false,
|
|
71
|
+
openWorldHint: false,
|
|
72
|
+
readOnlyHint: false,
|
|
73
|
+
});
|
|
74
|
+
/** Exact upstream descriptions and annotations exposed by the signed Computer Use helper. */
|
|
75
|
+
export const OFFICIAL_TOOL_METADATA = Object.freeze({
|
|
76
|
+
list_apps: {
|
|
77
|
+
description: "List the apps on this computer. Returns the set of apps that are currently running, as well as any that have been used in the last 14 days, including details on usage frequency",
|
|
78
|
+
annotations: readAnnotations,
|
|
79
|
+
},
|
|
80
|
+
get_app_state: {
|
|
81
|
+
description: "Start an app use session if needed, then get the state of the app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app",
|
|
82
|
+
annotations: readAnnotations,
|
|
83
|
+
},
|
|
84
|
+
click: {
|
|
85
|
+
description: "Click an element by index or pixel coordinates from screenshot",
|
|
86
|
+
annotations: actionAnnotations,
|
|
87
|
+
},
|
|
88
|
+
perform_secondary_action: {
|
|
89
|
+
description: "Invoke a secondary accessibility action exposed by an element",
|
|
90
|
+
annotations: actionAnnotations,
|
|
91
|
+
},
|
|
92
|
+
set_value: {
|
|
93
|
+
description: "Set the value of a settable accessibility element",
|
|
94
|
+
annotations: actionAnnotations,
|
|
95
|
+
},
|
|
96
|
+
select_text: {
|
|
97
|
+
description: "Select text inside a text element, or place the text cursor before or after it. Provide text exactly as it appears in the accessibility tree, including any Markdown formatting. If the text is not unique, provide surrounding prefix or suffix text to disambiguate it.",
|
|
98
|
+
annotations: actionAnnotations,
|
|
99
|
+
},
|
|
100
|
+
scroll: {
|
|
101
|
+
description: "Scroll an element in a direction by a number of pages",
|
|
102
|
+
annotations: actionAnnotations,
|
|
103
|
+
},
|
|
104
|
+
drag: {
|
|
105
|
+
description: "Drag from one point to another using pixel coordinates",
|
|
106
|
+
annotations: actionAnnotations,
|
|
107
|
+
},
|
|
108
|
+
press_key: {
|
|
109
|
+
description: "Press a key or key-combination on the keyboard, including modifier and navigation keys.\n - This supports xdotool's `key` syntax.\n - Examples: \"a\", \"Return\", \"Tab\", \"super+c\", \"Up\", \"KP_0\" (for the numpad 0 key).",
|
|
110
|
+
annotations: actionAnnotations,
|
|
111
|
+
},
|
|
112
|
+
type_text: {
|
|
113
|
+
description: "Type literal text using keyboard input",
|
|
114
|
+
annotations: actionAnnotations,
|
|
115
|
+
},
|
|
116
|
+
});
|
|
117
|
+
export const READ_ONLY_METHODS = new Set(OFFICIAL_METHODS.filter((method) => OFFICIAL_TOOL_METADATA[method].annotations.readOnlyHint));
|
|
48
118
|
export const MUTATING_METHODS = new Set(OFFICIAL_METHODS.filter((method) => !READ_ONLY_METHODS.has(method)));
|
|
49
119
|
export function isDirectMethod(value) {
|
|
50
120
|
return OFFICIAL_METHODS.includes(value);
|
package/dist/tools.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2DAA2D,CAAC,CAAC;AAC7F,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC,CAAC;AAE3E,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;IAChC,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACzC,KAAK,EAAE,CAAC;SACN,MAAM,CAAC;QACP,GAAG;QACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC,QAAQ,EAAE;QACvG,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE;QACvE,YAAY,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,0CAA0C,CAAC,CAAC,QAAQ,EAAE;QACjH,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC,CAAC,QAAQ,EAAE;QACjF,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC,CAAC,QAAQ,EAAE;KACjF,CAAC;SACD,MAAM,EAAE;IACV,wBAAwB,EAAE,CAAC;SACzB,MAAM,CAAC;QACP,GAAG;QACH,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QACxD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;KAClE,CAAC;SACD,MAAM,EAAE;IACV,SAAS,EAAE,CAAC;SACV,MAAM,CAAC;QACP,GAAG;QACH,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QACxD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;KAC7C,CAAC;SACD,MAAM,EAAE;IACV,WAAW,EAAE,CAAC;SACZ,MAAM,CAAC;QACP,GAAG,EAAE,aAAa;QAClB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;QAC7D,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gDAAgD,CAAC;QAC3E,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oFAAoF,CAAC,CAAC,QAAQ,EAAE;QAC5H,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,eAAe,EAAE,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,sFAAsF,CAAC,CAAC,QAAQ,EAAE;QACxK,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mFAAmF,CAAC,CAAC,QAAQ,EAAE;KAC3H,CAAC;SACD,MAAM,EAAE;IACV,MAAM,EAAE,CAAC;SACP,MAAM,CAAC;QACP,GAAG;QACH,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QACxD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4CAA4C,CAAC;QAC5E,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2EAA2E,CAAC,CAAC,QAAQ,EAAE;KAClH,CAAC;SACD,MAAM,EAAE;IACV,IAAI,EAAE,CAAC;SACL,MAAM,CAAC;QACP,GAAG;QACH,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QACjD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QACjD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;QAC7C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;KAC7C,CAAC;SACD,MAAM,EAAE;IACV,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;IAClG,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;CAC/E,CAAC;AAKX,MAAM,CAAC,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAmB,CAAC,CAAC;AAYlG,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;IACrC,eAAe,EAAE,KAAK;IACtB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,YAAY,EAAE,IAAI;CAClB,CAAC,CAAC;AACH,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CAAC;IACvC,eAAe,EAAE,KAAK;IACtB,cAAc,EAAE,KAAK;IACrB,aAAa,EAAE,KAAK;IACpB,YAAY,EAAE,KAAK;CACnB,CAAC,CAAC;AAEH,6FAA6F;AAC7F,MAAM,CAAC,MAAM,sBAAsB,GAAyD,MAAM,CAAC,MAAM,CAAC;IACzG,SAAS,EAAE;QACV,WAAW,EAAE,kLAAkL;QAC/L,WAAW,EAAE,eAAe;KAC5B;IACD,aAAa,EAAE;QACd,WAAW,EAAE,4MAA4M;QACzN,WAAW,EAAE,eAAe;KAC5B;IACD,KAAK,EAAE;QACN,WAAW,EAAE,gEAAgE;QAC7E,WAAW,EAAE,iBAAiB;KAC9B;IACD,wBAAwB,EAAE;QACzB,WAAW,EAAE,+DAA+D;QAC5E,WAAW,EAAE,iBAAiB;KAC9B;IACD,SAAS,EAAE;QACV,WAAW,EAAE,mDAAmD;QAChE,WAAW,EAAE,iBAAiB;KAC9B;IACD,WAAW,EAAE;QACZ,WAAW,EAAE,2QAA2Q;QACxR,WAAW,EAAE,iBAAiB;KAC9B;IACD,MAAM,EAAE;QACP,WAAW,EAAE,uDAAuD;QACpE,WAAW,EAAE,iBAAiB;KAC9B;IACD,IAAI,EAAE;QACL,WAAW,EAAE,wDAAwD;QACrE,WAAW,EAAE,iBAAiB;KAC9B;IACD,SAAS,EAAE;QACV,WAAW,EAAE,qOAAqO;QAClP,WAAW,EAAE,iBAAiB;KAC9B;IACD,SAAS,EAAE;QACV,WAAW,EAAE,wCAAwC;QACrD,WAAW,EAAE,iBAAiB;KAC9B;CACD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAe,gBAAgB,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;AACrJ,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAe,gBAAgB,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAE3H,MAAM,UAAU,cAAc,CAAC,KAAa;IAC3C,OAAO,gBAAgB,CAAC,QAAQ,CAAC,KAAqB,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,WAAW,GAAqC;IACrD,GAAG,EAAE,QAAQ;IACb,OAAO,EAAE,QAAQ;IACjB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,WAAW;IACjB,OAAO,EAAE,WAAW;IACpB,KAAK,EAAE,SAAS;IAChB,GAAG,EAAE,OAAO;IACZ,MAAM,EAAE,OAAO;IACf,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,QAAQ;IAChB,GAAG,EAAE,QAAQ;IACb,MAAM,EAAE,QAAQ;IAChB,SAAS,EAAE,WAAW;IACtB,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,SAAS;IACjB,QAAQ,EAAE,WAAW;IACrB,OAAO,EAAE,IAAI;IACb,SAAS,EAAE,MAAM;IACjB,SAAS,EAAE,MAAM;IACjB,UAAU,EAAE,OAAO;CACnB,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,KAAa;IACnD,OAAO,KAAK;SACV,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QACjD,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;QACxB,OAAO,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IAClE,CAAC,CAAC;SACD,IAAI,CAAC,GAAG,CAAC,CAAC;AACb,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAAoB,EAAE,KAAc;IAC3E,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,KAAK,CAAwB,CAAC;IAC/E,IAAI,MAAM,KAAK,WAAW;QAAE,OAAO,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,sBAAsB,CAAC,MAAM,CAAC,GAAa,CAAC,EAAE,CAAC;IACpG,OAAO,MAAM,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAoB,EAAE,IAAyB;IAC3E,OAAO,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAE,IAAI,CAAC,GAA0B,CAAC;AAC9E,CAAC;AAED,sGAAsG;AACtG,MAAM,CAAC,MAAM,+BAA+B,GAA4C;IACvF,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,EAAE,oBAAoB,EAAE,KAAK,EAAE;IAC1E,aAAa,EAAE;QACd,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,2DAA2D,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;QACjH,QAAQ,EAAE,CAAC,KAAK,CAAC;QACjB,oBAAoB,EAAE,KAAK;KAC3B;IACD,KAAK,EAAE;QACN,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACX,GAAG,EAAE,EAAE,WAAW,EAAE,2DAA2D,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjG,WAAW,EAAE,EAAE,WAAW,EAAE,iCAAiC,EAAE,IAAI,EAAE,SAAS,EAAE;YAChF,aAAa,EAAE,EAAE,WAAW,EAAE,wBAAwB,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxE,YAAY,EAAE,EAAE,WAAW,EAAE,0CAA0C,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC5H,CAAC,EAAE,EAAE,WAAW,EAAE,8CAA8C,EAAE,IAAI,EAAE,QAAQ,EAAE;YAClF,CAAC,EAAE,EAAE,WAAW,EAAE,8CAA8C,EAAE,IAAI,EAAE,QAAQ,EAAE;SAClF;QACD,QAAQ,EAAE,CAAC,KAAK,CAAC;QACjB,oBAAoB,EAAE,KAAK;KAC3B;IACD,wBAAwB,EAAE;QACzB,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACX,MAAM,EAAE,EAAE,WAAW,EAAE,qCAAqC,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9E,GAAG,EAAE,EAAE,WAAW,EAAE,2DAA2D,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjG,aAAa,EAAE,EAAE,WAAW,EAAE,oBAAoB,EAAE,IAAI,EAAE,QAAQ,EAAE;SACpE;QACD,QAAQ,EAAE,CAAC,KAAK,EAAE,eAAe,EAAE,QAAQ,CAAC;QAC5C,oBAAoB,EAAE,KAAK;KAC3B;IACD,SAAS,EAAE;QACV,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACX,GAAG,EAAE,EAAE,WAAW,EAAE,2DAA2D,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjG,aAAa,EAAE,EAAE,WAAW,EAAE,oBAAoB,EAAE,IAAI,EAAE,QAAQ,EAAE;YACpE,KAAK,EAAE,EAAE,WAAW,EAAE,iBAAiB,EAAE,IAAI,EAAE,QAAQ,EAAE;SACzD;QACD,QAAQ,EAAE,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,CAAC;QAC3C,oBAAoB,EAAE,KAAK;KAC3B;IACD,WAAW,EAAE;QACZ,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACX,GAAG,EAAE,EAAE,WAAW,EAAE,+BAA+B,EAAE,IAAI,EAAE,QAAQ,EAAE;YACrE,aAAa,EAAE,EAAE,WAAW,EAAE,yBAAyB,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzE,MAAM,EAAE,EAAE,WAAW,EAAE,oFAAoF,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC7H,SAAS,EAAE,EAAE,WAAW,EAAE,sFAAsF,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,eAAe,EAAE,cAAc,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;YACnL,MAAM,EAAE,EAAE,WAAW,EAAE,mFAAmF,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC5H,IAAI,EAAE,EAAE,WAAW,EAAE,gDAAgD,EAAE,IAAI,EAAE,QAAQ,EAAE;SACvF;QACD,QAAQ,EAAE,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,CAAC;QAC1C,oBAAoB,EAAE,KAAK;KAC3B;IACD,MAAM,EAAE;QACP,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACX,GAAG,EAAE,EAAE,WAAW,EAAE,2DAA2D,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjG,SAAS,EAAE,EAAE,WAAW,EAAE,4CAA4C,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxF,aAAa,EAAE,EAAE,WAAW,EAAE,oBAAoB,EAAE,IAAI,EAAE,QAAQ,EAAE;YACpE,KAAK,EAAE,EAAE,WAAW,EAAE,2EAA2E,EAAE,IAAI,EAAE,QAAQ,EAAE;SACnH;QACD,QAAQ,EAAE,CAAC,KAAK,EAAE,eAAe,EAAE,WAAW,CAAC;QAC/C,oBAAoB,EAAE,KAAK;KAC3B;IACD,IAAI,EAAE;QACL,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACX,GAAG,EAAE,EAAE,WAAW,EAAE,2DAA2D,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjG,MAAM,EAAE,EAAE,WAAW,EAAE,oBAAoB,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC7D,MAAM,EAAE,EAAE,WAAW,EAAE,oBAAoB,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC7D,IAAI,EAAE,EAAE,WAAW,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzD,IAAI,EAAE,EAAE,WAAW,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE;SACzD;QACD,QAAQ,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;QACrD,oBAAoB,EAAE,KAAK;KAC3B;IACD,SAAS,EAAE;QACV,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACX,GAAG,EAAE,EAAE,WAAW,EAAE,2DAA2D,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjG,GAAG,EAAE,EAAE,WAAW,EAAE,iCAAiC,EAAE,IAAI,EAAE,QAAQ,EAAE;SACvE;QACD,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC;QACxB,oBAAoB,EAAE,KAAK;KAC3B;IACD,SAAS,EAAE;QACV,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACX,GAAG,EAAE,EAAE,WAAW,EAAE,2DAA2D,EAAE,IAAI,EAAE,QAAQ,EAAE;YACjG,IAAI,EAAE,EAAE,WAAW,EAAE,sBAAsB,EAAE,IAAI,EAAE,QAAQ,EAAE;SAC7D;QACD,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;QACzB,oBAAoB,EAAE,KAAK;KAC3B;CACD,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# ADR 0001: Keep Code Mode in a standalone MCP process
|
|
2
|
+
|
|
3
|
+
- **Status:** Accepted
|
|
4
|
+
- **Date:** 2026-07-15
|
|
5
|
+
|
|
6
|
+
## Context
|
|
7
|
+
|
|
8
|
+
Codex can compose MCP tools through JavaScript Code Mode. Pi's progressive tool disclosure reduces schema context but does not let generated code invoke arbitrary Pi tools. Adding an executor, loader, or batch method here would also break this repository's exact ten-tool Computer Use contract.
|
|
9
|
+
|
|
10
|
+
We considered existing Pi and MCP implementations:
|
|
11
|
+
|
|
12
|
+
- `pi-mcp-adapter` has the strongest Pi-facing MCP configuration, OAuth, elicitation, lifecycle, metadata, and rich-result handling, but no Code Mode executor.
|
|
13
|
+
- `dmmulroy/pi-mcp` has a clean MCP manager but no Code Mode, no published package, and no detected licence.
|
|
14
|
+
- `Hor1zonZzz/pi-codeMode` demonstrates Pi-side JavaScript orchestration but supports only reconstructed built-ins and has no MCP support or tests.
|
|
15
|
+
- `pi-code-tool` is a tested Pi-native pattern, but uses Python/Monty and does not aggregate MCP servers.
|
|
16
|
+
- Cloudflare's Code Mode packages are strong design references, but their secure executor depends on Cloudflare Dynamic Workers; `cloudflare/mcp` is specific to the Cloudflare API.
|
|
17
|
+
- `tool-sandbox` and `tool-sandbox-mcp` provide QuickJS-based execution, but impose a separate capability boundary and the server primarily targets one upstream HTTP gateway.
|
|
18
|
+
- `cmcp` is the closest standalone multi-server implementation, but lacks the OAuth, elicitation, sampling, roots, and notification fidelity required here.
|
|
19
|
+
|
|
20
|
+
## Decision
|
|
21
|
+
|
|
22
|
+
If we add Code Mode for Pi, implement it as a **separate stdio MCP server** loaded through Pi's existing MCP adapter.
|
|
23
|
+
|
|
24
|
+
- Do not modify Pi core.
|
|
25
|
+
- Do not add Code Mode methods to `codex-computer-use-mcp`; its public Computer Use surface remains the exact ten official tools.
|
|
26
|
+
- Let the Code Mode process connect to upstream MCP servers and expose a compact code-execution surface for discovery and composition.
|
|
27
|
+
- Give generated code the same filesystem, network, environment, and process authority as the Code Mode server's surrounding Pi environment. Do not add a stricter capability/security sandbox solely around Code Mode.
|
|
28
|
+
- Use the standalone MCP process as the fault-containment boundary. Pi can time out, terminate, and restart it if generated code wedges the process. Do not add a child process or isolate per execution without operational evidence that the server boundary is insufficient.
|
|
29
|
+
- Preserve upstream MCP behavior across the nested boundary, especially OAuth, cancellation, rich content, and elicitation responses. Never fabricate `accept` or `decline`; unsupported interaction returns `cancel`.
|
|
30
|
+
|
|
31
|
+
Normal Pi tools remain available directly. This decision adds MCP composition; it does not attempt to turn every Pi tool into a nested Code Mode function.
|
|
32
|
+
|
|
33
|
+
## Rationale
|
|
34
|
+
|
|
35
|
+
The process boundary provides the practical reliability benefit—one malformed snippet cannot freeze Pi—without creating an artificial authority mismatch between Pi and Code Mode. A separate security sandbox would constrain only one route while the same agent can already use Pi's deliberately unrestricted tools.
|
|
36
|
+
|
|
37
|
+
Keeping Code Mode separate also preserves this package's upstream contract, works with agent-agnostic MCP clients, and avoids coupling the implementation to Pi internals.
|
|
38
|
+
|
|
39
|
+
## Consequences
|
|
40
|
+
|
|
41
|
+
- Code Mode is a separate package and lifecycle from Computer Use.
|
|
42
|
+
- The first implementation must close the protocol-fidelity gaps found in existing standalone proxies rather than adopting one unchanged.
|
|
43
|
+
- Pi's MCP adapter remains responsible for the outer client connection and UI; the Code Mode server remains responsible for its upstream MCP connections and execution lifecycle.
|
|
44
|
+
- Process timeout, termination, restart, cleanup, and end-to-end elicitation tests are release requirements.
|
|
45
|
+
- Additional execution isolation remains an evidence-driven future option, not a default architectural requirement.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Pi integration
|
|
2
2
|
|
|
3
|
-
The native Pi adapter is the primary 0.
|
|
3
|
+
The native Pi adapter is the primary 0.3 path. It registers ten namespaced typed tools so Pi itself chooses each official Computer Use method and argument. Pi 0.80.7 or newer exposes them progressively without changing the ten-tool contract.
|
|
4
4
|
|
|
5
5
|
## Source checkout acceptance
|
|
6
6
|
|
|
@@ -13,18 +13,23 @@ CODEX_COMPUTER_USE_HOME="$(mktemp -d)" \
|
|
|
13
13
|
|
|
14
14
|
`-ne` prevents an installed 0.1 adapter from loading at the same time. This source workflow does not install or switch live Pi configuration.
|
|
15
15
|
|
|
16
|
-
For normal installation, use the exact version 0.
|
|
16
|
+
For normal installation, use the exact version 0.3.0 npm package:
|
|
17
17
|
|
|
18
18
|
```bash
|
|
19
|
-
pi install npm:codex-computer-use-mcp@0.
|
|
19
|
+
pi install npm:codex-computer-use-mcp@0.3.0
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
Use the source workflow only when you need to test an exact reviewed commit. Follow the rollback procedure in `MIGRATION.md` when replacing version 0.1.
|
|
23
23
|
|
|
24
24
|
## Registered surface
|
|
25
25
|
|
|
26
|
+
Initially active:
|
|
27
|
+
|
|
26
28
|
- `computer_use_list_apps`
|
|
27
29
|
- `computer_use_get_app_state`
|
|
30
|
+
|
|
31
|
+
Registered and activated after a successful `get_app_state` call:
|
|
32
|
+
|
|
28
33
|
- `computer_use_click`
|
|
29
34
|
- `computer_use_perform_secondary_action`
|
|
30
35
|
- `computer_use_set_value`
|
|
@@ -33,13 +38,18 @@ Use the source workflow only when you need to test an exact reviewed commit. Fol
|
|
|
33
38
|
- `computer_use_drag`
|
|
34
39
|
- `computer_use_press_key`
|
|
35
40
|
- `computer_use_type_text`
|
|
41
|
+
|
|
42
|
+
Command:
|
|
43
|
+
|
|
36
44
|
- `/computer-use-status`
|
|
37
45
|
|
|
38
|
-
|
|
46
|
+
The session-start narrowing preserves every active tool owned by Pi or another extension. Activation after inspection is purely additive: both inspection tools remain active, and the adapter does not remove or replace any tool in that call. The lazily activated tools rely on their official descriptions and omit `promptSnippet` and `promptGuidelines`, so native deferred schema loading does not rebuild the system prompt. Supported Anthropic and OpenAI models receive the definitions at the inspection result; other models receive Pi's normal active-tool fallback on the next request.
|
|
47
|
+
|
|
48
|
+
No-permissions is the only policy: all ten tools are registered and available through this lifecycle with no wrapper permission prompts, mode selector, or app/intent/action gate. The signed host runs with Codex Full access (`approvalPolicy: "never"`, `sandbox: "danger-full-access"`), so normal empty-schema Computer Use app approvals are accepted by Codex before they reach Pi. Pi renders any form, OpenAI-form, or URL elicitation app-server does emit. The user's `accept`, `decline`, or `cancel` response is returned unchanged; the adapter never fabricates one.
|
|
39
49
|
|
|
40
50
|
## Generic MCP gateway
|
|
41
51
|
|
|
42
|
-
Merge `mcp.json.example` only for the exact 0.
|
|
52
|
+
Merge `mcp.json.example` only for the exact 0.3.0 package or after building an exact reviewed source commit. `directTools: false` is intentional; it keeps this powerful generic MCP surface behind Pi's gateway.
|
|
43
53
|
|
|
44
54
|
For a source checkout:
|
|
45
55
|
|
|
@@ -59,6 +69,6 @@ For a source checkout:
|
|
|
59
69
|
|
|
60
70
|
Do not load the native adapter and generic MCP adapter into the same acceptance process unless tool names are intentionally isolated.
|
|
61
71
|
|
|
62
|
-
The generic MCP path uses the same durable no-permissions policy: all ten methods
|
|
72
|
+
The generic MCP path uses the same durable no-permissions and official Full access policy: all ten methods and no wrapper permission gate. App-access approvals resolve inside Codex. Any standard form or URL elicitation app-server emits is forwarded to the invoking MCP client; an unsupported or headless client returns `cancel` rather than a fabricated decision.
|
|
63
73
|
|
|
64
74
|
See the root `MIGRATION.md` before replacing an installed 0.1 adapter.
|
package/integrations/pi/index.ts
CHANGED
|
@@ -4,52 +4,95 @@ import { getAgentDir, truncateHead, type ExtensionAPI } from "@earendil-works/pi
|
|
|
4
4
|
import { Text } from "@earendil-works/pi-tui";
|
|
5
5
|
import { Type, type TSchema } from "typebox";
|
|
6
6
|
import { executeDirectTool, getDirectStatus } from "../../dist/direct-service.js";
|
|
7
|
-
import type {
|
|
7
|
+
import type {
|
|
8
|
+
DirectBrokerElicitationRequest,
|
|
9
|
+
DirectBrokerElicitationResponse,
|
|
10
|
+
} from "../../dist/direct-broker.js";
|
|
11
|
+
import { OFFICIAL_TOOL_METADATA, type DirectMethod } from "../../dist/tools.js";
|
|
8
12
|
|
|
9
|
-
const App = Type.String({
|
|
10
|
-
const Element = Type.String({ minLength: 1, maxLength: 200, description: "Element identifier from computer_use_get_app_state" });
|
|
11
|
-
const Coordinate = Type.Number({ minimum: 0, maximum: 1_000_000 });
|
|
13
|
+
const App = Type.String({ description: "App name, full app path, or unambiguous bundle identifier" });
|
|
12
14
|
|
|
13
15
|
const ToolParameters: Record<DirectMethod, TSchema> = {
|
|
14
16
|
list_apps: Type.Object({}, { additionalProperties: false }),
|
|
15
17
|
get_app_state: Type.Object({ app: App }, { additionalProperties: false }),
|
|
16
18
|
click: Type.Object({
|
|
17
19
|
app: App,
|
|
18
|
-
click_count: Type.Optional(Type.Integer({
|
|
19
|
-
element_index: Type.Optional(Element),
|
|
20
|
-
mouse_button: Type.Optional(StringEnum(["left", "right", "middle"] as const)),
|
|
21
|
-
x: Type.Optional(
|
|
22
|
-
y: Type.Optional(
|
|
20
|
+
click_count: Type.Optional(Type.Integer({ description: "Number of clicks. Defaults to 1" })),
|
|
21
|
+
element_index: Type.Optional(Type.String({ description: "Element index to click" })),
|
|
22
|
+
mouse_button: Type.Optional(StringEnum(["left", "right", "middle"] as const, { description: "Mouse button to click. Defaults to left." })),
|
|
23
|
+
x: Type.Optional(Type.Number({ description: "X coordinate in screenshot pixel coordinates" })),
|
|
24
|
+
y: Type.Optional(Type.Number({ description: "Y coordinate in screenshot pixel coordinates" })),
|
|
23
25
|
}, { additionalProperties: false }),
|
|
24
|
-
perform_secondary_action: Type.Object({
|
|
25
|
-
|
|
26
|
-
|
|
26
|
+
perform_secondary_action: Type.Object({
|
|
27
|
+
app: App,
|
|
28
|
+
element_index: Type.String({ description: "Element identifier" }),
|
|
29
|
+
action: Type.String({ description: "Secondary accessibility action name" }),
|
|
30
|
+
}, { additionalProperties: false }),
|
|
31
|
+
set_value: Type.Object({
|
|
27
32
|
app: App,
|
|
28
|
-
element_index: Element,
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
+
element_index: Type.String({ description: "Element identifier" }),
|
|
34
|
+
value: Type.String({ description: "Value to assign" }),
|
|
35
|
+
}, { additionalProperties: false }),
|
|
36
|
+
select_text: Type.Object({
|
|
37
|
+
app: Type.String({ description: "App name or bundle identifier" }),
|
|
38
|
+
element_index: Type.String({ description: "Text element identifier" }),
|
|
39
|
+
text: Type.String({ description: "Target text as shown in the accessibility tree" }),
|
|
40
|
+
prefix: Type.Optional(Type.String({ description: "Optional text immediately before the target, used to disambiguate repeated matches" })),
|
|
41
|
+
selection: Type.Optional(StringEnum(["text", "cursor_before", "cursor_after"] as const, { description: "Whether to select the text or place the cursor before or after it. Defaults to text." })),
|
|
42
|
+
suffix: Type.Optional(Type.String({ description: "Optional text immediately after the target, used to disambiguate repeated matches" })),
|
|
33
43
|
}, { additionalProperties: false }),
|
|
34
44
|
scroll: Type.Object({
|
|
35
45
|
app: App,
|
|
36
|
-
element_index: Element,
|
|
37
|
-
direction:
|
|
38
|
-
pages: Type.Optional(Type.Number({
|
|
46
|
+
element_index: Type.String({ description: "Element identifier" }),
|
|
47
|
+
direction: Type.String({ description: "Scroll direction: up, down, left, or right" }),
|
|
48
|
+
pages: Type.Optional(Type.Number({ description: "Number of pages to scroll. Fractional values are supported. Defaults to 1" })),
|
|
39
49
|
}, { additionalProperties: false }),
|
|
40
|
-
drag: Type.Object({
|
|
41
|
-
|
|
42
|
-
|
|
50
|
+
drag: Type.Object({
|
|
51
|
+
app: App,
|
|
52
|
+
from_x: Type.Number({ description: "Start X coordinate" }),
|
|
53
|
+
from_y: Type.Number({ description: "Start Y coordinate" }),
|
|
54
|
+
to_x: Type.Number({ description: "End X coordinate" }),
|
|
55
|
+
to_y: Type.Number({ description: "End Y coordinate" }),
|
|
56
|
+
}, { additionalProperties: false }),
|
|
57
|
+
press_key: Type.Object({ app: App, key: Type.String({ description: "Key or key combination to press" }) }, { additionalProperties: false }),
|
|
58
|
+
type_text: Type.Object({ app: App, text: Type.String({ description: "Literal text to type" }) }, { additionalProperties: false }),
|
|
43
59
|
};
|
|
44
60
|
|
|
45
61
|
function toolDescription(method: DirectMethod): string {
|
|
46
|
-
return
|
|
62
|
+
return OFFICIAL_TOOL_METADATA[method].description;
|
|
47
63
|
}
|
|
48
64
|
|
|
49
65
|
function titleFor(method: DirectMethod): string {
|
|
50
66
|
return method.split("_").map((part) => `${part.slice(0, 1).toUpperCase()}${part.slice(1)}`).join(" ");
|
|
51
67
|
}
|
|
52
68
|
|
|
69
|
+
const INSPECTION_METHODS = new Set<DirectMethod>(["list_apps", "get_app_state"]);
|
|
70
|
+
export const INSPECTION_TOOL_NAMES = [
|
|
71
|
+
"computer_use_list_apps",
|
|
72
|
+
"computer_use_get_app_state",
|
|
73
|
+
] as const;
|
|
74
|
+
export const INTERACTION_TOOL_NAMES = [
|
|
75
|
+
"computer_use_click",
|
|
76
|
+
"computer_use_perform_secondary_action",
|
|
77
|
+
"computer_use_set_value",
|
|
78
|
+
"computer_use_select_text",
|
|
79
|
+
"computer_use_scroll",
|
|
80
|
+
"computer_use_drag",
|
|
81
|
+
"computer_use_press_key",
|
|
82
|
+
"computer_use_type_text",
|
|
83
|
+
] as const;
|
|
84
|
+
|
|
85
|
+
export function setInitialComputerUseTools(pi: Pick<ExtensionAPI, "getActiveTools" | "setActiveTools">): void {
|
|
86
|
+
const interactionTools = new Set<string>(INTERACTION_TOOL_NAMES);
|
|
87
|
+
const preserved = pi.getActiveTools().filter((name) => !interactionTools.has(name));
|
|
88
|
+
pi.setActiveTools([...new Set([...preserved, ...INSPECTION_TOOL_NAMES])]);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export function activateInteractionTools(pi: Pick<ExtensionAPI, "getActiveTools" | "setActiveTools">): void {
|
|
92
|
+
const active = pi.getActiveTools();
|
|
93
|
+
pi.setActiveTools([...new Set([...active, ...INTERACTION_TOOL_NAMES])]);
|
|
94
|
+
}
|
|
95
|
+
|
|
53
96
|
function toPiContent(content: Array<Record<string, unknown>>): Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> {
|
|
54
97
|
const result: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> = [];
|
|
55
98
|
for (const block of content) {
|
|
@@ -75,6 +118,74 @@ function errorText(content: Array<Record<string, unknown>>): string {
|
|
|
75
118
|
.slice(0, 2_000) || "Official Computer Use returned an error";
|
|
76
119
|
}
|
|
77
120
|
|
|
121
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
122
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function initialFormContent(schema: unknown): Record<string, unknown> {
|
|
126
|
+
if (!isRecord(schema) || !isRecord(schema.properties)) return {};
|
|
127
|
+
const content: Record<string, unknown> = {};
|
|
128
|
+
for (const [key, rawProperty] of Object.entries(schema.properties)) {
|
|
129
|
+
if (!isRecord(rawProperty)) continue;
|
|
130
|
+
if (rawProperty.default !== undefined) content[key] = rawProperty.default;
|
|
131
|
+
else if (Array.isArray(rawProperty.enum) && rawProperty.enum.length > 0) content[key] = rawProperty.enum[0];
|
|
132
|
+
else if (rawProperty.type === "boolean") content[key] = false;
|
|
133
|
+
else if (rawProperty.type === "number" || rawProperty.type === "integer") content[key] = 0;
|
|
134
|
+
else if (rawProperty.type === "array") content[key] = [];
|
|
135
|
+
else content[key] = "";
|
|
136
|
+
}
|
|
137
|
+
return content;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export async function handleOfficialElicitation(
|
|
141
|
+
request: DirectBrokerElicitationRequest,
|
|
142
|
+
ctx: { hasUI: boolean; ui: {
|
|
143
|
+
select(title: string, options: string[]): Promise<string | undefined>;
|
|
144
|
+
editor(title: string, prefill?: string): Promise<string | undefined>;
|
|
145
|
+
notify(message: string, type?: "info" | "warning" | "error"): void;
|
|
146
|
+
} },
|
|
147
|
+
openUrl: (url: string) => Promise<boolean>,
|
|
148
|
+
): Promise<DirectBrokerElicitationResponse> {
|
|
149
|
+
if (!ctx.hasUI) return { action: "cancel" };
|
|
150
|
+
const message = typeof request.message === "string" ? request.message : "Official Computer Use requests input";
|
|
151
|
+
|
|
152
|
+
if (request.mode === "url") {
|
|
153
|
+
if (typeof request.url !== "string") return { action: "cancel" };
|
|
154
|
+
const choice = await ctx.ui.select(`${message}\n${request.url}`, ["Open URL", "Decline", "Cancel"]);
|
|
155
|
+
if (choice === "Decline") return { action: "decline" };
|
|
156
|
+
if (choice !== "Open URL") return { action: "cancel" };
|
|
157
|
+
if (!await openUrl(request.url)) {
|
|
158
|
+
ctx.ui.notify("Could not open the official Computer Use URL.", "error");
|
|
159
|
+
return { action: "cancel" };
|
|
160
|
+
}
|
|
161
|
+
return { action: "accept" };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
if (request.mode !== undefined && request.mode !== "form" && request.mode !== "openai/form") {
|
|
165
|
+
ctx.ui.notify(`Official Computer Use sent an unsupported elicitation mode: ${request.mode}`, "warning");
|
|
166
|
+
return { action: "cancel" };
|
|
167
|
+
}
|
|
168
|
+
const openAiForm = request.mode === "openai/form";
|
|
169
|
+
if (request.requestedSchema === undefined || (!openAiForm && !isRecord(request.requestedSchema))) return { action: "cancel" };
|
|
170
|
+
const choice = await ctx.ui.select(message, ["Respond", "Decline", "Cancel"]);
|
|
171
|
+
if (choice === "Decline") return { action: "decline" };
|
|
172
|
+
if (choice !== "Respond") return { action: "cancel" };
|
|
173
|
+
let prefill = JSON.stringify(initialFormContent(request.requestedSchema), null, 2);
|
|
174
|
+
const title = `${message}\nSchema: ${JSON.stringify(request.requestedSchema)}`;
|
|
175
|
+
while (true) {
|
|
176
|
+
const edited = await ctx.ui.editor(title, prefill);
|
|
177
|
+
if (edited === undefined) return { action: "cancel" };
|
|
178
|
+
prefill = edited;
|
|
179
|
+
try {
|
|
180
|
+
const content = JSON.parse(edited);
|
|
181
|
+
if (!openAiForm && !isRecord(content)) throw new Error("Response must be a JSON object");
|
|
182
|
+
return { action: "accept", content };
|
|
183
|
+
} catch (error) {
|
|
184
|
+
ctx.ui.notify(error instanceof Error ? error.message : "Response must be valid JSON", "error");
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
78
189
|
export default function directComputerUse(pi: ExtensionAPI) {
|
|
79
190
|
const stateRoot = process.env.CODEX_COMPUTER_USE_HOME || path.join(getAgentDir(), "direct-computer-use");
|
|
80
191
|
|
|
@@ -89,27 +200,39 @@ export default function directComputerUse(pi: ExtensionAPI) {
|
|
|
89
200
|
const methods = Object.keys(ToolParameters) as DirectMethod[];
|
|
90
201
|
for (const method of methods) {
|
|
91
202
|
const piName = `computer_use_${method}`;
|
|
92
|
-
|
|
93
|
-
name: piName,
|
|
94
|
-
label: titleFor(method),
|
|
95
|
-
description: toolDescription(method),
|
|
203
|
+
const inspectionPromptMetadata = INSPECTION_METHODS.has(method) ? {
|
|
96
204
|
promptSnippet: `${titleFor(method)} through the official signed macOS Computer Use service`,
|
|
97
205
|
promptGuidelines: [
|
|
98
206
|
`${piName} is a direct typed tool: choose its arguments yourself; it does not invoke a nested planner or model.`,
|
|
99
|
-
|
|
100
|
-
`${piName} must not be used for credentials, authentication, payments, external messages, or destructive actions without the user's explicit request; this wrapper will not open a permission prompt.`,
|
|
207
|
+
`Call computer_use_get_app_state once per assistant turn before interacting with an app.`,
|
|
101
208
|
],
|
|
209
|
+
} : {};
|
|
210
|
+
pi.registerTool({
|
|
211
|
+
name: piName,
|
|
212
|
+
label: titleFor(method),
|
|
213
|
+
description: toolDescription(method),
|
|
214
|
+
...inspectionPromptMetadata,
|
|
102
215
|
parameters: ToolParameters[method] as any,
|
|
103
|
-
async execute(_toolCallId, params, signal, onUpdate) {
|
|
216
|
+
async execute(_toolCallId, params, signal, onUpdate, ctx) {
|
|
104
217
|
const response = await executeDirectTool(
|
|
105
218
|
{ method, arguments: params as Record<string, unknown> },
|
|
106
219
|
{
|
|
107
220
|
stateRoot,
|
|
108
221
|
signal,
|
|
222
|
+
supportsOpenAiFormElicitation: true,
|
|
223
|
+
onElicitation: (request) => handleOfficialElicitation(
|
|
224
|
+
request,
|
|
225
|
+
ctx,
|
|
226
|
+
async (url) => {
|
|
227
|
+
const result = await pi.exec("/usr/bin/open", ["--", url], { signal, timeout: 15_000 });
|
|
228
|
+
return result.code === 0;
|
|
229
|
+
},
|
|
230
|
+
),
|
|
109
231
|
onProgress: (message) => onUpdate?.({ content: [{ type: "text", text: message }], details: { status: "running" } }),
|
|
110
232
|
},
|
|
111
233
|
);
|
|
112
234
|
if (response.isError) throw new Error(errorText(response.content));
|
|
235
|
+
if (method === "get_app_state") activateInteractionTools(pi);
|
|
113
236
|
return { content: toPiContent(response.content), details: response.details };
|
|
114
237
|
},
|
|
115
238
|
renderCall(args, theme) {
|
|
@@ -119,4 +242,6 @@ export default function directComputerUse(pi: ExtensionAPI) {
|
|
|
119
242
|
},
|
|
120
243
|
});
|
|
121
244
|
}
|
|
245
|
+
|
|
246
|
+
pi.on("session_start", () => setInitialComputerUseTools(pi));
|
|
122
247
|
}
|