@gleanwork/mcp-server-tester 2.0.0-beta.4 → 2.0.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +4609 -944
- package/dist/fixtures/mcp.d.ts +3 -0
- package/dist/fixtures/mcp.js +1 -1
- package/dist/fixtures/mcp.js.map +1 -1
- package/dist/{index-CwFHKIoN.d.cts → index-BTEdKnrn.d.cts} +245 -4
- package/dist/{index-CwFHKIoN.d.ts → index-BTEdKnrn.d.ts} +245 -4
- package/dist/index.cjs +4753 -1075
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +4757 -1079
- package/dist/index.js.map +1 -1
- package/dist/reporters/mcpReporter.cjs +5 -1
- package/dist/reporters/mcpReporter.cjs.map +1 -1
- package/dist/reporters/mcpReporter.d.cts +3 -0
- package/dist/reporters/mcpReporter.d.ts +3 -0
- package/dist/reporters/mcpReporter.js +5 -1
- package/dist/reporters/mcpReporter.js.map +1 -1
- package/dist/types/index.d.cts +1 -1
- package/dist/types/index.d.ts +1 -1
- package/package.json +5 -1
- package/scripts/chatgpt_linux.py +672 -0
- package/scripts/chatgpt_linux_contract.json +56 -0
- package/scripts/cowork_computer_use.py +124 -14
|
@@ -0,0 +1,672 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Bounded ChatGPT AT-SPI setup/submission. MST owns the app and draft hand-off.
|
|
3
|
+
|
|
4
|
+
No inference API, shell, arbitrary keyboard input, answer extraction, or action retry.
|
|
5
|
+
Input is JSON on stdin; output never contains query text or accessible names.
|
|
6
|
+
Draft opening is one request to the MST parent over --open-fd; MST performs the
|
|
7
|
+
codex://new deep-link hand-off and replies with a fixed receipt.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import select
|
|
16
|
+
import stat
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
import time
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class DriverFailure(RuntimeError):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
BUTTONS = {'button', 'push button'}
|
|
28
|
+
SURFACES = {'chatgpt-work': 'ChatGPT Work', 'codex': 'Codex'}
|
|
29
|
+
MODE_LABELS = {surface: 'Switch mode, current mode: ' + label
|
|
30
|
+
for surface, label in SURFACES.items()}
|
|
31
|
+
MENU_LABELS = {'chatgpt-work': 'ChatGPT Work Create, learn, and explore',
|
|
32
|
+
'codex': 'Codex Build, debug, and ship'}
|
|
33
|
+
EDITOR_ROLES = {'entry', 'text', 'text area', 'editable text', 'paragraph'}
|
|
34
|
+
XDOTOOL = '/usr/bin/xdotool'
|
|
35
|
+
INPUT_LIMIT = 2 * 1024 * 1024
|
|
36
|
+
TEXT_NODE_LIMIT = 256
|
|
37
|
+
TEXT_DEPTH_LIMIT = 16
|
|
38
|
+
OUTPUT_LIMIT = 1024
|
|
39
|
+
OPEN_TIMEOUT = 30
|
|
40
|
+
# Shared with the Node adapter.
|
|
41
|
+
CONTRACT = json.loads(Path(__file__).with_name('chatgpt_linux_contract.json').read_text('utf-8'))
|
|
42
|
+
SESSION_KEYS = tuple(CONTRACT['sessionEnvironment'] + CONTRACT['profileEnvironment']
|
|
43
|
+
+ CONTRACT['helperEnvironment'])
|
|
44
|
+
MAX_ACTIONS = CONTRACT['maxActions']
|
|
45
|
+
ERROR_CODES = frozenset(CONTRACT['errorCodes'])
|
|
46
|
+
# After the one deep-link hand-off the draft usually appears in 1-3 s, but live
|
|
47
|
+
# runs rarely exceeded 10 s. Waiting is read-only; the hand-off is never repeated.
|
|
48
|
+
DRAFT_POLLS = 300
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def error_code(error, glib_error=()):
|
|
52
|
+
if isinstance(error, DriverFailure) and str(error) in ERROR_CODES:
|
|
53
|
+
return str(error)
|
|
54
|
+
if isinstance(error, AttributeError):
|
|
55
|
+
return 'desktop_attribute_error'
|
|
56
|
+
if isinstance(error, TypeError):
|
|
57
|
+
return 'desktop_type_error'
|
|
58
|
+
if isinstance(error, glib_error):
|
|
59
|
+
return 'desktop_glib_error'
|
|
60
|
+
return 'desktop_driver_failed'
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def has_interface(interfaces, name):
|
|
64
|
+
return name in interfaces or 'org.a11y.atspi.' + name in interfaces
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Desktop:
|
|
68
|
+
def __init__(self):
|
|
69
|
+
import gi
|
|
70
|
+
gi.require_version('Atspi', '2.0')
|
|
71
|
+
from gi.repository import Atspi, GLib
|
|
72
|
+
self.glib_error = GLib.Error
|
|
73
|
+
self.api = Atspi
|
|
74
|
+
self.context = GLib.MainContext.default()
|
|
75
|
+
self.deadline = float('inf')
|
|
76
|
+
self.open_fd = None
|
|
77
|
+
|
|
78
|
+
def require_helpers(self):
|
|
79
|
+
# The MST parent owns the deep-link hand-off. Without its channel no
|
|
80
|
+
# draft can be opened, so fail before any UI action.
|
|
81
|
+
fd = self.open_fd
|
|
82
|
+
try:
|
|
83
|
+
if type(fd) is not int or fd < 3 or not stat.S_ISSOCK(os.fstat(fd).st_mode):
|
|
84
|
+
raise OSError
|
|
85
|
+
except OSError:
|
|
86
|
+
raise DriverFailure('helper_missing') from None
|
|
87
|
+
|
|
88
|
+
def remaining(self, cap=2):
|
|
89
|
+
remaining = self.deadline - time.monotonic()
|
|
90
|
+
if remaining <= 0:
|
|
91
|
+
raise DriverFailure('deadline_exceeded')
|
|
92
|
+
return min(cap, remaining)
|
|
93
|
+
|
|
94
|
+
def snapshot(self):
|
|
95
|
+
# Retry only a read-only traversal, discarding every partial tree.
|
|
96
|
+
for attempt in range(3):
|
|
97
|
+
try:
|
|
98
|
+
return self._snapshot()
|
|
99
|
+
except getattr(self, 'glib_error', ()):
|
|
100
|
+
if attempt == 2:
|
|
101
|
+
raise
|
|
102
|
+
self.remaining()
|
|
103
|
+
|
|
104
|
+
def _snapshot(self):
|
|
105
|
+
# AT-SPI cache invalidation runs on the default GLib context. Never
|
|
106
|
+
# authorize actions from a snapshot while that event queue is still busy.
|
|
107
|
+
for _ in range(256):
|
|
108
|
+
if not self.context.pending():
|
|
109
|
+
break
|
|
110
|
+
self.context.iteration(False)
|
|
111
|
+
if self.context.pending():
|
|
112
|
+
raise DriverFailure('accessibility_event_budget')
|
|
113
|
+
root = self.api.get_desktop(0)
|
|
114
|
+
apps = [root.get_child_at_index(i) for i in range(min(root.get_child_count(), 128))]
|
|
115
|
+
apps = [a for a in apps if a and (a.get_name() or '').casefold()
|
|
116
|
+
in {'chatgpt', 'codex', 'codex-launcher'}]
|
|
117
|
+
if not apps:
|
|
118
|
+
return []
|
|
119
|
+
if len(apps) != 1:
|
|
120
|
+
raise DriverFailure('desktop_ambiguous')
|
|
121
|
+
pending = [(apps[0], ())]
|
|
122
|
+
nodes = []
|
|
123
|
+
while pending:
|
|
124
|
+
node, ancestors = pending.pop()
|
|
125
|
+
if node is None:
|
|
126
|
+
continue
|
|
127
|
+
if len(nodes) >= 5000:
|
|
128
|
+
raise DriverFailure('accessibility_tree_budget')
|
|
129
|
+
index = len(nodes)
|
|
130
|
+
# A stale or incomplete tree cannot authorize an action.
|
|
131
|
+
states = node.get_state_set()
|
|
132
|
+
showing = states.contains(self.api.StateType.SHOWING)
|
|
133
|
+
visible = states.contains(self.api.StateType.VISIBLE)
|
|
134
|
+
enabled = states.contains(self.api.StateType.ENABLED)
|
|
135
|
+
sensitive = states.contains(self.api.StateType.SENSITIVE)
|
|
136
|
+
role = node.get_role_name()
|
|
137
|
+
editable_state = states.contains(self.api.StateType.EDITABLE)
|
|
138
|
+
# Unsupported interface RPCs can fail even on otherwise valid nodes.
|
|
139
|
+
# Query only candidate roles and use the advertised public interfaces.
|
|
140
|
+
text_interface = False
|
|
141
|
+
editable_interface = False
|
|
142
|
+
if role in EDITOR_ROLES and editable_state:
|
|
143
|
+
interfaces = node.get_interfaces()
|
|
144
|
+
text_interface = (has_interface(interfaces, 'Text')
|
|
145
|
+
and node.get_text_iface() is not None)
|
|
146
|
+
editable_interface = (has_interface(interfaces, 'EditableText')
|
|
147
|
+
and node.get_editable_text_iface() is not None)
|
|
148
|
+
editable = role in EDITOR_ROLES and editable_state and text_interface
|
|
149
|
+
nodes.append({'node': node, 'ancestors': ancestors, 'name': node.get_name() or '',
|
|
150
|
+
'role': role, 'showing': showing, 'visible': visible,
|
|
151
|
+
'enabled': enabled, 'sensitive': sensitive, 'editable': editable,
|
|
152
|
+
'editableState': editable_state, 'editableInterface': editable_interface,
|
|
153
|
+
'textInterface': text_interface,
|
|
154
|
+
'selected': states.contains(self.api.StateType.CHECKED)
|
|
155
|
+
or states.contains(self.api.StateType.SELECTED)})
|
|
156
|
+
pending.extend((node.get_child_at_index(i), ancestors + (index,))
|
|
157
|
+
for i in range(node.get_child_count()))
|
|
158
|
+
return nodes
|
|
159
|
+
|
|
160
|
+
def activate(self, control, allowed):
|
|
161
|
+
action = control['node'].get_action_iface()
|
|
162
|
+
if action is None:
|
|
163
|
+
raise DriverFailure('action_unavailable')
|
|
164
|
+
matches = [i for i in range(action.get_n_actions()) if action.get_action_name(i) in allowed]
|
|
165
|
+
if len(matches) != 1:
|
|
166
|
+
raise DriverFailure('action_missing_or_ambiguous')
|
|
167
|
+
if not action.do_action(matches[0]):
|
|
168
|
+
raise DriverFailure('action_acknowledgement_uncertain')
|
|
169
|
+
|
|
170
|
+
def text(self, control, limit=None):
|
|
171
|
+
# Prefer public Hypertext references. The structural fallback follows
|
|
172
|
+
# only a single direct child of a whole-marker owned composer node.
|
|
173
|
+
# Bound both source reads and expanded output; never export UI content.
|
|
174
|
+
byte_limit = INPUT_LIMIT if limit is None else min(limit, INPUT_LIMIT)
|
|
175
|
+
nodes_read = 0
|
|
176
|
+
bytes_read = 0
|
|
177
|
+
output_bytes = 0
|
|
178
|
+
fragments = []
|
|
179
|
+
|
|
180
|
+
def append(value):
|
|
181
|
+
nonlocal output_bytes
|
|
182
|
+
output_bytes += len(value.encode('utf-8'))
|
|
183
|
+
if output_bytes > byte_limit:
|
|
184
|
+
raise DriverFailure('composer_text_unavailable')
|
|
185
|
+
fragments.append(value)
|
|
186
|
+
|
|
187
|
+
def structural(node, role, ancestors, owned):
|
|
188
|
+
if not owned:
|
|
189
|
+
return False
|
|
190
|
+
if not ancestors and (role not in EDITOR_ROLES or not
|
|
191
|
+
node.get_state_set().contains(self.api.StateType.EDITABLE)):
|
|
192
|
+
return False
|
|
193
|
+
count = node.get_child_count()
|
|
194
|
+
if type(count) is not int or count < 0:
|
|
195
|
+
raise DriverFailure('composer_text_unavailable')
|
|
196
|
+
if count != 1:
|
|
197
|
+
return False # No separators or child order can be inferred.
|
|
198
|
+
child = node.get_child_at_index(0)
|
|
199
|
+
if child is None:
|
|
200
|
+
raise DriverFailure('composer_text_unavailable')
|
|
201
|
+
child_role = child.get_role_name()
|
|
202
|
+
static = role == 'paragraph' and child_role in {'static', 'static text'}
|
|
203
|
+
if child_role not in {'paragraph', 'text'} and not static:
|
|
204
|
+
raise DriverFailure('composer_text_unavailable')
|
|
205
|
+
expand(child, ancestors + (node,), owned=True, static=static)
|
|
206
|
+
return True
|
|
207
|
+
|
|
208
|
+
def expand(node, ancestors, owned=False, static=False):
|
|
209
|
+
nonlocal nodes_read, bytes_read
|
|
210
|
+
self.remaining()
|
|
211
|
+
if (node is None or len(ancestors) >= TEXT_DEPTH_LIMIT
|
|
212
|
+
or nodes_read >= TEXT_NODE_LIMIT or node in ancestors):
|
|
213
|
+
raise DriverFailure('composer_text_unavailable')
|
|
214
|
+
nodes_read += 1
|
|
215
|
+
interfaces = node.get_interfaces()
|
|
216
|
+
role = node.get_role_name()
|
|
217
|
+
if static:
|
|
218
|
+
count = node.get_child_count()
|
|
219
|
+
states = node.get_state_set()
|
|
220
|
+
if (type(count) is not int or count != 0
|
|
221
|
+
or role not in {'static', 'static text'}
|
|
222
|
+
or not states.contains(self.api.StateType.SHOWING)
|
|
223
|
+
or not states.contains(self.api.StateType.VISIBLE)):
|
|
224
|
+
raise DriverFailure('composer_text_unavailable')
|
|
225
|
+
if not has_interface(interfaces, 'Text'):
|
|
226
|
+
# Only a visible static leaf under an owned paragraph has
|
|
227
|
+
# a name that represents content, not an arbitrary label.
|
|
228
|
+
value = node.get_name()
|
|
229
|
+
if not isinstance(value, str):
|
|
230
|
+
raise DriverFailure('composer_text_unavailable')
|
|
231
|
+
bytes_read += len(value.encode('utf-8'))
|
|
232
|
+
if bytes_read > byte_limit:
|
|
233
|
+
raise DriverFailure('composer_text_unavailable')
|
|
234
|
+
append(value)
|
|
235
|
+
return
|
|
236
|
+
if (role in {'image', 'password text'}
|
|
237
|
+
or not has_interface(interfaces, 'Text')
|
|
238
|
+
or node.get_text_iface() is None):
|
|
239
|
+
raise DriverFailure('composer_text_unavailable')
|
|
240
|
+
# Accessible.get_text_iface() may return the Accessible itself.
|
|
241
|
+
# Always use unbound Text methods to avoid the PyGObject collision.
|
|
242
|
+
count = self.api.Text.get_character_count(node)
|
|
243
|
+
if type(count) is not int or not 0 <= count <= byte_limit - bytes_read:
|
|
244
|
+
raise DriverFailure('composer_text_unavailable')
|
|
245
|
+
value = self.api.Text.get_text(node, 0, count)
|
|
246
|
+
if not isinstance(value, str) or len(value) != count:
|
|
247
|
+
raise DriverFailure('composer_text_unavailable')
|
|
248
|
+
bytes_read += len(value.encode('utf-8'))
|
|
249
|
+
if bytes_read > byte_limit:
|
|
250
|
+
raise DriverFailure('composer_text_unavailable')
|
|
251
|
+
if not has_interface(interfaces, 'Hypertext'):
|
|
252
|
+
if value == '\ufffc' and structural(node, role, ancestors, owned):
|
|
253
|
+
return
|
|
254
|
+
append(value)
|
|
255
|
+
return
|
|
256
|
+
start = 0
|
|
257
|
+
for offset, character in enumerate(value):
|
|
258
|
+
if character != '\ufffc':
|
|
259
|
+
continue
|
|
260
|
+
self.remaining()
|
|
261
|
+
index = self.api.Hypertext.get_link_index(node, offset)
|
|
262
|
+
if type(index) is not int or index < -1:
|
|
263
|
+
raise DriverFailure('composer_text_unavailable')
|
|
264
|
+
if index == -1:
|
|
265
|
+
if value == '\ufffc' and structural(node, role, ancestors, owned):
|
|
266
|
+
return
|
|
267
|
+
continue # An unresolved object character is not empty.
|
|
268
|
+
append(value[start:offset])
|
|
269
|
+
link = self.api.Hypertext.get_link(node, index)
|
|
270
|
+
if link is None:
|
|
271
|
+
raise DriverFailure('composer_text_unavailable')
|
|
272
|
+
anchors = self.api.Hyperlink.get_n_anchors(link)
|
|
273
|
+
if type(anchors) is not int or anchors != 1:
|
|
274
|
+
raise DriverFailure('composer_text_unavailable')
|
|
275
|
+
child = self.api.Hyperlink.get_object(link, 0)
|
|
276
|
+
expand(child, ancestors + (node,))
|
|
277
|
+
start = offset + 1
|
|
278
|
+
append(value[start:])
|
|
279
|
+
|
|
280
|
+
try:
|
|
281
|
+
expand(control['node'], (), owned=True)
|
|
282
|
+
return ''.join(fragments)
|
|
283
|
+
except DriverFailure:
|
|
284
|
+
raise
|
|
285
|
+
except Exception:
|
|
286
|
+
# RPC errors can contain private text. Do not propagate their values.
|
|
287
|
+
raise DriverFailure('composer_text_unavailable') from None
|
|
288
|
+
|
|
289
|
+
def environment(self):
|
|
290
|
+
return {key: os.environ[key] for key in SESSION_KEYS if key in os.environ}
|
|
291
|
+
|
|
292
|
+
def open_prompt(self, prompt):
|
|
293
|
+
# One request to MST, bound to the exact draft by its UTF-8 SHA-256.
|
|
294
|
+
# MST knows the prompt; it is never echoed. No argv, shell, or retry.
|
|
295
|
+
if not isinstance(prompt, str):
|
|
296
|
+
raise DriverFailure('invalid_prompt')
|
|
297
|
+
try:
|
|
298
|
+
encoded = prompt.encode('utf-8', errors='strict')
|
|
299
|
+
except UnicodeEncodeError:
|
|
300
|
+
raise DriverFailure('invalid_prompt') from None
|
|
301
|
+
if len(encoded) > INPUT_LIMIT:
|
|
302
|
+
raise DriverFailure('input_too_large')
|
|
303
|
+
request = json.dumps({'open': hashlib.sha256(encoded).hexdigest()}).encode('ascii') + b'\n'
|
|
304
|
+
self._open(request)
|
|
305
|
+
|
|
306
|
+
def _open(self, request):
|
|
307
|
+
fd = self.open_fd
|
|
308
|
+
if getattr(self, 'open_used', False):
|
|
309
|
+
raise DriverFailure('helper_failed')
|
|
310
|
+
self.open_used = True
|
|
311
|
+
expires = time.monotonic() + self.remaining(OPEN_TIMEOUT)
|
|
312
|
+
|
|
313
|
+
def remaining():
|
|
314
|
+
seconds = expires - time.monotonic()
|
|
315
|
+
if seconds <= 0:
|
|
316
|
+
raise DriverFailure('helper_timeout')
|
|
317
|
+
return seconds
|
|
318
|
+
|
|
319
|
+
try:
|
|
320
|
+
offset = 0
|
|
321
|
+
while offset < len(request):
|
|
322
|
+
_, writable, _ = select.select([], [fd], [], remaining())
|
|
323
|
+
if not writable:
|
|
324
|
+
raise DriverFailure('helper_timeout')
|
|
325
|
+
offset += os.write(fd, request[offset:])
|
|
326
|
+
output = bytearray()
|
|
327
|
+
while b'\n' not in output:
|
|
328
|
+
readable, _, _ = select.select([fd], [], [], remaining())
|
|
329
|
+
if not readable:
|
|
330
|
+
raise DriverFailure('helper_timeout')
|
|
331
|
+
chunk = os.read(fd, OUTPUT_LIMIT + 1 - len(output))
|
|
332
|
+
if not chunk:
|
|
333
|
+
raise DriverFailure('helper_failed')
|
|
334
|
+
output.extend(chunk)
|
|
335
|
+
if len(output) > OUTPUT_LIMIT:
|
|
336
|
+
raise DriverFailure('helper_failed')
|
|
337
|
+
line, _, rest = bytes(output).partition(b'\n')
|
|
338
|
+
# Preserve pairs to reject duplicate keys as well as extra keys and
|
|
339
|
+
# non-boolean values. Only this receipt can acknowledge dispatch.
|
|
340
|
+
receipt = json.loads(line.decode('utf-8'), object_pairs_hook=list)
|
|
341
|
+
if rest or receipt != [('opened', True)] or type(receipt[0][1]) is not bool:
|
|
342
|
+
raise DriverFailure('helper_failed')
|
|
343
|
+
except (OSError, ValueError):
|
|
344
|
+
raise DriverFailure('helper_failed') from None
|
|
345
|
+
|
|
346
|
+
def select_profession(self, nodes, choice):
|
|
347
|
+
# Primary selection path: the observed radio's WINDOW extents plus its
|
|
348
|
+
# top-level frame's SCREEN origin. Never reuse guessed coordinates.
|
|
349
|
+
if not (os.path.isfile(XDOTOOL) and os.access(XDOTOOL, os.X_OK)):
|
|
350
|
+
raise DriverFailure('helper_missing')
|
|
351
|
+
frames = [nodes[i] for i in choice['ancestors'] if nodes[i]['role'] == 'frame'
|
|
352
|
+
and available(nodes[i], enabled=False)]
|
|
353
|
+
frame = unique(frames, 'profession_geometry_invalid')
|
|
354
|
+
for control in (choice, frame):
|
|
355
|
+
if 'Component' not in control['node'].get_interfaces():
|
|
356
|
+
raise DriverFailure('profession_geometry_invalid')
|
|
357
|
+
target = self.api.Component.get_extents(choice['node'], self.api.CoordType.WINDOW)
|
|
358
|
+
bounds = self.api.Component.get_extents(frame['node'], self.api.CoordType.SCREEN)
|
|
359
|
+
if any(type(value) is not int for rect in (target, bounds)
|
|
360
|
+
for value in (rect.x, rect.y, rect.width, rect.height)):
|
|
361
|
+
raise DriverFailure('profession_geometry_invalid')
|
|
362
|
+
if (target.x < 0 or target.y < 0 or target.width <= 0 or target.height <= 0
|
|
363
|
+
or bounds.width <= 0 or bounds.height <= 0
|
|
364
|
+
or target.x + target.width > bounds.width
|
|
365
|
+
or target.y + target.height > bounds.height):
|
|
366
|
+
raise DriverFailure('profession_geometry_invalid')
|
|
367
|
+
x = bounds.x + target.x + target.width // 2
|
|
368
|
+
y = bounds.y + target.y + target.height // 2
|
|
369
|
+
if x < 0 or y < 0:
|
|
370
|
+
raise DriverFailure('profession_geometry_invalid')
|
|
371
|
+
# Public Component.contains is a hit test on this actual accessible.
|
|
372
|
+
contains = getattr(self.api.Component, 'contains', None)
|
|
373
|
+
if contains is not None and not contains(choice['node'], x, y, self.api.CoordType.SCREEN):
|
|
374
|
+
raise DriverFailure('profession_geometry_invalid')
|
|
375
|
+
try:
|
|
376
|
+
subprocess.run([XDOTOOL, 'mousemove', str(x), str(y), 'click', '1'],
|
|
377
|
+
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
|
378
|
+
stderr=subprocess.DEVNULL, env=self.environment(),
|
|
379
|
+
timeout=self.remaining(), check=True, shell=False)
|
|
380
|
+
except subprocess.TimeoutExpired:
|
|
381
|
+
raise DriverFailure('helper_timeout') from None
|
|
382
|
+
except (OSError, subprocess.CalledProcessError):
|
|
383
|
+
raise DriverFailure('helper_failed') from None
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def available(node, enabled=True):
|
|
387
|
+
return (node['showing'] and node['visible']
|
|
388
|
+
and (not enabled or (node['enabled'] and node['sensitive'])))
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def controls(nodes, names, roles=BUTTONS, enabled=True):
|
|
392
|
+
return [n for n in nodes if available(n, enabled)
|
|
393
|
+
and n['role'] in roles and n['name'] in names]
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def unique(nodes, code):
|
|
397
|
+
if len(nodes) != 1:
|
|
398
|
+
raise DriverFailure(code)
|
|
399
|
+
return nodes[0]
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def editor_roots(nodes):
|
|
403
|
+
# Chromium exposes an editable entry and editable paragraphs inside it.
|
|
404
|
+
# Select the containing editor by live ancestry, not role priority or order.
|
|
405
|
+
eligible = {i: n for i, n in enumerate(nodes) if available(n) and n['editable']}
|
|
406
|
+
return [n for n in eligible.values()
|
|
407
|
+
if not any(parent in eligible for parent in n['ancestors'])]
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def composer(nodes):
|
|
411
|
+
# Separate editors (including dialogs) remain ambiguous and cannot authorize input.
|
|
412
|
+
return unique(editor_roots(nodes), 'composer_missing_or_ambiguous')
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def matches_prompt(text, prompt):
|
|
416
|
+
# Same bounded representation as native exact_prompt correlation. Do not
|
|
417
|
+
# trim text or alter the prompt handed to MST for the deep link.
|
|
418
|
+
return text == prompt or text == prompt + '\n'
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
class Driver:
|
|
422
|
+
def __init__(self, desktop, timeout_ms, max_actions):
|
|
423
|
+
self.desktop = desktop
|
|
424
|
+
self.started = time.monotonic()
|
|
425
|
+
self.deadline = self.started + timeout_ms / 1000
|
|
426
|
+
self.desktop.deadline = self.deadline
|
|
427
|
+
self.max_actions = max_actions
|
|
428
|
+
self.actions = 0
|
|
429
|
+
self.phase = None
|
|
430
|
+
self.step = None
|
|
431
|
+
self.last_snapshot = None
|
|
432
|
+
|
|
433
|
+
def check(self):
|
|
434
|
+
if time.monotonic() >= self.deadline:
|
|
435
|
+
raise DriverFailure('deadline_exceeded')
|
|
436
|
+
|
|
437
|
+
def action(self, operation):
|
|
438
|
+
self.check()
|
|
439
|
+
if self.actions >= self.max_actions:
|
|
440
|
+
raise DriverFailure('action_budget_exhausted')
|
|
441
|
+
self.actions += 1
|
|
442
|
+
operation()
|
|
443
|
+
|
|
444
|
+
def click(self, node, actions=frozenset({'click', 'press'})):
|
|
445
|
+
self.action(lambda: self.desktop.activate(node, actions))
|
|
446
|
+
|
|
447
|
+
def snapshot(self):
|
|
448
|
+
self.check()
|
|
449
|
+
nodes = self.desktop.snapshot()
|
|
450
|
+
self.last_snapshot = nodes
|
|
451
|
+
self.check()
|
|
452
|
+
return nodes
|
|
453
|
+
|
|
454
|
+
def wait(self, predicate, polls=100):
|
|
455
|
+
# Poll only; an acknowledged action that does not change state is NOT retried.
|
|
456
|
+
# The cold-app wait gets 30 seconds; all waits share the overall deadline.
|
|
457
|
+
wait_deadline = min(self.deadline, time.monotonic() + polls * 0.1)
|
|
458
|
+
ambiguity = None
|
|
459
|
+
for _ in range(polls):
|
|
460
|
+
try:
|
|
461
|
+
nodes = self.snapshot()
|
|
462
|
+
except DriverFailure as error:
|
|
463
|
+
if str(error) == 'deadline_exceeded' and ambiguity is not None:
|
|
464
|
+
raise ambiguity from None
|
|
465
|
+
raise
|
|
466
|
+
if time.monotonic() >= wait_deadline:
|
|
467
|
+
break
|
|
468
|
+
try:
|
|
469
|
+
matched = predicate(nodes)
|
|
470
|
+
except DriverFailure as error:
|
|
471
|
+
# Only predicate reads may tolerate transient editor/Send overlap.
|
|
472
|
+
# Snapshot failures and every action remain fail-closed, with no retry.
|
|
473
|
+
if str(error) not in {'composer_missing_or_ambiguous', 'send_missing_or_ambiguous'}:
|
|
474
|
+
raise
|
|
475
|
+
if ambiguity is None:
|
|
476
|
+
ambiguity = error
|
|
477
|
+
else:
|
|
478
|
+
ambiguity = None
|
|
479
|
+
if matched:
|
|
480
|
+
return nodes
|
|
481
|
+
time.sleep(min(0.1, max(0, wait_deadline - time.monotonic())))
|
|
482
|
+
if ambiguity is not None:
|
|
483
|
+
raise ambiguity
|
|
484
|
+
raise DriverFailure('state_transition_unobserved')
|
|
485
|
+
|
|
486
|
+
def selected(self, nodes, surface):
|
|
487
|
+
switches = controls(nodes, set(MODE_LABELS.values()))
|
|
488
|
+
return len(switches) == 1 and switches[0]['name'] == MODE_LABELS[surface]
|
|
489
|
+
|
|
490
|
+
def select_surface(self, nodes, surface):
|
|
491
|
+
if self.selected(nodes, surface):
|
|
492
|
+
return nodes
|
|
493
|
+
switch = unique(controls(nodes, set(MODE_LABELS.values())), 'mode_missing_or_ambiguous')
|
|
494
|
+
self.click(switch, {'open'})
|
|
495
|
+
nodes = self.wait(lambda ns: bool(controls(ns, {MENU_LABELS[surface]}, {'menu item'})))
|
|
496
|
+
item = unique(controls(nodes, {MENU_LABELS[surface]}, {'menu item'}), 'surface_item_ambiguous')
|
|
497
|
+
self.click(item, {'select'})
|
|
498
|
+
return self.wait(lambda ns: self.selected(ns, surface))
|
|
499
|
+
|
|
500
|
+
def ready(self, nodes, surface):
|
|
501
|
+
if not self.selected(nodes, surface):
|
|
502
|
+
return False
|
|
503
|
+
editors = editor_roots(nodes)
|
|
504
|
+
sends = controls(nodes, {'Send'}, enabled=False)
|
|
505
|
+
if not editors or not sends:
|
|
506
|
+
return False
|
|
507
|
+
unique(editors, 'composer_missing_or_ambiguous')
|
|
508
|
+
unique(sends, 'send_missing_or_ambiguous')
|
|
509
|
+
return True
|
|
510
|
+
|
|
511
|
+
def prepare(self, surface):
|
|
512
|
+
if surface not in SURFACES:
|
|
513
|
+
raise DriverFailure('invalid_surface')
|
|
514
|
+
self.desktop.require_helpers()
|
|
515
|
+
self.phase = 'waiting-for-initial-ui'
|
|
516
|
+
nodes = self.wait(lambda ns: bool(controls(ns, {'Engineering'}, {'radio button', 'toggle button'})
|
|
517
|
+
or controls(ns, {'Leave a note on my Desktop', 'Go to ChatGPT',
|
|
518
|
+
*MODE_LABELS.values()})), polls=300)
|
|
519
|
+
engineering = controls(nodes, {'Engineering'}, {'radio button', 'toggle button'})
|
|
520
|
+
if engineering:
|
|
521
|
+
self.phase = 'profession-selected'
|
|
522
|
+
choice = unique(engineering, 'profession_ambiguous')
|
|
523
|
+
if not choice['selected']:
|
|
524
|
+
self.action(lambda: self.desktop.select_profession(nodes, choice))
|
|
525
|
+
# Chromium can keep the checked bit stale after a real selection.
|
|
526
|
+
# Observe the same profession page and enabled Continue, not that bit.
|
|
527
|
+
self.phase = 'continue-ready'
|
|
528
|
+
nodes = self.wait(lambda ns: len(controls(
|
|
529
|
+
ns, {'Engineering'}, {'radio button', 'toggle button'})) == 1
|
|
530
|
+
and bool(controls(ns, {'Continue'})))
|
|
531
|
+
self.click(unique(controls(nodes, {'Continue'}), 'continue_missing_or_ambiguous'))
|
|
532
|
+
self.phase = 'intro-dismiss'
|
|
533
|
+
nodes = self.wait(lambda ns: not controls(ns, {'Engineering'}, {'radio button', 'toggle button'})
|
|
534
|
+
and bool(controls(ns, {'Leave a note on my Desktop', 'Go to ChatGPT',
|
|
535
|
+
*MODE_LABELS.values()})))
|
|
536
|
+
self.phase = 'intro-dismiss'
|
|
537
|
+
# Only skip the observed, specific product-introduction screen.
|
|
538
|
+
if (controls(nodes, {'Leave a note on my Desktop'})
|
|
539
|
+
and controls(nodes, {'Turn this spreadsheet into a chart'})):
|
|
540
|
+
self.click(unique(controls(nodes, {'Skip'}), 'skip_missing_or_ambiguous'))
|
|
541
|
+
nodes = self.wait(lambda ns: bool(controls(ns, {'Go to ChatGPT'})))
|
|
542
|
+
if controls(nodes, {'Go to ChatGPT'}) and controls(nodes, {'Keep setting up'}):
|
|
543
|
+
self.click(unique(controls(nodes, {'Go to ChatGPT'}), 'intro_confirmation_ambiguous'))
|
|
544
|
+
nodes = self.wait(lambda ns: bool(controls(ns, set(MODE_LABELS.values()))))
|
|
545
|
+
self.phase = 'surface'
|
|
546
|
+
self.select_surface(nodes, surface)
|
|
547
|
+
self.phase = 'composer'
|
|
548
|
+
self.wait(lambda ns: self.ready(ns, surface))
|
|
549
|
+
self.step = 'draft-open'
|
|
550
|
+
self.action(lambda: self.desktop.open_prompt(''))
|
|
551
|
+
self.step = 'draft-surface'
|
|
552
|
+
# Setup has no user prompt to match. A new chat can expose a visible
|
|
553
|
+
# placeholder as Text; ready means controls/surface, not verified emptiness.
|
|
554
|
+
nodes = self.wait(lambda ns: any(self.ready(ns, candidate) for candidate in SURFACES),
|
|
555
|
+
polls=DRAFT_POLLS)
|
|
556
|
+
self.select_surface(nodes, surface)
|
|
557
|
+
self.step = 'draft-readback'
|
|
558
|
+
self.wait(lambda ns: self.ready(ns, surface))
|
|
559
|
+
self.step = None
|
|
560
|
+
return self.receipt('ready', surface)
|
|
561
|
+
|
|
562
|
+
def submit(self, prompt, surface):
|
|
563
|
+
if not isinstance(prompt, str) or not prompt.strip():
|
|
564
|
+
raise DriverFailure('invalid_prompt')
|
|
565
|
+
if surface not in SURFACES:
|
|
566
|
+
raise DriverFailure('invalid_surface')
|
|
567
|
+
self.desktop.require_helpers()
|
|
568
|
+
# Setup ran once for the batch. A changed surface blocks submission.
|
|
569
|
+
self.phase = 'surface'
|
|
570
|
+
nodes = self.snapshot()
|
|
571
|
+
if not self.ready(nodes, surface):
|
|
572
|
+
raise DriverFailure('surface_mismatch')
|
|
573
|
+
self.phase = 'composer'
|
|
574
|
+
self.step = 'draft-open'
|
|
575
|
+
self.action(lambda: self.desktop.open_prompt(prompt))
|
|
576
|
+
self.step = 'draft-surface'
|
|
577
|
+
nodes = self.wait(lambda ns: any(self.ready(ns, candidate) for candidate in SURFACES)
|
|
578
|
+
and matches_prompt(self.desktop.text(composer(ns)), prompt),
|
|
579
|
+
polls=DRAFT_POLLS)
|
|
580
|
+
# A deep link may change mode. One fixed UI selection is allowed, but it
|
|
581
|
+
# must preserve the draft. Never reopen, refill, or fall back on loss.
|
|
582
|
+
self.select_surface(nodes, surface)
|
|
583
|
+
self.step = 'draft-readback'
|
|
584
|
+
nodes = self.wait(lambda ns: self.ready(ns, surface)
|
|
585
|
+
and matches_prompt(self.desktop.text(composer(ns)), prompt)
|
|
586
|
+
and len(controls(ns, {'Send'})) == 1)
|
|
587
|
+
# Exactly one send. No retry, Enter fallback, or resubmission on missing trace.
|
|
588
|
+
self.step = 'send'
|
|
589
|
+
self.click(unique(controls(nodes, {'Send'}), 'send_missing_or_ambiguous'))
|
|
590
|
+
self.step = None
|
|
591
|
+
return self.receipt('submitted', surface)
|
|
592
|
+
|
|
593
|
+
def draft_state(self):
|
|
594
|
+
# Failure-only, read-only evidence from the last complete snapshot. Never
|
|
595
|
+
# serialize nodes, names, URLs, prompt/config values, or exception details.
|
|
596
|
+
nodes = self.last_snapshot
|
|
597
|
+
if nodes is None:
|
|
598
|
+
return None
|
|
599
|
+
roots = editor_roots(nodes)
|
|
600
|
+
switches = controls(nodes, set(MODE_LABELS.values()))
|
|
601
|
+
observed = 'unknown'
|
|
602
|
+
if len(switches) > 1:
|
|
603
|
+
observed = 'ambiguous'
|
|
604
|
+
elif len(switches) == 1:
|
|
605
|
+
observed = next(surface for surface, label in MODE_LABELS.items()
|
|
606
|
+
if switches[0]['name'] == label)
|
|
607
|
+
state = {'observedSurface': observed, 'composerRootCount': len(roots),
|
|
608
|
+
'sendControlCount': len(controls(nodes, {'Send'}, enabled=False)),
|
|
609
|
+
'textReadable': False}
|
|
610
|
+
if len(roots) != 1:
|
|
611
|
+
return state
|
|
612
|
+
try:
|
|
613
|
+
text = self.desktop.text(roots[0], limit=INPUT_LIMIT)
|
|
614
|
+
if not isinstance(text, str) or len(text) > INPUT_LIMIT:
|
|
615
|
+
return state
|
|
616
|
+
encoded = text.encode('utf-8')
|
|
617
|
+
if len(encoded) > INPUT_LIMIT:
|
|
618
|
+
return state
|
|
619
|
+
# Length/counts are Unicode code points; hash is exact UTF-8 bytes.
|
|
620
|
+
state.update(textReadable=True, textLength=len(text),
|
|
621
|
+
textSha256=hashlib.sha256(encoded).hexdigest(),
|
|
622
|
+
embeddedObjectCount=text.count('\ufffc'), newlineCount=text.count('\n'))
|
|
623
|
+
except Exception:
|
|
624
|
+
# Diagnostics must not replace the original failure or expose its text.
|
|
625
|
+
pass
|
|
626
|
+
return state
|
|
627
|
+
|
|
628
|
+
def receipt(self, status, surface=None):
|
|
629
|
+
draft_state = self.draft_state() if status == 'failed' else None
|
|
630
|
+
return {'status': status, 'action_count': self.actions,
|
|
631
|
+
'duration_ms': (time.monotonic() - self.started) * 1000,
|
|
632
|
+
**({'surface': surface} if surface else {}),
|
|
633
|
+
**({'phase': self.phase} if status == 'failed' and self.phase else {}),
|
|
634
|
+
**({'step': self.step} if status == 'failed' and self.step else {}),
|
|
635
|
+
**({'draftState': draft_state} if draft_state is not None else {})}
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def main():
|
|
639
|
+
parser = argparse.ArgumentParser()
|
|
640
|
+
parser.add_argument('--mode', choices=['prepare', 'submit'], required=True)
|
|
641
|
+
parser.add_argument('--timeout-ms', type=int, required=True)
|
|
642
|
+
parser.add_argument('--max-actions', type=int, default=MAX_ACTIONS['default'])
|
|
643
|
+
parser.add_argument('--open-fd', type=int)
|
|
644
|
+
args = parser.parse_args()
|
|
645
|
+
driver = None
|
|
646
|
+
try:
|
|
647
|
+
if args.timeout_ms <= 0 or not 1 <= args.max_actions <= MAX_ACTIONS['max']:
|
|
648
|
+
raise DriverFailure('invalid_budget')
|
|
649
|
+
data = sys.stdin.buffer.read(INPUT_LIMIT + 1)
|
|
650
|
+
if len(data) > INPUT_LIMIT:
|
|
651
|
+
raise DriverFailure('input_too_large')
|
|
652
|
+
payload = json.loads(data)
|
|
653
|
+
if not isinstance(payload, dict):
|
|
654
|
+
raise DriverFailure('invalid_input')
|
|
655
|
+
desktop = Desktop()
|
|
656
|
+
desktop.open_fd = args.open_fd
|
|
657
|
+
driver = Driver(desktop, args.timeout_ms, args.max_actions)
|
|
658
|
+
surface = payload.get('surface')
|
|
659
|
+
result = driver.prepare(surface) if args.mode == 'prepare' else driver.submit(payload.get('prompt'), surface)
|
|
660
|
+
print(json.dumps(result))
|
|
661
|
+
return 0
|
|
662
|
+
except Exception as error:
|
|
663
|
+
result = driver.receipt('failed') if driver else {'status': 'failed', 'action_count': 0, 'duration_ms': 0}
|
|
664
|
+
# Desktop owns the GLib error type; never map through module globals.
|
|
665
|
+
glib_error = getattr(driver.desktop, 'glib_error', ()) if driver else ()
|
|
666
|
+
result['error'] = error_code(error, glib_error)
|
|
667
|
+
print(json.dumps(result))
|
|
668
|
+
return 1
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
if __name__ == '__main__':
|
|
672
|
+
raise SystemExit(main())
|