@zenalexa/unicli 0.223.3 → 0.224.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/README.md +3 -3
- package/README.zh-CN.md +3 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +3 -0
- package/dist/cli.js.map +1 -1
- package/dist/commands/architecture.d.ts +21 -0
- package/dist/commands/architecture.d.ts.map +1 -0
- package/dist/commands/architecture.js +47 -0
- package/dist/commands/architecture.js.map +1 -0
- package/dist/commands/compute.d.ts.map +1 -1
- package/dist/commands/compute.js +31 -6
- package/dist/commands/compute.js.map +1 -1
- package/dist/commands/doctor-compute.d.ts.map +1 -1
- package/dist/commands/doctor-compute.js +88 -1
- package/dist/commands/doctor-compute.js.map +1 -1
- package/dist/compute/action-execution.d.ts +30 -0
- package/dist/compute/action-execution.d.ts.map +1 -0
- package/dist/compute/action-execution.js +112 -0
- package/dist/compute/action-execution.js.map +1 -0
- package/dist/compute/capture-reference.d.ts.map +1 -1
- package/dist/compute/capture-reference.js +6 -1
- package/dist/compute/capture-reference.js.map +1 -1
- package/dist/compute/capture.d.ts +2 -0
- package/dist/compute/capture.d.ts.map +1 -1
- package/dist/compute/capture.js +6 -1
- package/dist/compute/capture.js.map +1 -1
- package/dist/compute/cursor-visual-style.d.ts +35 -0
- package/dist/compute/cursor-visual-style.d.ts.map +1 -0
- package/dist/compute/cursor-visual-style.js +39 -0
- package/dist/compute/cursor-visual-style.js.map +1 -0
- package/dist/compute/linux-overlay.d.ts +38 -0
- package/dist/compute/linux-overlay.d.ts.map +1 -0
- package/dist/compute/linux-overlay.js +274 -0
- package/dist/compute/linux-overlay.js.map +1 -0
- package/dist/compute/macos-overlay.d.ts +64 -0
- package/dist/compute/macos-overlay.d.ts.map +1 -0
- package/dist/compute/macos-overlay.js +590 -0
- package/dist/compute/macos-overlay.js.map +1 -0
- package/dist/compute/overlay-daemon.d.ts +47 -0
- package/dist/compute/overlay-daemon.d.ts.map +1 -0
- package/dist/compute/overlay-daemon.js +206 -0
- package/dist/compute/overlay-daemon.js.map +1 -0
- package/dist/compute/overlay.d.ts +42 -0
- package/dist/compute/overlay.d.ts.map +1 -0
- package/dist/compute/overlay.js +111 -0
- package/dist/compute/overlay.js.map +1 -0
- package/dist/compute/platform-overlays.d.ts +20 -0
- package/dist/compute/platform-overlays.d.ts.map +1 -0
- package/dist/compute/platform-overlays.js +31 -0
- package/dist/compute/platform-overlays.js.map +1 -0
- package/dist/compute/visual-timeline.d.ts +132 -0
- package/dist/compute/visual-timeline.d.ts.map +1 -0
- package/dist/compute/visual-timeline.js +431 -0
- package/dist/compute/visual-timeline.js.map +1 -0
- package/dist/compute/windows-overlay.d.ts +38 -0
- package/dist/compute/windows-overlay.d.ts.map +1 -0
- package/dist/compute/windows-overlay.js +282 -0
- package/dist/compute/windows-overlay.js.map +1 -0
- package/dist/core/architecture-tree.d.ts +68 -0
- package/dist/core/architecture-tree.d.ts.map +1 -0
- package/dist/core/architecture-tree.js +215 -0
- package/dist/core/architecture-tree.js.map +1 -0
- package/dist/discovery/core-catalog.d.ts.map +1 -1
- package/dist/discovery/core-catalog.js +14 -0
- package/dist/discovery/core-catalog.js.map +1 -1
- package/dist/discovery/loader.d.ts +12 -6
- package/dist/discovery/loader.d.ts.map +1 -1
- package/dist/discovery/loader.js +37 -10
- package/dist/discovery/loader.js.map +1 -1
- package/dist/discovery/search.d.ts +27 -28
- package/dist/discovery/search.d.ts.map +1 -1
- package/dist/discovery/search.js +118 -120
- package/dist/discovery/search.js.map +1 -1
- package/dist/fast-path/handlers/discovery.d.ts +12 -5
- package/dist/fast-path/handlers/discovery.d.ts.map +1 -1
- package/dist/fast-path/handlers/discovery.js +42 -7
- package/dist/fast-path/handlers/discovery.js.map +1 -1
- package/dist/manifest.json +1 -1
- package/dist/mcp/profiles/computer-use.d.ts.map +1 -1
- package/dist/mcp/profiles/computer-use.js +76 -8
- package/dist/mcp/profiles/computer-use.js.map +1 -1
- package/dist/registry.d.ts +14 -5
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +33 -6
- package/dist/registry.js.map +1 -1
- package/dist/transport/cascade.d.ts +1 -0
- package/dist/transport/cascade.d.ts.map +1 -1
- package/dist/transport/cascade.js +2 -2
- package/dist/transport/cascade.js.map +1 -1
- package/docs/operate/compute.md +66 -1
- package/docs/operate/troubleshooting.md +42 -0
- package/package.json +9 -5
- package/server.json +3 -3
- package/skills/unicli/SKILL.md +1 -1
- package/skills/unicli-claude-code/SKILL.md +1 -1
- package/skills/unicli-hermes/SKILL.md +1 -1
- package/dist/manifest-search.json +0 -1
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @owner src/compute/linux-overlay.ts
|
|
3
|
+
* @does Render compute visual_action evidence through a Linux GTK full-screen HUD.
|
|
4
|
+
* @needs Python 3, PyGObject GTK 3, Cairo, src/compute/overlay-daemon.ts
|
|
5
|
+
* @feeds platform compute overlay selection, doctor compute, computer-use HUD rendering
|
|
6
|
+
* @breaks A non-click-through HUD can block AT-SPI or display-server action dispatch.
|
|
7
|
+
* @invariants The HUD is visual-only, keep-above, input-shaped empty, and driven by visual_action samples.
|
|
8
|
+
* @side-effects launches a Python/GTK helper process on Linux.
|
|
9
|
+
* @perf one persistent Python process per provider instance.
|
|
10
|
+
* @concurrency daemon session serializes render requests.
|
|
11
|
+
* @test tests/unit/compute-linux-overlay.test.ts
|
|
12
|
+
* @stability experimental
|
|
13
|
+
* @since 0.224.0
|
|
14
|
+
*/
|
|
15
|
+
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
16
|
+
import { tmpdir } from "node:os";
|
|
17
|
+
import { join } from "node:path";
|
|
18
|
+
import { computeOverlayRequestFromAction, } from "./overlay.js";
|
|
19
|
+
import { StdioComputeOverlayDaemonSession, } from "./overlay-daemon.js";
|
|
20
|
+
export class LinuxGtkOverlayDaemonProvider {
|
|
21
|
+
provider = "linux-gtk";
|
|
22
|
+
platform;
|
|
23
|
+
scriptPath;
|
|
24
|
+
sessionFactory;
|
|
25
|
+
session;
|
|
26
|
+
tmpRoot;
|
|
27
|
+
lastPoint;
|
|
28
|
+
constructor(opts = {}) {
|
|
29
|
+
this.platform = opts.platform ?? process.platform;
|
|
30
|
+
this.scriptPath = opts.scriptPath;
|
|
31
|
+
this.sessionFactory = opts.sessionFactory;
|
|
32
|
+
}
|
|
33
|
+
currentPoint() {
|
|
34
|
+
return this.lastPoint;
|
|
35
|
+
}
|
|
36
|
+
async render(action) {
|
|
37
|
+
const request = computeOverlayRequestFromAction(action);
|
|
38
|
+
if (!request)
|
|
39
|
+
return { provider: this.provider, status: "not_requested" };
|
|
40
|
+
if (this.platform !== "linux") {
|
|
41
|
+
return { provider: this.provider, status: "unavailable" };
|
|
42
|
+
}
|
|
43
|
+
try {
|
|
44
|
+
const session = await this.ensureSession();
|
|
45
|
+
const status = await session.render(request, Math.max(1_000, request.duration_ms + 1_500));
|
|
46
|
+
if (status.status === "arrived") {
|
|
47
|
+
this.lastPoint = pointFromOverlayTarget(request);
|
|
48
|
+
}
|
|
49
|
+
return status;
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
return {
|
|
53
|
+
provider: this.provider,
|
|
54
|
+
status: "failed",
|
|
55
|
+
error: errorMessage(error),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
async close() {
|
|
60
|
+
await this.session?.close();
|
|
61
|
+
this.session = undefined;
|
|
62
|
+
if (this.tmpRoot) {
|
|
63
|
+
await rm(this.tmpRoot, { recursive: true, force: true });
|
|
64
|
+
this.tmpRoot = undefined;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
async ensureSession() {
|
|
68
|
+
if (this.session)
|
|
69
|
+
return this.session;
|
|
70
|
+
if (this.sessionFactory) {
|
|
71
|
+
this.session = await this.sessionFactory();
|
|
72
|
+
return this.session;
|
|
73
|
+
}
|
|
74
|
+
const scriptPath = this.scriptPath ??
|
|
75
|
+
join((this.tmpRoot = await mkdtemp(join(tmpdir(), "unicli-overlay-linux-"))), "overlay.py");
|
|
76
|
+
if (!this.scriptPath) {
|
|
77
|
+
await writeFile(scriptPath, buildLinuxOverlayPythonScript(), "utf8");
|
|
78
|
+
}
|
|
79
|
+
this.session = new StdioComputeOverlayDaemonSession("python3", [
|
|
80
|
+
scriptPath,
|
|
81
|
+
]);
|
|
82
|
+
return this.session;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
export function buildLinuxOverlayPythonScript() {
|
|
86
|
+
return String.raw `#!/usr/bin/env python3
|
|
87
|
+
import json
|
|
88
|
+
import math
|
|
89
|
+
import sys
|
|
90
|
+
import threading
|
|
91
|
+
import time
|
|
92
|
+
|
|
93
|
+
import cairo
|
|
94
|
+
import gi
|
|
95
|
+
gi.require_version("Gtk", "3.0")
|
|
96
|
+
from gi.repository import Gdk, GLib, Gtk
|
|
97
|
+
|
|
98
|
+
request = None
|
|
99
|
+
started_at = 0.0
|
|
100
|
+
trail = []
|
|
101
|
+
windows = []
|
|
102
|
+
cursor_visual_style = "mac-glass-pointer-v1"
|
|
103
|
+
cursor_skin = "mac-pointer"
|
|
104
|
+
|
|
105
|
+
def write_protocol(payload):
|
|
106
|
+
sys.stdout.write(json.dumps(payload, separators=(",", ":")) + "\n")
|
|
107
|
+
sys.stdout.flush()
|
|
108
|
+
|
|
109
|
+
def sample_at(elapsed_ms):
|
|
110
|
+
global request
|
|
111
|
+
samples = request.get("samples") or []
|
|
112
|
+
if not samples:
|
|
113
|
+
return request["target"]
|
|
114
|
+
previous = samples[0]
|
|
115
|
+
for sample in samples[1:]:
|
|
116
|
+
if float(sample.get("at_ms", 0)) >= elapsed_ms:
|
|
117
|
+
span = max(1.0, float(sample.get("at_ms", 0)) - float(previous.get("at_ms", 0)))
|
|
118
|
+
t = min(1.0, max(0.0, (elapsed_ms - float(previous.get("at_ms", 0))) / span))
|
|
119
|
+
return {
|
|
120
|
+
"at_ms": elapsed_ms,
|
|
121
|
+
"x": float(previous.get("x", 0)) + ((float(sample.get("x", 0)) - float(previous.get("x", 0))) * t),
|
|
122
|
+
"y": float(previous.get("y", 0)) + ((float(sample.get("y", 0)) - float(previous.get("y", 0))) * t),
|
|
123
|
+
}
|
|
124
|
+
previous = sample
|
|
125
|
+
return request["target"]
|
|
126
|
+
|
|
127
|
+
class OverlayWindow(Gtk.Window):
|
|
128
|
+
def __init__(self, monitor):
|
|
129
|
+
super().__init__(type=Gtk.WindowType.POPUP)
|
|
130
|
+
self.monitor = monitor
|
|
131
|
+
geometry = monitor.get_geometry()
|
|
132
|
+
scale = monitor.get_scale_factor()
|
|
133
|
+
self.origin_x = geometry.x
|
|
134
|
+
self.origin_y = geometry.y
|
|
135
|
+
self.set_app_paintable(True)
|
|
136
|
+
self.set_decorated(False)
|
|
137
|
+
self.set_keep_above(True)
|
|
138
|
+
self.set_skip_taskbar_hint(True)
|
|
139
|
+
self.set_skip_pager_hint(True)
|
|
140
|
+
self.set_accept_focus(False)
|
|
141
|
+
self.set_focus_on_map(False)
|
|
142
|
+
self.move(geometry.x, geometry.y)
|
|
143
|
+
self.resize(geometry.width * scale, geometry.height * scale)
|
|
144
|
+
self.connect("draw", self.on_draw)
|
|
145
|
+
self.connect("realize", self.on_realize)
|
|
146
|
+
|
|
147
|
+
def on_realize(self, _widget):
|
|
148
|
+
gdkwindow = self.get_window()
|
|
149
|
+
if gdkwindow is not None:
|
|
150
|
+
gdkwindow.set_override_redirect(True)
|
|
151
|
+
gdkwindow.input_shape_combine_region(cairo.Region(), 0, 0)
|
|
152
|
+
|
|
153
|
+
def on_draw(self, _widget, cr):
|
|
154
|
+
if request is None:
|
|
155
|
+
return False
|
|
156
|
+
elapsed = (time.monotonic() - started_at) * 1000.0
|
|
157
|
+
duration_ms = max(120.0, float(request.get("duration_ms", 120)))
|
|
158
|
+
press_progress = elapsed / duration_ms
|
|
159
|
+
point = sample_at(elapsed)
|
|
160
|
+
x = float(point.get("x", 0)) - self.origin_x
|
|
161
|
+
y = float(point.get("y", 0)) - self.origin_y
|
|
162
|
+
cr.set_antialias(cairo.ANTIALIAS_BEST)
|
|
163
|
+
cr.set_source_rgba(0.76, 0.60, 0.32, 0.52)
|
|
164
|
+
cr.set_line_width(2.0)
|
|
165
|
+
cr.set_line_cap(cairo.LINE_CAP_ROUND)
|
|
166
|
+
if len(trail) > 1:
|
|
167
|
+
cr.move_to(float(trail[0]["x"]) - self.origin_x, float(trail[0]["y"]) - self.origin_y)
|
|
168
|
+
for item in trail[1:]:
|
|
169
|
+
cr.line_to(float(item["x"]) - self.origin_x, float(item["y"]) - self.origin_y)
|
|
170
|
+
cr.stroke()
|
|
171
|
+
hotspot = (x, y)
|
|
172
|
+
pointerPath = [
|
|
173
|
+
(x, y),
|
|
174
|
+
(x, y + 45),
|
|
175
|
+
(x + 13, y + 32),
|
|
176
|
+
(x + 21, y + 55),
|
|
177
|
+
(x + 31, y + 51),
|
|
178
|
+
(x + 23, y + 29),
|
|
179
|
+
(x + 43, y + 29),
|
|
180
|
+
]
|
|
181
|
+
cr.move_to(pointerPath[0][0], pointerPath[0][1])
|
|
182
|
+
for px, py in pointerPath[1:]:
|
|
183
|
+
cr.line_to(px, py)
|
|
184
|
+
cr.close_path()
|
|
185
|
+
cr.set_source_rgba(0.98, 0.96, 0.90, 0.98)
|
|
186
|
+
cr.fill_preserve()
|
|
187
|
+
cr.set_source_rgba(0.09, 0.07, 0.05, 0.96)
|
|
188
|
+
cr.set_line_width(2.25)
|
|
189
|
+
cr.stroke()
|
|
190
|
+
cr.set_source_rgba(1.0, 1.0, 1.0, 0.72)
|
|
191
|
+
cr.set_line_width(1.0)
|
|
192
|
+
cr.move_to(x + 6, y + 8)
|
|
193
|
+
cr.line_to(x + 6, y + 34)
|
|
194
|
+
cr.stroke()
|
|
195
|
+
state = str(request.get("state") or "")
|
|
196
|
+
affordance = request.get("affordance") or {}
|
|
197
|
+
halo = str(affordance.get("halo") or "")
|
|
198
|
+
click_ripple = bool(affordance.get("click_ripple"))
|
|
199
|
+
is_pressure = (state == "press" or halo == "pressure-bloom" or click_ripple) and press_progress >= 0.78
|
|
200
|
+
is_busy_orbit = state == "wait" or halo == "busy-orbit"
|
|
201
|
+
if is_pressure or is_busy_orbit:
|
|
202
|
+
cr.set_source_rgba(0.76, 0.60, 0.32, 0.78)
|
|
203
|
+
cr.set_line_width(1.5)
|
|
204
|
+
cr.set_dash([4.0, 5.0] if is_busy_orbit else [])
|
|
205
|
+
cr.arc(hotspot[0], hotspot[1], 15, 0, math.pi * 2)
|
|
206
|
+
cr.stroke()
|
|
207
|
+
cr.set_dash([])
|
|
208
|
+
return False
|
|
209
|
+
|
|
210
|
+
def tick():
|
|
211
|
+
global request, trail
|
|
212
|
+
if request is None:
|
|
213
|
+
return True
|
|
214
|
+
elapsed = (time.monotonic() - started_at) * 1000.0
|
|
215
|
+
duration = max(120.0, float(request.get("duration_ms", 120)))
|
|
216
|
+
point = sample_at(elapsed)
|
|
217
|
+
trail.append(point)
|
|
218
|
+
del trail[:-10]
|
|
219
|
+
for window in windows:
|
|
220
|
+
window.queue_draw()
|
|
221
|
+
if elapsed >= duration:
|
|
222
|
+
write_protocol({"provider": "linux-gtk", "status": "arrived", "acknowledged_at_ms": int(duration)})
|
|
223
|
+
request = None
|
|
224
|
+
return True
|
|
225
|
+
|
|
226
|
+
def render(next_request):
|
|
227
|
+
global request, started_at, trail
|
|
228
|
+
request = next_request
|
|
229
|
+
started_at = time.monotonic()
|
|
230
|
+
trail = []
|
|
231
|
+
return False
|
|
232
|
+
|
|
233
|
+
def read_stdin():
|
|
234
|
+
for line in sys.stdin:
|
|
235
|
+
try:
|
|
236
|
+
parsed = json.loads(line)
|
|
237
|
+
GLib.idle_add(render, parsed)
|
|
238
|
+
except Exception:
|
|
239
|
+
write_protocol({"provider": "linux-gtk", "status": "failed", "error": "invalid request"})
|
|
240
|
+
GLib.idle_add(Gtk.main_quit)
|
|
241
|
+
|
|
242
|
+
display = Gdk.Display.get_default()
|
|
243
|
+
if display is None:
|
|
244
|
+
write_protocol({"provider": "linux-gtk", "status": "failed", "error": "no display"})
|
|
245
|
+
sys.exit(1)
|
|
246
|
+
for index in range(display.get_n_monitors()):
|
|
247
|
+
window = OverlayWindow(display.get_monitor(index))
|
|
248
|
+
windows.append(window)
|
|
249
|
+
window.show_all()
|
|
250
|
+
|
|
251
|
+
GLib.timeout_add(16, tick)
|
|
252
|
+
write_protocol({"provider": "linux-gtk", "status": "ready"})
|
|
253
|
+
'{"provider":"linux-gtk","status":"ready"}'
|
|
254
|
+
threading.Thread(target=read_stdin, daemon=True).start()
|
|
255
|
+
Gtk.main()
|
|
256
|
+
`;
|
|
257
|
+
}
|
|
258
|
+
function pointFromOverlayTarget(request) {
|
|
259
|
+
return {
|
|
260
|
+
x: request.target.x,
|
|
261
|
+
y: request.target.y,
|
|
262
|
+
coordinate_space: {
|
|
263
|
+
kind: "screen-pixels",
|
|
264
|
+
origin: "top-left",
|
|
265
|
+
...(request.target.screenIndex !== undefined
|
|
266
|
+
? { screenIndex: request.target.screenIndex }
|
|
267
|
+
: {}),
|
|
268
|
+
},
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
function errorMessage(error) {
|
|
272
|
+
return error instanceof Error ? error.message : String(error);
|
|
273
|
+
}
|
|
274
|
+
//# sourceMappingURL=linux-overlay.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"linux-overlay.js","sourceRoot":"","sources":["../../src/compute/linux-overlay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EACL,+BAA+B,GAGhC,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,gCAAgC,GAEjC,MAAM,qBAAqB,CAAC;AAa7B,MAAM,OAAO,6BAA6B;IAC/B,QAAQ,GAAG,WAAW,CAAC;IAEf,QAAQ,CAAkB;IAC1B,UAAU,CAAqB;IAC/B,cAAc,CAEjB;IACN,OAAO,CAA0C;IACjD,OAAO,CAAqB;IAC5B,SAAS,CAAuC;IAExD,YAAY,OAA6C,EAAE;QACzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC;QAClD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;IAC5C,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,MAAM,CACV,MAA2B;QAE3B,MAAM,OAAO,GAAG,+BAA+B,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;QAC1E,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YAC9B,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;QAC5D,CAAC;QACD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CACjC,OAAO,EACP,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,WAAW,GAAG,KAAK,CAAC,CAC7C,CAAC;YACF,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChC,IAAI,CAAC,SAAS,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;YACnD,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,QAAQ;gBAChB,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC;aAC3B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QACzB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAC3B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,OAAO,CAAC;QACtC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3C,OAAO,IAAI,CAAC,OAAO,CAAC;QACtB,CAAC;QAED,MAAM,UAAU,GACd,IAAI,CAAC,UAAU;YACf,IAAI,CACF,CAAC,IAAI,CAAC,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,CAAC,CAAC,CAAC,EACvE,YAAY,CACb,CAAC;QACJ,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,SAAS,CAAC,UAAU,EAAE,6BAA6B,EAAE,EAAE,MAAM,CAAC,CAAC;QACvE,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,gCAAgC,CAAC,SAAS,EAAE;YAC7D,UAAU;SACX,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAED,MAAM,UAAU,6BAA6B;IAC3C,OAAO,MAAM,CAAC,GAAG,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0KlB,CAAC;AACF,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAA8B;IAE9B,OAAO;QACL,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACnB,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACnB,gBAAgB,EAAE;YAChB,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,UAAU;YAClB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS;gBAC1C,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE;gBAC7C,CAAC,CAAC,EAAE,CAAC;SACR;KACF,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @owner src/compute/macos-overlay.ts
|
|
3
|
+
* @does Render compute visual_action evidence through a macOS AppKit full-screen HUD.
|
|
4
|
+
* @needs Swift runtime, AppKit, src/compute/overlay.ts
|
|
5
|
+
* @feeds future computer-use native overlay orchestration
|
|
6
|
+
* @breaks Non-click-through or key windows can steal focus from the real action transport.
|
|
7
|
+
* @invariants The overlay is visual-only, click-through, non-key, and spans every visible screen.
|
|
8
|
+
* @side-effects launches a Swift/AppKit helper on macOS and may keep a daemon process per provider.
|
|
9
|
+
* @perf one persistent Swift process per daemon provider; one-shot provider spawns per render.
|
|
10
|
+
* @concurrency daemon provider serializes render requests; native AppKit state is process-local
|
|
11
|
+
* @test tests/unit/compute-macos-overlay-swift.test.ts
|
|
12
|
+
* @stability experimental
|
|
13
|
+
* @since 0.224.0
|
|
14
|
+
*/
|
|
15
|
+
import { type ComputeOverlayProvider } from "./overlay.js";
|
|
16
|
+
import { type ComputeOverlayDaemonSession } from "./overlay-daemon.js";
|
|
17
|
+
import type { ComputeVisualAction, ComputeVisualCursorPoint, ComputeVisualOverlayStatus } from "./visual-timeline.js";
|
|
18
|
+
export interface MacosOverlayShell {
|
|
19
|
+
run(command: string, args: readonly string[], opts?: {
|
|
20
|
+
input?: string;
|
|
21
|
+
timeoutMs?: number;
|
|
22
|
+
}): Promise<{
|
|
23
|
+
stdout: string;
|
|
24
|
+
stderr: string;
|
|
25
|
+
}>;
|
|
26
|
+
}
|
|
27
|
+
export interface MacosAppKitOverlayProviderOptions {
|
|
28
|
+
platform?: NodeJS.Platform;
|
|
29
|
+
shell?: MacosOverlayShell;
|
|
30
|
+
scriptPath?: string;
|
|
31
|
+
}
|
|
32
|
+
export type MacosOverlayDaemonSession = ComputeOverlayDaemonSession;
|
|
33
|
+
export interface MacosAppKitOverlayDaemonProviderOptions {
|
|
34
|
+
platform?: NodeJS.Platform;
|
|
35
|
+
scriptPath?: string;
|
|
36
|
+
sessionFactory?: () => Promise<MacosOverlayDaemonSession>;
|
|
37
|
+
}
|
|
38
|
+
export declare class MacosAppKitOverlayProvider implements ComputeOverlayProvider {
|
|
39
|
+
readonly provider = "macos-appkit";
|
|
40
|
+
private readonly platform;
|
|
41
|
+
private readonly shell;
|
|
42
|
+
private readonly scriptPath;
|
|
43
|
+
private lastPoint;
|
|
44
|
+
constructor(opts?: MacosAppKitOverlayProviderOptions);
|
|
45
|
+
currentPoint(): ComputeVisualCursorPoint | undefined;
|
|
46
|
+
render(action: ComputeVisualAction): Promise<ComputeVisualOverlayStatus>;
|
|
47
|
+
}
|
|
48
|
+
export declare class MacosAppKitOverlayDaemonProvider implements ComputeOverlayProvider {
|
|
49
|
+
readonly provider = "macos-appkit";
|
|
50
|
+
private readonly platform;
|
|
51
|
+
private readonly scriptPath;
|
|
52
|
+
private readonly sessionFactory;
|
|
53
|
+
private session;
|
|
54
|
+
private tmpRoot;
|
|
55
|
+
private lastPoint;
|
|
56
|
+
constructor(opts?: MacosAppKitOverlayDaemonProviderOptions);
|
|
57
|
+
currentPoint(): ComputeVisualCursorPoint | undefined;
|
|
58
|
+
render(action: ComputeVisualAction): Promise<ComputeVisualOverlayStatus>;
|
|
59
|
+
close(): Promise<void>;
|
|
60
|
+
private ensureSession;
|
|
61
|
+
}
|
|
62
|
+
export declare function buildMacosOverlaySwiftScript(): string;
|
|
63
|
+
export declare function buildMacosOverlayDaemonSwiftScript(): string;
|
|
64
|
+
//# sourceMappingURL=macos-overlay.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"macos-overlay.d.ts","sourceRoot":"","sources":["../../src/compute/macos-overlay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAOH,OAAO,EAGL,KAAK,sBAAsB,EAC5B,MAAM,cAAc,CAAC;AACtB,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EACV,mBAAmB,EACnB,wBAAwB,EACxB,0BAA0B,EAC3B,MAAM,sBAAsB,CAAC;AAE9B,MAAM,WAAW,iBAAiB;IAChC,GAAG,CACD,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAC5C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC;IAC3B,KAAK,CAAC,EAAE,iBAAiB,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,yBAAyB,GAAG,2BAA2B,CAAC;AAEpE,MAAM,WAAW,uCAAuC;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,OAAO,CAAC,yBAAyB,CAAC,CAAC;CAC3D;AAED,qBAAa,0BAA2B,YAAW,sBAAsB;IACvE,QAAQ,CAAC,QAAQ,kBAAkB;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAkB;IAC3C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoB;IAC1C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,SAAS,CAAuC;gBAE5C,IAAI,GAAE,iCAAsC;IAMxD,YAAY,IAAI,wBAAwB,GAAG,SAAS;IAI9C,MAAM,CACV,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,0BAA0B,CAAC;CAwCvC;AAED,qBAAa,gCAAiC,YAAW,sBAAsB;IAC7E,QAAQ,CAAC,QAAQ,kBAAkB;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAkB;IAC3C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAEjB;IACd,OAAO,CAAC,OAAO,CAAwC;IACvD,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,SAAS,CAAuC;gBAE5C,IAAI,GAAE,uCAA4C;IAM9D,YAAY,IAAI,wBAAwB,GAAG,SAAS;IAI9C,MAAM,CACV,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,0BAA0B,CAAC;IA4BhC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YASd,aAAa;CAmB5B;AAED,wBAAgB,4BAA4B,IAAI,MAAM,CAyLrD;AAED,wBAAgB,kCAAkC,IAAI,MAAM,CA8M3D"}
|