@phren/cli 0.3.56 → 0.3.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bridge/claude-suggestion.d.ts +9 -0
- package/dist/bridge/claude-suggestion.js +30 -4
- package/dist/bridge/computer-identity.d.ts +4 -2
- package/dist/bridge/computer-identity.js +7 -3
- package/dist/bridge/conductor-group.d.ts +1 -1
- package/dist/bridge/conductor-group.js +2 -2
- package/dist/bridge/conductor-sets.d.ts +6 -0
- package/dist/bridge/conductor-sets.js +8 -2
- package/dist/bridge/dispatch-command.js +3 -3
- package/dist/bridge/hand-off.d.ts +1 -0
- package/dist/bridge/hand-off.js +5 -1
- package/dist/bridge/pair.d.ts +22 -0
- package/dist/bridge/pair.js +74 -1
- package/dist/bridge/prompt-deliveries.d.ts +9 -0
- package/dist/bridge/prompt-deliveries.js +67 -4
- package/dist/bridge/server-launch.d.ts +4 -0
- package/dist/bridge/server-launch.js +4 -1
- package/dist/bridge/server-pane-routes.d.ts +12 -0
- package/dist/bridge/server-pane-routes.js +112 -0
- package/dist/bridge/server-routes.d.ts +2 -0
- package/dist/bridge/server-routes.js +1 -1
- package/dist/bridge/server.js +8 -2
- package/dist/bridge/session-move.d.ts +25 -0
- package/dist/bridge/session-move.js +128 -9
- package/dist/bridge/simulator-stream.d.ts +118 -0
- package/dist/bridge/simulator-stream.js +284 -0
- package/dist/bridge/simulators.d.ts +12 -0
- package/dist/bridge/simulators.js +32 -26
- package/dist/bridge/speech-transcribe.js +28 -3
- package/dist/bridge/terminal-herdr.js +28 -8
- package/dist/bridge-hook.mjs +1941 -1289
- package/dist/cli-registry.js +2 -2
- package/dist/modules/registry.js +2 -2
- package/dist/tools/dispatch.js +3 -2
- package/package.json +1 -1
- package/starter/global/skills/conductor/SKILL.md +7 -0
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
import { spawn as spawnChild } from "node:child_process";
|
|
2
|
+
import { BridgeError } from "./protocol.js";
|
|
3
|
+
import { bootedSimulators, SIMULATOR_UDID, verifiedHelper } from "./simulators.js";
|
|
4
|
+
/**
|
|
5
|
+
* `WS /v1/simulators/stream?udid=&fps=&size=&quality=`: one booted
|
|
6
|
+
* simulator's screen, live. The native helper (native/simcast.swift) reads the
|
|
7
|
+
* device's framebuffer through CoreSimulator, as `simctl io` does, and writes
|
|
8
|
+
* each changed frame as a length-prefixed JPEG; this side forwards them.
|
|
9
|
+
*
|
|
10
|
+
* Why JPEG frames from the framebuffer, not the alternatives:
|
|
11
|
+
* - `simctl io screenshot` took about 1.1 s a shot (Xcode 27.1, M1 Max),
|
|
12
|
+
* which is today's 1 fps poll and no more.
|
|
13
|
+
* - `simctl io recordVideo` writes a QuickTime movie that is only readable
|
|
14
|
+
* once finalized, so it cannot feed a live socket.
|
|
15
|
+
* - ScreenCaptureKit on the Simulator window needs a Screen Recording grant
|
|
16
|
+
* (which macOS re-asks for monthly), captures the window chrome, and stops
|
|
17
|
+
* when the window is minimized; the framebuffer needs no grant.
|
|
18
|
+
* - H.264 halves the bytes but ties frames together: a dropped frame corrupts
|
|
19
|
+
* the picture until the next keyframe, so "always the latest frame" would
|
|
20
|
+
* need keyframe requests across the socket. Every JPEG stands alone, so a
|
|
21
|
+
* frame the phone can't take yet is simply replaced by the next one.
|
|
22
|
+
* - The helper sends only frames whose framebuffer changed (its IOSurface
|
|
23
|
+
* seed moved): a still screen costs nothing on the wire.
|
|
24
|
+
*
|
|
25
|
+
* Frames: a text `{ type: "start", format: "jpeg", fps, size, quality, ack }`,
|
|
26
|
+
* then one binary message per frame, each a whole JPEG. A phone that asks
|
|
27
|
+
* with `ack=1` sends `{ type: "ack" }` for each frame it receives, and at
|
|
28
|
+
* most two frames are unacknowledged: the SSH channel and TCP buffers between
|
|
29
|
+
* here and the phone then never hold more than that, so what the phone shows
|
|
30
|
+
* is at most two frames old however slow the link. Without it, a frame is
|
|
31
|
+
* done once written to the gateway's socket. When the stream
|
|
32
|
+
* ends, the socket closes with the reason (1008 for a bad request, 1011 when
|
|
33
|
+
* capture failed); the phone then falls back to `/v1/simulators/screenshot`.
|
|
34
|
+
* Closing the socket stops the helper. Taps and keys stay on
|
|
35
|
+
* `/v1/simulators/action`.
|
|
36
|
+
*/
|
|
37
|
+
/** A JPEG larger than this is a broken helper, not a frame. */
|
|
38
|
+
export const MAX_FRAME_BYTES = 4 * 1024 * 1024;
|
|
39
|
+
/** The wire budget: about 24 Mbit/s, so animations at full rate on Wi-Fi, and
|
|
40
|
+
* the phone asks for a smaller `size` on a slower link. */
|
|
41
|
+
export const STREAM_BYTES_PER_SECOND = 3 * 1024 * 1024;
|
|
42
|
+
/** Each stream runs its own helper; a few phones may watch at once. */
|
|
43
|
+
export const MAX_STREAMS = 4;
|
|
44
|
+
const OPEN = 1;
|
|
45
|
+
/** Frames in flight when the phone acknowledges them: one decoding on the
|
|
46
|
+
* phone while the next crosses the network. */
|
|
47
|
+
export const ACK_WINDOW = 2;
|
|
48
|
+
/** The query, validated before anything runs. */
|
|
49
|
+
export function streamOptions(query) {
|
|
50
|
+
const udid = query.get("udid") ?? "";
|
|
51
|
+
if (!SIMULATOR_UDID.test(udid))
|
|
52
|
+
throw new BridgeError(400, "Invalid simulator identifier.");
|
|
53
|
+
const int = (name, fallback, min, max) => {
|
|
54
|
+
const raw = query.get(name);
|
|
55
|
+
if (raw === null)
|
|
56
|
+
return fallback;
|
|
57
|
+
if (!/^\d{1,5}$/.test(raw) || Number(raw) < min || Number(raw) > max)
|
|
58
|
+
throw new BridgeError(400, `${name} must be a whole number from ${min} to ${max}.`);
|
|
59
|
+
return Number(raw);
|
|
60
|
+
};
|
|
61
|
+
// size is the longest side in pixels; quality is the JPEG quality in percent.
|
|
62
|
+
const ack = query.get("ack");
|
|
63
|
+
if (ack !== null && ack !== "1" && ack !== "0")
|
|
64
|
+
throw new BridgeError(400, "ack must be 1 or 0.");
|
|
65
|
+
return { udid: udid.toUpperCase(), fps: int("fps", 24, 1, 30), size: int("size", 1280, 240, 1600), quality: int("quality", 60, 30, 90), ack: ack === "1" };
|
|
66
|
+
}
|
|
67
|
+
/** Splits the helper's stdout into frames: a 4-byte big-endian length, then
|
|
68
|
+
* that many bytes. Pipes deliver arbitrary chunks, so a frame may span many. */
|
|
69
|
+
export class FrameReader {
|
|
70
|
+
buffered = [];
|
|
71
|
+
bytes = 0;
|
|
72
|
+
push(chunk) {
|
|
73
|
+
this.buffered.push(chunk);
|
|
74
|
+
this.bytes += chunk.length;
|
|
75
|
+
const frames = [];
|
|
76
|
+
while (this.bytes >= 4) {
|
|
77
|
+
const head = this.buffered[0].length >= 4 ? this.buffered[0] : Buffer.concat(this.buffered);
|
|
78
|
+
const length = head.readUInt32BE(0);
|
|
79
|
+
if (length === 0 || length > MAX_FRAME_BYTES)
|
|
80
|
+
throw new Error(`frame of ${length} bytes`);
|
|
81
|
+
if (this.bytes < 4 + length)
|
|
82
|
+
break;
|
|
83
|
+
const all = this.buffered.length === 1 ? this.buffered[0] : Buffer.concat(this.buffered);
|
|
84
|
+
frames.push(all.subarray(4, 4 + length));
|
|
85
|
+
const rest = all.subarray(4 + length);
|
|
86
|
+
this.buffered = rest.length ? [rest] : [];
|
|
87
|
+
this.bytes = rest.length;
|
|
88
|
+
}
|
|
89
|
+
return frames;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
/** Sends the newest frame only. At most `window` frames are in flight (until
|
|
93
|
+
* written, or until acknowledged when `acked`); frames that arrive meanwhile
|
|
94
|
+
* replace the one waiting, so a slow link sees fewer frames, never older
|
|
95
|
+
* ones. A byte budget, refilled continuously with one second of burst, bounds
|
|
96
|
+
* the bitrate the same way. */
|
|
97
|
+
export class LatestFrame {
|
|
98
|
+
send;
|
|
99
|
+
options;
|
|
100
|
+
now;
|
|
101
|
+
waiting;
|
|
102
|
+
inFlight = 0;
|
|
103
|
+
tokens;
|
|
104
|
+
refilled;
|
|
105
|
+
timer;
|
|
106
|
+
closed = false;
|
|
107
|
+
sent = 0;
|
|
108
|
+
dropped = 0;
|
|
109
|
+
constructor(send, options = {}, now = Date.now) {
|
|
110
|
+
this.send = send;
|
|
111
|
+
this.options = options;
|
|
112
|
+
this.now = now;
|
|
113
|
+
this.tokens = this.bytesPerSecond;
|
|
114
|
+
this.refilled = now();
|
|
115
|
+
}
|
|
116
|
+
offer(frame) {
|
|
117
|
+
if (this.closed)
|
|
118
|
+
return;
|
|
119
|
+
if (this.waiting)
|
|
120
|
+
this.dropped++;
|
|
121
|
+
this.waiting = frame;
|
|
122
|
+
this.pump();
|
|
123
|
+
}
|
|
124
|
+
/** The phone has a frame; an acknowledgement it owes nothing for is ignored. */
|
|
125
|
+
ack() {
|
|
126
|
+
if (!this.options.acked || this.inFlight === 0)
|
|
127
|
+
return;
|
|
128
|
+
this.inFlight--;
|
|
129
|
+
this.pump();
|
|
130
|
+
}
|
|
131
|
+
close() { this.closed = true; this.waiting = undefined; clearTimeout(this.timer); }
|
|
132
|
+
get bytesPerSecond() { return this.options.bytesPerSecond ?? STREAM_BYTES_PER_SECOND; }
|
|
133
|
+
pump() {
|
|
134
|
+
if (this.closed || this.inFlight >= (this.options.window ?? 1) || !this.waiting || this.timer)
|
|
135
|
+
return;
|
|
136
|
+
const now = this.now();
|
|
137
|
+
this.tokens = Math.min(this.bytesPerSecond, this.tokens + (now - this.refilled) * this.bytesPerSecond / 1000);
|
|
138
|
+
this.refilled = now;
|
|
139
|
+
if (this.tokens < 0) {
|
|
140
|
+
// Over budget: the frame waits (and may be replaced) until it refills.
|
|
141
|
+
this.timer = setTimeout(() => { this.timer = undefined; this.pump(); }, Math.ceil(-this.tokens * 1000 / this.bytesPerSecond));
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
const frame = this.waiting;
|
|
145
|
+
this.waiting = undefined;
|
|
146
|
+
this.inFlight++;
|
|
147
|
+
this.tokens -= frame.length;
|
|
148
|
+
this.send(frame, error => {
|
|
149
|
+
if (error) {
|
|
150
|
+
this.close();
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
this.sent++;
|
|
154
|
+
if (this.options.acked)
|
|
155
|
+
return;
|
|
156
|
+
this.inFlight--;
|
|
157
|
+
this.pump();
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
const defaultDeps = {
|
|
162
|
+
booted: bootedSimulators,
|
|
163
|
+
helper: () => verifiedHelper("simcast"),
|
|
164
|
+
spawn: (binary, args) => spawnChild(binary, args, { stdio: ["ignore", "pipe", "pipe"] }),
|
|
165
|
+
platform: () => process.platform,
|
|
166
|
+
};
|
|
167
|
+
let active = 0;
|
|
168
|
+
/** Streams running now, for tests and health. */
|
|
169
|
+
export function activeSimulatorStreams() { return active; }
|
|
170
|
+
/** WebSocket close reasons are at most 123 bytes. */
|
|
171
|
+
function reason(text) { return Buffer.from(text).subarray(0, 120).toString("utf8").replace(/�+$/, ""); }
|
|
172
|
+
export async function streamSimulator(client, query, deps = defaultDeps) {
|
|
173
|
+
let options;
|
|
174
|
+
try {
|
|
175
|
+
options = streamOptions(query);
|
|
176
|
+
}
|
|
177
|
+
catch (error) {
|
|
178
|
+
client.close(1008, reason(error.message));
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
if (deps.platform() !== "darwin") {
|
|
182
|
+
client.close(1008, "Simulators run on macOS only.");
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
if (active >= MAX_STREAMS) {
|
|
186
|
+
client.close(1013, "Too many simulator streams; try again.");
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
// The slot is held until the work it pays for is over: discovery and the
|
|
190
|
+
// build still running after the socket closed, and a helper that takes up
|
|
191
|
+
// to 2 s to exit. Releasing it on close would let connect-close cycles pile
|
|
192
|
+
// up simctl and helper processes past MAX_STREAMS.
|
|
193
|
+
active++;
|
|
194
|
+
let released = false;
|
|
195
|
+
const release = () => { if (!released) {
|
|
196
|
+
released = true;
|
|
197
|
+
active--;
|
|
198
|
+
} };
|
|
199
|
+
let ended = false, child, killer;
|
|
200
|
+
const latest = new LatestFrame((frame, done) => {
|
|
201
|
+
if (client.readyState !== OPEN) {
|
|
202
|
+
done(new Error("closed"));
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
client.send(frame, { binary: true }, done);
|
|
206
|
+
}, { bytesPerSecond: deps.bytesPerSecond, window: options.ack ? ACK_WINDOW : 1, acked: options.ack });
|
|
207
|
+
client.on("message", (data, isBinary) => {
|
|
208
|
+
if (isBinary || data.length > 64)
|
|
209
|
+
return;
|
|
210
|
+
try {
|
|
211
|
+
if (JSON.parse(data.toString("utf8")).type === "ack")
|
|
212
|
+
latest.ack();
|
|
213
|
+
}
|
|
214
|
+
catch { /* not ours */ }
|
|
215
|
+
});
|
|
216
|
+
const end = (code, why) => {
|
|
217
|
+
if (ended)
|
|
218
|
+
return;
|
|
219
|
+
ended = true;
|
|
220
|
+
latest.close();
|
|
221
|
+
if (child) {
|
|
222
|
+
const running = child;
|
|
223
|
+
running.kill("SIGTERM");
|
|
224
|
+
// A helper stuck in CoreSimulator still goes.
|
|
225
|
+
killer = setTimeout(() => running.kill("SIGKILL"), 2_000);
|
|
226
|
+
killer.unref?.();
|
|
227
|
+
running.once("exit", () => clearTimeout(killer));
|
|
228
|
+
}
|
|
229
|
+
if (code !== undefined && client.readyState === OPEN)
|
|
230
|
+
client.close(code, reason(why ?? "Simulator stream ended."));
|
|
231
|
+
};
|
|
232
|
+
client.once("close", () => end());
|
|
233
|
+
client.once("error", () => end());
|
|
234
|
+
try {
|
|
235
|
+
if (!(await deps.booted()).some(device => device.udid === options.udid)) {
|
|
236
|
+
end(1008, "That simulator is not booted.");
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
// A socket that closed during discovery doesn't go on to build the helper.
|
|
240
|
+
if (ended || client.readyState !== OPEN) {
|
|
241
|
+
end();
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
const binary = await deps.helper();
|
|
245
|
+
if (ended || client.readyState !== OPEN) {
|
|
246
|
+
end();
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
child = deps.spawn(binary, [options.udid, String(options.fps), String(options.size), (options.quality / 100).toFixed(2)]);
|
|
250
|
+
child.once("exit", release);
|
|
251
|
+
child.once("error", release);
|
|
252
|
+
}
|
|
253
|
+
catch (error) {
|
|
254
|
+
end(1011, error instanceof BridgeError ? error.message : "Couldn't start the simulator stream.");
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
finally {
|
|
258
|
+
if (!child)
|
|
259
|
+
release();
|
|
260
|
+
}
|
|
261
|
+
const reader = new FrameReader();
|
|
262
|
+
let stderr = "";
|
|
263
|
+
child.stderr?.on("data", (chunk) => { stderr = (stderr + chunk.toString("utf8")).slice(-400); });
|
|
264
|
+
child.stdout?.on("data", (chunk) => {
|
|
265
|
+
if (ended)
|
|
266
|
+
return;
|
|
267
|
+
let frames;
|
|
268
|
+
try {
|
|
269
|
+
frames = reader.push(chunk);
|
|
270
|
+
}
|
|
271
|
+
catch {
|
|
272
|
+
end(1011, "The simulator stream sent a broken frame.");
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
for (const frame of frames)
|
|
276
|
+
latest.offer(frame);
|
|
277
|
+
});
|
|
278
|
+
child.once("error", () => end(1011, "Couldn't start the simulator stream."));
|
|
279
|
+
child.once("exit", () => {
|
|
280
|
+
const last = stderr.trim().split("\n").at(-1);
|
|
281
|
+
end(1011, last ? `Simulator stream ended: ${last}` : "Simulator stream ended.");
|
|
282
|
+
});
|
|
283
|
+
client.send(JSON.stringify({ type: "start", format: "jpeg", fps: options.fps, size: options.size, quality: options.quality, ack: options.ack }), { binary: false });
|
|
284
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Json } from "./protocol.js";
|
|
2
|
+
export declare const SIMULATOR_UDID: RegExp;
|
|
2
3
|
/** The iOS simulators booted on this Mac, from `simctl`; none elsewhere. */
|
|
3
4
|
export declare function bootedSimulators(): Promise<Json[]>;
|
|
4
5
|
/** A PNG of one booted simulator's screen. */
|
|
@@ -23,6 +24,17 @@ export type SimulatorAction = {
|
|
|
23
24
|
text: string;
|
|
24
25
|
submit?: boolean;
|
|
25
26
|
};
|
|
27
|
+
/** The native helpers, compiled once per Hook version with swiftc, which any
|
|
28
|
+
* Mac with the simulators has. simtap (native/simtap.swift) finds the
|
|
29
|
+
* Simulator window through the Accessibility API and posts touches and keys
|
|
30
|
+
* with CGEvent. AppleScript would need Automation consent, which a launchd
|
|
31
|
+
* agent can never be prompted for; the helper needs one Accessibility grant.
|
|
32
|
+
* Its `prompt` command puts it in the Accessibility list and shows the system
|
|
33
|
+
* dialog. simcast (native/simcast.swift) streams a simulator's framebuffer
|
|
34
|
+
* for simulator-stream.ts and needs no grant. */
|
|
35
|
+
export type NativeHelper = "simtap" | "simcast";
|
|
36
|
+
/** A helper, hashed again right before it runs. */
|
|
37
|
+
export declare function verifiedHelper(name?: NativeHelper): Promise<string>;
|
|
26
38
|
/** The steps for the Accessibility grant, for the 403 and the docs. */
|
|
27
39
|
export declare function accessibilityHelp(binary: string): string;
|
|
28
40
|
export declare function simulatorAct(udid: string, request: SimulatorAction): Promise<Json>;
|
|
@@ -5,8 +5,12 @@ import path from "node:path";
|
|
|
5
5
|
import { createHash, randomUUID } from "node:crypto";
|
|
6
6
|
import { promisify } from "node:util";
|
|
7
7
|
import { BridgeError, bridgeRoot } from "./protocol.js";
|
|
8
|
-
const
|
|
9
|
-
|
|
8
|
+
const execFileAsync = promisify(execFile);
|
|
9
|
+
/** Every simulator child is killed outright at its timeout. SIGTERM, the
|
|
10
|
+
* default, let one that ignored it hold its promise, and with it a stream's
|
|
11
|
+
* slot, for good. */
|
|
12
|
+
const exec = (file, args, options) => execFileAsync(file, args, { ...options, killSignal: "SIGKILL", encoding: "utf8" });
|
|
13
|
+
export const SIMULATOR_UDID = /^[A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12}$/i;
|
|
10
14
|
/** The iOS simulators booted on this Mac, from `simctl`; none elsewhere. */
|
|
11
15
|
export async function bootedSimulators() {
|
|
12
16
|
if (process.platform !== "darwin")
|
|
@@ -24,7 +28,7 @@ export async function bootedSimulators() {
|
|
|
24
28
|
// com.apple.CoreSimulator.SimRuntime.iOS-26-1 → iOS 26.1
|
|
25
29
|
const runtime = runtimeID.replace(/^.*SimRuntime\./, "").replace(/-(\d+)-(\d+)$/, " $1.$2").replace(/-/g, " ");
|
|
26
30
|
for (const device of devices ?? []) {
|
|
27
|
-
if (device.state !== "Booted" || !device.udid || !
|
|
31
|
+
if (device.state !== "Booted" || !device.udid || !SIMULATOR_UDID.test(device.udid))
|
|
28
32
|
continue;
|
|
29
33
|
list.push({ udid: device.udid.toUpperCase(), name: String(device.name ?? "Simulator").slice(0, 120), runtime });
|
|
30
34
|
if (list.length >= 16)
|
|
@@ -35,7 +39,7 @@ export async function bootedSimulators() {
|
|
|
35
39
|
}
|
|
36
40
|
/** A PNG of one booted simulator's screen. */
|
|
37
41
|
export async function simulatorScreenshot(udid) {
|
|
38
|
-
if (!
|
|
42
|
+
if (!SIMULATOR_UDID.test(udid))
|
|
39
43
|
throw new BridgeError(400, "Invalid simulator identifier.");
|
|
40
44
|
if (process.platform !== "darwin")
|
|
41
45
|
throw new BridgeError(404, "Simulators run on macOS only.");
|
|
@@ -53,22 +57,24 @@ export async function simulatorScreenshot(udid) {
|
|
|
53
57
|
await unlink(file).catch(() => undefined);
|
|
54
58
|
}
|
|
55
59
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
60
|
+
const HELPER_LABEL = { simtap: "simulator input helper", simcast: "simulator stream helper" };
|
|
61
|
+
const building = new Map();
|
|
62
|
+
function helper(name) {
|
|
63
|
+
let build = building.get(name);
|
|
64
|
+
if (!build) {
|
|
65
|
+
build = buildHelper(name).finally(() => building.delete(name));
|
|
66
|
+
building.set(name, build);
|
|
67
|
+
}
|
|
68
|
+
return build;
|
|
69
|
+
}
|
|
70
|
+
async function helperSource(name) {
|
|
71
|
+
const bundled = name === "simtap" ? (typeof SIMTAP_SOURCE === "string" ? SIMTAP_SOURCE : undefined) : (typeof SIMCAST_SOURCE === "string" ? SIMCAST_SOURCE : undefined);
|
|
72
|
+
return bundled ?? await readFile(new URL(`./native/${name}.swift`, import.meta.url), "utf8");
|
|
67
73
|
}
|
|
68
|
-
async function buildHelper() {
|
|
74
|
+
async function buildHelper(name) {
|
|
69
75
|
const dir = path.join(bridgeRoot(), "native");
|
|
70
|
-
const binary = path.join(dir,
|
|
71
|
-
const code =
|
|
76
|
+
const binary = path.join(dir, name);
|
|
77
|
+
const code = await helperSource(name);
|
|
72
78
|
const digest = createHash("sha256").update(code).digest("hex");
|
|
73
79
|
// Rebuilt only when the source changes: the Accessibility grant is tied
|
|
74
80
|
// to this binary, and a needless rebuild would make macOS ask again.
|
|
@@ -76,12 +82,12 @@ async function buildHelper() {
|
|
|
76
82
|
if (await stat(binary).catch(() => undefined) && saved?.sourceSha === digest)
|
|
77
83
|
return binary;
|
|
78
84
|
await mkdir(dir, { recursive: true, mode: 0o700 });
|
|
79
|
-
const temporary = path.join(dir,
|
|
85
|
+
const temporary = path.join(dir, `${name}.${randomUUID()}`), source = temporary + ".swift";
|
|
80
86
|
const sidecar = temporary + ".sha256";
|
|
81
87
|
try {
|
|
82
88
|
await writeFile(source, code, { mode: 0o600, flag: "wx" });
|
|
83
89
|
await exec("/usr/bin/swiftc", ["-O", "-o", temporary, source], { timeout: 180_000 }).catch(() => {
|
|
84
|
-
throw new BridgeError(409,
|
|
90
|
+
throw new BridgeError(409, `Couldn't build the ${HELPER_LABEL[name]} (needs Xcode's swiftc).`);
|
|
85
91
|
});
|
|
86
92
|
await chmod(temporary, 0o700);
|
|
87
93
|
const binarySha = createHash("sha256").update(await readFile(temporary)).digest("hex");
|
|
@@ -100,13 +106,13 @@ function simtap(name, ...args) {
|
|
|
100
106
|
inputQueue = task;
|
|
101
107
|
return task;
|
|
102
108
|
}
|
|
103
|
-
/**
|
|
104
|
-
async function verifiedHelper() {
|
|
105
|
-
const binary = await helper();
|
|
109
|
+
/** A helper, hashed again right before it runs. */
|
|
110
|
+
export async function verifiedHelper(name = "simtap") {
|
|
111
|
+
const binary = await helper(name);
|
|
106
112
|
const saved = await readFile(binary + ".sha256", "utf8").then(JSON.parse).catch(() => undefined);
|
|
107
113
|
const binarySha = createHash("sha256").update(await readFile(binary)).digest("hex");
|
|
108
114
|
if (!saved || saved.binarySha !== binarySha)
|
|
109
|
-
throw new BridgeError(409,
|
|
115
|
+
throw new BridgeError(409, `${HELPER_LABEL[name][0].toUpperCase()}${HELPER_LABEL[name].slice(1)} integrity check failed.`);
|
|
110
116
|
return binary;
|
|
111
117
|
}
|
|
112
118
|
async function executeSimtap(name, ...args) {
|
|
@@ -148,7 +154,7 @@ const ACCESSIBILITY_PANE = "x-apple.systempreferences:com.apple.preference.secur
|
|
|
148
154
|
// macOS virtual key codes.
|
|
149
155
|
const KEY_H = "4", KEY_L = "37";
|
|
150
156
|
export async function simulatorAct(udid, request) {
|
|
151
|
-
if (!
|
|
157
|
+
if (!SIMULATOR_UDID.test(udid))
|
|
152
158
|
throw new BridgeError(400, "Invalid simulator identifier.");
|
|
153
159
|
if (!["boot", "shutdown", "launch", "openurl", "home", "lock", "type", "tap", "screenshot-ready", "accessibility-settings"].includes(request.action))
|
|
154
160
|
throw new BridgeError(400, "Unknown simulator action.");
|
|
@@ -216,7 +222,7 @@ async function deviceName(udid) {
|
|
|
216
222
|
}
|
|
217
223
|
/** The apps a simulator has, for the launcher. */
|
|
218
224
|
export async function simulatorApps(udid) {
|
|
219
|
-
if (!
|
|
225
|
+
if (!SIMULATOR_UDID.test(udid))
|
|
220
226
|
throw new BridgeError(400, "Invalid simulator identifier.");
|
|
221
227
|
if (process.platform !== "darwin")
|
|
222
228
|
return [];
|
|
@@ -51,17 +51,37 @@ export function failRelay(client) {
|
|
|
51
51
|
}
|
|
52
52
|
export function relayTranscription(client, query, options = {}) {
|
|
53
53
|
const connect = options.connect ?? ((url, key) => new WebSocket(url, { headers: { "xi-api-key": key } }));
|
|
54
|
+
// The phone sends the audio it held while connecting as soon as the
|
|
55
|
+
// socket opens, before the key and region below are read: listen now and
|
|
56
|
+
// replay those frames once the relay is set up, so no opening word is lost.
|
|
57
|
+
let receive;
|
|
58
|
+
let early = [], earlyBytes = 0, hungUp = false;
|
|
59
|
+
client.on("message", (data, isBinary) => {
|
|
60
|
+
if (receive) {
|
|
61
|
+
receive(data, isBinary);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
// Past the relay's own limit the replay ends the session anyway.
|
|
65
|
+
if (early && (earlyBytes += data.length) <= MAX_PENDING_BYTES)
|
|
66
|
+
early.push([data, isBinary]);
|
|
67
|
+
});
|
|
68
|
+
client.on("close", () => { hungUp = true; });
|
|
54
69
|
return (async () => {
|
|
55
70
|
const key = await (options.key ?? readSpeechKey)();
|
|
56
71
|
const tell = (frame) => { if (client.readyState === OPEN)
|
|
57
72
|
client.send(JSON.stringify(frame)); };
|
|
73
|
+
if (hungUp)
|
|
74
|
+
return;
|
|
58
75
|
if (!key) {
|
|
59
76
|
tell({ type: "error", code: "transcribe-unconfigured", error: "This computer has no ElevenLabs key." });
|
|
60
77
|
client.close(1008, "No ElevenLabs key");
|
|
61
78
|
return;
|
|
62
79
|
}
|
|
63
80
|
const origin = options.origin ?? (await resolveSpeechRegion()).origin;
|
|
81
|
+
if (hungUp)
|
|
82
|
+
return;
|
|
64
83
|
const upstream = connect(transcribeURL(query, origin), key);
|
|
84
|
+
// Held until ElevenLabs answers; null is the phone's stop, kept in its place.
|
|
65
85
|
let pending = [], pendingBytes = 0, closed = false;
|
|
66
86
|
const finish = (code = 1000, reason = "") => {
|
|
67
87
|
if (closed)
|
|
@@ -82,13 +102,15 @@ export function relayTranscription(client, query, options = {}) {
|
|
|
82
102
|
const chunk = (audio, commit = false) => JSON.stringify({
|
|
83
103
|
message_type: "input_audio_chunk", audio_base_64: audio.toString("base64"), commit, sample_rate: TRANSCRIBE_RATE
|
|
84
104
|
});
|
|
85
|
-
|
|
105
|
+
receive = (data, isBinary) => {
|
|
86
106
|
if (closed)
|
|
87
107
|
return;
|
|
88
108
|
if (!isBinary) {
|
|
89
109
|
// The phone's only text frame: the person stopped, commit what's left.
|
|
90
110
|
if (upstream.readyState === OPEN)
|
|
91
111
|
upstream.send(chunk(Buffer.alloc(0), true));
|
|
112
|
+
else
|
|
113
|
+
pending.push(null);
|
|
92
114
|
return;
|
|
93
115
|
}
|
|
94
116
|
if (upstream.readyState === OPEN) {
|
|
@@ -102,12 +124,15 @@ export function relayTranscription(client, query, options = {}) {
|
|
|
102
124
|
return;
|
|
103
125
|
}
|
|
104
126
|
pending.push(data);
|
|
105
|
-
}
|
|
127
|
+
};
|
|
128
|
+
for (const [data, isBinary] of early ?? [])
|
|
129
|
+
receive(data, isBinary);
|
|
130
|
+
early = undefined;
|
|
106
131
|
client.on("close", () => finish());
|
|
107
132
|
client.on("error", () => finish());
|
|
108
133
|
upstream.on("open", () => {
|
|
109
134
|
for (const audio of pending)
|
|
110
|
-
upstream.send(chunk(audio));
|
|
135
|
+
upstream.send(audio ? chunk(audio) : chunk(Buffer.alloc(0), true));
|
|
111
136
|
pending = [];
|
|
112
137
|
pendingBytes = 0;
|
|
113
138
|
if (!closed)
|
|
@@ -18,13 +18,31 @@ export function phrenCommandLine(args) {
|
|
|
18
18
|
return [["phren", ...args].map(shellWord).join(" "),
|
|
19
19
|
`herdr pane release-agent "$HERDR_PANE_ID" --source ${PHREN_REPORT_SOURCE} --agent phren >/dev/null 2>&1`].join("; ");
|
|
20
20
|
}
|
|
21
|
-
/**
|
|
22
|
-
|
|
21
|
+
/** Herdr's `agent.prompt` and `agent.send_keys` take only agents it started
|
|
22
|
+
* and still holds as active. A worker launched with its brief as the
|
|
23
|
+
* harness's first prompt goes straight to work and can stay `launch_pending`
|
|
24
|
+
* there, and an agent restarted in its pane (or alive across a Herdr restart)
|
|
25
|
+
* is detected but never named again; Herdr refuses all of them with
|
|
26
|
+
* `agent_not_ready` for good. Such a pane is typed into instead, the way the
|
|
27
|
+
* terminal view types, when its foreground program is the harness Herdr
|
|
28
|
+
* detects there (phren-agent counts by report alone, as Herdr cannot detect
|
|
29
|
+
* it). For a prompt Herdr must also read the agent at rest or at work, so
|
|
30
|
+
* text never lands in a starting harness or answers a dialog it draws. */
|
|
31
|
+
async function typableAgent(server, pane, forPrompt) {
|
|
23
32
|
const info = object((await rpc(server, "pane.get", { pane_id: pane }).catch(() => ({}))).pane);
|
|
24
|
-
|
|
33
|
+
if (info.agent === "phren")
|
|
34
|
+
return true;
|
|
35
|
+
if (typeof info.agent !== "string" || !info.agent)
|
|
36
|
+
return false;
|
|
37
|
+
if (forPrompt && !["idle", "working", "done"].includes(String(info.agent_status)))
|
|
38
|
+
return false;
|
|
39
|
+
const processes = object((await rpc(server, "pane.process_info", { pane_id: pane }).catch(() => ({}))).process_info);
|
|
40
|
+
// The shell in the foreground would run the text as a command.
|
|
41
|
+
if (!Number.isSafeInteger(processes.foreground_process_group_id) || processes.foreground_process_group_id === processes.shell_pid)
|
|
42
|
+
return false;
|
|
43
|
+
return objects(processes.foreground_processes).some(p => typeof p.cmdline === "string" && agentFromCommand(p.cmdline) === info.agent);
|
|
25
44
|
}
|
|
26
|
-
/**
|
|
27
|
-
* a phren-agent pane it knows by report gets the pane calls instead. */
|
|
45
|
+
/** The refusal `typableAgent` answers: an agent Herdr does not hold as named. */
|
|
28
46
|
function unnamedAgent(error) {
|
|
29
47
|
return error instanceof BridgeError && error.details?.herdrCode === "agent_not_ready" && /not an active named agent/.test(error.message);
|
|
30
48
|
}
|
|
@@ -88,7 +106,7 @@ export const herdrTerminal = {
|
|
|
88
106
|
await rpc(server, "agent.send_keys", { target: pane, keys });
|
|
89
107
|
}
|
|
90
108
|
catch (error) {
|
|
91
|
-
if (!unnamedAgent(error) || !await
|
|
109
|
+
if (!unnamedAgent(error) || !await typableAgent(server, pane, false))
|
|
92
110
|
throw error;
|
|
93
111
|
await rpc(server, "pane.send_keys", { pane_id: pane, keys });
|
|
94
112
|
}
|
|
@@ -115,9 +133,11 @@ export const herdrTerminal = {
|
|
|
115
133
|
await rpc(server, "agent.prompt", { target: pane, text });
|
|
116
134
|
}
|
|
117
135
|
catch (error) {
|
|
118
|
-
if (!unnamedAgent(error) || !await
|
|
136
|
+
if (!unnamedAgent(error) || !await typableAgent(server, pane, true))
|
|
119
137
|
throw error;
|
|
120
|
-
// One bracketed paste keeps the newlines in
|
|
138
|
+
// One bracketed paste keeps the newlines in the agent's composer; Enter submits it.
|
|
139
|
+
// The route's delivery record (by deliveryId) is already armed: the agent's
|
|
140
|
+
// UserPromptSubmit settles it, as for a prompt Herdr typed.
|
|
121
141
|
await rpc(server, "pane.send_text", { pane_id: pane, text: `\x1b[200~${text}\x1b[201~` }, signal);
|
|
122
142
|
await new Promise(resolve => setTimeout(resolve, 150));
|
|
123
143
|
await rpc(server, "pane.send_keys", { pane_id: pane, keys: ["Enter"] }, signal);
|