@tapflowio/ios-agent 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/DeviceChromeLoader.d.ts +2 -45
- package/dist/DeviceChromeLoader.d.ts.map +1 -1
- package/dist/DeviceChromeLoader.js.map +1 -1
- package/dist/IOSAgent.d.ts +65 -0
- package/dist/IOSAgent.d.ts.map +1 -1
- package/dist/IOSAgent.js +471 -109
- package/dist/IOSAgent.js.map +1 -1
- package/dist/KeyboardHelperDaemon.d.ts.map +1 -1
- package/dist/KeyboardHelperDaemon.js +2 -1
- package/dist/KeyboardHelperDaemon.js.map +1 -1
- package/dist/MjpegStreamer.d.ts.map +1 -1
- package/dist/MjpegStreamer.js +9 -1
- package/dist/MjpegStreamer.js.map +1 -1
- package/dist/SimctlWrapper.d.ts +46 -1
- package/dist/SimctlWrapper.d.ts.map +1 -1
- package/dist/SimctlWrapper.js +86 -3
- package/dist/SimctlWrapper.js.map +1 -1
- package/dist/TouchHelper.d.ts +53 -14
- package/dist/TouchHelper.d.ts.map +1 -1
- package/dist/TouchHelper.js +237 -59
- package/dist/TouchHelper.js.map +1 -1
- package/dist/childProcessKill.d.ts +3 -0
- package/dist/childProcessKill.d.ts.map +1 -0
- package/dist/childProcessKill.js +19 -0
- package/dist/childProcessKill.js.map +1 -0
- package/package.json +9 -8
package/dist/IOSAgent.js
CHANGED
|
@@ -5,10 +5,26 @@ import { tmpdir } from 'os';
|
|
|
5
5
|
import { randomUUID } from 'crypto';
|
|
6
6
|
import { spawnSync } from 'child_process';
|
|
7
7
|
import { WebSocket } from 'ws';
|
|
8
|
-
import { createLogger, PlatformError, ValidationError } from '@tapflowio/agent-core';
|
|
8
|
+
import { createLogger, PlatformError, ValidationError, bootAbandonMessage, BOOT_NO_SESSION_STATE } from '@tapflowio/agent-core';
|
|
9
9
|
const logger = createLogger('ios-agent');
|
|
10
10
|
// Typed so a typo cannot ship silently — the viewer gates the whole clipboard bridge on this.
|
|
11
11
|
const AGENT_CAPABILITIES = ['clipboard'];
|
|
12
|
+
// Human prose for each reason. Not in `@tapflowio/protocol`: that package's main entry must stay
|
|
13
|
+
// runtime-free so it erases under `import type` and never reaches the dashboard bundle — a lookup
|
|
14
|
+
// table is a runtime value. The `reason` on the wire is the machine-readable half; this is only what
|
|
15
|
+
// a human reads, and each agent owns its own wording.
|
|
16
|
+
const INPUT_ERROR_MESSAGES = {
|
|
17
|
+
'not-booted': 'device not booted',
|
|
18
|
+
'channel-unavailable': 'input channel not ready',
|
|
19
|
+
'channel-starting': 'the input channel is still starting — retry in a moment',
|
|
20
|
+
'dispatch-failed': 'the device rejected the input',
|
|
21
|
+
unsupported: 'this input is not supported on the active connection to the device',
|
|
22
|
+
malformed: 'this input does not fit what the device is doing',
|
|
23
|
+
'no-gesture': 'no gesture is in progress to complete — start a new one',
|
|
24
|
+
// The relay is the only producer of this one — an agent has no way to know which socket asked. Present
|
|
25
|
+
// because the map is exhaustive, which is what makes a new reason a decision rather than an omission.
|
|
26
|
+
'not-session-owner': 'this input was addressed to a session the sender does not hold',
|
|
27
|
+
};
|
|
12
28
|
// Cross-platform button name → iOS device-chrome button name. Chrome uses
|
|
13
29
|
// hyphens and "power" (not "lock"); MCP's vocabulary uses underscores. Names
|
|
14
30
|
// not listed here (incl. the raw chrome names the dashboard sends) pass through.
|
|
@@ -18,7 +34,7 @@ export const IOS_BUTTON_ALIASES = {
|
|
|
18
34
|
volume_down: 'volume-down',
|
|
19
35
|
};
|
|
20
36
|
import { createResourceSampler, registerStreamWs, disableNagle, createKeyframeAwareSender, pickMaxSize, createRateLimitedDropWarn, createThroughputSampler, createSleepBlocker, getMachineId, isLocalhostWss, DEFAULT_BACKPRESSURE_BYTES, writeEnvelopeHeader, rewriteLowLatencySpsInFrame, CODEC_JPEG, CODEC_H264, CODEC_AUDIO, sendAudioYieldingToVideo, } from '@tapflowio/agent-core/utils';
|
|
21
|
-
import { SimctlWrapper, isDeviceMissingError, ClipboardTooLargeError } from './SimctlWrapper.js';
|
|
37
|
+
import { SimctlWrapper, isDeviceMissingError, ClipboardTooLargeError, firstLine } from './SimctlWrapper.js';
|
|
22
38
|
import { MAX_CLIPBOARD_BYTES, clipboardByteLength, CLIPBOARD_SENTINEL_PREFIX as SENTINEL_PREFIX, isClipboardSentinel as isSentinel, CLIPBOARD_COPY_DEADLINE_MS, CLIPBOARD_WRITE_DEADLINE_MS, CLIPBOARD_RESTORE_DEADLINE_MS, CLIPBOARD_POLL_MS, createKeyedSerialQueue, } from '@tapflowio/agent-core';
|
|
23
39
|
import { ScreenCaptureStreamer } from './ScreenCaptureStreamer.js';
|
|
24
40
|
import { AudioCaptureStreamer, readSimVolume, applyGain } from './AudioCaptureStreamer.js';
|
|
@@ -45,6 +61,34 @@ export class IOSAgent {
|
|
|
45
61
|
token;
|
|
46
62
|
handshakeTimeoutMs;
|
|
47
63
|
ws = null;
|
|
64
|
+
/** Send on the control socket, if there is one. The `?.` is the point: 66 call sites relied on a send
|
|
65
|
+
* being a no-op between reconnects, and this preserves that exactly.
|
|
66
|
+
*
|
|
67
|
+
* Typed with `AgentControlOutbound`, which is why this exists — an agent's literal used to reach `ws.send`
|
|
68
|
+
* with nothing checking it, and #489/#490 are what that cost. *
|
|
69
|
+
* **The guard is not defensive tidying.** `ws.send` on anything other than OPEN takes the `sendAfterClose`
|
|
70
|
+
* path, which adds the payload to a buffer nobody will flush and neither throws nor emits — so a reply
|
|
71
|
+
* sent to a closing socket is indistinguishable from a delivered one at the call site. `reportResources`
|
|
72
|
+
* has always checked; this did not, and the boot answers this file now owes a caller are exactly the
|
|
73
|
+
* messages whose purpose is to end someone's wait.
|
|
74
|
+
*
|
|
75
|
+
* The body is held character for character by `scripts/__tests__/agentSendTyped.test.mjs`, so the comment
|
|
76
|
+
* lives out here: that check strips comments and matches the body exactly, having watched three earlier
|
|
77
|
+
* drafts get bypassed by a renamed socket. */
|
|
78
|
+
sendMsg(msg) {
|
|
79
|
+
if (this.ws?.readyState !== WebSocket.OPEN)
|
|
80
|
+
return;
|
|
81
|
+
this.ws.send(JSON.stringify(msg));
|
|
82
|
+
}
|
|
83
|
+
/** Send on a socket the caller has already established. Takes the socket **as an argument** rather than
|
|
84
|
+
* reading `this.ws`, and that is deliberate: the call sites that use it sit behind an entry guard
|
|
85
|
+
* (`if (!this.ws) return`), and today deleting one of those guards is a compile error. Reading
|
|
86
|
+
* `this.ws` here — or asserting it with `!` — would make the guard optional to the compiler and turn
|
|
87
|
+
* its removal into a runtime `TypeError` instead. It also serves the `agent:register` send, which runs
|
|
88
|
+
* inside `onopen` on a local socket. */
|
|
89
|
+
sendOn(ws, msg) {
|
|
90
|
+
ws.send(JSON.stringify(msg));
|
|
91
|
+
}
|
|
48
92
|
deviceStates = new Map();
|
|
49
93
|
// Last app launched per device (deviceId → bundleId). The XCUITest tree backend
|
|
50
94
|
// queries by bundleId; kept outside DeviceState so it survives a relay reconnect
|
|
@@ -103,7 +147,7 @@ export class IOSAgent {
|
|
|
103
147
|
}, this.handshakeTimeoutMs);
|
|
104
148
|
ws.once('open', () => {
|
|
105
149
|
disableNagle(ws);
|
|
106
|
-
|
|
150
|
+
this.sendOn(ws, {
|
|
107
151
|
type: 'agent:register',
|
|
108
152
|
platform: 'ios',
|
|
109
153
|
// Lets a viewer tell a clipboard-capable agent from one that predates the
|
|
@@ -118,7 +162,7 @@ export class IOSAgent {
|
|
|
118
162
|
status: d.status,
|
|
119
163
|
osVersion: d.osVersion,
|
|
120
164
|
})),
|
|
121
|
-
})
|
|
165
|
+
});
|
|
122
166
|
});
|
|
123
167
|
ws.once('message', (data) => {
|
|
124
168
|
let msg;
|
|
@@ -140,7 +184,14 @@ export class IOSAgent {
|
|
|
140
184
|
this.initDeviceStates(msg.registeredSessions);
|
|
141
185
|
ws.on('message', (d) => {
|
|
142
186
|
try {
|
|
143
|
-
|
|
187
|
+
const m = JSON.parse(d.toString());
|
|
188
|
+
// Every type `handleRelayMessage` dispatches is session-scoped, and the relay resolves the
|
|
189
|
+
// session before forwarding — so a message without one did not come from that path. Rejecting
|
|
190
|
+
// it here is what lets the dispatcher declare `sessionId: string` instead of threading an
|
|
191
|
+
// optional through 30 sends and asserting it with `!` at each one.
|
|
192
|
+
if (typeof m.type !== 'string' || typeof m.sessionId !== 'string')
|
|
193
|
+
return;
|
|
194
|
+
this.handleRelayMessage(m);
|
|
144
195
|
}
|
|
145
196
|
catch { /* ignore malformed */ }
|
|
146
197
|
});
|
|
@@ -183,6 +234,8 @@ export class IOSAgent {
|
|
|
183
234
|
streamReader: null,
|
|
184
235
|
captureStreamer: null,
|
|
185
236
|
bootSeq: 0,
|
|
237
|
+
bootAbandon: new Map(),
|
|
238
|
+
bootsInFlight: new Set(),
|
|
186
239
|
orientation: 'portrait',
|
|
187
240
|
loadedChrome: null,
|
|
188
241
|
softKeyboardVisible: false,
|
|
@@ -208,6 +261,7 @@ export class IOSAgent {
|
|
|
208
261
|
this.resourcesTimer = null;
|
|
209
262
|
}
|
|
210
263
|
for (const state of this.deviceStates.values()) {
|
|
264
|
+
this.bumpBootSeq(state, 'relay-lost');
|
|
211
265
|
this.cleanupDeviceState(state);
|
|
212
266
|
}
|
|
213
267
|
this.deviceStates.clear();
|
|
@@ -226,6 +280,13 @@ export class IOSAgent {
|
|
|
226
280
|
this.resourcesTimer = null;
|
|
227
281
|
}
|
|
228
282
|
for (const state of this.deviceStates.values()) {
|
|
283
|
+
// Invalidate any boot still in flight against this state. The map is dropped just below, but a
|
|
284
|
+
// `handleDeviceBoot` awaiting simctl holds its own reference and its seq would still match, so it
|
|
285
|
+
// would go on to build a TouchHelper on a state nobody owns. That used to leak one child process;
|
|
286
|
+
// a self-reviving helper would respawn for the life of the agent with no reference left to stop it.
|
|
287
|
+
// Bumped here rather than inside `cleanupDeviceState` so that all three retirements name a reason —
|
|
288
|
+
// and so Android, whose boot path calls its own cleanup mid-boot, can hold the same shape.
|
|
289
|
+
this.bumpBootSeq(state, 'relay-lost');
|
|
229
290
|
this.cleanupDeviceState(state);
|
|
230
291
|
}
|
|
231
292
|
this.deviceStates.clear();
|
|
@@ -254,7 +315,7 @@ export class IOSAgent {
|
|
|
254
315
|
const bootedCount = Array.from(this.deviceStates.values()).filter((s) => s.streamReader !== null).length;
|
|
255
316
|
const slotsTotal = this.deviceStates.size;
|
|
256
317
|
const { memUsedMB, memTotalMB } = this.resources.getMemoryUsage();
|
|
257
|
-
this.
|
|
318
|
+
this.sendOn(this.ws, {
|
|
258
319
|
type: 'agent:resources',
|
|
259
320
|
resources: {
|
|
260
321
|
cpuPercent: this.resources.getCpuPercent(),
|
|
@@ -264,7 +325,7 @@ export class IOSAgent {
|
|
|
264
325
|
slotsTotal,
|
|
265
326
|
reportedAt: Date.now(),
|
|
266
327
|
},
|
|
267
|
-
})
|
|
328
|
+
});
|
|
268
329
|
}
|
|
269
330
|
cleanupDeviceState(state) {
|
|
270
331
|
void state.streamReader?.cancel();
|
|
@@ -285,31 +346,39 @@ export class IOSAgent {
|
|
|
285
346
|
state.streamWs = null;
|
|
286
347
|
}
|
|
287
348
|
sendChromeData(state, device) {
|
|
288
|
-
|
|
349
|
+
// `readyState`, not presence: this runs mid-boot, and a socket that closed since the entry guard
|
|
350
|
+
// takes the payload into a buffer nobody flushes while `device:ready` is dropped by `sendMsg`'s own
|
|
351
|
+
// check — leaving the caller with neither the data nor an answer.
|
|
352
|
+
if (this.ws?.readyState !== WebSocket.OPEN)
|
|
289
353
|
return;
|
|
354
|
+
// Stop the outgoing helper before dropping the reference. This used to leak a child process;
|
|
355
|
+
// now that a helper revives itself on death, an orphan would also keep respawning with
|
|
356
|
+
// nobody left holding a reference to stop it.
|
|
357
|
+
state.touchHelper?.stop();
|
|
290
358
|
state.touchHelper = new TouchHelper(device.id);
|
|
291
359
|
state.touchHelper.start();
|
|
292
|
-
this.
|
|
360
|
+
this.sendOn(this.ws, {
|
|
293
361
|
type: 'session:deviceInfo',
|
|
294
362
|
sessionId: state.sessionId,
|
|
295
363
|
payload: {
|
|
296
364
|
deviceName: device.name,
|
|
297
365
|
osVersion: device.osVersion ?? '',
|
|
298
366
|
},
|
|
299
|
-
})
|
|
367
|
+
});
|
|
300
368
|
state.loadedChrome = this.chromeLoader.load(device.typeId ?? device.name);
|
|
301
369
|
if (!state.loadedChrome)
|
|
302
370
|
return;
|
|
303
|
-
this.
|
|
371
|
+
this.sendOn(this.ws, {
|
|
304
372
|
type: 'session:chrome',
|
|
305
373
|
sessionId: state.sessionId,
|
|
306
374
|
payload: state.loadedChrome,
|
|
307
|
-
})
|
|
375
|
+
});
|
|
308
376
|
}
|
|
309
377
|
startBinaryStream(state, streamWs) {
|
|
310
378
|
// H.264 is the default; opt out per-agent with TAPFLOW_IOS_CODEC=jpeg. It also needs
|
|
311
379
|
// a browser that reported it can decode it (device:boot acceptH264) — otherwise JPEG.
|
|
312
|
-
// Only on the ScreenCaptureStreamer path — the MjpegStreamer fallback is always JPEG
|
|
380
|
+
// Only on the ScreenCaptureStreamer path — the MjpegStreamer fallback is always JPEG, and now
|
|
381
|
+
// genuinely produces it rather than only being stamped that way.
|
|
313
382
|
const envAllowsH264 = process.env.TAPFLOW_IOS_CODEC !== 'jpeg';
|
|
314
383
|
const useH264 = this.intervalMs === undefined && envAllowsH264 && state.acceptH264;
|
|
315
384
|
const codec = useH264 ? CODEC_H264 : CODEC_JPEG;
|
|
@@ -398,29 +467,86 @@ export class IOSAgent {
|
|
|
398
467
|
await registerStreamWs(streamWs, state.sessionId);
|
|
399
468
|
return streamWs;
|
|
400
469
|
}
|
|
401
|
-
|
|
470
|
+
/** Retire the boot that holds the current seq, recording what retired it, and return the new seq.
|
|
471
|
+
*
|
|
472
|
+
* **Three callers, and the third is why this is a method.** A new boot supersedes the one in flight, a
|
|
473
|
+
* shutdown abandons it, and losing the relay invalidates it — that last one lives in the reconnect path
|
|
474
|
+
* (`cleanupDeviceState`'s callers) and is easy to miss when reading `handleDeviceBoot` alone. Spreading
|
|
475
|
+
* the reason across three `state.bootSeq++` lines makes forgetting one silent; here it is a parameter. */
|
|
476
|
+
bumpBootSeq(state, reason) {
|
|
477
|
+
if (state.bootsInFlight.has(state.bootSeq))
|
|
478
|
+
state.bootAbandon.set(state.bootSeq, reason);
|
|
479
|
+
return ++state.bootSeq;
|
|
480
|
+
}
|
|
481
|
+
/** Answer a boot this agent has stopped running. Called at every point that abandons one.
|
|
482
|
+
*
|
|
483
|
+
* **Only when a correlator exists**, which is the same rule the input path settled on (#489): a reply
|
|
484
|
+
* nobody is waiting for is not an answer, and the dashboard reports every *uncorrelated*
|
|
485
|
+
* `device:boot-error` as a failure — deliberately, since Android's dead-stream report has no id it could
|
|
486
|
+
* carry (#426). So sending one for a boot with no correlator would put an error toast on a tester's
|
|
487
|
+
* screen for a device that is booting normally. */
|
|
488
|
+
abandonBoot(state, seq, sessionId, requestId) {
|
|
489
|
+
const reason = state.bootAbandon.get(seq) ?? 'superseded';
|
|
490
|
+
state.bootAbandon.delete(seq);
|
|
491
|
+
// **A state the agent has stopped holding cannot be answered, and trying is worse than silence.**
|
|
492
|
+
// `relay-lost` retires a boot by dropping the whole map, so by the time the parked `await` resumes this
|
|
493
|
+
// `state` is an object nobody is registered against — and whether the socket is back yet decides
|
|
494
|
+
// between dropping the reply and sending it to a *new* relay naming a session id it has never heard
|
|
495
|
+
// of. Neither is an answer, and which one happens is a race between a poll interval and a backoff.
|
|
496
|
+
if (this.deviceStates.get(sessionId) !== state) {
|
|
497
|
+
logger.warn(`boot for ${sessionId} abandoned (${reason}) after the session was dropped — nothing to answer`);
|
|
498
|
+
return;
|
|
499
|
+
}
|
|
500
|
+
if (!requestId) {
|
|
501
|
+
logger.warn(`boot for ${sessionId} abandoned (${reason}) with no requestId — nothing to answer`);
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
this.sendMsg({ type: 'device:boot-error', sessionId, requestId, message: bootAbandonMessage(reason) });
|
|
505
|
+
}
|
|
506
|
+
// `requestId` is a parameter, never a field on `state`: `bootSeq` exists because two boots overlap,
|
|
507
|
+
// and a correlator hoisted onto shared state would answer the first request with the second's id.
|
|
508
|
+
// Optional because the relay's idle timer boots nothing — but every *browser* boot carries one.
|
|
509
|
+
async handleDeviceBoot(sessionId, deviceId, fullErase = false, acceptH264 = false, tier, requestId) {
|
|
402
510
|
const state = this.deviceStates.get(sessionId);
|
|
403
|
-
|
|
511
|
+
// Split, because the two halves are not the same kind of nothing. No open control channel means the
|
|
512
|
+
// answer itself has nowhere to go, so this is the one abandonment that stays silent — and the caller
|
|
513
|
+
// learns from the relay instead, which declares the agent away and terminates the session. A missing
|
|
514
|
+
// `state` is answerable and now answered (#489's reasoning, on the boot path).
|
|
515
|
+
const ws = this.ws;
|
|
516
|
+
if (!ws || ws.readyState !== WebSocket.OPEN)
|
|
517
|
+
return;
|
|
518
|
+
if (!state) {
|
|
519
|
+
if (requestId)
|
|
520
|
+
this.sendMsg({ type: 'device:boot-error', sessionId, requestId, message: BOOT_NO_SESSION_STATE });
|
|
404
521
|
return;
|
|
522
|
+
}
|
|
405
523
|
state.acceptH264 = acceptH264;
|
|
406
524
|
if (tier) {
|
|
407
525
|
state.secureContext = tier.secureContext;
|
|
408
526
|
state.external = tier.external;
|
|
409
527
|
}
|
|
410
|
-
const seq =
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
state.streamWs?.close();
|
|
418
|
-
state.streamWs = null;
|
|
419
|
-
this.ws.send(JSON.stringify({ type: 'device:booting', sessionId }));
|
|
528
|
+
const seq = this.bumpBootSeq(state, 'superseded');
|
|
529
|
+
state.bootsInFlight.add(seq);
|
|
530
|
+
// **The teardown runs inside the try, and that is a fix rather than tidying.** It ran before it, so a
|
|
531
|
+
// throw there rejected this handler into the bare `.catch(logger.error)` at its dispatch site: no
|
|
532
|
+
// `device:booting`, no ready, no error — and the seq was already bumped, so the boot this one
|
|
533
|
+
// superseded was gone as well. The caller then waited out the very deadline this change exists to
|
|
534
|
+
// end. `finally` is what stops `bootAbandon` outliving the boot that would have read it.
|
|
420
535
|
try {
|
|
536
|
+
void state.streamReader?.cancel();
|
|
537
|
+
state.streamReader = null;
|
|
538
|
+
state.captureStreamer = null; // reader.cancel() kills the helper proc; drop the ref so a stale requestKeyframe() no-ops
|
|
539
|
+
state.touchHelper?.stop();
|
|
540
|
+
state.touchHelper = null;
|
|
541
|
+
state.booted = false;
|
|
542
|
+
state.streamWs?.close();
|
|
543
|
+
state.streamWs = null;
|
|
544
|
+
this.sendOn(ws, { type: 'device:booting', sessionId });
|
|
421
545
|
const devices = await this.simctl.listDevices();
|
|
422
|
-
if (seq !== state.bootSeq)
|
|
546
|
+
if (seq !== state.bootSeq) {
|
|
547
|
+
this.abandonBoot(state, seq, sessionId, requestId);
|
|
423
548
|
return;
|
|
549
|
+
}
|
|
424
550
|
const target = devices.find((d) => d.id === deviceId);
|
|
425
551
|
if (!target)
|
|
426
552
|
throw new PlatformError(`Device not found: ${deviceId}`);
|
|
@@ -443,8 +569,10 @@ export class IOSAgent {
|
|
|
443
569
|
// The shutdown is an await, so a newer boot for this device may have overtaken us while it
|
|
444
570
|
// ran. Without this check the superseded boot goes on to erase a device the tester has
|
|
445
571
|
// since re-picked with the toggle off — the exact wipe-with-no-click this issue is about.
|
|
446
|
-
if (seq !== state.bootSeq)
|
|
572
|
+
if (seq !== state.bootSeq) {
|
|
573
|
+
this.abandonBoot(state, seq, sessionId, requestId);
|
|
447
574
|
return;
|
|
575
|
+
}
|
|
448
576
|
try {
|
|
449
577
|
await this.simctl.erase(deviceId);
|
|
450
578
|
}
|
|
@@ -469,20 +597,44 @@ export class IOSAgent {
|
|
|
469
597
|
}
|
|
470
598
|
await this.simctl.boot(deviceId);
|
|
471
599
|
}
|
|
472
|
-
else
|
|
600
|
+
else {
|
|
601
|
+
// Unconditionally, including when the list above said `booted`. That reading is already
|
|
602
|
+
// stale by the time we act on it — a tester can ⌘Q the simulator, or run `simctl shutdown`,
|
|
603
|
+
// in the width of one `xcrun` round trip — and skipping the boot on it is the only way to
|
|
604
|
+
// reach the wait below with *nothing bringing the device up*, which then costs the full
|
|
605
|
+
// deadline to discover. `SimctlWrapper.boot` swallows `Unable to boot device in current
|
|
606
|
+
// state: Booted`, so re-issuing it for a device that really is up costs one no-op
|
|
607
|
+
// subprocess and makes "the wait only ever runs after a boot was accepted" true.
|
|
473
608
|
await this.bootWithZombieRecovery(deviceId);
|
|
474
609
|
}
|
|
475
|
-
if (seq !== state.bootSeq)
|
|
610
|
+
if (seq !== state.bootSeq) {
|
|
611
|
+
this.abandonBoot(state, seq, sessionId, requestId);
|
|
476
612
|
return;
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
613
|
+
}
|
|
614
|
+
// `simctl boot` returns on initiation, so everything below used to run while the device was
|
|
615
|
+
// still coming up — measured 7.6s early (#486). The status sent with the chrome data was
|
|
616
|
+
// hardcoded `'booted'` over a value that had just been fetched and discarded, and
|
|
617
|
+
// `device:ready` went out a few lines later. A human is slower than the gap and rarely
|
|
618
|
+
// notices; `mcp-server` installs and taps the moment it sees ready, which is what #440's
|
|
619
|
+
// "No devices are booted" was.
|
|
620
|
+
// `isStale` rather than a check on the far side alone: this handler is fire-and-forget, so a
|
|
621
|
+
// shutdown arriving mid-wait bumps `bootSeq` and leaves the poll spawning `xcrun simctl list`
|
|
622
|
+
// twice a second against a device that is now deliberately off — for the rest of the deadline,
|
|
623
|
+
// with nothing left that wants the answer.
|
|
624
|
+
const bootedDevice = await this.simctl.waitUntilBooted(deviceId, { isStale: () => seq !== state.bootSeq });
|
|
625
|
+
// Another await — a multi-second one — and `sendChromeData` below starts a helper process. A
|
|
626
|
+
// shutdown or a newer boot arriving in this gap would otherwise get a helper installed for the
|
|
627
|
+
// device it is taking down — and one that revives itself, so the stale reference outlives the
|
|
628
|
+
// boot that returns just below.
|
|
629
|
+
if (seq !== state.bootSeq) {
|
|
630
|
+
this.abandonBoot(state, seq, sessionId, requestId);
|
|
631
|
+
return;
|
|
632
|
+
}
|
|
633
|
+
this.sendChromeData(state, bootedDevice);
|
|
483
634
|
const streamWs = await this.openStreamWs(state);
|
|
484
635
|
if (seq !== state.bootSeq) {
|
|
485
636
|
streamWs.close();
|
|
637
|
+
this.abandonBoot(state, seq, sessionId, requestId);
|
|
486
638
|
return;
|
|
487
639
|
}
|
|
488
640
|
this.startBinaryStream(state, streamWs);
|
|
@@ -491,7 +643,7 @@ export class IOSAgent {
|
|
|
491
643
|
if (this.audioEnabled())
|
|
492
644
|
this.startAudioCapture(state, streamWs, deviceId);
|
|
493
645
|
state.booted = true;
|
|
494
|
-
this.
|
|
646
|
+
this.sendMsg({ type: 'device:ready', sessionId, requestId, payload: { deviceId } });
|
|
495
647
|
// Sync AppleKeyboards after ready — fire-and-forget so streaming isn't delayed.
|
|
496
648
|
// hw=Automatic lets the hardware layout follow the active input source on LANG1/CapsLock.
|
|
497
649
|
// By the time the user navigates to a text field the sync has already completed.
|
|
@@ -500,10 +652,22 @@ export class IOSAgent {
|
|
|
500
652
|
});
|
|
501
653
|
}
|
|
502
654
|
catch (e) {
|
|
503
|
-
if (seq !== state.bootSeq)
|
|
655
|
+
if (seq !== state.bootSeq) {
|
|
656
|
+
this.abandonBoot(state, seq, sessionId, requestId);
|
|
504
657
|
return;
|
|
505
|
-
|
|
506
|
-
|
|
658
|
+
}
|
|
659
|
+
// `firstLine`, not `e.message`: this reaches a toast, and node's first line is
|
|
660
|
+
// `Command failed: xcrun simctl boot <UDID>`, which says nothing and echoes the udid. That
|
|
661
|
+
// matters more now that the boot is issued unconditionally — a device caught mid `Shutting
|
|
662
|
+
// Down` refuses the boot, and refusing is correct (swallowing it would put us back to waiting
|
|
663
|
+
// on a device nobody is bringing up), so the refusal is what the tester reads.
|
|
664
|
+
this.sendMsg({ type: 'device:boot-error', sessionId, requestId, message: firstLine(e) });
|
|
665
|
+
}
|
|
666
|
+
finally {
|
|
667
|
+
state.bootsInFlight.delete(seq);
|
|
668
|
+
// Its own key, in the one case nothing reads it: a newer boot arriving after this boot's last
|
|
669
|
+
// checkpoint but before it returns.
|
|
670
|
+
state.bootAbandon.delete(seq);
|
|
507
671
|
}
|
|
508
672
|
}
|
|
509
673
|
// Boot a device, auto-recovering from a vanished data dir. simctl lists the device
|
|
@@ -521,11 +685,11 @@ export class IOSAgent {
|
|
|
521
685
|
await this.simctl.boot(deviceId);
|
|
522
686
|
}
|
|
523
687
|
}
|
|
524
|
-
async handleDeviceShutdown(sessionId, deviceId) {
|
|
688
|
+
async handleDeviceShutdown(sessionId, deviceId, requestId) {
|
|
525
689
|
const state = this.deviceStates.get(sessionId);
|
|
526
690
|
if (!state)
|
|
527
691
|
return;
|
|
528
|
-
state
|
|
692
|
+
this.bumpBootSeq(state, 'shut-down');
|
|
529
693
|
void state.streamReader?.cancel();
|
|
530
694
|
state.streamReader = null;
|
|
531
695
|
state.captureStreamer = null; // reader.cancel() kills the helper proc; drop the ref so a stale requestKeyframe() no-ops
|
|
@@ -540,11 +704,12 @@ export class IOSAgent {
|
|
|
540
704
|
this.uiTreeReader.stopIfDevice(deviceId);
|
|
541
705
|
try {
|
|
542
706
|
await this.simctl.shutdown(deviceId);
|
|
543
|
-
this.
|
|
707
|
+
this.sendMsg({
|
|
544
708
|
type: 'device:shutdown-done',
|
|
545
709
|
sessionId,
|
|
710
|
+
requestId,
|
|
546
711
|
payload: { deviceId },
|
|
547
|
-
})
|
|
712
|
+
});
|
|
548
713
|
}
|
|
549
714
|
catch (e) {
|
|
550
715
|
const message = e instanceof Error ? e.message : String(e);
|
|
@@ -578,14 +743,89 @@ export class IOSAgent {
|
|
|
578
743
|
sentinelParked(deviceId) {
|
|
579
744
|
return (this.parkedSentinels.get(deviceId) ?? 0) > 0;
|
|
580
745
|
}
|
|
581
|
-
//
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
746
|
+
// A terminal input naming a session this agent holds no state for. Reachable in a way that is
|
|
747
|
+
// genuinely disputed: `sessionRebind.test.ts` records that a restarted agent is re-seeded from
|
|
748
|
+
// `agent:registered`, so `!state` should never fire — but `registeredSessions` carries one entry
|
|
749
|
+
// per *device* (`RelayServer.ts`, `byDeviceId`), and the relay's own comment there notes that one
|
|
750
|
+
// device can now sit behind two sessions, which leaves the second unseeded. Answering costs four
|
|
751
|
+
// lines; staying silent costs a terminal input swallowed and the caller's own fallback reporting
|
|
752
|
+
// success. Android already answers, and `channel-unavailable` is what it maps this to.
|
|
753
|
+
/** The correlator on an input an ack answers, or `null` if the frame cannot be attributed.
|
|
754
|
+
*
|
|
755
|
+
* A local capture rather than a guard at the top of the dispatcher, and that is the point: a guard there
|
|
756
|
+
* would not narrow `msg.requestId` inside the case, so the shortest way to satisfy the reply's required
|
|
757
|
+
* field would be `msg.requestId!` — the assertion removed from `open-url` and then from clipboard. This
|
|
758
|
+
* hands back a `string` the case can close over.
|
|
759
|
+
*
|
|
760
|
+
* It is unvalidated JSON, so the check is real work rather than ceremony: the declaration is required and
|
|
761
|
+
* every in-repo sender is typed against it, and #444 made the relay refuse an empty one — but this agent may
|
|
762
|
+
* be talking to a relay older than that, and `mcp-server`'s tool
|
|
763
|
+
* schemas are bare `z.string()` so a model can produce `''`. */
|
|
764
|
+
correlatorOf(msg) {
|
|
765
|
+
if (typeof msg.requestId === 'string' && msg.requestId !== '')
|
|
766
|
+
return msg.requestId;
|
|
767
|
+
logger.warn(`${msg.type} without a usable requestId — dropped, its ack could not be attributed`);
|
|
768
|
+
return null;
|
|
769
|
+
}
|
|
770
|
+
ackNoSession(sessionId, requestId) {
|
|
771
|
+
// The `if (!sessionId) return` this used to open with is gone: the dispatcher now declares
|
|
772
|
+
// `sessionId: string`, so there is no undefined to guard against and the guard would have been a
|
|
773
|
+
// silent drop with nothing left that could reach it.
|
|
774
|
+
this.sendMsg({
|
|
775
|
+
type: 'input:error', sessionId, requestId,
|
|
776
|
+
message: INPUT_ERROR_MESSAGES['channel-unavailable'],
|
|
777
|
+
reason: 'channel-unavailable',
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
/**
|
|
781
|
+
* Why a write was refused. The helper's boolean says only "no".
|
|
782
|
+
*
|
|
783
|
+
* `kind` matters because the two questions were decided at different times. Readiness is about now;
|
|
784
|
+
* a continuation frame's fate was decided when the gesture opened. A gesture whose opening frame
|
|
785
|
+
* was refused inside the start-up window owns nothing, so its terminal frame can never land no
|
|
786
|
+
* matter how ready the helper has since become — and reading readiness first answered `malformed`
|
|
787
|
+
* ("never retry") for precisely the case `channel-starting` exists to serve. Ownership is therefore
|
|
788
|
+
* checked first for continuations, which is also the right answer when the helper died mid-gesture
|
|
789
|
+
* and its replacement is still starting: waiting would not help either.
|
|
790
|
+
*
|
|
791
|
+
* Safe to read at the ack site rather than inside the write, because iOS writes are synchronous —
|
|
792
|
+
* but only for the question readiness answers. That is why ownership is asked separately.
|
|
793
|
+
*/
|
|
794
|
+
refusalReason(helper, kind) {
|
|
795
|
+
const state = helper?.inputState() ?? 'unavailable';
|
|
796
|
+
// No channel at all dominates: the caller must reconnect, and telling it to re-open a gesture
|
|
797
|
+
// would send it round a loop that cannot succeed.
|
|
798
|
+
if (state === 'unavailable')
|
|
799
|
+
return 'channel-unavailable';
|
|
800
|
+
// Then ownership, and only for a continuation. A gesture whose opening frame never landed owns
|
|
801
|
+
// nothing, so its terminal frame can never be delivered however ready the helper has become —
|
|
802
|
+
// which is why readiness must not be consulted first here.
|
|
803
|
+
//
|
|
804
|
+
// A consequence worth stating: `channel-starting` is unreachable for a continuation. Owning a
|
|
805
|
+
// gesture requires an opening frame to have landed, which requires the helper to have been ready,
|
|
806
|
+
// so a continuation is never refused *merely* because the channel is still coming up. Standalone
|
|
807
|
+
// inputs — a key, a button — are the ones that get that answer.
|
|
808
|
+
if (kind === 'continuation' && !helper?.ownsGesture())
|
|
809
|
+
return 'no-gesture';
|
|
810
|
+
return state === 'starting' ? 'channel-starting' : 'no-gesture';
|
|
811
|
+
}
|
|
812
|
+
// Ack a terminal input. `input:done` = dispatched to a booted device (not a landing guarantee —
|
|
813
|
+
// HID is fire-and-forget). Otherwise `input:error` carries a machine-readable `reason` so a caller
|
|
814
|
+
// can tell "retry in a moment" from "reconnect" from "never retry"; `message` stays human prose.
|
|
815
|
+
// Off the sync inject path, so start/end pairing is unaffected.
|
|
816
|
+
//
|
|
817
|
+
// `requestId` is a parameter, never read from shared state: a gesture is dozens of frames and two can
|
|
818
|
+
// overlap, so a correlator hoisted out of per-request scope would answer one input with another's id —
|
|
819
|
+
// which is #499 rebuilt inside the agent. Same reason `seq` is passed in on the Android side.
|
|
820
|
+
async ackInput(state, outcome, requestId) {
|
|
821
|
+
const reason = outcome !== 'delivered'
|
|
822
|
+
? outcome
|
|
823
|
+
: (state.booted || (await this.isBooted(state.deviceId))) ? null : 'not-booted';
|
|
824
|
+
if (reason === null)
|
|
585
825
|
state.booted = true; // cache the post-reconnect verify so later inputs skip simctl
|
|
586
|
-
this.
|
|
587
|
-
? { type: 'input:done', sessionId: state.sessionId }
|
|
588
|
-
: { type: 'input:error', sessionId: state.sessionId, message:
|
|
826
|
+
this.sendMsg(reason === null
|
|
827
|
+
? { type: 'input:done', sessionId: state.sessionId, requestId }
|
|
828
|
+
: { type: 'input:error', sessionId: state.sessionId, requestId, message: INPUT_ERROR_MESSAGES[reason], reason });
|
|
589
829
|
}
|
|
590
830
|
async isBooted(deviceId) {
|
|
591
831
|
try {
|
|
@@ -601,39 +841,55 @@ export class IOSAgent {
|
|
|
601
841
|
case 'device:boot': {
|
|
602
842
|
const { deviceId, resetMode, acceptH264, secureContext, external } = msg.payload;
|
|
603
843
|
const sessionId = msg.sessionId;
|
|
604
|
-
this.handleDeviceBoot(sessionId, deviceId, resetMode === 'full-erase', acceptH264 === true, { secureContext: !!secureContext, external: !!external })
|
|
844
|
+
this.handleDeviceBoot(sessionId, deviceId, resetMode === 'full-erase', acceptH264 === true, { secureContext: !!secureContext, external: !!external }, msg.requestId)
|
|
605
845
|
.catch((e) => logger.error('handleDeviceBoot failed:', e));
|
|
606
846
|
break;
|
|
607
847
|
}
|
|
608
848
|
case 'device:shutdown': {
|
|
609
849
|
const { deviceId } = msg.payload;
|
|
610
850
|
const sessionId = msg.sessionId;
|
|
611
|
-
this.handleDeviceShutdown(sessionId, deviceId)
|
|
851
|
+
this.handleDeviceShutdown(sessionId, deviceId, msg.requestId)
|
|
612
852
|
.catch((e) => logger.error('handleDeviceShutdown failed:', e));
|
|
613
853
|
break;
|
|
614
854
|
}
|
|
615
855
|
case 'app:install': {
|
|
616
856
|
const { filePath, bundleId } = msg.payload;
|
|
617
857
|
const sessionId = msg.sessionId;
|
|
858
|
+
const { requestId } = msg;
|
|
859
|
+
if (typeof requestId !== 'string' || requestId === '') {
|
|
860
|
+
console.warn('[tapflow] app:install without a requestId — dropped, cannot correlate a reply');
|
|
861
|
+
break;
|
|
862
|
+
}
|
|
863
|
+
// `...body` first: with the ids last, a body *variable* carrying one would override the real
|
|
864
|
+
// correlator, because excess-property checking does not fire on variables. See `open-url`.
|
|
865
|
+
const respond = (body) => this.sendMsg({ ...body, sessionId, requestId });
|
|
618
866
|
const installState = this.deviceStates.get(sessionId);
|
|
619
867
|
if (!installState) {
|
|
620
|
-
|
|
868
|
+
respond({ type: 'app:install-error', message: 'No booted device' });
|
|
621
869
|
break;
|
|
622
870
|
}
|
|
623
871
|
this.installBuild(installState.deviceId, filePath, bundleId)
|
|
624
|
-
.then(() =>
|
|
872
|
+
.then(() => respond({ type: 'app:install-done' }))
|
|
625
873
|
.catch((e) => {
|
|
626
874
|
const message = e instanceof Error ? e.message : String(e);
|
|
627
|
-
|
|
875
|
+
respond({ type: 'app:install-error', message });
|
|
628
876
|
});
|
|
629
877
|
break;
|
|
630
878
|
}
|
|
631
879
|
case 'app:launch': {
|
|
632
880
|
const { bundleId } = msg.payload;
|
|
633
881
|
const sessionId = msg.sessionId;
|
|
882
|
+
const { requestId } = msg;
|
|
883
|
+
if (typeof requestId !== 'string' || requestId === '') {
|
|
884
|
+
console.warn('[tapflow] app:launch without a requestId — dropped, cannot correlate a reply');
|
|
885
|
+
break;
|
|
886
|
+
}
|
|
887
|
+
// `...body` first: with the ids last, a body *variable* carrying one would override the real
|
|
888
|
+
// correlator, because excess-property checking does not fire on variables. See `open-url`.
|
|
889
|
+
const respond = (body) => this.sendMsg({ ...body, sessionId, requestId });
|
|
634
890
|
const launchState = this.deviceStates.get(sessionId);
|
|
635
891
|
if (!launchState) {
|
|
636
|
-
|
|
892
|
+
respond({ type: 'app:launch-error', message: 'No booted device' });
|
|
637
893
|
break;
|
|
638
894
|
}
|
|
639
895
|
this.simctl.launchApp(launchState.deviceId, bundleId)
|
|
@@ -642,11 +898,11 @@ export class IOSAgent {
|
|
|
642
898
|
this.lastBundleIds.set(launchState.deviceId, bundleId);
|
|
643
899
|
// Audio: the whole-sim tap's poll picks up the launched app process within one interval;
|
|
644
900
|
// no per-launch helper needed.
|
|
645
|
-
|
|
901
|
+
respond({ type: 'app:launch-done' });
|
|
646
902
|
})
|
|
647
903
|
.catch((e) => {
|
|
648
904
|
const message = e instanceof Error ? e.message : String(e);
|
|
649
|
-
|
|
905
|
+
respond({ type: 'app:launch-error', message });
|
|
650
906
|
});
|
|
651
907
|
break;
|
|
652
908
|
}
|
|
@@ -668,11 +924,19 @@ export class IOSAgent {
|
|
|
668
924
|
break;
|
|
669
925
|
}
|
|
670
926
|
case 'input:touch:end': {
|
|
927
|
+
const requestId = this.correlatorOf(msg);
|
|
928
|
+
if (requestId === null)
|
|
929
|
+
break;
|
|
671
930
|
const state = this.deviceStates.get(msg.sessionId);
|
|
672
|
-
if (!state)
|
|
931
|
+
if (!state) {
|
|
932
|
+
this.ackNoSession(msg.sessionId, requestId);
|
|
673
933
|
break;
|
|
674
|
-
|
|
675
|
-
|
|
934
|
+
}
|
|
935
|
+
// The helper's answer, not its existence: a helper whose process has died reports every
|
|
936
|
+
// write as dropped, and that is what the caller needs to hear (#482).
|
|
937
|
+
const helper = state.touchHelper;
|
|
938
|
+
// terminal of a tap/swipe → ack the gesture
|
|
939
|
+
void this.ackInput(state, helper?.touchEnd() ? 'delivered' : this.refusalReason(helper, 'continuation'), requestId);
|
|
676
940
|
break;
|
|
677
941
|
}
|
|
678
942
|
case 'input:pinch:start': {
|
|
@@ -693,11 +957,16 @@ export class IOSAgent {
|
|
|
693
957
|
break;
|
|
694
958
|
}
|
|
695
959
|
case 'input:pinch:end': {
|
|
960
|
+
const requestId = this.correlatorOf(msg);
|
|
961
|
+
if (requestId === null)
|
|
962
|
+
break;
|
|
696
963
|
const state = this.deviceStates.get(msg.sessionId);
|
|
697
|
-
if (!state)
|
|
964
|
+
if (!state) {
|
|
965
|
+
this.ackNoSession(msg.sessionId, requestId);
|
|
698
966
|
break;
|
|
699
|
-
|
|
700
|
-
|
|
967
|
+
}
|
|
968
|
+
const pinchHelper = state.touchHelper;
|
|
969
|
+
void this.ackInput(state, pinchHelper?.pinchEnd() ? 'delivered' : this.refusalReason(pinchHelper, 'continuation'), requestId);
|
|
701
970
|
break;
|
|
702
971
|
}
|
|
703
972
|
case 'input:rotate': {
|
|
@@ -724,24 +993,27 @@ export class IOSAgent {
|
|
|
724
993
|
: this.simctl.hideSoftwareKeyboard(state.deviceId);
|
|
725
994
|
op.then(() => {
|
|
726
995
|
state.softKeyboardVisible = showing;
|
|
727
|
-
this.
|
|
996
|
+
this.sendMsg({
|
|
728
997
|
type: 'keyboard:toggled',
|
|
729
998
|
sessionId: state.sessionId,
|
|
730
999
|
payload: { visible: showing },
|
|
731
|
-
})
|
|
1000
|
+
});
|
|
732
1001
|
}).catch((e) => {
|
|
733
1002
|
logger.error('keyboard toggle failed:', e);
|
|
734
1003
|
});
|
|
735
1004
|
break;
|
|
736
1005
|
}
|
|
737
1006
|
case 'input:type': {
|
|
1007
|
+
const requestId = this.correlatorOf(msg);
|
|
1008
|
+
if (requestId === null)
|
|
1009
|
+
break;
|
|
738
1010
|
const sessionId = msg.sessionId;
|
|
739
1011
|
const state = this.deviceStates.get(sessionId);
|
|
740
1012
|
if (state)
|
|
741
1013
|
this.ensureTouchHelper(state);
|
|
742
1014
|
const { text } = (msg.payload ?? {});
|
|
743
1015
|
if (!state?.touchHelper) {
|
|
744
|
-
this.
|
|
1016
|
+
this.sendMsg({ type: 'input:type-error', sessionId, requestId, message: 'No booted device' });
|
|
745
1017
|
break;
|
|
746
1018
|
}
|
|
747
1019
|
// simctl pbcopy → Cmd+V paste. Works for arbitrary Unicode (unlike a
|
|
@@ -758,92 +1030,155 @@ export class IOSAgent {
|
|
|
758
1030
|
state.softKeyboardVisible = false;
|
|
759
1031
|
await this.simctl.hideSoftwareKeyboard(state.deviceId).catch(() => { });
|
|
760
1032
|
}
|
|
761
|
-
|
|
1033
|
+
// Throwing routes into the input:type-error below. Without it a dropped chord still
|
|
1034
|
+
// answers input:type-done: the text sits on the device pasteboard, nothing was pasted
|
|
1035
|
+
// into the app, and the caller moves on. Unlike the copy path in clipboard:read there
|
|
1036
|
+
// is no pasteboard deadline here to fail loudly in its place.
|
|
1037
|
+
if (!state.touchHelper?.sendKey(KEY_CODE_MAP['KeyV'], MODIFIER_BITS['MetaLeft'])) {
|
|
1038
|
+
throw new PlatformError('Cannot paste the text — no input channel to the device');
|
|
1039
|
+
}
|
|
762
1040
|
};
|
|
763
1041
|
// Ack on completion so a following input step (e.g. pressKey Enter) is
|
|
764
1042
|
// only sent after the paste has actually landed.
|
|
765
1043
|
// Shares the clipboard queue: this writes the pasteboard, so running it alongside a
|
|
766
1044
|
// clipboard:read would overwrite that read's sentinel and be returned as "copied".
|
|
767
1045
|
this.clipboardQueue(state.deviceId, doType)
|
|
768
|
-
.then(() => this.
|
|
1046
|
+
.then(() => this.sendMsg({ type: 'input:type-done', sessionId, requestId }))
|
|
769
1047
|
.catch((e) => {
|
|
770
1048
|
const message = e instanceof Error ? e.message : String(e);
|
|
771
1049
|
logger.error('input:type (pbcopy+paste) failed:', e);
|
|
772
|
-
this.
|
|
1050
|
+
this.sendMsg({ type: 'input:type-error', sessionId, requestId, message });
|
|
773
1051
|
});
|
|
774
1052
|
break;
|
|
775
1053
|
}
|
|
776
1054
|
case 'input:key': {
|
|
1055
|
+
const requestId = this.correlatorOf(msg);
|
|
1056
|
+
if (requestId === null)
|
|
1057
|
+
break;
|
|
777
1058
|
const state = this.deviceStates.get(msg.sessionId);
|
|
778
|
-
if (!state)
|
|
1059
|
+
if (!state) {
|
|
1060
|
+
this.ackNoSession(msg.sessionId, requestId);
|
|
779
1061
|
break;
|
|
1062
|
+
}
|
|
780
1063
|
this.ensureTouchHelper(state);
|
|
781
1064
|
const { code, modifiers } = msg.payload;
|
|
782
1065
|
const usage = KEY_CODE_MAP[code];
|
|
783
1066
|
if (usage === undefined) {
|
|
784
|
-
|
|
1067
|
+
// Prose preserved (it names the code), reason added: `unsupported` tells a caller never to
|
|
1068
|
+
// retry, which the message alone could not.
|
|
1069
|
+
this.sendMsg({
|
|
1070
|
+
type: 'input:error', sessionId: msg.sessionId, requestId,
|
|
1071
|
+
message: `unknown key code: ${code}`, reason: 'unsupported',
|
|
1072
|
+
});
|
|
785
1073
|
break;
|
|
786
1074
|
}
|
|
787
1075
|
if (state.softKeyboardVisible) {
|
|
788
1076
|
// Hide the SW keyboard first so iOS re-initialises the HW keyboard
|
|
789
1077
|
// context. Skipping this causes input-source desync (qks / ㅂㅏㄴ symptoms).
|
|
790
1078
|
state.softKeyboardVisible = false;
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
1079
|
+
// The ack belongs after the deferred chord, not beside it. Acking out here ran before
|
|
1080
|
+
// the key was ever written, so the answer could only be a guess — and the guess was
|
|
1081
|
+
// "delivered". The hide failing is not fatal (the key still goes out), so the catch
|
|
1082
|
+
// resolves and the send happens exactly once either way.
|
|
1083
|
+
void this.simctl.hideSoftwareKeyboard(state.deviceId)
|
|
1084
|
+
.catch((e) => { logger.error('hideSoftwareKeyboard (on key) failed:', e); })
|
|
1085
|
+
.then(() => {
|
|
1086
|
+
const h = state.touchHelper;
|
|
1087
|
+
return this.ackInput(state, h?.sendKey(usage, modifiers ?? 0) ? 'delivered' : this.refusalReason(h, 'standalone'), requestId);
|
|
796
1088
|
});
|
|
797
1089
|
}
|
|
798
1090
|
else {
|
|
799
|
-
state.touchHelper
|
|
1091
|
+
const h = state.touchHelper;
|
|
1092
|
+
void this.ackInput(state, h?.sendKey(usage, modifiers ?? 0) ? 'delivered' : this.refusalReason(h, 'standalone'), requestId);
|
|
800
1093
|
}
|
|
801
|
-
void this.ackInput(state, state.touchHelper !== null);
|
|
802
1094
|
break;
|
|
803
1095
|
}
|
|
804
1096
|
case 'input:button': {
|
|
1097
|
+
const requestId = this.correlatorOf(msg);
|
|
1098
|
+
if (requestId === null)
|
|
1099
|
+
break;
|
|
805
1100
|
const state = this.deviceStates.get(msg.sessionId);
|
|
806
|
-
if (!state)
|
|
1101
|
+
if (!state) {
|
|
1102
|
+
this.ackNoSession(msg.sessionId, requestId);
|
|
807
1103
|
break;
|
|
1104
|
+
}
|
|
808
1105
|
this.ensureTouchHelper(state);
|
|
809
1106
|
const { name, phase } = msg.payload;
|
|
810
1107
|
// Map the cross-platform button vocabulary (used by MCP) onto this
|
|
811
1108
|
// device's actual chrome button names. Dashboard already sends the raw
|
|
812
1109
|
// chrome names (e.g. "volume-up"), which pass through unchanged.
|
|
813
1110
|
const chromeName = IOS_BUTTON_ALIASES[name] ?? name;
|
|
1111
|
+
// Two branches below deliberately write nothing — a home press-down, and a button this
|
|
1112
|
+
// device's chrome has no HID mapping for. Neither is a dropped input, so they answer
|
|
1113
|
+
// with the channel's health instead of a write result; reporting an error there would
|
|
1114
|
+
// trade this issue's false success for a false failure.
|
|
1115
|
+
const btnHelper = state.touchHelper;
|
|
1116
|
+
const wrote = (ok) => ok ? 'delivered' : this.refusalReason(btnHelper, 'standalone');
|
|
1117
|
+
// The two branches that write nothing answer from the channel's state, which is `delivered`
|
|
1118
|
+
// when it is ready. A button this device's chrome lacks stays a *success* — the device
|
|
1119
|
+
// genuinely has no such button, so an error would be a false failure (#484). That is a
|
|
1120
|
+
// decision, and it is why iOS never sends `unsupported` for a button while Android does.
|
|
1121
|
+
let outcome = btnHelper?.inputState() === 'ready'
|
|
1122
|
+
? 'delivered'
|
|
1123
|
+
: this.refusalReason(btnHelper, 'standalone');
|
|
814
1124
|
if (chromeName === 'home') {
|
|
815
1125
|
// Home has no HID down/up split — always a single legacy press. Send once on release
|
|
816
1126
|
// (or on a phase-less legacy message) so a down+up pair doesn't fire it twice.
|
|
817
1127
|
if (phase !== 'down')
|
|
818
|
-
|
|
1128
|
+
outcome = wrote(btnHelper?.pressLegacyButton(0) ?? false);
|
|
819
1129
|
}
|
|
820
1130
|
else {
|
|
821
1131
|
const btn = state.loadedChrome?.buttons.find((b) => b.name === chromeName);
|
|
822
1132
|
if (btn && btn.usagePage > 0 && btn.usage > 0) {
|
|
823
1133
|
if (phase === 'down')
|
|
824
|
-
|
|
1134
|
+
outcome = wrote(btnHelper?.pressButtonDown(btn.usagePage, btn.usage) ?? false);
|
|
825
1135
|
else if (phase === 'up')
|
|
826
|
-
|
|
1136
|
+
outcome = wrote(btnHelper?.pressButtonUp(btn.usagePage, btn.usage) ?? false);
|
|
827
1137
|
else
|
|
828
|
-
|
|
1138
|
+
outcome = wrote(btnHelper?.pressButton(btn.usagePage, btn.usage) ?? false);
|
|
829
1139
|
}
|
|
830
1140
|
}
|
|
831
|
-
void this.ackInput(state,
|
|
1141
|
+
void this.ackInput(state, outcome, requestId);
|
|
832
1142
|
break;
|
|
833
1143
|
}
|
|
834
1144
|
case 'open-url': {
|
|
835
1145
|
const { url } = msg.payload;
|
|
836
1146
|
const sessionId = msg.sessionId;
|
|
1147
|
+
const { requestId } = msg;
|
|
1148
|
+
// No fallback by design (see the note above `OpenUrl`), so an uncorrelatable request cannot be
|
|
1149
|
+
// answered correlatably either — and inventing an id would make this agent's reply look like a
|
|
1150
|
+
// response to a request nobody made. Every in-repo sender supplies one, and the `fixed` version
|
|
1151
|
+
// group means there is no in-repo skew window; validating third-party frames at the relay's door
|
|
1152
|
+
// is #444, which will take this over. Until then a drop with a log beats a reply that lies.
|
|
1153
|
+
if (typeof requestId !== 'string' || requestId === '') {
|
|
1154
|
+
console.warn('[tapflow] open-url without a requestId — dropped, cannot correlate a reply');
|
|
1155
|
+
break;
|
|
1156
|
+
}
|
|
1157
|
+
// Every exit merges the correlation ids here. What that buys, precisely — review measured 13
|
|
1158
|
+
// attacks and the type caught 3:
|
|
1159
|
+
//
|
|
1160
|
+
// - **Omitting the correlator is a compile error.** That comes from `requestId: string` being
|
|
1161
|
+
// required on `OpenUrlDone`/`OpenUrlError`, reached through `sendMsg`'s `AgentControlOutbound`
|
|
1162
|
+
// — not from `OpenUrlReplyBody`, whose one contribution is rejecting a fresh id written as a
|
|
1163
|
+
// literal at the `respond(...)` call.
|
|
1164
|
+
// - **`...body` goes first on purpose.** With the ids last, a body *variable* carrying a
|
|
1165
|
+
// `requestId` cannot override them — excess-property checking does not fire on variables, so
|
|
1166
|
+
// the earlier `{ sessionId, requestId, ...body }` let a wrong id win.
|
|
1167
|
+
//
|
|
1168
|
+
// What it does **not** buy: `sendMsg` accepts any `string` here, so a site that bypasses
|
|
1169
|
+
// `respond` type-checks. The echo tests in both agents' suites are what catch that, and each
|
|
1170
|
+
// remaining correlation pair needs its own — this helper does not remove that work.
|
|
1171
|
+
const respond = (body) => this.sendMsg({ ...body, sessionId, requestId });
|
|
837
1172
|
const state = this.deviceStates.get(sessionId);
|
|
838
1173
|
if (!state) {
|
|
839
|
-
|
|
1174
|
+
respond({ type: 'open-url:error', message: 'no booted device' });
|
|
840
1175
|
break;
|
|
841
1176
|
}
|
|
842
1177
|
this.simctl.openUrl(state.deviceId, url)
|
|
843
|
-
.then(() =>
|
|
1178
|
+
.then(() => respond({ type: 'open-url:done' }))
|
|
844
1179
|
.catch((e) => {
|
|
845
1180
|
const message = e instanceof Error ? e.message : String(e);
|
|
846
|
-
|
|
1181
|
+
respond({ type: 'open-url:error', message });
|
|
847
1182
|
});
|
|
848
1183
|
break;
|
|
849
1184
|
}
|
|
@@ -853,7 +1188,7 @@ export class IOSAgent {
|
|
|
853
1188
|
const sessionId = msg.sessionId;
|
|
854
1189
|
const shotState = this.deviceStates.get(sessionId);
|
|
855
1190
|
if (!shotState) {
|
|
856
|
-
this.
|
|
1191
|
+
this.sendMsg({ type: 'screenshot:error', sessionId, requestId, message: 'No booted device' });
|
|
857
1192
|
break;
|
|
858
1193
|
}
|
|
859
1194
|
// Note for anyone widening this signature again: before the udid parameter existed this
|
|
@@ -861,32 +1196,38 @@ export class IOSAgent {
|
|
|
861
1196
|
// type error — the format string simply became the device id. The compiler cannot help
|
|
862
1197
|
// here; the test that scans exec arguments is what does.
|
|
863
1198
|
this.simctl.screenshot(shotState.deviceId, format ?? 'png')
|
|
864
|
-
.then((buf) => this.
|
|
1199
|
+
.then((buf) => this.sendMsg({
|
|
865
1200
|
type: 'screenshot:done',
|
|
866
1201
|
sessionId,
|
|
867
1202
|
requestId,
|
|
868
1203
|
format: format ?? 'png',
|
|
869
1204
|
data: buf.toString('base64'),
|
|
870
|
-
}))
|
|
1205
|
+
}))
|
|
871
1206
|
.catch((e) => {
|
|
872
1207
|
const message = e instanceof Error ? e.message : String(e);
|
|
873
|
-
this.
|
|
1208
|
+
this.sendMsg({ type: 'screenshot:error', sessionId, requestId, message });
|
|
874
1209
|
});
|
|
875
1210
|
break;
|
|
876
1211
|
}
|
|
877
1212
|
case 'app:clear-state': {
|
|
878
1213
|
const { bundleId } = (msg.payload ?? {});
|
|
879
1214
|
const sessionId = msg.sessionId;
|
|
1215
|
+
const { requestId } = msg;
|
|
1216
|
+
if (typeof requestId !== 'string' || requestId === '') {
|
|
1217
|
+
console.warn('[tapflow] app:clear-state without a requestId — dropped, cannot correlate a reply');
|
|
1218
|
+
break;
|
|
1219
|
+
}
|
|
1220
|
+
const respond = (body) => this.sendMsg({ ...body, sessionId, requestId });
|
|
880
1221
|
const state = this.deviceStates.get(sessionId);
|
|
881
1222
|
if (!state || !bundleId) {
|
|
882
|
-
|
|
1223
|
+
respond({ type: 'app:clear-state-error', message: !state ? 'No booted device' : 'bundleId missing' });
|
|
883
1224
|
break;
|
|
884
1225
|
}
|
|
885
1226
|
this.simctl.clearAppData(state.deviceId, bundleId)
|
|
886
|
-
.then(() =>
|
|
1227
|
+
.then(() => respond({ type: 'app:clear-state-done' }))
|
|
887
1228
|
.catch((e) => {
|
|
888
1229
|
const message = e instanceof Error ? e.message : String(e);
|
|
889
|
-
|
|
1230
|
+
respond({ type: 'app:clear-state-error', message });
|
|
890
1231
|
});
|
|
891
1232
|
break;
|
|
892
1233
|
}
|
|
@@ -895,14 +1236,22 @@ export class IOSAgent {
|
|
|
895
1236
|
// await hideSoftwareKeyboard first), and reading too early returns the PREVIOUS
|
|
896
1237
|
// pasteboard — a stale value the user would never notice.
|
|
897
1238
|
case 'clipboard:read': {
|
|
1239
|
+
// `requestId: string`, matching the screenshot and ui:tree casts a few cases below — clipboard was
|
|
1240
|
+
// the only one reading it as optional, for the same wire guarantee. `ClipboardRequest.requestId` is
|
|
1241
|
+
// required and the only requester goes through a typed `send()`, so nothing in-repo omits it.
|
|
1242
|
+
//
|
|
1243
|
+
// It is still an assertion about unvalidated JSON, as those other two are: a third-party client
|
|
1244
|
+
// could omit it, and then the reply would carry `undefined` and be uncorrelatable. That is inbound
|
|
1245
|
+
// validation (#444), not producer typing, and making it optional here instead would just move the
|
|
1246
|
+
// same hole into every reply this case sends.
|
|
898
1247
|
const { requestId } = msg;
|
|
899
1248
|
const sessionId = msg.sessionId;
|
|
900
1249
|
const state = this.deviceStates.get(sessionId);
|
|
901
1250
|
if (!state) {
|
|
902
|
-
this.
|
|
1251
|
+
this.sendMsg({
|
|
903
1252
|
type: 'clipboard:error', sessionId, requestId, message: 'No booted device',
|
|
904
1253
|
payload: { sentinelParked: false },
|
|
905
|
-
})
|
|
1254
|
+
});
|
|
906
1255
|
break;
|
|
907
1256
|
}
|
|
908
1257
|
const { press } = (msg.payload ?? {});
|
|
@@ -910,7 +1259,7 @@ export class IOSAgent {
|
|
|
910
1259
|
// executes AFTER this — so the viewer hears back as soon as the outcome is known and
|
|
911
1260
|
// does not wait out the restore window. The queue is still held until the restore
|
|
912
1261
|
// finishes, which is what actually matters: the next operation must not see a sentinel.
|
|
913
|
-
const respond = (body) => this.
|
|
1262
|
+
const respond = (body) => this.sendMsg({ sessionId, requestId, ...body });
|
|
914
1263
|
const read = async () => {
|
|
915
1264
|
if (!press) {
|
|
916
1265
|
respond({ type: 'clipboard:data', payload: { text: await this.simctl.getPasteboard(state.deviceId) } });
|
|
@@ -1023,23 +1372,31 @@ export class IOSAgent {
|
|
|
1023
1372
|
break;
|
|
1024
1373
|
}
|
|
1025
1374
|
case 'clipboard:write': {
|
|
1375
|
+
// `requestId: string`, matching the screenshot and ui:tree casts a few cases below — clipboard was
|
|
1376
|
+
// the only one reading it as optional, for the same wire guarantee. `ClipboardRequest.requestId` is
|
|
1377
|
+
// required and the only requester goes through a typed `send()`, so nothing in-repo omits it.
|
|
1378
|
+
//
|
|
1379
|
+
// It is still an assertion about unvalidated JSON, as those other two are: a third-party client
|
|
1380
|
+
// could omit it, and then the reply would carry `undefined` and be uncorrelatable. That is inbound
|
|
1381
|
+
// validation (#444), not producer typing, and making it optional here instead would just move the
|
|
1382
|
+
// same hole into every reply this case sends.
|
|
1026
1383
|
const { requestId } = msg;
|
|
1027
1384
|
const sessionId = msg.sessionId;
|
|
1028
1385
|
const state = this.deviceStates.get(sessionId);
|
|
1029
1386
|
if (!state) {
|
|
1030
|
-
this.
|
|
1387
|
+
this.sendMsg({
|
|
1031
1388
|
type: 'clipboard:error', sessionId, requestId, message: 'No booted device',
|
|
1032
1389
|
payload: { sentinelParked: false },
|
|
1033
|
-
})
|
|
1390
|
+
});
|
|
1034
1391
|
break;
|
|
1035
1392
|
}
|
|
1036
1393
|
const { text, pasteAfter } = (msg.payload ?? {});
|
|
1037
1394
|
if (clipboardByteLength(text ?? '') > MAX_CLIPBOARD_BYTES) {
|
|
1038
|
-
this.
|
|
1395
|
+
this.sendMsg({
|
|
1039
1396
|
type: 'clipboard:error', sessionId, requestId,
|
|
1040
1397
|
message: `Clipboard is too large (max ${Math.floor(MAX_CLIPBOARD_BYTES / 1024)} KB)`,
|
|
1041
1398
|
payload: { sentinelParked: this.sentinelParked(state.deviceId) },
|
|
1042
|
-
})
|
|
1399
|
+
});
|
|
1043
1400
|
break;
|
|
1044
1401
|
}
|
|
1045
1402
|
const write = async () => {
|
|
@@ -1064,17 +1421,22 @@ export class IOSAgent {
|
|
|
1064
1421
|
state.softKeyboardVisible = false;
|
|
1065
1422
|
await this.simctl.hideSoftwareKeyboard(state.deviceId).catch(() => { });
|
|
1066
1423
|
}
|
|
1067
|
-
|
|
1424
|
+
// Same reason as input:type: the deadline above proves the pasteboard took the text,
|
|
1425
|
+
// not that the chord reached the device, so a dead channel here would answer
|
|
1426
|
+
// clipboard:write-done having pasted nothing.
|
|
1427
|
+
if (!state.touchHelper?.sendKey(KEY_CODE_MAP['KeyV'], MODIFIER_BITS['MetaLeft'])) {
|
|
1428
|
+
throw new PlatformError('Cannot press paste — no input channel to the device');
|
|
1429
|
+
}
|
|
1068
1430
|
};
|
|
1069
1431
|
// Ack only once the write (and the paste, when asked for) actually landed.
|
|
1070
1432
|
this.clipboardQueue(state.deviceId, write)
|
|
1071
|
-
.then(() => this.
|
|
1433
|
+
.then(() => this.sendMsg({ type: 'clipboard:write-done', sessionId, requestId }))
|
|
1072
1434
|
.catch((e) => {
|
|
1073
1435
|
const message = e instanceof Error ? e.message : String(e);
|
|
1074
|
-
this.
|
|
1436
|
+
this.sendMsg({
|
|
1075
1437
|
type: 'clipboard:error', sessionId, requestId, message,
|
|
1076
1438
|
payload: { sentinelParked: this.sentinelParked(state.deviceId) },
|
|
1077
|
-
})
|
|
1439
|
+
});
|
|
1078
1440
|
});
|
|
1079
1441
|
break;
|
|
1080
1442
|
}
|
|
@@ -1084,14 +1446,14 @@ export class IOSAgent {
|
|
|
1084
1446
|
const sessionId = msg.sessionId;
|
|
1085
1447
|
const state = this.deviceStates.get(sessionId);
|
|
1086
1448
|
if (!state) {
|
|
1087
|
-
this.
|
|
1449
|
+
this.sendMsg({ type: 'ui:tree:error', sessionId, requestId, message: 'No booted device' });
|
|
1088
1450
|
break;
|
|
1089
1451
|
}
|
|
1090
1452
|
this.readUITree(state)
|
|
1091
|
-
.then((elements) => this.
|
|
1453
|
+
.then((elements) => this.sendMsg({ type: 'ui:tree:response', sessionId, requestId, elements }))
|
|
1092
1454
|
.catch((e) => {
|
|
1093
1455
|
const message = e instanceof Error ? e.message : String(e);
|
|
1094
|
-
this.
|
|
1456
|
+
this.sendMsg({ type: 'ui:tree:error', sessionId, requestId, message });
|
|
1095
1457
|
});
|
|
1096
1458
|
break;
|
|
1097
1459
|
}
|