@houwert/conductor 0.25.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/add-media.js +36 -0
- package/dist/commands/airplane-mode.js +46 -0
- package/dist/commands/assert-screenshot.js +68 -0
- package/dist/commands/assert-true.js +30 -0
- package/dist/commands/copy-text-from.js +63 -0
- package/dist/commands/record-video.js +126 -0
- package/dist/commands/set-permissions.js +54 -0
- package/dist/commands/stream-server.js +28 -0
- package/dist/commands/tap.js +41 -18
- package/dist/commands/travel.js +45 -0
- package/dist/daemon/h264-annexb.js +264 -0
- package/dist/daemon/server.js +47 -0
- package/dist/daemon/video-hub.js +58 -0
- package/dist/daemon/video-protocol.js +38 -0
- package/dist/daemon/video-server.js +88 -0
- package/dist/daemon/video-source.js +142 -0
- package/dist/drivers/android.js +4 -0
- package/dist/drivers/bootstrap.js +38 -0
- package/dist/drivers/flow-runner.js +1 -0
- package/dist/index.js +90 -0
- package/dist/runner.js +45 -0
- package/package.json +4 -1
- package/skills/conductor-device-interact/SKILL.md +24 -2
- package/skills/conductor-inspect/SKILL.md +3 -1
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* H.264 Annex B streaming parser (daemon side).
|
|
4
|
+
*
|
|
5
|
+
* Splits the raw Annex B byte stream from the capture backend into access units,
|
|
6
|
+
* extracts SPS/PPS to build the stream config, and preserves each access unit
|
|
7
|
+
* **in Annex B form** (start codes intact) for forwarding to subscribers — the
|
|
8
|
+
* wire contract is "config + a sequence of Annex B access units, keyframe-led".
|
|
9
|
+
*
|
|
10
|
+
* Ported from Argus's electron/services/simulator/h264-parser.ts; the SPS
|
|
11
|
+
* bit-reader is identical, but access units are emitted as Annex B rather than
|
|
12
|
+
* converted to AVCC length-prefix (that conversion is left to WebCodecs clients).
|
|
13
|
+
*/
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.H264AnnexBParser = void 0;
|
|
16
|
+
const NAL_SLICE = 1;
|
|
17
|
+
const NAL_IDR = 5;
|
|
18
|
+
const NAL_SPS = 7;
|
|
19
|
+
const NAL_PPS = 8;
|
|
20
|
+
// ── Exp-Golomb reader (minimal, for SPS parsing) ────────────────────────────
|
|
21
|
+
class ExpGolombReader {
|
|
22
|
+
constructor(data) {
|
|
23
|
+
this.data = data;
|
|
24
|
+
this.byteOffset = 0;
|
|
25
|
+
this.bitOffset = 0;
|
|
26
|
+
}
|
|
27
|
+
readBit() {
|
|
28
|
+
if (this.byteOffset >= this.data.length)
|
|
29
|
+
return 0;
|
|
30
|
+
const bit = (this.data[this.byteOffset] >> (7 - this.bitOffset)) & 1;
|
|
31
|
+
this.bitOffset++;
|
|
32
|
+
if (this.bitOffset === 8) {
|
|
33
|
+
this.bitOffset = 0;
|
|
34
|
+
this.byteOffset++;
|
|
35
|
+
}
|
|
36
|
+
return bit;
|
|
37
|
+
}
|
|
38
|
+
readBits(n) {
|
|
39
|
+
let val = 0;
|
|
40
|
+
for (let i = 0; i < n; i++)
|
|
41
|
+
val = (val << 1) | this.readBit();
|
|
42
|
+
return val;
|
|
43
|
+
}
|
|
44
|
+
readUE() {
|
|
45
|
+
let zeros = 0;
|
|
46
|
+
while (this.readBit() === 0 && zeros < 31)
|
|
47
|
+
zeros++;
|
|
48
|
+
if (zeros === 0)
|
|
49
|
+
return 0;
|
|
50
|
+
return (1 << zeros) - 1 + this.readBits(zeros);
|
|
51
|
+
}
|
|
52
|
+
readSE() {
|
|
53
|
+
const val = this.readUE();
|
|
54
|
+
return val % 2 === 0 ? -(val >> 1) : (val + 1) >> 1;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** Parse the SPS fields needed for config. `body` is the NAL after the header byte. */
|
|
58
|
+
function parseSps(body) {
|
|
59
|
+
const r = new ExpGolombReader(body);
|
|
60
|
+
const profileIdc = r.readBits(8);
|
|
61
|
+
const constraintFlags = r.readBits(8);
|
|
62
|
+
const levelIdc = r.readBits(8);
|
|
63
|
+
r.readUE(); // seq_parameter_set_id
|
|
64
|
+
if ([100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134].includes(profileIdc)) {
|
|
65
|
+
const chromaFormatIdc = r.readUE();
|
|
66
|
+
if (chromaFormatIdc === 3)
|
|
67
|
+
r.readBits(1);
|
|
68
|
+
r.readUE(); // bit_depth_luma_minus8
|
|
69
|
+
r.readUE(); // bit_depth_chroma_minus8
|
|
70
|
+
r.readBits(1); // qpprime_y_zero_transform_bypass_flag
|
|
71
|
+
if (r.readBits(1)) {
|
|
72
|
+
const count = chromaFormatIdc !== 3 ? 8 : 12;
|
|
73
|
+
for (let i = 0; i < count; i++) {
|
|
74
|
+
if (r.readBits(1)) {
|
|
75
|
+
const size = i < 6 ? 16 : 64;
|
|
76
|
+
let lastScale = 8;
|
|
77
|
+
let nextScale = 8;
|
|
78
|
+
for (let j = 0; j < size; j++) {
|
|
79
|
+
if (nextScale !== 0)
|
|
80
|
+
nextScale = (lastScale + r.readSE() + 256) % 256;
|
|
81
|
+
lastScale = nextScale === 0 ? lastScale : nextScale;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
r.readUE(); // log2_max_frame_num_minus4
|
|
88
|
+
const picOrderCntType = r.readUE();
|
|
89
|
+
if (picOrderCntType === 0) {
|
|
90
|
+
r.readUE();
|
|
91
|
+
}
|
|
92
|
+
else if (picOrderCntType === 1) {
|
|
93
|
+
r.readBits(1);
|
|
94
|
+
r.readSE();
|
|
95
|
+
r.readSE();
|
|
96
|
+
const n = r.readUE();
|
|
97
|
+
for (let i = 0; i < n; i++)
|
|
98
|
+
r.readSE();
|
|
99
|
+
}
|
|
100
|
+
r.readUE(); // max_num_ref_frames
|
|
101
|
+
r.readBits(1); // gaps_in_frame_num_value_allowed_flag
|
|
102
|
+
const picWidthInMbsMinus1 = r.readUE();
|
|
103
|
+
const picHeightInMapUnitsMinus1 = r.readUE();
|
|
104
|
+
const frameMbsOnlyFlag = r.readBits(1);
|
|
105
|
+
let width = (picWidthInMbsMinus1 + 1) * 16;
|
|
106
|
+
let height = (2 - frameMbsOnlyFlag) * (picHeightInMapUnitsMinus1 + 1) * 16;
|
|
107
|
+
if (!frameMbsOnlyFlag)
|
|
108
|
+
r.readBits(1);
|
|
109
|
+
r.readBits(1); // direct_8x8_inference_flag
|
|
110
|
+
if (r.readBits(1)) {
|
|
111
|
+
// frame_cropping_flag — 4:2:0 crop units are 2px
|
|
112
|
+
const cl = r.readUE();
|
|
113
|
+
const cr = r.readUE();
|
|
114
|
+
const ct = r.readUE();
|
|
115
|
+
const cb = r.readUE();
|
|
116
|
+
width -= (cl + cr) * 2;
|
|
117
|
+
height -= (ct + cb) * 2;
|
|
118
|
+
}
|
|
119
|
+
return { profileIdc, constraintFlags, levelIdc, width, height };
|
|
120
|
+
}
|
|
121
|
+
function buildCodecString(info) {
|
|
122
|
+
const pp = info.profileIdc.toString(16).padStart(2, '0');
|
|
123
|
+
const cc = info.constraintFlags.toString(16).padStart(2, '0');
|
|
124
|
+
const ll = info.levelIdc.toString(16).padStart(2, '0');
|
|
125
|
+
return `avc1.${pp}${cc}${ll}`;
|
|
126
|
+
}
|
|
127
|
+
/** AVCDecoderConfigurationRecord from raw SPS/PPS NAL bodies (with NAL header byte). */
|
|
128
|
+
function buildAvcC(sps, pps) {
|
|
129
|
+
const buf = Buffer.alloc(11 + sps.length + pps.length);
|
|
130
|
+
buf[0] = 1; // configurationVersion
|
|
131
|
+
buf[1] = sps[1]; // profile
|
|
132
|
+
buf[2] = sps[2]; // compatibility
|
|
133
|
+
buf[3] = sps[3]; // level
|
|
134
|
+
buf[4] = 0xff; // lengthSizeMinusOne = 3
|
|
135
|
+
buf[5] = 0xe1; // numOfSPS = 1
|
|
136
|
+
buf.writeUInt16BE(sps.length, 6);
|
|
137
|
+
Buffer.from(sps).copy(buf, 8);
|
|
138
|
+
const off = 8 + sps.length;
|
|
139
|
+
buf[off] = 1; // numOfPPS
|
|
140
|
+
buf.writeUInt16BE(pps.length, off + 1);
|
|
141
|
+
Buffer.from(pps).copy(buf, off + 3);
|
|
142
|
+
return buf;
|
|
143
|
+
}
|
|
144
|
+
/** Find the next Annex B start code (00 00 01 or 00 00 00 01). */
|
|
145
|
+
function findStartCode(buf, offset) {
|
|
146
|
+
for (let i = offset; i < buf.length - 2; i++) {
|
|
147
|
+
if (buf[i] === 0 && buf[i + 1] === 0) {
|
|
148
|
+
if (buf[i + 2] === 1)
|
|
149
|
+
return { index: i, length: 3 };
|
|
150
|
+
if (buf[i + 2] === 0 && i + 3 < buf.length && buf[i + 3] === 1)
|
|
151
|
+
return { index: i, length: 4 };
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Streaming Annex B parser. Feed raw capture bytes with push(); emits config
|
|
158
|
+
* once SPS+PPS are seen, then one Annex B access unit per VCL frame.
|
|
159
|
+
*/
|
|
160
|
+
class H264AnnexBParser {
|
|
161
|
+
constructor(cbs, fps = 30) {
|
|
162
|
+
this.cbs = cbs;
|
|
163
|
+
this.buffer = Buffer.alloc(0);
|
|
164
|
+
this.sps = null;
|
|
165
|
+
this.pps = null;
|
|
166
|
+
this.config = null;
|
|
167
|
+
// NALs (each Annex-B, start-code prefixed) buffered for the current access unit.
|
|
168
|
+
this.au = [];
|
|
169
|
+
this.auIsKey = false;
|
|
170
|
+
this.fps = fps;
|
|
171
|
+
}
|
|
172
|
+
reset() {
|
|
173
|
+
this.buffer = Buffer.alloc(0);
|
|
174
|
+
this.sps = null;
|
|
175
|
+
this.pps = null;
|
|
176
|
+
this.config = null;
|
|
177
|
+
this.au = [];
|
|
178
|
+
this.auIsKey = false;
|
|
179
|
+
}
|
|
180
|
+
push(chunk) {
|
|
181
|
+
this.buffer = this.buffer.length === 0 ? chunk : Buffer.concat([this.buffer, chunk]);
|
|
182
|
+
this.processBuffer();
|
|
183
|
+
}
|
|
184
|
+
processBuffer() {
|
|
185
|
+
let sc = findStartCode(this.buffer, 0);
|
|
186
|
+
if (!sc)
|
|
187
|
+
return;
|
|
188
|
+
while (true) {
|
|
189
|
+
const next = findStartCode(this.buffer, sc.index + sc.length);
|
|
190
|
+
if (!next)
|
|
191
|
+
break; // incomplete NAL — wait for more data
|
|
192
|
+
const nal = this.buffer.subarray(sc.index + sc.length, next.index);
|
|
193
|
+
// Keep the whole unit including start code so subscribers get Annex B.
|
|
194
|
+
const withStartCode = this.buffer.subarray(sc.index, next.index);
|
|
195
|
+
this.processNal(nal, withStartCode);
|
|
196
|
+
sc = next;
|
|
197
|
+
}
|
|
198
|
+
// Retain the trailing (unterminated) NAL for the next push.
|
|
199
|
+
if (sc.index > 0)
|
|
200
|
+
this.buffer = Buffer.from(this.buffer.subarray(sc.index));
|
|
201
|
+
// Guard against unbounded growth on a stuck stream.
|
|
202
|
+
if (this.buffer.length > 4 * 1024 * 1024) {
|
|
203
|
+
this.buffer = Buffer.from(this.buffer.subarray(-1024 * 1024));
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
processNal(nal, withStartCode) {
|
|
207
|
+
if (nal.length === 0)
|
|
208
|
+
return;
|
|
209
|
+
const nalType = nal[0] & 0x1f;
|
|
210
|
+
if (nalType === NAL_SPS) {
|
|
211
|
+
this.sps = Buffer.from(nal);
|
|
212
|
+
this.emitConfigIfReady();
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
if (nalType === NAL_PPS) {
|
|
216
|
+
this.pps = Buffer.from(nal);
|
|
217
|
+
this.emitConfigIfReady();
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
if (nalType === NAL_IDR || nalType === NAL_SLICE) {
|
|
221
|
+
// Flush any previously buffered AU before starting this VCL frame.
|
|
222
|
+
if (this.au.length > 0)
|
|
223
|
+
this.emitAccessUnit();
|
|
224
|
+
this.auIsKey = nalType === NAL_IDR;
|
|
225
|
+
this.au.push(Buffer.from(withStartCode));
|
|
226
|
+
this.emitAccessUnit();
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
// SEI / AUD / other — attach to the current access unit.
|
|
230
|
+
this.au.push(Buffer.from(withStartCode));
|
|
231
|
+
}
|
|
232
|
+
emitAccessUnit() {
|
|
233
|
+
if (this.au.length === 0)
|
|
234
|
+
return;
|
|
235
|
+
if (!this.config) {
|
|
236
|
+
// Can't decode without config yet — drop until SPS/PPS arrive.
|
|
237
|
+
this.au = [];
|
|
238
|
+
this.auIsKey = false;
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
const data = this.au.length === 1 ? this.au[0] : Buffer.concat(this.au);
|
|
242
|
+
this.cbs.onFrame(data, this.auIsKey);
|
|
243
|
+
this.au = [];
|
|
244
|
+
this.auIsKey = false;
|
|
245
|
+
}
|
|
246
|
+
emitConfigIfReady() {
|
|
247
|
+
if (this.config || !this.sps || !this.pps)
|
|
248
|
+
return;
|
|
249
|
+
const info = parseSps(this.sps.subarray(1));
|
|
250
|
+
this.config = {
|
|
251
|
+
codec: 'h264',
|
|
252
|
+
width: info.width,
|
|
253
|
+
height: info.height,
|
|
254
|
+
rotation: 0,
|
|
255
|
+
fps: this.fps,
|
|
256
|
+
codecString: buildCodecString(info),
|
|
257
|
+
sps: this.sps,
|
|
258
|
+
pps: this.pps,
|
|
259
|
+
avcC: buildAvcC(this.sps, this.pps),
|
|
260
|
+
};
|
|
261
|
+
this.cbs.onConfig(this.config);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
exports.H264AnnexBParser = H264AnnexBParser;
|
package/dist/daemon/server.js
CHANGED
|
@@ -28,6 +28,8 @@ const input_server_js_1 = require("./input-server.js");
|
|
|
28
28
|
const input_router_js_1 = require("./input-router.js");
|
|
29
29
|
const input_backends_js_1 = require("./input-backends.js");
|
|
30
30
|
const ios_hid_js_1 = require("../drivers/ios-hid.js");
|
|
31
|
+
const video_server_js_1 = require("./video-server.js");
|
|
32
|
+
const video_source_js_1 = require("./video-source.js");
|
|
31
33
|
const log_collector_js_1 = require("./log-collector.js");
|
|
32
34
|
const session_js_1 = require("../session.js");
|
|
33
35
|
const sessionName = process.argv[2] ?? 'default';
|
|
@@ -86,6 +88,8 @@ let logCollector = null;
|
|
|
86
88
|
let inputServer = null;
|
|
87
89
|
let inputPort = null;
|
|
88
90
|
let hidClient = null;
|
|
91
|
+
let videoServer = null;
|
|
92
|
+
let streamPort = null;
|
|
89
93
|
/**
|
|
90
94
|
* Start the streaming-input WebSocket server for the current device, once its
|
|
91
95
|
* driver is up. iOS/tvOS/Android only — web keeps its per-event REST path.
|
|
@@ -139,6 +143,33 @@ async function startInputServerForPlatform() {
|
|
|
139
143
|
inputPort = port;
|
|
140
144
|
dlog(`input server listening on ${port}`);
|
|
141
145
|
}
|
|
146
|
+
/**
|
|
147
|
+
* Start the streaming-video WebSocket server for the current device, once its
|
|
148
|
+
* driver is up. iOS/tvOS only for now (host-side SimulatorKit capture); the
|
|
149
|
+
* binary must be built/downloaded. Capture is lazy: it doesn't spawn until the
|
|
150
|
+
* first subscriber connects, and stops when the last one leaves.
|
|
151
|
+
*/
|
|
152
|
+
async function startVideoServerForPlatform() {
|
|
153
|
+
if (videoServer)
|
|
154
|
+
return;
|
|
155
|
+
if (driverPlatform !== 'ios' && driverPlatform !== 'tvos')
|
|
156
|
+
return;
|
|
157
|
+
const binary = await (0, bootstrap_js_1.getCaptureBinaryPath)();
|
|
158
|
+
if (!binary) {
|
|
159
|
+
dlog('video capture binary not present — stream server disabled for this device');
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
const port = await (0, bootstrap_js_1.getStreamPort)(sessionName);
|
|
163
|
+
videoServer = await (0, video_server_js_1.startVideoServer)({
|
|
164
|
+
port,
|
|
165
|
+
device: sessionName,
|
|
166
|
+
platform: driverPlatform,
|
|
167
|
+
makeSource: (hub) => new video_source_js_1.IOSCaptureSource(binary, sessionName, hub, dlog),
|
|
168
|
+
dlog,
|
|
169
|
+
});
|
|
170
|
+
streamPort = port;
|
|
171
|
+
dlog(`video server listening on ${port}`);
|
|
172
|
+
}
|
|
142
173
|
const DRIVER_HEALTH_INTERVAL_MS = 10000; // Check driver health every 10s
|
|
143
174
|
let _restartInProgress = false;
|
|
144
175
|
let _driverStarted = false;
|
|
@@ -239,6 +270,15 @@ async function main() {
|
|
|
239
270
|
}
|
|
240
271
|
inputServer = null;
|
|
241
272
|
}
|
|
273
|
+
if (videoServer) {
|
|
274
|
+
try {
|
|
275
|
+
await videoServer.close();
|
|
276
|
+
}
|
|
277
|
+
catch {
|
|
278
|
+
/* ok */
|
|
279
|
+
}
|
|
280
|
+
videoServer = null;
|
|
281
|
+
}
|
|
242
282
|
if (hidClient) {
|
|
243
283
|
hidClient.stop();
|
|
244
284
|
hidClient = null;
|
|
@@ -366,6 +406,7 @@ async function main() {
|
|
|
366
406
|
platform: driverPlatform,
|
|
367
407
|
driverPort,
|
|
368
408
|
inputPort,
|
|
409
|
+
streamPort,
|
|
369
410
|
cdpUrl: cdpUrl ?? null,
|
|
370
411
|
cdpTargetId: cdpTargetId ?? null,
|
|
371
412
|
chromiumCdpPort: driverPlatform === 'web' ? (0, web_server_js_1.getCdpPort)() : null,
|
|
@@ -419,6 +460,12 @@ async function main() {
|
|
|
419
460
|
catch (err) {
|
|
420
461
|
dlog(`Input server startup error: ${err instanceof Error ? err.message : String(err)}`);
|
|
421
462
|
}
|
|
463
|
+
try {
|
|
464
|
+
await startVideoServerForPlatform();
|
|
465
|
+
}
|
|
466
|
+
catch (err) {
|
|
467
|
+
dlog(`Video server startup error: ${err instanceof Error ? err.message : String(err)}`);
|
|
468
|
+
}
|
|
422
469
|
}
|
|
423
470
|
// Start collecting logs once the driver is (or was already) running.
|
|
424
471
|
if (_driverStarted) {
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Fan-out hub for a single device's H.264 stream: one capture, N subscribers.
|
|
4
|
+
*
|
|
5
|
+
* Mirrors Argus's videoForward.ts. The last-seen config and the last keyframe
|
|
6
|
+
* access unit are cached so a late subscriber can configure its decoder and
|
|
7
|
+
* start rendering immediately, without waiting for the next natural keyframe.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.VideoHub = void 0;
|
|
11
|
+
class VideoHub {
|
|
12
|
+
constructor() {
|
|
13
|
+
this.listeners = new Set();
|
|
14
|
+
this.lastConfig = null;
|
|
15
|
+
this.lastKeyframe = null;
|
|
16
|
+
}
|
|
17
|
+
get subscriberCount() {
|
|
18
|
+
return this.listeners.size;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Subscribe. If a config (and keyframe) are already cached they are delivered
|
|
22
|
+
* synchronously so the decoder starts without waiting. Returns an unsubscribe.
|
|
23
|
+
*/
|
|
24
|
+
subscribe(listener) {
|
|
25
|
+
this.listeners.add(listener);
|
|
26
|
+
if (this.lastConfig) {
|
|
27
|
+
listener.onConfig(this.lastConfig);
|
|
28
|
+
if (this.lastKeyframe)
|
|
29
|
+
listener.onFrame(this.lastKeyframe, true);
|
|
30
|
+
}
|
|
31
|
+
return () => this.listeners.delete(listener);
|
|
32
|
+
}
|
|
33
|
+
emitConfig(config) {
|
|
34
|
+
this.lastConfig = config;
|
|
35
|
+
// A config change invalidates the cached keyframe (dimensions may differ).
|
|
36
|
+
this.lastKeyframe = null;
|
|
37
|
+
for (const l of this.listeners)
|
|
38
|
+
l.onConfig(config);
|
|
39
|
+
}
|
|
40
|
+
emitFrame(annexB, keyFrame) {
|
|
41
|
+
if (keyFrame)
|
|
42
|
+
this.lastKeyframe = annexB;
|
|
43
|
+
for (const l of this.listeners)
|
|
44
|
+
l.onFrame(annexB, keyFrame);
|
|
45
|
+
}
|
|
46
|
+
/** Current coded dimensions, if known. */
|
|
47
|
+
dims() {
|
|
48
|
+
return this.lastConfig
|
|
49
|
+
? { width: this.lastConfig.width, height: this.lastConfig.height }
|
|
50
|
+
: null;
|
|
51
|
+
}
|
|
52
|
+
/** Drop cached state when capture ends. */
|
|
53
|
+
clear() {
|
|
54
|
+
this.lastConfig = null;
|
|
55
|
+
this.lastKeyframe = null;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
exports.VideoHub = VideoHub;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Streaming device-video protocol (conductor ⇄ subscribers).
|
|
4
|
+
*
|
|
5
|
+
* The capture counterpart to input-protocol.ts. A persistent per-device
|
|
6
|
+
* WebSocket carries a low-latency H.264 stream: on connect the server sends one
|
|
7
|
+
* JSON `config` frame (codec + dimensions + SPS/PPS), then a sequence of
|
|
8
|
+
* **binary** frames, each a single H.264 Annex B access unit (keyframe-led). One
|
|
9
|
+
* capture feeds N subscribers (fan-out); a late joiner always gets the cached
|
|
10
|
+
* config + a fresh keyframe immediately.
|
|
11
|
+
*
|
|
12
|
+
* Web devices advertise a different codec (JPEG over CDP screencast) in the same
|
|
13
|
+
* `config` frame so clients pick a decoder from the wire, not the URL.
|
|
14
|
+
*
|
|
15
|
+
* See docs/device-video-stream.md for the design.
|
|
16
|
+
*/
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.VIDEO_PROTOCOL_VERSION = void 0;
|
|
19
|
+
exports.toConfigFrame = toConfigFrame;
|
|
20
|
+
exports.VIDEO_PROTOCOL_VERSION = 1;
|
|
21
|
+
/** Serialize an in-daemon config into the JSON frame sent to a subscriber. */
|
|
22
|
+
function toConfigFrame(cfg, device, platform) {
|
|
23
|
+
return {
|
|
24
|
+
t: 'config',
|
|
25
|
+
protocol: exports.VIDEO_PROTOCOL_VERSION,
|
|
26
|
+
device,
|
|
27
|
+
platform,
|
|
28
|
+
codec: cfg.codec,
|
|
29
|
+
width: cfg.width,
|
|
30
|
+
height: cfg.height,
|
|
31
|
+
rotation: cfg.rotation,
|
|
32
|
+
fps: cfg.fps,
|
|
33
|
+
codecString: cfg.codecString,
|
|
34
|
+
sps: cfg.sps ? cfg.sps.toString('base64') : undefined,
|
|
35
|
+
pps: cfg.pps ? cfg.pps.toString('base64') : undefined,
|
|
36
|
+
avcC: cfg.avcC ? cfg.avcC.toString('base64') : undefined,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.startVideoServer = startVideoServer;
|
|
4
|
+
/**
|
|
5
|
+
* Streaming video WebSocket server (one per device daemon).
|
|
6
|
+
*
|
|
7
|
+
* The capture counterpart to input-server.ts. Bound to loopback; N clients open
|
|
8
|
+
* a socket and receive one JSON `config` frame followed by binary H.264 Annex B
|
|
9
|
+
* access units. All subscribers share one capture via a VideoHub — the first
|
|
10
|
+
* connection starts the capture backend, the last one out stops it. A late
|
|
11
|
+
* joiner gets the cached config + keyframe immediately (hub responsibility).
|
|
12
|
+
*/
|
|
13
|
+
const ws_1 = require("ws");
|
|
14
|
+
const video_protocol_js_1 = require("./video-protocol.js");
|
|
15
|
+
const video_hub_js_1 = require("./video-hub.js");
|
|
16
|
+
function sendJson(ws, frame) {
|
|
17
|
+
if (ws.readyState === ws_1.WebSocket.OPEN)
|
|
18
|
+
ws.send(JSON.stringify(frame));
|
|
19
|
+
}
|
|
20
|
+
function startVideoServer(opts) {
|
|
21
|
+
const host = opts.host ?? '127.0.0.1';
|
|
22
|
+
const dlog = opts.dlog ?? (() => { });
|
|
23
|
+
const hub = new video_hub_js_1.VideoHub();
|
|
24
|
+
const source = opts.makeSource(hub);
|
|
25
|
+
// Serialize start/stop so a rapid connect/disconnect churn can't race the
|
|
26
|
+
// capture backend into an inconsistent state.
|
|
27
|
+
let lifecycle = Promise.resolve();
|
|
28
|
+
const runExclusive = (fn) => {
|
|
29
|
+
lifecycle = lifecycle.then(fn, fn);
|
|
30
|
+
return lifecycle;
|
|
31
|
+
};
|
|
32
|
+
return new Promise((resolve, reject) => {
|
|
33
|
+
const wss = new ws_1.WebSocketServer({ host, port: opts.port });
|
|
34
|
+
wss.on('error', reject);
|
|
35
|
+
wss.on('listening', () => {
|
|
36
|
+
const addr = wss.address();
|
|
37
|
+
const boundPort = typeof addr === 'object' && addr ? addr.port : opts.port;
|
|
38
|
+
dlog(`video socket ready at ws://${host}:${boundPort}/stream (${opts.platform})`);
|
|
39
|
+
resolve({
|
|
40
|
+
port: boundPort,
|
|
41
|
+
hub,
|
|
42
|
+
close: () => runExclusive(async () => {
|
|
43
|
+
for (const client of wss.clients)
|
|
44
|
+
client.close();
|
|
45
|
+
await source.stop().catch(() => { });
|
|
46
|
+
await new Promise((res) => wss.close(() => res()));
|
|
47
|
+
}),
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
wss.on('connection', (ws) => {
|
|
51
|
+
const listener = {
|
|
52
|
+
onConfig: (config) => sendJson(ws, (0, video_protocol_js_1.toConfigFrame)(config, opts.device, opts.platform)),
|
|
53
|
+
onFrame: (annexB, _keyFrame) => {
|
|
54
|
+
if (ws.readyState === ws_1.WebSocket.OPEN)
|
|
55
|
+
ws.send(annexB);
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
const unsubscribe = hub.subscribe(listener);
|
|
59
|
+
const wasFirst = hub.subscriberCount === 1;
|
|
60
|
+
if (wasFirst) {
|
|
61
|
+
void runExclusive(async () => {
|
|
62
|
+
try {
|
|
63
|
+
await source.start();
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
dlog(`capture start error: ${err instanceof Error ? err.message : String(err)}`);
|
|
67
|
+
sendJson(ws, {
|
|
68
|
+
t: 'notice',
|
|
69
|
+
code: 'capture_failed',
|
|
70
|
+
msg: 'capture backend failed to start',
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
const teardown = () => {
|
|
76
|
+
unsubscribe();
|
|
77
|
+
if (hub.subscriberCount === 0) {
|
|
78
|
+
void runExclusive(() => source.stop().catch(() => { }));
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
ws.on('close', teardown);
|
|
82
|
+
ws.on('error', (err) => {
|
|
83
|
+
dlog(`video socket error: ${err instanceof Error ? err.message : String(err)}`);
|
|
84
|
+
});
|
|
85
|
+
// Clients are pure subscribers; inbound messages are ignored.
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.IOSCaptureSource = void 0;
|
|
7
|
+
/**
|
|
8
|
+
* Capture sources — the producer half of the video stream. A source drives one
|
|
9
|
+
* platform's capture and feeds decoded config/access-units into a VideoHub. The
|
|
10
|
+
* video server ref-counts subscribers and calls start()/stop() so exactly one
|
|
11
|
+
* capture runs while ≥1 subscriber is connected (last one out stops it).
|
|
12
|
+
*
|
|
13
|
+
* iOS/tvOS: spawns the host-side `conductor-capture` binary (packages/ios-capture),
|
|
14
|
+
* which captures the Simulator framebuffer via SimulatorKit, VideoToolbox-encodes
|
|
15
|
+
* H.264, and serves a raw Annex B TCP stream. We connect to that port, parse the
|
|
16
|
+
* elementary stream, and forward.
|
|
17
|
+
*/
|
|
18
|
+
const child_process_1 = require("child_process");
|
|
19
|
+
const net_1 = __importDefault(require("net"));
|
|
20
|
+
const readline_1 = __importDefault(require("readline"));
|
|
21
|
+
const h264_annexb_js_1 = require("./h264-annexb.js");
|
|
22
|
+
/** Restart the capture backend at most this often if the TCP stream drops. */
|
|
23
|
+
const RESTART_BACKOFF_MS = 1000;
|
|
24
|
+
class IOSCaptureSource {
|
|
25
|
+
constructor(binaryPath, udid, hub, dlog = () => { }) {
|
|
26
|
+
this.binaryPath = binaryPath;
|
|
27
|
+
this.udid = udid;
|
|
28
|
+
this.hub = hub;
|
|
29
|
+
this.dlog = dlog;
|
|
30
|
+
this.proc = null;
|
|
31
|
+
this.rl = null;
|
|
32
|
+
this.socket = null;
|
|
33
|
+
this.parser = null;
|
|
34
|
+
this.pending = [];
|
|
35
|
+
this.active = false;
|
|
36
|
+
this.restarting = false;
|
|
37
|
+
}
|
|
38
|
+
async start() {
|
|
39
|
+
if (this.active)
|
|
40
|
+
return;
|
|
41
|
+
this.active = true;
|
|
42
|
+
await this.spawnAndCapture();
|
|
43
|
+
}
|
|
44
|
+
async spawnAndCapture() {
|
|
45
|
+
this.proc = (0, child_process_1.spawn)(this.binaryPath, [], { stdio: ['pipe', 'pipe', 'inherit'] });
|
|
46
|
+
this.rl = readline_1.default.createInterface({ input: this.proc.stdout });
|
|
47
|
+
this.rl.on('line', (line) => {
|
|
48
|
+
const resolve = this.pending.shift();
|
|
49
|
+
if (!resolve)
|
|
50
|
+
return;
|
|
51
|
+
try {
|
|
52
|
+
resolve(JSON.parse(line));
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
resolve({ ok: false, error: `bad response: ${line}` });
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
this.proc.on('exit', (code) => {
|
|
59
|
+
this.dlog(`capture backend exited (code=${code ?? '?'})`);
|
|
60
|
+
this.proc = null;
|
|
61
|
+
this.rl = null;
|
|
62
|
+
while (this.pending.length)
|
|
63
|
+
this.pending.shift()({ ok: false, error: 'capture exited' });
|
|
64
|
+
this.maybeRestart();
|
|
65
|
+
});
|
|
66
|
+
const res = await this.send({ cmd: 'start_capture', udid: this.udid });
|
|
67
|
+
if (!res.ok || typeof res.port !== 'number') {
|
|
68
|
+
throw new Error(`start_capture failed: ${res.error ?? 'no port returned'}`);
|
|
69
|
+
}
|
|
70
|
+
this.connectStream(res.port);
|
|
71
|
+
this.dlog(`capture started, streaming H.264 from 127.0.0.1:${res.port}`);
|
|
72
|
+
}
|
|
73
|
+
connectStream(port) {
|
|
74
|
+
const parser = new h264_annexb_js_1.H264AnnexBParser({
|
|
75
|
+
onConfig: (config) => {
|
|
76
|
+
this.dlog(`H.264 config ${config.codecString} ${config.width}x${config.height}`);
|
|
77
|
+
this.hub.emitConfig(config);
|
|
78
|
+
},
|
|
79
|
+
onFrame: (annexB, keyFrame) => this.hub.emitFrame(annexB, keyFrame),
|
|
80
|
+
});
|
|
81
|
+
this.parser = parser;
|
|
82
|
+
const socket = net_1.default.connect(port, '127.0.0.1', () => {
|
|
83
|
+
this.dlog(`connected to capture stream on ${port}`);
|
|
84
|
+
});
|
|
85
|
+
socket.on('data', (chunk) => parser.push(chunk));
|
|
86
|
+
socket.on('error', (err) => this.dlog(`capture stream error: ${err.message}`));
|
|
87
|
+
socket.on('close', () => {
|
|
88
|
+
this.dlog('capture stream closed');
|
|
89
|
+
this.maybeRestart();
|
|
90
|
+
});
|
|
91
|
+
this.socket = socket;
|
|
92
|
+
}
|
|
93
|
+
/** If capture died while subscribers still want it, relaunch after a short backoff. */
|
|
94
|
+
maybeRestart() {
|
|
95
|
+
if (!this.active || this.restarting)
|
|
96
|
+
return;
|
|
97
|
+
this.restarting = true;
|
|
98
|
+
this.teardownTransport();
|
|
99
|
+
setTimeout(() => {
|
|
100
|
+
this.restarting = false;
|
|
101
|
+
if (!this.active)
|
|
102
|
+
return;
|
|
103
|
+
this.dlog('restarting capture backend');
|
|
104
|
+
this.spawnAndCapture().catch((err) => this.dlog(`capture restart failed: ${err.message}`));
|
|
105
|
+
}, RESTART_BACKOFF_MS).unref?.();
|
|
106
|
+
}
|
|
107
|
+
teardownTransport() {
|
|
108
|
+
this.socket?.destroy();
|
|
109
|
+
this.socket = null;
|
|
110
|
+
this.parser?.reset();
|
|
111
|
+
this.parser = null;
|
|
112
|
+
}
|
|
113
|
+
async stop() {
|
|
114
|
+
if (!this.active)
|
|
115
|
+
return;
|
|
116
|
+
this.active = false;
|
|
117
|
+
try {
|
|
118
|
+
if (this.proc)
|
|
119
|
+
await this.send({ cmd: 'stop_capture' }).catch(() => { });
|
|
120
|
+
}
|
|
121
|
+
finally {
|
|
122
|
+
this.teardownTransport();
|
|
123
|
+
this.rl?.close();
|
|
124
|
+
this.rl = null;
|
|
125
|
+
this.proc?.kill();
|
|
126
|
+
this.proc = null;
|
|
127
|
+
this.hub.clear();
|
|
128
|
+
this.dlog('capture stopped');
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
send(req) {
|
|
132
|
+
return new Promise((resolve) => {
|
|
133
|
+
if (!this.proc || !this.proc.stdin?.writable) {
|
|
134
|
+
resolve({ ok: false, error: 'capture process not running' });
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
this.pending.push(resolve);
|
|
138
|
+
this.proc.stdin.write(JSON.stringify(req) + '\n');
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
exports.IOSCaptureSource = IOSCaptureSource;
|
package/dist/drivers/android.js
CHANGED
|
@@ -75,6 +75,10 @@ class AndroidDriver {
|
|
|
75
75
|
this._recordingProcess = null;
|
|
76
76
|
this._recordingOutputPath = '';
|
|
77
77
|
}
|
|
78
|
+
/** adb serial of the target device (used by out-of-process helpers like record-video). */
|
|
79
|
+
get serial() {
|
|
80
|
+
return this.deviceId;
|
|
81
|
+
}
|
|
78
82
|
async connect() {
|
|
79
83
|
const packageDef = loadPackageDef();
|
|
80
84
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|