@appium/coresim 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +14 -0
- package/binding.gyp +7 -0
- package/lib/src/commands/video-recording.d.ts +14 -7
- package/lib/src/commands/video-recording.d.ts.map +1 -1
- package/lib/src/commands/video-recording.js +36 -11
- package/lib/src/commands/video-recording.js.map +1 -1
- package/lib/src/commands/video-stream.d.ts +55 -7
- package/lib/src/commands/video-stream.d.ts.map +1 -1
- package/lib/src/commands/video-stream.js +37 -35
- package/lib/src/commands/video-stream.js.map +1 -1
- package/lib/src/native-simctl.js +12 -1
- package/lib/src/native-simctl.js.map +1 -1
- package/lib/src/types.d.ts +72 -10
- package/lib/src/types.d.ts.map +1 -1
- package/lib/src/utils/index.d.ts +1 -1
- package/lib/src/utils/index.d.ts.map +1 -1
- package/lib/src/utils/index.js +1 -1
- package/lib/src/utils/index.js.map +1 -1
- package/lib/src/utils/run-catching.d.ts +4 -0
- package/lib/src/utils/run-catching.d.ts.map +1 -1
- package/lib/src/utils/run-catching.js +11 -0
- package/lib/src/utils/run-catching.js.map +1 -1
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/@appium+coresim.node +0 -0
- package/src/commands/video-recording.ts +49 -19
- package/src/commands/video-stream.ts +48 -39
- package/src/coresim.mm +547 -118
- package/src/native/audio_encoder.h +65 -0
- package/src/native/audio_encoder.mm +270 -0
- package/src/native/av_recording.h +52 -0
- package/src/native/av_recording.mm +424 -0
- package/src/native/av_stream.h +70 -0
- package/src/native/av_stream.mm +217 -0
- package/src/native/monotonic_clock.h +20 -0
- package/src/native/sim_audio_tap.h +67 -0
- package/src/native/sim_audio_tap.mm +350 -0
- package/src/native/sim_process.h +10 -0
- package/src/native/sim_process.mm +81 -18
- package/src/native/sim_video_stream.h +14 -11
- package/src/native/sim_video_stream.mm +27 -383
- package/src/native/video_encoder.h +76 -0
- package/src/native/video_encoder.mm +417 -0
- package/src/native-simctl.ts +12 -1
- package/src/types.ts +84 -11
- package/src/utils/index.ts +1 -1
- package/src/utils/run-catching.ts +11 -0
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
#include <sys/un.h>
|
|
7
7
|
|
|
8
8
|
#include <cstring>
|
|
9
|
+
#include <string>
|
|
9
10
|
#include <vector>
|
|
10
11
|
|
|
11
12
|
namespace coresim {
|
|
@@ -25,48 +26,96 @@ NSString* ExecutablePath(pid_t pid) {
|
|
|
25
26
|
return size > 0 ? [NSString stringWithUTF8String:pathBuf] : nil;
|
|
26
27
|
}
|
|
27
28
|
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
|
|
29
|
+
// Layout: argc, exec path, NUL padding, then argv[0..argc-1], then envp[...], each NUL-terminated
|
|
30
|
+
// (envp has no guaranteed empty-string sentinel within the buffer — just runs to `size`).
|
|
31
|
+
std::vector<char> ProcArgs2(pid_t pid) {
|
|
31
32
|
int mib[3] = {CTL_KERN, KERN_PROCARGS2, pid};
|
|
32
33
|
size_t size = 0;
|
|
33
34
|
if (sysctl(mib, 3, nullptr, &size, nullptr, 0) != 0 || size < sizeof(int)) {
|
|
34
|
-
return
|
|
35
|
+
return {};
|
|
35
36
|
}
|
|
36
37
|
std::vector<char> buffer(size);
|
|
37
38
|
if (sysctl(mib, 3, buffer.data(), &size, nullptr, 0) != 0) {
|
|
39
|
+
return {};
|
|
40
|
+
}
|
|
41
|
+
return buffer;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const char* SkipExecPathAndPadding(const char* cursor, const char* end) {
|
|
45
|
+
cursor += strnlen(cursor, static_cast<size_t>(end - cursor)); // skip the exec path
|
|
46
|
+
while (cursor < end && *cursor == '\0') cursor++; // skip the NUL padding
|
|
47
|
+
return cursor;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// Advances past `count` further NUL-terminated entries (e.g. argv), landing just after the last
|
|
51
|
+
// one's terminator — or at `end` if there weren't that many.
|
|
52
|
+
const char* SkipEntries(const char* cursor, const char* end, int count) {
|
|
53
|
+
for (int i = 0; i < count && cursor < end; i++) {
|
|
54
|
+
cursor += strnlen(cursor, static_cast<size_t>(end - cursor));
|
|
55
|
+
if (cursor < end) cursor++;
|
|
56
|
+
}
|
|
57
|
+
return cursor;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// argv[1] via the unprivileged KERN_PROCARGS2 sysctl (same as `ps`/`lsof`) — for `launchd_sim`
|
|
61
|
+
// that's its bootstrap plist path, which embeds the device's UDID.
|
|
62
|
+
NSString* FirstArgument(pid_t pid) {
|
|
63
|
+
std::vector<char> buffer = ProcArgs2(pid);
|
|
64
|
+
if (buffer.empty()) {
|
|
38
65
|
return nil;
|
|
39
66
|
}
|
|
40
|
-
// Layout: argc, exec path, NUL padding, then argv[0], argv[1], ..., each NUL-terminated.
|
|
41
67
|
int argc = 0;
|
|
42
68
|
std::memcpy(&argc, buffer.data(), sizeof(argc));
|
|
43
|
-
const char*
|
|
44
|
-
const char*
|
|
45
|
-
cursor += strnlen(cursor, static_cast<size_t>(end - cursor)); // skip the exec path
|
|
46
|
-
while (cursor < end && *cursor == '\0') cursor++; // skip the NUL padding
|
|
47
|
-
cursor += strnlen(cursor, static_cast<size_t>(end - cursor)); // skip argv[0]
|
|
48
|
-
if (cursor < end) cursor++; // skip its NUL terminator
|
|
69
|
+
const char* end = buffer.data() + buffer.size();
|
|
70
|
+
const char* cursor = SkipEntries(SkipExecPathAndPadding(buffer.data() + sizeof(argc), end), end, 1);
|
|
49
71
|
if (argc < 2 || cursor >= end) {
|
|
50
72
|
return nil;
|
|
51
73
|
}
|
|
52
74
|
return [NSString stringWithUTF8String:cursor];
|
|
53
75
|
}
|
|
54
76
|
|
|
55
|
-
//
|
|
56
|
-
|
|
77
|
+
// The value of `key=...` in `pid`'s environment, past the end of its argv — same sysctl as
|
|
78
|
+
// FirstArgument, walked further.
|
|
79
|
+
NSString* FindEnvValue(pid_t pid, NSString* key) {
|
|
80
|
+
std::vector<char> buffer = ProcArgs2(pid);
|
|
81
|
+
if (buffer.empty()) {
|
|
82
|
+
return nil;
|
|
83
|
+
}
|
|
84
|
+
int argc = 0;
|
|
85
|
+
std::memcpy(&argc, buffer.data(), sizeof(argc));
|
|
86
|
+
const char* end = buffer.data() + buffer.size();
|
|
87
|
+
const char* cursor = SkipEntries(SkipExecPathAndPadding(buffer.data() + sizeof(argc), end), end, argc);
|
|
88
|
+
std::string prefix = std::string(key.UTF8String) + "=";
|
|
89
|
+
while (cursor < end && *cursor != '\0') {
|
|
90
|
+
size_t entryLen = strnlen(cursor, static_cast<size_t>(end - cursor));
|
|
91
|
+
if (entryLen > prefix.size() && std::memcmp(cursor, prefix.data(), prefix.size()) == 0) {
|
|
92
|
+
return [NSString stringWithUTF8String:cursor + prefix.size()];
|
|
93
|
+
}
|
|
94
|
+
cursor += entryLen;
|
|
95
|
+
if (cursor < end) cursor++;
|
|
96
|
+
}
|
|
97
|
+
return nil;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Every live pid on the system, via the unprivileged proc_listpids sysctl.
|
|
101
|
+
std::vector<pid_t> AllPids() {
|
|
57
102
|
int neededBytes = proc_listpids(PROC_ALL_PIDS, 0, nullptr, 0);
|
|
58
103
|
if (neededBytes <= 0) {
|
|
59
|
-
return
|
|
104
|
+
return {};
|
|
60
105
|
}
|
|
61
106
|
// Headroom for processes started between the sizing call above and the listing call below.
|
|
62
107
|
std::vector<pid_t> pids(neededBytes / sizeof(pid_t) + 64);
|
|
63
108
|
int bytes = proc_listpids(PROC_ALL_PIDS, 0, pids.data(), static_cast<int>(pids.size() * sizeof(pid_t)));
|
|
64
109
|
if (bytes <= 0) {
|
|
65
|
-
return
|
|
110
|
+
return {};
|
|
66
111
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
112
|
+
pids.resize(static_cast<size_t>(bytes) / sizeof(pid_t));
|
|
113
|
+
return pids;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Each booted simulator has its own `launchd_sim`; finds the one owning `udid`.
|
|
117
|
+
pid_t FindLaunchdSimPid(NSString* udid) {
|
|
118
|
+
for (pid_t pid : AllPids()) {
|
|
70
119
|
if (pid <= 0) {
|
|
71
120
|
continue;
|
|
72
121
|
}
|
|
@@ -136,4 +185,18 @@ NSString* FindWebInspectorSocket(NSString* udid, NSError** error) {
|
|
|
136
185
|
return socketPath;
|
|
137
186
|
}
|
|
138
187
|
|
|
188
|
+
std::vector<pid_t> FindGuestProcessPids(NSString* udid) {
|
|
189
|
+
std::vector<pid_t> result;
|
|
190
|
+
for (pid_t pid : AllPids()) {
|
|
191
|
+
if (pid <= 0) {
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
NSString* value = FindEnvValue(pid, @"SIMULATOR_UDID");
|
|
195
|
+
if (value != nil && [value caseInsensitiveCompare:udid] == NSOrderedSame) {
|
|
196
|
+
result.push_back(pid);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return result;
|
|
200
|
+
}
|
|
201
|
+
|
|
139
202
|
} // namespace coresim
|
|
@@ -7,9 +7,9 @@
|
|
|
7
7
|
#include <memory>
|
|
8
8
|
#include <vector>
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
#include "video_encoder.h"
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
namespace coresim {
|
|
13
13
|
|
|
14
14
|
// One encoded frame — Annex-B NAL units, concatenated. A keyframe's `data` has parameter sets
|
|
15
15
|
// (SPS/PPS, or VPS/SPS/PPS for HEVC) prepended, so every keyframe is self-decodable alone.
|
|
@@ -21,20 +21,16 @@ struct VideoAccessUnit {
|
|
|
21
21
|
int64_t timestampMicros = 0;
|
|
22
22
|
};
|
|
23
23
|
|
|
24
|
-
struct VideoStreamOptions {
|
|
25
|
-
VideoStreamCodec codec = VideoStreamCodec::kH264;
|
|
26
|
-
NSString* displayId = nil;
|
|
27
|
-
double fps = 15.0;
|
|
28
|
-
int bitrate = 2000000;
|
|
29
|
-
};
|
|
30
|
-
|
|
31
24
|
// Polls the live display IOSurface (sim_screenshot.h) on a serial queue and encodes changed
|
|
32
25
|
// frames via the public VideoToolbox API — unlike StartVideoRecording's private, file-only
|
|
33
26
|
// recorder, this delivers access units live. Full thread-safety contract: see CLAUDE.md.
|
|
34
27
|
class VideoStreamSession {
|
|
35
28
|
public:
|
|
36
|
-
|
|
37
|
-
|
|
29
|
+
// `onAbortDelivery`, if set, is invoked by AbortDelivery() below — not called by this class on
|
|
30
|
+
// its own.
|
|
31
|
+
VideoStreamSession(id device, VideoEncoderOptions options, std::function<void(VideoAccessUnit)> onAccessUnit,
|
|
32
|
+
std::function<void(NSError*)> onError, std::function<void()> onEnd,
|
|
33
|
+
std::function<void()> onAbortDelivery = nullptr);
|
|
38
34
|
~VideoStreamSession();
|
|
39
35
|
|
|
40
36
|
VideoStreamSession(const VideoStreamSession&) = delete;
|
|
@@ -48,6 +44,13 @@ class VideoStreamSession {
|
|
|
48
44
|
// onError/onEnd — same queue this blocks on, so it would deadlock.
|
|
49
45
|
void Stop();
|
|
50
46
|
|
|
47
|
+
// Runs the constructor's `onAbortDelivery` (if any) — e.g. aborting a bounded delivery queue so
|
|
48
|
+
// a producer thread blocked pushing into it unblocks. Call before Stop() when the caller can't
|
|
49
|
+
// rely on anything else draining that queue concurrently (e.g. process-exit cleanup running
|
|
50
|
+
// synchronously on the same thread `Stop()` would otherwise wait on) — a normal Stop() doesn't
|
|
51
|
+
// need this. Safe from any thread; a no-op if unset.
|
|
52
|
+
void AbortDelivery();
|
|
53
|
+
|
|
51
54
|
// Forces the next encoded frame to be a keyframe (self-decodable, parameter sets included) —
|
|
52
55
|
// e.g. so a consumer that just resynced after dropping frames can resume cleanly instead of
|
|
53
56
|
// waiting for the next periodic one. Safe from any thread; just sets a flag.
|
|
@@ -1,372 +1,34 @@
|
|
|
1
1
|
#include "sim_video_stream.h"
|
|
2
2
|
|
|
3
|
-
#import <CoreMedia/CoreMedia.h>
|
|
4
|
-
#import <CoreVideo/CoreVideo.h>
|
|
5
|
-
#import <IOSurface/IOSurface.h>
|
|
6
|
-
#import <VideoToolbox/VideoToolbox.h>
|
|
7
|
-
|
|
8
|
-
#include <algorithm>
|
|
9
3
|
#include <atomic>
|
|
10
|
-
#include <ctime>
|
|
11
4
|
|
|
12
|
-
#include "
|
|
13
|
-
#include "safe_dispatch.h"
|
|
14
|
-
#include "sim_screenshot.h"
|
|
5
|
+
#include "video_encoder.h"
|
|
15
6
|
|
|
16
7
|
namespace coresim {
|
|
17
8
|
|
|
18
|
-
namespace {
|
|
19
|
-
|
|
20
|
-
NSString* const kVideoStreamErrorDomain = @"com.appium.coresim.VideoStream";
|
|
21
|
-
|
|
22
|
-
NSError* MakeError(NSInteger code, NSString* message) {
|
|
23
|
-
return [NSError errorWithDomain:kVideoStreamErrorDomain code:code userInfo:@{NSLocalizedDescriptionKey : message}];
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
NSError* MakeStatusError(NSInteger code, NSString* what, OSStatus status) {
|
|
27
|
-
return MakeError(code, [NSString stringWithFormat:@"%@ (OSStatus %d)", what, static_cast<int>(status)]);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
// CFAbsoluteTimeGetCurrent() is wall time — subject to backward NTP/manual clock corrections,
|
|
31
|
-
// which would violate VTCompressionSessionEncodeFrame's requirement of strictly increasing PTS.
|
|
32
|
-
// CLOCK_MONOTONIC_RAW never goes backward and isn't adjusted by NTP.
|
|
33
|
-
double MonotonicSeconds() {
|
|
34
|
-
struct timespec ts;
|
|
35
|
-
clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
|
|
36
|
-
return static_cast<double>(ts.tv_sec) + static_cast<double>(ts.tv_nsec) / 1e9;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
void AppendAnnexB(std::vector<uint8_t>& out, const uint8_t* nal, size_t length) {
|
|
40
|
-
static const uint8_t kStartCode[4] = {0, 0, 0, 1};
|
|
41
|
-
out.insert(out.end(), kStartCode, kStartCode + 4);
|
|
42
|
-
out.insert(out.end(), nal, nal + length);
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
// VideoToolbox's compressed output is AVCC-framed (a 4-byte big-endian length prefix per NAL,
|
|
46
|
-
// no start codes) — rewrites it into Annex-B, matching VideoAccessUnit's documented wire format.
|
|
47
|
-
void AppendSampleBufferNALs(std::vector<uint8_t>& out, CMSampleBufferRef sampleBuffer) {
|
|
48
|
-
CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sampleBuffer);
|
|
49
|
-
if (block == nullptr) {
|
|
50
|
-
return;
|
|
51
|
-
}
|
|
52
|
-
size_t totalLength = CMBlockBufferGetDataLength(block);
|
|
53
|
-
if (totalLength == 0) {
|
|
54
|
-
return;
|
|
55
|
-
}
|
|
56
|
-
// CMBlockBufferGetDataPointer's pointer only covers the contiguous region starting at the given
|
|
57
|
-
// offset, which for a segmented buffer (multiple backing memory blocks — CoreMedia's documented
|
|
58
|
-
// contract, not just a VideoToolbox implementation detail) can be far shorter than totalLength;
|
|
59
|
-
// reading up to totalLength through it would run past that region. CopyDataBytes stitches
|
|
60
|
-
// segments together into a caller-owned, guaranteed-contiguous copy instead.
|
|
61
|
-
std::vector<uint8_t> data(totalLength);
|
|
62
|
-
if (CMBlockBufferCopyDataBytes(block, 0, totalLength, data.data()) != kCMBlockBufferNoErr) {
|
|
63
|
-
return;
|
|
64
|
-
}
|
|
65
|
-
const uint8_t* dataPointer = data.data();
|
|
66
|
-
size_t offset = 0;
|
|
67
|
-
while (offset + 4 <= totalLength) {
|
|
68
|
-
uint32_t nalLength = (static_cast<uint32_t>(dataPointer[offset]) << 24) |
|
|
69
|
-
(static_cast<uint32_t>(dataPointer[offset + 1]) << 16) |
|
|
70
|
-
(static_cast<uint32_t>(dataPointer[offset + 2]) << 8) | dataPointer[offset + 3];
|
|
71
|
-
offset += 4;
|
|
72
|
-
if (nalLength == 0 || offset + nalLength > totalLength) {
|
|
73
|
-
break;
|
|
74
|
-
}
|
|
75
|
-
AppendAnnexB(out, dataPointer + offset, nalLength);
|
|
76
|
-
offset += nalLength;
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
using ParameterSetAtIndexFn = OSStatus (*)(CMFormatDescriptionRef, size_t, const uint8_t**, size_t*, size_t*, int*);
|
|
81
|
-
|
|
82
|
-
void AppendParameterSets(std::vector<uint8_t>& out, CMFormatDescriptionRef format, ParameterSetAtIndexFn getAtIndex) {
|
|
83
|
-
size_t count = 0;
|
|
84
|
-
if (getAtIndex(format, 0, nullptr, nullptr, &count, nullptr) != noErr) {
|
|
85
|
-
return;
|
|
86
|
-
}
|
|
87
|
-
for (size_t i = 0; i < count; i++) {
|
|
88
|
-
const uint8_t* bytes = nullptr;
|
|
89
|
-
size_t size = 0;
|
|
90
|
-
if (getAtIndex(format, i, &bytes, &size, nullptr, nullptr) == noErr) {
|
|
91
|
-
AppendAnnexB(out, bytes, size);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
} // namespace
|
|
97
|
-
|
|
98
9
|
class VideoStreamSession::Impl {
|
|
99
10
|
public:
|
|
100
|
-
Impl(id device,
|
|
101
|
-
std::function<void(NSError*)> onError, std::function<void()> onEnd)
|
|
102
|
-
:
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
onEnd_(std::move(onEnd)) {
|
|
107
|
-
queue_ = dispatch_queue_create("com.appium.coresim.videoStream", DISPATCH_QUEUE_SERIAL);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
~Impl() { Stop(); }
|
|
111
|
-
|
|
112
|
-
void Start() {
|
|
113
|
-
NSError* error = nil;
|
|
114
|
-
id descriptor = ResolveCaptureDisplay(device_, options_.displayId, &error);
|
|
115
|
-
if (descriptor == nil) {
|
|
116
|
-
throw NSErrorException(error);
|
|
117
|
-
}
|
|
118
|
-
id surfaceObj = CurrentDisplaySurface(descriptor);
|
|
119
|
-
if (surfaceObj == nil) {
|
|
120
|
-
throw NSErrorException(MakeError(4, @"The device's display surface is not available yet"));
|
|
121
|
-
}
|
|
122
|
-
IOSurfaceRef surface = (__bridge IOSurfaceRef)surfaceObj;
|
|
123
|
-
// Set up synchronously (not lazily on the first Tick()) so a setup failure rejects Start()
|
|
124
|
-
// directly rather than only reaching onError, which the caller may not be listening for yet.
|
|
125
|
-
NSError* setupError = nil;
|
|
126
|
-
if (!SetUpSession(surface, &setupError)) {
|
|
127
|
-
throw NSErrorException(setupError);
|
|
128
|
-
}
|
|
129
|
-
// Must be set before EncodeSurface below — both it and HandleEncodedSample measure elapsed
|
|
130
|
-
// time from this.
|
|
131
|
-
startTime_ = MonotonicSeconds();
|
|
132
|
-
// Encode immediately rather than waiting for a *changed* seed on the first tick, or the
|
|
133
|
-
// stream would stay silent until the display changes again. running_ is set true before this
|
|
134
|
-
// call (not after), since VTCompressionSessionEncodeFrame's output callback can in principle
|
|
135
|
-
// fire on another thread before this one returns — HandleEncodedSample discards samples while
|
|
136
|
-
// running_ is false, which would otherwise silently drop the stream's very first (keyframe)
|
|
137
|
-
// access unit. A failure resets it and tears session_ down itself here, rather than going
|
|
138
|
-
// through Stop()/onEnd_ (see coresim.mm — onEnd_ firing this early would double-release its
|
|
139
|
-
// ThreadSafeFunctions).
|
|
140
|
-
running_ = true;
|
|
141
|
-
bool encoded = false;
|
|
142
|
-
try {
|
|
143
|
-
encoded = EncodeSurface(surface);
|
|
144
|
-
} catch (...) {
|
|
145
|
-
running_ = false;
|
|
146
|
-
VTCompressionSessionInvalidate(session_);
|
|
147
|
-
CFRelease(session_);
|
|
148
|
-
session_ = nullptr;
|
|
149
|
-
throw;
|
|
150
|
-
}
|
|
151
|
-
// Only commit the seed once a frame was actually submitted — a transient pixel-buffer
|
|
152
|
-
// creation failure (EncodeSurface returning false) otherwise leaves lastSeed_ at its default
|
|
153
|
-
// 0, so the first Tick() sees the real seed as "changed" and retries automatically instead of
|
|
154
|
-
// the stream going silent forever on a display that never changes again.
|
|
155
|
-
if (encoded) {
|
|
156
|
-
lastSeed_ = IOSurfaceGetSeed(surface);
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
double interval = 1.0 / std::max(options_.fps, 1.0);
|
|
160
|
-
dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, queue_);
|
|
161
|
-
dispatch_source_set_timer(timer, dispatch_time(DISPATCH_TIME_NOW, 0),
|
|
162
|
-
static_cast<uint64_t>(interval * NSEC_PER_SEC),
|
|
163
|
-
static_cast<uint64_t>(interval * NSEC_PER_SEC / 10));
|
|
164
|
-
// `this` outlives the timer: Stop()/StopFromQueue() always drain or outrun it before `this`
|
|
165
|
-
// can be destroyed (see their comments below).
|
|
166
|
-
dispatch_source_set_event_handler(timer, ^{
|
|
167
|
-
Tick();
|
|
168
|
-
});
|
|
169
|
-
timer_ = timer;
|
|
170
|
-
dispatch_resume(timer_);
|
|
11
|
+
Impl(id device, VideoEncoderOptions options, std::function<void(VideoAccessUnit)> onAccessUnit,
|
|
12
|
+
std::function<void(NSError*)> onError, std::function<void()> onEnd, std::function<void()> onAbortDelivery)
|
|
13
|
+
: options_(options), onAccessUnit_(std::move(onAccessUnit)), onAbortDelivery_(std::move(onAbortDelivery)) {
|
|
14
|
+
encoder_ = std::make_unique<VideoFrameEncoder>(
|
|
15
|
+
device, options, [this](CMSampleBufferRef sampleBuffer) { HandleEncodedSample(sampleBuffer); },
|
|
16
|
+
std::move(onError), std::move(onEnd));
|
|
171
17
|
}
|
|
172
18
|
|
|
173
|
-
|
|
174
|
-
void Stop() {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
if (timer_ != nullptr) {
|
|
179
|
-
dispatch_source_cancel(timer_);
|
|
180
|
-
// Blocks until any in-flight Tick() finishes — by then running_ is already false, so it
|
|
181
|
-
// won't touch session_ again.
|
|
182
|
-
dispatch_sync(queue_, ^{
|
|
183
|
-
});
|
|
184
|
-
timer_ = nullptr;
|
|
19
|
+
void Start() { encoder_->Start(); }
|
|
20
|
+
void Stop() { encoder_->Stop(); }
|
|
21
|
+
void AbortDelivery() {
|
|
22
|
+
if (onAbortDelivery_) {
|
|
23
|
+
onAbortDelivery_();
|
|
185
24
|
}
|
|
186
|
-
TearDownSessionAndFireEnd();
|
|
187
25
|
}
|
|
188
|
-
|
|
189
|
-
void RequestKeyFrame() { forceKeyFrame_ = true; }
|
|
26
|
+
void RequestKeyFrame() { encoder_->RequestKeyFrame(); }
|
|
190
27
|
|
|
191
28
|
private:
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
if (!running_.exchange(false)) {
|
|
196
|
-
return; // idempotent — e.g. an external Stop() already won this race
|
|
197
|
-
}
|
|
198
|
-
if (timer_ != nullptr) {
|
|
199
|
-
dispatch_source_cancel(timer_);
|
|
200
|
-
timer_ = nullptr;
|
|
201
|
-
}
|
|
202
|
-
TearDownSessionAndFireEnd();
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
void TearDownSessionAndFireEnd() {
|
|
206
|
-
if (session_ != nullptr) {
|
|
207
|
-
// Flushes and blocks until every already-submitted frame's callback has returned — without
|
|
208
|
-
// this, a frame submitted just before Stop() could fire after onEnd_ releases the
|
|
209
|
-
// ThreadSafeFunctions below (a use-after-release; see CLAUDE.md).
|
|
210
|
-
VTCompressionSessionCompleteFrames(session_, kCMTimeInvalid);
|
|
211
|
-
VTCompressionSessionInvalidate(session_);
|
|
212
|
-
CFRelease(session_);
|
|
213
|
-
session_ = nullptr;
|
|
214
|
-
}
|
|
215
|
-
if (onEnd_) {
|
|
216
|
-
onEnd_();
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
void Tick() {
|
|
221
|
-
if (!running_) {
|
|
222
|
-
return;
|
|
223
|
-
}
|
|
224
|
-
if (pendingErrorTeardown_) {
|
|
225
|
-
// HandleEncodedSample (below) can run on VideoToolbox's own callback thread, where it's
|
|
226
|
-
// unsafe to tear down directly — StopFromQueue()/CompleteFrames() are only safe already
|
|
227
|
-
// serialized on `queue_` (here), and CompleteFrames specifically would deadlock waiting on
|
|
228
|
-
// its own still-executing callback. It sets this flag instead; picked up on the very next
|
|
229
|
-
// tick (queue_-serialized, safe) rather than via a raw cross-thread dispatch, since nothing
|
|
230
|
-
// here can outlive `this` the way a block captured on another thread otherwise could.
|
|
231
|
-
StopFromQueue();
|
|
232
|
-
return;
|
|
233
|
-
}
|
|
234
|
-
@autoreleasepool {
|
|
235
|
-
try {
|
|
236
|
-
// Re-resolved every tick (like CaptureScreenshot does), not cached once in Start(), so a
|
|
237
|
-
// deleted device or disconnected display surfaces a real error instead of Tick() quietly
|
|
238
|
-
// doing nothing forever.
|
|
239
|
-
NSError* resolveError = nil;
|
|
240
|
-
id descriptor = ResolveCaptureDisplay(device_, options_.displayId, &resolveError);
|
|
241
|
-
if (descriptor == nil) {
|
|
242
|
-
if (onError_) {
|
|
243
|
-
onError_(resolveError);
|
|
244
|
-
}
|
|
245
|
-
StopFromQueue();
|
|
246
|
-
return;
|
|
247
|
-
}
|
|
248
|
-
id surfaceObj = CurrentDisplaySurface(descriptor);
|
|
249
|
-
if (surfaceObj == nil) {
|
|
250
|
-
return; // transient — the connection may not have a frame ready yet, try again next tick
|
|
251
|
-
}
|
|
252
|
-
IOSurfaceRef surface = (__bridge IOSurfaceRef)surfaceObj;
|
|
253
|
-
uint32_t seed = IOSurfaceGetSeed(surface);
|
|
254
|
-
if (seed == lastSeed_) {
|
|
255
|
-
return; // unchanged since the last tick — mirrors CoreSimulator's own recorder, which
|
|
256
|
-
// only encodes a frame when the display actually changes (see CLAUDE.md)
|
|
257
|
-
}
|
|
258
|
-
// Only commit the new seed once EncodeSurface actually submits it — a transient failure
|
|
259
|
-
// (pixel-buffer creation) must leave lastSeed_ stale so the next tick retries this same
|
|
260
|
-
// frame instead of silently going quiet until the display changes again.
|
|
261
|
-
if (EncodeSurface(surface)) {
|
|
262
|
-
lastSeed_ = seed;
|
|
263
|
-
}
|
|
264
|
-
} catch (const std::exception& e) {
|
|
265
|
-
// Without this, an exception here (e.g. a dropped display-proxy connection) would escape
|
|
266
|
-
// this bare GCD timer handler uncaught and crash the whole process (see CLAUDE.md).
|
|
267
|
-
if (onError_) {
|
|
268
|
-
onError_(MakeError(3, [NSString stringWithFormat:@"Video stream encoding failed: %s", e.what()]));
|
|
269
|
-
}
|
|
270
|
-
StopFromQueue();
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
bool SetUpSession(IOSurfaceRef surface, NSError** error) {
|
|
276
|
-
int32_t width = static_cast<int32_t>(IOSurfaceGetWidth(surface));
|
|
277
|
-
int32_t height = static_cast<int32_t>(IOSurfaceGetHeight(surface));
|
|
278
|
-
CMVideoCodecType codecType =
|
|
279
|
-
options_.codec == VideoStreamCodec::kHEVC ? kCMVideoCodecType_HEVC : kCMVideoCodecType_H264;
|
|
280
|
-
OSStatus status = VTCompressionSessionCreate(kCFAllocatorDefault, width, height, codecType, nullptr, nullptr,
|
|
281
|
-
kCFAllocatorDefault, OutputCallback, this, &session_);
|
|
282
|
-
if (status != noErr) {
|
|
283
|
-
*error = MakeStatusError(1, @"Failed to create a VTCompressionSession", status);
|
|
284
|
-
return false;
|
|
285
|
-
}
|
|
286
|
-
// Clamped like Start()'s timer interval — an unvalidated 0 here would set MaxKeyFrameInterval
|
|
287
|
-
// to an out-of-spec value.
|
|
288
|
-
double fps = std::max(options_.fps, 1.0);
|
|
289
|
-
status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
|
|
290
|
-
if (status == noErr) {
|
|
291
|
-
status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse);
|
|
292
|
-
}
|
|
293
|
-
if (status == noErr) {
|
|
294
|
-
status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_AverageBitRate,
|
|
295
|
-
(__bridge CFNumberRef) @(options_.bitrate));
|
|
296
|
-
}
|
|
297
|
-
if (status == noErr) {
|
|
298
|
-
status =
|
|
299
|
-
VTSessionSetProperty(session_, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFNumberRef) @(fps));
|
|
300
|
-
}
|
|
301
|
-
if (status == noErr) {
|
|
302
|
-
status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_MaxKeyFrameInterval,
|
|
303
|
-
(__bridge CFNumberRef) @(static_cast<int>(fps * 2)));
|
|
304
|
-
}
|
|
305
|
-
if (status != noErr) {
|
|
306
|
-
*error = MakeStatusError(2, @"Failed to configure the VTCompressionSession", status);
|
|
307
|
-
VTCompressionSessionInvalidate(session_);
|
|
308
|
-
CFRelease(session_);
|
|
309
|
-
session_ = nullptr;
|
|
310
|
-
return false;
|
|
311
|
-
}
|
|
312
|
-
VTCompressionSessionPrepareToEncodeFrames(session_);
|
|
313
|
-
return true;
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
// Returns whether a frame was actually submitted to the encoder — false for a transient
|
|
317
|
-
// pixel-buffer creation failure the caller should retry, as opposed to a real encode failure
|
|
318
|
-
// (thrown, not returned, since that tears down the whole stream).
|
|
319
|
-
bool EncodeSurface(IOSurfaceRef surface) {
|
|
320
|
-
CVPixelBufferRef pixelBuffer = nullptr;
|
|
321
|
-
CVReturn cvStatus = CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, surface, nullptr, &pixelBuffer);
|
|
322
|
-
if (cvStatus != kCVReturnSuccess || pixelBuffer == nullptr) {
|
|
323
|
-
return false; // transient — try again next tick rather than tearing down the whole stream
|
|
324
|
-
}
|
|
325
|
-
CMTime pts = CMTimeMake(static_cast<int64_t>((MonotonicSeconds() - startTime_) * 1000000), 1000000);
|
|
326
|
-
NSDictionary* frameProperties = nil;
|
|
327
|
-
if (forceKeyFrame_.exchange(false)) {
|
|
328
|
-
frameProperties = @{(__bridge NSString*)kVTEncodeFrameOptionKey_ForceKeyFrame : @YES};
|
|
329
|
-
}
|
|
330
|
-
OSStatus status = VTCompressionSessionEncodeFrame(session_, pixelBuffer, pts, kCMTimeInvalid,
|
|
331
|
-
(__bridge CFDictionaryRef)frameProperties, nullptr, nullptr);
|
|
332
|
-
CVPixelBufferRelease(pixelBuffer);
|
|
333
|
-
if (status != noErr) {
|
|
334
|
-
throw std::runtime_error([[NSString stringWithFormat:@"VTCompressionSessionEncodeFrame failed (OSStatus %d)",
|
|
335
|
-
static_cast<int>(status)] UTF8String]);
|
|
336
|
-
}
|
|
337
|
-
return true;
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
static void OutputCallback(void* outputCallbackRefCon, void* /*sourceFrameRefCon*/, OSStatus status,
|
|
341
|
-
VTEncodeInfoFlags /*infoFlags*/, CMSampleBufferRef sampleBuffer) {
|
|
342
|
-
static_cast<Impl*>(outputCallbackRefCon)->HandleEncodedSample(status, sampleBuffer);
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
void HandleEncodedSample(OSStatus status, CMSampleBufferRef sampleBuffer) {
|
|
346
|
-
if (!running_) {
|
|
347
|
-
return;
|
|
348
|
-
}
|
|
349
|
-
if (status != noErr) {
|
|
350
|
-
// Only the first failure is reported — pendingErrorTeardown_ doubles as the report-once
|
|
351
|
-
// gate, since Tick() (the only place that consumes it) only ever needs to see it once too.
|
|
352
|
-
if (!pendingErrorTeardown_.exchange(true)) {
|
|
353
|
-
if (onError_) {
|
|
354
|
-
onError_(MakeStatusError(3, @"VideoToolbox reported an encoding failure", status));
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
return;
|
|
358
|
-
}
|
|
359
|
-
if (sampleBuffer == nullptr) {
|
|
360
|
-
return;
|
|
361
|
-
}
|
|
362
|
-
bool isKeyFrame = true;
|
|
363
|
-
CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, false);
|
|
364
|
-
if (attachments != nullptr && CFArrayGetCount(attachments) > 0) {
|
|
365
|
-
CFDictionaryRef attachment = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
|
|
366
|
-
isKeyFrame = !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
// From the sample's own presentation timestamp (as submitted in EncodeSurface), not a fresh
|
|
29
|
+
void HandleEncodedSample(CMSampleBufferRef sampleBuffer) {
|
|
30
|
+
bool isKeyFrame = IsKeyFrame(sampleBuffer);
|
|
31
|
+
// From the sample's own presentation timestamp (as submitted to the encoder), not a fresh
|
|
370
32
|
// wall-clock read here — this callback can fire well after encoding actually happened, and
|
|
371
33
|
// resampling "now" would report a later, jittery timestamp than when the frame was captured.
|
|
372
34
|
CMTime pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
|
|
@@ -374,47 +36,27 @@ class VideoStreamSession::Impl {
|
|
|
374
36
|
unit.isKeyFrame = isKeyFrame;
|
|
375
37
|
unit.sequence = sequence_++;
|
|
376
38
|
unit.timestampMicros = pts.timescale != 0 ? (pts.value * 1000000 / pts.timescale) : 0;
|
|
377
|
-
|
|
378
|
-
CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer);
|
|
379
|
-
if (format != nullptr) {
|
|
380
|
-
if (options_.codec == VideoStreamCodec::kHEVC) {
|
|
381
|
-
AppendParameterSets(unit.data, format, CMVideoFormatDescriptionGetHEVCParameterSetAtIndex);
|
|
382
|
-
} else {
|
|
383
|
-
AppendParameterSets(unit.data, format, CMVideoFormatDescriptionGetH264ParameterSetAtIndex);
|
|
384
|
-
}
|
|
385
|
-
}
|
|
386
|
-
}
|
|
387
|
-
AppendSampleBufferNALs(unit.data, sampleBuffer);
|
|
39
|
+
RepackAsAnnexB(unit.data, sampleBuffer, isKeyFrame, options_.codec);
|
|
388
40
|
if (onAccessUnit_) {
|
|
389
41
|
onAccessUnit_(std::move(unit));
|
|
390
42
|
}
|
|
391
43
|
}
|
|
392
44
|
|
|
393
|
-
|
|
394
|
-
VideoStreamOptions options_;
|
|
45
|
+
VideoEncoderOptions options_;
|
|
395
46
|
std::function<void(VideoAccessUnit)> onAccessUnit_;
|
|
396
|
-
std::function<void(
|
|
397
|
-
std::
|
|
398
|
-
|
|
399
|
-
dispatch_queue_t queue_ = nullptr;
|
|
400
|
-
dispatch_source_t timer_ = nullptr;
|
|
401
|
-
VTCompressionSessionRef session_ = nullptr;
|
|
402
|
-
uint32_t lastSeed_ = 0;
|
|
47
|
+
std::function<void()> onAbortDelivery_;
|
|
48
|
+
std::unique_ptr<VideoFrameEncoder> encoder_;
|
|
403
49
|
// VideoToolbox's output callback isn't documented as single-threaded, so this is read-modify-
|
|
404
50
|
// written atomically rather than assuming HandleEncodedSample never runs concurrently.
|
|
405
51
|
std::atomic<uint64_t> sequence_{0};
|
|
406
|
-
double startTime_ = 0;
|
|
407
|
-
std::atomic<bool> running_{false};
|
|
408
|
-
// Set by HandleEncodedSample (possibly off queue_) on an encoder failure, consumed by the next
|
|
409
|
-
// Tick() (on queue_) — see both for why teardown can't just happen inline there.
|
|
410
|
-
std::atomic<bool> pendingErrorTeardown_{false};
|
|
411
|
-
std::atomic<bool> forceKeyFrame_{false};
|
|
412
52
|
};
|
|
413
53
|
|
|
414
|
-
VideoStreamSession::VideoStreamSession(id device,
|
|
54
|
+
VideoStreamSession::VideoStreamSession(id device, VideoEncoderOptions options,
|
|
415
55
|
std::function<void(VideoAccessUnit)> onAccessUnit,
|
|
416
|
-
std::function<void(NSError*)> onError, std::function<void()> onEnd
|
|
417
|
-
|
|
56
|
+
std::function<void(NSError*)> onError, std::function<void()> onEnd,
|
|
57
|
+
std::function<void()> onAbortDelivery)
|
|
58
|
+
: impl_(std::make_unique<Impl>(device, options, std::move(onAccessUnit), std::move(onError), std::move(onEnd),
|
|
59
|
+
std::move(onAbortDelivery))) {}
|
|
418
60
|
|
|
419
61
|
VideoStreamSession::~VideoStreamSession() = default;
|
|
420
62
|
|
|
@@ -422,6 +64,8 @@ void VideoStreamSession::Start() { impl_->Start(); }
|
|
|
422
64
|
|
|
423
65
|
void VideoStreamSession::Stop() { impl_->Stop(); }
|
|
424
66
|
|
|
67
|
+
void VideoStreamSession::AbortDelivery() { impl_->AbortDelivery(); }
|
|
68
|
+
|
|
425
69
|
void VideoStreamSession::RequestKeyFrame() { impl_->RequestKeyFrame(); }
|
|
426
70
|
|
|
427
71
|
} // namespace coresim
|