@appium/coresim 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/README.md +14 -0
  3. package/binding.gyp +7 -0
  4. package/lib/src/commands/video-recording.d.ts +14 -7
  5. package/lib/src/commands/video-recording.d.ts.map +1 -1
  6. package/lib/src/commands/video-recording.js +36 -11
  7. package/lib/src/commands/video-recording.js.map +1 -1
  8. package/lib/src/commands/video-stream.d.ts +55 -7
  9. package/lib/src/commands/video-stream.d.ts.map +1 -1
  10. package/lib/src/commands/video-stream.js +37 -35
  11. package/lib/src/commands/video-stream.js.map +1 -1
  12. package/lib/src/native-simctl.js +12 -1
  13. package/lib/src/native-simctl.js.map +1 -1
  14. package/lib/src/types.d.ts +72 -10
  15. package/lib/src/types.d.ts.map +1 -1
  16. package/lib/src/utils/index.d.ts +1 -1
  17. package/lib/src/utils/index.d.ts.map +1 -1
  18. package/lib/src/utils/index.js +1 -1
  19. package/lib/src/utils/index.js.map +1 -1
  20. package/lib/src/utils/run-catching.d.ts +4 -0
  21. package/lib/src/utils/run-catching.d.ts.map +1 -1
  22. package/lib/src/utils/run-catching.js +11 -0
  23. package/lib/src/utils/run-catching.js.map +1 -1
  24. package/package.json +1 -1
  25. package/prebuilds/darwin-arm64/@appium+coresim.node +0 -0
  26. package/src/commands/video-recording.ts +49 -19
  27. package/src/commands/video-stream.ts +48 -39
  28. package/src/coresim.mm +547 -118
  29. package/src/native/audio_encoder.h +65 -0
  30. package/src/native/audio_encoder.mm +270 -0
  31. package/src/native/av_recording.h +52 -0
  32. package/src/native/av_recording.mm +424 -0
  33. package/src/native/av_stream.h +70 -0
  34. package/src/native/av_stream.mm +217 -0
  35. package/src/native/monotonic_clock.h +20 -0
  36. package/src/native/sim_audio_tap.h +67 -0
  37. package/src/native/sim_audio_tap.mm +350 -0
  38. package/src/native/sim_process.h +10 -0
  39. package/src/native/sim_process.mm +81 -18
  40. package/src/native/sim_video_stream.h +14 -11
  41. package/src/native/sim_video_stream.mm +27 -383
  42. package/src/native/video_encoder.h +76 -0
  43. package/src/native/video_encoder.mm +417 -0
  44. package/src/native-simctl.ts +12 -1
  45. package/src/types.ts +84 -11
  46. package/src/utils/index.ts +1 -1
  47. package/src/utils/run-catching.ts +11 -0
@@ -6,6 +6,7 @@
6
6
  #include <sys/un.h>
7
7
 
8
8
  #include <cstring>
9
+ #include <string>
9
10
  #include <vector>
10
11
 
11
12
  namespace coresim {
@@ -25,48 +26,96 @@ NSString* ExecutablePath(pid_t pid) {
25
26
  return size > 0 ? [NSString stringWithUTF8String:pathBuf] : nil;
26
27
  }
27
28
 
28
- // argv[1] via the unprivileged KERN_PROCARGS2 sysctl (same as `ps`/`lsof`) — for `launchd_sim`
29
- // that's its bootstrap plist path, which embeds the device's UDID.
30
- NSString* FirstArgument(pid_t pid) {
29
+ // Layout: argc, exec path, NUL padding, then argv[0..argc-1], then envp[...], each NUL-terminated
30
+ // (envp has no guaranteed empty-string sentinel within the buffer — just runs to `size`).
31
+ std::vector<char> ProcArgs2(pid_t pid) {
31
32
  int mib[3] = {CTL_KERN, KERN_PROCARGS2, pid};
32
33
  size_t size = 0;
33
34
  if (sysctl(mib, 3, nullptr, &size, nullptr, 0) != 0 || size < sizeof(int)) {
34
- return nil;
35
+ return {};
35
36
  }
36
37
  std::vector<char> buffer(size);
37
38
  if (sysctl(mib, 3, buffer.data(), &size, nullptr, 0) != 0) {
39
+ return {};
40
+ }
41
+ return buffer;
42
+ }
43
+
44
+ const char* SkipExecPathAndPadding(const char* cursor, const char* end) {
45
+ cursor += strnlen(cursor, static_cast<size_t>(end - cursor)); // skip the exec path
46
+ while (cursor < end && *cursor == '\0') cursor++; // skip the NUL padding
47
+ return cursor;
48
+ }
49
+
50
+ // Advances past `count` further NUL-terminated entries (e.g. argv), landing just after the last
51
+ // one's terminator — or at `end` if there weren't that many.
52
+ const char* SkipEntries(const char* cursor, const char* end, int count) {
53
+ for (int i = 0; i < count && cursor < end; i++) {
54
+ cursor += strnlen(cursor, static_cast<size_t>(end - cursor));
55
+ if (cursor < end) cursor++;
56
+ }
57
+ return cursor;
58
+ }
59
+
60
+ // argv[1] via the unprivileged KERN_PROCARGS2 sysctl (same as `ps`/`lsof`) — for `launchd_sim`
61
+ // that's its bootstrap plist path, which embeds the device's UDID.
62
+ NSString* FirstArgument(pid_t pid) {
63
+ std::vector<char> buffer = ProcArgs2(pid);
64
+ if (buffer.empty()) {
38
65
  return nil;
39
66
  }
40
- // Layout: argc, exec path, NUL padding, then argv[0], argv[1], ..., each NUL-terminated.
41
67
  int argc = 0;
42
68
  std::memcpy(&argc, buffer.data(), sizeof(argc));
43
- const char* cursor = buffer.data() + sizeof(argc);
44
- const char* end = buffer.data() + size;
45
- cursor += strnlen(cursor, static_cast<size_t>(end - cursor)); // skip the exec path
46
- while (cursor < end && *cursor == '\0') cursor++; // skip the NUL padding
47
- cursor += strnlen(cursor, static_cast<size_t>(end - cursor)); // skip argv[0]
48
- if (cursor < end) cursor++; // skip its NUL terminator
69
+ const char* end = buffer.data() + buffer.size();
70
+ const char* cursor = SkipEntries(SkipExecPathAndPadding(buffer.data() + sizeof(argc), end), end, 1);
49
71
  if (argc < 2 || cursor >= end) {
50
72
  return nil;
51
73
  }
52
74
  return [NSString stringWithUTF8String:cursor];
53
75
  }
54
76
 
55
- // Each booted simulator has its own `launchd_sim`; finds the one owning `udid`.
56
- pid_t FindLaunchdSimPid(NSString* udid) {
77
+ // The value of `key=...` in `pid`'s environment, past the end of its argv — same sysctl as
78
+ // FirstArgument, walked further.
79
+ NSString* FindEnvValue(pid_t pid, NSString* key) {
80
+ std::vector<char> buffer = ProcArgs2(pid);
81
+ if (buffer.empty()) {
82
+ return nil;
83
+ }
84
+ int argc = 0;
85
+ std::memcpy(&argc, buffer.data(), sizeof(argc));
86
+ const char* end = buffer.data() + buffer.size();
87
+ const char* cursor = SkipEntries(SkipExecPathAndPadding(buffer.data() + sizeof(argc), end), end, argc);
88
+ std::string prefix = std::string(key.UTF8String) + "=";
89
+ while (cursor < end && *cursor != '\0') {
90
+ size_t entryLen = strnlen(cursor, static_cast<size_t>(end - cursor));
91
+ if (entryLen > prefix.size() && std::memcmp(cursor, prefix.data(), prefix.size()) == 0) {
92
+ return [NSString stringWithUTF8String:cursor + prefix.size()];
93
+ }
94
+ cursor += entryLen;
95
+ if (cursor < end) cursor++;
96
+ }
97
+ return nil;
98
+ }
99
+
100
+ // Every live pid on the system, via the unprivileged proc_listpids sysctl.
101
+ std::vector<pid_t> AllPids() {
57
102
  int neededBytes = proc_listpids(PROC_ALL_PIDS, 0, nullptr, 0);
58
103
  if (neededBytes <= 0) {
59
- return -1;
104
+ return {};
60
105
  }
61
106
  // Headroom for processes started between the sizing call above and the listing call below.
62
107
  std::vector<pid_t> pids(neededBytes / sizeof(pid_t) + 64);
63
108
  int bytes = proc_listpids(PROC_ALL_PIDS, 0, pids.data(), static_cast<int>(pids.size() * sizeof(pid_t)));
64
109
  if (bytes <= 0) {
65
- return -1;
110
+ return {};
66
111
  }
67
- int count = bytes / static_cast<int>(sizeof(pid_t));
68
- for (int i = 0; i < count; i++) {
69
- pid_t pid = pids[i];
112
+ pids.resize(static_cast<size_t>(bytes) / sizeof(pid_t));
113
+ return pids;
114
+ }
115
+
116
+ // Each booted simulator has its own `launchd_sim`; finds the one owning `udid`.
117
+ pid_t FindLaunchdSimPid(NSString* udid) {
118
+ for (pid_t pid : AllPids()) {
70
119
  if (pid <= 0) {
71
120
  continue;
72
121
  }
@@ -136,4 +185,18 @@ NSString* FindWebInspectorSocket(NSString* udid, NSError** error) {
136
185
  return socketPath;
137
186
  }
138
187
 
188
+ std::vector<pid_t> FindGuestProcessPids(NSString* udid) {
189
+ std::vector<pid_t> result;
190
+ for (pid_t pid : AllPids()) {
191
+ if (pid <= 0) {
192
+ continue;
193
+ }
194
+ NSString* value = FindEnvValue(pid, @"SIMULATOR_UDID");
195
+ if (value != nil && [value caseInsensitiveCompare:udid] == NSOrderedSame) {
196
+ result.push_back(pid);
197
+ }
198
+ }
199
+ return result;
200
+ }
201
+
139
202
  } // namespace coresim
@@ -7,9 +7,9 @@
7
7
  #include <memory>
8
8
  #include <vector>
9
9
 
10
- namespace coresim {
10
+ #include "video_encoder.h"
11
11
 
12
- enum class VideoStreamCodec { kH264, kHEVC };
12
+ namespace coresim {
13
13
 
14
14
  // One encoded frame — Annex-B NAL units, concatenated. A keyframe's `data` has parameter sets
15
15
  // (SPS/PPS, or VPS/SPS/PPS for HEVC) prepended, so every keyframe is self-decodable alone.
@@ -21,20 +21,16 @@ struct VideoAccessUnit {
21
21
  int64_t timestampMicros = 0;
22
22
  };
23
23
 
24
- struct VideoStreamOptions {
25
- VideoStreamCodec codec = VideoStreamCodec::kH264;
26
- NSString* displayId = nil;
27
- double fps = 15.0;
28
- int bitrate = 2000000;
29
- };
30
-
31
24
  // Polls the live display IOSurface (sim_screenshot.h) on a serial queue and encodes changed
32
25
  // frames via the public VideoToolbox API — unlike StartVideoRecording's private, file-only
33
26
  // recorder, this delivers access units live. Full thread-safety contract: see CLAUDE.md.
34
27
  class VideoStreamSession {
35
28
  public:
36
- VideoStreamSession(id device, VideoStreamOptions options, std::function<void(VideoAccessUnit)> onAccessUnit,
37
- std::function<void(NSError*)> onError, std::function<void()> onEnd);
29
+ // `onAbortDelivery`, if set, is invoked by AbortDelivery() below — not called by this class on
30
+ // its own.
31
+ VideoStreamSession(id device, VideoEncoderOptions options, std::function<void(VideoAccessUnit)> onAccessUnit,
32
+ std::function<void(NSError*)> onError, std::function<void()> onEnd,
33
+ std::function<void()> onAbortDelivery = nullptr);
38
34
  ~VideoStreamSession();
39
35
 
40
36
  VideoStreamSession(const VideoStreamSession&) = delete;
@@ -48,6 +44,13 @@ class VideoStreamSession {
48
44
  // onError/onEnd — same queue this blocks on, so it would deadlock.
49
45
  void Stop();
50
46
 
47
+ // Runs the constructor's `onAbortDelivery` (if any) — e.g. aborting a bounded delivery queue so
48
+ // a producer thread blocked pushing into it unblocks. Call before Stop() when the caller can't
49
+ // rely on anything else draining that queue concurrently (e.g. process-exit cleanup running
50
+ // synchronously on the same thread `Stop()` would otherwise wait on) — a normal Stop() doesn't
51
+ // need this. Safe from any thread; a no-op if unset.
52
+ void AbortDelivery();
53
+
51
54
  // Forces the next encoded frame to be a keyframe (self-decodable, parameter sets included) —
52
55
  // e.g. so a consumer that just resynced after dropping frames can resume cleanly instead of
53
56
  // waiting for the next periodic one. Safe from any thread; just sets a flag.
@@ -1,372 +1,34 @@
1
1
  #include "sim_video_stream.h"
2
2
 
3
- #import <CoreMedia/CoreMedia.h>
4
- #import <CoreVideo/CoreVideo.h>
5
- #import <IOSurface/IOSurface.h>
6
- #import <VideoToolbox/VideoToolbox.h>
7
-
8
- #include <algorithm>
9
3
  #include <atomic>
10
- #include <ctime>
11
4
 
12
- #include "nserror_bridge.h"
13
- #include "safe_dispatch.h"
14
- #include "sim_screenshot.h"
5
+ #include "video_encoder.h"
15
6
 
16
7
  namespace coresim {
17
8
 
18
- namespace {
19
-
20
- NSString* const kVideoStreamErrorDomain = @"com.appium.coresim.VideoStream";
21
-
22
- NSError* MakeError(NSInteger code, NSString* message) {
23
- return [NSError errorWithDomain:kVideoStreamErrorDomain code:code userInfo:@{NSLocalizedDescriptionKey : message}];
24
- }
25
-
26
- NSError* MakeStatusError(NSInteger code, NSString* what, OSStatus status) {
27
- return MakeError(code, [NSString stringWithFormat:@"%@ (OSStatus %d)", what, static_cast<int>(status)]);
28
- }
29
-
30
- // CFAbsoluteTimeGetCurrent() is wall time — subject to backward NTP/manual clock corrections,
31
- // which would violate VTCompressionSessionEncodeFrame's requirement of strictly increasing PTS.
32
- // CLOCK_MONOTONIC_RAW never goes backward and isn't adjusted by NTP.
33
- double MonotonicSeconds() {
34
- struct timespec ts;
35
- clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
36
- return static_cast<double>(ts.tv_sec) + static_cast<double>(ts.tv_nsec) / 1e9;
37
- }
38
-
39
- void AppendAnnexB(std::vector<uint8_t>& out, const uint8_t* nal, size_t length) {
40
- static const uint8_t kStartCode[4] = {0, 0, 0, 1};
41
- out.insert(out.end(), kStartCode, kStartCode + 4);
42
- out.insert(out.end(), nal, nal + length);
43
- }
44
-
45
- // VideoToolbox's compressed output is AVCC-framed (a 4-byte big-endian length prefix per NAL,
46
- // no start codes) — rewrites it into Annex-B, matching VideoAccessUnit's documented wire format.
47
- void AppendSampleBufferNALs(std::vector<uint8_t>& out, CMSampleBufferRef sampleBuffer) {
48
- CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sampleBuffer);
49
- if (block == nullptr) {
50
- return;
51
- }
52
- size_t totalLength = CMBlockBufferGetDataLength(block);
53
- if (totalLength == 0) {
54
- return;
55
- }
56
- // CMBlockBufferGetDataPointer's pointer only covers the contiguous region starting at the given
57
- // offset, which for a segmented buffer (multiple backing memory blocks — CoreMedia's documented
58
- // contract, not just a VideoToolbox implementation detail) can be far shorter than totalLength;
59
- // reading up to totalLength through it would run past that region. CopyDataBytes stitches
60
- // segments together into a caller-owned, guaranteed-contiguous copy instead.
61
- std::vector<uint8_t> data(totalLength);
62
- if (CMBlockBufferCopyDataBytes(block, 0, totalLength, data.data()) != kCMBlockBufferNoErr) {
63
- return;
64
- }
65
- const uint8_t* dataPointer = data.data();
66
- size_t offset = 0;
67
- while (offset + 4 <= totalLength) {
68
- uint32_t nalLength = (static_cast<uint32_t>(dataPointer[offset]) << 24) |
69
- (static_cast<uint32_t>(dataPointer[offset + 1]) << 16) |
70
- (static_cast<uint32_t>(dataPointer[offset + 2]) << 8) | dataPointer[offset + 3];
71
- offset += 4;
72
- if (nalLength == 0 || offset + nalLength > totalLength) {
73
- break;
74
- }
75
- AppendAnnexB(out, dataPointer + offset, nalLength);
76
- offset += nalLength;
77
- }
78
- }
79
-
80
- using ParameterSetAtIndexFn = OSStatus (*)(CMFormatDescriptionRef, size_t, const uint8_t**, size_t*, size_t*, int*);
81
-
82
- void AppendParameterSets(std::vector<uint8_t>& out, CMFormatDescriptionRef format, ParameterSetAtIndexFn getAtIndex) {
83
- size_t count = 0;
84
- if (getAtIndex(format, 0, nullptr, nullptr, &count, nullptr) != noErr) {
85
- return;
86
- }
87
- for (size_t i = 0; i < count; i++) {
88
- const uint8_t* bytes = nullptr;
89
- size_t size = 0;
90
- if (getAtIndex(format, i, &bytes, &size, nullptr, nullptr) == noErr) {
91
- AppendAnnexB(out, bytes, size);
92
- }
93
- }
94
- }
95
-
96
- } // namespace
97
-
98
9
  class VideoStreamSession::Impl {
99
10
  public:
100
- Impl(id device, VideoStreamOptions options, std::function<void(VideoAccessUnit)> onAccessUnit,
101
- std::function<void(NSError*)> onError, std::function<void()> onEnd)
102
- : device_(device),
103
- options_(options),
104
- onAccessUnit_(std::move(onAccessUnit)),
105
- onError_(std::move(onError)),
106
- onEnd_(std::move(onEnd)) {
107
- queue_ = dispatch_queue_create("com.appium.coresim.videoStream", DISPATCH_QUEUE_SERIAL);
108
- }
109
-
110
- ~Impl() { Stop(); }
111
-
112
- void Start() {
113
- NSError* error = nil;
114
- id descriptor = ResolveCaptureDisplay(device_, options_.displayId, &error);
115
- if (descriptor == nil) {
116
- throw NSErrorException(error);
117
- }
118
- id surfaceObj = CurrentDisplaySurface(descriptor);
119
- if (surfaceObj == nil) {
120
- throw NSErrorException(MakeError(4, @"The device's display surface is not available yet"));
121
- }
122
- IOSurfaceRef surface = (__bridge IOSurfaceRef)surfaceObj;
123
- // Set up synchronously (not lazily on the first Tick()) so a setup failure rejects Start()
124
- // directly rather than only reaching onError, which the caller may not be listening for yet.
125
- NSError* setupError = nil;
126
- if (!SetUpSession(surface, &setupError)) {
127
- throw NSErrorException(setupError);
128
- }
129
- // Must be set before EncodeSurface below — both it and HandleEncodedSample measure elapsed
130
- // time from this.
131
- startTime_ = MonotonicSeconds();
132
- // Encode immediately rather than waiting for a *changed* seed on the first tick, or the
133
- // stream would stay silent until the display changes again. running_ is set true before this
134
- // call (not after), since VTCompressionSessionEncodeFrame's output callback can in principle
135
- // fire on another thread before this one returns — HandleEncodedSample discards samples while
136
- // running_ is false, which would otherwise silently drop the stream's very first (keyframe)
137
- // access unit. A failure resets it and tears session_ down itself here, rather than going
138
- // through Stop()/onEnd_ (see coresim.mm — onEnd_ firing this early would double-release its
139
- // ThreadSafeFunctions).
140
- running_ = true;
141
- bool encoded = false;
142
- try {
143
- encoded = EncodeSurface(surface);
144
- } catch (...) {
145
- running_ = false;
146
- VTCompressionSessionInvalidate(session_);
147
- CFRelease(session_);
148
- session_ = nullptr;
149
- throw;
150
- }
151
- // Only commit the seed once a frame was actually submitted — a transient pixel-buffer
152
- // creation failure (EncodeSurface returning false) otherwise leaves lastSeed_ at its default
153
- // 0, so the first Tick() sees the real seed as "changed" and retries automatically instead of
154
- // the stream going silent forever on a display that never changes again.
155
- if (encoded) {
156
- lastSeed_ = IOSurfaceGetSeed(surface);
157
- }
158
-
159
- double interval = 1.0 / std::max(options_.fps, 1.0);
160
- dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, queue_);
161
- dispatch_source_set_timer(timer, dispatch_time(DISPATCH_TIME_NOW, 0),
162
- static_cast<uint64_t>(interval * NSEC_PER_SEC),
163
- static_cast<uint64_t>(interval * NSEC_PER_SEC / 10));
164
- // `this` outlives the timer: Stop()/StopFromQueue() always drain or outrun it before `this`
165
- // can be destroyed (see their comments below).
166
- dispatch_source_set_event_handler(timer, ^{
167
- Tick();
168
- });
169
- timer_ = timer;
170
- dispatch_resume(timer_);
11
+ Impl(id device, VideoEncoderOptions options, std::function<void(VideoAccessUnit)> onAccessUnit,
12
+ std::function<void(NSError*)> onError, std::function<void()> onEnd, std::function<void()> onAbortDelivery)
13
+ : options_(options), onAccessUnit_(std::move(onAccessUnit)), onAbortDelivery_(std::move(onAbortDelivery)) {
14
+ encoder_ = std::make_unique<VideoFrameEncoder>(
15
+ device, options, [this](CMSampleBufferRef sampleBuffer) { HandleEncodedSample(sampleBuffer); },
16
+ std::move(onError), std::move(onEnd));
171
17
  }
172
18
 
173
- // Callable from any thread except `queue_` itself (would deadlock on the dispatch_sync below).
174
- void Stop() {
175
- if (!running_.exchange(false)) {
176
- return; // idempotent
177
- }
178
- if (timer_ != nullptr) {
179
- dispatch_source_cancel(timer_);
180
- // Blocks until any in-flight Tick() finishes — by then running_ is already false, so it
181
- // won't touch session_ again.
182
- dispatch_sync(queue_, ^{
183
- });
184
- timer_ = nullptr;
19
+ void Start() { encoder_->Start(); }
20
+ void Stop() { encoder_->Stop(); }
21
+ void AbortDelivery() {
22
+ if (onAbortDelivery_) {
23
+ onAbortDelivery_();
185
24
  }
186
- TearDownSessionAndFireEnd();
187
25
  }
188
-
189
- void RequestKeyFrame() { forceKeyFrame_ = true; }
26
+ void RequestKeyFrame() { encoder_->RequestKeyFrame(); }
190
27
 
191
28
  private:
192
- // Same as Stop() minus the dispatch_sync barrier — only safe from within Tick() itself, already
193
- // serialized on `queue_`; would race a concurrent Tick() from any other thread.
194
- void StopFromQueue() {
195
- if (!running_.exchange(false)) {
196
- return; // idempotent — e.g. an external Stop() already won this race
197
- }
198
- if (timer_ != nullptr) {
199
- dispatch_source_cancel(timer_);
200
- timer_ = nullptr;
201
- }
202
- TearDownSessionAndFireEnd();
203
- }
204
-
205
- void TearDownSessionAndFireEnd() {
206
- if (session_ != nullptr) {
207
- // Flushes and blocks until every already-submitted frame's callback has returned — without
208
- // this, a frame submitted just before Stop() could fire after onEnd_ releases the
209
- // ThreadSafeFunctions below (a use-after-release; see CLAUDE.md).
210
- VTCompressionSessionCompleteFrames(session_, kCMTimeInvalid);
211
- VTCompressionSessionInvalidate(session_);
212
- CFRelease(session_);
213
- session_ = nullptr;
214
- }
215
- if (onEnd_) {
216
- onEnd_();
217
- }
218
- }
219
-
220
- void Tick() {
221
- if (!running_) {
222
- return;
223
- }
224
- if (pendingErrorTeardown_) {
225
- // HandleEncodedSample (below) can run on VideoToolbox's own callback thread, where it's
226
- // unsafe to tear down directly — StopFromQueue()/CompleteFrames() are only safe already
227
- // serialized on `queue_` (here), and CompleteFrames specifically would deadlock waiting on
228
- // its own still-executing callback. It sets this flag instead; picked up on the very next
229
- // tick (queue_-serialized, safe) rather than via a raw cross-thread dispatch, since nothing
230
- // here can outlive `this` the way a block captured on another thread otherwise could.
231
- StopFromQueue();
232
- return;
233
- }
234
- @autoreleasepool {
235
- try {
236
- // Re-resolved every tick (like CaptureScreenshot does), not cached once in Start(), so a
237
- // deleted device or disconnected display surfaces a real error instead of Tick() quietly
238
- // doing nothing forever.
239
- NSError* resolveError = nil;
240
- id descriptor = ResolveCaptureDisplay(device_, options_.displayId, &resolveError);
241
- if (descriptor == nil) {
242
- if (onError_) {
243
- onError_(resolveError);
244
- }
245
- StopFromQueue();
246
- return;
247
- }
248
- id surfaceObj = CurrentDisplaySurface(descriptor);
249
- if (surfaceObj == nil) {
250
- return; // transient — the connection may not have a frame ready yet, try again next tick
251
- }
252
- IOSurfaceRef surface = (__bridge IOSurfaceRef)surfaceObj;
253
- uint32_t seed = IOSurfaceGetSeed(surface);
254
- if (seed == lastSeed_) {
255
- return; // unchanged since the last tick — mirrors CoreSimulator's own recorder, which
256
- // only encodes a frame when the display actually changes (see CLAUDE.md)
257
- }
258
- // Only commit the new seed once EncodeSurface actually submits it — a transient failure
259
- // (pixel-buffer creation) must leave lastSeed_ stale so the next tick retries this same
260
- // frame instead of silently going quiet until the display changes again.
261
- if (EncodeSurface(surface)) {
262
- lastSeed_ = seed;
263
- }
264
- } catch (const std::exception& e) {
265
- // Without this, an exception here (e.g. a dropped display-proxy connection) would escape
266
- // this bare GCD timer handler uncaught and crash the whole process (see CLAUDE.md).
267
- if (onError_) {
268
- onError_(MakeError(3, [NSString stringWithFormat:@"Video stream encoding failed: %s", e.what()]));
269
- }
270
- StopFromQueue();
271
- }
272
- }
273
- }
274
-
275
- bool SetUpSession(IOSurfaceRef surface, NSError** error) {
276
- int32_t width = static_cast<int32_t>(IOSurfaceGetWidth(surface));
277
- int32_t height = static_cast<int32_t>(IOSurfaceGetHeight(surface));
278
- CMVideoCodecType codecType =
279
- options_.codec == VideoStreamCodec::kHEVC ? kCMVideoCodecType_HEVC : kCMVideoCodecType_H264;
280
- OSStatus status = VTCompressionSessionCreate(kCFAllocatorDefault, width, height, codecType, nullptr, nullptr,
281
- kCFAllocatorDefault, OutputCallback, this, &session_);
282
- if (status != noErr) {
283
- *error = MakeStatusError(1, @"Failed to create a VTCompressionSession", status);
284
- return false;
285
- }
286
- // Clamped like Start()'s timer interval — an unvalidated 0 here would set MaxKeyFrameInterval
287
- // to an out-of-spec value.
288
- double fps = std::max(options_.fps, 1.0);
289
- status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
290
- if (status == noErr) {
291
- status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse);
292
- }
293
- if (status == noErr) {
294
- status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_AverageBitRate,
295
- (__bridge CFNumberRef) @(options_.bitrate));
296
- }
297
- if (status == noErr) {
298
- status =
299
- VTSessionSetProperty(session_, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFNumberRef) @(fps));
300
- }
301
- if (status == noErr) {
302
- status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_MaxKeyFrameInterval,
303
- (__bridge CFNumberRef) @(static_cast<int>(fps * 2)));
304
- }
305
- if (status != noErr) {
306
- *error = MakeStatusError(2, @"Failed to configure the VTCompressionSession", status);
307
- VTCompressionSessionInvalidate(session_);
308
- CFRelease(session_);
309
- session_ = nullptr;
310
- return false;
311
- }
312
- VTCompressionSessionPrepareToEncodeFrames(session_);
313
- return true;
314
- }
315
-
316
- // Returns whether a frame was actually submitted to the encoder — false for a transient
317
- // pixel-buffer creation failure the caller should retry, as opposed to a real encode failure
318
- // (thrown, not returned, since that tears down the whole stream).
319
- bool EncodeSurface(IOSurfaceRef surface) {
320
- CVPixelBufferRef pixelBuffer = nullptr;
321
- CVReturn cvStatus = CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, surface, nullptr, &pixelBuffer);
322
- if (cvStatus != kCVReturnSuccess || pixelBuffer == nullptr) {
323
- return false; // transient — try again next tick rather than tearing down the whole stream
324
- }
325
- CMTime pts = CMTimeMake(static_cast<int64_t>((MonotonicSeconds() - startTime_) * 1000000), 1000000);
326
- NSDictionary* frameProperties = nil;
327
- if (forceKeyFrame_.exchange(false)) {
328
- frameProperties = @{(__bridge NSString*)kVTEncodeFrameOptionKey_ForceKeyFrame : @YES};
329
- }
330
- OSStatus status = VTCompressionSessionEncodeFrame(session_, pixelBuffer, pts, kCMTimeInvalid,
331
- (__bridge CFDictionaryRef)frameProperties, nullptr, nullptr);
332
- CVPixelBufferRelease(pixelBuffer);
333
- if (status != noErr) {
334
- throw std::runtime_error([[NSString stringWithFormat:@"VTCompressionSessionEncodeFrame failed (OSStatus %d)",
335
- static_cast<int>(status)] UTF8String]);
336
- }
337
- return true;
338
- }
339
-
340
- static void OutputCallback(void* outputCallbackRefCon, void* /*sourceFrameRefCon*/, OSStatus status,
341
- VTEncodeInfoFlags /*infoFlags*/, CMSampleBufferRef sampleBuffer) {
342
- static_cast<Impl*>(outputCallbackRefCon)->HandleEncodedSample(status, sampleBuffer);
343
- }
344
-
345
- void HandleEncodedSample(OSStatus status, CMSampleBufferRef sampleBuffer) {
346
- if (!running_) {
347
- return;
348
- }
349
- if (status != noErr) {
350
- // Only the first failure is reported — pendingErrorTeardown_ doubles as the report-once
351
- // gate, since Tick() (the only place that consumes it) only ever needs to see it once too.
352
- if (!pendingErrorTeardown_.exchange(true)) {
353
- if (onError_) {
354
- onError_(MakeStatusError(3, @"VideoToolbox reported an encoding failure", status));
355
- }
356
- }
357
- return;
358
- }
359
- if (sampleBuffer == nullptr) {
360
- return;
361
- }
362
- bool isKeyFrame = true;
363
- CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, false);
364
- if (attachments != nullptr && CFArrayGetCount(attachments) > 0) {
365
- CFDictionaryRef attachment = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
366
- isKeyFrame = !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
367
- }
368
-
369
- // From the sample's own presentation timestamp (as submitted in EncodeSurface), not a fresh
29
+ void HandleEncodedSample(CMSampleBufferRef sampleBuffer) {
30
+ bool isKeyFrame = IsKeyFrame(sampleBuffer);
31
+ // From the sample's own presentation timestamp (as submitted to the encoder), not a fresh
370
32
  // wall-clock read here — this callback can fire well after encoding actually happened, and
371
33
  // resampling "now" would report a later, jittery timestamp than when the frame was captured.
372
34
  CMTime pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
@@ -374,47 +36,27 @@ class VideoStreamSession::Impl {
374
36
  unit.isKeyFrame = isKeyFrame;
375
37
  unit.sequence = sequence_++;
376
38
  unit.timestampMicros = pts.timescale != 0 ? (pts.value * 1000000 / pts.timescale) : 0;
377
- if (isKeyFrame) {
378
- CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer);
379
- if (format != nullptr) {
380
- if (options_.codec == VideoStreamCodec::kHEVC) {
381
- AppendParameterSets(unit.data, format, CMVideoFormatDescriptionGetHEVCParameterSetAtIndex);
382
- } else {
383
- AppendParameterSets(unit.data, format, CMVideoFormatDescriptionGetH264ParameterSetAtIndex);
384
- }
385
- }
386
- }
387
- AppendSampleBufferNALs(unit.data, sampleBuffer);
39
+ RepackAsAnnexB(unit.data, sampleBuffer, isKeyFrame, options_.codec);
388
40
  if (onAccessUnit_) {
389
41
  onAccessUnit_(std::move(unit));
390
42
  }
391
43
  }
392
44
 
393
- id device_;
394
- VideoStreamOptions options_;
45
+ VideoEncoderOptions options_;
395
46
  std::function<void(VideoAccessUnit)> onAccessUnit_;
396
- std::function<void(NSError*)> onError_;
397
- std::function<void()> onEnd_;
398
-
399
- dispatch_queue_t queue_ = nullptr;
400
- dispatch_source_t timer_ = nullptr;
401
- VTCompressionSessionRef session_ = nullptr;
402
- uint32_t lastSeed_ = 0;
47
+ std::function<void()> onAbortDelivery_;
48
+ std::unique_ptr<VideoFrameEncoder> encoder_;
403
49
  // VideoToolbox's output callback isn't documented as single-threaded, so this is read-modify-
404
50
  // written atomically rather than assuming HandleEncodedSample never runs concurrently.
405
51
  std::atomic<uint64_t> sequence_{0};
406
- double startTime_ = 0;
407
- std::atomic<bool> running_{false};
408
- // Set by HandleEncodedSample (possibly off queue_) on an encoder failure, consumed by the next
409
- // Tick() (on queue_) — see both for why teardown can't just happen inline there.
410
- std::atomic<bool> pendingErrorTeardown_{false};
411
- std::atomic<bool> forceKeyFrame_{false};
412
52
  };
413
53
 
414
- VideoStreamSession::VideoStreamSession(id device, VideoStreamOptions options,
54
+ VideoStreamSession::VideoStreamSession(id device, VideoEncoderOptions options,
415
55
  std::function<void(VideoAccessUnit)> onAccessUnit,
416
- std::function<void(NSError*)> onError, std::function<void()> onEnd)
417
- : impl_(std::make_unique<Impl>(device, options, std::move(onAccessUnit), std::move(onError), std::move(onEnd))) {}
56
+ std::function<void(NSError*)> onError, std::function<void()> onEnd,
57
+ std::function<void()> onAbortDelivery)
58
+ : impl_(std::make_unique<Impl>(device, options, std::move(onAccessUnit), std::move(onError), std::move(onEnd),
59
+ std::move(onAbortDelivery))) {}
418
60
 
419
61
  VideoStreamSession::~VideoStreamSession() = default;
420
62
 
@@ -422,6 +64,8 @@ void VideoStreamSession::Start() { impl_->Start(); }
422
64
 
423
65
  void VideoStreamSession::Stop() { impl_->Stop(); }
424
66
 
67
+ void VideoStreamSession::AbortDelivery() { impl_->AbortDelivery(); }
68
+
425
69
  void VideoStreamSession::RequestKeyFrame() { impl_->RequestKeyFrame(); }
426
70
 
427
71
  } // namespace coresim