@appium/coresim 1.4.0 → 1.6.0

package/src/native/sim_process.mm

@@ -6,6 +6,7 @@
 #include <sys/un.h>
 
 #include <cstring>
+#include <string>
 #include <vector>
 
 namespace coresim {
@@ -25,48 +26,96 @@ NSString* ExecutablePath(pid_t pid) {
   return size > 0 ? [NSString stringWithUTF8String:pathBuf] : nil;
 }
 
-// argv[1] via the unprivileged KERN_PROCARGS2 sysctl (same as `ps`/`lsof`) — for `launchd_sim`
-// that's its bootstrap plist path, which embeds the device's UDID.
-NSString* FirstArgument(pid_t pid) {
+// Layout: argc, exec path, NUL padding, then argv[0..argc-1], then envp[...], each NUL-terminated
+// (envp has no guaranteed empty-string sentinel within the buffer — just runs to `size`).
+std::vector<char> ProcArgs2(pid_t pid) {
   int mib[3] = {CTL_KERN, KERN_PROCARGS2, pid};
   size_t size = 0;
   if (sysctl(mib, 3, nullptr, &size, nullptr, 0) != 0 || size < sizeof(int)) {
-    return nil;
+    return {};
   }
   std::vector<char> buffer(size);
   if (sysctl(mib, 3, buffer.data(), &size, nullptr, 0) != 0) {
+    return {};
+  }
+  return buffer;
+}
+
+const char* SkipExecPathAndPadding(const char* cursor, const char* end) {
+  cursor += strnlen(cursor, static_cast<size_t>(end - cursor));  // skip the exec path
+  while (cursor < end && *cursor == '\0') cursor++;              // skip the NUL padding
+  return cursor;
+}
+
+// Advances past `count` further NUL-terminated entries (e.g. argv), landing just after the last
+// one's terminator — or at `end` if there weren't that many.
+const char* SkipEntries(const char* cursor, const char* end, int count) {
+  for (int i = 0; i < count && cursor < end; i++) {
+    cursor += strnlen(cursor, static_cast<size_t>(end - cursor));
+    if (cursor < end) cursor++;
+  }
+  return cursor;
+}
+
+// argv[1] via the unprivileged KERN_PROCARGS2 sysctl (same as `ps`/`lsof`) — for `launchd_sim`
+// that's its bootstrap plist path, which embeds the device's UDID.
+NSString* FirstArgument(pid_t pid) {
+  std::vector<char> buffer = ProcArgs2(pid);
+  if (buffer.empty()) {
     return nil;
   }
-  // Layout: argc, exec path, NUL padding, then argv[0], argv[1], ..., each NUL-terminated.
   int argc = 0;
   std::memcpy(&argc, buffer.data(), sizeof(argc));
-  const char* cursor = buffer.data() + sizeof(argc);
-  const char* end = buffer.data() + size;
-  cursor += strnlen(cursor, static_cast<size_t>(end - cursor));  // skip the exec path
-  while (cursor < end && *cursor == '\0') cursor++;              // skip the NUL padding
-  cursor += strnlen(cursor, static_cast<size_t>(end - cursor));  // skip argv[0]
-  if (cursor < end) cursor++;                                    // skip its NUL terminator
+  const char* end = buffer.data() + buffer.size();
+  const char* cursor = SkipEntries(SkipExecPathAndPadding(buffer.data() + sizeof(argc), end), end, 1);
   if (argc < 2 || cursor >= end) {
     return nil;
   }
   return [NSString stringWithUTF8String:cursor];
 }
 
-// Each booted simulator has its own `launchd_sim`; finds the one owning `udid`.
-pid_t FindLaunchdSimPid(NSString* udid) {
+// The value of `key=...` in `pid`'s environment, past the end of its argv — same sysctl as
+// FirstArgument, walked further.
+NSString* FindEnvValue(pid_t pid, NSString* key) {
+  std::vector<char> buffer = ProcArgs2(pid);
+  if (buffer.empty()) {
+    return nil;
+  }
+  int argc = 0;
+  std::memcpy(&argc, buffer.data(), sizeof(argc));
+  const char* end = buffer.data() + buffer.size();
+  const char* cursor = SkipEntries(SkipExecPathAndPadding(buffer.data() + sizeof(argc), end), end, argc);
+  std::string prefix = std::string(key.UTF8String) + "=";
+  while (cursor < end && *cursor != '\0') {
+    size_t entryLen = strnlen(cursor, static_cast<size_t>(end - cursor));
+    if (entryLen > prefix.size() && std::memcmp(cursor, prefix.data(), prefix.size()) == 0) {
+      return [NSString stringWithUTF8String:cursor + prefix.size()];
+    }
+    cursor += entryLen;
+    if (cursor < end) cursor++;
+  }
+  return nil;
+}
+
+// Every live pid on the system, via the unprivileged proc_listpids sysctl.
+std::vector<pid_t> AllPids() {
   int neededBytes = proc_listpids(PROC_ALL_PIDS, 0, nullptr, 0);
   if (neededBytes <= 0) {
-    return -1;
+    return {};
   }
   // Headroom for processes started between the sizing call above and the listing call below.
   std::vector<pid_t> pids(neededBytes / sizeof(pid_t) + 64);
   int bytes = proc_listpids(PROC_ALL_PIDS, 0, pids.data(), static_cast<int>(pids.size() * sizeof(pid_t)));
   if (bytes <= 0) {
-    return -1;
+    return {};
   }
-  int count = bytes / static_cast<int>(sizeof(pid_t));
-  for (int i = 0; i < count; i++) {
-    pid_t pid = pids[i];
+  pids.resize(static_cast<size_t>(bytes) / sizeof(pid_t));
+  return pids;
+}
+
+// Each booted simulator has its own `launchd_sim`; finds the one owning `udid`.
+pid_t FindLaunchdSimPid(NSString* udid) {
+  for (pid_t pid : AllPids()) {
     if (pid <= 0) {
       continue;
     }
@@ -136,4 +185,18 @@ NSString* FindWebInspectorSocket(NSString* udid, NSError** error) {
   return socketPath;
 }
 
+std::vector<pid_t> FindGuestProcessPids(NSString* udid) {
+  std::vector<pid_t> result;
+  for (pid_t pid : AllPids()) {
+    if (pid <= 0) {
+      continue;
+    }
+    NSString* value = FindEnvValue(pid, @"SIMULATOR_UDID");
+    if (value != nil && [value caseInsensitiveCompare:udid] == NSOrderedSame) {
+      result.push_back(pid);
+    }
+  }
+  return result;
+}
+
 }  // namespace coresim

package/src/native/sim_screenshot.h

@@ -12,6 +12,16 @@ namespace coresim {
 // array means the device has an IO client but no renderable display.
 NSArray<NSDictionary*>* ListDisplays(id device, NSError** error);
 
+// Resolves the display descriptor CaptureScreenshot itself reads from, without capturing a
+// screenshot — same displayId/fallback semantics, exposed for callers that need the descriptor
+// object itself (e.g. StartVideoRecording's `screen` argument).
+id ResolveCaptureDisplay(id device, NSString* displayId, NSError** error);
+
+// The descriptor's current framebuffer as an `IOSurfaceRef` (bridge-cast the returned `id`).
+// Returns nil if not available yet (e.g. connection just dropped) — not an error, since that can
+// legitimately happen from one call to the next on a live device.
+id CurrentDisplaySurface(id descriptor);
+
 enum class ScreenshotFormat { kPNG, kJPEG };
 
 // Captures a display as an image, reading the same in-process framebuffer surface `simctl io

package/src/native/sim_screenshot.mm

@@ -144,6 +144,16 @@ NSString* const kJPEGUTI = @"public.jpeg";
 
 }  // namespace
 
+id ResolveCaptureDisplay(id device, NSString* displayId, NSError** error) {
+  NSArray<NSDictionary*>* candidates = RenderableDisplayCandidates(device, error);
+  if (candidates == nil) {
+    return nil;
+  }
+  return ResolveDisplayDescriptor(candidates, displayId, error);
+}
+
+id CurrentDisplaySurface(id descriptor) { return RenderableSurface(descriptor); }
+
 NSArray<NSDictionary*>* ListDisplays(id device, NSError** error) {
   NSArray<NSDictionary*>* candidates = RenderableDisplayCandidates(device, error);
   if (candidates == nil) {
@@ -163,12 +173,7 @@ NSArray<NSDictionary*>* ListDisplays(id device, NSError** error) {
 
 NSData* CaptureScreenshot(id device, NSString* displayId, ScreenshotFormat format, NSNumber* jpegQualityPercent,
                           NSError** error) {
-  NSArray<NSDictionary*>* candidates = RenderableDisplayCandidates(device, error);
-  if (candidates == nil) {
-    return nil;
-  }
-
-  id descriptor = ResolveDisplayDescriptor(candidates, displayId, error);
+  id descriptor = ResolveCaptureDisplay(device, displayId, error);
   if (descriptor == nil) {
     return nil;
   }

package/src/native/sim_video_recording.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#import <Foundation/Foundation.h>
+#import <dispatch/dispatch.h>
+
+namespace coresim {
+
+// maskPolicy for a non-rectangular display. 0/1/2 = ignored/alpha/black, confirmed empirically —
+// alpha renders identically to black (see CLAUDE.md).
+enum class VideoMaskPolicy : long long {
+  kIgnored = 0,
+  kAlpha = 1,
+  kBlack = 2,
+};
+
+// Starts recording `displayId` (nil = primary) to `outputFile`, an absolute path, not a URL (see
+// CLAUDE.md). Returns NO+*error on synchronous resolution failure (`handler` never called then);
+// otherwise `handler` fires once the first frame is recorded, non-nil NSError on failure. Throws
+// NativeSimUnavailableError instead if this CoreSimulator has no video capture service at all.
+BOOL StartVideoRecording(id device, NSString* displayId, VideoMaskPolicy mask, NSDictionary* assetWriterOutputSettings,
+                         NSString* outputFile, dispatch_queue_t queue, void (^handler)(NSError*), NSError** error);
+
+// Stops the recording started by StartVideoRecording. Must not be called before its `handler` has
+// already fired — see CLAUDE.md for the race that otherwise causes a silent empty-file failure.
+BOOL StopVideoRecording(id device, dispatch_queue_t queue, void (^handler)(NSError*), NSError** error);
+
+}  // namespace coresim

package/src/native/sim_video_recording.mm

@@ -0,0 +1,87 @@
+#include "sim_video_recording.h"
+
+#import <objc/message.h>
+
+#include "objc_runtime.h"
+#include "safe_dispatch.h"
+#include "sim_screenshot.h"
+
+namespace coresim {
+
+namespace {
+
+NSString* const kVideoRecordingErrorDomain = @"com.appium.coresim.VideoRecording";
+
+NSError* MakeError(NSInteger code, NSString* message) {
+  return [NSError errorWithDomain:kVideoRecordingErrorDomain code:code userInfo:@{NSLocalizedDescriptionKey : message}];
+}
+
+id IdGetter(id target, const std::string& selectorName) {
+  RequireSelector(target, selectorName);
+  SEL selector = SelectorNamed(selectorName);
+  return SafeInvoke([&] {
+    using Fn = id (*)(id, SEL);
+    return ((Fn)objc_msgSend)(target, selector);
+  });
+}
+
+// The device-wide "capture service" port (real protocol: SimScreenCaptureService — see CLAUDE.md),
+// distinct from the display descriptor passed as `screen` below. Found by scanning ioPorts since
+// no header exists. Throws NativeSimUnavailableError (not NSError**) if absent.
+id ResolveVideoCaptureService(id device, NSError** error) {
+  static const std::string kStartRecordingSelector =
+      "startRecordingFromScreen:maskPolicy:assetWriterOutputSettings:outputFile:completionQueue:completionHandler:";
+  id ioClient = IdGetter(device, "io");
+  if (ioClient == nil) {
+    *error = MakeError(1, @"Device has no IO client available — is it booted?");
+    return nil;
+  }
+  NSArray* ports = IdGetter(ioClient, "ioPorts");
+  SEL selector = NSSelectorFromString(@(kStartRecordingSelector.c_str()));
+  for (id port in ports) {
+    id descriptor = IdGetter(port, "descriptor");
+    if (descriptor != nil && [descriptor respondsToSelector:selector]) {
+      return descriptor;
+    }
+  }
+  throw NativeSimUnavailableError("selector", kStartRecordingSelector, CoreSimulatorFrameworkVersion());
+}
+
+}  // namespace
+
+BOOL StartVideoRecording(id device, NSString* displayId, VideoMaskPolicy mask, NSDictionary* assetWriterOutputSettings,
+                         NSString* outputFile, dispatch_queue_t queue, void (^handler)(NSError*), NSError** error) {
+  id captureService = ResolveVideoCaptureService(device, error);
+  if (captureService == nil) {
+    return NO;
+  }
+  id screen = ResolveCaptureDisplay(device, displayId, error);
+  if (screen == nil) {
+    return NO;
+  }
+  static const std::string kSelectorName =
+      "startRecordingFromScreen:maskPolicy:assetWriterOutputSettings:outputFile:completionQueue:completionHandler:";
+  SEL selector = NSSelectorFromString(@(kSelectorName.c_str()));
+  return SafeInvoke([&] {
+    using Fn = void (*)(id, SEL, id, long long, NSDictionary*, NSString*, dispatch_queue_t, void (^)(NSError*));
+    ((Fn)objc_msgSend)(captureService, selector, screen, static_cast<long long>(mask), assetWriterOutputSettings,
+                       outputFile, queue, handler);
+    return YES;
+  });
+}
+
+BOOL StopVideoRecording(id device, dispatch_queue_t queue, void (^handler)(NSError*), NSError** error) {
+  id captureService = ResolveVideoCaptureService(device, error);
+  if (captureService == nil) {
+    return NO;
+  }
+  static const std::string kSelectorName = "stopRecordingWithCompletionQueue:completionHandler:";
+  SEL selector = NSSelectorFromString(@(kSelectorName.c_str()));
+  return SafeInvoke([&] {
+    using Fn = void (*)(id, SEL, dispatch_queue_t, void (^)(NSError*));
+    ((Fn)objc_msgSend)(captureService, selector, queue, handler);
+    return YES;
+  });
+}
+
+}  // namespace coresim

package/src/native/sim_video_stream.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#import <Foundation/Foundation.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include "video_encoder.h"
+
+namespace coresim {
+
+// One encoded frame — Annex-B NAL units, concatenated. A keyframe's `data` has parameter sets
+// (SPS/PPS, or VPS/SPS/PPS for HEVC) prepended, so every keyframe is self-decodable alone.
+struct VideoAccessUnit {
+  std::vector<uint8_t> data;
+  bool isKeyFrame = false;
+  uint64_t sequence = 0;
+  // Microseconds since the stream started.
+  int64_t timestampMicros = 0;
+};
+
+// Polls the live display IOSurface (sim_screenshot.h) on a serial queue and encodes changed
+// frames via the public VideoToolbox API — unlike StartVideoRecording's private, file-only
+// recorder, this delivers access units live. Full thread-safety contract: see CLAUDE.md.
+class VideoStreamSession {
+ public:
+  // `onAbortDelivery`, if set, is invoked by AbortDelivery() below — not called by this class on
+  // its own.
+  VideoStreamSession(id device, VideoEncoderOptions options, std::function<void(VideoAccessUnit)> onAccessUnit,
+                     std::function<void(NSError*)> onError, std::function<void()> onEnd,
+                     std::function<void()> onAbortDelivery = nullptr);
+  ~VideoStreamSession();
+
+  VideoStreamSession(const VideoStreamSession&) = delete;
+  VideoStreamSession& operator=(const VideoStreamSession&) = delete;
+
+  // Resolves the display and starts the polling loop; throws synchronously on resolution/setup
+  // failure (`onEnd` never called then). Later failures go to `onError`, then `onEnd`.
+  void Start();
+
+  // Idempotent; blocks until the loop has fully stopped. Never call from inside onAccessUnit/
+  // onError/onEnd — same queue this blocks on, so it would deadlock.
+  void Stop();
+
+  // Runs the constructor's `onAbortDelivery` (if any) — e.g. aborting a bounded delivery queue so
+  // a producer thread blocked pushing into it unblocks. Call before Stop() when the caller can't
+  // rely on anything else draining that queue concurrently (e.g. process-exit cleanup running
+  // synchronously on the same thread `Stop()` would otherwise wait on) — a normal Stop() doesn't
+  // need this. Safe from any thread; a no-op if unset.
+  void AbortDelivery();
+
+  // Forces the next encoded frame to be a keyframe (self-decodable, parameter sets included) —
+  // e.g. so a consumer that just resynced after dropping frames can resume cleanly instead of
+  // waiting for the next periodic one. Safe from any thread; just sets a flag.
+  void RequestKeyFrame();
+
+ private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace coresim

package/src/native/sim_video_stream.mm

@@ -0,0 +1,71 @@
+#include "sim_video_stream.h"
+
+#include <atomic>
+
+#include "video_encoder.h"
+
+namespace coresim {
+
+class VideoStreamSession::Impl {
+ public:
+  Impl(id device, VideoEncoderOptions options, std::function<void(VideoAccessUnit)> onAccessUnit,
+       std::function<void(NSError*)> onError, std::function<void()> onEnd, std::function<void()> onAbortDelivery)
+      : options_(options), onAccessUnit_(std::move(onAccessUnit)), onAbortDelivery_(std::move(onAbortDelivery)) {
+    encoder_ = std::make_unique<VideoFrameEncoder>(
+        device, options, [this](CMSampleBufferRef sampleBuffer) { HandleEncodedSample(sampleBuffer); },
+        std::move(onError), std::move(onEnd));
+  }
+
+  void Start() { encoder_->Start(); }
+  void Stop() { encoder_->Stop(); }
+  void AbortDelivery() {
+    if (onAbortDelivery_) {
+      onAbortDelivery_();
+    }
+  }
+  void RequestKeyFrame() { encoder_->RequestKeyFrame(); }
+
+ private:
+  void HandleEncodedSample(CMSampleBufferRef sampleBuffer) {
+    bool isKeyFrame = IsKeyFrame(sampleBuffer);
+    // From the sample's own presentation timestamp (as submitted to the encoder), not a fresh
+    // wall-clock read here — this callback can fire well after encoding actually happened, and
+    // resampling "now" would report a later, jittery timestamp than when the frame was captured.
+    CMTime pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
+    VideoAccessUnit unit;
+    unit.isKeyFrame = isKeyFrame;
+    unit.sequence = sequence_++;
+    unit.timestampMicros = pts.timescale != 0 ? (pts.value * 1000000 / pts.timescale) : 0;
+    RepackAsAnnexB(unit.data, sampleBuffer, isKeyFrame, options_.codec);
+    if (onAccessUnit_) {
+      onAccessUnit_(std::move(unit));
+    }
+  }
+
+  VideoEncoderOptions options_;
+  std::function<void(VideoAccessUnit)> onAccessUnit_;
+  std::function<void()> onAbortDelivery_;
+  std::unique_ptr<VideoFrameEncoder> encoder_;
+  // VideoToolbox's output callback isn't documented as single-threaded, so this is read-modify-
+  // written atomically rather than assuming HandleEncodedSample never runs concurrently.
+  std::atomic<uint64_t> sequence_{0};
+};
+
+VideoStreamSession::VideoStreamSession(id device, VideoEncoderOptions options,
+                                       std::function<void(VideoAccessUnit)> onAccessUnit,
+                                       std::function<void(NSError*)> onError, std::function<void()> onEnd,
+                                       std::function<void()> onAbortDelivery)
+    : impl_(std::make_unique<Impl>(device, options, std::move(onAccessUnit), std::move(onError), std::move(onEnd),
+                                   std::move(onAbortDelivery))) {}
+
+VideoStreamSession::~VideoStreamSession() = default;
+
+void VideoStreamSession::Start() { impl_->Start(); }
+
+void VideoStreamSession::Stop() { impl_->Stop(); }
+
+void VideoStreamSession::AbortDelivery() { impl_->AbortDelivery(); }
+
+void VideoStreamSession::RequestKeyFrame() { impl_->RequestKeyFrame(); }
+
+}  // namespace coresim

package/src/native/video_encoder.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#import <CoreMedia/CoreMedia.h>
+#import <Foundation/Foundation.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <vector>
+
+namespace coresim {
+
+enum class VideoStreamCodec { kH264, kHEVC };
+
+struct VideoEncoderOptions {
+  VideoStreamCodec codec = VideoStreamCodec::kH264;
+  NSString* displayId = nil;
+  double fps = 15.0;
+  int bitrate = 2000000;
+};
+
+// Polls the live display IOSurface (sim_screenshot.h) on a serial queue and encodes changed
+// frames via the public VideoToolbox API, delivering each encoded `CMSampleBufferRef` live.
+// Extracted from VideoStreamSession (video_stream.h) so both it and the combined AV
+// recording/streaming sessions (av_recording.h/av_stream.h) share one VTCompressionSession driver
+// instead of each re-deriving IOSurface-poll + compression-session setup from scratch. Callers
+// that need Annex-B-framed access units (VideoAccessUnit's wire format) repack the delivered
+// sample buffers themselves (see RepackAsAnnexB below) — this class only drives the encoder.
+class VideoFrameEncoder {
+ public:
+  // `onSample`'s CMSampleBufferRef is only valid for the duration of the call (VideoToolbox owns
+  // it) — a caller that needs the data beyond that must copy it out (or CFRetain it) before
+  // returning, never stash the raw pointer.
+  //
+  // `sharedClockOrigin`, if non-null, is used as this encoder's PTS-zero instant (from
+  // monotonic_clock.h) instead of capturing its own at Start() — pass the same pointer to an
+  // AudioEncoder started around the same time so both tracks' presentation timestamps measure
+  // elapsed time from one common reference (av_recording.h/av_stream.h do this; the standalone
+  // VideoStreamSession, video-only, has no such peer and leaves this null).
+  VideoFrameEncoder(id device, VideoEncoderOptions options, std::function<void(CMSampleBufferRef)> onSample,
+                    std::function<void(NSError*)> onError, std::function<void()> onEnd,
+                    const double* sharedClockOrigin = nullptr);
+  ~VideoFrameEncoder();
+
+  VideoFrameEncoder(const VideoFrameEncoder&) = delete;
+  VideoFrameEncoder& operator=(const VideoFrameEncoder&) = delete;
+
+  // Resolves the display and starts the polling loop; throws synchronously on resolution/setup
+  // failure (`onEnd` never called then). Later failures go to `onError`, then `onEnd`.
+  void Start();
+
+  // Idempotent; blocks until the loop has fully stopped. Never call from inside onSample/onError/
+  // onEnd — same queue this blocks on, so it would deadlock.
+  void Stop();
+
+  // Forces the next encoded frame to be a keyframe (self-decodable, parameter sets included) —
+  // e.g. so a consumer that just resynced after dropping frames can resume cleanly instead of
+  // waiting for the next periodic one. Safe from any thread; just sets a flag.
+  void RequestKeyFrame();
+
+ private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+// Whether `sampleBuffer` is a sync (key) frame, from its sample attachments.
+bool IsKeyFrame(CMSampleBufferRef sampleBuffer);
+
+// VideoToolbox's compressed output is AVCC-framed (a 4-byte big-endian length prefix per NAL, no
+// start codes) — rewrites `sampleBuffer` into Annex-B, appending to `out`. A keyframe's format
+// description (SPS/PPS, or VPS/SPS/PPS for HEVC) is prepended first when `isKeyFrame`, so every
+// keyframe is self-decodable alone. Shared by VideoStreamSession and the AV streaming path so both
+// produce byte-identical framing for the same encoder output.
+void RepackAsAnnexB(std::vector<uint8_t>& out, CMSampleBufferRef sampleBuffer, bool isKeyFrame, VideoStreamCodec codec);
+
+}  // namespace coresim