@appium/coresim 1.5.0 → 1.7.0

package/src/native/video_encoder.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#import <CoreMedia/CoreMedia.h>
+#import <Foundation/Foundation.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <vector>
+
+namespace coresim {
+
+enum class VideoStreamCodec { kH264, kHEVC };
+
+struct VideoEncoderOptions {
+  VideoStreamCodec codec = VideoStreamCodec::kH264;
+  NSString* displayId = nil;
+  double fps = 15.0;
+  int bitrate = 2000000;
+};
+
+// Polls the live display IOSurface (sim_screenshot.h) on a serial queue and encodes changed
+// frames via the public VideoToolbox API, delivering each encoded `CMSampleBufferRef` live.
+// Extracted from VideoStreamSession (video_stream.h) so both it and the combined AV
+// recording/streaming sessions (av_recording.h/av_stream.h) share one VTCompressionSession driver
+// instead of each re-deriving IOSurface-poll + compression-session setup from scratch. Callers
+// that need Annex-B-framed access units (VideoAccessUnit's wire format) repack the delivered
+// sample buffers themselves (see RepackAsAnnexB below) — this class only drives the encoder.
+class VideoFrameEncoder {
+ public:
+  // `onSample`'s CMSampleBufferRef is only valid for the duration of the call (VideoToolbox owns
+  // it) — a caller that needs the data beyond that must copy it out (or CFRetain it) before
+  // returning, never stash the raw pointer.
+  //
+  // `sharedClockOrigin`, if non-null, is used as this encoder's PTS-zero instant (from
+  // monotonic_clock.h) instead of capturing its own at Start() — pass the same pointer to an
+  // AudioEncoder started around the same time so both tracks' presentation timestamps measure
+  // elapsed time from one common reference (av_recording.h/av_stream.h do this; the standalone
+  // VideoStreamSession, video-only, has no such peer and leaves this null).
+  VideoFrameEncoder(id device, VideoEncoderOptions options, std::function<void(CMSampleBufferRef)> onSample,
+                    std::function<void(NSError*)> onError, std::function<void()> onEnd,
+                    const double* sharedClockOrigin = nullptr);
+  ~VideoFrameEncoder();
+
+  VideoFrameEncoder(const VideoFrameEncoder&) = delete;
+  VideoFrameEncoder& operator=(const VideoFrameEncoder&) = delete;
+
+  // Resolves the display and starts the polling loop; throws synchronously on resolution/setup
+  // failure (`onEnd` never called then). Later failures go to `onError`, then `onEnd`.
+  void Start();
+
+  // Idempotent; blocks until the loop has fully stopped. Never call from inside onSample/onError/
+  // onEnd — same queue this blocks on, so it would deadlock.
+  void Stop();
+
+  // Forces the next encoded frame to be a keyframe (self-decodable, parameter sets included) —
+  // e.g. so a consumer that just resynced after dropping frames can resume cleanly instead of
+  // waiting for the next periodic one. Safe from any thread; just sets a flag.
+  void RequestKeyFrame();
+
+ private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+// Whether `sampleBuffer` is a sync (key) frame, from its sample attachments.
+bool IsKeyFrame(CMSampleBufferRef sampleBuffer);
+
+// VideoToolbox's compressed output is AVCC-framed (a 4-byte big-endian length prefix per NAL, no
+// start codes) — rewrites `sampleBuffer` into Annex-B, appending to `out`. A keyframe's format
+// description (SPS/PPS, or VPS/SPS/PPS for HEVC) is prepended first when `isKeyFrame`, so every
+// keyframe is self-decodable alone. Shared by VideoStreamSession and the AV streaming path so both
+// produce byte-identical framing for the same encoder output.
+void RepackAsAnnexB(std::vector<uint8_t>& out, CMSampleBufferRef sampleBuffer, bool isKeyFrame, VideoStreamCodec codec);
+
+}  // namespace coresim

package/src/native/video_encoder.mm

@@ -0,0 +1,417 @@
+#include "video_encoder.h"
+
+#import <CoreVideo/CoreVideo.h>
+#import <IOSurface/IOSurface.h>
+#import <VideoToolbox/VideoToolbox.h>
+
+#include <algorithm>
+#include <atomic>
+
+#include "monotonic_clock.h"
+#include "nserror_bridge.h"
+#include "safe_dispatch.h"
+#include "sim_screenshot.h"
+
+namespace coresim {
+
+namespace {
+
+NSString* const kVideoEncoderErrorDomain = @"com.appium.coresim.VideoEncoder";
+
+NSError* MakeError(NSInteger code, NSString* message) {
+  return [NSError errorWithDomain:kVideoEncoderErrorDomain code:code userInfo:@{NSLocalizedDescriptionKey : message}];
+}
+
+NSError* MakeStatusError(NSInteger code, NSString* what, OSStatus status) {
+  return MakeError(code, [NSString stringWithFormat:@"%@ (OSStatus %d)", what, static_cast<int>(status)]);
+}
+
+void AppendAnnexB(std::vector<uint8_t>& out, const uint8_t* nal, size_t length) {
+  static const uint8_t kStartCode[4] = {0, 0, 0, 1};
+  out.insert(out.end(), kStartCode, kStartCode + 4);
+  out.insert(out.end(), nal, nal + length);
+}
+
+// CMBlockBufferGetDataPointer's pointer only covers the contiguous region starting at the given
+// offset, which for a segmented buffer (multiple backing memory blocks — CoreMedia's documented
+// contract, not just a VideoToolbox implementation detail) can be far shorter than totalLength;
+// reading up to totalLength through it would run past that region. CopyDataBytes stitches
+// segments together into a caller-owned, guaranteed-contiguous copy instead.
+void AppendSampleBufferNALs(std::vector<uint8_t>& out, CMSampleBufferRef sampleBuffer) {
+  CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sampleBuffer);
+  if (block == nullptr) {
+    return;
+  }
+  size_t totalLength = CMBlockBufferGetDataLength(block);
+  if (totalLength == 0) {
+    return;
+  }
+  std::vector<uint8_t> data(totalLength);
+  if (CMBlockBufferCopyDataBytes(block, 0, totalLength, data.data()) != kCMBlockBufferNoErr) {
+    return;
+  }
+  const uint8_t* dataPointer = data.data();
+  size_t offset = 0;
+  while (offset + 4 <= totalLength) {
+    uint32_t nalLength = (static_cast<uint32_t>(dataPointer[offset]) << 24) |
+                         (static_cast<uint32_t>(dataPointer[offset + 1]) << 16) |
+                         (static_cast<uint32_t>(dataPointer[offset + 2]) << 8) | dataPointer[offset + 3];
+    offset += 4;
+    if (nalLength == 0 || offset + nalLength > totalLength) {
+      break;
+    }
+    AppendAnnexB(out, dataPointer + offset, nalLength);
+    offset += nalLength;
+  }
+}
+
+using ParameterSetAtIndexFn = OSStatus (*)(CMFormatDescriptionRef, size_t, const uint8_t**, size_t*, size_t*, int*);
+
+void AppendParameterSets(std::vector<uint8_t>& out, CMFormatDescriptionRef format, ParameterSetAtIndexFn getAtIndex) {
+  size_t count = 0;
+  if (getAtIndex(format, 0, nullptr, nullptr, &count, nullptr) != noErr) {
+    return;
+  }
+  for (size_t i = 0; i < count; i++) {
+    const uint8_t* bytes = nullptr;
+    size_t size = 0;
+    if (getAtIndex(format, i, &bytes, &size, nullptr, nullptr) == noErr) {
+      AppendAnnexB(out, bytes, size);
+    }
+  }
+}
+
+}  // namespace
+
+bool IsKeyFrame(CMSampleBufferRef sampleBuffer) {
+  CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, false);
+  if (attachments == nullptr || CFArrayGetCount(attachments) == 0) {
+    return true;
+  }
+  CFDictionaryRef attachment = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
+  return !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
+}
+
+void RepackAsAnnexB(std::vector<uint8_t>& out, CMSampleBufferRef sampleBuffer, bool isKeyFrame,
+                    VideoStreamCodec codec) {
+  if (isKeyFrame) {
+    CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer);
+    if (format != nullptr) {
+      if (codec == VideoStreamCodec::kHEVC) {
+        AppendParameterSets(out, format, CMVideoFormatDescriptionGetHEVCParameterSetAtIndex);
+      } else {
+        AppendParameterSets(out, format, CMVideoFormatDescriptionGetH264ParameterSetAtIndex);
+      }
+    }
+  }
+  AppendSampleBufferNALs(out, sampleBuffer);
+}
+
+class VideoFrameEncoder::Impl {
+ public:
+  Impl(id device, VideoEncoderOptions options, std::function<void(CMSampleBufferRef)> onSample,
+       std::function<void(NSError*)> onError, std::function<void()> onEnd, const double* sharedClockOrigin)
+      : device_(device),
+        options_(options),
+        onSample_(std::move(onSample)),
+        onError_(std::move(onError)),
+        onEnd_(std::move(onEnd)),
+        sharedClockOrigin_(sharedClockOrigin) {
+    queue_ = dispatch_queue_create("com.appium.coresim.videoEncoder", DISPATCH_QUEUE_SERIAL);
+  }
+
+  ~Impl() { Stop(); }
+
+  void Start() {
+    NSError* error = nil;
+    id descriptor = ResolveCaptureDisplay(device_, options_.displayId, &error);
+    if (descriptor == nil) {
+      throw NSErrorException(error);
+    }
+    id surfaceObj = CurrentDisplaySurface(descriptor);
+    if (surfaceObj == nil) {
+      throw NSErrorException(MakeError(4, @"The device's display surface is not available yet"));
+    }
+    IOSurfaceRef surface = (__bridge IOSurfaceRef)surfaceObj;
+    // Set up synchronously (not lazily on the first Tick()) so a setup failure rejects Start()
+    // directly rather than only reaching onError, which the caller may not be listening for yet.
+    NSError* setupError = nil;
+    if (!SetUpSession(surface, &setupError)) {
+      throw NSErrorException(setupError);
+    }
+    // Must be set before EncodeSurface below — both it and HandleEncodedSample measure elapsed
+    // time from this. A caller-supplied origin (see the constructor) is used as-is, not offset
+    // further — the gap between it being captured and this line running is itself the correct,
+    // meaningful startup latency to bake into this encoder's PTS zero, for sync with a peer
+    // AudioEncoder given the same origin (av_recording.h/av_stream.h).
+    startTime_ = sharedClockOrigin_ != nullptr ? *sharedClockOrigin_ : MonotonicSeconds();
+    // Encode immediately rather than waiting for a *changed* seed on the first tick, or the
+    // stream would stay silent until the display changes again. running_ is set true before this
+    // call (not after), since VTCompressionSessionEncodeFrame's output callback can in principle
+    // fire on another thread before this one returns — HandleEncodedSample discards samples while
+    // running_ is false, which would otherwise silently drop the stream's very first (keyframe)
+    // sample. A failure resets it and tears session_ down itself here, rather than going through
+    // Stop()/onEnd_ (see coresim.mm — onEnd_ firing this early would double-release its
+    // ThreadSafeFunctions).
+    running_ = true;
+    bool encoded = false;
+    try {
+      encoded = EncodeSurface(surface);
+    } catch (...) {
+      running_ = false;
+      VTCompressionSessionInvalidate(session_);
+      CFRelease(session_);
+      session_ = nullptr;
+      throw;
+    }
+    // Only commit the seed once a frame was actually submitted — a transient pixel-buffer
+    // creation failure (EncodeSurface returning false) otherwise leaves lastSeed_ at its default
+    // 0, so the first Tick() sees the real seed as "changed" and retries automatically instead of
+    // the stream going silent forever on a display that never changes again.
+    if (encoded) {
+      lastSeed_ = IOSurfaceGetSeed(surface);
+    }
+
+    double interval = 1.0 / std::max(options_.fps, 1.0);
+    dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, queue_);
+    dispatch_source_set_timer(timer, dispatch_time(DISPATCH_TIME_NOW, 0),
+                              static_cast<uint64_t>(interval * NSEC_PER_SEC),
+                              static_cast<uint64_t>(interval * NSEC_PER_SEC / 10));
+    // `this` outlives the timer: Stop()/StopFromQueue() always drain or outrun it before `this`
+    // can be destroyed (see their comments below).
+    dispatch_source_set_event_handler(timer, ^{
+      Tick();
+    });
+    timer_ = timer;
+    dispatch_resume(timer_);
+  }
+
+  // Callable from any thread except `queue_` itself (would deadlock on the dispatch_sync below).
+  void Stop() {
+    if (!running_.exchange(false)) {
+      return;  // idempotent
+    }
+    if (timer_ != nullptr) {
+      dispatch_source_cancel(timer_);
+      // Blocks until any in-flight Tick() finishes — by then running_ is already false, so it
+      // won't touch session_ again.
+      dispatch_sync(queue_, ^{
+                    });
+      timer_ = nullptr;
+    }
+    TearDownSessionAndFireEnd();
+  }
+
+  void RequestKeyFrame() { forceKeyFrame_ = true; }
+
+ private:
+  // Same as Stop() minus the dispatch_sync barrier — only safe from within Tick() itself, already
+  // serialized on `queue_`; would race a concurrent Tick() from any other thread.
+  void StopFromQueue() {
+    if (!running_.exchange(false)) {
+      return;  // idempotent — e.g. an external Stop() already won this race
+    }
+    if (timer_ != nullptr) {
+      dispatch_source_cancel(timer_);
+      timer_ = nullptr;
+    }
+    TearDownSessionAndFireEnd();
+  }
+
+  void TearDownSessionAndFireEnd() {
+    if (session_ != nullptr) {
+      // Flushes and blocks until every already-submitted frame's callback has returned — without
+      // this, a frame submitted just before Stop() could fire after onEnd_ releases whatever
+      // resources the caller tied to it (e.g. ThreadSafeFunctions — see CLAUDE.md).
+      VTCompressionSessionCompleteFrames(session_, kCMTimeInvalid);
+      VTCompressionSessionInvalidate(session_);
+      CFRelease(session_);
+      session_ = nullptr;
+    }
+    if (onEnd_) {
+      onEnd_();
+    }
+  }
+
+  void Tick() {
+    if (!running_) {
+      return;
+    }
+    if (pendingErrorTeardown_) {
+      // HandleEncodedSample (below) can run on VideoToolbox's own callback thread, where it's
+      // unsafe to tear down directly — StopFromQueue()/CompleteFrames() are only safe already
+      // serialized on `queue_` (here), and CompleteFrames specifically would deadlock waiting on
+      // its own still-executing callback. It sets this flag instead; picked up on the very next
+      // tick (queue_-serialized, safe) rather than via a raw cross-thread dispatch, since nothing
+      // here can outlive `this` the way a block captured on another thread otherwise could.
+      StopFromQueue();
+      return;
+    }
+    @autoreleasepool {
+      try {
+        // Re-resolved every tick (like CaptureScreenshot does), not cached once in Start(), so a
+        // deleted device or disconnected display surfaces a real error instead of Tick() quietly
+        // doing nothing forever.
+        NSError* resolveError = nil;
+        id descriptor = ResolveCaptureDisplay(device_, options_.displayId, &resolveError);
+        if (descriptor == nil) {
+          if (onError_) {
+            onError_(resolveError);
+          }
+          StopFromQueue();
+          return;
+        }
+        id surfaceObj = CurrentDisplaySurface(descriptor);
+        if (surfaceObj == nil) {
+          return;  // transient — the connection may not have a frame ready yet, try again next tick
+        }
+        IOSurfaceRef surface = (__bridge IOSurfaceRef)surfaceObj;
+        uint32_t seed = IOSurfaceGetSeed(surface);
+        if (seed == lastSeed_) {
+          return;  // unchanged since the last tick — mirrors CoreSimulator's own recorder, which
+                   // only encodes a frame when the display actually changes (see CLAUDE.md)
+        }
+        // Only commit the new seed once EncodeSurface actually submits it — a transient failure
+        // (pixel-buffer creation) must leave lastSeed_ stale so the next tick retries this same
+        // frame instead of silently going quiet until the display changes again.
+        if (EncodeSurface(surface)) {
+          lastSeed_ = seed;
+        }
+      } catch (const std::exception& e) {
+        // Without this, an exception here (e.g. a dropped display-proxy connection) would escape
+        // this bare GCD timer handler uncaught and crash the whole process (see CLAUDE.md).
+        if (onError_) {
+          onError_(MakeError(3, [NSString stringWithFormat:@"Video encoding failed: %s", e.what()]));
+        }
+        StopFromQueue();
+      }
+    }
+  }
+
+  bool SetUpSession(IOSurfaceRef surface, NSError** error) {
+    int32_t width = static_cast<int32_t>(IOSurfaceGetWidth(surface));
+    int32_t height = static_cast<int32_t>(IOSurfaceGetHeight(surface));
+    CMVideoCodecType codecType =
+        options_.codec == VideoStreamCodec::kHEVC ? kCMVideoCodecType_HEVC : kCMVideoCodecType_H264;
+    OSStatus status = VTCompressionSessionCreate(kCFAllocatorDefault, width, height, codecType, nullptr, nullptr,
+                                                 kCFAllocatorDefault, OutputCallback, this, &session_);
+    if (status != noErr) {
+      *error = MakeStatusError(1, @"Failed to create a VTCompressionSession", status);
+      return false;
+    }
+    // Clamped like Start()'s timer interval — an unvalidated 0 here would set MaxKeyFrameInterval
+    // to an out-of-spec value.
+    double fps = std::max(options_.fps, 1.0);
+    status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
+    if (status == noErr) {
+      status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse);
+    }
+    if (status == noErr) {
+      status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_AverageBitRate,
+                                    (__bridge CFNumberRef) @(options_.bitrate));
+    }
+    if (status == noErr) {
+      status =
+          VTSessionSetProperty(session_, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFNumberRef) @(fps));
+    }
+    if (status == noErr) {
+      status = VTSessionSetProperty(session_, kVTCompressionPropertyKey_MaxKeyFrameInterval,
+                                    (__bridge CFNumberRef) @(static_cast<int>(fps * 2)));
+    }
+    if (status != noErr) {
+      *error = MakeStatusError(2, @"Failed to configure the VTCompressionSession", status);
+      VTCompressionSessionInvalidate(session_);
+      CFRelease(session_);
+      session_ = nullptr;
+      return false;
+    }
+    VTCompressionSessionPrepareToEncodeFrames(session_);
+    return true;
+  }
+
+  // Returns whether a frame was actually submitted to the encoder — false for a transient
+  // pixel-buffer creation failure the caller should retry, as opposed to a real encode failure
+  // (thrown, not returned, since that tears down the whole session).
+  bool EncodeSurface(IOSurfaceRef surface) {
+    CVPixelBufferRef pixelBuffer = nullptr;
+    CVReturn cvStatus = CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, surface, nullptr, &pixelBuffer);
+    if (cvStatus != kCVReturnSuccess || pixelBuffer == nullptr) {
+      return false;  // transient — try again next tick rather than tearing down the whole session
+    }
+    CMTime pts = CMTimeMake(static_cast<int64_t>((MonotonicSeconds() - startTime_) * 1000000), 1000000);
+    NSDictionary* frameProperties = nil;
+    if (forceKeyFrame_.exchange(false)) {
+      frameProperties = @{(__bridge NSString*)kVTEncodeFrameOptionKey_ForceKeyFrame : @YES};
+    }
+    OSStatus status = VTCompressionSessionEncodeFrame(session_, pixelBuffer, pts, kCMTimeInvalid,
+                                                      (__bridge CFDictionaryRef)frameProperties, nullptr, nullptr);
+    CVPixelBufferRelease(pixelBuffer);
+    if (status != noErr) {
+      throw std::runtime_error([[NSString stringWithFormat:@"VTCompressionSessionEncodeFrame failed (OSStatus %d)",
+                                                           static_cast<int>(status)] UTF8String]);
+    }
+    return true;
+  }
+
+  static void OutputCallback(void* outputCallbackRefCon, void* /*sourceFrameRefCon*/, OSStatus status,
+                             VTEncodeInfoFlags /*infoFlags*/, CMSampleBufferRef sampleBuffer) {
+    static_cast<Impl*>(outputCallbackRefCon)->HandleEncodedSample(status, sampleBuffer);
+  }
+
+  void HandleEncodedSample(OSStatus status, CMSampleBufferRef sampleBuffer) {
+    if (!running_) {
+      return;
+    }
+    if (status != noErr) {
+      // Only the first failure is reported — pendingErrorTeardown_ doubles as the report-once
+      // gate, since Tick() (the only place that consumes it) only ever needs to see it once too.
+      if (!pendingErrorTeardown_.exchange(true)) {
+        if (onError_) {
+          onError_(MakeStatusError(3, @"VideoToolbox reported an encoding failure", status));
+        }
+      }
+      return;
+    }
+    if (sampleBuffer == nullptr) {
+      return;
+    }
+    if (onSample_) {
+      onSample_(sampleBuffer);
+    }
+  }
+
+  id device_;
+  VideoEncoderOptions options_;
+  std::function<void(CMSampleBufferRef)> onSample_;
+  std::function<void(NSError*)> onError_;
+  std::function<void()> onEnd_;
+  const double* sharedClockOrigin_;
+
+  dispatch_queue_t queue_ = nullptr;
+  dispatch_source_t timer_ = nullptr;
+  VTCompressionSessionRef session_ = nullptr;
+  uint32_t lastSeed_ = 0;
+  double startTime_ = 0;
+  std::atomic<bool> running_{false};
+  // Set by HandleEncodedSample (possibly off queue_) on an encoder failure, consumed by the next
+  // Tick() (on queue_) — see both for why teardown can't just happen inline there.
+  std::atomic<bool> pendingErrorTeardown_{false};
+  std::atomic<bool> forceKeyFrame_{false};
+};
+
+VideoFrameEncoder::VideoFrameEncoder(id device, VideoEncoderOptions options,
+                                     std::function<void(CMSampleBufferRef)> onSample,
+                                     std::function<void(NSError*)> onError, std::function<void()> onEnd,
+                                     const double* sharedClockOrigin)
+    : impl_(std::make_unique<Impl>(device, options, std::move(onSample), std::move(onError), std::move(onEnd),
+                                   sharedClockOrigin)) {}
+
+VideoFrameEncoder::~VideoFrameEncoder() = default;
+
+void VideoFrameEncoder::Start() { impl_->Start(); }
+
+void VideoFrameEncoder::Stop() { impl_->Stop(); }
+
+void VideoFrameEncoder::RequestKeyFrame() { impl_->RequestKeyFrame(); }
+
+}  // namespace coresim

package/src/native-simctl.ts

@@ -281,7 +281,18 @@ const loadNative = util.memoize(function loadNative(): NativeCoreSimModule {
       'n/a',
     );
   }
-  return require('node-gyp-build')(getPkgRoot()) as NativeCoreSimModule;
+  const native = require('node-gyp-build')(getPkgRoot()) as NativeCoreSimModule;
+  // A forgotten (never explicitly stopped) AV recording/stream otherwise silently loses data —
+  // or just leaks a live encoder — the instant a caller force-exits via `process.exit()`, since
+  // Node's own cleanup hooks (coresim.mm's CleanupActiveSessions, registered against the exact
+  // same condition) are confirmed to NOT run in that path, only on a natural empty-event-loop
+  // exit or a Worker's own termination. `process.on('exit', ...)` does fire for `process.exit()`
+  // too, and (per Node's own contract) may run synchronous code — flushActiveSessions() qualifies:
+  // it's a single blocking native call, not new async JS work. Registered once, lazily, here
+  // rather than at module import time, so merely importing this package on a non-macOS platform
+  // never touches `process` for something it'll never need.
+  process.on('exit', () => native.flushActiveSessions());
+  return native;
 });
 
 const DEFAULT_DEVELOPER_DIR_TIMEOUT_MS = 15_000;

package/src/types.ts

@@ -140,9 +140,44 @@ export interface VideoRecordingOptions {
   /**
    * For a non-rectangular display (e.g. a Dynamic Island cutout): `'ignored'` (default) saves the
    * unmasked framebuffer, `'black'` renders the mask black, `'alpha'` is not supported and
-   * behaves like `'black'`.
+   * behaves like `'black'`. Only applies when neither `audio` nor `fps` is set — see their doc
+   * comments.
    */
   mask?: 'ignored' | 'alpha' | 'black';
+  /**
+   * Also capture the device's audio into the same file, muxed as a second track. Defaults to
+   * `false`. Requires macOS 14.2+ (Core Audio process taps), the host's "System Audio Recording
+   * Only" privacy permission (System Settings > Privacy & Security — cannot be granted
+   * programmatically; a denial isn't a thrown error, it surfaces as a silent, audio-less/near-
+   * silent recording), a default audio output device on the host, and a booted device that has
+   * produced audio at least once. See the README's "Screen capture" section for the full
+   * requirements list and known failure modes.
+   *
+   * Like an explicit `fps`, this switches the implementation to this addon's own VideoToolbox +
+   * Core Audio encoders instead of CoreSimulator's private recorder, which can't mux audio.
+   */
+  audio?: boolean;
+  /**
+   * Max frames/sec to poll the framebuffer at — see {@link VideoStreamOptions} `fps` for the
+   * identical semantics. Meaningless against CoreSimulator's private recorder (it captures on its
+   * own cadence, not one we poll), so setting `fps` — even without `audio` — switches this
+   * recording to the same own-encoder implementation `audio` does. That switch costs `mask`
+   * support, which only the private recorder implements.
+   */
+  fps?: number;
+  /** Target average bitrate, in bits/sec. Respected on either implementation. */
+  bitrate?: number;
+}
+
+/** Options for `NativeSimctl.stopVideoRecording`. */
+export interface StopVideoRecordingOptions {
+  /**
+   * Best-effort: still attempts the native stop, but releases this device's tracked-active-
+   * recording bookkeeping regardless of whether that attempt succeeds, instead of leaving it
+   * retryable — see the doc comment above `NativeSimctl.stopVideoRecording` for when a caller
+   * needs this over a plain retry.
+   */
+  force?: boolean;
 }
 
 /** Options for `NativeSimctl.startVideoStream`. */
@@ -161,18 +196,30 @@ export interface VideoStreamOptions {
   fps?: number;
   /** Target average bitrate, in bits/sec. Defaults to 2,000,000 (2 Mbps). */
   bitrate?: number;
+  /**
+   * Also stream the device's audio, interleaved into the same `accessUnits()` sequence. Defaults
+   * to `false`. Same requirements and failure modes as {@link VideoRecordingOptions.audio} — see
+   * its doc comment and the README's "Screen capture" section.
+   */
+  audio?: boolean;
 }
 
 /**
- * One encoded frame from `VideoStream.accessUnits()` — Annex-B NAL units. A keyframe's `data` has
- * parameter sets (SPS/PPS, or VPS/SPS/PPS for HEVC) prepended, so it's self-decodable alone.
+ * One encoded unit from `VideoStream.accessUnits()`, discriminated by `track`: a video unit
+ * (Annex-B NAL units — a keyframe's `data` has parameter sets, SPS/PPS or VPS/SPS/PPS for HEVC,
+ * prepended, so it's self-decodable alone) or, when {@link VideoStreamOptions.audio} was set, an
+ * interleaved audio unit (an ADTS-framed AAC-LC packet — the 7-byte ADTS header carries sample
+ * rate/channel count itself, so no separate decoder-config exchange is needed; always
+ * independently decodable, so `isKeyFrame` is always `true`). Without `audio`, every unit has
+ * `track: 'video'`.
  */
 export interface VideoAccessUnit {
+  track: 'video' | 'audio';
   data: Buffer;
   isKeyFrame: boolean;
-  /** Monotonically increasing per stream, starting at 0. */
+  /** Monotonically increasing per track, starting at 0 — independent between `'video'` and `'audio'`. */
   sequence: number;
-  /** Microseconds since the stream started. */
+  /** Microseconds since the stream started, on one shared clock across both tracks. */
   timestampMicros: number;
 }
 
@@ -280,6 +327,7 @@ export type NativeSpawnExitCallback = (code: number | null, signal: number | nul
 
 /** Raw shape of an access unit as the native addon delivers it — see {@link VideoAccessUnit}. */
 export interface NativeVideoAccessUnit {
+  track: 'video' | 'audio';
   data: Buffer;
   isKeyFrame: boolean;
   sequence: number;
@@ -289,13 +337,29 @@ export interface NativeVideoAccessUnit {
 export type NativeVideoAccessUnitCallback = (unit: NativeVideoAccessUnit) => void;
 export type NativeVideoErrorCallback = (err: Error) => void;
 
-/** A live encoder session, wrapped by `coresim.mm`'s `NativeVideoStream` — what `NativeDeviceHandle.startVideoStream()` resolves to. */
+/**
+ * A live encoder session, wrapped by `coresim.mm`'s `NativeVideoStream` (video only) or
+ * `NativeAVStream` (`audio: true` — see {@link VideoStreamOptions}) — either way, what
+ * `NativeDeviceHandle.startVideoStream()` resolves to; the two native wrapper classes expose the
+ * identical shape below, so callers never need to know which one they got.
+ */
 export interface NativeVideoStreamHandle {
   stop(): Promise<void>;
-  /** Forces the next encoded frame to be a keyframe — trivial in-memory flag, so synchronous. */
+  /** Forces the next encoded video frame to be a keyframe — trivial in-memory flag, so synchronous. */
   requestKeyFrame(): void;
 }
 
+/**
+ * A live recording, wrapped by `coresim.mm`'s `NativePrivateRecordingHandle` (video only,
+ * addressing CoreSimulator's own internally-tracked private recorder) or `NativeAVRecording`
+ * (`audio: true` — see {@link VideoRecordingOptions}, a real local resource with no server-side
+ * counterpart) — either way, what `NativeDeviceHandle.startVideoRecording()` resolves to.
+ */
+export interface NativeVideoRecordingHandle {
+  /** Resolves once the output file has been finalized on disk and is safe to read. */
+  stop(): Promise<void>;
+}
+
 /** A `SimDevice`, wrapped by `coresim.mm`'s `NativeDevice` — what `NativeSimctl`'s `_findDevice()` resolves to. */
 export interface NativeDeviceHandle {
   // Trivial in-memory accessors — kept synchronous on the native side (see coresim.mm), never a
@@ -344,13 +408,26 @@ export interface NativeDeviceHandle {
   getWebInspectorSocket(): Promise<string>;
   screenshot(options?: {format?: 'png' | 'jpeg'; displayId?: string; quality?: number}): Promise<Buffer>;
   getDisplays(): Promise<SimDisplayInfo[]>;
+  // `mask` only applies without `audio`/`fps`; `bitrate` applies either way — see
+  // VideoRecordingOptions's own doc comments for why. `onError` is only ever invoked on the
+  // `audio`/`fps` (own-encoder) path — a live mid-recording failure, which the private recorder
+  // has no channel to report.
   startVideoRecording(
     outputFile: string,
-    options?: {displayId?: string; codec?: 'h264' | 'hevc'; mask?: 'ignored' | 'alpha' | 'black'},
-  ): Promise<void>;
-  stopVideoRecording(): Promise<void>;
+    options:
+      | {
+          displayId?: string;
+          codec?: 'h264' | 'hevc';
+          mask?: 'ignored' | 'alpha' | 'black';
+          audio?: boolean;
+          fps?: number;
+          bitrate?: number;
+        }
+      | undefined,
+    onError: NativeVideoErrorCallback,
+  ): Promise<NativeVideoRecordingHandle>;
   startVideoStream(
-    options: {displayId?: string; codec?: 'h264' | 'hevc'; fps?: number; bitrate?: number} | undefined,
+    options: {displayId?: string; codec?: 'h264' | 'hevc'; fps?: number; bitrate?: number; audio?: boolean} | undefined,
     onAccessUnit: NativeVideoAccessUnitCallback,
     onError: NativeVideoErrorCallback,
   ): Promise<NativeVideoStreamHandle>;
@@ -376,4 +453,11 @@ export interface NativeServiceContextHandle {
 export interface NativeCoreSimModule {
   sharedServiceContext(developerDir: string): Promise<NativeServiceContextHandle>;
   frameworkVersion(): Promise<string>;
+  /**
+   * Synchronously (not a Promise) stops every still-live video/AV stream or AV recording, blocking
+   * until each has released its resources. Meant to be called from a `process.on('exit', ...)`
+   * listener (see native-simctl.ts) — cleanup hooks alone don't run under `process.exit()` on the
+   * main process/thread, only on a natural empty-event-loop exit or a Worker's own termination.
+   */
+  flushActiveSessions(): void;
 }