@appium/coresim 1.4.0 → 1.6.0

package/src/native/av_recording.mm

@@ -0,0 +1,424 @@
+#include "av_recording.h"
+
+#include <atomic>
+#include <mutex>
+#include <vector>
+
+#include "audio_encoder.h"
+#include "monotonic_clock.h"
+#include "nserror_bridge.h"
+#include "sim_audio_tap.h"
+
+namespace coresim {
+
+namespace {
+
+NSString* const kAVRecordingErrorDomain = @"com.appium.coresim.AVRecording";
+
+NSError* MakeError(NSInteger code, NSString* message) {
+  return [NSError errorWithDomain:kAVRecordingErrorDomain code:code userInfo:@{NSLocalizedDescriptionKey : message}];
+}
+
+constexpr size_t kMaxPendingAudioSamples = 500;  // a few seconds of AAC packets — see HandleAudioSample
+
+}  // namespace
+
+class AVRecordingSession::Impl {
+ public:
+  Impl(id device, NSString* udid, VideoEncoderOptions videoOptions, NSString* outputFile, bool captureAudio)
+      : device_(device),
+        udid_(udid),
+        videoOptions_(videoOptions),
+        outputFile_(outputFile),
+        captureAudio_(captureAudio) {}
+
+  ~Impl() { TearDownIfNeeded(); }
+
+  void Start(std::function<void()> onFirstSample, std::function<void(NSError*)> onError, std::function<void()> onEnd) {
+    onFirstSample_ = std::move(onFirstSample);
+    onError_ = std::move(onError);
+    onEnd_ = std::move(onEnd);
+    clockOrigin_ = MonotonicSeconds();
+
+    [[NSFileManager defaultManager] removeItemAtPath:outputFile_ error:nil];
+    NSError* writerError = nil;
+    writer_ = [AVAssetWriter assetWriterWithURL:[NSURL fileURLWithPath:outputFile_]
+                                       fileType:AVFileTypeMPEG4
+                                          error:&writerError];
+    if (writer_ == nil) {
+      throw NSErrorException(writerError ?: MakeError(1, @"Failed to create an AVAssetWriter"));
+    }
+
+    try {
+      if (captureAudio_) {
+        audioTap_ = std::make_unique<AudioTapSession>(
+            udid_, [this](const AudioBufferList* data, const AudioTimeStamp* time) { HandleAudioPCM(data, time); },
+            [this](NSError* error) {
+              // A failed process-list refresh (kAudioTapNonFatalProcessListRefreshErrorCode) isn't
+              // fatal — the tap keeps running. Everything else is.
+              if ([error.domain isEqualToString:kAudioTapErrorDomain] &&
+                  error.code == kAudioTapNonFatalProcessListRefreshErrorCode) {
+                return;
+              }
+              Fail(error, /*fromVideo=*/false);
+            },
+            [] {});
+        audioTap_->Start();
+
+        // Start() can invoke HandleAudioPCM before returning, so audioEncoder_ is written under
+        // mutex_ (same lock HandleAudioPCM reads it under) to avoid a torn/racy pointer read.
+        auto encoder = std::make_unique<AudioEncoder>(
+            audioTap_->Format(), [this](CMSampleBufferRef sampleBuffer) { HandleAudioSample(sampleBuffer); },
+            &clockOrigin_);
+        {
+          std::lock_guard<std::mutex> lock(mutex_);
+          audioEncoder_ = std::move(encoder);
+        }
+        // Audio's format is known immediately (unlike video's, learned from its first sample), so
+        // its input can be added to the writer right away — well before startWriting is called.
+        audioInput_ = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio
+                                                         outputSettings:nil
+                                                       sourceFormatHint:audioEncoder_->OutputFormatDescription()];
+        audioInput_.expectsMediaDataInRealTime = YES;
+        [writer_ addInput:audioInput_];
+      }
+
+      videoEncoder_ = std::make_unique<VideoFrameEncoder>(
+          device_, videoOptions_, [this](CMSampleBufferRef sampleBuffer) { HandleVideoSample(sampleBuffer); },
+          [this](NSError* error) { Fail(error, /*fromVideo=*/true); }, [] {}, &clockOrigin_);
+      videoEncoder_->Start();
+    } catch (...) {
+      TearDownIfNeeded();
+      throw;
+    }
+  }
+
+  // Must be fully idempotent, including concurrently: the env cleanup hook (coresim.mm) also
+  // unconditionally calls Stop() on every still-registered session, including one already stopped
+  // moments earlier whose JS wrapper hasn't GC'd yet. Finalizing an already-`.completed`
+  // AVAssetWriter throws an uncaught NSException and aborts the process — so only the FIRST call
+  // ever touches the writer/encoders; every later (or concurrent) call just replays that outcome.
+  void Stop(std::function<void(NSError*)> onFinished) {
+    bool isFirstCall;
+    bool resultReady;
+    NSError* resultError = nil;
+    {
+      std::lock_guard<std::mutex> lock(mutex_);
+      isFirstCall = !stopped_;
+      stopped_ = true;
+      resultReady = stopResultReady_;
+      resultError = stopResultError_;
+      if (!isFirstCall && !resultReady) {
+        pendingStopCallbacks_.push_back(std::move(onFinished));
+        return;
+      }
+    }
+    if (!isFirstCall) {
+      if (onFinished) {
+        onFinished(resultError);
+      }
+      return;
+    }
+
+    // Blocks until each has fully torn down — safe to finalize the writer afterward without racing
+    // a further sample callback, and also the point past which no new Fail() can start (both
+    // encoders' onError channels are now silenced) — which is what makes the stopResultReady_
+    // check below race-free against a concurrent Fail() (e.g. the audio tap's own poll timer).
+    if (videoEncoder_) {
+      videoEncoder_->Stop();
+    }
+    if (audioTap_) {
+      audioTap_->Stop();
+    }
+
+    bool started;
+    bool alreadyFailed;
+    NSError* failedError = nil;
+    {
+      std::lock_guard<std::mutex> lock(mutex_);
+      started = writerStarted_;
+      alreadyFailed = stopResultReady_;
+      failedError = stopResultError_;
+      ClearPendingAudioLocked();
+    }
+    if (alreadyFailed) {
+      // A concurrent Fail() already recorded the outcome and (if started) cancelled the writer —
+      // replay that outcome instead of touching it again; finishWriting on a cancelled writer
+      // throws an uncaught NSException.
+      if (onFinished) {
+        onFinished(failedError);
+      }
+      FireEndOnce();
+      return;
+    }
+    if (!started) {
+      FinishStop(MakeError(2, @"No audio or video was captured before the recording was stopped"),
+                 std::move(onFinished));
+      return;
+    }
+    [videoInput_ markAsFinished];
+    [audioInput_ markAsFinished];  // no-op if `captureAudio_` was never set (audioInput_ stays nil)
+    AVAssetWriter* writer = writer_;
+    [writer finishWritingWithCompletionHandler:^{
+      NSError* finishError = (writer.status == AVAssetWriterStatusCompleted) ? nil : writer.error;
+      FinishStop(finishError, std::move(onFinished));
+    }];
+  }
+
+ private:
+  void HandleVideoSample(CMSampleBufferRef sampleBuffer) {
+    NSError* writingError = nil;
+    bool justStarted = false;
+    {
+      std::lock_guard<std::mutex> lock(mutex_);
+      if (stopped_) {
+        return;
+      }
+      if (!writerStarted_) {
+        CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer);
+        videoInput_ = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo
+                                                         outputSettings:nil
+                                                       sourceFormatHint:format];
+        videoInput_.expectsMediaDataInRealTime = YES;
+        [writer_ addInput:videoInput_];
+        if ([writer_ startWriting]) {
+          [writer_ startSessionAtSourceTime:CMSampleBufferGetPresentationTimeStamp(sampleBuffer)];
+          writerStarted_ = true;
+          justStarted = true;
+          AppendVideoLocked(sampleBuffer);
+          for (CMSampleBufferRef pending : pendingAudio_) {
+            AppendAudioLocked(pending);
+            CFRelease(pending);
+          }
+          pendingAudio_.clear();
+        } else {
+          writingError = writer_.error ?: MakeError(3, @"AVAssetWriter startWriting failed");
+        }
+      } else {
+        AppendVideoLocked(sampleBuffer);
+      }
+    }
+    if (writingError != nil) {
+      // Detected while handling a video callback — see Fail()'s doc comment for why this can only
+      // safely stop the audio side directly from here, not video's own encoder.
+      Fail(writingError, /*fromVideo=*/true);
+      return;
+    }
+    if (justStarted) {
+      FireFirstSampleOnce();
+    }
+  }
+
+  void HandleAudioSample(CMSampleBufferRef sampleBuffer) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (stopped_) {
+      return;
+    }
+    if (!writerStarted_) {
+      // Buffered until video's first sample establishes the writer's format/session (see
+      // HandleVideoSample) — capped so a display that never resolves can't grow this unboundedly.
+      CFRetain(sampleBuffer);
+      pendingAudio_.push_back(sampleBuffer);
+      if (pendingAudio_.size() > kMaxPendingAudioSamples) {
+        CFRelease(pendingAudio_.front());
+        pendingAudio_.erase(pendingAudio_.begin());
+      }
+      return;
+    }
+    AppendAudioLocked(sampleBuffer);
+  }
+
+  // Runs on AudioTapSession's own queue; audioEncoder_ is guarded by mutex_ (see Start()). A throw
+  // from EncodePCM is left to propagate — AudioTapSession::HandleBuffer catches it and self-stops
+  // (see sim_audio_tap.h's onBuffer doc) — this method must never call audioTap_->Stop() itself.
+  void HandleAudioPCM(const AudioBufferList* data, const AudioTimeStamp* time) {
+    AudioEncoder* encoder;
+    {
+      std::lock_guard<std::mutex> lock(mutex_);
+      if (stopped_) {
+        return;
+      }
+      encoder = audioEncoder_.get();
+    }
+    if (encoder != nullptr) {
+      encoder->EncodePCM(data, time);
+    }
+  }
+
+  // Caller holds mutex_.
+  void AppendVideoLocked(CMSampleBufferRef sampleBuffer) {
+    if (videoInput_.isReadyForMoreMediaData) {
+      [videoInput_ appendSampleBuffer:sampleBuffer];
+    }
+  }
+
+  // Caller holds mutex_.
+  void AppendAudioLocked(CMSampleBufferRef sampleBuffer) {
+    if (audioInput_.isReadyForMoreMediaData) {
+      [audioInput_ appendSampleBuffer:sampleBuffer];
+    }
+  }
+
+  // Caller holds mutex_.
+  void ClearPendingAudioLocked() {
+    for (CMSampleBufferRef pending : pendingAudio_) {
+      CFRelease(pending);
+    }
+    pendingAudio_.clear();
+  }
+
+  void FireFirstSampleOnce() {
+    if (!firstSampleFired_.exchange(true)) {
+      if (onFirstSample_) {
+        onFirstSample_();
+      }
+    }
+  }
+
+  void FireEndOnce() {
+    if (!endFired_.exchange(true)) {
+      if (onEnd_) {
+        onEnd_();
+      }
+    }
+  }
+
+  // Caller holds mutex_. Records the session's single, final outcome — a no-op if already
+  // recorded. Returns any Stop() calls queued while the outcome was unknown; caller must invoke
+  // each with `error`, unlocked.
+  std::vector<std::function<void(NSError*)>> RecordResultLocked(NSError* error) {
+    if (stopResultReady_) {
+      return {};
+    }
+    stopResultReady_ = true;
+    stopResultError_ = error;
+    std::vector<std::function<void(NSError*)>> pending;
+    pending.swap(pendingStopCallbacks_);
+    return pending;
+  }
+
+  // Caller holds mutex_. Returns any queued Stop() calls to flush (see RecordResultLocked).
+  std::vector<std::function<void(NSError*)>> FailLocked(NSError* error) {
+    if (stopResultReady_) {
+      return {};  // already resolved (a prior Fail() or Stop() completion)
+    }
+    stopped_ = true;
+    if (writerStarted_) {
+      [writer_ cancelWriting];
+    }
+    ClearPendingAudioLocked();
+    auto pending = RecordResultLocked(error);
+    if (onError_) {
+      onError_(error);
+    }
+    return pending;
+  }
+
+  // Called from either encoder's onError, or from HandleVideoSample on a writer-level failure —
+  // not holding mutex_ (could deadlock against that same encoder's own queue). Stops only the
+  // OTHER (still-running) encoder — the one whose callback we're inside is already tearing itself
+  // down on its own error path, and calling its own blocking Stop() here would deadlock.
+  void Fail(NSError* error, bool fromVideo) {
+    if (fromVideo) {
+      if (audioTap_) {
+        audioTap_->Stop();
+      }
+    } else {
+      if (videoEncoder_) {
+        videoEncoder_->Stop();
+      }
+    }
+    std::vector<std::function<void(NSError*)>> pending;
+    {
+      std::lock_guard<std::mutex> lock(mutex_);
+      pending = FailLocked(error);
+    }
+    for (auto& callback : pending) {
+      if (callback) {
+        callback(error);
+      }
+    }
+    FireEndOnce();
+  }
+
+  // The single point where a real (first-ever) Stop() attempt's outcome becomes known — records
+  // it, replies to `onFinished` and any callers that arrived while it was still in flight, then
+  // fires onEnd_. Never called a second time for the same session (see Stop()'s own guard).
+  void FinishStop(NSError* error, std::function<void(NSError*)> onFinished) {
+    std::vector<std::function<void(NSError*)>> pending;
+    {
+      std::lock_guard<std::mutex> lock(mutex_);
+      pending = RecordResultLocked(error);
+    }
+    if (onFinished) {
+      onFinished(error);
+    }
+    for (auto& callback : pending) {
+      if (callback) {
+        callback(error);
+      }
+    }
+    FireEndOnce();
+  }
+
+  // Best-effort, synchronous cleanup for the destructor path (Start() throwing partway through,
+  // or the session being destroyed without an explicit Stop()) — no completion callback, unlike
+  // the public Stop().
+  void TearDownIfNeeded() {
+    if (videoEncoder_) {
+      videoEncoder_->Stop();
+    }
+    if (audioTap_) {
+      audioTap_->Stop();
+    }
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (writerStarted_ && writer_ != nil) {
+      [writer_ cancelWriting];
+    }
+    ClearPendingAudioLocked();
+  }
+
+  id device_;
+  NSString* udid_;
+  VideoEncoderOptions videoOptions_;
+  NSString* outputFile_;
+  bool captureAudio_;
+
+  std::function<void()> onFirstSample_;
+  std::function<void(NSError*)> onError_;
+  std::function<void()> onEnd_;
+
+  double clockOrigin_ = 0;
+  std::unique_ptr<VideoFrameEncoder> videoEncoder_;
+  std::unique_ptr<AudioTapSession> audioTap_;
+  std::unique_ptr<AudioEncoder> audioEncoder_;
+
+  AVAssetWriter* writer_ = nil;
+  AVAssetWriterInput* videoInput_ = nil;
+  AVAssetWriterInput* audioInput_ = nil;
+
+  std::mutex mutex_;
+  bool writerStarted_ = false;
+  bool stopped_ = false;  // true from the first Stop() (or Fail()) call onward
+  bool stopResultReady_ = false;
+  NSError* stopResultError_ = nil;
+  std::vector<std::function<void(NSError*)>> pendingStopCallbacks_;  // Stop() calls awaiting stopResultReady_
+  std::vector<CMSampleBufferRef> pendingAudio_;
+  std::atomic<bool> firstSampleFired_{false};
+  std::atomic<bool> endFired_{false};
+};
+
+AVRecordingSession::AVRecordingSession(id device, NSString* udid, VideoEncoderOptions videoOptions,
+                                       NSString* outputFile, bool captureAudio)
+    : impl_(std::make_unique<Impl>(device, udid, videoOptions, outputFile, captureAudio)) {}
+
+AVRecordingSession::~AVRecordingSession() = default;
+
+void AVRecordingSession::Start(std::function<void()> onFirstSample, std::function<void(NSError*)> onError,
+                               std::function<void()> onEnd) {
+  impl_->Start(std::move(onFirstSample), std::move(onError), std::move(onEnd));
+}
+
+void AVRecordingSession::Stop(std::function<void(NSError*)> onFinished) { impl_->Stop(std::move(onFinished)); }
+
+}  // namespace coresim

package/src/native/av_stream.h

@@ -0,0 +1,70 @@
+#pragma once
+
+#import <Foundation/Foundation.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include "video_encoder.h"
+
+namespace coresim {
+
+enum class AVTrack { kVideo, kAudio };
+
+// One encoded unit from AVStreamSession::onAccessUnit — either an Annex-B video access unit
+// (same wire format as VideoAccessUnit) or a raw AAC-LC packet, discriminated by `track`.
+// `isKeyFrame` is video-only (always true for audio — every AAC-LC packet is independently
+// decodable); `sequence` counts per track, not globally, since interleaving order between the two
+// tracks carries no meaning of its own — only `timestampMicros`, on the shared session clock
+// (monotonic_clock.h), does.
+struct AVAccessUnit {
+  AVTrack track = AVTrack::kVideo;
+  std::vector<uint8_t> data;
+  bool isKeyFrame = false;
+  uint64_t sequence = 0;
+  int64_t timestampMicros = 0;
+};
+
+// Combined audio+video real-time streaming — the streaming counterpart to AVRecordingSession
+// (av_recording.h). Drives the same VideoFrameEncoder/AudioTapSession/AudioEncoder trio on one
+// shared PTS clock, but delivers each encoded unit live via `onAccessUnit` (interleaved, in
+// production order) instead of muxing into a file.
+//
+// IMPORTANT: needs the host's "System Audio Recording Only" TCC permission — see sim_audio_tap.h.
+class AVStreamSession {
+ public:
+  // `onAbortDelivery`, if set, is invoked by AbortDelivery() below — not called by this class on
+  // its own. See VideoStreamSession's identical parameter for why it exists.
+  AVStreamSession(id device, NSString* udid, VideoEncoderOptions videoOptions,
+                  std::function<void(AVAccessUnit)> onAccessUnit, std::function<void(NSError*)> onError,
+                  std::function<void()> onEnd, std::function<void()> onAbortDelivery = nullptr);
+  ~AVStreamSession();
+
+  AVStreamSession(const AVStreamSession&) = delete;
+  AVStreamSession& operator=(const AVStreamSession&) = delete;
+
+  // Resolves the display and starts the audio tap + both encoders; throws synchronously on setup
+  // failure (`onAccessUnit`/`onError`/`onEnd` never called then). Later failures from either
+  // encoder go to `onError`, then `onEnd` — mirrors VideoStreamSession's contract.
+  void Start();
+
+  // Idempotent; blocks until both encoders have fully stopped. Never call from inside
+  // onAccessUnit/onError/onEnd — same queues this blocks on, so it would deadlock.
+  void Stop();
+
+  // See VideoStreamSession::AbortDelivery — same contract, call before Stop() only when nothing
+  // else will drain the delivery queue concurrently.
+  void AbortDelivery();
+
+  // Forces the next encoded video frame to be a keyframe — see VideoFrameEncoder::RequestKeyFrame.
+  // No audio equivalent (every AAC-LC packet already self-decodable).
+  void RequestKeyFrame();
+
+ private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace coresim