@appium/coresim 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +5 -3
- package/binding.gyp +3 -1
- package/lib/src/commands/jpeg-stream.d.ts +44 -0
- package/lib/src/commands/jpeg-stream.d.ts.map +1 -0
- package/lib/src/commands/jpeg-stream.js +198 -0
- package/lib/src/commands/jpeg-stream.js.map +1 -0
- package/lib/src/commands/ui.d.ts +21 -0
- package/lib/src/commands/ui.d.ts.map +1 -1
- package/lib/src/commands/ui.js +22 -0
- package/lib/src/commands/ui.js.map +1 -1
- package/lib/src/index.d.ts +2 -1
- package/lib/src/index.d.ts.map +1 -1
- package/lib/src/index.js +2 -1
- package/lib/src/index.js.map +1 -1
- package/lib/src/native-simctl.d.ts +2 -0
- package/lib/src/native-simctl.d.ts.map +1 -1
- package/lib/src/native-simctl.js +6 -1
- package/lib/src/native-simctl.js.map +1 -1
- package/lib/src/types.d.ts +64 -2
- package/lib/src/types.d.ts.map +1 -1
- package/lib/src/types.js +8 -0
- package/lib/src/types.js.map +1 -1
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/@appium+coresim.node +0 -0
- package/src/commands/jpeg-stream.ts +233 -0
- package/src/commands/ui.ts +27 -0
- package/src/coresim.mm +199 -6
- package/src/index.ts +4 -0
- package/src/native/audio_encoder.mm +1 -1
- package/src/native/av_recording.h +1 -1
- package/src/native/av_recording.mm +76 -27
- package/src/native/sim_audio_tap.mm +3 -3
- package/src/native/sim_device.h +11 -0
- package/src/native/sim_device.mm +53 -0
- package/src/native/sim_jpeg_stream.h +69 -0
- package/src/native/sim_jpeg_stream.mm +322 -0
- package/src/native/sim_orientation.h +16 -0
- package/src/native/sim_orientation.mm +58 -0
- package/src/native/sim_pasteboard.mm +3 -3
- package/src/native/sim_process.mm +1 -1
- package/src/native/sim_screenshot.h +13 -0
- package/src/native/sim_screenshot.mm +50 -25
- package/src/native/sim_video_recording.mm +2 -43
- package/src/native/tcc_privacy.mm +1 -1
- package/src/native/video_encoder.h +9 -2
- package/src/native/video_encoder.mm +241 -21
- package/src/native-simctl.ts +7 -0
- package/src/types.ts +69 -2
package/src/native/sim_device.mm
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
#include "sim_device.h"
|
|
2
2
|
|
|
3
|
+
#import <mach/mach.h>
|
|
3
4
|
#import <objc/message.h>
|
|
4
5
|
|
|
6
|
+
#include <cstring>
|
|
7
|
+
|
|
5
8
|
#include "objc_runtime.h"
|
|
6
9
|
#include "safe_dispatch.h"
|
|
7
10
|
|
|
@@ -9,6 +12,12 @@ namespace coresim {
|
|
|
9
12
|
|
|
10
13
|
namespace {
|
|
11
14
|
|
|
15
|
+
NSString* const kOrientationErrorDomain = @"io.appium.coresim.Orientation";
|
|
16
|
+
|
|
17
|
+
NSError* MakeOrientationError(NSInteger code, NSString* message) {
|
|
18
|
+
return [NSError errorWithDomain:kOrientationErrorDomain code:code userInfo:@{NSLocalizedDescriptionKey : message}];
|
|
19
|
+
}
|
|
20
|
+
|
|
12
21
|
// Shared shapes reused across the many identical-signature getters/setters below.
|
|
13
22
|
id IdGetter(id target, const std::string& selectorName) {
|
|
14
23
|
RequireSelector(target, selectorName);
|
|
@@ -53,6 +62,7 @@ NSString* DeviceName(id device) { return IdGetter(device, "name"); }
|
|
|
53
62
|
id DeviceDeviceType(id device) { return IdGetter(device, "deviceType"); }
|
|
54
63
|
id DeviceRuntime(id device) { return IdGetter(device, "runtime"); }
|
|
55
64
|
NSString* DeviceDataPath(id device) { return IdGetter(device, "dataPath"); }
|
|
65
|
+
NSDate* DeviceLastBootedAt(id device) { return IdGetter(device, "lastBootedAt"); }
|
|
56
66
|
|
|
57
67
|
unsigned long long DeviceState(id device) {
|
|
58
68
|
static const std::string kSelectorName = "state";
|
|
@@ -115,6 +125,49 @@ unsigned int LookupMachPort(id device, NSString* serviceName, NSError** error) {
|
|
|
115
125
|
});
|
|
116
126
|
}
|
|
117
127
|
|
|
128
|
+
BOOL SetDeviceOrientation(id device, int32_t orientation, NSError** error) {
|
|
129
|
+
NSError* lookupError = nil;
|
|
130
|
+
unsigned int purplePort = LookupMachPort(device, @"PurpleWorkspacePort", &lookupError);
|
|
131
|
+
// The port is the authoritative success signal, not the error (see LookupMachPort's own callers,
|
|
132
|
+
// e.g. sim_pasteboard.mm).
|
|
133
|
+
if (purplePort == 0) {
|
|
134
|
+
*error = lookupError ?: MakeOrientationError(1, @"PurpleWorkspacePort is not available — is the device booted?");
|
|
135
|
+
return NO;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// GSEvent wire format, undocumented (see CLAUDE.md): mach_msg_header_t + a fixed-layout event
|
|
139
|
+
// record, in a 112-byte buffer (8-byte aligned, >= the 108-byte message mach_msg sends).
|
|
140
|
+
constexpr uint32_t kGSEventTypeDeviceOrientationChanged = 50;
|
|
141
|
+
constexpr uint32_t kGSEventHostFlag = 0x20000;
|
|
142
|
+
constexpr mach_msg_id_t kGSEventMachMessageID = 0x7B;
|
|
143
|
+
constexpr mach_msg_timeout_t kSendTimeoutMs = 2000;
|
|
144
|
+
|
|
145
|
+
uint8_t buffer[112] = {0};
|
|
146
|
+
auto* header = reinterpret_cast<mach_msg_header_t*>(buffer);
|
|
147
|
+
header->msgh_bits = 0x13; // MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 0)
|
|
148
|
+
header->msgh_size = 108; // align4(4 + 0x6B) — matches Simulator.app's own sendPurpleEvent:
|
|
149
|
+
header->msgh_remote_port = purplePort;
|
|
150
|
+
header->msgh_local_port = MACH_PORT_NULL;
|
|
151
|
+
header->msgh_id = kGSEventMachMessageID;
|
|
152
|
+
|
|
153
|
+
uint32_t type = kGSEventTypeDeviceOrientationChanged | kGSEventHostFlag;
|
|
154
|
+
std::memcpy(buffer + 0x18, &type, sizeof(type));
|
|
155
|
+
uint32_t recordInfoSize = 4;
|
|
156
|
+
std::memcpy(buffer + 0x48, &recordInfoSize, sizeof(recordInfoSize));
|
|
157
|
+
uint32_t orientationValue = static_cast<uint32_t>(orientation);
|
|
158
|
+
std::memcpy(buffer + 0x4C, &orientationValue, sizeof(orientationValue));
|
|
159
|
+
|
|
160
|
+
kern_return_t kr = mach_msg(header, MACH_SEND_MSG | MACH_SEND_TIMEOUT, header->msgh_size, 0, MACH_PORT_NULL,
|
|
161
|
+
kSendTimeoutMs, MACH_PORT_NULL);
|
|
162
|
+
if (kr != KERN_SUCCESS) {
|
|
163
|
+
*error = MakeOrientationError(
|
|
164
|
+
2, [NSString stringWithFormat:@"Failed to send the orientation change (kern_return_t %d): %s", kr,
|
|
165
|
+
mach_error_string(kr)]);
|
|
166
|
+
return NO;
|
|
167
|
+
}
|
|
168
|
+
return YES;
|
|
169
|
+
}
|
|
170
|
+
|
|
118
171
|
BOOL InstallApp(id device, NSURL* appURL, NSDictionary* options, NSError** error) {
|
|
119
172
|
static const std::string kSelectorName = "installApplication:withOptions:error:";
|
|
120
173
|
RequireSelector(device, kSelectorName);
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#import <Foundation/Foundation.h>
|
|
4
|
+
|
|
5
|
+
#include <cstdint>
|
|
6
|
+
#include <functional>
|
|
7
|
+
#include <memory>
|
|
8
|
+
#include <vector>
|
|
9
|
+
|
|
10
|
+
namespace coresim {
|
|
11
|
+
|
|
12
|
+
struct JpegStreamOptions {
|
|
13
|
+
NSString* displayId = nil;
|
|
14
|
+
double fps = 60.0;
|
|
15
|
+
// 0-100 percent; same semantics as CaptureScreenshot's jpegQualityPercent (sim_screenshot.h),
|
|
16
|
+
// nil-able for ImageIO's own default (near-lossless, so a much larger frame) — but defaulted to
|
|
17
|
+
// 80 by coresim.mm rather than left nil, since a continuous live stream should default to
|
|
18
|
+
// noticeably smaller frames, unlike a one-off CaptureScreenshot call.
|
|
19
|
+
NSNumber* jpegQualityPercent = @80;
|
|
20
|
+
// 0.0-1.0 fraction of the original frame's width/height; 1.0 (default) performs no scaling.
|
|
21
|
+
// Already normalized from the JS-facing 1-100 percent option by coresim.mm.
|
|
22
|
+
double scale = 1.0;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
// One JPEG-encoded frame. Unlike VideoAccessUnit, every frame is independently decodable — there's
|
|
26
|
+
// no keyframe/interframe distinction, so no resync semantics are needed on the consuming side.
|
|
27
|
+
struct JpegFrame {
|
|
28
|
+
std::vector<uint8_t> data;
|
|
29
|
+
uint64_t sequence = 0;
|
|
30
|
+
// Microseconds since the stream started.
|
|
31
|
+
int64_t timestampMicros = 0;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
// Polls the live display IOSurface (sim_screenshot.h) on a serial queue and JPEG-encodes changed
|
|
35
|
+
// frames via ImageIO, delivering each live — see CLAUDE.md for how this relates to
|
|
36
|
+
// VideoStreamSession and CaptureScreenshot.
|
|
37
|
+
class JpegStreamSession {
|
|
38
|
+
public:
|
|
39
|
+
// `onAbortDelivery`, if set, is invoked by AbortDelivery() below — not called by this class on
|
|
40
|
+
// its own.
|
|
41
|
+
JpegStreamSession(id device, JpegStreamOptions options, std::function<void(JpegFrame)> onFrame,
|
|
42
|
+
std::function<void(NSError*)> onError, std::function<void()> onEnd,
|
|
43
|
+
std::function<void()> onAbortDelivery = nullptr);
|
|
44
|
+
~JpegStreamSession();
|
|
45
|
+
|
|
46
|
+
JpegStreamSession(const JpegStreamSession&) = delete;
|
|
47
|
+
JpegStreamSession& operator=(const JpegStreamSession&) = delete;
|
|
48
|
+
|
|
49
|
+
// Resolves the display and starts the polling loop; throws synchronously on resolution/setup
|
|
50
|
+
// failure (`onEnd` never called then). Later failures go to `onError`, then `onEnd`.
|
|
51
|
+
void Start();
|
|
52
|
+
|
|
53
|
+
// Idempotent; blocks until the loop has fully stopped. Never call from inside onFrame/onError/
|
|
54
|
+
// onEnd — same queue this blocks on, so it would deadlock.
|
|
55
|
+
void Stop();
|
|
56
|
+
|
|
57
|
+
// Runs the constructor's `onAbortDelivery` (if any) — e.g. aborting a bounded delivery queue so
|
|
58
|
+
// a producer thread blocked pushing into it unblocks. Call before Stop() when the caller can't
|
|
59
|
+
// rely on anything else draining that queue concurrently (e.g. process-exit cleanup running
|
|
60
|
+
// synchronously on the same thread `Stop()` would otherwise wait on) — a normal Stop() doesn't
|
|
61
|
+
// need this. Safe from any thread; a no-op if unset.
|
|
62
|
+
void AbortDelivery();
|
|
63
|
+
|
|
64
|
+
private:
|
|
65
|
+
class Impl;
|
|
66
|
+
std::unique_ptr<Impl> impl_;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
} // namespace coresim
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
#include "sim_jpeg_stream.h"
|
|
2
|
+
|
|
3
|
+
#import <CoreImage/CoreImage.h>
|
|
4
|
+
#import <IOSurface/IOSurface.h>
|
|
5
|
+
|
|
6
|
+
#include <algorithm>
|
|
7
|
+
#include <atomic>
|
|
8
|
+
#include <cmath>
|
|
9
|
+
|
|
10
|
+
#include "monotonic_clock.h"
|
|
11
|
+
#include "nserror_bridge.h"
|
|
12
|
+
#include "sim_screenshot.h"
|
|
13
|
+
|
|
14
|
+
namespace coresim {
|
|
15
|
+
|
|
16
|
+
namespace {
|
|
17
|
+
|
|
18
|
+
NSString* const kJpegStreamErrorDomain = @"io.appium.coresim.JpegStream";
|
|
19
|
+
|
|
20
|
+
// A scale this close to 1.0 (from a 1-100 percent option divided by 100.0 — never exactly 1.0
|
|
21
|
+
// except at 100%) is treated as "no scaling" — comparing doubles for exact equality is unreliable.
|
|
22
|
+
constexpr double kScaleEpsilon = 1e-9;
|
|
23
|
+
|
|
24
|
+
// If no frame can be produced for this long — the display surface staying unavailable, or the
|
|
25
|
+
// CIImage/CGImage render failing — for that whole stretch, something is genuinely wrong (a
|
|
26
|
+
// permanently dropped connection, say) rather than a one-off transient hiccup; escalate to a real
|
|
27
|
+
// error instead of polling forever with nothing to show for it and no error ever reported.
|
|
28
|
+
constexpr double kMaxStallSeconds = 10.0;
|
|
29
|
+
|
|
30
|
+
NSError* MakeError(NSInteger code, NSString* message) {
|
|
31
|
+
return [NSError errorWithDomain:kJpegStreamErrorDomain code:code userInfo:@{NSLocalizedDescriptionKey : message}];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
} // namespace
|
|
35
|
+
|
|
36
|
+
class JpegStreamSession::Impl {
|
|
37
|
+
public:
|
|
38
|
+
Impl(id device, JpegStreamOptions options, std::function<void(JpegFrame)> onFrame,
|
|
39
|
+
std::function<void(NSError*)> onError, std::function<void()> onEnd, std::function<void()> onAbortDelivery)
|
|
40
|
+
: device_(device),
|
|
41
|
+
options_(options),
|
|
42
|
+
onFrame_(std::move(onFrame)),
|
|
43
|
+
onError_(std::move(onError)),
|
|
44
|
+
onEnd_(std::move(onEnd)),
|
|
45
|
+
onAbortDelivery_(std::move(onAbortDelivery)) {
|
|
46
|
+
queue_ = dispatch_queue_create("io.appium.coresim.jpegStream", DISPATCH_QUEUE_SERIAL);
|
|
47
|
+
// Persistent, unlike CaptureScreenshot's own one-shot CIContext (see sim_screenshot.mm) — that
|
|
48
|
+
// file's comment calls this out as exactly the tradeoff a continuous streaming path should
|
|
49
|
+
// make instead.
|
|
50
|
+
context_ = [CIContext contextWithOptions:nil];
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
~Impl() { Stop(); }
|
|
54
|
+
|
|
55
|
+
void Start() {
|
|
56
|
+
NSError* error = nil;
|
|
57
|
+
id descriptor = ResolveCaptureDisplay(device_, options_.displayId, &error);
|
|
58
|
+
if (descriptor == nil) {
|
|
59
|
+
throw NSErrorException(error);
|
|
60
|
+
}
|
|
61
|
+
id surfaceObj = CurrentDisplaySurface(descriptor);
|
|
62
|
+
if (surfaceObj == nil) {
|
|
63
|
+
throw NSErrorException(MakeError(1, @"The device's display surface is not available yet"));
|
|
64
|
+
}
|
|
65
|
+
IOSurfaceRef surface = (__bridge IOSurfaceRef)surfaceObj;
|
|
66
|
+
|
|
67
|
+
startTime_ = MonotonicSeconds();
|
|
68
|
+
// Set before the initial encode below — EmitFrame measures elapsed time from it, and running_
|
|
69
|
+
// gates whether a frame is delivered at all (see Tick()'s own check).
|
|
70
|
+
running_ = true;
|
|
71
|
+
// Encode immediately rather than waiting for a *changed* seed on the first tick, or the stream
|
|
72
|
+
// would stay silent until the display changes again — unlike Tick(), this doesn't check
|
|
73
|
+
// lastSeed_ first (it's still its default 0), so EncodeCurrentSeedAndEmit always runs once here.
|
|
74
|
+
NSError* encodeError = nil;
|
|
75
|
+
EncodeCurrentSeedAndEmit(surface, &encodeError);
|
|
76
|
+
if (encodeError != nil) {
|
|
77
|
+
running_ = false;
|
|
78
|
+
throw NSErrorException(encodeError);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
double interval = 1.0 / std::max(options_.fps, 1.0);
|
|
82
|
+
dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, queue_);
|
|
83
|
+
dispatch_source_set_timer(timer, dispatch_time(DISPATCH_TIME_NOW, 0),
|
|
84
|
+
static_cast<uint64_t>(interval * NSEC_PER_SEC),
|
|
85
|
+
static_cast<uint64_t>(interval * NSEC_PER_SEC / 10));
|
|
86
|
+
// `this` outlives the timer: Stop()/StopFromQueue() always cancel or outrun it before `this`
|
|
87
|
+
// can be destroyed (see their comments below).
|
|
88
|
+
dispatch_source_set_event_handler(timer, ^{
|
|
89
|
+
Tick();
|
|
90
|
+
});
|
|
91
|
+
timer_ = timer;
|
|
92
|
+
dispatch_resume(timer_);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Callable from any thread except `queue_` itself (would deadlock on the dispatch_sync below).
|
|
96
|
+
void Stop() {
|
|
97
|
+
if (!running_.exchange(false)) {
|
|
98
|
+
return; // idempotent
|
|
99
|
+
}
|
|
100
|
+
if (timer_ != nullptr) {
|
|
101
|
+
dispatch_source_cancel(timer_);
|
|
102
|
+
// dispatch_source_cancel doesn't preempt a currently-executing handler — an already-running
|
|
103
|
+
// Tick() (past its own `running_` check) can still run to completion and emit one last frame
|
|
104
|
+
// via onFrame_ before this returns. This blocks until that happens, so onEnd_ below (which
|
|
105
|
+
// the caller uses to release resources onFrame_ needs, e.g. a ThreadSafeFunction) never races
|
|
106
|
+
// a still-in-flight emit — not, as such, a guarantee that no further frame is ever emitted.
|
|
107
|
+
dispatch_sync(queue_, ^{
|
|
108
|
+
});
|
|
109
|
+
timer_ = nullptr;
|
|
110
|
+
}
|
|
111
|
+
if (onEnd_) {
|
|
112
|
+
onEnd_();
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
void AbortDelivery() {
|
|
117
|
+
if (onAbortDelivery_) {
|
|
118
|
+
onAbortDelivery_();
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
private:
|
|
123
|
+
// Same as Stop() minus the dispatch_sync barrier — only safe from within Tick() itself, already
|
|
124
|
+
// serialized on `queue_`; would race a concurrent Tick() from any other thread.
|
|
125
|
+
void StopFromQueue() {
|
|
126
|
+
if (!running_.exchange(false)) {
|
|
127
|
+
return; // idempotent — e.g. an external Stop() already won this race
|
|
128
|
+
}
|
|
129
|
+
if (timer_ != nullptr) {
|
|
130
|
+
dispatch_source_cancel(timer_);
|
|
131
|
+
timer_ = nullptr;
|
|
132
|
+
}
|
|
133
|
+
if (onEnd_) {
|
|
134
|
+
onEnd_();
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
void Tick() {
|
|
139
|
+
if (!running_) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
@autoreleasepool {
|
|
143
|
+
try {
|
|
144
|
+
// Re-resolved every tick (like CaptureScreenshot does), not cached once in Start(), so a
|
|
145
|
+
// deleted device or disconnected display surfaces a real error instead of Tick() quietly
|
|
146
|
+
// doing nothing forever.
|
|
147
|
+
NSError* resolveError = nil;
|
|
148
|
+
id descriptor = ResolveCaptureDisplay(device_, options_.displayId, &resolveError);
|
|
149
|
+
if (descriptor == nil) {
|
|
150
|
+
if (onError_) {
|
|
151
|
+
onError_(resolveError);
|
|
152
|
+
}
|
|
153
|
+
StopFromQueue();
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
id surfaceObj = CurrentDisplaySurface(descriptor);
|
|
157
|
+
if (surfaceObj == nil) {
|
|
158
|
+
ReportIfStalledTooLong(); // transient — the connection may not have a frame ready yet
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
IOSurfaceRef surface = (__bridge IOSurfaceRef)surfaceObj;
|
|
162
|
+
uint32_t seed = IOSurfaceGetSeed(surface);
|
|
163
|
+
if (seed == lastSeed_) {
|
|
164
|
+
// Unchanged since the last tick (mirrors VideoFrameEncoder's own seed check) — the
|
|
165
|
+
// surface itself is fine, just nothing new to encode, so this resolves any stall.
|
|
166
|
+
stalledSince_ = 0;
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
NSError* encodeError = nil;
|
|
170
|
+
bool encoded = EncodeCurrentSeedAndEmit(surface, &encodeError);
|
|
171
|
+
if (encodeError != nil) {
|
|
172
|
+
if (onError_) {
|
|
173
|
+
onError_(encodeError);
|
|
174
|
+
}
|
|
175
|
+
StopFromQueue();
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
if (encoded) {
|
|
179
|
+
stalledSince_ = 0;
|
|
180
|
+
} else {
|
|
181
|
+
ReportIfStalledTooLong(); // transient — e.g. a momentary CIImage/CGImage render failure
|
|
182
|
+
}
|
|
183
|
+
} catch (const std::exception& e) {
|
|
184
|
+
// Without this, an exception here (e.g. a dropped display-proxy connection, which surfaces
|
|
185
|
+
// as NativeSimUnavailableError/ObjCException — both std::exception subtypes — via
|
|
186
|
+
// ResolveCaptureDisplay's own dynamic dispatch) would escape this bare GCD timer handler
|
|
187
|
+
// uncaught and crash the whole process (see CLAUDE.md; mirrors VideoFrameEncoder::Tick()).
|
|
188
|
+
if (onError_) {
|
|
189
|
+
onError_(MakeError(4, [NSString stringWithFormat:@"JPEG streaming failed: %s", e.what()]));
|
|
190
|
+
}
|
|
191
|
+
StopFromQueue();
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Called from Tick() on a tick that produced nothing but also wasn't a genuine (reported) error —
|
|
197
|
+
// starts a stall timer on the first such tick, and escalates to a real onError_/StopFromQueue()
|
|
198
|
+
// once it's run past kMaxStallSeconds without a single successful tick (a frame, or an unchanged-
|
|
199
|
+
// seed check) in between. Without this, a display surface that never comes back (or a
|
|
200
|
+
// CIImage/CGImage render that never succeeds again) would poll forever with nothing to show for
|
|
201
|
+
// it and no error ever reported (see CLAUDE.md).
|
|
202
|
+
void ReportIfStalledTooLong() {
|
|
203
|
+
double now = MonotonicSeconds();
|
|
204
|
+
if (stalledSince_ == 0) {
|
|
205
|
+
stalledSince_ = now;
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (now - stalledSince_ > kMaxStallSeconds) {
|
|
209
|
+
if (onError_) {
|
|
210
|
+
onError_(MakeError(5, @"No JPEG frame could be produced for too long"));
|
|
211
|
+
}
|
|
212
|
+
StopFromQueue();
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// Reads `surface`'s current seed, attempts one encode, and — only on success — commits exactly
|
|
217
|
+
// that pre-encode seed to lastSeed_ and emits the frame. Reading the seed before encoding (never
|
|
218
|
+
// a value re-read afterward) matters: if the display changes again while EncodeSurface() is still
|
|
219
|
+
// running, the pre-encode seed still correctly identifies which content this frame captures, so
|
|
220
|
+
// the next caller sees the surface's now-newer seed as "changed" and retries — reading it after
|
|
221
|
+
// encoding instead would wrongly commit the *newer* seed against the *older* frame just emitted,
|
|
222
|
+
// permanently losing that update (the following unchanged-seed check would then treat it as
|
|
223
|
+
// already delivered, silently, since it looks identical to a genuinely static display). Shared by
|
|
224
|
+
// Start() (always called once, regardless of lastSeed_) and Tick() (only once a changed seed was
|
|
225
|
+
// already observed) so this ordering can't independently drift between the two again.
|
|
226
|
+
//
|
|
227
|
+
// Returns whether a frame was produced. Sets *error only for a genuine encode failure that
|
|
228
|
+
// should end the whole session; a false return with *error left nil means "transient, retry
|
|
229
|
+
// next tick" (mirrors VideoFrameEncoder::EncodeSurface's identical CVPixelBufferCreateWithIOSurface
|
|
230
|
+
// transient-failure contract in video_encoder.mm) — lastSeed_ is deliberately left stale then, so
|
|
231
|
+
// the next attempt retries this same content instead of silently skipping it forever.
|
|
232
|
+
bool EncodeCurrentSeedAndEmit(IOSurfaceRef surface, NSError** error) {
|
|
233
|
+
uint32_t seed = IOSurfaceGetSeed(surface);
|
|
234
|
+
NSData* data = nil;
|
|
235
|
+
bool encoded = EncodeSurface(surface, &data, error);
|
|
236
|
+
if (encoded) {
|
|
237
|
+
lastSeed_ = seed;
|
|
238
|
+
EmitFrame(data);
|
|
239
|
+
}
|
|
240
|
+
return encoded;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// Returns whether a frame was produced. Sets *error only for a genuine encode failure that
|
|
244
|
+
// should end the whole session; a false return with *error left nil means "transient, retry
|
|
245
|
+
// next tick" (mirrors VideoFrameEncoder::EncodeSurface's identical CVPixelBufferCreateWithIOSurface
|
|
246
|
+
// transient-failure contract in video_encoder.mm).
|
|
247
|
+
bool EncodeSurface(IOSurfaceRef surface, NSData** outData, NSError** error) {
|
|
248
|
+
CIImage* ciImage = [CIImage imageWithIOSurface:surface];
|
|
249
|
+
if (ciImage == nil) {
|
|
250
|
+
return false;
|
|
251
|
+
}
|
|
252
|
+
// Scaling the CIImage before rendering (rather than resizing an already-encoded JPEG
|
|
253
|
+
// afterward) means the CGImage/JPEG below is produced at the target resolution directly — no
|
|
254
|
+
// extra decode/resize/re-encode round trip.
|
|
255
|
+
if (std::fabs(options_.scale - 1.0) > kScaleEpsilon) {
|
|
256
|
+
ciImage = [ciImage imageByApplyingTransform:CGAffineTransformMakeScale(options_.scale, options_.scale)];
|
|
257
|
+
}
|
|
258
|
+
CGImageRef cgImage = [context_ createCGImage:ciImage fromRect:ciImage.extent];
|
|
259
|
+
if (cgImage == nil) {
|
|
260
|
+
return false;
|
|
261
|
+
}
|
|
262
|
+
// Same CGImageDestination-based encode CaptureScreenshot's kJPEG format uses (sim_screenshot.mm).
|
|
263
|
+
NSData* imageData = EncodeImage(cgImage, ScreenshotFormat::kJPEG, options_.jpegQualityPercent, error);
|
|
264
|
+
CGImageRelease(cgImage);
|
|
265
|
+
if (imageData == nil) {
|
|
266
|
+
return false;
|
|
267
|
+
}
|
|
268
|
+
*outData = imageData;
|
|
269
|
+
return true;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// Called either from Start() (on whichever thread calls it, before the timer/queue_ even starts
|
|
273
|
+
// running) or from Tick() (already serialized on queue_, once the timer is live) — never both at
|
|
274
|
+
// once, since Start() always finishes (and only then resumes the timer) before Tick() can fire.
|
|
275
|
+
// So sequence_ needs no synchronization, unlike VideoStreamSession's (whose encoder callback can
|
|
276
|
+
// run concurrently with the poll loop on a different thread).
|
|
277
|
+
void EmitFrame(NSData* data) {
|
|
278
|
+
JpegFrame frame;
|
|
279
|
+
frame.sequence = sequence_++;
|
|
280
|
+
frame.timestampMicros = static_cast<int64_t>((MonotonicSeconds() - startTime_) * 1000000);
|
|
281
|
+
const uint8_t* bytes = static_cast<const uint8_t*>(data.bytes);
|
|
282
|
+
frame.data.assign(bytes, bytes + data.length);
|
|
283
|
+
if (onFrame_) {
|
|
284
|
+
onFrame_(std::move(frame));
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
id device_;
|
|
289
|
+
JpegStreamOptions options_;
|
|
290
|
+
std::function<void(JpegFrame)> onFrame_;
|
|
291
|
+
std::function<void(NSError*)> onError_;
|
|
292
|
+
std::function<void()> onEnd_;
|
|
293
|
+
std::function<void()> onAbortDelivery_;
|
|
294
|
+
|
|
295
|
+
dispatch_queue_t queue_ = nullptr;
|
|
296
|
+
dispatch_source_t timer_ = nullptr;
|
|
297
|
+
CIContext* context_ = nil;
|
|
298
|
+
uint32_t lastSeed_ = 0;
|
|
299
|
+
double startTime_ = 0;
|
|
300
|
+
uint64_t sequence_ = 0;
|
|
301
|
+
// 0 means "no stall in progress" — set to MonotonicSeconds() by ReportIfStalledTooLong() on the
|
|
302
|
+
// first unproductive tick of a run, cleared back to 0 by any tick that makes real progress
|
|
303
|
+
// (a produced frame, or an unchanged-seed check confirming the surface itself is still fine).
|
|
304
|
+
double stalledSince_ = 0;
|
|
305
|
+
std::atomic<bool> running_{false};
|
|
306
|
+
};
|
|
307
|
+
|
|
308
|
+
JpegStreamSession::JpegStreamSession(id device, JpegStreamOptions options, std::function<void(JpegFrame)> onFrame,
|
|
309
|
+
std::function<void(NSError*)> onError, std::function<void()> onEnd,
|
|
310
|
+
std::function<void()> onAbortDelivery)
|
|
311
|
+
: impl_(std::make_unique<Impl>(device, options, std::move(onFrame), std::move(onError), std::move(onEnd),
|
|
312
|
+
std::move(onAbortDelivery))) {}
|
|
313
|
+
|
|
314
|
+
JpegStreamSession::~JpegStreamSession() = default;
|
|
315
|
+
|
|
316
|
+
void JpegStreamSession::Start() { impl_->Start(); }
|
|
317
|
+
|
|
318
|
+
void JpegStreamSession::Stop() { impl_->Stop(); }
|
|
319
|
+
|
|
320
|
+
void JpegStreamSession::AbortDelivery() { impl_->AbortDelivery(); }
|
|
321
|
+
|
|
322
|
+
} // namespace coresim
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#import <Foundation/Foundation.h>
|
|
4
|
+
|
|
5
|
+
#include <cstdint>
|
|
6
|
+
|
|
7
|
+
namespace coresim {
|
|
8
|
+
|
|
9
|
+
// A live orientation read straight off backboardd's own persisted preference file on the host side
|
|
10
|
+
// (see CLAUDE.md) — reflects a rotation from any source, not just this process's own
|
|
11
|
+
// SetDeviceOrientation calls. Calls `handler` once, async, with a DeviceOrientation wire value
|
|
12
|
+
// (types.ts) — always 1 (portrait) if the device hasn't rotated yet this boot, or on any read
|
|
13
|
+
// failure.
|
|
14
|
+
void ReadGuestOrientation(id device, void (^handler)(int32_t orientation));
|
|
15
|
+
|
|
16
|
+
} // namespace coresim
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
#include "sim_orientation.h"
|
|
2
|
+
|
|
3
|
+
#include "sim_device.h"
|
|
4
|
+
|
|
5
|
+
namespace coresim {
|
|
6
|
+
|
|
7
|
+
namespace {
|
|
8
|
+
|
|
9
|
+
// backboardd's persisted GraphicsOrientation swaps landscape left/right vs. our own
|
|
10
|
+
// DeviceOrientation wire values — confirmed empirically (see CLAUDE.md).
|
|
11
|
+
int32_t TranslateGraphicsOrientation(NSInteger graphicsOrientation) {
|
|
12
|
+
switch (graphicsOrientation) {
|
|
13
|
+
case 3:
|
|
14
|
+
return 4;
|
|
15
|
+
case 4:
|
|
16
|
+
return 3;
|
|
17
|
+
default:
|
|
18
|
+
return static_cast<int32_t>(graphicsOrientation);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// The live digitizer entry for the current boot, or nil if it hasn't rotated yet this boot (or
|
|
23
|
+
// `plist` is nil/malformed). Reading straight from the file (a real binary plist) gives properly
|
|
24
|
+
// typed values — no ASCII-vs-binary parsing ambiguity to handle.
|
|
25
|
+
NSNumber* CurrentGraphicsOrientation(NSDictionary* plist) {
|
|
26
|
+
NSArray* entries = [plist[@"BKDigitizerPersistentServiceProperties"] isKindOfClass:[NSArray class]]
|
|
27
|
+
? plist[@"BKDigitizerPersistentServiceProperties"]
|
|
28
|
+
: nil;
|
|
29
|
+
NSDictionary* last = [entries.lastObject isKindOfClass:[NSDictionary class]] ? entries.lastObject : nil;
|
|
30
|
+
NSDictionary* props = [last[@"props"] isKindOfClass:[NSDictionary class]] ? last[@"props"] : nil;
|
|
31
|
+
return [props[@"GraphicsOrientation"] isKindOfClass:[NSNumber class]] ? props[@"GraphicsOrientation"] : nil;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
} // namespace
|
|
35
|
+
|
|
36
|
+
void ReadGuestOrientation(id device, void (^handler)(int32_t orientation)) {
|
|
37
|
+
NSString* plistPath =
|
|
38
|
+
[DeviceDataPath(device) stringByAppendingPathComponent:@"Library/Preferences/com.apple.backboardd.plist"];
|
|
39
|
+
NSDate* lastBootedAt = DeviceLastBootedAt(device);
|
|
40
|
+
|
|
41
|
+
dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{
|
|
42
|
+
NSDate* modified = [[NSFileManager defaultManager] attributesOfItemAtPath:plistPath
|
|
43
|
+
error:nil][NSFileModificationDate];
|
|
44
|
+
// The whole file predates this boot, so none of its entries — including the last one — can
|
|
45
|
+
// belong to it (confirmed empirically: a reboot leaves the previous boot's entry in place until
|
|
46
|
+
// backboardd rewrites the file, sometimes several seconds after boot completes — see CLAUDE.md).
|
|
47
|
+
// Treat exactly like "never rotated this boot" rather than trusting stale content.
|
|
48
|
+
if (lastBootedAt != nil && modified != nil && [modified compare:lastBootedAt] == NSOrderedAscending) {
|
|
49
|
+
handler(1);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
NSDictionary* plist = [NSDictionary dictionaryWithContentsOfFile:plistPath];
|
|
53
|
+
NSNumber* graphicsOrientation = CurrentGraphicsOrientation(plist);
|
|
54
|
+
handler(graphicsOrientation != nil ? TranslateGraphicsOrientation(graphicsOrientation.integerValue) : 1);
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
} // namespace coresim
|
|
@@ -69,7 +69,7 @@ namespace coresim {
|
|
|
69
69
|
|
|
70
70
|
namespace {
|
|
71
71
|
|
|
72
|
-
NSString* const kPasteboardErrorDomain = @"
|
|
72
|
+
NSString* const kPasteboardErrorDomain = @"io.appium.coresim.Pasteboard";
|
|
73
73
|
|
|
74
74
|
// The UTI both pasteboard mechanisms below exchange plain text under.
|
|
75
75
|
NSString* const kPlainTextUTI = @"public.utf8-plain-text";
|
|
@@ -166,7 +166,7 @@ NSString* PullPasteboardStringModern(id device, NSError** error) {
|
|
|
166
166
|
CoresimPasteboardDelegate* delegate = [CoresimPasteboardDelegate new];
|
|
167
167
|
delegate.activeSema = dispatch_semaphore_create(0);
|
|
168
168
|
delegate.appliedSema = dispatch_semaphore_create(0);
|
|
169
|
-
dispatch_queue_t queue = dispatch_queue_create("
|
|
169
|
+
dispatch_queue_t queue = dispatch_queue_create("io.appium.coresim.pasteboard.pull", DISPATCH_QUEUE_SERIAL);
|
|
170
170
|
|
|
171
171
|
id interface = ConnectPasteboardInterface(device, pasteboard, delegate, queue);
|
|
172
172
|
SafeInvoke([&] {
|
|
@@ -200,7 +200,7 @@ BOOL PushPasteboardStringModern(id device, NSString* content, NSError** error) {
|
|
|
200
200
|
CoresimPasteboardDelegate* delegate = [CoresimPasteboardDelegate new];
|
|
201
201
|
delegate.activeSema = dispatch_semaphore_create(0);
|
|
202
202
|
delegate.appliedSema = dispatch_semaphore_create(0);
|
|
203
|
-
dispatch_queue_t queue = dispatch_queue_create("
|
|
203
|
+
dispatch_queue_t queue = dispatch_queue_create("io.appium.coresim.pasteboard.push", DISPATCH_QUEUE_SERIAL);
|
|
204
204
|
|
|
205
205
|
id interface = ConnectPasteboardInterface(device, pasteboard, delegate, queue);
|
|
206
206
|
SafeInvoke([&] {
|
|
@@ -13,7 +13,7 @@ namespace coresim {
|
|
|
13
13
|
|
|
14
14
|
namespace {
|
|
15
15
|
|
|
16
|
-
NSString* const kProcessErrorDomain = @"
|
|
16
|
+
NSString* const kProcessErrorDomain = @"io.appium.coresim.Process";
|
|
17
17
|
NSString* const kWebInspectorSocketSuffix = @"com.apple.webinspectord_sim.socket";
|
|
18
18
|
|
|
19
19
|
NSError* MakeError(NSInteger code, NSString* message) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#pragma once
|
|
2
2
|
|
|
3
|
+
#import <CoreGraphics/CoreGraphics.h>
|
|
3
4
|
#import <Foundation/Foundation.h>
|
|
4
5
|
|
|
5
6
|
namespace coresim {
|
|
@@ -17,6 +18,11 @@ NSArray<NSDictionary*>* ListDisplays(id device, NSError** error);
|
|
|
17
18
|
// object itself (e.g. StartVideoRecording's `screen` argument).
|
|
18
19
|
id ResolveCaptureDisplay(id device, NSString* displayId, NSError** error);
|
|
19
20
|
|
|
21
|
+
// The device-wide "capture service" port (real protocol: SimScreenCaptureService — see CLAUDE.md),
|
|
22
|
+
// distinct from a display descriptor (see ResolveCaptureDisplay above). Found by scanning ioPorts
|
|
23
|
+
// since no header exists. Throws NativeSimUnavailableError (not NSError**) if absent.
|
|
24
|
+
id ResolveScreenCaptureService(id device, NSError** error);
|
|
25
|
+
|
|
20
26
|
// The descriptor's current framebuffer as an `IOSurfaceRef` (bridge-cast the returned `id`).
|
|
21
27
|
// Returns nil if not available yet (e.g. connection just dropped) — not an error, since that can
|
|
22
28
|
// legitimately happen from one call to the next on a live device.
|
|
@@ -24,6 +30,13 @@ id CurrentDisplaySurface(id descriptor);
|
|
|
24
30
|
|
|
25
31
|
enum class ScreenshotFormat { kPNG, kJPEG };
|
|
26
32
|
|
|
33
|
+
// Encodes `cgImage` via ImageIO into `format` — the same CGImageDestination-based encode
|
|
34
|
+
// CaptureScreenshot uses for its own final step, exposed for a caller that already has a
|
|
35
|
+
// CGImageRef from its own CIContext (e.g. JpegStreamSession's persistent one — see
|
|
36
|
+
// sim_jpeg_stream.mm). `jpegQualityPercent` semantics match CaptureScreenshot's own parameter of
|
|
37
|
+
// the same name. Returns nil (and sets *error) on encode failure. Does not release `cgImage`.
|
|
38
|
+
NSData* EncodeImage(CGImageRef cgImage, ScreenshotFormat format, NSNumber* jpegQualityPercent, NSError** error);
|
|
39
|
+
|
|
27
40
|
// Captures a display as an image, reading the same in-process framebuffer surface `simctl io
|
|
28
41
|
// <udid> screenshot` itself reads (see sim_screenshot.mm) — no temp file, no subprocess.
|
|
29
42
|
// `displayId`, if non-nil, selects a specific port by its UUID (see ListDisplays); nil selects the
|