@stim-cli/server 1.11.0 → 1.13.0

package/dist/stim-frames/main.swift

@@ -10,11 +10,20 @@ import IOSurface
 //   stim-frames android <serial>
 //
 // stdin takes one JSON object per line: {"fps": n, "maxEdge": px, "quality": 0-1,
-// "jpeg": bool, "jpegFps": n, "video": bool, "bitrate": bits per second}, or
-// {"keyframe": true} to make the next video frame a keyframe.
+// "jpeg": bool, "jpegFps": n, "video": bool, "bitrate": bits per second}, where fps 0
+// pauses frames; {"keyframe": true} to make the next video frame a keyframe; or an input:
+// {"input": "touch", "phase": "down|move|up", "x": 0-1, "y": 0-1, "display": n} with x
+// and y on the upright screen; {"input": "text", "text": s}, printable ASCII where "\n"
+// is Return, "\t" is Tab and "\u{8}" is Delete; and {"input": "button", "button":
+// "home|lock"}, or on an emulator also "back|app-switch"; {"input": "rotate", "direction":
+// "left|right"}; and on an emulator {"input": "posture", "posture":
+// "folded|half-open|unfolded"}. An emulator types and presses buttons only with a hardware
+// keyboard.
 // stdout carries messages framed as a 4-byte big-endian length, then a kind byte:
 // 1 is a frame (2-byte width, 2-byte height, JPEG bytes), 2 is a JSON notice
-// ({"error": message} before a failed exit), 3 is an H.264 access unit (1-byte
+// ({"error": message} before a failed exit, {"inputError": message}, on a simulator
+// with several displays {"display": n} when display n is the one lit and streamed, or
+// on an emulator {"keyboard": "yes|no"} once it reports its hardware), 3 is an H.264 access unit (1-byte
 // flags with bit 0 set on a keyframe, 8-byte big-endian float capture time in
 // milliseconds since the epoch, 2-byte width, 2-byte height, Annex-B bytes).
 // The helper exits when stdin closes.
@@ -80,7 +89,7 @@ enum Output {
     }
   }
 
-  static func notice(_ object: [String: String]) {
+  static func notice(_ object: [String: Any]) {
     guard let json = try? JSONSerialization.data(withJSONObject: object) else { return }
     writer.sync { write(Data([2]) + json) }
   }
@@ -135,7 +144,7 @@ final class Pacer {
   }
 
   private func schedule() {
-    guard dirty, !scheduled else { return }
+    guard dirty, !scheduled, config.fps > 0 else { return }
     let next = last + .nanoseconds(Int(1_000_000_000 / max(config.fps, 0.1)))
     scheduled = true
     queue.asyncAfter(deadline: max(next, .now())) {
@@ -179,9 +188,13 @@ final class JpegGate {
 func now() -> Double { Date().timeIntervalSince1970 * 1000 }
 
 final class SimulatorSource {
-  private let udid: String
+  let udid: String
+  let inputQueue = DispatchQueue(label: "stim.frames.input")
+  var hid: SimulatorHID?
   private let pacer: Pacer
-  private var display: SimDisplay?
+  private var displays: [SimDisplay] = []
+  private var displayIndex = 0
+  private var reportedDisplay: Int?
   private let callbackID = NSUUID()
   private let video = videoEncoder()
   private let jpegGate = JpegGate()
@@ -196,16 +209,61 @@ final class SimulatorSource {
   var queue: DispatchQueue { pacer.queue }
 
   func start(deadline: Date = Date().addingTimeInterval(30)) {
-    guard let display = CoreSimulator.displays(udid: udid).first else {
+    let found = CoreSimulator.displays(udid: udid)
+    guard !found.isEmpty else {
       if Date() > deadline { fail("Simulator \(udid) has no display with a framebuffer. Is it booted?") }
       queue.asyncAfter(deadline: .now() + 1) { self.start(deadline: deadline) }
       return
     }
-    self.display = display
-    display.registerDamageCallback(callbackID) { [weak self] _ in self?.pacer.changed() }
-    display.registerSurfacesCallback(callbackID) { [weak self] _ in self?.pacer.changed() }
-    display.registerPropertiesCallback(callbackID) { [weak self] _ in self?.pacer.changed() }
+    displays = found
+    displayIndex = 0
+    for display in found {
+      display.registerDamageCallback(callbackID) { [weak self] _ in self?.pacer.changed() }
+      display.registerSurfacesCallback(callbackID) { [weak self] _ in self?.pacer.changed() }
+      display.registerPropertiesCallback(callbackID) { [weak self] _ in self?.pacer.changed() }
+    }
     pacer.changed()
+    watch()
+  }
+
+  // A simulator that shuts down and boots again gets new display objects, and
+  // the old ones stop reporting damage; so does its HID client. CoreSimulator
+  // returns a new proxy for the same display on every lookup, so the watch
+  // follows the device state instead, where 3 is SimDeviceStateBooted.
+  private func watch() {
+    queue.asyncAfter(deadline: .now() + 2) {
+      guard CoreSimulator.device(udid: self.udid)?.value(forKey: "state") as? Int != 3 else {
+        if self.displays.count > 1 { self.pacer.changed() }
+        return self.watch()
+      }
+      for display in self.displays {
+        display.unregisterDamageCallback(self.callbackID)
+        display.unregisterSurfacesCallback(self.callbackID)
+        display.unregisterPropertiesCallback(self.callbackID)
+      }
+      self.displays = []
+      self.reportedDisplay = nil
+      self.inputQueue.async { self.hid = nil }
+      self.start()
+    }
+  }
+
+  // CoreSimulator keeps the built-in display an iPhone Duo's posture turned
+  // off, the cover or the inner one, all black, and sends no damage when it
+  // turns off, so watch() also re-renders a device with several displays.
+  private func litDisplay() -> SimDisplay? {
+    let panels = displays.indices.filter { displays[$0].screenProperties?.screenType == 0 }
+    guard panels.count > 1 else { return displays.first }
+    let lit = { (index: Int) in self.displays[index].framebufferSurface.map { !isBlack($0) } ?? false }
+    guard let index = lit(displayIndex) ? displayIndex : panels.first(where: lit) else {
+      return displays[displayIndex]
+    }
+    displayIndex = index
+    if reportedDisplay != index {
+      reportedDisplay = index
+      Output.notice(["display": index])
+    }
+    return displays[index]
   }
 
   func configure(_ config: Config) {
@@ -224,10 +282,10 @@ final class SimulatorSource {
   // uiOrientation is a UIInterfaceOrientation; the framebuffer stays in the
   // display's native portrait orientation, so the image is turned upright.
   private func render(_ config: Config) {
-    guard let surface = display?.framebufferSurface else { return }
+    guard let display = litDisplay(), let surface = display.framebufferSurface else { return }
     let orientation: CGImagePropertyOrientation
     let quarterTurns: Int
-    switch display?.screenProperties?.uiOrientation ?? 1 {
+    switch display.screenProperties?.uiOrientation ?? 1 {
     case 2: (orientation, quarterTurns) = (.down, 2)
     case 3: (orientation, quarterTurns) = (.right, 3)
     case 4: (orientation, quarterTurns) = (.left, 1)
@@ -249,12 +307,15 @@ final class SimulatorSource {
 }
 
 final class EmulatorSource {
-  private let serial: String
-  private let queue = DispatchQueue(label: "stim.frames.emulator")
+  let serial: String
+  let queue = DispatchQueue(label: "stim.frames.emulator")
+  let inputQueue = DispatchQueue(label: "stim.frames.emulator-input")
+  var input: EmulatorInput?
+  private var status: (size: CGSize?, keyboard: Bool)?
   private var stream: ScreenshotStream?
   private var generation = 0
   private var config = Config()
-  private var latest: EmulatorFrame?
+  var latest: EmulatorFrame?
   private var pacer: Pacer!
   private let video = videoEncoder()
   private let jpegGate = JpegGate()
@@ -269,6 +330,11 @@ final class EmulatorSource {
 
   func start() {
     queue.async { self.connect() }
+    inputQueue.async {
+      if let status = self.inputClient().flatMap(self.readStatus) {
+        Output.notice(["keyboard": status.keyboard ? "yes" : "no"])
+      }
+    }
   }
 
   func configure(_ config: Config) {
@@ -329,20 +395,58 @@ final class EmulatorSource {
   }
 }
 
-func parseConfig(_ line: Substring, base: Config) -> Config? {
+enum Command {
+  case config(Config)
+  case keyframe
+  case touch(TouchPhase, CGPoint, display: Int)
+  case text(String)
+  case button(String)
+  case rotate(clockwise: Bool)
+  case posture(EmulatorPosture)
+}
+
+func parseCommand(_ line: String, base: Config) -> Command? {
   guard let object = try? JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any] else { return nil }
-  var config = base
-  if let fps = object["fps"] as? Double, fps > 0 { config.fps = min(fps, 60) }
-  if let edge = object["maxEdge"] as? Int, edge > 0 { config.maxEdge = min(edge, 4096) }
-  if let quality = object["quality"] as? Double, quality > 0, quality <= 1 { config.quality = quality }
-  if let jpeg = object["jpeg"] as? Bool { config.jpeg = jpeg }
-  if let jpegFps = object["jpegFps"] as? Double, jpegFps > 0 { config.jpegFps = min(jpegFps, 60) }
-  if let video = object["video"] as? Bool { config.video = video }
-  if let bitrate = object["bitrate"] as? Int, bitrate > 0 { config.bitrate = bitrate }
-  return config
+  if object["keyframe"] as? Bool == true { return .keyframe }
+  switch object["input"] as? String {
+  case "touch":
+    let phases: [String: TouchPhase] = ["down": .down, "move": .move, "up": .up]
+    guard let phase = (object["phase"] as? String).flatMap({ phases[$0] }),
+      let x = object["x"] as? Double, let y = object["y"] as? Double, (0...1).contains(x), (0...1).contains(y)
+    else { return nil }
+    return .touch(phase, CGPoint(x: x, y: y), display: object["display"] as? Int ?? 0)
+  case "text":
+    return (object["text"] as? String).map { .text($0) }
+  case "button":
+    return (object["button"] as? String).map { .button($0) }
+  case "rotate":
+    let directions = ["left": false, "right": true]
+    return (object["direction"] as? String).flatMap { directions[$0] }.map { .rotate(clockwise: $0) }
+  case "posture":
+    let postures: [String: EmulatorPosture] = ["folded": .closed, "half-open": .halfOpened, "unfolded": .opened]
+    return (object["posture"] as? String).flatMap { postures[$0] }.map { .posture($0) }
+  case nil:
+    var config = base
+    if let fps = object["fps"] as? Double, fps >= 0 { config.fps = min(fps, 60) }
+    if let edge = object["maxEdge"] as? Int, edge > 0 { config.maxEdge = min(edge, 4096) }
+    if let quality = object["quality"] as? Double, quality > 0, quality <= 1 { config.quality = quality }
+    if let jpeg = object["jpeg"] as? Bool { config.jpeg = jpeg }
+    if let jpegFps = object["jpegFps"] as? Double, jpegFps > 0 { config.jpegFps = min(jpegFps, 60) }
+    if let video = object["video"] as? Bool { config.video = video }
+    if let bitrate = object["bitrate"] as? Int, bitrate > 0 { config.bitrate = bitrate }
+    return .config(config)
+  default:
+    return nil
+  }
+}
+
+protocol Source: AnyObject {
+  func configure(_ config: Config)
+  func keyframe()
+  func input(_ command: Command)
 }
 
-func readCommands(_ apply: @escaping (Config) -> Void, keyframe: @escaping () -> Void) {
+func readCommands(_ source: Source) {
   Thread.detachNewThread {
     var buffer = Data()
     var config = Config()
@@ -353,19 +457,178 @@ func readCommands(_ apply: @escaping (Config) -> Void, keyframe: @escaping () ->
       while let newline = buffer.firstIndex(of: 0x0a) {
         let line = String(decoding: buffer[buffer.startIndex..<newline], as: UTF8.self)
         buffer.removeSubrange(buffer.startIndex...newline)
-        if line.contains("\"keyframe\"") {
-          if (try? JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any])?["keyframe"] as? Bool == true {
-            keyframe()
-          }
-        } else if let parsed = parseConfig(Substring(line), base: config) {
+        switch parseCommand(line, base: config) {
+        case .config(let parsed)?:
           config = parsed
-          apply(config)
+          source.configure(config)
+        case .keyframe?:
+          source.keyframe()
+        case let command?:
+          source.input(command)
+        case nil:
+          Output.notice(["inputError": "stim-frames could not read \(line.prefix(80))"])
         }
       }
     }
   }
 }
 
+// macOS virtual key codes of a US keyboard, which SimulatorKit's
+// hidUsageForCGKeyCode turns into HID usages and the emulator turns into evdev
+// keys; shifted characters also hold Shift (0x38).
+let keyCodes: [Character: (code: UInt16, shift: Bool)] = {
+  var map: [Character: (UInt16, Bool)] = [:]
+  let plain: [(String, UInt16)] = [
+    ("a", 0x00), ("s", 0x01), ("d", 0x02), ("f", 0x03), ("h", 0x04), ("g", 0x05), ("z", 0x06), ("x", 0x07),
+    ("c", 0x08), ("v", 0x09), ("b", 0x0B), ("q", 0x0C), ("w", 0x0D), ("e", 0x0E), ("r", 0x0F), ("y", 0x10),
+    ("t", 0x11), ("1", 0x12), ("2", 0x13), ("3", 0x14), ("4", 0x15), ("6", 0x16), ("5", 0x17), ("=", 0x18),
+    ("9", 0x19), ("7", 0x1A), ("-", 0x1B), ("8", 0x1C), ("0", 0x1D), ("]", 0x1E), ("o", 0x1F), ("u", 0x20),
+    ("[", 0x21), ("i", 0x22), ("p", 0x23), ("l", 0x25), ("j", 0x26), ("'", 0x27), ("k", 0x28), (";", 0x29),
+    ("\\", 0x2A), (",", 0x2B), ("/", 0x2C), ("n", 0x2D), ("m", 0x2E), (".", 0x2F), ("`", 0x32), (" ", 0x31),
+    ("\n", 0x24), ("\t", 0x30), ("\u{8}", 0x33),
+  ]
+  for (text, code) in plain { map[Character(text)] = (code, false) }
+  for letter in "abcdefghijklmnopqrstuvwxyz" { map[Character(letter.uppercased())] = (map[letter]!.0, true) }
+  let shifted: [(Character, Character)] = [
+    ("!", "1"), ("@", "2"), ("#", "3"), ("$", "4"), ("%", "5"), ("^", "6"), ("&", "7"), ("*", "8"), ("(", "9"),
+    (")", "0"), ("_", "-"), ("+", "="), ("{", "["), ("}", "]"), ("|", "\\"), (":", ";"), ("\"", "'"), ("<", ","),
+    (">", "."), ("?", "/"), ("~", "`"),
+  ]
+  for (character, base) in shifted { map[character] = (map[base]!.0, true) }
+  return map
+}()
+
+extension SimulatorSource: Source {
+  func input(_ command: Command) {
+    inputQueue.async { self.apply(command) }
+  }
+
+  private func apply(_ command: Command) {
+    if case .rotate(let clockwise) = command {
+      if !SimulatorRotation.rotate(udid: udid, clockwise: clockwise) {
+        Output.notice(["inputError": "\(udid) did not take the rotation."])
+      }
+      return
+    }
+    if hid?.isConnected != true { hid = SimulatorHID(udid: udid) }
+    guard let hid else { return Output.notice(["inputError": "\(udid) could not be opened for input."]) }
+    switch command {
+    case .touch(let phase, let point, let index):
+      let displays = CoreSimulator.displays(udid: udid)
+      guard displays.indices.contains(index), let properties = displays[index].screenProperties else {
+        return Output.notice(["inputError": "\(udid) has no display \(index)."])
+      }
+      hid.touch(phase, at: nativeScreenPoint(point, orientation: properties.uiOrientation), screenID: properties.screenID)
+    case .text(let text):
+      for character in text {
+        guard let key = keyCodes[character] else { continue }
+        if key.shift { hid.hardwareKey(code: 0x38, down: true) }
+        hid.hardwareKey(code: key.code, down: true)
+        usleep(10_000)
+        hid.hardwareKey(code: key.code, down: false)
+        if key.shift { hid.hardwareKey(code: 0x38, down: false) }
+        usleep(15_000)
+      }
+    case .button(let name):
+      let buttons: [String: SimulatorButton] = ["home": .home, "lock": .lock]
+      guard let button = buttons[name] else { return Output.notice(["inputError": "iOS has no \(name) button."]) }
+      hid.button(button, down: true)
+      usleep(100_000)
+      hid.button(button, down: false)
+    case .config, .keyframe, .rotate, .posture:
+      break
+    }
+  }
+}
+
+private final class Applied: @unchecked Sendable {
+  var value = false
+}
+
+extension EmulatorSource: Source {
+  func input(_ command: Command) {
+    inputQueue.async { self.apply(command) }
+  }
+
+  private func apply(_ command: Command) {
+    guard let input = inputClient() else { return Output.notice(["inputError": "\(serial) has no gRPC endpoint for input."]) }
+    switch command {
+    case .touch(let phase, let point, let index):
+      guard index == 0 else { return Output.notice(["inputError": "Input goes to the emulator's main display only."]) }
+      guard let size = readStatus(input)?.size else {
+        return Output.notice(["inputError": "\(serial) did not report its display size."])
+      }
+      let rotation = queue.sync { latest?.rotation ?? 0 }
+      let pixel = displayPixel(point, rotation: rotation, displaySize: size)
+      input.call("sendTouch", InputMessages.touch(x: pixel.x, y: pixel.y, pressed: phase != .up))
+    case .text(let text):
+      guard readStatus(input)?.keyboard == true else { return noKeyboard() }
+      for character in text {
+        guard let key = keyCodes[character] else { continue }
+        if key.shift { send(input, InputMessages.key(macKeyCode: 0x38, down: true)) }
+        send(input, InputMessages.key(macKeyCode: key.code, down: true))
+        send(input, InputMessages.key(macKeyCode: key.code, down: false))
+        if key.shift { send(input, InputMessages.key(macKeyCode: 0x38, down: false)) }
+        // The emulator reorders a shifted key and the next one when they arrive back to back.
+        usleep(30_000)
+      }
+    case .button(let name):
+      guard readStatus(input)?.keyboard == true else { return noKeyboard() }
+      let keys = ["home": "GoHome", "back": "GoBack", "app-switch": "AppSwitch", "lock": "Power"]
+      guard let key = keys[name] else { return Output.notice(["inputError": "Android has no \(name) button."]) }
+      input.call("sendKey", InputMessages.namedKey(key))
+    case .rotate(let clockwise):
+      wait("rotation") { await EmulatorRotation.rotate(serial: self.serial, clockwise: clockwise) }
+    case .posture(let posture):
+      wait("posture") { await posture.apply(serial: self.serial) }
+    case .config, .keyframe:
+      break
+    }
+  }
+
+  private func wait(_ change: String, _ operation: @escaping () async -> Bool) {
+    let done = DispatchSemaphore(value: 0)
+    let applied = Applied()
+    Task {
+      applied.value = await operation()
+      done.signal()
+    }
+    if done.wait(timeout: .now() + 10) == .timedOut || !applied.value {
+      Output.notice(["inputError": "\(serial) did not take the \(change) change."])
+    }
+  }
+
+  private func send(_ input: EmulatorInput, _ key: Data) {
+    let done = DispatchSemaphore(value: 0)
+    input.call("sendKey", key) { _ in done.signal() }
+    _ = done.wait(timeout: .now() + 5)
+  }
+
+  private func noKeyboard() {
+    Output.notice(["inputError": "\(serial) has no hardware keyboard (hw.keyboard=no), so it drops key events."])
+  }
+
+  private func inputClient() -> EmulatorInput? {
+    if input == nil, let endpoint = EmulatorDiscovery.endpoint(serial: serial) { input = EmulatorInput(endpoint: endpoint) }
+    return input
+  }
+
+  private func readStatus(_ input: EmulatorInput) -> (size: CGSize?, keyboard: Bool)? {
+    if let status { return status }
+    let done = DispatchSemaphore(value: 0)
+    var response: Data?
+    input.call("getStatus", Data()) { reply in
+      response = reply
+      done.signal()
+    }
+    _ = done.wait(timeout: .now() + 5)
+    guard let response else { return nil }
+    let size = InputMessages.displaySize(fromStatus: response).map { CGSize(width: $0.width, height: $0.height) }
+    status = (size, InputMessages.hasKeyboard(fromStatus: response))
+    return status
+  }
+}
+
 setvbuf(stdout, nil, _IONBF, 0)
 signal(SIGPIPE, SIG_IGN)
 let arguments = CommandLine.arguments
@@ -376,12 +639,12 @@ case "ios":
   guard CoreSimulator.deviceSet != nil else { fail("CoreSimulator could not be loaded from \(CoreSimulator.developerDir).") }
   let source = SimulatorSource(udid: arguments[2])
   Output.requestKeyframe = source.keyframe
-  readCommands({ source.configure($0) }, keyframe: source.keyframe)
+  readCommands(source)
   source.queue.async { source.start() }
 case "android":
   let source = EmulatorSource(serial: arguments[2])
   Output.requestKeyframe = source.keyframe
-  readCommands({ source.configure($0) }, keyframe: source.keyframe)
+  readCommands(source)
   source.start()
 default:
   fail("usage: stim-frames ios <udid> | android <serial>")

package/dist/stim-frames/sim-fold.entitlements

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>com.apple.springboard.sbdisplay.service</key>
+  <true/>
+</dict>
+</plist>

package/dist/stim-frames/sim-fold.m

@@ -0,0 +1,47 @@
+// Folds or unfolds a booted iPhone Duo simulator. It runs inside the
+// simulator through `xcrun simctl spawn <udid> <path>`.
+//
+// SBSDisplayToolService is SpringBoardServices SPI that iOS 27.1 ships. Its
+// swapDisplayWithSettings:completion: sweeps the simulated hinge to the other
+// posture, and SpringBoard accepts it only from a process with the
+// com.apple.springboard.sbdisplay.service entitlement, which simulator
+// processes carry in their __TEXT,__entitlements section. The settings layout
+// matches the method's type encoding {?=qdqqBqdd}.
+#import <Foundation/Foundation.h>
+#import <objc/message.h>
+#include <dlfcn.h>
+
+typedef struct {
+  long long state;
+  double duration;
+  long long orientation;
+  long long profile;
+  BOOL wait;
+  long long targetMode;
+  double position;
+  double lockedUntil;
+} SwapSettings;
+
+int main(void) {
+  dlopen("/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices", RTLD_NOW);
+  Class serviceClass = NSClassFromString(@"SBSDisplayToolService");
+  SEL swap = NSSelectorFromString(@"swapDisplayWithSettings:completion:");
+  if (![serviceClass instancesRespondToSelector:swap]) {
+    fprintf(stderr, "SBSDisplayToolService is not available in this runtime\n");
+    return 2;
+  }
+  id service = [[serviceClass alloc] init];
+  SwapSettings settings = {.duration = 1, .wait = YES};
+  dispatch_semaphore_t done = dispatch_semaphore_create(0);
+  ((void (*)(id, SEL, SwapSettings, id))objc_msgSend)(service, swap, settings, ^{
+    dispatch_semaphore_signal(done);
+  });
+  long timedOut = dispatch_semaphore_wait(done, dispatch_time(DISPATCH_TIME_NOW, 20 * NSEC_PER_SEC));
+  SEL invalidate = NSSelectorFromString(@"invalidate");
+  if ([service respondsToSelector:invalidate]) ((void (*)(id, SEL))objc_msgSend)(service, invalidate);
+  if (timedOut) {
+    fprintf(stderr, "SpringBoard did not finish the fold within 20 seconds\n");
+    return 1;
+  }
+  return 0;
+}