@houwert/conductor 0.25.0 → 0.27.0

package/dist/drivers/bootstrap.js

@@ -6,8 +6,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.detectPlatform = detectPlatform;
 exports.getDriverPort = getDriverPort;
 exports.getInputPort = getInputPort;
+exports.getStreamPort = getStreamPort;
 exports.getInprocDylibPath = getInprocDylibPath;
 exports.getHidBinaryPath = getHidBinaryPath;
+exports.getCaptureBinaryPath = getCaptureBinaryPath;
 exports.installDriver = installDriver;
 exports.isSimulatorBooted = isSimulatorBooted;
 exports.isPortOpen = isPortOpen;
@@ -93,6 +95,7 @@ const ANDROID_BASE_PORT = 3763;
 const WEB_BASE_PORT = 4075;
 const VEGA_BASE_PORT = 5075;
 const INPUT_BASE_PORT = 7075;
+const STREAM_BASE_PORT = 8075;
 const PORT_FILE = path_1.default.join(os_1.default.homedir(), '.conductor', 'ports.json');
 const PORT_LOCK = PORT_FILE + '.lock';
 const PORT_LOCK_TIMEOUT_MS = 5000;
@@ -200,6 +203,27 @@ async function getInputPort(deviceId) {
         return port;
     });
 }
+/**
+ * Assign and persist a streaming-video WebSocket port for a device. Its own
+ * namespace: a device has a driver port (control), an input port (pointer/key
+ * frames), and this stream port (H.264 video fan-out).
+ */
+async function getStreamPort(deviceId) {
+    return withPortLock(() => {
+        const state = readPortState();
+        if (!state.streamAssignments)
+            state.streamAssignments = {};
+        if (state.nextStreamPort === undefined)
+            state.nextStreamPort = STREAM_BASE_PORT;
+        if (state.streamAssignments[deviceId] !== undefined) {
+            return state.streamAssignments[deviceId];
+        }
+        const port = state.nextStreamPort++;
+        state.streamAssignments[deviceId] = port;
+        writePortState(state);
+        return port;
+    });
+}
 // ── Driver paths (bundled dev fallback + runtime download cache) ──────────────
 /**
  * Walk up from __dirname to find the package root (the directory containing
@@ -274,6 +298,20 @@ async function getHidBinaryPath() {
     const p = path_1.default.join(dir, 'ios-hid', 'conductor-hid');
     return fs_1.default.existsSync(p) ? p : null;
 }
+/**
+ * Absolute path to the host-side Simulator video capture binary
+ * (`ios-capture/conductor-capture`), built by
+ * `packages/ios-capture/tools/build-capture.sh`. Captures the framebuffer via
+ * SimulatorKit and serves a VideoToolbox H.264 Annex B stream. Returns null if
+ * it hasn't been built (streaming falls back to unavailable).
+ */
+async function getCaptureBinaryPath() {
+    const dir = await getDriversDir().catch(() => null);
+    if (!dir)
+        return null;
+    const p = path_1.default.join(dir, 'ios-capture', 'conductor-capture');
+    return fs_1.default.existsSync(p) ? p : null;
+}
 async function ensureDriversCache(pkgRoot) {
     const pkgJsonPath = path_1.default.join(pkgRoot, 'package.json');
     const pkg = JSON.parse(fs_1.default.readFileSync(pkgJsonPath, 'utf-8'));

package/dist/drivers/flow-runner.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.parseFlowFile = parseFlowFile;
 exports.parseFlowString = parseFlowString;
 exports.executeFlow = executeFlow;
+exports.resolvePoint = resolvePoint;
 /**
  * Native Conductor YAML flow parser and executor.
  * Parses flow YAML files and executes commands directly using IOSDriver / AndroidDriver.

package/dist/index.js

@@ -30,6 +30,7 @@ const press_key_js_1 = require("./commands/press-key.js");
 const session_js_1 = require("./commands/session.js");
 const daemon_js_1 = require("./commands/daemon.js");
 const input_server_js_1 = require("./commands/input-server.js");
+const stream_server_js_1 = require("./commands/stream-server.js");
 const install_js_1 = require("./commands/install.js");
 const init_js_1 = require("./commands/init.js");
 const device_pool_js_1 = require("./commands/device-pool.js");
@@ -65,6 +66,14 @@ const metro_js_1 = require("./commands/metro.js");
 const clipboard_js_1 = require("./commands/clipboard.js");
 const options_js_1 = require("./commands/options.js");
 const web_targets_js_1 = require("./commands/web-targets.js");
+const copy_text_from_js_1 = require("./commands/copy-text-from.js");
+const assert_true_js_1 = require("./commands/assert-true.js");
+const set_permissions_js_1 = require("./commands/set-permissions.js");
+const add_media_js_1 = require("./commands/add-media.js");
+const airplane_mode_js_1 = require("./commands/airplane-mode.js");
+const travel_js_1 = require("./commands/travel.js");
+const record_video_js_1 = require("./commands/record-video.js");
+const assert_screenshot_js_1 = require("./commands/assert-screenshot.js");
 const session_js_2 = require("./session.js");
 const device_picker_js_1 = require("./device-picker.js");
 const cdp_discovery_js_1 = require("./drivers/cdp-discovery.js");
@@ -108,6 +117,7 @@ const COMMAND_HELP = {
     'clear-state': clear_state_js_1.HELP,
     'uninstall-app': uninstall_app_js_1.HELP,
     'tap-on': tap_js_1.HELP,
+    'copy-text-from': copy_text_from_js_1.HELP,
     'input-text': type_js_1.HELP,
     'erase-text': erase_text_js_1.HELP,
     back: back_js_1.HELP,
@@ -118,8 +128,15 @@ const COMMAND_HELP = {
     'scroll-until-visible': scroll_until_visible_js_1.HELP,
     'assert-visible': assert_visible_js_1.HELP,
     'assert-not-visible': assert_not_visible_js_1.HELP,
+    'assert-true': assert_true_js_1.HELP,
+    'assert-screenshot': assert_screenshot_js_1.HELP,
     'open-link': open_link_js_1.HELP,
     'set-location': set_location_js_1.HELP,
+    'set-permissions': set_permissions_js_1.HELP,
+    'add-media': add_media_js_1.HELP,
+    'set-airplane-mode': airplane_mode_js_1.HELP,
+    travel: travel_js_1.HELP,
+    'record-video': record_video_js_1.HELP,
     'set-orientation': set_orientation_js_1.HELP,
     'set-viewport': set_viewport_js_1.HELP,
     'take-screenshot': screenshot_js_1.HELP,
@@ -135,6 +152,7 @@ const COMMAND_HELP = {
     'daemon-stop': daemon_js_1.HELP_DAEMON_STOP,
     'daemon-status': daemon_js_1.HELP_DAEMON_STATUS,
     'input-server': input_server_js_1.HELP,
+    'stream-server': stream_server_js_1.HELP,
     'device-pool': device_pool_js_1.HELP,
     'run-parallel': run_parallel_js_1.HELP,
     'run-sequence': run_sequence_js_1.HELP,
@@ -204,6 +222,7 @@ async function main() {
             'global',
             'force',
             'yes',
+            'update',
         ],
         string: [
             'device',
@@ -268,6 +287,11 @@ async function main() {
             'react-tag',
             'path',
             'value',
+            'repeat',
+            'delay',
+            'speed',
+            'threshold',
+            'reference',
         ],
         alias: { h: 'help', v: 'verbose', V: 'version', o: 'output', y: 'yes' },
     });
@@ -313,6 +337,8 @@ async function main() {
         'workspace',
         'list-options',
         'web-targets',
+        // assert-true evaluates a pure JS expression in the flow sandbox — no device involved.
+        'assert-true',
         // `logs --list` and `logs --source metro` only query Metro on localhost — no device needed
         // `logs` always needs a device session — Metro discovery is device-scoped.
         // `daemon-stop --all` stops every daemon — no device needed
@@ -585,6 +611,9 @@ async function main() {
             exitCode = await (0, tap_js_1.tap)(element, opts, sessionName, {
                 id: argv['id'],
                 text: argv['text'],
+                at: argv['at'],
+                repeat: argv['repeat'] !== undefined ? Number(argv['repeat']) : undefined,
+                delay: argv['delay'] !== undefined ? Number(argv['delay']) : undefined,
                 index: argv['index'] !== undefined ? Number(argv['index']) : undefined,
                 longPress: argv['long-press'],
                 doubleTap: argv['double-tap'],
@@ -700,6 +729,35 @@ async function main() {
             });
             break;
         }
+        case 'assert-true': {
+            const expr = rest.join(' ');
+            const env = {};
+            const envArgs = [].concat(argv['env'] ?? []);
+            for (const e of envArgs) {
+                const idx = e.indexOf('=');
+                if (idx > 0)
+                    env[e.slice(0, idx)] = e.slice(idx + 1);
+            }
+            exitCode = await (0, assert_true_js_1.assertTrue)(expr, opts, env);
+            break;
+        }
+        case 'assert-screenshot': {
+            const reference = rest[0] ?? argv['reference'] ?? '';
+            exitCode = await (0, assert_screenshot_js_1.assertScreenshot)(reference, opts, sessionName, {
+                threshold: argv['threshold'] !== undefined ? Number(argv['threshold']) : undefined,
+                update: argv['update'],
+            });
+            break;
+        }
+        case 'copy-text-from': {
+            const element = rest.join(' ');
+            exitCode = await (0, copy_text_from_js_1.copyTextFrom)(element, opts, sessionName, {
+                id: argv['id'],
+                text: argv['text'],
+                index: argv['index'] !== undefined ? Number(argv['index']) : undefined,
+            });
+            break;
+        }
         case 'open-link': {
             const url = rest[0] ?? argv['url'] ?? '';
             exitCode = await (0, open_link_js_1.openLink)(url, opts, sessionName);
@@ -717,6 +775,35 @@ async function main() {
             }
             break;
         }
+        case 'set-permissions': {
+            exitCode = await (0, set_permissions_js_1.setPermissions)(rest.map(String), opts, sessionName);
+            break;
+        }
+        case 'add-media': {
+            exitCode = await (0, add_media_js_1.addMedia)(rest.map(String), opts, sessionName);
+            break;
+        }
+        case 'set-airplane-mode': {
+            const value = rest[0] ?? '';
+            exitCode = await (0, airplane_mode_js_1.setAirplaneMode)(value, opts, sessionName);
+            break;
+        }
+        case 'toggle-airplane-mode': {
+            exitCode = await (0, airplane_mode_js_1.toggleAirplaneMode)(opts, sessionName);
+            break;
+        }
+        case 'travel': {
+            exitCode = await (0, travel_js_1.travel)(rest.map(String), opts, sessionName, {
+                speed: argv['speed'] !== undefined ? Number(argv['speed']) : undefined,
+            });
+            break;
+        }
+        case 'record-video': {
+            exitCode = await (0, record_video_js_1.recordVideo)(rest[0] ?? '', opts, sessionName, {
+                out: argv['out'],
+            });
+            break;
+        }
         case 'set-orientation': {
             const orientation = (rest[0] ??
                 argv['orientation'] ??
@@ -847,6 +934,9 @@ async function main() {
         case 'input-server':
             exitCode = await (0, input_server_js_1.inputServer)(opts, sessionName);
             break;
+        case 'stream-server':
+            exitCode = await (0, stream_server_js_1.streamServer)(opts, sessionName);
+            break;
         case 'device-pool': {
             const acquire = argv['acquire'];
             const release = argv['release'];

package/dist/runner.js

@@ -4,6 +4,7 @@ exports.detectFirstDevice = detectFirstDevice;
 exports.getDriver = getDriver;
 exports.prewarmDriver = prewarmDriver;
 exports.inputServerInfo = inputServerInfo;
+exports.streamServerInfo = streamServerInfo;
 exports.runDirect = runDirect;
 exports.spawnCommand = spawnCommand;
 exports.runInlineFlow = runInlineFlow;
@@ -276,6 +277,50 @@ async function inputServerInfo(sessionName = 'default') {
     }
     throw new Error(`Input server for ${deviceId} did not come up within timeout.`);
 }
+/**
+ * Resolve the streaming-video socket for a session's device, starting the
+ * daemon (and its driver + video server) if needed. Returns the loopback
+ * WebSocket URL a viewer subscribes to. Throws if the platform has no live
+ * stream, the capture binary isn't built, or the port never comes up.
+ */
+async function streamServerInfo(sessionName = 'default') {
+    const deviceId = await resolveDeviceId(sessionName);
+    if (!deviceId) {
+        throw new Error('No device found. Connect a device or start a simulator, then run again.');
+    }
+    const platform = await (0, bootstrap_js_1.detectPlatform)(deviceId);
+    if (platform !== 'ios' && platform !== 'tvos') {
+        throw new Error(`Live video streaming is not yet available for ${platform} devices.`);
+    }
+    await (0, client_js_1.startDaemon)(deviceId);
+    // The video server starts just after the input server — poll status until its port appears.
+    const start = Date.now();
+    const deadline = start + 60000;
+    while (Date.now() < deadline) {
+        const status = await (0, client_js_1.fetchDaemonStatus)(deviceId);
+        if (status && typeof status.streamPort === 'number') {
+            return {
+                device: deviceId,
+                platform,
+                streamPort: status.streamPort,
+                url: `ws://127.0.0.1:${status.streamPort}/stream?device=${encodeURIComponent(deviceId)}&platform=${platform}`,
+                codec: 'h264',
+            };
+        }
+        // A daemon whose input server is up but that still reports no streamPort
+        // after a grace period means the capture binary isn't built/available —
+        // fail fast rather than waiting out the full timeout.
+        if (status &&
+            status.streamPort === null &&
+            typeof status.inputPort === 'number' &&
+            Date.now() - start > 5000) {
+            throw new Error(`Video capture backend is not available for ${deviceId} ` +
+                `(the conductor-capture binary is missing from the installed drivers).`);
+        }
+        await new Promise((r) => setTimeout(r, 500));
+    }
+    throw new Error(`Video server for ${deviceId} did not come up within timeout.`);
+}
 /**
  * Execute a function with the driver for the given session.
  * Returns a RunResult for consistent error handling across commands.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@houwert/conductor",
-  "version": "0.25.0",
+  "version": "0.27.0",
   "description": "CLI tool for mobile app interactions — optimized for AI agents",
   "license": "MIT",
   "repository": {
@@ -33,13 +33,16 @@
     "@grpc/proto-loader": "^0.7.15",
     "js-yaml": "^4.1.1",
     "minimist": "^1.2.8",
+    "pixelmatch": "^7.2.0",
     "playwright-core": "^1.52.0",
+    "pngjs": "^7.0.0",
     "ws": "^8.20.0"
   },
   "devDependencies": {
     "@types/js-yaml": "^4.0.9",
     "@types/minimist": "^1.2.5",
     "@types/node": "^20.0.0",
+    "@types/pngjs": "^6.0.5",
     "@types/ws": "^8.18.1",
     "@typescript-eslint/eslint-plugin": "^8.56.1",
     "@typescript-eslint/parser": "^8.56.1",

package/skills/conductor-device-interact/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: conductor-device-interact
-description: Drive a running iOS simulator, Android emulator, tvOS simulator, Vega (Amazon Fire TV) virtual device, or Playwright web app with the conductor CLI. Use when launching apps, tapping UI elements, typing text, scrolling/swiping, performing gestures, pressing hardware/keyboard/remote keys, opening URLs or deep links, navigating back, or verifying an app change in the real running app.
+description: Drive a running iOS simulator, Android emulator, tvOS simulator, Vega (Amazon Fire TV) virtual device, or Playwright web app with the conductor CLI. Use when launching apps, tapping UI elements (by selector or raw coordinate), typing text, scrolling/swiping, performing gestures, pressing hardware/keyboard/remote keys, opening URLs or deep links, navigating back, granting/denying app permissions, adding media to the gallery, setting GPS location or a travel route, toggling airplane mode, recording a screen video, or verifying an app change in the real running app.
 ---
 
 # Conductor — device interaction
@@ -36,7 +36,9 @@ conductor assert-visible "Dashboard"
 |---|---|
 | `conductor launch-app <appId>` | Launch app (saved to session). `--no-stop-app` resumes; `--argument key=value` passes launch args |
 | `conductor stop-app [<appId>]` | Stop the app |
-| `conductor tap-on <element>` | Tap by text, id, or `@eN`. `--long-press`, `--double-tap`, `--optional`, `--index <n>` |
+| `conductor tap-on <element>` | Tap by text, id, or `@eN`. `--long-press`, `--double-tap`, `--optional`, `--index <n>`, `--repeat <n> --delay <ms>` |
+| `conductor tap-on --at <x,y>` | Tap a raw coordinate (px `100,200`, percent `50%,50%`, or `0-1` fraction) — no element match |
+| `conductor copy-text-from <element>` | Print an element's text (and copy to the iOS clipboard) |
 | `conductor input-text <text>` | Type into the focused field |
 | `conductor erase-text [n]` | Erase n characters (default 50) |
 | `conductor press-key <key>` | Press a key (Enter, Backspace, Home, …) or a remote button (`Remote Dpad Up/Down/Left/Right/Center`, `Remote Menu`) for tvOS / Android TV / vega. `--long-press` / `--duration <seconds>` holds it |
@@ -49,9 +51,15 @@ conductor assert-visible "Dashboard"
 | `conductor pinch [--scale N] [--center x,y]` | Two-finger pinch (scale<1 out, >1 in) |
 | `conductor rotate-gesture [--degrees N] [--center x,y]` | Two-finger rotate |
 | `conductor gesture <json\|--file path>` | Play a multi-touch path |
+| `conductor set-permissions <perm=value>...` | Grant/deny app permissions (`camera=allow photos=deny`, `all=allow`) |
+| `conductor add-media <path>...` | Push image/video files into the device gallery |
+| `conductor set-airplane-mode <on\|off>` / `toggle-airplane-mode` | Airplane mode (Android only) |
+| `conductor travel <lat,lng>... [--speed <m/s>]` | Move GPS through a route of coordinates |
+| `conductor record-video start [--out <path>]` / `record-video stop` | Record a screen **video** (iOS/Android). Distinct from `flow record` (which records a YAML flow) |
 | `conductor clipboard read` / `clipboard write <text>` / `paste` | Clipboard (iOS) |
 | `conductor list-options [command]` | List valid values for enumerated params |
 | `conductor input-server` | Start (if needed) and print the streaming-input WebSocket URL for the device |
+| `conductor stream-server` | Start (if needed) and print the live video-stream WebSocket URL for the device |
 
 ## Streaming input (host IDEs)
 
@@ -65,6 +73,20 @@ accepts normalized (0..1) frames: `pointer{id,phase,x,y}`, `key{code,mods,down}`
 owns coord→device translation and keymaps. For scripted, one-off actions use the
 discrete commands above — this is for interactive host UIs.
 
+## Streaming video (host IDEs / device viewers)
+
+For live device mirroring, `conductor stream-server` ensures the daemon + capture
+backend are up and prints the loopback URL
+(`ws://127.0.0.1:<port>/stream?device=<id>&platform=<ios|tvos|android|web>`; also
+in `daemon-status --json` as `streamPort`). One capture fans out to N subscribers.
+On connect the server sends a JSON `config` frame
+(`{t:"config",codec:"h264",width,height,rotation,fps,sps,pps,avcC,codecString}`),
+then **binary** frames — each one H.264 Annex B access unit, keyframe-led; a late
+joiner gets the cached config + a keyframe immediately. iOS/tvOS only for now
+(host-side SimulatorKit → VideoToolbox capture); Android/web are follow-ons.
+This is capture only — input stays on `input-server`. For a still image use
+`screenshot` / `capture-ui`; this socket is for continuous low-latency mirroring.
+
 ## Discovering valid values
 
 Several commands only accept a fixed set of values (`press-key <key>`,

package/skills/conductor-inspect/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: conductor-inspect
-description: Read the live UI state of a running app with the conductor CLI — view hierarchy, accessibility snapshot, screenshots, focused element, and element refs. Use when you need to see what's on screen, find an element's id/text/coordinates, take a screenshot, check focus, or assert that something is (or isn't) visible before or after acting.
+description: Read the live UI state of a running app with the conductor CLI — view hierarchy, accessibility snapshot, screenshots, focused element, and element refs. Use when you need to see what's on screen, find an element's id/text/coordinates, take a screenshot, check focus, assert that something is (or isn't) visible, assert a JavaScript expression, or do visual-regression screenshot comparison before or after acting.
 ---
 
 # Conductor — inspection & assertions
@@ -78,6 +78,8 @@ conductor native-image 816,286,288,288 --output /tmp/avatar.png
 |---|---|
 | `conductor assert-visible <element> [--timeout ms]` | Assert element is visible (non-zero exit on failure) |
 | `conductor assert-not-visible <element> [--timeout ms]` | Assert element is absent |
+| `conductor assert-true <expr> [--env K=V]` | Assert a JavaScript expression is truthy (no device needed) |
+| `conductor assert-screenshot <reference.png> [--threshold <0-1>] [--update]` | Visual regression vs a baseline image; `--update` (re)writes the baseline. Writes `<ref>.diff.png` on mismatch |
 
 Both take the same selectors as `tap-on`: `--id`, `--text`, `--index`,
 `--below` / `--above` / `--left-of` / `--right-of`, `--focused`, `--enabled`,