@elizaos/plugin-computeruse 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/mobile/ios-bridge.d.ts

@@ -0,0 +1,359 @@
+/**
+ * iOS computer-use bridge — Capacitor JS↔Swift contract.
+ *
+ * iOS does not let third-party apps drive other apps. The only surfaces
+ * available to us are:
+ *
+ *   1. ReplayKit foreground capture     — own-app screen frames
+ *   2. ReplayKit broadcast extension    — system-wide capture (~50MB ceiling,
+ *                                         iOS 26/26.1 beta regression noted)
+ *   3. Apple Vision OCR                 — VNRecognizeTextRequest, on-device
+ *   4. App Intents                      — invoke other apps' Shortcuts-style
+ *                                         intents (the only sanctioned UI driver)
+ *   5. UIAccessibility (own app only)
+ *   6. Apple Foundation Models (iOS 26) — opportunistic on-device LLM
+ *
+ * What is NOT exposed (and never will be on stock iOS):
+ *   - Cross-app input or pixel scraping
+ *   - Process listing of other running apps
+ *   - Background inference past ~30s outside `BGProcessingTask`
+ *
+ * See `eliza/plugins/plugin-computeruse/docs/IOS_CONSTRAINTS.md` for the
+ * honest scope discussion and per-method validation checklist.
+ *
+ * The Swift counterpart lives at
+ * `apps/app/ios/App/App/ComputerUseBridge.swift`. The `// MARK: - Contract`
+ * block in that file mirrors the signatures below verbatim — keep them in
+ * lock-step or the bridge silently drifts.
+ */
+/**
+ * Standard result envelope for every bridge call.
+ *
+ * `ok=true` means the native side completed successfully and `data` is shaped
+ * to the per-method type. `ok=false` carries a `code` (machine-readable) and
+ * `message` (human-readable). No fallbacks, no silent defaults — if the call
+ * failed, the JS caller sees it.
+ */
+export type IosBridgeResult<T> = {
+    ok: true;
+    data: T;
+} | {
+    ok: false;
+    code: IosBridgeErrorCode;
+    message: string;
+};
+export type IosBridgeErrorCode = "unsupported_platform" | "permission_denied" | "permission_pending" | "extension_unavailable" | "extension_died" | "memory_pressure" | "intent_not_found" | "intent_invocation_failed" | "vision_no_text" | "foundation_model_unavailable" | "internal_error";
+/**
+ * Configures `RPScreenRecorder.shared().startCapture(handler:completionHandler:)`.
+ *
+ * `frameRate` caps how often the Swift side forwards frames to JS — Apple
+ * delivers at display refresh rate, we throttle. `maxDurationSec` is a
+ * hard upper bound; anything past 30s requires `BGProcessingTask` and is
+ * not supported by this method.
+ */
+export interface ReplayKitForegroundOptions {
+    /** Hard cap, default 1. Apple delivers at display refresh; we drop frames. */
+    readonly frameRate?: number;
+    /** Max session duration in seconds, default 30. Capped at 30 server-side. */
+    readonly maxDurationSec?: number;
+    /** Include audio sample buffers (mic). Default false. */
+    readonly includeAudio?: boolean;
+}
+export interface ReplayKitForegroundFrame {
+    /** Monotonic capture timestamp from CMSampleBuffer presentation time, ns. */
+    readonly timestampNs: number;
+    /** Pixel width in image-buffer coordinates. */
+    readonly width: number;
+    /** Pixel height. */
+    readonly height: number;
+    /** Base64-encoded JPEG of the frame at quality 0.7. */
+    readonly jpegBase64: string;
+}
+export interface ReplayKitForegroundHandle {
+    /** Opaque session id; pass to `replayKitForegroundStop`. */
+    readonly sessionId: string;
+    /** Echoed effective options after server-side clamping. */
+    readonly effective: Required<ReplayKitForegroundOptions>;
+}
+/**
+ * Handshake with the broadcast extension. Returns whether the extension is
+ * registered, the App Group container path used to stream frames, and a
+ * pre-flight memory headroom check.
+ *
+ * The extension itself runs in a separate process with a ~50MB ceiling. The
+ * main app cannot start the extension programmatically — the user must tap
+ * the system share-sheet broadcast picker. This call only verifies the
+ * pipeline is wired and the App Group container is writable.
+ *
+ * iOS 26 / 26.1 beta regression: extensions are killed within ~3 seconds even
+ * when memory is well under the limit. The bridge surfaces this as
+ * `extension_died` with `details.regressionFB` referencing the Apple feedback
+ * id when known. The caller should fall back to foreground capture.
+ */
+export interface BroadcastHandshakeResult {
+    /** True if the broadcast extension target is bundled with this build. */
+    readonly extensionInstalled: boolean;
+    /** App Group identifier (e.g. `group.com.elizaai.eliza`). */
+    readonly appGroupId: string;
+    /** Absolute path to the App Group container used for IPC frames. */
+    readonly sharedContainerPath: string;
+    /** Available memory in MB the extension would have at start time. */
+    readonly availableMemoryMb: number;
+    /** True if iOS reports an active broadcast session. */
+    readonly broadcastActive: boolean;
+    /** Last known iOS-26-beta regression status, if observed. */
+    readonly regression?: {
+        readonly observed: boolean;
+        readonly note: string;
+    };
+}
+export interface VisionOcrOptions {
+    /**
+     * ISO 639-1 language hints. Vision falls back to autodetect when omitted.
+     * Examples: ["en-US"], ["zh-Hans","en-US"].
+     */
+    readonly languages?: readonly string[];
+    /**
+     * "fast" uses a smaller model with lower latency (~80ms typical),
+     * "accurate" runs the higher-quality model (~250ms typical).
+     */
+    readonly recognitionLevel?: "fast" | "accurate";
+    /** Minimum text height as a fraction of image height. Default 0.0 (off). */
+    readonly minimumTextHeight?: number;
+    /**
+     * Whether to use language-correction post-pass. Default true on iOS 16+.
+     * Adds ~30ms but materially improves CJK and handwriting accuracy.
+     */
+    readonly usesLanguageCorrection?: boolean;
+}
+export interface VisionOcrLine {
+    readonly text: string;
+    readonly confidence: number;
+    /** Normalized 0..1 origin-bottom-left rect, matching VNRectangleObservation. */
+    readonly boundingBox: {
+        readonly x: number;
+        readonly y: number;
+        readonly width: number;
+        readonly height: number;
+    };
+}
+export interface VisionOcrResult {
+    readonly lines: readonly VisionOcrLine[];
+    readonly fullText: string;
+    /** Total time spent in VNImageRequestHandler.perform, ms. */
+    readonly elapsedMs: number;
+    /** Languages Vision actually used. */
+    readonly languagesUsed: readonly string[];
+}
+/**
+ * Parameter spec for an AppIntent. Mirrors the subset of `IntentParameter`
+ * shapes we can portably express across iOS 16+ AppIntent and the legacy
+ * Intents framework. The Swift side validates and rejects extras.
+ */
+export interface IntentParameterSpec {
+    readonly name: string;
+    readonly type: "string" | "number" | "boolean" | "date" | "url" | "enum";
+    readonly required: boolean;
+    /** Present when type === "enum". */
+    readonly enumValues?: readonly string[];
+    /** Human-readable description for the planner. */
+    readonly description?: string;
+}
+export interface IntentSpec {
+    /** Bundle id of the owning app, e.g. `com.apple.mobilenotes`. */
+    readonly bundleId: string;
+    /** Reverse-DNS intent identifier, e.g. `com.apple.mobilenotes.create-note`. */
+    readonly id: string;
+    /** Display name surfaced in the planner. */
+    readonly displayName: string;
+    /** Free-form summary used as planner context. */
+    readonly summary: string;
+    readonly parameters: readonly IntentParameterSpec[];
+    /**
+     * `donated` means the app has run at least once and donated this intent to
+     * Shortcuts on this device. `system` is a known-stable Apple intent that
+     * ships with the OS.
+     */
+    readonly source: "donated" | "system";
+}
+export interface IntentInvocationRequest {
+    readonly intentId: string;
+    readonly parameters: Readonly<Record<string, IntentParameterValue>>;
+}
+export type IntentParameterValue = string | number | boolean | {
+    readonly kind: "date";
+    readonly iso: string;
+} | {
+    readonly kind: "url";
+    readonly url: string;
+};
+export interface IntentInvocationResult {
+    readonly intentId: string;
+    /** Whether iOS reported the intent ran end-to-end. */
+    readonly success: boolean;
+    /** Optional structured response payload from the intent. */
+    readonly response?: Readonly<Record<string, unknown>>;
+    /** Time from invocation to completion, ms. */
+    readonly elapsedMs: number;
+}
+export interface AccessibilitySnapshotNode {
+    readonly id: string;
+    readonly role: string;
+    readonly label?: string;
+    readonly value?: string;
+    readonly isFocused: boolean;
+    readonly children: readonly AccessibilitySnapshotNode[];
+}
+export interface AccessibilitySnapshotResult {
+    readonly screenName: string;
+    readonly tree: AccessibilitySnapshotNode;
+    readonly capturedAt: number;
+}
+export interface FoundationModelOptions {
+    /** Sampling temperature in [0,1]. Default 0.2 for deterministic-ish output. */
+    readonly temperature?: number;
+    /** Max generated tokens. Default 256. */
+    readonly maxTokens?: number;
+    /** System-style instruction; mapped to the iOS 26 system role. */
+    readonly instruction?: string;
+}
+export interface FoundationModelResult {
+    readonly text: string;
+    readonly tokensIn: number;
+    readonly tokensOut: number;
+    readonly elapsedMs: number;
+}
+/**
+ * Strict TS interface for the Capacitor plugin. Every method returns an
+ * `IosBridgeResult<T>` — Swift never throws across the bridge.
+ *
+ * The Capacitor plugin name on the Swift side is `ComputerUseBridge`. The
+ * `jsName` is `ComputerUse`. Method names below are the exact strings
+ * registered with `CAPPluginMethod`.
+ */
+export interface IosComputerUseBridge {
+    /** `{ available: true }` if iOS 26+ with all four targets bundled. */
+    readonly probe: () => Promise<IosBridgeResult<IosBridgeProbe>>;
+    readonly replayKitForegroundStart: (options?: ReplayKitForegroundOptions) => Promise<IosBridgeResult<ReplayKitForegroundHandle>>;
+    /** Stops capture and discards the in-flight buffer. */
+    readonly replayKitForegroundStop: (args: {
+        readonly sessionId: string;
+    }) => Promise<IosBridgeResult<{
+        readonly sessionId: string;
+    }>>;
+    /**
+     * Drains the next batch of frames since last drain. Returns up to `max`
+     * frames. The Swift side ring-buffers at most 30 frames; older ones drop.
+     */
+    readonly replayKitForegroundDrain: (args: {
+        readonly sessionId: string;
+        readonly max?: number;
+    }) => Promise<IosBridgeResult<{
+        readonly frames: readonly ReplayKitForegroundFrame[];
+    }>>;
+    readonly broadcastExtensionHandshake: () => Promise<IosBridgeResult<BroadcastHandshakeResult>>;
+    readonly visionOcr: (args: {
+        /** Base64-encoded image (PNG or JPEG), no data URI prefix. */
+        readonly imageBase64: string;
+        readonly options?: VisionOcrOptions;
+    }) => Promise<IosBridgeResult<VisionOcrResult>>;
+    /** Lists intents iOS knows about for the given bundle ids (or all donated). */
+    readonly appIntentList: (args: {
+        readonly bundleIds?: readonly string[];
+    }) => Promise<IosBridgeResult<{
+        readonly intents: readonly IntentSpec[];
+    }>>;
+    readonly appIntentInvoke: (request: IntentInvocationRequest) => Promise<IosBridgeResult<IntentInvocationResult>>;
+    readonly accessibilitySnapshot: () => Promise<IosBridgeResult<AccessibilitySnapshotResult>>;
+    readonly foundationModelGenerate: (args: {
+        readonly prompt: string;
+        readonly options?: FoundationModelOptions;
+    }) => Promise<IosBridgeResult<FoundationModelResult>>;
+    /**
+     * One-shot read of `os_proc_available_memory()` plus a digest of recent
+     * `UIApplicationDidReceiveMemoryWarningNotification` events. The arbiter
+     * also subscribes to push events via `addPressureListener`.
+     */
+    readonly memoryPressureProbe: () => Promise<IosBridgeResult<MemoryPressureSample>>;
+}
+export interface IosBridgeProbe {
+    readonly platform: "ios";
+    /** e.g. "26.1" — `UIDevice.current.systemVersion`. */
+    readonly osVersion: string;
+    /** Capability matrix discovered at probe time. */
+    readonly capabilities: {
+        readonly replayKitForeground: boolean;
+        readonly broadcastExtension: boolean;
+        readonly visionOcr: boolean;
+        readonly appIntents: boolean;
+        readonly accessibilityRead: boolean;
+        readonly foundationModel: boolean;
+    };
+}
+/**
+ * Thin contract WS1 owns. The iOS bridge implements the producer side; WS1's
+ * arbiter consumes via this shape. Defined here so plugin-computeruse can
+ * publish samples without taking a runtime dep on the arbiter package.
+ */
+export interface IPressureSignal {
+    readonly source: "ios-uikit" | "android-low-memory" | "host-os" | "synthetic";
+    readonly capturedAt: number;
+    /** 0 = nominal, 1 = critical. iOS warning notification → 0.7. */
+    readonly severity: number;
+    /** Available process memory in MB, when the source can report it. */
+    readonly availableMb?: number;
+    /** Free-form details for diagnostics. Never load-bearing for arbiter logic. */
+    readonly details?: Readonly<Record<string, string | number | boolean>>;
+}
+export interface MemoryPressureSample extends IPressureSignal {
+    readonly source: "ios-uikit";
+    /** Last received `UIApplicationDidReceiveMemoryWarningNotification`, ms epoch. */
+    readonly lastWarningAt?: number;
+    /** Available process memory in MB from `os_proc_available_memory`. */
+    readonly availableMb: number;
+    /** True if the broadcast extension is currently active. */
+    readonly broadcastActive: boolean;
+}
+/** App Group identifier — must match the canonical
+ *  `eliza/packages/app-core/platforms/ios/App/App/App.entitlements`. */
+export declare const IOS_APP_GROUP_ID: "group.ai.elizaos.app";
+/** Capacitor plugin jsName (Capacitor injects `Capacitor.Plugins.ComputerUse`). */
+export declare const IOS_BRIDGE_JS_NAME: "ComputerUse";
+/** Maximum frames buffered server-side for ReplayKit foreground draining. */
+export declare const REPLAYKIT_FOREGROUND_MAX_BUFFER: 30;
+/** Hard cap on session duration; anything beyond requires BGProcessingTask. */
+export declare const REPLAYKIT_FOREGROUND_MAX_SESSION_SEC: 30;
+/**
+ * Result of `featureCheck()`. Callers use this to decide whether to invoke
+ * any other method on the bridge — when `supported` is false they should fall
+ * back to OCR / external orchestration instead of letting the call throw.
+ */
+export interface IosFeatureCheckResult {
+    readonly supported: boolean;
+    readonly reason?: string;
+}
+/**
+ * Runtime feature-detect for the iOS bridge. Synchronous and side-effect-free
+ * so it can run during planner setup. Returns `{supported:false}` whenever
+ * the JS↔Swift bridge cannot be reached:
+ *
+ *   - Running on a non-iOS host (desktop, Android, browser).
+ *   - Capacitor is not initialized (no `globalThis.Capacitor`).
+ *   - The Swift `ComputerUseBridge` plugin is not registered.
+ *
+ * This does NOT call `bridge.probe()` — that is async and may prompt the user
+ * for permissions. Callers that need the per-capability matrix should `await
+ * bridge.probe()` after `featureCheck().supported === true`.
+ *
+ * PARITY: untested on hardware — the lookup path mirrors what the Capacitor
+ * runtime does, but has not been validated against a real iOS build.
+ */
+export declare function featureCheck(): IosFeatureCheckResult;
+/**
+ * Resolve the live bridge handle, or `null` when unavailable. Pairs with
+ * `featureCheck()` for callers that want a typed handle in one step.
+ *
+ * PARITY: untested on hardware — feature-detect at runtime.
+ */
+export declare function getIosBridge(): IosComputerUseBridge | null;
+//# sourceMappingURL=ios-bridge.d.ts.map

package/dist/mobile/ios-bridge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ios-bridge.d.ts","sourceRoot":"","sources":["../../src/mobile/ios-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAIH;;;;;;;GAOG;AACH,MAAM,MAAM,eAAe,CAAC,CAAC,IACzB;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,CAAC,CAAA;CAAE,GACrB;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,kBAAkB,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAE7D,MAAM,MAAM,kBAAkB,GAC1B,sBAAsB,GACtB,mBAAmB,GACnB,oBAAoB,GACpB,uBAAuB,GACvB,gBAAgB,GAChB,iBAAiB,GACjB,kBAAkB,GAClB,0BAA0B,GAC1B,gBAAgB,GAChB,8BAA8B,GAC9B,gBAAgB,CAAC;AAIrB;;;;;;;GAOG;AACH,MAAM,WAAW,0BAA0B;IACzC,8EAA8E;IAC9E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,6EAA6E;IAC7E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,yDAAyD;IACzD,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;CACjC;AAED,MAAM,WAAW,wBAAwB;IACvC,6EAA6E;IAC7E,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+CAA+C;IAC/C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,oBAAoB;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,yBAAyB;IACxC,4DAA4D;IAC5D,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,2DAA2D;IAC3D,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,0BAA0B,CAAC,CAAC;CAC1D;AAID;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,wBAAwB;IACvC,yEAAyE;IACzE,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,6DAA6D;IAC7D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,qEAAqE;IACrE,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,uDAAuD;IACvD,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAClC,6DAA6D;IAC7D,QAAQ,CAAC,UAAU,CAAC,EAAE;QACpB,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;QAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAID,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC;;;OAGG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;IAChD,4EAA4E;IAC5E,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;;;OAGG;IACH,QAAQ,CAAC,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAC3C;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,gFAAgF;IAChF,QAAQ,CAAC,WAAW,EAAE;QACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,SAAS,aAAa,EAAE,CAAC;IACzC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,6DAA6D;IAC7D,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,sCAAsC;IACtC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;CAC3C;AAID;;;;GAIG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;IACzE,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,oCAAoC;IACpC,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,kDAAkD;IAClD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,UAAU;IACzB,iEAAiE;IACjE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,4CAA4C;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,iDAAiD;IACjD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACpD;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,QAAQ,CAAC;CACvC;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAC;CACrE;AAED,MAAM,MAAM,oBAAoB,GAC5B,MAAM,GACN,MAAM,GACN,OAAO,GACP;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAC/C;IAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnD,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,sDAAsD;IACtD,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,4DAA4D;IAC5D,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,8CAA8C;IAC9C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAID,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,SAAS,yBAAyB,EAAE,CAAC;CACzD;AAED,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,yBAAyB,CAAC;IACzC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAID,MAAM,WAAW,sBAAsB;IACrC,+EAA+E;IAC/E,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,yCAAyC;IACzC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,kEAAkE;IAClE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAID;;;;;;;GAOG;AACH,MAAM,WAAW,oBAAoB;IACnC,sEAAsE;IACtE,QAAQ,CAAC,KAAK,EAAE,MAAM,OAAO,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,CAAC;IAG/D,QAAQ,CAAC,wBAAwB,EAAE,CACjC,OAAO,CAAC,EAAE,0BAA0B,KACjC,OAAO,CAAC,eAAe,CAAC,yBAAyB,CAAC,CAAC,CAAC;IACzD,uDAAuD;IACvD,QAAQ,CAAC,uBAAuB,EAAE,CAAC,IAAI,EAAE;QACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,KAAK,OAAO,CAAC,eAAe,CAAC;QAAE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;IAC/D;;;OAGG;IACH,QAAQ,CAAC,wBAAwB,EAAE,CAAC,IAAI,EAAE;QACxC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;KACvB,KAAK,OAAO,CACX,eAAe,CAAC;QAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,wBAAwB,EAAE,CAAA;KAAE,CAAC,CAC1E,CAAC;IAGF,QAAQ,CAAC,2BAA2B,EAAE,MAAM,OAAO,CACjD,eAAe,CAAC,wBAAwB,CAAC,CAC1C,CAAC;IAGF,QAAQ,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE;QACzB,8DAA8D;QAC9D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC;KACrC,KAAK,OAAO,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC,CAAC;IAGhD,+EAA+E;IAC/E,QAAQ,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE;QAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACxC,KAAK,OAAO,CAAC,eAAe,CAAC;QAAE,QAAQ,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,CAAA;KAAE,CAAC,CAAC,CAAC;IAC5E,QAAQ,CAAC,eAAe,EAAE,CACxB,OAAO,EAAE,uBAAuB,KAC7B,OAAO,CAAC,eAAe,CAAC,sBAAsB,CAAC,CAAC,CAAC;IAGtD,QAAQ,CAAC,qBAAqB,EAAE,MAAM,OAAO,CAC3C,eAAe,CAAC,2BAA2B,CAAC,CAC7C,CAAC;IAGF,QAAQ,CAAC,uBAAuB,EAAE,CAAC,IAAI,EAAE;QACvC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,sBAAsB,CAAC;KAC3C,KAAK,OAAO,CAAC,eAAe,CAAC,qBAAqB,CAAC,CAAC,CAAC;IAGtD;;;;OAIG;IACH,QAAQ,CAAC,mBAAmB,EAAE,MAAM,OAAO,CACzC,eAAe,CAAC,oBAAoB,CAAC,CACtC,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC;IACzB,sDAAsD;IACtD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,kDAAkD;IAClD,QAAQ,CAAC,YAAY,EAAE;QACrB,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;QACtC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;QACrC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;QAC5B,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;QAC7B,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;QACpC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;KACnC,CAAC;CACH;AAID;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,oBAAoB,GAAG,SAAS,GAAG,WAAW,CAAC;IAC9E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,iEAAiE;IACjE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,qEAAqE;IACrE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,+EAA+E;IAC/E,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;CACxE;AAED,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC3D,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,kFAAkF;IAClF,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,sEAAsE;IACtE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2DAA2D;IAC3D,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAID;wEACwE;AACxE,eAAO,MAAM,gBAAgB,EAAG,sBAA+B,CAAC;AAEhE,mFAAmF;AACnF,eAAO,MAAM,kBAAkB,EAAG,aAAsB,CAAC;AAEzD,6EAA6E;AAC7E,eAAO,MAAM,+BAA+B,EAAG,EAAW,CAAC;AAE3D,+EAA+E;AAC/E,eAAO,MAAM,oCAAoC,EAAG,EAAW,CAAC;AAIhE;;;;GAIG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAYD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,YAAY,IAAI,qBAAqB,CAgBpD;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,IAAI,oBAAoB,GAAG,IAAI,CAK1D"}

package/dist/mobile/ios-computer-interface.d.ts

@@ -0,0 +1,160 @@
+/**
+ * WS7 ↔ WS9 — `IosComputerInterface` adapts the WS7 `ComputerInterface`
+ * port to the iOS Capacitor `ComputerUse` plugin.
+ *
+ * iOS is fundamentally NOT a click-and-keyboard surface for third-party
+ * apps. The only sanctioned UI driver is App Intents (Shortcuts-style
+ * invocations). The mapping below reflects that — cascade dispatch
+ * primitives that exist on Android (tap / swipe / type) throw a clear
+ * "use invokeAppIntent" message on iOS.
+ *
+ * What *is* supported:
+ *
+ *   screenshot()             → drain one frame from `replayKitForegroundDrain`
+ *                              (caller is responsible for having started a
+ *                              session via `replayKitForegroundStart`).
+ *   getScreenSize, coord     → metadata, no bridge call.
+ *   getAccessibilityTree()   → scene-provided tree from WS6; iOS itself
+ *                              cannot read cross-app AX, so this is
+ *                              own-app only (`accessibilitySnapshot`).
+ *   invokeAppIntent(...)     → the canonical iOS "action" — planner picks
+ *                              an intent id + parameters, this method calls
+ *                              `bridge.appIntentInvoke(...)`.
+ *
+ * What is NOT supported (and throws with a redirect message):
+ *
+ *   leftClick / rightClick / doubleClick / dragTo / drag / mouseDown /
+ *   mouseUp / keyDown / keyUp / typeText / pressKey / hotkey / scroll
+ *
+ * The cascade detects iOS via the platform-capabilities probe and is
+ * expected to plan in terms of `invokeAppIntent`, not synthetic taps. The
+ * `IosComputerInterface.invokeAppIntent` accessor surfaces that contract
+ * cleanly so callers don't have to reach into the raw Capacitor bridge.
+ */
+import type { ComputerInterface, CursorPosition, DisplayPoint, DragPath, MouseButton, ScreenshotResult, ScrollDelta } from "../actor/computer-interface.js";
+import type { Scene, SceneAxNode } from "../scene/scene-types.js";
+import type { DisplayDescriptor } from "../types.js";
+import type { IntentInvocationRequest, IntentInvocationResult, IosComputerUseBridge } from "./ios-bridge.js";
+/**
+ * Stable logical display id for the iOS phone screen. Mirrors the
+ * Android convention (`ANDROID_LOGICAL_DISPLAY_ID = 0`) so the cascade
+ * doesn't branch on platform for display selection — there's exactly
+ * one logical display on iOS.
+ */
+export declare const IOS_LOGICAL_DISPLAY_ID: 0;
+export interface IosComputerInterfaceDeps {
+    /** Capacitor plugin handle (null when off-platform). */
+    getBridge: () => IosComputerUseBridge | null;
+    /** Latest scene accessor — used for `getAccessibilityTree`. */
+    getScene?: () => Scene | null;
+    /**
+     * Active ReplayKit session id. The arbiter / scene-builder owns the
+     * session lifecycle (start before cascade tick, stop when idle); this
+     * interface only drains frames from it.
+     */
+    getReplayKitSessionId?: () => string | null;
+    /** Display descriptor; iOS has exactly one logical display. */
+    getDisplay?: () => DisplayDescriptor;
+    /** Internal cursor-position state, mostly for tests. */
+    cursorState?: {
+        current: CursorPosition;
+    };
+    /** Override JPEG decode (defaults to base64 → Buffer). */
+    decodeJpeg?: (b64: string) => Buffer;
+}
+/**
+ * iOS adapter implementing the WS7 `ComputerInterface`. All input-bearing
+ * methods throw with a "use invokeAppIntent" redirect; metadata + the
+ * screenshot path are functional.
+ */
+export declare class IosComputerInterface implements ComputerInterface {
+    private readonly deps;
+    private readonly cursorState;
+    constructor(deps: IosComputerInterfaceDeps);
+    screenshot(opts?: {
+        displayId?: number;
+    }): Promise<ScreenshotResult>;
+    mouseDown(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    mouseUp(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    leftClick(point: DisplayPoint): Promise<void>;
+    rightClick(point: DisplayPoint): Promise<void>;
+    doubleClick(point: DisplayPoint): Promise<void>;
+    moveCursor(point: DisplayPoint): Promise<void>;
+    dragTo(point: DisplayPoint): Promise<void>;
+    drag(path: DragPath): Promise<void>;
+    keyDown(args: {
+        key: string;
+    }): Promise<void>;
+    keyUp(args: {
+        key: string;
+    }): Promise<void>;
+    typeText(args: {
+        text: string;
+    }): Promise<void>;
+    pressKey(args: {
+        key: string;
+    }): Promise<void>;
+    hotkey(args: {
+        keys: string[];
+    }): Promise<void>;
+    scroll(delta: ScrollDelta): Promise<void>;
+    scrollUp(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    scrollDown(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    getScreenSize(_args: {
+        displayId: number;
+    }): {
+        w: number;
+        h: number;
+    };
+    getCursorPosition(): CursorPosition;
+    toScreenCoordinates(args: {
+        displayId: number;
+        imgX: number;
+        imgY: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        x: number;
+        y: number;
+    };
+    toScreenshotCoordinates(args: {
+        displayId: number;
+        x: number;
+        y: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        imgX: number;
+        imgY: number;
+    };
+    getAccessibilityTree(args: {
+        displayId?: number;
+    }): SceneAxNode[];
+    /**
+     * Invoke an AppIntent through the bridge. This is the iOS-only addition
+     * to the WS7 surface — it does not exist on Android/desktop because
+     * those platforms have direct input. Cascade planners that detect iOS
+     * via the platform-capabilities probe should plan in `invokeAppIntent`
+     * terms (intent id + parameters) instead of `leftClick`.
+     */
+    invokeAppIntent(request: IntentInvocationRequest): Promise<IntentInvocationResult>;
+    private refuseInput;
+    private refuseKeyboard;
+    private requireBridge;
+    private requireDisplayId;
+    private requireFiniteCoords;
+    private getDisplay;
+}
+/** Convenience factory mirroring the Android side. */
+export declare function makeIosComputerInterface(deps: IosComputerInterfaceDeps): IosComputerInterface;
+//# sourceMappingURL=ios-computer-interface.d.ts.map

package/dist/mobile/ios-computer-interface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ios-computer-interface.d.ts","sourceRoot":"","sources":["../../src/mobile/ios-computer-interface.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAGH,OAAO,KAAK,EACV,iBAAiB,EACjB,cAAc,EACd,YAAY,EACZ,QAAQ,EACR,WAAW,EACX,gBAAgB,EAChB,WAAW,EACZ,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EACV,uBAAuB,EACvB,sBAAsB,EAEtB,oBAAoB,EAErB,MAAM,iBAAiB,CAAC;AAEzB;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB,EAAG,CAAU,CAAC;AAUjD,MAAM,WAAW,wBAAwB;IACvC,wDAAwD;IACxD,SAAS,EAAE,MAAM,oBAAoB,GAAG,IAAI,CAAC;IAC7C,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,IAAI,CAAC;IAC9B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,MAAM,GAAG,IAAI,CAAC;IAC5C,+DAA+D;IAC/D,UAAU,CAAC,EAAE,MAAM,iBAAiB,CAAC;IACrC,wDAAwD;IACxD,WAAW,CAAC,EAAE;QAAE,OAAO,EAAE,cAAc,CAAA;KAAE,CAAC;IAC1C,0DAA0D;IAC1D,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;CACtC;AAED;;;;GAIG;AACH,qBAAa,oBAAqB,YAAW,iBAAiB;IAC5D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA2B;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8B;gBAE9C,IAAI,EAAE,wBAAwB;IASpC,UAAU,CACd,IAAI,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAO,GAChC,OAAO,CAAC,gBAAgB,CAAC;IAgCtB,SAAS,CACb,KAAK,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAC7C,OAAO,CAAC,IAAI,CAAC;IAIV,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAItE,SAAS,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7C,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9C,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/C,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ9C,MAAM,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAI1C,IAAI,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAInC,OAAO,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7C,KAAK,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3C,QAAQ,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/C,QAAQ,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9C,MAAM,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/C,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAQzC,QAAQ,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAIpE,UAAU,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM5E,aAAa,CAAC,KAAK,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE;IAKrE,iBAAiB,IAAI,cAAc;IAInC,mBAAmB,CAAC,IAAI,EAAE;QACxB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACd,GAAG;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE;IAgB5B,uBAAuB,CAAC,IAAI,EAAE;QAC5B,SAAS,EAAE,MAAM,CAAC;QAClB,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACd,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAgBlC,oBAAoB,CAAC,IAAI,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,WAAW,EAAE;IASjE;;;;;;OAMG;IACG,eAAe,CACnB,OAAO,EAAE,uBAAuB,GAC/B,OAAO,CAAC,sBAAsB,CAAC;IAclC,OAAO,CAAC,WAAW;IASnB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,gBAAgB;IASxB,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,UAAU;CAUnB;AAED,sDAAsD;AACtD,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,wBAAwB,GAC7B,oBAAoB,CAEtB"}