@phone-use/sdk 0.1.1 → 0.3.0

package/dist/index.d.mts

@@ -477,6 +477,137 @@ declare function executeAction(core: DeviceCore, action: Action, opts?: ActOptio
  */
 declare function createAgentDeviceBackend(config?: DeviceConfig): DeviceBackend;
 //#endregion
+//#region src/backends/cloud-sandbox.d.ts
+type SandboxRpcMethod = 'snapshot' | 'screenshot' | 'press' | 'longPress' | 'fill' | 'typeText' | 'pressKey' | 'scroll' | 'pan' | 'installApp' | 'waitForText' | 'systemAlert' | 'home' | 'back' | 'openApp' | 'listApps' | 'closeSession';
+type SandboxRpcRequest = {
+  method: SandboxRpcMethod;
+  args: unknown[];
+};
+type SandboxRpcSuccess = {
+  ok: true;
+  result: unknown;
+};
+type SandboxRpcFailure = {
+  ok: false;
+  error: {
+    message: string;
+    code?: string | undefined;
+    retryable?: boolean | undefined;
+  };
+};
+type SandboxRpcResponse = SandboxRpcSuccess | SandboxRpcFailure;
+type CloudSandboxBackendOptions = {
+  endpoint: string;
+  token: string;
+  capabilities?: Capability[] | undefined;
+  fetch?: typeof globalThis.fetch | undefined;
+};
+declare class CloudSandboxBackend extends BaseDeviceBackend {
+  private readonly endpoint;
+  private readonly token;
+  private readonly fetchImpl;
+  constructor(opts: CloudSandboxBackendOptions);
+  private rpc;
+  snapshot(opts?: {
+    interactiveOnly?: boolean;
+    depth?: number;
+  }): Promise<Snapshot>;
+  screenshot(opts: {
+    path: string;
+    overlayRefs?: boolean;
+  }): Promise<{
+    path: string;
+  }>;
+  press(target: PressTarget): Promise<void>;
+  longPress(ref: string, durationMs?: number): Promise<void>;
+  fill(ref: string, text: string): Promise<void>;
+  typeText(text: string): Promise<void>;
+  pressKey(key: 'return'): Promise<void>;
+  scroll(direction: ScrollDirection): Promise<void>;
+  pan(x: number, y: number, dx: number, dy: number, durationMs?: number): Promise<void>;
+  waitForText(text: string, timeoutMs?: number): Promise<void>;
+  systemAlert(action: AlertAction): Promise<BackendAlertResult>;
+  home(): Promise<void>;
+  back(): Promise<void>;
+  openApp(opts: {
+    app?: string;
+    url?: string;
+    relaunch?: boolean;
+  }): Promise<OpenAppResult>;
+  listApps(): Promise<string[]>;
+  closeSession(): Promise<void>;
+  /** Upload a zipped .app bundle (base64) and install it on the sandbox device. */
+  installApp(base64Zip: string): Promise<{
+    installed?: string;
+  }>;
+}
+/**
+ * Build a CloudSandboxBackend from explicit options or the environment:
+ * PHONE_USE_SANDBOX_URL + PHONE_USE_SANDBOX_TOKEN (printed by
+ * `phone-use sandbox env <id>`).
+ */
+declare function createCloudSandboxBackend(config?: Partial<CloudSandboxBackendOptions>): CloudSandboxBackend;
+//#endregion
+//#region src/backends/device-runner.d.ts
+/**
+ * A device-runner is addressed purely by URL, so it takes no DeviceConfig
+ * platform/udid selection — only where to reach the runner and how to auth.
+ */
+type DeviceRunnerConfig = {
+  endpoint?: string | undefined;
+  token?: string | undefined;
+  timeoutMs?: number | undefined;
+};
+/**
+ * Backend that speaks to an on-device runner: an XCTest-hosted JSON-RPC server
+ * running ON the iPhone itself, which holds the automation privileges iOS
+ * denies to ordinary apps.
+ *
+ * The endpoint is just a URL, so the same backend serves every topology:
+ *   - `http://127.0.0.1:45678`  — port-forwarded from a paired host
+ *   - `http://<phone-ip>:45678` — straight over the LAN / tailnet
+ *   - `https://relay.example/d/<id>` — the runner dials out to a cloud relay,
+ *     which is what lets an agent anywhere drive the phone with no inbound
+ *     ports and no Mac in the loop.
+ *
+ * The wire format matches the shape proven by rounak/PhoneAgent: newline-free
+ * JSON request/response over HTTP POST, one method per call.
+ */
+declare class DeviceRunnerBackend extends BaseDeviceBackend {
+  #private;
+  constructor(config?: DeviceRunnerConfig);
+  snapshot(opts?: {
+    interactiveOnly?: boolean | undefined;
+    depth?: number | undefined;
+  }): Promise<Snapshot>;
+  screenshot(opts: {
+    path: string;
+    overlayRefs?: boolean | undefined;
+  }): Promise<{
+    path: string;
+  }>;
+  press(target: PressTarget): Promise<void>;
+  fill(ref: string, text: string): Promise<void>;
+  typeText(text: string): Promise<void>;
+  scroll(direction: ScrollDirection): Promise<void>;
+  pan(x: number, y: number, dx: number, dy: number, durationMs?: number): Promise<void>;
+  home(): Promise<void>;
+  openApp(opts: {
+    app?: string | undefined;
+    url?: string | undefined;
+    relaunch?: boolean | undefined;
+  }): Promise<OpenAppResult>;
+  systemAlert(action: AlertAction): Promise<BackendAlertResult>;
+  closeSession(): Promise<void>;
+  /** Liveness probe used by `phone-use doctor` and the relay health check. */
+  ping(): Promise<{
+    ok: boolean;
+    ios?: string;
+    device?: string;
+  }>;
+}
+declare const createDeviceRunnerBackend: (config?: DeviceRunnerConfig) => DeviceRunnerBackend;
+//#endregion
 //#region src/lifecycle.d.ts
 /** The platform a {@link Device} runs on. */
 type DevicePlatform = 'ios' | 'android';
@@ -683,7 +814,7 @@ declare const ios: {
  * deliberately not re-exported here.
  */
 /** The published package version (kept in sync with package.json by the release flow). */
-declare const VERSION = "0.1.1";
+declare const VERSION = "0.3.0";
 //#endregion
-export { ALL_CAPABILITIES, AbortedError, type ActOptions, type Action, type ActionEvidence, ActionFailedError, type ActionResult, type ActionVerb, type AlertAction, type AlertOutcome, type AndroidDeviceConfig, type BackendAlertResult, type BackendFactory, BaseDeviceBackend, type Capability, type CommonDeviceConfig, type CompiledSkill, type CreateDeviceHandleOptions, type Device, type DeviceBackend, type DeviceConfig, DeviceCore, DeviceInUseError, DeviceNotFoundError, type DevicePlatform, type DeviceStatus, type ElementQuery, type IosConnectOptions, type IosDeviceConfig, type IosLaunchOptions, type Observation, type ObserveResult, type ObservedElement, type OpenAppResult, PhoneUseError, type PhoneUseErrorCode, type PhoneUseErrorDetails, type PressTarget, type Rect, type RenderState, type Resolution, type ResolveOpts, type ScrollDirection, SecretStore, SessionNotFoundError, type Snapshot, type SnapshotNode, TimeoutError, type UiElement, UnsupportedCapabilityError, VERSION, buildObserveResult, createAgentDeviceBackend, createDeviceHandle, describeError, executeAction, getBackendFactory, ios, labelMatches, listBackends, matchInElements, registerBackend, toActions, toPhoneUseError };
+export { ALL_CAPABILITIES, AbortedError, type ActOptions, type Action, type ActionEvidence, ActionFailedError, type ActionResult, type ActionVerb, type AlertAction, type AlertOutcome, type AndroidDeviceConfig, type BackendAlertResult, type BackendFactory, BaseDeviceBackend, type Capability, CloudSandboxBackend, type CloudSandboxBackendOptions, type CommonDeviceConfig, type CompiledSkill, type CreateDeviceHandleOptions, type Device, type DeviceBackend, type DeviceConfig, DeviceCore, DeviceInUseError, DeviceNotFoundError, type DevicePlatform, DeviceRunnerBackend, type DeviceStatus, type ElementQuery, type IosConnectOptions, type IosDeviceConfig, type IosLaunchOptions, type Observation, type ObserveResult, type ObservedElement, type OpenAppResult, PhoneUseError, type PhoneUseErrorCode, type PhoneUseErrorDetails, type PressTarget, type Rect, type RenderState, type Resolution, type ResolveOpts, type SandboxRpcMethod, type SandboxRpcRequest, type SandboxRpcResponse, type ScrollDirection, SecretStore, SessionNotFoundError, type Snapshot, type SnapshotNode, TimeoutError, type UiElement, UnsupportedCapabilityError, VERSION, buildObserveResult, createAgentDeviceBackend, createCloudSandboxBackend, createDeviceHandle, createDeviceRunnerBackend, describeError, executeAction, getBackendFactory, ios, labelMatches, listBackends, matchInElements, registerBackend, toActions, toPhoneUseError };
 //# sourceMappingURL=index.d.mts.map

package/dist/index.mjs

@@ -1,5 +1,6 @@
 import { a as registerBackend, c as DeviceInUseError, d as SessionNotFoundError, f as TimeoutError, i as listBackends, l as DeviceNotFoundError, m as toPhoneUseError, n as BaseDeviceBackend, o as AbortedError, p as UnsupportedCapabilityError, r as getBackendFactory, s as ActionFailedError, t as ALL_CAPABILITIES, u as PhoneUseError } from "./device-Xsy_LPUF.mjs";
 import { createAgentDeviceClient } from "agent-device";
+import { writeFile } from "node:fs/promises";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 //#region src/observe.ts
@@ -1453,6 +1454,281 @@ function createAgentDeviceBackend(config) {
 	return new AgentDeviceBackend(config);
 }
 //#endregion
+//#region src/backends/cloud-sandbox.ts
+var CloudSandboxBackend = class extends BaseDeviceBackend {
+	endpoint;
+	token;
+	fetchImpl;
+	constructor(opts) {
+		super("phone-use-cloud", opts.capabilities ?? ALL_CAPABILITIES);
+		this.endpoint = opts.endpoint.replace(/\/+$/, "");
+		this.token = opts.token;
+		this.fetchImpl = opts.fetch ?? globalThis.fetch;
+	}
+	async rpc(method, ...args) {
+		const response = await this.fetchImpl(`${this.endpoint}/rpc`, {
+			method: "POST",
+			headers: {
+				authorization: `Bearer ${this.token}`,
+				"content-type": "application/json"
+			},
+			body: JSON.stringify({
+				method,
+				args
+			})
+		});
+		const body = await response.json().catch(() => ({}));
+		if (!response.ok || !("ok" in body)) throw new PhoneUseError(("error" in body && typeof body.error === "string" ? body.error : void 0) ?? ("message" in body ? body.message : void 0) ?? `HTTP ${response.status}`, {
+			code: "UNKNOWN",
+			retryable: false
+		});
+		if (!body.ok) throw new PhoneUseError(body.error.message, {
+			code: normalizeErrorCode(body.error.code),
+			retryable: body.error.retryable ?? false
+		});
+		return body.result;
+	}
+	snapshot(opts) {
+		return this.rpc("snapshot", opts);
+	}
+	async screenshot(opts) {
+		const result = await this.rpc("screenshot", { overlayRefs: opts.overlayRefs });
+		await writeFile(opts.path, Buffer.from(result.base64, "base64"));
+		return { path: opts.path };
+	}
+	press(target) {
+		return this.rpc("press", target);
+	}
+	longPress(ref, durationMs) {
+		return this.rpc("longPress", ref, durationMs);
+	}
+	fill(ref, text) {
+		return this.rpc("fill", ref, text);
+	}
+	typeText(text) {
+		return this.rpc("typeText", text);
+	}
+	pressKey(key) {
+		return this.rpc("pressKey", key);
+	}
+	scroll(direction) {
+		return this.rpc("scroll", direction);
+	}
+	pan(x, y, dx, dy, durationMs) {
+		return this.rpc("pan", x, y, dx, dy, durationMs);
+	}
+	waitForText(text, timeoutMs) {
+		return this.rpc("waitForText", text, timeoutMs);
+	}
+	systemAlert(action) {
+		return this.rpc("systemAlert", action);
+	}
+	home() {
+		return this.rpc("home");
+	}
+	back() {
+		return this.rpc("back");
+	}
+	openApp(opts) {
+		return this.rpc("openApp", opts);
+	}
+	listApps() {
+		return this.rpc("listApps");
+	}
+	closeSession() {
+		return this.rpc("closeSession");
+	}
+	/** Upload a zipped .app bundle (base64) and install it on the sandbox device. */
+	installApp(base64Zip) {
+		return this.rpc("installApp", base64Zip);
+	}
+};
+/**
+* Build a CloudSandboxBackend from explicit options or the environment:
+* PHONE_USE_SANDBOX_URL + PHONE_USE_SANDBOX_TOKEN (printed by
+* `phone-use sandbox env <id>`).
+*/
+function createCloudSandboxBackend(config) {
+	const endpoint = config?.endpoint ?? process.env.PHONE_USE_SANDBOX_URL;
+	const token = config?.token ?? process.env.PHONE_USE_SANDBOX_TOKEN;
+	if (!endpoint || !token) throw new PhoneUseError("cloud sandbox backend needs PHONE_USE_SANDBOX_URL and PHONE_USE_SANDBOX_TOKEN (see `phone-use sandbox env <id>`)", {
+		code: "BACKEND_NOT_FOUND",
+		retryable: false
+	});
+	return new CloudSandboxBackend({
+		...config,
+		endpoint,
+		token
+	});
+}
+function normalizeErrorCode(code) {
+	return (/* @__PURE__ */ new Set([
+		"DEVICE_NOT_FOUND",
+		"DEVICE_IN_USE",
+		"SESSION_NOT_FOUND",
+		"TIMEOUT",
+		"ACTION_FAILED",
+		"UNSUPPORTED_CAPABILITY",
+		"BACKEND_NOT_FOUND",
+		"ABORTED",
+		"UNKNOWN"
+	])).has(code ?? "") ? code : "UNKNOWN";
+}
+//#endregion
+//#region src/backends/device-runner.ts
+const RUNNER_CAPABILITIES = [
+	"snapshot",
+	"screenshot",
+	"press",
+	"fill",
+	"type",
+	"scroll",
+	"pan",
+	"openApp",
+	"home"
+];
+/**
+* Backend that speaks to an on-device runner: an XCTest-hosted JSON-RPC server
+* running ON the iPhone itself, which holds the automation privileges iOS
+* denies to ordinary apps.
+*
+* The endpoint is just a URL, so the same backend serves every topology:
+*   - `http://127.0.0.1:45678`  — port-forwarded from a paired host
+*   - `http://<phone-ip>:45678` — straight over the LAN / tailnet
+*   - `https://relay.example/d/<id>` — the runner dials out to a cloud relay,
+*     which is what lets an agent anywhere drive the phone with no inbound
+*     ports and no Mac in the loop.
+*
+* The wire format matches the shape proven by rounak/PhoneAgent: newline-free
+* JSON request/response over HTTP POST, one method per call.
+*/
+var DeviceRunnerBackend = class extends BaseDeviceBackend {
+	#endpoint;
+	#token;
+	#timeoutMs;
+	constructor(config) {
+		super("device-runner", RUNNER_CAPABILITIES);
+		const endpoint = config?.endpoint ?? process.env.PHONE_USE_RUNNER_URL ?? "http://127.0.0.1:45678";
+		this.#endpoint = endpoint.replace(/\/+$/, "");
+		this.#token = config?.token ?? process.env.PHONE_USE_RUNNER_TOKEN;
+		this.#timeoutMs = config?.timeoutMs ?? 3e4;
+	}
+	async #rpc(method, params = {}) {
+		const controller = new AbortController();
+		const timer = setTimeout(() => controller.abort(), this.#timeoutMs);
+		let res;
+		try {
+			res = await fetch(this.#endpoint, {
+				method: "POST",
+				headers: {
+					"content-type": "application/json",
+					...this.#token ? { authorization: `Bearer ${this.#token}` } : {}
+				},
+				body: JSON.stringify({
+					jsonrpc: "2.0",
+					id: Date.now(),
+					method,
+					params
+				}),
+				signal: controller.signal
+			});
+		} catch (cause) {
+			if (controller.signal.aborted) throw new TimeoutError(`runner did not answer ${method} within ${this.#timeoutMs}ms`);
+			throw new DeviceNotFoundError(`cannot reach the on-device runner at ${this.#endpoint} — is it activated on the phone?`, { cause });
+		} finally {
+			clearTimeout(timer);
+		}
+		if (!res.ok) throw new ActionFailedError(`runner returned HTTP ${res.status} for ${method}`);
+		const body = await res.json();
+		if (body.error) throw new ActionFailedError(body.error.message ?? `runner rejected ${method}`);
+		return body.result;
+	}
+	async snapshot(opts) {
+		const wire = await this.#rpc("get_tree", {
+			interactiveOnly: opts?.interactiveOnly ?? false,
+			depth: opts?.depth
+		});
+		return {
+			appBundleId: wire.app,
+			appName: wire.app,
+			nodes: (wire.elements ?? []).map((e) => ({
+				ref: e.ref,
+				role: e.role,
+				type: e.role,
+				label: e.label,
+				value: e.value,
+				enabled: e.enabled,
+				rect: e.rect ? {
+					x: e.rect.x,
+					y: e.rect.y,
+					width: e.rect.w,
+					height: e.rect.h
+				} : void 0
+			}))
+		};
+	}
+	async screenshot(opts) {
+		const { base64 } = await this.#rpc("get_screen_image", { overlayRefs: opts.overlayRefs ?? false });
+		const { writeFile } = await import("node:fs/promises");
+		await writeFile(opts.path, Buffer.from(base64, "base64"));
+		return { path: opts.path };
+	}
+	async press(target) {
+		if ("ref" in target) {
+			await this.#rpc("tap_element", { ref: target.ref });
+			return;
+		}
+		await this.#rpc("tap", {
+			x: target.x,
+			y: target.y
+		});
+	}
+	async fill(ref, text) {
+		await this.#rpc("enter_text", {
+			ref,
+			text,
+			replace: true
+		});
+	}
+	async typeText(text) {
+		await this.#rpc("enter_text", {
+			text,
+			replace: false
+		});
+	}
+	async scroll(direction) {
+		await this.#rpc("scroll", { direction });
+	}
+	async pan(x, y, dx, dy, durationMs = 300) {
+		await this.#rpc("swipe", {
+			x,
+			y,
+			dx,
+			dy,
+			durationMs
+		});
+	}
+	async home() {
+		await this.#rpc("home");
+	}
+	async openApp(opts) {
+		return this.#rpc("open_app", {
+			app: opts.app,
+			url: opts.url,
+			relaunch: opts.relaunch ?? false
+		});
+	}
+	async systemAlert(action) {
+		return this.#rpc("alert", { action });
+	}
+	async closeSession() {}
+	/** Liveness probe used by `phone-use doctor` and the relay health check. */
+	async ping() {
+		return this.#rpc("get_context");
+	}
+};
+const createDeviceRunnerBackend = (config) => new DeviceRunnerBackend(config);
+//#endregion
 //#region src/exec.ts
 const pExecFile = promisify(execFile);
 const defaultExecRunner = async (file, args, opts) => {
@@ -1840,9 +2116,9 @@ const ios = {
 * deliberately not re-exported here.
 */
 /** The published package version (kept in sync with package.json by the release flow). */
-const VERSION = "0.1.1";
+const VERSION = "0.3.0";
 registerBackend("agent-device", createAgentDeviceBackend);
 //#endregion
-export { ALL_CAPABILITIES, AbortedError, ActionFailedError, BaseDeviceBackend, DeviceCore, DeviceInUseError, DeviceNotFoundError, PhoneUseError, SecretStore, SessionNotFoundError, TimeoutError, UnsupportedCapabilityError, VERSION, buildObserveResult, createAgentDeviceBackend, createDeviceHandle, describeError, executeAction, getBackendFactory, ios, labelMatches, listBackends, matchInElements, registerBackend, toActions, toPhoneUseError };
+export { ALL_CAPABILITIES, AbortedError, ActionFailedError, BaseDeviceBackend, CloudSandboxBackend, DeviceCore, DeviceInUseError, DeviceNotFoundError, DeviceRunnerBackend, PhoneUseError, SecretStore, SessionNotFoundError, TimeoutError, UnsupportedCapabilityError, VERSION, buildObserveResult, createAgentDeviceBackend, createCloudSandboxBackend, createDeviceHandle, createDeviceRunnerBackend, describeError, executeAction, getBackendFactory, ios, labelMatches, listBackends, matchInElements, registerBackend, toActions, toPhoneUseError };
 
 //# sourceMappingURL=index.mjs.map