@phone-use/sdk 0.1.1 → 0.3.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@phone-use/sdk",
-  "version": "0.1.1",
-  "description": "Typed mobile-device SDK — Device, backends, errors, actions",
+  "version": "0.3.0",
+  "description": "Typed mobile-device SDK \u2014 Device, backends, errors, actions",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,

package/src/backends/cloud-sandbox.ts

@@ -0,0 +1,202 @@
+import { writeFile } from 'node:fs/promises';
+import { BaseDeviceBackend } from '../backend.ts';
+import {
+  ALL_CAPABILITIES,
+  type AlertAction,
+  type BackendAlertResult,
+  type Capability,
+  type OpenAppResult,
+  type PressTarget,
+  type ScrollDirection,
+  type Snapshot,
+} from '../device.ts';
+import { PhoneUseError } from '../errors.ts';
+
+// ---------------------------------------------------------------------------
+// CloudSandboxBackend: drives a phone-use cloud sandbox (a simulator on a
+// remote worker) over the worker's sandbox-scoped RPC endpoint. This is the
+// client half of packages/cloud's worker protocol; it lives in the SDK so the
+// CLI/MCP/agent can attach to a sandbox without depending on @phone-use/cloud
+// (which also contains the server side and is not published).
+//
+// Wire protocol (shared with packages/cloud — keep in sync):
+//   POST <endpoint>/rpc  {method, args}  ->  {ok:true,result} | {ok:false,error}
+// ---------------------------------------------------------------------------
+
+export type SandboxRpcMethod =
+  | 'snapshot'
+  | 'screenshot'
+  | 'press'
+  | 'longPress'
+  | 'fill'
+  | 'typeText'
+  | 'pressKey'
+  | 'scroll'
+  | 'pan'
+  | 'installApp'
+  | 'waitForText'
+  | 'systemAlert'
+  | 'home'
+  | 'back'
+  | 'openApp'
+  | 'listApps'
+  | 'closeSession';
+
+export type SandboxRpcRequest = { method: SandboxRpcMethod; args: unknown[] };
+export type SandboxRpcSuccess = { ok: true; result: unknown };
+export type SandboxRpcFailure = {
+  ok: false;
+  error: { message: string; code?: string | undefined; retryable?: boolean | undefined };
+};
+export type SandboxRpcResponse = SandboxRpcSuccess | SandboxRpcFailure;
+
+export type CloudSandboxBackendOptions = {
+  endpoint: string;
+  token: string;
+  capabilities?: Capability[] | undefined;
+  fetch?: typeof globalThis.fetch | undefined;
+};
+
+export class CloudSandboxBackend extends BaseDeviceBackend {
+  private readonly endpoint: string;
+  private readonly token: string;
+  private readonly fetchImpl: typeof globalThis.fetch;
+
+  constructor(opts: CloudSandboxBackendOptions) {
+    super('phone-use-cloud', opts.capabilities ?? ALL_CAPABILITIES);
+    this.endpoint = opts.endpoint.replace(/\/+$/, '');
+    this.token = opts.token;
+    this.fetchImpl = opts.fetch ?? globalThis.fetch;
+  }
+
+  private async rpc<T>(method: SandboxRpcMethod, ...args: unknown[]): Promise<T> {
+    const response = await this.fetchImpl(`${this.endpoint}/rpc`, {
+      method: 'POST',
+      headers: {
+        authorization: `Bearer ${this.token}`,
+        'content-type': 'application/json',
+      },
+      body: JSON.stringify({ method, args } satisfies SandboxRpcRequest),
+    });
+    const body = (await response.json().catch(() => ({}))) as
+      | SandboxRpcResponse
+      | { error?: string; message?: string };
+    if (!response.ok || !('ok' in body)) {
+      const message =
+        ('error' in body && typeof body.error === 'string' ? body.error : undefined) ??
+        ('message' in body ? body.message : undefined) ??
+        `HTTP ${response.status}`;
+      throw new PhoneUseError(message, { code: 'UNKNOWN', retryable: false });
+    }
+    if (!body.ok) {
+      throw new PhoneUseError(body.error.message, {
+        code: normalizeErrorCode(body.error.code),
+        retryable: body.error.retryable ?? false,
+      });
+    }
+    return body.result as T;
+  }
+
+  override snapshot(opts?: { interactiveOnly?: boolean; depth?: number }): Promise<Snapshot> {
+    return this.rpc('snapshot', opts);
+  }
+
+  override async screenshot(opts: { path: string; overlayRefs?: boolean }): Promise<{ path: string }> {
+    const result = await this.rpc<{ base64: string }>('screenshot', {
+      overlayRefs: opts.overlayRefs,
+    });
+    await writeFile(opts.path, Buffer.from(result.base64, 'base64'));
+    return { path: opts.path };
+  }
+
+  override press(target: PressTarget): Promise<void> {
+    return this.rpc('press', target);
+  }
+  override longPress(ref: string, durationMs?: number): Promise<void> {
+    return this.rpc('longPress', ref, durationMs);
+  }
+  override fill(ref: string, text: string): Promise<void> {
+    return this.rpc('fill', ref, text);
+  }
+  override typeText(text: string): Promise<void> {
+    return this.rpc('typeText', text);
+  }
+  override pressKey(key: 'return'): Promise<void> {
+    return this.rpc('pressKey', key);
+  }
+  override scroll(direction: ScrollDirection): Promise<void> {
+    return this.rpc('scroll', direction);
+  }
+  override pan(x: number, y: number, dx: number, dy: number, durationMs?: number): Promise<void> {
+    return this.rpc('pan', x, y, dx, dy, durationMs);
+  }
+  override waitForText(text: string, timeoutMs?: number): Promise<void> {
+    return this.rpc('waitForText', text, timeoutMs);
+  }
+  override systemAlert(action: AlertAction): Promise<BackendAlertResult> {
+    return this.rpc('systemAlert', action);
+  }
+  override home(): Promise<void> {
+    return this.rpc('home');
+  }
+  override back(): Promise<void> {
+    return this.rpc('back');
+  }
+  override openApp(opts: { app?: string; url?: string; relaunch?: boolean }): Promise<OpenAppResult> {
+    return this.rpc('openApp', opts);
+  }
+  override listApps(): Promise<string[]> {
+    return this.rpc('listApps');
+  }
+  override closeSession(): Promise<void> {
+    return this.rpc('closeSession');
+  }
+
+  /** Upload a zipped .app bundle (base64) and install it on the sandbox device. */
+  installApp(base64Zip: string): Promise<{ installed?: string }> {
+    return this.rpc('installApp', base64Zip);
+  }
+}
+
+/**
+ * Build a CloudSandboxBackend from explicit options or the environment:
+ * PHONE_USE_SANDBOX_URL + PHONE_USE_SANDBOX_TOKEN (printed by
+ * `phone-use sandbox env <id>`).
+ */
+export function createCloudSandboxBackend(config?: Partial<CloudSandboxBackendOptions>): CloudSandboxBackend {
+  const endpoint = config?.endpoint ?? process.env.PHONE_USE_SANDBOX_URL;
+  const token = config?.token ?? process.env.PHONE_USE_SANDBOX_TOKEN;
+  if (!endpoint || !token) {
+    throw new PhoneUseError(
+      'cloud sandbox backend needs PHONE_USE_SANDBOX_URL and PHONE_USE_SANDBOX_TOKEN (see `phone-use sandbox env <id>`)',
+      { code: 'BACKEND_NOT_FOUND', retryable: false },
+    );
+  }
+  return new CloudSandboxBackend({ ...config, endpoint, token });
+}
+
+function normalizeErrorCode(
+  code: string | undefined,
+):
+  | 'DEVICE_NOT_FOUND'
+  | 'DEVICE_IN_USE'
+  | 'SESSION_NOT_FOUND'
+  | 'TIMEOUT'
+  | 'ACTION_FAILED'
+  | 'UNSUPPORTED_CAPABILITY'
+  | 'BACKEND_NOT_FOUND'
+  | 'ABORTED'
+  | 'UNKNOWN' {
+  const codes = new Set([
+    'DEVICE_NOT_FOUND',
+    'DEVICE_IN_USE',
+    'SESSION_NOT_FOUND',
+    'TIMEOUT',
+    'ACTION_FAILED',
+    'UNSUPPORTED_CAPABILITY',
+    'BACKEND_NOT_FOUND',
+    'ABORTED',
+    'UNKNOWN',
+  ]);
+  return codes.has(code ?? '') ? (code as ReturnType<typeof normalizeErrorCode>) : 'UNKNOWN';
+}

package/src/backends/device-runner.ts

@@ -0,0 +1,205 @@
+import { BaseDeviceBackend } from '../backend.ts';
+import type {
+  AlertAction,
+  BackendAlertResult,
+  Capability,
+  OpenAppResult,
+  PressTarget,
+  ScrollDirection,
+  Snapshot,
+} from '../device.ts';
+import { ActionFailedError, DeviceNotFoundError, TimeoutError } from '../errors.ts';
+
+/**
+ * A device-runner is addressed purely by URL, so it takes no DeviceConfig
+ * platform/udid selection — only where to reach the runner and how to auth.
+ */
+export type DeviceRunnerConfig = {
+  endpoint?: string | undefined;
+  token?: string | undefined;
+  timeoutMs?: number | undefined;
+};
+
+const RUNNER_CAPABILITIES: readonly Capability[] = [
+  'snapshot',
+  'screenshot',
+  'press',
+  'fill',
+  'type',
+  'scroll',
+  'pan',
+  'openApp',
+  'home',
+];
+
+/**
+ * Backend that speaks to an on-device runner: an XCTest-hosted JSON-RPC server
+ * running ON the iPhone itself, which holds the automation privileges iOS
+ * denies to ordinary apps.
+ *
+ * The endpoint is just a URL, so the same backend serves every topology:
+ *   - `http://127.0.0.1:45678`  — port-forwarded from a paired host
+ *   - `http://<phone-ip>:45678` — straight over the LAN / tailnet
+ *   - `https://relay.example/d/<id>` — the runner dials out to a cloud relay,
+ *     which is what lets an agent anywhere drive the phone with no inbound
+ *     ports and no Mac in the loop.
+ *
+ * The wire format matches the shape proven by rounak/PhoneAgent: newline-free
+ * JSON request/response over HTTP POST, one method per call.
+ */
+export class DeviceRunnerBackend extends BaseDeviceBackend {
+  readonly #endpoint: string;
+  readonly #token: string | undefined;
+  readonly #timeoutMs: number;
+
+  constructor(config?: DeviceRunnerConfig) {
+    // BaseDeviceBackend owns backendName/capabilities — set them via super()
+    // rather than redeclaring the fields.
+    super('device-runner', RUNNER_CAPABILITIES);
+    const endpoint = config?.endpoint ?? process.env.PHONE_USE_RUNNER_URL ?? 'http://127.0.0.1:45678';
+    this.#endpoint = endpoint.replace(/\/+$/, '');
+    this.#token = config?.token ?? process.env.PHONE_USE_RUNNER_TOKEN;
+    this.#timeoutMs = config?.timeoutMs ?? 30_000;
+  }
+
+  async #rpc<T>(method: string, params: Record<string, unknown> = {}): Promise<T> {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.#timeoutMs);
+    let res: Response;
+    try {
+      res = await fetch(this.#endpoint, {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          ...(this.#token ? { authorization: `Bearer ${this.#token}` } : {}),
+        },
+        body: JSON.stringify({ jsonrpc: '2.0', id: Date.now(), method, params }),
+        signal: controller.signal,
+      });
+    } catch (cause) {
+      if (controller.signal.aborted) {
+        throw new TimeoutError(`runner did not answer ${method} within ${this.#timeoutMs}ms`);
+      }
+      throw new DeviceNotFoundError(
+        `cannot reach the on-device runner at ${this.#endpoint} — is it activated on the phone?`,
+        { cause },
+      );
+    } finally {
+      clearTimeout(timer);
+    }
+
+    if (!res.ok) {
+      throw new ActionFailedError(`runner returned HTTP ${res.status} for ${method}`);
+    }
+    const body = (await res.json()) as { result?: T; error?: { message?: string } };
+    if (body.error) {
+      throw new ActionFailedError(body.error.message ?? `runner rejected ${method}`);
+    }
+    return body.result as T;
+  }
+
+  override async snapshot(opts?: {
+    interactiveOnly?: boolean | undefined;
+    depth?: number | undefined;
+  }): Promise<Snapshot> {
+    // The runner speaks its own compact wire shape; map it onto the SDK's
+    // Snapshot contract so every consumer (CLI, MCP, agent, bench) is unaware
+    // it is talking to a phone rather than a simulator.
+    const wire = await this.#rpc<{
+      app?: string;
+      elements: Array<{
+        ref: string;
+        role?: string;
+        label?: string;
+        value?: string;
+        enabled?: boolean;
+        rect?: { x: number; y: number; w: number; h: number };
+      }>;
+    }>('get_tree', {
+      interactiveOnly: opts?.interactiveOnly ?? false,
+      depth: opts?.depth,
+    });
+    return {
+      appBundleId: wire.app,
+      appName: wire.app,
+      nodes: (wire.elements ?? []).map((e) => ({
+        ref: e.ref,
+        role: e.role,
+        type: e.role,
+        label: e.label,
+        value: e.value,
+        enabled: e.enabled,
+        rect: e.rect ? { x: e.rect.x, y: e.rect.y, width: e.rect.w, height: e.rect.h } : undefined,
+      })),
+    };
+  }
+
+  override async screenshot(opts: {
+    path: string;
+    overlayRefs?: boolean | undefined;
+  }): Promise<{ path: string }> {
+    const { base64 } = await this.#rpc<{ base64: string }>('get_screen_image', {
+      overlayRefs: opts.overlayRefs ?? false,
+    });
+    const { writeFile } = await import('node:fs/promises');
+    await writeFile(opts.path, Buffer.from(base64, 'base64'));
+    return { path: opts.path };
+  }
+
+  override async press(target: PressTarget): Promise<void> {
+    // PressTarget is {ref} | {x,y} — never a bare string, so narrow on the key.
+    if ('ref' in target) {
+      await this.#rpc('tap_element', { ref: target.ref });
+      return;
+    }
+    await this.#rpc('tap', { x: target.x, y: target.y });
+  }
+
+  override async fill(ref: string, text: string): Promise<void> {
+    await this.#rpc('enter_text', { ref, text, replace: true });
+  }
+
+  override async typeText(text: string): Promise<void> {
+    await this.#rpc('enter_text', { text, replace: false });
+  }
+
+  override async scroll(direction: ScrollDirection): Promise<void> {
+    await this.#rpc('scroll', { direction });
+  }
+
+  override async pan(x: number, y: number, dx: number, dy: number, durationMs = 300): Promise<void> {
+    await this.#rpc('swipe', { x, y, dx, dy, durationMs });
+  }
+
+  override async home(): Promise<void> {
+    await this.#rpc('home');
+  }
+
+  override async openApp(opts: {
+    app?: string | undefined;
+    url?: string | undefined;
+    relaunch?: boolean | undefined;
+  }): Promise<OpenAppResult> {
+    return this.#rpc<OpenAppResult>('open_app', {
+      app: opts.app,
+      url: opts.url,
+      relaunch: opts.relaunch ?? false,
+    });
+  }
+
+  override async systemAlert(action: AlertAction): Promise<BackendAlertResult> {
+    return this.#rpc<BackendAlertResult>('alert', { action });
+  }
+
+  override async closeSession(): Promise<void> {
+    // The runner outlives any single client; nothing to tear down.
+  }
+
+  /** Liveness probe used by `phone-use doctor` and the relay health check. */
+  async ping(): Promise<{ ok: boolean; ios?: string; device?: string }> {
+    return this.#rpc('get_context');
+  }
+}
+
+export const createDeviceRunnerBackend = (config?: DeviceRunnerConfig): DeviceRunnerBackend =>
+  new DeviceRunnerBackend(config);

package/src/index.ts

@@ -7,7 +7,7 @@
  * deliberately not re-exported here.
  */
 /** The published package version (kept in sync with package.json by the release flow). */
-export const VERSION = '0.1.1';
+export const VERSION = '0.3.0';
 
 export {
   type Action,
@@ -31,6 +31,15 @@ export {
   registerBackend,
 } from './backend.ts';
 export { createAgentDeviceBackend } from './backends/agent-device.ts';
+export {
+  CloudSandboxBackend,
+  type CloudSandboxBackendOptions,
+  createCloudSandboxBackend,
+  type SandboxRpcMethod,
+  type SandboxRpcRequest,
+  type SandboxRpcResponse,
+} from './backends/cloud-sandbox.ts';
+export { createDeviceRunnerBackend, DeviceRunnerBackend } from './backends/device-runner.ts';
 export { type IosConnectOptions, type IosLaunchOptions, ios } from './backends/ios.ts';
 export type { AndroidDeviceConfig, CommonDeviceConfig, DeviceConfig, IosDeviceConfig } from './config.ts';
 export type {