@xopcai/xopc 0.0.414 → 0.0.416

package/dist/src/phone/adb-driver.d.ts

@@ -1,4 +1,4 @@
-import { type PhoneApplication, type DiscoveredPhone, type PhoneAction, type PhoneElement, type PhoneObservation } from './contracts.js';
+import { type PhoneApplication, type PhoneTextInputStatus, type DiscoveredPhone, type PhoneAction, type PhoneElement, type PhoneObservation } from './contracts.js';
 export declare function resolveAdbPath(configured?: string, stateDir?: string): string;
 export declare function runAdb(path: string, args: string[], signal?: AbortSignal, timeout?: number): Promise<Buffer>;
 export declare function parseAdbDevices(text: string): DiscoveredPhone[];
@@ -7,12 +7,14 @@ export declare function parseAndroidLayout(raw: string): PhoneElement[];
 export interface PhoneDriver {
     discover(signal?: AbortSignal): Promise<DiscoveredPhone[]>;
     applications?(serial: string, signal: AbortSignal): Promise<PhoneApplication[]>;
+    textInputStatus?(serial: string, signal: AbortSignal): Promise<PhoneTextInputStatus>;
     observe(serial: string, signal: AbortSignal): Promise<PhoneObservation>;
     capture?(serial: string, signal: AbortSignal): Promise<PhoneObservation>;
     observeForText?(serial: string, signal: AbortSignal): Promise<PhoneObservation>;
     execute(serial: string, action: PhoneAction, observation: PhoneObservation, signal: AbortSignal, options?: {
         settleMs: number;
         resetTask?: boolean;
+        settingsPage?: 'about-phone';
     }): Promise<void>;
 }
 /** Read only the active native editor, never historical input-method entries. */
@@ -22,7 +24,9 @@ export declare class AdbPhoneDriver implements PhoneDriver {
     private readonly command;
     private readonly useAndroidCli;
     private readonly readApplications?;
-    constructor(adbPath?: string, command?: typeof runAdb, useAndroidCli?: boolean, readApplications?: (signal: AbortSignal) => Promise<PhoneApplication[]>);
+    private readonly pasteText?;
+    constructor(adbPath?: string, command?: typeof runAdb, useAndroidCli?: boolean, readApplications?: (signal: AbortSignal) => Promise<PhoneApplication[]>, pasteText?: (text: string, signal: AbortSignal) => Promise<void>);
+    textInputStatus(serial: string, signal: AbortSignal): Promise<PhoneTextInputStatus>;
     discover(signal?: AbortSignal): Promise<DiscoveredPhone[]>;
     applications(serial: string, signal: AbortSignal): Promise<PhoneApplication[]>;
     foreground(serial: string, signal: AbortSignal): Promise<string>;
@@ -31,8 +35,10 @@ export declare class AdbPhoneDriver implements PhoneDriver {
     observeForText(serial: string, signal: AbortSignal): Promise<PhoneObservation>;
     /** Pixel capture avoids accessibility dumps on the human interaction path. */
     capture(serial: string, signal: AbortSignal): Promise<PhoneObservation>;
+    private confirmText;
     execute(serial: string, action: PhoneAction, observation: PhoneObservation, signal: AbortSignal, options?: {
         settleMs: number;
         resetTask?: boolean;
+        settingsPage?: 'about-phone';
     }): Promise<void>;
 }

package/dist/src/phone/adb-driver.js

@@ -132,12 +132,32 @@ function parseAndroidFocusedEditor(raw, app) {
 		};
 	}
 }
+function sameTextEditor(target, node) {
+	return node.focused && Boolean(node.editable) && !node.password && (target.resourceId ? node.resourceId === target.resourceId : node.bounds.every((value, index) => value === target.bounds[index]));
+}
 var AdbPhoneDriver = class {
-	constructor(adbPath = resolveAdbPath(), command = runAdb, useAndroidCli = true, readApplications) {
+	constructor(adbPath = resolveAdbPath(), command = runAdb, useAndroidCli = true, readApplications, pasteText) {
 		this.adbPath = adbPath;
 		this.command = command;
 		this.useAndroidCli = useAndroidCli;
 		this.readApplications = readApplications;
+		this.pasteText = pasteText;
+	}
+	async textInputStatus(serial, signal) {
+		const androidSdk = Number((await this.shell(serial, ["getprop", "ro.build.version.sdk"], signal)).trim()) || 0;
+		let ready = false;
+		try {
+			await readScrcpyServer();
+			ready = true;
+		} catch {
+			signal.throwIfAborted();
+		}
+		return {
+			androidSdk,
+			unicode: androidSdk < 24 ? "unsupported" : ready ? "available" : "setup_required",
+			method: "scrcpy-clipboard",
+			keyboardRequired: false
+		};
 	}
 	async discover(signal) {
 		return parseAdbDevices((await this.command(this.adbPath, ["devices", "-l"], signal)).toString());
@@ -198,25 +218,29 @@ var AdbPhoneDriver = class {
 			}
 			if (!elements.length) await setTimeout(350, void 0, { signal });
 		}
-		if (!elements.length) try {
-			const path = `/data/local/tmp/xopc-phone-${randomUUID()}.xml`;
+		if (!elements.length) for (const flags of [[], ["--compressed"]]) {
 			try {
-				await this.shell(serial, [
-					"uiautomator",
-					"dump",
-					path
-				], signal);
-				elements = parsePhoneUi(await this.shell(serial, ["cat", path], signal));
-			} finally {
-				await this.shell(serial, [
-					"rm",
-					"-f",
-					path
-				], signal).catch(() => {});
+				const path = `/data/local/tmp/xopc-phone-${randomUUID()}.xml`;
+				try {
+					await this.shell(serial, [
+						"uiautomator",
+						"dump",
+						...flags,
+						path
+					], signal);
+					elements = parsePhoneUi(await this.shell(serial, ["cat", path], signal));
+				} finally {
+					await this.shell(serial, [
+						"rm",
+						"-f",
+						path
+					], signal).catch(() => {});
+				}
+			} catch (error) {
+				signal.throwIfAborted();
+				if (error instanceof Error && error.message === "PHONE_UI_INVALID") throw error;
 			}
-		} catch (error) {
-			signal.throwIfAborted();
-			if (error instanceof Error && error.message === "PHONE_UI_INVALID") throw error;
+			if (elements.length) break;
 		}
 		return {
 			...await this.capture(serial, signal),
@@ -285,11 +309,40 @@ var AdbPhoneDriver = class {
 			throw error;
 		}
 	}
+	async confirmText(serial, app, target, matches, signal) {
+		for (let attempt = 0; attempt < 3; attempt++) {
+			await setTimeout(150, void 0, { signal });
+			const after = await this.observe(serial, signal);
+			try {
+				if (after.package !== app) break;
+				const candidates = after.elements.filter((node) => target.resourceId === "android:active-editor" ? node.focused && node.editable && !node.password : sameTextEditor(target, node));
+				if (candidates.length === 1 && matches(candidates[0].text)) return;
+				if (candidates.length !== 1) break;
+			} finally {
+				after.frame.fill(0);
+			}
+		}
+		throw new PhoneError("PHONE_TEXT_INPUT_UNCONFIRMED", "Text was sent but the target field value could not be verified. Take over and inspect the input field before retrying.", 409);
+	}
 	async execute(serial, action, observation, signal, options) {
 		const point = ([x, y]) => [String(Math.min(observation.width - 1, Math.round(x * observation.width / 1e3))), String(Math.min(observation.height - 1, Math.round(y * observation.height / 1e3)))];
 		const input = (args) => this.shell(serial, ["input", ...args], signal);
 		switch (action.type) {
+			case "tapElement": throw new PhoneError("PHONE_SEMANTIC_UNRESOLVED", "Resolve the native element against a fresh observation before dispatch", 409);
 			case "launch": {
+				if (options?.settingsPage) {
+					if (action.package !== "com.android.settings") throw new PhoneError("PHONE_APP_DENIED", "About phone requires the Settings application");
+					const output = await this.shell(serial, [
+						"am",
+						"start",
+						"-a",
+						"android.settings.DEVICE_INFO_SETTINGS",
+						"-p",
+						action.package
+					], signal);
+					if (/Error|Exception/.test(output)) throw new PhoneError("PHONE_LAUNCH_FAILED", "Unable to open About phone");
+					break;
+				}
 				const component = (await this.shell(serial, [
 					"cmd",
 					"package",
@@ -334,47 +387,64 @@ var AdbPhoneDriver = class {
 					String(action.durationMs)
 				]);
 				break;
-			case "typeText":
-				if (!observation.elements.some((node) => node.focused && node.editable && !node.password)) throw new PhoneError("PHONE_INPUT_NOT_FOCUSED", "Focus a non-password editable field before typing");
-				if (/^[a-zA-Z0-9 .,_@+-]+$/.test(action.text) && !action.text.includes("%")) await input(["text", action.text.replaceAll(" ", "%s")]);
-				else {
-					const previous = (await this.shell(serial, [
-						"settings",
-						"get",
-						"secure",
-						"default_input_method"
-					], signal)).trim();
-					const keyboard = "com.android.adbkeyboard/.AdbIME";
-					if (!(await this.shell(serial, [
-						"ime",
-						"list",
-						"-s"
-					], signal)).split(/\s+/).includes(keyboard)) throw new PhoneError("PHONE_KEYBOARD_REQUIRED", "Unicode input requires ADB Keyboard. Install and enable it on the device first.");
+			case "typeText": {
+				const editors = observation.elements.filter((node) => node.focused && node.editable && !node.password);
+				if (editors.length !== 1) throw new PhoneError("PHONE_INPUT_NOT_FOCUSED", "Focus one non-password editable field before typing");
+				const ascii = /^[a-zA-Z0-9 .,_@+-]+$/.test(action.text) && !action.text.includes("%");
+				if (!ascii) {
+					if ((await this.textInputStatus(serial, signal)).unicode !== "available") throw new PhoneError("PHONE_TEXT_INPUT_UNAVAILABLE", "Automatic text input is unavailable. Check Phone settings or take over and type with the phone keyboard.", 409);
+				}
+				let target = editors[0];
+				if (action.mode === "replace") {
+					if (target.resourceId === "android:active-editor") throw new PhoneError("PHONE_TEXT_INPUT_UNAVAILABLE", "Replacing text requires a visible editable field. Take over and inspect the field.", 409);
+					const fresh = await this.observe(serial, signal);
 					try {
-						await this.shell(serial, [
-							"ime",
-							"set",
-							keyboard
-						], signal);
-						const result = await this.shell(serial, [
-							"am",
-							"broadcast",
-							"-a",
-							"ADB_INPUT_B64",
-							"--es",
-							"msg",
-							Buffer.from(action.text).toString("base64")
-						], signal);
-						if (!/Broadcast completed/.test(result)) throw new PhoneError("PHONE_INPUT_FAILED", "Keyboard input was not acknowledged");
+						const current = fresh.elements.filter((node) => sameTextEditor(target, node));
+						if (fresh.package !== observation.package || current.length !== 1) throw new PhoneError("PHONE_INPUT_NOT_FOCUSED", "The focused input field changed before replacement", 409);
+						target = current[0];
 					} finally {
-						if (/^[\w.]+\/[\w.$]+$/.test(previous)) await this.shell(serial, [
-							"ime",
-							"set",
-							previous
-						], AbortSignal.timeout(3e3));
+						fresh.frame.fill(0);
+					}
+					if (target.text) {
+						if (!(await this.shell(serial, ["input"], signal)).includes("keycombination")) throw new PhoneError("PHONE_TEXT_INPUT_UNAVAILABLE", "This phone cannot safely select all text. Take over and replace the field manually.", 409);
+						try {
+							await input([
+								"keycombination",
+								"KEYCODE_CTRL_LEFT",
+								"KEYCODE_A"
+							]);
+							const selected = await this.observe(serial, signal);
+							try {
+								if (selected.package !== observation.package || selected.elements.filter((node) => sameTextEditor(target, node) && node.text === target.text).length !== 1) throw new Error("Editor changed during selection");
+							} finally {
+								selected.frame.fill(0);
+							}
+						} catch {
+							signal.throwIfAborted();
+							throw new PhoneError("PHONE_TEXT_INPUT_UNCONFIRMED", "Text selection started but the target field could not be verified. Take over and inspect the input field before retrying.", 409);
+						}
 					}
 				}
+				try {
+					if (ascii) await input(["text", action.text.replaceAll(" ", "%s")]);
+					else if (this.pasteText) await this.pasteText(action.text, signal);
+					else {
+						const { LocalScrcpyProvider } = await import("./scrcpy.js");
+						const provider = new LocalScrcpyProvider(this.adbPath, serial, () => {}, void 0, true);
+						try {
+							await provider.pasteText(action.text, signal);
+						} finally {
+							await provider.close();
+						}
+					}
+					await this.confirmText(serial, observation.package, target, (text) => action.mode === "replace" ? text === action.text : text.includes(action.text), signal);
+				} catch (error) {
+					signal.throwIfAborted();
+					if (error instanceof PhoneError && error.code === "PHONE_TEXT_INPUT_UNCONFIRMED") throw error;
+					throw new PhoneError("PHONE_TEXT_INPUT_UNCONFIRMED", "Text may have been sent but insertion could not be verified. Take over and inspect the input field before retrying.", 409);
+				}
 				break;
+			}
 			case "home":
 				await input(["keyevent", "KEYCODE_HOME"]);
 				break;

package/dist/src/phone/cloud-adb.js

@@ -266,6 +266,14 @@ var CloudAdbConnection = class CloudAdbConnection {
 			} finally {
 				listing.dispose();
 			}
+		}, async (text, signal) => {
+			const { CloudScrcpyProvider } = await import("./cloud-scrcpy.js");
+			const provider = new CloudScrcpyProvider(this.endpoint, Buffer.from(this.key), () => {}, void 0, true);
+			try {
+				await provider.pasteText(text, signal);
+			} finally {
+				await provider.close();
+			}
 		});
 	}
 };

package/dist/src/phone/cloud-scrcpy.d.ts

@@ -3,6 +3,7 @@ import { ScrcpyProvider } from './scrcpy.js';
 /** Dedicated authenticated ADB transport: no host ADB key, server or port forwarding. */
 export declare class CloudScrcpyProvider extends ScrcpyProvider {
     private endpoint;
+    private textOnly;
     private connection;
     private process?;
     private video?;
@@ -10,11 +11,12 @@ export declare class CloudScrcpyProvider extends ScrcpyProvider {
     private writer?;
     private remote;
     private scid;
-    constructor(endpoint: string, key: Buffer, onFailure: () => void, connect?: (address: string, credential: Buffer) => CloudAdbConnection);
+    constructor(endpoint: string, key: Buffer, onFailure: () => void, connect?: (address: string, credential: Buffer) => CloudAdbConnection, textOnly?: boolean);
     foreground(signal: AbortSignal): Promise<string>;
     protected startTransport(): Promise<void>;
     private drain;
     private readVideo;
+    private readControl;
     protected write(buffer: Buffer): Promise<void>;
     protected cleanup(): Promise<void>;
 }

package/dist/src/phone/cloud-scrcpy.js

@@ -17,9 +17,10 @@ var CloudScrcpyProvider = class extends ScrcpyProvider {
 	writer;
 	remote = "";
 	scid = "";
-	constructor(endpoint, key, onFailure, connect = (address, credential) => new CloudAdbConnection(address, credential)) {
+	constructor(endpoint, key, onFailure, connect = (address, credential) => new CloudAdbConnection(address, credential), textOnly = false) {
 		super(onFailure);
 		this.endpoint = endpoint;
+		this.textOnly = textOnly;
 		this.connection = connect(endpoint, key);
 	}
 	foreground(signal) {
@@ -48,7 +49,7 @@ var CloudScrcpyProvider = class extends ScrcpyProvider {
 				await sync.dispose();
 			}
 			if (this.closing) throw new Error("Video closed");
-			this.process = await adb.createSocket(`exec:CLASSPATH=${this.remote} app_process / com.genymobile.scrcpy.Server ${SCRCPY_VERSION} scid=${this.scid} tunnel_forward=true audio=false control=true cleanup=false max_size=960 max_fps=30 video_bit_rate=2000000 video_codec=h264 video_codec_options=i-frame-interval=1 send_dummy_byte=false send_device_meta=false send_codec_meta=true send_frame_meta=true clipboard_autosync=false`);
+			this.process = await adb.createSocket(`exec:CLASSPATH=${this.remote} app_process / com.genymobile.scrcpy.Server ${SCRCPY_VERSION} scid=${this.scid} tunnel_forward=true audio=false control=true cleanup=false max_size=960 max_fps=30 video_bit_rate=2000000 video_codec=h264 video_codec_options=i-frame-interval=1 send_dummy_byte=false send_device_meta=false send_codec_meta=true send_frame_meta=true clipboard_autosync=false${this.textOnly ? " video=false power_on=false" : ""}`);
 			this.drain(this.process);
 			for (let attempt = 0; attempt < 60; attempt++) {
 				if (this.closing) throw new Error("Video closed");
@@ -60,9 +61,16 @@ var CloudScrcpyProvider = class extends ScrcpyProvider {
 				}
 			}
 			if (!this.video) throw new Error("Scrcpy connection timed out");
+			if (this.textOnly) {
+				this.control = this.video;
+				this.video = void 0;
+				this.writer = this.control.writable.getWriter();
+				this.readControl(this.control);
+				return;
+			}
 			this.control = await adb.createSocket(`localabstract:${socketName}`);
 			this.writer = this.control.writable.getWriter();
-			this.drain(this.control);
+			this.readControl(this.control);
 			this.readVideo(this.video);
 		} finally {
 			clearTimeout(timer);
@@ -81,6 +89,12 @@ var CloudScrcpyProvider = class extends ScrcpyProvider {
 		} catch {}
 		if (!this.closing) this.fail();
 	}
+	async readControl(socket) {
+		try {
+			for await (const chunk of socket.readable) this.textChannel.receive(Buffer.from(chunk));
+		} catch {}
+		if (!this.closing) this.fail();
+	}
 	async write(buffer) {
 		if (!this.writer) throw new PhoneError("PHONE_INPUT_DISCONNECTED", "Cloud phone input channel unavailable");
 		const timer = setTimeout(() => this.connection.close(), 2e3);

package/dist/src/phone/contracts.d.ts

@@ -14,6 +14,8 @@ export declare const MOCK_PHONE_GOAL = "Open Android Settings and inspect About
 export declare const PhoneConfigSchema: z.ZodPrefault<z.ZodObject<{
     enabled: z.ZodDefault<z.ZodBoolean>;
     adbPath: z.ZodOptional<z.ZodString>;
+    semanticControls: z.ZodDefault<z.ZodBoolean>;
+    hierarchicalPlanning: z.ZodDefault<z.ZodBoolean>;
     maxActionsPerSession: z.ZodDefault<z.ZodNumber>;
     maxModelRequests: z.ZodDefault<z.ZodNumber>;
     maxOperationDurationMs: z.ZodDefault<z.ZodNumber>;
@@ -29,6 +31,10 @@ export declare const PhoneDeviceSchema: z.ZodObject<{
         all: "all";
         selected: "selected";
     }>>;
+    textInput: z.ZodOptional<z.ZodEnum<{
+        manual: "manual";
+        auto: "auto";
+    }>>;
     allowedApps: z.ZodDefault<z.ZodArray<z.ZodString>>;
     connection: z.ZodOptional<z.ZodObject<{
         kind: z.ZodLiteral<"aliyun-adb">;
@@ -43,18 +49,42 @@ export type PhoneApplication = {
     packageName: string;
     label: string;
 };
+export type PhoneTextInputStatus = {
+    androidSdk: number;
+    unicode: 'available' | 'setup_required' | 'unsupported';
+    method: 'scrcpy-clipboard' | 'provider-native';
+    keyboardRequired: false;
+};
 export type DiscoveredPhone = {
     serial: string;
     state: string;
     name: string;
     emulator: boolean;
 };
+export declare const PhoneElementSelectorSchema: z.ZodObject<{
+    resourceId: z.ZodOptional<z.ZodString>;
+    text: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+export type PhoneElementSelector = z.infer<typeof PhoneElementSelectorSchema>;
+export type PhoneCapabilities = {
+    semanticControls?: boolean;
+    hierarchicalPlanning?: boolean;
+};
 export declare const PhoneActionSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     type: z.ZodLiteral<"launch">;
     package: z.ZodString;
 }, z.core.$strict>, z.ZodObject<{
     type: z.ZodLiteral<"tap">;
     point: z.ZodArray<z.ZodNumber>;
+}, z.core.$strict>, z.ZodObject<{
+    type: z.ZodLiteral<"tapElement">;
+    frameId: z.ZodString;
+    selector: z.ZodObject<{
+        resourceId: z.ZodOptional<z.ZodString>;
+        text: z.ZodOptional<z.ZodString>;
+        description: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
 }, z.core.$strict>, z.ZodObject<{
     type: z.ZodLiteral<"doubleTap">;
     point: z.ZodArray<z.ZodNumber>;
@@ -70,6 +100,10 @@ export declare const PhoneActionSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
 }, z.core.$strict>, z.ZodObject<{
     type: z.ZodLiteral<"typeText">;
     text: z.ZodString;
+    mode: z.ZodOptional<z.ZodEnum<{
+        replace: "replace";
+        insert: "insert";
+    }>>;
 }, z.core.$strict>, z.ZodObject<{
     type: z.ZodEnum<{
         home: "home";
@@ -88,6 +122,14 @@ export declare const PhoneDecisionSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     }, z.core.$strict>, z.ZodObject<{
         type: z.ZodLiteral<"tap">;
         point: z.ZodArray<z.ZodNumber>;
+    }, z.core.$strict>, z.ZodObject<{
+        type: z.ZodLiteral<"tapElement">;
+        frameId: z.ZodString;
+        selector: z.ZodObject<{
+            resourceId: z.ZodOptional<z.ZodString>;
+            text: z.ZodOptional<z.ZodString>;
+            description: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>;
     }, z.core.$strict>, z.ZodObject<{
         type: z.ZodLiteral<"doubleTap">;
         point: z.ZodArray<z.ZodNumber>;
@@ -103,6 +145,10 @@ export declare const PhoneDecisionSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     }, z.core.$strict>, z.ZodObject<{
         type: z.ZodLiteral<"typeText">;
         text: z.ZodString;
+        mode: z.ZodOptional<z.ZodEnum<{
+            replace: "replace";
+            insert: "insert";
+        }>>;
     }, z.core.$strict>, z.ZodObject<{
         type: z.ZodEnum<{
             home: "home";
@@ -140,6 +186,17 @@ export type PhoneObservation = {
     frame: Buffer;
     elements: PhoneElement[];
 };
+export declare const PhoneCompletionSchema: z.ZodObject<{
+    appPackage: z.ZodOptional<z.ZodString>;
+    textAny: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    textAll: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    field: z.ZodOptional<z.ZodObject<{
+        resourceId: z.ZodString;
+        value: z.ZodString;
+    }, z.core.$strict>>;
+    requireChange: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strict>;
+export type PhoneCompletion = z.infer<typeof PhoneCompletionSchema>;
 export declare const PhoneInputSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     op: z.ZodLiteral<"discover">;
 }, z.core.$strict>, z.ZodObject<{
@@ -150,6 +207,7 @@ export declare const PhoneInputSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
         control: "control";
     }>;
     appPackage: z.ZodOptional<z.ZodString>;
+    settingsPage: z.ZodOptional<z.ZodLiteral<"about-phone">>;
 }, z.core.$strict>, z.ZodObject<{
     op: z.ZodLiteral<"observe">;
 }, z.core.$strict>, z.ZodObject<{
@@ -160,6 +218,16 @@ export declare const PhoneInputSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     goal: z.ZodString;
     scenario: z.ZodOptional<z.ZodLiteral<"settings-about">>;
     expectText: z.ZodOptional<z.ZodString>;
+    completion: z.ZodOptional<z.ZodObject<{
+        appPackage: z.ZodOptional<z.ZodString>;
+        textAny: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        textAll: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        field: z.ZodOptional<z.ZodObject<{
+            resourceId: z.ZodString;
+            value: z.ZodString;
+        }, z.core.$strict>>;
+        requireChange: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strict>>;
     maxActions: z.ZodDefault<z.ZodNumber>;
 }, z.core.$strict>, z.ZodObject<{
     op: z.ZodEnum<{
@@ -204,6 +272,23 @@ export type PhoneReport = {
     controlEpoch?: number;
     handoff?: 'idle' | 'acquiring' | 'returning';
     currentAction?: PhoneAction['type'];
+    diagnostics?: {
+        stalePredictions: number;
+        protocolRepairs: number;
+        unchangedActions: number;
+        completionCorrections: number;
+        semanticActions?: number;
+        semanticRejections?: number;
+        plannedSubgoals?: number;
+        completedSubgoals?: number;
+        backReplans?: number;
+        focusReplans?: number;
+        focusRecoveries?: number;
+        resumedTextInputs?: number;
+        numericActions?: number;
+        numericConfirmedSteps?: number;
+        numericMismatches?: number;
+    };
 };
 export declare class PhoneError extends Error {
     readonly code: string;

package/dist/src/phone/contracts.js

@@ -1,7 +1,7 @@
 import { __esmMin } from "../../_virtual/_rolldown/runtime.js";
 import { z } from "zod";
 //#region src/phone/contracts.ts
-var PhoneProfileSchema, PhoneModelMetadataSchema, MOCK_PHONE_MODEL, MOCK_PHONE_GOAL, PhoneConfigSchema, PhoneDeviceSchema, Point, PhoneActionSchema, PhoneDecisionSchema, PhoneInputSchema, PhoneError;
+var PhoneProfileSchema, PhoneModelMetadataSchema, MOCK_PHONE_MODEL, MOCK_PHONE_GOAL, PhoneConfigSchema, PhoneDeviceSchema, Point, PhoneElementSelectorSchema, PhoneActionSchema, PhoneDecisionSchema, PhoneCompletionSchema, PhoneInputSchema, PhoneError;
 var init_contracts = __esmMin((() => {
 	PhoneProfileSchema = z.enum(["autoglm-phone-v1", "structured-phone-v1"]);
 	PhoneModelMetadataSchema = z.object({ profile: PhoneProfileSchema }).strict();
@@ -10,6 +10,8 @@ var init_contracts = __esmMin((() => {
 	PhoneConfigSchema = z.object({
 		enabled: z.boolean().default(false),
 		adbPath: z.string().min(1).optional(),
+		semanticControls: z.boolean().default(false),
+		hierarchicalPlanning: z.boolean().default(false),
 		maxActionsPerSession: z.number().int().min(1).max(80).default(40),
 		maxModelRequests: z.number().int().min(1).max(120).default(60),
 		maxOperationDurationMs: z.number().int().min(1e3).max(9e5).default(3e5),
@@ -21,6 +23,7 @@ var init_contracts = __esmMin((() => {
 		name: z.string().trim().min(1).max(100),
 		enabled: z.boolean().default(true),
 		appAccess: z.enum(["all", "selected"]).optional(),
+		textInput: z.enum(["auto", "manual"]).optional(),
 		allowedApps: z.array(z.string().regex(/^[a-zA-Z][\w]*(?:\.[\w]+)+$/)).max(500).default([]),
 		connection: z.object({
 			kind: z.literal("aliyun-adb"),
@@ -31,6 +34,11 @@ var init_contracts = __esmMin((() => {
 		revision: z.number().int().positive()
 	}).strict();
 	Point = z.array(z.number().finite().min(0).max(1e3)).length(2);
+	PhoneElementSelectorSchema = z.object({
+		resourceId: z.string().min(1).max(300).optional(),
+		text: z.string().min(1).max(4e3).optional(),
+		description: z.string().min(1).max(1e3).optional()
+	}).strict().refine((value) => Boolean(value.resourceId || value.text || value.description), "Specify an exact native element identity");
 	PhoneActionSchema = z.discriminatedUnion("type", [
 		z.object({
 			type: z.literal("launch"),
@@ -40,6 +48,11 @@ var init_contracts = __esmMin((() => {
 			type: z.literal("tap"),
 			point: Point
 		}).strict(),
+		z.object({
+			type: z.literal("tapElement"),
+			frameId: z.string().min(1).max(200),
+			selector: PhoneElementSelectorSchema
+		}).strict(),
 		z.object({
 			type: z.literal("doubleTap"),
 			point: Point
@@ -57,7 +70,8 @@ var init_contracts = __esmMin((() => {
 		}).strict(),
 		z.object({
 			type: z.literal("typeText"),
-			text: z.string().min(1).max(4e3)
+			text: z.string().min(1).max(4e3),
+			mode: z.enum(["insert", "replace"]).optional()
 		}).strict(),
 		z.object({ type: z.enum(["back", "home"]) }).strict(),
 		z.object({
@@ -79,20 +93,32 @@ var init_contracts = __esmMin((() => {
 			reason: z.string().max(1e3)
 		}).strict()
 	]);
+	PhoneCompletionSchema = z.object({
+		appPackage: z.string().regex(/^[a-zA-Z][\w]*(?:\.[\w]+)+$/).optional(),
+		textAny: z.array(z.string().min(1).max(200)).min(1).max(20).optional(),
+		textAll: z.array(z.string().min(1).max(200)).min(1).max(20).optional(),
+		field: z.object({
+			resourceId: z.string().min(1).max(300),
+			value: z.string().max(4e3)
+		}).strict().optional(),
+		requireChange: z.boolean().optional()
+	}).strict().refine((value) => value.appPackage || value.textAny || value.textAll || value.field, "Specify at least one UI condition");
 	PhoneInputSchema = z.discriminatedUnion("op", [
 		z.object({ op: z.literal("discover") }).strict(),
 		z.object({
 			op: z.literal("open"),
 			deviceId: z.string().uuid().optional(),
 			mode: z.enum(["observe", "control"]),
-			appPackage: z.string().regex(/^[a-zA-Z][\w]*(?:\.[\w]+)+$/).optional()
-		}).strict(),
+			appPackage: z.string().regex(/^[a-zA-Z][\w]*(?:\.[\w]+)+$/).optional(),
+			settingsPage: z.literal("about-phone").optional()
+		}).strict().refine((input) => !input.settingsPage || input.mode === "control" && input.appPackage === "com.android.settings", "About phone requires authorized Settings control"),
 		z.object({ op: z.literal("observe") }).strict(),
 		z.object({
 			op: z.enum(["run", "step"]),
 			goal: z.string().trim().min(1).max(2e3),
 			scenario: z.literal("settings-about").optional(),
 			expectText: z.string().min(1).max(200).optional(),
+			completion: PhoneCompletionSchema.optional(),
 			maxActions: z.number().int().min(1).max(12).default(6)
 		}).strict(),
 		z.object({ op: z.enum([
@@ -113,4 +139,4 @@ var init_contracts = __esmMin((() => {
 }));
 //#endregion
 init_contracts();
-export { MOCK_PHONE_GOAL, MOCK_PHONE_MODEL, PhoneActionSchema, PhoneConfigSchema, PhoneDecisionSchema, PhoneDeviceSchema, PhoneError, PhoneInputSchema, PhoneModelMetadataSchema, PhoneProfileSchema, init_contracts };
+export { MOCK_PHONE_GOAL, MOCK_PHONE_MODEL, PhoneActionSchema, PhoneCompletionSchema, PhoneConfigSchema, PhoneDecisionSchema, PhoneDeviceSchema, PhoneElementSelectorSchema, PhoneError, PhoneInputSchema, PhoneModelMetadataSchema, PhoneProfileSchema, init_contracts };