@xopcai/xopc 0.0.415 → 0.0.416

package/dist/src/phone/runtime.js

@@ -8,10 +8,15 @@ import { PhoneInteractiveSession } from "./interactive-session.js";
 import { AdbPhoneDriver, resolveAdbPath } from "./adb-driver.js";
 import { LocalScrcpyProvider } from "./scrcpy.js";
 import { CloudPhones } from "./cloud-phones.js";
+import { advancePhonePlan, buildPhonePlan, createPhonePlanState } from "./planning.js";
+import { canDispatchPhoneAction, observationSignature, phoneCompletionSatisfied } from "./reliability.js";
+import { groundPhoneCoordinateAction, resolvePhoneSemanticAction } from "./semantic-controls.js";
 import { createPhoneModel, isAboutScreen } from "./model-adapter.js";
 import { PhoneRepository } from "./repository.js";
-import { canDispatchPhoneAction, observationSignature, phoneCompletionSatisfied } from "./reliability.js";
+import { confirmPhoneFocusRecovery, phoneBackRecovery, phoneUnfocusedTypingHint, preparePhoneFocusRecovery } from "./recovery.js";
+import { createPhoneNumericEntry, nextPhoneNumericEntry } from "./numeric-entry.js";
 import { createHash, randomUUID } from "node:crypto";
+import { setTimeout } from "node:timers/promises";
 //#region src/phone/runtime.ts
 init_logger();
 init_contracts();
@@ -226,7 +231,7 @@ var PhoneRuntime = class {
 							await this.dispatch(session, {
 								type: "launch",
 								package: input.appPackage
-							}, observation, context, signal, epoch);
+							}, observation, context, signal, epoch, false, input.settingsPage);
 						} finally {
 							observation.frame.fill(0);
 						}
@@ -315,7 +320,29 @@ var PhoneRuntime = class {
 			let repairs = 0;
 			let completionCorrections = 0;
 			let recoveryHint;
+			const enhancements = {
+				semanticControls: context.config.phone.semanticControls,
+				hierarchicalPlanning: context.config.phone.hierarchicalPlanning
+			};
+			const derivedPlan = enhancements.hierarchicalPlanning ? buildPhonePlan(input.goal, completion) : void 0;
+			let planning = derivedPlan ? createPhonePlanState(derivedPlan) : void 0;
+			if (enhancements.hierarchicalPlanning) {
+				session.report.diagnostics.plannedSubgoals = planning?.steps.length ?? 0;
+				session.report.diagnostics.completedSubgoals = 0;
+			}
 			let changed = false;
+			const entryObservation = {
+				...observation,
+				frame: Buffer.alloc(0),
+				elements: structuredClone(observation.elements)
+			};
+			const trajectoryStart = session.trajectoryCount;
+			let pendingFocus;
+			let numericIntent;
+			let numericPending;
+			const focusedIntents = /* @__PURE__ */ new Set();
+			let backReplans = 0;
+			let focusReplans = 0;
 			let previousSignature = observationSignature(observation);
 			let controlEpoch = session.report.controlEpoch;
 			const attempts = /* @__PURE__ */ new Map();
@@ -331,12 +358,51 @@ var PhoneRuntime = class {
 				if (controlEpoch !== epoch) {
 					attempts.clear();
 					recoveryHint = "The user handed back control. Re-evaluate the fresh screen before continuing.";
+					pendingFocus = void 0;
+					numericIntent = void 0;
+					numericPending = void 0;
 					controlEpoch = epoch;
 				}
 				const signature = observationSignature(observation);
 				if (signature !== previousSignature) changed = true;
 				previousSignature = signature;
 				this.recordObservation(session, observation);
+				if (planning) {
+					planning = advancePhonePlan(planning, observation);
+					session.report.diagnostics.completedSubgoals = planning.verifiedSteps;
+				}
+				let numericStep;
+				if (enhancements.hierarchicalPlanning) {
+					numericIntent ??= createPhoneNumericEntry(input.goal, completion, observation);
+					if (numericIntent) {
+						let next = nextPhoneNumericEntry(numericIntent, observation, numericPending);
+						for (let poll = 0; numericPending && next.kind === "blocked" && poll < 2; poll++) {
+							await setTimeout(150, void 0, { signal });
+							observation.frame.fill(0);
+							observation = await this.observe(session, signal);
+							this.guard(session, context);
+							this.allowedScreen(session, observation);
+							if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") break;
+							next = nextPhoneNumericEntry(numericIntent, observation, numericPending);
+						}
+						if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") continue;
+						this.recordObservation(session, observation);
+						if (next.kind === "blocked") {
+							session.report.diagnostics.numericMismatches = (session.report.diagnostics.numericMismatches ?? 0) + 1;
+							session.report.status = "paused";
+							session.report.controlOwner = "none";
+							session.report.controlEpoch++;
+							session.report.stopReason = "numeric_state_mismatch";
+							session.report.message = "Numeric input was not confirmed. Inspect the displayed value before continuing; no key was retried.";
+							session.budget?.pause();
+							await session.stream?.revoke();
+							return this.publish(session);
+						}
+						if (numericPending) session.report.diagnostics.numericConfirmedSteps = (session.report.diagnostics.numericConfirmedSteps ?? 0) + 1;
+						numericPending = void 0;
+						if (next.kind === "action") numericStep = next.step;
+					}
+				}
 				if (session.report.mock ? isAboutScreen(observation) : Boolean(completion && phoneCompletionSatisfied(completion, observation, changed))) {
 					const confirmation = await this.observe(session, signal);
 					try {
@@ -350,6 +416,10 @@ var PhoneRuntime = class {
 							continue;
 						}
 						this.recordObservation(session, confirmation);
+						if (planning) {
+							planning = advancePhonePlan(planning, confirmation);
+							session.report.diagnostics.completedSubgoals = planning.verifiedSteps;
+						}
 						session.report.verified = true;
 						session.report.verificationLevel = "ui";
 						session.report.status = "completed";
@@ -364,52 +434,72 @@ var PhoneRuntime = class {
 					session.report.stopReason = "action_limit";
 					return this.publish(session);
 				}
-				if (session.report.modelRequests >= context.config.phone.maxModelRequests) throw new PhoneError("PHONE_MODEL_LIMIT", "Phone model request limit reached");
-				session.report.modelRequests++;
-				session.report.currentAction = void 0;
-				this.publish(session);
-				session.prediction = new AbortController();
-				let decision;
-				try {
-					decision = await session.adapter.predict({
-						goal: input.goal,
-						observation,
-						completion,
-						allowedApps: session.device.allowedApps,
-						appAccess: phoneAppAccess(session.device),
-						applications: session.applications.filter((app) => phoneAllowsApp(session.device, app.packageName)),
-						history: session.report.steps.map((step) => step.action),
-						trajectory: session.trajectory,
-						progressSummary: JSON.stringify({
-							dispatched: session.trajectoryCount,
-							earlier: [...session.trajectorySummary, ...session.trajectory.slice(0, -8).map((entry) => JSON.stringify({
-								action: entry.action,
-								effect: entry.effect,
-								afterApp: entry.afterApp
-							}).slice(0, 250))].slice(-12)
-						}).slice(-1500),
-						recoveryHint,
-						protocolFeedback,
-						scenario: input.scenario
-					}, AbortSignal.any([signal, session.prediction.signal]));
-				} catch (error) {
-					if (!signal.aborted && epoch !== session.report.controlEpoch) {
-						await this.waitForAgent(session, signal);
-						observation.frame.fill(0);
-						observation = await this.observe(session, signal);
-						continue;
-					}
-					if (!signal.aborted && error instanceof PhoneError && ["PHONE_MODEL_PROTOCOL", "PHONE_MODEL_TRUNCATED"].includes(error.code) && repairs < 1 && session.report.modelRequests < context.config.phone.maxModelRequests) {
-						repairs++;
-						session.report.diagnostics.protocolRepairs++;
-						protocolFeedback = "Your previous output was invalid or truncated. Return exactly one complete action in the specified format with no extra prose.";
-						observation.frame.fill(0);
-						observation = await this.observe(session, signal);
-						continue;
+				let decision = numericStep ? {
+					kind: "action",
+					action: numericStep.action
+				} : void 0;
+				let resumedText = false;
+				if (!decision && pendingFocus) {
+					const pending = pendingFocus;
+					pendingFocus = void 0;
+					const intended = confirmPhoneFocusRecovery(pending, observation);
+					if (intended) {
+						decision = {
+							kind: "action",
+							action: intended
+						};
+						resumedText = true;
+					} else recoveryHint = "The focus transition was not confirmed. No text was sent. Reinspect the intended editable field before another action.";
+				}
+				if (!decision) {
+					if (session.report.modelRequests >= context.config.phone.maxModelRequests) throw new PhoneError("PHONE_MODEL_LIMIT", "Phone model request limit reached");
+					session.report.modelRequests++;
+					session.report.currentAction = void 0;
+					this.publish(session);
+					session.prediction = new AbortController();
+					try {
+						decision = await session.adapter.predict({
+							goal: input.goal,
+							observation,
+							completion,
+							allowedApps: session.device.allowedApps,
+							appAccess: phoneAppAccess(session.device),
+							applications: session.applications.filter((app) => phoneAllowsApp(session.device, app.packageName)),
+							history: session.report.steps.map((step) => step.action),
+							trajectory: session.trajectory,
+							progressSummary: JSON.stringify({
+								dispatched: session.trajectoryCount,
+								earlier: [...session.trajectorySummary, ...session.trajectory.slice(0, -8).map((entry) => JSON.stringify({
+									action: entry.action,
+									effect: entry.effect,
+									afterApp: entry.afterApp
+								}).slice(0, 250))].slice(-12)
+							}).slice(-1500),
+							recoveryHint,
+							protocolFeedback,
+							scenario: input.scenario,
+							enhancements,
+							planning
+						}, AbortSignal.any([signal, session.prediction.signal]));
+					} catch (error) {
+						if (!signal.aborted && epoch !== session.report.controlEpoch) {
+							await this.waitForAgent(session, signal);
+							observation.frame.fill(0);
+							observation = await this.observe(session, signal);
+							continue;
+						}
+						if (!signal.aborted && error instanceof PhoneError && ["PHONE_MODEL_PROTOCOL", "PHONE_MODEL_TRUNCATED"].includes(error.code) && repairs < 1 && session.report.modelRequests < context.config.phone.maxModelRequests) {
+							repairs++;
+							session.report.diagnostics.protocolRepairs++;
+							protocolFeedback = "Your previous output was invalid or truncated. Return exactly one complete action in the specified format with no extra prose.";
+							observation.frame.fill(0);
+							observation = await this.observe(session, signal);
+							continue;
+						}
+						throw error;
+					} finally {
+						session.prediction = void 0;
 					}
-					throw error;
-				} finally {
-					session.prediction = void 0;
 				}
 				signal.throwIfAborted();
 				this.guard(session, context);
@@ -420,6 +510,11 @@ var PhoneRuntime = class {
 					continue;
 				}
 				protocolFeedback = void 0;
+				if (enhancements.hierarchicalPlanning && !planning && decision.plan) {
+					planning = advancePhonePlan(createPhonePlanState(decision.plan), observation);
+					session.report.diagnostics.plannedSubgoals = planning.steps.length;
+					session.report.diagnostics.completedSubgoals = planning.verifiedSteps;
+				}
 				if (decision.kind !== "action") {
 					if (decision.kind === "complete" && !session.report.mock && (input.completion || input.expectText) && completionCorrections < 1 && session.report.modelRequests < context.config.phone.maxModelRequests) {
 						completionCorrections++;
@@ -447,10 +542,77 @@ var PhoneRuntime = class {
 					}
 					return this.publish(session);
 				}
+				let preparedFocus;
+				if (enhancements.hierarchicalPlanning) {
+					const operationActions = session.trajectoryCount - trajectoryStart;
+					const back = phoneBackRecovery({
+						action: decision.action,
+						observation,
+						entryObservation,
+						completion,
+						trajectory: operationActions ? session.trajectory.slice(-operationActions) : []
+					});
+					if (back) {
+						session.report.diagnostics.backReplans = (session.report.diagnostics.backReplans ?? 0) + 1;
+						if (backReplans++ >= 2) {
+							session.report.status = "paused";
+							session.report.controlOwner = "none";
+							session.report.controlEpoch++;
+							session.report.stopReason = "back_boundary";
+							session.report.message = back.hint;
+							session.budget?.pause();
+							await session.stream?.revoke();
+							return this.publish(session);
+						}
+						recoveryHint = back.hint;
+						observation.frame.fill(0);
+						observation = await this.observe(session, signal);
+						continue;
+					}
+					const focusHint = phoneUnfocusedTypingHint(decision.action, observation, completion);
+					if (focusHint) {
+						const key = JSON.stringify(decision.action);
+						preparedFocus = !focusedIntents.has(key) ? preparePhoneFocusRecovery(decision.action, observation, completion) : void 0;
+						if (preparedFocus) decision = {
+							kind: "action",
+							action: preparedFocus.focusAction
+						};
+						else {
+							session.report.diagnostics.focusReplans = (session.report.diagnostics.focusReplans ?? 0) + 1;
+							if (focusReplans++ >= 2) throw new PhoneError("PHONE_INPUT_NOT_FOCUSED", "The requested editable field could not be focused; inspect before typing", 409);
+							recoveryHint = focusHint;
+							observation.frame.fill(0);
+							observation = await this.observe(session, signal);
+							continue;
+						}
+					}
+				}
 				const fresh = await this.observe(session, signal);
 				try {
 					this.allowedScreen(session, fresh);
-					if (!canDispatchPhoneAction(decision.action, observation, fresh)) {
+					if (decision.action.type !== "tapElement" && !canDispatchPhoneAction(decision.action, observation, fresh)) {
+						session.report.diagnostics.stalePredictions++;
+						recoveryHint = "The proposed action was not dispatched because its target changed. Locate it again on the fresh screen.";
+						observation.frame.fill(0);
+						observation = fresh;
+						continue;
+					}
+					let action = enhancements.semanticControls ? groundPhoneCoordinateAction(decision.action, observation) : decision.action;
+					const semanticAction = action.type === "tapElement";
+					if (action.type === "tapElement") {
+						if (!enhancements.semanticControls && !preparedFocus) throw new PhoneError("PHONE_SEMANTIC_DISABLED", "Semantic controls are disabled");
+						try {
+							action = resolvePhoneSemanticAction(action, observation, fresh);
+						} catch (error) {
+							if (!(error instanceof PhoneError) || !error.code.startsWith("PHONE_SEMANTIC_")) throw error;
+							session.report.diagnostics.semanticRejections = (session.report.diagnostics.semanticRejections ?? 0) + 1;
+							recoveryHint = "The semantic target was not dispatched: select one unique fully visible non-password control from the fresh native UI.";
+							observation.frame.fill(0);
+							observation = fresh;
+							continue;
+						}
+					}
+					if (!canDispatchPhoneAction(action, observation, fresh)) {
 						session.report.diagnostics.stalePredictions++;
 						recoveryHint = "The proposed action was not dispatched because its target changed. Locate it again on the fresh screen.";
 						observation.frame.fill(0);
@@ -458,9 +620,9 @@ var PhoneRuntime = class {
 						continue;
 					}
 					if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") continue;
-					const attemptKey = `${observationSignature(fresh)}:${JSON.stringify(decision.action)}`;
+					const attemptKey = `${observationSignature(fresh)}:${JSON.stringify(action)}`;
 					const repetitions = attempts.get(attemptKey) ?? 0;
-					if (repetitions >= (decision.action.type === "wait" ? 3 : 2)) {
+					if (repetitions >= (action.type === "wait" ? 3 : 2)) {
 						session.report.status = "paused";
 						session.report.controlOwner = "none";
 						session.report.controlEpoch++;
@@ -471,7 +633,20 @@ var PhoneRuntime = class {
 						return this.publish(session);
 					}
 					attempts.set(attemptKey, repetitions + 1);
-					if (await this.dispatch(session, decision.action, fresh, context, signal, epoch)) actions++;
+					if (await this.dispatch(session, action, fresh, context, signal, epoch)) {
+						actions++;
+						if (semanticAction && enhancements.semanticControls) session.report.diagnostics.semanticActions = (session.report.diagnostics.semanticActions ?? 0) + 1;
+						if (preparedFocus) {
+							pendingFocus = preparedFocus;
+							focusedIntents.add(JSON.stringify(preparedFocus.intendedAction));
+							session.report.diagnostics.focusRecoveries = (session.report.diagnostics.focusRecoveries ?? 0) + 1;
+						}
+						if (resumedText) session.report.diagnostics.resumedTextInputs = (session.report.diagnostics.resumedTextInputs ?? 0) + 1;
+						if (numericStep) {
+							numericPending = numericStep;
+							session.report.diagnostics.numericActions = (session.report.diagnostics.numericActions ?? 0) + 1;
+						}
+					}
 				} finally {
 					if (observation !== fresh) fresh.frame.fill(0);
 				}
@@ -565,7 +740,7 @@ var PhoneRuntime = class {
 		session.report.foregroundApp = observation.package;
 		session.report.facts = [...new Set(observation.elements.flatMap((node) => node.password ? [] : [node.text, node.description]).filter(Boolean))].slice(0, 60);
 	}
-	async dispatch(session, action, observation, context, signal, epoch = session.report.controlEpoch, resetTask = false) {
+	async dispatch(session, action, observation, context, signal, epoch = session.report.controlEpoch, resetTask = false, settingsPage) {
 		return this.io(session, async () => {
 			signal.throwIfAborted();
 			this.guard(session, context);
@@ -610,7 +785,10 @@ var PhoneRuntime = class {
 				session.trajectorySummary = session.trajectorySummary.slice(-12);
 			}
 			try {
-				await session.driver.execute(session.device.serial, action, observation, signal, session.human ? {
+				await session.driver.execute(session.device.serial, action, observation, signal, settingsPage ? {
+					settleMs: session.human ? 0 : 350,
+					settingsPage
+				} : session.human ? {
 					settleMs: 0,
 					...resetTask ? { resetTask: true } : {}
 				} : void 0);
@@ -1023,7 +1201,7 @@ var PhoneRuntime = class {
 				const video = input.geometry !== void 0 && session.stream?.verifyGeometry(input.geometry);
 				if (!video && (!frame || Date.now() - frame.capturedAt > 5e3)) throw new PhoneError("PHONE_FRAME_STALE", "Wait for a fresh phone image before interacting", 409);
 				if (video && input.action?.type !== "typeText" && input.action?.type !== "back") throw new PhoneError("PHONE_INVALID_INPUT", "Use the continuous input channel for video gestures", 409);
-				if (!input.action || input.action.type === "launch" || input.action.type === "home" || input.action.type === "wait") throw new PhoneError("PHONE_INVALID_INPUT", "Unsupported manual input");
+				if (!input.action || input.action.type === "launch" || input.action.type === "home" || input.action.type === "wait" || input.action.type === "tapElement") throw new PhoneError("PHONE_INVALID_INPUT", "Unsupported manual input");
 				const observation = input.action.type === "typeText" ? await this.observe(session, session.controller.signal, true) : await this.capture(session, session.controller.signal);
 				try {
 					if (input.epoch !== session.report.controlEpoch || session.human !== client || Date.now() > (session.humanExpiresAt ?? 0)) throw new PhoneError("PHONE_CONTROL_CHANGED", "Manual control expired while observing the phone", 409);

package/dist/src/phone/semantic-controls.d.ts

@@ -0,0 +1,13 @@
+import { type PhoneAction, type PhoneElementSelector, type PhoneObservation } from './contracts.js';
+export type PhonePrimitiveAction = Exclude<PhoneAction, {
+    type: 'tapElement';
+}>;
+/** Offer exact identities only when one fully visible native node matches. */
+export declare function phoneSemanticControls(screen: PhoneObservation): Array<{
+    frameId: string;
+    selector: PhoneElementSelector;
+}>;
+/** Ground ordinary model taps only when native hit-testing identifies one bounded control. */
+export declare function groundPhoneCoordinateAction(action: PhoneAction, screen: PhoneObservation): PhoneAction;
+/** Resolve against the prediction snapshot and a fresh snapshot before any dispatch. */
+export declare function resolvePhoneSemanticAction(action: PhoneAction, predicted: PhoneObservation, fresh?: PhoneObservation): PhonePrimitiveAction;

package/dist/src/phone/semantic-controls.js

@@ -0,0 +1,76 @@
+import { PhoneActionSchema, PhoneError, init_contracts } from "./contracts.js";
+import { canDispatchPhoneAction } from "./reliability.js";
+//#region src/phone/semantic-controls.ts
+init_contracts();
+function visible(node, screen) {
+	const [left, top, right, bottom] = node.bounds;
+	return !node.password && node.bounds.every(Number.isFinite) && left >= 0 && top >= 0 && right <= screen.width && bottom <= screen.height && right > left && bottom > top;
+}
+function matches(node, selector) {
+	return (selector.resourceId === void 0 || node.resourceId === selector.resourceId) && (selector.text === void 0 || node.text === selector.text) && (selector.description === void 0 || node.description === selector.description);
+}
+function candidates(screen, selector) {
+	return screen.elements.filter((node) => visible(node, screen) && matches(node, selector));
+}
+function exactSelector(node) {
+	const selector = {};
+	if (node.resourceId && node.resourceId.length <= 300) selector.resourceId = node.resourceId;
+	if (node.text && node.text.length <= 4e3) selector.text = node.text;
+	if (node.description && node.description.length <= 1e3) selector.description = node.description;
+	return selector;
+}
+/** Offer exact identities only when one fully visible native node matches. */
+function phoneSemanticControls(screen) {
+	return screen.elements.filter((node) => visible(node, screen)).flatMap((node) => {
+		const selector = exactSelector(node);
+		return Object.keys(selector).length && candidates(screen, selector).length === 1 ? [{
+			frameId: screen.frameId,
+			selector
+		}] : [];
+	}).slice(0, 40);
+}
+function containsPoint(node, x, y) {
+	return x >= node.bounds[0] && y >= node.bounds[1] && x < node.bounds[2] && y < node.bounds[3];
+}
+function containsNode(outer, inner) {
+	return outer.bounds[0] <= inner.bounds[0] && outer.bounds[1] <= inner.bounds[1] && outer.bounds[2] >= inner.bounds[2] && outer.bounds[3] >= inner.bounds[3];
+}
+/** Ground ordinary model taps only when native hit-testing identifies one bounded control. */
+function groundPhoneCoordinateAction(action, screen) {
+	if (action.type !== "tap") return action;
+	const [x, y] = [action.point[0] * screen.width / 1e3, action.point[1] * screen.height / 1e3];
+	if (screen.elements.some((node) => node.password && containsPoint(node, x, y))) return action;
+	const actionable = screen.elements.filter((node) => visible(node, screen) && (node.clickable || node.editable));
+	const hits = actionable.filter((node) => containsPoint(node, x, y));
+	const smallest = hits.filter((node) => hits.every((other) => other === node || containsNode(other, node)));
+	if (smallest.length !== 1) return action;
+	const target = smallest[0];
+	if ((target.bounds[2] - target.bounds[0]) * (target.bounds[3] - target.bounds[1]) > screen.width * screen.height / 4) return action;
+	const selector = exactSelector(target);
+	if (!Object.keys(selector).length || candidates(screen, selector).length !== 1) return action;
+	const [centerX, centerY] = [(target.bounds[0] + target.bounds[2]) / 2, (target.bounds[1] + target.bounds[3]) / 2];
+	if (screen.elements.some((node) => node.password && containsPoint(node, centerX, centerY)) || actionable.some((node) => node !== target && containsPoint(node, centerX, centerY) && !containsNode(node, target))) return action;
+	return {
+		type: "tapElement",
+		frameId: screen.frameId,
+		selector
+	};
+}
+/** Resolve against the prediction snapshot and a fresh snapshot before any dispatch. */
+function resolvePhoneSemanticAction(action, predicted, fresh = predicted) {
+	if (action.type !== "tapElement") return action;
+	if (!PhoneActionSchema.safeParse(action).success) throw new PhoneError("PHONE_SEMANTIC_INVALID", "Invalid exact native element selector", 409);
+	if (action.frameId !== predicted.frameId) throw new PhoneError("PHONE_SEMANTIC_STALE", "Native element belongs to a different observation", 409);
+	const before = candidates(predicted, action.selector);
+	const after = candidates(fresh, action.selector);
+	if (before.length !== 1 || after.length !== 1) throw new PhoneError("PHONE_SEMANTIC_TARGET", "Native element selector must match exactly one fully visible non-password node", 409);
+	const target = after[0];
+	const resolved = {
+		type: "tap",
+		point: [500 * (target.bounds[0] + target.bounds[2]) / fresh.width, 500 * (target.bounds[1] + target.bounds[3]) / fresh.height]
+	};
+	if (JSON.stringify(before[0]) !== JSON.stringify(target) || !canDispatchPhoneAction(resolved, predicted, fresh)) throw new PhoneError("PHONE_SEMANTIC_STALE", "Native element or page changed before dispatch", 409);
+	return resolved;
+}
+//#endregion
+export { groundPhoneCoordinateAction, phoneSemanticControls, resolvePhoneSemanticAction };

package/dist/src/phone/setup/service.d.ts

@@ -0,0 +1,148 @@
+import type { PhoneSetupView } from '@xopcai/gateway-contract';
+import type { PhoneInput } from '../contracts.js';
+import type { PhoneRuntime } from '../runtime.js';
+type Context = {
+    conversationId: string;
+    agentId: string;
+};
+type Setup = Omit<PhoneSetupView, 'readiness' | 'stale' | 'settingsPath' | 'guidanceRefs'> & {
+    environmentRevision?: string;
+    calls: number;
+    owner?: string;
+};
+export interface PhoneSetupDeps {
+    runtime: PhoneRuntime;
+    inspect(context: Context, source?: 'local' | 'aliyun'): Promise<PhoneSetupView['readiness']>;
+    revision(context: Context, deviceId: string): string;
+    connected(deviceId: string, signal?: AbortSignal): Promise<boolean>;
+    leaseActive(deviceId: string): boolean;
+}
+/** Setup only orchestrates existing settings; secrets and configuration writes stay in product forms. */
+export declare class PhoneSetupService {
+    private readonly deps;
+    constructor(deps: PhoneSetupDeps);
+    private store;
+    current(context: Context): Setup;
+    private save;
+    private owned;
+    view(context: Context, setup: Setup): Promise<PhoneSetupView>;
+    command(command: string, args: Record<string, unknown>, context: Context, dryRun?: boolean): Promise<PhoneSetupView | {
+        dryRun?: boolean;
+        sceneId: string;
+        skillName: string;
+        readiness: {
+            enabled: boolean;
+            toolAllowed: boolean;
+            model: {
+                ref?: string;
+                chatRef?: string;
+                source?: "system" | "global" | "agent";
+                supported: boolean;
+                configured: boolean;
+            };
+            dependencies: Array<{
+                kind: "adb" | "scrcpy";
+                state: string;
+                version?: string;
+                canInstall: boolean;
+            }>;
+            devices: Array<{
+                id: string;
+                name: string;
+                source: "local" | "aliyun";
+                enabled: boolean;
+                settingsAllowed: boolean;
+            }>;
+            localDiscovery: Array<{
+                name: string;
+                state: string;
+                emulator: boolean;
+            }>;
+            diagnostic?: string;
+        };
+        setup: PhoneSetupView;
+        settingsPath: string;
+        guidanceRefs: {
+            sceneId: string;
+            path: string;
+            resourceVersion: string;
+        }[];
+    } | {
+        message: string;
+        id: string;
+        conversationId: string;
+        agentId: string;
+        revision: number;
+        state: "configuring" | "verifying" | "completed" | "needs_attention" | "cancelled";
+        source?: "local" | "aliyun";
+        error?: string;
+        stale: boolean;
+        settingsPath: string;
+        readiness: {
+            enabled: boolean;
+            toolAllowed: boolean;
+            model: {
+                ref?: string;
+                chatRef?: string;
+                source?: "system" | "global" | "agent";
+                supported: boolean;
+                configured: boolean;
+            };
+            dependencies: Array<{
+                kind: "adb" | "scrcpy";
+                state: string;
+                version?: string;
+                canInstall: boolean;
+            }>;
+            devices: Array<{
+                id: string;
+                name: string;
+                source: "local" | "aliyun";
+                enabled: boolean;
+                settingsAllowed: boolean;
+            }>;
+            localDiscovery: Array<{
+                name: string;
+                state: string;
+                emulator: boolean;
+            }>;
+            diagnostic?: string;
+        };
+        checks: {
+            configured: boolean;
+            read: boolean;
+            operated: boolean;
+            verified: boolean;
+            released: boolean;
+        };
+        attempt?: {
+            id: string;
+            deviceId: string;
+            modelRef: string;
+            sessionId?: string;
+            goal: string;
+            completion: {
+                appPackage: string;
+                textAny: string[];
+                requireChange: boolean;
+            };
+            maxActions: number;
+            expiresAt: number;
+            released: boolean;
+        };
+        guidanceRefs: Array<{
+            sceneId: "phone";
+            path: string;
+            resourceVersion: string;
+        }>;
+    }>;
+    source(context: Context, id: string, source: 'local' | 'aliyun', revision: number): Promise<PhoneSetupView>;
+    begin(context: Context, id: string, deviceId: string, revision: number, signal?: AbortSignal): Promise<PhoneSetupView>;
+    beforeInput(context: Context, input: PhoneInput): {
+        owner: string;
+        input: PhoneInput;
+    } | undefined;
+    afterInput(context: Context, owner: string, op: PhoneInput['op']): void;
+    release(context: Context, id: string): Promise<PhoneSetupView>;
+}
+export {};