@xopcai/xopc 0.0.414 → 0.0.416

package/dist/src/phone/runtime.js

@@ -8,9 +8,15 @@ import { PhoneInteractiveSession } from "./interactive-session.js";
 import { AdbPhoneDriver, resolveAdbPath } from "./adb-driver.js";
 import { LocalScrcpyProvider } from "./scrcpy.js";
 import { CloudPhones } from "./cloud-phones.js";
+import { advancePhonePlan, buildPhonePlan, createPhonePlanState } from "./planning.js";
+import { canDispatchPhoneAction, observationSignature, phoneCompletionSatisfied } from "./reliability.js";
+import { groundPhoneCoordinateAction, resolvePhoneSemanticAction } from "./semantic-controls.js";
 import { createPhoneModel, isAboutScreen } from "./model-adapter.js";
 import { PhoneRepository } from "./repository.js";
+import { confirmPhoneFocusRecovery, phoneBackRecovery, phoneUnfocusedTypingHint, preparePhoneFocusRecovery } from "./recovery.js";
+import { createPhoneNumericEntry, nextPhoneNumericEntry } from "./numeric-entry.js";
 import { createHash, randomUUID } from "node:crypto";
+import { setTimeout } from "node:timers/promises";
 //#region src/phone/runtime.ts
 init_logger();
 init_contracts();
@@ -58,6 +64,16 @@ var PhoneRuntime = class {
 		if (!driver.applications) throw new PhoneError("PHONE_APPS_UNAVAILABLE", "The phone provider cannot list applications");
 		return driver.applications(serial, signal);
 	}
+	async textInputStatus(id, config, signal) {
+		const device = this.repository.get(id);
+		if (!device.enabled) throw new PhoneError("PHONE_DEVICE_DENIED", "Phone access is disabled", 403);
+		const driver = await this.driverFor(device, config);
+		if (!driver.textInputStatus) throw new PhoneError("PHONE_TEXT_INPUT_UNAVAILABLE", "This phone provider does not expose text input diagnostics", 409);
+		return {
+			...await driver.textInputStatus(device.serial, signal),
+			policy: device.textInput ?? "auto"
+		};
+	}
 	async preview(id, config, signal) {
 		const device = this.repository.get(id);
 		if (!device.enabled) throw new PhoneError("PHONE_DEVICE_DENIED", "Phone access is disabled", 403);
@@ -168,6 +184,9 @@ var PhoneRuntime = class {
 						adbPath,
 						context,
 						applications,
+						trajectory: [],
+						trajectoryCount: 0,
+						trajectorySummary: [],
 						frames: new PhoneFrameCache(),
 						io: Promise.resolve(),
 						ioBusy: false,
@@ -194,7 +213,13 @@ var PhoneRuntime = class {
 							updatedAt: now,
 							verified: false,
 							verificationLevel: "none",
-							steps: []
+							steps: [],
+							diagnostics: {
+								stalePredictions: 0,
+								protocolRepairs: 0,
+								unchangedActions: 0,
+								completionCorrections: 0
+							}
 						}
 					};
 					this.sessions.set(owner, session);
@@ -206,7 +231,7 @@ var PhoneRuntime = class {
 							await this.dispatch(session, {
 								type: "launch",
 								package: input.appPackage
-							}, observation, context, signal, epoch);
+							}, observation, context, signal, epoch, false, input.settingsPage);
 						} finally {
 							observation.frame.fill(0);
 						}
@@ -259,6 +284,9 @@ var PhoneRuntime = class {
 		const startedAt = Date.now();
 		session.report.errorCode = void 0;
 		session.report.verified = false;
+		session.report.verificationLevel = "none";
+		session.report.stopReason = void 0;
+		session.report.message = void 0;
 		try {
 			await this.waitForAgent(session, signal);
 			session.report.status = "running";
@@ -282,7 +310,42 @@ var PhoneRuntime = class {
 			if (session.report.mode !== "control") throw new PhoneError("PHONE_OBSERVE_ONLY", "Reopen the phone in control mode to execute actions");
 			if (session.report.mock && input.scenario !== "settings-about") throw new PhoneError("PHONE_MOCK_SCENARIO_REQUIRED", "Mock mode only supports the explicit settings-about scenario");
 			const maxActions = input.op === "step" ? 1 : input.maxActions;
+			const requiredTexts = [...input.completion?.textAll ?? [], ...input.expectText ? [input.expectText] : []];
+			const completion = input.completion || input.expectText ? {
+				...input.completion,
+				textAll: requiredTexts.length ? requiredTexts : void 0
+			} : void 0;
 			let actions = 0;
+			let protocolFeedback;
+			let repairs = 0;
+			let completionCorrections = 0;
+			let recoveryHint;
+			const enhancements = {
+				semanticControls: context.config.phone.semanticControls,
+				hierarchicalPlanning: context.config.phone.hierarchicalPlanning
+			};
+			const derivedPlan = enhancements.hierarchicalPlanning ? buildPhonePlan(input.goal, completion) : void 0;
+			let planning = derivedPlan ? createPhonePlanState(derivedPlan) : void 0;
+			if (enhancements.hierarchicalPlanning) {
+				session.report.diagnostics.plannedSubgoals = planning?.steps.length ?? 0;
+				session.report.diagnostics.completedSubgoals = 0;
+			}
+			let changed = false;
+			const entryObservation = {
+				...observation,
+				frame: Buffer.alloc(0),
+				elements: structuredClone(observation.elements)
+			};
+			const trajectoryStart = session.trajectoryCount;
+			let pendingFocus;
+			let numericIntent;
+			let numericPending;
+			const focusedIntents = /* @__PURE__ */ new Set();
+			let backReplans = 0;
+			let focusReplans = 0;
+			let previousSignature = observationSignature(observation);
+			let controlEpoch = session.report.controlEpoch;
+			const attempts = /* @__PURE__ */ new Map();
 			while (true) {
 				if (await this.waitForAgent(session, signal)) {
 					observation.frame.fill(0);
@@ -292,45 +355,151 @@ var PhoneRuntime = class {
 				this.guard(session, context);
 				this.allowedScreen(session, observation);
 				const epoch = session.report.controlEpoch;
+				if (controlEpoch !== epoch) {
+					attempts.clear();
+					recoveryHint = "The user handed back control. Re-evaluate the fresh screen before continuing.";
+					pendingFocus = void 0;
+					numericIntent = void 0;
+					numericPending = void 0;
+					controlEpoch = epoch;
+				}
+				const signature = observationSignature(observation);
+				if (signature !== previousSignature) changed = true;
+				previousSignature = signature;
 				this.recordObservation(session, observation);
-				if (session.report.mock ? isAboutScreen(observation) : Boolean(input.expectText && session.report.facts?.some((text) => text.includes(input.expectText)))) {
-					session.report.verified = true;
-					session.report.verificationLevel = "ui";
-					session.report.status = "completed";
-					session.report.stopReason = actions ? "condition_satisfied" : "condition_present";
-					return this.publish(session);
+				if (planning) {
+					planning = advancePhonePlan(planning, observation);
+					session.report.diagnostics.completedSubgoals = planning.verifiedSteps;
+				}
+				let numericStep;
+				if (enhancements.hierarchicalPlanning) {
+					numericIntent ??= createPhoneNumericEntry(input.goal, completion, observation);
+					if (numericIntent) {
+						let next = nextPhoneNumericEntry(numericIntent, observation, numericPending);
+						for (let poll = 0; numericPending && next.kind === "blocked" && poll < 2; poll++) {
+							await setTimeout(150, void 0, { signal });
+							observation.frame.fill(0);
+							observation = await this.observe(session, signal);
+							this.guard(session, context);
+							this.allowedScreen(session, observation);
+							if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") break;
+							next = nextPhoneNumericEntry(numericIntent, observation, numericPending);
+						}
+						if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") continue;
+						this.recordObservation(session, observation);
+						if (next.kind === "blocked") {
+							session.report.diagnostics.numericMismatches = (session.report.diagnostics.numericMismatches ?? 0) + 1;
+							session.report.status = "paused";
+							session.report.controlOwner = "none";
+							session.report.controlEpoch++;
+							session.report.stopReason = "numeric_state_mismatch";
+							session.report.message = "Numeric input was not confirmed. Inspect the displayed value before continuing; no key was retried.";
+							session.budget?.pause();
+							await session.stream?.revoke();
+							return this.publish(session);
+						}
+						if (numericPending) session.report.diagnostics.numericConfirmedSteps = (session.report.diagnostics.numericConfirmedSteps ?? 0) + 1;
+						numericPending = void 0;
+						if (next.kind === "action") numericStep = next.step;
+					}
+				}
+				if (session.report.mock ? isAboutScreen(observation) : Boolean(completion && phoneCompletionSatisfied(completion, observation, changed))) {
+					const confirmation = await this.observe(session, signal);
+					try {
+						signal.throwIfAborted();
+						this.guard(session, context);
+						this.allowedScreen(session, confirmation);
+						const confirmed = session.report.mock ? isAboutScreen(confirmation) : Boolean(completion && phoneCompletionSatisfied(completion, confirmation, changed));
+						if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent" || !confirmed) {
+							observation.frame.fill(0);
+							observation = confirmation;
+							continue;
+						}
+						this.recordObservation(session, confirmation);
+						if (planning) {
+							planning = advancePhonePlan(planning, confirmation);
+							session.report.diagnostics.completedSubgoals = planning.verifiedSteps;
+						}
+						session.report.verified = true;
+						session.report.verificationLevel = "ui";
+						session.report.status = "completed";
+						session.report.stopReason = actions ? "condition_satisfied" : "condition_present";
+						return this.publish(session);
+					} finally {
+						if (observation !== confirmation) confirmation.frame.fill(0);
+					}
 				}
 				if (actions >= maxActions) {
 					session.report.status = "ready";
 					session.report.stopReason = "action_limit";
 					return this.publish(session);
 				}
-				if (session.report.modelRequests >= context.config.phone.maxModelRequests) throw new PhoneError("PHONE_MODEL_LIMIT", "Phone model request limit reached");
-				session.report.modelRequests++;
-				session.report.currentAction = void 0;
-				this.publish(session);
-				session.prediction = new AbortController();
-				let decision;
-				try {
-					decision = await session.adapter.predict({
-						goal: input.goal,
-						observation,
-						allowedApps: session.device.allowedApps,
-						appAccess: phoneAppAccess(session.device),
-						applications: session.applications.filter((app) => phoneAllowsApp(session.device, app.packageName)),
-						history: session.report.steps.map((step) => step.action),
-						scenario: input.scenario
-					}, AbortSignal.any([signal, session.prediction.signal]));
-				} catch (error) {
-					if (!signal.aborted && epoch !== session.report.controlEpoch) {
-						await this.waitForAgent(session, signal);
-						observation.frame.fill(0);
-						observation = await this.observe(session, signal);
-						continue;
+				let decision = numericStep ? {
+					kind: "action",
+					action: numericStep.action
+				} : void 0;
+				let resumedText = false;
+				if (!decision && pendingFocus) {
+					const pending = pendingFocus;
+					pendingFocus = void 0;
+					const intended = confirmPhoneFocusRecovery(pending, observation);
+					if (intended) {
+						decision = {
+							kind: "action",
+							action: intended
+						};
+						resumedText = true;
+					} else recoveryHint = "The focus transition was not confirmed. No text was sent. Reinspect the intended editable field before another action.";
+				}
+				if (!decision) {
+					if (session.report.modelRequests >= context.config.phone.maxModelRequests) throw new PhoneError("PHONE_MODEL_LIMIT", "Phone model request limit reached");
+					session.report.modelRequests++;
+					session.report.currentAction = void 0;
+					this.publish(session);
+					session.prediction = new AbortController();
+					try {
+						decision = await session.adapter.predict({
+							goal: input.goal,
+							observation,
+							completion,
+							allowedApps: session.device.allowedApps,
+							appAccess: phoneAppAccess(session.device),
+							applications: session.applications.filter((app) => phoneAllowsApp(session.device, app.packageName)),
+							history: session.report.steps.map((step) => step.action),
+							trajectory: session.trajectory,
+							progressSummary: JSON.stringify({
+								dispatched: session.trajectoryCount,
+								earlier: [...session.trajectorySummary, ...session.trajectory.slice(0, -8).map((entry) => JSON.stringify({
+									action: entry.action,
+									effect: entry.effect,
+									afterApp: entry.afterApp
+								}).slice(0, 250))].slice(-12)
+							}).slice(-1500),
+							recoveryHint,
+							protocolFeedback,
+							scenario: input.scenario,
+							enhancements,
+							planning
+						}, AbortSignal.any([signal, session.prediction.signal]));
+					} catch (error) {
+						if (!signal.aborted && epoch !== session.report.controlEpoch) {
+							await this.waitForAgent(session, signal);
+							observation.frame.fill(0);
+							observation = await this.observe(session, signal);
+							continue;
+						}
+						if (!signal.aborted && error instanceof PhoneError && ["PHONE_MODEL_PROTOCOL", "PHONE_MODEL_TRUNCATED"].includes(error.code) && repairs < 1 && session.report.modelRequests < context.config.phone.maxModelRequests) {
+							repairs++;
+							session.report.diagnostics.protocolRepairs++;
+							protocolFeedback = "Your previous output was invalid or truncated. Return exactly one complete action in the specified format with no extra prose.";
+							observation.frame.fill(0);
+							observation = await this.observe(session, signal);
+							continue;
+						}
+						throw error;
+					} finally {
+						session.prediction = void 0;
 					}
-					throw error;
-				} finally {
-					session.prediction = void 0;
 				}
 				signal.throwIfAborted();
 				this.guard(session, context);
@@ -340,9 +509,26 @@ var PhoneRuntime = class {
 					observation = await this.observe(session, signal);
 					continue;
 				}
+				protocolFeedback = void 0;
+				if (enhancements.hierarchicalPlanning && !planning && decision.plan) {
+					planning = advancePhonePlan(createPhonePlanState(decision.plan), observation);
+					session.report.diagnostics.plannedSubgoals = planning.steps.length;
+					session.report.diagnostics.completedSubgoals = planning.verifiedSteps;
+				}
 				if (decision.kind !== "action") {
+					if (decision.kind === "complete" && !session.report.mock && (input.completion || input.expectText) && completionCorrections < 1 && session.report.modelRequests < context.config.phone.maxModelRequests) {
+						completionCorrections++;
+						session.report.diagnostics.completionCorrections++;
+						recoveryHint = `Your completion claim is not verified. These required native UI conditions are not all satisfied: ${JSON.stringify({
+							expectText: input.expectText,
+							...input.completion
+						})}. Recheck the fresh screen, preserve user constraints, and continue if a safe authorized action can satisfy the goal; otherwise report unfinished.`;
+						observation.frame.fill(0);
+						observation = await this.observe(session, signal);
+						continue;
+					}
 					session.report.status = decision.kind === "needsUser" ? "needs_user" : "completed";
-					session.report.message = decision.kind === "needsUser" ? decision.reason : void 0;
+					session.report.message = decision.kind === "needsUser" ? decision.reason : decision.summary;
 					session.report.stopReason = decision.kind === "needsUser" ? "user_takeover" : "model_finished_unverified";
 					if (decision.kind === "needsUser" && !session.report.mock) {
 						session.report.controlOwner = "none";
@@ -356,24 +542,142 @@ var PhoneRuntime = class {
 					}
 					return this.publish(session);
 				}
+				let preparedFocus;
+				if (enhancements.hierarchicalPlanning) {
+					const operationActions = session.trajectoryCount - trajectoryStart;
+					const back = phoneBackRecovery({
+						action: decision.action,
+						observation,
+						entryObservation,
+						completion,
+						trajectory: operationActions ? session.trajectory.slice(-operationActions) : []
+					});
+					if (back) {
+						session.report.diagnostics.backReplans = (session.report.diagnostics.backReplans ?? 0) + 1;
+						if (backReplans++ >= 2) {
+							session.report.status = "paused";
+							session.report.controlOwner = "none";
+							session.report.controlEpoch++;
+							session.report.stopReason = "back_boundary";
+							session.report.message = back.hint;
+							session.budget?.pause();
+							await session.stream?.revoke();
+							return this.publish(session);
+						}
+						recoveryHint = back.hint;
+						observation.frame.fill(0);
+						observation = await this.observe(session, signal);
+						continue;
+					}
+					const focusHint = phoneUnfocusedTypingHint(decision.action, observation, completion);
+					if (focusHint) {
+						const key = JSON.stringify(decision.action);
+						preparedFocus = !focusedIntents.has(key) ? preparePhoneFocusRecovery(decision.action, observation, completion) : void 0;
+						if (preparedFocus) decision = {
+							kind: "action",
+							action: preparedFocus.focusAction
+						};
+						else {
+							session.report.diagnostics.focusReplans = (session.report.diagnostics.focusReplans ?? 0) + 1;
+							if (focusReplans++ >= 2) throw new PhoneError("PHONE_INPUT_NOT_FOCUSED", "The requested editable field could not be focused; inspect before typing", 409);
+							recoveryHint = focusHint;
+							observation.frame.fill(0);
+							observation = await this.observe(session, signal);
+							continue;
+						}
+					}
+				}
 				const fresh = await this.observe(session, signal);
 				try {
 					this.allowedScreen(session, fresh);
-					if (fresh.package !== observation.package || fresh.width !== observation.width || fresh.height !== observation.height || fresh.rotation !== observation.rotation || JSON.stringify(fresh.elements) !== JSON.stringify(observation.elements) || !fresh.elements.length && !fresh.frame.equals(observation.frame)) {
+					if (decision.action.type !== "tapElement" && !canDispatchPhoneAction(decision.action, observation, fresh)) {
+						session.report.diagnostics.stalePredictions++;
+						recoveryHint = "The proposed action was not dispatched because its target changed. Locate it again on the fresh screen.";
+						observation.frame.fill(0);
+						observation = fresh;
+						continue;
+					}
+					let action = enhancements.semanticControls ? groundPhoneCoordinateAction(decision.action, observation) : decision.action;
+					const semanticAction = action.type === "tapElement";
+					if (action.type === "tapElement") {
+						if (!enhancements.semanticControls && !preparedFocus) throw new PhoneError("PHONE_SEMANTIC_DISABLED", "Semantic controls are disabled");
+						try {
+							action = resolvePhoneSemanticAction(action, observation, fresh);
+						} catch (error) {
+							if (!(error instanceof PhoneError) || !error.code.startsWith("PHONE_SEMANTIC_")) throw error;
+							session.report.diagnostics.semanticRejections = (session.report.diagnostics.semanticRejections ?? 0) + 1;
+							recoveryHint = "The semantic target was not dispatched: select one unique fully visible non-password control from the fresh native UI.";
+							observation.frame.fill(0);
+							observation = fresh;
+							continue;
+						}
+					}
+					if (!canDispatchPhoneAction(action, observation, fresh)) {
+						session.report.diagnostics.stalePredictions++;
+						recoveryHint = "The proposed action was not dispatched because its target changed. Locate it again on the fresh screen.";
 						observation.frame.fill(0);
 						observation = fresh;
 						continue;
 					}
 					if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") continue;
-					if (await this.dispatch(session, decision.action, fresh, context, signal, epoch)) actions++;
+					const attemptKey = `${observationSignature(fresh)}:${JSON.stringify(action)}`;
+					const repetitions = attempts.get(attemptKey) ?? 0;
+					if (repetitions >= (action.type === "wait" ? 3 : 2)) {
+						session.report.status = "paused";
+						session.report.controlOwner = "none";
+						session.report.controlEpoch++;
+						session.report.stopReason = "no_progress";
+						session.report.message = "Repeated actions made no progress. Inspect the phone and hand back before continuing.";
+						session.budget?.pause();
+						await session.stream?.revoke();
+						return this.publish(session);
+					}
+					attempts.set(attemptKey, repetitions + 1);
+					if (await this.dispatch(session, action, fresh, context, signal, epoch)) {
+						actions++;
+						if (semanticAction && enhancements.semanticControls) session.report.diagnostics.semanticActions = (session.report.diagnostics.semanticActions ?? 0) + 1;
+						if (preparedFocus) {
+							pendingFocus = preparedFocus;
+							focusedIntents.add(JSON.stringify(preparedFocus.intendedAction));
+							session.report.diagnostics.focusRecoveries = (session.report.diagnostics.focusRecoveries ?? 0) + 1;
+						}
+						if (resumedText) session.report.diagnostics.resumedTextInputs = (session.report.diagnostics.resumedTextInputs ?? 0) + 1;
+						if (numericStep) {
+							numericPending = numericStep;
+							session.report.diagnostics.numericActions = (session.report.diagnostics.numericActions ?? 0) + 1;
+						}
+					}
 				} finally {
 					if (observation !== fresh) fresh.frame.fill(0);
 				}
+				const beforeSignature = observationSignature(observation);
 				observation.frame.fill(0);
 				observation = await this.observe(session, signal);
+				const entry = session.trajectory.at(-1);
+				if (entry && entry.actor === "agent" && epoch === session.report.controlEpoch) {
+					const unchanged = beforeSignature === observationSignature(observation);
+					entry.afterApp = observation.package;
+					entry.effect = unchanged ? "unchanged" : "changed";
+					entry.facts = observation.elements.filter((node) => !node.password).flatMap((node) => [node.text, node.description]).filter(Boolean).slice(0, 12).map((text) => text.slice(0, 160));
+					if (unchanged) session.report.diagnostics.unchangedActions++;
+					recoveryHint = unchanged ? "The last dispatched action produced no observable change. Wait if loading, otherwise try a different target or route; do not repeat an uncertain input." : void 0;
+				}
 			}
 		} catch (error) {
 			const failure = signal.aborted && signal.reason instanceof PhoneError ? signal.reason : error;
+			if (!signal.aborted && failure instanceof PhoneError && ["PHONE_TEXT_INPUT_UNAVAILABLE", "PHONE_TEXT_INPUT_UNCONFIRMED"].includes(failure.code)) {
+				session.report.status = "needs_user";
+				session.human = void 0;
+				session.report.controlOwner = "none";
+				session.report.controlEpoch++;
+				session.report.errorCode = failure.code;
+				session.report.message = failure.message;
+				session.report.currentAction = void 0;
+				session.report.stopReason = failure.code === "PHONE_TEXT_INPUT_UNAVAILABLE" ? "text_input_required" : "text_input_unconfirmed";
+				session.budget?.pause();
+				await session.stream?.revoke();
+				return this.publish(session);
+			}
 			if (failure instanceof PhoneError && failure.code === "PHONE_OPERATION_TIMEOUT" && !terminal.has(session.report.status) && session.report.status !== "paused") {
 				session.report.status = "paused";
 				session.report.controlOwner = "none";
@@ -427,7 +731,8 @@ var PhoneRuntime = class {
 				status: session.report.status,
 				stopReason: session.report.stopReason,
 				actions: session.report.actions,
-				modelRequests: session.report.modelRequests
+				modelRequests: session.report.modelRequests,
+				diagnostics: session.report.diagnostics
 			}, "Phone goal finished");
 		}
 	}
@@ -435,7 +740,7 @@ var PhoneRuntime = class {
 		session.report.foregroundApp = observation.package;
 		session.report.facts = [...new Set(observation.elements.flatMap((node) => node.password ? [] : [node.text, node.description]).filter(Boolean))].slice(0, 60);
 	}
-	async dispatch(session, action, observation, context, signal, epoch = session.report.controlEpoch, resetTask = false) {
+	async dispatch(session, action, observation, context, signal, epoch = session.report.controlEpoch, resetTask = false, settingsPage) {
 		return this.io(session, async () => {
 			signal.throwIfAborted();
 			this.guard(session, context);
@@ -445,6 +750,7 @@ var PhoneRuntime = class {
 			if (session.human && (session.report.humanActions ?? 0) >= 1200) throw new PhoneError("PHONE_ACTION_LIMIT", "Manual input limit reached; reopen the session");
 			if (action.type === "launch" && !phoneAllowsApp(session.device, action.package)) throw new PhoneError("PHONE_APP_DENIED", "Application is not authorized");
 			if (action.type === "home" && phoneAppAccess(session.device) !== "all") throw new PhoneError("PHONE_APP_DENIED", "Home leaves the app allowlist");
+			if (action.type === "typeText" && session.device.textInput === "manual") throw new PhoneError("PHONE_TEXT_INPUT_UNAVAILABLE", "Automatic text input is disabled for this phone. Ask the user to take over and type with the phone keyboard.", 409);
 			if (action.type === "typeText" && !observation.elements.some((node) => node.focused && node.editable && !node.password)) throw new PhoneError("PHONE_INPUT_NOT_FOCUSED", "Focus a non-password editable field before typing", 409);
 			const step = {
 				action: action.type,
@@ -460,14 +766,51 @@ var PhoneRuntime = class {
 				session.report.steps = session.report.steps.slice(-200);
 			}
 			this.publish(session);
+			const entry = {
+				action: structuredClone(action),
+				dispatch: "unknown",
+				actor: step.actor,
+				beforeApp: observation.package,
+				effect: "unknown"
+			};
+			session.trajectory.push(entry);
+			session.trajectoryCount++;
+			if (session.trajectory.length > 24) {
+				const earlier = session.trajectory.shift();
+				session.trajectorySummary.push(JSON.stringify({
+					action: earlier.action,
+					effect: earlier.effect,
+					afterApp: earlier.afterApp
+				}).slice(0, 250));
+				session.trajectorySummary = session.trajectorySummary.slice(-12);
+			}
 			try {
-				await session.driver.execute(session.device.serial, action, observation, signal, session.human ? {
+				await session.driver.execute(session.device.serial, action, observation, signal, settingsPage ? {
+					settleMs: session.human ? 0 : 350,
+					settingsPage
+				} : session.human ? {
 					settleMs: 0,
 					...resetTask ? { resetTask: true } : {}
 				} : void 0);
 				step.dispatch = "completed";
+				entry.dispatch = "completed";
 			} catch (error) {
-				session.report.status = "outcome_unknown";
+				const textFailure = error instanceof PhoneError && ["PHONE_TEXT_INPUT_UNAVAILABLE", "PHONE_TEXT_INPUT_UNCONFIRMED"].includes(error.code);
+				if (textFailure && error.code === "PHONE_TEXT_INPUT_UNAVAILABLE") {
+					session.report.steps.pop();
+					session.report.actions--;
+					session.trajectory.pop();
+					session.trajectoryCount--;
+					if (session.human) session.report.humanActions--;
+				} else session.report.status = "outcome_unknown";
+				if (textFailure && session.human) {
+					session.report.status = "paused";
+					session.report.errorCode = error.code;
+					session.report.message = error.message;
+					session.report.stopReason = error.code === "PHONE_TEXT_INPUT_UNAVAILABLE" ? "text_input_required" : "text_input_unconfirmed";
+					session.report.currentAction = void 0;
+					this.publish(session);
+				}
 				session.frames.clear();
 				throw error;
 			}
@@ -497,6 +840,8 @@ var PhoneRuntime = class {
 		if (op !== "pause") {
 			session.report.facts = void 0;
 			session.report.message = void 0;
+			session.trajectory = [];
+			session.trajectorySummary = [];
 			session.frames.clear();
 			session.adapter = { predict: async () => {
 				throw new PhoneError("PHONE_OPEN_REQUIRED", "Reopen the phone session");
@@ -625,6 +970,7 @@ var PhoneRuntime = class {
 			ownControl: Boolean(session.human === client),
 			allowedApps: [...session.device.allowedApps],
 			appAccess: phoneAppAccess(session.device),
+			textInput: session.device.textInput ?? "auto",
 			frame: session.frames.get() ? {
 				seq: session.frames.get().seq,
 				capturedAt: session.frames.get().capturedAt
@@ -855,7 +1201,7 @@ var PhoneRuntime = class {
 				const video = input.geometry !== void 0 && session.stream?.verifyGeometry(input.geometry);
 				if (!video && (!frame || Date.now() - frame.capturedAt > 5e3)) throw new PhoneError("PHONE_FRAME_STALE", "Wait for a fresh phone image before interacting", 409);
 				if (video && input.action?.type !== "typeText" && input.action?.type !== "back") throw new PhoneError("PHONE_INVALID_INPUT", "Use the continuous input channel for video gestures", 409);
-				if (!input.action || input.action.type === "launch" || input.action.type === "home" || input.action.type === "wait") throw new PhoneError("PHONE_INVALID_INPUT", "Unsupported manual input");
+				if (!input.action || input.action.type === "launch" || input.action.type === "home" || input.action.type === "wait" || input.action.type === "tapElement") throw new PhoneError("PHONE_INVALID_INPUT", "Unsupported manual input");
 				const observation = input.action.type === "typeText" ? await this.observe(session, session.controller.signal, true) : await this.capture(session, session.controller.signal);
 				try {
 					if (input.epoch !== session.report.controlEpoch || session.human !== client || Date.now() > (session.humanExpiresAt ?? 0)) throw new PhoneError("PHONE_CONTROL_CHANGED", "Manual control expired while observing the phone", 409);

package/dist/src/phone/scrcpy-text.d.ts

@@ -0,0 +1,11 @@
+/** Only supplied text is sent to Android; clipboard autosync and reads stay off. */
+export declare class ScrcpyTextChannel {
+    private write;
+    private buffer;
+    private sequence;
+    private pending;
+    constructor(write: (bytes: Buffer) => Promise<void>);
+    receive(bytes: Buffer): void;
+    paste(text: string, signal: AbortSignal): Promise<void>;
+    close(): void;
+}

package/dist/src/phone/scrcpy-text.js

@@ -0,0 +1,72 @@
+import { PhoneError, init_contracts } from "./contracts.js";
+//#region src/phone/scrcpy-text.ts
+init_contracts();
+/** Only supplied text is sent to Android; clipboard autosync and reads stay off. */
+var ScrcpyTextChannel = class {
+	buffer = Buffer.alloc(0);
+	sequence = 0n;
+	pending = /* @__PURE__ */ new Map();
+	constructor(write) {
+		this.write = write;
+	}
+	receive(bytes) {
+		this.buffer = Buffer.concat([this.buffer, bytes]);
+		while (this.buffer.length) {
+			const type = this.buffer[0];
+			let length;
+			if (type === 1) {
+				if (this.buffer.length < 9) return;
+				const seq = this.buffer.readBigUInt64BE(1);
+				this.pending.get(seq)?.resolve();
+				this.pending.delete(seq);
+				length = 9;
+			} else if (type === 0 || type === 2) {
+				if (this.buffer.length < 5) return;
+				length = 5 + (type === 0 ? this.buffer.readUInt32BE(1) : this.buffer.readUInt16BE(3));
+				if (length > 262144) throw new Error("Invalid device message");
+				if (this.buffer.length < length) return;
+			} else throw new Error("Unknown device message");
+			this.buffer = this.buffer.subarray(length);
+		}
+	}
+	async paste(text, signal) {
+		signal.throwIfAborted();
+		if (!text || text.length > 4e3 || this.pending.size) throw new PhoneError("PHONE_INVALID_INPUT", "Invalid or concurrent phone text input");
+		const sequence = ++this.sequence;
+		const payload = Buffer.from(text);
+		const bytes = Buffer.alloc(14 + payload.length);
+		bytes[0] = 9;
+		bytes.writeBigUInt64BE(sequence, 1);
+		bytes[9] = 1;
+		bytes.writeUInt32BE(payload.length, 10);
+		payload.copy(bytes, 14);
+		payload.fill(0);
+		const deadline = AbortSignal.any([signal, AbortSignal.timeout(1e4)]);
+		let abort;
+		const ack = new Promise((resolve, reject) => {
+			abort = () => reject(new PhoneError("PHONE_TEXT_INPUT_UNCONFIRMED", "Text may have been pasted. Take over and inspect the input field before retrying.", 409));
+			this.pending.set(sequence, {
+				resolve,
+				reject
+			});
+			deadline.addEventListener("abort", abort, { once: true });
+		});
+		try {
+			await Promise.all([this.write(bytes), ack]);
+		} catch {
+			throw new PhoneError("PHONE_TEXT_INPUT_UNCONFIRMED", "Text may have been pasted. Take over and inspect the input field before retrying.", 409);
+		} finally {
+			bytes.fill(0);
+			deadline.removeEventListener("abort", abort);
+			this.pending.delete(sequence);
+		}
+	}
+	close() {
+		for (const request of this.pending.values()) request.reject(new PhoneError("PHONE_TEXT_INPUT_UNCONFIRMED", "Phone text channel disconnected; inspect the field before retrying.", 409));
+		this.pending.clear();
+		this.buffer.fill(0);
+		this.buffer = Buffer.alloc(0);
+	}
+};
+//#endregion
+export { ScrcpyTextChannel };