@xopcai/xopc 0.0.433 → 0.0.435

package/dist/src/phone/runtime.d.ts

@@ -25,6 +25,8 @@ export declare class PhoneRuntime {
     private modelFactory;
     private interactiveFactory;
     readonly wireless: WirelessPhones;
+    private harmonyDriverFactory;
+    private harmonyVideoFactory;
     private reconnecting;
     private reconnectAfter;
     private cloudPhones?;
@@ -38,9 +40,9 @@ export declare class PhoneRuntime {
     private openings;
     private deviceOpenings;
     private maintenance?;
-    constructor(repository?: PhoneRepository, driverFactory?: (path: string) => PhoneDriver, modelFactory?: typeof createPhoneModel, interactiveFactory?: (adb: string, serial: string, failed: () => void) => PhoneInteractiveProvider, wireless?: WirelessPhones);
+    constructor(repository?: PhoneRepository, driverFactory?: (path: string) => PhoneDriver, modelFactory?: typeof createPhoneModel, interactiveFactory?: (adb: string, serial: string, failed: () => void) => PhoneInteractiveProvider, wireless?: WirelessPhones, harmonyDriverFactory?: (path: string) => PhoneDriver, harmonyVideoFactory?: (serial: string, config: Config['phone'], failed: () => void, validateScreen: (app: string) => void) => PhoneInteractiveProvider);
     driverFor(device: PhoneDevice, config: Config): Promise<PhoneDriver>;
-    discover(config: Config, signal?: AbortSignal): Promise<import("./contracts.js").DiscoveredPhone[]>;
+    discover(config: Config, signal?: AbortSignal, platform?: 'android' | 'harmonyos'): Promise<import("./contracts.js").DiscoveredPhone[]>;
     reconnectWireless(id: string, config: Config, signal?: AbortSignal, endpoint?: {
         host: string;
         port: number;
@@ -48,12 +50,13 @@ export declare class PhoneRuntime {
     applications(input: {
         deviceId?: string;
         serial?: string;
+        platform?: 'android' | 'harmonyos';
     }, config: Config, signal: AbortSignal): Promise<PhoneApplication[]>;
     textInputStatus(id: string, config: Config, signal: AbortSignal): Promise<{
         policy: "manual" | "auto";
-        androidSdk: number;
+        androidSdk?: number;
         unicode: "available" | "setup_required" | "unsupported";
-        method: "scrcpy-clipboard" | "provider-native";
+        method: "scrcpy-clipboard" | "provider-native" | "hdc-uitest";
         keyboardRequired: false;
     }>;
     preview(id: string, config: Config, signal: AbortSignal): Promise<PhoneObservation>;
@@ -76,6 +79,7 @@ export declare class PhoneRuntime {
     private busy;
     private finishTask;
     private perform;
+    private collectOutputs;
     private recordObservation;
     private modelRequestLimit;
     private readOutputFields;
@@ -160,6 +164,7 @@ export declare class PhoneRuntime {
         allowedApps: string[];
         appAccess: "all" | "selected";
         textInput: "manual" | "auto";
+        media: "snapshot" | "hdc-h264";
         frame: {
             seq: number;
             capturedAt: number;
@@ -172,6 +177,7 @@ export declare class PhoneRuntime {
     private expireHuman;
     liveFrame(owner: string, id: string, context: ExecutionContext, signal: AbortSignal): Promise<{
         frame: Buffer<ArrayBuffer>;
+        platform?: "android" | "harmonyos";
         height: number;
         capturedAt: number;
         package: string;

package/dist/src/phone/runtime.js

@@ -9,12 +9,15 @@ import { PhoneFrameCache, PhoneOperationBudget } from "./live-state.js";
 import { PhoneInteractiveSession } from "./interactive-session.js";
 import { LocalScrcpyProvider } from "./scrcpy.js";
 import { pendingPhoneAssistance, phoneAssistance, updatePhoneCheckpoint } from "./assistance.js";
+import { canDispatchPhoneAction, observationSignature, phoneCompletionSatisfied } from "./reliability.js";
 import { AdbPhoneDriver, resolveAdbPath } from "./adb-driver.js";
 import { CloudPhones } from "./cloud-phones.js";
 import { PhoneLocationEnvironment } from "./location-environment.js";
 import { WirelessPhones } from "./wireless.js";
+import { HdcPhoneDriver, resolveHdcPath } from "./hdc-driver.js";
+import { resolveHdcVideoTools } from "./hdc-video-tools.js";
+import { HdcVideoProvider } from "./hdc-video.js";
 import { advancePhonePlan, buildPhonePlan, createPhonePlanState } from "./planning.js";
-import { canDispatchPhoneAction, observationSignature, phoneCompletionSatisfied } from "./reliability.js";
 import { groundPhoneCoordinateAction, resolvePhoneSemanticAction } from "./semantic-controls.js";
 import { createPhoneModel, isAboutScreen } from "./model-adapter.js";
 import { PhoneRepository } from "./repository.js";
@@ -22,7 +25,7 @@ import { phoneCompletionGuidance, phoneCompletionRecoveryHint } from "./completi
 import { confirmPhoneFocusRecovery, phoneBackRecovery, phoneUnfocusedTypingHint, preparePhoneFocusRecovery } from "./recovery.js";
 import { createPhoneNumericEntry, nextPhoneNumericEntry } from "./numeric-entry.js";
 import { createHash, randomUUID } from "node:crypto";
-import { setTimeout } from "node:timers/promises";
+import { setTimeout as setTimeout$1 } from "node:timers/promises";
 //#region src/phone/runtime.ts
 init_logger();
 init_contracts();
@@ -82,19 +85,29 @@ var PhoneRuntime = class {
 	openings = /* @__PURE__ */ new Map();
 	deviceOpenings = /* @__PURE__ */ new Map();
 	maintenance;
-	constructor(repository = new PhoneRepository(), driverFactory = (path) => new AdbPhoneDriver(path), modelFactory = createPhoneModel, interactiveFactory = (adb, serial, failed) => new LocalScrcpyProvider(adb, serial, failed), wireless = new WirelessPhones()) {
+	constructor(repository = new PhoneRepository(), driverFactory = (path) => new AdbPhoneDriver(path), modelFactory = createPhoneModel, interactiveFactory = (adb, serial, failed) => new LocalScrcpyProvider(adb, serial, failed), wireless = new WirelessPhones(), harmonyDriverFactory = (path) => new HdcPhoneDriver(path), harmonyVideoFactory = (serial, config, failed, validateScreen) => new HdcVideoProvider(serial, config, failed, { validateScreen })) {
 		this.repository = repository;
 		this.driverFactory = driverFactory;
 		this.modelFactory = modelFactory;
 		this.interactiveFactory = interactiveFactory;
 		this.wireless = wireless;
+		this.harmonyDriverFactory = harmonyDriverFactory;
+		this.harmonyVideoFactory = harmonyVideoFactory;
 	}
 	async driverFor(device, config) {
-		return device.connection ? this.cloud.driver(device) : this.driverFactory(resolveAdbPath(config.phone.adbPath));
+		return device.platform === "harmonyos" ? this.harmonyDriverFactory(resolveHdcPath(config.phone.hdcPath)) : device.connection ? this.cloud.driver(device) : this.driverFactory(resolveAdbPath(config.phone.adbPath));
 	}
-	async discover(config, signal) {
+	async discover(config, signal, platform) {
+		if (platform === "harmonyos") return this.harmonyDriverFactory(resolveHdcPath(config.phone.hdcPath)).discover(signal);
 		const driver = this.driverFactory(resolveAdbPath(config.phone.adbPath));
-		let devices = await driver.discover(signal);
+		let devices = [];
+		let adbError;
+		try {
+			devices = await driver.discover(signal);
+		} catch (error) {
+			signal?.throwIfAborted();
+			adbError = error;
+		}
 		const missing = this.repository.list().filter((device) => device.enabled && device.wireless && !devices.some((item) => item.serial === device.serial && item.state === "device") && (this.reconnectAfter.get(device.id) ?? 0) < Date.now()).slice(0, 2);
 		const reconnectSignal = AbortSignal.any([AbortSignal.timeout(5e3), ...signal ? [signal] : []]);
 		await Promise.all(missing.map(async (device) => {
@@ -104,6 +117,15 @@ var PhoneRuntime = class {
 			});
 		}));
 		if (missing.length) devices = await driver.discover(signal);
+		let hdcAvailable = false;
+		if (platform !== "android" && (config.phone.hdcPath || this.repository.list().some((device) => device.platform === "harmonyos"))) try {
+			devices = [...devices, ...await this.harmonyDriverFactory(resolveHdcPath(config.phone.hdcPath)).discover(signal)];
+			hdcAvailable = true;
+		} catch (error) {
+			signal?.throwIfAborted();
+			if (!devices.length && this.repository.list().some((device) => device.platform === "harmonyos")) throw error;
+		}
+		if (adbError && !devices.length && !hdcAvailable) throw adbError;
 		return devices;
 	}
 	reconnectWireless(id, config, signal, endpoint) {
@@ -136,9 +158,9 @@ var PhoneRuntime = class {
 			driver = await this.driverFor(device, config);
 			serial = device.serial;
 		} else {
-			const found = (await this.discover(config, signal)).find((device) => device.serial === input.serial && device.state === "device");
+			const found = (await this.discover(config, signal, input.platform)).find((device) => (device.platform ?? "android") === (input.platform ?? "android") && device.serial === input.serial && device.state === "device");
 			if (!found) throw new PhoneError("PHONE_DEVICE_OFFLINE", "Choose a connected and authorized phone");
-			driver = this.driverFactory(resolveAdbPath(config.phone.adbPath));
+			driver = found.platform === "harmonyos" ? this.harmonyDriverFactory(resolveHdcPath(config.phone.hdcPath)) : this.driverFactory(resolveAdbPath(config.phone.adbPath));
 			serial = found.serial;
 		}
 		if (!driver.applications) throw new PhoneError("PHONE_APPS_UNAVAILABLE", "The phone provider cannot list applications");
@@ -571,7 +593,7 @@ var PhoneRuntime = class {
 					device = await this.reconnectWireless(device.id, context.config, signal);
 					found = (await driver.discover(signal)).find((item) => item.serial === device.serial);
 				}
-				if (found?.state !== "device") throw new PhoneError("PHONE_DEVICE_OFFLINE", "Phone is offline or has not authorized ADB");
+				if (found?.state !== "device") throw new PhoneError("PHONE_DEVICE_OFFLINE", "Phone is offline or has not authorized USB debugging");
 				if (modelRef === "phone-mock/settings-about" && !found.emulator) throw new PhoneError("PHONE_MOCK_EMULATOR_ONLY", "The deterministic mock model is only enabled for local Android emulators");
 				const lease = this.repository.acquire(device.serial);
 				try {
@@ -776,55 +798,7 @@ var PhoneRuntime = class {
 			session.controller.signal,
 			session.budget.controller.signal
 		]);
-		const operation = (async () => {
-			await this.perform(session, input, context, combined);
-			if ((input.op === "run" || input.op === "step") && ["ready", "completed"].includes(session.report.status) && session.report.task?.requestedOutputs.length) {
-				const observation = await this.observe(session, combined);
-				try {
-					this.guard(session, context);
-					this.allowedScreen(session, observation);
-					this.recordObservation(session, observation);
-					const conditions = input.completion || input.expectText ? {
-						...input.completion,
-						requireChange: false,
-						textAll: [...input.completion?.textAll ?? [], ...input.expectText ? [input.expectText] : []]
-					} : void 0;
-					if (session.report.verified && conditions && !phoneCompletionSatisfied({
-						...conditions,
-						textAll: conditions.textAll.length ? conditions.textAll : void 0
-					}, observation, true)) {
-						session.report.verified = false;
-						session.report.verificationLevel = "none";
-						session.report.status = "ready";
-						session.report.stopReason = "completion_changed";
-					}
-					const epoch = session.report.controlEpoch;
-					const visualOutputs = session.report.task.requestedOutputs.filter((request) => !request.selector);
-					if (visualOutputs.length > 1 && context.config.phone.batchReadings && session.adapter.readFields) await this.readOutputFields(session, observation, visualOutputs, context, combined);
-					for (const request of session.report.task.requestedOutputs) {
-						if (visualOutputs.length > 1 && context.config.phone.batchReadings && session.adapter.readFields) break;
-						if (request.selector) continue;
-						await this.readScreen(session, observation, request.description, context, combined);
-						if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") break;
-						const reading = session.report.readings?.at(-1);
-						if (reading?.source === "screenshot") {
-							const output = session.report.feedback.outputs.find((output) => output.key === request.key);
-							Object.assign(output, {
-								value: reading.text.join("\n") || void 0,
-								source: reading.source,
-								verification: reading.verification,
-								frameId: reading.frameId,
-								capturedAt: reading.capturedAt,
-								reason: reading.reason
-							});
-						}
-					}
-					this.publish(session);
-				} finally {
-					observation.frame.fill(0);
-				}
-			}
-		})();
+		const operation = this.perform(session, input, context, combined);
 		session.active = operation;
 		try {
 			await operation;
@@ -978,6 +952,16 @@ var PhoneRuntime = class {
 		let observation;
 		let heartbeat;
 		const startedAt = Date.now();
+		const finish = async () => {
+			if (observation && (input.op === "run" || input.op === "step") && ["ready", "completed"].includes(session.report.status) && session.report.task?.requestedOutputs.length) {
+				const status = session.report.status;
+				const epoch = session.report.controlEpoch;
+				session.report.status = "running";
+				await this.collectOutputs(session, input, observation, context, signal);
+				if (session.report.status === "running" && session.report.controlEpoch === epoch && session.report.controlOwner === "agent") session.report.status = status;
+			}
+			return this.publish(session);
+		};
 		session.report.errorCode = void 0;
 		session.report.verified = false;
 		session.report.verificationLevel = "none";
@@ -1003,7 +987,7 @@ var PhoneRuntime = class {
 				this.recordObservation(session, observation);
 				if (!session.report.facts?.length || input.goal) await this.readScreen(session, observation, input.goal ?? "Read the visible page text and values", context, signal);
 				if (session.report.controlOwner === "agent" && !signal.aborted) session.report.status = "ready";
-				return this.publish(session);
+				return await finish();
 			}
 			if (input.op !== "run" && input.op !== "step") throw new PhoneError("PHONE_INVALID_OPERATION", "Unsupported operation");
 			if (session.report.mode !== "control") throw new PhoneError("PHONE_OBSERVE_ONLY", "Reopen the phone in control mode to execute actions");
@@ -1076,7 +1060,7 @@ var PhoneRuntime = class {
 					if (numericIntent) {
 						let next = nextPhoneNumericEntry(numericIntent, observation, numericPending);
 						for (let poll = 0; numericPending && next.kind === "blocked" && poll < 2; poll++) {
-							await setTimeout(150, void 0, { signal });
+							await setTimeout$1(150, void 0, { signal });
 							observation.frame.fill(0);
 							observation = await this.observe(session, signal);
 							this.guard(session, context);
@@ -1095,7 +1079,7 @@ var PhoneRuntime = class {
 							session.report.message = "Numeric input was not confirmed. Inspect the displayed value before continuing; no key was retried.";
 							session.budget?.pause();
 							await session.stream?.revoke();
-							return this.publish(session);
+							return await finish();
 						}
 						if (numericPending) session.report.diagnostics.numericConfirmedSteps = (session.report.diagnostics.numericConfirmedSteps ?? 0) + 1;
 						numericPending = void 0;
@@ -1119,11 +1103,13 @@ var PhoneRuntime = class {
 							planning = advancePhonePlan(planning, confirmation);
 							session.report.diagnostics.completedSubgoals = planning.verifiedSteps;
 						}
+						observation.frame.fill(0);
+						observation = confirmation;
 						session.report.verified = true;
 						session.report.verificationLevel = "ui";
 						session.report.status = "completed";
 						session.report.stopReason = actions ? "condition_satisfied" : "condition_present";
-						return this.publish(session);
+						return await finish();
 					} finally {
 						if (observation !== confirmation) confirmation.frame.fill(0);
 					}
@@ -1131,12 +1117,12 @@ var PhoneRuntime = class {
 				if (session.report.actions - (session.report.humanActions ?? 0) - (session.taskStart?.actions ?? 0) >= (session.report.task?.budget?.maxActions ?? 80)) {
 					session.report.status = "ready";
 					session.report.stopReason = "task_action_limit";
-					return this.publish(session);
+					return await finish();
 				}
 				if (actions >= maxActions) {
 					session.report.status = "ready";
 					session.report.stopReason = "action_limit";
-					return this.publish(session);
+					return await finish();
 				}
 				let decision = numericStep ? {
 					kind: "action",
@@ -1251,13 +1237,21 @@ var PhoneRuntime = class {
 						session.report.message = "The required native UI conditions are still unmet. Inspect the current screen before resuming; no action was generated from the completion conditions.";
 						session.budget?.pause();
 						await session.stream?.revoke();
-						return this.publish(session);
+						return await finish();
 					}
 					session.report.message = decision.kind === "needsUser" ? decision.reason : decision.summary;
 					if (decision.kind === "complete") {
+						if (session.report.task?.requestedOutputs.length) {
+							observation.frame.fill(0);
+							observation = await this.observe(session, signal);
+							signal.throwIfAborted();
+							this.guard(session, context);
+							this.allowedScreen(session, observation);
+							if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") continue;
+						}
 						this.recordObservation(session, observation);
-						if (!context.config.phone.batchReadings || !session.adapter.readFields || !session.report.task?.requestedOutputs.some((request) => !request.selector)) await this.readScreen(session, observation, input.goal, context, signal);
-						if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") return this.publish(session);
+						if (!session.report.task?.requestedOutputs.length) await this.readScreen(session, observation, input.goal, context, signal);
+						if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") return await finish();
 					}
 					session.report.status = decision.kind === "needsUser" ? "needs_user" : "completed";
 					session.report.stopReason = decision.kind === "needsUser" ? "user_takeover" : "model_finished_unverified";
@@ -1267,9 +1261,9 @@ var PhoneRuntime = class {
 						session.budget?.pause();
 						session.report.assistanceId = context.onNeedsUser?.(this.checkpoint(session));
 						await session.stream?.revoke();
-						return this.publish(session);
+						return await finish();
 					}
-					return this.publish(session);
+					return await finish();
 				}
 				let preparedFocus;
 				if (enhancements.hierarchicalPlanning) {
@@ -1291,7 +1285,7 @@ var PhoneRuntime = class {
 							session.report.message = back.hint;
 							session.budget?.pause();
 							await session.stream?.revoke();
-							return this.publish(session);
+							return await finish();
 						}
 						recoveryHint = back.hint;
 						observation.frame.fill(0);
@@ -1316,14 +1310,17 @@ var PhoneRuntime = class {
 						}
 					}
 				}
-				const fresh = await this.observe(session, signal);
+				const fresh = await this.observe(session, signal, false, {
+					observation,
+					action: decision.action
+				});
 				try {
 					this.allowedScreen(session, fresh);
 					if (decision.action.type !== "tapElement" && !canDispatchPhoneAction(decision.action, observation, fresh)) {
 						session.report.diagnostics.stalePredictions++;
 						recoveryHint = "The proposed action was not dispatched because its target changed. Locate it again on the fresh screen.";
 						observation.frame.fill(0);
-						observation = fresh;
+						observation = fresh.frame.length ? fresh : await this.observe(session, signal);
 						continue;
 					}
 					let action = enhancements.semanticControls ? groundPhoneCoordinateAction(decision.action, observation) : decision.action;
@@ -1337,7 +1334,7 @@ var PhoneRuntime = class {
 							session.report.diagnostics.semanticRejections = (session.report.diagnostics.semanticRejections ?? 0) + 1;
 							recoveryHint = "The semantic target was not dispatched: select one unique fully visible non-password control from the fresh native UI.";
 							observation.frame.fill(0);
-							observation = fresh;
+							observation = fresh.frame.length ? fresh : await this.observe(session, signal);
 							continue;
 						}
 					}
@@ -1345,7 +1342,7 @@ var PhoneRuntime = class {
 						session.report.diagnostics.stalePredictions++;
 						recoveryHint = "The proposed action was not dispatched because its target changed. Locate it again on the fresh screen.";
 						observation.frame.fill(0);
-						observation = fresh;
+						observation = fresh.frame.length ? fresh : await this.observe(session, signal);
 						continue;
 					}
 					if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") continue;
@@ -1359,7 +1356,7 @@ var PhoneRuntime = class {
 						session.report.message = "Repeated actions made no progress. Inspect the phone and hand back before continuing.";
 						session.budget?.pause();
 						await session.stream?.revoke();
-						return this.publish(session);
+						return await finish();
 					}
 					attempts.set(attemptKey, repetitions + 1);
 					if (await this.dispatch(session, action, fresh, context, signal, epoch)) {
@@ -1406,7 +1403,7 @@ var PhoneRuntime = class {
 				session.budget?.pause();
 				session.report.assistanceId = context.onNeedsUser?.(this.checkpoint(session));
 				await session.stream?.revoke();
-				return this.publish(session);
+				return await finish();
 			}
 			if (failure instanceof PhoneError && failure.code === "PHONE_OPERATION_TIMEOUT" && !terminal.has(session.report.status) && session.report.status !== "paused" && session.report.status !== "outcome_unknown") {
 				session.report.status = "paused";
@@ -1427,7 +1424,7 @@ var PhoneRuntime = class {
 					errorCode: failure.code,
 					phase: "goal_timeout"
 				}, "Phone goal timed out; session remains paused for inspection");
-				return this.publish(session);
+				return await finish();
 			}
 			log.warn({
 				sessionId: session.report.sessionId,
@@ -1466,6 +1463,48 @@ var PhoneRuntime = class {
 			}, "Phone goal finished");
 		}
 	}
+	async collectOutputs(session, input, observation, context, signal) {
+		this.guard(session, context);
+		this.allowedScreen(session, observation);
+		this.recordObservation(session, observation);
+		const conditions = input.completion || input.expectText ? {
+			...input.completion,
+			requireChange: false,
+			textAll: [...input.completion?.textAll ?? [], ...input.expectText ? [input.expectText] : []]
+		} : void 0;
+		if (session.report.verified && conditions && !phoneCompletionSatisfied({
+			...conditions,
+			textAll: conditions.textAll.length ? conditions.textAll : void 0
+		}, observation, true)) {
+			session.report.verified = false;
+			session.report.verificationLevel = "none";
+			session.report.status = "ready";
+			session.report.stopReason = "completion_changed";
+		}
+		const epoch = session.report.controlEpoch;
+		signal.throwIfAborted();
+		const visualOutputs = session.report.task.requestedOutputs.filter((request) => !request.selector);
+		if (visualOutputs.length > 1 && context.config.phone.batchReadings && session.adapter.readFields) {
+			await this.readOutputFields(session, observation, visualOutputs, context, signal);
+			return;
+		}
+		for (const request of visualOutputs) {
+			await this.readScreen(session, observation, request.description, context, signal);
+			if (epoch !== session.report.controlEpoch || session.report.controlOwner !== "agent") break;
+			const reading = session.report.readings?.at(-1);
+			if (reading?.source === "screenshot") {
+				const output = session.report.feedback.outputs.find((output) => output.key === request.key);
+				Object.assign(output, {
+					value: reading.text.join("\n") || void 0,
+					source: reading.source,
+					verification: reading.verification,
+					frameId: reading.frameId,
+					capturedAt: reading.capturedAt,
+					reason: reading.reason
+				});
+			}
+		}
+	}
 	recordObservation(session, observation) {
 		if (session.report.task && session.report.feedback) session.report.feedback.outputs = session.report.task.requestedOutputs.map((output) => nativePhoneOutput(output, observation));
 		session.report.foregroundApp = observation.package;
@@ -1590,7 +1629,7 @@ var PhoneRuntime = class {
 			if (session.human && (session.report.humanActions ?? 0) >= 1200) throw new PhoneError("PHONE_ACTION_LIMIT", "Manual input limit reached; reopen the session");
 			if (action.type === "launch" && !phoneAllowsApp(session.device, action.package)) throw new PhoneError("PHONE_APP_DENIED", "Application is not authorized");
 			if (action.type === "home" && phoneAppAccess(session.device) !== "all") throw new PhoneError("PHONE_APP_DENIED", "Home leaves the app allowlist");
-			if (action.type === "typeText" && session.device.textInput === "manual") throw new PhoneError("PHONE_TEXT_INPUT_UNAVAILABLE", "Automatic text input is disabled for this phone. Ask the user to take over and type with the phone keyboard.", 409);
+			if (action.type === "typeText" && (session.device.platform === "harmonyos" || session.device.textInput === "manual")) throw new PhoneError("PHONE_TEXT_INPUT_UNAVAILABLE", "Automatic text input is disabled for this phone. Ask the user to take over and type with the phone keyboard.", 409);
 			if (action.type === "typeText" && !observation.elements.some((node) => node.focused && node.editable && !node.password)) throw new PhoneError("PHONE_INPUT_NOT_FOCUSED", "Focus a non-password editable field before typing", 409);
 			const step = {
 				action: action.type,
@@ -1674,8 +1713,10 @@ var PhoneRuntime = class {
 		session.report.controlEpoch++;
 		if (op === "pause") await session.stream?.revoke();
 		else {
+			clearTimeout(session.videoIdleTimer);
 			session.videoGeneration = void 0;
 			await session.stream?.close();
+			await session.streamClosing;
 			session.stream = void 0;
 		}
 		for (const wake of session.wake) wake();
@@ -1730,13 +1771,13 @@ var PhoneRuntime = class {
 		session.io = operation.catch(() => {});
 		return operation;
 	}
-	async observe(session, signal, forText = false) {
+	async observe(session, signal, forText = false, dispatchFrom) {
 		return this.io(session, async () => {
 			signal.throwIfAborted();
 			const observe = forText && session.driver.observeForText ? session.driver.observeForText : session.driver.observe;
-			const observation = await measurePhonePhase("observation", () => observe.call(session.driver, session.device.serial, signal));
+			const observation = await measurePhonePhase("observation", () => dispatchFrom && session.driver.observeForDispatch ? session.driver.observeForDispatch(session.device.serial, dispatchFrom.observation, dispatchFrom.action, signal) : observe.call(session.driver, session.device.serial, signal));
 			if (phoneAllowsApp(session.device, observation.package)) {
-				if (Date.now() - session.viewedAt < 3e4 && !terminal.has(session.report.status)) session.frames.set(observation);
+				if (observation.frame.length && Date.now() - session.viewedAt < 3e4 && !terminal.has(session.report.status)) session.frames.set(observation);
 			} else session.frames.clear();
 			return observation;
 		});
@@ -1790,6 +1831,7 @@ var PhoneRuntime = class {
 		this.maintenance = setInterval(() => {
 			for (const session of this.sessions.values()) {
 				if (terminal.has(session.report.status)) {
+					clearTimeout(session.videoIdleTimer);
 					session.frames.clear();
 					const stream = session.stream;
 					session.stream = void 0;
@@ -1837,7 +1879,8 @@ var PhoneRuntime = class {
 			ownControl: Boolean(session.human === client),
 			allowedApps: [...session.device.allowedApps],
 			appAccess: phoneAppAccess(session.device),
-			textInput: session.device.textInput ?? "auto",
+			textInput: session.device.platform === "harmonyos" ? "manual" : session.device.textInput ?? "auto",
+			media: session.device.platform === "harmonyos" ? resolveHdcVideoTools(context.config.phone) ? "hdc-h264" : "snapshot" : void 0,
 			frame: session.frames.get() ? {
 				seq: session.frames.get().seq,
 				capturedAt: session.frames.get().capturedAt
@@ -1846,6 +1889,8 @@ var PhoneRuntime = class {
 	}
 	async liveVideo(owner, id, context) {
 		const session = this.liveSession(owner, id, context);
+		clearTimeout(session.videoIdleTimer);
+		await session.streamClosing;
 		if (terminal.has(session.report.status)) throw new PhoneError("PHONE_OPEN_REQUIRED", "Phone session ended", 409);
 		if (session.streamOpening) return session.streamOpening;
 		const opening = this.openLiveVideo(session);
@@ -1867,7 +1912,10 @@ var PhoneRuntime = class {
 				session.videoGeneration = void 0;
 				this.stop(session.report.owner, "pause");
 			};
-			const provider = session.device.connection ? await this.cloud.interactive(session.device, failed) : this.interactiveFactory(session.adbPath, session.device.serial, failed);
+			const provider = session.device.platform === "harmonyos" ? this.harmonyVideoFactory(session.device.serial, session.context.config.phone, failed, (app) => {
+				this.guard(session, session.context, false);
+				if (!phoneAllowsApp(session.device, app)) throw new PhoneError("PHONE_APP_DENIED", "Current page is outside the authorized applications", 403);
+			}) : session.device.connection ? await this.cloud.interactive(session.device, failed) : this.interactiveFactory(session.adbPath, session.device.serial, failed);
 			if (terminal.has(session.report.status)) {
 				await provider.close();
 				throw new PhoneError("PHONE_VIDEO_ENDED", "Phone session ended", 409);
@@ -1911,8 +1959,26 @@ var PhoneRuntime = class {
 		if (!session || session.report.sessionId !== id) return;
 		if (session.stream?.viewerCount === 0) {
 			const stream = session.stream;
-			session.stream = void 0;
-			await stream.close();
+			if (session.device.platform === "harmonyos") {
+				clearTimeout(session.videoIdleTimer);
+				session.videoIdleTimer = setTimeout(() => {
+					if (session.stream !== stream || stream.viewerCount) return;
+					session.stream = void 0;
+					const closing = stream.close();
+					session.streamClosing = closing;
+					closing.catch((err) => log.warn({
+						err,
+						conversationId: owner,
+						phase: "video_cleanup"
+					}, "HarmonyOS video cleanup failed")).finally(() => {
+						if (session.streamClosing === closing) session.streamClosing = void 0;
+					});
+				}, 1500);
+				session.videoIdleTimer.unref();
+			} else {
+				session.stream = void 0;
+				await stream.close();
+			}
 		}
 	}
 	async clearExpiredHuman(session) {

package/dist/src/phone/setup/service.d.ts

@@ -22,7 +22,7 @@ export interface PhoneSetupMutation {
 }
 export interface PhoneSetupDeps {
     runtime: PhoneRuntime;
-    inspect(context: Context, source?: 'local' | 'aliyun'): Promise<PhoneSetupView['readiness']>;
+    inspect(context: Context, source?: 'local' | 'aliyun' | 'harmonyos'): Promise<PhoneSetupView['readiness']>;
     revision(context: Context, deviceId: string): string;
     connected(deviceId: string, signal?: AbortSignal): Promise<boolean>;
     leaseActive(deviceId: string): boolean;
@@ -33,8 +33,8 @@ export interface PhoneSetupDeps {
         configured: boolean;
     }>;
     configure?: (enabled: boolean, mutation?: PhoneSetupMutation) => Promise<void>;
-    install?: (kind: 'adb' | 'scrcpy', repair: boolean, mutation?: PhoneSetupMutation) => Promise<void>;
-    registerLocal?: (serial: string, name?: string, mutation?: PhoneSetupMutation) => Promise<void>;
+    install?: (kind: 'adb' | 'scrcpy' | 'hdc' | 'hoscrcpy' | 'java', repair: boolean, mutation?: PhoneSetupMutation) => Promise<void>;
+    registerLocal?: (serial: string, name?: string, mutation?: PhoneSetupMutation, platform?: 'android' | 'harmonyos') => Promise<void>;
 }
 /** Bounded product actions automate setup; credentials remain in secure product forms. */
 export declare class PhoneSetupService {
@@ -63,15 +63,17 @@ export declare class PhoneSetupService {
                 configured: boolean;
             };
             dependencies: Array<{
-                kind: "adb" | "scrcpy";
+                kind: "adb" | "scrcpy" | "hdc" | "hoscrcpy" | "java";
                 state: string;
                 version?: string;
                 canInstall: boolean;
+                distribution?: string;
+                downloadUrl?: string;
             }>;
             devices: Array<{
                 id: string;
                 name: string;
-                source: "local" | "aliyun";
+                source: "local" | "aliyun" | "harmonyos";
                 enabled: boolean;
                 settingsAllowed: boolean;
             }>;
@@ -98,7 +100,7 @@ export declare class PhoneSetupService {
             configured: boolean;
         }[];
     }>;
-    source(context: Context, id: string, source: 'local' | 'aliyun', revision: number): Promise<PhoneSetupView>;
+    source(context: Context, id: string, source: 'local' | 'aliyun' | 'harmonyos', revision: number): Promise<PhoneSetupView>;
     begin(context: Context, id: string, deviceId: string, revision: number, signal?: AbortSignal): Promise<PhoneSetupView>;
     beforeInput(context: Context, input: PhoneInput): {
         owner: string;