@xopcai/xopc 0.0.424 → 0.0.425

package/dist/src/phone/cloud-adb-connection.js

@@ -1,12 +1,14 @@
 import { init_paths, resolveCredentialsDir } from "../config/paths.js";
 import { PhoneError, init_contracts } from "./contracts.js";
+import { measurePhonePhase } from "./execution-trace.js";
 import { readScrcpyServer } from "./scrcpy-assets.js";
+import { PHONE_UI_DEX } from "./ui-dump-binary.js";
+import { PhoneUiSession } from "./ui-session.js";
 import { parsePhoneApplications, phoneApplicationsCommand } from "./applications.js";
 import { observeAdbConsumption } from "./cloud-adb-writable.js";
 import { protectPhoneCredential } from "./cloud-key-permissions.js";
-import { PHONE_UI_DEX } from "./ui-dump-binary.js";
 import { join } from "node:path";
-import { createPrivateKey, randomUUID } from "node:crypto";
+import { createHash, createPrivateKey, randomUUID } from "node:crypto";
 import { lstat, mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { connect } from "node:net";
 import { Adb, AdbDaemonTransport, AdbPacket, AdbPacketSerializeStream, AdbSignatureAuthenticator } from "@yume-chan/adb";
@@ -231,53 +233,122 @@ var CloudAdbConnection = class {
 			this.idle.unref();
 		}
 	}
-	async hierarchy(signal) {
+	async prepareUi(signal) {
 		const combined = AbortSignal.any([signal, AbortSignal.timeout(2e4)]);
 		const abort = () => this.close();
 		combined.addEventListener("abort", abort, { once: true });
-		const remote = `/data/local/tmp/xopc-ui-${randomUUID()}.dex`;
+		const binary = Buffer.from(PHONE_UI_DEX, "base64");
+		const hash = createHash("sha256").update(binary).digest("hex");
+		const remote = `/data/local/tmp/xopc-ui-${hash}.dex`;
+		let staging;
 		try {
 			combined.throwIfAborted();
-			const adb = await this.client();
-			combined.throwIfAborted();
-			const sync = await adb.sync();
-			try {
-				const binary = Buffer.from(PHONE_UI_DEX, "base64");
-				await sync.write({
-					filename: remote,
-					permission: 384,
-					file: new ReadableStream({ start(controller) {
-						controller.enqueue(binary);
-						controller.close();
-					} })
-				});
-			} finally {
-				await sync.dispose();
-			}
-			const result = (await this.command("", [
+			if ((await this.command("", [
 				"-s",
 				this.endpoint,
 				"shell",
-				"env",
-				`CLASSPATH=${remote}`,
-				"app_process",
-				"/",
-				"ai.xopc.phone.PhoneUiDump"
-			], combined)).toString();
-			if (!result.includes("<hierarchy") || !result.includes("</hierarchy>")) throw new PhoneError("PHONE_UI_UNAVAILABLE", "Unable to capture the active phone UI hierarchy", 502);
-			return result;
+				"sha256sum",
+				remote
+			], combined)).toString().trim().split(/\s+/)[0] !== hash) {
+				staging = `${remote}.${randomUUID()}.tmp`;
+				await measurePhonePhase("dex_upload", async () => {
+					const adb = await this.client();
+					combined.throwIfAborted();
+					const sync = await adb.sync();
+					try {
+						await sync.write({
+							filename: staging,
+							permission: 384,
+							file: new ReadableStream({ start(controller) {
+								controller.enqueue(binary);
+								controller.close();
+							} })
+						});
+					} finally {
+						await sync.dispose();
+					}
+					await this.command("", [
+						"-s",
+						this.endpoint,
+						"shell",
+						"chmod",
+						"400",
+						staging
+					], combined);
+					await this.command("", [
+						"-s",
+						this.endpoint,
+						"shell",
+						"mv",
+						"-f",
+						staging,
+						remote
+					], combined);
+					if ((await this.command("", [
+						"-s",
+						this.endpoint,
+						"shell",
+						"sha256sum",
+						remote
+					], combined)).toString().trim().split(/\s+/)[0] !== hash) throw new PhoneError("PHONE_UI_UNAVAILABLE", "Phone UI helper verification failed", 502);
+				});
+			}
+			return remote;
 		} finally {
 			combined.removeEventListener("abort", abort);
-			await this.command("", [
+			if (staging) await this.command("", [
 				"-s",
 				this.endpoint,
 				"shell",
 				"rm",
 				"-f",
-				remote
+				staging
 			], void 0, 3e3).catch(() => {});
 		}
 	}
+	async hierarchy(signal) {
+		const remote = await this.prepareUi(signal);
+		const result = (await this.command("", [
+			"-s",
+			this.endpoint,
+			"shell",
+			"env",
+			`CLASSPATH=${remote}`,
+			"app_process",
+			"/",
+			"ai.xopc.phone.PhoneUiDump"
+		], signal)).toString();
+		if (!result.includes("</hierarchy>")) throw new PhoneError("PHONE_UI_UNAVAILABLE", "Incomplete phone hierarchy", 502);
+		return result;
+	}
+	async observer(signal) {
+		const combined = AbortSignal.any([signal, AbortSignal.timeout(2e4)]);
+		const abort = () => this.close();
+		combined.addEventListener("abort", abort, { once: true });
+		try {
+			const remote = await this.prepareUi(combined);
+			const adb = await this.client();
+			combined.throwIfAborted();
+			const channel = await adb.createSocket(`exec:env CLASSPATH=${remote} app_process / ai.xopc.phone.PhoneUiDump --resident`);
+			if (combined.aborted) {
+				await channel.close();
+				combined.throwIfAborted();
+			}
+			clearTimeout(this.idle);
+			const writer = channel.writable.getWriter();
+			writer.closed.catch(() => {});
+			return new PhoneUiSession({
+				readable: channel.readable,
+				write: (data) => writer.write(data),
+				close: async () => {
+					await channel.close();
+					writer.releaseLock();
+				}
+			});
+		} finally {
+			combined.removeEventListener("abort", abort);
+		}
+	}
 	async applications(signal) {
 		const combined = AbortSignal.any([signal, AbortSignal.timeout(3e4)]);
 		const abort = () => this.close();

package/dist/src/phone/cloud-adb.js

@@ -10,15 +10,27 @@ var CloudAdbConnection = class CloudAdbConnection extends CloudAdbConnection$1 {
 			} finally {
 				listing.dispose();
 			}
-		}, async (text, signal) => {
+		}, void 0, this.hierarchy.bind(this), async (_serial, failed) => {
 			const { CloudScrcpyProvider } = await import("./cloud-scrcpy.js");
-			const provider = new CloudScrcpyProvider(this.endpoint, Buffer.from(this.key), () => {}, void 0, true);
+			return new CloudScrcpyProvider(this.endpoint, Buffer.from(this.key), failed, void 0, true);
+		}, async (signal) => {
+			const observation = new CloudAdbConnection(this.endpoint, Buffer.from(this.key));
 			try {
-				await provider.pasteText(text, signal);
-			} finally {
-				await provider.close();
+				const session = await observation.observer(signal);
+				const close = session.close.bind(session);
+				session.close = async () => {
+					try {
+						await close();
+					} finally {
+						observation.dispose();
+					}
+				};
+				return session;
+			} catch (error) {
+				observation.dispose();
+				throw error;
 			}
-		}, this.hierarchy.bind(this));
+		});
 	}
 };
 //#endregion

package/dist/src/phone/completion-recovery.d.ts

@@ -0,0 +1,6 @@
+import type { PhoneCompletion, PhoneObservation } from './contracts.js';
+export declare const PHONE_COMPLETION_CORRECTIONS = 2;
+/** Warn only when visible matching content could be mistaken for an absent exact input field. */
+export declare function phoneCompletionGuidance(completion: PhoneCompletion, screen: PhoneObservation, changed: boolean): string | undefined;
+/** Diagnostic feedback only; completion conditions never authorize generated device actions. */
+export declare function phoneCompletionRecoveryHint(completion: PhoneCompletion, screen: PhoneObservation, changed: boolean, attempt: number): string;

package/dist/src/phone/completion-recovery.js

@@ -0,0 +1,30 @@
+import { phoneCompletionSatisfied } from "./reliability.js";
+//#region src/phone/completion-recovery.ts
+const PHONE_COMPLETION_CORRECTIONS = 2;
+function completionHint(completion, screen, changed, prefix, concise = false) {
+	let evidence;
+	const preview = (value, limit = 60) => JSON.stringify(value.slice(0, limit));
+	if (completion.appPackage && completion.appPackage !== screen.package) evidence = `Required app ${preview(completion.appPackage, 90)}; observed ${preview(screen.package, 90)}.`;
+	else if (completion.field && !phoneCompletionSatisfied({ field: completion.field }, screen, changed)) {
+		const fields = screen.elements.filter((node) => node.editable && !node.password && node.resourceId === completion.field.resourceId);
+		evidence = `Editable field ${preview(completion.field.resourceId, 100)} must equal ${preview(completion.field.value)}; ` + (fields.length === 1 ? `observed ${preview(fields[0].text)}.` : `no unique non-password field (${fields.length} matches).`);
+	} else {
+		const texts = screen.elements.filter((node) => !node.password).flatMap((node) => [node.text, node.description]);
+		const missing = completion.textAll?.find((value) => !texts.some((text) => text.includes(value)));
+		evidence = missing ? `Required native text missing: ${preview(missing, 100)}.` : completion.textAny && !completion.textAny.some((value) => texts.some((text) => text.includes(value))) ? `No required native text alternative is visible: ${preview(completion.textAny[0], 100)}.` : "The required UI change has not been observed.";
+	}
+	const instruction = (completion.field ? "Matching visible content does not prove an exact field condition. " : "") + "Return one safe action authorized by the original goal, or request takeover if blocked. Do not finish without evidence." + (concise ? " Return only the action, without a screen summary." : "");
+	return `${prefix}${evidence.slice(0, 500 - prefix.length - instruction.length - 1)} ${instruction}`;
+}
+/** Warn only when visible matching content could be mistaken for an absent exact input field. */
+function phoneCompletionGuidance(completion, screen, changed) {
+	const field = completion.field;
+	if (!field?.value.trim() || screen.elements.some((node) => node.resourceId === field.resourceId)) return void 0;
+	return screen.elements.some((node) => !node.password && !node.editable && node.bounds[0] >= 0 && node.bounds[1] >= 0 && node.bounds[2] <= screen.width && node.bounds[3] <= screen.height && node.bounds[2] > node.bounds[0] && node.bounds[3] > node.bounds[1] && [node.text, node.description].some((text) => text.includes(field.value))) ? completionHint(completion, screen, changed, "Native completion: NOT MET. ", true) : void 0;
+}
+/** Diagnostic feedback only; completion conditions never authorize generated device actions. */
+function phoneCompletionRecoveryHint(completion, screen, changed, attempt) {
+	return completionHint(completion, screen, changed, `Completion rejected (${attempt}/2). `);
+}
+//#endregion
+export { PHONE_COMPLETION_CORRECTIONS, phoneCompletionGuidance, phoneCompletionRecoveryHint };

package/dist/src/phone/contracts.d.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import type { PhonePerformance } from './execution-trace.js';
 export declare const PhoneProfileSchema: z.ZodEnum<{
     "autoglm-phone-v1": "autoglm-phone-v1";
     "structured-phone-v1": "structured-phone-v1";
@@ -23,6 +24,24 @@ export declare const PhoneConfigSchema: z.ZodPrefault<z.ZodObject<{
         direct: "direct";
     }>>;
     adbPath: z.ZodOptional<z.ZodString>;
+    autoGlmPrompt: z.ZodDefault<z.ZodEnum<{
+        current: "current";
+        upstream: "upstream";
+    }>>;
+    autoGlmProtocol: z.ZodDefault<z.ZodEnum<{
+        current: "current";
+        upstream: "upstream";
+    }>>;
+    autoGlmSampling: z.ZodDefault<z.ZodEnum<{
+        current: "current";
+        upstream: "upstream";
+    }>>;
+    modelResponseMode: z.ZodDefault<z.ZodEnum<{
+        json: "json";
+        stream: "stream";
+    }>>;
+    batchReadings: z.ZodDefault<z.ZodBoolean>;
+    modelRepetitionGuard: z.ZodDefault<z.ZodBoolean>;
     semanticControls: z.ZodDefault<z.ZodBoolean>;
     hierarchicalPlanning: z.ZodDefault<z.ZodBoolean>;
     maxActionsPerSession: z.ZodDefault<z.ZodNumber>;
@@ -409,6 +428,7 @@ export type PhoneReport = {
     feedback?: PhoneTaskFeedback;
     sourceConversationId?: string;
     readings?: PhoneReading[];
+    performance?: PhonePerformance;
     steps: Array<{
         action: PhoneAction['type'];
         at: number;

package/dist/src/phone/contracts.js

@@ -12,6 +12,12 @@ var init_contracts = __esmMin((() => {
 		enabled: z.boolean().default(false),
 		executionMode: PhoneExecutionModeSchema.default("collaborative"),
 		adbPath: z.string().min(1).optional(),
+		autoGlmPrompt: z.enum(["current", "upstream"]).default("current"),
+		autoGlmProtocol: z.enum(["current", "upstream"]).default("current"),
+		autoGlmSampling: z.enum(["current", "upstream"]).default("current"),
+		modelResponseMode: z.enum(["json", "stream"]).default("json"),
+		batchReadings: z.boolean().default(false),
+		modelRepetitionGuard: z.boolean().default(false),
 		semanticControls: z.boolean().default(false),
 		hierarchicalPlanning: z.boolean().default(false),
 		maxActionsPerSession: z.number().int().min(1).max(80).default(40),

package/dist/src/phone/execution-trace.d.ts

@@ -0,0 +1,51 @@
+export type PhonePhase = 'observation' | 'hierarchy' | 'capture' | 'foreground' | 'screenshot' | 'display' | 'field_transaction' | 'dispatch' | 'input' | 'input_confirmation' | 'settle' | 'model_request' | 'model_parse' | 'screen_reading' | 'dex_upload';
+export type PhonePhaseMetric = {
+    count: number;
+    errors: number;
+    totalMs: number;
+    maxMs: number;
+};
+export type PhoneModelMetric = {
+    durationMs?: number;
+    preparationMs?: number;
+    firstByteMs?: number;
+    firstTokenMs?: number;
+    httpStatus?: number;
+    promptTokens?: number;
+    completionTokens?: number;
+    totalTokens?: number;
+    imageBytes?: number;
+    requestBytes?: number;
+    responseBytes?: number;
+    finishReason?: string;
+    errorCode?: string;
+    protocol?: string;
+    prompt?: string;
+    sampling?: string;
+    transport?: string;
+};
+export type PhonePerformance = {
+    operationId: string;
+    durationMs: number;
+    phases: Partial<Record<PhonePhase, PhonePhaseMetric>>;
+    modelCalls: PhoneModelMetric[];
+    droppedModelCalls: number;
+};
+/** Numeric, bounded diagnostics only; no screen text, prompts, model output or credentials. */
+export declare class PhoneExecutionTrace {
+    readonly id: `${string}-${string}-${string}-${string}-${string}`;
+    private started;
+    private phases;
+    private calls;
+    private dropped;
+    private endedAt?;
+    get active(): boolean;
+    finish(): void;
+    run<T>(work: () => T): T;
+    record(phase: PhonePhase, durationMs: number, failed: boolean): void;
+    model(metric: PhoneModelMetric): void;
+    snapshot(): PhonePerformance;
+}
+export declare function currentPhoneTrace(): PhoneExecutionTrace | undefined;
+/** Phases can overlap or nest; their sums must not be treated as wall-clock percentages. */
+export declare function measurePhonePhase<T>(phase: PhonePhase, work: () => Promise<T>): Promise<T>;

package/dist/src/phone/execution-trace.js

@@ -0,0 +1,78 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { randomUUID } from "node:crypto";
+import { performance } from "node:perf_hooks";
+//#region src/phone/execution-trace.ts
+const context = new AsyncLocalStorage();
+const rounded = (value) => Math.round(value * 100) / 100;
+/** Numeric, bounded diagnostics only; no screen text, prompts, model output or credentials. */
+var PhoneExecutionTrace = class {
+	id = randomUUID();
+	started = performance.now();
+	phases = {};
+	calls = [];
+	dropped = 0;
+	endedAt;
+	get active() {
+		return this.endedAt === void 0;
+	}
+	finish() {
+		this.endedAt ??= performance.now();
+	}
+	run(work) {
+		return context.run(this, work);
+	}
+	record(phase, durationMs, failed) {
+		if (!this.active) return;
+		const metric = this.phases[phase] ??= {
+			count: 0,
+			errors: 0,
+			totalMs: 0,
+			maxMs: 0
+		};
+		metric.count++;
+		metric.errors += Number(failed);
+		metric.totalMs += durationMs;
+		metric.maxMs = Math.max(metric.maxMs, durationMs);
+	}
+	model(metric) {
+		if (!this.active) return;
+		if (this.calls.length === 120) {
+			this.calls.shift();
+			this.dropped++;
+		}
+		this.calls.push({ ...metric });
+	}
+	snapshot() {
+		return {
+			operationId: this.id,
+			durationMs: rounded((this.endedAt ?? performance.now()) - this.started),
+			phases: Object.fromEntries(Object.entries(this.phases).map(([phase, metric]) => [phase, {
+				...metric,
+				totalMs: rounded(metric.totalMs),
+				maxMs: rounded(metric.maxMs)
+			}])),
+			modelCalls: structuredClone(this.calls),
+			droppedModelCalls: this.dropped
+		};
+	}
+};
+function currentPhoneTrace() {
+	const trace = context.getStore();
+	return trace?.active ? trace : void 0;
+}
+/** Phases can overlap or nest; their sums must not be treated as wall-clock percentages. */
+async function measurePhonePhase(phase, work) {
+	const trace = currentPhoneTrace();
+	if (!trace) return work();
+	const start = performance.now();
+	let failed = true;
+	try {
+		const result = await work();
+		failed = false;
+		return result;
+	} finally {
+		trace.record(phase, performance.now() - start, failed);
+	}
+}
+//#endregion
+export { PhoneExecutionTrace, currentPhoneTrace, measurePhonePhase };

package/dist/src/phone/model-adapter.d.ts

@@ -1,7 +1,7 @@
 import { type PhoneAppAccess } from './app-access.js';
 import { type PhonePlan, type PhonePlanState } from './planning.js';
 import type { Config } from '../config/schema.js';
-import { type PhoneAction, type PhoneApplication, type PhoneDecision, type PhoneObservation, type PhoneCompletion, type PhoneTaskContract } from './contracts.js';
+import { type PhoneAction, type PhoneApplication, type PhoneDecision, type PhoneObservation, type PhoneCompletion, type PhoneTaskContract, type PhoneRequestedOutput } from './contracts.js';
 export declare function phoneModelProfile(model: unknown): "autoglm-phone-v1" | "structured-phone-v1";
 export declare function validatePhoneModelChange(next: {
     primary: string;
@@ -28,6 +28,7 @@ export interface PhoneTrajectoryEntry {
 export interface PhonePrediction {
     task?: PhoneTaskContract;
     readOnly?: boolean;
+    readingFields?: PhoneRequestedOutput[];
     goal: string;
     observation: PhoneObservation;
     completion?: PhoneCompletion;
@@ -51,6 +52,9 @@ export type PhoneModelDecision = PhoneDecision & {
 };
 export interface PhoneModelAdapter {
     predict(input: PhonePrediction, signal: AbortSignal): Promise<PhoneModelDecision>;
+    recordAction?(action: PhoneAction, outcome: 'completed' | 'human'): void;
+    reset?(): void;
+    readFields?(input: PhonePrediction, fields: PhoneRequestedOutput[], signal: AbortSignal): Promise<Record<string, string | null>>;
     readScreen?(input: PhonePrediction, signal: AbortSignal): Promise<string>;
 }
 export declare function parsePhonePlanningResponse(raw: string, input: PhonePrediction): {
@@ -166,6 +170,10 @@ export declare function buildPhoneModelContext(input: PhonePrediction): {
     recoveryHint: string;
     protocolFeedback: string;
     task?: PhoneTaskContract;
+    readingFields?: {
+        key: string;
+        description: string;
+    }[];
 };
 export declare function isAboutScreen(observation: PhoneObservation): boolean;
 export declare class MockPhoneModel implements PhoneModelAdapter {