@xopcai/xopc 0.0.435 → 0.0.437

package/dist/src/computer/broker.js

@@ -2,7 +2,8 @@ import { createLogger } from "../utils/logger/index.js";
 import { init_logger } from "../utils/logger.js";
 import { ComputerConfigSchema, init_config } from "./config.js";
 import { ActionEnvelopeSchema, ComputerCommandSchema, computerActionInputCount, computerActionInputs, init_src } from "../../packages/computer-control-contract/src/index.js";
-import { ComputerTargetError } from "./errors.js";
+import { ComputerOperationError, ComputerTargetError, computerDiagnostic } from "./errors.js";
+import { ComputerStabilityTracker } from "./stability-diagnostics.js";
 import { createHash, randomUUID } from "node:crypto";
 import { setTimeout } from "node:timers/promises";
 //#region src/computer/broker.ts
@@ -221,7 +222,12 @@ var ComputerBroker = class {
 			s.controller.signal.removeEventListener("abort", abort);
 		}
 	}
-	async observe(s) {
+	async observe(s, afterAction) {
+		s.activity = {
+			phase: "observing",
+			...afterAction ? { action: afterAction } : {}
+		};
+		this.host.onStateChange?.();
 		let frame;
 		try {
 			frame = await this.driver.observe(s.target, s.controller.signal);
@@ -238,11 +244,14 @@ var ComputerBroker = class {
 			} catch {}
 			if (editable) {
 				let stable = false;
+				const stability = new ComputerStabilityTracker(afterAction ? "post_action" : "initial_capture", afterAction);
 				const fingerprint = () => JSON.stringify([frame.stateDigest, frame.target]);
 				const seen = new Set([fingerprint()]);
 				for (let sample = 0; sample < 5; sample++) {
 					this.active(s);
+					await setTimeout(100, void 0, { signal: s.controller.signal });
 					const next = await this.driver.observe(s.target, s.controller.signal);
+					stability.compare(frame, next);
 					const previous = frame;
 					frame = next;
 					previous.image.fill(0);
@@ -252,7 +261,7 @@ var ComputerBroker = class {
 					if (stable) break;
 					seen.add(current);
 				}
-				if (!stable) throw new Error("COMPUTER_UI_UNSTABLE");
+				if (!stable) throw new ComputerOperationError(stability.diagnostic("COMPUTER_UI_UNSTABLE", 6));
 			}
 			validateFrame();
 			s.target = frame.target;
@@ -270,6 +279,10 @@ var ComputerBroker = class {
 				stateDigest: frame.stateDigest
 			};
 			s.touchedAt = this.now();
+			s.activity = {
+				phase: "ready",
+				...afterAction ? { action: afterAction } : {}
+			};
 			return {
 				...this.result(s),
 				observation: s.observation,
@@ -280,7 +293,7 @@ var ComputerBroker = class {
 			};
 		} catch (error) {
 			frame?.image.fill(0);
-			s.errorCode ??= error instanceof Error && /^COMPUTER_[A-Z_0-9]+$/.test(error.message) ? error.message : "COMPUTER_OBSERVE_FAILED";
+			s.errorCode ??= computerDiagnostic(error)?.errorCode ?? (error instanceof Error && /^COMPUTER_[A-Z_0-9]+$/.test(error.message) ? error.message : "COMPUTER_OBSERVE_FAILED");
 			await this.stop().catch(() => {
 				log.warn({
 					sessionId: s.id,
@@ -345,6 +358,7 @@ var ComputerBroker = class {
 		const precheckStarted = this.durationNow();
 		let matched = false;
 		let precheckSamples = 0;
+		const stability = new ComputerStabilityTracker("pre_dispatch", e.action.kind);
 		for (let sample = 0; sample < 4; sample++) {
 			precheckSamples++;
 			if (sample) await setTimeout(100, void 0, { signal: s.controller.signal });
@@ -352,12 +366,14 @@ var ComputerBroker = class {
 			try {
 				current = await this.driver.observe(s.target, s.controller.signal);
 			} catch (error) {
-				s.errorCode ??= error instanceof Error && /^COMPUTER_[A-Z_0-9]+$/.test(error.message) ? error.message : "COMPUTER_OBSERVE_FAILED";
+				s.errorCode ??= computerDiagnostic(error)?.errorCode ?? (error instanceof Error && /^COMPUTER_[A-Z_0-9]+$/.test(error.message) ? error.message : "COMPUTER_OBSERVE_FAILED");
 				await this.stop().catch(() => {});
 				throw error;
 			}
 			try {
 				this.active(s);
+				stability.compare(observed, current);
+				if (e.deadlineAt <= this.now()) stability.deadline();
 				if (!(JSON.stringify(current.target) === JSON.stringify(observed.target)) || e.deadlineAt <= this.now()) break;
 				if (current.stateDigest !== observed.stateDigest) continue;
 				this.driver.validateAction?.(e.action);
@@ -373,7 +389,8 @@ var ComputerBroker = class {
 			s.status = "ready";
 			return {
 				...this.result(s),
-				errorCode: "COMPUTER_OBSERVATION_CHANGED"
+				errorCode: "COMPUTER_OBSERVATION_CHANGED",
+				diagnostic: stability.diagnostic("COMPUTER_OBSERVATION_CHANGED", precheckSamples)
 			};
 		}
 		const nativeMs = {
@@ -403,6 +420,11 @@ var ComputerBroker = class {
 		s.actions += computerActionInputCount(e.action);
 		s.pending = void 0;
 		s.status = "running";
+		s.activity = {
+			phase: "acting",
+			action: e.action.kind
+		};
+		this.host.onStateChange?.();
 		try {
 			const inputStarted = this.durationNow();
 			try {
@@ -426,7 +448,7 @@ var ComputerBroker = class {
 			const observationStarted = this.durationNow();
 			let after;
 			try {
-				after = await this.observe(s);
+				after = await this.observe(s, e.action.kind);
 			} finally {
 				nativeMs.postObserveMs = Math.max(0, Math.round(this.durationNow() - observationStarted));
 			}
@@ -438,7 +460,8 @@ var ComputerBroker = class {
 			};
 		} catch (error) {
 			if (receipt.dispatch === "started") receipt.dispatch = "unknown";
-			receipt.errorCode = error instanceof Error && /^COMPUTER_[A-Z_0-9]+$/.test(error.message) ? error.message : receipt.dispatch === "completed" ? "COMPUTER_POST_ACTION_OBSERVATION_FAILED" : "COMPUTER_NATIVE_ACTION_FAILED";
+			const diagnostic = computerDiagnostic(error);
+			receipt.errorCode = diagnostic?.errorCode ?? (error instanceof Error && /^COMPUTER_[A-Z_0-9]+$/.test(error.message) ? error.message : receipt.dispatch === "completed" ? "COMPUTER_POST_ACTION_OBSERVATION_FAILED" : "COMPUTER_NATIVE_ACTION_FAILED");
 			s.errorCode = receipt.errorCode;
 			await this.stop().catch(() => {
 				log.warn({
@@ -450,7 +473,8 @@ var ComputerBroker = class {
 			return {
 				...this.result(s),
 				receipt,
-				nativeMs
+				nativeMs,
+				...diagnostic ? { diagnostic } : {}
 			};
 		}
 	}
@@ -494,6 +518,7 @@ var ComputerBroker = class {
 			brokerEpoch: this.epoch,
 			appName: this.apps.get(s.appRef)?.name ?? s.appId,
 			status: s.status,
+			activity: s.activity ? { ...s.activity } : void 0,
 			target: s.target ? { ...s.target } : void 0,
 			signal: s.controller.signal
 		};

package/dist/src/computer/diagnostics.d.ts

@@ -67,6 +67,176 @@ export declare const ComputerFailureSchema: z.ZodObject<{
         invalid_envelope: "invalid_envelope";
         output_limit: "output_limit";
     }>>;
+    validation: z.ZodOptional<z.ZodObject<{
+        action: z.ZodEnum<{
+            type: "type";
+            key: "key";
+            unknown: "unknown";
+            answer: "answer";
+            wait: "wait";
+            typeText: "typeText";
+            batch: "batch";
+            terminate: "terminate";
+            activate: "activate";
+            showMenu: "showMenu";
+            click: "click";
+            drag: "drag";
+            setValue: "setValue";
+            pressKeys: "pressKeys";
+            scroll: "scroll";
+            activateSequence: "activateSequence";
+            interact: "interact";
+            takeover: "takeover";
+            finished: "finished";
+            button_plan: "button_plan";
+            activate_sequence: "activate_sequence";
+            show_menu: "show_menu";
+            set_value: "set_value";
+            left_click: "left_click";
+            right_click: "right_click";
+            double_click: "double_click";
+            hscroll: "hscroll";
+        }>;
+        stage: z.ZodEnum<{
+            plan: "plan";
+            schema: "schema";
+            converted_action: "converted_action";
+            typing_target: "typing_target";
+            keyboard: "keyboard";
+        }>;
+        issues: z.ZodArray<z.ZodObject<{
+            path: z.ZodArray<z.ZodUnion<readonly [z.ZodEnum<{
+                reason: "reason";
+                keys: "keys";
+                value: "value";
+                name: "name";
+                time: "time";
+                from: "from";
+                action: "action";
+                when: "when";
+                label: "label";
+                text: "text";
+                kind: "kind";
+                status: "status";
+                durationMs: "durationMs";
+                role: "role";
+                to: "to";
+                actions: "actions";
+                count: "count";
+                x: "x";
+                y: "y";
+                point: "point";
+                arguments: "arguments";
+                ref: "ref";
+                button: "button";
+                deltaX: "deltaX";
+                deltaY: "deltaY";
+                refs: "refs";
+                pixels: "pixels";
+                steps: "steps";
+                coordinate: "coordinate";
+                unknown_field: "unknown_field";
+                proposal: "proposal";
+                claimedSuccess: "claimedSuccess";
+            }>, z.ZodNumber]>>;
+            code: z.ZodEnum<{
+                custom: "custom";
+                invalid_type: "invalid_type";
+                unrecognized_keys: "unrecognized_keys";
+                invalid_union: "invalid_union";
+                invalid_value: "invalid_value";
+                invalid_format: "invalid_format";
+                too_big: "too_big";
+                too_small: "too_small";
+                not_multiple_of: "not_multiple_of";
+                missing_point: "missing_point";
+                outside_frame: "outside_frame";
+                editable_target_missing: "editable_target_missing";
+                editable_target_ambiguous: "editable_target_ambiguous";
+                invalid_context: "invalid_context";
+                unsupported_key: "unsupported_key";
+                plan_not_grounded: "plan_not_grounded";
+            }>;
+            action: z.ZodEnum<{
+                type: "type";
+                key: "key";
+                unknown: "unknown";
+                answer: "answer";
+                wait: "wait";
+                typeText: "typeText";
+                batch: "batch";
+                terminate: "terminate";
+                activate: "activate";
+                showMenu: "showMenu";
+                click: "click";
+                drag: "drag";
+                setValue: "setValue";
+                pressKeys: "pressKeys";
+                scroll: "scroll";
+                activateSequence: "activateSequence";
+                interact: "interact";
+                takeover: "takeover";
+                finished: "finished";
+                button_plan: "button_plan";
+                activate_sequence: "activate_sequence";
+                show_menu: "show_menu";
+                set_value: "set_value";
+                left_click: "left_click";
+                right_click: "right_click";
+                double_click: "double_click";
+                hscroll: "hscroll";
+            }>;
+        }, z.core.$strict>>;
+    }, z.core.$strict>>;
+    stability: z.ZodOptional<z.ZodObject<{
+        stage: z.ZodEnum<{
+            initial_capture: "initial_capture";
+            pre_dispatch: "pre_dispatch";
+            post_action: "post_action";
+        }>;
+        samples: z.ZodNumber;
+        action: z.ZodOptional<z.ZodEnum<{
+            type: "type";
+            key: "key";
+            unknown: "unknown";
+            answer: "answer";
+            wait: "wait";
+            typeText: "typeText";
+            batch: "batch";
+            terminate: "terminate";
+            activate: "activate";
+            showMenu: "showMenu";
+            click: "click";
+            drag: "drag";
+            setValue: "setValue";
+            pressKeys: "pressKeys";
+            scroll: "scroll";
+            activateSequence: "activateSequence";
+            interact: "interact";
+            takeover: "takeover";
+            finished: "finished";
+            button_plan: "button_plan";
+            activate_sequence: "activate_sequence";
+            show_menu: "show_menu";
+            set_value: "set_value";
+            left_click: "left_click";
+            right_click: "right_click";
+            double_click: "double_click";
+            hscroll: "hscroll";
+        }>>;
+        changes: z.ZodArray<z.ZodEnum<{
+            window_identity: "window_identity";
+            window_geometry: "window_geometry";
+            image_dimensions: "image_dimensions";
+            accessibility_structure: "accessibility_structure";
+            accessibility_text: "accessibility_text";
+            accessibility_state: "accessibility_state";
+            accessibility_geometry: "accessibility_geometry";
+            visual_or_unexposed_state: "visual_or_unexposed_state";
+            deadline: "deadline";
+        }>>;
+        changedElementIndices: z.ZodArray<z.ZodNumber>;
+    }, z.core.$strict>>;
     serviceErrorCode: z.ZodOptional<z.ZodEnum<{
         provider_error: "provider_error";
         max_input_tokens_exceeded: "max_input_tokens_exceeded";
@@ -176,6 +346,176 @@ export declare const ComputerSessionReportSchema: z.ZodObject<{
             invalid_envelope: "invalid_envelope";
             output_limit: "output_limit";
         }>>;
+        validation: z.ZodOptional<z.ZodObject<{
+            action: z.ZodEnum<{
+                type: "type";
+                key: "key";
+                unknown: "unknown";
+                answer: "answer";
+                wait: "wait";
+                typeText: "typeText";
+                batch: "batch";
+                terminate: "terminate";
+                activate: "activate";
+                showMenu: "showMenu";
+                click: "click";
+                drag: "drag";
+                setValue: "setValue";
+                pressKeys: "pressKeys";
+                scroll: "scroll";
+                activateSequence: "activateSequence";
+                interact: "interact";
+                takeover: "takeover";
+                finished: "finished";
+                button_plan: "button_plan";
+                activate_sequence: "activate_sequence";
+                show_menu: "show_menu";
+                set_value: "set_value";
+                left_click: "left_click";
+                right_click: "right_click";
+                double_click: "double_click";
+                hscroll: "hscroll";
+            }>;
+            stage: z.ZodEnum<{
+                plan: "plan";
+                schema: "schema";
+                converted_action: "converted_action";
+                typing_target: "typing_target";
+                keyboard: "keyboard";
+            }>;
+            issues: z.ZodArray<z.ZodObject<{
+                path: z.ZodArray<z.ZodUnion<readonly [z.ZodEnum<{
+                    reason: "reason";
+                    keys: "keys";
+                    value: "value";
+                    name: "name";
+                    time: "time";
+                    from: "from";
+                    action: "action";
+                    when: "when";
+                    label: "label";
+                    text: "text";
+                    kind: "kind";
+                    status: "status";
+                    durationMs: "durationMs";
+                    role: "role";
+                    to: "to";
+                    actions: "actions";
+                    count: "count";
+                    x: "x";
+                    y: "y";
+                    point: "point";
+                    arguments: "arguments";
+                    ref: "ref";
+                    button: "button";
+                    deltaX: "deltaX";
+                    deltaY: "deltaY";
+                    refs: "refs";
+                    pixels: "pixels";
+                    steps: "steps";
+                    coordinate: "coordinate";
+                    unknown_field: "unknown_field";
+                    proposal: "proposal";
+                    claimedSuccess: "claimedSuccess";
+                }>, z.ZodNumber]>>;
+                code: z.ZodEnum<{
+                    custom: "custom";
+                    invalid_type: "invalid_type";
+                    unrecognized_keys: "unrecognized_keys";
+                    invalid_union: "invalid_union";
+                    invalid_value: "invalid_value";
+                    invalid_format: "invalid_format";
+                    too_big: "too_big";
+                    too_small: "too_small";
+                    not_multiple_of: "not_multiple_of";
+                    missing_point: "missing_point";
+                    outside_frame: "outside_frame";
+                    editable_target_missing: "editable_target_missing";
+                    editable_target_ambiguous: "editable_target_ambiguous";
+                    invalid_context: "invalid_context";
+                    unsupported_key: "unsupported_key";
+                    plan_not_grounded: "plan_not_grounded";
+                }>;
+                action: z.ZodEnum<{
+                    type: "type";
+                    key: "key";
+                    unknown: "unknown";
+                    answer: "answer";
+                    wait: "wait";
+                    typeText: "typeText";
+                    batch: "batch";
+                    terminate: "terminate";
+                    activate: "activate";
+                    showMenu: "showMenu";
+                    click: "click";
+                    drag: "drag";
+                    setValue: "setValue";
+                    pressKeys: "pressKeys";
+                    scroll: "scroll";
+                    activateSequence: "activateSequence";
+                    interact: "interact";
+                    takeover: "takeover";
+                    finished: "finished";
+                    button_plan: "button_plan";
+                    activate_sequence: "activate_sequence";
+                    show_menu: "show_menu";
+                    set_value: "set_value";
+                    left_click: "left_click";
+                    right_click: "right_click";
+                    double_click: "double_click";
+                    hscroll: "hscroll";
+                }>;
+            }, z.core.$strict>>;
+        }, z.core.$strict>>;
+        stability: z.ZodOptional<z.ZodObject<{
+            stage: z.ZodEnum<{
+                initial_capture: "initial_capture";
+                pre_dispatch: "pre_dispatch";
+                post_action: "post_action";
+            }>;
+            samples: z.ZodNumber;
+            action: z.ZodOptional<z.ZodEnum<{
+                type: "type";
+                key: "key";
+                unknown: "unknown";
+                answer: "answer";
+                wait: "wait";
+                typeText: "typeText";
+                batch: "batch";
+                terminate: "terminate";
+                activate: "activate";
+                showMenu: "showMenu";
+                click: "click";
+                drag: "drag";
+                setValue: "setValue";
+                pressKeys: "pressKeys";
+                scroll: "scroll";
+                activateSequence: "activateSequence";
+                interact: "interact";
+                takeover: "takeover";
+                finished: "finished";
+                button_plan: "button_plan";
+                activate_sequence: "activate_sequence";
+                show_menu: "show_menu";
+                set_value: "set_value";
+                left_click: "left_click";
+                right_click: "right_click";
+                double_click: "double_click";
+                hscroll: "hscroll";
+            }>>;
+            changes: z.ZodArray<z.ZodEnum<{
+                window_identity: "window_identity";
+                window_geometry: "window_geometry";
+                image_dimensions: "image_dimensions";
+                accessibility_structure: "accessibility_structure";
+                accessibility_text: "accessibility_text";
+                accessibility_state: "accessibility_state";
+                accessibility_geometry: "accessibility_geometry";
+                visual_or_unexposed_state: "visual_or_unexposed_state";
+                deadline: "deadline";
+            }>>;
+            changedElementIndices: z.ZodArray<z.ZodNumber>;
+        }, z.core.$strict>>;
         serviceErrorCode: z.ZodOptional<z.ZodEnum<{
             provider_error: "provider_error";
             max_input_tokens_exceeded: "max_input_tokens_exceeded";

package/dist/src/computer/diagnostics.js

@@ -41,6 +41,8 @@ const ComputerFailureSchema = z.object({
 	httpStatus: z.number().int().min(100).max(599).optional(),
 	requestId: z.string().regex(/^(?:[a-f0-9]{32}|[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12})$/i).optional(),
 	validationReason: ComputerDiagnosticSchema.shape.validationReason,
+	validation: ComputerDiagnosticSchema.shape.validation,
+	stability: ComputerDiagnosticSchema.shape.stability,
 	serviceErrorCode: ComputerDiagnosticSchema.shape.serviceErrorCode
 }).strict();
 const categories = Object.create(null);
@@ -211,6 +213,8 @@ function classifyComputerFailure(error, phase, dispatch = "notStarted", aborted
 			httpStatus: diagnostic.httpStatus,
 			requestId: diagnostic.requestId,
 			validationReason: diagnostic.validationReason,
+			validation: diagnostic.validation,
+			stability: diagnostic.stability,
 			serviceErrorCode: diagnostic.serviceErrorCode
 		} : {}
 	};