@cydm/pie 2.2.0 → 2.4.0

package/dist/builtin/extensions/ask-user/index.js

@@ -1,9 +1,9 @@
 import { createRequire as __createRequire } from "node:module"; const require = __createRequire(import.meta.url);
 import {
   createAskUserCapability
-} from "../../../chunks/chunk-HH2Q2SNX.js";
-import "../../../chunks/chunk-QJH6YN2W.js";
-import "../../../chunks/chunk-IM43UYVV.js";
+} from "../../../chunks/chunk-FKUEIYFQ.js";
+import "../../../chunks/chunk-ZKHMNEXL.js";
+import "../../../chunks/chunk-YRXXRLMQ.js";
 import "../../../chunks/chunk-XXGYUYXI.js";
 import "../../../chunks/chunk-LZQXQOPJ.js";
 

package/dist/builtin/extensions/computer-use/index.js

@@ -3776,10 +3776,26 @@ function containsEditable(node) {
   if (node.canSetValue || node.role.toLowerCase().includes("text")) return true;
   return node.children.some(containsEditable);
 }
+function wireableInputDescendants(node) {
+  const matches = [];
+  const visit = (current) => {
+    for (const child of current.children) {
+      if (child.isTextInput && child.wireRef && !child.pictureOnly && !child.offscreen) matches.push(child);
+      visit(child);
+    }
+  };
+  visit(node);
+  return matches;
+}
 function recordActionFocus(action, state, node, coordinateFocus = false) {
   if (!["click", "press", "keypress", "typeText"].includes(action.action) || !action.ref && action.action !== "click") return;
   state.currentFocus = Boolean(node?.isTextInput && !node.pictureOnly) || coordinateFocus;
   state.focusRef = node?.isTextInput && node.wireRef && !node.pictureOnly ? node.ref : void 0;
+  if (!state.currentFocus && node && !node.pictureOnly && action.ref && (action.action === "click" || action.action === "press") && containsEditable(node)) {
+    const wireable = wireableInputDescendants(node);
+    state.currentFocus = true;
+    state.focusRef = wireable.length === 1 ? wireable[0].ref : void 0;
+  }
 }
 function editorAtPoint(nodes, x, y) {
   const matches = nodes.filter((node) => node.isTextInput && node.wireRef && !node.pictureOnly && !node.offscreen && node.rect && x >= node.rect.x && x < node.rect.x + node.rect.w && y >= node.rect.y && y < node.rect.y + node.rect.h);
@@ -3790,24 +3806,23 @@ function prepareAction(action, state, env) {
   const focusedKeyboard = !env.headless && state.currentFocus && !action.ref && (operation === "typeText" || operation === "keypress");
   const usesCurrentFocus = focusedKeyboard && !state.focusRef;
   const target = usesCurrentFocus ? focusedTarget(env) : nativeTarget(focusedKeyboard && state.focusRef ? { ...action, ref: state.focusRef } : action, operation, env);
-  const establishesFocus = !env.headless && Boolean(action.ref) && (operation === "click" || operation === "press") && containsEditable(env.node(action.ref));
   const needsForeground = !env.headless && (operation === "click" || operation === "press") && "x" in target;
   switch (operation) {
     case "press":
     case "click":
-      return { action: operation, target, params: { button: mouseButton(action.button), clickCount: clickCount(action.clickCount) }, establishesFocus, usesCurrentFocus: false, needsForeground };
+      return { action: operation, target, params: { button: mouseButton(action.button), clickCount: clickCount(action.clickCount) }, usesCurrentFocus: false, needsForeground };
     case "setText":
-      return { action: operation, target, params: { text: action.text ?? "" }, establishesFocus: false, usesCurrentFocus: false, needsForeground: false };
+      return { action: operation, target, params: { text: action.text ?? "" }, usesCurrentFocus: false, needsForeground: false };
     case "typeText":
-      return { action: operation, target, params: { text: action.text ?? "" }, establishesFocus: false, usesCurrentFocus, needsForeground: false };
+      return { action: operation, target, params: { text: action.text ?? "" }, usesCurrentFocus, needsForeground: false };
     case "keypress":
-      return { action: operation, target, params: { keys: keys(action.keys) }, establishesFocus: false, usesCurrentFocus, needsForeground: false };
+      return { action: operation, target, params: { keys: keys(action.keys) }, usesCurrentFocus, needsForeground: false };
     case "scroll":
-      return { action: operation, target, params: { scrollX: scrollDelta(action.scrollX), scrollY: scrollDelta(action.scrollY) }, establishesFocus: false, usesCurrentFocus: false, needsForeground: false };
+      return { action: operation, target, params: { scrollX: scrollDelta(action.scrollX), scrollY: scrollDelta(action.scrollY) }, usesCurrentFocus: false, needsForeground: false };
     case "drag":
-      return { action: operation, target, params: { path: path(action.path, env) }, establishesFocus: false, usesCurrentFocus: false, needsForeground: false };
+      return { action: operation, target, params: { path: path(action.path, env) }, usesCurrentFocus: false, needsForeground: false };
     case "moveMouse":
-      return { action: operation, target, params: {}, establishesFocus: false, usesCurrentFocus: false, needsForeground: false };
+      return { action: operation, target, params: {}, usesCurrentFocus: false, needsForeground: false };
   }
 }
 function canRetryInForeground(action, outcome, headless) {
@@ -4405,6 +4420,9 @@ var CdpTab = class _CdpTab {
   /** Evaluates a JS expression in the page and returns its primitive value. */
   async evaluate(expression) {
     const result = await this.send("Runtime.evaluate", { expression, returnByValue: true, timeout: COMMAND_TIMEOUT_MS, awaitPromise: true });
+    if (result?.exceptionDetails) {
+      throw new Error(`Evaluation failed: ${exceptionDetailsMessage(result.exceptionDetails)}`);
+    }
     return result?.result?.value;
   }
   async accessibilityTree() {
@@ -4429,6 +4447,9 @@ var CdpTab = class _CdpTab {
   async clickBackendNode(backendNodeId) {
     await this.withBackendNode(backendNodeId, "function(){ this.scrollIntoView({block:'center', inline:'center'}); this.click(); }");
   }
+  async focusBackendNode(backendNodeId) {
+    await this.withBackendNode(backendNodeId, "function(){ this.scrollIntoView({block:'center', inline:'center'}); this.focus(); }");
+  }
   async typeIntoBackendNode(backendNodeId, text, replace) {
     await this.withBackendNode(backendNodeId, "function(text, replace){ this.scrollIntoView({block:'center', inline:'center'}); this.focus(); if (replace) { if ('value' in this) this.value = ''; else this.textContent = ''; } if ('value' in this) this.value += text; else this.textContent = (this.textContent || '') + text; this.dispatchEvent(new InputEvent('input', {bubbles:true, inputType:'insertText', data:text})); this.dispatchEvent(new Event('change', {bubbles:true})); }", [text, replace]);
   }
@@ -4464,11 +4485,14 @@ var CdpTab = class _CdpTab {
     const resolved = await this.send("DOM.resolveNode", { backendNodeId });
     const objectId = resolved?.object?.objectId;
     if (typeof objectId !== "string") throw new Error(`CDP could not resolve backend node ${backendNodeId}.`);
-    await this.send("Runtime.callFunctionOn", {
+    const callResult = await this.send("Runtime.callFunctionOn", {
       objectId,
       functionDeclaration,
       arguments: args.map((value) => ({ value }))
     });
+    if (callResult?.exceptionDetails) {
+      throw new Error(`CDP call on backend node ${backendNodeId} failed: ${exceptionDetailsMessage(callResult.exceptionDetails)}`);
+    }
   }
   /** Screen bounds of the browser window containing this tab. */
   async windowBounds() {
@@ -4655,6 +4679,12 @@ async function cdpTypeFocusedForContext(contextId, text) {
     return true;
   }) === true;
 }
+async function cdpFocusForContext(contextId, backendNodeId) {
+  return await withCdpContextTab(contextId, async (tab) => {
+    await tab.focusBackendNode(backendNodeId);
+    return true;
+  }) === true;
+}
 async function cdpKeypressForContext(contextId, keys2) {
   return await withCdpContextTab(contextId, async (tab) => {
     await tab.keypress(keys2);
@@ -4758,6 +4788,13 @@ function axString(raw) {
   const value = raw?.value ?? raw;
   return typeof value === "string" ? value.trim() : "";
 }
+function exceptionDetailsMessage(details) {
+  const exception = details?.exception;
+  if (typeof exception?.description === "string" && exception.description) return exception.description;
+  if (typeof exception?.value === "string" && exception.value) return exception.value;
+  if (typeof details?.text === "string" && details.text) return details.text;
+  return "the script threw an exception.";
+}
 function cdpSnapshotOutline(snapshotId, nodes) {
   const records = /* @__PURE__ */ new Map();
   for (const raw of nodes) {
@@ -4774,7 +4811,7 @@ function cdpSnapshotOutline(snapshotId, nodes) {
     const actions = browserActionsForAxRole(role);
     const backendNodeId = Number.isFinite(raw?.backendDOMNodeId) ? Math.trunc(raw.backendDOMNodeId) : void 0;
     const wireRef = `cdp:${nodeId}`;
-    if (actions.length > 0 && name && (!actions.includes("click") || backendNodeId)) {
+    if (actions.length > 0 && (!actions.includes("click") || backendNodeId)) {
       targets.push({ ref: wireRef, source: "browser_ax", role, name, value: axString(raw?.value) || void 0, actions, backendNodeId });
     }
     const childIds = Array.isArray(raw?.childIds) ? raw.childIds.map(String) : [];
@@ -8713,7 +8750,7 @@ async function performBrowserTransaction(params, actions, signal) {
     if (!BROWSER_TRANSACTION_ACTIONS.has(action.action)) throw new Error(`Browser transactions do not support '${action.action}'.`);
     if (action.action === "click" && action.ref && action.button && action.button !== "left") throw new Error("Browser ref clicks support only the left button; use coordinate clicks for right or middle buttons.");
     const target = browserSnapshotTarget(params.stateId, trimOrUndefined(action.ref));
-    if ((action.action === "press" || action.action === "setText" || action.action === "click" && action.ref || action.action === "typeText" && action.ref) && !Number.isFinite(target?.backendNodeId)) {
+    if ((action.action === "press" || action.action === "setText" || action.action === "click" && action.ref || action.action === "typeText" && action.ref || action.action === "keypress" && action.ref) && !Number.isFinite(target?.backendNodeId)) {
       throw new Error(`Browser ${action.action} requires an actionable @e ref owned by ${params.stateId}.`);
     }
     if (action.ref && (!target || target.contextId !== contextId)) throw new Error(`Browser ${action.action} ref must be owned by ${params.stateId}.`);
@@ -8729,7 +8766,7 @@ async function performBrowserTransaction(params, actions, signal) {
         worked = await cdpMouseForContext(contextId, action.x, action.y, "mousePressed", action.button ?? "left", action.clickCount ?? 1) && await cdpMouseForContext(contextId, action.x, action.y, "mouseReleased", action.button ?? "left", action.clickCount ?? 1);
       } else if (action.action === "setText") worked = await cdpTypeForContext(contextId, target.backendNodeId, action.text ?? "", true);
       else if (action.action === "typeText") worked = target?.backendNodeId ? await cdpTypeForContext(contextId, target.backendNodeId, action.text ?? "", false) : await cdpTypeFocusedForContext(contextId, action.text ?? "");
-      else if (action.action === "keypress") worked = await cdpKeypressForContext(contextId, action.keys ?? []);
+      else if (action.action === "keypress") worked = target?.backendNodeId ? await cdpFocusForContext(contextId, target.backendNodeId) && await cdpKeypressForContext(contextId, action.keys ?? []) : await cdpKeypressForContext(contextId, action.keys ?? []);
       else if (action.action === "scroll") worked = await cdpScrollForContext(contextId, toFiniteNumber(action.scrollX, 0), toFiniteNumber(action.scrollY, 0), target?.backendNodeId);
       else if (action.action === "drag") worked = await cdpDragForContext(contextId, normalizeActionPath(action.path));
       else if (action.action === "moveMouse") worked = await cdpMouseForContext(contextId, action.x, action.y, "mouseMoved");

package/dist/builtin/extensions/computer-use/package.json

@@ -4,6 +4,9 @@
 	"description": "Desktop computer-use tools (observe/act) vendored from @injaneity/pi-computer-use",
 	"main": "index.js",
 	"type": "module",
+	"scripts": {
+		"test:native-retention": "node scripts/test-native-retention.mjs"
+	},
 	"pie": {
 		"name": "computer-use",
 		"main": "index.js"

package/dist/builtin/extensions/computer-use/scripts/build-native.mjs

@@ -11,6 +11,7 @@ const macosSourcePaths = [
 	"agent_cursor.swift",
 	"agent_cursor_motion.swift",
 	"bridge.swift",
+	"ref_store.swift",
 ].map((file) => path.join(rootDir, "native", "macos", file));
 const windowsCrateDir = path.join(rootDir, "native", "windows", "bridge-rs");
 const linuxCrateDir = path.join(rootDir, "native", "linux", "bridge-rs");
@@ -20,7 +21,7 @@ const archTriples = {
 	x64: "x86_64-apple-macosx",
 };
 const deploymentTarget = "14.0";
-const frameworks = ["ApplicationServices", "AppKit", "ScreenCaptureKit", "Foundation", "SwiftUI"];
+const frameworks = ["ApplicationServices", "AppKit", "ScreenCaptureKit", "Foundation", "SwiftUI", "CryptoKit"];
 const defaultCodeSignIdentifier = "com.pie.computer-use";
 
 async function exists(filePath) {

package/dist/builtin/extensions/computer-use/scripts/setup-helper.mjs

@@ -20,7 +20,7 @@ const windowsCrateDir = path.join(rootDir, "native", "windows", "bridge-rs");
 const windowsHelperDestPath = process.env.PIE_COMPUTER_USE_WINDOWS_HELPER_PATH || path.join(resolvePieConfigDir(), "helpers", "computer-use", "windows-bridge.exe");
 const linuxCrateDir = path.join(rootDir, "native", "linux", "bridge-rs");
 const linuxHelperDestPath = process.env.PIE_COMPUTER_USE_LINUX_HELPER_PATH || path.join(resolvePieConfigDir(), "helpers", "computer-use", "linux-bridge");
-const helperSourcePaths = ["agent_cursor.swift", "agent_cursor_motion.swift", "bridge.swift"]
+const helperSourcePaths = ["agent_cursor.swift", "agent_cursor_motion.swift", "bridge.swift", "ref_store.swift"]
 	.map((file) => path.join(rootDir, "native", "macos", file));
 const packageJsonPath = path.join(rootDir, "package.json");
 const localCodeSignCommonName = "pie-computer-use Local Signing (com.pie.computer-use)";
@@ -57,7 +57,7 @@ const archTriples = {
 	x64: "x86_64-apple-macosx",
 };
 const deploymentTarget = "14.0";
-const frameworks = ["ApplicationServices", "AppKit", "ScreenCaptureKit", "Foundation", "SwiftUI"];
+const frameworks = ["ApplicationServices", "AppKit", "ScreenCaptureKit", "Foundation", "SwiftUI", "CryptoKit"];
 const defaultCodeSignIdentifier = "com.pie.computer-use";
 
 function normalizeArch(arch) {

package/dist/builtin/extensions/computer-use/scripts/test-native-retention.mjs

@@ -0,0 +1,204 @@
+#!/usr/bin/env node
+
+// Compiles and runs the macOS native AXRefStore retention tests
+// (native/macos/retention_tests.swift) from source, then compiles the full
+// helper and drives its stdin JSON-lines transport with permission-free
+// requests to assert the bridge-level stale-ref/stale-look integration.
+//
+// Nothing here needs Accessibility or Screen Recording permission or real UI
+// interaction: the store tests use synthetic AXUIElement values, and the
+// helper protocol probes only exercise failure paths (unknown look/element
+// ids, missing windows). This script is macOS-only by design; on other
+// platforms it skips with an explicit message (Linux/Windows helpers have
+// their own native suites).
+
+import { spawn } from "node:child_process";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const nativeDir = path.join(rootDir, "native", "macos");
+const storeTestSources = ["ref_store.swift", "retention_tests.swift"].map((file) => path.join(nativeDir, file));
+const helperSources = ["agent_cursor.swift", "agent_cursor_motion.swift", "bridge.swift", "ref_store.swift"].map((file) => path.join(nativeDir, file));
+const deploymentTarget = "14.0";
+const helperFrameworks = ["ApplicationServices", "AppKit", "ScreenCaptureKit", "Foundation", "SwiftUI", "CryptoKit"];
+
+function run(command, args, options = {}) {
+	return new Promise((resolve, reject) => {
+		const child = spawn(command, args, { stdio: "inherit", ...options });
+		child.on("error", reject);
+		child.on("close", (code) => {
+			if (code === 0) {
+				resolve();
+				return;
+			}
+			reject(new Error(`Command failed (${code}): ${command} ${args.join(" ")}`));
+		});
+	});
+}
+
+function swiftcArgs(sources, output, moduleCache, frameworks) {
+	const args = [
+		"swiftc",
+		"-target",
+		`${process.arch === "arm64" ? "arm64" : "x86_64"}-apple-macosx${deploymentTarget}`,
+		"-module-cache-path",
+		moduleCache,
+		"-O",
+	];
+	for (const framework of frameworks) args.push("-framework", framework);
+	args.push(...sources, "-o", output);
+	return args;
+}
+
+/** Drives the helper over its stdin transport and resolves one response per request id. */
+function helperResponses(executable, requests, timeoutMs = 15_000) {
+	return new Promise((resolve, reject) => {
+		const child = spawn(executable, [], { stdio: ["pipe", "pipe", "ignore"] });
+		const pending = new Map(requests.map((request) => [request.id, request]));
+		const settled = new Map();
+		const fail = (error) => {
+			child.kill();
+			reject(error);
+		};
+		const timer = setTimeout(() => fail(new Error(`Helper did not answer all requests within ${timeoutMs}ms`)), timeoutMs);
+		let buffer = "";
+		child.stdout.setEncoding("utf8");
+		child.stdout.on("data", (chunk) => {
+			buffer += chunk;
+			let newline;
+			while ((newline = buffer.indexOf("\n")) >= 0) {
+				const line = buffer.slice(0, newline);
+				buffer = buffer.slice(newline + 1);
+				if (!line.trim()) continue;
+				let response;
+				try {
+					response = JSON.parse(line);
+				} catch {
+					continue;
+				}
+				if (typeof response.id === "string" && pending.delete(response.id)) {
+					settled.set(response.id, response);
+					if (pending.size === 0) {
+						clearTimeout(timer);
+						child.stdin.end();
+						child.on("close", () => resolve(settled));
+						child.kill();
+					}
+				}
+			}
+		});
+		child.on("error", fail);
+		for (const request of requests) child.stdin.write(`${JSON.stringify(request)}\n`);
+	});
+}
+
+/** Verifies the helper's true-exit path: shutdown answers, then the process exits zero on its own. */
+function helperShutdownExits(executable, timeoutMs = 8_000) {
+	return new Promise((resolve, reject) => {
+		const child = spawn(executable, [], { stdio: ["pipe", "pipe", "ignore"] });
+		let buffer = "";
+		let sawResponse = false;
+		const fail = (error) => {
+			child.kill();
+			reject(error);
+		};
+		const timer = setTimeout(() => fail(new Error(`Helper did not shut down within ${timeoutMs}ms`)), timeoutMs);
+		child.stdout.setEncoding("utf8");
+		child.stdout.on("data", (chunk) => {
+			buffer += chunk;
+			let newline;
+			while ((newline = buffer.indexOf("\n")) >= 0) {
+				const line = buffer.slice(0, newline);
+				buffer = buffer.slice(newline + 1);
+				if (!line.trim()) continue;
+				let response;
+				try {
+					response = JSON.parse(line);
+				} catch {
+					continue;
+				}
+				if (response.id === "shutdown" && response.ok === true && response.result?.shuttingDown === true) sawResponse = true;
+			}
+		});
+		child.on("error", fail);
+		child.on("close", (code, signal) => {
+			clearTimeout(timer);
+			if (!sawResponse) {
+				reject(new Error("Helper did not acknowledge shutdown before exiting"));
+				return;
+			}
+			if (code === 0) {
+				resolve();
+				return;
+			}
+			reject(new Error(`Helper exited with ${code ?? "null"}${signal ? ` (signal ${signal})` : ""} instead of 0`));
+		});
+		child.stdin.write(`${JSON.stringify({ id: "shutdown", cmd: "shutdown" })}\n`);
+	});
+}
+
+function assert(condition, message) {
+	if (!condition) throw new Error(`[pie-computer-use] bridge integration assertion failed: ${message}`);
+}
+
+async function runBridgeIntegration(helperPath) {
+	const responses = await helperResponses(helperPath, [
+		{ id: "diag", cmd: "diagnostics" },
+		{ id: "act-stale", cmd: "act", lookId: "look_999", pid: 123, action: "press", target: { ref: "e1" } },
+		{ id: "batch-stale", cmd: "actBatch", actions: [{ lookId: "look_999", pid: 123, action: "press", target: { ref: "e1" } }] },
+		{ id: "hit-stale", cmd: "hitTest", lookId: "look_999", x: 10, y: 10 },
+		{ id: "read-stale", cmd: "axReadText", elementRef: "e999999" },
+		{ id: "look-stale-base", cmd: "look", baseLookId: "look_999", windowId: 12345 },
+	]);
+	const diagnostics = responses.get("diag");
+	assert(diagnostics?.ok === true, "diagnostics must succeed");
+	assert(diagnostics.result?.protocolVersion === 6, "helper must keep protocol version 6");
+	assert(Array.isArray(diagnostics.result?.invariants) && diagnostics.result.invariants.includes("bounded-observation-history"), "helper must report the bounded-observation-history invariant");
+	for (const [id, expectedCode] of [
+		["act-stale", "stale_look"],
+		["batch-stale", "stale_look"],
+		["hit-stale", "stale_look"],
+		["read-stale", "element_ref_invalid"],
+		["look-stale-base", "stale_look"],
+	]) {
+		const response = responses.get(id);
+		assert(response?.ok === false, `${id} must fail`);
+		assert(response.error?.code === expectedCode, `${id} must fail with ${expectedCode}, got ${response.error?.code}`);
+	}
+	console.log("[pie-computer-use] bridge integration probes passed (stale look/ref failures, protocol v6).");
+}
+
+async function main() {
+	if (process.platform !== "darwin") {
+		console.log(`[pie-computer-use] skipping native retention tests: they are macOS-only and cannot run on ${process.platform}.`);
+		return;
+	}
+	let tempDir;
+	try {
+		tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "pie-computer-use-retention-"));
+		const moduleCache = path.join(tempDir, "module-cache");
+		const storeTestOutput = path.join(tempDir, "retention_tests");
+		console.log("[pie-computer-use] compiling native retention tests...");
+		await run("xcrun", swiftcArgs(storeTestSources, storeTestOutput, moduleCache, ["ApplicationServices", "CryptoKit"]));
+		console.log("[pie-computer-use] running native retention tests...");
+		await run(storeTestOutput, []);
+
+		const helperOutput = path.join(tempDir, "pie-computer-use");
+		console.log("[pie-computer-use] compiling native helper for bridge integration probes...");
+		await run("xcrun", swiftcArgs(helperSources, helperOutput, moduleCache, helperFrameworks));
+		await runBridgeIntegration(helperOutput);
+		await helperShutdownExits(helperOutput);
+		console.log("[pie-computer-use] helper shutdown exits cleanly with code 0.");
+		console.log("[pie-computer-use] native retention tests passed.");
+	} finally {
+		if (tempDir) await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {});
+	}
+}
+
+main().catch((error) => {
+	console.error(error instanceof Error ? error.message : String(error));
+	process.exit(1);
+});