@e2edev/e2e 0.12.0 → 0.14.0

package/dist/agent/operation-queue.js

@@ -0,0 +1,27 @@
+/**
+ * Call-order serialization for one step's observations and actions.
+ *
+ * An executor (or an AI SDK loop running parallel tool calls) that issues a
+ * second action before the first settles would otherwise resolve both
+ * targets against the same pre-action observation — exactly the wrong-node
+ * hazard the staleness rule exists to prevent. Queued, the second call sees
+ * the newest observation and a stale id fails loud instead of acting on the
+ * wrong node.
+ *
+ * Invariant: a queued body must never enqueue another operation itself — the
+ * inner call would wait behind its own caller and deadlock. Action bodies
+ * call the engine session directly, secret authorization never observes, and
+ * a relocation captures raw rather than through the queued observe.
+ */
+export class OperationQueue {
+    chain = Promise.resolve();
+    /** Runs `body` after every operation queued before it, in call order. */
+    run(body) {
+        // The chain is always already settled-to-undefined, so failures propagate
+        // to their own caller and never poison the queue.
+        const run = this.chain.then(body);
+        this.chain = run.then(() => undefined, () => undefined);
+        return run;
+    }
+}
+//# sourceMappingURL=operation-queue.js.map

package/dist/agent/operation-queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"operation-queue.js","sourceRoot":"","sources":["../../src/agent/operation-queue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,cAAc;IACjB,KAAK,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEpD,yEAAyE;IACzE,GAAG,CAAI,IAAsB;QAC3B,0EAA0E;QAC1E,kDAAkD;QAClD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,IAAI,CACnB,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CAChB,CAAC;QACF,OAAO,GAAG,CAAC;IACb,CAAC;CACF"}

package/dist/agent/phases.js

@@ -15,7 +15,7 @@ import { AgentError, toAgentError } from './error.js';
  * a clearly attributed failure, not the test budget.
  */
 export function boundedOperation(engine, actionTimeout, deadline) {
-    return engine.operation(Math.max(1, Math.min(actionTimeout, deadline.remaining())));
+    return { ...engine.operation(Math.max(1, Math.min(actionTimeout, deadline.remaining()))), origin: 'agent' };
 }
 /** Records one policy decision as a child event of the current step. */
 export function recordPolicyEvent(steps, name, decision, code) {

package/dist/agent/phases.js.map

@@ -1 +1 @@
-{"version":3,"file":"phases.js","sourceRoot":"","sources":["../../src/agent/phases.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAiB,MAAM,qBAAqB,CAAC;AAG7E,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEtD;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAqB,EACrB,aAAqB,EACrB,QAAkB;IAElB,OAAO,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,iBAAiB,CAC/B,KAAmB,EACnB,IAAY,EACZ,QAA8B,EAC9B,IAAa;IAEb,KAAK,CAAC,WAAW,CAAC;QAChB,IAAI,EAAE,QAAQ;QACd,SAAS,EAAE,SAAS,EAAE;QACtB,UAAU,EAAE,CAAC;QACb,MAAM,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACpD,IAAI;QACJ,QAAQ;QACR,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;KACxC,CAAC,CAAC;AACL,CAAC;AAiBD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAe,EACf,IAAe,EACf,IAA0B,EAC1B,MAA6C;IAE7C,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;IAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,MAAM,EAAE,QAAQ;YAChB,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;YACvD,GAAG,WAAW;SACf,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,MAAM,EAAE,KAAK,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;YAC3D,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;YACvD,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;IACzD,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,OAM9B;IACC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,UAAU,CAAC,WAAW,EAAE,GAAG,OAAO,CAAC,GAAG,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,UAAU,CAClB,cAAc,EACd,GAAG,OAAO,CAAC,GAAG,iBAAiB,OAAO,CAAC,SAAS,aAAa,EAC7D,MAAM,CACP,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAMrC;IACC,SAAS,CAAC;QACR,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrB,wEAAwE;YACxE,uEAAuE;YACvE,wDAAwD;YACxD,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YACzC,IAAI,WAAW,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;gBACxD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,KAAK,CAAC,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;AACH,CAAC;AAED,qFAAqF;AACrF,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,KAAK,YAAY,QAAQ;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC;IACjD,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,WAAW,KAAK,SAAS;QAAE,OAAO,WAAW,CAAC,IAAI,CAAC;IACvD,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;AACvD,CAAC"}
+{"version":3,"file":"phases.js","sourceRoot":"","sources":["../../src/agent/phases.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAiB,MAAM,qBAAqB,CAAC;AAG7E,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEtD;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAqB,EACrB,aAAqB,EACrB,QAAkB;IAElB,OAAO,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAC9G,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,iBAAiB,CAC/B,KAAmB,EACnB,IAAY,EACZ,QAA8B,EAC9B,IAAa;IAEb,KAAK,CAAC,WAAW,CAAC;QAChB,IAAI,EAAE,QAAQ;QACd,SAAS,EAAE,SAAS,EAAE;QACtB,UAAU,EAAE,CAAC;QACb,MAAM,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACpD,IAAI;QACJ,QAAQ;QACR,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;KACxC,CAAC,CAAC;AACL,CAAC;AAiBD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAe,EACf,IAAe,EACf,IAA0B,EAC1B,MAA6C;IAE7C,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;IAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,MAAM,EAAE,QAAQ;YAChB,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;YACvD,GAAG,WAAW;SACf,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,MAAM,EAAE,KAAK,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;YAC3D,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;YACvD,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;IACzD,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,OAM9B;IACC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,UAAU,CAAC,WAAW,EAAE,GAAG,OAAO,CAAC,GAAG,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,UAAU,CAClB,cAAc,EACd,GAAG,OAAO,CAAC,GAAG,iBAAiB,OAAO,CAAC,SAAS,aAAa,EAC7D,MAAM,CACP,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAMrC;IACC,SAAS,CAAC;QACR,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrB,wEAAwE;YACxE,uEAAuE;YACvE,wDAAwD;YACxD,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YACzC,IAAI,WAAW,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;gBACxD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,KAAK,CAAC,gBAAgB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;AACH,CAAC;AAED,qFAAqF;AACrF,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,KAAK,YAAY,QAAQ;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC;IACjD,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,WAAW,KAAK,SAAS;QAAE,OAAO,WAAW,CAAC,IAAI,CAAC;IACvD,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;AACvD,CAAC"}

package/dist/agent/pixels.d.ts

@@ -0,0 +1,21 @@
+/**
+ * The screenshot a model receives, sized for it. Engines capture at CSS or
+ * device scale; a 1280 by 720 web viewport or a 1170 by 2532 phone is more
+ * image than a model needs to place a tap, and every image pixel is paid
+ * for on every turn it stays in the conversation. The long side is capped
+ * and the image resampled, and `scale` (image pixels per CSS pixel) is
+ * reduced with it, so every coordinate a model reads off the image still
+ * maps onto the viewport through the same arithmetic. Masked regions stay
+ * masked: a box filter over black stays black.
+ */
+import type { ObservationPixels } from '../engine/surface.ts';
+/** Sizes a captured screenshot for the model; the input is returned as is when it already fits. */
+export declare function sizeForModel<P extends ObservationPixels>(pixels: P): P;
+/**
+ * Resamples the image so its longer side is at most `maxLongSide`, with a
+ * box filter over the source pixels each target pixel covers. Returns the
+ * input untouched when it already fits or cannot be decoded; whatever else
+ * the caller's pixels carry (a masked-region count) rides along unchanged.
+ */
+export declare function downscalePixels<P extends ObservationPixels>(pixels: P, maxLongSide: number): P;
+//# sourceMappingURL=pixels.d.ts.map

package/dist/agent/pixels.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pixels.d.ts","sourceRoot":"","sources":["../../src/agent/pixels.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAa9D,mGAAmG;AACnG,wBAAgB,YAAY,CAAC,CAAC,SAAS,iBAAiB,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,CAEtE;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,CAAC,SAAS,iBAAiB,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,MAAM,GAAG,CAAC,CAgB9F"}

package/dist/agent/pixels.js

@@ -0,0 +1,85 @@
+/**
+ * The screenshot a model receives, sized for it. Engines capture at CSS or
+ * device scale; a 1280 by 720 web viewport or a 1170 by 2532 phone is more
+ * image than a model needs to place a tap, and every image pixel is paid
+ * for on every turn it stays in the conversation. The long side is capped
+ * and the image resampled, and `scale` (image pixels per CSS pixel) is
+ * reduced with it, so every coordinate a model reads off the image still
+ * maps onto the viewport through the same arithmetic. Masked regions stay
+ * masked: a box filter over black stays black.
+ */
+import { PNG } from 'pngjs';
+/**
+ * Longest side a model-bound screenshot keeps, in image pixels. Measured on
+ * gpt-5.6 over a ten-key drawn keypad and an eight-screen drawn wizard, four
+ * runs per size: a 1280 by 720 capture costs about 1390 input tokens per
+ * turn, 1024 about 960, 768 about 680, 640 about 580, 512 about 500. Every
+ * size down to 640 passed every run; at 512 the model misread the keypad's
+ * display, needed retries, and once accepted a wrong code. 768 keeps a full
+ * step of margin above that cliff for 35% less per run than full size.
+ */
+const MAX_SCREENSHOT_LONG_SIDE = 768;
+/** Sizes a captured screenshot for the model; the input is returned as is when it already fits. */
+export function sizeForModel(pixels) {
+    return downscalePixels(pixels, MAX_SCREENSHOT_LONG_SIDE);
+}
+/**
+ * Resamples the image so its longer side is at most `maxLongSide`, with a
+ * box filter over the source pixels each target pixel covers. Returns the
+ * input untouched when it already fits or cannot be decoded; whatever else
+ * the caller's pixels carry (a masked-region count) rides along unchanged.
+ */
+export function downscalePixels(pixels, maxLongSide) {
+    const longSide = Math.max(pixels.width, pixels.height);
+    if (longSide <= maxLongSide || maxLongSide <= 0)
+        return pixels;
+    const factor = maxLongSide / longSide;
+    const width = Math.max(1, Math.round(pixels.width * factor));
+    const height = Math.max(1, Math.round(pixels.height * factor));
+    let source;
+    try {
+        source = PNG.sync.read(Buffer.from(pixels.data));
+    }
+    catch {
+        return pixels;
+    }
+    const target = new PNG({ width, height });
+    boxResample(source.data, source.width, source.height, target.data, width, height);
+    const data = new Uint8Array(PNG.sync.write(target));
+    return { ...pixels, data, width, height, scale: pixels.scale * (width / pixels.width) };
+}
+/** Averages, per channel, every source pixel a target pixel covers. RGBA, 8 bits per channel. */
+function boxResample(source, sourceWidth, sourceHeight, target, targetWidth, targetHeight) {
+    const xRatio = sourceWidth / targetWidth;
+    const yRatio = sourceHeight / targetHeight;
+    for (let ty = 0; ty < targetHeight; ty += 1) {
+        const y0 = Math.floor(ty * yRatio);
+        const y1 = Math.min(sourceHeight, Math.max(y0 + 1, Math.floor((ty + 1) * yRatio)));
+        for (let tx = 0; tx < targetWidth; tx += 1) {
+            const x0 = Math.floor(tx * xRatio);
+            const x1 = Math.min(sourceWidth, Math.max(x0 + 1, Math.floor((tx + 1) * xRatio)));
+            let r = 0;
+            let g = 0;
+            let b = 0;
+            let a = 0;
+            let count = 0;
+            for (let y = y0; y < y1; y += 1) {
+                let offset = (y * sourceWidth + x0) * 4;
+                for (let x = x0; x < x1; x += 1) {
+                    r += source[offset];
+                    g += source[offset + 1];
+                    b += source[offset + 2];
+                    a += source[offset + 3];
+                    offset += 4;
+                    count += 1;
+                }
+            }
+            const at = (ty * targetWidth + tx) * 4;
+            target[at] = Math.round(r / count);
+            target[at + 1] = Math.round(g / count);
+            target[at + 2] = Math.round(b / count);
+            target[at + 3] = Math.round(a / count);
+        }
+    }
+}
+//# sourceMappingURL=pixels.js.map

package/dist/agent/pixels.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pixels.js","sourceRoot":"","sources":["../../src/agent/pixels.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC;AAG5B;;;;;;;;GAQG;AACH,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAErC,mGAAmG;AACnG,MAAM,UAAU,YAAY,CAA8B,MAAS;IACjE,OAAO,eAAe,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;AAC3D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAA8B,MAAS,EAAE,WAAmB;IACzF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACvD,IAAI,QAAQ,IAAI,WAAW,IAAI,WAAW,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAC/D,MAAM,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;IAC/D,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IAC1C,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAClF,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACpD,OAAO,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AAC1F,CAAC;AAED,iGAAiG;AACjG,SAAS,WAAW,CAClB,MAAkB,EAClB,WAAmB,EACnB,YAAoB,EACpB,MAAkB,EAClB,WAAmB,EACnB,YAAoB;IAEpB,MAAM,MAAM,GAAG,WAAW,GAAG,WAAW,CAAC;IACzC,MAAM,MAAM,GAAG,YAAY,GAAG,YAAY,CAAC;IAC3C,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,YAAY,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACnF,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,WAAW,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;YAC3C,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC;YACnC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAClF,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChC,IAAI,MAAM,GAAG,CAAC,CAAC,GAAG,WAAW,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;gBACxC,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,CAAC,IAAI,MAAM,CAAC,MAAM,CAAE,CAAC;oBACrB,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;oBACzB,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;oBACzB,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;oBACzB,MAAM,IAAI,CAAC,CAAC;oBACZ,KAAK,IAAI,CAAC,CAAC;gBACb,CAAC;YACH,CAAC;YACD,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,WAAW,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;YACvC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;YACvC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/agent/point-tap.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Point taps: how a viewport point an executor names is routed onto the
+ * tree, and how the result reads back. A point that a listed, enabled
+ * control contains is tapped by that control's id, so policy, stale
+ * relocation, and the trace cache see an ordinary tap; a point on nothing
+ * listed goes to the engine as a bare point. Coordinates read off a
+ * screenshot are scaled into the observation's CSS pixels first.
+ */
+import type { SemanticNode, ViewportPoint } from '../engine/surface.ts';
+import type { ExecutorPixels } from './executor.ts';
+import { type AgentObservation } from './observation.ts';
+/**
+ * Scales a point read off a screenshot into the observation's CSS pixels and
+ * clamps it to the viewport. `pixels.scale` is image pixels per CSS pixel
+ * (3 on a device screenshot, 1 on a CSS-scale capture); the observation's
+ * own `viewport.scale` is not consulted, because a device engine reports its
+ * geometry in logical points regardless of the image.
+ */
+export declare function imagePointToViewport(point: {
+    readonly x: number;
+    readonly y: number;
+}, pixels: Pick<ExecutorPixels, 'width' | 'height' | 'scale'>, viewport: {
+    readonly width: number;
+    readonly height: number;
+}): ViewportPoint;
+export interface HitTest {
+    /** The innermost enabled control whose box contains the point, if any. */
+    readonly control: SemanticNode | undefined;
+    /** The innermost listed node of any role whose box contains the point, if any. */
+    readonly under: SemanticNode | undefined;
+}
+/**
+ * Finds what the newest observation lists at a viewport point. Innermost
+ * wins (deepest in the tree, then the smallest box); hidden nodes and nodes
+ * without a box are skipped. Every box is in the top-level viewport's CSS
+ * pixels, nested documents included (`SemanticNode.rect`), so a control
+ * inside an iframe is found like any other. Only an enabled interactive
+ * role counts as the control: tapping a node taps its center, which is what
+ * the caller asked for only when the node is the control itself, not a
+ * paragraph or region that happens to contain the point.
+ */
+export declare function hitTest(observation: AgentObservation, point: ViewportPoint): HitTest;
+/** The observation's own line for a node, as the model already reads it; the id alone when the line was cut. */
+export declare function nodeLine(observation: Pick<AgentObservation, 'text'>, id: string): string;
+/** What one point tap did, for the model that named the point. */
+export declare function describePointTap(input: {
+    readonly point: ViewportPoint;
+    readonly control: SemanticNode | undefined;
+    readonly under: SemanticNode | undefined;
+    readonly observation: Pick<AgentObservation, 'text'>;
+}): string;
+//# sourceMappingURL=point-tap.d.ts.map

package/dist/agent/point-tap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"point-tap.d.ts","sourceRoot":"","sources":["../../src/agent/point-tap.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAExE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAqB,KAAK,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAE5E;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE;IAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;CAAE,EACjD,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAC,EAC1D,QAAQ,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC5D,aAAa,CASf;AAED,MAAM,WAAW,OAAO;IACtB,0EAA0E;IAC1E,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,SAAS,CAAC;IAC3C,kFAAkF;IAClF,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,SAAS,CAAC;CAC1C;AAED;;;;;;;;;GASG;AACH,wBAAgB,OAAO,CAAC,WAAW,EAAE,gBAAgB,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAgBpF;AAiBD,gHAAgH;AAChH,wBAAgB,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,CAMxF;AAED,kEAAkE;AAClE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE;IACtC,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,SAAS,CAAC;IAC3C,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;CACtD,GAAG,MAAM,CAOT"}

package/dist/agent/point-tap.js

@@ -0,0 +1,88 @@
+/**
+ * Point taps: how a viewport point an executor names is routed onto the
+ * tree, and how the result reads back. A point that a listed, enabled
+ * control contains is tapped by that control's id, so policy, stale
+ * relocation, and the trace cache see an ordinary tap; a point on nothing
+ * listed goes to the engine as a bare point. Coordinates read off a
+ * screenshot are scaled into the observation's CSS pixels first.
+ */
+import { clamp, clampToViewport } from '../internal/geometry.js';
+import { INTERACTIVE_ROLES } from './observation.js';
+/**
+ * Scales a point read off a screenshot into the observation's CSS pixels and
+ * clamps it to the viewport. `pixels.scale` is image pixels per CSS pixel
+ * (3 on a device screenshot, 1 on a CSS-scale capture); the observation's
+ * own `viewport.scale` is not consulted, because a device engine reports its
+ * geometry in logical points regardless of the image.
+ */
+export function imagePointToViewport(point, pixels, viewport) {
+    const scale = pixels.scale > 0 ? pixels.scale : 1;
+    return clampToViewport({
+        x: clamp(point.x, 0, Math.max(0, pixels.width - 1)) / scale,
+        y: clamp(point.y, 0, Math.max(0, pixels.height - 1)) / scale,
+    }, viewport);
+}
+/**
+ * Finds what the newest observation lists at a viewport point. Innermost
+ * wins (deepest in the tree, then the smallest box); hidden nodes and nodes
+ * without a box are skipped. Every box is in the top-level viewport's CSS
+ * pixels, nested documents included (`SemanticNode.rect`), so a control
+ * inside an iframe is found like any other. Only an enabled interactive
+ * role counts as the control: tapping a node taps its center, which is what
+ * the caller asked for only when the node is the control itself, not a
+ * paragraph or region that happens to contain the point.
+ */
+export function hitTest(observation, point) {
+    let control;
+    let under;
+    for (const node of observation.nodes.values()) {
+        const rect = node.rect;
+        if (rect === undefined || rect.width <= 0 || rect.height <= 0)
+            continue;
+        if (node.states?.hidden === true)
+            continue;
+        if (point.x < rect.x || point.x >= rect.x + rect.width)
+            continue;
+        if (point.y < rect.y || point.y >= rect.y + rect.height)
+            continue;
+        const candidate = { node, depth: depthOf(node.ref.id, observation.parents), area: rect.width * rect.height };
+        if (inner(candidate, under))
+            under = candidate;
+        if (INTERACTIVE_ROLES.has(node.role ?? '') && node.states?.disabled !== true && inner(candidate, control)) {
+            control = candidate;
+        }
+    }
+    return { control: control?.node, under: under?.node };
+}
+function inner(candidate, best) {
+    if (best === undefined)
+        return true;
+    if (candidate.depth !== best.depth)
+        return candidate.depth > best.depth;
+    return candidate.area < best.area;
+}
+function depthOf(id, parents) {
+    let depth = 0;
+    for (let cursor = parents.get(id); cursor !== undefined; cursor = parents.get(cursor))
+        depth += 1;
+    return depth;
+}
+/** The observation's own line for a node, as the model already reads it; the id alone when the line was cut. */
+export function nodeLine(observation, id) {
+    for (const line of observation.text.split('\n')) {
+        const trimmed = line.trimStart();
+        if (trimmed.startsWith(`#${id} `) || trimmed === `#${id}`)
+            return trimmed;
+    }
+    return `#${id}`;
+}
+/** What one point tap did, for the model that named the point. */
+export function describePointTap(input) {
+    const at = `(${String(input.point.x)}, ${String(input.point.y)})`;
+    if (input.control !== undefined) {
+        return `Tapped ${nodeLine(input.observation, input.control.ref.id)}, the control at ${at}.`;
+    }
+    const under = input.under === undefined ? '' : ` (under it: ${nodeLine(input.observation, input.under.ref.id)})`;
+    return `Tapped the point ${at}; no listed control is there${under}.`;
+}
+//# sourceMappingURL=point-tap.js.map

package/dist/agent/point-tap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"point-tap.js","sourceRoot":"","sources":["../../src/agent/point-tap.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAEjE,OAAO,EAAE,iBAAiB,EAAyB,MAAM,kBAAkB,CAAC;AAE5E;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAAiD,EACjD,MAA0D,EAC1D,QAA6D;IAE7D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,OAAO,eAAe,CACpB;QACE,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK;QAC3D,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK;KAC7D,EACD,QAAQ,CACT,CAAC;AACJ,CAAC;AASD;;;;;;;;;GASG;AACH,MAAM,UAAU,OAAO,CAAC,WAA6B,EAAE,KAAoB;IACzE,IAAI,OAAwE,CAAC;IAC7E,IAAI,KAAsE,CAAC;IAC3E,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC;YAAE,SAAS;QACxE,IAAI,IAAI,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI;YAAE,SAAS;QAC3C,IAAI,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK;YAAE,SAAS;QACjE,IAAI,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM;YAAE,SAAS;QAClE,MAAM,SAAS,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC7G,IAAI,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC;YAAE,KAAK,GAAG,SAAS,CAAC;QAC/C,IAAI,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC;YAC1G,OAAO,GAAG,SAAS,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACxD,CAAC;AAED,SAAS,KAAK,CACZ,SAA0C,EAC1C,IAAiD;IAEjD,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACpC,IAAI,SAAS,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACxE,OAAO,SAAS,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;AACpC,CAAC;AAED,SAAS,OAAO,CAAC,EAAU,EAAE,OAAoC;IAC/D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,MAAM,KAAK,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;QAAE,KAAK,IAAI,CAAC,CAAC;IAClG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,gHAAgH;AAChH,MAAM,UAAU,QAAQ,CAAC,WAA2C,EAAE,EAAU;IAC9E,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QACjC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,OAAO,KAAK,IAAI,EAAE,EAAE;YAAE,OAAO,OAAO,CAAC;IAC5E,CAAC;IACD,OAAO,IAAI,EAAE,EAAE,CAAC;AAClB,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,gBAAgB,CAAC,KAKhC;IACC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,UAAU,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,oBAAoB,EAAE,GAAG,CAAC;IAC9F,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC;IACjH,OAAO,oBAAoB,EAAE,+BAA+B,KAAK,GAAG,CAAC;AACvE,CAAC"}

package/dist/agent/primitives.d.ts

@@ -35,7 +35,7 @@ export interface GrammarToolOptions {
      * (budget, timeout, cancel) propagates out of the tool; the chassis's guard
      * instead turns it into text and ends the loop.
      */
-    readonly guard?: (body: () => Promise<string>) => Promise<string>;
+    readonly guard?: <T>(body: () => Promise<T>) => Promise<T | string>;
     /**
      * Renders screens for the model and remembers what it has seen, so every
      * result after the first reports the changes rather than the whole tree.
@@ -44,6 +44,12 @@ export interface GrammarToolOptions {
      */
     readonly screen?: ScreenPresenter;
 }
+/**
+ * Every name `createGrammarTools` may hand out. The grammar owns these in the
+ * model's vocabulary whatever the engine declares, so a project tool cannot
+ * take one: it would be silently shadowed on one engine and live on another.
+ */
+export declare const GRAMMAR_TOOL_NAMES: ReadonlySet<string>;
 /**
  * AI SDK tools over the harness action grammar, limited to the verbs the
  * target's engine declared. A verb the surface cannot honor is not offered

package/dist/agent/primitives.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"primitives.d.ts","sourceRoot":"","sources":["../../src/agent/primitives.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpD,OAAO,EAAsC,KAAK,mBAAmB,EAAE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAE/G,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AA2CrD,wEAAwE;AACxE,eAAO,MAAM,aAAa,umBAG4N,CAAC;AAcvP,wDAAwD;AACxD,MAAM,WAAW,WAAW;IAC1B,uEAAuE;IACvE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,uEAAuE;IACvE,OAAO,IAAI,WAAW,GAAG,SAAS,CAAC;IACnC,yEAAyE;IACzE,SAAS,IAAI,OAAO,CAAC;CACtB;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,IAAI,WAAW,CA4C/C;AAED,8EAA8E;AAC9E,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAClE;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,eAAe,CAAC;CACnC;AAED;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,mBAAmB,EAC5B,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAuJT;AAED,+EAA+E;AAC/E,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,WAAW,IAAI,IAAI,CAAC;IACpB;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,kBAAkB,CAAC,GAAG,IAAI,CAAC;CAChF;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE;IAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAChE,gBAAgB,CAoBlB"}
+{"version":3,"file":"primitives.d.ts","sourceRoot":"","sources":["../../src/agent/primitives.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpD,OAAO,EAAsC,KAAK,mBAAmB,EAAE,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAI/G,OAAO,EAAE,eAAe,EAAqB,MAAM,oBAAoB,CAAC;AA4CxE,wEAAwE;AACxE,eAAO,MAAM,aAAa,umBAG4N,CAAC;AAqDvP,wDAAwD;AACxD,MAAM,WAAW,WAAW;IAC1B,uEAAuE;IACvE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,uEAAuE;IACvE,OAAO,IAAI,WAAW,GAAG,SAAS,CAAC;IACnC,yEAAyE;IACzE,SAAS,IAAI,OAAO,CAAC;CACtB;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,IAAI,WAAW,CA4C/C;AAED,8EAA8E;AAC9E,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IACpE;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,eAAe,CAAC;CACnC;AAED;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAWjD,CAAC;AAEH;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,mBAAmB,EAC5B,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAsLT;AAED,+EAA+E;AAC/E,MAAM,WAAW,gBAAgB;IAC/B,sEAAsE;IACtE,WAAW,IAAI,IAAI,CAAC;IACpB;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,kBAAkB,CAAC,GAAG,IAAI,CAAC;CAChF;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,mBAAmB,EAC5B,KAAK,CAAC,EAAE;IAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAChE,gBAAgB,CAoBlB"}

package/dist/agent/primitives.js

@@ -11,6 +11,8 @@
 import { z } from 'zod';
 import { isRuntimeHardStop, BLOCKABLE_CODES } from './executor.js';
 import { cacheTokenFields, readCost } from './model/sdk.js';
+import { OperationQueue } from './operation-queue.js';
+import { imagePointToViewport } from './point-tap.js';
 import { ScreenPresenter } from './screen-update.js';
 /** Codes the model may pick when concluding; runtime codes are runtime-assigned. */
 const MODEL_ERROR_CODES = [
@@ -20,6 +22,7 @@ const MODEL_ERROR_CODES = [
     'AUTHENTICATION_FAILED',
     'AUTH_CREDENTIAL_UNAVAILABLE',
     'AUTH_CREDENTIAL_INVALID',
+    'SECRET_UNAVAILABLE',
     'ENVIRONMENT_UNAVAILABLE',
     'SEED_DATA_MISSING',
     'TEST_SETUP_FAILED',
@@ -59,6 +62,30 @@ export const VERDICT_RULES = `Verdict rules:
 function schemaTool(definition) {
     return definition;
 }
+/**
+ * A grammar tool: its result is a rendered screen, text or text with the
+ * screenshot attached as a file part, so every one encodes its output the
+ * same way. A tool that skipped the encoder would hand the model the image
+ * bytes as JSON the first time an action ran in pixel mode.
+ */
+function screenTool(definition) {
+    return schemaTool({ ...definition, toModelOutput: screenModelOutput });
+}
+function screenModelOutput({ output }) {
+    if (typeof output === 'string')
+        return { type: 'text', value: output };
+    return {
+        type: 'content',
+        value: [
+            { type: 'text', text: output.text },
+            {
+                type: 'file',
+                data: { type: 'data', data: Buffer.from(output.pixels.data).toString('base64') },
+                mediaType: output.pixels.mediaType,
+            },
+        ],
+    };
+}
 /**
  * The `complete_step` verdict tool. A blocked verdict without a blockable
  * code is rejected back to the model, a passed verdict never carries a code,
@@ -103,6 +130,23 @@ export function createVerdictTool() {
         concluded: () => verdict !== undefined,
     };
 }
+/**
+ * Every name `createGrammarTools` may hand out. The grammar owns these in the
+ * model's vocabulary whatever the engine declares, so a project tool cannot
+ * take one: it would be silently shadowed on one engine and live on another.
+ */
+export const GRAMMAR_TOOL_NAMES = new Set([
+    'observe',
+    'tap',
+    'type',
+    'type_secret',
+    'press',
+    'select',
+    'scroll',
+    'navigate',
+    'screenshot',
+    'tap_at',
+]);
 /**
  * AI SDK tools over the harness action grammar, limited to the verbs the
  * target's engine declared. A verb the surface cannot honor is not offered
@@ -117,71 +161,71 @@ export function createGrammarTools(context, options = {}) {
     const guard = options.guard ?? ((body) => body());
     const screen = options.screen ?? new ScreenPresenter();
     const { verbs } = context.target;
-    // The chain is always already settled-to-undefined, so a failed body
-    // reaches its own caller and never poisons the queue (same idiom as the
-    // dispatch's own serialization).
-    let chain = Promise.resolve();
-    const inOrder = (body) => {
-        const run = chain.then(body);
-        chain = run.then(() => undefined, () => undefined);
-        return run;
-    };
+    // One queue for the tool bodies: an action and its look at the result run
+    // together, so a batched turn gets one coherent result per action.
+    const queue = new OperationQueue();
+    const inOrder = (body) => queue.run(body);
+    /**
+     * The screen after an action, as the model reads it: the changes since the
+     * screen it holds and, once the step is showing pixels, a fresh screenshot.
+     */
+    const present = async (lead, expectChange) => screen.present(await context.observe({ pixels: screen.showingPixels }), { lead, expectChange });
     /**
      * Performs one action and reads its result. A failed action still returns
      * the screen, so the model can act on a stale-id or not-found failure at
      * once instead of spending a turn to observe; runtime hard stops propagate
      * to the guard, which ends the loop. The guard runs inside the queue, so a
      * batched call that queued behind a hard stop or a verdict is skipped when
-     * its turn comes rather than acted on because it was queued in time.
+     * its turn comes rather than acted on because it was queued in time. The
+     * action may return its own lead line, for a result only it can describe.
      */
     const acting = (description, action, expectChange = true) => inOrder(() => guard(async () => {
+        let lead = description;
         try {
-            await action();
+            lead = (await action()) ?? description;
         }
         catch (cause) {
             if (isRuntimeHardStop(cause))
                 throw cause;
             const message = cause instanceof Error ? cause.message : String(cause);
-            const observation = await context.observe();
-            return screen.update(observation, { lead: `${description} failed: ${message}` });
+            return present(`${description} failed: ${message}`);
         }
-        const observation = await context.observe();
-        return screen.update(observation, { lead: description, expectChange });
+        return present(lead, expectChange);
     }));
     const target = z
         .string()
         .min(1)
         .describe('Node id from any screen in this conversation that is still present, e.g. "n42"');
     const tools = {
-        observe: schemaTool({
+        observe: screenTool({
             description: 'Look at the screen again and get what changed since the screen you last received. Action results already include their changes, so call this only after waiting for something in progress, never right after an action.',
             inputSchema: z.object({}),
-            execute: () => inOrder(() => guard(async () => screen.update(await context.observe()))),
+            execute: () => inOrder(() => guard(() => present('Observed.', false))),
         }),
     };
     if (verbs.has('tap')) {
-        tools['tap'] = schemaTool({
+        tools['tap'] = screenTool({
             description: 'Tap or click one node. The result waits for the effect (a navigation, a route change, a submit) and reports what changed.',
             inputSchema: z.object({ target }),
             execute: ({ target: id }) => acting(`Tapped #${id}.`, () => context.actions.tap({ id })),
         });
     }
     if (verbs.has('type')) {
-        tools['type'] = schemaTool({
+        tools['type'] = screenTool({
             description: 'Type a plain-text value into one input node, replacing its current value. Several fields can be typed in one turn.',
             inputSchema: z.object({ target, value: z.string() }),
             execute: ({ target: id, value }) => acting(`Typed into #${id}.`, () => context.actions.type({ id }, value)),
         });
     }
     if (verbs.has('press')) {
-        tools['press'] = schemaTool({
+        tools['press'] = screenTool({
             description: 'Send one key (e.g. "Enter", "Escape", "Tab") to one node.',
             inputSchema: z.object({ target, key: z.string().min(1).max(64) }),
             execute: ({ target: id, key }) => acting(`Pressed ${key} on #${id}.`, () => context.actions.press({ id }, key), !movesFocusOnly(key)),
         });
     }
     if (verbs.has('select')) {
-        tools['select'] = schemaTool({
+        tools['select'] = screenTool({
             description: 'Pick one option from a select-like control by its visible label.',
             inputSchema: z.object({ target, value: z.string().min(1) }),
             execute: ({ target: id, value }) => acting(`Selected "${value}" in #${id}.`, () => context.actions.select({ id }, value)),
@@ -208,33 +252,61 @@ export function createGrammarTools(context, options = {}) {
         const scrolled = (way, count) => count === 1 ? `Scrolled ${way}.` : `Scrolled ${way} ${String(count)} screens.`;
         // Node-targeted scrolling rides `perform`; without it only the viewport scrolls.
         tools['scroll'] = verbs.has('tap')
-            ? schemaTool({
+            ? screenTool({
                 description: 'Scroll the viewport, or one scrollable node when target is given. The result reports the rows that came into or left the tree.',
                 inputSchema: z.object({ direction, target: target.optional(), times }),
                 execute: ({ direction: way, target: id, times: count }) => acting(scrolled(way, count ?? 1), () => scrolling(way, id, count ?? 1), false),
             })
-            : schemaTool({
+            : screenTool({
                 description: 'Scroll the viewport. The result reports the rows that came into or left the tree.',
                 inputSchema: z.object({ direction, times }),
                 execute: ({ direction: way, times: count }) => acting(scrolled(way, count ?? 1), () => scrolling(way, undefined, count ?? 1), false),
             });
     }
     if (verbs.has('navigate')) {
-        tools['navigate'] = schemaTool({
+        tools['navigate'] = screenTool({
             description: 'Navigate to a URL or app-relative path within the allowed origins.',
             inputSchema: z.object({ url: z.string().min(1) }),
             execute: ({ url }) => acting(`Navigated to ${url}.`, () => context.actions.navigate(url)),
         });
     }
+    // The pixel verbs are offered while pixels can still leave the runner. Once
+    // a secret was filled in the attempt they could only decline, and a verb
+    // that is absent costs the model nothing where one that declines costs a
+    // turn. tap_at lands either as a tap by id or as a bare point, so it needs
+    // one of the two; screenshot needs only the observation every step has.
+    if (!context.pixelsTainted) {
+        tools['screenshot'] = screenTool({
+            description: 'Attach a screenshot of the current viewport. Use it when the screen lists too little to act on (a canvas, a map, an image, a game, a system sheet) or contradicts what you expect. From then on every action result carries a fresh screenshot too, so you can see what each action did.',
+            inputSchema: z.object({}),
+            execute: () => inOrder(() => guard(async () => screen.present(await context.observe({ pixels: true })))),
+        });
+        if (verbs.has('tap') || verbs.has('tapAt')) {
+            tools['tap_at'] = screenTool({
+                description: 'Tap a point in the latest screenshot, given as pixel coordinates in that image (x from the left edge, y from the top edge). Aim for the center of the target. A listed control under the point is tapped by its id; otherwise the bare point is tapped' +
+                    (verbs.has('tapAt') ? '.' : ', which this engine cannot do: the point must land on a listed control.') +
+                    ' Last resort: when the screen lists the target, tap it by id.',
+                inputSchema: z.object({ x: z.number(), y: z.number() }),
+                execute: ({ x, y }) => {
+                    const shot = screen.latestScreenshot;
+                    if (shot === undefined) {
+                        return Promise.resolve('No screenshot has been taken in this step: tap_at coordinates are pixels of the latest screenshot. Call screenshot first, or tap a listed node by id.');
+                    }
+                    const point = imagePointToViewport({ x, y }, shot.pixels, shot.viewport);
+                    return acting(`tap_at (${String(x)}, ${String(y)})`, async () => (await context.actions.tapAt(point)).summary);
+                },
+            });
+        }
+    }
     // Offered only when the step declared secrets and the surface can fill: an
     // empty vocabulary is better than a tool the model can only be rejected on.
     if (verbs.has('typeSecret') && context.step.secrets.length > 0) {
-        tools['type_secret'] = schemaTool({
-            description: 'Fill one declared secret credential into a secure input field; the plaintext never passes through you and never shows on screen. Available: ' +
+        tools['type_secret'] = screenTool({
+            description: 'Fill one declared secret into an input by its name; the plaintext never passes through you. A password fills only a password field; a generic-secret fills any editable input. Available: ' +
                 context.step.secrets.map((secret) => `"${secret.name}" (${secret.purpose})`).join(', ') +
                 '.',
             inputSchema: z.object({ target, name: z.string().min(1) }),
-            execute: ({ target: id, name }) => acting(`Filled secret "${name}" into #${id}; secure values never show on screen.`, () => context.actions.typeSecret({ id }, name), false),
+            execute: ({ target: id, name }) => acting(`Filled secret "${name}" into #${id}; its value is masked in every observation.`, () => context.actions.typeSecret({ id }, name), false),
         });
     }
     return tools;