effect-browser 0.3.0-beta.16 → 0.3.0-beta.19

package/src/internal/input/targets.ts

@@ -8,7 +8,7 @@ import { Effect } from "effect";
 import { InvalidRequest, NotActionable, StaleRef } from "../../BrowserError.ts";
 import { ResolvedTarget, type Target } from "../../Page.ts";
 import { scriptCall } from "../page/bridge.ts";
-import { decodeWith, failWith } from "../page/context.ts";
+import { decodeWith, failWith, refuse } from "../page/context.ts";
 import type { Viewport } from "../page/viewport.ts";
 import type { Dispatch } from "./dispatch.ts";
 import { type Approval, check, type Guard } from "./guard.ts";
@@ -31,18 +31,8 @@ const pointFailure = (
         : new NotActionable({ detail: result.detail }),
   );
 
-const resolvedTarget = (result: Script.ResolvedPoint): ResolvedTarget =>
-  new ResolvedTarget({
-    point: { x: result.x, y: result.y },
-    element: result.element,
-    tag: result.tag,
-    role: result.role,
-    name: result.name,
-    context: result.context,
-    box: result.box,
-    cursor: result.cursor,
-    ...(result.href === undefined ? {} : { href: result.href }),
-  });
+const resolvedTarget = ({ x, y, box, ...target }: Script.ResolvedPoint): ResolvedTarget =>
+  new ResolvedTarget({ point: { x, y }, box, ...target });
 
 export const make = (
   viewport: Viewport,
@@ -67,15 +57,9 @@ export const make = (
 
   const resolve = (operation: string, target: Target, approval?: Approval, scroll = true) => {
     if (typeof target === "string" && !/^e\d+$/.test(target))
-      return failWith(
-        operation,
-        new InvalidRequest({ detail: target + " is not a ref; refs look like e12" }),
-      );
+      return refuse(operation, target + " is not a ref; refs look like e12");
     if (typeof target !== "string" && (!Number.isFinite(target.x) || !Number.isFinite(target.y)))
-      return failWith(
-        operation,
-        new InvalidRequest({ detail: "point coordinates must be finite" }),
-      );
+      return refuse(operation, "point coordinates must be finite");
 
     return readPoint(operation, target, approval, scroll).pipe(
       Effect.flatMap((result) =>

package/src/internal/page/bridge.ts

@@ -90,7 +90,7 @@ const install = (
 ): PageApi => {
   const installed = globalThis.__effectBrowser;
 
-  if (installed !== undefined && installed.version === 19) return installed;
+  if (installed !== undefined && installed.version === 20) return installed;
   const named = makeNames();
   const walked = makeWalk(named);
   const placing = makeContext(named, walked);
@@ -108,7 +108,7 @@ const install = (
   const readiness = makeReady(walked, texts, kept, recorder);
 
   const api: PageApi = {
-    version: 19,
+    version: 20,
     snapshot: read.snapshot,
     find: subjected.find,
     until: makeExpectations(named, subjected).until,

package/src/internal/page/context.ts

@@ -17,7 +17,14 @@ import {
 import type { CDPSession, Page as PlaywrightPage } from "playwright-core";
 
 import type { CaptureSource } from "../../Browser.ts";
-import { BrowserError, Closed, Failed, type Reason, Timeout } from "../../BrowserError.ts";
+import {
+  BrowserError,
+  Closed,
+  Failed,
+  InvalidRequest,
+  type Reason,
+  Timeout,
+} from "../../BrowserError.ts";
 import type { BrowserEvent } from "../../BrowserEvent.ts";
 import type { InputGuard, Point } from "../../Page.ts";
 import type { Gate } from "../input/gate.ts";
@@ -159,6 +166,10 @@ export const undispatched = (operation: string, reason: Reason) =>
 export const failWith = (operation: string, reason: Reason) =>
   Effect.fail(undispatched(operation, reason));
 
+/** Refuse a request to `operation` that could never succeed, before sending anything. */
+export const refuse = (operation: string, detail: string) =>
+  failWith(operation, new InvalidRequest({ detail }));
+
 export const decodeWith =
   <A>(operation: string, schema: Schema.Codec<A, unknown>) =>
   (value: unknown) =>
@@ -308,6 +319,7 @@ export const make = (options: MakeOptions, scope: Scope.Scope) => {
     lane,
     pointer,
     now,
+    scope,
     owned,
     span,
     protocol,

package/src/internal/page/navigation.ts

@@ -5,17 +5,11 @@
  */
 import { Deferred, Duration, Effect, Schedule } from "effect";
 
-import {
-  BrowserError,
-  InvalidRequest,
-  NavigationFailed,
-  NotActionable,
-  NotFound,
-} from "../../BrowserError.ts";
+import { BrowserError, NavigationFailed, NotActionable, NotFound } from "../../BrowserError.ts";
 import type { Guard } from "../input/guard.ts";
 import type { InputMarks, Perform } from "../input/perform.ts";
 import { Current } from "../input/replies.ts";
-import { failWith, type PageContext } from "./context.ts";
+import { failWith, type PageContext, refuse } from "./context.ts";
 import * as Url from "./url.ts";
 
 export const make = (
@@ -157,10 +151,7 @@ export const make = (
       parsed === null ||
       !["http:", "https:", "about:", "data:", "file:"].includes(parsed.protocol)
     )
-      return failWith(
-        "navigate",
-        new InvalidRequest({ detail: `"${Url.redact(url)}" is not a URL` }),
-      );
+      return refuse("navigate", `"${Url.redact(url)}" is not a URL`);
 
     return navigation(
       "navigate",

package/src/internal/pictures/capture.ts

@@ -47,6 +47,8 @@ interface Options {
   readonly frameHistory: Duration.Duration;
   /** The viewport in CSS pixels; a capture is scaled to fit it, as screenshots are. */
   readonly viewport: Effect.Effect<Size, BrowserError>;
+  /** Copies the restored whole surface before native capture can restart. */
+  readonly restore: () => Promise<unknown>;
   readonly imageSize: (data: Uint8Array) => Size | undefined;
   readonly error: (cause: unknown) => BrowserError;
 }
@@ -589,6 +591,11 @@ export const make = (options: Options) =>
           };
 
           if (current === undefined || !resumable()) return;
+          // Screenshot replies restore emulation before the compositor has necessarily painted
+          // it. Copy the whole restored surface while stopped, retaining its actual native reply.
+          void pictures.track("picture restoration", options.restore(), failed(current));
+          yield* pictures.wait("picture restoration");
+          yield* healthy(current);
           const viewport = yield* readViewport(options);
 
           if (!resumable()) return;

package/src/internal/pictures/motion.ts

@@ -0,0 +1,65 @@
+/**
+ * Where a page's screen moves, from its screencast frames: each frame decoded and cut into
+ * 16-pixel blocks, a block moving where a few of its pixels, sampled every other one, changed by
+ * more than JPEG's noise in any color channel.
+ * What moved among the frames before an input is the page's background, such as an idle
+ * animation; foreground stillness ignores it.
+ *
+ * Frames are decoded with the JPEG decoder playwright-core bundles and exports as
+ * `playwright-core/lib/utilsBundle`: effect-browser depends on nothing else, and the runtime has
+ * no image decoder. That entry point has no types, so `utilsBundle.d.ts` declares what is used.
+ */
+// oxlint-disable-next-line typescript/triple-slash-reference -- a declaration of an untyped module cannot be imported, and every program that compiles this source needs it
+/// <reference path="./utilsBundle.d.ts" />
+import { jpegjs } from "playwright-core/lib/utilsBundle";
+
+import type { Frame } from "../../Frame.ts";
+
+/** A decoded frame: RGBA, `width` pixels a row. */
+export type Picture = ReturnType<typeof jpegjs.decode>;
+
+export const picture = (frame: Frame): Picture => jpegjs.decode(frame.data, { useTArray: true });
+
+/** A sample changed where any color channel moved more than this, beyond JPEG's noise. */
+const noise = 32;
+
+/** The blocks where two pictures differ, by `row * columns + column`; -1 for different sizes. */
+const moved = (from: Picture, to: Picture): ReadonlyArray<number> => {
+  if (from.width !== to.width || from.height !== to.height) return [-1];
+  const columns = Math.ceil(from.width / 16);
+  const counts = new Map<number, number>();
+  const differs = (at: number) => Math.abs((from.data[at] ?? 0) - (to.data[at] ?? 0)) > noise;
+
+  for (let y = 0; y < from.height; y += 2)
+    for (let x = 0; x < from.width; x += 2) {
+      const at = (y * from.width + x) * 4;
+      const block = Math.floor(y / 16) * columns + Math.floor(x / 16);
+
+      if (differs(at) || differs(at + 1) || differs(at + 2))
+        counts.set(block, (counts.get(block) ?? 0) + 1);
+    }
+
+  // At least three changed samples, so a stray one is no motion.
+  return [...counts].flatMap(([block, count]) => (count >= 3 ? [block] : []));
+};
+
+/** The blocks that moved from one picture to the next and their neighbours; none past a resize. */
+export const background = (pictures: ReadonlyArray<Picture>): ReadonlySet<number> => {
+  const columns = Math.ceil((pictures[0]?.width ?? 0) / 16);
+
+  const around = (block: number) =>
+    [-1, 0, 1].flatMap((row) =>
+      [-1, 0, 1]
+        .filter((step) => block >= 0 && (block % columns) + step >= 0)
+        .filter((step) => (block % columns) + step < columns)
+        .map((step) => block + row * columns + step),
+    );
+
+  return new Set(
+    pictures.flatMap((to, index) => moved(pictures[index - 1] ?? to, to).flatMap(around)),
+  );
+};
+
+/** Whether anything moved from one picture to another outside the background. */
+export const stirs = (from: Picture, to: Picture, mask: ReadonlySet<number>) =>
+  moved(from, to).some((block) => !mask.has(block));

package/src/internal/pictures/pictures.ts

@@ -11,7 +11,14 @@ import { Frame, Image, Screenshot } from "../../Frame.ts";
 import type { Point } from "../../Motion.ts";
 import { type FrameOptions, Region, type ScreenshotOptions, Zoom } from "../../Page.ts";
 import { type Bridge, scriptCall } from "../page/bridge.ts";
-import { decodeWith, failWith, type PageContext, reasonOf, undispatched } from "../page/context.ts";
+import {
+  decodeWith,
+  failWith,
+  type PageContext,
+  reasonOf,
+  refuse,
+  undispatched,
+} from "../page/context.ts";
 import type { Viewport } from "../page/viewport.ts";
 import * as Capture from "./capture.ts";
 import * as BrowserClock from "./clock.ts";
@@ -118,9 +125,7 @@ const calibrator = (page: PageContext, bridge: Bridge) => {
       yield* Effect.forEach(
         [1, 2, 3],
         () => Effect.map(probe(contextId), (one) => probes.push(one)),
-        {
-          discard: true,
-        },
+        { discard: true },
       ).pipe(
         within("calibrate", Duration.seconds(2)),
         Effect.catch((error) => (probes.length === 0 ? Effect.fail(error) : Effect.void)),
@@ -308,10 +313,7 @@ const reuse =
       const maxAge = millisOf(options.maxAge ?? defaultMaxAge);
 
       if (!(maxAge >= 0))
-        return yield* failWith(
-          operation,
-          new InvalidRequest({ detail: "maxAge must be a duration of zero or more" }),
-        );
+        return yield* refuse(operation, "maxAge must be a duration of zero or more");
       const latest = yield* capture.latest;
       const known = viewport.knownViewport();
       const { activity } = page;
@@ -371,6 +373,14 @@ export const make = Effect.fnUntraced(function* (
     watch: bridge.currentDocument("screencast"),
     frameHistory: settings.frameHistory,
     viewport: Effect.suspend(() => viewport.viewportFor("screencast")),
+    // An uncropped snapshot waits for the restored compositor surface. Its pixels are discarded;
+    // a renderer animation frame or viewport size alone cannot prove the compositor caught up.
+    restore: () =>
+      page.protocol.send("Page.captureScreenshot", {
+        format: "jpeg",
+        quality: 0,
+        optimizeForSpeed: true,
+      }),
     imageSize: jpegSize,
     error,
   });
@@ -468,10 +478,7 @@ export const make = Effect.fnUntraced(function* (
           const size = viewport.knownViewport() ?? (yield* viewport.viewportFor("zoom"));
 
           if (region.x + region.width > size.width || region.y + region.height > size.height)
-            return yield* failWith(
-              "zoom",
-              new InvalidRequest({ detail: "the crop must fit entirely within the viewport" }),
-            );
+            return yield* refuse("zoom", "the crop must fit entirely within the viewport");
           const image = yield* picture("zoom", 80, region);
 
           return new Zoom({ page: id, region, image });
@@ -485,10 +492,7 @@ export const make = Effect.fnUntraced(function* (
 
     return millis > 0 && millis <= Duration.toMillis(Capture.countsKept)
       ? capture.stats(millis)
-      : failWith(
-          "captureStats",
-          new InvalidRequest({ detail: "window must be over zero and at most a minute" }),
-        );
+      : refuse("captureStats", "window must be over zero and at most a minute");
   };
 
   // The browser's clock mapping, measured on this page only while the browser has none, as the

package/src/internal/pictures/utilsBundle.d.ts

@@ -0,0 +1,9 @@
+// What `motion.ts` uses of playwright-core's bundled JPEG decoder, which its untyped entry exports.
+declare module "playwright-core/lib/utilsBundle" {
+  export const jpegjs: {
+    decode(
+      data: Uint8Array,
+      options: { useTArray: true },
+    ): { readonly width: number; readonly height: number; readonly data: Uint8Array };
+  };
+}

package/src/internal/reading/reading.ts

@@ -5,11 +5,11 @@
  */
 import { Duration, Effect, Ref, Schema } from "effect";
 
-import { InvalidRequest, StaleRef, Timeout } from "../../BrowserError.ts";
+import { StaleRef, Timeout } from "../../BrowserError.ts";
 import { type FindQuery, Found, Text, type TextOptions, type WaitCondition } from "../../Page.ts";
 import { Snapshot, type SnapshotOptions } from "../../Snapshot.ts";
 import { type Bridge, scriptCall } from "../page/bridge.ts";
-import { contextGone, decodeWith, failWith, type PageContext } from "../page/context.ts";
+import { contextGone, decodeWith, failWith, type PageContext, refuse } from "../page/context.ts";
 import * as Url from "../page/url.ts";
 import { type ConditionRequest, ConditionResultSchema } from "./controls.inpage.ts";
 import type { Want } from "./expect.inpage.ts";
@@ -68,8 +68,7 @@ export const make = Effect.fnUntraced(function* (page: PageContext, bridge: Brid
           Effect.flatMap(decodeWith("snapshot", SnapshotResultSchema)),
         );
 
-        if ("invalid" in result)
-          return yield* failWith("snapshot", new InvalidRequest({ detail: result.invalid }));
+        if ("invalid" in result) return yield* refuse("snapshot", result.invalid);
         const { text, truncated, url } = result.snapshot;
 
         yield* counted(result.nextRef);
@@ -115,10 +114,7 @@ export const make = Effect.fnUntraced(function* (page: PageContext, bridge: Brid
 
     return (
       scope !== "viewport" && !/^e\d+$/.test(scope)
-        ? failWith(
-            "text",
-            new InvalidRequest({ detail: `${scope} is neither "viewport" nor a ref such as e12` }),
-          )
+        ? refuse("text", `${scope} is neither "viewport" nor a ref such as e12`)
         : texts(
             JSON.stringify(request),
             Effect.gen(function* () {
@@ -159,7 +155,7 @@ export const make = Effect.fnUntraced(function* (page: PageContext, bridge: Brid
       Effect.flatMap(decodeWith("waitFor", ConditionResultSchema)),
       Effect.flatMap((result) =>
         "invalid" in result
-          ? failWith("waitFor", new InvalidRequest({ detail: result.invalid }))
+          ? refuse("waitFor", result.invalid)
           : result.met
             ? Effect.void
             : unmet(),

package/src/internal/reading/ready.ts

@@ -10,16 +10,27 @@
  * its connection, and a frame that arrives first starts the spell again. Where the page's changes
  * are recorded, its evidence includes them: nothing in view may have changed for the spell, which a
  * DOM page's changes show better than its paint, and no stall on the way can hide. Content
- * stillness starts that record itself and ignores capture silence. Each new document starts a
+ * stillness starts that record itself and ignores capture silence. Foreground stillness lets
+ * frames keep coming but compares them, so that only motion outside what was moving before the
+ * page's latest input counts; it still waits out unanswered acknowledgements while no frame
+ * comes, and its last look follows the round trip too. Each new document starts a
  * new observation, while retries of destroyed script contexts share the caller's deadline.
  */
 import { Duration, Effect, Option, Schedule, Stream } from "effect";
 
-import { Timeout } from "../../BrowserError.ts";
+import { Failed, Timeout } from "../../BrowserError.ts";
+import type { Frame } from "../../Frame.ts";
 import type { ReadyOptions } from "../../Page.ts";
 import { type Bridge, scriptCall } from "../page/bridge.ts";
-import { contextGone, decodeWith, failWith, type PageContext } from "../page/context.ts";
+import {
+  contextGone,
+  decodeWith,
+  failWith,
+  type PageContext,
+  undispatched,
+} from "../page/context.ts";
 import type * as Capture from "../pictures/capture.ts";
+import * as Motion from "../pictures/motion.ts";
 import { ReadinessSchema } from "./ready.inpage.ts";
 
 export const make =
@@ -55,27 +66,68 @@ export const make =
     const quiet = (millis: number, until: number) =>
       Effect.gen(function* () {
         const running = yield* capture.active;
-        let { received } = yield* capture.stats();
+        const foreground = options.still === "foreground";
+        let counts = yield* capture.stats();
         let stirred = running ? page.now() : Number.POSITIVE_INFINITY;
+        let reference: Motion.Picture | undefined;
 
-        // How long the screen has been still: since a frame, or an unanswered acknowledgement.
+        const decoded = (frame: Frame) =>
+          Effect.try({
+            try: () => Motion.picture(frame),
+            catch: (error) =>
+              undispatched("ready", new Failed({ detail: `no picture: ${String(error)}` })),
+          });
+
+        // The background: what moved in up to eight frames of the document's 1.5 s before its
+        // latest input, as far back as the capture keeps them.
+        const { inputAt } = page.activity;
+        const document = Option.getOrUndefined(yield* capture.latest)?.document;
+
+        const before = (foreground ? yield* capture.recent : []).filter(
+          (frame) =>
+            frame.hostTime < inputAt &&
+            frame.hostTime >= inputAt - 1500 &&
+            frame.document === document,
+        );
+
+        const sampled = before.filter((_, index) => index % Math.ceil(before.length / 8) === 0);
+        const mask = Motion.background(yield* Effect.forEach(sampled, decoded));
+
+        // How long the screen has been still: since a frame, or an unanswered acknowledgement. In
+        // the foreground, since a frame moved outside the background, or acknowledgements went
+        // unanswered with no frame coming.
         const look = Effect.gen(function* () {
           const stats = yield* capture.stats();
+          const latest = Option.getOrUndefined(yield* capture.latest);
+          const arrived = stats.received !== counts.received;
+          const unseen = stats.accepted - counts.accepted;
 
-          if (
-            !Option.exists(yield* capture.latest, capture.current) ||
-            stats.received !== received ||
-            stats.ackBacklog > 0
-          ) {
-            received = stats.received;
+          counts = stats;
+          if (latest === undefined || !capture.current(latest)) stirred = page.now();
+          else if (!foreground ? arrived || stats.ackBacklog > 0 : !arrived && stats.ackBacklog > 0)
             stirred = page.now();
+          else if (foreground && (unseen > 0 || reference === undefined)) {
+            // A move and its return between looks both count. Bound decoding; any frames lost
+            // from history, or beyond this batch, deny a quiet spell rather than hiding motion.
+            const frames = (yield* capture.recent).slice(-Math.max(1, Math.min(8, unseen)));
+
+            if (unseen > frames.length) stirred = page.now();
+            for (const frame of frames) {
+              const seen = yield* decoded(frame);
+
+              if (reference === undefined || Motion.stirs(reference, seen, mask)) {
+                reference = seen;
+                stirred = page.now();
+              }
+            }
           }
 
           return page.now() - stirred;
         });
 
+        // Decode at a fixed cadence: adding a delay after a full batch lets its backlog grow.
         const spell = Effect.repeat(look, {
-          schedule: Schedule.spaced(Duration.millis(25)),
+          schedule: Schedule.fixed(Duration.millis(foreground ? 100 : 25)),
           until: (still) => still >= millis,
         });
 
@@ -86,10 +138,10 @@ export const make =
         const still = Effect.gen(function* () {
           for (;;) {
             yield* spell;
-            const before = received;
+            const before = counts.received;
 
             yield* Effect.all([settled(until), capture.sync], { concurrency: 2, discard: true });
-            if ((yield* look) >= millis && received === before) return;
+            if ((yield* look) >= millis && (foreground || counts.received === before)) return;
           }
         });
 

package/src/internal/timeline/events.ts

@@ -4,9 +4,9 @@
  */
 import { Cause, Deferred, Effect, Stream } from "effect";
 
-import { type BrowserError, EventHistoryExpired, InvalidRequest } from "../../BrowserError.ts";
+import { type BrowserError, EventHistoryExpired } from "../../BrowserError.ts";
 import { type BrowserEvent, RecordedEvent } from "../../BrowserEvent.ts";
-import { undispatched } from "../page/context.ts";
+import { refuse, undispatched } from "../page/context.ts";
 
 /** The owning Browser validates capacity and closes this timeline with its scope. */
 export const make = (capacity: number) => {
@@ -41,13 +41,10 @@ export const make = (capacity: number) => {
         let cursor = after ?? sequence;
 
         if (!Number.isSafeInteger(cursor) || cursor < 0 || cursor > sequence)
-          return Stream.fail(
-            undispatched(
+          return Stream.fromEffect(
+            refuse(
               "events",
-              new InvalidRequest({
-                detail:
-                  "the event cursor must be a nonnegative integer no greater than the latest sequence",
-              }),
+              "the event cursor must be a nonnegative integer no greater than the latest sequence",
             ),
           );
 

package/src/internal/timeline/window.ts

@@ -5,13 +5,13 @@
  */
 import { Duration, Effect, Option, Result, Schema } from "effect";
 
-import { type BrowserError, InvalidRequest } from "../../BrowserError.ts";
+import type { BrowserError } from "../../BrowserError.ts";
 import { type BrowserEvent, TrackEvent } from "../../BrowserEvent.ts";
 import type { Changes } from "../../Change.ts";
 import { Frame } from "../../Frame.ts";
 import { Window } from "../../Moment.ts";
 import type { ChangesOptions, WindowOptions } from "../../Page.ts";
-import { failWith, type PageContext } from "../page/context.ts";
+import { type PageContext, refuse } from "../page/context.ts";
 
 const isWindow = Schema.is(Window);
 const isFrame = Schema.is(Frame);
@@ -64,12 +64,7 @@ export const make =
       const asked = boundsOf({ since, until }, page.now());
 
       if (Option.isNone(asked))
-        return yield* failWith(
-          "window",
-          new InvalidRequest({
-            detail: "a window's bounds must be finite, its reach not negative",
-          }),
-        );
+        return yield* refuse("window", "a window's bounds must be finite, its reach not negative");
       const bounds = asked.value;
 
       // A previous window's changes go on exactly where they ended, on the page's own clock, and a