effect-browser 0.2.0-beta.9 → 0.3.0-beta.11

package/src/internal/reading/outline.inpage.ts

@@ -0,0 +1,260 @@
+/**
+ * In the page: the outline a model reads, with a ref for each control. See `names.inpage.ts` for
+ * what a page-side part may use.
+ */
+import { Schema } from "effect";
+
+import type { Control, Snapshot } from "../../Snapshot.ts";
+import type { addresses } from "../page/url.ts";
+import type { Controls } from "./controls.inpage.ts";
+import type { Names } from "./names.inpage.ts";
+import type { Subjects } from "./subjects.inpage.ts";
+import type { Texts } from "./text.inpage.ts";
+import type { Walk } from "./walk.inpage.ts";
+
+export interface SnapshotRequest {
+  readonly full: boolean;
+  readonly query: string | null;
+  /** A CSS selector: only what is inside the elements it matches is read. */
+  readonly within: string | null;
+  readonly maxChars: number;
+  readonly firstRef: number;
+}
+
+/**
+ * The outline as a `Snapshot` reads it, and the next ref the page may give, or why `within` is not
+ * a selector.
+ */
+export type SnapshotResult =
+  | { readonly snapshot: typeof Snapshot.Encoded; readonly nextRef: number }
+  | { readonly invalid: string };
+
+/** The rule every address the library reports goes by, which a link's address goes by too. */
+type Url = ReturnType<typeof addresses>;
+
+export const outline = (
+  names: Names,
+  walked: Walk,
+  subjects: Subjects,
+  texts: Texts,
+  controlling: Controls,
+  url: Url,
+) => {
+  const { clean, containers, isFrame, nameOf, refFor, refs, roleOf, textOf } = names;
+  const { visit } = walked;
+  const { isControl } = subjects;
+  const { cut } = texts;
+
+  const textBlocks =
+    /^(?:P|LI|TD|TH|DT|DD|LABEL|SPAN|BLOCKQUOTE|FIGCAPTION|CAPTION|PRE|STRONG|EM|B|I|SMALL|TIME|CODE|LEGEND)$/;
+
+  const nested =
+    "a[href],button,input,select,textarea,summary,[role],[onclick],[tabindex],[contenteditable],canvas,iframe";
+
+  const noise = /^[\s|•·\-–—,.:;()[\]{}"'/\\]*$/;
+
+  /** Text inside a control's label is already that control's name. */
+  const inControlLabel = (element: Element): boolean => {
+    const label = element.closest("label");
+
+    return label !== null && label.control !== null;
+  };
+
+  const hrefOf = (element: Element): string => {
+    if (element.tagName !== "A") return "";
+    const raw = element.getAttribute("href") ?? "";
+
+    if (raw === "" || raw.startsWith("javascript:")) return "";
+    try {
+      const { origin, href } = new URL(raw, element.ownerDocument.baseURI);
+      // Reported as every address is, before it is shortened, so no part of a credential shows.
+      const reported = url.redact(href);
+      const own = origin === location.origin && reported.startsWith(origin);
+
+      return ` -> ${clean(own ? reported.slice(origin.length) : reported, 80)}`;
+    } catch {
+      return "";
+    }
+  };
+
+  interface Place {
+    readonly depth: number;
+    readonly insidePointer: boolean;
+    /** Inside a control, only the controls in it are read. */
+    readonly controls: boolean;
+  }
+
+  const snapshot = (request: SnapshotRequest): SnapshotResult => {
+    const roots = request.within === null ? [null] : controlling.rootsOf(request.within);
+
+    if ("invalid" in roots) return roots;
+    if (refs.next < request.firstRef) refs.next = request.firstRef;
+    const width = window.innerWidth;
+    const height = window.innerHeight;
+    const query = request.query === null ? null : request.query.toLowerCase();
+    const lines: Array<string> = [];
+    const controls: Array<typeof Control.Encoded> = [];
+    let above = 0;
+    let below = 0;
+
+    const placement = (rect: DOMRect): "in" | "above" | "below" | "aside" => {
+      if (request.full) return "in";
+      if (rect.bottom <= 0) return "above";
+      if (rect.top >= height) return "below";
+      if (rect.right <= 0 || rect.left >= width) return "aside";
+
+      return "in";
+    };
+
+    const count = (rect: DOMRect) => {
+      const where = placement(rect);
+
+      if (where === "above") above++;
+      if (where === "below") below++;
+
+      return where;
+    };
+
+    const emit = (depth: number, line: string, rect: DOMRect): boolean => {
+      if (count(rect) !== "in") return false;
+      if (query !== null && !line.toLowerCase().includes(query)) return false;
+      lines.push(`${"  ".repeat(Math.min(depth, 8))}- ${line}`);
+
+      return true;
+    };
+
+    const control = (element: Element, rect: DOMRect, role: string | null, depth: number) => {
+      const name = nameOf(element, role);
+      const kind = role ?? "clickable";
+
+      const box =
+        role === "canvas" || role === "iframe"
+          ? ` ${Math.round(rect.width)}x${Math.round(rect.height)} at (${Math.round(rect.x)},${Math.round(rect.y)})`
+          : "";
+
+      const line =
+        placement(rect) === "in"
+          ? `${kind}${name === "" ? "" : ` ${JSON.stringify(name)}`} [ref=${refFor(element)}]${controlling.states(element, role)}${controlling.valueOf(element)}${hrefOf(element)}${box}`
+          : kind;
+
+      if (emit(depth, line, rect)) controls.push(controlling.controlOf(element, role, kind, name));
+      if (isFrame(element)) {
+        const body = element.contentDocument?.body;
+
+        if ((body === null || body === undefined) && placement(rect) === "in")
+          lines.push(
+            `${"  ".repeat(Math.min(depth + 1, 8))}- (another site's frame: use a screenshot and coordinates)`,
+          );
+
+        return { depth: depth + 1, insidePointer: false, controls: false };
+      }
+      if (["INPUT", "SELECT", "TEXTAREA", "CANVAS", "IMG"].includes(element.tagName))
+        return undefined;
+
+      return { depth: depth + 1, insidePointer: true, controls: true };
+    };
+
+    const enter = (
+      element: Element,
+      style: CSSStyleDeclaration,
+      rect: DOMRect,
+      { depth, insidePointer }: Place,
+    ): Place | undefined => {
+      const role = roleOf(element);
+
+      if (isControl(element, role, style, rect, insidePointer))
+        return control(element, rect, role, depth);
+      if (role === "heading") {
+        emit(
+          depth,
+          `heading ${JSON.stringify(nameOf(element, role))}${controlling.states(element, role)}`,
+          rect,
+        );
+
+        return undefined;
+      }
+      if (role === "img") {
+        const alt = nameOf(element, role);
+
+        if (alt !== "") emit(depth, `img ${JSON.stringify(alt)}`, rect);
+
+        return undefined;
+      }
+      let inner = depth;
+
+      if (role !== null && (containers[element.tagName] !== undefined || role === "dialog")) {
+        const label = element.getAttribute("aria-label");
+
+        if (
+          emit(depth, `${role}${label === null ? "" : ` ${JSON.stringify(clean(label))}`}:`, rect)
+        )
+          inner = depth + 1;
+      }
+
+      if (textBlocks.test(element.tagName) && element.querySelector(nested) === null) {
+        if (inControlLabel(element)) return undefined;
+        const text = clean(textOf(element), 300);
+
+        if (text !== "" && !noise.test(text)) emit(inner, `text: ${text}`, rect);
+
+        return undefined;
+      }
+
+      let own = "";
+
+      for (const node of Array.from(element.childNodes))
+        if (node.nodeType === Node.TEXT_NODE) own += node.textContent ?? "";
+      own = clean(own, 300);
+      if (own !== "" && !noise.test(own) && !inControlLabel(element))
+        emit(inner, `text: ${own}`, rect);
+
+      return {
+        depth: inner,
+        insidePointer: insidePointer || style.cursor === "pointer",
+        controls: false,
+      };
+    };
+
+    const visitor = {
+      enter,
+      admit: (child: Element, place: Place) =>
+        !place.controls || child.matches(nested) || child.querySelector(nested) !== null,
+      outside: count,
+    };
+
+    // Inside the viewport, a subtree out of view is skipped whole and counted as one part.
+    for (const root of roots)
+      visit(root, !request.full, { depth: 0, insidePointer: false, controls: false }, visitor);
+
+    const outlined = cut(lines.join("\n"), request.maxChars);
+
+    return {
+      snapshot: {
+        ...outlined,
+        url: location.href,
+        title: document.title,
+        above,
+        below,
+        viewport: { width, height },
+        scroll: {
+          y: Math.round(window.scrollY),
+          height: Math.round(document.documentElement.scrollHeight),
+        },
+        // Only the controls whose lines were kept.
+        controls: controls.filter((control) => outlined.text.includes(`[ref=${control.ref}]`)),
+      },
+      nextRef: refs.next,
+    };
+  };
+
+  const viewport = () => ({ width: window.innerWidth, height: window.innerHeight });
+
+  return { snapshot, viewport };
+};
+
+export type Outline = ReturnType<typeof outline>;
+
+export const ViewportResultSchema = Schema.Struct({
+  width: Schema.Finite,
+  height: Schema.Finite,
+});

package/src/internal/reading/reading.ts

@@ -0,0 +1,223 @@
+/**
+ * Reading a page: its outline with refs, the elements a query finds and the text it shows. Each
+ * read takes its turn on the page and shares its work with identical reads, as `lane.ts`
+ * describes. The viewport's text as it was last read is kept, for `Page.state`.
+ */
+import { Duration, Effect, Ref, Schema } from "effect";
+
+import { InvalidRequest, StaleRef, Timeout } from "../../BrowserError.ts";
+import { type FindQuery, Found, Text, type TextOptions, type WaitCondition } from "../../Page.ts";
+import { Snapshot, type SnapshotOptions } from "../../Snapshot.ts";
+import { type Bridge, scriptCall } from "../page/bridge.ts";
+import { contextGone, decodeWith, failWith, type PageContext } from "../page/context.ts";
+import * as Url from "../page/url.ts";
+import type { ConditionRequest } from "./controls.inpage.ts";
+import type { Want } from "./expect.inpage.ts";
+import type { FindRequest, Wanted } from "./match.inpage.ts";
+import type { SnapshotRequest } from "./outline.inpage.ts";
+import { type TextRequest, TextResultSchema } from "./text.inpage.ts";
+
+// What `snapshot` and `find` read back, each with the next ref the page may give, and why a
+// snapshot's selector is not CSS.
+const SnapshotResult = Schema.Union([
+  Schema.Struct({ snapshot: Snapshot, nextRef: Schema.Finite }),
+  Schema.Struct({ invalid: Schema.String }),
+]);
+
+const FindResults = Schema.Struct({ found: Schema.Array(Found), nextRef: Schema.Finite });
+
+const UntilResult = Schema.Struct({ met: Schema.Boolean, mark: Schema.String });
+
+const ConditionResult = Schema.Union([
+  Schema.Struct({ met: Schema.Boolean }),
+  Schema.Struct({ invalid: Schema.String }),
+]);
+
+// A pattern crosses into the page as its source and flags.
+const wanted = (value: string | RegExp | undefined): Wanted | null =>
+  value === undefined
+    ? null
+    : typeof value === "string"
+      ? value
+      : { source: value.source, flags: value.flags };
+
+/** A query as the page takes it, but for the first ref it may give. */
+const askedOf = (query: FindQuery, scope: "viewport" | "document") => ({
+  role: query.role ?? null,
+  name: wanted(query.name),
+  text: wanted(query.text),
+  near: query.near ?? null,
+  at: query.at ?? null,
+  scope: query.scope ?? scope,
+});
+
+export const make = Effect.fnUntraced(function* (page: PageContext, bridge: Bridge) {
+  const { now, span, owned, within, lane } = page;
+  const { evaluate } = bridge;
+  // Refs count up across the page's documents, so one never names two elements.
+  const nextRef = yield* Ref.make(1);
+  const counted = (next: number) => Ref.update(nextRef, (current) => Math.max(current, next));
+  const snapshots = lane.shared<Snapshot>("snapshot", true);
+  const finds = lane.shared<ReadonlyArray<Found>>("find", true);
+  const texts = lane.shared<Text>("text", true);
+  let viewed: Text | undefined;
+
+  const snapshot = (snapshotOptions: SnapshotOptions = {}) => {
+    const asked = {
+      full: snapshotOptions.full ?? false,
+      query: snapshotOptions.query ?? null,
+      within: snapshotOptions.within ?? null,
+      maxChars: snapshotOptions.maxChars ?? 12_000,
+    };
+
+    return snapshots(
+      JSON.stringify(asked),
+      Effect.gen(function* () {
+        const request: SnapshotRequest = { ...asked, firstRef: yield* Ref.get(nextRef) };
+
+        const result = yield* evaluate("snapshot", scriptCall("snapshot", request)).pipe(
+          Effect.flatMap(decodeWith("snapshot", SnapshotResult)),
+        );
+
+        if ("invalid" in result)
+          return yield* failWith("snapshot", new InvalidRequest({ detail: result.invalid }));
+        const { text, truncated, url } = result.snapshot;
+
+        yield* counted(result.nextRef);
+        yield* Effect.annotateCurrentSpan({ chars: text.length, truncated });
+
+        return new Snapshot({ ...result.snapshot, url: Url.redact(url) });
+      }).pipe(within("snapshot")),
+    ).pipe(span("Page.snapshot", { full: asked.full }), owned);
+  };
+
+  const find = (query: FindQuery = {}) => {
+    const asked = askedOf(query, "viewport");
+
+    return finds(
+      JSON.stringify(asked),
+      Effect.gen(function* () {
+        const request: FindRequest = { ...asked, firstRef: yield* Ref.get(nextRef) };
+
+        const result = yield* evaluate("find", scriptCall("find", request)).pipe(
+          Effect.flatMap(decodeWith("find", FindResults)),
+        );
+
+        yield* counted(result.nextRef);
+        yield* Effect.annotateCurrentSpan({ found: result.found.length });
+
+        return result.found;
+      }).pipe(within("find")),
+    ).pipe(
+      // The query's words are the caller's, and may be anything; only its shape is traced.
+      span("Page.find", { scope: asked.scope, role: asked.role ?? "" }),
+      owned,
+    );
+  };
+
+  const text = (options: TextOptions = {}) => {
+    const scope = options.scope ?? "viewport";
+
+    const request: TextRequest = {
+      ref: scope === "viewport" ? null : scope,
+      maxChars: options.maxChars ?? 12_000,
+      unmask: options.unmask ?? false,
+    };
+
+    return (
+      scope !== "viewport" && !/^e\d+$/.test(scope)
+        ? failWith(
+            "text",
+            new InvalidRequest({ detail: `${scope} is neither "viewport" nor a ref such as e12` }),
+          )
+        : texts(
+            JSON.stringify(request),
+            Effect.gen(function* () {
+              const result = yield* evaluate("text", scriptCall("text", request)).pipe(
+                Effect.flatMap(decodeWith("text", TextResultSchema)),
+              );
+
+              if (result === null) return yield* failWith("text", new StaleRef({ ref: scope }));
+              yield* Effect.annotateCurrentSpan({
+                chars: result.text.length,
+                truncated: result.truncated,
+              });
+              const shown = new Text({ ...result, url: Url.redact(result.url), at: now() });
+
+              if (request.ref === null) viewed = shown;
+
+              return shown;
+            }).pipe(within("text")),
+          )
+    ).pipe(span("Page.text", { scope: scope === "viewport" ? "viewport" : "ref" }), owned);
+  };
+
+  // One call, which the page answers once the condition holds or at the deadline. It follows the
+  // action in flight, as a read does, but holds no later one back while it waits, and it waits on
+  // in a document that replaces the one it began in.
+  const waitFor = (condition: WaitCondition, timeout: Duration.Input = Duration.seconds(10)) => {
+    const request: ConditionRequest = {
+      selector: condition.selector ?? null,
+      text: condition.text ?? null,
+      state: condition.state ?? "visible",
+    };
+
+    // JSON has no infinity; the deadline here ends a wait without one in any case.
+    const millis = Math.min(Duration.toMillis(timeout), Number.MAX_SAFE_INTEGER);
+    const unmet = () => failWith("waitFor", new Timeout({ millis }));
+
+    const look = evaluate("waitFor", scriptCall("waitUntil", request, millis)).pipe(
+      Effect.flatMap(decodeWith("waitFor", ConditionResult)),
+      Effect.flatMap((result) =>
+        "invalid" in result
+          ? failWith("waitFor", new InvalidRequest({ detail: result.invalid }))
+          : result.met
+            ? Effect.void
+            : unmet(),
+      ),
+    );
+
+    return lane
+      .read("waitFor")(Effect.void)
+      .pipe(
+        Effect.andThen(Effect.retry(look, { while: contextGone })),
+        Effect.timeoutOrElse({ duration: timeout, orElse: unmet }),
+        span("Page.waitFor", { state: request.state }),
+        owned,
+      );
+  };
+
+  // One call, which the page answers once `want` holds of what `query` matches, the whole document
+  // unless it says otherwise, or at the deadline, with the mark of what matched then. It follows
+  // the action in flight, as `waitFor` does, and the page has the action timeout past the deadline
+  // to answer.
+  const until = (query: FindQuery, want: Want, millis: number) => {
+    const look = Effect.gen(function* () {
+      const request: FindRequest = {
+        ...askedOf(query, "document"),
+        firstRef: yield* Ref.get(nextRef),
+      };
+
+      return yield* evaluate("expect", scriptCall("until", request, want, millis)).pipe(
+        Effect.flatMap(decodeWith("expect", UntilResult)),
+      );
+    });
+
+    return lane
+      .read("expect")(Effect.void)
+      .pipe(
+        Effect.andThen(Effect.retry(look, { while: contextGone })),
+        within("expect", Duration.sum(Duration.millis(millis), page.settings.actionTimeout)),
+        span("Page.expect", { want: typeof want === "string" ? want : "other" }),
+        owned,
+      );
+  };
+
+  /** The viewport's text as it was last read, if after `since`, as a document began. */
+  const viewedSince = (since: number) =>
+    viewed !== undefined && viewed.at > since ? viewed : undefined;
+
+  return { snapshot, find, text, waitFor, until, viewedSince };
+});
+
+export type Reading = Effect.Success<ReturnType<typeof make>>;

package/src/internal/reading/ready.inpage.ts

@@ -0,0 +1,164 @@
+/**
+ * In the page: whether it is ready to be shown, checked until it is or until a deadline. Its
+ * document is parsed and has painted since, nothing that ends is animating in view, its fonts and
+ * the images in view have loaded, nothing in view is marked busy (`aria-busy`), and the viewport
+ * shows something: text, a canvas drawn on, a picture or drawing larger than an icon, a video or a
+ * frame. So a canvas mounted blank, or a spinner alone, is still loading. A screen that only says
+ * "Loading…" in words reads as ready, and a WebGL canvas drawn once, without keeping its drawing,
+ * reads as blank. Asked for a quiet spell, where the page's changes are recorded, nothing in view
+ * may have changed for that long either. Content mode starts and refreshes the recorder and
+ * observes a full quiet interval from its first check in each document. After input, it waits a
+ * task and a frame. See `names.inpage.ts` for what a page-side part may use.
+ */
+import { Schema } from "effect";
+
+import type { History } from "../timeline/history.inpage.ts";
+import type { Recorder } from "../timeline/record.inpage.ts";
+import type { Texts } from "./text.inpage.ts";
+import type { Walk } from "./walk.inpage.ts";
+
+/** What the page is still waiting for; none when it is ready. */
+export const ReadinessSchema = Schema.Array(
+  Schema.Literals([
+    "load",
+    "paint",
+    "animations",
+    "images",
+    "fonts",
+    "busy",
+    "content",
+    "changing",
+  ]),
+);
+
+export type Readiness = typeof ReadinessSchema.Type;
+
+export const ready = (walked: Walk, texts: Texts, kept: History, recorder: Recorder) => {
+  const { inView, visible } = walked;
+  // A canvas is read through a small copy: reading it directly would give it a context of ours.
+  let copy: OffscreenCanvasRenderingContext2D | null | undefined;
+
+  const boxed = (element: Element): boolean => {
+    const rect = element.getBoundingClientRect();
+
+    return rect.width * rect.height > 0 && inView(rect);
+  };
+
+  const showing = <E extends Element>(elements: Iterable<E>): Array<E> =>
+    Array.from(elements).filter((element) => boxed(element) && visible(element));
+
+  // A canvas mounted blank is transparent all over.
+  const drawn = (canvas: HTMLCanvasElement): boolean => {
+    copy ??= new OffscreenCanvas(32, 32).getContext("2d", { willReadFrequently: true });
+    if (copy === null) return true;
+    try {
+      copy.clearRect(0, 0, 32, 32);
+      copy.imageSmoothingQuality = "high";
+      copy.drawImage(canvas, 0, 0, 32, 32);
+
+      return new Uint32Array(copy.getImageData(0, 0, 32, 32).data.buffer).some(
+        (pixel) => pixel !== 0,
+      );
+    } catch {
+      // Another site's pictures in it keep it from being read, and it has drawn them.
+      return true;
+    }
+  };
+
+  // A picture the size of an icon, such as a spinner, says nothing has come yet. Text costs a
+  // walk of the viewport, and a canvas a copy of it, so each is read only if needed.
+  const content = (): boolean =>
+    showing(document.querySelectorAll("video, iframe, embed, object")).length > 0 ||
+    showing(document.querySelectorAll("img, svg")).some((picture) => {
+      const { width, height } = picture.getBoundingClientRect();
+
+      return width > 64 && height > 64;
+    }) ||
+    (texts.read({ ref: null, maxChars: 1, unmask: false })?.text ?? "") !== "" ||
+    showing(document.querySelectorAll("canvas")).some(drawn);
+
+  // A hidden document paints no frames, so it is not ready to be shown.
+  const painted = (): Promise<boolean> => {
+    const { promise, resolve } = Promise.withResolvers<boolean>();
+
+    if (document.visibilityState === "hidden") resolve(false);
+    else requestAnimationFrame(() => resolve(true));
+
+    return promise;
+  };
+
+  // In a frame, before it is drawn: a canvas drawn every frame still holds its drawing then.
+  const check = (quietMillis: number, observed: number): Promise<Readiness> =>
+    painted().then((paints) => {
+      const waiting: Array<Readiness[number]> = [];
+
+      if (document.readyState === "loading") waiting.push("load");
+      if (!paints) waiting.push("paint");
+      // An endless animation, such as a pulsing dot, never ends, so only those that end count.
+      if (
+        document.getAnimations().some((animation) => {
+          const effect = animation.effect;
+
+          return (
+            animation.playState === "running" &&
+            effect !== null &&
+            Number.isFinite(effect.getComputedTiming().endTime) &&
+            "target" in effect &&
+            effect.target instanceof Element &&
+            boxed(effect.target)
+          );
+        })
+      )
+        waiting.push("animations");
+      if (Array.from(document.images).some((image) => !image.complete && boxed(image)))
+        waiting.push("images");
+      if (document.fonts.status !== "loaded") waiting.push("fonts");
+      if (showing(document.querySelectorAll("[aria-busy=true]")).length > 0) waiting.push("busy");
+      if (!content()) waiting.push("content");
+      if (
+        Math.max(observed, kept.changing()) >
+        performance.timeOrigin + performance.now() - quietMillis
+      )
+        waiting.push("changing");
+
+      return waiting;
+    });
+
+  /**
+   * Check until the page is ready, or for `millis`, and say what it is still waiting for; with
+   * `quietMillis`, until nothing the record saw in view has changed for that long, too.
+   */
+  const wait = async (millis: number, quietMillis = 0, contentOnly = false): Promise<Readiness> => {
+    const until = performance.now() + millis;
+    let observed = contentOnly ? undefined : Number.NEGATIVE_INFINITY;
+
+    for (;;) {
+      if (contentOnly) {
+        recorder.start();
+        if (kept.begun()) observed ??= performance.timeOrigin + performance.now();
+      }
+      const waiting = await check(quietMillis, observed ?? Number.POSITIVE_INFINITY);
+
+      if (waiting.length === 0 || performance.now() >= until) return waiting;
+      const { promise, resolve } = Promise.withResolvers<void>();
+
+      setTimeout(resolve, Math.min(50, until - performance.now()));
+      await promise;
+    }
+  };
+
+  /**
+   * One task, then a frame: what the page's handlers for input did at once has run, a navigation
+   * one of them started in a timer has been asked for, and their effect has painted. Says whether
+   * the page painted, which a hidden one does not.
+   */
+  const settle = (): Promise<boolean> => {
+    const { promise, resolve } = Promise.withResolvers<void>();
+
+    setTimeout(resolve, 0);
+
+    return promise.then(painted);
+  };
+
+  return { wait, settle };
+};