effect-browser 0.2.0-beta.9 → 0.3.0-beta.5

package/src/Presentation.ts

@@ -0,0 +1,195 @@
+/**
+ * Input performed for viewers. A presenter owns one drawn pointer and a pace; `view(page)` is the
+ * page with its actions performed: each waits as long as a person reacts, the pointer glides from
+ * where viewers last saw it, on whichever page, a field is clicked before typing, keys go at the
+ * pace's words a minute and the wheel turns in bursts, out to a target out of view. One view acts at
+ * a time, as one hand moves one pointer, and its presentation time is outside `actionTimeout`. The
+ * page itself stays plain, and plain input never waits for a view.
+ *
+ * ```ts
+ * const presenter = yield* Presentation.make({ motion: yield* HumanStrokes.motion })
+ * const performed = presenter.view(page)
+ * yield* performed.aim("e12")    // as the target appears in a streamed tool call
+ * yield* performed.click("e12")  // completes the glide, then clicks
+ * ```
+ *
+ * @since 0.3.0
+ */
+import { Clock, Duration, Effect, Fiber, MutableRef, Option, Semaphore } from "effect";
+
+import type { BrowserError } from "./BrowserError.ts";
+import * as Style from "./internal/input/style.ts";
+import { type Internals, internalsOf, viewOf } from "./internal/page/page.ts";
+import * as Motion from "./Motion.ts";
+import type { Page, Point, Target } from "./Page.ts";
+
+/** How long a person waits before acting, and how fast they type. */
+export interface Pacing {
+  /** The usual wait before an action that follows an expected change, as the next field of a form. */
+  readonly expected: Duration.Input;
+  /** The usual wait after the page moved to a new document. */
+  readonly surprise: Duration.Input;
+  /** The usual wait before a page's first action, or one after an action on another page. */
+  readonly unrelated: Duration.Input;
+  /** Typing speed, a word being five characters. */
+  readonly wordsPerMinute: number;
+}
+
+/**
+ * The research's medians: 280 ms to react to an expected change, 600 ms to a surprise and 1 second
+ * between unrelated actions, each drawn around its median, and typing at 70 words per minute.
+ */
+export const human: Pacing = {
+  expected: "280 millis",
+  surprise: "600 millis",
+  unrelated: "1 second",
+  wordsPerMinute: 70,
+};
+
+export interface Options {
+  /** Defaults to `human`. */
+  readonly pacing?: Pacing | undefined;
+  /** The pointer's paths. Defaults to `Motion.lognormal`. */
+  readonly motion?: Motion.Service | undefined;
+}
+
+/** A page whose actions are performed, which can also start the pointer toward a target early. */
+export interface View extends Page {
+  /**
+   * Start the glide toward `target` as soon as it is known, as in a model's streamed tool call. The
+   * next action waits for it when it acts on the same target, and otherwise stops it where it is.
+   * An aim presses nothing and records no action; under an input guard it does nothing.
+   */
+  readonly aim: (target: Target) => Effect.Effect<void>;
+}
+
+export interface Presenter {
+  /** `page`, a page a `Browser` opened, with its actions performed: the same view for the same page. */
+  readonly view: (page: Page) => View;
+}
+
+// Reactions spread as the research measured them: an expected change least.
+const spread = { expected: 0.3, surprise: 0.4, unrelated: 0.4 };
+
+const sameTarget = (left: Target, right: Target | undefined) =>
+  typeof left === "string" || typeof right !== "object"
+    ? left === right
+    : left.x === right.x && left.y === right.y;
+
+/** A presenter, for the scope: its pointer, its pace and the aims it has started. */
+export const make = Effect.fn("Presentation.make")(function* (options: Options = {}) {
+  const pacing = options.pacing ?? human;
+  const clock = yield* Clock.Clock;
+  const scope = yield* Effect.scope;
+  const now = () => Number(clock.monotonicTimeNanosUnsafe()) / 1e6;
+  const pointer = MutableRef.make(Option.none<Point>());
+  const hand = yield* Semaphore.make(1);
+
+  const style = Style.performed({
+    pointer,
+    motion: options.motion ?? Motion.lognormal,
+    wordsPerMinute: pacing.wordsPerMinute,
+  });
+
+  const medians = {
+    expected: Duration.toMillis(Duration.fromInputUnsafe(pacing.expected)),
+    surprise: Duration.toMillis(Duration.fromInputUnsafe(pacing.surprise)),
+    unrelated: Duration.toMillis(Duration.fromInputUnsafe(pacing.unrelated)),
+  };
+
+  // The latest performed action: its page, the document it began in, and when it ended.
+  let last: { readonly page: string; readonly document: number; readonly at: number } | undefined;
+
+  // A person reacts to what the last action left: an expected change, a new document, as a link
+  // opens, or another page altogether. Time already spent since it ended counts toward the wait.
+  const react = (page: Page, internals: Internals) =>
+    Effect.gen(function* () {
+      const kind =
+        last === undefined || last.page !== page.id
+          ? "unrelated"
+          : last.document === internals.document()
+            ? "expected"
+            : "surprise";
+
+      const left = (last?.at ?? now()) + (yield* Style.pause(medians[kind], spread[kind])) - now();
+
+      if (left > 0) yield* Effect.sleep(Duration.millis(left));
+    }).pipe(Effect.provideService(Clock.Clock, clock));
+
+  const views = new WeakMap<Page, View>();
+
+  const view = (page: Page): View => {
+    const known = views.get(page);
+
+    if (known !== undefined) return known;
+    const internals = internalsOf(page);
+
+    if (internals === undefined)
+      throw new TypeError("only a page that a Browser opened can be presented");
+    const { input } = internals;
+
+    let aiming:
+      | { readonly target: Target; readonly fiber: Fiber.Fiber<void, BrowserError> }
+      | undefined;
+
+    // An aim at the action's own target is completed; any other is stopped where it is.
+    const completeAim = (target: Target | undefined) =>
+      Effect.suspend(() => {
+        const current = aiming;
+
+        aiming = undefined;
+        if (current === undefined) return Effect.void;
+
+        return sameTarget(current.target, target)
+          ? Effect.asVoid(Fiber.await(current.fiber))
+          : Fiber.interrupt(current.fiber);
+      });
+
+    const perform =
+      (target: Target | undefined) =>
+      <A, E>(action: Effect.Effect<A, E>): Effect.Effect<A, E> =>
+        completeAim(target).pipe(
+          Effect.andThen(
+            hand.withPermits(1)(
+              Effect.gen(function* () {
+                yield* react(page, internals);
+                const document = internals.document();
+
+                return yield* action.pipe(
+                  Effect.ensuring(
+                    Effect.sync(() => {
+                      last = { page: page.id, document, at: now() };
+                    }),
+                  ),
+                );
+              }),
+            ),
+          ),
+        );
+
+    const performed: View = viewOf(page, {
+      ...page,
+      click: (target, clickOptions) => perform(target)(input.click(target, clickOptions, style)),
+      hover: (target) => perform(target)(input.hover(target, style)),
+      drag: (from, to) => perform(from)(input.drag(from, to, style)),
+      type: (text, typeOptions) => perform(typeOptions?.into)(input.type(text, typeOptions, style)),
+      press: (keys, pressOptions) => perform(undefined)(input.press(keys, pressOptions, style)),
+      scroll: (scrollOptions) => perform(scrollOptions?.at)(input.scroll(scrollOptions, style)),
+      select: (ref, values) => perform(ref)(input.select(ref, values)),
+      aim: (target) =>
+        Effect.gen(function* () {
+          yield* completeAim(undefined);
+          aiming = {
+            target,
+            fiber: yield* hand.withPermits(1)(input.aim(target, style)).pipe(Effect.forkIn(scope)),
+          };
+        }),
+    });
+
+    views.set(page, performed);
+
+    return performed;
+  };
+
+  return { view } satisfies Presenter;
+});

package/src/Snapshot.ts

@@ -0,0 +1,89 @@
+/**
+ * A compact, model-readable outline of what a page shows.
+ *
+ * Each control gets a ref such as `e12`. A ref stays valid while its element is on the page, and
+ * refs are never reused within a page, so a ref from an old snapshot fails as stale rather than
+ * naming a different element.
+ *
+ * @since 0.3.0
+ */
+import { Schema } from "effect";
+
+/** A control the outline shows, as a value: its ref, what it is and what it holds. */
+export class Control extends Schema.Class<Control>("effect-browser/Control")({
+  ref: Schema.String,
+  /** Its role, such as `button` or `textbox`, or `clickable` for an element without one. */
+  kind: Schema.String,
+  /** Its accessible name. */
+  name: Schema.String,
+  /** What it holds: a field's text, `••••` for a secret one, or a select's chosen labels. */
+  value: Schema.String,
+  /** A select's option labels, at most 64. */
+  options: Schema.optional(Schema.Array(Schema.String)),
+  /** How many options a select has, when it has options. */
+  optionCount: Schema.optional(Schema.Int),
+  disabled: Schema.optional(Schema.Boolean),
+  /** Whether it is checked, for what can be checked. */
+  checked: Schema.optional(Schema.Boolean),
+  /** True for a field that takes typed text. */
+  editable: Schema.optional(Schema.Boolean),
+}) {}
+
+export class Snapshot extends Schema.Class<Snapshot>("effect-browser/Snapshot")({
+  url: Schema.String,
+  title: Schema.String,
+  /** One line per control, heading or text block, indented by landmark. */
+  text: Schema.String,
+  /** True when `text` was cut at `maxChars`. */
+  truncated: Schema.Boolean,
+  /**
+   * Parts of the page left out because they lie above or below the viewport, each an element out
+   * of view with all it holds (viewport snapshots only).
+   */
+  above: Schema.Finite,
+  below: Schema.Finite,
+  viewport: Schema.Struct({ width: Schema.Finite, height: Schema.Finite }),
+  scroll: Schema.Struct({ y: Schema.Finite, height: Schema.Finite }),
+  /** The controls the outline shows, in its order. */
+  controls: Schema.Array(Control),
+}) {
+  /** The snapshot as one block of text for a model: header, outline and what was left out. */
+  get rendered(): string {
+    const position =
+      this.scroll.height > this.viewport.height
+        ? `, scrolled to ${this.scroll.y} of ${this.scroll.height - this.viewport.height}px`
+        : "";
+
+    const omitted: Array<string> = [];
+
+    if (this.above > 0) omitted.push(`${this.above} parts above`);
+    if (this.below > 0) omitted.push(`${this.below} parts below`);
+
+    const footer = [
+      ...(omitted.length > 0 ? [`(${omitted.join(", ")} the viewport)`] : []),
+      ...(this.truncated ? ["(cut short: narrow it with query)"] : []),
+    ];
+
+    return [
+      `Page: ${this.title === "" ? "(untitled)" : this.title}`,
+      `URL: ${this.url}`,
+      `Viewport: ${this.viewport.width}x${this.viewport.height}${position}`,
+      this.text === "" ? "(nothing readable in view)" : this.text,
+      ...footer,
+    ].join("\n");
+  }
+}
+
+export interface SnapshotOptions {
+  /** Read the whole page instead of the viewport. Defaults to false. */
+  readonly full?: boolean | undefined;
+  /** Keep only lines containing this text (case-insensitive). */
+  readonly query?: string | undefined;
+  /**
+   * Read only inside the elements this CSS selector matches, such as `form` or `#results`. A
+   * selector that matches nothing reads nothing; one that is not CSS is an `InvalidRequest`.
+   */
+  readonly within?: string | undefined;
+  /** Bound on the outline's length. Defaults to 12,000 characters. */
+  readonly maxChars?: number | undefined;
+}

package/src/Stage.ts

@@ -0,0 +1,223 @@
+/**
+ * The stage: whichever page's capture feeds a live output, switched between pages in one browser
+ * session or across two, each switch stamped with when it took effect on the frame clock. Make one
+ * per live output.
+ *
+ * `present(page, { at })` switches the output to `page` at `at`, host monotonic milliseconds on the
+ * clock that times frames, or at once. It starts the new page's capture ahead, waits for its first
+ * frame, then for any input already under way on the old page, and turns: from then on `frames`
+ * carries the new page's frames, its newest first, and the old page's capture stops. The captures
+ * overlap, so a switch shows no dark spell; on Browserbase two ran in one session at the frame
+ * rate of one. A first frame that does not come in time fails `present`, and the old page stays. A
+ * capture that fails restarts on the same page while its page and browser stand.
+ *
+ * What a live output does with frames stays the application's: a delay line, a liveness rule,
+ * redaction and repeating a held frame. A still page sends no frames until it changes, so the frame
+ * a switch begins with may be older than the frames before it; `Presented.at` says when it took
+ * effect.
+ *
+ * @since 0.3.0
+ */
+import {
+  Clock,
+  Deferred,
+  Duration,
+  Effect,
+  Exit,
+  Fiber,
+  Option,
+  PubSub,
+  Schedule,
+  Schema,
+  Semaphore,
+  Stream,
+} from "effect";
+
+import { type BrowserError, consequence, InvalidRequest, Timeout } from "./BrowserError.ts";
+import type { CaptureStats, Frame, ScreencastOptions } from "./Frame.ts";
+import { failWith } from "./internal/page/context.ts";
+import { internalsOf } from "./internal/page/page.ts";
+import type { Page } from "./Page.ts";
+
+export interface Options extends ScreencastOptions {
+  /** How long a switch waits for the new page's first frame. Defaults to 5 seconds. */
+  readonly firstFrame?: Duration.Input | undefined;
+}
+
+/** A switch that took effect. */
+export class Presented extends Schema.Class<Presented>("effect-browser/Presented")({
+  page: Schema.String,
+  /**
+   * When it took effect, in host monotonic milliseconds on the frame clock: no earlier than the
+   * time asked for, the frames it followed or the new page's first.
+   */
+  at: Schema.Finite,
+  /** How long after the time asked for it took effect. */
+  latency: Schema.Finite,
+}) {}
+
+export interface Stage {
+  readonly present: (
+    page: Page,
+    options?: { readonly at?: number | undefined },
+  ) => Effect.Effect<Presented, BrowserError>;
+  /**
+   * The presented page's frames, each naming its page, session, document and address. A reader
+   * more than 16 frames behind loses the oldest.
+   */
+  readonly frames: Stream.Stream<Frame>;
+  readonly current: Effect.Effect<Option.Option<Page>>;
+  /**
+   * The presented page's capture counts over the latest `window`, at most a minute: its frames,
+   * their largest gap, and those filtered out, as another capture's or a crop's.
+   */
+  readonly stats: (options?: {
+    readonly window?: Duration.Input | undefined;
+  }) => Effect.Effect<Option.Option<CaptureStats>, BrowserError>;
+}
+
+/** A page's capture for the stage: its newest frame, and whether its frames go out. */
+interface Feed {
+  readonly page: Page;
+  readonly first: Deferred.Deferred<void>;
+  latest: Frame | undefined;
+  live: boolean;
+  fiber: Fiber.Fiber<void, BrowserError> | undefined;
+}
+
+// A capture restarts unless its page or browser is gone, or it would fail the same way again.
+const restartable = (error: BrowserError) => {
+  const { lost, repeat } = consequence(error);
+
+  return lost === "nothing" && repeat !== "pointless";
+};
+
+export const make = Effect.fn("Stage.make")(function* (options: Options = {}) {
+  const clock = yield* Clock.Clock;
+  const scope = yield* Effect.scope;
+  const now = () => Number(clock.monotonicTimeNanosUnsafe()) / 1e6;
+  const waited = Duration.fromInput(options.firstFrame ?? Duration.seconds(5));
+
+  if (Option.isNone(waited) || !Duration.isPositive(waited.value))
+    return yield* failWith(
+      "stage",
+      new InvalidRequest({ detail: "firstFrame must be a positive duration" }),
+    );
+  const firstFrame = waited.value;
+  const settings = { quality: options.quality, size: options.size };
+  const output = yield* PubSub.sliding<Frame>(16);
+  const switching = yield* Semaphore.make(1);
+  // What is on the stage, which only `present` changes, and the paint of the latest frame out.
+  let shown: { readonly feed: Feed; readonly presented: Presented } | undefined;
+  let painted = Number.NEGATIVE_INFINITY;
+
+  const feed = (page: Page) =>
+    Effect.gen(function* () {
+      const fed: Feed = {
+        page,
+        first: yield* Deferred.make<void>(),
+        latest: undefined,
+        live: false,
+        fiber: undefined,
+      };
+
+      fed.fiber = yield* page.screencast(settings).pipe(
+        Stream.runForEach((frame) =>
+          Effect.sync(() => {
+            fed.latest = frame;
+            if (fed.live) {
+              painted = frame.hostTime;
+              PubSub.publishUnsafe(output, frame);
+            }
+            // Last: a switch waiting for this frame may turn, and send it, at once.
+            Deferred.doneUnsafe(fed.first, Exit.void);
+          }),
+        ),
+        Effect.retry({ while: restartable, schedule: Schedule.spaced(Duration.millis(500)) }),
+        Effect.forkIn(scope),
+      );
+
+      return fed;
+    });
+
+  // No frame of it goes out once this returns, and its capture stops.
+  const stop = (fed: Feed) =>
+    Effect.suspend(() => {
+      fed.live = false;
+
+      return fed.fiber === undefined ? Effect.void : Fiber.interrupt(fed.fiber);
+    });
+
+  // The new page's first frame; its capture's own failure, or `Timeout`.
+  const firstOf = (fed: Feed) =>
+    Effect.raceFirst(
+      Deferred.await(fed.first),
+      fed.fiber === undefined ? Effect.never : Effect.andThen(Fiber.join(fed.fiber), Effect.never),
+    ).pipe(
+      Effect.timeoutOrElse({
+        duration: firstFrame,
+        orElse: () => failWith("present", new Timeout({ millis: Duration.toMillis(firstFrame) })),
+      }),
+    );
+
+  // A timer can fire a fraction of a millisecond early, so the wait looks again.
+  const until = (at: number): Effect.Effect<void> =>
+    Effect.suspend(() =>
+      at > now()
+        ? Effect.andThen(Effect.sleep(Duration.millis(Math.ceil(at - now()))), until(at))
+        : Effect.void,
+    );
+
+  const present: Stage["present"] = (page, presentOptions = {}) =>
+    switching.withPermits(1)(
+      Effect.gen(function* () {
+        const asked = presentOptions.at ?? now();
+        const old = shown;
+
+        if (old?.feed.page === page) return old.presented;
+        const fed = yield* feed(page);
+
+        yield* Effect.gen(function* () {
+          yield* firstOf(fed);
+          yield* until(asked);
+          // Input under way on the old page ends first, within its action timeout.
+          if (old !== undefined)
+            yield* Effect.ignore(internalsOf(old.feed.page)?.inputEnded ?? Effect.void);
+        }).pipe(Effect.onError(() => stop(fed)));
+
+        const presented = yield* Effect.sync(() => {
+          const frame = fed.latest;
+          const at = Math.max(asked, frame?.hostTime ?? asked, painted);
+
+          if (old !== undefined) old.feed.live = false;
+          fed.live = true;
+          if (frame !== undefined) {
+            painted = frame.hostTime;
+            PubSub.publishUnsafe(output, frame);
+          }
+          shown = {
+            feed: fed,
+            presented: new Presented({ page: page.id, at, latency: at - asked }),
+          };
+
+          return shown.presented;
+        });
+
+        if (old !== undefined) yield* stop(old.feed);
+
+        return presented;
+      }).pipe(Effect.provideService(Clock.Clock, clock)),
+    );
+
+  return {
+    present,
+    frames: Stream.fromPubSub(output),
+    current: Effect.sync(() => Option.fromNullishOr(shown?.feed.page)),
+    stats: (statsOptions = {}) =>
+      shown === undefined
+        ? Effect.succeedNone
+        : Effect.asSome(
+            shown.feed.page.captureStats({ window: statsOptions.window ?? "1 second" }),
+          ),
+  } satisfies Stage;
+});