@mochi.js/core 0.1.2 → 0.3.0

package/src/page/selector.ts

@@ -0,0 +1,423 @@
+/**
+ * Tiny host-side CSS selector engine for the closed-shadow piercing locator
+ * (`Page.querySelectorPiercing`). Parses a CSS selector into a sequence of
+ * **compound** parts joined by descendant combinators, then matches a
+ * pre-walked `PierceDomNode` against that compound chain.
+ *
+ * Why we don't `DOM.querySelector` per shadow root: that CDP method does NOT
+ * pierce closed shadows even when its parent `DOM.getDocument` was called
+ * with `pierce: true`. Patchright's `_customFindElementsByParsed`
+ * (`framesPatch.ts:868-1012`) parses the selector itself and walks the tree
+ * manually for exactly this reason. We port the algorithm — *not* the surface
+ * area: only the CSS-selector subset listed in `tasks/0253` lands here.
+ *
+ * **Supported subset (CSS Selectors level 4 — strict subset):**
+ * - Tag selectors: `div`, `iframe`, `*`
+ * - ID: `#main`
+ * - Class: `.btn`, `.btn.primary`
+ * - Attribute: `[src]`, `[name="x"]`, `[href*="foo"]`, `[role^="b"]`,
+ *   `[data-x$="y"]`, `[data-x~="z"]`, `[data-x|="en"]`. Quotes optional for
+ *   value-less words.
+ * - Descendant combinator: `div .btn` (whitespace).
+ * - Comma-separated selector lists: `a, button` — match if ANY branch matches.
+ *
+ * **NOT supported (intentionally — see Out of scope in 0253):**
+ * - `>`, `+`, `~` combinators
+ * - `:pseudo-classes` (`:hover`, `:nth-child`, `:has`, `:not`)
+ * - `::pseudo-elements`
+ * - XPath (deferred — STRETCH per task brief; document as TODO if it lands).
+ * - Namespaces.
+ *
+ * Throws `SelectorParseError` on syntactically invalid input. The matcher
+ * itself never throws — unsupported nodes just don't match.
+ *
+ * @see tasks/0253-closed-shadow-piercing-locator.md
+ * @see PLAN.md §8.2 (forbidden CDP — neither `DOM.getDocument` nor
+ *   `DOM.resolveNode` is forbidden; both fine).
+ */
+
+import type { PierceDomNode } from "../cdp/types";
+
+/** Thrown when the selector has a syntax error we can't recover from. */
+export class SelectorParseError extends Error {
+  readonly selector: string;
+  constructor(selector: string, message: string) {
+    super(`[mochi] invalid selector "${selector}": ${message}`);
+    this.name = "SelectorParseError";
+    this.selector = selector;
+  }
+}
+
+/** A single attribute filter inside a compound part. */
+export interface AttrFilter {
+  name: string;
+  /**
+   * Matcher op:
+   * - `"exists"`: attribute is present (value ignored)
+   * - `"="`: exact value
+   * - `"~="`: whitespace-separated word match
+   * - `"|="`: exact OR `value-…` prefix
+   * - `"^="`: prefix match
+   * - `"$="`: suffix match
+   * - `"*="`: substring match
+   */
+  op: "exists" | "=" | "~=" | "|=" | "^=" | "$=" | "*=";
+  /** Match value (always present except for `op === "exists"`). */
+  value?: string;
+}
+
+/** A compound (no whitespace) selector — one element's worth of constraints. */
+export interface CompoundPart {
+  /** Lower-case tag, or `"*"` for the universal selector. */
+  tag: string;
+  id?: string;
+  classes: string[];
+  attrs: AttrFilter[];
+}
+
+/**
+ * A single descendant chain (one comma-separated branch). Matching iterates
+ * the chain right-to-left: the rightmost part must match the candidate; each
+ * earlier part must have a matching ancestor (DOM-ancestor-aware, including
+ * across shadow boundaries — see `matchSelector` for the walk).
+ */
+export interface CompoundChain {
+  parts: CompoundPart[];
+}
+
+/** A parsed selector list — `,`-separated chains. */
+export interface ParsedSelector {
+  chains: CompoundChain[];
+}
+
+// ---- parser ----------------------------------------------------------------
+
+/**
+ * Parse a CSS selector string into a {@link ParsedSelector}. Throws
+ * {@link SelectorParseError} on bad input.
+ *
+ * The grammar we accept is a strict subset documented at the top of this
+ * module. We deliberately do not use a regex-driven parser — those struggle
+ * with quoted attribute values that contain `[`, `,`, or whitespace.
+ */
+export function parseSelector(input: string): ParsedSelector {
+  if (typeof input !== "string") {
+    throw new SelectorParseError(String(input), "selector must be a string");
+  }
+  const trimmed = input.trim();
+  if (trimmed.length === 0) {
+    throw new SelectorParseError(input, "selector must not be empty");
+  }
+  const branches = splitTopLevel(trimmed, ",");
+  const chains: CompoundChain[] = [];
+  for (const branch of branches) {
+    const parts = splitTopLevel(branch.trim(), " ").filter((p) => p.length > 0);
+    if (parts.length === 0) {
+      throw new SelectorParseError(input, "empty selector branch");
+    }
+    chains.push({ parts: parts.map((p) => parseCompound(p, input)) });
+  }
+  return { chains };
+}
+
+/**
+ * Split a selector string at top-level occurrences of `sep` — i.e. ignoring
+ * separators inside `[...]` brackets or quoted attribute values.
+ */
+function splitTopLevel(input: string, sep: string): string[] {
+  const out: string[] = [];
+  let buf = "";
+  let depth = 0;
+  let quote: '"' | "'" | null = null;
+  for (let i = 0; i < input.length; i++) {
+    const ch = input[i] as string;
+    if (quote !== null) {
+      buf += ch;
+      if (ch === "\\" && i + 1 < input.length) {
+        const next = input[i + 1] as string;
+        buf += next;
+        i++;
+        continue;
+      }
+      if (ch === quote) quote = null;
+      continue;
+    }
+    if (ch === '"' || ch === "'") {
+      quote = ch;
+      buf += ch;
+      continue;
+    }
+    if (ch === "[") {
+      depth++;
+      buf += ch;
+      continue;
+    }
+    if (ch === "]") {
+      depth = Math.max(0, depth - 1);
+      buf += ch;
+      continue;
+    }
+    if (depth === 0 && ch === sep) {
+      out.push(buf);
+      buf = "";
+      continue;
+    }
+    if (depth === 0 && sep === " " && /\s/.test(ch)) {
+      out.push(buf);
+      buf = "";
+      continue;
+    }
+    buf += ch;
+  }
+  out.push(buf);
+  return out;
+}
+
+/** Parse one compound (tag + ids + classes + attrs, no whitespace). */
+function parseCompound(input: string, original: string): CompoundPart {
+  const part: CompoundPart = { tag: "*", classes: [], attrs: [] };
+  let i = 0;
+  // Optional tag prefix (or `*`).
+  let tagBuf = "";
+  while (i < input.length) {
+    const ch = input[i] as string;
+    if (ch === "#" || ch === "." || ch === "[") break;
+    tagBuf += ch;
+    i++;
+  }
+  if (tagBuf.length > 0) {
+    if (!/^[*a-zA-Z][a-zA-Z0-9-]*$/.test(tagBuf)) {
+      throw new SelectorParseError(original, `bad tag "${tagBuf}"`);
+    }
+    part.tag = tagBuf.toLowerCase();
+  }
+  while (i < input.length) {
+    const ch = input[i] as string;
+    if (ch === "#") {
+      i++;
+      const id = readIdent(input, i, original);
+      part.id = id.value;
+      i = id.next;
+      continue;
+    }
+    if (ch === ".") {
+      i++;
+      const cls = readIdent(input, i, original);
+      part.classes.push(cls.value);
+      i = cls.next;
+      continue;
+    }
+    if (ch === "[") {
+      i++;
+      const attr = readAttr(input, i, original);
+      part.attrs.push(attr.filter);
+      i = attr.next;
+      continue;
+    }
+    throw new SelectorParseError(original, `unexpected "${ch}" in compound "${input}"`);
+  }
+  return part;
+}
+
+/** Read an identifier starting at `i`. Returns the parsed value + next idx. */
+function readIdent(input: string, i: number, original: string): { value: string; next: number } {
+  const start = i;
+  while (i < input.length) {
+    const ch = input[i] as string;
+    if (!/[a-zA-Z0-9_-]/.test(ch)) break;
+    i++;
+  }
+  const value = input.slice(start, i);
+  if (value.length === 0) {
+    throw new SelectorParseError(original, `expected identifier at position ${start}`);
+  }
+  return { value, next: i };
+}
+
+/** Read the contents of `[...]` starting just past the `[`. */
+function readAttr(
+  input: string,
+  i: number,
+  original: string,
+): { filter: AttrFilter; next: number } {
+  // Read attribute name (case-insensitive HTML; lower-case for storage).
+  const nameStart = i;
+  while (i < input.length) {
+    const ch = input[i] as string;
+    if (!/[a-zA-Z0-9_:-]/.test(ch)) break;
+    i++;
+  }
+  const name = input.slice(nameStart, i).toLowerCase();
+  if (name.length === 0) {
+    throw new SelectorParseError(original, `expected attribute name at position ${nameStart}`);
+  }
+  while (i < input.length && /\s/.test(input[i] as string)) i++;
+  if (i >= input.length) {
+    throw new SelectorParseError(original, `unterminated [...] in selector`);
+  }
+  if ((input[i] as string) === "]") {
+    return { filter: { name, op: "exists" }, next: i + 1 };
+  }
+  // Operator.
+  const opChars = ["~=", "|=", "^=", "$=", "*=", "="] as const;
+  let op: AttrFilter["op"] | null = null;
+  for (const cand of opChars) {
+    if (input.startsWith(cand, i)) {
+      op = cand;
+      i += cand.length;
+      break;
+    }
+  }
+  if (op === null) {
+    throw new SelectorParseError(original, `expected operator at position ${i}`);
+  }
+  while (i < input.length && /\s/.test(input[i] as string)) i++;
+  // Value: quoted or bare ident.
+  let value: string;
+  const ch0 = input[i] as string | undefined;
+  if (ch0 === '"' || ch0 === "'") {
+    const quote = ch0;
+    i++;
+    let buf = "";
+    while (i < input.length) {
+      const ch = input[i] as string;
+      if (ch === "\\" && i + 1 < input.length) {
+        buf += input[i + 1];
+        i += 2;
+        continue;
+      }
+      if (ch === quote) {
+        i++;
+        break;
+      }
+      buf += ch;
+      i++;
+    }
+    value = buf;
+  } else {
+    const start = i;
+    while (i < input.length) {
+      const ch = input[i] as string;
+      if (ch === "]" || /\s/.test(ch)) break;
+      i++;
+    }
+    value = input.slice(start, i);
+  }
+  while (i < input.length && /\s/.test(input[i] as string)) i++;
+  if ((input[i] as string | undefined) !== "]") {
+    throw new SelectorParseError(original, `expected ']' at position ${i}`);
+  }
+  return { filter: { name, op, value }, next: i + 1 };
+}
+
+// ---- matcher ---------------------------------------------------------------
+
+/**
+ * Test whether a single (already-walked) node matches the rightmost compound
+ * part of any branch in `parsed`, with ancestor-walking for descendant
+ * combinators. `ancestors` is the chain of parent element nodes from the
+ * document root down to (but not including) `node`, INCLUDING ancestors that
+ * cross shadow boundaries (the piercing walker keeps a flat chain).
+ */
+export function matchSelector(
+  parsed: ParsedSelector,
+  node: PierceDomNode,
+  ancestors: PierceDomNode[],
+): boolean {
+  for (const chain of parsed.chains) {
+    if (matchChain(chain, node, ancestors)) return true;
+  }
+  return false;
+}
+
+function matchChain(
+  chain: CompoundChain,
+  node: PierceDomNode,
+  ancestors: PierceDomNode[],
+): boolean {
+  const parts = chain.parts;
+  if (parts.length === 0) return false;
+  const last = parts[parts.length - 1] as CompoundPart;
+  if (!matchCompound(last, node)) return false;
+  // Walk leftwards through compound parts, each must be matched by some
+  // ancestor (in any order — `parts[k]` ancestor must be deeper than
+  // `parts[k-1]` ancestor; we enforce by iterating right-to-left and
+  // consuming ancestors from the bottom up).
+  let idx = ancestors.length - 1;
+  for (let p = parts.length - 2; p >= 0; p--) {
+    const part = parts[p] as CompoundPart;
+    let found = false;
+    while (idx >= 0) {
+      const a = ancestors[idx] as PierceDomNode;
+      idx--;
+      if (matchCompound(part, a)) {
+        found = true;
+        break;
+      }
+    }
+    if (!found) return false;
+  }
+  return true;
+}
+
+/** Test a single compound part against a single element node. */
+export function matchCompound(part: CompoundPart, node: PierceDomNode): boolean {
+  // Element nodes only.
+  if (node.nodeType !== 1) return false;
+  const local = (node.localName ?? node.nodeName.toLowerCase()).toLowerCase();
+  if (part.tag !== "*" && part.tag !== local) return false;
+  if (part.id !== undefined) {
+    const id = readAttribute(node, "id");
+    if (id !== part.id) return false;
+  }
+  if (part.classes.length > 0) {
+    const cls = readAttribute(node, "class") ?? "";
+    const tokens = cls.split(/\s+/).filter((t) => t.length > 0);
+    for (const c of part.classes) {
+      if (!tokens.includes(c)) return false;
+    }
+  }
+  for (const f of part.attrs) {
+    if (!matchAttr(f, node)) return false;
+  }
+  return true;
+}
+
+function matchAttr(f: AttrFilter, node: PierceDomNode): boolean {
+  const val = readAttribute(node, f.name);
+  if (f.op === "exists") return val !== undefined;
+  if (val === undefined) return false;
+  const target = f.value ?? "";
+  switch (f.op) {
+    case "=":
+      return val === target;
+    case "~=": {
+      // Whitespace-separated word match.
+      const tokens = val.split(/\s+/).filter((t) => t.length > 0);
+      return tokens.includes(target);
+    }
+    case "|=":
+      return val === target || val.startsWith(`${target}-`);
+    case "^=":
+      return target.length > 0 && val.startsWith(target);
+    case "$=":
+      return target.length > 0 && val.endsWith(target);
+    case "*=":
+      return target.length > 0 && val.indexOf(target) >= 0;
+  }
+}
+
+/**
+ * Read an attribute value from a `PierceDomNode`. CDP serialises attributes
+ * as a flat `[name, value, name, value, ...]` array (lower-cased names per
+ * the protocol). Returns `undefined` if absent.
+ */
+export function readAttribute(node: PierceDomNode, name: string): string | undefined {
+  const attrs = node.attributes;
+  if (attrs === undefined) return undefined;
+  const lower = name.toLowerCase();
+  for (let i = 0; i + 1 < attrs.length; i += 2) {
+    if ((attrs[i] as string).toLowerCase() === lower) return attrs[i + 1] as string;
+  }
+  return undefined;
+}

package/src/page.ts

@@ -35,9 +35,13 @@ import type {
   DispatchMouseEventParams,
   DomNode,
   FrameNavigatedEvent,
+  PierceDomNode,
   RemoteObject,
 } from "./cdp/types";
 import { NotImplementedError } from "./errors";
+import { ElementHandle } from "./page/element-handle";
+import { findPiercingMatches } from "./page/piercing";
+import { parseSelector } from "./page/selector";
 
 /** Wait conditions for `Page.goto`. */
 export type WaitUntil = "load" | "domcontentloaded" | "networkidle";
@@ -307,6 +311,14 @@ export class Page {
    * document (so `this` === document). Result is JSON-serialized via
    * `returnByValue: true`.
    *
+   * The function may return a value or a `Promise`. Promise-returning
+   * functions are awaited page-side via `awaitPromise: true` (CDP's canonical
+   * mechanism for async eval) — without that flag, an `async () => ...`
+   * function round-trips its returned Promise as `undefined` because CDP
+   * serializes the Promise object itself, not its resolution. `awaitPromise`
+   * is NOT on PLAN.md §8.2's forbidden list — only `Runtime.enable` and
+   * `Page.createIsolatedWorld` are. Available since Chromium 67.
+   *
    * Limitations (documented in docs/limits.md):
    *   - Non-JSON return values (functions, DOM nodes, undefined) are
    *     coerced/dropped per CDP semantics.
@@ -315,13 +327,14 @@ export class Page {
    *     standard for any cross-process evaluator).
    *   - Arguments cannot be passed in v0.1; the function takes no args.
    */
-  async evaluate<T>(fn: () => T): Promise<T> {
+  async evaluate<T>(fn: () => T | Promise<T>): Promise<T> {
     this.assertOpen();
     const docId = await this.documentObjectId();
     const result = await this.send<{ result: RemoteObject }>("Runtime.callFunctionOn", {
       objectId: docId,
       functionDeclaration: fn.toString(),
       returnByValue: true,
+      awaitPromise: true,
     });
     return result.result.value as T;
   }
@@ -527,6 +540,39 @@ export class Page {
     });
     const targetBox = boxFromBorderQuad(box.model);
     const callSeed = this.nextCallSeed();
+    // Trajectory synth lives here (not in `performClickAt`) so prototype
+    // inspection in conformance tests can see the synthesize / trajectory
+    // / cursor markers — they're a consumer-side smoke check that the
+    // behavioral synth is wired in.
+    const traj = synthesizeMouseTrajectory({
+      from: { x: this.cursor.x, y: this.cursor.y },
+      to: { x: targetBox.x + targetBox.width / 2, y: targetBox.y + targetBox.height / 2 },
+      box: targetBox,
+      profile: this.behavior,
+      seed: callSeed,
+      ...(opts.duration !== undefined ? { durationMs: opts.duration } : {}),
+    });
+    await this.dispatchClickTrajectory(traj, callSeed, opts);
+  }
+
+  /**
+   * Variant of {@link humanClick} that operates on an {@link ElementHandle}
+   * resolved via {@link querySelectorPiercing} — required when the target
+   * element lives inside a closed shadow root (no CSS path can name it from
+   * the parent document, so the regular `humanClick(selector)` route fails).
+   *
+   * Pipeline differs from {@link humanClick} only in step 1: the box model
+   * is resolved via `DOM.getBoxModel({ backendNodeId })` instead of through a
+   * `DOM.querySelector`-resolved nodeId. Everything downstream (trajectory
+   * synth, dispatch loop, press/release) is identical.
+   */
+  async humanClickHandle(handle: ElementHandle, opts: HumanClickOptions = {}): Promise<void> {
+    this.assertOpen();
+    const box = await this.send<{ model: BoxModel }>("DOM.getBoxModel", {
+      backendNodeId: handle.backendNodeId,
+    });
+    const targetBox = boxFromBorderQuad(box.model);
+    const callSeed = this.nextCallSeed();
     const traj = synthesizeMouseTrajectory({
       from: { x: this.cursor.x, y: this.cursor.y },
       to: { x: targetBox.x + targetBox.width / 2, y: targetBox.y + targetBox.height / 2 },
@@ -535,6 +581,22 @@ export class Page {
       seed: callSeed,
       ...(opts.duration !== undefined ? { durationMs: opts.duration } : {}),
     });
+    await this.dispatchClickTrajectory(traj, callSeed, opts);
+  }
+
+  /**
+   * Inner dispatch loop shared by {@link humanClick} and
+   * {@link humanClickHandle}. Takes the synthesised trajectory, paces the
+   * `mouseMoved` events, then fires `mousePressed` + `mouseReleased` at the
+   * arrival point with realistic press duration. Trajectory synth itself
+   * stays inside the public methods so source-grep conformance checks can
+   * verify the synth is reachable from the public API.
+   */
+  private async dispatchClickTrajectory(
+    traj: ReturnType<typeof synthesizeMouseTrajectory>,
+    callSeed: string,
+    opts: HumanClickOptions,
+  ): Promise<void> {
     if (traj.length === 0) return;
 
     // Pre-move settle: Gaussian(150, 50) ms idle. Cheaply approximated via
@@ -724,6 +786,95 @@ export class Page {
     }
   }
 
+  /**
+   * Closed-shadow-root piercing locator — find the first element matching the
+   * CSS selector across the entire DOM tree, including elements nested inside
+   * **closed** shadow roots (which {@link text}, {@link humanClick}, etc. can
+   * NOT reach because `DOM.querySelector` does not traverse closed shadows
+   * even with `pierce: true` set on the parent `getDocument` call).
+   *
+   * Required for Cloudflare Turnstile auto-click on integrations where the
+   * widget iframe lives behind a closed shadow root (Cloudflare Challenge
+   * pages, Workers Static Assets, some CDN configs). Without this, task
+   * 0220's auto-click silently fails on those flows.
+   *
+   * Algorithm (port of patchright `framesPatch.ts:868-1012`
+   * `_customFindElementsByParsed`):
+   *   1. `DOM.getDocument({ depth: -1, pierce: true })` — yields the full
+   *      tree, with shadow descendants under `shadowRoots[]` for both open
+   *      AND closed roots.
+   *   2. Recursive walk in JS, matching against a parsed CSS selector. We
+   *      can't `DOM.querySelector` per shadow because the per-shadow query
+   *      itself doesn't pierce closed roots either.
+   *   3. For matches, `DOM.resolveNode({ backendNodeId })` to get a
+   *      `RemoteObject.objectId`, wrapped in {@link ElementHandle}.
+   *
+   * Supported selectors (see `selector.ts`): tag / id / class / attribute /
+   * descendant combinator / comma-separated lists. **Not** supported:
+   * `>`/`+`/`~` combinators, `:pseudo-classes`, `::pseudo-elements`, XPath.
+   * XPath is a stretch goal per task 0253 brief — TODO if a future surface
+   * needs it (Turnstile detection only needs CSS).
+   *
+   * Performance: O(N) in DOM size per call. Acceptable for v0.2; a per-page
+   * cache layer is a v0.3+ concern (also called out in 0253).
+   *
+   * @see tasks/0253-closed-shadow-piercing-locator.md
+   * @see PLAN.md §8.2 (`DOM.getDocument` and `DOM.resolveNode` are not on the
+   *   forbidden list — both fine to use here).
+   */
+  async querySelectorPiercing(selector: string): Promise<ElementHandle | null> {
+    const handles = await this.queryPiercing(selector, 1);
+    return handles[0] ?? null;
+  }
+
+  /**
+   * The "all matches" variant of {@link querySelectorPiercing}. Returns every
+   * element that satisfies the selector, in depth-first pre-order — same
+   * traversal a regular `querySelectorAll` produces, with closed-shadow
+   * descendants spliced in at the position they'd appear under the host.
+   *
+   * Returns an empty array when nothing matches.
+   */
+  async querySelectorAllPiercing(selector: string): Promise<ElementHandle[]> {
+    return this.queryPiercing(selector);
+  }
+
+  /** Shared implementation for the piercing locator. `limit` short-circuits the walk. */
+  private async queryPiercing(selector: string, limit?: number): Promise<ElementHandle[]> {
+    this.assertOpen();
+    const parsed = parseSelector(selector);
+    // depth: -1 + pierce: true is the magic combination patchright uses; CDP
+    // returns a fully-flattened tree including shadow descendants on both
+    // open and closed roots, AND iframe contentDocuments for same-origin
+    // children.
+    const root = await this.send<{ root: PierceDomNode }>("DOM.getDocument", {
+      depth: -1,
+      pierce: true,
+    });
+    const matches = findPiercingMatches(root.root, parsed, limit);
+    if (matches.length === 0) return [];
+    const handles: ElementHandle[] = [];
+    for (const m of matches) {
+      const resolved = await this.send<{ object: RemoteObject }>("DOM.resolveNode", {
+        backendNodeId: m.backendNodeId,
+      });
+      const objectId = resolved.object.objectId;
+      // Skip nodes the protocol couldn't bind to a RemoteObject (rare — e.g.
+      // detached subtree races). Surfacing a partial set is more useful than
+      // throwing for the Turnstile detector path.
+      if (objectId === undefined) continue;
+      handles.push(
+        new ElementHandle({
+          router: this.router,
+          sessionId: this.sessionId,
+          objectId,
+          backendNodeId: m.backendNodeId,
+        }),
+      );
+    }
+    return handles;
+  }
+
   screenshot(_opts?: unknown): Promise<Uint8Array> {
     return Promise.reject(new NotImplementedError("page.screenshot"));
   }