@swf2es/player 0.0.0-stage → 0.1.1

package/src/display/bounds.ts

@@ -0,0 +1,702 @@
+// A display object's bounds and hit tests (docs/architecture/rendering.md, "Bounds
+// and hit tests"): a SWF shape's recorded rectangles, a drawing's true
+// extent, a container's children's through their matrices.
+import type { Matrix } from "@swf2es/format";
+import { hitsGlyph } from "../text/static.js";
+import {
+  BitmapObject,
+  Clips,
+  Container,
+  type DisplayObject,
+  ShapeObject,
+  StaticTextObject,
+  TextObject,
+  VideoObject,
+} from "./display.js";
+import {
+  apply,
+  behindEye,
+  boxThrough,
+  concat,
+  contains,
+  invert,
+  isProjective,
+  overlaps,
+  type Rect,
+  transformRect,
+  union,
+} from "./geometry.js";
+import { sliceFor, sliceLayers, withinMask } from "./scale9.js";
+import {
+  facing,
+  flatten,
+  inside,
+  orientation,
+  type Path,
+  pointsOf,
+  type ShapeLayer,
+} from "./shapes.js";
+
+const TWIPS = 20;
+const IDENTITY: Matrix = { a: 1, b: 0, c: 0, d: 1, tx: 0, ty: 0 };
+
+/** The matrix from `d`'s space to the stage's (`stage` itself excluded), through every parent, scroll shifts and all. */
+export function toStage(d: DisplayObject, stage: DisplayObject | null): Matrix {
+  let m = d.placed;
+  for (let p = d.parent; p && p !== stage; p = p.parent) {
+    m = concat(m, p.placed);
+  }
+
+  return m;
+}
+
+/** What `d` itself draws, in its own space: null for nothing. `lines` counts the lines' widths. */
+export function ownBounds(d: DisplayObject, lines: boolean): Rect | null {
+  if (d.drawing) {
+    return d.drawing.bounds(lines);
+  }
+
+  if (d instanceof BitmapObject) {
+    const store = d.store;
+    return store && !store.disposed
+      ? { xMin: 0, yMin: 0, xMax: store.width, yMax: store.height }
+      : null;
+  }
+
+  if (d instanceof TextObject) {
+    return { xMin: d.left, yMin: d.top, xMax: d.left + d.width, yMax: d.top + d.height };
+  }
+
+  if (d instanceof VideoObject) {
+    return { xMin: 0, yMin: 0, xMax: d.boxWidth, yMax: d.boxHeight };
+  }
+
+  if (d instanceof StaticTextObject) {
+    const r = d.definition.definition.bounds;
+    return {
+      xMin: r.xMin / TWIPS,
+      yMin: r.yMin / TWIPS,
+      xMax: r.xMax / TWIPS,
+      yMax: r.yMax / TWIPS,
+    };
+  }
+
+  if (d instanceof ShapeObject && d.shape) {
+    const r = (lines ? null : d.shape.shape.edgeBounds) ?? d.shape.shape.bounds;
+    return {
+      xMin: r.xMin / TWIPS,
+      yMin: r.yMin / TWIPS,
+      xMax: r.xMax / TWIPS,
+      yMax: r.yMax / TWIPS,
+    };
+  }
+
+  return null;
+}
+
+/**
+ * `d`'s bounds in its own space, its children's included: null for an
+ * object with nothing in it. A scrolled object's are its scroll's size at
+ * (0, 0), whatever it draws, as Flash reports them; masks clip none.
+ * `drawn` takes a 3D object in another as drawn, not as reported
+ * (placedRect), for the bitmap the other is drawn into.
+ */
+export function bounds(d: DisplayObject, lines: boolean, drawn = false): Rect | null {
+  const scroll = d.scroll;
+  if (scroll) {
+    return { xMin: 0, yMin: 0, xMax: scroll.xMax - scroll.xMin, yMax: scroll.yMax - scroll.yMin };
+  }
+
+  let r = ownBounds(d, lines);
+  if (d instanceof Container) {
+    for (const child of d.children) {
+      const b = bounds(child, lines, drawn);
+      // A 3D child with a corner behind the eye is not drawn, and counts for nothing.
+      if (b && !(child.space && behindEye(b, child.placed))) {
+        r = union(r, placedRect(child, b, drawn));
+      }
+    }
+  }
+
+  return r;
+}
+
+/**
+ * `r`, in the space of `d`, through `d`'s matrix into its parent's space.
+ * A 3D object's is the box around what its plane draws in its projector's
+ * space, out to whole pixels, taken back to its parent's as a box, as Flash
+ * measures it: so under a rotated 2D parent its box there is the box of a
+ * box, larger than what is drawn (the `three-d-bounds` case). In a 3D
+ * parent, it is its box through its matrix3D's x and y rows, without
+ * perspective, as its width is in adl; or, `drawn`, its plane's there,
+ * which the parent's bitmap must hold.
+ */
+export function placedRect(d: DisplayObject, r: Rect, drawn = false): Rect {
+  if (!d.space) {
+    return transformRect(r, d.placed);
+  }
+
+  if (d.parent?.space) {
+    return transformRect(r, drawn ? d.placed : d.flatPlaced);
+  }
+
+  const { own, back } = d.planes();
+  const box = transformRect(r, own);
+  return back ? boxThrough(box, back) : box;
+}
+
+/** `d`'s bounds in `target`'s space (its own for null), through the stage. */
+export function boundsIn(
+  d: DisplayObject,
+  target: DisplayObject | null,
+  stage: DisplayObject | null,
+  lines: boolean,
+): Rect {
+  const own = bounds(d, lines) ?? { xMin: 0, yMin: 0, xMax: 0, yMax: 0 };
+  if (!target || target === d) {
+    return own;
+  }
+
+  const up = toStage(d, stage);
+  const down = target === stage ? IDENTITY : (invert(toStage(target, stage)) ?? IDENTITY);
+  if (!isProjective(up)) {
+    return transformRect(own, concat(up, down));
+  }
+
+  // Through a plane: a 2D object's only up to where it and the target meet, so that a plane and
+  // its inverse do not both go into it, out to whole pixels where they cancel.
+  if (!d.space) {
+    let common: DisplayObject | null = target;
+    while (common && !common.encloses(d)) {
+      common = common.parent;
+    }
+
+    if (!common || common === stage) {
+      return transformRect(own, concat(up, down));
+    }
+
+    const toTarget = target === common ? IDENTITY : invert(toStage(target, common));
+    return toTarget
+      ? transformRect(own, concat(toStage(d, common), toTarget))
+      : transformRect(own, concat(up, down));
+  }
+
+  // A 3D object's, as Flash measures it: the box on the stage, out to whole pixels, then that
+  // box taken into the target's space, a box of a box (the `three-d-bounds` case).
+  const box = transformRect(own, up);
+  return target === stage ? box : boxThrough(box, down);
+}
+
+/**
+ * Whether the point (x, y) is within `d`'s bounds, or, for `shape`, on what
+ * it or its children draw. The point is in the main root's space, as Flash
+ * has it (the corpus's `displayobject_hittestpoint_root`): moving the root
+ * moves nothing under the point, moving a loaded SWF's root does. The
+ * shape test asks for a root above `d`, the bounds test does not.
+ */
+export function hitsPoint(
+  d: DisplayObject,
+  x: number,
+  y: number,
+  shape: boolean,
+  root: DisplayObject | null,
+): boolean {
+  const up = toStage(d, root);
+  if (isProjective(up) && inHidden3D(d)) {
+    return false;
+  }
+
+  // In 3D, the bounds are the box around what is drawn on the stage, as Flash tests them.
+  if (!shape && isProjective(up)) {
+    const r = bounds(d, true);
+    return r !== null && !behindEye(r, up) && contains(transformRect(r, up), x, y);
+  }
+
+  const toLocal = invert(up);
+  if (!toLocal) {
+    return false;
+  }
+
+  if (!shape) {
+    const [lx, ly] = apply(toLocal, x, y);
+    const r = bounds(d, true);
+    return r !== null && contains(r, lx, ly);
+  }
+
+  // A mask is not drawn, and adl hits nothing of it, a script's or a timeline's, nor of what
+  // is in it. Nor the stage's shape test, though the stage is a root for what a script puts
+  // on it: the one root with no parent.
+  if (!underRoot(d) || withinMask(d) || (d.parent === null && d.loaderInfo !== null)) {
+    return false;
+  }
+
+  // Flash samples half a pixel to the left of the point and on its row: a
+  // point on a shape's right edge hits, one on its left, top or bottom edge
+  // does not (the corpus's displayobject_hittestpoint_boundary).
+  const [lx, ly] = apply(toLocal, x - 0.5, y);
+  const probe = { x: x - 0.5, y, root };
+  return drawnAt(d, lx, ly, probe, false) && !clippedAbove(d, probe);
+}
+
+/**
+ * What this object itself draws under a pointer, without asking its
+ * children. For the pointer's own pick (`clips`), the timeline's masks
+ * above it clip it too, as Flash picks (the corpus's `mouse_pick_masking`):
+ * a list scrolled under a mask layer takes no click where it is hidden.
+ */
+export function hitsOwnPoint(
+  d: DisplayObject,
+  x: number,
+  y: number,
+  root: DisplayObject,
+  clips = false,
+): boolean {
+  if (!underRoot(d) || d.maskOf) {
+    return false;
+  }
+
+  const toLocal = invert(toStage(d, root));
+  if (!toLocal) {
+    return false;
+  }
+
+  const probe = { x: x - 0.5, y, root };
+  const [lx, ly] = apply(toLocal, probe.x, probe.y);
+  return hitsOwnLocal(d, lx, ly, probe, clips);
+}
+
+/**
+ * hitsOwnPoint with the probe already in `d`'s own space, at (`lx`, `ly`):
+ * the pointer's pick carries it down the list, making no matrix per object.
+ */
+export function hitsOwnLocal(
+  d: DisplayObject,
+  lx: number,
+  ly: number,
+  probe: Probe,
+  clips = false,
+): boolean {
+  if (!underRoot(d) || d.maskOf) {
+    return false;
+  }
+
+  let hit: boolean;
+  if (d instanceof TextObject) {
+    const r = ownBounds(d, true);
+    const scroll = d.scroll;
+    hit =
+      !!r &&
+      contains(r, lx, ly) &&
+      (!scroll || contains(scroll, lx, ly)) &&
+      (!d.mask || inMask(d.mask, probe));
+  } else {
+    hit = drawnAt(d, lx, ly, probe, false, false);
+  }
+
+  // What clips it is asked only of a hit: a pick tests every object under its
+  // containers, and finding the mask layers over one walks its siblings.
+  return hit && !clippedAbove(d, probe) && !(clips && clippedByLayers(d, probe));
+}
+
+/** Whether a mask or a scroll above `d` leaves the probe out. */
+function clippedAbove(d: DisplayObject, probe: Probe): boolean {
+  for (let p = d.parent; p; p = p.parent) {
+    if (p.mask && !p.maskOf && !p.mask.encloses(p) && !inMask(p.mask, probe)) {
+      return true;
+    }
+
+    const scroll = p.scroll;
+    const toLocal = scroll && invert(toStage(p, probe.root));
+    if (scroll && toLocal) {
+      const [x, y] = apply(toLocal, probe.x, probe.y);
+      if (!(x >= scroll.xMin && x < scroll.xMax && y >= scroll.yMin && y < scroll.yMax)) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+/** Whether any of the first `n` clip layers open leaves the probe out. */
+function outsideLayers(clips: Clips, n: number, probe: Probe): boolean {
+  for (let i = 0; i < n; i++) {
+    if (!inMask(clips.masks[i], probe)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/** Whether a timeline's mask over `d` or over a container above it leaves the probe out. */
+function clippedByLayers(d: DisplayObject, probe: Probe): boolean {
+  for (let o = d; o.parent; o = o.parent) {
+    const clips = new Clips();
+    for (const child of o.parent.children) {
+      const n = clips.enter(child);
+      if (child !== o) {
+        continue;
+      }
+
+      if (outsideLayers(clips, n, probe)) {
+        return true;
+      }
+
+      break;
+    }
+  }
+
+  return false;
+}
+
+/** The point a shape test asks of, in the root's space: masks are found from it. */
+export interface Probe {
+  x: number;
+  y: number;
+  root: DisplayObject | null;
+}
+
+/** Whether the probe is on `mask`'s fills, where it is: its lines clip nothing; a 3D mask's, all of it. */
+function inMask(mask: DisplayObject, probe: Probe): boolean {
+  // A 3D mask clips nothing, as Flash draws it.
+  if (mask.space) {
+    return true;
+  }
+
+  const toLocal = invert(toStage(mask, probe.root));
+  if (!toLocal) {
+    return false;
+  }
+
+  const [x, y] = apply(toLocal, probe.x, probe.y);
+  return drawnAt(mask, x, y, probe, true);
+}
+
+/** Whether a SWF's root, the main one's or a loaded one's, is `d` or above it. */
+function underRoot(d: DisplayObject): boolean {
+  for (let o: DisplayObject | null = d; o; o = o.parent) {
+    if (o.loaderInfo || o.avm1Root) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Whether (x, y), in `d`'s space, is on a fill or a line of `d` or of a
+ * child, within the scroll and the masks that clip it. As a mask (`mask`)
+ * its lines count for nothing and its own mask is not asked, so that two
+ * masks of each other end.
+ */
+function drawnAt(
+  d: DisplayObject,
+  x: number,
+  y: number,
+  probe: Probe,
+  mask: boolean,
+  children = true,
+): boolean {
+  const scroll = d.scroll;
+  if (scroll && !(x >= scroll.xMin && x < scroll.xMax && y >= scroll.yMin && y < scroll.yMax)) {
+    return false;
+  }
+
+  if (!mask && d.mask && !d.maskOf && !d.mask.encloses(d) && !inMask(d.mask, probe)) {
+    return false;
+  }
+
+  // A Bitmap is hit, and masks, over its whole rectangle, transparent pixels and all.
+  if (d instanceof BitmapObject) {
+    const store = d.store;
+    return !!store && !store.disposed && x >= 0 && x < store.width && y >= 0 && y < store.height;
+  }
+
+  if (d instanceof StaticTextObject) {
+    return hitsGlyph(d.definition, d.glyphs.glyphs, x, y, mask);
+  }
+
+  const drawing = d.drawing;
+  let layers: ShapeLayer[] = drawing
+    ? facing(drawing.layers, drawing.culled && turnedOver(d), drawing.version)
+    : d instanceof ShapeObject
+      ? (d.shape?.layers ?? [])
+      : [];
+  // A 9-slice is hit where it is drawn, its layers as the renderer moves them, and only within
+  // its own bounds as drawn unsliced: adl misses an owner's fill that a wider child stretched
+  // past them. A mask is not sliced.
+  const own =
+    layers.length > 0 && !mask && (d.scale9Grid || d.parent?.scale9Grid) ? sliceFor(d) : null;
+  if (own) {
+    const r = ownBounds(d, true);
+    layers =
+      r && contains(r, x, y) ? sliceLayers(layers, own.slice, own.m, d.drawing?.version) : [];
+  }
+
+  for (const layer of layers) {
+    for (const { contours, winding } of layer.fills) {
+      // Inside by the fill's rule: an odd number of its contours around the
+      // point for even-odd, a non-zero sum of their orientations for non-zero.
+      let crossings = 0;
+      let sum = 0;
+      for (const contour of contours) {
+        const box = boxOf(contour);
+        if (!near(box, x, y, 0)) {
+          continue;
+        }
+
+        const flat = flatOf(contour, box);
+        if (inside(flat.points, x, y)) {
+          crossings++;
+          sum += flat.orientation;
+        }
+      }
+
+      if (winding === "nonZero" ? sum !== 0 : crossings % 2) {
+        return true;
+      }
+    }
+
+    if (layer.meshes?.some((mesh) => inMesh(mesh.positions, x, y))) {
+      return true;
+    }
+
+    if (mask) {
+      continue;
+    }
+
+    for (const { line, paths } of layer.strokes) {
+      const half = Math.max(line.width / TWIPS, 1) / 2;
+      for (const path of paths) {
+        const box = boxOf(path);
+        if (near(box, x, y, half) && nearPolyline(flatOf(path, box).points, x, y, half)) {
+          return true;
+        }
+      }
+    }
+  }
+
+  if (children && d instanceof Container) {
+    // Masks are not drawn; a timeline's clip no hit test, as Flash's do not. What a clip
+    // layer clips is hit only where the layer covers the probe: a clip that plays its mask
+    // to an empty frame hides a part from a shape test too (adl, avm1-hit-test-under-mask).
+    let clips: Clips | null = null;
+    for (const child of d.children) {
+      if (child.clipDepth > 0) {
+        clips ??= new Clips();
+      }
+
+      const n = clips ? clips.enter(child) : 0;
+      if (child.clipDepth !== 0 || child.maskOf !== null) {
+        continue;
+      }
+
+      const toChild = invert(child.placed);
+      if (
+        toChild &&
+        !(child.space && hidden3D(child)) &&
+        !(clips && !mask && outsideLayers(clips, n, probe)) &&
+        drawnAt(child, ...apply(toChild, x, y), probe, mask)
+      ) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Whether `d` is turned over on the stage, which decides the triangles a
+ * culled drawTriangles keeps there: as the renderer draws it, within a 3D
+ * object's own space.
+ */
+function turnedOver(d: DisplayObject): boolean {
+  let turned = false;
+  for (let o: DisplayObject | null = d; o && !o.space; o = o.parent) {
+    const m = o.matrix;
+    if (m.a * m.d - m.b * m.c < 0) {
+      turned = !turned;
+    }
+  }
+
+  return turned;
+}
+
+/** Whether (x, y) is in any of the triangles, three points of `positions` each. */
+function inMesh(positions: Float32Array, x: number, y: number): boolean {
+  const triangle = [0, 0, 0, 0, 0, 0];
+  for (let i = 0; i < positions.length; i += 6) {
+    for (let k = 0; k < 6; k++) {
+      triangle[k] = positions[i + k];
+    }
+
+    if (inside(triangle, x, y)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/** Whether `d` or a 3D object above it has a corner behind the eye, so that none of `d` is drawn. */
+function inHidden3D(d: DisplayObject): boolean {
+  for (let p: DisplayObject | null = d; p; p = p.parent) {
+    if (p.space && hidden3D(p)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/** Whether a 3D object has a corner behind the eye, which Flash draws none of, and hits none of. */
+export function hidden3D(d: DisplayObject): boolean {
+  const r = bounds(d, true);
+  return r !== null && behindEye(r, d.placed);
+}
+
+/** The box around a path, as of its length: a drawing's paths grow as lineTo adds to them. */
+interface Box {
+  length: number;
+  xMin: number;
+  yMin: number;
+  xMax: number;
+  yMax: number;
+}
+
+/** A path as hit tests read it: its polygon, the polygon's orientation, and the box around it. */
+interface Flat extends Box {
+  points: number[];
+  orientation: number;
+}
+
+/**
+ * The boxes of paths not flattened: their points' and control points',
+ * which hold their curves, made with nothing flattened. A drawing redrawn on
+ * every frame has new paths each time, most of them far from the pointer,
+ * which were flattened for their box alone.
+ */
+const hulls = new WeakMap<Path, Box>();
+
+/** The path's box: its flattening's where that was made, else its points' and control points'. */
+function boxOf(path: Path): Box {
+  const flat = flats.get(path);
+  if (flat && flat.length === path.length) {
+    return flat;
+  }
+
+  const kept = hulls.get(path);
+  if (kept && kept.length === path.length) {
+    return kept;
+  }
+
+  let xMin = Infinity;
+  let yMin = Infinity;
+  let xMax = -Infinity;
+  let yMax = -Infinity;
+  for (let i = 0; i < path.length; ) {
+    const end = i + 1 + pointsOf(path[i]) * 2;
+    for (let k = i + 1; k < end; k += 2) {
+      xMin = Math.min(xMin, path[k]);
+      xMax = Math.max(xMax, path[k]);
+      yMin = Math.min(yMin, path[k + 1]);
+      yMax = Math.max(yMax, path[k + 1]);
+    }
+
+    i = end;
+  }
+
+  const box = { length: path.length, xMin, yMin, xMax, yMax };
+  hulls.set(path, box);
+
+  return box;
+}
+
+/** The path flattened: `box` itself where boxOf gave its flattening. */
+function flatOf(path: Path, box: Box): Flat {
+  return "points" in box ? (box as Flat) : flattened(path);
+}
+
+/** Whether (x, y) is within `half` of the box, as it must be to be in or on its path. */
+function near(box: Box, x: number, y: number, half: number): boolean {
+  // Negated, so a box made NaN by a drawing's NaN point rules nothing out:
+  // the full test then skips only the NaN edges, as it did.
+  return !(
+    x < box.xMin - half ||
+    x > box.xMax + half ||
+    y < box.yMin - half ||
+    y > box.yMax + half
+  );
+}
+
+/**
+ * Paths flattened once, not on every hit test: the pointer asks on every
+ * move, of every shape under the point's ancestors, and a curve's chords
+ * cost more than a box test that rules most of them out.
+ */
+const flats = new WeakMap<Path, Flat>();
+
+function flattened(path: Path): Flat {
+  const cached = flats.get(path);
+  if (cached && cached.length === path.length) {
+    return cached;
+  }
+
+  const points = flatten(path);
+  let [xMin, yMin, xMax, yMax] = [Infinity, Infinity, -Infinity, -Infinity];
+  for (let i = 0; i < points.length; i += 2) {
+    xMin = Math.min(xMin, points[i]);
+    xMax = Math.max(xMax, points[i]);
+    yMin = Math.min(yMin, points[i + 1]);
+    yMax = Math.max(yMax, points[i + 1]);
+  }
+
+  const flat = {
+    length: path.length,
+    points,
+    orientation: orientation(points),
+    xMin,
+    yMin,
+    xMax,
+    yMax,
+  };
+  flats.set(path, flat);
+  return flat;
+}
+
+/** Whether (px, py) is within `half` of any segment of the open polyline. */
+function nearPolyline(points: number[], px: number, py: number, half: number): boolean {
+  for (let i = 2; i < points.length; i += 2) {
+    const ax = points[i - 2];
+    const ay = points[i - 1];
+    const bx = points[i];
+    const by = points[i + 1];
+    const dx = bx - ax;
+    const dy = by - ay;
+    const length2 = dx * dx + dy * dy;
+    const t =
+      length2 === 0 ? 0 : Math.max(0, Math.min(1, ((px - ax) * dx + (py - ay) * dy) / length2));
+    if (Math.hypot(px - (ax + t * dx), py - (ay + t * dy)) <= half) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/** Whether the two objects' bounds in the stage's space overlap. */
+export function hitsObject(
+  a: DisplayObject,
+  b: DisplayObject,
+  stage: DisplayObject | null,
+): boolean {
+  const ra = bounds(a, true);
+  const rb = bounds(b, true);
+  if (!ra || !rb) {
+    return false;
+  }
+
+  return overlaps(transformRect(ra, toStage(a, stage)), transformRect(rb, toStage(b, stage)));
+}