@quiesco/agent 0.0.0-stage → 0.1.1

package/src/actions/js.ts

@@ -0,0 +1,245 @@
+/**
+ * `js` action mode (requirements.md §6.3): call the components' callbacks directly. Fast and needs no
+ * native code, but skips hit testing. Limitations (documented):
+ * - tap / longPress / doubleTap call the nearest `onPress` / `onLongPress` found from the element up
+ *   to the root, together with that component's `onPressIn` / `onPressOut`. A component with
+ *   `disabled: true` is not pressed (like Pressability). Gesture-handler gestures (`Gesture.Tap()` ...)
+ *   have no such prop and are not supported.
+ * - type / clear call `onChangeText` (and `onChange`) of the nearest TextInput with the new full text.
+ *   Controlled inputs update through their `value`; uncontrolled inputs keep their native text.
+ *   `type` appends to the current `value` (or `defaultValue`) in a single call, not key by key.
+ * - scroll / swipe / scrollTo call `scrollTo({ x, y, animated: false })` on the nearest ScrollView
+ *   (FlatList / SectionList included). Other scrollables (pager views ...) are not supported. `from`
+ *   (a start point) has no meaning here and is ignored.
+ * - A tap at a point (`tap(t, { point })`, `tapAt`) presses the topmost element under that point.
+ * - A tap on a switch / checkbox without onPress calls `onValueChange(!value)` (unless disabled).
+ * - pressKey: `enter` calls `onSubmitEditing` (a multi-line input without it gets a newline);
+ *   `backspace` calls `onKeyPress` and removes the last character.
+ * - drag is not supported (it needs real touches).
+ */
+import type { Direction, Edge, KeyName, Point, Rect } from '@quiesco/protocol';
+
+import { QuiescoError } from '../errors';
+import type { HandlerRef, HostNode } from '../fiber-adapter';
+import { hitTest } from './hit-test';
+
+export interface JsActionContext {
+  platform: string;
+  /** Current host roots, for taps at a point (hit testing). */
+  roots?: () => readonly HostNode[];
+}
+
+/** Holding at least this long is a long press (`longPressAt`). */
+export const LONG_PRESS_THRESHOLD_MS = 500;
+
+/** Synthetic event passed to press handlers: the fields user code commonly reads. `at`: window point. */
+function pressEvent(node: HostNode, type: string, at?: Point): Record<string, unknown> {
+  const r = node.measure();
+  const cx = at?.x ?? (r === null ? 0 : r.x + r.width / 2);
+  const cy = at?.y ?? (r === null ? 0 : r.y + r.height / 2);
+  const nativeEvent = {
+    locationX: r === null ? 0 : cx - r.x,
+    locationY: r === null ? 0 : cy - r.y,
+    pageX: cx,
+    pageY: cy,
+    timestamp: Date.now(),
+    identifier: 0,
+    touches: [],
+    changedTouches: [],
+  };
+  let defaultPrevented = false;
+  return {
+    type,
+    nativeEvent,
+    timeStamp: nativeEvent.timestamp,
+    persist() {},
+    preventDefault() {
+      defaultPrevented = true;
+    },
+    isDefaultPrevented: () => defaultPrevented,
+    stopPropagation() {},
+    isPropagationStopped: () => false,
+  };
+}
+
+type JsAction = 'tap' | 'longPress' | 'doubleTap' | 'multiTap' | 'type' | 'replaceText' | 'clear' | 'pressKey' | 'scroll' | 'swipe' | 'scrollTo' | 'drag' | 'tapAt' | 'pinch' | 'rotate' | 'multiTouch';
+
+export function notSupported(action: JsAction, ctx: JsActionContext, hint: string): QuiescoError {
+  return new QuiescoError('ACTION_NOT_SUPPORTED', `${action} is not possible in js mode: ${hint}`, { action, platform: ctx.platform, hint });
+}
+
+function isDisabled(node: HostNode, owner: HandlerRef): boolean {
+  if (owner.props['disabled'] === true) return true;
+  const a11y = node.props['accessibilityState'] as { disabled?: unknown } | undefined;
+  return a11y?.disabled === true || node.props['aria-disabled'] === true;
+}
+
+function call(owner: HandlerRef, name: string, event: unknown): void {
+  const fn = owner.props[name];
+  if (typeof fn === 'function') (fn as (e: unknown) => unknown)(event);
+}
+
+/** A tap on a switch / checkbox without onPress toggles it through `onValueChange` (what the native control does). */
+function toggle(node: HostNode): boolean {
+  const owner = node.findHandler('onValueChange');
+  if (owner === null || typeof owner.props['value'] !== 'boolean') return false;
+  if (owner.props['disabled'] === true) return true;
+  owner.fn(!owner.props['value']);
+  return true;
+}
+
+function press(node: HostNode, handlerName: 'onPress' | 'onLongPress', action: JsAction, ctx: JsActionContext, at?: Point): void {
+  const owner = node.findHandler(handlerName);
+  if (owner === null && handlerName === 'onPress' && toggle(node)) return;
+  if (owner === null) throw notSupported(action, ctx, `no ${handlerName} handler on the element or its ancestors (gesture-handler gestures need native mode)`);
+  if (isDisabled(node, owner)) return;
+  call(owner, 'onPressIn', pressEvent(node, 'pressIn', at));
+  owner.fn(pressEvent(node, handlerName === 'onPress' ? 'press' : 'longPress', at));
+  call(owner, 'onPressOut', pressEvent(node, 'pressOut', at));
+}
+
+/** The topmost element at the window point `p` (hit testing like a real touch). */
+function elementAt(p: Point, action: JsAction, ctx: JsActionContext): HostNode {
+  const top = ctx.roots === undefined ? null : hitTest(ctx.roots(), p);
+  if (top === null) throw notSupported(action, ctx, `no element at (${Math.round(p.x)}, ${Math.round(p.y)})`);
+  return top;
+}
+
+/** `point` inside `node` → the topmost element there (a child may be on top of the target). */
+function pressAt(node: HostNode, point: Point | undefined, handler: 'onPress' | 'onLongPress', action: JsAction, ctx: JsActionContext): void {
+  if (point === undefined) return press(node, handler, action, ctx);
+  const r = node.measure();
+  const at = { x: (r?.x ?? 0) + point.x, y: (r?.y ?? 0) + point.y };
+  press(elementAt(at, action, ctx), handler, action, ctx, at);
+}
+
+export function tap(node: HostNode, ctx: JsActionContext, point?: Point): void {
+  pressAt(node, point, 'onPress', 'tap', ctx);
+}
+
+export function longPress(node: HostNode, ctx: JsActionContext, point?: Point): void {
+  pressAt(node, point, 'onLongPress', 'longPress', ctx);
+}
+
+export function multiTap(node: HostNode, count: number, ctx: JsActionContext): void {
+  for (let i = 0; i < count; i++) press(node, 'onPress', count === 2 ? 'doubleTap' : 'multiTap', ctx);
+}
+
+export function tapAt(p: Point, long: boolean, ctx: JsActionContext): void {
+  press(elementAt(p, 'tapAt', ctx), long ? 'onLongPress' : 'onPress', 'tapAt', ctx, p);
+}
+
+function changeText(node: HostNode, action: JsAction, next: (current: string) => string, ctx: JsActionContext): void {
+  const owner = node.findHandler('onChangeText') ?? node.findHandler('onChange');
+  if (owner === null) throw notSupported(action, ctx, 'no onChangeText / onChange handler on the element or its ancestors');
+  if (owner.props['editable'] === false || owner.props['readOnly'] === true) return;
+  const v = owner.props['value'] ?? owner.props['defaultValue'];
+  const text = next(typeof v === 'string' ? v : '');
+  call(owner, 'onFocus', { nativeEvent: { text: typeof v === 'string' ? v : '' } });
+  call(owner, 'onChange', { nativeEvent: { text, eventCount: 0, target: 0 } });
+  call(owner, 'onChangeText', text);
+}
+
+export function typeText(node: HostNode, text: string, ctx: JsActionContext): void {
+  changeText(node, 'type', (current) => current + text, ctx);
+}
+
+export function clearText(node: HostNode, ctx: JsActionContext): void {
+  changeText(node, 'clear', () => '', ctx);
+}
+
+export function replaceText(node: HostNode, text: string, ctx: JsActionContext): void {
+  changeText(node, 'replaceText', () => text, ctx);
+}
+
+export function pressKey(node: HostNode, key: KeyName, ctx: JsActionContext): void {
+  const input = node.findHandler('onChangeText') ?? node.findHandler('onSubmitEditing') ?? node.findHandler('onKeyPress');
+  if (input === null) throw notSupported('pressKey', ctx, 'no text input (onChangeText / onSubmitEditing / onKeyPress) on the element or its ancestors');
+  const value = input.props['value'] ?? input.props['defaultValue'];
+  const text = typeof value === 'string' ? value : '';
+  call(input, 'onKeyPress', { nativeEvent: { key: key === 'enter' ? 'Enter' : 'Backspace' } });
+  if (key === 'backspace') {
+    if (typeof input.props['onChangeText'] === 'function' || typeof input.props['onChange'] === 'function') changeText(node, 'pressKey', (t) => t.slice(0, -1), ctx);
+    return;
+  }
+  if (typeof input.props['onSubmitEditing'] === 'function') {
+    call(input, 'onSubmitEditing', { nativeEvent: { text } });
+  } else if (input.props['multiline'] === true) {
+    changeText(node, 'pressKey', (t) => `${t}\n`, ctx);
+  }
+}
+
+export interface Scrollable {
+  /** Current content offset. */
+  offset(): { x: number; y: number };
+  /** Size of the scroll viewport. */
+  viewport(): Rect | null;
+  /** The content container (its size is the scrollable extent). */
+  content(): Rect | null;
+  scrollTo(x: number, y: number): void;
+  /** The native scroll view host (its view tag hides the indicators). */
+  host(): HostNode | null;
+}
+
+/** The ScrollView that contains `node` (or is `node`), or null. */
+export function findScrollable(node: HostNode): Scrollable | null {
+  const inst = node.findInstance('scrollTo');
+  if (inst === null) return null;
+  // ScrollView renders: native scroll view host → content container host → children.
+  const host = (): { scroll: HostNode | null; content: HostNode | null } => {
+    const scroll = inst.hostNodes()[0] ?? null;
+    return { scroll, content: scroll?.children[0] ?? null };
+  };
+  return {
+    offset() {
+      const { scroll, content } = host();
+      const s = scroll?.measure();
+      const c = content?.measure();
+      if (s == null || c == null) return { x: 0, y: 0 };
+      return { x: s.x - c.x, y: s.y - c.y };
+    },
+    viewport: () => host().scroll?.measure() ?? null,
+    content: () => host().content?.measure() ?? null,
+    host: () => host().scroll,
+    scrollTo(x, y) {
+      inst.call({ x: Math.max(0, x), y: Math.max(0, y), animated: false });
+    },
+  };
+}
+
+/** Fraction of the viewport moved by one scroll / swipe step when no distance is given. */
+export const STEP_FRACTION = 0.8;
+
+/** `down` reveals content further down (offset grows), like Detox's scroll direction. */
+export function scrollBy(s: Scrollable, direction: Direction, distance?: number): void {
+  const vp = s.viewport();
+  const vertical = direction === 'up' || direction === 'down';
+  const d = distance ?? (vp === null ? 300 : (vertical ? vp.height : vp.width) * STEP_FRACTION);
+  const { x, y } = s.offset();
+  const sign = direction === 'down' || direction === 'right' ? 1 : -1;
+  s.scrollTo(vertical ? x : x + sign * d, vertical ? y + sign * d : y);
+}
+
+/** Jump to an edge of the current content (a virtualized list may grow afterwards; callers repeat). */
+export function scrollToEdge(s: Scrollable, edge: Edge): void {
+  const vp = s.viewport();
+  const content = s.content();
+  const { x, y } = s.offset();
+  const maxX = vp === null || content === null ? x : Math.max(0, content.width - vp.width);
+  const maxY = vp === null || content === null ? y : Math.max(0, content.height - vp.height);
+  if (edge === 'top') s.scrollTo(x, 0);
+  else if (edge === 'bottom') s.scrollTo(x, maxY);
+  else if (edge === 'left') s.scrollTo(0, y);
+  else s.scrollTo(maxX, y);
+}
+
+/** A swipe moves the finger; the content scrolls the opposite way (swipe up = scroll down). */
+export function swipeToScrollDirection(direction: Direction): Direction {
+  return ({ up: 'down', down: 'up', left: 'right', right: 'left' } as const)[direction];
+}
+
+export function requireScrollable(node: HostNode, action: 'scroll' | 'swipe' | 'scrollTo', ctx: JsActionContext): Scrollable {
+  const s = findScrollable(node);
+  if (s === null) throw notSupported(action, ctx, 'the element is not inside a ScrollView / FlatList');
+  return s;
+}

package/src/actions/multitouch.ts

@@ -0,0 +1,179 @@
+/**
+ * Multi-finger gestures (Phase 6.4): finger paths, pinch / rotate geometry and the frame format the
+ * native modules and the web host play back.
+ *
+ * Frames (flat numbers, `1 + 3 × fingers` per frame): `[tMs, down0, x0, y0, down1, x1, y1, ...]`, window
+ * coordinates, t from 0. A finger goes down in the first frame it is down and lifts in the first frame
+ * it is not; the last frame has every finger up (players also lift any finger still down after it).
+ */
+import type { Point, Rect, Speed } from '@quiesco/protocol';
+
+import type { Viewport } from '../fiber-adapter';
+
+/** One finger: window points with the time (ms from the gesture's start) it reaches them. */
+export type FingerPath = [Point, number][];
+
+/** Frame interval: one per display frame, like a touchscreen's report rate. */
+export const FRAME_MS = 16;
+/** Fingers rest before moving (the recognizers see both touches) and before lifting (no fling). */
+export const PRESS_MS = 80;
+export const REST_MS = 150;
+/** Fingers closer than this merge on a real screen; pinches keep them apart. */
+export const MIN_RADIUS = 20;
+/** Keep fingers this far inside the window. */
+const EDGE = 8;
+
+const PINCH_MS: Record<Speed, number> = { slow: 1200, normal: 600, fast: 300 };
+/** Rotation time per radian. */
+const ROTATE_MS_PER_RAD: Record<Speed, number> = { slow: 800, normal: 400, fast: 200 };
+
+function at(path: FingerPath, t: number): Point {
+  const first = path[0] as [Point, number];
+  if (t <= first[1]) return first[0];
+  for (let k = 1; k < path.length; k++) {
+    const [b, tb] = path[k] as [Point, number];
+    if (t <= tb) {
+      const [a, ta] = path[k - 1] as [Point, number];
+      const f = tb === ta ? 1 : (t - ta) / (tb - ta);
+      return { x: a.x + (b.x - a.x) * f, y: a.y + (b.y - a.y) * f };
+    }
+  }
+  return (path[path.length - 1] as [Point, number])[0];
+}
+
+/** Frames for fingers that may start and end at different times (each is down from its first to its last point). */
+export function framesFor(fingers: readonly FingerPath[], stepMs = FRAME_MS): number[] {
+  const spans = fingers.map((p) => [(p[0] as [Point, number])[1], (p[p.length - 1] as [Point, number])[1]] as const);
+  const times = new Set<number>();
+  const start = Math.min(...spans.map((s) => s[0]));
+  const end = Math.max(...spans.map((s) => s[1]));
+  for (let t = start; t < end; t += stepMs) times.add(Math.round(t));
+  for (const [s, e] of spans) {
+    times.add(s);
+    times.add(e);
+    times.add(e + 1); // the frame where a finger that ends early lifts
+  }
+  const out: number[] = [];
+  for (const t of [...times].sort((a, b) => a - b)) {
+    if (t > end + 1) continue;
+    out.push(t - start);
+    fingers.forEach((path, i) => {
+      const [s, e] = spans[i] as readonly [number, number];
+      if (t < s || t > e) {
+        out.push(0, 0, 0);
+      } else {
+        const p = at(path, t);
+        out.push(1, p.x, p.y);
+      }
+    });
+  }
+  return out;
+}
+
+/** Gestures one after another (all fingers up in between, `gapMs` apart), as one frame list. */
+export function framesForSequence(gestures: readonly (readonly FingerPath[])[], gapMs = 120): number[] {
+  const n = Math.max(...gestures.map((g) => g.length));
+  const out: number[] = [];
+  let offset = 0;
+  for (const g of gestures) {
+    const frames = framesFor(g);
+    const own = 1 + 3 * g.length;
+    let last = 0;
+    for (let k = 0; k < frames.length; k += own) {
+      last = (frames[k] as number) + offset;
+      out.push(last);
+      for (let i = 0; i < n; i++) {
+        if (i < g.length) out.push(frames[k + 1 + 3 * i] as number, frames[k + 2 + 3 * i] as number, frames[k + 3 + 3 * i] as number);
+        else out.push(0, 0, 0);
+      }
+    }
+    // framesFor ends with every finger up (the frame after the last end).
+    offset = last + gapMs;
+  }
+  return out;
+}
+
+const center = (r: Rect): Point => ({ x: r.x + r.width / 2, y: r.y + r.height / 2 });
+
+/** How far from `c` a finger can go along direction (dx, dy) and stay in the window. */
+function reach(c: Point, dx: number, dy: number, viewport: Viewport | null): number {
+  if (viewport === null) return Number.POSITIVE_INFINITY;
+  let r = Number.POSITIVE_INFINITY;
+  for (const [d, pos, size] of [
+    [dx, c.x, viewport.width],
+    [dy, c.y, viewport.height],
+  ] as const) {
+    if (d > 1e-9) r = Math.min(r, (size - EDGE - pos) / d);
+    else if (d < -1e-9) r = Math.min(r, (pos - EDGE) / -d);
+  }
+  return r;
+}
+
+/** Half the element's extent along the direction, inset: where fingers can go down on it. */
+function inside(rect: Rect, dx: number, dy: number): number {
+  const hx = Math.abs(dx) < 1e-9 ? Number.POSITIVE_INFINITY : rect.width / 2 / Math.abs(dx);
+  const hy = Math.abs(dy) < 1e-9 ? Number.POSITIVE_INFINITY : rect.height / 2 / Math.abs(dy);
+  return Math.min(hx, hy) * 0.85;
+}
+
+function twoFingers(c: Point, dx: number, dy: number, rStart: number, r0: number, r1: number, moveMs: number): FingerPath[] {
+  const p = (r: number, s: number): Point => ({ x: c.x + s * r * dx, y: c.y + s * r * dy });
+  // Lead-in (rStart → r0) at the same speed as the counted part.
+  const leadMs = Math.abs(r1 - r0) < 1e-9 ? 0 : (moveMs * Math.abs(r0 - rStart)) / Math.abs(r1 - r0);
+  return [1, -1].map((s) => [
+    [p(rStart, s), 0],
+    [p(rStart, s), PRESS_MS],
+    [p(r0, s), PRESS_MS + leadMs],
+    [p(r1, s), PRESS_MS + leadMs + moveMs],
+    [p(r1, s), PRESS_MS + leadMs + moveMs + REST_MS],
+  ]);
+}
+
+/**
+ * Pinches that scale the distance between two fingers by `scale` in total, centered on the element, the
+ * fingers on a line at `angle` (radians, 0 = horizontal). The fingers go down on the element and stay in
+ * the window; a factor that does not fit in one pinch is split into equal pinches (as a user repeats).
+ * `slop`: the distance change before the app's recognizer activates (and starts measuring); each pinch
+ * moves that much first, so the recognizer reports `scale`.
+ */
+export function pinchGestures(rect: Rect, scale: number, angle: number, speed: Speed, viewport: Viewport | null, slop = 0): FingerPath[][] {
+  const c = center(rect);
+  const dx = Math.cos(angle);
+  const dy = Math.sin(angle);
+  const half = inside(rect, dx, dy);
+  const maxR = Math.min(reach(c, dx, dy, viewport), reach(c, -dx, -dy, viewport));
+  const lead = slop / 2; // per finger
+  if (half < MIN_RADIUS * 1.2 + lead || maxR < MIN_RADIUS * 1.2 + lead) {
+    throw new RangeError(`the element (${Math.round(rect.width)}×${Math.round(rect.height)}) is too small for two fingers`);
+  }
+  // Zoom in: start close together on the element; zoom out: start wide apart on it.
+  const zoomIn = scale >= 1;
+  const rStart = zoomIn ? Math.max(MIN_RADIUS, half * 0.4 - lead) : Math.min(half, maxR);
+  const r0 = zoomIn ? rStart + lead : rStart - lead;
+  const limit = zoomIn ? maxR / r0 : MIN_RADIUS / r0;
+  const n = Math.max(1, Math.ceil(Math.log(scale) / Math.log(limit) - 1e-9));
+  const f = scale ** (1 / n);
+  const moveMs = PINCH_MS[speed];
+  return Array.from({ length: n }, () => twoFingers(c, dx, dy, rStart, r0, r0 * f, moveMs));
+}
+
+/**
+ * Two fingers turning `radians` (positive: clockwise on screen) around the element's center, on a
+ * circle that fits the element; sampled every frame so the fingers follow the arc. `slop`: the
+ * rotation before the app's recognizer activates, added to the turn.
+ */
+export function rotateGesture(rect: Rect, radians: number, speed: Speed, slop = 0): FingerPath[] {
+  const c = center(rect);
+  const r = Math.max(MIN_RADIUS * 1.5, Math.min(rect.width, rect.height) / 2 * 0.6);
+  // The recognizer measures from where it activates: turn `slop` more (same direction) in total.
+  const total = radians + Math.sign(radians) * slop;
+  const moveMs = Math.max(250, Math.abs(total) * ROTATE_MS_PER_RAD[speed]);
+  const steps = Math.max(2, Math.ceil(moveMs / FRAME_MS));
+  return [0, Math.PI].map((phase) => {
+    const p = (a: number): Point => ({ x: c.x + r * Math.cos(phase + a), y: c.y + r * Math.sin(phase + a) });
+    const path: FingerPath = [[p(0), 0]];
+    for (let k = 0; k <= steps; k++) path.push([p((total * k) / steps), PRESS_MS + (moveMs * k) / steps]);
+    path.push([p(total), PRESS_MS + moveMs + REST_MS]);
+    return path;
+  });
+}