@fabi-play/motion 0.4.0 → 0.7.0

package/README.md

@@ -16,6 +16,21 @@ const shape = shapeOf(body) // 'star', 'arms-up', … or null
 const { left, right } = hands(body)
 ```
 
+## System gestures and menus
+
+- `armsCrossed(body)`: the arms crossed in an X, Fabi Play's own gesture (held 1.5 s by the app: pause, exit), so
+  games never get it as a move and `shapeOf` is null while it is made. It counts in front of the chest and as a big
+  X above the head: each wrist past the middle of the body to the other side, the two forearms (elbow to wrist)
+  crossing, rising from the elbows; above the head, with both arms stretched. Clapping, arms up, folded arms, a
+  self-hug and hands on the head are not it. `crossOf(body)` says which X: `'chest'`, `'overhead'` or null. The
+  thresholds are in `CROSS`, tuned on a child's proportions. It reads one frame: when holding it, also want it still
+  (arms waving above the head cross now and then).
+- `armOut(body)`: one arm stretched out sideways at shoulder height, `'left'` or `'right'` as seen on the screen
+  (player space is a mirror: the side of the stretched wrist), for previous / next in menus. Null when both arms
+  are out (a star, arms wide, also with one drooping) and while the arms are crossed. The thresholds are in
+  `ARM_OUT`, in arm lengths.
+- `menuPointer(body)`: the raised hand as a cursor; `Hold` and `Dwell` for holding a gesture or staying on a button.
+
 Games get these movements as events from [`@fabi-play/sdk`](https://www.npmjs.com/package/@fabi-play/sdk); this
 package is for tools and for the Fabi Play apps. ESM only, types included; depends on `@fabi-play/protocol`.
 Docs: https://docs.fabiplay.app.

package/dist/index.d.ts

@@ -229,43 +229,96 @@ export declare function skeleton(body: Body$1): {
 export type Skeleton = ReturnType<typeof skeleton>;
 /** The shape the arms (and legs) make, or null when it's none of them. */
 export declare function shapeOf(body: Body$1): Shape | null;
+/** Where the arms make the X: in front of the chest, or big above the head. */
+export type Cross = "chest" | "overhead";
 /**
- * Arms crossed in an X in front of the chest: Fabi Play's system gesture (held 1.5 s: pause, exit), reserved, so
- * games never get it as a move. In player space, mirrored: the child's left shoulder is on the left of the image, so
- * crossing means the left wrist past the midline towards the right shoulder and the right wrist towards the left.
- * The side is read from the shoulders, not assumed. Measured in torsos (shoulders to hips; estimated from the
- * shoulders when the hips are out of view).
+ * Arms crossed in an X, in front of the chest or big above the head: Fabi Play's system gesture (held 1.5 s: pause,
+ * exit), reserved, so games never get it as a move. In player space, mirrored: the child's left shoulder is on the
+ * left of the image, so crossing means the left wrist past the midline towards the right shoulder and the right
+ * wrist towards the left. The side is read from the shoulders, not assumed. See crossOf for the rule.
+ *
+ * One frame at a time: arms waving above the head cross now and then. Whoever holds the gesture (1.5 s) should also
+ * want it still.
  */
 export declare function armsCrossed(body: Body$1): boolean;
-/** The thresholds of armsCrossed, in torsos (t: shoulders to hips). */
+/**
+ * Which X the arms make: 'chest', 'overhead', or null when they are not crossed (armsCrossed is crossOf !== null).
+ * The rule: the body faces the camera, each wrist went past the midline to the other side, and the two forearms
+ * (elbow to wrist) cross, rising from the elbows. Where they cross tells the X: from mid-chest up to the chin it is
+ * the chest X (elbows by the sides, not in front of the chest: a self-hug; hands not on the other arm); with both
+ * arms stretched up, the wrists clearly above the top of the head and the elbows above the nose, the overhead X
+ * (hands on the head have the elbows bent out and the wrists at the top of the head). Anything else is neither.
+ *
+ * Heights are in torsos (shoulders to hips; estimated from the shoulders when the hips are out of view), sideways
+ * distances in half shoulder widths, and the forearm's rise in forearms, so long and short arms read the same.
+ */
+export declare function crossOf(body: Body$1): Cross | null;
+/**
+ * The thresholds of crossOf and armsCrossed. Heights in torsos (t: shoulders to hips), sideways distances in half
+ * shoulder widths (h: shoulder to midline). A child (DISTANCE): h ≈ 0.38 t, nose ≈ 0.3 t and the top of the head
+ * ≈ 0.7 t above the shoulder line, upper arm ≈ 0.6 t, forearm ≈ 0.5 t.
+ */
 export declare const CROSS: {
-	/** Each wrist at least this far past the midline (between the shoulders)… */
-	readonly past: 0.12;
-	/** …and no farther than the opposite shoulder plus this (a wide X; self-hugs are told apart by the elbows). */
-	readonly pastShoulder: 0.5;
-	/** Each elbow at least this far from the midline, on its own side (in a self-hug they come in front of the chest). */
-	readonly elbowOut: 0.38;
+	/** Each wrist at least this far past the midline, on the other side (h). */
+	readonly past: 0.2;
+	/** Each wrist above its own elbow by at least this fraction of the forearm (about 15°: folded arms are level). */
+	readonly rise: 0.25;
+	/** The forearms count this much longer past the wrists (t: the hands), so wrists that only just cross still do. */
+	readonly hand: 0.08;
+	/** The chest X: the forearms cross from this far above the shoulder line (t: under the chin; not the face)… */
+	readonly chestTop: 0.1;
+	/** …down to this far below it (t: wrists at mid-chest, the forearms crossing a little lower). */
+	readonly chestBottom: 0.75;
+	/** The chest X: each elbow at least this far from the midline, on its own side (h)… */
+	readonly elbowOut: 0.3;
 	/**
-	 * Wrists between this far above the shoulder line (an X up to the chin and mouth; not higher: hands on the head and
-	 * forearms over the eyes, a child hiding the face, are not it)…
+	 * …and no wrist farther past the midline than this many times the nearer elbow is out (a self-hug: elbows together
+	 * in front of the chest, hands out on the other shoulder; in an X the wrists only just pass each other).
 	 */
-	readonly above: 0.6;
-	/** …and this far below it (mid-chest). */
-	readonly below: 0.7;
-	/** Each wrist at least this far above its own elbow (the forearm rises)… */
-	readonly rise: 0.15;
-	/** …and this far above the other arm's elbow. */
-	readonly overElbow: 0.1;
-	/** Elbows no higher than this above the shoulder line (arms crossed over the head are not it)… */
-	readonly elbowAbove: 0.15;
-	/** …and no lower than this below the hip line. */
+	readonly hugRatio: 1.5;
+	/** The chest X: no wrist farther than this past the midline (h: the opposite shoulder; hands on the upper arms are a hug). */
+	readonly reach: 1;
+	/** The chest X: elbows no lower than this below the hip line (t). */
 	readonly elbowBelowHips: 0.3;
+	/** The overhead X: the forearms cross at least this far above the shoulder line (t: about the top of the head)… */
+	readonly overShoulders: 0.55;
+	/** …both wrists at least this far above the shoulder line (t)… */
+	readonly wristsOverShoulders: 0.8;
+	/** …and this far above the nose (t), the elbows above the nose… */
+	readonly wristsOverNose: 0.45;
+	/** …and both arms this straight: shoulder to wrist over shoulder to elbow to wrist (hands on the head: ≈ 0.88). */
+	readonly straight: 0.93;
+	/** The nose above the shoulder line when it is not seen (t). */
+	readonly noseAbove: 0.33;
 	/** The visibility the wrist hidden behind the other needs; the other wrist, elbows and shoulders need the usual 0.5. */
 	readonly hiddenWristVisible: 0.25;
-	/** Shoulders at least this wide: facing the camera. */
+	/** Shoulders at least this wide (t): facing the camera. */
+	readonly minShoulderWidth: 0.3;
+	/** Shoulder width over torso (DISTANCE.shoulders / DISTANCE.torso), to estimate the torso when the hips are out of view. */
+	readonly shoulderPerTorso: 0.76;
+};
+/**
+ * One arm stretched out sideways, about level with the shoulder (for menus: previous / next, a left or right
+ * button). The side is the screen side of the stretched wrist, in player space (mirrored, as the child sees it): a
+ * wrist left of the shoulders' middle is 'left'. The arm must be on its own side (an arm reaching across the body is
+ * not out), its wrist far past its shoulder and near the shoulder's height, and fairly straight (the elbow between
+ * shoulder and wrist, near the line from one to the other). Null when no arm is out, when the other wrist is out
+ * sideways too (a star, arms wide, also with one arm drooping), and while the arms are crossed. Measured in the arm's
+ * own length (shoulder to elbow to wrist, as the camera sees it), so long and short arms read the same (see ARM_OUT).
+ */
+export declare function armOut(body: Body$1): "left" | "right" | null;
+/** The thresholds of armOut, in arm lengths (shoulder to elbow to wrist, as seen; a child's is ≈ 1.1 torsos). */
+export declare const ARM_OUT: {
+	/** The wrist at least this far out past its shoulder, sideways (straight out: 1; 45° up or down: 0.7). */
+	readonly reach: 0.6;
+	/** The wrist no farther than this above or below its shoulder (about 25°). */
+	readonly height: 0.4;
+	/** The elbow no farther than this above or below the line from shoulder to wrist (fairly straight). */
+	readonly bend: 0.2;
+	/** The other wrist reaching less than this out past its shoulder, at any height: not a star with one arm drooping. */
+	readonly otherReach: 0.35;
+	/** Shoulders at least this wide, in torsos: facing the camera. */
 	readonly minShoulderWidth: 0.3;
-	/** Shoulder width over torso, to estimate the torso when the hips are out of view. */
-	readonly shoulderPerTorso: 0.8;
 };
 /**
  * Something held for a while: progress 0–1 while active, fires once when full. Brief drop-outs (under graceMs, e.g.
@@ -328,8 +381,9 @@ export type Placement = "no-body" | "too-close" | "too-far" | "go-left" | "go-ri
  */
 export declare function placement(body: Body$1 | null, center?: number, radius?: number): Placement;
 /**
- * The hands as pointers on the screen (0–1): the space the arms reach, around the shoulders, stretched to the
- * whole screen, so a child reaches every corner without walking.
+ * The hands as pointers on the screen (0–1): the space the arms reach, around the shoulders (HAND_SPACE of
+ * @fabi-play/protocol), stretched to the whole screen, so a child reaches every corner without walking. The same
+ * stretch as the protocol's aimPoint in portrait (which games get from play.aim and engine.hands()).
  */
 export declare function hands(body: Body$1): {
 	left: Point | null;

package/dist/index.js

@@ -3,7 +3,7 @@ var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { en
 var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
 
 // src/index.ts
-import { P as P2 } from "@fabi-play/protocol";
+import { HAND_SPACE, P as P2 } from "@fabi-play/protocol";
 
 // src/distance.ts
 import { P } from "@fabi-play/protocol";
@@ -341,58 +341,133 @@ function shapeIn(s) {
 function armsCrossed(body) {
   return crossedIn(skeleton(body));
 }
-function crossedIn(s) {
-  if (!seen(s.leftShoulder, s.rightShoulder, s.leftElbow, s.rightElbow)) return false;
+function crossOf(body) {
+  return crossIn(skeleton(body));
+}
+var crossedIn = (s) => crossIn(s) !== null;
+var torsoOf = (s) => seen(s.leftHip, s.rightHip) ? s.torso : s.shoulderWidth / CROSS.shoulderPerTorso;
+var straightness = (shoulder, elbow, wrist) => dist(shoulder, wrist) / Math.max(dist(shoulder, elbow) + dist(elbow, wrist), 1e-6);
+function crossIn(s) {
+  if (!seen(s.leftShoulder, s.rightShoulder, s.leftElbow, s.rightElbow)) return null;
   const wv = [s.leftWrist.visibility, s.rightWrist.visibility];
-  if (Math.min(...wv) < CROSS.hiddenWristVisible || Math.max(...wv) < VISIBLE) return false;
-  const hipsSeen = seen(s.leftHip, s.rightHip);
-  const t = hipsSeen ? s.torso : s.shoulderWidth / CROSS.shoulderPerTorso;
-  if (s.shoulderWidth < CROSS.minShoulderWidth * t) return false;
+  if (Math.min(...wv) < CROSS.hiddenWristVisible || Math.max(...wv) < VISIBLE) return null;
+  const t = torsoOf(s);
+  if (s.shoulderWidth < CROSS.minShoulderWidth * t) return null;
   const dir = Math.sign(s.rightShoulder.x - s.leftShoulder.x);
   const m = s.shoulders.x;
+  const half = s.shoulderWidth / 2;
+  const pastL = dir * (s.leftWrist.x - m) / half;
+  const pastR = dir * (m - s.rightWrist.x) / half;
+  if (pastL < CROSS.past || pastR < CROSS.past) return null;
+  const rises = (e, w) => e.y - w.y >= CROSS.rise * dist(e, w);
+  if (!rises(s.leftElbow, s.leftWrist) || !rises(s.rightElbow, s.rightWrist)) return null;
+  const at = forearmsCross(s.leftElbow, s.leftWrist, s.rightElbow, s.rightWrist, CROSS.hand * t);
+  if (!at) return null;
   const top = s.shoulders.y;
-  const bottom = hipsSeen ? s.hips.y : top + t;
-  const crossL = dir * (s.leftWrist.x - m) / t;
-  const crossR = dir * (m - s.rightWrist.x) / t;
-  const elbowL = dir * (m - s.leftElbow.x) / t;
-  const elbowR = dir * (s.rightElbow.x - m) / t;
-  const reach = 0.5 * s.shoulderWidth / t + CROSS.pastShoulder;
-  if (crossL < CROSS.past || crossR < CROSS.past || crossL > reach || crossR > reach) return false;
-  if (elbowL < CROSS.elbowOut || elbowR < CROSS.elbowOut) return false;
-  for (const w of [s.leftWrist, s.rightWrist]) if (w.y < top - CROSS.above * t || w.y > top + CROSS.below * t) return false;
-  for (const e of [s.leftElbow, s.rightElbow]) if (e.y < top - CROSS.elbowAbove * t || e.y > bottom + CROSS.elbowBelowHips * t) return false;
-  const rises = (w, e) => (e.y - w.y) / t >= CROSS.rise;
-  const clear = (w, otherElbow) => (otherElbow.y - w.y) / t >= CROSS.overElbow;
-  return rises(s.leftWrist, s.leftElbow) && rises(s.rightWrist, s.rightElbow) && clear(s.leftWrist, s.rightElbow) && clear(s.rightWrist, s.leftElbow);
+  const up = (top - at.y) / t;
+  if (up <= CROSS.chestTop && up >= -CROSS.chestBottom) {
+    const elbowOut = Math.min(dir * (m - s.leftElbow.x), dir * (s.rightElbow.x - m)) / half;
+    const past = Math.max(pastL, pastR);
+    const bottom = seen(s.leftHip, s.rightHip) ? s.hips.y : top + t;
+    const elbowsLow = Math.max(s.leftElbow.y, s.rightElbow.y) > bottom + CROSS.elbowBelowHips * t;
+    return elbowOut >= CROSS.elbowOut && past <= CROSS.hugRatio * elbowOut && past <= CROSS.reach && !elbowsLow ? "chest" : null;
+  }
+  if (up < CROSS.overShoulders) return null;
+  const nose = s.nose.visibility >= VISIBLE ? s.nose.y : top - CROSS.noseAbove * t;
+  const wristsY = Math.max(s.leftWrist.y, s.rightWrist.y);
+  if ((top - wristsY) / t < CROSS.wristsOverShoulders || (nose - wristsY) / t < CROSS.wristsOverNose) return null;
+  if (Math.max(s.leftElbow.y, s.rightElbow.y) > nose) return null;
+  const straight = Math.min(straightness(s.leftShoulder, s.leftElbow, s.leftWrist), straightness(s.rightShoulder, s.rightElbow, s.rightWrist));
+  return straight >= CROSS.straight ? "overhead" : null;
+}
+function forearmsCross(e1, w1, e2, w2, hand) {
+  const reach = (e, w) => {
+    const k = 1 + hand / Math.max(dist(e, w), 1e-6);
+    return { x: (w.x - e.x) * k, y: (w.y - e.y) * k };
+  };
+  const r = reach(e1, w1);
+  const q = reach(e2, w2);
+  const den = r.x * q.y - r.y * q.x;
+  if (Math.abs(den) < 1e-12) return null;
+  const dx = e2.x - e1.x;
+  const dy = e2.y - e1.y;
+  const a = (dx * q.y - dy * q.x) / den;
+  const b = (dx * r.y - dy * r.x) / den;
+  if (a < 0 || a > 1 || b < 0 || b > 1) return null;
+  return { x: e1.x + a * r.x, y: e1.y + a * r.y };
 }
 var CROSS = {
-  /** Each wrist at least this far past the midline (between the shoulders)… */
-  past: 0.12,
-  /** …and no farther than the opposite shoulder plus this (a wide X; self-hugs are told apart by the elbows). */
-  pastShoulder: 0.5,
-  /** Each elbow at least this far from the midline, on its own side (in a self-hug they come in front of the chest). */
-  elbowOut: 0.38,
+  /** Each wrist at least this far past the midline, on the other side (h). */
+  past: 0.2,
+  /** Each wrist above its own elbow by at least this fraction of the forearm (about 15°: folded arms are level). */
+  rise: 0.25,
+  /** The forearms count this much longer past the wrists (t: the hands), so wrists that only just cross still do. */
+  hand: 0.08,
+  /** The chest X: the forearms cross from this far above the shoulder line (t: under the chin; not the face)… */
+  chestTop: 0.1,
+  /** …down to this far below it (t: wrists at mid-chest, the forearms crossing a little lower). */
+  chestBottom: 0.75,
+  /** The chest X: each elbow at least this far from the midline, on its own side (h)… */
+  elbowOut: 0.3,
   /**
-   * Wrists between this far above the shoulder line (an X up to the chin and mouth; not higher: hands on the head and
-   * forearms over the eyes, a child hiding the face, are not it)…
+   * …and no wrist farther past the midline than this many times the nearer elbow is out (a self-hug: elbows together
+   * in front of the chest, hands out on the other shoulder; in an X the wrists only just pass each other).
    */
-  above: 0.6,
-  /** …and this far below it (mid-chest). */
-  below: 0.7,
-  /** Each wrist at least this far above its own elbow (the forearm rises)… */
-  rise: 0.15,
-  /** …and this far above the other arm's elbow. */
-  overElbow: 0.1,
-  /** Elbows no higher than this above the shoulder line (arms crossed over the head are not it)… */
-  elbowAbove: 0.15,
-  /** …and no lower than this below the hip line. */
+  hugRatio: 1.5,
+  /** The chest X: no wrist farther than this past the midline (h: the opposite shoulder; hands on the upper arms are a hug). */
+  reach: 1,
+  /** The chest X: elbows no lower than this below the hip line (t). */
   elbowBelowHips: 0.3,
+  /** The overhead X: the forearms cross at least this far above the shoulder line (t: about the top of the head)… */
+  overShoulders: 0.55,
+  /** …both wrists at least this far above the shoulder line (t)… */
+  wristsOverShoulders: 0.8,
+  /** …and this far above the nose (t), the elbows above the nose… */
+  wristsOverNose: 0.45,
+  /** …and both arms this straight: shoulder to wrist over shoulder to elbow to wrist (hands on the head: ≈ 0.88). */
+  straight: 0.93,
+  /** The nose above the shoulder line when it is not seen (t). */
+  noseAbove: 0.33,
   /** The visibility the wrist hidden behind the other needs; the other wrist, elbows and shoulders need the usual 0.5. */
   hiddenWristVisible: 0.25,
-  /** Shoulders at least this wide: facing the camera. */
+  /** Shoulders at least this wide (t): facing the camera. */
   minShoulderWidth: 0.3,
-  /** Shoulder width over torso, to estimate the torso when the hips are out of view. */
-  shoulderPerTorso: 0.8
+  /** Shoulder width over torso (DISTANCE.shoulders / DISTANCE.torso), to estimate the torso when the hips are out of view. */
+  shoulderPerTorso: 0.76
+};
+function armOut(body) {
+  const s = skeleton(body);
+  if (!seen(s.leftShoulder, s.rightShoulder) || crossedIn(s)) return null;
+  if (s.shoulderWidth < ARM_OUT.minShoulderWidth * torsoOf(s)) return null;
+  const m = s.shoulders.x;
+  const arm = (shoulder, elbow, wrist) => {
+    if (!seen(elbow, wrist)) return { reach: 0, out: false };
+    const length = Math.max(dist(shoulder, elbow) + dist(elbow, wrist), 1e-6);
+    const side = Math.sign(shoulder.x - m);
+    const from = (p) => side * (p.x - m);
+    const reach = (from(wrist) - from(shoulder)) / length;
+    if (reach < ARM_OUT.reach || Math.abs(wrist.y - shoulder.y) / length > ARM_OUT.height) return { reach, out: false };
+    const k = (from(elbow) - from(shoulder)) / (reach * length);
+    const bend = Math.abs(elbow.y - (shoulder.y + k * (wrist.y - shoulder.y))) / length;
+    return { reach, out: k >= 0 && k <= 1 && bend <= ARM_OUT.bend };
+  };
+  const left = arm(s.leftShoulder, s.leftElbow, s.leftWrist);
+  const right = arm(s.rightShoulder, s.rightElbow, s.rightWrist);
+  const one = left.out && right.reach < ARM_OUT.otherReach ? s.leftWrist : right.out && left.reach < ARM_OUT.otherReach ? s.rightWrist : null;
+  if (!one) return null;
+  return one.x < m ? "left" : "right";
+}
+var ARM_OUT = {
+  /** The wrist at least this far out past its shoulder, sideways (straight out: 1; 45° up or down: 0.7). */
+  reach: 0.6,
+  /** The wrist no farther than this above or below its shoulder (about 25°). */
+  height: 0.4,
+  /** The elbow no farther than this above or below the line from shoulder to wrist (fairly straight). */
+  bend: 0.2,
+  /** The other wrist reaching less than this out past its shoulder, at any height: not a star with one arm drooping. */
+  otherReach: 0.35,
+  /** Shoulders at least this wide, in torsos: facing the camera. */
+  minShoulderWidth: 0.3
 };
 var HOLD_DRAIN = 0.5;
 var Hold = class {
@@ -512,7 +587,7 @@ function handSpace(body) {
   return spaceIn(skeleton(body));
 }
 function spaceIn(s) {
-  const box = { x: s.shoulders.x - 1.6 * s.torso, y: s.shoulders.y - 1.6 * s.torso, w: 3.2 * s.torso, h: 3 * s.torso };
+  const box = { x: s.shoulders.x - HAND_SPACE.left * s.torso, y: s.shoulders.y - HAND_SPACE.top * s.torso, w: HAND_SPACE.w * s.torso, h: HAND_SPACE.h * s.torso };
   return (p) => ({ x: (p.x - box.x) / box.w, y: (p.y - box.y) / box.h });
 }
 var MotionTracker = class {
@@ -684,6 +759,7 @@ function nearest(from, free, anchors, limit, pair) {
   }
 }
 export {
+  ARM_OUT,
   BodyTracker,
   CROSS,
   DISTANCE,
@@ -693,8 +769,10 @@ export {
   MotionTracker,
   RAISED,
   TooCloseGuard,
+  armOut,
   armsCrossed,
   bodyAtDistance,
+  crossOf,
   defaultFov,
   estimateDistance,
   handSpace,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fabi-play/motion",
-  "version": "0.4.0",
+  "version": "0.7.0",
   "description": "Body movements from pose landmarks: jumps, squats, shapes, lanes, freezing, hands as pointers.",
   "keywords": [
     "fabi-play",
@@ -32,7 +32,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@fabi-play/protocol": "^0.4.0"
+    "@fabi-play/protocol": "^0.7.0"
   },
   "devDependencies": {
     "dts-bundle-generator": "9.5.1",