@gtanczyk/diorama 0.1.0 → 0.2.0

package/app/web/main.ts

@@ -35,7 +35,7 @@ type Data = {
   world: World;
   places: Record<string, { geometry: Geometry | null; painting: string | null; backdrop?: string; layers?: Layers; weather?: Weather; reactions?: Reactions; glows?: Glow[]; issue?: string }>;
   /** Figure per hero or NPC key: a cut-out image, and its rig when `diorama cast` could cut it into parts. */
-  cast: Record<string, { image: string; height?: number; upright?: boolean; face?: { dir: string; info: FaceDefinition }; rig?: Rig; parts?: string; model?: { dir: string; info: ModelInfo }; sprites?: { dir: string; info: SpriteInfo } }>;
+  cast: Record<string, { image: string; height?: number; upright?: boolean; share?: number; face?: { dir: string; info: FaceDefinition }; rig?: Rig; parts?: string; model?: { dir: string; info: ModelInfo }; sprites?: { dir: string; info: SpriteInfo } }>;
   story: Story | null;
   /** Index into story.changes → the painting as it looks once that change holds. */
   changes?: Record<string, string>;
@@ -70,7 +70,7 @@ export type SpriteInfo = {
 /** walk: the cells a walk steps through, without the ones the sprites step found off the gait (a crouch, a sit). */
 type Sheet = { file: string; yaw: number; stand: number; img: HTMLImageElement; gestures: string[]; walk: number[] };
 /** height: metres standing (a grown-up 1.62, a cat 0.3), so each is drawn at its own size; upright false: an animal, drawn as painted. */
-type Figure = { img: HTMLImageElement; face?: PaintedFace; faceInfo?: FaceDefinition; height?: number; upright?: boolean; rig?: { rig: Rig; parts: Map<PartName, HTMLImageElement>; sheet?: HTMLImageElement; canvas: HTMLCanvasElement }; model?: Figure3D; sprites?: { info: SpriteInfo; sheets: Sheet[]; talk: Sheet[] } };
+type Figure = { img: HTMLImageElement; face?: PaintedFace; faceInfo?: FaceDefinition; height?: number; upright?: boolean; share?: number; rig?: { rig: Rig; parts: Map<PartName, HTMLImageElement>; sheet?: HTMLImageElement; canvas: HTMLCanvasElement }; model?: Figure3D; sprites?: { info: SpriteInfo; sheets: Sheet[]; talk: Sheet[] } };
 /** y: the floor row the cut stands on, which sets how far it slides with parallax. */
 /** Things that float and ride the swell, by their names. */
 const FLOATS = /\b(boat|rowboat|dinghy|skiff|canoe|kayak|raft|ship|barge|punt|gondola|sampan|junk|ferry|trawler|yacht|sailboat)s?\b/i;
@@ -386,7 +386,7 @@ class Game {
     await Promise.all(Object.entries(this.data.cast).map(async ([key, cast]) => {
       const img = await loadImage(files(cast.image)).catch(() => undefined);
       if (!img) return;
-      const figure: Figure = { img, height: cast.height, upright: cast.upright };
+      const figure: Figure = { img, height: cast.height, upright: cast.upright, share: cast.share };
       if (cast.rig && cast.parts) {
         const parts = new Map<PartName, HTMLImageElement>();
         await Promise.all(cast.rig.parts.map(async (p) => parts.set(p.name, await loadImage(files(`${cast.parts}/${p.file}`)))));
@@ -2057,15 +2057,19 @@ class Game {
       // ?people=0: the place alone, for a landscape take; everyone keeps playing, just unseen.
       if (nobody) continue;
       if (who.npc) {
+        this.bodyTop = undefined;
         const drawn = this.drawWho(who, lit, 'body');
+        const bodyTop = this.bodyTop;
         this.drawn.push({ hot: who.npc, rect: drawn });
         const body = who.npc.npc?.body;
-        this.occlude(drawn, who.y, gone, body?.x, body && (body.foot ?? stepFootY(g, body.x, who.y)));
+        this.occlude(drawn, who.y, gone, body?.x, body && (body.foot ?? stepFootY(g, body.x, who.y)), bodyTop);
       } else {
+        this.bodyTop = undefined;
         const drawn = this.drawWho(who, lit, 'body');
+        const bodyTop = this.bodyTop;
         if (who.body === this.hero) rect = drawn;
         // The depth map goes by where the feet are drawn: on a stair's tread, not at the walker's row.
-        this.occlude(drawn, who.y, gone, who.body?.x, who.body && (who.body.foot ?? stepFootY(g, who.body.x, who.y)));
+        this.occlude(drawn, who.y, gone, who.body?.x, who.body && (who.body.foot ?? stepFootY(g, who.body.x, who.y)), bodyTop);
       }
     }
     burn();
@@ -2136,14 +2140,14 @@ class Game {
    * than the feet. Painted figures and painted things are equally lit and equally sharp; this occlusion is
    * the cue that puts the figure in front. ctx is already shifted for parallax.
    */
-  private separate(x: number, y: number, h: number, paint: (c: CanvasRenderingContext2D) => number) {
+  private separate(x: number, y: number, h: number, paint: (c: CanvasRenderingContext2D) => number, tall = h) {
     const { focus: d, g } = this.place;
     const feet = y - g.scale.horizonY;
     if (!d || separateScale <= 0 || feet < 4 || h < 8) return;
     const blur = h * SEPARATE_BLUR;
     const pad = Math.ceil(blur * 2.5 + h * SEPARATE_SHIFT);
-    const [bw, bh] = [Math.ceil(h * 1.2) + 2 * pad, Math.ceil(h * 1.15) + 2 * pad];
-    const [ox, oy] = [Math.round(bw / 2), Math.ceil(pad + h * 1.1)];
+    const [bw, bh] = [Math.ceil(Math.max(h * 1.2, tall * 1.2)) + 2 * pad, Math.ceil(tall + h * 0.15) + 2 * pad];
+    const [ox, oy] = [Math.round(bw / 2), Math.ceil(pad + tall + h * 0.1)];
     const c = this.sepCanvas;
     if (c.width < bw || c.height < bh) Object.assign(c, { width: Math.max(c.width, bw), height: Math.max(c.height, bh) });
     const cc = c.getContext('2d')!;
@@ -2399,6 +2403,10 @@ class Game {
     // A rig walks its legs; a plain cut-out (or the stand-in) only bobs and rocks as it slides.
     const bob = creature ? creature.lift * h : rig || model || sprites || !walking ? 0 : Math.abs(Math.sin(phase * Math.PI * 2)) * h * 0.025;
     const rock = creature ? creature.lean * (facing < 0 ? -1 : 1) : rig || model || sprites || !walking ? 0 : Math.sin(phase * Math.PI * 2) * 0.035;
+    // How far above the feet the drawn figure reaches: the body's height, or more when it holds something over
+    // its head (a walk sheet cut with it, or a cut-out's cut.json share). Scratch canvases and boxes are sized by it.
+    const tall = rig ? Math.max(h, (h * rig.rig.ground) / rig.rig.height) : !model && !sprites && figure?.img && figure.share ? h / Math.max(0.2, Math.min(1, figure.share)) : h;
+    this.bodyTop = y - h * 1.04;
     let rigCanvas: CanvasImageSource | undefined;
     if (rig) {
       const at = pose(rig.rig, phase, walking);
@@ -2446,10 +2454,12 @@ class Game {
         return Math.max(ax, rig.canvas.width - ax) * s;
       }
       const img = figure?.img;
-      const w = img ? (h * img.width) / img.height : h * 0.36;
+      // A cut-out holding something over its head (cut.json share): the body keeps its height, the thing rises above.
+      const ih = h / Math.max(0.2, Math.min(1, figure?.share ?? 1));
+      const w = img ? (ih * img.width) / img.height : h * 0.36;
       // A creature squashes and stretches from the feet up, keeping its volume.
       const [sx2, sy2] = creature ? [1 / Math.sqrt(creature.stretch), creature.stretch] : [1, 1];
-      if (img) c.drawImage(figure?.face?.draw(img, '$image', 0, 1, 1, articulation, faceTime) ?? img, (-w / 2) * sx2, -h * sy2, w * sx2, h * sy2);
+      if (img) c.drawImage(figure?.face?.draw(img, '$image', 0, 1, 1, articulation, faceTime) ?? img, (-w / 2) * sx2, -ih * sy2, w * sx2, ih * sy2);
       else drawFigure(c, h, colour);
       return w / 2;
     };
@@ -2467,7 +2477,7 @@ class Game {
       const breath = 0.5 + 0.5 * Math.sin(t * ((2 * Math.PI) / 3.6));
       const m = c.getTransform();
       const density = Math.max(1, Math.min(1.5, Math.hypot(m.a, m.b)));
-      return idleWarp(c, paintRaw, token, h, density, sway, k * 0.02 * breath, 0.009 * k * Math.sin(t * ((2 * Math.PI) / 4.1) + 0.7));
+      return idleWarp(c, paintRaw, token, h, density, sway, k * 0.02 * breath, 0.009 * k * Math.sin(t * ((2 * Math.PI) / 4.1) + 0.7), tall);
     };
     // How much of the floor under the feet lies in a painted shadow: the person stands in it too.
     const shaded = Math.max(this.shadeAt(x, y, h), this.fireShadeAt(x, y, h));
@@ -2507,7 +2517,7 @@ class Game {
     const bright = this.lightAt(x, y - h * 0.5) * (1 - shaded * strength) * breath;
     // A party member is edged in its player's colour (twice, so the edge reads at a glance on a busy painting).
     const edge = glow ? ` drop-shadow(0 0 ${Math.max(2, h * 0.008).toFixed(1)}px ${glow}) drop-shadow(0 0 ${Math.max(2, h * 0.008).toFixed(1)}px ${glow})` : '';
-    this.separate(x, y, h, paintRaw);
+    this.separate(x, y, h, paintRaw, tall);
     ctx.filter = `brightness(${bright.toFixed(2)})${lit ? ' drop-shadow(0 0 5px rgba(255,225,150,0.95))' : ''}${edge}`;
     const frameAt = ctx.getTransform();
     ctx.translate(x, y - bob);
@@ -2520,9 +2530,10 @@ class Game {
       ctx.setTransform(feetAt);
       this.rim(h, paint, bright * (1 - shaded), light);
     }
-    this.shadeOver(paint, frameAt, feetAt, [x - half - 4, y - h * 1.06 - bob - 4, x + half + 4, y + Math.max(3, h * 0.04) + 4]);
+    this.shadeOver(paint, frameAt, feetAt, [x - half - 4, y - tall * 1.06 - bob - 4, x + half + 4, y + Math.max(3, h * 0.04) + 4]);
     ctx.restore();
-    return [x - half - 2 + sx, y - h * 1.04 - bob - 2 + sy, x + half + 2 + sx, y + Math.max(3, h * 0.04) + sy];
+    this.bodyTop = y - h * 1.04 - bob - 2 + sy;
+    return [x - half - 2 + sx, y - tall * 1.04 - bob - 2 + sy, x + half + 2 + sx, y + Math.max(3, h * 0.04) + sy];
   }
 
   /**
@@ -2933,16 +2944,18 @@ class Game {
     if (!light) return 1;
     const c = Math.min(light.cols - 1, Math.max(0, Math.floor(x / LIGHT_CELL)));
     const r = Math.min(light.rows - 1, Math.max(0, Math.floor(y / LIGHT_CELL)));
-    let sum = 0;
-    let n = 0;
-    for (let dr = -1; dr <= 1; dr++) for (let dc = -1; dc <= 1; dc++) {
+    // The light, not the colour of what lies there: a black tile, a dark table or a blue chair is dark paint in
+    // bright light, so the brighter cells around (the upper quartile of a 5 x 5) say how lit the spot is.
+    const around: number[] = [];
+    for (let dr = -2; dr <= 2; dr++) for (let dc = -2; dc <= 2; dc++) {
       const rr = r + dr;
       const cc = c + dc;
       if (rr < 0 || cc < 0 || rr >= light.rows || cc >= light.cols) continue;
-      sum += light.luma[rr * light.cols + cc];
-      n++;
+      around.push(light.luma[rr * light.cols + cc]);
     }
-    return Math.min(1.1, Math.max(0.4, 0.35 + (0.65 * sum) / n / 150));
+    around.sort((a, b) => a - b);
+    const lit = around[Math.floor(around.length * 0.75)] ?? 150;
+    return Math.min(1.1, Math.max(0.4, 0.35 + (0.65 * lit) / 150));
   }
 
   /** For the explorer: where the depth map puts the painting in front of feet at (x, feetY), over rect (src/qa/probe.ts). */
@@ -2955,6 +2968,8 @@ class Game {
   private snap?: HTMLCanvasElement;
   private snapped = false;
   private depthMask?: HTMLCanvasElement;
+  /** The top of the body of the person drawn last (drawPerson), below anything held over their head. */
+  private bodyTop?: number;
   private depthPatch?: HTMLCanvasElement;
 
   /**
@@ -3035,8 +3050,10 @@ class Game {
   }
 
   /** Puts the painting back over the hero, column by column, wherever an occluder stands in front of him. */
-  private occlude(rect: [number, number, number, number], feetY: number, gone: Set<string>, feetX?: number, drawnFeetY?: number) {
-    if (this.depthOcclusion) this.occludeDepth(rect, drawnFeetY ?? feetY, feetX);
+  private occlude(rect: [number, number, number, number], feetY: number, gone: Set<string>, feetX?: number, drawnFeetY?: number, bodyTop?: number) {
+    // The depth map hides only the body: what is held up over the head (a giant slice of toast) reaches where the
+    // map holds the near ceiling and the wall's top, which never stand in front of someone on the floor.
+    if (this.depthOcclusion) this.occludeDepth(bodyTop !== undefined && bodyTop > rect[1] ? [rect[0], bodyTop, rect[2], rect[3]] : rect, drawnFeetY ?? feetY, feetX);
     const support = (k: string) => this.place.byKey.get(k);
     for (const cut of this.place.cuts) {
       if (cut.occ.thing && gone.has(cut.occ.thing)) continue;
@@ -3680,8 +3697,9 @@ let idleOwner: { token: object; density: number; half: number } | undefined;
  * sway x height x u^1.7 (u = height above the feet as a share of the figure), so the feet stay planted and the head
  * moves most; the slices above the hips lift by up to `lift` x height (the breath), and the head nods by `nod`.
  */
-function idleWarp(c: CanvasRenderingContext2D, paint: (c: CanvasRenderingContext2D) => number, token: object, h: number, density: number, sway: number, lift: number, nod: number): number {
-  const W = Math.ceil(h * 1.6 * density), H = Math.ceil(h * 1.35 * density);
+function idleWarp(c: CanvasRenderingContext2D, paint: (c: CanvasRenderingContext2D) => number, token: object, h: number, density: number, sway: number, lift: number, nod: number, tall = h): number {
+  // Wide and high enough for what the figure holds over its head too (tall), else it is cut off flat.
+  const W = Math.ceil(Math.max(h * 1.6, tall * 1.2) * density), H = Math.ceil((tall + h * 0.35) * density);
   const foot = Math.ceil(h * 0.15 * density);
   if (!(W >= 8 && H >= 8 && W <= 4096 && H <= 4096)) return paint(c);
   const scratch = (idleScratch ??= document.createElement('canvas'));

package/docs/characters.md

@@ -4,7 +4,7 @@
 
 `build` also makes the characters, after the places (steps `cast`, `model`, `sprites`; each also runs alone):
 
-- `diorama cast <slug>` paints the hero and every NPC (cast/<key>.png, with what they hold: a lantern or a net SAM's person leaves out stays on, gaps it leaves are filled, and what floats beside them as theirs, a heart or the bubble they are in, comes along) and makes their 2D walk: SAM finds arms, legs and boots to place the joints, then the whole painting is bent by those bones as one fine mesh, not cut into parts, so a coat, an apron or a skirt stretches with the legs and never tears (src/scene/warp.ts). The walk is drawn once into a sheet (cast/<key>/walk.png, 8 frames and the untouched painting for standing; joints in cast/<key>/rig.json) and measured: a walk that opens holes or loses a piece is marked in rig.json `check` and the game slides the painted figure as one piece instead. `diorama rig <slug> [key...]` (Studio agent: `cast_rewalk`) makes the walk again from the paintings already there, with no Muse calls. A drawing SAM does not see as a person is asked as a character; when it still finds none, `cast_status` says why in `rigIssue` and `moves` says how the game moves them instead. Walk sheets whose frames left the pose are listed in `spriteIssues`; `sprites_redo` paints new takes. Spark also says how tall each one stands and whether upright like a person (cast/figures.json, a grown-up 1.62 m): a cat or a child is drawn at its own size, and a creature that is not upright (an animal, a round eye on tiny feet, a living teapot) gets no 2D rig or sprites: it moves as one body, hopping as it goes, squashing as it lands, leaning the way it goes, breathing and blinking (src/play/creature.ts), as its painted figure or, when it is in the sprites plan, as its own 3D model.
+- `diorama cast <slug>` paints the hero and every NPC (cast/<key>.png, with what they hold: a lantern or a net SAM's person leaves out stays on, gaps it leaves are filled, and what floats beside them as theirs, a heart or the bubble they are in, comes along) and makes their 2D walk: SAM finds arms, legs and boots to place the joints, then the whole painting is bent by those bones as one fine mesh, not cut into parts, so a coat, an apron or a skirt stretches with the legs and never tears (src/scene/warp.ts). The walk is drawn once into a sheet (cast/<key>/walk.png, 8 frames and the untouched painting for standing; joints in cast/<key>/rig.json) and measured: a walk that opens holes or loses a piece is marked in rig.json `check` and the game slides the painted figure as one piece instead. What the description says they hold or carry (one Spark call, only when the description says they hold something; the person's own list in cast/fixes.json `holds`, Studio agent: `cast_holds`) is found by SAM on its own, or as what stands out from the plain ground and touches them when SAM finds nothing, and kept on the cut-out and the walk however big it is: a slice of toast held over the head in both hands goes with the body and both arms carry it instead of swinging; cast/<key>/cut.json keeps the list and the share of the cut-out's height the body takes, so the game draws the body at its own height with the thing above it (src/scene/held.ts). `diorama rig <slug> [key...]` (Studio agent: `cast_rewalk`) makes the walk again from the paintings already there, with no Muse calls. A drawing SAM does not see as a person is asked as a character; when it still finds none, `cast_status` says why in `rigIssue` and `moves` says how the game moves them instead. Walk sheets whose frames left the pose are listed in `spriteIssues`; `sprites_redo` paints new takes. Spark also says how tall each one stands and whether upright like a person (cast/figures.json, a grown-up 1.62 m): a cat or a child is drawn at its own size, and a creature that is not upright (an animal, a round eye on tiny feet, a living teapot) gets no 2D rig or sprites: it moves as one body, hopping as it goes, squashing as it lands, leaning the way it goes, breathing and blinking (src/play/creature.ts), as its painted figure or, when it is in the sprites plan, as its own 3D model.
 - `diorama model <slug>` gives each character who needs sprites a rigged 3D model: Muse paints them from four sides, TRELLIS turns the views into a mesh and Make-It-Animatable adds a skeleton (both open Hugging Face Spaces; set `HUGGING_FACE_API_KEY` or `HF_TOKEN`, as they run on your account's GPU quota). The model only supplies poses. A creature (not upright) is painted from four sides as it is (no A-pose, no hands) and keeps TRELLIS's mesh with no skeleton (`model.json` `creature: true`): the game hops, squashes, leans and blinks the whole model, so a creature hero or NPC needs no sheets. Studio's Cast **Animate in 3D** (action `cast_animate`) builds a model for anyone the build skips, and the model step keeps building it. The figure is stood up straight in the file, and the poses start from the shape the skin was bound in (the reconstructed figure), not from whatever pose the rig saved its bones in (Make-It-Animatable's T-pose can sit a figure in a long coat down). What the character wears (a satchel, a backpack) is read off the front view and named in every other view's prompt with the side it shows on; Spark then checks all four together, and a view with it on the wrong side, or with anything standing on the ground at the feet, is painted again (up to 2 rounds; a wrong front repaints all four once). Things held in the hands are left out of the views: a suitcase put down beside the feet would be fused to a boot and swing with the leg in every sprite. A view the image safety filter refuses is asked once more in calm words (the looks left to the picture, the hands said plainly); refused again, the model made before stays, or the character walks with its 2D rig, and the build goes on.
 - Optional local Blender (3.6+, found on PATH, in /Applications, or at `DIORAMA_BLENDER`; on/off in Studio Setup, **3D animation**): after the rig, `src/model/blender/relax.py` puts the figure into a calm rest pose. Muse paints the reference with arms raised and feet wide, TRELLIS builds that pose, and the walk then had to turn each arm ~100° down from it, which pinched the shoulders, splayed the hands sideways and kept the legs straddled. Blender poses arms down by the sides with palms to the thighs and feet under the hips, smooths the bent shoulders and hips (Corrective Smooth) and makes that the rest pose (`model.json` `relaxed: true`, the rig as it came kept in `model.source.glb`). The original surface rides along as the `_REST` vertex attribute, so views fitted and painted in the original pose still land where they belong. `diorama model <slug> [key] --relax` (Studio agent: `cast_relax`) does it for existing models and draws their UV sprites again, with no calls. Without Blender nothing changes.
 - `diorama sprites <slug>` draws their walk and talk sheets (cast/<key>/sprites/): the model walks or gestures in a sheet of 8 frames, Muse draws the sheet again in the world's style, and each frame is cut out and lined up with its pose. A sheet that Muse handed back barely changed is drawn again over a different guide. A walk frame Muse painted out of the gait (a crouch, a sit: the painted figure under 88% of the sheet's usual height) is listed as `off` in sprites.json; the walk steps over it and standing still never shows it. Only what the story uses is drawn: the walk facings the hero's routes need, one talk sheet per speaker in the facing of their conversation, and the gesture Spark picks for each line (`--plan` prints the plan without drawing). Rendering the guides needs Playwright's Chromium (`diorama browsers`).

package/docs/setup.md

@@ -32,7 +32,7 @@ npm run doctor -- --now        # before building in front of people: is the netw
 npm run studio                 # the GUI
 npm run build -- <slug> "<idea>"
 npm run play -- <slug>
-npm run publish -- <slug>
+npm run pages -- <slug>        # diorama publish (npm keeps the script name publish for itself)
 npm run export -- <slug> --zip
 npm run browsers               # Playwright's Chromium, for explore, playthrough and sprites
 npm run diorama -- <command>   # any other command, e.g. npm run diorama -- masks <slug>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gtanczyk/diorama",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Turn a story prompt into playable painted point-and-click scenes with Meta Muse and SAM 3.1",
   "license": "Apache-2.0",
   "repository": {
@@ -42,7 +42,7 @@
     "studio": "node app/bin/diorama.mjs studio",
     "build": "node app/bin/diorama.mjs build",
     "play": "node app/bin/diorama.mjs play",
-    "publish": "node app/bin/diorama.mjs publish",
+    "pages": "node app/bin/diorama.mjs publish",
     "export": "node app/bin/diorama.mjs export",
     "browsers": "node app/bin/diorama.mjs browsers",
     "typecheck": "tsc --noEmit -p app && tsc --noEmit -p app/web && tsc --noEmit -p app/studio",