@gtanczyk/diorama 0.0.0-stage → 0.1.0

package/app/studio/sure.tsx

@@ -0,0 +1,54 @@
+// Asking before something big or hard to take back, in the Studio's own dialog instead of the browser's confirm.
+// `type` makes the person type a word (the world's slug) before the button wakes up, as GitHub does for deletes.
+import { signal } from '@preact/signals';
+import { useEffect, useRef, useState } from 'preact/hooks';
+
+type Ask = { title: string; body: string; go: string; danger?: boolean; type?: string; answer: (yes: boolean) => void };
+const asking = signal<Ask | null>(null);
+
+/** Resolves true when the person presses `go`, false on Cancel, Esc or a click outside. */
+export function askSure(o: Omit<Ask, 'answer'>): Promise<boolean> {
+  return new Promise((answer) => {
+    asking.value?.answer(false);
+    asking.value = { ...o, answer };
+  });
+}
+
+/** Mounted once, like the lightbox. */
+export function Sure() {
+  const a = asking.value;
+  const [typed, setTyped] = useState('');
+  const box = useRef<HTMLInputElement>(null);
+  const done = (yes: boolean) => {
+    a?.answer(yes);
+    asking.value = null;
+    setTyped('');
+  };
+  useEffect(() => {
+    if (!a) return;
+    box.current?.focus();
+    const esc = (e: KeyboardEvent) => e.key === 'Escape' && (e.stopImmediatePropagation(), done(false));
+    addEventListener('keydown', esc, true);
+    return () => removeEventListener('keydown', esc, true);
+  }, [a]);
+  if (!a) return null;
+  const ok = !a.type || typed.trim() === a.type;
+  return (
+    <div class="sure-back" onClick={(e) => e.target === e.currentTarget && done(false)}>
+      <form class="sure" role="alertdialog" aria-label={a.title} onSubmit={(e) => (e.preventDefault(), ok && done(true))}>
+        <h2>{a.title}</h2>
+        <p>{a.body}</p>
+        {a.type && (
+          <label><span>Type <code>{a.type}</code> to confirm</span>
+            <input ref={box} value={typed} autoComplete="off" spellcheck={false} onInput={(e) => setTyped(e.currentTarget.value)} />
+          </label>
+        )}
+        <div class="row">
+          <span class="grow" />
+          <button type="button" class="ghost" onClick={() => done(false)}>Cancel</button>
+          <button type="submit" class={a.danger ? 'danger-fill' : 'primary'} disabled={!ok}>{a.go}</button>
+        </div>
+      </form>
+    </div>
+  );
+}

package/app/studio/tip.tsx

@@ -0,0 +1,74 @@
+// Tooltips for the whole Studio: any element with a title gets a styled tip instead of the browser's slow grey
+// one. Mounted once; it watches the pointer and keyboard focus, so a screen just writes title="what it does".
+import { useEffect, useLayoutEffect, useRef, useState } from 'preact/hooks';
+
+type Shown = { text: string; rect: DOMRect };
+
+/** The game's frame and a select's options keep their own title. */
+const SKIP = new Set(['IFRAME', 'OPTION']);
+const WAIT_MS = 350;
+
+/** Moves an element's title into data-tip (so the browser's own tip stays away) and returns the text. */
+function tipOf(el: Element): string {
+  const title = el.getAttribute('title');
+  if (title) {
+    el.setAttribute('data-tip', title);
+    el.removeAttribute('title');
+    if (!el.hasAttribute('aria-label') && !el.textContent?.trim()) el.setAttribute('aria-label', title);
+  }
+  return el.getAttribute('data-tip') ?? '';
+}
+
+function target(from: EventTarget | null): Element | null {
+  const el = from instanceof Element ? from.closest('[title], [data-tip]') : null;
+  return el && !SKIP.has(el.tagName) ? el : null;
+}
+
+export function Tips() {
+  const [shown, setShown] = useState<Shown | null>(null);
+  const tip = useRef<HTMLDivElement>(null);
+  useEffect(() => {
+    let over: Element | null = null;
+    let timer = 0;
+    // Right after one tip, the next shows at once: running along a row of buttons reads them all.
+    let warmUntil = 0;
+    const enter = (el: Element | null) => {
+      if (el === over) return;
+      clearTimeout(timer);
+      if (over && !el) warmUntil = Date.now() + 500;
+      over = el;
+      if (!el) return setShown(null);
+      tipOf(el);
+      timer = window.setTimeout(() => {
+        const text = over === el && el.isConnected ? tipOf(el) : '';
+        setShown(text ? { text, rect: el.getBoundingClientRect() } : null);
+      }, Date.now() < warmUntil ? 0 : WAIT_MS);
+    };
+    const onOver = (e: PointerEvent) => e.pointerType !== 'touch' && enter(target(e.target));
+    const onFocus = (e: FocusEvent) => (e.target as Element)?.matches?.(':focus-visible') && enter(target(e.target));
+    const hide = () => enter(null);
+    const onKey = (e: KeyboardEvent) => e.key === 'Escape' && hide();
+    const on: Array<[EventTarget, string, EventListener, boolean?]> = [
+      [document, 'pointerover', onOver as EventListener], [document, 'focusin', onFocus as EventListener], [document, 'focusout', hide],
+      [document, 'pointerdown', hide, true], [document, 'keydown', onKey as EventListener], [window, 'scroll', hide, true], [window, 'blur', hide],
+    ];
+    for (const [t, name, fn, capture] of on) t.addEventListener(name, fn, capture);
+    return () => {
+      clearTimeout(timer);
+      for (const [t, name, fn, capture] of on) t.removeEventListener(name, fn, capture);
+    };
+  }, []);
+  // Above the element, or below it when there is no room; the edges stay 8 px inside the window.
+  useLayoutEffect(() => {
+    const el = tip.current;
+    if (!el || !shown) return;
+    const { rect } = shown;
+    const w = el.offsetWidth;
+    const h = el.offsetHeight;
+    const below = rect.top - h - 8 < 0;
+    el.style.left = `${Math.max(8, Math.min(innerWidth - w - 8, rect.left + rect.width / 2 - w / 2))}px`;
+    el.style.top = `${below ? rect.bottom + 8 : rect.top - h - 8}px`;
+    el.classList.add('on');
+  }, [shown]);
+  return shown ? <div ref={tip} class="tip" role="tooltip" key={shown.text}>{shown.text}</div> : null;
+}

package/app/studio/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": { "lib": ["ES2023", "DOM", "DOM.Iterable"], "types": [], "jsx": "react-jsx", "jsxImportSource": "preact" },
+  "include": ["."]
+}

package/app/studio/video.tsx

@@ -0,0 +1,73 @@
+import { useEffect, useRef, useState } from 'preact/hooks';
+import { call } from './api.ts';
+import { base64 } from './camera.tsx';
+import { Icon } from './icons.tsx';
+import { VIDEO_CHUNK, VIDEO_MAX } from '../src/world/video-limits.ts';
+export type VideoResult = { speech?: Array<{ at: number; text: string; provider: string; model: string; timing: string; speakerScope?: string; segments: Array<{ start: number; end: number; text: string; speaker?: string }> }>;  idea: string; photos: Array<{ name: string; data: string }>; totalFrames: number; observations: Array<{ at: number; text: string }> };
+/** Upload and interpretation are resumable independently; no world is built until the person reviews it. */
+export function VideoInput({ note, disabled, onResult, onBusy }: { note: string; disabled: boolean; onResult: (id: string, result: VideoResult) => void; onBusy: (busy: boolean) => void }) {
+  const [preview, setPreview] = useState('');
+  const picker = useRef<HTMLInputElement>(null);
+  const [filename, setFilename] = useState('');
+  useEffect(() => () => { if (preview) URL.revokeObjectURL(preview); }, [preview]);
+  const [status, setStatus] = useState('');
+  const [error, setError] = useState('');
+  const [busy, setBusy] = useState(false);
+  const [pending, setPending] = useState(false);
+  const work = useRef<{ id?: string; file: File; note: string }>();
+  const stop = useRef(false);
+  useEffect(() => () => { stop.current = true; }, []);
+  const run = async () => {
+    if (!work.current || busy) return;
+    stop.current = false; setBusy(true); onBusy(true); setError(''); setPending(true);
+    const task = work.current;
+    try {
+      if (!task.id) task.id = (await call<{ id: string }>('video_upload', { name: task.file.name, size: task.file.size })).id;
+      // Store the token so a refresh can resume: select the same source file again to continue.
+      try { localStorage.setItem('diorama-video-upload', JSON.stringify({ id: task.id, name: task.file.name, size: task.file.size, modified: task.file.lastModified, note: task.note })); } catch {}
+      let { received } = await call<{ received: number }>('video_upload', { id: task.id });
+      while (received < task.file.size) {
+        if (stop.current) return;
+        setStatus(`Uploading video · ${Math.floor(received / task.file.size * 100)}%`);
+        const data = (await base64(task.file.slice(received, received + VIDEO_CHUNK))).split(',')[1];
+        ({ received } = await call<{ received: number }>('video_upload', { id: task.id, offset: received, data }));
+      }
+      setStatus('Reading video duration and audio…');
+      const info = await call<{ duration: number; audio: boolean; parts: number }>('video_inspect', { id: task.id });
+      for (let index = 0; index < info.parts; index++) {
+        if (stop.current) return;
+        setStatus(`Understanding video · ${index + 1}/${info.parts} · ${info.audio ? 'pictures and speech' : 'pictures, no audio track'}. You can pause and resume.`);
+        await call('video_read', { id: task.id, index, note: task.note });
+      }
+      if (stop.current) return;
+      const result = await call<VideoResult>('video_result', { id: task.id });
+      if (stop.current) return;
+      onResult(task.id, result); setStatus('Video understood. Review the idea and selected pictures before building.');
+      setPending(false); try { localStorage.removeItem('diorama-video-upload'); } catch {} work.current = undefined;
+    } catch (e) { setError((e as Error).message); }
+    finally { setBusy(false); onBusy(false); }
+  };
+  const choose = (file?: File) => {
+    if (!file) return;
+    if (file.size > VIDEO_MAX) { setError('Choose a video up to 2 GB.'); return; }
+    let saved;
+    try { saved = JSON.parse(localStorage.getItem('diorama-video-upload') ?? 'null'); } catch {}
+    setPreview(URL.createObjectURL(file));
+    setFilename(file.name);
+    work.current = { file, note, ...(saved?.name === file.name && saved?.size === file.size && saved?.modified === file.lastModified ? { id: saved.id, note: saved.note } : {}) };
+    void run();
+  };
+  return <>
+    <button type="button" class="ghost bare" disabled={disabled || busy} title="Show your drawings and tell their story in a video. With or without sound, up to 2 GB / 4 hours." onClick={() => picker.current?.click()}><Icon name="film" /> Video</button>
+    <input ref={picker} class="video-picker" hidden type="file" aria-label="Choose a video" accept="video/*,.mkv,.m4v,.avi" disabled={disabled || busy} onChange={e => { choose(e.currentTarget.files?.[0]); e.currentTarget.value = ''; }} />
+    {(preview || error || status) && <div class="video-status">
+    {filename && <p class="video-filename"><Icon name="film" /> {filename}</p>}
+    {preview && <video src={preview} controls preload="metadata" style={{ maxWidth: '100%', maxHeight: '300px' }} />}
+    {status && <p role="status">{status}</p>}
+    {error && <p class="error" role="alert">{error}</p>}
+    {busy && <button type="button" class="ghost small" onClick={() => { stop.current = true; setStatus('Pausing after the current interval…'); }}>Pause</button>}
+    {!busy && pending && <button type="button" class="ghost small" onClick={() => void run()}>Resume video</button>}
+    <details class="detail"><summary>Video tips</summary>Hold each drawing still for at least ten seconds. You can pause and resume processing. Interpreting the video uses model calls.</details>
+    </div>}
+  </>;
+}

package/app/studio/world.tsx

@@ -0,0 +1,188 @@
+// A world in the atelier: the conversation with the agent that looks after it, told in cards (the story, the
+// places, the cast, what waits for you, what a turn changed, the game), beside a stage that shows what the
+// card is about. Click a thing on the stage and it opens; the agent is told what is open with each message.
+import type { ComponentChildren } from 'preact';
+import { useEffect, useState } from 'preact/hooks';
+import { call, fileUrl } from './api.ts';
+import { named } from './build.tsx';
+import { Icon } from './icons.tsx';
+
+export type StageView = { kind: 'build' | 'place' | 'cast' | 'story' | 'play' | 'work'; place?: string; step?: string; build?: { place?: string } };
+
+type Pic = { file: string; v: number | null } | null;
+export type CardPlace = { key: string; shows: string; image: Pic; stages: Record<string, number | null>; verdict: { ok: boolean } | null; trouble?: { why: string } | null };
+type Character = { key: string; role: string; looksNow: string; figure: Pic };
+
+/** A card in the conversation: what it shows decides the stage when clicked. */
+export function Card({ kind, by, on, onClick, children, tone, opening }: { kind: string; by?: string; on?: boolean; onClick?: () => void; children: ComponentChildren; tone?: 'ask' | 'play'; opening?: boolean }) {
+  return (
+    <div class={`wcard${on ? ' on' : ''}${tone ? ` ${tone}` : ''}${onClick ? ' click' : ''}`} data-opening={opening ? kind.toLowerCase() : undefined} onClick={onClick} role={onClick ? 'button' : undefined} tabIndex={onClick ? 0 : undefined}
+      onKeyDown={(e) => onClick && (e.key === 'Enter' || e.key === ' ') && (e.preventDefault(), onClick())}>
+      <div class="k">{kind}{by && <span class="by">{named(by)}</span>}</div>
+      {children}
+    </div>
+  );
+}
+
+const nice = (key: string) => key.replace(/[-_]+/g, ' ').replace(/^./, (c) => c.toUpperCase());
+export { nice };
+
+/** The places as small paintings; one not painted yet shows its name. */
+export function PlacesCard({ slug, places, pending = [], stage, onStage }: { slug: string; places: CardPlace[]; pending?: string[]; stage: StageView; onStage: (s: StageView) => void }) {
+  return (
+    <Card kind="Places" by="Muse Image · SAM 3.1" on={stage.kind === 'place'} opening>
+      <div class="thumbs">
+        {places.map((p) => (
+          <button type="button" class={`thumb${stage.kind === 'place' && stage.place === p.key ? ' on' : ''}${p.trouble ? ' trouble' : p.verdict?.ok ? ' ready' : ''}`} title={`${nice(p.key)}: ${p.shows}${p.verdict?.ok ? ' (playable)' : ''}`} onClick={() => onStage({ kind: 'place', place: p.key })}
+            style={p.image ? { backgroundImage: `url("${fileUrl(slug, p.image.file, p.image.v)}")` } : undefined}>
+            {!p.image && <span>{nice(p.key)}</span>}
+          </button>
+        ))}
+        {pending.map(key => <button type="button" class="thumb pending-place" title={`${nice(key)}: being built; not in the game yet. Show build progress`} onClick={() => onStage({ kind: 'build' })}><span>{nice(key)}<br />Building…</span></button>)}
+      </div>
+    </Card>
+  );
+}
+
+/** The characters standing in a row, as cut out. */
+export function CastCard({ slug, version, stage, onStage }: { slug: string; version: number; stage: StageView; onStage: (s: StageView) => void }) {
+  const cast = useCast(slug, version);
+  const shown = cast?.filter((c) => c.figure) ?? [];
+  if (!shown.length) return null;
+  return (
+    <Card kind="Cast" by="Muse Image · 3D" on={stage.kind === 'cast'} onClick={() => onStage({ kind: 'cast' })} opening>
+      <div class="lineup-mini">{shown.map((c) => <img src={fileUrl(slug, c.figure!.file, c.figure!.v)} alt={nice(c.key)} title={`${nice(c.key)}: ${c.looksNow}`} />)}</div>
+    </Card>
+  );
+}
+
+/**
+ * The world at a glance, pinned over a long conversation once its first cards scrolled away: the story, the places
+ * and the cast. The compact previews lead back to their full cards in the conversation.
+ */
+export function WorldStrip({ slug, version, title, places, pending = [], castDone, stage, onStage }: { slug: string; version: number; title?: string; places: CardPlace[]; pending?: string[]; castDone: boolean; stage: StageView; onStage: (s: StageView) => void }) {
+  const cast = useCast(slug, version)?.filter((c) => c.figure) ?? [];
+  const goToCard = (button: HTMLButtonElement, kind: 'places' | 'cast') => {
+    const scroller = button.closest('.agent')?.querySelector<HTMLElement>('.gc-scroll');
+    const card = scroller?.querySelector<HTMLElement>(`[data-opening="${kind}"]`);
+    if (!scroller || !card) return;
+    const target = kind === 'places' ? card.querySelector<HTMLElement>('button') : card;
+    target?.focus({ preventScroll: true });
+    scroller.scrollTo({ top: scroller.scrollTop + card.getBoundingClientRect().top - scroller.getBoundingClientRect().top - 8, behavior: matchMedia('(prefers-reduced-motion: reduce)').matches ? 'auto' : 'smooth' });
+  };
+  const placesLabel = `${places.length} ${places.length === 1 ? 'place' : 'places'}${pending.length ? `, ${pending.length} building` : ''}. Show places in the conversation`;
+  return (
+    <div class="world-strip">
+      {title && <button type="button" class={`ws-story${stage.kind === 'story' ? ' on' : ''}`} title="The story: places, items, people and endings" onClick={() => onStage({ kind: 'story' })}>{title}</button>}
+      <button type="button" class="ws-places" title={placesLabel} aria-label={placesLabel} onClick={(e) => goToCard(e.currentTarget, 'places')}>
+        <span class="ws-stack" aria-hidden="true">
+          {places.slice(0, 3).map((p) => <span class="ws-sheet" key={p.key} style={p.image ? { backgroundImage: `url("${fileUrl(slug, p.image.file, p.image.v)}")` } : undefined} />)}
+        </span>
+        <span class="ws-count">{places.length}</span>
+        {pending.length > 0 && <span class="ws-building" aria-hidden="true">+{pending.length} ◌</span>}
+      </button>
+      {castDone && cast.length > 0 && (
+        <button type="button" class="ws-cast" title="Show the cast in the conversation" aria-label={`Show ${cast.length} ${cast.length === 1 ? 'character' : 'characters'} in the conversation`} onClick={(e) => goToCard(e.currentTarget, 'cast')}>
+          {cast.slice(0, 5).map((c) => <img src={fileUrl(slug, c.figure!.file, c.figure!.v)} alt="" />)}
+          {cast.length > 5 && <span class="ws-count">+{cast.length - 5}</span>}
+        </button>
+      )}
+    </div>
+  );
+}
+
+function useCast(slug: string, version: number) {
+  const [cast, setCast] = useState<Character[] | null>(null);
+  useEffect(() => void call<Character[]>('cast_status', { slug }).then(setCast).catch(() => setCast([])), [slug, version]);
+  return cast;
+}
+
+type Box = [number, number, number, number];
+type Masks = { things?: Record<string, { box?: Box; phrase?: string }> };
+
+/**
+ * The place as a framed painting. Every thing SAM cut out is a spot: hovering lights up its mask,
+ * a click opens it (its mask, kind, stand spot: by hand).
+ */
+export function PlaceStage({ slug, place, version, onOpen, children }: { slug: string; place: CardPlace; version: number; onOpen?: (thing: string) => void; children?: ComponentChildren }) {
+  const [masks, setMasks] = useState<Masks | null>(null);
+  const [hover, setHover] = useState<string | null>(null);
+  useEffect(() => {
+    const controller = new AbortController();
+    setMasks(null);
+    setHover(null);
+    fetch(fileUrl(slug, `places/${place.key}/masks.json`, Date.now()), { cache: 'no-store', signal: controller.signal }).then((r) => (r.ok ? r.json() : null)).then((m) => { if (!controller.signal.aborted) setMasks(m as Masks | null); }).catch(() => {});
+    return () => controller.abort();
+  }, [slug, place.key, version]);
+  // Big things first, so a small one on top of a big one gets the pointer.
+  const spots = Object.entries(masks?.things ?? {}).filter(([, t]) => t.box).sort(([, a], [, b]) => area(b.box!) - area(a.box!));
+  return (
+    <div class="mat">
+      <div class="frame">
+        <div class="pic">
+          {place.image ? <img src={fileUrl(slug, place.image.file, place.image.v)} alt={place.shows} /> : <div class="empty"><strong>{nice(place.key)}</strong><span>{named('Muse Image paints it next')}</span></div>}
+          {hover && (
+            <div class="glow" style={{ maskImage: `url("${fileUrl(slug, `places/${place.key}/masks/${hover}.png`)}")`, WebkitMaskImage: `url("${fileUrl(slug, `places/${place.key}/masks/${hover}.png`)}")` }} />
+          )}
+          {spots.map(([key, t]) => {
+            const [x0, y0, x1, y1] = t.box!;
+            return (
+              <button type="button" class="spot" aria-label={`Open the ${nice(key).toLowerCase()}`} title={`${nice(key)}: click to open it`}
+                style={{ left: `${(x0 / 1200) * 100}%`, top: `${(y0 / 896) * 100}%`, width: `${((x1 - x0) / 1200) * 100}%`, height: `${((y1 - y0) / 896) * 100}%` }}
+                onMouseEnter={() => setHover(key)} onMouseLeave={() => setHover(null)} onFocus={() => setHover(key)} onBlur={() => setHover(null)}
+                onClick={() => onOpen?.(key)}>
+                {hover === key && <span class="name">{nice(key).toLowerCase()}</span>}
+              </button>
+            );
+          })}
+        </div>
+      </div>
+      {children}
+    </div>
+  );
+}
+const area = (b: Box) => (b[2] - b[0]) * (b[3] - b[1]);
+
+/** The cast standing in a row and their items on a shelf; a click opens one. */
+export function CastStage({ slug, version, onOpen }: { slug: string; version: number; onOpen?: (key: string) => void }) {
+  const cast = useCast(slug, version);
+  const [items, setItems] = useState<Item[]>([]);
+  useEffect(() => void call<Item[]>('icons_status', { slug }).then(setItems).catch(() => setItems([])), [slug, version]);
+  if (!cast) return <div class="mat"><p class="detail">Reading the cast…</p></div>;
+  const shown = cast.filter((c) => c.figure);
+  if (!shown.length) return <div class="mat"><p class="detail">Nobody is cast yet: the build paints them after the places.</p></div>;
+  return (
+    <div class="mat lineup">
+      {/* Any cast fits the stage: up to 6 a row, more rows when there are more, each figure smaller then. */}
+      <div class="figures" style={{ '--per-row': Math.min(shown.length, Math.ceil(shown.length / Math.ceil(shown.length / 6))), '--rows': Math.ceil(shown.length / 6) }}>
+      {shown.map((c) => (
+        <div class="who-at">
+          <button type="button" class="who" title={`${c.looksNow}. Click to open: looks, voice, 3D model and sprites`} onClick={() => onOpen?.(c.key)}>
+            <img src={fileUrl(slug, c.figure!.file, c.figure!.v)} alt="" />
+            <span>{nice(c.key)}{c.role === 'hero' ? ' · hero' : ''}</span>
+          </button>
+        </div>
+      ))}
+      </div>
+      {/* The story's items on a shelf: what the hero picks up or is handed; a click opens one. */}
+      {items.length > 0 && (
+        <div class="shelf" aria-label="Items">
+          {items.map((it) => (
+            <button type="button" class="item" title={`${it.name}: ${it.from ? `picked up in ${nice(it.from.place)}` : 'handed over'}. Click to open its icon`} onClick={() => onOpen?.(`item-${it.key}`)}>
+              {it.icon ? <img src={fileUrl(slug, it.icon.file, it.icon.v)} alt="" /> : <span class="noicon">?</span>}
+              <span>{it.name}</span>
+            </button>
+          ))}
+        </div>
+      )}
+    </div>
+  );
+}
+type Item = { key: string; name: string; from: { place: string; thing: string } | null; icon: Pic };
+
+/**
+ * What the person has open, sent ahead of each message so "this" means something to the agent; never shown.
+ * "[Looking at: …]" is the same from older conversations.
+ */
+export const screenPrefix = (on: string) => `[On screen: ${on}]\n`;
+export const shownScreen = (text: string) => text.replace(/^\[(?:On screen|Looking at): [^\]]+\]\n?/, '');

package/app/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2023",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "strict": true,
+    "noEmit": true,
+    "allowImportingTsExtensions": true,
+    "skipLibCheck": true,
+    "types": ["node"]
+  },
+  "include": ["src", "test"]
+}