@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/src/scene/face-calibration.ts

@@ -0,0 +1,65 @@
+// Human/agent-reviewed lip locations are tied to the exact inspected pixels.
+import fs from 'node:fs';
+import sharp from 'sharp';
+import { z } from 'zod';
+import type { Box, FacePose } from '../play/lipsync.ts';
+import { registeredFace, sha256 } from './faces-data.ts';
+import { FaceRevision, LipBox, MouthName } from './face-work.ts';
+import type { WorldFiles } from '../world/files.ts';
+
+export const CalibrationTarget = z.discriminatedUnion('kind', [
+  z.object({kind:z.literal('source'),view:z.string().min(1),pose:z.number().int().nonnegative()}),
+  z.object({kind:z.literal('patch'),view:z.string().min(1),mouth:MouthName}),
+]);
+type Target = z.infer<typeof CalibrationTarget>;
+const Review = z.object({box:LipBox,open:z.boolean()});
+export function reviewedLips(files:WorldFiles,key:string,image:Buffer) {
+  const file=files.path('cast',key,'lip-reviews',`${sha256(image)}.json`);
+  return fs.existsSync(file)?Review.parse(files.readJson(file)):undefined;
+}
+export function pixelCrop([x,y,w,h]:Box,width:number,height:number) {
+  const left=Math.max(0,Math.floor(x+1e-7)),top=Math.max(0,Math.floor(y+1e-7));
+  return {left,top,width:Math.max(1,Math.min(width-left,Math.ceil(x+w-1e-7)-left)),height:Math.max(1,Math.min(height-top,Math.ceil(y+h-1e-7)-top))};
+}
+export async function sourceLipCrop(data:Buffer,cols:number,rows:number,pose:FacePose,index:number) {
+  const meta=await sharp(data).metadata(),cw=meta.width!/cols,ch=meta.height!/rows;
+  const [mx,my,mw,mh]=pose.mouth;
+  const local=pixelCrop([mx-mw*.75,my-mh*1.25,mw*2.5,mh*2.75],Math.floor(cw),Math.floor(ch));
+  const rect={...local,left:local.left+index%cols*cw,top:local.top+Math.floor(index/cols)*ch};
+  const image=await sharp(data).extract(rect).resize(512,512,{fit:'fill'}).png().toBuffer();
+  return {image,rect,cw,ch};
+}
+export async function inspectFaceCalibration(files:WorldFiles,key:string,target?:Target) {
+  const face=registeredFace(files,key,'face-candidate.json');
+  if(!face)throw new Error('No current face candidate; prepare or inspect saved atlas work first');
+  const info=face.info, manifest=files.path('cast',key,'face-candidate.json');
+  const revision=sha256(fs.readFileSync(manifest));
+  const sources:Target[]=[];
+  if(info.imagePose&&!info.imagePose.lips)sources.push({kind:'source',view:'image',pose:0});
+  for(const [view,poses] of Object.entries(info.sheets)) if(!view.startsWith('rig:'))poses.forEach((p,i)=>{if(p&&!p.lips)sources.push({kind:'source',view,pose:i});});
+  const views=[...new Set([info.imagePose?.atlas,...Object.values(info.sheets).flatMap(p=>p.map(v=>v?.atlas))].filter((v):v is string=>!!v))];
+  const patches=views.filter(v=>!info.frames?.[v]).flatMap(view=>MouthName.options.map(mouth=>({kind:'patch' as const,view,mouth})));
+  if(!target)return {revision,pending:[...sources,...patches]};
+  let image:Buffer;
+  if(target.kind==='source') {
+    const pose=target.view==='image'?info.imagePose:info.sheets[target.view]?.[target.pose];
+    if(!pose || target.view==='image'&&target.pose!==0 || target.view.startsWith('rig:'))throw new Error('No current source pose');
+    const model=target.view.startsWith('model:');
+    const path=target.view==='image'?files.path('cast',`${key}.png`):files.path('cast',key,model?'model':'sprites',model?target.view.slice(6):target.view);
+    const grid=target.view==='image'||model?{cols:1,rows:1}:files.readJson(files.path('cast',key,'sprites','sprites.json')) as {cols:number;rows:number};
+    image=(await sourceLipCrop(fs.readFileSync(path),grid.cols,grid.rows,pose,target.pose)).image;
+  } else {
+    if(!views.includes(target.view))throw new Error('No current patch view');
+    const mouths=target.view==='default'?info.mouths:info.atlases?.[target.view];
+    if(!mouths)throw new Error('No current mouth patch');
+    const atlas=fs.readFileSync(files.path('cast',key,info.image)),meta=await sharp(atlas).metadata();
+    image=await sharp(atlas).extract(pixelCrop(mouths[target.mouth],meta.width!,meta.height!)).resize(512,512,{fit:'fill'}).png().toBuffer();
+  }
+  return {revision,target,imageHash:sha256(image),image,box:reviewedLips(files,key,image)?.box};
+}
+export async function setFaceCalibration(files:WorldFiles,key:string,target:Target,revision:string,imageHash:string,box:Box,open:boolean) {
+  FaceRevision.parse(revision);FaceRevision.parse(imageHash);LipBox.parse(box);
+  const current=await inspectFaceCalibration(files,key,target);
+  if(!current.image || current.revision!==revision || current.imageHash!==imageHash || sha256(fs.readFileSync(files.path('cast',key,'face-candidate.json')))!==revision)throw new Error('Face calibration changed; inspect it again');
+  files.write(files.path('cast',key,'lip-reviews',`${imageHash}.json`),{box,open});
+}

package/app/src/scene/face-registration.ts

@@ -0,0 +1,21 @@
+import sharp from 'sharp';
+import type { Box, FacePose } from '../play/lipsync.ts';
+
+/** The cut-out PNG is cropped; the rig retains its padded paint canvas. Their dimensions are not equivalent. */
+export async function alphaBounds(image: Buffer): Promise<Box> {
+  const { data, info } = await sharp(image).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+  let left = info.width, top = info.height, right = -1, bottom = -1;
+  for (let y = 0; y < info.height; y++) for (let x = 0; x < info.width; x++) if (data[(y * info.width + x) * 4 + 3] > 32) {
+    left = Math.min(left, x); top = Math.min(top, y); right = Math.max(right, x); bottom = Math.max(bottom, y);
+  }
+  if (right < left || bottom < top) throw new Error('Cannot register an empty figure');
+  return [left, top, right - left + 1, bottom - top + 1];
+}
+
+export async function registerRigFace(pose: FacePose, cropped: Buffer, standing: Buffer, sheetScale = 1): Promise<FacePose> {
+  const [a, b] = await Promise.all([alphaBounds(cropped), alphaBounds(standing)]);
+  const sx = b[2] / a[2] / sheetScale, sy = b[3] / a[3] / sheetScale;
+  const dx = b[0] / sheetScale - a[0] * sx, dy = b[1] / sheetScale - a[1] * sy;
+  const transform = ([x, y, w, h]: Box): Box => [x * sx + dx, y * sy + dy, w * sx, h * sy];
+  return { ...pose, mouth: transform(pose.mouth), eyes: pose.eyes.map(transform), ...(pose.lips ? { lips: transform(pose.lips) } : {}), ...(pose.suggestedLips ? {suggestedLips:transform(pose.suggestedLips)} : {}) };
+}

package/app/src/scene/face-work.ts

@@ -0,0 +1,71 @@
+// Source-bound, resumable work. Never served as a live face until all checks pass.
+import fs from 'node:fs';
+import { z } from 'zod';
+import { sha256, faceSourceHash } from './faces-data.ts';
+import type { WorldFiles } from '../world/files.ts';
+import { MOUTHS } from '../play/lipsync.ts';
+
+export const FaceWorkId = z.string().regex(/^[a-f0-9]{32}$/);
+export const FaceRevision = z.string().regex(/^[a-f0-9]{64}$/);
+export const LipBox = z.tuple([z.number().min(0).max(1), z.number().min(0).max(1), z.number().positive().max(1), z.number().positive().max(1)])
+  .refine(([x,y,w,h]) => x+w <= 1 && y+h <= 1, 'Lip box must fit inside the image');
+export const MouthName = z.enum(['X','A','B','C','D','E','F','G','H']);
+const Rect = z.tuple([z.number(), z.number(), z.number(), z.number()]);
+const Located = z.object({ head: Rect, mouth: Rect, eyes: z.array(Rect), confidence: z.number() });
+const Work = z.object({
+  id: FaceWorkId, view: z.string(), sourceHash: z.string(), referenceHash: z.string(), take: z.number(),
+  atlas: z.string().regex(/^face-work-[a-f0-9]{64}\.png$/).optional(),
+  accepted: z.boolean().default(false), poses: z.array(Located.nullable()).optional(),
+  lips: z.record(MouthName, LipBox).default({}), reviewed: z.array(MouthName).default([]),
+  stage: z.enum(['atlas','articulation','lips','ready']).default('atlas'), reason: z.string().optional(),
+  invalidFrames: z.array(z.number().int().min(0).max(8)).default([]),
+});
+export type FaceWork = z.infer<typeof Work>;
+const file = (files: WorldFiles, key: string, id: string) => files.path('cast', key, 'face-work', `${FaceWorkId.parse(id)}.json`);
+export const workRevision = (work: FaceWork) => sha256(JSON.stringify(Work.parse(work)));
+export function currentFaceSource(files: WorldFiles, key: string, view: string) {
+  if (view === 'image') return sha256(fs.readFileSync(files.path('cast', `${key}.png`)));
+  if (!/^(model:)?[a-zA-Z0-9_-]+\.png$/.test(view)) throw new Error('Invalid face view');
+  return faceSourceHash(files, key, view);
+}
+export function readFaceWork(files: WorldFiles, key: string, id: string, revision?: string): FaceWork {
+  const work = Work.parse(files.readJson(file(files,key,id)));
+  if (work.id !== id || currentFaceSource(files,key,work.view) !== work.sourceHash) throw new Error('Face source changed; prepare it again before repairing');
+  if (revision !== undefined && workRevision(work) !== FaceRevision.parse(revision)) throw new Error('Face work changed; inspect it again before applying this correction');
+  return work;
+}
+export function saveFaceWork(files: WorldFiles, key: string, work: FaceWork, atlas?: Buffer) {
+  if (atlas) {
+    work.atlas = `face-work-${sha256(atlas)}.png`;
+    files.write(files.path('cast',key,work.atlas),atlas);
+  }
+  const target = file(files,key,work.id);
+  files.write(`${target}.tmp`,work);
+  fs.renameSync(`${target}.tmp`,target);
+}
+export function openFaceWork(files: WorldFiles, key: string, view: string, take: number, reference: Buffer): FaceWork {
+  const sourceHash=currentFaceSource(files,key,view), referenceHash=sha256(reference);
+  const id=sha256(JSON.stringify(['face-work-v1',view,sourceHash,referenceHash,take])).slice(0,32);
+  if (fs.existsSync(file(files,key,id))) return readFaceWork(files,key,id);
+  const work=Work.parse({id,view,sourceHash,referenceHash,take});
+  files.write(files.path('cast',key,'face-work',`${id}-reference.png`),reference);
+  saveFaceWork(files,key,work);
+  return work;
+}
+export const faceWorkAtlas = (files: WorldFiles,key:string,work:FaceWork) => {
+  if (!work.atlas) throw new Error('No atlas yet; prepare the face first');
+  return fs.readFileSync(files.path('cast',key,work.atlas));
+};
+export const faceWorkReference = (files:WorldFiles,key:string,id:string) => fs.readFileSync(files.path('cast',key,'face-work',`${FaceWorkId.parse(id)}-reference.png`));
+export function faceWorkList(files:WorldFiles,key:string) {
+  const dir=files.path('cast',key,'face-work');
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir).filter(f=>/^[a-f0-9]{32}\.json$/.test(f)).flatMap(f=>{
+    try { const work=readFaceWork(files,key,f.slice(0,-5)); return [{id:work.id,revision:workRevision(work),view:work.view,take:work.take,stage:work.stage,reason:work.reason,accepted:work.accepted,invalidFrames:work.invalidFrames.map(i=>MOUTHS[i]),pendingLips:MOUTHS.filter(m=>!work.lips[m])}]; }
+    catch { return []; } // Old source-bound work is retained on disk, but cannot be applied to a repaint.
+  });
+}
+
+export class FacePreparationError extends Error {
+  constructor(readonly failures:string[]) { super(`${failures.length} face/audio preparation failures: ${failures.join('; ')}`); }
+}

package/app/src/scene/faces-data.ts

@@ -0,0 +1,48 @@
+// Validated, source-bound face assets. A repaint disables its old registrations, never another character.
+import fs from 'node:fs';
+import { createHash } from 'node:crypto';
+import { z } from 'zod';
+import type { WorldFiles } from '../world/files.ts';
+import { MOUTHS, type FaceDefinition } from '../play/lipsync.ts';
+
+const Rect = z.tuple([z.number().nonnegative(), z.number().nonnegative(), z.number().positive(), z.number().positive()]);
+const Pose = z.object({ mouth: Rect, eyes: z.array(Rect).max(2), lips: Rect.optional(), suggestedLips:Rect.optional(), lipConfidence:z.number().optional(), sourceOpen:z.boolean().optional(), atlas: z.string().optional() });
+const Mouths = z.object(Object.fromEntries(MOUTHS.map(m => [m, Rect])) as Record<typeof MOUTHS[number], typeof Rect>);
+export const FaceAsset = z.object({ image: z.string(), mouths: Mouths, atlases: z.record(Mouths).optional(), anchors: z.record(Rect).optional(), frames: z.record(Mouths).optional(),
+  sheets: z.record(z.array(Pose.nullable())), sheetHashes: z.record(z.string()).optional(), imagePose: Pose.optional(), rigPose: Pose.optional(), rigHash: z.string().optional(), imageHash: z.string().optional(), seed: z.number().optional(), sourceVersions: z.record(z.string()).optional() });
+export const sha256 = (data: Buffer | string) => createHash('sha256').update(data).digest('hex');
+export function rigFingerprint(files: WorldFiles, key: string): string | undefined {
+  const file = files.path('cast', key, 'rig.json');
+  if (!fs.existsSync(file)) return;
+  const rig = files.readJson(file) as { parts?: Array<{ file: string }> };
+  const parts = (rig.parts ?? []).map(p => files.path('cast', key, p.file));
+  if (parts.some(p => !fs.existsSync(p))) return;
+  return sha256(Buffer.concat([fs.readFileSync(file), ...parts.map(p => fs.readFileSync(p))]));
+}
+export function faceSourceHash(files: WorldFiles, key: string, source: string): string {
+  const rig = source.startsWith('rig:'), model = source.startsWith('model:');
+  const file = rig ? files.path('cast', key, source.slice(4)) : model ? files.path('cast', key, 'model', source.slice(6)) : files.path('cast', key, 'sprites', source);
+  const registration = rig ? files.path('cast', key, 'rig.json') : model ? files.path('cast', key, 'model', 'model.json') : undefined;
+  return sha256(Buffer.concat([fs.readFileSync(file), ...(registration ? [fs.readFileSync(registration)] : [])]));
+}
+const safe = (value: string) => !!value && !value.includes('..') && !value.startsWith('/') && !value.includes('\\');
+
+export function registeredFace(files: WorldFiles, key: string, manifest = 'face.json'): { dir: string; info: FaceDefinition } | undefined {
+  const file = files.path('cast', key, manifest);
+  if (!fs.existsSync(file)) return;
+  try {
+    const info = FaceAsset.parse(files.readJson(file));
+    if (!safe(info.image) || !fs.existsSync(files.path('cast', key, info.image))) return;
+    const source = files.path('cast', `${key}.png`);
+    const imagePose = fs.existsSync(source) && info.imageHash === sha256(fs.readFileSync(source)) ? info.imagePose : undefined;
+    const rigPose = imagePose && info.rigHash === rigFingerprint(files, key) ? info.rigPose : undefined;
+    const sheets = Object.fromEntries(Object.entries(info.sheets).filter(([sheet]) => {
+      const rig = sheet.startsWith('rig:'), model = sheet.startsWith('model:');
+      const name = rig ? sheet.slice(4) : model ? sheet.slice(6) : sheet;
+      if (!safe(name)) return false;
+      const source = rig ? files.path('cast', key, name) : files.path('cast', key, model ? 'model' : 'sprites', name);
+      return fs.existsSync(source) && (!rig || fs.existsSync(files.path('cast', key, 'rig.json'))) && (!model || fs.existsSync(files.path('cast', key, 'model', 'model.json'))) && info.sheetHashes?.[sheet] === faceSourceHash(files, key, sheet);
+    }));
+    if (imagePose || Object.keys(sheets).length) return { dir: `cast/${key}`, info: { ...info, imagePose, rigPose, sheets } };
+  } catch { /* An unfinished face asset keeps the previous figure playable. */ }
+}

package/app/src/scene/faces-settings.ts

@@ -0,0 +1,20 @@
+// Talking mouths (the faces step) on or off for this machine. Off by default: they were barely visible in play
+// and cost about half of a world's model calls and minutes of build time. DIORAMA_FACES=on|off overrides Setup.
+import fs from 'node:fs';
+import path from 'node:path';
+import { configDir } from '../secrets.ts';
+
+const file = () => path.join(configDir(), 'faces.json');
+const env = () => process.env.DIORAMA_FACES;
+
+export function facesState(): { enabled: boolean; env: string | null } {
+  const set = env();
+  if (set !== undefined) return { enabled: /^(1|on|yes|true)$/i.test(set), env: set };
+  try { return { enabled: (JSON.parse(fs.readFileSync(file(), 'utf8')) as { enabled?: boolean }).enabled === true, env: null }; }
+  catch { return { enabled: false, env: null }; }
+}
+
+export function writeFacesSettings(enabled: boolean) {
+  fs.mkdirSync(path.dirname(file()), { recursive: true, mode: 0o700 });
+  fs.writeFileSync(file(), `${JSON.stringify({ enabled }, null, 2)}\n`, { mode: 0o600 });
+}