@gtanczyk/diorama 0.0.0-stage → 0.1.0

package/app/src/meta/openrouter.ts

@@ -0,0 +1,93 @@
+// Other models through OpenRouter, for when a world should be made with something other than Meta's:
+// Gemini image models in place of Muse Image, any chat model in place of Spark. OPENROUTER_API_KEY holds
+// the key; in a cloud session the proxy supplies it. A Google key (GEMINI_API_KEY or GOOGLE_API_KEY) sends Gemini pictures straight to Google instead. Answers say what they cost.
+import sharp from 'sharp';
+import type { Paid } from './memo.ts';
+
+const URL = 'https://openrouter.ai/api/v1/chat/completions';
+
+/** Short names for the image models worth trying, as OpenRouter lists them (2026-10-28). */
+export const IMAGE_MODELS: Record<string, string> = {
+  'gemini-flash': 'google/gemini-3.1-flash-image',
+  'gemini-lite': 'google/gemini-3.1-flash-lite-image',
+  'nano-banana': 'google/gemini-nano-banana-2.1',
+  'gemini-pro': 'google/gemini-3-pro-image',
+};
+
+type Part = { type: 'text'; text: string } | { type: 'image_url'; image_url: { url: string } };
+type Answer = { choices?: { message?: { content?: string | null; images?: { image_url?: { url?: string } }[] } }[]; usage?: { cost?: number }; error?: { message?: string } };
+
+async function post(label: string, body: Record<string, unknown>): Promise<Answer> {
+  for (let n = 1; ; n++) {
+    const res = await fetch(URL, {
+      method: 'POST',
+      signal: AbortSignal.timeout(240_000),
+      headers: { Authorization: `Bearer ${process.env.OPENROUTER_API_KEY ?? 'proxy'}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    if (res.ok) return (await res.json()) as Answer;
+    if (n === 4 || ![408, 429, 500, 502, 503, 504].includes(res.status)) throw new Error(`${label} (${body.model}): ${res.status} ${(await res.text()).slice(0, 300)}`);
+    await new Promise((r) => setTimeout(r, 1000 * 2 ** n));
+  }
+}
+
+const dataUrl = (image: Buffer) => `data:image/${image[0] === 0xff && image[1] === 0xd8 ? 'jpeg' : 'png'};base64,${image.toString('base64')}`;
+const content = (prompt: string, images: Buffer[]): Part[] => [...images.map((i) => ({ type: 'image_url' as const, image_url: { url: dataUrl(i) } })), { type: 'text', text: prompt }];
+
+/** A chat model's text answer to a prompt and images. */
+export async function chat(label: string, model: string, prompt: string, images: Buffer[] = [], system?: string, schema?: Record<string, unknown>): Promise<Paid> {
+  const body = await post(label, {
+    model,
+    messages: [...(system ? [{ role: 'system', content: system }] : []), { role: 'user', content: content(prompt, images) }],
+    // Structured outputs: the model's answer is held to the shape, so it always parses.
+    ...(schema ? { response_format: { type: 'json_schema', json_schema: { name: 'answer', strict: true, schema } } } : {}),
+  });
+  const text = body.choices?.[0]?.message?.content ?? '';
+  if (!text.trim()) throw new Error(`${label} (${model}): empty answer${body.error?.message ? `: ${body.error.message}` : ''}`);
+  return { data: Buffer.from(text, 'utf8'), usd: body.usage?.cost, via: model };
+}
+
+const RATIOS = ['1:1', '2:3', '3:2', '3:4', '4:3', '4:5', '5:4', '9:16', '16:9', '21:9'];
+
+const googleKey = () => process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
+
+/** A Gemini picture straight from Google: the model id without OpenRouter's "google/". */
+async function paintGoogle(label: string, model: string, prompt: string, ratio: string, images: Buffer[]): Promise<Buffer> {
+  const parts = [...images.map((i) => ({ inlineData: { mimeType: i[0] === 0xff && i[1] === 0xd8 ? 'image/jpeg' : 'image/png', data: i.toString('base64') } })), { text: prompt }];
+  for (let n = 1; ; n++) {
+    const res = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`, {
+      method: 'POST',
+      signal: AbortSignal.timeout(240_000),
+      headers: { 'x-goog-api-key': googleKey()!, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ contents: [{ role: 'user', parts }], generationConfig: { responseModalities: ['IMAGE'], imageConfig: { aspectRatio: ratio } } }),
+    });
+    if (res.ok) {
+      const body = (await res.json()) as { candidates?: { content?: { parts?: { inlineData?: { data?: string } }[] } }[] };
+      const data = body.candidates?.[0]?.content?.parts?.find((p) => p.inlineData?.data)?.inlineData?.data;
+      if (!data) throw new Error(`${label} (${model}): no image in the answer`);
+      return Buffer.from(data, 'base64');
+    }
+    if (n === 4 || ![408, 429, 500, 502, 503, 504].includes(res.status)) throw new Error(`${label} (${model}): ${res.status} ${(await res.text()).slice(0, 300)}`);
+    await new Promise((r) => setTimeout(r, 1000 * 2 ** n));
+  }
+}
+
+/**
+ * Paints (no images) or edits (with images) like Muse Image. Muse answers a size request about 1.43x larger
+ * (1712x1280 for 1200x896), so the picture is brought to that size: everything after reads it as a Muse answer.
+ */
+export async function paint(label: string, model: string, prompt: string, size: string, images: Buffer[] = []): Promise<Paid> {
+  const [w, h] = size.split('x').map(Number);
+  const ratio = RATIOS.reduce((best, r) => {
+    const [a, b] = r.split(':').map(Number);
+    const [c, d] = best.split(':').map(Number);
+    return Math.abs(Math.log(a / b / (w / h))) < Math.abs(Math.log(c / d / (w / h))) ? r : best;
+  });
+  const fit = (image: Buffer) => sharp(image).resize(Math.round((w * 1712) / 1200), Math.round((h * 1280) / 896), { fit: 'fill' }).png().toBuffer();
+  // Google's price is not in its answer; the budget counts the call, not its dollars.
+  if (model.startsWith('google/') && googleKey()) return { data: await fit(await paintGoogle(label, model.slice(7), prompt, ratio, images)), via: model };
+  const body = await post(label, { model, modalities: ['image', 'text'], image_config: { aspect_ratio: ratio }, messages: [{ role: 'user', content: content(prompt, images) }] });
+  const url = body.choices?.[0]?.message?.images?.[0]?.image_url?.url;
+  if (!url?.startsWith('data:')) throw new Error(`${label} (${model}): no image in the answer${body.error?.message ? `: ${body.error.message}` : ''}`);
+  return { data: await fit(Buffer.from(url.slice(url.indexOf(',') + 1), 'base64')), usd: body.usage?.cost, via: model };
+}

package/app/src/meta/prices.ts

@@ -0,0 +1,25 @@
+// Meta Model API list prices (dev.meta.ai/docs/pricing-rate-limits, read 2026-10-28), so every call sent can say
+// what it cost. Contributor tier: same Spark, Meta may train on the prompts and answers.
+export const PRICES = {
+  /** Per image returned, generation or edit alike. */
+  museImage: 0.01,
+  /** Per image segmented. */
+  sam: 0.0025,
+  /** Per million tokens. */
+  spark: {
+    standard: { input: 1.25, cached: 0.15, output: 4.25 },
+    contributor: { input: 0.1, cached: 0.002, output: 0.2 },
+  },
+  /** Per hour of audio. */
+  voiceTranscribe: 0.18,
+} as const;
+
+export type SparkTier = keyof typeof PRICES.spark;
+export type Usage = { input_tokens?: number; output_tokens?: number; input_tokens_details?: { cached_tokens?: number } };
+
+export function sparkUsd(tier: SparkTier, usage: Usage | undefined): number | undefined {
+  if (!usage) return undefined;
+  const p = PRICES.spark[tier];
+  const cached = usage.input_tokens_details?.cached_tokens ?? 0;
+  return (((usage.input_tokens ?? 0) - cached) * p.input + cached * p.cached + (usage.output_tokens ?? 0) * p.output) / 1e6;
+}

package/app/src/meta/sam.ts

@@ -0,0 +1,83 @@
+// SAM 3.1: finds every instance of one concept in an image, or one object per box.
+import { buildObjectPrompt, decodeMaskToRaster, parseImageStream, recordsOfKind, type ResponsesEvent } from '@meta-sam/parser';
+import { emptyMask, imageSize, type Mask } from '../image/mask.ts';
+import { metaPost } from './http.ts';
+import { inputHash, type Memo } from './memo.ts';
+import { PRICES } from './prices.ts';
+
+export const SAM = 'sam-3.1';
+
+/** Pixel box; right and bottom are exclusive, as in @meta-sam/parser. */
+export type Box = { left: number; top: number; right: number; bottom: number };
+export type Found = { id: string; box: Box; confidence?: number; mask: Mask };
+/** At most this many boxes go in one request. */
+export const SAM_MAX_BOXES = 16;
+
+async function* serverEvents(res: Response): AsyncIterable<ResponsesEvent> {
+  const decoder = new TextDecoder();
+  let buffered = '';
+  for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
+    buffered += decoder.decode(chunk, { stream: true });
+    let nl;
+    while ((nl = buffered.indexOf('\n')) >= 0) {
+      const line = buffered.slice(0, nl).trim();
+      buffered = buffered.slice(nl + 1);
+      const data = line.startsWith('data:') ? line.slice(5).trim() : '';
+      if (data && data !== '[DONE]') yield JSON.parse(data) as ResponsesEvent;
+    }
+  }
+}
+
+async function* fromText(text: string): AsyncIterable<ResponsesEvent> {
+  const where = { item_id: 'memo', output_index: 0, content_index: 0 };
+  yield { type: 'response.output_text.delta', ...where, delta: text };
+  yield { type: 'response.output_text.done', ...where, text };
+  yield { type: 'response.completed' };
+}
+
+async function records(events: AsyncIterable<ResponsesEvent>) {
+  const stream = parseImageStream(events);
+  for await (const _snapshot of stream);
+  return stream.finalResult;
+}
+
+/** Turns SAM's output text into one full-size mask per instance found. */
+export async function readSamOutput(text: string, width: number, height: number): Promise<Found[]> {
+  const { records: all } = await records(fromText(text));
+  return recordsOfKind(all, 'mask').map((r) => {
+    const raster = decodeMaskToRaster(r.mask);
+    const { left, top, right, bottom } = r.bounds;
+    const mask = emptyMask(width, height);
+    // The raster covers the bounds box at its own resolution.
+    for (let y = Math.max(top, 0); y < Math.min(bottom, height); y++) {
+      const ry = Math.floor(((y - top) / (bottom - top)) * r.mask.height);
+      for (let x = Math.max(left, 0); x < Math.min(right, width); x++) {
+        const rx = Math.floor(((x - left) / (right - left)) * r.mask.width);
+        mask.bits[y * width + x] = raster[ry * r.mask.width + rx] ? 1 : 0;
+      }
+    }
+    return { id: r.objectId, box: { left, top, right, bottom }, confidence: r.confidence, mask };
+  });
+}
+
+/** Segments a PNG by a concept ("rowing boat") or by boxes. An empty list means nothing was found. */
+export async function samFind(memo: Memo, { label, image, prompt }: { label: string; image: Buffer; prompt: string | Box[] }): Promise<Found[]> {
+  const { width, height } = await imageSize(image);
+  if (Array.isArray(prompt) && prompt.length > SAM_MAX_BOXES) throw new Error(`sam ${label}: at most ${SAM_MAX_BOXES} boxes per request`);
+  const text = typeof prompt === 'string'
+    ? prompt
+    : buildObjectPrompt({ size: { width, height }, objects: prompt.map((box, id) => ({ id, prompts: [{ box }] })) });
+  const output = await memo.remember('sam', label, inputHash(SAM, text, image), 'txt', async () => {
+    const res = await metaPost('/responses', {
+      model: SAM,
+      input: [{ type: 'message', role: 'user', content: [
+        { type: 'input_text', text },
+        { type: 'input_image', image_url: `data:image/png;base64,${image.toString('base64')}` },
+      ] }],
+      metadata: { include_confidence: 'true' },
+      stream: true,
+    }, 'text/event-stream');
+    return { data: Buffer.from((await records(serverEvents(res))).rawOutput, 'utf8'), usd: PRICES.sam };
+  });
+  return readSamOutput(output.toString('utf8'), width, height);
+}

package/app/src/meta/schema.ts

@@ -0,0 +1,63 @@
+// A zod shape as the JSON Schema that structured outputs (Anthropic output_config.format, OpenRouter
+// response_format) hold a model's answer to: every object closed, no length or number limits (the API takes none;
+// zod still checks them after), fields with a default asked for outright so few stay optional.
+import { z } from 'zod';
+
+type Json = Record<string, unknown>;
+
+class Unsupported extends Error {}
+
+function convert(t: z.ZodTypeAny): Json {
+  const def = t._def as { typeName: string } & Record<string, any>;
+  switch (def.typeName) {
+    case 'ZodString':
+      return { type: 'string' };
+    case 'ZodNumber':
+      return { type: (def.checks as { kind: string }[]).some((c) => c.kind === 'int') ? 'integer' : 'number' };
+    case 'ZodBoolean':
+      return { type: 'boolean' };
+    case 'ZodNull':
+      return { type: 'null' };
+    case 'ZodLiteral':
+      return { const: def.value };
+    case 'ZodEnum':
+      return { type: 'string', enum: def.values };
+    case 'ZodArray':
+      return { type: 'array', items: convert(def.type) };
+    case 'ZodObject': {
+      if (def.unknownKeys === 'passthrough' || def.catchall?._def.typeName !== 'ZodNever') throw new Unsupported('open object');
+      const shape = (t as z.AnyZodObject).shape as Record<string, z.ZodTypeAny>;
+      const required = Object.keys(shape).filter((k) => shape[k]._def.typeName !== 'ZodOptional');
+      return { type: 'object', properties: Object.fromEntries(Object.entries(shape).map(([k, v]) => [k, convert(v)])), required, additionalProperties: false };
+    }
+    case 'ZodUnion':
+    case 'ZodDiscriminatedUnion':
+      return { anyOf: [...(def.options as z.ZodTypeAny[])].map(convert) };
+    case 'ZodNullable':
+      return { anyOf: [convert(def.innerType), { type: 'null' }] };
+    case 'ZodOptional':
+    case 'ZodDefault':
+    case 'ZodCatch':
+      return convert(def.innerType);
+    case 'ZodEffects':
+      return convert(def.schema);
+    case 'ZodBranded':
+      return convert(def.type);
+    case 'ZodPipeline':
+      return convert(def.in);
+    default:
+      // Records, tuples, any: no closed shape to hold the model to.
+      throw new Unsupported(def.typeName);
+  }
+}
+
+/** The JSON Schema for a structured answer, or undefined when the shape cannot be held (records, tuples, any). */
+export function answerSchema(t: z.ZodTypeAny): Json | undefined {
+  try {
+    const schema = convert(t);
+    return schema.type === 'object' ? schema : undefined;
+  } catch (error) {
+    if (error instanceof Unsupported) return undefined;
+    throw error;
+  }
+}

package/app/src/meta/spark.ts

@@ -0,0 +1,220 @@
+// Muse Spark: plans, briefs, locates things on images and judges them.
+import { metaPost } from './http.ts';
+import sharp from 'sharp';
+import { inputHash, type Memo, type Paid } from './memo.ts';
+import { envValue } from '../secrets.ts';
+import { sparkChoice } from './choice.ts';
+import { chat } from './openrouter.ts';
+import { sparkUsd, type SparkTier, type Usage } from './prices.ts';
+import { answerSchema } from './schema.ts';
+import type { ZodType } from 'zod';
+
+export const SPARK = 'muse-spark-1.3';
+
+/**
+ * Which Spark price tier a call goes to. DIORAMA_SPARK_TIER: "contributor" (Meta's cheaper tier, ~1/15 the price;
+ * Meta may train on the prompt and answer), "standard", or "auto" (the default): contributor, except calls that
+ * carry the person's own photos or drawings, which stay standard. Same model either way, so they share answers.
+ */
+export function sparkTier(personal = false): SparkTier {
+  const want = process.env.DIORAMA_SPARK_TIER ?? 'auto';
+  if (want === 'standard' || want === 'contributor') return want;
+  return personal ? 'standard' : 'contributor';
+}
+
+/**
+ * Which model answers Spark's asks (Studio Setup, or DIORAMA_SPARK_VIA): "meta" (the default), "claude" (Anthropic's API with
+ * DIORAMA_ANTHROPIC_API_KEY, model DIORAMA_CLAUDE_MODEL, default Sonnet 5.5) or "openrouter:<model id>".
+ * Another model's answers are kept under their own memo keys, so a Meta build never replays them.
+ */
+export function sparkVia(): string | null {
+  const via = sparkChoice();
+  if (!via || via === 'meta') return null;
+  return via === 'claude' ? `anthropic:${process.env.DIORAMA_CLAUDE_MODEL || 'claude-sonnet-5-5'}` : via;
+}
+
+export function imageDataUrl(image: Buffer): string {
+  const jpeg = image[0] === 0xff && image[1] === 0xd8;
+  return `data:image/${jpeg ? 'jpeg' : 'png'};base64,${image.toString('base64')}`;
+}
+
+/**
+ * personal: the images are the person's own photos or drawings (kept off the contributor tier by default).
+ * schema: the answer's shape. Claude (direct or via OpenRouter) is held to it by structured outputs, so its JSON
+ * always parses. The caller still checks the answer (zod limits, its own rules): structured outputs hold only the shape.
+ */
+export type SparkAsk<T = unknown> = { label: string; prompt: string; images?: Buffer[]; personal?: boolean; schema?: ZodType<T, any, any> };
+
+const sparkHash = ({ prompt, images = [] }: SparkAsk) => inputHash(sparkVia()?.replace(/^anthropic:/, '') ?? SPARK, prompt, ...images);
+
+async function askMeta({ label, prompt, images = [], personal }: SparkAsk): Promise<Paid> {
+  const tier = sparkTier(personal);
+  const input = images.length === 0
+    ? prompt
+    : [{ role: 'user', content: [{ type: 'input_text', text: prompt }, ...images.map((i) => ({ type: 'input_image', image_url: imageDataUrl(i) }))] }];
+  const res = await metaPost('/responses', { model: tier === 'contributor' ? `${SPARK}-contributor` : SPARK, input });
+  const body = (await res.json()) as { output?: { content?: { type: string; text?: string }[] }[]; usage?: Usage };
+  const text = (body.output ?? []).flatMap((o) => o.content ?? []).filter((c) => c.type === 'output_text').map((c) => c.text ?? '').join('');
+  if (!text.trim()) throw new Error(`spark ${label}: empty answer`);
+  return { data: Buffer.from(text, 'utf8'), usd: sparkUsd(tier, body.usage), via: tier === 'contributor' ? 'contributor' : undefined };
+}
+
+/** Claude takes images up to 5 MB each; a big painting goes as a JPEG. */
+async function claudeImage(image: Buffer) {
+  const jpeg = image[0] === 0xff && image[1] === 0xd8;
+  if (image.length < 3_500_000) return { type: 'base64', media_type: jpeg ? 'image/jpeg' : 'image/png', data: image.toString('base64') };
+  return { type: 'base64', media_type: 'image/jpeg', data: (await sharp(image).jpeg({ quality: 88 }).toBuffer()).toString('base64') };
+}
+
+/**
+ * Shapes the API would not compile into a grammar (the whole story is too large for it), asked plainly from then on:
+ * the JSON repair in jsonIn and askSparkJson's retries still hold those answers.
+ */
+const refused = new Set<string>();
+const heldTo = (schema?: ZodType) => {
+  const format = schema && answerSchema(schema);
+  return format && !refused.has(JSON.stringify(format)) ? format : undefined;
+};
+function refuse(label: string, format: object, why: string) {
+  refused.add(JSON.stringify(format));
+  // Said in plain words on the progress card; the API's own message only says which limit.
+  const limit = /too large|too complex/i.test(why) ? 'the answer\'s shape is too big for it' : 'it did not accept the answer\'s shape';
+  console.log(`  ${label}: Claude asked for plain JSON instead (${limit})`);
+}
+
+async function askOpenRouter(model: string, { label, prompt, images, schema }: SparkAsk): Promise<Paid> {
+  const format = heldTo(schema);
+  try {
+    return await chat(`spark ${label}`, model, prompt, images, CLAUDE_SYSTEM, format);
+  } catch (error) {
+    if (!format || !/: 400 /.test(String(error))) throw error;
+    refuse(label, format, String(error));
+    return chat(`spark ${label}`, model, prompt, images, CLAUDE_SYSTEM);
+  }
+}
+
+/** Spark's prompts were written for Spark; this keeps another model's answers in the shape they expect. */
+const CLAUDE_SYSTEM = 'You stand in for a vision-language model inside a game-making pipeline. When asked for JSON, reply with one valid JSON value and nothing else: no prose, no code fences. Inside JSON strings never use a plain double quote; for quotations in any language use the typographic quotes of that language (for Polish „ and ”) or single quotes. Keep the answer as short as the request allows.';
+
+async function askClaude(model: string, { label, prompt, images = [], schema }: SparkAsk): Promise<Paid> {
+  const key = envValue('DIORAMA_ANTHROPIC_API_KEY');
+  if (!key) throw new Error('DIORAMA_SPARK_VIA=claude needs DIORAMA_ANTHROPIC_API_KEY, ANTHROPIC_API_KEY or CLAUDE_API_KEY; openrouter:anthropic/claude-sonnet-5.5 goes without');
+  const content = [...(await Promise.all(images.map(async (i) => ({ type: 'image', source: await claudeImage(i) })))), { type: 'text', text: prompt }];
+  let format = heldTo(schema);
+  for (let n = 1, dropped = 0; ; n++) {
+    let res: Response;
+    let body: { content?: { type: string; text?: string }[]; stop_reason?: string } | undefined;
+    try {
+      res = await fetch('https://api.anthropic.com/v1/messages', {
+        method: 'POST',
+        signal: AbortSignal.timeout(240_000),
+        headers: { 'x-api-key': key, 'anthropic-version': '2023-06-01', 'content-type': 'application/json' },
+        body: JSON.stringify({
+          model,
+          max_tokens: 32000,
+          system: CLAUDE_SYSTEM,
+          messages: [{ role: 'user', content }],
+          ...(format ? { output_config: { format: { type: 'json_schema', schema: format } } } : {}),
+        }),
+      });
+      if (res.ok) body = (await res.json()) as typeof body;
+    } catch (error) {
+      // A dropped or stalled connection ("fetch failed": a busy uplink, Wi-Fi gone) is asked again, like metaPost
+      // does for Meta, instead of failing the whole step: up to ~3 min of backoff in all.
+      if (++dropped >= 6) throw error;
+      console.log(`  spark ${label} (claude): ${error instanceof Error && error.name === 'TimeoutError' ? 'no answer in 240 s' : error instanceof Error ? error.message.slice(0, 120) : error}; trying again (${dropped + 1}/6)`);
+      await new Promise((r) => setTimeout(r, Math.min(30_000, 2000 * 2 ** dropped)));
+      n--;
+      continue;
+    }
+    if (body) {
+      if (body.stop_reason === 'max_tokens') throw new Error(`spark ${label} (claude): the answer ran past ${32000} tokens and was cut off`);
+      const text = (body.content ?? []).filter((c) => c.type === 'text').map((c) => c.text ?? '').join('');
+      if (!text.trim()) throw new Error(`spark ${label} (claude): empty answer`);
+      const usage = (body as { usage?: { input_tokens?: number; output_tokens?: number } }).usage;
+      // Paid from Anthropic credits, not Meta's: the budget counts it apart. Sonnet 5.5 list price.
+      return { data: Buffer.from(text, 'utf8'), usd: ((usage?.input_tokens ?? 0) * 2 + (usage?.output_tokens ?? 0) * 10) / 1e6, via: model };
+    }
+    if (res.status === 400 && format) {
+      refuse(label, format, await res.text());
+      format = undefined;
+      continue;
+    }
+    if (n === 4 || ![429, 500, 502, 503, 529].includes(res.status)) throw new Error(`spark ${label} (claude): ${res.status} ${(await res.text()).slice(0, 300)}`);
+    await new Promise((r) => setTimeout(r, 1000 * 2 ** n));
+  }
+}
+
+export async function askSpark(memo: Memo, ask: SparkAsk): Promise<string> {
+  const via = sparkVia();
+  const answer = await memo.remember('spark', ask.label, sparkHash(ask), 'txt', () =>
+    !via ? askMeta(ask) : via.startsWith('anthropic:') ? askClaude(via.slice(10), ask) : askOpenRouter(via.replace(/^openrouter:/, ''), ask));
+  return answer.toString('utf8');
+}
+
+/**
+ * A bare `"` inside a string, escaped: models writing Polish or German quote „like this" close with a plain
+ * quote. A quote ends the string only when what follows can come after a string (`,` `:` `}` `]` or the end).
+ */
+export function escapeStrayQuotes(json: string): string {
+  let out = '';
+  let inString = false;
+  for (let i = 0; i < json.length; i++) {
+    const c = json[i];
+    if (inString && c === '\\') {
+      out += c + (json[++i] ?? '');
+      continue;
+    }
+    if (c === '"') {
+      if (!inString) inString = true;
+      else if (/^\s*([,:}\]]|$)/.test(json.slice(i + 1, i + 40))) inString = false;
+      else {
+        out += '\\"';
+        continue;
+      }
+    }
+    // A raw line break inside a string is not JSON either.
+    out += inString && c === '\n' ? '\\n' : c;
+  }
+  return out;
+}
+
+/** The first `{` to the last `}` of a reply, parsed. Spark often wraps JSON in prose or fences. */
+export function jsonIn<T = unknown>(text: string): T {
+  const from = text.indexOf('{');
+  const to = text.lastIndexOf('}');
+  if (from < 0 || to <= from) throw new Error(`no JSON object in: ${text.slice(0, 160)}`);
+  const body = text.slice(from, to + 1);
+  try {
+    return JSON.parse(body) as T;
+  } catch (error) {
+    try {
+      return JSON.parse(escapeStrayQuotes(body)) as T;
+    } catch {
+      throw error;
+    }
+  }
+}
+
+/**
+ * Asks for JSON. A reply that is not JSON is forgotten, so the next run asks again instead of replaying it, and is
+ * asked again at once (twice more), told what broke: a long reply in Polish often quotes a word with bare double
+ * quotes inside a string, and one broken story should not stop the whole build.
+ */
+export async function askSparkJson<T = unknown>(memo: Memo, ask: SparkAsk<T>): Promise<T> {
+  let last: unknown;
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const tried: SparkAsk<T> = attempt === 0 ? ask : {
+      ...ask,
+      prompt: `${ask.prompt}\n\nYour previous answer was not valid JSON (${last instanceof Error ? last.message.slice(0, 120) : 'unreadable'}). Answer with valid JSON only: inside a string, never use a bare double quote (write \\" or use „…” or single quotes).`,
+    };
+    const text = await askSpark(memo, tried);
+    try {
+      return jsonIn<T>(text);
+    } catch (error) {
+      memo.forget('spark', sparkHash(tried), 'txt');
+      last = error;
+    }
+  }
+  throw last;
+}

package/app/src/meta/step.ts

@@ -0,0 +1,14 @@
+// Which build step a model call belongs to. The build runs steps side by side, so "calls since the step began"
+// counts other steps' calls too; each call is tagged with the step that asked it instead (memo/calls.jsonl `step`).
+import { AsyncLocalStorage } from 'node:async_hooks';
+
+const current = new AsyncLocalStorage<string>();
+
+/** Runs `work` with every model call inside it counted to `step`. */
+export function inStep<T>(step: string, work: () => Promise<T>): Promise<T> {
+  return current.run(step, work);
+}
+
+export function currentStep(): string | undefined {
+  return current.getStore();
+}

package/app/src/meta/voice-file.ts

@@ -0,0 +1,78 @@
+// Meta Model API's file route: https://dev.meta.ai/docs/api-reference/voice/transcribe
+import { randomUUID } from 'node:crypto';
+import { z } from 'zod';
+import { apiKey, META_API, MetaHttpError } from './http.ts';
+import type { SpeechOptions, SttProvider, Transcript } from '../speech/types.ts';
+const MODEL = 'muse-voice-transcribe-1.0';
+const ResponseBody = z.object({ sessionId: z.string(), transcript: z.string(), audioDurationMs: z.number().nonnegative(), turns: z.array(z.object({ turnId: z.number(), startMs: z.number().nonnegative(), endMs: z.number().nonnegative(), transcript: z.string(), speaker: z.string().optional() })) });
+/** RIFF/WAVE is required by the file API. No ffmpeg conversion or lossy encoding is needed for PCM. */
+export function wav24k(pcm: Buffer): Buffer {
+  const h = Buffer.alloc(44);
+  h.write('RIFF'); h.writeUInt32LE(pcm.length + 36, 4); h.write('WAVEfmt ', 8); h.writeUInt32LE(16, 16);
+  h.writeUInt16LE(1, 20); h.writeUInt16LE(1, 22); h.writeUInt32LE(24000, 24); h.writeUInt32LE(48000, 28);
+  h.writeUInt16LE(2, 32); h.writeUInt16LE(16, 34); h.write('data', 36); h.writeUInt32LE(pcm.length, 40);
+  return Buffer.concat([h, pcm]);
+}
+async function fileTranscribe(pcm: Buffer, options: SpeechOptions): Promise<Transcript> {
+  const seconds = pcm.length / 48000;
+  // A clipped utterance can otherwise be returned with empty text. Silence lets the model finish the last turn.
+  const padding = Math.min(1, 600 - seconds);
+  const wav = wav24k(Buffer.concat([pcm, Buffer.alloc(Math.floor(padding * 48000))])), sessionId = randomUUID(), key = apiKey();
+  const request = { model: MODEL, audioEncoding: 'WAV', mode: 'DIARIZATION', ...(options.languages?.length && { languageBias: options.languages }), ...(options.keywords?.length && { keywords: options.keywords }) };
+  let serverErrors = 0;
+  for (let attempt = 0; ; attempt++) {
+    try {
+      const body = new FormData();
+      body.set('request', new Blob([JSON.stringify(request)], { type: 'application/json' }));
+      body.set('audio', new Blob([new Uint8Array(wav)], { type: 'audio/wav' }), 'speech.wav');
+      const response = await fetch(`${META_API}/asr/transcribe?sessionId=${sessionId}`, { method: 'POST', headers: { Authorization: `Bearer ${key}`, Accept: 'application/json' }, body, signal: AbortSignal.timeout(240000) });
+      if (!response.ok) throw new MetaHttpError(response.status, `Meta file STT: ${response.status} ${(await response.text()).slice(0, 300)}`);
+      const got = ResponseBody.parse(await response.json());
+      if (got.turns.some(t => t.endMs < t.startMs || t.endMs > (seconds + padding + .5) * 1000)) throw new z.ZodError([{ code: 'custom', path: ['turns'], message: 'STT timestamp lies outside its clip' }]);
+      const turns = got.turns.filter(t => t.transcript.trim() && t.startMs < seconds * 1000);
+      return { provider: 'meta', model: MODEL, text: got.transcript, timing: turns.length ? 'turn' : 'none', speakerScope: got.sessionId,
+        segments: turns.map(t => ({ start: t.startMs / 1000, end: Math.min(seconds, t.endMs / 1000), text: t.transcript, ...(t.speaker && { speaker: t.speaker }) })) };
+    } catch (error) {
+      // Retry server-budget errors once, and bounded rate-limit/network errors with backoff.
+      if (error instanceof z.ZodError || (error instanceof MetaHttpError && ![408, 429, 500, 502, 503, 504].includes(error.status))) throw error;
+      if (error instanceof MetaHttpError && error.status >= 500 && ++serverErrors > 1) throw error;
+      if (attempt >= 3) throw error;
+      await new Promise(r => setTimeout(r, 1000 * 2 ** attempt));
+    }
+  }
+}
+const QuickBody = z.object({ transcript: z.string() });
+// A few file requests at a time: the realtime route takes the clip's own length and one stream per process, the file
+// route answers a short line in 2-3 s and takes several at once.
+const QUICK_LANES = 4;
+let quickActive = 0;
+const quickWaiting: Array<() => void> = [];
+/**
+ * Plain transcript of one short clip on the file route (PUSH_TO_TALK: one speaker, no turns), for listening back to
+ * a voice take. It sometimes writes a word twice ("jest jest"), so callers confirm a mismatch on the realtime route.
+ */
+export async function quickTranscribe(pcm24k: Buffer): Promise<string> {
+  if (quickActive >= QUICK_LANES) await new Promise<void>((r) => quickWaiting.push(r));
+  quickActive++;
+  try {
+    const wav = wav24k(Buffer.concat([pcm24k, Buffer.alloc(48000)]));
+    const request = { model: MODEL, audioEncoding: 'WAV', mode: 'PUSH_TO_TALK' };
+    for (let attempt = 0; ; attempt++) {
+      try {
+        const body = new FormData();
+        body.set('request', new Blob([JSON.stringify(request)], { type: 'application/json' }));
+        body.set('audio', new Blob([new Uint8Array(wav)], { type: 'audio/wav' }), 'speech.wav');
+        const response = await fetch(`${META_API}/asr/transcribe?sessionId=${randomUUID()}`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey()}`, Accept: 'application/json' }, body, signal: AbortSignal.timeout(60000) });
+        if (!response.ok) throw new MetaHttpError(response.status, `Meta file STT: ${response.status} ${(await response.text()).slice(0, 300)}`);
+        return QuickBody.parse(await response.json()).transcript.trim();
+      } catch (error) {
+        if (error instanceof z.ZodError || (error instanceof MetaHttpError && ![408, 429, 500, 502, 503, 504].includes(error.status)) || attempt >= 2) throw error;
+        await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt));
+      }
+    }
+  } finally {
+    quickActive--;
+    quickWaiting.shift()?.();
+  }
+}
+export const metaFileStt: SttProvider = { id: 'meta', model: MODEL, capabilities: { timing: 'turn', speakers: true, maxSeconds: 600 }, transcribe: fileTranscribe };