@1agh/maude 0.43.0 → 0.44.0

package/apps/studio/draw/palette.ts

@@ -150,6 +150,40 @@ export function oklchToHex(o: Oklch): string {
   return toHex(oklchToRgb(o));
 }
 
+function srgb8ToLinear(c8: number): number {
+  const c = c8 / 255;
+  return c <= 0.04045 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4;
+}
+
+/**
+ * Convert clamped sRGB 0–255 to OKLCH — the exact inverse of {@link oklchToRgb}
+ * (same Björn Ottosson constants). Used by DDR-172's token importer to
+ * normalize hex/rgb() input colors into a DS's declared `oklch` colorSpace.
+ */
+export function rgbToOklch({ r, g, b }: Rgb): Oklch {
+  const rLin = srgb8ToLinear(r);
+  const gLin = srgb8ToLinear(g);
+  const bLin = srgb8ToLinear(b);
+
+  const lc = 0.4122214708 * rLin + 0.5363325363 * gLin + 0.0514459929 * bLin;
+  const mc = 0.2119034982 * rLin + 0.6806995451 * gLin + 0.1073969566 * bLin;
+  const sc = 0.0883024619 * rLin + 0.2817188376 * gLin + 0.6299787005 * bLin;
+
+  const l_ = Math.cbrt(lc);
+  const m_ = Math.cbrt(mc);
+  const s_ = Math.cbrt(sc);
+
+  const l = 0.2104542553 * l_ + 0.793617785 * m_ - 0.0040720468 * s_;
+  const a = 1.9779984951 * l_ - 2.428592205 * m_ + 0.4505937099 * s_;
+  const bComp = 0.0259040371 * l_ + 0.7827717662 * m_ - 0.808675766 * s_;
+
+  const c = Math.sqrt(a * a + bComp * bComp);
+  let h = (Math.atan2(bComp, a) * 180) / Math.PI;
+  if (h < 0) h += 360;
+
+  return { l, c, h };
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Ramp generation
 // ─────────────────────────────────────────────────────────────────────────────

package/apps/studio/footage/schema.test.ts

@@ -7,11 +7,13 @@
 import { describe, expect, test } from 'bun:test';
 
 import {
+  AI_GENERATED_TAG,
   EDL_VERSION,
   type Edl,
   emptyFootageAnalysis,
   FOOTAGE_ANALYSIS_VERSION,
   type FootageAnalysis,
+  generatedClipAnalysis,
   isEmptyAnalysis,
   validateEdl,
   validateFootageAnalysis,
@@ -192,4 +194,94 @@ describe('validateEdl', () => {
   test('rejects a music bed with a non-relative asset', () => {
     expect(validateEdl({ ...GOOD_EDL, music: { asset: 'http://x/y.mp3' } }).ok).toBe(false);
   });
+
+  // feature-ai-media-generation Phase 2 — layered audio tracks + captions.
+  test('accepts layered audioTracks (music/VO/SFX)', () => {
+    const r = validateEdl({
+      ...GOOD_EDL,
+      audioTracks: [
+        { asset: 'assets/aaaaaaaa.mp3', kind: 'music', gainDb: -12, fadeOutFrames: 30 },
+        { asset: 'assets/bbbbbbbb.mp3', kind: 'voiceover', startFrame: 15, durationFrames: 90 },
+        { asset: 'assets/cccccccc.mp3', kind: 'sfx', startFrame: 40, name: 'whoosh' },
+      ],
+    });
+    expect(r).toEqual({ ok: true, errors: [] });
+  });
+
+  test('rejects an audio track with a bad kind / non-relative asset / out-of-range gain', () => {
+    expect(
+      validateEdl({ ...GOOD_EDL, audioTracks: [{ asset: 'assets/a.mp3', kind: 'bass' }] }).ok
+    ).toBe(false);
+    expect(validateEdl({ ...GOOD_EDL, audioTracks: [{ asset: 'http://x/a.mp3' }] }).ok).toBe(false);
+    expect(
+      validateEdl({ ...GOOD_EDL, audioTracks: [{ asset: 'assets/a.mp3', gainDb: 999 }] }).ok
+    ).toBe(false);
+    expect(validateEdl({ ...GOOD_EDL, audioTracks: [{ kind: 'music' }] }).ok).toBe(false); // asset required
+  });
+
+  test('accepts a captions track', () => {
+    const r = validateEdl({
+      ...GOOD_EDL,
+      captions: {
+        style: 'lower-third',
+        cues: [
+          { startSec: 0, endSec: 0.8, text: 'Hello there.' },
+          { startSec: 1.0, endSec: 1.8, text: 'This is Maude.' },
+        ],
+      },
+    });
+    expect(r).toEqual({ ok: true, errors: [] });
+  });
+
+  test('rejects captions with a bad style, missing text, or non-array cues', () => {
+    expect(validateEdl({ ...GOOD_EDL, captions: { style: 'sideways', cues: [] } }).ok).toBe(false);
+    expect(validateEdl({ ...GOOD_EDL, captions: { cues: [{ startSec: 0, endSec: 1 }] } }).ok).toBe(
+      false
+    ); // text required
+    expect(validateEdl({ ...GOOD_EDL, captions: { cues: 'nope' } }).ok).toBe(false);
+  });
+
+  test('rejects an unknown top-level key (audioTracks typo)', () => {
+    expect(validateEdl({ ...GOOD_EDL, audioTrack: [] }).ok).toBe(false);
+  });
+});
+
+describe('generatedClipAnalysis (Task 3.2 — AI-clip provenance stub)', () => {
+  test('produces a VALID, empty-shots, ai-generated-tagged analysis', () => {
+    const a = generatedClipAnalysis('assets/deadbeef.mp4', {
+      provider: 'gemini',
+      model: 'veo-3.1-generate-preview',
+      prompt: 'a drone push over a mountain lake at dawn',
+    });
+    expect(validateFootageAnalysis(a)).toEqual({ ok: true, errors: [] });
+    expect(a.asset).toBe('assets/deadbeef.mp4');
+    // Empty shots ⇒ the reel's shot-aware cache still runs the analyst on it.
+    expect(isEmptyAnalysis(a)).toBe(true);
+    expect(a.tags).toContain(AI_GENERATED_TAG);
+    expect(a.tags).toContain('gemini');
+    expect(a.summary).toContain('mountain lake');
+  });
+
+  test('tolerates a missing prompt/provider and stays valid', () => {
+    const a = generatedClipAnalysis('assets/abcd1234.webm');
+    expect(validateFootageAnalysis(a)).toEqual({ ok: true, errors: [] });
+    expect(a.tags).toEqual([AI_GENERATED_TAG]);
+  });
+
+  test('maps C0/C1 control chars and caps an oversized prompt under the summary limit', () => {
+    const a = generatedClipAnalysis('assets/beadfeed.mp4', {
+      provider: 'gemini',
+      // C0 newline + C1 NEL (\u0085) must both collapse to a space — no fabricated lines.
+      prompt: `injected  line\n\u0085next${'x'.repeat(8000)}`,
+    });
+    expect(validateFootageAnalysis(a).ok).toBe(true);
+    expect(a.summary?.length ?? 0).toBeLessThanOrEqual(3901);
+    // No C0 (0x00–0x1f) or C1 (0x7f–0x9f) control byte survives into the sidecar.
+    expect(
+      [...(a.summary ?? '')].some((c) => {
+        const n = c.charCodeAt(0);
+        return n <= 0x1f || (n >= 0x7f && n <= 0x9f);
+      })
+    ).toBe(false);
+  });
 });

package/apps/studio/footage/schema.ts

@@ -197,6 +197,52 @@ export interface EdlMusic {
   fadeOutFrames?: number;
 }
 
+/** feature-ai-media-generation Phase 2 — the role an audio track plays in the cut. */
+export type EdlAudioKind = 'music' | 'voiceover' | 'sfx';
+
+/**
+ * One audio track in the cut (DDR-164 Phase 2). Generalizes the single `music`
+ * bed into a LIST so a reel can layer generated music + voiceover + per-beat
+ * SFX. `startFrame`/`durationFrames` place + trim it on the OUTPUT timeline (at
+ * the Edl's `fps`); omit both for a whole-reel bed. The codegen emits an
+ * `<Audio src="assets/…">` inside a `<Sequence from={startFrame}>`.
+ */
+export interface EdlAudioTrack {
+  /** `assets/<sha8>.<ext>` audio asset. */
+  asset: string;
+  /** Role — drives default gain/ducking guidance in the codegen. Default 'music'. */
+  kind?: EdlAudioKind;
+  /** Where the track enters, OUTPUT frames from the start. Default 0. */
+  startFrame?: number;
+  /** Trim length in OUTPUT frames. Omit ⇒ play to the end of the source. */
+  durationFrames?: number;
+  /** Volume adjustment in dB (negative = quieter; e.g. a −12 dB music bed under VO). */
+  gainDb?: number;
+  fadeInFrames?: number;
+  fadeOutFrames?: number;
+  /** Stable Timeline identity ("vo", "bed", "whoosh-3"). */
+  name?: string;
+}
+
+/** One rendered caption cue on the OUTPUT timeline — SECONDS (matches captions.ts). */
+export interface EdlCaptionCue {
+  startSec: number;
+  endSec: number;
+  text: string;
+}
+
+/**
+ * The caption/subtitle track (DDR-164 Phase 2), fed by `generation/captions.ts`
+ * (whisper.cpp local or ElevenLabs Scribe / Groq cloud). Stored as parsed cues
+ * (seconds) rather than a raw SRT file so the codegen can render them as overlays
+ * without re-parsing; the codegen converts seconds → frames at the Edl's `fps`.
+ */
+export interface EdlCaptions {
+  cues: EdlCaptionCue[];
+  /** Where the caption sits on screen. Default 'lower-third'. */
+  style?: 'lower-third' | 'centered' | 'top';
+}
+
 /**
  * The edit decision list — the director's output, one per cut. `beats` is the
  * ordered timeline; the composition's total length = sum(durationFrames) −
@@ -211,7 +257,12 @@ export interface Edl {
   width?: number;
   height?: number;
   beats?: EdlBeat[];
+  /** A single whole-reel music bed (kept for backwards compatibility). */
   music?: EdlMusic | null;
+  /** feature-ai-media-generation Phase 2 — layered audio (music/VO/SFX). */
+  audioTracks?: EdlAudioTrack[];
+  /** feature-ai-media-generation Phase 2 — the caption/subtitle track. */
+  captions?: EdlCaptions | null;
 }
 
 // ── shared helpers ───────────────────────────────────────────────────────────
@@ -231,6 +282,60 @@ export function emptyFootageAnalysis(asset?: string): FootageAnalysis {
   return { version: FOOTAGE_ANALYSIS_VERSION, ...(asset ? { asset } : {}), shots: [] };
 }
 
+/** The provenance tag every AI-generated clip's sidecar carries. The
+ *  footage-analyst preserves it when it enriches the stub with real shots. It is
+ *  an ADVISORY hint, not a trust boundary — the sidecar is versioned + peer-
+ *  editable (DDR-054), so a peer can add/strip it; nothing security-relevant keys
+ *  off it (it only labels a beat in the UI). */
+export const AI_GENERATED_TAG = 'ai-generated';
+
+/**
+ * A provenance `FootageAnalysis` stub for an AI-GENERATED clip (DDR-164 Phase 3,
+ * Task 3.2). Written by the `/_api/generate-jobs` route the moment a video job
+ * lands, so the generated clip is immediately KNOWN to the footage pipeline —
+ * `getAnalysis` returns non-null and the reel director can see the clip is
+ * synthetic (the `ai-generated` tag) and read what it was generated FOR (the
+ * prompt, as the summary) without re-watching.
+ *
+ * It carries NO shots on purpose: the `footage-analyst` still probes the
+ * keyframes and fills the real shot list (so `isEmptyAnalysis` is true and the
+ * reel's shot-aware cache re-analyzes it), but the provenance tag + summary
+ * survive. Duration/dimensions are unknown until that probe, so they are omitted.
+ */
+export function generatedClipAnalysis(
+  asset: string,
+  provenance: { provider?: string; model?: string; prompt?: string } = {}
+): FootageAnalysis {
+  const tags = [AI_GENERATED_TAG];
+  if (provenance.provider) tags.push(provenance.provider);
+  const label = [provenance.provider, provenance.model].filter(Boolean).join(' · ');
+  // This is the LOCAL generation request's prompt (not a peer file), but its text
+  // becomes the sidecar `summary`, which IS versioned + peer-syncable (DDR-054)
+  // and read by the footage agents downstream. Map C0/C1 control chars to spaces
+  // (matching generation's sanitizeReuseText) so it can't fabricate multi-line
+  // instructions, then collapse whitespace; bounded to the summary cap below. The
+  // agents' own "treat sidecar text as data" posture is the mitigation for
+  // arbitrary PEER-authored summaries — this only cleans what we write.
+  const prompt = [...(provenance.prompt ?? '')]
+    .map((ch) => {
+      const c = ch.charCodeAt(0);
+      return (c >= 0x00 && c <= 0x1f) || (c >= 0x7f && c <= 0x9f) ? ' ' : ch;
+    })
+    .join('')
+    .replace(/\s+/g, ' ')
+    .trim();
+  let summary = `AI-generated clip${label ? ` (${label})` : ''}${prompt ? ` — ${prompt}` : ''}`;
+  if (summary.length > 3900) summary = `${summary.slice(0, 3900)}…`;
+  return {
+    version: FOOTAGE_ANALYSIS_VERSION,
+    asset,
+    keyframes: 0,
+    shots: [],
+    summary,
+    tags,
+  };
+}
+
 // ── structural validation (dependency-free — no Ajv) ─────────────────────────
 // Mirrors photo/schema.ts: the `/_api/footage` route validates untrusted JSON
 // with these before persisting. Unknown keys / wrong types / out-of-range
@@ -432,7 +537,7 @@ export function validateEdl(input: unknown): ValidationResult {
   assertKeys(
     errors,
     input,
-    ['version', 'title', 'fps', 'width', 'height', 'beats', 'music'],
+    ['version', 'title', 'fps', 'width', 'height', 'beats', 'music', 'audioTracks', 'captions'],
     'root'
   );
   if ('version' in input && input.version != null && typeof input.version !== 'number')
@@ -480,5 +585,78 @@ export function validateEdl(input: unknown): ValidationResult {
     }
   }
 
+  // feature-ai-media-generation Phase 2 — layered audio tracks (music/VO/SFX).
+  if ('audioTracks' in input && input.audioTracks != null) {
+    if (!Array.isArray(input.audioTracks)) errors.push('audioTracks: must be an array');
+    else if (input.audioTracks.length > 64) errors.push('audioTracks: more than 64 items');
+    else
+      input.audioTracks.forEach((raw, i) => {
+        const where = `audioTracks[${i}]`;
+        if (!isPlainObject(raw)) {
+          errors.push(`${where}: must be an object`);
+          return;
+        }
+        assertKeys(
+          errors,
+          raw,
+          [
+            'asset',
+            'kind',
+            'startFrame',
+            'durationFrames',
+            'gainDb',
+            'fadeInFrames',
+            'fadeOutFrames',
+            'name',
+          ],
+          where
+        );
+        assetRel(errors, raw, 'asset', where);
+        if (!('asset' in raw) || raw.asset == null) errors.push(`${where}.asset: required`);
+        if (
+          'kind' in raw &&
+          raw.kind != null &&
+          !['music', 'voiceover', 'sfx'].includes(raw.kind as string)
+        )
+          errors.push(`${where}.kind: must be music|voiceover|sfx`);
+        intGe(errors, raw, 'startFrame', 0, where);
+        intGe(errors, raw, 'durationFrames', 1, where);
+        num(errors, raw, 'gainDb', -60, 24, where);
+        intGe(errors, raw, 'fadeInFrames', 0, where);
+        intGe(errors, raw, 'fadeOutFrames', 0, where);
+        str(errors, raw, 'name', 120, where);
+      });
+  }
+
+  // feature-ai-media-generation Phase 2 — the caption/subtitle track.
+  if ('captions' in input && input.captions != null) {
+    const c = input.captions;
+    if (!isPlainObject(c)) errors.push('captions: must be an object');
+    else {
+      assertKeys(errors, c, ['cues', 'style'], 'captions');
+      if (
+        'style' in c &&
+        c.style != null &&
+        !['lower-third', 'centered', 'top'].includes(c.style as string)
+      )
+        errors.push('captions.style: must be lower-third|centered|top');
+      if (!('cues' in c) || !Array.isArray(c.cues)) errors.push('captions.cues: must be an array');
+      else if (c.cues.length > 5000) errors.push('captions.cues: more than 5000 items');
+      else
+        c.cues.forEach((raw, i) => {
+          const where = `captions.cues[${i}]`;
+          if (!isPlainObject(raw)) {
+            errors.push(`${where}: must be an object`);
+            return;
+          }
+          assertKeys(errors, raw, ['startSec', 'endSec', 'text'], where);
+          num(errors, raw, 'startSec', 0, 24 * 3600, where);
+          num(errors, raw, 'endSec', 0, 24 * 3600, where);
+          str(errors, raw, 'text', 2000, where);
+          if (!('text' in raw) || raw.text == null) errors.push(`${where}.text: required`);
+        });
+    }
+  }
+
   return { ok: errors.length === 0, errors };
 }