@1agh/maude 0.43.0 → 0.45.0

package/apps/studio/generation/captions.ts

@@ -0,0 +1,132 @@
+// generation/captions.ts — word-timestamp JSON → SRT / VTT, with a re-flow pass
+// (DDR-164, Phase 2). SHARED by every speech-to-text provider: ElevenLabs Scribe
+// (adapters/elevenlabs.ts) and local whisper.cpp (bin/_transcribe.mjs) both emit
+// word-level timings; this is the single place they become subtitle cues, so the
+// two never drift on line-length / timing rules.
+//
+// Dependency-free (mirrors photo/schema.ts, footage/schema.ts, generation/
+// types.ts). Pure functions over plain data — no node:*, no provider SDK.
+
+/** One word with second-based start/end timings (the STT-normalized shape). */
+export interface CaptionWord {
+  text: string;
+  /** Seconds from the start of the audio. */
+  start: number;
+  end: number;
+}
+
+/** One rendered subtitle cue (a line or two of grouped words). */
+export interface CaptionCue {
+  start: number;
+  end: number;
+  text: string;
+}
+
+export interface ReflowOptions {
+  /** Max characters per cue before a new cue starts (subtitle legibility). */
+  maxChars?: number;
+  /** Max seconds a single cue may span. */
+  maxDuration?: number;
+  /** A silence gap (seconds) between words that forces a cue break. */
+  gapBreak?: number;
+}
+
+const DEFAULT_REFLOW: Required<ReflowOptions> = {
+  maxChars: 42, // one comfortable subtitle line
+  maxDuration: 6,
+  gapBreak: 0.8,
+};
+
+/**
+ * Group word-level timings into readable cues — breaking on a max line length, a
+ * max duration, a silence gap, or sentence-ending punctuation. Deterministic and
+ * allocation-light so both STT paths produce identical output for identical input.
+ */
+export function reflowWords(words: CaptionWord[], opts: ReflowOptions = {}): CaptionCue[] {
+  const o = { ...DEFAULT_REFLOW, ...opts };
+  const cues: CaptionCue[] = [];
+  let cur: CaptionWord[] = [];
+
+  const flush = (): void => {
+    if (cur.length === 0) return;
+    cues.push({
+      start: cur[0].start,
+      end: cur[cur.length - 1].end,
+      text: cur
+        .map((w) => w.text)
+        .join(' ')
+        .replace(/\s+([,.!?;:])/g, '$1') // no space before punctuation
+        .trim(),
+    });
+    cur = [];
+  };
+
+  for (const w of words) {
+    if (!w || typeof w.text !== 'string') continue;
+    const text = w.text.trim();
+    if (!text) continue;
+    const start = Number.isFinite(w.start) ? w.start : cur.length ? cur[cur.length - 1].end : 0;
+    const end = Number.isFinite(w.end) ? w.end : start;
+    const word: CaptionWord = { text, start, end };
+
+    if (cur.length > 0) {
+      const prev = cur[cur.length - 1];
+      const projected = cur.map((x) => x.text).join(' ').length + 1 + text.length;
+      const gap = start - prev.end;
+      const span = end - cur[0].start;
+      if (projected > o.maxChars || span > o.maxDuration || gap >= o.gapBreak) {
+        flush();
+      }
+    }
+    cur.push(word);
+    // Break AFTER sentence-ending punctuation so a cue ends on a natural clause.
+    if (/[.!?]$/.test(text)) flush();
+  }
+  flush();
+  return cues;
+}
+
+/** SRT timestamp — `HH:MM:SS,mmm`. */
+function srtTime(sec: number): string {
+  return formatTime(sec, ',');
+}
+/** VTT timestamp — `HH:MM:SS.mmm`. */
+function vttTime(sec: number): string {
+  return formatTime(sec, '.');
+}
+function formatTime(sec: number, msSep: ',' | '.'): string {
+  const s = Math.max(0, sec);
+  const h = Math.floor(s / 3600);
+  const m = Math.floor((s % 3600) / 60);
+  const ss = Math.floor(s % 60);
+  const ms = Math.round((s - Math.floor(s)) * 1000);
+  const pad = (n: number, w = 2) => String(n).padStart(w, '0');
+  return `${pad(h)}:${pad(m)}:${pad(ss)}${msSep}${pad(ms, 3)}`;
+}
+
+/** Render cues as an SRT document. */
+export function cuesToSrt(cues: CaptionCue[]): string {
+  const body = cues
+    .map((c, i) => `${i + 1}\n${srtTime(c.start)} --> ${srtTime(c.end)}\n${c.text}`)
+    .join('\n\n');
+  return `${body}\n`;
+}
+
+/** Render cues as a WebVTT document. */
+export function cuesToVtt(cues: CaptionCue[]): string {
+  return (
+    'WEBVTT\n\n' +
+    cues.map((c) => `${vttTime(c.start)} --> ${vttTime(c.end)}\n${c.text}`).join('\n\n') +
+    '\n'
+  );
+}
+
+/** Word timings → SRT in one call (reflow + render). */
+export function wordsToSrt(words: CaptionWord[], opts?: ReflowOptions): string {
+  return cuesToSrt(reflowWords(words, opts));
+}
+
+/** Word timings → VTT in one call (reflow + render). */
+export function wordsToVtt(words: CaptionWord[], opts?: ReflowOptions): string {
+  return cuesToVtt(reflowWords(words, opts));
+}

package/apps/studio/generation/download.ts

@@ -0,0 +1,76 @@
+// generation/download.ts — localize a produced GenAsset into the content-
+// addressed `assets/<sha8>.<ext>` store (DDR-088), so nothing generated is ever
+// referenced by an expiring provider URL or an un-sniffed blob.
+//
+// EVERY produced artifact goes through here before it is referenced anywhere:
+//   • inline bytes / base64 (sync providers like Gemini) → api.saveAsset(bytes),
+//     which magic-byte-sniffs + category-caps + content-addresses in ONE tested
+//     path (the same route drag-drop/paste upload uses).
+//   • an expiring provider URL (fal/Veo, async) → downloaded through the
+//     SSRF-hardened egress path (_fetch-asset.mjs discipline) THEN saveAsset.
+//     That URL path is added in Phase 1 (fal); Phase 0 (Gemini base64) never
+//     hits it, so it fails loud rather than shipping an unhardened fetch.
+//
+// The route/job wires `AdapterContext.localize` to `localizeGenAsset(asset, …)`.
+
+import type { GenAsset } from './types.ts';
+
+/** Minimal seam onto api.saveAsset so this module stays dependency-light. */
+export interface LocalizeDeps {
+  /** api.saveAsset — magic-byte sniff + category cap + sha8 content-address. */
+  saveAsset(bytes: Uint8Array): Promise<{ ok: boolean; path?: string; error?: string }>;
+}
+
+/** Decode a base64 string into bytes (data: URL prefix tolerated). */
+function decodeBase64(data: string): Uint8Array {
+  const comma = data.indexOf(',');
+  const b64 = data.startsWith('data:') && comma !== -1 ? data.slice(comma + 1) : data;
+  // Bun/Node Buffer decodes base64 without a browser atob dependency.
+  return new Uint8Array(Buffer.from(b64, 'base64'));
+}
+
+/**
+ * Localize one produced asset → returns the `assets/<sha8>.<ext>` rel path.
+ * Throws on an empty/rejected asset (saveAsset enforces the sniff + cap, so a
+ * provider that returns an SVG/HTML/script blob is rejected there).
+ */
+export async function localizeGenAsset(asset: GenAsset, deps: LocalizeDeps): Promise<string> {
+  // A transcription result is caption TEXT (SRT/VTT), not media — it must never
+  // hit the magic-byte-sniffed media store (saveAsset would reject it anyway).
+  // Captions are consumed by the `maude design transcribe` verb (which writes the
+  // SRT/VTT directly) and, once wired, the EDL caption track — never localized as
+  // an `assets/<sha8>` media path here (Phase 2.2 / 2.3). Fail loud rather than
+  // silently drop it, so a premature transcription-through-the-media-queue is
+  // caught at the seam instead of producing a mislabelled asset.
+  if (asset.kind === 'transcription' || typeof asset.text === 'string') {
+    throw new Error(
+      'transcription/caption output is not localized to the media asset store — ' +
+        'it is handled by the transcribe verb / EDL caption track (Phase 2.2/2.3)'
+    );
+  }
+
+  let bytes: Uint8Array | null = null;
+
+  const maybeBase64 = (asset as unknown as { base64?: unknown }).base64;
+  if (asset.bytes && asset.bytes.byteLength > 0) {
+    bytes = asset.bytes;
+  } else if (typeof maybeBase64 === 'string') {
+    // Some adapters stash base64 under `.base64`; tolerate it.
+    bytes = decodeBase64(maybeBase64);
+  } else if (asset.url) {
+    throw new Error(
+      'URL-result localization (expiring provider URL) is added in Phase 1 (fal) — ' +
+        'the SSRF-hardened egress path is not wired yet; Phase 0 uses inline bytes only.'
+    );
+  }
+
+  if (!bytes || bytes.byteLength === 0) {
+    throw new Error('generation produced an empty asset');
+  }
+
+  const saved = await deps.saveAsset(bytes);
+  if (!saved.ok || !saved.path) {
+    throw new Error(`asset rejected on save: ${saved.error ?? 'unknown'}`);
+  }
+  return saved.path;
+}

package/apps/studio/generation/elevenlabs.test.ts

@@ -0,0 +1,223 @@
+// generation/elevenlabs.test.ts — the ElevenLabs adapter proven per-capability
+// against stubbed HTTP (no real key, no network). Audio verbs yield audio bytes;
+// Scribe yields an SRT built from word timings.
+
+import { afterEach, describe, expect, test } from 'bun:test';
+
+import { createElevenLabsAdapter, parseHistory } from './adapters/elevenlabs.ts';
+import { createAdapter, providersForModality } from './registry.ts';
+import type { AdapterContext, GenRequest } from './types.ts';
+
+const realFetch = globalThis.fetch;
+afterEach(() => {
+  globalThis.fetch = realFetch;
+});
+
+function ctxWith(overrides: Partial<AdapterContext> = {}): AdapterContext {
+  return {
+    apiKey: 'sk-eleven-test',
+    localize: async () => 'assets/stub.mp3',
+    ...overrides,
+  };
+}
+
+/** Capture the last request + return a canned response. */
+function stub(makeRes: () => Response): { calls: Array<{ url: string; init: RequestInit }> } {
+  const calls: Array<{ url: string; init: RequestInit }> = [];
+  globalThis.fetch = (async (url: string, init: RequestInit) => {
+    calls.push({ url: String(url), init });
+    return makeRes();
+  }) as typeof fetch;
+  return { calls };
+}
+
+const MP3 = new Uint8Array([0xff, 0xfb, 0x90, 0x00, 1, 2, 3, 4]); // MP3 frame-sync-ish
+function audioRes(): Response {
+  return new Response(MP3, { status: 200, headers: { 'content-type': 'audio/mpeg' } });
+}
+
+async function firstAsset(adapter: ReturnType<typeof createElevenLabsAdapter>, req: GenRequest) {
+  const job = await adapter.submit(req);
+  const res = await job.result();
+  return { job, res };
+}
+
+describe('elevenlabs audio', () => {
+  test('music POSTs /v1/music and yields audio bytes', async () => {
+    const { calls } = stub(audioRes);
+    const { job, res } = await firstAsset(createElevenLabsAdapter(ctxWith()), {
+      modality: 'audio',
+      provider: 'elevenlabs',
+      prompt: 'a warm lo-fi loop',
+    });
+    expect(job.status()).toBe('done');
+    expect(calls[0].url).toContain('/v1/music');
+    expect(res.assets[0].kind).toBe('audio');
+    expect(res.assets[0].bytes?.byteLength).toBe(MP3.byteLength);
+    // The key rides the header only, never the URL.
+    expect(String(calls[0].init.headers['xi-api-key'])).toBe('sk-eleven-test');
+    expect(calls[0].url).not.toContain('sk-eleven-test');
+  });
+
+  test('sfx POSTs /v1/sound-generation with a duration', async () => {
+    const { calls } = stub(audioRes);
+    await firstAsset(createElevenLabsAdapter(ctxWith()), {
+      modality: 'audio',
+      provider: 'elevenlabs',
+      prompt: 'glass shatter',
+      params: { audioKind: 'sfx', durationSeconds: 2 },
+    });
+    expect(calls[0].url).toContain('/v1/sound-generation');
+    expect(JSON.parse(String(calls[0].init.body)).duration_seconds).toBe(2);
+  });
+
+  test('tts POSTs /v1/text-to-speech/{voice} and requires a voice_id', async () => {
+    const { calls } = stub(audioRes);
+    await firstAsset(createElevenLabsAdapter(ctxWith()), {
+      modality: 'audio',
+      provider: 'elevenlabs',
+      prompt: 'Welcome to Maude',
+      params: { audioKind: 'tts', voice_id: 'Rachel' },
+    });
+    expect(calls[0].url).toContain('/v1/text-to-speech/Rachel');
+
+    const noVoice = createElevenLabsAdapter(ctxWith());
+    const job = await noVoice.submit({
+      modality: 'audio',
+      provider: 'elevenlabs',
+      prompt: 'x',
+      params: { audioKind: 'tts' },
+    });
+    await expect(job.result()).rejects.toThrow(/voice_id/);
+  });
+
+  test('a missing key fails fast without a network call', async () => {
+    let called = false;
+    globalThis.fetch = (async () => {
+      called = true;
+      return audioRes();
+    }) as typeof fetch;
+    const job = await createElevenLabsAdapter(ctxWith({ apiKey: null })).submit({
+      modality: 'audio',
+      provider: 'elevenlabs',
+      prompt: 'x',
+    });
+    await expect(job.result()).rejects.toThrow(/no ElevenLabs key/);
+    expect(called).toBe(false);
+  });
+
+  test('an HTTP error surfaces the provider detail, never the key', async () => {
+    stub(
+      () =>
+        new Response(JSON.stringify({ detail: { message: 'quota exceeded' } }), {
+          status: 429,
+          headers: { 'content-type': 'application/json' },
+        })
+    );
+    const job = await createElevenLabsAdapter(ctxWith()).submit({
+      modality: 'audio',
+      provider: 'elevenlabs',
+      prompt: 'x',
+    });
+    await expect(job.result()).rejects.toThrow(/quota exceeded/);
+  });
+});
+
+describe('elevenlabs scribe (transcription)', () => {
+  test('word timings become an SRT text asset', async () => {
+    stub(
+      () =>
+        new Response(
+          JSON.stringify({
+            text: 'Hello there.',
+            words: [
+              { text: 'Hello', start: 0, end: 0.4, type: 'word' },
+              { text: ' ', start: 0.4, end: 0.4, type: 'spacing' },
+              { text: 'there.', start: 0.4, end: 0.8, type: 'word' },
+            ],
+          }),
+          { status: 200, headers: { 'content-type': 'application/json' } }
+        )
+    );
+    const adapter = createElevenLabsAdapter(
+      ctxWith({
+        readSourceAsset: async () => ({ bytes: new Uint8Array([1, 2, 3]), mime: 'audio/mpeg' }),
+      })
+    );
+    const { res } = await firstAsset(adapter, {
+      modality: 'transcription',
+      provider: 'elevenlabs',
+      sourceAsset: 'assets/deadbeef.mp3',
+    });
+    const asset = res.assets[0];
+    expect(asset.kind).toBe('transcription');
+    expect(asset.text).toContain('00:00:00,000 --> 00:00:00,800');
+    expect(asset.text).toContain('Hello there.');
+    // The spacing token is dropped, not emitted as a word.
+    expect(asset.text).not.toContain('Hello  there');
+  });
+
+  test('transcription without a source asset fails', async () => {
+    const job = await createElevenLabsAdapter(ctxWith()).submit({
+      modality: 'transcription',
+      provider: 'elevenlabs',
+      sourceAsset: 'assets/deadbeef.mp3',
+    });
+    // no readSourceAsset wired → hard error
+    await expect(job.result()).rejects.toThrow(/source-asset access/);
+  });
+});
+
+describe('registry', () => {
+  test('elevenlabs is registered for audio + transcription', () => {
+    expect(providersForModality('audio').some((p) => p.id === 'elevenlabs')).toBe(true);
+    expect(providersForModality('transcription').some((p) => p.id === 'elevenlabs')).toBe(true);
+    const adapter = createAdapter('elevenlabs', ctxWith());
+    expect(adapter.descriptor.id).toBe('elevenlabs');
+  });
+});
+
+describe('elevenlabs history (Task 2.5 — reuse-before-you-pay)', () => {
+  test('parseHistory maps items, drops id-less, converts date_unix → ISO', () => {
+    const items = parseHistory({
+      history: [
+        { history_item_id: 'h1', text: 'warm lofi loop', voice_id: 'v1', date_unix: 1_700_000_000 },
+        { text: 'no id — dropped' },
+        { history_item_id: 'h2', text: 'sfx whoosh' },
+      ],
+    });
+    expect(items.map((i) => i.id)).toEqual(['h1', 'h2']);
+    expect(items[0].text).toBe('warm lofi loop');
+    expect(items[0].voiceId).toBe('v1');
+    expect(items[0].at).toBe(new Date(1_700_000_000 * 1000).toISOString());
+    expect(items[1].at).toBeUndefined();
+  });
+
+  test('parseHistory tolerates an empty / missing history', () => {
+    expect(parseHistory({})).toEqual([]);
+    expect(parseHistory({ history: [] })).toEqual([]);
+  });
+
+  test('listHistory GETs /v1/history with the key in the header only', async () => {
+    const { calls } = stub(
+      () =>
+        new Response(JSON.stringify({ history: [{ history_item_id: 'h1', text: 'x' }] }), {
+          status: 200,
+          headers: { 'content-type': 'application/json' },
+        })
+    );
+    const items = await createElevenLabsAdapter(ctxWith()).listHistory?.();
+    expect(items?.[0].id).toBe('h1');
+    expect(calls[0].url).toContain('/v1/history');
+    expect(String(calls[0].init.headers['xi-api-key'])).toBe('sk-eleven-test');
+    expect(calls[0].url).not.toContain('sk-eleven-test');
+  });
+
+  test('fetchHistoryAudio GETs the item audio and yields audio bytes', async () => {
+    const { calls } = stub(audioRes);
+    const asset = await createElevenLabsAdapter(ctxWith()).fetchHistoryAudio?.('h1');
+    expect(asset?.kind).toBe('audio');
+    expect(asset?.bytes?.byteLength).toBe(MP3.byteLength);
+    expect(calls[0].url).toContain('/v1/history/h1/audio');
+  });
+});