@aura3d/audio 1.0.6 → 1.1.0

package/src/AudioWaveform.ts

@@ -0,0 +1,226 @@
+import type { AudioClip } from "./AudioClip";
+
+export interface AudioWaveformOptions {
+  readonly peakCount?: number;
+  readonly samplesPerPeak?: number;
+  readonly channel?: number | "mix";
+  readonly normalize?: boolean;
+}
+
+export interface AudioWaveformPeak {
+  readonly min: number;
+  readonly max: number;
+  readonly rms: number;
+}
+
+export interface AudioWaveformData {
+  readonly duration: number;
+  readonly sampleRate: number;
+  readonly channels: number;
+  readonly peakCount: number;
+  readonly samplesPerPeak: number;
+  readonly normalized: boolean;
+  readonly peaks: readonly AudioWaveformPeak[];
+}
+
+export interface AudioWaveformPathPoint {
+  readonly x: number;
+  readonly yMin: number;
+  readonly yMax: number;
+  readonly rmsHeight: number;
+}
+
+export interface AudioWaveformPathOptions {
+  readonly width: number;
+  readonly height: number;
+  readonly padding?: number;
+}
+
+export interface AudioWaveformReviewStem {
+  readonly id: string;
+  readonly label?: string;
+  readonly startTime: number;
+  readonly waveform: AudioWaveformData;
+}
+
+export interface AudioWaveformReviewStemView {
+  readonly id: string;
+  readonly label: string;
+  readonly startTime: number;
+  readonly duration: number;
+  readonly peakCount: number;
+  readonly path: readonly AudioWaveformPathPoint[];
+}
+
+export interface AudioWaveformReviewData {
+  readonly kind: "audio-waveform-review-data";
+  readonly stemCount: number;
+  readonly width: number;
+  readonly height: number;
+  readonly stems: readonly AudioWaveformReviewStemView[];
+}
+
+export type AudioWaveformInput = AudioBuffer | AudioClip;
+
+export function createAudioWaveform(input: AudioWaveformInput, options: AudioWaveformOptions = {}): AudioWaveformData {
+  const buffer = audioBufferFromInput(input);
+  const channels = buffer.numberOfChannels;
+  const sampleRate = finitePositive(buffer.sampleRate, "sampleRate");
+  const duration = Math.max(0, buffer.duration);
+  const totalSamples = Math.max(0, Math.round(duration * sampleRate));
+  const samplesPerPeak = resolveSamplesPerPeak(totalSamples, options);
+  const peakCount = totalSamples === 0 ? 0 : Math.ceil(totalSamples / samplesPerPeak);
+  const peaks: AudioWaveformPeak[] = [];
+  const sourceChannels = collectWaveformChannels(buffer, options.channel ?? "mix");
+
+  for (let peakIndex = 0; peakIndex < peakCount; peakIndex++) {
+    const start = peakIndex * samplesPerPeak;
+    const end = Math.min(totalSamples, start + samplesPerPeak);
+    let min = 0;
+    let max = 0;
+    let sumSquares = 0;
+    let sampleCount = 0;
+
+    for (let sampleIndex = start; sampleIndex < end; sampleIndex++) {
+      let value = 0;
+      for (const channel of sourceChannels) {
+        value += channel[sampleIndex] ?? 0;
+      }
+      value /= sourceChannels.length;
+      min = Math.min(min, value);
+      max = Math.max(max, value);
+      sumSquares += value * value;
+      sampleCount++;
+    }
+
+    peaks.push({
+      min,
+      max,
+      rms: sampleCount === 0 ? 0 : Math.sqrt(sumSquares / sampleCount)
+    });
+  }
+
+  const normalize = options.normalize ?? false;
+  const outputPeaks = normalize ? normalizePeaks(peaks) : peaks;
+  return {
+    duration,
+    sampleRate,
+    channels,
+    peakCount: outputPeaks.length,
+    samplesPerPeak,
+    normalized: normalize,
+    peaks: outputPeaks
+  };
+}
+
+export function sampleAudioWaveformAtTime(waveform: AudioWaveformData, timeSeconds: number): AudioWaveformPeak {
+  if (waveform.peaks.length === 0 || waveform.duration <= 0) {
+    return { min: 0, max: 0, rms: 0 };
+  }
+  const normalizedTime = clamp(timeSeconds / waveform.duration, 0, 1);
+  const index = Math.min(waveform.peaks.length - 1, Math.floor(normalizedTime * waveform.peaks.length));
+  return waveform.peaks[index] ?? { min: 0, max: 0, rms: 0 };
+}
+
+export function createAudioWaveformPath(waveform: AudioWaveformData, options: AudioWaveformPathOptions): readonly AudioWaveformPathPoint[] {
+  const width = finitePositive(options.width, "width");
+  const height = finitePositive(options.height, "height");
+  const padding = Math.max(0, options.padding ?? 0);
+  const drawableHeight = Math.max(1, height - padding * 2);
+  const centerY = padding + drawableHeight / 2;
+  const halfHeight = drawableHeight / 2;
+  const denominator = Math.max(1, waveform.peaks.length - 1);
+
+  return waveform.peaks.map((peak, index) => ({
+    x: waveform.peaks.length === 1 ? width / 2 : index / denominator * width,
+    yMin: centerY - clamp(peak.max, -1, 1) * halfHeight,
+    yMax: centerY - clamp(peak.min, -1, 1) * halfHeight,
+    rmsHeight: clamp(peak.rms, 0, 1) * halfHeight
+  }));
+}
+
+export function createAudioWaveformReviewData(
+  stems: readonly AudioWaveformReviewStem[],
+  options: AudioWaveformPathOptions
+): AudioWaveformReviewData {
+  const width = finitePositive(options.width, "width");
+  const height = finitePositive(options.height, "height");
+  return {
+    kind: "audio-waveform-review-data",
+    stemCount: stems.length,
+    width,
+    height,
+    stems: stems.map((stem) => ({
+      id: stem.id,
+      label: stem.label ?? stem.id,
+      startTime: Math.max(0, stem.startTime),
+      duration: stem.waveform.duration,
+      peakCount: stem.waveform.peakCount,
+      path: createAudioWaveformPath(stem.waveform, { ...options, width, height })
+    }))
+  };
+}
+
+export function audioWaveformPeakRange(waveform: AudioWaveformData): { readonly min: number; readonly max: number; readonly rmsMax: number } {
+  let min = 0;
+  let max = 0;
+  let rmsMax = 0;
+  for (const peak of waveform.peaks) {
+    min = Math.min(min, peak.min);
+    max = Math.max(max, peak.max);
+    rmsMax = Math.max(rmsMax, peak.rms);
+  }
+  return { min, max, rmsMax };
+}
+
+function audioBufferFromInput(input: AudioWaveformInput): AudioBuffer {
+  return "buffer" in input ? input.buffer : input;
+}
+
+function resolveSamplesPerPeak(totalSamples: number, options: AudioWaveformOptions): number {
+  if (options.samplesPerPeak !== undefined) {
+    return Math.max(1, Math.floor(finitePositive(options.samplesPerPeak, "samplesPerPeak")));
+  }
+  if (options.peakCount !== undefined) {
+    const peakCount = Math.max(1, Math.floor(finitePositive(options.peakCount, "peakCount")));
+    return Math.max(1, Math.ceil(totalSamples / peakCount));
+  }
+  return Math.max(1, Math.ceil(totalSamples / 1024));
+}
+
+function collectWaveformChannels(buffer: AudioBuffer, channel: number | "mix"): readonly Float32Array[] {
+  if (buffer.numberOfChannels <= 0) {
+    throw new Error("Audio waveform requires at least one channel");
+  }
+  if (channel === "mix") {
+    return Array.from({ length: buffer.numberOfChannels }, (_, index) => buffer.getChannelData(index));
+  }
+  if (!Number.isInteger(channel) || channel < 0 || channel >= buffer.numberOfChannels) {
+    throw new Error(`Audio waveform channel ${channel} is outside the buffer channel range`);
+  }
+  return [buffer.getChannelData(channel)];
+}
+
+function normalizePeaks(peaks: readonly AudioWaveformPeak[]): readonly AudioWaveformPeak[] {
+  let maxAbs = 0;
+  for (const peak of peaks) {
+    maxAbs = Math.max(maxAbs, Math.abs(peak.min), Math.abs(peak.max), peak.rms);
+  }
+  if (maxAbs <= 0) return peaks;
+  return peaks.map((peak) => ({
+    min: peak.min / maxAbs,
+    max: peak.max / maxAbs,
+    rms: peak.rms / maxAbs
+  }));
+}
+
+function finitePositive(value: number, label: string): number {
+  if (!Number.isFinite(value) || value <= 0) {
+    throw new Error(`Audio waveform ${label} must be a positive finite number`);
+  }
+  return value;
+}
+
+function clamp(value: number, min: number, max: number): number {
+  return Math.min(max, Math.max(min, value));
+}

package/src/index.ts

@@ -1,6 +1,46 @@
 export { AudioBus } from "./AudioBus";
 export { AudioClip } from "./AudioClip";
 export type { AudioClipOptions } from "./AudioClip";
+export { AudioFileManager, validateEpisodeAudioAssets } from "./AudioFileManager";
+export type {
+  AudioDecodeContextLike,
+  EpisodeAudioAssetDiagnostic,
+  EpisodeAudioAssetReadiness,
+  EpisodeAudioAssetRequirement,
+  AudioFileAssetLike,
+  AudioFileFetchResponseLike,
+  AudioFileInput,
+  AudioFileManagerOptions,
+  AudioFileRequest,
+  ResolvedAudioFileRequest
+} from "./AudioFileManager";
+export { createAudioTimelineMixSnapshot, defaultAudioTimelineBusForRole, validateAudioCaptionSync, AudioTimelineTrack } from "./AudioTimelineTrack";
+export type {
+  AudioCaptionCue,
+  AudioCaptionSyncIssue,
+  AudioCaptionSyncReport,
+  AudioTimelineBusMix,
+  AudioTimelineClip,
+  AudioTimelineClipOptions,
+  AudioTimelineEnvelopePoint,
+  AudioTimelineMixOptions,
+  AudioTimelineMixSnapshot,
+  AudioTimelineSample,
+  AudioTimelineTrackOptions,
+  AudioTimelineTrackRole
+} from "./AudioTimelineTrack";
+export { audioWaveformPeakRange, createAudioWaveform, createAudioWaveformPath, createAudioWaveformReviewData, sampleAudioWaveformAtTime } from "./AudioWaveform";
+export type {
+  AudioWaveformData,
+  AudioWaveformInput,
+  AudioWaveformOptions,
+  AudioWaveformPathOptions,
+  AudioWaveformPathPoint,
+  AudioWaveformPeak,
+  AudioWaveformReviewData,
+  AudioWaveformReviewStem,
+  AudioWaveformReviewStemView
+} from "./AudioWaveform";
 export { sampleAudioEffectsAnalysisFixture } from "./AudioEffectsAnalysisFixtures";
 export type {
   AudioChorusPreset,
@@ -21,7 +61,8 @@ export type { AudioContextLike, AudioContextManagerOptions, AudioContextState }
 export type { AudioEffect } from "./AudioEffect";
 export { AudioListener } from "./AudioListener";
 export type { Vec3Like } from "./AudioListener";
-export { AudioMixer } from "./AudioMixer";
+export { AudioMixer, createAudioMixerEvidence, createCartoonAudioMixer } from "./AudioMixer";
+export type { AudioMixerBusEvidence, AudioMixerEvidence, CartoonAudioMixer, CartoonAudioMixerOptions } from "./AudioMixer";
 export { AudioSource } from "./AudioSource";
 export type { AudioSourceOptions, AudioSourceState } from "./AudioSource";
 export { AudioSystem } from "./AudioSystem";

package/tests/audio.test.ts

@@ -1,7 +1,23 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 import { Scene } from "@aura3d/scene";
-import { AudioClip, AudioContextManager, AudioListener, AudioSource, SceneAudioBridge } from "../src/index";
+import {
+  AudioClip,
+  AudioContextManager,
+  AudioFileManager,
+  AudioListener,
+  AudioSource,
+  AudioTimelineTrack,
+  SceneAudioBridge,
+  createAudioTimelineMixSnapshot,
+  createAudioWaveformReviewData,
+  createCartoonAudioMixer,
+  createAudioWaveform,
+  createAudioWaveformPath,
+  sampleAudioWaveformAtTime,
+  validateAudioCaptionSync,
+  validateEpisodeAudioAssets
+} from "../src/index";
 
 class MockParam {
   value = 0;
@@ -36,6 +52,19 @@ class MockSource extends MockNode {
   }
 }
 
+function createMockAudioBuffer(channels: readonly Float32Array[], sampleRate: number): AudioBuffer {
+  return {
+    duration: channels.length === 0 ? 0 : (channels[0]?.length ?? 0) / sampleRate,
+    numberOfChannels: channels.length,
+    sampleRate,
+    getChannelData(index: number) {
+      const channel = channels[index];
+      if (!channel) throw new Error(`Unknown channel: ${index}`);
+      return channel;
+    }
+  } as AudioBuffer;
+}
+
 test("AudioContextManager unlocks, suspends, resumes, and disposes a mock context", async () => {
   const context = {
     state: "suspended",
@@ -119,3 +148,206 @@ test("SceneAudioBridge syncs listener and spatial source positions from scene no
   assert.deepEqual(listener.position, { x: 1, y: 2, z: 3 });
   assert.deepEqual(spatial.position, { x: 4, y: 5, z: 6 });
 });
+
+test("AudioFileManager loads typed audio assets once and caches decoded clips", async () => {
+  const decodedBuffer = createMockAudioBuffer([new Float32Array([0, 0.5, -0.5, 0])], 4);
+  let fetchCount = 0;
+  let decodeCount = 0;
+  const context = {
+    state: "running",
+    destination: new MockNode() as unknown as AudioNode,
+    currentTime: 0,
+    resume: async () => {},
+    suspend: async () => {},
+    close: async () => {},
+    createGain: () => new MockGain() as unknown as GainNode,
+    createBufferSource: () => new MockSource() as unknown as AudioBufferSourceNode,
+    createPanner: () => new MockNode() as unknown as PannerNode,
+    createBiquadFilter: () => new MockNode() as unknown as BiquadFilterNode,
+    createConvolver: () => new MockNode() as unknown as ConvolverNode,
+    decodeAudioData: async () => {
+      decodeCount++;
+      return decodedBuffer;
+    }
+  };
+  const manager = new AudioFileManager({
+    context,
+    fetch: async (url) => {
+      fetchCount++;
+      assert.equal(url, "/aura-assets/dialogue.wav");
+      return {
+        ok: true,
+        status: 200,
+        arrayBuffer: async () => new Uint8Array([1, 2, 3, 4]).buffer
+      };
+    }
+  });
+  const asset = {
+    kind: "aura-asset-ref",
+    id: "line1",
+    type: "audio",
+    format: "wav",
+    url: "/aura-assets/dialogue.wav"
+  };
+
+  const [first, second] = await Promise.all([manager.load(asset), manager.load(asset)]);
+
+  assert.equal(first, second);
+  assert.equal(first.name, "line1");
+  assert.equal(first.duration, 1);
+  assert.equal(fetchCount, 1);
+  assert.equal(decodeCount, 1);
+  assert.equal(manager.getCached(asset), first);
+  assert.throws(() => manager.resolve({ ...asset, type: "model" }), /not an audio asset/);
+});
+
+test("AudioWaveform computes deterministic peak data and canvas path points", () => {
+  const buffer = createMockAudioBuffer([new Float32Array([-1, 1, 0.5, -0.5])], 4);
+  const waveform = createAudioWaveform(buffer, { samplesPerPeak: 2 });
+
+  assert.equal(waveform.duration, 1);
+  assert.equal(waveform.peakCount, 2);
+  assert.deepEqual(waveform.peaks[0], { min: -1, max: 1, rms: 1 });
+  assert.deepEqual(waveform.peaks[1], { min: -0.5, max: 0.5, rms: 0.5 });
+  assert.deepEqual(sampleAudioWaveformAtTime(waveform, 0.75), waveform.peaks[1]);
+
+  const path = createAudioWaveformPath(waveform, { width: 100, height: 40 });
+  assert.equal(path.length, 2);
+  assert.equal(path[0]?.x, 0);
+  assert.equal(path[1]?.x, 100);
+  assert.equal(path[0]?.yMin, 0);
+  assert.equal(path[0]?.yMax, 40);
+});
+
+test("AudioTimelineTrack edits clips, routes buses, and ducks music under dialogue", () => {
+  const dialogue = new AudioTimelineTrack({ id: "dialogue", role: "dialogue" });
+  const music = new AudioTimelineTrack({ id: "music", role: "music", volume: 0.8 });
+  const sfx = new AudioTimelineTrack({ id: "sfx", role: "sfx" });
+
+  dialogue.addClip({
+    id: "line1",
+    startTime: 1,
+    duration: 2,
+    envelope: [
+      { time: 0, value: 0 },
+      { time: 0.5, value: 1 },
+      { time: 2, value: 1 }
+    ]
+  });
+  music.addClip({ id: "bed", startTime: 0, duration: 5 });
+  sfx.addClip({ id: "chime", startTime: 1.25, duration: 0.5, volume: 0.5 });
+
+  assert.equal(dialogue.busName, "voice");
+  assert.equal(sfx.busName, "sfx");
+  assert.equal(dialogue.sampleAt(1.25)[0]?.volume, 0.5);
+
+  const [left, right] = dialogue.splitClip("line1", 2, "line1b");
+  assert.equal(left.duration, 1);
+  assert.equal(right.startTime, 2);
+  assert.equal(right.trimStart, 1);
+  assert.equal(dialogue.activeClipsAt(2.25)[0]?.id, "line1b");
+
+  const moved = sfx.moveClip("chime", 1.5);
+  assert.equal(moved.startTime, 1.5);
+  const trimmed = music.trimClip("bed", 1, 4);
+  assert.equal(trimmed.duration, 3);
+
+  const snapshot = createAudioTimelineMixSnapshot([dialogue, music, sfx], 2.25, {
+    duckMusicDuringDialogue: true,
+    duckingRatio: 0.25,
+    masterVolume: 0.9
+  });
+  const musicBus = snapshot.buses.find((bus) => bus.busName === "music");
+  const voiceBus = snapshot.buses.find((bus) => bus.busName === "voice");
+
+  assert.equal(snapshot.dialogueActive, true);
+  assert.equal(snapshot.duckingApplied, true);
+  assert.equal(musicBus?.ducked, true);
+  assert.equal(musicBus?.volume, 0.2);
+  assert.equal(voiceBus?.volume, 1);
+  assert.deepEqual(snapshot.activeSamples.map((sample) => sample.clipId).sort(), ["bed", "line1b"]);
+});
+
+test("cartoon episode audio readiness reports missing typed audio assets", () => {
+  const readiness = validateEpisodeAudioAssets([
+    {
+      kind: "aura-asset-ref",
+      id: "moonGardenMusic",
+      type: "audio",
+      format: "ogg",
+      url: "/aura-assets/moon-garden-music.ogg",
+      license: "CC0"
+    },
+    {
+      kind: "aura-asset-ref",
+      id: "mikoLine1",
+      type: "model",
+      format: "glb",
+      url: "/aura-assets/miko.glb",
+      license: "CC0"
+    }
+  ], [
+    { id: "mikoLine1", role: "dialogue", requireLicense: true },
+    { id: "lumaLine1", role: "dialogue", requireLicense: true },
+    { id: "moonGardenMusic", role: "music", requireLicense: true }
+  ]);
+
+  assert.equal(readiness.ok, false);
+  assert.equal(readiness.requiredCount, 3);
+  assert.equal(readiness.readyCount, 1);
+  assert.deepEqual(readiness.missingAssetIds, ["lumaLine1"]);
+  assert.deepEqual(readiness.diagnostics.map((issue) => issue.code), [
+    "audio-asset-wrong-type",
+    "audio-asset-missing"
+  ]);
+});
+
+test("cartoon dialogue audio stems stay aligned with captions within one frame", () => {
+  const dialogue = new AudioTimelineTrack({ id: "dialogue", role: "dialogue" });
+  dialogue.addClip({ id: "miko-line-1", startTime: 1, duration: 1.5 });
+  dialogue.addClip({ id: "luma-line-1", startTime: 3, duration: 1 });
+
+  const synced = validateAudioCaptionSync([dialogue], [
+    { id: "caption-1", audioClipId: "miko-line-1", startTime: 1 + 1 / 60, endTime: 2.5 },
+    { id: "caption-2", audioClipId: "luma-line-1", startTime: 3, endTime: 4 }
+  ], { frameRate: 30, toleranceFrames: 1 });
+  const broken = validateAudioCaptionSync([dialogue], [
+    { id: "caption-3", audioClipId: "miko-line-1", startTime: 1.25, endTime: 2.5 }
+  ], { frameRate: 30, toleranceFrames: 1 });
+
+  assert.equal(synced.ok, true);
+  assert.equal(synced.checkedCueCount, 2);
+  assert.equal(broken.ok, false);
+  assert.equal(broken.issues[0]?.code, "caption-audio-start-out-of-sync");
+});
+
+test("cartoon audio waveform review data and mixer defaults expose route evidence", () => {
+  const waveform = createAudioWaveform(createMockAudioBuffer([new Float32Array([-1, 1, 0.5, -0.5])], 4), {
+    samplesPerPeak: 2
+  });
+  const reviewData = createAudioWaveformReviewData([
+    { id: "miko-line-1", label: "Miko line 1", startTime: 1, waveform }
+  ], { width: 120, height: 48 });
+  const context = {
+    state: "running",
+    destination: new MockNode() as unknown as AudioNode,
+    currentTime: 0,
+    resume: async () => {},
+    suspend: async () => {},
+    close: async () => {},
+    createGain: () => new MockGain() as unknown as GainNode,
+    createBufferSource: () => new MockSource() as unknown as AudioBufferSourceNode,
+    createPanner: () => new MockNode() as unknown as PannerNode,
+    createBiquadFilter: () => new MockNode() as unknown as BiquadFilterNode,
+    createConvolver: () => new MockNode() as unknown as ConvolverNode
+  };
+  const cartoonMixer = createCartoonAudioMixer(context, { musicVolume: 0.4 });
+  const evidence = cartoonMixer.evidence({ unlocked: true });
+
+  assert.equal(reviewData.kind, "audio-waveform-review-data");
+  assert.equal(reviewData.stemCount, 1);
+  assert.equal(reviewData.stems[0]?.path.length, 2);
+  assert.deepEqual(evidence.buses.map((bus) => bus.name).sort(), ["ambient", "master", "music", "sfx", "voice"]);
+  assert.equal(evidence.buses.find((bus) => bus.name === "music")?.volume, 0.4);
+  assert.equal(evidence.unlocked, true);
+});