@hraness/slopcamera 3.2.7 → 3.3.1

package/apps/desktop/html-overlay/audio-reactivity.ts

@@ -0,0 +1,127 @@
+import { z } from "zod";
+
+import type { AudioReactivityBands } from "../core/music-analysis";
+
+export const HTML_AUDIO_REACTIVITY_RESOURCE_NAME = "audio-reactivity";
+export const HTML_AUDIO_REACTIVITY_RESOURCE_URL_PATH = "slopcamera/audio-reactivity.json";
+export const HTML_AUDIO_REACTIVITY_MAX_BYTES = 4 * 1024 * 1024;
+
+export type HtmlOverlayAudioReactivity = Readonly<{
+  kind: "slopcamera.audio-reactivity";
+  schemaVersion: 1;
+  profile: "bands-v1";
+  source: Readonly<{ bytes: number; sha256: string }>;
+  pcm: Readonly<{ sampleCount: number; sha256: string }>;
+  durationUs: number;
+  rateHz: 60;
+  bands: AudioReactivityBands;
+}>;
+
+export type HtmlOverlayAudioSample = Readonly<{
+  bass: number; mid: number; treble: number; energy: number;
+}>;
+export type PreparedHtmlOverlayAudioReactivity = Readonly<{
+  durationUs: number;
+  sample: (timeUs: number) => HtmlOverlayAudioSample;
+}>;
+
+const LIMITS = Object.freeze({ maxSamples: 14_400_000, maxSourceBytes: 512 * 1024 * 1024,
+  sampleRateHz: 24_000, hopSize: 400, rateHz: 60, maxTimeUs: 3_600_000_000 });
+
+/** One implementation validates the SDK and browser resource, before frame sampling. */
+function createAudioReactivityRuntime(limits: typeof LIMITS) {
+  const object = (value: unknown, keys: readonly string[]): Record<string, unknown> => {
+    if (value === null || typeof value !== "object" || Array.isArray(value)) {
+      throw new TypeError("Audio reactivity requires a strict JSON object.");
+    }
+    const descriptors = Object.getOwnPropertyDescriptors(value);
+    if (Reflect.ownKeys(value).length !== keys.length
+      || keys.some(key => descriptors[key] === undefined || !("value" in descriptors[key]!))) {
+      throw new TypeError("Audio reactivity contains missing, unknown, or accessor fields.");
+    }
+    return value as Record<string, unknown>;
+  };
+  const integer = (value: unknown, minimum: number, maximum: number): number => {
+    if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum || value > maximum) {
+      throw new RangeError("Audio reactivity integer is outside its bound.");
+    }
+    return value;
+  };
+  const digest = (value: unknown): string => {
+    if (typeof value !== "string" || !/^[a-f0-9]{64}$/u.test(value)) {
+      throw new TypeError("Audio reactivity requires a SHA-256 digest.");
+    }
+    return value;
+  };
+  const parse = (value: unknown): HtmlOverlayAudioReactivity => {
+    const input = object(value, ["kind", "schemaVersion", "profile", "source", "pcm", "durationUs", "rateHz", "bands"]);
+    if (input.kind !== "slopcamera.audio-reactivity" || input.schemaVersion !== 1
+      || input.profile !== "bands-v1" || input.rateHz !== limits.rateHz) {
+      throw new TypeError("Unsupported audio reactivity profile or version.");
+    }
+    const source = object(input.source, ["bytes", "sha256"]);
+    const pcm = object(input.pcm, ["sampleCount", "sha256"]);
+    const sampleCount = integer(pcm.sampleCount, 1, limits.maxSamples);
+    const durationUs = Math.round(sampleCount * 1_000_000 / limits.sampleRateHz);
+    if (input.durationUs !== durationUs) throw new RangeError("Audio reactivity duration differs from its PCM sample count.");
+    const length = Math.ceil(sampleCount / limits.hopSize) + 1;
+    const bands = object(input.bands, ["bass", "mid", "treble", "energy"]);
+    const channel = (value: unknown): readonly number[] => {
+      const keys = value !== null && typeof value === "object" ? Reflect.ownKeys(value) : [];
+      if (!Array.isArray(value) || value.length !== length || keys.length !== length + 1
+        || keys.some(key => key !== "length" && (typeof key !== "string" || !/^(?:0|[1-9][0-9]*)$/u.test(key)))) {
+        throw new RangeError("Audio reactivity bands must be dense arrays covering the declared duration.");
+      }
+      const copy: number[] = [];
+      for (let index = 0; index < length; index++) {
+        const descriptor = Object.getOwnPropertyDescriptor(value, String(index));
+        const sample: unknown = descriptor !== undefined && "value" in descriptor ? descriptor.value : undefined;
+        if (typeof sample !== "number" || !Number.isFinite(sample) || sample < 0 || sample > 1) {
+          throw new RangeError("Audio reactivity band values must be finite numbers in [0, 1].");
+        }
+        copy.push(sample === 0 ? 0 : sample);
+      }
+      if (copy.at(-1) !== 0) throw new RangeError("Audio reactivity requires a zero endpoint at the exact duration.");
+      return Object.freeze(copy);
+    };
+    return Object.freeze({
+      kind: "slopcamera.audio-reactivity", schemaVersion: 1, profile: "bands-v1", rateHz: 60, durationUs,
+      source: Object.freeze({ bytes: integer(source.bytes, 1, limits.maxSourceBytes), sha256: digest(source.sha256) }),
+      pcm: Object.freeze({ sampleCount, sha256: digest(pcm.sha256) }),
+      bands: Object.freeze({ bass: channel(bands.bass), mid: channel(bands.mid), treble: channel(bands.treble), energy: channel(bands.energy) }),
+    });
+  };
+  const prepareAudioReactivity = (input: unknown): PreparedHtmlOverlayAudioReactivity => {
+    const data = parse(input);
+    const zero = Object.freeze({ bass: 0, mid: 0, treble: 0, energy: 0 });
+    return Object.freeze({ durationUs: data.durationUs, sample(timeUs: number): HtmlOverlayAudioSample {
+      integer(timeUs, -limits.maxTimeUs, limits.maxTimeUs);
+      if (timeUs < 0 || timeUs >= data.durationUs) return zero;
+      const index = Math.min(data.bands.bass.length - 2, Math.floor(timeUs * limits.rateHz / 1_000_000));
+      const startUs = index * 1_000_000 / limits.rateHz;
+      const endUs = Math.min(data.durationUs, (index + 1) * 1_000_000 / limits.rateHz);
+      const mix = Math.max(0, Math.min(1, (timeUs - startUs) / (endUs - startUs)));
+      const interpolate = (band: readonly number[]) => band[index]! * (1 - mix) + band[index + 1]! * mix;
+      return Object.freeze({ bass: interpolate(data.bands.bass), mid: interpolate(data.bands.mid),
+        treble: interpolate(data.bands.treble), energy: interpolate(data.bands.energy) });
+    } });
+  };
+  return { parse, prepareAudioReactivity };
+}
+
+const runtime = createAudioReactivityRuntime(LIMITS);
+export const HtmlOverlayAudioReactivitySchema = z.unknown().transform((input, context): HtmlOverlayAudioReactivity => {
+  try { return runtime.parse(input); } catch (error) {
+    context.addIssue({ code: "custom", message: error instanceof Error ? error.message : "Invalid audio reactivity." });
+    return z.NEVER;
+  }
+});
+
+/** Validate and copy once. Samples depend only on absolute source time, in integer microseconds. */
+export function prepareHtmlOverlayAudioReactivity(input: unknown): PreparedHtmlOverlayAudioReactivity {
+  return runtime.prepareAudioReactivity(input);
+}
+
+export function createHtmlOverlayAudioReactivityRuntimeSource(): string {
+  return `(() => { const runtime = (${createAudioReactivityRuntime.toString()})(${JSON.stringify(LIMITS)}); return { prepareAudioReactivity: runtime.prepareAudioReactivity }; })()`;
+}

package/apps/desktop/html-overlay/index.ts

@@ -33,6 +33,15 @@ export {
   type HtmlOverlayTiming,
 } from "./contracts";
 export { HtmlSceneInputSchema, type HtmlSceneInput } from "./scene";
+export {
+  HTML_AUDIO_REACTIVITY_RESOURCE_NAME,
+  HTML_AUDIO_REACTIVITY_RESOURCE_URL_PATH,
+  HtmlOverlayAudioReactivitySchema,
+  prepareHtmlOverlayAudioReactivity,
+  type HtmlOverlayAudioReactivity,
+  type HtmlOverlayAudioSample,
+  type PreparedHtmlOverlayAudioReactivity,
+} from "./audio-reactivity";
 export {
   HTML_OVERLAY_SCAFFOLD_CLOCK_INTEGRATIONS,
   HTML_OVERLAY_SCAFFOLD_KINDS,

package/apps/desktop/html-overlay/runtime.ts

@@ -11,6 +11,7 @@ import {
 import { htmlOverlayAssetLocalUrl } from "./libraries";
 import { HTML_OVERLAY_RANDOM_ALGORITHM } from "./random";
 import { createHtmlOverlayMusicClockRuntimeSource } from "./music-clock";
+import { createHtmlOverlayAudioReactivityRuntimeSource } from "./audio-reactivity";
 
 const Uint32Schema = z.number().int().safe().min(0).max(0xffff_ffff);
 
@@ -713,6 +714,7 @@ export function createHtmlOverlayBrowserRuntimeSource(
       };
     },
     ...${createHtmlOverlayMusicClockRuntimeSource()},
+    ...${createHtmlOverlayAudioReactivityRuntimeSource()},
     parameters: deepFreeze(config.parameters),
     random,
     randomFor,

package/apps/desktop/html-overlay/scene.ts

@@ -6,6 +6,8 @@ import {
 } from "./contracts";
 import { HtmlOverlayExecutionProfileSchema } from "./execution-profile";
 import { HtmlOverlayLibrarySelectionSchema } from "./libraries";
+import { AUDIO_REACTIVITY_MAX_DURATION_US } from "../core/music-analysis";
+import { HTML_AUDIO_REACTIVITY_RESOURCE_NAME, HTML_AUDIO_REACTIVITY_RESOURCE_URL_PATH } from "./audio-reactivity";
 
 const ResourceSchema = HtmlOverlayDeclaredResourceSchema.omit({ bytes: true, sha256: true }).extend({
   path: RepositoryRelativePathSchema,
@@ -27,8 +29,35 @@ export const HtmlSceneInputSchema = z.strictObject({
   executionProfile: HtmlOverlayExecutionProfileSchema.optional(),
   audio: z.strictObject({
     path: z.string().min(1).max(4096).refine(path => !/[\0\r\n]/u.test(path), "Audio path contains a control character."),
+    reactivity: z.strictObject({
+      profile: z.literal("bands-v1"),
+      resource: z.literal(HTML_AUDIO_REACTIVITY_RESOURCE_NAME).optional(),
+      sha256: z.string().regex(/^[a-f0-9]{64}$/u).optional(),
+    }).optional(),
   }).optional(),
 }).superRefine((input, context) => {
+  if (input.audio?.reactivity !== undefined && input.timing.durationUs > AUDIO_REACTIVITY_MAX_DURATION_US) {
+    context.addIssue({ code: "custom", message: "Audio reactivity is limited to ten minutes per scene." });
+  }
+  if (input.audio?.reactivity?.resource === undefined && input.audio?.reactivity !== undefined && (
+    input.resources.length >= 64
+    || input.resources.some(resource => resource.name === HTML_AUDIO_REACTIVITY_RESOURCE_NAME
+      || resource.urlPath === HTML_AUDIO_REACTIVITY_RESOURCE_URL_PATH)
+  )) {
+    context.addIssue({ code: "custom", message: "Audio reactivity requires one free resource slot and an unused audio-reactivity name and slopcamera/audio-reactivity.json URL path." });
+  }
+  if (input.audio?.reactivity?.resource === undefined && input.audio?.reactivity?.sha256 !== undefined) {
+    context.addIssue({ code: "custom", message: "An audio-reactivity sidecar digest requires its retained resource name." });
+  }
+  if (input.audio?.reactivity?.resource !== undefined && !input.resources.some(resource => (
+    resource.name === HTML_AUDIO_REACTIVITY_RESOURCE_NAME && resource.urlPath === HTML_AUDIO_REACTIVITY_RESOURCE_URL_PATH
+      && resource.mediaType === "application/json" && resource.transport === "fetch"
+  ))) {
+    context.addIssue({ code: "custom", message: "A retained audio-reactivity source must declare its fixed fetched JSON resource." });
+  }
+  if (input.audio?.reactivity?.resource !== undefined && input.audio.reactivity.sha256 === undefined) {
+    context.addIssue({ code: "custom", message: "A retained audio-reactivity source must record its sidecar SHA-256 digest." });
+  }
   const authoring = HtmlOverlayAuthoringInputSchema.safeParse({
     kind: "slopcamera.html-overlay", schemaVersion: 1, canvas: input.canvas, timing: input.timing,
     libraries: input.libraries, parameters: input.parameters, seed: input.seed,