@hraness/slopcamera 3.2.7 → 3.3.1

package/apps/desktop/core/index.ts

@@ -2,6 +2,7 @@ export * from "./auto-zoom";
 export * from "./audio-alignment";
 export * from "./alignment-apply";
 export * from "./canonical-json";
+export * from "./cinema-plan";
 export * from "./emoji";
 export * from "./events";
 export * from "./face-framing";

package/apps/desktop/core/music-analysis.ts

@@ -89,7 +89,7 @@ function validatePcm(pcm: MonoPcm): void {
   }
 }
 
-function fftPowerSpectrum(samples: readonly number[]): Float64Array {
+export function fftPowerSpectrum(samples: readonly number[]): Float64Array {
   let size = 1;
   while (size < samples.length) size *= 2;
   const real = new Float64Array(size);
@@ -145,6 +145,129 @@ function fftPowerSpectrum(samples: readonly number[]): Float64Array {
   return power;
 }
 
+export const AUDIO_REACTIVITY_PROFILE = "bands-v1" as const;
+export const AUDIO_REACTIVITY_SAMPLE_RATE_HZ = 24_000;
+export const AUDIO_REACTIVITY_RATE_HZ = 60;
+export const AUDIO_REACTIVITY_MAX_DURATION_US = 600_000_000;
+
+const AUDIO_REACTIVITY_WINDOW_SAMPLES = 2_048;
+const AUDIO_REACTIVITY_HOP_SAMPLES = AUDIO_REACTIVITY_SAMPLE_RATE_HZ / AUDIO_REACTIVITY_RATE_HZ;
+const AUDIO_REACTIVITY_MAXIMUM_SAMPLES = AUDIO_REACTIVITY_MAX_DURATION_US * AUDIO_REACTIVITY_SAMPLE_RATE_HZ / 1_000_000;
+const AUDIO_REACTIVITY_MAXIMUM_PCM_AMPLITUDE = 16;
+// Suppress quiet input below -80 dBFS RMS, and band leakage below 2% of the
+// simultaneous broadband RMS. Subtracting the floor avoids a hard onset step.
+const AUDIO_REACTIVITY_ABSOLUTE_RMS_FLOOR = 0.0001;
+const AUDIO_REACTIVITY_LEAKAGE_RMS_RATIO = 0.02;
+const AUDIO_REACTIVITY_FLAT_RANGE_RATIO = 0.001;
+const AUDIO_REACTIVITY_ATTACK_SECONDS = 0.03;
+const AUDIO_REACTIVITY_RELEASE_SECONDS = 0.18;
+// The FFT already applies a symmetric Hann window and divides power by N².
+// Correct its mean squared gain when recovering one-sided RMS energy.
+const AUDIO_REACTIVITY_HANN_POWER_GAIN = 3 * (AUDIO_REACTIVITY_WINDOW_SAMPLES - 1) / (8 * AUDIO_REACTIVITY_WINDOW_SAMPLES);
+
+export interface AudioReactivityBands {
+  readonly bass: readonly number[];
+  readonly mid: readonly number[];
+  readonly treble: readonly number[];
+  readonly energy: readonly number[];
+}
+
+type MutableAudioReactivityBands = { [Band in keyof AudioReactivityBands]: number[] };
+const AUDIO_REACTIVITY_BAND_NAMES = ["bass", "mid", "treble", "energy"] as const;
+
+function audioReactivityPercentile(ordered: readonly number[], fraction: number): number {
+  const position = (ordered.length - 1) * fraction;
+  const left = Math.floor(position);
+  const a = ordered[left]!;
+  return a + ((ordered[Math.min(left + 1, ordered.length - 1)] ?? a) - a) * (position - left);
+}
+
+function normalizeAndSmoothAudioReactivity(values: readonly number[], energy: readonly number[], band: boolean): readonly number[] {
+  const gated = values.map((value, index) => Math.max(0, value - Math.max(
+    AUDIO_REACTIVITY_ABSOLUTE_RMS_FLOOR,
+    band ? energy[index]! * AUDIO_REACTIVITY_LEAKAGE_RMS_RATIO : 0,
+  )));
+  const ordered = [...gated].sort((left, right) => left - right);
+  let low = audioReactivityPercentile(ordered, 0.05);
+  let high = audioReactivityPercentile(ordered, 0.95);
+  // Events occupying less than 5% of the track still have a useful range.
+  if (high <= EPSILON) high = ordered.at(-1)!;
+  // A sustained tone has energy even when its useful percentile span is flat.
+  if (high - low <= Math.max(EPSILON, high * AUDIO_REACTIVITY_FLAT_RANGE_RATIO)) low = 0;
+  const span = high - low;
+  const attack = 1 - Math.exp(-1 / (AUDIO_REACTIVITY_RATE_HZ * AUDIO_REACTIVITY_ATTACK_SECONDS));
+  const release = 1 - Math.exp(-1 / (AUDIO_REACTIVITY_RATE_HZ * AUDIO_REACTIVITY_RELEASE_SECONDS));
+  let previous = 0;
+  const result = gated.map(value => {
+    const target = span <= EPSILON ? 0 : Math.min(1, Math.max(0, (value - low) / span));
+    previous += (target - previous) * (target > previous ? attack : release);
+    return Math.min(1, Math.max(0, previous));
+  });
+  // This endpoint is at the exact PCM duration, not the next regular grid tick.
+  result.push(0);
+  return Object.freeze(result);
+}
+
+/**
+ * Analyze already-decoded mono PCM on a fixed absolute timeline. Each band has
+ * ceil(sampleCount / 400) samples at i / 60 seconds, then a zero endpoint at
+ * durationUs. Windows are centered with zero padding; their 42.7 ms lookahead
+ * and offline normalization are intentional. This detects energy, not beats.
+ */
+export function analyzeAudioReactivity(pcm: {
+  readonly sampleRateHz: number;
+  readonly samples: readonly number[] | Float32Array;
+}): { readonly durationUs: number; readonly bands: AudioReactivityBands } {
+  if (pcm.sampleRateHz !== AUDIO_REACTIVITY_SAMPLE_RATE_HZ) {
+    throw new TypeError(`Audio reactivity PCM must use exactly ${AUDIO_REACTIVITY_SAMPLE_RATE_HZ} Hz.`);
+  }
+  if ((!Array.isArray(pcm.samples) && !(pcm.samples instanceof Float32Array))
+    || pcm.samples.length === 0 || pcm.samples.length > AUDIO_REACTIVITY_MAXIMUM_SAMPLES) {
+    throw new TypeError(`Audio reactivity PCM must contain 1–${AUDIO_REACTIVITY_MAXIMUM_SAMPLES} samples.`);
+  }
+  let nonzero = false;
+  for (let index = 0; index < pcm.samples.length; index++) {
+    const sample = pcm.samples[index]!;
+    if (!Number.isFinite(sample) || Math.abs(sample) > AUDIO_REACTIVITY_MAXIMUM_PCM_AMPLITUDE) {
+      throw new TypeError(`Audio reactivity PCM samples must be finite numbers within ±${AUDIO_REACTIVITY_MAXIMUM_PCM_AMPLITUDE}.`);
+    }
+    nonzero ||= sample !== 0;
+  }
+  const durationUs = Math.round(pcm.samples.length * 1_000_000 / AUDIO_REACTIVITY_SAMPLE_RATE_HZ);
+  const count = Math.ceil(pcm.samples.length / AUDIO_REACTIVITY_HOP_SAMPLES);
+  if (!nonzero) {
+    const silence = Object.freeze(Array<number>(count + 1).fill(0));
+    return Object.freeze({ durationUs, bands: Object.freeze({ bass: silence, mid: silence, treble: silence, energy: silence }) });
+  }
+  const raw: MutableAudioReactivityBands = { bass: [], mid: [], treble: [], energy: [] };
+  const window = Array<number>(AUDIO_REACTIVITY_WINDOW_SAMPLES).fill(0);
+  for (let frame = 0; frame < count; frame++) {
+    const start = frame * AUDIO_REACTIVITY_HOP_SAMPLES - AUDIO_REACTIVITY_WINDOW_SAMPLES / 2;
+    for (let index = 0; index < AUDIO_REACTIVITY_WINDOW_SAMPLES; index++) {
+      const source = start + index;
+      window[index] = source < 0 || source >= pcm.samples.length ? 0 : pcm.samples[source]!;
+    }
+    const spectrum = fftPowerSpectrum(window);
+    const power = { bass: 0, mid: 0, treble: 0, energy: 0 };
+    for (let bin = 0; bin < spectrum.length; bin++) {
+      const frequencyHz = bin * AUDIO_REACTIVITY_SAMPLE_RATE_HZ / AUDIO_REACTIVITY_WINDOW_SAMPLES;
+      const value = spectrum[bin]! * (bin === 0 || bin === spectrum.length - 1 ? 1 : 2) / AUDIO_REACTIVITY_HANN_POWER_GAIN;
+      power.energy += value;
+      if (frequencyHz >= 35 && frequencyHz < 180) power.bass += value;
+      else if (frequencyHz >= 180 && frequencyHz < 2_000) power.mid += value;
+      else if (frequencyHz >= 2_000 && frequencyHz <= 12_000) power.treble += value;
+    }
+    for (const name of AUDIO_REACTIVITY_BAND_NAMES) raw[name].push(Math.sqrt(power[name]));
+  }
+  const bands = Object.freeze({
+    bass: normalizeAndSmoothAudioReactivity(raw.bass, raw.energy, true),
+    mid: normalizeAndSmoothAudioReactivity(raw.mid, raw.energy, true),
+    treble: normalizeAndSmoothAudioReactivity(raw.treble, raw.energy, true),
+    energy: normalizeAndSmoothAudioReactivity(raw.energy, raw.energy, false),
+  });
+  return Object.freeze({ durationUs, bands });
+}
+
 /** Fold pitched energy from C2 through B6 into twelve pitch classes. */
 export function computeChroma(frame: readonly number[], sampleRateHz: number): readonly number[] {
   if (!Number.isSafeInteger(sampleRateHz) || sampleRateHz < 8_000 || sampleRateHz > 192_000) {

package/apps/desktop/core/storage.ts

@@ -18,6 +18,7 @@ import {
   FaceAnalysisV1Schema,
   MusicAnalysisV1Schema,
   ProjectInactivityAnalysisV1Schema,
+  ProjectCinemaPlanV1Schema,
   ProjectEditPlanV1Schema,
   RecordingManifestV1Schema,
   RepositoryRelativePathSchema,
@@ -30,6 +31,7 @@ import {
   type FaceAnalysisV1,
   type MusicAnalysisV1,
   type ProjectInactivityAnalysisV1,
+  type ProjectCinemaPlanV1,
   type ProjectEditPlanV1,
   type RecordingManifestV1,
   type SceneAnalysisV1,
@@ -41,6 +43,7 @@ import { canonicalJson, sha256Hex } from "./canonical-json";
 export const RECORDING_MANIFEST_PATH = "manifest.json";
 export const VIDEO_PROJECT_PATH = "project.json";
 export const CURRENT_PROJECT_EDIT_PLAN_PATH = "edits/current.json";
+export const CURRENT_PROJECT_CINEMA_PLAN_PATH = "cinema/current.json";
 const MAXIMUM_STRUCTURED_FILE_BYTES = 256 * 1024 * 1024;
 const IMMUTABLE_COPY_TEMP_PREFIX = ".slopcamera-copy-";
 
@@ -89,6 +92,7 @@ export interface BundleFileIntegrity {
 export type SlopcameraPersistenceDocument =
   | AnalysisArtifact
   | EditPlanV1
+  | ProjectCinemaPlanV1
   | ProjectEditPlanV1
   | RecordingManifestV1
   | VideoProjectV1;
@@ -264,6 +268,77 @@ export async function saveProjectEditRevision(
   return path;
 }
 
+export function projectCinemaRevisionPath(artifactSha256: string): string {
+  const sha256 = Sha256Schema.parse(artifactSha256);
+  return RepositoryRelativePathSchema.parse(`cinema/revisions/${sha256}.json`);
+}
+
+export function projectCinemaRenderPlanPath(planSha256: string): string {
+  const sha256 = Sha256Schema.parse(planSha256);
+  return RepositoryRelativePathSchema.parse(`cinema/plans/${sha256}.json`);
+}
+
+/**
+ * Load the live cinema sidecar, or null when the project has none. Its absence
+ * is meaningful: project render planning must stay byte-identical without it.
+ */
+export async function loadProjectCinemaPlan(
+  fileSystem: BundleFileSystem,
+  path = CURRENT_PROJECT_CINEMA_PLAN_PATH,
+): Promise<ProjectCinemaPlanV1 | null> {
+  RepositoryRelativePathSchema.parse(path);
+  let text: string;
+  try {
+    text = await fileSystem.readText(path);
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "ENOENT") return null;
+    throw error;
+  }
+  const input: unknown = JSON.parse(text);
+  return ProjectCinemaPlanV1Schema.parse(input);
+}
+
+export async function saveProjectCinemaPlan(
+  fileSystem: BundleFileSystem,
+  plan: ProjectCinemaPlanV1,
+  path = CURRENT_PROJECT_CINEMA_PLAN_PATH,
+): Promise<void> {
+  RepositoryRelativePathSchema.parse(path);
+  const parsed = ProjectCinemaPlanV1Schema.parse(canonicalSlopcameraPersistenceDocument(
+    ProjectCinemaPlanV1Schema.parse(plan),
+  ));
+  await fileSystem.writeTextAtomic(path, `${canonicalJson(parsed)}\n`);
+}
+
+/** Publish one content-addressed cinema sidecar revision without replacement. */
+export async function saveProjectCinemaRevision(
+  fileSystem: BundleFileSystem,
+  contents: string,
+  expectedArtifactSha256: string,
+): Promise<string> {
+  const sha256 = Sha256Schema.parse(expectedArtifactSha256);
+  const actualSha256 = sha256Hex(contents);
+  if (actualSha256 !== sha256) {
+    throw new Error(
+      `Project cinema revision artifact hash mismatch: expected ${sha256}, received ${actualSha256}.`,
+    );
+  }
+  if (fileSystem.writeTextNoReplace === undefined) {
+    throw new Error("Bundle file system does not support immutable no-replace publication.");
+  }
+  const path = projectCinemaRevisionPath(sha256);
+  const disposition = await fileSystem.writeTextNoReplace(path, contents);
+  const installed = await fileSystem.readText(path);
+  if (installed !== contents) {
+    throw new Error(
+      disposition === "exists"
+        ? `Project cinema revision path already contains different bytes: ${path}`
+        : `Published project cinema revision failed read-back verification: ${path}`,
+    );
+  }
+  return path;
+}
+
 export type AnalysisArtifact =
   | AudioAlignmentAnalysisV1
   | FaceAnalysisV1