@hyperframes/studio 0.8.107 → 0.8.108

package/src/player/components/clipMenuDuck.ts

@@ -0,0 +1,142 @@
+import {
+  HF_AUDIO_FX_ATTR,
+  parseAudioFxChain,
+  serializeAudioFxChain,
+  type HfAudioFxChain,
+} from "@hyperframes/core/audio-fx";
+import {
+  clipsOverlap,
+  DEFAULT_CARVE,
+  HF_AUDIO_CARVE_ATTR,
+  normalizeCarveSettings,
+  type HfCarveSettings,
+} from "@hyperframes/core/audio-carve";
+import { resolveCarveSourceIds } from "@hyperframes/core/audio-groups";
+import { parseAutomation, type HfAutomation } from "@hyperframes/core/audio-automation";
+import {
+  carveBedRoles,
+  carverAgainst,
+  collectCarveCandidates,
+  CARVE_ABORTED,
+  isPromiseLike,
+  resolveNextCarveSettings,
+} from "../../components/editor/useFxCarveGrouping";
+import {
+  dropCarveOutput,
+  resolveCarveVoices,
+  withoutCarveLanes,
+} from "../../components/editor/carveOutput";
+import { carveLanes, measureCarve, mintCarveNodes } from "../../components/editor/useFxCarveNodes";
+import { readClipClock } from "../../components/editor/clipAudioClock";
+import {
+  automationAttrValue,
+  HF_AUDIO_AUTOMATION_ATTR,
+} from "../../components/editor/propertyPanelAutomation";
+import { spanOf } from "../../components/editor/propertyPanelAudioFxGroupUtils";
+
+export type WriteBedAttribute = (attr: string, value: string | null) => Promise<void>;
+export type GroupClips = (clipIds: readonly string[], groupId: string) => Promise<void>;
+
+export function readBedCarve(bed: Element): HfCarveSettings | null {
+  const raw = bed.getAttribute(HF_AUDIO_CARVE_ATTR);
+  if (!raw) return null;
+  try {
+    return normalizeCarveSettings(JSON.parse(raw));
+  } catch {
+    return null;
+  }
+}
+
+function readChain(bed: Element): HfAudioFxChain {
+  const raw = bed.getAttribute(HF_AUDIO_FX_ATTR);
+  try {
+    return raw ? parseAudioFxChain(raw) : { version: 1, nodes: [] };
+  } catch {
+    return { version: 1, nodes: [] };
+  }
+}
+
+function readAutomation(bed: Element): HfAutomation {
+  const raw = bed.getAttribute(HF_AUDIO_AUTOMATION_ATTR);
+  try {
+    return raw ? parseAutomation(raw) : { version: 1, lanes: [] };
+  } catch {
+    return { version: 1, lanes: [] };
+  }
+}
+
+export function isDuckableBed(bed: Element | null): bed is Element {
+  if (bed === null || !carveBedRoles(bed.id, bed).couldBeBed) return false;
+  return carverAgainst(bed.ownerDocument, bed.id) === null;
+}
+
+/** Every overlapping clip with speech, including a video's own sound; never the bed itself. */
+export function duckVoiceSources(doc: Document, bed: Element): string[] {
+  const bedSpan = spanOf(bed.getAttribute("data-start"), bed.getAttribute("data-duration"));
+  const others = Array.from(doc.querySelectorAll("audio[id], video[id]")).filter(
+    (el) => el.id !== bed.id,
+  );
+  const overlaps = (el: Element) =>
+    clipsOverlap(bedSpan, spanOf(el.getAttribute("data-start"), el.getAttribute("data-duration")));
+  return collectCarveCandidates(doc, others, overlaps, bed.id)
+    .filter((candidate) => candidate.kind === "voice" || candidate.kind === "unknown")
+    .map((candidate) => candidate.id);
+}
+
+async function writeMeasuredCarve(
+  doc: Document,
+  bed: Element,
+  settings: HfCarveSettings,
+  write: WriteBedAttribute,
+): Promise<void> {
+  const expanded = resolveCarveSourceIds(doc, settings.sources).filter((id) => id !== bed.id);
+  const voices = resolveCarveVoices(doc, expanded);
+  if (voices.length === 0) return;
+  const measured = await measureCarve(doc, voices, settings.strength, {
+    src: bed.getAttribute("src"),
+    start: bed.getAttribute("data-start"),
+    clock: readClipClock((name) => bed.getAttribute(name)),
+  });
+  if (!measured) return;
+  const chain = readChain(bed);
+  const automation = readAutomation(bed);
+  const { next, carvedNodes, duckNode } = mintCarveNodes(chain, measured.carved, measured.duck);
+  await write(HF_AUDIO_FX_ATTR, serializeAudioFxChain(next));
+  const lanes = carveLanes(carvedNodes, duckNode, measured.duck, measured.voiceMix, measured.bands);
+  const carriedOver = withoutCarveLanes(automation, chain);
+  const nextAutomation = { version: 1, lanes: [...carriedOver.lanes, ...lanes] };
+  await write(HF_AUDIO_AUTOMATION_ATTR, automationAttrValue(nextAutomation) || null);
+}
+
+export type DuckOutcome = "ducked" | "no-voice" | "aborted" | "off";
+
+/**
+ * Switch the carve on a music bed. On: carve against every overlapping voice at
+ * core's default strength (several voices are grouped first, as the carve lint
+ * requires). Off: `enabled:false`, which drops the generated filters and lanes.
+ */
+export async function setDuckUnderVoice(
+  doc: Document,
+  bed: Element,
+  on: boolean,
+  write: WriteBedAttribute,
+  groupClips?: GroupClips,
+): Promise<DuckOutcome> {
+  const current = readBedCarve(bed);
+  if (!on) {
+    await dropCarveOutput(readChain(bed), readAutomation(bed), write);
+    await write(
+      HF_AUDIO_CARVE_ATTR,
+      JSON.stringify({ ...(current ?? DEFAULT_CARVE), enabled: false }),
+    );
+    return "off";
+  }
+  const sources = duckVoiceSources(doc, bed);
+  if (sources.length === 0) return "no-voice";
+  const resolved = resolveNextCarveSettings({ ...DEFAULT_CARVE, sources }, doc, groupClips);
+  const settings = isPromiseLike(resolved) ? await resolved : resolved;
+  if (settings === CARVE_ABORTED || settings === null) return "aborted";
+  await write(HF_AUDIO_CARVE_ATTR, JSON.stringify(settings));
+  await writeMeasuredCarve(doc, bed, settings, write);
+  return "ducked";
+}

package/src/player/components/clipMenuNormalize.test.ts

@@ -0,0 +1,95 @@
+// @vitest-environment happy-dom
+import { describe, expect, it, vi } from "vitest";
+import {
+  clipHasSound,
+  normalizeRequestBody,
+  normalizeToastText,
+  requestNormalizePlan,
+} from "./clipMenuNormalize";
+import type { TimelineElement } from "../store/timelineElement";
+
+const clip = (over: Partial<TimelineElement>): TimelineElement => ({
+  id: "vo",
+  tag: "audio",
+  start: 0,
+  duration: 4,
+  track: 0,
+  ...over,
+});
+
+describe("clipHasSound", () => {
+  it("is true for audio and for a video with sound, false for silent or muted video", () => {
+    expect(clipHasSound(clip({}))).toBe(true);
+    expect(clipHasSound(clip({ tag: "video", hasAudio: true }))).toBe(true);
+    expect(clipHasSound(clip({ tag: "video", hasAudio: false }))).toBe(false);
+    expect(clipHasSound(clip({ tag: "video", hasAudio: true, muted: true }))).toBe(false);
+    expect(clipHasSound(clip({ tag: "img" }))).toBe(false);
+  });
+});
+
+describe("normalizeRequestBody", () => {
+  it("sends the played source window and the current gain", () => {
+    const body = normalizeRequestBody(
+      clip({ domId: "a-roll", playbackStart: 2, playbackRate: 2, volume: 0.5, duration: 3 }),
+      "talk.mp4",
+    );
+    expect(body).toEqual({
+      id: "a-roll",
+      src: "talk.mp4",
+      mediaStart: 2,
+      duration: 3,
+      playbackRate: 2,
+      volume: 0.5,
+    });
+  });
+});
+
+describe("normalizeToastText", () => {
+  it("names the target and the change", () => {
+    expect(
+      normalizeToastText({
+        targetLufs: -16,
+        projectedLufs: -16,
+        volume: 1.44,
+        changeDb: 3.2,
+        limitedBy: null,
+      }),
+    ).toBe("Normalized to −16 LUFS (+3.2 dB)");
+  });
+
+  it("says so when a limit held it back", () => {
+    expect(
+      normalizeToastText({
+        targetLufs: -16,
+        projectedLufs: -18.4,
+        volume: 4,
+        changeDb: 12,
+        limitedBy: "gain-ceiling",
+      }),
+    ).toBe("Raised to −18 LUFS (+12.0 dB), the most the +12 dB ceiling allows");
+  });
+});
+
+describe("requestNormalizePlan", () => {
+  it("posts the project-relative source and returns the plan", async () => {
+    const plan = { targetLufs: -16, projectedLufs: -16, volume: 2, changeDb: 6, limitedBy: null };
+    const fetchImpl = vi.fn<typeof fetch>(async () => Response.json({ plan }));
+    const origin = window.location.origin;
+    const el = clip({ src: `${origin}/api/projects/p1/preview/assets/vo.wav` });
+    await expect(requestNormalizePlan("p1", el, fetchImpl)).resolves.toEqual(plan);
+    const [url, init] = fetchImpl.mock.calls[0] ?? [];
+    expect(url).toBe("/api/projects/p1/loudness/normalize");
+    expect(JSON.parse(String(init?.body))).toEqual(
+      expect.objectContaining({ src: "assets/vo.wav" }),
+    );
+  });
+
+  it("surfaces the server's reason", async () => {
+    const fetchImpl = vi.fn<typeof fetch>(async () =>
+      Response.json({ error: "#vo is silent" }, { status: 422 }),
+    );
+    await expect(
+      requestNormalizePlan("p1", clip({ src: "assets/vo.wav" }), fetchImpl),
+    ).rejects.toThrow(/silent/);
+  });
+});

package/src/player/components/clipMenuNormalize.ts

@@ -0,0 +1,96 @@
+import { buildProjectApiPath } from "../../utils/projectRouting";
+import { resolvePreviewRelative } from "../../utils/previewRelativePath";
+import { resolveMediaPreviewUrl } from "./thumbnailUtils";
+import type { TimelineElement } from "../store/timelineElement";
+import { parseAutomation, VOLUME_TARGET } from "@hyperframes/core/audio-automation";
+import type { TimelineEditOutcome } from "../../hooks/timelineEditPermission";
+
+export interface NormalizePlan {
+  targetLufs: number;
+  projectedLufs: number;
+  volume: number;
+  changeDb: number;
+  limitedBy: "gain-ceiling" | "true-peak" | null;
+}
+
+export function clipHasSound(el: TimelineElement): boolean {
+  if (el.tag === "audio") return true;
+  return el.tag === "video" && el.hasAudio === true && el.muted !== true;
+}
+
+export const VOLUME_LANE_REFUSAL =
+  "This clip's volume is automated. Remove its volume automation to normalize it.";
+
+export function volumeLaneOwnsGain(el: TimelineElement): boolean {
+  if (!el.automation) return false;
+  try {
+    return parseAutomation(el.automation).lanes.some(
+      (lane) => lane.target === VOLUME_TARGET && lane.points.length > 0,
+    );
+  } catch {
+    return false;
+  }
+}
+
+export class TimelineSaveError extends Error {}
+
+export function throwUnlessSaved(outcome: TimelineEditOutcome | void): void {
+  if (outcome && outcome.status !== "saved") throw new TimelineSaveError(outcome.reason);
+}
+
+export function normalizeRequestBody(el: TimelineElement, projectRelativeSrc: string) {
+  return {
+    id: el.domId ?? el.id,
+    src: projectRelativeSrc,
+    mediaStart: el.playbackStart ?? 0,
+    duration: el.duration > 0 ? el.duration : undefined,
+    playbackRate: el.playbackRate ?? 1,
+    volume: el.volume ?? 1,
+  };
+}
+
+const signedDb = (db: number) => `${db >= 0 ? "+" : "−"}${Math.abs(db).toFixed(1)} dB`;
+const lufsText = (lufs: number) => `${lufs < 0 ? "−" : ""}${Math.abs(lufs).toFixed(0)} LUFS`;
+
+const LIMIT_SUFFIX = {
+  "gain-ceiling": "the most the +12 dB ceiling allows",
+  "true-peak": "held back to keep peaks under −1.5 dBTP",
+} satisfies Record<NonNullable<NormalizePlan["limitedBy"]>, string>;
+
+export function normalizeToastText(plan: NormalizePlan): string {
+  if (!plan.limitedBy)
+    return `Normalized to ${lufsText(plan.targetLufs)} (${signedDb(plan.changeDb)})`;
+  const reached = `${lufsText(Math.round(plan.projectedLufs))}`;
+  return `Raised to ${reached} (${signedDb(plan.changeDb)}), ${LIMIT_SUFFIX[plan.limitedBy]}`;
+}
+
+function isNormalizePlan(value: unknown): value is NormalizePlan {
+  if (typeof value !== "object" || value === null) return false;
+  return ["targetLufs", "projectedLufs", "volume", "changeDb"].every(
+    (key) => typeof Reflect.get(value, key) === "number",
+  );
+}
+
+export async function requestNormalizePlan(
+  projectId: string,
+  el: TimelineElement,
+  fetchImpl: typeof fetch = fetch,
+): Promise<NormalizePlan> {
+  const origin = window.location.origin;
+  const relative = resolvePreviewRelative(
+    resolveMediaPreviewUrl(el.src ?? "", projectId, origin),
+    projectId,
+    origin,
+  );
+  if (!relative) throw new Error("Only project media can be normalized");
+  const res = await fetchImpl(buildProjectApiPath(projectId, "/loudness/normalize"), {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(normalizeRequestBody(el, relative)),
+  });
+  const body: unknown = await res.json().catch(() => null);
+  const plan = typeof body === "object" && body !== null ? Reflect.get(body, "plan") : null;
+  if (res.ok && isNormalizePlan(plan)) return plan;
+  const error = typeof body === "object" && body !== null ? Reflect.get(body, "error") : null;
+  throw new Error(typeof error === "string" ? error : "Could not measure this clip");
+}

package/src/player/components/clipMenuToolItems.test.tsx

@@ -0,0 +1,139 @@
+// @vitest-environment happy-dom
+
+import { act } from "react";
+import { createRoot, type Root } from "react-dom/client";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { TimelineElement } from "../store/playerStore";
+import { TimelineEditProvider } from "../../contexts/TimelineEditContext";
+import { useCropPresetBarStore } from "../../components/editor/cropPresetStore";
+import { ClipMenuToolItems, type ClipMenuToolGroup } from "./clipMenuToolItems";
+
+Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", {
+  configurable: true,
+  value: true,
+});
+
+let root: Root | null = null;
+
+afterEach(() => {
+  act(() => root?.unmount());
+  root = null;
+  document.body.innerHTML = "";
+  useCropPresetBarStore.getState().close();
+});
+
+const video: TimelineElement = {
+  id: "talk",
+  tag: "video",
+  start: 0,
+  duration: 6,
+  track: 0,
+  hasAudio: true,
+};
+
+function renderItems(group: ClipMenuToolGroup, element: TimelineElement, currentTime = 2) {
+  const setQuiet = vi.fn(
+    async (_el: TimelineElement, _attr: string, _value: string | null, _label: string) => undefined,
+  );
+  const onClose = vi.fn();
+  const freeze = vi.fn((_el: TimelineElement, _time: number) => undefined);
+  const host = document.createElement("div");
+  document.body.append(host);
+  root = createRoot(host);
+  act(() => {
+    root?.render(
+      <TimelineEditProvider value={{ onSetElementAttributeQuiet: setQuiet, onFreezeFrame: freeze }}>
+        <ClipMenuToolItems
+          group={group}
+          element={element}
+          currentTime={currentTime}
+          onClose={onClose}
+        />
+      </TimelineEditProvider>,
+    );
+  });
+  return { setQuiet, onClose, freeze };
+}
+
+function openSubmenu(label: string) {
+  const row = Array.from(document.querySelectorAll<HTMLButtonElement>('[role="menuitem"]')).find(
+    (button) => button.textContent?.startsWith(label),
+  );
+  act(() => row?.click());
+}
+
+function pick(text: string) {
+  const item = Array.from(
+    document.querySelectorAll<HTMLButtonElement>('[role="menuitemradio"]'),
+  ).find((button) => button.textContent?.endsWith(text));
+  act(() => item?.click());
+}
+
+describe("ClipMenuToolItems", () => {
+  it("Voice writes a single-choice data-fx-chain on a video with sound", () => {
+    const { setQuiet, onClose } = renderItems("sound", video);
+    openSubmenu("Voice");
+    pick("Clean");
+    const [element, attr, value] = setQuiet.mock.calls[0] ?? [];
+    expect(element).toBe(video);
+    expect(attr).toBe("data-fx-chain");
+    expect(String(value)).toContain('"fromPreset":"voice-clean"');
+    expect(onClose).toHaveBeenCalled();
+  });
+
+  it("Voice None drops the attribute", () => {
+    const { setQuiet } = renderItems("sound", video);
+    openSubmenu("Voice");
+    pick("None");
+    expect(setQuiet.mock.calls[0]?.slice(1, 3)).toEqual(["data-fx-chain", null]);
+  });
+
+  it("has no Voice on a video without sound", () => {
+    renderItems("sound", { ...video, hasAudio: false });
+    expect(document.body.textContent).toBe("");
+  });
+
+  it("Look writes the preset form and None removes it", () => {
+    const { setQuiet } = renderItems("picture", video);
+    openSubmenu("Look");
+    pick("Warm daylight");
+    openSubmenu("Look");
+    pick("None");
+    expect(setQuiet.mock.calls.map((call) => call.slice(1, 3))).toEqual([
+      ["data-color-grading", '{"preset":"warm-daylight","intensity":1}'],
+      ["data-color-grading", null],
+    ]);
+  });
+
+  it("Crop opens the preset bar for this clip", () => {
+    renderItems("picture", video);
+    const crop = Array.from(document.querySelectorAll<HTMLButtonElement>("button")).find(
+      (button) => button.textContent === "Crop",
+    );
+    act(() => crop?.click());
+    expect(useCropPresetBarStore.getState().openFor).toEqual({ hfId: undefined, id: "talk" });
+  });
+
+  it("offers no picture tools on an audio clip", () => {
+    renderItems("picture", { ...video, tag: "audio" });
+    expect(document.body.textContent).toBe("");
+  });
+
+  it("Freeze frame calls the freeze mutation at the playhead on a video", () => {
+    const { freeze } = renderItems("time", video, 3.2);
+    const item = document.querySelector<HTMLButtonElement>('[role="menuitem"]');
+    expect(item?.textContent).toBe("Freeze frame");
+    act(() => item?.click());
+    expect(freeze).toHaveBeenCalledWith(video, 3.2);
+  });
+
+  it("Freeze frame is disabled outside the clip and absent on images", () => {
+    renderItems("time", video, 9);
+    expect(document.querySelector<HTMLButtonElement>('[role="menuitem"]')?.disabled).toBe(true);
+    act(() => root?.unmount());
+    root = null;
+    document.body.innerHTML = "";
+    renderItems("time", { ...video, tag: "img" });
+    expect(document.body.textContent).toBe("");
+  });
+});