@hyperframes/studio 0.8.106 → 0.8.108

package/src/components/editor/mediaAudioEdits.ts

@@ -0,0 +1,282 @@
+import {
+  HF_AUDIO_AUTOMATION_ATTR,
+  RATE_TARGET,
+  parseAutomation,
+  serializeAutomation,
+  type HfAutomationLane,
+} from "@hyperframes/core/audio-automation";
+import { HF_AUDIO_CARVE_ATTR } from "@hyperframes/core/audio-carve";
+import { HF_AUDIO_FADE_IN_ATTR, HF_AUDIO_FADE_OUT_ATTR } from "@hyperframes/core/audio-fade";
+import { HF_AUDIO_FX_ATTR } from "@hyperframes/core/audio-fx";
+import { HF_AUDIO_GROUP_ATTR } from "@hyperframes/core/audio-groups";
+import {
+  escapeHtmlAttribute,
+  findTagByTarget,
+  applyPatchByTarget,
+  readAttributeByTarget,
+  unescapeHtmlAttribute,
+  type PatchOperation,
+  type PatchTarget,
+} from "../../utils/sourcePatcher";
+import { collectHtmlIds, resolveAssetHasAudio } from "../../utils/studioHelpers";
+import type { CommitDomAttributeBatch } from "../../hooks/domEditCommitTypes";
+import { buildDomEditPatchTarget, type DomEditSelection } from "./domEditing";
+import { generateId } from "../../utils/generateId";
+
+const MEDIA_LINK_ATTR = "data-link";
+const SOUND_KEPT_ON_LINKED_AUDIO_STAGE = "Background removed. Sound kept on a linked audio track.";
+
+const SOUND_NOT_KEPT_MESSAGE =
+  "Background removed, but the sound could not be kept on a linked audio track. Undo to restore the original clip.";
+
+const MOVED_SOUND_ATTRS = [
+  "data-volume",
+  HF_AUDIO_FADE_IN_ATTR,
+  HF_AUDIO_FADE_OUT_ATTR,
+  HF_AUDIO_FX_ATTR,
+  HF_AUDIO_CARVE_ATTR,
+  HF_AUDIO_GROUP_ATTR,
+];
+
+const COPIED_TIMING_ATTRS = [
+  "start",
+  "end",
+  "duration",
+  "media-start",
+  "playback-start",
+  "playback-rate",
+  "source-duration",
+];
+
+const htmlAttr = (property: string, value: string | null): PatchOperation => ({
+  type: "html-attribute",
+  property,
+  value,
+});
+
+const dataAttr = (property: string, value: string | null): PatchOperation => ({
+  type: "attribute",
+  property,
+  value,
+});
+
+function stripDataPrefix(name: string): string {
+  return name.startsWith("data-") ? name.slice("data-".length) : name;
+}
+
+function probedHasAudioValue(probedHasAudio: boolean | null): string | null {
+  if (probedHasAudio === null) return null;
+  return probedHasAudio ? "true" : "false";
+}
+
+export function mutedToggleOps(input: {
+  isVideo: boolean;
+  nextMuted: boolean;
+  probedHasAudio: boolean | null;
+}): PatchOperation[] {
+  const muted = htmlAttr("muted", input.nextMuted ? "true" : null);
+  if (!input.isVideo) return [muted];
+  const hasAudio = input.nextMuted ? null : probedHasAudioValue(input.probedHasAudio);
+  return [muted, dataAttr("has-audio", hasAudio)];
+}
+
+export function hasAudioToggleOps(next: boolean): PatchOperation[] {
+  return next
+    ? [dataAttr("has-audio", "true"), htmlAttr("muted", null)]
+    : [dataAttr("has-audio", null), htmlAttr("muted", "true")];
+}
+
+export function cutoutOps(input: { isVideo: boolean; cutoutSrc: string }): PatchOperation[] {
+  const src = htmlAttr("src", input.cutoutSrc);
+  return input.isVideo ? [src, htmlAttr("muted", "true"), dataAttr("has-audio", null)] : [src];
+}
+
+function firstFreeName(base: string, taken: Set<string>): string {
+  if (!taken.has(base)) return base;
+  let n = 2;
+  while (taken.has(`${base}-${n}`)) n += 1;
+  return `${base}-${n}`;
+}
+
+export function mintLinkId(doc: Document): string {
+  const taken = new Set<string>();
+  for (const el of Array.from(doc.querySelectorAll(`[id], [${MEDIA_LINK_ATTR}]`))) {
+    if (el.id) taken.add(el.id);
+    const link = el.getAttribute(MEDIA_LINK_ATTR);
+    if (link) taken.add(link);
+  }
+  return firstFreeName("link", taken);
+}
+
+function firstFreeTrackIndex(source: string): number {
+  const used = new Set(
+    Array.from(source.matchAll(/\bdata-track-index=["'](-?\d+)["']/g), (m) => Number(m[1])),
+  );
+  let track = 0;
+  while (used.has(track)) track += 1;
+  return track;
+}
+
+function splitAutomation(raw: string): { videoKeeps: string | null; audioTakes: string } {
+  let lanes: HfAutomationLane[];
+  try {
+    lanes = parseAutomation(raw).lanes;
+  } catch {
+    return { videoKeeps: null, audioTakes: raw };
+  }
+  const rateLanes = lanes.filter((lane) => lane.target === RATE_TARGET);
+  const soundLanes = lanes.filter((lane) => lane.target !== RATE_TARGET);
+  return {
+    videoKeeps: rateLanes.length > 0 ? serializeAutomation({ version: 1, lanes: rateLanes }) : null,
+    audioTakes: serializeAutomation({ version: 1, lanes: [...soundLanes, ...rateLanes] }),
+  };
+}
+
+function formatAttrs(attrs: Array<[string, string]>): string {
+  return attrs.map(([name, value]) => `${name}="${escapeHtmlAttribute(value)}"`).join(" ");
+}
+
+function readAuthoredSrc(source: string, target: PatchTarget): string {
+  const tag = findTagByTarget(source, target);
+  if (!tag) return "";
+  const own = /\bsrc=(["'])([^"']*)\1/.exec(tag.tag);
+  if (own?.[2]) return unescapeHtmlAttribute(own[2]);
+  const firstSource = /^\s*<source\b[^>]*?\bsrc=(["'])([^"']*)\1/i.exec(source.slice(tag.end + 1));
+  return firstSource?.[2] ? unescapeHtmlAttribute(firstSource[2]) : "";
+}
+
+function insertBeforeTarget(source: string, target: PatchTarget, markup: string): string {
+  const match = findTagByTarget(source, target);
+  if (!match) return source;
+  const lineStart = source.lastIndexOf("\n", match.start) + 1;
+  const indent = /^[ \t]*$/.test(source.slice(lineStart, match.start))
+    ? source.slice(lineStart, match.start)
+    : "";
+  const separator = indent ? `\n${indent}` : "";
+  return `${source.slice(0, match.start)}${markup}${separator}${source.slice(match.start)}`;
+}
+
+export interface KeepSoundCutoutEdit {
+  ops: PatchOperation[];
+  prepareContent: (source: string) => string;
+  audioInserted: () => boolean;
+}
+
+export function buildKeepSoundCutoutEdit(input: {
+  video: Element;
+  videoId: string | null;
+  target: PatchTarget;
+  cutoutSrc: string;
+}): KeepSoundCutoutEdit {
+  const { video, target } = input;
+  const linkId = mintLinkId(video.ownerDocument);
+  let audioInserted = false;
+
+  const movedSound: Array<[string, string]> = [];
+  const cutoutSrcOp = htmlAttr("src", input.cutoutSrc);
+  const ops: PatchOperation[] = [htmlAttr("muted", "true"), dataAttr("has-audio", null)];
+  for (const name of MOVED_SOUND_ATTRS) {
+    const value = video.getAttribute(name);
+    if (value === null) continue;
+    movedSound.push([name, value]);
+    ops.push(dataAttr(stripDataPrefix(name), null));
+  }
+  const automation = video.getAttribute(HF_AUDIO_AUTOMATION_ATTR);
+  if (automation !== null) {
+    const { videoKeeps, audioTakes } = splitAutomation(automation);
+    movedSound.push([HF_AUDIO_AUTOMATION_ATTR, audioTakes]);
+    ops.push(dataAttr(stripDataPrefix(HF_AUDIO_AUTOMATION_ATTR), videoKeeps));
+  }
+  ops.push(dataAttr(stripDataPrefix(MEDIA_LINK_ATTR), linkId));
+
+  const prepareContent = (source: string): string => {
+    const originalSrc = readAuthoredSrc(source, target);
+    const audioId = firstFreeName(
+      `${input.videoId || "video"}-audio`,
+      new Set(collectHtmlIds(source)),
+    );
+    const timing: Array<[string, string]> = [];
+    for (const name of COPIED_TIMING_ATTRS) {
+      const value = readAttributeByTarget(source, target, name);
+      if (value !== undefined) timing.push([`data-${name}`, value]);
+    }
+    const attrs: Array<[string, string]> = [
+      ["id", audioId],
+      ["data-hf-id", `hf-${generateId()}`],
+      ["class", "clip"],
+      ["src", originalSrc],
+      ...timing,
+      ["data-track-index", String(firstFreeTrackIndex(source))],
+      [MEDIA_LINK_ATTR, linkId],
+      ...movedSound,
+    ];
+    if (video.hasAttribute("loop")) attrs.push(["loop", ""]);
+    const cut = applyPatchByTarget(source, target, cutoutSrcOp);
+    const inserted = insertBeforeTarget(cut, target, `<audio ${formatAttrs(attrs)}></audio>`);
+    audioInserted = inserted !== cut;
+    return inserted;
+  };
+
+  return { ops, prepareContent, audioInserted: () => audioInserted };
+}
+
+export interface MediaEditContext {
+  element: DomEditSelection;
+  projectId: string | null;
+  projectSrc: string;
+  commit: CommitDomAttributeBatch;
+}
+
+const isVideoSelection = (element: DomEditSelection) => element.tagName === "video";
+
+export async function commitMutedToggle(ctx: MediaEditContext, nextMuted: boolean): Promise<void> {
+  const isVideo = isVideoSelection(ctx.element);
+  const probedHasAudio =
+    isVideo && !nextMuted && ctx.projectId && ctx.projectSrc
+      ? await resolveAssetHasAudio(ctx.projectId, ctx.projectSrc)
+      : null;
+  await ctx.commit(ctx.element, mutedToggleOps({ isVideo, nextMuted, probedHasAudio }), {
+    label: "Edit muted",
+  });
+}
+
+export async function commitHasAudioToggle(ctx: MediaEditContext, next: boolean): Promise<void> {
+  await ctx.commit(ctx.element, hasAudioToggleOps(next), { label: "Edit has audio" });
+}
+
+export async function commitCutout(
+  ctx: MediaEditContext,
+  cutoutSrc: string,
+  keepSound: boolean,
+): Promise<string> {
+  const isVideo = isVideoSelection(ctx.element);
+  const label = "Remove background";
+  const keep = isVideo && keepSound;
+  if (!keep) {
+    const applied = await ctx.commit(ctx.element, cutoutOps({ isVideo, cutoutSrc }), { label });
+    if (!applied) throw new Error("Couldn't apply the cutout");
+    return "Applied cutout";
+  }
+  const { landed, soundKept } = await commitKeepSoundCutout(ctx, cutoutSrc, label);
+  if (!landed) throw new Error("Couldn't apply the cutout");
+  if (!soundKept) throw new Error(SOUND_NOT_KEPT_MESSAGE);
+  return SOUND_KEPT_ON_LINKED_AUDIO_STAGE;
+}
+
+async function commitKeepSoundCutout(
+  ctx: MediaEditContext,
+  cutoutSrc: string,
+  label: string,
+): Promise<{ landed: boolean; soundKept: boolean }> {
+  const edit = buildKeepSoundCutoutEdit({
+    video: ctx.element.element,
+    videoId: ctx.element.id ?? null,
+    target: buildDomEditPatchTarget(ctx.element),
+    cutoutSrc,
+  });
+  const landed = await ctx.commit(ctx.element, edit.ops, {
+    label,
+    prepareContent: edit.prepareContent,
+  });
+  return { landed, soundKept: landed && edit.audioInserted() };
+}

package/src/components/editor/propertyPanelAudioFxGroup.test.tsx

@@ -103,11 +103,11 @@ function mount(dataAttributes: Record<string, string>, alone = false, voices = 2
   return { host, onSetAttributeQuiet, onSetAttributeLive };
 }
 
-function mountGroup(memberStart: number) {
+function mountGroup(memberStart: number, memberTag: "audio" | "video" = "audio") {
   const bus = document.createElement("hf-audio-group");
   bus.id = "voiceover";
   document.body.append(bus);
-  const member = document.createElement("audio");
+  const member = document.createElement(memberTag);
   member.id = "vo-1";
   member.setAttribute("data-audio-group", "voiceover");
   member.setAttribute("data-start", String(memberStart));
@@ -649,6 +649,16 @@ describe("AudioFxGroup dynamic carve", () => {
       leaveShelf(host);
     });
 
+    it("seeks a group audition to an audible video member's span", () => {
+      act(() =>
+        usePlayerStore.setState({ isPlaying: false, currentTime: 2, requestedSeekTime: null }),
+      );
+      const host = mountGroup(7, "video");
+      hoverPreset(host);
+      expect(store().requestedSeekTime).toBe(7);
+      leaveShelf(host);
+    });
+
     it("leaves a transport the author started alone", () => {
       // Stopping their playback because they passed over a preset would be the
       // panel taking a decision nobody offered it.
@@ -1621,7 +1631,14 @@ describe("AudioFxGroup carve by default", () => {
 describe("AudioFxGroup carve source list", () => {
   /** Mount a bed alongside tracks named however the test needs. */
   const mountWith = (
-    tracks: { id: string; src?: string; start?: string; duration?: string }[],
+    tracks: {
+      id: string;
+      src?: string;
+      start?: string;
+      duration?: string;
+      tag?: "audio" | "video";
+      muted?: boolean;
+    }[],
     bedAttrs: Record<string, string> = {},
   ) => {
     const bed = document.createElement("audio");
@@ -1629,8 +1646,9 @@ describe("AudioFxGroup carve source list", () => {
     for (const [k, v] of Object.entries(bedAttrs)) bed.setAttribute(`data-${k}`, v);
     document.body.append(bed);
     for (const t of tracks) {
-      const el = document.createElement("audio");
+      const el = document.createElement(t.tag ?? "audio");
       el.id = t.id;
+      if (t.muted) el.setAttribute("muted", "");
       if (t.src) el.setAttribute("src", t.src);
       if (t.start !== undefined) el.setAttribute("data-start", t.start);
       if (t.duration !== undefined) el.setAttribute("data-duration", t.duration);
@@ -1660,6 +1678,16 @@ describe("AudioFxGroup carve source list", () => {
     return { host, options, boxes, onSetAttributeQuiet };
   };
 
+  it("offers an audible video as a voice and never a muted one", () => {
+    const { options } = mountWith([
+      { id: "a-roll", tag: "video" },
+      { id: "b-roll", tag: "video", muted: true },
+      { id: "narration" },
+    ]);
+    expect(options).toContain("a-roll");
+    expect(options).not.toContain("b-roll");
+  });
+
   it("reads the one voice out instead of offering a picker with one entry", () => {
     // A question with one answer is not a question. It is also the common case: a
     // narration and a bed, with a couple of stings that cannot be the voice.

package/src/components/editor/propertyPanelAudioFxGroup.tsx

@@ -54,6 +54,7 @@ import {
   resolveAudioGroups,
 } from "@hyperframes/core/audio-groups";
 import { useFxLevelling } from "./useFxLevelling.js";
+import { isCarveVoiceElement } from "./useFxCarveGrouping.js";
 
 function auditionSpan(startRaw: string | undefined, durationRaw: string | undefined) {
   const start = Number.parseFloat(startRaw ?? "");
@@ -70,8 +71,9 @@ function auditionSpansFor(element: DomEditSelection): AuditionSpan[] {
   if (element.tagName?.toLowerCase() !== HF_AUDIO_GROUP_TAG || !element.id) return [];
   const doc = element.element?.ownerDocument;
   if (!doc) return [];
-  return [...doc.querySelectorAll(`audio[${HF_AUDIO_GROUP_ATTR}]`)]
+  return [...doc.querySelectorAll(`[${HF_AUDIO_GROUP_ATTR}]`)]
     .filter((member) => member.getAttribute(HF_AUDIO_GROUP_ATTR) === element.id)
+    .filter(isCarveVoiceElement)
     .flatMap((member) => {
       const span = auditionSpan(
         member.getAttribute("data-start") ?? undefined,