@hyperframes/studio 0.8.107 → 0.8.109

package/src/components/editor/mediaAudioEdits.ts

@@ -12,7 +12,9 @@ import { HF_AUDIO_GROUP_ATTR } from "@hyperframes/core/audio-groups";
 import {
   escapeHtmlAttribute,
   findTagByTarget,
+  applyPatchByTarget,
   readAttributeByTarget,
+  unescapeHtmlAttribute,
   type PatchOperation,
   type PatchTarget,
 } from "../../utils/sourcePatcher";
@@ -21,10 +23,17 @@ import type { CommitDomAttributeBatch } from "../../hooks/domEditCommitTypes";
 import { buildDomEditPatchTarget, type DomEditSelection } from "./domEditing";
 import { generateId } from "../../utils/generateId";
 
-const MEDIA_LINK_ATTR = "data-link";
+import {
+  MEDIA_LINK_ATTR,
+  SYNC_ORIGIN_ATTR,
+  mintLinkId as mintLinkIdFrom,
+} from "@hyperframes/core/media-link";
 const SOUND_KEPT_ON_LINKED_AUDIO_STAGE = "Background removed. Sound kept on a linked audio track.";
 
-const MOVED_SOUND_ATTRS = [
+const SOUND_NOT_KEPT_MESSAGE =
+  "Background removed, but the sound could not be kept on a linked audio track. Undo to restore the original clip.";
+
+export const MOVED_SOUND_ATTRS = [
   "data-volume",
   HF_AUDIO_FADE_IN_ATTR,
   HF_AUDIO_FADE_OUT_ATTR,
@@ -33,7 +42,7 @@ const MOVED_SOUND_ATTRS = [
   HF_AUDIO_GROUP_ATTR,
 ];
 
-const COPIED_TIMING_ATTRS = [
+export const COPIED_TIMING_ATTRS = [
   "start",
   "end",
   "duration",
@@ -86,7 +95,7 @@ export function cutoutOps(input: { isVideo: boolean; cutoutSrc: string }): Patch
   return input.isVideo ? [src, htmlAttr("muted", "true"), dataAttr("has-audio", null)] : [src];
 }
 
-function firstFreeName(base: string, taken: Set<string>): string {
+export function firstFreeName(base: string, taken: Set<string>): string {
   if (!taken.has(base)) return base;
   let n = 2;
   while (taken.has(`${base}-${n}`)) n += 1;
@@ -95,12 +104,16 @@ function firstFreeName(base: string, taken: Set<string>): string {
 
 export function mintLinkId(doc: Document): string {
   const taken = new Set<string>();
-  for (const el of Array.from(doc.querySelectorAll(`[id], [${MEDIA_LINK_ATTR}]`))) {
-    if (el.id) taken.add(el.id);
-    const link = el.getAttribute(MEDIA_LINK_ATTR);
-    if (link) taken.add(link);
+  const selector = `[id], [${MEDIA_LINK_ATTR}], [${SYNC_ORIGIN_ATTR}]`;
+  for (const el of Array.from(doc.querySelectorAll(selector))) {
+    for (const value of [
+      el.id,
+      el.getAttribute(MEDIA_LINK_ATTR),
+      el.getAttribute(SYNC_ORIGIN_ATTR),
+    ])
+      if (value) taken.add(value);
   }
-  return firstFreeName("link", taken);
+  return mintLinkIdFrom(taken);
 }
 
 function firstFreeTrackIndex(source: string): number {
@@ -112,7 +125,7 @@ function firstFreeTrackIndex(source: string): number {
   return track;
 }
 
-function splitAutomation(raw: string): { videoKeeps: string | null; audioTakes: string } {
+export function splitAutomation(raw: string): { videoKeeps: string | null; audioTakes: string } {
   let lanes: HfAutomationLane[];
   try {
     lanes = parseAutomation(raw).lanes;
@@ -127,11 +140,20 @@ function splitAutomation(raw: string): { videoKeeps: string | null; audioTakes:
   };
 }
 
-function formatAttrs(attrs: Array<[string, string]>): string {
+export function formatAttrs(attrs: Array<[string, string]>): string {
   return attrs.map(([name, value]) => `${name}="${escapeHtmlAttribute(value)}"`).join(" ");
 }
 
-function insertBeforeTarget(source: string, target: PatchTarget, markup: string): string {
+export function readAuthoredSrc(source: string, target: PatchTarget): string {
+  const tag = findTagByTarget(source, target);
+  if (!tag) return "";
+  const own = /\bsrc=(["'])([^"']*)\1/.exec(tag.tag);
+  if (own?.[2]) return unescapeHtmlAttribute(own[2]);
+  const firstSource = /^\s*<source\b[^>]*?\bsrc=(["'])([^"']*)\1/i.exec(source.slice(tag.end + 1));
+  return firstSource?.[2] ? unescapeHtmlAttribute(firstSource[2]) : "";
+}
+
+export function insertBeforeTarget(source: string, target: PatchTarget, markup: string): string {
   const match = findTagByTarget(source, target);
   if (!match) return source;
   const lineStart = source.lastIndexOf("\n", match.start) + 1;
@@ -145,6 +167,7 @@ function insertBeforeTarget(source: string, target: PatchTarget, markup: string)
 export interface KeepSoundCutoutEdit {
   ops: PatchOperation[];
   prepareContent: (source: string) => string;
+  audioInserted: () => boolean;
 }
 
 export function buildKeepSoundCutoutEdit(input: {
@@ -155,10 +178,11 @@ export function buildKeepSoundCutoutEdit(input: {
 }): KeepSoundCutoutEdit {
   const { video, target } = input;
   const linkId = mintLinkId(video.ownerDocument);
-  const originalSrc = video.getAttribute("src") ?? "";
+  let audioInserted = false;
 
   const movedSound: Array<[string, string]> = [];
-  const ops: PatchOperation[] = [...cutoutOps({ isVideo: true, cutoutSrc: input.cutoutSrc })];
+  const cutoutSrcOp = htmlAttr("src", input.cutoutSrc);
+  const ops: PatchOperation[] = [htmlAttr("muted", "true"), dataAttr("has-audio", null)];
   for (const name of MOVED_SOUND_ATTRS) {
     const value = video.getAttribute(name);
     if (value === null) continue;
@@ -171,9 +195,13 @@ export function buildKeepSoundCutoutEdit(input: {
     movedSound.push([HF_AUDIO_AUTOMATION_ATTR, audioTakes]);
     ops.push(dataAttr(stripDataPrefix(HF_AUDIO_AUTOMATION_ATTR), videoKeeps));
   }
-  ops.push(dataAttr(stripDataPrefix(MEDIA_LINK_ATTR), linkId));
+  ops.push(
+    dataAttr(stripDataPrefix(MEDIA_LINK_ATTR), linkId),
+    dataAttr(stripDataPrefix(SYNC_ORIGIN_ATTR), linkId),
+  );
 
   const prepareContent = (source: string): string => {
+    const originalSrc = readAuthoredSrc(source, target);
     const audioId = firstFreeName(
       `${input.videoId || "video"}-audio`,
       new Set(collectHtmlIds(source)),
@@ -191,12 +219,18 @@ export function buildKeepSoundCutoutEdit(input: {
       ...timing,
       ["data-track-index", String(firstFreeTrackIndex(source))],
       [MEDIA_LINK_ATTR, linkId],
+      [SYNC_ORIGIN_ATTR, linkId],
       ...movedSound,
     ];
-    return insertBeforeTarget(source, target, `<audio ${formatAttrs(attrs)}></audio>`);
+    if (video.hasAttribute("loop")) attrs.push(["loop", ""]);
+    if (video.hasAttribute("data-hidden")) attrs.push(["data-hidden", ""]);
+    const cut = applyPatchByTarget(source, target, cutoutSrcOp);
+    const inserted = insertBeforeTarget(cut, target, `<audio ${formatAttrs(attrs)}></audio>`);
+    audioInserted = inserted !== cut;
+    return inserted;
   };
 
-  return { ops, prepareContent };
+  return { ops, prepareContent, audioInserted: () => audioInserted };
 }
 
 export interface MediaEditContext {
@@ -231,23 +265,31 @@ export async function commitCutout(
   const isVideo = isVideoSelection(ctx.element);
   const label = "Remove background";
   const keep = isVideo && keepSound;
-  const landed = keep
-    ? await commitKeepSoundCutout(ctx, cutoutSrc, label)
-    : await ctx.commit(ctx.element, cutoutOps({ isVideo, cutoutSrc }), { label });
+  if (!keep) {
+    const applied = await ctx.commit(ctx.element, cutoutOps({ isVideo, cutoutSrc }), { label });
+    if (!applied) throw new Error("Couldn't apply the cutout");
+    return "Applied cutout";
+  }
+  const { landed, soundKept } = await commitKeepSoundCutout(ctx, cutoutSrc, label);
   if (!landed) throw new Error("Couldn't apply the cutout");
-  return keep ? SOUND_KEPT_ON_LINKED_AUDIO_STAGE : "Applied cutout";
+  if (!soundKept) throw new Error(SOUND_NOT_KEPT_MESSAGE);
+  return SOUND_KEPT_ON_LINKED_AUDIO_STAGE;
 }
 
-function commitKeepSoundCutout(
+async function commitKeepSoundCutout(
   ctx: MediaEditContext,
   cutoutSrc: string,
   label: string,
-): Promise<boolean> {
+): Promise<{ landed: boolean; soundKept: boolean }> {
   const edit = buildKeepSoundCutoutEdit({
     video: ctx.element.element,
     videoId: ctx.element.id ?? null,
     target: buildDomEditPatchTarget(ctx.element),
     cutoutSrc,
   });
-  return ctx.commit(ctx.element, edit.ops, { label, prepareContent: edit.prepareContent });
+  const landed = await ctx.commit(ctx.element, edit.ops, {
+    label,
+    prepareContent: edit.prepareContent,
+  });
+  return { landed, soundKept: landed && edit.audioInserted() };
 }

package/src/components/editor/mediaLinkEdits.test.ts

@@ -0,0 +1,321 @@
+// @vitest-environment happy-dom
+import { describe, expect, it } from "vitest";
+import {
+  canDetachAudio,
+  canLinkPair,
+  detachAudioInSource,
+  linkInSource,
+  findMergePair,
+  mergeAudioInSource,
+  pickDetachedAudioTrack,
+  removeElementInSource,
+  setLinkInSource,
+} from "./mediaLinkEdits";
+
+const automation = JSON.stringify({
+  version: 1,
+  lanes: [
+    { target: "volume", points: [{ t: 0, v: 1 }] },
+    { target: "rate", points: [{ t: 0, v: 2 }] },
+  ],
+}).replaceAll('"', """);
+
+const VIDEO_TAG =
+  '<video id="talk" class="clip" src="assets/talk.mp4" data-start="2" data-duration="6" data-media-start="1" data-playback-rate="1" data-track-index="0" data-has-audio="true" data-volume="0.5" data-fade-in="0.3" data-fade-out="0.2" data-fx-chain="eq" data-fx-carve="carve" data-audio-group="voiceover" data-automation="' +
+  automation +
+  '"></video>';
+
+const SOURCE = [
+  '<div data-composition-id="main" data-duration="10">',
+  `  ${VIDEO_TAG}`,
+  '  <audio id="music" class="clip" src="assets/bgm.mp3" data-start="0" data-duration="10" data-track-index="1"></audio>',
+  "</div>",
+].join("\n");
+
+const video = { id: "talk" };
+
+function attrsOf(source: string, selector: string): Record<string, string> {
+  const el = new DOMParser().parseFromString(source, "text/html").querySelector(selector);
+  if (!el) throw new Error(`missing ${selector}`);
+  return Object.fromEntries(Array.from(el.attributes, (a) => [a.name, a.value]));
+}
+
+describe("detachAudioInSource", () => {
+  const result = detachAudioInSource(SOURCE, { target: video, videoId: "talk", track: 2 });
+
+  it("inserts a linked audio with the video's file, window and sound", () => {
+    expect(result).not.toBeNull();
+    const audio = attrsOf(result?.html ?? "", "audio#talk-audio");
+    expect(audio).toMatchObject({
+      src: "assets/talk.mp4",
+      "data-start": "2",
+      "data-duration": "6",
+      "data-media-start": "1",
+      "data-playback-rate": "1",
+      "data-track-index": "2",
+      "data-link": result?.linkId,
+      "data-sync-origin": result?.linkId,
+      "data-volume": "0.5",
+      "data-fade-in": "0.3",
+      "data-fade-out": "0.2",
+      "data-fx-chain": "eq",
+      "data-fx-carve": "carve",
+      "data-audio-group": "voiceover",
+    });
+    expect(audio["data-automation"]).toContain("volume");
+  });
+
+  it("mutes the video, strips its sound attributes and links it", () => {
+    const after = attrsOf(result?.html ?? "", "video#talk");
+    expect(after).toHaveProperty("muted");
+    expect(after["data-link"]).toBe(result?.linkId);
+    expect(after["data-sync-origin"]).toBe(result?.linkId);
+    for (const gone of [
+      "data-has-audio",
+      "data-volume",
+      "data-fade-in",
+      "data-fade-out",
+      "data-fx-chain",
+      "data-fx-carve",
+      "data-audio-group",
+    ]) {
+      expect(after).not.toHaveProperty(gone);
+    }
+    expect(after["data-automation"]).toContain("rate");
+    expect(after["data-automation"]).not.toContain("volume");
+  });
+
+  it("suffixes the audio id when taken", () => {
+    const taken = SOURCE.replace('id="music"', 'id="talk-audio"');
+    expect(detachAudioInSource(taken, { target: video, videoId: "talk", track: 2 })?.audioId).toBe(
+      "talk-audio-2",
+    );
+  });
+});
+
+describe("mergeAudioInSource", () => {
+  it("round-trips detach back to the original attributes", () => {
+    const detached = detachAudioInSource(SOURCE, { target: video, videoId: "talk", track: 2 });
+    const merged = mergeAudioInSource(detached?.html ?? "", {
+      videoTarget: video,
+      audioTarget: { id: "talk-audio" },
+    });
+    const original = attrsOf(SOURCE, "video#talk");
+    const roundTrip = attrsOf(merged ?? "", "video#talk");
+    const lanes = (raw: string | undefined) =>
+      JSON.parse(raw ?? "{}").lanes.map((lane: { target: string }) => lane.target);
+    expect(lanes(roundTrip["data-automation"]).sort()).toEqual(
+      lanes(original["data-automation"]).sort(),
+    );
+    delete original["data-automation"];
+    delete roundTrip["data-automation"];
+    expect(roundTrip).toEqual(original);
+    expect(merged).not.toContain('<audio id="talk-audio"');
+    expect(merged).toContain('id="music"');
+  });
+
+  it("converts a legacy a-roll + a-roll-audio pair into one video", () => {
+    const legacy = [
+      "<div>",
+      '  <video id="a-roll" src="a.mp4" muted data-start="0" data-duration="4"></video>',
+      '  <audio id="a-roll-audio" src="a.mp4" data-start="0" data-duration="4" data-volume="0.8"></audio>',
+      "</div>",
+    ].join("\n");
+    const merged = mergeAudioInSource(legacy, {
+      videoTarget: { id: "a-roll" },
+      audioTarget: { id: "a-roll-audio" },
+    });
+    expect(merged).toBe(
+      [
+        "<div>",
+        '  <video id="a-roll" src="a.mp4" data-start="0" data-duration="4" data-volume="0.8" data-has-audio="true"></video>',
+        "</div>",
+      ].join("\n"),
+    );
+  });
+});
+
+describe("mergeAudioInSource refusals", () => {
+  const pair = (videoAttrs: string, audioAttrs: string) =>
+    [
+      "<div>",
+      `  <video id="v" src="assets/one/talk.mp4" muted data-start="0" data-duration="4"${videoAttrs}></video>`,
+      `  <audio id="a" src="assets/one/talk.mp4" data-start="0" data-duration="4"${audioAttrs}></audio>`,
+      "</div>",
+    ].join("\n");
+  const merge = (source: string) =>
+    mergeAudioInSource(source, { videoTarget: { id: "v" }, audioTarget: { id: "a" } });
+
+  it("refuses an audio of a different file with the same name", () => {
+    expect(
+      merge(
+        pair("", "").replace('<audio id="a" src="assets/one/', '<audio id="a" src="assets/two/'),
+      ),
+    ).toBeNull();
+  });
+
+  it("refuses when only one member is hidden, which would change what is heard", () => {
+    expect(merge(pair("", " data-hidden"))).toBeNull();
+    expect(merge(pair(" data-hidden", ""))).toBeNull();
+    expect(merge(pair(" data-hidden", " data-hidden"))).not.toBeNull();
+  });
+});
+
+describe("detachAudioInSource on a hidden video", () => {
+  it("keeps the detached sound silent", () => {
+    const hidden = SOURCE.replace('<video id="talk"', '<video id="talk" data-hidden');
+    const result = detachAudioInSource(hidden, { target: video, videoId: "talk", track: 2 });
+    expect(attrsOf(result?.html ?? "", "audio#talk-audio")).toHaveProperty("data-hidden");
+    expect(attrsOf(result?.html ?? "", "video#talk")).toHaveProperty("data-hidden");
+  });
+});
+
+describe("removeElementInSource", () => {
+  it("removes the element and its whole line", () => {
+    expect(removeElementInSource('<div>\n  <audio id="a"></audio>\n</div>', { id: "a" })).toBe(
+      "<div>\n</div>",
+    );
+  });
+});
+
+describe("linkInSource", () => {
+  it("mints an id no element or link already uses", () => {
+    const src = '<video id="lk-1" data-link="lk-2"></video><audio id="a"></audio>';
+    expect(linkInSource(src, [{ id: "a" }])).toContain(
+      '<audio id="a" data-link="lk-3" data-sync-origin="lk-3">',
+    );
+  });
+
+  it("skips ids already used as a sync origin", () => {
+    const src = '<video id="v" data-sync-origin="lk-1"></video><audio id="a"></audio>';
+    expect(linkInSource(src, [{ id: "a" }])).toContain('data-link="lk-2"');
+  });
+});
+
+describe("setLinkInSource", () => {
+  it("writes and removes data-link on every target", () => {
+    const src = '<video id="v"></video><audio id="a"></audio>';
+    const linked = setLinkInSource(src, [{ id: "v" }, { id: "a" }], "lk-1");
+    expect(linked).toBe(
+      '<video id="v" data-link="lk-1"></video><audio id="a" data-link="lk-1"></audio>',
+    );
+    expect(setLinkInSource(linked, [{ id: "v" }, { id: "a" }], null)).toBe(src);
+  });
+
+  it("keeps the sync origin when unlinking", () => {
+    const linked = linkInSource('<video id="v"></video><audio id="a"></audio>', [
+      { id: "v" },
+      { id: "a" },
+    ]);
+    const unlinked = setLinkInSource(linked, [{ id: "v" }, { id: "a" }], null);
+    expect(unlinked).toBe(
+      '<video id="v" data-sync-origin="lk-1"></video><audio id="a" data-sync-origin="lk-1"></audio>',
+    );
+  });
+});
+
+const el = (id: string, tag: string, extra: Record<string, unknown> = {}) => ({
+  id,
+  tag,
+  src: "assets/talk.mp4",
+  start: 2,
+  duration: 6,
+  track: 0,
+  ...extra,
+});
+
+describe("predicates", () => {
+  it("detaches only an unmuted video with declared audio", () => {
+    expect(canDetachAudio(el("v", "video", { hasAudio: true }))).toBe(true);
+    expect(canDetachAudio(el("v", "video", { hasAudio: true, muted: true }))).toBe(false);
+    expect(canDetachAudio(el("v", "video"))).toBe(false);
+    expect(canDetachAudio(el("a", "audio", { hasAudio: true }))).toBe(false);
+  });
+
+  it("finds the merge pair from either member, linked or identically timed", () => {
+    const v = el("v", "video", { muted: true });
+    const a = el("a", "audio");
+    expect(findMergePair(v, [v, a])).toEqual({ video: v, audio: a });
+    expect(findMergePair(a, [v, a])).toEqual({ video: v, audio: a });
+    const drifted = el("a", "audio", { start: 3 });
+    expect(findMergePair(v, [v, drifted])).toBeNull();
+    const linked = [
+      el("v", "video", { muted: true, link: "lk-1" }),
+      el("a", "audio", { start: 3, link: "lk-1" }),
+    ];
+    expect(findMergePair(linked[0] ?? v, linked)?.audio.id).toBe("a");
+    expect(findMergePair(el("v", "video"), [el("v", "video"), a])).toBeNull();
+    expect(findMergePair(v, [v, el("a", "audio", { src: "other.mp4" })])).toBeNull();
+  });
+
+  it("links exactly one video and one audio of the same file and timing", () => {
+    expect(canLinkPair([el("v", "video"), el("a", "audio")])).toBe(true);
+    expect(canLinkPair([el("v", "video"), el("a", "audio", { start: 3 })])).toBe(false);
+    expect(canLinkPair([el("v", "video"), el("w", "video")])).toBe(false);
+    expect(canLinkPair([el("v", "video", { link: "x" }), el("a", "audio", { link: "x" })])).toBe(
+      false,
+    );
+  });
+});
+
+describe("predicates compare the whole asset path", () => {
+  const one = { src: "assets/one/talk.mp4" };
+  const two = { src: "assets/two/talk.mp4" };
+
+  it("does not pair same-named files in different folders", () => {
+    const v = el("v", "video", { muted: true, ...one });
+    const a = el("a", "audio", two);
+    expect(findMergePair(v, [v, a])).toBeNull();
+    expect(canLinkPair([el("v", "video", one), el("a", "audio", two)])).toBe(false);
+  });
+
+  it("resolves each src against its own source file, and links only inside one", () => {
+    const v = el("v", "video", { src: "../assets/talk.mp4", sourceFile: "scenes/a.html" });
+    const a = el("a", "audio", { src: "./assets/talk.mp4", sourceFile: "index.html" });
+    expect(canLinkPair([v, a])).toBe(false);
+    expect(
+      canLinkPair([v, { ...a, src: "../assets/./talk.mp4", sourceFile: "scenes/a.html" }]),
+    ).toBe(true);
+  });
+
+  it("does not take a same-id link from another source file as the merge partner", () => {
+    const v = el("v", "video", { muted: true, link: "lk-1", sourceFile: "index.html" });
+    const a = el("a", "audio", { start: 3, link: "lk-1", sourceFile: "child.html" });
+    expect(findMergePair(v, [v, a])).toBeNull();
+  });
+
+  it("does not link or merge a pair split across inline compositions", () => {
+    const v = el("v", "video", { muted: true, compositionScope: "main" });
+    const a = el("a", "audio", { compositionScope: "child" });
+    expect(canLinkPair([v, a])).toBe(false);
+    expect(findMergePair(v, [v, a])).toBeNull();
+  });
+
+  it("does not merge a hidden audio back into a visible video", () => {
+    const v = el("v", "video", { muted: true });
+    expect(findMergePair(v, [v, el("a", "audio", { hidden: true })])).toBeNull();
+  });
+});
+
+describe("pickDetachedAudioTrack", () => {
+  it("uses the first free audio track below the visual tracks", () => {
+    const elements = [el("v", "video", { track: 0 }), el("t", "div", { track: 1 })];
+    expect(pickDetachedAudioTrack(elements, { start: 2, duration: 6 })).toBe(2);
+  });
+
+  it("reuses an audio track that is free in the clip's window", () => {
+    const elements = [
+      el("v", "video", { track: 0 }),
+      el("m", "audio", { track: 1, start: 0, duration: 2 }),
+    ];
+    expect(pickDetachedAudioTrack(elements, { start: 2, duration: 6 })).toBe(1);
+  });
+
+  it("opens a new bottom track when every audio track is busy", () => {
+    const elements = [
+      el("v", "video", { track: 0 }),
+      el("m", "audio", { track: 1, start: 0, duration: 10 }),
+    ];
+    expect(pickDetachedAudioTrack(elements, { start: 2, duration: 6 })).toBe(2);
+  });
+});