@hyperframes/studio 0.8.105 → 0.8.107

package/src/components/editor/mediaAudioEdits.test.ts

@@ -0,0 +1,212 @@
+// @vitest-environment happy-dom
+import { describe, expect, it } from "vitest";
+import { applyPatchByTarget } from "../../utils/sourcePatcher";
+import {
+  buildKeepSoundCutoutEdit,
+  cutoutOps,
+  hasAudioToggleOps,
+  mintLinkId,
+  mutedToggleOps,
+} from "./mediaAudioEdits";
+
+const attr = (property: string, value: string | null) => ({
+  type: "attribute" as const,
+  property,
+  value,
+});
+const html = (property: string, value: string | null) => ({
+  type: "html-attribute" as const,
+  property,
+  value,
+});
+
+describe("mutedToggleOps", () => {
+  it("mutes a video and removes has-audio instead of writing an empty marker", () => {
+    expect(mutedToggleOps({ isVideo: true, nextMuted: true, probedHasAudio: null })).toEqual([
+      html("muted", "true"),
+      attr("has-audio", null),
+    ]);
+  });
+
+  it("unmutes a video the probe hears as audible", () => {
+    expect(mutedToggleOps({ isVideo: true, nextMuted: false, probedHasAudio: true })).toEqual([
+      html("muted", null),
+      attr("has-audio", "true"),
+    ]);
+  });
+
+  it("unmutes a silent-probe video with an explicit has-audio=false", () => {
+    expect(mutedToggleOps({ isVideo: true, nextMuted: false, probedHasAudio: false })).toEqual([
+      html("muted", null),
+      attr("has-audio", "false"),
+    ]);
+  });
+
+  it("leaves has-audio absent when the probe fails", () => {
+    expect(mutedToggleOps({ isVideo: true, nextMuted: false, probedHasAudio: null })).toEqual([
+      html("muted", null),
+      attr("has-audio", null),
+    ]);
+  });
+
+  it("toggles only muted on an audio element", () => {
+    expect(mutedToggleOps({ isVideo: false, nextMuted: true, probedHasAudio: null })).toEqual([
+      html("muted", "true"),
+    ]);
+    expect(mutedToggleOps({ isVideo: false, nextMuted: false, probedHasAudio: null })).toEqual([
+      html("muted", null),
+    ]);
+  });
+});
+
+describe("hasAudioToggleOps", () => {
+  it("turns sound on by stamping has-audio and clearing muted", () => {
+    expect(hasAudioToggleOps(true)).toEqual([attr("has-audio", "true"), html("muted", null)]);
+  });
+
+  it("turns sound off by removing has-audio rather than writing an empty value", () => {
+    expect(hasAudioToggleOps(false)).toEqual([attr("has-audio", null), html("muted", "true")]);
+  });
+});
+
+describe("cutoutOps", () => {
+  it("swaps the src of an image and nothing else", () => {
+    expect(cutoutOps({ isVideo: false, cutoutSrc: "assets/cut.png" })).toEqual([
+      html("src", "assets/cut.png"),
+    ]);
+  });
+
+  it("swaps a silent video's src and keeps it muted without a has-audio marker", () => {
+    expect(cutoutOps({ isVideo: true, cutoutSrc: "assets/cut.webm" })).toEqual([
+      html("src", "assets/cut.webm"),
+      html("muted", "true"),
+      attr("has-audio", null),
+    ]);
+  });
+});
+
+describe("mintLinkId", () => {
+  it("dedupes against element ids and existing links", () => {
+    const doc = document.implementation.createHTMLDocument("t");
+    expect(mintLinkId(doc)).toBe("link");
+    doc.body.innerHTML = '<div id="link"></div><video data-link="link-2"></video>';
+    expect(mintLinkId(doc)).toBe("link-3");
+  });
+});
+
+function makeVideo(attrs: Record<string, string>): HTMLVideoElement {
+  const doc = document.implementation.createHTMLDocument("t");
+  const video = doc.createElement("video");
+  for (const [name, value] of Object.entries(attrs)) video.setAttribute(name, value);
+  doc.body.append(video);
+  return video;
+}
+
+function applyEdit(source: string, edit: ReturnType<typeof buildKeepSoundCutoutEdit>): string {
+  const target = { id: "clip" };
+  const patched = edit.ops.reduce((acc, op) => applyPatchByTarget(acc, target, op), source);
+  return edit.prepareContent(patched);
+}
+
+function parse(source: string): Document {
+  return new DOMParser().parseFromString(source, "text/html");
+}
+
+describe("buildKeepSoundCutoutEdit", () => {
+  const automation = JSON.stringify({
+    version: 1,
+    lanes: [
+      { target: "volume", points: [{ t: 0, v: 1 }] },
+      { target: "rate", points: [{ t: 0, v: 2 }] },
+    ],
+  });
+
+  const source = [
+    '<div data-composition-id="main" data-duration="10">',
+    '  <video id="clip" class="clip" src="assets/talk.mp4" data-start="1" data-duration="4" data-media-start="2" data-track-index="0" data-has-audio="true" data-volume="0.5" data-fade-in="0.3" data-audio-group="voiceover"></video>',
+    '  <audio id="music" class="clip" src="assets/bgm.mp3" data-start="0" data-duration="10" data-track-index="1"></audio>',
+    "</div>",
+  ].join("\n");
+
+  const liveVideo = () =>
+    makeVideo({
+      id: "clip",
+      src: "assets/talk.mp4",
+      "data-has-audio": "true",
+      "data-volume": "0.5",
+      "data-fade-in": "0.3",
+      "data-audio-group": "voiceover",
+      "data-automation": automation,
+    });
+
+  it("mutes the video onto the cutout and moves its sound to a linked sibling audio", () => {
+    const edit = buildKeepSoundCutoutEdit({
+      video: liveVideo(),
+      videoId: "clip",
+      target: { id: "clip" },
+      cutoutSrc: "assets/talk-cutout.webm",
+    });
+    const doc = parse(applyEdit(source, edit));
+    const video = doc.getElementById("clip");
+    const audio = doc.getElementById("clip-audio");
+    if (!video || !audio) throw new Error("expected both clips");
+
+    expect(video.getAttribute("src")).toBe("assets/talk-cutout.webm");
+    expect(video.hasAttribute("muted")).toBe(true);
+    expect(video.hasAttribute("data-has-audio")).toBe(false);
+    expect(video.hasAttribute("data-volume")).toBe(false);
+    expect(video.hasAttribute("data-fade-in")).toBe(false);
+    expect(video.hasAttribute("data-audio-group")).toBe(false);
+    expect(JSON.parse(video.getAttribute("data-automation") ?? "").lanes).toEqual([
+      { target: "rate", points: [{ t: 0, v: 2 }] },
+    ]);
+
+    expect(audio.tagName).toBe("AUDIO");
+    expect(audio.getAttribute("src")).toBe("assets/talk.mp4");
+    expect(audio.getAttribute("data-start")).toBe("1");
+    expect(audio.getAttribute("data-duration")).toBe("4");
+    expect(audio.getAttribute("data-media-start")).toBe("2");
+    expect(audio.hasAttribute("data-playback-rate")).toBe(false);
+    expect(audio.getAttribute("data-track-index")).toBe("2");
+    expect(audio.getAttribute("data-volume")).toBe("0.5");
+    expect(audio.getAttribute("data-fade-in")).toBe("0.3");
+    expect(audio.getAttribute("data-audio-group")).toBe("voiceover");
+    expect(JSON.parse(audio.getAttribute("data-automation") ?? "").lanes).toEqual([
+      { target: "volume", points: [{ t: 0, v: 1 }] },
+      { target: "rate", points: [{ t: 0, v: 2 }] },
+    ]);
+    expect(audio.getAttribute("data-link")).toBe(video.getAttribute("data-link"));
+    expect(video.getAttribute("data-link")).toBe("link");
+    expect(audio.parentElement).toBe(video.parentElement);
+  });
+
+  it("drops the video's automation attribute when it held no rate lane", () => {
+    const video = makeVideo({
+      id: "clip",
+      src: "assets/talk.mp4",
+      "data-automation": JSON.stringify({
+        version: 1,
+        lanes: [{ target: "volume", points: [{ t: 0, v: 1 }] }],
+      }),
+    });
+    const edit = buildKeepSoundCutoutEdit({
+      video,
+      videoId: "clip",
+      target: { id: "clip" },
+      cutoutSrc: "assets/cut.webm",
+    });
+    expect(edit.ops).toContainEqual(attr("automation", null));
+  });
+
+  it("suffixes the audio id when <video-id>-audio is taken", () => {
+    const edit = buildKeepSoundCutoutEdit({
+      video: liveVideo(),
+      videoId: "clip",
+      target: { id: "clip" },
+      cutoutSrc: "assets/cut.webm",
+    });
+    const taken = source.replace('id="music"', 'id="clip-audio"');
+    const doc = parse(applyEdit(taken, edit));
+    expect(doc.getElementById("clip-audio-2")?.getAttribute("src")).toBe("assets/talk.mp4");
+  });
+});

package/src/components/editor/mediaAudioEdits.ts

@@ -0,0 +1,253 @@
+import {
+  HF_AUDIO_AUTOMATION_ATTR,
+  RATE_TARGET,
+  parseAutomation,
+  serializeAutomation,
+  type HfAutomationLane,
+} from "@hyperframes/core/audio-automation";
+import { HF_AUDIO_CARVE_ATTR } from "@hyperframes/core/audio-carve";
+import { HF_AUDIO_FADE_IN_ATTR, HF_AUDIO_FADE_OUT_ATTR } from "@hyperframes/core/audio-fade";
+import { HF_AUDIO_FX_ATTR } from "@hyperframes/core/audio-fx";
+import { HF_AUDIO_GROUP_ATTR } from "@hyperframes/core/audio-groups";
+import {
+  escapeHtmlAttribute,
+  findTagByTarget,
+  readAttributeByTarget,
+  type PatchOperation,
+  type PatchTarget,
+} from "../../utils/sourcePatcher";
+import { collectHtmlIds, resolveAssetHasAudio } from "../../utils/studioHelpers";
+import type { CommitDomAttributeBatch } from "../../hooks/domEditCommitTypes";
+import { buildDomEditPatchTarget, type DomEditSelection } from "./domEditing";
+import { generateId } from "../../utils/generateId";
+
+const MEDIA_LINK_ATTR = "data-link";
+const SOUND_KEPT_ON_LINKED_AUDIO_STAGE = "Background removed. Sound kept on a linked audio track.";
+
+const MOVED_SOUND_ATTRS = [
+  "data-volume",
+  HF_AUDIO_FADE_IN_ATTR,
+  HF_AUDIO_FADE_OUT_ATTR,
+  HF_AUDIO_FX_ATTR,
+  HF_AUDIO_CARVE_ATTR,
+  HF_AUDIO_GROUP_ATTR,
+];
+
+const COPIED_TIMING_ATTRS = [
+  "start",
+  "end",
+  "duration",
+  "media-start",
+  "playback-start",
+  "playback-rate",
+  "source-duration",
+];
+
+const htmlAttr = (property: string, value: string | null): PatchOperation => ({
+  type: "html-attribute",
+  property,
+  value,
+});
+
+const dataAttr = (property: string, value: string | null): PatchOperation => ({
+  type: "attribute",
+  property,
+  value,
+});
+
+function stripDataPrefix(name: string): string {
+  return name.startsWith("data-") ? name.slice("data-".length) : name;
+}
+
+function probedHasAudioValue(probedHasAudio: boolean | null): string | null {
+  if (probedHasAudio === null) return null;
+  return probedHasAudio ? "true" : "false";
+}
+
+export function mutedToggleOps(input: {
+  isVideo: boolean;
+  nextMuted: boolean;
+  probedHasAudio: boolean | null;
+}): PatchOperation[] {
+  const muted = htmlAttr("muted", input.nextMuted ? "true" : null);
+  if (!input.isVideo) return [muted];
+  const hasAudio = input.nextMuted ? null : probedHasAudioValue(input.probedHasAudio);
+  return [muted, dataAttr("has-audio", hasAudio)];
+}
+
+export function hasAudioToggleOps(next: boolean): PatchOperation[] {
+  return next
+    ? [dataAttr("has-audio", "true"), htmlAttr("muted", null)]
+    : [dataAttr("has-audio", null), htmlAttr("muted", "true")];
+}
+
+export function cutoutOps(input: { isVideo: boolean; cutoutSrc: string }): PatchOperation[] {
+  const src = htmlAttr("src", input.cutoutSrc);
+  return input.isVideo ? [src, htmlAttr("muted", "true"), dataAttr("has-audio", null)] : [src];
+}
+
+function firstFreeName(base: string, taken: Set<string>): string {
+  if (!taken.has(base)) return base;
+  let n = 2;
+  while (taken.has(`${base}-${n}`)) n += 1;
+  return `${base}-${n}`;
+}
+
+export function mintLinkId(doc: Document): string {
+  const taken = new Set<string>();
+  for (const el of Array.from(doc.querySelectorAll(`[id], [${MEDIA_LINK_ATTR}]`))) {
+    if (el.id) taken.add(el.id);
+    const link = el.getAttribute(MEDIA_LINK_ATTR);
+    if (link) taken.add(link);
+  }
+  return firstFreeName("link", taken);
+}
+
+function firstFreeTrackIndex(source: string): number {
+  const used = new Set(
+    Array.from(source.matchAll(/\bdata-track-index=["'](-?\d+)["']/g), (m) => Number(m[1])),
+  );
+  let track = 0;
+  while (used.has(track)) track += 1;
+  return track;
+}
+
+function splitAutomation(raw: string): { videoKeeps: string | null; audioTakes: string } {
+  let lanes: HfAutomationLane[];
+  try {
+    lanes = parseAutomation(raw).lanes;
+  } catch {
+    return { videoKeeps: null, audioTakes: raw };
+  }
+  const rateLanes = lanes.filter((lane) => lane.target === RATE_TARGET);
+  const soundLanes = lanes.filter((lane) => lane.target !== RATE_TARGET);
+  return {
+    videoKeeps: rateLanes.length > 0 ? serializeAutomation({ version: 1, lanes: rateLanes }) : null,
+    audioTakes: serializeAutomation({ version: 1, lanes: [...soundLanes, ...rateLanes] }),
+  };
+}
+
+function formatAttrs(attrs: Array<[string, string]>): string {
+  return attrs.map(([name, value]) => `${name}="${escapeHtmlAttribute(value)}"`).join(" ");
+}
+
+function insertBeforeTarget(source: string, target: PatchTarget, markup: string): string {
+  const match = findTagByTarget(source, target);
+  if (!match) return source;
+  const lineStart = source.lastIndexOf("\n", match.start) + 1;
+  const indent = /^[ \t]*$/.test(source.slice(lineStart, match.start))
+    ? source.slice(lineStart, match.start)
+    : "";
+  const separator = indent ? `\n${indent}` : "";
+  return `${source.slice(0, match.start)}${markup}${separator}${source.slice(match.start)}`;
+}
+
+export interface KeepSoundCutoutEdit {
+  ops: PatchOperation[];
+  prepareContent: (source: string) => string;
+}
+
+export function buildKeepSoundCutoutEdit(input: {
+  video: Element;
+  videoId: string | null;
+  target: PatchTarget;
+  cutoutSrc: string;
+}): KeepSoundCutoutEdit {
+  const { video, target } = input;
+  const linkId = mintLinkId(video.ownerDocument);
+  const originalSrc = video.getAttribute("src") ?? "";
+
+  const movedSound: Array<[string, string]> = [];
+  const ops: PatchOperation[] = [...cutoutOps({ isVideo: true, cutoutSrc: input.cutoutSrc })];
+  for (const name of MOVED_SOUND_ATTRS) {
+    const value = video.getAttribute(name);
+    if (value === null) continue;
+    movedSound.push([name, value]);
+    ops.push(dataAttr(stripDataPrefix(name), null));
+  }
+  const automation = video.getAttribute(HF_AUDIO_AUTOMATION_ATTR);
+  if (automation !== null) {
+    const { videoKeeps, audioTakes } = splitAutomation(automation);
+    movedSound.push([HF_AUDIO_AUTOMATION_ATTR, audioTakes]);
+    ops.push(dataAttr(stripDataPrefix(HF_AUDIO_AUTOMATION_ATTR), videoKeeps));
+  }
+  ops.push(dataAttr(stripDataPrefix(MEDIA_LINK_ATTR), linkId));
+
+  const prepareContent = (source: string): string => {
+    const audioId = firstFreeName(
+      `${input.videoId || "video"}-audio`,
+      new Set(collectHtmlIds(source)),
+    );
+    const timing: Array<[string, string]> = [];
+    for (const name of COPIED_TIMING_ATTRS) {
+      const value = readAttributeByTarget(source, target, name);
+      if (value !== undefined) timing.push([`data-${name}`, value]);
+    }
+    const attrs: Array<[string, string]> = [
+      ["id", audioId],
+      ["data-hf-id", `hf-${generateId()}`],
+      ["class", "clip"],
+      ["src", originalSrc],
+      ...timing,
+      ["data-track-index", String(firstFreeTrackIndex(source))],
+      [MEDIA_LINK_ATTR, linkId],
+      ...movedSound,
+    ];
+    return insertBeforeTarget(source, target, `<audio ${formatAttrs(attrs)}></audio>`);
+  };
+
+  return { ops, prepareContent };
+}
+
+export interface MediaEditContext {
+  element: DomEditSelection;
+  projectId: string | null;
+  projectSrc: string;
+  commit: CommitDomAttributeBatch;
+}
+
+const isVideoSelection = (element: DomEditSelection) => element.tagName === "video";
+
+export async function commitMutedToggle(ctx: MediaEditContext, nextMuted: boolean): Promise<void> {
+  const isVideo = isVideoSelection(ctx.element);
+  const probedHasAudio =
+    isVideo && !nextMuted && ctx.projectId && ctx.projectSrc
+      ? await resolveAssetHasAudio(ctx.projectId, ctx.projectSrc)
+      : null;
+  await ctx.commit(ctx.element, mutedToggleOps({ isVideo, nextMuted, probedHasAudio }), {
+    label: "Edit muted",
+  });
+}
+
+export async function commitHasAudioToggle(ctx: MediaEditContext, next: boolean): Promise<void> {
+  await ctx.commit(ctx.element, hasAudioToggleOps(next), { label: "Edit has audio" });
+}
+
+export async function commitCutout(
+  ctx: MediaEditContext,
+  cutoutSrc: string,
+  keepSound: boolean,
+): Promise<string> {
+  const isVideo = isVideoSelection(ctx.element);
+  const label = "Remove background";
+  const keep = isVideo && keepSound;
+  const landed = keep
+    ? await commitKeepSoundCutout(ctx, cutoutSrc, label)
+    : await ctx.commit(ctx.element, cutoutOps({ isVideo, cutoutSrc }), { label });
+  if (!landed) throw new Error("Couldn't apply the cutout");
+  return keep ? SOUND_KEPT_ON_LINKED_AUDIO_STAGE : "Applied cutout";
+}
+
+function commitKeepSoundCutout(
+  ctx: MediaEditContext,
+  cutoutSrc: string,
+  label: string,
+): Promise<boolean> {
+  const edit = buildKeepSoundCutoutEdit({
+    video: ctx.element.element,
+    videoId: ctx.element.id ?? null,
+    target: buildDomEditPatchTarget(ctx.element),
+    cutoutSrc,
+  });
+  return ctx.commit(ctx.element, edit.ops, { label, prepareContent: edit.prepareContent });
+}

package/src/components/editor/overlayFrameLoop.test.ts

@@ -3,8 +3,10 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import {
   IDLE_POLL_MS,
+  IDLE_SLICE_MS,
   requestOverlayFrames,
   resetOverlayFrameLoopForTests,
+  runWhenInputIdle,
   subscribeOverlayFrame,
 } from "./overlayFrameLoop";
 import { usePlayerStore } from "../../player/store/playerStore";
@@ -71,6 +73,31 @@ describe("overlay frame loop", () => {
     expect(runs).toBeGreaterThan(0);
   });
 
+  it("runs idle work in capped slices, only while no input arrives", () => {
+    const budgets: number[] = [];
+    requestOverlayFrames();
+    runWhenInputIdle((timeLeft) => {
+      budgets.push(timeLeft());
+      return budgets.length === 3;
+    });
+    vi.advanceTimersByTime(300);
+    expect(budgets).toEqual([]);
+    vi.advanceTimersByTime(200);
+    expect(budgets).toHaveLength(3);
+    expect(Math.max(...budgets)).toBeLessThanOrEqual(IDLE_SLICE_MS);
+
+    // Input during a slice holds the next one until the loop has been idle again.
+    const slices: number[] = [];
+    runWhenInputIdle(() => {
+      slices.push(performance.now());
+      if (slices.length === 1) requestOverlayFrames();
+      return slices.length === 2;
+    });
+    vi.advanceTimersByTime(1000);
+    expect(slices).toHaveLength(2);
+    expect(slices[1] - slices[0]).toBeGreaterThanOrEqual(400);
+  });
+
   it("runs every subscriber on one frame, not one frame each", () => {
     const calls: string[] = [];
     subscribeOverlayFrame(() => calls.push("a"));

package/src/components/editor/overlayFrameLoop.ts

@@ -143,6 +143,27 @@ function schedule(): void {
   }, IDLE_POLL_MS);
 }
 
+/** Longest idle slice, so input that arrives during one waits at most this long for its frame. */
+export const IDLE_SLICE_MS = 3;
+
+/**
+ * Runs `step` in idle slices once no input has woken the loop for AWAKE_MS, so it never shares a gesture's
+ * frames. `step` works while `timeLeft()` is positive and returns true when it has finished.
+ */
+export function runWhenInputIdle(step: (timeLeft: () => number) => boolean): void {
+  const wait = awakeUntil - performance.now();
+  if (wait > 0) {
+    setTimeout(() => runWhenInputIdle(step), wait);
+    return;
+  }
+  const idle = window.requestIdleCallback ?? ((callback: () => void) => setTimeout(callback, 0));
+  idle(() => {
+    const end = performance.now() + IDLE_SLICE_MS;
+    if (performance.now() < awakeUntil || !step(() => end - performance.now()))
+      runWhenInputIdle(step);
+  });
+}
+
 /** Something moved, or might have. Run frames at full rate for a moment. */
 export function requestOverlayFrames(): void {
   awakeUntil = performance.now() + AWAKE_MS;