@hyperframes/studio 0.8.106 → 0.8.107

package/src/components/editor/manualEditsDom.ts

@@ -35,13 +35,26 @@ import {
 import { gsapAnimatesProperty } from "./gsapAnimatesProperty";
 import { splitTopLevelWhitespace } from "./manualEditsStyleHelpers";
 import { roundTo3, roundToLayoutPx } from "../../utils/rounding";
+import { BOX_SIZE_STYLE_PROPS } from "./manualEditsDomPatches";
 
 /* ── Gesture tracking ─────────────────────────────────────────────── */
 let studioManualEditGestureId = 0;
 
-export function beginStudioManualEditGesture(element: HTMLElement): string {
+export type StudioGestureDraws = "move" | "resize" | "rotate" | "edit";
+const MOVE_DRAWS = ["translate", STUDIO_OFFSET_X_PROP, STUDIO_OFFSET_Y_PROP];
+const GESTURE_DRAWS: Record<StudioGestureDraws, readonly string[]> = {
+  move: MOVE_DRAWS,
+  resize: [...MOVE_DRAWS, STUDIO_WIDTH_PROP, STUDIO_HEIGHT_PROP, ...BOX_SIZE_STYLE_PROPS],
+  rotate: ["rotate", "transform", "transform-origin", "display", STUDIO_ROTATION_PROP],
+  edit: [],
+};
+
+export function beginStudioManualEditGesture(
+  element: HTMLElement,
+  draws: StudioGestureDraws,
+): string {
   studioManualEditGestureId += 1;
-  const token = `gesture-${studioManualEditGestureId}`;
+  const token = `gesture-${studioManualEditGestureId}:${draws}`;
   element.setAttribute(STUDIO_MANUAL_EDIT_GESTURE_ATTR, token);
   return token;
 }
@@ -78,10 +91,20 @@ export function studioManualEditSavesIn(doc: Document): number {
   return gestureSaves.get(doc) ?? 0;
 }
 
-/** A gesture's save in `element`'s document: a reload requested before it shows the file before it. */
-export function noteStudioManualEditSave(element: HTMLElement): void {
+/** Runs a gesture's save, counted as it starts and as it settles: a reload requested before shows the old file. */
+export function countStudioManualEditSave<R>(element: HTMLElement, save: () => R): R {
   const doc = element.ownerDocument;
-  gestureSaves.set(doc, studioManualEditSavesIn(doc) + 1);
+  const count = () => void gestureSaves.set(doc, studioManualEditSavesIn(doc) + 1);
+  count();
+  const result = save();
+  void Promise.resolve(result).then(count, count);
+  return result;
+}
+
+export function studioGestureDraws(element: Element): readonly string[] | null {
+  const token = element.getAttribute(STUDIO_MANUAL_EDIT_GESTURE_ATTR);
+  if (token === null) return null;
+  return GESTURE_DRAWS[token.split(":")[1] as StudioGestureDraws] ?? [];
 }
 
 function isStudioManualEditGestureActive(element: HTMLElement): boolean {

package/src/components/editor/manualEditsDomPatches.ts

@@ -1,4 +1,5 @@
 import type { PatchOperation } from "../../utils/sourcePatcher";
+import { gsapRendersTransform } from "./gsapAnimatesProperty";
 import {
   STUDIO_OFFSET_X_PROP,
   STUDIO_OFFSET_Y_PROP,
@@ -127,7 +128,10 @@ const BOX_SIZE_ORIG_ATTRS: ReadonlyArray<[string, string]> = [
 export function buildBoxSizePatches(element: HTMLElement): PatchOperation[] {
   const ops: PatchOperation[] = [];
   collectInlineStyleOps(element, [STUDIO_WIDTH_PROP, STUDIO_HEIGHT_PROP], ops);
-  collectInlineStyleOps(element, BOX_SIZE_STYLE_PROPS, ops);
+  const withoutGsapScaleMask = gsapRendersTransform(element)
+    ? BOX_SIZE_STYLE_PROPS.filter((prop) => prop !== "scale")
+    : BOX_SIZE_STYLE_PROPS;
+  collectInlineStyleOps(element, withoutGsapScaleMask, ops);
   ops.push({ type: "attribute", property: STUDIO_BOX_SIZE_ATTR, value: "true" });
   collectAttributeOps(
     element,

package/src/components/editor/manualOffsetDrag.ts

@@ -356,7 +356,10 @@ export function createManualOffsetDragMember(input: {
   }
 
   const initialPathOffset = captureStudioPathOffset(input.element);
-  const gestureToken = beginStudioManualEditGesture(input.element);
+  const gestureToken = beginStudioManualEditGesture(
+    input.element,
+    input.gesture === "resize" ? "resize" : "move",
+  );
   const measured = measureManualOffsetDragScreenToOffsetMatrix(input.element, initialOffset, {
     scaleX: input.rect.editScaleX,
     scaleY: input.rect.editScaleY,

package/src/components/editor/mediaAudioEdits.test.ts

@@ -0,0 +1,212 @@
+// @vitest-environment happy-dom
+import { describe, expect, it } from "vitest";
+import { applyPatchByTarget } from "../../utils/sourcePatcher";
+import {
+  buildKeepSoundCutoutEdit,
+  cutoutOps,
+  hasAudioToggleOps,
+  mintLinkId,
+  mutedToggleOps,
+} from "./mediaAudioEdits";
+
+const attr = (property: string, value: string | null) => ({
+  type: "attribute" as const,
+  property,
+  value,
+});
+const html = (property: string, value: string | null) => ({
+  type: "html-attribute" as const,
+  property,
+  value,
+});
+
+describe("mutedToggleOps", () => {
+  it("mutes a video and removes has-audio instead of writing an empty marker", () => {
+    expect(mutedToggleOps({ isVideo: true, nextMuted: true, probedHasAudio: null })).toEqual([
+      html("muted", "true"),
+      attr("has-audio", null),
+    ]);
+  });
+
+  it("unmutes a video the probe hears as audible", () => {
+    expect(mutedToggleOps({ isVideo: true, nextMuted: false, probedHasAudio: true })).toEqual([
+      html("muted", null),
+      attr("has-audio", "true"),
+    ]);
+  });
+
+  it("unmutes a silent-probe video with an explicit has-audio=false", () => {
+    expect(mutedToggleOps({ isVideo: true, nextMuted: false, probedHasAudio: false })).toEqual([
+      html("muted", null),
+      attr("has-audio", "false"),
+    ]);
+  });
+
+  it("leaves has-audio absent when the probe fails", () => {
+    expect(mutedToggleOps({ isVideo: true, nextMuted: false, probedHasAudio: null })).toEqual([
+      html("muted", null),
+      attr("has-audio", null),
+    ]);
+  });
+
+  it("toggles only muted on an audio element", () => {
+    expect(mutedToggleOps({ isVideo: false, nextMuted: true, probedHasAudio: null })).toEqual([
+      html("muted", "true"),
+    ]);
+    expect(mutedToggleOps({ isVideo: false, nextMuted: false, probedHasAudio: null })).toEqual([
+      html("muted", null),
+    ]);
+  });
+});
+
+describe("hasAudioToggleOps", () => {
+  it("turns sound on by stamping has-audio and clearing muted", () => {
+    expect(hasAudioToggleOps(true)).toEqual([attr("has-audio", "true"), html("muted", null)]);
+  });
+
+  it("turns sound off by removing has-audio rather than writing an empty value", () => {
+    expect(hasAudioToggleOps(false)).toEqual([attr("has-audio", null), html("muted", "true")]);
+  });
+});
+
+describe("cutoutOps", () => {
+  it("swaps the src of an image and nothing else", () => {
+    expect(cutoutOps({ isVideo: false, cutoutSrc: "assets/cut.png" })).toEqual([
+      html("src", "assets/cut.png"),
+    ]);
+  });
+
+  it("swaps a silent video's src and keeps it muted without a has-audio marker", () => {
+    expect(cutoutOps({ isVideo: true, cutoutSrc: "assets/cut.webm" })).toEqual([
+      html("src", "assets/cut.webm"),
+      html("muted", "true"),
+      attr("has-audio", null),
+    ]);
+  });
+});
+
+describe("mintLinkId", () => {
+  it("dedupes against element ids and existing links", () => {
+    const doc = document.implementation.createHTMLDocument("t");
+    expect(mintLinkId(doc)).toBe("link");
+    doc.body.innerHTML = '<div id="link"></div><video data-link="link-2"></video>';
+    expect(mintLinkId(doc)).toBe("link-3");
+  });
+});
+
+function makeVideo(attrs: Record<string, string>): HTMLVideoElement {
+  const doc = document.implementation.createHTMLDocument("t");
+  const video = doc.createElement("video");
+  for (const [name, value] of Object.entries(attrs)) video.setAttribute(name, value);
+  doc.body.append(video);
+  return video;
+}
+
+function applyEdit(source: string, edit: ReturnType<typeof buildKeepSoundCutoutEdit>): string {
+  const target = { id: "clip" };
+  const patched = edit.ops.reduce((acc, op) => applyPatchByTarget(acc, target, op), source);
+  return edit.prepareContent(patched);
+}
+
+function parse(source: string): Document {
+  return new DOMParser().parseFromString(source, "text/html");
+}
+
+describe("buildKeepSoundCutoutEdit", () => {
+  const automation = JSON.stringify({
+    version: 1,
+    lanes: [
+      { target: "volume", points: [{ t: 0, v: 1 }] },
+      { target: "rate", points: [{ t: 0, v: 2 }] },
+    ],
+  });
+
+  const source = [
+    '<div data-composition-id="main" data-duration="10">',
+    '  <video id="clip" class="clip" src="assets/talk.mp4" data-start="1" data-duration="4" data-media-start="2" data-track-index="0" data-has-audio="true" data-volume="0.5" data-fade-in="0.3" data-audio-group="voiceover"></video>',
+    '  <audio id="music" class="clip" src="assets/bgm.mp3" data-start="0" data-duration="10" data-track-index="1"></audio>',
+    "</div>",
+  ].join("\n");
+
+  const liveVideo = () =>
+    makeVideo({
+      id: "clip",
+      src: "assets/talk.mp4",
+      "data-has-audio": "true",
+      "data-volume": "0.5",
+      "data-fade-in": "0.3",
+      "data-audio-group": "voiceover",
+      "data-automation": automation,
+    });
+
+  it("mutes the video onto the cutout and moves its sound to a linked sibling audio", () => {
+    const edit = buildKeepSoundCutoutEdit({
+      video: liveVideo(),
+      videoId: "clip",
+      target: { id: "clip" },
+      cutoutSrc: "assets/talk-cutout.webm",
+    });
+    const doc = parse(applyEdit(source, edit));
+    const video = doc.getElementById("clip");
+    const audio = doc.getElementById("clip-audio");
+    if (!video || !audio) throw new Error("expected both clips");
+
+    expect(video.getAttribute("src")).toBe("assets/talk-cutout.webm");
+    expect(video.hasAttribute("muted")).toBe(true);
+    expect(video.hasAttribute("data-has-audio")).toBe(false);
+    expect(video.hasAttribute("data-volume")).toBe(false);
+    expect(video.hasAttribute("data-fade-in")).toBe(false);
+    expect(video.hasAttribute("data-audio-group")).toBe(false);
+    expect(JSON.parse(video.getAttribute("data-automation") ?? "").lanes).toEqual([
+      { target: "rate", points: [{ t: 0, v: 2 }] },
+    ]);
+
+    expect(audio.tagName).toBe("AUDIO");
+    expect(audio.getAttribute("src")).toBe("assets/talk.mp4");
+    expect(audio.getAttribute("data-start")).toBe("1");
+    expect(audio.getAttribute("data-duration")).toBe("4");
+    expect(audio.getAttribute("data-media-start")).toBe("2");
+    expect(audio.hasAttribute("data-playback-rate")).toBe(false);
+    expect(audio.getAttribute("data-track-index")).toBe("2");
+    expect(audio.getAttribute("data-volume")).toBe("0.5");
+    expect(audio.getAttribute("data-fade-in")).toBe("0.3");
+    expect(audio.getAttribute("data-audio-group")).toBe("voiceover");
+    expect(JSON.parse(audio.getAttribute("data-automation") ?? "").lanes).toEqual([
+      { target: "volume", points: [{ t: 0, v: 1 }] },
+      { target: "rate", points: [{ t: 0, v: 2 }] },
+    ]);
+    expect(audio.getAttribute("data-link")).toBe(video.getAttribute("data-link"));
+    expect(video.getAttribute("data-link")).toBe("link");
+    expect(audio.parentElement).toBe(video.parentElement);
+  });
+
+  it("drops the video's automation attribute when it held no rate lane", () => {
+    const video = makeVideo({
+      id: "clip",
+      src: "assets/talk.mp4",
+      "data-automation": JSON.stringify({
+        version: 1,
+        lanes: [{ target: "volume", points: [{ t: 0, v: 1 }] }],
+      }),
+    });
+    const edit = buildKeepSoundCutoutEdit({
+      video,
+      videoId: "clip",
+      target: { id: "clip" },
+      cutoutSrc: "assets/cut.webm",
+    });
+    expect(edit.ops).toContainEqual(attr("automation", null));
+  });
+
+  it("suffixes the audio id when <video-id>-audio is taken", () => {
+    const edit = buildKeepSoundCutoutEdit({
+      video: liveVideo(),
+      videoId: "clip",
+      target: { id: "clip" },
+      cutoutSrc: "assets/cut.webm",
+    });
+    const taken = source.replace('id="music"', 'id="clip-audio"');
+    const doc = parse(applyEdit(taken, edit));
+    expect(doc.getElementById("clip-audio-2")?.getAttribute("src")).toBe("assets/talk.mp4");
+  });
+});

package/src/components/editor/mediaAudioEdits.ts

@@ -0,0 +1,253 @@
+import {
+  HF_AUDIO_AUTOMATION_ATTR,
+  RATE_TARGET,
+  parseAutomation,
+  serializeAutomation,
+  type HfAutomationLane,
+} from "@hyperframes/core/audio-automation";
+import { HF_AUDIO_CARVE_ATTR } from "@hyperframes/core/audio-carve";
+import { HF_AUDIO_FADE_IN_ATTR, HF_AUDIO_FADE_OUT_ATTR } from "@hyperframes/core/audio-fade";
+import { HF_AUDIO_FX_ATTR } from "@hyperframes/core/audio-fx";
+import { HF_AUDIO_GROUP_ATTR } from "@hyperframes/core/audio-groups";
+import {
+  escapeHtmlAttribute,
+  findTagByTarget,
+  readAttributeByTarget,
+  type PatchOperation,
+  type PatchTarget,
+} from "../../utils/sourcePatcher";
+import { collectHtmlIds, resolveAssetHasAudio } from "../../utils/studioHelpers";
+import type { CommitDomAttributeBatch } from "../../hooks/domEditCommitTypes";
+import { buildDomEditPatchTarget, type DomEditSelection } from "./domEditing";
+import { generateId } from "../../utils/generateId";
+
+const MEDIA_LINK_ATTR = "data-link";
+const SOUND_KEPT_ON_LINKED_AUDIO_STAGE = "Background removed. Sound kept on a linked audio track.";
+
+const MOVED_SOUND_ATTRS = [
+  "data-volume",
+  HF_AUDIO_FADE_IN_ATTR,
+  HF_AUDIO_FADE_OUT_ATTR,
+  HF_AUDIO_FX_ATTR,
+  HF_AUDIO_CARVE_ATTR,
+  HF_AUDIO_GROUP_ATTR,
+];
+
+const COPIED_TIMING_ATTRS = [
+  "start",
+  "end",
+  "duration",
+  "media-start",
+  "playback-start",
+  "playback-rate",
+  "source-duration",
+];
+
+const htmlAttr = (property: string, value: string | null): PatchOperation => ({
+  type: "html-attribute",
+  property,
+  value,
+});
+
+const dataAttr = (property: string, value: string | null): PatchOperation => ({
+  type: "attribute",
+  property,
+  value,
+});
+
+function stripDataPrefix(name: string): string {
+  return name.startsWith("data-") ? name.slice("data-".length) : name;
+}
+
+function probedHasAudioValue(probedHasAudio: boolean | null): string | null {
+  if (probedHasAudio === null) return null;
+  return probedHasAudio ? "true" : "false";
+}
+
+export function mutedToggleOps(input: {
+  isVideo: boolean;
+  nextMuted: boolean;
+  probedHasAudio: boolean | null;
+}): PatchOperation[] {
+  const muted = htmlAttr("muted", input.nextMuted ? "true" : null);
+  if (!input.isVideo) return [muted];
+  const hasAudio = input.nextMuted ? null : probedHasAudioValue(input.probedHasAudio);
+  return [muted, dataAttr("has-audio", hasAudio)];
+}
+
+export function hasAudioToggleOps(next: boolean): PatchOperation[] {
+  return next
+    ? [dataAttr("has-audio", "true"), htmlAttr("muted", null)]
+    : [dataAttr("has-audio", null), htmlAttr("muted", "true")];
+}
+
+export function cutoutOps(input: { isVideo: boolean; cutoutSrc: string }): PatchOperation[] {
+  const src = htmlAttr("src", input.cutoutSrc);
+  return input.isVideo ? [src, htmlAttr("muted", "true"), dataAttr("has-audio", null)] : [src];
+}
+
+function firstFreeName(base: string, taken: Set<string>): string {
+  if (!taken.has(base)) return base;
+  let n = 2;
+  while (taken.has(`${base}-${n}`)) n += 1;
+  return `${base}-${n}`;
+}
+
+export function mintLinkId(doc: Document): string {
+  const taken = new Set<string>();
+  for (const el of Array.from(doc.querySelectorAll(`[id], [${MEDIA_LINK_ATTR}]`))) {
+    if (el.id) taken.add(el.id);
+    const link = el.getAttribute(MEDIA_LINK_ATTR);
+    if (link) taken.add(link);
+  }
+  return firstFreeName("link", taken);
+}
+
+function firstFreeTrackIndex(source: string): number {
+  const used = new Set(
+    Array.from(source.matchAll(/\bdata-track-index=["'](-?\d+)["']/g), (m) => Number(m[1])),
+  );
+  let track = 0;
+  while (used.has(track)) track += 1;
+  return track;
+}
+
+function splitAutomation(raw: string): { videoKeeps: string | null; audioTakes: string } {
+  let lanes: HfAutomationLane[];
+  try {
+    lanes = parseAutomation(raw).lanes;
+  } catch {
+    return { videoKeeps: null, audioTakes: raw };
+  }
+  const rateLanes = lanes.filter((lane) => lane.target === RATE_TARGET);
+  const soundLanes = lanes.filter((lane) => lane.target !== RATE_TARGET);
+  return {
+    videoKeeps: rateLanes.length > 0 ? serializeAutomation({ version: 1, lanes: rateLanes }) : null,
+    audioTakes: serializeAutomation({ version: 1, lanes: [...soundLanes, ...rateLanes] }),
+  };
+}
+
+function formatAttrs(attrs: Array<[string, string]>): string {
+  return attrs.map(([name, value]) => `${name}="${escapeHtmlAttribute(value)}"`).join(" ");
+}
+
+function insertBeforeTarget(source: string, target: PatchTarget, markup: string): string {
+  const match = findTagByTarget(source, target);
+  if (!match) return source;
+  const lineStart = source.lastIndexOf("\n", match.start) + 1;
+  const indent = /^[ \t]*$/.test(source.slice(lineStart, match.start))
+    ? source.slice(lineStart, match.start)
+    : "";
+  const separator = indent ? `\n${indent}` : "";
+  return `${source.slice(0, match.start)}${markup}${separator}${source.slice(match.start)}`;
+}
+
+export interface KeepSoundCutoutEdit {
+  ops: PatchOperation[];
+  prepareContent: (source: string) => string;
+}
+
+export function buildKeepSoundCutoutEdit(input: {
+  video: Element;
+  videoId: string | null;
+  target: PatchTarget;
+  cutoutSrc: string;
+}): KeepSoundCutoutEdit {
+  const { video, target } = input;
+  const linkId = mintLinkId(video.ownerDocument);
+  const originalSrc = video.getAttribute("src") ?? "";
+
+  const movedSound: Array<[string, string]> = [];
+  const ops: PatchOperation[] = [...cutoutOps({ isVideo: true, cutoutSrc: input.cutoutSrc })];
+  for (const name of MOVED_SOUND_ATTRS) {
+    const value = video.getAttribute(name);
+    if (value === null) continue;
+    movedSound.push([name, value]);
+    ops.push(dataAttr(stripDataPrefix(name), null));
+  }
+  const automation = video.getAttribute(HF_AUDIO_AUTOMATION_ATTR);
+  if (automation !== null) {
+    const { videoKeeps, audioTakes } = splitAutomation(automation);
+    movedSound.push([HF_AUDIO_AUTOMATION_ATTR, audioTakes]);
+    ops.push(dataAttr(stripDataPrefix(HF_AUDIO_AUTOMATION_ATTR), videoKeeps));
+  }
+  ops.push(dataAttr(stripDataPrefix(MEDIA_LINK_ATTR), linkId));
+
+  const prepareContent = (source: string): string => {
+    const audioId = firstFreeName(
+      `${input.videoId || "video"}-audio`,
+      new Set(collectHtmlIds(source)),
+    );
+    const timing: Array<[string, string]> = [];
+    for (const name of COPIED_TIMING_ATTRS) {
+      const value = readAttributeByTarget(source, target, name);
+      if (value !== undefined) timing.push([`data-${name}`, value]);
+    }
+    const attrs: Array<[string, string]> = [
+      ["id", audioId],
+      ["data-hf-id", `hf-${generateId()}`],
+      ["class", "clip"],
+      ["src", originalSrc],
+      ...timing,
+      ["data-track-index", String(firstFreeTrackIndex(source))],
+      [MEDIA_LINK_ATTR, linkId],
+      ...movedSound,
+    ];
+    return insertBeforeTarget(source, target, `<audio ${formatAttrs(attrs)}></audio>`);
+  };
+
+  return { ops, prepareContent };
+}
+
+export interface MediaEditContext {
+  element: DomEditSelection;
+  projectId: string | null;
+  projectSrc: string;
+  commit: CommitDomAttributeBatch;
+}
+
+const isVideoSelection = (element: DomEditSelection) => element.tagName === "video";
+
+export async function commitMutedToggle(ctx: MediaEditContext, nextMuted: boolean): Promise<void> {
+  const isVideo = isVideoSelection(ctx.element);
+  const probedHasAudio =
+    isVideo && !nextMuted && ctx.projectId && ctx.projectSrc
+      ? await resolveAssetHasAudio(ctx.projectId, ctx.projectSrc)
+      : null;
+  await ctx.commit(ctx.element, mutedToggleOps({ isVideo, nextMuted, probedHasAudio }), {
+    label: "Edit muted",
+  });
+}
+
+export async function commitHasAudioToggle(ctx: MediaEditContext, next: boolean): Promise<void> {
+  await ctx.commit(ctx.element, hasAudioToggleOps(next), { label: "Edit has audio" });
+}
+
+export async function commitCutout(
+  ctx: MediaEditContext,
+  cutoutSrc: string,
+  keepSound: boolean,
+): Promise<string> {
+  const isVideo = isVideoSelection(ctx.element);
+  const label = "Remove background";
+  const keep = isVideo && keepSound;
+  const landed = keep
+    ? await commitKeepSoundCutout(ctx, cutoutSrc, label)
+    : await ctx.commit(ctx.element, cutoutOps({ isVideo, cutoutSrc }), { label });
+  if (!landed) throw new Error("Couldn't apply the cutout");
+  return keep ? SOUND_KEPT_ON_LINKED_AUDIO_STAGE : "Applied cutout";
+}
+
+function commitKeepSoundCutout(
+  ctx: MediaEditContext,
+  cutoutSrc: string,
+  label: string,
+): Promise<boolean> {
+  const edit = buildKeepSoundCutoutEdit({
+    video: ctx.element.element,
+    videoId: ctx.element.id ?? null,
+    target: buildDomEditPatchTarget(ctx.element),
+    cutoutSrc,
+  });
+  return ctx.commit(ctx.element, edit.ops, { label, prepareContent: edit.prepareContent });
+}