@hyperframes/studio 0.8.57 → 0.8.59

package/src/player/hooks/usePlayerHandle.test.tsx

@@ -0,0 +1,152 @@
+// @vitest-environment happy-dom
+
+import React, { useEffect } from "react";
+import { act } from "react";
+import { createRoot } from "react-dom/client";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { NLEProvider, useNLEContext, type NLEContextValue } from "../../components/nle/NLEContext";
+import { liveTime, usePlayerStore } from "../store/playerStore";
+import { makeAdapterWindow, makeFakeIframe } from "./timelinePlayerTestHarness";
+import { usePlayerHandle, type PlayerHandle } from "./usePlayerHandle";
+
+Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true });
+
+function renderHandle() {
+  let latest: PlayerHandle | null = null;
+  let context: NLEContextValue | null = null;
+  let renders = 0;
+  const host = document.createElement("div");
+  document.body.append(host);
+  const root = createRoot(host);
+  act(() => {
+    root.render(
+      <NLEProvider projectId="player-handle-test">
+        <HandleHarness
+          onContext={(value) => (context = value)}
+          onValue={(value) => (latest = value)}
+          onRender={() => renders++}
+        />
+      </NLEProvider>,
+    );
+  });
+  if (!latest || !context) throw new Error("usePlayerHandle did not mount");
+  return {
+    root,
+    getHandle: () => latest as PlayerHandle,
+    getContext: () => context as NLEContextValue,
+    getRenderCount: () => renders,
+  };
+}
+
+function HandleHarness({
+  onContext,
+  onValue,
+  onRender,
+}: {
+  onContext: (value: NLEContextValue) => void;
+  onValue: (value: PlayerHandle) => void;
+  onRender: () => void;
+}) {
+  onRender();
+  const context = useNLEContext();
+  const handle = usePlayerHandle();
+  useEffect(() => {
+    onContext(context);
+    onValue(handle);
+  }, [context, handle, onContext, onValue]);
+  return null;
+}
+
+afterEach(() => {
+  usePlayerStore.getState().reset();
+  document.body.innerHTML = "";
+});
+
+describe("usePlayerHandle", () => {
+  it("refuses seek before readiness and notifies structural changes independently", () => {
+    const { root, getHandle } = renderHandle();
+    const handle = getHandle();
+
+    expect(handle.ready).toBe(false);
+    expect(handle.seek(4)).toBe(false);
+
+    const structuralListener = vi.fn();
+    const unsubscribe = handle.subscribe(structuralListener);
+    act(() => usePlayerStore.setState({ timelineReady: true }));
+    expect(structuralListener).toHaveBeenCalledTimes(1);
+    act(() => usePlayerStore.setState({ duration: 8 }));
+    expect(structuralListener).toHaveBeenCalledTimes(2);
+    act(() =>
+      usePlayerStore.setState({
+        elements: [
+          {
+            id: "clip",
+            tag: "div",
+            start: 2,
+            duration: 3,
+            track: 0,
+            selector: "#clip",
+            sourceFile: "index.html",
+            compositionSrc: "compositions/intro.html",
+          },
+        ],
+      }),
+    );
+    expect(structuralListener).toHaveBeenCalledTimes(3);
+    expect(getHandle().elements).toEqual([
+      {
+        id: "clip",
+        start: 2,
+        end: 5,
+        selector: "#clip",
+        sourceFile: "index.html",
+        compositionSrc: "compositions/intro.html",
+      },
+    ]);
+
+    unsubscribe();
+    act(() => root.unmount());
+  });
+
+  it("uses the shell player to seek a real preview adapter after readiness", async () => {
+    const { root, getContext, getHandle } = renderHandle();
+    const { adapter, win } = makeAdapterWindow();
+    await act(async () => {
+      getContext().iframeRef.current = makeFakeIframe(win);
+      getContext().onIframeLoad();
+      await Promise.resolve();
+    });
+
+    expect(usePlayerStore.getState().timelineReady).toBe(true);
+    expect(getHandle().seek(4)).toBe(true);
+    expect(adapter.getTime()).toBe(4);
+    expect(usePlayerStore.getState().currentTime).toBe(4);
+    act(() => getHandle().play());
+    expect(getHandle().playing).toBe(true);
+
+    act(() => root.unmount());
+  });
+
+  it("keeps playback time out of React renders and forwards live time notifications", () => {
+    const { root, getHandle, getRenderCount } = renderHandle();
+    const initialRenderCount = getRenderCount();
+    const timeListener = vi.fn();
+    const structuralListener = vi.fn();
+    const unsubscribeTime = getHandle().subscribeTime(timeListener);
+    const unsubscribe = getHandle().subscribe(structuralListener);
+
+    act(() => {
+      usePlayerStore.setState({ currentTime: 2.5 });
+      liveTime.notify(7.25);
+    });
+
+    expect(getRenderCount()).toBe(initialRenderCount);
+    expect(getHandle().currentTime).toBe(2.5);
+    expect(timeListener).toHaveBeenCalledWith(7.25);
+    expect(structuralListener).not.toHaveBeenCalled();
+
+    unsubscribeTime();
+    unsubscribe();
+    act(() => root.unmount());
+  });
+});

package/src/player/hooks/usePlayerHandle.ts

@@ -0,0 +1,94 @@
+import { useCallback, useMemo } from "react";
+import { liveTime, usePlayerStore, type TimelineElement } from "../store/playerStore";
+import { useNLEContext } from "../../components/nle/NLEContext";
+
+export interface PlayerHandleElement {
+  id: string;
+  start: number;
+  end: number;
+  selector?: string;
+  sourceFile?: string;
+  compositionSrc?: string;
+}
+
+export type PlayerHandleListener = () => void;
+export type PlayerHandleTimeListener = (time: number) => void;
+
+export interface PlayerHandle {
+  ready: boolean;
+  playing: boolean;
+  seek: (seconds: number) => boolean;
+  play: () => void;
+  pause: () => void;
+  togglePlay: () => void;
+  readonly currentTime: number;
+  duration: number;
+  elements: PlayerHandleElement[];
+  subscribe: (listener: PlayerHandleListener) => () => void;
+  subscribeTime: (listener: PlayerHandleTimeListener) => () => void;
+}
+
+function toHandleElement(element: TimelineElement): PlayerHandleElement {
+  return {
+    id: element.id,
+    start: element.start,
+    end: element.start + element.duration,
+    selector: element.selector,
+    sourceFile: element.sourceFile,
+    compositionSrc: element.compositionSrc,
+  };
+}
+
+/**
+ * Reads and controls the shell player. Call this under EditorShell/NLEProvider;
+ * the provider owns the mounted preview iframe and the single player instance.
+ */
+export function usePlayerHandle(): PlayerHandle {
+  const { play, pause, togglePlay, seek: seekPlayer } = useNLEContext();
+  const ready = usePlayerStore((state) => state.timelineReady);
+  const playing = usePlayerStore((state) => state.isPlaying);
+  const duration = usePlayerStore((state) => state.duration);
+  const elements = usePlayerStore((state) => state.elements);
+
+  const seek = useCallback(
+    (seconds: number) => {
+      if (!ready) return false;
+      return seekPlayer(seconds);
+    },
+    [ready, seekPlayer],
+  );
+
+  const subscribe = useCallback((listener: PlayerHandleListener) => {
+    let previous = usePlayerStore.getState();
+    return usePlayerStore.subscribe((state) => {
+      const structuralChange =
+        state.timelineReady !== previous.timelineReady ||
+        state.isPlaying !== previous.isPlaying ||
+        state.duration !== previous.duration ||
+        state.elements !== previous.elements;
+      previous = state;
+      if (structuralChange) listener();
+    });
+  }, []);
+
+  const subscribeTime = useCallback((listener: PlayerHandleTimeListener) => {
+    return liveTime.subscribe(listener);
+  }, []);
+  const handleElements = useMemo(() => elements.map(toHandleElement), [elements]);
+
+  return {
+    ready,
+    playing,
+    seek,
+    play,
+    pause,
+    togglePlay,
+    get currentTime() {
+      return usePlayerStore.getState().currentTime;
+    },
+    duration,
+    elements: handleElements,
+    subscribe,
+    subscribeTime,
+  };
+}

package/src/player/index.ts

@@ -7,6 +7,13 @@ export { CompositionThumbnail } from "./components/CompositionThumbnail";
 
 // Hooks
 export { useTimelinePlayer } from "./hooks/useTimelinePlayer";
+export { usePlayerHandle } from "./hooks/usePlayerHandle";
+export type {
+  PlayerHandle,
+  PlayerHandleElement,
+  PlayerHandleListener,
+  PlayerHandleTimeListener,
+} from "./hooks/usePlayerHandle";
 export { resolveIframe } from "./lib/timelineDOM";
 
 // Store

package/src/player/lib/timelineElementHelpers.ts

@@ -12,6 +12,7 @@ import type { ClipManifestClip } from "./playbackTypes";
 import { isFinitePositive } from "./playbackAdapter";
 import { getSourceScopedSelectorIndex } from "../../utils/sourceScopedSelectorIndex";
 import { HF_AUDIO_GROUP_TAG } from "@hyperframes/core/audio-groups";
+import { readElementFades } from "@hyperframes/core/audio-fade";
 
 // ---------------------------------------------------------------------------
 // Layer-reveal lift transparency
@@ -173,8 +174,27 @@ function applyPlaybackMetadataFromElement(entry: TimelineElement, el: Element):
   }
 }
 
+/** Sets or clears an optional field, so a re-parse after the attribute is removed drops it. */
+function setOptional<K extends keyof TimelineElement>(
+  entry: TimelineElement,
+  key: K,
+  value: TimelineElement[K] | undefined,
+): void {
+  if (value === undefined) delete entry[key];
+  else entry[key] = value;
+}
+
+/** `data-has-audio` and the clip-edge fades: what the mixer hears and how it enters and leaves. */
+function applyAudioMetadataFromElement(entry: TimelineElement, el: Element): void {
+  setOptional(entry, "hasAudio", el.getAttribute("data-has-audio") === "true" ? true : undefined);
+  const fades = readElementFades(el);
+  setOptional(entry, "fadeIn", fades.fadeIn > 0 ? fades.fadeIn : undefined);
+  setOptional(entry, "fadeOut", fades.fadeOut > 0 ? fades.fadeOut : undefined);
+}
+
 export function applyMediaMetadataFromElement(entry: TimelineElement, el: Element): void {
   applyPlaybackMetadataFromElement(entry, el);
+  applyAudioMetadataFromElement(entry, el);
 
   const mediaEl = resolveMediaElement(el);
   if (!mediaEl) return;

package/src/player/store/timelineElement.ts

@@ -54,6 +54,11 @@ export interface TimelineElement {
   playbackRate?: number;
   sourceDuration?: number;
   volume?: number;
+  /** `data-has-audio="true"` on a video: its sound is mixed, so it takes audio affordances (fades). */
+  hasAudio?: boolean;
+  /** Clip-edge fades from `data-fade-in` / `data-fade-out`, seconds; absent means none. */
+  fadeIn?: number;
+  fadeOut?: number;
   /** Verbatim `data-fx-chain` / `data-automation`; see automationLaneData. */
   fxChain?: string;
   automation?: string;

package/src/styles/theme.css

@@ -69,6 +69,10 @@
   --timeline-clip-border-hover: rgba(60, 230, 172, 0.75);
   --timeline-clip-border-active: rgba(60, 230, 172, 0.55);
   --timeline-handle: rgba(255, 255, 255, 0.55);
+  /* Audio clip fades: the shaded wedge over the faded gain, and the corner dots. */
+  --timeline-fade-shade: rgba(0, 0, 0, 0.38);
+  --timeline-fade-dot-active: rgba(255, 255, 255, 1);
+  --timeline-fade-dot-ring: rgba(0, 0, 0, 0.55);
   --timeline-resize-seam: rgba(255, 255, 255, 0.12);
   --timeline-resize-active: rgba(255, 255, 255, 0.24);
   --timeline-track-clip-fill: rgba(255, 255, 255, 0.055);

package/src/utils/audioMeterMath.test.ts

@@ -0,0 +1,44 @@
+import { describe, expect, it } from "vitest";
+import {
+  levelToFraction,
+  markFraction,
+  SILENT_CHANNEL,
+  stepChannel,
+  stepPair,
+} from "./audioMeterMath";
+
+describe("levelToFraction", () => {
+  it("puts each dB mark on its equally spaced stop", () => {
+    [1, 0.8, 0.6, 0.4, 0.2].forEach((f, i) =>
+      expect(markFraction([0, -3, -6, -12, -24][i]!)).toBeCloseTo(f, 6),
+    );
+  });
+  it("clamps silence and overs", () => {
+    expect(levelToFraction(0)).toBe(0);
+    expect(levelToFraction(4)).toBe(1);
+  });
+  it("is monotonic between marks", () => {
+    expect(levelToFraction(0.5)).toBeGreaterThan(markFraction(-12));
+    expect(levelToFraction(0.5)).toBeLessThan(markFraction(-3));
+  });
+});
+
+describe("stepChannel", () => {
+  it("attacks instantly, holds the peak, then falls", () => {
+    let ch = stepChannel(SILENT_CHANNEL, 1, 0, 16);
+    expect(ch.level).toBe(1);
+    ch = stepChannel(ch, 0, 500, 16);
+    expect(ch.peak).toBe(1);
+    expect(ch.level).toBeLessThan(1);
+    ch = stepChannel(ch, 0, 2000, 500);
+    expect(ch.peak).toBeLessThan(1);
+  });
+});
+
+describe("stepPair", () => {
+  it("returns the same pair while silent at rest, and moves on sound", () => {
+    const rest = stepPair(undefined, undefined, 0, 16);
+    expect(stepPair(rest, { l: 0, r: 0 }, 16, 16)).toBe(rest);
+    expect(stepPair(rest, { l: 1, r: 0 }, 32, 16)).not.toBe(rest);
+  });
+});

package/src/utils/audioMeterMath.ts

@@ -0,0 +1,79 @@
+import { useMemo } from "react";
+import { audioDbToGain, audioGainToDb } from "@hyperframes/core/audio-gain";
+import { usePlayerStore, type TimelineElement } from "../player";
+import { isAudioTimelineElement } from "./timelineInspector";
+
+/** dB stops of the meter scale, top to bottom; equal spacing between stops. */
+export const METER_DB_MARKS = [0, -3, -6, -12, -24] as const;
+const FLOOR_DB = -60;
+const STOPS = [...METER_DB_MARKS, FLOOR_DB];
+
+const FALL_PER_SECOND = 0.9;
+const HOLD_MS = 1200;
+
+/** Linear peak (0..1) to a 0..1 bar fraction along the piecewise-linear dB scale. */
+export function levelToFraction(linear: number): number {
+  const db = linear > 0 ? audioGainToDb(linear) : FLOOR_DB;
+  if (db >= 0) return 1;
+  if (db <= FLOOR_DB) return 0;
+  const j = STOPS.findIndex((stop) => db >= stop);
+  const hi = STOPS[j - 1] as number;
+  const lo = STOPS[j] as number;
+  return (STOPS.length - 1 - j + (db - lo) / (hi - lo)) / (STOPS.length - 1);
+}
+
+/** Fraction of the bar height at which a dB mark sits, 0 at the bottom. */
+export function markFraction(db: number): number {
+  return levelToFraction(audioDbToGain(db));
+}
+
+export interface MeterChannel {
+  level: number;
+  peak: number;
+  peakAt: number;
+}
+
+export const SILENT_CHANNEL: MeterChannel = { level: 0, peak: 0, peakAt: 0 };
+
+/** Instant attack, steady fall; the peak holds for HOLD_MS, then falls with the bar. */
+export function stepChannel(
+  prev: MeterChannel,
+  linear: number,
+  now: number,
+  dtMs: number,
+): MeterChannel {
+  const fall = (dtMs / 1000) * FALL_PER_SECOND;
+  const target = levelToFraction(linear);
+  const level = Math.max(target, prev.level - fall);
+  if (target >= prev.peak) return { level, peak: target, peakAt: now };
+  if (now - prev.peakAt < HOLD_MS) return { level, peak: prev.peak, peakAt: prev.peakAt };
+  return { level, peak: Math.max(target, prev.peak - fall), peakAt: prev.peakAt };
+}
+
+export type MeterPair = [MeterChannel, MeterChannel];
+
+const atRest = (ch: MeterChannel): boolean => ch.level === 0 && ch.peak === 0;
+
+/** Steps both channels; returns `prev` itself while silent and at rest, so callers can skip the paint. */
+export function stepPair(
+  prev: MeterPair | undefined,
+  raw: { l: number; r: number } | undefined,
+  now: number,
+  dtMs: number,
+): MeterPair {
+  const [l, r] = prev ?? [SILENT_CHANNEL, SILENT_CHANNEL];
+  const silent = !raw?.l && !raw?.r;
+  if (prev && silent && atRest(l) && atRest(r)) return prev;
+  return [stepChannel(l, raw?.l ?? 0, now, dtMs), stepChannel(r, raw?.r ?? 0, now, dtMs)];
+}
+
+/** The meters appear only for a project with at least one audio clip or group. */
+function hasProjectAudio(elements: readonly TimelineElement[]): boolean {
+  return elements.some((el) => isAudioTimelineElement(el) || el.audioGroup !== undefined);
+}
+
+/** Scans once per `elements` change, not on every store update (the playhead ticks each frame). */
+export function useProjectHasAudio(): boolean {
+  const elements = usePlayerStore((s) => s.elements);
+  return useMemo(() => hasProjectAudio(elements), [elements]);
+}

package/src/utils/audioMeterVisibility.ts

@@ -0,0 +1,14 @@
+import { create } from "zustand";
+import { readStudioUiPreferences, writeStudioUiPreferences } from "./studioUiPreferences";
+
+/** Whether the audio meter strip is shown; persisted, on unless the user hid it. */
+export const useAudioMetersVisible = create<{
+  visible: boolean;
+  setVisible: (visible: boolean) => void;
+}>((set) => ({
+  visible: readStudioUiPreferences().audioMetersVisible ?? true,
+  setVisible: (visible) => {
+    writeStudioUiPreferences({ audioMetersVisible: visible });
+    set({ visible });
+  },
+}));

package/src/utils/studioHelpers.ts

@@ -377,6 +377,27 @@ export async function resolveDroppedAssetDuration(
   return duration;
 }
 
+export function mediaMetadataUrl(projectId: string, assetPath: string): string {
+  return `/api/projects/${encodeURIComponent(projectId)}/media/metadata?path=${encodeURIComponent(assetPath)}`;
+}
+
+/** Dropped video audio stream from the metadata endpoint. Failure answers false so the drop still lands muted. */
+export async function resolveDroppedAssetHasAudio(
+  projectId: string,
+  assetPath: string,
+  kind: TimelineAssetKind,
+): Promise<boolean> {
+  if (kind !== "video") return false;
+  try {
+    const response = await fetch(mediaMetadataUrl(projectId, assetPath));
+    if (!response.ok) return false;
+    const data = (await response.json()) as { metadata?: { hasAudio?: boolean } } | null;
+    return data?.metadata?.hasAudio === true;
+  } catch {
+    return false;
+  }
+}
+
 export async function resolveDroppedAssetDimensions(
   projectId: string,
   assetPath: string,

package/src/utils/studioUiPreferences.test.ts

@@ -161,3 +161,13 @@ describe("timeline zoom pin persistence", () => {
     expect(prefs.timelineManualZoomPercent).toBeUndefined();
   });
 });
+
+describe("audioMetersVisible preference", () => {
+  it("round-trips and ignores a non-boolean", () => {
+    const storage = createStorage();
+    writeStudioUiPreferences({ audioMetersVisible: false }, storage);
+    expect(readStudioUiPreferences(storage).audioMetersVisible).toBe(false);
+    storage.setItem("hf-studio-ui-preferences", JSON.stringify({ audioMetersVisible: "no" }));
+    expect(readStudioUiPreferences(storage).audioMetersVisible).toBeUndefined();
+  });
+});

package/src/utils/studioUiPreferences.ts

@@ -25,6 +25,8 @@ export interface StudioUiPreferences {
   snapToGrid?: boolean;
   /** Timeline magnet: snap clip drags/trims/drops to playhead, clip edges, and beats. */
   timelineSnapEnabled?: boolean;
+  /** Audio level meters at the timeline's right edge; shown unless hidden here. */
+  audioMetersVisible?: boolean;
   /** Keeps the main track gapless: deleting a clip closes the gap. Distinct
    *  from `timelineSnapEnabled` ("Magnet", drag/trim snapping). */
   rippleEditEnabled?: boolean;
@@ -140,6 +142,9 @@ function readStorage(storage: Storage | null, key: string): StudioUiPreferences
     if (typeof parsed.timelineSnapEnabled === "boolean") {
       preferences.timelineSnapEnabled = parsed.timelineSnapEnabled;
     }
+    if (typeof parsed.audioMetersVisible === "boolean") {
+      preferences.audioMetersVisible = parsed.audioMetersVisible;
+    }
     if (typeof parsed.rippleEditEnabled === "boolean") {
       preferences.rippleEditEnabled = parsed.rippleEditEnabled;
     }

package/src/utils/timelineAssetDrop.test.ts

@@ -198,6 +198,38 @@ describe("insertTimelineAssetIntoSource", () => {
   });
 });
 
+describe("buildTimelineAssetInsertHtml — video audio", () => {
+  const base = {
+    id: "clip_asset",
+    hfId: "hf-vid-1",
+    assetPath: "assets/clip.mp4",
+    kind: "video" as const,
+    start: 0,
+    duration: 8,
+    track: 1,
+    zIndex: 2,
+  };
+
+  it("inserts a video muted when nothing says it carries audio", () => {
+    const html = buildTimelineAssetInsertHtml(base);
+    expect(html).toContain(" muted ");
+    expect(html).not.toContain("data-has-audio");
+  });
+
+  it("inserts a video with an audio stream audible: data-has-audio and no muted", () => {
+    const html = buildTimelineAssetInsertHtml({ ...base, hasAudio: true });
+    expect(html).toContain('data-has-audio="true"');
+    expect(html).not.toContain("muted");
+    expect(html).toContain("playsinline");
+  });
+
+  it("keeps a video without an audio stream muted", () => {
+    const html = buildTimelineAssetInsertHtml({ ...base, hasAudio: false });
+    expect(html).toContain(" muted ");
+    expect(html).not.toContain("data-has-audio");
+  });
+});
+
 describe("buildTimelineAssetInsertHtml markup quality", () => {
   const base = {
     id: "clip_1",

package/src/utils/timelineAssetDrop.ts

@@ -116,6 +116,8 @@ export function buildTimelineAssetInsertHtml(input: {
   track: number;
   zIndex: number;
   geometry?: { left: number; top: number; width: number; height: number };
+  /** Video only: true inserts `data-has-audio="true"` with no `muted`. Unknown or false stays muted. */
+  hasAudio?: boolean;
 }): string {
   const sharedAttrs = `id="${input.id}" data-hf-id="${input.hfId}" class="clip" src="${input.assetPath}" data-start="${input.start}" data-duration="${input.duration}" data-track-index="${input.track}"`;
   const geometry = input.geometry ?? { left: 0, top: 0, width: 640, height: 360 };
@@ -126,7 +128,10 @@ export function buildTimelineAssetInsertHtml(input: {
   }
 
   if (input.kind === "video") {
-    return `<video ${sharedAttrs} muted playsinline style="${visualStyles}"></video>`;
+    // `muted` and `data-has-audio="true"` are mutually exclusive by the lint
+    // contract (video_has_audio_but_muted): an audible drop takes the latter.
+    const audio = input.hasAudio ? 'data-has-audio="true"' : "muted";
+    return `<video ${sharedAttrs} ${audio} playsinline style="${visualStyles}"></video>`;
   }
 
   return `<audio ${sharedAttrs} data-volume="1" style="z-index: ${input.zIndex}"></audio>`;