@dailies-studio/shot-editor 0.1.0 → 0.2.0

package/src/panels/library/LibrarySoundPanel.tsx

@@ -0,0 +1,293 @@
+// The Library's right column for a SOUND PACK: the pack's files in manifest
+// order, an audition per file, the picked file's waveform, the Add that places
+// that one file as a clip in the open Shot, and the Pin that lands it as a
+// sound card. A pack is never a card itself (the validator refuses a `library`
+// card of kind "sounds"), so this column adds one sound at a time and the
+// browser's `onAdd` never sees a pack.
+//
+// ONE AUDITION AT A TIME, through ONE `Audio` element owned by this panel. A
+// second play stops the first; closing the Library or picking another item
+// unmounts the panel, which stops it. It follows the tab's monitor settings
+// (`previewMuted`, `previewVolume`), the ones the transport's speaker sets,
+// because an author who muted the preview did not mean "except the library".
+//
+// THE BYTES COME FROM THE LIBRARY ROUTE, `libraryFileUrl` (`GET /api/library/
+// items/<id>/files/<name>`), never the Project's asset store: auditioning a
+// sound must not import the pack. The waveform is the timeline's own
+// `loadPeaks` and `drawClipWaveform`, keyed on that URL. `loadPeaks` caches by
+// `asset.path` and fetches `/studio/assets/<path>` by default, so the URL rides
+// in as the path and `fetchBytes` fetches it as it is: a root-relative URL
+// can never collide with a Project asset path, so the shared cache stays
+// honest without a second waveform module.
+import { useEffect, useRef, useState } from "react";
+import { libraryFileLabel, libraryFileUrl } from "@dailies-studio/shot/doc";
+import { drawClipWaveform, loadPeaks } from "@dailies-studio/timeline";
+import type { PeakData } from "@dailies-studio/timeline";
+import type { LibraryItem, LibraryItemDetail } from "../../api.js";
+import { ADD_ICON, CHEVRON_DOWN_ICON, PLAY_ICON, STOP_ICON } from "../../icons.js";
+import { useShotEditor, useStore } from "../../state/context.js";
+import { openBoardForPin, placeInShotRefusal } from "../../state/store.js";
+import { formatDuration } from "../timeline/model.js";
+import { displayNameOf, plural } from "./libraryModel.js";
+
+type SoundFile = NonNullable<LibraryItemDetail["sounds"]>["files"][number];
+
+const NONE: never[] = [];
+
+/** Rows shown before "Show all": Paper's fold, enough to read the pack's
+ *  range without pushing the waveform and the Add off a laptop screen. */
+export const SOUND_FOLD = 7;
+
+/** The bare file name `addStoryboardLibrarySound` matches the import on. */
+const fileNameOf = (file: SoundFile): string => file.path.slice(file.path.lastIndexOf("/") + 1);
+
+export function LibrarySoundPanel({
+  item,
+  canAdd,
+  adding,
+}: {
+  item: LibraryItem;
+  canAdd: boolean;
+  adding: boolean;
+}) {
+  const store = useStore();
+  const detail = useShotEditor((s) => s.libraryItems[item.id]);
+  const muted = useShotEditor((s) => s.previewMuted);
+  const volume = useShotEditor((s) => s.previewVolume);
+  const [failed, setFailed] = useState(false);
+  const [picked, setPicked] = useState<string | null>(null);
+  const [playing, setPlaying] = useState<string | null>(null);
+  const [showAll, setShowAll] = useState(false);
+  // ponytail: this add's in-flight flag is the panel's own, so the browser's
+  // `adding` does not see it; lift it into the browser if a second add door
+  // ever shares this column.
+  const [busy, setBusy] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const placeRefusal = useShotEditor((s) => placeInShotRefusal(s, "sound"));
+  const boardOpen = useShotEditor((s) => s.storyboardOpen);
+  const audio = useRef<HTMLAudioElement | null>(null);
+
+  useEffect(() => {
+    let alive = true;
+    void store.loadLibraryItem(item.id).then((d) => {
+      if (alive && d === null) setFailed(true);
+    });
+    return () => {
+      alive = false;
+    };
+  }, [item.id, store]);
+  // Stop on unmount: the Library closed, or another item was picked.
+  useEffect(
+    () => () => {
+      audio.current?.pause();
+      audio.current = null;
+    },
+    [],
+  );
+  // The monitor settings follow live, mid-audition included.
+  useEffect(() => {
+    if (audio.current === null) return;
+    audio.current.muted = muted;
+    audio.current.volume = volume;
+  }, [muted, volume]);
+
+  const files = detail?.sounds?.files ?? NONE;
+  const name = displayNameOf(item);
+  const selected = files.find((f) => f.path === picked) ?? files[0];
+  const shown = showAll ? files : files.slice(0, SOUND_FOLD);
+
+  const stop = (): void => {
+    audio.current?.pause();
+    audio.current = null;
+    setPlaying(null);
+  };
+  const audition = (file: SoundFile): void => {
+    setPicked(file.path);
+    if (playing === file.path) {
+      stop();
+      return;
+    }
+    audio.current?.pause();
+    const el = new Audio(libraryFileUrl(item.id, file.path));
+    el.muted = muted;
+    el.volume = volume;
+    const done = (): void => {
+      if (audio.current !== el) return;
+      audio.current = null;
+      setPlaying(null);
+    };
+    el.addEventListener("ended", done);
+    el.addEventListener("error", done);
+    audio.current = el;
+    setPlaying(file.path);
+    // A refused play (autoplay policy, an unreadable file) ends the audition
+    // quietly: the row's button goes back to Play, which is the whole truth.
+    void el.play().catch(done);
+  };
+  // ADD places the sound as a clip at the playhead of the open Shot (D2); PIN
+  // makes it a sound card for the agent. Both import that one file only.
+  const run = (act: () => Promise<unknown>, placed: boolean): void => {
+    setBusy(true);
+    setError(null);
+    // The store reports its own refusals to the shell's error strip, which
+    // sits under this dialog, so the strip is cleared first and whatever the
+    // add leaves there is printed here too.
+    store.dismissError();
+    void act()
+      .then(() => {
+        const message = store.getState().error;
+        if (message !== null) setError(message);
+        // A placed sound is on the timeline under this dialog: show it.
+        else if (placed && store.getState().libraryBrowserOpen) store.setLibraryBrowserOpen(false);
+      })
+      .finally(() => setBusy(false));
+  };
+  const add = (file: SoundFile): void => run(() => store.dropLibraryOnFrame(item.id, fileNameOf(file)), true);
+  const pin = (file: SoundFile): void =>
+    run(async () => {
+      if (openBoardForPin(store)) await store.addStoryboardLibrarySound(null, item.id, fileNameOf(file));
+    }, false);
+
+  let body;
+  if (detail === undefined) {
+    body = <p className="library-preview-note">{failed ? "The sounds could not be read." : "Loading the sounds…"}</p>;
+  } else if (files.length === 0) {
+    body = <p className="library-preview-note">This pack ships no sounds.</p>;
+  } else {
+    body = (
+      <>
+        <ul className="library-sound-list" aria-label={`Sounds in ${name}`}>
+          {shown.map((file) => {
+            const label = libraryFileLabel(file);
+            const on = file === selected;
+            const live = playing === file.path;
+            return (
+              <li key={file.path} className={on ? "library-sound-row is-selected" : "library-sound-row"} data-file={file.path}>
+                <button
+                  type="button"
+                  className="library-sound-play"
+                  aria-label={live ? `Stop ${label}` : `Play ${label}`}
+                  onClick={() => audition(file)}
+                >
+                  {live ? STOP_ICON : PLAY_ICON}
+                </button>
+                <button type="button" className="library-sound-pick" aria-pressed={on} onClick={() => setPicked(file.path)}>
+                  <span className="library-sound-name">{label}</span>
+                  <span className="library-sound-length">{formatDuration(file.durationMs / 1000)}</span>
+                </button>
+              </li>
+            );
+          })}
+        </ul>
+        {files.length > SOUND_FOLD && !showAll ? (
+          <button type="button" className="library-sound-more" onClick={() => setShowAll(true)}>
+            Show all {files.length} sounds
+            {CHEVRON_DOWN_ICON}
+          </button>
+        ) : null}
+        {selected === undefined ? null : (
+          <div className="library-sound-selected" data-testid="library-sound-selected">
+            <p className="library-sound-selected-name">
+              <span>{libraryFileLabel(selected)}</span>
+              <span className="library-sound-length">{formatDuration(selected.durationMs / 1000)}</span>
+            </p>
+            <SoundWave url={libraryFileUrl(item.id, selected.path)} />
+            <button
+              type="button"
+              className="btn is-primary is-lg"
+              data-testid="library-add-sound"
+              disabled={!canAdd || adding || busy}
+              aria-disabled={placeRefusal !== null || undefined}
+              title={!canAdd ? "Open a Project to add library items." : (placeRefusal ?? "Import this sound and place it at the playhead of the open Shot.")}
+              onClick={() => {
+                if (placeRefusal === null) add(selected);
+              }}
+            >
+              {ADD_ICON}
+              {busy ? "Adding…" : "Add to shot"}
+            </button>
+            <button
+              type="button"
+              className="btn is-outline is-lg"
+              data-testid="library-pin-sound"
+              disabled={!canAdd || adding || busy}
+              title={canAdd ? `Import this sound and pin it as a card the agent reads.${boardOpen ? "" : " Opens the storyboard."}` : "Open a Project to add library items."}
+              onClick={() => pin(selected)}
+            >
+              Pin to board
+            </button>
+            {error === null ? null : (
+              <p className="library-preview-error" role="alert" data-testid="library-add-error">
+                {error}
+              </p>
+            )}
+            <p className="library-preview-hint">Add places it in the open Shot. Pin saves it for later.</p>
+          </div>
+        )}
+      </>
+    );
+  }
+
+  return (
+    <aside className="library-preview is-sounds" aria-label={`Preview: ${name}`} data-testid="library-preview">
+      <div className="library-preview-purpose">
+        <h2 className="library-preview-name">{name}</h2>
+        {files.length === 0 ? null : <p className="library-preview-note">{plural(files.length, "sound")} · Choose one to preview</p>}
+      </div>
+      {body}
+      <footer className="library-preview-meta">
+        <span>Sound pack</span>
+        {item.version === undefined ? null : <span>v{item.version}</span>}
+      </footer>
+    </aside>
+  );
+}
+
+/** The picked sound's envelope. Decoration: a file that will not decode
+ *  leaves the well empty, and the canvas is hidden from assistive tech. */
+export function SoundWave({ url }: { url: string }) {
+  const canvas = useRef<HTMLCanvasElement | null>(null);
+  const [data, setData] = useState<PeakData | null>(null);
+  useEffect(() => {
+    let alive = true;
+    setData(null);
+    loadPeaks(
+      { path: url },
+      {
+        fetchBytes: async () => {
+          // A stock sound's file is on its site's CDN: asked for with no
+          // cookie and no page address, as the search is. A pack sound is
+          // the studio's own, read as every asset is.
+          const own = new URL(url, location.href).origin === location.origin;
+          const res = await fetch(url, own ? {} : { credentials: "omit", referrerPolicy: "no-referrer" });
+          if (!res.ok) throw new Error(`${url}: ${res.status}`);
+          return await res.arrayBuffer();
+        },
+      },
+    )
+      .then((d) => {
+        if (alive) setData(d);
+      })
+      .catch(() => {});
+    return () => {
+      alive = false;
+    };
+  }, [url]);
+  useEffect(() => {
+    const el = canvas.current;
+    if (el === null || data === null || data.peaks.length === 0) return;
+    drawClipWaveform(el, {
+      data,
+      fromSeconds: 0,
+      toSeconds: data.sourceSeconds || 1,
+      widthPx: el.clientWidth || 240,
+      heightPx: el.clientHeight || 46,
+      color: getComputedStyle(el).color,
+    });
+  }, [data]);
+  return (
+    <div className="library-sound-wave">
+      <canvas ref={canvas} aria-hidden="true" data-testid="library-sound-wave" data-drawn={data !== null && data.peaks.length > 0 ? "true" : "false"} />
+    </div>
+  );
+}

package/src/panels/library/SoundHistory.tsx

@@ -0,0 +1,245 @@
+// The Library's Generated sounds: the workspace's sound history. The sound
+// effects, music beds and voice lines this member made, or that were made for
+// a Project they may view: hosted, every one added to a Project, and the rest
+// for a day after they were made; locally every take in the store, since a
+// take there is a file. It fills the catalog and preview columns, like Stock
+// sounds.
+//
+// ADD places the take as a bed clip at the playhead (`placeTake`, the Generate
+// drawer's own verb), a scored bed at frame 0 where its sections line up, so a
+// take is reused without paying for it again. DELETE takes it out of the
+// history: hosted its waiting bytes go, locally its file stays for any Project
+// that uses it.
+//
+// A PAGE CAN BE EMPTY AND STILL GO ON: hosted, a page reads a bounded number
+// of rows and keeps the ones this member may see, so "Load more" shows
+// whenever there is a `next`, list or no list.
+import { useEffect, useRef, useState } from "react";
+import { SegmentedControl } from "@dailies-studio/ui";
+import { deleteGeneration, generationHistory } from "../../api.js";
+import type { GenerationKind, GenerationTake } from "../../api.js";
+import { ADD_ICON, PLAY_ICON, STOP_ICON } from "../../icons.js";
+import { useShotEditor, useStore } from "../../state/context.js";
+import { formatDuration } from "../timeline/model.js";
+
+type Filter = "all" | GenerationKind;
+
+const KIND_WORDS: Record<GenerationKind, string> = { sfx: "Sound effect", music: "Music", voice: "Voice" };
+
+export function SoundHistory({ canAdd }: { canAdd: boolean }) {
+  const store = useStore();
+  const muted = useShotEditor((s) => s.previewMuted);
+  const volume = useShotEditor((s) => s.previewVolume);
+  const [takes, setTakes] = useState<GenerationTake[] | null>(null);
+  const [next, setNext] = useState<string | null>(null);
+  /** A page on its way: a second "Load more" waits for it. */
+  const [loading, setLoading] = useState(false);
+  const [failed, setFailed] = useState<string | null>(null);
+  const [filter, setFilter] = useState<Filter>("all");
+  const [picked, setPicked] = useState<string | null>(null);
+  const [playing, setPlaying] = useState<string | null>(null);
+  const [busy, setBusy] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const audio = useRef<HTMLAudioElement | null>(null);
+
+  const load = (before?: string): void => {
+    setLoading(true);
+    void generationHistory(before)
+      .then(
+        (page) => {
+          setTakes((held) => (before === undefined ? page.takes : [...(held ?? []), ...page.takes]));
+          setNext(page.next);
+        },
+        (e: unknown) => setFailed(e instanceof Error ? e.message : String(e)),
+      )
+      .finally(() => setLoading(false));
+  };
+  useEffect(() => {
+    load();
+    return () => {
+      audio.current?.pause();
+      audio.current = null;
+    };
+  }, []);
+  useEffect(() => {
+    if (audio.current === null) return;
+    audio.current.muted = muted;
+    audio.current.volume = volume;
+  }, [muted, volume]);
+
+  const shown = (takes ?? []).filter((t) => filter === "all" || t.kind === filter);
+  const selected = shown.find((t) => t.id === picked) ?? shown[0] ?? null;
+
+  const audition = (take: GenerationTake): void => {
+    setPicked(take.id);
+    audio.current?.pause();
+    if (playing === take.id) {
+      audio.current = null;
+      setPlaying(null);
+      return;
+    }
+    const el = new Audio(take.url);
+    el.muted = muted;
+    el.volume = volume;
+    const done = (): void => {
+      if (audio.current !== el) return;
+      audio.current = null;
+      setPlaying(null);
+    };
+    el.addEventListener("ended", done);
+    el.addEventListener("error", done);
+    audio.current = el;
+    setPlaying(take.id);
+    void el.play().catch(done);
+  };
+
+  const add = (take: GenerationTake): void => {
+    setBusy(true);
+    setError(null);
+    void store
+      .placeTake(take, undefined, take.scored === true ? 0 : undefined)
+      .then(() => {
+        if (store.getState().libraryBrowserOpen) store.setLibraryBrowserOpen(false);
+      })
+      .catch((e: unknown) => setError(e instanceof Error ? e.message : String(e)))
+      .finally(() => setBusy(false));
+  };
+
+  const remove = (take: GenerationTake): void => {
+    setBusy(true);
+    setError(null);
+    void deleteGeneration(take.id)
+      .then(() => setTakes((held) => (held ?? []).filter((t) => t.id !== take.id)))
+      .catch((e: unknown) => setError(e instanceof Error ? e.message : String(e)))
+      .finally(() => setBusy(false));
+  };
+
+  const more =
+    next === null ? null : (
+      <button type="button" className="btn is-outline" data-testid="history-more" disabled={loading} onClick={() => load(next)}>
+        {loading ? "Loading…" : "Load more"}
+      </button>
+    );
+  let body;
+  if (takes === null) {
+    body = (
+      <p className="library-preview-note" role="status">
+        {failed ?? "Reading the sound history…"}
+      </p>
+    );
+  } else if (shown.length === 0) {
+    body = (
+      <>
+        <p className="library-preview-note" role="status">
+          {takes.length === 0
+            ? next === null
+              ? "Nothing generated yet. Sound mode's Generate makes effects, music and voice lines."
+              : "None of the newest takes is yours to see. Load more to look further back."
+            : "No takes of this kind yet."}
+        </p>
+        {more}
+      </>
+    );
+  } else {
+    body = (
+      <>
+        <ul className="library-sound-list" aria-label="Generated sounds" data-testid="history-list">
+          {shown.map((take) => {
+            const live = playing === take.id;
+            const on = take === selected;
+            return (
+              <li key={take.id} className={on ? "library-sound-row is-selected" : "library-sound-row"} data-id={take.id}>
+                <button type="button" className="library-sound-play" aria-label={live ? `Stop ${take.prompt}` : `Play ${take.prompt}`} onClick={() => audition(take)}>
+                  {live ? STOP_ICON : PLAY_ICON}
+                </button>
+                <button type="button" className="library-sound-pick" aria-pressed={on} onClick={() => setPicked(take.id)}>
+                  <span className="library-sound-name">{take.prompt}</span>
+                  <span className="stock-sound-license">{KIND_WORDS[take.kind]}</span>
+                  {take.seconds === null ? null : <span className="library-sound-length">{formatDuration(take.seconds)}</span>}
+                </button>
+              </li>
+            );
+          })}
+        </ul>
+        {more}
+      </>
+    );
+  }
+
+  return (
+    <>
+      <section className="library-catalog" aria-label="Generated sounds" data-testid="sound-history">
+        <header className="library-catalog-head">
+          <div>
+            <h2>Generated sounds</h2>
+            <p>Generated sounds you can open: every one added to a Project, and the rest for a day.</p>
+          </div>
+        </header>
+        <SegmentedControl
+          ariaLabel="Kind of take"
+          className="stock-kind"
+          value={filter}
+          options={[
+            { value: "all", label: "All" },
+            { value: "sfx", label: "Effects" },
+            { value: "music", label: "Music" },
+            { value: "voice", label: "Voice" },
+          ]}
+          onChange={(next) => setFilter(next as Filter)}
+        />
+        {body}
+      </section>
+      <aside className="library-preview is-sounds" aria-label="Preview: generated sound" data-testid="history-preview">
+        {selected === null ? (
+          <p className="library-preview-empty">Pick a take to preview it here.</p>
+        ) : (
+          <>
+            <div className="library-preview-purpose">
+              <h2 className="library-preview-name">{selected.prompt}</h2>
+              <p className="library-preview-note">
+                {KIND_WORDS[selected.kind]}
+                {selected.seconds === null ? "" : ` · ${formatDuration(selected.seconds)}`} · {new Date(selected.createdAt).toLocaleDateString()}
+              </p>
+            </div>
+            <div className="library-sound-selected" data-testid="history-selected">
+              <button
+                type="button"
+                className="btn is-primary is-lg"
+                data-testid="history-add"
+                disabled={!canAdd || busy}
+                title={
+                  !canAdd
+                    ? "Open a Project to add sounds."
+                    : selected.scored === true
+                      ? "Scored to a film's cuts: added at frame 0, where its sections line up."
+                      : "Add this take at the playhead."
+                }
+                onClick={() => add(selected)}
+              >
+                {ADD_ICON}
+                {busy ? "Working…" : selected.scored === true ? "Add at the start" : "Add at playhead"}
+              </button>
+              <button type="button" className="btn is-outline is-lg" data-testid="history-delete" disabled={busy} onClick={() => remove(selected)}>
+                Delete from history
+              </button>
+              {error === null ? null : (
+                <p className="library-preview-error" role="alert" data-testid="history-error">
+                  {error}
+                </p>
+              )}
+              <p className="library-preview-hint">
+                {selected.kept === true
+                  ? "Kept: it stays here until you delete it."
+                  : "Not added to a Project yet: it leaves the history a day after it was made."}
+              </p>
+            </div>
+          </>
+        )}
+        <footer className="library-preview-meta">
+          <span>Sound history</span>
+          <span>{takes === null ? "" : `${takes.length}${next === null ? "" : "+"} take${takes.length === 1 ? "" : "s"}`}</span>
+        </footer>
+      </aside>
+    </>
+  );
+}