@dailies-studio/shot-editor 0.1.0 → 0.2.0

package/src/panels/timeline/GenerateDrawer.tsx

@@ -0,0 +1,407 @@
+// THE GENERATE DRAWER: Sound mode's right column, swapped in for the mixer.
+//
+// A sentence becomes a sound effect, a music bed or a voice line, through
+// `/studio/audio/generate` on either host (api.ts). The take is previewed
+// with the browser's own <audio> before it touches the document, and nothing
+// is placed until "Add to <track> at playhead", which is one undo entry.
+//
+// Locally the take is already a stored asset (the CLI stores it as it
+// generates), so placing it is `placeStoredAudio`. Hosted, the take waits in
+// R2 for a day and placing it is `importAudio` of its bytes, which uploads it
+// into the Project like a dropped file. Either way a past take in the recent
+// list is placed again without generating it twice.
+//
+// It runs on the user's own ElevenLabs key: the environment's locally, the
+// workspace's hosted, which an Owner or Admin adds in Settings, AI keys.
+//
+// A SOUND EFFECT IS TWO TAKES by default: both are made at once, measured in
+// this tab and ranked (`judgeTakes`, a chunk of its own), so the one that
+// measures better leads, with the reason and a tone check when it skews
+// bright. Both stay playable and addable: the ranking advises, the ear decides.
+//
+// MUSIC CAN BE SCORED TO THE PROJECT: "Score to the cuts" sends the film's
+// sections (`scoreSectionsOf`), so the track changes where the film does, and
+// adds it at frame 0, where its sections line up.
+import { useEffect, useMemo, useState } from "react";
+import { SegmentedControl } from "@dailies-studio/ui";
+import { audioTrackOf, audioTracksOf, scoreSectionsOf } from "@dailies-studio/shot/doc";
+import { generateAudioTake, generationStatus } from "../../api.js";
+import type { GenerationBody, GenerationKind, GenerationStatus, GenerationTake } from "../../api.js";
+import { CLOSE_ICON, SPARKLES_ICON } from "../../icons.js";
+import { useShotEditor, useStore } from "../../state/context.js";
+
+const KIND_LABEL: Record<GenerationKind, string> = { sfx: "Sound effect", music: "Music", voice: "Voice" };
+const PLACEHOLDER: Record<GenerationKind, string> = {
+  sfx: "Short airy whoosh, left to right",
+  music: "Warm lo-fi beat, soft keys, 90 bpm",
+  voice: "What the voice says, as it should be spoken.",
+};
+
+function errorText(e: unknown): string {
+  return e instanceof Error ? e.message : String(e);
+}
+
+/** The dashboard's AI keys tab. The editor host sends `/projects/*` to the
+ *  main host, so the relative link works there (as `MANAGE_FONTS_HREF`). */
+export const ADD_KEY_HREF = "/projects/?view=settings-keys";
+
+/** Whose ElevenLabs key a take runs on, under the Generate button. */
+export function keyLine(status: GenerationStatus, takes = 1): string {
+  const whose = status.host === "local" ? "Uses ELEVENLABS_API_KEY on this machine" : "Uses this workspace's ElevenLabs key";
+  return `${whose}${takes > 1 ? " · two takes are two generations" : ""}`;
+}
+
+export function GenerateDrawer(props: { onClose: () => void }) {
+  const store = useStore();
+  const doc = useShotEditor((s) => s.doc);
+  const project = useShotEditor((s) => s.name);
+  const selected = useShotEditor((s) => s.selectedAudio);
+  const [status, setStatus] = useState<GenerationStatus | null>(null);
+  const [kind, setKind] = useState<GenerationKind>("sfx");
+  const [prompts, setPrompts] = useState<Record<GenerationKind, string>>({ sfx: "", music: "", voice: "" });
+  const [seconds, setSeconds] = useState<{ sfx: number; music: number }>({ sfx: 4, music: 30 });
+  const [voice, setVoice] = useState<string | null>(null);
+  const [instrumental, setInstrumental] = useState(true);
+  const [loop, setLoop] = useState(false);
+  /** Sound effects: two takes, the better kept (the default). */
+  const [twoTakes, setTwoTakes] = useState(true);
+  /** Music: compose to the Project's cuts instead of a length. */
+  const [fit, setFit] = useState(false);
+  const [busy, setBusy] = useState<"generate" | "place" | null>(null);
+  const [error, setError] = useState<string | null>(null);
+  const [made, setMade] = useState<{ order: GenerationTake[]; reasons: string[]; tone: string | null; scored: boolean } | null>(null);
+  const [trackId, setTrackId] = useState<string | null>(null);
+
+  useEffect(() => {
+    if (project === null) return;
+    let live = true;
+    generationStatus(project).then(
+      (s) => {
+        if (!live) return;
+        setStatus(s);
+        setSeconds({ sfx: s.caps.sfx.defaultSeconds, music: s.caps.music.defaultSeconds });
+      },
+      (e: unknown) => {
+        if (live) setError(errorText(e));
+      },
+    );
+    return () => {
+      live = false;
+    };
+  }, [project]);
+
+  const tracks = doc === null ? [] : audioTracksOf(doc);
+  // The selected clip's track, else the first: where "Add" lands unless the
+  // author picks another.
+  const defaultTrack = useMemo(() => {
+    if (doc === null) return undefined;
+    const clip = selected.length > 0 ? doc.audio?.[selected[0]!] : undefined;
+    return audioTrackOf(doc, clip?.track).id;
+  }, [doc, selected]);
+  const target = tracks.find((t) => t.id === trackId) ?? tracks.find((t) => t.id === defaultTrack) ?? tracks[0];
+  const sections = useMemo(() => (doc === null ? null : scoreSectionsOf(doc)), [doc]);
+
+  if (doc === null || project === null) return null;
+  const prompt = prompts[kind];
+  const available = status?.kinds[kind];
+  const voices = status?.voices ?? [];
+  const pickedVoice = voice ?? voices[0]?.id;
+  // Scoring needs a film music can be as long as: under 3 s there is no
+  // section, and past the music cap there is no take.
+  const filmMs = sections === null ? 0 : sections.reduce((sum, s) => sum + s.durationMs, 0);
+  const scoreRefusal =
+    sections === null
+      ? "The film is shorter than 3 seconds, the shortest a section can be"
+      : status !== null && filmMs > status.caps.music.maxSeconds * 1000
+        ? `The film is ${Math.round(filmMs / 1000)} s; music can be at most ${status.caps.music.maxSeconds} s`
+        : null;
+  const scored = kind === "music" && fit && sections !== null && scoreRefusal === null;
+  const takes = kind === "sfx" && twoTakes ? 2 : 1;
+  const body: GenerationBody = {
+    project,
+    kind,
+    prompt,
+    ...(kind === "voice"
+      ? pickedVoice === undefined
+        ? {}
+        : { voice: pickedVoice }
+      : scored
+        ? {
+            seconds: filmMs / 1000,
+            instrumental,
+            sections: sections.map(({ name, durationMs }) => ({ name, durationMs })),
+          }
+        : { seconds: seconds[kind], ...(kind === "music" ? { instrumental } : { loop }) }),
+  };
+  const maxChars = status === null ? 0 : kind === "voice" ? status.caps.voice.maxChars : status.caps.promptMaxChars;
+  const caps = status !== null && kind !== "voice" ? status.caps[kind] : null;
+  const canGenerate = busy === null && available?.ok === true && prompt.trim() !== "";
+
+  const generate = async (): Promise<void> => {
+    setBusy("generate");
+    setError(null);
+    try {
+      const asked = { ...body };
+      const settled = await Promise.allSettled(Array.from({ length: takes }, () => generateAudioTake(asked)));
+      const arrived = settled.flatMap((r) => (r.status === "fulfilled" ? [r.value.take] : []));
+      if (arrived.length === 0) throw (settled[0] as PromiseRejectedResult).reason;
+      // The judge is its own chunk, fetched the first time a sound effect
+      // lands: it ranks two takes, and gives one take its tone check. A chunk
+      // that does not arrive (a deploy replaced it under this tab) leaves the
+      // takes shown, unranked, rather than lost behind an error.
+      let ranked: { order: GenerationTake[]; reasons: string[]; tone: string | null } = { order: arrived, reasons: [], tone: null };
+      if (kind === "sfx") {
+        try {
+          ranked = await (await import("./takeMeasure.js")).judgeTakes(arrived, asked.seconds ?? 0);
+        } catch {
+          if (arrived.length > 1) ranked = { ...ranked, reasons: ["These takes could not be measured here, so listen to both."] };
+        }
+      }
+      const failed = settled.flatMap((r) => (r.status === "rejected" ? [errorText(r.reason)] : []));
+      setMade({ ...ranked, reasons: [...ranked.reasons, ...failed.map((f) => `A take failed: ${f}`)], scored });
+      setStatus((s) =>
+        s === null ? s : { ...s, recent: [...arrived, ...s.recent.filter((r) => !arrived.some((a) => a.id === r.id))] },
+      );
+    } catch (e) {
+      setError(errorText(e));
+    } finally {
+      setBusy(null);
+    }
+  };
+
+  const place = async (t: GenerationTake, atStart = false): Promise<void> => {
+    setBusy("place");
+    setError(null);
+    try {
+      await store.placeTake(t, target?.id, atStart ? 0 : undefined);
+    } catch (e) {
+      setError(errorText(e));
+    } finally {
+      setBusy(null);
+    }
+  };
+
+  return (
+    <div className="sound-gen" data-testid="generate-drawer">
+      <div className="sound-gen-head">
+        <span className="sound-gen-glyph" aria-hidden="true">
+          {SPARKLES_ICON}
+        </span>
+        <span className="sound-gen-title">Generate</span>
+        <button type="button" className="btn is-ghost is-icon" aria-label="Close" title="Back to the mixer" onClick={props.onClose}>
+          {CLOSE_ICON}
+        </button>
+      </div>
+      <div className="sound-gen-body">
+        <SegmentedControl
+          className="sound-gen-kinds"
+          ariaLabel="What to generate"
+          value={kind}
+          onChange={(k) => setKind(k === "music" || k === "voice" ? k : "sfx")}
+          options={(["sfx", "music", "voice"] as const).map((k) => ({
+            value: k,
+            label: KIND_LABEL[k],
+            testId: `generate-kind-${k}`,
+          }))}
+        />
+        <textarea
+          className="text-input sound-gen-prompt"
+          data-testid="generate-prompt"
+          aria-label={kind === "voice" ? "Script" : "Describe the sound"}
+          placeholder={PLACEHOLDER[kind]}
+          value={prompt}
+          maxLength={maxChars > 0 ? maxChars : undefined}
+          rows={kind === "voice" ? 4 : 2}
+          onChange={(e) => {
+            const value = e.currentTarget.value;
+            setPrompts((p) => ({ ...p, [kind]: value }));
+          }}
+          onKeyDown={(e) => {
+            // The dock's keymap must not see typing (Space plays, Q/W trim).
+            e.stopPropagation();
+            if (e.key === "Enter" && (e.metaKey || e.ctrlKey) && canGenerate) {
+              e.preventDefault();
+              void generate();
+            }
+          }}
+        />
+        {kind === "voice" ? (
+          <label className="sound-gen-field">
+            <span className="sound-gen-label">Voice</span>
+            <select
+              className="select sound-gen-select"
+              data-testid="generate-voice"
+              value={pickedVoice ?? ""}
+              onChange={(e) => setVoice(e.currentTarget.value)}
+            >
+              {voices.map((v) => (
+                <option key={v.id} value={v.id}>
+                  {v.name}, {v.description}
+                </option>
+              ))}
+            </select>
+          </label>
+        ) : (
+          <>
+            <div className="sound-gen-row">
+              <label className="sound-gen-field">
+                <span className="sound-gen-label">Seconds</span>
+                <input
+                  className="text-input sound-gen-seconds"
+                  data-testid="generate-seconds"
+                  type="number"
+                  min={caps?.minSeconds}
+                  max={caps?.maxSeconds}
+                  step={kind === "sfx" ? 0.5 : 1}
+                  disabled={scored}
+                  value={scored ? (body.seconds ?? 0) : seconds[kind]}
+                  onKeyDown={(e) => e.stopPropagation()}
+                  onChange={(e) => {
+                    const n = Number(e.currentTarget.value);
+                    if (!Number.isFinite(n) || caps === null) return;
+                    setSeconds((s) => ({ ...s, [kind]: Math.min(caps.maxSeconds, Math.max(caps.minSeconds, n)) }));
+                  }}
+                />
+              </label>
+              <label className="sound-gen-check">
+                <input
+                  type="checkbox"
+                  checked={kind === "music" ? instrumental : loop}
+                  onChange={(e) => (kind === "music" ? setInstrumental : setLoop)(e.currentTarget.checked)}
+                />
+                {kind === "music" ? "Instrumental" : "Loop"}
+              </label>
+            </div>
+            {kind === "sfx" ? (
+              <label className="sound-gen-check" title="Make two takes and lead with the one that measures better">
+                <input type="checkbox" data-testid="generate-two-takes" checked={twoTakes} onChange={(e) => setTwoTakes(e.currentTarget.checked)} />
+                Two takes, keep the better
+              </label>
+            ) : (
+              <label
+                className="sound-gen-check"
+                title={scoreRefusal ?? "Compose to the Project's cuts and named markers"}
+              >
+                <input
+                  type="checkbox"
+                  data-testid="generate-fit"
+                  checked={scored}
+                  disabled={scoreRefusal !== null}
+                  onChange={(e) => setFit(e.currentTarget.checked)}
+                />
+                Score to the cuts{sections === null ? "" : ` (${sections.length} section${sections.length === 1 ? "" : "s"})`}
+              </label>
+            )}
+          </>
+        )}
+        <div className="sound-gen-price" data-testid="generate-price">
+          {status === null
+            ? "Checking what this studio can make…"
+            : available?.ok !== true
+              ? (available?.reason ?? "Not available here")
+              : keyLine(status, takes)}
+          {status?.needsKey === "add" ? (
+            <>
+              {" "}
+              <a className="sound-gen-key-link" data-testid="generate-add-key" href={ADD_KEY_HREF} target="_blank" rel="noopener">
+                Open AI keys ↗
+              </a>
+            </>
+          ) : null}
+        </div>
+        <button
+          type="button"
+          className="btn is-primary sound-gen-go"
+          data-testid="generate-go"
+          disabled={!canGenerate}
+          onClick={() => void generate()}
+        >
+          {busy === "generate" ? "Generating…" : `Generate ${KIND_LABEL[kind].toLowerCase()}`}
+        </button>
+        {error === null ? null : (
+          <p className="sound-gen-error" role="alert">
+            {error}
+          </p>
+        )}
+        {made === null ? null : (
+          <div className="sound-gen-take" data-testid="generate-take">
+            {made.order.map((take, i) => (
+              <div key={take.id} className="sound-gen-take-row" data-testid="generate-take-row">
+                <span className="sound-gen-take-name" title={take.prompt}>
+                  {made.order.length > 1 ? (i === 0 ? "Kept · " : "Other take · ") : ""}
+                  {take.prompt}
+                </span>
+                <audio className="sound-gen-audio" controls preload="metadata" src={take.url} />
+                <button
+                  type="button"
+                  className={i === 0 ? "btn is-primary" : "btn is-outline"}
+                  data-testid={i === 0 ? "generate-place" : "generate-place-other"}
+                  disabled={busy !== null || target === undefined}
+                  onClick={() => void place(take, made.scored)}
+                >
+                  Add to {target?.name ?? "track"} {made.scored ? "at the start" : "at playhead"}
+                </button>
+              </div>
+            ))}
+            {made.reasons.length === 0 ? null : (
+              <p className="sound-gen-why" data-testid="generate-why">
+                {made.reasons.join(" ")}
+              </p>
+            )}
+            {made.tone === null ? null : (
+              <p className="sound-gen-why" data-testid="generate-tone">
+                {made.tone}
+              </p>
+            )}
+            <div className="sound-gen-take-actions">
+              <button type="button" className="btn is-outline" disabled={!canGenerate} onClick={() => void generate()}>
+                {takes > 1 ? "Two more takes" : "Another take"}
+              </button>
+            </div>
+          </div>
+        )}
+        {tracks.length > 1 ? (
+          <label className="sound-gen-field">
+            <span className="sound-gen-label">Track</span>
+            <select
+              className="select sound-gen-select"
+              data-testid="generate-track"
+              value={target?.id ?? ""}
+              onChange={(e) => setTrackId(e.currentTarget.value)}
+            >
+              {tracks.map((t) => (
+                <option key={t.id} value={t.id}>
+                  {t.name}
+                </option>
+              ))}
+            </select>
+          </label>
+        ) : null}
+        {status !== null && status.recent.length > 0 ? (
+          <div className="sound-gen-recent">
+            <span className="sound-gen-label">Recent</span>
+            <ul className="sound-gen-list">
+              {status.recent.map((r) => (
+                <li key={r.id} className="sound-gen-item">
+                  <span className="sound-gen-item-kind">{KIND_LABEL[r.kind]}</span>
+                  <span className="sound-gen-item-prompt" title={r.prompt}>
+                    {r.prompt}
+                  </span>
+                  <button
+                    type="button"
+                    className="btn is-ghost sound-gen-item-add"
+                    disabled={busy !== null || target === undefined}
+                    title={`Add to ${target?.name ?? "track"} ${r.scored === true ? "at the start, where its sections line up" : "at the playhead"}`}
+                    onClick={() => void place(r, r.scored === true)}
+                  >
+                    Add
+                  </button>
+                </li>
+              ))}
+            </ul>
+          </div>
+        ) : null}
+      </div>
+    </div>
+  );
+}

package/src/panels/timeline/KeyStretch.tsx

@@ -0,0 +1,115 @@
+// THE STRETCH BRACKET (Rows face): the selected keys' span, with a grab tab at
+// each end. Dragging one scales every selected key's frame about the other end
+// (`dragKeyStretch`, `keyStretchMoves`), across tracks, layers and Shots. Graph
+// mode has its own bracket over one curve; both run `retimeLandings`.
+//
+// It is how a motion preset is retimed without the document remembering one:
+// a stamp selects the keys it wrote, and this bracket stands over them.
+//
+// TWO HALVES, because the dock scrolls under a sticky ruler. The tabs and the
+// readout live IN THE RULER (`KeyStretchTabs`): always in sight, and above the
+// tracks, where a tab in the rows would scroll away or slide under the ruler.
+// The guides through the rows live in `.dock-overlay` (`KeyStretchGuides`) and
+// take no pointer. The tabs sit just inside each end of a band over the span,
+// so the ruler's clip never hides one at frame 0. Both read the dock's frames:
+// a Shot's keys plus its start, less the shot context's re-zero. A pointer
+// gesture, like Graph mode's bracket: the keyboard retimes keys by nudging.
+import { useState } from "react";
+import type { PointerEvent as ReactPointerEvent } from "react";
+import { frameToPx, startDrag } from "@dailies-studio/timeline";
+import { isDefId, sceneStart } from "@dailies-studio/shot/doc";
+import { useShotEditor, useStore } from "../../state/context.js";
+import { frameDelta } from "./model.js";
+
+interface Span {
+  min: number;
+  max: number;
+  left: number;
+  right: number;
+}
+
+/** The selected keys' span in dock frames and pixels, or null when there is
+ *  no bracket to draw: not the Rows face, a viewer, or under two frames. */
+function useStretchSpan(ppf: number, origin: number): Span | null {
+  const keys = useShotEditor((s) => s.selectedKeyframes);
+  const doc = useShotEditor((s) => s.doc);
+  const rows = useShotEditor((s) => s.dockMode === "rows");
+  const readOnly = useShotEditor((s) => s.readOnly);
+  if (!rows || readOnly || doc === null || keys.length < 2) return null;
+  const at = keys.map((k) => k.frame + (isDefId(k.containerId) ? 0 : (sceneStart(doc, k.containerId) ?? 0)));
+  const min = Math.min(...at);
+  const max = Math.max(...at);
+  if (min === max) return null;
+  return { min, max, left: frameToPx(min - origin, ppf), right: frameToPx(max - origin, ppf) };
+}
+
+/** The guides through the rows, in `.dock-overlay`. */
+export function KeyStretchGuides({ ppf, origin }: { ppf: number; origin: number }) {
+  const span = useStretchSpan(ppf, origin);
+  if (span === null) return null;
+  return (
+    <>
+      <span className="dock-key-stretch-guide" style={{ left: span.left }} />
+      <span className="dock-key-stretch-guide" style={{ left: span.right }} />
+    </>
+  );
+}
+
+/** The grab tabs and the drag's readout, in the ruler's track area. */
+export function KeyStretchTabs({ ppf, origin }: { ppf: number; origin: number }) {
+  const store = useStore();
+  const span = useStretchSpan(ppf, origin);
+  const [chip, setChip] = useState<{ edge: "left" | "right"; label: string } | null>(null);
+  if (span === null) return null;
+
+  const grab = (edge: "left" | "right") => (e: ReactPointerEvent<HTMLElement>): void => {
+    if (e.button !== 0) return;
+    e.preventDefault();
+    // The ruler's own press seeks: a tab is not the ruler.
+    e.stopPropagation();
+    if (!store.beginBlockDrag()) return;
+    const startX = e.clientX;
+    const width = span.max - span.min;
+    setChip({ edge, label: "×1.00" });
+    startDrag(e, {
+      onMove(ev) {
+        const factor = store.dragKeyStretch(edge, frameDelta(ev.clientX - startX, ppf));
+        if (factor !== null) setChip({ edge, label: `×${factor.toFixed(2)} · ${Math.round(width * factor)}f` });
+      },
+      onEnd(commit) {
+        setChip(null);
+        store.endBlockDrag(commit);
+      },
+    });
+  };
+
+  return (
+    <>
+      <span className="dock-key-stretch-band" style={{ left: span.left, width: span.right - span.left }} />
+      <span
+        className="dock-key-stretch-handle is-left"
+        data-testid="key-stretch-left"
+        title="Drag to stretch the selected keyframes"
+        style={{ left: span.left }}
+        onPointerDown={grab("left")}
+      />
+      <span
+        className="dock-key-stretch-handle is-right"
+        data-testid="key-stretch-right"
+        title="Drag to stretch the selected keyframes"
+        style={{ left: span.right }}
+        onPointerDown={grab("right")}
+      />
+      {chip === null ? null : (
+        <span
+          className="stagger-chip dock-key-stretch-chip"
+          data-testid="key-stretch-readout"
+          role="status"
+          style={{ left: chip.edge === "left" ? span.left : span.right }}
+        >
+          <span className="stagger-chip-step">{chip.label}</span>
+        </span>
+      )}
+    </>
+  );
+}