@motionscript/audio 0.0.0-stage → 0.1.0-alpha.10

package/src/waveform/live-waveform.ts

@@ -0,0 +1,206 @@
+import {
+  Types,
+  component2d,
+  fillOps,
+  type Fill,
+  type FillResolved,
+  type NodeOf,
+  type RenderContext,
+  // Named so the emitted declaration reaches these through the sdk, not core.
+  type Node2D as _Node2D,
+  type Node2DProps as _Node2DProps,
+} from "@motionscript/sdk"
+
+import {
+  EMPTY_ENVELOPE,
+  forSpeaker,
+  sampleEnvelope,
+  type AudioEnvelope,
+  type EnvelopeBar,
+} from "./envelope"
+import {
+  clamp01,
+  groupByPaint,
+  layOutBars,
+  livePaintKey,
+  paintBars,
+  voiceColor,
+  type WaveformAlignment,
+  type WaveformStyle,
+} from "./shared"
+
+/**
+ * A **live audio visualiser**: bars that move with the recording as it plays.
+ *
+ * The thing every podcast-to-video tool puts beside the artwork. It is not a
+ * picture of the file — that is {@link FullWaveform} — it is a picture of *this
+ * moment*, and what it is for is making a static frame of a talking head feel
+ * like it is playing.
+ *
+ * ## "Live" without listening
+ *
+ * Nothing here reads an audio device. The bars are sampled out of a baked
+ * envelope at the node's own elapsed time, which is what makes the visualiser
+ * survive the thing it exists for: an **export**, where there is no playback to
+ * analyse and frames may be rendered out of order or in parallel. A frame is a
+ * pure function of its time, so scrubbing back and forth over the visualiser
+ * redraws exactly what it drew the first time — which an analyser node, by
+ * construction, cannot promise. See {@link AudioEnvelope}.
+ *
+ * ## The speaker filter
+ *
+ * `speaker` narrows the node to one voice of a diarized recording, and it
+ * is the reason this node knows about transcripts at all. Two visualisers beside
+ * two headshots, each filtered to its own host, is a podcast edit that would
+ * otherwise be done by hand for every turn — and because the filter *silences*
+ * rather than skips (see {@link forSpeaker}), the quiet host visibly stops
+ * talking instead of vanishing. Nothing about that requires the words: only who
+ * is talking when, which is what the diarizer produces.
+ *
+ * A filter naming a voice the recording doesn't have draws a resting row rather
+ * than everything — a picture of "this person never speaks here", which is
+ * true, where falling back to the unfiltered mix would be a silent lie about
+ * whose voice you were watching.
+ */
+export const LiveWaveform = component2d({
+  key: "liveWaveform",
+  state: {
+    /** The baked recording. See {@link AudioEnvelope} for why it is baked. */
+    envelope: Types.custom<AudioEnvelope>(EMPTY_ENVELOPE),
+    /** How many bars the visualiser is made of. */
+    bars: 48,
+    /** Space between bars, as a fraction of the pitch. */
+    gap: 0.35,
+    radius: 4,
+    /** How many seconds of audio are on screen at once. */
+    window: 3,
+    /** Seconds into the file the node's first frame shows. */
+    offset: 0,
+    /** Advance with the scene clock. Off freezes the picture at `offset`. */
+    playing: true,
+    /** Multiplies every magnitude before it is clamped. */
+    gain: 1.4,
+    /** How much of the box a silent bar still occupies, `0`–`1`. */
+    floor: 0.04,
+    align: Types.custom<WaveformAlignment>("center"),
+    style: Types.custom<WaveformStyle>("bars"),
+    /**
+     * Show only this voice's activity, by index into the recording's own
+     * numbering. `-1` — the default — is every voice.
+     */
+    speaker: -1,
+    /** Colour each bar by whoever is speaking in it. */
+    byVoice: false,
+    /** A colour per voice index. Short or empty falls back to the palette. */
+    voiceColors: Types.custom<string[]>([]),
+    fill: Types.custom<Fill, FillResolved[]>("#6366f1", { resolve: fillOps.resolve, lerp: fillOps.lerp }),
+  },
+  // Nothing declares a component's paints but itself.
+  load: (loader, state) => loader.fill(state.fill),
+  render: (ctx, state) => {
+    drawBars(ctx, state)
+    ctx.children()
+  },
+})
+
+export type LiveWaveform = NodeOf<typeof LiveWaveform>
+export type LiveWaveformProps = NonNullable<Parameters<typeof LiveWaveform>[0]>
+
+interface LiveState {
+  readonly envelope: AudioEnvelope
+  readonly bars: number
+  readonly gap: number
+  readonly radius: number
+  readonly window: number
+  readonly offset: number
+  readonly playing: boolean
+  readonly gain: number
+  readonly floor: number
+  readonly align: WaveformAlignment
+  readonly style: WaveformStyle
+  readonly speaker: number
+  readonly byVoice: boolean
+  readonly voiceColors: string[]
+  readonly fill: FillResolved[]
+}
+
+/**
+ * The second of the recording the visualiser is showing.
+ *
+ * Off the node's own clock rather than the scene's, which is what "from the
+ * moment this node appeared" means — the same clock the engine's own
+ * `Video` times its picture from, and the same one that makes a node dropped
+ * halfway through a clip start its recording at its entrance rather than
+ * partway in.
+ */
+function at(ctx: RenderContext, s: LiveState): number {
+  return s.offset + (s.playing ? Math.max(0, ctx.time.elapsed) : 0)
+}
+
+function drawBars(ctx: RenderContext, s: LiveState): void {
+  const { width, height } = ctx.size
+  if (width <= 0 || height <= 0) return
+
+  const count = Math.max(1, Math.round(s.bars))
+  const span = Math.max(0.05, s.window)
+  const now = at(ctx, s)
+  const voice = s.speaker < 0 ? null : Math.round(s.speaker)
+
+  // The window is centred on the playhead, so the bars run *toward* the
+  // middle and away again — the arrangement every mixer's meter uses, and the
+  // one that reads as sound arriving rather than as a strip scrolling past.
+  const sampled = forSpeaker(
+    sampleEnvelope(s.envelope, now - span / 2, now + span / 2, count),
+    voice
+  )
+
+  const shaped = s.style === "line" ? taper(sampled) : sampled
+  const boxes = layOutBars(shaped, {
+    width,
+    height,
+    gap: s.style === "blocks" ? 0.12 : s.gap,
+    align: s.align,
+    gain: s.gain,
+    floor: s.floor,
+  })
+  if (boxes.length === 0) return
+
+  const groups = groupByPaint(boxes, (i) =>
+    livePaintKey(shaped[i].speaker, s.byVoice, voice)
+  )
+
+  for (const [key, run] of groups) {
+    const graphics = paintBars(run, paintFor(s, key), s.radius)
+    if (graphics) ctx.draw(graphics)
+  }
+}
+
+/** See `FullWaveform`'s paints — a voice is a hue, never a gradient. */
+function paintFor(s: LiveState, key: string): Fill {
+  if (key === "all") return s.fill
+  const voice = Number(key.slice("voice:".length))
+  if (!Number.isFinite(voice) || voice < 0) return s.fill
+  return s.voiceColors[voice] ?? voiceColor(voice)
+}
+
+/**
+ * Rolls the ends of a window off toward silence.
+ *
+ * What separates the `line` style from `bars`: a hard-edged window makes the
+ * loud bar entering at the right edge appear from nothing, which reads as a
+ * glitch rather than as sound arriving. Fading the outer fifth turns the same
+ * data into a shape that grows out of the baseline — the profile a spectrum
+ * display has, and the reason it looks like one object rather than a row of
+ * separate meters.
+ */
+function taper(bars: readonly EnvelopeBar[]): EnvelopeBar[] {
+  const count = bars.length
+  if (count < 3) return [...bars]
+  const edge = Math.max(1, Math.floor(count / 5))
+
+  return bars.map((bar, i) => {
+    const fromEnd = Math.min(i, count - 1 - i)
+    const weight = clamp01(fromEnd / edge)
+    return { magnitude: bar.magnitude * weight, speaker: bar.speaker }
+  })
+}

package/src/waveform/shared.ts

@@ -0,0 +1,197 @@
+import { Graphics2D, type Fill, type FillResolved } from "@motionscript/sdk"
+
+import type { EnvelopeBar } from "./envelope"
+
+/**
+ * Geometry and paint shared by the two waveform nodes.
+ *
+ * They draw the same object — a row of bars whose heights are loudness — and
+ * differ only in what decides the heights: the whole file for one, a window
+ * around the playhead for the other. Everything about *how a row of bars is
+ * laid out and painted* is therefore common, and lives here so the two cannot
+ * drift into looking like different components.
+ */
+
+/** How a bar is grown out of its baseline. */
+export const WAVEFORM_ALIGNMENTS = ["center", "bottom"] as const
+export type WaveformAlignment = (typeof WAVEFORM_ALIGNMENTS)[number]
+
+/** What the bars are shaped like. */
+export const WAVEFORM_STYLES = ["bars", "line", "blocks"] as const
+export type WaveformStyle = (typeof WAVEFORM_STYLES)[number]
+
+/**
+ * The colours voices are drawn in when nobody has assigned any.
+ *
+ * Eight hues that can be told apart at a glance, handed out by voice index. They
+ * are literals rather than looked up, because a scene has to build in a
+ * headless render with no database in scope; a host that names its voices
+ * supplies its own colours instead.
+ */
+export const VOICE_PALETTE = [
+  "#6366f1",
+  "#14b8a6",
+  "#f59e0b",
+  "#ec4899",
+  "#22c55e",
+  "#3b82f6",
+  "#f97316",
+  "#a855f7",
+] as const
+
+/** The colour a voice takes, wrapping past the end of the palette. */
+export function voiceColor(index: number): string {
+  return VOICE_PALETTE[
+    ((index % VOICE_PALETTE.length) + VOICE_PALETTE.length) %
+      VOICE_PALETTE.length
+  ]
+}
+
+/**
+ * Where one bar sits and how tall it came out: its top-left corner and size, in
+ * the node's own pixels with **y pointing down** from the middle of the box.
+ * {@link paintBars} turns that into the centred, y-up rect the renderer draws.
+ */
+export interface BarBox {
+  x: number
+  y: number
+  width: number
+  height: number
+}
+
+/**
+ * Lays a row of bars out across a box.
+ *
+ * The gap is stated as a *fraction of the pitch* rather than in pixels, and
+ * that is the one decision here worth stating out loud: the bar count is a
+ * control, so a gap in pixels would silently become a solid block at 200 bars
+ * and a picket fence at 20. As a fraction, the row keeps the same rhythm at
+ * every density — which is what lets the count be a slider anyone can drag.
+ *
+ * A bar is never thinner than a hairline and never shorter than one either: a
+ * silent passage has to read as "audio, and nothing here" rather than as a hole
+ * in the drawing.
+ */
+export function layOutBars(
+  bars: readonly EnvelopeBar[],
+  options: {
+    width: number
+    height: number
+    gap: number
+    align: WaveformAlignment
+    /** Multiplies every magnitude before it is clamped back into 0…1. */
+    gain: number
+    /** Fraction of the box a silent bar still occupies. */
+    floor: number
+  }
+): BarBox[] {
+  const { width, height, gap, align, gain, floor } = options
+  const count = bars.length
+  if (count === 0 || width <= 0 || height <= 0) return []
+
+  const pitch = width / count
+  const barWidth = Math.max(1, pitch * (1 - clamp01(gap)))
+  // The tallest a bar may be. Centre-aligned bars grow both ways from the
+  // middle, so the *half* is what a magnitude of 1 fills.
+  const full = align === "center" ? height / 2 : height
+
+  const out: BarBox[] = new Array(count)
+  for (let i = 0; i < count; i++) {
+    const magnitude = clamp01(bars[i].magnitude * gain)
+    const reach = Math.max(full * clamp01(floor), full * magnitude)
+    const x = -width / 2 + pitch * i + (pitch - barWidth) / 2
+
+    // A hairline minimum, taken before the position so the bar stays inside the
+    // box: a silent bar 1px tall that kept the position of a zero-height one
+    // would hang half a pixel (centred) or a whole pixel (bottom) outside it.
+    const h = Math.max(1, align === "center" ? reach * 2 : reach)
+    out[i] =
+      align === "center"
+        ? { x, y: -h / 2, width: barWidth, height: h }
+        : { x, y: height / 2 - h, width: barWidth, height: h }
+  }
+  return out
+}
+
+/**
+ * Draws a run of bars in one paint.
+ *
+ * Batched by colour by the caller, and this is why: every `Graphics2D` handed to
+ * `ctx.draw` is a surface the renderer composites, so a bar apiece would be a
+ * thousand surfaces a frame. Chaining the rects of one colour into a single
+ * `Graphics2D` before filling makes it one — and, as a bonus, a gradient fill
+ * maps across the whole run rather than restarting inside every bar, which is
+ * the behaviour anyone reaching for a gradient here actually wants.
+ */
+export function paintBars(
+  boxes: readonly BarBox[],
+  fill: Fill,
+  radius: number
+): Graphics2D | null {
+  if (boxes.length === 0) return null
+  if ((fill as FillResolved[]).length === 0) return null
+
+  const graphics = new Graphics2D()
+  for (const box of boxes) {
+    graphics.rect({
+      x: box.x + box.width / 2,
+      // The box is stated y-down and the rect is drawn y-up. A centred bar is
+      // symmetric about the middle, so only a bottom-aligned one shows the flip.
+      y: -(box.y + box.height / 2),
+      width: box.width,
+      height: box.height,
+      // Never rounder than the bar is wide, or a thin bar becomes a lozenge
+      // whose ends taper away to nothing.
+      cornerRadius: Math.min(radius, box.width / 2, box.height / 2),
+    })
+  }
+  return graphics.fill(fill)
+}
+
+/**
+ * Groups bars by the paint they take, preserving order within each group.
+ *
+ * The seam that makes {@link paintBars} worth batching: a diarized waveform has
+ * as many colours as it has voices, not as many as it has bars, so a
+ * conversation of two people is two draws however long it runs.
+ */
+export function groupByPaint<K>(
+  boxes: readonly BarBox[],
+  keyOf: (index: number) => K
+): Map<K, BarBox[]> {
+  const groups = new Map<K, BarBox[]>()
+  for (let i = 0; i < boxes.length; i++) {
+    const key = keyOf(i)
+    const run = groups.get(key)
+    if (run) run.push(boxes[i])
+    else groups.set(key, [boxes[i]])
+  }
+  return groups
+}
+
+export function clamp01(value: number): number {
+  return value < 0 ? 0 : value > 1 ? 1 : value
+}
+
+/**
+ * The {@link groupByPaint} key one bar of a live visualiser takes.
+ *
+ * Filtered to one voice, every bar is that voice regardless of `byVoice` or
+ * whose turn the bucket originally belonged to. That second part is the one
+ * worth spelling out: {@link forSpeaker} silences a bar by zeroing its
+ * magnitude but leaves `speaker` exactly as it was, so a bar sitting in the
+ * middle of a *different* voice's turn is still stamped with that voice's
+ * index — grouping by `byVoice`'s usual per-bar key would paint those silent
+ * stretches in whoever used to be talking there, in flickers too short to
+ * read as anything but noise. Forcing every bar under the filtered voice's own
+ * key is the true picture instead: the node is drawing exactly one person, on
+ * or off, never anyone else's colour.
+ */
+export function livePaintKey(
+  barSpeaker: number,
+  byVoice: boolean,
+  filteredVoice: number | null
+): string {
+  if (filteredVoice !== null) return `voice:${filteredVoice}`
+  return byVoice ? `voice:${barSpeaker}` : "all"
+}