@motionscript/audio 0.0.0-stage → 0.1.0-alpha.0

package/src/waveform/envelope.ts

@@ -0,0 +1,143 @@
+/**
+ * The shape of a recording, as the waveform nodes draw it.
+ *
+ * ## Why the data is baked rather than listened to
+ *
+ * A scene has to render identically in the preview and in a headless export,
+ * and an export has no audio device, no `AudioContext` and no playback to
+ * analyse. So a "live" visualiser cannot read a Web Audio analyser: what it
+ * reads is a *pre-computed envelope*, indexed by the scene clock. The picture is
+ * then a pure function of time, which is what makes scrubbing backwards, jumping
+ * to frame 900, and rendering out of order all produce the same frame.
+ *
+ * The decode happens in the app, once per file, and arrives here through the
+ * build (see `SceneAsset.envelope`) — the same arrangement a chart's rows and a
+ * protein's atoms already use, and for the same reason: the build is
+ * synchronous and cannot await a fetch.
+ *
+ * ## Why magnitude rather than min/max
+ *
+ * An editor's waveform is drawn from per-bucket minima *and* maxima, because an
+ * editor is a measuring instrument. These nodes are neither — they are a bar
+ * chart of loudness that happens to be shaped like a recording — and audio is
+ * near enough symmetric that a mirrored magnitude is indistinguishable at the
+ * size a node draws. Half the numbers, and one array to reason about.
+ */
+
+/**
+ * A decoded recording, reduced to a uniform grid of loudness — plus, when the
+ * app knows it, who is talking in each bucket.
+ *
+ * The speaker track is **not** something a decoder can produce; it comes from a
+ * diarized transcript and is folded in by the app before the build. It lives
+ * here rather than in a parallel structure because every reader of it is also a
+ * reader of the magnitudes at the same index, and two arrays that must stay the
+ * same length are better as one object than as two arguments.
+ */
+export interface AudioEnvelope {
+  /** Peak magnitude per bucket, 0…1. Buckets span {@link duration} evenly. */
+  magnitude: Float32Array
+  /** The decoded length in seconds. */
+  duration: number
+  /**
+   * Which voice is speaking in each bucket, as an index into the recording's
+   * own numbering, or `-1` for "nobody identified". Parallel to
+   * {@link magnitude}, or `null` when the track was never diarized.
+   */
+  speaker: Int16Array | null
+  /** How many distinct voices the diarizer found. Zero when it did not run. */
+  speakers: number
+}
+
+/** The envelope of a track that hasn't loaded — or isn't there. */
+export const EMPTY_ENVELOPE: AudioEnvelope = {
+  magnitude: new Float32Array(0),
+  duration: 0,
+  speaker: null,
+  speakers: 0,
+}
+
+/** One drawn bar: how loud, and whose. */
+export interface EnvelopeBar {
+  /** 0…1. */
+  magnitude: number
+  /** The voice index, or -1. */
+  speaker: number
+}
+
+/**
+ * Resamples `[from, to)` seconds of an envelope down to `bars` columns.
+ *
+ * Peak-preserving rather than averaging, which is the whole difference between
+ * a waveform and a blur: averaging a hundred buckets into one column flattens
+ * every transient, and transients are what makes a drawn recording read as
+ * speech rather than as noise. The speaker of a column is the speaker of its
+ * *loudest* bucket, on the same reasoning — a column straddling a turn belongs
+ * to whoever is actually audible in it.
+ *
+ * A window past either end of the recording yields silent, unattributed
+ * columns rather than clamping onto the nearest real one. That matters for the
+ * live visualiser, which is handed a window centred on the playhead and would
+ * otherwise smear the first bucket across the whole run-up to a track's start.
+ */
+export function sampleEnvelope(
+  envelope: AudioEnvelope,
+  from: number,
+  to: number,
+  bars: number
+): EnvelopeBar[] {
+  const count = Math.max(0, Math.floor(bars))
+  const out: EnvelopeBar[] = new Array(count)
+  for (let i = 0; i < count; i++) out[i] = { magnitude: 0, speaker: -1 }
+
+  const buckets = envelope.magnitude.length
+  if (count === 0 || buckets === 0 || envelope.duration <= 0) return out
+
+  const perSecond = buckets / envelope.duration
+  const span = (to - from) / count
+
+  for (let i = 0; i < count; i++) {
+    const start = (from + span * i) * perSecond
+    const end = (from + span * (i + 1)) * perSecond
+
+    // At least one bucket per column, so a window zoomed in past the grid draws
+    // the sample it is standing on rather than an empty band.
+    const first = Math.floor(start)
+    const last = Math.max(first, Math.ceil(end) - 1)
+
+    let loudest = 0
+    let speaker = -1
+    for (let b = first; b <= last; b++) {
+      if (b < 0 || b >= buckets) continue
+      const value = envelope.magnitude[b]
+      if (value < loudest) continue
+      loudest = value
+      speaker = envelope.speaker ? envelope.speaker[b] : -1
+    }
+
+    out[i] = { magnitude: loudest, speaker }
+  }
+
+  return out
+}
+
+/**
+ * The same window, with everything that isn't `voice` silenced.
+ *
+ * The point of the speaker filter, and the reason it silences rather than
+ * *drops*: a podcast visualiser filtered to one host should go quiet while the
+ * other one talks, not compress their turn out of existence. Keeping the bars
+ * and zeroing them is what makes the picture read as "this person is not
+ * speaking right now" instead of as a jump cut.
+ *
+ * `voice` of `null` returns the bars untouched, which is the unfiltered node.
+ */
+export function forSpeaker(
+  bars: EnvelopeBar[],
+  voice: number | null
+): EnvelopeBar[] {
+  if (voice === null) return bars
+  return bars.map((bar) =>
+    bar.speaker === voice ? bar : { magnitude: 0, speaker: bar.speaker }
+  )
+}

package/src/waveform/full-waveform.ts

@@ -0,0 +1,218 @@
+import {
+  Node2D,
+  node,
+  command,
+  easeInOut,
+  fillOps,
+  property,
+  type AssetScope,
+  type CommandArgs,
+  type Fill,
+  type FillResolved,
+  type Node2DProps,
+  type RenderContext2D,
+} from "@motionscript/core"
+
+import { declarePaints } from "@motionscript/core/component"
+import type { Seekable } from "@motionscript/core/component"
+import { EMPTY_ENVELOPE, sampleEnvelope, type AudioEnvelope } from "./envelope"
+import {
+  clamp01,
+  groupByPaint,
+  layOutBars,
+  paintBars,
+  voiceColor,
+  type WaveformAlignment,
+} from "./shared"
+
+export interface FullWaveformProps extends Node2DProps {
+  /** The baked recording. See {@link AudioEnvelope} for why it is baked. */
+  envelope: AudioEnvelope
+  /** How many columns the whole file is drawn as. */
+  bars: number
+  /** Space between bars, as a fraction of the pitch. See {@link layOutBars}. */
+  gap: number
+  /** Corner radius of a bar, capped at half its width. */
+  radius: number
+  /** How much of the file reads as played, `0`–`1`. Tweenable. */
+  progress: number
+  /** Drive {@link progress} from the node's own clock instead of the prop. */
+  follow: boolean
+  /** Multiplies every magnitude — a quiet track pulled up to fill its box. */
+  gain: number
+  /** How much of the box a silent bar still occupies, `0`–`1`. */
+  floor: number
+  align: WaveformAlignment
+  /** Colour each voice's turns rather than splitting on played / not played. */
+  byVoice: boolean
+  /** A colour per voice index. Short or empty falls back to the palette. */
+  voiceColors: string[]
+  /** The bars ahead of the playhead. */
+  fill: Fill
+  /** …and the bars behind it. */
+  playedFill: Fill
+}
+
+/**
+ * A **waveform overview**: the whole recording's shape, drawn at once.
+ *
+ * The picture every audio editor opens with, and the one a podcast clip puts
+ * under its caption — a static silhouette of the file, with a played portion
+ * that advances. It answers "how long is this and where are the loud parts",
+ * which is a question about the *file*, not about the moment; its sibling
+ * {@link LiveWaveform} answers the other one.
+ *
+ * ## Progress is authored, not assumed
+ *
+ * {@link follow} is off by default, and that default is the considered one. A
+ * scene has no idea where the recording it is drawing sits on the video's audio
+ * lane — a clip is placed, trimmed and sped up out there, and none of that is
+ * visible from in here. So the honest default is that the fill is *animation*:
+ * the author sweeps it with a command, the same as any other tween, and it lands
+ * wherever they put it. `follow` is the shortcut for the common arrangement
+ * where the node appears exactly as the track starts, and it says so by being a
+ * switch someone has to turn on rather than a behaviour that quietly disagrees
+ * with the mix.
+ *
+ * ## Voices instead of progress
+ *
+ * With {@link byVoice} on, the played/ahead split is replaced by one colour per
+ * speaking voice — the drawing the timeline's clip bars already make, at a size
+ * you can read. For a two-hander it is a map of the conversation: who has the
+ * floor, where the turns are, how long each one ran. It replaces the progress
+ * colours rather than combining with them, because a bar cannot mean two things
+ * at once and "played, and also Grace" is a legend nobody can hold in their
+ * head.
+ */
+@node({
+  key: "fullWaveform",
+  parentKey: "node",
+  forkable: true,
+  layout: {
+    children: "freeform",
+    defaultWidthMode: "fixed",
+    defaultHeightMode: "fixed",
+    acceptsChildren: true,
+  },
+  seed: {
+    width: 900,
+    height: 160,
+  },
+})
+export class FullWaveform extends Node2D<FullWaveformProps> {
+  @property({ default: EMPTY_ENVELOPE }) declare envelope: AudioEnvelope
+  @property({ default: 120 }) declare bars: number
+  @property({ default: 0.3 }) declare gap: number
+  @property({ default: 2 }) declare radius: number
+  @property({ default: 0 }) declare progress: number
+  @property({ default: false }) declare follow: boolean
+  @property({ default: 1 }) declare gain: number
+  @property({ default: 0.015 }) declare floor: number
+  @property({ default: "center" }) declare align: WaveformAlignment
+  @property({ default: false }) declare byVoice: boolean
+  @property({ default: [] }) declare voiceColors: string[]
+
+  @property({
+    default: "#5b5b7a",
+    mapper: fillOps.resolve,
+    tween: fillOps.lerp,
+  })
+  declare fill: Fill
+
+  @property({
+    default: "#6366f1",
+    mapper: fillOps.resolve,
+    tween: fillOps.lerp,
+  })
+  declare playedFill: Fill
+
+  /**
+   * Fill the waveform in, left to right — the node's one command, since
+   * everything else about it is expressible as an ordinary `to`.
+   */
+  @command()
+  sweep(args: CommandArgs<Record<string, never>> & { duration: number }): Seekable {
+    return this.to({
+      data: { progress: 1 } as Partial<FullWaveformProps>,
+      duration: args.duration,
+      easing: args.easing ?? easeInOut(),
+    })
+  }
+
+  /**
+   * This node extends `Node2D` rather than `ShapeNode`, so nothing declares the
+   * paint props it carries — see {@link declarePaints}. `playedFill` is named
+   * past the conventional slots, so it is passed explicitly.
+   */
+  override declareAssets(assets: AssetScope): void {
+    super.declareAssets(assets)
+    declarePaints(this, assets, { fills: [this.playedFill as FillResolved[]] })
+  }
+
+  /** Where the fill has reached, from whichever source is in charge. */
+  private get played(): number {
+    if (!this.follow) return clamp01(this.progress)
+    const length = this.envelope.duration
+    if (length <= 0) return 0
+    return clamp01(this.time.elapsed / length)
+  }
+
+  protected renderSelf(ctx: RenderContext2D): void {
+    const rect = this.layoutBounds
+    const width = rect?.width ?? 0
+    const height = rect?.height ?? 0
+    if (width <= 0 || height <= 0) return
+
+    const count = Math.max(1, Math.round(this.bars))
+    const sampled = sampleEnvelope(
+      this.envelope,
+      0,
+      this.envelope.duration,
+      count
+    )
+    const boxes = layOutBars(sampled, {
+      width,
+      height,
+      gap: this.gap,
+      align: this.align,
+      gain: this.gain,
+      floor: this.floor,
+    })
+    if (boxes.length === 0) return
+
+    // One draw per colour rather than per bar — see {@link paintBars}. The key
+    // is a string because it has to be comparable; the paint it stands for is
+    // resolved after the grouping.
+    const edge = this.played * boxes.length
+    const groups = groupByPaint(boxes, (i) =>
+      this.byVoice
+        ? `voice:${sampled[i].speaker}`
+        : i < edge
+          ? "played"
+          : "ahead"
+    )
+
+    for (const [key, run] of groups) {
+      const graphics = paintBars(run, this.paintFor(key), this.radius)
+      if (graphics) ctx.draw(graphics)
+    }
+  }
+
+  /**
+   * The paint a group takes.
+   *
+   * A voice's colour is a **colour string**, never one of the node's two fill
+   * props, and that is deliberate: a voice is identified by hue, so a gradient
+   * or an image there would defeat the only thing the colour is for. It also
+   * means these paints declare no assets, which is why {@link declareAssets}
+   * only has to account for the two real fill props.
+   */
+  private paintFor(key: string): Fill {
+    if (key === "played") return this.playedFill
+    if (key === "ahead") return this.fill
+
+    const voice = Number(key.slice("voice:".length))
+    if (!Number.isFinite(voice) || voice < 0) return this.fill
+    return this.voiceColors[voice] ?? voiceColor(voice)
+  }
+}

package/src/waveform/index.ts

@@ -0,0 +1,30 @@
+/**
+ * The waveform family: one subject, two questions.
+ *
+ * - {@link FullWaveform} draws the **file** — the whole recording's silhouette,
+ *   with a played portion that advances. "How long is this, and where are the
+ *   loud parts."
+ * - {@link LiveWaveform} draws the **moment** — bars moving with playback, the
+ *   visualiser a podcast clip puts beside the artwork. "Is anything happening
+ *   right now, and whose voice is it."
+ *
+ * Both read the same baked {@link AudioEnvelope}, which is why they cannot
+ * disagree about a recording, and why neither listens to anything — see that
+ * module for why a scene that has to export cannot use a live analyser.
+ */
+export { FullWaveform } from "./full-waveform"
+export type { FullWaveformProps } from "./full-waveform"
+export { LiveWaveform } from "./live-waveform"
+export type { LiveWaveformProps } from "./live-waveform"
+export { EMPTY_ENVELOPE, forSpeaker, sampleEnvelope } from "./envelope"
+export type { AudioEnvelope, EnvelopeBar } from "./envelope"
+export {
+  VOICE_PALETTE,
+  WAVEFORM_ALIGNMENTS,
+  WAVEFORM_STYLES,
+  groupByPaint,
+  layOutBars,
+  paintBars,
+  voiceColor,
+} from "./shared"
+export type { BarBox, WaveformAlignment, WaveformStyle } from "./shared"

package/src/waveform/live-waveform.ts

@@ -0,0 +1,221 @@
+import {
+  Node2D,
+  node,
+  fillOps,
+  property,
+  type AssetScope,
+  type Fill,
+  type Node2DProps,
+  type RenderContext2D,
+} from "@motionscript/core"
+
+import { declarePaints } from "@motionscript/core/component"
+import {
+  EMPTY_ENVELOPE,
+  forSpeaker,
+  sampleEnvelope,
+  type AudioEnvelope,
+  type EnvelopeBar,
+} from "./envelope"
+import {
+  clamp01,
+  groupByPaint,
+  layOutBars,
+  livePaintKey,
+  paintBars,
+  voiceColor,
+  type WaveformAlignment,
+  type WaveformStyle,
+} from "./shared"
+
+export interface LiveWaveformProps extends Node2DProps {
+  /** The baked recording. See {@link AudioEnvelope} for why it is baked. */
+  envelope: AudioEnvelope
+  /** How many bars the visualiser is made of. */
+  bars: number
+  /** Space between bars, as a fraction of the pitch. */
+  gap: number
+  radius: number
+  /** How many seconds of audio are on screen at once. */
+  window: number
+  /** Seconds into the file the node's first frame shows. */
+  offset: number
+  /** Advance with the scene clock. Off freezes the picture at {@link offset}. */
+  playing: boolean
+  /** Multiplies every magnitude before it is clamped. */
+  gain: number
+  /** How much of the box a silent bar still occupies, `0`–`1`. */
+  floor: number
+  align: WaveformAlignment
+  style: WaveformStyle
+  /**
+   * Show only this voice's activity, by index into the recording's own
+   * numbering. `-1` — the default — is every voice.
+   */
+  speaker: number
+  /** Colour each bar by whoever is speaking in it. */
+  byVoice: boolean
+  /** A colour per voice index. Short or empty falls back to the palette. */
+  voiceColors: string[]
+  fill: Fill
+}
+
+/**
+ * A **live audio visualiser**: bars that move with the recording as it plays.
+ *
+ * The thing every podcast-to-video tool puts beside the artwork. It is not a
+ * picture of the file — that is {@link FullWaveform} — it is a picture of *this
+ * moment*, and what it is for is making a static frame of a talking head feel
+ * like it is playing.
+ *
+ * ## "Live" without listening
+ *
+ * Nothing here reads an audio device. The bars are sampled out of a baked
+ * envelope at the node's own elapsed time, which is what makes the visualiser
+ * survive the thing it exists for: an **export**, where there is no playback to
+ * analyse and frames may be rendered out of order or in parallel. A frame is a
+ * pure function of its time, so scrubbing back and forth over the visualiser
+ * redraws exactly what it drew the first time — which an analyser node, by
+ * construction, cannot promise. See {@link AudioEnvelope}.
+ *
+ * ## The speaker filter
+ *
+ * {@link speaker} narrows the node to one voice of a diarized recording, and it
+ * is the reason this node knows about transcripts at all. Two visualisers beside
+ * two headshots, each filtered to its own host, is a podcast edit that would
+ * otherwise be done by hand for every turn — and because the filter *silences*
+ * rather than skips (see {@link forSpeaker}), the quiet host visibly stops
+ * talking instead of vanishing. Nothing about that requires the words: only who
+ * is talking when, which is what the diarizer produces.
+ *
+ * A filter naming a voice the recording doesn't have draws a resting row rather
+ * than everything — a picture of "this person never speaks here", which is
+ * true, where falling back to the unfiltered mix would be a silent lie about
+ * whose voice you were watching.
+ */
+@node({
+  key: "liveWaveform",
+  parentKey: "node",
+  forkable: true,
+  layout: {
+    children: "freeform",
+    defaultWidthMode: "fixed",
+    defaultHeightMode: "fixed",
+    acceptsChildren: true,
+  },
+  seed: {
+    width: 420,
+    height: 220,
+  },
+})
+export class LiveWaveform extends Node2D<LiveWaveformProps> {
+  @property({ default: EMPTY_ENVELOPE }) declare envelope: AudioEnvelope
+  @property({ default: 48 }) declare bars: number
+  @property({ default: 0.35 }) declare gap: number
+  @property({ default: 4 }) declare radius: number
+  @property({ default: 3 }) declare window: number
+  @property({ default: 0 }) declare offset: number
+  @property({ default: true }) declare playing: boolean
+  @property({ default: 1.4 }) declare gain: number
+  @property({ default: 0.04 }) declare floor: number
+  @property({ default: "center" }) declare align: WaveformAlignment
+  @property({ default: "bars" }) declare style: WaveformStyle
+  @property({ default: -1 }) declare speaker: number
+  @property({ default: false }) declare byVoice: boolean
+  @property({ default: [] }) declare voiceColors: string[]
+
+  @property({
+    default: "#6366f1",
+    mapper: fillOps.resolve,
+    tween: fillOps.lerp,
+  })
+  declare fill: Fill
+
+  override declareAssets(assets: AssetScope): void {
+    super.declareAssets(assets)
+    declarePaints(this, assets)
+  }
+
+  /**
+   * The second of the recording the visualiser is showing.
+   *
+   * Off the node's own clock rather than the scene's, which is what "from the
+   * moment this node appeared" means — the same clock motion-script's own
+   * `Video` times its picture from, and the same one that makes a node dropped
+   * halfway through a scene start its recording at its entrance rather than
+   * partway in.
+   */
+  private get at(): number {
+    return this.offset + (this.playing ? Math.max(0, this.time.elapsed) : 0)
+  }
+
+  protected renderSelf(ctx: RenderContext2D): void {
+    const rect = this.layoutBounds
+    const width = rect?.width ?? 0
+    const height = rect?.height ?? 0
+    if (width <= 0 || height <= 0) return
+
+    const count = Math.max(1, Math.round(this.bars))
+    const span = Math.max(0.05, this.window)
+    const at = this.at
+    const voice = this.speaker < 0 ? null : Math.round(this.speaker)
+
+    // The window is centred on the playhead, so the bars run *toward* the
+    // middle and away again — the arrangement every mixer's meter uses, and the
+    // one that reads as sound arriving rather than as a strip scrolling past.
+    const sampled = forSpeaker(
+      sampleEnvelope(this.envelope, at - span / 2, at + span / 2, count),
+      voice
+    )
+
+    const shaped = this.style === "line" ? taper(sampled) : sampled
+    const boxes = layOutBars(shaped, {
+      width,
+      height,
+      gap: this.style === "blocks" ? 0.12 : this.gap,
+      align: this.align,
+      gain: this.gain,
+      floor: this.floor,
+    })
+    if (boxes.length === 0) return
+
+    const groups = groupByPaint(boxes, (i) =>
+      livePaintKey(shaped[i].speaker, this.byVoice, voice)
+    )
+
+    for (const [key, run] of groups) {
+      const graphics = paintBars(run, this.paintFor(key), this.radius)
+      if (graphics) ctx.draw(graphics)
+    }
+  }
+
+  /** See {@link FullWaveform.paintFor} — a voice is a hue, never a gradient. */
+  private paintFor(key: string): Fill {
+    if (key === "all") return this.fill
+    const voice = Number(key.slice("voice:".length))
+    if (!Number.isFinite(voice) || voice < 0) return this.fill
+    return this.voiceColors[voice] ?? voiceColor(voice)
+  }
+}
+
+/**
+ * Rolls the ends of a window off toward silence.
+ *
+ * What separates the `line` style from `bars`: a hard-edged window makes the
+ * loud bar entering at the right edge appear from nothing, which reads as a
+ * glitch rather than as sound arriving. Fading the outer fifth turns the same
+ * data into a shape that grows out of the baseline — the profile a spectrum
+ * display has, and the reason it looks like one object rather than a row of
+ * separate meters.
+ */
+function taper(bars: readonly EnvelopeBar[]): EnvelopeBar[] {
+  const count = bars.length
+  if (count < 3) return [...bars]
+  const edge = Math.max(1, Math.floor(count / 5))
+
+  return bars.map((bar, i) => {
+    const fromEnd = Math.min(i, count - 1 - i)
+    const weight = clamp01(fromEnd / edge)
+    return { magnitude: bar.magnitude * weight, speaker: bar.speaker }
+  })
+}