@motionscript/audio 0.1.0-alpha.0 → 0.1.0-alpha.10

package/src/waveform/envelope.ts

@@ -1,6 +1,37 @@
 /**
  * The shape of a recording, as the waveform nodes draw it.
  *
+ * ## How an envelope reaches a node: the host supplies it
+ *
+ * Nothing in the engine, and nothing in this kit, decodes audio. A waveform node
+ * does not read a `sound` node's `src`, declares no audio asset and is not tied
+ * to a clip: it draws the {@link AudioEnvelope} it is handed in its `envelope`
+ * prop. Decoding a file into an envelope is the host's job, done once per file
+ * before the document is built. Where the recording sits on the audio lane is
+ * the author's timing too, which is why `FullWaveform.follow` is opt-in.
+ *
+ * ### The JSON form
+ *
+ * A document holds the envelope as plain JSON, because a prop value in a
+ * document is JSON. The typed arrays in {@link AudioEnvelope} are one spelling of
+ * it; `ArrayLike<number>` also accepts the plain arrays a document stores:
+ *
+ * ```json
+ * {
+ *   "magnitude": [0, 0.25, 0.5, 1, 0.5, 0],
+ *   "duration": 6,
+ *   "speaker": [0, 0, 0, 1, 1, -1],
+ *   "speakers": 2
+ * }
+ * ```
+ *
+ * `magnitude` is the peak loudness, `0`…`1`, of buckets that divide `duration`
+ * seconds evenly. `speaker` is `null` when nobody was identified; otherwise it is parallel to `magnitude`, one voice
+ * index per bucket, `-1` for "nobody". `speakers` is the voice count, `0` when
+ * `speaker` is `null`. A `Float32Array` or `Int16Array` is not JSON-serialisable
+ * (`JSON.stringify` writes an object of indices), so a host that holds the typed
+ * form converts with `Array.from` before storing it in a document.
+ *
  * ## Why the data is baked rather than listened to
  *
  * A scene has to render identically in the preview and in a headless export,
@@ -10,10 +41,9 @@
  * then a pure function of time, which is what makes scrubbing backwards, jumping
  * to frame 900, and rendering out of order all produce the same frame.
  *
- * The decode happens in the app, once per file, and arrives here through the
- * build (see `SceneAsset.envelope`) — the same arrangement a chart's rows and a
- * protein's atoms already use, and for the same reason: the build is
- * synchronous and cannot await a fetch.
+ * The decode happens in the host, once per file, and arrives here through the
+ * build — the same arrangement a chart's rows and a protein's atoms already use,
+ * and for the same reason: the build is synchronous and cannot await a fetch.
  *
  * ## Why magnitude rather than min/max
  *
@@ -26,17 +56,20 @@
 
 /**
  * A decoded recording, reduced to a uniform grid of loudness — plus, when the
- * app knows it, who is talking in each bucket.
+ * host knows it, who is talking in each bucket.
  *
  * The speaker track is **not** something a decoder can produce; it comes from a
- * diarized transcript and is folded in by the app before the build. It lives
+ * diarized transcript and is folded in by the host before the build. It lives
  * here rather than in a parallel structure because every reader of it is also a
  * reader of the magnitudes at the same index, and two arrays that must stay the
  * same length are better as one object than as two arguments.
  */
 export interface AudioEnvelope {
-  /** Peak magnitude per bucket, 0…1. Buckets span {@link duration} evenly. */
-  magnitude: Float32Array
+  /**
+   * Peak magnitude per bucket, 0…1. Buckets span {@link duration} evenly. A
+   * `Float32Array`, or the plain `number[]` a document stores.
+   */
+  magnitude: ArrayLike<number>
   /** The decoded length in seconds. */
   duration: number
   /**
@@ -44,7 +77,7 @@ export interface AudioEnvelope {
    * own numbering, or `-1` for "nobody identified". Parallel to
    * {@link magnitude}, or `null` when the track was never diarized.
    */
-  speaker: Int16Array | null
+  speaker: ArrayLike<number> | null
   /** How many distinct voices the diarizer found. Zero when it did not run. */
   speakers: number
 }

package/src/waveform/full-waveform.ts

@@ -1,20 +1,17 @@
 import {
-  Node2D,
-  node,
-  command,
+  Types,
+  component2d,
   easeInOut,
   fillOps,
-  property,
-  type AssetScope,
-  type CommandArgs,
   type Fill,
   type FillResolved,
-  type Node2DProps,
-  type RenderContext2D,
-} from "@motionscript/core"
+  type NodeOf,
+  type RenderContext,
+  // Named so the emitted declaration reaches these through the sdk, not core.
+  type Node2D as _Node2D,
+  type Node2DProps as _Node2DProps,
+} from "@motionscript/sdk"
 
-import { declarePaints } from "@motionscript/core/component"
-import type { Seekable } from "@motionscript/core/component"
 import { EMPTY_ENVELOPE, sampleEnvelope, type AudioEnvelope } from "./envelope"
 import {
   clamp01,
@@ -25,33 +22,9 @@ import {
   type WaveformAlignment,
 } from "./shared"
 
-export interface FullWaveformProps extends Node2DProps {
-  /** The baked recording. See {@link AudioEnvelope} for why it is baked. */
-  envelope: AudioEnvelope
-  /** How many columns the whole file is drawn as. */
-  bars: number
-  /** Space between bars, as a fraction of the pitch. See {@link layOutBars}. */
-  gap: number
-  /** Corner radius of a bar, capped at half its width. */
-  radius: number
-  /** How much of the file reads as played, `0`–`1`. Tweenable. */
-  progress: number
-  /** Drive {@link progress} from the node's own clock instead of the prop. */
-  follow: boolean
-  /** Multiplies every magnitude — a quiet track pulled up to fill its box. */
-  gain: number
-  /** How much of the box a silent bar still occupies, `0`–`1`. */
-  floor: number
-  align: WaveformAlignment
-  /** Colour each voice's turns rather than splitting on played / not played. */
-  byVoice: boolean
-  /** A colour per voice index. Short or empty falls back to the palette. */
-  voiceColors: string[]
-  /** The bars ahead of the playhead. */
-  fill: Fill
-  /** …and the bars behind it. */
-  playedFill: Fill
-}
+/** A paint entry as the waveforms declare theirs: resolved and tweened like a fill, its kind read off the default. */
+const paint = (initial: string) =>
+  Types.custom<Fill, FillResolved[]>(initial, { resolve: fillOps.resolve, lerp: fillOps.lerp })
 
 /**
  * A **waveform overview**: the whole recording's shape, drawn at once.
@@ -64,7 +37,7 @@ export interface FullWaveformProps extends Node2DProps {
  *
  * ## Progress is authored, not assumed
  *
- * {@link follow} is off by default, and that default is the considered one. A
+ * `follow` is off by default, and that default is the considered one. A
  * scene has no idea where the recording it is drawing sits on the video's audio
  * lane — a clip is placed, trimmed and sped up out there, and none of that is
  * visible from in here. So the honest default is that the fill is *animation*:
@@ -76,143 +49,140 @@ export interface FullWaveformProps extends Node2DProps {
  *
  * ## Voices instead of progress
  *
- * With {@link byVoice} on, the played/ahead split is replaced by one colour per
- * speaking voice — the drawing the timeline's clip bars already make, at a size
- * you can read. For a two-hander it is a map of the conversation: who has the
+ * With `byVoice` on, the played/ahead split is replaced by one colour per
+ * speaking voice, at a size you can read. For a two-hander it is a map of the conversation: who has the
  * floor, where the turns are, how long each one ran. It replaces the progress
  * colours rather than combining with them, because a bar cannot mean two things
  * at once and "played, and also Grace" is a legend nobody can hold in their
  * head.
  */
-@node({
+export const FullWaveform = component2d({
   key: "fullWaveform",
-  parentKey: "node",
-  forkable: true,
-  layout: {
-    children: "freeform",
-    defaultWidthMode: "fixed",
-    defaultHeightMode: "fixed",
-    acceptsChildren: true,
+  state: {
+    /** The baked recording. See {@link AudioEnvelope} for why it is baked. */
+    envelope: Types.custom<AudioEnvelope>(EMPTY_ENVELOPE),
+    /** How many columns the whole file is drawn as. */
+    bars: 120,
+    /** Space between bars, as a fraction of the pitch. See {@link layOutBars}. */
+    gap: 0.3,
+    /** Corner radius of a bar, capped at half its width. */
+    radius: 2,
+    /** How much of the file reads as played, `0`–`1`. Tweenable. */
+    progress: 0,
+    /** Drive `progress` from the node's own clock instead of the prop. */
+    follow: false,
+    /** Multiplies every magnitude — a quiet track pulled up to fill its box. */
+    gain: 1,
+    /** How much of the box a silent bar still occupies, `0`–`1`. */
+    floor: 0.015,
+    align: Types.custom<WaveformAlignment>("center"),
+    /** Colour each voice's turns rather than splitting on played / not played. */
+    byVoice: false,
+    /** A colour per voice index. Short or empty falls back to the palette. */
+    voiceColors: Types.custom<string[]>([]),
+    /** The bars ahead of the playhead. */
+    fill: paint("#5b5b7a"),
+    /** …and the bars behind it. */
+    playedFill: paint("#6366f1"),
+  },
+  // Nothing declares a component's paints but itself: `fill` and `playedFill` both.
+  load: (loader, state) => {
+    loader.fill(state.fill)
+    loader.fill(state.playedFill)
+  },
+  commands: {
+    /**
+     * Fill the waveform in, left to right — the node's one command, since
+     * everything else about it is expressible as an ordinary `to`. Eased in-out
+     * unless it is given an easing of its own.
+     */
+    sweep: (timeline, state) => {
+      const ease = timeline.easing ? null : easeInOut()
+      const from = state.progress
+      return timeline.animate((t) => ({ progress: from + (1 - from) * (ease ? ease(t) : t) }))
+    },
   },
-  seed: {
-    width: 900,
-    height: 160,
+  render: (ctx, state) => {
+    drawBars(ctx, state)
+    ctx.children()
   },
 })
-export class FullWaveform extends Node2D<FullWaveformProps> {
-  @property({ default: EMPTY_ENVELOPE }) declare envelope: AudioEnvelope
-  @property({ default: 120 }) declare bars: number
-  @property({ default: 0.3 }) declare gap: number
-  @property({ default: 2 }) declare radius: number
-  @property({ default: 0 }) declare progress: number
-  @property({ default: false }) declare follow: boolean
-  @property({ default: 1 }) declare gain: number
-  @property({ default: 0.015 }) declare floor: number
-  @property({ default: "center" }) declare align: WaveformAlignment
-  @property({ default: false }) declare byVoice: boolean
-  @property({ default: [] }) declare voiceColors: string[]
-
-  @property({
-    default: "#5b5b7a",
-    mapper: fillOps.resolve,
-    tween: fillOps.lerp,
-  })
-  declare fill: Fill
 
-  @property({
-    default: "#6366f1",
-    mapper: fillOps.resolve,
-    tween: fillOps.lerp,
-  })
-  declare playedFill: Fill
-
-  /**
-   * Fill the waveform in, left to right — the node's one command, since
-   * everything else about it is expressible as an ordinary `to`.
-   */
-  @command()
-  sweep(args: CommandArgs<Record<string, never>> & { duration: number }): Seekable {
-    return this.to({
-      data: { progress: 1 } as Partial<FullWaveformProps>,
-      duration: args.duration,
-      easing: args.easing ?? easeInOut(),
-    })
-  }
-
-  /**
-   * This node extends `Node2D` rather than `ShapeNode`, so nothing declares the
-   * paint props it carries — see {@link declarePaints}. `playedFill` is named
-   * past the conventional slots, so it is passed explicitly.
-   */
-  override declareAssets(assets: AssetScope): void {
-    super.declareAssets(assets)
-    declarePaints(this, assets, { fills: [this.playedFill as FillResolved[]] })
-  }
+export type FullWaveform = NodeOf<typeof FullWaveform>
+export type FullWaveformProps = NonNullable<Parameters<typeof FullWaveform>[0]>
 
-  /** Where the fill has reached, from whichever source is in charge. */
-  private get played(): number {
-    if (!this.follow) return clamp01(this.progress)
-    const length = this.envelope.duration
-    if (length <= 0) return 0
-    return clamp01(this.time.elapsed / length)
-  }
+interface FullState {
+  readonly envelope: AudioEnvelope
+  readonly bars: number
+  readonly gap: number
+  readonly radius: number
+  readonly progress: number
+  readonly follow: boolean
+  readonly gain: number
+  readonly floor: number
+  readonly align: WaveformAlignment
+  readonly byVoice: boolean
+  readonly voiceColors: string[]
+  readonly fill: FillResolved[]
+  readonly playedFill: FillResolved[]
+}
 
-  protected renderSelf(ctx: RenderContext2D): void {
-    const rect = this.layoutBounds
-    const width = rect?.width ?? 0
-    const height = rect?.height ?? 0
-    if (width <= 0 || height <= 0) return
+function drawBars(ctx: RenderContext, s: FullState): void {
+  const { width, height } = ctx.size
+  if (width <= 0 || height <= 0) return
 
-    const count = Math.max(1, Math.round(this.bars))
-    const sampled = sampleEnvelope(
-      this.envelope,
-      0,
-      this.envelope.duration,
-      count
-    )
-    const boxes = layOutBars(sampled, {
-      width,
-      height,
-      gap: this.gap,
-      align: this.align,
-      gain: this.gain,
-      floor: this.floor,
-    })
-    if (boxes.length === 0) return
+  const count = Math.max(1, Math.round(s.bars))
+  const sampled = sampleEnvelope(s.envelope, 0, s.envelope.duration, count)
+  const boxes = layOutBars(sampled, {
+    width,
+    height,
+    gap: s.gap,
+    align: s.align,
+    gain: s.gain,
+    floor: s.floor,
+  })
+  if (boxes.length === 0) return
 
-    // One draw per colour rather than per bar — see {@link paintBars}. The key
-    // is a string because it has to be comparable; the paint it stands for is
-    // resolved after the grouping.
-    const edge = this.played * boxes.length
-    const groups = groupByPaint(boxes, (i) =>
-      this.byVoice
-        ? `voice:${sampled[i].speaker}`
-        : i < edge
-          ? "played"
-          : "ahead"
-    )
+  // One draw per colour rather than per bar — see {@link paintBars}. The key
+  // is a string because it has to be comparable; the paint it stands for is
+  // resolved after the grouping.
+  const edge = played(ctx, s) * boxes.length
+  const groups = groupByPaint(boxes, (i) =>
+    s.byVoice
+      ? `voice:${sampled[i].speaker}`
+      : i < edge
+        ? "played"
+        : "ahead"
+  )
 
-    for (const [key, run] of groups) {
-      const graphics = paintBars(run, this.paintFor(key), this.radius)
-      if (graphics) ctx.draw(graphics)
-    }
+  for (const [key, run] of groups) {
+    const graphics = paintBars(run, paintFor(s, key), s.radius)
+    if (graphics) ctx.draw(graphics)
   }
+}
+
+/** Where the fill has reached, from whichever source is in charge. */
+function played(ctx: RenderContext, s: FullState): number {
+  if (!s.follow) return clamp01(s.progress)
+  const length = s.envelope.duration
+  if (length <= 0) return 0
+  return clamp01(ctx.time.elapsed / length)
+}
 
-  /**
-   * The paint a group takes.
-   *
-   * A voice's colour is a **colour string**, never one of the node's two fill
-   * props, and that is deliberate: a voice is identified by hue, so a gradient
-   * or an image there would defeat the only thing the colour is for. It also
-   * means these paints declare no assets, which is why {@link declareAssets}
-   * only has to account for the two real fill props.
-   */
-  private paintFor(key: string): Fill {
-    if (key === "played") return this.playedFill
-    if (key === "ahead") return this.fill
+/**
+ * The paint a group takes.
+ *
+ * A voice's colour is a **colour string**, never one of the node's two fill
+ * props, and that is deliberate: a voice is identified by hue, so a gradient
+ * or an image there would defeat the only thing the colour is for. It also
+ * means these paints declare no assets, which is why `load` only has to account
+ * for the two real fill props.
+ */
+function paintFor(s: FullState, key: string): Fill {
+  if (key === "played") return s.playedFill
+  if (key === "ahead") return s.fill
 
-    const voice = Number(key.slice("voice:".length))
-    if (!Number.isFinite(voice) || voice < 0) return this.fill
-    return this.voiceColors[voice] ?? voiceColor(voice)
-  }
+  const voice = Number(key.slice("voice:".length))
+  if (!Number.isFinite(voice) || voice < 0) return s.fill
+  return s.voiceColors[voice] ?? voiceColor(voice)
 }