@motionscript/audio 0.0.0-stage → 0.1.0-alpha.10

package/package.json

@@ -1,6 +1,75 @@
 {
   "name": "@motionscript/audio",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.0-alpha.10",
+  "type": "module",
+  "description": "Waveform components for Motion Script: a full recording drawn as bars, and a live meter that moves with playback. Both draw a loudness envelope the host supplies.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/motionscript-dev/motionscript.git",
+    "directory": "packages/components/audio"
+  },
+  "homepage": "https://motionscript.dev",
+  "bugs": {
+    "url": "https://github.com/motionscript-dev/motionscript/issues"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "sideEffects": false,
+  "files": [
+    "NOTICE",
+    "dist",
+    "src",
+    "registry.json",
+    "CHANGELOG.md",
+    "!dist/**/*.tsbuildinfo",
+    "!src/**/tests/**",
+    "!src/**/*.test.ts",
+    "!src/**/*.test.tsx",
+    "!src/**/*.fixtures.ts"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./kit": {
+      "types": "./dist/kit.d.ts",
+      "default": "./dist/kit.js"
+    }
+  },
+  "keywords": [
+    "motion-script",
+    "motion script",
+    "audio",
+    "waveform",
+    "animation"
+  ],
+  "license": "Apache-2.0",
+  "author": "Motionscript",
+  "peerDependencies": {
+    "@motionscript/sdk": "^0.1.0-alpha.10"
+  },
+  "devDependencies": {
+    "@types/node": "^25.6.2",
+    "eslint": "^9.39.4",
+    "eslint-import-resolver-typescript": "^4.4.4",
+    "eslint-plugin-import-x": "^4.16.1",
+    "tsc-alias": "^1.8.16",
+    "typescript": "^6.0.3",
+    "typescript-eslint": "^8.48.0",
+    "vitest": "^4.1.5",
+    "@motionscript/core": "0.1.0-alpha.10",
+    "@motionscript/sdk": "0.1.0-alpha.10"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json --resolve-full-paths && node ../../../scripts/build-browser.mjs",
+    "dev": "tsc -p tsconfig.build.json --watch & tsc-alias -p tsconfig.build.json --resolve-full-paths -w",
+    "clean": "rimraf --glob dist .turbo *.tsbuildinfo",
+    "test": "vitest",
+    "lint": "eslint .",
+    "typecheck": "tsc -p tsconfig.json --noEmit"
+  }
 }

package/registry.json

@@ -0,0 +1,28 @@
+{
+  "kit": {
+    "barrel": "src/kit.ts",
+    "specifier": "@motionscript/audio/kit"
+  },
+  "items": [
+    {
+      "name": "full-waveform",
+      "description": "A whole recording drawn as bars, with a played portion that advances.",
+      "root": "src/waveform",
+      "files": [
+        "envelope.ts",
+        "full-waveform.ts",
+        "shared.ts"
+      ]
+    },
+    {
+      "name": "live-waveform",
+      "description": "A visualiser whose bars move with the recording as it plays, coloured by voice.",
+      "root": "src/waveform",
+      "files": [
+        "envelope.ts",
+        "live-waveform.ts",
+        "shared.ts"
+      ]
+    }
+  ]
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./waveform"
+export { NODES } from "./nodes";

package/src/kit.ts

@@ -0,0 +1,22 @@
+/**
+ * What the two waveform nodes share, importable without copying it.
+ *
+ * A registry item copies `envelope.ts` and `shared.ts` beside its node: they
+ * hold no class and no registered state, so a fork owning its own copy splits
+ * no identity. This is the same surface for a host that samples envelopes or
+ * lays bars out itself, and for a fork that keeps only its node file.
+ */
+export { EMPTY_ENVELOPE, forSpeaker, sampleEnvelope } from "./waveform/envelope"
+export type { AudioEnvelope, EnvelopeBar } from "./waveform/envelope"
+export {
+  VOICE_PALETTE,
+  WAVEFORM_ALIGNMENTS,
+  WAVEFORM_STYLES,
+  clamp01,
+  groupByPaint,
+  layOutBars,
+  livePaintKey,
+  paintBars,
+  voiceColor,
+} from "./waveform/shared"
+export type { BarBox, WaveformAlignment, WaveformStyle } from "./waveform/shared"

package/src/nodes.ts

@@ -0,0 +1,17 @@
+import { FullWaveform } from "./waveform/full-waveform";
+import { LiveWaveform } from "./waveform/live-waveform";
+
+/**
+ * Every node type this package publishes.
+ *
+ * A host registers these by handing the array to an engine —
+ * `new Engine(platform, { nodes: NODES })` — which reads each component's key.
+ * Defining a component only *declares* that key: nothing is registered by
+ * importing a module, so a registry holds exactly what its owner asked for.
+ *
+ * Listing the component **values** is also what survives bundling: this package
+ * declares `sideEffects: false`, and a document names a node type by string, so
+ * an import kept only for its side effect is exactly what gets shaken out.
+ */
+// Typed by name, so the declaration names each component rather than spelling out its type through core.
+export const NODES: (typeof FullWaveform | typeof LiveWaveform)[] = [FullWaveform, LiveWaveform];

package/src/waveform/envelope.ts

@@ -0,0 +1,176 @@
+/**
+ * The shape of a recording, as the waveform nodes draw it.
+ *
+ * ## How an envelope reaches a node: the host supplies it
+ *
+ * Nothing in the engine, and nothing in this kit, decodes audio. A waveform node
+ * does not read a `sound` node's `src`, declares no audio asset and is not tied
+ * to a clip: it draws the {@link AudioEnvelope} it is handed in its `envelope`
+ * prop. Decoding a file into an envelope is the host's job, done once per file
+ * before the document is built. Where the recording sits on the audio lane is
+ * the author's timing too, which is why `FullWaveform.follow` is opt-in.
+ *
+ * ### The JSON form
+ *
+ * A document holds the envelope as plain JSON, because a prop value in a
+ * document is JSON. The typed arrays in {@link AudioEnvelope} are one spelling of
+ * it; `ArrayLike<number>` also accepts the plain arrays a document stores:
+ *
+ * ```json
+ * {
+ *   "magnitude": [0, 0.25, 0.5, 1, 0.5, 0],
+ *   "duration": 6,
+ *   "speaker": [0, 0, 0, 1, 1, -1],
+ *   "speakers": 2
+ * }
+ * ```
+ *
+ * `magnitude` is the peak loudness, `0`…`1`, of buckets that divide `duration`
+ * seconds evenly. `speaker` is `null` when nobody was identified; otherwise it is parallel to `magnitude`, one voice
+ * index per bucket, `-1` for "nobody". `speakers` is the voice count, `0` when
+ * `speaker` is `null`. A `Float32Array` or `Int16Array` is not JSON-serialisable
+ * (`JSON.stringify` writes an object of indices), so a host that holds the typed
+ * form converts with `Array.from` before storing it in a document.
+ *
+ * ## Why the data is baked rather than listened to
+ *
+ * A scene has to render identically in the preview and in a headless export,
+ * and an export has no audio device, no `AudioContext` and no playback to
+ * analyse. So a "live" visualiser cannot read a Web Audio analyser: what it
+ * reads is a *pre-computed envelope*, indexed by the scene clock. The picture is
+ * then a pure function of time, which is what makes scrubbing backwards, jumping
+ * to frame 900, and rendering out of order all produce the same frame.
+ *
+ * The decode happens in the host, once per file, and arrives here through the
+ * build — the same arrangement a chart's rows and a protein's atoms already use,
+ * and for the same reason: the build is synchronous and cannot await a fetch.
+ *
+ * ## Why magnitude rather than min/max
+ *
+ * An editor's waveform is drawn from per-bucket minima *and* maxima, because an
+ * editor is a measuring instrument. These nodes are neither — they are a bar
+ * chart of loudness that happens to be shaped like a recording — and audio is
+ * near enough symmetric that a mirrored magnitude is indistinguishable at the
+ * size a node draws. Half the numbers, and one array to reason about.
+ */
+
+/**
+ * A decoded recording, reduced to a uniform grid of loudness — plus, when the
+ * host knows it, who is talking in each bucket.
+ *
+ * The speaker track is **not** something a decoder can produce; it comes from a
+ * diarized transcript and is folded in by the host before the build. It lives
+ * here rather than in a parallel structure because every reader of it is also a
+ * reader of the magnitudes at the same index, and two arrays that must stay the
+ * same length are better as one object than as two arguments.
+ */
+export interface AudioEnvelope {
+  /**
+   * Peak magnitude per bucket, 0…1. Buckets span {@link duration} evenly. A
+   * `Float32Array`, or the plain `number[]` a document stores.
+   */
+  magnitude: ArrayLike<number>
+  /** The decoded length in seconds. */
+  duration: number
+  /**
+   * Which voice is speaking in each bucket, as an index into the recording's
+   * own numbering, or `-1` for "nobody identified". Parallel to
+   * {@link magnitude}, or `null` when the track was never diarized.
+   */
+  speaker: ArrayLike<number> | null
+  /** How many distinct voices the diarizer found. Zero when it did not run. */
+  speakers: number
+}
+
+/** The envelope of a track that hasn't loaded — or isn't there. */
+export const EMPTY_ENVELOPE: AudioEnvelope = {
+  magnitude: new Float32Array(0),
+  duration: 0,
+  speaker: null,
+  speakers: 0,
+}
+
+/** One drawn bar: how loud, and whose. */
+export interface EnvelopeBar {
+  /** 0…1. */
+  magnitude: number
+  /** The voice index, or -1. */
+  speaker: number
+}
+
+/**
+ * Resamples `[from, to)` seconds of an envelope down to `bars` columns.
+ *
+ * Peak-preserving rather than averaging, which is the whole difference between
+ * a waveform and a blur: averaging a hundred buckets into one column flattens
+ * every transient, and transients are what makes a drawn recording read as
+ * speech rather than as noise. The speaker of a column is the speaker of its
+ * *loudest* bucket, on the same reasoning — a column straddling a turn belongs
+ * to whoever is actually audible in it.
+ *
+ * A window past either end of the recording yields silent, unattributed
+ * columns rather than clamping onto the nearest real one. That matters for the
+ * live visualiser, which is handed a window centred on the playhead and would
+ * otherwise smear the first bucket across the whole run-up to a track's start.
+ */
+export function sampleEnvelope(
+  envelope: AudioEnvelope,
+  from: number,
+  to: number,
+  bars: number
+): EnvelopeBar[] {
+  const count = Math.max(0, Math.floor(bars))
+  const out: EnvelopeBar[] = new Array(count)
+  for (let i = 0; i < count; i++) out[i] = { magnitude: 0, speaker: -1 }
+
+  const buckets = envelope.magnitude.length
+  if (count === 0 || buckets === 0 || envelope.duration <= 0) return out
+
+  const perSecond = buckets / envelope.duration
+  const span = (to - from) / count
+
+  for (let i = 0; i < count; i++) {
+    const start = (from + span * i) * perSecond
+    const end = (from + span * (i + 1)) * perSecond
+
+    // At least one bucket per column, so a window zoomed in past the grid draws
+    // the sample it is standing on rather than an empty band.
+    const first = Math.floor(start)
+    const last = Math.max(first, Math.ceil(end) - 1)
+
+    let loudest = 0
+    let speaker = -1
+    for (let b = first; b <= last; b++) {
+      if (b < 0 || b >= buckets) continue
+      const value = envelope.magnitude[b]
+      if (value < loudest) continue
+      loudest = value
+      speaker = envelope.speaker ? envelope.speaker[b] : -1
+    }
+
+    out[i] = { magnitude: loudest, speaker }
+  }
+
+  return out
+}
+
+/**
+ * The same window, with everything that isn't `voice` silenced.
+ *
+ * The point of the speaker filter, and the reason it silences rather than
+ * *drops*: a podcast visualiser filtered to one host should go quiet while the
+ * other one talks, not compress their turn out of existence. Keeping the bars
+ * and zeroing them is what makes the picture read as "this person is not
+ * speaking right now" instead of as a jump cut.
+ *
+ * `voice` of `null` returns the bars untouched, which is the unfiltered node.
+ */
+export function forSpeaker(
+  bars: EnvelopeBar[],
+  voice: number | null
+): EnvelopeBar[] {
+  if (voice === null) return bars
+  return bars.map((bar) =>
+    bar.speaker === voice ? bar : { magnitude: 0, speaker: bar.speaker }
+  )
+}

package/src/waveform/full-waveform.ts

@@ -0,0 +1,188 @@
+import {
+  Types,
+  component2d,
+  easeInOut,
+  fillOps,
+  type Fill,
+  type FillResolved,
+  type NodeOf,
+  type RenderContext,
+  // Named so the emitted declaration reaches these through the sdk, not core.
+  type Node2D as _Node2D,
+  type Node2DProps as _Node2DProps,
+} from "@motionscript/sdk"
+
+import { EMPTY_ENVELOPE, sampleEnvelope, type AudioEnvelope } from "./envelope"
+import {
+  clamp01,
+  groupByPaint,
+  layOutBars,
+  paintBars,
+  voiceColor,
+  type WaveformAlignment,
+} from "./shared"
+
+/** A paint entry as the waveforms declare theirs: resolved and tweened like a fill, its kind read off the default. */
+const paint = (initial: string) =>
+  Types.custom<Fill, FillResolved[]>(initial, { resolve: fillOps.resolve, lerp: fillOps.lerp })
+
+/**
+ * A **waveform overview**: the whole recording's shape, drawn at once.
+ *
+ * The picture every audio editor opens with, and the one a podcast clip puts
+ * under its caption — a static silhouette of the file, with a played portion
+ * that advances. It answers "how long is this and where are the loud parts",
+ * which is a question about the *file*, not about the moment; its sibling
+ * {@link LiveWaveform} answers the other one.
+ *
+ * ## Progress is authored, not assumed
+ *
+ * `follow` is off by default, and that default is the considered one. A
+ * scene has no idea where the recording it is drawing sits on the video's audio
+ * lane — a clip is placed, trimmed and sped up out there, and none of that is
+ * visible from in here. So the honest default is that the fill is *animation*:
+ * the author sweeps it with a command, the same as any other tween, and it lands
+ * wherever they put it. `follow` is the shortcut for the common arrangement
+ * where the node appears exactly as the track starts, and it says so by being a
+ * switch someone has to turn on rather than a behaviour that quietly disagrees
+ * with the mix.
+ *
+ * ## Voices instead of progress
+ *
+ * With `byVoice` on, the played/ahead split is replaced by one colour per
+ * speaking voice, at a size you can read. For a two-hander it is a map of the conversation: who has the
+ * floor, where the turns are, how long each one ran. It replaces the progress
+ * colours rather than combining with them, because a bar cannot mean two things
+ * at once and "played, and also Grace" is a legend nobody can hold in their
+ * head.
+ */
+export const FullWaveform = component2d({
+  key: "fullWaveform",
+  state: {
+    /** The baked recording. See {@link AudioEnvelope} for why it is baked. */
+    envelope: Types.custom<AudioEnvelope>(EMPTY_ENVELOPE),
+    /** How many columns the whole file is drawn as. */
+    bars: 120,
+    /** Space between bars, as a fraction of the pitch. See {@link layOutBars}. */
+    gap: 0.3,
+    /** Corner radius of a bar, capped at half its width. */
+    radius: 2,
+    /** How much of the file reads as played, `0`–`1`. Tweenable. */
+    progress: 0,
+    /** Drive `progress` from the node's own clock instead of the prop. */
+    follow: false,
+    /** Multiplies every magnitude — a quiet track pulled up to fill its box. */
+    gain: 1,
+    /** How much of the box a silent bar still occupies, `0`–`1`. */
+    floor: 0.015,
+    align: Types.custom<WaveformAlignment>("center"),
+    /** Colour each voice's turns rather than splitting on played / not played. */
+    byVoice: false,
+    /** A colour per voice index. Short or empty falls back to the palette. */
+    voiceColors: Types.custom<string[]>([]),
+    /** The bars ahead of the playhead. */
+    fill: paint("#5b5b7a"),
+    /** …and the bars behind it. */
+    playedFill: paint("#6366f1"),
+  },
+  // Nothing declares a component's paints but itself: `fill` and `playedFill` both.
+  load: (loader, state) => {
+    loader.fill(state.fill)
+    loader.fill(state.playedFill)
+  },
+  commands: {
+    /**
+     * Fill the waveform in, left to right — the node's one command, since
+     * everything else about it is expressible as an ordinary `to`. Eased in-out
+     * unless it is given an easing of its own.
+     */
+    sweep: (timeline, state) => {
+      const ease = timeline.easing ? null : easeInOut()
+      const from = state.progress
+      return timeline.animate((t) => ({ progress: from + (1 - from) * (ease ? ease(t) : t) }))
+    },
+  },
+  render: (ctx, state) => {
+    drawBars(ctx, state)
+    ctx.children()
+  },
+})
+
+export type FullWaveform = NodeOf<typeof FullWaveform>
+export type FullWaveformProps = NonNullable<Parameters<typeof FullWaveform>[0]>
+
+interface FullState {
+  readonly envelope: AudioEnvelope
+  readonly bars: number
+  readonly gap: number
+  readonly radius: number
+  readonly progress: number
+  readonly follow: boolean
+  readonly gain: number
+  readonly floor: number
+  readonly align: WaveformAlignment
+  readonly byVoice: boolean
+  readonly voiceColors: string[]
+  readonly fill: FillResolved[]
+  readonly playedFill: FillResolved[]
+}
+
+function drawBars(ctx: RenderContext, s: FullState): void {
+  const { width, height } = ctx.size
+  if (width <= 0 || height <= 0) return
+
+  const count = Math.max(1, Math.round(s.bars))
+  const sampled = sampleEnvelope(s.envelope, 0, s.envelope.duration, count)
+  const boxes = layOutBars(sampled, {
+    width,
+    height,
+    gap: s.gap,
+    align: s.align,
+    gain: s.gain,
+    floor: s.floor,
+  })
+  if (boxes.length === 0) return
+
+  // One draw per colour rather than per bar — see {@link paintBars}. The key
+  // is a string because it has to be comparable; the paint it stands for is
+  // resolved after the grouping.
+  const edge = played(ctx, s) * boxes.length
+  const groups = groupByPaint(boxes, (i) =>
+    s.byVoice
+      ? `voice:${sampled[i].speaker}`
+      : i < edge
+        ? "played"
+        : "ahead"
+  )
+
+  for (const [key, run] of groups) {
+    const graphics = paintBars(run, paintFor(s, key), s.radius)
+    if (graphics) ctx.draw(graphics)
+  }
+}
+
+/** Where the fill has reached, from whichever source is in charge. */
+function played(ctx: RenderContext, s: FullState): number {
+  if (!s.follow) return clamp01(s.progress)
+  const length = s.envelope.duration
+  if (length <= 0) return 0
+  return clamp01(ctx.time.elapsed / length)
+}
+
+/**
+ * The paint a group takes.
+ *
+ * A voice's colour is a **colour string**, never one of the node's two fill
+ * props, and that is deliberate: a voice is identified by hue, so a gradient
+ * or an image there would defeat the only thing the colour is for. It also
+ * means these paints declare no assets, which is why `load` only has to account
+ * for the two real fill props.
+ */
+function paintFor(s: FullState, key: string): Fill {
+  if (key === "played") return s.playedFill
+  if (key === "ahead") return s.fill
+
+  const voice = Number(key.slice("voice:".length))
+  if (!Number.isFinite(voice) || voice < 0) return s.fill
+  return s.voiceColors[voice] ?? voiceColor(voice)
+}

package/src/waveform/index.ts

@@ -0,0 +1,30 @@
+/**
+ * The waveform family: one subject, two questions.
+ *
+ * - {@link FullWaveform} draws the **file** — the whole recording's silhouette,
+ *   with a played portion that advances. "How long is this, and where are the
+ *   loud parts."
+ * - {@link LiveWaveform} draws the **moment** — bars moving with playback, the
+ *   visualiser a podcast clip puts beside the artwork. "Is anything happening
+ *   right now, and whose voice is it."
+ *
+ * Both read the same baked {@link AudioEnvelope}, which is why they cannot
+ * disagree about a recording, and why neither listens to anything — see that
+ * module for why a scene that has to export cannot use a live analyser.
+ */
+export { FullWaveform } from "./full-waveform"
+export type { FullWaveformProps } from "./full-waveform"
+export { LiveWaveform } from "./live-waveform"
+export type { LiveWaveformProps } from "./live-waveform"
+export { EMPTY_ENVELOPE, forSpeaker, sampleEnvelope } from "./envelope"
+export type { AudioEnvelope, EnvelopeBar } from "./envelope"
+export {
+  VOICE_PALETTE,
+  WAVEFORM_ALIGNMENTS,
+  WAVEFORM_STYLES,
+  groupByPaint,
+  layOutBars,
+  paintBars,
+  voiceColor,
+} from "./shared"
+export type { BarBox, WaveformAlignment, WaveformStyle } from "./shared"