@tensordoc/prism 0.1.0

package/src/registry.ts

@@ -0,0 +1,70 @@
+// registry.ts — share-token lookup for catalog content.
+//
+// Every catalog entry on prism.scott.ai has a 6-char base62 token (the
+// short_id). Consumers pass one to `player.load(...)` or via the URL
+// param `?g=<short_id>` and the player resolves it to a PrismGraph
+// offline — no network call. The map is generated from the catalog at
+// build-index time; see scripts/prism/commands/build-index.ts.
+
+import registryData from "./registry.generated.json";
+import { SCHEMA_VERSION, type NodeDef, type NodeType, type PrismGraph } from "./types";
+
+export interface RegistryEntry {
+  name: string;
+  source_type: "milkdrop" | "shadertoy" | "isf" | "wgsl";
+  source_loader: "url" | "npm-butterchurn-presets";
+  source_url?: string;
+  source_ref?: string;
+  default_image?: string;
+}
+
+const registry = registryData as Record<string, RegistryEntry>;
+
+/** Resolve a 6-char short_id to a registry entry. Returns null when
+ *  the id isn't known — callers decide whether to fall back, error,
+ *  or just keep the synthetic cold-open running. */
+export function lookup(shortId: string): RegistryEntry | null {
+  return registry[shortId] ?? null;
+}
+
+/** All known short_ids. Mostly useful for tests + the rotation pool. */
+export function shortIds(): string[] {
+  return Object.keys(registry);
+}
+
+/** Build a minimal PrismGraph that plays the entry behind `shortId`.
+ *  Returns null when the id isn't in the registry. */
+export function shortIdToGraph(shortId: string): PrismGraph | null {
+  const entry = lookup(shortId);
+  if (!entry) return null;
+  return entryToGraph(shortId, entry);
+}
+
+function entryToGraph(shortId: string, entry: RegistryEntry): PrismGraph {
+  const mainParams: Record<string, string> = {};
+  let mainType: NodeType;
+  if (entry.source_type === "shadertoy") {
+    mainType = "lf.shadertoy";
+    if (entry.source_url) mainParams.shader_url = entry.source_url;
+    if (entry.default_image) mainParams.image_url = entry.default_image;
+  } else {
+    mainType = "lf.milkdrop";
+    if (entry.source_loader === "url" && entry.source_url) {
+      mainParams.preset_url = entry.source_url;
+    } else if (entry.source_ref) {
+      mainParams.preset_name = entry.source_ref;
+    }
+  }
+  const nodes: Record<string, NodeDef> = {
+    audio: { type: "signal.audio" },
+    main: { type: mainType, params: mainParams, inputs: { audio: "audio.signal" } },
+    screen: { type: "sink.display", inputs: { frame: "main.frame" } },
+  };
+  return {
+    schema: SCHEMA_VERSION,
+    id: `g:${shortId}`,
+    intent: entry.name,
+    nodes,
+    output: "screen",
+  };
+}

package/src/runtime.ts

@@ -0,0 +1,100 @@
+// runtime.ts — graph executor for prism.graph/0.1.
+//
+// Walks a PrismGraph, finds the light-field generator node, dispatches
+// to the appropriate backend (milkdrop / shadertoy / ...). Future M9
+// will add op.* nodes (blend / displace) and a real compositor; today
+// it's single-backend-at-a-time and swaps as needed.
+//
+// The runtime owns visibility of the two background canvases — only the
+// active backend's canvas is shown, the other is hidden. URL-loaded
+// presets (favorites + shadertoys) load asynchronously; runtime.apply
+// returns synchronously after kicking off the load and updating state.
+
+import type { MilkdropBg } from "./backends/milkdrop";
+import type { ShadertoyBg } from "./backends/shadertoy";
+import { nodesByRole, type PrismGraph } from "./types";
+
+export interface RuntimeContext {
+  milkdrop: MilkdropBg;
+  shadertoy: ShadertoyBg;
+  /** Called to toggle which backend's canvas is visible.
+   *  Implementations set CSS opacity/display on the two canvases. */
+  setActiveBackend: (which: "milkdrop" | "shadertoy") => void;
+}
+
+export interface ApplyResult {
+  ok: boolean;
+  error?: string;
+  /** Display name of the preset that was loaded, if any. */
+  presetName?: string;
+  /** Which backend handled this graph. */
+  backend?: "milkdrop" | "shadertoy";
+}
+
+export class GraphRuntime {
+  private active: PrismGraph | null = null;
+
+  constructor(private readonly ctx: RuntimeContext) {}
+
+  apply(graph: PrismGraph, blendSecondsOverride?: number): ApplyResult {
+    const lfNodes = nodesByRole(graph, "lf");
+    if (lfNodes.length === 0) {
+      return { ok: false, error: "graph has no light-field generator" };
+    }
+    const [, node] = lfNodes[0];
+
+    if (node.type === "lf.milkdrop") {
+      const presetName = node.params?.preset_name;
+      const presetUrl = node.params?.preset_url;
+      const blendSeconds =
+        blendSecondsOverride ??
+        (typeof node.params?.blend_seconds === "number" ? node.params.blend_seconds : 2.5);
+      this.ctx.setActiveBackend("milkdrop");
+      // URL takes precedence (favorites). Fall back to name (npm bundle / legacy).
+      if (typeof presetUrl === "string") {
+        void this.ctx.milkdrop.loadFromUrl(presetUrl, blendSeconds).catch((err: Error) => {
+          console.warn("[runtime] milkdrop loadFromUrl failed:", err.message);
+        });
+        this.active = graph;
+        const name = presetUrl.split("/").pop()?.replace(/\.milk$/, "") ?? presetUrl;
+        return { ok: true, presetName: name, backend: "milkdrop" };
+      }
+      if (typeof presetName !== "string") {
+        return { ok: false, error: "lf.milkdrop missing preset_name or preset_url" };
+      }
+      const loaded = this.ctx.milkdrop.loadByName(presetName, blendSeconds);
+      if (loaded === null) {
+        return { ok: false, error: `preset not found: ${presetName}` };
+      }
+      this.active = graph;
+      return { ok: true, presetName: loaded, backend: "milkdrop" };
+    }
+
+    if (node.type === "lf.shadertoy") {
+      const url = node.params?.shader_url;
+      if (typeof url !== "string") {
+        return { ok: false, error: "lf.shadertoy missing shader_url" };
+      }
+      const imageUrl = node.params?.image_url;
+      this.ctx.setActiveBackend("shadertoy");
+      void this.ctx.shadertoy.loadFromUrl(url).catch((err: Error) => {
+        console.warn("[runtime] shadertoy loadFromUrl failed:", err.message);
+      });
+      // Bind the entry's default image to iChannel1 if specified. The
+      // shader has a 1x1 grey placeholder until this resolves.
+      void this.ctx.shadertoy.bindImage(typeof imageUrl === "string" ? imageUrl : null)
+        .catch((err: Error) => {
+          console.warn("[runtime] shadertoy bindImage failed:", err.message);
+        });
+      this.active = graph;
+      const name = url.split("/").pop()?.replace(/\.glsl$/, "") ?? url;
+      return { ok: true, presetName: name, backend: "shadertoy" };
+    }
+
+    return { ok: false, error: `unsupported lf type: ${node.type}` };
+  }
+
+  get current(): PrismGraph | null {
+    return this.active;
+  }
+}

package/src/synth.ts

@@ -0,0 +1,251 @@
+// synthetic-signal.ts — silent "fractal" audio driver.
+// Generates a pink-noise-like spectrum (octave-spaced oscillators, amplitude
+// ∝ 1/√f) plus periodic beat envelopes and slow frequency mutations. The
+// signal is read by butterchurn's analyser to keep the Milkdrop preset
+// animated while no real audio is shared. Output never reaches ctx.destination,
+// so nothing is audible.
+
+type Band = "bass" | "mid" | "treble";
+type Voice = {
+  base: number;
+  baseGain: number;
+  band: Band;
+  osc: OscillatorNode;
+  lfo: OscillatorNode;
+  lfoGain: GainNode;
+  gain: GainNode;
+};
+
+export class SyntheticSignal {
+  private readonly ctx: AudioContext;
+  private readonly output: GainNode;
+  private readonly voices: Voice[] = [];
+  private readonly analyser: AnalyserNode;
+  private readonly fft: Uint8Array;
+  private beatTimer: number | null = null;
+  private mutationTimer: number | null = null;
+  private stopped = false;
+  private lastCursorBeatAt = 0;
+
+  constructor(ctx: AudioContext) {
+    this.ctx = ctx;
+    this.output = ctx.createGain();
+    this.output.gain.value = 0.65;
+
+    // Octave-spaced voices producing a 1/√f spectrum — convincingly "musical"
+    // to the analyser without being recognisable as anything in particular.
+    //   freqHz, type, baseGain, lfoHz, lfoDepth
+    const seeds: Array<[number, OscillatorType, number, number, number]> = [
+      [ 50,   "sine",      0.55, 0.06,  18 ],
+      [ 110,  "sine",      0.40, 0.09,  24 ],
+      [ 230,  "triangle",  0.28, 0.12,  60 ],
+      [ 460,  "sawtooth",  0.18, 0.17, 140 ],
+      [ 950,  "triangle",  0.12, 0.21, 280 ],
+      [ 1900, "sine",      0.08, 0.27, 520 ],
+      [ 3900, "triangle",  0.05, 0.33, 1100 ],
+    ];
+    for (const [f, type, g, lfoHz, lfoD] of seeds) this.addVoice(f, type, g, lfoHz, lfoD);
+
+    // Internal analyser so callers can read our energy without colliding with
+    // butterchurn's own analyser.
+    this.analyser = ctx.createAnalyser();
+    this.analyser.fftSize = 256;
+    this.analyser.smoothingTimeConstant = 0.78;
+    this.output.connect(this.analyser);
+    this.fft = new Uint8Array(this.analyser.frequencyBinCount);
+
+    this.scheduleBeats();
+    this.scheduleMutations();
+  }
+
+  /** The node to feed into a consumer (e.g. butterchurn.connectAudio). */
+  getOutput(): AudioNode {
+    return this.output;
+  }
+
+  /** Returns a smoothed 0..1 energy reading from our own analyser. */
+  readEnergy(): number {
+    if (this.stopped) return 0;
+    this.analyser.getByteFrequencyData(this.fft as unknown as Uint8Array<ArrayBuffer>);
+    let s = 0;
+    for (let i = 0; i < this.fft.length; i++) s += this.fft[i];
+    return s / 255 / this.fft.length;
+  }
+
+  /** Returns smoothed bass/mid/treble bands (0..1 each). */
+  readBands(): { bass: number; mid: number; treble: number } {
+    if (this.stopped) return { bass: 0, mid: 0, treble: 0 };
+    this.analyser.getByteFrequencyData(this.fft as unknown as Uint8Array<ArrayBuffer>);
+    const n = this.fft.length;
+    const bassEnd = Math.floor(n * 0.10);
+    const midEnd = Math.floor(n * 0.45);
+    let b = 0, m = 0, t = 0;
+    for (let i = 0; i < n; i++) {
+      const v = this.fft[i] / 255;
+      if (i < bassEnd) b += v;
+      else if (i < midEnd) m += v;
+      else t += v;
+    }
+    return {
+      bass: b / Math.max(1, bassEnd),
+      mid: m / Math.max(1, midEnd - bassEnd),
+      treble: t / Math.max(1, n - midEnd),
+    };
+  }
+
+  stop(): void {
+    if (this.stopped) return;
+    this.stopped = true;
+    if (this.beatTimer != null) clearTimeout(this.beatTimer);
+    if (this.mutationTimer != null) clearTimeout(this.mutationTimer);
+    for (const v of this.voices) {
+      try { v.osc.stop(); v.lfo.stop(); } catch { /* ignore */ }
+      try { v.osc.disconnect(); v.lfo.disconnect(); v.lfoGain.disconnect(); v.gain.disconnect(); }
+      catch { /* ignore */ }
+    }
+    try { this.output.disconnect(); } catch { /* ignore */ }
+    try { this.analyser.disconnect(); } catch { /* ignore */ }
+  }
+
+  private addVoice(
+    freq: number,
+    type: OscillatorType,
+    gain: number,
+    lfoHz: number,
+    lfoDepth: number,
+  ): void {
+    const ctx = this.ctx;
+    const osc = ctx.createOscillator();
+    osc.frequency.value = freq;
+    osc.type = type;
+    const g = ctx.createGain();
+    g.gain.value = gain;
+    osc.connect(g).connect(this.output);
+
+    const lfo = ctx.createOscillator();
+    lfo.frequency.value = lfoHz;
+    lfo.type = "sine";
+    const lfoGain = ctx.createGain();
+    lfoGain.gain.value = lfoDepth;
+    lfo.connect(lfoGain).connect(osc.frequency);
+
+    lfo.start();
+    osc.start();
+    const band: Band = freq < 200 ? "bass" : freq < 1100 ? "mid" : "treble";
+    this.voices.push({ base: freq, baseGain: gain, band, osc, lfo, lfoGain, gain: g });
+  }
+
+  /**
+   * Modulate the synth from the cursor. Position controls the bass/treble
+   * balance (top-right = bright/airy; bottom-left = bassy/warm); velocity
+   * triggers a beat envelope on the master gain. The cursor becomes the
+   * instrument driving the milkdrop preset.
+   *
+   *   @param x01      cursor X normalised to 0..1 (left .. right)
+   *   @param y01      cursor Y normalised to 0..1 (top .. bottom)
+   *   @param vel01    cursor speed normalised to 0..1
+   */
+  setCursorModulation(x01: number, y01: number, vel01: number): void {
+    if (this.stopped) return;
+    const now = this.ctx.currentTime;
+    const ramp = 0.12;
+    const x = Math.max(0, Math.min(1, x01));
+    const y = Math.max(0, Math.min(1, y01));
+    const v = Math.max(0, Math.min(1, vel01));
+
+    // bass: bottom of screen = louder
+    const bassMul = 0.55 + (1 - y) * 0.90 + v * 0.15;
+    // treble: right of screen = brighter
+    const trebleMul = 0.55 + x * 0.90 + v * 0.20;
+    // mid: rises with velocity (gives "presence" to motion)
+    const midMul = 0.7 + v * 0.7;
+
+    for (const voice of this.voices) {
+      const target =
+        voice.band === "bass" ? voice.baseGain * bassMul :
+        voice.band === "treble" ? voice.baseGain * trebleMul :
+        voice.baseGain * midMul;
+      voice.gain.gain.cancelScheduledValues(now);
+      voice.gain.gain.setValueAtTime(voice.gain.gain.value, now);
+      voice.gain.gain.linearRampToValueAtTime(target, now + ramp);
+    }
+
+    // Velocity-burst → master gain envelope (rate-limited so a steady fast
+    // glide doesn't constantly hammer the analyser).
+    if (v > 0.45 && now - this.lastCursorBeatAt > 0.22) {
+      const peak = 0.95 + v * 0.45;
+      this.output.gain.cancelScheduledValues(now);
+      this.output.gain.setValueAtTime(this.output.gain.value, now);
+      this.output.gain.linearRampToValueAtTime(peak, now + 0.035);
+      this.output.gain.exponentialRampToValueAtTime(0.52, now + 0.22);
+      this.lastCursorBeatAt = now;
+    }
+  }
+
+  /**
+   * Fire a bass-kick envelope — for hooking up heartbeats, MIDI clicks, or
+   * any external rhythmic trigger. Sharp attack + bass voice swell so the
+   * spectrum butterchurn sees registers as a clean bass beat.
+   *
+   *   @param intensity  0..1, scales the peak
+   */
+  pulseBeat(intensity = 0.8): void {
+    if (this.stopped) return;
+    const now = this.ctx.currentTime;
+    const i = Math.max(0, Math.min(1, intensity));
+
+    // Master gain — sharp attack, exponential decay (kick-drum shape)
+    const masterPeak = 0.95 + i * 0.55;
+    this.output.gain.cancelScheduledValues(now);
+    this.output.gain.setValueAtTime(this.output.gain.value, now);
+    this.output.gain.linearRampToValueAtTime(masterPeak, now + 0.035);
+    this.output.gain.exponentialRampToValueAtTime(0.42, now + 0.26);
+
+    // Bass voices get an extra swell so the kick lands in the low band
+    // (which is where butterchurn's preset reactivity usually lives).
+    for (const voice of this.voices) {
+      if (voice.band !== "bass") continue;
+      const peak = voice.baseGain * (1.6 + i * 1.4);
+      voice.gain.gain.cancelScheduledValues(now);
+      voice.gain.gain.setValueAtTime(voice.gain.gain.value, now);
+      voice.gain.gain.linearRampToValueAtTime(peak, now + 0.025);
+      voice.gain.gain.exponentialRampToValueAtTime(
+        Math.max(0.01, voice.baseGain), now + 0.34,
+      );
+    }
+  }
+
+  /** Periodic "beat" — short amplitude burst on the master gain. */
+  private scheduleBeats(): void {
+    const tick = (): void => {
+      if (this.stopped) return;
+      const now = this.ctx.currentTime;
+      const peak = 0.95 + Math.random() * 0.35;
+      this.output.gain.cancelScheduledValues(now);
+      this.output.gain.setValueAtTime(this.output.gain.value, now);
+      this.output.gain.linearRampToValueAtTime(peak, now + 0.04);
+      this.output.gain.exponentialRampToValueAtTime(0.45, now + 0.28);
+      // BPM-ish range 60–110, jittered
+      const nextMs = 600 + Math.random() * 1000;
+      this.beatTimer = window.setTimeout(tick, nextMs);
+    };
+    this.beatTimer = window.setTimeout(tick, 900);
+  }
+
+  /** Every ~20–35 s, smoothly mutate one voice's base frequency over 8 s. */
+  private scheduleMutations(): void {
+    const tick = (): void => {
+      if (this.stopped) return;
+      const v = this.voices[Math.floor(Math.random() * this.voices.length)];
+      const now = this.ctx.currentTime;
+      const ratio = 0.65 + Math.random() * 0.7; // 0.65× .. 1.35×
+      const target = Math.max(20, Math.min(6000, v.base * ratio));
+      v.osc.frequency.cancelScheduledValues(now);
+      v.osc.frequency.setValueAtTime(v.osc.frequency.value, now);
+      v.osc.frequency.linearRampToValueAtTime(target, now + 7.5);
+      v.base = target;
+      this.mutationTimer = window.setTimeout(tick, 18000 + Math.random() * 16000);
+    };
+    this.mutationTimer = window.setTimeout(tick, 6000);
+  }
+}

package/src/types.ts

@@ -0,0 +1,98 @@
+// prism.graph/0.1 — node-graph schema for visualizations.
+//
+// Every Prism visualization is a JSON document of this shape. Nodes fall
+// into five role-tagged categories that mirror the fundamental abstraction:
+//
+//   signal source → signal xform → light field generator → lf operator → sink
+//
+// In English: Prism computes light fields from signals.
+//
+// M1 only exercises the minimal graph (audio → milkdrop → display); the
+// schema is intentionally roomy so M8/M9 can add more generators and
+// compositor ops without rewriting it.
+
+export const SCHEMA_VERSION = "prism.graph/0.1" as const;
+
+export type NodeRole = "signal" | "xform" | "lf" | "op" | "sink";
+
+export type NodeType =
+  // signal sources — produce a streaming signal
+  | "signal.audio"
+  | "signal.cursor"
+  | "signal.heartbeat"
+  | "signal.synth"
+  // signal transformers — signal → signal
+  | "xform.gain"
+  | "xform.beat"
+  // light field generators — produce a frame from signals
+  | "lf.milkdrop"
+  | "lf.shadertoy"
+  | "lf.isf"
+  // light field operators — frame(s) → frame
+  | "op.blend"
+  | "op.displace"
+  | "op.feedback"
+  // sinks — terminate the graph
+  | "sink.display"
+  | "sink.recorder";
+
+export type NodeParam =
+  | string
+  | number
+  | boolean
+  | string[]
+  | number[]
+  | Record<string, string | number | boolean>;
+
+/** A single node in the graph. Inputs reference upstream node outputs by
+ *  the string "<node_id>.<output_name>". The runtime resolves these. */
+export interface NodeDef {
+  type: NodeType;
+  params?: Record<string, NodeParam>;
+  inputs?: Record<string, string>;
+}
+
+export interface PrismGraph {
+  schema: typeof SCHEMA_VERSION;
+  /** Stable id — enables share-by-URL in M6. */
+  id: string;
+  /** Human-readable summary of what the graph does. Shown in the SKILL
+   *  readout; lets the AI explain its choice. */
+  intent: string;
+  nodes: Record<string, NodeDef>;
+  /** Node id of the terminal sink. */
+  output: string;
+}
+
+/** Extract the role prefix from a node type. */
+export function roleOf(type: NodeType): NodeRole {
+  return type.split(".")[0] as NodeRole;
+}
+
+/** Iterate nodes of a given role. */
+export function nodesByRole(graph: PrismGraph, role: NodeRole): Array<[string, NodeDef]> {
+  return Object.entries(graph.nodes).filter(([, n]) => roleOf(n.type) === role);
+}
+
+/** Build the M1 minimal graph: audio_in → lf.milkdrop(preset) → sink.display. */
+export function makeMilkdropGraph(
+  presetName: string,
+  intent: string,
+  blendSeconds = 2.5,
+): PrismGraph {
+  return {
+    schema: SCHEMA_VERSION,
+    id: `g_${Math.random().toString(36).slice(2, 10)}`,
+    intent,
+    nodes: {
+      audio_in: { type: "signal.audio", params: {} },
+      main: {
+        type: "lf.milkdrop",
+        params: { preset_name: presetName, blend_seconds: blendSeconds },
+        inputs: { audio: "audio_in.signal" },
+      },
+      out: { type: "sink.display", inputs: { frame: "main.frame" } },
+    },
+    output: "out",
+  };
+}