castle-web-cli 0.4.128 → 0.4.129

package/kits/physics-2d/engine/media.js

@@ -1,6 +1,8 @@
 // The pool of playing media elements: one `<audio>` / `<video>` per actor per
-// file. Behaviors (Sound, Video) ask for theirs each frame and set the state
-// they want on it; this module owns creating, reusing, and reaping them.
+// file. Behaviors (SoundPlayer with `stream` on, Video) ask for theirs each frame
+// and set the state they want on it; this module owns creating, reusing, and
+// reaping them. Short sound effects don't come through here at all -- they are
+// decoded buffers, in engine/sound.js.
 //
 // The pool is MODULE-level, not per-SceneRuntime, because the editor builds a
 // throwaway runtime every frame to draw the edit preview -- a per-runtime pool
@@ -15,7 +17,7 @@
 // editor's video preview draws a paused element's first frame.
 
 import { initialMedia } from './files';
-import { getAudioContext, resumeAudioContext } from './audioContext';
+import { getAudioContext, resumeAudioContext, unlockTargets } from './audioContext';
 
 // Reaped after this long without being asked for. Long enough to survive a
 // scene's worth of frames where an actor is briefly not drawn (offscreen,
@@ -24,10 +26,6 @@ import { getAudioContext, resumeAudioContext } from './audioContext';
 const IDLE_REAP_MS = 1500;
 
 const entries = new Map();
-// One-shot plays: copies of an element that overlap the original instead of
-// cutting it off (a sound effect fired again before the last one finished).
-// They are not keyed by anything -- they exist until they end.
-const oneShots = new Set();
 // Elements routed through WebAudio for panning, and their panner nodes. An
 // element can only be connected to the graph ONCE, so this is also the record
 // of which ones already are.
@@ -63,8 +61,10 @@ function bindUnlock() {
       );
     }
   };
-  window.addEventListener('pointerdown', retry, { passive: true });
-  window.addEventListener('keydown', retry, { passive: true });
+  for (const target of unlockTargets()) {
+    target.addEventListener('pointerdown', retry, { passive: true });
+    target.addEventListener('keydown', retry, { passive: true });
+  }
 }
 
 // Play, tolerating the autoplay policy: a refused play is queued for the next
@@ -81,6 +81,26 @@ export function playMedia(element) {
   }
 }
 
+// Retune a sound along with its speed, the way a tape does -- and the way the
+// buffer path's playbackRate already does.
+//
+// An element does the OPPOSITE by default. `preservesPitch` starts TRUE, so it
+// time-stretches to hold the pitch steady while the speed changes: right for a
+// podcast at 1.5x, wrong for a game sound, and audibly a phase vocoder rather
+// than a clean retune. Measured on a 110Hz tone at rate 2: 111Hz with the
+// default, 220Hz with this off -- the latter matching an AudioBufferSourceNode
+// exactly, which is what makes one `playbackRate` prop mean one thing across
+// both of this kit's playback paths.
+//
+// The prefixed spellings are for browsers older than the standard property;
+// assigning one that doesn't exist is harmless.
+export function setMediaPlaybackRate(element, rate) {
+  element.preservesPitch = false;
+  element.webkitPreservesPitch = false;
+  element.mozPreservesPitch = false;
+  element.playbackRate = rate;
+}
+
 // Pan a sound left/right (-1..1). The element has no panning of its own, so the
 // first non-zero pan routes it through a WebAudio graph -- permanently, since a
 // media element can only be connected once. Which is why pan 0 (the default)
@@ -107,31 +127,6 @@ export function setMediaPan(element, pan) {
   resumeAudioContext();
 }
 
-// Play a copy of this sound, overlapping whatever is already playing. Returns a
-// promise that resolves when the copy finishes, so a caller can wait for it.
-// Copies are never reused: a one-shot is done when it's done.
-export function playOneShot(path, { volume = 1, playbackRate = 1, pan = 0 } = {}) {
-  const url = mediaUrl(path);
-  if (!url) return Promise.resolve();
-  const element = document.createElement('audio');
-  element.src = url;
-  element.volume = volume;
-  element.playbackRate = playbackRate;
-  oneShots.add(element);
-  const finished = new Promise((resolve) => {
-    const done = () => {
-      oneShots.delete(element);
-      element.removeAttribute('src');
-      resolve();
-    };
-    element.addEventListener('ended', done, { once: true });
-    element.addEventListener('error', done, { once: true });
-  });
-  if (pan !== 0) setMediaPan(element, pan);
-  playMedia(element);
-  return finished;
-}
-
 export function stopMedia(element) {
   blocked.delete(element);
   element.pause();
@@ -202,11 +197,6 @@ export function stopAllMedia() {
     stopMedia(entry.element);
     entry.element.removeAttribute('src');
   }
-  for (const element of oneShots) {
-    stopMedia(element);
-    element.removeAttribute('src');
-  }
   entries.clear();
-  oneShots.clear();
   blocked.clear();
 }

package/kits/physics-2d/engine/sound.js

@@ -0,0 +1,316 @@
+// The deck's sound effects: audio files decoded into WebAudio buffers ahead of
+// time, then fired as overlapping one-shots from anywhere.
+//
+// This is the SFX half of the kit's audio. The other half is `engine/media.js`,
+// which plays an `<audio>` element -- the right thing for a long track, whose
+// decoded PCM would be tens of megabytes held in memory for as long as the deck
+// runs (a minute of 44.1kHz stereo is ~21MB). Short sounds go here, long ones
+// stream there, and `behaviors/SoundPlayer.jsx` is where a deck chooses.
+//
+// Deliberately decoupled from the rest of the kit: the file map arrives through
+// `initSounds` rather than being imported, and nothing here knows about actors,
+// scenes or blueprints. The one kit import is the shared AudioContext, which has
+// to be shared -- a second context could not mix with the element path. So
+// moving this into the SDK or a package other kits can use is a file move, not a
+// rewrite.
+//
+// A context starts SUSPENDED until the page has been interacted with, but
+// decoding does not need a running one, so preloading begins at load and only
+// playback waits for the unlock (see engine/audioContext.js).
+
+import { bindAudioResume, getAudioContext, resumeAudioContext } from './audioContext';
+
+// Generous on purpose: a voice is ~3 nodes, so this is a backstop against a
+// runaway loop, not a mixing decision. Sounds stack and clip if a deck asks them
+// to -- that is the deck's business.
+const MAX_VOICES = 128;
+
+// Preloaded sounds live here, in the deck's own tree or an import's (so a kit
+// can ship its own effects). Everything else in the deck is still playable and
+// still listed in the inspector -- this decides what is held in memory, not what
+// exists. Keeping it a DIRECTORY rather than a size cutoff means the answer is
+// visible in the file tree instead of buried in a constant here.
+const PRELOAD_DIR = /^(?:imports\/[^/]+\/)?assets\/sounds\//;
+
+// Where an import's files live at the deck root, mirroring castle-web-sdk's
+// IMPORTS_DIR. Spelled out rather than imported so this module keeps depending on
+// nothing but the shared AudioContext.
+const IMPORTS_PREFIX = 'imports/';
+const isImported = (path) => path.startsWith(IMPORTS_PREFIX);
+
+// A preloaded sound longer than this is probably meant to stream. Warned about,
+// never excluded: what preloads is the file tree's business, and a rule that
+// silently dropped a file would be exactly the hidden internal the directory
+// convention exists to avoid.
+const LONG_SOUND_SECONDS = 15;
+
+const DEV = Boolean(import.meta.env?.DEV);
+
+let media = {};
+let stems = new Map();
+const buffers = new Map();
+const loading = new Map();
+// Insertion-ordered, which is what makes the cap's "steal the oldest" a lookup
+// rather than a search.
+const voices = new Set();
+const preloadSet = new Set();
+const warned = new Set();
+let master = null;
+let ready = null;
+let preloaded = false;
+
+function warnOnce(key, message) {
+  if (!DEV || warned.has(key)) return;
+  warned.add(key);
+  console.warn(`[sound] ${message}`);
+}
+
+function clamp(value, min, max, fallback) {
+  return Number.isFinite(value) ? Math.min(max, Math.max(min, value)) : fallback;
+}
+
+// Stems within one deck: `hit` for `assets/sounds/hit.wav`. A stem shared by two
+// of that deck's files resolves to NEITHER -- picking one arbitrarily would make
+// the winner a function of sort order, which the deck author cannot see. The full
+// path always works and is how such a pair is told apart.
+function uniqueStems(paths, ambiguous) {
+  const found = new Map();
+  for (const path of paths) {
+    const stem = path.slice(path.lastIndexOf('/') + 1).replace(/\.[^.]+$/, '');
+    if (found.has(stem)) ambiguous.add(stem);
+    else found.set(stem, path);
+  }
+  for (const stem of ambiguous) found.delete(stem);
+  return found;
+}
+
+// Short names for sounds, resolved deck-first: a sound the deck owns beats one of
+// the same name from an import, the same precedence editors/behaviorRegistry.js
+// gives a deck's behaviors over a dependency's. Without that, a kit that shipped
+// its own `click.wav` would make every importing deck's `play('click')` ambiguous
+// -- a kit could break decks by adding a file.
+function buildStems(paths) {
+  const ownAmbiguous = new Set();
+  const importAmbiguous = new Set();
+  const imported = uniqueStems(paths.filter(isImported), importAmbiguous);
+  const own = uniqueStems(paths.filter((path) => !isImported(path)), ownAmbiguous);
+  const merged = new Map([...imported, ...own]);
+  for (const stem of ownAmbiguous) {
+    merged.delete(stem);
+    warnOnce(`stem:${stem}`, `This deck has more than one sound named "${stem}" -- play it by full path instead.`);
+  }
+  for (const stem of importAmbiguous) {
+    if (own.has(stem)) continue;
+    merged.delete(stem);
+    warnOnce(`stem:${stem}`, `An import has more than one sound named "${stem}" -- play it by full path instead.`);
+  }
+  return merged;
+}
+
+function resolvePath(name) {
+  const key = typeof name === 'string' ? name.replace(/^\.\//, '') : '';
+  if (media[key]) return key;
+  return stems.get(key) ?? null;
+}
+
+// Everything is mixed through one gain node, so a master volume and a fade on
+// stop have somewhere to live later. Unity, and no compressor: a deck that
+// stacks forty copies of one sample is meant to clip.
+function masterGain() {
+  const ctx = getAudioContext();
+  if (!ctx) return null;
+  if (!master || master.context !== ctx) {
+    master = ctx.createGain();
+    master.gain.value = 1;
+    master.connect(ctx.destination);
+  }
+  return master;
+}
+
+async function decode(path) {
+  const url = media[path];
+  const ctx = getAudioContext();
+  if (!url || !ctx) return null;
+  try {
+    const response = await fetch(url);
+    const buffer = await ctx.decodeAudioData(await response.arrayBuffer());
+    buffers.set(path, buffer);
+    return buffer;
+  } catch (error) {
+    warnOnce(`decode:${path}`, `Could not decode ${path}: ${error?.message ?? error}`);
+    return null;
+  }
+}
+
+// One decode per file, however many callers ask at once -- two `play`s of an
+// unloaded sound in the same frame must not decode it twice.
+function load(path) {
+  if (buffers.has(path)) return Promise.resolve(buffers.get(path));
+  const inFlight = loading.get(path);
+  if (inFlight) return inFlight;
+  const started = decode(path).finally(() => loading.delete(path));
+  loading.set(path, started);
+  return started;
+}
+
+async function preload(path) {
+  const buffer = await load(path);
+  if (buffer && buffer.duration > LONG_SOUND_SECONDS) {
+    warnOnce(
+      `long:${path}`,
+      `${path} is ${Math.round(buffer.duration)}s and is held decoded in memory. ` +
+        `Long tracks belong outside assets/sounds/, played by a SoundPlayer with stream on.`
+    );
+  }
+}
+
+/** Whether a deck path is one this library preloads. */
+export function isPreloadedPath(path) {
+  return PRELOAD_DIR.test(String(path ?? ''));
+}
+
+/**
+ * Point the library at the deck's audio files and start preloading.
+ *
+ * @param audioFiles path -> URL, every audio file in the deck (see
+ *   engine/files.js -- URLs, not paths, so this keeps working once the deck is
+ *   bundled into a single file and the assets are `data:` URIs).
+ */
+export function initSounds(audioFiles) {
+  media = { ...audioFiles };
+  stems = buildStems(Object.keys(media));
+  preloadSet.clear();
+  for (const path of Object.keys(media)) {
+    if (PRELOAD_DIR.test(path)) preloadSet.add(path);
+  }
+  // No sounds to preload means no AudioContext either: a deck that makes no
+  // sound should not build an audio graph just by loading the kit.
+  if (preloadSet.size === 0) {
+    preloaded = true;
+    ready = Promise.resolve();
+    return ready;
+  }
+  ready = Promise.all([...preloadSet].map(preload)).then(() => {
+    preloaded = true;
+  });
+  return ready;
+}
+
+function stopVoice(voice) {
+  voices.delete(voice);
+  try {
+    voice.source.stop();
+  } catch {
+    // Already ended -- `ended` has fired or the source never started.
+  }
+}
+
+function startVoice(buffer, { volume, pan, playbackRate }) {
+  const ctx = getAudioContext();
+  const out = masterGain();
+  if (!ctx || !out) return null;
+  // The context starts suspended until the page has been interacted with. This
+  // asks now, and binds a listener so the next gesture asks again -- a deck with
+  // no streaming sound has nothing else that would.
+  bindAudioResume();
+  resumeAudioContext();
+  if (voices.size >= MAX_VOICES) stopVoice(voices.values().next().value);
+
+  const source = ctx.createBufferSource();
+  source.buffer = buffer;
+  source.playbackRate.value = clamp(playbackRate, 0.01, 10, 1);
+  const gain = ctx.createGain();
+  gain.gain.value = clamp(volume, 0, 1, 1);
+  source.connect(gain);
+  let tail = gain;
+  const wanted = clamp(pan, -1, 1, 0);
+  if (wanted !== 0 && ctx.createStereoPanner) {
+    const panner = ctx.createStereoPanner();
+    panner.pan.value = wanted;
+    gain.connect(panner);
+    tail = panner;
+  }
+  tail.connect(out);
+
+  const voice = { source };
+  voices.add(voice);
+  const finished = new Promise((resolve) => {
+    source.addEventListener('ended', () => {
+      voices.delete(voice);
+      resolve();
+    });
+  });
+  source.start();
+  return {
+    stop() {
+      stopVoice(voice);
+    },
+    finished,
+  };
+}
+
+/**
+ * Play a sound, overlapping whatever else is playing.
+ *
+ * `name` is a deck-relative path (`assets/sounds/hit.wav`) or the bare filename
+ * without its extension (`hit`).
+ *
+ * Returns `{ stop, finished }`, or null when nothing played -- an unknown name,
+ * or a sound whose buffer is not decoded yet. A play that lands before its sound
+ * is ready is DROPPED rather than deferred: a sound effect that arrives late is
+ * worse than one that never arrives, and deferring it would fire it at a moment
+ * that has already passed.
+ */
+export function playSound(name, { volume = 1, pan = 0, playbackRate = 1 } = {}) {
+  const path = resolvePath(name);
+  if (!path) {
+    warnOnce(`unknown:${name}`, `No sound named "${name}".`);
+    return null;
+  }
+  const buffer = buffers.get(path);
+  if (buffer) return startVoice(buffer, { volume, pan, playbackRate });
+  // Not decoded. Start it now so the NEXT play works -- which for a sound
+  // outside assets/sounds/ is what makes the first play the only silent one.
+  void load(path);
+  if (!preloadSet.has(path)) {
+    warnOnce(
+      `cold:${path}`,
+      `${path} is not preloaded, so this first play is silent. ` +
+        `Move it under assets/sounds/ to have it ready before the deck starts.`
+    );
+  }
+  return null;
+}
+
+/** Every sound effect, off. Long tracks are the element path's to stop. */
+export function stopAllSounds() {
+  for (const voice of [...voices]) stopVoice(voice);
+  voices.clear();
+}
+
+// The deck-facing library, also reachable as `scene.sound` (see systems/media.js)
+// so a behavior needs no import to make a noise.
+export const Sound = {
+  /** True once every preloaded sound has been decoded (or failed to). */
+  get isLoaded() {
+    return preloaded;
+  },
+  /** Resolves when preloading settles, so a caller need not poll `isLoaded`. */
+  get whenLoaded() {
+    return ready ?? Promise.resolve();
+  },
+  /** Whether a name resolves to a sound this deck has. */
+  has(name) {
+    return resolvePath(name) !== null;
+  },
+  /**
+   * How many voices are sounding right now. Worth watching if effects seem to
+   * cut each other off: at MAX_VOICES the oldest is stolen to make room, and
+   * nothing else says so.
+   */
+  get voiceCount() {
+    return voices.size;
+  },
+  play: playSound,
+  stopAll: stopAllSounds,
+};