dewdrop-sdk 0.4.1 → 0.5.0

package/README.md

@@ -20,7 +20,7 @@ export default { plugins: [dew()] };
 
 `npx dewctl init` sets up a new game this way, with the Dewdrop CLI beside it.
 To update a project, run `npm install -D dewdrop-sdk@latest`; saves need 0.4.0
-or later.
+or later, voice 0.5.0.
 
 ## Saves locally
 
@@ -58,6 +58,51 @@ included. Call `saves.save()` then to keep what the player has done. `npm
 run dev` ends no session ahead of time, so `session.ending` stays `null`
 there.
 
+## Voice
+
+```js
+import { backend, voice } from 'dewdrop-sdk';
+
+// This player's microphone, a chunk at a time, sent to the backend each frame.
+const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+const capture = voice.capture(stream);                 // capture.stop() to go quiet
+function frame() {
+  for (const chunk of capture.read()) backend.send(chunk);  // Int16Array, 800 samples at 48 kHz
+  requestAnimationFrame(frame);
+}
+requestAnimationFrame(frame);
+
+// Another player's voice, as the backend sends it on: from their figure,
+// loud up close, fading with distance.
+const source = voice.source(listener.context);
+const sound = new PositionalAudio(listener);           // on their avatar
+sound.setNodeSource(source.node);                      // or source.node.connect(anywhere)
+backend.onmessage = ({ data }) => source.write(new Int16Array(data, 4));  // behind the backend's own header
+```
+
+Dewdrop carries no voice between players itself: your `backend.js`
+decides who hears whom — those within earshot, a team, everyone — and
+sends each chunk on through its ordinary messages, so a game can mix,
+mute, record or transcribe as it sees fit. `voice.capture(stream)` takes
+the live microphone stream from `getUserMedia` and hands back the
+capture: `read()` returns the chunks that arrived since the last read,
+each an `Int16Array` of 800 samples, mono, at `voice.SAMPLE_RATE`
+(48 kHz), and `stop()` ends it; one capture at a time, a second throws
+an `InvalidStateError`. A read is at most eight chunks of 1,600 bytes,
+which a client's frame sends to its backend whole within its 16 KiB
+with room beside, so a game may send every chunk it reads; a backend's
+frame sends some 160 copies on within its 256 KiB, so a crowd all
+speaking at once is more than one frame carries and a backend sheds the
+rest — voice is late or nothing — or compresses first. `voice.source(context)`
+is plain Web Audio: `node` is a node of `context` for `connect` or a
+three.js `PositionalAudio`'s `setNodeSource`, `write(samples)` plays an
+`Int16Array` or `Float32Array` right after those written before it —
+the first two chunks ahead of the clock, so a chunk a little early or
+late plays without a gap — dropping what runs eight chunks ahead, and
+`close()` leaves the graph. `npm run dev` reads your browser's
+microphone through an audio worklet and hands your game the same chunks,
+so voice works between tabs locally as it does on Dewdrop.
+
 ## Docs
 
 How to build your game, from the look to model calls, multiplayer, saves and

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dewdrop-sdk",
-  "version": "0.4.1",
-  "description": "Build a three.js game for Dewdrop: the Vite plugin, model calls, multiplayer and saves.",
+  "version": "0.5.0",
+  "description": "Build a three.js game for Dewdrop: the Vite plugin, model calls, multiplayer, saves and voice.",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/index.js

@@ -5,11 +5,13 @@
 // runs, `clients`, in that backend, over its port back, and `saves` into
 // the browser's own storage, which the plugin's script empties as each
 // page loads; `session` says when a session ends ahead of time, which here
-// none does.
+// none does; and `voice` reads the microphone through the browser's own
+// audio worklet, the chunks Dewdrop's server would decode.
 export { llm } from './llm.js';
 export { backend } from './backend.js';
 export { clients } from './clients.js';
 export { session } from './session.js';
+export { voice } from './voice.js';
 import { attachBackend, joining } from './backend.js';
 import { load, save } from './saves.js';
 

package/src/voice.js

@@ -0,0 +1,192 @@
+// `voice` on a developer's machine: the runtime's surface — the player's
+// microphone a chunk at a time, and a source that plays such chunks
+// through the game's own audio graph — where Dewdrop carries no voice
+// between players itself: a game sends what it captures to its backend
+// through its messages, and the backend sends it on to the players it
+// chooses. On Dewdrop the server decodes the microphone the browser
+// streams onto the game's voice stream; here the browser's own audio
+// worklet reads the microphone and hands the game the same chunks, so a
+// game captures in `npm run dev` as it captures on Dewdrop. `source` is
+// plain Web Audio, the same function as Dewdrop's, and plays here as
+// there.
+
+// Hand-copied from the runtime, which tests/sync.test.mjs pins: the rate
+// of the samples a capture reads and a source plays, the samples in one
+// of a capture's chunks, 1/60 s, and the chunks that wait for a game that
+// reads late, the newest kept.
+export const SAMPLE_RATE = 48000;
+export const VOICE_CHUNK_SAMPLES = 800;
+export const VOICE_CHUNKS_HELD = 8;
+// How far ahead of the context's clock a source begins what is written
+// to it — the first chunk, and the first after it ran dry — in seconds:
+// two chunks, which a later chunk may come late by and still follow the
+// last without a gap; and how far ahead writes may run before the newest
+// is dropped instead, so a voice falls no further behind.
+const SOURCE_LEAD = 2 * VOICE_CHUNK_SAMPLES / SAMPLE_RATE;
+const MAX_SOURCE_LEAD = 8 * VOICE_CHUNK_SAMPLES / SAMPLE_RATE;
+
+// The processor that reads the microphone in the browser's audio thread:
+// it takes the microphone's frames at the context's rate, steps through
+// them at SAMPLE_RATE, and posts each VOICE_CHUNK_SAMPLES of them as 16-bit
+// samples, the buffer handed over rather than copied. Its output is
+// silence, so it can hang off the destination, which is what has it run.
+export const CAPTURE_PROCESSOR = 'dew-voice-capture';
+export const CAPTURE_WORKLET = `
+class DewVoiceCapture extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.chunk = new Int16Array(${VOICE_CHUNK_SAMPLES});
+    this.at = 0;
+    // The microphone's frames per sample, and where in the block the next
+    // sample falls.
+    this.step = sampleRate / ${SAMPLE_RATE};
+    this.phase = 0;
+  }
+  process(inputs) {
+    const frames = inputs[0][0];
+    if (frames === undefined) return true;
+    while (this.phase < frames.length) {
+      const i = Math.floor(this.phase);
+      const next = i + 1 < frames.length ? frames[i + 1] : frames[i];
+      const sample = frames[i] + (next - frames[i]) * (this.phase - i);
+      const clamped = sample < -1 ? -1 : sample > 1 ? 1 : sample;
+      this.chunk[this.at++] = Math.round(clamped * 32767);
+      if (this.at === this.chunk.length) {
+        this.port.postMessage(this.chunk, [this.chunk.buffer]);
+        this.chunk = new Int16Array(${VOICE_CHUNK_SAMPLES});
+        this.at = 0;
+      }
+      this.phase += this.step;
+    }
+    this.phase -= frames.length;
+    return true;
+  }
+}
+registerProcessor('${CAPTURE_PROCESSOR}', DewVoiceCapture);
+`;
+
+// The capture under way, if any.
+let capturing = null;
+
+// A stream with a live audio track, as `getUserMedia` resolves one.
+function liveMicrophone(stream) {
+  return stream !== null && typeof stream === 'object' && typeof stream.getAudioTracks === 'function'
+    && stream.getAudioTracks().some((track) => track.readyState === 'live');
+}
+
+// The player's microphone, a chunk at a time: given the stream
+// `getUserMedia` resolved, while its track is live, hands back the
+// capture, whose `read()` returns the chunks that arrived since the last
+// read — each an Int16Array of VOICE_CHUNK_SAMPLES samples, mono, at
+// SAMPLE_RATE, in order, the newest VOICE_CHUNKS_HELD kept when the game
+// reads late — and whose `stop()` ends it, closing the context at once.
+// One at a time. The browser's own graph reads the stream: a context at
+// the sample rate where the browser allows one, the worklet between the
+// stream and the destination.
+function capture(stream) {
+  if (!liveMicrophone(stream)) {
+    throw new TypeError('voice.capture: stream is not a live microphone stream from navigator.mediaDevices.getUserMedia');
+  }
+  if (capturing !== null) throw new DOMException('voice.capture: the microphone is captured already; stop that first', 'InvalidStateError');
+  if (typeof AudioContext === 'undefined' || typeof AudioWorkletNode === 'undefined') {
+    throw new DOMException('voice.capture is unavailable here: the browser has no AudioWorklet', 'NotSupportedError');
+  }
+  let context;
+  try {
+    context = new AudioContext({ sampleRate: SAMPLE_RATE });
+  } catch {
+    context = new AudioContext();
+  }
+  let chunks = [];
+  let live = true;
+  const worklet = URL.createObjectURL(new Blob([CAPTURE_WORKLET], { type: 'text/javascript' }));
+  const ready = context.audioWorklet.addModule(worklet).then(() => {
+    URL.revokeObjectURL(worklet);
+    if (!live) return undefined;
+    const node = new AudioWorkletNode(context, CAPTURE_PROCESSOR, {
+      numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1], channelCount: 1, channelCountMode: 'explicit',
+    });
+    node.port.onmessage = ({ data }) => {
+      if (!live) return;
+      chunks.push(data);
+      if (chunks.length > VOICE_CHUNKS_HELD) chunks.splice(0, chunks.length - VOICE_CHUNKS_HELD);
+    };
+    context.createMediaStreamSource(stream).connect(node).connect(context.destination);
+    // A context the browser keeps suspended until the player's next
+    // gesture starts then; nothing here waits on it.
+    context.resume().catch(() => {});
+  });
+  // A worklet the browser refuses leaves the capture silent; the reason
+  // goes to the console, where the developer is.
+  ready.catch((error) => console.error('voice.capture: the microphone could not be read: ' + error.message));
+  const mine = Object.freeze({
+    read() {
+      if (capturing !== mine || chunks.length === 0) return [];
+      const read = chunks;
+      chunks = [];
+      return read;
+    },
+    stop() {
+      if (capturing !== mine) return;
+      capturing = null;
+      live = false;
+      chunks = [];
+      // Closed at once, whether or not the worklet has loaded or the
+      // context ever ran: a setup still to come finds `live` false and
+      // makes nothing.
+      context.close().catch(() => {});
+    },
+  });
+  capturing = mine;
+  return mine;
+}
+
+function isObjectLike(value) { return value !== null && (typeof value === 'object' || typeof value === 'function'); }
+
+// A source for `context`'s graph that plays the chunks written to it as
+// they come — Int16Array or Float32Array samples at SAMPLE_RATE, mono,
+// as `capture` reads them or as another player's reached this game —
+// each right after the last: `node` is the source, for `connect`
+// or a three.js PositionalAudio's `setNodeSource`; `write` plays a chunk
+// after those before it; `close` leaves the graph. Plain Web Audio — a
+// buffer source a write, begun SOURCE_LEAD ahead of the context's clock
+// and following the last, and nothing of Dewdrop's — so a game may play a
+// voice its own way instead. The runtime's web/voice.js carries this same
+// function, which tests/sync.test.mjs pins line for line.
+function source(context) {
+  if (!isObjectLike(context) || typeof context.createBufferSource !== 'function') {
+    throw new TypeError('voice.source: context is not an AudioContext');
+  }
+  const node = context.createGain();
+  // When the next chunk written plays, on the context's clock.
+  let at = 0;
+  return Object.freeze({
+    node,
+    write(samples) {
+      const tag = Object.prototype.toString.call(samples);
+      if (tag !== '[object Int16Array]' && tag !== '[object Float32Array]') {
+        throw new TypeError('voice.source: write takes an Int16Array or a Float32Array of samples');
+      }
+      if (samples.length === 0) return;
+      const now = context.currentTime;
+      // The first write, or one after the chunks before it played out:
+      // SOURCE_LEAD ahead of the clock. While they still play, right
+      // after them, however early or late this one came, which is what
+      // rides out the jitter; too far ahead of the clock: dropped.
+      if (at <= now) at = now + SOURCE_LEAD;
+      else if (at > now + MAX_SOURCE_LEAD) return;
+      const buffer = context.createBuffer(1, samples.length, SAMPLE_RATE);
+      const channel = buffer.getChannelData(0);
+      if (tag === '[object Float32Array]') channel.set(samples);
+      else for (let i = 0; i < samples.length; i++) channel[i] = samples[i] / 32768;
+      const player = context.createBufferSource();
+      player.buffer = buffer;
+      player.connect(node);
+      player.start(at);
+      at += buffer.duration;
+    },
+    close() { node.disconnect(); },
+  });
+}
+
+export const voice = Object.freeze({ capture, source, SAMPLE_RATE });