dsh-realtime-audio-ws 0.1.2 → 0.1.3

package/lib/client/bundle.js

@@ -0,0 +1,278 @@
+window.__ModuleLoader__.load({ id: "dsh-realtime-audio-ws",
+  factory: (require) => {
+    const exports = {}
+    const module = { exports }
+    "use strict";
+    /**
+     * The browser face: microphone in, speaker out, over the host route.
+     *
+     * This is the client half of the transport. `dsh-realtime-audio-ws` claims the socket on the host and
+     * bridges it to the agent's audio events; this file opens that socket, captures the microphone into it, and
+     * plays what comes back.
+     *
+     * **One file, on purpose.** A client face is served as a single resource (`<package>/client.js`), so a
+     * relative value import would emit a second file the loader never serves. Everything therefore lives here,
+     * including the API shapes it needs.
+     *
+     * **No DOM lib, also on purpose.** The browser APIs are described structurally and read off `globalThis`
+     * through one cast, so this compiles in the package's ordinary program — no second tsconfig, no DOM lib,
+     * no bundler — and every path is testable by injecting fakes. `standard-browser-types` would have bought
+     * accurate types for six objects at the cost of a whole second build face; this is the cheaper trade.
+     *
+     * **`ScriptProcessorNode`, and why.** The modern capture path is an `AudioWorklet`, whose module must be
+     * fetched from a URL — a `blob:` URL, which a page's Content-Security-Policy can refuse. The desktop app
+     * demonstrably *has* a CSP (a `connect-src 'none'` string is in its bundle, governing the shell pages), and
+     * I have not read the app page's policy. `ScriptProcessorNode` needs no URL, no module fetch and no policy
+     * grant, and it runs in the page, so the whole capture path is testable. It is deprecated and its latency is
+     * worse. That is a deliberate, disclosed trade: swap it for a worklet once the app page's `script-src` has
+     * actually been read, rather than shipping a capture path that a policy might silently kill.
+     */
+    Object.defineProperty(exports, "__esModule", { value: true });
+    exports.INJECTED_KEY = exports.GLOBAL_KEY = exports.MAX_BACKLOG_SECONDS = exports.CAPTURE_BUFFER = exports.SAMPLE_RATE = exports.DEFAULT_PATH = exports.inject = exports.name = void 0;
+    exports.socketUrl = socketUrl;
+    exports.pcm16FromFloat32 = pcm16FromFloat32;
+    exports.float32FromPcm16 = float32FromPcm16;
+    exports.defaultDeps = defaultDeps;
+    exports.createAudioClient = createAudioClient;
+    exports.apply = apply;
+    /** Browser-side plugin name. */
+    exports.name = 'realtime-audio-client';
+    /**
+     * No client services. The socket, the device and the audio graph are all this plugin's own, so it injects
+     * nothing — and a client face that needs nothing cannot be broken by another plugin's absence.
+     */
+    exports.inject = [];
+    /** Where the host route lives. Duplicated from the host package because a client face cannot import it. */
+    exports.DEFAULT_PATH = '/dsh-realtime/audio';
+    /** The session's declared rate. The graph is opened at this rate or the client refuses to run. */
+    exports.SAMPLE_RATE = 24_000;
+    /** Capture chunk in samples. 1024 at 24 kHz is ~43 ms: the 2^10 floor the API allows, so latency is lowest. */
+    exports.CAPTURE_BUFFER = 1024;
+    /** Backlog past which incoming speech is dropped rather than queued — the same rule the host carries. */
+    exports.MAX_BACKLOG_SECONDS = 0.5;
+    /** Global the client publishes itself on. There is no UI surface yet; this is how it is started. */
+    exports.GLOBAL_KEY = '__dshRealtimeAudio';
+    /** Optional host-injected settings. A future index-injection row may set the path; the default matches. */
+    exports.INJECTED_KEY = '__DSH_REALTIME_AUDIO__';
+    // ---- pure helpers --------------------------------------------------------------------------------------
+    /**
+     * The socket URL for the host route.
+     *
+     * @param location - the page's location, or undefined when there is none.
+     * @param path - the route pathname.
+     * @returns the URL, or undefined when this page cannot host a socket at all.
+     */
+    function socketUrl(location, path) {
+        if (location === undefined || location.host === '')
+            return undefined;
+        // An https page may not open a ws:// socket — the browser blocks it as mixed content — so the scheme has
+        // to follow the page rather than being decided here.
+        const scheme = location.protocol === 'https:' ? 'wss' : 'ws';
+        return `${scheme}://${location.host}${path}`;
+    }
+    /**
+     * Convert one captured float block to the wire's PCM16.
+     *
+     * Clamped, not wrapped: a sample above 1.0 is a loud peak, and letting it wrap turns a loud voice into
+     * noise. Asymmetric on purpose — 32767 is the largest Int16 and -32768 the smallest, so scaling both by the
+     * same factor would clip every negative peak one step early.
+     *
+     * @param input - samples in the range [-1, 1].
+     * @returns the same samples as little-endian PCM16 bytes.
+     */
+    function pcm16FromFloat32(input) {
+        const out = new Int16Array(input.length);
+        // `entries()` rather than an index: under `noUncheckedIndexedAccess` an indexed read is
+        // `number | undefined`, and guarding that would add a branch that can never be taken at runtime.
+        for (const [index, raw] of input.entries()) {
+            const sample = Math.max(-1, Math.min(1, raw));
+            out[index] = Math.round(sample < 0 ? sample * 32768 : sample * 32767);
+        }
+        return out;
+    }
+    /**
+     * Read one PCM16 frame as playback samples.
+     *
+     * @param bytes - the frame.
+     * @returns samples in [-1, 1).
+     */
+    function float32FromPcm16(bytes) {
+        const samples = new Int16Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 2));
+        const out = new Float32Array(samples.length);
+        for (const [index, sample] of samples.entries())
+            out[index] = sample / 32768;
+        return out;
+    }
+    /**
+     * Read the real browser APIs, and any settings the host injected.
+     *
+     * @param scope - the global scope, injectable so both the present and absent cases are testable without
+     * mutating the process's own globals.
+     * @returns the production dependencies. Every one may be absent, which is why `start` reports rather than
+     * throws: a page without a microphone API is a fact about the page, not a defect in the caller.
+     */
+    function defaultDeps(scope = globalThis) {
+        const media = scope.navigator?.mediaDevices;
+        const Audio = scope.AudioContext;
+        const Socket = scope.WebSocket;
+        return {
+            location: scope.location,
+            getUserMedia: media === undefined ? undefined : (constraints) => media.getUserMedia(constraints),
+            createAudioContext: Audio === undefined ? undefined : (rate) => new Audio({ sampleRate: rate }),
+            createSocket: Socket === undefined ? undefined : (url) => new Socket(url),
+            injected: scope.__DSH_REALTIME_AUDIO__,
+        };
+    }
+    /**
+     * The client: one microphone in, one socket out, one speaker fed.
+     *
+     * @param deps - the outside world, injected so every path is testable.
+     * @returns the start/stop/state handle.
+     */
+    function createAudioClient(deps) {
+        let current = { kind: 'idle' };
+        let socket;
+        let context;
+        let stream;
+        let processor;
+        let silence;
+        let playsAt = 0;
+        const fail = (reason) => {
+            current = { kind: 'failed', reason };
+            return current;
+        };
+        /** Release everything held. Idempotent, because stop, an error and a close all reach it. */
+        const release = () => {
+            socket?.close();
+            socket = undefined;
+            processor?.disconnect();
+            processor = undefined;
+            silence?.disconnect();
+            silence = undefined;
+            for (const track of stream?.getTracks() ?? [])
+                track.stop();
+            stream = undefined;
+            const closing = context;
+            context = undefined;
+            if (closing !== undefined)
+                void closing.close().catch(() => undefined);
+            playsAt = 0;
+        };
+        /** Play one frame, dropping it rather than queueing when we are already behind. */
+        const play = (bytes) => {
+            if (context === undefined)
+                return;
+            const samples = float32FromPcm16(bytes);
+            if (samples.length === 0)
+                return;
+            const buffer = context.createBuffer(1, samples.length, exports.SAMPLE_RATE);
+            buffer.getChannelData(0).set(samples);
+            const now = context.currentTime;
+            const at = Math.max(now, playsAt);
+            // Unbuffered, like the host: audio that is already late is dropped rather than played behind, because a
+            // queue that grows converts a stutter into a permanent delay that never recovers.
+            if (at - now > exports.MAX_BACKLOG_SECONDS)
+                return;
+            const source = context.createBufferSource();
+            source.buffer = buffer;
+            source.connect(context.destination);
+            source.start(at);
+            playsAt = at + buffer.duration;
+        };
+        /** Build the capture graph and start shipping frames. */
+        const attachCapture = () => {
+            if (context === undefined || stream === undefined || socket === undefined) {
+                return fail('the audio graph went away before the socket opened');
+            }
+            const source = context.createMediaStreamSource(stream);
+            const node = context.createScriptProcessor(exports.CAPTURE_BUFFER, 1, 1);
+            node.onaudioprocess = (event) => {
+                const pcm16 = pcm16FromFloat32(event.inputBuffer.getChannelData(0));
+                socket?.send(new Uint8Array(pcm16.buffer));
+            };
+            // A ScriptProcessorNode only fires while connected to the destination — and connecting the microphone
+            // there is a feedback loop. A zero-gain node keeps it running with nothing audible, which is the whole
+            // reason this chain looks the way it does.
+            const mute = context.createGain();
+            mute.gain.value = 0;
+            source.connect(node);
+            node.connect(mute);
+            mute.connect(context.destination);
+            processor = node;
+            silence = mute;
+            current = { kind: 'live' };
+            return current;
+        };
+        const stop = () => {
+            release();
+            current = { kind: 'idle' };
+        };
+        const start = async () => {
+            if (current.kind === 'live')
+                return current;
+            const url = socketUrl(deps.location, deps.injected?.path ?? exports.DEFAULT_PATH);
+            if (url === undefined)
+                return fail('this page has no location to open a socket against');
+            if (deps.getUserMedia === undefined)
+                return fail('this page has no microphone API');
+            if (deps.createAudioContext === undefined)
+                return fail('this page has no audio API');
+            if (deps.createSocket === undefined)
+                return fail('this page has no WebSocket API');
+            try {
+                stream = await deps.getUserMedia({
+                    audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
+                });
+            }
+            catch (error) {
+                // A refused permission is the ordinary case here, not an error to hide: the reason is what tells the
+                // user whether to grant it or to look somewhere else.
+                return fail(error instanceof Error ? error.message : String(error));
+            }
+            context = deps.createAudioContext(exports.SAMPLE_RATE);
+            if (context.sampleRate !== exports.SAMPLE_RATE) {
+                // Refuse rather than mislabel. Streaming 48 kHz samples declared as 24 kHz arrives at half speed and
+                // sounds like a provider fault, which is the most expensive possible way to learn about a resampler.
+                const opened = context.sampleRate;
+                release();
+                return fail(`the audio graph opened at ${String(opened)} Hz, not ${String(exports.SAMPLE_RATE)}`);
+            }
+            const opened = deps.createSocket(url);
+            socket = opened;
+            opened.binaryType = 'arraybuffer';
+            opened.onmessage = (event) => {
+                if (event.data instanceof ArrayBuffer)
+                    play(new Uint8Array(event.data));
+            };
+            opened.onclose = () => {
+                release();
+                current = { kind: 'idle' };
+            };
+            return await new Promise((resolve) => {
+                opened.onopen = () => { resolve(attachCapture()); };
+                opened.onerror = () => { resolve(fail('the host refused the audio socket')); };
+            });
+        };
+        return { start, stop, state: () => current };
+    }
+    /**
+     * The client plugin. Publishes the client on a global and releases it with the plugin.
+     *
+     * There is no UI surface yet, so a global is how it is driven — a settings card is its own increment and a
+     * bigger one than this. `inject` is empty for the same reason: nothing here needs another plugin.
+     *
+     * @param ctx - the client context, used only to own the lifetime.
+     */
+    function apply(ctx) {
+        const client = createAudioClient(defaultDeps());
+        globalThis[exports.GLOBAL_KEY] = {
+            start: () => client.start(),
+            stop: () => { client.stop(); },
+            state: () => client.state(),
+        };
+        // A held microphone and a live audio graph must not outlive the plugin that opened them.
+        ctx.effect(() => () => { client.stop(); }, 'realtime-audio-client');
+    }
+    return exports
+  },
+})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-realtime-audio-ws",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "The host end of the client half's transport: a WebSocket upgrade route that bridges microphone audio in and agent speech out.",
   "type": "module",
   "main": "lib/index.js",
@@ -12,7 +12,7 @@
     },
     "./client": {
       "types": "./lib/client/index.d.ts",
-      "default": "./lib/client/index.js"
+      "default": "./lib/client/bundle.js"
     },
     "./src/*": "./src/*",
     "./package.json": "./package.json"
@@ -64,7 +64,7 @@
     "@types/ws": "^8.18.2"
   },
   "scripts": {
-    "build": "tsc -b && tsc -p tsconfig.client.json",
+    "build": "tsc -b && tsc -p tsconfig.client.json && node ../../scripts/client-bundle.mjs .",
     "typecheck": "tsc -b --dry"
   }
 }