@4players/odin-plugin-web 1.0.0-alpha.21 → 1.0.0-alpha.22

package/lib/worker/unshared/capture.js

@@ -0,0 +1,66 @@
+/// <reference lib="WebWorker" />
+import { createEncoderInformationBuffer, freeEncoderInformationBuffer, setEncoderVolume, } from '../buffers';
+import { VadPreload } from '../config';
+import { send } from '../messages';
+import { Packet } from '../packet';
+import { free, malloc } from '../memory';
+export class Capture {
+    constructor(sampleRate, wasm) {
+        this.sampleRate = sampleRate;
+        this.wasm = wasm;
+        this.sequenceNumber = (Math.random() * Math.pow(2, 32)) >> 0;
+        this.timestamp = (Math.random() * Math.pow(2, 32)) >> 0;
+        this.connections = [];
+        this.active = false;
+        this.encoder = new wasm.Encoder(sampleRate, 
+        /* fec */ 15, 
+        /* voip */ true, VadPreload);
+        this.information = createEncoderInformationBuffer(this.wasm);
+        this.inBuffer = malloc(this.wasm, 'f32', sampleRate * 0.02);
+        this.outBuffer = malloc(this.wasm, 'u8', 1270 /* educated guess */);
+        setEncoderVolume(this.information, 1);
+    }
+    setVad(config) {
+        var _a, _b, _c, _d, _e, _f, _g, _h;
+        this.encoder.set_volume_gate(config.volumeGate !== undefined, (_b = (_a = config.volumeGate) === null || _a === void 0 ? void 0 : _a.attackThreshold) !== null && _b !== void 0 ? _b : 0, (_d = (_c = config.volumeGate) === null || _c === void 0 ? void 0 : _c.releaseThreshold) !== null && _d !== void 0 ? _d : 0);
+        this.encoder.set_speech_detection(config.voiceActivity !== undefined, (_f = (_e = config.voiceActivity) === null || _e === void 0 ? void 0 : _e.attackThreshold) !== null && _f !== void 0 ? _f : 0, (_h = (_g = config.voiceActivity) === null || _g === void 0 ? void 0 : _g.releaseThreshold) !== null && _h !== void 0 ? _h : 0);
+    }
+    setMedias(connections) {
+        this.connections = connections.filter(({ mediaIds }) => mediaIds.length > 0);
+    }
+    setVolume(value) {
+        setEncoderVolume(this.information, value);
+    }
+    close() {
+        freeEncoderInformationBuffer(this.wasm, this.information);
+        free(this.wasm, this.inBuffer);
+        free(this.wasm, this.outBuffer);
+        this.encoder.delete();
+    }
+    tick(buffer) {
+        this.inBuffer.set(buffer, 0);
+        this.encoder.push(this.inBuffer.byteOffset, this.information.values.byteOffset);
+        while (true) {
+            const n = this.encoder.pop(this.connections.length === 0 ? 0 : this.outBuffer.byteOffset, this.outBuffer.length);
+            if (n === 0)
+                break;
+            if (n < 0) {
+                console.error(`failed encoding packet: ${n}`);
+                return;
+            }
+            this.timestamp = (this.timestamp + 1800) >> 0; /* 20ms 90kHz */
+            if (n === 1)
+                continue;
+            this.sequenceNumber = (this.sequenceNumber + 1) >> 0;
+            const isEndOfSpurt = this.active && !this.encoder.active;
+            this.active = this.encoder.active;
+            for (const connection of this.connections) {
+                const packet = new Packet(connection.mediaIds, this.sequenceNumber, this.timestamp, isEndOfSpurt, this.outBuffer.subarray(0, n));
+                send(globalThis, [
+                    'sendPacket',
+                    { connectionId: connection.id, packet: packet.toBytes() },
+                ]);
+            }
+        }
+    }
+}

package/lib/worker/unshared/index.d.ts

@@ -0,0 +1,3 @@
+import { CaptureSink, CaptureWorker, PlaybackWorker } from '../common';
+export declare function createCaptureWorker(context: AudioContext, sink: CaptureSink, refreshRate: number, workerPromise: Promise<Worker>, workletPromise: Promise<AudioWorkletNode>): Promise<CaptureWorker>;
+export declare function createPlaybackWorker(audioContext: AudioContext, refreshRate: number, workerPromise: Promise<Worker>, workletPromise: Promise<AudioWorkletNode>): Promise<PlaybackWorker>;

package/lib/worker/unshared/index.js

@@ -0,0 +1,170 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { Silence, mergeIds, } from '../common';
+import { MaxNumberOfDecoders } from '../config';
+import { send, waitFor } from '../messages';
+class CaptureWorkerImpl {
+    constructor(worklet, worker, sink) {
+        this.worklet = worklet;
+        this.worker = worker;
+        this.sink = sink;
+        this._Activity = Silence;
+        this._SpeechProbability = 0;
+        worker.addEventListener('message', this._OnMessages.bind(this));
+    }
+    get context() {
+        return this.worklet.context;
+    }
+    get activity() {
+        return this._Activity;
+    }
+    get speechProbability() {
+        return this._SpeechProbability;
+    }
+    setMedias(connections) {
+        send(this.worker, ['captureSetMedias', { connections }]);
+    }
+    setVad(config) {
+        send(this.worker, ['captureSetVad', config]);
+    }
+    setVolume(value) {
+        send(this.worker, ['captureSetVolume', { value }]);
+    }
+    close() {
+        send(this.worklet.port, ['close', undefined]);
+        send(this.worker, ['close', undefined]);
+    }
+    _OnMessages(event) {
+        const [name, values] = event.data;
+        switch (name) {
+            case 'sendPacket':
+                this.sink(values.connectionId, values.packet);
+                break;
+            case 'captureUpdate':
+                this._Activity = {
+                    isSilent: !values.active,
+                    rmsDBFS: values.rmsDBFs,
+                };
+                this._SpeechProbability = values.speechProbability;
+        }
+    }
+}
+export function createCaptureWorker(context, sink, refreshRate, workerPromise, workletPromise) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { port1, port2 } = new MessageChannel();
+        const [worker, worklet] = yield Promise.all([workerPromise, workletPromise]);
+        send(worker, [
+            'captureInit',
+            { sampleRate: context.sampleRate, refreshRate, workletPort: port1 },
+        ], [port1]);
+        yield waitFor(worker, 'captureReady');
+        send(worklet.port, ['init', { workerPort: port2 }], [port2]);
+        return new CaptureWorkerImpl(worklet, worker, sink);
+    });
+}
+class PlaybackWorkerImpl {
+    constructor(audioContext, worklet, worker) {
+        this.audioContext = audioContext;
+        this.worklet = worklet;
+        this.worker = worker;
+        this.availableIndexes = [];
+        this.medias = new Map();
+        this.activities = new Float32Array();
+        worker.addEventListener('message', this._OnMessages.bind(this));
+    }
+    receivePacket(connectionId, packet) {
+        send(this.worker, ['playbackReceivePacket', { connectionId, packet }], [packet.buffer]);
+    }
+    addMedias(connectionId, mediaIds) {
+        const added = [];
+        for (const mediaId of mediaIds) {
+            const index = this._GetFreeIndex();
+            if (index === undefined)
+                continue;
+            this.medias.set(mergeIds({ connectionId, mediaId }), { index });
+            added.push({ connectionId, mediaId, index });
+        }
+        send(this.worker, [
+            'playbackUpdateMedias',
+            { added, removed: [] },
+        ]);
+    }
+    removeMedias(connectionId, mediaIds) {
+        const removed = [];
+        for (const mediaId of mediaIds) {
+            const id = mergeIds({ connectionId, mediaId });
+            const media = this.medias.get(id);
+            if (media === undefined)
+                continue;
+            this.medias.delete(id);
+            this.availableIndexes.push(media.index);
+            removed.push({ connectionId, mediaId, index: media.index });
+        }
+        send(this.worker, [
+            'playbackUpdateMedias',
+            { added: [], removed },
+        ]);
+    }
+    getActivity(id) {
+        const entry = this.medias.get(mergeIds(id));
+        if (entry === undefined)
+            return undefined;
+        const value = this.activities.at(entry.index);
+        if (value === undefined)
+            return undefined;
+        return value === 0 ? Silence : { isSilent: false, rmsDBFS: value };
+    }
+    setGlobalVolume(value) {
+        send(this.worker, [
+            'playbackSetVolume',
+            { index: 'global', value },
+        ]);
+    }
+    setVolume(id, value) {
+        const entry = this.medias.get(mergeIds(id));
+        if (entry !== undefined) {
+            send(this.worker, [
+                'playbackSetVolume',
+                { index: entry.index, value },
+            ]);
+        }
+    }
+    close() {
+        send(this.worker, ['close', undefined]);
+        send(this.worklet.port, ['close', undefined]);
+    }
+    _GetFreeIndex() {
+        const result = this.availableIndexes.pop();
+        if (result !== undefined)
+            return result;
+        if (this.medias.size < MaxNumberOfDecoders)
+            return this.medias.size;
+        return undefined;
+    }
+    _OnMessages(event) {
+        const [name, values] = event.data;
+        switch (name) {
+            case 'playbackUpdate':
+                this.activities = values.activities;
+                break;
+        }
+    }
+}
+export function createPlaybackWorker(audioContext, refreshRate, workerPromise, workletPromise) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { port1, port2 } = new MessageChannel();
+        const [worker, worklet] = yield Promise.all([workerPromise, workletPromise]);
+        send(worker, ['playbackInit', { workletPort: port1, refreshRate }], [port1]);
+        yield waitFor(worker, 'playbackReady');
+        send(worklet.port, ['init', { workerPort: port2 }], [port2]);
+        worklet.connect(audioContext.destination);
+        return new PlaybackWorkerImpl(audioContext, worklet, worker);
+    });
+}

package/lib/worker/unshared/messages.d.ts

@@ -0,0 +1,81 @@
+import { VadConfig } from '../common';
+import { Inflate } from '../messages';
+export type CaptureWorkletMessages = {
+    init: {
+        workerPort: MessagePort;
+    };
+    close: void;
+};
+export type PlaybackWorkletMessages = {
+    init: {
+        workerPort: MessagePort;
+    };
+    samples: {
+        id: number;
+        buffer: Float32Array;
+    };
+    close: void;
+};
+export type WorkerMessages = {
+    captureInit: {
+        sampleRate: number;
+        refreshRate: number;
+        workletPort: MessagePort;
+    };
+    captureSetMedias: {
+        connections: Array<{
+            id: number;
+            mediaIds: number[];
+        }>;
+    };
+    captureSetVad: VadConfig;
+    captureSetVolume: {
+        value: number;
+    };
+    captureTick: {
+        buffer: Float32Array;
+    };
+    playbackInit: {
+        refreshRate: number;
+        workletPort: MessagePort;
+    };
+    playbackUpdateMedias: {
+        added: Array<{
+            index: number;
+            connectionId: number;
+            mediaId: number;
+        }>;
+        removed: Array<{
+            index: number;
+            connectionId: number;
+            mediaId: number;
+        }>;
+    };
+    playbackSetVolume: {
+        index: 'global' | number;
+        value: number;
+    };
+    playbackReceivePacket: {
+        connectionId: number;
+        packet: Uint8Array;
+    };
+    playbackTick: void;
+    close: void;
+};
+export type MainMessages = {
+    sendPacket: {
+        connectionId: number;
+        packet: Uint8Array;
+    };
+    captureReady: void;
+    playbackReady: void;
+    captureUpdate: {
+        active: boolean;
+        rmsDBFs: number;
+        speechProbability: number;
+    };
+    playbackUpdate: {
+        activities: Float32Array;
+    };
+};
+export declare function onMessage(event: MessageEvent<Inflate<WorkerMessages>>): void;

package/lib/worker/unshared/messages.js

@@ -0,0 +1,80 @@
+import LoadOdinWasm from '../../../lib/wasm';
+import { getDecoderRmsDBFS, getEncoderActive, getEncoderSpeechProbability, } from '../buffers';
+import { send } from '../messages';
+import { Capture } from './capture';
+import { Playback } from './playback';
+let playback;
+let playbackInterval;
+let capture;
+let captureInterval;
+export function onMessage(event) {
+    handle(event.data);
+}
+function handle(message) {
+    const [name, values] = message;
+    switch (name) {
+        case 'playbackInit':
+            playback = new Playback(LoadOdinWasm(), values.workletPort);
+            playbackInterval = setInterval(() => {
+                const activities = new Float32Array(playback.information.length * 2);
+                for (let i = 0; i <= playback.largestIndex; i++) {
+                    activities[i] = getDecoderRmsDBFS(playback.information, i);
+                }
+                send(globalThis, ['playbackUpdate', { activities }], [activities.buffer]);
+            }, values.refreshRate);
+            values.workletPort.onmessage = (event) => onMessage(event);
+            send(globalThis, ['playbackReady', undefined]);
+            break;
+        case 'playbackReceivePacket':
+            playback === null || playback === void 0 ? void 0 : playback.receivePacket(values.connectionId, values.packet);
+            break;
+        case 'playbackUpdateMedias':
+            for (const { index, connectionId, mediaId } of values.removed) {
+                playback === null || playback === void 0 ? void 0 : playback.removeMedia(index, connectionId, mediaId);
+            }
+            for (const { index, connectionId, mediaId } of values.added) {
+                playback === null || playback === void 0 ? void 0 : playback.insertMedia(index, connectionId, mediaId);
+            }
+            break;
+        case 'playbackSetVolume':
+            playback === null || playback === void 0 ? void 0 : playback.setVolume(values.index, values.value);
+            break;
+        case 'playbackTick':
+            playback === null || playback === void 0 ? void 0 : playback.tick();
+            break;
+        case 'captureInit':
+            capture = new Capture(values.sampleRate, LoadOdinWasm());
+            captureInterval = setInterval(() => {
+                send(globalThis, [
+                    'captureUpdate',
+                    {
+                        active: getEncoderActive(capture.information),
+                        rmsDBFs: getEncoderSpeechProbability(capture.information),
+                        speechProbability: getEncoderSpeechProbability(capture.information),
+                    },
+                ]);
+            }, values.refreshRate);
+            values.workletPort.onmessage = (event) => onMessage(event);
+            send(globalThis, ['captureReady', undefined]);
+            break;
+        case 'captureSetVad':
+            capture === null || capture === void 0 ? void 0 : capture.setVad(values);
+            break;
+        case 'captureSetMedias':
+            capture === null || capture === void 0 ? void 0 : capture.setMedias(values.connections);
+            break;
+        case 'captureSetVolume':
+            capture === null || capture === void 0 ? void 0 : capture.setVolume(values.value);
+            break;
+        case 'captureTick':
+            capture === null || capture === void 0 ? void 0 : capture.tick(values.buffer);
+            break;
+        case 'close':
+            playback === null || playback === void 0 ? void 0 : playback.close();
+            clearInterval(playbackInterval);
+            capture === null || capture === void 0 ? void 0 : capture.close();
+            clearInterval(captureInterval);
+            self.close();
+            break;
+    }
+}

package/lib/worker/unshared/playback.d.ts

@@ -0,0 +1,22 @@
+import type { OdinWasm } from '../../../lib/wasm';
+import { DecoderInformationBuffer } from '../buffers';
+export declare class Playback {
+    readonly wasm: typeof OdinWasm;
+    readonly workletPort: MessagePort;
+    readonly medias: Map<number, {
+        index: number;
+        decoder: OdinWasm.Decoder;
+    }>;
+    readonly decoders: Uint32Array;
+    readonly information: DecoderInformationBuffer;
+    readonly outBuffer: Float32Array;
+    largestIndex: number;
+    id: number;
+    constructor(wasm: typeof OdinWasm, workletPort: MessagePort);
+    close(): void;
+    receivePacket(connectionId: number, packetBytes: Uint8Array): void;
+    insertMedia(index: number, connectionId: number, mediaId: number): void;
+    removeMedia(index: number, connectionId: number, mediaId: number): void;
+    setVolume(index: number | 'global', value: number): void;
+    tick(): void;
+}

package/lib/worker/unshared/playback.js

@@ -0,0 +1,83 @@
+/// <reference lib="WebWorker" />
+import { createDecoderInformationBuffer, freeDecoderInformationBuffer, setDecoderRmsDBFS, setDecoderVolume, setGlobalDecoderVolume, } from '../buffers';
+import { mergeIds } from '../common';
+import { MaxNumberOfDecoders, PlaybackPreload } from '../config';
+import { malloc } from '../memory';
+import { send } from '../messages';
+import { Packet } from '../packet';
+export class Playback {
+    constructor(wasm, workletPort) {
+        this.wasm = wasm;
+        this.workletPort = workletPort;
+        this.medias = new Map();
+        this.largestIndex = -1;
+        this.id = PlaybackPreload;
+        this.decoders = malloc(this.wasm, 'u32', MaxNumberOfDecoders);
+        this.information = createDecoderInformationBuffer(this.wasm, MaxNumberOfDecoders);
+        this.outBuffer = malloc(this.wasm, 'f32', 48000 * 0.02 * 2);
+        setGlobalDecoderVolume(this.information, 1);
+    }
+    close() {
+        freeDecoderInformationBuffer(this.wasm, this.information);
+        for (const { decoder } of this.medias.values()) {
+            decoder.delete();
+        }
+    }
+    receivePacket(connectionId, packetBytes) {
+        const packet = Packet.parse(packetBytes);
+        if (packet.type === 'Failure') {
+            console.error(`failed decoding audio packet; ${packet.reason}`);
+            return;
+        }
+        for (const mediaId of packet.value.mediaIds) {
+            const media = this.medias.get(mergeIds({ connectionId, mediaId }));
+            if (media === undefined)
+                continue;
+            // the decoder keeps ownership of the payload
+            const payload = malloc(this.wasm, 'u8', packet.value.payload.length);
+            payload.set(packet.value.payload);
+            media.decoder.push(packet.value.sequenceNumber, packet.value.timeStamp, packet.value.isEndOfSpurt, payload.byteOffset, payload.length);
+        }
+    }
+    insertMedia(index, connectionId, mediaId) {
+        const id = mergeIds({ connectionId, mediaId });
+        const decoder = new this.wasm.Decoder();
+        this.medias.set(id, { index, decoder });
+        this.decoders[index] = decoder.$$.ptr;
+        this.largestIndex = Math.max(this.largestIndex, index);
+        setDecoderVolume(this.information, index, 1);
+    }
+    removeMedia(index, connectionId, mediaId) {
+        const id = mergeIds({ connectionId, mediaId });
+        const entry = this.medias.get(id);
+        if (entry === undefined || entry.index !== index)
+            return;
+        this.medias.delete(id);
+        setDecoderRmsDBFS(this.information, index, 0);
+        this.decoders[index] = 0;
+        entry.decoder.delete();
+        if (index === this.largestIndex) {
+            for (index = index - 1; index >= 0; index--) {
+                if (this.decoders[index] !== 0)
+                    break;
+            }
+            this.largestIndex = index;
+        }
+    }
+    setVolume(index, value) {
+        if (index === 'global') {
+            setGlobalDecoderVolume(this.information, value);
+        }
+        else {
+            setDecoderVolume(this.information, index, value);
+        }
+    }
+    tick() {
+        const id = this.id++;
+        const hasAudio = this.wasm.pop_many(this.decoders.byteOffset, this.information.values.byteOffset, this.largestIndex + 1, this.outBuffer.byteOffset);
+        if (hasAudio) {
+            const buffer = this.outBuffer.slice();
+            send(this.workletPort, ['samples', { id: this.id, buffer }], [buffer.buffer]);
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@4players/odin-plugin-web",
-  "version": "1.0.0-alpha.21",
+  "version": "1.0.0-alpha.22",
   "description": "A backend plugin providing the custom WebAssembly audio processing module for ODIN web clients",
   "homepage": "https://www.4players.io",
   "repository": {
@@ -12,36 +12,33 @@
   "types": "./lib/plugin/index.d.ts",
   "scripts": {
     "clean": "rimraf build lib",
-    "configure": "npm-run-all configure:*",
+    "configure": "npm-run-all -pl configure:*",
     "configure:shared": "emcmake cmake -B build/shared -DCMAKE_BUILD_TYPE=Release -DSHARED_MEMORY=1 -S .",
     "configure:unshared": "emcmake cmake -B build/unshared -DCMAKE_BUILD_TYPE=Release -S .",
-    "build": "npm-run-all -s configure build:genversion build:wasm build:bundle build:types",
-    "build:genversion": "genversion -se src/common/version.ts",
-    "build:wasm": "npm-run-all -pl build:wasm:*",
-    "build:wasm:shared": "emmake cmake --build build/shared -j 8",
-    "build:wasm:unshared": "emmake cmake --build build/unshared -j 8",
-    "build:bundle": "ts-node src/bundle.ts",
-    "build:types": "dts-bundle-generator --fail-on-class --no-banner -o lib/plugin/index.d.ts src/plugin/index.ts",
+    "build": "npm-run-all -s configure build:wasm build:plugin build:worker",
+    "build:wasm": "npm-run-all -pl build:wasm:* && node install-wasm.js",
+    "build:wasm:shared": "emmake cmake --build build/shared",
+    "build:wasm:unshared": "emmake cmake --build build/unshared",
+    "build:worker": "ts-node src/build.ts && tsc -b ./tsconfig.worker.json",
+    "build:plugin": "tsc --build --verbose --force",
     "format": "prettier 'src/**/*.ts' --write",
     "lint": "eslint . --ext .ts",
     "test": "npm --prefix test run start",
     "bundle": "npm run build && esbuild lib/plugin/index.js --bundle --outfile=dist/plugin.js"
   },
   "dependencies": {
-    "@4players/odin-common": "~2.8.0"
+  "@4players/odin-common": "~2.8.0"
   },
   "devDependencies": {
     "@types/node": "~20.14.0",
     "@typescript-eslint/eslint-plugin": "~7.16.0",
     "@typescript-eslint/parser": "~7.16.0",
-    "dts-bundle-generator": "^9.5.1",
     "esbuild": "~0.23.0",
     "eslint": "~8.57.0",
     "eslint-config-semistandard": "~17.0.0",
     "eslint-plugin-import": "~2.29.0",
     "eslint-plugin-node": "~11.1.0",
     "eslint-plugin-promise": "~6.1.0",
-    "genversion": "^3.2.0",
     "npm-run-all2": "~6.1.0",
     "prettier": "~3.3.0",
     "rimraf": "~5.0.0",