@4players/odin-plugin-web 1.0.0-alpha.2 → 1.0.0-alpha.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/common/msgpack.d.ts +2 -0
- package/lib/common/msgpack.js +117 -0
- package/{backend/lib → lib/plugin}/config.d.ts +5 -4
- package/lib/plugin/config.js +22 -0
- package/{backend/lib → lib/plugin}/connectionPool.d.ts +3 -3
- package/{backend/lib → lib/plugin}/connectionPool.js +32 -13
- package/lib/plugin/deviceManager.d.ts +12 -0
- package/lib/plugin/deviceManager.js +120 -0
- package/lib/plugin/h3/mainConnection.d.ts +31 -0
- package/lib/plugin/h3/mainConnection.js +195 -0
- package/{backend/lib → lib/plugin}/index.d.ts +30 -13
- package/lib/plugin/index.js +404 -0
- package/lib/plugin/mainConnection.d.ts +30 -0
- package/lib/plugin/mainConnection.js +21 -0
- package/{backend/lib/audioController.d.ts → lib/plugin/mediaController.d.ts} +12 -48
- package/lib/plugin/mediaController.js +215 -0
- package/lib/plugin/medias/audio-capture.d.ts +30 -0
- package/lib/plugin/medias/audio-capture.js +224 -0
- package/lib/plugin/medias/audio-playback.d.ts +13 -0
- package/lib/plugin/medias/audio-playback.js +40 -0
- package/lib/plugin/medias/video-capture.d.ts +15 -0
- package/lib/plugin/medias/video-capture.js +69 -0
- package/lib/plugin/medias/video-playback.d.ts +13 -0
- package/lib/plugin/medias/video-playback.js +20 -0
- package/{backend/lib → lib/plugin}/roomConnection.d.ts +9 -6
- package/{backend/lib → lib/plugin}/roomConnection.js +74 -53
- package/lib/plugin/rpcConnection.d.ts +57 -0
- package/{backend/lib → lib/plugin}/rpcConnection.js +129 -83
- package/lib/plugin/token.d.ts +13 -0
- package/{backend/lib → lib/plugin}/token.js +8 -9
- package/lib/plugin/webrtc/mainConnection.d.ts +53 -0
- package/lib/plugin/webrtc/mainConnection.js +455 -0
- package/lib/plugin/webrtc/websocket.d.ts +3 -0
- package/lib/plugin/webrtc/websocket.js +56 -0
- package/lib/worker/buffers.d.ts +42 -0
- package/lib/worker/buffers.js +79 -0
- package/{worker/lib/index.d.ts → lib/worker/common.d.ts} +7 -8
- package/lib/worker/common.js +7 -0
- package/lib/worker/config.d.ts +8 -0
- package/lib/worker/config.js +11 -0
- package/lib/worker/index.d.ts +3 -0
- package/lib/worker/index.js +1 -0
- package/lib/worker/memory.d.ts +9 -0
- package/lib/worker/memory.js +29 -0
- package/lib/worker/messages.d.ts +14 -0
- package/lib/worker/messages.js +15 -0
- package/lib/worker/packet.d.ts +11 -0
- package/lib/worker/packet.js +59 -0
- package/lib/worker/shared/capture.d.ts +27 -0
- package/lib/worker/shared/capture.js +63 -0
- package/lib/worker/shared/index.d.ts +3 -0
- package/lib/worker/shared/index.js +147 -0
- package/lib/worker/shared/messages.d.ts +67 -0
- package/lib/worker/shared/messages.js +64 -0
- package/lib/worker/shared/playback.d.ts +20 -0
- package/lib/worker/shared/playback.js +78 -0
- package/lib/worker/unshared/capture.d.ts +27 -0
- package/lib/worker/unshared/capture.js +66 -0
- package/lib/worker/unshared/index.d.ts +3 -0
- package/lib/worker/unshared/index.js +170 -0
- package/{worker/lib → lib/worker/unshared}/messages.d.ts +3 -15
- package/lib/worker/unshared/messages.js +80 -0
- package/lib/worker/unshared/playback.d.ts +22 -0
- package/lib/worker/unshared/playback.js +83 -0
- package/package.json +29 -26
- package/backend/lib/audioController.d.ts.map +0 -1
- package/backend/lib/audioController.js +0 -442
- package/backend/lib/audioController.js.map +0 -1
- package/backend/lib/config.d.ts.map +0 -1
- package/backend/lib/config.js +0 -11
- package/backend/lib/config.js.map +0 -1
- package/backend/lib/connectionPool.d.ts.map +0 -1
- package/backend/lib/connectionPool.js.map +0 -1
- package/backend/lib/index.d.ts.map +0 -1
- package/backend/lib/index.js +0 -293
- package/backend/lib/index.js.map +0 -1
- package/backend/lib/mainConnection.d.ts +0 -40
- package/backend/lib/mainConnection.d.ts.map +0 -1
- package/backend/lib/mainConnection.js +0 -294
- package/backend/lib/mainConnection.js.map +0 -1
- package/backend/lib/roomConnection.d.ts.map +0 -1
- package/backend/lib/roomConnection.js.map +0 -1
- package/backend/lib/rpcConnection.d.ts +0 -49
- package/backend/lib/rpcConnection.d.ts.map +0 -1
- package/backend/lib/rpcConnection.js.map +0 -1
- package/backend/lib/token.d.ts +0 -16
- package/backend/lib/token.d.ts.map +0 -1
- package/backend/lib/token.js.map +0 -1
- package/backend/package.json +0 -13
- package/wasm/lib/index.js +0 -19
- package/wasm/package.json +0 -14
- package/worker/lib/common.d.ts +0 -27
- package/worker/lib/index.js +0 -2
- package/worker/package.json +0 -13
- /package/{wasm/lib → lib/wasm}/index.d.ts +0 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/// <reference lib="WebWorker" />
|
|
2
|
+
import { createEncoderInformationBuffer, freeEncoderInformationBuffer, setEncoderVolume, } from '../buffers';
|
|
3
|
+
import { VadPreload } from '../config';
|
|
4
|
+
import { send } from '../messages';
|
|
5
|
+
import { Packet } from '../packet';
|
|
6
|
+
import { free, malloc } from '../memory';
|
|
7
|
+
export class Capture {
|
|
8
|
+
constructor(sampleRate, wasm) {
|
|
9
|
+
this.sampleRate = sampleRate;
|
|
10
|
+
this.wasm = wasm;
|
|
11
|
+
this.sequenceNumber = (Math.random() * Math.pow(2, 32)) >> 0;
|
|
12
|
+
this.timestamp = (Math.random() * Math.pow(2, 32)) >> 0;
|
|
13
|
+
this.connections = [];
|
|
14
|
+
this.active = false;
|
|
15
|
+
this.encoder = new wasm.Encoder(sampleRate,
|
|
16
|
+
/* fec */ 15,
|
|
17
|
+
/* voip */ true, VadPreload);
|
|
18
|
+
this.information = createEncoderInformationBuffer(this.wasm);
|
|
19
|
+
this.inBuffer = malloc(this.wasm, 'f32', sampleRate * 0.02);
|
|
20
|
+
this.outBuffer = malloc(this.wasm, 'u8', 1270 /* educated guess */);
|
|
21
|
+
setEncoderVolume(this.information, 1);
|
|
22
|
+
}
|
|
23
|
+
setVad(config) {
|
|
24
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
25
|
+
this.encoder.set_volume_gate(config.volumeGate !== undefined, (_b = (_a = config.volumeGate) === null || _a === void 0 ? void 0 : _a.attackThreshold) !== null && _b !== void 0 ? _b : 0, (_d = (_c = config.volumeGate) === null || _c === void 0 ? void 0 : _c.releaseThreshold) !== null && _d !== void 0 ? _d : 0);
|
|
26
|
+
this.encoder.set_speech_detection(config.voiceActivity !== undefined, (_f = (_e = config.voiceActivity) === null || _e === void 0 ? void 0 : _e.attackThreshold) !== null && _f !== void 0 ? _f : 0, (_h = (_g = config.voiceActivity) === null || _g === void 0 ? void 0 : _g.releaseThreshold) !== null && _h !== void 0 ? _h : 0);
|
|
27
|
+
}
|
|
28
|
+
setMedias(connections) {
|
|
29
|
+
this.connections = connections.filter(({ mediaIds }) => mediaIds.length > 0);
|
|
30
|
+
}
|
|
31
|
+
setVolume(value) {
|
|
32
|
+
setEncoderVolume(this.information, value);
|
|
33
|
+
}
|
|
34
|
+
close() {
|
|
35
|
+
freeEncoderInformationBuffer(this.wasm, this.information);
|
|
36
|
+
free(this.wasm, this.inBuffer);
|
|
37
|
+
free(this.wasm, this.outBuffer);
|
|
38
|
+
this.encoder.delete();
|
|
39
|
+
}
|
|
40
|
+
tick(buffer) {
|
|
41
|
+
this.inBuffer.set(buffer, 0);
|
|
42
|
+
this.encoder.push(this.inBuffer.byteOffset, this.information.values.byteOffset);
|
|
43
|
+
while (true) {
|
|
44
|
+
const n = this.encoder.pop(this.connections.length === 0 ? 0 : this.outBuffer.byteOffset, this.outBuffer.length);
|
|
45
|
+
if (n === 0)
|
|
46
|
+
break;
|
|
47
|
+
if (n < 0) {
|
|
48
|
+
console.error(`failed encoding packet: ${n}`);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
this.timestamp = (this.timestamp + 1800) >> 0; /* 20ms 90kHz */
|
|
52
|
+
if (n === 1)
|
|
53
|
+
continue;
|
|
54
|
+
this.sequenceNumber = (this.sequenceNumber + 1) >> 0;
|
|
55
|
+
const isEndOfSpurt = this.active && !this.encoder.active;
|
|
56
|
+
this.active = this.encoder.active;
|
|
57
|
+
for (const connection of this.connections) {
|
|
58
|
+
const packet = new Packet(connection.mediaIds, this.sequenceNumber, this.timestamp, isEndOfSpurt, this.outBuffer.subarray(0, n));
|
|
59
|
+
send(globalThis, [
|
|
60
|
+
'sendPacket',
|
|
61
|
+
{ connectionId: connection.id, packet: packet.toBytes() },
|
|
62
|
+
]);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { CaptureSink, CaptureWorker, PlaybackWorker } from '../common';
|
|
2
|
+
export declare function createCaptureWorker(context: AudioContext, sink: CaptureSink, refreshRate: number, workerPromise: Promise<Worker>, workletPromise: Promise<AudioWorkletNode>): Promise<CaptureWorker>;
|
|
3
|
+
export declare function createPlaybackWorker(audioContext: AudioContext, refreshRate: number, workerPromise: Promise<Worker>, workletPromise: Promise<AudioWorkletNode>): Promise<PlaybackWorker>;
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { Silence, mergeIds, } from '../common';
|
|
11
|
+
import { MaxNumberOfDecoders } from '../config';
|
|
12
|
+
import { send, waitFor } from '../messages';
|
|
13
|
+
class CaptureWorkerImpl {
|
|
14
|
+
constructor(worklet, worker, sink) {
|
|
15
|
+
this.worklet = worklet;
|
|
16
|
+
this.worker = worker;
|
|
17
|
+
this.sink = sink;
|
|
18
|
+
this._Activity = Silence;
|
|
19
|
+
this._SpeechProbability = 0;
|
|
20
|
+
worker.addEventListener('message', this._OnMessages.bind(this));
|
|
21
|
+
}
|
|
22
|
+
get context() {
|
|
23
|
+
return this.worklet.context;
|
|
24
|
+
}
|
|
25
|
+
get activity() {
|
|
26
|
+
return this._Activity;
|
|
27
|
+
}
|
|
28
|
+
get speechProbability() {
|
|
29
|
+
return this._SpeechProbability;
|
|
30
|
+
}
|
|
31
|
+
setMedias(connections) {
|
|
32
|
+
send(this.worker, ['captureSetMedias', { connections }]);
|
|
33
|
+
}
|
|
34
|
+
setVad(config) {
|
|
35
|
+
send(this.worker, ['captureSetVad', config]);
|
|
36
|
+
}
|
|
37
|
+
setVolume(value) {
|
|
38
|
+
send(this.worker, ['captureSetVolume', { value }]);
|
|
39
|
+
}
|
|
40
|
+
close() {
|
|
41
|
+
send(this.worklet.port, ['close', undefined]);
|
|
42
|
+
send(this.worker, ['close', undefined]);
|
|
43
|
+
}
|
|
44
|
+
_OnMessages(event) {
|
|
45
|
+
const [name, values] = event.data;
|
|
46
|
+
switch (name) {
|
|
47
|
+
case 'sendPacket':
|
|
48
|
+
this.sink(values.connectionId, values.packet);
|
|
49
|
+
break;
|
|
50
|
+
case 'captureUpdate':
|
|
51
|
+
this._Activity = {
|
|
52
|
+
isSilent: !values.active,
|
|
53
|
+
rmsDBFS: values.rmsDBFs,
|
|
54
|
+
};
|
|
55
|
+
this._SpeechProbability = values.speechProbability;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
export function createCaptureWorker(context, sink, refreshRate, workerPromise, workletPromise) {
|
|
60
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
61
|
+
const { port1, port2 } = new MessageChannel();
|
|
62
|
+
const [worker, worklet] = yield Promise.all([workerPromise, workletPromise]);
|
|
63
|
+
send(worker, [
|
|
64
|
+
'captureInit',
|
|
65
|
+
{ sampleRate: context.sampleRate, refreshRate, workletPort: port1 },
|
|
66
|
+
], [port1]);
|
|
67
|
+
yield waitFor(worker, 'captureReady');
|
|
68
|
+
send(worklet.port, ['init', { workerPort: port2 }], [port2]);
|
|
69
|
+
return new CaptureWorkerImpl(worklet, worker, sink);
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
class PlaybackWorkerImpl {
|
|
73
|
+
constructor(audioContext, worklet, worker) {
|
|
74
|
+
this.audioContext = audioContext;
|
|
75
|
+
this.worklet = worklet;
|
|
76
|
+
this.worker = worker;
|
|
77
|
+
this.availableIndexes = [];
|
|
78
|
+
this.medias = new Map();
|
|
79
|
+
this.activities = new Float32Array();
|
|
80
|
+
worker.addEventListener('message', this._OnMessages.bind(this));
|
|
81
|
+
}
|
|
82
|
+
receivePacket(connectionId, packet) {
|
|
83
|
+
send(this.worker, ['playbackReceivePacket', { connectionId, packet }], [packet.buffer]);
|
|
84
|
+
}
|
|
85
|
+
addMedias(connectionId, mediaIds) {
|
|
86
|
+
const added = [];
|
|
87
|
+
for (const mediaId of mediaIds) {
|
|
88
|
+
const index = this._GetFreeIndex();
|
|
89
|
+
if (index === undefined)
|
|
90
|
+
continue;
|
|
91
|
+
this.medias.set(mergeIds({ connectionId, mediaId }), { index });
|
|
92
|
+
added.push({ connectionId, mediaId, index });
|
|
93
|
+
}
|
|
94
|
+
send(this.worker, [
|
|
95
|
+
'playbackUpdateMedias',
|
|
96
|
+
{ added, removed: [] },
|
|
97
|
+
]);
|
|
98
|
+
}
|
|
99
|
+
removeMedias(connectionId, mediaIds) {
|
|
100
|
+
const removed = [];
|
|
101
|
+
for (const mediaId of mediaIds) {
|
|
102
|
+
const id = mergeIds({ connectionId, mediaId });
|
|
103
|
+
const media = this.medias.get(id);
|
|
104
|
+
if (media === undefined)
|
|
105
|
+
continue;
|
|
106
|
+
this.medias.delete(id);
|
|
107
|
+
this.availableIndexes.push(media.index);
|
|
108
|
+
removed.push({ connectionId, mediaId, index: media.index });
|
|
109
|
+
}
|
|
110
|
+
send(this.worker, [
|
|
111
|
+
'playbackUpdateMedias',
|
|
112
|
+
{ added: [], removed },
|
|
113
|
+
]);
|
|
114
|
+
}
|
|
115
|
+
getActivity(id) {
|
|
116
|
+
const entry = this.medias.get(mergeIds(id));
|
|
117
|
+
if (entry === undefined)
|
|
118
|
+
return undefined;
|
|
119
|
+
const value = this.activities.at(entry.index);
|
|
120
|
+
if (value === undefined)
|
|
121
|
+
return undefined;
|
|
122
|
+
return value === 0 ? Silence : { isSilent: false, rmsDBFS: value };
|
|
123
|
+
}
|
|
124
|
+
setGlobalVolume(value) {
|
|
125
|
+
send(this.worker, [
|
|
126
|
+
'playbackSetVolume',
|
|
127
|
+
{ index: 'global', value },
|
|
128
|
+
]);
|
|
129
|
+
}
|
|
130
|
+
setVolume(id, value) {
|
|
131
|
+
const entry = this.medias.get(mergeIds(id));
|
|
132
|
+
if (entry !== undefined) {
|
|
133
|
+
send(this.worker, [
|
|
134
|
+
'playbackSetVolume',
|
|
135
|
+
{ index: entry.index, value },
|
|
136
|
+
]);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
close() {
|
|
140
|
+
send(this.worker, ['close', undefined]);
|
|
141
|
+
send(this.worklet.port, ['close', undefined]);
|
|
142
|
+
}
|
|
143
|
+
_GetFreeIndex() {
|
|
144
|
+
const result = this.availableIndexes.pop();
|
|
145
|
+
if (result !== undefined)
|
|
146
|
+
return result;
|
|
147
|
+
if (this.medias.size < MaxNumberOfDecoders)
|
|
148
|
+
return this.medias.size;
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
_OnMessages(event) {
|
|
152
|
+
const [name, values] = event.data;
|
|
153
|
+
switch (name) {
|
|
154
|
+
case 'playbackUpdate':
|
|
155
|
+
this.activities = values.activities;
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
export function createPlaybackWorker(audioContext, refreshRate, workerPromise, workletPromise) {
|
|
161
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
162
|
+
const { port1, port2 } = new MessageChannel();
|
|
163
|
+
const [worker, worklet] = yield Promise.all([workerPromise, workletPromise]);
|
|
164
|
+
send(worker, ['playbackInit', { workletPort: port1, refreshRate }], [port1]);
|
|
165
|
+
yield waitFor(worker, 'playbackReady');
|
|
166
|
+
send(worklet.port, ['init', { workerPort: port2 }], [port2]);
|
|
167
|
+
worklet.connect(audioContext.destination);
|
|
168
|
+
return new PlaybackWorkerImpl(audioContext, worklet, worker);
|
|
169
|
+
});
|
|
170
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
1
|
+
import { VadConfig } from '../common';
|
|
2
|
+
import { Inflate } from '../messages';
|
|
3
3
|
export type CaptureWorkletMessages = {
|
|
4
4
|
init: {
|
|
5
5
|
workerPort: MessagePort;
|
|
@@ -78,16 +78,4 @@ export type MainMessages = {
|
|
|
78
78
|
activities: Float32Array;
|
|
79
79
|
};
|
|
80
80
|
};
|
|
81
|
-
export
|
|
82
|
-
export type Message<Name, Values> = [Name, Values];
|
|
83
|
-
export type Inflate<T extends Messages> = {
|
|
84
|
-
[Name in keyof T]: Message<Name, T[Name]>;
|
|
85
|
-
}[keyof T];
|
|
86
|
-
export type MessageResult<T extends Messages> = Result<Inflate<T> | 'closed'>;
|
|
87
|
-
export declare function send<T extends Messages>(port: {
|
|
88
|
-
postMessage(message: unknown, transfer?: Transferable[]): void;
|
|
89
|
-
}, message: Inflate<T>, transfer?: Transferable[]): void;
|
|
90
|
-
export declare function waitFor<T extends Messages, Name extends Extract<keyof T, string>>(port: {
|
|
91
|
-
addEventListener(type: 'message', listener: (event: MessageEvent) => void): void;
|
|
92
|
-
removeEventListener(type: 'message', listener: (event: MessageEvent) => void): void;
|
|
93
|
-
}, name: Name): Promise<T[Name]>;
|
|
81
|
+
export declare function onMessage(event: MessageEvent<Inflate<WorkerMessages>>): void;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import LoadOdinWasm from '../../../lib/wasm';
|
|
2
|
+
import { getDecoderRmsDBFS, getEncoderActive, getEncoderSpeechProbability, } from '../buffers';
|
|
3
|
+
import { send } from '../messages';
|
|
4
|
+
import { Capture } from './capture';
|
|
5
|
+
import { Playback } from './playback';
|
|
6
|
+
let playback;
|
|
7
|
+
let playbackInterval;
|
|
8
|
+
let capture;
|
|
9
|
+
let captureInterval;
|
|
10
|
+
export function onMessage(event) {
|
|
11
|
+
handle(event.data);
|
|
12
|
+
}
|
|
13
|
+
function handle(message) {
|
|
14
|
+
const [name, values] = message;
|
|
15
|
+
switch (name) {
|
|
16
|
+
case 'playbackInit':
|
|
17
|
+
playback = new Playback(LoadOdinWasm(), values.workletPort);
|
|
18
|
+
playbackInterval = setInterval(() => {
|
|
19
|
+
const activities = new Float32Array(playback.information.length * 2);
|
|
20
|
+
for (let i = 0; i <= playback.largestIndex; i++) {
|
|
21
|
+
activities[i] = getDecoderRmsDBFS(playback.information, i);
|
|
22
|
+
}
|
|
23
|
+
send(globalThis, ['playbackUpdate', { activities }], [activities.buffer]);
|
|
24
|
+
}, values.refreshRate);
|
|
25
|
+
values.workletPort.onmessage = (event) => onMessage(event);
|
|
26
|
+
send(globalThis, ['playbackReady', undefined]);
|
|
27
|
+
break;
|
|
28
|
+
case 'playbackReceivePacket':
|
|
29
|
+
playback === null || playback === void 0 ? void 0 : playback.receivePacket(values.connectionId, values.packet);
|
|
30
|
+
break;
|
|
31
|
+
case 'playbackUpdateMedias':
|
|
32
|
+
for (const { index, connectionId, mediaId } of values.removed) {
|
|
33
|
+
playback === null || playback === void 0 ? void 0 : playback.removeMedia(index, connectionId, mediaId);
|
|
34
|
+
}
|
|
35
|
+
for (const { index, connectionId, mediaId } of values.added) {
|
|
36
|
+
playback === null || playback === void 0 ? void 0 : playback.insertMedia(index, connectionId, mediaId);
|
|
37
|
+
}
|
|
38
|
+
break;
|
|
39
|
+
case 'playbackSetVolume':
|
|
40
|
+
playback === null || playback === void 0 ? void 0 : playback.setVolume(values.index, values.value);
|
|
41
|
+
break;
|
|
42
|
+
case 'playbackTick':
|
|
43
|
+
playback === null || playback === void 0 ? void 0 : playback.tick();
|
|
44
|
+
break;
|
|
45
|
+
case 'captureInit':
|
|
46
|
+
capture = new Capture(values.sampleRate, LoadOdinWasm());
|
|
47
|
+
captureInterval = setInterval(() => {
|
|
48
|
+
send(globalThis, [
|
|
49
|
+
'captureUpdate',
|
|
50
|
+
{
|
|
51
|
+
active: getEncoderActive(capture.information),
|
|
52
|
+
rmsDBFs: getEncoderSpeechProbability(capture.information),
|
|
53
|
+
speechProbability: getEncoderSpeechProbability(capture.information),
|
|
54
|
+
},
|
|
55
|
+
]);
|
|
56
|
+
}, values.refreshRate);
|
|
57
|
+
values.workletPort.onmessage = (event) => onMessage(event);
|
|
58
|
+
send(globalThis, ['captureReady', undefined]);
|
|
59
|
+
break;
|
|
60
|
+
case 'captureSetVad':
|
|
61
|
+
capture === null || capture === void 0 ? void 0 : capture.setVad(values);
|
|
62
|
+
break;
|
|
63
|
+
case 'captureSetMedias':
|
|
64
|
+
capture === null || capture === void 0 ? void 0 : capture.setMedias(values.connections);
|
|
65
|
+
break;
|
|
66
|
+
case 'captureSetVolume':
|
|
67
|
+
capture === null || capture === void 0 ? void 0 : capture.setVolume(values.value);
|
|
68
|
+
break;
|
|
69
|
+
case 'captureTick':
|
|
70
|
+
capture === null || capture === void 0 ? void 0 : capture.tick(values.buffer);
|
|
71
|
+
break;
|
|
72
|
+
case 'close':
|
|
73
|
+
playback === null || playback === void 0 ? void 0 : playback.close();
|
|
74
|
+
clearInterval(playbackInterval);
|
|
75
|
+
capture === null || capture === void 0 ? void 0 : capture.close();
|
|
76
|
+
clearInterval(captureInterval);
|
|
77
|
+
self.close();
|
|
78
|
+
break;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { OdinWasm } from '../../../lib/wasm';
|
|
2
|
+
import { DecoderInformationBuffer } from '../buffers';
|
|
3
|
+
export declare class Playback {
|
|
4
|
+
readonly wasm: typeof OdinWasm;
|
|
5
|
+
readonly workletPort: MessagePort;
|
|
6
|
+
readonly medias: Map<number, {
|
|
7
|
+
index: number;
|
|
8
|
+
decoder: OdinWasm.Decoder;
|
|
9
|
+
}>;
|
|
10
|
+
readonly decoders: Uint32Array;
|
|
11
|
+
readonly information: DecoderInformationBuffer;
|
|
12
|
+
readonly outBuffer: Float32Array;
|
|
13
|
+
largestIndex: number;
|
|
14
|
+
id: number;
|
|
15
|
+
constructor(wasm: typeof OdinWasm, workletPort: MessagePort);
|
|
16
|
+
close(): void;
|
|
17
|
+
receivePacket(connectionId: number, packetBytes: Uint8Array): void;
|
|
18
|
+
insertMedia(index: number, connectionId: number, mediaId: number): void;
|
|
19
|
+
removeMedia(index: number, connectionId: number, mediaId: number): void;
|
|
20
|
+
setVolume(index: number | 'global', value: number): void;
|
|
21
|
+
tick(): void;
|
|
22
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/// <reference lib="WebWorker" />
|
|
2
|
+
import { createDecoderInformationBuffer, freeDecoderInformationBuffer, setDecoderRmsDBFS, setDecoderVolume, setGlobalDecoderVolume, } from '../buffers';
|
|
3
|
+
import { mergeIds } from '../common';
|
|
4
|
+
import { MaxNumberOfDecoders, PlaybackPreload } from '../config';
|
|
5
|
+
import { malloc } from '../memory';
|
|
6
|
+
import { send } from '../messages';
|
|
7
|
+
import { Packet } from '../packet';
|
|
8
|
+
export class Playback {
|
|
9
|
+
constructor(wasm, workletPort) {
|
|
10
|
+
this.wasm = wasm;
|
|
11
|
+
this.workletPort = workletPort;
|
|
12
|
+
this.medias = new Map();
|
|
13
|
+
this.largestIndex = -1;
|
|
14
|
+
this.id = PlaybackPreload;
|
|
15
|
+
this.decoders = malloc(this.wasm, 'u32', MaxNumberOfDecoders);
|
|
16
|
+
this.information = createDecoderInformationBuffer(this.wasm, MaxNumberOfDecoders);
|
|
17
|
+
this.outBuffer = malloc(this.wasm, 'f32', 48000 * 0.02 * 2);
|
|
18
|
+
setGlobalDecoderVolume(this.information, 1);
|
|
19
|
+
}
|
|
20
|
+
close() {
|
|
21
|
+
freeDecoderInformationBuffer(this.wasm, this.information);
|
|
22
|
+
for (const { decoder } of this.medias.values()) {
|
|
23
|
+
decoder.delete();
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
receivePacket(connectionId, packetBytes) {
|
|
27
|
+
const packet = Packet.parse(packetBytes);
|
|
28
|
+
if (packet.type === 'Failure') {
|
|
29
|
+
console.error(`failed decoding audio packet; ${packet.reason}`);
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
for (const mediaId of packet.value.mediaIds) {
|
|
33
|
+
const media = this.medias.get(mergeIds({ connectionId, mediaId }));
|
|
34
|
+
if (media === undefined)
|
|
35
|
+
continue;
|
|
36
|
+
// the decoder keeps ownership of the payload
|
|
37
|
+
const payload = malloc(this.wasm, 'u8', packet.value.payload.length);
|
|
38
|
+
payload.set(packet.value.payload);
|
|
39
|
+
media.decoder.push(packet.value.sequenceNumber, packet.value.timeStamp, packet.value.isEndOfSpurt, payload.byteOffset, payload.length);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
insertMedia(index, connectionId, mediaId) {
|
|
43
|
+
const id = mergeIds({ connectionId, mediaId });
|
|
44
|
+
const decoder = new this.wasm.Decoder();
|
|
45
|
+
this.medias.set(id, { index, decoder });
|
|
46
|
+
this.decoders[index] = decoder.$$.ptr;
|
|
47
|
+
this.largestIndex = Math.max(this.largestIndex, index);
|
|
48
|
+
setDecoderVolume(this.information, index, 1);
|
|
49
|
+
}
|
|
50
|
+
removeMedia(index, connectionId, mediaId) {
|
|
51
|
+
const id = mergeIds({ connectionId, mediaId });
|
|
52
|
+
const entry = this.medias.get(id);
|
|
53
|
+
if (entry === undefined || entry.index !== index)
|
|
54
|
+
return;
|
|
55
|
+
this.medias.delete(id);
|
|
56
|
+
setDecoderRmsDBFS(this.information, index, 0);
|
|
57
|
+
this.decoders[index] = 0;
|
|
58
|
+
entry.decoder.delete();
|
|
59
|
+
if (index === this.largestIndex) {
|
|
60
|
+
for (index = index - 1; index >= 0; index--) {
|
|
61
|
+
if (this.decoders[index] !== 0)
|
|
62
|
+
break;
|
|
63
|
+
}
|
|
64
|
+
this.largestIndex = index;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
setVolume(index, value) {
|
|
68
|
+
if (index === 'global') {
|
|
69
|
+
setGlobalDecoderVolume(this.information, value);
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
setDecoderVolume(this.information, index, value);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
tick() {
|
|
76
|
+
const id = this.id++;
|
|
77
|
+
const hasAudio = this.wasm.pop_many(this.decoders.byteOffset, this.information.values.byteOffset, this.largestIndex + 1, this.outBuffer.byteOffset);
|
|
78
|
+
if (hasAudio) {
|
|
79
|
+
const buffer = this.outBuffer.slice();
|
|
80
|
+
send(this.workletPort, ['samples', { id: this.id, buffer }], [buffer.buffer]);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@4players/odin-plugin-web",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.20",
|
|
4
4
|
"description": "A backend plugin providing the custom WebAssembly audio processing module for ODIN web clients",
|
|
5
5
|
"homepage": "https://www.4players.io",
|
|
6
6
|
"repository": {
|
|
@@ -8,41 +8,44 @@
|
|
|
8
8
|
"url": "https://github.com/4Players/odin-sdk-web.git"
|
|
9
9
|
},
|
|
10
10
|
"license": "UNLICENSED",
|
|
11
|
-
"main": "
|
|
12
|
-
"types": "
|
|
11
|
+
"main": "./lib/plugin/index.js",
|
|
12
|
+
"types": "./lib/plugin/index.d.ts",
|
|
13
13
|
"scripts": {
|
|
14
|
-
"clean": "
|
|
15
|
-
"
|
|
14
|
+
"clean": "rimraf build lib",
|
|
15
|
+
"configure": "npm-run-all -pl configure:*",
|
|
16
|
+
"configure:shared": "emcmake cmake -B build/shared -DCMAKE_BUILD_TYPE=Release -DSHARED_MEMORY=1 -S .",
|
|
17
|
+
"configure:unshared": "emcmake cmake -B build/unshared -DCMAKE_BUILD_TYPE=Release -S .",
|
|
18
|
+
"build": "npm-run-all -s configure build:wasm build:plugin build:worker",
|
|
19
|
+
"build:wasm": "npm-run-all -pl build:wasm:* && node install-wasm.js",
|
|
20
|
+
"build:wasm:shared": "emmake cmake --build build/shared",
|
|
21
|
+
"build:wasm:unshared": "emmake cmake --build build/unshared",
|
|
22
|
+
"build:worker": "ts-node src/build.ts && tsc -b ./tsconfig.worker.json",
|
|
23
|
+
"build:plugin": "tsc --build --verbose --force",
|
|
24
|
+
"format": "prettier 'src/**/*.ts' --write",
|
|
16
25
|
"lint": "eslint . --ext .ts",
|
|
17
|
-
"test": "npm --prefix test run
|
|
26
|
+
"test": "npm --prefix test run start",
|
|
27
|
+
"bundle": "npm run build && esbuild lib/plugin/index.js --bundle --outfile=dist/plugin.js"
|
|
18
28
|
},
|
|
19
29
|
"dependencies": {
|
|
20
|
-
|
|
21
|
-
"@4players/odin-plugin-web-backend": "file:./backend/",
|
|
22
|
-
"@4players/odin-plugin-web-wasm": "file:./wasm/",
|
|
23
|
-
"@4players/odin-plugin-web-worker": "file:./worker/",
|
|
24
|
-
"@msgpack/msgpack": "~3.0.0-beta2",
|
|
25
|
-
"uuid": "~9.0.0",
|
|
26
|
-
"zod": "~3.22.0"
|
|
30
|
+
"@4players/odin-common": "~2.7.0"
|
|
27
31
|
},
|
|
28
32
|
"devDependencies": {
|
|
29
|
-
"@types/node": "~20.
|
|
30
|
-
"@
|
|
31
|
-
"@typescript-eslint/
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"eslint": "~8.56.0",
|
|
33
|
+
"@types/node": "~20.14.0",
|
|
34
|
+
"@typescript-eslint/eslint-plugin": "~7.16.0",
|
|
35
|
+
"@typescript-eslint/parser": "~7.16.0",
|
|
36
|
+
"esbuild": "~0.23.0",
|
|
37
|
+
"eslint": "~8.57.0",
|
|
35
38
|
"eslint-config-semistandard": "~17.0.0",
|
|
36
39
|
"eslint-plugin-import": "~2.29.0",
|
|
37
40
|
"eslint-plugin-node": "~11.1.0",
|
|
38
41
|
"eslint-plugin-promise": "~6.1.0",
|
|
42
|
+
"npm-run-all2": "~6.1.0",
|
|
43
|
+
"prettier": "~3.3.0",
|
|
44
|
+
"rimraf": "~5.0.0",
|
|
39
45
|
"ts-node": "~10.9.0",
|
|
40
|
-
"typescript": "~5.
|
|
46
|
+
"typescript": "~5.5.0"
|
|
41
47
|
},
|
|
42
|
-
"
|
|
43
|
-
"
|
|
44
|
-
|
|
45
|
-
"backend",
|
|
46
|
-
"test"
|
|
47
|
-
]
|
|
48
|
+
"peerDependencies": {
|
|
49
|
+
"zod": "*"
|
|
50
|
+
}
|
|
48
51
|
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"audioController.d.ts","sourceRoot":"","sources":["../src/audioController.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,OAAO,EAEP,MAAM,EAKP,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,GAAG,CAAC;AACrC,OAAO,EACL,WAAW,EAEX,UAAU,EACV,cAAc,EAIf,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;AACpC,MAAM,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;AACxC,MAAM,MAAM,6BAA6B,GACvC,OAAO,CAAC,6BAA6B,CAAC;AACxC,MAAM,MAAM,4BAA4B,GACtC,OAAO,CAAC,4BAA4B,GAAG;IACrC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AACJ,MAAM,MAAM,mBAAmB,GAAG,CAChC,UAAU,EAAE,MAAM,GAAG,SAAS,KAC3B,OAAO,CAAC,YAAY,CAAC,CAAC;AAE3B,MAAM,WAAW,aAAc,SAAQ,UAAU;CAAG;AAEpD,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,gBAAgB;QACxB,IAAI,EAAE,MAAM,CAAC;KACd;IAED,UAAU,YAAY;QACpB,SAAS,CAAC,EAAE,CACV,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,gBAAgB,KAC1C,OAAO,CAAC,SAAS,CAAC,CAAC;KACzB;CACF;AAuBD,qBAAa,sBAAsB;IAOrB,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB;IAN7D,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC;QACtB,UAAU,EAAE,SAAS,GAAG,MAAM,CAAC;QAC/B,YAAY,EAAE,YAAY,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAM;gBAEa,mBAAmB,EAAE,mBAAmB;IAEvD,OAAO,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC;IAsB9D,OAAO,CAAC,YAAY,EAAE,gBAAgB;IAa5C,KAAK;CAKN;AAED,qBAAa,eAAe;IAsBxB,QAAQ,CAAC,aAAa,EAAE,sBAAsB;IAC9C,QAAQ,CAAC,IAAI,EAAE,WAAW;IAtB5B,cAAc,EAAE,cAAc,GAAG,SAAS,CAAC;IAC3C,YAAY,SAAK;IACjB,SAAS;YAGD,aAAa,GAAG,SAAS;gBACrB,MAAM;kBACJ,aAAa;OAEvB;IAEJ,iBAAiB,SAAgB;IAEpB,MAAM,CACjB,aAAa,EAAE,sBAAsB,EACrC,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,eAAe,CAAC;gBAKhB,aAAa,EAAE,sBAAsB,EACrC,IAAI,EAAE,WAAW;IAG5B,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU;IAIhD,mBAAmB,CACvB,UAAU,EAAE,6BAA6B,GACxC,OAAO,CAAC,aAAa,CAAC;IAenB,kBAAkB,CACtB,UAAU,EAAE,4BAA4B,GACvC,OAAO,CAAC,YAAY,CAAC;IAMlB,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IA0B3C,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS;IAsB5C,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM;IAK7C,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,aAAa,GAAG,IAAI;IAWnD,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,aAAa,GAAG,IAAI;IAQlD,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ;IAM1C,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IASpD,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAKtC,aAAa,CAAC,GAAG,EAAE,MAAM;IAOzB,KAAK,IAAI,IAAI;CAId;AAED,qBAAa,MAAO,YAAW,OAAO,CAAC,MAAM;IAC/B,QAAQ,CAAC,MAAM,EAAE,eAAe;IAAE,QAAQ,CAAC,SAAS,EAAE,OAAO;gBAApD,MAAM,EAAE,eAAe,EAAW,SAAS,GAAE,OAAe;IAEjF,IAAI,IAAI,qCASP;IAED,IAAI,IAAI,WAEP;CACF;AAED,qBAAa,aAAc,YAAW,OAAO,CAAC,aAAa;IAUvD,QAAQ,CAAC,UAAU,EAAE,eAAe;IACpC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG;IAV3B,IAAI,MAAM,mBAET;IAED,IAAI,QAAQ,qBAEX;gBAGU,UAAU,EAAE,eAAe,EAC3B,GAAG,EAAE,OAAO,CAAC,GAAG;IAG3B,KAAK,CAAC,EAAE,EAAE,aAAa,GAAG,IAAI;IAI9B,IAAI,CAAC,EAAE,EAAE,aAAa,GAAG,IAAI;IAI7B,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,GAAG,IAAI;IAItC,KAAK,IAAI,IAAI;CAGd;AAED,qBAAa,YAAa,YAAW,OAAO,CAAC,YAAY;IAmCrD,QAAQ,CAAC,UAAU,EAAE,eAAe;IAlCtC,OAAO,CAAC,QAAQ,CAKF;IAEd,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,OAAO,CAAC,IAAI,CAAc;IAC1B,OAAO,CAAC,IAAI,CAAc;IAC1B,OAAO,CAAC,OAAO,CAAa;IAC5B,OAAO,CAAC,iBAAiB,CAAC,CAAmB;IAE7C,IAAI,GAAG,sBAEN;IAED,IAAI,GAAG,sBAEN;IAED,IAAI,MAAM,mBAET;IAED,IAAI,QAAQ,wDAEX;IAED,IAAI,iBAAiB,WAEpB;gBAGU,UAAU,EAAE,eAAe,EACpC,UAAU,EAAE,4BAA4B;IAa1C,KAAK,CAAC,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,GAAG,IAAI;IAkBzE,IAAI,CAAC,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,GAAG,IAAI;IAWtD,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,GAAG,IAAI;IAOhC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAOhD,SAAS,CAAC,UAAU,EAAE,gBAAgB;IAqC5C,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,GAAG,IAAI;IAOtC,KAAK,IAAI,IAAI;IASP,aAAa;CAQpB"}
|