@4players/odin-plugin-web 1.0.0-alpha.7 → 1.0.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/plugin/audioController.d.ts +9 -4
- package/lib/plugin/audioController.js +52 -6
- package/lib/plugin/config.d.ts +0 -1
- package/lib/plugin/config.js +0 -1
- package/lib/plugin/connectionPool.d.ts +0 -1
- package/lib/plugin/connectionPool.js +0 -1
- package/lib/plugin/index.d.ts +0 -1
- package/lib/plugin/index.js +0 -1
- package/lib/plugin/mainConnection.d.ts +0 -1
- package/lib/plugin/mainConnection.js +0 -1
- package/lib/plugin/roomConnection.d.ts +0 -1
- package/lib/plugin/roomConnection.js +10 -6
- package/lib/plugin/rpcConnection.d.ts +0 -1
- package/lib/plugin/rpcConnection.js +0 -1
- package/lib/plugin/token.d.ts +2 -1
- package/lib/plugin/token.js +12 -1
- package/lib/worker/buffers.d.ts +42 -0
- package/lib/worker/buffers.js +79 -0
- package/lib/worker/common.d.ts +44 -27
- package/lib/worker/common.js +2 -95
- package/lib/worker/config.d.ts +8 -0
- package/lib/worker/config.js +11 -0
- package/lib/worker/index.d.ts +1 -45
- package/lib/worker/index.js +1 -2
- package/lib/worker/memory.d.ts +9 -0
- package/lib/worker/memory.js +29 -0
- package/lib/worker/messages.d.ts +0 -80
- package/lib/worker/messages.js +0 -1
- package/lib/worker/packet.d.ts +11 -0
- package/lib/worker/packet.js +59 -0
- package/lib/worker/shared/capture.d.ts +28 -0
- package/lib/worker/shared/capture.js +63 -0
- package/lib/worker/shared/index.d.ts +3 -0
- package/lib/worker/shared/index.js +148 -0
- package/lib/worker/shared/messages.d.ts +67 -0
- package/lib/worker/shared/messages.js +64 -0
- package/lib/worker/shared/playback.d.ts +21 -0
- package/lib/worker/shared/playback.js +78 -0
- package/lib/worker/unshared/capture.d.ts +28 -0
- package/lib/worker/unshared/capture.js +66 -0
- package/lib/worker/unshared/index.d.ts +3 -0
- package/lib/worker/unshared/index.js +171 -0
- package/lib/worker/unshared/messages.d.ts +81 -0
- package/lib/worker/unshared/messages.js +80 -0
- package/lib/worker/unshared/playback.d.ts +23 -0
- package/lib/worker/unshared/playback.js +83 -0
- package/package.json +15 -11
- package/lib/wasm/index.js +0 -19
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/// <reference lib="WebWorker" />
|
|
2
|
+
import { createDecoderInformationBuffer, createSampleBuffer, freeDecoderInformationBuffer, freeSampleBuffer, getSamples, setDecoderRmsDBFS, setHeader, } from '../buffers';
|
|
3
|
+
import { mergeIds } from '../common';
|
|
4
|
+
import { AdditionalPlaybackBlocks, MaxNumberOfDecoders } from '../config';
|
|
5
|
+
import { malloc } from '../memory';
|
|
6
|
+
import { Packet } from '../packet';
|
|
7
|
+
export class Playback {
|
|
8
|
+
constructor(wasm) {
|
|
9
|
+
this.wasm = wasm;
|
|
10
|
+
this.medias = new Map();
|
|
11
|
+
this.largestIndex = -1;
|
|
12
|
+
const numberOfBuffers = 2 + AdditionalPlaybackBlocks;
|
|
13
|
+
this.sampleBuffer = createSampleBuffer(wasm, 48000, numberOfBuffers, true);
|
|
14
|
+
this.sampleBuffer.samples.fill(0);
|
|
15
|
+
const prefill = numberOfBuffers - 2;
|
|
16
|
+
for (let i = 0; i < prefill; i++) {
|
|
17
|
+
setHeader(this.sampleBuffer, i, { id: i });
|
|
18
|
+
}
|
|
19
|
+
this.id = prefill;
|
|
20
|
+
this.decoders = malloc(this.wasm, 'u32', MaxNumberOfDecoders);
|
|
21
|
+
this.information = createDecoderInformationBuffer(this.wasm, MaxNumberOfDecoders);
|
|
22
|
+
}
|
|
23
|
+
close() {
|
|
24
|
+
freeSampleBuffer(this.wasm, this.sampleBuffer);
|
|
25
|
+
freeDecoderInformationBuffer(this.wasm, this.information);
|
|
26
|
+
for (const { decoder } of this.medias.values()) {
|
|
27
|
+
decoder.delete();
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
receivePacket(connectionId, packetBytes) {
|
|
31
|
+
const packet = Packet.parse(packetBytes);
|
|
32
|
+
if (packet.type === 'Failure') {
|
|
33
|
+
console.error(`failed decoding audio packet; ${packet.reason}`);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
for (const mediaId of packet.value.mediaIds) {
|
|
37
|
+
const media = this.medias.get(mergeIds({ connectionId, mediaId }));
|
|
38
|
+
if (media === undefined)
|
|
39
|
+
continue;
|
|
40
|
+
// the decoder keeps ownership of the payload
|
|
41
|
+
const payload = malloc(this.wasm, 'u8', packet.value.payload.length);
|
|
42
|
+
payload.set(packet.value.payload);
|
|
43
|
+
media.decoder.push(packet.value.sequenceNumber, packet.value.timeStamp, packet.value.isEndOfSpurt, payload.byteOffset, payload.length);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
insertMedia(index, connectionId, mediaId) {
|
|
47
|
+
const id = mergeIds({ connectionId, mediaId });
|
|
48
|
+
const decoder = new this.wasm.Decoder();
|
|
49
|
+
this.medias.set(id, { index, decoder });
|
|
50
|
+
this.decoders[index] = decoder.$$.ptr;
|
|
51
|
+
this.largestIndex = Math.max(this.largestIndex, index);
|
|
52
|
+
}
|
|
53
|
+
removeMedia(index, connectionId, mediaId) {
|
|
54
|
+
const id = mergeIds({ connectionId, mediaId });
|
|
55
|
+
const entry = this.medias.get(id);
|
|
56
|
+
if (entry === undefined || entry.index !== index)
|
|
57
|
+
return;
|
|
58
|
+
this.medias.delete(id);
|
|
59
|
+
setDecoderRmsDBFS(this.information, index, 0);
|
|
60
|
+
this.decoders[index] = 0;
|
|
61
|
+
entry.decoder.delete();
|
|
62
|
+
if (index === this.largestIndex) {
|
|
63
|
+
for (index = index - 1; index >= 0; index--) {
|
|
64
|
+
if (this.decoders[index] !== 0)
|
|
65
|
+
break;
|
|
66
|
+
}
|
|
67
|
+
this.largestIndex = index;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
tick() {
|
|
71
|
+
const id = this.id++;
|
|
72
|
+
const index = id % this.sampleBuffer.length;
|
|
73
|
+
const hasAudio = this.wasm.pop_many(this.decoders.byteOffset, this.information.values.byteOffset, this.largestIndex + 1, getSamples(this.sampleBuffer, index).byteOffset);
|
|
74
|
+
if (hasAudio) {
|
|
75
|
+
setHeader(this.sampleBuffer, index, { id });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/// <reference lib="webworker" />
|
|
2
|
+
import type { OdinWasm } from '../../../lib/wasm';
|
|
3
|
+
import type { VadConfig } from '../common';
|
|
4
|
+
import { EncoderInformationBuffer } from '../buffers';
|
|
5
|
+
export declare class Capture {
|
|
6
|
+
readonly sampleRate: number;
|
|
7
|
+
readonly wasm: typeof OdinWasm;
|
|
8
|
+
readonly encoder: OdinWasm.Encoder;
|
|
9
|
+
readonly information: EncoderInformationBuffer;
|
|
10
|
+
readonly inBuffer: Float32Array;
|
|
11
|
+
readonly outBuffer: Uint8Array;
|
|
12
|
+
sequenceNumber: number;
|
|
13
|
+
timestamp: number;
|
|
14
|
+
connections: Array<{
|
|
15
|
+
id: number;
|
|
16
|
+
mediaIds: number[];
|
|
17
|
+
}>;
|
|
18
|
+
active: boolean;
|
|
19
|
+
constructor(sampleRate: number, wasm: typeof OdinWasm);
|
|
20
|
+
setVad(config: VadConfig): void;
|
|
21
|
+
setMedias(connections: Array<{
|
|
22
|
+
id: number;
|
|
23
|
+
mediaIds: number[];
|
|
24
|
+
}>): void;
|
|
25
|
+
setVolume(value: number): void;
|
|
26
|
+
close(): void;
|
|
27
|
+
tick(buffer: Float32Array): void;
|
|
28
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/// <reference lib="WebWorker" />
|
|
2
|
+
import { createEncoderInformationBuffer, freeEncoderInformationBuffer, setEncoderVolume, } from '../buffers';
|
|
3
|
+
import { VadPreload } from '../config';
|
|
4
|
+
import { send } from '../messages';
|
|
5
|
+
import { Packet } from '../packet';
|
|
6
|
+
import { free, malloc } from '../memory';
|
|
7
|
+
export class Capture {
|
|
8
|
+
constructor(sampleRate, wasm) {
|
|
9
|
+
this.sampleRate = sampleRate;
|
|
10
|
+
this.wasm = wasm;
|
|
11
|
+
this.sequenceNumber = (Math.random() * Math.pow(2, 32)) >> 0;
|
|
12
|
+
this.timestamp = (Math.random() * Math.pow(2, 32)) >> 0;
|
|
13
|
+
this.connections = [];
|
|
14
|
+
this.active = false;
|
|
15
|
+
this.encoder = new wasm.Encoder(sampleRate,
|
|
16
|
+
/* fec */ 15,
|
|
17
|
+
/* voip */ true, VadPreload);
|
|
18
|
+
this.information = createEncoderInformationBuffer(this.wasm);
|
|
19
|
+
this.inBuffer = malloc(this.wasm, 'f32', sampleRate * 0.02);
|
|
20
|
+
this.outBuffer = malloc(this.wasm, 'u8', 1270 /* educated guess */);
|
|
21
|
+
setEncoderVolume(this.information, 1);
|
|
22
|
+
}
|
|
23
|
+
setVad(config) {
|
|
24
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
25
|
+
this.encoder.set_volume_gate(config.volumeGate !== undefined, (_b = (_a = config.volumeGate) === null || _a === void 0 ? void 0 : _a.attackThreshold) !== null && _b !== void 0 ? _b : 0, (_d = (_c = config.volumeGate) === null || _c === void 0 ? void 0 : _c.releaseThreshold) !== null && _d !== void 0 ? _d : 0);
|
|
26
|
+
this.encoder.set_speech_detection(config.voiceActivity !== undefined, (_f = (_e = config.voiceActivity) === null || _e === void 0 ? void 0 : _e.attackThreshold) !== null && _f !== void 0 ? _f : 0, (_h = (_g = config.voiceActivity) === null || _g === void 0 ? void 0 : _g.releaseThreshold) !== null && _h !== void 0 ? _h : 0);
|
|
27
|
+
}
|
|
28
|
+
setMedias(connections) {
|
|
29
|
+
this.connections = connections.filter(({ mediaIds }) => mediaIds.length > 0);
|
|
30
|
+
}
|
|
31
|
+
setVolume(value) {
|
|
32
|
+
setEncoderVolume(this.information, value);
|
|
33
|
+
}
|
|
34
|
+
close() {
|
|
35
|
+
freeEncoderInformationBuffer(this.wasm, this.information);
|
|
36
|
+
free(this.wasm, this.inBuffer);
|
|
37
|
+
free(this.wasm, this.outBuffer);
|
|
38
|
+
this.encoder.delete();
|
|
39
|
+
}
|
|
40
|
+
tick(buffer) {
|
|
41
|
+
this.inBuffer.set(buffer, 0);
|
|
42
|
+
this.encoder.push(this.inBuffer.byteOffset, this.information.values.byteOffset);
|
|
43
|
+
while (true) {
|
|
44
|
+
const n = this.encoder.pop(this.connections.length === 0 ? 0 : this.outBuffer.byteOffset, this.outBuffer.length);
|
|
45
|
+
if (n === 0)
|
|
46
|
+
break;
|
|
47
|
+
if (n < 0) {
|
|
48
|
+
console.error(`failed encoding packet: ${n}`);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
this.timestamp = (this.timestamp + 1800) >> 0; /* 20ms 90kHz */
|
|
52
|
+
if (n === 1)
|
|
53
|
+
continue;
|
|
54
|
+
this.sequenceNumber = (this.sequenceNumber + 1) >> 0;
|
|
55
|
+
const isEndOfSpurt = this.active && !this.encoder.active;
|
|
56
|
+
this.active = this.encoder.active;
|
|
57
|
+
for (const connection of this.connections) {
|
|
58
|
+
const packet = new Packet(connection.mediaIds, this.sequenceNumber, this.timestamp, isEndOfSpurt, this.outBuffer.subarray(0, n));
|
|
59
|
+
send(globalThis, [
|
|
60
|
+
'sendPacket',
|
|
61
|
+
{ connectionId: connection.id, packet: packet.toBytes() },
|
|
62
|
+
]);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { CaptureSink, CaptureWorker, PlaybackWorker } from '../common';
|
|
2
|
+
export declare function createCaptureWorker(context: AudioContext, sink: CaptureSink, refreshRate: number, workerPromise: Promise<Worker>, workletPromise: Promise<AudioWorkletNode>): Promise<CaptureWorker>;
|
|
3
|
+
export declare function createPlaybackWorker(audioContext: AudioContext, refreshRate: number, workerPromise: Promise<Worker>, workletPromise: Promise<AudioWorkletNode>): Promise<PlaybackWorker>;
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { Silence, mergeIds, } from '../common';
|
|
11
|
+
import { MaxNumberOfDecoders } from '../config';
|
|
12
|
+
import { send, waitFor } from '../messages';
|
|
13
|
+
class CaptureWorkerImpl {
|
|
14
|
+
constructor(worklet, worker, sink) {
|
|
15
|
+
this.worklet = worklet;
|
|
16
|
+
this.worker = worker;
|
|
17
|
+
this.sink = sink;
|
|
18
|
+
this._Activity = Silence;
|
|
19
|
+
this._SpeechProbability = 0;
|
|
20
|
+
worker.addEventListener('message', this._OnMessages.bind(this));
|
|
21
|
+
}
|
|
22
|
+
get context() {
|
|
23
|
+
return this.worklet.context;
|
|
24
|
+
}
|
|
25
|
+
get activity() {
|
|
26
|
+
return this._Activity;
|
|
27
|
+
}
|
|
28
|
+
get speechProbability() {
|
|
29
|
+
return this._SpeechProbability;
|
|
30
|
+
}
|
|
31
|
+
setMedias(connections) {
|
|
32
|
+
send(this.worker, ['captureSetMedias', { connections }]);
|
|
33
|
+
}
|
|
34
|
+
setVad(config) {
|
|
35
|
+
send(this.worker, ['captureSetVad', config]);
|
|
36
|
+
}
|
|
37
|
+
setVolume(value) {
|
|
38
|
+
send(this.worker, ['captureSetVolume', { value }]);
|
|
39
|
+
}
|
|
40
|
+
close() {
|
|
41
|
+
send(this.worklet.port, ['close', undefined]);
|
|
42
|
+
send(this.worker, ['close', undefined]);
|
|
43
|
+
}
|
|
44
|
+
_OnMessages(event) {
|
|
45
|
+
const [name, values] = event.data;
|
|
46
|
+
switch (name) {
|
|
47
|
+
case 'sendPacket':
|
|
48
|
+
this.sink(values.connectionId, values.packet);
|
|
49
|
+
break;
|
|
50
|
+
case 'captureUpdate':
|
|
51
|
+
this._Activity = {
|
|
52
|
+
isSilent: !values.active,
|
|
53
|
+
rmsDBFS: values.rmsDBFs,
|
|
54
|
+
};
|
|
55
|
+
this._SpeechProbability = values.speechProbability;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
export function createCaptureWorker(context, sink, refreshRate, workerPromise, workletPromise) {
|
|
60
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
61
|
+
const { port1, port2 } = new MessageChannel();
|
|
62
|
+
const [worker, worklet] = yield Promise.all([workerPromise, workletPromise]);
|
|
63
|
+
send(worker, [
|
|
64
|
+
'captureInit',
|
|
65
|
+
{ sampleRate: context.sampleRate, refreshRate, workletPort: port1 },
|
|
66
|
+
], [port1]);
|
|
67
|
+
yield waitFor(worker, 'captureReady');
|
|
68
|
+
send(worklet.port, ['init', { workerPort: port2 }], [port2]);
|
|
69
|
+
return new CaptureWorkerImpl(worklet, worker, sink);
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
class PlaybackWorkerImpl {
|
|
73
|
+
constructor(audioContext, worklet, worker) {
|
|
74
|
+
this.audioContext = audioContext;
|
|
75
|
+
this.worklet = worklet;
|
|
76
|
+
this.worker = worker;
|
|
77
|
+
this.availableIndexes = [];
|
|
78
|
+
this.medias = new Map();
|
|
79
|
+
this.activities = new Float32Array();
|
|
80
|
+
worker.addEventListener('message', this._OnMessages.bind(this));
|
|
81
|
+
}
|
|
82
|
+
receivePacket(connectionId, packet) {
|
|
83
|
+
send(this.worker, ['playbackReceivePacket', { connectionId, packet }], [packet.buffer]);
|
|
84
|
+
}
|
|
85
|
+
addMedias(connectionId, mediaIds) {
|
|
86
|
+
console.log('addMedias', connectionId, mediaIds);
|
|
87
|
+
const added = [];
|
|
88
|
+
for (const mediaId of mediaIds) {
|
|
89
|
+
const index = this._GetFreeIndex();
|
|
90
|
+
if (index === undefined)
|
|
91
|
+
continue;
|
|
92
|
+
this.medias.set(mergeIds({ connectionId, mediaId }), { index });
|
|
93
|
+
added.push({ connectionId, mediaId, index });
|
|
94
|
+
}
|
|
95
|
+
send(this.worker, [
|
|
96
|
+
'playbackUpdateMedias',
|
|
97
|
+
{ added, removed: [] },
|
|
98
|
+
]);
|
|
99
|
+
}
|
|
100
|
+
removeMedias(connectionId, mediaIds) {
|
|
101
|
+
const removed = [];
|
|
102
|
+
for (const mediaId of mediaIds) {
|
|
103
|
+
const id = mergeIds({ connectionId, mediaId });
|
|
104
|
+
const media = this.medias.get(id);
|
|
105
|
+
if (media === undefined)
|
|
106
|
+
continue;
|
|
107
|
+
this.medias.delete(id);
|
|
108
|
+
this.availableIndexes.push(media.index);
|
|
109
|
+
removed.push({ connectionId, mediaId, index: media.index });
|
|
110
|
+
}
|
|
111
|
+
send(this.worker, [
|
|
112
|
+
'playbackUpdateMedias',
|
|
113
|
+
{ added: [], removed },
|
|
114
|
+
]);
|
|
115
|
+
}
|
|
116
|
+
getActivity(id) {
|
|
117
|
+
const entry = this.medias.get(mergeIds(id));
|
|
118
|
+
if (entry === undefined)
|
|
119
|
+
return undefined;
|
|
120
|
+
const value = this.activities.at(entry.index);
|
|
121
|
+
if (value === undefined)
|
|
122
|
+
return undefined;
|
|
123
|
+
return value === 0 ? Silence : { isSilent: false, rmsDBFS: value };
|
|
124
|
+
}
|
|
125
|
+
setGlobalVolume(value) {
|
|
126
|
+
send(this.worker, [
|
|
127
|
+
'playbackSetVolume',
|
|
128
|
+
{ index: 'global', value },
|
|
129
|
+
]);
|
|
130
|
+
}
|
|
131
|
+
setVolume(id, value) {
|
|
132
|
+
const entry = this.medias.get(mergeIds(id));
|
|
133
|
+
if (entry !== undefined) {
|
|
134
|
+
send(this.worker, [
|
|
135
|
+
'playbackSetVolume',
|
|
136
|
+
{ index: entry.index, value },
|
|
137
|
+
]);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
close() {
|
|
141
|
+
send(this.worker, ['close', undefined]);
|
|
142
|
+
send(this.worklet.port, ['close', undefined]);
|
|
143
|
+
}
|
|
144
|
+
_GetFreeIndex() {
|
|
145
|
+
const result = this.availableIndexes.pop();
|
|
146
|
+
if (result !== undefined)
|
|
147
|
+
return result;
|
|
148
|
+
if (this.medias.size < MaxNumberOfDecoders)
|
|
149
|
+
return this.medias.size;
|
|
150
|
+
return undefined;
|
|
151
|
+
}
|
|
152
|
+
_OnMessages(event) {
|
|
153
|
+
const [name, values] = event.data;
|
|
154
|
+
switch (name) {
|
|
155
|
+
case 'playbackUpdate':
|
|
156
|
+
this.activities = values.activities;
|
|
157
|
+
break;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
export function createPlaybackWorker(audioContext, refreshRate, workerPromise, workletPromise) {
|
|
162
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
163
|
+
const { port1, port2 } = new MessageChannel();
|
|
164
|
+
const [worker, worklet] = yield Promise.all([workerPromise, workletPromise]);
|
|
165
|
+
send(worker, ['playbackInit', { workletPort: port1, refreshRate }], [port1]);
|
|
166
|
+
yield waitFor(worker, 'playbackReady');
|
|
167
|
+
send(worklet.port, ['init', { workerPort: port2 }], [port2]);
|
|
168
|
+
worklet.connect(audioContext.destination);
|
|
169
|
+
return new PlaybackWorkerImpl(audioContext, worklet, worker);
|
|
170
|
+
});
|
|
171
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { VadConfig } from '../common';
|
|
2
|
+
import { Inflate } from '../messages';
|
|
3
|
+
export type CaptureWorkletMessages = {
|
|
4
|
+
init: {
|
|
5
|
+
workerPort: MessagePort;
|
|
6
|
+
};
|
|
7
|
+
close: void;
|
|
8
|
+
};
|
|
9
|
+
export type PlaybackWorkletMessages = {
|
|
10
|
+
init: {
|
|
11
|
+
workerPort: MessagePort;
|
|
12
|
+
};
|
|
13
|
+
samples: {
|
|
14
|
+
id: number;
|
|
15
|
+
buffer: Float32Array;
|
|
16
|
+
};
|
|
17
|
+
close: void;
|
|
18
|
+
};
|
|
19
|
+
export type WorkerMessages = {
|
|
20
|
+
captureInit: {
|
|
21
|
+
sampleRate: number;
|
|
22
|
+
refreshRate: number;
|
|
23
|
+
workletPort: MessagePort;
|
|
24
|
+
};
|
|
25
|
+
captureSetMedias: {
|
|
26
|
+
connections: Array<{
|
|
27
|
+
id: number;
|
|
28
|
+
mediaIds: number[];
|
|
29
|
+
}>;
|
|
30
|
+
};
|
|
31
|
+
captureSetVad: VadConfig;
|
|
32
|
+
captureSetVolume: {
|
|
33
|
+
value: number;
|
|
34
|
+
};
|
|
35
|
+
captureTick: {
|
|
36
|
+
buffer: Float32Array;
|
|
37
|
+
};
|
|
38
|
+
playbackInit: {
|
|
39
|
+
refreshRate: number;
|
|
40
|
+
workletPort: MessagePort;
|
|
41
|
+
};
|
|
42
|
+
playbackUpdateMedias: {
|
|
43
|
+
added: Array<{
|
|
44
|
+
index: number;
|
|
45
|
+
connectionId: number;
|
|
46
|
+
mediaId: number;
|
|
47
|
+
}>;
|
|
48
|
+
removed: Array<{
|
|
49
|
+
index: number;
|
|
50
|
+
connectionId: number;
|
|
51
|
+
mediaId: number;
|
|
52
|
+
}>;
|
|
53
|
+
};
|
|
54
|
+
playbackSetVolume: {
|
|
55
|
+
index: 'global' | number;
|
|
56
|
+
value: number;
|
|
57
|
+
};
|
|
58
|
+
playbackReceivePacket: {
|
|
59
|
+
connectionId: number;
|
|
60
|
+
packet: Uint8Array;
|
|
61
|
+
};
|
|
62
|
+
playbackTick: void;
|
|
63
|
+
close: void;
|
|
64
|
+
};
|
|
65
|
+
export type MainMessages = {
|
|
66
|
+
sendPacket: {
|
|
67
|
+
connectionId: number;
|
|
68
|
+
packet: Uint8Array;
|
|
69
|
+
};
|
|
70
|
+
captureReady: void;
|
|
71
|
+
playbackReady: void;
|
|
72
|
+
captureUpdate: {
|
|
73
|
+
active: boolean;
|
|
74
|
+
rmsDBFs: number;
|
|
75
|
+
speechProbability: number;
|
|
76
|
+
};
|
|
77
|
+
playbackUpdate: {
|
|
78
|
+
activities: Float32Array;
|
|
79
|
+
};
|
|
80
|
+
};
|
|
81
|
+
export declare function onMessage(event: MessageEvent<Inflate<WorkerMessages>>): void;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import LoadOdinWasm from '../../../lib/wasm';
|
|
2
|
+
import { getDecoderRmsDBFS, getEncoderActive, getEncoderSpeechProbability, } from '../buffers';
|
|
3
|
+
import { send } from '../messages';
|
|
4
|
+
import { Capture } from './capture';
|
|
5
|
+
import { Playback } from './playback';
|
|
6
|
+
let playback;
|
|
7
|
+
let playbackInterval;
|
|
8
|
+
let capture;
|
|
9
|
+
let captureInterval;
|
|
10
|
+
export function onMessage(event) {
|
|
11
|
+
handle(event.data);
|
|
12
|
+
}
|
|
13
|
+
function handle(message) {
|
|
14
|
+
const [name, values] = message;
|
|
15
|
+
switch (name) {
|
|
16
|
+
case 'playbackInit':
|
|
17
|
+
playback = new Playback(LoadOdinWasm(), values.workletPort);
|
|
18
|
+
playbackInterval = setInterval(() => {
|
|
19
|
+
const activities = new Float32Array(playback.information.length * 2);
|
|
20
|
+
for (let i = 0; i <= playback.largestIndex; i++) {
|
|
21
|
+
activities[i] = getDecoderRmsDBFS(playback.information, i);
|
|
22
|
+
}
|
|
23
|
+
send(globalThis, ['playbackUpdate', { activities }], [activities.buffer]);
|
|
24
|
+
}, values.refreshRate);
|
|
25
|
+
values.workletPort.onmessage = (event) => onMessage(event);
|
|
26
|
+
send(globalThis, ['playbackReady', undefined]);
|
|
27
|
+
break;
|
|
28
|
+
case 'playbackReceivePacket':
|
|
29
|
+
playback === null || playback === void 0 ? void 0 : playback.receivePacket(values.connectionId, values.packet);
|
|
30
|
+
break;
|
|
31
|
+
case 'playbackUpdateMedias':
|
|
32
|
+
for (const { index, connectionId, mediaId } of values.removed) {
|
|
33
|
+
playback === null || playback === void 0 ? void 0 : playback.removeMedia(index, connectionId, mediaId);
|
|
34
|
+
}
|
|
35
|
+
for (const { index, connectionId, mediaId } of values.added) {
|
|
36
|
+
playback === null || playback === void 0 ? void 0 : playback.insertMedia(index, connectionId, mediaId);
|
|
37
|
+
}
|
|
38
|
+
break;
|
|
39
|
+
case 'playbackSetVolume':
|
|
40
|
+
playback === null || playback === void 0 ? void 0 : playback.setVolume(values.index, values.value);
|
|
41
|
+
break;
|
|
42
|
+
case 'playbackTick':
|
|
43
|
+
playback === null || playback === void 0 ? void 0 : playback.tick();
|
|
44
|
+
break;
|
|
45
|
+
case 'captureInit':
|
|
46
|
+
capture = new Capture(values.sampleRate, LoadOdinWasm());
|
|
47
|
+
captureInterval = setInterval(() => {
|
|
48
|
+
send(globalThis, [
|
|
49
|
+
'captureUpdate',
|
|
50
|
+
{
|
|
51
|
+
active: getEncoderActive(capture.information),
|
|
52
|
+
rmsDBFs: getEncoderSpeechProbability(capture.information),
|
|
53
|
+
speechProbability: getEncoderSpeechProbability(capture.information),
|
|
54
|
+
},
|
|
55
|
+
]);
|
|
56
|
+
}, values.refreshRate);
|
|
57
|
+
values.workletPort.onmessage = (event) => onMessage(event);
|
|
58
|
+
send(globalThis, ['captureReady', undefined]);
|
|
59
|
+
break;
|
|
60
|
+
case 'captureSetVad':
|
|
61
|
+
capture === null || capture === void 0 ? void 0 : capture.setVad(values);
|
|
62
|
+
break;
|
|
63
|
+
case 'captureSetMedias':
|
|
64
|
+
capture === null || capture === void 0 ? void 0 : capture.setMedias(values.connections);
|
|
65
|
+
break;
|
|
66
|
+
case 'captureSetVolume':
|
|
67
|
+
capture === null || capture === void 0 ? void 0 : capture.setVolume(values.value);
|
|
68
|
+
break;
|
|
69
|
+
case 'captureTick':
|
|
70
|
+
capture === null || capture === void 0 ? void 0 : capture.tick(values.buffer);
|
|
71
|
+
break;
|
|
72
|
+
case 'close':
|
|
73
|
+
playback === null || playback === void 0 ? void 0 : playback.close();
|
|
74
|
+
clearInterval(playbackInterval);
|
|
75
|
+
capture === null || capture === void 0 ? void 0 : capture.close();
|
|
76
|
+
clearInterval(captureInterval);
|
|
77
|
+
self.close();
|
|
78
|
+
break;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/// <reference lib="webworker" />
|
|
2
|
+
import type { OdinWasm } from '../../../lib/wasm';
|
|
3
|
+
import { DecoderInformationBuffer } from '../buffers';
|
|
4
|
+
export declare class Playback {
|
|
5
|
+
readonly wasm: typeof OdinWasm;
|
|
6
|
+
readonly workletPort: MessagePort;
|
|
7
|
+
readonly medias: Map<number, {
|
|
8
|
+
index: number;
|
|
9
|
+
decoder: OdinWasm.Decoder;
|
|
10
|
+
}>;
|
|
11
|
+
readonly decoders: Uint32Array;
|
|
12
|
+
readonly information: DecoderInformationBuffer;
|
|
13
|
+
readonly outBuffer: Float32Array;
|
|
14
|
+
largestIndex: number;
|
|
15
|
+
id: number;
|
|
16
|
+
constructor(wasm: typeof OdinWasm, workletPort: MessagePort);
|
|
17
|
+
close(): void;
|
|
18
|
+
receivePacket(connectionId: number, packetBytes: Uint8Array): void;
|
|
19
|
+
insertMedia(index: number, connectionId: number, mediaId: number): void;
|
|
20
|
+
removeMedia(index: number, connectionId: number, mediaId: number): void;
|
|
21
|
+
setVolume(index: number | 'global', value: number): void;
|
|
22
|
+
tick(): void;
|
|
23
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/// <reference lib="WebWorker" />
|
|
2
|
+
import { createDecoderInformationBuffer, freeDecoderInformationBuffer, setDecoderRmsDBFS, setDecoderVolume, setGlobalDecoderVolume, } from '../buffers';
|
|
3
|
+
import { mergeIds } from '../common';
|
|
4
|
+
import { MaxNumberOfDecoders, PlaybackPreload } from '../config';
|
|
5
|
+
import { malloc } from '../memory';
|
|
6
|
+
import { send } from '../messages';
|
|
7
|
+
import { Packet } from '../packet';
|
|
8
|
+
export class Playback {
|
|
9
|
+
constructor(wasm, workletPort) {
|
|
10
|
+
this.wasm = wasm;
|
|
11
|
+
this.workletPort = workletPort;
|
|
12
|
+
this.medias = new Map();
|
|
13
|
+
this.largestIndex = -1;
|
|
14
|
+
this.id = PlaybackPreload;
|
|
15
|
+
this.decoders = malloc(this.wasm, 'u32', MaxNumberOfDecoders);
|
|
16
|
+
this.information = createDecoderInformationBuffer(this.wasm, MaxNumberOfDecoders);
|
|
17
|
+
this.outBuffer = malloc(this.wasm, 'f32', 48000 * 0.02 * 2);
|
|
18
|
+
setGlobalDecoderVolume(this.information, 1);
|
|
19
|
+
}
|
|
20
|
+
close() {
|
|
21
|
+
freeDecoderInformationBuffer(this.wasm, this.information);
|
|
22
|
+
for (const { decoder } of this.medias.values()) {
|
|
23
|
+
decoder.delete();
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
receivePacket(connectionId, packetBytes) {
|
|
27
|
+
const packet = Packet.parse(packetBytes);
|
|
28
|
+
if (packet.type === 'Failure') {
|
|
29
|
+
console.error(`failed decoding audio packet; ${packet.reason}`);
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
for (const mediaId of packet.value.mediaIds) {
|
|
33
|
+
const media = this.medias.get(mergeIds({ connectionId, mediaId }));
|
|
34
|
+
if (media === undefined)
|
|
35
|
+
continue;
|
|
36
|
+
// the decoder keeps ownership of the payload
|
|
37
|
+
const payload = malloc(this.wasm, 'u8', packet.value.payload.length);
|
|
38
|
+
payload.set(packet.value.payload);
|
|
39
|
+
media.decoder.push(packet.value.sequenceNumber, packet.value.timeStamp, packet.value.isEndOfSpurt, payload.byteOffset, payload.length);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
insertMedia(index, connectionId, mediaId) {
|
|
43
|
+
const id = mergeIds({ connectionId, mediaId });
|
|
44
|
+
const decoder = new this.wasm.Decoder();
|
|
45
|
+
this.medias.set(id, { index, decoder });
|
|
46
|
+
this.decoders[index] = decoder.$$.ptr;
|
|
47
|
+
this.largestIndex = Math.max(this.largestIndex, index);
|
|
48
|
+
setDecoderVolume(this.information, index, 1);
|
|
49
|
+
}
|
|
50
|
+
removeMedia(index, connectionId, mediaId) {
|
|
51
|
+
const id = mergeIds({ connectionId, mediaId });
|
|
52
|
+
const entry = this.medias.get(id);
|
|
53
|
+
if (entry === undefined || entry.index !== index)
|
|
54
|
+
return;
|
|
55
|
+
this.medias.delete(id);
|
|
56
|
+
setDecoderRmsDBFS(this.information, index, 0);
|
|
57
|
+
this.decoders[index] = 0;
|
|
58
|
+
entry.decoder.delete();
|
|
59
|
+
if (index === this.largestIndex) {
|
|
60
|
+
for (index = index - 1; index >= 0; index--) {
|
|
61
|
+
if (this.decoders[index] !== 0)
|
|
62
|
+
break;
|
|
63
|
+
}
|
|
64
|
+
this.largestIndex = index;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
setVolume(index, value) {
|
|
68
|
+
if (index === 'global') {
|
|
69
|
+
setGlobalDecoderVolume(this.information, value);
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
setDecoderVolume(this.information, index, value);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
tick() {
|
|
76
|
+
const id = this.id++;
|
|
77
|
+
const hasAudio = this.wasm.pop_many(this.decoders.byteOffset, this.information.values.byteOffset, this.largestIndex + 1, this.outBuffer.byteOffset);
|
|
78
|
+
if (hasAudio) {
|
|
79
|
+
const buffer = this.outBuffer.slice();
|
|
80
|
+
send(this.workletPort, ['samples', { id: this.id, buffer }], [buffer.buffer]);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@4players/odin-plugin-web",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.9",
|
|
4
4
|
"description": "A backend plugin providing the custom WebAssembly audio processing module for ODIN web clients",
|
|
5
5
|
"homepage": "https://www.4players.io",
|
|
6
6
|
"repository": {
|
|
@@ -8,30 +8,34 @@
|
|
|
8
8
|
"url": "https://github.com/4Players/odin-sdk-web.git"
|
|
9
9
|
},
|
|
10
10
|
"license": "UNLICENSED",
|
|
11
|
-
"main": "lib/plugin/index.js",
|
|
12
|
-
"types": "lib/plugin/index.d.ts",
|
|
11
|
+
"main": "./lib/plugin/index.js",
|
|
12
|
+
"types": "./lib/plugin/index.d.ts",
|
|
13
13
|
"scripts": {
|
|
14
14
|
"clean": "rm -rf build lib",
|
|
15
|
-
"configure": "
|
|
15
|
+
"configure": "npm-run-all -pl configure:*",
|
|
16
|
+
"configure:shared": "emcmake cmake -B build/shared -DCMAKE_BUILD_TYPE=Release -DSHARED_MEMORY=1 -S .",
|
|
17
|
+
"configure:unshared": "emcmake cmake -B build/unshared -DCMAKE_BUILD_TYPE=Release -S .",
|
|
16
18
|
"build": "npm-run-all -s configure build:wasm build:plugin build:worker",
|
|
17
|
-
"build:
|
|
18
|
-
"build:wasm": "emmake
|
|
19
|
+
"build:wasm": "npm-run-all -pl build:wasm:* && node install-wasm.js",
|
|
20
|
+
"build:wasm:shared": "emmake cmake --build build/shared",
|
|
21
|
+
"build:wasm:unshared": "emmake cmake --build build/unshared",
|
|
19
22
|
"build:worker": "ts-node src/build.ts && tsc -b ./tsconfig.worker.json",
|
|
23
|
+
"build:plugin": "tsc",
|
|
20
24
|
"lint": "eslint . --ext .ts",
|
|
21
25
|
"test": "npm --prefix test run serve"
|
|
22
26
|
},
|
|
23
27
|
"dependencies": {
|
|
24
|
-
"@4players/odin-common": "~1.
|
|
28
|
+
"@4players/odin-common": "~1.4.0",
|
|
25
29
|
"@msgpack/msgpack": "~3.0.0-beta2",
|
|
26
30
|
"uuid": "~9.0.0",
|
|
27
31
|
"zod": "~3.22.0"
|
|
28
32
|
},
|
|
29
33
|
"devDependencies": {
|
|
30
|
-
"@types/node": "~20.
|
|
34
|
+
"@types/node": "~20.11.0",
|
|
31
35
|
"@types/uuid": "~9.0.0",
|
|
32
|
-
"@typescript-eslint/eslint-plugin": "~6.
|
|
33
|
-
"@typescript-eslint/parser": "~6.
|
|
34
|
-
"esbuild": "~0.
|
|
36
|
+
"@typescript-eslint/eslint-plugin": "~6.20.0",
|
|
37
|
+
"@typescript-eslint/parser": "~6.20.0",
|
|
38
|
+
"esbuild": "~0.20.0",
|
|
35
39
|
"eslint": "~8.56.0",
|
|
36
40
|
"eslint-config-semistandard": "~17.0.0",
|
|
37
41
|
"eslint-plugin-import": "~2.29.0",
|