@dtelecom/server-sdk-node 0.2.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -4
- package/dist/index.d.ts +35 -2
- package/dist/index.js +192 -151
- package/dist/index.js.map +1 -1
- package/package.json +4 -7
- package/patches/werift+0.22.8.patch +0 -17
package/README.md
CHANGED
|
@@ -8,15 +8,17 @@ Node.js RTC SDK for dTelecom — join rooms as a WebRTC participant for AI bots.
|
|
|
8
8
|
- **Audio receive** — subscribe to remote audio as PCM16 streams (16kHz for STT)
|
|
9
9
|
- **Audio send** — publish PCM16 audio as Opus (from TTS or any source)
|
|
10
10
|
- **Data channels** — send/receive messages via reliable or lossy channels
|
|
11
|
-
- **Pure TypeScript WebRTC** — uses [werift](https://github.com/
|
|
12
|
-
- **Opus codec** —
|
|
11
|
+
- **Pure TypeScript WebRTC** — uses [werift](https://github.com/shinyoshiaki/werift-webrtc) (no native WebRTC deps)
|
|
12
|
+
- **Opus codec** — [opusscript](https://github.com/abalabahaha/opusscript) (libopus compiled to WASM; nothing to build at install)
|
|
13
13
|
|
|
14
14
|
## Install
|
|
15
15
|
|
|
16
16
|
```bash
|
|
17
|
-
npm install @dtelecom/server-sdk-node
|
|
17
|
+
npm install @dtelecom/server-sdk-node
|
|
18
18
|
```
|
|
19
19
|
|
|
20
|
+
Requires Node.js 22 or later. Nothing is compiled at install time.
|
|
21
|
+
|
|
20
22
|
## Quick Start: Echo Bot
|
|
21
23
|
|
|
22
24
|
```typescript
|
|
@@ -89,6 +91,16 @@ room.on('trackSubscribed', async (remoteTrack) => {
|
|
|
89
91
|
});
|
|
90
92
|
```
|
|
91
93
|
|
|
94
|
+
`AudioStream` preserves silence across Opus DTX and lost-packet timestamp gaps.
|
|
95
|
+
After the first decoded packet, a stalled sender produces zero PCM with a 60 ms
|
|
96
|
+
jitter grace period, allowing downstream VAD/turn detection to observe elapsed
|
|
97
|
+
silence. Normal packets are emitted immediately. Silence already generated by
|
|
98
|
+
the timer is not inserted again when the next RTP packet arrives; accepted
|
|
99
|
+
resumed speech is preserved in full. Close the stream (or stop the remote track)
|
|
100
|
+
to stop its timer. The close log reports `timestamp_silence_ms` and
|
|
101
|
+
`stalled_silence_ms` for diagnosis. These counts include intentional sender
|
|
102
|
+
silence suppression and are not packet-loss measurements.
|
|
103
|
+
|
|
92
104
|
### Data Channels
|
|
93
105
|
|
|
94
106
|
```typescript
|
|
@@ -122,7 +134,7 @@ npx tsx examples/send-audio.ts --room test --input hello.pcm
|
|
|
122
134
|
| Package | Purpose | Native? |
|
|
123
135
|
|---|---|---|
|
|
124
136
|
| `werift` | WebRTC (ICE, DTLS, SRTP) | No (pure TS) |
|
|
125
|
-
|
|
|
137
|
+
| `opusscript` | Opus encode and decode | No (WASM) |
|
|
126
138
|
| `ws` | WebSocket client | No |
|
|
127
139
|
| `protobufjs` | Protobuf encode/decode | No |
|
|
128
140
|
|
package/dist/index.d.ts
CHANGED
|
@@ -446,6 +446,12 @@ declare class RTCEngine extends TypedEmitter<EngineEvents> {
|
|
|
446
446
|
private joinResponse;
|
|
447
447
|
private publishWaiters;
|
|
448
448
|
private _negotiateResolve;
|
|
449
|
+
/** Negotiations run one at a time: there is one slot for the pending answer. */
|
|
450
|
+
private negotiateChain;
|
|
451
|
+
/** The publisher has an applied answer, so its data channels can open. */
|
|
452
|
+
private publisherNegotiated;
|
|
453
|
+
/** A negotiation started only to open the data channels, shared by concurrent senders. */
|
|
454
|
+
private dataNegotiation;
|
|
449
455
|
private _publisherConnectedResolve;
|
|
450
456
|
get isConnected(): boolean;
|
|
451
457
|
get publisherPC(): RTCPeerConnection | null;
|
|
@@ -465,9 +471,19 @@ declare class RTCEngine extends TypedEmitter<EngineEvents> {
|
|
|
465
471
|
disableDtx?: boolean;
|
|
466
472
|
muted?: boolean;
|
|
467
473
|
}): Promise<TrackPublishedResponse>;
|
|
474
|
+
/** Offers the publisher's current tracks and data channels; one negotiation at a time. */
|
|
468
475
|
negotiate(): Promise<void>;
|
|
476
|
+
private runNegotiate;
|
|
469
477
|
/** Wait for publisher ICE to reach connected state (DTLS+SRTP ready). */
|
|
470
478
|
waitForPublisherConnected(timeoutMs?: number): Promise<void>;
|
|
479
|
+
/**
|
|
480
|
+
* Opens a publisher data channel before the first message. With subscriber-primary joins the
|
|
481
|
+
* publisher connection is only negotiated when something is published, so a bot that has not
|
|
482
|
+
* published a track negotiates it here (the offer carries the data channels). Throws when the
|
|
483
|
+
* channel does not open in time.
|
|
484
|
+
*/
|
|
485
|
+
ensureDataChannelOpen(kind: 'reliable' | 'lossy', timeoutMs?: number): Promise<void>;
|
|
486
|
+
/** Sends on an open publisher data channel; throws when it is not open. */
|
|
471
487
|
sendData(data: Uint8Array, kind: 'reliable' | 'lossy'): void;
|
|
472
488
|
disconnect(): Promise<void>;
|
|
473
489
|
private buildIceServers;
|
|
@@ -575,12 +591,20 @@ declare class AudioSource {
|
|
|
575
591
|
|
|
576
592
|
declare class AudioStream implements AsyncIterable<AudioFrame> {
|
|
577
593
|
private decoder;
|
|
578
|
-
private depacketizer;
|
|
579
594
|
private queue;
|
|
580
595
|
private readonly outputSampleRate;
|
|
581
596
|
private readonly outputChannels;
|
|
582
597
|
private track;
|
|
583
598
|
private _closed;
|
|
599
|
+
private silenceTimer;
|
|
600
|
+
private lastSequence;
|
|
601
|
+
private lastTimestamp;
|
|
602
|
+
private lastFrameSamples;
|
|
603
|
+
private lastPacketAt;
|
|
604
|
+
private concealedSamples;
|
|
605
|
+
private currentSsrc;
|
|
606
|
+
private timestampSilenceSamples;
|
|
607
|
+
private stalledSilenceSamples;
|
|
584
608
|
/**
|
|
585
609
|
* @param track The remote audio track to stream from
|
|
586
610
|
* @param sampleRate Desired output sample rate (default: 16000 for STT)
|
|
@@ -596,6 +620,8 @@ declare class AudioStream implements AsyncIterable<AudioFrame> {
|
|
|
596
620
|
private packetCount;
|
|
597
621
|
private start;
|
|
598
622
|
private processRtpPacket;
|
|
623
|
+
private fillStalledInput;
|
|
624
|
+
private emitSilence;
|
|
599
625
|
private emitFrame;
|
|
600
626
|
}
|
|
601
627
|
|
|
@@ -702,6 +728,8 @@ declare abstract class Participant extends TypedEmitter<ParticipantEvents> {
|
|
|
702
728
|
name: string;
|
|
703
729
|
metadata: string;
|
|
704
730
|
state: ParticipantInfo_State;
|
|
731
|
+
/** What the participant's token allows, as the node reports it. */
|
|
732
|
+
permissions?: ParticipantPermission;
|
|
705
733
|
protected _trackPublications: Map<string, TrackPublication>;
|
|
706
734
|
constructor(sid: string, identity: string, name?: string, metadata?: string);
|
|
707
735
|
get trackPublications(): Map<string, TrackPublication>;
|
|
@@ -715,6 +743,8 @@ interface DataPublishOptions {
|
|
|
715
743
|
destinationSids?: string[];
|
|
716
744
|
/** Topic for the data message */
|
|
717
745
|
topic?: string;
|
|
746
|
+
/** How long to wait for the data channel to open on first use (default 15 s). */
|
|
747
|
+
timeoutMs?: number;
|
|
718
748
|
}
|
|
719
749
|
declare class LocalParticipant extends Participant {
|
|
720
750
|
private engine;
|
|
@@ -737,8 +767,11 @@ declare class LocalParticipant extends Participant {
|
|
|
737
767
|
/**
|
|
738
768
|
* Publish data to the room.
|
|
739
769
|
*
|
|
770
|
+
* Works before any track is published: the first call opens the data channel. Rejects when the
|
|
771
|
+
* participant may not publish data, or the channel does not open within `timeoutMs`.
|
|
772
|
+
*
|
|
740
773
|
* @param data The data payload
|
|
741
|
-
* @param options Delivery options (kind, destinations, topic)
|
|
774
|
+
* @param options Delivery options (kind, destinations, topic, timeoutMs)
|
|
742
775
|
*/
|
|
743
776
|
publishData(data: Uint8Array, options?: DataPublishOptions): Promise<void>;
|
|
744
777
|
}
|
package/dist/index.js
CHANGED
|
@@ -1450,6 +1450,11 @@ var SignalClient = class extends TypedEmitter {
|
|
|
1450
1450
|
|
|
1451
1451
|
// src/engine.ts
|
|
1452
1452
|
var log2 = createLogger("RTCEngine");
|
|
1453
|
+
function parseTrickleCandidate(candidateInit) {
|
|
1454
|
+
const init = JSON.parse(candidateInit);
|
|
1455
|
+
if (init.sdpMid === "") delete init.sdpMid;
|
|
1456
|
+
return new import_werift.RTCIceCandidate(init);
|
|
1457
|
+
}
|
|
1453
1458
|
var RTCEngine = class extends TypedEmitter {
|
|
1454
1459
|
signal;
|
|
1455
1460
|
publisher = null;
|
|
@@ -1464,6 +1469,12 @@ var RTCEngine = class extends TypedEmitter {
|
|
|
1464
1469
|
joinResponse = null;
|
|
1465
1470
|
publishWaiters = /* @__PURE__ */ new Map();
|
|
1466
1471
|
_negotiateResolve = null;
|
|
1472
|
+
/** Negotiations run one at a time: there is one slot for the pending answer. */
|
|
1473
|
+
negotiateChain = Promise.resolve();
|
|
1474
|
+
/** The publisher has an applied answer, so its data channels can open. */
|
|
1475
|
+
publisherNegotiated = false;
|
|
1476
|
+
/** A negotiation started only to open the data channels, shared by concurrent senders. */
|
|
1477
|
+
dataNegotiation = null;
|
|
1467
1478
|
_publisherConnectedResolve = null;
|
|
1468
1479
|
get isConnected() {
|
|
1469
1480
|
return this._isConnected;
|
|
@@ -1559,7 +1570,14 @@ var RTCEngine = class extends TypedEmitter {
|
|
|
1559
1570
|
this.signal.sendAddTrack(request);
|
|
1560
1571
|
});
|
|
1561
1572
|
}
|
|
1562
|
-
|
|
1573
|
+
/** Offers the publisher's current tracks and data channels; one negotiation at a time. */
|
|
1574
|
+
negotiate() {
|
|
1575
|
+
const run = this.negotiateChain.then(() => this.runNegotiate());
|
|
1576
|
+
this.negotiateChain = run.catch(() => {
|
|
1577
|
+
});
|
|
1578
|
+
return run;
|
|
1579
|
+
}
|
|
1580
|
+
async runNegotiate() {
|
|
1563
1581
|
if (!this.publisher) {
|
|
1564
1582
|
throw new Error("Publisher PC not initialized");
|
|
1565
1583
|
}
|
|
@@ -1600,11 +1618,49 @@ var RTCEngine = class extends TypedEmitter {
|
|
|
1600
1618
|
};
|
|
1601
1619
|
});
|
|
1602
1620
|
}
|
|
1621
|
+
/**
|
|
1622
|
+
* Opens a publisher data channel before the first message. With subscriber-primary joins the
|
|
1623
|
+
* publisher connection is only negotiated when something is published, so a bot that has not
|
|
1624
|
+
* published a track negotiates it here (the offer carries the data channels). Throws when the
|
|
1625
|
+
* channel does not open in time.
|
|
1626
|
+
*/
|
|
1627
|
+
async ensureDataChannelOpen(kind, timeoutMs = 15e3) {
|
|
1628
|
+
const channel = kind === "reliable" ? this.reliableChannel : this.lossyChannel;
|
|
1629
|
+
if (!channel) throw new Error("Not connected");
|
|
1630
|
+
if (channel.readyState === "open") return;
|
|
1631
|
+
if (channel.readyState === "closing" || channel.readyState === "closed") {
|
|
1632
|
+
throw new Error(`Data channel ${kind} is closed`);
|
|
1633
|
+
}
|
|
1634
|
+
if (!this.publisherNegotiated) {
|
|
1635
|
+
if (!this.dataNegotiation) {
|
|
1636
|
+
log2.debug("Negotiating the publisher to open its data channels");
|
|
1637
|
+
this.dataNegotiation = this.negotiate().finally(() => {
|
|
1638
|
+
this.dataNegotiation = null;
|
|
1639
|
+
});
|
|
1640
|
+
}
|
|
1641
|
+
await this.dataNegotiation;
|
|
1642
|
+
}
|
|
1643
|
+
if (channel.readyState === "open") return;
|
|
1644
|
+
await new Promise((resolve, reject) => {
|
|
1645
|
+
const timer = setTimeout(() => {
|
|
1646
|
+
subscription.unSubscribe();
|
|
1647
|
+
reject(new Error(`Data channel ${kind} did not open within ${timeoutMs}ms`));
|
|
1648
|
+
}, timeoutMs);
|
|
1649
|
+
const subscription = channel.stateChanged.subscribe((state) => {
|
|
1650
|
+
if (state === "open" || state === "closed") {
|
|
1651
|
+
clearTimeout(timer);
|
|
1652
|
+
subscription.unSubscribe();
|
|
1653
|
+
if (state === "open") resolve();
|
|
1654
|
+
else reject(new Error(`Data channel ${kind} is closed`));
|
|
1655
|
+
}
|
|
1656
|
+
});
|
|
1657
|
+
});
|
|
1658
|
+
}
|
|
1659
|
+
/** Sends on an open publisher data channel; throws when it is not open. */
|
|
1603
1660
|
sendData(data, kind) {
|
|
1604
1661
|
const channel = kind === "reliable" ? this.reliableChannel : this.lossyChannel;
|
|
1605
1662
|
if (!channel || channel.readyState !== "open") {
|
|
1606
|
-
|
|
1607
|
-
return;
|
|
1663
|
+
throw new Error(`Data channel ${kind} not open`);
|
|
1608
1664
|
}
|
|
1609
1665
|
channel.send(Buffer.from(data));
|
|
1610
1666
|
}
|
|
@@ -1779,6 +1835,7 @@ var RTCEngine = class extends TypedEmitter {
|
|
|
1779
1835
|
);
|
|
1780
1836
|
log2.debug("Set publisher remote description");
|
|
1781
1837
|
this.flushPendingCandidates(0 /* PUBLISHER */);
|
|
1838
|
+
this.publisherNegotiated = true;
|
|
1782
1839
|
if (this._negotiateResolve) {
|
|
1783
1840
|
this._negotiateResolve();
|
|
1784
1841
|
this._negotiateResolve = null;
|
|
@@ -1789,8 +1846,7 @@ var RTCEngine = class extends TypedEmitter {
|
|
|
1789
1846
|
});
|
|
1790
1847
|
this.signal.on("trickle", async (trickle) => {
|
|
1791
1848
|
try {
|
|
1792
|
-
const
|
|
1793
|
-
const candidate = new import_werift.RTCIceCandidate(candidateInit);
|
|
1849
|
+
const candidate = parseTrickleCandidate(trickle.candidateInit);
|
|
1794
1850
|
const pc = trickle.target === 0 /* PUBLISHER */ ? this.publisher : this.subscriber;
|
|
1795
1851
|
if (!pc) return;
|
|
1796
1852
|
if (pc.remoteDescription) {
|
|
@@ -1893,33 +1949,12 @@ var AudioFrame = class _AudioFrame {
|
|
|
1893
1949
|
};
|
|
1894
1950
|
|
|
1895
1951
|
// src/audio/opus-encoder.ts
|
|
1952
|
+
var import_opusscript = __toESM(require("opusscript"));
|
|
1896
1953
|
var log3 = createLogger("OpusEncoder");
|
|
1897
1954
|
var OPUS_FRAME_DURATION_MS = 20;
|
|
1898
1955
|
var OPUS_SAMPLE_RATE = 48e3;
|
|
1899
1956
|
var OPUS_FRAME_SIZE = OPUS_SAMPLE_RATE * OPUS_FRAME_DURATION_MS / 1e3;
|
|
1900
1957
|
var VALID_OPUS_RATES = [8e3, 12e3, 16e3, 24e3, 48e3];
|
|
1901
|
-
var OpusEncoderClass = null;
|
|
1902
|
-
var opusLibraryName = "";
|
|
1903
|
-
function getOpusEncoder() {
|
|
1904
|
-
if (!OpusEncoderClass) {
|
|
1905
|
-
try {
|
|
1906
|
-
const opus = require("@discordjs/opus");
|
|
1907
|
-
OpusEncoderClass = opus.OpusEncoder;
|
|
1908
|
-
opusLibraryName = "@discordjs/opus (native)";
|
|
1909
|
-
} catch {
|
|
1910
|
-
try {
|
|
1911
|
-
const OpusScript = require("opusscript");
|
|
1912
|
-
OpusEncoderClass = OpusScript;
|
|
1913
|
-
opusLibraryName = "opusscript (WASM)";
|
|
1914
|
-
} catch {
|
|
1915
|
-
throw new Error(
|
|
1916
|
-
"No Opus library found. Install @discordjs/opus (recommended) or opusscript (WASM fallback)."
|
|
1917
|
-
);
|
|
1918
|
-
}
|
|
1919
|
-
}
|
|
1920
|
-
}
|
|
1921
|
-
return OpusEncoderClass;
|
|
1922
|
-
}
|
|
1923
1958
|
var OpusEncoder = class {
|
|
1924
1959
|
encoder;
|
|
1925
1960
|
channels;
|
|
@@ -1937,22 +1972,16 @@ var OpusEncoder = class {
|
|
|
1937
1972
|
}
|
|
1938
1973
|
this.channels = channels;
|
|
1939
1974
|
this.frameSize = sampleRate * OPUS_FRAME_DURATION_MS / 1e3;
|
|
1940
|
-
|
|
1941
|
-
this.encoder
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
log3.info(`Opus encoder: ${opusLibraryName}, ${sampleRate}Hz, ${channels}ch, ${bitrate / 1e3}kbps, frameSize=${this.frameSize}, DTX=on`);
|
|
1949
|
-
} catch (err) {
|
|
1950
|
-
log3.warn("Failed to enable DTX:", err);
|
|
1951
|
-
log3.info(`Opus encoder: ${opusLibraryName}, ${sampleRate}Hz, ${channels}ch, ${bitrate / 1e3}kbps, frameSize=${this.frameSize}`);
|
|
1952
|
-
}
|
|
1953
|
-
} else {
|
|
1954
|
-
log3.info(`Opus encoder: ${opusLibraryName}, ${sampleRate}Hz, ${channels}ch, ${bitrate / 1e3}kbps, frameSize=${this.frameSize}`);
|
|
1975
|
+
this.encoder = new import_opusscript.default(sampleRate, channels, import_opusscript.default.Application.AUDIO);
|
|
1976
|
+
this.encoder.setBitrate(bitrate);
|
|
1977
|
+
let dtx = false;
|
|
1978
|
+
try {
|
|
1979
|
+
this.encoder.encoderCTL(4016, 1);
|
|
1980
|
+
dtx = true;
|
|
1981
|
+
} catch (err) {
|
|
1982
|
+
log3.warn("Failed to enable DTX:", err);
|
|
1955
1983
|
}
|
|
1984
|
+
log3.info(`Opus encoder: opusscript, ${sampleRate}Hz, ${channels}ch, ${bitrate / 1e3}kbps, frameSize=${this.frameSize}${dtx ? ", DTX=on" : ""}`);
|
|
1956
1985
|
}
|
|
1957
1986
|
/**
|
|
1958
1987
|
* Encode a PCM16 frame to Opus.
|
|
@@ -1961,13 +1990,12 @@ var OpusEncoder = class {
|
|
|
1961
1990
|
*/
|
|
1962
1991
|
encode(pcm) {
|
|
1963
1992
|
const buf = pcm instanceof Int16Array ? Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength) : pcm;
|
|
1993
|
+
if (!this.encoder) throw new Error("Opus encoder destroyed");
|
|
1964
1994
|
return this.encoder.encode(buf, this.frameSize);
|
|
1965
1995
|
}
|
|
1966
|
-
/**
|
|
1996
|
+
/** Free the WASM encoder. */
|
|
1967
1997
|
destroy() {
|
|
1968
|
-
|
|
1969
|
-
this.encoder.delete();
|
|
1970
|
-
}
|
|
1998
|
+
this.encoder?.delete();
|
|
1971
1999
|
this.encoder = null;
|
|
1972
2000
|
}
|
|
1973
2001
|
};
|
|
@@ -2038,86 +2066,6 @@ var OpusDecoder = class {
|
|
|
2038
2066
|
}
|
|
2039
2067
|
};
|
|
2040
2068
|
|
|
2041
|
-
// src/audio/rtp-opus.ts
|
|
2042
|
-
var log5 = createLogger("RTP-Opus");
|
|
2043
|
-
var OPUS_PAYLOAD_TYPE = 111;
|
|
2044
|
-
var OPUS_TIMESTAMP_INCREMENT = OPUS_FRAME_SIZE;
|
|
2045
|
-
var OpusRtpPacketizer = class {
|
|
2046
|
-
sequenceNumber;
|
|
2047
|
-
timestamp;
|
|
2048
|
-
ssrc;
|
|
2049
|
-
payloadType;
|
|
2050
|
-
constructor(ssrc = 0, payloadType = OPUS_PAYLOAD_TYPE) {
|
|
2051
|
-
this.sequenceNumber = Math.floor(Math.random() * 65535);
|
|
2052
|
-
this.timestamp = Math.floor(Math.random() * 4294967295);
|
|
2053
|
-
this.ssrc = ssrc;
|
|
2054
|
-
this.payloadType = payloadType;
|
|
2055
|
-
}
|
|
2056
|
-
/**
|
|
2057
|
-
* Get the next RTP header values for an Opus frame.
|
|
2058
|
-
* Call this for each 20ms frame to get incrementing seq/ts.
|
|
2059
|
-
*/
|
|
2060
|
-
nextPacketInfo() {
|
|
2061
|
-
const info = {
|
|
2062
|
-
sequenceNumber: this.sequenceNumber & 65535,
|
|
2063
|
-
timestamp: this.timestamp >>> 0,
|
|
2064
|
-
ssrc: this.ssrc
|
|
2065
|
-
};
|
|
2066
|
-
this.sequenceNumber = this.sequenceNumber + 1 & 65535;
|
|
2067
|
-
this.timestamp = this.timestamp + OPUS_TIMESTAMP_INCREMENT >>> 0;
|
|
2068
|
-
return info;
|
|
2069
|
-
}
|
|
2070
|
-
/** Reset the packetizer state */
|
|
2071
|
-
reset() {
|
|
2072
|
-
this.sequenceNumber = Math.floor(Math.random() * 65535);
|
|
2073
|
-
this.timestamp = Math.floor(Math.random() * 4294967295);
|
|
2074
|
-
}
|
|
2075
|
-
};
|
|
2076
|
-
var OpusRtpDepacketizer = class {
|
|
2077
|
-
lastSequenceNumber = -1;
|
|
2078
|
-
lastTimestamp = -1;
|
|
2079
|
-
lostPackets = 0;
|
|
2080
|
-
/**
|
|
2081
|
-
* Process an incoming RTP packet containing an Opus frame.
|
|
2082
|
-
* Returns the Opus payload and metadata.
|
|
2083
|
-
*/
|
|
2084
|
-
processPacket(payload, sequenceNumber, timestamp) {
|
|
2085
|
-
let lost = 0;
|
|
2086
|
-
const isFirst = this.lastSequenceNumber === -1;
|
|
2087
|
-
if (!isFirst) {
|
|
2088
|
-
const expectedSeq = this.lastSequenceNumber + 1 & 65535;
|
|
2089
|
-
if (sequenceNumber !== expectedSeq) {
|
|
2090
|
-
if (sequenceNumber > this.lastSequenceNumber) {
|
|
2091
|
-
lost = sequenceNumber - this.lastSequenceNumber - 1;
|
|
2092
|
-
} else {
|
|
2093
|
-
lost = 65535 - this.lastSequenceNumber + sequenceNumber;
|
|
2094
|
-
}
|
|
2095
|
-
this.lostPackets += lost;
|
|
2096
|
-
if (lost > 0) {
|
|
2097
|
-
log5.debug(`Lost ${lost} packets (seq ${this.lastSequenceNumber} \u2192 ${sequenceNumber})`);
|
|
2098
|
-
}
|
|
2099
|
-
}
|
|
2100
|
-
}
|
|
2101
|
-
this.lastSequenceNumber = sequenceNumber;
|
|
2102
|
-
this.lastTimestamp = timestamp;
|
|
2103
|
-
return {
|
|
2104
|
-
opusFrame: payload,
|
|
2105
|
-
lost,
|
|
2106
|
-
isFirst
|
|
2107
|
-
};
|
|
2108
|
-
}
|
|
2109
|
-
/** Total packets lost since creation */
|
|
2110
|
-
get totalLost() {
|
|
2111
|
-
return this.lostPackets;
|
|
2112
|
-
}
|
|
2113
|
-
/** Reset depacketizer state */
|
|
2114
|
-
reset() {
|
|
2115
|
-
this.lastSequenceNumber = -1;
|
|
2116
|
-
this.lastTimestamp = -1;
|
|
2117
|
-
this.lostPackets = 0;
|
|
2118
|
-
}
|
|
2119
|
-
};
|
|
2120
|
-
|
|
2121
2069
|
// src/audio/resampler.ts
|
|
2122
2070
|
function downsample(input, fromRate, toRate, channels = 1) {
|
|
2123
2071
|
if (fromRate === toRate) {
|
|
@@ -2243,15 +2191,25 @@ var AsyncQueue = class {
|
|
|
2243
2191
|
};
|
|
2244
2192
|
|
|
2245
2193
|
// src/audio/audio-stream.ts
|
|
2246
|
-
var
|
|
2194
|
+
var log5 = createLogger("AudioStream");
|
|
2195
|
+
var SILENCE_GRACE_MS = 60;
|
|
2196
|
+
var MAX_TIMESTAMP_GAP_SAMPLES = OPUS_SAMPLE_RATE * 3;
|
|
2247
2197
|
var AudioStream = class {
|
|
2248
2198
|
decoder = null;
|
|
2249
|
-
depacketizer;
|
|
2250
2199
|
queue;
|
|
2251
2200
|
outputSampleRate;
|
|
2252
2201
|
outputChannels;
|
|
2253
2202
|
track = null;
|
|
2254
2203
|
_closed = false;
|
|
2204
|
+
silenceTimer = null;
|
|
2205
|
+
lastSequence = null;
|
|
2206
|
+
lastTimestamp = null;
|
|
2207
|
+
lastFrameSamples = 0;
|
|
2208
|
+
lastPacketAt = 0;
|
|
2209
|
+
concealedSamples = 0;
|
|
2210
|
+
currentSsrc = null;
|
|
2211
|
+
timestampSilenceSamples = 0;
|
|
2212
|
+
stalledSilenceSamples = 0;
|
|
2255
2213
|
/**
|
|
2256
2214
|
* @param track The remote audio track to stream from
|
|
2257
2215
|
* @param sampleRate Desired output sample rate (default: 16000 for STT)
|
|
@@ -2260,7 +2218,6 @@ var AudioStream = class {
|
|
|
2260
2218
|
constructor(track, sampleRate = 16e3, channels = 1) {
|
|
2261
2219
|
this.outputSampleRate = sampleRate;
|
|
2262
2220
|
this.outputChannels = channels;
|
|
2263
|
-
this.depacketizer = new OpusRtpDepacketizer();
|
|
2264
2221
|
this.queue = new AsyncQueue();
|
|
2265
2222
|
this.track = track;
|
|
2266
2223
|
this.start();
|
|
@@ -2272,13 +2229,15 @@ var AudioStream = class {
|
|
|
2272
2229
|
close() {
|
|
2273
2230
|
if (this._closed) return;
|
|
2274
2231
|
this._closed = true;
|
|
2232
|
+
if (this.silenceTimer) clearInterval(this.silenceTimer);
|
|
2233
|
+
this.silenceTimer = null;
|
|
2275
2234
|
this.queue.close();
|
|
2276
2235
|
if (this.decoder) {
|
|
2277
2236
|
this.decoder.destroy();
|
|
2278
2237
|
this.decoder = null;
|
|
2279
2238
|
}
|
|
2280
2239
|
this.track = null;
|
|
2281
|
-
|
|
2240
|
+
log5.info(`AudioStream closed: packets=${this.packetCount}, timestamp_silence_ms=${Math.round(this.timestampSilenceSamples / 48)}, stalled_silence_ms=${Math.round(this.stalledSilenceSamples / 48)}`);
|
|
2282
2241
|
}
|
|
2283
2242
|
[Symbol.asyncIterator]() {
|
|
2284
2243
|
return this.queue[Symbol.asyncIterator]();
|
|
@@ -2294,19 +2253,19 @@ var AudioStream = class {
|
|
|
2294
2253
|
this.packetCount++;
|
|
2295
2254
|
if (this.firstSsrc === null) {
|
|
2296
2255
|
this.firstSsrc = rtpPacket.header.ssrc;
|
|
2297
|
-
|
|
2256
|
+
log5.info(`AudioStream first RTP: ssrc=${this.firstSsrc}, trackUuid=${trackUuid}`);
|
|
2298
2257
|
} else if (rtpPacket.header.ssrc !== this.firstSsrc && this.packetCount < 20) {
|
|
2299
|
-
|
|
2258
|
+
log5.warn(`AudioStream SSRC changed: ${this.firstSsrc} \u2192 ${rtpPacket.header.ssrc}, trackUuid=${trackUuid}`);
|
|
2300
2259
|
}
|
|
2301
2260
|
try {
|
|
2302
2261
|
this.processRtpPacket(rtpPacket);
|
|
2303
2262
|
} catch (err) {
|
|
2304
|
-
|
|
2263
|
+
log5.error("Failed to process RTP packet", err);
|
|
2305
2264
|
}
|
|
2306
2265
|
});
|
|
2307
2266
|
this.track.onReceiveRtp.once(() => {
|
|
2308
2267
|
});
|
|
2309
|
-
|
|
2268
|
+
log5.debug(`AudioStream started, output: ${this.outputSampleRate}Hz ${this.outputChannels}ch, trackUuid=${trackUuid}`);
|
|
2310
2269
|
}
|
|
2311
2270
|
processRtpPacket(rtp) {
|
|
2312
2271
|
if (!rtp.payload || rtp.payload.length === 0) {
|
|
@@ -2314,31 +2273,66 @@ var AudioStream = class {
|
|
|
2314
2273
|
}
|
|
2315
2274
|
const payloadCopy = Buffer.from(rtp.payload);
|
|
2316
2275
|
if (payloadCopy.length > 1500) {
|
|
2317
|
-
|
|
2276
|
+
log5.warn(`Skipping oversized RTP payload: ${payloadCopy.length} bytes`);
|
|
2318
2277
|
return;
|
|
2319
2278
|
}
|
|
2320
|
-
if (
|
|
2321
|
-
this.
|
|
2279
|
+
if (this.currentSsrc !== rtp.header.ssrc) {
|
|
2280
|
+
this.currentSsrc = rtp.header.ssrc;
|
|
2281
|
+
this.lastSequence = this.lastTimestamp = null;
|
|
2282
|
+
this.concealedSamples = 0;
|
|
2283
|
+
this.decoder?.destroy();
|
|
2284
|
+
this.decoder = null;
|
|
2322
2285
|
}
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
rtp.header.timestamp
|
|
2327
|
-
);
|
|
2328
|
-
for (let i = 0; i < lost && i < 3; i++) {
|
|
2329
|
-
const silence = Buffer.alloc(OPUS_FRAME_SIZE * this.outputChannels * 2);
|
|
2330
|
-
this.emitFrame(silence);
|
|
2286
|
+
if (this.lastSequence !== null) {
|
|
2287
|
+
const delta = rtp.header.sequenceNumber - this.lastSequence & 65535;
|
|
2288
|
+
if (delta === 0 || delta >= 32768) return;
|
|
2331
2289
|
}
|
|
2332
|
-
if (
|
|
2333
|
-
|
|
2290
|
+
if (!this.decoder) {
|
|
2291
|
+
this.decoder = new OpusDecoder(OPUS_SAMPLE_RATE, this.outputChannels);
|
|
2334
2292
|
}
|
|
2335
2293
|
try {
|
|
2336
|
-
const opusCopy = Buffer.
|
|
2337
|
-
opusFrame.copy(opusCopy);
|
|
2294
|
+
const opusCopy = Buffer.from(payloadCopy);
|
|
2338
2295
|
const pcmBuffer = this.decoder.decode(opusCopy);
|
|
2296
|
+
if (this.lastTimestamp !== null) {
|
|
2297
|
+
const timestampDelta = rtp.header.timestamp - this.lastTimestamp | 0;
|
|
2298
|
+
const missing = timestampDelta - this.lastFrameSamples - this.concealedSamples;
|
|
2299
|
+
if (missing > 0 && missing <= MAX_TIMESTAMP_GAP_SAMPLES) {
|
|
2300
|
+
this.emitSilence(missing);
|
|
2301
|
+
this.timestampSilenceSamples += missing;
|
|
2302
|
+
} else if (missing > MAX_TIMESTAMP_GAP_SAMPLES || timestampDelta < 0) {
|
|
2303
|
+
log5.warn("Audio RTP timestamp discontinuity; rebasing receive timeline");
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
2339
2306
|
this.emitFrame(pcmBuffer);
|
|
2307
|
+
this.lastSequence = rtp.header.sequenceNumber;
|
|
2308
|
+
this.lastTimestamp = rtp.header.timestamp;
|
|
2309
|
+
this.lastFrameSamples = pcmBuffer.length / (2 * this.outputChannels);
|
|
2310
|
+
this.lastPacketAt = performance.now();
|
|
2311
|
+
this.concealedSamples = 0;
|
|
2312
|
+
if (!this.silenceTimer) {
|
|
2313
|
+
this.silenceTimer = setInterval(() => this.fillStalledInput(), 20);
|
|
2314
|
+
this.silenceTimer.unref();
|
|
2315
|
+
}
|
|
2340
2316
|
} catch (err) {
|
|
2341
|
-
|
|
2317
|
+
log5.error(`Opus decode failed (${payloadCopy.length} bytes)`, err);
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
fillStalledInput() {
|
|
2321
|
+
if (this._closed || this.lastTimestamp === null) return;
|
|
2322
|
+
const elapsedSamples = Math.floor((performance.now() - this.lastPacketAt - SILENCE_GRACE_MS) * OPUS_SAMPLE_RATE / 1e3);
|
|
2323
|
+
const missing = elapsedSamples - this.lastFrameSamples - this.concealedSamples;
|
|
2324
|
+
const samples = Math.min(OPUS_FRAME_SIZE * 5, Math.floor(missing / OPUS_FRAME_SIZE) * OPUS_FRAME_SIZE);
|
|
2325
|
+
if (samples > 0) {
|
|
2326
|
+
this.emitSilence(samples);
|
|
2327
|
+
this.concealedSamples += samples;
|
|
2328
|
+
this.stalledSilenceSamples += samples;
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2331
|
+
emitSilence(samples) {
|
|
2332
|
+
while (samples > 0) {
|
|
2333
|
+
const count = Math.min(samples, OPUS_FRAME_SIZE);
|
|
2334
|
+
this.emitFrame(Buffer.alloc(count * this.outputChannels * 2));
|
|
2335
|
+
samples -= count;
|
|
2342
2336
|
}
|
|
2343
2337
|
}
|
|
2344
2338
|
emitFrame(pcm48k) {
|
|
@@ -2356,6 +2350,42 @@ var AudioStream = class {
|
|
|
2356
2350
|
}
|
|
2357
2351
|
};
|
|
2358
2352
|
|
|
2353
|
+
// src/audio/rtp-opus.ts
|
|
2354
|
+
var log6 = createLogger("RTP-Opus");
|
|
2355
|
+
var OPUS_PAYLOAD_TYPE = 111;
|
|
2356
|
+
var OPUS_TIMESTAMP_INCREMENT = OPUS_FRAME_SIZE;
|
|
2357
|
+
var OpusRtpPacketizer = class {
|
|
2358
|
+
sequenceNumber;
|
|
2359
|
+
timestamp;
|
|
2360
|
+
ssrc;
|
|
2361
|
+
payloadType;
|
|
2362
|
+
constructor(ssrc = 0, payloadType = OPUS_PAYLOAD_TYPE) {
|
|
2363
|
+
this.sequenceNumber = Math.floor(Math.random() * 65535);
|
|
2364
|
+
this.timestamp = Math.floor(Math.random() * 4294967295);
|
|
2365
|
+
this.ssrc = ssrc;
|
|
2366
|
+
this.payloadType = payloadType;
|
|
2367
|
+
}
|
|
2368
|
+
/**
|
|
2369
|
+
* Get the next RTP header values for an Opus frame.
|
|
2370
|
+
* Call this for each 20ms frame to get incrementing seq/ts.
|
|
2371
|
+
*/
|
|
2372
|
+
nextPacketInfo() {
|
|
2373
|
+
const info = {
|
|
2374
|
+
sequenceNumber: this.sequenceNumber & 65535,
|
|
2375
|
+
timestamp: this.timestamp >>> 0,
|
|
2376
|
+
ssrc: this.ssrc
|
|
2377
|
+
};
|
|
2378
|
+
this.sequenceNumber = this.sequenceNumber + 1 & 65535;
|
|
2379
|
+
this.timestamp = this.timestamp + OPUS_TIMESTAMP_INCREMENT >>> 0;
|
|
2380
|
+
return info;
|
|
2381
|
+
}
|
|
2382
|
+
/** Reset the packetizer state */
|
|
2383
|
+
reset() {
|
|
2384
|
+
this.sequenceNumber = Math.floor(Math.random() * 65535);
|
|
2385
|
+
this.timestamp = Math.floor(Math.random() * 4294967295);
|
|
2386
|
+
}
|
|
2387
|
+
};
|
|
2388
|
+
|
|
2359
2389
|
// src/track.ts
|
|
2360
2390
|
var log7 = createLogger("Track");
|
|
2361
2391
|
var TrackPublication = class extends TypedEmitter {
|
|
@@ -2532,6 +2562,8 @@ var Participant = class extends TypedEmitter {
|
|
|
2532
2562
|
name;
|
|
2533
2563
|
metadata;
|
|
2534
2564
|
state;
|
|
2565
|
+
/** What the participant's token allows, as the node reports it. */
|
|
2566
|
+
permissions;
|
|
2535
2567
|
_trackPublications = /* @__PURE__ */ new Map();
|
|
2536
2568
|
constructor(sid, identity, name = "", metadata = "") {
|
|
2537
2569
|
super();
|
|
@@ -2552,6 +2584,7 @@ var Participant = class extends TypedEmitter {
|
|
|
2552
2584
|
this.name = info.name;
|
|
2553
2585
|
this.metadata = info.metadata;
|
|
2554
2586
|
this.state = info.state;
|
|
2587
|
+
if (info.permission) this.permissions = info.permission;
|
|
2555
2588
|
if (metadataChanged) {
|
|
2556
2589
|
this.emit("metadataChanged", info.metadata);
|
|
2557
2590
|
}
|
|
@@ -2627,10 +2660,16 @@ var LocalParticipant = class extends Participant {
|
|
|
2627
2660
|
/**
|
|
2628
2661
|
* Publish data to the room.
|
|
2629
2662
|
*
|
|
2663
|
+
* Works before any track is published: the first call opens the data channel. Rejects when the
|
|
2664
|
+
* participant may not publish data, or the channel does not open within `timeoutMs`.
|
|
2665
|
+
*
|
|
2630
2666
|
* @param data The data payload
|
|
2631
|
-
* @param options Delivery options (kind, destinations, topic)
|
|
2667
|
+
* @param options Delivery options (kind, destinations, topic, timeoutMs)
|
|
2632
2668
|
*/
|
|
2633
2669
|
async publishData(data, options = {}) {
|
|
2670
|
+
if (this.permissions && !this.permissions.canPublishData) {
|
|
2671
|
+
throw new Error("Participant lacks canPublishData");
|
|
2672
|
+
}
|
|
2634
2673
|
const kind = options.kind ?? 0 /* RELIABLE */;
|
|
2635
2674
|
const destinationSids = options.destinationSids ?? [];
|
|
2636
2675
|
const topic = options.topic;
|
|
@@ -2645,6 +2684,7 @@ var LocalParticipant = class extends Participant {
|
|
|
2645
2684
|
};
|
|
2646
2685
|
const encoded = DataPacket.encode(packet).finish();
|
|
2647
2686
|
const channelKind = kind === 0 /* RELIABLE */ ? "reliable" : "lossy";
|
|
2687
|
+
await this.engine.ensureDataChannelOpen(channelKind, options.timeoutMs);
|
|
2648
2688
|
this.engine.sendData(new Uint8Array(encoded), channelKind);
|
|
2649
2689
|
}
|
|
2650
2690
|
};
|
|
@@ -2859,6 +2899,7 @@ var Room = class extends TypedEmitter {
|
|
|
2859
2899
|
join.participant.name,
|
|
2860
2900
|
join.participant.metadata
|
|
2861
2901
|
);
|
|
2902
|
+
this.localParticipant.permissions = join.participant.permission;
|
|
2862
2903
|
}
|
|
2863
2904
|
for (const info of join.otherParticipants) {
|
|
2864
2905
|
this.getOrCreateParticipant(info);
|