@livekit/agents 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/CHANGELOG.md +40 -0
- package/dist/audio.d.ts +1 -4
- package/dist/audio.d.ts.map +1 -1
- package/dist/audio.js +30 -12
- package/dist/audio.js.map +1 -1
- package/dist/cli.d.ts +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +41 -17
- package/dist/cli.js.map +1 -1
- package/dist/generator.d.ts +5 -0
- package/dist/generator.d.ts.map +1 -1
- package/dist/generator.js +11 -0
- package/dist/generator.js.map +1 -1
- package/dist/http_server.d.ts +1 -0
- package/dist/http_server.d.ts.map +1 -1
- package/dist/http_server.js +13 -0
- package/dist/http_server.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/ipc/job_main.js +9 -1
- package/dist/ipc/job_main.js.map +1 -1
- package/dist/ipc/proc_pool.d.ts.map +1 -1
- package/dist/ipc/proc_pool.js +1 -0
- package/dist/ipc/proc_pool.js.map +1 -1
- package/dist/job.d.ts +1 -0
- package/dist/job.d.ts.map +1 -1
- package/dist/job.js +30 -1
- package/dist/job.js.map +1 -1
- package/dist/multimodal/agent_playout.d.ts +34 -0
- package/dist/multimodal/agent_playout.d.ts.map +1 -0
- package/dist/multimodal/agent_playout.js +221 -0
- package/dist/multimodal/agent_playout.js.map +1 -0
- package/dist/multimodal/index.d.ts +3 -0
- package/dist/multimodal/index.d.ts.map +1 -0
- package/dist/multimodal/index.js +6 -0
- package/dist/multimodal/index.js.map +1 -0
- package/dist/multimodal/multimodal_agent.d.ts +47 -0
- package/dist/multimodal/multimodal_agent.d.ts.map +1 -0
- package/dist/multimodal/multimodal_agent.js +329 -0
- package/dist/multimodal/multimodal_agent.js.map +1 -0
- package/dist/transcription.d.ts +22 -0
- package/dist/transcription.d.ts.map +1 -0
- package/dist/transcription.js +112 -0
- package/dist/transcription.js.map +1 -0
- package/dist/utils.d.ts +29 -1
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +117 -15
- package/dist/utils.js.map +1 -1
- package/dist/worker.d.ts +3 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +49 -9
- package/dist/worker.js.map +1 -1
- package/package.json +6 -4
- package/src/audio.ts +21 -20
- package/src/cli.ts +42 -17
- package/src/generator.ts +14 -0
- package/src/http_server.ts +6 -0
- package/src/index.ts +3 -1
- package/src/ipc/job_main.ts +9 -2
- package/src/ipc/proc_pool.ts +1 -0
- package/src/job.ts +37 -1
- package/src/multimodal/agent_playout.ts +254 -0
- package/src/multimodal/index.ts +5 -0
- package/src/multimodal/multimodal_agent.ts +426 -0
- package/src/transcription.ts +129 -0
- package/src/utils.ts +151 -12
- package/src/worker.ts +60 -14
- package/tsconfig.json +1 -1
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
|
2
|
+
if (kind === "m") throw new TypeError("Private method is not writable");
|
|
3
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
|
4
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
|
5
|
+
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
6
|
+
};
|
|
7
|
+
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
|
8
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
|
9
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
10
|
+
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
11
|
+
};
|
|
12
|
+
var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
13
|
+
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
14
|
+
var m = o[Symbol.asyncIterator], i;
|
|
15
|
+
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
|
|
16
|
+
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
|
|
17
|
+
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
18
|
+
};
|
|
19
|
+
var _MultimodalAgent_instances, _MultimodalAgent_participant, _MultimodalAgent_agentPublication, _MultimodalAgent_localTrackSid, _MultimodalAgent_localSource, _MultimodalAgent_agentPlayout, _MultimodalAgent_playingHandle, _MultimodalAgent_logger, _MultimodalAgent_session, _MultimodalAgent_fncCtx, _MultimodalAgent__started, _MultimodalAgent__pendingFunctionCalls, _MultimodalAgent__speaking, _MultimodalAgent_pendingFunctionCalls_get, _MultimodalAgent_pendingFunctionCalls_set, _MultimodalAgent_speaking_get, _MultimodalAgent_speaking_set, _MultimodalAgent_started_get, _MultimodalAgent_started_set, _MultimodalAgent_linkParticipant, _MultimodalAgent_subscribeToMicrophone, _MultimodalAgent_getLocalTrackSid, _MultimodalAgent_publishTranscription, _MultimodalAgent_updateState, _MultimodalAgent_setState;
|
|
20
|
+
import { AudioSource, AudioStream, LocalAudioTrack, RoomEvent, TrackPublishOptions, TrackSource, } from '@livekit/rtc-node';
|
|
21
|
+
import { EventEmitter } from 'events';
|
|
22
|
+
import { AudioByteStream } from '../audio.js';
|
|
23
|
+
import { log } from '../log.js';
|
|
24
|
+
import { BasicTranscriptionForwarder } from '../transcription.js';
|
|
25
|
+
import { findMicroTrackId } from '../utils.js';
|
|
26
|
+
import { AgentPlayout } from './agent_playout.js';
|
|
27
|
+
/**
|
|
28
|
+
* @internal
|
|
29
|
+
* @beta
|
|
30
|
+
*/
|
|
31
|
+
export class RealtimeSession extends EventEmitter {
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* @internal
|
|
35
|
+
* @beta
|
|
36
|
+
*/
|
|
37
|
+
export class RealtimeModel {
|
|
38
|
+
}
|
|
39
|
+
export const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';
|
|
40
|
+
/** @beta */
|
|
41
|
+
export class MultimodalAgent {
|
|
42
|
+
constructor({ model, fncCtx, }) {
|
|
43
|
+
_MultimodalAgent_instances.add(this);
|
|
44
|
+
this.room = null;
|
|
45
|
+
this.linkedParticipant = null;
|
|
46
|
+
this.subscribedTrack = null;
|
|
47
|
+
this.readMicroTask = null;
|
|
48
|
+
_MultimodalAgent_participant.set(this, null);
|
|
49
|
+
_MultimodalAgent_agentPublication.set(this, null);
|
|
50
|
+
_MultimodalAgent_localTrackSid.set(this, null);
|
|
51
|
+
_MultimodalAgent_localSource.set(this, null);
|
|
52
|
+
_MultimodalAgent_agentPlayout.set(this, null);
|
|
53
|
+
_MultimodalAgent_playingHandle.set(this, undefined);
|
|
54
|
+
_MultimodalAgent_logger.set(this, log());
|
|
55
|
+
_MultimodalAgent_session.set(this, null);
|
|
56
|
+
_MultimodalAgent_fncCtx.set(this, undefined);
|
|
57
|
+
_MultimodalAgent__started.set(this, false);
|
|
58
|
+
_MultimodalAgent__pendingFunctionCalls.set(this, new Set());
|
|
59
|
+
_MultimodalAgent__speaking.set(this, false);
|
|
60
|
+
this.model = model;
|
|
61
|
+
__classPrivateFieldSet(this, _MultimodalAgent_fncCtx, fncCtx, "f");
|
|
62
|
+
}
|
|
63
|
+
get fncCtx() {
|
|
64
|
+
return __classPrivateFieldGet(this, _MultimodalAgent_fncCtx, "f");
|
|
65
|
+
}
|
|
66
|
+
set fncCtx(ctx) {
|
|
67
|
+
__classPrivateFieldSet(this, _MultimodalAgent_fncCtx, ctx, "f");
|
|
68
|
+
if (__classPrivateFieldGet(this, _MultimodalAgent_session, "f")) {
|
|
69
|
+
__classPrivateFieldGet(this, _MultimodalAgent_session, "f").fncCtx = ctx;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
start(room, participant = null) {
|
|
73
|
+
return new Promise(async (resolve, reject) => {
|
|
74
|
+
var _a;
|
|
75
|
+
if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_started_get)) {
|
|
76
|
+
reject(new Error('MultimodalAgent already started'));
|
|
77
|
+
}
|
|
78
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
|
|
79
|
+
room.on(RoomEvent.ParticipantConnected, (participant) => {
|
|
80
|
+
if (!this.linkedParticipant) {
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant.identity);
|
|
84
|
+
});
|
|
85
|
+
this.room = room;
|
|
86
|
+
__classPrivateFieldSet(this, _MultimodalAgent_participant, participant, "f");
|
|
87
|
+
__classPrivateFieldSet(this, _MultimodalAgent_localSource, new AudioSource(this.model.sampleRate, this.model.numChannels), "f");
|
|
88
|
+
__classPrivateFieldSet(this, _MultimodalAgent_agentPlayout, new AgentPlayout(__classPrivateFieldGet(this, _MultimodalAgent_localSource, "f"), this.model.sampleRate, this.model.numChannels, this.model.inFrameSize, this.model.outFrameSize), "f");
|
|
89
|
+
const track = LocalAudioTrack.createAudioTrack('assistant_voice', __classPrivateFieldGet(this, _MultimodalAgent_localSource, "f"));
|
|
90
|
+
const options = new TrackPublishOptions();
|
|
91
|
+
options.source = TrackSource.SOURCE_MICROPHONE;
|
|
92
|
+
__classPrivateFieldSet(this, _MultimodalAgent_agentPublication, (await ((_a = room.localParticipant) === null || _a === void 0 ? void 0 : _a.publishTrack(track, options))) || null, "f");
|
|
93
|
+
if (!__classPrivateFieldGet(this, _MultimodalAgent_agentPublication, "f")) {
|
|
94
|
+
__classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Failed to publish track');
|
|
95
|
+
reject(new Error('Failed to publish track'));
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
await __classPrivateFieldGet(this, _MultimodalAgent_agentPublication, "f").waitForSubscription();
|
|
99
|
+
if (participant) {
|
|
100
|
+
if (typeof participant === 'string') {
|
|
101
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant);
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant.identity);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
// No participant specified, try to find the first participant in the room
|
|
109
|
+
for (const participant of room.remoteParticipants.values()) {
|
|
110
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant.identity);
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
__classPrivateFieldSet(this, _MultimodalAgent_session, this.model.session({ fncCtx: __classPrivateFieldGet(this, _MultimodalAgent_fncCtx, "f") }), "f");
|
|
115
|
+
__classPrivateFieldSet(this, _MultimodalAgent_instances, true, "a", _MultimodalAgent_started_set);
|
|
116
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
117
|
+
__classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('response_content_added', (message) => {
|
|
118
|
+
var _a;
|
|
119
|
+
// openai.realtime.RealtimeContent
|
|
120
|
+
const trFwd = new BasicTranscriptionForwarder(this.room, this.room.localParticipant.identity, __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_getLocalTrackSid).call(this), message.responseId);
|
|
121
|
+
const handle = (_a = __classPrivateFieldGet(this, _MultimodalAgent_agentPlayout, "f")) === null || _a === void 0 ? void 0 : _a.play(message.itemId, message.contentIndex, trFwd, message.textStream, message.audioStream);
|
|
122
|
+
if (handle) {
|
|
123
|
+
__classPrivateFieldSet(this, _MultimodalAgent_instances, true, "a", _MultimodalAgent_speaking_set);
|
|
124
|
+
handle.on('done', () => {
|
|
125
|
+
if (__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f") == handle) {
|
|
126
|
+
__classPrivateFieldSet(this, _MultimodalAgent_instances, false, "a", _MultimodalAgent_speaking_set);
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
__classPrivateFieldSet(this, _MultimodalAgent_playingHandle, handle, "f");
|
|
131
|
+
});
|
|
132
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
133
|
+
__classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('input_speech_committed', (ev) => {
|
|
134
|
+
var _a, _b;
|
|
135
|
+
// openai.realtime.InputSpeechCommittedEvent
|
|
136
|
+
const participantIdentity = (_a = this.linkedParticipant) === null || _a === void 0 ? void 0 : _a.identity;
|
|
137
|
+
const trackSid = (_b = this.subscribedTrack) === null || _b === void 0 ? void 0 : _b.sid;
|
|
138
|
+
if (participantIdentity && trackSid) {
|
|
139
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_publishTranscription).call(this, participantIdentity, trackSid, '…', false, ev.itemId);
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
__classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Participant or track not set');
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
146
|
+
__classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('input_speech_transcription_completed', (ev) => {
|
|
147
|
+
var _a, _b;
|
|
148
|
+
// openai.realtime.InputSpeechTranscriptionCompletedEvent
|
|
149
|
+
const transcription = ev.transcript;
|
|
150
|
+
const participantIdentity = (_a = this.linkedParticipant) === null || _a === void 0 ? void 0 : _a.identity;
|
|
151
|
+
const trackSid = (_b = this.subscribedTrack) === null || _b === void 0 ? void 0 : _b.sid;
|
|
152
|
+
if (participantIdentity && trackSid) {
|
|
153
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_publishTranscription).call(this, participantIdentity, trackSid, transcription, true, ev.itemId);
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
__classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Participant or track not set');
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
__classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('input_speech_started', (ev) => {
|
|
160
|
+
var _a, _b;
|
|
161
|
+
if (__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f") && !__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").done) {
|
|
162
|
+
__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").interrupt();
|
|
163
|
+
__classPrivateFieldGet(this, _MultimodalAgent_session, "f").defaultConversation.item.truncate(__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").itemId, __classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").contentIndex, Math.floor((__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").audioSamples / 24000) * 1000));
|
|
164
|
+
__classPrivateFieldSet(this, _MultimodalAgent_playingHandle, undefined, "f");
|
|
165
|
+
}
|
|
166
|
+
const participantIdentity = (_a = this.linkedParticipant) === null || _a === void 0 ? void 0 : _a.identity;
|
|
167
|
+
const trackSid = (_b = this.subscribedTrack) === null || _b === void 0 ? void 0 : _b.sid;
|
|
168
|
+
if (participantIdentity && trackSid) {
|
|
169
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_publishTranscription).call(this, participantIdentity, trackSid, '…', false, ev.itemId);
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
173
|
+
__classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('function_call_started', (ev) => {
|
|
174
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).add(ev.callId);
|
|
175
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
|
|
176
|
+
});
|
|
177
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
178
|
+
__classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('function_call_completed', (ev) => {
|
|
179
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).delete(ev.callId);
|
|
180
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
|
|
181
|
+
});
|
|
182
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
183
|
+
__classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('function_call_failed', (ev) => {
|
|
184
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).delete(ev.callId);
|
|
185
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
|
|
186
|
+
});
|
|
187
|
+
resolve(__classPrivateFieldGet(this, _MultimodalAgent_session, "f"));
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
_MultimodalAgent_participant = new WeakMap(), _MultimodalAgent_agentPublication = new WeakMap(), _MultimodalAgent_localTrackSid = new WeakMap(), _MultimodalAgent_localSource = new WeakMap(), _MultimodalAgent_agentPlayout = new WeakMap(), _MultimodalAgent_playingHandle = new WeakMap(), _MultimodalAgent_logger = new WeakMap(), _MultimodalAgent_session = new WeakMap(), _MultimodalAgent_fncCtx = new WeakMap(), _MultimodalAgent__started = new WeakMap(), _MultimodalAgent__pendingFunctionCalls = new WeakMap(), _MultimodalAgent__speaking = new WeakMap(), _MultimodalAgent_instances = new WeakSet(), _MultimodalAgent_pendingFunctionCalls_get = function _MultimodalAgent_pendingFunctionCalls_get() {
|
|
192
|
+
return __classPrivateFieldGet(this, _MultimodalAgent__pendingFunctionCalls, "f");
|
|
193
|
+
}, _MultimodalAgent_pendingFunctionCalls_set = function _MultimodalAgent_pendingFunctionCalls_set(calls) {
|
|
194
|
+
__classPrivateFieldSet(this, _MultimodalAgent__pendingFunctionCalls, calls, "f");
|
|
195
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
|
|
196
|
+
}, _MultimodalAgent_speaking_get = function _MultimodalAgent_speaking_get() {
|
|
197
|
+
return __classPrivateFieldGet(this, _MultimodalAgent__speaking, "f");
|
|
198
|
+
}, _MultimodalAgent_speaking_set = function _MultimodalAgent_speaking_set(isSpeaking) {
|
|
199
|
+
__classPrivateFieldSet(this, _MultimodalAgent__speaking, isSpeaking, "f");
|
|
200
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
|
|
201
|
+
}, _MultimodalAgent_started_get = function _MultimodalAgent_started_get() {
|
|
202
|
+
return __classPrivateFieldGet(this, _MultimodalAgent__started, "f");
|
|
203
|
+
}, _MultimodalAgent_started_set = function _MultimodalAgent_started_set(started) {
|
|
204
|
+
__classPrivateFieldSet(this, _MultimodalAgent__started, started, "f");
|
|
205
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
|
|
206
|
+
}, _MultimodalAgent_linkParticipant = function _MultimodalAgent_linkParticipant(participantIdentity) {
|
|
207
|
+
if (!this.room) {
|
|
208
|
+
__classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Room is not set');
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;
|
|
212
|
+
if (!this.linkedParticipant) {
|
|
213
|
+
__classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error(`Participant with identity ${participantIdentity} not found`);
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
if (this.linkedParticipant.trackPublications.size > 0) {
|
|
217
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_subscribeToMicrophone).call(this);
|
|
218
|
+
}
|
|
219
|
+
else {
|
|
220
|
+
this.room.on(RoomEvent.TrackPublished, () => {
|
|
221
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_subscribeToMicrophone).call(this);
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
}, _MultimodalAgent_subscribeToMicrophone = function _MultimodalAgent_subscribeToMicrophone() {
|
|
225
|
+
const readAudioStreamTask = async (audioStream) => {
|
|
226
|
+
var _a, e_1, _b, _c;
|
|
227
|
+
const bstream = new AudioByteStream(this.model.sampleRate, this.model.numChannels, this.model.inFrameSize);
|
|
228
|
+
try {
|
|
229
|
+
for (var _d = true, audioStream_1 = __asyncValues(audioStream), audioStream_1_1; audioStream_1_1 = await audioStream_1.next(), _a = audioStream_1_1.done, !_a; _d = true) {
|
|
230
|
+
_c = audioStream_1_1.value;
|
|
231
|
+
_d = false;
|
|
232
|
+
const frame = _c;
|
|
233
|
+
const audioData = frame.data;
|
|
234
|
+
for (const frame of bstream.write(audioData.buffer)) {
|
|
235
|
+
__classPrivateFieldGet(this, _MultimodalAgent_session, "f").inputAudioBuffer.append(frame);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
240
|
+
finally {
|
|
241
|
+
try {
|
|
242
|
+
if (!_d && !_a && (_b = audioStream_1.return)) await _b.call(audioStream_1);
|
|
243
|
+
}
|
|
244
|
+
finally { if (e_1) throw e_1.error; }
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
if (!this.linkedParticipant) {
|
|
248
|
+
__classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Participant is not set');
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
for (const publication of this.linkedParticipant.trackPublications.values()) {
|
|
252
|
+
if (publication.source !== TrackSource.SOURCE_MICROPHONE) {
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
if (!publication.subscribed) {
|
|
256
|
+
publication.setSubscribed(true);
|
|
257
|
+
}
|
|
258
|
+
const track = publication.track;
|
|
259
|
+
if (track && track !== this.subscribedTrack) {
|
|
260
|
+
this.subscribedTrack = track;
|
|
261
|
+
if (this.readMicroTask) {
|
|
262
|
+
this.readMicroTask.cancel();
|
|
263
|
+
}
|
|
264
|
+
let cancel;
|
|
265
|
+
this.readMicroTask = {
|
|
266
|
+
promise: new Promise((resolve, reject) => {
|
|
267
|
+
cancel = () => {
|
|
268
|
+
reject(new Error('Task cancelled'));
|
|
269
|
+
};
|
|
270
|
+
readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
|
|
271
|
+
.then(resolve)
|
|
272
|
+
.catch(reject);
|
|
273
|
+
}),
|
|
274
|
+
cancel: () => cancel(),
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}, _MultimodalAgent_getLocalTrackSid = function _MultimodalAgent_getLocalTrackSid() {
|
|
279
|
+
var _a;
|
|
280
|
+
if (!__classPrivateFieldGet(this, _MultimodalAgent_localTrackSid, "f") && this.room && this.room.localParticipant) {
|
|
281
|
+
__classPrivateFieldSet(this, _MultimodalAgent_localTrackSid, findMicroTrackId(this.room, (_a = this.room.localParticipant) === null || _a === void 0 ? void 0 : _a.identity), "f");
|
|
282
|
+
}
|
|
283
|
+
return __classPrivateFieldGet(this, _MultimodalAgent_localTrackSid, "f");
|
|
284
|
+
}, _MultimodalAgent_publishTranscription = function _MultimodalAgent_publishTranscription(participantIdentity, trackSid, text, isFinal, id) {
|
|
285
|
+
var _a;
|
|
286
|
+
__classPrivateFieldGet(this, _MultimodalAgent_logger, "f").debug(`Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`);
|
|
287
|
+
if (!((_a = this.room) === null || _a === void 0 ? void 0 : _a.localParticipant)) {
|
|
288
|
+
__classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Room or local participant not set');
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
this.room.localParticipant.publishTranscription({
|
|
292
|
+
participantIdentity,
|
|
293
|
+
trackSid,
|
|
294
|
+
segments: [
|
|
295
|
+
{
|
|
296
|
+
text,
|
|
297
|
+
final: isFinal,
|
|
298
|
+
id,
|
|
299
|
+
startTime: BigInt(0),
|
|
300
|
+
endTime: BigInt(0),
|
|
301
|
+
language: '',
|
|
302
|
+
},
|
|
303
|
+
],
|
|
304
|
+
});
|
|
305
|
+
}, _MultimodalAgent_updateState = function _MultimodalAgent_updateState() {
|
|
306
|
+
let newState = 'initializing';
|
|
307
|
+
if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).size > 0) {
|
|
308
|
+
newState = 'thinking';
|
|
309
|
+
}
|
|
310
|
+
else if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_speaking_get)) {
|
|
311
|
+
newState = 'speaking';
|
|
312
|
+
}
|
|
313
|
+
else if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_started_get)) {
|
|
314
|
+
newState = 'listening';
|
|
315
|
+
}
|
|
316
|
+
__classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_setState).call(this, newState);
|
|
317
|
+
}, _MultimodalAgent_setState = function _MultimodalAgent_setState(state) {
|
|
318
|
+
var _a;
|
|
319
|
+
if (((_a = this.room) === null || _a === void 0 ? void 0 : _a.isConnected) && this.room.localParticipant) {
|
|
320
|
+
const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];
|
|
321
|
+
if (currentState !== state) {
|
|
322
|
+
this.room.localParticipant.setAttributes({
|
|
323
|
+
[AGENT_STATE_ATTRIBUTE]: state,
|
|
324
|
+
});
|
|
325
|
+
__classPrivateFieldGet(this, _MultimodalAgent_logger, "f").debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
};
|
|
329
|
+
//# sourceMappingURL=multimodal_agent.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multimodal_agent.js","sourceRoot":"","sources":["../../src/multimodal/multimodal_agent.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AASA,OAAO,EACL,WAAW,EACX,WAAW,EACX,eAAe,EACf,SAAS,EACT,mBAAmB,EACnB,WAAW,GACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAChC,OAAO,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAsB,MAAM,oBAAoB,CAAC;AAEtE;;;GAGG;AACH,MAAM,OAAgB,eAAgB,SAAQ,YAAY;CAMzD;AAED;;;GAGG;AACH,MAAM,OAAgB,aAAa;CAQlC;AAGD,MAAM,CAAC,MAAM,qBAAqB,GAAG,gBAAgB,CAAC;AAEtD,YAAY;AACZ,MAAM,OAAO,eAAe;IAO1B,YAAY,EACV,KAAK,EACL,MAAM,GAIP;;QAXD,SAAI,GAAgB,IAAI,CAAC;QACzB,sBAAiB,GAA6B,IAAI,CAAC;QACnD,oBAAe,GAA4B,IAAI,CAAC;QAChD,kBAAa,GAA0D,IAAI,CAAC;QAa5E,uCAAkD,IAAI,EAAC;QACvD,4CAAkD,IAAI,EAAC;QACvD,yCAAgC,IAAI,EAAC;QACrC,uCAAmC,IAAI,EAAC;QACxC,wCAAqC,IAAI,EAAC;QAC1C,yCAA4C,SAAS,EAAC;QACtD,kCAAU,GAAG,EAAE,EAAC;QAChB,mCAAmC,IAAI,EAAC;QACxC,kCAA2C,SAAS,EAAC;QAErD,oCAAqB,KAAK,EAAC;QAC3B,iDAAsC,IAAI,GAAG,EAAE,EAAC;QAChD,qCAAsB,KAAK,EAAC;QAhB1B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,uBAAA,IAAI,2BAAW,MAAM,MAAA,CAAC;IACxB,CAAC;IAgBD,IAAI,MAAM;QACR,OAAO,uBAAA,IAAI,+BAAQ,CAAC;IACtB,CAAC;IAED,IAAI,MAAM,CAAC,GAAoC;QAC7C,uBAAA,IAAI,2BAAW,GAAG,MAAA,CAAC;QACnB,IAAI,uBAAA,IAAI,gCAAS,EAAE,CAAC;YAClB,uBAAA,IAAI,gCAAS,CAAC,MAAM,GAAG,GAAG,CAAC;QAC7B,CAAC;IACH,CAAC;IA6BD,KAAK,CACH,IAAU,EACV,cAAiD,IAAI;QAErD,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;;YAC3C,IAAI,uBAAA,IAAI,gEAAS,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAC;YACvD,CAAC;YACD,uBAAA,IAAI,gEAAa,MAAjB,IAAI,CAAe,CAAC;YAEpB,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,oBAAoB,EAAE,CAAC,WAA8B,EAAE,EAAE;gBACzE,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAC5B,OAAO;gBACT,CAAC;gBAED,uBAAA,IAAI,oEAAiB,MAArB,IAAI,EAAkB,WAAW,CAAC,QAAQ,CAAC,CAAC;YAC9C,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,uBAAA,IAAI,gCAAgB,WAAW,MAAA,CAAC;YAEhC,uBAAA,IAAI,gCAAgB,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAA,CAAC;YACnF,uBAAA,IAAI,iCAAiB,IAAI,YAAY,CACnC,uBAAA,IAAI,oCAAa,EACjB,IAAI,CAAC,KAAK,CAAC,UAAU,EACrB,IAAI,CAAC,KAAK,CAAC,WAAW,EACtB,IAAI,CAAC,KAAK,CAAC,WAAW,EACtB,IAAI,CAAC,KAAK,CAAC,YAAY,CACxB,MAAA,CAAC;YACF,MAAM,KAAK,GAAG,eAAe,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,uBAAA,IAAI,oCAAa,CAAC,CAAC;YACrF,MAAM,OAAO,GAAG,IAAI,mBAAmB,EAAE,CAAC;YAC1C,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC,iBAAiB,CAAC;YAC/C,uBAAA,IAAI,qCAAqB,CAAC,MAAM,CAAA,MAAA,IAAI,CAAC,gBAAgB,0CAAE,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA,CAAC,IAAI,IAAI,MAAA,CAAC;YAC7F,IAAI,CAAC,uBAAA,IAAI,yCAAkB,EAAE,CAAC;gBAC5B,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;gBAC9C,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC,CAAC;gBAC7C,OAAO;YACT,CAAC;YAED,MAAM,uBAAA,IAAI,yCAAkB,CAAC,mBAAmB,EAAE,CAAC;YAEnD,IAAI,WAAW,EAAE,CAAC;gBAChB,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;oBACpC,uBAAA,IAAI,oEAAiB,MAArB,IAAI,EAAkB,WAAW,CAAC,CAAC;gBACrC,CAAC;qBAAM,CAAC;oBACN,uBAAA,IAAI,oEAAiB,MAArB,IAAI,EAAkB,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,0EAA0E;gBAC1E,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,EAAE,CAAC;oBAC3D,uBAAA,IAAI,oEAAiB,MAArB,IAAI,EAAkB,WAAW,CAAC,QAAQ,CAAC,CAAC;oBAC5C,MAAM;gBACR,CAAC;YACH,CAAC;YAED,uBAAA,IAAI,4BAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,uBAAA,IAAI,+BAAQ,EAAE,CAAC,MAAA,CAAC;YAC7D,uBAAA,IAAI,8BAAY,IAAI,oCAAA,CAAC;YAErB,8DAA8D;YAC9D,uBAAA,IAAI,gCAAS,CAAC,EAAE,CAAC,wBAAwB,EAAE,CAAC,OAAY,EAAE,EAAE;;gBAC1D,kCAAkC;gBAClC,MAAM,KAAK,GAAG,IAAI,2BAA2B,CAC3C,IAAI,CAAC,IAAK,EACV,IAAI,CAAC,IAAK,CAAC,gBAAiB,CAAC,QAAQ,EACrC,uBAAA,IAAI,qEAAkB,MAAtB,IAAI,CAAqB,EACzB,OAAO,CAAC,UAAU,CACnB,CAAC;gBAEF,MAAM,MAAM,GAAG,MAAA,uBAAA,IAAI,qCAAc,0CAAE,IAAI,CACrC,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,YAAY,EACpB,KAAK,EACL,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,WAAW,CACpB,CAAC;gBACF,IAAI,MAAM,EAAE,CAAC;oBACX,uBAAA,IAAI,8BAAa,IAAI,qCAAA,CAAC;oBACtB,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;wBACrB,IAAI,uBAAA,IAAI,sCAAe,IAAI,MAAM,EAAE,CAAC;4BAClC,uBAAA,IAAI,8BAAa,KAAK,qCAAA,CAAC;wBACzB,CAAC;oBACH,CAAC,CAAC,CAAC;gBACL,CAAC;gBACD,uBAAA,IAAI,kCAAkB,MAAM,MAAA,CAAC;YAC/B,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,uBAAA,IAAI,gCAAS,CAAC,EAAE,CAAC,wBAAwB,EAAE,CAAC,EAAO,EAAE,EAAE;;gBACrD,4CAA4C;gBAC5C,MAAM,mBAAmB,GAAG,MAAA,IAAI,CAAC,iBAAiB,0CAAE,QAAQ,CAAC;gBAC7D,MAAM,QAAQ,GAAG,MAAA,IAAI,CAAC,eAAe,0CAAE,GAAG,CAAC;gBAC3C,IAAI,mBAAmB,IAAI,QAAQ,EAAE,CAAC;oBACpC,uBAAA,IAAI,yEAAsB,MAA1B,IAAI,EAAuB,mBAAmB,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;gBACnF,CAAC;qBAAM,CAAC;oBACN,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;gBACrD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,uBAAA,IAAI,gCAAS,CAAC,EAAE,CAAC,sCAAsC,EAAE,CAAC,EAAO,EAAE,EAAE;;gBACnE,yDAAyD;gBACzD,MAAM,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC;gBACpC,MAAM,mBAAmB,GAAG,MAAA,IAAI,CAAC,iBAAiB,0CAAE,QAAQ,CAAC;gBAC7D,MAAM,QAAQ,GAAG,MAAA,IAAI,CAAC,eAAe,0CAAE,GAAG,CAAC;gBAC3C,IAAI,mBAAmB,IAAI,QAAQ,EAAE,CAAC;oBACpC,uBAAA,IAAI,yEAAsB,MAA1B,IAAI,EAAuB,mBAAmB,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;gBAC5F,CAAC;qBAAM,CAAC;oBACN,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;gBACrD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,uBAAA,IAAI,gCAAS,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,EAAO,EAAE,EAAE;;gBACnD,IAAI,uBAAA,IAAI,sCAAe,IAAI,CAAC,uBAAA,IAAI,sCAAe,CAAC,IAAI,EAAE,CAAC;oBACrD,uBAAA,IAAI,sCAAe,CAAC,SAAS,EAAE,CAAC;oBAEhC,uBAAA,IAAI,gCAAU,CAAC,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAC9C,uBAAA,IAAI,sCAAe,CAAC,MAAM,EAC1B,uBAAA,IAAI,sCAAe,CAAC,YAAY,EAChC,IAAI,CAAC,KAAK,CAAC,CAAC,uBAAA,IAAI,sCAAe,CAAC,YAAY,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,CAC9D,CAAC;oBAEF,uBAAA,IAAI,kCAAkB,SAAS,MAAA,CAAC;gBAClC,CAAC;gBAED,MAAM,mBAAmB,GAAG,MAAA,IAAI,CAAC,iBAAiB,0CAAE,QAAQ,CAAC;gBAC7D,MAAM,QAAQ,GAAG,MAAA,IAAI,CAAC,eAAe,0CAAE,GAAG,CAAC;gBAC3C,IAAI,mBAAmB,IAAI,QAAQ,EAAE,CAAC;oBACpC,uBAAA,IAAI,yEAAsB,MAA1B,IAAI,EAAuB,mBAAmB,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;gBACnF,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,uBAAA,IAAI,gCAAS,CAAC,EAAE,CAAC,uBAAuB,EAAE,CAAC,EAAO,EAAE,EAAE;gBACpD,uBAAA,IAAI,6EAAsB,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAC1C,uBAAA,IAAI,gEAAa,MAAjB,IAAI,CAAe,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,uBAAA,IAAI,gCAAS,CAAC,EAAE,CAAC,yBAAyB,EAAE,CAAC,EAAO,EAAE,EAAE;gBACtD,uBAAA,IAAI,6EAAsB,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAC7C,uBAAA,IAAI,gEAAa,MAAjB,IAAI,CAAe,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,uBAAA,IAAI,gCAAS,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,EAAO,EAAE,EAAE;gBACnD,uBAAA,IAAI,6EAAsB,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBAC7C,uBAAA,IAAI,gEAAa,MAAjB,IAAI,CAAe,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,uBAAA,IAAI,gCAAS,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC;CAqJF;;IAtUG,OAAO,uBAAA,IAAI,8CAAuB,CAAC;AACrC,CAAC,iGAEyB,KAAkB;IAC1C,uBAAA,IAAI,0CAA0B,KAAK,MAAA,CAAC;IACpC,uBAAA,IAAI,gEAAa,MAAjB,IAAI,CAAe,CAAC;AACtB,CAAC;IAGC,OAAO,uBAAA,IAAI,kCAAW,CAAC;AACzB,CAAC,yEAEa,UAAmB;IAC/B,uBAAA,IAAI,8BAAc,UAAU,MAAA,CAAC;IAC7B,uBAAA,IAAI,gEAAa,MAAjB,IAAI,CAAe,CAAC;AACtB,CAAC;IAGC,OAAO,uBAAA,IAAI,iCAAU,CAAC;AACxB,CAAC,uEAEY,OAAgB;IAC3B,uBAAA,IAAI,6BAAa,OAAO,MAAA,CAAC;IACzB,uBAAA,IAAI,gEAAa,MAAjB,IAAI,CAAe,CAAC;AACtB,CAAC,+EAkKgB,mBAA2B;IAC1C,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACf,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACtC,OAAO;IACT,CAAC;IAED,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,IAAI,CAAC;IACvF,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC5B,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAAC,6BAA6B,mBAAmB,YAAY,CAAC,CAAC;QACjF,OAAO;IACT,CAAC;IAED,IAAI,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACtD,uBAAA,IAAI,0EAAuB,MAA3B,IAAI,CAAyB,CAAC;IAChC,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,cAAc,EAAE,GAAG,EAAE;YAC1C,uBAAA,IAAI,0EAAuB,MAA3B,IAAI,CAAyB,CAAC;QAChC,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;IAGC,MAAM,mBAAmB,GAAG,KAAK,EAAE,WAAwB,EAAE,EAAE;;QAC7D,MAAM,OAAO,GAAG,IAAI,eAAe,CACjC,IAAI,CAAC,KAAK,CAAC,UAAU,EACrB,IAAI,CAAC,KAAK,CAAC,WAAW,EACtB,IAAI,CAAC,KAAK,CAAC,WAAW,CACvB,CAAC;;YAEF,KAA0B,eAAA,gBAAA,cAAA,WAAW,CAAA,iBAAA,2FAAE,CAAC;gBAAd,2BAAW;gBAAX,WAAW;gBAA1B,MAAM,KAAK,KAAA,CAAA;gBACpB,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC;gBAC7B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;oBACpD,uBAAA,IAAI,gCAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;;;;;;;;;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC5B,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;QAC5E,IAAI,WAAW,CAAC,MAAM,KAAK,WAAW,CAAC,iBAAiB,EAAE,CAAC;YACzD,SAAS;QACX,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;YAC5B,WAAW,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC;QAEhC,IAAI,KAAK,IAAI,KAAK,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;YAC5C,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;YAE7B,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvB,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC9B,CAAC;YAED,IAAI,MAAkB,CAAC;YACvB,IAAI,CAAC,aAAa,GAAG;gBACnB,OAAO,EAAE,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBAC7C,MAAM,GAAG,GAAG,EAAE;wBACZ,MAAM,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;oBACtC,CAAC,CAAC;oBACF,mBAAmB,CACjB,IAAI,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CACtE;yBACE,IAAI,CAAC,OAAO,CAAC;yBACb,KAAK,CAAC,MAAM,CAAC,CAAC;gBACnB,CAAC,CAAC;gBACF,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE;aACvB,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;;IAGC,IAAI,CAAC,uBAAA,IAAI,sCAAe,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACpE,uBAAA,IAAI,kCAAkB,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,MAAA,IAAI,CAAC,IAAI,CAAC,gBAAgB,0CAAE,QAAQ,CAAC,MAAA,CAAC;IAC1F,CAAC;IACD,OAAO,uBAAA,IAAI,sCAAe,CAAC;AAC7B,CAAC,yFAGC,mBAA2B,EAC3B,QAAgB,EAChB,IAAY,EACZ,OAAgB,EAChB,EAAU;;IAEV,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAChB,4BAA4B,mBAAmB,IAAI,QAAQ,IAAI,IAAI,IAAI,OAAO,IAAI,EAAE,EAAE,CACvF,CAAC;IACF,IAAI,CAAC,CAAA,MAAA,IAAI,CAAC,IAAI,0CAAE,gBAAgB,CAAA,EAAE,CAAC;QACjC,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACxD,OAAO;IACT,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC;QAC9C,mBAAmB;QACnB,QAAQ;QACR,QAAQ,EAAE;YACR;gBACE,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,EAAE;gBACF,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;gBACpB,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;gBAClB,QAAQ,EAAE,EAAE;aACb;SACF;KACF,CAAC,CAAC;AACL,CAAC;IAGC,IAAI,QAAQ,GAAe,cAAc,CAAC;IAC1C,IAAI,uBAAA,IAAI,6EAAsB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACxC,QAAQ,GAAG,UAAU,CAAC;IACxB,CAAC;SAAM,IAAI,uBAAA,IAAI,iEAAU,EAAE,CAAC;QAC1B,QAAQ,GAAG,UAAU,CAAC;IACxB,CAAC;SAAM,IAAI,uBAAA,IAAI,gEAAS,EAAE,CAAC;QACzB,QAAQ,GAAG,WAAW,CAAC;IACzB,CAAC;IAED,uBAAA,IAAI,6DAAU,MAAd,IAAI,EAAW,QAAQ,CAAC,CAAC;AAC3B,CAAC,iEAES,KAAiB;;IACzB,IAAI,CAAA,MAAA,IAAI,CAAC,IAAI,0CAAE,WAAW,KAAI,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACzD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QAClF,IAAI,YAAY,KAAK,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;gBACvC,CAAC,qBAAqB,CAAC,EAAE,KAAK;aAC/B,CAAC,CAAC;YACH,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAAC,GAAG,qBAAqB,KAAK,YAAY,MAAM,KAAK,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { AudioFrame, Room } from '@livekit/rtc-node';
|
|
2
|
+
export interface TranscriptionForwarder {
|
|
3
|
+
start(): void;
|
|
4
|
+
pushAudio(frame: AudioFrame): void;
|
|
5
|
+
pushText(text: string): void;
|
|
6
|
+
markTextComplete(): void;
|
|
7
|
+
markAudioComplete(): void;
|
|
8
|
+
close(interrupt: boolean): Promise<void>;
|
|
9
|
+
currentCharacterIndex: number;
|
|
10
|
+
}
|
|
11
|
+
export declare class BasicTranscriptionForwarder implements TranscriptionForwarder {
|
|
12
|
+
#private;
|
|
13
|
+
currentCharacterIndex: number;
|
|
14
|
+
constructor(room: Room, participantIdentity: string, trackSid: string, messageId: string);
|
|
15
|
+
start(): void;
|
|
16
|
+
pushAudio(frame: AudioFrame): void;
|
|
17
|
+
pushText(text: string): void;
|
|
18
|
+
markTextComplete(): void;
|
|
19
|
+
markAudioComplete(): void;
|
|
20
|
+
close(interrupt: boolean): Promise<void>;
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=transcription.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transcription.d.ts","sourceRoot":"","sources":["../src/transcription.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAG1D,MAAM,WAAW,sBAAsB;IACrC,KAAK,IAAI,IAAI,CAAC;IACd,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,IAAI,IAAI,CAAC;IACzB,iBAAiB,IAAI,IAAI,CAAC;IAC1B,KAAK,CAAC,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,qBAAa,2BAA4B,YAAW,sBAAsB;;IAYxE,qBAAqB,EAAE,MAAM,CAAK;gBAEtB,IAAI,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IAOxF,KAAK,IAAI,IAAI;IAUb,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAIlC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAO5B,gBAAgB,IAAI,IAAI;IAKxB,iBAAiB,IAAI,IAAI;IAwDnB,KAAK,CAAC,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;CAS/C"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
|
2
|
+
if (kind === "m") throw new TypeError("Private method is not writable");
|
|
3
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
|
4
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
|
5
|
+
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
6
|
+
};
|
|
7
|
+
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
|
8
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
|
9
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
10
|
+
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
11
|
+
};
|
|
12
|
+
var _BasicTranscriptionForwarder_instances, _BasicTranscriptionForwarder_room, _BasicTranscriptionForwarder_participantIdentity, _BasicTranscriptionForwarder_trackSid, _BasicTranscriptionForwarder_currentText, _BasicTranscriptionForwarder_totalAudioDuration, _BasicTranscriptionForwarder_currentPlayoutTime, _BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND, _BasicTranscriptionForwarder_charsPerSecond, _BasicTranscriptionForwarder_messageId, _BasicTranscriptionForwarder_isRunning, _BasicTranscriptionForwarder_logger, _BasicTranscriptionForwarder_textIsComplete, _BasicTranscriptionForwarder_audioIsComplete, _BasicTranscriptionForwarder_adjustTimingIfBothFinished, _BasicTranscriptionForwarder_computeSleepInterval, _BasicTranscriptionForwarder_startPublishingLoop, _BasicTranscriptionForwarder_publishTranscription;
|
|
13
|
+
import { log } from './log.js';
|
|
14
|
+
export class BasicTranscriptionForwarder {
|
|
15
|
+
constructor(room, participantIdentity, trackSid, messageId) {
|
|
16
|
+
_BasicTranscriptionForwarder_instances.add(this);
|
|
17
|
+
_BasicTranscriptionForwarder_room.set(this, void 0);
|
|
18
|
+
_BasicTranscriptionForwarder_participantIdentity.set(this, void 0);
|
|
19
|
+
_BasicTranscriptionForwarder_trackSid.set(this, void 0);
|
|
20
|
+
_BasicTranscriptionForwarder_currentText.set(this, '');
|
|
21
|
+
_BasicTranscriptionForwarder_totalAudioDuration.set(this, 0);
|
|
22
|
+
_BasicTranscriptionForwarder_currentPlayoutTime.set(this, 0);
|
|
23
|
+
_BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND.set(this, 16);
|
|
24
|
+
_BasicTranscriptionForwarder_charsPerSecond.set(this, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND, "f"));
|
|
25
|
+
_BasicTranscriptionForwarder_messageId.set(this, void 0);
|
|
26
|
+
_BasicTranscriptionForwarder_isRunning.set(this, false);
|
|
27
|
+
_BasicTranscriptionForwarder_logger.set(this, log());
|
|
28
|
+
this.currentCharacterIndex = 0;
|
|
29
|
+
_BasicTranscriptionForwarder_textIsComplete.set(this, false);
|
|
30
|
+
_BasicTranscriptionForwarder_audioIsComplete.set(this, false);
|
|
31
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_room, room, "f");
|
|
32
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_participantIdentity, participantIdentity, "f");
|
|
33
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_trackSid, trackSid, "f");
|
|
34
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_messageId, messageId, "f");
|
|
35
|
+
}
|
|
36
|
+
start() {
|
|
37
|
+
if (!__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f")) {
|
|
38
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, true, "f");
|
|
39
|
+
__classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_startPublishingLoop).call(this).catch((error) => {
|
|
40
|
+
__classPrivateFieldGet(this, _BasicTranscriptionForwarder_logger, "f").error('Error in publishing loop:', error);
|
|
41
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, false, "f");
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
pushAudio(frame) {
|
|
46
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_totalAudioDuration, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_totalAudioDuration, "f") + frame.samplesPerChannel / frame.sampleRate, "f");
|
|
47
|
+
}
|
|
48
|
+
pushText(text) {
|
|
49
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_currentText, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f") + text, "f");
|
|
50
|
+
}
|
|
51
|
+
markTextComplete() {
|
|
52
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_textIsComplete, true, "f");
|
|
53
|
+
__classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_adjustTimingIfBothFinished).call(this);
|
|
54
|
+
}
|
|
55
|
+
markAudioComplete() {
|
|
56
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_audioIsComplete, true, "f");
|
|
57
|
+
__classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_adjustTimingIfBothFinished).call(this);
|
|
58
|
+
}
|
|
59
|
+
async close(interrupt) {
|
|
60
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, false, "f");
|
|
61
|
+
// Publish whatever we had as final
|
|
62
|
+
if (!interrupt) {
|
|
63
|
+
this.currentCharacterIndex = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length;
|
|
64
|
+
}
|
|
65
|
+
await __classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_publishTranscription).call(this, true);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
_BasicTranscriptionForwarder_room = new WeakMap(), _BasicTranscriptionForwarder_participantIdentity = new WeakMap(), _BasicTranscriptionForwarder_trackSid = new WeakMap(), _BasicTranscriptionForwarder_currentText = new WeakMap(), _BasicTranscriptionForwarder_totalAudioDuration = new WeakMap(), _BasicTranscriptionForwarder_currentPlayoutTime = new WeakMap(), _BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND = new WeakMap(), _BasicTranscriptionForwarder_charsPerSecond = new WeakMap(), _BasicTranscriptionForwarder_messageId = new WeakMap(), _BasicTranscriptionForwarder_isRunning = new WeakMap(), _BasicTranscriptionForwarder_logger = new WeakMap(), _BasicTranscriptionForwarder_textIsComplete = new WeakMap(), _BasicTranscriptionForwarder_audioIsComplete = new WeakMap(), _BasicTranscriptionForwarder_instances = new WeakSet(), _BasicTranscriptionForwarder_adjustTimingIfBothFinished = function _BasicTranscriptionForwarder_adjustTimingIfBothFinished() {
|
|
69
|
+
if (__classPrivateFieldGet(this, _BasicTranscriptionForwarder_textIsComplete, "f") && __classPrivateFieldGet(this, _BasicTranscriptionForwarder_audioIsComplete, "f")) {
|
|
70
|
+
const actualDuration = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_totalAudioDuration, "f");
|
|
71
|
+
if (actualDuration > 0 && __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length > 0) {
|
|
72
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_charsPerSecond, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length / actualDuration, "f");
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}, _BasicTranscriptionForwarder_computeSleepInterval = function _BasicTranscriptionForwarder_computeSleepInterval() {
|
|
76
|
+
return Math.min(Math.max(1 / __classPrivateFieldGet(this, _BasicTranscriptionForwarder_charsPerSecond, "f"), 0.0625), 0.5);
|
|
77
|
+
}, _BasicTranscriptionForwarder_startPublishingLoop = async function _BasicTranscriptionForwarder_startPublishingLoop() {
|
|
78
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, true, "f");
|
|
79
|
+
let sleepInterval = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_computeSleepInterval).call(this);
|
|
80
|
+
let isComplete = false;
|
|
81
|
+
while (__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f") && !isComplete) {
|
|
82
|
+
__classPrivateFieldSet(this, _BasicTranscriptionForwarder_currentPlayoutTime, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentPlayoutTime, "f") + sleepInterval, "f");
|
|
83
|
+
this.currentCharacterIndex = Math.floor(__classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentPlayoutTime, "f") * __classPrivateFieldGet(this, _BasicTranscriptionForwarder_charsPerSecond, "f"));
|
|
84
|
+
isComplete = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_textIsComplete, "f") && this.currentCharacterIndex >= __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length;
|
|
85
|
+
await __classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_publishTranscription).call(this, false);
|
|
86
|
+
if (__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f") && !isComplete) {
|
|
87
|
+
sleepInterval = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_computeSleepInterval).call(this);
|
|
88
|
+
await new Promise((resolve) => setTimeout(resolve, sleepInterval * 1000));
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f")) {
|
|
92
|
+
this.close(false);
|
|
93
|
+
}
|
|
94
|
+
}, _BasicTranscriptionForwarder_publishTranscription = async function _BasicTranscriptionForwarder_publishTranscription(final) {
|
|
95
|
+
var _a;
|
|
96
|
+
const textToPublish = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").slice(0, this.currentCharacterIndex);
|
|
97
|
+
await ((_a = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_room, "f").localParticipant) === null || _a === void 0 ? void 0 : _a.publishTranscription({
|
|
98
|
+
participantIdentity: __classPrivateFieldGet(this, _BasicTranscriptionForwarder_participantIdentity, "f"),
|
|
99
|
+
trackSid: __classPrivateFieldGet(this, _BasicTranscriptionForwarder_trackSid, "f"),
|
|
100
|
+
segments: [
|
|
101
|
+
{
|
|
102
|
+
text: textToPublish,
|
|
103
|
+
final: final,
|
|
104
|
+
id: __classPrivateFieldGet(this, _BasicTranscriptionForwarder_messageId, "f"),
|
|
105
|
+
startTime: BigInt(0),
|
|
106
|
+
endTime: BigInt(0),
|
|
107
|
+
language: '',
|
|
108
|
+
},
|
|
109
|
+
],
|
|
110
|
+
}));
|
|
111
|
+
};
|
|
112
|
+
//# sourceMappingURL=transcription.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transcription.js","sourceRoot":"","sources":["../src/transcription.ts"],"names":[],"mappings":";;;;;;;;;;;;AAIA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAY/B,MAAM,OAAO,2BAA2B;IActC,YAAY,IAAU,EAAE,mBAA2B,EAAE,QAAgB,EAAE,SAAiB;;QAbxF,oDAAY;QACZ,mEAA6B;QAC7B,wDAAkB;QAClB,mDAAuB,EAAE,EAAC;QAC1B,0DAA8B,CAAC,EAAC;QAChC,0DAA8B,CAAC,EAAC;QAChC,gEAA4B,EAAE,EAAC;QAC/B,sDAA0B,uBAAA,IAAI,6DAA0B,EAAC;QACzD,yDAAmB;QACnB,iDAAsB,KAAK,EAAC;QAC5B,8CAAU,GAAG,EAAE,EAAC;QAChB,0BAAqB,GAAW,CAAC,CAAC;QA2BlC,sDAA2B,KAAK,EAAC;QACjC,uDAA4B,KAAK,EAAC;QAzBhC,uBAAA,IAAI,qCAAS,IAAI,MAAA,CAAC;QAClB,uBAAA,IAAI,oDAAwB,mBAAmB,MAAA,CAAC;QAChD,uBAAA,IAAI,yCAAa,QAAQ,MAAA,CAAC;QAC1B,uBAAA,IAAI,0CAAc,SAAS,MAAA,CAAC;IAC9B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,uBAAA,IAAI,8CAAW,EAAE,CAAC;YACrB,uBAAA,IAAI,0CAAc,IAAI,MAAA,CAAC;YACvB,uBAAA,IAAI,gGAAqB,MAAzB,IAAI,CAAuB,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC1C,uBAAA,IAAI,2CAAQ,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;gBACvD,uBAAA,IAAI,0CAAc,KAAK,MAAA,CAAC;YAC1B,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,SAAS,CAAC,KAAiB;QACzB,mKAA4B,KAAK,CAAC,iBAAiB,GAAG,KAAK,CAAC,UAAU,MAAA,CAAC;IACzE,CAAC;IAED,QAAQ,CAAC,IAAY;QACnB,qJAAqB,IAAI,MAAA,CAAC;IAC5B,CAAC;IAKD,gBAAgB;QACd,uBAAA,IAAI,+CAAmB,IAAI,MAAA,CAAC;QAC5B,uBAAA,IAAI,uGAA4B,MAAhC,IAAI,CAA8B,CAAC;IACrC,CAAC;IAED,iBAAiB;QACf,uBAAA,IAAI,gDAAoB,IAAI,MAAA,CAAC;QAC7B,uBAAA,IAAI,uGAA4B,MAAhC,IAAI,CAA8B,CAAC;IACrC,CAAC;IAqDD,KAAK,CAAC,KAAK,CAAC,SAAkB;QAC5B,uBAAA,IAAI,0CAAc,KAAK,MAAA,CAAC;QAExB,mCAAmC;QACnC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,IAAI,CAAC,qBAAqB,GAAG,uBAAA,IAAI,gDAAa,CAAC,MAAM,CAAC;QACxD,CAAC;QACD,MAAM,uBAAA,IAAI,iGAAsB,MAA1B,IAAI,EAAuB,IAAI,CAAC,CAAC;IACzC,CAAC;CACF;;IA3DG,IAAI,uBAAA,IAAI,mDAAgB,IAAI,uBAAA,IAAI,oDAAiB,EAAE,CAAC;QAClD,MAAM,cAAc,GAAG,uBAAA,IAAI,uDAAoB,CAAC;QAChD,IAAI,cAAc,GAAG,CAAC,IAAI,uBAAA,IAAI,gDAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,uBAAA,IAAI,+CAAmB,uBAAA,IAAI,gDAAa,CAAC,MAAM,GAAG,cAAc,MAAA,CAAC;QACnE,CAAC;IACH,CAAC;AACH,CAAC;IAGC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,uBAAA,IAAI,mDAAgB,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;AACnE,CAAC,qDAED,KAAK;IACH,uBAAA,IAAI,0CAAc,IAAI,MAAA,CAAC;IACvB,IAAI,aAAa,GAAG,uBAAA,IAAI,iGAAsB,MAA1B,IAAI,CAAwB,CAAC;IACjD,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,OAAO,uBAAA,IAAI,8CAAW,IAAI,CAAC,UAAU,EAAE,CAAC;QACtC,mKAA4B,aAAa,MAAA,CAAC;QAC1C,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,uBAAA,IAAI,uDAAoB,GAAG,uBAAA,IAAI,mDAAgB,CAAC,CAAC;QACzF,UAAU,GAAG,uBAAA,IAAI,mDAAgB,IAAI,IAAI,CAAC,qBAAqB,IAAI,uBAAA,IAAI,gDAAa,CAAC,MAAM,CAAC;QAC5F,MAAM,uBAAA,IAAI,iGAAsB,MAA1B,IAAI,EAAuB,KAAK,CAAC,CAAC;QACxC,IAAI,uBAAA,IAAI,8CAAW,IAAI,CAAC,UAAU,EAAE,CAAC;YACnC,aAAa,GAAG,uBAAA,IAAI,iGAAsB,MAA1B,IAAI,CAAwB,CAAC;YAC7C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,IAAI,uBAAA,IAAI,8CAAW,EAAE,CAAC;QACpB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;AACH,CAAC,sDAED,KAAK,4DAAuB,KAAc;;IACxC,MAAM,aAAa,GAAG,uBAAA,IAAI,gDAAa,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC7E,MAAM,CAAA,MAAA,uBAAA,IAAI,yCAAM,CAAC,gBAAgB,0CAAE,oBAAoB,CAAC;QACtD,mBAAmB,EAAE,uBAAA,IAAI,wDAAqB;QAC9C,QAAQ,EAAE,uBAAA,IAAI,6CAAU;QACxB,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,aAAa;gBACnB,KAAK,EAAE,KAAK;gBACZ,EAAE,EAAE,uBAAA,IAAI,8CAAW;gBACnB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;gBACpB,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;gBAClB,QAAQ,EAAE,EAAE;aACb;SACF;KACF,CAAC,CAAA,CAAC;AACL,CAAC"}
|
package/dist/utils.d.ts
CHANGED
|
@@ -22,6 +22,8 @@ export declare class Mutex {
|
|
|
22
22
|
/** @internal */
|
|
23
23
|
export declare class Queue<T> {
|
|
24
24
|
#private;
|
|
25
|
+
/** @internal */
|
|
26
|
+
items: T[];
|
|
25
27
|
constructor(limit?: number);
|
|
26
28
|
get(): Promise<T>;
|
|
27
29
|
put(item: T): Promise<void>;
|
|
@@ -29,8 +31,34 @@ export declare class Queue<T> {
|
|
|
29
31
|
/** @internal */
|
|
30
32
|
export declare class Future {
|
|
31
33
|
#private;
|
|
34
|
+
constructor();
|
|
32
35
|
get await(): Promise<void>;
|
|
36
|
+
get done(): boolean;
|
|
33
37
|
resolve(): void;
|
|
34
|
-
reject(
|
|
38
|
+
reject(error: Error): void;
|
|
39
|
+
}
|
|
40
|
+
/** @internal */
|
|
41
|
+
export declare class CancellablePromise<T> {
|
|
42
|
+
#private;
|
|
43
|
+
constructor(executor: (resolve: (value: T | PromiseLike<T>) => void, reject: (reason?: any) => void, onCancel: (cancelFn: () => void) => void) => void);
|
|
44
|
+
get isCancelled(): boolean;
|
|
45
|
+
get error(): Error | null;
|
|
46
|
+
then<TResult1 = T, TResult2 = never>(onfulfilled?: ((value: T) => TResult1 | Promise<TResult1>) | null, onrejected?: ((reason: any) => TResult2 | Promise<TResult2>) | null): Promise<TResult1 | TResult2>;
|
|
47
|
+
catch<TResult = never>(onrejected?: ((reason: any) => TResult | Promise<TResult>) | null): Promise<T | TResult>;
|
|
48
|
+
finally(onfinally?: (() => void) | null): Promise<T>;
|
|
49
|
+
cancel(): void;
|
|
50
|
+
static from<T>(promise: Promise<T>): CancellablePromise<T>;
|
|
51
|
+
}
|
|
52
|
+
/** @internal */
|
|
53
|
+
export declare function gracefullyCancel<T>(promise: CancellablePromise<T>): Promise<void>;
|
|
54
|
+
/** @internal */
|
|
55
|
+
export declare class AsyncIterableQueue<T> implements AsyncIterable<T> {
|
|
56
|
+
private queue;
|
|
57
|
+
private closed;
|
|
58
|
+
private static readonly QUEUE_END_MARKER;
|
|
59
|
+
constructor();
|
|
60
|
+
put(item: T): void;
|
|
61
|
+
close(): void;
|
|
62
|
+
[Symbol.asyncIterator](): AsyncIterator<T>;
|
|
35
63
|
}
|
|
36
64
|
//# sourceMappingURL=utils.d.ts.map
|
package/dist/utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAGV,IAAI,EAEL,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,UAAU,EAAe,MAAM,mBAAmB,CAAC;AAG5D,0DAA0D;AAC1D,MAAM,MAAM,WAAW,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC;AAEpD;;;;;;;GAOG;AACH,eAAO,MAAM,WAAW,WAAY,WAAW,KAAG,UA6BjD,CAAC;AAEF,eAAO,MAAM,gBAAgB,SAAU,IAAI,YAAY,MAAM,KAAG,MAyB/D,CAAC;AAEF,gBAAgB;AAChB,qBAAa,KAAK;;gBAKJ,KAAK,SAAI;IAMrB,QAAQ,IAAI,OAAO;IAIb,IAAI,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC;CAiBlC;AAED,gBAAgB;AAChB,qBAAa,KAAK,CAAC,CAAC;;
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAGV,IAAI,EAEL,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,UAAU,EAAe,MAAM,mBAAmB,CAAC;AAG5D,0DAA0D;AAC1D,MAAM,MAAM,WAAW,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC;AAEpD;;;;;;;GAOG;AACH,eAAO,MAAM,WAAW,WAAY,WAAW,KAAG,UA6BjD,CAAC;AAEF,eAAO,MAAM,gBAAgB,SAAU,IAAI,YAAY,MAAM,KAAG,MAyB/D,CAAC;AAEF,gBAAgB;AAChB,qBAAa,KAAK;;gBAKJ,KAAK,SAAI;IAMrB,QAAQ,IAAI,OAAO;IAIb,IAAI,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC;CAiBlC;AAED,gBAAgB;AAChB,qBAAa,KAAK,CAAC,CAAC;;IAClB,gBAAgB;IAChB,KAAK,EAAE,CAAC,EAAE,CAAM;gBAIJ,KAAK,CAAC,EAAE,MAAM;IAIpB,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC;IASjB,GAAG,CAAC,IAAI,EAAE,CAAC;CAOlB;AAED,gBAAgB;AAChB,qBAAa,MAAM;;;IAajB,IAAI,KAAK,kBAER;IAED,IAAI,IAAI,YAEP;IAED,OAAO;IAKP,MAAM,CAAC,KAAK,EAAE,KAAK;CAIpB;AAED,gBAAgB;AAChB,qBAAa,kBAAkB,CAAC,CAAC;;gBAO7B,QAAQ,EAAE,CACR,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI,EAC5C,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE,GAAG,KAAK,IAAI,EAC9B,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,IAAI,KACrC,IAAI;IAuBX,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,IAAI,KAAK,IAAI,KAAK,GAAG,IAAI,CAExB;IAED,IAAI,CAAC,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,KAAK,EACjC,WAAW,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,EACjE,UAAU,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,KAAK,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,GAClE,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAI/B,KAAK,CAAC,OAAO,GAAG,KAAK,EACnB,UAAU,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,IAAI,GAChE,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC;IAIvB,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC;IAIpD,MAAM,IAAI,IAAI;IAId,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC,CAAC;CAK3D;AAED,gBAAgB;AAChB,wBAAsB,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CASvF;AAED,gBAAgB;AAChB,qBAAa,kBAAkB,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,CAAC;IAC5D,OAAO,CAAC,KAAK,CAAwD;IACrE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAA8B;;IAMtE,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI;IAOlB,KAAK,IAAI,IAAI;IAKb,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;CAc3C"}
|