@anam-ai/js-sdk 4.8.0-alpha.sam.0 → 4.8.0-alpha.sam.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/main/AnamClient.d.ts +1 -0
- package/dist/main/AnamClient.d.ts.map +1 -1
- package/dist/main/AnamClient.js +1 -0
- package/dist/main/AnamClient.js.map +1 -1
- package/dist/main/modules/StreamingClient.js +1 -1
- package/dist/main/modules/StreamingClient.js.map +1 -1
- package/dist/main/modules/ThoughtHistoryClient.d.ts +10 -0
- package/dist/main/modules/ThoughtHistoryClient.d.ts.map +1 -0
- package/dist/main/modules/ThoughtHistoryClient.js +34 -0
- package/dist/main/modules/ThoughtHistoryClient.js.map +1 -0
- package/dist/main/modules/index.d.ts +1 -0
- package/dist/main/modules/index.d.ts.map +1 -1
- package/dist/main/modules/index.js +3 -1
- package/dist/main/modules/index.js.map +1 -1
- package/dist/main/types/events/internal/InternalEvent.d.ts +2 -1
- package/dist/main/types/events/internal/InternalEvent.d.ts.map +1 -1
- package/dist/main/types/events/internal/InternalEvent.js +1 -0
- package/dist/main/types/events/internal/InternalEvent.js.map +1 -1
- package/dist/main/types/events/internal/InternalEventCallbacks.d.ts.map +1 -1
- package/dist/main/types/events/internal/InternalEventCallbacks.js.map +1 -1
- package/dist/main/types/events/public/AnamEvent.d.ts +2 -1
- package/dist/main/types/events/public/AnamEvent.d.ts.map +1 -1
- package/dist/main/types/events/public/AnamEvent.js +2 -1
- package/dist/main/types/events/public/AnamEvent.js.map +1 -1
- package/dist/main/types/events/public/EventCallbacks.d.ts +0 -1
- package/dist/main/types/events/public/EventCallbacks.d.ts.map +1 -1
- package/dist/main/types/events/public/EventCallbacks.js.map +1 -1
- package/dist/main/types/messageHistory/Thought.d.ts +5 -0
- package/dist/main/types/messageHistory/Thought.d.ts.map +1 -0
- package/dist/main/types/messageHistory/Thought.js +3 -0
- package/dist/main/types/messageHistory/Thought.js.map +1 -0
- package/dist/main/types/messageHistory/ThoughtStreamEvent.d.ts +6 -0
- package/dist/main/types/messageHistory/ThoughtStreamEvent.d.ts.map +1 -0
- package/dist/main/types/messageHistory/ThoughtStreamEvent.js +3 -0
- package/dist/main/types/messageHistory/ThoughtStreamEvent.js.map +1 -0
- package/dist/main/types/messageHistory/index.d.ts +2 -0
- package/dist/main/types/messageHistory/index.d.ts.map +1 -1
- package/dist/main/types/streaming/WebRtcReasoningTextMessageEvent.d.ts +1 -0
- package/dist/main/types/streaming/WebRtcReasoningTextMessageEvent.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/module/AnamClient.d.ts +0 -64
- package/dist/module/AnamClient.d.ts.map +0 -1
- package/dist/module/AnamClient.js +0 -461
- package/dist/module/AnamClient.js.map +0 -1
- package/dist/module/index.d.ts +0 -26
- package/dist/module/index.d.ts.map +0 -1
- package/dist/module/index.js +0 -27
- package/dist/module/index.js.map +0 -1
- package/dist/module/lib/ClientError.d.ts +0 -19
- package/dist/module/lib/ClientError.d.ts.map +0 -1
- package/dist/module/lib/ClientError.js +0 -30
- package/dist/module/lib/ClientError.js.map +0 -1
- package/dist/module/lib/ClientMetrics.d.ts +0 -59
- package/dist/module/lib/ClientMetrics.d.ts.map +0 -1
- package/dist/module/lib/ClientMetrics.js +0 -305
- package/dist/module/lib/ClientMetrics.js.map +0 -1
- package/dist/module/lib/constants.d.ts +0 -10
- package/dist/module/lib/constants.d.ts.map +0 -1
- package/dist/module/lib/constants.js +0 -11
- package/dist/module/lib/constants.js.map +0 -1
- package/dist/module/lib/correlationId.d.ts +0 -2
- package/dist/module/lib/correlationId.d.ts.map +0 -1
- package/dist/module/lib/correlationId.js +0 -9
- package/dist/module/lib/correlationId.js.map +0 -1
- package/dist/module/lib/validateApiGatewayConfig.d.ts +0 -8
- package/dist/module/lib/validateApiGatewayConfig.d.ts.map +0 -1
- package/dist/module/lib/validateApiGatewayConfig.js +0 -31
- package/dist/module/lib/validateApiGatewayConfig.js.map +0 -1
- package/dist/module/lib/version.d.ts +0 -3
- package/dist/module/lib/version.d.ts.map +0 -1
- package/dist/module/lib/version.js +0 -3
- package/dist/module/lib/version.js.map +0 -1
- package/dist/module/modules/CoreApiRestClient.d.ts +0 -18
- package/dist/module/modules/CoreApiRestClient.d.ts.map +0 -1
- package/dist/module/modules/CoreApiRestClient.js +0 -149
- package/dist/module/modules/CoreApiRestClient.js.map +0 -1
- package/dist/module/modules/EngineApiRestClient.d.ts +0 -9
- package/dist/module/modules/EngineApiRestClient.d.ts.map +0 -1
- package/dist/module/modules/EngineApiRestClient.js +0 -56
- package/dist/module/modules/EngineApiRestClient.js.map +0 -1
- package/dist/module/modules/InternalEventEmitter.d.ts +0 -9
- package/dist/module/modules/InternalEventEmitter.d.ts.map +0 -1
- package/dist/module/modules/InternalEventEmitter.js +0 -24
- package/dist/module/modules/InternalEventEmitter.js.map +0 -1
- package/dist/module/modules/MessageHistoryClient.d.ts +0 -13
- package/dist/module/modules/MessageHistoryClient.d.ts.map +0 -1
- package/dist/module/modules/MessageHistoryClient.js +0 -66
- package/dist/module/modules/MessageHistoryClient.js.map +0 -1
- package/dist/module/modules/PublicEventEmitter.d.ts +0 -9
- package/dist/module/modules/PublicEventEmitter.d.ts.map +0 -1
- package/dist/module/modules/PublicEventEmitter.js +0 -33
- package/dist/module/modules/PublicEventEmitter.js.map +0 -1
- package/dist/module/modules/SignallingClient.d.ts +0 -34
- package/dist/module/modules/SignallingClient.d.ts.map +0 -1
- package/dist/module/modules/SignallingClient.js +0 -251
- package/dist/module/modules/SignallingClient.js.map +0 -1
- package/dist/module/modules/StreamingClient.d.ts +0 -81
- package/dist/module/modules/StreamingClient.d.ts.map +0 -1
- package/dist/module/modules/StreamingClient.js +0 -706
- package/dist/module/modules/StreamingClient.js.map +0 -1
- package/dist/module/modules/ToolCallManager.d.ts +0 -8
- package/dist/module/modules/ToolCallManager.d.ts.map +0 -1
- package/dist/module/modules/ToolCallManager.js +0 -17
- package/dist/module/modules/ToolCallManager.js.map +0 -1
- package/dist/module/modules/index.d.ts +0 -9
- package/dist/module/modules/index.d.ts.map +0 -1
- package/dist/module/modules/index.js +0 -9
- package/dist/module/modules/index.js.map +0 -1
- package/dist/module/types/AgentAudioInputStream.d.ts +0 -28
- package/dist/module/types/AgentAudioInputStream.d.ts.map +0 -1
- package/dist/module/types/AgentAudioInputStream.js +0 -50
- package/dist/module/types/AgentAudioInputStream.js.map +0 -1
- package/dist/module/types/AnamClientOptions.d.ts +0 -4
- package/dist/module/types/AnamClientOptions.d.ts.map +0 -1
- package/dist/module/types/AnamClientOptions.js +0 -2
- package/dist/module/types/AnamClientOptions.js.map +0 -1
- package/dist/module/types/AnamInternalClientOptions.d.ts +0 -4
- package/dist/module/types/AnamInternalClientOptions.d.ts.map +0 -1
- package/dist/module/types/AnamInternalClientOptions.js +0 -2
- package/dist/module/types/AnamInternalClientOptions.js.map +0 -1
- package/dist/module/types/AnamPublicClientOptions.d.ts +0 -20
- package/dist/module/types/AnamPublicClientOptions.d.ts.map +0 -1
- package/dist/module/types/AnamPublicClientOptions.js +0 -2
- package/dist/module/types/AnamPublicClientOptions.js.map +0 -1
- package/dist/module/types/ApiGatewayConfig.d.ts +0 -45
- package/dist/module/types/ApiGatewayConfig.d.ts.map +0 -1
- package/dist/module/types/ApiGatewayConfig.js +0 -2
- package/dist/module/types/ApiGatewayConfig.js.map +0 -1
- package/dist/module/types/InputAudioState.d.ts +0 -11
- package/dist/module/types/InputAudioState.d.ts.map +0 -1
- package/dist/module/types/InputAudioState.js +0 -8
- package/dist/module/types/InputAudioState.js.map +0 -1
- package/dist/module/types/PersonaConfig.d.ts +0 -13
- package/dist/module/types/PersonaConfig.d.ts.map +0 -1
- package/dist/module/types/PersonaConfig.js +0 -4
- package/dist/module/types/PersonaConfig.js.map +0 -1
- package/dist/module/types/TalkMessageStream.d.ts +0 -18
- package/dist/module/types/TalkMessageStream.d.ts.map +0 -1
- package/dist/module/types/TalkMessageStream.js +0 -89
- package/dist/module/types/TalkMessageStream.js.map +0 -1
- package/dist/module/types/TalkMessageStreamState.d.ts +0 -7
- package/dist/module/types/TalkMessageStreamState.d.ts.map +0 -1
- package/dist/module/types/TalkMessageStreamState.js +0 -8
- package/dist/module/types/TalkMessageStreamState.js.map +0 -1
- package/dist/module/types/VoiceDetectionOptions.d.ts +0 -4
- package/dist/module/types/VoiceDetectionOptions.d.ts.map +0 -1
- package/dist/module/types/VoiceDetectionOptions.js +0 -2
- package/dist/module/types/VoiceDetectionOptions.js.map +0 -1
- package/dist/module/types/coreApi/ApiOptions.d.ts +0 -7
- package/dist/module/types/coreApi/ApiOptions.d.ts.map +0 -1
- package/dist/module/types/coreApi/ApiOptions.js +0 -2
- package/dist/module/types/coreApi/ApiOptions.js.map +0 -1
- package/dist/module/types/coreApi/StartSessionOptions.d.ts +0 -5
- package/dist/module/types/coreApi/StartSessionOptions.d.ts.map +0 -1
- package/dist/module/types/coreApi/StartSessionOptions.js +0 -2
- package/dist/module/types/coreApi/StartSessionOptions.js.map +0 -1
- package/dist/module/types/coreApi/StartSessionResponse.d.ts +0 -13
- package/dist/module/types/coreApi/StartSessionResponse.d.ts.map +0 -1
- package/dist/module/types/coreApi/StartSessionResponse.js +0 -2
- package/dist/module/types/coreApi/StartSessionResponse.js.map +0 -1
- package/dist/module/types/coreApi/index.d.ts +0 -5
- package/dist/module/types/coreApi/index.d.ts.map +0 -1
- package/dist/module/types/coreApi/index.js +0 -2
- package/dist/module/types/coreApi/index.js.map +0 -1
- package/dist/module/types/engineApi/EngineApiRestClientOptions.d.ts +0 -4
- package/dist/module/types/engineApi/EngineApiRestClientOptions.d.ts.map +0 -1
- package/dist/module/types/engineApi/EngineApiRestClientOptions.js +0 -2
- package/dist/module/types/engineApi/EngineApiRestClientOptions.js.map +0 -1
- package/dist/module/types/engineApi/index.d.ts +0 -1
- package/dist/module/types/engineApi/index.d.ts.map +0 -1
- package/dist/module/types/engineApi/index.js +0 -2
- package/dist/module/types/engineApi/index.js.map +0 -1
- package/dist/module/types/events/EventCallback.d.ts +0 -2
- package/dist/module/types/events/EventCallback.d.ts.map +0 -1
- package/dist/module/types/events/EventCallback.js +0 -2
- package/dist/module/types/events/EventCallback.js.map +0 -1
- package/dist/module/types/events/index.d.ts +0 -7
- package/dist/module/types/events/index.d.ts.map +0 -1
- package/dist/module/types/events/index.js +0 -4
- package/dist/module/types/events/index.js.map +0 -1
- package/dist/module/types/events/internal/InternalEvent.d.ts +0 -7
- package/dist/module/types/events/internal/InternalEvent.d.ts.map +0 -1
- package/dist/module/types/events/internal/InternalEvent.js +0 -8
- package/dist/module/types/events/internal/InternalEvent.js.map +0 -1
- package/dist/module/types/events/internal/InternalEventCallbacks.d.ts +0 -8
- package/dist/module/types/events/internal/InternalEventCallbacks.d.ts.map +0 -1
- package/dist/module/types/events/internal/InternalEventCallbacks.js +0 -2
- package/dist/module/types/events/internal/InternalEventCallbacks.js.map +0 -1
- package/dist/module/types/events/public/AnamEvent.d.ts +0 -20
- package/dist/module/types/events/public/AnamEvent.d.ts.map +0 -1
- package/dist/module/types/events/public/AnamEvent.js +0 -21
- package/dist/module/types/events/public/AnamEvent.js.map +0 -1
- package/dist/module/types/events/public/ConnectionClosedCodes.d.ts +0 -8
- package/dist/module/types/events/public/ConnectionClosedCodes.d.ts.map +0 -1
- package/dist/module/types/events/public/ConnectionClosedCodes.js +0 -9
- package/dist/module/types/events/public/ConnectionClosedCodes.js.map +0 -1
- package/dist/module/types/events/public/EventCallbacks.d.ts +0 -22
- package/dist/module/types/events/public/EventCallbacks.d.ts.map +0 -1
- package/dist/module/types/events/public/EventCallbacks.js +0 -2
- package/dist/module/types/events/public/EventCallbacks.js.map +0 -1
- package/dist/module/types/index.d.ts +0 -18
- package/dist/module/types/index.d.ts.map +0 -1
- package/dist/module/types/index.js +0 -9
- package/dist/module/types/index.js.map +0 -1
- package/dist/module/types/messageHistory/Message.d.ts +0 -8
- package/dist/module/types/messageHistory/Message.d.ts.map +0 -1
- package/dist/module/types/messageHistory/Message.js +0 -2
- package/dist/module/types/messageHistory/Message.js.map +0 -1
- package/dist/module/types/messageHistory/MessageRole.d.ts +0 -5
- package/dist/module/types/messageHistory/MessageRole.d.ts.map +0 -1
- package/dist/module/types/messageHistory/MessageRole.js +0 -6
- package/dist/module/types/messageHistory/MessageRole.js.map +0 -1
- package/dist/module/types/messageHistory/MessageStreamEvent.d.ts +0 -9
- package/dist/module/types/messageHistory/MessageStreamEvent.d.ts.map +0 -1
- package/dist/module/types/messageHistory/MessageStreamEvent.js +0 -2
- package/dist/module/types/messageHistory/MessageStreamEvent.js.map +0 -1
- package/dist/module/types/messageHistory/index.d.ts +0 -4
- package/dist/module/types/messageHistory/index.d.ts.map +0 -1
- package/dist/module/types/messageHistory/index.js +0 -2
- package/dist/module/types/messageHistory/index.js.map +0 -1
- package/dist/module/types/signalling/AgentAudioInputConfig.d.ts +0 -9
- package/dist/module/types/signalling/AgentAudioInputConfig.d.ts.map +0 -1
- package/dist/module/types/signalling/AgentAudioInputConfig.js +0 -2
- package/dist/module/types/signalling/AgentAudioInputConfig.js.map +0 -1
- package/dist/module/types/signalling/AgentAudioInputPayload.d.ts +0 -13
- package/dist/module/types/signalling/AgentAudioInputPayload.d.ts.map +0 -1
- package/dist/module/types/signalling/AgentAudioInputPayload.js +0 -2
- package/dist/module/types/signalling/AgentAudioInputPayload.js.map +0 -1
- package/dist/module/types/signalling/SignalMessage.d.ts +0 -19
- package/dist/module/types/signalling/SignalMessage.d.ts.map +0 -1
- package/dist/module/types/signalling/SignalMessage.js +0 -15
- package/dist/module/types/signalling/SignalMessage.js.map +0 -1
- package/dist/module/types/signalling/SignallingClientOptions.d.ts +0 -12
- package/dist/module/types/signalling/SignallingClientOptions.d.ts.map +0 -1
- package/dist/module/types/signalling/SignallingClientOptions.js +0 -2
- package/dist/module/types/signalling/SignallingClientOptions.js.map +0 -1
- package/dist/module/types/signalling/TalkMessageStreamPayload.d.ts +0 -7
- package/dist/module/types/signalling/TalkMessageStreamPayload.d.ts.map +0 -1
- package/dist/module/types/signalling/TalkMessageStreamPayload.js +0 -2
- package/dist/module/types/signalling/TalkMessageStreamPayload.js.map +0 -1
- package/dist/module/types/signalling/TalkStreamInterruptedSignalMessage.d.ts +0 -5
- package/dist/module/types/signalling/TalkStreamInterruptedSignalMessage.d.ts.map +0 -1
- package/dist/module/types/signalling/TalkStreamInterruptedSignalMessage.js +0 -7
- package/dist/module/types/signalling/TalkStreamInterruptedSignalMessage.js.map +0 -1
- package/dist/module/types/signalling/index.d.ts +0 -6
- package/dist/module/types/signalling/index.d.ts.map +0 -1
- package/dist/module/types/signalling/index.js +0 -2
- package/dist/module/types/signalling/index.js.map +0 -1
- package/dist/module/types/streaming/ClientToolEvent.d.ts +0 -10
- package/dist/module/types/streaming/ClientToolEvent.d.ts.map +0 -1
- package/dist/module/types/streaming/ClientToolEvent.js +0 -2
- package/dist/module/types/streaming/ClientToolEvent.js.map +0 -1
- package/dist/module/types/streaming/DataChannelMessage.d.ts +0 -6
- package/dist/module/types/streaming/DataChannelMessage.d.ts.map +0 -1
- package/dist/module/types/streaming/DataChannelMessage.js +0 -7
- package/dist/module/types/streaming/DataChannelMessage.js.map +0 -1
- package/dist/module/types/streaming/InputAudioOptions.d.ts +0 -8
- package/dist/module/types/streaming/InputAudioOptions.d.ts.map +0 -1
- package/dist/module/types/streaming/InputAudioOptions.js +0 -2
- package/dist/module/types/streaming/InputAudioOptions.js.map +0 -1
- package/dist/module/types/streaming/StreamingClientOptions.d.ts +0 -16
- package/dist/module/types/streaming/StreamingClientOptions.d.ts.map +0 -1
- package/dist/module/types/streaming/StreamingClientOptions.js +0 -2
- package/dist/module/types/streaming/StreamingClientOptions.js.map +0 -1
- package/dist/module/types/streaming/WebRtcClientToolEvent.d.ts +0 -11
- package/dist/module/types/streaming/WebRtcClientToolEvent.d.ts.map +0 -1
- package/dist/module/types/streaming/WebRtcClientToolEvent.js +0 -2
- package/dist/module/types/streaming/WebRtcClientToolEvent.js.map +0 -1
- package/dist/module/types/streaming/WebRtcReasoningTextMessageEvent.d.ts +0 -7
- package/dist/module/types/streaming/WebRtcReasoningTextMessageEvent.d.ts.map +0 -1
- package/dist/module/types/streaming/WebRtcReasoningTextMessageEvent.js +0 -2
- package/dist/module/types/streaming/WebRtcReasoningTextMessageEvent.js.map +0 -1
- package/dist/module/types/streaming/WebRtcTextMessageEvent.d.ts +0 -9
- package/dist/module/types/streaming/WebRtcTextMessageEvent.d.ts.map +0 -1
- package/dist/module/types/streaming/WebRtcTextMessageEvent.js +0 -2
- package/dist/module/types/streaming/WebRtcTextMessageEvent.js.map +0 -1
- package/dist/module/types/streaming/index.d.ts +0 -8
- package/dist/module/types/streaming/index.d.ts.map +0 -1
- package/dist/module/types/streaming/index.js +0 -2
- package/dist/module/types/streaming/index.js.map +0 -1
- package/dist/umd/anam.js +0 -2
- package/dist/umd/anam.js.LICENSE.txt +0 -8
|
@@ -1,706 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { ClientMetricMeasurement, createRTCStatsReport, sendClientMetric, } from '../lib/ClientMetrics';
|
|
11
|
-
import { EngineApiRestClient, SignallingClient, ToolCallManager, } from '../modules';
|
|
12
|
-
import { AnamEvent, AudioPermissionState, ConnectionClosedCode, DataChannelMessage, InternalEvent, SignalMessageAction, } from '../types';
|
|
13
|
-
import { AgentAudioInputStream } from '../types/AgentAudioInputStream';
|
|
14
|
-
import { TalkMessageStream } from '../types/TalkMessageStream';
|
|
15
|
-
const SUCCESS_METRIC_POLLING_TIMEOUT_MS = 15000; // After this time we will stop polling for the first frame and consider the session a failure.
|
|
16
|
-
const STATS_COLLECTION_INTERVAL_MS = 5000;
|
|
17
|
-
const ICE_CANDIDATE_POOL_SIZE = 2; // Optimisation to speed up connection time
|
|
18
|
-
export class StreamingClient {
|
|
19
|
-
constructor(sessionId, options, publicEventEmitter, internalEventEmitter) {
|
|
20
|
-
var _a, _b, _c, _d;
|
|
21
|
-
this.peerConnection = null;
|
|
22
|
-
this.connectionReceivedAnswer = false;
|
|
23
|
-
this.remoteIceCandidateBuffer = [];
|
|
24
|
-
this.inputAudioStream = null;
|
|
25
|
-
this.dataChannel = null;
|
|
26
|
-
this.videoElement = null;
|
|
27
|
-
this.videoStream = null;
|
|
28
|
-
this.audioStream = null;
|
|
29
|
-
this.inputAudioState = {
|
|
30
|
-
isMuted: false,
|
|
31
|
-
permissionState: AudioPermissionState.NOT_REQUESTED,
|
|
32
|
-
};
|
|
33
|
-
this.successMetricPoller = null;
|
|
34
|
-
this.successMetricFired = false;
|
|
35
|
-
this.showPeerConnectionStatsReport = false;
|
|
36
|
-
this.peerConnectionStatsReportOutputFormat = 'console';
|
|
37
|
-
this.statsCollectionInterval = null;
|
|
38
|
-
this.agentAudioInputStream = null;
|
|
39
|
-
this.publicEventEmitter = publicEventEmitter;
|
|
40
|
-
this.internalEventEmitter = internalEventEmitter;
|
|
41
|
-
this.apiGatewayConfig = options.apiGateway;
|
|
42
|
-
// initialize input audio state
|
|
43
|
-
const { inputAudio } = options;
|
|
44
|
-
this.inputAudioState = inputAudio.inputAudioState;
|
|
45
|
-
if (options.inputAudio.userProvidedMediaStream) {
|
|
46
|
-
this.inputAudioStream = options.inputAudio.userProvidedMediaStream;
|
|
47
|
-
}
|
|
48
|
-
this.disableInputAudio = options.inputAudio.disableInputAudio === true;
|
|
49
|
-
// register event handlers
|
|
50
|
-
this.internalEventEmitter.addListener(InternalEvent.WEB_SOCKET_OPEN, this.onSignallingClientConnected.bind(this));
|
|
51
|
-
this.internalEventEmitter.addListener(InternalEvent.SIGNAL_MESSAGE_RECEIVED, this.onSignalMessage.bind(this));
|
|
52
|
-
// set ice servers
|
|
53
|
-
this.iceServers = options.iceServers;
|
|
54
|
-
// initialize signalling client
|
|
55
|
-
this.signallingClient = new SignallingClient(sessionId, options.signalling, this.publicEventEmitter, this.internalEventEmitter, this.apiGatewayConfig);
|
|
56
|
-
// initialize engine API client
|
|
57
|
-
this.engineApiRestClient = new EngineApiRestClient(options.engine.baseUrl, sessionId, this.apiGatewayConfig);
|
|
58
|
-
this.audioDeviceId = options.inputAudio.audioDeviceId;
|
|
59
|
-
this.showPeerConnectionStatsReport =
|
|
60
|
-
(_b = (_a = options.metrics) === null || _a === void 0 ? void 0 : _a.showPeerConnectionStatsReport) !== null && _b !== void 0 ? _b : false;
|
|
61
|
-
this.peerConnectionStatsReportOutputFormat =
|
|
62
|
-
(_d = (_c = options.metrics) === null || _c === void 0 ? void 0 : _c.peerConnectionStatsReportOutputFormat) !== null && _d !== void 0 ? _d : 'console';
|
|
63
|
-
}
|
|
64
|
-
onInputAudioStateChange(oldState, newState) {
|
|
65
|
-
// changed microphone mute state
|
|
66
|
-
if (oldState.isMuted !== newState.isMuted) {
|
|
67
|
-
if (newState.isMuted) {
|
|
68
|
-
this.muteAllAudioTracks();
|
|
69
|
-
}
|
|
70
|
-
else {
|
|
71
|
-
this.unmuteAllAudioTracks();
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
muteAllAudioTracks() {
|
|
76
|
-
var _a;
|
|
77
|
-
(_a = this.inputAudioStream) === null || _a === void 0 ? void 0 : _a.getAudioTracks().forEach((track) => {
|
|
78
|
-
track.enabled = false;
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
unmuteAllAudioTracks() {
|
|
82
|
-
var _a;
|
|
83
|
-
(_a = this.inputAudioStream) === null || _a === void 0 ? void 0 : _a.getAudioTracks().forEach((track) => {
|
|
84
|
-
track.enabled = true;
|
|
85
|
-
});
|
|
86
|
-
}
|
|
87
|
-
startStatsCollection() {
|
|
88
|
-
if (this.statsCollectionInterval) {
|
|
89
|
-
return;
|
|
90
|
-
}
|
|
91
|
-
// Send stats every STATS_COLLECTION_INTERVAL_MS seconds
|
|
92
|
-
this.statsCollectionInterval = setInterval(() => __awaiter(this, void 0, void 0, function* () {
|
|
93
|
-
if (!this.peerConnection ||
|
|
94
|
-
!this.dataChannel ||
|
|
95
|
-
this.dataChannel.readyState !== 'open') {
|
|
96
|
-
return;
|
|
97
|
-
}
|
|
98
|
-
try {
|
|
99
|
-
const stats = yield this.peerConnection.getStats();
|
|
100
|
-
this.sendClientSideMetrics(stats);
|
|
101
|
-
}
|
|
102
|
-
catch (error) {
|
|
103
|
-
console.error('Failed to collect and send stats:', error);
|
|
104
|
-
}
|
|
105
|
-
}), STATS_COLLECTION_INTERVAL_MS);
|
|
106
|
-
}
|
|
107
|
-
sendClientSideMetrics(stats) {
|
|
108
|
-
stats.forEach((report) => {
|
|
109
|
-
// Process inbound-rtp stats for both video and audio
|
|
110
|
-
if (report.type === 'inbound-rtp') {
|
|
111
|
-
const metrics = {
|
|
112
|
-
message_type: 'remote_rtp_stats',
|
|
113
|
-
data: report,
|
|
114
|
-
};
|
|
115
|
-
// Send the metrics via data channel
|
|
116
|
-
if (this.dataChannel && this.dataChannel.readyState === 'open') {
|
|
117
|
-
this.dataChannel.send(JSON.stringify(metrics));
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
});
|
|
121
|
-
}
|
|
122
|
-
startSuccessMetricPolling() {
|
|
123
|
-
if (this.successMetricPoller || this.successMetricFired) {
|
|
124
|
-
return;
|
|
125
|
-
}
|
|
126
|
-
const timeoutId = setTimeout(() => {
|
|
127
|
-
if (this.successMetricPoller) {
|
|
128
|
-
console.warn('No video frames received, there is a problem with the connection.');
|
|
129
|
-
clearInterval(this.successMetricPoller);
|
|
130
|
-
this.successMetricPoller = null;
|
|
131
|
-
}
|
|
132
|
-
}, SUCCESS_METRIC_POLLING_TIMEOUT_MS);
|
|
133
|
-
this.successMetricPoller = setInterval(() => __awaiter(this, void 0, void 0, function* () {
|
|
134
|
-
if (!this.peerConnection || this.successMetricFired) {
|
|
135
|
-
if (this.successMetricPoller) {
|
|
136
|
-
clearInterval(this.successMetricPoller);
|
|
137
|
-
}
|
|
138
|
-
clearTimeout(timeoutId);
|
|
139
|
-
return;
|
|
140
|
-
}
|
|
141
|
-
try {
|
|
142
|
-
const stats = yield this.peerConnection.getStats();
|
|
143
|
-
let videoDetected = false;
|
|
144
|
-
let detectionMethod = null;
|
|
145
|
-
stats.forEach((report) => {
|
|
146
|
-
// Find the report for inbound video
|
|
147
|
-
if (report.type === 'inbound-rtp' && report.kind === 'video') {
|
|
148
|
-
// Method 1: Try framesDecoded (most reliable when available)
|
|
149
|
-
if (report.framesDecoded !== undefined &&
|
|
150
|
-
report.framesDecoded > 0) {
|
|
151
|
-
videoDetected = true;
|
|
152
|
-
detectionMethod = 'framesDecoded';
|
|
153
|
-
}
|
|
154
|
-
else if (report.framesReceived !== undefined &&
|
|
155
|
-
report.framesReceived > 0) {
|
|
156
|
-
videoDetected = true;
|
|
157
|
-
detectionMethod = 'framesReceived';
|
|
158
|
-
}
|
|
159
|
-
else if (report.bytesReceived > 0 &&
|
|
160
|
-
report.packetsReceived > 0 &&
|
|
161
|
-
// Additional check: ensure we've received enough data for actual video
|
|
162
|
-
report.bytesReceived > 100000 // rough threshold
|
|
163
|
-
) {
|
|
164
|
-
videoDetected = true;
|
|
165
|
-
detectionMethod = 'bytesReceived';
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
});
|
|
169
|
-
if (videoDetected && !this.successMetricFired) {
|
|
170
|
-
this.successMetricFired = true;
|
|
171
|
-
sendClientMetric(ClientMetricMeasurement.CLIENT_METRIC_MEASUREMENT_SESSION_SUCCESS, '1', detectionMethod ? { detectionMethod } : undefined);
|
|
172
|
-
if (this.successMetricPoller) {
|
|
173
|
-
clearInterval(this.successMetricPoller);
|
|
174
|
-
}
|
|
175
|
-
clearTimeout(timeoutId);
|
|
176
|
-
this.successMetricPoller = null;
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
catch (error) { }
|
|
180
|
-
}), 500);
|
|
181
|
-
}
|
|
182
|
-
muteInputAudio() {
|
|
183
|
-
const oldAudioState = this.inputAudioState;
|
|
184
|
-
const newAudioState = Object.assign(Object.assign({}, this.inputAudioState), { isMuted: true });
|
|
185
|
-
this.inputAudioState = newAudioState;
|
|
186
|
-
this.onInputAudioStateChange(oldAudioState, newAudioState);
|
|
187
|
-
return this.inputAudioState;
|
|
188
|
-
}
|
|
189
|
-
unmuteInputAudio() {
|
|
190
|
-
const oldAudioState = this.inputAudioState;
|
|
191
|
-
const newAudioState = Object.assign(Object.assign({}, this.inputAudioState), { isMuted: false });
|
|
192
|
-
this.inputAudioState = newAudioState;
|
|
193
|
-
this.onInputAudioStateChange(oldAudioState, newAudioState);
|
|
194
|
-
return this.inputAudioState;
|
|
195
|
-
}
|
|
196
|
-
getInputAudioState() {
|
|
197
|
-
return this.inputAudioState;
|
|
198
|
-
}
|
|
199
|
-
getPeerConnection() {
|
|
200
|
-
return this.peerConnection;
|
|
201
|
-
}
|
|
202
|
-
changeAudioInputDevice(deviceId) {
|
|
203
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
204
|
-
if (!this.peerConnection) {
|
|
205
|
-
throw new Error('StreamingClient - changeAudioInputDevice: peer connection is not initialized. Start streaming first.');
|
|
206
|
-
}
|
|
207
|
-
if (deviceId === null || deviceId === undefined) {
|
|
208
|
-
throw new Error('StreamingClient - changeAudioInputDevice: deviceId is required');
|
|
209
|
-
}
|
|
210
|
-
// Store the current mute state to preserve it
|
|
211
|
-
const wasMuted = this.inputAudioState.isMuted;
|
|
212
|
-
try {
|
|
213
|
-
// Stop the current audio stream tracks
|
|
214
|
-
if (this.inputAudioStream) {
|
|
215
|
-
this.inputAudioStream.getAudioTracks().forEach((track) => {
|
|
216
|
-
track.stop();
|
|
217
|
-
});
|
|
218
|
-
}
|
|
219
|
-
// Request new audio stream with the new device ID
|
|
220
|
-
const audioConstraints = {
|
|
221
|
-
echoCancellation: true,
|
|
222
|
-
deviceId: {
|
|
223
|
-
exact: deviceId,
|
|
224
|
-
},
|
|
225
|
-
};
|
|
226
|
-
this.inputAudioStream = yield navigator.mediaDevices.getUserMedia({
|
|
227
|
-
audio: audioConstraints,
|
|
228
|
-
});
|
|
229
|
-
// Update the stored device ID
|
|
230
|
-
this.audioDeviceId = deviceId;
|
|
231
|
-
// Replace the audio track in the peer connection
|
|
232
|
-
yield this.setupAudioTrack();
|
|
233
|
-
// Restore the mute state
|
|
234
|
-
if (wasMuted) {
|
|
235
|
-
this.muteAllAudioTracks();
|
|
236
|
-
}
|
|
237
|
-
// Emit event to notify that the device has changed
|
|
238
|
-
this.publicEventEmitter.emit(AnamEvent.INPUT_AUDIO_DEVICE_CHANGED, deviceId);
|
|
239
|
-
}
|
|
240
|
-
catch (error) {
|
|
241
|
-
console.error('Failed to change audio input device:', error);
|
|
242
|
-
throw new Error(`StreamingClient - changeAudioInputDevice: ${error instanceof Error ? error.message : String(error)}`);
|
|
243
|
-
}
|
|
244
|
-
});
|
|
245
|
-
}
|
|
246
|
-
getInputAudioStream() {
|
|
247
|
-
return this.inputAudioStream;
|
|
248
|
-
}
|
|
249
|
-
getVideoStream() {
|
|
250
|
-
return this.videoStream;
|
|
251
|
-
}
|
|
252
|
-
getAudioStream() {
|
|
253
|
-
return this.audioStream;
|
|
254
|
-
}
|
|
255
|
-
sendDataMessage(message) {
|
|
256
|
-
if (this.dataChannel && this.dataChannel.readyState === 'open') {
|
|
257
|
-
this.dataChannel.send(message);
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
setMediaStreamTargetById(videoElementId) {
|
|
261
|
-
// set up streaming targets
|
|
262
|
-
if (videoElementId) {
|
|
263
|
-
const videoElement = document.getElementById(videoElementId);
|
|
264
|
-
if (!videoElement) {
|
|
265
|
-
throw new Error(`StreamingClient: video element with id ${videoElementId} not found`);
|
|
266
|
-
}
|
|
267
|
-
this.videoElement = videoElement;
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
startConnection() {
|
|
271
|
-
try {
|
|
272
|
-
if (this.peerConnection) {
|
|
273
|
-
console.error('StreamingClient - startConnection: peer connection already exists');
|
|
274
|
-
return;
|
|
275
|
-
}
|
|
276
|
-
// start the connection
|
|
277
|
-
this.signallingClient.connect();
|
|
278
|
-
}
|
|
279
|
-
catch (error) {
|
|
280
|
-
console.error('StreamingClient - startConnection: error', error);
|
|
281
|
-
this.handleWebrtcFailure(error);
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
stopConnection() {
|
|
285
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
286
|
-
yield this.shutdown();
|
|
287
|
-
});
|
|
288
|
-
}
|
|
289
|
-
sendTalkCommand(content) {
|
|
290
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
291
|
-
if (!this.peerConnection) {
|
|
292
|
-
throw new Error('StreamingClient - sendTalkCommand: peer connection is null');
|
|
293
|
-
}
|
|
294
|
-
yield this.engineApiRestClient.sendTalkCommand(content);
|
|
295
|
-
return;
|
|
296
|
-
});
|
|
297
|
-
}
|
|
298
|
-
startTalkMessageStream(correlationId) {
|
|
299
|
-
if (!correlationId) {
|
|
300
|
-
// generate a random correlation uuid
|
|
301
|
-
correlationId = Math.random().toString(36).substring(2, 15);
|
|
302
|
-
}
|
|
303
|
-
return new TalkMessageStream(correlationId, this.internalEventEmitter, this.signallingClient);
|
|
304
|
-
}
|
|
305
|
-
createAgentAudioInputStream(config) {
|
|
306
|
-
this.agentAudioInputStream = new AgentAudioInputStream(config, this.signallingClient);
|
|
307
|
-
return this.agentAudioInputStream;
|
|
308
|
-
}
|
|
309
|
-
getAgentAudioInputStream() {
|
|
310
|
-
return this.agentAudioInputStream;
|
|
311
|
-
}
|
|
312
|
-
initPeerConnection() {
|
|
313
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
314
|
-
this.peerConnection = new RTCPeerConnection({
|
|
315
|
-
iceServers: this.iceServers,
|
|
316
|
-
iceCandidatePoolSize: ICE_CANDIDATE_POOL_SIZE,
|
|
317
|
-
});
|
|
318
|
-
// set event handlers
|
|
319
|
-
this.peerConnection.onicecandidate = this.onIceCandidate.bind(this);
|
|
320
|
-
this.peerConnection.oniceconnectionstatechange =
|
|
321
|
-
this.onIceConnectionStateChange.bind(this);
|
|
322
|
-
this.peerConnection.onconnectionstatechange =
|
|
323
|
-
this.onConnectionStateChange.bind(this);
|
|
324
|
-
this.peerConnection.addEventListener('track', this.onTrackEventHandler.bind(this));
|
|
325
|
-
// set up data channels
|
|
326
|
-
yield this.setupDataChannels();
|
|
327
|
-
// add transceivers
|
|
328
|
-
this.peerConnection.addTransceiver('video', { direction: 'recvonly' });
|
|
329
|
-
if (this.disableInputAudio) {
|
|
330
|
-
this.peerConnection.addTransceiver('audio', { direction: 'recvonly' });
|
|
331
|
-
}
|
|
332
|
-
else {
|
|
333
|
-
this.peerConnection.addTransceiver('audio', { direction: 'sendrecv' });
|
|
334
|
-
// Handle audio setup after transceivers are configured
|
|
335
|
-
if (this.inputAudioStream) {
|
|
336
|
-
// User provided an audio stream, set it up immediately
|
|
337
|
-
yield this.setupAudioTrack();
|
|
338
|
-
}
|
|
339
|
-
else {
|
|
340
|
-
// No user stream, start microphone permission request asynchronously
|
|
341
|
-
// Don't await - let it run in parallel with connection setup
|
|
342
|
-
this.requestMicrophonePermissionAsync().catch((error) => {
|
|
343
|
-
console.error('Async microphone permission request failed:', error);
|
|
344
|
-
});
|
|
345
|
-
}
|
|
346
|
-
}
|
|
347
|
-
});
|
|
348
|
-
}
|
|
349
|
-
onSignalMessage(signalMessage) {
|
|
350
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
351
|
-
if (!this.peerConnection) {
|
|
352
|
-
console.error('StreamingClient - onSignalMessage: peerConnection is not initialized');
|
|
353
|
-
return;
|
|
354
|
-
}
|
|
355
|
-
switch (signalMessage.actionType) {
|
|
356
|
-
case SignalMessageAction.ANSWER:
|
|
357
|
-
const answer = signalMessage.payload;
|
|
358
|
-
yield this.peerConnection.setRemoteDescription(answer);
|
|
359
|
-
this.connectionReceivedAnswer = true;
|
|
360
|
-
// flush the remote buffer
|
|
361
|
-
this.flushRemoteIceCandidateBuffer();
|
|
362
|
-
break;
|
|
363
|
-
case SignalMessageAction.ICE_CANDIDATE:
|
|
364
|
-
const iceCandidateConfig = signalMessage.payload;
|
|
365
|
-
const candidate = new RTCIceCandidate(iceCandidateConfig);
|
|
366
|
-
if (this.connectionReceivedAnswer) {
|
|
367
|
-
yield this.peerConnection.addIceCandidate(candidate);
|
|
368
|
-
}
|
|
369
|
-
else {
|
|
370
|
-
this.remoteIceCandidateBuffer.push(candidate);
|
|
371
|
-
}
|
|
372
|
-
break;
|
|
373
|
-
case SignalMessageAction.END_SESSION:
|
|
374
|
-
const reason = signalMessage.payload;
|
|
375
|
-
this.publicEventEmitter.emit(AnamEvent.CONNECTION_CLOSED, ConnectionClosedCode.SERVER_CLOSED_CONNECTION, reason);
|
|
376
|
-
// close the peer connection
|
|
377
|
-
this.shutdown();
|
|
378
|
-
break;
|
|
379
|
-
case SignalMessageAction.WARNING:
|
|
380
|
-
const message = signalMessage.payload;
|
|
381
|
-
console.warn('Warning received from server: ' + message);
|
|
382
|
-
this.publicEventEmitter.emit(AnamEvent.SERVER_WARNING, message);
|
|
383
|
-
break;
|
|
384
|
-
case SignalMessageAction.TALK_STREAM_INTERRUPTED:
|
|
385
|
-
const chatMessage = signalMessage.payload;
|
|
386
|
-
this.publicEventEmitter.emit(AnamEvent.TALK_STREAM_INTERRUPTED, chatMessage.correlationId);
|
|
387
|
-
break;
|
|
388
|
-
case SignalMessageAction.SESSION_READY:
|
|
389
|
-
const sessionId = signalMessage.sessionId;
|
|
390
|
-
this.publicEventEmitter.emit(AnamEvent.SESSION_READY, sessionId);
|
|
391
|
-
break;
|
|
392
|
-
case SignalMessageAction.HEARTBEAT:
|
|
393
|
-
break;
|
|
394
|
-
default:
|
|
395
|
-
console.warn('StreamingClient - onSignalMessage: unknown signal message action type. Is your anam-sdk version up to date?', signalMessage);
|
|
396
|
-
}
|
|
397
|
-
});
|
|
398
|
-
}
|
|
399
|
-
onSignallingClientConnected() {
|
|
400
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
401
|
-
if (!this.peerConnection) {
|
|
402
|
-
try {
|
|
403
|
-
yield this.initPeerConnectionAndSendOffer();
|
|
404
|
-
}
|
|
405
|
-
catch (err) {
|
|
406
|
-
console.error('StreamingClient - onSignallingClientConnected: Error initializing peer connection', err);
|
|
407
|
-
this.handleWebrtcFailure(err);
|
|
408
|
-
}
|
|
409
|
-
}
|
|
410
|
-
});
|
|
411
|
-
}
|
|
412
|
-
flushRemoteIceCandidateBuffer() {
|
|
413
|
-
this.remoteIceCandidateBuffer.forEach((candidate) => {
|
|
414
|
-
var _a;
|
|
415
|
-
(_a = this.peerConnection) === null || _a === void 0 ? void 0 : _a.addIceCandidate(candidate);
|
|
416
|
-
});
|
|
417
|
-
this.remoteIceCandidateBuffer = [];
|
|
418
|
-
}
|
|
419
|
-
/**
|
|
420
|
-
* ICE Candidate Trickle
|
|
421
|
-
* As each ICE candidate is gathered from the STUN server it is sent to the
|
|
422
|
-
* webRTC server immediately in an effort to reduce time to connection.
|
|
423
|
-
*/
|
|
424
|
-
onIceCandidate(event) {
|
|
425
|
-
if (event.candidate) {
|
|
426
|
-
this.signallingClient.sendIceCandidate(event.candidate);
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
onIceConnectionStateChange() {
|
|
430
|
-
var _a, _b;
|
|
431
|
-
if (((_a = this.peerConnection) === null || _a === void 0 ? void 0 : _a.iceConnectionState) === 'connected' ||
|
|
432
|
-
((_b = this.peerConnection) === null || _b === void 0 ? void 0 : _b.iceConnectionState) === 'completed') {
|
|
433
|
-
this.publicEventEmitter.emit(AnamEvent.CONNECTION_ESTABLISHED);
|
|
434
|
-
// Start collecting stats every 5 seconds
|
|
435
|
-
this.startStatsCollection();
|
|
436
|
-
}
|
|
437
|
-
}
|
|
438
|
-
onConnectionStateChange() {
|
|
439
|
-
var _a;
|
|
440
|
-
if (((_a = this.peerConnection) === null || _a === void 0 ? void 0 : _a.connectionState) === 'closed') {
|
|
441
|
-
console.error('StreamingClient - onConnectionStateChange: Connection closed');
|
|
442
|
-
this.handleWebrtcFailure('The connection to our servers was lost. Please try again.');
|
|
443
|
-
}
|
|
444
|
-
}
|
|
445
|
-
handleWebrtcFailure(err) {
|
|
446
|
-
console.error({ message: 'StreamingClient - handleWebrtcFailure: ', err });
|
|
447
|
-
if (err.name === 'NotAllowedError' && err.message === 'Permission denied') {
|
|
448
|
-
this.publicEventEmitter.emit(AnamEvent.CONNECTION_CLOSED, ConnectionClosedCode.MICROPHONE_PERMISSION_DENIED);
|
|
449
|
-
}
|
|
450
|
-
else {
|
|
451
|
-
this.publicEventEmitter.emit(AnamEvent.CONNECTION_CLOSED, ConnectionClosedCode.WEBRTC_FAILURE);
|
|
452
|
-
}
|
|
453
|
-
try {
|
|
454
|
-
this.stopConnection();
|
|
455
|
-
}
|
|
456
|
-
catch (error) {
|
|
457
|
-
console.error('StreamingClient - handleWebrtcFailure: error stopping connection', error);
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
onTrackEventHandler(event) {
|
|
461
|
-
if (event.track.kind === 'video') {
|
|
462
|
-
// start polling stats to detect successful video data received
|
|
463
|
-
this.startSuccessMetricPolling();
|
|
464
|
-
this.videoStream = event.streams[0];
|
|
465
|
-
this.publicEventEmitter.emit(AnamEvent.VIDEO_STREAM_STARTED, this.videoStream);
|
|
466
|
-
if (this.videoElement) {
|
|
467
|
-
this.videoElement.srcObject = this.videoStream;
|
|
468
|
-
const handle = this.videoElement.requestVideoFrameCallback(() => {
|
|
469
|
-
var _a;
|
|
470
|
-
// unregister the callback after the first frame
|
|
471
|
-
(_a = this.videoElement) === null || _a === void 0 ? void 0 : _a.cancelVideoFrameCallback(handle);
|
|
472
|
-
this.publicEventEmitter.emit(AnamEvent.VIDEO_PLAY_STARTED);
|
|
473
|
-
if (!this.successMetricFired) {
|
|
474
|
-
this.successMetricFired = true;
|
|
475
|
-
sendClientMetric(ClientMetricMeasurement.CLIENT_METRIC_MEASUREMENT_SESSION_SUCCESS, '1', { detectionMethod: 'videoElement' });
|
|
476
|
-
}
|
|
477
|
-
});
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
else if (event.track.kind === 'audio') {
|
|
481
|
-
this.audioStream = event.streams[0];
|
|
482
|
-
this.publicEventEmitter.emit(AnamEvent.AUDIO_STREAM_STARTED, this.audioStream);
|
|
483
|
-
}
|
|
484
|
-
}
|
|
485
|
-
/**
|
|
486
|
-
* Set up the data channels for sending and receiving messages
|
|
487
|
-
*/
|
|
488
|
-
setupDataChannels() {
|
|
489
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
490
|
-
if (!this.peerConnection) {
|
|
491
|
-
console.error('StreamingClient - setupDataChannels: peer connection is not initialized');
|
|
492
|
-
return;
|
|
493
|
-
}
|
|
494
|
-
/**
|
|
495
|
-
* Audio - Validate user-provided stream only
|
|
496
|
-
*
|
|
497
|
-
* If the user provided an audio stream, validate it has audio tracks
|
|
498
|
-
* Microphone permission request will be handled asynchronously
|
|
499
|
-
*/
|
|
500
|
-
if (!this.disableInputAudio && this.inputAudioStream) {
|
|
501
|
-
// verify the user provided stream has audio tracks
|
|
502
|
-
if (!this.inputAudioStream.getAudioTracks().length) {
|
|
503
|
-
throw new Error('StreamingClient - setupDataChannels: user provided stream does not have audio tracks');
|
|
504
|
-
}
|
|
505
|
-
}
|
|
506
|
-
/**
|
|
507
|
-
* Text
|
|
508
|
-
*
|
|
509
|
-
* Create the data channel for sending and receiving text.
|
|
510
|
-
* There is no input stream for text, instead the sending of data is triggered by a UI interaction.
|
|
511
|
-
*/
|
|
512
|
-
const dataChannel = this.peerConnection.createDataChannel('session', {
|
|
513
|
-
ordered: true,
|
|
514
|
-
});
|
|
515
|
-
dataChannel.onopen = () => {
|
|
516
|
-
this.dataChannel = dataChannel !== null && dataChannel !== void 0 ? dataChannel : null;
|
|
517
|
-
};
|
|
518
|
-
dataChannel.onclose = () => { };
|
|
519
|
-
// pass text message to the message history client
|
|
520
|
-
dataChannel.onmessage = (event) => {
|
|
521
|
-
try {
|
|
522
|
-
const message = JSON.parse(event.data);
|
|
523
|
-
// Handle known message types
|
|
524
|
-
switch (message.messageType) {
|
|
525
|
-
case DataChannelMessage.SPEECH_TEXT:
|
|
526
|
-
this.internalEventEmitter.emit(InternalEvent.WEBRTC_CHAT_MESSAGE_RECEIVED, message.data);
|
|
527
|
-
break;
|
|
528
|
-
case DataChannelMessage.CLIENT_TOOL_EVENT:
|
|
529
|
-
const webRtcToolEvent = message.data;
|
|
530
|
-
this.internalEventEmitter.emit(InternalEvent.WEBRTC_CLIENT_TOOL_EVENT_RECEIVED, webRtcToolEvent);
|
|
531
|
-
const clientToolEvent = ToolCallManager.WebRTCClientToolEventToClientToolEvent(webRtcToolEvent);
|
|
532
|
-
this.publicEventEmitter.emit(AnamEvent.CLIENT_TOOL_EVENT_RECEIVED, clientToolEvent);
|
|
533
|
-
break;
|
|
534
|
-
case DataChannelMessage.REASONING_TEXT:
|
|
535
|
-
this.publicEventEmitter.emit(AnamEvent.REASONING_TEXT_MESSAGE_RECEIVED, message.data);
|
|
536
|
-
break;
|
|
537
|
-
// Unknown message types are silently ignored to maintain forward compatibility
|
|
538
|
-
default:
|
|
539
|
-
break;
|
|
540
|
-
}
|
|
541
|
-
}
|
|
542
|
-
catch (error) {
|
|
543
|
-
console.error('Failed to parse data channel message:', error);
|
|
544
|
-
}
|
|
545
|
-
};
|
|
546
|
-
});
|
|
547
|
-
}
|
|
548
|
-
/**
|
|
549
|
-
* Request microphone permission asynchronously without blocking connection
|
|
550
|
-
*/
|
|
551
|
-
requestMicrophonePermissionAsync() {
|
|
552
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
553
|
-
if (this.inputAudioState.permissionState === AudioPermissionState.PENDING) {
|
|
554
|
-
return; // Already requesting
|
|
555
|
-
}
|
|
556
|
-
this.inputAudioState = Object.assign(Object.assign({}, this.inputAudioState), { permissionState: AudioPermissionState.PENDING });
|
|
557
|
-
this.publicEventEmitter.emit(AnamEvent.MIC_PERMISSION_PENDING);
|
|
558
|
-
try {
|
|
559
|
-
const audioConstraints = {
|
|
560
|
-
echoCancellation: true,
|
|
561
|
-
};
|
|
562
|
-
// If an audio device ID is provided in the options, use it
|
|
563
|
-
if (this.audioDeviceId) {
|
|
564
|
-
audioConstraints.deviceId = {
|
|
565
|
-
exact: this.audioDeviceId,
|
|
566
|
-
};
|
|
567
|
-
}
|
|
568
|
-
this.inputAudioStream = yield navigator.mediaDevices.getUserMedia({
|
|
569
|
-
audio: audioConstraints,
|
|
570
|
-
});
|
|
571
|
-
this.inputAudioState = Object.assign(Object.assign({}, this.inputAudioState), { permissionState: AudioPermissionState.GRANTED });
|
|
572
|
-
this.publicEventEmitter.emit(AnamEvent.MIC_PERMISSION_GRANTED);
|
|
573
|
-
// Now add the audio track to the existing connection
|
|
574
|
-
yield this.setupAudioTrack();
|
|
575
|
-
}
|
|
576
|
-
catch (error) {
|
|
577
|
-
console.error('Failed to get microphone permission:', error);
|
|
578
|
-
this.inputAudioState = Object.assign(Object.assign({}, this.inputAudioState), { permissionState: AudioPermissionState.DENIED });
|
|
579
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
580
|
-
this.publicEventEmitter.emit(AnamEvent.MIC_PERMISSION_DENIED, errorMessage);
|
|
581
|
-
}
|
|
582
|
-
});
|
|
583
|
-
}
|
|
584
|
-
/**
|
|
585
|
-
* Set up audio track and add it to the peer connection using replaceTrack
|
|
586
|
-
*/
|
|
587
|
-
setupAudioTrack() {
|
|
588
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
589
|
-
if (!this.peerConnection || !this.inputAudioStream) {
|
|
590
|
-
return;
|
|
591
|
-
}
|
|
592
|
-
// verify the stream has audio tracks
|
|
593
|
-
if (!this.inputAudioStream.getAudioTracks().length) {
|
|
594
|
-
console.error('StreamingClient - setupAudioTrack: stream does not have audio tracks');
|
|
595
|
-
return;
|
|
596
|
-
}
|
|
597
|
-
// mute the audio tracks if the user has muted the microphone
|
|
598
|
-
if (this.inputAudioState.isMuted) {
|
|
599
|
-
this.muteAllAudioTracks();
|
|
600
|
-
}
|
|
601
|
-
const audioTrack = this.inputAudioStream.getAudioTracks()[0];
|
|
602
|
-
// Find the audio sender
|
|
603
|
-
const existingSenders = this.peerConnection.getSenders();
|
|
604
|
-
const audioSender = existingSenders.find((sender) => {
|
|
605
|
-
var _a;
|
|
606
|
-
return ((_a = sender.track) === null || _a === void 0 ? void 0 : _a.kind) === 'audio' ||
|
|
607
|
-
(sender.track === null && sender.dtmf !== null);
|
|
608
|
-
});
|
|
609
|
-
if (audioSender) {
|
|
610
|
-
// Replace existing track (or null track) with our audio track
|
|
611
|
-
try {
|
|
612
|
-
yield audioSender.replaceTrack(audioTrack);
|
|
613
|
-
}
|
|
614
|
-
catch (error) {
|
|
615
|
-
console.error('Failed to replace audio track:', error);
|
|
616
|
-
// Fallback: add track normally
|
|
617
|
-
this.peerConnection.addTrack(audioTrack, this.inputAudioStream);
|
|
618
|
-
}
|
|
619
|
-
}
|
|
620
|
-
else {
|
|
621
|
-
// No audio sender found, add track normally
|
|
622
|
-
this.peerConnection.addTrack(audioTrack, this.inputAudioStream);
|
|
623
|
-
}
|
|
624
|
-
// pass the stream to the callback
|
|
625
|
-
this.publicEventEmitter.emit(AnamEvent.INPUT_AUDIO_STREAM_STARTED, this.inputAudioStream);
|
|
626
|
-
});
|
|
627
|
-
}
|
|
628
|
-
initPeerConnectionAndSendOffer() {
|
|
629
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
630
|
-
yield this.initPeerConnection();
|
|
631
|
-
if (!this.peerConnection) {
|
|
632
|
-
console.error('StreamingClient - initPeerConnectionAndSendOffer: peer connection is not initialized');
|
|
633
|
-
return;
|
|
634
|
-
}
|
|
635
|
-
// create offer and set local description
|
|
636
|
-
try {
|
|
637
|
-
const offer = yield this.peerConnection.createOffer();
|
|
638
|
-
yield this.peerConnection.setLocalDescription(offer);
|
|
639
|
-
}
|
|
640
|
-
catch (error) {
|
|
641
|
-
console.error('StreamingClient - initPeerConnectionAndSendOffer: error creating offer', error);
|
|
642
|
-
}
|
|
643
|
-
if (!this.peerConnection.localDescription) {
|
|
644
|
-
throw new Error('StreamingClient - initPeerConnectionAndSendOffer: local description is null');
|
|
645
|
-
}
|
|
646
|
-
yield this.signallingClient.sendOffer(this.peerConnection.localDescription);
|
|
647
|
-
});
|
|
648
|
-
}
|
|
649
|
-
shutdown() {
|
|
650
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
651
|
-
var _a;
|
|
652
|
-
if (this.showPeerConnectionStatsReport) {
|
|
653
|
-
const stats = yield ((_a = this.peerConnection) === null || _a === void 0 ? void 0 : _a.getStats());
|
|
654
|
-
if (stats) {
|
|
655
|
-
const report = createRTCStatsReport(stats, this.peerConnectionStatsReportOutputFormat);
|
|
656
|
-
if (report) {
|
|
657
|
-
console.log(report, undefined, 2);
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
}
|
|
661
|
-
// stop stats collection
|
|
662
|
-
if (this.statsCollectionInterval) {
|
|
663
|
-
clearInterval(this.statsCollectionInterval);
|
|
664
|
-
this.statsCollectionInterval = null;
|
|
665
|
-
}
|
|
666
|
-
// reset video frame polling
|
|
667
|
-
if (this.successMetricPoller) {
|
|
668
|
-
clearInterval(this.successMetricPoller);
|
|
669
|
-
this.successMetricPoller = null;
|
|
670
|
-
}
|
|
671
|
-
this.successMetricFired = false;
|
|
672
|
-
// stop the input audio stream
|
|
673
|
-
try {
|
|
674
|
-
if (this.inputAudioStream) {
|
|
675
|
-
this.inputAudioStream.getTracks().forEach((track) => {
|
|
676
|
-
track.stop();
|
|
677
|
-
});
|
|
678
|
-
}
|
|
679
|
-
this.inputAudioStream = null;
|
|
680
|
-
}
|
|
681
|
-
catch (error) {
|
|
682
|
-
console.error('StreamingClient - shutdown: error stopping input audio stream', error);
|
|
683
|
-
}
|
|
684
|
-
// stop the signalling client
|
|
685
|
-
try {
|
|
686
|
-
this.signallingClient.stop();
|
|
687
|
-
}
|
|
688
|
-
catch (error) {
|
|
689
|
-
console.error('StreamingClient - shutdown: error stopping signallilng', error);
|
|
690
|
-
}
|
|
691
|
-
// close the peer connection
|
|
692
|
-
try {
|
|
693
|
-
if (this.peerConnection &&
|
|
694
|
-
this.peerConnection.connectionState !== 'closed') {
|
|
695
|
-
this.peerConnection.onconnectionstatechange = null;
|
|
696
|
-
this.peerConnection.close();
|
|
697
|
-
this.peerConnection = null;
|
|
698
|
-
}
|
|
699
|
-
}
|
|
700
|
-
catch (error) {
|
|
701
|
-
console.error('StreamingClient - shutdown: error closing peer connection', error);
|
|
702
|
-
}
|
|
703
|
-
});
|
|
704
|
-
}
|
|
705
|
-
}
|
|
706
|
-
//# sourceMappingURL=StreamingClient.js.map
|