@unith-ai/core-client 2.0.0 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +224 -0
  2. package/dist/index.d.ts +88 -0
  3. package/dist/index.d.ts.map +1 -0
  4. package/dist/lib.js +2 -0
  5. package/dist/lib.js.map +1 -0
  6. package/dist/lib.module.js +2 -0
  7. package/dist/lib.module.js.map +1 -0
  8. package/dist/lib.umd.js +2 -0
  9. package/dist/lib.umd.js.map +1 -0
  10. package/dist/modules/audio.d.ts +18 -0
  11. package/dist/modules/audio.d.ts.map +1 -0
  12. package/dist/modules/av.d.ts +23 -0
  13. package/dist/modules/av.d.ts.map +1 -0
  14. package/dist/modules/connection.d.ts +21 -0
  15. package/dist/modules/connection.d.ts.map +1 -0
  16. package/dist/modules/environment.d.ts +4 -0
  17. package/dist/modules/environment.d.ts.map +1 -0
  18. package/dist/modules/event.d.ts +3 -0
  19. package/dist/modules/event.d.ts.map +1 -0
  20. package/dist/modules/idle-video.d.ts +9 -0
  21. package/dist/modules/idle-video.d.ts.map +1 -0
  22. package/dist/modules/microphone.d.ts +46 -0
  23. package/dist/modules/microphone.d.ts.map +1 -0
  24. package/dist/modules/monitor.d.ts +33 -0
  25. package/dist/modules/monitor.d.ts.map +1 -0
  26. package/dist/modules/sync.d.ts +15 -0
  27. package/dist/modules/sync.d.ts.map +1 -0
  28. package/dist/modules/user.d.ts +28 -0
  29. package/dist/modules/user.d.ts.map +1 -0
  30. package/dist/modules/video.d.ts +62 -0
  31. package/dist/modules/video.d.ts.map +1 -0
  32. package/dist/modules/vp8.d.ts +42 -0
  33. package/dist/modules/vp8.d.ts.map +1 -0
  34. package/dist/types/Conversation.d.ts +67 -0
  35. package/dist/types/Conversation.d.ts.map +1 -0
  36. package/dist/types/User.d.ts +44 -0
  37. package/dist/types/User.d.ts.map +1 -0
  38. package/dist/types/audio.d.ts +17 -0
  39. package/dist/types/audio.d.ts.map +1 -0
  40. package/dist/types/chat.d.ts +29 -0
  41. package/dist/types/chat.d.ts.map +1 -0
  42. package/dist/types/connection.d.ts +14 -0
  43. package/dist/types/connection.d.ts.map +1 -0
  44. package/dist/types/environment.d.ts +2 -0
  45. package/dist/types/environment.d.ts.map +1 -0
  46. package/dist/types/event.d.ts +142 -0
  47. package/dist/types/event.d.ts.map +1 -0
  48. package/dist/types/idle-video.d.ts +8 -0
  49. package/dist/types/idle-video.d.ts.map +1 -0
  50. package/dist/types/microphone.d.ts +26 -0
  51. package/dist/types/microphone.d.ts.map +1 -0
  52. package/dist/types/sync.d.ts +24 -0
  53. package/dist/types/sync.d.ts.map +1 -0
  54. package/dist/types/vp8.d.ts +49 -0
  55. package/dist/types/vp8.d.ts.map +1 -0
  56. package/dist/utils/audio.d.ts +9 -0
  57. package/dist/utils/audio.d.ts.map +1 -0
  58. package/dist/utils/microphone.d.ts +21 -0
  59. package/dist/utils/microphone.d.ts.map +1 -0
  60. package/dist/utils/storage.d.ts +5 -0
  61. package/dist/utils/storage.d.ts.map +1 -0
  62. package/dist/utils/sync.d.ts +3 -0
  63. package/dist/utils/sync.d.ts.map +1 -0
  64. package/dist/utils/worklet-loader.d.ts +2 -0
  65. package/dist/utils/worklet-loader.d.ts.map +1 -0
  66. package/package.json +31 -23
  67. package/eslint.config.cjs +0 -1
@@ -0,0 +1,46 @@
1
+ import { MicrophoneEvents, MicrophoneStatus, MicrophoneProvider, ElevenLabsOptions, MicTokenResponseType } from "../types/microphone";
2
+ import { User } from "./user";
3
+ import { HeadType } from "..";
4
+ export declare class Microphone {
5
+ provider: MicrophoneProvider;
6
+ options: MicrophoneEvents;
7
+ elevenLabsOptions: ElevenLabsOptions;
8
+ user: User;
9
+ private readonly headInfo;
10
+ private readonly sendMessage;
11
+ private microphoneStatus;
12
+ private microphoneAccess;
13
+ private recognizer;
14
+ private connection;
15
+ private tokenObj;
16
+ private micBeep;
17
+ /**
18
+ * This static method initializes the Microphone instance.
19
+ * It gets the tokens from our endpoints and sets up the necessary configurations.
20
+ * @param options
21
+ * @param provider
22
+ * @param elevenLabsOptions
23
+ * @param user
24
+ * @param headInfo
25
+ * @param microphoneAccess
26
+ * @param sendMessage
27
+ * @returns
28
+ */
29
+ static initializeMicrophone(options: MicrophoneEvents, provider: MicrophoneProvider, elevenLabsOptions: ElevenLabsOptions, user: User, headInfo: HeadType, microphoneAccess: boolean, sendMessage: (message: string) => void): Promise<Microphone>;
30
+ constructor(provider: MicrophoneProvider, options: MicrophoneEvents, elevenLabsOptions: ElevenLabsOptions, user: User, headInfo: HeadType, microphoneAccess: boolean, token: MicTokenResponseType, sendMessage: (message: string) => void);
31
+ private updateMicrophoneStatus;
32
+ private handleRecognitionResult;
33
+ private startAzureMicrophone;
34
+ private stopAzureMicrophone;
35
+ private startElevenLabsMicrophone;
36
+ private stopElevenLabsMicrophone;
37
+ /**
38
+ * Toggles the microphone on or off.
39
+ */
40
+ toggleMicrophone(): Promise<void>;
41
+ /**
42
+ * Gets the current status of the microphone.
43
+ */
44
+ status(): MicrophoneStatus;
45
+ }
46
+ //# sourceMappingURL=microphone.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"microphone.d.ts","sourceRoot":"","sources":["../../src/modules/microphone.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,oBAAoB,EACrB,MAAM,qBAAqB,CAAC;AAQ7B,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAS9B,qBAAa,UAAU;IA2DZ,QAAQ,EAAE,kBAAkB;IAC5B,OAAO,EAAE,gBAAgB;IACzB,iBAAiB,EAAE,iBAAiB;IACpC,IAAI,EAAE,IAAI;IACjB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAGzB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAjE9B,OAAO,CAAC,gBAAgB,CAA2B;IACnD,OAAO,CAAC,gBAAgB,CAAkB;IAC1C,OAAO,CAAC,UAAU,CAAiC;IACnD,OAAO,CAAC,UAAU,CAAmC;IACrD,OAAO,CAAC,QAAQ,CAId;IACF,OAAO,CAAC,OAAO,CAA6D;IAC5E;;;;;;;;;;;OAWG;WACiB,oBAAoB,CACtC,OAAO,EAAE,gBAAgB,EACzB,QAAQ,EAAE,kBAAkB,EAC5B,iBAAiB,EAAE,iBAAiB,EACpC,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,QAAQ,EAClB,gBAAgB,EAAE,OAAO,EACzB,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,GACrC,OAAO,CAAC,UAAU,CAAC;gBA4Bb,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,EAAE,gBAAgB,EACzB,iBAAiB,EAAE,iBAAiB,EACpC,IAAI,EAAE,IAAI,EACA,QAAQ,EAAE,QAAQ,EACnC,gBAAgB,EAAE,OAAO,EACzB,KAAK,EAAE,oBAAoB,EACV,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI;IAWzD,OAAO,CAAC,sBAAsB;IAK9B,OAAO,CAAC,uBAAuB;YAOjB,oBAAoB;YAuDpB,mBAAmB;YAenB,yBAAyB;YAkDzB,wBAAwB;IAetC;;OAEG;IACU,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IA8B9C;;OAEG;IACI,MAAM;CAGd"}
@@ -0,0 +1,33 @@
1
+ import { Connection } from "./connection";
2
+ type LatencyStatus = "good" | "moderate" | "poor";
3
+ interface LatencyStats {
4
+ rtt: number;
5
+ average: number;
6
+ status: LatencyStatus;
7
+ }
8
+ type OnUpdateCallback = (stats: LatencyStats) => void;
9
+ export declare class LatencyMonitor {
10
+ connection: Connection;
11
+ private intervalId;
12
+ private pingInterval;
13
+ private timeout;
14
+ private lastPingTimestamp;
15
+ private history;
16
+ private maxHistory;
17
+ private onUpdate?;
18
+ constructor(connection: Connection, options?: {
19
+ pingInterval?: number;
20
+ timeout?: number;
21
+ maxHistory?: number;
22
+ onUpdate?: OnUpdateCallback;
23
+ });
24
+ start(): void;
25
+ stop(): void;
26
+ destroy(): void;
27
+ private sendPing;
28
+ private handleMessage;
29
+ private recordLatency;
30
+ private classifyLatency;
31
+ }
32
+ export {};
33
+ //# sourceMappingURL=monitor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"monitor.d.ts","sourceRoot":"","sources":["../../src/modules/monitor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,KAAK,aAAa,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,CAAC;AAElD,UAAU,YAAY;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,aAAa,CAAC;CACvB;AAED,KAAK,gBAAgB,GAAG,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;AAEtD,qBAAa,cAAc;IAUhB,UAAU,EAAE,UAAU;IAT/B,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,iBAAiB,CAAe;IACxC,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAC,CAAmB;gBAG3B,UAAU,EAAE,UAAU,EAC7B,OAAO,CAAC,EAAE;QACR,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,CAAC,EAAE,gBAAgB,CAAC;KAC7B;IAUH,KAAK;IAQL,IAAI;IAKJ,OAAO;IAIP,OAAO,CAAC,QAAQ;IAqBhB,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,aAAa;IAmBrB,OAAO,CAAC,eAAe;CAKxB"}
@@ -0,0 +1,15 @@
1
+ import { SyncConfig } from "../types/sync";
2
+ export declare class SyncController {
3
+ private config;
4
+ private driftHistory;
5
+ private correctionInProgress;
6
+ private lastAudioTiming;
7
+ private lastVideoTiming;
8
+ constructor(config: SyncConfig);
9
+ updateAudioTime(relativeTime: number): void;
10
+ updateVideoTime(relativeTime: number): void;
11
+ resetTiming(): void;
12
+ checkSync(): Promise<void>;
13
+ private recordDrift;
14
+ }
15
+ //# sourceMappingURL=sync.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/modules/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,UAAU,EAAc,MAAM,eAAe,CAAC;AAEpE,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,YAAY,CAAqB;IACzC,OAAO,CAAC,oBAAoB,CAAkB;IAC9C,OAAO,CAAC,eAAe,CAA2B;IAClD,OAAO,CAAC,eAAe,CAA2B;gBAEtC,MAAM,EAAE,UAAU;IAIvB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAO3C,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAO3C,WAAW,IAAI,IAAI;IAKb,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBvC,OAAO,CAAC,WAAW;CAapB"}
@@ -0,0 +1,28 @@
1
+ import { ApiAsrTokenType, HeadType, TokenResponseType } from "../types/User";
2
+ export declare class User {
3
+ readonly id: string;
4
+ username: string;
5
+ readonly password: string;
6
+ readonly orgId: string;
7
+ readonly headId: string;
8
+ readonly apiBase: string;
9
+ static loginUser(username: string, password: string, apiUrl: string, orgId: string, headId: string): Promise<User>;
10
+ private EXPIRATION_OFFSET;
11
+ accessToken: string;
12
+ tokenType: string;
13
+ sessionId: number;
14
+ private constructor();
15
+ getHeadDetails(apiKey: string): Promise<HeadType>;
16
+ getAuthToken(username: string, password: string): Promise<TokenResponseType>;
17
+ getProviderToken(auth_jwt: string, provider: "elevenlabs" | "azure"): Promise<ApiAsrTokenType>;
18
+ getAccessToken(): Promise<{
19
+ access_token: string;
20
+ user_id: string | null | undefined;
21
+ session_id: number;
22
+ }>;
23
+ getAsrToken(provider: "elevenlabs" | "azure"): Promise<{
24
+ token: string;
25
+ region: string;
26
+ }>;
27
+ }
28
+ //# sourceMappingURL=user.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"user.d.ts","sourceRoot":"","sources":["../../src/modules/user.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EAGf,QAAQ,EACR,iBAAiB,EAClB,MAAM,eAAe,CAAC;AAIvB,qBAAa,IAAI;aA0DG,EAAE,EAAE,MAAM;IACnB,QAAQ,EAAE,MAAM;aACP,QAAQ,EAAE,MAAM;aAChB,KAAK,EAAE,MAAM;aACb,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,MAAM;WA9Db,SAAS,CAC3B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;IA2ChB,OAAO,CAAC,iBAAiB,CAAkB;IACpC,WAAW,EAAE,MAAM,CAAM;IACzB,SAAS,EAAE,MAAM,CAAM;IACvB,SAAS,EAAE,MAAM,CAAK;IAE7B,OAAO;IAoBM,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;IA4BjD,YAAY,CACvB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,iBAAiB,CAAC;IAyBhB,gBAAgB,CAC3B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,YAAY,GAAG,OAAO,GAC/B,OAAO,CAAC,eAAe,CAAC;IAsBd,cAAc;;;;;IA+Dd,WAAW,CAAC,QAAQ,EAAE,YAAY,GAAG,OAAO;;;;CAuC1D"}
@@ -0,0 +1,62 @@
1
+ import { IdleVideoConfig } from "../types/idle-video";
2
+ import { VideoFormatConfig, VideoTransitionType } from "../types/vp8";
3
+ export interface VideoEffectConfig {
4
+ background?: BackgroundConfig;
5
+ }
6
+ export interface BackgroundConfig {
7
+ type: "color" | "image";
8
+ value: number | string;
9
+ }
10
+ export declare class VideoOutput {
11
+ private videoOutput;
12
+ private container;
13
+ private idleVideo;
14
+ private cachedVideo;
15
+ private idleVideoConfig;
16
+ private videoTransitionConfig;
17
+ private readonly CROSSFADE_DURATION;
18
+ private isTransitioning;
19
+ private isShowingIdleVideo;
20
+ private bufferCheckAnimationId;
21
+ private lastBufferCheckTime;
22
+ private sessionStarted;
23
+ private isShowingCachedVideo;
24
+ private onIdleVideoShown?;
25
+ private onIdleVideoHidden?;
26
+ private onSpeakingStartCallback;
27
+ private onSpeakingEndCallback;
28
+ static createVideoOutput(container: HTMLElement, config: VideoFormatConfig & {
29
+ effects?: VideoEffectConfig;
30
+ idleVideo: IdleVideoConfig;
31
+ transition: VideoTransitionType;
32
+ }): Promise<VideoOutput>;
33
+ private constructor();
34
+ private setupIdleVideo;
35
+ private setupCachedVideo;
36
+ onSpeakingStart(speakingStartCallback: () => void): void;
37
+ onSpeakingEnd(speakingEndCallback: () => void): void;
38
+ startStreaming(init?: boolean): void;
39
+ stopBufferMonitoring(): void;
40
+ getBufferLength(): number;
41
+ startBufferMonitoring(): void;
42
+ toggleCacheVideoMute(): void;
43
+ playCachedVideo(cachedVideoUrl: string): Promise<void>;
44
+ private crossfadeFromIdleToCached;
45
+ private crossfadeFromCachedToIdle;
46
+ private cleanupCachedVideo;
47
+ private hideIdleVideo;
48
+ private hideIdleVideoBeforeStream;
49
+ setEventCallbacks(callbacks: {
50
+ onIdleVideoShown?: () => void;
51
+ onIdleVideoHidden?: () => void;
52
+ }): void;
53
+ getStreamingStatus(): boolean;
54
+ showIdleVideo(): Promise<void>;
55
+ showIdleVideoAfterStream(): Promise<void>;
56
+ addFrame(frameData: Blob | ArrayBuffer, timeStamp: number, isKeyframe?: boolean): Promise<void>;
57
+ clearFrame(): void;
58
+ toggleStream(status: boolean): Promise<void>;
59
+ interrupt(fadeOut?: boolean): void;
60
+ destroy(): void;
61
+ }
62
+ //# sourceMappingURL=video.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"video.d.ts","sourceRoot":"","sources":["../../src/modules/video.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EACL,iBAAiB,EAEjB,mBAAmB,EACpB,MAAM,cAAc,CAAC;AAEtB,MAAM,WAAW,iBAAiB;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,WAAW,CAAiB;IACpC,OAAO,CAAC,SAAS,CAAc;IAC/B,OAAO,CAAC,SAAS,CAAiC;IAClD,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,eAAe,CAAgC;IAGvD,OAAO,CAAC,qBAAqB,CAAoC;IACjE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAO;IAC1C,OAAO,CAAC,eAAe,CAAkB;IAGzC,OAAO,CAAC,kBAAkB,CAAiB;IAC3C,OAAO,CAAC,sBAAsB,CAAuB;IACrD,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,cAAc,CAAkB;IAGxC,OAAO,CAAC,oBAAoB,CAAkB;IAG9C,OAAO,CAAC,gBAAgB,CAAC,CAAa;IACtC,OAAO,CAAC,iBAAiB,CAAC,CAAa;IAEvC,OAAO,CAAC,uBAAuB,CAA6B;IAC5D,OAAO,CAAC,qBAAqB,CAA6B;WAEtC,iBAAiB,CACnC,SAAS,EAAE,WAAW,EACtB,MAAM,EAAE,iBAAiB,GAAG;QAC1B,OAAO,CAAC,EAAE,iBAAiB,CAAC;QAC5B,SAAS,EAAE,eAAe,CAAC;QAC3B,UAAU,EAAE,mBAAmB,CAAC;KACjC;IAiBH,OAAO;YAWO,cAAc;YA2Cd,gBAAgB;IA4BvB,eAAe,CAAC,qBAAqB,EAAE,MAAM,IAAI,GAAG,IAAI;IAIxD,aAAa,CAAC,mBAAmB,EAAE,MAAM,IAAI,GAAG,IAAI;IAIpD,cAAc,CAAC,IAAI,UAAQ,GAAG,IAAI;IAMlC,oBAAoB,IAAI,IAAI;IAS5B,eAAe,IAAI,MAAM;IAIzB,qBAAqB,IAAI,IAAI;IAyB7B,oBAAoB,IAAI,IAAI;IAMtB,eAAe,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YA4DrD,yBAAyB;YA4BzB,yBAAyB;YA+BzB,kBAAkB;IAUhC,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,yBAAyB;IAS1B,iBAAiB,CAAC,SAAS,EAAE;QAClC,gBAAgB,CAAC,EAAE,MAAM,IAAI,CAAC;QAC9B,iBAAiB,CAAC,EAAE,MAAM,IAAI,CAAC;KAChC,GAAG,IAAI;IAKD,kBAAkB,IAAI,OAAO;IAIvB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAiB9B,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBzC,QAAQ,CACnB,SAAS,EAAE,IAAI,GAAG,WAAW,EAC7B,SAAS,EAAE,MAAM,EACjB,UAAU,CAAC,EAAE,OAAO,GACnB,OAAO,CAAC,IAAI,CAAC;IAST,UAAU,IAAI,IAAI;IAKZ,YAAY,CAAC,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAIlD,SAAS,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,IAAI;IAIlC,OAAO,IAAI,IAAI;CAqBvB"}
@@ -0,0 +1,42 @@
1
+ import { VideoState, VideoFormatConfig } from "../types/vp8";
2
+ export declare class Vp8VideoOutput {
3
+ private canvas;
4
+ private ctx;
5
+ private container;
6
+ private config;
7
+ private resizeObserver;
8
+ private decoder;
9
+ private state;
10
+ private isProcessingFrame;
11
+ private isStreaming;
12
+ private startTime;
13
+ private frameBuffer;
14
+ private currentSequenceId;
15
+ private animationFrameId;
16
+ private renderLoop;
17
+ private fpsCounter;
18
+ private handleContextLoss;
19
+ private handleContextRestore;
20
+ private constructor();
21
+ static create(container: HTMLElement, config: VideoFormatConfig): Promise<Vp8VideoOutput>;
22
+ private initializeDecoder;
23
+ private setupResizeHandling;
24
+ private handleResize;
25
+ getBufferLength(): number;
26
+ getStreamingStatus(): boolean;
27
+ addFrame(frameData: Uint8Array<ArrayBuffer>, timestamp: number, isKeyframe?: boolean): Promise<void>;
28
+ toggleStream(status: boolean): Promise<void>;
29
+ clearFrame(): void;
30
+ interrupt(fadeOut?: boolean): void;
31
+ destroy(): void;
32
+ getState(): VideoState;
33
+ startRenderingStreamingVideo(init: boolean): void;
34
+ private render;
35
+ private processNextFrame;
36
+ private decodeVP8Frame;
37
+ private renderVideoFrame;
38
+ private clearCanvas;
39
+ private fadeOutCanvas;
40
+ private setupContextLossHandling;
41
+ }
42
+ //# sourceMappingURL=vp8.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vp8.d.ts","sourceRoot":"","sources":["../../src/modules/vp8.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,UAAU,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAExE,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,GAAG,CAA2B;IACtC,OAAO,CAAC,SAAS,CAAc;IAC/B,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,cAAc,CAA+B;IAGrD,OAAO,CAAC,OAAO,CAA6B;IAG5C,OAAO,CAAC,KAAK,CAAuC;IACpD,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,SAAS,CAAa;IAI9B,OAAO,CAAC,WAAW,CAAmB;IACtC,OAAO,CAAC,iBAAiB,CAAa;IAGtC,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,UAAU,CAAkB;IAGpC,OAAO,CAAC,UAAU,CAGhB;IAGF,OAAO,CAAC,iBAAiB,CAAoC;IAC7D,OAAO,CAAC,oBAAoB,CAAoC;IAEhE,OAAO;WAoBa,MAAM,CACxB,SAAS,EAAE,WAAW,EACtB,MAAM,EAAE,iBAAiB,GACxB,OAAO,CAAC,cAAc,CAAC;YAsCZ,iBAAiB;IAkC/B,OAAO,CAAC,mBAAmB;IAe3B,OAAO,CAAC,YAAY,CAelB;IAEK,eAAe,IAAI,MAAM;IAIzB,kBAAkB,IAAI,OAAO;IAIvB,QAAQ,CACnB,SAAS,EAAE,UAAU,CAAC,WAAW,CAAC,EAClC,SAAS,EAAE,MAAM,EACjB,UAAU,GAAE,OAAe,GAC1B,OAAO,CAAC,IAAI,CAAC;IAoCH,YAAY,CAAC,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAIlD,UAAU,IAAI,IAAI;IAsClB,SAAS,CAAC,OAAO,GAAE,OAAe,GAAG,IAAI;IAyBzC,OAAO,IAAI,IAAI;IAoCf,QAAQ,IAAI,UAAU;IAItB,4BAA4B,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI;IASxD,OAAO,CAAC,MAAM;YAwBA,gBAAgB;YAkChB,cAAc;IAgB5B,OAAO,CAAC,gBAAgB;IAiCxB,OAAO,CAAC,WAAW;YAIL,aAAa;IAkB3B,OAAO,CAAC,wBAAwB;CAmBjC"}
@@ -0,0 +1,67 @@
1
+ import { Environment } from "./environment";
2
+ import { MessageEventData, OnDisconnectCallback } from "./event";
3
+ import { ElevenLabsOptions, MicrophoneEvents, MicrophoneProvider } from "./microphone";
4
+ import { ConnectHeadType } from "./User";
5
+ import { VideoTransitionType } from "./vp8";
6
+ export type Language = "ar-AE" | "bg-BG" | "bs-BA" | "cs-CZ" | "de-DE" | "en-US" | "es-ES" | "fr-FR" | "hu-HU" | "id-ID" | "it-IT" | "ka-GE" | "kk-KZ" | "lt-LT" | "lv-LV" | "nl-NL" | "pl-PL" | "pt-PT" | "ro-RO" | "ru-RU" | "sk-SK" | "sr-RS" | "th-TH" | "uk-UA";
7
+ export type Status = "connecting" | "connected" | "disconnecting" | "disconnected";
8
+ export type DigitalHumanOptions = {
9
+ username: string;
10
+ apiKey: string;
11
+ environment: Environment;
12
+ mode: ConversationMode;
13
+ language?: Language;
14
+ allowWakeLock?: boolean;
15
+ fadeTransitionsType?: VideoTransitionType;
16
+ };
17
+ export type HeadOptions = {
18
+ orgId: string;
19
+ headId: string;
20
+ };
21
+ export type VideoHtmlElement = {
22
+ element: HTMLElement;
23
+ };
24
+ export type KeepSessionEventData = {
25
+ granted: boolean;
26
+ };
27
+ export type ConversationOptions = DigitalHumanOptions & HeadOptions;
28
+ export type MicrophoneType = {
29
+ microphoneProvider: MicrophoneProvider;
30
+ microphoneOptions: MicrophoneEvents;
31
+ elevenLabsOptions?: ElevenLabsOptions;
32
+ };
33
+ export type ConversationEvents = {
34
+ onStatusChange: (prop: {
35
+ status: Status;
36
+ }) => void;
37
+ onConnect: (prop: {
38
+ userId: string;
39
+ headInfo: ConnectHeadType;
40
+ microphoneAccess: boolean;
41
+ }) => void;
42
+ onDisconnect: OnDisconnectCallback;
43
+ onMessage: (prop: MessageEventData) => void;
44
+ onSuggestions: (prop: {
45
+ suggestions: string[];
46
+ }) => void;
47
+ onTimeoutWarning: () => void;
48
+ onTimeout: () => void;
49
+ onMuteStatusChange: (prop: {
50
+ isMuted: boolean;
51
+ }) => void;
52
+ onSpeakingStart: () => void;
53
+ onSpeakingEnd: () => void;
54
+ onStoppingEnd: () => void;
55
+ onKeepSession: (prop: KeepSessionEventData) => void;
56
+ onError: (prop: {
57
+ message: string;
58
+ endConversation: boolean;
59
+ type: "toast" | "modal" | "silent";
60
+ }) => void;
61
+ };
62
+ export type ConversationMode = "chat" | "audio" | "video" | "default";
63
+ export type Role = "user" | "ai";
64
+ export type Mode = "speaking" | "listening" | "thinking" | "stopping";
65
+ export type StreamingType = "jpg" | "vp8";
66
+ export type QualityType = "low" | "standard" | "high" | "premium";
67
+ //# sourceMappingURL=Conversation.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Conversation.d.ts","sourceRoot":"","sources":["../../src/types/Conversation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAGL,gBAAgB,EAChB,oBAAoB,EAErB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,kBAAkB,EACnB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAY,MAAM,QAAQ,CAAC;AACnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,OAAO,CAAC;AAE5C,MAAM,MAAM,QAAQ,GAChB,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,CAAC;AAEZ,MAAM,MAAM,MAAM,GACd,YAAY,GACZ,WAAW,GACX,eAAe,GACf,cAAc,CAAC;AAEnB,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,WAAW,CAAC;IACzB,IAAI,EAAE,gBAAgB,CAAC;IACvB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CAC3C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,WAAW,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,mBAAmB,GAAG,WAAW,CAAC;AAEpE,MAAM,MAAM,cAAc,GAAG;IAC3B,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,iBAAiB,EAAE,gBAAgB,CAAC;IACpC,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACnD,SAAS,EAAE,CAAC,IAAI,EAAE;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,eAAe,CAAC;QAC1B,gBAAgB,EAAE,OAAO,CAAC;KAC3B,KAAK,IAAI,CAAC;IACX,YAAY,EAAE,oBAAoB,CAAC;IACnC,SAAS,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC5C,aAAa,EAAE,CAAC,IAAI,EAAE;QAAE,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,KAAK,IAAI,CAAC;IACzD,gBAAgB,EAAE,MAAM,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,IAAI,CAAC;IACtB,kBAAkB,EAAE,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;IACzD,eAAe,EAAE,MAAM,IAAI,CAAC;IAC5B,aAAa,EAAE,MAAM,IAAI,CAAC;IAC1B,aAAa,EAAE,MAAM,IAAI,CAAC;IAC1B,aAAa,EAAE,CAAC,IAAI,EAAE,oBAAoB,KAAK,IAAI,CAAC;IACpD,OAAO,EAAE,CAAC,IAAI,EAAE;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,eAAe,EAAE,OAAO,CAAC;QACzB,IAAI,EAAE,OAAO,GAAG,OAAO,GAAG,QAAQ,CAAC;KACpC,KAAK,IAAI,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAEtE,MAAM,MAAM,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;AAEjC,MAAM,MAAM,IAAI,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,UAAU,CAAC;AAEtE,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,KAAK,CAAC;AAE1C,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,UAAU,GAAG,MAAM,GAAG,SAAS,CAAC"}
@@ -0,0 +1,44 @@
1
+ import { Language } from "./Conversation";
2
+ export type ApiErrorType = {
3
+ response?: {
4
+ data?: {
5
+ detail?: string;
6
+ };
7
+ status_code?: number;
8
+ status?: number;
9
+ };
10
+ name?: string;
11
+ };
12
+ export type TokenResponseType = {
13
+ access_token: string;
14
+ user_id: string;
15
+ token_type: string;
16
+ };
17
+ export type HeadType = {
18
+ alias: string;
19
+ icon_config: undefined;
20
+ lang_speech_recognition: string;
21
+ language: Language;
22
+ name: string;
23
+ operation_mode: "ttt" | "oc" | "doc_qa";
24
+ phrases: string[];
25
+ streaming_enabled: boolean;
26
+ allowed_iframe_origins: string[];
27
+ avatarSrc?: string;
28
+ };
29
+ export type ConnectHeadType = {
30
+ name: string;
31
+ phrases: string[];
32
+ language: Language;
33
+ avatar?: string;
34
+ };
35
+ export type AuthTokenType = {
36
+ exp: number;
37
+ username: string;
38
+ admin: boolean;
39
+ };
40
+ export type ApiAsrTokenType = {
41
+ token: string;
42
+ region: string;
43
+ };
44
+ //# sourceMappingURL=User.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"User.d.ts","sourceRoot":"","sources":["../../src/types/User.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,EAAE;QACT,IAAI,CAAC,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,SAAS,CAAC;IACvB,uBAAuB,EAAE,MAAM,CAAC;IAChC,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,KAAK,GAAG,IAAI,GAAG,QAAQ,CAAC;IACxC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,sBAAsB,EAAE,MAAM,EAAE,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAE9E,MAAM,MAAM,eAAe,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC"}
@@ -0,0 +1,17 @@
1
+ export type AudioFormatConfig = {
2
+ sampleRate: number;
3
+ format: "pcm" | "wav" | "ulaw";
4
+ };
5
+ export type ResultType = {
6
+ text: string;
7
+ confidence: string;
8
+ isFinal: boolean;
9
+ };
10
+ export declare enum STATUS_TYPES {
11
+ INACTIVE = "INACTIVE",
12
+ INITIALIZING = "INITIALIZING",
13
+ READY = "READY",
14
+ LISTENING = "LISTENING",
15
+ ERROR = "ERROR"
16
+ }
17
+ //# sourceMappingURL=audio.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"audio.d.ts","sourceRoot":"","sources":["../../src/types/audio.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,iBAAiB,GAAG;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,oBAAY,YAAY;IACtB,QAAQ,aAAa;IACrB,YAAY,iBAAiB;IAC7B,KAAK,UAAU;IACf,SAAS,cAAc;IACvB,KAAK,UAAU;CAChB"}
@@ -0,0 +1,29 @@
1
+ export type ChatItem = {
2
+ user_id?: string;
3
+ username?: string;
4
+ speaker?: "user" | "backend";
5
+ text?: string;
6
+ event?: string;
7
+ error_reason?: string;
8
+ timestamp?: Date;
9
+ url?: string;
10
+ session_id?: string;
11
+ type?: string;
12
+ media?: string[];
13
+ is_last?: boolean;
14
+ messages?: ChatItem[];
15
+ id?: number | string;
16
+ isSent?: boolean;
17
+ disabled?: boolean;
18
+ visible?: boolean;
19
+ matches_suggestion?: boolean;
20
+ userFeedback?: boolean;
21
+ input_message_id?: string;
22
+ part_order?: number;
23
+ received_message_id?: number;
24
+ streaming?: boolean;
25
+ stream_id?: string;
26
+ pointer?: string[];
27
+ };
28
+ export type ChatHistory = Map<string, ChatItem>;
29
+ //# sourceMappingURL=chat.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/types/chat.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,QAAQ,GAAG;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGtB,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAGlB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC"}
@@ -0,0 +1,14 @@
1
+ import { ConversationMode, Language } from "./Conversation";
2
+ import { Environment } from "./environment";
3
+ export type WebsocketConnectionConfig = {
4
+ environment: Environment;
5
+ orgId: string;
6
+ headId: string;
7
+ token: string;
8
+ apiKey: string;
9
+ mode: ConversationMode;
10
+ language: Language;
11
+ onJoin?: () => void;
12
+ onError?: (error: string) => void;
13
+ };
14
+ //# sourceMappingURL=connection.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"connection.d.ts","sourceRoot":"","sources":["../../src/types/connection.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,QAAQ,EAGT,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,MAAM,yBAAyB,GAAG;IACtC,WAAW,EAAE,WAAW,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,gBAAgB,CAAC;IACvB,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,IAAI,CAAC;IACpB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACnC,CAAC"}
@@ -0,0 +1,2 @@
1
+ export type Environment = "production" | "staging" | "development";
2
+ //# sourceMappingURL=environment.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"environment.d.ts","sourceRoot":"","sources":["../../src/types/environment.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,WAAW,GAAG,YAAY,GAAG,SAAS,GAAG,aAAa,CAAC"}
@@ -0,0 +1,142 @@
1
+ export declare enum EventType {
2
+ TEXT = "text",
3
+ CONVERSATION_END = "conversation_end",
4
+ JOIN = "join",
5
+ ERROR = "error",
6
+ TIME_OUT = "timeout",
7
+ UNITH_NLP_EXCEPTION = "unith_nlp_exception",
8
+ ANALYTICS = "analytics-userFeedback",
9
+ CHOICE = "choice",
10
+ TIMEOUT_WARNING = "timeout_warning",
11
+ KEEP_SESSION = "keep_session",
12
+ RESPONSE = "response",
13
+ STREAMING = "streaming",
14
+ PING = "ping",
15
+ PONG = "pong",
16
+ BINARY = "binary"
17
+ }
18
+ export declare enum StreamingEventType {
19
+ VIDEO_FRAME = "video_frame",
20
+ AUDIO_FRAME = "audio_frame",
21
+ METADATA = "metadata",
22
+ ERROR = "error",
23
+ CACHE = "cache"
24
+ }
25
+ export interface Message {
26
+ id: number;
27
+ timestamp: string;
28
+ speaker: string;
29
+ text: string;
30
+ isSent: boolean;
31
+ user_id: string;
32
+ username?: string;
33
+ event: EventType.TEXT | EventType.KEEP_SESSION;
34
+ session_id?: string;
35
+ visible: boolean;
36
+ }
37
+ export type JoinEventData = {
38
+ granted: boolean;
39
+ };
40
+ export type GlobalEventData = {
41
+ event: EventType;
42
+ user_id: string;
43
+ username: string;
44
+ type?: string;
45
+ };
46
+ export type BinaryEventData = {
47
+ data: ArrayBuffer;
48
+ };
49
+ export type SpeakerType = "user" | "ai";
50
+ export type TextEventData = {
51
+ id: string;
52
+ timestamp: Date;
53
+ speaker: SpeakerType;
54
+ text: string;
55
+ isSent: boolean;
56
+ session_id: string;
57
+ visible: boolean;
58
+ is_last?: boolean;
59
+ part_order?: number;
60
+ input_message_id?: string;
61
+ suggestions: string[];
62
+ has_error?: boolean;
63
+ error_code?: number;
64
+ error_message?: string;
65
+ media?: string[];
66
+ video?: string;
67
+ stream_id?: string;
68
+ };
69
+ export type ConversationEndEventData = {
70
+ reason: string;
71
+ duration: number;
72
+ };
73
+ export type TimeoutEventData = {
74
+ remaining_time: number;
75
+ };
76
+ export type AnalyticsEventData = {
77
+ feedback_type: string;
78
+ rating: number;
79
+ comment?: string;
80
+ };
81
+ export type ChoiceEventData = {
82
+ choices: string[];
83
+ selected_choice?: string;
84
+ };
85
+ export type StreamingEventData = {
86
+ message?: string;
87
+ metadata_type?: string;
88
+ session_id?: string;
89
+ suggestions: string[];
90
+ event_timestamp_ms: number;
91
+ type: StreamingEventType;
92
+ video_url?: string;
93
+ text?: string;
94
+ error_type?: "resource_exhausted" | "deadline_exceeded" | "inactivity_timeout";
95
+ format?: "jpeg" | "wav";
96
+ frame_data?: string;
97
+ height?: number;
98
+ width?: string;
99
+ sample_rate?: string;
100
+ duration_ms?: number;
101
+ is_keyframe?: boolean;
102
+ };
103
+ export type PingEventData = {
104
+ event: EventType.PING;
105
+ timestamp: string;
106
+ id: string;
107
+ };
108
+ export type PongEventData = {
109
+ type: EventType.PONG;
110
+ timestamp: string;
111
+ id: string;
112
+ };
113
+ export type MessageEventData = {
114
+ timestamp: Date;
115
+ sender: SpeakerType;
116
+ text: string;
117
+ visible: boolean;
118
+ };
119
+ export type IncomingSocketEvent<T extends EventType = EventType> = T extends EventType.BINARY ? GlobalEventData & BinaryEventData : T extends EventType.JOIN ? GlobalEventData & JoinEventData : T extends EventType.TEXT ? GlobalEventData & TextEventData : T extends EventType.RESPONSE ? GlobalEventData & TextEventData : T extends EventType.CONVERSATION_END ? GlobalEventData & ConversationEndEventData : T extends EventType.STREAMING ? GlobalEventData & StreamingEventData : T extends EventType.TIME_OUT ? GlobalEventData & TimeoutEventData : T extends EventType.TIMEOUT_WARNING ? GlobalEventData & TimeoutEventData : T extends EventType.KEEP_SESSION ? GlobalEventData & JoinEventData : T extends EventType.ANALYTICS ? GlobalEventData & AnalyticsEventData : T extends EventType.PING ? GlobalEventData & PingEventData : T extends EventType.PONG ? GlobalEventData & PongEventData : T extends EventType.CHOICE ? GlobalEventData & ChoiceEventData : GlobalEventData;
120
+ export type AnyIncomingSocketEvent = IncomingSocketEvent<EventType.JOIN> | IncomingSocketEvent<EventType.TEXT> | IncomingSocketEvent<EventType.RESPONSE> | IncomingSocketEvent<EventType.STREAMING> | IncomingSocketEvent<EventType.BINARY> | IncomingSocketEvent<EventType.CONVERSATION_END> | IncomingSocketEvent<EventType.TIME_OUT> | IncomingSocketEvent<EventType.TIMEOUT_WARNING> | IncomingSocketEvent<EventType.KEEP_SESSION> | IncomingSocketEvent<EventType.ANALYTICS> | IncomingSocketEvent<EventType.CHOICE> | IncomingSocketEvent<EventType.PING> | IncomingSocketEvent<EventType.PONG> | (GlobalEventData & {
121
+ event: Exclude<EventType, EventType.JOIN | EventType.TEXT | EventType.CONVERSATION_END | EventType.TIME_OUT | EventType.TIMEOUT_WARNING | EventType.ANALYTICS | EventType.CHOICE | EventType.RESPONSE | EventType.STREAMING | EventType.PING | EventType.KEEP_SESSION | EventType.BINARY>;
122
+ });
123
+ export type DisconnectionDetails = {
124
+ reason: "error" | "user";
125
+ message?: string;
126
+ context?: Event;
127
+ };
128
+ export type OnDisconnectCallback = (details: DisconnectionDetails) => void;
129
+ export type OnMessageCallback = (event: AnyIncomingSocketEvent) => void;
130
+ export type OnPingPongCallback = (event: PongEventData) => void;
131
+ export declare function isJoinEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.JOIN>;
132
+ export declare function isTextEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.TEXT>;
133
+ export declare function isResponseEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.RESPONSE>;
134
+ export declare function isStreamingEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.STREAMING>;
135
+ export declare function isStreamingErrorEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.STREAMING>;
136
+ export declare function isBinaryEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.BINARY>;
137
+ export declare function isConversationEndEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.CONVERSATION_END>;
138
+ export declare function isPongEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.PONG>;
139
+ export declare function isTimeoutWarningEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.TIMEOUT_WARNING>;
140
+ export declare function isTimeoutEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.TIME_OUT>;
141
+ export declare function isKeepSessionEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.KEEP_SESSION>;
142
+ //# sourceMappingURL=event.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"event.d.ts","sourceRoot":"","sources":["../../src/types/event.ts"],"names":[],"mappings":"AAAA,oBAAY,SAAS;IACnB,IAAI,SAAS;IACb,gBAAgB,qBAAqB;IACrC,IAAI,SAAS;IACb,KAAK,UAAU;IACf,QAAQ,YAAY;IACpB,mBAAmB,wBAAwB;IAC3C,SAAS,2BAA2B;IACpC,MAAM,WAAW;IACjB,eAAe,oBAAoB;IACnC,YAAY,iBAAiB;IAC7B,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,MAAM,WAAW;CAClB;AAED,oBAAY,kBAAkB;IAC5B,WAAW,gBAAgB;IAC3B,WAAW,gBAAgB;IAC3B,QAAQ,aAAa;IACrB,KAAK,UAAU;IACf,KAAK,UAAU;CAChB;AACD,MAAM,WAAW,OAAO;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,SAAS,CAAC,IAAI,GAAG,SAAS,CAAC,YAAY,CAAC;IAE/C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,SAAS,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,IAAI,CAAC;AAExC,MAAM,MAAM,aAAa,GAAG;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,WAAW,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAGF,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,IAAI,EAAE,kBAAkB,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EACP,oBAAoB,GACpB,mBAAmB,GACnB,oBAAoB,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,SAAS,EAAE,IAAI,CAAC;IAChB,MAAM,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,mBAAmB,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS,IAC7D,CAAC,SAAS,SAAS,CAAC,MAAM,GACtB,eAAe,GAAG,eAAe,GACjC,CAAC,SAAS,SAAS,CAAC,IAAI,GACtB,eAAe,GAAG,aAAa,GAC/B,CAAC,SAAS,SAAS,CAAC,IAAI,GACtB,eAAe,GAAG,aAAa,GAC/B,CAAC,SAAS,SAAS,CAAC,QAAQ,GAC1B,eAAe,GAAG,aAAa,GAC/B,CAAC,SAAS,SAAS,CAAC,gBAAgB,GAClC,eAAe,GAAG,wBAAwB,GAC1C,CAAC,SAAS,SAAS,CAAC,SAAS,GAC3B,eAAe,GAAG,kBAAkB,GACpC,CAAC,SAAS,SAAS,CAAC,QAAQ,GAC1B,eAAe,GAAG,gBAAgB,GAClC,CAAC,SAAS,SAAS,CAAC,eAAe,GACjC,eAAe,GAAG,gBAAgB,GAClC,CAAC,SAAS,SAAS,CAAC,YAAY,GAC9B,eAAe,GAAG,aAAa,GAC/B,CAAC,SAAS,SAAS,CAAC,SAAS,GAC3B,eAAe,GAAG,kBAAkB,GACpC,CAAC,SAAS,SAAS,CAAC,IAAI,GACtB,eAAe,GAAG,aAAa,GAC/B,CAAC,SAAS,SAAS,CAAC,IAAI,GACtB,eAAe,GAAG,aAAa,GAC/B,CAAC,SAAS,SAAS,CAAC,MAAM,GACxB,eAAe,GAAG,eAAe,GACjC,eAAe,CAAC;AAE9C,MAAM,MAAM,sBAAsB,GAC9B,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,GACnC,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,GACnC,mBAAmB,CAAC,SAAS,CAAC,QAAQ,CAAC,GACvC,mBAAmB,CAAC,SAAS,CAAC,SAAS,CAAC,GACxC,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,GACrC,mBAAmB,CAAC,SAAS,CAAC,gBAAgB,CAAC,GAC/C,mBAAmB,CAAC,SAAS,CAAC,QAAQ,CAAC,GACvC,mBAAmB,CAAC,SAAS,CAAC,eAAe,CAAC,GAC9C,mBAAmB,CAAC,SAAS,CAAC,YAAY,CAAC,GAC3C,mBAAmB,CAAC,SAAS,CAAC,SAAS,CAAC,GACxC,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,GACrC,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,GACnC,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,GACnC,CAAC,eAAe,GAAG;IACjB,KAAK,EAAE,OAAO,CACZ,SAAS,EACP,SAAS,CAAC,IAAI,GACd,SAAS,CAAC,IAAI,GACd,SAAS,CAAC,gBAAgB,GAC1B,SAAS,CAAC,QAAQ,GAClB,SAAS,CAAC,eAAe,GACzB,SAAS,CAAC,SAAS,GACnB,SAAS,CAAC,MAAM,GAChB,SAAS,CAAC,QAAQ,GAClB,SAAS,CAAC,SAAS,GACnB,SAAS,CAAC,IAAI,GACd,SAAS,CAAC,YAAY,GACtB,SAAS,CAAC,MAAM,CACnB,CAAC;CACH,CAAC,CAAC;AAEP,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,KAAK,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,EAAE,oBAAoB,KAAK,IAAI,CAAC;AAC3E,MAAM,MAAM,iBAAiB,GAAG,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;AACxE,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAGhE,wBAAgB,WAAW,CACzB,KAAK,EAAE,sBAAsB,GAC5B,KAAK,IAAI,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,CAE9C;AAED,wBAAgB,WAAW,CACzB,KAAK,EAAE,sBAAsB,GAC5B,KAAK,IAAI,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,CAE9C;AAED,wBAAgB,eAAe,CAC7B,KAAK,EAAE,sBAAsB,GAC5B,KAAK,IAAI,mBAAmB,CAAC,SAAS,CAAC,QAAQ,CAAC,CAElD;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,sBAAsB,GAC5B,KAAK,IAAI,mBAAmB,CAAC,SAAS,CAAC,SAAS,CAAC,CAEnD;AACD,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,sBAAsB,GAC5B,KAAK,IAAI,mBAAmB,CAAC,SAAS,CAAC,SAAS,CAAC,CAKnD;AACD,wBAAgB,aAAa,CAC3B,KAAK,EAAE,sBAAsB,GAC5B,KAAK,IAAI,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,CAEhD;AAED,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,sBAAsB,GAC5B,KAAK,IAAI,mBAAmB,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAE1D;AAED,wBAAgB,WAAW,CACzB,KAAK,EAAE,sBAAsB,GAC5B,KAAK,IAAI,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,CAE9C;AAED,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,sBAAsB,GAC5B,KAAK,IAAI,mBAAmB,CAAC,SAAS,CAAC,eAAe,CAAC,CAEzD;AAED,wBAAgB,cAAc,CAC5B,KAAK,EAAE,sBAAsB,GAC5B,KAAK,IAAI,mBAAmB,CAAC,SAAS,CAAC,QAAQ,CAAC,CAElD;AAED,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,sBAAsB,GAC5B,KAAK,IAAI,mBAAmB,CAAC,SAAS,CAAC,YAAY,CAAC,CAEtD"}
@@ -0,0 +1,8 @@
1
+ export type ObjectOfStrings = {
2
+ [key: string]: string;
3
+ };
4
+ export type IdleVideoConfig = {
5
+ src: string;
6
+ enabled: boolean;
7
+ };
8
+ //# sourceMappingURL=idle-video.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"idle-video.d.ts","sourceRoot":"","sources":["../../src/types/idle-video.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,eAAe,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,CAAC;AAExD,MAAM,MAAM,eAAe,GAAG;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC"}