agora-appbuilder-core 4.1.26 → 4.1.27-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/template/agora-rn-uikit/src/Rtc/Create.tsx +9 -0
- package/template/android/app/src/main/assets/fonts/icomoon.ttf +0 -0
- package/template/customization-api/index.ts +1 -0
- package/template/customization-api/logger.ts +33 -0
- package/template/defaultConfig.js +2 -2
- package/template/src/assets/font-styles.css +4 -0
- package/template/src/assets/fonts/icomoon.ttf +0 -0
- package/template/src/assets/selection.json +1 -1
- package/template/src/atoms/CustomIcon.tsx +1 -0
- package/template/src/logger/AppBuilderLogger.tsx +3 -4
- package/template/src/subComponents/caption/Transcript.tsx +37 -0
- package/template/src/subComponents/caption/__tests__/transcriptJourney.test.ts +7 -0
- package/template/src/subComponents/caption/useCaption.tsx +144 -42
- package/template/src/subComponents/caption/useSTTAPI.tsx +13 -12
- package/template/src/subComponents/caption/useStreamMessageUtils.ts +109 -8
|
@@ -38,6 +38,8 @@ export declare const StatusTypes: {
|
|
|
38
38
|
|
|
39
39
|
export type StatusType = (typeof StatusTypes)[keyof typeof StatusTypes];
|
|
40
40
|
|
|
41
|
+
export type CustomizationLogType = string;
|
|
42
|
+
|
|
41
43
|
export enum LogSource {
|
|
42
44
|
AgoraSDK = 'Agora-SDK',
|
|
43
45
|
/** Logs related to all features */
|
|
@@ -107,10 +109,7 @@ type LogType = {
|
|
|
107
109
|
| 'recording_delete'
|
|
108
110
|
| 'ban_user';
|
|
109
111
|
[LogSource.Events]: 'CUSTOM_EVENTS' | 'RTM_EVENTS';
|
|
110
|
-
[LogSource.CustomizationAPI]:
|
|
111
|
-
| 'Log'
|
|
112
|
-
| 'AI_AGENT_CUSTOMIZATION'
|
|
113
|
-
| 'EXTERNAL_CUSTOMIZATION';
|
|
112
|
+
[LogSource.CustomizationAPI]: CustomizationLogType;
|
|
114
113
|
[LogSource.SDK]: 'Log' | 'Event';
|
|
115
114
|
};
|
|
116
115
|
|
|
@@ -23,6 +23,8 @@ import {
|
|
|
23
23
|
import {TranscriptHeader} from '../../pages/video-call/SidePanelHeader';
|
|
24
24
|
import {useContent} from 'customization-api';
|
|
25
25
|
import {TranscriptItem, useCaption} from './useCaption';
|
|
26
|
+
import {logger, LogSource} from '../../logger/AppBuilderLogger';
|
|
27
|
+
import {TRANSCRIPT_JOURNEY} from './transcriptJourney';
|
|
26
28
|
import {TranscriptText} from './TranscriptText';
|
|
27
29
|
import PrimaryButton from '../../atoms/PrimaryButton';
|
|
28
30
|
import ThemeConfig from '../../theme';
|
|
@@ -54,6 +56,7 @@ const Transcript = (props: TranscriptProps) => {
|
|
|
54
56
|
isLangChangeInProgress,
|
|
55
57
|
getBotOwnerUid,
|
|
56
58
|
isSTTActive,
|
|
59
|
+
isSTTListenerAdded,
|
|
57
60
|
selectedTranslationLanguage,
|
|
58
61
|
transcriptViewMode,
|
|
59
62
|
} = useCaption();
|
|
@@ -69,6 +72,40 @@ const Transcript = (props: TranscriptProps) => {
|
|
|
69
72
|
const data = meetingTranscript; // Object.entries(transcript);
|
|
70
73
|
console.log('[STT_GLOBAL] meetingTranscript data: ', data);
|
|
71
74
|
|
|
75
|
+
const hasLoggedFirstEntryRef = React.useRef(false);
|
|
76
|
+
const transcriptEntryCountRef = React.useRef(data.length);
|
|
77
|
+
transcriptEntryCountRef.current = data.length;
|
|
78
|
+
|
|
79
|
+
React.useEffect(() => {
|
|
80
|
+
logger.log(
|
|
81
|
+
LogSource.Internals,
|
|
82
|
+
'TRANSCRIPT',
|
|
83
|
+
`${TRANSCRIPT_JOURNEY} transcript panel state`,
|
|
84
|
+
{
|
|
85
|
+
stage: 'transcript_panel',
|
|
86
|
+
isSTTActive,
|
|
87
|
+
listenerRegistered: isSTTListenerAdded,
|
|
88
|
+
transcriptEntryCount: transcriptEntryCountRef.current,
|
|
89
|
+
},
|
|
90
|
+
);
|
|
91
|
+
}, [isSTTActive, isSTTListenerAdded]);
|
|
92
|
+
|
|
93
|
+
React.useEffect(() => {
|
|
94
|
+
if (data.length > 0 && !hasLoggedFirstEntryRef.current) {
|
|
95
|
+
hasLoggedFirstEntryRef.current = true;
|
|
96
|
+
logger.log(
|
|
97
|
+
LogSource.Internals,
|
|
98
|
+
'TRANSCRIPT',
|
|
99
|
+
`${TRANSCRIPT_JOURNEY} first transcript entry available to panel`,
|
|
100
|
+
{
|
|
101
|
+
stage: 'transcript_panel_data',
|
|
102
|
+
outcome: 'success',
|
|
103
|
+
transcriptEntryCount: data.length,
|
|
104
|
+
},
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
}, [data.length]);
|
|
108
|
+
|
|
72
109
|
const [showButton, setShowButton] = React.useState(false);
|
|
73
110
|
|
|
74
111
|
const contentHeightRef = React.useRef(0);
|
|
@@ -625,22 +625,21 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
625
625
|
try {
|
|
626
626
|
setIsLangChangeInProgress(true);
|
|
627
627
|
const result = await start(localBotUidRef.current, newConfig);
|
|
628
|
-
console.log('[STT] start result: ', result);
|
|
629
628
|
if (result.success) {
|
|
630
629
|
setIsSTTError(false);
|
|
631
630
|
logger.log(
|
|
632
631
|
LogSource.NetworkRest,
|
|
633
632
|
'stt',
|
|
634
|
-
|
|
635
|
-
result.data,
|
|
633
|
+
`${TRANSCRIPT_JOURNEY} STT started successfully`,
|
|
634
|
+
result.data ?? null,
|
|
636
635
|
);
|
|
637
636
|
} else {
|
|
638
637
|
setIsSTTError(true);
|
|
639
638
|
logger.error(
|
|
640
639
|
LogSource.NetworkRest,
|
|
641
640
|
'stt',
|
|
642
|
-
|
|
643
|
-
result.error,
|
|
641
|
+
`${TRANSCRIPT_JOURNEY} failed to start STT`,
|
|
642
|
+
result.error ?? null,
|
|
644
643
|
);
|
|
645
644
|
Toast.show({
|
|
646
645
|
leadingIconName: 'alert',
|
|
@@ -657,7 +656,12 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
657
656
|
} catch (error: any) {
|
|
658
657
|
setIsLangChangeInProgress(false);
|
|
659
658
|
setIsSTTError(true);
|
|
660
|
-
logger.error(
|
|
659
|
+
logger.error(
|
|
660
|
+
LogSource.NetworkRest,
|
|
661
|
+
'stt',
|
|
662
|
+
`${TRANSCRIPT_JOURNEY} STT start error`,
|
|
663
|
+
error ?? null,
|
|
664
|
+
);
|
|
661
665
|
// Show error toast: text1 = translated label, text2 = exception error
|
|
662
666
|
Toast.show({
|
|
663
667
|
leadingIconName: 'alert',
|
|
@@ -702,16 +706,16 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
702
706
|
logger.log(
|
|
703
707
|
LogSource.NetworkRest,
|
|
704
708
|
'stt',
|
|
705
|
-
|
|
706
|
-
result.data,
|
|
709
|
+
`${TRANSCRIPT_JOURNEY} STT updated successfully`,
|
|
710
|
+
result.data ?? null,
|
|
707
711
|
);
|
|
708
712
|
} else {
|
|
709
713
|
setIsSTTError(true);
|
|
710
714
|
logger.error(
|
|
711
715
|
LogSource.NetworkRest,
|
|
712
716
|
'stt',
|
|
713
|
-
|
|
714
|
-
result.error,
|
|
717
|
+
`${TRANSCRIPT_JOURNEY} failed to update STT`,
|
|
718
|
+
result.error ?? null,
|
|
715
719
|
);
|
|
716
720
|
// Show error toast: text1 = translated label, text2 = API error
|
|
717
721
|
Toast.show({
|
|
@@ -729,7 +733,12 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
729
733
|
} catch (error: any) {
|
|
730
734
|
setIsLangChangeInProgress(false);
|
|
731
735
|
setIsSTTError(true);
|
|
732
|
-
logger.error(
|
|
736
|
+
logger.error(
|
|
737
|
+
LogSource.NetworkRest,
|
|
738
|
+
'stt',
|
|
739
|
+
`${TRANSCRIPT_JOURNEY} STT update error`,
|
|
740
|
+
error ?? null,
|
|
741
|
+
);
|
|
733
742
|
// Show error toast: text1 = translated label, text2 = exception error
|
|
734
743
|
Toast.show({
|
|
735
744
|
leadingIconName: 'alert',
|
|
@@ -748,10 +757,20 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
748
757
|
};
|
|
749
758
|
|
|
750
759
|
const stopSTTBotSession = React.useCallback(async () => {
|
|
751
|
-
|
|
760
|
+
logger.log(
|
|
761
|
+
LogSource.Internals,
|
|
762
|
+
'TRANSCRIPT',
|
|
763
|
+
`${TRANSCRIPT_JOURNEY} STT stop requested`,
|
|
764
|
+
{stage: 'stt_stop', localBotUid: localBotUidRef.current ?? null},
|
|
765
|
+
);
|
|
752
766
|
|
|
753
767
|
if (!localBotUidRef.current) {
|
|
754
|
-
|
|
768
|
+
logger.warn(
|
|
769
|
+
LogSource.Internals,
|
|
770
|
+
'TRANSCRIPT',
|
|
771
|
+
`${TRANSCRIPT_JOURNEY} STT stop skipped because bot UID is missing`,
|
|
772
|
+
{stage: 'stt_stop', outcome: 'skipped'},
|
|
773
|
+
);
|
|
755
774
|
return;
|
|
756
775
|
}
|
|
757
776
|
|
|
@@ -767,16 +786,16 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
767
786
|
logger.log(
|
|
768
787
|
LogSource.NetworkRest,
|
|
769
788
|
'stt',
|
|
770
|
-
|
|
771
|
-
result.data,
|
|
789
|
+
`${TRANSCRIPT_JOURNEY} STT stopped successfully`,
|
|
790
|
+
result.data ?? null,
|
|
772
791
|
);
|
|
773
792
|
} else {
|
|
774
793
|
setIsSTTError(true);
|
|
775
794
|
logger.error(
|
|
776
795
|
LogSource.NetworkRest,
|
|
777
796
|
'stt',
|
|
778
|
-
|
|
779
|
-
result.error,
|
|
797
|
+
`${TRANSCRIPT_JOURNEY} failed to stop STT`,
|
|
798
|
+
result.error ?? null,
|
|
780
799
|
);
|
|
781
800
|
}
|
|
782
801
|
} catch (error) {
|
|
@@ -784,8 +803,8 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
784
803
|
logger.error(
|
|
785
804
|
LogSource.NetworkRest,
|
|
786
805
|
'stt',
|
|
787
|
-
|
|
788
|
-
error,
|
|
806
|
+
`${TRANSCRIPT_JOURNEY} error in stopSTTBotSession`,
|
|
807
|
+
error ?? null,
|
|
789
808
|
);
|
|
790
809
|
}
|
|
791
810
|
}, [stop]);
|
|
@@ -856,12 +875,29 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
856
875
|
// Queue
|
|
857
876
|
enqueueSttEvent(updatedState, true, targetChange);
|
|
858
877
|
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
878
|
+
logger.log(
|
|
879
|
+
LogSource.Internals,
|
|
880
|
+
'TRANSCRIPT',
|
|
881
|
+
`${TRANSCRIPT_JOURNEY} local spoken-language state queued`,
|
|
882
|
+
{
|
|
883
|
+
stage: 'global_state_queue',
|
|
884
|
+
outcome: 'queued',
|
|
885
|
+
spokenLanguage: updatedState.globalSpokenLanguage ?? null,
|
|
886
|
+
translationTargetCount:
|
|
887
|
+
updatedState.globalTranslationTargets.length,
|
|
888
|
+
},
|
|
862
889
|
);
|
|
863
890
|
} catch (error) {
|
|
864
|
-
|
|
891
|
+
logger.error(
|
|
892
|
+
LogSource.Internals,
|
|
893
|
+
'TRANSCRIPT',
|
|
894
|
+
`${TRANSCRIPT_JOURNEY} local spoken-language state failed`,
|
|
895
|
+
{
|
|
896
|
+
stage: 'global_state_queue',
|
|
897
|
+
outcome: 'failure',
|
|
898
|
+
error: error ?? null,
|
|
899
|
+
},
|
|
900
|
+
);
|
|
865
901
|
}
|
|
866
902
|
},
|
|
867
903
|
[localUid], // only real dependency
|
|
@@ -916,15 +952,24 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
916
952
|
// Queues all local + remote stt events
|
|
917
953
|
const enqueueSttEvent = React.useCallback(
|
|
918
954
|
(state: GlobalSttState, isLocal: boolean, targetChange?: TargetChange) => {
|
|
919
|
-
console.log(
|
|
920
|
-
'[STT_GLOBAL] inside enqueueSttEvent - sttDepsReadyRef flag',
|
|
921
|
-
sttDepsReadyRef.current,
|
|
922
|
-
);
|
|
923
955
|
sttEventQueueRef.current.push({
|
|
924
956
|
state,
|
|
925
957
|
isLocal,
|
|
926
958
|
targetChange,
|
|
927
959
|
});
|
|
960
|
+
logger.log(
|
|
961
|
+
LogSource.Internals,
|
|
962
|
+
'TRANSCRIPT',
|
|
963
|
+
`${TRANSCRIPT_JOURNEY} STT global state queued`,
|
|
964
|
+
{
|
|
965
|
+
stage: 'global_state_queue',
|
|
966
|
+
outcome: 'queued',
|
|
967
|
+
stateOrigin: isLocal ? 'local' : 'remote',
|
|
968
|
+
sttEnabled: state.globalSttEnabled,
|
|
969
|
+
sttDepsReady: sttDepsReadyRef.current,
|
|
970
|
+
queueSize: sttEventQueueRef.current.length,
|
|
971
|
+
},
|
|
972
|
+
);
|
|
928
973
|
if (sttDepsReadyRef.current) {
|
|
929
974
|
processSttEventQueue();
|
|
930
975
|
}
|
|
@@ -951,7 +996,12 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
951
996
|
!globalSttStateRef.current.globalSttEnabled &&
|
|
952
997
|
sttEventQueueRef.current.length === 0
|
|
953
998
|
) {
|
|
954
|
-
|
|
999
|
+
logger.log(
|
|
1000
|
+
LogSource.Internals,
|
|
1001
|
+
'TRANSCRIPT',
|
|
1002
|
+
`${TRANSCRIPT_JOURNEY} STT auto-start state injected`,
|
|
1003
|
+
{stage: 'global_state_queue', outcome: 'queued'},
|
|
1004
|
+
);
|
|
955
1005
|
|
|
956
1006
|
sttAutoStartGuardRef.current = true;
|
|
957
1007
|
|
|
@@ -978,7 +1028,12 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
978
1028
|
const {state: newState, isLocal, targetChange} = item;
|
|
979
1029
|
const prevState = globalSttStateRef.current;
|
|
980
1030
|
if (isSameState(prevState, newState)) {
|
|
981
|
-
|
|
1031
|
+
logger.log(
|
|
1032
|
+
LogSource.Internals,
|
|
1033
|
+
'TRANSCRIPT',
|
|
1034
|
+
`${TRANSCRIPT_JOURNEY} duplicate STT global state skipped`,
|
|
1035
|
+
{stage: 'global_state_queue', outcome: 'skipped'},
|
|
1036
|
+
);
|
|
982
1037
|
continue; // no call to processGlobalSttSingleEvent
|
|
983
1038
|
}
|
|
984
1039
|
const ok = await processGlobalSttSingleEvent(
|
|
@@ -988,16 +1043,22 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
988
1043
|
);
|
|
989
1044
|
if (!ok) {
|
|
990
1045
|
if (isLocal) {
|
|
991
|
-
|
|
992
|
-
|
|
1046
|
+
logger.warn(
|
|
1047
|
+
LogSource.Internals,
|
|
1048
|
+
'TRANSCRIPT',
|
|
1049
|
+
`${TRANSCRIPT_JOURNEY} local global state update skipped because API failed`,
|
|
1050
|
+
{stage: 'global_state_apply', outcome: 'skipped'},
|
|
993
1051
|
);
|
|
994
1052
|
continue;
|
|
995
1053
|
}
|
|
996
1054
|
// A received RTM event is authoritative for the meeting-level state.
|
|
997
1055
|
// Failure to provision this participant's bot must not make the UI
|
|
998
1056
|
// report that meeting STT is disabled while captions are arriving.
|
|
999
|
-
|
|
1000
|
-
|
|
1057
|
+
logger.warn(
|
|
1058
|
+
LogSource.Internals,
|
|
1059
|
+
'TRANSCRIPT',
|
|
1060
|
+
`${TRANSCRIPT_JOURNEY} remote global state applied after local STT API failure`,
|
|
1061
|
+
{stage: 'global_state_apply', outcome: 'applied_with_api_failure'},
|
|
1001
1062
|
);
|
|
1002
1063
|
}
|
|
1003
1064
|
// update global state AFTER processing
|
|
@@ -1022,10 +1083,24 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1022
1083
|
const isStopOperation = wasEnabledBefore && !isEnabledNow;
|
|
1023
1084
|
try {
|
|
1024
1085
|
if (isStartOperation) {
|
|
1025
|
-
|
|
1086
|
+
logger.log(
|
|
1087
|
+
LogSource.Internals,
|
|
1088
|
+
'TRANSCRIPT',
|
|
1089
|
+
`${TRANSCRIPT_JOURNEY} STT global state selected start operation`,
|
|
1090
|
+
{
|
|
1091
|
+
stage: 'global_state_operation',
|
|
1092
|
+
outcome: 'started',
|
|
1093
|
+
stateOrigin: isLocal ? 'local' : 'remote',
|
|
1094
|
+
},
|
|
1095
|
+
);
|
|
1026
1096
|
// Start guard starts
|
|
1027
1097
|
if (sttStartGuardRef.current) {
|
|
1028
|
-
|
|
1098
|
+
logger.log(
|
|
1099
|
+
LogSource.Internals,
|
|
1100
|
+
'TRANSCRIPT',
|
|
1101
|
+
`${TRANSCRIPT_JOURNEY} STT start skipped by start guard`,
|
|
1102
|
+
{stage: 'global_state_operation', outcome: 'skipped'},
|
|
1103
|
+
);
|
|
1029
1104
|
return;
|
|
1030
1105
|
}
|
|
1031
1106
|
sttStartGuardRef.current = true;
|
|
@@ -1096,7 +1171,16 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1096
1171
|
}
|
|
1097
1172
|
return true;
|
|
1098
1173
|
} else if (isUpdateOperation) {
|
|
1099
|
-
|
|
1174
|
+
logger.log(
|
|
1175
|
+
LogSource.Internals,
|
|
1176
|
+
'TRANSCRIPT',
|
|
1177
|
+
`${TRANSCRIPT_JOURNEY} STT global state selected update operation`,
|
|
1178
|
+
{
|
|
1179
|
+
stage: 'global_state_operation',
|
|
1180
|
+
outcome: 'started',
|
|
1181
|
+
stateOrigin: isLocal ? 'local' : 'remote',
|
|
1182
|
+
},
|
|
1183
|
+
);
|
|
1100
1184
|
const result = await updateSTTBotSession(
|
|
1101
1185
|
{
|
|
1102
1186
|
source: [newState.globalSpokenLanguage],
|
|
@@ -1160,7 +1244,16 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1160
1244
|
}
|
|
1161
1245
|
return true;
|
|
1162
1246
|
} else if (isStopOperation) {
|
|
1163
|
-
|
|
1247
|
+
logger.log(
|
|
1248
|
+
LogSource.Internals,
|
|
1249
|
+
'TRANSCRIPT',
|
|
1250
|
+
`${TRANSCRIPT_JOURNEY} STT global state selected stop operation`,
|
|
1251
|
+
{
|
|
1252
|
+
stage: 'global_state_operation',
|
|
1253
|
+
outcome: 'started',
|
|
1254
|
+
stateOrigin: isLocal ? 'local' : 'remote',
|
|
1255
|
+
},
|
|
1256
|
+
);
|
|
1164
1257
|
await stopSTTBotSession();
|
|
1165
1258
|
sttStartGuardRef.current = false;
|
|
1166
1259
|
return true;
|
|
@@ -1173,8 +1266,8 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1173
1266
|
logger.error(
|
|
1174
1267
|
LogSource.Internals,
|
|
1175
1268
|
'STT',
|
|
1176
|
-
|
|
1177
|
-
error,
|
|
1269
|
+
`${TRANSCRIPT_JOURNEY} error handling STT_GLOBAL_STATE event`,
|
|
1270
|
+
error ?? null,
|
|
1178
1271
|
);
|
|
1179
1272
|
return false;
|
|
1180
1273
|
}
|
|
@@ -1186,7 +1279,16 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1186
1279
|
React.useEffect(() => {
|
|
1187
1280
|
const handleGlobalSTTChange = async (evt: any) => {
|
|
1188
1281
|
const {payload} = evt || {};
|
|
1189
|
-
|
|
1282
|
+
logger.log(
|
|
1283
|
+
LogSource.Internals,
|
|
1284
|
+
'TRANSCRIPT',
|
|
1285
|
+
`${TRANSCRIPT_JOURNEY} STT_GLOBAL_STATE event received`,
|
|
1286
|
+
{
|
|
1287
|
+
stage: 'global_state_event',
|
|
1288
|
+
outcome: 'received',
|
|
1289
|
+
payloadPresent: typeof payload === 'string' && payload.length > 0,
|
|
1290
|
+
},
|
|
1291
|
+
);
|
|
1190
1292
|
let newState: GlobalSttState;
|
|
1191
1293
|
try {
|
|
1192
1294
|
newState = JSON.parse(payload);
|
|
@@ -1194,8 +1296,8 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1194
1296
|
logger.error(
|
|
1195
1297
|
LogSource.Internals,
|
|
1196
1298
|
'STT',
|
|
1197
|
-
|
|
1198
|
-
error,
|
|
1299
|
+
`${TRANSCRIPT_JOURNEY} failed to parse STT_GLOBAL_STATE event payload`,
|
|
1300
|
+
error ?? null,
|
|
1199
1301
|
);
|
|
1200
1302
|
return;
|
|
1201
1303
|
}
|
|
@@ -7,6 +7,7 @@ import {logger, LogSource} from '../../logger/AppBuilderLogger';
|
|
|
7
7
|
import getUniqueID from '../../utils/getUniqueID';
|
|
8
8
|
import {buildSTTRequestBody, type STTMethod} from './sttRequestBody';
|
|
9
9
|
import {ensureSTTSessionId} from './sttSessionId';
|
|
10
|
+
import {TRANSCRIPT_JOURNEY} from './transcriptJourney';
|
|
10
11
|
|
|
11
12
|
export interface STTAPIResponse {
|
|
12
13
|
success: boolean;
|
|
@@ -99,12 +100,12 @@ const useSTTAPI = (): IuseSTTAPI => {
|
|
|
99
100
|
});
|
|
100
101
|
|
|
101
102
|
console.log(
|
|
102
|
-
|
|
103
|
+
`${TRANSCRIPT_JOURNEY} [STT_BOT_SUBSCRIPTION] ${method.toUpperCase()} - Bot UID: ${botUid} will subscribe to User UID: ${ownerUid}`,
|
|
103
104
|
{
|
|
104
105
|
method,
|
|
105
106
|
botUid,
|
|
106
107
|
ownerUid,
|
|
107
|
-
translationConfig,
|
|
108
|
+
translationConfig: translationConfig ?? null,
|
|
108
109
|
},
|
|
109
110
|
);
|
|
110
111
|
const response = await fetch(`${STT_API_URL}/${method}`, {
|
|
@@ -146,10 +147,10 @@ const useSTTAPI = (): IuseSTTAPI => {
|
|
|
146
147
|
logger.error(
|
|
147
148
|
LogSource.NetworkRest,
|
|
148
149
|
'stt',
|
|
149
|
-
|
|
150
|
-
errorPayload,
|
|
150
|
+
`${TRANSCRIPT_JOURNEY} STT API Failure - Called ${method}`,
|
|
151
|
+
errorPayload ?? {message},
|
|
151
152
|
{
|
|
152
|
-
responseData: res,
|
|
153
|
+
responseData: res ?? null,
|
|
153
154
|
httpStatus,
|
|
154
155
|
requestId,
|
|
155
156
|
startReqTs,
|
|
@@ -172,9 +173,9 @@ const useSTTAPI = (): IuseSTTAPI => {
|
|
|
172
173
|
logger.log(
|
|
173
174
|
LogSource.NetworkRest,
|
|
174
175
|
'stt',
|
|
175
|
-
|
|
176
|
+
`${TRANSCRIPT_JOURNEY} STT API Success - Called ${method}`,
|
|
176
177
|
{
|
|
177
|
-
responseData: res,
|
|
178
|
+
responseData: res ?? null,
|
|
178
179
|
httpStatus,
|
|
179
180
|
requestId,
|
|
180
181
|
startReqTs,
|
|
@@ -194,8 +195,8 @@ const useSTTAPI = (): IuseSTTAPI => {
|
|
|
194
195
|
logger.error(
|
|
195
196
|
LogSource.NetworkRest,
|
|
196
197
|
'stt',
|
|
197
|
-
|
|
198
|
-
error,
|
|
198
|
+
`${TRANSCRIPT_JOURNEY} STT API Failure - Called ${method}`,
|
|
199
|
+
error ?? null,
|
|
199
200
|
{
|
|
200
201
|
requestId,
|
|
201
202
|
startReqTs,
|
|
@@ -235,14 +236,14 @@ const useSTTAPI = (): IuseSTTAPI => {
|
|
|
235
236
|
logger.log(
|
|
236
237
|
LogSource.NetworkRest,
|
|
237
238
|
'stt',
|
|
238
|
-
|
|
239
|
+
`${TRANSCRIPT_JOURNEY} Retrying STT start after transient failure`,
|
|
239
240
|
{
|
|
240
241
|
attempt,
|
|
241
242
|
maxAttempts: STT_START_MAX_ATTEMPTS,
|
|
242
243
|
retryDelayMs,
|
|
243
244
|
botUid,
|
|
244
|
-
httpStatus: result.httpStatus,
|
|
245
|
-
errorCode: result.error?.code,
|
|
245
|
+
httpStatus: result.httpStatus ?? null,
|
|
246
|
+
errorCode: result.error?.code ?? null,
|
|
246
247
|
},
|
|
247
248
|
);
|
|
248
249
|
await wait(retryDelayMs);
|
|
@@ -3,6 +3,8 @@ import protoRoot from './proto/ptoto';
|
|
|
3
3
|
import PQueue from 'p-queue';
|
|
4
4
|
import type {CaptionObj, TranscriptItem} from './useCaption';
|
|
5
5
|
import type {LanguageType} from './utils';
|
|
6
|
+
import {logger, LogSource} from '../../logger/AppBuilderLogger';
|
|
7
|
+
import {TRANSCRIPT_JOURNEY} from './transcriptJourney';
|
|
6
8
|
|
|
7
9
|
type StreamMessageCallback = (args: [number, Uint8Array]) => void;
|
|
8
10
|
type FinalListType = {
|
|
@@ -42,8 +44,29 @@ const useStreamMessageUtils = ({
|
|
|
42
44
|
const finalTranscriptListRef = React.useRef<FinalListType>({});
|
|
43
45
|
const finalTranslationListRef = React.useRef<FinalTranslationListType>({});
|
|
44
46
|
const queue = React.useMemo(() => new PQueue({concurrency: 1}), []);
|
|
47
|
+
const messagesReceivedRef = React.useRef(0);
|
|
48
|
+
const messagesProcessedRef = React.useRef(0);
|
|
49
|
+
const finalizedSegmentsRef = React.useRef(0);
|
|
50
|
+
const speakersWithFinalizedSegmentsRef = React.useRef(new Set<string>());
|
|
45
51
|
|
|
46
52
|
const streamMessageCallback: StreamMessageCallback = args => {
|
|
53
|
+
const [receivedBotUid, receivedPayload] = args;
|
|
54
|
+
const messageNumber = ++messagesReceivedRef.current;
|
|
55
|
+
if (messageNumber === 1 || messageNumber % 50 === 0) {
|
|
56
|
+
logger.log(
|
|
57
|
+
LogSource.Internals,
|
|
58
|
+
'TRANSCRIPT',
|
|
59
|
+
`${TRANSCRIPT_JOURNEY} stream message received`,
|
|
60
|
+
{
|
|
61
|
+
stage: 'stream_message_received',
|
|
62
|
+
outcome: 'success',
|
|
63
|
+
botUid: receivedBotUid ?? null,
|
|
64
|
+
payloadByteLength: receivedPayload?.byteLength || 0,
|
|
65
|
+
messagesReceived: messageNumber,
|
|
66
|
+
messagesProcessed: messagesProcessedRef.current,
|
|
67
|
+
},
|
|
68
|
+
);
|
|
69
|
+
}
|
|
47
70
|
const queueCallback = (args1: [number, Uint8Array]) => {
|
|
48
71
|
const finalList = finalListRef.current;
|
|
49
72
|
const finalTranscriptList = finalTranscriptListRef.current;
|
|
@@ -221,11 +244,29 @@ const useStreamMessageUtils = ({
|
|
|
221
244
|
finalList[textstream.uid].push(finalText);
|
|
222
245
|
finalTranscriptList[textstream.uid].push(finalText);
|
|
223
246
|
currentFinalText = finalText;
|
|
224
|
-
|
|
247
|
+
const speakerUid = textstream.uid ?? null;
|
|
248
|
+
const speakerKey = String(speakerUid ?? 'unknown');
|
|
249
|
+
const isFirstFinalizedSegmentForSpeaker =
|
|
250
|
+
!speakersWithFinalizedSegmentsRef.current.has(speakerKey);
|
|
251
|
+
finalizedSegmentsRef.current += 1;
|
|
225
252
|
const duration = performance.now() - captionStartTimeRef.current;
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
253
|
+
if (isFirstFinalizedSegmentForSpeaker) {
|
|
254
|
+
speakersWithFinalizedSegmentsRef.current.add(speakerKey);
|
|
255
|
+
logger.log(
|
|
256
|
+
LogSource.Internals,
|
|
257
|
+
'TRANSCRIPT',
|
|
258
|
+
`${TRANSCRIPT_JOURNEY} first transcript segment finalized for speaker`,
|
|
259
|
+
{
|
|
260
|
+
stage: 'segment_finalized',
|
|
261
|
+
outcome: 'success',
|
|
262
|
+
botUid: botUid ?? null,
|
|
263
|
+
speakerUid,
|
|
264
|
+
finalCharacterCount: currentFinalText.length,
|
|
265
|
+
finalizeDurationMs: duration,
|
|
266
|
+
finalizedSegments: finalizedSegmentsRef.current,
|
|
267
|
+
},
|
|
268
|
+
);
|
|
269
|
+
}
|
|
229
270
|
captionStartTimeRef.current = 0; // Reset start time
|
|
230
271
|
}
|
|
231
272
|
|
|
@@ -364,14 +405,74 @@ const useStreamMessageUtils = ({
|
|
|
364
405
|
};
|
|
365
406
|
void queue
|
|
366
407
|
.add(() => queueCallback(args))
|
|
367
|
-
.then(() =>
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
408
|
+
.then(() => {
|
|
409
|
+
const processedCount = ++messagesProcessedRef.current;
|
|
410
|
+
if (processedCount === 1 || processedCount % 50 === 0) {
|
|
411
|
+
logger.log(
|
|
412
|
+
LogSource.Internals,
|
|
413
|
+
'TRANSCRIPT',
|
|
414
|
+
`${TRANSCRIPT_JOURNEY} stream message processed`,
|
|
415
|
+
{
|
|
416
|
+
stage: 'stream_message_processed',
|
|
417
|
+
outcome: 'success',
|
|
418
|
+
botUid: receivedBotUid ?? null,
|
|
419
|
+
messagesReceived: messagesReceivedRef.current,
|
|
420
|
+
messagesProcessed: processedCount,
|
|
421
|
+
queueSize: queue.size,
|
|
422
|
+
},
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
})
|
|
426
|
+
.catch(error => {
|
|
427
|
+
logger.error(
|
|
428
|
+
LogSource.Internals,
|
|
429
|
+
'TRANSCRIPT',
|
|
430
|
+
`${TRANSCRIPT_JOURNEY} failed to process stream message`,
|
|
431
|
+
{
|
|
432
|
+
stage: 'stream_message_processing',
|
|
433
|
+
outcome: 'failure',
|
|
434
|
+
botUid: receivedBotUid ?? null,
|
|
435
|
+
messageNumber,
|
|
436
|
+
messagesReceived: messagesReceivedRef.current,
|
|
437
|
+
messagesProcessed: messagesProcessedRef.current,
|
|
438
|
+
error: error ?? null,
|
|
439
|
+
},
|
|
440
|
+
);
|
|
441
|
+
});
|
|
371
442
|
};
|
|
372
443
|
|
|
373
444
|
const flushStreamMessageQueue = React.useCallback(async () => {
|
|
445
|
+
logger.log(
|
|
446
|
+
LogSource.Internals,
|
|
447
|
+
'TRANSCRIPT',
|
|
448
|
+
`${TRANSCRIPT_JOURNEY} stream queue flush requested`,
|
|
449
|
+
{
|
|
450
|
+
stage: 'stream_queue_flush',
|
|
451
|
+
outcome: 'started',
|
|
452
|
+
messagesReceived: messagesReceivedRef.current,
|
|
453
|
+
messagesProcessed: messagesProcessedRef.current,
|
|
454
|
+
finalizedSegments: finalizedSegmentsRef.current,
|
|
455
|
+
speakersWithFinalizedSegments:
|
|
456
|
+
speakersWithFinalizedSegmentsRef.current.size,
|
|
457
|
+
queueSize: queue.size,
|
|
458
|
+
},
|
|
459
|
+
);
|
|
374
460
|
await queue.onIdle();
|
|
461
|
+
logger.log(
|
|
462
|
+
LogSource.Internals,
|
|
463
|
+
'TRANSCRIPT',
|
|
464
|
+
`${TRANSCRIPT_JOURNEY} stream queue flush completed`,
|
|
465
|
+
{
|
|
466
|
+
stage: 'stream_queue_flush',
|
|
467
|
+
outcome: 'success',
|
|
468
|
+
messagesReceived: messagesReceivedRef.current,
|
|
469
|
+
messagesProcessed: messagesProcessedRef.current,
|
|
470
|
+
finalizedSegments: finalizedSegmentsRef.current,
|
|
471
|
+
speakersWithFinalizedSegments:
|
|
472
|
+
speakersWithFinalizedSegmentsRef.current.size,
|
|
473
|
+
queueSize: queue.size,
|
|
474
|
+
},
|
|
475
|
+
);
|
|
375
476
|
}, [queue]);
|
|
376
477
|
|
|
377
478
|
return {
|