agora-appbuilder-core 4.1.24-beta.2 → 4.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/template/defaultConfig.js +2 -2
- package/template/src/subComponents/caption/Transcript.tsx +0 -36
- package/template/src/subComponents/caption/useCaption.tsx +46 -215
- package/template/src/subComponents/caption/useSTTAPI.tsx +12 -13
- package/template/src/subComponents/caption/useStreamMessageUtils.ts +8 -109
- package/template/src/subComponents/caption/__tests__/transcriptJourney.test.ts +0 -7
- package/template/src/subComponents/caption/transcriptJourney.ts +0 -1
package/package.json
CHANGED
|
@@ -77,8 +77,8 @@ const DefaultConfig = {
|
|
|
77
77
|
CHAT_ORG_NAME: "",
|
|
78
78
|
CHAT_APP_NAME: "",
|
|
79
79
|
CHAT_URL: "",
|
|
80
|
-
CLI_VERSION: "3.1.24
|
|
81
|
-
CORE_VERSION: "4.1.24
|
|
80
|
+
CLI_VERSION: "3.1.24",
|
|
81
|
+
CORE_VERSION: "4.1.24",
|
|
82
82
|
DISABLE_LANDSCAPE_MODE: false,
|
|
83
83
|
STT_AUTO_START: false,
|
|
84
84
|
CLOUD_RECORDING_AUTO_START: false,
|
|
@@ -40,8 +40,6 @@ import {
|
|
|
40
40
|
sttTranscriptPanelSearchText,
|
|
41
41
|
sttTranscriptPanelViewLatestText,
|
|
42
42
|
} from '../../../src/language/default-labels/videoCallScreenLabels';
|
|
43
|
-
import {logger, LogSource} from '../../logger/AppBuilderLogger';
|
|
44
|
-
import {TRANSCRIPT_JOURNEY} from './transcriptJourney';
|
|
45
43
|
|
|
46
44
|
export interface TranscriptProps {
|
|
47
45
|
showHeader?: boolean;
|
|
@@ -58,7 +56,6 @@ const Transcript = (props: TranscriptProps) => {
|
|
|
58
56
|
isSTTActive,
|
|
59
57
|
selectedTranslationLanguage,
|
|
60
58
|
transcriptViewMode,
|
|
61
|
-
isSTTListenerAdded,
|
|
62
59
|
} = useCaption();
|
|
63
60
|
|
|
64
61
|
const settingSpokenLanguageLabel = useString(sttSettingSpokenLanguageText)();
|
|
@@ -71,39 +68,6 @@ const Transcript = (props: TranscriptProps) => {
|
|
|
71
68
|
|
|
72
69
|
const data = meetingTranscript; // Object.entries(transcript);
|
|
73
70
|
console.log('[STT_GLOBAL] meetingTranscript data: ', data);
|
|
74
|
-
const hasLoggedFirstEntryRef = React.useRef(false);
|
|
75
|
-
const transcriptEntryCountRef = React.useRef(data.length);
|
|
76
|
-
transcriptEntryCountRef.current = data.length;
|
|
77
|
-
|
|
78
|
-
React.useEffect(() => {
|
|
79
|
-
logger.log(
|
|
80
|
-
LogSource.Internals,
|
|
81
|
-
'TRANSCRIPT',
|
|
82
|
-
`${TRANSCRIPT_JOURNEY} transcript panel state`,
|
|
83
|
-
{
|
|
84
|
-
stage: 'transcript_panel',
|
|
85
|
-
isSTTActive,
|
|
86
|
-
listenerRegistered: isSTTListenerAdded,
|
|
87
|
-
transcriptEntryCount: transcriptEntryCountRef.current,
|
|
88
|
-
},
|
|
89
|
-
);
|
|
90
|
-
}, [isSTTActive, isSTTListenerAdded]);
|
|
91
|
-
|
|
92
|
-
React.useEffect(() => {
|
|
93
|
-
if (data.length > 0 && !hasLoggedFirstEntryRef.current) {
|
|
94
|
-
hasLoggedFirstEntryRef.current = true;
|
|
95
|
-
logger.log(
|
|
96
|
-
LogSource.Internals,
|
|
97
|
-
'TRANSCRIPT',
|
|
98
|
-
`${TRANSCRIPT_JOURNEY} first transcript entry available to panel`,
|
|
99
|
-
{
|
|
100
|
-
stage: 'transcript_panel_data',
|
|
101
|
-
outcome: 'success',
|
|
102
|
-
transcriptEntryCount: data.length,
|
|
103
|
-
},
|
|
104
|
-
);
|
|
105
|
-
}
|
|
106
|
-
}, [data.length]);
|
|
107
71
|
|
|
108
72
|
const [showButton, setShowButton] = React.useState(false);
|
|
109
73
|
|
|
@@ -21,7 +21,6 @@ import {useRoomInfo} from '../../components/room-info/useRoomInfo';
|
|
|
21
21
|
import {useContent, useRtc} from 'customization-api';
|
|
22
22
|
import useStreamMessageUtils from './useStreamMessageUtils';
|
|
23
23
|
import {isWebInternal} from '../../utils/common';
|
|
24
|
-
import {TRANSCRIPT_JOURNEY} from './transcriptJourney';
|
|
25
24
|
|
|
26
25
|
// Types
|
|
27
26
|
type GlobalSttState = {
|
|
@@ -349,58 +348,17 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
349
348
|
return;
|
|
350
349
|
}
|
|
351
350
|
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
`${TRANSCRIPT_JOURNEY} stream listener registration requested`,
|
|
356
|
-
{stage: 'listener_registration', outcome: 'started'},
|
|
351
|
+
transcriptListenerSubscriptionRef.current = RtcEngineUnsafe.addListener(
|
|
352
|
+
'onStreamMessage',
|
|
353
|
+
handleStreamMessageCallback,
|
|
357
354
|
);
|
|
358
|
-
|
|
359
|
-
transcriptListenerSubscriptionRef.current = RtcEngineUnsafe.addListener(
|
|
360
|
-
'onStreamMessage',
|
|
361
|
-
handleStreamMessageCallback,
|
|
362
|
-
);
|
|
363
|
-
setIsSTTListenerAdded(true);
|
|
364
|
-
logger.log(
|
|
365
|
-
LogSource.Internals,
|
|
366
|
-
'TRANSCRIPT',
|
|
367
|
-
`${TRANSCRIPT_JOURNEY} stream listener registered`,
|
|
368
|
-
{stage: 'listener_registration', outcome: 'success'},
|
|
369
|
-
);
|
|
370
|
-
} catch (error) {
|
|
371
|
-
logger.error(
|
|
372
|
-
LogSource.Internals,
|
|
373
|
-
'TRANSCRIPT',
|
|
374
|
-
`${TRANSCRIPT_JOURNEY} stream listener registration failed`,
|
|
375
|
-
{
|
|
376
|
-
stage: 'listener_registration',
|
|
377
|
-
outcome: 'failure',
|
|
378
|
-
error: error ?? null,
|
|
379
|
-
},
|
|
380
|
-
);
|
|
381
|
-
}
|
|
355
|
+
setIsSTTListenerAdded(true);
|
|
382
356
|
}, [RtcEngineUnsafe, callActive, handleStreamMessageCallback]);
|
|
383
357
|
|
|
384
358
|
React.useEffect(() => {
|
|
385
359
|
return () => {
|
|
386
|
-
logger.log(
|
|
387
|
-
LogSource.Internals,
|
|
388
|
-
'TRANSCRIPT',
|
|
389
|
-
`${TRANSCRIPT_JOURNEY} stream listener cleanup requested`,
|
|
390
|
-
{
|
|
391
|
-
stage: 'listener_cleanup',
|
|
392
|
-
outcome: 'started',
|
|
393
|
-
listenerReferencePresent: !!transcriptListenerSubscriptionRef.current,
|
|
394
|
-
},
|
|
395
|
-
);
|
|
396
360
|
transcriptListenerSubscriptionRef.current?.remove();
|
|
397
361
|
transcriptListenerSubscriptionRef.current = null;
|
|
398
|
-
logger.log(
|
|
399
|
-
LogSource.Internals,
|
|
400
|
-
'TRANSCRIPT',
|
|
401
|
-
`${TRANSCRIPT_JOURNEY} stream listener subscription remove called and local reference cleared`,
|
|
402
|
-
{stage: 'listener_cleanup', outcome: 'completed'},
|
|
403
|
-
);
|
|
404
362
|
};
|
|
405
363
|
}, []);
|
|
406
364
|
|
|
@@ -451,31 +409,6 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
451
409
|
sttDepsReadyRef.current = sttDepsReady;
|
|
452
410
|
}, [sttDepsReady]);
|
|
453
411
|
|
|
454
|
-
React.useEffect(() => {
|
|
455
|
-
logger.log(
|
|
456
|
-
LogSource.Internals,
|
|
457
|
-
'TRANSCRIPT',
|
|
458
|
-
`${TRANSCRIPT_JOURNEY} STT readiness changed`,
|
|
459
|
-
{
|
|
460
|
-
stage: 'stt_readiness',
|
|
461
|
-
outcome: sttDepsReady ? 'ready' : 'waiting',
|
|
462
|
-
localUidReady: !!localUid,
|
|
463
|
-
localBotUidReady: !!localBotUid,
|
|
464
|
-
rtmJoined: !!hasUserJoinedRTM,
|
|
465
|
-
callActive: !!callActive,
|
|
466
|
-
roomInfoReady: !!(roomId?.host || roomId?.attendee),
|
|
467
|
-
listenerRegistered: !!transcriptListenerSubscriptionRef.current,
|
|
468
|
-
},
|
|
469
|
-
);
|
|
470
|
-
}, [
|
|
471
|
-
callActive,
|
|
472
|
-
hasUserJoinedRTM,
|
|
473
|
-
localBotUid,
|
|
474
|
-
localUid,
|
|
475
|
-
roomId,
|
|
476
|
-
sttDepsReady,
|
|
477
|
-
]);
|
|
478
|
-
|
|
479
412
|
React.useEffect(() => {
|
|
480
413
|
if (sttDepsReadyRef.current && !hasFlushedSttQueueRef.current) {
|
|
481
414
|
hasFlushedSttQueueRef.current = true;
|
|
@@ -593,21 +526,22 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
593
526
|
try {
|
|
594
527
|
setIsLangChangeInProgress(true);
|
|
595
528
|
const result = await start(localBotUidRef.current, newConfig);
|
|
529
|
+
console.log('[STT] start result: ', result);
|
|
596
530
|
if (result.success) {
|
|
597
531
|
setIsSTTError(false);
|
|
598
532
|
logger.log(
|
|
599
533
|
LogSource.NetworkRest,
|
|
600
534
|
'stt',
|
|
601
|
-
|
|
602
|
-
result.data
|
|
535
|
+
'STT started successfully',
|
|
536
|
+
result.data,
|
|
603
537
|
);
|
|
604
538
|
} else {
|
|
605
539
|
setIsSTTError(true);
|
|
606
540
|
logger.error(
|
|
607
541
|
LogSource.NetworkRest,
|
|
608
542
|
'stt',
|
|
609
|
-
|
|
610
|
-
result.error
|
|
543
|
+
'Failed to start STT',
|
|
544
|
+
result.error,
|
|
611
545
|
);
|
|
612
546
|
Toast.show({
|
|
613
547
|
leadingIconName: 'alert',
|
|
@@ -624,12 +558,7 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
624
558
|
} catch (error: any) {
|
|
625
559
|
setIsLangChangeInProgress(false);
|
|
626
560
|
setIsSTTError(true);
|
|
627
|
-
logger.error(
|
|
628
|
-
LogSource.NetworkRest,
|
|
629
|
-
'stt',
|
|
630
|
-
`${TRANSCRIPT_JOURNEY} STT start error`,
|
|
631
|
-
error ?? null,
|
|
632
|
-
);
|
|
561
|
+
logger.error(LogSource.NetworkRest, 'stt', 'STT start error', error);
|
|
633
562
|
// Show error toast: text1 = translated label, text2 = exception error
|
|
634
563
|
Toast.show({
|
|
635
564
|
leadingIconName: 'alert',
|
|
@@ -674,16 +603,16 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
674
603
|
logger.log(
|
|
675
604
|
LogSource.NetworkRest,
|
|
676
605
|
'stt',
|
|
677
|
-
|
|
678
|
-
result.data
|
|
606
|
+
'STT updated successfully',
|
|
607
|
+
result.data,
|
|
679
608
|
);
|
|
680
609
|
} else {
|
|
681
610
|
setIsSTTError(true);
|
|
682
611
|
logger.error(
|
|
683
612
|
LogSource.NetworkRest,
|
|
684
613
|
'stt',
|
|
685
|
-
|
|
686
|
-
result.error
|
|
614
|
+
'Failed to update STT',
|
|
615
|
+
result.error,
|
|
687
616
|
);
|
|
688
617
|
// Show error toast: text1 = translated label, text2 = API error
|
|
689
618
|
Toast.show({
|
|
@@ -701,12 +630,7 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
701
630
|
} catch (error: any) {
|
|
702
631
|
setIsLangChangeInProgress(false);
|
|
703
632
|
setIsSTTError(true);
|
|
704
|
-
logger.error(
|
|
705
|
-
LogSource.NetworkRest,
|
|
706
|
-
'stt',
|
|
707
|
-
`${TRANSCRIPT_JOURNEY} STT update error`,
|
|
708
|
-
error ?? null,
|
|
709
|
-
);
|
|
633
|
+
logger.error(LogSource.NetworkRest, 'stt', 'STT update error', error);
|
|
710
634
|
// Show error toast: text1 = translated label, text2 = exception error
|
|
711
635
|
Toast.show({
|
|
712
636
|
leadingIconName: 'alert',
|
|
@@ -725,20 +649,10 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
725
649
|
};
|
|
726
650
|
|
|
727
651
|
const stopSTTBotSession = React.useCallback(async () => {
|
|
728
|
-
|
|
729
|
-
LogSource.Internals,
|
|
730
|
-
'TRANSCRIPT',
|
|
731
|
-
`${TRANSCRIPT_JOURNEY} STT stop requested`,
|
|
732
|
-
{stage: 'stt_stop', localBotUid: localBotUidRef.current ?? null},
|
|
733
|
-
);
|
|
652
|
+
console.log('[STT] stopSTTBotSession called');
|
|
734
653
|
|
|
735
654
|
if (!localBotUidRef.current) {
|
|
736
|
-
|
|
737
|
-
LogSource.Internals,
|
|
738
|
-
'TRANSCRIPT',
|
|
739
|
-
`${TRANSCRIPT_JOURNEY} STT stop skipped because bot UID is missing`,
|
|
740
|
-
{stage: 'stt_stop', outcome: 'skipped'},
|
|
741
|
-
);
|
|
655
|
+
console.warn('[STT] Missing botUid for stop');
|
|
742
656
|
return;
|
|
743
657
|
}
|
|
744
658
|
|
|
@@ -754,16 +668,16 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
754
668
|
logger.log(
|
|
755
669
|
LogSource.NetworkRest,
|
|
756
670
|
'stt',
|
|
757
|
-
|
|
758
|
-
result.data
|
|
671
|
+
'STT stopped successfully',
|
|
672
|
+
result.data,
|
|
759
673
|
);
|
|
760
674
|
} else {
|
|
761
675
|
setIsSTTError(true);
|
|
762
676
|
logger.error(
|
|
763
677
|
LogSource.NetworkRest,
|
|
764
678
|
'stt',
|
|
765
|
-
|
|
766
|
-
result.error
|
|
679
|
+
'Failed to stop STT',
|
|
680
|
+
result.error,
|
|
767
681
|
);
|
|
768
682
|
}
|
|
769
683
|
} catch (error) {
|
|
@@ -771,8 +685,8 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
771
685
|
logger.error(
|
|
772
686
|
LogSource.NetworkRest,
|
|
773
687
|
'stt',
|
|
774
|
-
|
|
775
|
-
error
|
|
688
|
+
'Error in stopSTTBotSession',
|
|
689
|
+
error,
|
|
776
690
|
);
|
|
777
691
|
}
|
|
778
692
|
}, [stop]);
|
|
@@ -843,29 +757,12 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
843
757
|
// Queue
|
|
844
758
|
enqueueSttEvent(updatedState, true, targetChange);
|
|
845
759
|
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
`${TRANSCRIPT_JOURNEY} local spoken-language state queued`,
|
|
850
|
-
{
|
|
851
|
-
stage: 'global_state_queue',
|
|
852
|
-
outcome: 'queued',
|
|
853
|
-
spokenLanguage: updatedState.globalSpokenLanguage ?? null,
|
|
854
|
-
translationTargetCount:
|
|
855
|
-
updatedState.globalTranslationTargets.length,
|
|
856
|
-
},
|
|
760
|
+
console.log(
|
|
761
|
+
'[STT_GLOBAL] confirmSpokenLanguageChange sent STT_GLOBAL_STATE: ',
|
|
762
|
+
updatedState,
|
|
857
763
|
);
|
|
858
764
|
} catch (error) {
|
|
859
|
-
|
|
860
|
-
LogSource.Internals,
|
|
861
|
-
'TRANSCRIPT',
|
|
862
|
-
`${TRANSCRIPT_JOURNEY} local spoken-language state failed`,
|
|
863
|
-
{
|
|
864
|
-
stage: 'global_state_queue',
|
|
865
|
-
outcome: 'failure',
|
|
866
|
-
error: error ?? null,
|
|
867
|
-
},
|
|
868
|
-
);
|
|
765
|
+
console.log('[STT_GLOBAL] confirmSpokenLanguageChange error: ', error);
|
|
869
766
|
}
|
|
870
767
|
},
|
|
871
768
|
[localUid], // only real dependency
|
|
@@ -920,24 +817,15 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
920
817
|
// Queues all local + remote stt events
|
|
921
818
|
const enqueueSttEvent = React.useCallback(
|
|
922
819
|
(state: GlobalSttState, isLocal: boolean, targetChange?: TargetChange) => {
|
|
820
|
+
console.log(
|
|
821
|
+
'[STT_GLOBAL] inside enqueueSttEvent - sttDepsReadyRef flag',
|
|
822
|
+
sttDepsReadyRef.current,
|
|
823
|
+
);
|
|
923
824
|
sttEventQueueRef.current.push({
|
|
924
825
|
state,
|
|
925
826
|
isLocal,
|
|
926
827
|
targetChange,
|
|
927
828
|
});
|
|
928
|
-
logger.log(
|
|
929
|
-
LogSource.Internals,
|
|
930
|
-
'TRANSCRIPT',
|
|
931
|
-
`${TRANSCRIPT_JOURNEY} STT global state queued`,
|
|
932
|
-
{
|
|
933
|
-
stage: 'global_state_queue',
|
|
934
|
-
outcome: 'queued',
|
|
935
|
-
stateOrigin: isLocal ? 'local' : 'remote',
|
|
936
|
-
sttEnabled: state.globalSttEnabled,
|
|
937
|
-
sttDepsReady: sttDepsReadyRef.current,
|
|
938
|
-
queueSize: sttEventQueueRef.current.length,
|
|
939
|
-
},
|
|
940
|
-
);
|
|
941
829
|
if (sttDepsReadyRef.current) {
|
|
942
830
|
processSttEventQueue();
|
|
943
831
|
}
|
|
@@ -964,12 +852,7 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
964
852
|
!globalSttStateRef.current.globalSttEnabled &&
|
|
965
853
|
sttEventQueueRef.current.length === 0
|
|
966
854
|
) {
|
|
967
|
-
|
|
968
|
-
LogSource.Internals,
|
|
969
|
-
'TRANSCRIPT',
|
|
970
|
-
`${TRANSCRIPT_JOURNEY} STT auto-start state injected`,
|
|
971
|
-
{stage: 'global_state_queue', outcome: 'queued'},
|
|
972
|
-
);
|
|
855
|
+
console.log('[STT] AUTO_START injecting auto start state');
|
|
973
856
|
|
|
974
857
|
sttAutoStartGuardRef.current = true;
|
|
975
858
|
|
|
@@ -996,12 +879,7 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
996
879
|
const {state: newState, isLocal, targetChange} = item;
|
|
997
880
|
const prevState = globalSttStateRef.current;
|
|
998
881
|
if (isSameState(prevState, newState)) {
|
|
999
|
-
|
|
1000
|
-
LogSource.Internals,
|
|
1001
|
-
'TRANSCRIPT',
|
|
1002
|
-
`${TRANSCRIPT_JOURNEY} duplicate STT global state skipped`,
|
|
1003
|
-
{stage: 'global_state_queue', outcome: 'skipped'},
|
|
1004
|
-
);
|
|
882
|
+
console.log('[STT] Skipped duplicate STT_GLOBAL_STATE');
|
|
1005
883
|
continue; // no call to processGlobalSttSingleEvent
|
|
1006
884
|
}
|
|
1007
885
|
const ok = await processGlobalSttSingleEvent(
|
|
@@ -1011,22 +889,16 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1011
889
|
);
|
|
1012
890
|
if (!ok) {
|
|
1013
891
|
if (isLocal) {
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
'TRANSCRIPT',
|
|
1017
|
-
`${TRANSCRIPT_JOURNEY} local global state update skipped because API failed`,
|
|
1018
|
-
{stage: 'global_state_apply', outcome: 'skipped'},
|
|
892
|
+
console.warn(
|
|
893
|
+
'[STT] Skipping local global state update because API failed.',
|
|
1019
894
|
);
|
|
1020
895
|
continue;
|
|
1021
896
|
}
|
|
1022
897
|
// A received RTM event is authoritative for the meeting-level state.
|
|
1023
898
|
// Failure to provision this participant's bot must not make the UI
|
|
1024
899
|
// report that meeting STT is disabled while captions are arriving.
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
'TRANSCRIPT',
|
|
1028
|
-
`${TRANSCRIPT_JOURNEY} remote global state applied after local STT API failure`,
|
|
1029
|
-
{stage: 'global_state_apply', outcome: 'applied_with_api_failure'},
|
|
900
|
+
console.warn(
|
|
901
|
+
'[STT] Applying remote global state after local STT API failure.',
|
|
1030
902
|
);
|
|
1031
903
|
}
|
|
1032
904
|
// update global state AFTER processing
|
|
@@ -1051,24 +923,10 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1051
923
|
const isStopOperation = wasEnabledBefore && !isEnabledNow;
|
|
1052
924
|
try {
|
|
1053
925
|
if (isStartOperation) {
|
|
1054
|
-
|
|
1055
|
-
LogSource.Internals,
|
|
1056
|
-
'TRANSCRIPT',
|
|
1057
|
-
`${TRANSCRIPT_JOURNEY} STT global state selected start operation`,
|
|
1058
|
-
{
|
|
1059
|
-
stage: 'global_state_operation',
|
|
1060
|
-
outcome: 'started',
|
|
1061
|
-
stateOrigin: isLocal ? 'local' : 'remote',
|
|
1062
|
-
},
|
|
1063
|
-
);
|
|
926
|
+
console.log('[STT] Remote global STT -> starting session', newState);
|
|
1064
927
|
// Start guard starts
|
|
1065
928
|
if (sttStartGuardRef.current) {
|
|
1066
|
-
|
|
1067
|
-
LogSource.Internals,
|
|
1068
|
-
'TRANSCRIPT',
|
|
1069
|
-
`${TRANSCRIPT_JOURNEY} STT start skipped by start guard`,
|
|
1070
|
-
{stage: 'global_state_operation', outcome: 'skipped'},
|
|
1071
|
-
);
|
|
929
|
+
console.log('[STT] Start skipped (already started)');
|
|
1072
930
|
return;
|
|
1073
931
|
}
|
|
1074
932
|
sttStartGuardRef.current = true;
|
|
@@ -1139,16 +997,7 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1139
997
|
}
|
|
1140
998
|
return true;
|
|
1141
999
|
} else if (isUpdateOperation) {
|
|
1142
|
-
|
|
1143
|
-
LogSource.Internals,
|
|
1144
|
-
'TRANSCRIPT',
|
|
1145
|
-
`${TRANSCRIPT_JOURNEY} STT global state selected update operation`,
|
|
1146
|
-
{
|
|
1147
|
-
stage: 'global_state_operation',
|
|
1148
|
-
outcome: 'started',
|
|
1149
|
-
stateOrigin: isLocal ? 'local' : 'remote',
|
|
1150
|
-
},
|
|
1151
|
-
);
|
|
1000
|
+
console.log('[STT] Global STT -> updating session', newState);
|
|
1152
1001
|
const result = await updateSTTBotSession(
|
|
1153
1002
|
{
|
|
1154
1003
|
source: [newState.globalSpokenLanguage],
|
|
@@ -1212,16 +1061,7 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1212
1061
|
}
|
|
1213
1062
|
return true;
|
|
1214
1063
|
} else if (isStopOperation) {
|
|
1215
|
-
|
|
1216
|
-
LogSource.Internals,
|
|
1217
|
-
'TRANSCRIPT',
|
|
1218
|
-
`${TRANSCRIPT_JOURNEY} STT global state selected stop operation`,
|
|
1219
|
-
{
|
|
1220
|
-
stage: 'global_state_operation',
|
|
1221
|
-
outcome: 'started',
|
|
1222
|
-
stateOrigin: isLocal ? 'local' : 'remote',
|
|
1223
|
-
},
|
|
1224
|
-
);
|
|
1064
|
+
console.log('[STT] Global STT -> stopping session', newState);
|
|
1225
1065
|
await stopSTTBotSession();
|
|
1226
1066
|
sttStartGuardRef.current = false;
|
|
1227
1067
|
return true;
|
|
@@ -1234,8 +1074,8 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1234
1074
|
logger.error(
|
|
1235
1075
|
LogSource.Internals,
|
|
1236
1076
|
'STT',
|
|
1237
|
-
|
|
1238
|
-
error
|
|
1077
|
+
'Error handling STT_GLOBAL_STATE event',
|
|
1078
|
+
error,
|
|
1239
1079
|
);
|
|
1240
1080
|
return false;
|
|
1241
1081
|
}
|
|
@@ -1247,16 +1087,7 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1247
1087
|
React.useEffect(() => {
|
|
1248
1088
|
const handleGlobalSTTChange = async (evt: any) => {
|
|
1249
1089
|
const {payload} = evt || {};
|
|
1250
|
-
|
|
1251
|
-
LogSource.Internals,
|
|
1252
|
-
'TRANSCRIPT',
|
|
1253
|
-
`${TRANSCRIPT_JOURNEY} STT_GLOBAL_STATE event received`,
|
|
1254
|
-
{
|
|
1255
|
-
stage: 'global_state_event',
|
|
1256
|
-
outcome: 'received',
|
|
1257
|
-
payloadPresent: typeof payload === 'string' && payload.length > 0,
|
|
1258
|
-
},
|
|
1259
|
-
);
|
|
1090
|
+
console.log('[STT] STT_GLOBAL_STATE event received: ', evt);
|
|
1260
1091
|
let newState: GlobalSttState;
|
|
1261
1092
|
try {
|
|
1262
1093
|
newState = JSON.parse(payload);
|
|
@@ -1264,8 +1095,8 @@ const CaptionProvider: React.FC<CaptionProviderProps> = ({
|
|
|
1264
1095
|
logger.error(
|
|
1265
1096
|
LogSource.Internals,
|
|
1266
1097
|
'STT',
|
|
1267
|
-
|
|
1268
|
-
error
|
|
1098
|
+
'Failed to parse STT_GLOBAL_STATE event payload',
|
|
1099
|
+
error,
|
|
1269
1100
|
);
|
|
1270
1101
|
return;
|
|
1271
1102
|
}
|
|
@@ -7,7 +7,6 @@ import {logger, LogSource} from '../../logger/AppBuilderLogger';
|
|
|
7
7
|
import getUniqueID from '../../utils/getUniqueID';
|
|
8
8
|
import {buildSTTRequestBody, type STTMethod} from './sttRequestBody';
|
|
9
9
|
import {ensureSTTSessionId} from './sttSessionId';
|
|
10
|
-
import {TRANSCRIPT_JOURNEY} from './transcriptJourney';
|
|
11
10
|
|
|
12
11
|
export interface STTAPIResponse {
|
|
13
12
|
success: boolean;
|
|
@@ -100,12 +99,12 @@ const useSTTAPI = (): IuseSTTAPI => {
|
|
|
100
99
|
});
|
|
101
100
|
|
|
102
101
|
console.log(
|
|
103
|
-
|
|
102
|
+
`[STT_BOT_SUBSCRIPTION] ${method.toUpperCase()} - Bot UID: ${botUid} will subscribe to User UID: ${ownerUid}`,
|
|
104
103
|
{
|
|
105
104
|
method,
|
|
106
105
|
botUid,
|
|
107
106
|
ownerUid,
|
|
108
|
-
translationConfig
|
|
107
|
+
translationConfig,
|
|
109
108
|
},
|
|
110
109
|
);
|
|
111
110
|
const response = await fetch(`${STT_API_URL}/${method}`, {
|
|
@@ -147,10 +146,10 @@ const useSTTAPI = (): IuseSTTAPI => {
|
|
|
147
146
|
logger.error(
|
|
148
147
|
LogSource.NetworkRest,
|
|
149
148
|
'stt',
|
|
150
|
-
|
|
151
|
-
errorPayload
|
|
149
|
+
`STT API Failure - Called ${method}`,
|
|
150
|
+
errorPayload,
|
|
152
151
|
{
|
|
153
|
-
responseData: res
|
|
152
|
+
responseData: res,
|
|
154
153
|
httpStatus,
|
|
155
154
|
requestId,
|
|
156
155
|
startReqTs,
|
|
@@ -173,9 +172,9 @@ const useSTTAPI = (): IuseSTTAPI => {
|
|
|
173
172
|
logger.log(
|
|
174
173
|
LogSource.NetworkRest,
|
|
175
174
|
'stt',
|
|
176
|
-
|
|
175
|
+
`STT API Success - Called ${method}`,
|
|
177
176
|
{
|
|
178
|
-
responseData: res
|
|
177
|
+
responseData: res,
|
|
179
178
|
httpStatus,
|
|
180
179
|
requestId,
|
|
181
180
|
startReqTs,
|
|
@@ -195,8 +194,8 @@ const useSTTAPI = (): IuseSTTAPI => {
|
|
|
195
194
|
logger.error(
|
|
196
195
|
LogSource.NetworkRest,
|
|
197
196
|
'stt',
|
|
198
|
-
|
|
199
|
-
error
|
|
197
|
+
`STT API Failure - Called ${method}`,
|
|
198
|
+
error,
|
|
200
199
|
{
|
|
201
200
|
requestId,
|
|
202
201
|
startReqTs,
|
|
@@ -236,14 +235,14 @@ const useSTTAPI = (): IuseSTTAPI => {
|
|
|
236
235
|
logger.log(
|
|
237
236
|
LogSource.NetworkRest,
|
|
238
237
|
'stt',
|
|
239
|
-
|
|
238
|
+
'Retrying STT start after transient failure',
|
|
240
239
|
{
|
|
241
240
|
attempt,
|
|
242
241
|
maxAttempts: STT_START_MAX_ATTEMPTS,
|
|
243
242
|
retryDelayMs,
|
|
244
243
|
botUid,
|
|
245
|
-
httpStatus: result.httpStatus
|
|
246
|
-
errorCode: result.error?.code
|
|
244
|
+
httpStatus: result.httpStatus,
|
|
245
|
+
errorCode: result.error?.code,
|
|
247
246
|
},
|
|
248
247
|
);
|
|
249
248
|
await wait(retryDelayMs);
|
|
@@ -3,8 +3,6 @@ import protoRoot from './proto/ptoto';
|
|
|
3
3
|
import PQueue from 'p-queue';
|
|
4
4
|
import type {CaptionObj, TranscriptItem} from './useCaption';
|
|
5
5
|
import type {LanguageType} from './utils';
|
|
6
|
-
import {logger, LogSource} from '../../logger/AppBuilderLogger';
|
|
7
|
-
import {TRANSCRIPT_JOURNEY} from './transcriptJourney';
|
|
8
6
|
|
|
9
7
|
type StreamMessageCallback = (args: [number, Uint8Array]) => void;
|
|
10
8
|
type FinalListType = {
|
|
@@ -44,29 +42,8 @@ const useStreamMessageUtils = ({
|
|
|
44
42
|
const finalTranscriptListRef = React.useRef<FinalListType>({});
|
|
45
43
|
const finalTranslationListRef = React.useRef<FinalTranslationListType>({});
|
|
46
44
|
const queue = React.useMemo(() => new PQueue({concurrency: 1}), []);
|
|
47
|
-
const messagesReceivedRef = React.useRef(0);
|
|
48
|
-
const messagesProcessedRef = React.useRef(0);
|
|
49
|
-
const finalizedSegmentsRef = React.useRef(0);
|
|
50
|
-
const speakersWithFinalizedSegmentsRef = React.useRef(new Set<string>());
|
|
51
45
|
|
|
52
46
|
const streamMessageCallback: StreamMessageCallback = args => {
|
|
53
|
-
const [receivedBotUid, receivedPayload] = args;
|
|
54
|
-
const messageNumber = ++messagesReceivedRef.current;
|
|
55
|
-
if (messageNumber === 1 || messageNumber % 50 === 0) {
|
|
56
|
-
logger.log(
|
|
57
|
-
LogSource.Internals,
|
|
58
|
-
'TRANSCRIPT',
|
|
59
|
-
`${TRANSCRIPT_JOURNEY} stream message received`,
|
|
60
|
-
{
|
|
61
|
-
stage: 'stream_message_received',
|
|
62
|
-
outcome: 'success',
|
|
63
|
-
botUid: receivedBotUid ?? null,
|
|
64
|
-
payloadByteLength: receivedPayload?.byteLength || 0,
|
|
65
|
-
messagesReceived: messageNumber,
|
|
66
|
-
messagesProcessed: messagesProcessedRef.current,
|
|
67
|
-
},
|
|
68
|
-
);
|
|
69
|
-
}
|
|
70
47
|
const queueCallback = (args1: [number, Uint8Array]) => {
|
|
71
48
|
const finalList = finalListRef.current;
|
|
72
49
|
const finalTranscriptList = finalTranscriptListRef.current;
|
|
@@ -244,29 +221,11 @@ const useStreamMessageUtils = ({
|
|
|
244
221
|
finalList[textstream.uid].push(finalText);
|
|
245
222
|
finalTranscriptList[textstream.uid].push(finalText);
|
|
246
223
|
currentFinalText = finalText;
|
|
247
|
-
|
|
248
|
-
const speakerKey = String(speakerUid ?? 'unknown');
|
|
249
|
-
const isFirstFinalizedSegmentForSpeaker =
|
|
250
|
-
!speakersWithFinalizedSegmentsRef.current.has(speakerKey);
|
|
251
|
-
finalizedSegmentsRef.current += 1;
|
|
224
|
+
// log info to show measure the duration of passes in which a sentence gets finalized
|
|
252
225
|
const duration = performance.now() - captionStartTimeRef.current;
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
LogSource.Internals,
|
|
257
|
-
'TRANSCRIPT',
|
|
258
|
-
`${TRANSCRIPT_JOURNEY} first transcript segment finalized for speaker`,
|
|
259
|
-
{
|
|
260
|
-
stage: 'segment_finalized',
|
|
261
|
-
outcome: 'success',
|
|
262
|
-
botUid: botUid ?? null,
|
|
263
|
-
speakerUid,
|
|
264
|
-
finalCharacterCount: currentFinalText.length,
|
|
265
|
-
finalizeDurationMs: duration,
|
|
266
|
-
finalizedSegments: finalizedSegmentsRef.current,
|
|
267
|
-
},
|
|
268
|
-
);
|
|
269
|
-
}
|
|
226
|
+
console.log(
|
|
227
|
+
`stt-Time taken to finalize caption ${currentFinalText}: ${duration}ms`,
|
|
228
|
+
);
|
|
270
229
|
captionStartTimeRef.current = 0; // Reset start time
|
|
271
230
|
}
|
|
272
231
|
|
|
@@ -405,74 +364,14 @@ const useStreamMessageUtils = ({
|
|
|
405
364
|
};
|
|
406
365
|
void queue
|
|
407
366
|
.add(() => queueCallback(args))
|
|
408
|
-
.then(() =>
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
LogSource.Internals,
|
|
413
|
-
'TRANSCRIPT',
|
|
414
|
-
`${TRANSCRIPT_JOURNEY} stream message processed`,
|
|
415
|
-
{
|
|
416
|
-
stage: 'stream_message_processed',
|
|
417
|
-
outcome: 'success',
|
|
418
|
-
botUid: receivedBotUid ?? null,
|
|
419
|
-
messagesReceived: messagesReceivedRef.current,
|
|
420
|
-
messagesProcessed: processedCount,
|
|
421
|
-
queueSize: queue.size,
|
|
422
|
-
},
|
|
423
|
-
);
|
|
424
|
-
}
|
|
425
|
-
})
|
|
426
|
-
.catch(error => {
|
|
427
|
-
logger.error(
|
|
428
|
-
LogSource.Internals,
|
|
429
|
-
'TRANSCRIPT',
|
|
430
|
-
`${TRANSCRIPT_JOURNEY} failed to process stream message`,
|
|
431
|
-
{
|
|
432
|
-
stage: 'stream_message_processing',
|
|
433
|
-
outcome: 'failure',
|
|
434
|
-
botUid: receivedBotUid ?? null,
|
|
435
|
-
messageNumber,
|
|
436
|
-
messagesReceived: messagesReceivedRef.current,
|
|
437
|
-
messagesProcessed: messagesProcessedRef.current,
|
|
438
|
-
error: error ?? null,
|
|
439
|
-
},
|
|
440
|
-
);
|
|
441
|
-
});
|
|
367
|
+
.then(() => console.log('stt- using pq queue'))
|
|
368
|
+
.catch(error =>
|
|
369
|
+
console.error('Failed to process STT stream message', error),
|
|
370
|
+
);
|
|
442
371
|
};
|
|
443
372
|
|
|
444
373
|
const flushStreamMessageQueue = React.useCallback(async () => {
|
|
445
|
-
logger.log(
|
|
446
|
-
LogSource.Internals,
|
|
447
|
-
'TRANSCRIPT',
|
|
448
|
-
`${TRANSCRIPT_JOURNEY} stream queue flush requested`,
|
|
449
|
-
{
|
|
450
|
-
stage: 'stream_queue_flush',
|
|
451
|
-
outcome: 'started',
|
|
452
|
-
messagesReceived: messagesReceivedRef.current,
|
|
453
|
-
messagesProcessed: messagesProcessedRef.current,
|
|
454
|
-
finalizedSegments: finalizedSegmentsRef.current,
|
|
455
|
-
speakersWithFinalizedSegments:
|
|
456
|
-
speakersWithFinalizedSegmentsRef.current.size,
|
|
457
|
-
queueSize: queue.size,
|
|
458
|
-
},
|
|
459
|
-
);
|
|
460
374
|
await queue.onIdle();
|
|
461
|
-
logger.log(
|
|
462
|
-
LogSource.Internals,
|
|
463
|
-
'TRANSCRIPT',
|
|
464
|
-
`${TRANSCRIPT_JOURNEY} stream queue flush completed`,
|
|
465
|
-
{
|
|
466
|
-
stage: 'stream_queue_flush',
|
|
467
|
-
outcome: 'success',
|
|
468
|
-
messagesReceived: messagesReceivedRef.current,
|
|
469
|
-
messagesProcessed: messagesProcessedRef.current,
|
|
470
|
-
finalizedSegments: finalizedSegmentsRef.current,
|
|
471
|
-
speakersWithFinalizedSegments:
|
|
472
|
-
speakersWithFinalizedSegmentsRef.current.size,
|
|
473
|
-
queueSize: queue.size,
|
|
474
|
-
},
|
|
475
|
-
);
|
|
476
375
|
}, [queue]);
|
|
477
376
|
|
|
478
377
|
return {
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export const TRANSCRIPT_JOURNEY = '[TRANSCRIPT_JOURNEY]';
|