agora-appbuilder-core 4.1.23 → 4.1.24-beta.2

package/template/src/subComponents/caption/useStreamMessageUtils.ts

@@ -3,6 +3,8 @@ import protoRoot from './proto/ptoto';
 import PQueue from 'p-queue';
 import type {CaptionObj, TranscriptItem} from './useCaption';
 import type {LanguageType} from './utils';
+import {logger, LogSource} from '../../logger/AppBuilderLogger';
+import {TRANSCRIPT_JOURNEY} from './transcriptJourney';
 
 type StreamMessageCallback = (args: [number, Uint8Array]) => void;
 type FinalListType = {
@@ -42,8 +44,29 @@ const useStreamMessageUtils = ({
   const finalTranscriptListRef = React.useRef<FinalListType>({});
   const finalTranslationListRef = React.useRef<FinalTranslationListType>({});
   const queue = React.useMemo(() => new PQueue({concurrency: 1}), []);
+  const messagesReceivedRef = React.useRef(0);
+  const messagesProcessedRef = React.useRef(0);
+  const finalizedSegmentsRef = React.useRef(0);
+  const speakersWithFinalizedSegmentsRef = React.useRef(new Set<string>());
 
   const streamMessageCallback: StreamMessageCallback = args => {
+    const [receivedBotUid, receivedPayload] = args;
+    const messageNumber = ++messagesReceivedRef.current;
+    if (messageNumber === 1 || messageNumber % 50 === 0) {
+      logger.log(
+        LogSource.Internals,
+        'TRANSCRIPT',
+        `${TRANSCRIPT_JOURNEY} stream message received`,
+        {
+          stage: 'stream_message_received',
+          outcome: 'success',
+          botUid: receivedBotUid ?? null,
+          payloadByteLength: receivedPayload?.byteLength || 0,
+          messagesReceived: messageNumber,
+          messagesProcessed: messagesProcessedRef.current,
+        },
+      );
+    }
     const queueCallback = (args1: [number, Uint8Array]) => {
       const finalList = finalListRef.current;
       const finalTranscriptList = finalTranscriptListRef.current;
@@ -221,11 +244,29 @@ const useStreamMessageUtils = ({
         finalList[textstream.uid].push(finalText);
         finalTranscriptList[textstream.uid].push(finalText);
         currentFinalText = finalText;
-        // log info to show measure the duration of passes in which a sentence gets finalized
+        const speakerUid = textstream.uid ?? null;
+        const speakerKey = String(speakerUid ?? 'unknown');
+        const isFirstFinalizedSegmentForSpeaker =
+          !speakersWithFinalizedSegmentsRef.current.has(speakerKey);
+        finalizedSegmentsRef.current += 1;
         const duration = performance.now() - captionStartTimeRef.current;
-        console.log(
-          `stt-Time taken to finalize caption ${currentFinalText}: ${duration}ms`,
-        );
+        if (isFirstFinalizedSegmentForSpeaker) {
+          speakersWithFinalizedSegmentsRef.current.add(speakerKey);
+          logger.log(
+            LogSource.Internals,
+            'TRANSCRIPT',
+            `${TRANSCRIPT_JOURNEY} first transcript segment finalized for speaker`,
+            {
+              stage: 'segment_finalized',
+              outcome: 'success',
+              botUid: botUid ?? null,
+              speakerUid,
+              finalCharacterCount: currentFinalText.length,
+              finalizeDurationMs: duration,
+              finalizedSegments: finalizedSegmentsRef.current,
+            },
+          );
+        }
         captionStartTimeRef.current = 0; // Reset start time
       }
 
@@ -364,14 +405,74 @@ const useStreamMessageUtils = ({
     };
     void queue
       .add(() => queueCallback(args))
-      .then(() => console.log('stt- using pq queue'))
-      .catch(error =>
-        console.error('Failed to process STT stream message', error),
-      );
+      .then(() => {
+        const processedCount = ++messagesProcessedRef.current;
+        if (processedCount === 1 || processedCount % 50 === 0) {
+          logger.log(
+            LogSource.Internals,
+            'TRANSCRIPT',
+            `${TRANSCRIPT_JOURNEY} stream message processed`,
+            {
+              stage: 'stream_message_processed',
+              outcome: 'success',
+              botUid: receivedBotUid ?? null,
+              messagesReceived: messagesReceivedRef.current,
+              messagesProcessed: processedCount,
+              queueSize: queue.size,
+            },
+          );
+        }
+      })
+      .catch(error => {
+        logger.error(
+          LogSource.Internals,
+          'TRANSCRIPT',
+          `${TRANSCRIPT_JOURNEY} failed to process stream message`,
+          {
+            stage: 'stream_message_processing',
+            outcome: 'failure',
+            botUid: receivedBotUid ?? null,
+            messageNumber,
+            messagesReceived: messagesReceivedRef.current,
+            messagesProcessed: messagesProcessedRef.current,
+            error: error ?? null,
+          },
+        );
+      });
   };
 
   const flushStreamMessageQueue = React.useCallback(async () => {
+    logger.log(
+      LogSource.Internals,
+      'TRANSCRIPT',
+      `${TRANSCRIPT_JOURNEY} stream queue flush requested`,
+      {
+        stage: 'stream_queue_flush',
+        outcome: 'started',
+        messagesReceived: messagesReceivedRef.current,
+        messagesProcessed: messagesProcessedRef.current,
+        finalizedSegments: finalizedSegmentsRef.current,
+        speakersWithFinalizedSegments:
+          speakersWithFinalizedSegmentsRef.current.size,
+        queueSize: queue.size,
+      },
+    );
     await queue.onIdle();
+    logger.log(
+      LogSource.Internals,
+      'TRANSCRIPT',
+      `${TRANSCRIPT_JOURNEY} stream queue flush completed`,
+      {
+        stage: 'stream_queue_flush',
+        outcome: 'success',
+        messagesReceived: messagesReceivedRef.current,
+        messagesProcessed: messagesProcessedRef.current,
+        finalizedSegments: finalizedSegmentsRef.current,
+        speakersWithFinalizedSegments:
+          speakersWithFinalizedSegmentsRef.current.size,
+        queueSize: queue.size,
+      },
+    );
   }, [queue]);
 
   return {

package/template/src/subComponents/screenshare/ScreenshareConfigure.native.tsx

@@ -52,6 +52,11 @@ import {timeNow} from '../../rtm/utils';
 import LocalEventEmitter, {
   LocalEventsEnum,
 } from '../../rtm-events-api/LocalEvents';
+import {
+  captureScreenshareRecoveryCandidate,
+  getScreenshareRecoveryDecision,
+  ScreenshareRecoveryCandidate,
+} from './screenshareInterruptionRecovery';
 
 export const ScreenshareContextConsumer = ScreenshareContext.Consumer;
 
@@ -82,6 +87,14 @@ export const ScreenshareConfigure = (props: {children: React.ReactNode}) => {
     secondaryPinnedUid: secondaryPinnedUid,
   });
   const screenShareDataRef = useRef({screenShareData: screenShareData});
+  const recoveryCandidateRef = useRef<ScreenshareRecoveryCandidate | null>(
+    null,
+  );
+  const previousLayoutStateRef = useRef({
+    activeUids,
+    pinnedUid,
+    currentLayout,
+  });
   const localMute = useMuteToggleLocal();
   const {video} = useLocalUserInfo();
   useEffect(() => {
@@ -163,6 +176,71 @@ export const ScreenshareConfigure = (props: {children: React.ReactNode}) => {
     }
   }, [activeUids, screenShareData, triggerChangeLayout]);
 
+  useEffect(() => {
+    const previousLayoutState = previousLayoutStateRef.current;
+    const existingCandidate = recoveryCandidateRef.current;
+    const capturedCandidate = captureScreenshareRecoveryCandidate({
+      previousActiveUids: previousLayoutState.activeUids,
+      currentActiveUids: activeUids,
+      previousPinnedUid: previousLayoutState.pinnedUid,
+      previousLayout: previousLayoutState.currentLayout,
+      screenShareData,
+      detectedAt: Date.now(),
+    });
+
+    if (!existingCandidate && capturedCandidate) {
+      recoveryCandidateRef.current = capturedCandidate;
+    }
+
+    const candidate = recoveryCandidateRef.current;
+    if (candidate) {
+      const decision = getScreenshareRecoveryDecision({
+        candidate,
+        activeUids,
+        pinnedUid,
+        isVideoPublished: defaultContent?.[candidate.uid]?.video === 1,
+        screenShareData,
+      });
+
+      if (decision === 'waiting_for_video') {
+        candidate.joinedLogged = true;
+      } else if (decision === 'restore') {
+        isPinned.current = candidate.uid;
+        dispatch({type: 'UserPin', value: [candidate.uid]});
+        if (
+          candidate.previousLayout === getPinnedLayoutName() &&
+          currentLayout !== getPinnedLayoutName()
+        ) {
+          setPinnedLayout();
+        }
+        recoveryCandidateRef.current = null;
+      } else if (decision.startsWith('cancel_')) {
+        recoveryCandidateRef.current = null;
+      }
+    }
+
+    previousLayoutStateRef.current = {
+      activeUids,
+      pinnedUid,
+      currentLayout,
+    };
+  }, [
+    activeUids,
+    currentLayout,
+    defaultContent,
+    dispatch,
+    pinnedUid,
+    screenShareData,
+    setPinnedLayout,
+  ]);
+
+  useEffect(
+    () => () => {
+      recoveryCandidateRef.current = null;
+    },
+    [],
+  );
+
   useEffect(() => {
     /**
      * When user kicked off by remote host then stop the screenshare
@@ -170,7 +248,8 @@ export const ScreenshareConfigure = (props: {children: React.ReactNode}) => {
     const unsubKickUser = LocalEventEmitter.on(
       LocalEventsEnum.USER_KICKED_OFF_BY_REMOTE_HOST,
       () => {
-        stopScreenshare(false, true);
+        // Force native capture cleanup even if React state has not caught up.
+        forceStopScreenshare();
       },
     );
     /**
@@ -179,7 +258,8 @@ export const ScreenshareConfigure = (props: {children: React.ReactNode}) => {
     const unsubKickScreenshare = events.on(
       controlMessageEnum.kickScreenshare,
       () => {
-        stopScreenshare(false, true);
+        // A host-initiated removal must not depend on local active state.
+        forceStopScreenshare();
       },
     );
     const unsubScreenShareAttribute = events.on(
@@ -428,34 +508,39 @@ export const ScreenshareConfigure = (props: {children: React.ReactNode}) => {
     unpublishScreenshare,
   ]);
 
-  const stopScreenshare = useCallback(
-    async (enableVideo: boolean = false, forceStop: boolean = false) => {
-      if (isScreenshareActive || forceStop) {
-        logger.log(
-          LogSource.Internals,
-          'SCREENSHARE',
-          'Trying to stop native screenshare',
-        );
-        try {
-          engine?.current?.stopScreenCapture();
-        } catch (error) {
-          logger.error(
-            LogSource.Internals,
-            'SCREENSHARE',
-            'native screenshare error on -> stopScreenCapture',
-            error,
-          );
-        }
-      } else {
-        logger.debug(
-          LogSource.Internals,
-          'SCREENSHARE',
-          'native screenshare -> no screenshare is active',
-        );
-      }
-    },
-    [engine, isScreenshareActive],
-  );
+  const stopNativeScreenCapture = useCallback(async () => {
+    logger.log(
+      LogSource.Internals,
+      'SCREENSHARE',
+      'Trying to stop native screenshare',
+    );
+    try {
+      engine?.current?.stopScreenCapture();
+    } catch (error) {
+      logger.error(
+        LogSource.Internals,
+        'SCREENSHARE',
+        'native screenshare error on -> stopScreenCapture',
+        error,
+      );
+    }
+  }, [engine]);
+
+  const stopScreenshare = useCallback(async () => {
+    if (isScreenshareActive) {
+      await stopNativeScreenCapture();
+    } else {
+      logger.debug(
+        LogSource.Internals,
+        'SCREENSHARE',
+        'native screenshare -> no screenshare is active',
+      );
+    }
+  }, [isScreenshareActive, stopNativeScreenCapture]);
+
+  const forceStopScreenshare = useCallback(async () => {
+    await stopNativeScreenCapture();
+  }, [stopNativeScreenCapture]);
 
   const onLocalVideoStateChanged = useCallback(
     (source: VideoSourceType, state: LocalVideoStreamState, error) => {