botframework-webchat-api 4.18.2 → 4.19.1-main.20260526.b9ad8fa

package/src/providers/GroupActivities/private/createDefaultGroupActivitiesMiddleware.ts

@@ -0,0 +1,113 @@
+import { getOrgSchemaMessage, type GlobalScopePonyfill, type WebChatActivity } from 'botframework-webchat-core';
+import { IdentifierSchema } from 'botframework-webchat-core/graph.js';
+import { safeParse } from 'valibot';
+
+import { querySendStatusFromOutgoingActivity } from 'botframework-webchat-core/activity.js';
+import type GroupActivitiesMiddleware from '../../../types/GroupActivitiesMiddleware';
+import { type SendStatus } from '../../../types/SendStatus';
+
+function bin<T>(items: readonly T[], grouping: (last: T, current: T) => boolean): readonly (readonly T[])[] {
+  let lastBin: T[];
+  const bins: T[][] = [];
+  let lastItem: T;
+
+  for (const item of items) {
+    if (lastItem && grouping(lastItem, item)) {
+      lastBin.push(item);
+    } else {
+      lastBin = [item];
+      bins.push(lastBin);
+    }
+
+    lastItem = item;
+  }
+
+  for (const bin in bins) {
+    Object.freeze(bin);
+  }
+
+  return Object.freeze(bins);
+}
+
+function isSending(activity: WebChatActivity): SendStatus | undefined {
+  if (activity.from.role === 'user') {
+    return querySendStatusFromOutgoingActivity(activity);
+  }
+}
+
+function createShouldGroupTimestamp(groupTimestamp: boolean | number, { Date }: GlobalScopePonyfill) {
+  return (activityX: WebChatActivity, activityY: WebChatActivity): boolean => {
+    if (groupTimestamp === false) {
+      // Hide timestamp for all activities.
+      return true;
+    } else if (activityX && activityY) {
+      if (isSending(activityX) !== isSending(activityY)) {
+        return false;
+      }
+
+      groupTimestamp = typeof groupTimestamp === 'number' ? groupTimestamp : Infinity;
+
+      const timeX = new Date(activityX.timestamp).getTime();
+      const timeY = new Date(activityY.timestamp).getTime();
+
+      return Math.abs(timeX - timeY) <= groupTimestamp;
+    }
+
+    return false;
+  };
+}
+
+function shouldGroupSender(x: WebChatActivity, y: WebChatActivity): boolean {
+  const {
+    from: { role: roleX, id: idX }
+  } = x;
+  const {
+    from: { role: roleY, id: idY }
+  } = y;
+  return roleX === roleY && idX === idY;
+}
+
+export default function createDefaultGroupActivitiesMiddleware({
+  groupTimestamp,
+  ponyfill
+}: Readonly<{
+  groupTimestamp: boolean | number;
+  ponyfill: GlobalScopePonyfill;
+}>): readonly GroupActivitiesMiddleware[] {
+  return Object.freeze([
+    type =>
+      type === 'sender' || typeof type === 'undefined'
+        ? next =>
+            ({ activities }) => ({ ...next({ activities }), sender: bin(activities, shouldGroupSender) })
+        : undefined,
+    type =>
+      type === 'status' || typeof type === 'undefined'
+        ? next =>
+            ({ activities }) => ({
+              ...next({ activities }),
+              status: bin(activities, createShouldGroupTimestamp(groupTimestamp, ponyfill))
+            })
+        : undefined,
+    type =>
+      type === 'part'
+        ? next =>
+            ({ activities }) => {
+              const messages = activities.map(activity => [getOrgSchemaMessage(activity.entities), activity] as const);
+
+              return {
+                ...next({ activities }),
+                part: bin(messages, ([last], [current]) => {
+                  const lastPartIdResult = safeParse(IdentifierSchema, last?.isPartOf[0]?.['@id']);
+                  const currentPartIdResult = safeParse(IdentifierSchema, current?.isPartOf[0]?.['@id']);
+
+                  return (
+                    lastPartIdResult.success &&
+                    currentPartIdResult.success &&
+                    lastPartIdResult.output === currentPartIdResult.output
+                  );
+                }).map(bin => bin.map(([, activity]) => activity))
+              };
+            }
+        : undefined
+  ]);
+}

package/src/providers/GroupActivities/private/isGroupingValid.ts

@@ -0,0 +1,41 @@
+import { type WebChatActivity } from 'botframework-webchat-core';
+
+export default function isGroupingValid(
+  source: readonly WebChatActivity[],
+  bins: readonly (readonly WebChatActivity[])[]
+): boolean {
+  const set = new Set(source);
+
+  if (source.length !== set.size) {
+    console.warn('botframework-webchat: Cannot validate activity grouping because some activities are duplicated');
+
+    return false;
+  }
+
+  for (const bin of bins) {
+    for (const activityInBin of bin) {
+      if (!set.has(activityInBin)) {
+        console.warn(
+          'botframework-webchat: All binned items must be originate from the source list, check groupingActivityMiddleware to make sure it bin from the source list',
+          {
+            activityInBin
+          }
+        );
+
+        return false;
+      }
+
+      set.delete(activityInBin);
+    }
+  }
+
+  if (set.size) {
+    console.warn(
+      'botframework-webchat: Not every activity is binned, check groupingActivityMiddleware to make sure it is binning every activity passed'
+    );
+
+    return false;
+  }
+
+  return true;
+}

package/src/providers/GroupActivities/private/useGroupActivitiesContext.ts

@@ -0,0 +1,6 @@
+import { useContext } from 'react';
+import GroupActivitiesContext, { type GroupActivitiesContextType } from './GroupActivitiesContext';
+
+export default function useGroupActivitiesContext(): GroupActivitiesContextType {
+  return useContext(GroupActivitiesContext);
+}

package/src/providers/GroupActivities/useGroupActivities.ts

@@ -0,0 +1,20 @@
+import { type WebChatActivity } from 'botframework-webchat-core';
+
+import useGroupActivitiesContext from './private/useGroupActivitiesContext';
+
+type GroupedActivities = readonly (readonly WebChatActivity[])[];
+
+/**
+ * This hook will return a callback function. When called with `activities`, the callback function will run the `groupActivitiesMiddleware` and will return all groupings.
+ *
+ * @deprecated This function is deprecated and will be removed on or after 2027-05-04. Developers should migrate to [`useGroupActivitiesByName`](#usegroupactivitiesbyname) for performance reason.
+ */
+export default function useGroupActivities(): ({
+  activities
+}: Readonly<{
+  activities: readonly WebChatActivity[];
+}>) => Readonly<{
+  [key: string]: GroupedActivities;
+}> {
+  return useGroupActivitiesContext().groupActivities;
+}

package/src/providers/GroupActivities/useGroupActivitiesByName.ts

@@ -0,0 +1,17 @@
+import { type WebChatActivity } from 'botframework-webchat-core';
+
+import useGroupActivitiesContext from './private/useGroupActivitiesContext';
+
+/**
+ * This hook will return a callback function. When called with `activities`, the callback function will run the `groupActivitiesMiddleware` for the specified grouping name.
+ *
+ * Unlike the [`useGroupActivities`](#usegroupactivities) hook which provide result for all groupings, this hook only provide result for the specified grouping name and the grouping name must be one of the name specified in `styleOptions.groupActivitiesBy`.
+ *
+ * @returns Result of `groupActivitiesMiddleware` for the specified grouping name
+ */
+export default function useGroupActivitiesBy(): (
+  activities: readonly WebChatActivity[],
+  name: string
+) => readonly (readonly WebChatActivity[])[] {
+  return useGroupActivitiesContext().groupActivitiesByName;
+}

package/src/providers/SpeechToSpeech/SpeechToSpeechComposer.tsx

@@ -0,0 +1,20 @@
+import React, { type ReactNode } from 'react';
+import { VoiceHandlerBridge } from './private/VoiceHandlerBridge';
+import { VoiceRecorderBridge } from './private/VoiceRecorderBridge';
+
+/**
+ * SpeechToSpeechComposer sets up the speech-to-speech infrastructure.
+ *
+ * This component renders invisible bridge components that:
+ * 1. VoiceHandlerBridge - registers audio player functions with Redux
+ * 2. VoiceRecorderBridge - reacts to recording state and manages microphone
+ *
+ * Use the `useVoiceState`, `useStartVoice`, and `useStopVoice` hooks to access state and controls.
+ */
+export const SpeechToSpeechComposer: React.FC<{ readonly children: ReactNode }> = ({ children }) => (
+  <React.Fragment>
+    <VoiceHandlerBridge />
+    <VoiceRecorderBridge />
+    {children}
+  </React.Fragment>
+);

package/src/providers/SpeechToSpeech/private/VoiceHandlerBridge.tsx

@@ -0,0 +1,23 @@
+import { useEffect } from 'react';
+import { useAudioPlayer } from './useAudioPlayer';
+import useRegisterVoiceHandler from '../../../hooks/internal/useRegisterVoiceHandler';
+import useShouldShowMicrophoneButton from '../../../hooks/internal/useShouldShowMicrophoneButton';
+
+/**
+ * VoiceHandlerBridge is an invisible component that registers the audio player
+ * functions (queueAudio, stopAllAudio) with Redux
+ */
+export const VoiceHandlerBridge = () => {
+  const { queueAudio, stopAllAudio } = useAudioPlayer();
+  const registerVoiceHandler = useRegisterVoiceHandler();
+  const shouldShowMicrophoneButton = useShouldShowMicrophoneButton();
+
+  useEffect(() => {
+    if (!shouldShowMicrophoneButton) {
+      return;
+    }
+    return registerVoiceHandler({ queueAudio, stopAllAudio });
+  }, [queueAudio, registerVoiceHandler, shouldShowMicrophoneButton, stopAllAudio]);
+
+  return null;
+};

package/src/providers/SpeechToSpeech/private/VoiceRecorderBridge.tsx

@@ -0,0 +1,49 @@
+import { useEffect, useCallback } from 'react';
+import { useRecorder } from './useRecorder';
+import usePostVoiceActivity from '../../../hooks/internal/usePostVoiceActivity';
+import useVoiceRecordingMuted from '../../../hooks/useVoiceRecordingMuted';
+import useVoiceState from '../../../hooks/useVoiceState';
+
+/**
+ * VoiceRecorderBridge is an invisible component that bridges the Redux recording state
+ * with the actual microphone recording functionality.
+ */
+export function VoiceRecorderBridge(): null {
+  const [muted] = useVoiceRecordingMuted();
+  const [voiceState] = useVoiceState();
+  const postVoiceActivity = usePostVoiceActivity();
+
+  // Derive recording state from voiceState - recording is active when not idle
+  const recording = voiceState !== 'idle';
+
+  const handleAudioChunk = useCallback(
+    (base64: string, timestamp: string) => {
+      postVoiceActivity({
+        name: 'media.chunk',
+        type: 'event',
+        value: {
+          contentType: 'audio/webm',
+          content: base64,
+          timestamp
+        }
+      } as any);
+    },
+    [postVoiceActivity]
+  );
+
+  const { mute, record } = useRecorder(handleAudioChunk);
+
+  useEffect(() => {
+    if (recording) {
+      return record();
+    }
+  }, [record, recording]);
+
+  useEffect(() => {
+    if (muted) {
+      return mute();
+    }
+  }, [muted, mute]);
+
+  return null;
+}

package/src/providers/SpeechToSpeech/private/useAudioPlayer.spec.tsx

@@ -0,0 +1,355 @@
+/** @jest-environment @happy-dom/jest-environment */
+/// <reference types="jest" />
+/// <reference types="node" />
+
+import { render, type RenderResult } from '@testing-library/react';
+import React, { type ComponentType } from 'react';
+import { useAudioPlayer } from './useAudioPlayer';
+
+// Mock setVoiceState function
+const mockSetVoiceState = jest.fn();
+
+// Mock useVoiceStateWritable hook - returns [state, setVoiceState] array
+jest.mock('../../../hooks/internal/useVoiceStateWritable', () => ({
+  __esModule: true,
+  default: jest.fn(() => [undefined, mockSetVoiceState])
+}));
+
+jest.mock('../../Capabilities/useCapabilities', () => ({
+  __esModule: true,
+  default: jest.fn((selector: (caps: { voiceConfiguration?: { sampleRate: number } }) => unknown) =>
+    selector({ voiceConfiguration: { sampleRate: 24000 } })
+  )
+}));
+
+// Mock AudioContext and related APIs
+const mockAudioContext = {
+  close: jest.fn().mockResolvedValue(undefined),
+  createBuffer: jest.fn(),
+  createBufferSource: jest.fn(),
+  currentTime: 0,
+  destination: {},
+  resume: jest.fn().mockResolvedValue(undefined),
+  sampleRate: 24000,
+  state: 'running'
+};
+
+const mockAudioBuffer = {
+  duration: 0.1, // 100ms
+  getChannelData: jest.fn().mockReturnValue(new Float32Array(2400))
+};
+
+// Factory to create unique buffer source mocks
+const createMockBufferSource = () => ({
+  buffer: null as typeof mockAudioBuffer | null,
+  connect: jest.fn(),
+  disconnect: jest.fn(),
+  onended: null as (() => void) | null,
+  start: jest.fn(),
+  stop: jest.fn()
+});
+
+// Track all created buffer sources for assertions
+let createdBufferSources: ReturnType<typeof createMockBufferSource>[] = [];
+
+// Mock global AudioContext
+global.AudioContext = jest.fn(() => mockAudioContext) as unknown as typeof AudioContext;
+global.atob = jest.fn(str => str); // Simple mock for base64 decode
+
+type UseAudioPlayerReturn = ReturnType<typeof useAudioPlayer>;
+
+describe('useAudioPlayer', () => {
+  let HookApp: ComponentType;
+  let hookData: UseAudioPlayerReturn | undefined;
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  let renderResult: RenderResult;
+  const originalAudioContext = global.AudioContext;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    createdBufferSources = [];
+    mockAudioContext.currentTime = 0;
+    mockAudioContext.createBuffer.mockReturnValue(mockAudioBuffer);
+    mockAudioContext.createBufferSource.mockImplementation(() => {
+      const source = createMockBufferSource();
+      createdBufferSources.push(source);
+      return source;
+    });
+
+    HookApp = () => {
+      hookData = useAudioPlayer();
+      return null;
+    };
+  });
+
+  afterEach(() => {
+    global.AudioContext = originalAudioContext;
+  });
+
+  describe('Initialization', () => {
+    test('should return queueAudio and stopAllAudio functions', () => {
+      render(<HookApp />);
+
+      expect(typeof hookData?.queueAudio).toBe('function');
+      expect(typeof hookData?.stopAllAudio).toBe('function');
+    });
+
+    test('should create AudioContext on first queueAudio call', async () => {
+      render(<HookApp />);
+
+      await hookData?.queueAudio('dGVzdA=='); // base64 for 'test'
+
+      expect(AudioContext).toHaveBeenCalledWith({ sampleRate: 24000 });
+    });
+
+    test('should reuse existing AudioContext on subsequent calls', async () => {
+      render(<HookApp />);
+
+      await hookData?.queueAudio('dGVzdA==');
+      await hookData?.queueAudio('dGVzdDI=');
+
+      expect(AudioContext).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('Audio playback', () => {
+    beforeEach(() => {
+      renderResult = render(<HookApp />);
+    });
+
+    test('should process base64 audio data correctly', async () => {
+      await hookData?.queueAudio('dGVzdA==');
+
+      expect(global.atob).toHaveBeenCalledWith('dGVzdA==');
+      expect(mockAudioContext.createBuffer).toHaveBeenCalledWith(1, expect.any(Number), 24000);
+      expect(mockAudioContext.createBufferSource).toHaveBeenCalled();
+    });
+
+    test('should set up audio buffer source correctly', async () => {
+      await hookData?.queueAudio('dGVzdA==');
+
+      const [source] = createdBufferSources;
+      expect(source.connect).toHaveBeenCalledWith(mockAudioContext.destination);
+      expect(source.start).toHaveBeenCalled();
+      expect(source.buffer).toBe(mockAudioBuffer);
+    });
+
+    test('should resume AudioContext if needed', async () => {
+      await hookData?.queueAudio('dGVzdA==');
+
+      expect(mockAudioContext.resume).toHaveBeenCalled();
+    });
+
+    test('should queue multiple audio chunks correctly', async () => {
+      mockAudioBuffer.duration = 0.1; // 100ms
+
+      await hookData?.queueAudio('dGVzdA==');
+      await hookData?.queueAudio('dGVzdDI=');
+
+      expect(createdBufferSources).toHaveLength(2);
+      // First chunk starts at currentTime (0), second at 0.1
+      expect(createdBufferSources[0].start).toHaveBeenCalledWith(0);
+      expect(createdBufferSources[1].start).toHaveBeenCalledWith(0.1);
+    });
+  });
+
+  describe('Voice state management', () => {
+    beforeEach(() => {
+      renderResult = render(<HookApp />);
+    });
+
+    test('should set voice state to bot_speaking on first chunk', async () => {
+      mockAudioContext.currentTime = 0;
+
+      await hookData?.queueAudio('dGVzdA==');
+
+      expect(mockSetVoiceState).toHaveBeenCalledWith('bot_speaking');
+    });
+
+    test('should not set bot_speaking on subsequent chunks while playing', async () => {
+      mockAudioContext.currentTime = 0;
+      mockAudioBuffer.duration = 0.1;
+
+      await hookData?.queueAudio('dGVzdA=='); // First chunk
+      mockSetVoiceState.mockClear();
+
+      await hookData?.queueAudio('dGVzdDI='); // Second chunk (while first is still playing)
+
+      expect(mockSetVoiceState).not.toHaveBeenCalledWith('bot_speaking');
+    });
+
+    test('should set voice state to listening when last audio ends', async () => {
+      await hookData?.queueAudio('dGVzdA==');
+      mockSetVoiceState.mockClear();
+
+      // Simulate audio ended
+      const [source] = createdBufferSources;
+      if (source.onended) {
+        source.onended();
+      }
+
+      expect(mockSetVoiceState).toHaveBeenCalledWith('listening');
+    });
+
+    test('should only trigger listening on the last source ended', async () => {
+      mockAudioBuffer.duration = 0.1;
+
+      await hookData?.queueAudio('dGVzdA==');
+      await hookData?.queueAudio('dGVzdDI=');
+      mockSetVoiceState.mockClear();
+
+      const [firstSource, lastSource] = createdBufferSources;
+
+      // Simulate first chunk ended (should not trigger listening)
+      if (firstSource.onended) {
+        firstSource.onended();
+      }
+
+      expect(mockSetVoiceState).not.toHaveBeenCalledWith('listening');
+
+      // Simulate last chunk ended (should trigger listening)
+      if (lastSource.onended) {
+        lastSource.onended();
+      }
+
+      expect(mockSetVoiceState).toHaveBeenCalledWith('listening');
+    });
+  });
+
+  describe('Audio cleanup', () => {
+    beforeEach(() => {
+      renderResult = render(<HookApp />);
+    });
+
+    test('should clean up buffer source on ended', async () => {
+      await hookData?.queueAudio('dGVzdA==');
+
+      const [source] = createdBufferSources;
+      // Simulate audio ended
+      if (source.onended) {
+        source.onended();
+      }
+
+      expect(source.disconnect).toHaveBeenCalled();
+      expect(source.buffer).toBeNull();
+    });
+
+    test('should stop all audio and close context', async () => {
+      await hookData?.queueAudio('dGVzdA==');
+
+      hookData?.stopAllAudio();
+
+      expect(mockAudioContext.close).toHaveBeenCalled();
+    });
+
+    test('should clear lastSourceRef onended callback on stop', async () => {
+      await hookData?.queueAudio('dGVzdA==');
+      const [source] = createdBufferSources;
+      const onEndedBefore = source.onended;
+
+      expect(onEndedBefore).not.toBeNull();
+
+      hookData?.stopAllAudio();
+
+      // After stopAllAudio, the onended should be cleared
+      expect(source.onended).toBeNull();
+    });
+  });
+
+  describe('Error handling', () => {
+    beforeEach(() => {
+      renderResult = render(<HookApp />);
+    });
+
+    test('should handle invalid base64 data gracefully', async () => {
+      await expect(async () => {
+        await hookData?.queueAudio('invalid-base64!@#');
+      }).not.toThrow();
+    });
+
+    test('should handle AudioContext creation failure', async () => {
+      global.AudioContext = jest.fn(() => {
+        throw new Error('AudioContext not supported');
+      }) as unknown as typeof AudioContext;
+
+      await expect(async () => {
+        await hookData?.queueAudio('dGVzdA==');
+      }).rejects.toThrow('AudioContext not supported');
+    });
+  });
+
+  describe('Real-world scenarios', () => {
+    beforeEach(() => {
+      renderResult = render(<HookApp />);
+    });
+
+    test('should handle streaming audio chunks', async () => {
+      mockAudioBuffer.duration = 0.05; // 50ms chunks
+
+      // Simulate streaming 5 chunks
+      await Promise.all(Array.from({ length: 5 }, (_, i) => hookData?.queueAudio(`chunk${i}`)));
+
+      expect(createdBufferSources).toHaveLength(5);
+      createdBufferSources.forEach(source => {
+        expect(source.start).toHaveBeenCalled();
+      });
+      // Should only call bot_speaking once (first chunk)
+      expect(mockSetVoiceState).toHaveBeenCalledWith('bot_speaking');
+      expect(mockSetVoiceState).toHaveBeenCalledTimes(1);
+    });
+
+    test('should handle playback interruption', async () => {
+      await hookData?.queueAudio('dGVzdA==');
+
+      hookData?.stopAllAudio();
+
+      expect(mockAudioContext.close).toHaveBeenCalled();
+    });
+
+    test('should handle resume after stop', async () => {
+      // Play, stop, then play again
+      await hookData?.queueAudio('dGVzdA==');
+      hookData?.stopAllAudio();
+      await hookData?.queueAudio('dGVzdDI=');
+
+      expect(AudioContext).toHaveBeenCalledTimes(2); // New context after stop
+    });
+
+    test('should reset nextPlayTime after stop allowing immediate playback', async () => {
+      mockAudioBuffer.duration = 0.1;
+
+      await hookData?.queueAudio('dGVzdA==');
+      hookData?.stopAllAudio();
+      mockSetVoiceState.mockClear();
+
+      await hookData?.queueAudio('dGVzdDI=');
+
+      // Should trigger bot_speaking again since it's a fresh start
+      expect(mockSetVoiceState).toHaveBeenCalledWith('bot_speaking');
+    });
+  });
+
+  describe('Performance considerations', () => {
+    beforeEach(() => {
+      renderResult = render(<HookApp />);
+    });
+
+    test('should handle large audio data', async () => {
+      const largeBase64 = 'A'.repeat(10000);
+
+      await expect(async () => {
+        await hookData?.queueAudio(largeBase64);
+      }).not.toThrow();
+    });
+
+    test('should handle rapid successive calls', async () => {
+      // Ensure the mock "base64" data has an even length as Int16Array requires multiples of 2 bytes
+      await Promise.all(Array.from({ length: 100 }, (_, i) => hookData?.queueAudio(`chunk${i}`.padEnd(8, ' '))));
+
+      expect(createdBufferSources).toHaveLength(100);
+      createdBufferSources.forEach(source => {
+        expect(source.start).toHaveBeenCalled();
+      });
+    });
+  });
+});