botframework-webchat 4.13.0 → 4.15.0

package/src/speech/createMicrophoneAudioConfigAndAudioContext.ts

@@ -0,0 +1,146 @@
+import { ChunkedArrayBufferStream } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common/Exports';
+import { PcmRecorder } from 'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports';
+
+import { AudioStreamNode, DeviceInfo, Format } from './CustomAudioInputStream';
+import bytesPerSample from './bytesPerSample';
+import createAudioConfig from './createAudioConfig';
+import createAudioContext from './createAudioContext';
+import getUserMedia from './getUserMedia';
+
+// This is how often we are flushing audio buffer to the network. Modify this value will affect latency.
+const DEFAULT_BUFFER_DURATION_IN_MS = 100;
+
+// TODO: [P2] #3975 We should consider building our own PcmRecorder:
+//       - Use Audio Worklet via blob URL
+//       - Not hardcoding the sample rate or other values
+// PcmRecorder always downscale to 16000 Hz. We cannot use the dynamic value from MediaConstraints or MediaTrackSettings.
+const PCM_RECORDER_HARDCODED_SETTINGS: MediaTrackSettings = Object.freeze({
+  channelCount: 1,
+  sampleRate: 16000,
+  sampleSize: 16
+});
+
+const PCM_RECORDER_HARDCODED_FORMAT: Format = Object.freeze({
+  bitsPerSample: PCM_RECORDER_HARDCODED_SETTINGS.sampleSize,
+  channels: PCM_RECORDER_HARDCODED_SETTINGS.channelCount,
+  samplesPerSec: PCM_RECORDER_HARDCODED_SETTINGS.sampleRate
+});
+
+type MicrophoneAudioInputStreamOptions = {
+  /** Specifies the constraints for selecting an audio device. */
+  audioConstraints?: true | MediaTrackConstraints;
+
+  /** Specifies the `AudioContext` to use. This object must be primed and ready to use. */
+  audioContext: AudioContext;
+
+  /** Specifies the buffering delay on how often to flush audio data to network. Increasing the value will increase audio latency. Default is 100 ms. */
+  bufferDurationInMS?: number;
+
+  /** Specifies whether to display diagnostic information. */
+  debug?: true;
+
+  /** Specifies if telemetry data should be sent. If not specified, telemetry data will NOT be sent. */
+  enableTelemetry?: true;
+
+  /** Specifies the `AudioWorklet` URL for `PcmRecorder`. If not specified, will use script processor on UI thread instead. */
+  pcmRecorderWorkletUrl?: string;
+};
+
+function createMicrophoneAudioConfig(options: MicrophoneAudioInputStreamOptions) {
+  const { audioConstraints, audioContext, debug, enableTelemetry, pcmRecorderWorkletUrl } = options;
+  const bufferDurationInMS = options.bufferDurationInMS || DEFAULT_BUFFER_DURATION_IN_MS;
+
+  const pcmRecorder = new PcmRecorder();
+
+  pcmRecorderWorkletUrl && pcmRecorder.setWorkletUrl(pcmRecorderWorkletUrl);
+
+  return createAudioConfig({
+    async attach(audioNodeId: string): Promise<{
+      audioStreamNode: AudioStreamNode;
+      deviceInfo: DeviceInfo;
+      format: Format;
+    }> {
+      // We need to get new MediaStream on every attach().
+      // This is because PcmRecorder.releaseMediaResources() disconnected/stopped them.
+      const mediaStream = await getUserMedia({ audio: audioConstraints, video: false });
+
+      const [firstAudioTrack] = mediaStream.getAudioTracks();
+
+      if (!firstAudioTrack) {
+        throw new Error('No audio device is found.');
+      }
+
+      const outputStream = new ChunkedArrayBufferStream(
+        // Speech SDK quirks: PcmRecorder hardcoded sample rate of 16000 Hz.
+        bytesPerSample(PCM_RECORDER_HARDCODED_SETTINGS) *
+          // eslint-disable-next-line no-magic-numbers
+          ((bufferDurationInMS || DEFAULT_BUFFER_DURATION_IN_MS) / 1000),
+        audioNodeId
+      );
+
+      pcmRecorder.record(audioContext, mediaStream, outputStream);
+
+      return {
+        audioStreamNode: {
+          // Speech SDK quirks: In SDK's original MicAudioSource implementation, it call turnOff() during detach().
+          //                    That means, it call turnOff(), then detach(), then turnOff() again. Seems redundant.
+          //                    When using with Direct Line Speech, turnOff() is never called.
+          detach: (): Promise<void> => {
+            // Speech SDK quirks: In SDK, it call outputStream.close() in turnOff() before outputStream.readEnded() in detach().
+            //                    I think it make sense to call readEnded() before close().
+            outputStream.readEnded();
+            outputStream.close();
+
+            // PcmRecorder.releaseMediaResources() will disconnect/stop the MediaStream.
+            // We cannot use MediaStream again after turned off.
+            pcmRecorder.releaseMediaResources(audioContext);
+
+            // MediaStream will become inactive after all tracks are removed.
+            mediaStream.getTracks().forEach(track => mediaStream.removeTrack(track));
+
+            // ESLint: "return" is required by TypeScript
+            // eslint-disable-next-line no-useless-return
+            return;
+          },
+          id: () => audioNodeId,
+          read: () => outputStream.read()
+        },
+        deviceInfo: {
+          manufacturer: 'Bot Framework Web Chat',
+          model: enableTelemetry ? firstAudioTrack.label : '',
+          type: enableTelemetry ? 'Microphones' : 'Unknown'
+        },
+        // Speech SDK quirks: PcmRecorder hardcoded sample rate of 16000 Hz.
+        //                    We cannot obtain this number other than looking at their source code.
+        //                    I.e. no getter property.
+        // PcmRecorder always downscale to 16000 Hz. We cannot use the dynamic value from MediaConstraints or MediaTrackSettings.
+        format: PCM_RECORDER_HARDCODED_FORMAT
+      };
+    },
+    debug
+  });
+}
+
+export default function createMicrophoneAudioConfigAndAudioContext({
+  audioContext,
+  audioInputDeviceId,
+  enableTelemetry
+}: {
+  audioContext?: AudioContext;
+  audioInputDeviceId?: string;
+  enableTelemetry?: true;
+}) {
+  // Web Chat has an implementation of AudioConfig for microphone that would enable better support on Safari:
+  // - Maintain same instance of `AudioContext` across recognitions;
+  // - Resume suspended `AudioContext` on user gestures.
+  audioContext || (audioContext = createAudioContext());
+
+  return {
+    audioConfig: createMicrophoneAudioConfig({
+      audioConstraints: audioInputDeviceId ? { deviceId: audioInputDeviceId } : true,
+      audioContext,
+      enableTelemetry: enableTelemetry ? true : undefined
+    }),
+    audioContext
+  };
+}

package/src/speech/getUserMedia.ts

@@ -0,0 +1,14 @@
+export default function getUserMedia(constraints: MediaStreamConstraints): Promise<MediaStream> {
+  const { navigator } = window;
+
+  if (typeof navigator.mediaDevices !== 'undefined') {
+    return navigator.mediaDevices.getUserMedia(constraints);
+  }
+
+  // Although getUserMedia has vendor prefix, they are only used in very old version of browsers.
+  if (typeof navigator.getUserMedia !== 'undefined') {
+    return new Promise((resolve, reject) => navigator.getUserMedia(constraints, resolve, reject));
+  }
+
+  throw new Error('This browser does not support Web Audio API.');
+}

package/src/tsconfig.json

@@ -11,5 +11,5 @@
     "skipLibCheck": true,
     "sourceMap": true
   },
-  "files": ["index.tsx", "index-es5.tsx", "index-minimal.tsx"]
+  "files": ["index.ts", "index-es5.ts", "index-minimal.ts"]
 }

package/src/types/AdaptiveCardsPackage.ts

@@ -0,0 +1,23 @@
+import {
+  AdaptiveCard,
+  GlobalSettings,
+  HorizontalAlignment,
+  HostConfig,
+  TextSize,
+  TextWeight,
+  SerializationContext,
+  Version
+} from 'adaptivecards';
+
+type AdaptiveCardsPackage = {
+  AdaptiveCard: typeof AdaptiveCard;
+  GlobalSettings: typeof GlobalSettings;
+  HorizontalAlignment: typeof HorizontalAlignment;
+  HostConfig: typeof HostConfig;
+  TextSize: typeof TextSize;
+  TextWeight: typeof TextWeight;
+  SerializationContext: typeof SerializationContext;
+  Version: typeof Version;
+};
+
+export default AdaptiveCardsPackage;

package/src/types/CognitiveServicesAudioOutputFormat.ts

@@ -0,0 +1,28 @@
+type CognitiveServicesAudioOutputFormat =
+  | 'audio-16khz-128kbitrate-mono-mp3'
+  | 'audio-16khz-32kbitrate-mono-mp3'
+  | 'audio-16khz-64kbitrate-mono-mp3'
+  | 'audio-24khz-160kbitrate-mono-mp3'
+  | 'audio-24khz-48kbitrate-mono-mp3'
+  | 'audio-24khz-96kbitrate-mono-mp3'
+  | 'audio-48khz-192kbitrate-mono-mp3'
+  | 'audio-48khz-96kbitrate-mono-mp3'
+  | 'ogg-16khz-16bit-mono-opus'
+  | 'ogg-24khz-16bit-mono-opus'
+  | 'ogg-48khz-16bit-mono-opus'
+  | 'raw-16khz-16bit-mono-pcm'
+  | 'raw-16khz-16bit-mono-truesilk'
+  | 'raw-24khz-16bit-mono-pcm'
+  | 'raw-24khz-16bit-mono-truesilk'
+  | 'raw-48khz-16bit-mono-pcm'
+  | 'raw-8khz-8bit-mono-alaw'
+  | 'raw-8khz-8bit-mono-mulaw'
+  | 'riff-16khz-16bit-mono-pcm'
+  | 'riff-24khz-16bit-mono-pcm'
+  | 'riff-48khz-16bit-mono-pcm'
+  | 'riff-8khz-8bit-mono-alaw'
+  | 'riff-8khz-8bit-mono-mulaw'
+  | 'webm-16khz-16bit-mono-opus'
+  | 'webm-24khz-16bit-mono-opus';
+
+export default CognitiveServicesAudioOutputFormat;

package/src/types/CognitiveServicesCredentials.ts

@@ -0,0 +1,28 @@
+type CognitiveServicesAuthorizationToken = {
+  authorizationToken: string;
+};
+
+type CognitiveServicesSubscriptionKey = {
+  subscriptionKey: string;
+};
+
+type CognitiveServicesRegion = {
+  region: string;
+};
+
+type CognitiveServicesSovereignCloud = {
+  customVoiceHostname: string;
+  speechRecognitionHostname: string;
+  speechSynthesisHostname: string;
+};
+
+type CognitiveServicesBaseCredentials = (CognitiveServicesAuthorizationToken | CognitiveServicesSubscriptionKey) &
+  (CognitiveServicesRegion | CognitiveServicesSovereignCloud);
+
+type CognitiveServicesCredentials =
+  | CognitiveServicesBaseCredentials
+  | Promise<CognitiveServicesBaseCredentials>
+  | (() => CognitiveServicesBaseCredentials)
+  | (() => Promise<CognitiveServicesBaseCredentials>);
+
+export default CognitiveServicesCredentials;

package/src/types/CognitiveServicesTextNormalization.ts

@@ -0,0 +1,3 @@
+type CognitiveServicesTextNormalization = 'display' | 'itn' | 'lexical' | 'maskeditn';
+
+export default CognitiveServicesTextNormalization;

package/src/{FullBundleStyleOptions.ts → types/FullBundleStyleOptions.ts}

@@ -1,6 +1,6 @@
 import { StrictStyleOptions, StyleOptions } from 'botframework-webchat-api';
 
-import AdaptiveCardStyleOptions, { StrictAdaptiveCardsStyleOptions } from './adaptiveCards/AdaptiveCardsStyleOptions';
+import AdaptiveCardStyleOptions, { StrictAdaptiveCardsStyleOptions } from '../adaptiveCards/AdaptiveCardsStyleOptions';
 
 type FullBundleStyleOptions = StyleOptions & AdaptiveCardStyleOptions;
 type StrictFullBundleStyleOptions = StrictStyleOptions & StrictAdaptiveCardsStyleOptions;

package/src/{useComposerProps.js → useComposerProps.ts}

@@ -1,4 +1,4 @@
-import { concatMiddleware } from 'botframework-webchat-component';
+import { AttachmentForScreenReaderMiddleware, AttachmentMiddleware } from 'botframework-webchat-api';
 import { useMemo } from 'react';
 
 import createAdaptiveCardsAttachmentForScreenReaderMiddleware from './adaptiveCards/createAdaptiveCardsAttachmentForScreenReaderMiddleware';
@@ -12,32 +12,42 @@ export default function useComposerProps({
   renderMarkdown,
   styleOptions,
   styleSet
-}) {
+}: {
+  attachmentForScreenReaderMiddleware: AttachmentForScreenReaderMiddleware[];
+  attachmentMiddleware: AttachmentMiddleware[];
+  renderMarkdown?: (markdown: string, { markdownRespectCRLF: boolean }, { externalLinkAlt: string }) => string;
+  styleOptions: any;
+  styleSet: any;
+}): {
+  attachmentForScreenReaderMiddleware: AttachmentForScreenReaderMiddleware[];
+  attachmentMiddleware: AttachmentMiddleware[];
+  renderMarkdown: (markdown: string, { markdownRespectCRLF: boolean }, { externalLinkAlt: string }) => string;
+  extraStyleSet: any;
+} {
   const patchedAttachmentMiddleware = useMemo(
-    () => concatMiddleware(attachmentMiddleware, createAdaptiveCardsAttachmentMiddleware()),
+    () => [...attachmentMiddleware, createAdaptiveCardsAttachmentMiddleware()],
     [attachmentMiddleware]
   );
 
   const patchedAttachmentForScreenReaderMiddleware = useMemo(
-    () =>
-      concatMiddleware(attachmentForScreenReaderMiddleware, createAdaptiveCardsAttachmentForScreenReaderMiddleware()),
+    () => [...attachmentForScreenReaderMiddleware, createAdaptiveCardsAttachmentForScreenReaderMiddleware()],
     [attachmentForScreenReaderMiddleware]
   );
 
   // When styleSet is not specified, the styleOptions will be used to create Adaptive Cards styleSet and merged into useStyleSet.
-  const extraStyleSet = useMemo(() => (styleSet ? undefined : createAdaptiveCardsStyleSet(styleOptions)), [
-    styleOptions,
-    styleSet
-  ]);
+  const extraStyleSet = useMemo(
+    () => (styleSet ? undefined : createAdaptiveCardsStyleSet(styleOptions)),
+    [styleOptions, styleSet]
+  );
 
   const patchedRenderMarkdown = useMemo(
-    () => (typeof renderMarkdown === 'undefined' ? (...args) => defaultRenderMarkdown(...args) : renderMarkdown),
+    () => (typeof renderMarkdown === 'undefined' ? defaultRenderMarkdown : renderMarkdown),
     [renderMarkdown]
   );
 
   return {
-    attachmentMiddleware: patchedAttachmentMiddleware,
     attachmentForScreenReaderMiddleware: patchedAttachmentForScreenReaderMiddleware,
+    attachmentMiddleware: patchedAttachmentMiddleware,
     extraStyleSet,
     renderMarkdown: patchedRenderMarkdown
   };

package/webpack.config.js

@@ -14,6 +14,27 @@ let config = {
     'webchat-minimal': './lib/index-minimal.js'
   },
   mode: 'production',
+  module: {
+    rules: [
+      {
+        // To speed up bundling, we are limiting Babel to a number of packages which does not publish ES5 bits.
+        test: /\/node_modules\/(botframework-streaming|buffer|nanoid|postcss|punycode|sanitize-html)\//iu,
+        use: {
+          loader: 'babel-loader',
+          options: {
+            presets: [
+              [
+                '@babel/preset-env',
+                {
+                  modules: 'commonjs'
+                }
+              ]
+            ]
+          }
+        }
+      }
+    ]
+  },
   optimization: {
     minimizer: [
       // Webpack use terser for minification
@@ -30,9 +51,7 @@ let config = {
     ]
   },
   output: {
-    filename: '[name].js',
-    libraryTarget: 'umd',
-    path: resolve(__dirname, 'dist')
+    libraryTarget: 'umd'
   },
   plugins: [
     new StatsWriterPlugin({
@@ -42,14 +61,34 @@ let config = {
   ],
   resolve: {
     alias: {
-      'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioConfig': resolve(
+      // TODO: [P1] #3914 It is smaller to use /lib/ instead of /es2015/ with Webpack.
+      //       Verifies if /es2015/ is better when moving to esbuild.
+      'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports': resolve(
+        __dirname,
+        'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.browser/Exports.js'
+      ),
+      'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.speech/Exports': resolve(
+        __dirname,
+        'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.speech/Exports.js'
+      ),
+      'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common/Exports': resolve(
+        __dirname,
+        'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common/Exports.js'
+      ),
+      'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioStreamFormat': resolve(
         __dirname,
-        'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioConfig.js'
+        'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioStreamFormat.js'
+      ),
+      'microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Exports': resolve(
+        __dirname,
+        'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Exports.js'
       ),
       'microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk': resolve(
         __dirname,
         'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk.js'
       ),
+
+      // This line must be placed after other specific imports.
       'microsoft-cognitiveservices-speech-sdk': resolve(
         __dirname,
         'node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/microsoft.cognitiveservices.speech.sdk.js'
@@ -57,7 +96,8 @@ let config = {
       react: resolve(__dirname, 'node_modules/isomorphic-react/dist/react.js'),
       'react-dom': resolve(__dirname, 'node_modules/isomorphic-react-dom/dist/react-dom.js')
     }
-  }
+  },
+  target: ['web', 'es5']
 };
 
 // VSTS always emits uppercase environment variables.

package/.eslintignore

@@ -1,9 +0,0 @@
-/src/__tests__/**/*
-/src/**/*.spec.js
-/src/**/*.spec.jsx
-/src/**/*.spec.ts
-/src/**/*.spec.tsx
-/src/**/*.test.js
-/src/**/*.test.jsx
-/src/**/*.test.ts
-/src/**/*.test.tsx

package/lib/FullBundleStyleOptions.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"FullBundleStyleOptions.d.ts","sourceRoot":"","sources":["../src/FullBundleStyleOptions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAE5E,OAAO,wBAAwB,EAAE,EAAE,+BAA+B,EAAE,MAAM,2CAA2C,CAAC;AAEtH,aAAK,sBAAsB,GAAG,YAAY,GAAG,wBAAwB,CAAC;AACtE,aAAK,4BAA4B,GAAG,kBAAkB,GAAG,+BAA+B,CAAC;AAEzF,eAAe,sBAAsB,CAAC;AACtC,OAAO,EAAE,4BAA4B,EAAE,CAAC"}

package/src/FullComposer.js

@@ -1,40 +0,0 @@
-import { Components } from 'botframework-webchat-component';
-import PropTypes from 'prop-types';
-import React from 'react';
-
-import AdaptiveCardsComposer from './adaptiveCards/AdaptiveCardsComposer';
-import useComposerProps from './useComposerProps';
-
-const { Composer } = Components;
-
-const FullComposer = props => {
-  const { adaptiveCardsHostConfig, adaptiveCardsPackage, children, ...otherProps } = props;
-  const composerProps = useComposerProps(props);
-
-  return (
-    <AdaptiveCardsComposer
-      adaptiveCardsHostConfig={adaptiveCardsHostConfig}
-      adaptiveCardsPackage={adaptiveCardsPackage}
-    >
-      <Composer {...otherProps} {...composerProps}>
-        {children}
-      </Composer>
-    </AdaptiveCardsComposer>
-  );
-};
-
-FullComposer.defaultProps = {
-  ...Composer.defaultProps,
-  adaptiveCardsHostConfig: undefined,
-  adaptiveCardsPackage: undefined,
-  children: undefined
-};
-
-FullComposer.propTypes = {
-  ...Composer.propTypes,
-  adaptiveCardsHostConfig: PropTypes.any,
-  adaptiveCardsPackage: PropTypes.any,
-  children: PropTypes.any
-};
-
-export default FullComposer;

package/src/FullReactWebChat.js

@@ -1,47 +0,0 @@
-import BasicWebChat from 'botframework-webchat-component';
-import PropTypes from 'prop-types';
-import React, { useEffect } from 'react';
-
-import AdaptiveCardsComposer from './adaptiveCards/AdaptiveCardsComposer';
-import useComposerProps from './useComposerProps';
-
-// Add additional props to <WebChat>, so it support additional features
-const FullReactWebChat = props => {
-  const { adaptiveCardHostConfig, adaptiveCardsHostConfig, adaptiveCardsPackage, ...otherProps } = props;
-
-  useEffect(() => {
-    adaptiveCardHostConfig &&
-      console.warn(
-        'Web Chat: "adaptiveCardHostConfig" is deprecated. Please use "adaptiveCardsHostConfig" instead. "adaptiveCardHostConfig" will be removed on or after 2022-01-01.'
-      );
-  }, [adaptiveCardHostConfig]);
-
-  const composerProps = useComposerProps(props);
-
-  return (
-    <AdaptiveCardsComposer
-      adaptiveCardsHostConfig={adaptiveCardHostConfig || adaptiveCardsHostConfig}
-      adaptiveCardsPackage={adaptiveCardsPackage}
-    >
-      <BasicWebChat {...otherProps} {...composerProps} />
-    </AdaptiveCardsComposer>
-  );
-};
-
-FullReactWebChat.defaultProps = {
-  ...BasicWebChat.defaultProps,
-  adaptiveCardHostConfig: undefined,
-  adaptiveCardsHostConfig: undefined,
-  adaptiveCardsPackage: undefined,
-  renderMarkdown: undefined
-};
-
-FullReactWebChat.propTypes = {
-  ...BasicWebChat.propTypes,
-  adaptiveCardHostConfig: PropTypes.any,
-  adaptiveCardsHostConfig: PropTypes.any,
-  adaptiveCardsPackage: PropTypes.any,
-  renderMarkdown: PropTypes.func
-};
-
-export default FullReactWebChat;

package/src/adaptiveCards/AdaptiveCardsContext.js

@@ -1,5 +0,0 @@
-import { createContext } from 'react';
-
-const AdaptiveCardHostConfigContext = createContext();
-
-export default AdaptiveCardHostConfigContext;