botframework-webchat-core 4.18.1 → 4.19.0

package/src/utils/getOrgSchemaMessage.ts

@@ -1,21 +1,21 @@
-import { type WebChatActivity } from '../types/WebChatActivity';
-import { parseCreativeWork, type CreativeWork } from '../types/external/OrgSchema/CreativeWork';
-import { parseThing } from '../types/external/OrgSchema/Thing';
+import { parse } from 'valibot';
+import { orgSchemaCreativeWorkSchema, type OrgSchemaCreativeWork } from '@msinternal/botframework-webchat-core-json-ld';
+import type { WebChatActivity } from '../types/WebChatActivity';
 
-export default function getOrgSchemaMessage(
-  graph: readonly WebChatActivity['entities'][0][]
-): CreativeWork | undefined {
-  const messageEntity = (graph || []).find(entity => {
-    const isThing = entity.type?.startsWith('https://schema.org/');
+type EntityType = NonNullable<WebChatActivity['entities']>[number];
 
-    if (isThing) {
-      const thing = parseThing(entity);
+export default function getOrgSchemaMessage(graph: readonly EntityType[]): OrgSchemaCreativeWork | undefined {
+  for (const entity of graph ?? []) {
+    const isPossiblySelfMessage =
+      entity &&
+      typeof entity === 'object' &&
+      'type' in entity &&
+      entity.type?.startsWith('https://schema.org/') &&
+      '@id' in entity &&
+      entity['@id'] === '';
 
-      return thing['@id'] === '';
+    if (isPossiblySelfMessage) {
+      return parse(orgSchemaCreativeWorkSchema, entity);
     }
-  });
-
-  const message = messageEntity && parseCreativeWork(messageEntity);
-
-  return message && parseCreativeWork(message);
+  }
 }

package/src/utils/onErrorResumeNext.ts

@@ -9,7 +9,7 @@ export default function onErrorResumeNext<T, U = T>(fn: () => T, defaultValue: U
 export default function onErrorResumeNext<T>(fn: () => T, defaultValue?: T): T | undefined {
   try {
     return fn();
-  } catch (error) {
+  } catch {
     return defaultValue;
   }
 }

package/src/utils/voiceActivity/getVoiceActivityRole.spec.ts

@@ -0,0 +1,150 @@
+import getVoiceActivityRole from './getVoiceActivityRole';
+import { WebChatActivity } from '../../types/WebChatActivity';
+
+// Mock activity factory for testing
+const createMockActivity = (type: string = 'event', name?: string, value?: any, valueType?: string): WebChatActivity =>
+  ({
+    type: type as any,
+    id: 'test-activity-id',
+    from: { id: 'test-user' },
+    channelData: {
+      'webchat:sequence-id': 1
+    },
+    ...(name && { name }),
+    ...(value && { value }),
+    ...(valueType && { valueType })
+  }) as WebChatActivity;
+
+const createMockVoiceActivity = (
+  name: string,
+  value: Record<string, any>,
+  valueType: string = 'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+): WebChatActivity => createMockActivity('event', name, value, valueType);
+
+const createMockTranscriptActivity = (
+  origin: 'user' | 'agent',
+  transcription: string = 'test',
+  valueType: string = 'application/vnd.microsoft.activity.azure.directline.audio.transcript'
+): WebChatActivity => createMockActivity('event', 'media.end', { transcription, origin }, valueType);
+
+describe('getVoiceActivityRole', () => {
+  describe.each([
+    ['user', 'user', 'Hello world'],
+    ['user', 'user', '']
+  ] as const)('Voice transcript activities with origin %s', (expectedRole, origin, transcription) => {
+    test(`should return "${expectedRole}" for media.end with origin ${origin}${transcription ? '' : ' and empty transcription'}`, () => {
+      const activity = createMockTranscriptActivity(origin, transcription);
+
+      const result = getVoiceActivityRole(activity);
+
+      expect(result).toBe(expectedRole);
+    });
+  });
+
+  describe.each([
+    ['bot', 'agent', 'Hello, how can I help you?'],
+    ['bot', 'agent', '']
+  ] as const)('Voice transcript activities with origin %s', (expectedRole, origin, transcription) => {
+    test(`should return "${expectedRole}" for media.end with origin ${origin}${transcription ? '' : ' and empty transcription'}`, () => {
+      const activity = createMockTranscriptActivity(origin, transcription);
+
+      const result = getVoiceActivityRole(activity);
+
+      expect(result).toBe(expectedRole);
+    });
+  });
+
+  describe('Non-transcript voice activities', () => {
+    test.each([
+      ['media.chunk', { content: 'base64' }, 'application/vnd.microsoft.activity.azure.directline.audio.chunk'],
+      ['request.update', { state: 'detected' }, 'application/vnd.microsoft.activity.azure.directline.audio.state']
+    ])('should return undefined for %s', (name, value, valueType) => {
+      const activity = createMockVoiceActivity(name, value, valueType);
+
+      const result = getVoiceActivityRole(activity);
+
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('Non-voice activities', () => {
+    test.each([
+      ['message', 'regular message activity'],
+      ['typing', 'typing activity']
+    ])('should return undefined for %s', type => {
+      const activity = createMockActivity(type);
+
+      const result = getVoiceActivityRole(activity);
+
+      expect(result).toBeUndefined();
+    });
+
+    test('should return undefined for event activity without audio valueType', () => {
+      const activity = createMockActivity('event', 'test', { someOtherData: 'test' }, 'application/json');
+
+      const result = getVoiceActivityRole(activity);
+
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('Real-world scenarios', () => {
+    test('should correctly identify user transcript in conversation flow', () => {
+      const userActivities = [
+        createMockVoiceActivity(
+          'request.update',
+          { state: 'detected' },
+          'application/vnd.microsoft.activity.azure.directline.audio.state'
+        ),
+        createMockVoiceActivity(
+          'media.chunk',
+          { content: 'base64' },
+          'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+        ),
+        createMockTranscriptActivity('user', 'What is the weather today?')
+      ];
+
+      const roles = userActivities.map(activity => getVoiceActivityRole(activity));
+
+      expect(roles).toEqual([undefined, undefined, 'user']);
+    });
+
+    test('should correctly identify bot transcript in conversation flow', () => {
+      const botActivities = [
+        createMockVoiceActivity(
+          'media.chunk',
+          { content: 'chunk1' },
+          'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+        ),
+        createMockVoiceActivity(
+          'media.chunk',
+          { content: 'chunk2' },
+          'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+        ),
+        createMockTranscriptActivity('agent', 'Today will be sunny with a high of 75 degrees.'),
+        createMockVoiceActivity(
+          'request.update',
+          { state: 'processing' },
+          'application/vnd.microsoft.activity.azure.directline.audio.state'
+        )
+      ];
+
+      const roles = botActivities.map(activity => getVoiceActivityRole(activity));
+
+      expect(roles).toEqual([undefined, undefined, 'bot', undefined]);
+    });
+
+    test('should handle mixed activity types in conversation', () => {
+      const mixedActivities = [
+        createMockActivity('message'),
+        createMockTranscriptActivity('user', 'Hello'),
+        createMockActivity('typing'),
+        createMockTranscriptActivity('agent', 'Hi there!')
+      ];
+
+      const roles = mixedActivities.map(activity => getVoiceActivityRole(activity));
+
+      expect(roles).toEqual([undefined, 'user', undefined, 'bot']);
+    });
+  });
+});

package/src/utils/voiceActivity/getVoiceActivityRole.ts

@@ -0,0 +1,16 @@
+import { WebChatActivity } from '../../types/WebChatActivity';
+import isVoiceTranscriptActivity from './isVoiceTranscriptActivity';
+
+const getVoiceActivityRole = (activity: WebChatActivity): 'bot' | 'user' | undefined => {
+  if (isVoiceTranscriptActivity(activity)) {
+    if (activity.value.origin === 'agent') {
+      return 'bot';
+    } else if (activity.value.origin === 'user') {
+      return 'user';
+    }
+  }
+
+  return undefined;
+};
+
+export default getVoiceActivityRole;

package/src/utils/voiceActivity/getVoiceActivityText.spec.ts

@@ -0,0 +1,100 @@
+import getVoiceActivityText from './getVoiceActivityText';
+import { WebChatActivity } from '../../types/WebChatActivity';
+
+// Mock activity factory for testing
+const createMockActivity = (type: string = 'event', name?: string, value?: any, valueType?: string): WebChatActivity =>
+  ({
+    type: type as any,
+    id: 'test-activity-id',
+    from: { id: 'test-user' },
+    channelData: {
+      'webchat:sequence-id': 1
+    },
+    ...(name && { name }),
+    ...(value && { value }),
+    ...(valueType && { valueType })
+  }) as WebChatActivity;
+
+const createMockTranscriptActivity = (
+  transcription: string | undefined,
+  origin: 'user' | 'agent' = 'user',
+  valueType: string = 'application/vnd.microsoft.activity.azure.directline.audio.transcript'
+): WebChatActivity =>
+  createMockActivity(
+    'event',
+    'media.end',
+    transcription !== undefined ? { transcription, origin } : { origin },
+    valueType
+  );
+
+describe('getVoiceActivityText', () => {
+  describe('Voice transcript activities', () => {
+    test.each([
+      ['Hello world', 'Hello world'],
+      ['How can I help you today?', 'How can I help you today?'],
+      ['', '']
+    ])('should return %p for media.end with transcription %p', (expected, transcription) => {
+      const activity = createMockTranscriptActivity(transcription);
+
+      const result = getVoiceActivityText(activity);
+
+      expect(result).toBe(expected);
+    });
+
+    test('should return undefined for media.end without transcript property', () => {
+      const activity = createMockTranscriptActivity(undefined);
+
+      const result = getVoiceActivityText(activity);
+
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('Non-transcript voice activities', () => {
+    test.each([['media.chunk'], ['request.update']])('should return undefined for %s activity', name => {
+      const activity = createMockActivity(
+        'event',
+        name,
+        { content: 'base64' },
+        'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+      );
+
+      const result = getVoiceActivityText(activity);
+
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('Non-voice activities', () => {
+    test.each([
+      ['message', undefined, undefined, undefined],
+      ['event', undefined, { someData: 'test' }, undefined]
+    ])('should return undefined for %s', (type, name, value, valueType) => {
+      const activity = createMockActivity(type, name, value, valueType);
+
+      const result = getVoiceActivityText(activity);
+
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('Edge cases', () => {
+    test('should handle transcript with whitespace only', () => {
+      const activity = createMockTranscriptActivity('   ');
+
+      const result = getVoiceActivityText(activity);
+
+      expect(result).toBe('   ');
+    });
+
+    test('should handle very long transcript', () => {
+      const longText = 'A'.repeat(10000);
+      const activity = createMockTranscriptActivity(longText);
+
+      const result = getVoiceActivityText(activity);
+
+      expect(result).toBe(longText);
+      expect(result?.length).toBe(10000);
+    });
+  });
+});

package/src/utils/voiceActivity/getVoiceActivityText.ts

@@ -0,0 +1,11 @@
+import { WebChatActivity } from '../../types/WebChatActivity';
+import isVoiceTranscriptActivity from './isVoiceTranscriptActivity';
+
+const getVoiceActivityText = (activity: WebChatActivity): string | undefined => {
+  if (isVoiceTranscriptActivity(activity)) {
+    return activity.value.transcription;
+  }
+  return undefined;
+};
+
+export default getVoiceActivityText;

package/src/utils/voiceActivity/isVoiceActivity.spec.ts

@@ -0,0 +1,202 @@
+import isVoiceActivity from './isVoiceActivity';
+import { WebChatActivity } from '../../types/WebChatActivity';
+
+// Mock activity factory for testing
+const createMockActivity = (type: string = 'event', name?: string, value?: any, valueType?: string): WebChatActivity =>
+  ({
+    type: type as any,
+    id: 'test-activity-id',
+    from: { id: 'test-user' },
+    channelData: {
+      'webchat:sequence-id': 1
+    },
+    ...(name && { name }),
+    ...(value && { value }),
+    ...(valueType && { valueType })
+  }) as WebChatActivity;
+
+const createMockVoiceActivity = (
+  name: string,
+  value: Record<string, any>,
+  valueType: string = 'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+): WebChatActivity => createMockActivity('event', name, value, valueType);
+
+const createMockDtmfActivity = (name: string, value: Record<string, any>): WebChatActivity =>
+  createMockActivity('event', name, value, 'application/vnd.microsoft.activity.ccv2.dtmf');
+
+describe('isVoiceActivity', () => {
+  describe('Valid voice activities', () => {
+    test('should return true for event activity with azure directline audio valueType', () => {
+      const activity = createMockVoiceActivity(
+        'media.chunk',
+        { content: 'base64', contentType: 'audio/webm' },
+        'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+      );
+
+      const result = isVoiceActivity(activity);
+
+      expect(result).toBe(true);
+    });
+
+    test('should return true for event activity with ccv2 audio valueType', () => {
+      const activity = createMockVoiceActivity(
+        'media.chunk',
+        { content: 'base64' },
+        'application/vnd.microsoft.activity.ccv2.audio.chunk'
+      );
+
+      const result = isVoiceActivity(activity);
+
+      expect(result).toBe(true);
+    });
+
+    test('should return true for event activity with dtmf valueType', () => {
+      const activity = createMockDtmfActivity('media.end', { key: '1' });
+
+      const result = isVoiceActivity(activity);
+
+      expect(result).toBe(true);
+    });
+
+    test('should return true for request.update with audio.state valueType', () => {
+      const activity = createMockVoiceActivity(
+        'request.update',
+        { state: 'detected', message: 'Your request is identified' },
+        'application/vnd.microsoft.activity.azure.directline.audio.state'
+      );
+
+      const result = isVoiceActivity(activity);
+
+      expect(result).toBe(true);
+    });
+  });
+
+  describe('Invalid activities', () => {
+    const testCases = [
+      {
+        name: 'message activity with audio valueType',
+        activity: () =>
+          createMockActivity(
+            'message',
+            'media.chunk',
+            { content: 'base64' },
+            'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+          )
+      },
+      {
+        name: 'typing activity',
+        activity: () => createMockActivity('typing')
+      },
+      {
+        name: 'event activity with non-audio valueType',
+        activity: () => createMockActivity('event', 'test', { data: 'test' }, 'application/json')
+      },
+      {
+        name: 'event activity without valueType',
+        activity: () => createMockActivity('event', 'test', { someData: 'value' })
+      },
+      {
+        name: 'event activity with no value',
+        activity: () =>
+          createMockActivity(
+            'event',
+            'test',
+            undefined,
+            'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+          )
+      },
+      {
+        name: 'event activity with no name',
+        activity: () =>
+          createMockActivity(
+            'event',
+            undefined,
+            { data: 'test' },
+            'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+          )
+      }
+    ];
+
+    test.each(testCases)('should return false for $name', ({ activity }) => {
+      const result = isVoiceActivity(activity());
+
+      expect(result).toBe(false);
+    });
+  });
+
+  describe('Real-world voice activity scenarios', () => {
+    const voiceScenarios = [
+      {
+        name: 'request.update with speech detected state',
+        eventName: 'request.update',
+        value: { state: 'detected', message: 'Your request is identified' },
+        valueType: 'application/vnd.microsoft.activity.azure.directline.audio.state'
+      },
+      {
+        name: 'request.update with processing state',
+        eventName: 'request.update',
+        value: { state: 'processing', message: 'Your request is being processed' },
+        valueType: 'application/vnd.microsoft.activity.azure.directline.audio.state'
+      },
+      {
+        name: 'media.end with user transcription',
+        eventName: 'media.end',
+        value: { transcription: 'My destination is bangalore', origin: 'user' },
+        valueType: 'application/vnd.microsoft.activity.azure.directline.audio.transcript'
+      },
+      {
+        name: 'media.chunk with server audio response',
+        eventName: 'media.chunk',
+        value: { content: 'base64chunk', contentType: 'audio/webm' },
+        valueType: 'application/vnd.microsoft.activity.azure.directline.audio.chunk'
+      },
+      {
+        name: 'media.end with bot transcription',
+        eventName: 'media.end',
+        value: { transcription: 'Your destination is at 1000m above sea level', origin: 'agent' },
+        valueType: 'application/vnd.microsoft.activity.azure.directline.audio.transcript'
+      }
+    ];
+
+    test.each(voiceScenarios)('should return true for $name', ({ eventName, value, valueType }) => {
+      const activity = createMockVoiceActivity(eventName, value, valueType);
+
+      const result = isVoiceActivity(activity);
+
+      expect(result).toBe(true);
+    });
+  });
+
+  describe('Real-world DTMF activity scenarios', () => {
+    const dtmfScenarios = [
+      {
+        name: 'DTMF with digit 1',
+        eventName: 'media.end',
+        value: { key: '1' }
+      },
+      {
+        name: 'DTMF with star key',
+        eventName: 'media.end',
+        value: { key: '*' }
+      },
+      {
+        name: 'DTMF with hash key',
+        eventName: 'media.end',
+        value: { key: '#' }
+      },
+      {
+        name: 'DTMF with digit 5',
+        eventName: 'media.end',
+        value: { key: '5' }
+      }
+    ];
+
+    test.each(dtmfScenarios)('should return true for $name', ({ eventName, value }) => {
+      const activity = createMockDtmfActivity(eventName, value);
+
+      const result = isVoiceActivity(activity);
+
+      expect(result).toBe(true);
+    });
+  });
+});

package/src/utils/voiceActivity/isVoiceActivity.ts

@@ -0,0 +1,21 @@
+import { check, is, literal, looseObject, object, pipe, string, type InferOutput } from 'valibot';
+
+import { WebChatActivity } from '../../types/WebChatActivity';
+
+// Activity spec proposal - https://github.com/microsoft/Agents/issues/416
+// valueType: contains 'audio' or 'dtmf' (works with any server prefix like azure.directline, ccv2, etc.)
+const VoiceActivitySchema = object({
+  name: string(),
+  type: literal('event'),
+  value: looseObject({}),
+  valueType: pipe(
+    string(),
+    check(value => value.includes('audio') || value.includes('dtmf'))
+  )
+});
+
+const isVoiceActivity = (
+  activity: WebChatActivity
+): activity is WebChatActivity & InferOutput<typeof VoiceActivitySchema> => is(VoiceActivitySchema, activity);
+
+export default isVoiceActivity;