@microsoft/agents-copilotstudio-client 1.1.0-alpha.9.g154c2c8a32 → 1.1.4-g8d884129e7

package/src/copilotStudioClient.ts

@@ -3,8 +3,8 @@
  * Licensed under the MIT License.
  */
 
+import { createEventSource, EventSourceClient } from 'eventsource-client'
 import { ConnectionSettings } from './connectionSettings'
-import axios, { AxiosInstance, AxiosRequestConfig } from 'axios'
 import { getCopilotStudioConnectionUrl, getTokenAudience } from './powerPlatformEnvironment'
 import { Activity, ActivityTypes, ConversationAccount } from '@microsoft/agents-activity'
 import { ExecuteTurnRequest } from './executeTurnRequest'
@@ -14,11 +14,6 @@ import os from 'os'
 
 const logger = debug('copilot-studio:client')
 
-interface streamRead {
-  done: boolean,
-  value: string
-}
-
 /**
  * Client for interacting with Microsoft Copilot Studio services.
  * Provides functionality to start conversations and send messages to Copilot Studio bots.
@@ -33,8 +28,8 @@ export class CopilotStudioClient {
   private conversationId: string = ''
   /** The connection settings for the client. */
   private readonly settings: ConnectionSettings
-  /** The Axios instance used for HTTP requests. */
-  private readonly client: AxiosInstance
+  /** The authenticaton token. */
+  private readonly token: string
 
   /**
    * Returns the scope URL needed to connect to Copilot Studio from the connection settings.
@@ -51,79 +46,97 @@ export class CopilotStudioClient {
    */
   constructor (settings: ConnectionSettings, token: string) {
     this.settings = settings
-    this.client = axios.create()
-    this.client.defaults.headers.common.Authorization = `Bearer ${token}`
-    this.client.defaults.headers.common['User-Agent'] = CopilotStudioClient.getProductInfo()
+    this.token = token
   }
 
-  private async postRequestAsync (axiosConfig: AxiosRequestConfig): Promise<Activity[]> {
-    const activities: Activity[] = []
-
-    logger.debug(`>>> SEND TO ${axiosConfig.url}`)
-
-    const response = await this.client(axiosConfig)
-
-    if (this.settings.useExperimentalEndpoint && !this.settings.directConnectUrl?.trim()) {
-      const islandExperimentalUrl = response.headers?.[CopilotStudioClient.islandExperimentalUrlHeaderKey]
-      if (islandExperimentalUrl) {
-        this.settings.directConnectUrl = islandExperimentalUrl
-        logger.debug(`Island Experimental URL: ${islandExperimentalUrl}`)
-      }
-    }
-
-    this.conversationId = response.headers?.[CopilotStudioClient.conversationIdHeaderKey] ?? ''
-    if (this.conversationId) {
-      logger.debug(`Conversation ID: ${this.conversationId}`)
-    }
+  /**
+   * Streams activities from the Copilot Studio service using eventsource-client.
+   * @param url The connection URL for Copilot Studio.
+   * @param body Optional. The request body (for POST).
+   * @param method Optional. The HTTP method (default: POST).
+   * @returns An async generator yielding the Agent's Activities.
+   */
+  private async * postRequestAsync (url: string, body?: any, method: string = 'POST'): AsyncGenerator<Activity> {
+    logger.debug(`>>> SEND TO ${url}`)
 
-    const sanitizedHeaders = { ...response.headers }
-    delete sanitizedHeaders['Authorization']
-    delete sanitizedHeaders[CopilotStudioClient.conversationIdHeaderKey]
-    logger.debug('Headers received:', sanitizedHeaders)
+    const streamMap = new Map<string, { text: string, sequence: number }[]>()
 
-    const stream = response.data
-    const reader = stream.pipeThrough(new TextDecoderStream()).getReader()
-    let result: string = ''
-    const results: string[] = []
-
-    const processEvents = async ({ done, value }: streamRead): Promise<string[]> => {
-      if (done) {
-        logger.debug('Stream complete')
-        result += value
-        results.push(result)
-        return results
+    const eventSource: EventSourceClient = createEventSource({
+      url,
+      headers: {
+        Authorization: `Bearer ${this.token}`,
+        'User-Agent': CopilotStudioClient.getProductInfo(),
+        'Content-Type': 'application/json',
+        Accept: 'text/event-stream'
+      },
+      body: body ? JSON.stringify(body) : undefined,
+      method,
+      fetch: async (url, init) => {
+        const response = await fetch(url, init)
+        this.processResponseHeaders(response.headers)
+        return response
       }
-      logger.info('Agent is typing ...')
-      result += value
+    })
 
-      return await processEvents(await reader.read())
-    }
+    try {
+      for await (const { data, event } of eventSource) {
+        if (data && event === 'activity') {
+          try {
+            const activity = Activity.fromJson(data)
 
-    const events: string[] = await reader.read().then(processEvents)
-
-    events.forEach(event => {
-      const values: string[] = event.toString().split('\n')
-      const validEvents = values.filter(e => e.substring(0, 4) === 'data' && e !== 'data: end\r')
-      validEvents.forEach(ve => {
-        try {
-          const act = Activity.fromJson(ve.substring(5, ve.length))
-          if (act.type === ActivityTypes.Message) {
-            activities.push(act)
-            if (!this.conversationId.trim()) {
-              // Did not get it from the header.
-              this.conversationId = act.conversation?.id ?? ''
-              logger.debug(`Conversation ID: ${this.conversationId}`)
+            // check to see if this activity is part of the streamed response, in which case we need to accumulate the text
+            const streamingEntity = activity.entities?.find(e => e.type === 'streaminfo' && e.streamType === 'streaming')
+            switch (activity.type) {
+              case ActivityTypes.Message:
+                if (!this.conversationId.trim()) { // Did not get it from the header.
+                  this.conversationId = activity.conversation?.id ?? ''
+                  logger.debug(`Conversation ID: ${this.conversationId}`)
+                }
+                yield activity
+                break
+              case ActivityTypes.Typing:
+                logger.debug(`Activity type: ${activity.type}`)
+                // Accumulate the text as it comes in from the stream.
+                // This also accounts for the "old style" of streaming where the stream info is in channelData.
+                if (streamingEntity || activity.channelData?.streamType === 'streaming') {
+                  const text = activity.text ?? ''
+                  const id = (streamingEntity?.streamId ?? activity.channelData?.streamId)
+                  const sequence = (streamingEntity?.streamSequence ?? activity.channelData?.streamSequence)
+                  // Accumulate the text chunks based on stream ID and sequence number.
+                  if (id && sequence) {
+                    if (streamMap.has(id)) {
+                      const existing = streamMap.get(id)!
+                      existing.push({ text, sequence })
+                      streamMap.set(id, existing)
+                    } else {
+                      streamMap.set(id, [{ text, sequence }])
+                    }
+                    activity.text = streamMap.get(id)?.sort((a, b) => a.sequence - b.sequence).map(item => item.text).join('') || ''
+                  }
+                }
+                yield activity
+                break
+              default:
+                logger.debug(`Activity type: ${activity.type}`)
+                yield activity
+                break
             }
-          } else {
-            logger.debug(`Activity type: ${act.type}`)
+          } catch (error) {
+            logger.error('Failed to parse activity:', error)
           }
-        } catch (e) {
-          logger.error('Error: ', e)
-          throw e
+        } else if (event === 'end') {
+          logger.debug('Stream complete')
+          break
         }
-      })
-    })
-    return activities
+
+        if (eventSource.readyState === 'closed') {
+          logger.debug('Connection closed')
+          break
+        }
+      }
+    } finally {
+      eventSource.close()
+    }
   }
 
   /**
@@ -146,43 +159,83 @@ export class CopilotStudioClient {
     return userAgent
   }
 
+  private processResponseHeaders (responseHeaders: Headers): void {
+    if (this.settings.useExperimentalEndpoint && !this.settings.directConnectUrl?.trim()) {
+      const islandExperimentalUrl = responseHeaders?.get(CopilotStudioClient.islandExperimentalUrlHeaderKey)
+      if (islandExperimentalUrl) {
+        this.settings.directConnectUrl = islandExperimentalUrl
+        logger.debug(`Island Experimental URL: ${islandExperimentalUrl}`)
+      }
+    }
+
+    this.conversationId = responseHeaders?.get(CopilotStudioClient.conversationIdHeaderKey) ?? ''
+    if (this.conversationId) {
+      logger.debug(`Conversation ID: ${this.conversationId}`)
+    }
+
+    const sanitizedHeaders = new Headers()
+    responseHeaders.forEach((value, key) => {
+      if (key.toLowerCase() !== 'authorization' && key.toLowerCase() !== CopilotStudioClient.conversationIdHeaderKey.toLowerCase()) {
+        sanitizedHeaders.set(key, value)
+      }
+    })
+    logger.debug('Headers received:', sanitizedHeaders)
+  }
+
   /**
    * Starts a new conversation with the Copilot Studio service.
    * @param emitStartConversationEvent Whether to emit a start conversation event. Defaults to true.
-   * @returns A promise that resolves to the initial activity of the conversation.
+   * @returns An async generator yielding the Agent's Activities.
    */
-  public async startConversationAsync (emitStartConversationEvent: boolean = true): Promise<Activity> {
+  public async * startConversationStreaming (emitStartConversationEvent: boolean = true): AsyncGenerator<Activity> {
     const uriStart: string = getCopilotStudioConnectionUrl(this.settings)
     const body = { emitStartConversationEvent }
 
-    const config: AxiosRequestConfig = {
-      method: 'post',
-      url: uriStart,
-      headers: {
-        Accept: 'text/event-stream',
-        'Content-Type': 'application/json',
-      },
-      data: body,
-      responseType: 'stream',
-      adapter: 'fetch'
-    }
-
     logger.info('Starting conversation ...')
-    const values = await this.postRequestAsync(config)
-    const act = values[0]
-    logger.info(`Conversation '${act.conversation?.id}' started. Received ${values.length} activities.`, values)
-    return act
+
+    yield * this.postRequestAsync(uriStart, body, 'POST')
+  }
+
+  /**
+   * Sends an activity to the Copilot Studio service and retrieves the response activities.
+   * @param activity The activity to send.
+   * @param conversationId The ID of the conversation. Defaults to the current conversation ID.
+   * @returns An async generator yielding the Agent's Activities.
+   */
+  public async * sendActivityStreaming (activity: Activity, conversationId: string = this.conversationId) : AsyncGenerator<Activity> {
+    const localConversationId = activity.conversation?.id ?? conversationId
+    const uriExecute = getCopilotStudioConnectionUrl(this.settings, localConversationId)
+    const qbody: ExecuteTurnRequest = new ExecuteTurnRequest(activity)
+
+    logger.info('Sending activity...', activity)
+    yield * this.postRequestAsync(uriExecute, qbody, 'POST')
+  }
+
+  /**
+   * Starts a new conversation with the Copilot Studio service.
+   * @param emitStartConversationEvent Whether to emit a start conversation event. Defaults to true.
+   * @returns A promise yielding an array of activities.
+   * @deprecated Use startConversationStreaming instead.
+   */
+  public async startConversationAsync (emitStartConversationEvent: boolean = true): Promise<Activity[]> {
+    const result: Activity[] = []
+    for await (const value of this.startConversationStreaming(emitStartConversationEvent)) {
+      result.push(value)
+    }
+    return result
   }
 
   /**
    * Sends a question to the Copilot Studio service and retrieves the response activities.
    * @param question The question to ask.
    * @param conversationId The ID of the conversation. Defaults to the current conversation ID.
-   * @returns A promise that resolves to an array of activities containing the responses.
+   * @returns A promise yielding an array of activities.
+   * @deprecated Use sendActivityStreaming instead.
    */
-  public async askQuestionAsync (question: string, conversationId: string = this.conversationId) {
+  public async askQuestionAsync (question: string, conversationId?: string) : Promise<Activity[]> {
+    const localConversationId = conversationId?.trim() ? conversationId : this.conversationId
     const conversationAccount: ConversationAccount = {
-      id: conversationId
+      id: localConversationId
     }
     const activityObj = {
       type: 'message',
@@ -191,34 +244,25 @@ export class CopilotStudioClient {
     }
     const activity = Activity.fromObject(activityObj)
 
-    return this.sendActivity(activity)
+    const result: Activity[] = []
+    for await (const value of this.sendActivityStreaming(activity, conversationId)) {
+      result.push(value)
+    }
+    return result
   }
 
   /**
    * Sends an activity to the Copilot Studio service and retrieves the response activities.
    * @param activity The activity to send.
    * @param conversationId The ID of the conversation. Defaults to the current conversation ID.
-   * @returns A promise that resolves to an array of activities containing the responses.
+   * @returns A promise yielding an array of activities.
+   * @deprecated Use sendActivityStreaming instead.
    */
-  public async sendActivity (activity: Activity, conversationId: string = this.conversationId) {
-    const localConversationId = activity.conversation?.id ?? conversationId
-    const uriExecute = getCopilotStudioConnectionUrl(this.settings, localConversationId)
-    const qbody: ExecuteTurnRequest = new ExecuteTurnRequest(activity)
-
-    const config: AxiosRequestConfig = {
-      method: 'post',
-      url: uriExecute,
-      headers: {
-        Accept: 'text/event-stream',
-        'Content-Type': 'application/json',
-      },
-      data: qbody,
-      responseType: 'stream',
-      adapter: 'fetch'
+  public async sendActivity (activity: Activity, conversationId: string = this.conversationId) : Promise<Activity[]> {
+    const result: Activity[] = []
+    for await (const value of this.sendActivityStreaming(activity, conversationId)) {
+      result.push(value)
     }
-    logger.info('Sending activity...', activity)
-    const values = await this.postRequestAsync(config)
-    logger.info(`Received ${values.length} activities.`, values)
-    return values
+    return result
   }
 }

package/src/copilotStudioWebChat.ts

@@ -5,7 +5,7 @@
 
 import { v4 as uuid } from 'uuid'
 
-import { Activity, ConversationAccount } from '@microsoft/agents-activity'
+import { Activity, Attachment, ConversationAccount } from '@microsoft/agents-activity'
 import { Observable, BehaviorSubject, type Subscriber } from 'rxjs'
 
 import { CopilotStudioClient } from './copilotStudioClient'
@@ -201,7 +201,7 @@ export class CopilotStudioWebChat {
   static createConnection (
     client: CopilotStudioClient,
     settings?: CopilotStudioWebChatSettings
-  ):CopilotStudioWebChatConnection {
+  ): CopilotStudioWebChatConnection {
     logger.info('--> Creating connection between Copilot Studio and WebChat ...')
     let sequence = 0
     let activitySubscriber: Subscriber<Partial<Activity>> | undefined
@@ -211,17 +211,25 @@ export class CopilotStudioWebChat {
     const activity$ = createObservable<Partial<Activity>>(async (subscriber) => {
       activitySubscriber = subscriber
 
-      if (connectionStatus$.value < 2) {
+      const handleAcknowledgementOnce = async (): Promise<void> => {
         connectionStatus$.next(2)
-        return
+        await 0 // Webchat requires an extra tick to process the connection status change
       }
 
       logger.debug('--> Connection established.')
       notifyTyping()
-      const activity = await client.startConversationAsync()
-      conversation = activity.conversation
-      sequence = 0
-      notifyActivity(activity)
+
+      if (connectionStatus$.value < 2) {
+        for await (const activity of client.startConversationStreaming()) {
+          delete activity.replyToId
+          if (!conversation && activity.conversation) {
+            conversation = activity.conversation
+            await handleAcknowledgementOnce()
+          }
+
+          notifyActivity(activity)
+        }
+      }
     })
 
     const notifyActivity = (activity: Partial<Activity>) => {
@@ -230,9 +238,10 @@ export class CopilotStudioWebChat {
         timestamp: new Date().toISOString(),
         channelData: {
           ...activity.channelData,
-          'webchat:sequence-id': sequence++,
+          'webchat:sequence-id': sequence,
         },
       }
+      sequence++
       logger.debug(`Notify '${newActivity.type}' activity to WebChat:`, newActivity)
       activitySubscriber?.next(newActivity)
     }
@@ -264,22 +273,26 @@ export class CopilotStudioWebChat {
 
         return createObservable<string>(async (subscriber) => {
           try {
-            const id = uuid()
-
             logger.info('--> Sending activity to Copilot Studio ...')
+            const newActivity = Activity.fromObject({
+              ...activity,
+              id: uuid(),
+              attachments: await processAttachments(activity)
+            })
 
-            notifyActivity({ ...activity, id })
+            notifyActivity(newActivity)
             notifyTyping()
 
-            const activities = await client.sendActivity(activity)
+            // Notify WebChat immediately that the message was sent
+            subscriber.next(newActivity.id!)
 
-            for (const responseActivity of activities) {
+            // Stream the agent's response, but don't block the UI
+            for await (const responseActivity of client.sendActivityStreaming(newActivity)) {
               notifyActivity(responseActivity)
+              logger.info('<-- Activity received correctly from Copilot Studio.')
             }
 
-            subscriber.next(id)
             subscriber.complete()
-            logger.info('--> Activity received correctly from Copilot Studio.')
           } catch (error) {
             logger.error('Error sending Activity to Copilot Studio:', error)
             subscriber.error(error)
@@ -299,6 +312,74 @@ export class CopilotStudioWebChat {
   }
 }
 
+/**
+ * Processes activity attachments.
+ * @param activity The activity to process for attachments.
+ * @returns A promise that resolves to the activity with all attachments converted.
+ */
+async function processAttachments (activity: Activity): Promise<Attachment[]> {
+  if (activity.type !== 'message' || !activity.attachments?.length) {
+    return activity.attachments || []
+  }
+
+  const attachments: Attachment[] = []
+  for (const attachment of activity.attachments) {
+    const processed = await processBlobAttachment(attachment)
+    attachments.push(processed)
+  }
+
+  return attachments
+}
+
+/**
+ * Processes a blob attachment to convert its content URL to a data URL.
+ * @param attachment The attachment to process.
+ * @returns A promise that resolves to the processed attachment.
+ */
+async function processBlobAttachment (attachment: Attachment): Promise<Attachment> {
+  let newContentUrl = attachment.contentUrl
+  if (!newContentUrl?.startsWith('blob:')) {
+    return attachment
+  }
+
+  try {
+    const response = await fetch(newContentUrl)
+    if (!response.ok) {
+      throw new Error(`Failed to fetch blob URL: ${response.status} ${response.statusText}`)
+    }
+
+    const blob = await response.blob()
+    const arrayBuffer = await blob.arrayBuffer()
+    const base64 = arrayBufferToBase64(arrayBuffer)
+    newContentUrl = `data:${blob.type};base64,${base64}`
+  } catch (error) {
+    newContentUrl = attachment.contentUrl
+    logger.error('Error processing blob attachment:', newContentUrl, error)
+  }
+
+  return { ...attachment, contentUrl: newContentUrl }
+}
+
+/**
+ * Converts an ArrayBuffer to a base64 string.
+ * @param buffer The ArrayBuffer to convert.
+ * @returns The base64 encoded string.
+ */
+function arrayBufferToBase64 (buffer: ArrayBuffer): string {
+  // Node.js environment
+  const BufferClass = typeof globalThis.Buffer === 'function' ? globalThis.Buffer : undefined
+  if (BufferClass && typeof BufferClass.from === 'function') {
+    return BufferClass.from(buffer).toString('base64')
+  }
+
+  // Browser environment
+  let binary = ''
+  for (const byte of new Uint8Array(buffer)) {
+    binary += String.fromCharCode(byte)
+  }
+  return btoa(binary)
+}
+
 /**
  * Creates an RxJS Observable that wraps an asynchronous function execution.
  *