@microsoft/agents-hosting 1.1.4-geb1c05c291 → 1.2.0-alpha.19.g9aeee229e8

package/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package.json",
   "name": "@microsoft/agents-hosting",
-  "version": "1.1.4-geb1c05c291",
+  "version": "1.2.0-alpha.19.g9aeee229e8",
   "homepage": "https://github.com/microsoft/Agents-for-js",
   "repository": {
     "type": "git",
@@ -20,10 +20,10 @@
   "types": "dist/src/index.d.ts",
   "dependencies": {
     "@azure/core-auth": "^1.10.1",
-    "@azure/msal-node": "^3.8.2",
-    "@microsoft/agents-activity": "1.1.4-geb1c05c291",
+    "@azure/msal-node": "^3.8.4",
+    "@microsoft/agents-activity": "1.2.0-alpha.19.g9aeee229e8",
     "axios": "^1.13.2",
-    "jsonwebtoken": "^9.0.2",
+    "jsonwebtoken": "^9.0.3",
     "jwks-rsa": "^3.2.0",
     "object-path": "^0.11.8"
   },

package/src/activityHandler.ts

@@ -4,7 +4,8 @@
  */
 import { debug } from '@microsoft/agents-activity/logger'
 import { TurnContext } from './turnContext'
-import { Activity, ActivityTypes, Channels } from '@microsoft/agents-activity'
+import { Activity, ActivityTypes, Channels, ExceptionHelper } from '@microsoft/agents-activity'
+import { Errors } from './errorHelper'
 import { StatusCodes } from './statusCodes'
 import { InvokeResponse } from './invoke/invokeResponse'
 import { InvokeException } from './invoke/invokeException'
@@ -254,11 +255,13 @@ export class ActivityHandler {
    * @throws Error if context is missing, activity is missing, or activity type is missing
    */
   async run (context: TurnContext): Promise<void> {
-    if (!context) throw new Error('Missing TurnContext parameter')
-    if (!context.activity) throw new Error('TurnContext does not include an activity')
-    if (!context.activity.type) throw new Error('Activity is missing its type')
+    if (!context) throw ExceptionHelper.generateException(Error, Errors.MissingTurnContext)
+    if (!context.activity) throw ExceptionHelper.generateException(Error, Errors.TurnContextMissingActivity)
+    if (!context.activity.type) throw ExceptionHelper.generateException(Error, Errors.ActivityMissingType)
 
-    await this.onTurnActivity(context)
+    await this.handle(context, 'Turn', async () => {
+      await this.onTurnActivity(context)
+    })
   }
 
   /**

package/src/app/index.ts

@@ -18,3 +18,4 @@ export * from './extensions'
 export * from './adaptiveCards'
 export * from './streaming/streamingResponse'
 export * from './streaming/citation'
+export * from './teamsAttachmentDownloader'

package/src/app/streaming/streamingResponse.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { Activity, addAIToActivity, Attachment, Entity, ClientCitation, SensitivityUsageInfo } from '@microsoft/agents-activity'
+import { Activity, addAIToActivity, Attachment, Entity, ClientCitation, SensitivityUsageInfo, DeliveryModes, Channels } from '@microsoft/agents-activity'
 import { TurnContext } from '../../turnContext'
 import { Citation } from './citation'
 import { CitationUtil } from './citationUtil'
@@ -11,6 +11,28 @@ import { debug } from '@microsoft/agents-activity/logger'
 
 const logger = debug('agents:streamingResponse')
 
+/**
+ * Results for streaming response operations.
+ */
+export enum StreamingResponseResult {
+  /**
+   * The operation was successful.
+   */
+  Success = 'success',
+  /**
+   * The stream has already ended.
+   */
+  AlreadyEnded = 'alreadyEnded',
+  /**
+   * The user canceled the streaming response.
+   */
+  UserCanceled = 'userCanceled',
+  /**
+   * An error occurred during the streaming response.
+   */
+  Error = 'error'
+}
+
 /**
  * A helper class for streaming responses to the client.
  *
@@ -29,7 +51,11 @@ export class StreamingResponse {
   private _message: string = ''
   private _attachments?: Attachment[]
   private _ended = false
-  private _delayInMs = 1500
+  private _delayInMs = 250
+  private _isStreamingChannel: boolean = true
+  private _finalMessage?: Activity
+  private _canceled = false
+  private _userCanceled = false
 
   // Queue for outgoing activities
   private _queue: Array<() => Activity> = []
@@ -44,23 +70,24 @@ export class StreamingResponse {
   private _sensitivityLabel?: SensitivityUsageInfo
 
   /**
-     * Creates a new StreamingResponse instance.
-     *
-     * @param {TurnContext} context - Context for the current turn of conversation with the user.
-     * @returns {TurnContext} - The context for the current turn of conversation with the user.
-     */
+   * Creates a new StreamingResponse instance.
+   *
+   * @param {TurnContext} context - Context for the current turn of conversation with the user.
+   * @returns {TurnContext} - The context for the current turn of conversation with the user.
+   */
   public constructor (context: TurnContext) {
     this._context = context
+    this.loadDefaults(context.activity)
   }
 
   /**
-     * Gets the stream ID of the current response.
-     *
-     * @returns {string | undefined} - The stream ID of the current response.
-     *
-     * @remarks
-     * Assigned after the initial update is sent.
-     */
+   * Gets the stream ID of the current response.
+   *
+   * @returns {string | undefined} - The stream ID of the current response.
+   *
+   * @remarks
+   * Assigned after the initial update is sent.
+   */
   public get streamId (): string | undefined {
     return this._streamId
   }
@@ -73,27 +100,42 @@ export class StreamingResponse {
   }
 
   /**
-     * Gets the number of updates sent for the stream.
-     *
-     * @returns {number} - The number of updates sent for the stream.
-     */
+   * Gets the number of updates sent for the stream.
+   *
+   * @returns {number} - The number of updates sent for the stream.
+   */
   public get updatesSent (): number {
     return this._nextSequence - 1
   }
 
   /**
    * Gets the delay in milliseconds between chunks.
+   * @remarks
+   * Teams default: 1000 ms
+   * Web Chat / Direct Line default: 500 ms
+   * Other channels: 250 ms
    */
   public get delayInMs (): number {
     return this._delayInMs
   }
 
   /**
-     * Queues an informative update to be sent to the client.
-     *
-     * @param {string} text Text of the update to send.
-     */
+   * Gets whether the channel supports streaming.
+   */
+  public get isStreamingChannel (): boolean {
+    return this._isStreamingChannel
+  }
+
+  /**
+   * Queues an informative update to be sent to the client.
+   *
+   * @param {string} text Text of the update to send.
+   */
   public queueInformativeUpdate (text: string): void {
+    if (!this.isStreamingChannel || !text.trim() || this._canceled) {
+      return
+    }
+
     if (this._ended) {
       throw new Error('The stream has already ended.')
     }
@@ -111,17 +153,21 @@ export class StreamingResponse {
   }
 
   /**
-     * Queues a chunk of partial message text to be sent to the client
-     *
-     * @param {string} text Partial text of the message to send.
-     * @param {Citation[]} citations Citations to be included in the message.
-     *
-     * @remarks
-     * The text we be sent as quickly as possible to the client. Chunks may be combined before
-     * delivery to the client.
-     *
-     */
+   * Queues a chunk of partial message text to be sent to the client
+   *
+   * @param {string} text Partial text of the message to send.
+   * @param {Citation[]} citations Citations to be included in the message.
+   *
+   * @remarks
+   * The text we be sent as quickly as possible to the client. Chunks may be combined before
+   * delivery to the client.
+   *
+   */
   public queueTextChunk (text: string, citations?: Citation[]): void {
+    if (!text.trim() || this._canceled) {
+      return
+    }
+
     if (this._ended) {
       throw new Error('The stream has already ended.')
     }
@@ -132,51 +178,92 @@ export class StreamingResponse {
     // If there are citations, modify the content so that the sources are numbers instead of [doc1], [doc2], etc.
     this._message = CitationUtil.formatCitationsResponse(this._message)
 
+    if (!this.isStreamingChannel) {
+      return
+    }
+
     // Queue the next chunk
     this.queueNextChunk()
   }
 
   /**
-     * Ends the stream by sending the final message to the client.
-     *
-     * @returns {Promise<void>} - A promise representing the async operation
-     */
-  public endStream (): Promise<void> {
+   * Ends the stream by sending the final message to the client.
+   *
+   * @returns {Promise<StreamingResponseResult>} - StreamingResponseResult with the result of the streaming response.
+   */
+  public async endStream (): Promise<StreamingResponseResult> {
     if (this._ended) {
-      throw new Error('The stream has already ended.')
+      return StreamingResponseResult.AlreadyEnded
+    }
+
+    if (this._canceled) {
+      return this._userCanceled ? StreamingResponseResult.UserCanceled : StreamingResponseResult.Error
     }
 
     // Queue final message
     this._ended = true
+
+    if (!this.isStreamingChannel) {
+      await this.sendActivity(this.createFinalMessage())
+      return StreamingResponseResult.Success
+    }
+
+    // Queue final message
     this.queueNextChunk()
 
     // Wait for the queue to drain
-    return this.waitForQueue()
+    await this.waitForQueue()
+    return StreamingResponseResult.Success
   }
 
   /**
-     * Sets the attachments to attach to the final chunk.
-     *
-     * @param attachments List of attachments.
-     */
+   * Resets the streaming response to its initial state.
+   * If the stream is still running, this will wait for completion.
+   */
+  public async reset () : Promise<void> {
+    await this.waitForQueue()
+
+    this._queueSync = undefined
+    this._queue = []
+    this._chunkQueued = false
+    this._ended = false
+    this._canceled = false
+    this._userCanceled = false
+    this._message = ''
+    this._nextSequence = 1
+    this._streamId = undefined
+  }
+
+  /**
+   * Set Activity that will be (optionally) used for the final streaming message.
+   */
+  public setFinalMessage (activity: Activity): void {
+    this._finalMessage = activity
+  }
+
+  /**
+   * Sets the attachments to attach to the final chunk.
+   *
+   * @param attachments List of attachments.
+   */
   public setAttachments (attachments: Attachment[]): void {
     this._attachments = attachments
   }
 
   /**
-     * Sets the sensitivity label to attach to the final chunk.
-     *
-     * @param sensitivityLabel The sensitivty label.
-     */
+   * Sets the sensitivity label to attach to the final chunk.
+   *
+   * @param sensitivityLabel The sensitivty label.
+   */
   public setSensitivityLabel (sensitivityLabel: SensitivityUsageInfo): void {
     this._sensitivityLabel = sensitivityLabel
   }
 
   /**
-     * Sets the citations for the full message.
-     *
-     * @param {Citation[]} citations Citations to be included in the message.
-     */
+   * Sets the citations for the full message.
+   *
+   * @param {Citation[]} citations Citations to be included in the message.
+   */
   public setCitations (citations: Citation[]): void {
     if (citations.length > 0) {
       if (!this._citations) {
@@ -202,31 +289,31 @@ export class StreamingResponse {
   }
 
   /**
-     * Sets the Feedback Loop in Teams that allows a user to
-     * give thumbs up or down to a response.
-     * Default is `false`.
-     *
-     * @param enableFeedbackLoop If true, the feedback loop is enabled.
-     */
+   * Sets the Feedback Loop in Teams that allows a user to
+   * give thumbs up or down to a response.
+   * Default is `false`.
+   *
+   * @param enableFeedbackLoop If true, the feedback loop is enabled.
+   */
   public setFeedbackLoop (enableFeedbackLoop: boolean): void {
     this._enableFeedbackLoop = enableFeedbackLoop
   }
 
   /**
-     * Sets the type of UI to use for the feedback loop.
-     *
-     * @param feedbackLoopType The type of the feedback loop.
-     */
+   * Sets the type of UI to use for the feedback loop.
+   *
+   * @param feedbackLoopType The type of the feedback loop.
+   */
   public setFeedbackLoopType (feedbackLoopType: 'default' | 'custom'): void {
     this._feedbackLoopType = feedbackLoopType
   }
 
   /**
-     * Sets the the Generated by AI label in Teams
-     * Default is `false`.
-     *
-     * @param enableGeneratedByAILabel If true, the label is added.
-     */
+   * Sets the the Generated by AI label in Teams
+   * Default is `false`.
+   *
+   * @param enableGeneratedByAILabel If true, the label is added.
+   */
   public setGeneratedByAILabel (enableGeneratedByAILabel: boolean): void {
     this._enableGeneratedByAILabel = enableGeneratedByAILabel
   }
@@ -240,28 +327,28 @@ export class StreamingResponse {
   }
 
   /**
-     * Returns the most recently streamed message.
-     *
-     * @returns The streamed message.
-     */
+   * Returns the most recently streamed message.
+   *
+   * @returns The streamed message.
+   */
   public getMessage (): string {
     return this._message
   }
 
   /**
-     * Waits for the outgoing activity queue to be empty.
-     *
-     * @returns {Promise<void>} - A promise representing the async operation.
-     */
+   * Waits for the outgoing activity queue to be empty.
+   *
+   * @returns {Promise<void>} - A promise representing the async operation.
+   */
   private waitForQueue (): Promise<void> {
     return this._queueSync || Promise.resolve()
   }
 
   /**
-     * Queues the next chunk of text to be sent to the client.
-     *
-     * @private
-     */
+   * Queues the next chunk of text to be sent to the client.
+   *
+   * @private
+   */
   private queueNextChunk (): void {
     // Are we already waiting to send a chunk?
     if (this._chunkQueued) {
@@ -274,16 +361,7 @@ export class StreamingResponse {
       this._chunkQueued = false
       if (this._ended) {
         // Send final message
-        return Activity.fromObject({
-          type: 'message',
-          text: this._message || 'end of stream response',
-          attachments: this._attachments,
-          entities: [{
-            type: 'streaminfo',
-            streamType: 'final',
-            streamSequence: this._nextSequence++
-          }]
-        })
+        return this.createFinalMessage()
       } else {
         // Send typing activity
         return Activity.fromObject({
@@ -300,8 +378,8 @@ export class StreamingResponse {
   }
 
   /**
-     * Queues an activity to be sent to the client.
-     */
+   * Queues an activity to be sent to the client.
+   */
   private queueActivity (factory: () => Activity): void {
     this._queue.push(factory)
 
@@ -315,11 +393,11 @@ export class StreamingResponse {
   }
 
   /**
-     * Sends any queued activities to the client until the queue is empty.
-     *
-     * @returns {Promise<void>} - A promise that will be resolved once the queue is empty.
-     * @private
-     */
+   * Sends any queued activities to the client until the queue is empty.
+   *
+   * @returns {Promise<void>} - A promise that will be resolved once the queue is empty.
+   * @private
+   */
   private async drainQueue (): Promise<void> {
     // eslint-disable-next-line no-async-promise-executor
     return new Promise<void>(async (resolve, reject) => {
@@ -341,12 +419,38 @@ export class StreamingResponse {
   }
 
   /**
-     * Sends an activity to the client and saves the stream ID returned.
-     *
-     * @param {Activity} activity - The activity to send.
-     * @returns {Promise<void>} - A promise representing the async operation.
-     * @private
-     */
+   * Creates the final message to be sent at the end of the stream.
+   */
+  private createFinalMessage (): Activity {
+    const activity = this._finalMessage ?? new Activity('message')
+    activity.type = 'message'
+
+    if (!this._finalMessage) {
+      activity.text = this._message || 'end of stream response'
+    }
+
+    activity.entities ??= []
+    activity.attachments = this._attachments
+    this._nextSequence++ // Increment sequence for final message, even if not streaming.
+
+    if (this.isStreamingChannel) {
+      activity.entities.push({
+        type: 'streaminfo',
+        streamType: 'final',
+        streamSequence: this._nextSequence
+      })
+    }
+
+    return activity
+  }
+
+  /**
+   * Sends an activity to the client and saves the stream ID returned.
+   *
+   * @param {Activity} activity - The activity to send.
+   * @returns {Promise<void>} - A promise representing the async operation.
+   * @private
+   */
   private async sendActivity (activity: Activity): Promise<void> {
     // Set activity ID to the assigned stream ID
     if (this._streamId) {
@@ -385,13 +489,54 @@ export class StreamingResponse {
       }
     }
 
-    // Send activity
-    const response = await this._context.sendActivity(activity)
-    await new Promise((resolve) => setTimeout(resolve, this.delayInMs))
+    try {
+      const response = await this._context.sendActivity(activity)
+      if (!this._streamId) {
+        this._streamId = response?.id
+      }
+      await new Promise((resolve) => setTimeout(resolve, this.delayInMs))
+    } catch (error) {
+      const { message } = error as Error
+      this._canceled = true
+      this._queueSync = undefined
+      this._queue = []
+
+      // MS Teams code list: https://learn.microsoft.com/en-us/microsoftteams/platform/bots/streaming-ux?tabs=jsts#error-codes
+      if (message.includes('ContentStreamNotAllowed')) {
+        logger.warn('Streaming content is not allowed by the client side.', { originalError: message })
+        this._userCanceled = true
+      } else if (message.includes('BadArgument') && message.toLowerCase().includes('streaming api is not enabled')) {
+        logger.warn('Interaction does not support streaming. Defaulting to non-streaming response.', { originalError: message })
+        this._canceled = false
+        this._isStreamingChannel = false
+      }
+    }
+  }
 
-    // Save assigned stream ID
-    if (!this._streamId) {
-      this._streamId = response?.id
+  /**
+   * Loads default values for the streaming response.
+   */
+  private loadDefaults (activity: Activity) {
+    if (!activity) {
+      return
+    }
+
+    if (activity.deliveryMode === DeliveryModes.ExpectReplies) {
+      this._isStreamingChannel = false
+    } else if (Channels.Msteams === activity.channelId) {
+      if (activity.isAgenticRequest()) {
+        // Agentic requests do not support streaming responses at this time.
+        // TODO: Enable streaming for agentic requests when supported.
+        this._isStreamingChannel = false
+      } else {
+        this._isStreamingChannel = true
+        this._delayInMs = 1000
+      }
+    } else if (Channels.Webchat === activity.channelId || Channels.Directline === activity.channelId) {
+      this._isStreamingChannel = true
+      this._delayInMs = 500
+    } else {
+      this._isStreamingChannel = false
     }
   }
 }

package/src/app/teamsAttachmentDownloader.ts

@@ -0,0 +1,110 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { Attachment, Channels } from '@microsoft/agents-activity'
+import { debug } from '@microsoft/agents-activity/logger'
+import { ConnectorClient } from '../connector-client'
+import { InputFile, InputFileDownloader } from './inputFileDownloader'
+import { TurnContext } from '../turnContext'
+import { TurnState } from './turnState'
+import axios, { AxiosInstance } from 'axios'
+import { z } from 'zod'
+
+const logger = debug('agents:teamsAttachmentDownloader')
+
+/**
+ * Downloads attachments from Teams using the bots access token.
+ */
+export class TeamsAttachmentDownloader<TState extends TurnState = TurnState> implements InputFileDownloader<TState> {
+  private _httpClient: AxiosInstance
+  private _stateKey: string
+
+  public constructor (stateKey: string = 'inputFiles') {
+    this._httpClient = axios.create()
+    this._stateKey = stateKey
+  }
+
+  /**
+   * Download any files relative to the current user's input.
+   *
+   * @param {TurnContext} context Context for the current turn of conversation.
+   * @returns {Promise<InputFile[]>} Promise that resolves to an array of downloaded input files.
+   */
+  public async downloadFiles (context: TurnContext): Promise<InputFile[]> {
+    if (context.activity.channelId !== Channels.Msteams && context.activity.channelId !== Channels.M365Copilot) {
+      return Promise.resolve([])
+    }
+    // Filter out HTML attachments
+    const attachments = context.activity.attachments?.filter((a) => a.contentType && !a.contentType.startsWith('text/html'))
+    if (!attachments || attachments.length === 0) {
+      return Promise.resolve([])
+    }
+
+    const connectorClient : ConnectorClient = context.turnState.get<ConnectorClient>(context.adapter.ConnectorClientKey)
+    this._httpClient.defaults.headers = connectorClient.axiosInstance.defaults.headers
+
+    const files: InputFile[] = []
+    for (const attachment of attachments) {
+      const file = await this.downloadFile(attachment)
+      if (file) {
+        files.push(file)
+      }
+    }
+
+    return files
+  }
+
+  /**
+   * @private
+   * @param {Attachment} attachment - Attachment to download.
+   * @returns {Promise<InputFile>} - Promise that resolves to the downloaded input file.
+   */
+  private async downloadFile (attachment: Attachment): Promise<InputFile | undefined> {
+    let inputFile: InputFile | undefined
+
+    if (attachment.contentUrl && attachment.contentUrl.startsWith('https://')) {
+      try {
+        const contentSchema = z.object({ downloadUrl: z.string().url() })
+        const parsed = contentSchema.safeParse(attachment.content)
+        const downloadUrl = parsed.success ? parsed.data.downloadUrl : attachment.contentUrl
+        const response = await this._httpClient.get(downloadUrl, { responseType: 'arraybuffer' })
+
+        const content = Buffer.from(response.data, 'binary')
+        const contentType = response.headers['content-type'] || 'application/octet-stream'
+        inputFile = { content, contentType, contentUrl: attachment.contentUrl }
+      } catch (error) {
+        logger.error(`Failed to download Teams attachment: ${error}`)
+        return undefined
+      }
+    } else {
+      if (!attachment.content) {
+        logger.error('Attachment missing content')
+        return undefined
+      }
+      if (!(attachment.content instanceof ArrayBuffer) && !Buffer.isBuffer(attachment.content)) {
+        logger.error('Attachment content is not ArrayBuffer or Buffer')
+        return undefined
+      }
+      inputFile = {
+        content: Buffer.from(attachment.content as ArrayBuffer),
+        contentType: attachment.contentType,
+        contentUrl: attachment.contentUrl
+      }
+    }
+    return inputFile
+  }
+
+  /**
+   * Downloads files from the attachments in the current turn context and stores them in state.
+   *
+   * @param context The turn context containing the activity with attachments.
+   * @param state The turn state to store the files in.
+   * @returns A promise that resolves when the downloaded files are stored.
+   */
+  public async downloadAndStoreFiles (context: TurnContext, state: TState): Promise<void> {
+    const files = await this.downloadFiles(context)
+    state.setValue(this._stateKey, files)
+  }
+}