@microsoft/agents-hosting 0.5.1-g2e246ff274 → 0.5.12-g2d752e9b13

package/src/app/streaming/citationUtil.ts

@@ -0,0 +1,76 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ClientCitation } from '@microsoft/agents-activity/src/entity/AIEntity'
+
+// import { stringify } from 'yaml'
+
+// import { Tokenizer } from './tokenizers'
+
+/**
+ * Utility functions for manipulating text and citations.
+ */
+export class CitationUtil {
+  /**
+     *
+     * Clips the text to a maximum length in case it exceeds the limit.
+     * @param {string} text The text to clip.
+     * @param {number} maxLength The maximum length of the text to return, cutting off the last whole word.
+     * @returns {string} The modified text
+     */
+  public static snippet (text: string, maxLength: number): string {
+    if (text.length <= maxLength) {
+      return text
+    }
+    let snippet = text.slice(0, maxLength)
+    snippet = snippet.slice(0, Math.min(snippet.length, snippet.lastIndexOf(' ')))
+    snippet += '...'
+    return snippet
+  }
+
+  /**
+     * Convert citation tags `[doc(s)n]` to `[n]` where n is a number.
+     * @param {string} text The text to format.
+     * @returns {string} The formatted text.
+     */
+  public static formatCitationsResponse (text: string): string {
+    return text.replace(/\[docs?(\d+)\]/gi, '[$1]')
+  }
+
+  /**
+     * Get the citations used in the text. This will remove any citations that are included in the citations array from the response but not referenced in the text.
+     * @param {string} text - The text to search for citation references, i.e. [1], [2], etc.
+     * @param {ClientCitation[]} citations - The list of citations to search for.
+     * @returns {ClientCitation[] | undefined} The list of citations used in the text.
+     */
+  public static getUsedCitations (text: string, citations: ClientCitation[]): ClientCitation[] | undefined {
+    const regex = /\[(\d+)\]/gi
+    const matches = text.match(regex)
+
+    if (!matches) {
+      return undefined
+    }
+
+    // Remove duplicates
+    const filteredMatches = new Set()
+    matches.forEach((match) => {
+      if (filteredMatches.has(match)) {
+        return
+      }
+
+      filteredMatches.add(match)
+    })
+
+    // Add citations
+    const usedCitations: ClientCitation[] = []
+    filteredMatches.forEach((match) => {
+      const found = citations.find((citation) => `[${citation.position}]` === match)
+      if (found) {
+        usedCitations.push(found)
+      }
+    })
+    return usedCitations
+  }
+}

package/src/app/streaming/streamingResponse.ts

@@ -0,0 +1,407 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { Activity, addAIToActivity, Attachment, Entity } from '@microsoft/agents-activity'
+import { TurnContext } from '../../turnContext'
+import { Citation } from './citation'
+import { CitationUtil } from './citationUtil'
+import { debug } from '../../logger'
+import { ClientCitation, SensitivityUsageInfo } from '@microsoft/agents-activity/src/entity/AIEntity'
+
+const logger = debug('agents:streamingResponse')
+
+/**
+ * A helper class for streaming responses to the client.
+ * @remarks
+ * This class is used to send a series of updates to the client in a single response. The expected
+ * sequence of calls is:
+ *
+ * `queueInformativeUpdate()`, `queueTextChunk()`, `queueTextChunk()`, ..., `endStream()`.
+ *
+ * Once `endStream()` is called, the stream is considered ended and no further updates can be sent.
+ */
+export class StreamingResponse {
+  private readonly _context: TurnContext
+  private _nextSequence: number = 1
+  private _streamId?: string
+  private _message: string = ''
+  private _attachments?: Attachment[]
+  private _ended = false
+
+  // Queue for outgoing activities
+  private _queue: Array<() => Activity> = []
+  private _queueSync: Promise<void> | undefined
+  private _chunkQueued = false
+
+  // Powered by AI feature flags
+  private _enableFeedbackLoop = false
+  private _feedbackLoopType?: 'default' | 'custom'
+  private _enableGeneratedByAILabel = false
+  private _citations?: ClientCitation[] = []
+  private _sensitivityLabel?: SensitivityUsageInfo
+
+  /**
+     * Creates a new StreamingResponse instance.
+     * @param {TurnContext} context - Context for the current turn of conversation with the user.
+     * @returns {TurnContext} - The context for the current turn of conversation with the user.
+     */
+  public constructor (context: TurnContext) {
+    this._context = context
+  }
+
+  /**
+     * Gets the stream ID of the current response.
+     * @returns {string | undefined} - The stream ID of the current response.
+     * @remarks
+     * Assigned after the initial update is sent.
+     */
+  public get streamId (): string | undefined {
+    return this._streamId
+  }
+
+  /**
+     * Gets the citations of the current response.
+     */
+  public get citations (): ClientCitation[] | undefined {
+    return this._citations
+  }
+
+  /**
+     * Gets the number of updates sent for the stream.
+     * @returns {number} - The number of updates sent for the stream.
+     */
+  public get updatesSent (): number {
+    return this._nextSequence - 1
+  }
+
+  /**
+     * Queues an informative update to be sent to the client.
+     * @param {string} text Text of the update to send.
+     */
+  public queueInformativeUpdate (text: string): void {
+    if (this._ended) {
+      throw new Error('The stream has already ended.')
+    }
+
+    // Queue a typing activity
+    this.queueActivity(() => Activity.fromObject({
+      type: 'typing',
+      text,
+      channelData: {
+        streamType: 'informative',
+        streamSequence: this._nextSequence++
+      } as StreamingChannelData
+    }))
+  }
+
+  /**
+     * Queues a chunk of partial message text to be sent to the client
+     * @remarks
+     * The text we be sent as quickly as possible to the client. Chunks may be combined before
+     * delivery to the client.
+     * @param {string} text Partial text of the message to send.
+     * @param {Citation[]} citations Citations to be included in the message.
+     */
+  public queueTextChunk (text: string, citations?: Citation[]): void {
+    if (this._ended) {
+      throw new Error('The stream has already ended.')
+    }
+
+    // Update full message text
+    this._message += text
+
+    // If there are citations, modify the content so that the sources are numbers instead of [doc1], [doc2], etc.
+    this._message = CitationUtil.formatCitationsResponse(this._message)
+
+    // Queue the next chunk
+    this.queueNextChunk()
+  }
+
+  /**
+     * Ends the stream by sending the final message to the client.
+     * @returns {Promise<void>} - A promise representing the async operation
+     */
+  public endStream (): Promise<void> {
+    if (this._ended) {
+      throw new Error('The stream has already ended.')
+    }
+
+    // Queue final message
+    this._ended = true
+    this.queueNextChunk()
+
+    // Wait for the queue to drain
+    return this.waitForQueue()
+  }
+
+  /**
+     * Sets the attachments to attach to the final chunk.
+     * @param attachments List of attachments.
+     */
+  public setAttachments (attachments: Attachment[]): void {
+    this._attachments = attachments
+  }
+
+  /**
+     * Sets the sensitivity label to attach to the final chunk.
+     * @param sensitivityLabel The sensitivty label.
+     */
+  public setSensitivityLabel (sensitivityLabel: SensitivityUsageInfo): void {
+    this._sensitivityLabel = sensitivityLabel
+  }
+
+  /**
+     * Sets the citations for the full message.
+     * @param {Citation[]} citations Citations to be included in the message.
+     */
+  public setCitations (citations: Citation[]): void {
+    if (citations.length > 0) {
+      if (!this._citations) {
+        this._citations = []
+      }
+      let currPos = this._citations.length
+
+      for (const citation of citations) {
+        const clientCitation: ClientCitation = {
+          '@type': 'Claim',
+          position: currPos + 1,
+          appearance: {
+            '@type': 'DigitalDocument',
+            name: citation.title || `Document #${currPos + 1}`,
+            abstract: CitationUtil.snippet(citation.content, 477)
+          }
+        }
+        currPos++
+        this._citations.push(clientCitation)
+      }
+    }
+  }
+
+  /**
+     * Sets the Feedback Loop in Teams that allows a user to
+     * give thumbs up or down to a response.
+     * Default is `false`.
+     * @param enableFeedbackLoop If true, the feedback loop is enabled.
+     */
+  public setFeedbackLoop (enableFeedbackLoop: boolean): void {
+    this._enableFeedbackLoop = enableFeedbackLoop
+  }
+
+  /**
+     * Sets the type of UI to use for the feedback loop.
+     * @param feedbackLoopType The type of the feedback loop.
+     */
+  public setFeedbackLoopType (feedbackLoopType: 'default' | 'custom'): void {
+    this._feedbackLoopType = feedbackLoopType
+  }
+
+  /**
+     * Sets the the Generated by AI label in Teams
+     * Default is `false`.
+     * @param enableGeneratedByAILabel If true, the label is added.
+     */
+  public setGeneratedByAILabel (enableGeneratedByAILabel: boolean): void {
+    this._enableGeneratedByAILabel = enableGeneratedByAILabel
+  }
+
+  /**
+     * Returns the most recently streamed message.
+     * @returns The streamed message.
+     */
+  public getMessage (): string {
+    return this._message
+  }
+
+  /**
+     * Waits for the outgoing activity queue to be empty.
+     * @returns {Promise<void>} - A promise representing the async operation.
+     */
+  public waitForQueue (): Promise<void> {
+    return this._queueSync || Promise.resolve()
+  }
+
+  /**
+     * Queues the next chunk of text to be sent to the client.
+     * @private
+     */
+  private queueNextChunk (): void {
+    // Are we already waiting to send a chunk?
+    if (this._chunkQueued) {
+      return
+    }
+
+    // Queue a chunk of text to be sent
+    this._chunkQueued = true
+    this.queueActivity(() => {
+      this._chunkQueued = false
+      if (this._ended) {
+        // Send final message
+        return Activity.fromObject({
+          type: 'message',
+          text: this._message || 'end strean response',
+          attachments: this._attachments,
+          channelData: {
+            streamType: 'final',
+            streamSequence: this._nextSequence++
+          } as StreamingChannelData
+        })
+      } else {
+        // Send typing activity
+        return Activity.fromObject({
+          type: 'typing',
+          text: this._message,
+          channelData: {
+            streamType: 'streaming',
+            streamSequence: this._nextSequence++
+          } as StreamingChannelData
+        })
+      }
+    })
+  }
+
+  /**
+     * Queues an activity to be sent to the client.
+     */
+  private queueActivity (factory: () => Activity): void {
+    this._queue.push(factory)
+
+    // If there's no sync in progress, start one
+    if (!this._queueSync) {
+      this._queueSync = this.drainQueue().catch((err) => {
+        logger.error(`Error occured when sending activity while streaming: "${err}".`)
+        // throw err
+      })
+    }
+  }
+
+  /**
+     * Sends any queued activities to the client until the queue is empty.
+     * @returns {Promise<void>} - A promise that will be resolved once the queue is empty.
+     * @private
+     */
+  private async drainQueue (): Promise<void> {
+    // eslint-disable-next-line no-async-promise-executor
+    return new Promise<void>(async (resolve, reject) => {
+      try {
+        logger.debug(`Draining queue with ${this._queue.length} activities.`)
+        while (this._queue.length > 0) {
+          const factory = this._queue.shift()!
+          const activity = factory()
+          await this.sendActivity(activity)
+        }
+
+        resolve()
+      } catch (err) {
+        reject(err)
+      } finally {
+        this._queueSync = undefined
+      }
+    })
+  }
+
+  /**
+     * Sends an activity to the client and saves the stream ID returned.
+     * @param {Activity} activity - The activity to send.
+     * @returns {Promise<void>} - A promise representing the async operation.
+     * @private
+     */
+  private async sendActivity (activity: Activity): Promise<void> {
+    // Set activity ID to the assigned stream ID
+    if (this._streamId) {
+      activity.id = this._streamId
+      activity.channelData = Object.assign({}, activity.channelData, { streamId: this._streamId })
+    }
+
+    activity.entities = [
+      {
+        type: 'streaminfo',
+        ...activity.channelData
+      } as Entity
+    ]
+
+    if (this._citations && this._citations.length > 0 && !this._ended) {
+      // Filter out the citations unused in content.
+      const currCitations = CitationUtil.getUsedCitations(this._message, this._citations) ?? undefined
+      activity.entities.push({
+        type: 'https://schema.org/Message',
+        '@type': 'Message',
+        '@context': 'https://schema.org',
+        '@id': '',
+        citation: currCitations
+      } as unknown as Entity)
+    }
+
+    // Add in Powered by AI feature flags
+    if (this._ended) {
+      if (this._enableFeedbackLoop && this._feedbackLoopType) {
+        activity.channelData = Object.assign({}, activity.channelData, {
+          feedbackLoop: { type: this._feedbackLoopType }
+        })
+      } else {
+        activity.channelData = Object.assign({}, activity.channelData, {
+          feedbackLoopEnabled: this._enableFeedbackLoop
+        })
+      }
+
+      // Add in Generated by AI
+      if (this._enableGeneratedByAILabel) {
+        addAIToActivity(activity, this._citations, this._sensitivityLabel)
+        // activity.entities.push({
+        //   type: 'https://schema.org/Message',
+        //   '@type': 'Message',
+        //   '@context': 'https://schema.org',
+        //   '@id': '',
+        //   additionalType: ['AIGeneratedContent'],
+        //   citation: this._citations && this._citations.length > 0 ? this._citations : [],
+        //   usageInfo: this._sensitivityLabel
+        // } as unknown as Entity)
+      }
+    }
+
+    // Send activity
+    const response = await this._context.sendActivity(activity)
+    await new Promise((resolve) => setTimeout(resolve, 1500))
+
+    // Save assigned stream ID
+    if (!this._streamId) {
+      this._streamId = response?.id
+    }
+  }
+}
+
+/**
+ * @private
+ * Structure of the outgoing channelData field for streaming responses.
+ * @remarks
+ * The expected sequence of streamTypes is:
+ *
+ * `informative`, `streaming`, `streaming`, ..., `final`.
+ *
+ * Once a `final` message is sent, the stream is considered ended.
+ */
+interface StreamingChannelData {
+  /**
+     * The type of message being sent.
+     * @remarks
+     * `informative` - An informative update.
+     * `streaming` - A chunk of partial message text.
+     * `final` - The final message.
+     */
+  streamType: 'informative' | 'streaming' | 'final';
+
+  /**
+     * Sequence number of the message in the stream.
+     * @remarks
+     * Starts at 1 for the first message and increments from there.
+     */
+  streamSequence: number;
+
+  /**
+     * ID of the stream.
+     * @remarks
+     * Assigned after the initial update is sent.
+     */
+  streamId?: string;
+}

package/src/cards/cardFactory.ts

@@ -13,8 +13,7 @@ import { O365ConnectorCard } from './o365ConnectorCard'
 import { ThumbnailCard } from './thumbnailCard'
 import { VideoCard } from './videoCard'
 import { CardImage } from './cardImage'
-import { OAuthCard } from '../oauth/oAuthCard'
-import { SigningResource } from '../oauth/signingResource'
+import { OAuthCard, SignInResource } from '../oauth/userTokenClient.types'
 import { SigninCard } from './signinCard'
 
 /**
@@ -218,7 +217,7 @@ export class CardFactory {
    * @param signingResource The signing resource.
    * @returns The OAuth card attachment.
    */
-  static oauthCard (connectionName: string, title: string, text: string, signingResource: SigningResource) : Attachment {
+  static oauthCard (connectionName: string, title: string, text: string, signingResource: SignInResource) : Attachment {
     const card: Partial<OAuthCard> = {
       buttons: [{
         type: ActionTypes.Signin,

package/src/oauth/index.ts

@@ -1,6 +1,3 @@
-export * from './oAuthCard'
-export * from './signingResource'
 export * from './userTokenClient'
-export * from './tokenExchangeRequest'
 export * from './oAuthFlow'
-export * from './tokenResponse'
+export * from './userTokenClient.types'

package/src/oauth/oAuthFlow.ts

@@ -4,20 +4,18 @@ import { debug } from './../logger'
 import { ActivityTypes, Attachment } from '@microsoft/agents-activity'
 import {
   CardFactory,
-  AgentStatePropertyAccessor,
-  UserState,
   TurnContext,
+  Storage,
   MessageFactory,
-  TokenExchangeRequest,
-  UserTokenClient
 } from '../'
-import { TokenRequestStatus, TokenResponse } from './tokenResponse'
+import { UserTokenClient } from './userTokenClient'
+import { TokenExchangeRequest, TokenResponse } from './userTokenClient.types'
 
 const logger = debug('agents:oauth-flow')
 
-export class FlowState {
-  public flowStarted: boolean = false
-  public flowExpires: number = 0
+export interface FlowState {
+  flowStarted: boolean,
+  flowExpires: number
 }
 
 interface TokenVerifyState {
@@ -35,12 +33,7 @@ export class OAuthFlow {
   /**
    * The current state of the OAuth flow.
    */
-  state: FlowState | null
-
-  /**
-   * The accessor for managing the flow state in user state.
-   */
-  flowStateAccessor: AgentStatePropertyAccessor<FlowState | null>
+  state: FlowState
 
   /**
    * The ID of the token exchange request, used to deduplicate requests.
@@ -66,9 +59,8 @@ export class OAuthFlow {
    * Creates a new instance of OAuthFlow.
    * @param userState The user state.
    */
-  constructor (userState: UserState, absOauthConnectionName: string, tokenClient?: UserTokenClient, cardTitle?: string, cardText?: string) {
-    this.state = new FlowState()
-    this.flowStateAccessor = userState.createProperty('flowState')
+  constructor (private storage: Storage, absOauthConnectionName: string, tokenClient?: UserTokenClient, cardTitle?: string, cardText?: string) {
+    this.state = { flowExpires: 0, flowStarted: false }
     this.absOauthConnectionName = absOauthConnectionName
     this.userTokenClient = tokenClient ?? null!
     this.cardTitle = cardTitle ?? this.cardTitle
@@ -97,7 +89,7 @@ export class OAuthFlow {
    * @param context The turn context.
    * @returns A promise that resolves to the user token.
    */
-  public async beginFlow (context: TurnContext): Promise<TokenResponse> {
+  public async beginFlow (context: TurnContext): Promise<TokenResponse | undefined> {
     this.state = await this.getUserState(context)
     if (this.absOauthConnectionName === '') {
       throw new Error('connectionName is not set')
@@ -105,27 +97,20 @@ export class OAuthFlow {
     logger.info('Starting OAuth flow for connectionName:', this.absOauthConnectionName)
     await this.initializeTokenClient(context)
 
-    const tokenResponse = await this.userTokenClient.getUserToken(this.absOauthConnectionName, context.activity.channelId!, context.activity.from?.id!)
-    if (tokenResponse?.status === TokenRequestStatus.Success) {
+    const act = context.activity
+    const output = await this.userTokenClient.getTokenOrSignInResource(act.from?.id!, this.absOauthConnectionName, act.channelId!, act.getConversationReference(), act.relatesTo!, undefined!)
+    if (output && output.tokenResponse) {
       this.state.flowStarted = false
       this.state.flowExpires = 0
-      await this.flowStateAccessor.set(context, this.state)
-      logger.info('User token retrieved successfully from service')
-      return tokenResponse
+      await this.storage.write({ [this.getFlowStateKey(context)]: this.state })
+      return output.tokenResponse
     }
-
-    const authConfig = context.adapter.authConfig
-    const signingResource = await this.userTokenClient.getSignInResource(authConfig.clientId!, this.absOauthConnectionName, context.activity.getConversationReference(), context.activity.relatesTo)
-    const oCard: Attachment = CardFactory.oauthCard(this.absOauthConnectionName, this.cardTitle, this.cardText, signingResource)
+    const oCard: Attachment = CardFactory.oauthCard(this.absOauthConnectionName, this.cardTitle, this.cardText, output.signInResource)
     await context.sendActivity(MessageFactory.attachment(oCard))
     this.state.flowStarted = true
     this.state.flowExpires = Date.now() + 30000
-    await this.flowStateAccessor.set(context, this.state)
-    logger.info('OAuth begin flow completed, waiting for user to sign in')
-    return {
-      token: undefined,
-      status: TokenRequestStatus.InProgress
-    }
+    await this.storage.write({ [this.getFlowStateKey(context)]: this.state })
+    return undefined
   }
 
   /**
@@ -140,7 +125,7 @@ export class OAuthFlow {
       logger.warn('Flow expired')
       this.state!.flowStarted = false
       await context.sendActivity(MessageFactory.text('Sign-in session expired. Please try again.'))
-      return { status: TokenRequestStatus.Expired, token: undefined }
+      return { token: undefined }
     }
     const contFlowActivity = context.activity
     if (contFlowActivity.type === ActivityTypes.Message) {
@@ -161,22 +146,22 @@ export class OAuthFlow {
       logger.info('Continuing OAuth flow with tokenExchange')
       const tokenExchangeRequest = contFlowActivity.value as TokenExchangeRequest
       if (this.tokenExchangeId === tokenExchangeRequest.id) { // dedupe
-        return { status: TokenRequestStatus.InProgress, token: undefined }
+        return { token: undefined }
       }
       this.tokenExchangeId = tokenExchangeRequest.id!
       const userTokenResp = await this.userTokenClient?.exchangeTokenAsync(contFlowActivity.from?.id!, this.absOauthConnectionName, contFlowActivity.channelId!, tokenExchangeRequest)
-      if (userTokenResp?.status === TokenRequestStatus.Success) {
+      if (userTokenResp && userTokenResp.token) {
         logger.info('Token exchanged')
         this.state!.flowStarted = false
-        await this.flowStateAccessor.set(context, this.state)
+        await this.storage.write({ [this.getFlowStateKey(context)]: this.state })
         return userTokenResp
       } else {
         logger.warn('Token exchange failed')
         this.state!.flowStarted = true
-        return { status: TokenRequestStatus.Failed, token: undefined }
+        return { token: undefined }
       }
     }
-    return { status: TokenRequestStatus.Failed, token: undefined }
+    return { token: undefined }
   }
 
   /**
@@ -189,7 +174,7 @@ export class OAuthFlow {
     await this.initializeTokenClient(context)
     await this.userTokenClient?.signOut(context.activity.from?.id as string, this.absOauthConnectionName, context.activity.channelId as string)
     this.state!.flowExpires = 0
-    await this.flowStateAccessor.set(context, this.state)
+    this.storage.write({ [this.getFlowStateKey(context)]: this.state })
     logger.info('User signed out successfully')
   }
 
@@ -199,10 +184,9 @@ export class OAuthFlow {
    * @returns A promise that resolves to the user state.
    */
   private async getUserState (context: TurnContext) {
-    let userProfile: FlowState | null = await this.flowStateAccessor.get(context, null)
-    if (userProfile === null) {
-      userProfile = new FlowState()
-    }
+    const key = this.getFlowStateKey(context)
+    const data = await this.storage.read([key])
+    const userProfile: FlowState = data[key] ?? { flowStarted: false, flowExpires: 0 }
     return userProfile
   }
 
@@ -210,7 +194,17 @@ export class OAuthFlow {
     if (this.userTokenClient === undefined || this.userTokenClient === null) {
       const scope = 'https://api.botframework.com'
       const accessToken = await context.adapter.authProvider.getAccessToken(context.adapter.authConfig, scope)
-      this.userTokenClient = new UserTokenClient(accessToken)
+      this.userTokenClient = new UserTokenClient(accessToken, context.adapter.authConfig.clientId!)
+    }
+  }
+
+  private getFlowStateKey (context: TurnContext): string {
+    const channelId = context.activity.channelId
+    const conversationId = context.activity.conversation?.id
+    const userId = context.activity.from?.id
+    if (!channelId || !conversationId || !userId) {
+      throw new Error('ChannelId, conversationId, and userId must be set in the activity')
     }
+    return `oauth/${channelId}/${conversationId}/${userId}/flowState`
   }
 }