@ray-js/t-agent-plugin-aistream 0.2.0-beta-8 → 0.2.0-beta-10

package/dist/utils/AIStream.js

@@ -445,6 +445,9 @@ export class AIStreamEvent {
     });
   }
   async abort(options) {
+    if (this.closed) {
+      return;
+    }
     if (this.sessionId) {
       await sendEventChatBreak({
         eventId: this.eventId,

package/dist/utils/defaultMock.js

@@ -112,35 +112,7 @@ mock.hooks.hook('createSession', context => {
     sessionId: generateId(),
     sendDataChannels: ['audio', 'video', 'text', 'image'],
     revDataChannels: ['text', 'audio'],
-    currentEvent: null,
-    replyText: function (streamFlag) {
-      let data = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : null;
-      if (session.closed) {
-        return;
-      }
-      if (!session.currentEvent) {
-        throw new Error('replyText event not exists');
-      }
-      dispatch('onTextReceived', {
-        dataChannel: 'text',
-        sessionIdList: [session.sessionId],
-        streamFlag,
-        text: data ? JSON.stringify(data) : ''
-      });
-    },
-    replyEvent: eventType => {
-      if (session.closed) {
-        return;
-      }
-      if (!session.currentEvent) {
-        throw new Error('replyEvent event not exists');
-      }
-      dispatch('onEventReceived', {
-        eventId: session.currentEvent.eventId,
-        sessionId: session.sessionId,
-        eventType
-      });
-    }
+    currentEvent: null
   };
   map.set(session.sessionId, session);
   context.result = {
@@ -193,17 +165,47 @@ mock.hooks.hook('sendEventStart', context => {
     throw new Error('sendEventStart already in event');
   }
   const eventId = generateId();
-  session.currentEvent = {
-    asrStatus: {
-      controller: new AbortController(),
+  const event = {
+    replyText: function (streamFlag) {
+      let data = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : null;
+      if (session.closed || event.controller.signal.aborted) {
+        return;
+      }
+      if (!session.currentEvent || session.currentEvent.eventId !== eventId) {
+        throw new Error('replyEvent event not exists or not in current event');
+      }
+      dispatch('onTextReceived', {
+        dataChannel: 'text',
+        sessionIdList: [session.sessionId],
+        streamFlag,
+        text: data ? JSON.stringify(data) : ''
+      });
+    },
+    replyEvent: eventType => {
+      if (session.closed || event.controller.signal.aborted) {
+        return;
+      }
+      if (!session.currentEvent || session.currentEvent.eventId !== eventId) {
+        throw new Error('replyEvent event not exists or not in current event');
+      }
+      dispatch('onEventReceived', {
+        eventId,
+        sessionId: session.sessionId,
+        eventType
+      });
+    },
+    asr: {
+      finishController: new AbortController(),
       promise: Promise.resolve()
     },
     eventId,
     data: [],
     controller: new AbortController()
   };
+  session.currentEvent = event;
   context.result = {
-    eventId
+    eventId,
+    sessionId
   };
 });
 mock.hooks.hook('registerRecordAmplitudes', context => {
@@ -218,9 +220,12 @@ mock.hooks.hook('sendEventEnd', context => {
   const session = getSession(context.options.sessionId, context.options.eventId);
   context.result = true;
   const event = session.currentEvent;
+  if (event.eventId !== context.options.eventId) {
+    throw new Error('sendEventEnd eventId mismatch');
+  }
   (async () => {
     var _ctx$responseSkills;
-    await event.asrStatus.promise;
+    await event.asr.promise;
     const ctx = {
       data: event.data,
       responseText: ''
@@ -238,9 +243,13 @@ mock.hooks.hook('sendEventEnd', context => {
       return;
     }
     if (ctx.data.length === 0) {
-      session.replyEvent(EventType.EVENT_START);
+      event.replyEvent(EventType.EVENT_START);
       await mock.sleep(100);
-      session.replyEvent(EventType.EVENT_END);
+      if (event.controller.signal.aborted) {
+        return;
+      }
+      event.replyEvent(EventType.EVENT_END);
+      console.log('No data to send, ending event, session.currentEvent = null');
       session.currentEvent = null;
       return;
     }
@@ -249,19 +258,19 @@ mock.hooks.hook('sendEventEnd', context => {
     if (event.controller.signal.aborted) {
       return;
     }
-    session.replyEvent(EventType.EVENT_START);
+    event.replyEvent(EventType.EVENT_START);
     await mock.sleep(500);
     if (event.controller.signal.aborted) {
       return;
     }
     const bizId = generateId();
-    session.replyText(StreamFlag.START);
+    event.replyText(StreamFlag.START);
     for (const word of words) {
       await mock.sleep(100);
       if (event.controller.signal.aborted) {
         return;
       }
-      session.replyText(StreamFlag.IN_PROGRESS, {
+      event.replyText(StreamFlag.IN_PROGRESS, {
         bizType: ReceivedTextPacketType.NLG,
         eof: ReceivedTextPacketEof.CONTINUE,
         bizId,
@@ -276,7 +285,7 @@ mock.hooks.hook('sendEventEnd', context => {
     if (event.controller.signal.aborted) {
       return;
     }
-    session.replyText(StreamFlag.IN_PROGRESS, {
+    event.replyText(StreamFlag.IN_PROGRESS, {
       bizType: ReceivedTextPacketType.NLG,
       eof: ReceivedTextPacketEof.END,
       bizId,
@@ -296,7 +305,7 @@ mock.hooks.hook('sendEventEnd', context => {
         if (event.controller.signal.aborted) {
           return;
         }
-        session.replyText(StreamFlag.IN_PROGRESS, {
+        event.replyText(StreamFlag.IN_PROGRESS, {
           bizType: ReceivedTextPacketType.SKILL,
           eof: ReceivedTextPacketEof.END,
           bizId,
@@ -304,12 +313,13 @@ mock.hooks.hook('sendEventEnd', context => {
         });
       }
     }
-    session.replyText(StreamFlag.END);
+    event.replyText(StreamFlag.END);
     await mock.sleep(500);
     if (event.controller.signal.aborted) {
       return;
     }
-    session.replyEvent(EventType.EVENT_END);
+    event.replyEvent(EventType.EVENT_END);
+    console.log('finish sendEventEnd session.currentEvent = null');
     session.currentEvent = null;
   })();
 });
@@ -320,6 +330,7 @@ mock.hooks.hook('sendEventPayloadEnd', context => {
 mock.hooks.hook('sendEventChatBreak', context => {
   const session = getSession(context.options.sessionId, context.options.eventId);
   session.currentEvent.controller.abort(new Error('sendEventChatBreak'));
+  console.log('sendEventChatBreak currentEvent = null');
   session.currentEvent = null;
   context.result = true;
 });
@@ -328,11 +339,12 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
   if (!session.currentEvent) {
     throw new Error('startRecordAndSendAudioData event not exists');
   }
+  const event = session.currentEvent;
   context.result = true;
   (async () => {
     const {
-      controller
-    } = session.currentEvent.asrStatus;
+      finishController
+    } = event.asr;
     const count = mock.data.get('recordAmplitudesCount');
     if (count) {
       const id = setInterval(() => {
@@ -340,9 +352,11 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
           amplitudes: new Array(count).fill(0).map(() => Math.random())
         });
       }, 100);
-      controller.signal.addEventListener('abort', () => {
+      finishController.signal.addEventListener('abort', () => {
+        clearInterval(id);
+      });
+      event.controller.signal.addEventListener('abort', () => {
         clearInterval(id);
-        mock.data.delete('recordAmplitudesCountId');
       });
     }
     const result = {
@@ -352,23 +366,23 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
     const responseText = result.responseText || '';
     const bizId = generateId();
     let finishResolve;
-    session.currentEvent.asrStatus.promise = new Promise(resolve => {
+    event.asr.promise = new Promise(resolve => {
       finishResolve = resolve;
     });
     const parts = splitString(responseText);
     let text = '';
-    controller.signal.addEventListener('abort', async () => {
+    finishController.signal.addEventListener('abort', async () => {
       var _finishResolve;
-      if (!session.currentEvent) {
+      if (event.controller.signal.aborted) {
         return;
       }
 
       // 终止识别到出完整结果的延迟
       await mock.sleep(5000);
-      if (!session.currentEvent) {
+      if (event.controller.signal.aborted) {
         return;
       }
-      session.replyText(StreamFlag.IN_PROGRESS, {
+      event.replyText(StreamFlag.IN_PROGRESS, {
         bizType: ReceivedTextPacketType.ASR,
         eof: ReceivedTextPacketEof.END,
         bizId,
@@ -377,10 +391,10 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
         }
       });
       await mock.sleep(100);
-      if (!session.currentEvent) {
+      if (event.controller.signal.aborted) {
         return;
       }
-      session.replyText(StreamFlag.END);
+      event.replyText(StreamFlag.END);
       if (session.currentEvent && text) {
         session.currentEvent.data.push({
           type: 'text',
@@ -392,10 +406,10 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
 
     // 识别 ASR 的延迟
     await mock.sleep(100);
-    if (controller.signal.aborted) {
+    if (finishController.signal.aborted) {
       return;
     }
-    session.replyText(StreamFlag.START, {
+    event.replyText(StreamFlag.START, {
       bizType: ReceivedTextPacketType.ASR,
       eof: ReceivedTextPacketEof.CONTINUE,
       bizId,
@@ -404,15 +418,15 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
       }
     });
     for (const part of parts) {
-      if (controller.signal.aborted) {
+      if (finishController.signal.aborted) {
         break;
       }
       await mock.sleep(300);
-      if (controller.signal.aborted) {
+      if (finishController.signal.aborted) {
         break;
       }
       text += part;
-      session.replyText(StreamFlag.IN_PROGRESS, {
+      event.replyText(StreamFlag.IN_PROGRESS, {
         bizType: ReceivedTextPacketType.ASR,
         eof: ReceivedTextPacketEof.CONTINUE,
         bizId,
@@ -428,7 +442,7 @@ mock.hooks.hook('stopRecordAndSendAudioData', context => {
   if (!session.currentEvent) {
     throw new Error('stopRecordAndSendAudioData event not exists');
   }
-  session.currentEvent.asrStatus.controller.abort(new Error('stopRecordAndSendAudioData'));
+  session.currentEvent.asr.finishController.abort(new Error('stopRecordAndSendAudioData'));
   context.result = true;
 });
 mock.hooks.hook('startRecordAndSendVideoData', context => {

package/dist/utils/sendMessage.d.ts

@@ -6,6 +6,7 @@ export interface SendBlocksToAIStreamParams {
     blocks: InputBlock[];
     session: AIStreamSession;
     attribute?: AIStreamChatAttribute;
+    signal?: AbortSignal;
 }
 export declare class AIStreamSessionError extends Error {
     readonly code: number;

package/dist/utils/sendMessage.js

@@ -23,7 +23,8 @@ export class AIStreamSessionError extends Error {
 export function sendBlocksToAIStream(params) {
   const {
     session,
-    blocks
+    blocks,
+    signal
   } = params;
   let audioEmitter = null;
   let amplitudeCount = 0;
@@ -64,6 +65,11 @@ export function sendBlocksToAIStream(params) {
           value: JSON.stringify(attribute)
         }]
       }));
+      if (signal !== null && signal !== void 0 && signal.aborted) {
+        canceled = true;
+        controller.close();
+        return;
+      }
       if (error) {
         controller.error(error);
         closed = true;
@@ -237,7 +243,10 @@ export function sendBlocksToAIStream(params) {
           });
           audioEmitter.addEventListener('cancel', async () => {
             await s.stop();
-            await event.abort();
+            if (!canceled) {
+              canceled = true;
+              await event.abort();
+            }
             resolve();
           });
           s.start();
@@ -246,9 +255,11 @@ export function sendBlocksToAIStream(params) {
       await event.end();
     },
     async cancel() {
-      var _event;
-      canceled = true;
-      await ((_event = event) === null || _event === void 0 ? void 0 : _event.abort());
+      if (!canceled) {
+        var _event;
+        canceled = true;
+        await ((_event = event) === null || _event === void 0 ? void 0 : _event.abort());
+      }
     }
   });
   return {

package/dist/withAIStream.d.ts

@@ -16,8 +16,6 @@ export interface AIStreamOptions {
     };
     /** 历史消息数量，默认1000，为0的话，则已分页的方式取全部数据 */
     historySize?: number;
-    /** 是否支持多模态 */
-    multiModal?: boolean;
     /** 是否将输入块传递给智能体，默认为 true，设置为 false 时，需要你自己编写 onInputBlocksPush Hook 来处理输入块 */
     wireInput?: boolean;
     /** 索引ID，如果传了 createChatHistoryStore，则会覆盖 */

package/dist/withAIStream.js

@@ -30,7 +30,6 @@ export function withAIStream() {
       onAgentDispose,
       onTileEvent
     } = agent;
-    agent.session.initValue('multiModal', options.multiModal);
     const ui = agent.plugins.ui;
     onAgentStart(async () => {
       /**
@@ -197,6 +196,7 @@ export function withAIStream() {
       const result = sendBlocksToAIStream({
         blocks,
         session: streamSession,
+        signal,
         attribute: _objectSpread({}, extraOptions)
       });
       signal === null || signal === void 0 || signal.addEventListener('abort', event => {
@@ -253,7 +253,9 @@ export function withAIStream() {
               return;
             }
             if (!event.detail.text) {
-              await userMsg.remove();
+              if (userMsg.isShow) {
+                await userMsg.remove();
+              }
               reject(new Error('No text found in audio event'));
               return;
             }
@@ -272,7 +274,9 @@ export function withAIStream() {
           const onCancel = async () => {
             end = true;
             if (!response.started) {
-              await userMsg.remove();
+              if (userMsg.isShow) {
+                await userMsg.remove();
+              }
             }
             audioEmitter.removeEventListener('cancel', onCancel);
             reject(new Error('User cancel'));
@@ -283,7 +287,9 @@ export function withAIStream() {
           });
           const onError = async () => {
             end = true;
-            await userMsg.remove();
+            if (userMsg.isShow) {
+              await userMsg.remove();
+            }
             audioEmitter.removeEventListener('error', onError);
             reject(new Error('Audio emitter error'));
           };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.0-beta-8",
+  "version": "0.2.0-beta-10",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -35,5 +35,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "586a5010616634911334e31f9c1e1993231dfb95"
+  "gitHead": "a17ba630a9ec71afe6a0fc46c427f664d46acf82"
 }