@ray-js/t-agent-plugin-aistream 0.2.0-beta-11 → 0.2.0-beta-13

package/dist/utils/AIStream.js

@@ -34,7 +34,8 @@ export class AIStreamConnection {
     _defineProperty(this, "promise", null);
     _defineProperty(this, "observer", null);
     _defineProperty(this, "onStateChanged", entry => {
-      if (entry.type === 'connectionState' && entry.body.connectionId === this.connectionId) {
+      // 一个小程序只能启动一个 connection，所以这里不判断 id
+      if (entry.type === 'connectionState') {
         this.state = entry.body.connectState;
         this.activeSessions.forEach(session => {
           if (session.sessionId) {
@@ -347,7 +348,7 @@ export class AIStreamEvent {
           userData: chunk.userData
         });
       }
-      await sendEventPayloadEnd({
+      sendEventPayloadEnd({
         eventId: this.eventId,
         sessionId: this.sessionId,
         dataChannel
@@ -380,7 +381,7 @@ export class AIStreamEvent {
         stream.started = true;
         if (source.type === 'audio') {
           if (source.amplitudeCount) {
-            await registerRecordAmplitudes({
+            registerRecordAmplitudes({
               count: source.amplitudeCount
             });
           }
@@ -404,7 +405,7 @@ export class AIStreamEvent {
         }
         if (source.type === 'audio') {
           if (source.amplitudeCount) {
-            await unregisterVoiceAmplitudes({
+            unregisterVoiceAmplitudes({
               count: source.amplitudeCount
             });
           }
@@ -421,7 +422,7 @@ export class AIStreamEvent {
           //   userData: source.userData,
           // });
         }
-        await sendEventPayloadEnd({
+        sendEventPayloadEnd({
           eventId: this.eventId,
           sessionId: this.sessionId,
           dataChannel
@@ -449,7 +450,8 @@ export class AIStreamEvent {
       return;
     }
     if (this.sessionId) {
-      await sendEventChatBreak({
+      // 故意不等，直接发送中断事件
+      sendEventChatBreak({
         eventId: this.eventId,
         sessionId: this.sessionId,
         userData: options === null || options === void 0 ? void 0 : options.userData

package/dist/utils/defaultMock.js

@@ -153,14 +153,18 @@ mock.hooks.hook('closeSession', context => {
   map.delete(sessionId);
   context.result = true;
 });
-mock.hooks.hook('sendEventStart', context => {
+mock.hooks.hook('sendEventStart', async context => {
   const {
     options
   } = context;
   const {
     sessionId
   } = options;
+
+  // throw new Error('sendEventStart is deprecated, use sendEventStartV2 instead');
+
   const session = getSession(sessionId);
+  await mock.sleep(200);
   if (session.currentEvent) {
     throw new Error('sendEventStart already in event');
   }
@@ -216,13 +220,14 @@ mock.hooks.hook('unregisterVoiceAmplitudes', context => {
   mock.data.delete('recordAmplitudesCount');
   context.result = true;
 });
-mock.hooks.hook('sendEventEnd', context => {
+mock.hooks.hook('sendEventEnd', async context => {
   const session = getSession(context.options.sessionId, context.options.eventId);
   context.result = true;
   const event = session.currentEvent;
   if (event.eventId !== context.options.eventId) {
     throw new Error('sendEventEnd eventId mismatch');
   }
+  await mock.sleep(100);
   (async () => {
     var _ctx$responseSkills;
     await event.asr.promise;
@@ -249,7 +254,6 @@ mock.hooks.hook('sendEventEnd', context => {
         return;
       }
       event.replyEvent(EventType.EVENT_END);
-      console.log('No data to send, ending event, session.currentEvent = null');
       session.currentEvent = null;
       return;
     }
@@ -319,28 +323,28 @@ mock.hooks.hook('sendEventEnd', context => {
       return;
     }
     event.replyEvent(EventType.EVENT_END);
-    console.log('finish sendEventEnd session.currentEvent = null');
     session.currentEvent = null;
   })();
 });
-mock.hooks.hook('sendEventPayloadEnd', context => {
+mock.hooks.hook('sendEventPayloadEnd', async context => {
   getSession(context.options.sessionId, context.options.eventId);
   context.result = true;
+  await mock.sleep(100);
 });
-mock.hooks.hook('sendEventChatBreak', context => {
+mock.hooks.hook('sendEventChatBreak', async context => {
   const session = getSession(context.options.sessionId, context.options.eventId);
   session.currentEvent.controller.abort(new Error('sendEventChatBreak'));
-  console.log('sendEventChatBreak currentEvent = null');
   session.currentEvent = null;
   context.result = true;
 });
-mock.hooks.hook('startRecordAndSendAudioData', context => {
+mock.hooks.hook('startRecordAndSendAudioData', async context => {
   const session = getSession(context.options.sessionId);
   if (!session.currentEvent) {
     throw new Error('startRecordAndSendAudioData event not exists');
   }
   const event = session.currentEvent;
   context.result = true;
+  await mock.sleep(100);
   (async () => {
     const {
       finishController
@@ -378,7 +382,7 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
       }
 
       // 终止识别到出完整结果的延迟
-      await mock.sleep(5000);
+      await mock.sleep(500);
       if (event.controller.signal.aborted) {
         return;
       }
@@ -437,13 +441,14 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
     }
   })();
 });
-mock.hooks.hook('stopRecordAndSendAudioData', context => {
+mock.hooks.hook('stopRecordAndSendAudioData', async context => {
   const session = getSession(context.options.sessionId);
   if (!session.currentEvent) {
     throw new Error('stopRecordAndSendAudioData event not exists');
   }
   session.currentEvent.asr.finishController.abort(new Error('stopRecordAndSendAudioData'));
   context.result = true;
+  await mock.sleep(100);
 });
 mock.hooks.hook('startRecordAndSendVideoData', context => {
   context.result = true;

package/dist/utils/sendMessage.js

@@ -7,6 +7,7 @@ import { AIStreamAttributePayloadType, AIStreamAttributeType, AIStreamChatSysWor
 import { EmitterEvent, generateId, safeParseJSON, StreamResponse } from '@ray-js/t-agent';
 import { tryCatch } from './misc';
 import { AIStreamConnectionError } from './errors';
+import logger from './logger';
 const mimeTypeToFormatMap = {
   'video/mp4': FileFormat.MP4,
   'text/json': FileFormat.JSON,
@@ -21,6 +22,7 @@ export class AIStreamSessionError extends Error {
   }
 }
 export function sendBlocksToAIStream(params) {
+  logger.debug('sendBlocksToAIStream start');
   const {
     session,
     blocks,
@@ -52,11 +54,59 @@ export function sendBlocksToAIStream(params) {
   const stream = new ReadableStream({
     async start(controller) {
       const enqueue = part => {
-        if (canceled || closed) {
+        if (signal !== null && signal !== void 0 && signal.aborted || canceled || closed) {
           return;
         }
         controller.enqueue(part);
       };
+      let aborting = false;
+      const abort = async () => {
+        if (aborting || canceled || closed) {
+          return;
+        }
+        aborting = true;
+        canceled = true;
+        if (event) {
+          await event.abort();
+        }
+        aborting = false;
+      };
+      const close = () => {
+        if (!closed) {
+          logger.debug('sendBlocksToAIStream close');
+          closed = true;
+          if (!canceled && !(signal !== null && signal !== void 0 && signal.aborted)) {
+            controller.close();
+          }
+        }
+      };
+
+      // signal?.addEventListener('abort', async () => {
+      //   logger.debug('sendBlocksToAIStream signal aborted');
+      //   abortPromise = abort();
+      //   await abortPromise;
+      //   logger.debug('sendBlocksToAIStream signal aborted done');
+      //   close();
+      // });
+
+      let pendingCancel = false;
+      if (audioEmitter) {
+        audioEmitter.addEventListener('confirm', () => {
+          // 在确认发送时，如果事件还没创建，则取消发送
+          if (!event) {
+            logger.debug('sendBlocksToAIStream audioEmitter confirm before event start');
+            // event 留到后面再关
+            pendingCancel = true;
+          }
+        });
+        audioEmitter.addEventListener('cancel', () => {
+          if (!event) {
+            logger.debug('sendBlocksToAIStream audioEmitter cancel before event start');
+            // event 留到后面再关
+            pendingCancel = true;
+          }
+        });
+      }
       let error;
       [error, event] = await tryCatch(() => session.startEvent({
         userData: [{
@@ -65,15 +115,26 @@ export function sendBlocksToAIStream(params) {
           value: JSON.stringify(attribute)
         }]
       }));
-      if (signal !== null && signal !== void 0 && signal.aborted) {
-        canceled = true;
-        controller.close();
+      if (error) {
+        const e = new AIStreamSessionError(error.message, error.code || 0);
+        enqueue({
+          type: 'error',
+          error: e,
+          level: 'error',
+          meta: {}
+        });
+        if (audioEmitter) {
+          audioEmitter.dispatchEvent(new EmitterEvent('error', {
+            detail: e
+          }));
+        }
+        close();
         return;
       }
-      if (error) {
-        controller.error(error);
-        closed = true;
-        controller.close();
+      if (signal !== null && signal !== void 0 && signal.aborted || pendingCancel) {
+        logger.debug('sendBlocksToAIStream pendingCancel aborted');
+        await abort();
+        close();
         return;
       }
       const meta = {
@@ -91,6 +152,7 @@ export function sendBlocksToAIStream(params) {
           }
           const packet = safeParseJSON(data.body.text);
           if (packet.bizType === ReceivedTextPacketType.NLG) {
+            logger.debug('sendBlocksToAIStream Receive NLG', packet.data.content);
             const text = prevText + packet.data.content;
             enqueue({
               type: 'text',
@@ -100,6 +162,7 @@ export function sendBlocksToAIStream(params) {
             });
             prevText = text;
           } else if (packet.bizType === ReceivedTextPacketType.SKILL) {
+            logger.debug('sendBlocksToAIStream Receive SKILL', packet.data);
             enqueue({
               id: generateId(),
               type: 'attachment',
@@ -110,11 +173,15 @@ export function sendBlocksToAIStream(params) {
           } else if (packet.bizType === ReceivedTextPacketType.ASR && audioEmitter) {
             if (packet.eof === ReceivedTextPacketEof.END) {
               if (packet.data.text === '') {
+                logger.debug('sendBlocksToAIStream Receive ASR EOF EMPTY');
                 // 没识别出任何文本
                 audioEmitter.dispatchEvent(new EmitterEvent('error', {
-                  detail: {}
+                  detail: {
+                    name: 'AsrEmptyError'
+                  }
                 }));
               } else {
+                logger.debug('sendBlocksToAIStream Receive ASR EOF', packet.data.text);
                 audioEmitter.dispatchEvent(new EmitterEvent('finish', {
                   detail: {
                     text: packet.data.text
@@ -122,6 +189,7 @@ export function sendBlocksToAIStream(params) {
                 }));
               }
             } else {
+              logger.debug('sendBlocksToAIStream Receive ASR', packet.data.text);
               audioEmitter.dispatchEvent(new EmitterEvent('update', {
                 detail: {
                   text: packet.data.text
@@ -165,35 +233,48 @@ export function sendBlocksToAIStream(params) {
           }
         } else if (data.type === 'sessionState') {
           if (data.body.sessionState === SessionState.CLOSED || data.body.sessionState === SessionState.CREATE_FAILED) {
+            const e = new AIStreamSessionError('Session closed', data.body.code);
             enqueue({
               type: 'error',
-              error: new AIStreamSessionError('Session closed', data.body.code),
+              error: e,
               level: 'error',
               meta
             });
+            if (audioEmitter) {
+              audioEmitter.dispatchEvent(new EmitterEvent('error', {
+                detail: e
+              }));
+            }
+            close();
           }
         } else if (data.type === 'connectionState') {
           if (data.body.connectState === ConnectState.DISCONNECTED || data.body.connectState === ConnectState.CLOSED) {
+            const e = new AIStreamConnectionError('Connection disconnected', data.body.code);
             enqueue({
               type: 'error',
-              error: new AIStreamConnectionError('Connection disconnected', data.body.code),
+              error: e,
               level: 'error',
               meta
             });
+            if (audioEmitter) {
+              audioEmitter.dispatchEvent(new EmitterEvent('error', {
+                detail: e
+              }));
+            }
+            close();
           }
         }
       });
       event.on('close', () => {
-        if (!canceled && !closed) {
-          // 当取消后，不需要关闭控制器
-          controller.close();
+        logger.debug('sendBlocksToAIStream event closed');
+        if (!canceled) {
+          close();
         }
       });
       event.on('finish', () => {
+        logger.debug('sendBlocksToAIStream event finished');
         if (!canceled) {
-          closed = true;
-          // 当取消后，不需要关闭控制器
-          controller.close();
+          close();
         }
       });
       event.on('error', error => {
@@ -238,26 +319,40 @@ export function sendBlocksToAIStream(params) {
             type: 'audio',
             amplitudeCount
           });
-          audioEmitter.addEventListener('confirm', () => {
-            s.stop().then(resolve);
+          audioEmitter.addEventListener('confirm', async () => {
+            if (!canceled) {
+              await s.stop();
+            }
+            resolve();
           });
           audioEmitter.addEventListener('cancel', async () => {
-            await s.stop();
             if (!canceled) {
-              canceled = true;
-              await event.abort();
+              await s.stop();
             }
+            logger.debug('sendBlocksToAIStream audio cancel aborted');
+            await abort();
             resolve();
           });
           s.start();
         });
+        if (signal !== null && signal !== void 0 && signal.aborted) {
+          logger.debug('sendBlocksToAIStream after audio aborted');
+          await abort();
+          close();
+          return;
+        }
       }
       await event.end();
+      if (signal !== null && signal !== void 0 && signal.aborted) {
+        await abort();
+        close();
+      }
     },
-    async cancel() {
+    async cancel(reason) {
       if (!canceled) {
         var _event;
         canceled = true;
+        logger.debug('sendBlocksToAIStream stream canceled', reason);
         await ((_event = event) === null || _event === void 0 ? void 0 : _event.abort());
       }
     }

package/dist/withAIStream.js

@@ -209,6 +209,7 @@ export function withAIStream() {
       return result;
     };
     const chat = async (blocks, signal, options) => {
+      logger.log('withAIStream chat agent.chat start');
       const {
         sendBy = 'user',
         responseBy = 'assistant',
@@ -287,7 +288,7 @@ export function withAIStream() {
           });
           const onError = async () => {
             end = true;
-            if (userMsg.isShow) {
+            if (userMsg.isShow && userMsg.status !== ChatMessageStatus.FINISH) {
               await userMsg.remove();
             }
             audioEmitter.removeEventListener('error', onError);
@@ -303,6 +304,7 @@ export function withAIStream() {
         await userMsg.show();
         await userMsg.persist();
       }
+      logger.log('withAIStream chat agent.chat send');
       const {
         response,
         metaPromise
@@ -310,6 +312,7 @@ export function withAIStream() {
       if (audioPromise) {
         try {
           await audioPromise;
+          logger.log('withAIStream chat agent.chat audioPromise finish');
         } catch (e) {
           // 没有识别到文字
           return [];
@@ -320,6 +323,7 @@ export function withAIStream() {
         role: responseBy
       });
       const skills = [];
+      logger.debug('withAIStream chat agent.flushStreamToShow');
       const result = {
         messages: await agent.flushStreamToShow(message, response, {
           attachmentCompose: async (respMsg, part) => {
@@ -334,18 +338,15 @@ export function withAIStream() {
           }
         })
       };
+      logger.debug('withAIStream chat agent.flushStreamToShow end');
       if (skills.length) {
         await hooks.callHook('onSkillsEnd', skills, message, result);
         await message.update();
       }
-      if (message.bubble.status === BubbleTileStatus.ABORTED) {
-        if (message.bubble.text) {
-          await message.persist();
-        } else {
-          await message.remove();
-        }
-      } else {
+      if (message.bubble.text) {
         await message.persist();
+      } else {
+        await message.remove();
       }
       return [userMsg, ...result.messages];
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.0-beta-11",
+  "version": "0.2.0-beta-13",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -35,5 +35,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "dd56b31b0095fb82d0fb5d4f58581d00ae2970ee"
+  "gitHead": "ad4d4b4637a4a5396bcf5adf21adcbf215561436"
 }