@ray-js/t-agent-plugin-aistream 0.2.0-beta-14 → 0.2.0-beta-16

package/dist/AIStreamTypes.d.ts

@@ -1434,24 +1434,38 @@ export type ReceivedTextNlgPacket = ReceivedTextPacketBase<ReceivedTextPacketTyp
     finish: boolean;
 }>;
 export declare enum BuildInSkillCode {
-    SEARCH_KNOWLEDGE = "searchKnowledge"
+    SEARCH_KNOWLEDGE = "searchKnowledge",
+    SMART_HOME = "smart_home"
 }
-export type ReceivedTextSkillPacketBody<C = BuildInSkillCode | string, T = any> = {
-    code: C;
-    skillContent: T;
-    text?: string[];
+export type ReceivedTextSkillPacketBody<Code = BuildInSkillCode | string, G = any, C = any> = {
+    code: Code;
+    general?: G;
+    custom?: C;
 };
-export type ReceivedTextSkillPacket<C = BuildInSkillCode | string, T = any> = ReceivedTextPacketBase<ReceivedTextPacketType.SKILL, ReceivedTextSkillPacketBody<C, T>>;
+export type ReceivedTextSkillPacket<Code = BuildInSkillCode | string, G = any, C = any> = ReceivedTextPacketBase<ReceivedTextPacketType.SKILL, ReceivedTextSkillPacketBody<Code, G, C>>;
 export type ReceivedTextPacket = ReceivedTextAsrPacket | ReceivedTextNlgPacket | ReceivedTextSkillPacket;
-export type ReceivedSearchKnowledgeSkillContent = {
-    custom: {
-        documents: Array<{
-            libCode: string;
-            itemId: string;
-            itemType: 'KNOWLEDGE';
-            title: string;
-            url: string;
-            contentBody: string;
-        }>;
+export type ReceivedSearchKnowledgeSkill = ReceivedTextSkillPacketBody<BuildInSkillCode.SEARCH_KNOWLEDGE, any, {
+    documents: Array<{
+        libCode: string;
+        itemId: string;
+        itemType: 'KNOWLEDGE';
+        title: string;
+        url: string;
+        contentBody: string;
+    }>;
+}>;
+export interface ReceivedSmartHomeSkillDevice {
+    devId: string;
+    devIcon: string;
+    dps: Record<string, string>;
+    devName: string;
+}
+export declare enum ReceivedSmartHomeSkillAction {
+    CONTROL_DEVICE = "control_device"
+}
+export type ReceivedSmartHomeSkill = ReceivedTextSkillPacketBody<BuildInSkillCode.SEARCH_KNOWLEDGE, {
+    data: {
+        devices: ReceivedSmartHomeSkillDevice[];
     };
-};
+    action: ReceivedSmartHomeSkillAction;
+}, any>;

package/dist/AIStreamTypes.js

@@ -217,5 +217,10 @@ export let ReceivedTextPacketEof = /*#__PURE__*/function (ReceivedTextPacketEof)
 }({});
 export let BuildInSkillCode = /*#__PURE__*/function (BuildInSkillCode) {
   BuildInSkillCode["SEARCH_KNOWLEDGE"] = "searchKnowledge";
+  BuildInSkillCode["SMART_HOME"] = "smart_home";
   return BuildInSkillCode;
+}({});
+export let ReceivedSmartHomeSkillAction = /*#__PURE__*/function (ReceivedSmartHomeSkillAction) {
+  ReceivedSmartHomeSkillAction["CONTROL_DEVICE"] = "control_device";
+  return ReceivedSmartHomeSkillAction;
 }({});

package/dist/buildIn/withBuildIn.js

@@ -1,7 +1,7 @@
 import "core-js/modules/web.dom-collections.iterator.js";
 import { getMiniAppConfig } from '../utils';
 import logger from '../utils/logger';
-import { BuildInSkillCode } from '../AIStreamTypes';
+import { BuildInSkillCode, ReceivedSmartHomeSkillAction } from '../AIStreamTypes';
 export function withBuildIn() {
   return _agent => {
     if (!_agent.plugins.aiStream || !_agent.plugins.ui) {
@@ -54,7 +54,7 @@ export function withBuildIn() {
           if (skill.code !== BuildInSkillCode.SEARCH_KNOWLEDGE) {
             continue;
           }
-          const content = skill.skillContent;
+          const content = skill;
           if (!((_content$custom = content.custom) !== null && _content$custom !== void 0 && _content$custom.documents)) {
             continue;
           }
@@ -70,6 +70,44 @@ export function withBuildIn() {
         }
       });
     })();
+    (() => {
+      const intentMap = {
+        [ReceivedSmartHomeSkillAction.CONTROL_DEVICE]: 'controlDevice'
+      };
+      onSkillsEnd((skills, responseMessage) => {
+        if (!responseMessage) {
+          return;
+        }
+        const data = {
+          deviceInfo: [],
+          sceneInfo: [],
+          changeInfo: []
+        };
+        for (const skill of skills) {
+          var _content$general;
+          if (skill.code !== BuildInSkillCode.SMART_HOME) {
+            continue;
+          }
+          const content = skill;
+          if (!(content !== null && content !== void 0 && (_content$general = content.general) !== null && _content$general !== void 0 && (_content$general = _content$general.data) !== null && _content$general !== void 0 && (_content$general = _content$general.devices) !== null && _content$general !== void 0 && _content$general.length)) {
+            continue;
+          }
+          const intent = intentMap[content.general.action] || 'controlDevice';
+          for (const dev of content.general.data.devices) {
+            data.deviceInfo.push({
+              icon: dev.devIcon,
+              name: dev.devName,
+              deviceId: dev.devId,
+              intent,
+              success: null
+            });
+          }
+        }
+        if (data.deviceInfo.length) {
+          responseMessage.bubble.addTile('operateCard', data);
+        }
+      });
+    })();
     return {};
   };
 }

package/dist/utils/AIStream.d.ts

@@ -120,7 +120,7 @@ export declare class AIStreamEvent {
     }): Promise<void>;
     abort(options?: {
         userData?: Attribute[];
-    }): Promise<void>;
+    }): void;
     on(name: 'close', callback: () => void): void;
     on(name: 'finish', callback: () => void): void;
     on(name: 'error', callback: (error: any) => void): void;

package/dist/utils/AIStream.js

@@ -374,6 +374,7 @@ export class AIStreamEvent {
     if (this.streams[dataChannel]) {
       throw new AIStreamEventError("".concat(dataChannel, " stream already exists"), AIStreamErrorCode.AIStreamError);
     }
+    let startPromise = null;
     const stream = {
       dataChannel,
       type: source.type,
@@ -392,11 +393,12 @@ export class AIStreamEvent {
               count: source.amplitudeCount
             });
           }
-          await startRecordAndSendAudioData({
+          startPromise = startRecordAndSendAudioData({
             sessionId: this.sessionId,
             dataChannel,
             userData: source.userData
           });
+          await startPromise;
         } else if (source.type === 'video') {
           // const cameraType = source.cameraType || VideoCameraType.BACK;
           // await startRecordAndSendVideoData({
@@ -410,13 +412,22 @@ export class AIStreamEvent {
         if (!stream.started || this.closed) {
           return;
         }
+        // 一定要等录音开始，才能结束
+        if (startPromise) {
+          try {
+            await startPromise;
+          } catch (e) {
+            // pass
+          }
+          startPromise = null;
+        }
         if (source.type === 'audio') {
           if (source.amplitudeCount) {
             unregisterVoiceAmplitudes({
               count: source.amplitudeCount
             });
           }
-          await stopRecordAndSendAudioData({
+          stopRecordAndSendAudioData({
             sessionId: this.sessionId,
             dataChannel,
             userData: source.userData
@@ -452,7 +463,7 @@ export class AIStreamEvent {
       userData: options === null || options === void 0 ? void 0 : options.userData
     });
   }
-  async abort(options) {
+  abort(options) {
     if (this.closed) {
       return;
     }

package/dist/utils/defaultMock.js

@@ -124,7 +124,7 @@ mock.hooks.hook('createSession', context => {
   // 用于测试断开连接;
   // setTimeout(() => {
   //   dispatch('onConnectStateChanged', {
-  //     connectionId: getCurrentConnection().connectionId,
+  //     connectionId: '',
   //     connectState: ConnectState.DISCONNECTED,
   //     code: 200,
   //   });
@@ -134,7 +134,7 @@ mock.hooks.hook('createSession', context => {
   //     s.closed = true;
   //   });
   //   mock.data.set('sessionMap', new Map());
-  // }, 3000);
+  // }, 5000);
 
   // 用于测试断开会话
   // setTimeout(() => {

package/dist/utils/sendMessage.js

@@ -60,14 +60,14 @@ export function sendBlocksToAIStream(params) {
         controller.enqueue(part);
       };
       let aborting = false;
-      const abort = async () => {
+      const abort = () => {
         if (aborting || canceled || closed) {
           return;
         }
         aborting = true;
         canceled = true;
         if (event) {
-          await event.abort();
+          event.abort();
         }
         aborting = false;
       };
@@ -80,15 +80,6 @@ export function sendBlocksToAIStream(params) {
           }
         }
       };
-
-      // signal?.addEventListener('abort', async () => {
-      //   logger.debug('sendBlocksToAIStream signal aborted');
-      //   abortPromise = abort();
-      //   await abortPromise;
-      //   logger.debug('sendBlocksToAIStream signal aborted done');
-      //   close();
-      // });
-
       let pendingCancel = false;
       if (audioEmitter) {
         audioEmitter.addEventListener('confirm', () => {
@@ -133,7 +124,7 @@ export function sendBlocksToAIStream(params) {
       }
       if (signal !== null && signal !== void 0 && signal.aborted || pendingCancel) {
         logger.debug('sendBlocksToAIStream pendingCancel aborted');
-        await abort();
+        abort();
         close();
         return;
       }
@@ -330,21 +321,21 @@ export function sendBlocksToAIStream(params) {
               await s.stop();
             }
             logger.debug('sendBlocksToAIStream audio cancel aborted');
-            await abort();
+            abort();
             resolve();
           });
           s.start();
         });
         if (signal !== null && signal !== void 0 && signal.aborted) {
           logger.debug('sendBlocksToAIStream after audio aborted');
-          await abort();
+          abort();
           close();
           return;
         }
       }
       await event.end();
       if (signal !== null && signal !== void 0 && signal.aborted) {
-        await abort();
+        abort();
         close();
       }
     },
@@ -353,7 +344,7 @@ export function sendBlocksToAIStream(params) {
         var _event;
         canceled = true;
         logger.debug('sendBlocksToAIStream stream canceled', reason);
-        await ((_event = event) === null || _event === void 0 ? void 0 : _event.abort());
+        (_event = event) === null || _event === void 0 || _event.abort();
       }
     }
   });

package/dist/withAIStream.js

@@ -235,6 +235,7 @@ export function withAIStream() {
         let end = false;
         audioPromise = new Promise((resolve, reject) => {
           // 当确认发送时，展示 loading
+          let userMsgShow = false;
           const onConfirm = async () => {
             if (end) {
               return;
@@ -243,6 +244,7 @@ export function withAIStream() {
               status: ChatMessageStatus.START
             });
             userMsg.bubble.setStatus(BubbleTileStatus.NORMAL);
+            userMsgShow = true; // show 会延迟变更 isShow 状态，所以使用 userMsgShow
             await userMsg.show();
             audioEmitter.removeEventListener('confirm', onConfirm);
           };
@@ -253,10 +255,8 @@ export function withAIStream() {
             if (end) {
               return;
             }
-            if (!event.detail.text) {
-              if (userMsg.isShow) {
-                await userMsg.remove();
-              }
+            if (!event.detail.text && userMsgShow) {
+              await userMsg.remove();
               reject(new Error('No text found in audio event'));
               return;
             }
@@ -274,21 +274,20 @@ export function withAIStream() {
           audioEmitter.addEventListener('finish', onFinished);
           const onCancel = async () => {
             end = true;
-            if (!response.started) {
-              if (userMsg.isShow) {
-                await userMsg.remove();
-              }
+            if (!response.started && userMsgShow) {
+              await userMsg.remove();
             }
             audioEmitter.removeEventListener('cancel', onCancel);
             reject(new Error('User cancel'));
           };
           audioEmitter.addEventListener('cancel', onCancel);
           signal.addEventListener('abort', () => {
+            logger.log('withAIStream chat agent.chat audioEmitter onAbort');
             audioEmitter.dispatchEvent(new EmitterEvent('cancel'));
           });
           const onError = async () => {
             end = true;
-            if (userMsg.isShow && userMsg.status !== ChatMessageStatus.FINISH) {
+            if (userMsgShow && userMsg.status !== ChatMessageStatus.FINISH) {
               await userMsg.remove();
             }
             audioEmitter.removeEventListener('error', onError);
@@ -419,7 +418,7 @@ export function withAIStream() {
       await hooks.callHook('onTTTAction', tile, result);
       if (result.action) {
         await runTTTAction(result.action, {
-          onSendSkills: async () => {
+          onSendSkills: async (block, options) => {
             throw new Error('send skill not support');
           },
           onSendMessages: async (blocks, sendImmediately) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.0-beta-14",
+  "version": "0.2.0-beta-16",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -35,5 +35,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "2fdaf9e5abc36632f003b7060ada22390cda132b"
+  "gitHead": "d9abc8e193dc1db6a7f9fca9ebeba2ff6462b713"
 }