@ray-js/t-agent-plugin-aistream 0.2.0-beta-6 → 0.2.0-beta-8

package/dist/AIStreamTypes.d.ts

@@ -1433,9 +1433,25 @@ export type ReceivedTextNlgPacket = ReceivedTextPacketBase<ReceivedTextPacketTyp
     appendMode: 'append';
     finish: boolean;
 }>;
-export type ReceivedTextSkillPacket<T = any> = ReceivedTextPacketBase<ReceivedTextPacketType.SKILL, {
-    code: string;
+export declare enum BuildInSkillCode {
+    SEARCH_KNOWLEDGE = "searchKnowledge"
+}
+export type ReceivedTextSkillPacketBody<C = BuildInSkillCode | string, T = any> = {
+    code: C;
     skillContent: T;
-    text: string[];
-}>;
+    text?: string[];
+};
+export type ReceivedTextSkillPacket<C = BuildInSkillCode | string, T = any> = ReceivedTextPacketBase<ReceivedTextPacketType.SKILL, ReceivedTextSkillPacketBody<C, T>>;
 export type ReceivedTextPacket = ReceivedTextAsrPacket | ReceivedTextNlgPacket | ReceivedTextSkillPacket;
+export type ReceivedSearchKnowledgeSkillContent = {
+    custom: {
+        documents: Array<{
+            libCode: string;
+            itemId: string;
+            itemType: 'KNOWLEDGE';
+            title: string;
+            url: string;
+            contentBody: string;
+        }>;
+    };
+};

package/dist/AIStreamTypes.js

@@ -214,4 +214,8 @@ export let ReceivedTextPacketEof = /*#__PURE__*/function (ReceivedTextPacketEof)
   ReceivedTextPacketEof[ReceivedTextPacketEof["CONTINUE"] = 0] = "CONTINUE";
   ReceivedTextPacketEof[ReceivedTextPacketEof["END"] = 1] = "END";
   return ReceivedTextPacketEof;
+}({});
+export let BuildInSkillCode = /*#__PURE__*/function (BuildInSkillCode) {
+  BuildInSkillCode["SEARCH_KNOWLEDGE"] = "searchKnowledge";
+  return BuildInSkillCode;
 }({});

package/dist/buildIn/index.d.ts

@@ -0,0 +1 @@
+export * from './withBuildIn';

package/dist/buildIn/index.js

@@ -0,0 +1 @@
+export * from './withBuildIn';

package/dist/buildIn/transform.d.ts

@@ -0,0 +1,10 @@
+import { TTTAction } from '../utils';
+import { ButtonRouterType, Recommendation } from '../AIStreamTypes';
+export declare const getActionFromRouter: (type: ButtonRouterType) => {
+    i18nKey: string;
+    action: TTTAction;
+} | null;
+export declare const getActionFromRecommendation: (item: Recommendation) => {
+    text: string;
+    action: TTTAction;
+} | null;

package/dist/buildIn/transform.js

@@ -0,0 +1,179 @@
+import { ButtonRouterType } from '../AIStreamTypes';
+export const getActionFromRouter = type => {
+  switch (type) {
+    case ButtonRouterType.CREATE_SCENE:
+      return {
+        i18nKey: 't-agent.build-in.button.create_scene_manually',
+        action: {
+          type: 'openRoute',
+          url: 'thingSmart://thing_add_scene?biz_type=ai_smart&action=add'
+        }
+      };
+    case ButtonRouterType.FAMILY_MANAGE:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_home_manage',
+        action: {
+          type: 'openRoute',
+          url: 'thingSmart://family_manage'
+        }
+      };
+    case ButtonRouterType.ROOM_MANAGE:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_room_manage',
+        action: {
+          type: 'openRoute',
+          url: 'thingSmart://room_manage'
+        }
+      };
+    case ButtonRouterType.MESSAGE_WARN:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_alarm_message',
+        action: {
+          type: 'openRoute',
+          url: 'thingSmart://messageCenter?category=0'
+        }
+      };
+    case ButtonRouterType.MESSAGE_HOME:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_home_message',
+        action: {
+          type: 'openRoute',
+          url: 'thingSmart://messageCenter?category=1'
+        }
+      };
+    case ButtonRouterType.MESSAGE_NOTICE:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_bulletin',
+        action: {
+          type: 'openRoute',
+          url: 'thingSmart://messageCenter?category=2'
+        }
+      };
+    case ButtonRouterType.MESSAGE_SETTING:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_notification_setting',
+        action: {
+          type: 'openRoute',
+          url: 'thingSmart://push_setting'
+        }
+      };
+    case ButtonRouterType.USER_INFO:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_personal_information',
+        action: {
+          type: 'openRoute',
+          url: 'thingSmart://personal_info'
+        }
+      };
+    case ButtonRouterType.ACCOUNT_SECURITY:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_account_security',
+        action: {
+          type: 'openRoute',
+          url: 'thingSmart://account_and_safety'
+        }
+      };
+    case ButtonRouterType.SETTING:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_setting',
+        action: {
+          type: 'openRoute',
+          url: 'thingSmart://thing_user_setting'
+        }
+      };
+    case ButtonRouterType.PARING:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_paring',
+        action: {
+          type: 'openRoute',
+          url: 'thingSmart://config_device'
+        }
+      };
+    case ButtonRouterType.DEVICE_SHARE:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_share_device',
+        action: {
+          type: 'openRoute',
+          // TODO: add devId
+          url: "thingSmart://thing_single_device_share?devId="
+        }
+      };
+    case ButtonRouterType.TICKETS:
+      return {
+        i18nKey: 't-agent.build-in.button.enter_faq_feedback',
+        action: {
+          type: 'buildIn',
+          name: 'ty.openHelpCenter'
+        }
+      };
+    case ButtonRouterType.QUESTIONNAIRE:
+      return {
+        i18nKey: 't-agent.build-in.button.questionnaire_take',
+        action: {
+          type: 'buildIn',
+          name: 'questionnaire'
+        }
+      };
+    case ButtonRouterType.HOME_LOCATION:
+      return {
+        i18nKey: 't-agent.build-in.button.set_home_location',
+        action: {
+          type: 'buildIn',
+          name: 'homeLocation'
+        }
+      };
+    default:
+      return undefined;
+  }
+};
+export const getActionFromRecommendation = item => {
+  const {
+    text
+  } = item;
+  switch (item.type) {
+    case 1:
+      {
+        return {
+          text,
+          action: {
+            type: 'sendMessage',
+            blocks: [{
+              type: 'text',
+              text
+            }]
+          }
+        };
+      }
+    case 2:
+      {
+        return {
+          text,
+          action: {
+            type: 'openH5',
+            url: item.h5Url
+          }
+        };
+      }
+    case 3:
+      {
+        return {
+          text,
+          action: {
+            type: 'openRoute',
+            url: item.miniProgramUrl
+          }
+        };
+      }
+    case 4:
+      {
+        return {
+          text,
+          action: {
+            type: 'openRoute',
+            url: item.nativeUrl
+          }
+        };
+      }
+  }
+  return null;
+};

package/dist/buildIn/withBuildIn.d.ts

@@ -0,0 +1,5 @@
+import { ChatAgent } from '@ray-js/t-agent';
+interface WithBuildInOptions {
+}
+export declare function withBuildIn(options?: WithBuildInOptions): (_agent: ChatAgent) => {};
+export {};

package/dist/buildIn/withBuildIn.js

@@ -0,0 +1,75 @@
+import "core-js/modules/web.dom-collections.iterator.js";
+import { getMiniAppConfig } from '../utils';
+import logger from '../utils/logger';
+import { BuildInSkillCode } from '../AIStreamTypes';
+export function withBuildIn() {
+  return _agent => {
+    if (!_agent.plugins.aiStream || !_agent.plugins.ui) {
+      throw new Error('withBuildIn must be used after withAIStream and withUI');
+    }
+    const agent = _agent;
+    const {
+      session,
+      onTileEvent,
+      onAgentStart,
+      createMessage
+    } = agent;
+    const {
+      onSkillsEnd
+    } = agent.plugins.aiStream;
+    onAgentStart(async () => {
+      const agentId = session.get('AIStream.agentId');
+      if (agentId) {
+        const _conf = await session.get('AIAssistant.projectConfig');
+        if (!_conf) {
+          try {
+            const config = (await getMiniAppConfig({})).config || {};
+            const projectConfig = config[agentId];
+            if (projectConfig) {
+              logger.debug('getMiniAppConfig projectConfig', {
+                agentId,
+                projectConfig
+              });
+              await session.set('AIAssistant.projectConfig', projectConfig);
+            }
+          } catch (error) {
+            logger.warn('getMiniAppConfig error', error);
+          }
+        }
+      }
+    });
+
+    // 关联文档
+
+    (() => {
+      onSkillsEnd((skills, responseMessage) => {
+        if (!responseMessage) {
+          return;
+        }
+        const data = {
+          documents: []
+        };
+        for (const skill of skills) {
+          var _content$custom;
+          if (skill.code !== BuildInSkillCode.SEARCH_KNOWLEDGE) {
+            continue;
+          }
+          const content = skill.skillContent;
+          if (!((_content$custom = content.custom) !== null && _content$custom !== void 0 && _content$custom.documents)) {
+            continue;
+          }
+          for (const doc of content.custom.documents) {
+            data.documents.push({
+              title: doc.title,
+              url: doc.url
+            });
+          }
+        }
+        if (data.documents.length) {
+          responseMessage.bubble.addTile('documents', data);
+        }
+      });
+    })();
+    return {};
+  };
+}

package/dist/index.d.ts

@@ -2,3 +2,4 @@ export * from './utils';
 export * from './AIStreamTypes';
 export * from './withAIStream';
 export * from './ChatHistoryStore';
+export * from './buildIn';

package/dist/index.js

@@ -1,4 +1,5 @@
 export * from './utils';
 export * from './AIStreamTypes';
 export * from './withAIStream';
-export * from './ChatHistoryStore';
+export * from './ChatHistoryStore';
+export * from './buildIn';

package/dist/utils/defaultMock.js

@@ -219,6 +219,7 @@ mock.hooks.hook('sendEventEnd', context => {
   context.result = true;
   const event = session.currentEvent;
   (async () => {
+    var _ctx$responseSkills;
     await event.asrStatus.promise;
     const ctx = {
       data: event.data,
@@ -236,6 +237,13 @@ mock.hooks.hook('sendEventEnd', context => {
       map.delete(session.sessionId);
       return;
     }
+    if (ctx.data.length === 0) {
+      session.replyEvent(EventType.EVENT_START);
+      await mock.sleep(100);
+      session.replyEvent(EventType.EVENT_END);
+      session.currentEvent = null;
+      return;
+    }
     const text = ctx.responseText || '⚠️ No mock text response matched!';
     const words = splitString(text);
     if (event.controller.signal.aborted) {
@@ -282,6 +290,20 @@ mock.hooks.hook('sendEventEnd', context => {
     if (event.controller.signal.aborted) {
       return;
     }
+    if ((_ctx$responseSkills = ctx.responseSkills) !== null && _ctx$responseSkills !== void 0 && _ctx$responseSkills.length) {
+      for (const skill of ctx.responseSkills) {
+        await mock.sleep(100);
+        if (event.controller.signal.aborted) {
+          return;
+        }
+        session.replyText(StreamFlag.IN_PROGRESS, {
+          bizType: ReceivedTextPacketType.SKILL,
+          eof: ReceivedTextPacketEof.END,
+          bizId,
+          data: skill
+        });
+      }
+    }
     session.replyText(StreamFlag.END);
     await mock.sleep(500);
     if (event.controller.signal.aborted) {
@@ -327,7 +349,7 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
       responseText: ''
     };
     await mock.hooks.callHook('asrDetection', result);
-    const responseText = result.responseText || '⚠️ No mock asr response matched!';
+    const responseText = result.responseText || '';
     const bizId = generateId();
     let finishResolve;
     session.currentEvent.asrStatus.promise = new Promise(resolve => {
@@ -337,8 +359,15 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
     let text = '';
     controller.signal.addEventListener('abort', async () => {
       var _finishResolve;
+      if (!session.currentEvent) {
+        return;
+      }
+
       // 终止识别到出完整结果的延迟
-      await mock.sleep(300);
+      await mock.sleep(5000);
+      if (!session.currentEvent) {
+        return;
+      }
       session.replyText(StreamFlag.IN_PROGRESS, {
         bizType: ReceivedTextPacketType.ASR,
         eof: ReceivedTextPacketEof.END,
@@ -348,8 +377,11 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
         }
       });
       await mock.sleep(100);
+      if (!session.currentEvent) {
+        return;
+      }
       session.replyText(StreamFlag.END);
-      if (session.currentEvent) {
+      if (session.currentEvent && text) {
         session.currentEvent.data.push({
           type: 'text',
           text

package/dist/utils/index.d.ts

@@ -1,7 +1,7 @@
+import { mock } from './mock';
 import './defaultMock';
 export * from './abort';
 export * from './logger';
-export * from './mock';
 export * from './promisify';
 export * from './ttt';
 export * from './url';
@@ -11,3 +11,4 @@ export * from './sendMessage';
 export * from './version';
 export * from './createAsrAgent';
 export * from './actions';
+export { mock };

package/dist/utils/index.js

@@ -1,7 +1,7 @@
+import { mock } from './mock';
 import './defaultMock';
 export * from './abort';
 export * from './logger';
-export * from './mock';
 export * from './promisify';
 export * from './ttt';
 export * from './url';
@@ -10,4 +10,5 @@ export * from './observer';
 export * from './sendMessage';
 export * from './version';
 export * from './createAsrAgent';
-export * from './actions';
+export * from './actions';
+export { mock };

package/dist/utils/mock.d.ts

@@ -1,10 +1,10 @@
 import { Emitter } from '@ray-js/t-agent';
-import { Attribute } from '../AIStreamTypes';
+import { Attribute, ReceivedTextSkillPacketBody } from '../AIStreamTypes';
 interface TTTCallContext {
     options: any;
     result: any;
 }
-interface SendToAIStreamContext {
+export interface SendToAIStreamContext {
     data: Array<{
         type: 'text';
         text: string;
@@ -15,6 +15,7 @@ interface SendToAIStreamContext {
         userData?: Attribute[];
     }>;
     responseText: string | undefined;
+    responseSkills?: ReceivedTextSkillPacketBody[];
 }
 interface AsrDetectionContext {
     responseText: string | undefined;

package/dist/utils/sendMessage.js

@@ -103,11 +103,18 @@ export function sendBlocksToAIStream(params) {
             });
           } else if (packet.bizType === ReceivedTextPacketType.ASR && audioEmitter) {
             if (packet.eof === ReceivedTextPacketEof.END) {
-              audioEmitter.dispatchEvent(new EmitterEvent('finish', {
-                detail: {
-                  text: packet.data.text
-                }
-              }));
+              if (packet.data.text === '') {
+                // 没识别出任何文本
+                audioEmitter.dispatchEvent(new EmitterEvent('error', {
+                  detail: {}
+                }));
+              } else {
+                audioEmitter.dispatchEvent(new EmitterEvent('finish', {
+                  detail: {
+                    text: packet.data.text
+                  }
+                }));
+              }
             } else {
               audioEmitter.dispatchEvent(new EmitterEvent('update', {
                 detail: {

package/dist/withAIStream.d.ts

@@ -1,5 +1,5 @@
-import { ChatAgent, ChatMessage, ComposeHandler, InputBlock } from '@ray-js/t-agent';
-import { ConnectClientType } from './AIStreamTypes';
+import { ChatAgent, ChatMessage, GetChatPluginHandler, InputBlock } from '@ray-js/t-agent';
+import { ConnectClientType, ReceivedTextSkillPacketBody } from './AIStreamTypes';
 import { ChatHistoryLocalStore, StoredMessageObject } from './ChatHistoryStore';
 export interface AIStreamOptions {
     /** client 类型: 1-作为设备代理, 2-作为 App */
@@ -29,11 +29,15 @@ export interface AIStreamOptions {
     /** 自定义消息存储, 返回的实例需要实现 ChatHistoryLocalStore 接口, 返回null则不存储历史聊天记录 */
     createChatHistoryStore?: (agent: ChatAgent) => ChatHistoryLocalStore | null;
 }
+export type AIStreamPlugin = GetChatPluginHandler<typeof withAIStream>;
 export interface AIStreamHooks {
     onMessageParse: (msgItem: StoredMessageObject, result: {
         messages: ChatMessage[];
     }) => void;
-    onSkillCompose: (skill: any, respMsg: ChatMessage, result: {
+    onSkillCompose: (skill: ReceivedTextSkillPacketBody[], respMsg: ChatMessage, result: {
+        messages: ChatMessage[];
+    }) => void;
+    onSkillsEnd: (skills: ReceivedTextSkillPacketBody[], respMsg: ChatMessage, result: {
         messages: ChatMessage[];
     }) => void;
 }
@@ -49,10 +53,11 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
             responseBy?: string | undefined;
             extraOptions?: Record<string, any> | undefined;
         } | undefined) => Promise<ChatMessage[]>;
-        composeHandler: ComposeHandler;
         options: AIStreamOptions;
         removeMessage: (message: ChatMessage) => Promise<void>;
         clearAllMessages: () => Promise<void>;
+        onSkillCompose: (fn: AIStreamHooks['onSkillCompose']) => () => void;
+        onSkillsEnd: (fn: AIStreamHooks['onSkillsEnd']) => () => void;
         feedback: ({ requestId, type }: {
             requestId: string;
             type: string;

package/dist/withAIStream.js

@@ -7,7 +7,7 @@ import "core-js/modules/es.array.unscopables.flat.js";
 import "core-js/modules/esnext.iterator.constructor.js";
 import "core-js/modules/esnext.iterator.map.js";
 import "core-js/modules/web.dom-collections.iterator.js";
-import { BubbleTileStatus, ChatMessageStatus, createHooks } from '@ray-js/t-agent';
+import { BubbleTileStatus, ChatMessageStatus, createHooks, EmitterEvent } from '@ray-js/t-agent';
 import { messageAppraise } from './utils/apis';
 import { AIStreamObserver, getAccountInfo, getCurrentHomeInfo, runTTTAction, sendBlocksToAIStream } from './utils';
 import { BizCode, ConnectClientType, ConnectState } from './AIStreamTypes';
@@ -192,17 +192,6 @@ export function withAIStream() {
         await removeMessage(message);
       }
     });
-    const composeHandler = {
-      attachmentCompose: async (respMsg, part) => {
-        const result = {
-          messages: []
-        };
-        if (part.attachmentType === 'skill') {
-          await hooks.callHook('onSkillCompose', part.attachment, respMsg, result, 'stream');
-        }
-        return result.messages;
-      }
-    };
     const send = (blocks, signal, extraOptions) => {
       const streamSession = session.get('AIStream.streamSession');
       const result = sendBlocksToAIStream({
@@ -211,7 +200,9 @@ export function withAIStream() {
         attribute: _objectSpread({}, extraOptions)
       });
       signal === null || signal === void 0 || signal.addEventListener('abort', event => {
-        result === null || result === void 0 || result.response.cancel(event.reason);
+        if (result.response.started) {
+          result.response.cancel(event.reason);
+        }
         agent.hooks.callHook('onUserAbort', event.reason);
       });
       signal === null || signal === void 0 || signal.throwIfAborted();
@@ -278,12 +269,25 @@ export function withAIStream() {
             resolve();
           };
           audioEmitter.addEventListener('finish', onFinished);
-          const onCancel = () => {
+          const onCancel = async () => {
             end = true;
+            if (!response.started) {
+              await userMsg.remove();
+            }
             audioEmitter.removeEventListener('cancel', onCancel);
             reject(new Error('User cancel'));
           };
           audioEmitter.addEventListener('cancel', onCancel);
+          signal.addEventListener('abort', () => {
+            audioEmitter.dispatchEvent(new EmitterEvent('cancel'));
+          });
+          const onError = async () => {
+            end = true;
+            await userMsg.remove();
+            audioEmitter.removeEventListener('error', onError);
+            reject(new Error('Audio emitter error'));
+          };
+          audioEmitter.addEventListener('error', onError);
         });
       } else {
         userMsg.set({
@@ -309,7 +313,25 @@ export function withAIStream() {
       const message = createMessage({
         role: responseBy
       });
-      const messages = await agent.flushStreamToShow(message, response, composeHandler);
+      const skills = [];
+      const result = {
+        messages: await agent.flushStreamToShow(message, response, {
+          attachmentCompose: async (respMsg, part) => {
+            const result = {
+              messages: []
+            };
+            if (part.attachmentType === 'skill') {
+              skills.push(part.attachment);
+              await hooks.callHook('onSkillCompose', part.attachment, respMsg, result);
+            }
+            return result.messages;
+          }
+        })
+      };
+      if (skills.length) {
+        await hooks.callHook('onSkillsEnd', skills, message, result);
+        await message.update();
+      }
       if (message.bubble.status === BubbleTileStatus.ABORTED) {
         if (message.bubble.text) {
           await message.persist();
@@ -319,7 +341,7 @@ export function withAIStream() {
       } else {
         await message.persist();
       }
-      return [userMsg, ...messages];
+      return [userMsg, ...result.messages];
     };
     onInputBlocksPush(chat);
     const feedback = async _ref => {
@@ -442,7 +464,6 @@ export function withAIStream() {
       aiStream: {
         send,
         chat,
-        composeHandler,
         options,
         removeMessage,
         clearAllMessages: async () => {
@@ -455,6 +476,12 @@ export function withAIStream() {
             await historyStore.removeAll();
           }
         },
+        onSkillCompose: fn => {
+          return hooks.hook('onSkillCompose', fn);
+        },
+        onSkillsEnd: fn => {
+          return hooks.hook('onSkillsEnd', fn);
+        },
         feedback
       }
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.0-beta-6",
+  "version": "0.2.0-beta-8",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -35,5 +35,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "a09b74682949c5860f34105b4a5f8f60fc34a582"
+  "gitHead": "586a5010616634911334e31f9c1e1993231dfb95"
 }