@ray-js/t-agent-plugin-aistream 0.2.2-beta-1 → 0.2.3-beta-1

package/dist/AIStreamTypes.d.ts

@@ -1573,7 +1573,8 @@ export type ReceivedTextAsrPacket = ReceivedTextPacketBase<ReceivedTextPacketTyp
     text: string;
 }>;
 export type ReceivedTextNlgPacket = ReceivedTextPacketBase<ReceivedTextPacketType.NLG, {
-    content: string;
+    reasoningContent?: string;
+    content?: string;
     appendMode: 'append';
     finish: boolean;
 }>;

package/dist/ChatHistoryLocalStore.d.ts

@@ -0,0 +1,22 @@
+import { ChatMessageObject } from '@ray-js/t-agent';
+import { ChatHistoryStore, ChatHistoryStoreOptions, StoredMessageObject, ChatHistoryStorePagination } from './ChatHistoryStore';
+export declare class ChatHistoryLocalStore implements ChatHistoryStore {
+    /** 版本号，会作为内部索引的一部分 */
+    private readonly version;
+    private options;
+    constructor(options: ChatHistoryStoreOptions);
+    private itemToMessage;
+    private getRecordIndex;
+    private getQueryCondition;
+    query(id: number): Promise<StoredMessageObject | null>;
+    queryAll(pagination: ChatHistoryStorePagination): Promise<{
+        total: number;
+        records: StoredMessageObject[];
+    }>;
+    update(id: number, body: ChatMessageObject): Promise<void>;
+    remove(id: number): Promise<void>;
+    removeAll(ids?: number[]): Promise<void>;
+    insert(message: ChatMessageObject): Promise<{
+        id: number;
+    }>;
+}

package/dist/ChatHistoryLocalStore.js

@@ -0,0 +1,191 @@
+import _objectWithoutProperties from "@babel/runtime/helpers/esm/objectWithoutProperties";
+import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
+import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
+const _excluded = ["id"];
+import "core-js/modules/es.array.sort.js";
+import "core-js/modules/es.json.stringify.js";
+import "core-js/modules/esnext.iterator.constructor.js";
+import "core-js/modules/esnext.iterator.filter.js";
+import "core-js/modules/esnext.iterator.map.js";
+import { safeParseJSON } from '@ray-js/t-agent';
+import logger from './utils/logger';
+import { deleteRecordList, insertRecord, queryRecordList, updateRecord } from './utils';
+export class ChatHistoryLocalStore {
+  constructor(options) {
+    /** 版本号，会作为内部索引的一部分 */
+    _defineProperty(this, "version", 'v1');
+    if (!options.agentId) {
+      throw new Error('agentId is required');
+    }
+    if (!options.bizCode) {
+      throw new Error('bizCode is required');
+    }
+    this.options = _objectSpread({
+      indexId: 'default',
+      homeId: 0
+    }, options);
+  }
+  itemToMessage(item) {
+    const messageInfo = safeParseJSON(item.data);
+    if (!messageInfo) {
+      logger.error('ChatHistoryLocalStore failed parse item: ', item);
+      return null;
+    }
+    messageInfo.message.meta.id = item.id;
+    return {
+      id: item.id,
+      indexId: item.index2,
+      agentId: item.solutionCode,
+      deviceId: item.devId,
+      homeId: item.homeId,
+      createdAt: messageInfo.createdAt,
+      updatedAt: messageInfo.updatedAt,
+      message: messageInfo.message
+    };
+  }
+  getRecordIndex() {
+    return {
+      bizCode: this.options.bizCode,
+      solutionCode: this.options.agentId,
+      homeId: this.options.homeId,
+      index: "t-agent-".concat(this.version),
+      index2: this.options.indexId,
+      devId: this.options.deviceId
+    };
+  }
+  getQueryCondition() {
+    const indexes = this.getRecordIndex();
+    const params = {
+      bizCode: [indexes.bizCode],
+      solutionCode: [indexes.solutionCode],
+      homeId: [indexes.homeId],
+      index: [indexes.index],
+      index2: [indexes.index2]
+    };
+    if (indexes.devId) {
+      params.devId = [indexes.devId];
+    }
+    return params;
+  }
+  async query(id) {
+    if (typeof id !== 'number') {
+      logger.warn('ChatHistoryLocalStore query id is not number', {
+        id
+      });
+      return null;
+    }
+    const queryCondition = _objectSpread(_objectSpread({}, this.getQueryCondition()), {}, {
+      id: [id],
+      offset: 0,
+      limit: 1,
+      sortType: 1
+    });
+    const {
+      records
+    } = await queryRecordList(queryCondition);
+    return records[0] ? this.itemToMessage(records[0]) : null;
+  }
+  async queryAll(pagination) {
+    const queryCondition = _objectSpread(_objectSpread({}, this.getQueryCondition()), {}, {
+      offset: 0,
+      sortType: 1
+    });
+    const {
+      offset,
+      limit
+    } = pagination || {};
+    if (offset) {
+      queryCondition.offset = offset;
+    }
+    if (limit) {
+      queryCondition.limit = limit;
+    }
+    queryCondition.sortType = pagination.sort === 'desc' ? 0 : 1;
+    try {
+      const result = await queryRecordList(queryCondition);
+      const list = result.records.map(this.itemToMessage).filter(item => !!item);
+      return {
+        total: result.total,
+        records: list
+      };
+    } catch (error) {
+      logger.error('ChatHistoryLocalStore queryRecordList error', error);
+      return {
+        total: 0,
+        records: []
+      };
+    }
+  }
+  async update(id, body) {
+    if (typeof id !== 'number') {
+      throw new Error('query id is not number');
+    }
+    const stored = await this.query(id);
+    if (!stored) {
+      throw new Error("could not find message: ".concat(id));
+    }
+    const nowTime = Date.now();
+    const msgObject = {
+      message: body,
+      createdAt: stored.createdAt,
+      updatedAt: nowTime
+    };
+    stored.message.meta.updatedAt = nowTime;
+    return updateRecord({
+      id,
+      data: JSON.stringify(msgObject)
+    });
+  }
+  async remove(id) {
+    if (typeof id !== 'number') {
+      throw new Error('query id is not number');
+    }
+    const stored = await this.query(id);
+    if (!stored) {
+      logger.warn('ChatHistoryLocalStore not find by id', {
+        id
+      });
+    }
+    await deleteRecordList({
+      id: [id]
+    });
+  }
+  async removeAll(ids) {
+    if (ids) {
+      if (!Array.isArray(ids)) {
+        throw new Error('ids is not array');
+      }
+      if (ids.length === 0) {
+        return;
+      }
+      await deleteRecordList(_objectSpread({
+        id: ids
+      }, this.getQueryCondition()));
+    } else {
+      await deleteRecordList(this.getQueryCondition());
+    }
+  }
+  async insert(message) {
+    const nowTime = Date.now();
+
+    // 先把 id 提取出来不存，id 始终是在数据库里自增的
+    const _message$meta = message.meta,
+      {
+        id
+      } = _message$meta,
+      meta = _objectWithoutProperties(_message$meta, _excluded);
+    const msgObject = {
+      message: _objectSpread(_objectSpread({}, message), {}, {
+        meta: _objectSpread(_objectSpread(_objectSpread({}, meta), this.options), {}, {
+          createdAt: nowTime,
+          updatedAt: nowTime
+        })
+      }),
+      createdAt: nowTime,
+      updatedAt: nowTime
+    };
+    return insertRecord(_objectSpread(_objectSpread({}, this.getRecordIndex()), {}, {
+      data: JSON.stringify(msgObject)
+    }));
+  }
+}

package/dist/ChatHistoryStore.d.ts

@@ -10,7 +10,7 @@ export interface StoredMessageObject {
     homeId?: number;
     indexId?: string;
 }
-interface Pagination {
+export interface ChatHistoryStorePagination {
     sort?: 'asc' | 'desc';
     offset: number;
     limit: number;
@@ -30,7 +30,7 @@ export interface ChatHistoryStore<Key = number> {
     /** 查询消息 */
     query(id: Key): Promise<StoredMessageObject>;
     /** 批量查询 */
-    queryAll(pagination: Pagination): Promise<{
+    queryAll(pagination: ChatHistoryStorePagination): Promise<{
         total: number;
         records: StoredMessageObject[];
     }>;
@@ -45,24 +45,3 @@ export interface ChatHistoryStore<Key = number> {
         id: Key;
     }>;
 }
-export declare class ChatHistoryLocalStore implements ChatHistoryStore {
-    /** 版本号，会作为内部索引的一部分 */
-    private readonly version;
-    private options;
-    constructor(options: ChatHistoryStoreOptions);
-    private itemToMessage;
-    private getRecordIndex;
-    private getQueryCondition;
-    query(id: number): Promise<StoredMessageObject | null>;
-    queryAll(pagination: Pagination): Promise<{
-        total: number;
-        records: StoredMessageObject[];
-    }>;
-    update(id: number, body: ChatMessageObject): Promise<void>;
-    remove(id: number): Promise<void>;
-    removeAll(ids?: number[]): Promise<void>;
-    insert(message: ChatMessageObject): Promise<{
-        id: number;
-    }>;
-}
-export {};

package/dist/ChatHistoryStore.js

@@ -1,191 +1 @@
-import _objectWithoutProperties from "@babel/runtime/helpers/esm/objectWithoutProperties";
-import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-const _excluded = ["id"];
-import "core-js/modules/es.array.sort.js";
-import "core-js/modules/es.json.stringify.js";
-import "core-js/modules/esnext.iterator.constructor.js";
-import "core-js/modules/esnext.iterator.filter.js";
-import "core-js/modules/esnext.iterator.map.js";
-import { safeParseJSON } from '@ray-js/t-agent';
-import { insertRecord, deleteRecordList, updateRecord, queryRecordList } from './utils';
-import logger from './utils/logger';
-export class ChatHistoryLocalStore {
-  constructor(options) {
-    /** 版本号，会作为内部索引的一部分 */
-    _defineProperty(this, "version", 'v1');
-    if (!options.agentId) {
-      throw new Error('agentId is required');
-    }
-    if (!options.bizCode) {
-      throw new Error('indexId is required');
-    }
-    this.options = _objectSpread({
-      indexId: 'default',
-      homeId: 0
-    }, options);
-  }
-  itemToMessage(item) {
-    const messageInfo = safeParseJSON(item.data);
-    if (!messageInfo) {
-      logger.error('ChatHistoryLocalStore failed parse item: ', item);
-      return null;
-    }
-    messageInfo.message.meta.id = item.id;
-    return {
-      id: item.id,
-      indexId: item.index2,
-      agentId: item.solutionCode,
-      deviceId: item.devId,
-      homeId: item.homeId,
-      createdAt: messageInfo.createdAt,
-      updatedAt: messageInfo.updatedAt,
-      message: messageInfo.message
-    };
-  }
-  getRecordIndex() {
-    return {
-      bizCode: this.options.bizCode,
-      solutionCode: this.options.agentId,
-      homeId: this.options.homeId,
-      index: "t-agent-".concat(this.version),
-      index2: this.options.indexId,
-      deviceId: this.options.deviceId
-    };
-  }
-  getQueryCondition() {
-    const indexes = this.getRecordIndex();
-    const params = {
-      bizCode: [indexes.bizCode],
-      solutionCode: [indexes.solutionCode],
-      homeId: [indexes.homeId],
-      index: [indexes.index],
-      index2: [indexes.index2]
-    };
-    if (indexes.deviceId) {
-      params.devId = [indexes.deviceId];
-    }
-    return params;
-  }
-  async query(id) {
-    if (typeof id !== 'number') {
-      logger.warn('ChatHistoryLocalStore query id is not number', {
-        id
-      });
-      return null;
-    }
-    const queryCondition = _objectSpread(_objectSpread({}, this.getQueryCondition()), {}, {
-      id: [id],
-      offset: 0,
-      limit: 1,
-      sortType: 1
-    });
-    const {
-      records
-    } = await queryRecordList(queryCondition);
-    return records[0] ? this.itemToMessage(records[0]) : null;
-  }
-  async queryAll(pagination) {
-    const queryCondition = _objectSpread(_objectSpread({}, this.getQueryCondition()), {}, {
-      offset: 0,
-      sortType: 1
-    });
-    const {
-      offset,
-      limit
-    } = pagination || {};
-    if (offset) {
-      queryCondition.offset = offset;
-    }
-    if (limit) {
-      queryCondition.limit = limit;
-    }
-    queryCondition.sortType = pagination.sort === 'desc' ? 0 : 1;
-    try {
-      const result = await queryRecordList(queryCondition);
-      const list = result.records.map(this.itemToMessage).filter(item => !!item);
-      return {
-        total: result.total,
-        records: list
-      };
-    } catch (error) {
-      logger.error('ChatHistoryLocalStore queryRecordList error', error);
-      return {
-        total: 0,
-        records: []
-      };
-    }
-  }
-  async update(id, body) {
-    if (typeof id !== 'number') {
-      throw new Error('query id is not number');
-    }
-    const stored = await this.query(id);
-    if (!stored) {
-      throw new Error("could not find message: ".concat(id));
-    }
-    const nowTime = Date.now();
-    const msgObject = {
-      message: body,
-      createdAt: stored.createdAt,
-      updatedAt: nowTime
-    };
-    stored.message.meta.updatedAt = nowTime;
-    return updateRecord({
-      id,
-      data: JSON.stringify(msgObject)
-    });
-  }
-  async remove(id) {
-    if (typeof id !== 'number') {
-      throw new Error('query id is not number');
-    }
-    const stored = await this.query(id);
-    if (!stored) {
-      logger.warn('ChatHistoryLocalStore not find by id', {
-        id
-      });
-    }
-    await deleteRecordList({
-      id: [id]
-    });
-  }
-  async removeAll(ids) {
-    if (ids) {
-      if (!Array.isArray(ids)) {
-        throw new Error('ids is not array');
-      }
-      if (ids.length === 0) {
-        return;
-      }
-      await deleteRecordList(_objectSpread({
-        id: ids
-      }, this.getQueryCondition()));
-    } else {
-      await deleteRecordList(this.getQueryCondition());
-    }
-  }
-  async insert(message) {
-    const nowTime = Date.now();
-
-    // 先把 id 提取出来不存，id 始终是在数据库里自增的
-    const _message$meta = message.meta,
-      {
-        id
-      } = _message$meta,
-      meta = _objectWithoutProperties(_message$meta, _excluded);
-    const msgObject = {
-      message: _objectSpread(_objectSpread({}, message), {}, {
-        meta: _objectSpread(_objectSpread(_objectSpread({}, meta), this.options), {}, {
-          createdAt: nowTime,
-          updatedAt: nowTime
-        })
-      }),
-      createdAt: nowTime,
-      updatedAt: nowTime
-    };
-    return insertRecord(_objectSpread(_objectSpread({}, this.getRecordIndex()), {}, {
-      data: JSON.stringify(msgObject)
-    }));
-  }
-}
+export {};

package/dist/buildIn/withBuildIn.js

@@ -41,41 +41,18 @@ export function withBuildIn() {
 
     // 关联文档
 
+    // 关联智能家居卡片
     (() => {
       onSkillsEnd((skills, responseMessage) => {
         if (!responseMessage) {
           return;
         }
-        const data = {
-          documents: []
+        const operateData = {
+          deviceInfo: [],
+          sceneInfo: [],
+          changeInfo: []
         };
-        for (const skill of skills) {
-          var _content$custom;
-          if (skill.code !== BuildInSkillCode.SEARCH_KNOWLEDGE) {
-            continue;
-          }
-          const content = skill;
-          if (!((_content$custom = content.custom) !== null && _content$custom !== void 0 && (_content$custom = _content$custom.data) !== null && _content$custom !== void 0 && _content$custom.documents)) {
-            continue;
-          }
-          for (const doc of content.custom.data.documents) {
-            data.documents.push({
-              title: doc.title,
-              url: doc.url
-            });
-          }
-        }
-        if (data.documents.length) {
-          responseMessage.bubble.addTile('documents', data);
-        }
-      });
-    })();
-    (() => {
-      onSkillsEnd((skills, responseMessage) => {
-        if (!responseMessage) {
-          return;
-        }
-        const data = {
+        const executeData = {
           deviceInfo: [],
           sceneInfo: [],
           changeInfo: []
@@ -87,7 +64,7 @@ export function withBuildIn() {
           const content = skill;
           if (content.general.action === ReceivedSmartHomeSkillAction.QUERY_SCENE) {
             for (const scene of content.general.data.scenes) {
-              data.sceneInfo.push({
+              executeData.sceneInfo.push({
                 type: scene.type,
                 enabled: scene.enable,
                 icon: scene.icon,
@@ -101,7 +78,7 @@ export function withBuildIn() {
             }
           } else if (content.general.action === ReceivedSmartHomeSkillAction.CONTROL_SCENE) {
             for (const scene of content.general.data.scenes) {
-              data.sceneInfo.push({
+              operateData.sceneInfo.push({
                 type: scene.type,
                 icon: scene.icon,
                 name: scene.name,
@@ -114,7 +91,7 @@ export function withBuildIn() {
             }
           } else if (content.general.action === ReceivedSmartHomeSkillAction.QUERY_DEVICE) {
             for (const dev of content.general.data.devices) {
-              data.deviceInfo.push({
+              executeData.deviceInfo.push({
                 icon: dev.icon,
                 name: dev.name,
                 deviceId: dev.deviceId,
@@ -125,7 +102,7 @@ export function withBuildIn() {
             }
           } else if (content.general.action === ReceivedSmartHomeSkillAction.CONTROL_DEVICE) {
             for (const dev of content.general.data.devices) {
-              data.deviceInfo.push({
+              operateData.deviceInfo.push({
                 icon: dev.icon,
                 name: dev.name,
                 deviceId: dev.deviceId,
@@ -135,8 +112,41 @@ export function withBuildIn() {
             }
           }
         }
-        if (data.deviceInfo.length) {
-          responseMessage.bubble.addTile('operateCard', data);
+        if (executeData.deviceInfo.length || executeData.sceneInfo.length || executeData.changeInfo.length) {
+          responseMessage.bubble.addTile('executeCard', executeData);
+        }
+        if (operateData.deviceInfo.length || operateData.sceneInfo.length || operateData.changeInfo.length) {
+          responseMessage.bubble.addTile('operateCard', operateData);
+        }
+      });
+    })();
+    // 最后是关联文档
+    (() => {
+      onSkillsEnd((skills, responseMessage) => {
+        if (!responseMessage) {
+          return;
+        }
+        const data = {
+          documents: []
+        };
+        for (const skill of skills) {
+          var _content$custom;
+          if (skill.code !== BuildInSkillCode.SEARCH_KNOWLEDGE) {
+            continue;
+          }
+          const content = skill;
+          if (!((_content$custom = content.custom) !== null && _content$custom !== void 0 && (_content$custom = _content$custom.data) !== null && _content$custom !== void 0 && _content$custom.documents)) {
+            continue;
+          }
+          for (const doc of content.custom.data.documents) {
+            data.documents.push({
+              title: doc.title,
+              url: doc.url
+            });
+          }
+        }
+        if (data.documents.length) {
+          responseMessage.bubble.addTile('documents', data);
         }
       });
     })();

package/dist/index.d.ts

@@ -3,3 +3,4 @@ export * from './AIStreamTypes';
 export * from './withAIStream';
 export * from './ChatHistoryStore';
 export * from './buildIn';
+export * from './ChatHistoryLocalStore';

package/dist/index.js

@@ -2,4 +2,5 @@ export * from './utils';
 export * from './AIStreamTypes';
 export * from './withAIStream';
 export * from './ChatHistoryStore';
-export * from './buildIn';
+export * from './buildIn';
+export * from './ChatHistoryLocalStore';

package/dist/utils/AIStream.d.ts

@@ -86,7 +86,6 @@ type AIStreamEventWriteChunk = {
 type AIStreamEventSource = {
     type: 'audio';
     dataChannel?: string;
-    amplitudeCount?: number;
     userData?: Attribute[];
 } | {
     type: 'video';

package/dist/utils/AIStream.js

@@ -8,7 +8,7 @@ import "core-js/modules/esnext.iterator.for-each.js";
 import "core-js/modules/esnext.iterator.map.js";
 import "core-js/modules/web.dom-collections.iterator.js";
 import { AIStreamAttributeType, AIStreamErrorCode, AIStreamServerErrorCode, BizTag, ConnectClientType, ConnectState, EventType, NetworkType, SessionState } from '../AIStreamTypes';
-import { closeSession, connect, createSession, disconnect, getCurrentHomeInfo, getNetworkType, isConnected, queryAgentToken, registerRecordAmplitudes, sendEventChatBreak, sendEventEnd, sendEventPayloadEnd, sendEventStart, sendImageData, sendTextData, startRecordAndSendAudioData, stopRecordAndSendAudioData, unregisterVoiceAmplitudes } from './ttt';
+import { closeSession, connect, createSession, disconnect, getCurrentHomeInfo, getNetworkType, isConnected, queryAgentToken, sendEventChatBreak, sendEventEnd, sendEventPayloadEnd, sendEventStart, sendImageData, sendTextData, startRecordAndSendAudioData, stopRecordAndSendAudioData } from './ttt';
 import { AIStreamObserver, AIStreamObserverPool } from './observer';
 import { isAbortError } from '@ray-js/t-agent';
 import logger from './logger';
@@ -456,11 +456,6 @@ export class AIStreamEvent {
         }
         stream.started = true;
         if (source.type === 'audio') {
-          if (source.amplitudeCount) {
-            registerRecordAmplitudes({
-              count: source.amplitudeCount
-            });
-          }
           startPromise = startRecordAndSendAudioData({
             sessionId: this.sessionId,
             dataChannel,
@@ -486,11 +481,6 @@ export class AIStreamEvent {
           startPromise = null;
         }
         if (source.type === 'audio') {
-          if (source.amplitudeCount) {
-            unregisterVoiceAmplitudes({
-              count: source.amplitudeCount
-            });
-          }
           stopRecordAndSendAudioData({
             sessionId: this.sessionId,
             dataChannel,

package/dist/utils/sendMessage.js

@@ -22,14 +22,12 @@ export function sendBlocksToAIStream(params) {
     signal
   } = params;
   let audioEmitter = null;
-  let amplitudeCount = 0;
   for (const block of blocks) {
     if (block.type === 'audio') {
       if (audioEmitter) {
         throw new Error('only one audio emitter is allowed');
       }
       audioEmitter = block.audio_emitter;
-      amplitudeCount = block.amplitude_count || 0;
     }
   }
   const attribute = _objectSpread({
@@ -144,11 +142,13 @@ export function sendBlocksToAIStream(params) {
           }
           const packet = safeParseJSON(data.body.text);
           if (packet.bizType === ReceivedTextPacketType.NLG) {
-            logger.debug('sendBlocksToAIStream Receive NLG', packet.data.content);
-            const text = prevText + packet.data.content;
+            // TODO: 处理 reasoningContent 推理能力
+            const delta = packet.data.content || '';
+            logger.debug('sendBlocksToAIStream Receive NLG', delta);
+            const text = prevText + delta;
             enqueue({
               type: 'text',
-              delta: packet.data.content,
+              delta,
               text,
               meta
             });
@@ -291,8 +291,7 @@ export function sendBlocksToAIStream(params) {
       if (audioEmitter) {
         await new Promise(resolve => {
           const s = event.stream({
-            type: 'audio',
-            amplitudeCount
+            type: 'audio'
           });
           audioEmitter.addEventListener('confirm', async () => {
             if (!canceled) {

package/dist/withAIStream.d.ts

@@ -1,6 +1,7 @@
-import { ChatAgent, ChatMessage, GetChatPluginHandler, InputBlock } from '@ray-js/t-agent';
+import { ChatAgent, ChatCardObject, ChatMessage, ChatTile, GetChatPluginHandler, InputBlock } from '@ray-js/t-agent';
+import { TTTAction } from './utils';
 import { ConnectClientType, ReceivedTextSkillPacketBody } from './AIStreamTypes';
-import { ChatHistoryLocalStore, StoredMessageObject } from './ChatHistoryStore';
+import { ChatHistoryStore, StoredMessageObject } from './ChatHistoryStore';
 export interface AIStreamOptions {
     /** client 类型: 1-作为设备代理, 2-作为 App */
     clientType?: ConnectClientType;
@@ -24,8 +25,8 @@ export interface AIStreamOptions {
     homeId?: number;
     /** 是否在 onAgentStart 阶段就建立连接 */
     earlyStart?: boolean;
-    /** 自定义消息存储, 返回的实例需要实现 ChatHistoryLocalStore 接口, 返回null则不存储历史聊天记录 */
-    createChatHistoryStore?: (agent: ChatAgent) => ChatHistoryLocalStore | null;
+    /** 自定义消息存储, 返回的实例需要实现 ChatHistoryStore 接口, 返回null则不存储历史聊天记录 */
+    createChatHistoryStore?: (agent: ChatAgent) => ChatHistoryStore | null;
     /** 是否开启音频合成 */
     enableTts?: boolean;
 }
@@ -40,9 +41,15 @@ export interface AIStreamHooks {
     onSkillsEnd: (skills: ReceivedTextSkillPacketBody[], respMsg: ChatMessage, result: {
         messages: ChatMessage[];
     }) => void;
+    onTTTAction: (tile: ChatTile | null, result: {
+        action?: TTTAction;
+    }) => void;
+    onCardsReceived: (skills: ReceivedTextSkillPacketBody[], result: {
+        cards: ChatCardObject[];
+    }) => void;
 }
 export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAgent) => {
-    hooks: import("hookable").Hookable<any, string>;
+    hooks: import("hookable").Hookable<AIStreamHooks, import("hookable").HookKeys<AIStreamHooks>>;
     aiStream: {
         send: (blocks: InputBlock[], signal?: AbortSignal, extraOptions?: Record<string, any>) => {
             response: import("@ray-js/t-agent").StreamResponse;
@@ -56,8 +63,6 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
         options: AIStreamOptions;
         removeMessage: (message: ChatMessage) => Promise<void>;
         clearAllMessages: () => Promise<void>;
-        onSkillCompose: (fn: AIStreamHooks['onSkillCompose']) => () => void;
-        onSkillsEnd: (fn: AIStreamHooks['onSkillsEnd']) => () => void;
         feedback: ({ requestId, type }: {
             requestId: string;
             type: string;
@@ -65,5 +70,9 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
             thingjson?: any;
             data: string;
         }>;
+        onSkillCompose: (fn: AIStreamHooks['onSkillCompose']) => () => void;
+        onSkillsEnd: (fn: AIStreamHooks['onSkillsEnd']) => () => void;
+        onCardsReceived: (fn: AIStreamHooks['onCardsReceived']) => () => void;
+        onTTTAction: (fn: AIStreamHooks['onTTTAction']) => () => void;
     };
 };

package/dist/withAIStream.js

@@ -11,9 +11,9 @@ import { BubbleTileStatus, ChatMessageStatus, createHooks, EmitterEvent } from '
 import { messageAppraise } from './utils/apis';
 import { getAccountInfo, getCurrentHomeInfo, runTTTAction, sendBlocksToAIStream } from './utils';
 import { BizCode, ConnectClientType } from './AIStreamTypes';
-import { ChatHistoryLocalStore } from './ChatHistoryStore';
 import { DEFAULT_TOKEN_API, DEFAULT_TOKEN_API_VERSION, globalAIStreamClient } from './global';
 import logger from './utils/logger';
+import { ChatHistoryLocalStore } from './ChatHistoryLocalStore';
 export function withAIStream() {
   let options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
   const hooks = createHooks();
@@ -38,11 +38,24 @@ export function withAIStream() {
       const {
         deviceId,
         agentId,
+        clientType = ConnectClientType.APP,
         tokenOptions = {
           api: DEFAULT_TOKEN_API,
           version: DEFAULT_TOKEN_API_VERSION
         }
       } = options;
+      if (!agentId) {
+        throw new Error('agentId is required');
+      }
+      if (!tokenOptions.api || !tokenOptions.version) {
+        throw new Error('tokenOptions.api and tokenOptions.version are required');
+      }
+      if (clientType === ConnectClientType.DEVICE && !deviceId) {
+        throw new Error('deviceId is required when clientType is device');
+      }
+      if (clientType === ConnectClientType.APP && deviceId) {
+        throw new Error('deviceId is not allowed when clientType is app');
+      }
       let homeId = options.homeId;
       if (!homeId) {
         const info = await getCurrentHomeInfo();
@@ -56,7 +69,7 @@ export function withAIStream() {
       /*
        * 2. 获取历史消息
        */
-      let historyStore = null;
+      let historyStore;
       if (typeof options.createChatHistoryStore === 'function') {
         historyStore = options.createChatHistoryStore(agent);
       } else {
@@ -87,21 +100,18 @@ export function withAIStream() {
 
       // 创建 streamSession
       const connection = globalAIStreamClient.getConnection({
-        clientType: options.clientType || ConnectClientType.APP,
-        deviceId: options.deviceId
+        clientType,
+        deviceId
       });
-      if (options.clientType === ConnectClientType.DEVICE && !options.deviceId) {
-        throw new Error('deviceId is required when clientType is device');
-      }
       const streamSession = connection.createSession({
-        ownerId: options.clientType === ConnectClientType.DEVICE ? options.deviceId : "".concat(homeId),
+        ownerId: clientType === ConnectClientType.DEVICE ? deviceId : "".concat(homeId),
         api: tokenOptions.api,
         apiVersion: tokenOptions.version,
         solutionCode: agentId,
-        extParams: _objectSpread(_objectSpread({}, tokenOptions.extParams), {}, {
+        extParams: _objectSpread({
           needTts: !!options.enableTts,
-          deviceId: options.deviceId || undefined
-        })
+          deviceId
+        }, tokenOptions.extParams)
       });
       await session.set('AIStream.streamSession', streamSession);
       if (options.earlyStart) {
@@ -331,12 +341,24 @@ export function withAIStream() {
         await hooks.callHook('onSkillsEnd', skills, message, result);
         await message.update();
       }
+      let valid = false;
       if (message.bubble.text) {
+        valid = true;
         await message.persist();
       } else if (message.bubble.status === BubbleTileStatus.NORMAL) {
-        await message.remove();
+        valid = false;
       } else {
+        valid = true;
+      }
+      if (valid) {
         await message.persist();
+        const [_, ...rest] = result.messages;
+        for (const m of rest) {
+          await m.show();
+          await m.persist();
+        }
+      } else {
+        await message.remove();
       }
       return [userMsg, ...result.messages];
     };
@@ -452,6 +474,43 @@ export function withAIStream() {
         }
       }
     });
+    const onSkillsEnd = fn => {
+      return hooks.hook('onSkillsEnd', fn);
+    };
+    onSkillsEnd(async (skills, respMsg, result) => {
+      const cards = [];
+      for (const skill of skills) {
+        var _skill$custom, _skill$general;
+        if ((_skill$custom = skill.custom) !== null && _skill$custom !== void 0 && (_skill$custom = _skill$custom.data) !== null && _skill$custom !== void 0 && _skill$custom.aiCards) {
+          for (const card of skill.custom.data.aiCards) {
+            cards.push(card);
+          }
+        }
+        if ((_skill$general = skill.general) !== null && _skill$general !== void 0 && (_skill$general = _skill$general.data) !== null && _skill$general !== void 0 && _skill$general.aiCards) {
+          for (const card of skill.custom.data.aiCards) {
+            cards.push(card);
+          }
+        }
+      }
+      const r = {
+        cards: cards
+      };
+      await hooks.callHook('onCardsReceived', skills, r);
+      for (const card of cards) {
+        const m = createMessage({
+          role: respMsg.role,
+          status: ChatMessageStatus.FINISH,
+          meta: {
+            eventId: respMsg.meta.eventId,
+            sessionId: respMsg.meta.sessionId
+          }
+        });
+        m.addTile('card', {
+          card
+        });
+        result.messages.push(m);
+      }
+    });
     return {
       hooks,
       aiStream: {
@@ -469,13 +528,17 @@ export function withAIStream() {
             await historyStore.removeAll();
           }
         },
+        feedback,
         onSkillCompose: fn => {
           return hooks.hook('onSkillCompose', fn);
         },
-        onSkillsEnd: fn => {
-          return hooks.hook('onSkillsEnd', fn);
+        onSkillsEnd,
+        onCardsReceived: fn => {
+          return hooks.hook('onCardsReceived', fn);
         },
-        feedback
+        onTTTAction: fn => {
+          return hooks.hook('onTTTAction', fn);
+        }
       }
     };
   };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.2-beta-1",
+  "version": "0.2.3-beta-1",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -35,5 +35,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "0dd513ad93a351ad5c00a1ee8993c7b62f03640f"
+  "gitHead": "ed9810075246e1261af2fdaa41550070313a98dc"
 }