@ray-js/t-agent-plugin-aistream 0.2.0-beta-1

package/dist/withAIStream.js

@@ -0,0 +1,420 @@
+import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
+import _objectWithoutProperties from "@babel/runtime/helpers/esm/objectWithoutProperties";
+const _excluded = ["message"];
+import "core-js/modules/es.array.flat.js";
+import "core-js/modules/es.array.unscopables.flat.js";
+import "core-js/modules/esnext.iterator.constructor.js";
+import "core-js/modules/esnext.iterator.for-each.js";
+import "core-js/modules/esnext.iterator.map.js";
+import "core-js/modules/web.dom-collections.iterator.js";
+import { BubbleTileStatus, ChatMessageStatus, createHooks } from '@ray-js/t-agent';
+import { getAccountInfo } from './utils/ttt';
+import { tAgentMessageAppraise } from './utils/apis';
+import { AIStreamObserver, getCurrentHomeInfo, sendBlocksToAIStream } from './utils';
+import { ConnectClientType, ConnectState } from './AIStreamTypes';
+import { ChatHistoryLocalStore, ChatHistoryStore } from './ChatHistoryStore';
+import { DEFAULT_TOKEN_API, DEFAULT_TOKEN_API_VERSION, globalAIStreamClient } from './global';
+import { runTTTAction } from '@ray-js/t-agent-plugin-assistant';
+import logger from './utils/logger';
+export function withAIStream() {
+  let options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
+  const hooks = createHooks();
+  return agent => {
+    const {
+      onAgentStart,
+      onChatStart,
+      onChatResume,
+      onInputBlocksPush,
+      onMessagePersist,
+      onMessageChange,
+      session,
+      createMessage,
+      onAgentDispose,
+      onTileEvent
+    } = agent;
+    const ui = agent.plugins.ui;
+    onAgentStart(async () => {
+      /**
+       * 1. 管理配置数据
+       */
+      const {
+        deviceId,
+        agentId,
+        tokenApi,
+        tokenApiVersion
+      } = options;
+      let homeId = options.homeId;
+      if (!homeId) {
+        const info = await getCurrentHomeInfo();
+        homeId = +info.homeId;
+      }
+      const indexId = options.indexId || 'default';
+      await agent.session.set('AIStream.indexId', indexId);
+      await agent.session.set('AIStream.agentId', agentId);
+      await agent.session.set('AIStream.deviceId', deviceId);
+
+      /*
+       * 2. 获取历史消息
+       */
+      let historyStore = null;
+      if (typeof options.createChatHistoryStore === 'function') {
+        historyStore = options.createChatHistoryStore(agent);
+        if (!(historyStore instanceof ChatHistoryStore)) {
+          throw new Error('createChatHistoryStore must return an instance of ChatHistoryStore');
+        }
+      } else {
+        historyStore = new ChatHistoryLocalStore({
+          agentId,
+          deviceId,
+          homeId,
+          indexId
+        });
+      }
+      await agent.session.set('AIStream.historyStore', historyStore);
+      const {
+        records: historyMsgList
+      } = await historyStore.queryBy({
+        solutionCode: options.agentId ? [options.agentId] : null,
+        devId: options.deviceId ? [options.deviceId] : null,
+        homeId: [homeId],
+        index: [indexId]
+      }, {
+        sort: 'asc',
+        pageSize: options.historySize || 1000,
+        pageNo: 0
+      });
+      await session.set('AIStream.rawMessages', historyMsgList);
+      session.isNewChat = historyMsgList.length === 0;
+      session.sessionId = indexId;
+
+      // 创建 streamSession
+      const connection = globalAIStreamClient.getConnection({
+        clientType: options.clientType || ConnectClientType.APP,
+        deviceId: options.deviceId
+      });
+      const streamSession = connection.createSession({
+        api: tokenApi || DEFAULT_TOKEN_API,
+        apiVersion: tokenApiVersion || DEFAULT_TOKEN_API_VERSION,
+        solutionCode: agentId
+      });
+      await session.set('AIStream.streamSession', streamSession);
+      const observer = new AIStreamObserver(data => {
+        if (data.type === 'amplitudes') {
+          ui.emitEvent('amplitudes', data);
+        } else if (data.type === 'connectionState') {
+          ui.emitEvent('networkChange', {
+            online: data.body.connectState === ConnectState.CONNECTED
+          });
+        }
+      }, globalAIStreamClient.pool);
+      observer.observe({
+        connectionState: true,
+        amplitudes: true
+      });
+      await session.set('AIStream.observer', observer);
+    });
+    onAgentDispose(async () => {
+      const observer = session.get('AIStream.observer');
+      if (observer) {
+        observer.disconnect();
+      }
+      const streamSession = session.get('AIStream.streamSession');
+
+      /**
+       * 关闭session
+       */
+      await streamSession.close();
+    });
+    const persist = async message => {
+      console.log('persist', message);
+      // 对消息进行持久化，已存在的数据进行更新
+      const historyStore = await session.get('AIStream.historyStore');
+      const storeId = message.meta.id;
+      const messageObj = message.toObject();
+      if (storeId) {
+        await historyStore.update(storeId, messageObj);
+        if (message.isShow) {
+          await message.update();
+        }
+      } else {
+        delete messageObj.id;
+        const insertRes = await historyStore.insert(messageObj);
+        if (insertRes && insertRes.id) {
+          // 把 id 设置到 meta上，标识已经持久化
+          message.setMeta({
+            id: insertRes.id
+          });
+        }
+      }
+    };
+    onMessagePersist((payload, message) => persist(message));
+    onChatResume(async result => {
+      /** 格式化历史消息 */
+      const list = await session.get('AIStream.rawMessages');
+      if (list && list.length > 0) {
+        const messages = await Promise.all(list.map(async item => {
+          const {
+              message
+            } = item,
+            messageMeta = _objectWithoutProperties(item, _excluded);
+          const messageObj = createMessage(message);
+          messageObj.setMeta(messageMeta);
+          const messages = [];
+          if (messageObj.tiles.length) {
+            messages.push(messageObj);
+          }
+          const result = {
+            messages
+          };
+          await hooks.callHook('onMessageParse', item, result);
+          return result.messages;
+        }));
+        result.messages = messages.flat();
+      }
+    }, 'before');
+    const removeMessage = async message => {
+      const storeId = message.meta.id;
+      if (!storeId) {
+        return;
+      }
+      const historyStore = await session.get('AIStream.historyStore');
+      await historyStore.remove(storeId);
+    };
+    onMessageChange(async (type, message) => {
+      if (type === 'remove') {
+        await removeMessage(message);
+      }
+    });
+    const composeHandler = {
+      attachmentCompose: async () => {
+        // if (part.attachmentType === 'extensions') {
+        //   await hooks.callHook('onExtensionCompose', part.attachment, respMsg, result, 'stream');
+        // }
+        return {
+          messages: []
+        }.messages;
+      }
+    };
+    const send = (blocks, signal, extraOptions) => {
+      const streamSession = session.get('AIStream.streamSession');
+      const result = sendBlocksToAIStream({
+        blocks,
+        session: streamSession,
+        attribute: _objectSpread({}, extraOptions)
+      });
+      signal === null || signal === void 0 || signal.addEventListener('abort', event => {
+        result === null || result === void 0 || result.response.cancel(event.reason);
+        agent.hooks.callHook('onUserAbort', event.reason);
+      });
+      signal === null || signal === void 0 || signal.throwIfAborted();
+      return result;
+    };
+    const chat = async (blocks, signal, options) => {
+      const {
+        sendBy = 'user',
+        responseBy = 'assistant',
+        extraOptions
+      } = options || {};
+      let audioEmitter = null;
+      for (const block of blocks) {
+        if (block.type === 'audio') {
+          if (audioEmitter) {
+            throw new Error('only one audio emitter is allowed');
+          }
+          audioEmitter = block.audio_emitter;
+        }
+      }
+
+      // 创建用户消息
+      const userMsg = createMessage({
+        role: sendBy,
+        status: ChatMessageStatus.START
+      });
+      let audioPromise = null;
+      if (audioEmitter) {
+        let end = false;
+        audioPromise = new Promise((resolve, reject) => {
+          // 当确认发送时，展示 loading
+          const onConfirm = async () => {
+            if (end) {
+              return;
+            }
+            userMsg.set({
+              status: ChatMessageStatus.START
+            });
+            userMsg.bubble.setStatus(BubbleTileStatus.NORMAL);
+            await userMsg.show();
+            audioEmitter.removeEventListener('confirm', onConfirm);
+          };
+          audioEmitter.addEventListener('confirm', onConfirm);
+
+          // 当音频识别完毕时，展示文本
+          const onFinished = async event => {
+            if (end) {
+              return;
+            }
+            if (!event.detail.text) {
+              await userMsg.remove();
+              reject(new Error('No text found in audio event'));
+              return;
+            }
+            userMsg.bubble.initWithInputBlocks(blocks);
+            userMsg.bubble.setText(event.detail.text);
+            userMsg.set({
+              status: ChatMessageStatus.FINISH
+            });
+            await userMsg.update();
+            await userMsg.persist();
+            end = true;
+            audioEmitter.removeEventListener('finish', onFinished);
+            resolve();
+          };
+          audioEmitter.addEventListener('finish', onFinished);
+          const onCancel = () => {
+            end = true;
+            audioEmitter.removeEventListener('cancel', onCancel);
+            reject(new Error('User cancel'));
+          };
+          audioEmitter.addEventListener('cancel', onCancel);
+        });
+      } else {
+        userMsg.set({
+          status: ChatMessageStatus.FINISH
+        });
+        userMsg.bubble.initWithInputBlocks(blocks);
+        await userMsg.show();
+        await userMsg.persist();
+      }
+      const {
+        response,
+        metaPromise
+      } = send(blocks, signal, extraOptions);
+      if (audioPromise) {
+        try {
+          await audioPromise;
+        } catch (e) {
+          // 没有识别到文字
+          return [];
+        }
+      }
+      metaPromise.then(meta => userMsg.setMeta(meta).update());
+      const message = createMessage({
+        role: responseBy
+      });
+      const messages = await agent.flushStreamToShow(message, response, composeHandler);
+      if (message.bubble.status === BubbleTileStatus.ABORTED) {
+        if (!message.bubble.text) {
+          await message.remove();
+        }
+      } else {
+        await message.persist();
+      }
+      return [userMsg, ...messages];
+    };
+    onInputBlocksPush(chat);
+    const handleTTTAction = async (tile, tttAction) => {
+      const result = {
+        action: tttAction
+      };
+      await hooks.callHook('onTTTAction', tile, result);
+      if (result.action) {
+        await runTTTAction(result.action, {
+          onSendSkills: async () => {
+            throw new Error('send skill not support');
+          },
+          onSendMessages: async (blocks, sendImmediately) => {
+            if (sendImmediately) {
+              ui.emitEvent('sendMessage', {
+                blocks
+              });
+            } else {
+              ui.emitEvent('setInputBlocks', {
+                blocks
+              });
+            }
+          },
+          onBuildIn(name, data) {
+            if (name.startsWith('ty.')) {
+              const fn = name.slice(3);
+              if (ty[fn]) {
+                ty[fn](data);
+              } else {
+                logger.error('handleTTTAction onBuildIn can not find TTT method:', name);
+              }
+            } else {
+              logger.error('handleTTTAction onBuildIn dont support:', name);
+            }
+          }
+        });
+      }
+    };
+    onTileEvent(async (tile, payload) => {
+      if (payload !== null && payload !== void 0 && payload.tttAction) {
+        await handleTTTAction(tile, payload.tttAction);
+      }
+    });
+    onTileEvent(async (tile, payload) => {
+      if (tile.type === 'card' && (payload === null || payload === void 0 ? void 0 : payload.type) === '@buildIn/setCardState') {
+        const {
+          state,
+          options
+        } = payload;
+        tile.data.card.cardState = state;
+        await tile.update();
+        if (options.persist) {
+          await tile.message.persist();
+        }
+      }
+    });
+    return {
+      hooks,
+      assistant: {
+        send,
+        chat,
+        composeHandler,
+        options,
+        removeMessage,
+        clearAllMessages: async () => {
+          // 删除内存中的消息
+          const messages = agent.session.messages.values();
+          Array.from(messages).forEach(message => {
+            message.remove();
+          });
+          if (options !== null && options !== void 0 && options.indexId) {
+            return;
+          }
+          // 清空缓存的数据
+          const historyStore = await session.get('AIStream.historyStore');
+          historyStore.removeBy({
+            index: [options.indexId]
+          });
+        },
+        feedback: async _ref => {
+          var _accountInfo$miniProg;
+          let {
+            requestId,
+            type
+          } = _ref;
+          const aiPtChannel = session.get('AIStream.aiPtChannel');
+          if (!aiPtChannel) {
+            logger.error('aiPtChannel is not found');
+            return null;
+          }
+          const accountInfo = await getAccountInfo();
+          const appId = accountInfo === null || accountInfo === void 0 || (_accountInfo$miniProg = accountInfo.miniProgram) === null || _accountInfo$miniProg === void 0 ? void 0 : _accountInfo$miniProg.appId;
+          if (!appId) {
+            logger.error('appId is not found');
+            return null;
+          }
+          return tAgentMessageAppraise({
+            aiPtChannel,
+            miniProgramId: appId,
+            requestId,
+            /** 评价（1-有帮助，2-无帮助） */
+            approveStatus: type === 'like' ? 1 : 2
+          });
+        }
+      }
+    };
+  };
+}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@ray-js/t-agent-plugin-aistream",
+  "version": "0.2.0-beta-1",
+  "author": "Tuya.inc",
+  "license": "MIT",
+  "private": false,
+  "main": "dist/index.js",
+  "typings": "dist/index.d.ts",
+  "maintainers": [
+    "tuya_npm",
+    "tuyafe"
+  ],
+  "files": [
+    "dist",
+    "README.md",
+    "README-zh_CN.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "dev": "ray start --type=component --output dist --emit-declaration-dev",
+    "build": "ray build --type=component --output dist",
+    "clean": "rimraf ./dist"
+  },
+  "dependencies": {
+    "dayjs": "^1.10.4",
+    "url-parse": "^1.5.10",
+    "web-streams-polyfill": "^4.0.0"
+  },
+  "peerDependencies": {
+    "@ray-js/ray": ">=1.6.8",
+    "@ray-js/t-agent": "*"
+  },
+  "devDependencies": {
+    "@types/url-parse": "^1.4.11"
+  },
+  "gitHead": "98229f8a9cb090a6fa0d9b47fa70b1f44ec2b34a"
+}