@ray-js/t-agent-plugin-aistream 0.2.8-beta.2 → 0.2.8-beta.4

package/dist/utils/observer.d.ts

@@ -45,6 +45,12 @@ export declare class AIStreamObserverPool {
     private observerMap;
     constructor();
     private cancels;
+    private createObserverMap;
+    private isSessionAwareType;
+    private getEntrySessionIds;
+    private collectObservers;
+    private addObserver;
+    private removeObserver;
     start(): void;
     dispose(): void;
     connect(observer: AIStreamObserver): void;

package/dist/utils/observer.js

@@ -10,25 +10,102 @@ export class AIStreamObserverPool {
   constructor() {
     _defineProperty(this, "isStarted", false);
     _defineProperty(this, "cancels", []);
-    this.observerMap = {
-      connectionState: new Set(),
-      sessionState: new Set(),
-      sessionEvent: new Set(),
-      event: new Set(),
-      text: new Set(),
-      audio: new Set(),
-      video: new Set(),
-      file: new Set(),
-      image: new Set()
+    this.observerMap = this.createObserverMap();
+  }
+  createObserverMap() {
+    return {
+      connectionState: {
+        all: new Set(),
+        bySessionId: new Map()
+      },
+      sessionState: {
+        all: new Set(),
+        bySessionId: new Map()
+      },
+      sessionEvent: {
+        all: new Set(),
+        bySessionId: new Map()
+      },
+      event: {
+        all: new Set(),
+        bySessionId: new Map()
+      },
+      text: {
+        all: new Set(),
+        bySessionId: new Map()
+      },
+      audio: {
+        all: new Set(),
+        bySessionId: new Map()
+      },
+      video: {
+        all: new Set(),
+        bySessionId: new Map()
+      },
+      file: {
+        all: new Set(),
+        bySessionId: new Map()
+      },
+      image: {
+        all: new Set(),
+        bySessionId: new Map()
+      }
     };
   }
+  isSessionAwareType(type) {
+    return type !== 'connectionState';
+  }
+  getEntrySessionIds(entry) {
+    if (entry.type === 'event' || entry.type === 'sessionEvent' || entry.type === 'sessionState') {
+      return entry.body.sessionId ? [entry.body.sessionId] : [];
+    }
+    if (entry.type === 'text' || entry.type === 'audio' || entry.type === 'video' || entry.type === 'file' || entry.type === 'image') {
+      return entry.body.sessionIdList || [];
+    }
+    return [];
+  }
+  collectObservers(entry) {
+    const bucket = this.observerMap[entry.type];
+    const observers = new Set(bucket.all);
+    this.getEntrySessionIds(entry).forEach(sessionId => {
+      const sessionObservers = bucket.bySessionId.get(sessionId);
+      sessionObservers === null || sessionObservers === void 0 || sessionObservers.forEach(observer => {
+        observers.add(observer);
+      });
+    });
+    return observers;
+  }
+  addObserver(type, observer) {
+    const sessionId = observer.options.sessionId;
+    const bucket = this.observerMap[type];
+    if (!sessionId || !this.isSessionAwareType(type)) {
+      bucket.all.add(observer);
+      return;
+    }
+    let sessionObservers = bucket.bySessionId.get(sessionId);
+    if (!sessionObservers) {
+      sessionObservers = new Set();
+      bucket.bySessionId.set(sessionId, sessionObservers);
+    }
+    sessionObservers.add(observer);
+  }
+  removeObserver(type, observer) {
+    const bucket = this.observerMap[type];
+    bucket.all.delete(observer);
+    bucket.bySessionId.forEach((sessionObservers, sessionId) => {
+      sessionObservers.delete(observer);
+      if (sessionObservers.size === 0) {
+        bucket.bySessionId.delete(sessionId);
+      }
+    });
+  }
   start() {
     if (this.isStarted) {
       return;
     }
     const handle = entry => {
       var _entry$body;
-      const observers = this.observerMap[entry.type];
+      const observers = this.collectObservers(entry);
       let state = '';
       if (entry.type === 'connectionState') {
         state = "".concat(ConnectState[entry.body.connectState], " ").concat(entry.body.code || '');
@@ -47,18 +124,19 @@ export class AIStreamObserverPool {
             sessionId
           } = observer.options;
           let matched = true;
+          const sessionIdList = entry.body.sessionIdList || [];
           if (dataChannels !== null && dataChannels !== void 0 && dataChannels.length && !dataChannels.includes(entry.body.dataChannel)) {
             matched = false;
           }
-          if (sessionId && !entry.body.sessionIdList.includes(sessionId)) {
+          if (sessionId && !sessionIdList.includes(sessionId)) {
             matched = false;
           }
           if (matched) {
             observer.callback(entry, observer);
           }
-        } else {
+        } else if (!((entry.type === 'event' || entry.type === 'sessionEvent' || entry.type === 'sessionState') && observer.options.sessionId && observer.options.sessionId !== entry.body.sessionId)) {
           observer.callback(entry, observer);
-        }
+        } // noop
       });
     };
     this.cancels = [listenConnectStateChanged(body => handle({
@@ -98,36 +176,26 @@ export class AIStreamObserverPool {
     this.isStarted = false;
     this.cancels.forEach(cancel => cancel());
     this.cancels = [];
-    this.observerMap = {
-      connectionState: new Set(),
-      sessionState: new Set(),
-      sessionEvent: new Set(),
-      event: new Set(),
-      text: new Set(),
-      audio: new Set(),
-      video: new Set(),
-      file: new Set(),
-      image: new Set()
-    };
+    this.observerMap = this.createObserverMap();
   }
   connect(observer) {
     if (!this.isStarted) {
       this.start();
     }
-    types.forEach(key => {
-      const type = key;
+    types.forEach(type => {
       if (observer.options[type]) {
-        this.observerMap[type].add(observer);
+        this.addObserver(type, observer);
       }
     });
   }
   disconnect(observer) {
-    types.forEach(key => {
-      this.observerMap[key].delete(observer);
+    types.forEach(type => {
+      this.removeObserver(type, observer);
     });
     let empty = true;
-    types.forEach(key => {
-      if (this.observerMap[key].size > 0) {
+    types.forEach(type => {
+      const bucket = this.observerMap[type];
+      if (bucket.all.size > 0 || bucket.bySessionId.size > 0) {
         empty = false;
       }
     });

package/dist/utils/sendMessage.d.ts

@@ -6,7 +6,6 @@ export interface SendBlocksToAIStreamParams {
     blocks: InputBlock[];
     session: AIStreamSession;
     signal?: AbortSignalObject;
-    enableTts?: boolean;
     eventIdPrefix?: string;
     getUserData: () => Promise<AIStreamUserData>;
 }

package/dist/utils/sendMessage.js

@@ -219,15 +219,17 @@ export function sendBlocksToAIStream(params) {
           if (!imageId) {
             imageId = generateId();
           }
+          const attachment = {
+            streamFlag: data.body.streamFlag,
+            path: data.body.path,
+            width: data.body.width,
+            height: data.body.height
+          };
           enqueue({
             id: imageId,
             type: 'attachment',
             attachmentType: 'image',
-            attachment: {
-              path: data.body.path || null,
-              width: data.body.width || null,
-              height: data.body.height || null
-            },
+            attachment: attachment,
             meta
           });
           if (data.body.streamFlag === StreamFlag.END) {
@@ -237,13 +239,20 @@ export function sendBlocksToAIStream(params) {
           if (!audioId) {
             audioId = generateId();
           }
+          const attachment = {
+            streamFlag: data.body.streamFlag,
+            path: data.body.path,
+            pts: data.body.pts,
+            codecType: data.body.codecType,
+            sampleRate: data.body.sampleRate,
+            channels: data.body.channels,
+            bitDepth: data.body.bitDepth
+          };
           enqueue({
             id: audioId,
             type: 'attachment',
             attachmentType: 'audio',
-            attachment: {
-              path: data.body.path || null
-            },
+            attachment,
             meta
           });
           if (data.body.streamFlag === StreamFlag.END) {
@@ -380,6 +389,8 @@ export function sendBlocksToAIStream(params) {
     }
   });
   return {
+    // web-streams-polyfill's ReadableStream is runtime-compatible but its
+    // getReader() overloads differ from lib.dom's, so cast to satisfy the type.
     response: new StreamResponse(stream),
     metaPromise
   };

package/dist/utils/ttt.d.ts

@@ -72,6 +72,7 @@ export declare const createSession: (options?: Omit<CreateSessionParams, "succes
     sessionId: string;
     sendDataChannels: string[];
     revDataChannels: string[];
+    baseCacheDir: string;
 }>;
 export declare const closeSession: (options?: Omit<CloseSessionParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const sendEventStart: (options?: Omit<SendEventStartParams, "success" | "fail"> | undefined) => Promise<{

package/dist/utils/ttt.js

@@ -42,6 +42,8 @@ export const sendEventStart = promisify(ty.aistream.sendEventStart, true);
 export const sendEventPayloadEnd = promisify(ty.aistream.sendEventPayloadEnd, true);
 export const sendEventEnd = promisify(ty.aistream.sendEventEnd, true);
 export const sendEventChatBreak = promisify(ty.aistream.sendEventChatBreak, true);
+
+// @ts-ignore
 export const sendEventToAIStream = promisify(ty.aistream.sendEvent, true);
 export const startRecordAndSendAudioData = promisify(ty.aistream.startRecordAndSendAudioData, true);
 export const stopRecordAndSendAudioData = promisify(ty.aistream.stopRecordAndSendAudioData, true);

package/dist/withAIStream.d.ts

@@ -1,4 +1,4 @@
-import { AbortSignalObject, ChatAgent, ChatCardObject, ChatMessage, ChatMessageStatus, ChatTile, GetChatPluginHandler, InputBlock } from '@ray-js/t-agent';
+import { AbortSignalObject, AttachmentPart, ChatAgent, ChatCardObject, ChatMessage, ChatMessageStatus, ChatTile, GetChatPluginHandler, InputBlock } from '@ray-js/t-agent';
 import { TTTAction } from './utils';
 import { AIStreamUserData, ConnectClientType, ReceivedTextSkillPacketBody, SendAIStreamEventParams, SessionEventBody } from './AIStreamTypes';
 import { ChatHistoryStore, StoredMessageObject } from './ChatHistoryStore';
@@ -46,7 +46,13 @@ export interface AIStreamHooks {
     onTextCompose: (respMsg: ChatMessage, status: ChatMessageStatus, result: {
         text: string;
     }) => void;
-    onSkillCompose: (skill: ReceivedTextSkillPacketBody[], respMsg: ChatMessage, result: {
+    onSkillCompose: (skill: ReceivedTextSkillPacketBody, respMsg: ChatMessage, result: {
+        messages: ChatMessage[];
+    }) => void;
+    onAttachmentCompose: (part: AttachmentPart, respMsg: ChatMessage, result: {
+        messages: ChatMessage[];
+    }) => void;
+    onAttachmentsEnd: (parts: AttachmentPart[], respMsg: ChatMessage, result: {
         messages: ChatMessage[];
     }) => void;
     onSkillsEnd: (skills: ReceivedTextSkillPacketBody[], respMsg: ChatMessage, result: {
@@ -94,6 +100,8 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
         onCardsReceived: (fn: AIStreamHooks["onCardsReceived"]) => () => void;
         onUserDataRead: (fn: AIStreamHooks["onUserDataRead"]) => () => void;
         onSessionEventReceived: (fn: AIStreamHooks["onSessionEventReceived"]) => () => void;
+        onAttachmentCompose: (fn: AIStreamHooks["onAttachmentCompose"]) => () => void;
+        onAttachmentsEnd: (fn: AIStreamHooks["onAttachmentsEnd"]) => () => void;
         sendEvent: (params: SendAIStreamEventParams) => Promise<null>;
         onTTTAction: (fn: AIStreamHooks["onTTTAction"]) => () => void;
         getChatId: () => Promise<string>;

package/dist/withAIStream.js

@@ -93,65 +93,79 @@ export function withAIStream() {
         clientType,
         deviceId
       });
-      const streamSession = connection.createSession({
-        ownerId: clientType === ConnectClientType.DEVICE ? deviceId : "".concat(homeId),
-        api: tokenOptions.api,
-        apiVersion: tokenOptions.version,
-        solutionCode: agentId,
-        extParams: _objectSpread({
-          needTts: !!options.enableTts,
-          deviceId
-        }, tokenOptions.extParams),
-        getSessionUserData: async () => {
-          const result = {
-            userData: {}
-          };
-          await hooks.callHook('onUserDataRead', 'create-session', {}, result);
-          return result.userData;
-        }
-      });
-      streamSession.on('sessionEvent', event => {
-        hooks.callHook('onSessionEventReceived', event);
-      });
-      await session.set('AIStream.streamSession', streamSession);
-      if (options.earlyStart) {
-        // 故意异步，不阻塞消息列表加载
-        streamSession.ensureSession().catch(error => {
-          logger.error('earlyStart failed', error);
+      let streamSession = null;
+      try {
+        streamSession = connection.createSession({
+          ownerId: clientType === ConnectClientType.DEVICE ? deviceId : "".concat(homeId),
+          api: tokenOptions.api,
+          apiVersion: tokenOptions.version,
+          solutionCode: agentId,
+          extParams: _objectSpread({
+            needTts: !!options.enableTts,
+            deviceId
+          }, tokenOptions.extParams),
+          getSessionUserData: async () => {
+            const result = {
+              userData: {}
+            };
+            await hooks.callHook('onUserDataRead', 'create-session', {}, result);
+            return result.userData;
+          }
         });
-      }
-
-      /*
-       * 2. 获取历史消息
-       */
-      let historyStore;
-      if (typeof options.createChatHistoryStore === 'function') {
-        historyStore = options.createChatHistoryStore(agent);
-      } else {
-        historyStore = new ChatHistoryLocalStore({
-          bizCode: BizCode.CHAT,
-          agentId,
-          deviceId,
-          homeId,
-          indexId
+        streamSession.on('sessionEvent', event => {
+          hooks.callHook('onSessionEventReceived', event);
         });
-      }
-      if (historyStore) {
-        await agent.session.set('AIStream.historyStore', historyStore);
-        const {
-          records
-        } = await historyStore.queryAll({
-          sort: 'desc',
-          // id 倒序去获取最新的消息list，然后再reverse，保证最新的消息在最前面
-          limit: options.historySize || 1000,
-          offset: 0
+        await session.set('AIStream.connection', connection);
+        await session.set('AIStream.streamSession', streamSession);
+        if (options.earlyStart) {
+          // 故意异步，不阻塞消息列表加载
+          streamSession.ensureSession().catch(error => {
+            logger.error('earlyStart failed', error);
+          });
+        }
+
+        /*
+         * 2. 获取历史消息
+         */
+        let historyStore;
+        if (typeof options.createChatHistoryStore === 'function') {
+          historyStore = options.createChatHistoryStore(agent);
+        } else {
+          historyStore = new ChatHistoryLocalStore({
+            bizCode: BizCode.CHAT,
+            agentId,
+            deviceId,
+            homeId,
+            indexId
+          });
+        }
+        if (historyStore) {
+          await agent.session.set('AIStream.historyStore', historyStore);
+          const {
+            records
+          } = await historyStore.queryAll({
+            sort: 'desc',
+            // id 倒序去获取最新的消息list，然后再reverse，保证最新的消息在最前面
+            limit: options.historySize || 1000,
+            offset: 0
+          });
+          await session.set('AIStream.rawMessages', records.reverse());
+          session.isNewChat = records.length === 0;
+        } else {
+          session.isNewChat = true;
+        }
+        session.sessionId = indexId;
+      } catch (error) {
+        if (streamSession) {
+          await streamSession.close().catch(closeError => {
+            logger.error('withAIStream cleanup streamSession failed', closeError);
+          });
+        }
+        await connection.close().catch(closeError => {
+          logger.error('withAIStream cleanup connection failed', closeError);
         });
-        await session.set('AIStream.rawMessages', records.reverse());
-        session.isNewChat = records.length === 0;
-      } else {
-        session.isNewChat = true;
+        throw error;
       }
-      session.sessionId = indexId;
     });
     const getHistoryStore = () => {
       const historyStore = session.get('AIStream.historyStore');
@@ -431,6 +445,7 @@ export function withAIStream() {
       });
       await hooks.callHook('onChatMessageSent', userMsg, message);
       const skills = [];
+      const attachments = [];
       logger.debug('withAIStream chat agent.flushStreamToShow');
       const result = {
         messages: await agent.flushStreamToShow(message, response, {
@@ -445,17 +460,28 @@ export function withAIStream() {
             const result = {
               messages: []
             };
+            await hooks.callHook('onAttachmentCompose', part, respMsg, result);
+            attachments.push(part);
             if (part.attachmentType === 'skill') {
-              skills.push(part.attachment);
-              await hooks.callHook('onSkillCompose', part.attachment, respMsg, result);
+              const skill = part.attachment;
+              skills.push(skill);
+              await hooks.callHook('onSkillCompose', skill, respMsg, result);
             }
             return result.messages;
           }
         })
       };
       logger.debug('withAIStream chat agent.flushStreamToShow end');
+      let changed = false;
       if (skills.length) {
+        changed = true;
         await hooks.callHook('onSkillsEnd', skills, message, result);
+      }
+      if (attachments.length) {
+        changed = true;
+        await hooks.callHook('onAttachmentsEnd', attachments, message, result);
+      }
+      if (changed) {
         await message.update();
       }
       let valid = false;
@@ -533,6 +559,7 @@ export function withAIStream() {
     });
     onAgentDispose(async () => {
       const streamSession = session.get('AIStream.streamSession');
+      const connection = session.get('AIStream.connection');
 
       // 比较低的概率 onAgentStart 还没跑完（卡在获取获取家庭信息）就调用了 dispose
       if (streamSession) {
@@ -541,6 +568,9 @@ export function withAIStream() {
          */
         await streamSession.close();
       }
+      if (connection) {
+        await connection.close();
+      }
     });
     const handleTTTAction = async (tile, tttAction) => {
       const result = {
@@ -664,6 +694,12 @@ export function withAIStream() {
         onSessionEventReceived: fn => {
           return hooks.hook('onSessionEventReceived', fn);
         },
+        onAttachmentCompose: fn => {
+          return hooks.hook('onAttachmentCompose', fn);
+        },
+        onAttachmentsEnd: fn => {
+          return hooks.hook('onAttachmentsEnd', fn);
+        },
         sendEvent: params => sendEventToAIStream(params),
         onTTTAction: fn => {
           return hooks.hook('onTTTAction', fn);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.8-beta.2",
+  "version": "0.2.8-beta.4",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -26,7 +26,6 @@
     "test:coverage": "jest --runInBand --coverage"
   },
   "dependencies": {
-    "dayjs": "^1.10.4",
     "url-parse": "^1.5.10",
     "web-streams-polyfill": "^4.0.0"
   },
@@ -37,5 +36,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "450ca8d41536094b3bea57198fd7724ba6812438"
+  "gitHead": "556407c77bdeb9f879a8bb9e89184fc62e4c8bd0"
 }