@ray-js/t-agent-plugin-aistream 0.2.6 → 0.2.7-beta.2

package/README-zh_CN.md

@@ -771,6 +771,7 @@ const createAgent = () => {
 - `indexId` 索引 ID，默认为 'default'
 - `homeId` 家庭 ID，不填默认当前家庭
 - `earlyStart` 是否在 onAgentStart 阶段就建立连接
+- `eventIdPrefix` eventId 前缀，用于方便云端调试和日志追踪
 - `tokenOptions` 获取 agent token 的参数
   - `api` API 接口名
   - `version` 接口版本
@@ -779,14 +780,30 @@ const createAgent = () => {
 
 方法：
 
-- `agent.plugins.aiStream.send` 向智能体发送一条消息
-- `agent.plugins.aiStream.chat` 向智能体发送一条消息，并生成提问 ChatMessage 对象和 AI 回答 ChatMessage 对象，流式更新
+- `agent.plugins.aiStream.send(blocks, signal, userData)` 向智能体发送一条消息
+  - `blocks` 输入块数组
+  - `signal` 可选的 AbortSignal，用于中断请求
+  - `userData` 可选的用户数据，会附带在发送的消息中
+- `agent.plugins.aiStream.chat(blocks, signal, options)` 向智能体发送一条消息，并生成提问 ChatMessage 对象和 AI 回答 ChatMessage 对象，流式更新
+  - `blocks` 输入块数组
+  - `signal` 可选的 AbortSignal
+  - `options.sendBy` 发送者角色，默认为 'user'
+  - `options.responseBy` 响应者角色，默认为 'assistant'
+  - `options.userData` 可选的用户数据
+- `agent.plugins.aiStream.getChatId()` 获取当前会话的 chatId，返回 Promise<string>
 
 Hooks：
 
 - `onMessageParse` 当读取历史消息，解析消息时触发，可以在这个 Hook 里修改消息
   - `msgItem` 存储的消息对象
   - `result.messages` 解析后的消息列表
+- `onChatMessageSent` 当用户消息和响应消息创建后触发
+  - `userMessage` 用户发送的消息
+  - `respMessage` AI 的响应消息
+- `onTextCompose` 当收到文本数据时触发，用于处理文本的渲染
+  - `respMsg` 响应消息
+  - `status` 消息状态
+  - `result.text` 文本内容，可以修改
 - `onSkillCompose` 当收到技能数据时触发，用于处理技能的渲染
   - `skill` 技能数据数组 (ReceivedTextSkillPacketBody[])
   - `respMsg` 响应消息
@@ -801,6 +818,47 @@ Hooks：
 - `onCardsReceived` 当收到卡片数据时触发
   - `skills` 技能数据列表 (ReceivedTextSkillPacketBody[])
   - `result.cards` 卡片列表
+- `onUserDataRead` **(0.2.x 新增)** 当需要读取用户自定义数据时触发，用于向 AI 平台传递额外的上下文信息
+  - `type` 触发类型，可以是 'create-session' 或 'start-event'
+    - `'create-session'` 创建会话时触发，只触发一次
+    - `'start-event'` 每次发送消息时触发
+  - `data` 上下文数据
+    - `data.blocks` 当 type 为 'start-event' 时，包含本次发送的输入块
+  - `result.userData` 返回的用户数据对象，会被合并后发送给 AI 平台
+
+**使用示例**：
+
+```tsx
+const agent = createChatAgent(
+  withUI(),
+  withAIStream({
+    agentId: 'your-agent-id',
+  })
+);
+
+agent.plugins.aiStream.onUserDataRead((type, data, result) => {
+  // 在创建会话时传递用户信息
+  if (type === 'create-session') {
+    result.sessionAttributes = {
+      'custom.param': {
+        userName: { value: 'John' },
+        userLevel: { value: 'Plus' },
+      },
+    };
+    return;
+  }
+  // 在每次发送消息时传递动态上下文
+  if (type === 'start-event') {
+    result.userData = {
+      'custom.param': {
+        timestamp: { value: Date.now() },
+        pid: { value: '123456' },
+      },
+    };
+    return;
+  }
+});
+```
 
 ### withBuildIn 插件
 

package/README.md

@@ -766,6 +766,7 @@ Parameters:
 - `indexId` Index ID, defaults to 'default'
 - `homeId` Home ID, defaults to current home if not provided
 - `earlyStart` Whether to establish connection during onAgentStart phase
+- `eventIdPrefix` Event ID prefix for cloud debugging and log tracing
 - `tokenOptions` Parameters for getting agent token
   - `api` API interface name
   - `version` Interface version
@@ -774,14 +775,30 @@ Parameters:
 
 Methods:
 
-- `agent.plugins.aiStream.send`: Send a message to the agent
-- `agent.plugins.aiStream.chat`: Send a message to the agent while generating a ChatMessage object for the question and the AI's answer, updating in a streaming manner
+- `agent.plugins.aiStream.send(blocks, signal, userData)` Send a message to the agent
+  - `blocks` Input block array
+  - `signal` Optional AbortSignal for interrupting requests
+  - `userData` Optional user data to be included with the message
+- `agent.plugins.aiStream.chat(blocks, signal, options)` Send a message to the agent while generating a ChatMessage object for the question and the AI's answer, updating in a streaming manner
+  - `blocks` Input block array
+  - `signal` Optional AbortSignal
+  - `options.sendBy` Sender role, defaults to 'user'
+  - `options.responseBy` Responder role, defaults to 'assistant'
+  - `options.userData` Optional user data
+- `agent.plugins.aiStream.getChatId()` Get the current session's chatId, returns Promise<string>
 
 Hooks:
 
 - `onMessageParse` Triggered when reading history messages and parsing them, allowing for message modification in this Hook
   - `msgItem` Stored message object
   - `result.messages` Parsed message list
+- `onChatMessageSent` Triggered after user message and response message are created
+  - `userMessage` User's sent message
+  - `respMessage` AI's response message
+- `onTextCompose` Triggered when receiving text data, used for handling text rendering
+  - `respMsg` Response message
+  - `status` Message status
+  - `result.text` Text content, can be modified
 - `onSkillCompose` Triggered when receiving skill data, used for handling skill rendering
   - `skill` Skill data array (ReceivedTextSkillPacketBody[])
   - `respMsg` Response message
@@ -796,6 +813,47 @@ Hooks:
 - `onCardsReceived` Triggered when receiving card data
   - `skills` Skill data list (ReceivedTextSkillPacketBody[])
   - `result.cards` Card list
+- `onUserDataRead` **(0.2.x New)** Triggered when user custom data needs to be read, used to pass additional context information to the AI platform
+  - `type` Trigger type, can be 'create-session' or 'start-event'
+    - `'create-session'` Triggered when creating a session, only once
+    - `'start-event'` Triggered each time a message is sent
+  - `data` Context data
+    - `data.blocks` When type is 'start-event', contains the input blocks for this send
+  - `result.userData` Returned user data object, will be merged and sent to the AI platform
+
+**Usage Example**:
+
+```tsx
+const agent = createChatAgent(
+  withUI(),
+  withAIStream({
+    agentId: 'your-agent-id',
+  })
+);
+
+agent.plugins.aiStream.onUserDataRead((type, data, result) => {
+  // Pass user information when creating a session
+  if (type === 'create-session') {
+    result.sessionAttributes = {
+      'custom.param': {
+        userName: { value: 'John' },
+        userLevel: { value: 'Plus' },
+      },
+    };
+    return;
+  }
+  // Pass dynamic context each time a message is sent
+  if (type === 'start-event') {
+    result.userData = {
+      'custom.param': {
+        timestamp: { value: Date.now() },
+        pid: { value: '123456' },
+      },
+    };
+    return;
+  }
+});
+```
 
 ### withBuildIn Plugin
 

package/dist/AIStreamTypes.d.ts

@@ -216,6 +216,23 @@ export type MiniProgramAccountInfo = {
     /** 小程序图标 */
     appIcon: string;
 };
+export interface CustomParamItem {
+    value: any;
+    /** 生效次数，不传一直生效 */
+    effectiveCount?: number;
+}
+export type AIStreamUserData = {
+    sessionAttributes?: {
+        'custom.param'?: Record<string, CustomParamItem>;
+        [key: string]: any;
+    };
+    eventAttributes?: {
+        'custom.param'?: Record<string, CustomParamItem>;
+        [key: string]: any;
+    };
+    chatAttributes?: AIStreamChatAttribute;
+    [key: string]: any;
+};
 /**
  * 获取小程序账号信息
  */
@@ -630,9 +647,7 @@ export declare enum EventType {
     /** 聊天中断 */
     CHAT_BREAK = 4,
     /** 服务端 VAD */
-    SERVER_VAD = 5,
-    /** 服务端错误 */
-    SERVER_ERROR = 6
+    SERVER_VAD = 5
 }
 export declare enum StreamFlag {
     /** 仅一包 */
@@ -821,8 +836,6 @@ export type EventBody = {
     sessionId: string;
     /** 事件类型: 0-Event Start, 1-Event Payload End, 2-Event End, 3 - OneShot, 4-Chat Break, 5-Server VAD */
     eventType: EventType;
-    /** 属性列表 */
-    userData?: Attribute[];
 };
 export type AudioBody = {
     /** 接收数据通道 Code */
@@ -831,8 +844,6 @@ export type AudioBody = {
     streamFlag: number;
     /** 音频缓存路径 */
     path: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     /** SessionId 列表, 云端返回，可能为空 */
     sessionIdList?: string[];
     /**
@@ -945,8 +956,6 @@ export type VideoBody = {
     streamFlag: StreamFlag;
     /** 视频缓存路径，传输结束后可访问(streamFlag == 1 || StreamFlag == 3) */
     path: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     /** SessionId 列表, 云端返回，可能为空 */
     sessionIdList?: string[];
 };
@@ -961,8 +970,6 @@ export type ImageBody = {
     width: number;
     /** 图片高度 */
     height: number;
-    /** 扩展属性 */
-    userData?: Attribute[];
     /** SessionId 列表, 云端返回，可能为空 */
     sessionIdList?: string[];
 };
@@ -973,8 +980,6 @@ export type TextBody = {
     streamFlag: StreamFlag;
     /** 文本内容(JSON String，见云端文档定义) */
     text: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     /** SessionId 列表, 云端返回，可能为空 */
     sessionIdList?: string[];
 };
@@ -987,8 +992,6 @@ export type FileBody = {
     path: string;
     /** 文件类型: 1 - MP4, 2 - OGG_OPUS, 3 - PDF, 4 - JSON, 5 - IPC_LOG, 6 - SweeperMap */
     format: FileFormat;
-    /** 扩展属性 */
-    userData?: Attribute[];
     /** SessionId 列表, 云端返回，可能为空 */
     sessionIdList?: string[];
 };
@@ -1142,8 +1145,13 @@ export type CreateSessionParams = {
     sessionId?: string;
     /** 业务配置定义（通过 queryAgentToken 传递） */
     bizConfig: BizConfig;
-    /** 扩展属性 */
+    /**
+     * 扩展属性
+     * @deprecated
+     */
     userData?: Attribute[];
+    /** 扩展属性 */
+    userDataJson?: string;
     /** 是否复用数据通道，默认不复用 */
     reuseDataChannel?: boolean;
     success?: (params: {
@@ -1211,8 +1219,13 @@ export type SendEventStartParams = {
     /** 会话 id */
     sessionId: string;
     eventIdPrefix?: string;
-    /** 扩展属性 */
+    /**
+     * 扩展属性
+     * @deprecated
+     */
     userData?: Attribute[];
+    /** 扩展属性 */
+    userDataJson?: string;
     success?: (params: {
         /** 事件 id。此后 sendEventPayloadEnd、sendEventEnd、sendEventChatBreak 都需要使用该 eventId 直到下发一次 sendEventStart 生成新 eventId 之前。 */
         eventId: string;
@@ -1228,7 +1241,7 @@ export type SendEventStartParams = {
     complete?: () => void;
 };
 /**
- *@description 某个数据流传输结束事件
+ * @description 某个数据流传输结束事件
  */
 export type SendEventPayloadEndParams = {
     /** 事件 id */
@@ -1237,8 +1250,6 @@ export type SendEventPayloadEndParams = {
     sessionId: string;
     /** 发送结束 的 dataChannel */
     dataChannel: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1258,8 +1269,6 @@ export type SendEventEndParams = {
     eventId: string;
     /** 会话 id */
     sessionId: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1279,8 +1288,13 @@ export type SendEventChatBreakParams = {
     eventId: string;
     /** 会话 id */
     sessionId: string;
-    /** 扩展属性 */
+    /**
+     * 扩展属性
+     * @deprecated
+     */
     userData?: Attribute[];
+    /** 扩展属性 */
+    userDataJson?: string;
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1304,8 +1318,6 @@ export type StartRecordAndSendAudioDataParams = {
     sessionId: string;
     /** 下发数据通道，当音频只有单路的时候，可以不传 */
     dataChannel?: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1322,8 +1334,6 @@ export type StopRecordAndSendAudioDataParams = {
     sessionId: string;
     /** 下发数据通道，当音频只有单路的时候，可以不传 */
     dataChannel?: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: AIStreamAudioFile | null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1372,8 +1382,6 @@ export type StopRecordAndSendVideoDataParams = {
      * 1-前置摄像头, 2-后置摄像头
      */
     cameraType: VideoCameraType;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1392,8 +1400,6 @@ export type SendImageDataParams = {
     dataChannel?: string;
     /** 图片路径 */
     path: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1412,8 +1418,6 @@ export type SendTextDataParams = {
     dataChannel?: string;
     /** 文本内容 */
     text: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1447,8 +1451,6 @@ export type SendFileDataParams = {
     path: string;
     /** 文件类型: 1 - MP4, 2 - OGG_OPUS, 3 - PDF, 4 - JSON, 5 - IPC_LOG, 6 - SweeperMap */
     format: FileFormat;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;

package/dist/AIStreamTypes.js

@@ -100,7 +100,6 @@ export let EventType = /*#__PURE__*/function (EventType) {
   EventType[EventType["ONE_SHOT"] = 3] = "ONE_SHOT";
   EventType[EventType["CHAT_BREAK"] = 4] = "CHAT_BREAK";
   EventType[EventType["SERVER_VAD"] = 5] = "SERVER_VAD";
-  EventType[EventType["SERVER_ERROR"] = 6] = "SERVER_ERROR";
   return EventType;
 }({});
 export let StreamFlag = /*#__PURE__*/function (StreamFlag) {
@@ -228,7 +227,7 @@ export let NetworkType = /*#__PURE__*/function (NetworkType) {
  */
 
 /**
- *@description 某个数据流传输结束事件
+ * @description 某个数据流传输结束事件
  */
 
 /**

package/dist/asr/AsrAgent.js

@@ -177,7 +177,7 @@ export class AsrAgent {
       return;
     }
     const activeSession = await this.createSession();
-    const attribute = {
+    const chatAttributes = {
       'processing.interrupt': 'false',
       'asr.enableVad': 'false'
     };
@@ -186,8 +186,11 @@ export class AsrAgent {
       userData: [{
         type: AIStreamAttributeType.AI_CHAT,
         payloadType: AIStreamAttributePayloadType.STRING,
-        value: JSON.stringify(attribute)
-      }]
+        value: JSON.stringify(chatAttributes)
+      }],
+      userDataJson: JSON.stringify({
+        chatAttributes
+      })
     }));
     if (startEventError) {
       finish(startEventError);

package/dist/utils/AIStream.d.ts

@@ -1,4 +1,4 @@
-import { AIStreamAudioFile, Attribute, BizTag, ConnectClientType, ConnectState, FileFormat, VideoCameraType } from '../AIStreamTypes';
+import { AIStreamAudioFile, AIStreamUserData, Attribute, BizTag, ConnectClientType, ConnectState, FileFormat, VideoCameraType } from '../AIStreamTypes';
 import { AIStreamDataEntry, AIStreamObserverPool } from './observer';
 import { AIStreamError } from './errors';
 interface AIStreamConnectionOptions {
@@ -40,12 +40,13 @@ interface AIStreamSessionOptions {
     apiVersion?: string;
     /** 业务额外参数 */
     extParams?: any;
-    userData?: Attribute[];
+    getSessionUserData?: () => Promise<AIStreamUserData>;
 }
 interface AIStreamEventOptions {
     signal?: AbortSignal;
-    userData?: Attribute[];
     eventIdPrefix?: string;
+    userData?: Attribute[];
+    userDataJson?: string;
 }
 export declare class AIStreamSession {
     private connection;
@@ -75,30 +76,25 @@ type AIStreamEventWriteChunk = {
     type: 'text';
     text: string;
     dataChannel?: string;
-    userData?: Attribute[];
 } | {
     type: 'file';
     path: string;
     format: FileFormat;
     dataChannel?: string;
-    userData?: Attribute[];
 } | {
     type: 'image';
     path: string;
     dataChannel?: string;
-    userData?: Attribute[];
 };
 type AIStreamEventSource = {
     type: 'audio';
     dataChannel?: string;
-    userData?: Attribute[];
     saveFile?: boolean;
     sampleRate?: number;
 } | {
     type: 'video';
     dataChannel?: string;
     cameraType?: VideoCameraType;
-    userData?: Attribute[];
 };
 export interface AIStreamEventStream {
     type: 'video' | 'audio';
@@ -123,12 +119,8 @@ export declare class AIStreamEvent {
     private findFirstCode;
     write(chunk: AIStreamEventWriteChunk): Promise<void>;
     stream(source: AIStreamEventSource): AIStreamEventStream;
-    end(options?: {
-        userData?: Attribute[];
-    }): Promise<void>;
-    abort(options?: {
-        userData?: Attribute[];
-    }): void;
+    end(): Promise<void>;
+    abort(): void;
     on(name: 'close', callback: () => void): void;
     on(name: 'finish', callback: () => void): void;
     on(name: 'error', callback: (error: AIStreamError) => void): void;

package/dist/utils/AIStream.js

@@ -1,13 +1,12 @@
-import _objectWithoutProperties from "@babel/runtime/helpers/esm/objectWithoutProperties";
 import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-const _excluded = ["signal"];
+import "core-js/modules/es.json.stringify.js";
 import "core-js/modules/esnext.iterator.constructor.js";
 import "core-js/modules/esnext.iterator.find.js";
 import "core-js/modules/esnext.iterator.for-each.js";
 import "core-js/modules/esnext.iterator.map.js";
 import "core-js/modules/web.dom-collections.iterator.js";
-import { AIStreamAttributeType, AIStreamErrorCode, AIStreamServerErrorCode, BizTag, ConnectClientType, ConnectState, EventType, NetworkType, SessionState } from '../AIStreamTypes';
+import { AIStreamErrorCode, AIStreamServerErrorCode, BizTag, ConnectClientType, ConnectState, EventType, NetworkType, SessionState } from '../AIStreamTypes';
 import { closeSession, connect, createSession, disconnect, getCurrentHomeInfo, getNetworkType, isConnected, queryAgentToken, sendEventChatBreak, sendEventEnd, sendEventPayloadEnd, sendEventStart, sendImageData, sendTextData, startRecordAndSendAudioData, stopRecordAndSendAudioData } from './ttt';
 import { AIStreamObserver, AIStreamObserverPool } from './observer';
 import { isAbortError, safeParseJSON } from '@ray-js/t-agent';
@@ -180,27 +179,12 @@ export class AIStreamSession {
       (_this$activeEvent2 = this.activeEvent) === null || _this$activeEvent2 === void 0 || _this$activeEvent2.emit('data', entry);
       if (entry.type === 'event' && entry.body.eventId === this.activeEvent.eventId) {
         const {
-          eventType,
-          userData
+          eventType
         } = entry.body;
         if (eventType === EventType.EVENT_END || eventType === EventType.CHAT_BREAK || eventType === EventType.ONE_SHOT) {
           var _this$activeEvent3;
           (_this$activeEvent3 = this.activeEvent) === null || _this$activeEvent3 === void 0 || _this$activeEvent3.emit('finish');
           this.cleanupEvent();
-        } else if (eventType === EventType.SERVER_ERROR) {
-          var _this$activeEvent4;
-          let code = '';
-          let message = '';
-          for (const item of userData) {
-            if (item.type === AIStreamAttributeType.ERROR_CODE) {
-              code = item.value;
-            } else if (item.type === AIStreamAttributeType.ERROR_MESSAGE) {
-              message = item.value;
-            }
-          }
-          const error = new AIStreamError(message || 'Event error occurred', code || AIStreamErrorCode.UNKNOWN_ERROR);
-          (_this$activeEvent4 = this.activeEvent) === null || _this$activeEvent4 === void 0 || _this$activeEvent4.emit('error', error);
-          this.cleanupEvent();
         }
       }
     });
@@ -274,12 +258,16 @@ export class AIStreamSession {
       } else {
         this.tokenExtParamsResolvable.resolve({});
       }
+      let userData = {};
+      if (options.getSessionUserData) {
+        userData = await options.getSessionUserData();
+      }
       {
         const [err, res] = await tryCatchTTT(() => createSession({
           bizTag: options.bizTag,
           agentToken,
           bizConfig,
-          userData: options.userData
+          userDataJson: JSON.stringify(userData)
         }));
         if (err) {
           this.promise = null;
@@ -302,18 +290,23 @@ export class AIStreamSession {
       throw new AIStreamError('Cannot start a new event while another is active', AIStreamErrorCode.EVENT_EXISTS);
     }
     const {
-        signal
-      } = options,
-      rest = _objectWithoutProperties(options, _excluded);
+      signal,
+      userData,
+      eventIdPrefix,
+      userDataJson
+    } = options;
     await this.ensureSession();
     if (signal !== null && signal !== void 0 && signal.aborted) {
       const error = new AIStreamError('start event was aborted', AIStreamErrorCode.EVENT_ABORTED);
       error.name = 'AbortError';
       throw error;
     }
-    const [error, result] = await tryCatchTTT(() => sendEventStart(_objectSpread({
-      sessionId: this.sessionId
-    }, rest)));
+    const [error, result] = await tryCatchTTT(() => sendEventStart({
+      sessionId: this.sessionId,
+      userDataJson,
+      userData,
+      eventIdPrefix
+    }));
     if (error) {
       throw error;
     }
@@ -358,11 +351,11 @@ export class AIStreamSession {
     return this.activeEvent;
   }
   cleanupEvent() {
-    var _this$activeObserver, _this$activeEvent5;
+    var _this$activeObserver, _this$activeEvent4;
     logger.debug('AIStreamSession cleanupEvent');
     (_this$activeObserver = this.activeObserver) === null || _this$activeObserver === void 0 || _this$activeObserver.disconnect();
     this.activeObserver = null;
-    (_this$activeEvent5 = this.activeEvent) === null || _this$activeEvent5 === void 0 || _this$activeEvent5.emit('close');
+    (_this$activeEvent4 = this.activeEvent) === null || _this$activeEvent4 === void 0 || _this$activeEvent4.emit('close');
     this.activeEvent = null;
   }
   cleanup() {
@@ -420,8 +413,7 @@ export class AIStreamEvent {
         const [error] = await tryCatchTTT(() => sendTextData({
           sessionId: this.sessionId,
           text: chunk.text,
-          dataChannel,
-          userData: chunk.userData
+          dataChannel
         }));
         if (error) {
           throw error;
@@ -433,13 +425,11 @@ export class AIStreamEvent {
         //   path: chunk.path,
         //   format: chunk.format,
         //   dataChannel,
-        //   userData: chunk.userData,
         // });
       } else if (chunk.type === 'image') {
         const [error] = await tryCatchTTT(() => sendImageData({
           sessionId: this.sessionId,
-          path: chunk.path,
-          userData: chunk.userData
+          path: chunk.path
         }));
         if (error) {
           throw error;
@@ -485,7 +475,6 @@ export class AIStreamEvent {
           startPromise = startRecordAndSendAudioData({
             sessionId: this.sessionId,
             dataChannel,
-            userData: source.userData,
             recordInitParams: source.sampleRate ? {
               sampleRate: source.sampleRate
             } : undefined,
@@ -497,7 +486,6 @@ export class AIStreamEvent {
           // await startRecordAndSendVideoData({
           //   dataChannel,
           //   cameraType,
-          //   userData: source.userData,
           // });
         }
       },
@@ -514,8 +502,7 @@ export class AIStreamEvent {
         if (source.type === 'audio') {
           file = await stopRecordAndSendAudioData({
             sessionId: this.sessionId,
-            dataChannel,
-            userData: source.userData
+            dataChannel
           });
         } else if (source.type === 'video') {
           logger.warn('Video data sending is not implemented yet');
@@ -523,7 +510,6 @@ export class AIStreamEvent {
           // await stopRecordAndSendVideoData({
           //   dataChannel,
           //   cameraType,
-          //   userData: source.userData,
           // });
         }
         sendEventPayloadEnd({
@@ -539,18 +525,17 @@ export class AIStreamEvent {
     this.streams[dataChannel] = stream;
     return stream;
   }
-  async end(options) {
+  async end() {
     if (this.closed) {
       return;
     }
     await Promise.all([...Object.values(this.chains), ...Object.values(this.streams).map(s => s.stop())]);
     await sendEventEnd({
       eventId: this.eventId,
-      sessionId: this.sessionId,
-      userData: options === null || options === void 0 ? void 0 : options.userData
+      sessionId: this.sessionId
     });
   }
-  abort(options) {
+  abort() {
     if (this.closed) {
       return;
     }
@@ -558,8 +543,7 @@ export class AIStreamEvent {
       // 故意不等，直接发送中断事件
       sendEventChatBreak({
         eventId: this.eventId,
-        sessionId: this.sessionId,
-        userData: options === null || options === void 0 ? void 0 : options.userData
+        sessionId: this.sessionId
       });
     }
     const error = new AIStreamError('This operation was aborted', AIStreamErrorCode.EVENT_ABORTED);

package/dist/utils/defaultMock.js

@@ -504,8 +504,7 @@ mock.hooks.hook('sendImageData', context => {
   }
   session.currentEvent.data.push({
     type: 'image',
-    path: context.options.path,
-    userData: context.options.userData
+    path: context.options.path
   });
   context.result = true;
 });
@@ -517,8 +516,7 @@ mock.hooks.hook('sendTextData', context => {
   }
   session.currentEvent.data.push({
     type: 'text',
-    text: context.options.text,
-    userData: context.options.userData
+    text: context.options.text
   });
 });
 const filterRecords = query => {

package/dist/utils/mock.d.ts

@@ -1,5 +1,5 @@
 import { Emitter } from '@ray-js/t-agent';
-import { Attribute, ReceivedTextSkillPacketBody } from '../AIStreamTypes';
+import { ReceivedTextSkillPacketBody } from '../AIStreamTypes';
 interface TTTCallContext {
     options: any;
     result: any;
@@ -8,11 +8,9 @@ export interface SendToAIStreamContext {
     data: Array<{
         type: 'text';
         text: string;
-        userData?: Attribute[];
     } | {
         type: 'image';
         path: string;
-        userData?: Attribute[];
     }>;
     responseText: string | undefined;
     wordDelayMs: number;

package/dist/utils/object.d.ts

@@ -5,3 +5,4 @@ export interface Resolvable<T = void> {
     state: 'pending' | 'fulfilled' | 'rejected';
 }
 export declare const createResolvable: <T = void>() => Resolvable<T>;
+export declare function deepMerge(target: any, source: any): any;

package/dist/utils/object.js

@@ -22,4 +22,27 @@ export const createResolvable = () => {
     };
   });
   return ret;
-};
+};
+export function deepMerge(target, source) {
+  if (typeof target !== 'object' || target === null) {
+    return source; // 如果目标不是对象，直接返回来源
+  }
+  if (typeof source !== 'object' || source === null) {
+    return target; // 如果来源不是对象，返回目标
+  }
+  for (const key in source) {
+    if (Object.prototype.hasOwnProperty.call(source, key)) {
+      const sourceValue = source[key];
+      const targetValue = target[key];
+
+      // 如果是对象，递归合并
+      if (typeof sourceValue === 'object' && sourceValue !== null) {
+        target[key] = deepMerge(Array.isArray(targetValue) ? [] : targetValue || {}, sourceValue);
+      } else {
+        // 否则直接赋值
+        target[key] = sourceValue;
+      }
+    }
+  }
+  return target;
+}

package/dist/utils/sendMessage.d.ts

@@ -1,14 +1,14 @@
 import '../polyfill';
 import { AIStreamSession } from './AIStream';
-import { AIStreamChatAttribute } from '../AIStreamTypes';
+import { AIStreamUserData } from '../AIStreamTypes';
 import { InputBlock, StreamResponse } from '@ray-js/t-agent';
 export interface SendBlocksToAIStreamParams {
     blocks: InputBlock[];
     session: AIStreamSession;
-    attribute?: AIStreamChatAttribute;
     signal?: AbortSignal;
     enableTts?: boolean;
     eventIdPrefix?: string;
+    getUserData: () => Promise<AIStreamUserData>;
 }
 export declare function sendBlocksToAIStream(params: SendBlocksToAIStreamParams): {
     response: StreamResponse;

package/dist/utils/sendMessage.js

@@ -1,4 +1,3 @@
-import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
 import "core-js/modules/es.json.stringify.js";
 import "core-js/modules/web.dom-collections.iterator.js";
 import '../polyfill';
@@ -8,6 +7,7 @@ import { EmitterEvent, generateId, safeParseJSON, StreamResponse } from '@ray-js
 import { tryCatch } from './misc';
 import { AIStreamError, transformErrorCode } from './errors';
 import logger from './logger';
+import { deepMerge } from './object';
 const mimeTypeToFormatMap = {
   'video/mp4': FileFormat.MP4,
   'text/json': FileFormat.JSON,
@@ -20,7 +20,8 @@ export function sendBlocksToAIStream(params) {
     session,
     blocks,
     signal,
-    eventIdPrefix
+    eventIdPrefix,
+    getUserData
   } = params;
   let audioEmitter = null;
   for (const block of blocks) {
@@ -31,10 +32,6 @@ export function sendBlocksToAIStream(params) {
       audioEmitter = block.audio_emitter;
     }
   }
-  const attribute = _objectSpread({
-    'processing.interrupt': 'false',
-    'asr.enableVad': 'false'
-  }, params.attribute);
   let canceled = false;
   let closed = false;
   let event = null;
@@ -103,14 +100,26 @@ export function sendBlocksToAIStream(params) {
           }
         });
       }
+      const chatAttributes = {
+        'processing.interrupt': 'false',
+        'asr.enableVad': 'false'
+      };
+      let eventUserData = {};
+      if (getUserData) {
+        eventUserData = await getUserData();
+      }
+      const userData = deepMerge({
+        chatAttributes
+      }, eventUserData);
       let error;
       [error, event] = await tryCatch(() => session.startEvent({
         signal,
         eventIdPrefix,
+        userDataJson: JSON.stringify(userData),
         userData: [{
           type: AIStreamAttributeType.AI_CHAT,
           payloadType: AIStreamAttributePayloadType.STRING,
-          value: JSON.stringify(attribute)
+          value: JSON.stringify(chatAttributes)
         }]
       }));
       if (error) {

package/dist/utils/ttt.d.ts

@@ -77,6 +77,7 @@ export declare const closeSession: (options?: Omit<CloseSessionParams, "success"
 export declare const sendEventStart: (options?: Omit<SendEventStartParams, "success" | "fail"> | undefined) => Promise<{
     eventId: string;
 }>;
+/** @deprecated */
 export declare const sendEventPayloadEnd: (options?: Omit<SendEventPayloadEndParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const sendEventEnd: (options?: Omit<SendEventEndParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const sendEventChatBreak: (options?: Omit<SendEventChatBreakParams, "success" | "fail"> | undefined) => Promise<null>;

package/dist/utils/ttt.js

@@ -37,6 +37,8 @@ export const queryAgentToken = promisify(ty.aistream.queryAgentToken, true);
 export const createSession = promisify(ty.aistream.createSession, true);
 export const closeSession = promisify(ty.aistream.closeSession, true);
 export const sendEventStart = promisify(ty.aistream.sendEventStart, true);
+
+/** @deprecated */
 export const sendEventPayloadEnd = promisify(ty.aistream.sendEventPayloadEnd, true);
 export const sendEventEnd = promisify(ty.aistream.sendEventEnd, true);
 export const sendEventChatBreak = promisify(ty.aistream.sendEventChatBreak, true);

package/dist/withAIStream.d.ts

@@ -1,6 +1,6 @@
 import { ChatAgent, ChatCardObject, ChatMessage, ChatMessageStatus, ChatTile, GetChatPluginHandler, InputBlock } from '@ray-js/t-agent';
 import { TTTAction } from './utils';
-import { ConnectClientType, ReceivedTextSkillPacketBody } from './AIStreamTypes';
+import { AIStreamUserData, ConnectClientType, ReceivedTextSkillPacketBody } from './AIStreamTypes';
 import { ChatHistoryStore, StoredMessageObject } from './ChatHistoryStore';
 export interface AIStreamOptions {
     /** client 类型: 1-作为设备代理, 2-作为 App，3-作为开发者（行业 App） */
@@ -58,18 +58,23 @@ export interface AIStreamHooks {
     onCardsReceived: (skills: ReceivedTextSkillPacketBody[], result: {
         cards: ChatCardObject[];
     }) => void;
+    onUserDataRead: (type: 'create-session' | 'start-event', data: {
+        blocks?: InputBlock[];
+    }, result: {
+        userData: AIStreamUserData;
+    }) => void;
 }
 export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAgent) => {
     hooks: import("hookable").Hookable<AIStreamHooks, import("hookable").HookKeys<AIStreamHooks>>;
     aiStream: {
-        send: (blocks: InputBlock[], signal?: AbortSignal, extraOptions?: Record<string, any>) => {
+        send: (blocks: InputBlock[], signal?: AbortSignal, eventUserData?: AIStreamUserData) => {
             response: import("@ray-js/t-agent").StreamResponse;
             metaPromise: Promise<Record<string, any>>;
         };
         chat: (blocks: InputBlock[], signal?: AbortSignal, options?: {
             sendBy?: string | undefined;
             responseBy?: string | undefined;
-            extraOptions?: Record<string, any> | undefined;
+            userData?: AIStreamUserData | undefined;
         } | undefined) => Promise<ChatMessage[]>;
         options: AIStreamOptions;
         removeMessage: (message: ChatMessage) => Promise<void>;
@@ -86,6 +91,7 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
         onTextCompose: (fn: AIStreamHooks['onTextCompose']) => () => void;
         onSkillsEnd: (fn: AIStreamHooks['onSkillsEnd']) => () => void;
         onCardsReceived: (fn: AIStreamHooks['onCardsReceived']) => () => void;
+        onUserDataRead: (fn: AIStreamHooks['onUserDataRead']) => () => void;
         onTTTAction: (fn: AIStreamHooks['onTTTAction']) => () => void;
         getChatId: () => Promise<string>;
     };

package/dist/withAIStream.js

@@ -14,6 +14,7 @@ import { BizCode, ConnectClientType } from './AIStreamTypes';
 import { DEFAULT_TOKEN_API, DEFAULT_TOKEN_API_VERSION, globalAIStreamClient } from './global';
 import logger from './utils/logger';
 import { ChatHistoryLocalStore } from './ChatHistoryLocalStore';
+import { deepMerge } from './utils/object';
 export function withAIStream() {
   let options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
   const hooks = createHooks();
@@ -88,7 +89,14 @@ export function withAIStream() {
         extParams: _objectSpread({
           needTts: !!options.enableTts,
           deviceId
-        }, tokenOptions.extParams)
+        }, tokenOptions.extParams),
+        getSessionUserData: async () => {
+          const result = {
+            userData: {}
+          };
+          await hooks.callHook('onUserDataRead', 'create-session', {}, result);
+          return result.userData;
+        }
       });
       await session.set('AIStream.streamSession', streamSession);
       if (options.earlyStart) {
@@ -221,14 +229,25 @@ export function withAIStream() {
         };
       }
     });
-    const send = (blocks, signal, extraOptions) => {
+    const send = (blocks, signal, eventUserData) => {
       const streamSession = session.get('AIStream.streamSession');
       const result = sendBlocksToAIStream({
         blocks,
         session: streamSession,
         signal,
         eventIdPrefix: options.eventIdPrefix,
-        attribute: _objectSpread({}, extraOptions)
+        getUserData: async () => {
+          const userDataResult = {
+            userData: {}
+          };
+          await hooks.callHook('onUserDataRead', 'start-event', {
+            blocks
+          }, userDataResult);
+          const userData = {};
+          deepMerge(userData, userDataResult.userData);
+          deepMerge(userData, eventUserData);
+          return userData;
+        }
       });
       signal === null || signal === void 0 || signal.addEventListener('abort', event => {
         logger.debug('withAIStream signal aborted, response.started:', result.response.started);
@@ -245,7 +264,7 @@ export function withAIStream() {
       const {
         sendBy = 'user',
         responseBy = 'assistant',
-        extraOptions
+        userData
       } = options || {};
       let audioEmitter = null;
       for (const block of blocks) {
@@ -345,7 +364,7 @@ export function withAIStream() {
       const {
         response,
         metaPromise
-      } = send(blocks, signal, extraOptions);
+      } = send(blocks, signal, userData);
       if (audioPromise) {
         try {
           await audioPromise;
@@ -585,6 +604,9 @@ export function withAIStream() {
         onCardsReceived: fn => {
           return hooks.hook('onCardsReceived', fn);
         },
+        onUserDataRead: fn => {
+          return hooks.hook('onUserDataRead', fn);
+        },
         onTTTAction: fn => {
           return hooks.hook('onTTTAction', fn);
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.6",
+  "version": "0.2.7-beta.2",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -35,5 +35,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "6fc9a8195706d72c2bab10479e8d65a26854bdde"
+  "gitHead": "c600e01e2bc977e22d9c5d114c721f6d87d21149"
 }