@ray-js/t-agent-plugin-aistream 0.2.6 → 0.2.7-beta.10

package/README-zh_CN.md

@@ -771,6 +771,7 @@ const createAgent = () => {
 - `indexId` 索引 ID，默认为 'default'
 - `homeId` 家庭 ID，不填默认当前家庭
 - `earlyStart` 是否在 onAgentStart 阶段就建立连接
+- `eventIdPrefix` eventId 前缀，用于方便云端调试和日志追踪
 - `tokenOptions` 获取 agent token 的参数
   - `api` API 接口名
   - `version` 接口版本
@@ -779,14 +780,30 @@ const createAgent = () => {
 
 方法：
 
-- `agent.plugins.aiStream.send` 向智能体发送一条消息
-- `agent.plugins.aiStream.chat` 向智能体发送一条消息，并生成提问 ChatMessage 对象和 AI 回答 ChatMessage 对象，流式更新
+- `agent.plugins.aiStream.send(blocks, signal, userData)` 向智能体发送一条消息
+  - `blocks` 输入块数组
+  - `signal` 可选的 AbortSignal，用于中断请求
+  - `userData` 可选的用户数据，会附带在发送的消息中
+- `agent.plugins.aiStream.chat(blocks, signal, options)` 向智能体发送一条消息，并生成提问 ChatMessage 对象和 AI 回答 ChatMessage 对象，流式更新
+  - `blocks` 输入块数组
+  - `signal` 可选的 AbortSignal
+  - `options.sendBy` 发送者角色，默认为 'user'
+  - `options.responseBy` 响应者角色，默认为 'assistant'
+  - `options.userData` 可选的用户数据
+- `agent.plugins.aiStream.getChatId()` 获取当前会话的 chatId，返回 Promise<string>
 
 Hooks：
 
 - `onMessageParse` 当读取历史消息，解析消息时触发，可以在这个 Hook 里修改消息
   - `msgItem` 存储的消息对象
   - `result.messages` 解析后的消息列表
+- `onChatMessageSent` 当用户消息和响应消息创建后触发
+  - `userMessage` 用户发送的消息
+  - `respMessage` AI 的响应消息
+- `onTextCompose` 当收到文本数据时触发，用于处理文本的渲染
+  - `respMsg` 响应消息
+  - `status` 消息状态
+  - `result.text` 文本内容，可以修改
 - `onSkillCompose` 当收到技能数据时触发，用于处理技能的渲染
   - `skill` 技能数据数组 (ReceivedTextSkillPacketBody[])
   - `respMsg` 响应消息
@@ -801,6 +818,47 @@ Hooks：
 - `onCardsReceived` 当收到卡片数据时触发
   - `skills` 技能数据列表 (ReceivedTextSkillPacketBody[])
   - `result.cards` 卡片列表
+- `onUserDataRead` **(0.2.x 新增)** 当需要读取用户自定义数据时触发，用于向 AI 平台传递额外的上下文信息
+  - `type` 触发类型，可以是 'create-session' 或 'start-event'
+    - `'create-session'` 创建会话时触发，只触发一次
+    - `'start-event'` 每次发送消息时触发
+  - `data` 上下文数据
+    - `data.blocks` 当 type 为 'start-event' 时，包含本次发送的输入块
+  - `result.userData` 返回的用户数据对象，会被合并后发送给 AI 平台
+
+**使用示例**：
+
+```tsx
+const agent = createChatAgent(
+  withUI(),
+  withAIStream({
+    agentId: 'your-agent-id',
+  })
+);
+
+agent.plugins.aiStream.onUserDataRead((type, data, result) => {
+  // 在创建会话时传递用户信息
+  if (type === 'create-session') {
+    result.sessionAttributes = {
+      'custom.param': {
+        userName: { value: 'John' },
+        userLevel: { value: 'Plus' },
+      },
+    };
+    return;
+  }
+  // 在每次发送消息时传递动态上下文
+  if (type === 'start-event') {
+    result.userData = {
+      'custom.param': {
+        timestamp: { value: Date.now() },
+        pid: { value: '123456' },
+      },
+    };
+    return;
+  }
+});
+```
 
 ### withBuildIn 插件
 

package/README.md

@@ -766,6 +766,7 @@ Parameters:
 - `indexId` Index ID, defaults to 'default'
 - `homeId` Home ID, defaults to current home if not provided
 - `earlyStart` Whether to establish connection during onAgentStart phase
+- `eventIdPrefix` Event ID prefix for cloud debugging and log tracing
 - `tokenOptions` Parameters for getting agent token
   - `api` API interface name
   - `version` Interface version
@@ -774,14 +775,30 @@ Parameters:
 
 Methods:
 
-- `agent.plugins.aiStream.send`: Send a message to the agent
-- `agent.plugins.aiStream.chat`: Send a message to the agent while generating a ChatMessage object for the question and the AI's answer, updating in a streaming manner
+- `agent.plugins.aiStream.send(blocks, signal, userData)` Send a message to the agent
+  - `blocks` Input block array
+  - `signal` Optional AbortSignal for interrupting requests
+  - `userData` Optional user data to be included with the message
+- `agent.plugins.aiStream.chat(blocks, signal, options)` Send a message to the agent while generating a ChatMessage object for the question and the AI's answer, updating in a streaming manner
+  - `blocks` Input block array
+  - `signal` Optional AbortSignal
+  - `options.sendBy` Sender role, defaults to 'user'
+  - `options.responseBy` Responder role, defaults to 'assistant'
+  - `options.userData` Optional user data
+- `agent.plugins.aiStream.getChatId()` Get the current session's chatId, returns Promise<string>
 
 Hooks:
 
 - `onMessageParse` Triggered when reading history messages and parsing them, allowing for message modification in this Hook
   - `msgItem` Stored message object
   - `result.messages` Parsed message list
+- `onChatMessageSent` Triggered after user message and response message are created
+  - `userMessage` User's sent message
+  - `respMessage` AI's response message
+- `onTextCompose` Triggered when receiving text data, used for handling text rendering
+  - `respMsg` Response message
+  - `status` Message status
+  - `result.text` Text content, can be modified
 - `onSkillCompose` Triggered when receiving skill data, used for handling skill rendering
   - `skill` Skill data array (ReceivedTextSkillPacketBody[])
   - `respMsg` Response message
@@ -796,6 +813,47 @@ Hooks:
 - `onCardsReceived` Triggered when receiving card data
   - `skills` Skill data list (ReceivedTextSkillPacketBody[])
   - `result.cards` Card list
+- `onUserDataRead` **(0.2.x New)** Triggered when user custom data needs to be read, used to pass additional context information to the AI platform
+  - `type` Trigger type, can be 'create-session' or 'start-event'
+    - `'create-session'` Triggered when creating a session, only once
+    - `'start-event'` Triggered each time a message is sent
+  - `data` Context data
+    - `data.blocks` When type is 'start-event', contains the input blocks for this send
+  - `result.userData` Returned user data object, will be merged and sent to the AI platform
+
+**Usage Example**:
+
+```tsx
+const agent = createChatAgent(
+  withUI(),
+  withAIStream({
+    agentId: 'your-agent-id',
+  })
+);
+
+agent.plugins.aiStream.onUserDataRead((type, data, result) => {
+  // Pass user information when creating a session
+  if (type === 'create-session') {
+    result.sessionAttributes = {
+      'custom.param': {
+        userName: { value: 'John' },
+        userLevel: { value: 'Plus' },
+      },
+    };
+    return;
+  }
+  // Pass dynamic context each time a message is sent
+  if (type === 'start-event') {
+    result.userData = {
+      'custom.param': {
+        timestamp: { value: Date.now() },
+        pid: { value: '123456' },
+      },
+    };
+    return;
+  }
+});
+```
 
 ### withBuildIn Plugin
 

package/dist/AIStreamTypes.d.ts

@@ -216,6 +216,23 @@ export type MiniProgramAccountInfo = {
     /** 小程序图标 */
     appIcon: string;
 };
+export interface CustomParamItem {
+    value: any;
+    /** 生效次数，不传一直生效 */
+    effectiveCount?: number;
+}
+export type AIStreamUserData = {
+    sessionAttributes?: {
+        'custom.param'?: Record<string, CustomParamItem>;
+        [key: string]: any;
+    };
+    eventAttributes?: {
+        'custom.param'?: Record<string, CustomParamItem>;
+        [key: string]: any;
+    };
+    chatAttributes?: AIStreamChatAttribute;
+    [key: string]: any;
+};
 /**
  * 获取小程序账号信息
  */
@@ -630,9 +647,7 @@ export declare enum EventType {
     /** 聊天中断 */
     CHAT_BREAK = 4,
     /** 服务端 VAD */
-    SERVER_VAD = 5,
-    /** 服务端错误 */
-    SERVER_ERROR = 6
+    SERVER_VAD = 5
 }
 export declare enum StreamFlag {
     /** 仅一包 */
@@ -821,8 +836,6 @@ export type EventBody = {
     sessionId: string;
     /** 事件类型: 0-Event Start, 1-Event Payload End, 2-Event End, 3 - OneShot, 4-Chat Break, 5-Server VAD */
     eventType: EventType;
-    /** 属性列表 */
-    userData?: Attribute[];
 };
 export type AudioBody = {
     /** 接收数据通道 Code */
@@ -831,8 +844,6 @@ export type AudioBody = {
     streamFlag: number;
     /** 音频缓存路径 */
     path: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     /** SessionId 列表, 云端返回，可能为空 */
     sessionIdList?: string[];
     /**
@@ -945,8 +956,6 @@ export type VideoBody = {
     streamFlag: StreamFlag;
     /** 视频缓存路径，传输结束后可访问(streamFlag == 1 || StreamFlag == 3) */
     path: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     /** SessionId 列表, 云端返回，可能为空 */
     sessionIdList?: string[];
 };
@@ -961,8 +970,6 @@ export type ImageBody = {
     width: number;
     /** 图片高度 */
     height: number;
-    /** 扩展属性 */
-    userData?: Attribute[];
     /** SessionId 列表, 云端返回，可能为空 */
     sessionIdList?: string[];
 };
@@ -973,8 +980,6 @@ export type TextBody = {
     streamFlag: StreamFlag;
     /** 文本内容(JSON String，见云端文档定义) */
     text: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     /** SessionId 列表, 云端返回，可能为空 */
     sessionIdList?: string[];
 };
@@ -987,8 +992,6 @@ export type FileBody = {
     path: string;
     /** 文件类型: 1 - MP4, 2 - OGG_OPUS, 3 - PDF, 4 - JSON, 5 - IPC_LOG, 6 - SweeperMap */
     format: FileFormat;
-    /** 扩展属性 */
-    userData?: Attribute[];
     /** SessionId 列表, 云端返回，可能为空 */
     sessionIdList?: string[];
 };
@@ -1037,6 +1040,22 @@ export type GetNetworkTypeParams = {
         };
     }) => void;
 };
+export type EventParams = {
+    /** 事件id */
+    eventId: string;
+    /** 事件点对象 */
+    event: Record<string, any>;
+    complete?: () => void;
+    success?: (params: null) => void;
+    fail?: (params: {
+        errorMsg: string;
+        errorCode: string | number;
+        innerError: {
+            errorCode: string | number;
+            errorMsg: string;
+        };
+    }) => void;
+};
 export type CheckConnectParams = {
     /** client 类型: 1-作为设备代理, 2-作为 App */
     clientType: ConnectClientType;
@@ -1142,8 +1161,13 @@ export type CreateSessionParams = {
     sessionId?: string;
     /** 业务配置定义（通过 queryAgentToken 传递） */
     bizConfig: BizConfig;
-    /** 扩展属性 */
+    /**
+     * 扩展属性
+     * @deprecated
+     */
     userData?: Attribute[];
+    /** 扩展属性 */
+    userDataJson?: string;
     /** 是否复用数据通道，默认不复用 */
     reuseDataChannel?: boolean;
     success?: (params: {
@@ -1211,8 +1235,13 @@ export type SendEventStartParams = {
     /** 会话 id */
     sessionId: string;
     eventIdPrefix?: string;
-    /** 扩展属性 */
+    /**
+     * 扩展属性
+     * @deprecated
+     */
     userData?: Attribute[];
+    /** 扩展属性 */
+    userDataJson?: string;
     success?: (params: {
         /** 事件 id。此后 sendEventPayloadEnd、sendEventEnd、sendEventChatBreak 都需要使用该 eventId 直到下发一次 sendEventStart 生成新 eventId 之前。 */
         eventId: string;
@@ -1228,7 +1257,7 @@ export type SendEventStartParams = {
     complete?: () => void;
 };
 /**
- *@description 某个数据流传输结束事件
+ * @description 某个数据流传输结束事件
  */
 export type SendEventPayloadEndParams = {
     /** 事件 id */
@@ -1237,8 +1266,6 @@ export type SendEventPayloadEndParams = {
     sessionId: string;
     /** 发送结束 的 dataChannel */
     dataChannel: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1258,8 +1285,6 @@ export type SendEventEndParams = {
     eventId: string;
     /** 会话 id */
     sessionId: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1279,8 +1304,13 @@ export type SendEventChatBreakParams = {
     eventId: string;
     /** 会话 id */
     sessionId: string;
-    /** 扩展属性 */
+    /**
+     * 扩展属性
+     * @deprecated
+     */
     userData?: Attribute[];
+    /** 扩展属性 */
+    userDataJson?: string;
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1304,8 +1334,6 @@ export type StartRecordAndSendAudioDataParams = {
     sessionId: string;
     /** 下发数据通道，当音频只有单路的时候，可以不传 */
     dataChannel?: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1322,8 +1350,6 @@ export type StopRecordAndSendAudioDataParams = {
     sessionId: string;
     /** 下发数据通道，当音频只有单路的时候，可以不传 */
     dataChannel?: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: AIStreamAudioFile | null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1372,8 +1398,6 @@ export type StopRecordAndSendVideoDataParams = {
      * 1-前置摄像头, 2-后置摄像头
      */
     cameraType: VideoCameraType;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1392,8 +1416,6 @@ export type SendImageDataParams = {
     dataChannel?: string;
     /** 图片路径 */
     path: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1412,8 +1434,6 @@ export type SendTextDataParams = {
     dataChannel?: string;
     /** 文本内容 */
     text: string;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;
@@ -1447,8 +1467,6 @@ export type SendFileDataParams = {
     path: string;
     /** 文件类型: 1 - MP4, 2 - OGG_OPUS, 3 - PDF, 4 - JSON, 5 - IPC_LOG, 6 - SweeperMap */
     format: FileFormat;
-    /** 扩展属性 */
-    userData?: Attribute[];
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;

package/dist/AIStreamTypes.js

@@ -100,7 +100,6 @@ export let EventType = /*#__PURE__*/function (EventType) {
   EventType[EventType["ONE_SHOT"] = 3] = "ONE_SHOT";
   EventType[EventType["CHAT_BREAK"] = 4] = "CHAT_BREAK";
   EventType[EventType["SERVER_VAD"] = 5] = "SERVER_VAD";
-  EventType[EventType["SERVER_ERROR"] = 6] = "SERVER_ERROR";
   return EventType;
 }({});
 export let StreamFlag = /*#__PURE__*/function (StreamFlag) {
@@ -228,7 +227,7 @@ export let NetworkType = /*#__PURE__*/function (NetworkType) {
  */
 
 /**
- *@description 某个数据流传输结束事件
+ * @description 某个数据流传输结束事件
  */
 
 /**

package/dist/asr/AsrAgent.js

@@ -177,7 +177,7 @@ export class AsrAgent {
       return;
     }
     const activeSession = await this.createSession();
-    const attribute = {
+    const chatAttributes = {
       'processing.interrupt': 'false',
       'asr.enableVad': 'false'
     };
@@ -186,8 +186,11 @@ export class AsrAgent {
       userData: [{
         type: AIStreamAttributeType.AI_CHAT,
         payloadType: AIStreamAttributePayloadType.STRING,
-        value: JSON.stringify(attribute)
-      }]
+        value: JSON.stringify(chatAttributes)
+      }],
+      userDataJson: JSON.stringify({
+        chatAttributes
+      })
     }));
     if (startEventError) {
       finish(startEventError);

package/dist/asr/createAsrAgent.js

@@ -1,4 +1,5 @@
 import { AsrAgent } from './AsrAgent';
+import { trackEvent } from '../utils/track';
 
 /**
  * 创建一个AsrAgent实例，用于语音转文本
@@ -31,5 +32,8 @@ import { AsrAgent } from './AsrAgent';
  */
 export function createAsrAgent(options) {
   const asrAgent = new AsrAgent(options);
+  trackEvent(options.agentId, 'create_asr_agent', {
+    recordingOptions: options.recordingOptions
+  });
   return asrAgent;
 }

package/dist/buildIn/withBuildIn.js

@@ -1,4 +1,3 @@
-import "core-js/modules/web.dom-collections.iterator.js";
 import { BuildInSkillCode, ReceivedSmartHomeSkillAction } from '../AIStreamTypes';
 export function withBuildIn() {
   return _agent => {
@@ -56,13 +55,15 @@ export function withBuildIn() {
           sceneInfo: [],
           changeInfo: []
         };
-        for (const skill of skills) {
+        for (let i = 0; i < skills.length; i++) {
+          const skill = skills[i];
           if (skill.code !== BuildInSkillCode.SMART_HOME) {
             continue;
           }
           const content = skill;
           if (content.general.action === ReceivedSmartHomeSkillAction.QUERY_SCENE) {
-            for (const scene of content.general.data.scenes) {
+            for (let i1 = 0; i1 < content.general.data.scenes.length; i1++) {
+              const scene = content.general.data.scenes[i1];
               executeData.sceneInfo.push({
                 type: scene.type,
                 enabled: scene.enable,
@@ -76,7 +77,8 @@ export function withBuildIn() {
               });
             }
           } else if (content.general.action === ReceivedSmartHomeSkillAction.CONTROL_SCENE) {
-            for (const scene of content.general.data.scenes) {
+            for (let i1 = 0; i1 < content.general.data.scenes.length; i1++) {
+              const scene = content.general.data.scenes[i1];
               operateData.sceneInfo.push({
                 type: scene.type,
                 icon: scene.icon,
@@ -89,7 +91,8 @@ export function withBuildIn() {
               });
             }
           } else if (content.general.action === ReceivedSmartHomeSkillAction.QUERY_DEVICE) {
-            for (const dev of content.general.data.devices) {
+            for (let i1 = 0; i1 < content.general.data.devices.length; i1++) {
+              const dev = content.general.data.devices[i1];
               executeData.deviceInfo.push({
                 icon: dev.icon,
                 name: dev.name,
@@ -100,7 +103,8 @@ export function withBuildIn() {
               });
             }
           } else if (content.general.action === ReceivedSmartHomeSkillAction.CONTROL_DEVICE) {
-            for (const dev of content.general.data.devices) {
+            for (let i1 = 0; i1 < content.general.data.devices.length; i1++) {
+              const dev = content.general.data.devices[i1];
               operateData.deviceInfo.push({
                 icon: dev.icon,
                 name: dev.name,
@@ -128,8 +132,9 @@ export function withBuildIn() {
         const data = {
           documents: []
         };
-        for (const skill of skills) {
+        for (let i = 0; i < skills.length; i++) {
           var _content$custom;
+          const skill = skills[i];
           if (skill.code !== BuildInSkillCode.SEARCH_KNOWLEDGE) {
             continue;
           }
@@ -137,7 +142,8 @@ export function withBuildIn() {
           if (!((_content$custom = content.custom) !== null && _content$custom !== void 0 && (_content$custom = _content$custom.data) !== null && _content$custom !== void 0 && _content$custom.documents)) {
             continue;
           }
-          for (const doc of content.custom.data.documents) {
+          for (let i1 = 0; i1 < content.custom.data.documents.length; i1++) {
+            const doc = content.custom.data.documents[i1];
             data.documents.push({
               title: doc.title,
               url: doc.url

package/dist/utils/AIStream.d.ts

@@ -1,5 +1,6 @@
-import { AIStreamAudioFile, Attribute, BizTag, ConnectClientType, ConnectState, FileFormat, VideoCameraType } from '../AIStreamTypes';
+import { AIStreamAudioFile, AIStreamUserData, Attribute, BizTag, ConnectClientType, ConnectState, FileFormat, VideoCameraType } from '../AIStreamTypes';
 import { AIStreamDataEntry, AIStreamObserverPool } from './observer';
+import { AbortSignalObject } from '@ray-js/t-agent';
 import { AIStreamError } from './errors';
 interface AIStreamConnectionOptions {
     /** client 类型: 1-作为设备代理, 2-作为 App */
@@ -40,12 +41,13 @@ interface AIStreamSessionOptions {
     apiVersion?: string;
     /** 业务额外参数 */
     extParams?: any;
-    userData?: Attribute[];
+    getSessionUserData?: () => Promise<AIStreamUserData>;
 }
 interface AIStreamEventOptions {
-    signal?: AbortSignal;
-    userData?: Attribute[];
+    signal?: AbortSignalObject;
     eventIdPrefix?: string;
+    userData?: Attribute[];
+    userDataJson?: string;
 }
 export declare class AIStreamSession {
     private connection;
@@ -75,30 +77,25 @@ type AIStreamEventWriteChunk = {
     type: 'text';
     text: string;
     dataChannel?: string;
-    userData?: Attribute[];
 } | {
     type: 'file';
     path: string;
     format: FileFormat;
     dataChannel?: string;
-    userData?: Attribute[];
 } | {
     type: 'image';
     path: string;
     dataChannel?: string;
-    userData?: Attribute[];
 };
 type AIStreamEventSource = {
     type: 'audio';
     dataChannel?: string;
-    userData?: Attribute[];
     saveFile?: boolean;
     sampleRate?: number;
 } | {
     type: 'video';
     dataChannel?: string;
     cameraType?: VideoCameraType;
-    userData?: Attribute[];
 };
 export interface AIStreamEventStream {
     type: 'video' | 'audio';
@@ -123,12 +120,8 @@ export declare class AIStreamEvent {
     private findFirstCode;
     write(chunk: AIStreamEventWriteChunk): Promise<void>;
     stream(source: AIStreamEventSource): AIStreamEventStream;
-    end(options?: {
-        userData?: Attribute[];
-    }): Promise<void>;
-    abort(options?: {
-        userData?: Attribute[];
-    }): void;
+    end(): Promise<void>;
+    abort(): void;
     on(name: 'close', callback: () => void): void;
     on(name: 'finish', callback: () => void): void;
     on(name: 'error', callback: (error: AIStreamError) => void): void;