@ray-js/t-agent-plugin-aistream 0.2.7 → 0.2.8-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh_CN.md +286 -95
- package/dist/AIStreamTypes.d.ts +55 -18
- package/dist/AIStreamTypes.js +5 -0
- package/dist/asr/AsrAgent.d.ts +3 -3
- package/dist/asr/AsrAgent.js +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -1
- package/dist/mcp/McpServer.d.ts +11 -0
- package/dist/mcp/McpServer.js +162 -0
- package/dist/mcp/index.d.ts +3 -0
- package/dist/mcp/index.js +3 -0
- package/dist/mcp/types.d.ts +114 -0
- package/dist/mcp/types.js +1 -0
- package/dist/mcp/withMCP.d.ts +15 -0
- package/dist/mcp/withMCP.js +98 -0
- package/dist/polyfill.js +1 -0
- package/dist/utils/AIStream.d.ts +15 -2
- package/dist/utils/AIStream.js +36 -4
- package/dist/utils/defaultMock.js +285 -79
- package/dist/utils/errors.js +3 -1
- package/dist/utils/mock.d.ts +17 -3
- package/dist/utils/object.d.ts +0 -1
- package/dist/utils/object.js +1 -24
- package/dist/utils/observer.d.ts +6 -2
- package/dist/utils/observer.js +17 -6
- package/dist/utils/promisify.d.ts +1 -1
- package/dist/utils/sendMessage.js +9 -7
- package/dist/utils/track.js +2 -2
- package/dist/utils/ttt.d.ts +7 -6
- package/dist/utils/ttt.js +2 -1
- package/dist/withAIStream.d.ts +15 -12
- package/dist/withAIStream.js +12 -6
- package/package.json +5 -3
package/README-zh_CN.md
CHANGED
|
@@ -28,6 +28,8 @@ yarn add @ray-js/t-agent @ray-js/t-agent-plugin-aistream @ray-js/t-agent-ui-ray
|
|
|
28
28
|
}
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
+
> 使用 MCP 功能时,`AIStreamKit` 需要 `2.2.1` 或以上版本。
|
|
32
|
+
|
|
31
33
|
## package.json 依赖要求
|
|
32
34
|
|
|
33
35
|
```json
|
|
@@ -59,59 +61,70 @@ yarn run miniapp
|
|
|
59
61
|
import React from 'react';
|
|
60
62
|
import { View } from '@ray-js/components';
|
|
61
63
|
import { createChatAgent, withDebug, withUI } from '@ray-js/t-agent';
|
|
62
|
-
import {
|
|
64
|
+
import {
|
|
65
|
+
ChatContainer,
|
|
66
|
+
defaultRenderOptions,
|
|
67
|
+
MessageInput,
|
|
68
|
+
MessageList,
|
|
69
|
+
MessageActionBar,
|
|
70
|
+
} from '@ray-js/t-agent-ui-ray';
|
|
63
71
|
import { withAIStream, withBuildIn } from '@ray-js/t-agent-plugin-aistream';
|
|
64
72
|
|
|
65
73
|
const createAgent = () => {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
74
|
+
const agent = createChatAgent(
|
|
75
|
+
withUI(),
|
|
76
|
+
withAIStream({
|
|
77
|
+
earlyStart: true,
|
|
78
|
+
agentId: 'your-agent-id',
|
|
79
|
+
tokenOptions: {
|
|
80
|
+
api: 'm.life.ai.agent.token.get',
|
|
81
|
+
version: '1.0',
|
|
82
|
+
extParams: {
|
|
83
|
+
dialogueMode: 1,
|
|
84
|
+
},
|
|
85
|
+
},
|
|
86
|
+
}),
|
|
87
|
+
withDebug(),
|
|
88
|
+
withBuildIn()
|
|
89
|
+
);
|
|
78
90
|
|
|
79
|
-
|
|
80
|
-
|
|
91
|
+
agent.plugins.aiStream.onUserDataRead((type, data, result) => {
|
|
92
|
+
if (type === 'start-event') {
|
|
93
|
+
result.userData = {
|
|
94
|
+
sessionAttributes: {
|
|
95
|
+
'custom.param': {
|
|
96
|
+
'custom.app.scene': {
|
|
97
|
+
value: 'chat-page',
|
|
98
|
+
},
|
|
99
|
+
},
|
|
100
|
+
},
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
});
|
|
81
104
|
|
|
82
|
-
|
|
83
|
-
onChatStart(async result => {
|
|
84
|
-
const hello = createMessage({
|
|
85
|
-
role: 'assistant',
|
|
86
|
-
});
|
|
105
|
+
const { onChatStart, createMessage } = agent;
|
|
87
106
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
await hello.persist();
|
|
107
|
+
onChatStart(async result => {
|
|
108
|
+
const hello = createMessage({
|
|
109
|
+
role: 'assistant',
|
|
92
110
|
});
|
|
93
111
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
});
|
|
112
|
+
hello.bubble.setText('Hello, world!');
|
|
113
|
+
result.messages.push(hello);
|
|
114
|
+
await hello.persist();
|
|
115
|
+
});
|
|
99
116
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
} catch (error) {
|
|
106
|
-
console.error('Agent creation failed:', error);
|
|
107
|
-
throw error;
|
|
108
|
-
}
|
|
117
|
+
return agent;
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
const renderOptions = {
|
|
121
|
+
...defaultRenderOptions,
|
|
109
122
|
};
|
|
110
123
|
|
|
111
124
|
export default function ChatPage() {
|
|
112
125
|
return (
|
|
113
126
|
<View style={{ height: '100vh' }}>
|
|
114
|
-
<ChatContainer createAgent={createAgent}>
|
|
127
|
+
<ChatContainer createAgent={createAgent} renderOptions={renderOptions}>
|
|
115
128
|
<MessageList />
|
|
116
129
|
<MessageInput />
|
|
117
130
|
<MessageActionBar />
|
|
@@ -790,6 +803,8 @@ const createAgent = () => {
|
|
|
790
803
|
- `options.sendBy` 发送者角色,默认为 'user'
|
|
791
804
|
- `options.responseBy` 响应者角色,默认为 'assistant'
|
|
792
805
|
- `options.userData` 可选的用户数据
|
|
806
|
+
- `agent.plugins.aiStream.removeMessage(message)` 删除一条历史消息
|
|
807
|
+
- `agent.plugins.aiStream.clearAllMessages()` 清空当前会话的所有消息和本地历史
|
|
793
808
|
- `agent.plugins.aiStream.getChatId()` 获取当前会话的 chatId,返回 Promise<string>
|
|
794
809
|
|
|
795
810
|
Hooks:
|
|
@@ -805,7 +820,7 @@ Hooks:
|
|
|
805
820
|
- `status` 消息状态
|
|
806
821
|
- `result.text` 文本内容,可以修改
|
|
807
822
|
- `onSkillCompose` 当收到技能数据时触发,用于处理技能的渲染
|
|
808
|
-
- `skill`
|
|
823
|
+
- `skill` 当前技能数据 (ReceivedTextSkillPacketBody)
|
|
809
824
|
- `respMsg` 响应消息
|
|
810
825
|
- `result.messages` 消息列表
|
|
811
826
|
- `onSkillsEnd` 当所有技能处理完成时触发
|
|
@@ -826,8 +841,99 @@ Hooks:
|
|
|
826
841
|
- `data.blocks` 当 type 为 'start-event' 时,包含本次发送的输入块
|
|
827
842
|
- `result.userData` 返回的用户数据对象,会被合并后发送给 AI 平台
|
|
828
843
|
|
|
844
|
+
### withBuildIn 插件
|
|
845
|
+
|
|
846
|
+
提供了一些内置的功能,比如智能家居、知识库搜索等。
|
|
847
|
+
|
|
848
|
+
**支持的技能**:
|
|
849
|
+
|
|
850
|
+
- **智能家居**:设备控制、场景管理
|
|
851
|
+
- **知识库搜索**:关联文档展示
|
|
852
|
+
|
|
853
|
+
### withMCP 插件
|
|
854
|
+
|
|
855
|
+
提供 MCP 服务暴露能力,可以实现让 Agent 调用小程序里面提供的 tool,实现操作蓝牙设备,调用自定义 api,读取本地文件等功能,使用时先需要在智能体节点上开启“设备 MCP”,此外,MCP tool 的名字必须以 `device.` 开头。
|
|
856
|
+
|
|
857
|
+

|
|
858
|
+
|
|
859
|
+
**使用前提**:
|
|
860
|
+
|
|
861
|
+
- 必须在 `withAIStream` 之后使用
|
|
862
|
+
- 使用 MCP 功能时,`AIStreamKit` 需要 `2.2.1` 或以上版本
|
|
863
|
+
- `createServer` 必须同步返回 `McpServer`
|
|
864
|
+
|
|
865
|
+
参数:
|
|
866
|
+
|
|
867
|
+
- `createServer(agent)` 创建 MCP 服务实例
|
|
868
|
+
- `createContext(event, agent)` 可选,为每次 MCP 请求补充上下文
|
|
869
|
+
|
|
829
870
|
**使用示例**:
|
|
830
871
|
|
|
872
|
+
```tsx
|
|
873
|
+
import { createChatAgent, withUI } from '@ray-js/t-agent';
|
|
874
|
+
import { McpServer, withAIStream, withMCP } from '@ray-js/t-agent-plugin-aistream';
|
|
875
|
+
|
|
876
|
+
const createAgent = () => {
|
|
877
|
+
const server = new McpServer({
|
|
878
|
+
name: 'demo-mcp',
|
|
879
|
+
version: '1.0.0',
|
|
880
|
+
});
|
|
881
|
+
|
|
882
|
+
server.registerTool(
|
|
883
|
+
'device.echo',
|
|
884
|
+
{
|
|
885
|
+
description: '回显输入参数',
|
|
886
|
+
inputSchema: {
|
|
887
|
+
type: 'object',
|
|
888
|
+
properties: {
|
|
889
|
+
text: { type: 'string' },
|
|
890
|
+
},
|
|
891
|
+
},
|
|
892
|
+
},
|
|
893
|
+
async (args, context) => {
|
|
894
|
+
return {
|
|
895
|
+
content: [
|
|
896
|
+
{
|
|
897
|
+
type: 'text',
|
|
898
|
+
text: JSON.stringify({
|
|
899
|
+
text: args.text,
|
|
900
|
+
sessionId: context.sessionId,
|
|
901
|
+
}),
|
|
902
|
+
},
|
|
903
|
+
],
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
);
|
|
907
|
+
|
|
908
|
+
return createChatAgent(
|
|
909
|
+
withUI(),
|
|
910
|
+
withAIStream({
|
|
911
|
+
agentId: 'your-agent-id',
|
|
912
|
+
}),
|
|
913
|
+
withMCP({
|
|
914
|
+
createServer: () => server,
|
|
915
|
+
createContext: event => ({
|
|
916
|
+
traceId: event.eventId,
|
|
917
|
+
}),
|
|
918
|
+
})
|
|
919
|
+
);
|
|
920
|
+
};
|
|
921
|
+
```
|
|
922
|
+
|
|
923
|
+
启用后,插件会:
|
|
924
|
+
|
|
925
|
+
- 在创建 session 时注入 `sessionAttributes.deviceMcp`
|
|
926
|
+
- 在收到 `MCP_CMD` 事件时调用对应 tool
|
|
927
|
+
- 将 tool 执行结果按 JSON-RPC 响应格式回传给 AIStream
|
|
928
|
+
|
|
929
|
+
### 自定义变量传入
|
|
930
|
+
|
|
931
|
+
某些情况下,你可能需要自定义变量到 agent 作为流程中的变量,你需要在工作流里配置自定义变量的引用处,然后再在小程序里传入自定义变量:
|
|
932
|
+
|
|
933
|
+

|
|
934
|
+
|
|
935
|
+
**小程序中传入自定义变量**:
|
|
936
|
+
|
|
831
937
|
```tsx
|
|
832
938
|
const agent = createChatAgent(
|
|
833
939
|
withUI(),
|
|
@@ -839,10 +945,14 @@ const agent = createChatAgent(
|
|
|
839
945
|
agent.plugins.aiStream.onUserDataRead((type, data, result) => {
|
|
840
946
|
// 在创建会话时传递用户信息
|
|
841
947
|
if (type === 'create-session') {
|
|
842
|
-
result.
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
948
|
+
result.userData = {
|
|
949
|
+
sessionAttributes: {
|
|
950
|
+
'custom.param': {
|
|
951
|
+
'custom.app.test_me': {
|
|
952
|
+
// 此处为平台上配置接收的参数名
|
|
953
|
+
value: `with value test me ${type} ${Date.now()}`,
|
|
954
|
+
},
|
|
955
|
+
},
|
|
846
956
|
},
|
|
847
957
|
};
|
|
848
958
|
return;
|
|
@@ -850,9 +960,13 @@ agent.plugins.aiStream.onUserDataRead((type, data, result) => {
|
|
|
850
960
|
// 在每次发送消息时传递动态上下文
|
|
851
961
|
if (type === 'start-event') {
|
|
852
962
|
result.userData = {
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
963
|
+
sessionAttributes: {
|
|
964
|
+
'custom.param': {
|
|
965
|
+
'custom.app.test_me': {
|
|
966
|
+
// 此处为平台上配置接收的参数名
|
|
967
|
+
value: `with value test me ${type} ${Date.now()}`,
|
|
968
|
+
},
|
|
969
|
+
},
|
|
856
970
|
},
|
|
857
971
|
};
|
|
858
972
|
return;
|
|
@@ -860,15 +974,6 @@ agent.plugins.aiStream.onUserDataRead((type, data, result) => {
|
|
|
860
974
|
});
|
|
861
975
|
```
|
|
862
976
|
|
|
863
|
-
### withBuildIn 插件
|
|
864
|
-
|
|
865
|
-
提供了一些内置的功能,比如智能家居、知识库搜索等。
|
|
866
|
-
|
|
867
|
-
**支持的技能**:
|
|
868
|
-
|
|
869
|
-
- **智能家居**:设备控制、场景管理
|
|
870
|
-
- **知识库搜索**:关联文档展示
|
|
871
|
-
|
|
872
977
|
## mock 机制
|
|
873
978
|
|
|
874
979
|
为了方便开发,我们提供了一个 mock 机制,可以在开发时不用连接小程序 AI 智能体平台,直接使用 mock 数据进行开发。
|
|
@@ -918,6 +1023,92 @@ mock.hooks.hook('asrDetection', context => {
|
|
|
918
1023
|
});
|
|
919
1024
|
```
|
|
920
1025
|
|
|
1026
|
+
### mock MCP
|
|
1027
|
+
|
|
1028
|
+
推荐做法是在 mock 的 `sendToAIStream` 钩子里,根据输入内容决定要调用哪个 MCP tool,再通过 `context.callMCPTool()` 直接走一遍完整的 MCP 调用链。
|
|
1029
|
+
|
|
1030
|
+
```tsx
|
|
1031
|
+
import { mock } from '@ray-js/t-agent-plugin-aistream';
|
|
1032
|
+
|
|
1033
|
+
const getToolCall = (text: string) => {
|
|
1034
|
+
if (text.includes('报错') || text.includes('失败') || text.includes('异常')) {
|
|
1035
|
+
return {
|
|
1036
|
+
name: 'device.home.energy.summary.fail',
|
|
1037
|
+
arguments: {
|
|
1038
|
+
reason: 'mock-error-case',
|
|
1039
|
+
},
|
|
1040
|
+
};
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
if (text.includes('能耗') || text.includes('电量') || text.includes('数据')) {
|
|
1044
|
+
return {
|
|
1045
|
+
name: 'device.home.energy.summary.get',
|
|
1046
|
+
arguments: {
|
|
1047
|
+
period: 'today',
|
|
1048
|
+
},
|
|
1049
|
+
};
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
return {
|
|
1053
|
+
name: 'device.app.open',
|
|
1054
|
+
arguments: {
|
|
1055
|
+
category: 'music',
|
|
1056
|
+
},
|
|
1057
|
+
};
|
|
1058
|
+
};
|
|
1059
|
+
|
|
1060
|
+
export const setupMockMCP = () => {
|
|
1061
|
+
mock.hooks.hook('sendToAIStream', async context => {
|
|
1062
|
+
const text = context.data
|
|
1063
|
+
.filter(item => item.type === 'text')
|
|
1064
|
+
.map(item => item.text)
|
|
1065
|
+
.join(' ');
|
|
1066
|
+
|
|
1067
|
+
if (!/mcp|应用|能耗|电量|数据|报错|失败|异常/i.test(text)) {
|
|
1068
|
+
return;
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
const toolCall = getToolCall(text);
|
|
1072
|
+
|
|
1073
|
+
await context.writeText(`MCP mock 正在调用 ${toolCall.name}...`);
|
|
1074
|
+
|
|
1075
|
+
try {
|
|
1076
|
+
const result = await context.callMCPTool(toolCall.name, toolCall.arguments, {
|
|
1077
|
+
delayMs: 80,
|
|
1078
|
+
});
|
|
1079
|
+
await context.writeText(
|
|
1080
|
+
`工具 ${toolCall.name} 已返回:${
|
|
1081
|
+
typeof result === 'string' ? result : JSON.stringify(result == null ? {} : result)
|
|
1082
|
+
}`
|
|
1083
|
+
);
|
|
1084
|
+
} catch (error) {
|
|
1085
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1086
|
+
await context.writeText(`MCP mock 已收到工具报错:${message}`);
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
await context.end();
|
|
1090
|
+
});
|
|
1091
|
+
};
|
|
1092
|
+
```
|
|
1093
|
+
|
|
1094
|
+
使用时在创建 agent 前注册一次即可:
|
|
1095
|
+
|
|
1096
|
+
```tsx
|
|
1097
|
+
const createAgent = () => {
|
|
1098
|
+
setupMockMCP();
|
|
1099
|
+
|
|
1100
|
+
return createChatAgent(
|
|
1101
|
+
withUI(),
|
|
1102
|
+
withAIStream({
|
|
1103
|
+
agentId: 'your-agent-id',
|
|
1104
|
+
}),
|
|
1105
|
+
withMCP({
|
|
1106
|
+
createServer: () => server,
|
|
1107
|
+
})
|
|
1108
|
+
);
|
|
1109
|
+
};
|
|
1110
|
+
```
|
|
1111
|
+
|
|
921
1112
|
## 附带的 utils 工具(现在不稳定,还在开发中)
|
|
922
1113
|
|
|
923
1114
|
### AbortController
|
|
@@ -1112,17 +1303,30 @@ yarn add @ray-js/t-agent-ui-ray
|
|
|
1112
1303
|
## 使用
|
|
1113
1304
|
|
|
1114
1305
|
```tsx
|
|
1115
|
-
import
|
|
1306
|
+
import React from 'react';
|
|
1307
|
+
import { View } from '@ray-js/components';
|
|
1308
|
+
import {
|
|
1309
|
+
ChatContainer,
|
|
1310
|
+
defaultRenderOptions,
|
|
1311
|
+
MessageActionBar,
|
|
1312
|
+
MessageInput,
|
|
1313
|
+
MessageList,
|
|
1314
|
+
} from '@ray-js/t-agent-ui-ray';
|
|
1116
1315
|
// createAgent 实现参见 t-agent 的使用示例
|
|
1117
1316
|
import { createAgent } from './createAgent';
|
|
1118
1317
|
|
|
1318
|
+
const renderOptions = {
|
|
1319
|
+
...defaultRenderOptions,
|
|
1320
|
+
};
|
|
1321
|
+
|
|
1119
1322
|
export default function ChatPage() {
|
|
1120
1323
|
// createAgent 必须返回一个 ChatAgent 应用过 withUI、withAIStream 插件的实例
|
|
1121
1324
|
return (
|
|
1122
1325
|
<View style={{ height: '100vh' }}>
|
|
1123
|
-
<ChatContainer createAgent={createAgent}>
|
|
1326
|
+
<ChatContainer createAgent={createAgent} renderOptions={renderOptions}>
|
|
1124
1327
|
<MessageList />
|
|
1125
1328
|
<MessageInput />
|
|
1329
|
+
<MessageActionBar />
|
|
1126
1330
|
</ChatContainer>
|
|
1127
1331
|
</View>
|
|
1128
1332
|
);
|
|
@@ -1181,7 +1385,16 @@ props:
|
|
|
1181
1385
|
|
|
1182
1386
|
- `className` 输入框的类名
|
|
1183
1387
|
- `placeholder` 输入框的占位符
|
|
1388
|
+
- `placeholderStyle` 输入框 placeholder 的样式
|
|
1184
1389
|
- `renderTop` 用于渲染输入框上方的内容
|
|
1390
|
+
- `style` 输入框容器样式
|
|
1391
|
+
- `attachment` 是否启用附件上传,或传入 `{ image, video, imageCount, videoCount }` 精细控制
|
|
1392
|
+
- `maxTextLength` 文本最大长度,默认 200
|
|
1393
|
+
- `maxAudioMs` 录音最大时长
|
|
1394
|
+
- `onStateChange` 输入框状态变化回调,状态值见 `MessageInputState`
|
|
1395
|
+
- `amplitudeCount` 语音波形振幅采样数量
|
|
1396
|
+
|
|
1397
|
+
`MessageInput` 当前默认导出的是 `MessageInputAIStream`,适用于已接入 `withAIStream` 的场景。
|
|
1185
1398
|
|
|
1186
1399
|
### MessageActionBar(0.2.x 新增)
|
|
1187
1400
|
|
|
@@ -1261,7 +1474,12 @@ const MyTile = ({ notifyHeightChanged }) => {
|
|
|
1261
1474
|
- bubble 气泡
|
|
1262
1475
|
- buttons 按钮组
|
|
1263
1476
|
- card 卡片
|
|
1477
|
+
- divider 分隔线
|
|
1478
|
+
- documents 关联文档
|
|
1479
|
+
- executeCard 执行结果卡片
|
|
1480
|
+
- file 文件
|
|
1264
1481
|
- image 图片
|
|
1482
|
+
- operateCard 操作结果卡片
|
|
1265
1483
|
- recommendations 推荐行动
|
|
1266
1484
|
- text 文本,包含 markdown 支持
|
|
1267
1485
|
- time 时间标识
|
|
@@ -1269,10 +1487,6 @@ const MyTile = ({ notifyHeightChanged }) => {
|
|
|
1269
1487
|
- video 视频
|
|
1270
1488
|
- workflow 工作选项
|
|
1271
1489
|
|
|
1272
|
-
### 内置 card
|
|
1273
|
-
|
|
1274
|
-
- WorkflowReplyCard 工作流回复卡片
|
|
1275
|
-
|
|
1276
1490
|
## React Hooks
|
|
1277
1491
|
|
|
1278
1492
|
### useChatAgent
|
|
@@ -1337,16 +1551,23 @@ const MyTilePart = () => {
|
|
|
1337
1551
|
|
|
1338
1552
|
### useSendAction
|
|
1339
1553
|
|
|
1340
|
-
发送一个 TTTAction
|
|
1554
|
+
发送一个 TTTAction。
|
|
1555
|
+
|
|
1556
|
+
如果当前组件位于 tile 或 card 内部,会优先走 `emitTileEvent`。
|
|
1557
|
+
如果当前组件只是 `ChatContainer` 下的普通子组件,则会退化为触发 UI 事件 `runTTTAction`。
|
|
1341
1558
|
|
|
1342
1559
|
```tsx
|
|
1343
1560
|
import { useSendAction } from '@ray-js/t-agent-ui-ray';
|
|
1344
1561
|
|
|
1345
|
-
const
|
|
1562
|
+
const ActionPanel = () => {
|
|
1346
1563
|
const sendAction = useSendAction();
|
|
1347
1564
|
|
|
1348
1565
|
const handleClick = () => {
|
|
1349
|
-
sendAction({
|
|
1566
|
+
sendAction({
|
|
1567
|
+
type: 'sendMessage',
|
|
1568
|
+
blocks: [{ type: 'text', text: 'hello' }],
|
|
1569
|
+
sendImmediately: true,
|
|
1570
|
+
});
|
|
1350
1571
|
};
|
|
1351
1572
|
|
|
1352
1573
|
return <button onClick={handleClick}>Send Message</button>;
|
|
@@ -1753,33 +1974,3 @@ const renderOptions = {
|
|
|
1753
1974
|
| t-agent.error.stream-exists | 错误提示 | 消息发送异常,请稍后再试 |
|
|
1754
1975
|
| t-agent.error.timeout | 错误提示 | 发送超时 |
|
|
1755
1976
|
| t-agent.error.asr-empty | 错误提示 | 语音识别结果为空 |
|
|
1756
|
-
|
|
1757
|
-
# 更新日志
|
|
1758
|
-
|
|
1759
|
-
## 0.2.x 版本
|
|
1760
|
-
|
|
1761
|
-
### @ray-js/t-agent
|
|
1762
|
-
|
|
1763
|
-
- **Hook 机制增强**:新增 `onMessageFeedback` 和 `onClearHistory` 等生命周期钩子
|
|
1764
|
-
- **消息状态管理**:优化消息状态管理,提供更精确的消息状态控制
|
|
1765
|
-
- **错误处理**:增强错误处理机制,支持更详细的错误信息和错误分类
|
|
1766
|
-
- **性能优化**:优化内存管理和垃圾回收机制
|
|
1767
|
-
|
|
1768
|
-
### @ray-js/t-agent-plugin-aistream
|
|
1769
|
-
|
|
1770
|
-
- **全新插件**:替代废弃的 assistant 插件,提供更强大的功能
|
|
1771
|
-
- **连接优化**:新增 `earlyStart` 参数,支持提前建立连接,减少首次响应时间
|
|
1772
|
-
- **Token 管理**:优化 Token 获取机制,支持自定义 `tokenOptions`
|
|
1773
|
-
- **语音识别**:新增 AsrAgent 语音识别功能,支持实时语音转文字
|
|
1774
|
-
- **Mock 机制**:改进的 mock 机制,支持更灵活的测试场景
|
|
1775
|
-
- **多模态支持**:默认支持文本、图片、语音等多种输入类型
|
|
1776
|
-
|
|
1777
|
-
### @ray-js/t-agent-ui-ray
|
|
1778
|
-
|
|
1779
|
-
- **消息操作栏**:新增 MessageActionBar 组件,支持多选操作和批量删除
|
|
1780
|
-
- **虚拟滚动**:集成 LazyScrollView,提供虚拟滚动和性能优化
|
|
1781
|
-
- **国际化系统**:完整的国际化系统,支持中文简繁体、英文、日文等 8 种语言
|
|
1782
|
-
- **翻译 Hook**:新增 useTranslate Hook,简化多语言使用
|
|
1783
|
-
- **自定义渲染**:扩展 renderOptions,支持更多自定义渲染选项
|
|
1784
|
-
- **交互优化**:优化长按菜单功能,支持复制、删除、多选、点赞等操作
|
|
1785
|
-
- **UI 完善**:新增多语言键值对,覆盖所有 UI 交互场景
|
package/dist/AIStreamTypes.d.ts
CHANGED
|
@@ -647,7 +647,9 @@ export declare enum EventType {
|
|
|
647
647
|
/** 聊天中断 */
|
|
648
648
|
CHAT_BREAK = 4,
|
|
649
649
|
/** 服务端 VAD */
|
|
650
|
-
SERVER_VAD = 5
|
|
650
|
+
SERVER_VAD = 5,
|
|
651
|
+
/** MCP 系统指令 */
|
|
652
|
+
MCP_CMD = 1000
|
|
651
653
|
}
|
|
652
654
|
export declare enum StreamFlag {
|
|
653
655
|
/** 仅一包 */
|
|
@@ -690,18 +692,18 @@ export declare enum AIStreamErrorCode {
|
|
|
690
692
|
ASR_EMPTY = "asr-empty"
|
|
691
693
|
}
|
|
692
694
|
export declare enum AIStreamAppErrorCode {
|
|
693
|
-
GENERIC_ERROR = 39001
|
|
694
|
-
INVALID_PARAMS = 39002
|
|
695
|
-
HTTP_REQUEST_FAILED = 39003
|
|
696
|
-
CONNECTION_INVALID = 39004
|
|
697
|
-
SESSION_ID_INVALID = 39005
|
|
698
|
-
EVENT_ID_INVALID = 39006
|
|
699
|
-
DATA_CHANNEL_NOT_FOUND = 39007
|
|
700
|
-
INVALID_DATA_PACKET = 39008
|
|
701
|
-
FILE_DATA_READ_ERROR = 39009
|
|
702
|
-
DATA_SEND_FAILED = 39010
|
|
703
|
-
CONNECTION_CLOSED_BY_REMOTE = 39012
|
|
704
|
-
AUDIO_UNEXPECTEDLY_RESET = 39014
|
|
695
|
+
GENERIC_ERROR = 39001,// 通用错误,用于一些杂项错误
|
|
696
|
+
INVALID_PARAMS = 39002,// 参数不合法,详见备注
|
|
697
|
+
HTTP_REQUEST_FAILED = 39003,// HTTP 请求失败
|
|
698
|
+
CONNECTION_INVALID = 39004,// Connection 未连接
|
|
699
|
+
SESSION_ID_INVALID = 39005,// SessionId 不存在
|
|
700
|
+
EVENT_ID_INVALID = 39006,// EventId 为空
|
|
701
|
+
DATA_CHANNEL_NOT_FOUND = 39007,// DataChannel 不存在
|
|
702
|
+
INVALID_DATA_PACKET = 39008,// 数据包不合法
|
|
703
|
+
FILE_DATA_READ_ERROR = 39009,// 文件数据读取异常
|
|
704
|
+
DATA_SEND_FAILED = 39010,// 发送数据失败
|
|
705
|
+
CONNECTION_CLOSED_BY_REMOTE = 39012,// Connection 被远端关闭
|
|
706
|
+
AUDIO_UNEXPECTEDLY_RESET = 39014,// 音频组件被重置
|
|
705
707
|
RECONNECT = 39016
|
|
706
708
|
}
|
|
707
709
|
export declare enum AIStreamServerErrorCode {
|
|
@@ -835,8 +837,10 @@ export type EventBody = {
|
|
|
835
837
|
eventId: string;
|
|
836
838
|
/** 会话 id */
|
|
837
839
|
sessionId: string;
|
|
838
|
-
/** 事件类型: 0-Event Start, 1-Event Payload End, 2-Event End, 3 - OneShot, 4-Chat Break, 5-Server VAD */
|
|
840
|
+
/** 事件类型: 0-Event Start, 1-Event Payload End, 2-Event End, 3 - OneShot, 4-Chat Break, 5-Server VAD, 6-MCP_CMD */
|
|
839
841
|
eventType: EventType;
|
|
842
|
+
/** MCP 事件附带的 JSON-RPC payload */
|
|
843
|
+
payload?: string;
|
|
840
844
|
};
|
|
841
845
|
export type AudioBody = {
|
|
842
846
|
/** 接收数据通道 Code */
|
|
@@ -1012,12 +1016,22 @@ export type SessionStateBody = {
|
|
|
1012
1016
|
/** 会话被关闭原因 */
|
|
1013
1017
|
code?: AIStreamServerErrorCode;
|
|
1014
1018
|
};
|
|
1019
|
+
export type SessionEventBody = {
|
|
1020
|
+
/** 事件 id,这个 id 和普通的 event 可能不一样 */
|
|
1021
|
+
eventId: string;
|
|
1022
|
+
/** 会话 id */
|
|
1023
|
+
sessionId: string;
|
|
1024
|
+
/** 事件类型: 1000 MCP_CMD */
|
|
1025
|
+
eventType: EventType;
|
|
1026
|
+
/** 事件附带的 payload */
|
|
1027
|
+
payload?: string;
|
|
1028
|
+
};
|
|
1015
1029
|
export type RecordAmplitudesBody = {
|
|
1016
1030
|
/** 振幅数据 */
|
|
1017
1031
|
amplitudes: number[];
|
|
1018
1032
|
};
|
|
1019
1033
|
export declare enum NetworkType {
|
|
1020
|
-
NONE = "none"
|
|
1034
|
+
NONE = "none",// 无网络
|
|
1021
1035
|
CELL_2G = "2g",
|
|
1022
1036
|
CELL_3G = "3g",
|
|
1023
1037
|
CELL_4G = "4g",
|
|
@@ -1323,6 +1337,29 @@ export type SendEventChatBreakParams = {
|
|
|
1323
1337
|
}) => void;
|
|
1324
1338
|
complete?: () => void;
|
|
1325
1339
|
};
|
|
1340
|
+
/**
|
|
1341
|
+
* @description 发送 AIStream 原始事件
|
|
1342
|
+
*/
|
|
1343
|
+
export type SendAIStreamEventParams = {
|
|
1344
|
+
/** 会话 id */
|
|
1345
|
+
sessionId: string;
|
|
1346
|
+
/** 事件 id */
|
|
1347
|
+
eventId: string;
|
|
1348
|
+
/** 事件类型 */
|
|
1349
|
+
eventType: EventType;
|
|
1350
|
+
/** 扩展属性 */
|
|
1351
|
+
payload?: string;
|
|
1352
|
+
success?: (params: null) => void;
|
|
1353
|
+
fail?: (params: {
|
|
1354
|
+
errorMsg: string;
|
|
1355
|
+
errorCode: string | number;
|
|
1356
|
+
innerError: {
|
|
1357
|
+
errorCode: string | number;
|
|
1358
|
+
errorMsg: string;
|
|
1359
|
+
};
|
|
1360
|
+
}) => void;
|
|
1361
|
+
complete?: () => void;
|
|
1362
|
+
};
|
|
1326
1363
|
/**
|
|
1327
1364
|
*@description 开始录制并发送音频数据
|
|
1328
1365
|
*/
|
|
@@ -1666,8 +1703,8 @@ export type InsertRecordParams = {
|
|
|
1666
1703
|
complete?: () => void;
|
|
1667
1704
|
};
|
|
1668
1705
|
export declare enum ReceivedTextPacketType {
|
|
1669
|
-
ASR = "ASR"
|
|
1670
|
-
SKILL = "SKILL"
|
|
1706
|
+
ASR = "ASR",// ASR 识别的文本
|
|
1707
|
+
SKILL = "SKILL",// 数据
|
|
1671
1708
|
NLG = "NLG"
|
|
1672
1709
|
}
|
|
1673
1710
|
export declare enum ReceivedTextPacketEof {
|
|
@@ -1694,7 +1731,7 @@ export type ReceivedTextNlgPacket = ReceivedTextPacketBase<ReceivedTextPacketTyp
|
|
|
1694
1731
|
finish: boolean;
|
|
1695
1732
|
}>;
|
|
1696
1733
|
export declare enum BuildInSkillCode {
|
|
1697
|
-
SEARCH_KNOWLEDGE = "searchKnowledge"
|
|
1734
|
+
SEARCH_KNOWLEDGE = "searchKnowledge",// 知识库搜索
|
|
1698
1735
|
SMART_HOME = "smart_home"
|
|
1699
1736
|
}
|
|
1700
1737
|
export type ReceivedTextSkillPacketBody<Code = BuildInSkillCode | string, G = any, C = any> = {
|
package/dist/AIStreamTypes.js
CHANGED
|
@@ -100,6 +100,7 @@ export let EventType = /*#__PURE__*/function (EventType) {
|
|
|
100
100
|
EventType[EventType["ONE_SHOT"] = 3] = "ONE_SHOT";
|
|
101
101
|
EventType[EventType["CHAT_BREAK"] = 4] = "CHAT_BREAK";
|
|
102
102
|
EventType[EventType["SERVER_VAD"] = 5] = "SERVER_VAD";
|
|
103
|
+
EventType[EventType["MCP_CMD"] = 1000] = "MCP_CMD";
|
|
103
104
|
return EventType;
|
|
104
105
|
}({});
|
|
105
106
|
export let StreamFlag = /*#__PURE__*/function (StreamFlag) {
|
|
@@ -239,6 +240,10 @@ export let NetworkType = /*#__PURE__*/function (NetworkType) {
|
|
|
239
240
|
*@description 打断事件。用于:1. 中止自己的发送,2. 中止接收云端的回复
|
|
240
241
|
*/
|
|
241
242
|
|
|
243
|
+
/**
|
|
244
|
+
* @description 发送 AIStream 原始事件
|
|
245
|
+
*/
|
|
246
|
+
|
|
242
247
|
/**
|
|
243
248
|
*@description 开始录制并发送音频数据
|
|
244
249
|
*/
|
package/dist/asr/AsrAgent.d.ts
CHANGED
|
@@ -36,9 +36,9 @@ export interface AsrAgentOptions {
|
|
|
36
36
|
eventIdPrefix?: string;
|
|
37
37
|
}
|
|
38
38
|
export declare enum AsrAgentStatus {
|
|
39
|
-
PENDING = "pending"
|
|
40
|
-
RECORDING = "recording"
|
|
41
|
-
ASR = "asr"
|
|
39
|
+
PENDING = "pending",// 等待开始
|
|
40
|
+
RECORDING = "recording",// 录音中
|
|
41
|
+
ASR = "asr",// ASR 处理中
|
|
42
42
|
ABORTING = "aborting"
|
|
43
43
|
}
|
|
44
44
|
export declare class AsrAgent {
|
package/dist/asr/AsrAgent.js
CHANGED
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED