@ray-js/t-agent-plugin-aistream 0.2.0-beta-5 → 0.2.0-beta-7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AIStreamTypes.d.ts +24 -4
- package/dist/AIStreamTypes.js +4 -0
- package/dist/buildIn/index.d.ts +1 -0
- package/dist/buildIn/index.js +1 -0
- package/dist/buildIn/transform.d.ts +10 -0
- package/dist/buildIn/transform.js +179 -0
- package/dist/buildIn/withBuildIn.d.ts +5 -0
- package/dist/buildIn/withBuildIn.js +75 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -1
- package/dist/utils/defaultMock.js +49 -5
- package/dist/utils/index.d.ts +2 -1
- package/dist/utils/index.js +3 -2
- package/dist/utils/mock.d.ts +3 -2
- package/dist/utils/sendMessage.js +12 -5
- package/dist/withAIStream.d.ts +9 -4
- package/dist/withAIStream.js +44 -17
- package/package.json +2 -2
package/dist/AIStreamTypes.d.ts
CHANGED
|
@@ -86,6 +86,7 @@ export type ImagePathInputBlock = {
|
|
|
86
86
|
type: 'image_path';
|
|
87
87
|
image_path: {
|
|
88
88
|
path: string;
|
|
89
|
+
size: number;
|
|
89
90
|
};
|
|
90
91
|
};
|
|
91
92
|
export type FileUrlInputBlock = {
|
|
@@ -101,12 +102,15 @@ export type FilePathInputBlock = {
|
|
|
101
102
|
file_path: {
|
|
102
103
|
mimeType: string;
|
|
103
104
|
path: string;
|
|
105
|
+
size: number;
|
|
104
106
|
};
|
|
105
107
|
};
|
|
106
108
|
export type VideoPathInputBlock = {
|
|
107
109
|
type: 'video_path';
|
|
108
110
|
video_path: {
|
|
109
111
|
path: string;
|
|
112
|
+
thumb_path: string;
|
|
113
|
+
size: number;
|
|
110
114
|
};
|
|
111
115
|
};
|
|
112
116
|
/**
|
|
@@ -1429,9 +1433,25 @@ export type ReceivedTextNlgPacket = ReceivedTextPacketBase<ReceivedTextPacketTyp
|
|
|
1429
1433
|
appendMode: 'append';
|
|
1430
1434
|
finish: boolean;
|
|
1431
1435
|
}>;
|
|
1432
|
-
export
|
|
1433
|
-
|
|
1436
|
+
export declare enum BuildInSkillCode {
|
|
1437
|
+
SEARCH_KNOWLEDGE = "searchKnowledge"
|
|
1438
|
+
}
|
|
1439
|
+
export type ReceivedTextSkillPacketBody<C = BuildInSkillCode | string, T = any> = {
|
|
1440
|
+
code: C;
|
|
1434
1441
|
skillContent: T;
|
|
1435
|
-
text
|
|
1436
|
-
}
|
|
1442
|
+
text?: string[];
|
|
1443
|
+
};
|
|
1444
|
+
export type ReceivedTextSkillPacket<C = BuildInSkillCode | string, T = any> = ReceivedTextPacketBase<ReceivedTextPacketType.SKILL, ReceivedTextSkillPacketBody<C, T>>;
|
|
1437
1445
|
export type ReceivedTextPacket = ReceivedTextAsrPacket | ReceivedTextNlgPacket | ReceivedTextSkillPacket;
|
|
1446
|
+
export type ReceivedSearchKnowledgeSkillContent = {
|
|
1447
|
+
custom: {
|
|
1448
|
+
documents: Array<{
|
|
1449
|
+
libCode: string;
|
|
1450
|
+
itemId: string;
|
|
1451
|
+
itemType: 'KNOWLEDGE';
|
|
1452
|
+
title: string;
|
|
1453
|
+
url: string;
|
|
1454
|
+
contentBody: string;
|
|
1455
|
+
}>;
|
|
1456
|
+
};
|
|
1457
|
+
};
|
package/dist/AIStreamTypes.js
CHANGED
|
@@ -214,4 +214,8 @@ export let ReceivedTextPacketEof = /*#__PURE__*/function (ReceivedTextPacketEof)
|
|
|
214
214
|
ReceivedTextPacketEof[ReceivedTextPacketEof["CONTINUE"] = 0] = "CONTINUE";
|
|
215
215
|
ReceivedTextPacketEof[ReceivedTextPacketEof["END"] = 1] = "END";
|
|
216
216
|
return ReceivedTextPacketEof;
|
|
217
|
+
}({});
|
|
218
|
+
export let BuildInSkillCode = /*#__PURE__*/function (BuildInSkillCode) {
|
|
219
|
+
BuildInSkillCode["SEARCH_KNOWLEDGE"] = "searchKnowledge";
|
|
220
|
+
return BuildInSkillCode;
|
|
217
221
|
}({});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './withBuildIn';
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './withBuildIn';
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { TTTAction } from '../utils';
|
|
2
|
+
import { ButtonRouterType, Recommendation } from '../AIStreamTypes';
|
|
3
|
+
export declare const getActionFromRouter: (type: ButtonRouterType) => {
|
|
4
|
+
i18nKey: string;
|
|
5
|
+
action: TTTAction;
|
|
6
|
+
} | null;
|
|
7
|
+
export declare const getActionFromRecommendation: (item: Recommendation) => {
|
|
8
|
+
text: string;
|
|
9
|
+
action: TTTAction;
|
|
10
|
+
} | null;
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { ButtonRouterType } from '../AIStreamTypes';
|
|
2
|
+
export const getActionFromRouter = type => {
|
|
3
|
+
switch (type) {
|
|
4
|
+
case ButtonRouterType.CREATE_SCENE:
|
|
5
|
+
return {
|
|
6
|
+
i18nKey: 't-agent.build-in.button.create_scene_manually',
|
|
7
|
+
action: {
|
|
8
|
+
type: 'openRoute',
|
|
9
|
+
url: 'thingSmart://thing_add_scene?biz_type=ai_smart&action=add'
|
|
10
|
+
}
|
|
11
|
+
};
|
|
12
|
+
case ButtonRouterType.FAMILY_MANAGE:
|
|
13
|
+
return {
|
|
14
|
+
i18nKey: 't-agent.build-in.button.enter_home_manage',
|
|
15
|
+
action: {
|
|
16
|
+
type: 'openRoute',
|
|
17
|
+
url: 'thingSmart://family_manage'
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
case ButtonRouterType.ROOM_MANAGE:
|
|
21
|
+
return {
|
|
22
|
+
i18nKey: 't-agent.build-in.button.enter_room_manage',
|
|
23
|
+
action: {
|
|
24
|
+
type: 'openRoute',
|
|
25
|
+
url: 'thingSmart://room_manage'
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
case ButtonRouterType.MESSAGE_WARN:
|
|
29
|
+
return {
|
|
30
|
+
i18nKey: 't-agent.build-in.button.enter_alarm_message',
|
|
31
|
+
action: {
|
|
32
|
+
type: 'openRoute',
|
|
33
|
+
url: 'thingSmart://messageCenter?category=0'
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
case ButtonRouterType.MESSAGE_HOME:
|
|
37
|
+
return {
|
|
38
|
+
i18nKey: 't-agent.build-in.button.enter_home_message',
|
|
39
|
+
action: {
|
|
40
|
+
type: 'openRoute',
|
|
41
|
+
url: 'thingSmart://messageCenter?category=1'
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
case ButtonRouterType.MESSAGE_NOTICE:
|
|
45
|
+
return {
|
|
46
|
+
i18nKey: 't-agent.build-in.button.enter_bulletin',
|
|
47
|
+
action: {
|
|
48
|
+
type: 'openRoute',
|
|
49
|
+
url: 'thingSmart://messageCenter?category=2'
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
case ButtonRouterType.MESSAGE_SETTING:
|
|
53
|
+
return {
|
|
54
|
+
i18nKey: 't-agent.build-in.button.enter_notification_setting',
|
|
55
|
+
action: {
|
|
56
|
+
type: 'openRoute',
|
|
57
|
+
url: 'thingSmart://push_setting'
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
case ButtonRouterType.USER_INFO:
|
|
61
|
+
return {
|
|
62
|
+
i18nKey: 't-agent.build-in.button.enter_personal_information',
|
|
63
|
+
action: {
|
|
64
|
+
type: 'openRoute',
|
|
65
|
+
url: 'thingSmart://personal_info'
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
case ButtonRouterType.ACCOUNT_SECURITY:
|
|
69
|
+
return {
|
|
70
|
+
i18nKey: 't-agent.build-in.button.enter_account_security',
|
|
71
|
+
action: {
|
|
72
|
+
type: 'openRoute',
|
|
73
|
+
url: 'thingSmart://account_and_safety'
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
case ButtonRouterType.SETTING:
|
|
77
|
+
return {
|
|
78
|
+
i18nKey: 't-agent.build-in.button.enter_setting',
|
|
79
|
+
action: {
|
|
80
|
+
type: 'openRoute',
|
|
81
|
+
url: 'thingSmart://thing_user_setting'
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
case ButtonRouterType.PARING:
|
|
85
|
+
return {
|
|
86
|
+
i18nKey: 't-agent.build-in.button.enter_paring',
|
|
87
|
+
action: {
|
|
88
|
+
type: 'openRoute',
|
|
89
|
+
url: 'thingSmart://config_device'
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
case ButtonRouterType.DEVICE_SHARE:
|
|
93
|
+
return {
|
|
94
|
+
i18nKey: 't-agent.build-in.button.enter_share_device',
|
|
95
|
+
action: {
|
|
96
|
+
type: 'openRoute',
|
|
97
|
+
// TODO: add devId
|
|
98
|
+
url: "thingSmart://thing_single_device_share?devId="
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
case ButtonRouterType.TICKETS:
|
|
102
|
+
return {
|
|
103
|
+
i18nKey: 't-agent.build-in.button.enter_faq_feedback',
|
|
104
|
+
action: {
|
|
105
|
+
type: 'buildIn',
|
|
106
|
+
name: 'ty.openHelpCenter'
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
case ButtonRouterType.QUESTIONNAIRE:
|
|
110
|
+
return {
|
|
111
|
+
i18nKey: 't-agent.build-in.button.questionnaire_take',
|
|
112
|
+
action: {
|
|
113
|
+
type: 'buildIn',
|
|
114
|
+
name: 'questionnaire'
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
case ButtonRouterType.HOME_LOCATION:
|
|
118
|
+
return {
|
|
119
|
+
i18nKey: 't-agent.build-in.button.set_home_location',
|
|
120
|
+
action: {
|
|
121
|
+
type: 'buildIn',
|
|
122
|
+
name: 'homeLocation'
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
default:
|
|
126
|
+
return undefined;
|
|
127
|
+
}
|
|
128
|
+
};
|
|
129
|
+
export const getActionFromRecommendation = item => {
|
|
130
|
+
const {
|
|
131
|
+
text
|
|
132
|
+
} = item;
|
|
133
|
+
switch (item.type) {
|
|
134
|
+
case 1:
|
|
135
|
+
{
|
|
136
|
+
return {
|
|
137
|
+
text,
|
|
138
|
+
action: {
|
|
139
|
+
type: 'sendMessage',
|
|
140
|
+
blocks: [{
|
|
141
|
+
type: 'text',
|
|
142
|
+
text
|
|
143
|
+
}]
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
case 2:
|
|
148
|
+
{
|
|
149
|
+
return {
|
|
150
|
+
text,
|
|
151
|
+
action: {
|
|
152
|
+
type: 'openH5',
|
|
153
|
+
url: item.h5Url
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
case 3:
|
|
158
|
+
{
|
|
159
|
+
return {
|
|
160
|
+
text,
|
|
161
|
+
action: {
|
|
162
|
+
type: 'openRoute',
|
|
163
|
+
url: item.miniProgramUrl
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
case 4:
|
|
168
|
+
{
|
|
169
|
+
return {
|
|
170
|
+
text,
|
|
171
|
+
action: {
|
|
172
|
+
type: 'openRoute',
|
|
173
|
+
url: item.nativeUrl
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return null;
|
|
179
|
+
};
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import "core-js/modules/web.dom-collections.iterator.js";
|
|
2
|
+
import { getMiniAppConfig } from '../utils';
|
|
3
|
+
import logger from '../utils/logger';
|
|
4
|
+
import { BuildInSkillCode } from '../AIStreamTypes';
|
|
5
|
+
export function withBuildIn() {
|
|
6
|
+
return _agent => {
|
|
7
|
+
if (!_agent.plugins.aiStream || !_agent.plugins.ui) {
|
|
8
|
+
throw new Error('withBuildIn must be used after withAIStream and withUI');
|
|
9
|
+
}
|
|
10
|
+
const agent = _agent;
|
|
11
|
+
const {
|
|
12
|
+
session,
|
|
13
|
+
onTileEvent,
|
|
14
|
+
onAgentStart,
|
|
15
|
+
createMessage
|
|
16
|
+
} = agent;
|
|
17
|
+
const {
|
|
18
|
+
onSkillsEnd
|
|
19
|
+
} = agent.plugins.aiStream;
|
|
20
|
+
onAgentStart(async () => {
|
|
21
|
+
const agentId = session.get('AIStream.agentId');
|
|
22
|
+
if (agentId) {
|
|
23
|
+
const _conf = await session.get('AIAssistant.projectConfig');
|
|
24
|
+
if (!_conf) {
|
|
25
|
+
try {
|
|
26
|
+
const config = (await getMiniAppConfig({})).config || {};
|
|
27
|
+
const projectConfig = config[agentId];
|
|
28
|
+
if (projectConfig) {
|
|
29
|
+
logger.debug('getMiniAppConfig projectConfig', {
|
|
30
|
+
agentId,
|
|
31
|
+
projectConfig
|
|
32
|
+
});
|
|
33
|
+
await session.set('AIAssistant.projectConfig', projectConfig);
|
|
34
|
+
}
|
|
35
|
+
} catch (error) {
|
|
36
|
+
logger.warn('getMiniAppConfig error', error);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
// 关联文档
|
|
43
|
+
|
|
44
|
+
(() => {
|
|
45
|
+
onSkillsEnd((skills, responseMessage) => {
|
|
46
|
+
if (!responseMessage) {
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
const data = {
|
|
50
|
+
documents: []
|
|
51
|
+
};
|
|
52
|
+
for (const skill of skills) {
|
|
53
|
+
var _content$custom;
|
|
54
|
+
if (skill.code !== BuildInSkillCode.SEARCH_KNOWLEDGE) {
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
const content = skill.skillContent;
|
|
58
|
+
if (!((_content$custom = content.custom) !== null && _content$custom !== void 0 && _content$custom.documents)) {
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
for (const doc of content.custom.documents) {
|
|
62
|
+
data.documents.push({
|
|
63
|
+
title: doc.title,
|
|
64
|
+
url: doc.url
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if (data.documents.length) {
|
|
69
|
+
responseMessage.bubble.addTile('documents', data);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
})();
|
|
73
|
+
return {};
|
|
74
|
+
};
|
|
75
|
+
}
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -8,8 +8,9 @@ import "core-js/modules/esnext.iterator.map.js";
|
|
|
8
8
|
import "core-js/modules/web.dom-collections.iterator.js";
|
|
9
9
|
import { mock } from './mock';
|
|
10
10
|
import { EmitterEvent, generateId } from '@ray-js/t-agent';
|
|
11
|
-
import { BizCode, ConnectState, EventType, ReceivedTextPacketEof, ReceivedTextPacketType, StreamFlag } from '../AIStreamTypes';
|
|
11
|
+
import { AIStreamErrorCode, BizCode, ConnectState, EventType, ReceivedTextPacketEof, ReceivedTextPacketType, SessionState, StreamFlag } from '../AIStreamTypes';
|
|
12
12
|
import AbortController from './abort';
|
|
13
|
+
import { tryCatch } from './misc';
|
|
13
14
|
function splitString(input) {
|
|
14
15
|
return input.match(/[a-zA-Z0-9]+\s*|[\u4e00-\u9fff]+\s*|[^\w\s\u4e00-\u9fff]+\s*|[\s]+/g) || [];
|
|
15
16
|
}
|
|
@@ -218,12 +219,31 @@ mock.hooks.hook('sendEventEnd', context => {
|
|
|
218
219
|
context.result = true;
|
|
219
220
|
const event = session.currentEvent;
|
|
220
221
|
(async () => {
|
|
222
|
+
var _ctx$responseSkills;
|
|
221
223
|
await event.asrStatus.promise;
|
|
222
224
|
const ctx = {
|
|
223
225
|
data: event.data,
|
|
224
226
|
responseText: ''
|
|
225
227
|
};
|
|
226
|
-
await mock.hooks.callHook('sendToAIStream', ctx);
|
|
228
|
+
const [error] = await tryCatch(() => mock.hooks.callHook('sendToAIStream', ctx));
|
|
229
|
+
if (error) {
|
|
230
|
+
dispatch('onSessionStateChanged', {
|
|
231
|
+
sessionId: session.sessionId,
|
|
232
|
+
sessionState: SessionState.CLOSED,
|
|
233
|
+
code: AIStreamErrorCode.InternalServerError
|
|
234
|
+
});
|
|
235
|
+
session.closed = true;
|
|
236
|
+
const map = mock.data.get('sessionMap');
|
|
237
|
+
map.delete(session.sessionId);
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
if (ctx.data.length === 0) {
|
|
241
|
+
session.replyEvent(EventType.EVENT_START);
|
|
242
|
+
await mock.sleep(100);
|
|
243
|
+
session.replyEvent(EventType.EVENT_END);
|
|
244
|
+
session.currentEvent = null;
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
227
247
|
const text = ctx.responseText || '⚠️ No mock text response matched!';
|
|
228
248
|
const words = splitString(text);
|
|
229
249
|
if (event.controller.signal.aborted) {
|
|
@@ -270,6 +290,20 @@ mock.hooks.hook('sendEventEnd', context => {
|
|
|
270
290
|
if (event.controller.signal.aborted) {
|
|
271
291
|
return;
|
|
272
292
|
}
|
|
293
|
+
if ((_ctx$responseSkills = ctx.responseSkills) !== null && _ctx$responseSkills !== void 0 && _ctx$responseSkills.length) {
|
|
294
|
+
for (const skill of ctx.responseSkills) {
|
|
295
|
+
await mock.sleep(100);
|
|
296
|
+
if (event.controller.signal.aborted) {
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
session.replyText(StreamFlag.IN_PROGRESS, {
|
|
300
|
+
bizType: ReceivedTextPacketType.SKILL,
|
|
301
|
+
eof: ReceivedTextPacketEof.END,
|
|
302
|
+
bizId,
|
|
303
|
+
data: skill
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
}
|
|
273
307
|
session.replyText(StreamFlag.END);
|
|
274
308
|
await mock.sleep(500);
|
|
275
309
|
if (event.controller.signal.aborted) {
|
|
@@ -315,7 +349,7 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
|
|
|
315
349
|
responseText: ''
|
|
316
350
|
};
|
|
317
351
|
await mock.hooks.callHook('asrDetection', result);
|
|
318
|
-
const responseText = result.responseText || '
|
|
352
|
+
const responseText = result.responseText || '';
|
|
319
353
|
const bizId = generateId();
|
|
320
354
|
let finishResolve;
|
|
321
355
|
session.currentEvent.asrStatus.promise = new Promise(resolve => {
|
|
@@ -325,8 +359,15 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
|
|
|
325
359
|
let text = '';
|
|
326
360
|
controller.signal.addEventListener('abort', async () => {
|
|
327
361
|
var _finishResolve;
|
|
362
|
+
if (!session.currentEvent) {
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
|
|
328
366
|
// 终止识别到出完整结果的延迟
|
|
329
|
-
await mock.sleep(
|
|
367
|
+
await mock.sleep(5000);
|
|
368
|
+
if (!session.currentEvent) {
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
330
371
|
session.replyText(StreamFlag.IN_PROGRESS, {
|
|
331
372
|
bizType: ReceivedTextPacketType.ASR,
|
|
332
373
|
eof: ReceivedTextPacketEof.END,
|
|
@@ -336,8 +377,11 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
|
|
|
336
377
|
}
|
|
337
378
|
});
|
|
338
379
|
await mock.sleep(100);
|
|
380
|
+
if (!session.currentEvent) {
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
339
383
|
session.replyText(StreamFlag.END);
|
|
340
|
-
if (session.currentEvent) {
|
|
384
|
+
if (session.currentEvent && text) {
|
|
341
385
|
session.currentEvent.data.push({
|
|
342
386
|
type: 'text',
|
|
343
387
|
text
|
package/dist/utils/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import { mock } from './mock';
|
|
1
2
|
import './defaultMock';
|
|
2
3
|
export * from './abort';
|
|
3
4
|
export * from './logger';
|
|
4
|
-
export * from './mock';
|
|
5
5
|
export * from './promisify';
|
|
6
6
|
export * from './ttt';
|
|
7
7
|
export * from './url';
|
|
@@ -11,3 +11,4 @@ export * from './sendMessage';
|
|
|
11
11
|
export * from './version';
|
|
12
12
|
export * from './createAsrAgent';
|
|
13
13
|
export * from './actions';
|
|
14
|
+
export { mock };
|
package/dist/utils/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import { mock } from './mock';
|
|
1
2
|
import './defaultMock';
|
|
2
3
|
export * from './abort';
|
|
3
4
|
export * from './logger';
|
|
4
|
-
export * from './mock';
|
|
5
5
|
export * from './promisify';
|
|
6
6
|
export * from './ttt';
|
|
7
7
|
export * from './url';
|
|
@@ -10,4 +10,5 @@ export * from './observer';
|
|
|
10
10
|
export * from './sendMessage';
|
|
11
11
|
export * from './version';
|
|
12
12
|
export * from './createAsrAgent';
|
|
13
|
-
export * from './actions';
|
|
13
|
+
export * from './actions';
|
|
14
|
+
export { mock };
|
package/dist/utils/mock.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { Emitter } from '@ray-js/t-agent';
|
|
2
|
-
import { Attribute } from '../AIStreamTypes';
|
|
2
|
+
import { Attribute, ReceivedTextSkillPacketBody } from '../AIStreamTypes';
|
|
3
3
|
interface TTTCallContext {
|
|
4
4
|
options: any;
|
|
5
5
|
result: any;
|
|
6
6
|
}
|
|
7
|
-
interface SendToAIStreamContext {
|
|
7
|
+
export interface SendToAIStreamContext {
|
|
8
8
|
data: Array<{
|
|
9
9
|
type: 'text';
|
|
10
10
|
text: string;
|
|
@@ -15,6 +15,7 @@ interface SendToAIStreamContext {
|
|
|
15
15
|
userData?: Attribute[];
|
|
16
16
|
}>;
|
|
17
17
|
responseText: string | undefined;
|
|
18
|
+
responseSkills?: ReceivedTextSkillPacketBody[];
|
|
18
19
|
}
|
|
19
20
|
interface AsrDetectionContext {
|
|
20
21
|
responseText: string | undefined;
|
|
@@ -103,11 +103,18 @@ export function sendBlocksToAIStream(params) {
|
|
|
103
103
|
});
|
|
104
104
|
} else if (packet.bizType === ReceivedTextPacketType.ASR && audioEmitter) {
|
|
105
105
|
if (packet.eof === ReceivedTextPacketEof.END) {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
106
|
+
if (packet.data.text === '') {
|
|
107
|
+
// 没识别出任何文本
|
|
108
|
+
audioEmitter.dispatchEvent(new EmitterEvent('error', {
|
|
109
|
+
detail: {}
|
|
110
|
+
}));
|
|
111
|
+
} else {
|
|
112
|
+
audioEmitter.dispatchEvent(new EmitterEvent('finish', {
|
|
113
|
+
detail: {
|
|
114
|
+
text: packet.data.text
|
|
115
|
+
}
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
111
118
|
} else {
|
|
112
119
|
audioEmitter.dispatchEvent(new EmitterEvent('update', {
|
|
113
120
|
detail: {
|
package/dist/withAIStream.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { ChatAgent, ChatMessage,
|
|
2
|
-
import { ConnectClientType } from './AIStreamTypes';
|
|
1
|
+
import { ChatAgent, ChatMessage, GetChatPluginHandler, InputBlock } from '@ray-js/t-agent';
|
|
2
|
+
import { ConnectClientType, ReceivedTextSkillPacketBody } from './AIStreamTypes';
|
|
3
3
|
import { ChatHistoryLocalStore, StoredMessageObject } from './ChatHistoryStore';
|
|
4
4
|
export interface AIStreamOptions {
|
|
5
5
|
/** client 类型: 1-作为设备代理, 2-作为 App */
|
|
@@ -29,11 +29,15 @@ export interface AIStreamOptions {
|
|
|
29
29
|
/** 自定义消息存储, 返回的实例需要实现 ChatHistoryLocalStore 接口, 返回null则不存储历史聊天记录 */
|
|
30
30
|
createChatHistoryStore?: (agent: ChatAgent) => ChatHistoryLocalStore | null;
|
|
31
31
|
}
|
|
32
|
+
export type AIStreamPlugin = GetChatPluginHandler<typeof withAIStream>;
|
|
32
33
|
export interface AIStreamHooks {
|
|
33
34
|
onMessageParse: (msgItem: StoredMessageObject, result: {
|
|
34
35
|
messages: ChatMessage[];
|
|
35
36
|
}) => void;
|
|
36
|
-
onSkillCompose: (skill:
|
|
37
|
+
onSkillCompose: (skill: ReceivedTextSkillPacketBody[], respMsg: ChatMessage, result: {
|
|
38
|
+
messages: ChatMessage[];
|
|
39
|
+
}) => void;
|
|
40
|
+
onSkillsEnd: (skills: ReceivedTextSkillPacketBody[], respMsg: ChatMessage, result: {
|
|
37
41
|
messages: ChatMessage[];
|
|
38
42
|
}) => void;
|
|
39
43
|
}
|
|
@@ -49,10 +53,11 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
|
|
|
49
53
|
responseBy?: string | undefined;
|
|
50
54
|
extraOptions?: Record<string, any> | undefined;
|
|
51
55
|
} | undefined) => Promise<ChatMessage[]>;
|
|
52
|
-
composeHandler: ComposeHandler;
|
|
53
56
|
options: AIStreamOptions;
|
|
54
57
|
removeMessage: (message: ChatMessage) => Promise<void>;
|
|
55
58
|
clearAllMessages: () => Promise<void>;
|
|
59
|
+
onSkillCompose: (fn: AIStreamHooks['onSkillCompose']) => () => void;
|
|
60
|
+
onSkillsEnd: (fn: AIStreamHooks['onSkillsEnd']) => () => void;
|
|
56
61
|
feedback: ({ requestId, type }: {
|
|
57
62
|
requestId: string;
|
|
58
63
|
type: string;
|
package/dist/withAIStream.js
CHANGED
|
@@ -7,7 +7,7 @@ import "core-js/modules/es.array.unscopables.flat.js";
|
|
|
7
7
|
import "core-js/modules/esnext.iterator.constructor.js";
|
|
8
8
|
import "core-js/modules/esnext.iterator.map.js";
|
|
9
9
|
import "core-js/modules/web.dom-collections.iterator.js";
|
|
10
|
-
import { BubbleTileStatus, ChatMessageStatus, createHooks } from '@ray-js/t-agent';
|
|
10
|
+
import { BubbleTileStatus, ChatMessageStatus, createHooks, EmitterEvent } from '@ray-js/t-agent';
|
|
11
11
|
import { messageAppraise } from './utils/apis';
|
|
12
12
|
import { AIStreamObserver, getAccountInfo, getCurrentHomeInfo, runTTTAction, sendBlocksToAIStream } from './utils';
|
|
13
13
|
import { BizCode, ConnectClientType, ConnectState } from './AIStreamTypes';
|
|
@@ -192,17 +192,6 @@ export function withAIStream() {
|
|
|
192
192
|
await removeMessage(message);
|
|
193
193
|
}
|
|
194
194
|
});
|
|
195
|
-
const composeHandler = {
|
|
196
|
-
attachmentCompose: async (respMsg, part) => {
|
|
197
|
-
const result = {
|
|
198
|
-
messages: []
|
|
199
|
-
};
|
|
200
|
-
if (part.attachmentType === 'skill') {
|
|
201
|
-
await hooks.callHook('onSkillCompose', part.attachment, respMsg, result, 'stream');
|
|
202
|
-
}
|
|
203
|
-
return result.messages;
|
|
204
|
-
}
|
|
205
|
-
};
|
|
206
195
|
const send = (blocks, signal, extraOptions) => {
|
|
207
196
|
const streamSession = session.get('AIStream.streamSession');
|
|
208
197
|
const result = sendBlocksToAIStream({
|
|
@@ -211,7 +200,9 @@ export function withAIStream() {
|
|
|
211
200
|
attribute: _objectSpread({}, extraOptions)
|
|
212
201
|
});
|
|
213
202
|
signal === null || signal === void 0 || signal.addEventListener('abort', event => {
|
|
214
|
-
|
|
203
|
+
if (result.response.started) {
|
|
204
|
+
result.response.cancel(event.reason);
|
|
205
|
+
}
|
|
215
206
|
agent.hooks.callHook('onUserAbort', event.reason);
|
|
216
207
|
});
|
|
217
208
|
signal === null || signal === void 0 || signal.throwIfAborted();
|
|
@@ -278,12 +269,25 @@ export function withAIStream() {
|
|
|
278
269
|
resolve();
|
|
279
270
|
};
|
|
280
271
|
audioEmitter.addEventListener('finish', onFinished);
|
|
281
|
-
const onCancel = () => {
|
|
272
|
+
const onCancel = async () => {
|
|
282
273
|
end = true;
|
|
274
|
+
if (!response.started) {
|
|
275
|
+
await userMsg.remove();
|
|
276
|
+
}
|
|
283
277
|
audioEmitter.removeEventListener('cancel', onCancel);
|
|
284
278
|
reject(new Error('User cancel'));
|
|
285
279
|
};
|
|
286
280
|
audioEmitter.addEventListener('cancel', onCancel);
|
|
281
|
+
signal.addEventListener('abort', () => {
|
|
282
|
+
audioEmitter.dispatchEvent(new EmitterEvent('cancel'));
|
|
283
|
+
});
|
|
284
|
+
const onError = async () => {
|
|
285
|
+
end = true;
|
|
286
|
+
await userMsg.remove();
|
|
287
|
+
audioEmitter.removeEventListener('error', onError);
|
|
288
|
+
reject(new Error('Audio emitter error'));
|
|
289
|
+
};
|
|
290
|
+
audioEmitter.addEventListener('error', onError);
|
|
287
291
|
});
|
|
288
292
|
} else {
|
|
289
293
|
userMsg.set({
|
|
@@ -309,7 +313,25 @@ export function withAIStream() {
|
|
|
309
313
|
const message = createMessage({
|
|
310
314
|
role: responseBy
|
|
311
315
|
});
|
|
312
|
-
const
|
|
316
|
+
const skills = [];
|
|
317
|
+
const result = {
|
|
318
|
+
messages: await agent.flushStreamToShow(message, response, {
|
|
319
|
+
attachmentCompose: async (respMsg, part) => {
|
|
320
|
+
const result = {
|
|
321
|
+
messages: []
|
|
322
|
+
};
|
|
323
|
+
if (part.attachmentType === 'skill') {
|
|
324
|
+
skills.push(part.attachment);
|
|
325
|
+
await hooks.callHook('onSkillCompose', part.attachment, respMsg, result);
|
|
326
|
+
}
|
|
327
|
+
return result.messages;
|
|
328
|
+
}
|
|
329
|
+
})
|
|
330
|
+
};
|
|
331
|
+
if (skills.length) {
|
|
332
|
+
await hooks.callHook('onSkillsEnd', skills, message, result);
|
|
333
|
+
await message.update();
|
|
334
|
+
}
|
|
313
335
|
if (message.bubble.status === BubbleTileStatus.ABORTED) {
|
|
314
336
|
if (message.bubble.text) {
|
|
315
337
|
await message.persist();
|
|
@@ -319,7 +341,7 @@ export function withAIStream() {
|
|
|
319
341
|
} else {
|
|
320
342
|
await message.persist();
|
|
321
343
|
}
|
|
322
|
-
return [userMsg, ...messages];
|
|
344
|
+
return [userMsg, ...result.messages];
|
|
323
345
|
};
|
|
324
346
|
onInputBlocksPush(chat);
|
|
325
347
|
const feedback = async _ref => {
|
|
@@ -442,7 +464,6 @@ export function withAIStream() {
|
|
|
442
464
|
aiStream: {
|
|
443
465
|
send,
|
|
444
466
|
chat,
|
|
445
|
-
composeHandler,
|
|
446
467
|
options,
|
|
447
468
|
removeMessage,
|
|
448
469
|
clearAllMessages: async () => {
|
|
@@ -455,6 +476,12 @@ export function withAIStream() {
|
|
|
455
476
|
await historyStore.removeAll();
|
|
456
477
|
}
|
|
457
478
|
},
|
|
479
|
+
onSkillCompose: fn => {
|
|
480
|
+
return hooks.hook('onSkillCompose', fn);
|
|
481
|
+
},
|
|
482
|
+
onSkillsEnd: fn => {
|
|
483
|
+
return hooks.hook('onSkillsEnd', fn);
|
|
484
|
+
},
|
|
458
485
|
feedback
|
|
459
486
|
}
|
|
460
487
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ray-js/t-agent-plugin-aistream",
|
|
3
|
-
"version": "0.2.0-beta-
|
|
3
|
+
"version": "0.2.0-beta-7",
|
|
4
4
|
"author": "Tuya.inc",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"private": false,
|
|
@@ -35,5 +35,5 @@
|
|
|
35
35
|
"devDependencies": {
|
|
36
36
|
"@types/url-parse": "^1.4.11"
|
|
37
37
|
},
|
|
38
|
-
"gitHead": "
|
|
38
|
+
"gitHead": "622a593292ba773b94c6257f7c117d6a2c7baff9"
|
|
39
39
|
}
|