@ray-js/t-agent-plugin-aistream 0.2.8-beta.1 → 0.2.8-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh_CN.md +70 -11
- package/README.md +916 -706
- package/dist/AIStreamTypes.d.ts +36 -17
- package/dist/asr/AsrAgent.d.ts +4 -4
- package/dist/asr/AsrAgent.js +8 -5
- package/dist/buildIn/withBuildIn.d.ts +1 -0
- package/dist/buildIn/withBuildIn.js +162 -25
- package/dist/polyfill.js +1 -0
- package/dist/utils/AIStream.d.ts +16 -3
- package/dist/utils/AIStream.js +84 -41
- package/dist/utils/defaultMock.js +90 -3
- package/dist/utils/errors.js +3 -1
- package/dist/utils/mock.d.ts +18 -3
- package/dist/utils/observer.d.ts +6 -0
- package/dist/utils/observer.js +100 -32
- package/dist/utils/promisify.d.ts +1 -1
- package/dist/utils/sendMessage.d.ts +0 -1
- package/dist/utils/sendMessage.js +26 -13
- package/dist/utils/ttt.d.ts +5 -4
- package/dist/utils/ttt.js +2 -0
- package/dist/withAIStream.d.ts +22 -14
- package/dist/withAIStream.js +94 -58
- package/package.json +5 -4
|
@@ -9,7 +9,7 @@ import "core-js/modules/esnext.iterator.map.js";
|
|
|
9
9
|
import "core-js/modules/web.dom-collections.iterator.js";
|
|
10
10
|
import { mock } from './mock';
|
|
11
11
|
import { EmitterEvent, generateId, safeParseJSON } from '@ray-js/t-agent';
|
|
12
|
-
import { AIStreamAppErrorCode, AIStreamServerErrorCode, BizCode, ConnectState, EventType, NetworkType, ReceivedTextPacketEof, ReceivedTextPacketType, SessionState, StreamFlag } from '../AIStreamTypes';
|
|
12
|
+
import { AIStreamAppErrorCode, AIStreamServerErrorCode, AudioPlayChangedState, BizCode, ConnectState, EventType, NetworkType, ReceivedTextPacketEof, ReceivedTextPacketType, SessionState, StreamFlag } from '../AIStreamTypes';
|
|
13
13
|
import AbortController from './abort';
|
|
14
14
|
import { tryCatch } from './misc';
|
|
15
15
|
import { TTTError } from './errors';
|
|
@@ -309,6 +309,40 @@ mock.hooks.hook('unregisterVoiceAmplitudes', context => {
|
|
|
309
309
|
mock.data.delete('recordAmplitudesCount');
|
|
310
310
|
context.result = true;
|
|
311
311
|
});
|
|
312
|
+
|
|
313
|
+
// 模拟 TTS 音频播放:开始时下发 START,3 秒后下发 COMPLETED。
|
|
314
|
+
mock.hooks.hook('startPlayAudio', context => {
|
|
315
|
+
var _context$options;
|
|
316
|
+
const path = ((_context$options = context.options) === null || _context$options === void 0 ? void 0 : _context$options.path) || '';
|
|
317
|
+
|
|
318
|
+
// 用播放 id 标记本次播放,被主动停止或被新的播放替换时即失效(同一时刻只播一条)
|
|
319
|
+
const playId = generateId();
|
|
320
|
+
mock.data.set('audioPlayId', playId);
|
|
321
|
+
dispatch('onAudioPlayChanged', {
|
|
322
|
+
path,
|
|
323
|
+
state: AudioPlayChangedState.START
|
|
324
|
+
});
|
|
325
|
+
context.result = true;
|
|
326
|
+
|
|
327
|
+
// 模拟 3 秒播放时长,结束后下发 COMPLETED(不阻塞 startPlayAudio 的返回)
|
|
328
|
+
|
|
329
|
+
(async () => {
|
|
330
|
+
await mock.sleep(3000);
|
|
331
|
+
if (mock.data.get('audioPlayId') !== playId) {
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
mock.data.delete('audioPlayId');
|
|
335
|
+
dispatch('onAudioPlayChanged', {
|
|
336
|
+
path,
|
|
337
|
+
state: AudioPlayChangedState.COMPLETED
|
|
338
|
+
});
|
|
339
|
+
})();
|
|
340
|
+
});
|
|
341
|
+
mock.hooks.hook('stopPlayAudio', context => {
|
|
342
|
+
// 主动停止:让当前播放失效,不再下发 COMPLETED(与真实主动停止一致)
|
|
343
|
+
mock.data.delete('audioPlayId');
|
|
344
|
+
context.result = true;
|
|
345
|
+
});
|
|
312
346
|
mock.hooks.hook('sendEventEnd', async context => {
|
|
313
347
|
const map = mock.data.get('sessionMap');
|
|
314
348
|
const session = map.get(context.options.sessionId);
|
|
@@ -430,6 +464,43 @@ mock.hooks.hook('sendEventEnd', async context => {
|
|
|
430
464
|
data: skill
|
|
431
465
|
});
|
|
432
466
|
},
|
|
467
|
+
writeAudio: async (audio, options) => {
|
|
468
|
+
if (!audio) {
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
471
|
+
if (options !== null && options !== void 0 && options.delayMs) {
|
|
472
|
+
await mock.sleep(options.delayMs);
|
|
473
|
+
}
|
|
474
|
+
if (!isEventActive()) {
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
hasStreamedResponse = true;
|
|
478
|
+
|
|
479
|
+
// START 包:携带完整格式头
|
|
480
|
+
dispatch('onAudioReceived', {
|
|
481
|
+
dataChannel: 'audio',
|
|
482
|
+
sessionIdList: [session.sessionId],
|
|
483
|
+
streamFlag: StreamFlag.START,
|
|
484
|
+
path: audio.path,
|
|
485
|
+
pts: audio.pts,
|
|
486
|
+
codecType: audio.codecType,
|
|
487
|
+
sampleRate: audio.sampleRate,
|
|
488
|
+
channels: audio.channels,
|
|
489
|
+
bitDepth: audio.bitDepth
|
|
490
|
+
});
|
|
491
|
+
if (!isEventActive()) {
|
|
492
|
+
return;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// END 包:只带最终 path/pts,不再携带格式头(与真实协议一致)
|
|
496
|
+
dispatch('onAudioReceived', {
|
|
497
|
+
dataChannel: 'audio',
|
|
498
|
+
sessionIdList: [session.sessionId],
|
|
499
|
+
streamFlag: StreamFlag.END,
|
|
500
|
+
path: audio.path,
|
|
501
|
+
pts: audio.pts
|
|
502
|
+
});
|
|
503
|
+
},
|
|
433
504
|
end: async options => {
|
|
434
505
|
await finishEvent((options === null || options === void 0 ? void 0 : options.delayMs) || 0);
|
|
435
506
|
},
|
|
@@ -532,6 +603,22 @@ mock.hooks.hook('sendEventEnd', async context => {
|
|
|
532
603
|
await ctx.writeText(text, {
|
|
533
604
|
wordDelayMs: ctx.wordDelayMs
|
|
534
605
|
});
|
|
606
|
+
|
|
607
|
+
// 开启 TTS 时,模拟下发一段 TTS 音频包,与真实协议保持一致
|
|
608
|
+
if (session.tokenConfig.needTts) {
|
|
609
|
+
await ctx.writeAudio({
|
|
610
|
+
path: "/mock/tts-".concat(generateId(), ".mp3"),
|
|
611
|
+
codecType: 109,
|
|
612
|
+
// MP3
|
|
613
|
+
sampleRate: 16000,
|
|
614
|
+
channels: 0,
|
|
615
|
+
// 单声道
|
|
616
|
+
bitDepth: 16,
|
|
617
|
+
pts: 0
|
|
618
|
+
}, {
|
|
619
|
+
delayMs: 100
|
|
620
|
+
});
|
|
621
|
+
}
|
|
535
622
|
if ((_ctx$responseSkills = ctx.responseSkills) !== null && _ctx$responseSkills !== void 0 && _ctx$responseSkills.length) {
|
|
536
623
|
for (let i = 0; i < ctx.responseSkills.length; i++) {
|
|
537
624
|
await ctx.writeSkill(ctx.responseSkills[i], {
|
|
@@ -569,8 +656,8 @@ mock.hooks.hook('sendEventChatBreak', async context => {
|
|
|
569
656
|
context.result = true;
|
|
570
657
|
});
|
|
571
658
|
mock.hooks.hook('sendEvent', async context => {
|
|
572
|
-
var _context$
|
|
573
|
-
if (((_context$
|
|
659
|
+
var _context$options2;
|
|
660
|
+
if (((_context$options2 = context.options) === null || _context$options2 === void 0 ? void 0 : _context$options2.eventType) !== EventType.MCP_CMD) {
|
|
574
661
|
context.result = true;
|
|
575
662
|
return;
|
|
576
663
|
}
|
package/dist/utils/errors.js
CHANGED
|
@@ -96,9 +96,11 @@ export async function tryCatchTTT(fn) {
|
|
|
96
96
|
code,
|
|
97
97
|
message
|
|
98
98
|
} = error;
|
|
99
|
-
|
|
99
|
+
const originalCode = code;
|
|
100
|
+
message = "".concat(message, " (original_code: ").concat(originalCode || '-', ")");
|
|
100
101
|
code = transformErrorCode(code);
|
|
101
102
|
const e = removeTopStackFrame(new AIStreamError(message, code));
|
|
103
|
+
e.originalCode = originalCode;
|
|
102
104
|
return [e, null];
|
|
103
105
|
}
|
|
104
106
|
return [null, result];
|
package/dist/utils/mock.d.ts
CHANGED
|
@@ -23,6 +23,21 @@ export interface SendToAIStreamContext {
|
|
|
23
23
|
writeSkill: (skill: ReceivedTextSkillPacketBody, options?: {
|
|
24
24
|
delayMs?: number;
|
|
25
25
|
}) => Promise<void>;
|
|
26
|
+
/**
|
|
27
|
+
* 模拟一段流式音频(TTS)下发。
|
|
28
|
+
* 真实协议里格式头(codecType/sampleRate/channels/bitDepth)只在 START 包携带,
|
|
29
|
+
* END 包通常只带最终 path/pts,这里据实模拟,分两包下发。
|
|
30
|
+
*/
|
|
31
|
+
writeAudio: (audio: {
|
|
32
|
+
path: string;
|
|
33
|
+
codecType?: number;
|
|
34
|
+
sampleRate?: number;
|
|
35
|
+
channels?: number;
|
|
36
|
+
bitDepth?: number;
|
|
37
|
+
pts?: number;
|
|
38
|
+
}, options?: {
|
|
39
|
+
delayMs?: number;
|
|
40
|
+
}) => Promise<void>;
|
|
26
41
|
end: (options?: {
|
|
27
42
|
delayMs?: number;
|
|
28
43
|
}) => Promise<void>;
|
|
@@ -54,9 +69,9 @@ declare const mock: {
|
|
|
54
69
|
emitter: Emitter;
|
|
55
70
|
hooks: import("hookable").Hookable<MockHooks, string>;
|
|
56
71
|
getRecords: <T = any>() => Row<T>[];
|
|
57
|
-
getRecord: <
|
|
58
|
-
setRecord: <
|
|
59
|
-
updateRecord: <
|
|
72
|
+
getRecord: <T = any>(id: number) => Row<T> | undefined;
|
|
73
|
+
setRecord: <T = any>(entry: Row<T>) => void;
|
|
74
|
+
updateRecord: <T = any>(entry: Row<T>) => void;
|
|
60
75
|
deleteRecord: (id: number) => void;
|
|
61
76
|
clearRecords: () => void;
|
|
62
77
|
getId: () => number;
|
package/dist/utils/observer.d.ts
CHANGED
|
@@ -45,6 +45,12 @@ export declare class AIStreamObserverPool {
|
|
|
45
45
|
private observerMap;
|
|
46
46
|
constructor();
|
|
47
47
|
private cancels;
|
|
48
|
+
private createObserverMap;
|
|
49
|
+
private isSessionAwareType;
|
|
50
|
+
private getEntrySessionIds;
|
|
51
|
+
private collectObservers;
|
|
52
|
+
private addObserver;
|
|
53
|
+
private removeObserver;
|
|
48
54
|
start(): void;
|
|
49
55
|
dispose(): void;
|
|
50
56
|
connect(observer: AIStreamObserver): void;
|
package/dist/utils/observer.js
CHANGED
|
@@ -10,25 +10,102 @@ export class AIStreamObserverPool {
|
|
|
10
10
|
constructor() {
|
|
11
11
|
_defineProperty(this, "isStarted", false);
|
|
12
12
|
_defineProperty(this, "cancels", []);
|
|
13
|
-
this.observerMap =
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
13
|
+
this.observerMap = this.createObserverMap();
|
|
14
|
+
}
|
|
15
|
+
createObserverMap() {
|
|
16
|
+
return {
|
|
17
|
+
connectionState: {
|
|
18
|
+
all: new Set(),
|
|
19
|
+
bySessionId: new Map()
|
|
20
|
+
},
|
|
21
|
+
sessionState: {
|
|
22
|
+
all: new Set(),
|
|
23
|
+
bySessionId: new Map()
|
|
24
|
+
},
|
|
25
|
+
sessionEvent: {
|
|
26
|
+
all: new Set(),
|
|
27
|
+
bySessionId: new Map()
|
|
28
|
+
},
|
|
29
|
+
event: {
|
|
30
|
+
all: new Set(),
|
|
31
|
+
bySessionId: new Map()
|
|
32
|
+
},
|
|
33
|
+
text: {
|
|
34
|
+
all: new Set(),
|
|
35
|
+
bySessionId: new Map()
|
|
36
|
+
},
|
|
37
|
+
audio: {
|
|
38
|
+
all: new Set(),
|
|
39
|
+
bySessionId: new Map()
|
|
40
|
+
},
|
|
41
|
+
video: {
|
|
42
|
+
all: new Set(),
|
|
43
|
+
bySessionId: new Map()
|
|
44
|
+
},
|
|
45
|
+
file: {
|
|
46
|
+
all: new Set(),
|
|
47
|
+
bySessionId: new Map()
|
|
48
|
+
},
|
|
49
|
+
image: {
|
|
50
|
+
all: new Set(),
|
|
51
|
+
bySessionId: new Map()
|
|
52
|
+
}
|
|
23
53
|
};
|
|
24
54
|
}
|
|
55
|
+
isSessionAwareType(type) {
|
|
56
|
+
return type !== 'connectionState';
|
|
57
|
+
}
|
|
58
|
+
getEntrySessionIds(entry) {
|
|
59
|
+
if (entry.type === 'event' || entry.type === 'sessionEvent' || entry.type === 'sessionState') {
|
|
60
|
+
return entry.body.sessionId ? [entry.body.sessionId] : [];
|
|
61
|
+
}
|
|
62
|
+
if (entry.type === 'text' || entry.type === 'audio' || entry.type === 'video' || entry.type === 'file' || entry.type === 'image') {
|
|
63
|
+
return entry.body.sessionIdList || [];
|
|
64
|
+
}
|
|
65
|
+
return [];
|
|
66
|
+
}
|
|
67
|
+
collectObservers(entry) {
|
|
68
|
+
const bucket = this.observerMap[entry.type];
|
|
69
|
+
const observers = new Set(bucket.all);
|
|
70
|
+
this.getEntrySessionIds(entry).forEach(sessionId => {
|
|
71
|
+
const sessionObservers = bucket.bySessionId.get(sessionId);
|
|
72
|
+
sessionObservers === null || sessionObservers === void 0 || sessionObservers.forEach(observer => {
|
|
73
|
+
observers.add(observer);
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
return observers;
|
|
77
|
+
}
|
|
78
|
+
addObserver(type, observer) {
|
|
79
|
+
const sessionId = observer.options.sessionId;
|
|
80
|
+
const bucket = this.observerMap[type];
|
|
81
|
+
if (!sessionId || !this.isSessionAwareType(type)) {
|
|
82
|
+
bucket.all.add(observer);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
let sessionObservers = bucket.bySessionId.get(sessionId);
|
|
86
|
+
if (!sessionObservers) {
|
|
87
|
+
sessionObservers = new Set();
|
|
88
|
+
bucket.bySessionId.set(sessionId, sessionObservers);
|
|
89
|
+
}
|
|
90
|
+
sessionObservers.add(observer);
|
|
91
|
+
}
|
|
92
|
+
removeObserver(type, observer) {
|
|
93
|
+
const bucket = this.observerMap[type];
|
|
94
|
+
bucket.all.delete(observer);
|
|
95
|
+
bucket.bySessionId.forEach((sessionObservers, sessionId) => {
|
|
96
|
+
sessionObservers.delete(observer);
|
|
97
|
+
if (sessionObservers.size === 0) {
|
|
98
|
+
bucket.bySessionId.delete(sessionId);
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
}
|
|
25
102
|
start() {
|
|
26
103
|
if (this.isStarted) {
|
|
27
104
|
return;
|
|
28
105
|
}
|
|
29
106
|
const handle = entry => {
|
|
30
107
|
var _entry$body;
|
|
31
|
-
const observers = this.
|
|
108
|
+
const observers = this.collectObservers(entry);
|
|
32
109
|
let state = '';
|
|
33
110
|
if (entry.type === 'connectionState') {
|
|
34
111
|
state = "".concat(ConnectState[entry.body.connectState], " ").concat(entry.body.code || '');
|
|
@@ -47,18 +124,19 @@ export class AIStreamObserverPool {
|
|
|
47
124
|
sessionId
|
|
48
125
|
} = observer.options;
|
|
49
126
|
let matched = true;
|
|
127
|
+
const sessionIdList = entry.body.sessionIdList || [];
|
|
50
128
|
if (dataChannels !== null && dataChannels !== void 0 && dataChannels.length && !dataChannels.includes(entry.body.dataChannel)) {
|
|
51
129
|
matched = false;
|
|
52
130
|
}
|
|
53
|
-
if (sessionId && !
|
|
131
|
+
if (sessionId && !sessionIdList.includes(sessionId)) {
|
|
54
132
|
matched = false;
|
|
55
133
|
}
|
|
56
134
|
if (matched) {
|
|
57
135
|
observer.callback(entry, observer);
|
|
58
136
|
}
|
|
59
|
-
} else {
|
|
137
|
+
} else if (!((entry.type === 'event' || entry.type === 'sessionEvent' || entry.type === 'sessionState') && observer.options.sessionId && observer.options.sessionId !== entry.body.sessionId)) {
|
|
60
138
|
observer.callback(entry, observer);
|
|
61
|
-
}
|
|
139
|
+
} // noop
|
|
62
140
|
});
|
|
63
141
|
};
|
|
64
142
|
this.cancels = [listenConnectStateChanged(body => handle({
|
|
@@ -98,36 +176,26 @@ export class AIStreamObserverPool {
|
|
|
98
176
|
this.isStarted = false;
|
|
99
177
|
this.cancels.forEach(cancel => cancel());
|
|
100
178
|
this.cancels = [];
|
|
101
|
-
this.observerMap =
|
|
102
|
-
connectionState: new Set(),
|
|
103
|
-
sessionState: new Set(),
|
|
104
|
-
sessionEvent: new Set(),
|
|
105
|
-
event: new Set(),
|
|
106
|
-
text: new Set(),
|
|
107
|
-
audio: new Set(),
|
|
108
|
-
video: new Set(),
|
|
109
|
-
file: new Set(),
|
|
110
|
-
image: new Set()
|
|
111
|
-
};
|
|
179
|
+
this.observerMap = this.createObserverMap();
|
|
112
180
|
}
|
|
113
181
|
connect(observer) {
|
|
114
182
|
if (!this.isStarted) {
|
|
115
183
|
this.start();
|
|
116
184
|
}
|
|
117
|
-
types.forEach(
|
|
118
|
-
const type = key;
|
|
185
|
+
types.forEach(type => {
|
|
119
186
|
if (observer.options[type]) {
|
|
120
|
-
this.
|
|
187
|
+
this.addObserver(type, observer);
|
|
121
188
|
}
|
|
122
189
|
});
|
|
123
190
|
}
|
|
124
191
|
disconnect(observer) {
|
|
125
|
-
types.forEach(
|
|
126
|
-
this.
|
|
192
|
+
types.forEach(type => {
|
|
193
|
+
this.removeObserver(type, observer);
|
|
127
194
|
});
|
|
128
195
|
let empty = true;
|
|
129
|
-
types.forEach(
|
|
130
|
-
|
|
196
|
+
types.forEach(type => {
|
|
197
|
+
const bucket = this.observerMap[type];
|
|
198
|
+
if (bucket.all.size > 0 || bucket.bySessionId.size > 0) {
|
|
131
199
|
empty = false;
|
|
132
200
|
}
|
|
133
201
|
});
|
|
@@ -13,6 +13,6 @@ interface AsyncTTTFnParams<P> {
|
|
|
13
13
|
}
|
|
14
14
|
export declare const getEnableMock: () => boolean;
|
|
15
15
|
export declare const setEnableMock: (enable: boolean) => boolean;
|
|
16
|
-
export declare function promisify<T extends AsyncTTTFnParams<any>>(fn: (options: any) => void, enableMock?: boolean): (options?: Omit<T,
|
|
16
|
+
export declare function promisify<T extends AsyncTTTFnParams<any>>(fn: (options: any) => void, enableMock?: boolean): (options?: Omit<T, "success" | "fail">) => Promise<Parameters<NonNullable<T["success"]>>[0]>;
|
|
17
17
|
export declare function listening<T>(on: (listener: (params: T) => void) => void, off: (listener: (params: T) => void) => void, enableMock?: boolean): (listener: (params: T) => void) => () => void;
|
|
18
18
|
export {};
|
|
@@ -65,8 +65,10 @@ export function sendBlocksToAIStream(params) {
|
|
|
65
65
|
if (!closed) {
|
|
66
66
|
logger.debug('sendBlocksToAIStream close');
|
|
67
67
|
closed = true;
|
|
68
|
-
|
|
68
|
+
try {
|
|
69
69
|
controller.close();
|
|
70
|
+
} catch (error) {
|
|
71
|
+
logger.debug('sendBlocksToAIStream close ignored', error);
|
|
70
72
|
}
|
|
71
73
|
}
|
|
72
74
|
};
|
|
@@ -217,15 +219,17 @@ export function sendBlocksToAIStream(params) {
|
|
|
217
219
|
if (!imageId) {
|
|
218
220
|
imageId = generateId();
|
|
219
221
|
}
|
|
222
|
+
const attachment = {
|
|
223
|
+
streamFlag: data.body.streamFlag,
|
|
224
|
+
path: data.body.path,
|
|
225
|
+
width: data.body.width,
|
|
226
|
+
height: data.body.height
|
|
227
|
+
};
|
|
220
228
|
enqueue({
|
|
221
229
|
id: imageId,
|
|
222
230
|
type: 'attachment',
|
|
223
231
|
attachmentType: 'image',
|
|
224
|
-
attachment:
|
|
225
|
-
path: data.body.path || null,
|
|
226
|
-
width: data.body.width || null,
|
|
227
|
-
height: data.body.height || null
|
|
228
|
-
},
|
|
232
|
+
attachment: attachment,
|
|
229
233
|
meta
|
|
230
234
|
});
|
|
231
235
|
if (data.body.streamFlag === StreamFlag.END) {
|
|
@@ -235,13 +239,20 @@ export function sendBlocksToAIStream(params) {
|
|
|
235
239
|
if (!audioId) {
|
|
236
240
|
audioId = generateId();
|
|
237
241
|
}
|
|
242
|
+
const attachment = {
|
|
243
|
+
streamFlag: data.body.streamFlag,
|
|
244
|
+
path: data.body.path,
|
|
245
|
+
pts: data.body.pts,
|
|
246
|
+
codecType: data.body.codecType,
|
|
247
|
+
sampleRate: data.body.sampleRate,
|
|
248
|
+
channels: data.body.channels,
|
|
249
|
+
bitDepth: data.body.bitDepth
|
|
250
|
+
};
|
|
238
251
|
enqueue({
|
|
239
252
|
id: audioId,
|
|
240
253
|
type: 'attachment',
|
|
241
254
|
attachmentType: 'audio',
|
|
242
|
-
attachment
|
|
243
|
-
path: data.body.path || null
|
|
244
|
-
},
|
|
255
|
+
attachment,
|
|
245
256
|
meta
|
|
246
257
|
});
|
|
247
258
|
if (data.body.streamFlag === StreamFlag.END) {
|
|
@@ -251,7 +262,7 @@ export function sendBlocksToAIStream(params) {
|
|
|
251
262
|
if (data.body.sessionState === SessionState.CLOSED || data.body.sessionState === SessionState.CREATE_FAILED) {
|
|
252
263
|
const msg = SessionState[data.body.sessionState];
|
|
253
264
|
const e = new AIStreamError("Session Error: ".concat(msg, ", (original_code: ").concat(data.body.code, ")"), transformErrorCode(data.body.code));
|
|
254
|
-
emitError(
|
|
265
|
+
emitError(e);
|
|
255
266
|
if (audioEmitter) {
|
|
256
267
|
audioEmitter.dispatchEvent(new EmitterEvent('error', {
|
|
257
268
|
detail: e
|
|
@@ -262,7 +273,7 @@ export function sendBlocksToAIStream(params) {
|
|
|
262
273
|
} else if (data.type === 'connectionState') {
|
|
263
274
|
if (data.body.connectState === ConnectState.DISCONNECTED || data.body.connectState === ConnectState.CLOSED) {
|
|
264
275
|
const e = new AIStreamError("Session disconnected, (original_code: ".concat(data.body.code, ")"), transformErrorCode(data.body.code));
|
|
265
|
-
emitError(
|
|
276
|
+
emitError(e);
|
|
266
277
|
if (audioEmitter) {
|
|
267
278
|
audioEmitter.dispatchEvent(new EmitterEvent('error', {
|
|
268
279
|
detail: e
|
|
@@ -303,14 +314,14 @@ export function sendBlocksToAIStream(params) {
|
|
|
303
314
|
} else if (block.type === 'file_path') {
|
|
304
315
|
event.write({
|
|
305
316
|
type: 'file',
|
|
306
|
-
format: mimeTypeToFormatMap[block.
|
|
317
|
+
format: mimeTypeToFormatMap[block.file_path.mimeType] || 0,
|
|
307
318
|
path: block.file_path.path
|
|
308
319
|
}).catch(emitError);
|
|
309
320
|
} else if (block.type === 'video_path') {
|
|
310
321
|
event.write({
|
|
311
322
|
type: 'file',
|
|
312
323
|
format: FileFormat.MP4,
|
|
313
|
-
path: block.
|
|
324
|
+
path: block.video_path.path
|
|
314
325
|
}).catch(emitError);
|
|
315
326
|
}
|
|
316
327
|
}
|
|
@@ -378,6 +389,8 @@ export function sendBlocksToAIStream(params) {
|
|
|
378
389
|
}
|
|
379
390
|
});
|
|
380
391
|
return {
|
|
392
|
+
// web-streams-polyfill's ReadableStream is runtime-compatible but its
|
|
393
|
+
// getReader() overloads differ from lib.dom's, so cast to satisfy the type.
|
|
381
394
|
response: new StreamResponse(stream),
|
|
382
395
|
metaPromise
|
|
383
396
|
};
|
package/dist/utils/ttt.d.ts
CHANGED
|
@@ -36,11 +36,11 @@ export declare const getAppInfo: (options?: Omit<GetAppInfoParams, "success" | "
|
|
|
36
36
|
regionCode: string;
|
|
37
37
|
appName: string;
|
|
38
38
|
appIcon: string;
|
|
39
|
-
appEnv?: number
|
|
39
|
+
appEnv?: number;
|
|
40
40
|
appBundleId: string;
|
|
41
41
|
appScheme: string;
|
|
42
42
|
appId: string;
|
|
43
|
-
clientId?: string
|
|
43
|
+
clientId?: string;
|
|
44
44
|
}>;
|
|
45
45
|
export declare const canIUseRouter: (options?: Omit<CanIUseRouterParams, "success" | "fail"> | undefined) => Promise<{
|
|
46
46
|
result: boolean;
|
|
@@ -57,7 +57,7 @@ export declare const openMiniWidget: (options?: Omit<OpenMiniWidgetParams, "succ
|
|
|
57
57
|
export declare const isConnected: (options?: Omit<IsConnectedParams, "success" | "fail"> | undefined) => Promise<{
|
|
58
58
|
connected: boolean;
|
|
59
59
|
state: number;
|
|
60
|
-
connectionId?: string
|
|
60
|
+
connectionId?: string;
|
|
61
61
|
}>;
|
|
62
62
|
export declare const connect: (options?: Omit<ConnectParams, "success" | "fail"> | undefined) => Promise<{
|
|
63
63
|
connectionId: string;
|
|
@@ -66,12 +66,13 @@ export declare const disconnect: (options?: Omit<DisconnectParams, "success" | "
|
|
|
66
66
|
export declare const queryAgentToken: (options?: Omit<QueryAgentTokenParams, "success" | "fail"> | undefined) => Promise<{
|
|
67
67
|
agentToken: string;
|
|
68
68
|
bizConfig: import("../AIStreamTypes").BizConfig;
|
|
69
|
-
extParams?: string
|
|
69
|
+
extParams?: string;
|
|
70
70
|
}>;
|
|
71
71
|
export declare const createSession: (options?: Omit<CreateSessionParams, "success" | "fail"> | undefined) => Promise<{
|
|
72
72
|
sessionId: string;
|
|
73
73
|
sendDataChannels: string[];
|
|
74
74
|
revDataChannels: string[];
|
|
75
|
+
baseCacheDir: string;
|
|
75
76
|
}>;
|
|
76
77
|
export declare const closeSession: (options?: Omit<CloseSessionParams, "success" | "fail"> | undefined) => Promise<null>;
|
|
77
78
|
export declare const sendEventStart: (options?: Omit<SendEventStartParams, "success" | "fail"> | undefined) => Promise<{
|
package/dist/utils/ttt.js
CHANGED
|
@@ -42,6 +42,8 @@ export const sendEventStart = promisify(ty.aistream.sendEventStart, true);
|
|
|
42
42
|
export const sendEventPayloadEnd = promisify(ty.aistream.sendEventPayloadEnd, true);
|
|
43
43
|
export const sendEventEnd = promisify(ty.aistream.sendEventEnd, true);
|
|
44
44
|
export const sendEventChatBreak = promisify(ty.aistream.sendEventChatBreak, true);
|
|
45
|
+
|
|
46
|
+
// @ts-ignore
|
|
45
47
|
export const sendEventToAIStream = promisify(ty.aistream.sendEvent, true);
|
|
46
48
|
export const startRecordAndSendAudioData = promisify(ty.aistream.startRecordAndSendAudioData, true);
|
|
47
49
|
export const stopRecordAndSendAudioData = promisify(ty.aistream.stopRecordAndSendAudioData, true);
|
package/dist/withAIStream.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AbortSignalObject, ChatAgent, ChatCardObject, ChatMessage, ChatMessageStatus, ChatTile, GetChatPluginHandler, InputBlock } from '@ray-js/t-agent';
|
|
1
|
+
import { AbortSignalObject, AttachmentPart, ChatAgent, ChatCardObject, ChatMessage, ChatMessageStatus, ChatTile, GetChatPluginHandler, InputBlock } from '@ray-js/t-agent';
|
|
2
2
|
import { TTTAction } from './utils';
|
|
3
3
|
import { AIStreamUserData, ConnectClientType, ReceivedTextSkillPacketBody, SendAIStreamEventParams, SessionEventBody } from './AIStreamTypes';
|
|
4
4
|
import { ChatHistoryStore, StoredMessageObject } from './ChatHistoryStore';
|
|
@@ -46,7 +46,13 @@ export interface AIStreamHooks {
|
|
|
46
46
|
onTextCompose: (respMsg: ChatMessage, status: ChatMessageStatus, result: {
|
|
47
47
|
text: string;
|
|
48
48
|
}) => void;
|
|
49
|
-
onSkillCompose: (skill: ReceivedTextSkillPacketBody
|
|
49
|
+
onSkillCompose: (skill: ReceivedTextSkillPacketBody, respMsg: ChatMessage, result: {
|
|
50
|
+
messages: ChatMessage[];
|
|
51
|
+
}) => void;
|
|
52
|
+
onAttachmentCompose: (part: AttachmentPart, respMsg: ChatMessage, result: {
|
|
53
|
+
messages: ChatMessage[];
|
|
54
|
+
}) => void;
|
|
55
|
+
onAttachmentsEnd: (parts: AttachmentPart[], respMsg: ChatMessage, result: {
|
|
50
56
|
messages: ChatMessage[];
|
|
51
57
|
}) => void;
|
|
52
58
|
onSkillsEnd: (skills: ReceivedTextSkillPacketBody[], respMsg: ChatMessage, result: {
|
|
@@ -73,10 +79,10 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
|
|
|
73
79
|
metaPromise: Promise<Record<string, any>>;
|
|
74
80
|
};
|
|
75
81
|
chat: (blocks: InputBlock[], signal?: AbortSignalObject, options?: {
|
|
76
|
-
sendBy?: string
|
|
77
|
-
responseBy?: string
|
|
78
|
-
userData?: AIStreamUserData
|
|
79
|
-
}
|
|
82
|
+
sendBy?: string;
|
|
83
|
+
responseBy?: string;
|
|
84
|
+
userData?: AIStreamUserData;
|
|
85
|
+
}) => Promise<ChatMessage[]>;
|
|
80
86
|
options: AIStreamOptions;
|
|
81
87
|
removeMessage: (message: ChatMessage) => Promise<void>;
|
|
82
88
|
clearAllMessages: () => Promise<void>;
|
|
@@ -87,15 +93,17 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
|
|
|
87
93
|
thingjson?: any;
|
|
88
94
|
data: string;
|
|
89
95
|
}>;
|
|
90
|
-
onSkillCompose: (fn: AIStreamHooks[
|
|
91
|
-
onChatMessageSent: (fn: AIStreamHooks[
|
|
92
|
-
onTextCompose: (fn: AIStreamHooks[
|
|
93
|
-
onSkillsEnd: (fn: AIStreamHooks[
|
|
94
|
-
onCardsReceived: (fn: AIStreamHooks[
|
|
95
|
-
onUserDataRead: (fn: AIStreamHooks[
|
|
96
|
-
onSessionEventReceived: (fn: AIStreamHooks[
|
|
96
|
+
onSkillCompose: (fn: AIStreamHooks["onSkillCompose"]) => () => void;
|
|
97
|
+
onChatMessageSent: (fn: AIStreamHooks["onChatMessageSent"]) => () => void;
|
|
98
|
+
onTextCompose: (fn: AIStreamHooks["onTextCompose"]) => () => void;
|
|
99
|
+
onSkillsEnd: (fn: AIStreamHooks["onSkillsEnd"]) => () => void;
|
|
100
|
+
onCardsReceived: (fn: AIStreamHooks["onCardsReceived"]) => () => void;
|
|
101
|
+
onUserDataRead: (fn: AIStreamHooks["onUserDataRead"]) => () => void;
|
|
102
|
+
onSessionEventReceived: (fn: AIStreamHooks["onSessionEventReceived"]) => () => void;
|
|
103
|
+
onAttachmentCompose: (fn: AIStreamHooks["onAttachmentCompose"]) => () => void;
|
|
104
|
+
onAttachmentsEnd: (fn: AIStreamHooks["onAttachmentsEnd"]) => () => void;
|
|
97
105
|
sendEvent: (params: SendAIStreamEventParams) => Promise<null>;
|
|
98
|
-
onTTTAction: (fn: AIStreamHooks[
|
|
106
|
+
onTTTAction: (fn: AIStreamHooks["onTTTAction"]) => () => void;
|
|
99
107
|
getChatId: () => Promise<string>;
|
|
100
108
|
};
|
|
101
109
|
};
|