@ray-js/t-agent-plugin-aistream 0.2.8-beta.1 → 0.2.8-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,7 +9,7 @@ import "core-js/modules/esnext.iterator.map.js";
9
9
  import "core-js/modules/web.dom-collections.iterator.js";
10
10
  import { mock } from './mock';
11
11
  import { EmitterEvent, generateId, safeParseJSON } from '@ray-js/t-agent';
12
- import { AIStreamAppErrorCode, AIStreamServerErrorCode, BizCode, ConnectState, EventType, NetworkType, ReceivedTextPacketEof, ReceivedTextPacketType, SessionState, StreamFlag } from '../AIStreamTypes';
12
+ import { AIStreamAppErrorCode, AIStreamServerErrorCode, AudioPlayChangedState, BizCode, ConnectState, EventType, NetworkType, ReceivedTextPacketEof, ReceivedTextPacketType, SessionState, StreamFlag } from '../AIStreamTypes';
13
13
  import AbortController from './abort';
14
14
  import { tryCatch } from './misc';
15
15
  import { TTTError } from './errors';
@@ -309,6 +309,40 @@ mock.hooks.hook('unregisterVoiceAmplitudes', context => {
309
309
  mock.data.delete('recordAmplitudesCount');
310
310
  context.result = true;
311
311
  });
312
+
313
+ // 模拟 TTS 音频播放:开始时下发 START,3 秒后下发 COMPLETED。
314
+ mock.hooks.hook('startPlayAudio', context => {
315
+ var _context$options;
316
+ const path = ((_context$options = context.options) === null || _context$options === void 0 ? void 0 : _context$options.path) || '';
317
+
318
+ // 用播放 id 标记本次播放,被主动停止或被新的播放替换时即失效(同一时刻只播一条)
319
+ const playId = generateId();
320
+ mock.data.set('audioPlayId', playId);
321
+ dispatch('onAudioPlayChanged', {
322
+ path,
323
+ state: AudioPlayChangedState.START
324
+ });
325
+ context.result = true;
326
+
327
+ // 模拟 3 秒播放时长,结束后下发 COMPLETED(不阻塞 startPlayAudio 的返回)
328
+
329
+ (async () => {
330
+ await mock.sleep(3000);
331
+ if (mock.data.get('audioPlayId') !== playId) {
332
+ return;
333
+ }
334
+ mock.data.delete('audioPlayId');
335
+ dispatch('onAudioPlayChanged', {
336
+ path,
337
+ state: AudioPlayChangedState.COMPLETED
338
+ });
339
+ })();
340
+ });
341
+ mock.hooks.hook('stopPlayAudio', context => {
342
+ // 主动停止:让当前播放失效,不再下发 COMPLETED(与真实主动停止一致)
343
+ mock.data.delete('audioPlayId');
344
+ context.result = true;
345
+ });
312
346
  mock.hooks.hook('sendEventEnd', async context => {
313
347
  const map = mock.data.get('sessionMap');
314
348
  const session = map.get(context.options.sessionId);
@@ -430,6 +464,43 @@ mock.hooks.hook('sendEventEnd', async context => {
430
464
  data: skill
431
465
  });
432
466
  },
467
+ writeAudio: async (audio, options) => {
468
+ if (!audio) {
469
+ return;
470
+ }
471
+ if (options !== null && options !== void 0 && options.delayMs) {
472
+ await mock.sleep(options.delayMs);
473
+ }
474
+ if (!isEventActive()) {
475
+ return;
476
+ }
477
+ hasStreamedResponse = true;
478
+
479
+ // START 包:携带完整格式头
480
+ dispatch('onAudioReceived', {
481
+ dataChannel: 'audio',
482
+ sessionIdList: [session.sessionId],
483
+ streamFlag: StreamFlag.START,
484
+ path: audio.path,
485
+ pts: audio.pts,
486
+ codecType: audio.codecType,
487
+ sampleRate: audio.sampleRate,
488
+ channels: audio.channels,
489
+ bitDepth: audio.bitDepth
490
+ });
491
+ if (!isEventActive()) {
492
+ return;
493
+ }
494
+
495
+ // END 包:只带最终 path/pts,不再携带格式头(与真实协议一致)
496
+ dispatch('onAudioReceived', {
497
+ dataChannel: 'audio',
498
+ sessionIdList: [session.sessionId],
499
+ streamFlag: StreamFlag.END,
500
+ path: audio.path,
501
+ pts: audio.pts
502
+ });
503
+ },
433
504
  end: async options => {
434
505
  await finishEvent((options === null || options === void 0 ? void 0 : options.delayMs) || 0);
435
506
  },
@@ -532,6 +603,22 @@ mock.hooks.hook('sendEventEnd', async context => {
532
603
  await ctx.writeText(text, {
533
604
  wordDelayMs: ctx.wordDelayMs
534
605
  });
606
+
607
+ // 开启 TTS 时,模拟下发一段 TTS 音频包,与真实协议保持一致
608
+ if (session.tokenConfig.needTts) {
609
+ await ctx.writeAudio({
610
+ path: "/mock/tts-".concat(generateId(), ".mp3"),
611
+ codecType: 109,
612
+ // MP3
613
+ sampleRate: 16000,
614
+ channels: 0,
615
+ // 单声道
616
+ bitDepth: 16,
617
+ pts: 0
618
+ }, {
619
+ delayMs: 100
620
+ });
621
+ }
535
622
  if ((_ctx$responseSkills = ctx.responseSkills) !== null && _ctx$responseSkills !== void 0 && _ctx$responseSkills.length) {
536
623
  for (let i = 0; i < ctx.responseSkills.length; i++) {
537
624
  await ctx.writeSkill(ctx.responseSkills[i], {
@@ -569,8 +656,8 @@ mock.hooks.hook('sendEventChatBreak', async context => {
569
656
  context.result = true;
570
657
  });
571
658
  mock.hooks.hook('sendEvent', async context => {
572
- var _context$options;
573
- if (((_context$options = context.options) === null || _context$options === void 0 ? void 0 : _context$options.eventType) !== EventType.MCP_CMD) {
659
+ var _context$options2;
660
+ if (((_context$options2 = context.options) === null || _context$options2 === void 0 ? void 0 : _context$options2.eventType) !== EventType.MCP_CMD) {
574
661
  context.result = true;
575
662
  return;
576
663
  }
@@ -96,9 +96,11 @@ export async function tryCatchTTT(fn) {
96
96
  code,
97
97
  message
98
98
  } = error;
99
- message = "".concat(message, " (original_code: ").concat(code || '-', ")");
99
+ const originalCode = code;
100
+ message = "".concat(message, " (original_code: ").concat(originalCode || '-', ")");
100
101
  code = transformErrorCode(code);
101
102
  const e = removeTopStackFrame(new AIStreamError(message, code));
103
+ e.originalCode = originalCode;
102
104
  return [e, null];
103
105
  }
104
106
  return [null, result];
@@ -23,6 +23,21 @@ export interface SendToAIStreamContext {
23
23
  writeSkill: (skill: ReceivedTextSkillPacketBody, options?: {
24
24
  delayMs?: number;
25
25
  }) => Promise<void>;
26
+ /**
27
+ * 模拟一段流式音频(TTS)下发。
28
+ * 真实协议里格式头(codecType/sampleRate/channels/bitDepth)只在 START 包携带,
29
+ * END 包通常只带最终 path/pts,这里据实模拟,分两包下发。
30
+ */
31
+ writeAudio: (audio: {
32
+ path: string;
33
+ codecType?: number;
34
+ sampleRate?: number;
35
+ channels?: number;
36
+ bitDepth?: number;
37
+ pts?: number;
38
+ }, options?: {
39
+ delayMs?: number;
40
+ }) => Promise<void>;
26
41
  end: (options?: {
27
42
  delayMs?: number;
28
43
  }) => Promise<void>;
@@ -54,9 +69,9 @@ declare const mock: {
54
69
  emitter: Emitter;
55
70
  hooks: import("hookable").Hookable<MockHooks, string>;
56
71
  getRecords: <T = any>() => Row<T>[];
57
- getRecord: <T_1 = any>(id: number) => Row<T_1> | undefined;
58
- setRecord: <T_2 = any>(entry: Row<T_2>) => void;
59
- updateRecord: <T_3 = any>(entry: Row<T_3>) => void;
72
+ getRecord: <T = any>(id: number) => Row<T> | undefined;
73
+ setRecord: <T = any>(entry: Row<T>) => void;
74
+ updateRecord: <T = any>(entry: Row<T>) => void;
60
75
  deleteRecord: (id: number) => void;
61
76
  clearRecords: () => void;
62
77
  getId: () => number;
@@ -45,6 +45,12 @@ export declare class AIStreamObserverPool {
45
45
  private observerMap;
46
46
  constructor();
47
47
  private cancels;
48
+ private createObserverMap;
49
+ private isSessionAwareType;
50
+ private getEntrySessionIds;
51
+ private collectObservers;
52
+ private addObserver;
53
+ private removeObserver;
48
54
  start(): void;
49
55
  dispose(): void;
50
56
  connect(observer: AIStreamObserver): void;
@@ -10,25 +10,102 @@ export class AIStreamObserverPool {
10
10
  constructor() {
11
11
  _defineProperty(this, "isStarted", false);
12
12
  _defineProperty(this, "cancels", []);
13
- this.observerMap = {
14
- connectionState: new Set(),
15
- sessionState: new Set(),
16
- sessionEvent: new Set(),
17
- event: new Set(),
18
- text: new Set(),
19
- audio: new Set(),
20
- video: new Set(),
21
- file: new Set(),
22
- image: new Set()
13
+ this.observerMap = this.createObserverMap();
14
+ }
15
+ createObserverMap() {
16
+ return {
17
+ connectionState: {
18
+ all: new Set(),
19
+ bySessionId: new Map()
20
+ },
21
+ sessionState: {
22
+ all: new Set(),
23
+ bySessionId: new Map()
24
+ },
25
+ sessionEvent: {
26
+ all: new Set(),
27
+ bySessionId: new Map()
28
+ },
29
+ event: {
30
+ all: new Set(),
31
+ bySessionId: new Map()
32
+ },
33
+ text: {
34
+ all: new Set(),
35
+ bySessionId: new Map()
36
+ },
37
+ audio: {
38
+ all: new Set(),
39
+ bySessionId: new Map()
40
+ },
41
+ video: {
42
+ all: new Set(),
43
+ bySessionId: new Map()
44
+ },
45
+ file: {
46
+ all: new Set(),
47
+ bySessionId: new Map()
48
+ },
49
+ image: {
50
+ all: new Set(),
51
+ bySessionId: new Map()
52
+ }
23
53
  };
24
54
  }
55
+ isSessionAwareType(type) {
56
+ return type !== 'connectionState';
57
+ }
58
+ getEntrySessionIds(entry) {
59
+ if (entry.type === 'event' || entry.type === 'sessionEvent' || entry.type === 'sessionState') {
60
+ return entry.body.sessionId ? [entry.body.sessionId] : [];
61
+ }
62
+ if (entry.type === 'text' || entry.type === 'audio' || entry.type === 'video' || entry.type === 'file' || entry.type === 'image') {
63
+ return entry.body.sessionIdList || [];
64
+ }
65
+ return [];
66
+ }
67
+ collectObservers(entry) {
68
+ const bucket = this.observerMap[entry.type];
69
+ const observers = new Set(bucket.all);
70
+ this.getEntrySessionIds(entry).forEach(sessionId => {
71
+ const sessionObservers = bucket.bySessionId.get(sessionId);
72
+ sessionObservers === null || sessionObservers === void 0 || sessionObservers.forEach(observer => {
73
+ observers.add(observer);
74
+ });
75
+ });
76
+ return observers;
77
+ }
78
+ addObserver(type, observer) {
79
+ const sessionId = observer.options.sessionId;
80
+ const bucket = this.observerMap[type];
81
+ if (!sessionId || !this.isSessionAwareType(type)) {
82
+ bucket.all.add(observer);
83
+ return;
84
+ }
85
+ let sessionObservers = bucket.bySessionId.get(sessionId);
86
+ if (!sessionObservers) {
87
+ sessionObservers = new Set();
88
+ bucket.bySessionId.set(sessionId, sessionObservers);
89
+ }
90
+ sessionObservers.add(observer);
91
+ }
92
+ removeObserver(type, observer) {
93
+ const bucket = this.observerMap[type];
94
+ bucket.all.delete(observer);
95
+ bucket.bySessionId.forEach((sessionObservers, sessionId) => {
96
+ sessionObservers.delete(observer);
97
+ if (sessionObservers.size === 0) {
98
+ bucket.bySessionId.delete(sessionId);
99
+ }
100
+ });
101
+ }
25
102
  start() {
26
103
  if (this.isStarted) {
27
104
  return;
28
105
  }
29
106
  const handle = entry => {
30
107
  var _entry$body;
31
- const observers = this.observerMap[entry.type];
108
+ const observers = this.collectObservers(entry);
32
109
  let state = '';
33
110
  if (entry.type === 'connectionState') {
34
111
  state = "".concat(ConnectState[entry.body.connectState], " ").concat(entry.body.code || '');
@@ -47,18 +124,19 @@ export class AIStreamObserverPool {
47
124
  sessionId
48
125
  } = observer.options;
49
126
  let matched = true;
127
+ const sessionIdList = entry.body.sessionIdList || [];
50
128
  if (dataChannels !== null && dataChannels !== void 0 && dataChannels.length && !dataChannels.includes(entry.body.dataChannel)) {
51
129
  matched = false;
52
130
  }
53
- if (sessionId && !entry.body.sessionIdList.includes(sessionId)) {
131
+ if (sessionId && !sessionIdList.includes(sessionId)) {
54
132
  matched = false;
55
133
  }
56
134
  if (matched) {
57
135
  observer.callback(entry, observer);
58
136
  }
59
- } else {
137
+ } else if (!((entry.type === 'event' || entry.type === 'sessionEvent' || entry.type === 'sessionState') && observer.options.sessionId && observer.options.sessionId !== entry.body.sessionId)) {
60
138
  observer.callback(entry, observer);
61
- }
139
+ } // noop
62
140
  });
63
141
  };
64
142
  this.cancels = [listenConnectStateChanged(body => handle({
@@ -98,36 +176,26 @@ export class AIStreamObserverPool {
98
176
  this.isStarted = false;
99
177
  this.cancels.forEach(cancel => cancel());
100
178
  this.cancels = [];
101
- this.observerMap = {
102
- connectionState: new Set(),
103
- sessionState: new Set(),
104
- sessionEvent: new Set(),
105
- event: new Set(),
106
- text: new Set(),
107
- audio: new Set(),
108
- video: new Set(),
109
- file: new Set(),
110
- image: new Set()
111
- };
179
+ this.observerMap = this.createObserverMap();
112
180
  }
113
181
  connect(observer) {
114
182
  if (!this.isStarted) {
115
183
  this.start();
116
184
  }
117
- types.forEach(key => {
118
- const type = key;
185
+ types.forEach(type => {
119
186
  if (observer.options[type]) {
120
- this.observerMap[type].add(observer);
187
+ this.addObserver(type, observer);
121
188
  }
122
189
  });
123
190
  }
124
191
  disconnect(observer) {
125
- types.forEach(key => {
126
- this.observerMap[key].delete(observer);
192
+ types.forEach(type => {
193
+ this.removeObserver(type, observer);
127
194
  });
128
195
  let empty = true;
129
- types.forEach(key => {
130
- if (this.observerMap[key].size > 0) {
196
+ types.forEach(type => {
197
+ const bucket = this.observerMap[type];
198
+ if (bucket.all.size > 0 || bucket.bySessionId.size > 0) {
131
199
  empty = false;
132
200
  }
133
201
  });
@@ -13,6 +13,6 @@ interface AsyncTTTFnParams<P> {
13
13
  }
14
14
  export declare const getEnableMock: () => boolean;
15
15
  export declare const setEnableMock: (enable: boolean) => boolean;
16
- export declare function promisify<T extends AsyncTTTFnParams<any>>(fn: (options: any) => void, enableMock?: boolean): (options?: Omit<T, 'success' | 'fail'>) => Promise<Parameters<NonNullable<T["success"]>>[0]>;
16
+ export declare function promisify<T extends AsyncTTTFnParams<any>>(fn: (options: any) => void, enableMock?: boolean): (options?: Omit<T, "success" | "fail">) => Promise<Parameters<NonNullable<T["success"]>>[0]>;
17
17
  export declare function listening<T>(on: (listener: (params: T) => void) => void, off: (listener: (params: T) => void) => void, enableMock?: boolean): (listener: (params: T) => void) => () => void;
18
18
  export {};
@@ -6,7 +6,6 @@ export interface SendBlocksToAIStreamParams {
6
6
  blocks: InputBlock[];
7
7
  session: AIStreamSession;
8
8
  signal?: AbortSignalObject;
9
- enableTts?: boolean;
10
9
  eventIdPrefix?: string;
11
10
  getUserData: () => Promise<AIStreamUserData>;
12
11
  }
@@ -65,8 +65,10 @@ export function sendBlocksToAIStream(params) {
65
65
  if (!closed) {
66
66
  logger.debug('sendBlocksToAIStream close');
67
67
  closed = true;
68
- if (!canceled && !(signal !== null && signal !== void 0 && signal.aborted)) {
68
+ try {
69
69
  controller.close();
70
+ } catch (error) {
71
+ logger.debug('sendBlocksToAIStream close ignored', error);
70
72
  }
71
73
  }
72
74
  };
@@ -217,15 +219,17 @@ export function sendBlocksToAIStream(params) {
217
219
  if (!imageId) {
218
220
  imageId = generateId();
219
221
  }
222
+ const attachment = {
223
+ streamFlag: data.body.streamFlag,
224
+ path: data.body.path,
225
+ width: data.body.width,
226
+ height: data.body.height
227
+ };
220
228
  enqueue({
221
229
  id: imageId,
222
230
  type: 'attachment',
223
231
  attachmentType: 'image',
224
- attachment: {
225
- path: data.body.path || null,
226
- width: data.body.width || null,
227
- height: data.body.height || null
228
- },
232
+ attachment: attachment,
229
233
  meta
230
234
  });
231
235
  if (data.body.streamFlag === StreamFlag.END) {
@@ -235,13 +239,20 @@ export function sendBlocksToAIStream(params) {
235
239
  if (!audioId) {
236
240
  audioId = generateId();
237
241
  }
242
+ const attachment = {
243
+ streamFlag: data.body.streamFlag,
244
+ path: data.body.path,
245
+ pts: data.body.pts,
246
+ codecType: data.body.codecType,
247
+ sampleRate: data.body.sampleRate,
248
+ channels: data.body.channels,
249
+ bitDepth: data.body.bitDepth
250
+ };
238
251
  enqueue({
239
252
  id: audioId,
240
253
  type: 'attachment',
241
254
  attachmentType: 'audio',
242
- attachment: {
243
- path: data.body.path || null
244
- },
255
+ attachment,
245
256
  meta
246
257
  });
247
258
  if (data.body.streamFlag === StreamFlag.END) {
@@ -251,7 +262,7 @@ export function sendBlocksToAIStream(params) {
251
262
  if (data.body.sessionState === SessionState.CLOSED || data.body.sessionState === SessionState.CREATE_FAILED) {
252
263
  const msg = SessionState[data.body.sessionState];
253
264
  const e = new AIStreamError("Session Error: ".concat(msg, ", (original_code: ").concat(data.body.code, ")"), transformErrorCode(data.body.code));
254
- emitError(error);
265
+ emitError(e);
255
266
  if (audioEmitter) {
256
267
  audioEmitter.dispatchEvent(new EmitterEvent('error', {
257
268
  detail: e
@@ -262,7 +273,7 @@ export function sendBlocksToAIStream(params) {
262
273
  } else if (data.type === 'connectionState') {
263
274
  if (data.body.connectState === ConnectState.DISCONNECTED || data.body.connectState === ConnectState.CLOSED) {
264
275
  const e = new AIStreamError("Session disconnected, (original_code: ".concat(data.body.code, ")"), transformErrorCode(data.body.code));
265
- emitError(error);
276
+ emitError(e);
266
277
  if (audioEmitter) {
267
278
  audioEmitter.dispatchEvent(new EmitterEvent('error', {
268
279
  detail: e
@@ -303,14 +314,14 @@ export function sendBlocksToAIStream(params) {
303
314
  } else if (block.type === 'file_path') {
304
315
  event.write({
305
316
  type: 'file',
306
- format: mimeTypeToFormatMap[block.file_url.mimeType] || 0,
317
+ format: mimeTypeToFormatMap[block.file_path.mimeType] || 0,
307
318
  path: block.file_path.path
308
319
  }).catch(emitError);
309
320
  } else if (block.type === 'video_path') {
310
321
  event.write({
311
322
  type: 'file',
312
323
  format: FileFormat.MP4,
313
- path: block.file_path.path
324
+ path: block.video_path.path
314
325
  }).catch(emitError);
315
326
  }
316
327
  }
@@ -378,6 +389,8 @@ export function sendBlocksToAIStream(params) {
378
389
  }
379
390
  });
380
391
  return {
392
+ // web-streams-polyfill's ReadableStream is runtime-compatible but its
393
+ // getReader() overloads differ from lib.dom's, so cast to satisfy the type.
381
394
  response: new StreamResponse(stream),
382
395
  metaPromise
383
396
  };
@@ -36,11 +36,11 @@ export declare const getAppInfo: (options?: Omit<GetAppInfoParams, "success" | "
36
36
  regionCode: string;
37
37
  appName: string;
38
38
  appIcon: string;
39
- appEnv?: number | undefined;
39
+ appEnv?: number;
40
40
  appBundleId: string;
41
41
  appScheme: string;
42
42
  appId: string;
43
- clientId?: string | undefined;
43
+ clientId?: string;
44
44
  }>;
45
45
  export declare const canIUseRouter: (options?: Omit<CanIUseRouterParams, "success" | "fail"> | undefined) => Promise<{
46
46
  result: boolean;
@@ -57,7 +57,7 @@ export declare const openMiniWidget: (options?: Omit<OpenMiniWidgetParams, "succ
57
57
  export declare const isConnected: (options?: Omit<IsConnectedParams, "success" | "fail"> | undefined) => Promise<{
58
58
  connected: boolean;
59
59
  state: number;
60
- connectionId?: string | undefined;
60
+ connectionId?: string;
61
61
  }>;
62
62
  export declare const connect: (options?: Omit<ConnectParams, "success" | "fail"> | undefined) => Promise<{
63
63
  connectionId: string;
@@ -66,12 +66,13 @@ export declare const disconnect: (options?: Omit<DisconnectParams, "success" | "
66
66
  export declare const queryAgentToken: (options?: Omit<QueryAgentTokenParams, "success" | "fail"> | undefined) => Promise<{
67
67
  agentToken: string;
68
68
  bizConfig: import("../AIStreamTypes").BizConfig;
69
- extParams?: string | undefined;
69
+ extParams?: string;
70
70
  }>;
71
71
  export declare const createSession: (options?: Omit<CreateSessionParams, "success" | "fail"> | undefined) => Promise<{
72
72
  sessionId: string;
73
73
  sendDataChannels: string[];
74
74
  revDataChannels: string[];
75
+ baseCacheDir: string;
75
76
  }>;
76
77
  export declare const closeSession: (options?: Omit<CloseSessionParams, "success" | "fail"> | undefined) => Promise<null>;
77
78
  export declare const sendEventStart: (options?: Omit<SendEventStartParams, "success" | "fail"> | undefined) => Promise<{
package/dist/utils/ttt.js CHANGED
@@ -42,6 +42,8 @@ export const sendEventStart = promisify(ty.aistream.sendEventStart, true);
42
42
  export const sendEventPayloadEnd = promisify(ty.aistream.sendEventPayloadEnd, true);
43
43
  export const sendEventEnd = promisify(ty.aistream.sendEventEnd, true);
44
44
  export const sendEventChatBreak = promisify(ty.aistream.sendEventChatBreak, true);
45
+
46
+ // @ts-ignore
45
47
  export const sendEventToAIStream = promisify(ty.aistream.sendEvent, true);
46
48
  export const startRecordAndSendAudioData = promisify(ty.aistream.startRecordAndSendAudioData, true);
47
49
  export const stopRecordAndSendAudioData = promisify(ty.aistream.stopRecordAndSendAudioData, true);
@@ -1,4 +1,4 @@
1
- import { AbortSignalObject, ChatAgent, ChatCardObject, ChatMessage, ChatMessageStatus, ChatTile, GetChatPluginHandler, InputBlock } from '@ray-js/t-agent';
1
+ import { AbortSignalObject, AttachmentPart, ChatAgent, ChatCardObject, ChatMessage, ChatMessageStatus, ChatTile, GetChatPluginHandler, InputBlock } from '@ray-js/t-agent';
2
2
  import { TTTAction } from './utils';
3
3
  import { AIStreamUserData, ConnectClientType, ReceivedTextSkillPacketBody, SendAIStreamEventParams, SessionEventBody } from './AIStreamTypes';
4
4
  import { ChatHistoryStore, StoredMessageObject } from './ChatHistoryStore';
@@ -46,7 +46,13 @@ export interface AIStreamHooks {
46
46
  onTextCompose: (respMsg: ChatMessage, status: ChatMessageStatus, result: {
47
47
  text: string;
48
48
  }) => void;
49
- onSkillCompose: (skill: ReceivedTextSkillPacketBody[], respMsg: ChatMessage, result: {
49
+ onSkillCompose: (skill: ReceivedTextSkillPacketBody, respMsg: ChatMessage, result: {
50
+ messages: ChatMessage[];
51
+ }) => void;
52
+ onAttachmentCompose: (part: AttachmentPart, respMsg: ChatMessage, result: {
53
+ messages: ChatMessage[];
54
+ }) => void;
55
+ onAttachmentsEnd: (parts: AttachmentPart[], respMsg: ChatMessage, result: {
50
56
  messages: ChatMessage[];
51
57
  }) => void;
52
58
  onSkillsEnd: (skills: ReceivedTextSkillPacketBody[], respMsg: ChatMessage, result: {
@@ -73,10 +79,10 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
73
79
  metaPromise: Promise<Record<string, any>>;
74
80
  };
75
81
  chat: (blocks: InputBlock[], signal?: AbortSignalObject, options?: {
76
- sendBy?: string | undefined;
77
- responseBy?: string | undefined;
78
- userData?: AIStreamUserData | undefined;
79
- } | undefined) => Promise<ChatMessage[]>;
82
+ sendBy?: string;
83
+ responseBy?: string;
84
+ userData?: AIStreamUserData;
85
+ }) => Promise<ChatMessage[]>;
80
86
  options: AIStreamOptions;
81
87
  removeMessage: (message: ChatMessage) => Promise<void>;
82
88
  clearAllMessages: () => Promise<void>;
@@ -87,15 +93,17 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
87
93
  thingjson?: any;
88
94
  data: string;
89
95
  }>;
90
- onSkillCompose: (fn: AIStreamHooks['onSkillCompose']) => () => void;
91
- onChatMessageSent: (fn: AIStreamHooks['onChatMessageSent']) => () => void;
92
- onTextCompose: (fn: AIStreamHooks['onTextCompose']) => () => void;
93
- onSkillsEnd: (fn: AIStreamHooks['onSkillsEnd']) => () => void;
94
- onCardsReceived: (fn: AIStreamHooks['onCardsReceived']) => () => void;
95
- onUserDataRead: (fn: AIStreamHooks['onUserDataRead']) => () => void;
96
- onSessionEventReceived: (fn: AIStreamHooks['onSessionEventReceived']) => () => void;
96
+ onSkillCompose: (fn: AIStreamHooks["onSkillCompose"]) => () => void;
97
+ onChatMessageSent: (fn: AIStreamHooks["onChatMessageSent"]) => () => void;
98
+ onTextCompose: (fn: AIStreamHooks["onTextCompose"]) => () => void;
99
+ onSkillsEnd: (fn: AIStreamHooks["onSkillsEnd"]) => () => void;
100
+ onCardsReceived: (fn: AIStreamHooks["onCardsReceived"]) => () => void;
101
+ onUserDataRead: (fn: AIStreamHooks["onUserDataRead"]) => () => void;
102
+ onSessionEventReceived: (fn: AIStreamHooks["onSessionEventReceived"]) => () => void;
103
+ onAttachmentCompose: (fn: AIStreamHooks["onAttachmentCompose"]) => () => void;
104
+ onAttachmentsEnd: (fn: AIStreamHooks["onAttachmentsEnd"]) => () => void;
97
105
  sendEvent: (params: SendAIStreamEventParams) => Promise<null>;
98
- onTTTAction: (fn: AIStreamHooks['onTTTAction']) => () => void;
106
+ onTTTAction: (fn: AIStreamHooks["onTTTAction"]) => () => void;
99
107
  getChatId: () => Promise<string>;
100
108
  };
101
109
  };