@ray-js/t-agent-plugin-aistream 0.2.0-beta-17 → 0.2.0-beta.18

package/dist/utils/errors.js

@@ -1,3 +1,6 @@
+import "core-js/modules/web.dom-collections.iterator.js";
+import { AIStreamAppErrorCode, AIStreamErrorCode, AIStreamServerErrorCode } from '../AIStreamTypes';
+import { tryCatch } from './misc';
 export class BaseError extends Error {
   constructor(message, code) {
     super(message);
@@ -16,27 +19,87 @@ export class TTTError extends BaseError {
     this.name = 'TTTError';
   }
 }
-export class AIStreamConnectionError extends BaseError {
+export class AIStreamError extends BaseError {
   constructor(message, code) {
     super(message, code);
     this.message = message;
     this.code = code;
-    this.name = 'AIStreamConnectionError';
+    this.name = 'AIStreamError';
   }
 }
-export class AIStreamSessionError extends BaseError {
-  constructor(message, code) {
-    super(message, code);
-    this.message = message;
-    this.code = code;
-    this.name = 'AIStreamSessionError';
+
+/**
+ * 移除错误对象的顶部栈帧
+ *
+ * @param error 需要处理的错误对象
+ * @returns 移除顶部栈帧后的错误对象
+ */
+function removeTopStackFrame(error) {
+  if (!error.stack) {
+    return error;
+  }
+
+  // 按行分割错误栈信息
+  const stackLines = error.stack.split('\n');
+
+  // 检查是否有足够的行来移除
+  if (stackLines.length <= 1) {
+    return error;
   }
+
+  // 保留第一行错误信息，移除第二行(顶部栈帧)
+  const newStack = [stackLines[0]].concat(stackLines.slice(2)).join('\n');
+  error.stack = newStack;
+  return error;
 }
-export class AIStreamEventError extends BaseError {
-  constructor(message, code) {
-    super(message, code);
-    this.message = message;
-    this.code = code;
-    this.name = 'AIStreamEventError';
+export const transformErrorCode = code => {
+  switch (code) {
+    case AIStreamAppErrorCode.GENERIC_ERROR:
+      return AIStreamErrorCode.UNKNOWN_ERROR;
+    case AIStreamAppErrorCode.INVALID_PARAMS:
+    case AIStreamAppErrorCode.CONNECTION_INVALID:
+    case AIStreamAppErrorCode.SESSION_ID_INVALID:
+    case AIStreamAppErrorCode.EVENT_ID_INVALID:
+    case AIStreamAppErrorCode.INVALID_DATA_PACKET:
+    case AIStreamAppErrorCode.FILE_DATA_READ_ERROR:
+    case AIStreamAppErrorCode.DATA_CHANNEL_NOT_FOUND:
+      return AIStreamErrorCode.INVALID_PARAMS;
+    case AIStreamAppErrorCode.HTTP_REQUEST_FAILED:
+      return AIStreamErrorCode.SESSION_CREATE_FAILED;
+    case AIStreamAppErrorCode.CONNECTION_CLOSED_BY_REMOTE:
+      return AIStreamErrorCode.CONNECTION_CLOSED;
+    case AIStreamServerErrorCode.OK:
+    case AIStreamServerErrorCode.CLOSE_BY_CLIENT:
+    case AIStreamServerErrorCode.CLOSE_BY_EXPIRE:
+    case AIStreamServerErrorCode.CLOSE_BY_IO:
+    case AIStreamServerErrorCode.CLOSE_BY_KEEPALIVE:
+    case AIStreamServerErrorCode.CLOSE_BY_REUSE:
+      return AIStreamErrorCode.CONNECTION_CLOSED;
+    case AIStreamServerErrorCode.BAD_REQUEST:
+    case AIStreamServerErrorCode.NOT_FOUND:
+    case AIStreamServerErrorCode.UNAUTHENTICATED:
+      return AIStreamErrorCode.INVALID_PARAMS;
+    case AIStreamServerErrorCode.INTERNAL_SERVER_ERROR:
+      return AIStreamErrorCode.UNKNOWN_ERROR;
+    case AIStreamServerErrorCode.REQUEST_TIMEOUT:
+    case AIStreamServerErrorCode.GATEWAY_TIMEOUT:
+      return AIStreamErrorCode.TIMEOUT;
+  }
+  return AIStreamErrorCode.UNKNOWN_ERROR;
+};
+
+// 函数实现
+export async function tryCatchTTT(fn) {
+  const [error, result] = await tryCatch(fn);
+  if (error) {
+    let {
+      code,
+      message
+    } = error;
+    message = "".concat(message, " (original_code: ").concat(code || '-', ")");
+    code = transformErrorCode(code);
+    const e = removeTopStackFrame(new AIStreamError(message, code));
+    return [e, null];
   }
+  return [null, result];
 }

package/dist/utils/sendMessage.d.ts

@@ -7,10 +7,7 @@ export interface SendBlocksToAIStreamParams {
     session: AIStreamSession;
     attribute?: AIStreamChatAttribute;
     signal?: AbortSignal;
-}
-export declare class AIStreamSessionError extends Error {
-    readonly code: number;
-    constructor(message: string, code: number);
+    enableTts?: boolean;
 }
 export declare function sendBlocksToAIStream(params: SendBlocksToAIStreamParams): {
     response: StreamResponse;

package/dist/utils/sendMessage.js

@@ -3,10 +3,10 @@ import "core-js/modules/es.json.stringify.js";
 import "core-js/modules/web.dom-collections.iterator.js";
 import '../polyfill';
 import { ReadableStream } from 'web-streams-polyfill';
-import { AIStreamAttributePayloadType, AIStreamAttributeType, AIStreamChatSysWorkflow, ConnectState, FileFormat, ReceivedTextPacketEof, ReceivedTextPacketType, SessionState, StreamFlag } from '../AIStreamTypes';
+import { AIStreamAttributePayloadType, AIStreamAttributeType, AIStreamErrorCode, ConnectState, FileFormat, ReceivedTextPacketEof, ReceivedTextPacketType, SessionState, StreamFlag } from '../AIStreamTypes';
 import { EmitterEvent, generateId, safeParseJSON, StreamResponse } from '@ray-js/t-agent';
 import { tryCatch } from './misc';
-import { AIStreamConnectionError } from './errors';
+import { AIStreamError, transformErrorCode } from './errors';
 import logger from './logger';
 const mimeTypeToFormatMap = {
   'video/mp4': FileFormat.MP4,
@@ -14,13 +14,6 @@ const mimeTypeToFormatMap = {
   'application/json': FileFormat.JSON,
   'application/pdf': FileFormat.PDF
 };
-export class AIStreamSessionError extends Error {
-  constructor(message, code) {
-    super(message);
-    this.name = 'AIStreamSessionError';
-    this.code = code;
-  }
-}
 export function sendBlocksToAIStream(params) {
   logger.debug('sendBlocksToAIStream start');
   const {
@@ -41,8 +34,7 @@ export function sendBlocksToAIStream(params) {
   }
   const attribute = _objectSpread({
     'processing.interrupt': 'false',
-    'asr.enableVad': 'false',
-    'sys.workflow': audioEmitter ? AIStreamChatSysWorkflow.ASR_LLM : AIStreamChatSysWorkflow.LLM
+    'asr.enableVad': 'false'
   }, params.attribute);
   let canceled = false;
   let closed = false;
@@ -80,6 +72,20 @@ export function sendBlocksToAIStream(params) {
           }
         }
       };
+      const emitError = error => {
+        if (!error) {
+          error = new AIStreamError('Unknown error', AIStreamErrorCode.UNKNOWN_ERROR);
+        }
+        if (!(error instanceof AIStreamError)) {
+          error = new AIStreamError("AIStream Error: ".concat(error.message || 'Unknown error'), transformErrorCode(error.code));
+        }
+        enqueue({
+          type: 'error',
+          level: 'error',
+          error,
+          meta: {}
+        });
+      };
       let pendingCancel = false;
       if (audioEmitter) {
         audioEmitter.addEventListener('confirm', () => {
@@ -108,16 +114,10 @@ export function sendBlocksToAIStream(params) {
         }]
       }));
       if (error) {
-        const e = new AIStreamSessionError(error.message, error.code || 0);
-        enqueue({
-          type: 'error',
-          error: e,
-          level: 'error',
-          meta: {}
-        });
+        emitError(error);
         if (audioEmitter) {
           audioEmitter.dispatchEvent(new EmitterEvent('error', {
-            detail: e
+            detail: error
           }));
         }
         close();
@@ -167,10 +167,9 @@ export function sendBlocksToAIStream(params) {
               if (packet.data.text === '') {
                 logger.debug('sendBlocksToAIStream Receive ASR EOF EMPTY');
                 // 没识别出任何文本
+                const e = new AIStreamError('ASR data is empty', AIStreamErrorCode.ASR_EMPTY);
                 audioEmitter.dispatchEvent(new EmitterEvent('error', {
-                  detail: {
-                    name: 'AsrEmptyError'
-                  }
+                  detail: e
                 }));
               } else {
                 logger.debug('sendBlocksToAIStream Receive ASR EOF', packet.data.text);
@@ -225,13 +224,9 @@ export function sendBlocksToAIStream(params) {
           }
         } else if (data.type === 'sessionState') {
           if (data.body.sessionState === SessionState.CLOSED || data.body.sessionState === SessionState.CREATE_FAILED) {
-            const e = new AIStreamSessionError('Session closed', data.body.code);
-            enqueue({
-              type: 'error',
-              error: e,
-              level: 'error',
-              meta
-            });
+            const msg = SessionState[data.body.sessionState];
+            const e = new AIStreamError("Session Error: ".concat(msg, ", (original_code: ").concat(data.body.code, ")"), transformErrorCode(data.body.code));
+            emitError(error);
             if (audioEmitter) {
               audioEmitter.dispatchEvent(new EmitterEvent('error', {
                 detail: e
@@ -241,13 +236,8 @@ export function sendBlocksToAIStream(params) {
           }
         } else if (data.type === 'connectionState') {
           if (data.body.connectState === ConnectState.DISCONNECTED || data.body.connectState === ConnectState.CLOSED) {
-            const e = new AIStreamConnectionError('Connection disconnected', data.body.code);
-            enqueue({
-              type: 'error',
-              error: e,
-              level: 'error',
-              meta
-            });
+            const e = new AIStreamError("Session disconnected, (original_code: ".concat(data.body.code, ")"), transformErrorCode(data.body.code));
+            emitError(error);
             if (audioEmitter) {
               audioEmitter.dispatchEvent(new EmitterEvent('error', {
                 detail: e
@@ -269,14 +259,7 @@ export function sendBlocksToAIStream(params) {
           close();
         }
       });
-      event.on('error', error => {
-        enqueue({
-          type: 'error',
-          level: 'error',
-          error: error instanceof Error ? error : new Error('Unknown error'),
-          meta: {}
-        });
-      });
+      event.on('error', emitError);
       for (const block of blocks) {
         if (block.type === 'text') {
           event.write({
@@ -285,24 +268,24 @@ export function sendBlocksToAIStream(params) {
               type: 'text',
               message: block.text
             })
-          });
+          }).catch(emitError);
         } else if (block.type === 'image_path') {
           event.write({
             type: 'image',
             path: block.image_path.path
-          });
+          }).catch(emitError);
         } else if (block.type === 'file_path') {
           event.write({
             type: 'file',
             format: mimeTypeToFormatMap[block.file_url.mimeType] || 0,
             path: block.file_path.path
-          });
+          }).catch(emitError);
         } else if (block.type === 'video_path') {
           event.write({
             type: 'file',
             format: FileFormat.MP4,
             path: block.file_path.path
-          });
+          }).catch(emitError);
         }
       }
       if (audioEmitter) {

package/dist/utils/ttt.d.ts

@@ -1,10 +1,11 @@
-import { ApiRequestByAtopParams, ApiRequestByHighwayParams, AudioBody, AuthorizeParams, AuthorizePolicyStatusParams, CanIUseRouterParams, CloseSessionParams, ConnectParams, ConnectStateBody, CreateSessionParams, DeleteRecordListParams, DisconnectParams, EventBody, EventChannelMessageParams, GetAppInfoParams, GetCurrentHomeInfoParams, GetMiniAppConfigParams, GetAccountInfoParams, ImageBody, InsertRecordParams, NavigateToMiniProgramParams, OpenInnerH5Params, OpenMiniWidgetParams, QueryAgentTokenParams, QueryRecordListParams, RecordAmplitudesBody, RegisterChannelParams, RouterParams, SendEventChatBreakParams, SendEventEndParams, SendEventPayloadEndParams, SendEventStartParams, SendImageDataParams, SendTextDataParams, SessionStateBody, StartRecordAndSendAudioDataParams, StopRecordAndSendAudioDataParams, TextBody, UpdateRecordParams, IsConnectedParams } from '../AIStreamTypes';
+import { ApiRequestByAtopParams, ApiRequestByHighwayParams, AudioBody, AuthorizeParams, AuthorizePolicyStatusParams, CanIUseRouterParams, CloseSessionParams, ConnectParams, ConnectStateBody, CreateSessionParams, DeleteRecordListParams, DisconnectParams, EventBody, EventChannelMessageParams, GetAppInfoParams, GetCurrentHomeInfoParams, GetMiniAppConfigParams, GetAccountInfoParams, ImageBody, InsertRecordParams, NavigateToMiniProgramParams, OpenInnerH5Params, OpenMiniWidgetParams, QueryAgentTokenParams, QueryRecordListParams, RecordAmplitudesBody, RegisterChannelParams, RouterParams, SendEventChatBreakParams, SendEventEndParams, SendEventPayloadEndParams, SendEventStartParams, SendImageDataParams, SendTextDataParams, SessionStateBody, StartRecordAndSendAudioDataParams, StopRecordAndSendAudioDataParams, TextBody, UpdateRecordParams, IsConnectedParams, GetNetworkTypeParams, StartPlayAudioParams } from '../AIStreamTypes';
 export declare const getMiniAppConfig: (options?: Omit<GetMiniAppConfigParams, "success" | "fail"> | undefined) => Promise<{
     config: any;
 }>;
 export declare const getAccountInfo: (options?: Omit<GetAccountInfoParams, "success" | "fail"> | undefined) => Promise<{
     miniProgram: import("../AIStreamTypes").MiniProgramAccountInfo;
 }>;
+export declare const systemInfo: any;
 export declare const isDevTools: () => boolean;
 export declare const apiRequestByAtop: (options?: Omit<ApiRequestByAtopParams, "success" | "fail"> | undefined) => Promise<{
     thing_json_?: any;
@@ -85,7 +86,11 @@ export declare const registerRecordAmplitudes: (options?: Omit<{
     count: number;
 }, "success" | "fail"> | undefined) => Promise<never>;
 export declare const unregisterVoiceAmplitudes: (options?: Omit<any, "success" | "fail"> | undefined) => Promise<unknown>;
+export declare const startPlayAudio: (options?: Omit<StartPlayAudioParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const listenEventReceived: (listener: (params: EventBody) => void) => () => void;
+export declare const listenAudioPlayEnd: (listener: (params: {
+    path: string;
+}) => void) => () => void;
 export declare const listenAudioReceived: (listener: (params: AudioBody) => void) => () => void;
 export declare const listenTextReceived: (listener: (params: TextBody) => void) => () => void;
 export declare const listenImageReceived: (listener: (params: ImageBody) => void) => () => void;
@@ -101,3 +106,7 @@ export declare const updateRecord: (options?: Omit<UpdateRecordParams, "success"
 export declare const insertRecord: (options?: Omit<InsertRecordParams, "success" | "fail"> | undefined) => Promise<{
     id: number;
 }>;
+export declare const getNetworkType: (options?: Omit<GetNetworkTypeParams, "success" | "fail"> | undefined) => Promise<{
+    networkType: import("../AIStreamTypes").NetworkType;
+    signalStrength: number;
+}>;

package/dist/utils/ttt.js

@@ -1,13 +1,14 @@
 import { listening, promisify } from './promisify';
 export const getMiniAppConfig = promisify(ty.getMiniAppConfig, true);
 export const getAccountInfo = promisify(ty.getAccountInfo);
+export const systemInfo = ty.getSystemInfoSync();
 export const isDevTools = () => {
-  return ty.getSystemInfoSync().brand === 'devtools';
+  return systemInfo.brand === 'devtools';
 };
 export const apiRequestByAtop = promisify(ty.apiRequestByAtop);
 export const apiRequestByHighway = promisify(ty.apiRequestByHighway);
 export const createNativeEventManager = options => {
-  if (isDevTools) {
+  if (isDevTools()) {
     return {
       onListener: () => null,
       offerListener: () => null
@@ -55,10 +56,12 @@ export const sendImageData = promisify(ty.aistream.sendImageData, true);
 export const sendTextData = promisify(ty.aistream.sendTextData, true);
 export const registerRecordAmplitudes = promisify(ty.aistream.registerRecordAmplitudes, true);
 export const unregisterVoiceAmplitudes = promisify(ty.aistream.unregisterVoiceAmplitudes, true);
+export const startPlayAudio = promisify(ty.aistream.startPlayAudio, true);
 
 // export const sendFileData = promisify<SendFileDataParams>(ty.aistream.sendFileData);
 
 export const listenEventReceived = listening(ty.aistream.onEventReceived, ty.aistream.offEventReceived, true);
+export const listenAudioPlayEnd = listening(ty.aistream.onAudioPlayEnd, ty.aistream.onAudioPlayEnd, true);
 export const listenAudioReceived = listening(ty.aistream.onAudioReceived, ty.aistream.offAudioReceived, true);
 
 // export const listenVideoReceived = listening<VideoBody>(
@@ -82,4 +85,5 @@ export const listenRecordAmplitudes = listening(ty.aistream.onRecordAmplitudes,
 export const queryRecordList = promisify(ty.aistream.queryRecordList, true);
 export const deleteRecordList = promisify(ty.aistream.deleteRecordList, true);
 export const updateRecord = promisify(ty.aistream.updateRecord, true);
-export const insertRecord = promisify(ty.aistream.insertRecord, true);
+export const insertRecord = promisify(ty.aistream.insertRecord, true);
+export const getNetworkType = promisify(ty.getNetworkType, true);

package/dist/withAIStream.d.ts

@@ -26,6 +26,8 @@ export interface AIStreamOptions {
     earlyStart?: boolean;
     /** 自定义消息存储, 返回的实例需要实现 ChatHistoryLocalStore 接口, 返回null则不存储历史聊天记录 */
     createChatHistoryStore?: (agent: ChatAgent) => ChatHistoryLocalStore | null;
+    /** 是否开启音频合成 */
+    enableTts?: boolean;
 }
 export type AIStreamPlugin = GetChatPluginHandler<typeof withAIStream>;
 export interface AIStreamHooks {

package/dist/withAIStream.js

@@ -1,5 +1,5 @@
-import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
 import _objectWithoutProperties from "@babel/runtime/helpers/esm/objectWithoutProperties";
+import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
 const _excluded = ["message"];
 import "core-js/modules/es.array.flat.js";
 import "core-js/modules/es.array.reverse.js";
@@ -98,7 +98,10 @@ export function withAIStream() {
         api: tokenOptions.api,
         apiVersion: tokenOptions.version,
         solutionCode: agentId,
-        extParams: tokenOptions.extParams
+        extParams: _objectSpread(_objectSpread({}, tokenOptions.extParams), {}, {
+          needTts: !!options.enableTts,
+          deviceId: options.deviceId || undefined
+        })
       });
       await session.set('AIStream.streamSession', streamSession);
       if (options.earlyStart) {
@@ -344,8 +347,10 @@ export function withAIStream() {
       }
       if (message.bubble.text) {
         await message.persist();
-      } else {
+      } else if (message.bubble.status === BubbleTileStatus.NORMAL) {
         await message.remove();
+      } else {
+        await message.persist();
       }
       return [userMsg, ...result.messages];
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.0-beta-17",
+  "version": "0.2.0-beta.18",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -35,5 +35,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "32e3d3ae017d11825e0bfd4dc4441abe67bd8ef0"
+  "gitHead": "3826cdeac61109618670dcbbeafd4103113f04d7"
 }