@ray-js/t-agent-plugin-aistream 0.2.0-beta-2 → 0.2.0-beta-4

package/dist/utils/createAsrAgent.js

@@ -0,0 +1,197 @@
+import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
+import "core-js/modules/es.json.stringify.js";
+import { authorize } from './ttt';
+import { DEFAULT_TOKEN_API, DEFAULT_TOKEN_API_VERSION, globalAIStreamClient } from '../global';
+import { ConnectClientType, AIStreamAttributePayloadType, AIStreamAttributeType, AIStreamChatSysWorkflow } from '../AIStreamTypes';
+import logger from './logger';
+class AsrAgent {
+  constructor(options) {
+    /** 数据链接，一个agent保持一个链接就可以 */
+    /** 当前请求的session，每次请求都需要是一个新的session */
+    /** 音频流监听 */
+    /** 录音时长定时器 */
+    _defineProperty(this, "recordDurationTimer", null);
+    this.options = options;
+  }
+
+  /** 获取录音权限 */
+  getRecordScope() {
+    return authorize({
+      scope: 'scope.record'
+    }).then(() => true, () => false);
+  }
+
+  /** 获取数据链接，一般只有一个链接就可以 */
+  getConnection(clientType, deviceId) {
+    if (this.streamConn) {
+      return this.streamConn;
+    }
+    // 创建 activeSession
+    this.streamConn = globalAIStreamClient.getConnection({
+      clientType: clientType || ConnectClientType.APP,
+      deviceId: deviceId
+    });
+    return this.streamConn;
+  }
+  createSession() {
+    var _this$activeSession;
+    // 每次新的请求，都需要重新创建一个Session
+    const {
+      clientType,
+      deviceId,
+      tokenApi,
+      tokenApiVersion,
+      agentId
+    } = this.options;
+    const streamConn = this.getConnection(clientType, deviceId);
+    // 如果有激活的，需要释放
+    if ((_this$activeSession = this.activeSession) !== null && _this$activeSession !== void 0 && _this$activeSession.sessionId) {
+      this.activeSession.close();
+    }
+    const activeSession = streamConn.createSession({
+      api: tokenApi || DEFAULT_TOKEN_API,
+      apiVersion: tokenApiVersion || DEFAULT_TOKEN_API_VERSION,
+      solutionCode: agentId
+    });
+    this.activeSession = activeSession;
+    return this.activeSession;
+  }
+
+  /** 开始录音时长监听 */
+  startRecordTimer() {
+    const {
+      maxDuration
+    } = this.options;
+    if (maxDuration) {
+      if (this.recordDurationTimer) {
+        clearTimeout(this.recordDurationTimer);
+      }
+      this.recordDurationTimer = setTimeout(() => {
+        this.stop(true);
+      }, maxDuration * 1000);
+    }
+  }
+  async start() {
+    const hasScope = await this.getRecordScope();
+    if (!hasScope) {
+      throw new Error('authorize failed');
+    }
+    const {
+      onMessage,
+      onFinish,
+      onError
+    } = this.options || {};
+    const activeSession = await this.createSession();
+    const attribute = {
+      'processing.interrupt': 'false',
+      'asr.enableVad': 'false',
+      'sys.workflow': AIStreamChatSysWorkflow.ASR
+    };
+    const activeEvent = await activeSession.startEvent({
+      userData: [{
+        type: AIStreamAttributeType.AI_CHAT,
+        payloadType: AIStreamAttributePayloadType.STRING,
+        value: JSON.stringify(attribute)
+      }]
+    });
+    this.activeEvent = activeEvent;
+    const audioStream = activeEvent.stream({
+      type: 'audio'
+    });
+    this.audioStream = audioStream;
+    await audioStream.start();
+    this.startRecordTimer();
+    activeEvent.on('data', entry => {
+      if (entry.type === 'text') {
+        console.log('text', entry, JSON.parse(entry.body.text));
+        let data = {
+          text: ''
+        };
+        try {
+          data = JSON.parse(entry.body.text) || {};
+        } catch (error) {
+          logger.error('JSON.parse error', error);
+        }
+        typeof onMessage === 'function' && onMessage(data);
+      }
+    });
+    activeEvent.on('finish', () => {
+      this.stop();
+      typeof onFinish === 'function' && onFinish();
+    });
+    activeEvent.on('error', error => {
+      typeof onError === 'function' && onError(error);
+    });
+  }
+  async stop(isAbort) {
+    var _this$activeSession2;
+    if (this.recordDurationTimer) {
+      clearTimeout(this.recordDurationTimer);
+      this.recordDurationTimer = null;
+    }
+    if (this.audioStream) {
+      await this.audioStream.stop();
+      this.audioStream = null;
+    }
+    if (this.activeEvent) {
+      if (isAbort) {
+        await this.activeEvent.abort();
+      } else {
+        await this.activeEvent.end();
+      }
+      this.activeEvent = null;
+    }
+    if (isAbort) {
+      const {
+        onAbort
+      } = this.options || {};
+      typeof onAbort === 'function' && onAbort();
+    }
+    await ((_this$activeSession2 = this.activeSession) === null || _this$activeSession2 === void 0 ? void 0 : _this$activeSession2.close());
+  }
+  async abort() {
+    await this.stop(true);
+  }
+}
+
+/**
+ * 创建一个AsrAgent实例，用于语音转文本
+ * @param options AsrAgentOptions
+ * @param options.agentId 必填 语音转文本的agentId
+ * @param options.tokenApi 语音转文本的tokenApi
+ * @param options.tokenApiVersion 语音转文本的tokenApiVersion
+ * @param options.clientType 语音转文本的clientType
+ * @param options.deviceId 语音转文本的deviceId
+ * @example
+ * const asrAgent = createAsrAgent({
+ *   agentId: 'asr-agent',
+ *   tokenApi: 'm.thing.aigc.basic.server.token',
+ *   tokenApiVersion: '1.0',
+ *   clientType: ConnectClientType.APP,
+ *   deviceId: 'deviceId',
+ *   maxDuration: 60,
+ *   onMessage: (message) => {
+ *     console.log('onMessage', message);
+ *   },
+ *   onFinish: () => {
+ *     console.log('onFinish');
+ *   },
+ *   onError: (error) => {
+ *     console.log('onError', error);
+ *   },
+ *   onAbort: () => {
+ *     console.log('onAbort');
+ *   },
+ * });
+ * // 开始录音
+ * asrAgent.start()
+ * // 停止录音
+ * asrAgent.stop()
+ * // 中断录音
+ * asrAgent.abort()
+ * @returns
+ */
+export function createAsrAgent(options) {
+  const asrAgent = new AsrAgent(options);
+  return asrAgent;
+}

package/dist/utils/defaultMock.js

@@ -1,3 +1,4 @@
+import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
 import "core-js/modules/es.array.sort.js";
 import "core-js/modules/es.json.stringify.js";
 import "core-js/modules/es.regexp.exec.js";
@@ -34,19 +35,20 @@ mock.hooks.hook('getMiniAppConfig', context => {
     config: {}
   };
 });
+const getCurrentConnection = () => {
+  return mock.data.get('currentConnection');
+};
 mock.hooks.hook('connect', context => {
-  const {
-    options
-  } = context;
-  const key = "connection-".concat(options.clientType, "-").concat(options.deviceId || '');
-  if (!mock.data.has(key)) {
-    const connectionId = generateId();
-    mock.data.set(key, {
-      connectionId
-    });
-    mock.data.set("connection-".concat(connectionId), key);
+  let connection = getCurrentConnection();
+  if (connection) {
+    throw new Error('already connected');
+  } else {
+    connection = {
+      connectionId: generateId()
+    };
+    mock.data.set('currentConnection', connection);
   }
-  context.result = mock.data.get(key);
+  context.result = connection;
 });
 mock.hooks.hook('disconnect', context => {
   const {
@@ -54,9 +56,15 @@ mock.hooks.hook('disconnect', context => {
       connectionId
     }
   } = context;
-  const key = mock.data.get("connection-".concat(connectionId));
-  mock.data.delete(key);
-  mock.data.delete("connection-".concat(connectionId));
+  const connection = getCurrentConnection();
+  if (!connection) {
+    throw new Error('not connected');
+  }
+  if (connection.connectionId !== connectionId) {
+    throw new Error('connectionId mismatch');
+  }
+  mock.data.set('currentConnection', null);
+  mock.data.set('sessionMap', new Map());
 });
 mock.hooks.hook('queryAgentToken', context => {
   context.result = {
@@ -106,6 +114,14 @@ mock.hooks.hook('createSession', context => {
     sendDataChannels: session.sendDataChannels,
     revDataChannels: session.revDataChannels
   };
+
+  // setTimeout(() => {
+  //   dispatch('onConnectStateChanged', {
+  //     connectionId: getCurrentConnection().connectionId,
+  //     connectState: ConnectState.DISCONNECTED,
+  //     code: 200,
+  //   });
+  // }, 1000);
 });
 mock.hooks.hook('closeSession', context => {
   const map = mock.data.get('sessionMap');
@@ -149,21 +165,31 @@ mock.hooks.hook('unregisterVoiceAmplitudes', context => {
 mock.hooks.hook('sendEventEnd', context => {
   const session = getSession(context.options.sessionId, context.options.eventId);
   context.result = true;
+  const event = session.currentEvent;
   (async () => {
+    await event.asrStatus.promise;
     const ctx = {
-      data: session.currentEvent.data,
+      data: event.data,
       responseText: ''
     };
-    await session.currentEvent.asrStatus.promise;
     await mock.hooks.callHook('sendToAIStream', ctx);
     const text = ctx.responseText || '⚠️ No mock text response matched!';
     const words = splitString(text);
+    if (event.controller.signal.aborted) {
+      return;
+    }
     session.replyEvent(EventType.EVENT_START);
     await mock.sleep(500);
+    if (event.controller.signal.aborted) {
+      return;
+    }
     const bizId = generateId();
     session.replyText(StreamFlag.START);
     for (const word of words) {
       await mock.sleep(100);
+      if (event.controller.signal.aborted) {
+        return;
+      }
       session.replyText(StreamFlag.IN_PROGRESS, {
         bizType: ReceivedTextPacketType.NLG,
         eof: ReceivedTextPacketEof.CONTINUE,
@@ -175,6 +201,10 @@ mock.hooks.hook('sendEventEnd', context => {
         }
       });
     }
+    await mock.sleep(100);
+    if (event.controller.signal.aborted) {
+      return;
+    }
     session.replyText(StreamFlag.IN_PROGRESS, {
       bizType: ReceivedTextPacketType.NLG,
       eof: ReceivedTextPacketEof.END,
@@ -186,8 +216,14 @@ mock.hooks.hook('sendEventEnd', context => {
       }
     });
     await mock.sleep(10);
+    if (event.controller.signal.aborted) {
+      return;
+    }
     session.replyText(StreamFlag.END);
     await mock.sleep(500);
+    if (event.controller.signal.aborted) {
+      return;
+    }
     session.replyEvent(EventType.EVENT_END);
     session.currentEvent = null;
   })();
@@ -238,6 +274,7 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
     let text = '';
     controller.signal.addEventListener('abort', async () => {
       var _finishResolve;
+      // 终止识别到出完整结果的延迟
       await mock.sleep(300);
       session.replyText(StreamFlag.IN_PROGRESS, {
         bizType: ReceivedTextPacketType.ASR,
@@ -249,8 +286,16 @@ mock.hooks.hook('startRecordAndSendAudioData', context => {
       });
       await mock.sleep(100);
       session.replyText(StreamFlag.END);
+      if (session.currentEvent) {
+        session.currentEvent.data.push({
+          type: 'text',
+          text
+        });
+      }
       (_finishResolve = finishResolve) === null || _finishResolve === void 0 || _finishResolve();
     });
+
+    // 识别 ASR 的延迟
     await mock.sleep(100);
     if (controller.signal.aborted) {
       return;
@@ -426,4 +471,24 @@ mock.hooks.hook('insertRecord', context => {
   context.result = {
     id
   };
+});
+mock.hooks.hook('updateRecord', context => {
+  const options = context.options;
+  const id = options.id;
+
+  // 默认倒序
+  const records = filterRecords({
+    id: [id]
+  });
+  if (!records.length) {
+    throw new Error('record not exists');
+  }
+  const newRecord = _objectSpread(_objectSpread({}, records[0]), options);
+  mock.setRecord({
+    id,
+    record: newRecord
+  });
+  context.result = {
+    newRecord
+  };
 });

package/dist/utils/index.d.ts

@@ -9,3 +9,5 @@ export * from './AIStream';
 export * from './observer';
 export * from './sendMessage';
 export * from './version';
+export * from './createAsrAgent';
+export * from './actions';

package/dist/utils/index.js

@@ -8,4 +8,6 @@ export * from './url';
 export * from './AIStream';
 export * from './observer';
 export * from './sendMessage';
-export * from './version';
+export * from './version';
+export * from './createAsrAgent';
+export * from './actions';

package/dist/utils/mock.d.ts

@@ -41,6 +41,7 @@ declare const mock: {
     getRecords: <T = any>() => Row<T>[];
     getRecord: <T_1 = any>(id: number) => Row<T_1> | undefined;
     setRecord: <T_2 = any>(entry: Row<T_2>) => void;
+    updateRecord: <T_3 = any>(entry: Row<T_3>) => void;
     deleteRecord: (id: number) => void;
     clearRecords: () => void;
     getId: () => number;

package/dist/utils/mock.js

@@ -52,6 +52,14 @@ const mock = {
       data: JSON.stringify(mock.getRecords())
     });
   },
+  updateRecord: entry => {
+    mock.getRecords();
+    mock.database.set(entry.id, entry);
+    ty.setStorageSync({
+      key: 'AIStreamMockDatabase',
+      data: JSON.stringify(mock.getRecords())
+    });
+  },
   deleteRecord: id => {
     mock.getRecords();
     mock.database.delete(id);

package/dist/utils/promisify.d.ts

@@ -11,6 +11,12 @@ interface AsyncTTTFnParams<P> {
     }) => void;
     [key: string]: any;
 }
+export declare class TTTError extends Error {
+    message: string;
+    errorCode: string | number;
+    constructor(message: string, errorCode: string | number);
+    toString(): string;
+}
 export declare const getEnableMock: () => boolean;
 export declare const setEnableMock: (enable: boolean) => boolean;
 export declare function promisify<T extends AsyncTTTFnParams<any>>(fn: (options: any) => void, enableMock?: boolean): (options?: Omit<T, 'success' | 'fail'>) => Promise<Parameters<NonNullable<T["success"]>>[0]>;

package/dist/utils/promisify.js

@@ -2,30 +2,40 @@ import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
 import { isDevTools } from './ttt';
 import logger from './logger';
 import { mock } from './mock';
+export class TTTError extends Error {
+  constructor(message, errorCode) {
+    super(message);
+    this.message = message;
+    this.errorCode = errorCode;
+  }
+  toString() {
+    return "TTTError: ".concat(this.message, ", errorCode: ").concat(this.errorCode);
+  }
+}
 let callId = 100000;
 export const getEnableMock = () => {
   try {
     const result = ty.getStorageSync({
-      key: 'AIAssistantEnableMock'
+      key: 'AIStreamEnableMock'
     });
     // @ts-ignore
     if (result && (result === 'true' || result.data === 'true')) {
       return true;
     }
   } catch (e) {
-    console.error('获取AIAssistantEnableMock配置失败', e);
+    console.error('获取AIStreamEnableMock配置失败', e);
   }
   return isDevTools();
 };
 export const setEnableMock = enable => {
   try {
     ty.setStorageSync({
-      key: 'AIAssistantEnableMock',
+      key: 'AIStreamEnableMock',
       data: String(enable)
     });
     return true;
   } catch (e) {
-    console.error('设置AIAssistantEnableMock配置失败', e);
+    console.error('设置AIStreamEnableMock配置失败', e);
     return false;
   }
 };
@@ -44,6 +54,16 @@ export function promisify(fn) {
       };
       const reject = error => {
         logger.debug("TTT error #".concat(id, " %c").concat(fn.name), 'background: red; color: white', error);
+
+        // 这是 TTT 的错误
+        if (error.errorCode != null && error.errorMsg != null) {
+          var _error$innerError;
+          if (((_error$innerError = error.innerError) === null || _error$innerError === void 0 ? void 0 : _error$innerError.errorCode) != null && error.innerError.errorMsg != null) {
+            error = new TTTError(error.innerError.errorMsg, error.innerError.errorCode);
+          } else {
+            error = new TTTError(error.errorMsg, error.errorCode);
+          }
+        }
         _reject(error);
       };
       const context = {

package/dist/utils/ttt.js

@@ -2,7 +2,7 @@ import { listening, promisify } from './promisify';
 import { ConnectClientType, ConnectState } from '../AIStreamTypes';
 import { generateId } from '@ray-js/t-agent';
 export const getMiniAppConfig = promisify(ty.getMiniAppConfig, true);
-export const getAccountInfo = promisify(ty.getAccountInfo, true);
+export const getAccountInfo = promisify(ty.getAccountInfo);
 export const isDevTools = () => {
   return ty.getSystemInfoSync().brand === 'devtools';
 };

package/dist/withAIStream.d.ts

@@ -1,33 +1,33 @@
 import { ChatAgent, ChatMessage, ComposeHandler, InputBlock } from '@ray-js/t-agent';
 import { ConnectClientType } from './AIStreamTypes';
-import { ChatHistoryStore, StoredMessageObject } from './ChatHistoryStore';
+import { ChatHistoryLocalStore, StoredMessageObject } from './ChatHistoryStore';
 export interface AIStreamOptions {
     /** client 类型: 1-作为设备代理, 2-作为 App */
     clientType?: ConnectClientType;
     /** 代理的设备ID, clientType == 1 时必传 */
     deviceId?: string;
+    /** Agent ID */
     agentId?: string;
-    channel?: string;
-    /** 打开小程序的链接，取链接参数字段 aiPtChannel */
-    aiPtChannel?: string;
-    sessionId?: string;
-    tokenApi?: string;
-    tokenApiVersion?: string;
+    /** 获取 agent token 的参数 */
+    tokenOptions?: {
+        api: string;
+        version: string;
+        extParams?: Record<string, any>;
+    };
     /** 历史消息数量，默认1000，为0的话，则已分页的方式取全部数据 */
     historySize?: number;
     /** 是否支持多模态 */
     multiModal?: boolean;
+    /** 是否将输入块传递给智能体，默认为 true，设置为 false 时，需要你自己编写 onInputBlocksPush Hook 来处理输入块 */
     wireInput?: boolean;
     /** 索引ID，如果传了 createChatHistoryStore，则会覆盖 */
     indexId?: string;
     /** 家庭id，不填默认当前家庭 */
     homeId?: number;
-    /** 自定义消息存储, 返回的实例需要实现 ChatHistoryStore 接口 */
-    createChatHistoryStore?: (agent: ChatAgent) => ChatHistoryStore;
-    messageOptions?: {
-        device_id?: string;
-        [key: string]: string;
-    };
+    /** 是否在 onAgentStart 阶段就建立连接 */
+    earlyStart?: boolean;
+    /** 自定义消息存储, 返回的实例需要实现 ChatHistoryLocalStore 接口, 返回null则不存储历史聊天记录 */
+    createChatHistoryStore?: (agent: ChatAgent) => ChatHistoryLocalStore | null;
 }
 export interface AIStreamHooks {
     onMessageParse: (msgItem: StoredMessageObject, result: {
@@ -39,7 +39,7 @@ export interface AIStreamHooks {
 }
 export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAgent) => {
     hooks: import("hookable").Hookable<any, string>;
-    assistant: {
+    aiStream: {
         send: (blocks: InputBlock[], signal?: AbortSignal, extraOptions?: Record<string, any>) => {
             response: import("@ray-js/t-agent").StreamResponse;
             metaPromise: Promise<Record<string, any>>;
@@ -53,12 +53,12 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
         options: AIStreamOptions;
         removeMessage: (message: ChatMessage) => Promise<void>;
         clearAllMessages: () => Promise<void>;
-        feedback: ({ requestId, type, }: {
+        feedback: ({ requestId, type }: {
             requestId: string;
-            type: 'like' | 'unlike';
+            type: string;
         }) => Promise<{
             thingjson?: any;
             data: string;
-        } | null>;
+        }>;
     };
 };