@ray-js/t-agent-plugin-aistream 0.2.3-beta-2 → 0.2.3-beta-4

package/dist/AIStreamTypes.d.ts

@@ -791,6 +791,18 @@ export type StartPlayAudioParams = {
     }) => void;
     complete?: () => void;
 };
+export type StopPlayAudioParams = {
+    success?: (params: null) => void;
+    fail?: (params: {
+        errorMsg: string;
+        errorCode: string | number;
+        innerError: {
+            errorCode: string | number;
+            errorMsg: string;
+        };
+    }) => void;
+    complete?: () => void;
+};
 export type Attribute = {
     /** Attribute 类型 */
     type: AIStreamAttributeType;
@@ -862,6 +874,67 @@ export type AudioBody = {
      */
     bitDepth?: number;
 };
+export declare enum AudioPlayChangedState {
+    /** 播放开始 */
+    START = 1,
+    /** 缓冲中 */
+    BUFFERING = 2,
+    /** 缓存完成继续播放 */
+    BUFFERED = 3,
+    /** 播放完成 */
+    COMPLETED = 4,
+    /** 播放异常 */
+    ERROR = 5
+}
+export type AudioPlayChangedBody = {
+    path: string;
+    /** 播放状态: 1-播放开始, 2-缓冲中, 3-缓存完成 继续播放, 4-播放完成, 5-播放异常 */
+    state: AudioPlayChangedState;
+    /** 播放异常码: 1-主动停止播放, 2-播放被异常中断, ... */
+    code?: number;
+};
+export interface AIStreamAudioFile {
+    /** 音频缓存路径 */
+    path: string;
+    /**
+     * 音频编码类型，表示音频数据的编码格式
+     * - 100: ADPCM
+     * - 101: PCM
+     * - 102: AACRaw
+     * - 103: AACADTS
+     * - 104: AACLATM
+     * - 105: G711U
+     * - 106: G711A
+     * - 107: G726
+     * - 108: SPEEX
+     * - 109: MP3
+     * - 110: G722
+     * - 111: Opus
+     */
+    codecType: number;
+    /**
+     * 音频采样率，单位Hz
+     * 表示每秒采样次数，常见值：8000, 16000, 44100等
+     */
+    sampleRate: number;
+    /**
+     * 音频位深，表示每个采样点的位数
+     * 常见值：8, 16, 24, 32等
+     */
+    bitDepth: number;
+    /**
+     * 音频通道数
+     * - 0: 单声道
+     * - 1: 立体声
+     */
+    channels: number;
+}
+export type RecordAndSendAudioFailBody = {
+    /**
+     * 录制发送异常: 1-发送数据异常, 2-录制音频异常
+     */
+    code: 0 | 1;
+};
 export type VideoBody = {
     /** 接收数据通道 */
     dataChannel: string;
@@ -1217,6 +1290,10 @@ export type SendEventChatBreakParams = {
  *@description 开始录制并发送音频数据
  */
 export type StartRecordAndSendAudioDataParams = {
+    saveFile?: boolean;
+    recordInitParams?: {
+        sampleRate?: number;
+    };
     /** 会话 id */
     sessionId: string;
     /** 下发数据通道，当音频只有单路的时候，可以不传 */
@@ -1241,7 +1318,7 @@ export type StopRecordAndSendAudioDataParams = {
     dataChannel?: string;
     /** 扩展属性 */
     userData?: Attribute[];
-    success?: (params: null) => void;
+    success?: (params: AIStreamAudioFile | null) => void;
     fail?: (params: {
         errorMsg: string;
         errorCode: string | number;
@@ -1342,6 +1419,19 @@ export type SendTextDataParams = {
     }) => void;
     complete?: () => void;
 };
+export type InitAudioRecorderParams = {
+    sampleRate?: number;
+    success?: (params: null) => void;
+    fail?: (params: {
+        errorMsg: string;
+        errorCode: string | number;
+        innerError: {
+            errorCode: string | number;
+            errorMsg: string;
+        };
+    }) => void;
+    complete?: () => void;
+};
 export type SendFileDataParams = {
     /** 会话 id */
     sessionId: string;

package/dist/AIStreamTypes.js

@@ -187,6 +187,14 @@ export let AIStreamAttributePayloadType = /*#__PURE__*/function (AIStreamAttribu
   AIStreamAttributePayloadType[AIStreamAttributePayloadType["STRING"] = 6] = "STRING";
   return AIStreamAttributePayloadType;
 }({});
+export let AudioPlayChangedState = /*#__PURE__*/function (AudioPlayChangedState) {
+  AudioPlayChangedState[AudioPlayChangedState["START"] = 1] = "START";
+  AudioPlayChangedState[AudioPlayChangedState["BUFFERING"] = 2] = "BUFFERING";
+  AudioPlayChangedState[AudioPlayChangedState["BUFFERED"] = 3] = "BUFFERED";
+  AudioPlayChangedState[AudioPlayChangedState["COMPLETED"] = 4] = "COMPLETED";
+  AudioPlayChangedState[AudioPlayChangedState["ERROR"] = 5] = "ERROR";
+  return AudioPlayChangedState;
+}({});
 export let NetworkType = /*#__PURE__*/function (NetworkType) {
   NetworkType["NONE"] = "none";
   NetworkType["CELL_2G"] = "2g";

package/dist/asr/AsrAgent.d.ts

@@ -0,0 +1,59 @@
+import { AIStreamAudioFile, ConnectClientType } from '../AIStreamTypes';
+export interface AsrAgentOptions {
+    agentId: string;
+    /** 获取 agent token 的参数 */
+    tokenOptions?: {
+        api: string;
+        version: string;
+        extParams?: Record<string, any>;
+    };
+    /** 是否在构造时就初始化连接和录音参数 */
+    earlyStart?: boolean;
+    homeId?: number;
+    clientType?: ConnectClientType;
+    deviceId?: string;
+    onMessage?: (message: {
+        type: 'text';
+        text: string;
+    } | {
+        type: 'file';
+        file: AIStreamAudioFile;
+    }) => void;
+    onFinish?: () => void;
+    onError?: (error: any) => void;
+    /** 主动中断或者超过录制时长，会调用onAbort */
+    onAbort?: () => void;
+    recordingOptions?: {
+        /** 是否需要保存音频 */
+        saveFile?: boolean;
+        /** 保存的音频采样率，单位：hz */
+        sampleRate?: number;
+        /** 最长录制时长，单位：毫秒 */
+        maxDuration?: number;
+    };
+    enableLog?: boolean;
+}
+export declare class AsrAgent {
+    /** 数据链接，一个agent保持一个链接就可以 */
+    private streamConn;
+    /** 当前请求的session */
+    private activeSession;
+    private activeEvent;
+    /** 音频流监听 */
+    private audioStream;
+    options: AsrAgentOptions;
+    /** 录音时长定时器 */
+    private recordDurationTimer;
+    constructor(options: AsrAgentOptions);
+    /** 获取录音权限 */
+    getRecordScope(): Promise<boolean>;
+    /** 获取数据链接，一般只有一个链接就可以 */
+    private getConnection;
+    private createSession;
+    /** 开始录音时长监听 */
+    private startRecordTimer;
+    start(): Promise<void>;
+    stop(isAbort?: boolean): Promise<void>;
+    abort(): Promise<void>;
+    dispose(): void;
+}

package/dist/asr/AsrAgent.js

@@ -0,0 +1,235 @@
+import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
+import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
+import "core-js/modules/es.json.stringify.js";
+import { authorize, getCurrentHomeInfo, initAudioRecorder } from '../utils';
+import { AIStreamAttributePayloadType, AIStreamAttributeType, ConnectClientType, ConnectState, ReceivedTextPacketType, SessionState } from '../AIStreamTypes';
+import { DEFAULT_TOKEN_API, DEFAULT_TOKEN_API_VERSION, globalAIStreamClient } from '../global';
+import { Logger, safeParseJSON } from '@ray-js/t-agent';
+import logger from '../utils/logger';
+export class AsrAgent {
+  constructor(options) {
+    /** 数据链接，一个agent保持一个链接就可以 */
+    /** 当前请求的session */
+    /** 音频流监听 */
+    /** 录音时长定时器 */
+    _defineProperty(this, "recordDurationTimer", null);
+    this.options = options;
+    if (options.earlyStart) {
+      const {
+        recordingOptions
+      } = options;
+      const sampleRate = recordingOptions === null || recordingOptions === void 0 ? void 0 : recordingOptions.sampleRate;
+      // 如果需要提前启动，直接初始化和创建连接
+      this.createSession().then(() => initAudioRecorder(sampleRate ? {
+        sampleRate
+      } : {})).catch(error => {
+        logger.error('EarlyStart Failed to create ASR session:', error);
+      });
+    }
+    if (this.options.enableLog) {
+      Logger.setLogLevel('debug');
+    }
+  }
+
+  /** 获取录音权限 */
+  getRecordScope() {
+    return authorize({
+      scope: 'scope.record'
+    }).then(() => true, () => false);
+  }
+
+  /** 获取数据链接，一般只有一个链接就可以 */
+  getConnection(clientType, deviceId) {
+    if (this.streamConn) {
+      return this.streamConn;
+    }
+    // 创建 activeSession
+    this.streamConn = globalAIStreamClient.getConnection({
+      clientType: clientType || ConnectClientType.APP,
+      deviceId: deviceId
+    });
+    return this.streamConn;
+  }
+  async createSession() {
+    var _this$activeSession;
+    // 如果有激活的 Session，直接返回
+    if ((_this$activeSession = this.activeSession) !== null && _this$activeSession !== void 0 && _this$activeSession.sessionId) {
+      return this.activeSession;
+    }
+    const {
+      clientType,
+      deviceId,
+      tokenOptions,
+      agentId
+    } = this.options;
+    const streamConn = this.getConnection(clientType, deviceId);
+    let homeId = this.options.homeId;
+    if (!homeId) {
+      const info = await getCurrentHomeInfo();
+      homeId = +info.homeId;
+    }
+    this.activeSession = streamConn.createSession({
+      ownerId: clientType === ConnectClientType.DEVICE ? deviceId : "".concat(homeId),
+      api: (tokenOptions === null || tokenOptions === void 0 ? void 0 : tokenOptions.api) || DEFAULT_TOKEN_API,
+      apiVersion: (tokenOptions === null || tokenOptions === void 0 ? void 0 : tokenOptions.version) || DEFAULT_TOKEN_API_VERSION,
+      solutionCode: agentId,
+      extParams: _objectSpread({
+        onlyAsr: true
+      }, tokenOptions === null || tokenOptions === void 0 ? void 0 : tokenOptions.extParams)
+    });
+    return this.activeSession;
+  }
+
+  /** 开始录音时长监听 */
+  startRecordTimer() {
+    const {
+      maxDuration
+    } = this.options.recordingOptions || {};
+    if (maxDuration) {
+      if (this.recordDurationTimer) {
+        clearTimeout(this.recordDurationTimer);
+      }
+      this.recordDurationTimer = setTimeout(() => {
+        this.stop(true);
+      }, maxDuration);
+    }
+  }
+  async start() {
+    const hasScope = await this.getRecordScope();
+    if (!hasScope) {
+      throw new Error('authorize failed');
+    }
+    const {
+      onMessage,
+      onFinish,
+      onError
+    } = this.options || {};
+    const activeSession = await this.createSession();
+    const activeEvent = await activeSession.startEvent({
+      userData: [{
+        type: AIStreamAttributeType.AI_CHAT,
+        payloadType: AIStreamAttributePayloadType.STRING,
+        value: JSON.stringify({
+          'processing.interrupt': 'false',
+          'asr.enableVad': 'false'
+        })
+      }]
+    });
+    this.activeEvent = activeEvent;
+    const {
+      recordingOptions
+    } = this.options || {};
+    const {
+      sampleRate,
+      saveFile
+    } = recordingOptions || {};
+    const audioStream = activeEvent.stream({
+      type: 'audio',
+      sampleRate,
+      saveFile
+    });
+    this.audioStream = audioStream;
+    activeEvent.on('data', entry => {
+      if (entry.type === 'text') {
+        const packet = safeParseJSON(entry.body.text);
+        if (!packet || !packet.data) {
+          return;
+        }
+        if (packet.bizType !== ReceivedTextPacketType.ASR) {
+          return;
+        }
+        typeof onMessage === 'function' && onMessage({
+          type: 'text',
+          text: packet.data.text
+        });
+      } else if (entry.type === 'connectionState') {
+        if (entry.body.connectState === ConnectState.DISCONNECTED || entry.body.connectState === ConnectState.CLOSED) {
+          this.stop();
+          typeof onError === 'function' && onError(new Error('Connection closed'));
+        }
+      } else if (entry.type === 'sessionState') {
+        if (entry.body.sessionState === SessionState.CLOSED || entry.body.sessionState === SessionState.CREATE_FAILED) {
+          this.stop();
+          typeof onError === 'function' && onError(new Error('Session closed'));
+        }
+      }
+    });
+    let finished = false;
+    activeEvent.on('finish', () => {
+      if (finished) {
+        return;
+      }
+      this.stop();
+      typeof onFinish === 'function' && onFinish();
+      finished = true;
+    });
+    activeEvent.on('close', () => {
+      if (finished) {
+        return;
+      }
+      this.stop();
+      typeof onFinish === 'function' && onFinish();
+      finished = true;
+    });
+    activeEvent.on('error', error => {
+      typeof onError === 'function' && onError(error);
+    });
+    await audioStream.start();
+    this.startRecordTimer();
+  }
+  async stop(isAbort) {
+    if (this.recordDurationTimer) {
+      clearTimeout(this.recordDurationTimer);
+      this.recordDurationTimer = null;
+    }
+    if (this.audioStream) {
+      const file = await this.audioStream.stop();
+      console.log('Audio file result:', file);
+      if (file !== null && file !== void 0 && file.path) {
+        this.options.onMessage({
+          type: 'file',
+          file
+        });
+      }
+      this.audioStream = null;
+    }
+    if (this.activeEvent) {
+      if (isAbort) {
+        await this.activeEvent.abort();
+      } else {
+        await this.activeEvent.end();
+      }
+      this.activeEvent = null;
+    }
+    if (isAbort) {
+      const {
+        onAbort
+      } = this.options || {};
+      typeof onAbort === 'function' && onAbort();
+    }
+  }
+  async abort() {
+    await this.stop(true);
+  }
+  dispose() {
+    if (this.recordDurationTimer) {
+      clearTimeout(this.recordDurationTimer);
+      this.recordDurationTimer = null;
+    }
+    if (this.audioStream && this.audioStream.started) {
+      this.audioStream.stop();
+    }
+    this.audioStream = null;
+    if (this.activeEvent) {
+      this.activeEvent.abort();
+      this.activeEvent = null;
+    }
+    if (this.activeSession) {
+      this.activeSession.close();
+      this.activeSession = null;
+    }
+    if (this.streamConn) {
+      this.streamConn = null;
+    }
+  }
+}

package/dist/asr/createAsrAgent.d.ts

@@ -0,0 +1,31 @@
+import { AsrAgent, AsrAgentOptions } from './AsrAgent';
+/**
+ * 创建一个AsrAgent实例，用于语音转文本
+ * @example
+ * const asrAgent = createAsrAgent({
+ *   agentId: 'your_agent_id',
+ *   clientType: ConnectClientType.APP,
+ *   deviceId: 'deviceId',
+ *   maxDuration: 60,
+ *   onMessage: (message) => {
+ *     console.log('onMessage', message);
+ *   },
+ *   onFinish: () => {
+ *     console.log('onFinish');
+ *   },
+ *   onError: (error) => {
+ *     console.log('onError', error);
+ *   },
+ *   onAbort: () => {
+ *     console.log('onAbort');
+ *   },
+ * });
+ * // 开始录音
+ * asrAgent.start()
+ * // 停止录音
+ * asrAgent.stop()
+ * // 中断录音
+ * asrAgent.abort()
+ * @returns
+ */
+export declare function createAsrAgent(options: AsrAgentOptions): AsrAgent;

package/dist/asr/createAsrAgent.js

@@ -0,0 +1,35 @@
+import { AsrAgent } from './AsrAgent';
+
+/**
+ * 创建一个AsrAgent实例，用于语音转文本
+ * @example
+ * const asrAgent = createAsrAgent({
+ *   agentId: 'your_agent_id',
+ *   clientType: ConnectClientType.APP,
+ *   deviceId: 'deviceId',
+ *   maxDuration: 60,
+ *   onMessage: (message) => {
+ *     console.log('onMessage', message);
+ *   },
+ *   onFinish: () => {
+ *     console.log('onFinish');
+ *   },
+ *   onError: (error) => {
+ *     console.log('onError', error);
+ *   },
+ *   onAbort: () => {
+ *     console.log('onAbort');
+ *   },
+ * });
+ * // 开始录音
+ * asrAgent.start()
+ * // 停止录音
+ * asrAgent.stop()
+ * // 中断录音
+ * asrAgent.abort()
+ * @returns
+ */
+export function createAsrAgent(options) {
+  const asrAgent = new AsrAgent(options);
+  return asrAgent;
+}

package/dist/asr/index.d.ts

@@ -0,0 +1,3 @@
+import { AsrAgent } from './AsrAgent';
+import { createAsrAgent } from './createAsrAgent';
+export { AsrAgent, createAsrAgent };

package/dist/asr/index.js

@@ -0,0 +1,3 @@
+import { AsrAgent } from './AsrAgent';
+import { createAsrAgent } from './createAsrAgent';
+export { AsrAgent, createAsrAgent };

package/dist/buildIn/withBuildIn.js

@@ -1,6 +1,4 @@
 import "core-js/modules/web.dom-collections.iterator.js";
-import { getMiniAppConfig } from '../utils';
-import logger from '../utils/logger';
 import { BuildInSkillCode, ReceivedSmartHomeSkillAction } from '../AIStreamTypes';
 export function withBuildIn() {
   return _agent => {
@@ -17,27 +15,28 @@ export function withBuildIn() {
     const {
       onSkillsEnd
     } = agent.plugins.aiStream;
-    onAgentStart(async () => {
-      const agentId = session.get('AIStream.agentId');
-      if (agentId) {
-        const _conf = await session.get('AIAssistant.projectConfig');
-        if (!_conf) {
-          try {
-            const config = (await getMiniAppConfig({})).config || {};
-            const projectConfig = config[agentId];
-            if (projectConfig) {
-              logger.debug('getMiniAppConfig projectConfig', {
-                agentId,
-                projectConfig
-              });
-              await session.set('AIAssistant.projectConfig', projectConfig);
-            }
-          } catch (error) {
-            logger.warn('getMiniAppConfig error', error);
-          }
-        }
-      }
-    });
+
+    // onAgentStart(async () => {
+    //   const agentId = session.get('AIStream.agentId');
+    //   if (agentId) {
+    //     const _conf = session.get('AIStream.projectConfig');
+    //     if (!_conf) {
+    //       try {
+    //         const config: Record<string, ProjectConfig> = (await getMiniAppConfig({})).config || {};
+    //         const projectConfig = config[agentId];
+    //         if (projectConfig) {
+    //           logger.debug('getMiniAppConfig projectConfig', {
+    //             agentId,
+    //             projectConfig,
+    //           });
+    //           await session.set('AIAssistant.projectConfig', projectConfig);
+    //         }
+    //       } catch (error) {
+    //         logger.warn('getMiniAppConfig error', error);
+    //       }
+    //     }
+    //   }
+    // });
 
     // 关联文档
 

package/dist/index.d.ts

@@ -4,3 +4,4 @@ export * from './withAIStream';
 export * from './ChatHistoryStore';
 export * from './buildIn';
 export * from './ChatHistoryLocalStore';
+export * from './asr';

package/dist/index.js

@@ -3,4 +3,5 @@ export * from './AIStreamTypes';
 export * from './withAIStream';
 export * from './ChatHistoryStore';
 export * from './buildIn';
-export * from './ChatHistoryLocalStore';
+export * from './ChatHistoryLocalStore';
+export * from './asr';

package/dist/utils/AIStream.d.ts

@@ -1,4 +1,4 @@
-import { Attribute, BizTag, ConnectClientType, ConnectState, FileFormat, VideoCameraType } from '../AIStreamTypes';
+import { AIStreamAudioFile, Attribute, BizTag, ConnectClientType, ConnectState, FileFormat, VideoCameraType } from '../AIStreamTypes';
 import { AIStreamDataEntry, AIStreamObserverPool } from './observer';
 import { AIStreamError } from './errors';
 interface AIStreamConnectionOptions {
@@ -87,6 +87,8 @@ type AIStreamEventSource = {
     type: 'audio';
     dataChannel?: string;
     userData?: Attribute[];
+    saveFile?: boolean;
+    sampleRate?: number;
 } | {
     type: 'video';
     dataChannel?: string;
@@ -98,7 +100,7 @@ export interface AIStreamEventStream {
     dataChannel: string;
     started: boolean;
     start: () => Promise<void>;
-    stop: () => Promise<void>;
+    stop: () => Promise<AIStreamAudioFile | null>;
 }
 export declare class AIStreamEvent {
     readonly eventId: string;

package/dist/utils/AIStream.js

@@ -459,7 +459,11 @@ export class AIStreamEvent {
           startPromise = startRecordAndSendAudioData({
             sessionId: this.sessionId,
             dataChannel,
-            userData: source.userData
+            userData: source.userData,
+            recordInitParams: source.sampleRate ? {
+              sampleRate: source.sampleRate
+            } : undefined,
+            saveFile: source.saveFile
           });
           await startPromise;
         } else if (source.type === 'video') {
@@ -473,15 +477,16 @@ export class AIStreamEvent {
       },
       stop: async () => {
         if (!stream.started || this.closed) {
-          return;
+          return null;
         }
         // 一定要等录音开始，才能结束
         if (startPromise) {
           await tryCatchTTT(() => startPromise);
           startPromise = null;
         }
+        let file = null;
         if (source.type === 'audio') {
-          stopRecordAndSendAudioData({
+          file = await stopRecordAndSendAudioData({
             sessionId: this.sessionId,
             dataChannel,
             userData: source.userData
@@ -502,6 +507,7 @@ export class AIStreamEvent {
         });
         delete this.streams[dataChannel];
         stream.started = false;
+        return file;
       }
     };
     this.streams[dataChannel] = stream;

package/dist/utils/defaultMock.js

@@ -16,6 +16,7 @@ function splitString(input) {
   return input.match(/[a-zA-Z0-9]+\s*|[\u4e00-\u9fff]+\s*|[^\w\s\u4e00-\u9fff]+\s*|[\s]+/g) || [];
 }
 mock.data.set('sessionMap', new Map());
+mock.data.set('tokenMap', new Map());
 const getSession = (sessionId, eventId) => {
   const connection = getCurrentConnection();
   if (!connection) {
@@ -94,8 +95,15 @@ mock.hooks.hook('disconnect', context => {
   mock.data.set('sessionMap', new Map());
 });
 mock.hooks.hook('queryAgentToken', context => {
+  var _context$options$extP, _context$options$extP2;
+  const agentToken = generateId();
+  const map = mock.data.get('tokenMap');
+  map.set(agentToken, {
+    onlyAsr: ((_context$options$extP = context.options.extParams) === null || _context$options$extP === void 0 ? void 0 : _context$options$extP.onlyAsr) || false,
+    needTts: ((_context$options$extP2 = context.options.extParams) === null || _context$options$extP2 === void 0 ? void 0 : _context$options$extP2.needTts) || false
+  });
   context.result = {
-    agentToken: generateId(),
+    agentToken,
     bizConfig: {
       bizCode: BizCode.CHAT,
       sendData: ['audio', 'video', 'text', 'image'],
@@ -114,12 +122,15 @@ mock.hooks.hook('createSession', context => {
     throw new TTTError('not connect', AIStreamAppErrorCode.CONNECTION_INVALID);
   }
   const map = mock.data.get('sessionMap');
+  const tokenMap = mock.data.get('tokenMap');
+  const tokenConfig = tokenMap.get(context.options.agentToken);
   const session = {
     closed: false,
     sessionId: generateId(),
     sendDataChannels: ['audio', 'video', 'text', 'image'],
     revDataChannels: ['text', 'audio'],
-    currentEvent: null
+    currentEvent: null,
+    tokenConfig
   };
   map.set(session.sessionId, session);
   context.result = {
@@ -238,6 +249,11 @@ mock.hooks.hook('sendEventEnd', async context => {
   (async () => {
     var _ctx$responseSkills;
     await event.asr.promise;
+    if (session.tokenConfig.onlyAsr) {
+      event.replyEvent(EventType.EVENT_END);
+      session.currentEvent = null;
+      return;
+    }
     const ctx = {
       data: event.data,
       responseText: ''
@@ -454,7 +470,7 @@ mock.hooks.hook('stopRecordAndSendAudioData', async context => {
     throw new TTTError('stopRecordAndSendAudioData event not exists', AIStreamAppErrorCode.SESSION_ID_INVALID);
   }
   session.currentEvent.asr.finishController.abort(new Error('stopRecordAndSendAudioData'));
-  context.result = true;
+  context.result = null;
   await mock.sleep(100);
 });
 mock.hooks.hook('startRecordAndSendVideoData', context => {

package/dist/utils/index.d.ts

@@ -9,6 +9,5 @@ export * from './AIStream';
 export * from './observer';
 export * from './sendMessage';
 export * from './version';
-export * from './createAsrAgent';
 export * from './actions';
 export { mock };

package/dist/utils/index.js

@@ -9,6 +9,5 @@ export * from './AIStream';
 export * from './observer';
 export * from './sendMessage';
 export * from './version';
-export * from './createAsrAgent';
 export * from './actions';
 export { mock };

package/dist/utils/ttt.d.ts

@@ -1,4 +1,4 @@
-import { ApiRequestByAtopParams, ApiRequestByHighwayParams, AudioBody, AuthorizeParams, AuthorizePolicyStatusParams, CanIUseRouterParams, CloseSessionParams, ConnectParams, ConnectStateBody, CreateSessionParams, DeleteRecordListParams, DisconnectParams, EventBody, EventChannelMessageParams, GetAppInfoParams, GetCurrentHomeInfoParams, GetMiniAppConfigParams, GetAccountInfoParams, ImageBody, InsertRecordParams, NavigateToMiniProgramParams, OpenInnerH5Params, OpenMiniWidgetParams, QueryAgentTokenParams, QueryRecordListParams, RecordAmplitudesBody, RegisterChannelParams, RouterParams, SendEventChatBreakParams, SendEventEndParams, SendEventPayloadEndParams, SendEventStartParams, SendImageDataParams, SendTextDataParams, SessionStateBody, StartRecordAndSendAudioDataParams, StopRecordAndSendAudioDataParams, TextBody, UpdateRecordParams, IsConnectedParams, GetNetworkTypeParams, StartPlayAudioParams } from '../AIStreamTypes';
+import { ApiRequestByAtopParams, ApiRequestByHighwayParams, AudioBody, AuthorizeParams, AuthorizePolicyStatusParams, CanIUseRouterParams, CloseSessionParams, ConnectParams, ConnectStateBody, CreateSessionParams, DeleteRecordListParams, DisconnectParams, EventBody, EventChannelMessageParams, GetAppInfoParams, GetCurrentHomeInfoParams, GetMiniAppConfigParams, GetAccountInfoParams, ImageBody, InsertRecordParams, NavigateToMiniProgramParams, OpenInnerH5Params, OpenMiniWidgetParams, QueryAgentTokenParams, QueryRecordListParams, RecordAmplitudesBody, RegisterChannelParams, RouterParams, SendEventChatBreakParams, SendEventEndParams, SendEventPayloadEndParams, SendEventStartParams, SendImageDataParams, SendTextDataParams, SessionStateBody, StartRecordAndSendAudioDataParams, StopRecordAndSendAudioDataParams, TextBody, UpdateRecordParams, IsConnectedParams, GetNetworkTypeParams, StartPlayAudioParams, InitAudioRecorderParams, StopPlayAudioParams, AudioPlayChangedBody } from '../AIStreamTypes';
 export declare const getMiniAppConfig: (options?: Omit<GetMiniAppConfigParams, "success" | "fail"> | undefined) => Promise<{
     config: any;
 }>;
@@ -79,18 +79,18 @@ export declare const sendEventPayloadEnd: (options?: Omit<SendEventPayloadEndPar
 export declare const sendEventEnd: (options?: Omit<SendEventEndParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const sendEventChatBreak: (options?: Omit<SendEventChatBreakParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const startRecordAndSendAudioData: (options?: Omit<StartRecordAndSendAudioDataParams, "success" | "fail"> | undefined) => Promise<null>;
-export declare const stopRecordAndSendAudioData: (options?: Omit<StopRecordAndSendAudioDataParams, "success" | "fail"> | undefined) => Promise<null>;
+export declare const stopRecordAndSendAudioData: (options?: Omit<StopRecordAndSendAudioDataParams, "success" | "fail"> | undefined) => Promise<import("../AIStreamTypes").AIStreamAudioFile | null>;
 export declare const sendImageData: (options?: Omit<SendImageDataParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const sendTextData: (options?: Omit<SendTextDataParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const registerRecordAmplitudes: (options?: Omit<{
     count: number;
 }, "success" | "fail"> | undefined) => Promise<never>;
 export declare const unregisterVoiceAmplitudes: (options?: Omit<any, "success" | "fail"> | undefined) => Promise<unknown>;
+export declare const initAudioRecorder: (options?: Omit<InitAudioRecorderParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const startPlayAudio: (options?: Omit<StartPlayAudioParams, "success" | "fail"> | undefined) => Promise<null>;
+export declare const stopPlayAudio: (options?: Omit<StopPlayAudioParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const listenEventReceived: (listener: (params: EventBody) => void) => () => void;
-export declare const listenAudioPlayEnd: (listener: (params: {
-    path: string;
-}) => void) => () => void;
+export declare const listenAudioPlayChanged: (listener: (params: AudioPlayChangedBody) => void) => () => void;
 export declare const listenAudioReceived: (listener: (params: AudioBody) => void) => () => void;
 export declare const listenTextReceived: (listener: (params: TextBody) => void) => () => void;
 export declare const listenImageReceived: (listener: (params: ImageBody) => void) => () => void;

package/dist/utils/ttt.js

@@ -56,12 +56,14 @@ export const sendImageData = promisify(ty.aistream.sendImageData, true);
 export const sendTextData = promisify(ty.aistream.sendTextData, true);
 export const registerRecordAmplitudes = promisify(ty.aistream.registerRecordAmplitudes, true);
 export const unregisterVoiceAmplitudes = promisify(ty.aistream.unregisterVoiceAmplitudes, true);
+export const initAudioRecorder = promisify(ty.aistream.initAudioRecorder, true);
 export const startPlayAudio = promisify(ty.aistream.startPlayAudio, true);
+export const stopPlayAudio = promisify(ty.aistream.stopPlayAudio, true);
 
 // export const sendFileData = promisify<SendFileDataParams>(ty.aistream.sendFileData);
 
 export const listenEventReceived = listening(ty.aistream.onEventReceived, ty.aistream.offEventReceived, true);
-export const listenAudioPlayEnd = listening(ty.aistream.onAudioPlayEnd, ty.aistream.onAudioPlayEnd, true);
+export const listenAudioPlayChanged = listening(ty.aistream.onAudioPlayChanged, ty.aistream.onAudioPlayChanged, true);
 export const listenAudioReceived = listening(ty.aistream.onAudioReceived, ty.aistream.offAudioReceived, true);
 
 // export const listenVideoReceived = listening<VideoBody>(

package/dist/withAIStream.js

@@ -190,6 +190,28 @@ export function withAIStream() {
         await removeMessage(message);
       }
     });
+    const clearAllMessages = async () => {
+      // 删除内存中的消息
+      const messages = agent.session.messages.values();
+      await Promise.all(Array.from(messages).map(message => message.remove()));
+      // 清空缓存的数据
+      const historyStore = getHistoryStore();
+      if (historyStore) {
+        await historyStore.removeAll();
+      }
+    };
+    ui.hook('onClearHistory', async context => {
+      try {
+        await clearAllMessages();
+        context.result = {
+          success: true
+        };
+      } catch (e) {
+        context.result = {
+          success: false
+        };
+      }
+    });
     const send = (blocks, signal, extraOptions) => {
       const streamSession = session.get('AIStream.streamSession');
       const result = sendBlocksToAIStream({
@@ -517,16 +539,7 @@ export function withAIStream() {
         chat,
         options,
         removeMessage,
-        clearAllMessages: async () => {
-          // 删除内存中的消息
-          const messages = agent.session.messages.values();
-          await Promise.all(Array.from(messages).map(message => message.remove()));
-          // 清空缓存的数据
-          const historyStore = getHistoryStore();
-          if (historyStore) {
-            await historyStore.removeAll();
-          }
-        },
+        clearAllMessages,
         feedback,
         onSkillCompose: fn => {
           return hooks.hook('onSkillCompose', fn);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.3-beta-2",
+  "version": "0.2.3-beta-4",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -35,5 +35,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "13c4284de3e8b067813efee40be01c1992b448b3"
+  "gitHead": "952d287025737224227ead3ecf6dd4c432fec6a2"
 }

package/dist/utils/createAsrAgent.d.ts

@@ -1,79 +0,0 @@
-import { ConnectClientType } from '../AIStreamTypes';
-export interface AsrAgentOptions {
-    agentId: string;
-    tokenApi?: string;
-    tokenApiVersion?: string;
-    clientType?: ConnectClientType;
-    deviceId?: string;
-    /** 最长录制时长，单位：秒 */
-    maxDuration?: number;
-    onMessage?: (message: {
-        text: string;
-    }) => void;
-    onFinish?: () => void;
-    onError?: (error: any) => void;
-    /** 主动中断或者超过录制时长，会调用onAbort */
-    onAbort?: () => void;
-}
-declare class AsrAgent {
-    /** 数据链接，一个agent保持一个链接就可以 */
-    private streamConn;
-    /** 当前请求的session，每次请求都需要是一个新的session */
-    private activeSession;
-    private activeEvent;
-    /** 音频流监听 */
-    private audioStream;
-    options: AsrAgentOptions;
-    /** 录音时长定时器 */
-    private recordDurationTimer;
-    constructor(options: any);
-    /** 获取录音权限 */
-    getRecordScope(): Promise<boolean>;
-    /** 获取数据链接，一般只有一个链接就可以 */
-    private getConnection;
-    private createSession;
-    /** 开始录音时长监听 */
-    private startRecordTimer;
-    start(): Promise<void>;
-    stop(isAbort?: boolean): Promise<void>;
-    abort(): Promise<void>;
-}
-/**
- * 创建一个AsrAgent实例，用于语音转文本
- * @param options AsrAgentOptions
- * @param options.agentId 必填 语音转文本的agentId
- * @param options.tokenApi 语音转文本的tokenApi
- * @param options.tokenApiVersion 语音转文本的tokenApiVersion
- * @param options.clientType 语音转文本的clientType
- * @param options.deviceId 语音转文本的deviceId
- * @example
- * const asrAgent = createAsrAgent({
- *   agentId: 'asr-agent',
- *   tokenApi: 'xxxx',
- *   tokenApiVersion: '1.0',
- *   clientType: ConnectClientType.APP,
- *   deviceId: 'deviceId',
- *   maxDuration: 60,
- *   onMessage: (message) => {
- *     console.log('onMessage', message);
- *   },
- *   onFinish: () => {
- *     console.log('onFinish');
- *   },
- *   onError: (error) => {
- *     console.log('onError', error);
- *   },
- *   onAbort: () => {
- *     console.log('onAbort');
- *   },
- * });
- * // 开始录音
- * asrAgent.start()
- * // 停止录音
- * asrAgent.stop()
- * // 中断录音
- * asrAgent.abort()
- * @returns
- */
-export declare function createAsrAgent(options: AsrAgentOptions): AsrAgent;
-export {};

package/dist/utils/createAsrAgent.js

@@ -1,198 +0,0 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import "core-js/modules/es.json.stringify.js";
-import { authorize } from './ttt';
-import { DEFAULT_TOKEN_API, DEFAULT_TOKEN_API_VERSION, globalAIStreamClient } from '../global';
-import { ConnectClientType, AIStreamAttributePayloadType, AIStreamAttributeType } from '../AIStreamTypes';
-import logger from './logger';
-class AsrAgent {
-  constructor(options) {
-    /** 数据链接，一个agent保持一个链接就可以 */
-    /** 当前请求的session，每次请求都需要是一个新的session */
-    /** 音频流监听 */
-    /** 录音时长定时器 */
-    _defineProperty(this, "recordDurationTimer", null);
-    this.options = options;
-  }
-
-  /** 获取录音权限 */
-  getRecordScope() {
-    return authorize({
-      scope: 'scope.record'
-    }).then(() => true, () => false);
-  }
-
-  /** 获取数据链接，一般只有一个链接就可以 */
-  getConnection(clientType, deviceId) {
-    if (this.streamConn) {
-      return this.streamConn;
-    }
-    // 创建 activeSession
-    this.streamConn = globalAIStreamClient.getConnection({
-      clientType: clientType || ConnectClientType.APP,
-      deviceId: deviceId
-    });
-    return this.streamConn;
-  }
-  createSession() {
-    var _this$activeSession;
-    // 每次新的请求，都需要重新创建一个Session
-    const {
-      clientType,
-      deviceId,
-      tokenApi,
-      tokenApiVersion,
-      agentId
-    } = this.options;
-    const streamConn = this.getConnection(clientType, deviceId);
-    // 如果有激活的，需要释放
-    if ((_this$activeSession = this.activeSession) !== null && _this$activeSession !== void 0 && _this$activeSession.sessionId) {
-      this.activeSession.close();
-    }
-    const activeSession = streamConn.createSession({
-      api: tokenApi || DEFAULT_TOKEN_API,
-      apiVersion: tokenApiVersion || DEFAULT_TOKEN_API_VERSION,
-      solutionCode: agentId,
-      extParams: {
-        onlyAsr: true
-      }
-    });
-    this.activeSession = activeSession;
-    return this.activeSession;
-  }
-
-  /** 开始录音时长监听 */
-  startRecordTimer() {
-    const {
-      maxDuration
-    } = this.options;
-    if (maxDuration) {
-      if (this.recordDurationTimer) {
-        clearTimeout(this.recordDurationTimer);
-      }
-      this.recordDurationTimer = setTimeout(() => {
-        this.stop(true);
-      }, maxDuration * 1000);
-    }
-  }
-  async start() {
-    const hasScope = await this.getRecordScope();
-    if (!hasScope) {
-      throw new Error('authorize failed');
-    }
-    const {
-      onMessage,
-      onFinish,
-      onError
-    } = this.options || {};
-    const activeSession = this.createSession();
-    const activeEvent = await activeSession.startEvent({
-      userData: [{
-        type: AIStreamAttributeType.AI_CHAT,
-        payloadType: AIStreamAttributePayloadType.STRING,
-        value: JSON.stringify({
-          'processing.interrupt': 'false',
-          'asr.enableVad': 'false'
-        })
-      }]
-    });
-    this.activeEvent = activeEvent;
-    const audioStream = activeEvent.stream({
-      type: 'audio'
-    });
-    this.audioStream = audioStream;
-    await audioStream.start();
-    this.startRecordTimer();
-    activeEvent.on('data', entry => {
-      if (entry.type === 'text') {
-        console.log('text', entry, JSON.parse(entry.body.text));
-        let data = {
-          text: ''
-        };
-        try {
-          data = JSON.parse(entry.body.text) || {};
-        } catch (error) {
-          logger.error('JSON.parse error', error);
-        }
-        typeof onMessage === 'function' && onMessage(data);
-      }
-    });
-    activeEvent.on('finish', () => {
-      this.stop();
-      typeof onFinish === 'function' && onFinish();
-    });
-    activeEvent.on('error', error => {
-      typeof onError === 'function' && onError(error);
-    });
-  }
-  async stop(isAbort) {
-    var _this$activeSession2;
-    if (this.recordDurationTimer) {
-      clearTimeout(this.recordDurationTimer);
-      this.recordDurationTimer = null;
-    }
-    if (this.audioStream) {
-      await this.audioStream.stop();
-      this.audioStream = null;
-    }
-    if (this.activeEvent) {
-      if (isAbort) {
-        await this.activeEvent.abort();
-      } else {
-        await this.activeEvent.end();
-      }
-      this.activeEvent = null;
-    }
-    if (isAbort) {
-      const {
-        onAbort
-      } = this.options || {};
-      typeof onAbort === 'function' && onAbort();
-    }
-    await ((_this$activeSession2 = this.activeSession) === null || _this$activeSession2 === void 0 ? void 0 : _this$activeSession2.close());
-  }
-  async abort() {
-    await this.stop(true);
-  }
-}
-
-/**
- * 创建一个AsrAgent实例，用于语音转文本
- * @param options AsrAgentOptions
- * @param options.agentId 必填 语音转文本的agentId
- * @param options.tokenApi 语音转文本的tokenApi
- * @param options.tokenApiVersion 语音转文本的tokenApiVersion
- * @param options.clientType 语音转文本的clientType
- * @param options.deviceId 语音转文本的deviceId
- * @example
- * const asrAgent = createAsrAgent({
- *   agentId: 'asr-agent',
- *   tokenApi: 'xxxx',
- *   tokenApiVersion: '1.0',
- *   clientType: ConnectClientType.APP,
- *   deviceId: 'deviceId',
- *   maxDuration: 60,
- *   onMessage: (message) => {
- *     console.log('onMessage', message);
- *   },
- *   onFinish: () => {
- *     console.log('onFinish');
- *   },
- *   onError: (error) => {
- *     console.log('onError', error);
- *   },
- *   onAbort: () => {
- *     console.log('onAbort');
- *   },
- * });
- * // 开始录音
- * asrAgent.start()
- * // 停止录音
- * asrAgent.stop()
- * // 中断录音
- * asrAgent.abort()
- * @returns
- */
-export function createAsrAgent(options) {
-  const asrAgent = new AsrAgent(options);
-  return asrAgent;
-}