@ray-js/t-agent-plugin-aistream 0.2.3-beta-4 → 0.2.4-beta-1

package/dist/asr/AsrAgent.d.ts

@@ -23,6 +23,7 @@ export interface AsrAgentOptions {
     onError?: (error: any) => void;
     /** 主动中断或者超过录制时长，会调用onAbort */
     onAbort?: () => void;
+    onStatusChanged?: (status: AsrAgentStatus) => void;
     recordingOptions?: {
         /** 是否需要保存音频 */
         saveFile?: boolean;
@@ -33,6 +34,12 @@ export interface AsrAgentOptions {
     };
     enableLog?: boolean;
 }
+export declare enum AsrAgentStatus {
+    PENDING = "pending",
+    RECORDING = "recording",
+    ASR = "asr",
+    ABORTING = "aborting"
+}
 export declare class AsrAgent {
     /** 数据链接，一个agent保持一个链接就可以 */
     private streamConn;
@@ -41,19 +48,28 @@ export declare class AsrAgent {
     private activeEvent;
     /** 音频流监听 */
     private audioStream;
-    options: AsrAgentOptions;
+    private options;
     /** 录音时长定时器 */
     private recordDurationTimer;
+    private startPromise;
+    private authorized;
+    _status: AsrAgentStatus;
+    private disposed;
+    get status(): AsrAgentStatus;
+    set status(status: AsrAgentStatus);
     constructor(options: AsrAgentOptions);
     /** 获取录音权限 */
-    getRecordScope(): Promise<boolean>;
+    private getRecordScope;
     /** 获取数据链接，一般只有一个链接就可以 */
     private getConnection;
     private createSession;
     /** 开始录音时长监听 */
     private startRecordTimer;
+    private _start;
+    private _abort;
+    private _stop;
     start(): Promise<void>;
-    stop(isAbort?: boolean): Promise<void>;
+    stop(): Promise<void>;
     abort(): Promise<void>;
     dispose(): void;
 }

package/dist/asr/AsrAgent.js

@@ -1,18 +1,44 @@
 import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
 import "core-js/modules/es.json.stringify.js";
+import "core-js/modules/es.promise.finally.js";
+import "core-js/modules/web.dom-collections.iterator.js";
 import { authorize, getCurrentHomeInfo, initAudioRecorder } from '../utils';
 import { AIStreamAttributePayloadType, AIStreamAttributeType, ConnectClientType, ConnectState, ReceivedTextPacketType, SessionState } from '../AIStreamTypes';
 import { DEFAULT_TOKEN_API, DEFAULT_TOKEN_API_VERSION, globalAIStreamClient } from '../global';
 import { Logger, safeParseJSON } from '@ray-js/t-agent';
 import logger from '../utils/logger';
+import { tryCatch } from '../utils/misc';
+export let AsrAgentStatus = /*#__PURE__*/function (AsrAgentStatus) {
+  AsrAgentStatus["PENDING"] = "pending";
+  AsrAgentStatus["RECORDING"] = "recording";
+  AsrAgentStatus["ASR"] = "asr";
+  AsrAgentStatus["ABORTING"] = "aborting";
+  return AsrAgentStatus;
+}({});
 export class AsrAgent {
+  get status() {
+    return this._status;
+  }
+  set status(status) {
+    this._status = status;
+    const {
+      onStatusChanged
+    } = this.options || {};
+    if (typeof onStatusChanged === 'function') {
+      onStatusChanged(status);
+    }
+  }
   constructor(options) {
     /** 数据链接，一个agent保持一个链接就可以 */
     /** 当前请求的session */
     /** 音频流监听 */
     /** 录音时长定时器 */
     _defineProperty(this, "recordDurationTimer", null);
+    _defineProperty(this, "startPromise", null);
+    _defineProperty(this, "authorized", null);
+    _defineProperty(this, "_status", AsrAgentStatus.PENDING);
+    _defineProperty(this, "disposed", false);
     this.options = options;
     if (options.earlyStart) {
       const {
@@ -20,7 +46,7 @@ export class AsrAgent {
       } = options;
       const sampleRate = recordingOptions === null || recordingOptions === void 0 ? void 0 : recordingOptions.sampleRate;
       // 如果需要提前启动，直接初始化和创建连接
-      this.createSession().then(() => initAudioRecorder(sampleRate ? {
+      this.createSession().then(session => session.ensureSession()).then(() => initAudioRecorder(sampleRate ? {
         sampleRate
       } : {})).catch(error => {
         logger.error('EarlyStart Failed to create ASR session:', error);
@@ -33,9 +59,15 @@ export class AsrAgent {
 
   /** 获取录音权限 */
   getRecordScope() {
+    if (this.authorized !== null) {
+      return Promise.resolve(this.authorized);
+    }
     return authorize({
       scope: 'scope.record'
-    }).then(() => true, () => false);
+    }).then(() => true, () => false).then(authorized => {
+      this.authorized = authorized;
+      return authorized;
+    });
   }
 
   /** 获取数据链接，一般只有一个链接就可以 */
@@ -90,31 +122,60 @@ export class AsrAgent {
         clearTimeout(this.recordDurationTimer);
       }
       this.recordDurationTimer = setTimeout(() => {
-        this.stop(true);
+        this._stop();
       }, maxDuration);
     }
   }
-  async start() {
-    const hasScope = await this.getRecordScope();
-    if (!hasScope) {
-      throw new Error('authorize failed');
-    }
+  async _start() {
+    this.status = AsrAgentStatus.RECORDING;
+    let finished = false;
     const {
       onMessage,
       onFinish,
-      onError
+      onError,
+      onAbort
     } = this.options || {};
+    const finish = error => {
+      if (this.disposed) {
+        return;
+      }
+      if (finished) {
+        return;
+      }
+      this.audioStream = null;
+      this.activeEvent = null;
+      finished = true;
+      const prevStatus = this.status;
+      this.status = AsrAgentStatus.PENDING;
+      if (prevStatus === AsrAgentStatus.ABORTING) {
+        typeof onAbort === 'function' && onAbort();
+      } else if (error) {
+        typeof onError === 'function' && onError(error);
+      } else {
+        typeof onFinish === 'function' && onFinish();
+      }
+    };
+    const hasScope = await this.getRecordScope();
+    if (!hasScope) {
+      finish(new Error('No record permission granted'));
+      return;
+    }
     const activeSession = await this.createSession();
-    const activeEvent = await activeSession.startEvent({
+    const attribute = {
+      'processing.interrupt': 'false',
+      'asr.enableVad': 'false'
+    };
+    const [startEventError, activeEvent] = await tryCatch(() => activeSession.startEvent({
       userData: [{
         type: AIStreamAttributeType.AI_CHAT,
         payloadType: AIStreamAttributePayloadType.STRING,
-        value: JSON.stringify({
-          'processing.interrupt': 'false',
-          'asr.enableVad': 'false'
-        })
+        value: JSON.stringify(attribute)
       }]
-    });
+    }));
+    if (startEventError) {
+      finish(startEventError);
+      return;
+    }
     this.activeEvent = activeEvent;
     const {
       recordingOptions
@@ -144,92 +205,109 @@ export class AsrAgent {
         });
       } else if (entry.type === 'connectionState') {
         if (entry.body.connectState === ConnectState.DISCONNECTED || entry.body.connectState === ConnectState.CLOSED) {
-          this.stop();
-          typeof onError === 'function' && onError(new Error('Connection closed'));
+          finish(new Error('Connection closed'));
         }
       } else if (entry.type === 'sessionState') {
         if (entry.body.sessionState === SessionState.CLOSED || entry.body.sessionState === SessionState.CREATE_FAILED) {
-          this.stop();
-          typeof onError === 'function' && onError(new Error('Session closed'));
+          finish(new Error('Session closed'));
         }
       }
     });
-    let finished = false;
     activeEvent.on('finish', () => {
-      if (finished) {
-        return;
-      }
-      this.stop();
-      typeof onFinish === 'function' && onFinish();
-      finished = true;
+      finish();
     });
     activeEvent.on('close', () => {
-      if (finished) {
-        return;
-      }
-      this.stop();
-      typeof onFinish === 'function' && onFinish();
-      finished = true;
+      finish();
     });
     activeEvent.on('error', error => {
-      typeof onError === 'function' && onError(error);
+      finish(error);
     });
-    await audioStream.start();
+    const [error] = await tryCatch(() => audioStream.start());
+    if (error) {
+      finish(error);
+      return;
+    }
     this.startRecordTimer();
   }
-  async stop(isAbort) {
+  async _abort() {
+    this.status = AsrAgentStatus.ABORTING;
+    if (this.recordDurationTimer) {
+      clearTimeout(this.recordDurationTimer);
+      this.recordDurationTimer = null;
+    }
+    if (this.startPromise) {
+      await this.startPromise;
+    }
+    this.activeEvent.abort();
+  }
+  async _stop() {
     if (this.recordDurationTimer) {
       clearTimeout(this.recordDurationTimer);
       this.recordDurationTimer = null;
     }
+    if (this.startPromise) {
+      await this.startPromise;
+    }
     if (this.audioStream) {
       const file = await this.audioStream.stop();
-      console.log('Audio file result:', file);
-      if (file !== null && file !== void 0 && file.path) {
+      if (file !== null && file !== void 0 && file.path && typeof this.options.onMessage === 'function') {
         this.options.onMessage({
           type: 'file',
           file
         });
       }
-      this.audioStream = null;
     }
     if (this.activeEvent) {
-      if (isAbort) {
-        await this.activeEvent.abort();
-      } else {
-        await this.activeEvent.end();
-      }
-      this.activeEvent = null;
+      await this.activeEvent.end();
+      this.status = AsrAgentStatus.ASR;
     }
-    if (isAbort) {
-      const {
-        onAbort
-      } = this.options || {};
-      typeof onAbort === 'function' && onAbort();
+  }
+  start() {
+    if (this.disposed) {
+      throw new Error('AsrAgent has been disposed');
+    }
+    if (this.status !== AsrAgentStatus.PENDING) {
+      throw new Error('AsrAgent is already started or in progress');
+    }
+    if (!this.startPromise) {
+      this.startPromise = this._start().finally(() => {
+        this.startPromise = null;
+      });
     }
+    return this.startPromise;
+  }
+  async stop() {
+    if (this.disposed) {
+      throw new Error('AsrAgent has been disposed');
+    }
+    if (this.status !== AsrAgentStatus.RECORDING) {
+      throw new Error('AsrAgent is not in recording state');
+    }
+    await this._stop();
   }
   async abort() {
-    await this.stop(true);
+    if (this.disposed) {
+      throw new Error('AsrAgent has been disposed');
+    }
+    if (this.status !== AsrAgentStatus.RECORDING && this.status !== AsrAgentStatus.ASR) {
+      throw new Error('AsrAgent is not in recording or asr state');
+    }
+    return this._abort();
   }
   dispose() {
     if (this.recordDurationTimer) {
       clearTimeout(this.recordDurationTimer);
       this.recordDurationTimer = null;
     }
-    if (this.audioStream && this.audioStream.started) {
-      this.audioStream.stop();
-    }
-    this.audioStream = null;
-    if (this.activeEvent) {
-      this.activeEvent.abort();
-      this.activeEvent = null;
+    if (this.disposed) {
+      return;
     }
     if (this.activeSession) {
       this.activeSession.close();
-      this.activeSession = null;
-    }
-    if (this.streamConn) {
-      this.streamConn = null;
     }
+    this.audioStream = null;
+    this.activeEvent = null;
+    this.streamConn = null;
+    this.activeSession = null;
   }
 }

package/dist/asr/index.d.ts

@@ -1,3 +1,3 @@
-import { AsrAgent } from './AsrAgent';
+import { AsrAgent, AsrAgentStatus } from './AsrAgent';
 import { createAsrAgent } from './createAsrAgent';
-export { AsrAgent, createAsrAgent };
+export { AsrAgent, createAsrAgent, AsrAgentStatus };

package/dist/asr/index.js

@@ -1,3 +1,3 @@
-import { AsrAgent } from './AsrAgent';
+import { AsrAgent, AsrAgentStatus } from './AsrAgent';
 import { createAsrAgent } from './createAsrAgent';
-export { AsrAgent, createAsrAgent };
+export { AsrAgent, createAsrAgent, AsrAgentStatus };

package/dist/utils/authorizePolicy.d.ts

@@ -0,0 +1 @@
+export declare function authorizeAssistantPolicy(): Promise<boolean>;

package/dist/utils/authorizePolicy.js

@@ -0,0 +1,51 @@
+import { authorizePolicyStatus, listenReceiveMessage, openMiniWidget, registerChannel } from './ttt';
+import logger from './logger';
+import { AuthorizePolicySign } from '../AIStreamTypes';
+export async function authorizeAssistantPolicy() {
+  // @ts-ignore
+  if (!ty.authorizePolicyStatus) {
+    logger.warn('authorizeAssistantPolicy: ty.authorizePolicyStatus not found, check if BaseKit >= 3.20.4');
+    // 不支持权限状态查询，直接返回true
+    return true;
+  }
+  const {
+    sign
+  } = await authorizePolicyStatus({
+    type: 'ai_algorithm'
+  });
+  if (sign === AuthorizePolicySign.AUTHORIZED) {
+    // 已授权，直接返回true
+    return true;
+  }
+
+  // 未授权，打开小程序授权
+  await registerChannel({
+    eventName: 'AIAuthorization'
+  });
+  const resultPromise = new Promise(resolve => {
+    // 一次性事件
+    const cancel = listenReceiveMessage(res => {
+      const {
+        status
+      } = res.event || {};
+      if (status) {
+        // 同意，直接进入
+        resolve(true);
+      } else {
+        resolve(false);
+      }
+      cancel();
+    });
+  });
+
+  // 打开小程序授权
+  await openMiniWidget({
+    appId: 'ty9mvmbhtvwrp5zbla',
+    pagePath: 'cards/index',
+    autoDismiss: false,
+    autoCache: false,
+    // versionType: ty.WidgetVersionType.preview,
+    style: '0.68'
+  });
+  return resultPromise;
+}

package/dist/utils/defaultMock.js

@@ -4,6 +4,7 @@ import "core-js/modules/es.json.stringify.js";
 import "core-js/modules/es.regexp.exec.js";
 import "core-js/modules/esnext.iterator.constructor.js";
 import "core-js/modules/esnext.iterator.filter.js";
+import "core-js/modules/esnext.iterator.for-each.js";
 import "core-js/modules/esnext.iterator.map.js";
 import "core-js/modules/web.dom-collections.iterator.js";
 import { mock } from './mock';
@@ -140,19 +141,19 @@ mock.hooks.hook('createSession', context => {
   };
 
   // 用于测试断开连接;
-  // setTimeout(() => {
-  //   dispatch('onConnectStateChanged', {
-  //     connectionId: '',
-  //     connectState: ConnectState.DISCONNECTED,
-  //     code: 200,
-  //   });
-  //   mock.data.set('currentConnection', null);
-  //   const map: Map<string, MockSession> = mock.data.get('sessionMap');
-  //   map.forEach(s => {
-  //     s.closed = true;
-  //   });
-  //   mock.data.set('sessionMap', new Map());
-  // }, 5000);
+  setTimeout(() => {
+    dispatch('onConnectStateChanged', {
+      connectionId: '',
+      connectState: ConnectState.DISCONNECTED,
+      code: 200
+    });
+    mock.data.set('currentConnection', null);
+    const map = mock.data.get('sessionMap');
+    map.forEach(s => {
+      s.closed = true;
+    });
+    mock.data.set('sessionMap', new Map());
+  }, 5000);
 
   // 用于测试断开会话
   // setTimeout(() => {
@@ -168,6 +169,8 @@ mock.hooks.hook('createSession', context => {
 mock.hooks.hook('closeSession', context => {
   const map = mock.data.get('sessionMap');
   const sessionId = context.options.sessionId;
+  const session = map.get(sessionId);
+  session.closed = true;
   map.delete(sessionId);
   context.result = true;
 });
@@ -273,7 +276,7 @@ mock.hooks.hook('sendEventEnd', async context => {
     if (ctx.data.length === 0) {
       event.replyEvent(EventType.EVENT_START);
       await mock.sleep(100);
-      if (event.controller.signal.aborted) {
+      if (event.controller.signal.aborted || session.closed) {
         return;
       }
       event.replyEvent(EventType.EVENT_END);
@@ -282,19 +285,19 @@ mock.hooks.hook('sendEventEnd', async context => {
     }
     const text = ctx.responseText || '⚠️ No mock text response matched!';
     const words = splitString(text);
-    if (event.controller.signal.aborted) {
+    if (event.controller.signal.aborted || session.closed) {
       return;
     }
     event.replyEvent(EventType.EVENT_START);
     await mock.sleep(500);
-    if (event.controller.signal.aborted) {
+    if (event.controller.signal.aborted || session.closed) {
       return;
     }
     const bizId = generateId();
     event.replyText(StreamFlag.START);
     for (const word of words) {
       await mock.sleep(100);
-      if (event.controller.signal.aborted) {
+      if (event.controller.signal.aborted || session.closed) {
         return;
       }
       event.replyText(StreamFlag.IN_PROGRESS, {
@@ -309,7 +312,7 @@ mock.hooks.hook('sendEventEnd', async context => {
       });
     }
     await mock.sleep(100);
-    if (event.controller.signal.aborted) {
+    if (event.controller.signal.aborted || session.closed) {
       return;
     }
     event.replyText(StreamFlag.IN_PROGRESS, {
@@ -323,13 +326,13 @@ mock.hooks.hook('sendEventEnd', async context => {
       }
     });
     await mock.sleep(10);
-    if (event.controller.signal.aborted) {
+    if (event.controller.signal.aborted || session.closed) {
       return;
     }
     if ((_ctx$responseSkills = ctx.responseSkills) !== null && _ctx$responseSkills !== void 0 && _ctx$responseSkills.length) {
       for (const skill of ctx.responseSkills) {
         await mock.sleep(100);
-        if (event.controller.signal.aborted) {
+        if (event.controller.signal.aborted || session.closed) {
           return;
         }
         event.replyText(StreamFlag.IN_PROGRESS, {
@@ -342,7 +345,7 @@ mock.hooks.hook('sendEventEnd', async context => {
     }
     event.replyText(StreamFlag.END);
     await mock.sleep(500);
-    if (event.controller.signal.aborted) {
+    if (event.controller.signal.aborted || session.closed) {
       return;
     }
     event.replyEvent(EventType.EVENT_END);
@@ -400,13 +403,13 @@ mock.hooks.hook('startRecordAndSendAudioData', async context => {
     let text = '';
     finishController.signal.addEventListener('abort', async () => {
       var _finishResolve;
-      if (event.controller.signal.aborted) {
+      if (event.controller.signal.aborted || session.closed) {
         return;
       }
 
       // 终止识别到出完整结果的延迟
       await mock.sleep(500);
-      if (event.controller.signal.aborted) {
+      if (event.controller.signal.aborted || session.closed) {
         return;
       }
       event.replyText(StreamFlag.IN_PROGRESS, {
@@ -418,7 +421,7 @@ mock.hooks.hook('startRecordAndSendAudioData', async context => {
         }
       });
       await mock.sleep(100);
-      if (event.controller.signal.aborted) {
+      if (event.controller.signal.aborted || session.closed) {
         return;
       }
       event.replyText(StreamFlag.END);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.3-beta-4",
+  "version": "0.2.4-beta-1",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -35,5 +35,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "952d287025737224227ead3ecf6dd4c432fec6a2"
+  "gitHead": "83f32cc9add5ebd4c30b426b3593d51c1893b2eb"
 }