@ray-js/t-agent-plugin-aistream 0.2.0-beta-1

package/dist/utils/AIStream.js

@@ -0,0 +1,486 @@
+import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
+import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
+import "core-js/modules/esnext.iterator.constructor.js";
+import "core-js/modules/esnext.iterator.find.js";
+import "core-js/modules/esnext.iterator.for-each.js";
+import "core-js/modules/esnext.iterator.map.js";
+import "core-js/modules/web.dom-collections.iterator.js";
+import { AIStreamErrorCode, BizTag, ConnectClientType, ConnectState, EventType, SessionState } from '../AIStreamTypes';
+import { closeSession, connect, createSession, disconnect, getCurrentHomeInfo, queryAgentToken, registerRecordAmplitudes, sendEventChatBreak, sendEventEnd, sendEventPayloadEnd, sendEventStart, sendImageData, sendTextData, startRecordAndSendAudioData, stopRecordAndSendAudioData, unregisterVoiceAmplitudes } from './ttt';
+import { AIStreamObserver, AIStreamObserverPool } from './observer';
+import { isAbortError } from '@ray-js/t-agent';
+export class AIStreamClient {
+  constructor() {
+    _defineProperty(this, "pool", new AIStreamObserverPool());
+    _defineProperty(this, "cached", new Map());
+  }
+  getConnection(options) {
+    const key = "".concat(options.clientType, "|").concat(options.deviceId || '');
+    let connection = this.cached.get(key);
+    if (!connection) {
+      connection = new AIStreamConnection(this.pool, options);
+    }
+    return connection;
+  }
+}
+export class AIStreamConnection {
+  constructor(pool, options) {
+    _defineProperty(this, "connectionId", null);
+    _defineProperty(this, "state", ConnectState.INIT);
+    _defineProperty(this, "activeSessions", new Set());
+    _defineProperty(this, "promise", null);
+    _defineProperty(this, "observer", null);
+    _defineProperty(this, "onStateChanged", entry => {
+      if (entry.type === 'connectionState' && entry.body.connectionId === this.connectionId) {
+        this.activeSessions.forEach(session => {
+          if (session.sessionId) {
+            session._onStateChanged(entry);
+          }
+        });
+        if (entry.body.connectState === ConnectState.DISCONNECTED || entry.body.connectState === ConnectState.CLOSED) {
+          // 事件触发的时候，只做清理
+          this.state = entry.body.connectState;
+          this.cleanup();
+        }
+      }
+      if (entry.type === 'sessionState') {
+        this.activeSessions.forEach(session => {
+          if (session.sessionId && session.sessionId === entry.body.sessionId) {
+            session._onStateChanged(entry);
+            if (entry.body.sessionState === SessionState.CLOSED || entry.body.sessionState === SessionState.CREATE_FAILED) {
+              this.activeSessions.delete(session);
+            }
+          }
+        });
+      }
+    });
+    this.pool = pool;
+    this.options = options;
+  }
+  _ensureConnected() {
+    if (this.promise) {
+      return this.promise;
+    }
+    if (this.state === ConnectState.CONNECTED) {
+      return Promise.resolve();
+    }
+    this.promise = (async () => {
+      // 监听断开事件，重置 state & 清理
+      this.observer = new AIStreamObserver(this.onStateChanged, this.pool);
+      this.observer.observe({
+        connectionState: true,
+        sessionState: true
+      });
+      // 调用 SDK connect
+      const res = await connect(this.options);
+      this.connectionId = res.connectionId;
+      this.state = ConnectState.CONNECTED;
+      this.promise = null;
+    })();
+    return this.promise;
+  }
+  cleanup() {
+    var _this$observer;
+    (_this$observer = this.observer) === null || _this$observer === void 0 || _this$observer.disconnect();
+    this.observer = null;
+    this.connectionId = null;
+    this.promise = null;
+    this.activeSessions.forEach(s => s.cleanup());
+    this.activeSessions.clear();
+  }
+  createSession(options) {
+    const session = new AIStreamSession(this, this.pool, options);
+    this.activeSessions.add(session);
+    return session;
+  }
+  async closeSession(session) {
+    if (session.sessionId && this.connectionId) {
+      await closeSession({
+        sessionId: session.sessionId,
+        code: AIStreamErrorCode.OK
+      });
+    }
+    this.activeSessions.delete(session);
+  }
+
+  // 断连
+  async close() {
+    if (this.connectionId) {
+      // 只有设备端才需要断开连接，App 端由 App 来维持
+      if (this.options.clientType === ConnectClientType.DEVICE) {
+        await disconnect({
+          connectionId: this.connectionId
+        });
+      }
+      this.state = ConnectState.CLOSED;
+      this.cleanup();
+    }
+  }
+}
+export class AIStreamSession {
+  constructor(connection, pool, options) {
+    _defineProperty(this, "sessionId", null);
+    _defineProperty(this, "sendDataChannels", []);
+    _defineProperty(this, "revDataChannels", []);
+    _defineProperty(this, "promise", null);
+    _defineProperty(this, "_onStateChanged", entry => {
+      var _this$activeEvent;
+      (_this$activeEvent = this.activeEvent) === null || _this$activeEvent === void 0 || _this$activeEvent.emit('data', entry);
+      if (entry.type === 'sessionState' && (entry.body.sessionState === SessionState.CLOSED || entry.body.sessionState === SessionState.CREATE_FAILED)) {
+        this.cleanup();
+      }
+    });
+    _defineProperty(this, "onDataEntry", entry => {
+      var _this$activeEvent2;
+      if (!this.activeEvent) {
+        return;
+      }
+      (_this$activeEvent2 = this.activeEvent) === null || _this$activeEvent2 === void 0 || _this$activeEvent2.emit('data', entry);
+      if (entry.type === 'event' && entry.body.eventId === this.activeEvent.eventId) {
+        const {
+          eventType
+        } = entry.body;
+        if (eventType === EventType.EVENT_END || eventType === EventType.CHAT_BREAK || eventType === EventType.ONE_SHOT) {
+          var _this$activeEvent3;
+          (_this$activeEvent3 = this.activeEvent) === null || _this$activeEvent3 === void 0 || _this$activeEvent3.emit('finish');
+          this.cleanupEvent();
+        }
+      }
+    });
+    this.connection = connection;
+    this.pool = pool;
+    this.options = options;
+  }
+  ensureSession() {
+    if (this.promise) {
+      return this.promise;
+    }
+    if (this.sessionId) {
+      return Promise.resolve();
+    }
+    this.promise = (async () => {
+      await this.connection._ensureConnected();
+      let ownerId = this.options.ownerId;
+      if (!ownerId) {
+        if (this.connection.options.clientType === ConnectClientType.APP) {
+          const {
+            homeId
+          } = await getCurrentHomeInfo();
+          ownerId = homeId;
+        } else {
+          ownerId = this.connection.options.deviceId;
+        }
+        this.options.ownerId = ownerId;
+      }
+      const options = _objectSpread({
+        bizTag: BizTag.DEFAULT,
+        ownerId,
+        extParams: {}
+      }, this.options);
+      const {
+        agentToken,
+        bizConfig
+      } = await queryAgentToken(options);
+      const res = await createSession({
+        bizTag: options.bizTag,
+        agentToken,
+        bizConfig,
+        userData: options.userData
+      });
+      this.sessionId = res.sessionId;
+      this.sendDataChannels = res.sendDataChannels;
+      this.revDataChannels = res.revDataChannels;
+    })();
+    return this.promise;
+  }
+  async startEvent() {
+    let options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
+    if (this.activeEvent) {
+      throw new Error('Cannot start a new event while another is active');
+    }
+    await this.ensureSession();
+    const {
+      eventId
+    } = await sendEventStart(_objectSpread({
+      sessionId: this.sessionId
+    }, options));
+    this.activeEvent = new AIStreamEvent({
+      eventId,
+      sessionId: this.sessionId,
+      sendDataChannels: this.sendDataChannels
+    });
+    const onError = error => {
+      if (isAbortError(error)) {
+        this.cleanupEvent();
+      }
+    };
+    this.activeEvent.on('error', onError);
+    const observerOptions = {
+      event: true,
+      dataChannels: this.revDataChannels,
+      sessionId: this.sessionId
+    };
+    this.activeObserver = new AIStreamObserver(this.onDataEntry, this.pool);
+    for (const revDataChannel of this.revDataChannels) {
+      if (revDataChannel.startsWith('text')) {
+        observerOptions.text = true;
+      }
+      if (revDataChannel.startsWith('audio')) {
+        observerOptions.audio = true;
+      }
+      if (revDataChannel.startsWith('video')) {
+        observerOptions.video = true;
+      }
+      if (revDataChannel.startsWith('file')) {
+        observerOptions.file = true;
+      }
+      if (revDataChannel.startsWith('image')) {
+        observerOptions.image = true;
+      }
+    }
+    this.activeObserver.observe(observerOptions);
+    return this.activeEvent;
+  }
+  cleanupEvent() {
+    var _this$activeObserver, _this$activeEvent4;
+    (_this$activeObserver = this.activeObserver) === null || _this$activeObserver === void 0 || _this$activeObserver.disconnect();
+    this.activeObserver = null;
+    (_this$activeEvent4 = this.activeEvent) === null || _this$activeEvent4 === void 0 || _this$activeEvent4.emit('close');
+    this.activeEvent = null;
+  }
+  cleanup() {
+    this.cleanupEvent();
+    this.sessionId = null;
+    this.sendDataChannels = [];
+    this.revDataChannels = [];
+  }
+
+  // 会话关闭清理
+  async close() {
+    await this.connection.closeSession(this);
+    if (this.sessionId) {
+      this.cleanup();
+    }
+  }
+}
+export class AIStreamEvent {
+  constructor(init) {
+    _defineProperty(this, "closed", false);
+    _defineProperty(this, "chains", Object.create(null));
+    _defineProperty(this, "streams", Object.create(null));
+    _defineProperty(this, "listeners", {
+      finish: new Set(),
+      error: new Set(),
+      data: new Set(),
+      close: new Set()
+    });
+    this.eventId = init.eventId;
+    this.sessionId = init.sessionId;
+    this.sendDataChannels = init.sendDataChannels;
+  }
+  findFirstCode(type) {
+    const code = this.sendDataChannels.find(code => code.startsWith(type));
+    if (!code) {
+      throw new Error("No available data code for type: ".concat(type));
+    }
+    return code;
+  }
+  write(chunk) {
+    if (this.closed) {
+      throw new Error('Cannot write to a closed event');
+    }
+    const dataChannel = chunk.dataChannel || this.findFirstCode(chunk.type);
+    let promise = this.chains[dataChannel];
+    if (!promise) {
+      promise = Promise.resolve();
+    }
+    this.chains[dataChannel] = promise.then(async () => {
+      if (this.closed) {
+        return;
+      }
+      if (chunk.type === 'text') {
+        await sendTextData({
+          sessionId: this.sessionId,
+          text: chunk.text,
+          dataChannel,
+          userData: chunk.userData
+        });
+      } else if (chunk.type === 'file') {
+        // await sendFileData({
+        //   sessionId: this.sessionId,
+        //   path: chunk.path,
+        //   format: chunk.format,
+        //   dataChannel,
+        //   userData: chunk.userData,
+        // });
+      } else if (chunk.type === 'image') {
+        await sendImageData({
+          sessionId: this.sessionId,
+          path: chunk.path,
+          userData: chunk.userData
+        });
+      }
+      await sendEventPayloadEnd({
+        eventId: this.eventId,
+        sessionId: this.sessionId,
+        dataChannel
+      });
+    }).catch(error => {
+      this.emit('error', error);
+      throw error;
+    });
+    return this.chains[dataChannel];
+  }
+  stream(source) {
+    if (this.closed) {
+      throw new Error('Cannot stream to a closed event');
+    }
+    const dataChannel = source.dataChannel || this.findFirstCode(source.type);
+    if (this.streams[dataChannel]) {
+      throw new Error("".concat(dataChannel, " stream already exists"));
+    }
+    const stream = {
+      dataChannel,
+      type: source.type,
+      started: false,
+      start: async () => {
+        if (this.closed) {
+          throw new Error('Cannot stream to a closed event');
+        }
+        if (stream.started) {
+          return;
+        }
+        stream.started = true;
+        if (source.type === 'audio') {
+          if (source.amplitudeCount) {
+            await registerRecordAmplitudes({
+              count: source.amplitudeCount
+            });
+          }
+          await startRecordAndSendAudioData({
+            sessionId: this.sessionId,
+            dataChannel,
+            userData: source.userData
+          });
+        } else if (source.type === 'video') {
+          // const cameraType = source.cameraType || VideoCameraType.BACK;
+          // await startRecordAndSendVideoData({
+          //   dataChannel,
+          //   cameraType,
+          //   userData: source.userData,
+          // });
+        }
+      },
+      stop: async () => {
+        if (!stream.started || this.closed) {
+          return;
+        }
+        if (source.type === 'audio') {
+          if (source.amplitudeCount) {
+            await unregisterVoiceAmplitudes({
+              count: source.amplitudeCount
+            });
+          }
+          await stopRecordAndSendAudioData({
+            sessionId: this.sessionId,
+            dataChannel,
+            userData: source.userData
+          });
+        } else if (source.type === 'video') {
+          // const cameraType = source.cameraType || VideoCameraType.BACK;
+          // await stopRecordAndSendVideoData({
+          //   dataChannel,
+          //   cameraType,
+          //   userData: source.userData,
+          // });
+        }
+        await sendEventPayloadEnd({
+          eventId: this.eventId,
+          sessionId: this.sessionId,
+          dataChannel
+        });
+        delete this.streams[dataChannel];
+        stream.started = false;
+      }
+    };
+    this.streams[dataChannel] = stream;
+    return stream;
+  }
+  async end(options) {
+    if (this.closed) {
+      return;
+    }
+    await Promise.all([...Object.values(this.chains), ...Object.values(this.streams).map(s => s.stop())]);
+    await sendEventEnd({
+      eventId: this.eventId,
+      sessionId: this.sessionId,
+      userData: options === null || options === void 0 ? void 0 : options.userData
+    });
+  }
+  async abort(options) {
+    if (this.sessionId) {
+      await sendEventChatBreak({
+        eventId: this.eventId,
+        sessionId: this.sessionId,
+        userData: options === null || options === void 0 ? void 0 : options.userData
+      });
+    }
+    const error = new Error('This operation was aborted');
+    error.name = 'AbortError';
+    // 发送 break 后，stream 自动会关掉，在这里提前关闭
+    Object.values(this.streams).forEach(s => {
+      s.started = false;
+    });
+    this.emit('error', error);
+    this.emit('close');
+  }
+  on(name, callback) {
+    if (name === 'finish') {
+      this.listeners.finish.add(callback);
+    } else if (name === 'error') {
+      this.listeners.error.add(callback);
+    } else if (name === 'data') {
+      this.listeners.data.add(callback);
+    } else if (name === 'close') {
+      this.listeners.close.add(callback);
+    }
+  }
+  off(name, callback) {
+    if (name === 'finish') {
+      this.listeners.finish.delete(callback);
+    } else if (name === 'error') {
+      this.listeners.error.delete(callback);
+    } else if (name === 'data') {
+      this.listeners.data.delete(callback);
+    } else if (name === 'close') {
+      this.listeners.close.delete(callback);
+    }
+  }
+  emit(name, data) {
+    if (this.closed) {
+      return;
+    }
+    if (name === 'finish') {
+      this.listeners.finish.forEach(cb => cb());
+      this.emit('close');
+    } else if (name === 'error') {
+      this.listeners.error.forEach(cb => cb(data));
+    } else if (name === 'data') {
+      this.listeners.data.forEach(cb => cb(data));
+    } else if (name === 'close') {
+      this.closed = true;
+      this.listeners.close.forEach(cb => cb());
+      this.dispose();
+    }
+  }
+  dispose() {
+    this.sessionId = null;
+    this.listeners.finish.clear();
+    this.listeners.error.clear();
+    this.listeners.data.clear();
+    this.listeners.close.clear();
+    Object.values(this.streams).forEach(s => s.stop());
+    this.streams = Object.create(null);
+    this.chains = Object.create(null);
+  }
+}

package/dist/utils/abort.d.ts

@@ -0,0 +1,38 @@
+import { Emitter, EmitterEvent, AbortSignalObject } from '@ray-js/t-agent';
+type EventCallback = (event: EmitterEvent) => void;
+export declare class AbortSignal extends Emitter implements AbortSignalObject {
+    aborted: boolean;
+    onabort: EventCallback;
+    reason: any;
+    constructor();
+    toString(): string;
+    dispatchEvent(event: EmitterEvent): boolean;
+    /**
+     * @see {@link https://developer.mozilla.org/zh-CN/docs/Web/API/AbortSignal/throwIfAborted}
+     */
+    throwIfAborted(): void;
+    /**
+     * @see {@link https://developer.mozilla.org/zh-CN/docs/Web/API/AbortSignal/timeout_static}
+     * @param {number} time The "active" time in milliseconds before the returned {@link AbortSignalObject} will abort.
+     *                      The value must be within range of 0 and {@link Number.MAX_SAFE_INTEGER}.
+     * @returns {AbortSignalObject} The signal will abort with its {@link AbortSignalObject.reason} property set to a `TimeoutError` {@link DOMException} on timeout,
+     *                        or an `AbortError` {@link DOMException} if the operation was user-triggered.
+     */
+    static timeout(time: number): AbortSignal;
+    /**
+     * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/any_static}
+     * @param {Iterable<AbortSignalObject>} iterable An {@link Iterable} (such as an {@link Array}) of abort signals.
+     * @returns {AbortSignalObject} - **Already aborted**, if any of the abort signals given is already aborted.
+     *                          The returned {@link AbortSignalObject}'s reason will be already set to the `reason` of the first abort signal that was already aborted.
+     *                        - **Asynchronously aborted**, when any abort signal in `iterable` aborts.
+     *                          The `reason` will be set to the reason of the first abort signal that is aborted.
+     */
+    static any(iterable: Iterable<AbortSignal>): AbortSignal;
+}
+export declare class AbortController {
+    signal: AbortSignal;
+    constructor();
+    abort(reason: any): void;
+    toString(): string;
+}
+export default AbortController;

package/dist/utils/abort.js

@@ -0,0 +1,177 @@
+import "core-js/modules/es.symbol.description.js";
+import "core-js/modules/web.dom-collections.iterator.js";
+// source: https://github.com/mo/abortcontroller-polyfill
+import { Emitter } from '@ray-js/t-agent';
+function createAbortEvent(reason) {
+  let event;
+  try {
+    event = new Event('abort');
+  } catch (e) {
+    if (typeof document !== 'undefined') {
+      if (!document.createEvent) {
+        // For Internet Explorer 8:
+        // @ts-ignore
+        event = document.createEventObject();
+        event.type = 'abort';
+      } else {
+        // For Internet Explorer 11:
+        event = document.createEvent('Event');
+        event.initEvent('abort', false, false);
+      }
+    } else {
+      // Fallback where document isn't available:
+      event = {
+        type: 'abort',
+        bubbles: false,
+        cancelable: false
+      };
+    }
+  }
+  event.reason = reason;
+  return event;
+}
+function normalizeAbortReason(reason) {
+  if (reason === undefined) {
+    if (typeof document === 'undefined') {
+      reason = new Error('This operation was aborted');
+      reason.name = 'AbortError';
+    } else {
+      try {
+        reason = new DOMException('signal is aborted without reason');
+        // The DOMException does not support setting the name property directly.
+        Object.defineProperty(reason, 'name', {
+          value: 'AbortError'
+        });
+      } catch (err) {
+        // IE 11 does not support calling the DOMException constructor, use a
+        // regular error object on it instead.
+        reason = new Error('This operation was aborted');
+        reason.name = 'AbortError';
+      }
+    }
+  }
+  return reason;
+}
+export class AbortSignal extends Emitter {
+  constructor() {
+    super();
+    // Some versions of babel does not transpile super() correctly for IE <= 10, if the parent
+    // constructor has failed to run, then "this.listeners" will still be undefined and then we call
+    // the parent constructor directly instead as a workaround. For general details, see babel bug:
+    // https://github.com/babel/babel/issues/3041
+    // This hack was added as a fix for the issue described here:
+    // https://github.com/Financial-Times/polyfill-library/pull/59#issuecomment-477558042
+    if (!this.listeners) {
+      Emitter.call(this);
+    }
+
+    // Compared to assignment, Object.defineProperty makes properties non-enumerable by default and
+    // we want Object.keys(new AbortController().signal) to be [] for compat with the native impl
+    Object.defineProperty(this, 'aborted', {
+      value: false,
+      writable: true,
+      configurable: true
+    });
+    Object.defineProperty(this, 'onabort', {
+      value: null,
+      writable: true,
+      configurable: true
+    });
+    Object.defineProperty(this, 'reason', {
+      value: undefined,
+      writable: true,
+      configurable: true
+    });
+  }
+  toString() {
+    return '[object AbortSignal]';
+  }
+  dispatchEvent(event) {
+    if (event.type === 'abort') {
+      this.aborted = true;
+      if (typeof this.onabort === 'function') {
+        this.onabort.call(this, event);
+      }
+    }
+    return super.dispatchEvent(event);
+  }
+
+  /**
+   * @see {@link https://developer.mozilla.org/zh-CN/docs/Web/API/AbortSignal/throwIfAborted}
+   */
+  throwIfAborted() {
+    const {
+      aborted,
+      reason = 'Aborted'
+    } = this;
+    if (!aborted) return;
+    throw reason;
+  }
+
+  /**
+   * @see {@link https://developer.mozilla.org/zh-CN/docs/Web/API/AbortSignal/timeout_static}
+   * @param {number} time The "active" time in milliseconds before the returned {@link AbortSignalObject} will abort.
+   *                      The value must be within range of 0 and {@link Number.MAX_SAFE_INTEGER}.
+   * @returns {AbortSignalObject} The signal will abort with its {@link AbortSignalObject.reason} property set to a `TimeoutError` {@link DOMException} on timeout,
+   *                        or an `AbortError` {@link DOMException} if the operation was user-triggered.
+   */
+  static timeout(time) {
+    const controller = new AbortController();
+    setTimeout(() => controller.abort(new DOMException("This signal is timeout in ".concat(time, "ms"), 'TimeoutError')), time);
+    return controller.signal;
+  }
+
+  /**
+   * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/any_static}
+   * @param {Iterable<AbortSignalObject>} iterable An {@link Iterable} (such as an {@link Array}) of abort signals.
+   * @returns {AbortSignalObject} - **Already aborted**, if any of the abort signals given is already aborted.
+   *                          The returned {@link AbortSignalObject}'s reason will be already set to the `reason` of the first abort signal that was already aborted.
+   *                        - **Asynchronously aborted**, when any abort signal in `iterable` aborts.
+   *                          The `reason` will be set to the reason of the first abort signal that is aborted.
+   */
+  static any(iterable) {
+    const controller = new AbortController();
+    function clean() {
+      for (const signal of iterable) signal.removeEventListener('abort', abort);
+    }
+    /**
+     * @this AbortSignalObject
+     */
+    function abort() {
+      controller.abort(this.reason);
+      clean();
+    }
+    for (const signal of iterable) if (signal.aborted) {
+      controller.abort(signal.reason);
+      break;
+    } else signal.addEventListener('abort', abort);
+    return controller.signal;
+  }
+}
+export class AbortController {
+  constructor() {
+    // Compared to assignment, Object.defineProperty makes properties non-enumerable by default and
+    // we want Object.keys(new AbortController()) to be [] for compat with the native impl
+    Object.defineProperty(this, 'signal', {
+      value: new AbortSignal(),
+      writable: true,
+      configurable: true
+    });
+  }
+  abort(reason) {
+    const signalReason = normalizeAbortReason(reason);
+    const event = createAbortEvent(signalReason);
+    this.signal.reason = signalReason;
+    this.signal.dispatchEvent(event);
+  }
+  toString() {
+    return '[object AbortController]';
+  }
+}
+if (typeof Symbol !== 'undefined' && Symbol.toStringTag) {
+  // These are necessary to make sure that we get correct output for:
+  // Object.prototype.toString.call(new AbortController())
+  AbortController.prototype[Symbol.toStringTag] = 'AbortController';
+  AbortSignal.prototype[Symbol.toStringTag] = 'AbortSignal';
+}
+export default AbortController;