@ray-js/t-agent-plugin-aistream 0.2.0-beta-1

package/dist/utils/mock.js

@@ -0,0 +1,72 @@
+import "core-js/modules/es.array.flat.js";
+import "core-js/modules/es.array.unscopables.flat.js";
+import "core-js/modules/es.json.stringify.js";
+import "core-js/modules/web.dom-collections.iterator.js";
+import { createHooks, Emitter } from '@ray-js/t-agent';
+const mock = {
+  loaded: false,
+  lastId: 1,
+  database: new Map(),
+  data: new Map(),
+  sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),
+  generateInt: (min, max) => Math.floor(Math.random() * (max - min + 1)) + min,
+  emitter: new Emitter(),
+  hooks: createHooks(),
+  getRecords: () => {
+    var _rows;
+    if (mock.loaded) {
+      return Array.from(mock.database.values());
+    }
+    const ret = ty.getStorageSync({
+      key: 'AIStreamMockDatabase'
+    });
+    mock.loaded = true;
+    let rows = [];
+    if (typeof ret === 'string') {
+      rows = JSON.parse(ret);
+    } else if (typeof (ret === null || ret === void 0 ? void 0 : ret.data) === 'string') {
+      rows = JSON.parse(ret === null || ret === void 0 ? void 0 : ret.data);
+    }
+    if (Array.isArray((_rows = rows) === null || _rows === void 0 ? void 0 : _rows[0])) {
+      rows = rows.flat();
+    }
+    if (!rows) {
+      return [];
+    }
+    for (const row of rows) {
+      mock.database.set(row.id, row);
+      mock.lastId = Math.max(mock.lastId, row.id);
+    }
+    mock.lastId += 1;
+    return rows;
+  },
+  getRecord: id => {
+    mock.getRecords();
+    return mock.database.get(id);
+  },
+  setRecord: entry => {
+    mock.getRecords();
+    mock.database.set(entry.id, entry);
+    ty.setStorageSync({
+      key: 'AIStreamMockDatabase',
+      data: JSON.stringify(mock.getRecords())
+    });
+  },
+  deleteRecord: id => {
+    mock.getRecords();
+    mock.database.delete(id);
+    ty.setStorageSync({
+      key: 'AIStreamMockDatabase',
+      data: JSON.stringify(mock.getRecords())
+    });
+  },
+  clearRecords: () => {
+    mock.database.clear();
+  },
+  getId: () => {
+    const id = mock.lastId;
+    mock.lastId += 1;
+    return id;
+  }
+};
+export { mock };

package/dist/utils/observer.d.ts

@@ -0,0 +1,61 @@
+import { AudioBody, ConnectStateBody, EventBody, FileBody, ImageBody, RecordAmplitudesBody, SessionStateBody, TextBody, VideoBody } from '../AIStreamTypes';
+export type AIStreamDataEntry = {
+    type: 'text';
+    body: TextBody;
+} | {
+    type: 'image';
+    body: ImageBody;
+} | {
+    type: 'video';
+    body: VideoBody;
+} | {
+    type: 'file';
+    body: FileBody;
+} | {
+    type: 'audio';
+    body: AudioBody;
+} | {
+    type: 'event';
+    body: EventBody;
+} | {
+    type: 'connectionState';
+    body: ConnectStateBody;
+} | {
+    type: 'sessionState';
+    body: SessionStateBody;
+} | {
+    type: 'amplitudes';
+    body: RecordAmplitudesBody;
+};
+export interface AIStreamObserverOptions {
+    sessionState?: boolean;
+    connectionState?: boolean;
+    event?: boolean;
+    text?: boolean;
+    audio?: boolean;
+    video?: boolean;
+    file?: boolean;
+    image?: boolean;
+    dataChannels?: string[];
+    sessionId?: string;
+    amplitudes?: boolean;
+}
+export declare class AIStreamObserverPool {
+    isStarted: boolean;
+    private observerMap;
+    constructor();
+    private cancels;
+    start(): void;
+    dispose(): void;
+    connect(observer: AIStreamObserver): void;
+    disconnect(observer: AIStreamObserver): void;
+}
+export declare class AIStreamObserver {
+    readonly callback: (entry: AIStreamDataEntry, observer: AIStreamObserver) => void;
+    private readonly pool;
+    get options(): AIStreamObserverOptions;
+    private _options;
+    constructor(callback: (entry: AIStreamDataEntry, observer: AIStreamObserver) => void, pool: AIStreamObserverPool);
+    observe(options: AIStreamObserverOptions): void;
+    disconnect(): void;
+}

package/dist/utils/observer.js

@@ -0,0 +1,152 @@
+import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
+import "core-js/modules/esnext.iterator.constructor.js";
+import "core-js/modules/esnext.iterator.for-each.js";
+import "core-js/modules/web.dom-collections.iterator.js";
+import { ConnectState, EventType, SessionState, StreamFlag } from '../AIStreamTypes';
+import { listenAudioReceived, listenConnectStateChanged, listenEventReceived, listenImageReceived, listenRecordAmplitudes, listenSessionStateChanged, listenTextReceived } from './ttt';
+import logger from './logger';
+const types = ['connectionState', 'sessionState', 'event', 'text', 'audio', 'video', 'file', 'image', 'amplitudes'];
+export class AIStreamObserverPool {
+  constructor() {
+    _defineProperty(this, "isStarted", false);
+    _defineProperty(this, "cancels", []);
+    this.observerMap = {
+      connectionState: new Set(),
+      sessionState: new Set(),
+      event: new Set(),
+      text: new Set(),
+      audio: new Set(),
+      video: new Set(),
+      file: new Set(),
+      image: new Set(),
+      amplitudes: new Set()
+    };
+  }
+  start() {
+    if (this.isStarted) {
+      return;
+    }
+    const handle = entry => {
+      const observers = this.observerMap[entry.type];
+      let state = '';
+      if (entry.type !== 'amplitudes') {
+        var _entry$body;
+        if (entry.type === 'connectionState') {
+          state = "".concat(ConnectState[entry.body.connectState], " ").concat(entry.body.code || '');
+        } else if (entry.type === 'sessionState') {
+          state = "".concat(SessionState[entry.body.sessionState], " ").concat(entry.body.code || '');
+        } else if (entry.type === 'event') {
+          state = "".concat(EventType[entry.body.eventType]);
+        } else if ((_entry$body = entry.body) !== null && _entry$body !== void 0 && _entry$body.streamFlag) {
+          state = "".concat(StreamFlag[entry.body.streamFlag]);
+        }
+        logger.debug("Pool received: %c".concat(observers.size ? '⊕' : '×').concat(entry.type, " ").concat(state), 'background: black; color: white', entry);
+      }
+      observers.forEach(observer => {
+        if (entry.type === 'text' || entry.type === 'audio' || entry.type === 'video' || entry.type === 'file' || entry.type === 'image') {
+          const {
+            dataChannels,
+            sessionId
+          } = observer.options;
+          let matched = true;
+          if (dataChannels !== null && dataChannels !== void 0 && dataChannels.length && !dataChannels.includes(entry.body.dataChannel)) {
+            matched = false;
+          }
+          if (sessionId && !entry.body.sessionIdList.includes(sessionId)) {
+            matched = false;
+          }
+          if (matched) {
+            observer.callback(entry, observer);
+          }
+        } else {
+          observer.callback(entry, observer);
+        }
+      });
+    };
+    this.cancels = [listenConnectStateChanged(body => handle({
+      type: 'connectionState',
+      body
+    })), listenSessionStateChanged(body => handle({
+      type: 'sessionState',
+      body
+    })), listenEventReceived(body => handle({
+      type: 'event',
+      body
+    })), listenTextReceived(body => handle({
+      type: 'text',
+      body
+    })), listenAudioReceived(body => handle({
+      type: 'audio',
+      body
+    })), listenImageReceived(body => handle({
+      type: 'image',
+      body
+    })), listenRecordAmplitudes(body => handle({
+      type: 'amplitudes',
+      body
+    }))
+    // listenVideoReceived(body => handle({ type: 'video', body })),
+    // listenFileReceived(body => handle({ type: 'file', body })),
+    ];
+    this.isStarted = true;
+  }
+  dispose() {
+    this.isStarted = false;
+    this.cancels.forEach(cancel => cancel());
+    this.cancels = [];
+    this.observerMap = {
+      connectionState: new Set(),
+      sessionState: new Set(),
+      event: new Set(),
+      text: new Set(),
+      audio: new Set(),
+      video: new Set(),
+      file: new Set(),
+      image: new Set(),
+      amplitudes: new Set()
+    };
+  }
+  connect(observer) {
+    if (!this.isStarted) {
+      this.start();
+    }
+    types.forEach(key => {
+      const type = key;
+      if (observer.options[type]) {
+        this.observerMap[type].add(observer);
+      }
+    });
+  }
+  disconnect(observer) {
+    types.forEach(key => {
+      this.observerMap[key].delete(observer);
+    });
+    let empty = true;
+    types.forEach(key => {
+      if (this.observerMap[key].size > 0) {
+        empty = false;
+      }
+    });
+    if (empty) {
+      this.dispose();
+    }
+  }
+}
+export class AIStreamObserver {
+  get options() {
+    return this._options;
+  }
+  constructor(callback, pool) {
+    _defineProperty(this, "_options", {});
+    this.callback = callback;
+    this.pool = pool;
+  }
+  observe(options) {
+    this.pool.disconnect(this);
+    this._options = options;
+    this.pool.connect(this);
+  }
+  disconnect() {
+    this.pool.disconnect(this);
+  }
+}

package/dist/utils/parsers.d.ts

@@ -0,0 +1,10 @@
+/**
+ * 数字转化为整数
+ * @param v 带转化的数字
+ * @param defaultValue 兜底值，默认为0
+ * @returns number 转化后的整数
+ * @example
+ * parseInt(1.1) // 1
+ * parseInt(1.9) // 1
+ */
+export declare function safeParseInt(v: number | string, defaultValue?: number): number;

package/dist/utils/parsers.js

@@ -0,0 +1,13 @@
+/**
+ * 数字转化为整数
+ * @param v 带转化的数字
+ * @param defaultValue 兜底值，默认为0
+ * @returns number 转化后的整数
+ * @example
+ * parseInt(1.1) // 1
+ * parseInt(1.9) // 1
+ */
+export function safeParseInt(v) {
+  let defaultValue = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 0;
+  return v ? Number.parseInt(String(v), 10) || defaultValue : defaultValue;
+}

package/dist/utils/promisify.d.ts

@@ -0,0 +1,18 @@
+interface AsyncTTTFnParams<P> {
+    complete?: () => void;
+    success?: (params: P) => void;
+    fail?: (params: {
+        errorMsg: string;
+        errorCode: string | number;
+        innerError: {
+            errorCode: string | number;
+            errorMsg: string;
+        };
+    }) => void;
+    [key: string]: any;
+}
+export declare const getEnableMock: () => boolean;
+export declare const setEnableMock: (enable: boolean) => boolean;
+export declare function promisify<T extends AsyncTTTFnParams<any>>(fn: (options: any) => void, enableMock?: boolean): (options?: Omit<T, 'success' | 'fail'>) => Promise<Parameters<NonNullable<T["success"]>>[0]>;
+export declare function listening<T>(on: (listener: (params: T) => void) => void, off: (listener: (params: T) => void) => void, enableMock?: boolean): (listener: (params: T) => void) => () => void;
+export {};

package/dist/utils/promisify.js

@@ -0,0 +1,82 @@
+import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
+import { isDevTools } from './ttt';
+import logger from './logger';
+import { mock } from './mock';
+let callId = 100000;
+export const getEnableMock = () => {
+  try {
+    const result = ty.getStorageSync({
+      key: 'AIAssistantEnableMock'
+    });
+    // @ts-ignore
+    if (result && (result === 'true' || result.data === 'true')) {
+      return true;
+    }
+  } catch (e) {
+    console.error('获取AIAssistantEnableMock配置失败', e);
+  }
+  return isDevTools();
+};
+export const setEnableMock = enable => {
+  try {
+    ty.setStorageSync({
+      key: 'AIAssistantEnableMock',
+      data: String(enable)
+    });
+    return true;
+  } catch (e) {
+    console.error('设置AIAssistantEnableMock配置失败', e);
+    return false;
+  }
+};
+export function promisify(fn) {
+  let enableMock = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;
+  return options => {
+    return new Promise((_resolve, _reject) => {
+      if (!fn) {
+        _reject(new Error('fn is not a function'));
+      }
+      const id = callId++;
+      logger.debug("TTT call  #".concat(id, " %c").concat(fn.name), 'background: blue; color: white', options);
+      const resolve = result => {
+        logger.debug("TTT resp  #".concat(id, " %c").concat(fn.name), 'background: green; color: white', result);
+        _resolve(result);
+      };
+      const reject = error => {
+        logger.debug("TTT error #".concat(id, " %c").concat(fn.name), 'background: red; color: white', error);
+        _reject(error);
+      };
+      const context = {
+        options,
+        result: undefined
+      };
+      if (getEnableMock() && enableMock) {
+        mock.hooks.callHook(fn.name, context).then(() => resolve(context.result), reject);
+      } else {
+        fn.call(ty, _objectSpread(_objectSpread({}, options), {}, {
+          success: resolve,
+          fail: reject
+        }));
+      }
+    });
+  };
+}
+export function listening(on, off) {
+  let enableMock = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : false;
+  return listener => {
+    if (getEnableMock() && enableMock) {
+      const cb = event => {
+        logger.debug('listening on event', on.name, event.detail);
+        listener(event.detail);
+      };
+      mock.emitter.addEventListener(on.name, cb);
+      return () => {
+        mock.emitter.removeEventListener(on.name, cb);
+      };
+    }
+    on(listener);
+    return () => {
+      off(listener);
+    };
+  };
+}

package/dist/utils/sendMessage.d.ts

@@ -0,0 +1,21 @@
+import '../polyfill';
+import { AIStreamSession } from './AIStream';
+import { AIStreamChatAttribute } from '../AIStreamTypes';
+import { InputBlock, StreamResponse } from '@ray-js/t-agent';
+export interface SendBlocksToAIStreamParams {
+    blocks: InputBlock[];
+    session: AIStreamSession;
+    attribute?: AIStreamChatAttribute;
+}
+export declare class AIStreamSessionError extends Error {
+    readonly code: number;
+    constructor(message: string, code: number);
+}
+export declare class AIStreamConnectionError extends Error {
+    readonly code: number;
+    constructor(message: string, code: number);
+}
+export declare function sendBlocksToAIStream(params: SendBlocksToAIStreamParams): {
+    response: StreamResponse;
+    metaPromise: Promise<Record<string, any>>;
+};

package/dist/utils/sendMessage.js

@@ -0,0 +1,241 @@
+import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
+import "core-js/modules/es.json.stringify.js";
+import "core-js/modules/web.dom-collections.iterator.js";
+import '../polyfill';
+import { ReadableStream } from 'web-streams-polyfill';
+import { AIStreamAttributePayloadType, AIStreamAttributeType, AIStreamChatSysWorkflow, ConnectState, FileFormat, ReceivedTextPacketEof, ReceivedTextPacketType, SessionState, StreamFlag } from '../AIStreamTypes';
+import { EmitterEvent, generateId, safeParseJSON, StreamResponse } from '@ray-js/t-agent';
+const mimeTypeToFormatMap = {
+  'video/mp4': FileFormat.MP4,
+  'text/json': FileFormat.JSON,
+  'application/json': FileFormat.JSON,
+  'application/pdf': FileFormat.PDF
+};
+export class AIStreamSessionError extends Error {
+  constructor(message, code) {
+    super(message);
+    this.name = 'AIStreamSessionError';
+    this.code = code;
+  }
+}
+export class AIStreamConnectionError extends Error {
+  constructor(message, code) {
+    super(message);
+    this.name = 'AIStreamConnectionError';
+    this.code = code;
+  }
+}
+export function sendBlocksToAIStream(params) {
+  const {
+    session,
+    blocks
+  } = params;
+  let audioEmitter = null;
+  let amplitudeCount = 0;
+  for (const block of blocks) {
+    if (block.type === 'audio') {
+      if (audioEmitter) {
+        throw new Error('only one audio emitter is allowed');
+      }
+      audioEmitter = block.audio_emitter;
+      amplitudeCount = block.amplitude_count || 0;
+    }
+  }
+  const attribute = _objectSpread({
+    'processing.interrupt': 'false',
+    'asr.enableVad': 'false',
+    'sys.workflow': audioEmitter ? AIStreamChatSysWorkflow.ASR_LLM : AIStreamChatSysWorkflow.LLM
+  }, params.attribute);
+  let canceled = false;
+  let event = null;
+  let metaResolve;
+  const metaPromise = new Promise(resolve => {
+    metaResolve = resolve;
+  });
+  const stream = new ReadableStream({
+    async start(controller) {
+      const enqueue = part => {
+        if (canceled) {
+          return;
+        }
+        controller.enqueue(part);
+      };
+      event = await session.startEvent({
+        userData: [{
+          type: AIStreamAttributeType.AI_CHAT,
+          payloadType: AIStreamAttributePayloadType.STRING,
+          value: JSON.stringify(attribute)
+        }]
+      });
+      const meta = {
+        sessionId: event.sessionId,
+        eventId: event.eventId
+      };
+      metaResolve(meta);
+      let prevText = '';
+      let imageId = null;
+      let audioId = null;
+      event.on('data', data => {
+        if (data.type === 'text') {
+          if (!data.body.text) {
+            return;
+          }
+          const packet = safeParseJSON(data.body.text);
+          if (packet.bizType === ReceivedTextPacketType.NLG) {
+            const text = prevText + packet.data.content;
+            enqueue({
+              type: 'text',
+              delta: packet.data.content,
+              text,
+              meta
+            });
+            prevText = text;
+          } else if (packet.bizType === ReceivedTextPacketType.SKILL) {
+            enqueue({
+              id: generateId(),
+              type: 'attachment',
+              attachmentType: 'skill',
+              attachment: packet.data,
+              meta
+            });
+          } else if (packet.bizType === ReceivedTextPacketType.ASR && audioEmitter) {
+            if (packet.eof === ReceivedTextPacketEof.END) {
+              audioEmitter.dispatchEvent(new EmitterEvent('finish', {
+                detail: {
+                  text: packet.data.text
+                }
+              }));
+            } else {
+              audioEmitter.dispatchEvent(new EmitterEvent('update', {
+                detail: {
+                  text: packet.data.text
+                }
+              }));
+            }
+          }
+        } else if (data.type === 'image') {
+          if (!imageId) {
+            imageId = generateId();
+          }
+          enqueue({
+            id: imageId,
+            type: 'attachment',
+            attachmentType: 'image',
+            attachment: {
+              path: data.body.path || null,
+              width: data.body.width || null,
+              height: data.body.height || null
+            },
+            meta
+          });
+          if (data.body.streamFlag === StreamFlag.END) {
+            imageId = null;
+          }
+        } else if (data.type === 'audio') {
+          if (!audioId) {
+            audioId = generateId();
+          }
+          enqueue({
+            id: audioId,
+            type: 'attachment',
+            attachmentType: 'audio',
+            attachment: {
+              path: data.body.path || null
+            },
+            meta
+          });
+          if (data.body.streamFlag === StreamFlag.END) {
+            audioId = null;
+          }
+        } else if (data.type === 'sessionState') {
+          if (data.body.sessionState === SessionState.CLOSED) {
+            enqueue({
+              type: 'error',
+              error: new AIStreamSessionError('Session closed', data.body.code),
+              level: 'error',
+              meta
+            });
+          }
+        } else if (data.type === 'connectionState') {
+          if (data.body.connectState === ConnectState.DISCONNECTED) {
+            enqueue({
+              type: 'error',
+              error: new AIStreamConnectionError('Connection disconnected', data.body.code),
+              level: 'error',
+              meta
+            });
+          }
+        }
+      });
+      event.on('finish', () => {
+        if (!canceled) {
+          // 当取消后，不需要关闭控制器
+          controller.close();
+        }
+      });
+      event.on('error', error => {
+        enqueue({
+          type: 'error',
+          level: 'error',
+          error: error instanceof Error ? error : new Error('Unknown error'),
+          meta: {}
+        });
+      });
+      for (const block of blocks) {
+        if (block.type === 'text') {
+          event.write({
+            type: 'text',
+            text: JSON.stringify({
+              type: 'text',
+              message: block.text
+            })
+          });
+        } else if (block.type === 'image_path') {
+          event.write({
+            type: 'image',
+            path: block.image_path.path
+          });
+        } else if (block.type === 'file_path') {
+          event.write({
+            type: 'file',
+            format: mimeTypeToFormatMap[block.file_url.mimeType] || 0,
+            path: block.file_path.path
+          });
+        } else if (block.type === 'video_path') {
+          event.write({
+            type: 'file',
+            format: FileFormat.MP4,
+            path: block.file_path.path
+          });
+        }
+      }
+      if (audioEmitter) {
+        await new Promise(resolve => {
+          const s = event.stream({
+            type: 'audio',
+            amplitudeCount
+          });
+          audioEmitter.addEventListener('confirm', () => {
+            s.stop().then(resolve);
+          });
+          audioEmitter.addEventListener('cancel', async () => {
+            await s.stop();
+            await event.abort();
+            resolve();
+          });
+          s.start();
+        });
+      }
+      await event.end();
+    },
+    async cancel() {
+      var _event;
+      canceled = true;
+      await ((_event = event) === null || _event === void 0 ? void 0 : _event.abort());
+    }
+  });
+  return {
+    response: new StreamResponse(stream),
+    metaPromise
+  };
+}