@ray-js/t-agent-plugin-aistream 0.2.3-beta-3 → 0.2.3-beta-4

package/dist/AIStreamTypes.d.ts

@@ -791,6 +791,18 @@ export type StartPlayAudioParams = {
     }) => void;
     complete?: () => void;
 };
+export type StopPlayAudioParams = {
+    success?: (params: null) => void;
+    fail?: (params: {
+        errorMsg: string;
+        errorCode: string | number;
+        innerError: {
+            errorCode: string | number;
+            errorMsg: string;
+        };
+    }) => void;
+    complete?: () => void;
+};
 export type Attribute = {
     /** Attribute 类型 */
     type: AIStreamAttributeType;
@@ -862,6 +874,67 @@ export type AudioBody = {
      */
     bitDepth?: number;
 };
+export declare enum AudioPlayChangedState {
+    /** 播放开始 */
+    START = 1,
+    /** 缓冲中 */
+    BUFFERING = 2,
+    /** 缓存完成继续播放 */
+    BUFFERED = 3,
+    /** 播放完成 */
+    COMPLETED = 4,
+    /** 播放异常 */
+    ERROR = 5
+}
+export type AudioPlayChangedBody = {
+    path: string;
+    /** 播放状态: 1-播放开始, 2-缓冲中, 3-缓存完成 继续播放, 4-播放完成, 5-播放异常 */
+    state: AudioPlayChangedState;
+    /** 播放异常码: 1-主动停止播放, 2-播放被异常中断, ... */
+    code?: number;
+};
+export interface AIStreamAudioFile {
+    /** 音频缓存路径 */
+    path: string;
+    /**
+     * 音频编码类型，表示音频数据的编码格式
+     * - 100: ADPCM
+     * - 101: PCM
+     * - 102: AACRaw
+     * - 103: AACADTS
+     * - 104: AACLATM
+     * - 105: G711U
+     * - 106: G711A
+     * - 107: G726
+     * - 108: SPEEX
+     * - 109: MP3
+     * - 110: G722
+     * - 111: Opus
+     */
+    codecType: number;
+    /**
+     * 音频采样率，单位Hz
+     * 表示每秒采样次数，常见值：8000, 16000, 44100等
+     */
+    sampleRate: number;
+    /**
+     * 音频位深，表示每个采样点的位数
+     * 常见值：8, 16, 24, 32等
+     */
+    bitDepth: number;
+    /**
+     * 音频通道数
+     * - 0: 单声道
+     * - 1: 立体声
+     */
+    channels: number;
+}
+export type RecordAndSendAudioFailBody = {
+    /**
+     * 录制发送异常: 1-发送数据异常, 2-录制音频异常
+     */
+    code: 0 | 1;
+};
 export type VideoBody = {
     /** 接收数据通道 */
     dataChannel: string;
@@ -1217,6 +1290,10 @@ export type SendEventChatBreakParams = {
  *@description 开始录制并发送音频数据
  */
 export type StartRecordAndSendAudioDataParams = {
+    saveFile?: boolean;
+    recordInitParams?: {
+        sampleRate?: number;
+    };
     /** 会话 id */
     sessionId: string;
     /** 下发数据通道，当音频只有单路的时候，可以不传 */
@@ -1241,9 +1318,7 @@ export type StopRecordAndSendAudioDataParams = {
     dataChannel?: string;
     /** 扩展属性 */
     userData?: Attribute[];
-    success?: (params: {
-        filePath: string;
-    } | null) => void;
+    success?: (params: AIStreamAudioFile | null) => void;
     fail?: (params: {
         errorMsg: string;
         errorCode: string | number;
@@ -1345,6 +1420,7 @@ export type SendTextDataParams = {
     complete?: () => void;
 };
 export type InitAudioRecorderParams = {
+    sampleRate?: number;
     success?: (params: null) => void;
     fail?: (params: {
         errorMsg: string;

package/dist/AIStreamTypes.js

@@ -187,6 +187,14 @@ export let AIStreamAttributePayloadType = /*#__PURE__*/function (AIStreamAttribu
   AIStreamAttributePayloadType[AIStreamAttributePayloadType["STRING"] = 6] = "STRING";
   return AIStreamAttributePayloadType;
 }({});
+export let AudioPlayChangedState = /*#__PURE__*/function (AudioPlayChangedState) {
+  AudioPlayChangedState[AudioPlayChangedState["START"] = 1] = "START";
+  AudioPlayChangedState[AudioPlayChangedState["BUFFERING"] = 2] = "BUFFERING";
+  AudioPlayChangedState[AudioPlayChangedState["BUFFERED"] = 3] = "BUFFERED";
+  AudioPlayChangedState[AudioPlayChangedState["COMPLETED"] = 4] = "COMPLETED";
+  AudioPlayChangedState[AudioPlayChangedState["ERROR"] = 5] = "ERROR";
+  return AudioPlayChangedState;
+}({});
 export let NetworkType = /*#__PURE__*/function (NetworkType) {
   NetworkType["NONE"] = "none";
   NetworkType["CELL_2G"] = "2g";

package/dist/asr/AsrAgent.d.ts

@@ -1,4 +1,4 @@
-import { ConnectClientType } from '../AIStreamTypes';
+import { AIStreamAudioFile, ConnectClientType } from '../AIStreamTypes';
 export interface AsrAgentOptions {
     agentId: string;
     /** 获取 agent token 的参数 */
@@ -13,9 +13,11 @@ export interface AsrAgentOptions {
     clientType?: ConnectClientType;
     deviceId?: string;
     onMessage?: (message: {
+        type: 'text';
         text: string;
     } | {
-        filePath: string;
+        type: 'file';
+        file: AIStreamAudioFile;
     }) => void;
     onFinish?: () => void;
     onError?: (error: any) => void;
@@ -23,7 +25,7 @@ export interface AsrAgentOptions {
     onAbort?: () => void;
     recordingOptions?: {
         /** 是否需要保存音频 */
-        needSave?: boolean;
+        saveFile?: boolean;
         /** 保存的音频采样率，单位：hz */
         sampleRate?: number;
         /** 最长录制时长，单位：毫秒 */

package/dist/asr/AsrAgent.js

@@ -14,9 +14,15 @@ export class AsrAgent {
     /** 录音时长定时器 */
     _defineProperty(this, "recordDurationTimer", null);
     this.options = options;
-    if (this.options.earlyStart) {
+    if (options.earlyStart) {
+      const {
+        recordingOptions
+      } = options;
+      const sampleRate = recordingOptions === null || recordingOptions === void 0 ? void 0 : recordingOptions.sampleRate;
       // 如果需要提前启动，直接初始化和创建连接
-      this.createSession().then(() => initAudioRecorder()).catch(error => {
+      this.createSession().then(() => initAudioRecorder(sampleRate ? {
+        sampleRate
+      } : {})).catch(error => {
         logger.error('EarlyStart Failed to create ASR session:', error);
       });
     }
@@ -113,9 +119,15 @@ export class AsrAgent {
     const {
       recordingOptions
     } = this.options || {};
-    const audioStream = activeEvent.stream(_objectSpread({
-      type: 'audio'
-    }, recordingOptions));
+    const {
+      sampleRate,
+      saveFile
+    } = recordingOptions || {};
+    const audioStream = activeEvent.stream({
+      type: 'audio',
+      sampleRate,
+      saveFile
+    });
     this.audioStream = audioStream;
     activeEvent.on('data', entry => {
       if (entry.type === 'text') {
@@ -126,7 +138,10 @@ export class AsrAgent {
         if (packet.bizType !== ReceivedTextPacketType.ASR) {
           return;
         }
-        typeof onMessage === 'function' && onMessage(packet.data);
+        typeof onMessage === 'function' && onMessage({
+          type: 'text',
+          text: packet.data.text
+        });
       } else if (entry.type === 'connectionState') {
         if (entry.body.connectState === ConnectState.DISCONNECTED || entry.body.connectState === ConnectState.CLOSED) {
           this.stop();
@@ -168,10 +183,12 @@ export class AsrAgent {
       this.recordDurationTimer = null;
     }
     if (this.audioStream) {
-      const result = await this.audioStream.stop();
-      if (result !== null && result !== void 0 && result.filePath) {
+      const file = await this.audioStream.stop();
+      console.log('Audio file result:', file);
+      if (file !== null && file !== void 0 && file.path) {
         this.options.onMessage({
-          filePath: result.filePath
+          type: 'file',
+          file
         });
       }
       this.audioStream = null;

package/dist/asr/index.d.ts

@@ -0,0 +1,3 @@
+import { AsrAgent } from './AsrAgent';
+import { createAsrAgent } from './createAsrAgent';
+export { AsrAgent, createAsrAgent };

package/dist/asr/index.js

@@ -0,0 +1,3 @@
+import { AsrAgent } from './AsrAgent';
+import { createAsrAgent } from './createAsrAgent';
+export { AsrAgent, createAsrAgent };

package/dist/index.d.ts

@@ -4,3 +4,4 @@ export * from './withAIStream';
 export * from './ChatHistoryStore';
 export * from './buildIn';
 export * from './ChatHistoryLocalStore';
+export * from './asr';

package/dist/index.js

@@ -3,4 +3,5 @@ export * from './AIStreamTypes';
 export * from './withAIStream';
 export * from './ChatHistoryStore';
 export * from './buildIn';
-export * from './ChatHistoryLocalStore';
+export * from './ChatHistoryLocalStore';
+export * from './asr';

package/dist/utils/AIStream.d.ts

@@ -1,4 +1,4 @@
-import { Attribute, BizTag, ConnectClientType, ConnectState, FileFormat, VideoCameraType } from '../AIStreamTypes';
+import { AIStreamAudioFile, Attribute, BizTag, ConnectClientType, ConnectState, FileFormat, VideoCameraType } from '../AIStreamTypes';
 import { AIStreamDataEntry, AIStreamObserverPool } from './observer';
 import { AIStreamError } from './errors';
 interface AIStreamConnectionOptions {
@@ -87,7 +87,7 @@ type AIStreamEventSource = {
     type: 'audio';
     dataChannel?: string;
     userData?: Attribute[];
-    needSave?: boolean;
+    saveFile?: boolean;
     sampleRate?: number;
 } | {
     type: 'video';
@@ -100,9 +100,7 @@ export interface AIStreamEventStream {
     dataChannel: string;
     started: boolean;
     start: () => Promise<void>;
-    stop: () => Promise<{
-        filePath?: string;
-    } | null>;
+    stop: () => Promise<AIStreamAudioFile | null>;
 }
 export declare class AIStreamEvent {
     readonly eventId: string;

package/dist/utils/AIStream.js

@@ -459,7 +459,11 @@ export class AIStreamEvent {
           startPromise = startRecordAndSendAudioData({
             sessionId: this.sessionId,
             dataChannel,
-            userData: source.userData
+            userData: source.userData,
+            recordInitParams: source.sampleRate ? {
+              sampleRate: source.sampleRate
+            } : undefined,
+            saveFile: source.saveFile
           });
           await startPromise;
         } else if (source.type === 'video') {
@@ -480,9 +484,9 @@ export class AIStreamEvent {
           await tryCatchTTT(() => startPromise);
           startPromise = null;
         }
-        let result = null;
+        let file = null;
         if (source.type === 'audio') {
-          result = await stopRecordAndSendAudioData({
+          file = await stopRecordAndSendAudioData({
             sessionId: this.sessionId,
             dataChannel,
             userData: source.userData
@@ -503,7 +507,7 @@ export class AIStreamEvent {
         });
         delete this.streams[dataChannel];
         stream.started = false;
-        return result;
+        return file;
       }
     };
     this.streams[dataChannel] = stream;

package/dist/utils/defaultMock.js

@@ -470,7 +470,7 @@ mock.hooks.hook('stopRecordAndSendAudioData', async context => {
     throw new TTTError('stopRecordAndSendAudioData event not exists', AIStreamAppErrorCode.SESSION_ID_INVALID);
   }
   session.currentEvent.asr.finishController.abort(new Error('stopRecordAndSendAudioData'));
-  context.result = true;
+  context.result = null;
   await mock.sleep(100);
 });
 mock.hooks.hook('startRecordAndSendVideoData', context => {

package/dist/utils/index.d.ts

@@ -9,6 +9,5 @@ export * from './AIStream';
 export * from './observer';
 export * from './sendMessage';
 export * from './version';
-export * from '../asr/createAsrAgent';
 export * from './actions';
 export { mock };

package/dist/utils/index.js

@@ -9,6 +9,5 @@ export * from './AIStream';
 export * from './observer';
 export * from './sendMessage';
 export * from './version';
-export * from '../asr/createAsrAgent';
 export * from './actions';
 export { mock };

package/dist/utils/ttt.d.ts

@@ -1,4 +1,4 @@
-import { ApiRequestByAtopParams, ApiRequestByHighwayParams, AudioBody, AuthorizeParams, AuthorizePolicyStatusParams, CanIUseRouterParams, CloseSessionParams, ConnectParams, ConnectStateBody, CreateSessionParams, DeleteRecordListParams, DisconnectParams, EventBody, EventChannelMessageParams, GetAppInfoParams, GetCurrentHomeInfoParams, GetMiniAppConfigParams, GetAccountInfoParams, ImageBody, InsertRecordParams, NavigateToMiniProgramParams, OpenInnerH5Params, OpenMiniWidgetParams, QueryAgentTokenParams, QueryRecordListParams, RecordAmplitudesBody, RegisterChannelParams, RouterParams, SendEventChatBreakParams, SendEventEndParams, SendEventPayloadEndParams, SendEventStartParams, SendImageDataParams, SendTextDataParams, SessionStateBody, StartRecordAndSendAudioDataParams, StopRecordAndSendAudioDataParams, TextBody, UpdateRecordParams, IsConnectedParams, GetNetworkTypeParams, StartPlayAudioParams, InitAudioRecorderParams } from '../AIStreamTypes';
+import { ApiRequestByAtopParams, ApiRequestByHighwayParams, AudioBody, AuthorizeParams, AuthorizePolicyStatusParams, CanIUseRouterParams, CloseSessionParams, ConnectParams, ConnectStateBody, CreateSessionParams, DeleteRecordListParams, DisconnectParams, EventBody, EventChannelMessageParams, GetAppInfoParams, GetCurrentHomeInfoParams, GetMiniAppConfigParams, GetAccountInfoParams, ImageBody, InsertRecordParams, NavigateToMiniProgramParams, OpenInnerH5Params, OpenMiniWidgetParams, QueryAgentTokenParams, QueryRecordListParams, RecordAmplitudesBody, RegisterChannelParams, RouterParams, SendEventChatBreakParams, SendEventEndParams, SendEventPayloadEndParams, SendEventStartParams, SendImageDataParams, SendTextDataParams, SessionStateBody, StartRecordAndSendAudioDataParams, StopRecordAndSendAudioDataParams, TextBody, UpdateRecordParams, IsConnectedParams, GetNetworkTypeParams, StartPlayAudioParams, InitAudioRecorderParams, StopPlayAudioParams, AudioPlayChangedBody } from '../AIStreamTypes';
 export declare const getMiniAppConfig: (options?: Omit<GetMiniAppConfigParams, "success" | "fail"> | undefined) => Promise<{
     config: any;
 }>;
@@ -79,9 +79,7 @@ export declare const sendEventPayloadEnd: (options?: Omit<SendEventPayloadEndPar
 export declare const sendEventEnd: (options?: Omit<SendEventEndParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const sendEventChatBreak: (options?: Omit<SendEventChatBreakParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const startRecordAndSendAudioData: (options?: Omit<StartRecordAndSendAudioDataParams, "success" | "fail"> | undefined) => Promise<null>;
-export declare const stopRecordAndSendAudioData: (options?: Omit<StopRecordAndSendAudioDataParams, "success" | "fail"> | undefined) => Promise<{
-    filePath: string;
-} | null>;
+export declare const stopRecordAndSendAudioData: (options?: Omit<StopRecordAndSendAudioDataParams, "success" | "fail"> | undefined) => Promise<import("../AIStreamTypes").AIStreamAudioFile | null>;
 export declare const sendImageData: (options?: Omit<SendImageDataParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const sendTextData: (options?: Omit<SendTextDataParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const registerRecordAmplitudes: (options?: Omit<{
@@ -90,10 +88,9 @@ export declare const registerRecordAmplitudes: (options?: Omit<{
 export declare const unregisterVoiceAmplitudes: (options?: Omit<any, "success" | "fail"> | undefined) => Promise<unknown>;
 export declare const initAudioRecorder: (options?: Omit<InitAudioRecorderParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const startPlayAudio: (options?: Omit<StartPlayAudioParams, "success" | "fail"> | undefined) => Promise<null>;
+export declare const stopPlayAudio: (options?: Omit<StopPlayAudioParams, "success" | "fail"> | undefined) => Promise<null>;
 export declare const listenEventReceived: (listener: (params: EventBody) => void) => () => void;
-export declare const listenAudioPlayEnd: (listener: (params: {
-    path: string;
-}) => void) => () => void;
+export declare const listenAudioPlayChanged: (listener: (params: AudioPlayChangedBody) => void) => () => void;
 export declare const listenAudioReceived: (listener: (params: AudioBody) => void) => () => void;
 export declare const listenTextReceived: (listener: (params: TextBody) => void) => () => void;
 export declare const listenImageReceived: (listener: (params: ImageBody) => void) => () => void;

package/dist/utils/ttt.js

@@ -58,11 +58,12 @@ export const registerRecordAmplitudes = promisify(ty.aistream.registerRecordAmpl
 export const unregisterVoiceAmplitudes = promisify(ty.aistream.unregisterVoiceAmplitudes, true);
 export const initAudioRecorder = promisify(ty.aistream.initAudioRecorder, true);
 export const startPlayAudio = promisify(ty.aistream.startPlayAudio, true);
+export const stopPlayAudio = promisify(ty.aistream.stopPlayAudio, true);
 
 // export const sendFileData = promisify<SendFileDataParams>(ty.aistream.sendFileData);
 
 export const listenEventReceived = listening(ty.aistream.onEventReceived, ty.aistream.offEventReceived, true);
-export const listenAudioPlayEnd = listening(ty.aistream.onAudioPlayEnd, ty.aistream.onAudioPlayEnd, true);
+export const listenAudioPlayChanged = listening(ty.aistream.onAudioPlayChanged, ty.aistream.onAudioPlayChanged, true);
 export const listenAudioReceived = listening(ty.aistream.onAudioReceived, ty.aistream.offAudioReceived, true);
 
 // export const listenVideoReceived = listening<VideoBody>(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.3-beta-3",
+  "version": "0.2.3-beta-4",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -35,5 +35,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "84e4e7b2ec8f91fe56b3b8499d549175893f9fe3"
+  "gitHead": "952d287025737224227ead3ecf6dd4c432fec6a2"
 }