@ray-js/t-agent-plugin-aistream 0.2.0-beta.0 → 0.2.0-beta.19

package/dist/AIStreamTypes.d.ts

@@ -628,7 +628,9 @@ export declare enum EventType {
     /** 聊天中断 */
     CHAT_BREAK = 4,
     /** 服务端 VAD */
-    SERVER_VAD = 5
+    SERVER_VAD = 5,
+    /** 服务端错误 */
+    SERVER_ERROR = 6
 }
 export declare enum StreamFlag {
     /** 仅一包 */
@@ -718,7 +720,9 @@ export type BizConfig = {
     revData: string[];
 };
 export declare enum AIStreamAttributeType {
-    AI_CHAT = 1003
+    AI_CHAT = 1003,
+    ERROR_MESSAGE = 1006,
+    ERROR_CODE = 1007
 }
 export declare enum AIStreamAttributePayloadType {
     U_INT8 = 1,
@@ -728,17 +732,9 @@ export declare enum AIStreamAttributePayloadType {
     BYTES = 5,
     STRING = 6
 }
-export declare enum AIStreamChatSysWorkflow {
-    ASR_LLM_TTS = "asr-llm-tts",
-    ASR_LLM = "asr-llm",
-    ASR = "asr",
-    LLM = "llm",
-    LLM_TTS = "llm-tts"
-}
 export interface AIStreamChatAttribute {
     'asr.enableVad'?: 'true' | 'false';
     'processing.interrupt'?: 'true' | 'false';
-    'sys.workflow'?: AIStreamChatSysWorkflow;
     [key: string]: string;
 }
 export type StartPlayAudioParams = {
@@ -1605,17 +1601,33 @@ export type ReceivedSearchKnowledgeSkill = ReceivedTextSkillPacketBody<BuildInSk
     };
 }>;
 export interface ReceivedSmartHomeSkillDevice {
-    devId: string;
-    devIcon: string;
-    dps: Record<string, string>;
-    devName: string;
+    deviceId: string;
+    icon: string;
+    dps?: Record<string, string>;
+    name: string;
+    room: string;
+    success: boolean;
+}
+export interface ReceivedSmartHomeSkillScene {
+    sceneId: string;
+    name: string;
+    icon: string;
+    type: number;
+    valid: boolean;
+    displayColor: string;
+    enable?: boolean;
+    success?: boolean;
 }
 export declare enum ReceivedSmartHomeSkillAction {
-    CONTROL_DEVICE = "control_device"
+    QUERY_DEVICE = "query_device",
+    CONTROL_DEVICE = "control_device",
+    QUERY_SCENE = "query_scene",
+    CONTROL_SCENE = "control_scene"
 }
 export type ReceivedSmartHomeSkill = ReceivedTextSkillPacketBody<BuildInSkillCode.SEARCH_KNOWLEDGE, {
     data: {
         devices: ReceivedSmartHomeSkillDevice[];
+        scenes: ReceivedSmartHomeSkillScene[];
     };
     action: ReceivedSmartHomeSkillAction;
 }, any>;

package/dist/AIStreamTypes.js

@@ -99,6 +99,7 @@ export let EventType = /*#__PURE__*/function (EventType) {
   EventType[EventType["ONE_SHOT"] = 3] = "ONE_SHOT";
   EventType[EventType["CHAT_BREAK"] = 4] = "CHAT_BREAK";
   EventType[EventType["SERVER_VAD"] = 5] = "SERVER_VAD";
+  EventType[EventType["SERVER_ERROR"] = 6] = "SERVER_ERROR";
   return EventType;
 }({});
 export let StreamFlag = /*#__PURE__*/function (StreamFlag) {
@@ -173,6 +174,8 @@ export let BizTag = /*#__PURE__*/function (BizTag) {
 }({});
 export let AIStreamAttributeType = /*#__PURE__*/function (AIStreamAttributeType) {
   AIStreamAttributeType[AIStreamAttributeType["AI_CHAT"] = 1003] = "AI_CHAT";
+  AIStreamAttributeType[AIStreamAttributeType["ERROR_MESSAGE"] = 1006] = "ERROR_MESSAGE";
+  AIStreamAttributeType[AIStreamAttributeType["ERROR_CODE"] = 1007] = "ERROR_CODE";
   return AIStreamAttributeType;
 }({});
 export let AIStreamAttributePayloadType = /*#__PURE__*/function (AIStreamAttributePayloadType) {
@@ -184,14 +187,6 @@ export let AIStreamAttributePayloadType = /*#__PURE__*/function (AIStreamAttribu
   AIStreamAttributePayloadType[AIStreamAttributePayloadType["STRING"] = 6] = "STRING";
   return AIStreamAttributePayloadType;
 }({});
-export let AIStreamChatSysWorkflow = /*#__PURE__*/function (AIStreamChatSysWorkflow) {
-  AIStreamChatSysWorkflow["ASR_LLM_TTS"] = "asr-llm-tts";
-  AIStreamChatSysWorkflow["ASR_LLM"] = "asr-llm";
-  AIStreamChatSysWorkflow["ASR"] = "asr";
-  AIStreamChatSysWorkflow["LLM"] = "llm";
-  AIStreamChatSysWorkflow["LLM_TTS"] = "llm-tts";
-  return AIStreamChatSysWorkflow;
-}({});
 export let NetworkType = /*#__PURE__*/function (NetworkType) {
   NetworkType["NONE"] = "none";
   NetworkType["CELL_2G"] = "2g";
@@ -260,6 +255,9 @@ export let BuildInSkillCode = /*#__PURE__*/function (BuildInSkillCode) {
   return BuildInSkillCode;
 }({});
 export let ReceivedSmartHomeSkillAction = /*#__PURE__*/function (ReceivedSmartHomeSkillAction) {
+  ReceivedSmartHomeSkillAction["QUERY_DEVICE"] = "query_device";
   ReceivedSmartHomeSkillAction["CONTROL_DEVICE"] = "control_device";
+  ReceivedSmartHomeSkillAction["QUERY_SCENE"] = "query_scene";
+  ReceivedSmartHomeSkillAction["CONTROL_SCENE"] = "control_scene";
   return ReceivedSmartHomeSkillAction;
 }({});

package/dist/buildIn/withBuildIn.js

@@ -71,9 +71,6 @@ export function withBuildIn() {
       });
     })();
     (() => {
-      const intentMap = {
-        [ReceivedSmartHomeSkillAction.CONTROL_DEVICE]: 'controlDevice'
-      };
       onSkillsEnd((skills, responseMessage) => {
         if (!responseMessage) {
           return;
@@ -84,23 +81,58 @@ export function withBuildIn() {
           changeInfo: []
         };
         for (const skill of skills) {
-          var _content$general;
           if (skill.code !== BuildInSkillCode.SMART_HOME) {
             continue;
           }
           const content = skill;
-          if (!(content !== null && content !== void 0 && (_content$general = content.general) !== null && _content$general !== void 0 && (_content$general = _content$general.data) !== null && _content$general !== void 0 && (_content$general = _content$general.devices) !== null && _content$general !== void 0 && _content$general.length)) {
-            continue;
-          }
-          const intent = intentMap[content.general.action] || 'controlDevice';
-          for (const dev of content.general.data.devices) {
-            data.deviceInfo.push({
-              icon: dev.devIcon,
-              name: dev.devName,
-              deviceId: dev.devId,
-              intent,
-              success: null
-            });
+          if (content.general.action === ReceivedSmartHomeSkillAction.QUERY_SCENE) {
+            for (const scene of content.general.data.scenes) {
+              data.sceneInfo.push({
+                type: scene.type,
+                enabled: scene.enable,
+                icon: scene.icon,
+                name: scene.name,
+                sceneId: scene.sceneId,
+                intent: 'queryScene',
+                valid: scene.valid,
+                displayColor: scene.displayColor,
+                success: null
+              });
+            }
+          } else if (content.general.action === ReceivedSmartHomeSkillAction.CONTROL_SCENE) {
+            for (const scene of content.general.data.scenes) {
+              data.sceneInfo.push({
+                type: scene.type,
+                icon: scene.icon,
+                name: scene.name,
+                sceneId: scene.sceneId,
+                intent: 'controlScene',
+                valid: scene.valid,
+                displayColor: scene.displayColor,
+                success: scene.success
+              });
+            }
+          } else if (content.general.action === ReceivedSmartHomeSkillAction.QUERY_DEVICE) {
+            for (const dev of content.general.data.devices) {
+              data.deviceInfo.push({
+                icon: dev.icon,
+                name: dev.name,
+                deviceId: dev.deviceId,
+                intent: 'queryDevice',
+                room: dev.room,
+                success: dev.success
+              });
+            }
+          } else if (content.general.action === ReceivedSmartHomeSkillAction.CONTROL_DEVICE) {
+            for (const dev of content.general.data.devices) {
+              data.deviceInfo.push({
+                icon: dev.icon,
+                name: dev.name,
+                deviceId: dev.deviceId,
+                intent: 'controlDevice',
+                success: null
+              });
+            }
           }
         }
         if (data.deviceInfo.length) {

package/dist/global.d.ts

@@ -1,4 +1,4 @@
 import { AIStreamClient } from './utils';
 export declare const globalAIStreamClient: AIStreamClient;
-export declare const DEFAULT_TOKEN_API = "m.thing.aigc.basic.server.token";
+export declare const DEFAULT_TOKEN_API = "m.life.ai.token.get";
 export declare const DEFAULT_TOKEN_API_VERSION = "1.0";

package/dist/global.js

@@ -1,4 +1,4 @@
 import { AIStreamClient } from './utils';
 export const globalAIStreamClient = new AIStreamClient();
-export const DEFAULT_TOKEN_API = 'm.thing.aigc.basic.server.token';
+export const DEFAULT_TOKEN_API = 'm.life.ai.token.get';
 export const DEFAULT_TOKEN_API_VERSION = '1.0';

package/dist/utils/AIStream.js

@@ -7,7 +7,7 @@ import "core-js/modules/esnext.iterator.find.js";
 import "core-js/modules/esnext.iterator.for-each.js";
 import "core-js/modules/esnext.iterator.map.js";
 import "core-js/modules/web.dom-collections.iterator.js";
-import { AIStreamErrorCode, AIStreamServerErrorCode, BizTag, ConnectClientType, ConnectState, EventType, NetworkType, SessionState } from '../AIStreamTypes';
+import { AIStreamAttributeType, AIStreamErrorCode, AIStreamServerErrorCode, BizTag, ConnectClientType, ConnectState, EventType, NetworkType, SessionState } from '../AIStreamTypes';
 import { closeSession, connect, createSession, disconnect, getCurrentHomeInfo, getNetworkType, isConnected, queryAgentToken, registerRecordAmplitudes, sendEventChatBreak, sendEventEnd, sendEventPayloadEnd, sendEventStart, sendImageData, sendTextData, startRecordAndSendAudioData, stopRecordAndSendAudioData, unregisterVoiceAmplitudes } from './ttt';
 import { AIStreamObserver, AIStreamObserverPool } from './observer';
 import { isAbortError } from '@ray-js/t-agent';
@@ -178,12 +178,27 @@ export class AIStreamSession {
       (_this$activeEvent2 = this.activeEvent) === null || _this$activeEvent2 === void 0 || _this$activeEvent2.emit('data', entry);
       if (entry.type === 'event' && entry.body.eventId === this.activeEvent.eventId) {
         const {
-          eventType
+          eventType,
+          userData
         } = entry.body;
         if (eventType === EventType.EVENT_END || eventType === EventType.CHAT_BREAK || eventType === EventType.ONE_SHOT) {
           var _this$activeEvent3;
           (_this$activeEvent3 = this.activeEvent) === null || _this$activeEvent3 === void 0 || _this$activeEvent3.emit('finish');
           this.cleanupEvent();
+        } else if (eventType === EventType.SERVER_ERROR) {
+          var _this$activeEvent4;
+          let code = '';
+          let message = '';
+          for (const item of userData) {
+            if (item.type === AIStreamAttributeType.ERROR_CODE) {
+              code = item.value;
+            } else if (item.type === AIStreamAttributeType.ERROR_MESSAGE) {
+              message = item.value;
+            }
+          }
+          const error = new AIStreamError(message || 'Event error occurred', code || AIStreamErrorCode.UNKNOWN_ERROR);
+          (_this$activeEvent4 = this.activeEvent) === null || _this$activeEvent4 === void 0 || _this$activeEvent4.emit('error', error);
+          this.cleanupEvent();
         }
       }
     });
@@ -320,11 +335,11 @@ export class AIStreamSession {
     return this.activeEvent;
   }
   cleanupEvent() {
-    var _this$activeObserver, _this$activeEvent4;
+    var _this$activeObserver, _this$activeEvent5;
     logger.debug('AIStreamSession cleanupEvent');
     (_this$activeObserver = this.activeObserver) === null || _this$activeObserver === void 0 || _this$activeObserver.disconnect();
     this.activeObserver = null;
-    (_this$activeEvent4 = this.activeEvent) === null || _this$activeEvent4 === void 0 || _this$activeEvent4.emit('close');
+    (_this$activeEvent5 = this.activeEvent) === null || _this$activeEvent5 === void 0 || _this$activeEvent5.emit('close');
     this.activeEvent = null;
   }
   cleanup() {

package/dist/utils/createAsrAgent.d.ts

@@ -49,7 +49,7 @@ declare class AsrAgent {
  * @example
  * const asrAgent = createAsrAgent({
  *   agentId: 'asr-agent',
- *   tokenApi: 'm.thing.aigc.basic.server.token',
+ *   tokenApi: 'xxxx',
  *   tokenApiVersion: '1.0',
  *   clientType: ConnectClientType.APP,
  *   deviceId: 'deviceId',

package/dist/utils/createAsrAgent.js

@@ -2,7 +2,7 @@ import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
 import "core-js/modules/es.json.stringify.js";
 import { authorize } from './ttt';
 import { DEFAULT_TOKEN_API, DEFAULT_TOKEN_API_VERSION, globalAIStreamClient } from '../global';
-import { ConnectClientType, AIStreamAttributePayloadType, AIStreamAttributeType, AIStreamChatSysWorkflow } from '../AIStreamTypes';
+import { ConnectClientType, AIStreamAttributePayloadType, AIStreamAttributeType } from '../AIStreamTypes';
 import logger from './logger';
 class AsrAgent {
   constructor(options) {
@@ -51,7 +51,10 @@ class AsrAgent {
     const activeSession = streamConn.createSession({
       api: tokenApi || DEFAULT_TOKEN_API,
       apiVersion: tokenApiVersion || DEFAULT_TOKEN_API_VERSION,
-      solutionCode: agentId
+      solutionCode: agentId,
+      extParams: {
+        onlyAsr: true
+      }
     });
     this.activeSession = activeSession;
     return this.activeSession;
@@ -81,17 +84,15 @@ class AsrAgent {
       onFinish,
       onError
     } = this.options || {};
-    const activeSession = await this.createSession();
-    const attribute = {
-      'processing.interrupt': 'false',
-      'asr.enableVad': 'false',
-      'sys.workflow': AIStreamChatSysWorkflow.ASR
-    };
+    const activeSession = this.createSession();
     const activeEvent = await activeSession.startEvent({
       userData: [{
         type: AIStreamAttributeType.AI_CHAT,
         payloadType: AIStreamAttributePayloadType.STRING,
-        value: JSON.stringify(attribute)
+        value: JSON.stringify({
+          'processing.interrupt': 'false',
+          'asr.enableVad': 'false'
+        })
       }]
     });
     this.activeEvent = activeEvent;
@@ -165,7 +166,7 @@ class AsrAgent {
  * @example
  * const asrAgent = createAsrAgent({
  *   agentId: 'asr-agent',
- *   tokenApi: 'm.thing.aigc.basic.server.token',
+ *   tokenApi: 'xxxx',
  *   tokenApiVersion: '1.0',
  *   clientType: ConnectClientType.APP,
  *   deviceId: 'deviceId',

package/dist/utils/sendMessage.d.ts

@@ -7,6 +7,7 @@ export interface SendBlocksToAIStreamParams {
     session: AIStreamSession;
     attribute?: AIStreamChatAttribute;
     signal?: AbortSignal;
+    enableTts?: boolean;
 }
 export declare function sendBlocksToAIStream(params: SendBlocksToAIStreamParams): {
     response: StreamResponse;

package/dist/utils/sendMessage.js

@@ -3,7 +3,7 @@ import "core-js/modules/es.json.stringify.js";
 import "core-js/modules/web.dom-collections.iterator.js";
 import '../polyfill';
 import { ReadableStream } from 'web-streams-polyfill';
-import { AIStreamAttributePayloadType, AIStreamAttributeType, AIStreamChatSysWorkflow, AIStreamErrorCode, ConnectState, FileFormat, ReceivedTextPacketEof, ReceivedTextPacketType, SessionState, StreamFlag } from '../AIStreamTypes';
+import { AIStreamAttributePayloadType, AIStreamAttributeType, AIStreamErrorCode, ConnectState, FileFormat, ReceivedTextPacketEof, ReceivedTextPacketType, SessionState, StreamFlag } from '../AIStreamTypes';
 import { EmitterEvent, generateId, safeParseJSON, StreamResponse } from '@ray-js/t-agent';
 import { tryCatch } from './misc';
 import { AIStreamError, transformErrorCode } from './errors';
@@ -34,8 +34,7 @@ export function sendBlocksToAIStream(params) {
   }
   const attribute = _objectSpread({
     'processing.interrupt': 'false',
-    'asr.enableVad': 'false',
-    'sys.workflow': audioEmitter ? AIStreamChatSysWorkflow.ASR_LLM : AIStreamChatSysWorkflow.LLM
+    'asr.enableVad': 'false'
   }, params.attribute);
   let canceled = false;
   let closed = false;

package/dist/withAIStream.d.ts

@@ -26,6 +26,8 @@ export interface AIStreamOptions {
     earlyStart?: boolean;
     /** 自定义消息存储, 返回的实例需要实现 ChatHistoryLocalStore 接口, 返回null则不存储历史聊天记录 */
     createChatHistoryStore?: (agent: ChatAgent) => ChatHistoryLocalStore | null;
+    /** 是否开启音频合成 */
+    enableTts?: boolean;
 }
 export type AIStreamPlugin = GetChatPluginHandler<typeof withAIStream>;
 export interface AIStreamHooks {
@@ -43,7 +45,7 @@ export declare function withAIStream(options?: AIStreamOptions): (agent: ChatAge
     hooks: import("hookable").Hookable<any, string>;
     aiStream: {
         send: (blocks: InputBlock[], signal?: AbortSignal, extraOptions?: Record<string, any>) => {
-            response: import("@ray-js/t-agent").StreamResponse; /** 是否将输入块传递给智能体，默认为 true，设置为 false 时，需要你自己编写 onInputBlocksPush Hook 来处理输入块 */
+            response: import("@ray-js/t-agent").StreamResponse;
             metaPromise: Promise<Record<string, any>>;
         };
         chat: (blocks: InputBlock[], signal?: AbortSignal, options?: {

package/dist/withAIStream.js

@@ -1,5 +1,5 @@
-import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
 import _objectWithoutProperties from "@babel/runtime/helpers/esm/objectWithoutProperties";
+import _objectSpread from "@babel/runtime/helpers/esm/objectSpread2";
 const _excluded = ["message"];
 import "core-js/modules/es.array.flat.js";
 import "core-js/modules/es.array.reverse.js";
@@ -98,7 +98,10 @@ export function withAIStream() {
         api: tokenOptions.api,
         apiVersion: tokenOptions.version,
         solutionCode: agentId,
-        extParams: tokenOptions.extParams
+        extParams: _objectSpread(_objectSpread({}, tokenOptions.extParams), {}, {
+          needTts: !!options.enableTts,
+          deviceId: options.deviceId || undefined
+        })
       });
       await session.set('AIStream.streamSession', streamSession);
       if (options.earlyStart) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ray-js/t-agent-plugin-aistream",
-  "version": "0.2.0-beta.0",
+  "version": "0.2.0-beta.19",
   "author": "Tuya.inc",
   "license": "MIT",
   "private": false,
@@ -35,5 +35,5 @@
   "devDependencies": {
     "@types/url-parse": "^1.4.11"
   },
-  "gitHead": "0b5d552289721444563828b03a51a6e8695779db"
+  "gitHead": "0db50eef90b217b739bab1d824e4c6b2d495a169"
 }