@ray-js/t-agent 0.0.5-beta-1

package/README.md

@@ -0,0 +1,1345 @@
+English | [简体中文](./README-zh_CN.md)
+
+# AI Intelligent Agent SDK
+
+## Installation (ray mini-program)
+
+```shell
+yarn add @ray-js/t-agent @ray-js/t-agent-plugin-assistant @ray-js/t-agent-ui-ray
+```
+
+> Ensure that the versions of `@ray-js/t-agent`, `@ray-js/t-agent-plugin-assistant`, and `@ray-js/t-agent-ui-ray` are consistent.
+
+## Mini-Program Kit Requirements
+
+```json
+{
+  "dependencies": {
+    "BaseKit": "3.12.0",
+    "BizKit": "4.10.0",
+    "DeviceKit": "4.6.1",
+    "HomeKit": "3.4.0",
+    "MiniKit": "3.12.1"
+  },
+  "baseversion": "2.21.10"
+}
+```
+
+## package.json Dependency Requirements
+
+```json
+{
+  "dependencies": {
+    "@ray-js/ray": ">=1.6.8"
+  }
+}
+```
+
+## Development
+
+```shell
+# Develop the SDK
+yarn run dev
+
+# Develop the template mini-program
+yarn run miniapp
+```
+
+## Usage Example
+
+Implementing a chat page using ray UI
+
+![ChatPage1](https://static1.tuyacn.com/static/txp-ray-TAgent/ChatPage1.png)
+
+```tsx
+// ChatPage.tsx
+import React from 'react';
+import { View } from '@ray-js/components';
+import { createChatAgent, withDebug, withUI } from '@ray-js/t-agent';
+import { ChatContainer, MessageInput, MessageList } from '@ray-js/t-agent-ui-ray';
+import { withAssistant, withBuildIn } from '@ray-js/t-agent-plugin-assistant';
+
+const createAgent = () => {
+  // The order of applying plugins matters
+  const agent = createChatAgent(
+    withUI(), // The withUI plugin provides default UI behavior, essential
+    withAssistant({
+      // Connects to the mini-program AI agent platform, essential in mini-program
+      channel: '', // Enter your agent ID
+      multiModal: false, // Whether to enable multi-modal capabilities
+    }),
+    withDebug(), // Enables logging to the console
+    withBuildIn() // Provides built-in features
+  );
+
+  // Lifecycle hooks for custom behavior registration
+  const { onChatStart, createMessage, onChatResume, onError, onInputBlocksPush, session } = agent;
+
+  // Initialize chat with a message
+  onChatStart(async result => {
+    const hello = createMessage({
+      role: 'assistant',
+    });
+
+    hello.bubble.setText('Hello, world!');
+    result.messages.push(hello);
+    // Persist message for display upon next entry
+    await hello.persist('createText');
+  });
+
+  // Message upon chat resume
+  onChatResume(async result => {
+    const welcomeBack = createMessage({
+      role: 'assistant',
+    });
+
+    welcomeBack.bubble.setText('Welcome back');
+    result.messages.push(welcomeBack);
+    await welcomeBack.persist('createText');
+  });
+  return agent;
+};
+
+export default function ChatPage() {
+  return (
+    <View style={{ height: '100vh' }}>
+      <ChatContainer createAgent={createAgent}>
+        <MessageList />
+        <MessageInput />
+      </ChatContainer>
+    </View>
+  );
+}
+```
+
+# t-agent
+
+The t-agent package, crafted in TypeScript, is a dialogue agent SDK for building feature-rich dialogue interactions, supporting plugins to enhance functionality. As a pure SDK, it doesn’t contain UI components, allowing flexibility to work with any framework.
+
+## Basic Concepts
+
+![flow.png](https://static1.tuyacn.com/static/txp-ray-TAgent/flow-en.png)
+
+### ChatAgent Dialogue Agent
+
+The core class handles the dialogue lifecycle, message creation, persistence, and supports plugins and hooks to extend functionality.
+
+Create a ChatAgent instance using `createChatAgent`:
+
+```tsx
+import { createChatAgent } from '@ray-js/t-agent';
+const createAgent = () => {
+  /* Apply plugins in the createChatAgent function. Plugin order matters. */
+  const agent = createChatAgent();
+
+  return agent;
+};
+```
+
+Core Attributes:
+
+- `agent.session` ChatSession container for session data
+- `agent.plugins` Contains applied plugins' methods and hooks
+
+Core Methods:
+
+- `agent.start()` Activates the agent
+- `agent.dispose()` Deactivates the agent
+- `agent.pushInputBlocks(blocks, signal)` Pushes user messages to ChatAgent
+- `agent.createMessage(data)` Creates a message linked to the agent
+- `agent.emitTileEvent(tileId: string, payload: any)` Emits tile events
+- `agent.removeMessage(messageId: string)` Removes messages
+- `agent.flushStreamToShow(message: ChatMessage, response: StreamResponse, composer: ComposeHandler)` Handles stream updates
+
+### Hooks Mechanism
+
+ChatAgent implements a framework and data structure. Behavioral details are executed using a hook-driven, event-model approach by registering specific callbacks.
+
+```tsx
+import { createChatAgent } from '@ray-js/t-agent';
+const createAgent = () => {
+  const agent = createChatAgent();
+
+  const { onChatStart } = agent;
+
+  // onChatStart Hook triggers on dialogue start
+  onChatStart(result => {
+    console.log('Chat start', result);
+  });
+  return agent;
+};
+```
+
+ChatAgent's Central Hooks and Parameters:
+
+- `agent.onAgentStart` Agent initialization
+- `agent.onChatStart` Triggered at dialogue initiation
+  - `result.messages` Initializes message lists
+- `agent.onChatResume` Triggered on dialogue continuation
+  - `result.messages` Restored message lists
+- `agent.onMessageListInit` On message list initialization
+  - `result.messages` Rendering message list
+- `agent.onInputBlocksPush` On message block input
+  - `blocks` Input message blocks
+  - `signal` Interrupt signal
+- `agent.onMessageChange` On message updates
+  - `type` Update types: `show`, `update`, `remove`
+  - `message` Updated message instance
+- `agent.onMessagePersist` At message persistence
+  - `payload` Persistence parameters
+  - `message` The message to persist
+- `agent.onTileEvent` After message persistence
+  - `tile` Event-triggering tile
+  - `payload` Event data
+- `agent.onAgentDispose` On agent disposal
+- `agent.onUserAbort` On user interruption
+  - `reason` Interruption reason
+- `agent.onError` On error occurrence
+  - `error` Error details
+
+These hooks accept a callback parameter, invoked upon their criteria. Callbacks can be synchronous or asynchronous. To modify results, adjust the result parameter or message object directly.
+
+### ChatSession Session Container
+
+ChatSession holds message lists, session-related data along with intelligent interactions, generated alongside ChatAgent.
+
+Main Properties:
+
+- `session.messages` List of messages
+- `session.sessionId` Unique session ID
+- `session.isNewChat` To distinguish fresh or ongoing chats
+
+Principal Methods:
+
+- `session.set` Assign session data (any type)
+- `session.get` Retrieve session data
+- `session.getData` Fetch session data in object format
+- `session.getLatestMessage` Access the latest message
+
+Hook:
+
+- `session.onChange` Register for session data updates
+
+### ChatMessage Dialogue Message
+
+An abstraction representing each dialogue's content, status with utility methods for message control. Composed of multiple ChatTiles for varied display elements.
+
+Creating a Message with `createMessage`:
+
+```tsx
+const createAgent = () => {
+  const agent = createChatAgent();
+
+  const { createMessage, onChatStart } = agent;
+
+  // Initialize chat with an assistant's message
+  onChatStart(async result => {
+    const message = createMessage({
+      role: 'assistant',
+    });
+
+    // Quickly create text bubble
+    message.bubble.setText('Hello!');
+    result.messages.push(message);
+  });
+
+  return agent;
+};
+```
+
+Core Properties:
+
+- `message.id` Unique message identifier
+- `message.role` Either assistant or user
+- `message.tiles` List of message tiles
+- `message.status` Message state, `ChatMessageStatus` enum: `START`, `UPDATING`, `FINISH`
+- `message.meta` Supplementary data
+- `message.isShow` Display status
+- `message.bubble` Shortcut for bubble tile
+
+Key Methods:
+
+- `message.show` Display message
+- `message.update` Refresh current message
+- `message.remove` Delete current message
+- `message.persist` Save message
+- `message.addTile` Add a tile
+- `message.removeTile` Delete a tile
+- `message.setTilesLocked` Lock/unlock all tiles
+- `message.set` Configure message attributes
+- `message.setMetaValue` Assign meta attribute by key
+- `message.deleteMetaValue` Remove meta attribute
+- `message.setMeta` Comprehensive meta object assignment
+- `message.findTileByType` Locate tile by type
+
+#### ChatTile Dialogue Message Block
+
+ChatTiles render specific content types fairly like text, images, and cards.
+
+Adding a Tile with `addTile`:
+
+```tsx
+const message = createMessage({
+  role: 'assistant',
+});
+
+// Introduce an image tile
+message.addTile('image', {
+  src: '/image.jpg',
+});
+
+await message.show();
+```
+
+Chief Attributes of ChatTile:
+
+- `tile.id` Tile's identifier
+- `tile.type` Type of tile
+- `tile.data` Content held by tile
+- `tile.children` Nested tiles
+- `tile.locked` Tile lock status
+- `tile.fallback` Alternative content for non-displayed tile
+- `tile.message` Parent message of tile
+
+Primary Methods of ChatTile:
+
+- `tile.update` Equivalent to `tile.message.update`
+- `tile.show` Synonymous with `tile.message.show`
+- `tile.setLocked` Adjust tile's lock status
+- `tile.addTile` Add a sub-tile
+- `tile.setData` Update tile content
+- `tile.setFallback` Update fallback content
+- `tile.findByType` Locate sub-tile by type
+
+Accessing `message.bubble` enables swift bubble tile addition:
+
+```tsx
+const message = createMessage({
+  role: 'assistant',
+});
+
+message.bubble.setText('Hello, world!');
+// Equivalent to
+message.addTile('bubble', {}).addTile('text', { text: 'Hello, world!' });
+
+await message.show();
+```
+
+For bubble messages, additional properties and methods include:
+
+- `message.bubble.text` Reads bubble text
+- `message.bubble.setText` Assigns bubble text
+- `message.bubble.isMarkdown` Determines markdown formatting
+- `message.bubble.setIsMarkdown` Sets markdown format
+- `message.bubble.status` Indicates bubble status
+- `message.bubble.setStatus` Defines bubble status
+- `message.bubble.info` Relays bubble info
+- `message.bubble.setInfo` Configures bubble info
+- `message.bubble.initWithInputBlocks` Initializes bubble with input blocks
+
+### Lifecycle
+
+ChatAgent phases trigger hooks enabling developers to integrate custom behavior efficiently. Below is the lifecycle depiction:
+
+```mermaid
+sequenceDiagram
+  participant AI as Backend(AI)
+  participant A as ChatAgent
+  participant UI as UI Interface
+  actor User as User
+
+  User ->> UI: Open chat interface
+  UI ->> UI: Create Agent object agent
+  UI ->> A: agent.start() Start Agent
+  rect rgba(255,255,0,0.1)
+    note over A: Hook: onAgentStart
+    A ->> AI: Check if new session
+    AI ->> A: Return
+  end
+  alt Is new session
+    A ->> AI: Start a new session
+    note over A: Hook: onChatStart
+  else Is resume session
+    A ->> AI: Read session history
+    AI ->> A: Return history messages
+    note over A: Hook: onChatResume
+  end
+  rect rgba(255,255,0,0.1)
+    note over A: Hook: onMessageListInit
+    A ->> UI: Assemble messages to display
+    UI ->> User: Display messages
+  end
+  opt Conversation
+    User ->> UI: Enter and send message
+    UI ->> A: agent.pushInputBlocks()
+    rect rgba(255,255,0,0.1)
+      note over A: Hook: onInputBlocksPush
+      A ->> A: Create Message object msg\nSet msg as user's input text\nmsg.show()
+      note over A: Hook: onMessageChange show
+      A -->> UI: Update interface
+      UI -->> User: Display new message
+      A ->> A: Create Message object respMsg\nSet respMsg as "loading"\nrespMsg.show()
+      note over A: Hook: onMessageChange show
+      A -->> UI: Update interface
+      UI -->> User: Display loading message
+      A ->> AI: Invoke AI
+      AI ->> A: Return message stream
+      loop Streaming messages
+        AI ->> A: Message packet
+        A ->> A: Update respMsg data\nrespMsg.update()
+        note over A: Hook: onMessageChange update
+        A -->> UI: Update interface
+        UI -->> User: Display typing effect/card
+      end
+    end
+   end
+
+    opt Operate message action points (e.g., single selection)
+       User ->> UI: Select option
+       UI ->> A: agent.emitTileEvent()
+       rect rgba(255,255,0,0.1)
+          note over A: Hook: onTileEvent
+          A ->> A: Set msg status\nmsg.persist()
+          note over A: Hook: onMessagePersist
+          A ->> AI: Persist message
+          AI ->> A: Return result
+          A ->> A: msg.update()
+          note over A: Hook: onMessageChange update
+          A ->> UI: Update interface
+          UI ->> User: Message greyed out, highlight selection
+       end
+    end
+    opt Delete message
+       User ->> UI: Delete message
+       UI ->> A: agent.removeMessage()
+       rect rgba(255,255,0,0.1)
+          A ->> A: msg.remove()
+          A ->> AI: Mark message deleted
+          note over A: Hook: onMessageChange remove
+          A ->> UI: Update interface
+          UI ->> User: Message disappears
+       end
+    end
+```
+
+### Plugin Mechanism
+
+Plugins are implemented based on the above Hook mechanism and can enhance the functionality of chat agents, such as interfacing with AI platforms, providing UI interfaces, etc. Plugins can also expose methods and properties for developers to use.
+
+```tsx
+import { createChatAgent, withUI } from '@ray-js/t-agent';
+const createAgent = () => {
+  const agent = createChatAgent(
+    withUI() // withUI plugin provides methods for rendering UI
+  );
+
+  return agent;
+};
+
+// ScrollToBottom.tsx
+import React from 'react';
+import { Button } from '@ray-js/components';
+import { useChatAgent } from '@ray-js/t-agent-ui-ray';
+const ScrollToBottom = () => {
+  const agent = useChatAgent();
+
+  const scroll = () => {
+    // Use methods exposed by the plugin
+    agent.plugins.ui.emitEvent('scrollToBottom', { animation: false });
+  };
+
+  return <Button onClick={scroll}>Scroll to bottom</Button>;
+};
+```
+
+#### Writing a Plugin
+
+A plugin is a higher-order function that takes options and returns a function. This function receives a ChatAgent object, within which hooks can be registered.
+
+```tsx
+import { ChatAgent, createHooks, Hookable } from '@ray-js/t-agent';
+
+// Implementing MyPlugin
+export type MyPlugin = GetChatPluginHandler<typeof withMyPlugin>;
+
+export const withMyPlugin = (options: any) => {
+  // Create a Hookable object
+  const hooks = createHooks();
+
+  return (agent: ChatAgent) => {
+    const { onChatStart } = agent;
+
+    onChatStart(async () => {
+      console.log('Chat start');
+      // Trigger plugin hook
+      await hooks.callHook('onMyHook', 'Hello, world!');
+    });
+
+    // Expose plugin methods and hooks
+    return {
+      hooks,
+      myPlugin: {
+        // Expose a method
+        myMethod() {
+          console.log('My method');
+        },
+        // Expose a hook
+        onMyHook: fn => {
+          return hooks.hook('onMyHook', fn);
+        },
+      },
+    };
+  };
+};
+```
+
+```tsx
+// Using the plugin
+import { createChatAgent } from '@ray-js/t-agent';
+
+const createAgent = () => {
+  const agent = createChatAgent(
+    // Use the plugin
+    withMyPlugin({})
+  );
+
+  // Call methods exposed by the plugin
+  agent.plugins.myPlugin.myMethod();
+
+  // Register plugin hooks
+  agent.plugins.myPlugin.onMyHook(msg => {
+    console.log(msg);
+  });
+
+  return agent;
+};
+```
+
+## Built-in Plugins
+
+### withDebug
+
+The withDebug plugin logs messages to the console for easy debugging.
+
+```tsx
+const agent = createChatAgent(
+  withDebug({
+    autoStart: true, // Whether to start automatically, default is true
+  })
+);
+
+// Start
+agent.plugins.debug.start();
+
+// Stop
+agent.plugins.debug.stop();
+```
+
+### withUI
+
+The withUI plugin provides default UI behaviors such as message display, message deletion, etc., and a message bus.
+
+```tsx
+const agent = createChatAgent(withUI());
+
+agent.plugins.ui.emitter; // Message bus
+
+// Scroll to bottom
+agent.plugins.ui.emitEvent('scrollToBottom', { animation: false });
+
+// Listen to events
+const off = agent.plugins.ui.onEvent('scrollToBottom', payload => {
+  console.log('scroll to bottom', payload.animation);
+});
+
+// Unlisten to events
+off();
+```
+
+The main events of the UI plugin are as follows, and you can freely register additional events you need:
+
+- `messageListInit`: Initialize the message list
+  - `payload.messages: ChatMessageObject[]`: Message list
+- `messageChange`: Message changes
+  - `payload.type: 'show' | 'update' | 'remove'`: Change type
+  - `payload.message: ChatMessageObject`: Message
+- `scrollToBottom`: Scroll to bottom
+  - `payload.animation: boolean`: Whether to use animation
+- `sendMessage`: Send message, triggering UI update
+  - `payload.blocks: InputBlock[]`: Input blocks
+- `setInputBlocks`: Set input blocks in MessageInput
+  - `payload.blocks: InputBlock[]`: Input blocks
+
+> Note: `ChatMessageObject` here is a message object, not a `ChatMessage` type.
+> It contains various attributes and methods to ensure changes are made in the ChatAgent, avoiding conflicts with the UI layer.
+
+## Included utils
+
+### getLogger(prefix: string): Logger
+
+Create a logger for logging messages.
+
+```tsx
+import { getLogger } from '@ray-js/t-agent';
+const logger = getLogger('MyPlugin');
+logger.debug('Hello, world!');
+```
+
+### Emitter
+
+Emitter is an event bus for registering and triggering events.
+
+```tsx
+import { Emitter, EmitterEvent } from '@ray-js/t-agent';
+const emitter = new Emitter();
+
+// Register event
+const cb = event => console.log('detail', event.detail);
+emitter.addEventListener('event', cb);
+
+// Trigger event
+emitter.dispatchEvent(new EmitterEvent('event', { detail: 'Hello, world!' }));
+
+// Remove event
+emitter.removeEventListener('event', cb);
+```
+
+### StreamResponse
+
+StreamResponse is a streaming response object for handling streaming messages.
+
+```tsx
+import { StreamResponse } from '@ray-js/t-agent';
+
+const partStream = await getPartStream(); // Obtain stream data
+const response = new StreamResponse(partStream);
+
+const parts = response.parts();
+for await (const part of parts) {
+  console.log('part', part);
+}
+```
+
+### createHooks, Hookable
+
+Refer to the `hookable` npm package for more details.
+
+### isAbortError
+
+Determine if it is an abort error.
+
+### safeParseJSON
+
+Safely parse JSON strings, returning `undefined` on failure.
+
+```tsx
+import { safeParseJSON } from '@ray-js/t-agent';
+
+const obj = safeParseJSON<{ a: number }>('{"a": 1}');
+
+console.log(obj.a); // 1
+```
+
+# t-agent-plugin-assistant
+
+t-agent-plugin-assistant is a plugin designed for integration with applet AI agent platforms, offering capabilities to connect to these platforms.
+
+## Installation
+
+```shell
+yarn add @ray-js/t-agent-plugin-assistant
+```
+
+## Usage
+
+```tsx
+import { createChatAgent, withUI } from '@ray-js/t-agent';
+import { withAssistant, withBuildIn } from '@ray-js/t-agent-plugin-assistant';
+
+const createAgent = () => {
+  const agent = createChatAgent(
+    withUI(), // Typically, the withUI plugin is necessary
+    withAssistant({
+      channel: 'your-channel-id', // Provide your agent ID
+      multiModal: false, // Enable multi-modal? Defaults to true
+    }),
+    withBuildIn()
+  );
+
+  return agent;
+};
+```
+
+## Included Plugins
+
+### withAssistant Plugin
+
+This plugin facilitates communication with applet AI agent platforms.
+
+Parameters:
+
+- `channel`: Agent ID
+- `multiModal`: Enable multi-modal? Defaults to true
+- `wireInputToAssistant`: Should input blocks be passed to the agent? Defaults to true. Set to false if you plan to handle input blocks manually with the onInputBlocksPush Hook.
+- `historySize`: Size of message history, default is 100.
+
+Methods:
+
+- `agent.plugins.assistant.send`: Send a message to the agent.
+- `agent.plugins.assistant.chat`: Send a message to the agent while generating a ChatMessage object for the question and the AI's answer, updating in a streaming manner. Useful for onInputBlocksPush.
+
+Hooks:
+
+- `onSocketStatusChange`: Triggered when the network status changes.
+- `onRawMessageParse`: Triggered during history message parsing, allowing for message modification in this Hook.
+  - `rawItems`: Original message data
+  - `result.messages`: Message list, typically generating one question message (`messages[0]`) and one answer message (`messages[1]`) per history record. You can modify or extend this list.
+- `onExtensionCompose`: When parsing message extension data, this hook activates.
+  - `extension`: Extension data
+  - `responseMessage`: Response message
+  - `result.messages`: Message list
+
+### withBuildIn Plugin
+
+Offers built-in features like workflows, buttons, recommended actions, etc., currently under development.
+
+### withAssistantCopyHistory Plugin
+
+Utilizing this plugin in real machine previews automatically copies history records to the clipboard for easier debugging.
+
+## Mock Mechanism
+
+To facilitate development, we offer a mock mechanism allowing mock data to be used without connecting to applet AI platforms.
+
+### Mock a ttt Interface
+
+```tsx
+import { mock } from '@ray-js/t-agent-plugin-assistant';
+
+// Mock the interface for retrieving historical data
+mock.hooks.hook('getAIAssistantGroupHistory', context => {
+  context.result = yourMockData;
+});
+```
+
+### Mock AI Assistant Response
+
+````
+```tsx
+import { mock } from '@ray-js/t-agent-plugin-assistant';
+
+mock.hooks.hook('sendToAssistant', context => {
+  if (context.options.block?.includes('hello')) {
+    context.responseText = 'Hello, who are you?';
+  }
+
+  if (context.options.block?.includes('卡片')) {
+    context.responseText = 'This is a card example';
+    context.responseExtensions = {
+      aiCards: [
+        {
+          cardCode: 'myCard',
+          cardType: ChatCardType.CUSTOM,
+          cardData: { title: 'Card 1' },
+        },
+        {
+          cardCode: 'myCard',
+          cardType: ChatCardType.CUSTOM,
+          cardData: { title: 'Card 2' },
+        },
+      ],
+    };
+  }
+
+  if (context.options.block?.includes('工作流')) {
+    context.responseText = 'This is a workflow';
+    context.responseExtensions = {
+      workflowAskOptions: {
+        options: [
+          {
+            name: 'Option 1',
+            value: 'Option 1',
+          },
+          {
+            name: 'Option 2',
+            value: 'Option 2',
+          },
+          {
+            name: 'Option 3',
+            value: 'Option 3',
+          },
+        ],
+      },
+    };
+  }
+});
+````
+
+### mock ASR Speech Recognition
+
+```tsx
+import { mock } from '@ray-js/t-agent-plugin-assistant';
+
+mock.hooks.hook('asrDetection', context => {
+  context.responseText = 'Hello world!, I am a virtual assistant.';
+});
+```
+
+## Additional Utils Tools (Beta Tools)
+
+### AbortController
+
+This is an AbortController ponyfill for mini-programs, refer to mdn
+
+### runTTTAction
+
+Run a TTTAction to handle user actions, currently supporting the following actions
+
+- `openRoute` Open a route
+- `openMiniApp` Open a mini program
+- `openH5` Open an H5 page
+- `sendMessage` Send a message
+- `buildIn` Built-in action
+
+### Asr
+
+ASR Speech Recognition encapsulation, used for recognizing user's voice input
+
+Usage:
+
+```tsx
+import { Asr, AsrDetectResultState, AsrError } from '@ray-js/t-agent-plugin-assistant';
+
+async function startAsr() {
+  // Request user permission first
+  await Asr.authorize();
+
+  // Initial text
+  let initial = '';
+
+  // Previously recognized text
+  let last = initial;
+
+  // Generate a recognizer
+  const asr = Asr.detect(async res => {
+    if (res.state === AsrDetectResultState.STARTING || res.state === AsrDetectResultState.END) {
+      // Full text
+      const full = initial + res.text;
+
+      // Recently recognized text
+      const incoming = full.slice(last.length);
+
+      // Previously recognized text
+      last = full;
+    }
+
+    if (res.state === AsrDetectResultState.ERROR) {
+      onError(new AsrError(res.errorCode));
+    }
+  });
+
+  // Start recognition
+  await asr.start();
+}
+```
+
+### promisify TTT
+
+A promisify method for the built-in TTT API, used to convert the TTT API into a Promise, supporting mock concurrently
+
+Usage:
+
+```tsx
+import { promisify } from '@ray-js/t-agent-plugin-assistant';
+
+interface RouterParams {
+  /** Route link */
+  url: string;
+  complete?: () => void;
+  success?: (params: null) => void;
+  fail?: (params: {
+    errorMsg: string;
+    errorCode: string | number;
+    innerError: {
+      errorCode: string | number;
+      errorMsg: string;
+    };
+  }) => void;
+}
+const router = promisify<RouterParams>(ty.router);
+
+// mock, only effective under IDE
+mock.hooks.hook('router', context => {
+  console.log('call router', context.options);
+});
+
+// invoke
+await router({ url: '/pages/index/index' });
+```
+
+### sendBlockToAssistant
+
+Send a message to the Assistant
+
+```tsx
+import { sendBlockToAssistant, getAIAssistantRequestId } from '@ray-js/t-agent-ui-ray';
+
+const send = async () => {
+  const requestId = await getAIAssistantRequestId();
+  const result = sendBlockToAssistant({
+    channel: 'your-channel-id',
+    sessionId: 'your-session-id',
+    requestId,
+    blocks: [{ type: 'text', text: 'hello' }],
+  });
+
+  // Get metadata after sending
+  const meta = await result.metaPromise;
+
+  // Get streamed message
+  const parts = result.parts();
+
+  for await (const part of parts) {
+    console.log('part', part);
+  }
+};
+```
+
+### sendSkillToAssistant
+
+Send skills to the Assistant
+
+```tsx
+import { sendSkillToAssistant } from '@ray-js/t-agent-ui-ray';
+
+const send = async () => {
+  const result = sendSkillToAssistant({
+    channel: 'your-channel-id',
+    sessionId: 'your-session-id',
+    options: {
+      domain: 'string',
+      intent: 'string',
+    },
+  });
+
+  // Get metadata after sending
+  const meta = await result.metaPromise;
+
+  // Get streamed messages
+  const parts = result.parts();
+
+  for await (const part of parts) {
+    console.log('part', part);
+  }
+};
+```
+
+### authorizeAssistantPolicy
+
+Popup to remind users to accept the AI privacy policy, a security requirement that users must agree to access AI features in the app
+
+```tsx
+import { authorizeAssistantPolicy } from '@ray-js/t-agent-plugin-assistant';
+
+// Call this function after the page load completes to pop up the agreement prompt
+const authorize = async () => {
+  try {
+    const result = await authorizeAssistantPolicy();
+
+    if (result) {
+      // Already agreed or clicked the agree button
+      console.log('User agreed to the AI privacy policy');
+    } else {
+      ty.exitMiniProgram({});
+      console.log('User declined the AI privacy policy');
+    }
+  } catch (e) {
+    // Popup error message
+    ty.showToast({
+      title: I18n.t('get_sign_error'),
+      icon: 'error',
+    });
+    // Delay to allow the user to read clearly
+    setTimeout(() => {
+      ty.exitMiniProgram({});
+    }, 1000);
+  }
+};
+```
+
+### Functions Related to Media Files
+
+`uploadMedia`, `uploadVideo`, `uploadImage` are used to upload media files, with caching supported.
+
+`isFullLink` is used to determine whether it is a link with http(s) protocol
+
+`parseCloudKey` is used to parse the key of cloud storage in the URL
+
+`isLinkExpired` is used to determine whether the link is expired
+
+`getUrlByCloudKey` is used to get the signed download URL of cloud storage from the cache, otherwise returns `undefined`
+
+`setUrlByCloudKey` is used to set the signed download URL of cloud storage into the cache
+
+`resetUrlByCloudKey` is used to reset the cache of the signed download URL of cloud storage
+
+`chooseImage`, `chooseVideo` are used to select media files
+
+```tsx
+import { useEffect } from 'react';
+
+async function getPictureList() {
+  // Retrieve the user's private picture list from the cloud
+  const list = await pictureListRequest();
+  // Set into the cache this way, so it won’t need to request next time
+  for (const item of list) {
+    setUrlByCloudKey(item.path, item.displayUrl);
+  }
+}
+```
+
+# t-agent-ui-ray
+
+t-agent-ui-ray is a UI component library based on ray, containing some commonly used dialogue interface components, such as message list, message input box, etc.
+
+## Installation
+
+```shell
+yarn add @ray-js/t-agent-ui-ray
+```
+
+## Usage
+
+```tsx
+import { ChatContainer, MessageList, MessageInput } from '@ray-js/t-agent-ui-ray';
+// see the use case of t-agent for createAgent implementation
+import { createAgent } from './createAgent';
+
+export default function ChatPage() {
+  // createAgent must return a ChatAgent instance applied with withUI, withAssistant plugins
+  return (
+    <View style={{ height: '100vh' }}>
+      <ChatContainer createAgent={createAgent}>
+        <MessageList />
+        <MessageInput />
+      </ChatContainer>
+    </View>
+  );
+}
+```
+
+## Components
+
+### ChatContainer
+
+Dialog container, used to wrap message lists and message input boxes, provides the context for `ChatAgent`
+
+props:
+
+- `className` Class name of the container
+- `createAgent` Function to create `ChatAgent`, which will call this function to create `ChatAgent` instance after `ChatContainer` mount
+- `renderOptions` Rendering options, used to decide the rendering method of each element in `MessageList`, refer to the renderOptions custom rendering part below
+  - `renderTileAs` This function decides how to render tiles in the messages
+  - `customBlockTypes` Custom block types, only block types registered here will be rendered by `renderCustomBlockAs`
+  - `renderCustomBlockAs` This function decides how to render custom blocks in markdown bubble messages, by default supports `echarts`
+  - `renderCardAs` This function decides how to render cards in messages, generally no need to customize this item
+  - `customCardMap` Custom card mapping, no need to modify the `renderCardAs` function, just register the card type and corresponding component here
+  - `getStaticResourceBizType` Get static resource `bizType` used to fetch static resources
+
+### MessageList
+
+Message list, used to display messages
+
+props:
+
+- `className` Class name of the list
+- `roleSide` Alignment of message roles, default `{ user: 'end', assistant: 'start' }`
+
+### MessageInput
+
+Message input box, used to input messages, upload attachments, ASR speech recognition
+
+props:
+
+- `className` Class name of the input box
+- `placeholder` Placeholder of the input box
+- `renderTop` Used to render content above the input box
+
+### PrivateImage
+
+Private image component, used to display private images, props are the same as Image, adding bizType parameter
+
+### Built-in tile components
+
+- bubble Bubble
+- buttons Button group
+- card Card
+- image Image
+- recommendations Recommended actions
+- text Text, including markdown support
+- time Time marker
+- tip Tip
+- video Video
+- workflow Workflow options
+
+### Built-in card
+
+- WorkflowReplyCard Workflow reply card
+
+## React Hooks
+
+### useChatAgent
+
+Get `ChatAgent` instance in the `ChatContainer` context
+
+### useAgentMessage
+
+Get `messages: ChatMessageObject[]` list in the `ChatContainer` context
+
+### useRenderOptions
+
+Get `renderOptions` object in the `ChatContainer` context
+
+### useOnEvent
+
+Register UI events in the `ChatContainer` context, and automatically unregister them when the component is unloaded
+
+```tsx
+import { useOnEvent } from '@ray-js/t-agent-ui-ray';
+
+const MyComponent = () => {
+  useOnEvent('scrollToBottom', payload => {
+    console.log('scroll to bottom', payload.animation);
+  });
+
+  return <div>My Component</div>;
+};
+```
+
+### useEmitEvent
+
+Trigger UI events in the `ChatContainer` context
+
+```tsx
+import { useEmitEvent } from '@ray-js/t-agent-ui-ray';
+
+const MyComponent = () => {
+  const emitEvent = useEmitEvent();
+
+  const scroll = () => {
+    emitEvent('scrollToBottom', { animation: false });
+  };
+
+  return <button onClick={scroll}>Scroll to bottom</button>;
+};
+```
+
+### useTileProps
+
+Get the `TileProps` object in the tile component; if it's a tile, you can directly obtain it from props.
+
+```tsx
+import { useTileProps } from '@ray-js/t-agent-ui-ray';
+
+const MyTilePart = () => {
+  const { message, agent, tile, emitEvent } = useTileProps();
+
+  return <div>My Tile</div>;
+};
+```
+
+### useSendAction
+
+Send a TTTAction, note that it can only be used inside tile components (or cards).
+
+```tsx
+import { useSendAction } from '@ray-js/t-agent-ui-ray';
+
+const MyTilePart = () => {
+  const sendAction = useSendAction();
+
+  const handleClick = () => {
+    sendAction({ type: 'sendMessage', blocks: [{ type: 'text', text: 'hello' }] });
+  };
+
+  return <button onClick={handleClick}>Send Message</button>;
+};
+```
+
+## renderOptions Custom Rendering
+
+### Replace or Add Tile
+
+If you need to replace a tile with your own implementation, or add a new tile, you can override `renderTileAs`, for example:
+
+```tsx
+import { ImageTileData } from '@ray-js/t-agent';
+import { Image } from '@ray-js/ray';
+import { defaultRenderOptions, TileProps } from '@ray-js/t-agent-ui-ray';
+
+function MyImageTile(props: TileProps<ImageTileData>) {
+  // Implement your own ImageTile
+  return <Image src={props.tile.data.src}></Image>;
+}
+
+const renderOptions = {
+  ...defaultRenderOptions,
+  renderTileAs: (props: TileProps) => {
+    if (props.tile.type === 'image') {
+      return <MyImageTile {...props} />;
+    }
+    // Maintain default behavior
+    return defaultRenderOptions.renderTileAs(props);
+  },
+};
+```
+
+### Customize Cards
+
+Cards can be categorized into three types: built-in cards (buildIn), custom cards (custom), and low-code cards (lowCode). Currently, low-code cards are still under development. Custom cards can be registered using `customCardMap`.
+
+The data structure of the card is as follows:
+
+```tsx
+enum ChatCardType {
+  CUSTOM = 'custom', // Custom card available for business use
+  BUILD_IN = 'buildIn', // Platform built-in card
+  LOW_CODE = 'lowCode', // Low-code card
+}
+
+interface ChatCardObject<T = any> {
+  cardCode: string; // Unique identifier for the card
+  cardType: ChatCardType; // Card type
+  cardData: T; // Data carried by the card
+}
+```
+
+Register a custom card:
+
+```tsx
+import {
+  ChatCardObject,
+  ChatCardType,
+  defaultRenderOptions,
+  useTileProps,
+  useSendAction,
+} from '@ray-js/t-agent-ui-ray';
+import { View, Text, Button } from '@ray-js/ray';
+
+function MyCard(props: { card: ChatCardObject }) {
+  // If you need to access attributes like agent, message, tile, emitEvent, use useTileProps
+  const { message, agent, tile, emitEvent } = useTileProps();
+
+  // If you need to send a TTTAction, use useSendAction
+  const sendAction = useSendAction();
+
+  return (
+    <View>
+      <Text>My Card</Text>
+      <Button
+        onClick={() =>
+          sendAction({ type: 'sendMessage', blocks: [{ type: 'text', text: 'hello' }] })
+        }
+      >
+        Fill text into input
+      </Button>
+    </View>
+  );
+}
+
+const renderOptions = {
+  ...defaultRenderOptions,
+  customCardMap: {
+    myCard: MyCard,
+  },
+};
+```
+
+### Customize Block
+
+In `TextTile`, the markdown renderer supports custom blocks. You can register and render custom blocks through `customBlockTypes` and `renderCustomBlockAs`.
+
+The data structure of the block is as follows:
+
+```tsx
+export interface MarkdownBlock {
+  id: string;
+  type: string;
+  children: string; // Content in the block
+}
+```
+
+Register a custom block:
+
+```tsx
+import { defaultRenderOptions, MarkdownBlock } from '@ray-js/t-agent-ui-ray';
+import { View, Text } from '@ray-js/ray';
+
+const renderOptions = {
+  ...defaultRenderOptions,
+  customBlockTypes: ['my-block'],
+  renderCustomBlockAs: (block: MarkdownBlock) => {
+    if (block.type === 'my-block') {
+      return (
+        <View>
+          <View>This is My Block</View>
+          <View>{block.children}</View>
+        </View>
+      );
+    }
+    return defaultRenderOptions.renderCustomBlockAs(block);
+  },
+};
+```
+
+If a markdown text containing a `fence` of type `my-block` is sent by an AI, since we registered `my-block` above, this `fence` can be treated as a custom block.
+
+````markdown
+This is my custom block!
+
+```my-block
+Hello, world!
+```
+
+The following fence is not registered, and will not be rendered as a custom block, but just as a regular code block.
+
+```javascript
+console.log('Hello, world!');
+```
+````
+
+The rendered result is as follows:
+
+```
+This is my custom block!
+This is My Block
+Hello, world!
+The following fence is not registered, and will not be rendered as a custom block, but just as a regular code block.
+----------------------------------
+| console.log('Hello, world!');  |
+----------------------------------
+```
+
+### getStaticResourceBizType
+
+If your static resources need to include `bizType`, it can be obtained via `getStaticResourceBizType`.
+
+```tsx
+import { defaultRenderOptions } from '@ray-js/t-agent-ui-ray';
+
+const renderOptions = {
+  ...defaultRenderOptions,
+  getStaticResourceBizType: (src: string, scene: string) => 'bizType',
+};
+```
+
+For different scenarios, you may require different `bizType`. You can return different `bizType` based on `src` and `scene`.
+
+The built-in `scene` includes the following:
+
+- `image:view` Image viewing
+- `image:upload` Image uploading
+- `video:view` Video viewing
+- `video:upload` Video uploading
+- `videoThumb:view` Video thumbnail viewing
+- `videoThumb:upload` Video thumbnail uploading