@mcp-b/embedded-agent 0.0.7 → 0.0.8-beta.0

package/README.md

@@ -2,286 +2,330 @@
 
 React components for embedding an AI agent UI with MCP (Model Context Protocol) tool support and voice mode.
 
-## Architecture Overview
+## Table of Contents
+
+- [Quick Start](#quick-start)
+- [Architecture Overview](#architecture-overview)
+- [Core API](#core-api)
+- [Widgets](#widgets)
+- [Voice Mode](#voice-mode)
+- [MCP Tools](#mcp-tools)
+- [Legacy API](#legacy-api)
+- [Development](#development)
+- [Design Documents](#design-documents)
 
-The package follows a **core/widgets** architecture:
+## Quick Start
 
-- **Core** (`src/core/`): Data layer - providers and hooks, no UI
-- **Widgets** (`src/widgets/`): UI layer - independent components that consume core
+### Option 1: Web Component (Easiest)
 
-```
-src/
-├── core/                 # DATA LAYER
-│   ├── providers/
-│   │   ├── AgentProvider.tsx    # Unified provider (START HERE)
-│   │   └── index.ts
-│   ├── hooks/
-│   │   ├── useAgent.ts          # Main facade hook (START HERE)
-│   │   └── index.ts
-│   └── index.ts
-│
-├── widgets/              # UI LAYER
-│   ├── pill/                    # Compact floating UI
-│   │   └── index.ts             # Exports AgentPill, PillContainer, etc.
-│   ├── modal/                   # Full chat modal
-│   │   └── index.ts             # Exports AssistantModal, Thread, etc.
-│   ├── shared/                  # Shared components
-│   │   └── index.ts             # Exports ActionList, SummaryBlock, etc.
-│   └── index.ts
-│
-├── components/           # IMPLEMENTATION (internal)
-│   ├── pill/                    # Pill component implementations
-│   │   ├── AgentPill.tsx        # Main pill component
-│   │   ├── PillContainer.tsx    # Morphing container
-│   │   ├── PillVoice.tsx        # Voice mode UI
-│   │   ├── ActionList.tsx       # Action display
-│   │   └── ...
-│   └── ...                      # Other component implementations
-│
-├── hooks/                # INDIVIDUAL HOOKS (internal)
-│   ├── useActions.ts            # Derives actions from tool calls
-│   ├── useVoiceActions.ts       # Voice mode action tracking
-│   ├── useVoiceSummary.ts       # Voice session summary
-│   ├── useVoiceMode.ts          # Voice mode state machine
-│   └── ...
-│
-├── providers/            # INDIVIDUAL PROVIDERS (internal)
-│   ├── MCPToolsProvider.tsx     # MCP tool registry
-│   ├── VoiceModeProvider.tsx    # Voice mode context
-│   └── VoiceMCPBridge.tsx       # Connects MCP tools to voice
-│
-├── services/             # EXTERNAL SERVICES
-│   └── realtime/                # OpenAI Realtime API via WebRTC
-│       ├── openai-realtime-service.ts
-│       ├── webrtc-manager.ts
-│       └── ...
-│
-├── lib/                  # UTILITIES
-│   ├── constants.ts             # Magic numbers live here
-│   ├── utils.ts                 # cn(), etc.
-│   └── ...
-│
-└── index.ts              # PUBLIC EXPORTS
+```html
+<script src="@mcp-b/embedded-agent/standalone"></script>
+<webmcp-agent
+  app-id="your-app-id"
+  api-base="https://your-worker.workers.dev"
+  view-mode="pill"
+></webmcp-agent>
 ```
 
-## Quick Start
+### Option 2: React Component
+
+```tsx
+import { EmbeddedAgent } from '@mcp-b/embedded-agent'
+import '@mcp-b/embedded-agent/styles'
+
+function App() {
+  return (
+    <EmbeddedAgent
+      appId="your-app-id"
+      apiBase="https://your-worker.workers.dev"
+      viewMode="pill"
+    />
+  )
+}
+```
 
-### Recommended: AgentProvider + Widget
+### Option 3: Composable (Recommended for Custom UIs)
 
 ```tsx
-import { AgentPill, AgentProvider } from '@mcp-b/embedded-agent'
+import { AgentProvider, AgentPill } from '@mcp-b/embedded-agent'
+import '@mcp-b/embedded-agent/styles'
 
 function App() {
-	return (
-		<AgentProvider apiBase="https://your-worker.workers.dev">
-			<AgentPill position="bottom-center" showVoiceButton />
-		</AgentProvider>
-	)
+  return (
+    <AgentProvider apiBase="https://your-worker.workers.dev">
+      <AgentPill position="bottom-center" showVoiceButton />
+    </AgentProvider>
+  )
 }
 ```
 
-### Custom UI with useAgent
+### Option 4: Fully Custom UI
 
 ```tsx
 import { AgentProvider, useAgent } from '@mcp-b/embedded-agent'
 
 function CustomUI() {
-	const agent = useAgent()
-
-	return (
-		<div>
-			{agent.isRunning && <p>Processing...</p>}
-			{agent.activeActions.map((action) => (
-				<div key={action.id}>{action.label}</div>
-			))}
-			{agent.voice?.isActive && <button onClick={agent.voice.stop}>End Voice</button>}
-		</div>
-	)
+  const agent = useAgent()
+
+  return (
+    <div>
+      {agent.isRunning && <p>Processing...</p>}
+      {agent.activeActions.map((action) => (
+        <div key={action.id}>{action.label}</div>
+      ))}
+      {agent.voice?.isActive && (
+        <button onClick={agent.voice.stop}>End Voice</button>
+      )}
+    </div>
+  )
 }
 
 function App() {
-	return (
-		<AgentProvider apiBase="https://...">
-			<CustomUI />
-		</AgentProvider>
-	)
+  return (
+    <AgentProvider apiBase="https://...">
+      <CustomUI />
+    </AgentProvider>
+  )
 }
 ```
 
-## Core API
+## Architecture Overview
 
-### AgentProvider
+The package follows a **core/widgets** architecture with strict separation between data and UI layers:
 
-**File:** `src/core/providers/AgentProvider.tsx`
+```
+src/
+├── core/                 # DATA LAYER (no UI)
+│   ├── providers/
+│   │   └── AgentProvider.tsx    # Unified provider
+│   └── hooks/
+│       └── useAgent.ts          # Main facade hook
+│
+├── widgets/              # UI LAYER
+│   ├── pill/             # Compact floating UI
+│   ├── modal/            # Full chat modal
+│   └── shared/           # Shared components
+│
+├── providers/            # Individual providers
+│   ├── MCPToolsProvider.tsx     # MCP tool registry
+│   ├── VoiceModeProvider.tsx    # Voice mode context
+│   └── VoiceMCPBridge.tsx       # MCP-to-voice bridge
+│
+├── hooks/                # Individual hooks
+│   ├── useActions.ts            # Derive actions from tool calls
+│   ├── useVoiceMode.ts          # Voice session state
+│   └── ...
+│
+├── services/realtime/    # OpenAI Realtime API
+│   ├── openai-realtime-service.ts
+│   ├── webrtc-manager.ts
+│   └── ...
+│
+└── components/           # Internal implementations
+```
 
-Unified provider that sets up:
+For detailed architecture information, see the [Design Documents](#design-documents).
 
-- MCP tool registry (`MCPToolsProvider`)
-- Voice mode bridge (`VoiceMCPBridge`)
-- Chat runtime (`AssistantRuntimeProvider` from @assistant-ui/react)
+## Core API
+
+### AgentProvider
+
+Unified provider that sets up all agent infrastructure.
 
 ```tsx
 interface AgentProviderProps {
-	children: ReactNode
-	apiBase?: string // Backend URL
-	tokenEndpoint?: string // Voice token endpoint (defaults to {apiBase}/api/realtime/session)
-	autoConnectLocal?: boolean // Auto-connect to local MCP source (default: true)
-	onToolsChange?: (tools: ToolWithSource[]) => void
-	onVoiceError?: (error: string) => void
-	onVoiceConnect?: () => void
-	onVoiceDisconnect?: (duration: number) => void
+  children: ReactNode
+  apiBase?: string                              // Backend URL
+  tokenEndpoint?: string                        // Voice token endpoint (auto-computed)
+  autoConnectLocal?: boolean                    // Auto-connect to local MCP (default: true)
+  onToolsChange?: (tools: ToolWithSource[]) => void
+  onVoiceError?: (error: string) => void
+  onVoiceConnect?: () => void
+  onVoiceDisconnect?: (duration: number) => void
 }
 ```
 
-### useAgent Hook
+**What it provides:**
+- MCP tool registry (`MCPToolsProvider`)
+- Voice mode with MCP tools (`VoiceMCPBridge`)
+- Chat runtime (`AssistantRuntimeProvider` from @assistant-ui/react)
 
-**File:** `src/core/hooks/useAgent.ts`
+### useAgent Hook
 
-Main facade hook - combines all agent capabilities into one interface.
+Main facade hook combining all agent capabilities.
 
 ```tsx
-interface AgentState {
-	// Thread
-	messages: ReadonlyArray<ThreadMessage>
-	isRunning: boolean
-	hasMessages: boolean
-
-	// Actions (from tool calls)
-	actions: Action[] // All text mode actions
-	currentAction: Action | null // Currently running
-	recentActions: Action[] // Last 3 completed
-
-	// Voice mode actions
-	voiceActions: Action[] // Actions from voice mode
-	activeActions: Action[] // Voice or text depending on mode
-
-	// Summary
-	summary: string | null // Latest summary text
-	voiceSummary: VoiceSummary | null
-
-	// Voice controls (null if not configured)
-	voice: {
-		isActive: boolean
-		isConnecting: boolean
-		isError: boolean
-		isMuted: boolean
-		error?: string
-		audioLevel?: AudioLevelData
-		transcript?: TranscriptData
-		start: () => Promise<void>
-		stop: () => void
-		toggleMute: (muted?: boolean) => void
-		sendMessage: (text: string) => void
-	} | null
-
-	// MCP tools (null if not in context)
-	tools: {
-		list: ToolWithSource[]
-		call: (name: string, args: Record<string, unknown>) => Promise<CallToolResult>
-	} | null
-
-	// Derived
-	isVoiceActive: boolean
-}
+const agent = useAgent()
+
+// Thread state
+agent.messages        // All messages
+agent.isRunning       // Is agent processing?
+agent.hasMessages     // Has any messages?
+
+// Actions (from tool calls)
+agent.actions         // All text mode actions
+agent.currentAction   // Currently running action
+agent.recentActions   // Last 3 completed
+agent.voiceActions    // Voice mode actions
+agent.activeActions   // Voice or text (auto-switches)
+
+// Summaries
+agent.summary         // Latest text summary
+agent.voiceSummary    // Voice session summary
+
+// Voice controls (null if not configured)
+agent.voice?.isActive
+agent.voice?.isConnecting
+agent.voice?.isMuted
+agent.voice?.start()
+agent.voice?.stop()
+agent.voice?.toggleMute()
+agent.voice?.sendMessage(text)
+agent.voice?.audioLevel    // Real-time levels
+agent.voice?.transcript    // Current transcript
+
+// MCP tools (null if not in context)
+agent.tools?.list
+agent.tools?.call(name, args)
+
+// Derived
+agent.isVoiceActive
 ```
 
 ## Widgets
 
 ### AgentPill
 
-**File:** `src/components/pill/AgentPill.tsx`
-
-Compact, morphing floating UI. Shows actions, voice mode, and composer.
+Compact, morphing floating UI with action-first design.
 
 ```tsx
-interface AgentPillProps {
-	position?: 'bottom-center' | 'bottom-right'
-	onOpenHistory?: () => void
-	showVoiceButton?: boolean
-	autoCollapse?: boolean // Auto-collapse after 30s inactivity
-	className?: string
-}
+import { AgentPill } from '@mcp-b/embedded-agent'
+
+<AgentPill
+  position="bottom-center"     // or "bottom-right"
+  showVoiceButton={true}
+  autoCollapse={true}          // Collapse after 30s inactivity
+  onOpenHistory={() => {}}
+  className="custom-class"
+/>
 ```
 
-### AssistantModal
+**States:**
+- `collapsed` - Tiny bar (ambient)
+- `hovered` - Shows keyboard hint
+- `composing` - User typing
+- `active` - Agent working
+- `expanded` - Shows summary/results
 
-**File:** `src/components/assistant-modal.tsx`
+### AssistantModal
 
 Traditional chat modal with full message thread.
 
 ```tsx
-// No props - uses context from AgentProvider
-<AssistantModal />
+import { AssistantModal } from '@mcp-b/embedded-agent'
+
+<AssistantModal />  // Uses context from AgentProvider
 ```
 
-## Key Implementation Details
+### Shared Components
+
+For building custom UIs:
+
+```tsx
+import {
+  ActionList,
+  CurrentActivity,
+  SummaryBlock,
+  LiveWaveform,
+  VoiceIndicator,
+  ToolStatusBorder,
+} from '@mcp-b/embedded-agent'
+```
 
-### Action Tracking
+## Voice Mode
 
-**Files:** `src/hooks/useActions.ts`, `src/hooks/useVoiceActions.ts`
+Voice mode uses WebRTC to connect to OpenAI's Realtime API with MCP tool integration.
 
-Actions are derived from tool calls in assistant messages. The `useActions` hook:
+### How It Works
 
-1. Subscribes to thread messages via `useThread`
-2. Extracts tool calls from assistant messages
-3. Maps tool calls to Action objects with status (running/success/error)
+1. **VoiceMCPBridge** converts MCP tools to OpenAI's function format
+2. **WebRTCManager** establishes peer connection with OpenAI
+3. **ToolManager** executes tool calls through MCP when voice requests them
+4. **AudioAnalyzer** provides real-time audio levels for visualization
+
+### Requirements
+
+- Backend endpoint at `{apiBase}/api/realtime/session` that returns ephemeral tokens
+- Browser with WebRTC and getUserMedia support
+
+### Checking Support
+
+```tsx
+const agent = useAgent()
 
-Voice actions work similarly but track tool calls from the realtime voice API.
+if (agent.voice) {
+  // Voice is available
+  await agent.voice.start()
+}
+```
 
-### Voice Mode
+## MCP Tools
 
-**Files:**
+### How Tools Are Registered
 
-- `src/hooks/useVoiceMode.ts` - State machine for voice sessions
-- `src/services/realtime/` - OpenAI Realtime API integration
-- `src/providers/VoiceMCPBridge.tsx` - Connects MCP tools to voice mode
+1. `MCPToolsProvider` manages connections to MCP sources
+2. `MCPToolRegistry` bridges MCP tools to the chat runtime
+3. Tools are tagged with `_sourceId` for auto-routing
 
-Voice mode uses WebRTC to connect to OpenAI's Realtime API. The VoiceMCPBridge:
+### Multiple Sources
 
-1. Watches MCP tools from MCPToolsProvider
-2. Converts them to RegisteredTool format
-3. Provides tool executor that calls through MCP
+```tsx
+const { addSource, tools, callTool } = useMCPTools()
 
-### MCP Tools
+// Add remote server
+await addSource('remote', {
+  type: 'http',
+  url: 'http://localhost:8888/mcp'
+})
 
-**File:** `src/providers/MCPToolsProvider.tsx`
+// Tools from all sources are aggregated
+console.log(tools)  // [...localTools, ...remoteTools]
 
-Manages MCP tool sources (local tab, remote servers). Key exports:
+// Calls auto-route to correct source
+await callTool('some_tool', { arg: 'value' })
+```
 
-- `useMCPTools()` - Required context hook
-- `useOptionalMCPTools()` - Returns null if not in provider
+### Prompts
 
-### Constants
+MCP prompts are also supported:
 
-**File:** `src/lib/constants.ts`
+```tsx
+const { prompts, getPrompt } = useMCPTools()
 
-All magic numbers are centralized here:
+// List available prompts
+console.log(prompts)
 
-- `VOICE_ACTIONS_RETENTION_MS` (3000ms) - How long voice actions persist
-- `VOICE_SUMMARY_RETENTION_MS` (30000ms) - How long voice summary persists
-- `TOOL_CALL_DISPLAY_DURATION_MS` (2000ms) - Tool status display time
-- etc.
+// Get expanded prompt content
+const result = await getPrompt('my-prompt', { arg: 'value' })
+```
 
 ## Legacy API
 
-The `EmbeddedAgent` component is preserved for backward compatibility but wraps the new architecture internally.
+The `EmbeddedAgent` component wraps the new architecture for backward compatibility:
 
 ```tsx
-// Legacy (still works)
 import { EmbeddedAgent } from '@mcp-b/embedded-agent'
 
 <EmbeddedAgent
   appId="your-app"
   apiBase="https://..."
-  viewMode="pill"
+  viewMode="pill"              // or "modal"
+  autoConnectLocal={true}
+  onToolsChange={(tools) => {}}
+  onVoiceError={(error) => {}}
+  onVoiceConnect={() => {}}
+  onVoiceDisconnect={(duration) => {}}
 />
-
-// New (recommended)
-import { AgentProvider, AgentPill } from '@mcp-b/embedded-agent'
-
-<AgentProvider apiBase="https://...">
-  <AgentPill />
-</AgentProvider>
 ```
 
 ## Development
@@ -303,28 +347,77 @@ pnpm test
 pnpm play
 ```
 
+## Design Documents
+
+For detailed architecture and implementation information:
+
+- **[High-Level Design](./docs/HIGH_LEVEL_DESIGN.md)** - Architecture overview, data flow, design principles
+- **[Low-Level Design](./docs/LOW_LEVEL_DESIGN.md)** - Implementation details, file organization, patterns
+
 ## Exports Reference
 
-### From `@mcp-b/embedded-agent`
+### Main Entry (`@mcp-b/embedded-agent`)
 
-**Core:**
+```typescript
+// Main component
+export { EmbeddedAgent, registerWebMCPAgent }
+export type { EmbeddedAgentProps, AgentViewMode }
+```
+
+### Core (`@mcp-b/embedded-agent/core`) *
+
+```typescript
+// Provider
+export { AgentProvider }
+export type { AgentProviderProps }
 
-- `AgentProvider`, `AgentProviderProps`
-- `useAgent`, `AgentState`, `AgentVoice`, `AgentTools`
-- `useActions`, `useCurrentAction`, `useRecentActions`, `Action`
-- `useVoiceActions`, `useVoiceSummary`, `VoiceSummary`
-- `useMCPTools`, `useOptionalMCPTools`, `MCPToolsContextValue`
+// Main hook
+export { useAgent }
+export type { AgentState, AgentVoice, AgentTools }
 
-**Widgets:**
+// Action hooks
+export { useActions, useCurrentAction, useRecentActions, humanizeToolName }
+export type { Action }
 
-- `AgentPill`, `AgentPillProps`, `PillPosition`
-- `AssistantModal`
-- `ActionList`, `SummaryBlock`, `CurrentActivity`
-- `PillContainer`, `PillComposer`, `PillVoice`
-- `LiveWaveform`, `VoiceIndicator`, `ToolStatusBorder`
+// Voice hooks
+export { useVoiceActions, useVoiceSummary, useHasVoiceSummary, useVoiceMode }
+export type { VoiceSummary, UseVoiceModeOptions, UseVoiceModeReturn }
 
-**Legacy:**
+// Prompt hooks
+export { usePrompts }
+export type { UsePromptsResult }
 
-- `EmbeddedAgent`, `EmbeddedAgentProps`
+// Provider access
+export { useMCPTools, useOptionalMCPTools, useOptionalVoiceModeContext }
+export type { MCPToolsContextValue, VoiceModeContextValue }
+
+// Constants
+export {
+  VOICE_ACTIONS_RETENTION_MS,
+  VOICE_SUMMARY_RETENTION_MS,
+  TOOL_CALL_DISPLAY_DURATION_MS,
+  TOOL_CALL_ERROR_DISPLAY_DURATION_MS,
+}
+```
+
+### Widgets (`@mcp-b/embedded-agent/widgets`) *
+
+```typescript
+// Pill widget
+export { AgentPill, PillContainer, PillComposer, PillVoice, HistorySidebar }
+export { useVoiceBorderStatus, PromptBadge, PromptSuggestions }
+export type { AgentPillProps, PillPosition, PillMode, Conversation }
+export type { PromptBadgeProps, PromptSuggestionsProps }
+
+// Modal widget
+export { AssistantModal, Thread, Composer, ThreadWithVoice }
+export { UserMessage, AssistantMessage }
+
+// Shared components
+export { ActionList, ActionSummary, CurrentActivity, ActionStatusIcon }
+export { IdleIndicator, SummaryBlock, WelcomeMessage, ThreadContent }
+export { useLatestSummary, LiveWaveform, VoiceIndicator }
+export { ToolStatusBorder, PillMarkdown }
+```
 
-See `src/index.ts` for the complete export list.
+\* *These are internal module paths available for advanced usage. The main entry point re-exports commonly used items.*