@telnyx/ai-agent-lib 0.4.5 → 0.5.0-beta.0

package/README.md

@@ -184,6 +184,9 @@ Returns the `TelnyxAIAgent` instance for direct API access.
 - `startConversation(options?)` - Start a new conversation with optional caller metadata and headers
 - `endConversation()` - End the current conversation
 - `sendConversationMessage(message: string)` - Send a text message during an active conversation
+- `registerClientTool(name, handler)` - Register a client-side tool the assistant can invoke (see [Client-Side Tools](#client-side-tools))
+- `unregisterClientTool(name)` - Remove a registered client-side tool
+- `getClientTools()` - List registered client-side tool names
 - `setRemoteStream(stream: MediaStream)` - Manually set the remote audio stream for monitoring (useful when `call.remoteStream` is not available)
 - `transcript` - Get current transcript array
 
@@ -342,6 +345,121 @@ agent.on('conversation.update', (notification) => {
 - The agent will receive and process text messages just like spoken input
 - Text messages may appear in the transcript depending on the agent configuration
 
+### Client-Side Tools
+
+Client-side tools let the AI assistant call functions that run **in the
+browser / client** during a conversation. The assistant decides when to call a
+tool, the library executes your handler, and the return value is sent back to
+the assistant so it can continue the conversation. This implements the ACA
+`client_side_tool` execution path over the Voice SDK Proxy (VSP) WebSocket.
+
+Typical uses: looking up data the page already has, reading client-side state,
+triggering UI actions, or calling an API the browser is authenticated for.
+
+#### Registering tools
+
+Register tools up front via the constructor, or at any time with
+`registerClientTool`:
+
+```typescript
+import { TelnyxAIAgent } from '@telnyx/ai-agent-lib';
+
+const agent = new TelnyxAIAgent({
+  agentId: 'your-agent-id',
+  // Register at construction time:
+  clientTools: {
+    lookup_order: async (args, context) => {
+      // args is the parsed `arguments` object from the assistant
+      const order = await fetchOrder(args.orderId);
+      return { status: 'found', orderId: order.id, total: order.total };
+    },
+  },
+  // Optional: per-tool execution timeout (default 30000ms)
+  clientToolTimeoutMs: 15000,
+});
+
+// ...or register/replace later:
+agent.registerClientTool('get_cart', () => ({ items: cart.items.length }));
+
+// Remove a tool:
+agent.unregisterClientTool('get_cart');
+
+// Inspect registered tools:
+agent.getClientTools(); // ['lookup_order']
+```
+
+In React you can do the same via `useClient()`:
+
+```tsx
+function ToolRegistration() {
+  const client = useClient();
+  useEffect(() => {
+    client.registerClientTool('lookup_order', async (args) => {
+      return { status: 'found', orderId: args.orderId };
+    });
+    return () => client.unregisterClientTool('lookup_order');
+  }, [client]);
+  return null;
+}
+```
+
+#### Handler contract
+
+```typescript
+type ClientSideToolHandler = (
+  args: unknown,                  // parsed JSON arguments (or undefined if empty)
+  context: ClientSideToolContext, // { callId, toolName, rawArguments }
+) => unknown | Promise<unknown>;
+```
+
+- The return value is serialized and sent back as the tool output. Strings are
+  sent verbatim; anything else is `JSON.stringify`-ed.
+- Handlers may be async. They are run with a timeout (`clientToolTimeoutMs`).
+- The tool name and `call_id` must round-trip back to the assistant — the
+  library handles that for you.
+
+#### Safety & robustness
+
+The library always returns a result to the assistant so the conversation never
+hangs, and it never executes a tool twice for the same call:
+
+- **Unknown tool** → safe error output `{ "error": "unknown_tool" }`.
+- **Invalid JSON arguments** → `{ "error": "invalid_arguments" }`.
+- **Handler throws / rejects** → `{ "error": "handler_error" }`.
+- **Handler exceeds timeout** → `{ "error": "timeout" }`.
+- **Disconnected before output can be sent** → `{ "error": "shutdown" }` event,
+  output dropped (ACA has already timed out its waiter).
+- **Duplicate `call_id`** while a tool is in-flight → ignored (no double run).
+- On disconnect, in-flight bookkeeping is cleared so reconnects start clean.
+
+> Tool arguments and outputs are **never logged** — they may contain customer
+> data. Only safe correlation fields (`call_id`, tool name) appear in debug logs.
+
+#### Observing tool lifecycle
+
+Three events let you observe tool execution without touching payloads:
+
+```typescript
+agent.on('client.tool.invoked', ({ callId, toolName }) => {
+  console.log(`tool ${toolName} invoked (${callId})`);
+});
+agent.on('client.tool.completed', ({ callId, toolName, isError }) => {
+  console.log(`tool ${toolName} completed, isError=${isError}`);
+});
+agent.on('client.tool.error', ({ callId, toolName, reason }) => {
+  console.warn(`tool ${toolName} failed: ${reason}`);
+});
+```
+
+#### EVA adapter usage
+
+The EVA adapter (`telnyx-voice-ai-eva-adapter`) is expected to consume this same
+public API — registering its EVA-specific tools through `registerClientTool`
+(or the `clientTools` constructor option) rather than reimplementing the
+PR-531 protocol. The core client-side tool support lives here and in
+`@telnyx/webrtc`, so it works for any VSP-connected Voice SDK client
+independently of the EVA adapter.
+
 ### Latency Measurement
 
 The library automatically measures round-trip latency using client-side Voice Activity Detection (VAD). This provides accurate timing from when the user stops speaking until the agent's response audio begins.
@@ -498,6 +616,9 @@ The `TelnyxAIAgent` class extends `EventEmitter` and provides a comprehensive se
 | `conversation.update` | `INotification` | Emitted when conversation state changes |
 | `conversation.agent.state` | `AgentStateData` | Emitted when agent state changes (listening/speaking/thinking) |
 | `agent.audio.mute` | `boolean` | Emitted when agent audio is muted or unmuted |
+| `client.tool.invoked` | `{ callId, toolName }` | Emitted when a client-side tool starts executing |
+| `client.tool.completed` | `{ callId, toolName, isError }` | Emitted when a client-side tool output is sent back to the assistant |
+| `client.tool.error` | `{ callId, toolName, reason }` | Emitted when a client-side tool fails and a safe error output is sent |
 
 ### Data Types
 

package/dist/client-tools.d.ts

@@ -0,0 +1,112 @@
+import type { Call } from "@telnyx/webrtc";
+import type { ClientSideToolErrorReason, ClientSideToolHandler, ClientSideToolMap } from "./types";
+/** Default per-tool execution timeout (ms). */
+export declare const DEFAULT_CLIENT_TOOL_TIMEOUT_MS = 30000;
+type ToolLifecycleEvents = {
+    onInvoked: (info: {
+        callId: string;
+        toolName: string;
+    }) => void;
+    onCompleted: (info: {
+        callId: string;
+        toolName: string;
+        isError: boolean;
+    }) => void;
+    onError: (info: {
+        callId: string;
+        toolName: string;
+        reason: ClientSideToolErrorReason;
+    }) => void;
+};
+type ClientToolManagerOptions = {
+    /**
+     * Returns the currently active {@link Call}, or `null` when no call is
+     * connected. The manager uses it to send `function_call_output` back to ACA.
+     */
+    getActiveCall: () => Call | null;
+    /** Per-tool execution timeout in milliseconds. */
+    timeoutMs?: number;
+    /** Lifecycle event hooks (wired to the TelnyxAIAgent EventEmitter). */
+    events: ToolLifecycleEvents;
+};
+/**
+ * Manages registration and PR-531 execution of client-side tools.
+ *
+ * Responsibilities:
+ * - hold the tool registry
+ * - subscribe to inbound `function_call` items (via {@link handleEvent})
+ * - parse `arguments`, run the matching handler with a timeout
+ * - de-duplicate concurrent AND already-completed invocations sharing a
+ *   `call_id` (idempotency across VSP/ACA re-delivery)
+ * - always send a `function_call_output` (result or safe error) back to ACA
+ * - clean up in-flight/completed bookkeeping on disconnect, and drop late
+ *   handler resolutions that belong to a previous session generation
+ *
+ * Tool arguments and outputs are NEVER logged (they may contain customer data).
+ */
+export declare class ClientToolManager {
+    private readonly registry;
+    private readonly inFlight;
+    /**
+     * `call_id`s that have already produced a `function_call_output` in the
+     * current session. Retained (until {@link reset}) so a re-delivered
+     * `function_call` for an already-handled id is ignored instead of re-running
+     * a handler with side effects.
+     */
+    private readonly completed;
+    /**
+     * Monotonic session generation. Bumped on every {@link reset}/{@link destroy}
+     * so a handler that resolves after a disconnect can detect that its session
+     * is gone and avoid sending a stale output against a reconnected call.
+     */
+    private generation;
+    private readonly getActiveCall;
+    private readonly timeoutMs;
+    private readonly events;
+    constructor(options: ClientToolManagerOptions);
+    /** Registers (or replaces) a handler for `name`. */
+    register(name: string, handler: ClientSideToolHandler): void;
+    /** Bulk-registers handlers from a map (used by the constructor option). */
+    registerAll(tools: ClientSideToolMap): void;
+    /** Removes a handler. Returns true when a handler was removed. */
+    unregister(name: string): boolean;
+    /** True when a handler is registered for `name`. */
+    has(name: string): boolean;
+    /** Registered tool names. */
+    list(): string[];
+    /**
+     * Inbound `telnyx.ai.conversation` event handler. Ignores anything that is
+     * not a PR-531 `function_call` so transcript / state messages are untouched.
+     */
+    handleEvent: (event: unknown) => void;
+    /**
+     * Drops all in-flight/completed bookkeeping and advances the session
+     * generation. Called on disconnect so a later reconnect starts clean, and so
+     * a handler still running from the old session drops its (now stale) output
+     * instead of sending it against the new call.
+     */
+    reset(): void;
+    /** Clears registry and all session state. Called on full teardown. */
+    destroy(): void;
+    /**
+     * Records a terminal `call_id` for idempotency, but only if the session has
+     * not been reset since the invocation started — otherwise the id belongs to a
+     * dead generation and must not leak into the next session's dedupe set.
+     */
+    private markCompleted;
+    private execute;
+    private withTimeout;
+    /**
+     * Sends a `function_call_output` back to ACA via the active call. When the
+     * session is gone (disconnected / hung up) the output is dropped with a
+     * shutdown error event rather than queued — ACA has its own waiter timeout
+     * and would reject a stale late result anyway.
+     *
+     * If the session generation has advanced since the invocation started (a
+     * reset/disconnect happened while the handler was running), the result is
+     * considered stale and is dropped — sending it would leak the old result
+     * into a freshly reconnected call/session.
+     */
+    private sendOutput;
+}
+export {};

package/dist/client.d.ts

@@ -1,6 +1,6 @@
 import { Call } from "@telnyx/webrtc";
 import EventEmitter from "eventemitter3";
-import type { AIAgentEvents, TranscriptItem, VADOptions } from "./types";
+import type { AIAgentEvents, ClientSideToolHandler, ClientSideToolMap, TranscriptItem, VADOptions } from "./types";
 export type TelnyxAIAgentConstructorParams = {
     agentId: string;
     versionId?: string;
@@ -35,6 +35,27 @@ export type TelnyxAIAgentConstructorParams = {
      * @default true
      */
     skipLastVoiceSdkId?: boolean;
+    /**
+     * Client-side tools the AI assistant can invoke during a conversation
+     * (PR-531 `client_side_tool` execution). Each entry maps a tool name to a
+     * handler whose return value is sent back to the assistant as the tool
+     * output. Tools can also be added later with {@link TelnyxAIAgent.registerClientTool}.
+     *
+     * @example
+     * new TelnyxAIAgent({
+     *   agentId,
+     *   clientTools: {
+     *     lookup_order: async (args) => ({ status: "found", orderId: args.orderId }),
+     *   },
+     * });
+     */
+    clientTools?: ClientSideToolMap;
+    /**
+     * Per-tool execution timeout in milliseconds for client-side tools. After
+     * this elapses the handler result is abandoned and a safe timeout error is
+     * returned to the assistant. Defaults to 30000ms.
+     */
+    clientToolTimeoutMs?: number;
 };
 export declare class TelnyxAIAgent extends EventEmitter<AIAgentEvents> {
     private telnyxRTC;
@@ -48,6 +69,7 @@ export declare class TelnyxAIAgent extends EventEmitter<AIAgentEvents> {
     conversationId?: string;
     debug: boolean;
     private audioStreamMonitor;
+    private clientTools;
     activeCall: Call | null;
     /**
      * When true, the client operates in chat-only mode (no microphone).
@@ -75,6 +97,30 @@ export declare class TelnyxAIAgent extends EventEmitter<AIAgentEvents> {
      */
     callReportId: string | null;
     constructor(params: TelnyxAIAgentConstructorParams);
+    /**
+     * Registers (or replaces) a client-side tool the AI assistant can invoke.
+     *
+     * The handler receives the parsed `arguments` object and an invocation
+     * context, and its return value is serialized and sent back to the assistant.
+     * Throwing / rejecting is caught and reported to the assistant as a safe
+     * error output so the conversation continues.
+     *
+     * @example
+     * agent.registerClientTool("lookup_order", async (args) => {
+     *   return { status: "found", orderId: args.orderId };
+     * });
+     */
+    registerClientTool(name: string, handler: ClientSideToolHandler): void;
+    /**
+     * Removes a previously-registered client-side tool.
+     *
+     * @returns true if a handler was removed, false if none was registered.
+     */
+    unregisterClientTool(name: string): boolean;
+    /**
+     * Returns the names of all currently-registered client-side tools.
+     */
+    getClientTools(): string[];
     /**
      * Connects to the Telnyx WebRTC service and establishes a session with the AI agent.
      *