@lumavox/client 0.5.0 → 0.6.0

package/README.md

@@ -105,6 +105,17 @@ A tool marked "ask first" sends `tool.confirm` and does not run:
 `confirmTool(callId, yes)` answers with the person's tap — a turn of
 theirs, queued as `send` is, which runs the tool first on a yes.
 
+The agent's app tools run in your app: `connect`'s `tools` gives a
+handler for each, by name. The conversation names them when it
+authenticates, runs one when the model calls it, and sends back what it
+answers — or what it threw:
+
+```ts
+const conversation = client.connect(sessionId, {}, {
+  tools: { open_page: async ({ page }) => ({ opened: page }) },
+});
+```
+
 ### Speaking
 
 ```ts

package/dist/conversation.d.ts

@@ -35,7 +35,7 @@
  * shows — a `text.delta`, say — and none for audio, which is passed
  * through, never copied.
  */
-import type { ErrorFrame, InterruptedFrame, KnowledgeFrame, ReadyFrame, ServerFrame, ServerFrames, TranscriptFrame, TurnDoneFrame } from "./events.js";
+import type { ErrorFrame, InterruptedFrame, KnowledgeFrame, ReadyFrame, ServerFrame, ServerFrames, ToolCallFrame, TranscriptFrame, TurnDoneFrame } from "./events.js";
 import type { RelationInfo } from "./gen/index.js";
 import { type AudioFrame, type Warning, type WebSocketConstructor } from "./socket.js";
 import type { Tokens } from "./token.js";
@@ -169,6 +169,12 @@ export interface ConversationHandlers {
     /** Every frame, the ones above included: what a newer edge sends that this SDK does not know. */
     onEvent?: (frame: ServerFrame) => void;
 }
+/**
+ * One of your app's tools: given the model's arguments, it answers any
+ * JSON (or nothing, `null`), maybe as a promise. A throw or a rejection is
+ * the call's `error`, which the model is told.
+ */
+export type ToolHandler = (args: Record<string, unknown>, call: ToolCallFrame) => unknown;
 export interface ConnectOptions {
     /**
      * Every turn's `speech`. By default `false` when there is no `onAudio`
@@ -178,6 +184,12 @@ export interface ConnectOptions {
     speech?: boolean;
     /** Ask every turn for thinking summaries (`onThought`). */
     thoughts?: boolean;
+    /**
+     * The app's own tools, by name: the agent is offered those of its tools
+     * of kind `app` named here, and each call runs its handler here. At most
+     * `APP_TOOLS_MAX` (50).
+     */
+    tools?: Record<string, ToolHandler>;
 }
 /** Per turn: overrides `ConnectOptions`. */
 export interface TurnOptions {
@@ -308,6 +320,15 @@ export declare class Conversation {
     private stopTimers;
     private frame;
     private ready;
+    /**
+     * One of the app's tools, called by the model: its handler run, and its
+     * answer — or what it threw — sent back as `tool.result` on the socket
+     * the call came on. A reconnect in between ended the call at the edge,
+     * so nothing is sent on the new socket. An action-only call is answered
+     * by nobody: its handler runs, and one that throws is rethrown on its
+     * own, as any handler's is.
+     */
+    private runTool;
     /** The end of a turn: answers whether it was a second for the same turn, which says nothing new. */
     private terminal;
     private error;

package/dist/conversation.js

@@ -1,5 +1,5 @@
 import { wireBytes } from "./pcm.js";
-import { KEEPALIVE_SECONDS, MAX_AUDIO_BYTES, MAX_MESSAGE_CHARS, RECORDING_RATES, } from "./protocol.gen.js";
+import { KEEPALIVE_SECONDS, MAX_AUDIO_BYTES, MAX_MESSAGE_CHARS, RECORDING_RATES, TOOL_ERROR_MAX_CHARS, } from "./protocol.gen.js";
 import { dial, isFinalClose, reconnectAfter, socketUrl, } from "./socket.js";
 /** No `ready` this long after the dial — a proxy that took the upgrade and says nothing — is a dropped connection. */
 export const READY_DEADLINE_MS = 15000;
@@ -315,6 +315,7 @@ export class Conversation {
                 url: socketUrl(this.setup.socketUrl, this.sessionId),
                 key,
                 sessionId: this.sessionId,
+                tools: Object.keys(this.setup.options.tools ?? {}),
                 WebSocket: this.setup.WebSocket,
                 onFrame: (frame) => this.frame(frame),
                 onAudio: (frame) => {
@@ -508,6 +509,9 @@ export class Conversation {
             case "tool.end":
                 this.update({ tool: null });
                 break;
+            case "tool.call":
+                this.runTool(frame);
+                break;
             case "knowledge":
                 this.update({
                     knowledge: [...(this.snap.turn?.knowledge ?? []), frame],
@@ -563,6 +567,48 @@ export class Conversation {
         }
         this.sync();
     }
+    /**
+     * One of the app's tools, called by the model: its handler run, and its
+     * answer — or what it threw — sent back as `tool.result` on the socket
+     * the call came on. A reconnect in between ended the call at the edge,
+     * so nothing is sent on the new socket. An action-only call is answered
+     * by nobody: its handler runs, and one that throws is rethrown on its
+     * own, as any handler's is.
+     */
+    runTool(frame) {
+        const handler = this.setup.options.tools?.[frame.name];
+        const socket = this.socket;
+        const answer = (fields) => {
+            if (frame.action_only || !socket || this.socket !== socket)
+                return;
+            const sent = { type: "tool.result", call_id: frame.call_id };
+            try {
+                socket.send({ ...sent, ...fields });
+            }
+            catch {
+                // swallowed: a result JSON cannot carry (a cycle, a BigInt), told
+                // to the model as the failure it is.
+                socket.send({ ...sent, error: "The tool's answer is not JSON." });
+            }
+        };
+        new Promise((resolve) => {
+            if (!handler)
+                throw new Error(`This app has no tool ${frame.name}.`);
+            resolve(handler(frame.arguments, frame));
+        }).then((result) => answer({ result: result === undefined ? null : result }), (error) => {
+            const said = error instanceof Error ? error.message : String(error);
+            answer({
+                error: [...(said || "The tool failed.")]
+                    .slice(0, TOOL_ERROR_MAX_CHARS)
+                    .join(""),
+            });
+            if (frame.action_only) {
+                setTimeout(() => {
+                    throw error;
+                });
+            }
+        });
+    }
     /** The end of a turn: answers whether it was a second for the same turn, which says nothing new. */
     terminal(frame) {
         if (frame.turn === this.finished)

package/dist/events.d.ts

@@ -102,6 +102,19 @@ export interface ToolEndFrame extends Frame<"tool.end">, InTurn {
     ok: boolean;
     ms: number;
 }
+/**
+ * The model called one of your app's tools (`ConnectOptions.tools`): the
+ * conversation runs its handler and answers `tool.result` within
+ * `timeout_ms`. An `action_only` call is not waited for — the reply goes
+ * on at once.
+ */
+export interface ToolCallFrame extends Frame<"tool.call">, InTurn {
+    call_id: string;
+    name: string;
+    arguments: Record<string, unknown>;
+    action_only: boolean;
+    timeout_ms: number;
+}
 /**
  * A tool marked "ask first" waits for the person's yes: the agent asks in
  * its own words, and `confirmTool(call_id, yes)` answers with a tap.
@@ -244,6 +257,7 @@ export type ServerFrames = Exhaustive<{
     "thought.delta": ThoughtDeltaFrame;
     "tool.start": ToolStartFrame;
     "tool.end": ToolEndFrame;
+    "tool.call": ToolCallFrame;
     "tool.confirm": ToolConfirmFrame;
     knowledge: KnowledgeFrame;
     "audio.start": AudioStartFrame;

package/dist/index.d.ts

@@ -9,7 +9,7 @@
  */
 export type { AgentAudio } from "./audio.js";
 export { type Client, type ClientOptions, createClient, DEFAULT_BASE_URL, type Facts, type RelationListParams, type Relations, type SessionGetParams, type SessionListParams, type Sessions, } from "./client.js";
-export { type CloseInfo, type ConnectOptions, Conversation, type ConversationHandlers, type ConversationState, type Disconnect, type Dropped, READY_DEADLINE_MS, type Snapshot, type StartAudioOptions, type TranscriptOutcome, type TurnOptions, type TurnOutcome, type TurnSnapshot, } from "./conversation.js";
+export { type CloseInfo, type ConnectOptions, Conversation, type ConversationHandlers, type ConversationState, type Disconnect, type Dropped, READY_DEADLINE_MS, type Snapshot, type StartAudioOptions, type ToolHandler, type TranscriptOutcome, type TurnOptions, type TurnOutcome, type TurnSnapshot, } from "./conversation.js";
 export { AuthenticationError, ConflictError, ConnectionError, InsufficientCreditsError, NotFoundError, PermissionError, PlatformError, type PlatformErrorInit, RateLimitError, ServerError, TimeoutError, ValidationError, } from "./errors.js";
 export type * from "./events.js";
 export type * from "./gen/index.js";

package/dist/protocol.gen.d.ts

@@ -1,10 +1,10 @@
 /** The protocol, sent as the WebSocket subprotocol and echoed on `ready`. */
 export declare const PROTOCOL = "ino.v2";
 /** Every frame a client may send; the edge closes on any other (4009). */
-export declare const CLIENT_VERBS: readonly ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping", "tool.confirmed"];
+export declare const CLIENT_VERBS: readonly ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping", "tool.confirmed", "tool.result"];
 export type ClientVerb = (typeof CLIENT_VERBS)[number];
 /** Every event the edge sends. */
-export declare const SERVER_EVENTS: readonly ["ready", "turn.start", "heard", "heard.partial", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
+export declare const SERVER_EVENTS: readonly ["ready", "turn.start", "heard", "heard.partial", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
 export type ServerEventName = (typeof SERVER_EVENTS)[number];
 /** The events that end a turn: exactly one of them ends every turn. */
 export declare const TERMINAL_EVENTS: readonly ["turn.done", "interrupted"];
@@ -45,3 +45,5 @@ export declare const AUDIO_CODEC = "pcm_s16le";
 export declare const AUDIO_CHANNELS = 1;
 export declare const DEFAULT_SAMPLE_RATE = 24000;
 export declare const MAX_SOCKETS_PER_END_USER = 3;
+export declare const APP_TOOLS_MAX = 50;
+export declare const TOOL_ERROR_MAX_CHARS = 2000;

package/dist/protocol.gen.js

@@ -5,9 +5,9 @@
 /** The protocol, sent as the WebSocket subprotocol and echoed on `ready`. */
 export const PROTOCOL = "ino.v2";
 /** Every frame a client may send; the edge closes on any other (4009). */
-export const CLIENT_VERBS = ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping", "tool.confirmed"];
+export const CLIENT_VERBS = ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping", "tool.confirmed", "tool.result"];
 /** Every event the edge sends. */
-export const SERVER_EVENTS = ["ready", "turn.start", "heard", "heard.partial", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
+export const SERVER_EVENTS = ["ready", "turn.start", "heard", "heard.partial", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
 /** The events that end a turn: exactly one of them ends every turn. */
 export const TERMINAL_EVENTS = ["turn.done", "interrupted"];
 /** A `ping` of exactly this text is answered by the edge's runtime with `KEEPALIVE_PONG`, which has no `seq`. */
@@ -47,3 +47,5 @@ export const AUDIO_CODEC = "pcm_s16le";
 export const AUDIO_CHANNELS = 1;
 export const DEFAULT_SAMPLE_RATE = 24000;
 export const MAX_SOCKETS_PER_END_USER = 3;
+export const APP_TOOLS_MAX = 50;
+export const TOOL_ERROR_MAX_CHARS = 2000;

package/dist/socket.d.ts

@@ -81,6 +81,8 @@ export interface DialOptions {
     url: string;
     key: string;
     sessionId: string;
+    /** The app tools this connection runs, by name: `auth.tools`. */
+    tools?: string[];
     WebSocket: WebSocketConstructor;
     onFrame(frame: ServerFrame): void;
     onAudio(frame: AudioFrame): void;

package/dist/socket.js

@@ -21,11 +21,13 @@ export function dial(options) {
         return true;
     };
     socket.onopen = () => {
-        // Two fields, and the edge refuses any third by name.
+        // The edge refuses a field it does not know by name, so `tools` goes
+        // only when there are some: no app tools, no field.
         send(JSON.stringify({
             type: "auth",
             key: options.key,
             session_id: options.sessionId,
+            ...(options.tools?.length ? { tools: options.tools } : {}),
         }));
     };
     socket.onmessage = ({ data }) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lumavox/client",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Talk to Lumavox agents from a browser, a React Native app or Node, with an end user's ephemeral key: conversations on the agent's socket, sessions, memory and relations. Zero dependencies.",
   "license": "MIT",
   "type": "module",