@lumavox/client 0.4.0 → 0.6.0

package/README.md

@@ -89,7 +89,7 @@ key, dials, authenticates, and from then on:
 
 | Handler | Frame |
 |---|---|
-| `onReady` `onTurnStart` `onHeard` `onHeardPartial` `onKnowledge` `onMood` `onThought` `onText` `onChunk` `onToolStart` `onToolEnd` `onUsage` `onMuted` `onAudioStart` `onAudioEnd` `onTranscript` `onDone` `onInterrupted` `onError` | the wire's frame, as it is |
+| `onReady` `onTurnStart` `onHeard` `onHeardPartial` `onKnowledge` `onMood` `onThought` `onText` `onChunk` `onToolStart` `onToolEnd` `onToolConfirm` `onUsage` `onMuted` `onAudioStart` `onAudioEnd` `onTranscript` `onDone` `onInterrupted` `onError` | the wire's frame, as it is |
 | `onAudio` | `{ pcm, chunk, turn, sample_rate, channels }`: raw 16-bit PCM, never copied |
 | `onRelation` | the relationship, each change once |
 | `onState` | `connecting` `ready` `busy` `reconnecting` `dormant` `closed` |
@@ -101,6 +101,21 @@ key, dials, authenticates, and from then on:
 With no `onAudio` handler every turn asks for no speech, so nothing pays
 for audio nobody hears; `connect`'s `{ speech }` overrides that.
 
+A tool marked "ask first" sends `tool.confirm` and does not run:
+`confirmTool(callId, yes)` answers with the person's tap — a turn of
+theirs, queued as `send` is, which runs the tool first on a yes.
+
+The agent's app tools run in your app: `connect`'s `tools` gives a
+handler for each, by name. The conversation names them when it
+authenticates, runs one when the model calls it, and sends back what it
+answers — or what it threw:
+
+```ts
+const conversation = client.connect(sessionId, {}, {
+  tools: { open_page: async ({ page }) => ({ opened: page }) },
+});
+```
+
 ### Speaking
 
 ```ts

package/dist/conversation.d.ts

@@ -35,7 +35,7 @@
  * shows — a `text.delta`, say — and none for audio, which is passed
  * through, never copied.
  */
-import type { ErrorFrame, InterruptedFrame, KnowledgeFrame, ReadyFrame, ServerFrame, ServerFrames, TranscriptFrame, TurnDoneFrame } from "./events.js";
+import type { ErrorFrame, InterruptedFrame, KnowledgeFrame, ReadyFrame, ServerFrame, ServerFrames, ToolCallFrame, TranscriptFrame, TurnDoneFrame } from "./events.js";
 import type { RelationInfo } from "./gen/index.js";
 import { type AudioFrame, type Warning, type WebSocketConstructor } from "./socket.js";
 import type { Tokens } from "./token.js";
@@ -142,6 +142,8 @@ export interface ConversationHandlers {
     onChunk?: On<"text.chunk">;
     onToolStart?: On<"tool.start">;
     onToolEnd?: On<"tool.end">;
+    /** A tool that asks first waits for the person: answer with `confirmTool`. */
+    onToolConfirm?: On<"tool.confirm">;
     onUsage?: On<"usage">;
     onMuted?: On<"muted">;
     onAudioStart?: On<"audio.start">;
@@ -167,6 +169,12 @@ export interface ConversationHandlers {
     /** Every frame, the ones above included: what a newer edge sends that this SDK does not know. */
     onEvent?: (frame: ServerFrame) => void;
 }
+/**
+ * One of your app's tools: given the model's arguments, it answers any
+ * JSON (or nothing, `null`), maybe as a promise. A throw or a rejection is
+ * the call's `error`, which the model is told.
+ */
+export type ToolHandler = (args: Record<string, unknown>, call: ToolCallFrame) => unknown;
 export interface ConnectOptions {
     /**
      * Every turn's `speech`. By default `false` when there is no `onAudio`
@@ -176,6 +184,12 @@ export interface ConnectOptions {
     speech?: boolean;
     /** Ask every turn for thinking summaries (`onThought`). */
     thoughts?: boolean;
+    /**
+     * The app's own tools, by name: the agent is offered those of its tools
+     * of kind `app` named here, and each call runs its handler here. At most
+     * `APP_TOOLS_MAX` (50).
+     */
+    tools?: Record<string, ToolHandler>;
 }
 /** Per turn: overrides `ConnectOptions`. */
 export interface TurnOptions {
@@ -250,6 +264,12 @@ export declare class Conversation {
      * 1 to 32,000 characters, or it is `dropped: refused` here.
      */
     send(text: string, options?: TurnOptions): Promise<TurnOutcome>;
+    /**
+     * Answer a `tool.confirm` with the person's tap: a turn of theirs — "Yes."
+     * or "No." in the transcript — which runs the tool first on a yes.
+     * Queued as `send` is; never rejects.
+     */
+    confirmTool(callId: string, yes: boolean, options?: TurnOptions): Promise<TurnOutcome>;
     /**
      * Open a recording at `rate`, which must be one of `RECORDING_RATES`.
      * Answers its id. A recording already open is abandoned, as the edge
@@ -300,6 +320,15 @@ export declare class Conversation {
     private stopTimers;
     private frame;
     private ready;
+    /**
+     * One of the app's tools, called by the model: its handler run, and its
+     * answer — or what it threw — sent back as `tool.result` on the socket
+     * the call came on. A reconnect in between ended the call at the edge,
+     * so nothing is sent on the new socket. An action-only call is answered
+     * by nobody: its handler runs, and one that throws is rethrown on its
+     * own, as any handler's is.
+     */
+    private runTool;
     /** The end of a turn: answers whether it was a second for the same turn, which says nothing new. */
     private terminal;
     private error;

package/dist/conversation.js

@@ -1,5 +1,5 @@
 import { wireBytes } from "./pcm.js";
-import { KEEPALIVE_SECONDS, MAX_AUDIO_BYTES, MAX_MESSAGE_CHARS, RECORDING_RATES, } from "./protocol.gen.js";
+import { KEEPALIVE_SECONDS, MAX_AUDIO_BYTES, MAX_MESSAGE_CHARS, RECORDING_RATES, TOOL_ERROR_MAX_CHARS, } from "./protocol.gen.js";
 import { dial, isFinalClose, reconnectAfter, socketUrl, } from "./socket.js";
 /** No `ready` this long after the dial — a proxy that took the upgrade and says nothing — is a dropped connection. */
 export const READY_DEADLINE_MS = 15000;
@@ -26,6 +26,7 @@ const HANDLER = {
     "text.chunk": "onChunk",
     "tool.start": "onToolStart",
     "tool.end": "onToolEnd",
+    "tool.confirm": "onToolConfirm",
     usage: "onUsage",
     muted: "onMuted",
     "audio.start": "onAudioStart",
@@ -138,6 +139,14 @@ export class Conversation {
         }
         return this.enqueue({ type: "user_message", text }, options);
     }
+    /**
+     * Answer a `tool.confirm` with the person's tap: a turn of theirs — "Yes."
+     * or "No." in the transcript — which runs the tool first on a yes.
+     * Queued as `send` is; never rejects.
+     */
+    confirmTool(callId, yes, options = {}) {
+        return this.enqueue({ type: "tool.confirmed", call_id: callId, yes }, options);
+    }
     /**
      * Open a recording at `rate`, which must be one of `RECORDING_RATES`.
      * Answers its id. A recording already open is abandoned, as the edge
@@ -306,6 +315,7 @@ export class Conversation {
                 url: socketUrl(this.setup.socketUrl, this.sessionId),
                 key,
                 sessionId: this.sessionId,
+                tools: Object.keys(this.setup.options.tools ?? {}),
                 WebSocket: this.setup.WebSocket,
                 onFrame: (frame) => this.frame(frame),
                 onAudio: (frame) => {
@@ -499,6 +509,9 @@ export class Conversation {
             case "tool.end":
                 this.update({ tool: null });
                 break;
+            case "tool.call":
+                this.runTool(frame);
+                break;
             case "knowledge":
                 this.update({
                     knowledge: [...(this.snap.turn?.knowledge ?? []), frame],
@@ -554,6 +567,48 @@ export class Conversation {
         }
         this.sync();
     }
+    /**
+     * One of the app's tools, called by the model: its handler run, and its
+     * answer — or what it threw — sent back as `tool.result` on the socket
+     * the call came on. A reconnect in between ended the call at the edge,
+     * so nothing is sent on the new socket. An action-only call is answered
+     * by nobody: its handler runs, and one that throws is rethrown on its
+     * own, as any handler's is.
+     */
+    runTool(frame) {
+        const handler = this.setup.options.tools?.[frame.name];
+        const socket = this.socket;
+        const answer = (fields) => {
+            if (frame.action_only || !socket || this.socket !== socket)
+                return;
+            const sent = { type: "tool.result", call_id: frame.call_id };
+            try {
+                socket.send({ ...sent, ...fields });
+            }
+            catch {
+                // swallowed: a result JSON cannot carry (a cycle, a BigInt), told
+                // to the model as the failure it is.
+                socket.send({ ...sent, error: "The tool's answer is not JSON." });
+            }
+        };
+        new Promise((resolve) => {
+            if (!handler)
+                throw new Error(`This app has no tool ${frame.name}.`);
+            resolve(handler(frame.arguments, frame));
+        }).then((result) => answer({ result: result === undefined ? null : result }), (error) => {
+            const said = error instanceof Error ? error.message : String(error);
+            answer({
+                error: [...(said || "The tool failed.")]
+                    .slice(0, TOOL_ERROR_MAX_CHARS)
+                    .join(""),
+            });
+            if (frame.action_only) {
+                setTimeout(() => {
+                    throw error;
+                });
+            }
+        });
+    }
     /** The end of a turn: answers whether it was a second for the same turn, which says nothing new. */
     terminal(frame) {
         if (frame.turn === this.finished)
@@ -686,8 +741,9 @@ export class Conversation {
                 next.resolve(dropped(recording.dropped ?? "closed"));
                 continue;
             }
-            // A message abandons an open recording, at the edge and here.
-            if (next.frame.type === "user_message" && this.open) {
+            // A message or a tap — a turn with no recording of its own — abandons
+            // an open recording, at the edge and here.
+            if (!recording && this.open) {
                 this.abandon("cancelled");
             }
             this.inFlight = next;

package/dist/events.d.ts

@@ -85,11 +85,59 @@ export interface TextChunkFrame extends Frame<"text.chunk">, InTurn {
 export interface ThoughtDeltaFrame extends Frame<"thought.delta">, InTurn {
     text: string;
 }
+/** Who runs a tool: the platform's own, your server's, or your app's. */
+export type ToolKind = "platform" | "webhook" | "app";
+/** The model called a tool; its `tool.end` follows. */
 export interface ToolStartFrame extends Frame<"tool.start">, InTurn {
     name: string;
+    kind: ToolKind;
+    /** Your tool's call, `call_…`; `null` for the platform's own. */
+    call_id: string | null;
 }
+/** A tool's call is over: whether it answered, and how long it took. */
 export interface ToolEndFrame extends Frame<"tool.end">, InTurn {
     name: string;
+    kind: ToolKind;
+    call_id: string | null;
+    ok: boolean;
+    ms: number;
+}
+/**
+ * The model called one of your app's tools (`ConnectOptions.tools`): the
+ * conversation runs its handler and answers `tool.result` within
+ * `timeout_ms`. An `action_only` call is not waited for — the reply goes
+ * on at once.
+ */
+export interface ToolCallFrame extends Frame<"tool.call">, InTurn {
+    call_id: string;
+    name: string;
+    arguments: Record<string, unknown>;
+    action_only: boolean;
+    timeout_ms: number;
+}
+/**
+ * A tool marked "ask first" waits for the person's yes: the agent asks in
+ * its own words, and `confirmTool(call_id, yes)` answers with a tap.
+ */
+export interface ToolConfirmFrame extends Frame<"tool.confirm">, InTurn {
+    call_id: string;
+    name: string;
+    arguments: Record<string, unknown>;
+}
+/** One call of your tool in a turn, as `turn.done` and the transcript keep it. */
+export interface ToolCall {
+    call_id: string;
+    name: string;
+    kind: "webhook" | "app";
+    arguments: Record<string, unknown>;
+    /** What it answered, as the model read it (16 KB at most); `null` for none. */
+    result: unknown;
+    ok: boolean;
+    /** What failed, as the model was told; `null` when nothing did. */
+    error: string | null;
+    ms: number;
+    /** Which of the reply's rounds, 1 to 3; 0 for a call the person confirmed with a tap. */
+    round: number;
 }
 export interface KnowledgeEntry {
     entry_id: string;
@@ -157,6 +205,8 @@ export interface TurnDoneFrame extends Frame<"turn.done">, InTurn {
     chunks_spoken: number;
     heard: string | null;
     heard_seconds: number | null;
+    /** Your tools it called, in order. */
+    tool_calls: ToolCall[];
     timings: TurnTimings;
 }
 /** The turn ended early; what was written before it stopped still stands. */
@@ -207,6 +257,8 @@ export type ServerFrames = Exhaustive<{
     "thought.delta": ThoughtDeltaFrame;
     "tool.start": ToolStartFrame;
     "tool.end": ToolEndFrame;
+    "tool.call": ToolCallFrame;
+    "tool.confirm": ToolConfirmFrame;
     knowledge: KnowledgeFrame;
     "audio.start": AudioStartFrame;
     "audio.end": AudioEndFrame;

package/dist/gen/index.d.ts

@@ -1 +1 @@
-export type { ClientOptions, CompactionInfo, CompactRequest, CompactSessionData, CompactSessionError, CompactSessionErrors, CompactSessionResponse, CompactSessionResponses, CreateSessionData, CreateSessionError, CreateSessionErrors, CreateSessionResponse, CreateSessionResponses, DeleteAllFactsData, DeleteAllFactsError, DeleteAllFactsErrors, DeleteAllFactsResponse, DeleteAllFactsResponses, DeleteFactData, DeleteFactError, DeleteFactErrors, DeleteFactResponse, DeleteFactResponses, DeleteSessionData, DeleteSessionError, DeleteSessionErrors, DeleteSessionResponse, DeleteSessionResponses, ErrorCode, ErrorInfo, ErrorResponse, ExtractSessionFactsData, ExtractSessionFactsError, ExtractSessionFactsErrors, ExtractSessionFactsResponse, ExtractSessionFactsResponses, FactInfo, FactPage, FactsDeleted, GetSessionData, GetSessionError, GetSessionErrors, GetSessionResponse, GetSessionResponses, KnowledgeCitation, KnowledgeLookupInfo, ListFactsData, ListFactsError, ListFactsErrors, ListFactsResponse, ListFactsResponses, ListRelationsData, ListRelationsError, ListRelationsErrors, ListRelationsResponse, ListRelationsResponses, ListSessionsData, ListSessionsError, ListSessionsErrors, ListSessionsResponse, ListSessionsResponses, MessageInfo, MessageKnowledge, MessagePage, RelationInfo, RelationPage, SessionCreateRequest, SessionDetail, SessionInfo, SessionPage, ValidationIssue } from './types.gen.js';
+export type { ClientOptions, CompactionInfo, CompactRequest, CompactSessionData, CompactSessionError, CompactSessionErrors, CompactSessionResponse, CompactSessionResponses, CreateSessionData, CreateSessionError, CreateSessionErrors, CreateSessionResponse, CreateSessionResponses, DeleteAllFactsData, DeleteAllFactsError, DeleteAllFactsErrors, DeleteAllFactsResponse, DeleteAllFactsResponses, DeleteFactData, DeleteFactError, DeleteFactErrors, DeleteFactResponse, DeleteFactResponses, DeleteSessionData, DeleteSessionError, DeleteSessionErrors, DeleteSessionResponse, DeleteSessionResponses, ErrorCode, ErrorInfo, ErrorResponse, ExtractSessionFactsData, ExtractSessionFactsError, ExtractSessionFactsErrors, ExtractSessionFactsResponse, ExtractSessionFactsResponses, FactInfo, FactPage, FactsDeleted, GetSessionData, GetSessionError, GetSessionErrors, GetSessionResponse, GetSessionResponses, KnowledgeCitation, KnowledgeLookupInfo, ListFactsData, ListFactsError, ListFactsErrors, ListFactsResponse, ListFactsResponses, ListRelationsData, ListRelationsError, ListRelationsErrors, ListRelationsResponse, ListRelationsResponses, ListSessionsData, ListSessionsError, ListSessionsErrors, ListSessionsResponse, ListSessionsResponses, MessageInfo, MessageKnowledge, MessagePage, MessageToolCall, RelationInfo, RelationPage, SessionCreateRequest, SessionDetail, SessionInfo, SessionPage, ValidationIssue } from './types.gen.js';

package/dist/gen/types.gen.d.ts

@@ -89,6 +89,10 @@ export type MessageInfo = {
      */
     mood_intensity: number | null;
     role: 'user' | 'assistant';
+    /**
+     * On a reply: the calls of your tools it made, in order, each with its arguments and what it answered. Empty when it called none, and on the person's messages.
+     */
+    tool_calls: Array<MessageToolCall>;
 };
 export type MessageKnowledge = {
     lookups: Array<KnowledgeLookupInfo>;
@@ -114,6 +118,40 @@ export type KnowledgeCitation = {
     score: number;
     title: string;
 };
+export type MessageToolCall = {
+    /**
+     * The call's id, `call_…`: what your server or app was sent.
+     */
+    call_id: string;
+    name: string;
+    kind: 'webhook' | 'app';
+    /**
+     * What the model called it with.
+     */
+    arguments: {
+        [key: string]: unknown;
+    };
+    /**
+     * What it answered, as the model read it — its JSON, or its text, cut at 16 KB. Null when it answered nothing.
+     */
+    result?: unknown;
+    /**
+     * It answered in time, with a 2xx or a result.
+     */
+    ok: boolean;
+    /**
+     * What failed, as the model was told — or, for a tool that asks first, that it waited for the person's yes. Null when nothing did.
+     */
+    error: string | null;
+    /**
+     * How long it took.
+     */
+    ms: number;
+    /**
+     * Which of the reply's rounds of calls, 1 to 3; 0 for a call the person confirmed with a tap.
+     */
+    round: number;
+};
 export type RelationInfo = {
     agent_id: string;
     /**

package/dist/index.d.ts

@@ -9,7 +9,7 @@
  */
 export type { AgentAudio } from "./audio.js";
 export { type Client, type ClientOptions, createClient, DEFAULT_BASE_URL, type Facts, type RelationListParams, type Relations, type SessionGetParams, type SessionListParams, type Sessions, } from "./client.js";
-export { type CloseInfo, type ConnectOptions, Conversation, type ConversationHandlers, type ConversationState, type Disconnect, type Dropped, READY_DEADLINE_MS, type Snapshot, type StartAudioOptions, type TranscriptOutcome, type TurnOptions, type TurnOutcome, type TurnSnapshot, } from "./conversation.js";
+export { type CloseInfo, type ConnectOptions, Conversation, type ConversationHandlers, type ConversationState, type Disconnect, type Dropped, READY_DEADLINE_MS, type Snapshot, type StartAudioOptions, type ToolHandler, type TranscriptOutcome, type TurnOptions, type TurnOutcome, type TurnSnapshot, } from "./conversation.js";
 export { AuthenticationError, ConflictError, ConnectionError, InsufficientCreditsError, NotFoundError, PermissionError, PlatformError, type PlatformErrorInit, RateLimitError, ServerError, TimeoutError, ValidationError, } from "./errors.js";
 export type * from "./events.js";
 export type * from "./gen/index.js";

package/dist/protocol.gen.d.ts

@@ -1,10 +1,10 @@
 /** The protocol, sent as the WebSocket subprotocol and echoed on `ready`. */
 export declare const PROTOCOL = "ino.v2";
 /** Every frame a client may send; the edge closes on any other (4009). */
-export declare const CLIENT_VERBS: readonly ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping"];
+export declare const CLIENT_VERBS: readonly ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping", "tool.confirmed", "tool.result"];
 export type ClientVerb = (typeof CLIENT_VERBS)[number];
 /** Every event the edge sends. */
-export declare const SERVER_EVENTS: readonly ["ready", "turn.start", "heard", "heard.partial", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
+export declare const SERVER_EVENTS: readonly ["ready", "turn.start", "heard", "heard.partial", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
 export type ServerEventName = (typeof SERVER_EVENTS)[number];
 /** The events that end a turn: exactly one of them ends every turn. */
 export declare const TERMINAL_EVENTS: readonly ["turn.done", "interrupted"];
@@ -45,3 +45,5 @@ export declare const AUDIO_CODEC = "pcm_s16le";
 export declare const AUDIO_CHANNELS = 1;
 export declare const DEFAULT_SAMPLE_RATE = 24000;
 export declare const MAX_SOCKETS_PER_END_USER = 3;
+export declare const APP_TOOLS_MAX = 50;
+export declare const TOOL_ERROR_MAX_CHARS = 2000;

package/dist/protocol.gen.js

@@ -5,9 +5,9 @@
 /** The protocol, sent as the WebSocket subprotocol and echoed on `ready`. */
 export const PROTOCOL = "ino.v2";
 /** Every frame a client may send; the edge closes on any other (4009). */
-export const CLIENT_VERBS = ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping"];
+export const CLIENT_VERBS = ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping", "tool.confirmed", "tool.result"];
 /** Every event the edge sends. */
-export const SERVER_EVENTS = ["ready", "turn.start", "heard", "heard.partial", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
+export const SERVER_EVENTS = ["ready", "turn.start", "heard", "heard.partial", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
 /** The events that end a turn: exactly one of them ends every turn. */
 export const TERMINAL_EVENTS = ["turn.done", "interrupted"];
 /** A `ping` of exactly this text is answered by the edge's runtime with `KEEPALIVE_PONG`, which has no `seq`. */
@@ -47,3 +47,5 @@ export const AUDIO_CODEC = "pcm_s16le";
 export const AUDIO_CHANNELS = 1;
 export const DEFAULT_SAMPLE_RATE = 24000;
 export const MAX_SOCKETS_PER_END_USER = 3;
+export const APP_TOOLS_MAX = 50;
+export const TOOL_ERROR_MAX_CHARS = 2000;

package/dist/socket.d.ts

@@ -81,6 +81,8 @@ export interface DialOptions {
     url: string;
     key: string;
     sessionId: string;
+    /** The app tools this connection runs, by name: `auth.tools`. */
+    tools?: string[];
     WebSocket: WebSocketConstructor;
     onFrame(frame: ServerFrame): void;
     onAudio(frame: AudioFrame): void;

package/dist/socket.js

@@ -21,11 +21,13 @@ export function dial(options) {
         return true;
     };
     socket.onopen = () => {
-        // Two fields, and the edge refuses any third by name.
+        // The edge refuses a field it does not know by name, so `tools` goes
+        // only when there are some: no app tools, no field.
         send(JSON.stringify({
             type: "auth",
             key: options.key,
             session_id: options.sessionId,
+            ...(options.tools?.length ? { tools: options.tools } : {}),
         }));
     };
     socket.onmessage = ({ data }) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lumavox/client",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "Talk to Lumavox agents from a browser, a React Native app or Node, with an end user's ephemeral key: conversations on the agent's socket, sessions, memory and relations. Zero dependencies.",
   "license": "MIT",
   "type": "module",