@oberik/sdk 0.60.0 → 0.62.0

package/dist/cjs/index.js

@@ -2096,6 +2096,208 @@ class AgentFramework {
             }),
         },
     };
+    // -- voice ---------------------------------------------------------------
+    /**
+     * Live voice calls.
+     *
+     * `open()` creates the call and connects the audio socket; everything after that is
+     * `send()` for microphone audio, `onAudio` for what to play, and events for what is
+     * happening. The token is the same end-user JWT everything else uses, and the turns
+     * behind the call are ordinary turns — same capability gate, same tool allow-list, same
+     * per-customer cost attribution.
+     *
+     * Needs a `WebSocket` global: present in browsers and in Node 22+. On older Node, pass
+     * one in as `webSocket` — the SDK deliberately has no dependencies, so it will not bring
+     * an implementation of its own.
+     */
+    voice = {
+        open: async (body = {}, handlers = {}, opts = {}) => {
+            const info = await this.request("POST", "/voice/sessions", { body });
+            return this.connectVoice(info, handlers, opts.webSocket);
+        },
+        /** Reconnect to a call that is already open — a page reload, a second screen watching.
+         *
+         *  Audio goes to whoever is connected; events go to everyone. So a supervisor can watch
+         *  a call without taking it over, which is what a call-centre floor actually needs. */
+        attach: async (id, handlers = {}, opts = {}) => {
+            const info = await this.request("GET", `/voice/sessions/${id}`);
+            return this.connectVoice(info, handlers, opts.webSocket);
+        },
+        /** Every call this token can see. */
+        list: () => this.request("GET", "/voice/sessions"),
+        /** What the frontend can do — the voices and roles a project may ask for. Read it rather
+         *  than hard-coding a list: asking for a voice the host does not have fails the call at
+         *  `open()`, which is a worse place to find out than a dropdown. */
+        frontend: () => this.request("GET", "/voice/frontend"),
+    };
+    connectVoice(info, handlers, webSocketImpl) {
+        const WS = (webSocketImpl ?? globalThis.WebSocket);
+        if (!WS) {
+            throw new AgentStreamError("no WebSocket available. Browsers and Node 22+ have one; on older Node pass " +
+                "`{ webSocket: WebSocket }` from a library of your choice. The SDK has no " +
+                "dependencies and will not choose one for you.");
+        }
+        const base = this.baseUrl.replace(/^http/, "ws");
+        const url = info.stream_url.startsWith("ws")
+            ? info.stream_url
+            : `${base}${info.stream_url}`;
+        const ws = new WS(url);
+        ws.binaryType = "arraybuffer";
+        const listeners = new Map();
+        // Audio the caller produced before the socket finished opening. Buffered rather than
+        // dropped: a browser's microphone starts before the connection does, and dropping the
+        // first quarter-second means the caller's first word is missing — which reads as the
+        // agent not listening.
+        const pending = [];
+        let closed = false;
+        ws.onopen = () => {
+            // The token goes in the first frame rather than in the URL. A query-string token ends
+            // up in proxy logs and in browser history, and a WebSocket cannot carry an
+            // Authorization header from a browser.
+            void (async () => {
+                const headers = await this.authHeaders();
+                const bearer = headers["Authorization"] ?? headers["X-API-Key"] ?? "";
+                ws.send(JSON.stringify({ type: "auth", token: bearer.replace(/^Bearer /, "") }));
+                for (const chunk of pending)
+                    ws.send(chunk);
+                pending.length = 0;
+            })();
+        };
+        ws.onmessage = (e) => {
+            if (typeof e.data !== "string") {
+                handlers.onAudio?.(e.data);
+                return;
+            }
+            let frame;
+            try {
+                frame = JSON.parse(e.data);
+            }
+            catch {
+                return;
+            }
+            handlers.onEvent?.(frame);
+            if (frame.event === "state")
+                handlers.onState?.(frame.data);
+            if (frame.event === "error")
+                handlers.onError?.(frame.data.detail);
+            for (const fn of listeners.get(frame.event) ?? [])
+                fn(frame.data);
+        };
+        ws.onerror = () => handlers.onError?.("the voice socket failed");
+        ws.onclose = (e) => {
+            closed = true;
+            handlers.onClose?.(e?.reason ?? "closed");
+        };
+        const post = async (path, body) => {
+            await this.request("POST", `/voice/sessions/${info.id}${path}`, { body: body ?? {} });
+        };
+        /** The same call, keeping the answer. For the routes whose answer is the point — a
+         *  disclosure the server may have refused to say, and the reason it did. */
+        const postFor = (path, body) => this.request("POST", `/voice/sessions/${info.id}${path}`, { body: body ?? {} });
+        /** A control frame down the socket rather than an HTTP call.
+         *
+         *  Because these are about *timing*: "the caller started talking" has to reach the server
+         *  in the same tens of milliseconds the audio does, and an HTTP round trip on the barge-in
+         *  path is the difference between the agent stopping and the agent talking over somebody.
+         *  Hold and transfer go over HTTP precisely because they are not on that path. */
+        const control = (msg) => {
+            if (ws.readyState === 1)
+                ws.send(JSON.stringify(msg));
+        };
+        return {
+            id: info.id,
+            sessionId: info.session_id,
+            rate: info.rate,
+            info,
+            get closed() {
+                return closed;
+            },
+            send: (pcm) => {
+                const bytes = pcm instanceof ArrayBuffer
+                    ? new Uint8Array(pcm)
+                    : new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+                // 1 === OPEN. Compared numerically because a polyfilled WebSocket may not expose
+                // the static constants, and `ws.OPEN` on an instance is not in the spec.
+                if (ws.readyState === 1)
+                    ws.send(bytes);
+                else if (!closed)
+                    pending.push(bytes);
+            },
+            on: (name, fn) => {
+                const set = listeners.get(name) ?? new Set();
+                listeners.set(name, set);
+                set.add(fn);
+                return () => set.delete(fn);
+            },
+            snapshot: () => this.request("GET", `/voice/sessions/${info.id}/state`),
+            hold: (o = {}) => post("/hold", o),
+            resume: (o = {}) => post("/resume", o),
+            holdUntil: async (work, o = {}) => {
+                // The hold goes on FIRST, and the work starts after. The other order leaves a window
+                // where the work is running and the caller is hearing nothing — short for a fast
+                // lookup, and exactly as long as the request takes for a slow one, which is the case
+                // this whole mechanism is for.
+                const maxSeconds = o.maxSeconds ?? 120;
+                await post("/hold", {
+                    reason: o.reason ?? "the app is waiting on something",
+                    listen: o.listen,
+                    max_seconds: maxSeconds,
+                });
+                // The caller ending the hold themselves. Watched rather than polled, so "the caller
+                // said never mind" and "the CRM answered" are a genuine race.
+                let onInterrupted;
+                const interrupted = new Promise((resolve) => {
+                    onInterrupted = () => resolve("interrupted");
+                });
+                const offHold = listeners.get("hold.interrupted") ?? new Set();
+                listeners.set("hold.interrupted", offHold);
+                const watcher = () => onInterrupted?.();
+                offHold.add(watcher);
+                let timer;
+                const expired = new Promise((resolve) => {
+                    timer = setTimeout(() => resolve("timeout"), maxSeconds * 1000);
+                });
+                try {
+                    const raced = await Promise.race([
+                        work().then((result) => ({ outcome: "done", result }), (error) => ({ outcome: "failed", error })),
+                        interrupted.then(() => ({ outcome: "interrupted" })),
+                        expired.then(() => ({ outcome: "timeout" })),
+                    ]);
+                    // Only resume for the outcomes where the caller is still waiting on us. An
+                    // interruption already took them off hold — resuming again would say "thanks for
+                    // waiting" to somebody who just said never mind.
+                    if (raced.outcome !== "interrupted") {
+                        await post("/resume", { lead: o.lead });
+                    }
+                    return raced;
+                }
+                finally {
+                    if (timer !== undefined)
+                        clearTimeout(timer);
+                    offHold.delete(watcher);
+                }
+            },
+            transfer: (to, context) => post("/transfer", { to, context }),
+            inject: (facts) => post("/context", { facts }),
+            say: (text, o = {}) => postFor("/speech", { text, interrupt: o.interrupt ?? true }),
+            speechStart: () => control({ type: "speech_start" }),
+            speechEnd: () => control({ type: "speech_end" }),
+            transcript: (text, final = true) => control({ type: "transcript", text, final }),
+            dtmf: (digit) => control({ type: "dtmf", digit }),
+            hangUp: () => control({ type: "hangup" }),
+            markPlayed: (mark) => control({ type: "mark", mark }),
+            bench: () => this.request("GET", `/voice/sessions/${info.id}/bench`),
+            close: () => {
+                closed = true;
+                try {
+                    ws.close();
+                }
+                catch {
+                    /* already gone */
+                }
+            },
+        };
+    }
     // -- documents -----------------------------------------------------------
     /** Agent Plugins — the skills this token can reach for, and the ones it may add.
      *
@@ -3176,6 +3378,14 @@ class OberikProject {
          * `set(await get())` — the obvious way to flip one flag — works. Anything else
          * unrecognised still errors: a capability nobody enforces must not be accepted in
          * silence, which is the failure this whole check exists for.
+         *
+         * That now includes the MEMBERS of the two modality lists, which take `image`,
+         * `audio`, `video` or `file` and refuse anything else by name. They used to be
+         * filtered — so `outputModalities: ["banana"]` and `outputModalities: ["image"]` on a
+         * text-only project were the same 200 storing `[]`, and a token that may produce
+         * nothing looks exactly like one that was saved correctly. A real modality no
+         * registered model supports is still dropped rather than granted, but the response
+         * says so in `warnings`.
          */
         set: (caps) => {
             const { supportedModalities: _readOnly, ...settable } = caps;
@@ -3300,6 +3510,11 @@ class OberikProject {
         /** Name a chat model AND an embedding model: the first lets the agent answer, the
          *  second lets it index. The response's `derived` says what was set for you.
          *
+         *  `values` must carry every field the provider's catalog entry declares without
+         *  `optional` — `api_base` for a self-hosted server, `api_key` for a hosted API — and a
+         *  missing one is refused, naming it. A vLLM provider with a key and no base URL used
+         *  to register happily and could not be reached by anything.
+         *
          *  `create`, like every other collection here. It was `add`, and `create` is what a
          *  developer guesses — a reporter wrote `providers.create` from memory, it threw
          *  mid-run, and they filed it against themselves (OBE-106). Two collections used