@vrealtime/server 0.0.0-stage → 0.2.0

package/dist/index.js

@@ -0,0 +1,480 @@
+import { startSession, } from '@vrealtime/core';
+import { Inbox, Outbox, decodeAudioFrame, decodeClientMessage, encodeAudioFrame, encodeMessage, negotiateCapabilities, PROTOCOL_VERSION, TOOL_STREAM_V1, } from '@vrealtime/protocol';
+import { supportedCapabilities } from './capabilities.js';
+import { legacyPreview, toolStreamMessage } from './tool-stream.js';
+const PCM24 = { encoding: 'pcm16', sampleRate: 24000, channels: 1 };
+const REFUSED = Symbol('refused');
+const FLUSH_INTERVAL_MS = 10;
+const OUTPUT = 2;
+const INPUT = 1;
+const systemClock = {
+    now: () => performance.now(),
+    setTimeout: (run, ms) => setTimeout(run, ms),
+    clearTimeout: (id) => clearTimeout(id),
+};
+export function toCoreFormat(format) {
+    if (format === undefined)
+        return undefined;
+    if (format.encoding === 'pcmu')
+        return { encoding: 'pcmu' };
+    if (format.encoding === 'opus')
+        return { encoding: 'opus', frameMs: 20 };
+    const rate = format.sampleRate;
+    return rate === 8000 || rate === 16000 || rate === 24000 || rate === 48000 ? { encoding: 'pcm16', sampleRate: rate, channels: 1 } : undefined;
+}
+export function toWireFormat(format) {
+    return format.encoding === 'pcm16' ? { encoding: 'pcm16', sampleRate: format.sampleRate } : { encoding: format.encoding };
+}
+function sameFormat(a, b) {
+    return a.encoding === b.encoding && (a.encoding !== 'pcm16' || (b.encoding === 'pcm16' && a.sampleRate === b.sampleRate));
+}
+/** Use the app's format when the model takes it as-is (pass-through), otherwise 24 kHz PCM16. */
+function negotiate(requested, supported) {
+    const format = toCoreFormat(requested);
+    if (format && supported.some((s) => sameFormat(s, format)))
+        return format;
+    return supported.find((s) => sameFormat(s, PCM24)) ?? supported[0] ?? PCM24;
+}
+/** One agent session and everything that must survive a reconnect. */
+class LiveSession {
+    hub;
+    principal;
+    id = crypto.randomUUID();
+    outbox = new Outbox();
+    inbox = new Inbox();
+    session;
+    input = PCM24;
+    output = PCM24;
+    /** Negotiated when the session starts; fixed for its lifetime, resumes included. */
+    capabilities = [];
+    socket;
+    expiry;
+    #audioSeq = 0;
+    #audioItem;
+    #queue = [];
+    #flushTimer;
+    constructor(hub, principal) {
+        this.hub = hub;
+        this.principal = principal;
+    }
+    bind(agentSession) {
+        this.session = agentSession;
+        agentSession.on('turn.state', (e) => this.send({ type: 'turn.state', turnId: e.turnId, epoch: e.epoch, state: e.state }));
+        agentSession.on('transcript', (e) => this.send({ type: 'transcript', role: e.role, itemId: e.itemId, text: e.text, final: e.final }));
+        agentSession.on('text', (e) => this.send({ type: 'text', responseId: e.responseId, delta: e.delta }));
+        agentSession.on('audio', (e) => this.#audio(e));
+        // `arguments` is a required member on the wire: a missing value is sent as null.
+        agentSession.on('tool.call', (e) => this.send({ type: 'tool.call', callId: e.callId, epoch: e.epoch, turnId: e.turnId, name: e.name, arguments: e.arguments ?? null }));
+        // One source for call progress: the full lifecycle with tool-stream-v1, the 1.0 preview without it, never both.
+        const toolStream = this.capabilities.includes(TOOL_STREAM_V1);
+        agentSession.on('tool.stream', (e) => {
+            const message = toolStream ? toolStreamMessage(e) : legacyPreview(e);
+            if (message)
+                this.send(message);
+        });
+        agentSession.on('interrupted', (e) => {
+            this.#dropAudio(e.epoch);
+            this.send({ type: 'interrupted', epoch: e.epoch });
+        });
+        agentSession.on('error', (e) => this.send({ type: 'error', code: 'AGENT_ERROR', message: e.error.message, recoverable: e.recoverable }));
+        agentSession.on('closed', () => {
+            this.hub.forget(this);
+            this.socket?.close(1000, 'session closed');
+        });
+    }
+    send(message, epoch) {
+        const stamped = this.outbox.stamp(message);
+        this.#write(encodeMessage(stamped), epoch);
+    }
+    /** Opens the session on the bound socket; `resumedFrom` is set on a resume. */
+    sendReady(resumedFrom) {
+        this.send({
+            type: 'ready',
+            sessionId: this.id,
+            input: toWireFormat(this.input),
+            output: toWireFormat(this.output),
+            ...(resumedFrom !== undefined ? { resumedFrom } : {}),
+            ...(this.capabilities.length > 0 ? { capabilities: [...this.capabilities] } : {}),
+        });
+    }
+    /** Re-sends everything after the client's last seen seq. False when that is no longer possible. */
+    resend(lastSeen) {
+        const missed = this.outbox.resendFrom(lastSeen);
+        if (missed === undefined)
+            return false;
+        for (const message of missed)
+            this.#write(encodeMessage(message));
+        return true;
+    }
+    detach() {
+        this.socket = undefined;
+        this.#queue = [];
+        this.#audioItem = undefined;
+    }
+    flush() {
+        this.#flushTimer = undefined;
+        const socket = this.socket;
+        if (!socket) {
+            this.#queue = [];
+            return;
+        }
+        while (this.#queue.length > 0 && !socket.congested())
+            socket.send(this.#queue.shift().data);
+        if (this.#queue.length > 0)
+            this.#scheduleFlush();
+    }
+    #audio(event) {
+        if (!this.socket)
+            return;
+        if (event.itemId !== this.#audioItem) {
+            this.#audioItem = event.itemId;
+            this.send({ type: 'audio.start', itemId: event.itemId, responseId: event.responseId, epoch: event.epoch }, event.epoch);
+        }
+        this.#write(encodeAudioFrame(OUTPUT, this.#audioSeq++, event.chunk.data), event.epoch);
+    }
+    /** Frames keep their order: once anything is queued, everything queues behind it. */
+    #write(data, epoch) {
+        const socket = this.socket;
+        if (!socket)
+            return;
+        if (this.#queue.length === 0 && !socket.congested()) {
+            socket.send(data);
+            return;
+        }
+        this.#queue.push(epoch === undefined ? { data } : { data, epoch });
+        this.#scheduleFlush();
+    }
+    #dropAudio(closedEpoch) {
+        this.#queue = this.#queue.filter((item) => item.epoch === undefined || item.epoch > closedEpoch);
+        this.#audioItem = undefined;
+    }
+    #scheduleFlush() {
+        if (this.#flushTimer === undefined)
+            this.#flushTimer = this.hub.clock.setTimeout(() => this.flush(), FLUSH_INTERVAL_MS);
+    }
+}
+class SocketConnection {
+    hub;
+    socket;
+    headers;
+    #live;
+    #starting;
+    #pending = [];
+    #closed = false;
+    #lastSeen;
+    #heartbeat;
+    #ackScheduled = false;
+    constructor(hub, socket, headers) {
+        this.hub = hub;
+        this.socket = socket;
+        this.headers = headers;
+        this.#lastSeen = hub.clock.now();
+        this.#checkHeartbeat();
+        socket.addEventListener?.('message', (event) => this.receive(event.data));
+        socket.addEventListener?.('close', () => this.closed());
+    }
+    get sessionId() {
+        return this.#live?.id;
+    }
+    congested() {
+        return (this.socket.bufferedAmount ?? 0) > this.hub.highWaterMark;
+    }
+    send(data) {
+        if (!this.#closed)
+            this.socket.send(data);
+    }
+    close(code, reason) {
+        if (this.#closed)
+            return;
+        this.socket.close(code, reason);
+        this.closed();
+    }
+    receive(data) {
+        if (this.#closed)
+            return;
+        this.#lastSeen = this.hub.clock.now();
+        if (typeof data !== 'string') {
+            this.#receiveAudio(data);
+            return;
+        }
+        const decoded = decodeClientMessage(data);
+        if (!decoded.ok) {
+            this.#error(decoded.error.code, decoded.error.message, true);
+            return;
+        }
+        const message = decoded.message;
+        if (message.type === 'ping') {
+            this.#sendDirect({ type: 'pong' });
+            return;
+        }
+        if (message.type === 'hello') {
+            void this.#hello(message);
+            return;
+        }
+        if (this.#starting || !this.#live) {
+            if (this.#starting)
+                this.#pending.push(message);
+            else
+                this.#error('NO_SESSION', 'Send hello first', false);
+            return;
+        }
+        this.#handle(this.#live, message);
+    }
+    closed() {
+        if (this.#closed)
+            return;
+        this.#closed = true;
+        this.hub.clock.clearTimeout(this.#heartbeat);
+        const live = this.#live;
+        if (live?.socket === this) {
+            live.detach();
+            this.hub.park(live);
+        }
+    }
+    #receiveAudio(data) {
+        const bytes = data instanceof Uint8Array ? data : data instanceof ArrayBuffer ? new Uint8Array(data) : undefined;
+        const frame = bytes && decodeAudioFrame(bytes);
+        if (!frame || frame.kind !== INPUT) {
+            this.#error('MALFORMED', 'Invalid audio frame', true);
+            return;
+        }
+        const live = this.#live;
+        if (live && !this.#starting)
+            live.session.sendAudio({ data: frame.payload, format: live.input });
+    }
+    async #hello(hello) {
+        if (this.#live || this.#starting) {
+            this.#error('DUPLICATE_HELLO', 'hello was already received', true);
+            return;
+        }
+        // Messages that arrive while authenticating or starting are queued in order.
+        let done;
+        this.#starting = new Promise((resolve) => { done = resolve; });
+        try {
+            const principal = await this.#authenticate(hello);
+            if (this.#closed)
+                return;
+            if (principal === REFUSED) {
+                this.#error('UNAUTHORIZED', 'Authentication failed', false);
+                this.close(4401, 'unauthorized');
+                return;
+            }
+            const live = hello.sessionId !== undefined ? this.#resume(hello, principal) : await this.#start(hello, principal);
+            if (!live)
+                return;
+            if (hello.context !== undefined)
+                this.hub.options.onContext?.(hello.context, live.session);
+            for (const message of this.#pending.splice(0))
+                this.#handle(live, message);
+        }
+        finally {
+            this.#starting = undefined;
+            done();
+        }
+    }
+    async #authenticate(hello) {
+        const authenticate = this.hub.options.authenticate;
+        if (!authenticate)
+            return undefined;
+        try {
+            const principal = await authenticate({ token: hello.auth, context: hello.context, sessionId: hello.sessionId, headers: this.headers });
+            return principal ? principal : REFUSED;
+        }
+        catch {
+            return REFUSED;
+        }
+    }
+    async #start(hello, principal) {
+        const live = new LiveSession(this.hub, principal);
+        const supported = this.hub.agent.model.capabilities;
+        live.input = negotiate(hello.input, supported.input.formats);
+        live.output = negotiate(hello.output, supported.output.formats);
+        live.capabilities = negotiateCapabilities(hello.capabilities, this.hub.capabilities);
+        try {
+            const configured = (await this.hub.options.configure?.(principal, { context: hello.context })) ?? {};
+            const session = await startSession(this.hub.agent, {
+                clientTools: hello.tools,
+                config: { ...configured, inputFormat: live.input, outputFormat: live.output },
+                clock: this.hub.clock,
+                data: principal,
+            });
+            live.bind(session);
+        }
+        catch (error) {
+            this.#error('START_FAILED', error instanceof Error ? error.message : String(error), false);
+            this.close(1011, 'session start failed');
+            return undefined;
+        }
+        this.hub.adopt(live);
+        if (this.#closed) {
+            // The socket went away while the model was connecting; park so a resume can still find it.
+            this.hub.park(live);
+            return undefined;
+        }
+        this.#bind(live);
+        live.sendReady();
+        live.resend(0); // anything the session emitted before the socket was bound
+        return live;
+    }
+    #resume(hello, principal) {
+        const live = this.hub.find(hello.sessionId);
+        // An unknown session and someone else's session get the same answer.
+        if (!live || live.principal?.id !== principal?.id) {
+            this.#error('RESUME_FAILED', 'Unknown or expired session', false);
+            this.close(4404, 'unknown session');
+            return undefined;
+        }
+        live.socket?.close(4000, 'replaced by a newer connection');
+        this.#bind(live);
+        if (live.session.closed)
+            return undefined;
+        live.sendReady(live.inbox.lastSeq);
+        if (!live.resend(hello.resumeFrom ?? 0)) {
+            this.#error('RESUME_FAILED', 'Cannot resend from resumeFrom: those messages were dropped or never sent', false);
+            void live.session.close();
+            return undefined;
+        }
+        return live;
+    }
+    #bind(live) {
+        this.hub.unpark(live);
+        this.#live = live;
+        live.socket = this;
+    }
+    #handle(live, message) {
+        if (message.type === 'ack') {
+            live.outbox.ack(message.upTo);
+            return;
+        }
+        if (!live.inbox.accept(message)) {
+            this.#scheduleAck(live);
+            return;
+        }
+        const session = live.session;
+        switch (message.type) {
+            case 'text':
+                session.sendText(message.text);
+                break;
+            case 'interrupt':
+                session.interrupt(message.reason);
+                break;
+            case 'playback':
+                session.playback(message.itemId, message.playedMs);
+                break;
+            case 'tool.result': {
+                const result = message.ok
+                    ? { ok: true, output: message.output }
+                    : { ok: false, code: message.code, message: message.message };
+                session.submitClientToolResult(message.callId, result);
+                break;
+            }
+            case 'context':
+                this.hub.options.onContext?.(message.context, session);
+                break;
+        }
+        this.#scheduleAck(live);
+    }
+    #scheduleAck(live) {
+        if (this.#ackScheduled)
+            return;
+        this.#ackScheduled = true;
+        queueMicrotask(() => {
+            this.#ackScheduled = false;
+            if (live.socket === this)
+                live.send({ type: 'ack', upTo: live.inbox.lastSeq });
+        });
+    }
+    #sendDirect(message) {
+        if (this.#live?.socket === this)
+            this.#live.send(message);
+        else
+            this.send(encodeMessage({ v: PROTOCOL_VERSION, seq: 0, ...message }));
+    }
+    #error(code, message, recoverable) {
+        this.#sendDirect({ type: 'error', code, message, recoverable });
+    }
+    #checkHeartbeat() {
+        const timeout = this.hub.heartbeatTimeoutMs;
+        this.#heartbeat = this.hub.clock.setTimeout(() => {
+            if (this.#closed)
+                return;
+            if (this.hub.clock.now() - this.#lastSeen >= timeout)
+                this.close(4408, 'heartbeat timeout');
+            else
+                this.#checkHeartbeat();
+        }, Math.max(1, timeout / 3));
+    }
+}
+/** Shares sessions between sockets, so a reconnecting app can resume. */
+export class Hub {
+    agent;
+    options;
+    clock;
+    highWaterMark;
+    heartbeatTimeoutMs;
+    /** Protocol capabilities sessions may negotiate: what this server implements and the agent's model supports. */
+    capabilities;
+    #sessions = new Map();
+    constructor(agent, options = {}) {
+        this.agent = agent;
+        this.options = options;
+        this.capabilities = supportedCapabilities(agent.model.capabilities);
+        this.clock = options.clock ?? systemClock;
+        this.highWaterMark = options.highWaterMark ?? 256 * 1024;
+        this.heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? 45_000;
+    }
+    /** `headers` (of the upgrade request) are passed to `authenticate`. */
+    attach(socket, info = {}) {
+        return new SocketConnection(this, socket, info.headers);
+    }
+    /** Live sessions, connected or waiting for a resume. */
+    get size() {
+        return this.#sessions.size;
+    }
+    async close() {
+        await Promise.all([...this.#sessions.values()].map((live) => live.session.close()));
+        this.#sessions.clear();
+    }
+    /** @internal */
+    adopt(live) {
+        this.#sessions.set(live.id, live);
+    }
+    /** @internal */
+    find(id) {
+        return this.#sessions.get(id);
+    }
+    /** @internal */
+    forget(live) {
+        this.unpark(live);
+        this.#sessions.delete(live.id);
+    }
+    /** @internal Keeps a socketless session for `resumeWindowMs`, then closes it. */
+    park(live) {
+        this.unpark(live);
+        live.expiry = this.clock.setTimeout(() => {
+            live.expiry = undefined;
+            if (!live.socket)
+                void live.session.close();
+        }, this.options.resumeWindowMs ?? 30_000);
+    }
+    /** @internal */
+    unpark(live) {
+        if (live.expiry !== undefined)
+            this.clock.clearTimeout(live.expiry);
+        live.expiry = undefined;
+    }
+}
+const hubs = new WeakMap();
+/**
+ * Serves `agent` over one WebSocket using protocol v1. Sockets attached for the
+ * same agent share a hub, so a reconnecting app can resume its session.
+ * Options apply when the agent's hub is first created.
+ */
+export function attach(agent, socket, options, info) {
+    let hub = hubs.get(agent);
+    if (!hub) {
+        hub = new Hub(agent, options);
+        hubs.set(agent, hub);
+    }
+    return hub.attach(socket, info);
+}

package/dist/node/debug-page.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Microphone debug page served by `serve()` at `/`. A dependency-free protocol v1
+ * client: text and voice in, transcripts and audio out, a barge-in interrupt and
+ * one app-side tool (`set_accent_color`) behind an epoch gate.
+ */
+export declare const DEBUG_PAGE = "<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">\n<title>vRealTime debug</title>\n<style>\n  :root { --accent: #2563eb; --bg: #fff; --fg: #111827; --muted: #6b7280; --line: #e5e7eb; color-scheme: light dark; }\n  @media (prefers-color-scheme: dark) { :root { --bg: #0b0f17; --fg: #e5e7eb; --muted: #9ca3af; --line: #1f2937; } }\n  body { margin: 0; background: var(--bg); color: var(--fg); font: 15px/1.5 system-ui, sans-serif; }\n  main { max-width: 720px; margin: 0 auto; padding: 16px; }\n  h1 { font-size: 20px; margin: 0 0 4px; color: var(--accent); }\n  .muted { color: var(--muted); font-size: 13px; }\n  .row { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; margin: 12px 0; }\n  button { background: var(--accent); color: #fff; border: 0; border-radius: 6px; padding: 8px 14px; font: inherit; cursor: pointer; }\n  button.secondary { background: transparent; color: var(--fg); border: 1px solid var(--line); }\n  form { display: flex; gap: 8px; }\n  input { flex: 1; min-width: 0; padding: 8px; border: 1px solid var(--line); border-radius: 6px; background: transparent; color: inherit; font: inherit; }\n  #state { padding: 2px 8px; border-radius: 99px; border: 1px solid var(--accent); color: var(--accent); font-size: 13px; }\n  #log { border-top: 1px solid var(--line); margin-top: 12px; padding-top: 8px; }\n  #log p { margin: 6px 0; } .user { font-weight: 600; } .note { color: var(--muted); font-size: 13px; }\n</style>\n</head>\n<body>\n<main>\n  <h1>vRealTime</h1>\n  <div class=\"muted\" id=\"status\">connecting\u2026</div>\n  <div class=\"row\">\n    <button id=\"mic\">Start mic</button>\n    <button id=\"stop\" class=\"secondary\">Interrupt</button>\n    <span id=\"state\">idle</span>\n  </div>\n  <form id=\"form\"><input id=\"text\" placeholder=\"Type a message\" autocomplete=\"off\"><button>Send</button></form>\n  <div id=\"log\"></div>\n</main>\n<script type=\"module\">\nconst RATE = 24000, FRAME = 480, TRANSIENT = ['hello', 'ack', 'ping'];\nconst $ = (id) => document.getElementById(id);\nlet seq = 0, lastSeq = 0, epoch = 0, closedEpoch = -1, audioSeq = 0;\n\nconst tools = {\n  set_accent_color: {\n    description: 'Set the page accent color to any CSS color.',\n    parameters: { type: 'object', properties: { color: { type: 'string' } }, required: ['color'] },\n    run: ({ color }) => { document.documentElement.style.setProperty('--accent', color); return { color }; },\n  },\n};\n\nconst ws = new WebSocket(location.origin.replace(/^http/, 'ws') + location.pathname);\nws.binaryType = 'arraybuffer';\nconst send = (m) => { if (ws.readyState === 1) ws.send(JSON.stringify({ v: 1, seq: TRANSIENT.includes(m.type) ? 0 : ++seq, ...m })); };\nws.onopen = () => send({\n  type: 'hello',\n  tools: Object.entries(tools).map(([name, t]) => ({ name, description: t.description, parameters: t.parameters })),\n  input: { encoding: 'pcm16', sampleRate: RATE }, output: { encoding: 'pcm16', sampleRate: RATE },\n});\nws.onclose = (e) => { $('status').textContent = 'disconnected (' + e.code + ')'; };\nsetInterval(() => { send({ type: 'ping' }); if (lastSeq) send({ type: 'ack', upTo: lastSeq }); }, 10000);\n\nconst lines = new Map();\nfunction line(id, role, text) {\n  let el = lines.get(id);\n  if (!el) { el = document.createElement('p'); el.className = role; $('log').append(el); lines.set(id, el); }\n  el.textContent = (role === 'user' ? 'You: ' : 'Agent: ') + text;\n  el.scrollIntoView({ block: 'end' });\n}\nfunction note(text) { const el = document.createElement('p'); el.className = 'note'; el.textContent = text; $('log').append(el); }\n\n// Playback: schedule PCM16 chunks back to back; track what was actually heard per item.\nlet ctx, playhead = 0, sources = [], item;\nconst audio = () => (ctx ??= new AudioContext({ sampleRate: RATE }));\nfunction play(bytes) {\n  const c = audio(), n = bytes.byteLength >> 1, view = new DataView(bytes.buffer, bytes.byteOffset, n * 2);\n  const buffer = c.createBuffer(1, n, RATE), data = buffer.getChannelData(0);\n  for (let i = 0; i < n; i++) data[i] = view.getInt16(i * 2, true) / 32768;\n  const src = c.createBufferSource();\n  src.buffer = buffer; src.connect(c.destination);\n  const at = Math.max(playhead, c.currentTime + 0.02);\n  if (item.start === undefined) item.start = at;\n  src.start(at); playhead = at + buffer.duration; item.end = playhead;\n  sources.push(src); src.onended = () => { sources = sources.filter((s) => s !== src); };\n}\nfunction stopPlayback() {\n  for (const s of sources) s.stop();\n  sources = []; playhead = 0;\n  if (item && item.start !== undefined) {\n    const heard = Math.max(0, Math.min(audio().currentTime, item.end) - item.start) * 1000;\n    send({ type: 'playback', itemId: item.id, playedMs: Math.round(heard) });\n  }\n  item = undefined;\n}\nfunction interrupt() {\n  closedEpoch = Math.max(closedEpoch, epoch);\n  stopPlayback();\n  send({ type: 'interrupt', epoch, reason: 'user' });\n}\n\nasync function runTool(m) {\n  const tool = tools[m.name];\n  if (m.epoch <= closedEpoch) return send({ type: 'tool.result', callId: m.callId, epoch: m.epoch, ok: false, code: 'INTERRUPTED', message: 'interrupted' });\n  if (!tool) return send({ type: 'tool.result', callId: m.callId, epoch: m.epoch, ok: false, code: 'FAILED', message: 'unknown tool' });\n  try {\n    const output = await tool.run(m.arguments);\n    send({ type: 'tool.result', callId: m.callId, epoch: m.epoch, ok: true, output });\n    note('tool ' + m.name + ' ' + JSON.stringify(m.arguments));\n  } catch (e) {\n    send({ type: 'tool.result', callId: m.callId, epoch: m.epoch, ok: false, code: 'FAILED', message: String(e) });\n  }\n}\n\nws.onmessage = (e) => {\n  if (typeof e.data !== 'string') {\n    const bytes = new Uint8Array(e.data);\n    if (bytes[0] === 2 && item) play(bytes.subarray(5));\n    return;\n  }\n  const m = JSON.parse(e.data);\n  if (m.seq > 0) { if (m.seq <= lastSeq) return; lastSeq = m.seq; }\n  switch (m.type) {\n    case 'ready': $('status').textContent = 'session ' + m.sessionId; break;\n    case 'turn.state':\n      epoch = Math.max(epoch, m.epoch); $('state').textContent = m.state;\n      if (m.state === 'listening' && sources.length) stopPlayback();\n      break;\n    case 'transcript': line(m.itemId, m.role, m.text); break;\n    case 'text': line(m.responseId, 'agent', ((lines.get(m.responseId)?.textContent ?? 'Agent: ').slice(7)) + m.delta); break;\n    case 'audio.start': item = m.epoch > closedEpoch ? { id: m.itemId } : undefined; break;\n    case 'tool.call': runTool(m); break;\n    case 'interrupted': closedEpoch = Math.max(closedEpoch, m.epoch); stopPlayback(); note('interrupted'); break;\n    case 'error': note('error ' + m.code + ': ' + m.message); break;\n  }\n};\n\n$('form').onsubmit = (e) => {\n  e.preventDefault();\n  const text = $('text').value.trim();\n  if (!text) return;\n  stopPlayback(); line('local-' + seq, 'user', text); send({ type: 'text', text }); $('text').value = '';\n};\n$('stop').onclick = interrupt;\n\n// Microphone: AudioWorklet \u2192 20 ms PCM16 frames \u2192 [kind=1][seq u32 BE][payload].\nconst WORKLET = `class Capture extends AudioWorkletProcessor {\n  constructor() { super(); this.buf = new Int16Array(${FRAME}); this.n = 0; }\n  process(inputs) {\n    const ch = inputs[0][0];\n    if (ch) for (let i = 0; i < ch.length; i++) {\n      const s = Math.max(-1, Math.min(1, ch[i]));\n      this.buf[this.n++] = s < 0 ? s * 32768 : s * 32767;\n      if (this.n === ${FRAME}) { this.port.postMessage(this.buf.buffer.slice(0)); this.n = 0; }\n    }\n    return true;\n  }\n}\nregisterProcessor('vrt-capture', Capture);`;\nlet mic;\n$('mic').onclick = async () => {\n  if (mic) { mic.getTracks().forEach((t) => t.stop()); mic = undefined; $('mic').textContent = 'Start mic'; return; }\n  const c = audio();\n  await c.resume();\n  mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, channelCount: 1 } });\n  await c.audioWorklet.addModule(URL.createObjectURL(new Blob([WORKLET], { type: 'text/javascript' })));\n  const node = new AudioWorkletNode(c, 'vrt-capture');\n  node.port.onmessage = ({ data }) => {\n    if (!mic || ws.readyState !== 1) return;\n    const frame = new Uint8Array(5 + data.byteLength);\n    frame[0] = 1; new DataView(frame.buffer).setUint32(1, audioSeq++ >>> 0); frame.set(new Uint8Array(data), 5);\n    ws.send(frame);\n  };\n  c.createMediaStreamSource(mic).connect(node);\n  $('mic').textContent = 'Stop mic';\n};\n</script>\n</body>\n</html>\n";

package/dist/node/debug-page.js

@@ -0,0 +1,181 @@
+/**
+ * Microphone debug page served by `serve()` at `/`. A dependency-free protocol v1
+ * client: text and voice in, transcripts and audio out, a barge-in interrupt and
+ * one app-side tool (`set_accent_color`) behind an epoch gate.
+ */
+export const DEBUG_PAGE = /* html */ `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width,initial-scale=1">
+<title>vRealTime debug</title>
+<style>
+  :root { --accent: #2563eb; --bg: #fff; --fg: #111827; --muted: #6b7280; --line: #e5e7eb; color-scheme: light dark; }
+  @media (prefers-color-scheme: dark) { :root { --bg: #0b0f17; --fg: #e5e7eb; --muted: #9ca3af; --line: #1f2937; } }
+  body { margin: 0; background: var(--bg); color: var(--fg); font: 15px/1.5 system-ui, sans-serif; }
+  main { max-width: 720px; margin: 0 auto; padding: 16px; }
+  h1 { font-size: 20px; margin: 0 0 4px; color: var(--accent); }
+  .muted { color: var(--muted); font-size: 13px; }
+  .row { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; margin: 12px 0; }
+  button { background: var(--accent); color: #fff; border: 0; border-radius: 6px; padding: 8px 14px; font: inherit; cursor: pointer; }
+  button.secondary { background: transparent; color: var(--fg); border: 1px solid var(--line); }
+  form { display: flex; gap: 8px; }
+  input { flex: 1; min-width: 0; padding: 8px; border: 1px solid var(--line); border-radius: 6px; background: transparent; color: inherit; font: inherit; }
+  #state { padding: 2px 8px; border-radius: 99px; border: 1px solid var(--accent); color: var(--accent); font-size: 13px; }
+  #log { border-top: 1px solid var(--line); margin-top: 12px; padding-top: 8px; }
+  #log p { margin: 6px 0; } .user { font-weight: 600; } .note { color: var(--muted); font-size: 13px; }
+</style>
+</head>
+<body>
+<main>
+  <h1>vRealTime</h1>
+  <div class="muted" id="status">connecting…</div>
+  <div class="row">
+    <button id="mic">Start mic</button>
+    <button id="stop" class="secondary">Interrupt</button>
+    <span id="state">idle</span>
+  </div>
+  <form id="form"><input id="text" placeholder="Type a message" autocomplete="off"><button>Send</button></form>
+  <div id="log"></div>
+</main>
+<script type="module">
+const RATE = 24000, FRAME = 480, TRANSIENT = ['hello', 'ack', 'ping'];
+const $ = (id) => document.getElementById(id);
+let seq = 0, lastSeq = 0, epoch = 0, closedEpoch = -1, audioSeq = 0;
+
+const tools = {
+  set_accent_color: {
+    description: 'Set the page accent color to any CSS color.',
+    parameters: { type: 'object', properties: { color: { type: 'string' } }, required: ['color'] },
+    run: ({ color }) => { document.documentElement.style.setProperty('--accent', color); return { color }; },
+  },
+};
+
+const ws = new WebSocket(location.origin.replace(/^http/, 'ws') + location.pathname);
+ws.binaryType = 'arraybuffer';
+const send = (m) => { if (ws.readyState === 1) ws.send(JSON.stringify({ v: 1, seq: TRANSIENT.includes(m.type) ? 0 : ++seq, ...m })); };
+ws.onopen = () => send({
+  type: 'hello',
+  tools: Object.entries(tools).map(([name, t]) => ({ name, description: t.description, parameters: t.parameters })),
+  input: { encoding: 'pcm16', sampleRate: RATE }, output: { encoding: 'pcm16', sampleRate: RATE },
+});
+ws.onclose = (e) => { $('status').textContent = 'disconnected (' + e.code + ')'; };
+setInterval(() => { send({ type: 'ping' }); if (lastSeq) send({ type: 'ack', upTo: lastSeq }); }, 10000);
+
+const lines = new Map();
+function line(id, role, text) {
+  let el = lines.get(id);
+  if (!el) { el = document.createElement('p'); el.className = role; $('log').append(el); lines.set(id, el); }
+  el.textContent = (role === 'user' ? 'You: ' : 'Agent: ') + text;
+  el.scrollIntoView({ block: 'end' });
+}
+function note(text) { const el = document.createElement('p'); el.className = 'note'; el.textContent = text; $('log').append(el); }
+
+// Playback: schedule PCM16 chunks back to back; track what was actually heard per item.
+let ctx, playhead = 0, sources = [], item;
+const audio = () => (ctx ??= new AudioContext({ sampleRate: RATE }));
+function play(bytes) {
+  const c = audio(), n = bytes.byteLength >> 1, view = new DataView(bytes.buffer, bytes.byteOffset, n * 2);
+  const buffer = c.createBuffer(1, n, RATE), data = buffer.getChannelData(0);
+  for (let i = 0; i < n; i++) data[i] = view.getInt16(i * 2, true) / 32768;
+  const src = c.createBufferSource();
+  src.buffer = buffer; src.connect(c.destination);
+  const at = Math.max(playhead, c.currentTime + 0.02);
+  if (item.start === undefined) item.start = at;
+  src.start(at); playhead = at + buffer.duration; item.end = playhead;
+  sources.push(src); src.onended = () => { sources = sources.filter((s) => s !== src); };
+}
+function stopPlayback() {
+  for (const s of sources) s.stop();
+  sources = []; playhead = 0;
+  if (item && item.start !== undefined) {
+    const heard = Math.max(0, Math.min(audio().currentTime, item.end) - item.start) * 1000;
+    send({ type: 'playback', itemId: item.id, playedMs: Math.round(heard) });
+  }
+  item = undefined;
+}
+function interrupt() {
+  closedEpoch = Math.max(closedEpoch, epoch);
+  stopPlayback();
+  send({ type: 'interrupt', epoch, reason: 'user' });
+}
+
+async function runTool(m) {
+  const tool = tools[m.name];
+  if (m.epoch <= closedEpoch) return send({ type: 'tool.result', callId: m.callId, epoch: m.epoch, ok: false, code: 'INTERRUPTED', message: 'interrupted' });
+  if (!tool) return send({ type: 'tool.result', callId: m.callId, epoch: m.epoch, ok: false, code: 'FAILED', message: 'unknown tool' });
+  try {
+    const output = await tool.run(m.arguments);
+    send({ type: 'tool.result', callId: m.callId, epoch: m.epoch, ok: true, output });
+    note('tool ' + m.name + ' ' + JSON.stringify(m.arguments));
+  } catch (e) {
+    send({ type: 'tool.result', callId: m.callId, epoch: m.epoch, ok: false, code: 'FAILED', message: String(e) });
+  }
+}
+
+ws.onmessage = (e) => {
+  if (typeof e.data !== 'string') {
+    const bytes = new Uint8Array(e.data);
+    if (bytes[0] === 2 && item) play(bytes.subarray(5));
+    return;
+  }
+  const m = JSON.parse(e.data);
+  if (m.seq > 0) { if (m.seq <= lastSeq) return; lastSeq = m.seq; }
+  switch (m.type) {
+    case 'ready': $('status').textContent = 'session ' + m.sessionId; break;
+    case 'turn.state':
+      epoch = Math.max(epoch, m.epoch); $('state').textContent = m.state;
+      if (m.state === 'listening' && sources.length) stopPlayback();
+      break;
+    case 'transcript': line(m.itemId, m.role, m.text); break;
+    case 'text': line(m.responseId, 'agent', ((lines.get(m.responseId)?.textContent ?? 'Agent: ').slice(7)) + m.delta); break;
+    case 'audio.start': item = m.epoch > closedEpoch ? { id: m.itemId } : undefined; break;
+    case 'tool.call': runTool(m); break;
+    case 'interrupted': closedEpoch = Math.max(closedEpoch, m.epoch); stopPlayback(); note('interrupted'); break;
+    case 'error': note('error ' + m.code + ': ' + m.message); break;
+  }
+};
+
+$('form').onsubmit = (e) => {
+  e.preventDefault();
+  const text = $('text').value.trim();
+  if (!text) return;
+  stopPlayback(); line('local-' + seq, 'user', text); send({ type: 'text', text }); $('text').value = '';
+};
+$('stop').onclick = interrupt;
+
+// Microphone: AudioWorklet → 20 ms PCM16 frames → [kind=1][seq u32 BE][payload].
+const WORKLET = \`class Capture extends AudioWorkletProcessor {
+  constructor() { super(); this.buf = new Int16Array(\${FRAME}); this.n = 0; }
+  process(inputs) {
+    const ch = inputs[0][0];
+    if (ch) for (let i = 0; i < ch.length; i++) {
+      const s = Math.max(-1, Math.min(1, ch[i]));
+      this.buf[this.n++] = s < 0 ? s * 32768 : s * 32767;
+      if (this.n === \${FRAME}) { this.port.postMessage(this.buf.buffer.slice(0)); this.n = 0; }
+    }
+    return true;
+  }
+}
+registerProcessor('vrt-capture', Capture);\`;
+let mic;
+$('mic').onclick = async () => {
+  if (mic) { mic.getTracks().forEach((t) => t.stop()); mic = undefined; $('mic').textContent = 'Start mic'; return; }
+  const c = audio();
+  await c.resume();
+  mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, channelCount: 1 } });
+  await c.audioWorklet.addModule(URL.createObjectURL(new Blob([WORKLET], { type: 'text/javascript' })));
+  const node = new AudioWorkletNode(c, 'vrt-capture');
+  node.port.onmessage = ({ data }) => {
+    if (!mic || ws.readyState !== 1) return;
+    const frame = new Uint8Array(5 + data.byteLength);
+    frame[0] = 1; new DataView(frame.buffer).setUint32(1, audioSeq++ >>> 0); frame.set(new Uint8Array(data), 5);
+    ws.send(frame);
+  };
+  c.createMediaStreamSource(mic).connect(node);
+  $('mic').textContent = 'Stop mic';
+};
+</script>
+</body>
+</html>
+`;