@memberjunction/ai-realtime-client 0.0.1 → 5.42.0

package/dist/generic/baseRealtimeClient.js

@@ -0,0 +1,226 @@
+import { REALTIME_AUDIO_BIN_COUNT } from '../audio/audioMeter.js';
+/**
+ * Abstract base class for **browser-side, provider-direct** realtime (voice) clients.
+ *
+ * This is the client-side mirror of the server's `BaseRealtimeModel` pattern: the server
+ * mints an ephemeral credential + session config (`ClientRealtimeSessionConfig`) via its
+ * provider driver, and the browser resolves the matching CLIENT driver through the
+ * MemberJunction `ClassFactory` using the config's `Provider` string as the registration
+ * key (e.g. `'openai'`):
+ *
+ * ```typescript
+ * const client = MJGlobal.Instance.ClassFactory.CreateInstance<BaseRealtimeClient>(
+ *     BaseRealtimeClient, startResult.Provider);
+ * ```
+ *
+ * Concrete drivers (e.g. {@link OpenAIRealtimeClient}) own ALL provider wire concerns:
+ * transport (WebRTC/WebSocket), event-name translation, the response state machine
+ * (never letting a tool-result reply collide with an in-flight response), narration-kind
+ * tagging, and audible-playback tracking. Hosts own POLICY: when to narrate, what
+ * instructions to speak, transcript persistence, and UI state.
+ *
+ * All `On*` methods register a single handler (matching the server `IRealtimeSession`
+ * style); registering again replaces the previous handler.
+ *
+ * ## DRIVER AUTHOR OBLIGATIONS
+ *
+ * Hard-won, provider-independent rules every client driver MUST honor (the client-side
+ * mirror of the obligations documented on the server's `BaseRealtimeModel` in
+ * `@memberjunction/ai`). Most were paid for in live debugging; do not relearn them:
+ *
+ * 1. **Silent exit from `'speaking'` after a tool call.** When the model emits a tool
+ *    call, the host typically shows its own busy ("thinking") indicator while the tool
+ *    executes. The driver must leave the `'speaking'` state *silently* (no state
+ *    emission) at tool-call time so the turn's trailing frames don't clobber the host's
+ *    indicator.
+ * 2. **Busy-flag release on tool-call emission (deadlock guard).** A tool-call frame
+ *    means the model has yielded the floor pending the result. Any internal "response
+ *    active" flag MUST be cleared then — otherwise the eventual {@link SendToolResult}
+ *    (or a queued send) deadlocks waiting for a turn boundary that will never arrive
+ *    until after the result is sent.
+ * 3. **Playback flush + honest {@link IsAudioPlaying} on interruption.** On a true
+ *    barge-in the driver must flush any locally-owned audio playback and report
+ *    `IsAudioPlaying === false` promptly — stale queued audio after an interruption is
+ *    a product bug, not a nicety. The same flush applies to {@link CancelActiveResponse}.
+ * 4. **{@link SendText} must NOT echo a user transcript — and implies barge-in.** The
+ *    driver must not synthesize a user-role transcript event for injected text (the host
+ *    owns the local echo and would render the message twice), and an active spoken
+ *    response is cancelled via the driver's own cancel path before the text is injected
+ *    (see {@link CancelActiveResponse}).
+ * 5. **Tool-result delivery invariant.** Every result fed back via {@link SendToolResult}
+ *    must EVENTUALLY be voiced/processed by the model and never dropped. If the provider
+ *    rejects overlapping generation triggers, the driver queues the result's trigger
+ *    behind the in-flight response and flushes it at the next turn boundary. The result
+ *    must also EXPLICITLY trigger generation on providers that don't auto-continue after
+ *    a tool response: OpenAI requires an explicit `response.create`; Gemini auto-continues
+ *    UNLESS prior client content (a context note's `turnComplete: false`) left the turn
+ *    open, in which case the driver must commit the turn or the model stays silent.
+ * 6. **Token/credential expiry surfaces as a FATAL error.** When the ephemeral credential
+ *    dies (expiry, revocation, unexpected socket loss), the driver must surface it through
+ *    {@link OnError} with `Fatal: true` so the host tears down cleanly instead of idling
+ *    forever on a dead connection.
+ * 7. **`'listening'` only after the session config is applied.** The driver must not
+ *    report the session as listening until the server-built session config (system prompt
+ *    + tools) has actually been applied to the provider socket — otherwise early turns run
+ *    against an unconfigured model.
+ * 8. **`SessionConfig` is a private pact.** The `ClientRealtimeSessionConfig.SessionConfig`
+ *    payload is authored by the same-keyed SERVER driver and consumed only by this client
+ *    driver; its shape may change between the two driver halves without notice. Hosts and
+ *    intermediaries treat it as an opaque blob — and so must any code outside the matching
+ *    driver pair.
+ * 9. **Audio metering is a CAPABILITY, not an obligation — but wire what you own.** A
+ *    driver that owns (or can tap) an audio plane should attach
+ *    {@link BaseRealtimeClient.attachInputAudioMeter} / {@link attachOutputAudioMeter} so
+ *    {@link GetAudioActivity} reports real levels: the mic stream everywhere
+ *    (`RealtimeAudioMeter.ForStream`), the shared `RealtimePcmPlayback.CreateMeter()` on
+ *    client-owned-audio drivers, the remote WebRTC stream on peer-connection drivers.
+ *    Meters must be released on disconnect ({@link closeAudioMeters}). A driver with no
+ *    tappable plane simply attaches nothing — hosts fall back to turn-state animation.
+ */
+export class BaseRealtimeClient {
+    constructor() {
+        // ── Audio-activity metering (capability surface — see driver obligation #9) ─
+        /** Meter over the USER's microphone, when the driver attached one. */
+        this.inputAudioMeter = null;
+        /** Meter over the AGENT's audio output, when the driver attached one. */
+        this.outputAudioMeter = null;
+    }
+    /**
+     * The session's current audible activity, or `null` when this driver attached no
+     * meters at all (hosts then keep their turn-state-driven visuals). Sampled by hosts
+     * per animation frame — implementations are cheap, allocation-light reads of an
+     * `AnalyserNode`; no per-call provider traffic.
+     */
+    GetAudioActivity() {
+        if (!this.inputAudioMeter && !this.outputAudioMeter) {
+            return null;
+        }
+        return {
+            InputLevel: this.inputAudioMeter ? this.inputAudioMeter.Level() : null,
+            OutputLevel: this.outputAudioMeter ? this.outputAudioMeter.Level() : null,
+            InputBins: this.inputAudioMeter ? this.inputAudioMeter.Bins(REALTIME_AUDIO_BIN_COUNT) : null,
+            OutputBins: this.outputAudioMeter ? this.outputAudioMeter.Bins(REALTIME_AUDIO_BIN_COUNT) : null
+        };
+    }
+    /** Attaches (replacing + closing any previous) the USER-microphone meter. `null` detaches. */
+    attachInputAudioMeter(meter) {
+        this.inputAudioMeter?.Close();
+        this.inputAudioMeter = meter;
+    }
+    /** Attaches (replacing + closing any previous) the AGENT-output meter. `null` detaches. */
+    attachOutputAudioMeter(meter) {
+        this.outputAudioMeter?.Close();
+        this.outputAudioMeter = meter;
+    }
+    /** Releases both meters — every driver calls this from its disconnect/teardown path. */
+    closeAudioMeters() {
+        this.attachInputAudioMeter(null);
+        this.attachOutputAudioMeter(null);
+    }
+    // ── Handler registration ──────────────────────────────────────────────────
+    /**
+     * Registers the (single) transcript handler. Receives interim deltas (`IsFinal: false`)
+     * and final turn transcripts (`IsFinal: true`) for both roles, tagged with their
+     * {@link RealtimeClientTranscript.Kind}.
+     */
+    OnTranscript(handler) {
+        this.transcriptHandler = handler;
+    }
+    /**
+     * Registers the (single) tool-call handler. The host executes the tool and calls
+     * {@link SendToolResult} with the outcome.
+     */
+    OnToolCall(handler) {
+        this.toolCallHandler = handler;
+    }
+    /**
+     * Registers the (single) state-change handler. States may be re-emitted (e.g. repeated
+     * `'listening'` on successive barge-ins); hosts should treat emissions as idempotent.
+     */
+    OnStateChange(handler) {
+        this.stateChangeHandler = handler;
+    }
+    /**
+     * Registers the (single) error handler. Receives both fatal transport errors (which are
+     * also followed by an `'error'` state change) and non-fatal provider error frames.
+     */
+    OnError(handler) {
+        this.errorHandler = handler;
+    }
+    /**
+     * Registers the (single) interruption handler.
+     *
+     * **True barge-in only:** fires ONLY when USER INPUT CUT OFF ACTIVE MODEL OUTPUT — a
+     * model response in flight or audio audibly playing when the user took the floor. A
+     * user simply taking their normal turn while the model is idle is NOT an interruption,
+     * and drivers must not report it as one (e.g. a raw "speech started" frame must be
+     * gated on whether a response is actually active or audio is playing).
+     *
+     * On interruption, drivers must also flush locally-owned playback and report
+     * {@link IsAudioPlaying} `=== false` promptly (driver obligation #3). Hosts typically
+     * use this hook to abort in-flight delegated work so a stale result is never narrated
+     * into a conversation that has moved on.
+     */
+    OnInterruption(handler) {
+        this.interruptionHandler = handler;
+    }
+    /**
+     * Registers the (single) usage handler.
+     *
+     * Emissions carry token **deltas** for the response/turn that just completed (see
+     * {@link RealtimeClientUsage} — deltas preferred; cumulative-only providers must convert
+     * in the driver). Hosts accumulate and relay/persist on their own cadence (e.g. the voice
+     * session service debounces a `RelayRealtimeUsage` mutation onto the co-agent prompt run).
+     *
+     * **Optional capability:** drivers whose provider exposes no usage telemetry simply never
+     * emit — registering a handler is always safe, but hosts must not assume emissions arrive.
+     * See {@link RealtimeClientUsage} for per-driver availability.
+     */
+    OnUsage(handler) {
+        this.usageHandler = handler;
+    }
+    /**
+     * Registers the (single) remote-VIDEO handler — the model/avatar's video track for a VIDEO session
+     * (a talking-head the host renders, e.g. as the agent's tile). Invoked once the provider publishes
+     * its video track.
+     *
+     * **Optional capability:** audio-only drivers (the default) never emit — registering is always safe,
+     * but hosts must not assume a video track arrives. Video-capable drivers
+     * ({@link BaseRealtimeModel.SupportsVideo}) call {@link emitRemoteVideo} when the track is live.
+     *
+     * @param handler Invoked with the remote video `MediaStream` when it becomes available.
+     */
+    OnRemoteVideo(handler) {
+        this.remoteVideoHandler = handler;
+    }
+    // ── Protected emit helpers for concrete drivers ───────────────────────────
+    /** Emits a transcript event to the registered handler (if any). */
+    emitTranscript(transcript) {
+        this.transcriptHandler?.(transcript);
+    }
+    /** Emits a tool-call request to the registered handler (if any). */
+    emitToolCall(call) {
+        this.toolCallHandler?.(call);
+    }
+    /** Emits a state change to the registered handler (if any). */
+    emitStateChange(state) {
+        this.stateChangeHandler?.(state);
+    }
+    /** Emits an error to the registered handler (if any). */
+    emitError(error) {
+        this.errorHandler?.(error);
+    }
+    /** Emits a true barge-in interruption to the registered handler (if any). */
+    emitInterruption() {
+        this.interruptionHandler?.();
+    }
+    /** Emits a usage update (token deltas for a completed response/turn) to the registered handler (if any). */
+    emitUsage(usage) {
+        this.usageHandler?.(usage);
+    }
+    /** Emits the model/avatar's remote video stream to the registered handler (video drivers only). */
+    emitRemoteVideo(stream) {
+        this.remoteVideoHandler?.(stream);
+    }
+}
+//# sourceMappingURL=baseRealtimeClient.js.map

package/dist/generic/baseRealtimeClient.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"baseRealtimeClient.js","sourceRoot":"","sources":["../../src/generic/baseRealtimeClient.ts"],"names":[],"mappings":"AACA,OAAO,EAAuB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AA2HpF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6EG;AACH,MAAM,OAAgB,kBAAkB;IAAxC;QAUI,+EAA+E;QAC/E,sEAAsE;QAC9D,oBAAe,GAA+B,IAAI,CAAC;QAC3D,yEAAyE;QACjE,qBAAgB,GAA+B,IAAI,CAAC;IAgRhE,CAAC;IA9QG;;;;;OAKG;IACI,gBAAgB;QACnB,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO;YACH,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YACtE,WAAW,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YACzE,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,IAAI;YAC5F,UAAU,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,IAAI;SAClG,CAAC;IACN,CAAC;IAED,8FAA8F;IACpF,qBAAqB,CAAC,KAAiC;QAC7D,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED,2FAA2F;IACjF,sBAAsB,CAAC,KAAiC;QAC9D,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAClC,CAAC;IAED,wFAAwF;IAC9E,gBAAgB;QACtB,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IAoHD,6EAA6E;IAE7E;;;;OAIG;IACI,YAAY,CAAC,OAAuD;QACvE,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC;IACrC,CAAC;IAED;;;OAGG;IACI,UAAU,CAAC,OAA+C;QAC7D,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;IACnC,CAAC;IAED;;;OAGG;IACI,aAAa,CAAC,OAA6C;QAC9D,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACtC,CAAC;IAED;;;OAGG;IACI,OAAO,CAAC,OAA6C;QACxD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;OAaG;IACI,cAAc,CAAC,OAAmB;QACrC,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;IACvC,CAAC;IAED;;;;;;;;;;;OAWG;IACI,OAAO,CAAC,OAA6C;QACxD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED;;;;;;;;;;OAUG;IACI,aAAa,CAAC,OAAsC;QACvD,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACtC,CAAC;IAED,6EAA6E;IAE7E,mEAAmE;IACzD,cAAc,CAAC,UAAoC;QACzD,IAAI,CAAC,iBAAiB,EAAE,CAAC,UAAU,CAAC,CAAC;IACzC,CAAC;IAED,oEAAoE;IAC1D,YAAY,CAAC,IAA4B;QAC/C,IAAI,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,+DAA+D;IACrD,eAAe,CAAC,KAA0B;QAChD,IAAI,CAAC,kBAAkB,EAAE,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,yDAAyD;IAC/C,SAAS,CAAC,KAA0B;QAC1C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,6EAA6E;IACnE,gBAAgB;QACtB,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;IACjC,CAAC;IAED,4GAA4G;IAClG,SAAS,CAAC,KAA0B;QAC1C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,mGAAmG;IACzF,eAAe,CAAC,MAAmB;QACzC,IAAI,CAAC,kBAAkB,EAAE,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;CACJ"}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+export * from './generic/baseRealtimeClient.js';
+export * from './audio/pcmUtils.js';
+export * from './audio/audioMeter.js';
+export * from './audio/pcmPlayback.js';
+export * from './audio/micCapture.js';
+export * from './drivers/openAIRealtimeClient.js';
+export * from './drivers/geminiRealtimeClient.js';
+export * from './drivers/elevenLabsRealtimeClient.js';
+export * from './drivers/assemblyAIRealtimeClient.js';
+export * from './drivers/xaiRealtimeClient.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,8BAA8B,CAAC;AAC7C,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oCAAoC,CAAC;AACnD,cAAc,oCAAoC,CAAC;AACnD,cAAc,6BAA6B,CAAC"}

package/dist/index.js

@@ -0,0 +1,11 @@
+export * from './generic/baseRealtimeClient.js';
+export * from './audio/pcmUtils.js';
+export * from './audio/audioMeter.js';
+export * from './audio/pcmPlayback.js';
+export * from './audio/micCapture.js';
+export * from './drivers/openAIRealtimeClient.js';
+export * from './drivers/geminiRealtimeClient.js';
+export * from './drivers/elevenLabsRealtimeClient.js';
+export * from './drivers/assemblyAIRealtimeClient.js';
+export * from './drivers/xaiRealtimeClient.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,8BAA8B,CAAC;AAC7C,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oCAAoC,CAAC;AACnD,cAAc,oCAAoC,CAAC;AACnD,cAAc,6BAA6B,CAAC"}

package/package.json

@@ -1,10 +1,31 @@
 {
   "name": "@memberjunction/ai-realtime-client",
-  "version": "0.0.1",
-  "description": "OIDC trusted publishing setup package for @memberjunction/ai-realtime-client",
-  "keywords": [
-    "oidc",
-    "trusted-publishing",
-    "setup"
-  ]
+  "type": "module",
+  "version": "5.42.0",
+  "description": "MemberJunction: Framework-agnostic browser client abstraction for provider-direct realtime (voice) sessions - BaseRealtimeClient + provider drivers",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "/dist"
+  ],
+  "scripts": {
+    "build": "tsc && tsc-alias -f",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "author": "MemberJunction.com",
+  "license": "ISC",
+  "dependencies": {
+    "@memberjunction/global": "5.42.0",
+    "@memberjunction/ai": "5.42.0",
+    "@google/genai": "^1.40.0"
+  },
+  "devDependencies": {
+    "@types/node": "24.10.11",
+    "typescript": "^5.9.3"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/MemberJunction/MJ"
+  }
 }