@memberjunction/ai-agents 6.2.0-edge.1 → 6.2.0-edge.3

package/dist/realtime/realtime-client-session-service.js

@@ -33,15 +33,28 @@ import { LogError, LogStatus, RunView } from '@memberjunction/core';
 import { MJGlobal, MJLruCache, UUIDsEqual, EscapeSQLString } from '@memberjunction/global';
 import { ActionEngineServer } from '@memberjunction/actions';
 import { RunActionParams } from '@memberjunction/actions-base';
-import { BaseRealtimeModel, GetAIAPIKey } from '@memberjunction/ai';
+import { BaseRealtimeModel, GetAIAPIKey, CredentialScopeAllows } from '@memberjunction/ai';
 import { FormatAppContextNote, ResolvePromptRunUserID } from '@memberjunction/ai-core-plus';
 import { AIEngine } from '@memberjunction/aiengine';
 import { AgentMemoryContextBuilder } from '../agent-memory-context-builder.js';
 import { AgentRunner } from '../AgentRunner.js';
+import { DelegationNarrator } from './realtime-delegation-narrator.js';
+import { FilterAllowedAgentsByCanRun } from './realtime-coagent-resolution.js';
 import { RealtimeToolBroker, INVOKE_TARGET_AGENT_TOOL_NAME, INVOKE_TARGET_AGENT_DESCRIPTION, BuildRealtimeAgentFraming } from './realtime-tool-broker.js';
 import { NARRATION_PROMPT_NAME, LEGACY_NARRATION_PROMPT_NAME, ResolveNarrationInstructionsTemplate } from './realtime-narration.js';
 import { BuildVoiceMannerSection, BuildAppRealtimeOverridesJson, DeepMergeConfigs, GetModelCatalogSessionSettings, GetDisclosureForTarget, GetNarrationPaceMs, GetProviderVoiceSettings, GetSessionTuningSettings, ResolveEffectiveRealtimeConfig, MatchProviderVoiceSettings, GetDirectActionsConfig, IsActionAllowedForDirectInvocation } from './realtime-coagent-config.js';
 import { SelectRealtimeVendorForModel } from './realtime-vendor-resolution.js';
+/** Runtime handles by their realtime session, so the layer that only holds the session can reach its runtime. */
+const bridgeRuntimes = new WeakMap();
+/**
+ * Returns the runtime wired onto a bridged realtime session by
+ * {@link RealtimeClientSessionService.WireBridgeRealtimeSession}, or `undefined` for a session that was never
+ * wired. Lets a host that only holds the {@link IRealtimeSession} (the telephony services) cancel delegations on
+ * barge-in and install its local tool handler.
+ */
+export function GetBridgeRealtimeRuntime(session) {
+    return bridgeRuntimes.get(session);
+}
 /**
  * Logs when a configuration authored per-provider voice settings but NONE matched the vendor that
  * actually ran — those settings are dropped, and pre-#3530 that happened with no trace at all: the
@@ -115,12 +128,27 @@ export class ToolArgumentsError extends Error {
  */
 export class RealtimeClientSessionService {
     constructor() {
-        /** Maps session id -> (wireName -> MJActionEntityExtended) built during tool projection. */
-        this.sessionWireActionMaps = new Map();
-        /** Maps targetAgentID -> (wireName -> MJActionEntityExtended) fallback built during tool projection. */
-        this.targetWireActionMaps = new Map();
-        /** Maps session id -> direct actions config resolved during session prep. */
-        this.sessionDirectConfigs = new Map();
+        /**
+         * Maps session id -> (wireName -> MJActionEntityExtended) built during tool projection.
+         * Bounded with `MJLruCache` (rather than a plain `Map`): the resolver holds ONE shared service
+         * instance for the process lifetime (see class doc above), and there is no "session ended" hook
+         * this service can key eviction off — every voice session ever prepared would otherwise leave an
+         * entry here forever. Same TTL/maxSize rationale as {@link promptRunWriteChains} below.
+         */
+        this.sessionWireActionMaps = new MJLruCache({
+            maxSize: 5_000,
+            ttlMs: 4 * 60 * 60 * 1000, // 4h — generous vs. any realistic voice-session duration
+        });
+        /** Maps targetAgentID -> (wireName -> MJActionEntityExtended) fallback built during tool projection. Bounded for the same reason as {@link sessionWireActionMaps}. */
+        this.targetWireActionMaps = new MJLruCache({
+            maxSize: 5_000,
+            ttlMs: 4 * 60 * 60 * 1000,
+        });
+        /** Maps session id -> direct actions config resolved during session prep. Bounded for the same reason as {@link sessionWireActionMaps}. */
+        this.sessionDirectConfigs = new MJLruCache({
+            maxSize: 5_000,
+            ttlMs: 4 * 60 * 60 * 1000,
+        });
         /**
          * IN-FLIGHT DELEGATION REGISTRY — the server half of the client-direct CANCEL channel.
          *
@@ -300,6 +328,16 @@ export class RealtimeClientSessionService {
             // Prep must have succeeded before wiring; degrade to a tool-error fallback rather than throw.
             return this.wireBridgeFallbackRuntime(session);
         }
+        // The delegation set the model may reach, narrowed to what THIS run-as user may run (the browser path
+        // applies the same filter; without it a bridged call would reach colleagues the caller cannot).
+        const allowedAgents = await FilterAllowedAgentsByCanRun(prep.EffectiveConfig?.realtime?.allowedAgents, contextUser);
+        // Spoken progress while delegated work runs — the same pacing/wording the generic session runner uses.
+        const narrator = new DelegationNarrator({
+            GetSession: () => session,
+            NarrationInstructionsTemplate: this.resolveNarrationInstructionsTemplate(),
+            NarrationPaceMs: GetNarrationPaceMs(prep.EffectiveConfig) ?? undefined,
+        });
+        let localToolHandler;
         const promptID = this.resolveCoAgentSystemPrompt(coAgent).PromptID;
         const obs = await this.createCoAgentObservabilityRun(coAgent, promptID, resolution.ModelID, resolution.VendorID, input.UserID || contextUser?.ID, input.AgentSessionID, contextUser, provider, input.ConversationID);
         let finalized = false;
@@ -313,15 +351,18 @@ export class RealtimeClientSessionService {
         // Tool calls → the shared delegation entry point, then hand the serialized result back to the model.
         session.OnToolCall(async (call) => {
             try {
-                const result = await this.ExecuteRelayedTool({
-                    AgentSessionID: input.AgentSessionID,
-                    ParentRunID: obs?.CoAgentRunID,
-                    TargetAgentID: input.TargetAgentID,
-                    AllowedAgents: prep.EffectiveConfig?.realtime?.allowedAgents,
-                    DirectActions: prep.EffectiveConfig?.realtime?.directActions,
-                    Call: call,
-                }, contextUser, provider);
-                await session.SendToolResult(call.CallID, result.ResultJson);
+                const resultJson = localToolHandler?.Handles(call.ToolName)
+                    ? await localToolHandler.Execute(call)
+                    : (await narrator.Track(() => this.ExecuteRelayedTool({
+                        AgentSessionID: input.AgentSessionID,
+                        ParentRunID: obs?.CoAgentRunID,
+                        TargetAgentID: input.TargetAgentID,
+                        AllowedAgents: allowedAgents,
+                        DirectActions: prep.EffectiveConfig?.realtime?.directActions,
+                        OnProgress: (progress) => narrator.HandleProgress(progress),
+                        Call: call,
+                    }, contextUser, provider))).ResultJson;
+                await session.SendToolResult(call.CallID, resultJson);
             }
             catch (error) {
                 const message = error instanceof Error ? error.message : String(error);
@@ -333,11 +374,24 @@ export class RealtimeClientSessionService {
         // through the idempotent finalizer, so double-fire is harmless.
         const originalClose = session.Close.bind(session);
         session.Close = async () => {
+            narrator.Cancel();
             await finalize(true);
             await originalClose();
         };
         session.OnClose?.(() => { void finalize(true); });
-        return { CoAgentRunID: obs?.CoAgentRunID, PromptRunID: obs?.PromptRunID, Finalize: finalize };
+        const runtime = {
+            CoAgentRunID: obs?.CoAgentRunID,
+            PromptRunID: obs?.PromptRunID,
+            Finalize: finalize,
+            CancelInFlightDelegations: () => {
+                narrator.Cancel(); // a stale "still working on it" line must not be spoken over the caller
+                return this.CancelInFlightDelegations(input.AgentSessionID);
+            },
+            CancelPendingNarration: () => narrator.Cancel(),
+            SetLocalToolHandler: (handler) => { localToolHandler = handler; },
+        };
+        bridgeRuntimes.set(session, runtime);
+        return runtime;
     }
     /**
      * Degenerate {@link BridgeRealtimeRuntime} for the rare case wiring is attempted without a resolved
@@ -348,7 +402,14 @@ export class RealtimeClientSessionService {
         session.OnToolCall((call) => {
             void session.SendToolResult(call.CallID, JSON.stringify({ success: false, error: 'Tool execution is unavailable — the co-agent did not resolve. Let the user know.' }));
         });
-        return { Finalize: async () => { } };
+        const runtime = {
+            Finalize: async () => { },
+            CancelInFlightDelegations: () => 0,
+            CancelPendingNarration: () => { },
+            SetLocalToolHandler: () => { },
+        };
+        bridgeRuntimes.set(session, runtime);
+        return runtime;
     }
     /**
      * **The single source of truth for realtime session prep.** Builds the {@link RealtimeSessionParams}
@@ -1024,16 +1085,36 @@ export class RealtimeClientSessionService {
      * @returns The resolution outcome (resolution or failure reason).
      */
     async resolveModelForSession(input, coAgent, effectiveConfig) {
+        // One resolver for every branch below, built here rather than per branch so the requested-model,
+        // configured-preference and default paths cannot disagree about which credentials the session
+        // may use. It answers with the session's RUN keys only: resolveVendorAndInstantiate layers this
+        // service's getAPIKeyForDriver seam under it, giving run key → seam → (the seam's default) env.
+        // A resolver that fell back to the environment itself would answer before the seam, so a
+        // subclass that overrides the seam would lose to AI_VENDOR_API_KEY__<driver>.
+        const resolveRunKey = this.buildRunKeyResolver(input.APIKeys);
+        const scope = input.CredentialScope ?? 'Any';
         if (input.PreferredModelID) {
-            return this.resolvePreferredRealtimeModel(input.PreferredModelID);
+            return this.resolvePreferredRealtimeModel(input.PreferredModelID, resolveRunKey, scope);
         }
-        const fromConfig = this.resolveConfiguredModelPreference(effectiveConfig);
+        const fromConfig = this.resolveConfiguredModelPreference(effectiveConfig, resolveRunKey, scope);
         if (fromConfig) {
             return { Resolution: fromConfig };
         }
-        const resolution = await this.resolveRealtimeModel(coAgent);
+        const resolution = await this.resolveRealtimeModel(coAgent, resolveRunKey, scope);
         return resolution ? { Resolution: resolution } : { ErrorMessage: this.noModelMessage() };
     }
+    /**
+     * The session's run-scoped keys as a resolver: the key for a driver class when the run carries
+     * one, else `undefined` — no platform fallback. The fallback belongs to
+     * {@link getAPIKeyForDriver}, which {@link resolveVendorAndInstantiate} consults for any class
+     * this does not key.
+     *
+     * @param apiKeys The session's run keys ({@link PrepareClientSessionInput.APIKeys}), if any.
+     * @returns A resolver over those keys only.
+     */
+    buildRunKeyResolver(apiKeys) {
+        return (driverClass) => apiKeys?.find(k => k.driverClass === driverClass)?.apiKey || undefined;
+    }
     /**
      * Resolves the effective config's `realtime.modelPreference` (an `MJ: AI Models` Name OR ID)
      * into a usable realtime model. TOLERANT by design — this is a METADATA preference, so any
@@ -1042,9 +1123,11 @@ export class RealtimeClientSessionService {
      * explicit runtime choice ({@link resolvePreferredRealtimeModel}), which fails loud.
      *
      * @param effectiveConfig The resolved effective configuration.
+     * @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
+     * @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
      * @returns The resolution, or `null` when no preference is configured or it can't be satisfied.
      */
-    resolveConfiguredModelPreference(effectiveConfig) {
+    resolveConfiguredModelPreference(effectiveConfig, resolve, credentialScope = 'Any') {
         const preference = effectiveConfig?.realtime?.modelPreference;
         if (!preference) {
             return null;
@@ -1060,7 +1143,7 @@ export class RealtimeClientSessionService {
                 `'${model.Name}' but it is not an Active Realtime model — falling through to default resolution.`);
             return null;
         }
-        const resolution = this.resolveVendorAndInstantiate(model);
+        const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope);
         if (!resolution) {
             LogError(`RealtimeClientSessionService: configured realtime model preference '${model.Name}' has no usable ` +
                 'vendor DriverClass/API key — falling through to default resolution.');
@@ -1092,9 +1175,11 @@ export class RealtimeClientSessionService {
      * the caller's user explicitly chose this one.
      *
      * @param preferredModelID The `MJ: AI Models.ID` the user chose.
+     * @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
+     * @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
      * @returns The resolution outcome (resolution or a specific failure reason).
      */
-    resolvePreferredRealtimeModel(preferredModelID) {
+    resolvePreferredRealtimeModel(preferredModelID, resolve, credentialScope = 'Any') {
         const model = this.findModelByID(preferredModelID);
         if (!model) {
             return { ErrorMessage: `The requested realtime model (id '${preferredModelID}') was not found in AI model metadata.` };
@@ -1105,7 +1190,7 @@ export class RealtimeClientSessionService {
         if (!this.isRealtimeModel(model)) {
             return { ErrorMessage: `The requested model '${model.Name}' is not a Realtime model (its type is '${model.AIModelType}').` };
         }
-        const resolution = this.resolveVendorAndInstantiate(model);
+        const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope);
         if (!resolution) {
             return {
                 ErrorMessage: `The requested model '${model.Name}' has no active vendor with a usable DriverClass/API key ` +
@@ -1137,9 +1222,11 @@ export class RealtimeClientSessionService {
      * step can't be satisfied.
      *
      * @param coAgent The co-agent being voiced (reserved for future per-agent model preference).
+     * @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
+     * @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
      * @returns The resolved model + identifiers, or `null`.
      */
-    async resolveRealtimeModel(coAgent) {
+    async resolveRealtimeModel(coAgent, resolve, credentialScope = 'Any') {
         // Walk candidates in descending PowerRank, returning the FIRST that fully resolves to a usable
         // client-direct driver (active vendor + API key + ClassFactory driver + SupportsClientDirect).
         // Single-pick dead-ended whenever the highest-power model lacked a key or client-direct support
@@ -1147,7 +1234,7 @@ export class RealtimeClientSessionService {
         // surfaced "No usable Realtime model" instead of falling through to a model that works.
         const candidates = this.selectRealtimeModelCandidates(coAgent);
         for (const model of candidates) {
-            const resolution = this.resolveVendorAndInstantiate(model);
+            const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope);
             if (resolution && resolution.Model.SupportsClientDirect) {
                 return resolution;
             }
@@ -1158,15 +1245,31 @@ export class RealtimeClientSessionService {
      * Shared tail of model resolution: picks the vendor (with a usable API key) for an
      * already-chosen model entity and instantiates its realtime driver.
      *
+     * Key precedence, per driver class: `resolve` (the session's run keys), then this service's
+     * {@link getAPIKeyForDriver} seam, whose default is the environment key. Vendor selection and the
+     * mint use that one chain, so a session is never routed to a vendor it cannot then be minted on.
+     *
      * @param model The chosen model entity.
+     * @param resolve The session's run-scoped key resolver. Expected to answer with run keys only:
+     *   one that falls back to the environment itself answers before {@link getAPIKeyForDriver} and
+     *   so bypasses an override of it.
+     * @param credentialScope A scope that rules out the `'Environment'` source (`'RuntimeOnly'`) drops the
+     *   {@link getAPIKeyForDriver} seam: the run's keys are the whole chain, so a vendor they do not key
+     *   is never selected on the platform's key.
      * @returns The full resolution, or `null` when no vendor/key/driver can be satisfied.
      */
-    resolveVendorAndInstantiate(model) {
-        const vendor = this.selectRealtimeVendor(model.ID);
+    resolveVendorAndInstantiate(model, resolve, credentialScope = 'Any') {
+        // The run's keys first, then this service's own seam (which subclasses and tests override) —
+        // so a run-scoped credential wins without taking that seam away from anyone who replaced it.
+        // A scope that rules out the environment has no second step: the seam's default is the platform key.
+        const resolveKey = CredentialScopeAllows(credentialScope, 'Environment')
+            ? (driverClass) => resolve?.(driverClass) ?? this.getAPIKeyForDriver(driverClass)
+            : (driverClass) => resolve?.(driverClass);
+        const vendor = this.selectRealtimeVendor(model.ID, resolveKey);
         if (!vendor) {
             return null;
         }
-        const apiKey = this.getAPIKeyForDriver(vendor.DriverClass);
+        const apiKey = resolveKey(vendor.DriverClass);
         if (!apiKey) {
             return null;
         }
@@ -1187,6 +1290,9 @@ export class RealtimeClientSessionService {
     /**
      * Resolves the API key for a vendor driver class. **Overridable seam** (wraps the module-level
      * {@link GetAIAPIKey}) so tests can simulate present/absent keys without environment setup.
+     * Consulted for every driver class the session's run keys do not cover, so an override (a vault,
+     * a different key, or `undefined` to disable a vendor) applies whether or not the environment
+     * also holds a key.
      *
      * @param driverClass The vendor's `DriverClass`.
      * @returns The API key, or a falsy value when none is configured.
@@ -1225,10 +1331,11 @@ export class RealtimeClientSessionService {
      * THIS service's {@link getAPIKeyForDriver} so subclasses and tests keep their key-resolution seam.
      *
      * @param modelID The chosen model's id.
+     * @param resolve The complete key chain to select against; defaults to {@link getAPIKeyForDriver}.
      * @returns The vendor driver/api identifiers, or `null` when none has a usable key.
      */
-    selectRealtimeVendor(modelID) {
-        return SelectRealtimeVendorForModel(modelID, (driverClass) => this.getAPIKeyForDriver(driverClass));
+    selectRealtimeVendor(modelID, resolve) {
+        return SelectRealtimeVendorForModel(modelID, resolve ?? ((driverClass) => this.getAPIKeyForDriver(driverClass)));
     }
     /**
      * Resolves the DB-driven progress-narration instruction template: the Active `MJ: AI Prompts`
@@ -1267,7 +1374,7 @@ export class RealtimeClientSessionService {
         const combinedExtra = directTools.length > 0
             ? [...(input.ExtraTools ?? []), ...directTools]
             : input.ExtraTools;
-        const tools = this.buildStableToolSet(combinedExtra);
+        const tools = this.appendHostTools(this.buildStableToolSet(combinedExtra), input.HostTools);
         // Hoisted (rather than built inline at the return) so the mint log below can report the voice
         // that ACTUALLY reached the driver — see the `voice=` field. Same bag, built once.
         const configBag = this.buildSessionConfigBag(input, effectiveConfig, driverClass, modelID, modelVendorID);
@@ -1362,6 +1469,7 @@ export class RealtimeClientSessionService {
         const colleagues = this.buildColleaguesFromConfig(effectiveConfig, input.TargetAgentID);
         const framing = BuildRealtimeAgentFraming(targetName, this.buildInteractiveSurfaceFraming(input.ExtraTools), colleagues, hasDirectTools);
         const meetingFraming = this.buildMeetingFraming(input);
+        const hostFraming = input.HostFraming?.trim() ?? '';
         const coAgentPrompt = this.getCoAgentSystemPromptText(coAgent);
         const voiceManner = BuildVoiceMannerSection(effectiveConfig);
         const targetIdentity = this.formatTargetIdentity(target);
@@ -1369,7 +1477,7 @@ export class RealtimeClientSessionService {
         const priorTranscript = this.formatPriorTranscript(input.PriorTranscript);
         const history = this.formatConversationHistory(input.ConversationMessages);
         const memoryContext = await this.assembleMemoryContext(input, coAgent, contextUser, provider);
-        return [framing, meetingFraming, coAgentPrompt, voiceManner, targetIdentity, appContextSection, priorTranscript, history, memoryContext]
+        return [framing, meetingFraming, hostFraming, coAgentPrompt, voiceManner, targetIdentity, appContextSection, priorTranscript, history, memoryContext]
             .filter(part => part && part.trim().length > 0)
             .join('\n\n');
     }
@@ -1695,6 +1803,21 @@ export class RealtimeClientSessionService {
         }
         return result;
     }
+    /** Appends the host-declared tools to a stable tool set, dropping any whose name is already taken. */
+    appendHostTools(tools, hostTools) {
+        if (!hostTools || hostTools.length === 0) {
+            return tools;
+        }
+        const taken = new Set(tools.map((t) => t.Name.toLowerCase()));
+        const merged = [...tools];
+        for (const tool of hostTools) {
+            if (!taken.has(tool.Name.toLowerCase())) {
+                taken.add(tool.Name.toLowerCase());
+                merged.push(tool);
+            }
+        }
+        return merged;
+    }
     /**
      * Builds the {@link RealtimeToolBroker} for a relayed tool call, wiring `DelegateToTarget` to a
      * target-agent run and `ExecuteTool` to direct-action execution or a structured "not available" placeholder.
@@ -2017,12 +2140,12 @@ export class RealtimeClientSessionService {
         const allowedActions = candidateActions.filter(action => IsActionAllowedForDirectInvocation(action.Name, directConfig));
         const wireMap = this.BuildWireActionMap(allowedActions);
         if (agentSessionID) {
-            this.sessionWireActionMaps.set(agentSessionID, wireMap);
+            this.sessionWireActionMaps.Set(agentSessionID, wireMap);
             if (directConfig) {
-                this.sessionDirectConfigs.set(agentSessionID, directConfig);
+                this.sessionDirectConfigs.Set(agentSessionID, directConfig);
             }
         }
-        this.targetWireActionMaps.set(targetAgentID, wireMap);
+        this.targetWireActionMaps.Set(targetAgentID, wireMap);
         const tools = [];
         for (const [wireName, action] of wireMap.entries()) {
             tools.push(this.mapActionToToolDefinition(action, wireName));
@@ -2107,7 +2230,7 @@ export class RealtimeClientSessionService {
         }
         const candidateActions = this.getTargetAgentActions(target.ID);
         const candidateWireMap = this.BuildWireActionMap(candidateActions);
-        const action = (input?.AgentSessionID ? this.sessionWireActionMaps.get(input.AgentSessionID)?.get(call.ToolName) : undefined)
+        const action = (input?.AgentSessionID ? this.sessionWireActionMaps.Get(input.AgentSessionID)?.get(call.ToolName) : undefined)
             ?? candidateWireMap.get(call.ToolName)
             ?? Array.from(candidateWireMap.entries()).find(([w]) => w.toLowerCase() === call.ToolName.trim().toLowerCase())?.[1];
         if (!action) {
@@ -2118,7 +2241,7 @@ export class RealtimeClientSessionService {
             };
         }
         const directConfig = input?.DirectActions
-            ?? (input?.AgentSessionID ? this.sessionDirectConfigs.get(input.AgentSessionID) : undefined)
+            ?? (input?.AgentSessionID ? this.sessionDirectConfigs.Get(input.AgentSessionID) : undefined)
             ?? GetDirectActionsConfig(this.resolveEffectiveConfig(target, undefined, target));
         if (!IsActionAllowedForDirectInvocation(action.Name, directConfig)) {
             return {