@memberjunction/ai-agents 5.41.0 → 5.43.0

package/dist/base-agent.js

@@ -11,7 +11,7 @@
  * @since 2.49.0
  */
 import { FileStorageEngineBase } from '@memberjunction/core-entities';
-import { Metadata, RunView, LogStatus, LogStatusEx, LogError, LogErrorEx, IsVerboseLoggingEnabled, DatabaseProviderBase, EntitySaveOptions } from '@memberjunction/core';
+import { Metadata, RunView, LogStatus, LogStatusEx, LogError, LogErrorEx, IsVerboseLoggingEnabled, DatabaseProviderBase } from '@memberjunction/core';
 import { AgentRunWatchdog } from './agent-run-watchdog.js';
 import { AIPromptRunner } from '@memberjunction/ai-prompts';
 import { BaseRealtimeModel, GetAIAPIKey } from '@memberjunction/ai';
@@ -19,12 +19,14 @@ import { BaseAgentType } from './agent-types/base-agent-type.js';
 import { CopyScalarsAndArrays, JSONValidator, MJGlobal, SafeExpressionEvaluator, UUIDsEqual } from '@memberjunction/global';
 import { RealtimeSessionRunner } from './realtime/realtime-session-runner.js';
 import { ResolveNarrationInstructionsTemplate } from './realtime/realtime-narration.js';
-import { BuildVoiceMannerSection, GetNarrationPaceMs, GetProviderVoiceSettings, ResolveEffectiveRealtimeConfig } from './realtime/realtime-coagent-config.js';
+import { BuildRealtimeOverridesJson, BuildVoiceMannerSection, GetNarrationPaceMs, GetProviderVoiceSettings, ResolveEffectiveRealtimeConfig } from './realtime/realtime-coagent-config.js';
+import { RealtimeClientSessionService } from './realtime/realtime-client-session-service.js';
+import { BuildRealtimeAgentFraming } from './realtime/realtime-tool-broker.js';
 import { AIEngine } from '@memberjunction/aiengine';
 import { ActionEngineServer } from '@memberjunction/actions';
 import { AIAgentPermissionHelper } from '@memberjunction/ai-engine-base';
 import { AgentMemoryContextBuilder } from './agent-memory-context-builder.js';
-import { AIPromptParams, ChildPromptParam, ConversationUtility, ParseFileOutputRef, parseAssignmentStrategy } from '@memberjunction/ai-core-plus';
+import { AIPromptParams, ChildPromptParam, ConversationUtility, ParseFileOutputRef, parseAssignmentStrategy, initAgentRunStep, finalizeAgentRunStep, AgentRunStepSaveQueue } from '@memberjunction/ai-core-plus';
 import { AgentRunner } from './AgentRunner.js';
 import { PayloadManager } from './PayloadManager.js';
 import { ScratchpadManager } from './ScratchpadManager.js';
@@ -92,30 +94,13 @@ export class BaseAgent {
          */
         this._promptRunner = new AIPromptRunner();
         /**
-         * List of pending database save Promises for observability step records.
-         * Awaited concurrently in finalizeAgentRun() to prevent blocking.
+         * Fire-and-forget save orchestration for this run's observability step records: the create INSERT is
+         * fired without blocking the agent flow (the PK is client-generated by `NewRecord()`), each finalize
+         * UPDATE chains after its step's INSERT and force-persists (`IgnoreDirtyState`), and all pending saves
+         * are flushed (`allSettled`) in {@link finalizeAgentRun}. The pattern lives once in
+         * {@link AgentRunStepSaveQueue} (shared with `@memberjunction/computer-use-engine`'s step tracker).
          */
-        this._pendingSaves = [];
-        /**
-         * Queue map to chain database saves sequentially per step entity.
-         * Prevents UPDATE queries running before INSERT queries on quick steps.
-         *
-         * Keyed by the step ENTITY INSTANCE, not its `ID`: a new step's `ID` is empty at create time and
-         * only gets populated during its INSERT `Save()`, so keying by `ID` would file the create and the
-         * finalize under different buckets and defeat the chain — letting the UPDATE race ahead of the
-         * INSERT on millisecond-fast steps (e.g. pipelines), which left them stuck at `Running`.
-         */
-        this._stepSavePromises = new Map();
-        /**
-         * Per-step 'started'-INSERT promises, keyed by the step entity instance. The create INSERT is
-         * fire-and-forget (the agent flow never blocks on it — the PK is client-generated by NewRecord() so
-         * the ID is valid immediately). Every UPDATE-phase save ({@link queueStepSave}) chains after this
-         * INSERT promise so the UPDATE never races ahead of the create, and force-persists with
-         * `IgnoreDirtyState` so a mutation absorbed by the INSERT's post-save dirty-reset (which silently
-         * no-op'd UPDATEs and left fast create→finalize steps stuck at Status='Running') is still written.
-         * A WeakMap so entries are GC'd with the entity.
-         */
-        this._stepInsertPromises = new WeakMap();
+        this._stepSaveQueue = new AgentRunStepSaveQueue();
         /**
          * Active per-request metadata provider, set at the start of Execute().
          * Defaults to the global Metadata.Provider; overridden when a per-request
@@ -1299,6 +1284,93 @@ export class BaseAgent {
             return await this.createFailureResult(msg, params.contextUser);
         }
     }
+    /**
+     * Opens a **raw** {@link IRealtimeSession} for this agent — the duplex model connection a Realtime
+     * Bridge hands to `AIBridgeEngine.StartBridgeSession` so the agent can talk + hear over a media
+     * transport (a LiveKit room, a Zoom/Teams meeting, a phone call). The bridge engine owns turn-taking
+     * and the transport seam, so this deliberately returns the **session itself**, NOT a
+     * {@link RealtimeSessionRunner} (which is the client-direct topology's own orchestration loop).
+     *
+     * It reuses the EXACT same resolution + assembly as {@link executeRealtimeSession} — model selection
+     * ({@link resolveRealtimeModel}), agent configuration ({@link loadAgentConfiguration}), effective-config
+     * persona/voice ({@link resolveRealtimeEffectiveConfig}), and the system-prompt + memory context
+     * ({@link buildRealtimeSessionParams}) — then opens the session via
+     * {@link BaseRealtimeModel.StartSession}. Tools are intentionally NOT pre-populated: the
+     * `invoke-target-agent` + interactive-surface tools are a runner concern; a bridge that needs them
+     * registers them on the returned session itself.
+     *
+     * @param params The execution parameters (agent + context user + the request-scoped provider). A fresh
+     *   bridge session typically passes an empty `conversationMessages` array.
+     * @returns The live realtime session.
+     * @throws When the agent configuration fails to load or no usable Realtime model can be resolved.
+     */
+    async StartBridgeRealtimeSession(params) {
+        // Mirror Execute()'s provider wiring so the realtime helpers operate on the request-scoped provider.
+        this._activeProvider = params.provider ?? Metadata.Provider;
+        const provider = params.provider ?? Metadata.Provider;
+        // A LiveKit / Zoom / Teams bridge is a thin TRANSPORT over the realtime co-agent — it does NOT build
+        // session prep itself. It CONSUMES the one shared producer
+        // ({@link RealtimeClientSessionService.PrepareRealtimeSessionParams}) so the agent's identity (it
+        // speaks first-person AS the target — Sage / Marketing Agent / …), the model + voice precedence
+        // cascade, the tool set (always incl. invoke-target-agent), and memory are byte-for-byte identical to
+        // the native realtime chat. Bridges differ ONLY in opening the session server-side (StartSession) and
+        // their media transport. See plans/realtime/realtime-core-host-convergence.md.
+        // ONE service instance: it produces the prep AND wires the long-lived runtime, so the in-flight
+        // delegation registry (barge-in cancel) is shared between them.
+        const service = new RealtimeClientSessionService();
+        const input = this.buildBridgePrepInput(params);
+        const contextUser = params.contextUser;
+        const prep = await service.PrepareRealtimeSessionParams(input, contextUser, provider);
+        if (!prep.Success || !prep.Resolution || !prep.SessionParams) {
+            throw new Error(prep.ErrorMessage ?? `Failed to prepare a realtime session for agent '${params.agent.Name}'. ` +
+                `Configure an Active AIModelType 'Realtime' model with an active vendor whose DriverClass has a ` +
+                `resolvable API key.`);
+        }
+        const session = await prep.Resolution.Model.StartSession(prep.SessionParams);
+        // Phase 2: wire the SAME core runtime the native chat uses — real `invoke-target-agent` delegation
+        // (target runs via AgentRunner, nested + tracked) + co-agent run/prompt-run observability, finalized
+        // when the bridge calls `session.Close()`. No host-local tool re-implementation. The runtime handle's
+        // side effects live on `session` (OnToolCall + a finalize-wrapped Close), so the bridge just owns the
+        // session. See plans/realtime/realtime-core-host-convergence.md (Phase 2).
+        await service.WireBridgeRealtimeSession(session, input, prep, contextUser, provider);
+        return session;
+    }
+    /**
+     * Adapts {@link ExecuteAgentParams} → the core {@link PrepareClientSessionInput} for a server-bridged
+     * session. The CO-AGENT is the executed agent; the TARGET agent + the per-session model/voice override
+     * ride `params.data` (the same conduit the native dev picker uses, funneled into the one
+     * `ConfigOverridesJson` cascade slot via {@link BuildRealtimeOverridesJson}). Tools are left empty — a
+     * bridge host injects its OWN UX tools (none for LiveKit audio today); identity/precedence/invoke-target
+     * come from the core. `AgentSessionID` groups this session's observability runs (see
+     * {@link RealtimeClientSessionService.WireBridgeRealtimeSession}).
+     *
+     * @param params The bridge execution parameters.
+     * @returns The core prep input.
+     */
+    buildBridgePrepInput(params) {
+        const modelID = params.data?.realtimeModelID?.trim() || undefined;
+        const voice = params.data?.realtimeVoice?.trim() || undefined;
+        const targetID = params.data?.targetAgentID?.trim() || '';
+        // Multi-agent meeting signal (set by the room coordinator when the agent joins a room that already
+        // has agents): disable the model's blind auto-response + add meeting discipline to the prompt so it
+        // hears everything but speaks only when addressed. SelfNames feed only the prompt phrasing; the
+        // addressing GATE is the bridge's matcher. See plans/realtime/multi-agent-meeting-turn-taking.md.
+        const meetingMode = params.data?.realtimeMeetingMode === true;
+        const selfNames = Array.isArray(params.data?.realtimeSelfNames)
+            ? (params.data?.realtimeSelfNames).filter((n) => typeof n === 'string')
+            : undefined;
+        return {
+            CoAgent: params.agent,
+            TargetAgentID: targetID,
+            AgentSessionID: params.data?.agentSessionId ?? '',
+            PreferredModelID: modelID,
+            ConfigOverridesJson: BuildRealtimeOverridesJson(modelID, voice) ?? undefined,
+            ConversationMessages: params.conversationMessages,
+            UserID: params.contextUser?.ID,
+            DisableAutoResponse: meetingMode || undefined,
+            SelfNames: selfNames,
+        };
+    }
     /**
      * Resolves the realtime model + vendor driver + API key for a session-driven run.
      *
@@ -1316,54 +1388,68 @@ export class BaseAgent {
      * @param params The execution parameters (for the agent + context user).
      * @returns The resolved model instance plus its model/vendor identifiers, or `null`.
      */
-    async resolveRealtimeModel(params) {
-        const model = this.selectRealtimeModelEntity(params.agent);
-        if (!model) {
-            return null;
-        }
-        const vendor = this.selectRealtimeVendor(model.ID);
-        if (!vendor) {
-            return null;
-        }
-        const apiKey = GetAIAPIKey(vendor.driverClass);
-        if (!apiKey) {
-            return null;
-        }
-        const instance = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeModel, vendor.driverClass, apiKey);
-        if (!instance) {
-            return null;
+    async resolveRealtimeModel(params, overrideModelID) {
+        // Walk candidates in resolution order (preference first, then highest PowerRank), returning the
+        // FIRST that FULLY resolves (active vendor + resolvable API key + ClassFactory driver). Single-pick
+        // would dead-end whenever the top model lacked a key — e.g. a power-11 model with no env key
+        // (Inworld/AssemblyAI) outranking GPT Realtime — and surface "No usable Realtime model" even though
+        // a usable model exists. This mirrors the same fix in RealtimeClientSessionService.
+        const candidates = this.selectRealtimeModelCandidates(params.agent, overrideModelID);
+        for (const model of candidates) {
+            const vendor = this.selectRealtimeVendor(model.ID);
+            if (!vendor) {
+                continue;
+            }
+            const apiKey = GetAIAPIKey(vendor.driverClass);
+            if (!apiKey) {
+                continue;
+            }
+            const instance = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeModel, vendor.driverClass, apiKey);
+            if (!instance) {
+                continue;
+            }
+            return { model: instance, modelID: model.ID, vendorID: vendor.vendorID, apiName: vendor.apiName, driverClass: vendor.driverClass };
         }
-        return { model: instance, modelID: model.ID, vendorID: vendor.vendorID, apiName: vendor.apiName, driverClass: vendor.driverClass };
+        return null;
     }
     /**
-     * Selects the `MJ: AI Models` row to use for a realtime session: the highest-power active
-     * model of AIModelType `Realtime`. Returns `null` when no `Realtime` model exists in metadata
-     * (expected before P4).
+     * The active `Realtime`-AIModelType models to try, in resolution order — the candidate list
+     * {@link resolveRealtimeModel} walks until one yields a usable vendor + key + driver. Returns ALL
+     * candidates (not just the top pick) so a keyless / undriveable higher-power model falls through to
+     * the next usable one instead of dead-ending the whole resolution.
+     *
+     * Ordering: an effective-config model preference (`realtime.modelPreference`, an MJ: AI Models Name
+     * or ID) goes FIRST when it resolves, followed by the rest by descending PowerRank (so even a keyless
+     * preferred model degrades gracefully). An unsatisfiable preference logs and is ignored.
      *
-     * @param agent The agent being executed (reserved for future per-agent model preference).
-     * @returns The chosen model entity, or `null`.
+     * @param agent The agent being executed.
+     * @returns The candidate models in resolution order (empty when none are active).
      */
-    selectRealtimeModelEntity(agent) {
+    selectRealtimeModelCandidates(agent, overrideModelID) {
         const isRealtime = (m) => typeof m.AIModelType === 'string' && m.AIModelType.trim().toLowerCase() === 'realtime';
         const realtimeModels = AIEngine.Instance.Models.filter(m => m.IsActive && isRealtime(m));
         if (realtimeModels.length === 0) {
-            return null;
+            return [];
         }
-        // Effective-config model preference (realtime.modelPreference, an MJ: AI Models Name or
-        // ID) participates first. METADATA preferences degrade gracefully: an unsatisfiable
-        // preference logs and falls through to the default highest-PowerRank selection.
-        const preference = this.resolveRealtimeEffectiveConfig(agent).realtime?.modelPreference;
+        const byPower = [...realtimeModels].sort((a, b) => (b.PowerRank ?? 0) - (a.PowerRank ?? 0));
+        // A per-session override (a dev picking a specific Realtime model for this bridged agent) wins over
+        // the config's modelPreference — same "preferred first, rest by power as fallback" semantics.
+        const preference = (overrideModelID && overrideModelID.trim().length > 0)
+            ? overrideModelID.trim()
+            : this.resolveRealtimeEffectiveConfig(agent).realtime?.modelPreference;
         if (preference) {
             const wanted = preference.trim().toLowerCase();
             const preferred = realtimeModels.find(m => UUIDsEqual(m.ID, preference))
                 ?? realtimeModels.find(m => m.Name?.trim().toLowerCase() === wanted);
             if (preferred) {
-                return preferred;
+                // Preference first, the rest (by power) as fallback so a keyless preferred model still
+                // falls through to a usable one rather than dead-ending.
+                return [preferred, ...byPower.filter(m => !UUIDsEqual(m.ID, preferred.ID))];
             }
             this.logError(`Realtime model preference '${preference}' for agent '${agent.Name}' matches no Active Realtime ` +
                 'model — falling through to default (highest-PowerRank) selection.', { agent, category: 'RealtimeSession' });
         }
-        return realtimeModels.sort((a, b) => (b.PowerRank ?? 0) - (a.PowerRank ?? 0))[0];
+        return byPower;
     }
     /**
      * Resolves the agent's EFFECTIVE realtime configuration — the agent TYPE's
@@ -1498,16 +1584,11 @@ export class BaseAgent {
      * @returns The session parameters.
      */
     async buildRealtimeSessionParams(params, config, modelApiName, effectiveConfig, driverClass) {
-        const framing = `You are the real-time voice for the agent "${params.agent.Name}". Hold a natural, ` +
-            `low-latency conversation with the user. When actual work is required, call the ` +
-            `'invoke-target-agent' tool and narrate progress while it runs — do not attempt to do ` +
-            `the work yourself. ONE EXCEPTION: besides 'invoke-target-agent' you may have been given ` +
-            `interactive-surface tools (for example 'browser_*' to drive a LIVE web browser the user ` +
-            `can watch, or 'Whiteboard_*' to draw on a shared board). Those surfaces are operated by ` +
-            `YOU, directly — when the user asks to use one (e.g. "open/show a browser", "go to a ` +
-            `site", "add to the whiteboard"), call the matching tool yourself immediately and narrate ` +
-            `what you're doing. NEVER route an interactive-surface request through 'invoke-target-agent', ` +
-            `and never claim you lack a session — calling the tool is all that's needed.`;
+        // Identity framing comes from the ONE shared producer so the agent speaks first-person AS the
+        // TARGET (Sage / Marketing Agent / …), identical to every other realtime host — not as the co-agent.
+        // See BuildRealtimeAgentFraming + plans/realtime/realtime-core-host-convergence.md.
+        const targetAgent = this.resolveRealtimeTargetAgent(params);
+        const framing = BuildRealtimeAgentFraming(targetAgent?.Name ?? 'the configured target agent');
         const basePrompt = config.systemPrompt?.TemplateText ? config.systemPrompt.TemplateText : '';
         // Effective-config voice persona (realtime.voice.default) → short "Voice & manner" section.
         const voiceManner = BuildVoiceMannerSection(effectiveConfig);
@@ -1941,8 +2022,13 @@ export class BaseAgent {
      * @protected
      */
     async initializeEngines(contextUser) {
-        await AIEngine.Instance.Config(false, contextUser);
+        // Load the Action engine BEFORE the AI engine. AIEngine.RefreshActions()
+        // (invoked by AIEngine.Config) reuses already-cached 'MJ: Actions'
+        // metadata via BaseEngineRegistry, so priming ActionEngineServer first
+        // lets AIEngine skip loading a second copy into ActionEngineBase —
+        // eliminating the duplicate-RunView telemetry warning at agent startup.
         await ActionEngineServer.Instance.Config(false, contextUser);
+        await AIEngine.Instance.Config(false, contextUser);
     }
     /**
      * Determine the scope label for a note based on its scope fields.
@@ -4796,7 +4882,9 @@ The context is now within limits. Please retry your request with the recovered c
                 SecondaryScopes: params.SecondaryScopes,
                 onAgentRunCreated: async (agentRunId) => {
                     stepEntity.TargetLogID = agentRunId;
-                    this.queueStepSave(stepEntity);
+                    // Re-apply post-INSERT: this callback can fire while the step's INSERT is still in flight,
+                    // and the INSERT's reload would otherwise revert TargetLogID back to null.
+                    this.queueStepSave(stepEntity, (s) => { s.TargetLogID = agentRunId; });
                 }
             });
             // Check if execution was successful
@@ -5547,43 +5635,38 @@ The context is now within limits. Please retry your request with the recovered c
         // steps link via ParentID and the post-create UPDATE-phase mutations reference this row, and the
         // create INSERT is fire-and-forget (the agent flow must not block on it).
         stepEntity.NewRecord();
-        stepEntity.AgentRunID = this._agentRun.ID;
         // Step number is based on current count of steps + 1
-        stepEntity.StepNumber = (this._agentRun.Steps?.length || 0) + 1;
-        stepEntity.StepType = params.stepType;
-        // Include hierarchy breadcrumb in StepName for better logging
-        stepEntity.StepName = this.formatHierarchicalMessage(params.stepName);
-        // check to see if targetId is a valid UUID
+        const stepNumber = (this._agentRun.Steps?.length || 0) + 1;
+        // Warn on a non-UUID targetId before delegating (initAgentRunStep silently ignores invalid ids).
         if (params.targetId && !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(params.targetId)) {
-            // If not valid, we can just ignore it, but console.warn
             console.warn(`Invalid target ID format: ${params.targetId}`);
         }
-        else {
-            stepEntity.TargetID = params.targetId || null;
-        }
-        stepEntity.TargetLogID = params.targetLogId || null;
-        stepEntity.ParentID = params.parentId || null; // Link to parent step (e.g., loop step)
-        stepEntity.Status = 'Running';
-        stepEntity.StartedAt = new Date();
-        stepEntity.PayloadAtStart = this.serializePayloadAtStart(params.payloadAtStart);
-        stepEntity.PayloadAtEnd = this.serializePayloadAtEnd(params.payloadAtEnd);
-        // Populate InputData if provided
-        if (params.inputData) {
-            stepEntity.InputData = JSON.stringify({
-                ...params.inputData,
-                context: {
-                    agentHierarchy: this._agentHierarchy,
-                    depth: this._depth,
-                    stepNumber: stepEntity.StepNumber
-                }
-            });
-        }
-        // Fire-and-forget the 'started' INSERT — the agent flow never blocks on a step save. Store the
-        // promise per-step so every later UPDATE (queueStepSave) runs only AFTER this INSERT commits
-        // (see _stepInsertPromises).
-        const insertPromise = this.saveStepRecord(stepEntity, 'insert');
-        this._stepInsertPromises.set(stepEntity, insertPromise);
-        this._pendingSaves.push(insertPromise);
+        // Populate the started fields via the shared single-source-of-truth helper. Instance-specific
+        // concerns (hierarchy breadcrumb, InputData context, payload serialization) are computed here.
+        initAgentRunStep(stepEntity, {
+            AgentRunID: this._agentRun.ID,
+            StepNumber: stepNumber,
+            StepType: params.stepType,
+            StepName: this.formatHierarchicalMessage(params.stepName), // include hierarchy breadcrumb
+            TargetID: params.targetId,
+            TargetLogID: params.targetLogId,
+            ParentID: params.parentId, // Link to parent step (e.g., loop step)
+            PayloadAtStart: this.serializePayloadAtStart(params.payloadAtStart),
+            PayloadAtEnd: this.serializePayloadAtEnd(params.payloadAtEnd),
+            InputData: params.inputData
+                ? JSON.stringify({
+                    ...params.inputData,
+                    context: {
+                        agentHierarchy: this._agentHierarchy,
+                        depth: this._depth,
+                        stepNumber
+                    }
+                })
+                : undefined
+        });
+        // Fire-and-forget the 'started' INSERT — the agent flow never blocks on a step save. The queue
+        // tracks the INSERT so every later UPDATE (queueStepSave) chains AFTER it commits.
+        this._stepSaveQueue.Insert(stepEntity);
         // Add the step to the agent run's Steps array
         if (this._agentRun) {
             this._agentRun.Steps.push(stepEntity);
@@ -5650,78 +5733,42 @@ The context is now within limits. Please retry your request with the recovered c
      */
     async finalizeStepEntity(stepEntity, success, errorMessage, outputData) {
         try {
-            // Apply the completion state to the in-memory entity NOW (so the run's Steps array / UI see
-            // Completed immediately), then fire-and-forget the UPDATE via queueStepSave — which chains after
-            // the INSERT and force-persists (IgnoreDirtyState). The agent flow never blocks on this UPDATE.
-            stepEntity.Status = success ? 'Completed' : 'Failed';
-            stepEntity.CompletedAt = new Date();
-            stepEntity.Success = success;
-            stepEntity.ErrorMessage = errorMessage || null;
-            // Populate OutputData if provided
-            if (outputData) {
-                stepEntity.OutputData = JSON.stringify({
-                    ...CopyScalarsAndArrays(outputData, true),
-                    context: {
-                        success,
-                        durationMs: stepEntity.CompletedAt.getTime() - stepEntity.StartedAt.getTime(),
-                        errorMessage
-                    }
-                });
-            }
-            this.queueStepSave(stepEntity);
+            // Capture the completion timestamp NOW so the duration is accurate regardless of when the
+            // mutation is actually applied/persisted.
+            const finalizeOpts = {
+                success,
+                errorMessage,
+                outputData: outputData ? CopyScalarsAndArrays(outputData, true) : undefined,
+                completedAt: new Date(),
+                // Capture any TargetLogID already stamped on the entity (e.g. a prompt-run / sub-agent-run id
+                // set before finalize) so the post-INSERT re-apply restores it too — otherwise the INSERT's
+                // reload could leave it null on a fast step.
+                targetLogID: stepEntity.TargetLogID ?? undefined
+            };
+            // Apply to the in-memory entity NOW so the run's Steps array / UI see the terminal state
+            // immediately. This in-memory copy can be reverted by the INSERT's post-save reload if the step
+            // finished while its INSERT was still in flight, which is why we ALSO re-apply it inside the
+            // post-INSERT continuation below (idempotent — same completedAt).
+            finalizeAgentRunStep(stepEntity, finalizeOpts);
+            // Fire-and-forget the UPDATE, but re-assert the finalize state AFTER the INSERT (and its reload)
+            // lands so the force-persisted UPDATE never writes stale pre-finalize values. The agent flow
+            // never blocks on this UPDATE.
+            this.queueStepSave(stepEntity, (s) => finalizeAgentRunStep(s, finalizeOpts));
         }
         catch (e) {
             LogError(`Failed to update agent run step record: ${e?.message ?? e}`, undefined, e);
         }
     }
-    /**
-     * Saves one step record and ALWAYS logs (never swallows, never verbose-gated) a failure via `LogError`
-     * with `LatestResult.CompleteMessage` — the log is observability, so a failure must surface but must
-     * not throw into the agent loop. The `update` phase force-saves with `IgnoreDirtyState` because a
-     * finalize/TargetLogID mutation applied while the INSERT was still in flight gets absorbed by the
-     * INSERT's post-save dirty-reset, leaving the new values only in memory (the "step stuck at Running"
-     * bug); forcing the UPDATE re-persists them. `insert` saves normally.
-     * @returns whether the row persisted.
-     */
-    async saveStepRecord(stepEntity, phase) {
-        try {
-            let options;
-            if (phase === 'update') {
-                options = new EntitySaveOptions();
-                options.IgnoreDirtyState = true;
-            }
-            const ok = await stepEntity.Save(options);
-            if (!ok) {
-                LogError(`Failed to ${phase} agent run step record ${stepEntity.ID || '(unsaved)'}: ${stepEntity.LatestResult?.CompleteMessage ?? 'unknown error'}`);
-            }
-            return ok;
-        }
-        catch (e) {
-            LogError(`Error on ${phase} of agent run step record ${stepEntity.ID || '(unsaved)'}: ${e?.message ?? e}`);
-            return false;
-        }
-    }
     /**
      * Queues a fire-and-forget UPDATE of a step entity whose fields the caller has ALREADY mutated.
-     *
-     * - The agent flow never awaits this (logging is fire-and-forget).
-     * - Chains after the step's 'started' INSERT and any prior queued save, so the UPDATE never races
-     *   ahead of the INSERT; updates to DIFFERENT steps run concurrently.
-     * - Force-persists (IgnoreDirtyState) because a mutation applied while the INSERT was in flight can be
-     *   absorbed by the INSERT's post-save dirty-reset, leaving the entity "clean" with the new values
-     *   only in memory — without the force, the UPDATE would silently no-op and the row would stay stuck
-     *   at Status='Running' / null TargetLogID.
-     * - Failures are logged (never thrown) and surfaced via `_pendingSaves` at run finalize.
+     * Delegates to {@link AgentRunStepSaveQueue.QueueUpdate} — the agent flow never awaits this; the UPDATE
+     * chains after the step's INSERT and force-persists (`IgnoreDirtyState`). Kept `protected` so driver
+     * subclasses that finalize their own steps get the same non-blocking behavior.
      *
      * @protected
      */
-    queueStepSave(stepEntity) {
-        // Chain on the entity INSTANCE (stable), NOT stepEntity.ID. Fall back to the INSERT promise so an
-        // UPDATE queued before the create lands still runs after it.
-        const previousSave = this._stepSavePromises.get(stepEntity) ?? this._stepInsertPromises.get(stepEntity) ?? Promise.resolve();
-        const currentSave = previousSave.then(() => this.saveStepRecord(stepEntity, 'update'));
-        this._stepSavePromises.set(stepEntity, currentSave);
-        this._pendingSaves.push(currentSave);
+    queueStepSave(stepEntity, applyMutation) {
+        this._stepSaveQueue.QueueUpdate(stepEntity, applyMutation);
     }
     /**
      * Maps an array through an async worker with bounded concurrency.
@@ -6092,7 +6139,9 @@ The context is now within limits. Please retry your request with the recovered c
             } : undefined;
             promptParams.onPromptRunCreated = async (promptRunId) => {
                 stepEntity.TargetLogID = promptRunId;
-                this.queueStepSave(stepEntity);
+                // Re-apply post-INSERT: onPromptRunCreated can fire before the step's INSERT lands, and the
+                // INSERT's reload would otherwise revert TargetLogID back to null.
+                this.queueStepSave(stepEntity, (s) => { s.TargetLogID = promptRunId; });
             };
             // Execute the prompt
             const promptResult = await this.executePrompt(promptParams);
@@ -7741,8 +7790,11 @@ The context is now within limits. Please retry your request with the recovered c
                     actionResult = await this.ExecuteSingleAction(params, aa, actionEntity, params.contextUser);
                     // Update step entity with ActionExecutionLog ID if available
                     if (actionResult.LogEntry?.ID) {
-                        stepEntity.TargetLogID = actionResult.LogEntry.ID;
-                        this.queueStepSave(stepEntity);
+                        const logId = actionResult.LogEntry.ID;
+                        stepEntity.TargetLogID = logId;
+                        // Re-apply post-INSERT: a fast action can finish before the step's INSERT lands, and
+                        // the INSERT's reload would otherwise revert TargetLogID back to null.
+                        this.queueStepSave(stepEntity, (s) => { s.TargetLogID = logId; });
                     }
                     // Prepare output data with action result
                     const outputData = {
@@ -8468,6 +8520,18 @@ The context is now within limits. Please retry your request with the recovered c
      * Strips "payload." prefix if present (for LLM convenience)
      */
     getCollectionFromPayload(payload, path) {
+        // Support a literal static collection: "static:[1,2,3,4,5]". This lets a ForEach iterate a
+        // fixed list/range without a prior step having to build the array in the payload first.
+        const trimmed = path.trim();
+        if (trimmed.toLowerCase().startsWith('static:')) {
+            try {
+                const parsed = JSON.parse(trimmed.substring(trimmed.indexOf(':') + 1).trim());
+                return Array.isArray(parsed) ? parsed : null;
+            }
+            catch {
+                return null;
+            }
+        }
         // Remove "payload." prefix if present
         const cleanPath = path.toLowerCase().startsWith('payload.')
             ? path.substring(8)
@@ -9018,27 +9082,15 @@ The context is now within limits. Please retry your request with the recovered c
      * @private
      */
     async finalizeAgentRun(finalStep, payload, contextUser) {
-        // Await every pending step save (success OR failure) and accumulate diagnostics.
-        // We use allSettled so a single failure doesn't shadow the rest, and we drain
-        // both queues afterwards so an instance reused for another run doesn't leak
-        // settled promises.
-        const pending = this._pendingSaves;
-        this._pendingSaves = [];
-        this._stepSavePromises.clear();
-        if (pending.length > 0) {
-            const settled = await Promise.allSettled(pending);
-            const rejections = settled.filter(s => s.status === 'rejected');
-            const falses = settled.filter(s => s.status === 'fulfilled' && s.value === false).length;
-            for (const r of rejections) {
-                LogError(`Pending step save rejected: ${r.reason instanceof Error ? r.reason.message : String(r.reason)}`);
-            }
-            const totalFailures = rejections.length + falses;
-            if (totalFailures > 0 && this._agentRun) {
-                const note = `${totalFailures} step record save(s) failed during this run; see logs for details.`;
-                this._agentRun.ErrorMessage = this._agentRun.ErrorMessage
-                    ? `${this._agentRun.ErrorMessage}\n${note}`
-                    : note;
-            }
+        // Flush every pending step save (success OR failure) via the shared queue, which allSettles so a
+        // single failure doesn't shadow the rest and drains itself so a reused instance doesn't leak
+        // settled promises. Surface the failure count on the run for visibility.
+        const { failures } = await this._stepSaveQueue.Flush();
+        if (failures > 0 && this._agentRun) {
+            const note = `${failures} step record save(s) failed during this run; see logs for details.`;
+            this._agentRun.ErrorMessage = this._agentRun.ErrorMessage
+                ? `${this._agentRun.ErrorMessage}\n${note}`
+                : note;
         }
         // Only resolve media placeholders for ROOT agents (depth === 0)
         // Sub-agents keep placeholders intact so parent agents don't get huge base64 in their context