@memberjunction/server 6.1.0-edge.1 → 6.1.0-edge.3

package/src/services/IntegrationSyncWorkerService.ts

@@ -0,0 +1,141 @@
+/**
+ * @fileoverview Worker-mode poll loop for durable integration sync runs (PR 1 item 8).
+ * @module MJServer/services
+ */
+
+import { LogError, LogStatus, UserInfo } from '@memberjunction/core';
+import { UserCache } from '@memberjunction/generic-database-provider';
+import { IntegrationEngine } from '@memberjunction/integration-engine';
+import { IntegrationSyncWorkerConfig } from '../config.js';
+
+/**
+ * Polls `CompanyIntegrationRun` for `Status='Queued'` rows, claims each one atomically,
+ * executes it, and releases — the enqueue/execute split that lets a sync outlive the
+ * request that asked for it.
+ *
+ * Multiple workers (in this process or others) may poll the same queue: the claim sproc
+ * admits exactly one, and every loser simply moves on to the next candidate. Nothing here
+ * assumes it is the only worker.
+ */
+export class IntegrationSyncWorkerService {
+    private config: IntegrationSyncWorkerConfig;
+    private systemUser: UserInfo | null = null;
+    private timer: NodeJS.Timeout | null = null;
+    private isRunning = false;
+    private polling = false;
+    /** Run IDs currently executing in THIS process — bounds concurrency and prevents self-collision. */
+    private inFlight = new Set<string>();
+
+    constructor(config: IntegrationSyncWorkerConfig) {
+        this.config = config;
+    }
+
+    public get IsEnabled(): boolean {
+        return this.config.enabled;
+    }
+
+    public get IsRunning(): boolean {
+        return this.isRunning;
+    }
+
+    /** Number of queued runs this worker is currently executing. */
+    public get InFlightCount(): number {
+        return this.inFlight.size;
+    }
+
+    /** Resolve the system user the worker executes syncs as. */
+    public async Initialize(): Promise<void> {
+        if (!this.config.enabled) return;
+
+        const email = this.config.systemUserEmail;
+        this.systemUser = UserCache.Users.find(u => u.Email?.toLowerCase() === email.toLowerCase()) ?? null;
+        if (!this.systemUser) {
+            throw new Error(`[IntegrationSyncWorker] System user not found with email: ${email}`);
+        }
+        await IntegrationEngine.Instance.Config(false, this.systemUser);
+    }
+
+    /** Begin polling the queue. */
+    public Start(): void {
+        if (!this.config.enabled || this.isRunning) return;
+        if (!this.systemUser) {
+            throw new Error('[IntegrationSyncWorker] Not initialized — call Initialize() first');
+        }
+
+        this.isRunning = true;
+        this.timer = setInterval(() => void this.pollOnce(), this.config.pollingIntervalMs);
+        // Standard level, NOT verboseOnly: this states a deployment-shaping fact — that this process
+        // executes queued syncs — and it's one line per process lifetime. Stop() already logs at
+        // standard level, so hiding Start() produced logs showing a worker stopping that never
+        // started, and no way to confirm from logs that a deployment is in worker mode at all.
+        LogStatus(
+            `🔄 Integration Sync Worker: polling every ${Math.round(this.config.pollingIntervalMs / 1000)}s, ` +
+            `up to ${this.config.maxConcurrentRuns} concurrent run(s)`
+        );
+        // Don't wait a full interval to drain a queue that already has work.
+        void this.pollOnce();
+    }
+
+    /**
+     * Stop polling. In-flight runs are left to finish on their own — their leases are
+     * renewed by the engine's heartbeat, so killing them here would only strand rows.
+     */
+    public Stop(): void {
+        if (!this.isRunning) return;
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+        this.isRunning = false;
+        LogStatus(`[IntegrationSyncWorker] Stopped polling (${this.inFlight.size} run(s) still in flight)`);
+    }
+
+    /**
+     * One poll pass: fetch claimable candidates up to the free concurrency slots and start
+     * each. Overlapping passes are suppressed so a slow query can't stack up timers.
+     */
+    private async pollOnce(): Promise<void> {
+        if (this.polling || !this.isRunning || !this.systemUser) return;
+
+        const slots = this.config.maxConcurrentRuns - this.inFlight.size;
+        if (slots <= 0) return;
+
+        this.polling = true;
+        try {
+            const candidates = await IntegrationEngine.PollQueuedRuns(this.systemUser, slots);
+            for (const candidate of candidates) {
+                if (this.inFlight.has(candidate.ID)) continue;
+                this.startRun(candidate.ID);
+            }
+        } catch (err) {
+            LogError(`[IntegrationSyncWorker] Poll failed`, undefined, err);
+        } finally {
+            this.polling = false;
+        }
+    }
+
+    /**
+     * Execute one queued run in the background. Losing the claim race is normal — the run
+     * is simply someone else's — so it is logged as information, never as an error.
+     */
+    private startRun(runID: string): void {
+        if (!this.systemUser) return;
+
+        this.inFlight.add(runID);
+        void IntegrationEngine.Instance
+            .ExecuteQueuedRun(runID, this.systemUser)
+            .then(result => {
+                if (result.Success) {
+                    LogStatus(`[IntegrationSyncWorker] Run ${runID} completed: ${result.RecordsProcessed} record(s) processed`);
+                } else {
+                    LogStatus(`[IntegrationSyncWorker] Run ${runID} did not complete: ${result.ErrorMessage ?? 'unknown reason'}`);
+                }
+            })
+            .catch(err => {
+                LogError(`[IntegrationSyncWorker] Run ${runID} threw`, undefined, err);
+            })
+            .finally(() => {
+                this.inFlight.delete(runID);
+            });
+    }
+}

package/src/services/ScheduledJobsService.ts

@@ -4,7 +4,7 @@
  */
 
 import { LogError, LogStatus, LogStatusEx, UserInfo } from '@memberjunction/core';
-import { UserCache } from '@memberjunction/sqlserver-dataprovider';
+import { UserCache } from '@memberjunction/generic-database-provider';
 import { SchedulingEngine } from '@memberjunction/scheduling-engine';
 import { ScheduledJobsConfig } from '../config.js';
 

package/src/services/StartTaskGraphDispatcher.ts

@@ -18,9 +18,14 @@
  */
 import { UserInfo } from '@memberjunction/core';
 import { LoadTaskGraphOperations, TaskGraphDispatcher } from '@memberjunction/task-graph';
+import { TaskGraphPromptRunner } from './TaskGraphPromptRunner.js';
 import sql from 'mssql';
+import { DurableEntityActionRegistry } from '@memberjunction/actions-base';
 import { CreateTaskGraphProviderFactory } from './TaskGraphProviderFactory.js';
+import { TaskGraphActionRunner } from './TaskGraphActionRunner.js';
 import { TaskGraphAgentRunner } from './TaskGraphAgentRunner.js';
+import { randomBytes } from 'node:crypto';
+import { DurableEntityActionTaskSubmitter } from './DurableEntityActionTaskSubmitter.js';
 import { TaskGraphContinuationDeliverer } from './TaskGraphContinuationDeliverer.js';
 import { PubSubManager } from '../generic/PubSubManager.js';
 import { TaskGraphFrameBroadcaster } from '../resolvers/TaskGraphFrameResolver.js';
@@ -35,6 +40,19 @@ import { TaskGraphFrameBroadcaster } from '../resolvers/TaskGraphFrameResolver.j
  *
  * The dispatcher self-registers with `ShutdownRegistry`, so the caller does not wire teardown.
  */
+/**
+ * A per-process identity for the claim protocol.
+ *
+ * Host and pid are kept because they are what a human reads in a log; the random suffix is what
+ * makes the value actually unique. Generated once per call, so a restart is a new instance — which
+ * is correct: the previous process's claims should expire rather than be inherited.
+ */
+function defaultInstanceID(): string {
+    const host = process.env.HOSTNAME ?? 'mjapi';
+    const entropy = randomBytes(4).toString('hex');
+    return `${host}-${process.pid}-${entropy}`;
+}
+
 export async function StartTaskGraphDispatcher(
     pool: sql.ConnectionPool,
     contextUser: UserInfo,
@@ -48,14 +66,33 @@ export async function StartTaskGraphDispatcher(
         providerFactory,
         new TaskGraphAgentRunner(),
         contextUser,
-        { InstanceID: instanceID ?? `${process.env.HOSTNAME ?? 'mjapi'}-${process.pid}` },
+        // A RANDOM SUFFIX, because host+pid is not unique in the deployments this runs in (C2).
+        // `HOSTNAME` is not exported to child processes under systemd or pm2, and a containerised
+        // process is routinely pid 1 — so two hosts default to the same `mjapi-1`. A shared instance
+        // id defeats every `ClaimedBy=@me` guard at once: A's heartbeat renews B's lease, and A's
+        // stale terminal write lands over B's live execution. An explicit `instanceID` still wins,
+        // for hosts that have a real identity to supply.
+        { InstanceID: instanceID ?? defaultInstanceID() },
         // Without this the dispatcher had nowhere to deliver: a finished graph logged its outcome,
         // marked itself delivered, and said nothing to the conversation that asked for it.
         new TaskGraphContinuationDeliverer(providerFactory, contextUser),
         // Live signal. Supplied only when a PubSub engine exists — in a worker or a test there is
         // nobody to publish to, and the dispatcher runs identically without one.
         pubSub ? new TaskGraphFrameBroadcaster(pubSub) : undefined,
+        // Action nodes — the third execution shape, beside agents and people. Durable After* entity
+        // actions land as these.
+        new TaskGraphActionRunner(),
+        // Prompt nodes. Without this a Flow agent with a Prompt step submits and then
+        // stalls on a task nothing can execute — which is why the shipped User Onboarding
+        // Flow Agent could not run at all before this seam existed.
+        new TaskGraphPromptRunner(),
     );
     await dispatcher.Start();
+
+    // Registered only AFTER the dispatcher is running, and only here. A submitter without a
+    // dispatcher writes Task rows nobody picks up — strictly worse than staying inline, because the
+    // work is now both undone and invisible in the place it used to be logged.
+    DurableEntityActionRegistry.Instance.Register(new DurableEntityActionTaskSubmitter(providerFactory));
+
     return dispatcher;
 }

package/src/services/TaskGraphActionRunner.ts

@@ -0,0 +1,99 @@
+/**
+ * @fileoverview MJServer's implementation of the task-graph `TaskActionRunner` seam.
+ *
+ * The counterpart to {@link TaskGraphAgentRunner}: the dispatcher knows *when* an action-assigned
+ * node should run and nothing about how an action executes, so execution is injected. Keeping it
+ * here rather than in `@memberjunction/task-graph` is not tidiness — the action engine depends on
+ * the entity layer the dispatcher also builds on, and importing it there would make every consumer
+ * of durable execution load the action engine as well.
+ *
+ * @module @memberjunction/server
+ */
+import { ActionEngineServer } from '@memberjunction/actions';
+import { ActionParam } from '@memberjunction/actions-base';
+import { LogError } from '@memberjunction/core';
+import { UUIDsEqual } from '@memberjunction/global';
+import type { TaskActionRunner, TaskActionRunParams, TaskActionRunResult } from '@memberjunction/task-graph';
+
+export class TaskGraphActionRunner implements TaskActionRunner {
+    public async RunActionForTask(params: TaskActionRunParams): Promise<TaskActionRunResult> {
+        try {
+            await ActionEngineServer.Instance.Config(false, params.ContextUser);
+            const action = ActionEngineServer.Instance.Actions.find((a) => UUIDsEqual(a.ID, params.ActionID));
+            if (!action) {
+                return { Success: false, ErrorMessage: `Action ${params.ActionID} is not in the engine's metadata.` };
+            }
+
+            const result = await ActionEngineServer.Instance.RunAction({
+                Action: action,
+                ContextUser: params.ContextUser,
+                Params: this.buildParams(params),
+                // No Filters. A durable node is work that a filter ALREADY let through — the entity
+                // action's gate ran at dispatch time, on the change context that no longer exists by
+                // the time the dispatcher picks the task up. Re-evaluating here would ask a
+                // transition question with nothing to answer it, and fail closed on every retry.
+                Filters: [],
+            });
+
+            return {
+                Success: result.Success,
+                Output: this.buildOutput(result.Params),
+                ErrorMessage: result.Success ? undefined : result.Message,
+                // The engine already wrote the log; this is the only place its id is in hand. Without
+                // carrying it out, a workflow's action step has no path back to its own execution
+                // record — the one thing anyone wants when an action misbehaves.
+                ActionLogID: result.LogEntry?.ID,
+            };
+        } catch (e) {
+            const message = e instanceof Error ? e.message : String(e);
+            LogError(`[TaskGraphActionRunner] Task ${params.TaskID} failed: ${message}`);
+            return { Success: false, ErrorMessage: message };
+        }
+    }
+
+    /**
+     * Projects the action's parameters onto the flat result a workflow step is expected to produce.
+     *
+     * **Why not return `result.Params` as-is.** That is an `ActionParam[]`, and every consumer of a
+     * step's output addresses fields BY NAME: an output mapping says `{"CurrentPrice": "stockPrice"}`,
+     * and a branch condition then reads `payload.stockPrice`. Handed an array, the mapping finds no
+     * `CurrentPrice`, writes nothing, and the condition evaluates against `undefined` — which is
+     * falsy rather than erroneous, so the workflow takes the other branch and reports success. That
+     * is exactly what the Demo workflow did: step 1 completed with an output payload of `{}` and
+     * every downstream branch was skipped, with nothing anywhere reporting a problem.
+     *
+     * Only `Output` and `Both` params are included. Echoing the inputs back would let a step's own
+     * parameters masquerade as its results, so an output mapping could "succeed" by reading the
+     * value the step was given rather than the one it produced.
+     */
+    private buildOutput(actionParams: ActionParam[]): Record<string, unknown> {
+        const output: Record<string, unknown> = {};
+        for (const p of actionParams ?? []) {
+            if (p.Type === 'Output' || p.Type === 'Both') output[p.Name] = p.Value;
+        }
+        return output;
+    }
+
+    /**
+     * Rebuilds the action's parameters from the task's stored payload.
+     *
+     * **The values here have been through redaction**, because `Task.InputPayload` is persistent,
+     * user-visible storage and nothing writes a raw `ActionParam[]` there. A parameter the binding
+     * marked as not-logged therefore arrives absent rather than secret — the action sees a missing
+     * value, which is the honest consequence of choosing not to persist it, and the reason durable
+     * dispatch is opt-in per binding rather than the default.
+     *
+     * Dependency outputs are merged underneath the task's own input so a node's explicit parameters
+     * always win over an upstream node that happened to emit the same key.
+     */
+    private buildParams(params: TaskActionRunParams): ActionParam[] {
+        const merged: Record<string, unknown> = {};
+        for (const [name, value] of params.DependencyOutputs) {
+            merged[name] = value;
+        }
+        if (params.InputPayload && typeof params.InputPayload === 'object') {
+            Object.assign(merged, params.InputPayload as Record<string, unknown>);
+        }
+        return Object.entries(merged).map(([Name, Value]) => ({ Name, Value, Type: 'Input' }));
+    }
+}

package/src/services/TaskGraphAgentRunner.ts

@@ -15,7 +15,8 @@
 import { AgentRunner } from '@memberjunction/ai-agents';
 import { ChatMessageRole } from '@memberjunction/ai';
 import { LogError } from '@memberjunction/core';
-import { MJAIAgentEntityExtended } from '@memberjunction/ai-core-plus';
+import { MJAIAgentEntityExtended, MJAIAgentRunEntityExtended } from '@memberjunction/ai-core-plus';
+import type { AgentExecutionProgressCallback } from '@memberjunction/ai-core-plus';
 import type { TaskAgentRunner, TaskAgentRunParams, TaskAgentRunResult } from '@memberjunction/task-graph';
 
 export class TaskGraphAgentRunner implements TaskAgentRunner {
@@ -30,6 +31,14 @@ export class TaskGraphAgentRunner implements TaskAgentRunner {
                 agent,
                 conversationMessages: [{ role: ChatMessageRole.user, content: this.buildPrompt(params) }],
                 contextUser: params.ContextUser,
+                // Inherited from the graph, so a graph this run goes on to emit is one hop deeper
+                // rather than starting the chain over. Without it MAX_REINVOKE_DEPTH never fires.
+                continuationDepth: params.ContinuationDepth ?? 0,
+                parentRun: await this.loadSubmittingRun(params),
+                // The bridge from the agent's own progress stream up to graph-level observers: the
+                // dispatcher turns these into rate-limited `NodeProgress` frames, which is how a
+                // running Agent step shows "what it is doing right now" on the workflow canvas.
+                onProgress: this.bridgeProgress(params),
             });
 
             const success = result?.success === true;
@@ -46,6 +55,54 @@ export class TaskGraphAgentRunner implements TaskAgentRunner {
         }
     }
 
+    /**
+     * Adapts the agent framework's progress callback onto the dispatcher's plain sink.
+     *
+     * Undefined when nobody is listening, so the agent run pays nothing for observability it is not
+     * providing. Never throws — a progress announcement must not be able to fail the step.
+     */
+    private bridgeProgress(params: TaskAgentRunParams): AgentExecutionProgressCallback | undefined {
+        const sink = params.OnProgress;
+        if (!sink) return undefined;
+        return (progress) => {
+            try {
+                sink(progress.message, progress.percentage);
+            } catch {
+                // The dispatcher's sink already guards itself; this is belt over braces.
+            }
+        };
+    }
+
+    /**
+     * The run that submitted this graph, as the entity `RunAgent` wants for `parentRun`.
+     *
+     * **Why this link is worth a load.** It sets `ParentRunID` on the spawned run, which turns a
+     * workflow's total cost into one indexed sum over `RootParentRunID` rather than a walk of the
+     * graph, and gives the run a "started by" trail someone can follow back. Without it every
+     * dispatched run looks like it began from nowhere.
+     *
+     * Undefined — never a throw — when there is no submitting run or it cannot be read. A graph
+     * submitted by a schedule, by MCP, or by a person genuinely has no parent run, and failing the
+     * step because its provenance is unavailable would trade real work for a bookkeeping detail.
+     */
+    private async loadSubmittingRun(params: TaskAgentRunParams): Promise<MJAIAgentRunEntityExtended | undefined> {
+        if (!params.SubmittingAgentRunID) return undefined;
+        try {
+            const run = await params.Provider.GetEntityObject<MJAIAgentRunEntityExtended>(
+                'MJ: AI Agent Runs',
+                params.ContextUser,
+            );
+            return (await run.Load(params.SubmittingAgentRunID)) ? run : undefined;
+        } catch (e) {
+            LogError(
+                `[TaskGraphAgentRunner] Task ${params.TaskID}: submitting run ` +
+                `${params.SubmittingAgentRunID} could not be loaded; the spawned run will have no ` +
+                `parent link. ${e instanceof Error ? e.message : String(e)}`,
+            );
+            return undefined;
+        }
+    }
+
     /**
      * Builds the agent's instruction from the task's own input plus its dependencies' outputs.
      *

package/src/services/TaskGraphContinuationDeliverer.ts

@@ -135,7 +135,17 @@ export class TaskGraphContinuationDeliverer implements TaskContinuationDeliverer
                 continuationDepth: params.ReinvokeDepth + 1,
             });
         } catch (e) {
-            LogError(`[TaskGraphContinuationDeliverer] Reinvoking for "${params.WorkflowName}" threw`, undefined, e);
+            // FALL BACK, don't just log (C3). Both load-failure paths above already post instead of
+            // reinvoking, on the principle that a completed workflow's outcome must reach the user
+            // somehow — but a throw out of `RunAgent` skipped that and lost the outcome entirely.
+            // The marker has been claimed by the time this runs, so nothing will look at the graph
+            // again: the work finished, and the only record of it is a server log line.
+            LogError(`[TaskGraphContinuationDeliverer] Reinvoking for "${params.WorkflowName}" threw — posting instead`, undefined, e);
+            try {
+                await this.PostMessage(params);
+            } catch (fallbackError) {
+                LogError(`[TaskGraphContinuationDeliverer] The fallback post for "${params.WorkflowName}" also failed`, undefined, fallbackError);
+            }
         }
     }
 

package/src/services/TaskGraphPromptRunner.ts

@@ -0,0 +1,144 @@
+/**
+ * @fileoverview MJServer's implementation of the task-graph `TaskPromptRunner` seam.
+ *
+ * The third sibling of {@link TaskGraphAgentRunner} and {@link TaskGraphActionRunner}, and here for
+ * the same reason: the dispatcher knows *when* a prompt-assigned node should run and nothing about
+ * how a prompt executes. The AI engine depends on the entity layer the dispatcher also builds on, so
+ * importing it into `@memberjunction/task-graph` would make every consumer of durable execution load
+ * the prompt engine too.
+ *
+ * **Why prompt nodes matter enough to have their own runner.** The `User Onboarding Flow Agent`
+ * ships with six of them. Until this existed, every one of its steps was refused at submission and
+ * the agent could not run at all — the refusal was correct (a persisted step nothing can execute
+ * waits forever) but it made a shipped workflow unusable rather than merely unfinished.
+ *
+ * @module @memberjunction/server
+ */
+import { AIPromptRunner } from '@memberjunction/ai-prompts';
+import { AIEngine } from '@memberjunction/aiengine';
+import { AIPromptParams } from '@memberjunction/ai-core-plus';
+import { LogError } from '@memberjunction/core';
+import { UUIDsEqual } from '@memberjunction/global';
+import type { TaskPromptRunner, TaskPromptRunParams, TaskPromptRunResult } from '@memberjunction/task-graph';
+
+/** The placeholder a workflow prompt uses to receive the payload (plan §5.7). */
+const CURRENT_PAYLOAD_PLACEHOLDER = '_CURRENT_PAYLOAD';
+
+export class TaskGraphPromptRunner implements TaskPromptRunner {
+    public async RunPromptForTask(params: TaskPromptRunParams): Promise<TaskPromptRunResult> {
+        try {
+            await AIEngine.Instance.Config(false, params.ContextUser);
+            const prompt = AIEngine.Instance.Prompts.find((p) => UUIDsEqual(p.ID, params.PromptID));
+            if (!prompt) {
+                return { Success: false, ErrorMessage: `Prompt ${params.PromptID} is not in the engine's metadata.` };
+            }
+
+            const promptParams = new AIPromptParams();
+            promptParams.prompt = prompt;
+            promptParams.contextUser = params.ContextUser;
+            promptParams.data = this.buildData(params);
+
+            const result = await new AIPromptRunner().ExecutePrompt(promptParams);
+            if (!result?.success) {
+                return {
+                    Success: false,
+                    ErrorMessage: result?.errorMessage ?? 'The prompt did not return a result.',
+                    PromptRunID: result?.promptRun?.ID,
+                };
+            }
+
+            const parsed = this.parseResponse(result.result);
+            return {
+                Success: true,
+                Output: parsed.payload,
+                PromptRunID: result.promptRun?.ID,
+                ChatMessage: parsed.chatMessage,
+            };
+        } catch (e) {
+            const message = e instanceof Error ? e.message : String(e);
+            LogError(`[TaskGraphPromptRunner] Task ${params.TaskID} failed: ${message}`);
+            return { Success: false, ErrorMessage: message };
+        }
+    }
+
+    /**
+     * What the prompt's template can see.
+     *
+     * The payload arrives under a **placeholder name** rather than being spread across the top level,
+     * so a workflow author writes `{{ _CURRENT_PAYLOAD }}` and gets the whole thing — and a payload
+     * key can never collide with a template parameter and silently win.
+     *
+     * `flowContext` carries what the step knows about its own position: what its prerequisites
+     * produced. Without it a prompt is reasoning about the payload alone and cannot refer to the step
+     * immediately before it.
+     */
+    private buildData(params: TaskPromptRunParams): Record<string, unknown> {
+        const payload = params.InputPayload && typeof params.InputPayload === 'object'
+            ? params.InputPayload as Record<string, unknown>
+            : {};
+
+        const dependencyOutputs: Record<string, unknown> = {};
+        for (const [taskID, output] of params.DependencyOutputs) dependencyOutputs[taskID] = output;
+
+        return {
+            ...(params.TemplateParameters ?? {}),
+            // JSON, not the object. A template writes `{{ _CURRENT_PAYLOAD }}` and expects to SEE
+            // the payload; handing the renderer an object yields "[object Object]", and the model
+            // then answers about a value it was never shown. That failure is invisible from the
+            // outside — the prompt run succeeds, the response is well-formed, and it is about
+            // nothing. It is what kept the Content Pipeline reviewer rejecting a draft it could not
+            // read, on every iteration, forever.
+            [CURRENT_PAYLOAD_PLACEHOLDER]: JSON.stringify(payload, null, 2),
+            flowContext: JSON.stringify({ dependencyOutputs }, null, 2),
+        };
+    }
+
+    /**
+     * Reads the model's response as a payload update, and notices when it wants to stop.
+     *
+     * **A prompt is the only node kind that can end the workflow early**, because it is the only one
+     * doing open-ended reasoning: it can conclude the remaining steps are unnecessary. That is
+     * expressed as a Chat-shaped response, and the dispatcher honours it by skipping what remains and
+     * settling the parent Complete — rather than treating an early finish as an abandoned graph.
+     *
+     * Non-JSON is not an error. A prompt that answers in prose has still answered; it simply carries
+     * no structured update, so it becomes a message rather than a payload merge.
+     */
+    private parseResponse(raw: unknown): { payload?: Record<string, unknown>; chatMessage?: string } {
+        if (raw == null) return {};
+
+        if (typeof raw === 'object') {
+            return this.interpret(raw as Record<string, unknown>);
+        }
+
+        if (typeof raw === 'string') {
+            const text = raw.trim();
+            const fenced = text.replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
+            try {
+                const parsed: unknown = JSON.parse(fenced);
+                if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+                    return this.interpret(parsed as Record<string, unknown>);
+                }
+            } catch {
+                // Prose, not JSON — see above.
+            }
+            return { chatMessage: text };
+        }
+
+        return {};
+    }
+
+    /** Separates "here is an update" from "we are done, tell them this". */
+    private interpret(obj: Record<string, unknown>): { payload?: Record<string, unknown>; chatMessage?: string } {
+        const nextStep = obj['nextStep'] as { type?: string } | undefined;
+        const wantsChat = nextStep?.type === 'Chat' || obj['taskComplete'] === true;
+        const message = typeof obj['message'] === 'string' ? obj['message'] : undefined;
+
+        if (wantsChat && message) {
+            // The payload still merges: a prompt that both concluded and produced data should not
+            // lose the data because it also had something to say.
+            return { payload: obj, chatMessage: message };
+        }
+        return { payload: obj };
+    }
+}

package/src/telephony/RingCentralTelephonyService.ts

@@ -19,7 +19,7 @@
 
 import { RunView, UserInfo, IMetadataProvider, Metadata, LogError, LogStatus } from '@memberjunction/core';
 import type { MJAIBridgeAgentIdentityEntity, MJAIBridgeProviderEntity } from '@memberjunction/core-entities';
-import { UserCache } from '@memberjunction/sqlserver-dataprovider';
+import { UserCache } from '@memberjunction/generic-database-provider';
 import { AIBridgeEngine } from '@memberjunction/ai-bridge-server';
 import { CreateBridgeRealtimeSession } from '@memberjunction/ai-agents';
 import {

package/src/telephony/TwilioTelephonyRouter.ts

@@ -20,7 +20,7 @@ import { Router, urlencoded, type Request, type Response } from 'express';
 import { WebSocketServer, type WebSocket } from 'ws';
 import { RegisterMediaUpgradeRoute } from './media-upgrade-router.js';
 import { LogError, LogStatus, UserInfo, IMetadataProvider, Metadata } from '@memberjunction/core';
-import { UserCache } from '@memberjunction/sqlserver-dataprovider';
+import { UserCache } from '@memberjunction/generic-database-provider';
 import {
     verifyTwilioSignature,
     resolveInboundCall,

package/src/telephony/VonageTelephonyRouter.ts

@@ -26,7 +26,7 @@ import { WebSocketServer, type WebSocket } from 'ws';
 import type { IncomingMessage } from 'node:http';
 import { RegisterMediaUpgradeRoute } from './media-upgrade-router.js';
 import { LogError, LogStatus, UserInfo, IMetadataProvider, Metadata } from '@memberjunction/core';
-import { UserCache } from '@memberjunction/sqlserver-dataprovider';
+import { UserCache } from '@memberjunction/generic-database-provider';
 import {
     verifyVonageSignature,
     verifyVonageJwt,

package/src/types.ts

@@ -107,6 +107,12 @@ export type RunViewGenericParams = {
    * happens. Propagated to `RunViewParams.BypassCache`.
    */
   bypassCache?: boolean;
+  /**
+   * Optional source-of-truth selector for entities that have a base-view materialization.
+   * 'Materialized' routes the read to the entity's materialized wrapper view; defaults to 'Live'.
+   * Propagated to `RunViewParams.DataSource`.
+   */
+  dataSource?: 'Live' | 'Materialized';
 };