@memberjunction/server 6.2.0-edge.1 → 6.2.0-edge.3

package/src/generic/FireAndForgetHeartbeat.ts

@@ -2,8 +2,22 @@ import { PubSubEngine } from 'type-graphql';
 import { LogError } from '@memberjunction/core';
 import { PublishStatusUpdate } from './PushStatusResolver.js';
 
-/** Default cadence for fire-and-forget liveness pulses (5 minutes). */
-export const DEFAULT_PULSE_INTERVAL_MS = 5 * 60 * 1000;
+/**
+ * Default cadence for fire-and-forget liveness pulses (60 seconds).
+ *
+ * MATCHED PAIR — do not change alone. The client's inactivity window
+ * (`DEFAULT_IDLE_TIMEOUT_MS` in GraphQLDataProvider's fireAndForgetHelper) is sized at roughly
+ * 3x this value, so a healthy run always refreshes the timer well before it expires. Lengthening
+ * this without lengthening that window makes every quiet stretch of a long agent run look like a
+ * stall and trips reconciliation; shortening the window without shortening this does the same.
+ *
+ * WAS 5 MINUTES (MJ #4222). Paired with the old 12-minute window, a client whose transport had
+ * silently died waited 12 minutes to notice — on runs that typically finish in 20 seconds. The
+ * pulse is a few dozen bytes and exists only while an operation is actually running, so the
+ * added traffic is negligible: a 40-minute run now sends ~40 pulses instead of ~8, and the
+ * 20-second run that motivated the change still sends none.
+ */
+export const DEFAULT_PULSE_INTERVAL_MS = 60 * 1000;
 
 /** The `type` discriminator carried by liveness pulse messages. */
 export const HEARTBEAT_MESSAGE_TYPE = 'Heartbeat';

package/src/generic/OrphanedConversationDetailReconciler.ts

@@ -0,0 +1,267 @@
+import { LogError, LogStatus, RunView } from '@memberjunction/core';
+import { UUIDsEqual } from '@memberjunction/global';
+import { UserCache } from '@memberjunction/sqlserver-dataprovider';
+import type { IMetadataProvider, UserInfo } from '@memberjunction/core';
+import type { MJAIAgentRunEntity, MJConversationDetailEntity } from '@memberjunction/core-entities';
+
+/**
+ * Grace period after a run reaches a terminal state before its conversation detail is considered
+ * abandoned. The normal path closes the detail moments after the run finishes; this waits well past
+ * that so a reconcile can never race a completion that is already on its way.
+ */
+export const ORPHAN_DETAIL_GRACE_MS = 2 * 60 * 1000;
+
+/** Ceiling on details closed per pass, so a large backlog cannot stall startup. */
+const MAX_DETAILS_PER_PASS = 200;
+
+/**
+ * Agent-run statuses that mean the run succeeded. `AwaitingFeedback` is the normal end of a chat
+ * turn: the agent replied and the next move is the user's.
+ */
+const SUCCESSFUL_RUN_STATUSES: MJAIAgentRunEntity['Status'][] = ['Completed', 'AwaitingFeedback'];
+
+/**
+ * Agent-run statuses that mean execution is over, whatever the outcome. `Paused` is absent: the
+ * run is parked on a workflow that is still executing.
+ */
+const TERMINAL_RUN_STATUSES: MJAIAgentRunEntity['Status'][] = [...SUCCESSFUL_RUN_STATUSES, 'Failed', 'Cancelled'];
+
+/**
+ * Close conversation details left `In-Progress` by a run that is already over (MJ #4222).
+ *
+ * WHY THIS EXISTS. `AgentRunner` closes the detail as the final step of a run, so a process that
+ * dies mid-run never reaches it. The agent-run watchdog repairs the RUN — force-failing it once the
+ * heartbeat goes stale — but nothing repairs the detail, and the detail is the row the chat renders
+ * from. The result is durable state that disagrees with itself: a terminal run beside a message
+ * that still says it is being generated, spinning forever for anyone who opens it.
+ *
+ * WHY NOT IN THE SWEEP PROC. `spSweepStaleAIAgentRuns` is a single set-based UPDATE, and extending
+ * it to the detail would be the obvious fix. It would also be the wrong one: `MJ: Conversation
+ * Details` declares `TrustServerCacheCompletely`, so the server's RunView cache relies on
+ * `BaseEntity` events for invalidation and a stored procedure fires none. The row would change
+ * while every instance kept serving `In-Progress` from cache — a worse failure than the one being
+ * fixed, because reconciliation itself would read the stale value back. Going through
+ * `BaseEntity.Save()` costs a per-row write and buys correct invalidation everywhere.
+ *
+ * Idempotent and safe to run concurrently on several instances: the write is a no-op once the
+ * status is terminal, and each pass re-reads state before deciding.
+ *
+ * @returns How many details were closed.
+ */
+export async function ReconcileOrphanedConversationDetails(
+    provider: IMetadataProvider,
+    contextUser: UserInfo,
+    graceMs: number = ORPHAN_DETAIL_GRACE_MS
+): Promise<number> {
+    try {
+        const rv = RunView.FromMetadataProvider(provider);
+
+        const detailResult = await rv.RunView<MJConversationDetailEntity>(
+            {
+                EntityName: 'MJ: Conversation Details',
+                ExtraFilter: `Status = 'In-Progress' AND Role = 'AI'${hasTerminalRunClause(provider)}`,
+                OrderBy: '__mj_CreatedAt ASC',
+                MaxRows: MAX_DETAILS_PER_PASS,
+                ResultType: 'entity_object',
+            },
+            contextUser
+        );
+        if (!detailResult.Success || !detailResult.Results?.length) {
+            return 0;
+        }
+
+        const runsByDetail = await loadTerminalRuns(rv, detailResult.Results.map(d => d.ID), contextUser);
+        if (runsByDetail.size === 0) {
+            return 0;
+        }
+
+        const ownersByConversation = await loadConversationOwners(
+            rv,
+            detailResult.Results.map(d => d.ConversationID).filter((id): id is string => !!id),
+            contextUser
+        );
+
+        const cutoff = Date.now() - graceMs;
+        let closed = 0;
+
+        for (const detail of detailResult.Results) {
+            const run = runsByDetail.get(detail.ID);
+            if (!run) {
+                // No run yet, or its run is still executing. Both are legitimately in progress.
+                continue;
+            }
+            const completedAt = run.CompletedAt ? new Date(run.CompletedAt).getTime() : NaN;
+            if (!Number.isFinite(completedAt) || completedAt > cutoff) {
+                // Inside the grace window — the owning process may still be closing this itself.
+                continue;
+            }
+
+            // Write as the conversation's OWNER, not as the system user.
+            //
+            // `MJConversationDetailEntityExtended.Save()` gates writes on conversation ownership: a
+            // non-owner without a resource grant is refused, returning false with no result
+            // registered anywhere — indistinguishable from a crash, and the reason this pass
+            // silently wrote nothing at all until it was traced. The system user owns no
+            // conversations, so it is exactly the caller that gate exists to stop. Saving as the
+            // owner satisfies the gate on its own terms rather than routing around it, and keeps
+            // the write attributable to the person whose message it is.
+            const owner = detail.ConversationID ? ownersByConversation.get(detail.ConversationID) : undefined;
+            if (!owner) {
+                LogError(`[OrphanDetailReconciler] No owner resolved for conversation ${detail.ConversationID}; leaving detail ${detail.ID}`);
+                continue;
+            }
+            const writable = await provider.GetEntityObject<MJConversationDetailEntity>('MJ: Conversation Details', owner);
+            if (!(await writable.Load(detail.ID))) {
+                LogError(`[OrphanDetailReconciler] Could not load detail ${detail.ID} for write`);
+                continue;
+            }
+            if (writable.Status !== 'In-Progress') {
+                // Closed by something else since the list query. Its outcome stands.
+                continue;
+            }
+            // The failure marker belongs only to the error branch. A completed run has nothing to
+            // report, so an empty message stays empty rather than contradicting its own status.
+            if (SUCCESSFUL_RUN_STATUSES.includes(run.Status)) {
+                writable.Status = 'Complete';
+            } else {
+                writable.Status = 'Error';
+                if (!writable.Message) {
+                    writable.Message = run.ErrorMessage ?? '❌ Failed';
+                }
+            }
+
+            if (await writable.Save()) {
+                closed++;
+            } else {
+                const result = writable.LatestResult;
+                LogError(
+                    `[OrphanDetailReconciler] Failed to close detail ${detail.ID}: ` +
+                    `${result?.CompleteMessage || result?.Message || 'no message'}`
+                );
+            }
+        }
+
+        if (closed > 0) {
+            LogStatus(`[OrphanDetailReconciler] Closed ${closed} conversation detail(s) left behind by finished runs`);
+        }
+        return closed;
+    } catch (err) {
+        // Maintenance work must never take the server down with it.
+        LogError(`[OrphanDetailReconciler] Pass failed: ${err instanceof Error ? err.message : String(err)}`);
+        return 0;
+    }
+}
+
+/** The quoting a database provider offers. Present on every server-side provider. */
+interface QuotingProvider {
+    QuoteIdentifier(name: string): string;
+    QuoteSchemaAndView(schemaName: string, objectName: string): string;
+}
+
+function isQuotingProvider(provider: IMetadataProvider): provider is IMetadataProvider & QuotingProvider {
+    const p = provider as IMetadataProvider & Partial<QuotingProvider>;
+    return typeof p.QuoteIdentifier === 'function' && typeof p.QuoteSchemaAndView === 'function';
+}
+
+/**
+ * Filter clause that keeps only details with at least one finished run.
+ *
+ * Details with no run, or whose run is still executing, are never closed here. Without this clause
+ * they fill the oldest-first window, and once 200 of them accumulate no closable orphan is reached.
+ * The newest-run check in {@link loadTerminalRuns} still decides each remaining detail.
+ *
+ * @returns An ` AND ...` clause, or an empty string when the provider cannot quote names.
+ */
+function hasTerminalRunClause(provider: IMetadataProvider): string {
+    const runs = provider.EntityByName('MJ: AI Agent Runs');
+    if (!runs || !isQuotingProvider(provider)) {
+        return '';
+    }
+    const statuses = TERMINAL_RUN_STATUSES.map(s => `'${s}'`).join(',');
+    return (
+        ` AND ID IN (SELECT ${provider.QuoteIdentifier('ConversationDetailID')}` +
+        ` FROM ${provider.QuoteSchemaAndView(runs.SchemaName, runs.BaseView)}` +
+        ` WHERE ${provider.QuoteIdentifier('Status')} IN (${statuses}))`
+    );
+}
+
+/**
+ * Newest terminal run per conversation detail. Details whose newest run is still executing are
+ * deliberately absent, so a retried message is judged on its latest attempt rather than an older
+ * finished one.
+ */
+async function loadTerminalRuns(
+    rv: RunView,
+    detailIds: string[],
+    contextUser: UserInfo
+): Promise<Map<string, MJAIAgentRunEntity>> {
+    const quoted = detailIds.map(id => `'${id.replace(/'/g, "''")}'`).join(',');
+    const result = await rv.RunView<MJAIAgentRunEntity>(
+        {
+            EntityName: 'MJ: AI Agent Runs',
+            ExtraFilter: `ConversationDetailID IN (${quoted})`,
+            OrderBy: '__mj_CreatedAt DESC',
+            ResultType: 'entity_object',
+        },
+        contextUser
+    );
+    if (!result.Success) {
+        return new Map();
+    }
+
+    const newest = new Map<string, MJAIAgentRunEntity>();
+    for (const run of result.Results ?? []) {
+        const detailId = run.ConversationDetailID;
+        if (!detailId || newest.has(detailId)) {
+            continue; // DESC order puts the newest run first
+        }
+        newest.set(detailId, run);
+    }
+
+    // Drop any detail whose newest attempt is still running.
+    for (const [detailId, run] of newest) {
+        if (!TERMINAL_RUN_STATUSES.includes(run.Status)) {
+            newest.delete(detailId);
+        }
+    }
+    return newest;
+}
+
+/**
+ * Owner `UserInfo` per conversation, resolved through {@link UserCache}. The reconciler writes as
+ * the owner so the conversation-detail permission gate is satisfied on its own terms.
+ */
+async function loadConversationOwners(
+    rv: RunView,
+    conversationIds: string[],
+    contextUser: UserInfo
+): Promise<Map<string, UserInfo>> {
+    const owners = new Map<string, UserInfo>();
+    if (conversationIds.length === 0) {
+        return owners;
+    }
+    const quoted = [...new Set(conversationIds)].map(id => `'${id.replace(/'/g, "''")}'`).join(',');
+    const result = await rv.RunView<{ ID: string; UserID: string }>(
+        {
+            EntityName: 'MJ: Conversations',
+            ExtraFilter: `ID IN (${quoted})`,
+            Fields: ['ID', 'UserID'],
+            ResultType: 'simple',
+        },
+        contextUser
+    );
+    if (!result.Success) {
+        return owners;
+    }
+    const users = UserCache.Instance.Users;
+    for (const row of result.Results ?? []) {
+        if (!row.UserID) {
+            continue;
+        }
+        const user = users.find(u => UUIDsEqual(u.ID, row.UserID));
+        if (user) {
+            owners.set(row.ID, user);
+        }
+    }
+    return owners;
+}

package/src/generic/PushStatusResolver.ts

@@ -1,5 +1,5 @@
 import { Arg, Field, ID, ObjectType, PubSubEngine, Resolver, ResolverFilterData, Root, Subscription } from 'type-graphql';
-import { UUIDsEqual } from '@memberjunction/global';
+import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
 import { UserPayload } from '../types.js';
 
 export const PUSH_STATUS_UPDATES_TOPIC = 'PUSH_STATUS_UPDATES';
@@ -32,6 +32,89 @@ export interface PushStatusNotificationPayload {
   sessionId: string;
   /** Authenticated user the update belongs to. Server-side filter key; never sent to the client. */
   ownerUserId: string;
+  /**
+   * `MJGlobal.ProcessUUID` of the instance that first published this update. Used only for
+   * cross-instance echo suppression: an instance that receives its own message back from the
+   * message bus must drop it, or every push is delivered twice.
+   */
+  SourceServerId?: string;
+}
+
+/** Receives every locally-published status update, for fan-out to other server instances. */
+export type PushStatusPublishHook = (payload: PushStatusNotificationPayload) => void;
+
+let _publishHook: PushStatusPublishHook | undefined;
+
+/**
+ * Register (or clear, with no arg) the cross-instance fan-out hook.
+ *
+ * Inversion of control, deliberately: this module must not know that Redis exists. The server
+ * registers the hook at startup only when a message bus is configured, so a single-instance
+ * deployment behaves exactly as before.
+ */
+export function SetPushStatusPublishHook(hook?: PushStatusPublishHook): void {
+  _publishHook = hook;
+}
+
+/**
+ * Validate an update received from the cross-instance message bus.
+ *
+ * The bus carries messages from other processes, so every field is type-checked before the update
+ * is republished onto the local topic, where {@link StatusUpdatesFilter} compares `ownerUserId`
+ * with the connection's user.
+ *
+ * @param raw The JSON text taken off the bus.
+ * @param localServerId This process's `MJGlobal.ProcessUUID`, used to drop our own echo.
+ * @returns The update to republish, or `null` to drop it.
+ */
+export function ParseReplicatedStatusUpdate(raw: string, localServerId: string): PushStatusNotificationPayload | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return null;
+  }
+  if (typeof parsed !== 'object' || parsed === null) {
+    return null;
+  }
+
+  const { sessionId, ownerUserId, message, SourceServerId } = parsed as Record<string, unknown>;
+  const source = typeof SourceServerId === 'string' ? SourceServerId : undefined;
+  const body = typeof message === 'string' ? message : undefined;
+  if ((SourceServerId != null && source === undefined) || (message != null && body === undefined)) {
+    return null; // a present field of the wrong type
+  }
+  if (source === localServerId) {
+    return null; // our own message, echoed back
+  }
+  if (typeof sessionId !== 'string' || !sessionId || typeof ownerUserId !== 'string' || !ownerUserId) {
+    return null; // fail closed: an update with no valid identity can never be routed safely
+  }
+
+  return { sessionId, ownerUserId, message: body, SourceServerId: source };
+}
+
+/**
+ * Whether an update is worth sending to other server instances.
+ *
+ * Streaming content is excluded. It arrives at hundreds of messages per second and each one carries
+ * a full serialized agent run, so replicating it would cost far more bandwidth than the delivery it
+ * buys — and a token delta is worthless to a client that has already missed the ones before it.
+ * Progress and completion are low-rate and each is individually meaningful, which is exactly the
+ * kind of message that must survive landing on the wrong replica.
+ */
+export function ShouldReplicateStatusUpdate(message?: string): boolean {
+  if (!message) {
+    return false;
+  }
+  try {
+    const parsed: unknown = JSON.parse(message);
+    const type = (parsed as { type?: unknown })?.type;
+    return type !== 'StreamingContent';
+  } catch {
+    // A plain-string message (not the resolvers' JSON envelope) is rare and low-rate. Replicate it.
+    return true;
+  }
 }
 
 interface PushStatusNotificationArgs {
@@ -60,8 +143,21 @@ export function PublishStatusUpdate(pubSub: PubSubEngine, params: StatusUpdatePa
     sessionId: params.sessionId,
     ownerUserId: params.ownerUserId,
     message: params.message,
+    SourceServerId: MJGlobal.Instance.ProcessUUID,
   };
   pubSub.publish(PUSH_STATUS_UPDATES_TOPIC, payload);
+
+  // Fan out to other instances. Only reached by updates originating HERE: a message arriving from
+  // the bus is republished straight onto the local topic and never comes back through this
+  // function, so there is no loop to break beyond the SourceServerId check on the receiving side.
+  if (_publishHook && ShouldReplicateStatusUpdate(params.message)) {
+    try {
+      _publishHook(payload);
+    } catch {
+      // Fan-out is an optimization over the durable tail query. A message bus that is down must
+      // never break local delivery, which is the path that works for the common case.
+    }
+  }
 }
 
 /** @deprecated Use {@link PublishStatusUpdate}. */