@memberjunction/server 6.2.0-edge.1 → 6.2.0-edge.2

package/src/generic/FireAndForgetHeartbeat.ts

@@ -2,8 +2,22 @@ import { PubSubEngine } from 'type-graphql';
 import { LogError } from '@memberjunction/core';
 import { PublishStatusUpdate } from './PushStatusResolver.js';
 
-/** Default cadence for fire-and-forget liveness pulses (5 minutes). */
-export const DEFAULT_PULSE_INTERVAL_MS = 5 * 60 * 1000;
+/**
+ * Default cadence for fire-and-forget liveness pulses (60 seconds).
+ *
+ * MATCHED PAIR — do not change alone. The client's inactivity window
+ * (`DEFAULT_IDLE_TIMEOUT_MS` in GraphQLDataProvider's fireAndForgetHelper) is sized at roughly
+ * 3x this value, so a healthy run always refreshes the timer well before it expires. Lengthening
+ * this without lengthening that window makes every quiet stretch of a long agent run look like a
+ * stall and trips reconciliation; shortening the window without shortening this does the same.
+ *
+ * WAS 5 MINUTES (MJ #4222). Paired with the old 12-minute window, a client whose transport had
+ * silently died waited 12 minutes to notice — on runs that typically finish in 20 seconds. The
+ * pulse is a few dozen bytes and exists only while an operation is actually running, so the
+ * added traffic is negligible: a 40-minute run now sends ~40 pulses instead of ~8, and the
+ * 20-second run that motivated the change still sends none.
+ */
+export const DEFAULT_PULSE_INTERVAL_MS = 60 * 1000;
 
 /** The `type` discriminator carried by liveness pulse messages. */
 export const HEARTBEAT_MESSAGE_TYPE = 'Heartbeat';

package/src/generic/OrphanedConversationDetailReconciler.ts

@@ -0,0 +1,267 @@
+import { LogError, LogStatus, RunView } from '@memberjunction/core';
+import { UUIDsEqual } from '@memberjunction/global';
+import { UserCache } from '@memberjunction/sqlserver-dataprovider';
+import type { IMetadataProvider, UserInfo } from '@memberjunction/core';
+import type { MJAIAgentRunEntity, MJConversationDetailEntity } from '@memberjunction/core-entities';
+
+/**
+ * Grace period after a run reaches a terminal state before its conversation detail is considered
+ * abandoned. The normal path closes the detail moments after the run finishes; this waits well past
+ * that so a reconcile can never race a completion that is already on its way.
+ */
+export const ORPHAN_DETAIL_GRACE_MS = 2 * 60 * 1000;
+
+/** Ceiling on details closed per pass, so a large backlog cannot stall startup. */
+const MAX_DETAILS_PER_PASS = 200;
+
+/**
+ * Agent-run statuses that mean the run succeeded. `AwaitingFeedback` is the normal end of a chat
+ * turn: the agent replied and the next move is the user's.
+ */
+const SUCCESSFUL_RUN_STATUSES: MJAIAgentRunEntity['Status'][] = ['Completed', 'AwaitingFeedback'];
+
+/**
+ * Agent-run statuses that mean execution is over, whatever the outcome. `Paused` is absent: the
+ * run is parked on a workflow that is still executing.
+ */
+const TERMINAL_RUN_STATUSES: MJAIAgentRunEntity['Status'][] = [...SUCCESSFUL_RUN_STATUSES, 'Failed', 'Cancelled'];
+
+/**
+ * Close conversation details left `In-Progress` by a run that is already over (MJ #4222).
+ *
+ * WHY THIS EXISTS. `AgentRunner` closes the detail as the final step of a run, so a process that
+ * dies mid-run never reaches it. The agent-run watchdog repairs the RUN — force-failing it once the
+ * heartbeat goes stale — but nothing repairs the detail, and the detail is the row the chat renders
+ * from. The result is durable state that disagrees with itself: a terminal run beside a message
+ * that still says it is being generated, spinning forever for anyone who opens it.
+ *
+ * WHY NOT IN THE SWEEP PROC. `spSweepStaleAIAgentRuns` is a single set-based UPDATE, and extending
+ * it to the detail would be the obvious fix. It would also be the wrong one: `MJ: Conversation
+ * Details` declares `TrustServerCacheCompletely`, so the server's RunView cache relies on
+ * `BaseEntity` events for invalidation and a stored procedure fires none. The row would change
+ * while every instance kept serving `In-Progress` from cache — a worse failure than the one being
+ * fixed, because reconciliation itself would read the stale value back. Going through
+ * `BaseEntity.Save()` costs a per-row write and buys correct invalidation everywhere.
+ *
+ * Idempotent and safe to run concurrently on several instances: the write is a no-op once the
+ * status is terminal, and each pass re-reads state before deciding.
+ *
+ * @returns How many details were closed.
+ */
+export async function ReconcileOrphanedConversationDetails(
+    provider: IMetadataProvider,
+    contextUser: UserInfo,
+    graceMs: number = ORPHAN_DETAIL_GRACE_MS
+): Promise<number> {
+    try {
+        const rv = RunView.FromMetadataProvider(provider);
+
+        const detailResult = await rv.RunView<MJConversationDetailEntity>(
+            {
+                EntityName: 'MJ: Conversation Details',
+                ExtraFilter: `Status = 'In-Progress' AND Role = 'AI'${hasTerminalRunClause(provider)}`,
+                OrderBy: '__mj_CreatedAt ASC',
+                MaxRows: MAX_DETAILS_PER_PASS,
+                ResultType: 'entity_object',
+            },
+            contextUser
+        );
+        if (!detailResult.Success || !detailResult.Results?.length) {
+            return 0;
+        }
+
+        const runsByDetail = await loadTerminalRuns(rv, detailResult.Results.map(d => d.ID), contextUser);
+        if (runsByDetail.size === 0) {
+            return 0;
+        }
+
+        const ownersByConversation = await loadConversationOwners(
+            rv,
+            detailResult.Results.map(d => d.ConversationID).filter((id): id is string => !!id),
+            contextUser
+        );
+
+        const cutoff = Date.now() - graceMs;
+        let closed = 0;
+
+        for (const detail of detailResult.Results) {
+            const run = runsByDetail.get(detail.ID);
+            if (!run) {
+                // No run yet, or its run is still executing. Both are legitimately in progress.
+                continue;
+            }
+            const completedAt = run.CompletedAt ? new Date(run.CompletedAt).getTime() : NaN;
+            if (!Number.isFinite(completedAt) || completedAt > cutoff) {
+                // Inside the grace window — the owning process may still be closing this itself.
+                continue;
+            }
+
+            // Write as the conversation's OWNER, not as the system user.
+            //
+            // `MJConversationDetailEntityExtended.Save()` gates writes on conversation ownership: a
+            // non-owner without a resource grant is refused, returning false with no result
+            // registered anywhere — indistinguishable from a crash, and the reason this pass
+            // silently wrote nothing at all until it was traced. The system user owns no
+            // conversations, so it is exactly the caller that gate exists to stop. Saving as the
+            // owner satisfies the gate on its own terms rather than routing around it, and keeps
+            // the write attributable to the person whose message it is.
+            const owner = detail.ConversationID ? ownersByConversation.get(detail.ConversationID) : undefined;
+            if (!owner) {
+                LogError(`[OrphanDetailReconciler] No owner resolved for conversation ${detail.ConversationID}; leaving detail ${detail.ID}`);
+                continue;
+            }
+            const writable = await provider.GetEntityObject<MJConversationDetailEntity>('MJ: Conversation Details', owner);
+            if (!(await writable.Load(detail.ID))) {
+                LogError(`[OrphanDetailReconciler] Could not load detail ${detail.ID} for write`);
+                continue;
+            }
+            if (writable.Status !== 'In-Progress') {
+                // Closed by something else since the list query. Its outcome stands.
+                continue;
+            }
+            // The failure marker belongs only to the error branch. A completed run has nothing to
+            // report, so an empty message stays empty rather than contradicting its own status.
+            if (SUCCESSFUL_RUN_STATUSES.includes(run.Status)) {
+                writable.Status = 'Complete';
+            } else {
+                writable.Status = 'Error';
+                if (!writable.Message) {
+                    writable.Message = run.ErrorMessage ?? '❌ Failed';
+                }
+            }
+
+            if (await writable.Save()) {
+                closed++;
+            } else {
+                const result = writable.LatestResult;
+                LogError(
+                    `[OrphanDetailReconciler] Failed to close detail ${detail.ID}: ` +
+                    `${result?.CompleteMessage || result?.Message || 'no message'}`
+                );
+            }
+        }
+
+        if (closed > 0) {
+            LogStatus(`[OrphanDetailReconciler] Closed ${closed} conversation detail(s) left behind by finished runs`);
+        }
+        return closed;
+    } catch (err) {
+        // Maintenance work must never take the server down with it.
+        LogError(`[OrphanDetailReconciler] Pass failed: ${err instanceof Error ? err.message : String(err)}`);
+        return 0;
+    }
+}
+
+/** The quoting a database provider offers. Present on every server-side provider. */
+interface QuotingProvider {
+    QuoteIdentifier(name: string): string;
+    QuoteSchemaAndView(schemaName: string, objectName: string): string;
+}
+
+function isQuotingProvider(provider: IMetadataProvider): provider is IMetadataProvider & QuotingProvider {
+    const p = provider as IMetadataProvider & Partial<QuotingProvider>;
+    return typeof p.QuoteIdentifier === 'function' && typeof p.QuoteSchemaAndView === 'function';
+}
+
+/**
+ * Filter clause that keeps only details with at least one finished run.
+ *
+ * Details with no run, or whose run is still executing, are never closed here. Without this clause
+ * they fill the oldest-first window, and once 200 of them accumulate no closable orphan is reached.
+ * The newest-run check in {@link loadTerminalRuns} still decides each remaining detail.
+ *
+ * @returns An ` AND ...` clause, or an empty string when the provider cannot quote names.
+ */
+function hasTerminalRunClause(provider: IMetadataProvider): string {
+    const runs = provider.EntityByName('MJ: AI Agent Runs');
+    if (!runs || !isQuotingProvider(provider)) {
+        return '';
+    }
+    const statuses = TERMINAL_RUN_STATUSES.map(s => `'${s}'`).join(',');
+    return (
+        ` AND ID IN (SELECT ${provider.QuoteIdentifier('ConversationDetailID')}` +
+        ` FROM ${provider.QuoteSchemaAndView(runs.SchemaName, runs.BaseView)}` +
+        ` WHERE ${provider.QuoteIdentifier('Status')} IN (${statuses}))`
+    );
+}
+
+/**
+ * Newest terminal run per conversation detail. Details whose newest run is still executing are
+ * deliberately absent, so a retried message is judged on its latest attempt rather than an older
+ * finished one.
+ */
+async function loadTerminalRuns(
+    rv: RunView,
+    detailIds: string[],
+    contextUser: UserInfo
+): Promise<Map<string, MJAIAgentRunEntity>> {
+    const quoted = detailIds.map(id => `'${id.replace(/'/g, "''")}'`).join(',');
+    const result = await rv.RunView<MJAIAgentRunEntity>(
+        {
+            EntityName: 'MJ: AI Agent Runs',
+            ExtraFilter: `ConversationDetailID IN (${quoted})`,
+            OrderBy: '__mj_CreatedAt DESC',
+            ResultType: 'entity_object',
+        },
+        contextUser
+    );
+    if (!result.Success) {
+        return new Map();
+    }
+
+    const newest = new Map<string, MJAIAgentRunEntity>();
+    for (const run of result.Results ?? []) {
+        const detailId = run.ConversationDetailID;
+        if (!detailId || newest.has(detailId)) {
+            continue; // DESC order puts the newest run first
+        }
+        newest.set(detailId, run);
+    }
+
+    // Drop any detail whose newest attempt is still running.
+    for (const [detailId, run] of newest) {
+        if (!TERMINAL_RUN_STATUSES.includes(run.Status)) {
+            newest.delete(detailId);
+        }
+    }
+    return newest;
+}
+
+/**
+ * Owner `UserInfo` per conversation, resolved through {@link UserCache}. The reconciler writes as
+ * the owner so the conversation-detail permission gate is satisfied on its own terms.
+ */
+async function loadConversationOwners(
+    rv: RunView,
+    conversationIds: string[],
+    contextUser: UserInfo
+): Promise<Map<string, UserInfo>> {
+    const owners = new Map<string, UserInfo>();
+    if (conversationIds.length === 0) {
+        return owners;
+    }
+    const quoted = [...new Set(conversationIds)].map(id => `'${id.replace(/'/g, "''")}'`).join(',');
+    const result = await rv.RunView<{ ID: string; UserID: string }>(
+        {
+            EntityName: 'MJ: Conversations',
+            ExtraFilter: `ID IN (${quoted})`,
+            Fields: ['ID', 'UserID'],
+            ResultType: 'simple',
+        },
+        contextUser
+    );
+    if (!result.Success) {
+        return owners;
+    }
+    const users = UserCache.Instance.Users;
+    for (const row of result.Results ?? []) {
+        if (!row.UserID) {
+            continue;
+        }
+        const user = users.find(u => UUIDsEqual(u.ID, row.UserID));
+        if (user) {
+            owners.set(row.ID, user);
+        }
+    }
+    return owners;
+}

package/src/generic/PushStatusResolver.ts

@@ -1,5 +1,5 @@
 import { Arg, Field, ID, ObjectType, PubSubEngine, Resolver, ResolverFilterData, Root, Subscription } from 'type-graphql';
-import { UUIDsEqual } from '@memberjunction/global';
+import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
 import { UserPayload } from '../types.js';
 
 export const PUSH_STATUS_UPDATES_TOPIC = 'PUSH_STATUS_UPDATES';
@@ -32,6 +32,89 @@ export interface PushStatusNotificationPayload {
   sessionId: string;
   /** Authenticated user the update belongs to. Server-side filter key; never sent to the client. */
   ownerUserId: string;
+  /**
+   * `MJGlobal.ProcessUUID` of the instance that first published this update. Used only for
+   * cross-instance echo suppression: an instance that receives its own message back from the
+   * message bus must drop it, or every push is delivered twice.
+   */
+  SourceServerId?: string;
+}
+
+/** Receives every locally-published status update, for fan-out to other server instances. */
+export type PushStatusPublishHook = (payload: PushStatusNotificationPayload) => void;
+
+let _publishHook: PushStatusPublishHook | undefined;
+
+/**
+ * Register (or clear, with no arg) the cross-instance fan-out hook.
+ *
+ * Inversion of control, deliberately: this module must not know that Redis exists. The server
+ * registers the hook at startup only when a message bus is configured, so a single-instance
+ * deployment behaves exactly as before.
+ */
+export function SetPushStatusPublishHook(hook?: PushStatusPublishHook): void {
+  _publishHook = hook;
+}
+
+/**
+ * Validate an update received from the cross-instance message bus.
+ *
+ * The bus carries messages from other processes, so every field is type-checked before the update
+ * is republished onto the local topic, where {@link StatusUpdatesFilter} compares `ownerUserId`
+ * with the connection's user.
+ *
+ * @param raw The JSON text taken off the bus.
+ * @param localServerId This process's `MJGlobal.ProcessUUID`, used to drop our own echo.
+ * @returns The update to republish, or `null` to drop it.
+ */
+export function ParseReplicatedStatusUpdate(raw: string, localServerId: string): PushStatusNotificationPayload | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return null;
+  }
+  if (typeof parsed !== 'object' || parsed === null) {
+    return null;
+  }
+
+  const { sessionId, ownerUserId, message, SourceServerId } = parsed as Record<string, unknown>;
+  const source = typeof SourceServerId === 'string' ? SourceServerId : undefined;
+  const body = typeof message === 'string' ? message : undefined;
+  if ((SourceServerId != null && source === undefined) || (message != null && body === undefined)) {
+    return null; // a present field of the wrong type
+  }
+  if (source === localServerId) {
+    return null; // our own message, echoed back
+  }
+  if (typeof sessionId !== 'string' || !sessionId || typeof ownerUserId !== 'string' || !ownerUserId) {
+    return null; // fail closed: an update with no valid identity can never be routed safely
+  }
+
+  return { sessionId, ownerUserId, message: body, SourceServerId: source };
+}
+
+/**
+ * Whether an update is worth sending to other server instances.
+ *
+ * Streaming content is excluded. It arrives at hundreds of messages per second and each one carries
+ * a full serialized agent run, so replicating it would cost far more bandwidth than the delivery it
+ * buys — and a token delta is worthless to a client that has already missed the ones before it.
+ * Progress and completion are low-rate and each is individually meaningful, which is exactly the
+ * kind of message that must survive landing on the wrong replica.
+ */
+export function ShouldReplicateStatusUpdate(message?: string): boolean {
+  if (!message) {
+    return false;
+  }
+  try {
+    const parsed: unknown = JSON.parse(message);
+    const type = (parsed as { type?: unknown })?.type;
+    return type !== 'StreamingContent';
+  } catch {
+    // A plain-string message (not the resolvers' JSON envelope) is rare and low-rate. Replicate it.
+    return true;
+  }
 }
 
 interface PushStatusNotificationArgs {
@@ -60,8 +143,21 @@ export function PublishStatusUpdate(pubSub: PubSubEngine, params: StatusUpdatePa
     sessionId: params.sessionId,
     ownerUserId: params.ownerUserId,
     message: params.message,
+    SourceServerId: MJGlobal.Instance.ProcessUUID,
   };
   pubSub.publish(PUSH_STATUS_UPDATES_TOPIC, payload);
+
+  // Fan out to other instances. Only reached by updates originating HERE: a message arriving from
+  // the bus is republished straight onto the local topic and never comes back through this
+  // function, so there is no loop to break beyond the SourceServerId check on the receiving side.
+  if (_publishHook && ShouldReplicateStatusUpdate(params.message)) {
+    try {
+      _publishHook(payload);
+    } catch {
+      // Fan-out is an optimization over the durable tail query. A message bus that is down must
+      // never break local delivery, which is the path that works for the common case.
+    }
+  }
 }
 
 /** @deprecated Use {@link PublishStatusUpdate}. */

package/src/generic/ResolverBase.ts

@@ -4,6 +4,7 @@ import {
   BaseEntityEvent,
   CompositeKey,
   DatabaseProviderBase,
+  EntityFieldInfo,
   EntityFieldTSType,
   EntityInfo,
   EntityPermissionType,
@@ -1692,7 +1693,7 @@ export class ResolverBase {
             await this.TestAndSetClientOldValuesToDBValues(input, clientNewValues, entityObject, userInfo);
           } else {
             // no OldValues, so we can just set the new values from input
-            entityObject.SetMany(input);
+            entityObject.SetMany(clientNewValues);
           }
         } else {
           // Use a generic message to avoid leaking whether a record exists — distinguishing
@@ -1704,8 +1705,11 @@ export class ResolverBase {
       } else {
         // we get here if we are NOT tracking changes and we DO have OldValues, so we can load from them
         const oldValues = {};
-        // for each item in the oldValues array, add it to the oldValues object
-        input.OldValues___?.forEach((item) => (oldValues[item.Key] = item.Value));
+        // for each item in the oldValues array, add it to the oldValues object, typed like the field
+        input.OldValues___?.forEach((item) => {
+          const field = entityObject.EntityInfo.Fields.find((f) => f.CodeName === item.Key);
+          oldValues[item.Key] = field ? this.ClientOldValueToFieldValue(field, item.Value) : item.Value;
+        });
 
         // 1) load the old values, this will be the initial state of the object
         await entityObject.LoadFromData(oldValues);
@@ -1772,6 +1776,11 @@ export class ResolverBase {
    *
    * Ordered so the boolean flag is evaluated last: the extra load lands only on entities that have
    * the feature switched on, which is almost none of them.
+   *
+   * `MJ: Record Changes` always loads from the database: its server class allows an update only when
+   * Comments is the one dirty field, and "dirty" compared against client-supplied OldValues lets a
+   * caller pin forged audit columns as both old and new values. It doesn't track its own changes, so
+   * nothing else forces the load.
    */
   protected MustLoadTruthFromDatabase(
     entityInfo: EntityInfo,
@@ -1781,6 +1790,7 @@ export class ResolverBase {
   ): boolean {
     return (
       entityInfo.TrackRecordChanges ||
+      entityInfo.Name.trim().toLowerCase() === 'mj: record changes' ||
       !input.OldValues___ ||
       hasDeniedReadFields ||
       hasNarrowedAuditPayload ||
@@ -1917,6 +1927,41 @@ export class ResolverBase {
     return true;
   }
 
+  /**
+   * Converts one client-sent old value (always a string on the wire; dates as epoch milliseconds,
+   * the GraphQL Timestamp form) to the field's TypeScript type, so it compares equal to the stored
+   * value. Both the OldValues comparison and the load-from-OldValues path use it: without it a date
+   * old value becomes an Invalid Date and an unchanged date field reads as edited.
+   */
+  protected ClientOldValueToFieldValue(field: EntityFieldInfo | undefined, raw: unknown): string | number | boolean | Date | null {
+    let val: unknown = raw;
+    if ((val === null || val === undefined) && field && field.DefaultValue !== null && field.DefaultValue !== undefined && !field.AllowsNull)
+      val = field.DefaultValue; // set default value as the field was never set and it does NOT allow nulls
+    if (val === undefined) val = null;
+    // A null old value stays null, except a boolean, which the comparison has always read as false.
+    if (field?.TSType === EntityFieldTSType.Boolean && val === null) return false;
+    if (val === null) return null;
+    if (!field) {
+      // No field metadata to convert by: pass scalars through, stringify anything else.
+      return typeof val === 'string' || typeof val === 'number' || typeof val === 'boolean' || val instanceof Date ? val : String(val);
+    }
+
+    const text = String(val);
+    switch (field.TSType) {
+      case EntityFieldTSType.Number: {
+        const integer = ['int', 'smallint', 'bigint', 'tinyint'].includes((field.Type as string).toLowerCase());
+        return integer ? parseInt(text) : parseFloat(text);
+      }
+      case EntityFieldTSType.Boolean:
+        return !(text === 'false' || text === '0' || parseInt(text) === 0);
+      case EntityFieldTSType.Date:
+        // Epoch milliseconds (the GraphQL Timestamp form) arrive as a numeric string.
+        return new Date(text.trim() !== '' && !isNaN(Number(text)) ? parseInt(text) : text);
+      default:
+        return text; // already a string
+    }
+  }
+
   /**
    * This routine compares the OldValues property in the input object to the values in the DB that we just loaded. If there are differences, we need to check to see if the client
    * is trying to update any of those fields (e.g. overlap). If there is overlap, we throw an error. If there is no overlap, we can proceed with the update even if the DB Values
@@ -1931,52 +1976,7 @@ export class ResolverBase {
     input.OldValues___.forEach((item) => {
       // we need to do a quick transform on the values to make sure they match the TS Type for the given field because item.Value will always be a string
       const field = entityObject.EntityInfo.Fields.find((f) => f.CodeName === item.Key);
-      let val = item.Value;
-      if ((val === null || val === undefined) && field.DefaultValue !== null && field.DefaultValue !== undefined && !field.AllowsNull)
-        val = field.DefaultValue; // set default value as the field was never set and it does NOT allow nulls
-
-      if (field) {
-        switch (field.TSType) {
-          case EntityFieldTSType.Number:
-            if (val == null && val == undefined) {
-              val = null;
-            }
-            else {
-              let typeLowered = (field.Type as string).toLowerCase();
-
-              switch (typeLowered) {
-                case 'int':
-                case 'smallint':
-                case 'bigint':
-                case 'tinyint':
-                  val = parseInt(val);
-                  break;
-                case 'money':
-                case 'smallmoney':
-                case 'decimal':
-                case 'numeric':
-                case 'float':
-                  val = parseFloat(val);
-                  break;
-                default:
-                  val = parseFloat(val);
-                  break;
-              }
-            }
-            break;
-          case EntityFieldTSType.Boolean:
-            val = val === null || val === undefined || val === 'false' || val === '0' || parseInt(val) === 0 ? false : true;
-            break;
-          case EntityFieldTSType.Date:
-            // first, if val is a string and it is actually a number (milliseconds since epoch), convert it to a number.
-            if (val !== null && val !== undefined && val.toString().trim() !== '' && !isNaN(val)) val = parseInt(val);
-
-            val = val !== null && val !== undefined ? new Date(val) : null;
-            break;
-          default:
-            break; // already a string
-        }
-      }
+      const val = this.ClientOldValueToFieldValue(field, item.Value);
       clientOldValues[item.Key] = val;
     });