@memberjunction/server 6.1.0-edge.2 → 6.1.0-edge.3

package/src/rest/UploadTokenManager.ts

@@ -0,0 +1,211 @@
+/**
+ * @fileoverview Ephemeral in-memory staged upload token manager.
+ *
+ * Provides a secure, single-use, time-to-live (TTL) bounded staging area for raw
+ * binary upload buffers received via REST streaming before being committed through
+ * the GraphQL `UploadStorageFile` pipeline.
+ *
+ * Security properties:
+ * - Single-use nonce: Token and buffer are atomically destroyed on first consumption.
+ * - User binding: Tokens are bound to the authenticated User ID that uploaded them.
+ * - Auto-eviction: Abandoned tokens are automatically evicted from RAM after the TTL.
+ * - Memory caps: Enforces per-file max size (default 100MB) and total pool capacity (default 500MB).
+ *
+ * @module @memberjunction/server/rest/UploadTokenManager
+ */
+
+import { BaseSingleton, UUIDsEqual } from '@memberjunction/global';
+import { LogError } from '@memberjunction/core';
+import { randomBytes } from 'node:crypto';
+
+/**
+ * An ephemeral in-memory staged upload record.
+ */
+export interface StagedUploadEntry {
+  token: string;
+  buffer: Buffer;
+  fileName: string;
+  mimeType: string;
+  contentLength: number;
+  userId: string;
+  createdAt: number;
+  expiresAt: number;
+  timer: NodeJS.Timeout;
+}
+
+/**
+ * Ephemeral In-Memory Staged Upload Token Manager.
+ *
+ * Architecture & Deployment Topology Note:
+ * This implementation maintains staged buffers in-process RAM with a 5-minute sliding TTL.
+ * In a clustered/multi-replica deployment behind a load balancer, sticky sessions (or routing
+ * the subsequent GraphQL `UploadStorageFile` commit to the same node) are recommended.
+ * For shared-memory multi-node clustering, an external ephemeral store (such as Redis)
+ * can be plugged in behind this interface.
+ */
+export class UploadTokenManager extends BaseSingleton<UploadTokenManager> {
+  private readonly _staged = new Map<string, StagedUploadEntry>();
+  private _totalMemoryBytes = 0;
+
+  public defaultTtlSeconds = 300; // 5 minutes
+  public maxFileSizeBytes = 100 * 1024 * 1024; // 100 MB
+  public maxUserMemoryBytes = 150 * 1024 * 1024; // 150 MB per user
+  public maxPoolMemoryBytes = 500 * 1024 * 1024; // 500 MB
+
+  // Public constructor required by BaseSingleton
+  public constructor() {
+    super();
+  }
+
+  public static get Instance(): UploadTokenManager {
+    return UploadTokenManager.getInstance<UploadTokenManager>();
+  }
+
+  /**
+   * Calculates total active staged bytes currently held for a given user.
+   */
+  public GetUserMemoryBytes(userId: string): number {
+    let total = 0;
+    for (const entry of this._staged.values()) {
+      if (UUIDsEqual(entry.userId, userId)) {
+        total += entry.contentLength;
+      }
+    }
+    return total;
+  }
+
+  /**
+   * Stages a raw binary buffer in RAM, returning a single-use cryptographic token.
+   *
+   * @throws Error if buffer exceeds single file limit, per-user quota, or global pool memory capacity.
+   */
+  public Stage(params: {
+    buffer: Buffer;
+    fileName: string;
+    mimeType: string;
+    userId: string;
+    ttlSeconds?: number;
+  }): string {
+    const size = params.buffer.length;
+
+    if (size > this.maxFileSizeBytes) {
+      throw new Error(
+        `Staged upload exceeds maximum allowed file size of ${Math.round(this.maxFileSizeBytes / (1024 * 1024))}MB (received ${Math.round(size / (1024 * 1024))}MB).`
+      );
+    }
+
+    const userCurrentBytes = this.GetUserMemoryBytes(params.userId);
+    if (userCurrentBytes + size > this.maxUserMemoryBytes) {
+      throw new Error(
+        `Per-user upload memory quota reached (${Math.round(userCurrentBytes / (1024 * 1024))}MB / ${Math.round(this.maxUserMemoryBytes / (1024 * 1024))}MB used). Please wait for active uploads to finalize.`
+      );
+    }
+
+    if (this._totalMemoryBytes + size > this.maxPoolMemoryBytes) {
+      throw new Error(
+        `Staged upload memory capacity reached (${Math.round(this._totalMemoryBytes / (1024 * 1024))}MB / ${Math.round(this.maxPoolMemoryBytes / (1024 * 1024))}MB used). Please try again shortly.`
+      );
+    }
+
+    const token = 'upt_' + randomBytes(32).toString('hex');
+    const ttl = params.ttlSeconds || this.defaultTtlSeconds;
+    const now = Date.now();
+    const expiresAt = now + ttl * 1000;
+
+    const timer = setTimeout(() => {
+      this.Evict(token);
+    }, ttl * 1000);
+
+    // Unref timer so it doesn't hold open process shutdown in tests or CLI
+    if (typeof timer.unref === 'function') {
+      timer.unref();
+    }
+
+    const entry: StagedUploadEntry = {
+      token,
+      buffer: params.buffer,
+      fileName: params.fileName,
+      mimeType: params.mimeType,
+      contentLength: size,
+      userId: params.userId,
+      createdAt: now,
+      expiresAt,
+      timer,
+    };
+
+    this._staged.set(token, entry);
+    this._totalMemoryBytes += size;
+
+    return token;
+  }
+
+  /**
+   * Atomically consumes and evicts a staged upload token.
+   *
+   * @param token The staged upload token.
+   * @param claimingUserId The user ID claiming this token. If provided, must match the user who uploaded it.
+   * @returns The staged buffer and metadata, or null if the token is invalid, expired, or claimed by the wrong user.
+   */
+  public Consume(
+    token: string,
+    claimingUserId?: string
+  ): { buffer: Buffer; fileName: string; mimeType: string; contentLength: number } | null {
+    if (!token || !this._staged.has(token)) {
+      return null;
+    }
+
+    const entry = this._staged.get(token)!;
+
+    // Security check: verify user ownership using UUIDsEqual for robust ID normalization
+    if (claimingUserId && entry.userId && !UUIDsEqual(entry.userId, claimingUserId)) {
+      LogError(`[UploadTokenManager] Security violation: User '${claimingUserId}' attempted to claim upload token owned by User '${entry.userId}'.`);
+      // Evict immediately to prevent further probing
+      this.Evict(token);
+      return null;
+    }
+
+    // Atomic consumption & memory reclamation
+    this.Evict(token);
+
+    return {
+      buffer: entry.buffer,
+      fileName: entry.fileName,
+      mimeType: entry.mimeType,
+      contentLength: entry.contentLength,
+    };
+  }
+
+  /**
+   * Evicts a token from RAM and reclaims memory.
+   */
+  public Evict(token: string): boolean {
+    const entry = this._staged.get(token);
+    if (!entry) {
+      return false;
+    }
+
+    clearTimeout(entry.timer);
+    this._staged.delete(token);
+    this._totalMemoryBytes = Math.max(0, this._totalMemoryBytes - entry.contentLength);
+    return true;
+  }
+
+  /** Total RAM in bytes currently consumed by all active staged uploads. */
+  public get TotalMemoryBytes(): number {
+    return this._totalMemoryBytes;
+  }
+
+  /** Count of currently active staged uploads. */
+  public get ActiveCount(): number {
+    return this._staged.size;
+  }
+
+  /** Clears all active staged uploads. */
+  public Clear(): void {
+    for (const [, entry] of this._staged) {
+      clearTimeout(entry.timer);
+    }
+    this._staged.clear();
+    this._totalMemoryBytes = 0;
+  }
+}

package/src/services/IntegrationSyncWorkerService.ts

@@ -0,0 +1,141 @@
+/**
+ * @fileoverview Worker-mode poll loop for durable integration sync runs (PR 1 item 8).
+ * @module MJServer/services
+ */
+
+import { LogError, LogStatus, UserInfo } from '@memberjunction/core';
+import { UserCache } from '@memberjunction/generic-database-provider';
+import { IntegrationEngine } from '@memberjunction/integration-engine';
+import { IntegrationSyncWorkerConfig } from '../config.js';
+
+/**
+ * Polls `CompanyIntegrationRun` for `Status='Queued'` rows, claims each one atomically,
+ * executes it, and releases — the enqueue/execute split that lets a sync outlive the
+ * request that asked for it.
+ *
+ * Multiple workers (in this process or others) may poll the same queue: the claim sproc
+ * admits exactly one, and every loser simply moves on to the next candidate. Nothing here
+ * assumes it is the only worker.
+ */
+export class IntegrationSyncWorkerService {
+    private config: IntegrationSyncWorkerConfig;
+    private systemUser: UserInfo | null = null;
+    private timer: NodeJS.Timeout | null = null;
+    private isRunning = false;
+    private polling = false;
+    /** Run IDs currently executing in THIS process — bounds concurrency and prevents self-collision. */
+    private inFlight = new Set<string>();
+
+    constructor(config: IntegrationSyncWorkerConfig) {
+        this.config = config;
+    }
+
+    public get IsEnabled(): boolean {
+        return this.config.enabled;
+    }
+
+    public get IsRunning(): boolean {
+        return this.isRunning;
+    }
+
+    /** Number of queued runs this worker is currently executing. */
+    public get InFlightCount(): number {
+        return this.inFlight.size;
+    }
+
+    /** Resolve the system user the worker executes syncs as. */
+    public async Initialize(): Promise<void> {
+        if (!this.config.enabled) return;
+
+        const email = this.config.systemUserEmail;
+        this.systemUser = UserCache.Users.find(u => u.Email?.toLowerCase() === email.toLowerCase()) ?? null;
+        if (!this.systemUser) {
+            throw new Error(`[IntegrationSyncWorker] System user not found with email: ${email}`);
+        }
+        await IntegrationEngine.Instance.Config(false, this.systemUser);
+    }
+
+    /** Begin polling the queue. */
+    public Start(): void {
+        if (!this.config.enabled || this.isRunning) return;
+        if (!this.systemUser) {
+            throw new Error('[IntegrationSyncWorker] Not initialized — call Initialize() first');
+        }
+
+        this.isRunning = true;
+        this.timer = setInterval(() => void this.pollOnce(), this.config.pollingIntervalMs);
+        // Standard level, NOT verboseOnly: this states a deployment-shaping fact — that this process
+        // executes queued syncs — and it's one line per process lifetime. Stop() already logs at
+        // standard level, so hiding Start() produced logs showing a worker stopping that never
+        // started, and no way to confirm from logs that a deployment is in worker mode at all.
+        LogStatus(
+            `🔄 Integration Sync Worker: polling every ${Math.round(this.config.pollingIntervalMs / 1000)}s, ` +
+            `up to ${this.config.maxConcurrentRuns} concurrent run(s)`
+        );
+        // Don't wait a full interval to drain a queue that already has work.
+        void this.pollOnce();
+    }
+
+    /**
+     * Stop polling. In-flight runs are left to finish on their own — their leases are
+     * renewed by the engine's heartbeat, so killing them here would only strand rows.
+     */
+    public Stop(): void {
+        if (!this.isRunning) return;
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+        this.isRunning = false;
+        LogStatus(`[IntegrationSyncWorker] Stopped polling (${this.inFlight.size} run(s) still in flight)`);
+    }
+
+    /**
+     * One poll pass: fetch claimable candidates up to the free concurrency slots and start
+     * each. Overlapping passes are suppressed so a slow query can't stack up timers.
+     */
+    private async pollOnce(): Promise<void> {
+        if (this.polling || !this.isRunning || !this.systemUser) return;
+
+        const slots = this.config.maxConcurrentRuns - this.inFlight.size;
+        if (slots <= 0) return;
+
+        this.polling = true;
+        try {
+            const candidates = await IntegrationEngine.PollQueuedRuns(this.systemUser, slots);
+            for (const candidate of candidates) {
+                if (this.inFlight.has(candidate.ID)) continue;
+                this.startRun(candidate.ID);
+            }
+        } catch (err) {
+            LogError(`[IntegrationSyncWorker] Poll failed`, undefined, err);
+        } finally {
+            this.polling = false;
+        }
+    }
+
+    /**
+     * Execute one queued run in the background. Losing the claim race is normal — the run
+     * is simply someone else's — so it is logged as information, never as an error.
+     */
+    private startRun(runID: string): void {
+        if (!this.systemUser) return;
+
+        this.inFlight.add(runID);
+        void IntegrationEngine.Instance
+            .ExecuteQueuedRun(runID, this.systemUser)
+            .then(result => {
+                if (result.Success) {
+                    LogStatus(`[IntegrationSyncWorker] Run ${runID} completed: ${result.RecordsProcessed} record(s) processed`);
+                } else {
+                    LogStatus(`[IntegrationSyncWorker] Run ${runID} did not complete: ${result.ErrorMessage ?? 'unknown reason'}`);
+                }
+            })
+            .catch(err => {
+                LogError(`[IntegrationSyncWorker] Run ${runID} threw`, undefined, err);
+            })
+            .finally(() => {
+                this.inFlight.delete(runID);
+            });
+    }
+}

package/src/services/StartTaskGraphDispatcher.ts

@@ -24,6 +24,7 @@ import { DurableEntityActionRegistry } from '@memberjunction/actions-base';
 import { CreateTaskGraphProviderFactory } from './TaskGraphProviderFactory.js';
 import { TaskGraphActionRunner } from './TaskGraphActionRunner.js';
 import { TaskGraphAgentRunner } from './TaskGraphAgentRunner.js';
+import { randomBytes } from 'node:crypto';
 import { DurableEntityActionTaskSubmitter } from './DurableEntityActionTaskSubmitter.js';
 import { TaskGraphContinuationDeliverer } from './TaskGraphContinuationDeliverer.js';
 import { PubSubManager } from '../generic/PubSubManager.js';
@@ -39,6 +40,19 @@ import { TaskGraphFrameBroadcaster } from '../resolvers/TaskGraphFrameResolver.j
  *
  * The dispatcher self-registers with `ShutdownRegistry`, so the caller does not wire teardown.
  */
+/**
+ * A per-process identity for the claim protocol.
+ *
+ * Host and pid are kept because they are what a human reads in a log; the random suffix is what
+ * makes the value actually unique. Generated once per call, so a restart is a new instance — which
+ * is correct: the previous process's claims should expire rather than be inherited.
+ */
+function defaultInstanceID(): string {
+    const host = process.env.HOSTNAME ?? 'mjapi';
+    const entropy = randomBytes(4).toString('hex');
+    return `${host}-${process.pid}-${entropy}`;
+}
+
 export async function StartTaskGraphDispatcher(
     pool: sql.ConnectionPool,
     contextUser: UserInfo,
@@ -52,7 +66,13 @@ export async function StartTaskGraphDispatcher(
         providerFactory,
         new TaskGraphAgentRunner(),
         contextUser,
-        { InstanceID: instanceID ?? `${process.env.HOSTNAME ?? 'mjapi'}-${process.pid}` },
+        // A RANDOM SUFFIX, because host+pid is not unique in the deployments this runs in (C2).
+        // `HOSTNAME` is not exported to child processes under systemd or pm2, and a containerised
+        // process is routinely pid 1 — so two hosts default to the same `mjapi-1`. A shared instance
+        // id defeats every `ClaimedBy=@me` guard at once: A's heartbeat renews B's lease, and A's
+        // stale terminal write lands over B's live execution. An explicit `instanceID` still wins,
+        // for hosts that have a real identity to supply.
+        { InstanceID: instanceID ?? defaultInstanceID() },
         // Without this the dispatcher had nowhere to deliver: a finished graph logged its outcome,
         // marked itself delivered, and said nothing to the conversation that asked for it.
         new TaskGraphContinuationDeliverer(providerFactory, contextUser),

package/src/services/TaskGraphAgentRunner.ts

@@ -16,6 +16,7 @@ import { AgentRunner } from '@memberjunction/ai-agents';
 import { ChatMessageRole } from '@memberjunction/ai';
 import { LogError } from '@memberjunction/core';
 import { MJAIAgentEntityExtended, MJAIAgentRunEntityExtended } from '@memberjunction/ai-core-plus';
+import type { AgentExecutionProgressCallback } from '@memberjunction/ai-core-plus';
 import type { TaskAgentRunner, TaskAgentRunParams, TaskAgentRunResult } from '@memberjunction/task-graph';
 
 export class TaskGraphAgentRunner implements TaskAgentRunner {
@@ -34,6 +35,10 @@ export class TaskGraphAgentRunner implements TaskAgentRunner {
                 // rather than starting the chain over. Without it MAX_REINVOKE_DEPTH never fires.
                 continuationDepth: params.ContinuationDepth ?? 0,
                 parentRun: await this.loadSubmittingRun(params),
+                // The bridge from the agent's own progress stream up to graph-level observers: the
+                // dispatcher turns these into rate-limited `NodeProgress` frames, which is how a
+                // running Agent step shows "what it is doing right now" on the workflow canvas.
+                onProgress: this.bridgeProgress(params),
             });
 
             const success = result?.success === true;
@@ -50,6 +55,24 @@ export class TaskGraphAgentRunner implements TaskAgentRunner {
         }
     }
 
+    /**
+     * Adapts the agent framework's progress callback onto the dispatcher's plain sink.
+     *
+     * Undefined when nobody is listening, so the agent run pays nothing for observability it is not
+     * providing. Never throws — a progress announcement must not be able to fail the step.
+     */
+    private bridgeProgress(params: TaskAgentRunParams): AgentExecutionProgressCallback | undefined {
+        const sink = params.OnProgress;
+        if (!sink) return undefined;
+        return (progress) => {
+            try {
+                sink(progress.message, progress.percentage);
+            } catch {
+                // The dispatcher's sink already guards itself; this is belt over braces.
+            }
+        };
+    }
+
     /**
      * The run that submitted this graph, as the entity `RunAgent` wants for `parentRun`.
      *

package/src/services/TaskGraphContinuationDeliverer.ts

@@ -135,7 +135,17 @@ export class TaskGraphContinuationDeliverer implements TaskContinuationDeliverer
                 continuationDepth: params.ReinvokeDepth + 1,
             });
         } catch (e) {
-            LogError(`[TaskGraphContinuationDeliverer] Reinvoking for "${params.WorkflowName}" threw`, undefined, e);
+            // FALL BACK, don't just log (C3). Both load-failure paths above already post instead of
+            // reinvoking, on the principle that a completed workflow's outcome must reach the user
+            // somehow — but a throw out of `RunAgent` skipped that and lost the outcome entirely.
+            // The marker has been claimed by the time this runs, so nothing will look at the graph
+            // again: the work finished, and the only record of it is a server log line.
+            LogError(`[TaskGraphContinuationDeliverer] Reinvoking for "${params.WorkflowName}" threw — posting instead`, undefined, e);
+            try {
+                await this.PostMessage(params);
+            } catch (fallbackError) {
+                LogError(`[TaskGraphContinuationDeliverer] The fallback post for "${params.WorkflowName}" also failed`, undefined, fallbackError);
+            }
         }
     }
 

package/src/types.ts

@@ -107,6 +107,12 @@ export type RunViewGenericParams = {
    * happens. Propagated to `RunViewParams.BypassCache`.
    */
   bypassCache?: boolean;
+  /**
+   * Optional source-of-truth selector for entities that have a base-view materialization.
+   * 'Materialized' routes the read to the entity's materialized wrapper view; defaults to 'Live'.
+   * Propagated to `RunViewParams.DataSource`.
+   */
+  dataSource?: 'Live' | 'Materialized';
 };