@memberjunction/schema-engine 6.1.0-edge.2 → 6.1.0-edge.4

package/dist/RuntimeSchemaManager.js

@@ -9,7 +9,7 @@
  * In-memory concurrency mutex (one operation at a time).
  */
 import { BaseSingleton } from '@memberjunction/global';
-import { LogError, Metadata } from '@memberjunction/core';
+import { LogError, Metadata, RunView, } from '@memberjunction/core';
 import { Octokit } from '@octokit/rest';
 import { RSUMetrics } from './RSUMetrics.js';
 import { GetDialect } from './DDLGenerator.js';
@@ -80,9 +80,6 @@ class RSUConfig {
     get IsAuditLogEnabled() {
         return process.env.RSU_AUDIT_LOG_ENABLED !== '0';
     }
-    get PendingWorkPath() {
-        return process.env.RSU_PENDING_WORK_PATH || '.rsu_pending';
-    }
     get ProtectedSchemas() {
         const envSchemas = process.env.RSU_PROTECTED_SCHEMAS;
         return envSchemas ? envSchemas.split(',').map((s) => s.trim()) : [];
@@ -145,9 +142,20 @@ export class RuntimeSchemaManager extends BaseSingleton {
         this._outOfSyncSince = null;
         this._lastRunAt = null;
         this._lastRunResult = null;
+        /** Names of deprecated members already warned about, so a hot path logs once, not per call. */
+        this.warnedDeprecations = new Set();
         this._currentStepName = null;
         this._currentStepIndex = null;
         this._stepTotal = null;
+        // ── Pipeline observer ──────────────────────────────────────────────
+        /**
+         * Optional observer notified of every step and run boundary. Set once at process startup (see
+         * the RSU progress bridge in MJServer) — a single observer is sufficient because RSU runs are
+         * serialized by the pipeline lock, so events can never interleave between runs.
+         *
+         * Never awaited, never allowed to fail the pipeline. See {@link RSUPipelineObserver}.
+         */
+        this.PipelineObserver = null;
         /** Waiters queued behind the current lock holder. */
         this._lockWaiters = [];
         this._dbLockId = null;
@@ -202,41 +210,164 @@ export class RuntimeSchemaManager extends BaseSingleton {
         return this._additionalSchemaInfoPath ?? rsuConfig.AdditionalSchemaInfoPath;
     }
     // ─── Pending Work (post-restart tasks) ─────────────────────────
-    /** Write pending work to disk so it can be processed after restart. */
-    async WritePendingWork(data) {
-        const { writeFileSync, mkdirSync } = await import('node:fs');
-        const { join } = await import('node:path');
-        const dir = join(rsuConfig.WorkDir, rsuConfig.PendingWorkPath);
-        mkdirSync(dir, { recursive: true });
-        const filePath = join(dir, `${Date.now()}.json`);
-        writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf-8');
-        this.rsuLog(`Wrote pending work to ${filePath}`);
-        return filePath;
+    /**
+     * Register pending work durably in `MJ: RSU Pending Works` so it survives the
+     * post-migration restart. Returns the new row's ID.
+     *
+     * Unlike the `.rsu_pending` files this replaces, the row is NOT consumed on read —
+     * it stays Pending until the consumer explicitly calls {@link CompletePendingWork}
+     * or {@link FailPendingWork}, so a crash mid-consumption leaves visible, resumable work.
+     */
+    async WritePendingWork(data, contextUser, provider) {
+        // `contextUser` is optional only to keep the published 1-arg signature compiling
+        // (see the deprecation note above the class). A durable row cannot be written
+        // without a user, and silently pretending to have persisted the work would defeat
+        // the entire point of this queue — so the 1-arg form fails loudly instead.
+        if (!contextUser) {
+            throw new Error('RuntimeSchemaManager.WritePendingWork now persists to `MJ: RSU Pending Works` and requires a contextUser. ' +
+                'Call WritePendingWork(data, contextUser) — the previous 1-argument form wrote a .rsu_pending file and no longer exists.');
+        }
+        const md = provider ?? new Metadata();
+        const row = await md.GetEntityObject('MJ: RSU Pending Works', contextUser);
+        row.NewRecord();
+        row.CompanyIntegrationID = data.CompanyIntegrationID;
+        row.PayloadJSON = JSON.stringify(data);
+        row.Status = 'Pending';
+        if (!(await row.Save())) {
+            throw new Error(`Failed to register RSU pending work: ${row.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+        }
+        this.rsuLog(`Registered pending work ${row.ID} for company integration ${data.CompanyIntegrationID}`);
+        return row.ID;
     }
-    /** Read all pending work files, return them, and delete them. */
-    async ReadAndClearPendingWork() {
-        const { readdirSync, readFileSync, unlinkSync, existsSync } = await import('node:fs');
-        const { join } = await import('node:path');
-        const dir = join(rsuConfig.WorkDir, rsuConfig.PendingWorkPath);
-        if (!existsSync(dir))
+    /**
+     * Read every still-Pending work row. Never deletes or mutates — the caller marks each
+     * row Completed/Failed once it has actually finished processing it. Rows whose payload
+     * fails to parse are reported as Failed so they stop being retried forever.
+     *
+     * @param staleAfterMinutes when set, rows created more than this many minutes ago are
+     *        logged as stale so an operator can see work that a previous process abandoned.
+     */
+    async ReadPendingWork(contextUser, staleAfterMinutes, provider) {
+        // Read through the SAME provider the caller writes through. Defaulting to the global
+        // provider here would read a different connection's rows than WritePendingWork just
+        // wrote, which is the whole reason the provider is threaded through these methods.
+        const rv = new RunView(provider);
+        const result = await rv.RunView({
+            EntityName: 'MJ: RSU Pending Works',
+            ExtraFilter: `Status = 'Pending'`,
+            OrderBy: '__mj_CreatedAt ASC',
+            ResultType: 'entity_object',
+            BypassCache: true,
+        }, contextUser);
+        if (!result.Success) {
+            LogError(`[RSU] Failed to read pending work: ${result.ErrorMessage}`);
             return [];
-        const files = readdirSync(dir).filter((f) => f.endsWith('.json'));
-        const results = [];
-        for (const file of files) {
-            const filePath = join(dir, file);
+        }
+        const items = [];
+        for (const row of result.Results) {
+            const createdAt = row.__mj_CreatedAt ?? new Date();
             try {
-                const data = JSON.parse(readFileSync(filePath, 'utf-8'));
-                results.push(data);
-                unlinkSync(filePath);
+                items.push({ ID: row.ID, Work: JSON.parse(row.PayloadJSON), CreatedAt: createdAt });
             }
-            catch {
-                /* skip corrupt files */
+            catch (err) {
+                await this.FailPendingWork(row.ID, `Corrupt PayloadJSON: ${err instanceof Error ? err.message : String(err)}`, contextUser, provider);
             }
         }
-        if (results.length > 0) {
-            this.rsuLog(`Found ${results.length} pending work item(s)`);
+        if (staleAfterMinutes !== undefined) {
+            const cutoff = Date.now() - staleAfterMinutes * 60_000;
+            const stale = items.filter((i) => i.CreatedAt.getTime() < cutoff);
+            if (stale.length > 0) {
+                LogError(`[RSU] ${stale.length} pending work item(s) older than ${staleAfterMinutes} minute(s) are still unprocessed: ${stale.map((s) => s.ID).join(', ')}`);
+            }
         }
-        return results;
+        if (items.length > 0)
+            this.rsuLog(`Found ${items.length} pending work item(s)`);
+        return items;
+    }
+    /**
+     * @deprecated Use {@link ReadPendingWork} instead, and mark each item terminal with
+     * {@link CompletePendingWork} / {@link FailPendingWork} once you have processed it.
+     *
+     * Retained with its original signature so consumers taking this minor upgrade still
+     * compile. It is NO LONGER FUNCTIONAL: pending work lives in `MJ: RSU Pending Works`
+     * rather than in `.rsu_pending` files, and reading it requires a context user this
+     * signature has no way to supply. Returns an empty array — the value that previously
+     * meant "no pending work" — rather than pretending to have drained a queue.
+     */
+    async ReadAndClearPendingWork() {
+        this.warnDeprecatedOnce('ReadAndClearPendingWork', 'RuntimeSchemaManager.ReadAndClearPendingWork() is deprecated and no longer functional. ' +
+            'Pending work is now durable in `MJ: RSU Pending Works`; call ReadPendingWork(contextUser) and then ' +
+            'CompletePendingWork/FailPendingWork per item. Returning an empty array.');
+        return [];
+    }
+    warnDeprecatedOnce(member, message) {
+        if (this.warnedDeprecations.has(member))
+            return;
+        this.warnedDeprecations.add(member);
+        LogError(`[RSU] ${message}`);
+    }
+    /** Mark a pending work row Completed. Call ONLY after the work actually succeeded. */
+    async CompletePendingWork(id, contextUser, provider) {
+        return this.setPendingWorkTerminalStatus(id, 'Completed', undefined, contextUser, provider);
+    }
+    /** Mark a pending work row Failed, recording why. */
+    async FailPendingWork(id, errorMessage, contextUser, provider) {
+        return this.setPendingWorkTerminalStatus(id, 'Failed', errorMessage, contextUser, provider);
+    }
+    async setPendingWorkTerminalStatus(id, status, errorMessage, contextUser, provider) {
+        const md = provider ?? new Metadata();
+        const row = await md.GetEntityObject('MJ: RSU Pending Works', contextUser);
+        if (!(await row.Load(id))) {
+            LogError(`[RSU] Pending work ${id} not found when marking ${status}`);
+            return false;
+        }
+        row.Status = status;
+        row.ErrorMessage = errorMessage ?? null;
+        row.ProcessedAt = new Date();
+        if (!(await row.Save())) {
+            LogError(`[RSU] Failed to mark pending work ${id} as ${status}: ${row.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Register every input's PendingWork rows. Called for successful migrations only,
+     * before the restart. Returns row IDs keyed by the input's index within `inputs`.
+     *
+     * Registration failures are collected, NOT swallowed. This step is the durability
+     * boundary: once the restart fires, work that never became a row is gone, and the
+     * post-restart consumer has nothing to find. A run whose rows failed to register
+     * therefore reports the failure on its result instead of appearing to have succeeded.
+     */
+    async registerPendingWork(inputs) {
+        const registered = new Map();
+        const errors = new Map();
+        const addError = (input, message) => {
+            LogError(`[RSU] ${message}`);
+            const list = errors.get(input) ?? [];
+            list.push(message);
+            errors.set(input, list);
+        };
+        for (const input of inputs) {
+            if (!input.PendingWork?.length)
+                continue;
+            if (!input.ContextUser) {
+                addError(input, `PendingWork supplied without ContextUser for "${input.Description}" — nothing was registered, so this work will NOT run after the restart`);
+                continue;
+            }
+            const ids = [];
+            for (const work of input.PendingWork) {
+                try {
+                    ids.push(await this.WritePendingWork(work, input.ContextUser));
+                }
+                catch (err) {
+                    addError(input, `Failed to register post-restart work for company integration ${work.CompanyIntegrationID}: ` +
+                        `${err instanceof Error ? err.message : String(err)} — this work will NOT run after the restart`);
+                }
+            }
+            registered.set(input, ids);
+        }
+        return { IDs: registered, Errors: errors };
     }
     // ─── Generic Post-Restart File Injection ──────────────────────
     /**
@@ -322,6 +453,70 @@ export class RuntimeSchemaManager extends BaseSingleton {
         this._currentStepIndex = null;
         this._stepTotal = null;
     }
+    /** Delivers an event to {@link PipelineObserver}, swallowing (but logging) any throw. */
+    notifyObserver(event) {
+        const observer = this.PipelineObserver;
+        if (!observer)
+            return;
+        try {
+            observer(event);
+        }
+        catch (error) {
+            // Progress reporting must never break a schema migration.
+            this.rsuLog(`Pipeline observer threw on ${event.Kind} (ignored): ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Records a completed step onto `steps` AND publishes it to the observer, so a step is never
+     * visible in the result but absent from the event stream. Every step recording goes through
+     * here — including the ones computed inline rather than via {@link runStep}.
+     */
+    recordStep(steps, step) {
+        steps.push(step);
+        this.notifyObserver({
+            Kind: 'step.end',
+            Name: step.Name,
+            Status: step.Status,
+            DurationMs: step.DurationMs,
+            Message: step.Message,
+            StepIndex: step.StepIndex,
+            StepTotal: step.StepTotal,
+        });
+    }
+    /**
+     * Maps a finished batch onto its terminal `run.end` event.
+     *
+     * `result` is undefined when a throw escaped the pipeline before a result existed — that case
+     * must still produce a FAILED run.end, otherwise an observer's run would hang in-flight forever.
+     *
+     * Public + static so the mapping is unit-testable without a live pipeline.
+     */
+    static BuildRunEndEvent(result, totalCount) {
+        if (!result) {
+            return {
+                Kind: 'run.end',
+                Success: false,
+                SuccessCount: 0,
+                FailureCount: totalCount,
+                TotalCount: totalCount,
+                ErrorMessage: 'Pipeline threw before producing a result',
+            };
+        }
+        const firstFailure = result.Results.find((r) => !r.Success);
+        return {
+            Kind: 'run.end',
+            Success: result.FailureCount === 0,
+            SuccessCount: result.SuccessCount,
+            FailureCount: result.FailureCount,
+            TotalCount: result.TotalCount,
+            ErrorMessage: firstFailure?.ErrorMessage,
+            ErrorStep: firstFailure?.ErrorStep,
+        };
+    }
+    /** Publishes the terminal run boundary. `result` is undefined when a throw escaped the pipeline. */
+    notifyRunEnd(result, totalCount) {
+        this.notifyObserver(RuntimeSchemaManager.BuildRunEndEvent(result, totalCount));
+    }
     // ─── Pipeline ────────────────────────────────────────────────────
     /**
      * Execute the RSU pipeline for a single input. Convenience wrapper
@@ -360,21 +555,35 @@ export class RuntimeSchemaManager extends BaseSingleton {
         const sharedSteps = [];
         // U11 — arm the determinate step counter (index of expected total) for this run.
         this.beginStepTracking(inputs.length);
+        this.notifyObserver({
+            Kind: 'run.start',
+            ItemCount: inputs.length,
+            Descriptions: inputs.map((i) => i.Description),
+            AffectedTables: [...new Set(inputs.flatMap((i) => i.AffectedTables))],
+            StepTotal: this._stepTotal ?? 0,
+        });
+        // Captured so the `finally` can publish the terminal run boundary on EVERY exit path —
+        // early validation failure, normal completion, or a throw (which leaves it undefined).
+        let batchResult;
         try {
             // Phase 1: Validate
             const validationFailure = await this.validateBatch(inputs, sharedSteps);
-            if (validationFailure)
-                return validationFailure;
+            if (validationFailure) {
+                batchResult = validationFailure;
+                return batchResult;
+            }
             // Phase 2: Execute migrations under lock
             const itemResults = await this.executeMigrations(inputs, sharedSteps);
             // Phase 3: Post-migration pipeline (CodeGen, compile, restart, git)
             const successfulItems = itemResults.filter((r) => r.Success);
             const postResult = await this.runPostMigrationPipeline(inputs, successfulItems, sharedSteps);
             // Phase 4: Build per-caller results
-            return this.buildPerCallerResults(itemResults, successfulItems, sharedSteps, postResult);
+            batchResult = this.buildPerCallerResults(itemResults, successfulItems, sharedSteps, postResult);
+            return batchResult;
         }
         finally {
             this.endStepTracking();
+            this.notifyRunEnd(batchResult, inputs.length);
         }
     }
     /** Phase 1: Validate environment and all migration SQL. Returns a batch failure result if validation fails, null on success. */
@@ -386,11 +595,11 @@ export class RuntimeSchemaManager extends BaseSingleton {
         for (const input of inputs) {
             const validation = ValidateMigrationSQL(input.MigrationSQL, this.getProtectedSchemas());
             if (!validation.Valid) {
-                sharedSteps.push({ Name: 'ValidateSQL', Status: 'failed', DurationMs: 0, Message: validation.Errors.join('; ') });
+                this.recordStep(sharedSteps, { Name: 'ValidateSQL', Status: 'failed', DurationMs: 0, Message: validation.Errors.join('; ') });
                 return this.buildBatchResult(inputs.map((i) => this.buildFailedResult(i, 'ValidateSQL', sharedSteps)));
             }
         }
-        sharedSteps.push({ Name: 'ValidateSQL', Status: 'success', DurationMs: 0, Message: `Validated ${inputs.length} migration(s)` });
+        this.recordStep(sharedSteps, { Name: 'ValidateSQL', Status: 'success', DurationMs: 0, Message: `Validated ${inputs.length} migration(s)` });
         return null;
     }
     /** Phase 2: Acquire lock, write and execute each migration, release lock. */
@@ -426,7 +635,7 @@ export class RuntimeSchemaManager extends BaseSingleton {
             }
             if (itemResults.some((r) => r.Success)) {
                 this.MarkOutOfSync();
-                sharedSteps.push({ Name: 'MarkOutOfSync', Status: 'success', DurationMs: 0, Message: 'DB changed, API out-of-sync until CodeGen completes' });
+                this.recordStep(sharedSteps, { Name: 'MarkOutOfSync', Status: 'success', DurationMs: 0, Message: 'DB changed, API out-of-sync until CodeGen completes' });
             }
         }
         finally {
@@ -473,6 +682,12 @@ export class RuntimeSchemaManager extends BaseSingleton {
         // Failed migrations should not trigger post-restart entity maps or syncs.
         const successfulInputs = successfulItems.map(r => r.Input);
         await this.writePostRestartFiles(successfulInputs);
+        // Register durable pending work for successful migrations. Rows stay Pending
+        // until the post-restart consumer completes them, so a crash between here and
+        // consumption leaves the work visible instead of losing it.
+        const pendingWork = await this.registerPendingWork(successfulInputs);
+        result.PendingWorkIDs = pendingWork.IDs;
+        result.PendingWorkErrors = pendingWork.Errors;
         // Restart LAST — PM2 restart kills this process, nothing runs after this
         if (!inputs.every((i) => i.SkipRestart)) {
             const restartOk = await this.runStep('RestartMJAPI', () => this.restartMJAPI(), sharedSteps);
@@ -493,19 +708,31 @@ export class RuntimeSchemaManager extends BaseSingleton {
             // A migration that executed but whose run-wide CodeGen failed is NOT a success —
             // the entity may have no spCreate/spUpdate procs and would silently skip on sync.
             const codeGenFailedThisCaller = codeGenFailed && item.Success;
+            // Same reasoning as the CodeGen case above: the migration ran, but work the caller
+            // asked to happen after the restart was never persisted, so the restart drops it.
+            // Reporting success here would tell the caller their sync is coming when it is not.
+            const pendingWorkErrors = postResult.PendingWorkErrors?.get(item.Input) ?? [];
+            const pendingWorkFailedThisCaller = pendingWorkErrors.length > 0 && item.Success;
             const result = {
-                Success: item.Success && successfulItems.length > 0 && !codeGenFailed,
+                Success: item.Success && successfulItems.length > 0 && !codeGenFailed && !pendingWorkFailedThisCaller,
                 MigrationFilePath: item.FilePath,
                 APIRestarted: postResult.ApiRestarted,
                 GitCommitSuccess: postResult.GitCommitSuccess,
                 BranchName: postResult.BranchName,
                 Steps: allSteps,
-                ErrorMessage: codeGenFailedThisCaller ? postResult.CodeGenError ?? item.Error : item.Error,
+                ErrorMessage: codeGenFailedThisCaller
+                    ? postResult.CodeGenError ?? item.Error
+                    : pendingWorkFailedThisCaller
+                        ? pendingWorkErrors.join('; ')
+                        : item.Error,
                 ErrorStep: codeGenFailedThisCaller
                     ? 'RunCodeGen'
-                    : item.Error
-                        ? item.Steps.find((s) => s.Status === 'failed')?.Name
-                        : undefined,
+                    : pendingWorkFailedThisCaller
+                        ? 'RegisterPendingWork'
+                        : item.Error
+                            ? item.Steps.find((s) => s.Status === 'failed')?.Name
+                            : undefined,
+                PendingWorkIDs: postResult.PendingWorkIDs?.get(item.Input),
             };
             this.writeAuditLog(item.Input, result).catch((err) => LogError(`[RSU] Audit log failed: ${err instanceof Error ? err.message : String(err)}`));
             this.recordMetrics(item, allSteps, result);
@@ -665,7 +892,6 @@ export class RuntimeSchemaManager extends BaseSingleton {
      * Verify ALL filesystem paths RSU may write to are actually writable.
      * Paths checked:
      *  - MigrationsPath     (migration SQL files)
-     *  - PendingWorkPath    (post-restart task JSONs)
      *  - AdditionalSchemaInfoPath parent dir (soft FK / soft PK config)
      *  - CodeGenDir         (temp codegen script)
      *  - WorkDir            (pipeline log file)
@@ -676,7 +902,6 @@ export class RuntimeSchemaManager extends BaseSingleton {
         const workDir = rsuConfig.WorkDir;
         const directoriesToCheck = [
             nodePath.join(workDir, rsuConfig.MigrationsPath),
-            nodePath.join(workDir, rsuConfig.PendingWorkPath),
             dirname(nodePath.join(workDir, this.additionalSchemaInfoPath)),
             rsuConfig.CodeGenDir,
             workDir, // pipeline log
@@ -1022,8 +1247,9 @@ export class RuntimeSchemaManager extends BaseSingleton {
         const processName = rsuConfig.PM2ProcessName;
         // When running inside the MJAPI process that PM2 manages, `pm2 restart`
         // sends SIGINT to THIS process. We will die before execAsync resolves.
-        // That's fine — the pending-work file (.rsu_pending/) was already written
-        // before this step, and the newly restarted process picks it up on boot.
+        // That's fine — the pending-work rows were already committed to
+        // `MJ: RSU Pending Works` before this step, and the newly restarted process
+        // picks them up on boot.
         //
         // Strategy: fire the restart command, catch the inevitable rejection
         // (our process gets killed mid-flight), and return true. If the process
@@ -1579,18 +1805,19 @@ export class RuntimeSchemaManager extends BaseSingleton {
         const stepIndex = this._currentStepIndex ?? undefined;
         const stepTotal = this._stepTotal ?? undefined;
         this.rsuLog(`▶ Starting step${stepIndex && stepTotal ? ` ${stepIndex}/${stepTotal}` : ''}: ${name}`);
+        this.notifyObserver({ Kind: 'step.start', Name: name, StepIndex: stepIndex, StepTotal: stepTotal });
         try {
             const result = await fn();
             const durationMs = Date.now() - start;
             const msg = `${name} completed successfully`;
-            steps.push({ Name: name, Status: 'success', DurationMs: durationMs, Message: msg, StepIndex: stepIndex, StepTotal: stepTotal });
+            this.recordStep(steps, { Name: name, Status: 'success', DurationMs: durationMs, Message: msg, StepIndex: stepIndex, StepTotal: stepTotal });
             this.rsuLog(`✓ ${name} — ${durationMs}ms`);
             return result;
         }
         catch (error) {
             const durationMs = Date.now() - start;
             const msg = error instanceof Error ? error.message : String(error);
-            steps.push({ Name: name, Status: 'failed', DurationMs: durationMs, Message: msg, StepIndex: stepIndex, StepTotal: stepTotal });
+            this.recordStep(steps, { Name: name, Status: 'failed', DurationMs: durationMs, Message: msg, StepIndex: stepIndex, StepTotal: stepTotal });
             this.rsuLog(`✗ ${name} — FAILED after ${durationMs}ms: ${msg}`);
             return undefined;
         }