@memberjunction/scheduling-engine 5.38.0 → 5.39.0

package/dist/ScheduledJobEngine.js

@@ -3,7 +3,8 @@
  * @module @memberjunction/scheduling-engine
  */
 import os from 'os';
-import { Metadata, LogError, LogStatusEx } from '@memberjunction/core';
+import { v4 as uuidv4 } from 'uuid';
+import { Metadata, LogError, LogStatusEx, LocalCacheManager, RunView } from '@memberjunction/core';
 import { BaseSingleton, MJGlobal, UUIDsEqual } from '@memberjunction/global';
 import { SchedulingEngineBase } from '@memberjunction/scheduling-engine-base';
 import { BaseScheduledJob } from './BaseScheduledJob.js';
@@ -38,6 +39,52 @@ export class SchedulingEngine extends BaseSingleton {
         this.hasInitialized = false;
         /** Job IDs we have already warned about for sub-threshold run frequency. */
         this.highFrequencyWarnedJobIds = new Set();
+        // ========================================================================
+        // DECOUPLING STATE (added in v5.39 — see plans/scheduled-job-engine-decoupling.md)
+        // ========================================================================
+        /**
+         * Maximum concurrent scheduled jobs on this engine instance. Default 5.
+         * Configurable via MJServer's `scheduledJobs.maxConcurrentJobs` config.
+         *
+         * SOFT CAP: under overlapping poll bodies the cap may be transiently
+         * exceeded by a small amount bounded by overlap count. See README
+         * "Cap and lease semantics" for tuning guidance.
+         */
+        this._maxConcurrentJobs = 5;
+        /**
+         * Lock lease duration. Default 10 minutes. Configurable via
+         * mj.config.cjs `scheduling.leaseTimeoutMinutes`.
+         *
+         * Public API uses minutes (operational readability); internal computations
+         * use _leaseTimeoutMs for precision and testability. Tests inject sub-second
+         * leases via `_setLeaseTimeoutMsForTest`.
+         *
+         * Constraint: must be > maximum expected runtime of any job. Setting too
+         * low causes healthy long jobs to be reclaimed and re-dispatched.
+         */
+        this._leaseTimeoutMs = 10 * 60 * 1000;
+        /**
+         * Promises for jobs currently dispatched but not yet settled. Keyed by job ID.
+         *
+         * Three purposes:
+         *   1. Bounded concurrency — DispatchScheduledJobs checks size vs MaxConcurrentJobs.
+         *   2. Sweep untracking — sweepStaleInflightJobs deletes by ID for jobs whose
+         *      lease has expired, freeing the cap slot even though the JS promise leaks
+         *      (see README "Leaked promise behavior").
+         *   3. Graceful shutdown — StopPolling can await all in-flight via .values().
+         *
+         * Self-cleans via identity-checked .finally() on each dispatched promise
+         * (no-op if a sweep + re-dispatch already replaced the entry).
+         *
+         * NOT used for double-dispatch prevention — that's the atomic lock sproc's job.
+         */
+        this.inflightJobPromises = new Map();
+        /**
+         * When false, DispatchScheduledJobs becomes a no-op. Set false in StopPolling
+         * BEFORE snapshotting inflightJobPromises for shutdown drain, so no new
+         * entries sneak in during the shutdown window.
+         */
+        this.acceptingDispatches = true;
     }
     /**
      * Get singleton instance
@@ -72,6 +119,52 @@ export class SchedulingEngine extends BaseSingleton {
     get ActivePollingInterval() {
         return this.Base.ActivePollingInterval;
     }
+    /**
+     * Maximum concurrent scheduled jobs on this engine instance. Default 5.
+     * Configurable via MJServer's `scheduledJobs.maxConcurrentJobs` config.
+     */
+    get MaxConcurrentJobs() {
+        return this._maxConcurrentJobs;
+    }
+    set MaxConcurrentJobs(value) {
+        if (!Number.isInteger(value) || value < 1) {
+            throw new Error(`MaxConcurrentJobs must be a positive integer, got ${value}`);
+        }
+        const old = this._maxConcurrentJobs;
+        this._maxConcurrentJobs = value;
+        this.log(`MaxConcurrentJobs changed from ${old} to ${value}`);
+    }
+    /**
+     * Lock lease duration in milliseconds. Default 600000 (10 minutes).
+     * Production callers should use this setter — matches the ms unit of
+     * MJServer's `scheduledJobs.defaultLockTimeout` config.
+     */
+    get LeaseTimeoutMs() {
+        return this._leaseTimeoutMs;
+    }
+    set LeaseTimeoutMs(value) {
+        if (!Number.isFinite(value) || value <= 0) {
+            throw new Error(`LeaseTimeoutMs must be a positive number, got ${value}`);
+        }
+        const old = this._leaseTimeoutMs;
+        this._leaseTimeoutMs = value;
+        this.log(`LeaseTimeoutMs changed from ${old} to ${value}`);
+    }
+    /**
+     * Convenience accessor — lease duration as integer minutes. Production
+     * code may use either this or `LeaseTimeoutMs`. The setter validates
+     * positive integer minutes (no fractional minutes via this path; use
+     * `LeaseTimeoutMs` for sub-minute precision, including tests).
+     */
+    get LeaseTimeoutMinutes() {
+        return Math.round(this._leaseTimeoutMs / 60_000);
+    }
+    set LeaseTimeoutMinutes(value) {
+        if (!Number.isInteger(value) || value < 1) {
+            throw new Error(`LeaseTimeoutMinutes must be a positive integer, got ${value}`);
+        }
+        this.LeaseTimeoutMs = value * 60 * 1000;
+    }
     /** Find a job type by name. */
     GetJobTypeByName(name) {
         return this.Base.GetJobTypeByName(name);
@@ -106,76 +199,110 @@ export class SchedulingEngine extends BaseSingleton {
     // EXECUTION METHODS
     // ========================================================================
     /**
-     * Start continuous polling for scheduled jobs
-     * Uses adaptive interval based on ActivePollingInterval
+     * Start continuous polling for scheduled jobs.
+     *
+     * Async (changed in v5.39) because upfront work — Config, initial-NextRunAt
+     * seeding, stale-lock cleanup, permission probe — runs ONCE before the
+     * first poll fires. Subsequent polls assume that work is complete.
+     *
+     * The poll callback re-arms its timer FIRST, before any awaited work, so
+     * that any hang downstream (Config, DispatchScheduledJobs, etc.) cannot
+     * prevent the next poll from firing on schedule. This is the load-bearing
+     * invariant of the decoupling fix (see plans/scheduled-job-engine-decoupling.md).
      *
      * @param contextUser - User context for execution
      */
-    StartPolling(contextUser) {
+    async StartPolling(contextUser) {
         if (this.isPolling) {
             this.log('Polling already started');
             return;
         }
+        // Upfront work — done ONCE before any poll fires.
+        // Order: Config first (so this.ScheduledJobs is populated), then
+        // initializeNextRunTimes / cleanupStaleLocks / permission probe.
+        await this.Config(false, contextUser);
+        await this.initializeNextRunTimes(contextUser);
+        await this.cleanupStaleLocks(contextUser);
+        await this.probeLockSprocPermissions();
+        if (this.ScheduledJobs.length === 0) {
+            console.log(`📅 Scheduled Jobs: No active jobs found, polling not started`);
+            return;
+        }
+        this.warnAboutHighFrequencyJobs();
         this.isPolling = true;
+        this.acceptingDispatches = true;
+        this.hasInitialized = true;
         const poll = async () => {
-            // Initialize NextRunAt and clean up stale locks on first poll only
-            if (!this.hasInitialized) {
-                await this.initializeNextRunTimes(contextUser);
-                await this.cleanupStaleLocks(contextUser);
-                // No need to force-reload: initializeNextRunTimes and cleanupStaleLocks
-                // modify and save the in-memory entity objects directly, so the cache
-                // already reflects the current DB state.
-                this.hasInitialized = true;
-                // Check if there are no jobs after initialization
-                if (this.ScheduledJobs.length === 0) {
-                    console.log(`📅 Scheduled Jobs: No active jobs found, stopping polling`);
-                    this.StopPolling();
+            // Re-arm IMMEDIATELY — load-bearing invariant. The next poll fires
+            // on schedule regardless of any await downstream.
+            if (this.isPolling) {
+                const interval = this.ActivePollingInterval;
+                if (interval === null) {
+                    console.log(`📅 Scheduled Jobs: All jobs removed, stopping polling`);
+                    // Schedule the cleanup via the same async path so we still
+                    // return cleanly from this poll body.
+                    this.StopPolling().catch(err => this.logError('Error during StopPolling', err));
                     return;
                 }
-                this.warnAboutHighFrequencyJobs();
+                this.pollingTimer = setTimeout(poll, interval);
             }
             try {
-                const runs = await this.ExecuteScheduledJobs(contextUser);
-                // Only log if jobs were actually executed
-                if (runs.length > 0) {
-                    console.log(`📅 Scheduled Jobs: Executed ${runs.length} job(s)`);
-                }
-                // Schedule next poll based on current ActivePollingInterval
-                if (this.isPolling) {
-                    const interval = this.ActivePollingInterval;
-                    // If interval is null (no jobs), stop polling
-                    if (interval === null) {
-                        console.log(`📅 Scheduled Jobs: All jobs removed, stopping polling`);
-                        this.StopPolling();
-                        return;
-                    }
-                    this.pollingTimer = setTimeout(poll, interval);
+                const result = await this.DispatchScheduledJobs(contextUser);
+                if (result.swept > 0 || result.dispatched > 0 || result.lockedOut > 0 || result.skippedAtCapacity > 0) {
+                    console.log(`📅 Scheduled Jobs: swept=${result.swept}, dispatched=${result.dispatched}, ` +
+                        `lockedOut=${result.lockedOut}, skippedAtCapacity=${result.skippedAtCapacity}, ` +
+                        `inflight=${this.inflightJobPromises.size}/${this.MaxConcurrentJobs}`);
                 }
             }
             catch (error) {
-                this.logError('Error during polling', error);
-                // Continue polling even after errors
-                if (this.isPolling) {
-                    this.pollingTimer = setTimeout(poll, 60000); // Fallback to 1 minute
-                }
+                this.logError('Error during DispatchScheduledJobs', error);
+                // No fallback timer needed — re-arm at top of function already
+                // scheduled the next poll before any await could fire.
             }
         };
-        // Start first poll immediately
-        poll();
+        // First poll fires on the timer (not immediate) so observers can stop
+        // us between StartPolling returning and the first poll firing.
+        this.pollingTimer = setTimeout(poll, this.ActivePollingInterval ?? 60_000);
+        this.log('Started scheduled job polling');
     }
     /**
-     * Stop continuous polling
+     * Stop continuous polling.
+     *
+     * Async (changed in v5.39). With opts.waitForInflight=true, awaits all
+     * currently-dispatched jobs to settle before returning. With opts.maxWaitMs,
+     * bounds that wait so a zombie can't make shutdown hang indefinitely.
+     *
+     * Order matters: sets acceptingDispatches=false FIRST so no new entries
+     * can be added to inflightJobPromises during the snapshot for allSettled.
+     *
+     * @param opts.waitForInflight - Await dispatched jobs before returning
+     * @param opts.maxWaitMs - Bound the wait (only meaningful with waitForInflight)
      */
-    StopPolling() {
-        if (!this.isPolling) {
+    async StopPolling(opts) {
+        if (!this.isPolling)
             return;
-        }
+        // Order matters: block new dispatches BEFORE snapshotting inflight.
+        this.acceptingDispatches = false;
         this.isPolling = false;
         if (this.pollingTimer) {
             clearTimeout(this.pollingTimer);
             this.pollingTimer = undefined;
         }
         this.log('Stopped scheduled job polling');
+        if (opts?.waitForInflight && this.inflightJobPromises.size > 0) {
+            const promises = [...this.inflightJobPromises.values()];
+            this.log(`Waiting for ${promises.length} in-flight job(s) to settle...`);
+            if (opts.maxWaitMs) {
+                await Promise.race([
+                    Promise.allSettled(promises),
+                    new Promise(resolve => setTimeout(resolve, opts.maxWaitMs))
+                ]);
+            }
+            else {
+                await Promise.allSettled(promises);
+            }
+            this.log(`Shutdown wait complete`);
+        }
     }
     /**
      * Check if polling is currently active
@@ -312,14 +439,30 @@ export class SchedulingEngine extends BaseSingleton {
         for (const job of this.ScheduledJobs) {
             console.log(`  - ${job.Name}: NextRunAt=${job.NextRunAt?.toISOString() || 'NULL'}, Status=${job.Status}`);
             if (this.isJobDue(job, evalTime)) {
-                console.log(`    ✓ Job is due, executing...`);
+                console.log(`    ✓ Job is due, attempting to acquire lock...`);
                 try {
-                    const run = await this.executeJob(job, contextUser);
-                    if (run) { // null if skipped
-                        runs.push(run);
+                    const lockResult = await this.tryAcquireLock(job.ID);
+                    if (!lockResult.acquired) {
+                        if (job.ConcurrencyMode === 'Queue') {
+                            this.log(`Job ${job.Name} is locked, queueing (ConcurrencyMode=Queue)`);
+                            const queuedRun = await this.createQueuedJobRun(job, contextUser);
+                            runs.push(queuedRun);
+                        }
+                        else if (job.ConcurrencyMode === 'Skip') {
+                            console.log(`    ⊘ Job is locked, skipping (ConcurrencyMode=Skip)`);
+                        }
+                        else {
+                            // Concurrent mode: proceed without lock
+                            console.log(`    ↪ Concurrent mode, proceeding without lock`);
+                            const run = await this.executeJobWithLock(job, null, contextUser);
+                            if (run)
+                                runs.push(run);
+                        }
                     }
                     else {
-                        console.log(`    ⊘ Job was skipped (locked or queued)`);
+                        const run = await this.executeJobWithLock(job, lockResult.token, contextUser);
+                        if (run)
+                            runs.push(run);
                     }
                 }
                 catch (error) {
@@ -348,7 +491,96 @@ export class SchedulingEngine extends BaseSingleton {
         if (!job) {
             throw new Error(`Scheduled job ${jobId} not found or not active`);
         }
-        return await this.executeJob(job, contextUser);
+        const lockResult = await this.tryAcquireLock(job.ID);
+        if (!lockResult.acquired) {
+            throw new Error(`Could not acquire lock for job ${jobId} — held by another holder`);
+        }
+        const run = await this.executeJobWithLock(job, lockResult.token, contextUser);
+        if (!run) {
+            throw new Error(`Job execution returned null for ${jobId}`);
+        }
+        return run;
+    }
+    /**
+     * Dispatch all currently-due scheduled jobs WITHOUT awaiting their completion.
+     *
+     * This is the polling-path entry point introduced in v5.39 as part of the
+     * scheduler decoupling fix (GH #2736). The poll loop calls this and re-arms
+     * its timer based on the synchronous-portion return; jobs run in the background.
+     *
+     * Two phases:
+     *
+     *   PHASE 1 — Stale-inflight sweep (decoupled from isJobDue AND from the cap):
+     *     Walks inflightJobPromises looking for jobs whose DB lease has expired.
+     *     Untracks each, frees its cap slot, and fire-and-forget marks any
+     *     orphaned `Status='Running'` run records as abandoned. Runs first so
+     *     it can free slots BEFORE the cap check throttles dispatch.
+     *
+     *   PHASE 2 — Cap-bounded dispatch loop:
+     *     For each due job, atomically acquire its lock via spAcquireScheduledJobLock.
+     *     Only jobs whose lock was acquired count against MaxConcurrentJobs.
+     *     Lock-failed jobs are reported via `lockedOut` counter.
+     *     If at-cap, remaining due jobs counted via `skippedAtCapacity` and
+     *     picked up by subsequent polls as slots free (no in-memory queueing).
+     *
+     * Same-instance double-dispatch is structurally prevented by the atomic
+     * lock sproc — its WHERE clause filters held-and-not-stale locks, so any
+     * second attempt against the same job ID returns Acquired=0.
+     *
+     * In-flight dispatched promises are tracked in `inflightJobPromises` so
+     * `StopPolling({ waitForInflight: true })` can perform graceful shutdown.
+     *
+     * @returns Counters for observability.
+     */
+    async DispatchScheduledJobs(contextUser, evalTime = new Date()) {
+        if (!this.acceptingDispatches) {
+            return { swept: 0, dispatched: 0, lockedOut: 0, skippedAtCapacity: 0 };
+        }
+        // PHASE 1: stale-inflight sweep. Decoupled from isJobDue and cap.
+        const swept = await this.sweepStaleInflightJobs(contextUser);
+        // PHASE 2: cap-bounded dispatch.
+        let dispatched = 0;
+        let lockedOut = 0;
+        let skippedAtCapacity = 0;
+        for (const job of this.ScheduledJobs) {
+            if (!this.isJobDue(job, evalTime))
+                continue;
+            if (this.inflightJobPromises.size >= this.MaxConcurrentJobs) {
+                skippedAtCapacity++;
+                continue;
+            }
+            const lockResult = await this.tryAcquireLock(job.ID);
+            if (!lockResult.acquired) {
+                if (job.ConcurrencyMode === 'Queue') {
+                    await this.createQueuedJobRun(job, contextUser);
+                }
+                lockedOut++;
+                continue;
+            }
+            // TDZ-safe identity tracking: set Map entry SYNCHRONOUSLY before
+            // attaching catch/finally. A synchronous throw in executeJobWithLock
+            // (e.g. ClassFactory.CreateInstance failing on a missing DriverClass)
+            // would otherwise fire .finally before .set runs and orphan the entry.
+            const promise = this.executeJobWithLock(job, lockResult.token, contextUser);
+            this.inflightJobPromises.set(job.ID, promise);
+            promise
+                .catch(error => {
+                this.logError(`Unexpected throw escaping executeJobWithLock for job ${job.Name} ` +
+                    `(indicates a bug in the engine itself, not a plugin)`, error);
+                return null;
+            })
+                .finally(() => {
+                // Identity check: only delete if the entry still refers to
+                // OUR promise. If a sweep + re-dispatch already replaced it,
+                // leave the new entry alone.
+                if (this.inflightJobPromises.get(job.ID) === promise) {
+                    this.inflightJobPromises.delete(job.ID);
+                }
+            });
+            dispatched++;
+        }
+        this.UpdatePollingInterval();
+        return { swept, dispatched, lockedOut, skippedAtCapacity };
     }
     /**
      * Determine if a job is currently due for execution
@@ -376,30 +608,31 @@ export class SchedulingEngine extends BaseSingleton {
     /**
      * Execute a single scheduled job
      *
-     * @param job - The job to execute
-     * @param contextUser - User context
-     * @returns The created job run record
+     * Execute a single scheduled job WITH a pre-acquired lock token.
+     *
+     * Caller (DispatchScheduledJobs / ExecuteScheduledJob / ExecuteScheduledJobs)
+     * is responsible for acquiring the lock atomically via tryAcquireLock and
+     * passing the resulting token. This method owns the lock's lifecycle from
+     * this point forward: every exit path (success, failure, exception)
+     * releases the lock via releaseLockIfTokenMatches.
+     *
+     * If lockToken is null, the job is running in `ConcurrencyMode='Concurrent'`
+     * (no lock acquired) — finally simply skips the release.
+     *
+     * @param job - The job entity. READ-ONLY from this method's perspective —
+     *              do not mutate or call Save on it. The shared entity in
+     *              this.ScheduledJobs must not be touched here.
+     * @param lockToken - The token returned by tryAcquireLock, or null for
+     *                    Concurrent mode where no lock was acquired.
+     * @param contextUser - User context for execution.
+     * @returns The created run record (Completed or Failed), or null if a
+     *          synchronous setup error prevented run creation.
      * @private
      */
-    async executeJob(job, contextUser) {
-        // Try to acquire lock for this job
-        const lockAcquired = await this.tryAcquireLock(job);
-        if (!lockAcquired) {
-            // Handle based on concurrency mode
-            if (job.ConcurrencyMode === 'Skip') {
-                this.log(`Job ${job.Name} is locked, skipping (ConcurrencyMode=Skip)`);
-                return null; // Skip this execution
-            }
-            else if (job.ConcurrencyMode === 'Queue') {
-                this.log(`Job ${job.Name} is locked, queueing (ConcurrencyMode=Queue)`);
-                // Create a queued run record for future processing
-                return await this.createQueuedJobRun(job, contextUser);
-            }
-            // Concurrent mode: proceed without lock
-        }
-        // Create run record
-        const run = await this.createJobRun(job, contextUser);
+    async executeJobWithLock(job, lockToken, contextUser) {
+        let run = null;
         try {
+            run = await this.createJobRun(job, contextUser);
             // Get job type
             const jobType = this.ScheduledJobTypes.find(t => UUIDsEqual(t.ID, job.JobTypeID));
             if (!jobType) {
@@ -426,7 +659,13 @@ export class SchedulingEngine extends BaseSingleton {
             run.Success = result.Success;
             run.ErrorMessage = result.ErrorMessage || null;
             run.Details = result.Details ? JSON.stringify(result.Details) : null;
-            await run.Save();
+            const runSaved = await run.Save();
+            if (!runSaved) {
+                this.logError(`Failed to save run record for job ${job.Name} (run ${run.ID}): ` +
+                    `${run.LatestResult?.CompleteMessage ?? 'unknown'}`, null);
+                // Continue — stats update + release are still important even if
+                // the run record save failed (best-effort persistence).
+            }
             // Update job statistics
             await this.updateJobStatistics(job, result.Success, run.ID);
             // Send notifications if configured
@@ -438,21 +677,28 @@ export class SchedulingEngine extends BaseSingleton {
             return run;
         }
         catch (error) {
-            // Update run with failure
-            run.CompletedAt = new Date();
-            run.Status = 'Failed';
-            run.Success = false;
-            run.ErrorMessage = error instanceof Error ? error.message : 'Unknown error';
-            await run.Save();
-            // Update job failure count
-            await this.updateJobStatistics(job, false, run.ID);
+            if (run) {
+                run.CompletedAt = new Date();
+                run.Status = 'Failed';
+                run.Success = false;
+                run.ErrorMessage = error instanceof Error ? error.message : 'Unknown error';
+                const runSaved = await run.Save();
+                if (!runSaved) {
+                    this.logError(`Failed to save failed-run record for job ${job.Name} (run ${run.ID}): ` +
+                        `${run.LatestResult?.CompleteMessage ?? 'unknown'}`, null);
+                    // Continue — stats update + release still need to happen.
+                }
+                await this.updateJobStatistics(job, false, run.ID);
+            }
             this.logError(`Job failed: ${job.Name}`, error);
             return run;
         }
         finally {
-            // Release lock if we acquired it
-            if (lockAcquired) {
-                await this.releaseLock(job);
+            // Token-checked release — safe under lease-expiry races.
+            // Sproc no-ops if our token no longer matches (another holder reclaimed).
+            // Skip entirely for Concurrent mode (no lock to release).
+            if (lockToken) {
+                await this.releaseLockIfTokenMatches(job.ID, lockToken);
             }
         }
     }
@@ -474,7 +720,11 @@ export class SchedulingEngine extends BaseSingleton {
      * Update job statistics after execution
      * @private
      */
-    async updateJobStatistics(job, success, runId) {
+    async updateJobStatistics(job, success, _runId) {
+        const now = new Date();
+        const nextRun = CronExpressionHelper.GetNextRunTime(job.CronExpression, job.Timezone);
+        // Update in-memory entity so callers reading job.RunCount/job.NextRunAt
+        // immediately after see the new values without a Load round-trip.
         job.RunCount++;
         if (success) {
             job.SuccessCount++;
@@ -482,9 +732,49 @@ export class SchedulingEngine extends BaseSingleton {
         else {
             job.FailureCount++;
         }
-        job.LastRunAt = new Date();
-        job.NextRunAt = CronExpressionHelper.GetNextRunTime(job.CronExpression, job.Timezone);
-        await job.Save();
+        job.LastRunAt = now;
+        job.NextRunAt = nextRun;
+        // Persist via the targeted sproc — touches ONLY the 5 stats columns.
+        // We deliberately do NOT call job.Save() here: a full-entity Save would
+        // overwrite the DB's lock columns with the entity's stale in-memory
+        // values (the entity was loaded BEFORE the atomic lock sproc set
+        // LockToken), blowing away the live lock. That manifested as a
+        // false-positive token mismatch on every successful job completion.
+        // See plans/scheduled-job-engine-decoupling.md and the migration
+        // V202606022027__v5.39.x__Scheduling_Engine_Atomic_Stats_Update.sql.
+        // MJ pattern: positional placeholders; see tryAcquireLock for rationale.
+        const provider = this.Base.ProviderToUse;
+        const schema = provider.MJCoreSchemaName;
+        await provider.ExecuteSQL(`EXEC [${schema}].[spUpdateScheduledJobStatistics] ` +
+            `@JobID=@p0, @Success=@p1, @LastRunAt=@p2, @NextRunAt=@p3`, [job.ID, success ? 1 : 0, now, nextRun], { isMutation: true, description: 'spUpdateScheduledJobStatistics' }, this.Base.ContextUser);
+        // Direct SQL bypasses BaseEntity.Save(), so it skips the save event that
+        // would normally drive LocalCacheManager invalidation. Without this call,
+        // any cached RunView for 'MJ: Scheduled Jobs' (e.g., the Scheduling
+        // Dashboard's filtered views) would keep showing stale RunCount /
+        // SuccessCount / NextRunAt until the cache TTL expired.
+        //
+        // We use full invalidation rather than in-place upsert (UpsertSingleEntity)
+        // because the dashboard reads through filtered/sorted views — MJ can't
+        // safely patch those in JS (it would need to evaluate the SQL filter to
+        // know whether the updated row still belongs in each cached result).
+        //
+        // See guides/CACHING_AND_PUBSUB_GUIDE.md and CLAUDE.md §"Server-Side
+        // Caching" — `BypassCache: true` is the documented opposite escape hatch
+        // for readers; this is the producer-side counterpart.
+        //
+        // Wrapped in try/catch: cache invalidation is best-effort observability
+        // hygiene, not load-bearing for job execution. If LocalCacheManager throws
+        // (e.g., uninitialized in a test environment, transient storage error),
+        // the stats UPDATE has already persisted — the worst case is one cached
+        // dashboard view stays stale until its TTL.
+        try {
+            if (LocalCacheManager.Instance.IsInitialized) {
+                await LocalCacheManager.Instance.InvalidateEntityCaches('MJ: Scheduled Jobs');
+            }
+        }
+        catch (cacheError) {
+            this.logError('Cache invalidation after stats update failed (non-fatal)', cacheError);
+        }
     }
     /**
      * Send notifications if configured
@@ -515,118 +805,196 @@ export class SchedulingEngine extends BaseSingleton {
      * Try to acquire a lock for job execution
      * @private
      */
-    async tryAcquireLock(job) {
-        console.log(`    🔒 tryAcquireLock: job.LockToken=${job.LockToken?.substring(0, 8) || 'NULL'}, ExpectedCompletionAt=${job.ExpectedCompletionAt?.toISOString() || 'NULL'}`);
-        if (job.LockToken != null) {
-            const now = new Date();
-            console.log(`      Lock exists! Checking if stale: ExpectedCompletionAt=${job.ExpectedCompletionAt?.toISOString()}, now=${now.toISOString()}`);
-            if (job.ExpectedCompletionAt && now > job.ExpectedCompletionAt) {
-                console.log(`      → Lock is STALE, cleaning up...`);
-                this.log(`Detected stale lock on job ${job.Name}, cleaning up`);
-                const cleaned = await this.cleanupStaleLock(job);
-                if (!cleaned) {
-                    console.log(`      ❌ Failed to clean up stale lock, skipping`);
-                    return false;
-                }
-                // Reload from DB to verify cleanup succeeded
-                await job.Load(job.ID);
-                if (job.LockToken != null) {
-                    console.log(`      ❌ Lock still present after cleanup (re-acquired by ${job.LockedByInstance}), skipping`);
-                    return false;
-                }
-                console.log(`      ✓ Stale lock cleaned successfully`);
-            }
-            else {
-                console.log(`      → Lock is ACTIVE (not stale), returning false`);
-                return false;
-            }
-        }
-        else {
-            console.log(`      → No lock exists, will try to acquire`);
+    /**
+     * Atomically acquire a lock on a job via spAcquireScheduledJobLock.
+     * The sproc's WHERE clause handles both the free-lock and stale-lease cases
+     * in a single statement — no TOCTOU window between check and write.
+     *
+     * Operates only on lock columns; never mutates the shared entity in
+     * this.ScheduledJobs. Caller passes jobId (string), not the entity object.
+     *
+     * @returns { acquired: true, token } on success; { acquired: false } otherwise.
+     * @private
+     */
+    async tryAcquireLock(jobId) {
+        // uuidv4 (cryptographic) — Math.random()-based GUIDs would risk collision
+        // between concurrent engine instances and break the lost-mutex protection
+        // that releaseLockIfTokenMatches relies on (token identity = execution identity).
+        const token = uuidv4();
+        const instance = this.getInstanceIdentifier();
+        const expectedCompletion = new Date(Date.now() + this._leaseTimeoutMs);
+        const provider = this.Base.ProviderToUse;
+        const schema = provider.MJCoreSchemaName;
+        // MJ pattern: positional parameter array with @p0/@p1/... placeholders.
+        // SQLServerDataProvider binds positional params by index (request.input('p0', val)).
+        // The sproc's own parameters are bound by name via `@SprocParam=@p0` syntax.
+        // See packages/SchemaEngine/src/RuntimeSchemaManager.ts for the canonical example.
+        const rows = await provider.ExecuteSQL(`EXEC [${schema}].[spAcquireScheduledJobLock] ` +
+            `@JobID=@p0, @Token=@p1, @Instance=@p2, @ExpectedCompletionAt=@p3`, [jobId, token, instance, expectedCompletion], { isMutation: true, description: 'spAcquireScheduledJobLock' }, this.Base.ContextUser);
+        const acquired = rows?.[0]?.Acquired === 1;
+        return acquired ? { acquired: true, token } : { acquired: false };
+    }
+    /**
+     * Atomically release a lock IF AND ONLY IF the current DB token matches
+     * expectedToken. Prevents the lost-mutex hazard under lease-expiry races:
+     * if a stale holder's execution eventually settles after the lease was
+     * reclaimed by a fresh holder, this no-ops (token mismatch).
+     *
+     * Idempotent — safe to call on an already-released lock (returns false).
+     *
+     * @returns true if released, false if token mismatch / already released
+     * @private
+     */
+    async releaseLockIfTokenMatches(jobId, expectedToken) {
+        const provider = this.Base.ProviderToUse;
+        const schema = provider.MJCoreSchemaName;
+        // MJ pattern: positional placeholders; see tryAcquireLock for rationale.
+        const rows = await provider.ExecuteSQL(`EXEC [${schema}].[spReleaseScheduledJobLockIfTokenMatches] ` +
+            `@JobID=@p0, @ExpectedToken=@p1`, [jobId, expectedToken], { isMutation: true, description: 'spReleaseScheduledJobLockIfTokenMatches' }, this.Base.ContextUser);
+        const released = rows?.[0]?.Released === 1;
+        if (!released) {
+            this.log(`Lock for job ${jobId.substring(0, 8)} was not released ` +
+                `(token mismatch — reclaimed by another holder, or already released)`);
         }
-        return this.attemptLockAcquisition(job);
+        return released;
     }
     /**
-     * Attempt to acquire a fresh lock on a job that is currently unlocked.
-     * Separated from tryAcquireLock to keep the stale-cleanup and acquisition
-     * paths distinct and easier to follow.
+     * Pre-flight: verify EXECUTE permission on lock sprocs. Fails LOUDLY at boot
+     * if the engine's DB principal lacks grants — much better than a silent
+     * runtime failure the next time a job tries to dispatch.
+     *
+     * Wrapped in try/catch: probe failure (e.g., non-SQL-Server provider where
+     * `sys.fn_my_permissions` doesn't exist) must NOT crash boot. We log and
+     * continue; any actual permission issue will surface at first sproc call.
+     *
      * @private
      */
-    async attemptLockAcquisition(job) {
-        const lockToken = this.generateGuid();
-        const instanceId = this.getInstanceIdentifier();
-        const expectedCompletion = new Date(Date.now() + 10 * 60 * 1000);
+    async probeLockSprocPermissions() {
         try {
-            // Reload to get latest DB state right before acquiring
-            await job.Load(job.ID);
-            if (job.LockToken != null) {
-                console.log(`      ❌ Lock was acquired by another process during reload`);
-                return false;
-            }
-            job.LockToken = lockToken;
-            job.LockedAt = new Date();
-            job.LockedByInstance = instanceId;
-            job.ExpectedCompletionAt = expectedCompletion;
-            console.log(`      → Attempting to save with lock: ${lockToken.substring(0, 8)}...`);
-            const saveResult = await job.Save();
-            if (saveResult) {
-                console.log(`      ✅ Lock acquired successfully!`);
-                return true;
+            const provider = this.Base.ProviderToUse;
+            const schema = provider.MJCoreSchemaName;
+            const sql = `SELECT permission_name FROM sys.fn_my_permissions(` +
+                `'${schema}.spAcquireScheduledJobLock', 'OBJECT') WHERE permission_name = 'EXECUTE'`;
+            const rows = await provider.ExecuteSQL(sql, [], { isMutation: false, description: 'Scheduling engine permission probe' }, this.Base.ContextUser);
+            if (!rows || rows.length === 0) {
+                this.logError(`⚠️  Scheduling engine DB principal lacks EXECUTE on ` +
+                    `${schema}.spAcquireScheduledJobLock. Job dispatch WILL fail. ` +
+                    `Grant cdp_Developer or cdp_Integration role to the principal and restart.`, null);
             }
             else {
-                console.log(`      ❌ Save failed: ${job.LatestResult?.CompleteMessage ?? 'unknown'}`);
-                this.clearInMemoryLockFields(job);
-                return false;
+                this.log(`Lock sproc permission check OK`);
             }
         }
-        catch (error) {
-            this.logError(`Failed to acquire lock for job ${job.Name}`, error);
-            this.clearInMemoryLockFields(job);
-            return false;
+        catch (probeError) {
+            // Probe itself failed (non-SQL-Server provider, or unexpected error).
+            // Don't crash boot. Log and continue.
+            this.log(`Permission probe skipped (provider may not support sys.fn_my_permissions): ${probeError}`);
         }
     }
     /**
-     * Clear in-memory lock fields without saving — used when a save attempt
-     * fails and we need the in-memory object to reflect "unlocked" so the
-     * next poll cycle doesn't see a phantom lock.
+     * Sweep stale inflight jobs. Runs unconditionally at top of every poll.
+     *
+     * SINGLE BATCH QUERY (not N round-trips). Returns only jobs whose lease has
+     * expired OR whose lock has already been cleared. In steady-state (no zombies)
+     * the query matches zero rows and the sweep is essentially free.
+     *
+     * For each stale entry:
+     *   - Untrack the leaked promise from inflightJobPromises (frees cap slot).
+     *   - FIRE-AND-FORGET abandon any orphaned `Status='Running'` run records.
+     *     NOT awaited because cleanup must not delay dispatch under a fleet-wide
+     *     hang event where the sweep finds many zombies at once.
+     *
+     * Decoupled from:
+     *   - isJobDue — irrelevant; we care about lease state, not cron.
+     *   - MaxConcurrentJobs — the sweep IS what frees the cap when saturated by hangs.
+     *
+     * See plans/scheduled-job-engine-decoupling.md for the full rationale.
+     *
+     * @returns count of inflight entries swept
      * @private
      */
-    clearInMemoryLockFields(job) {
-        job.LockToken = null;
-        job.LockedAt = null;
-        job.LockedByInstance = null;
-        job.ExpectedCompletionAt = null;
+    async sweepStaleInflightJobs(contextUser) {
+        if (this.inflightJobPromises.size === 0)
+            return 0;
+        const trackedIds = [...this.inflightJobPromises.keys()];
+        // ID values interpolated below are engine-generated GUIDs from
+        // this.inflightJobPromises keys (originally from this.ScheduledJobs[].ID),
+        // never user input. No SQL-injection vector. RunViewParams.ExtraFilter
+        // does not support parameterized binding in current MJCore.
+        const idList = trackedIds.map(id => `'${id}'`).join(',');
+        const nowIso = new Date().toISOString();
+        const rv = new RunView(this.Base.RunViewProviderToUse);
+        const result = await rv.RunView({
+            EntityName: 'MJ: Scheduled Jobs',
+            ExtraFilter: `ID IN (${idList}) AND (LockToken IS NULL OR ExpectedCompletionAt IS NULL OR ExpectedCompletionAt < '${nowIso}')`,
+            Fields: ['ID', 'Name', 'ExpectedCompletionAt'],
+            ResultType: 'simple',
+        }, contextUser);
+        if (!result.Success || result.Results.length === 0)
+            return 0;
+        let swept = 0;
+        for (const row of result.Results) {
+            const jobName = row.Name ?? row.ID;
+            this.log(`[sweep] Untracking inflight job ${jobName}: ` +
+                `lease=${row.ExpectedCompletionAt?.toISOString() ?? 'NULL'}, now=${nowIso}. ` +
+                `Original execution presumed hung. See README "Leaked promise behavior".`);
+            this.inflightJobPromises.delete(row.ID);
+            swept++;
+            // FIRE-AND-FORGET: cleanup must not block dispatch.
+            this.abandonOrphanedRunRecords(row.ID, contextUser).catch(err => this.logError(`Background abandon-orphaned-runs failed for ${row.ID}`, err));
+        }
+        return swept;
     }
     /**
-     * Release a lock after job execution
+     * Mark any Running run records for the given job as Failed/abandoned.
+     *
+     * IMPORTANT: the `Status='Running'` filter is LOAD-BEARING — not just for
+     * finding zombies. It also protects against a sweep/release race:
+     *
+     *   - Job completes normally.
+     *   - executeJobWithLock's finally calls releaseLockIfTokenMatches (clears LockToken).
+     *   - BEFORE that completes, a poll's sweep query sees LockToken IS NULL
+     *     and classifies the just-completed job as a zombie.
+     *   - But its run record is already Status='Completed' (set inside the try block,
+     *     before the finally), so THIS FILTER excludes it from abandonment.
+     *
+     * Removing or relaxing this filter would corrupt completed run records.
+     * If "optimizing" this method, preserve the Status='Running' filter.
+     *
      * @private
      */
-    async releaseLock(job) {
-        try {
-            job.LockToken = null;
-            job.LockedAt = null;
-            job.LockedByInstance = null;
-            job.ExpectedCompletionAt = null;
-            const saved = await job.Save();
+    async abandonOrphanedRunRecords(jobId, contextUser) {
+        // jobId is an engine-supplied GUID from the sweep's RunView result row,
+        // not user input. No SQL-injection vector.
+        const rv = new RunView(this.Base.RunViewProviderToUse);
+        const result = await rv.RunView({
+            EntityName: 'MJ: Scheduled Job Runs',
+            ExtraFilter: `ScheduledJobID='${jobId}' AND Status='Running'`,
+            ResultType: 'entity_object',
+        }, contextUser);
+        if (!result.Success || result.Results.length === 0)
+            return;
+        const now = new Date();
+        for (const run of result.Results) {
+            run.CompletedAt = now;
+            run.Status = 'Failed';
+            run.Success = false;
+            run.ErrorMessage =
+                `Execution abandoned by scheduling engine sweep: lease (started ` +
+                    `${run.StartedAt?.toISOString()}) expired and the original execution ` +
+                    `never settled. The hung promise was untracked so its concurrency slot ` +
+                    `could be reused. See packages/Scheduling/engine/README.md ` +
+                    `"Leaked promise behavior" for details.`;
+            const saved = await run.Save();
             if (!saved) {
-                this.logError(`Failed to release lock for job ${job.Name}: ${job.LatestResult?.CompleteMessage ?? 'Save returned false'}`);
+                this.logError(`Failed to abandon orphaned run ${run.ID}: ` +
+                    `${run.LatestResult?.CompleteMessage ?? 'unknown'}`, null);
+            }
+            else {
+                this.log(`[sweep] Abandoned orphaned run ${run.ID} for job ${jobId}`);
             }
-            return saved;
-        }
-        catch (error) {
-            this.logError(`Failed to release lock for job ${job.Name}`, error);
-            return false;
         }
     }
-    /**
-     * Clean up a stale lock. Returns true if the lock was successfully
-     * cleared in the database, false if the save failed.
-     * @private
-     */
-    async cleanupStaleLock(job) {
-        this.log(`Cleaning up stale lock on job ${job.Name} (locked by ${job.LockedByInstance})`);
-        return this.releaseLock(job);
-    }
     /**
      * Create a queued job run for later execution
      * @private
@@ -651,25 +1019,33 @@ export class SchedulingEngine extends BaseSingleton {
         return `${os.hostname()}-${process.pid}`;
     }
     /**
-     * Generate a GUID for lock tokens
-     * @private
-     */
-    generateGuid() {
-        return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
-            const r = (Math.random() * 16) | 0;
-            const v = c === 'x' ? r : (r & 0x3) | 0x8;
-            return v.toString(16);
-        });
-    }
-    /**
-     * Initialize NextRunAt for jobs that don't have it set
+     * Initialize NextRunAt for jobs that don't have it set.
+     *
+     * If a job has `RunImmediatelyIfNeverRun = true` AND has never run
+     * (`LastRunAt IS NULL`), `NextRunAt` is set to `now()` so the job
+     * executes on the next polling cycle instead of waiting for the next
+     * cron tick. Useful for freshly-seeded jobs that should not wait up
+     * to a full cron interval (e.g. 24h for a daily job) for their first run.
+     *
      * @private
      */
     async initializeNextRunTimes(contextUser) {
         for (const job of this.ScheduledJobs) {
             if (!job.NextRunAt) {
-                job.NextRunAt = CronExpressionHelper.GetNextRunTime(job.CronExpression, job.Timezone);
+                if (job.RunImmediatelyIfNeverRun && !job.LastRunAt) {
+                    job.NextRunAt = new Date();
+                    console.log(`  ⏱️  Job ${job.Name} flagged RunImmediatelyIfNeverRun — scheduling for immediate execution`);
+                }
+                else {
+                    job.NextRunAt = CronExpressionHelper.GetNextRunTime(job.CronExpression, job.Timezone);
+                }
                 try {
+                    // SAFE: this Save runs in StartPolling's upfront block, BEFORE
+                    // isPolling=true is set. No locks can have been acquired yet,
+                    // so the full-entity Save cannot clobber any live lock state.
+                    // If you ever move this call site outside the upfront block,
+                    // refactor to a targeted UPDATE sproc — see
+                    // updateJobStatistics for the pattern.
                     await job.Save();
                     console.log(`  ⚙️  Initialized NextRunAt for ${job.Name} -> ${job.NextRunAt.toISOString()}`);
                 }
@@ -680,56 +1056,62 @@ export class SchedulingEngine extends BaseSingleton {
         }
     }
     /**
-     * Clean up stale locks on startup
+     * Clean up stale locks on startup using atomic sprocs.
+     *
+     * For each job whose DB shows a stale lock (ExpectedCompletionAt < now OR
+     * ExpectedCompletionAt IS NULL while LockToken IS NOT NULL):
+     *   1. Atomically acquire the stale lock with a fresh token (sproc's WHERE
+     *      handles the stale-detection in a single statement).
+     *   2. Immediately release it with that same token.
+     *
+     * Net effect: stale lock cleared atomically with zero TOCTOU window. Uses
+     * the new sproc-backed pattern instead of load-compare-save on shared
+     * this.ScheduledJobs entities (see plans/scheduled-job-engine-decoupling.md
+     * for why the old pattern was unsafe once polling became concurrent).
+     *
      * @private
      */
-    async cleanupStaleLocks(contextUser) {
+    async cleanupStaleLocks(_contextUser) {
         const now = new Date();
-        let cleanedCount = 0;
         console.log(`  🔍 Checking for stale locks (current time: ${now.toISOString()})...`);
-        for (const job of this.ScheduledJobs) {
-            if (job.LockToken) {
-                console.log(`    Job "${job.Name}": LockToken=${job.LockToken?.substring(0, 8)}..., ExpectedCompletionAt=${job.ExpectedCompletionAt?.toISOString() || 'NULL'}`);
-                if (job.ExpectedCompletionAt) {
-                    const isStale = job.ExpectedCompletionAt < now;
-                    console.log(`      → Is stale? ${isStale} (${job.ExpectedCompletionAt.getTime()} < ${now.getTime()} = ${job.ExpectedCompletionAt.getTime() < now.getTime()})`);
-                    if (isStale) {
-                        console.log(`      🔓 Cleaning stale lock (locked by ${job.LockedByInstance})`);
-                        job.LockToken = null;
-                        job.LockedAt = null;
-                        job.LockedByInstance = null;
-                        job.ExpectedCompletionAt = null;
-                        try {
-                            await job.Save();
-                            cleanedCount++;
-                        }
-                        catch (error) {
-                            this.logError(`Failed to clean stale lock for job ${job.Name}`, error);
-                        }
-                    }
+        // Use the engine's already-loaded cache instead of round-tripping to
+        // the DB — this method runs in StartPolling's upfront block, IMMEDIATELY
+        // after Config() loads this.Base.ScheduledJobs, so the cached lock-column
+        // values are current (no stale-state risk that the sweep path has). Saves
+        // one query + silences the "already loaded by SchedulingEngineBase"
+        // telemetry warning.
+        //
+        // Caveat: only safe HERE because the cache was just loaded. The sweep
+        // path (sweepStaleInflightJobs) MUST hit the DB because by then the
+        // cache's lock columns are stale (atomic sprocs bypass the entity cache).
+        const stale = this.Base.ScheduledJobs.filter(job => job.LockToken != null &&
+            (job.ExpectedCompletionAt == null || job.ExpectedCompletionAt < now));
+        if (stale.length === 0) {
+            console.log(`  ✓ No stale locks found`);
+            return;
+        }
+        let cleanedCount = 0;
+        for (const job of stale) {
+            try {
+                // Atomic reclaim: acquire returns Acquired=1 if the lock was
+                // stale (per the sproc's WHERE clause). Then immediately release
+                // with the new token to leave the lock free.
+                const lockResult = await this.tryAcquireLock(job.ID);
+                if (lockResult.acquired) {
+                    await this.releaseLockIfTokenMatches(job.ID, lockResult.token);
+                    console.log(`    🔓 Cleared stale lock on "${job.Name}" (was held by ${job.LockedByInstance})`);
+                    cleanedCount++;
                 }
                 else {
-                    console.log(`      ⚠️  Lock exists but no ExpectedCompletionAt - clearing anyway`);
-                    job.LockToken = null;
-                    job.LockedAt = null;
-                    job.LockedByInstance = null;
-                    job.ExpectedCompletionAt = null;
-                    try {
-                        await job.Save();
-                        cleanedCount++;
-                    }
-                    catch (error) {
-                        this.logError(`Failed to clean stale lock for job ${job.Name}`, error);
-                    }
+                    // Another instance acquired between our cache load and acquire.
+                    console.log(`    ℹ️  Stale lock on "${job.Name}" was cleared by another holder`);
                 }
             }
+            catch (error) {
+                this.logError(`Failed to clean stale lock for job ${job.Name}`, error);
+            }
         }
-        if (cleanedCount > 0) {
-            console.log(`  ✅ Cleaned ${cleanedCount} stale lock(s)`);
-        }
-        else {
-            console.log(`  ✓ No stale locks found`);
-        }
+        console.log(`  ✅ Cleaned ${cleanedCount} stale lock(s)`);
     }
     log(message) {
         LogStatusEx({