nvent 0.4.4 → 0.5.0

package/dist/runtime/scheduler/scheduler.js

@@ -0,0 +1,623 @@
+import { CronJob } from "cron";
+import { getEventBus } from "../events/eventBus.js";
+import { resolveTimeAwait } from "../nitro/utils/awaitPatterns/time.js";
+import { resolveScheduleAwait } from "../nitro/utils/awaitPatterns/schedule.js";
+import { useNventLogger } from "#imports";
+export class Scheduler {
+  store;
+  keyPrefix;
+  lockTTL;
+  instanceId;
+  useIndexLocking;
+  jobs = /* @__PURE__ */ new Map();
+  jobConfigs = /* @__PURE__ */ new Map();
+  lockRenewalTimers = /* @__PURE__ */ new Map();
+  started = false;
+  logger = useNventLogger("scheduler");
+  constructor(options) {
+    this.store = options.store;
+    this.keyPrefix = options.keyPrefix || "nvent:scheduler";
+    this.lockTTL = options.lockTTL || 3e5;
+    this.instanceId = options.instanceId || `instance-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
+    this.useIndexLocking = options.useIndexLocking !== false;
+  }
+  async schedule(job) {
+    this.jobConfigs.set(job.id, job);
+    await this.persistJob(job);
+    if (job.type === "cron" && job.cron) {
+      const cronJob = new CronJob(
+        job.cron,
+        async () => {
+          await this.executeWithLock(job);
+        },
+        null,
+        job.enabled !== false && this.started,
+        job.timezone || "UTC"
+      );
+      this.jobs.set(job.id, cronJob);
+      const nextDate = cronJob.nextDate();
+      if (nextDate) {
+        job.nextRun = nextDate.toMillis();
+        await this.updateJobStats(job.id, { nextRun: job.nextRun });
+      }
+      return job.id;
+    }
+    if (job.type === "interval" && job.interval) {
+      if (job.enabled !== false && this.started) {
+        const intervalId = setInterval(
+          async () => {
+            await this.executeWithLock(job);
+          },
+          job.interval
+        );
+        this.jobs.set(job.id, intervalId);
+      }
+      job.nextRun = Date.now() + job.interval;
+      await this.updateJobStats(job.id, { nextRun: job.nextRun });
+      return job.id;
+    }
+    if (job.type === "one-time" && job.executeAt) {
+      const delay = job.executeAt - Date.now();
+      if (delay > 0) {
+        const timeoutId = setTimeout(
+          async () => {
+            await this.executeWithLock(job);
+            await this.unschedule(job.id);
+          },
+          delay
+        );
+        this.jobs.set(job.id, timeoutId);
+        job.nextRun = job.executeAt;
+        await this.updateJobStats(job.id, { nextRun: job.nextRun });
+      } else {
+        await this.executeWithLock(job);
+      }
+      return job.id;
+    }
+    throw new Error(`Invalid job configuration: ${job.id}`);
+  }
+  /**
+   * Execute job with distributed lock
+   * Prevents multiple instances from running the same job
+   */
+  async executeWithLock(job) {
+    const lockAcquired = await this.acquireLock(job.id);
+    if (!lockAcquired) {
+      return;
+    }
+    try {
+      this.startLockRenewal(job.id);
+      await this.executeJob(job);
+    } catch (error) {
+      this.logger.error("Job failed", { jobId: job.id, error: error.message });
+    } finally {
+      this.stopLockRenewal(job.id);
+      await this.releaseLock(job.id).catch((err) => {
+        this.logger.error("Failed to release lock", { jobId: job.id, error: err.message });
+      });
+    }
+  }
+  async executeJob(job) {
+    const config = this.jobConfigs.get(job.id);
+    if (!config) return;
+    try {
+      const now = Date.now();
+      config.lastRun = now;
+      config.runCount = (config.runCount || 0) + 1;
+      await job.handler();
+      if (job.type === "cron" && job.cron) {
+        const cronJob = this.jobs.get(job.id);
+        if (cronJob instanceof CronJob) {
+          const nextDate = cronJob.nextDate();
+          if (nextDate) {
+            config.nextRun = nextDate.toMillis();
+          }
+        }
+      } else if (job.type === "interval" && job.interval) {
+        config.nextRun = Date.now() + job.interval;
+      }
+      await this.updateJobStats(job.id, {
+        lastRun: now,
+        nextRun: config.nextRun,
+        runCount: config.runCount
+      });
+    } catch (error) {
+      config.failCount = (config.failCount || 0) + 1;
+      await this.updateJobStats(job.id, {
+        failCount: config.failCount,
+        lastError: error instanceof Error ? error.message : String(error)
+      });
+      throw error;
+    }
+  }
+  /**
+   * Acquire distributed lock using store adapter
+   */
+  async acquireLock(jobId) {
+    const lockKey = `${this.keyPrefix}:lock:${jobId}`;
+    const now = Date.now();
+    const expiresAt = now + this.lockTTL;
+    if (this.useIndexLocking && this.store.index.add) {
+      try {
+        const lockIndex = `${this.keyPrefix}:locks`;
+        await this.store.index.add(lockIndex, jobId, expiresAt, {
+          instanceId: this.instanceId,
+          acquiredAt: now,
+          expiresAt
+        });
+        return true;
+      } catch {
+        if (this.store.index.get) {
+          const existing = await this.store.index.get(`${this.keyPrefix}:locks`, jobId);
+          if (existing && existing.score < now) {
+            if (this.store.index.delete) {
+              await this.store.index.delete(`${this.keyPrefix}:locks`, jobId);
+              return this.acquireLock(jobId);
+            }
+          }
+        }
+        return false;
+      }
+    } else {
+      try {
+        const existingLock = await this.store.kv.get(lockKey);
+        if (existingLock) {
+          if (existingLock.expiresAt < now) {
+            await this.store.kv.delete(lockKey);
+            return this.acquireLock(jobId);
+          }
+          return false;
+        }
+        const lock = {
+          jobId,
+          instanceId: this.instanceId,
+          acquiredAt: now,
+          expiresAt
+        };
+        await this.store.kv.set(lockKey, lock, Math.ceil(this.lockTTL / 1e3));
+        return true;
+      } catch (error) {
+        this.logger.error("Error acquiring lock", { jobId, error: error.message });
+        return false;
+      }
+    }
+  }
+  /**
+   * Release distributed lock
+   */
+  async releaseLock(jobId) {
+    if (this.useIndexLocking && this.store.index.delete) {
+      const lockIndex = `${this.keyPrefix}:locks`;
+      await this.store.index.delete(lockIndex, jobId);
+    } else {
+      const lockKey = `${this.keyPrefix}:lock:${jobId}`;
+      await this.store.kv.delete(lockKey);
+    }
+  }
+  /**
+   * Start periodic lock renewal (for long-running jobs)
+   */
+  startLockRenewal(jobId) {
+    const renewalInterval = this.lockTTL / 2;
+    const timer = setInterval(async () => {
+      try {
+        if (this.useIndexLocking && this.store.index.update) {
+          const lockIndex = `${this.keyPrefix}:locks`;
+          const expiresAt = Date.now() + this.lockTTL;
+          await this.store.index.update(lockIndex, jobId, {
+            expiresAt
+          });
+        } else {
+          const lockKey = `${this.keyPrefix}:lock:${jobId}`;
+          const lock = await this.store.kv.get(lockKey);
+          if (lock && lock.instanceId === this.instanceId) {
+            lock.expiresAt = Date.now() + this.lockTTL;
+            await this.store.kv.set(lockKey, lock, Math.ceil(this.lockTTL / 1e3));
+          }
+        }
+      } catch (error) {
+        this.logger.error("Error renewing lock", { jobId, error: error.message });
+      }
+    }, renewalInterval);
+    this.lockRenewalTimers.set(jobId, timer);
+  }
+  /**
+   * Stop lock renewal
+   */
+  stopLockRenewal(jobId) {
+    const timer = this.lockRenewalTimers.get(jobId);
+    if (timer) {
+      clearInterval(timer);
+      this.lockRenewalTimers.delete(jobId);
+    }
+  }
+  /**
+   * Persist job configuration
+   */
+  async persistJob(job) {
+    const jobKey = `${this.keyPrefix}:jobs:${job.id}`;
+    const now = Date.now();
+    const jobData = {
+      id: job.id,
+      name: job.name,
+      type: job.type,
+      cron: job.cron,
+      interval: job.interval,
+      executeAt: job.executeAt,
+      timezone: job.timezone || "UTC",
+      enabled: job.enabled !== false,
+      metadata: job.metadata,
+      persistedAt: now
+    };
+    await this.store.kv.set(jobKey, jobData);
+    if (this.store.index.add) {
+      try {
+        const jobIndex = `${this.keyPrefix}:jobs`;
+        await this.store.index.add(jobIndex, job.id, now, jobData);
+      } catch {
+        if (this.store.index.update) {
+          try {
+            const jobIndex = `${this.keyPrefix}:jobs`;
+            await this.store.index.update(jobIndex, job.id, jobData);
+          } catch (updateError) {
+            this.logger.error("Failed to persist job to index", { jobId: job.id, error: updateError.message });
+          }
+        }
+      }
+    }
+  }
+  /**
+   * Update job statistics
+   */
+  async updateJobStats(jobId, stats) {
+    const statsKey = `${this.keyPrefix}:stats:${jobId}`;
+    try {
+      const existing = await this.store.kv.get(statsKey) || {};
+      await this.store.kv.set(statsKey, {
+        ...existing,
+        ...stats
+      });
+    } catch (error) {
+      this.logger.error("Error updating stats", {
+        jobId,
+        error: error.message
+      });
+    }
+  }
+  async unschedule(jobId) {
+    const job = this.jobs.get(jobId);
+    if (job instanceof CronJob) {
+      job.stop();
+    } else if (job) {
+      clearTimeout(job);
+      clearInterval(job);
+    }
+    this.jobs.delete(jobId);
+    this.jobConfigs.delete(jobId);
+    await this.store.kv.delete(`${this.keyPrefix}:jobs:${jobId}`);
+    await this.store.kv.delete(`${this.keyPrefix}:stats:${jobId}`);
+    if (this.store.index.delete) {
+      try {
+        await this.store.index.delete(`${this.keyPrefix}:jobs`, jobId);
+      } catch {
+      }
+    }
+    await this.releaseLock(jobId);
+    return !!job;
+  }
+  /**
+   * Recover jobs from store on startup
+   * Critical for:
+   * - Restarts: Re-create in-memory schedulers
+   * - Horizontal scaling: New instances pick up existing jobs
+   * - Orphaned awaits: Resume flows waiting on schedules
+   */
+  async recoverJobs() {
+    this.logger.info("Recovering jobs from store");
+    try {
+      if (this.store.index.read) {
+        const jobIndex = `${this.keyPrefix}:jobs`;
+        const jobEntries = await this.store.index.read(jobIndex, { limit: 1e4 });
+        this.logger.info("Found jobs in index", { count: jobEntries.length });
+        for (const entry of jobEntries) {
+          const jobData = entry.metadata;
+          if (jobData) {
+            await this.recoverJob(jobData);
+          }
+        }
+      } else {
+        this.logger.warn("Store does not support index read. Job recovery limited.");
+        this.logger.warn("For full job recovery, use Redis or Postgres store adapter.");
+        await this.recoverWellKnownJobs();
+      }
+    } catch (error) {
+      this.logger.error("Error during job recovery", { error: error.message });
+    }
+  }
+  /**
+   * Recover a single job from persisted data
+   *
+   * IMPORTANT: This method reconstructs in-memory schedulers (CronJob/setInterval)
+   * WITHOUT calling schedule() to avoid re-persisting handler functions (which
+   * cannot be serialized). Handlers are reconstructed from metadata where possible.
+   */
+  async recoverJob(jobData) {
+    try {
+      if (this.jobs.has(jobData.id)) {
+        return;
+      }
+      if (jobData.enabled === false) {
+        this.logger.debug("Skipping disabled job", { jobId: jobData.id });
+        return;
+      }
+      if (jobData.metadata?.type === "schedule-trigger" && jobData.metadata?.triggerName) {
+        jobData.handler = async () => {
+          const logger = useNventLogger("scheduler");
+          const eventBus = getEventBus();
+          const triggerName = jobData.metadata?.triggerName;
+          const scheduleConfig = jobData.metadata?.scheduleConfig;
+          logger.debug("Schedule trigger fired", { trigger: triggerName });
+          await eventBus.publish({
+            type: "trigger.fired",
+            triggerName,
+            data: {
+              scheduledAt: Date.now(),
+              timezone: scheduleConfig?.timezone || "UTC"
+            }
+          });
+        };
+      } else if (jobData.metadata?.component === "await-pattern") {
+        const { awaitType, runId, stepName, flowName, position } = jobData.metadata;
+        if (awaitType === "time") {
+          jobData.handler = async () => {
+            await resolveTimeAwait(runId, stepName, flowName, position, { delayCompleted: true });
+          };
+        } else if (awaitType === "schedule") {
+          jobData.handler = async () => {
+            await resolveScheduleAwait(runId, stepName, flowName, position, { scheduledAt: Date.now() });
+          };
+        } else if (awaitType === "webhook") {
+          jobData.handler = async () => {
+            const eventBus = getEventBus();
+            const timeout = jobData.metadata?.timeout;
+            const timeoutAction = jobData.metadata?.timeoutAction || "fail";
+            this.logger.warn("Webhook await timeout", {
+              runId,
+              stepName,
+              flowName,
+              timeout,
+              timeoutAction
+            });
+            eventBus.publish({
+              type: "await.timeout",
+              flowName,
+              runId,
+              stepName,
+              position,
+              awaitType: "webhook",
+              timeoutAction,
+              data: {
+                timeout,
+                registeredAt: Date.now() - (timeout || 0),
+                timedOutAt: Date.now()
+              }
+            });
+          };
+        } else if (awaitType === "event") {
+          jobData.handler = async () => {
+            const eventBus = getEventBus();
+            const timeout = jobData.metadata?.timeout;
+            const timeoutAction = jobData.metadata?.timeoutAction || "fail";
+            const eventName = jobData.metadata?.eventName;
+            this.logger.warn("Event await timeout", {
+              runId,
+              stepName,
+              flowName,
+              eventName,
+              timeout,
+              timeoutAction
+            });
+            eventBus.publish({
+              type: "await.timeout",
+              flowName,
+              runId,
+              stepName,
+              position,
+              awaitType: "event",
+              timeoutAction,
+              data: {
+                eventName,
+                timeout,
+                registeredAt: Date.now() - (timeout || 0),
+                timedOutAt: Date.now()
+              }
+            });
+          };
+        } else {
+          this.logger.warn("Cannot reconstruct await pattern", { awaitType, jobId: jobData.id });
+          return;
+        }
+        this.logger.info("Reconstructed await pattern handler", {
+          awaitType,
+          flowName,
+          runId
+        });
+      } else if (!jobData.handler) {
+        this.logger.debug("Skipping job - no handler available, waiting for re-registration", {
+          jobId: jobData.id
+        });
+        return;
+      }
+      this.jobConfigs.set(jobData.id, jobData);
+      if (jobData.type === "cron" && jobData.cron) {
+        const cronJob = new CronJob(
+          jobData.cron,
+          async () => {
+            await this.executeWithLock(jobData);
+          },
+          null,
+          true,
+          // Start immediately since scheduler.started is already true
+          jobData.timezone || "UTC"
+        );
+        this.jobs.set(jobData.id, cronJob);
+        this.logger.info("Recovered cron job", { jobId: jobData.id });
+      } else if (jobData.type === "interval" && jobData.interval) {
+        const intervalId = setInterval(
+          async () => {
+            await this.executeWithLock(jobData);
+          },
+          jobData.interval
+        );
+        this.jobs.set(jobData.id, intervalId);
+        this.logger.info("Recovered interval job", { jobId: jobData.id });
+      } else if (jobData.type === "one-time" && jobData.executeAt) {
+        const delay = jobData.executeAt - Date.now();
+        const isAwaitPattern = jobData.metadata?.component === "await-pattern";
+        if (delay > 0) {
+          const timeoutId = setTimeout(
+            async () => {
+              await this.executeWithLock(jobData);
+              await this.unschedule(jobData.id);
+            },
+            delay
+          );
+          this.jobs.set(jobData.id, timeoutId);
+          this.logger.info("Recovered one-time job", { jobId: jobData.id });
+        } else if (isAwaitPattern) {
+          this.logger.info("Executing overdue await pattern immediately", {
+            jobId: jobData.id,
+            awaitType: jobData.metadata?.awaitType,
+            flowName: jobData.metadata?.flowName
+          });
+          setImmediate(async () => {
+            try {
+              await jobData.handler();
+              await this.unschedule(jobData.id);
+            } catch (error) {
+              this.logger.error("Failed to execute overdue await pattern", {
+                jobId: jobData.id,
+                error: error.message
+              });
+            }
+          });
+        } else {
+          this.logger.debug("Skipping expired one-time job", { jobId: jobData.id });
+        }
+      }
+    } catch (error) {
+      this.logger.error("Failed to recover job", {
+        jobId: jobData.id,
+        error: error.message
+      });
+    }
+  }
+  /**
+   * Recover well-known job patterns when index scan is not available
+   * This includes:
+   * - stall-detection (always exists)
+   * - await-time-* (flows waiting on time delays)
+   * - await-schedule-* (flows waiting on cron schedules)
+   */
+  async recoverWellKnownJobs() {
+    const knownPatterns = [
+      "stall-detection"
+      // Note: await-* jobs have dynamic IDs, we can't easily recover them without scanning
+      // This is a limitation of KV-only stores
+    ];
+    for (const jobId of knownPatterns) {
+      try {
+        const jobKey = `${this.keyPrefix}:jobs:${jobId}`;
+        const jobData = await this.store.kv.get(jobKey);
+        if (jobData) {
+          await this.recoverJob(jobData);
+        }
+      } catch {
+      }
+    }
+    this.logger.info("Recovered well-known jobs. Await patterns may need manual recovery.");
+  }
+  async start() {
+    if (this.started) return;
+    this.started = true;
+    await this.recoverJobs();
+    this.logger.info("Started with active jobs", { count: this.jobs.size });
+  }
+  async stop() {
+    this.started = false;
+    for (const job of this.jobs.values()) {
+      if (job instanceof CronJob) {
+        job.stop();
+      } else {
+        clearTimeout(job);
+        clearInterval(job);
+      }
+    }
+    for (const timer of this.lockRenewalTimers.values()) {
+      clearInterval(timer);
+    }
+    this.lockRenewalTimers.clear();
+    if (this.useIndexLocking && this.store.index.read) {
+      const lockIndex = `${this.keyPrefix}:locks`;
+      const locks = await this.store.index.read(lockIndex, { limit: 1e3 });
+      for (const lock of locks) {
+        if (lock.metadata?.instanceId === this.instanceId) {
+          await this.releaseLock(lock.id);
+        }
+      }
+    }
+    this.jobs.clear();
+    this.jobConfigs.clear();
+  }
+  async getScheduledJobs() {
+    return Array.from(this.jobConfigs.values());
+  }
+  /**
+   * Get all persisted jobs from store (for debugging/monitoring)
+   * This shows ALL jobs across ALL instances with their runtime stats
+   */
+  async getAllPersistedJobs() {
+    const jobs = [];
+    try {
+      if (this.store.index.read) {
+        const jobIndex = `${this.keyPrefix}:jobs`;
+        const entries = await this.store.index.read(jobIndex, { limit: 1e4 });
+        for (const entry of entries) {
+          if (entry.metadata) {
+            const jobConfig = entry.metadata;
+            const statsKey = `${this.keyPrefix}:stats:${jobConfig.id}`;
+            try {
+              const stats = await this.store.kv.get(statsKey);
+              if (stats) {
+                jobs.push({
+                  ...jobConfig,
+                  lastRun: stats.lastRun,
+                  nextRun: stats.nextRun,
+                  runCount: stats.runCount,
+                  failCount: stats.failCount
+                });
+              } else {
+                jobs.push(jobConfig);
+              }
+            } catch {
+              jobs.push(jobConfig);
+            }
+          }
+        }
+      } else {
+        this.logger.warn("Cannot list all persisted jobs without index support");
+      }
+    } catch (error) {
+      this.logger.error("Error getting persisted jobs", { error: error.message });
+    }
+    return jobs;
+  }
+  async isHealthy() {
+    try {
+      await this.store.kv.get("__health_check__");
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}