nvent 0.5.3 → 0.5.5

package/dist/runtime/nitro/utils/useAwait.js

@@ -10,7 +10,34 @@ import {
   registerTimeAwait,
   resolveTimeAwait
 } from "./awaitPatterns/index.js";
-import { useStoreAdapter, useStreamTopics, useNventLogger } from "#imports";
+import { useStoreAdapter, useStreamTopics, useNventLogger, useRuntimeConfig } from "#imports";
+export function useAwaitDefaults() {
+  try {
+    const config = useRuntimeConfig();
+    const awaitDefaults = config?.nvent?.flow?.awaitDefaults;
+    return {
+      webhookTimeout: awaitDefaults?.webhookTimeout ?? 24 * 60 * 60 * 1e3,
+      // 24 hours
+      eventTimeout: awaitDefaults?.eventTimeout ?? 24 * 60 * 60 * 1e3,
+      // 24 hours
+      timeTimeout: awaitDefaults?.timeTimeout,
+      // undefined by default
+      scheduleTimeout: awaitDefaults?.scheduleTimeout,
+      // undefined by default
+      timeoutAction: awaitDefaults?.timeoutAction ?? "fail"
+    };
+  } catch {
+    return {
+      webhookTimeout: 24 * 60 * 60 * 1e3,
+      // 24 hours
+      eventTimeout: 24 * 60 * 60 * 1e3,
+      // 24 hours
+      timeTimeout: void 0,
+      scheduleTimeout: void 0,
+      timeoutAction: "fail"
+    };
+  }
+}
 export function useAwait() {
   const logger = useNventLogger("await");
   const store = useStoreAdapter();
@@ -62,7 +89,10 @@ export function useAwait() {
       }
       const awaitingSteps = entry.metadata.awaitingSteps || {};
       if (stepName) {
-        return awaitingSteps[stepName] || null;
+        const awaitKeyBefore = `${stepName}:before`;
+        const awaitKeyAfter = `${stepName}:after`;
+        const awaitState = awaitingSteps[awaitKeyBefore] || awaitingSteps[awaitKeyAfter] || awaitingSteps[stepName];
+        return awaitState || null;
       }
       return awaitingSteps;
     },
@@ -78,8 +108,8 @@ export function useAwait() {
      */
     async getAllActiveAwaits(flowName) {
       const activeAwaits = [];
-      if (!store.indexScan) {
-        logger.warn("Store does not support indexScan");
+      if (!store.index.read) {
+        logger.warn("Store does not support index read");
         return activeAwaits;
       }
       if (flowName) {

package/dist/runtime/nitro/utils/useFlow.js

@@ -20,9 +20,20 @@ export function useFlow() {
       );
       const opts = entryWorker?.queue?.defaultJobOptions || {};
       const flowId = randomUUID();
-      const id = await queueAdapter.enqueue(queueName, { name: flow.entry.step, data: { ...payload, flowId, flowName }, opts });
+      const id = await queueAdapter.enqueue(queueName, {
+        name: flow.entry.step,
+        data: { ...payload, flowId, flowName },
+        opts
+      });
       try {
-        await eventsManager.publishBus({ type: "flow.start", runId: flowId, flowName, data: { input: payload } });
+        await eventsManager.publishBus({
+          type: "flow.start",
+          runId: flowId,
+          flowName,
+          data: {
+            input: payload
+          }
+        });
       } catch {
       }
       return { id, queue: queueName, step: flow.entry.step, flowId };

package/dist/runtime/nitro/utils/useHookRegistry.d.ts

@@ -5,18 +5,24 @@
 export interface LifecycleHooks {
     /**
      * Called when await pattern is registered
-     * @param webhookUrl - Generated webhook URL (for webhook awaits) or event/schedule info
+     * @param hookData - Type-safe data based on await type (webhookUrl, eventName, etc.)
      * @param stepData - Current step data
-     * @param ctx - Worker context
+     * @param ctx - Worker context with awaitType and awaitConfig
      */
-    onAwaitRegister?: (webhookUrl: string, stepData: any, ctx: any) => Promise<void>;
+    onAwaitRegister?: (hookData: any, stepData: any, ctx: any) => Promise<void>;
     /**
      * Called when await pattern is resolved
      * @param resolvedData - Data from the trigger that resolved the await
      * @param stepData - Current step data
-     * @param ctx - Worker context
+     * @param ctx - Worker context with awaitType
      */
     onAwaitResolve?: (resolvedData: any, stepData: any, ctx: any) => Promise<void>;
+    /**
+     * Called when await pattern times out
+     * @param stepData - Current step data
+     * @param ctx - Worker context with awaitType and timeoutAction
+     */
+    onAwaitTimeout?: (stepData: any, ctx: any) => Promise<void>;
 }
 export declare function useHookRegistry(): {
     /**

package/dist/runtime/scheduler/scheduler.d.ts

@@ -44,6 +44,7 @@ export declare class Scheduler implements SchedulerAdapter {
     private lockRenewalTimers;
     private started;
     private logger;
+    private syncSubscription?;
     constructor(options: SchedulerOptions);
     schedule(job: ScheduledJob): Promise<string>;
     /**
@@ -60,6 +61,18 @@ export declare class Scheduler implements SchedulerAdapter {
      * Release distributed lock
      */
     private releaseLock;
+    /**
+     * Poll for new jobs created by other instances
+     * This ensures all instances have the same jobs in-memory
+     *
+     * NOTE: This is a simple but not optimal solution for distributed systems.
+     * TODO: For production multi-instance deployments, consider:
+     * - Pub/Sub notifications (Redis PUBSUB, Postgres NOTIFY/LISTEN)
+     * - Dedicated scheduler instance with queue-based execution
+     * - Service mesh with leader election
+     * See specs/distributed-scheduler.md for detailed architecture
+     */
+    private startJobSyncPolling;
     /**
      * Start periodic lock renewal (for long-running jobs)
      */
@@ -104,6 +117,12 @@ export declare class Scheduler implements SchedulerAdapter {
     start(): Promise<void>;
     stop(): Promise<void>;
     getScheduledJobs(): Promise<ScheduledJob[]>;
+    /**
+     * Get jobs by pattern (e.g., by runId)
+     * This queries the persisted store, not just in-memory jobs
+     * Works across all instances in a distributed setup
+     */
+    getJobsByPattern(pattern: string): Promise<ScheduledJob[]>;
     /**
      * Get all persisted jobs from store (for debugging/monitoring)
      * This shows ALL jobs across ALL instances with their runtime stats

package/dist/runtime/scheduler/scheduler.js

@@ -2,7 +2,8 @@ import { CronJob } from "cron";
 import { getEventBus } from "../events/eventBus.js";
 import { resolveTimeAwait } from "../nitro/utils/awaitPatterns/time.js";
 import { resolveScheduleAwait } from "../nitro/utils/awaitPatterns/schedule.js";
-import { useNventLogger } from "#imports";
+import { useNventLogger, useStoreAdapter, useRuntimeConfig } from "#imports";
+import { createStallDetector } from "../events/utils/stallDetector.js";
 export class Scheduler {
   store;
   keyPrefix;
@@ -14,6 +15,7 @@ export class Scheduler {
   lockRenewalTimers = /* @__PURE__ */ new Map();
   started = false;
   logger = useNventLogger("scheduler");
+  syncSubscription;
   constructor(options) {
     this.store = options.store;
     this.keyPrefix = options.keyPrefix || "nvent:scheduler";
@@ -194,6 +196,54 @@ export class Scheduler {
       await this.store.kv.delete(lockKey);
     }
   }
+  /**
+   * Poll for new jobs created by other instances
+   * This ensures all instances have the same jobs in-memory
+   *
+   * NOTE: This is a simple but not optimal solution for distributed systems.
+   * TODO: For production multi-instance deployments, consider:
+   * - Pub/Sub notifications (Redis PUBSUB, Postgres NOTIFY/LISTEN)
+   * - Dedicated scheduler instance with queue-based execution
+   * - Service mesh with leader election
+   * See specs/distributed-scheduler.md for detailed architecture
+   */
+  startJobSyncPolling() {
+    const pollInterval = setInterval(async () => {
+      if (!this.started) {
+        clearInterval(pollInterval);
+        return;
+      }
+      try {
+        if (!this.store.index.read) {
+          return;
+        }
+        const jobIndex = `${this.keyPrefix}:jobs`;
+        const entries = await this.store.index.read(jobIndex, { limit: 1e4 });
+        let syncedCount = 0;
+        for (const entry of entries) {
+          const jobData = entry.metadata;
+          if (jobData && !this.jobs.has(jobData.id) && jobData.enabled !== false) {
+            this.logger.debug("Syncing job from another instance", {
+              jobId: jobData.id
+            });
+            await this.recoverJob(jobData);
+            syncedCount++;
+          }
+        }
+        if (syncedCount > 0) {
+          this.logger.info("Synced jobs from other instances", {
+            count: syncedCount,
+            totalJobs: this.jobs.size
+          });
+        }
+      } catch (error) {
+        this.logger.error("Error during job sync polling", {
+          error: error.message
+        });
+      }
+    }, 3e4);
+    this.syncSubscription = () => clearInterval(pollInterval);
+  }
   /**
    * Start periodic lock renewal (for long-running jobs)
    */
@@ -350,6 +400,12 @@ export class Scheduler {
         this.logger.debug("Skipping disabled job", { jobId: jobData.id });
         return;
       }
+      if (jobData.executeAt && typeof jobData.executeAt === "string") {
+        jobData.executeAt = new Date(jobData.executeAt).getTime();
+      }
+      if (jobData.interval && typeof jobData.interval === "string") {
+        jobData.interval = Number(jobData.interval);
+      }
       if (jobData.metadata?.type === "schedule-trigger" && jobData.metadata?.triggerName) {
         jobData.handler = async () => {
           const logger = useNventLogger("scheduler");
@@ -442,6 +498,19 @@ export class Scheduler {
           flowName,
           runId
         });
+      } else if (jobData.metadata?.component === "stall-detector") {
+        const { flowName, runId } = jobData.metadata;
+        jobData.handler = async () => {
+          const store = useStoreAdapter();
+          const config = useRuntimeConfig();
+          const stallDetector = createStallDetector(store, config.nvent?.flow?.stallDetection);
+          this.logger.info(`Per-flow stall timeout fired for '${flowName}' runId '${runId}'`);
+          await stallDetector.markAsStalled(flowName, runId, "Stall timeout reached");
+        };
+        this.logger.info("Reconstructed stall detector handler", {
+          flowName,
+          runId
+        });
       } else if (!jobData.handler) {
         this.logger.debug("Skipping job - no handler available, waiting for re-registration", {
           jobId: jobData.id
@@ -472,9 +541,85 @@ export class Scheduler {
         this.jobs.set(jobData.id, intervalId);
         this.logger.info("Recovered interval job", { jobId: jobData.id });
       } else if (jobData.type === "one-time" && jobData.executeAt) {
-        const delay = jobData.executeAt - Date.now();
+        const now = Date.now();
+        const delay = jobData.executeAt - now;
         const isAwaitPattern = jobData.metadata?.component === "await-pattern";
-        if (delay > 0) {
+        const isStallDetector = jobData.metadata?.component === "stall-detector";
+        const awaitType = jobData.metadata?.awaitType;
+        this.logger.debug("Recovering one-time job", {
+          jobId: jobData.id,
+          component: jobData.metadata?.component,
+          awaitType,
+          executeAt: jobData.executeAt,
+          executeAtISO: new Date(jobData.executeAt).toISOString(),
+          now,
+          nowISO: new Date(now).toISOString(),
+          delay,
+          delayHours: (delay / 1e3 / 60 / 60).toFixed(2)
+        });
+        if (isAwaitPattern && (awaitType === "webhook" || awaitType === "event")) {
+          if (delay > 0) {
+            const timeoutId = setTimeout(
+              async () => {
+                await this.executeWithLock(jobData);
+                await this.unschedule(jobData.id);
+              },
+              delay
+            );
+            this.jobs.set(jobData.id, timeoutId);
+            this.logger.info("Recovered webhook/event await timeout", {
+              jobId: jobData.id,
+              awaitType,
+              flowName: jobData.metadata?.flowName,
+              remainingMs: delay
+            });
+          } else {
+            this.logger.warn("Webhook/event await timeout is overdue - flow may be stalled or orphaned", {
+              jobId: jobData.id,
+              awaitType,
+              flowName: jobData.metadata?.flowName,
+              stepName: jobData.metadata?.stepName,
+              runId: jobData.metadata?.runId,
+              scheduledFor: new Date(jobData.executeAt).toISOString(),
+              overdueBy: Math.abs(delay)
+            });
+            await this.unschedule(jobData.id);
+          }
+        } else if (isStallDetector) {
+          if (delay > 0) {
+            const timeoutId = setTimeout(
+              async () => {
+                await this.executeWithLock(jobData);
+                await this.unschedule(jobData.id);
+              },
+              delay
+            );
+            this.jobs.set(jobData.id, timeoutId);
+            this.logger.info("Recovered stall detection timeout", {
+              jobId: jobData.id,
+              flowName: jobData.metadata?.flowName,
+              remainingMs: delay
+            });
+          } else {
+            this.logger.info("Executing overdue stall detection immediately", {
+              jobId: jobData.id,
+              flowName: jobData.metadata?.flowName,
+              runId: jobData.metadata?.runId,
+              overdueBy: Math.abs(delay)
+            });
+            setImmediate(async () => {
+              try {
+                await jobData.handler();
+                await this.unschedule(jobData.id);
+              } catch (error) {
+                this.logger.error("Failed to execute overdue stall detection", {
+                  jobId: jobData.id,
+                  error: error.message
+                });
+              }
+            });
+          }
+        } else if (delay > 0) {
           const timeoutId = setTimeout(
             async () => {
               await this.executeWithLock(jobData);
@@ -483,12 +628,13 @@ export class Scheduler {
             delay
           );
           this.jobs.set(jobData.id, timeoutId);
-          this.logger.info("Recovered one-time job", { jobId: jobData.id });
-        } else if (isAwaitPattern) {
-          this.logger.info("Executing overdue await pattern immediately", {
+          this.logger.info("Recovered one-time job", { jobId: jobData.id, remainingMs: delay });
+        } else if (isAwaitPattern && (awaitType === "time" || awaitType === "schedule")) {
+          this.logger.info("Executing overdue time/schedule await immediately", {
             jobId: jobData.id,
-            awaitType: jobData.metadata?.awaitType,
-            flowName: jobData.metadata?.flowName
+            awaitType,
+            flowName: jobData.metadata?.flowName,
+            overdueBy: Math.abs(delay)
           });
           setImmediate(async () => {
             try {
@@ -541,10 +687,15 @@ export class Scheduler {
     if (this.started) return;
     this.started = true;
     await this.recoverJobs();
+    this.startJobSyncPolling();
     this.logger.info("Started with active jobs", { count: this.jobs.size });
   }
   async stop() {
     this.started = false;
+    if (this.syncSubscription) {
+      this.syncSubscription();
+      this.syncSubscription = void 0;
+    }
     for (const job of this.jobs.values()) {
       if (job instanceof CronJob) {
         job.stop();
@@ -572,6 +723,31 @@ export class Scheduler {
   async getScheduledJobs() {
     return Array.from(this.jobConfigs.values());
   }
+  /**
+   * Get jobs by pattern (e.g., by runId)
+   * This queries the persisted store, not just in-memory jobs
+   * Works across all instances in a distributed setup
+   */
+  async getJobsByPattern(pattern) {
+    const jobs = [];
+    try {
+      if (this.store.index.read) {
+        const jobIndex = `${this.keyPrefix}:jobs`;
+        const entries = await this.store.index.read(jobIndex, { limit: 1e4 });
+        for (const entry of entries) {
+          if (entry.metadata && entry.id.includes(pattern)) {
+            jobs.push(entry.metadata);
+          }
+        }
+      } else {
+        this.logger.warn("Store does not support index read, falling back to in-memory jobs");
+        return Array.from(this.jobConfigs.values()).filter((job) => job.id.includes(pattern));
+      }
+    } catch (error) {
+      this.logger.error("Error getting jobs by pattern", { pattern, error: error.message });
+    }
+    return jobs;
+  }
   /**
    * Get all persisted jobs from store (for debugging/monitoring)
    * This shows ALL jobs across ALL instances with their runtime stats

package/dist/runtime/scheduler/types.d.ts

@@ -90,6 +90,12 @@ export interface SchedulerAdapter {
      * Get all scheduled jobs (in-memory, for this instance)
      */
     getScheduledJobs(): Promise<ScheduledJob[]>;
+    /**
+     * Get jobs matching a pattern (e.g., by runId)
+     * Queries persisted store, works across all instances
+     * More efficient than getAllPersistedJobs() when filtering
+     */
+    getJobsByPattern(pattern: string): Promise<ScheduledJob[]>;
     /**
      * Get all persisted jobs from store (across all instances)
      * Useful for debugging and monitoring in horizontal setups

package/dist/runtime/worker/node/runner.js

@@ -5,13 +5,11 @@ import {
   useEventManager,
   useNventLogger,
   $useFunctionRegistry,
-  useAwait,
-  useHookRegistry,
-  useStreamTopics,
+  $useAnalyzedFlows,
   useStateAdapter,
-  useStoreAdapter,
-  useRunContext
+  useQueueAdapter
 } from "#imports";
+import { SYSTEM_HANDLERS } from "../system/index.js";
 const logger = useNventLogger("node-runner");
 const defaultState = {
   async get() {
@@ -136,49 +134,7 @@ export function createJobProcessor(handler, queueName) {
     const awaitAfter = stepMeta?.awaitAfter;
     const isAwaitResume = job.data?.awaitResolved === true;
     const awaitData = job.data?.awaitData;
-    if (awaitBefore && !isAwaitResume) {
-      const awaitLogger = useNventLogger("await-before");
-      awaitLogger.info("Step has awaitBefore, registering await pattern", {
-        flowName,
-        runId: flowId,
-        stepName: job.name,
-        awaitType: awaitBefore.type
-      });
-      try {
-        const { register } = useAwait();
-        const awaitResult = await register(
-          flowId || "unknown",
-          job.name,
-          flowName,
-          awaitBefore,
-          "before"
-          // Position: awaitBefore means wait before execution
-        );
-        const hookRegistry = useHookRegistry();
-        const hooks = hookRegistry.load(flowName, job.name);
-        if (hooks?.onAwaitRegister) {
-          try {
-            await hooks.onAwaitRegister(
-              awaitResult.webhookUrl || awaitResult.eventName || "",
-              job.data,
-              useRunContext({ flowId, flowName, stepName: job.name })
-            );
-          } catch (err) {
-            awaitLogger.error("onAwaitRegister hook failed", { error: err.message });
-          }
-        }
-        return {
-          awaiting: true,
-          awaitType: awaitBefore.type,
-          awaitConfig: awaitBefore
-        };
-      } catch (err) {
-        awaitLogger.error("Failed to register await pattern", {
-          error: err.message,
-          stack: err.stack
-        });
-      }
-    }
+    const awaitPosition = job.data?.awaitPosition;
     const ctx = buildContext({
       jobId: job.id,
       queue: queueName,
@@ -204,7 +160,11 @@ export function createJobProcessor(handler, queueName) {
         stepName: job.name,
         stepId: stepRunId,
         attempt,
-        data: { jobId: job.id, name: job.name, queue: queueName }
+        data: {
+          jobId: job.id,
+          name: job.name,
+          queue: queueName
+        }
       });
     } catch {
     }
@@ -275,50 +235,31 @@ export function createJobProcessor(handler, queueName) {
       });
     } catch {
     }
-    if (awaitAfter && !isAwaitResume) {
-      const awaitLogger = useNventLogger("await-after");
-      awaitLogger.info("Step has awaitAfter, registering await pattern", {
-        flowName,
-        runId: flowId,
-        stepName: job.name,
-        awaitType: awaitAfter.type
-      });
+    const shouldRegisterAwaitAfter = awaitAfter && (!isAwaitResume || awaitPosition === "before");
+    if (shouldRegisterAwaitAfter) {
       try {
-        const store = useStoreAdapter();
-        const { StoreSubjects } = useStreamTopics();
-        const streamName = StoreSubjects.flowRun(flowId || "unknown");
-        let _emitEvents = [];
-        if (store.stream.read) {
-          const recentEvents = await store.stream.read(streamName, { limit: 100 });
-          _emitEvents = recentEvents.filter(
-            (evt) => evt.type === "emit" && evt.stepName === job.name && evt.stepId === stepRunId
-          );
-        }
-        const { register } = useAwait();
-        const awaitResult = await register(
-          flowId || "unknown",
-          job.name,
-          flowName,
-          awaitAfter,
-          "after"
-        );
-        const hookRegistry = useHookRegistry();
-        const hooks = hookRegistry.load(flowName, job.name);
-        if (hooks?.onAwaitRegister) {
-          try {
-            await hooks.onAwaitRegister(
-              awaitResult.webhookUrl || awaitResult.eventName || "",
-              { ...job.data, result },
-              ctx
-            );
-          } catch (err) {
-            awaitLogger.error("onAwaitRegister hook failed", { error: err.message });
-          }
-        }
+        const queue = useQueueAdapter();
+        const analyzedFlows = $useAnalyzedFlows();
+        const flowDef = analyzedFlows.find((f) => f.id === flowName);
+        const analyzedAwaitStep = flowDef?.analyzed?.steps?.[job.name];
+        const awaitStepTimeout = analyzedAwaitStep?.stepTimeout;
+        await queue.enqueue(queueName, {
+          name: SYSTEM_HANDLERS.AWAIT_REGISTER,
+          data: {
+            flowId: flowId || "unknown",
+            flowName,
+            stepName: job.name,
+            position: "after",
+            awaitConfig: awaitAfter,
+            input: { ...job.data, result }
+          },
+          opts: { jobId: `${flowId}__${job.name}__await-register-after`, timeout: awaitStepTimeout }
+        });
       } catch (err) {
-        awaitLogger.error("Failed to register awaitAfter pattern", {
-          error: err.message,
-          stack: err.stack
+        logger.error("Failed to register awaitAfter pattern", {
+          flowName,
+          stepName: job.name,
+          error: err.message
         });
       }
     }

package/dist/runtime/worker/system/awaitHandlers.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Internal system handlers for await lifecycle events
+ * These handlers run in job context to handle await registration, resolution, and timeout
+ */
+import type { QueueJob } from '../node/runner.js';
+/**
+ * System handler for await registration
+ * Registers await pattern and calls onAwaitRegister hook
+ */
+export declare function awaitRegisterHandler(job: QueueJob): Promise<{
+    success: boolean;
+    awaitResult: any;
+}>;
+/**
+ * System handler for await resolution
+ * Calls onAwaitResolve hook and enqueues the actual step (for awaitBefore)
+ */
+export declare function awaitResolveHandler(job: QueueJob): Promise<{
+    success: boolean;
+}>;
+/**
+ * System handler for await timeout
+ * Calls onAwaitTimeout hook
+ */
+export declare function awaitTimeoutHandler(job: QueueJob): Promise<{
+    success: boolean;
+}>;