nvent 0.5.4 → 0.5.6

package/dist/runtime/scheduler/scheduler.d.ts

@@ -44,6 +44,7 @@ export declare class Scheduler implements SchedulerAdapter {
     private lockRenewalTimers;
     private started;
     private logger;
+    private syncSubscription?;
     constructor(options: SchedulerOptions);
     schedule(job: ScheduledJob): Promise<string>;
     /**
@@ -60,6 +61,18 @@ export declare class Scheduler implements SchedulerAdapter {
      * Release distributed lock
      */
     private releaseLock;
+    /**
+     * Poll for new jobs created by other instances
+     * This ensures all instances have the same jobs in-memory
+     *
+     * NOTE: This is a simple but not optimal solution for distributed systems.
+     * TODO: For production multi-instance deployments, consider:
+     * - Pub/Sub notifications (Redis PUBSUB, Postgres NOTIFY/LISTEN)
+     * - Dedicated scheduler instance with queue-based execution
+     * - Service mesh with leader election
+     * See specs/distributed-scheduler.md for detailed architecture
+     */
+    private startJobSyncPolling;
     /**
      * Start periodic lock renewal (for long-running jobs)
      */
@@ -104,6 +117,12 @@ export declare class Scheduler implements SchedulerAdapter {
     start(): Promise<void>;
     stop(): Promise<void>;
     getScheduledJobs(): Promise<ScheduledJob[]>;
+    /**
+     * Get jobs by pattern (e.g., by runId)
+     * This queries the persisted store, not just in-memory jobs
+     * Works across all instances in a distributed setup
+     */
+    getJobsByPattern(pattern: string): Promise<ScheduledJob[]>;
     /**
      * Get all persisted jobs from store (for debugging/monitoring)
      * This shows ALL jobs across ALL instances with their runtime stats

package/dist/runtime/scheduler/scheduler.js

@@ -2,7 +2,8 @@ import { CronJob } from "cron";
 import { getEventBus } from "../events/eventBus.js";
 import { resolveTimeAwait } from "../nitro/utils/awaitPatterns/time.js";
 import { resolveScheduleAwait } from "../nitro/utils/awaitPatterns/schedule.js";
-import { useNventLogger } from "#imports";
+import { useNventLogger, useStoreAdapter, useRuntimeConfig } from "#imports";
+import { createStallDetector } from "../events/utils/stallDetector.js";
 export class Scheduler {
   store;
   keyPrefix;
@@ -14,6 +15,7 @@ export class Scheduler {
   lockRenewalTimers = /* @__PURE__ */ new Map();
   started = false;
   logger = useNventLogger("scheduler");
+  syncSubscription;
   constructor(options) {
     this.store = options.store;
     this.keyPrefix = options.keyPrefix || "nvent:scheduler";
@@ -194,6 +196,54 @@ export class Scheduler {
       await this.store.kv.delete(lockKey);
     }
   }
+  /**
+   * Poll for new jobs created by other instances
+   * This ensures all instances have the same jobs in-memory
+   *
+   * NOTE: This is a simple but not optimal solution for distributed systems.
+   * TODO: For production multi-instance deployments, consider:
+   * - Pub/Sub notifications (Redis PUBSUB, Postgres NOTIFY/LISTEN)
+   * - Dedicated scheduler instance with queue-based execution
+   * - Service mesh with leader election
+   * See specs/distributed-scheduler.md for detailed architecture
+   */
+  startJobSyncPolling() {
+    const pollInterval = setInterval(async () => {
+      if (!this.started) {
+        clearInterval(pollInterval);
+        return;
+      }
+      try {
+        if (!this.store.index.read) {
+          return;
+        }
+        const jobIndex = `${this.keyPrefix}:jobs`;
+        const entries = await this.store.index.read(jobIndex, { limit: 1e4 });
+        let syncedCount = 0;
+        for (const entry of entries) {
+          const jobData = entry.metadata;
+          if (jobData && !this.jobs.has(jobData.id) && jobData.enabled !== false) {
+            this.logger.debug("Syncing job from another instance", {
+              jobId: jobData.id
+            });
+            await this.recoverJob(jobData);
+            syncedCount++;
+          }
+        }
+        if (syncedCount > 0) {
+          this.logger.info("Synced jobs from other instances", {
+            count: syncedCount,
+            totalJobs: this.jobs.size
+          });
+        }
+      } catch (error) {
+        this.logger.error("Error during job sync polling", {
+          error: error.message
+        });
+      }
+    }, 3e4);
+    this.syncSubscription = () => clearInterval(pollInterval);
+  }
   /**
    * Start periodic lock renewal (for long-running jobs)
    */
@@ -350,6 +400,12 @@ export class Scheduler {
         this.logger.debug("Skipping disabled job", { jobId: jobData.id });
         return;
       }
+      if (jobData.executeAt && typeof jobData.executeAt === "string") {
+        jobData.executeAt = new Date(jobData.executeAt).getTime();
+      }
+      if (jobData.interval && typeof jobData.interval === "string") {
+        jobData.interval = Number(jobData.interval);
+      }
       if (jobData.metadata?.type === "schedule-trigger" && jobData.metadata?.triggerName) {
         jobData.handler = async () => {
           const logger = useNventLogger("scheduler");
@@ -442,6 +498,19 @@ export class Scheduler {
           flowName,
           runId
         });
+      } else if (jobData.metadata?.component === "stall-detector") {
+        const { flowName, runId } = jobData.metadata;
+        jobData.handler = async () => {
+          const store = useStoreAdapter();
+          const config = useRuntimeConfig();
+          const stallDetector = createStallDetector(store, config.nvent?.flow?.stallDetection);
+          this.logger.info(`Per-flow stall timeout fired for '${flowName}' runId '${runId}'`);
+          await stallDetector.markAsStalled(flowName, runId, "Stall timeout reached");
+        };
+        this.logger.info("Reconstructed stall detector handler", {
+          flowName,
+          runId
+        });
       } else if (!jobData.handler) {
         this.logger.debug("Skipping job - no handler available, waiting for re-registration", {
           jobId: jobData.id
@@ -472,9 +541,85 @@ export class Scheduler {
         this.jobs.set(jobData.id, intervalId);
         this.logger.info("Recovered interval job", { jobId: jobData.id });
       } else if (jobData.type === "one-time" && jobData.executeAt) {
-        const delay = jobData.executeAt - Date.now();
+        const now = Date.now();
+        const delay = jobData.executeAt - now;
         const isAwaitPattern = jobData.metadata?.component === "await-pattern";
-        if (delay > 0) {
+        const isStallDetector = jobData.metadata?.component === "stall-detector";
+        const awaitType = jobData.metadata?.awaitType;
+        this.logger.debug("Recovering one-time job", {
+          jobId: jobData.id,
+          component: jobData.metadata?.component,
+          awaitType,
+          executeAt: jobData.executeAt,
+          executeAtISO: new Date(jobData.executeAt).toISOString(),
+          now,
+          nowISO: new Date(now).toISOString(),
+          delay,
+          delayHours: (delay / 1e3 / 60 / 60).toFixed(2)
+        });
+        if (isAwaitPattern && (awaitType === "webhook" || awaitType === "event")) {
+          if (delay > 0) {
+            const timeoutId = setTimeout(
+              async () => {
+                await this.executeWithLock(jobData);
+                await this.unschedule(jobData.id);
+              },
+              delay
+            );
+            this.jobs.set(jobData.id, timeoutId);
+            this.logger.info("Recovered webhook/event await timeout", {
+              jobId: jobData.id,
+              awaitType,
+              flowName: jobData.metadata?.flowName,
+              remainingMs: delay
+            });
+          } else {
+            this.logger.warn("Webhook/event await timeout is overdue - flow may be stalled or orphaned", {
+              jobId: jobData.id,
+              awaitType,
+              flowName: jobData.metadata?.flowName,
+              stepName: jobData.metadata?.stepName,
+              runId: jobData.metadata?.runId,
+              scheduledFor: new Date(jobData.executeAt).toISOString(),
+              overdueBy: Math.abs(delay)
+            });
+            await this.unschedule(jobData.id);
+          }
+        } else if (isStallDetector) {
+          if (delay > 0) {
+            const timeoutId = setTimeout(
+              async () => {
+                await this.executeWithLock(jobData);
+                await this.unschedule(jobData.id);
+              },
+              delay
+            );
+            this.jobs.set(jobData.id, timeoutId);
+            this.logger.info("Recovered stall detection timeout", {
+              jobId: jobData.id,
+              flowName: jobData.metadata?.flowName,
+              remainingMs: delay
+            });
+          } else {
+            this.logger.info("Executing overdue stall detection immediately", {
+              jobId: jobData.id,
+              flowName: jobData.metadata?.flowName,
+              runId: jobData.metadata?.runId,
+              overdueBy: Math.abs(delay)
+            });
+            setImmediate(async () => {
+              try {
+                await jobData.handler();
+                await this.unschedule(jobData.id);
+              } catch (error) {
+                this.logger.error("Failed to execute overdue stall detection", {
+                  jobId: jobData.id,
+                  error: error.message
+                });
+              }
+            });
+          }
+        } else if (delay > 0) {
           const timeoutId = setTimeout(
             async () => {
               await this.executeWithLock(jobData);
@@ -483,12 +628,13 @@ export class Scheduler {
             delay
           );
           this.jobs.set(jobData.id, timeoutId);
-          this.logger.info("Recovered one-time job", { jobId: jobData.id });
-        } else if (isAwaitPattern) {
-          this.logger.info("Executing overdue await pattern immediately", {
+          this.logger.info("Recovered one-time job", { jobId: jobData.id, remainingMs: delay });
+        } else if (isAwaitPattern && (awaitType === "time" || awaitType === "schedule")) {
+          this.logger.info("Executing overdue time/schedule await immediately", {
             jobId: jobData.id,
-            awaitType: jobData.metadata?.awaitType,
-            flowName: jobData.metadata?.flowName
+            awaitType,
+            flowName: jobData.metadata?.flowName,
+            overdueBy: Math.abs(delay)
           });
           setImmediate(async () => {
             try {
@@ -541,10 +687,15 @@ export class Scheduler {
     if (this.started) return;
     this.started = true;
     await this.recoverJobs();
+    this.startJobSyncPolling();
     this.logger.info("Started with active jobs", { count: this.jobs.size });
   }
   async stop() {
     this.started = false;
+    if (this.syncSubscription) {
+      this.syncSubscription();
+      this.syncSubscription = void 0;
+    }
     for (const job of this.jobs.values()) {
       if (job instanceof CronJob) {
         job.stop();
@@ -572,6 +723,31 @@ export class Scheduler {
   async getScheduledJobs() {
     return Array.from(this.jobConfigs.values());
   }
+  /**
+   * Get jobs by pattern (e.g., by runId)
+   * This queries the persisted store, not just in-memory jobs
+   * Works across all instances in a distributed setup
+   */
+  async getJobsByPattern(pattern) {
+    const jobs = [];
+    try {
+      if (this.store.index.read) {
+        const jobIndex = `${this.keyPrefix}:jobs`;
+        const entries = await this.store.index.read(jobIndex, { limit: 1e4 });
+        for (const entry of entries) {
+          if (entry.metadata && entry.id.includes(pattern)) {
+            jobs.push(entry.metadata);
+          }
+        }
+      } else {
+        this.logger.warn("Store does not support index read, falling back to in-memory jobs");
+        return Array.from(this.jobConfigs.values()).filter((job) => job.id.includes(pattern));
+      }
+    } catch (error) {
+      this.logger.error("Error getting jobs by pattern", { pattern, error: error.message });
+    }
+    return jobs;
+  }
   /**
    * Get all persisted jobs from store (for debugging/monitoring)
    * This shows ALL jobs across ALL instances with their runtime stats

package/dist/runtime/scheduler/types.d.ts

@@ -90,6 +90,12 @@ export interface SchedulerAdapter {
      * Get all scheduled jobs (in-memory, for this instance)
      */
     getScheduledJobs(): Promise<ScheduledJob[]>;
+    /**
+     * Get jobs matching a pattern (e.g., by runId)
+     * Queries persisted store, works across all instances
+     * More efficient than getAllPersistedJobs() when filtering
+     */
+    getJobsByPattern(pattern: string): Promise<ScheduledJob[]>;
     /**
      * Get all persisted jobs from store (across all instances)
      * Useful for debugging and monitoring in horizontal setups

package/dist/runtime/worker/node/runner.d.ts

@@ -1,4 +1,4 @@
-import { useFlow } from '#imports';
+import type { FlowStats, StartFlowResult, CancelFlowResult, RunningFlow } from '../../nitro/utils/useFlow.js';
 /**
  * Generic job interface that works with any queue adapter
  * Adapters should provide jobs in this format
@@ -22,6 +22,48 @@ export interface RunState {
     }): Promise<void>;
     delete(key: string): Promise<void>;
 }
+/**
+ * Flow context available within step handlers
+ * Provides context-aware versions of flow operations with auto-injected flowId/flowName
+ */
+export interface RunContextFlow {
+    /** Start a new flow with the given payload */
+    startFlow: (flowName: string, payload?: any) => Promise<StartFlowResult>;
+    /** Emit a trigger event (auto-injects flowId, flowName, stepName from context) */
+    emit: (trigger: string, payload?: any) => Promise<any[]>;
+    /** Cancel a specific flow by name and runId */
+    cancelFlow: (flowName: string, runId: string) => Promise<CancelFlowResult>;
+    /** Cancel the current flow (uses flowId from context) */
+    cancel: () => Promise<CancelFlowResult>;
+    /**
+     * Check if a flow is currently running
+     * @param flowName - Optional flow name (defaults to current flow)
+     * @param runId - Optional specific run ID to check
+     * @param options - Optional configuration (auto-excludes current flow if not specified)
+     */
+    isRunning: (flowName?: string, runId?: string, options?: {
+        excludeRunIds?: string[];
+    }) => Promise<boolean>;
+    /**
+     * Get all currently running flows
+     * @param flowName - Optional flow name (defaults to current flow)
+     * @param options - Optional configuration (auto-excludes current flow if not specified)
+     */
+    getRunningFlows: (flowName?: string, options?: {
+        excludeRunIds?: string[];
+    }) => Promise<RunningFlow[]>;
+    /** Get flow statistics by name */
+    getFlowStats: (flowName: string) => Promise<FlowStats | null>;
+    /** Get all flows with their statistics */
+    getAllFlowStats: (options?: {
+        sortBy?: 'registeredAt' | 'lastRunAt' | 'name';
+        order?: 'asc' | 'desc';
+        limit?: number;
+        offset?: number;
+    }) => Promise<FlowStats[]>;
+    /** Check if a flow has statistics in the index */
+    hasFlowStats: (flowName: string) => Promise<boolean>;
+}
 export interface RunContext {
     jobId?: string;
     queue?: string;
@@ -32,7 +74,7 @@ export interface RunContext {
     attempt?: number;
     logger: RunLogger;
     state: RunState;
-    flow: ReturnType<typeof useFlow>;
+    flow: RunContextFlow;
     /**
      * Resolved data from await pattern (awaitBefore only)
      * Available when step resumes after await resolution

package/dist/runtime/worker/node/runner.js

@@ -5,13 +5,11 @@ import {
   useEventManager,
   useNventLogger,
   $useFunctionRegistry,
-  useAwait,
-  useHookRegistry,
-  useStreamTopics,
+  $useAnalyzedFlows,
   useStateAdapter,
-  useStoreAdapter,
-  useRunContext
+  useQueueAdapter
 } from "#imports";
+import { SYSTEM_HANDLERS } from "../system/index.js";
 const logger = useNventLogger("node-runner");
 const defaultState = {
   async get() {
@@ -54,14 +52,15 @@ export function buildContext(partial) {
       log: (level, msg, meta) => {
         const runId = partial?.flowId || "unknown";
         const flowName = meta?.flowName || "unknown";
+        const metaObj = meta !== null && meta !== void 0 ? typeof meta === "object" && !Array.isArray(meta) ? meta : { value: meta } : {};
         void eventManager.publishBus({
           type: "log",
           runId,
           flowName,
-          stepName: meta?.stepName,
-          stepId: meta?.stepId || meta?.stepRunId,
-          attempt: meta?.attempt,
-          data: { level, message: msg, ...meta }
+          stepName: metaObj?.stepName,
+          stepId: metaObj?.stepId || metaObj?.stepRunId,
+          attempt: metaObj?.attempt,
+          data: { level, message: msg, ...metaObj }
         });
       }
     };
@@ -84,19 +83,21 @@ export function buildContext(partial) {
       }
       return baseFlowEngine.cancelFlow(partial.flowName, partial.flowId);
     },
-    isRunning: async (flowName, runId) => {
+    isRunning: async (flowName, runId, options) => {
       const targetFlowName = flowName || partial?.flowName;
       if (!targetFlowName) {
         throw new Error("flowName is required to check if flow is running");
       }
-      return baseFlowEngine.isRunning(targetFlowName, runId);
+      const effectiveOptions = options || (partial?.flowId ? { excludeRunIds: [partial.flowId] } : void 0);
+      return baseFlowEngine.isRunning(targetFlowName, runId, effectiveOptions);
     },
-    getRunningFlows: async (flowName) => {
+    getRunningFlows: async (flowName, options) => {
       const targetFlowName = flowName || partial?.flowName;
       if (!targetFlowName) {
         throw new Error("flowName is required to get running flows");
       }
-      return baseFlowEngine.getRunningFlows(targetFlowName);
+      const effectiveOptions = options || (partial?.flowId ? { excludeRunIds: [partial.flowId] } : void 0);
+      return baseFlowEngine.getRunningFlows(targetFlowName, effectiveOptions);
     }
   };
   return {
@@ -136,49 +137,7 @@ export function createJobProcessor(handler, queueName) {
     const awaitAfter = stepMeta?.awaitAfter;
     const isAwaitResume = job.data?.awaitResolved === true;
     const awaitData = job.data?.awaitData;
-    if (awaitBefore && !isAwaitResume) {
-      const awaitLogger = useNventLogger("await-before");
-      awaitLogger.info("Step has awaitBefore, registering await pattern", {
-        flowName,
-        runId: flowId,
-        stepName: job.name,
-        awaitType: awaitBefore.type
-      });
-      try {
-        const { register } = useAwait();
-        const awaitResult = await register(
-          flowId || "unknown",
-          job.name,
-          flowName,
-          awaitBefore,
-          "before"
-          // Position: awaitBefore means wait before execution
-        );
-        const hookRegistry = useHookRegistry();
-        const hooks = hookRegistry.load(flowName, job.name);
-        if (hooks?.onAwaitRegister) {
-          try {
-            await hooks.onAwaitRegister(
-              awaitResult.webhookUrl || awaitResult.eventName || "",
-              job.data,
-              useRunContext({ flowId, flowName, stepName: job.name })
-            );
-          } catch (err) {
-            awaitLogger.error("onAwaitRegister hook failed", { error: err.message });
-          }
-        }
-        return {
-          awaiting: true,
-          awaitType: awaitBefore.type,
-          awaitConfig: awaitBefore
-        };
-      } catch (err) {
-        awaitLogger.error("Failed to register await pattern", {
-          error: err.message,
-          stack: err.stack
-        });
-      }
-    }
+    const awaitPosition = job.data?.awaitPosition;
     const ctx = buildContext({
       jobId: job.id,
       queue: queueName,
@@ -192,7 +151,8 @@ export function createJobProcessor(handler, queueName) {
     });
     const attemptLogger = {
       log: (level, msg, meta) => {
-        const enriched = { ...meta || {}, stepName: job.name, attempt, stepRunId, flowName };
+        const metaObj = meta !== null && meta !== void 0 ? typeof meta === "object" && !Array.isArray(meta) ? meta : { value: meta } : {};
+        const enriched = { ...metaObj, stepName: job.name, attempt, stepRunId, flowName };
         ctx.logger.log(level, msg, enriched);
       }
     };
@@ -204,7 +164,11 @@ export function createJobProcessor(handler, queueName) {
         stepName: job.name,
         stepId: stepRunId,
         attempt,
-        data: { jobId: job.id, name: job.name, queue: queueName }
+        data: {
+          jobId: job.id,
+          name: job.name,
+          queue: queueName
+        }
       });
     } catch {
     }
@@ -275,50 +239,31 @@ export function createJobProcessor(handler, queueName) {
       });
     } catch {
     }
-    if (awaitAfter && !isAwaitResume) {
-      const awaitLogger = useNventLogger("await-after");
-      awaitLogger.info("Step has awaitAfter, registering await pattern", {
-        flowName,
-        runId: flowId,
-        stepName: job.name,
-        awaitType: awaitAfter.type
-      });
+    const shouldRegisterAwaitAfter = awaitAfter && (!isAwaitResume || awaitPosition === "before");
+    if (shouldRegisterAwaitAfter) {
       try {
-        const store = useStoreAdapter();
-        const { StoreSubjects } = useStreamTopics();
-        const streamName = StoreSubjects.flowRun(flowId || "unknown");
-        let _emitEvents = [];
-        if (store.stream.read) {
-          const recentEvents = await store.stream.read(streamName, { limit: 100 });
-          _emitEvents = recentEvents.filter(
-            (evt) => evt.type === "emit" && evt.stepName === job.name && evt.stepId === stepRunId
-          );
-        }
-        const { register } = useAwait();
-        const awaitResult = await register(
-          flowId || "unknown",
-          job.name,
-          flowName,
-          awaitAfter,
-          "after"
-        );
-        const hookRegistry = useHookRegistry();
-        const hooks = hookRegistry.load(flowName, job.name);
-        if (hooks?.onAwaitRegister) {
-          try {
-            await hooks.onAwaitRegister(
-              awaitResult.webhookUrl || awaitResult.eventName || "",
-              { ...job.data, result },
-              ctx
-            );
-          } catch (err) {
-            awaitLogger.error("onAwaitRegister hook failed", { error: err.message });
-          }
-        }
+        const queue = useQueueAdapter();
+        const analyzedFlows = $useAnalyzedFlows();
+        const flowDef = analyzedFlows.find((f) => f.id === flowName);
+        const analyzedAwaitStep = flowDef?.analyzed?.steps?.[job.name];
+        const awaitStepTimeout = analyzedAwaitStep?.stepTimeout;
+        await queue.enqueue(queueName, {
+          name: SYSTEM_HANDLERS.AWAIT_REGISTER,
+          data: {
+            flowId: flowId || "unknown",
+            flowName,
+            stepName: job.name,
+            position: "after",
+            awaitConfig: awaitAfter,
+            input: { ...job.data, result }
+          },
+          opts: { jobId: `${flowId}__${job.name}__await-register-after`, timeout: awaitStepTimeout }
+        });
       } catch (err) {
-        awaitLogger.error("Failed to register awaitAfter pattern", {
-          error: err.message,
-          stack: err.stack
+        logger.error("Failed to register awaitAfter pattern", {
+          flowName,
+          stepName: job.name,
+          error: err.message
         });
       }
     }

package/dist/runtime/worker/system/awaitHandlers.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Internal system handlers for await lifecycle events
+ * These handlers run in job context to handle await registration, resolution, and timeout
+ */
+import type { QueueJob } from '../node/runner.js';
+/**
+ * System handler for await registration
+ * Registers await pattern and calls onAwaitRegister hook
+ */
+export declare function awaitRegisterHandler(job: QueueJob): Promise<{
+    success: boolean;
+    awaitResult: any;
+}>;
+/**
+ * System handler for await resolution
+ * Calls onAwaitResolve hook and enqueues the actual step (for awaitBefore)
+ */
+export declare function awaitResolveHandler(job: QueueJob): Promise<{
+    success: boolean;
+}>;
+/**
+ * System handler for await timeout
+ * Calls onAwaitTimeout hook
+ */
+export declare function awaitTimeoutHandler(job: QueueJob): Promise<{
+    success: boolean;
+}>;