@kb-labs/workflow-engine 2.119.0 → 2.119.1-canary.b53bbee1d

package/dist/index.d.ts

@@ -43,6 +43,30 @@ declare class WorkflowLoader {
     private validate;
 }
 
+/**
+ * How long a daemon's liveness claim on this shared state store stays valid
+ * without a heartbeat renewal. This answers a completely different question
+ * from `RUN_TTL_MS` above: not "is this run's data still around" but "is the
+ * daemon that owns it still alive". `cleanupStaleRuns` uses ONLY this lease —
+ * never a run's or job's own elapsed running time — to decide whether a
+ * previous daemon instance is genuinely gone. A lease still on file this
+ * recently can only mean some process wrote it within the last
+ * DAEMON_LEASE_TTL_MS; if that process isn't the one calling
+ * `cleanupStaleRuns`, force-failing "running"/"queued" jobs would very likely
+ * be abandoning work a still-live sibling instance owns, not cleaning up
+ * after a dead one. Kept well short of RUN_TTL_MS on purpose — a run/job can
+ * legitimately run for tens of minutes, but no legitimate single daemon
+ * instance should ever go this long without renewing its own lease.
+ */
+declare const DAEMON_LEASE_TTL_MS = 45000;
+/** How often a live daemon should call `writeDaemonLease` to keep its lease
+ * fresh — comfortably under `DAEMON_LEASE_TTL_MS` so a normal GC pause or a
+ * slow cache round-trip never lets the lease lapse on its own. */
+declare const DAEMON_LEASE_HEARTBEAT_INTERVAL_MS = 15000;
+interface DaemonLease {
+    instanceId: string;
+    heartbeatAt: string;
+}
 declare class StateStore {
     private readonly logger;
     private readonly cache;
@@ -51,6 +75,22 @@ declare class StateStore {
     getRun(runId: string): Promise<WorkflowRun | null>;
     deleteRun(runId: string): Promise<void>;
     getAllRunIds(): Promise<string[]>;
+    /**
+     * Read the current daemon liveness lease, if one is still on file. Returns
+     * null both when no daemon has ever written one and when the last writer's
+     * lease has expired (per `DAEMON_LEASE_TTL_MS`) — the cache backend itself
+     * enforces that expiry, so a non-null result here is proof some process
+     * heartbeated within the last `DAEMON_LEASE_TTL_MS`.
+     */
+    getDaemonLease(): Promise<DaemonLease | null>;
+    /**
+     * Claim (or renew) the daemon liveness lease under `instanceId`, valid for
+     * `DAEMON_LEASE_TTL_MS`. Called once at startup after `cleanupStaleRuns`
+     * decides this instance is safe to proceed as the active daemon, and then
+     * periodically for as long as the daemon stays up, so its lease never
+     * lapses while it's genuinely alive.
+     */
+    writeDaemonLease(instanceId: string): Promise<void>;
     /**
      * Holds an exclusive per-run lock (see `withLock`) for the whole
      * read-modify-write, so `mutator` runs exactly once per call — no other
@@ -234,11 +274,21 @@ interface WorkflowEngineOptions {
     snapshotManager?: ISnapshotManager;
     /** Workspace root (monorepo root) - used for plugin execution context */
     workspaceRoot?: string;
+    /**
+     * Unique identifier for this daemon/engine process, used by
+     * `cleanupStaleRuns` to tell "genuinely a different daemon instance" apart
+     * from "this same instance calling in again". Defaults to a fresh
+     * `randomUUID()` per engine — override only in tests that need to assert
+     * on a known id or simulate two instances against the same store.
+     */
+    instanceId?: string;
 }
 declare class WorkflowEngine {
     private readonly options;
     readonly loader: WorkflowLoader;
     readonly maxWorkflowDepth: number;
+    /** See `WorkflowEngineOptions.instanceId`. */
+    readonly instanceId: string;
     private readonly logger;
     private readonly analytics?;
     private readonly stateStore;
@@ -372,6 +422,14 @@ declare class WorkflowEngine {
      * Get the scheduler for direct access (used by worker for gate re-enqueue).
      */
     getScheduler(): Scheduler;
+    /**
+     * Renew this instance's daemon liveness lease. Call once cleanupStaleRuns
+     * has decided this instance is safe to proceed, then periodically (see
+     * `DAEMON_LEASE_HEARTBEAT_INTERVAL_MS`) for as long as the daemon stays
+     * up — a live daemon's lease must never lapse, since `cleanupStaleRuns`
+     * treats a lapsed lease as proof its owner is genuinely gone.
+     */
+    renewDaemonLease(): Promise<void>;
     /**
      * Mark stale running/queued jobs as failed on daemon startup — their
      * executor process is gone, so they're unrecoverable. The run itself is
@@ -379,6 +437,20 @@ declare class WorkflowEngine {
      * a run with one abandoned job and one job legitimately parked on a human
      * approval or a child workflow stays 'running' (only the abandoned job is
      * failed) until the parked one resolves.
+     *
+     * Before touching anything, this checks the shared daemon liveness lease
+     * (see `StateStore.getDaemonLease`/`writeDaemonLease`). Being CALLED is not
+     * by itself evidence of a restart — that's exactly what let a stray or
+     * duplicate daemon process (e.g. a second launch racing a daemon that never
+     * actually restarted) force-fail a genuinely in-flight run out from under
+     * a still-alive sibling instance, purely because it, too, executed its own
+     * one-time startup sweep against the same shared state store. A lease still
+     * fresh under a DIFFERENT instanceId is real, recent (within
+     * `DAEMON_LEASE_TTL_MS`) evidence that some other process is currently
+     * alive and heartbeating — in that case this pass must not run at all, no
+     * matter how long any individual job has been "running": elapsed job
+     * runtime is never a substitute for genuine restart evidence, and is not
+     * even consulted here.
      */
     cleanupStaleRuns(): Promise<void>;
     /**
@@ -1058,4 +1130,4 @@ declare class JobManager implements IJobScheduler {
     private jobRecordToHandle;
 }
 
-export { type AcquireOptions, ArtifactMerger, type ArtifactMergerOptions, ConcurrencyManager, type CreateRunInput, type EngineLogger, EnvSecretProvider, type EnvSecretProviderOptions, EventBusBridge, JobManager, type JobManagerConfig, type JobQueueEntry, ManifestConverter, ManifestScanner, type ManifestScannerOptions, type RetryDecision, type RunContext, RunCoordinator, type RunCoordinatorOptions, type RunSnapshot, RunSnapshotStorage, Scheduler, type SchedulerOptions, type SecretProvider, StateStore, type ValidationResult, WorkflowEngine, type WorkflowEngineOptions, type WorkflowEvent, type WorkflowHandlerInfo, type WorkflowListOptions, WorkflowLoader, type WorkflowLoaderOptions, type WorkflowLoaderResult, WorkflowRegistry, type WorkflowRegistryEntry, type WorkflowRegistryOptions, WorkflowRepository, type WorkflowRepositoryOptions, type WorkflowRuntime, type WorkflowSchedule, WorkflowService, type WorkflowServiceListOptions, type WorkflowServiceOptions, type WorkflowStats, type WorkflowTrigger, type WorkflowTriggerType, calculateBackoff, createDefaultSecretProvider, shouldRetry };
+export { type AcquireOptions, ArtifactMerger, type ArtifactMergerOptions, ConcurrencyManager, type CreateRunInput, DAEMON_LEASE_HEARTBEAT_INTERVAL_MS, DAEMON_LEASE_TTL_MS, type DaemonLease, type EngineLogger, EnvSecretProvider, type EnvSecretProviderOptions, EventBusBridge, JobManager, type JobManagerConfig, type JobQueueEntry, ManifestConverter, ManifestScanner, type ManifestScannerOptions, type RetryDecision, type RunContext, RunCoordinator, type RunCoordinatorOptions, type RunSnapshot, RunSnapshotStorage, Scheduler, type SchedulerOptions, type SecretProvider, StateStore, type ValidationResult, WorkflowEngine, type WorkflowEngineOptions, type WorkflowEvent, type WorkflowHandlerInfo, type WorkflowListOptions, WorkflowLoader, type WorkflowLoaderOptions, type WorkflowLoaderResult, WorkflowRegistry, type WorkflowRegistryEntry, type WorkflowRegistryOptions, WorkflowRepository, type WorkflowRepositoryOptions, type WorkflowRuntime, type WorkflowSchedule, WorkflowService, type WorkflowServiceListOptions, type WorkflowServiceOptions, type WorkflowStats, type WorkflowTrigger, type WorkflowTriggerType, calculateBackoff, createDefaultSecretProvider, shouldRetry };

package/dist/index.js

@@ -121,6 +121,9 @@ async function withLock(cache, lockKey, fn) {
 
 // src/state-store.ts
 var RUN_TTL_MS = 24 * 60 * 60 * 1e3;
+var DAEMON_LEASE_TTL_MS = 45e3;
+var DAEMON_LEASE_HEARTBEAT_INTERVAL_MS = 15e3;
+var DAEMON_LEASE_KEY = "kb:daemon:lease";
 var StateStore = class {
   constructor(cache, logger) {
     this.logger = logger;
@@ -159,6 +162,36 @@ var StateStore = class {
     const runIds = await this.cache.zrangebyscore("workflow:runs:index", -Infinity, Infinity);
     return runIds ?? [];
   }
+  /**
+   * Read the current daemon liveness lease, if one is still on file. Returns
+   * null both when no daemon has ever written one and when the last writer's
+   * lease has expired (per `DAEMON_LEASE_TTL_MS`) — the cache backend itself
+   * enforces that expiry, so a non-null result here is proof some process
+   * heartbeated within the last `DAEMON_LEASE_TTL_MS`.
+   */
+  async getDaemonLease() {
+    const raw = await this.cache.get(DAEMON_LEASE_KEY);
+    if (!raw) {
+      return null;
+    }
+    try {
+      return JSON.parse(raw);
+    } catch (error) {
+      this.logger.error("Failed to parse stored daemon lease", error instanceof Error ? error : void 0);
+      return null;
+    }
+  }
+  /**
+   * Claim (or renew) the daemon liveness lease under `instanceId`, valid for
+   * `DAEMON_LEASE_TTL_MS`. Called once at startup after `cleanupStaleRuns`
+   * decides this instance is safe to proceed as the active daemon, and then
+   * periodically for as long as the daemon stays up, so its lease never
+   * lapses while it's genuinely alive.
+   */
+  async writeDaemonLease(instanceId) {
+    const lease = { instanceId, heartbeatAt: (/* @__PURE__ */ new Date()).toISOString() };
+    await this.cache.set(DAEMON_LEASE_KEY, JSON.stringify(lease), DAEMON_LEASE_TTL_MS);
+  }
   /**
    * Holds an exclusive per-run lock (see `withLock`) for the whole
    * read-modify-write, so `mutator` runs exactly once per call — no other
@@ -794,6 +827,7 @@ var WorkflowEngine = class {
     this.options = options;
     this.logger = options.logger;
     this.analytics = options.analytics;
+    this.instanceId = options.instanceId ?? randomUUID();
     this.stateStore = new StateStore(options.cache, this.logger);
     this.concurrency = new ConcurrencyManager(
       options.cache,
@@ -816,6 +850,8 @@ var WorkflowEngine = class {
   options;
   loader;
   maxWorkflowDepth;
+  /** See `WorkflowEngineOptions.instanceId`. */
+  instanceId;
   logger;
   analytics;
   stateStore;
@@ -1523,6 +1559,16 @@ var WorkflowEngine = class {
   getScheduler() {
     return this.scheduler;
   }
+  /**
+   * Renew this instance's daemon liveness lease. Call once cleanupStaleRuns
+   * has decided this instance is safe to proceed, then periodically (see
+   * `DAEMON_LEASE_HEARTBEAT_INTERVAL_MS`) for as long as the daemon stays
+   * up — a live daemon's lease must never lapse, since `cleanupStaleRuns`
+   * treats a lapsed lease as proof its owner is genuinely gone.
+   */
+  async renewDaemonLease() {
+    await this.stateStore.writeDaemonLease(this.instanceId);
+  }
   /**
    * Mark stale running/queued jobs as failed on daemon startup — their
    * executor process is gone, so they're unrecoverable. The run itself is
@@ -1530,8 +1576,36 @@ var WorkflowEngine = class {
    * a run with one abandoned job and one job legitimately parked on a human
    * approval or a child workflow stays 'running' (only the abandoned job is
    * failed) until the parked one resolves.
+   *
+   * Before touching anything, this checks the shared daemon liveness lease
+   * (see `StateStore.getDaemonLease`/`writeDaemonLease`). Being CALLED is not
+   * by itself evidence of a restart — that's exactly what let a stray or
+   * duplicate daemon process (e.g. a second launch racing a daemon that never
+   * actually restarted) force-fail a genuinely in-flight run out from under
+   * a still-alive sibling instance, purely because it, too, executed its own
+   * one-time startup sweep against the same shared state store. A lease still
+   * fresh under a DIFFERENT instanceId is real, recent (within
+   * `DAEMON_LEASE_TTL_MS`) evidence that some other process is currently
+   * alive and heartbeating — in that case this pass must not run at all, no
+   * matter how long any individual job has been "running": elapsed job
+   * runtime is never a substitute for genuine restart evidence, and is not
+   * even consulted here.
    */
   async cleanupStaleRuns() {
+    const priorLease = await this.stateStore.getDaemonLease();
+    if (priorLease && priorLease.instanceId !== this.instanceId) {
+      this.logger.error(
+        "cleanupStaleRuns: a daemon liveness lease held by a different instance is still fresh \u2014 skipping stale-run cleanup entirely to avoid abandoning runs that may genuinely still be in flight under that instance. If no other workflow daemon is actually running, this lease is simply stale past its own restart window and the next daemon start will clean up normally.",
+        void 0,
+        {
+          thisInstanceId: this.instanceId,
+          otherInstanceId: priorLease.instanceId,
+          otherHeartbeatAt: priorLease.heartbeatAt
+        }
+      );
+      return;
+    }
+    await this.stateStore.writeDaemonLease(this.instanceId);
     const runIds = await this.stateStore.getAllRunIds();
     const now = (/* @__PURE__ */ new Date()).toISOString();
     let count = 0;
@@ -3463,6 +3537,6 @@ var JobManager = class {
   }
 };
 
-export { ArtifactMerger, ConcurrencyManager, EnvSecretProvider, EventBusBridge, JobManager, ManifestConverter, ManifestScanner, RunCoordinator, RunSnapshotStorage, Scheduler, StateStore, WorkflowEngine, WorkflowLoader, WorkflowRegistry, WorkflowRepository, WorkflowService, calculateBackoff, createDefaultSecretProvider, shouldRetry };
+export { ArtifactMerger, ConcurrencyManager, DAEMON_LEASE_HEARTBEAT_INTERVAL_MS, DAEMON_LEASE_TTL_MS, EnvSecretProvider, EventBusBridge, JobManager, ManifestConverter, ManifestScanner, RunCoordinator, RunSnapshotStorage, Scheduler, StateStore, WorkflowEngine, WorkflowLoader, WorkflowRegistry, WorkflowRepository, WorkflowService, calculateBackoff, createDefaultSecretProvider, shouldRetry };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map