@cascade-flow/worker 0.1.0

package/dist/index.js

@@ -0,0 +1,841 @@
+// src/step-worker.ts
+import { FileSystemBackend } from "@cascade-flow/backend-filesystem";
+import { getMicrosecondTimestamp as getMicrosecondTimestamp5, MICROSECONDS_PER_MILLISECOND, convertZodToJSONSchema, projectStepState as projectStepState6, serializeError } from "@cascade-flow/backend-interface";
+import { hostname } from "node:os";
+import { join } from "node:path";
+import { discoverSteps, discoverWorkflows, executeStepInProcess, calculateWorkflowHash, getGitInfo } from "@cascade-flow/runner";
+import { Skip } from "@cascade-flow/workflow";
+
+// src/execution/dependency-resolver.ts
+import { projectStepState } from "@cascade-flow/backend-interface";
+import { isOptional } from "@cascade-flow/workflow";
+async function checkDependenciesSatisfied(step, backend, workflowSlug, runId) {
+  for (const [depKey, dep] of Object.entries(step.dependencies)) {
+    const depEvents = await backend.loadEvents(workflowSlug, runId, {
+      category: "step",
+      stepId: dep.id
+    });
+    if (depEvents.length === 0) {
+      const isOptionalDep2 = isOptional(step.dependencies[depKey]);
+      if (!isOptionalDep2) {
+        return false;
+      }
+      continue;
+    }
+    const depState = projectStepState(depEvents, workflowSlug);
+    const isOptionalDep = isOptional(step.dependencies[depKey]);
+    if (depState.status === "skipped" && isOptionalDep) {
+      continue;
+    }
+    if (depState.status !== "completed") {
+      return false;
+    }
+  }
+  return true;
+}
+async function loadDependencyOutputs(step, backend, workflowSlug, runId) {
+  const dependencyOutputs = {};
+  for (const [depKey, depStep] of Object.entries(step.dependencies)) {
+    const depEvents = await backend.loadEvents(workflowSlug, runId, {
+      category: "step",
+      stepId: depStep.id
+    });
+    if (depEvents.length === 0) {
+      const isOptionalDep2 = isOptional(step.dependencies[depKey]);
+      if (!isOptionalDep2) {
+        throw new Error(`Required dependency ${depStep.name} has no events`);
+      }
+      dependencyOutputs[depKey] = undefined;
+      continue;
+    }
+    const depState = projectStepState(depEvents, workflowSlug);
+    const isOptionalDep = isOptional(step.dependencies[depKey]);
+    if (depState.status === "skipped" && isOptionalDep) {
+      dependencyOutputs[depKey] = undefined;
+      continue;
+    }
+    if (depState.status !== "completed" || !depState.output) {
+      throw new Error(`Dependency ${depStep.name} is not completed`);
+    }
+    dependencyOutputs[depKey] = JSON.parse(depState.output);
+  }
+  return dependencyOutputs;
+}
+function findDependentSteps(targetStepId, allSteps, requiredOnly = false) {
+  return allSteps.filter((step) => {
+    const deps = Object.entries(step.dependencies);
+    return deps.some(([alias, dep]) => {
+      const dependsOnTarget = dep.id === targetStepId;
+      if (!dependsOnTarget)
+        return false;
+      if (requiredOnly) {
+        const isOptionalDep = isOptional(step.dependencies[alias]);
+        return !isOptionalDep;
+      }
+      return true;
+    });
+  });
+}
+
+// src/scheduling/skip-handler.ts
+import { getMicrosecondTimestamp, projectStepState as projectStepState2 } from "@cascade-flow/backend-interface";
+async function handleStepSkip(backend, workflowSlug, runId, stepId, skipError, duration, attemptNumber) {
+  await backend.saveStepSkipped(workflowSlug, runId, stepId, {
+    skipType: "primary",
+    reason: skipError.reason || skipError.message.replace("Step skipped: ", ""),
+    metadata: skipError.metadata,
+    duration,
+    attemptNumber
+  });
+}
+async function cascadeSkip(backend, workflowSlug, runId, skippedStepId, allSteps, workerId) {
+  const dependentSteps = findDependentSteps(skippedStepId, allSteps, true);
+  for (const dependentStep of dependentSteps) {
+    const stepEvents = await backend.loadEvents(workflowSlug, runId, {
+      category: "step",
+      stepId: dependentStep.id
+    });
+    const hasEvents = stepEvents.length > 0;
+    if (!hasEvents) {
+      const now = getMicrosecondTimestamp();
+      await backend.saveStepStart(workflowSlug, runId, dependentStep.id, workerId, {
+        dependencies: Object.keys(dependentStep.dependencies),
+        timestamp: now,
+        attemptNumber: 1
+      });
+      await backend.saveStepSkipped(workflowSlug, runId, dependentStep.id, {
+        skipType: "cascade",
+        reason: `Dependency '${skippedStepId}' was skipped`,
+        duration: 0,
+        attemptNumber: 1,
+        cascadedFrom: skippedStepId
+      });
+      await cascadeSkip(backend, workflowSlug, runId, dependentStep.id, allSteps, workerId);
+    } else {
+      const depState = projectStepState2(stepEvents, workflowSlug);
+      if (depState.status !== "completed" && depState.status !== "failed" && depState.status !== "skipped") {
+        await backend.saveStepSkipped(workflowSlug, runId, dependentStep.id, {
+          skipType: "cascade",
+          reason: `Dependency '${skippedStepId}' was skipped`,
+          duration: getMicrosecondTimestamp() - (depState.startTime || getMicrosecondTimestamp()),
+          attemptNumber: depState.attemptNumber,
+          cascadedFrom: skippedStepId
+        });
+        await cascadeSkip(backend, workflowSlug, runId, dependentStep.id, allSteps, workerId);
+      }
+    }
+  }
+}
+
+// src/scheduling/step-scheduler.ts
+import { getMicrosecondTimestamp as getMicrosecondTimestamp2, projectStepState as projectStepState3 } from "@cascade-flow/backend-interface";
+async function scheduleStep(backend, workflowSlug, runId, stepId, reason, attemptNumber, options) {
+  const now = getMicrosecondTimestamp2();
+  await backend.saveStepScheduled(workflowSlug, runId, stepId, {
+    availableAt: options?.availableAt ?? now,
+    reason,
+    attemptNumber,
+    retryDelayMs: options?.retryDelayMs
+  });
+}
+async function scheduleDownstreamSteps(backend, workflowSlug, runId, completedStepId, allSteps) {
+  const scheduledSteps = [];
+  const dependentSteps = findDependentSteps(completedStepId, allSteps, false);
+  for (const dependentStep of dependentSteps) {
+    const allDepsCompleted = await checkDependenciesSatisfied(dependentStep, backend, workflowSlug, runId);
+    if (allDepsCompleted) {
+      const stepEvents = await backend.loadEvents(workflowSlug, runId, {
+        category: "step",
+        stepId: dependentStep.id
+      });
+      if (stepEvents.length === 0) {
+        await scheduleStep(backend, workflowSlug, runId, dependentStep.id, "dependency-satisfied", 1);
+        scheduledSteps.push(dependentStep.name);
+      } else {
+        const state = projectStepState3(stepEvents, workflowSlug);
+        if (state.status === "pending") {
+          await scheduleStep(backend, workflowSlug, runId, dependentStep.id, "dependency-satisfied", 1);
+          scheduledSteps.push(dependentStep.name);
+        }
+      }
+    }
+  }
+  return scheduledSteps;
+}
+
+// src/utils/step-key.ts
+function createStepKey(workflowSlug, runId, stepId) {
+  return `${workflowSlug}/${runId}/${stepId}`;
+}
+function extractStepId(stepKey) {
+  const parts = stepKey.split("/");
+  return parts.slice(2).join("/");
+}
+
+// src/scheduling/workflow-lifecycle.ts
+import { getMicrosecondTimestamp as getMicrosecondTimestamp4, projectStepState as projectStepState5 } from "@cascade-flow/backend-interface";
+
+// src/utils/event-helpers.ts
+import { getMicrosecondTimestamp as getMicrosecondTimestamp3, projectStepState as projectStepState4 } from "@cascade-flow/backend-interface";
+async function loadWorkflowEvents(backend, workflowSlug, runId) {
+  return backend.loadEvents(workflowSlug, runId, { category: "workflow" });
+}
+function getCurrentWorkflowAttemptNumber(events) {
+  const retryEvents = events.filter((e) => e.type === "WorkflowRetryStarted");
+  if (retryEvents.length === 0)
+    return 1;
+  const mostRecentRetry = retryEvents[retryEvents.length - 1];
+  if (!mostRecentRetry)
+    return 1;
+  return mostRecentRetry.workflowAttemptNumber;
+}
+function getWorkflowTerminalState(events) {
+  const currentAttempt = getCurrentWorkflowAttemptNumber(events);
+  return {
+    hasCompleted: events.some((e) => e.type === "WorkflowCompleted" && e.workflowAttemptNumber === currentAttempt),
+    hasFailed: events.some((e) => e.type === "WorkflowFailed" && e.workflowAttemptNumber === currentAttempt),
+    hasCancelled: events.some((e) => e.type === "WorkflowCancelled" && e.workflowAttemptNumber === currentAttempt)
+  };
+}
+function calculateWorkflowDuration(events) {
+  const currentAttempt = getCurrentWorkflowAttemptNumber(events);
+  const workflowStartEvent = events.find((e) => e.type === "WorkflowStarted" && e.workflowAttemptNumber === currentAttempt);
+  const startTime = workflowStartEvent ? workflowStartEvent.timestampUs : 0;
+  const now = getMicrosecondTimestamp3();
+  return now - startTime;
+}
+function hasWorkflowStarted(events) {
+  const currentAttempt = getCurrentWorkflowAttemptNumber(events);
+  return events.some((e) => e.type === "WorkflowStarted" && e.workflowAttemptNumber === currentAttempt);
+}
+function hasWorkflowSubmitted(events) {
+  return events.some((e) => e.type === "RunSubmitted");
+}
+function hasWorkflowValidatedInput(events) {
+  const currentAttempt = getCurrentWorkflowAttemptNumber(events);
+  return events.some((e) => e.type === "WorkflowInputValidation" && e.workflowAttemptNumber === currentAttempt);
+}
+function getRunSubmittedEvent(events) {
+  return events.find((e) => e.type === "RunSubmitted");
+}
+function getWorkflowInputValidationEvent(events) {
+  const validationEvents = events.filter((e) => e.type === "WorkflowInputValidation");
+  return validationEvents.length > 0 ? validationEvents[validationEvents.length - 1] : undefined;
+}
+
+// src/scheduling/workflow-lifecycle.ts
+async function handleWorkflowStart(backend, workflowSlug, runId, metadata, workflowEvents) {
+  const workflowAttemptNumber = getCurrentWorkflowAttemptNumber(workflowEvents);
+  await backend.saveWorkflowStart(workflowSlug, runId, {
+    versionId: metadata.versionId,
+    workflowAttemptNumber,
+    hasInputSchema: metadata.hasInputSchema,
+    hasInput: metadata.hasInput
+  });
+}
+async function validateWorkflowInput(backend, workflowSlug, runId, workflowInput, inputSchema, workflowEvents) {
+  const workflowAttemptNumber = getCurrentWorkflowAttemptNumber(workflowEvents);
+  if (!inputSchema) {
+    if (workflowInput !== undefined) {
+      await backend.saveWorkflowInputValidation(workflowSlug, runId, {
+        workflowAttemptNumber,
+        hasSchema: false,
+        success: true
+      });
+    }
+    return { success: true };
+  }
+  const parseResult = inputSchema.safeParse(workflowInput ?? {});
+  if (!parseResult.success) {
+    const validationErrors = parseResult.error.issues.map((e) => ({
+      path: e.path.join("."),
+      message: e.message
+    }));
+    const errorMessage = validationErrors.map((e) => `  ${e.path}: ${e.message}`).join(`
+`);
+    const error = {
+      name: "ValidationError",
+      message: `Invalid workflow input:
+${errorMessage}`
+    };
+    await backend.saveWorkflowInputValidation(workflowSlug, runId, {
+      workflowAttemptNumber,
+      hasSchema: true,
+      success: false,
+      error,
+      validationErrors
+    });
+    const duration = calculateWorkflowDuration(workflowEvents);
+    await backend.saveWorkflowFailed(workflowSlug, runId, error, {
+      workflowAttemptNumber,
+      duration,
+      completedSteps: 0
+    }, "step-failed");
+    return { success: false, error, validationErrors };
+  }
+  await backend.saveWorkflowInputValidation(workflowSlug, runId, {
+    workflowAttemptNumber,
+    hasSchema: true,
+    success: true
+  });
+  return { success: true };
+}
+async function handleWorkflowCompletion(backend, workflowSlug, runId, steps, workflowEvents) {
+  const workflowAttemptNumber = getCurrentWorkflowAttemptNumber(workflowEvents);
+  const duration = calculateWorkflowDuration(workflowEvents);
+  const now = getMicrosecondTimestamp4();
+  const output = {};
+  for (const step of steps) {
+    const stepEvents = await backend.loadEvents(workflowSlug, runId, {
+      category: "step",
+      stepId: step.id
+    });
+    const state = projectStepState5(stepEvents, workflowSlug);
+    if (state.exportOutput && state.output) {
+      output[step.name] = JSON.parse(state.output);
+    }
+  }
+  await backend.saveWorkflowComplete(workflowSlug, runId, output, {
+    workflowAttemptNumber,
+    timestamp: now,
+    duration,
+    totalSteps: steps.length
+  });
+}
+async function handleWorkflowFailure(backend, workflowSlug, runId, steps, workflowEvents, failedStepName) {
+  const workflowAttemptNumber = getCurrentWorkflowAttemptNumber(workflowEvents);
+  const duration = calculateWorkflowDuration(workflowEvents);
+  let completedCount = 0;
+  for (const step of steps) {
+    const stepEvents = await backend.loadEvents(workflowSlug, runId, {
+      category: "step",
+      stepId: step.id
+    });
+    if (stepEvents.length > 0) {
+      const state = projectStepState5(stepEvents, workflowSlug);
+      if (state.status === "completed") {
+        completedCount++;
+      }
+    }
+  }
+  await backend.saveWorkflowFailed(workflowSlug, runId, {
+    message: failedStepName ? `Step ${failedStepName} failed` : "Workflow failed",
+    name: "StepFailure"
+  }, {
+    workflowAttemptNumber,
+    duration,
+    completedSteps: completedCount,
+    failedStep: failedStepName
+  }, "step-failed");
+}
+function parseWorkflowInput(workflowEvents) {
+  const runSubmittedEvent = getRunSubmittedEvent(workflowEvents);
+  if (runSubmittedEvent && runSubmittedEvent.input) {
+    return JSON.parse(runSubmittedEvent.input);
+  }
+  return;
+}
+
+// src/step-worker.ts
+class StepWorker {
+  workerId;
+  backend;
+  options;
+  running = false;
+  activeSteps = new Map;
+  stats;
+  workflowCache = new Map;
+  constructor(backend, options = {}) {
+    this.workerId = options.workerId || `step-worker-${hostname()}-${process.pid}-${getMicrosecondTimestamp5()}`;
+    this.backend = backend || new FileSystemBackend(options.baseDir || "./.runs");
+    this.options = {
+      workerId: this.workerId,
+      mode: options.mode ?? "unified",
+      concurrency: options.concurrency ?? 1,
+      pollInterval: options.pollInterval ?? 1000,
+      heartbeatInterval: options.heartbeatInterval ?? 5000,
+      staleThreshold: options.staleThreshold ?? 30000,
+      staleCheckInterval: options.staleCheckInterval ?? 1e4,
+      schedulerInterval: options.schedulerInterval ?? 5000,
+      shutdownTimeout: options.shutdownTimeout ?? 30000,
+      workflowsDir: options.workflowsDir || "./workflows",
+      baseDir: options.baseDir || "./.runs"
+    };
+    this.stats = {
+      workerId: this.workerId,
+      mode: this.options.mode,
+      startedAt: getMicrosecondTimestamp5(),
+      totalStepsProcessed: 0,
+      currentlyRunning: 0,
+      failedSteps: 0,
+      reclaimedSteps: 0
+    };
+  }
+  getStats() {
+    return { ...this.stats };
+  }
+  async start() {
+    if (this.running) {
+      throw new Error("Worker is already running");
+    }
+    this.running = true;
+    console.log(`[StepWorker ${this.workerId}] Starting in ${this.options.mode} mode...`);
+    this.setupSignalHandlers();
+    console.log(`[StepWorker ${this.workerId}] Loading workflows from ${this.options.workflowsDir}...`);
+    await this.loadWorkflows();
+    console.log(`[StepWorker ${this.workerId}] Registering workflows with backend...`);
+    await this.registerWorkflows();
+    const loops = [];
+    if (this.options.mode === "unified" || this.options.mode === "executor") {
+      loops.push(this.executorLoop());
+      console.log(`[StepWorker ${this.workerId}] Executor loop started (concurrency: ${this.options.concurrency})`);
+    }
+    if (this.options.mode === "unified" || this.options.mode === "scheduler") {
+      loops.push(this.schedulerLoop());
+      console.log(`[StepWorker ${this.workerId}] Scheduler loop started (interval: ${this.options.schedulerInterval}ms)`);
+      loops.push(this.reclamationLoop());
+      console.log(`[StepWorker ${this.workerId}] Reclamation loop started (threshold: ${this.options.staleThreshold}ms)`);
+    }
+    console.log(`[StepWorker ${this.workerId}] Started successfully`);
+    try {
+      await Promise.all(loops);
+    } finally {
+      clearInterval(this.heartbeatTimer);
+      console.log(`[StepWorker ${this.workerId}] Stopped`);
+    }
+  }
+  async stop() {
+    if (!this.running) {
+      return;
+    }
+    console.log(`[StepWorker ${this.workerId}] Stopping... (${this.activeSteps.size} active steps)`);
+    this.running = false;
+    const shutdownStart = Date.now();
+    while (this.activeSteps.size > 0 && Date.now() - shutdownStart < this.options.shutdownTimeout) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+    if (this.activeSteps.size > 0) {
+      console.warn(`[StepWorker ${this.workerId}] Forced shutdown with ${this.activeSteps.size} active steps`);
+    }
+    console.log(`[StepWorker ${this.workerId}] Stopped`);
+  }
+  async loadWorkflows() {
+    try {
+      const workflows = await discoverWorkflows(this.options.workflowsDir);
+      for (const workflow of workflows) {
+        const steps = await discoverSteps(workflow.stepsDir);
+        this.workflowCache.set(workflow.slug, {
+          metadata: workflow,
+          steps,
+          inputSchema: workflow.inputSchema
+        });
+      }
+      console.log(`[StepWorker ${this.workerId}] Loaded ${workflows.length} workflows`);
+    } catch (error) {
+      console.error(`[StepWorker ${this.workerId}] Failed to load workflows:`, error);
+      throw error;
+    }
+  }
+  async registerWorkflows() {
+    try {
+      for (const [slug, { metadata, steps, inputSchema }] of this.workflowCache) {
+        let inputSchemaJSON = undefined;
+        if (inputSchema) {
+          try {
+            inputSchemaJSON = convertZodToJSONSchema(inputSchema);
+          } catch (error) {
+            console.warn(`[StepWorker ${this.workerId}] Failed to convert schema for workflow ${slug}: ${error}`);
+          }
+        }
+        const stepDefinitions = steps.map((step) => ({
+          id: step.id,
+          name: step.name,
+          dependencies: Object.values(step.dependencies).map((dep) => dep.id),
+          exportOutput: step.exportOutput ?? false
+        }));
+        await this.backend.registerWorkflow({
+          slug,
+          name: metadata.name,
+          location: metadata.dir,
+          inputSchemaJSON,
+          steps: stepDefinitions
+        });
+        const versionId = await calculateWorkflowHash(metadata);
+        const git = await getGitInfo(metadata.dir);
+        const stepManifest = steps.map((s) => s.id);
+        await this.backend.createWorkflowVersion({
+          workflowSlug: slug,
+          versionId,
+          createdAt: getMicrosecondTimestamp5(),
+          stepManifest,
+          totalSteps: steps.length,
+          git
+        });
+      }
+      console.log(`[StepWorker ${this.workerId}] Registered ${this.workflowCache.size} workflows`);
+    } catch (error) {
+      console.error(`[StepWorker ${this.workerId}] Failed to register workflows:`, error);
+      throw error;
+    }
+  }
+  async executorLoop() {
+    while (this.running) {
+      try {
+        if (this.activeSteps.size >= this.options.concurrency) {
+          await new Promise((resolve) => setTimeout(resolve, this.options.pollInterval));
+          continue;
+        }
+        const limit = this.options.concurrency - this.activeSteps.size;
+        const scheduledSteps = await this.backend.listScheduledSteps({ limit });
+        if (scheduledSteps.length === 0) {
+          await new Promise((resolve) => setTimeout(resolve, this.options.pollInterval));
+          continue;
+        }
+        for (const { workflowSlug, runId, stepId } of scheduledSteps) {
+          if (this.activeSteps.size >= this.options.concurrency) {
+            break;
+          }
+          const isClaimable = await this.backend.isStepClaimable(workflowSlug, runId, stepId);
+          if (!isClaimable) {
+            continue;
+          }
+          this.executeStep(workflowSlug, runId, stepId).catch((err) => {
+            console.error(`[StepWorker ${this.workerId}] Error executing ${createStepKey(workflowSlug, runId, stepId)}:`, err);
+          });
+        }
+      } catch (error) {
+        console.error(`[StepWorker ${this.workerId}] Error in executor loop:`, error);
+        await new Promise((resolve) => setTimeout(resolve, this.options.pollInterval));
+      }
+    }
+  }
+  async schedulerLoop() {
+    while (this.running) {
+      try {
+        const activeWorkflows = await this.backend.listActiveWorkflows();
+        for (const workflowSlug of activeWorkflows) {
+          const workflowData = this.workflowCache.get(workflowSlug);
+          if (!workflowData) {
+            console.warn(`[StepWorker ${this.workerId}] Workflow ${workflowSlug} not in cache, skipping`);
+            continue;
+          }
+          let runIds = [];
+          try {
+            runIds = await this.backend.listRunIds(workflowSlug);
+          } catch {
+            continue;
+          }
+          for (const runId of runIds) {
+            try {
+              const workflowEvents = await loadWorkflowEvents(this.backend, workflowSlug, runId);
+              if (workflowEvents.length === 0)
+                continue;
+              const { hasCompleted, hasFailed, hasCancelled } = getWorkflowTerminalState(workflowEvents);
+              if (hasCompleted || hasFailed || hasCancelled) {
+                continue;
+              }
+              const hasSubmitted = hasWorkflowSubmitted(workflowEvents);
+              const hasStarted = hasWorkflowStarted(workflowEvents);
+              if (!hasSubmitted) {
+                continue;
+              }
+              if (!hasStarted) {
+                const runSubmittedEvent = getRunSubmittedEvent(workflowEvents);
+                if (!runSubmittedEvent) {
+                  throw new Error(`No RunSubmitted event found for ${workflowSlug}/${runId}`);
+                }
+                await handleWorkflowStart(this.backend, workflowSlug, runId, {
+                  versionId: runSubmittedEvent.versionId,
+                  hasInputSchema: workflowData.inputSchema !== undefined,
+                  hasInput: !!(runSubmittedEvent && runSubmittedEvent.input !== undefined)
+                }, workflowEvents);
+                console.log(`[StepWorker ${this.workerId}] Started workflow ${workflowSlug}/${runId}`);
+              }
+              const hasValidatedInput = hasWorkflowValidatedInput(workflowEvents);
+              if (!hasValidatedInput) {
+                const workflowInput = parseWorkflowInput(workflowEvents);
+                const validationResult = await validateWorkflowInput(this.backend, workflowSlug, runId, workflowInput, workflowData.inputSchema, workflowEvents);
+                if (!validationResult.success) {
+                  console.log(`[StepWorker ${this.workerId}] Workflow ${workflowSlug}/${runId} failed (input validation)`);
+                  continue;
+                }
+              }
+              let allStepsCompleted = true;
+              let anyStepFailed = false;
+              let failedStepName;
+              for (const step of workflowData.steps) {
+                const stepEvents = await this.backend.loadEvents(workflowSlug, runId, {
+                  category: "step",
+                  stepId: step.id
+                });
+                if (stepEvents.length === 0) {
+                  allStepsCompleted = false;
+                  const canSchedule = await checkDependenciesSatisfied(step, this.backend, workflowSlug, runId);
+                  if (canSchedule) {
+                    await scheduleStep(this.backend, workflowSlug, runId, step.id, Object.keys(step.dependencies).length === 0 ? "initial" : "dependency-satisfied", 1);
+                    console.log(`[StepWorker ${this.workerId}] Scheduled ${step.name} in ${workflowSlug}/${runId}`);
+                  }
+                } else {
+                  const state = projectStepState6(stepEvents, workflowSlug);
+                  if (state.status === "pending") {
+                    allStepsCompleted = false;
+                    const canSchedule = await checkDependenciesSatisfied(step, this.backend, workflowSlug, runId);
+                    if (canSchedule) {
+                      await scheduleStep(this.backend, workflowSlug, runId, step.id, "dependency-satisfied", 1);
+                      console.log(`[StepWorker ${this.workerId}] Scheduled pending step ${step.name} in ${workflowSlug}/${runId}`);
+                    }
+                  } else if (state.status === "failed" && state.terminal) {
+                    anyStepFailed = true;
+                    failedStepName = step.name;
+                    allStepsCompleted = false;
+                  } else if (state.status !== "completed" && state.status !== "skipped") {
+                    allStepsCompleted = false;
+                  }
+                }
+              }
+              if (allStepsCompleted) {
+                await handleWorkflowCompletion(this.backend, workflowSlug, runId, workflowData.steps, workflowEvents);
+                console.log(`[StepWorker ${this.workerId}] Workflow ${workflowSlug}/${runId} completed`);
+              } else if (anyStepFailed) {
+                await handleWorkflowFailure(this.backend, workflowSlug, runId, workflowData.steps, workflowEvents, failedStepName);
+                console.log(`[StepWorker ${this.workerId}] Workflow ${workflowSlug}/${runId} failed (step: ${failedStepName})`);
+              }
+            } catch (error) {
+              console.error(`[StepWorker ${this.workerId}] Error processing run ${runId}:`, error);
+            }
+          }
+        }
+        await new Promise((resolve) => setTimeout(resolve, this.options.schedulerInterval));
+      } catch (error) {
+        console.error(`[StepWorker ${this.workerId}] Error in scheduler loop:`, error);
+        await new Promise((resolve) => setTimeout(resolve, this.options.schedulerInterval));
+      }
+    }
+  }
+  async reclamationLoop() {
+    while (this.running) {
+      try {
+        const reclaimed = await this.backend.reclaimStaleSteps(this.options.staleThreshold * 1000, this.workerId);
+        if (reclaimed.length > 0) {
+          console.log(`[StepWorker ${this.workerId}] Reclaimed ${reclaimed.length} stale steps`);
+          this.stats.reclaimedSteps += reclaimed.length;
+        }
+        await new Promise((resolve) => setTimeout(resolve, this.options.staleCheckInterval));
+      } catch (error) {
+        console.error(`[StepWorker ${this.workerId}] Error in reclamation loop:`, error);
+        await new Promise((resolve) => setTimeout(resolve, this.options.staleCheckInterval));
+      }
+    }
+  }
+  async executeStep(workflowSlug, runId, stepId) {
+    const stepKey = createStepKey(workflowSlug, runId, stepId);
+    let startTime = getMicrosecondTimestamp5();
+    let attemptNumber = null;
+    let heartbeatTimer = null;
+    try {
+      const workflowData = this.workflowCache.get(workflowSlug);
+      if (!workflowData) {
+        throw new Error(`Workflow ${workflowSlug} not found in cache`);
+      }
+      const step = workflowData.steps.find((s) => s.id === stepId);
+      if (!step) {
+        throw new Error(`Step ${stepId} not found in workflow ${workflowSlug}`);
+      }
+      const claimResult = await this.backend.claimScheduledStep(workflowSlug, runId, stepId, this.workerId, {
+        dependencies: Object.keys(step.dependencies),
+        timestamp: startTime,
+        attemptNumber: 1
+      });
+      if (!claimResult) {
+        console.log(`[StepWorker ${this.workerId}] Skipped ${stepKey} (already claimed)`);
+        return;
+      }
+      attemptNumber = claimResult.attemptNumber;
+      startTime = getMicrosecondTimestamp5();
+      heartbeatTimer = setInterval(() => {
+        if (attemptNumber === null)
+          return;
+        this.backend.saveStepHeartbeat(workflowSlug, runId, stepId, this.workerId, attemptNumber).catch((err) => {
+          console.error(`[StepWorker ${this.workerId}] Error sending heartbeat for ${stepKey}:`, err);
+        });
+      }, this.options.heartbeatInterval);
+      const trackedTimer = heartbeatTimer;
+      this.activeSteps.set(stepKey, {
+        workflowSlug,
+        runId,
+        stepId,
+        attemptNumber,
+        heartbeatTimer: trackedTimer
+      });
+      this.stats.currentlyRunning++;
+      console.log(`[StepWorker ${this.workerId}] Executing ${step.name} (attempt ${attemptNumber})`);
+      const dependencyOutputs = await loadDependencyOutputs(step, this.backend, workflowSlug, runId);
+      const workflowEvents = await loadWorkflowEvents(this.backend, workflowSlug, runId);
+      const runSubmittedEvent = getRunSubmittedEvent(workflowEvents);
+      let workflowInput = undefined;
+      if (runSubmittedEvent && runSubmittedEvent.input !== undefined) {
+        workflowInput = JSON.parse(runSubmittedEvent.input);
+      }
+      if (workflowData.inputSchema) {
+        const validationEvent = getWorkflowInputValidationEvent(workflowEvents);
+        if (validationEvent && validationEvent.success) {
+          const parseResult = workflowData.inputSchema.safeParse(workflowInput ?? {});
+          if (parseResult.success) {
+            workflowInput = parseResult.data;
+          }
+        }
+      }
+      const ctx = {
+        runId,
+        workflow: {
+          slug: workflowSlug,
+          name: workflowData.metadata.name
+        },
+        input: workflowInput,
+        log: (...args) => {
+          console.log(`[${step.name}]`, ...args);
+        }
+      };
+      const stepFile = join(step.dir, "step.ts");
+      const submissionTimeoutMs = runSubmittedEvent && runSubmittedEvent.timeoutUs ? Math.floor(runSubmittedEvent.timeoutUs / 1000) : undefined;
+      const timeout = step.timeoutMs ?? submissionTimeoutMs ?? 300000;
+      const abortController = new AbortController;
+      const executionPromise = executeStepInProcess(stepFile, step.id, dependencyOutputs, ctx, attemptNumber, this.backend, async (log) => {
+        const logEvent = {
+          workflowSlug,
+          runId,
+          category: "step",
+          type: "LogEntry",
+          stepId,
+          stream: log.stream,
+          message: log.message,
+          attemptNumber,
+          timestampUs: log.timestamp
+        };
+        await this.backend.appendEvent(workflowSlug, runId, logEvent);
+      }, {
+        signal: abortController.signal
+      });
+      const timeoutTimer = setTimeout(() => {
+        abortController.abort(new Error(`Step timeout after ${timeout}ms`));
+      }, timeout);
+      const { result: output } = await executionPromise.finally(() => {
+        clearTimeout(timeoutTimer);
+      });
+      const endTime = getMicrosecondTimestamp5();
+      const duration = endTime - startTime;
+      await this.backend.saveStepComplete(workflowSlug, runId, stepId, output, {
+        timestamp: endTime,
+        duration,
+        attemptNumber,
+        output
+      }, step.exportOutput ?? false);
+      console.log(`[StepWorker ${this.workerId}] Completed ${step.name}`);
+      if (heartbeatTimer) {
+        clearInterval(heartbeatTimer);
+        heartbeatTimer = null;
+      }
+      this.activeSteps.delete(stepKey);
+      this.stats.currentlyRunning--;
+      this.stats.totalStepsProcessed++;
+      if (this.options.mode === "unified") {
+        const workflowData2 = this.workflowCache.get(workflowSlug);
+        if (workflowData2) {
+          const scheduled = await scheduleDownstreamSteps(this.backend, workflowSlug, runId, stepId, workflowData2.steps);
+          if (scheduled.length > 0) {
+            console.log(`[StepWorker ${this.workerId}] Scheduled downstream steps: ${scheduled.join(", ")}`);
+          }
+        }
+      }
+    } catch (error) {
+      const failureContext = attemptNumber === null ? "Failed to claim" : "Failed";
+      console.error(`[StepWorker ${this.workerId}] ${failureContext} ${stepKey}:`, error);
+      const activeStep = this.activeSteps.get(stepKey);
+      if (activeStep) {
+        clearInterval(activeStep.heartbeatTimer);
+        this.activeSteps.delete(stepKey);
+        this.stats.currentlyRunning--;
+      } else if (heartbeatTimer) {
+        clearInterval(heartbeatTimer);
+        heartbeatTimer = null;
+      }
+      if (attemptNumber === null) {
+        return;
+      }
+      this.stats.failedSteps++;
+      const workflowData = this.workflowCache.get(workflowSlug);
+      if (!workflowData) {
+        console.error(`[StepWorker ${this.workerId}] Workflow ${workflowSlug} not found in cache`);
+        return;
+      }
+      const stepId2 = extractStepId(stepKey);
+      if (!stepId2) {
+        console.error(`[StepWorker ${this.workerId}] Invalid stepKey format: ${stepKey}`);
+        return;
+      }
+      const step = workflowData.steps.find((s) => s.id === stepId2);
+      if (!step) {
+        console.error(`[StepWorker ${this.workerId}] Step ${stepId2} not found in workflow ${workflowSlug}`);
+        return;
+      }
+      const events = await this.backend.loadEvents(workflowSlug, runId, { category: "step", stepId: stepId2 });
+      const state = projectStepState6(events, workflowSlug);
+      const currentAttemptNumber = state.attemptNumber;
+      const endTime = getMicrosecondTimestamp5();
+      const duration = endTime - (state.startTime || startTime);
+      const stepError = serializeError(error);
+      if (error instanceof Error && (error.name === "Skip" || error instanceof Skip)) {
+        const skipError = error;
+        await handleStepSkip(this.backend, workflowSlug, runId, stepId2, skipError, duration, currentAttemptNumber);
+        console.log(`[StepWorker ${this.workerId}] Skipped ${step.name}: ${skipError.reason || skipError.message}`);
+        if (this.options.mode === "unified") {
+          const workflowData2 = this.workflowCache.get(workflowSlug);
+          if (workflowData2) {
+            await cascadeSkip(this.backend, workflowSlug, runId, stepId2, workflowData2.steps, this.workerId);
+          }
+        }
+        return;
+      }
+      const maxRetries = step.maxRetries ?? 0;
+      if (currentAttemptNumber <= maxRetries) {
+        const retryDelayMs = step.retryDelayMs ?? 0;
+        const nextRetryAt = getMicrosecondTimestamp5() + retryDelayMs * MICROSECONDS_PER_MILLISECOND;
+        await this.backend.saveStepFailedAndScheduleRetry(workflowSlug, runId, stepId2, stepError, {
+          duration,
+          attemptNumber: currentAttemptNumber,
+          nextRetryAt,
+          failureReason: "execution-error"
+        }, {
+          availableAt: nextRetryAt,
+          nextAttemptNumber: currentAttemptNumber + 1,
+          retryDelayMs,
+          maxRetries
+        });
+        console.log(`[StepWorker ${this.workerId}] Scheduled retry for ${step.name} (attempt ${currentAttemptNumber + 1}/${maxRetries + 1})`);
+      } else {
+        await this.backend.saveStepFailed(workflowSlug, runId, stepId2, stepError, {
+          duration,
+          attemptNumber: currentAttemptNumber,
+          terminal: true,
+          failureReason: "exhausted-retries"
+        });
+        console.log(`[StepWorker ${this.workerId}] Terminal failure for ${step.name} (retries exhausted)`);
+      }
+    }
+  }
+  setupSignalHandlers() {
+    const shutdown = async (signal) => {
+      console.log(`[StepWorker ${this.workerId}] Received ${signal}, shutting down gracefully...`);
+      await this.stop();
+      process.exit(0);
+    };
+    process.on("SIGTERM", () => shutdown("SIGTERM"));
+    process.on("SIGINT", () => shutdown("SIGINT"));
+  }
+  heartbeatTimer;
+}
+export {
+  StepWorker
+};
+
+//# debugId=94F7F5AE1B4A4B8564756E2164756E21