@cascade-flow/backend-postgres 0.2.16 → 0.2.19

package/dist/index.js

@@ -4849,6 +4849,9 @@ class DatabaseClient {
         let attemptNumber = null;
         let availableAtUs = null;
         let exportOutput = null;
+        let priority = null;
+        let slotIndex = null;
+        let workerConcurrency = null;
         let errorNameHash = "";
         let errorMessageHash = "";
         let errorStackExactHash = "";
@@ -4856,12 +4859,15 @@ class DatabaseClient {
         let errorStackPortableHash = "";
         if (se.type === "StepStarted" || se.type === "StepHeartbeat") {
           workerId = se.workerId;
+          slotIndex = se.slotIndex ?? null;
+          workerConcurrency = se.workerConcurrency ?? null;
         }
         if ("attemptNumber" in se) {
           attemptNumber = se.attemptNumber;
         }
         if (se.type === "StepScheduled") {
           availableAtUs = se.availableAtUs;
+          priority = se.priority ?? null;
         }
         if (se.type === "StepCompleted") {
           exportOutput = se.exportOutput;
@@ -4884,9 +4890,10 @@ class DatabaseClient {
           event_id, workflow_slug, run_id, step_id, timestamp_us, category, type, event_data,
           worker_id, attempt_number, available_at_us, export_output,
           error_name_hash, error_message_hash, error_stack_exact_hash,
-          error_stack_normalized_hash, error_stack_portable_hash, version_id
+          error_stack_normalized_hash, error_stack_portable_hash, version_id, priority,
+          slot_index, worker_concurrency
         )
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)`, [
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)`, [
           se.eventId,
           se.workflowSlug,
           se.runId,
@@ -4904,7 +4911,10 @@ class DatabaseClient {
           errorStackExactHash,
           errorStackNormalizedHash,
           errorStackPortableHash,
-          versionId
+          versionId,
+          priority,
+          slotIndex,
+          workerConcurrency
         ]);
       }
     } finally {
@@ -4966,6 +4976,9 @@ class DatabaseClient {
       let attemptNumber = null;
       let availableAtUs = null;
       let exportOutput = null;
+      let priority = null;
+      let slotIndex = null;
+      let workerConcurrency = null;
       let errorNameHash = "";
       let errorMessageHash = "";
       let errorStackExactHash = "";
@@ -4973,12 +4986,15 @@ class DatabaseClient {
       let errorStackPortableHash = "";
       if (se.type === "StepStarted" || se.type === "StepHeartbeat") {
         workerId = se.workerId;
+        slotIndex = se.slotIndex ?? null;
+        workerConcurrency = se.workerConcurrency ?? null;
       }
       if ("attemptNumber" in se) {
         attemptNumber = se.attemptNumber;
       }
       if (se.type === "StepScheduled") {
         availableAtUs = se.availableAtUs;
+        priority = se.priority ?? null;
       }
       if (se.type === "StepCompleted") {
         exportOutput = se.exportOutput;
@@ -5001,9 +5017,10 @@ class DatabaseClient {
           event_id, workflow_slug, run_id, step_id, timestamp_us, category, type, event_data,
           worker_id, attempt_number, available_at_us, export_output,
           error_name_hash, error_message_hash, error_stack_exact_hash,
-          error_stack_normalized_hash, error_stack_portable_hash, version_id
+          error_stack_normalized_hash, error_stack_portable_hash, version_id, priority,
+          slot_index, worker_concurrency
         )
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)`, [
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)`, [
         se.eventId,
         se.workflowSlug,
         se.runId,
@@ -5021,7 +5038,10 @@ class DatabaseClient {
         errorStackExactHash,
         errorStackNormalizedHash,
         errorStackPortableHash,
-        versionId
+        versionId,
+        priority,
+        slotIndex,
+        workerConcurrency
       ]);
     }
   }
@@ -5105,13 +5125,15 @@ class DatabaseClient {
       client.release();
     }
   }
-  async claimScheduledStep(workflowSlug, runId, stepId, workerId, eventToWrite) {
+  async claimScheduledStep(workflowSlug, runId, stepId, eventToWrite) {
     const client = await this.pool.connect();
     try {
       await client.query("BEGIN");
+      const currentTimeUs = Date.now() * 1000;
       const checkQuery = `
-      SELECT event_data FROM ${this.schema}.step_events
+      SELECT event_data, attempt_number, available_at_us FROM ${this.schema}.step_events
       WHERE workflow_slug = $1 AND run_id = $2 AND step_id = $3
+        AND type NOT IN ('LogEntry', 'StepCheckpoint', 'StepCheckpointFailed')
       ORDER BY timestamp_us DESC, event_id DESC
       LIMIT 1
       FOR UPDATE SKIP LOCKED
@@ -5119,30 +5141,42 @@ class DatabaseClient {
       const checkResult = await client.query(checkQuery, [workflowSlug, runId, stepId]);
       if (checkResult.rows.length === 0) {
         await client.query("ROLLBACK");
-        return false;
+        return null;
       }
-      const latestEvent = checkResult.rows[0].event_data;
+      const latestRow = checkResult.rows[0];
+      const latestEvent = latestRow.event_data;
       if (latestEvent.type !== "StepScheduled" && latestEvent.type !== "StepReclaimed" && latestEvent.type !== "StepRetrying") {
         await client.query("ROLLBACK");
-        return false;
+        return null;
       }
-      let workerId2 = null;
+      const availableAtUs = latestRow.available_at_us ?? latestEvent.availableAtUs ?? null;
+      if (availableAtUs !== null && availableAtUs > currentTimeUs) {
+        await client.query("ROLLBACK");
+        return null;
+      }
+      const resolvedAttemptNumber = latestRow.attempt_number ?? latestEvent.attemptNumber ?? (eventToWrite.type === "StepStarted" ? eventToWrite.attemptNumber : null) ?? 1;
+      const eventPayload = eventToWrite.type === "StepStarted" ? { ...eventToWrite, attemptNumber: resolvedAttemptNumber } : eventToWrite;
+      let eventWorkerId = null;
       let attemptNumber = null;
+      let slotIndex = null;
+      let workerConcurrency = null;
       let errorNameHash = "";
       let errorMessageHash = "";
       let errorStackExactHash = "";
       let errorStackNormalizedHash = "";
       let errorStackPortableHash = "";
-      if (eventToWrite.type === "StepStarted") {
-        workerId2 = eventToWrite.workerId;
-        attemptNumber = eventToWrite.attemptNumber;
-      }
-      if (eventToWrite.type === "StepFailed") {
-        errorNameHash = eventToWrite.errorFingerprints.nameHash;
-        errorMessageHash = eventToWrite.errorFingerprints.messageHash;
-        errorStackExactHash = eventToWrite.errorFingerprints.stackExactHash;
-        errorStackNormalizedHash = eventToWrite.errorFingerprints.stackNormalizedHash;
-        errorStackPortableHash = eventToWrite.errorFingerprints.stackPortableHash;
+      if (eventPayload.type === "StepStarted") {
+        eventWorkerId = eventPayload.workerId;
+        attemptNumber = eventPayload.attemptNumber;
+        slotIndex = eventPayload.slotIndex ?? null;
+        workerConcurrency = eventPayload.workerConcurrency ?? null;
+      }
+      if (eventPayload.type === "StepFailed") {
+        errorNameHash = eventPayload.errorFingerprints.nameHash;
+        errorMessageHash = eventPayload.errorFingerprints.messageHash;
+        errorStackExactHash = eventPayload.errorFingerprints.stackExactHash;
+        errorStackNormalizedHash = eventPayload.errorFingerprints.stackNormalizedHash;
+        errorStackPortableHash = eventPayload.errorFingerprints.stackPortableHash;
       }
       const versionResult = await client.query(`SELECT version_id FROM ${this.schema}.workflow_events
          WHERE workflow_slug = $1 AND run_id = $2
@@ -5155,18 +5189,19 @@ class DatabaseClient {
           event_id, workflow_slug, run_id, step_id, timestamp_us, category, type, event_data,
           worker_id, attempt_number, available_at_us, export_output,
           error_name_hash, error_message_hash, error_stack_exact_hash,
-          error_stack_normalized_hash, error_stack_portable_hash, version_id
+          error_stack_normalized_hash, error_stack_portable_hash, version_id,
+          slot_index, worker_concurrency
         )
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)`, [
-        eventToWrite.eventId,
-        eventToWrite.workflowSlug,
-        eventToWrite.runId,
-        eventToWrite.stepId,
-        eventToWrite.timestampUs,
-        eventToWrite.category,
-        eventToWrite.type,
-        JSON.stringify(stripEventIdFromJson(eventToWrite)),
-        workerId2,
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)`, [
+        eventPayload.eventId,
+        eventPayload.workflowSlug,
+        eventPayload.runId,
+        eventPayload.stepId,
+        eventPayload.timestampUs,
+        eventPayload.category,
+        eventPayload.type,
+        JSON.stringify(stripEventIdFromJson(eventPayload)),
+        eventWorkerId,
         attemptNumber,
         null,
         null,
@@ -5175,10 +5210,12 @@ class DatabaseClient {
         errorStackExactHash,
         errorStackNormalizedHash,
         errorStackPortableHash,
-        versionId
+        versionId,
+        slotIndex,
+        workerConcurrency
       ]);
       await client.query("COMMIT");
-      return true;
+      return resolvedAttemptNumber;
     } catch (error) {
       await client.query("ROLLBACK");
       throw error;
@@ -5194,7 +5231,7 @@ class DatabaseClient {
       let query = `
       WITH latest_step_events AS (
         SELECT DISTINCT ON (workflow_slug, run_id, step_id)
-          workflow_slug, run_id, step_id, type, available_at_us
+          workflow_slug, run_id, step_id, type, available_at_us, priority
         FROM ${this.schema}.step_events
         ${options?.workflowSlugs ? "WHERE workflow_slug = ANY($1)" : ""}
         ORDER BY workflow_slug, run_id, step_id, timestamp_us DESC, event_id DESC
@@ -5203,6 +5240,7 @@ class DatabaseClient {
       FROM latest_step_events
       WHERE type = ANY($${options?.workflowSlugs ? "2" : "1"})
         AND (available_at_us IS NULL OR available_at_us <= $${options?.workflowSlugs ? "3" : "2"})
+      ORDER BY priority DESC NULLS LAST, available_at_us ASC
       ${options?.limit ? `LIMIT $${options?.workflowSlugs ? "4" : "3"}` : ""}
     `;
       const params = [];
@@ -5231,21 +5269,25 @@ class DatabaseClient {
       const query = `
       WITH latest_step_events AS (
         SELECT DISTINCT ON (workflow_slug, run_id, step_id)
-          workflow_slug, run_id, step_id, type, timestamp_us, worker_id
+          workflow_slug, run_id, step_id, type, timestamp_us, worker_id, attempt_number
         FROM ${this.schema}.step_events
-        WHERE type IN ('StepStarted', 'StepHeartbeat')
+        WHERE type NOT IN ('LogEntry', 'StepCheckpoint', 'StepCheckpointFailed')
         ORDER BY workflow_slug, run_id, step_id, timestamp_us DESC, event_id DESC
       )
-      SELECT workflow_slug, run_id, step_id, worker_id
+      SELECT workflow_slug, run_id, step_id, worker_id, attempt_number, timestamp_us
       FROM latest_step_events
-      WHERE timestamp_us < $1 AND worker_id IS NOT NULL
+      WHERE type IN ('StepStarted', 'StepHeartbeat')
+        AND timestamp_us < $1
+        AND worker_id IS NOT NULL
     `;
       const result = await client.query(query, [currentTimeUs - staleThresholdUs]);
       return result.rows.map((row) => ({
         workflowSlug: row.workflow_slug,
         runId: row.run_id,
         stepId: row.step_id,
-        workerId: row.worker_id
+        workerId: row.worker_id,
+        attemptNumber: row.attempt_number ?? 1,
+        lastHeartbeatUs: Number(row.timestamp_us)
       }));
     } finally {
       client.release();
@@ -5602,6 +5644,15 @@ class DatabaseClient {
             type
           FROM ${this.schema}.workflow_events
           WHERE ($1::text IS NULL OR workflow_slug = $1)
+            AND type IN (
+              'RunSubmitted',
+              'WorkflowRetryStarted',
+              'WorkflowStarted',
+              'WorkflowResumed',
+              'WorkflowCompleted',
+              'WorkflowFailed',
+              'WorkflowCancelled'
+            )
           ORDER BY workflow_slug, run_id, timestamp_us DESC, event_id DESC
         )
         SELECT
@@ -5739,6 +5790,214 @@ class DatabaseClient {
       client.release();
     }
   }
+  async getActiveWorkersAggregation(_options) {
+    const client = await this.pool.connect();
+    try {
+      const runningRunsResult = await client.query(`
+        WITH latest_workflow_events AS (
+          SELECT DISTINCT ON (workflow_slug, run_id)
+            workflow_slug,
+            run_id,
+            type
+          FROM ${this.schema}.workflow_events
+          ORDER BY workflow_slug, run_id, timestamp_us DESC, event_id DESC
+        )
+        SELECT workflow_slug, run_id
+        FROM latest_workflow_events
+        WHERE type IN ('WorkflowStarted', 'WorkflowResumed')
+        LIMIT 500
+      `);
+      if (runningRunsResult.rows.length === 0) {
+        return {
+          workers: [],
+          totalActiveWorkers: 0,
+          totalRunningSteps: 0
+        };
+      }
+      const runIds = runningRunsResult.rows.map((r) => r.run_id);
+      const runningStepsResult = await client.query(`
+        WITH latest_step_events AS (
+          SELECT DISTINCT ON (workflow_slug, run_id, step_id)
+            workflow_slug,
+            run_id,
+            step_id,
+            type,
+            worker_id,
+            timestamp_us,
+            slot_index,
+            worker_concurrency
+          FROM ${this.schema}.step_events
+          WHERE run_id = ANY($1)
+            AND type NOT IN ('LogEntry', 'StepCheckpoint', 'StepCheckpointFailed')
+          ORDER BY workflow_slug, run_id, step_id, timestamp_us DESC, event_id DESC
+        )
+        SELECT
+          worker_id,
+          workflow_slug,
+          run_id,
+          step_id,
+          timestamp_us as last_heartbeat_us,
+          slot_index,
+          worker_concurrency
+        FROM latest_step_events
+        WHERE type IN ('StepStarted', 'StepHeartbeat')
+          AND worker_id IS NOT NULL
+        LIMIT 1000
+      `, [runIds]);
+      const workerMap = new Map;
+      for (const row of runningStepsResult.rows) {
+        let worker = workerMap.get(row.worker_id);
+        if (!worker) {
+          worker = {
+            workerId: row.worker_id,
+            lastSeenUs: parseInt(row.last_heartbeat_us, 10),
+            totalStepsProcessed: 0,
+            failedSteps: 0,
+            reclaimedFromCount: 0,
+            workerConcurrency: row.worker_concurrency ?? undefined,
+            activeSteps: []
+          };
+          workerMap.set(row.worker_id, worker);
+        }
+        const lastHeartbeatUs = parseInt(row.last_heartbeat_us, 10);
+        if (lastHeartbeatUs > worker.lastSeenUs) {
+          worker.lastSeenUs = lastHeartbeatUs;
+        }
+        if (row.worker_concurrency != null && worker.workerConcurrency == null) {
+          worker.workerConcurrency = row.worker_concurrency;
+        }
+        worker.activeSteps.push({
+          workflowSlug: row.workflow_slug,
+          runId: row.run_id,
+          stepId: row.step_id,
+          startedAtUs: lastHeartbeatUs,
+          lastHeartbeatUs,
+          slotIndex: row.slot_index ?? undefined
+        });
+      }
+      const workers = Array.from(workerMap.values()).sort((a, b) => {
+        if (b.activeSteps.length !== a.activeSteps.length) {
+          return b.activeSteps.length - a.activeSteps.length;
+        }
+        return b.lastSeenUs - a.lastSeenUs;
+      });
+      const totalRunningSteps = workers.reduce((sum, w) => sum + w.activeSteps.length, 0);
+      const totalActiveWorkers = workers.length;
+      return {
+        workers,
+        totalActiveWorkers,
+        totalRunningSteps
+      };
+    } finally {
+      client.release();
+    }
+  }
+  async getWorkerById(workerId) {
+    const client = await this.pool.connect();
+    try {
+      const result = await client.query(`
+        WITH worker_step_events AS (
+          -- Get all step events for this worker
+          SELECT
+            workflow_slug,
+            run_id,
+            step_id,
+            type,
+            timestamp_us,
+            slot_index,
+            worker_concurrency
+          FROM ${this.schema}.step_events
+          WHERE worker_id = $1
+            AND type IN ('StepStarted', 'StepHeartbeat')
+        ),
+        latest_per_step AS (
+          -- For each step this worker touched, get the latest event
+          SELECT DISTINCT ON (workflow_slug, run_id, step_id)
+            workflow_slug,
+            run_id,
+            step_id,
+            timestamp_us,
+            slot_index,
+            worker_concurrency
+          FROM worker_step_events
+          ORDER BY workflow_slug, run_id, step_id, timestamp_us DESC
+        ),
+        -- Check if these steps are still running (no completion/failure after our heartbeat)
+        still_running AS (
+          SELECT
+            lps.workflow_slug,
+            lps.run_id,
+            lps.step_id,
+            lps.timestamp_us as last_heartbeat_us,
+            lps.slot_index,
+            lps.worker_concurrency
+          FROM latest_per_step lps
+          WHERE NOT EXISTS (
+            SELECT 1 FROM ${this.schema}.step_events se
+            WHERE se.workflow_slug = lps.workflow_slug
+              AND se.run_id = lps.run_id
+              AND se.step_id = lps.step_id
+              AND se.timestamp_us > lps.timestamp_us
+              AND se.type IN ('StepCompleted', 'StepFailed', 'StepSkipped', 'StepReclaimed')
+          )
+        )
+        SELECT * FROM still_running
+        ORDER BY last_heartbeat_us DESC
+        LIMIT 100
+      `, [workerId]);
+      if (result.rows.length === 0) {
+        const lastSeenResult = await client.query(`
+          SELECT MAX(timestamp_us) as last_seen_us, MAX(worker_concurrency) as worker_concurrency
+          FROM ${this.schema}.step_events
+          WHERE worker_id = $1
+            AND type IN ('StepStarted', 'StepHeartbeat')
+        `, [workerId]);
+        if (!lastSeenResult.rows[0]?.last_seen_us) {
+          return null;
+        }
+        return {
+          workerId,
+          lastSeenUs: parseInt(lastSeenResult.rows[0].last_seen_us, 10),
+          totalStepsProcessed: 0,
+          failedSteps: 0,
+          reclaimedFromCount: 0,
+          workerConcurrency: lastSeenResult.rows[0].worker_concurrency ?? undefined,
+          activeSteps: []
+        };
+      }
+      let lastSeenUs = 0;
+      let workerConcurrency;
+      const activeSteps = [];
+      for (const row of result.rows) {
+        const heartbeatUs = parseInt(row.last_heartbeat_us, 10);
+        if (heartbeatUs > lastSeenUs) {
+          lastSeenUs = heartbeatUs;
+        }
+        if (row.worker_concurrency != null && workerConcurrency == null) {
+          workerConcurrency = row.worker_concurrency;
+        }
+        activeSteps.push({
+          workflowSlug: row.workflow_slug,
+          runId: row.run_id,
+          stepId: row.step_id,
+          startedAtUs: heartbeatUs,
+          lastHeartbeatUs: heartbeatUs,
+          slotIndex: row.slot_index ?? undefined
+        });
+      }
+      return {
+        workerId,
+        lastSeenUs,
+        totalStepsProcessed: 0,
+        failedSteps: 0,
+        reclaimedFromCount: 0,
+        workerConcurrency,
+        activeSteps
+      };
+    } finally {
+      client.release();
+    }
+  }
 }
 function createPool(connectionString) {
   return new Pool2({ connectionString });
@@ -6086,6 +6345,141 @@ async function migration008_addRunIdIndex(pool, schema) {
     client.release();
   }
 }
+async function migration009_addStepPriority(pool, schema) {
+  const client = await pool.connect();
+  try {
+    await client.query(`
+      ALTER TABLE ${schema}.step_events
+      ADD COLUMN IF NOT EXISTS priority INTEGER
+    `);
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_step_events_priority_queue
+      ON ${schema}.step_events (priority DESC NULLS LAST, available_at_us ASC, timestamp_us ASC)
+      WHERE type IN ('StepScheduled', 'StepReclaimed', 'StepRetrying')
+    `);
+    console.log("[Migration 009] Step priority column and index added successfully");
+  } catch (error) {
+    console.error("[Migration 009] Error adding step priority:", error);
+    throw error;
+  } finally {
+    client.release();
+  }
+}
+async function migration010_addSlotTracking(pool, schema) {
+  const client = await pool.connect();
+  try {
+    await client.query(`
+      ALTER TABLE ${schema}.step_events
+      ADD COLUMN IF NOT EXISTS slot_index INTEGER,
+      ADD COLUMN IF NOT EXISTS worker_concurrency INTEGER
+    `);
+    console.log("[Migration 010] Slot tracking columns added successfully");
+  } catch (error) {
+    console.error("[Migration 010] Error adding slot tracking columns:", error);
+    throw error;
+  } finally {
+    client.release();
+  }
+}
+async function migration011_addWorkerConcurrencyIndex(pool, schema) {
+  const client = await pool.connect();
+  try {
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_step_events_worker_concurrency
+      ON ${schema}.step_events (worker_id, timestamp_us DESC)
+      WHERE worker_id IS NOT NULL
+        AND worker_concurrency IS NOT NULL
+        AND type IN ('StepStarted', 'StepHeartbeat')
+    `);
+    console.log("[Migration 011] Worker concurrency index added successfully");
+  } catch (error) {
+    console.error("[Migration 011] Error adding worker concurrency index:", error);
+    throw error;
+  } finally {
+    client.release();
+  }
+}
+async function migration012_addWorkerAnalyticsIndexes(pool, schema) {
+  const client = await pool.connect();
+  try {
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_step_events_worker_activity
+      ON ${schema}.step_events (timestamp_us, type, worker_id)
+      WHERE worker_id IS NOT NULL
+        AND type IN ('StepStarted', 'StepHeartbeat')
+    `);
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_step_events_recent_by_step
+      ON ${schema}.step_events (timestamp_us DESC, workflow_slug, run_id, step_id, type, event_id DESC)
+      WHERE type NOT IN ('LogEntry', 'StepCheckpoint', 'StepCheckpointFailed')
+    `);
+    console.log("[Migration 012] Worker analytics indexes added successfully");
+  } catch (error) {
+    console.error("[Migration 012] Error adding worker analytics indexes:", error);
+    throw error;
+  } finally {
+    client.release();
+  }
+}
+async function migration013_addWorkersObservabilityIndexes(pool, schema) {
+  const client = await pool.connect();
+  try {
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_workflow_events_run_status
+      ON ${schema}.workflow_events (run_id, timestamp_us DESC, event_id DESC)
+      INCLUDE (workflow_slug, type)
+    `);
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_step_events_by_run
+      ON ${schema}.step_events (run_id, workflow_slug, step_id, timestamp_us DESC, event_id DESC)
+      INCLUDE (type, worker_id, slot_index, worker_concurrency)
+      WHERE type NOT IN ('LogEntry', 'StepCheckpoint', 'StepCheckpointFailed')
+    `);
+    console.log("[Migration 013] Workers observability indexes added successfully");
+  } catch (error) {
+    console.error("[Migration 013] Error adding workers observability indexes:", error);
+    throw error;
+  } finally {
+    client.release();
+  }
+}
+async function migration014_addWorkerHotPathIndexes(pool, schema) {
+  const client = await pool.connect();
+  try {
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_step_events_latest_cover
+      ON ${schema}.step_events (workflow_slug, run_id, step_id, timestamp_us DESC, event_id DESC)
+      INCLUDE (type, available_at_us, worker_id, attempt_number, priority)
+    `);
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_step_events_terminal_latest
+      ON ${schema}.step_events (workflow_slug, run_id, step_id, timestamp_us DESC, event_id DESC)
+      WHERE type IN ('StepCompleted', 'StepFailed', 'StepSkipped', 'StepReclaimed')
+    `);
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_workflow_events_active_latest
+      ON ${schema}.workflow_events (type, workflow_slug, run_id, timestamp_us DESC, event_id DESC)
+      WHERE type IN ('RunSubmitted', 'WorkflowRetryStarted', 'WorkflowStarted', 'WorkflowResumed')
+    `);
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_workflow_events_run_chrono
+      ON ${schema}.workflow_events (workflow_slug, run_id, timestamp_us ASC, event_id ASC)
+    `);
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_workflow_events_version_lookup
+      ON ${schema}.workflow_events (workflow_slug, run_id, timestamp_us DESC, event_id DESC)
+      INCLUDE (version_id)
+      WHERE type IN ('WorkflowStarted', 'RunSubmitted')
+        AND version_id IS NOT NULL
+    `);
+    console.log("[Migration 014] Worker hot-path indexes added successfully");
+  } catch (error) {
+    console.error("[Migration 014] Error adding worker hot-path indexes:", error);
+    throw error;
+  } finally {
+    client.release();
+  }
+}
 async function runMigrations(pool, schema = "cascadeflow") {
   console.log(`[Migrations] Starting database migrations in schema '${schema}'...`);
   try {
@@ -6098,6 +6492,12 @@ async function runMigrations(pool, schema = "cascadeflow") {
     await migration006_addDescIndexes(pool, schema);
     await migration007_addWorkerIndexes(pool, schema);
     await migration008_addRunIdIndex(pool, schema);
+    await migration009_addStepPriority(pool, schema);
+    await migration010_addSlotTracking(pool, schema);
+    await migration011_addWorkerConcurrencyIndex(pool, schema);
+    await migration012_addWorkerAnalyticsIndexes(pool, schema);
+    await migration013_addWorkersObservabilityIndexes(pool, schema);
+    await migration014_addWorkerHotPathIndexes(pool, schema);
     console.log("[Migrations] All migrations completed successfully");
   } catch (error) {
     console.error("[Migrations] Migration failed:", error);
@@ -6408,7 +6808,8 @@ class PostgresBackend extends Backend {
       availableAtUs: metadata.availableAt,
       reason: metadata.reason,
       attemptNumber: metadata.attemptNumber,
-      retryDelayMs: metadata.retryDelayMs
+      retryDelayMs: metadata.retryDelayMs,
+      priority: metadata.priority
     };
     await this.db.appendEvent("step_events", event);
   }
@@ -6426,7 +6827,9 @@ class PostgresBackend extends Backend {
       stepId,
       attemptNumber,
       workerId,
-      dependencies: metadata.dependencies
+      dependencies: metadata.dependencies,
+      slotIndex: metadata.slotIndex,
+      workerConcurrency: metadata.workerConcurrency
     };
     await this.db.appendEvent("step_events", event);
   }
@@ -6545,6 +6948,7 @@ class PostgresBackend extends Backend {
         reason: "retry",
         attemptNumber: scheduleMetadata.nextAttemptNumber,
         retryDelayMs: scheduleMetadata.retryDelayMs,
+        priority: scheduleMetadata.priority,
         policyIndex: scheduleMetadata.policyIndex,
         attemptInPolicy: scheduleMetadata.attemptInPolicy
       };
@@ -6614,7 +7018,7 @@ class PostgresBackend extends Backend {
     };
     await this.db.appendEvent("step_events", event);
   }
-  async saveStepHeartbeat(workflowSlug, runId, stepId, workerId, attemptNumber) {
+  async saveStepHeartbeat(workflowSlug, runId, stepId, workerId, attemptNumber, slotInfo) {
     const now = getMicrosecondTimestamp();
     const event = {
       category: "step",
@@ -6625,7 +7029,9 @@ class PostgresBackend extends Backend {
       runId,
       stepId,
       workerId,
-      attemptNumber
+      attemptNumber,
+      slotIndex: slotInfo?.slotIndex,
+      workerConcurrency: slotInfo?.workerConcurrency
     };
     await this.db.appendEvent("step_events", event);
   }
@@ -6957,16 +7363,6 @@ class PostgresBackend extends Backend {
     return !!(latestEvent && (latestEvent.type === "StepScheduled" || latestEvent.type === "StepReclaimed" || latestEvent.type === "StepRetrying"));
   }
   async claimScheduledStep(workflowSlug, runId, stepId, workerId, metadata) {
-    const initialEvents = await this.loadEvents(workflowSlug, runId, { category: "step", stepId });
-    if (initialEvents.length === 0) {
-      return null;
-    }
-    const now = getMicrosecondTimestamp();
-    const initialState = projectStepState(initialEvents, workflowSlug);
-    if (initialState.status !== "scheduled" || initialState.availableAt === undefined || initialState.availableAt > now) {
-      return null;
-    }
-    const attemptNumber = initialState.attemptNumber;
     const timestamp = getMicrosecondTimestamp();
     const event = {
       category: "step",
@@ -6978,41 +7374,34 @@ class PostgresBackend extends Backend {
       stepId,
       workerId,
       dependencies: metadata.dependencies,
-      attemptNumber
+      attemptNumber: metadata.attemptNumber,
+      slotIndex: metadata.slotIndex,
+      workerConcurrency: metadata.workerConcurrency
     };
-    const claimed = await this.db.claimScheduledStep(workflowSlug, runId, stepId, workerId, event);
-    return claimed ? { attemptNumber } : null;
+    const claimedAttemptNumber = await this.db.claimScheduledStep(workflowSlug, runId, stepId, event);
+    return claimedAttemptNumber !== null ? { attemptNumber: claimedAttemptNumber } : null;
   }
   async reclaimStaleSteps(staleThreshold, reclaimedBy) {
     const reclaimed = [];
     const now = getMicrosecondTimestamp();
     const staleSteps = await this.db.findStaleSteps(staleThreshold);
     for (const step of staleSteps) {
-      const events = await this.loadEvents(step.workflowSlug, step.runId, { category: "step", stepId: step.stepId });
-      if (events.length === 0)
-        continue;
-      const state = projectStepState(events, step.workflowSlug);
-      if (state.status !== "running")
-        continue;
-      const lastHeartbeat = state.lastHeartbeat || state.startTime || 0;
-      const staleDuration = now - lastHeartbeat;
-      if (staleDuration > staleThreshold) {
-        await this.saveStepReclaimed(step.workflowSlug, step.runId, step.stepId, {
-          originalWorkerId: state.claimedBy || "unknown",
-          reclaimedBy,
-          lastHeartbeat,
-          staleThreshold,
-          staleDuration,
-          attemptNumber: state.attemptNumber
-        });
-        await this.saveStepScheduled(step.workflowSlug, step.runId, step.stepId, {
-          availableAt: now,
-          reason: "retry",
-          attemptNumber: state.attemptNumber + 1,
-          retryDelayMs: 0
-        });
-        reclaimed.push({ workflowSlug: step.workflowSlug, runId: step.runId, stepId: step.stepId });
-      }
+      const staleDuration = now - step.lastHeartbeatUs;
+      await this.saveStepReclaimed(step.workflowSlug, step.runId, step.stepId, {
+        originalWorkerId: step.workerId,
+        reclaimedBy,
+        lastHeartbeat: step.lastHeartbeatUs,
+        staleThreshold,
+        staleDuration,
+        attemptNumber: step.attemptNumber
+      });
+      await this.saveStepScheduled(step.workflowSlug, step.runId, step.stepId, {
+        availableAt: now,
+        reason: "retry",
+        attemptNumber: step.attemptNumber + 1,
+        retryDelayMs: 0
+      });
+      reclaimed.push({ workflowSlug: step.workflowSlug, runId: step.runId, stepId: step.stepId });
     }
     return reclaimed;
   }
@@ -7422,9 +7811,48 @@ class PostgresBackend extends Backend {
       successRate
     };
   }
+  async getActiveWorkers(options) {
+    const staleThresholdUs = options?.staleThresholdUs ?? 30 * 1000 * 1000;
+    const result = await this.db.getActiveWorkersAggregation({
+      staleThresholdUs,
+      includeInactive: options?.includeInactive,
+      timeRange: options?.timeRange
+    });
+    return {
+      workers: result.workers.map((w) => ({
+        workerId: w.workerId,
+        lastSeenUs: w.lastSeenUs,
+        currentlyRunningSteps: w.activeSteps.length,
+        totalStepsProcessed: w.totalStepsProcessed,
+        failedSteps: w.failedSteps,
+        reclaimedFromCount: w.reclaimedFromCount,
+        workerConcurrency: w.workerConcurrency,
+        activeSteps: w.activeSteps
+      })),
+      totalActiveWorkers: result.totalActiveWorkers,
+      totalRunningSteps: result.totalRunningSteps,
+      staleThresholdUs
+    };
+  }
+  async getWorkerById(workerId) {
+    const result = await this.db.getWorkerById(workerId);
+    if (!result) {
+      return null;
+    }
+    return {
+      workerId: result.workerId,
+      lastSeenUs: result.lastSeenUs,
+      currentlyRunningSteps: result.activeSteps.length,
+      totalStepsProcessed: result.totalStepsProcessed,
+      failedSteps: result.failedSteps,
+      reclaimedFromCount: result.reclaimedFromCount,
+      workerConcurrency: result.workerConcurrency,
+      activeSteps: result.activeSteps
+    };
+  }
 }
 export {
   PostgresBackend
 };
 
-//# debugId=DA7768AC03A0F9DC64756E2164756E21
+//# debugId=43E020BB2B7D86DF64756E2164756E21