@cascade-flow/backend-postgres 0.2.16 → 0.2.17

package/dist/index.js

@@ -4849,6 +4849,9 @@ class DatabaseClient {
         let attemptNumber = null;
         let availableAtUs = null;
         let exportOutput = null;
+        let priority = null;
+        let slotIndex = null;
+        let workerConcurrency = null;
         let errorNameHash = "";
         let errorMessageHash = "";
         let errorStackExactHash = "";
@@ -4856,12 +4859,15 @@ class DatabaseClient {
         let errorStackPortableHash = "";
         if (se.type === "StepStarted" || se.type === "StepHeartbeat") {
           workerId = se.workerId;
+          slotIndex = se.slotIndex ?? null;
+          workerConcurrency = se.workerConcurrency ?? null;
         }
         if ("attemptNumber" in se) {
           attemptNumber = se.attemptNumber;
         }
         if (se.type === "StepScheduled") {
           availableAtUs = se.availableAtUs;
+          priority = se.priority ?? null;
         }
         if (se.type === "StepCompleted") {
           exportOutput = se.exportOutput;
@@ -4884,9 +4890,10 @@ class DatabaseClient {
           event_id, workflow_slug, run_id, step_id, timestamp_us, category, type, event_data,
           worker_id, attempt_number, available_at_us, export_output,
           error_name_hash, error_message_hash, error_stack_exact_hash,
-          error_stack_normalized_hash, error_stack_portable_hash, version_id
+          error_stack_normalized_hash, error_stack_portable_hash, version_id, priority,
+          slot_index, worker_concurrency
         )
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)`, [
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)`, [
           se.eventId,
           se.workflowSlug,
           se.runId,
@@ -4904,7 +4911,10 @@ class DatabaseClient {
           errorStackExactHash,
           errorStackNormalizedHash,
           errorStackPortableHash,
-          versionId
+          versionId,
+          priority,
+          slotIndex,
+          workerConcurrency
         ]);
       }
     } finally {
@@ -4966,6 +4976,9 @@ class DatabaseClient {
       let attemptNumber = null;
       let availableAtUs = null;
       let exportOutput = null;
+      let priority = null;
+      let slotIndex = null;
+      let workerConcurrency = null;
       let errorNameHash = "";
       let errorMessageHash = "";
       let errorStackExactHash = "";
@@ -4973,12 +4986,15 @@ class DatabaseClient {
       let errorStackPortableHash = "";
       if (se.type === "StepStarted" || se.type === "StepHeartbeat") {
         workerId = se.workerId;
+        slotIndex = se.slotIndex ?? null;
+        workerConcurrency = se.workerConcurrency ?? null;
       }
       if ("attemptNumber" in se) {
         attemptNumber = se.attemptNumber;
       }
       if (se.type === "StepScheduled") {
         availableAtUs = se.availableAtUs;
+        priority = se.priority ?? null;
       }
       if (se.type === "StepCompleted") {
         exportOutput = se.exportOutput;
@@ -5001,9 +5017,10 @@ class DatabaseClient {
           event_id, workflow_slug, run_id, step_id, timestamp_us, category, type, event_data,
           worker_id, attempt_number, available_at_us, export_output,
           error_name_hash, error_message_hash, error_stack_exact_hash,
-          error_stack_normalized_hash, error_stack_portable_hash, version_id
+          error_stack_normalized_hash, error_stack_portable_hash, version_id, priority,
+          slot_index, worker_concurrency
         )
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)`, [
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)`, [
         se.eventId,
         se.workflowSlug,
         se.runId,
@@ -5021,7 +5038,10 @@ class DatabaseClient {
         errorStackExactHash,
         errorStackNormalizedHash,
         errorStackPortableHash,
-        versionId
+        versionId,
+        priority,
+        slotIndex,
+        workerConcurrency
       ]);
     }
   }
@@ -5128,6 +5148,8 @@ class DatabaseClient {
       }
       let workerId2 = null;
       let attemptNumber = null;
+      let slotIndex = null;
+      let workerConcurrency = null;
       let errorNameHash = "";
       let errorMessageHash = "";
       let errorStackExactHash = "";
@@ -5136,6 +5158,8 @@ class DatabaseClient {
       if (eventToWrite.type === "StepStarted") {
         workerId2 = eventToWrite.workerId;
         attemptNumber = eventToWrite.attemptNumber;
+        slotIndex = eventToWrite.slotIndex ?? null;
+        workerConcurrency = eventToWrite.workerConcurrency ?? null;
       }
       if (eventToWrite.type === "StepFailed") {
         errorNameHash = eventToWrite.errorFingerprints.nameHash;
@@ -5155,9 +5179,10 @@ class DatabaseClient {
           event_id, workflow_slug, run_id, step_id, timestamp_us, category, type, event_data,
           worker_id, attempt_number, available_at_us, export_output,
           error_name_hash, error_message_hash, error_stack_exact_hash,
-          error_stack_normalized_hash, error_stack_portable_hash, version_id
+          error_stack_normalized_hash, error_stack_portable_hash, version_id,
+          slot_index, worker_concurrency
         )
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)`, [
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)`, [
         eventToWrite.eventId,
         eventToWrite.workflowSlug,
         eventToWrite.runId,
@@ -5175,7 +5200,9 @@ class DatabaseClient {
         errorStackExactHash,
         errorStackNormalizedHash,
         errorStackPortableHash,
-        versionId
+        versionId,
+        slotIndex,
+        workerConcurrency
       ]);
       await client.query("COMMIT");
       return true;
@@ -5194,7 +5221,7 @@ class DatabaseClient {
       let query = `
       WITH latest_step_events AS (
         SELECT DISTINCT ON (workflow_slug, run_id, step_id)
-          workflow_slug, run_id, step_id, type, available_at_us
+          workflow_slug, run_id, step_id, type, available_at_us, priority
         FROM ${this.schema}.step_events
         ${options?.workflowSlugs ? "WHERE workflow_slug = ANY($1)" : ""}
         ORDER BY workflow_slug, run_id, step_id, timestamp_us DESC, event_id DESC
@@ -5203,6 +5230,7 @@ class DatabaseClient {
       FROM latest_step_events
       WHERE type = ANY($${options?.workflowSlugs ? "2" : "1"})
         AND (available_at_us IS NULL OR available_at_us <= $${options?.workflowSlugs ? "3" : "2"})
+      ORDER BY priority DESC NULLS LAST, available_at_us ASC
       ${options?.limit ? `LIMIT $${options?.workflowSlugs ? "4" : "3"}` : ""}
     `;
       const params = [];
@@ -5739,6 +5767,222 @@ class DatabaseClient {
       client.release();
     }
   }
+  async getActiveWorkersAggregation(options) {
+    const client = await this.pool.connect();
+    try {
+      const nowUs = Date.now() * 1000;
+      const staleThresholdUs = options?.staleThresholdUs ?? 30 * 1000 * 1000;
+      const includeInactive = options?.includeInactive ?? false;
+      const staleTimestamp = nowUs - staleThresholdUs;
+      const workerStatsResult = await client.query(`
+        WITH worker_activity AS (
+          -- All worker activity from StepStarted and StepHeartbeat events
+          SELECT
+            worker_id,
+            MAX(timestamp_us) AS last_seen_us
+          FROM ${this.schema}.step_events
+          WHERE worker_id IS NOT NULL
+            AND type IN ('StepStarted', 'StepHeartbeat')
+            ${options?.timeRange ? `AND timestamp_us >= $1 AND timestamp_us <= $2` : ""}
+          GROUP BY worker_id
+        ),
+        -- Steps started by each worker
+        steps_started AS (
+          SELECT DISTINCT
+            se.worker_id,
+            se.workflow_slug,
+            se.run_id,
+            se.step_id,
+            se.attempt_number
+          FROM ${this.schema}.step_events se
+          WHERE se.type = 'StepStarted'
+            AND se.worker_id IS NOT NULL
+        ),
+        -- Steps completed
+        completed_steps AS (
+          SELECT
+            ss.worker_id,
+            COUNT(*) AS completed_count
+          FROM steps_started ss
+          INNER JOIN ${this.schema}.step_events se
+            ON se.workflow_slug = ss.workflow_slug
+            AND se.run_id = ss.run_id
+            AND se.step_id = ss.step_id
+            AND se.attempt_number = ss.attempt_number
+            AND se.type = 'StepCompleted'
+          GROUP BY ss.worker_id
+        ),
+        -- Steps failed
+        failed_steps AS (
+          SELECT
+            ss.worker_id,
+            COUNT(*) AS failed_count
+          FROM steps_started ss
+          INNER JOIN ${this.schema}.step_events se
+            ON se.workflow_slug = ss.workflow_slug
+            AND se.run_id = ss.run_id
+            AND se.step_id = ss.step_id
+            AND se.attempt_number = ss.attempt_number
+            AND se.type = 'StepFailed'
+          GROUP BY ss.worker_id
+        ),
+        -- Reclamation counts (times this worker's steps were reclaimed)
+        reclaimed_counts AS (
+          SELECT
+            (event_data->>'originalWorkerId') AS worker_id,
+            COUNT(*) AS reclaimed_count
+          FROM ${this.schema}.step_events
+          WHERE type = 'StepReclaimed'
+          GROUP BY (event_data->>'originalWorkerId')
+        )
+        SELECT
+          wa.worker_id,
+          wa.last_seen_us,
+          COALESCE(cs.completed_count, 0) AS total_steps_processed,
+          COALESCE(fs.failed_count, 0) AS failed_steps,
+          COALESCE(rc.reclaimed_count, 0) AS reclaimed_from_count
+        FROM worker_activity wa
+        LEFT JOIN completed_steps cs ON wa.worker_id = cs.worker_id
+        LEFT JOIN failed_steps fs ON wa.worker_id = fs.worker_id
+        LEFT JOIN reclaimed_counts rc ON wa.worker_id = rc.worker_id
+        ${!includeInactive ? `WHERE wa.last_seen_us >= ${staleTimestamp}` : ""}
+        ORDER BY wa.last_seen_us DESC
+      `, options?.timeRange ? [options.timeRange.startUs, options.timeRange.endUs] : []);
+      const runningStepsResult = await client.query(`
+        WITH latest_step_events AS (
+          -- Get the latest event per step (excluding LogEntry)
+          SELECT DISTINCT ON (workflow_slug, run_id, step_id)
+            workflow_slug,
+            run_id,
+            step_id,
+            type,
+            worker_id,
+            timestamp_us,
+            slot_index,
+            worker_concurrency
+          FROM ${this.schema}.step_events
+          WHERE type NOT IN ('LogEntry', 'StepCheckpoint', 'StepCheckpointFailed')
+          ORDER BY workflow_slug, run_id, step_id, timestamp_us DESC, event_id DESC
+        ),
+        -- Steps currently running (latest event is StepStarted or StepHeartbeat)
+        running_steps AS (
+          SELECT
+            workflow_slug,
+            run_id,
+            step_id,
+            worker_id,
+            slot_index,
+            worker_concurrency
+          FROM latest_step_events
+          WHERE type IN ('StepStarted', 'StepHeartbeat')
+            AND worker_id IS NOT NULL
+        ),
+        -- Get start time and last heartbeat for each running step
+        step_times AS (
+          SELECT
+            rs.workflow_slug,
+            rs.run_id,
+            rs.step_id,
+            rs.worker_id,
+            rs.slot_index,
+            rs.worker_concurrency,
+            MIN(se.timestamp_us) FILTER (WHERE se.type = 'StepStarted') AS started_at_us,
+            MAX(se.timestamp_us) FILTER (WHERE se.type IN ('StepStarted', 'StepHeartbeat')) AS last_heartbeat_us
+          FROM running_steps rs
+          INNER JOIN ${this.schema}.step_events se
+            ON se.workflow_slug = rs.workflow_slug
+            AND se.run_id = rs.run_id
+            AND se.step_id = rs.step_id
+            AND se.type IN ('StepStarted', 'StepHeartbeat')
+          GROUP BY rs.workflow_slug, rs.run_id, rs.step_id, rs.worker_id, rs.slot_index, rs.worker_concurrency
+        )
+        SELECT
+          worker_id,
+          workflow_slug,
+          run_id,
+          step_id,
+          started_at_us,
+          last_heartbeat_us,
+          slot_index,
+          worker_concurrency
+        FROM step_times
+        ORDER BY worker_id, last_heartbeat_us DESC
+      `);
+      const workerConcurrencyResult = await client.query(`
+        SELECT DISTINCT ON (worker_id)
+          worker_id,
+          worker_concurrency
+        FROM ${this.schema}.step_events
+        WHERE worker_id IS NOT NULL
+          AND worker_concurrency IS NOT NULL
+          AND type IN ('StepStarted', 'StepHeartbeat')
+        ORDER BY worker_id, timestamp_us DESC
+      `);
+      const workerConcurrencyMap = new Map;
+      for (const row of workerConcurrencyResult.rows) {
+        workerConcurrencyMap.set(row.worker_id, row.worker_concurrency);
+      }
+      const workerMap = new Map;
+      for (const row of workerStatsResult.rows) {
+        workerMap.set(row.worker_id, {
+          workerId: row.worker_id,
+          lastSeenUs: parseInt(row.last_seen_us, 10),
+          totalStepsProcessed: parseInt(row.total_steps_processed, 10),
+          failedSteps: parseInt(row.failed_steps, 10),
+          reclaimedFromCount: parseInt(row.reclaimed_from_count, 10),
+          activeSteps: []
+        });
+      }
+      for (const row of runningStepsResult.rows) {
+        let worker = workerMap.get(row.worker_id);
+        if (!worker) {
+          worker = {
+            workerId: row.worker_id,
+            lastSeenUs: parseInt(row.last_heartbeat_us, 10),
+            totalStepsProcessed: 0,
+            failedSteps: 0,
+            reclaimedFromCount: 0,
+            activeSteps: []
+          };
+          workerMap.set(row.worker_id, worker);
+        }
+        if (row.worker_concurrency != null && worker.workerConcurrency == null) {
+          worker.workerConcurrency = row.worker_concurrency;
+        }
+        worker.activeSteps.push({
+          workflowSlug: row.workflow_slug,
+          runId: row.run_id,
+          stepId: row.step_id,
+          startedAtUs: parseInt(row.started_at_us, 10),
+          lastHeartbeatUs: parseInt(row.last_heartbeat_us, 10),
+          slotIndex: row.slot_index ?? undefined
+        });
+      }
+      for (const worker of workerMap.values()) {
+        if (worker.workerConcurrency == null) {
+          const historicalConcurrency = workerConcurrencyMap.get(worker.workerId);
+          if (historicalConcurrency != null) {
+            worker.workerConcurrency = historicalConcurrency;
+          }
+        }
+      }
+      const workers = Array.from(workerMap.values()).sort((a, b) => {
+        if (b.activeSteps.length !== a.activeSteps.length) {
+          return b.activeSteps.length - a.activeSteps.length;
+        }
+        return b.lastSeenUs - a.lastSeenUs;
+      });
+      const totalRunningSteps = workers.reduce((sum, w) => sum + w.activeSteps.length, 0);
+      const totalActiveWorkers = workers.filter((w) => w.activeSteps.length > 0).length;
+      return {
+        workers,
+        totalActiveWorkers,
+        totalRunningSteps
+      };
+    } finally {
+      client.release();
+    }
+  }
 }
 function createPool(connectionString) {
   return new Pool2({ connectionString });
@@ -6086,6 +6330,60 @@ async function migration008_addRunIdIndex(pool, schema) {
     client.release();
   }
 }
+async function migration009_addStepPriority(pool, schema) {
+  const client = await pool.connect();
+  try {
+    await client.query(`
+      ALTER TABLE ${schema}.step_events
+      ADD COLUMN IF NOT EXISTS priority INTEGER
+    `);
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_step_events_priority_queue
+      ON ${schema}.step_events (priority DESC NULLS LAST, available_at_us ASC, timestamp_us ASC)
+      WHERE type IN ('StepScheduled', 'StepReclaimed', 'StepRetrying')
+    `);
+    console.log("[Migration 009] Step priority column and index added successfully");
+  } catch (error) {
+    console.error("[Migration 009] Error adding step priority:", error);
+    throw error;
+  } finally {
+    client.release();
+  }
+}
+async function migration010_addSlotTracking(pool, schema) {
+  const client = await pool.connect();
+  try {
+    await client.query(`
+      ALTER TABLE ${schema}.step_events
+      ADD COLUMN IF NOT EXISTS slot_index INTEGER,
+      ADD COLUMN IF NOT EXISTS worker_concurrency INTEGER
+    `);
+    console.log("[Migration 010] Slot tracking columns added successfully");
+  } catch (error) {
+    console.error("[Migration 010] Error adding slot tracking columns:", error);
+    throw error;
+  } finally {
+    client.release();
+  }
+}
+async function migration011_addWorkerConcurrencyIndex(pool, schema) {
+  const client = await pool.connect();
+  try {
+    await client.query(`
+      CREATE INDEX IF NOT EXISTS idx_step_events_worker_concurrency
+      ON ${schema}.step_events (worker_id, timestamp_us DESC)
+      WHERE worker_id IS NOT NULL
+        AND worker_concurrency IS NOT NULL
+        AND type IN ('StepStarted', 'StepHeartbeat')
+    `);
+    console.log("[Migration 011] Worker concurrency index added successfully");
+  } catch (error) {
+    console.error("[Migration 011] Error adding worker concurrency index:", error);
+    throw error;
+  } finally {
+    client.release();
+  }
+}
 async function runMigrations(pool, schema = "cascadeflow") {
   console.log(`[Migrations] Starting database migrations in schema '${schema}'...`);
   try {
@@ -6098,6 +6396,9 @@ async function runMigrations(pool, schema = "cascadeflow") {
     await migration006_addDescIndexes(pool, schema);
     await migration007_addWorkerIndexes(pool, schema);
     await migration008_addRunIdIndex(pool, schema);
+    await migration009_addStepPriority(pool, schema);
+    await migration010_addSlotTracking(pool, schema);
+    await migration011_addWorkerConcurrencyIndex(pool, schema);
     console.log("[Migrations] All migrations completed successfully");
   } catch (error) {
     console.error("[Migrations] Migration failed:", error);
@@ -6408,7 +6709,8 @@ class PostgresBackend extends Backend {
       availableAtUs: metadata.availableAt,
       reason: metadata.reason,
       attemptNumber: metadata.attemptNumber,
-      retryDelayMs: metadata.retryDelayMs
+      retryDelayMs: metadata.retryDelayMs,
+      priority: metadata.priority
     };
     await this.db.appendEvent("step_events", event);
   }
@@ -6426,7 +6728,9 @@ class PostgresBackend extends Backend {
       stepId,
       attemptNumber,
       workerId,
-      dependencies: metadata.dependencies
+      dependencies: metadata.dependencies,
+      slotIndex: metadata.slotIndex,
+      workerConcurrency: metadata.workerConcurrency
     };
     await this.db.appendEvent("step_events", event);
   }
@@ -6545,6 +6849,7 @@ class PostgresBackend extends Backend {
         reason: "retry",
         attemptNumber: scheduleMetadata.nextAttemptNumber,
         retryDelayMs: scheduleMetadata.retryDelayMs,
+        priority: scheduleMetadata.priority,
         policyIndex: scheduleMetadata.policyIndex,
         attemptInPolicy: scheduleMetadata.attemptInPolicy
       };
@@ -6614,7 +6919,7 @@ class PostgresBackend extends Backend {
     };
     await this.db.appendEvent("step_events", event);
   }
-  async saveStepHeartbeat(workflowSlug, runId, stepId, workerId, attemptNumber) {
+  async saveStepHeartbeat(workflowSlug, runId, stepId, workerId, attemptNumber, slotInfo) {
     const now = getMicrosecondTimestamp();
     const event = {
       category: "step",
@@ -6625,7 +6930,9 @@ class PostgresBackend extends Backend {
       runId,
       stepId,
       workerId,
-      attemptNumber
+      attemptNumber,
+      slotIndex: slotInfo?.slotIndex,
+      workerConcurrency: slotInfo?.workerConcurrency
     };
     await this.db.appendEvent("step_events", event);
   }
@@ -6978,7 +7285,9 @@ class PostgresBackend extends Backend {
       stepId,
       workerId,
       dependencies: metadata.dependencies,
-      attemptNumber
+      attemptNumber,
+      slotIndex: metadata.slotIndex,
+      workerConcurrency: metadata.workerConcurrency
     };
     const claimed = await this.db.claimScheduledStep(workflowSlug, runId, stepId, workerId, event);
     return claimed ? { attemptNumber } : null;
@@ -7422,9 +7731,32 @@ class PostgresBackend extends Backend {
       successRate
     };
   }
+  async getActiveWorkers(options) {
+    const staleThresholdUs = options?.staleThresholdUs ?? 30 * 1000 * 1000;
+    const result = await this.db.getActiveWorkersAggregation({
+      staleThresholdUs,
+      includeInactive: options?.includeInactive,
+      timeRange: options?.timeRange
+    });
+    return {
+      workers: result.workers.map((w) => ({
+        workerId: w.workerId,
+        lastSeenUs: w.lastSeenUs,
+        currentlyRunningSteps: w.activeSteps.length,
+        totalStepsProcessed: w.totalStepsProcessed,
+        failedSteps: w.failedSteps,
+        reclaimedFromCount: w.reclaimedFromCount,
+        workerConcurrency: w.workerConcurrency,
+        activeSteps: w.activeSteps
+      })),
+      totalActiveWorkers: result.totalActiveWorkers,
+      totalRunningSteps: result.totalRunningSteps,
+      staleThresholdUs
+    };
+  }
 }
 export {
   PostgresBackend
 };
 
-//# debugId=DA7768AC03A0F9DC64756E2164756E21
+//# debugId=9C62728A85A6463664756E2164756E21