@dbos-inc/dbos-sdk 4.26.7-preview → 4.26.8-preview

package/dist/src/system_database.js

@@ -564,14 +564,12 @@ class SystemDatabase {
         await this.pool.end();
     }
     // ==================== Workflow Status ====================
-    async initWorkflowStatus(initStatus, ownerXid, options) {
+    async initWorkflowStatus(initStatus, ownerXid) {
         const client = await this.pool.connect();
         let shouldCommit = false;
         try {
             await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
-            // Moving from enqueued to pending asks to increment recovery attempts... rather than in the recovery process
-            //  where it moves from pending back to enqueued.
-            const resRow = await this.insertWorkflowStatus(client, initStatus, ownerXid, !!options?.isRecoveryRequest || !!options?.isDequeuedRequest);
+            const resRow = await this.insertWorkflowStatus(client, initStatus, ownerXid);
             if (resRow.name !== initStatus.workflowName) {
                 const msg = `Workflow already exists with a different function name: ${resRow.name}, but the provided function name is: ${initStatus.workflowName}`;
                 throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
@@ -590,31 +588,12 @@ class SystemDatabase {
             }
             const status = resRow.status;
             const deadlineEpochMS = resRow.workflow_deadline_epoch_ms ?? undefined;
-            // If there is an existing DB record and we aren't here to recover it,
-            //  leave it be.  Roll back the change to max recovery attempts.
-            if (ownerXid !== resRow.owner_xid && !options?.isRecoveryRequest && !options?.isDequeuedRequest) {
-                // It is not clear if getting the handle should throw the error, or getting the result from the handle should error.
-                //  Current precedent is the former.
-                if (status === workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED) {
-                    throw new error_1.DBOSMaxRecoveryAttemptsExceededError(initStatus.workflowUUID, options?.maxRetries ?? -1);
-                }
+            // If there is an existing DB record and we aren't here to recover it, leave it be.
+            if (ownerXid !== resRow.owner_xid) {
                 return { status, deadlineEpochMS, shouldExecuteOnThisExecutor: false, serialization: resRow.serialization };
             }
-            // Upsert above already set executor assignment and incremented the recovery attempt
+            // Upsert above already set executor assignment
             shouldCommit = true;
-            // recovery_attempt means "attempts" (we kept the name for backward compatibility). It's default value is 1.
-            // Every time we init the status, we increment `recovery_attempts` by 1.
-            // Thus, when this number becomes equal to `maxRetries + 1`, we should mark the workflow as `MAX_RECOVERY_ATTEMPTS_EXCEEDED`.
-            const attempts = resRow.recovery_attempts;
-            if (options?.maxRetries && attempts > options?.maxRetries + 1) {
-                await this.updateWorkflowStatus(client, initStatus.workflowUUID, workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, {
-                    where: { status: workflow_1.StatusString.PENDING },
-                    throwOnFailure: false,
-                    update: { resetDeduplicationID: true },
-                });
-                throw new error_1.DBOSMaxRecoveryAttemptsExceededError(initStatus.workflowUUID, options.maxRetries);
-            }
-            this.logger.debug(`Workflow ${initStatus.workflowUUID} attempt number: ${attempts}.`);
             return {
                 status,
                 deadlineEpochMS,
@@ -637,6 +616,19 @@ class SystemDatabase {
             }
         }
     }
+    /** Move claimed workflows that exhausted their attempts off the queue, leaving rows others have moved on alone. */
+    async deadLetterWorkflows(workflowIDs, minRecoveryAttempts) {
+        if (workflowIDs.length === 0)
+            return;
+        await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
+       SET status = $1,
+           deduplication_id = NULL,
+           started_at_epoch_ms = NULL,
+           queue_name = NULL,
+           updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
+           completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
+       WHERE workflow_uuid = ANY($2::text[]) AND status = $3 AND recovery_attempts >= $4`, [workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, workflowIDs, workflow_1.StatusString.PENDING, minRecoveryAttempts]);
+    }
     /** Highest created_at among still-active rows per partition key, used to seed the in-memory cursor. */
     async #maxPartitionKeyCreatedAt(keys) {
         const maxima = new Map();
@@ -832,24 +824,17 @@ class SystemDatabase {
     }
     // Recovery re-enqueues rather than executing directly so the queue's atomic dequeue admits exactly one runner, and the executor ID predicate rejects sweeps for rows a live executor has already claimed.
     async reenqueueWorkflowsForRecovery(executorID, appVersion, recoveryQueueName) {
-        const params = [
-            workflow_1.StatusString.ENQUEUED,
-            Date.now(),
-            recoveryQueueName,
-            workflow_1.StatusString.PENDING,
-            executorID,
-            appVersion,
-        ];
+        const params = [workflow_1.StatusString.ENQUEUED, recoveryQueueName, workflow_1.StatusString.PENDING, executorID, appVersion];
         // executor_id defaults to "local", so it collides across applications.
         const scope = this.#appNameFilter('application_name', this.appName, params);
         const result = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
        SET started_at_epoch_ms = NULL,
            status = $1,
-           updated_at = $2,
-           queue_name = COALESCE(queue_name, $3)
-       WHERE status = $4
-         AND executor_id = $5
-         AND application_version = $6
+           updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
+           queue_name = COALESCE(queue_name, $2)
+       WHERE status = $3
+         AND executor_id = $4
+         AND application_version = $5
          AND ${scope}
        RETURNING workflow_uuid`, params);
         return result.rows.map((row) => row.workflow_uuid);
@@ -879,12 +864,33 @@ class SystemDatabase {
             return json ? JSON.parse(json) : null;
         }
     }
+    /** Max IDs per {@link getWorkflowStatuses} fetch: listWorkflows binds one parameter per ID. */
+    statusFetchChunkSize = 500;
+    // Retried per chunk so a reconnect refetches one chunk, not every chunk before it.
+    async fetchWorkflowStatusChunk(workflowIDs) {
+        return await this.listWorkflows({ workflowIDs, loadInput: true, loadOutput: false });
+    }
+    /** Fetch many statuses in as few round trips as possible. IDs with no row are omitted. */
+    async getWorkflowStatuses(workflowIDs) {
+        const statuses = new Map();
+        for (let start = 0; start < workflowIDs.length; start += this.statusFetchChunkSize) {
+            for (const status of await this.fetchWorkflowStatusChunk(workflowIDs.slice(start, start + this.statusFetchChunkSize))) {
+                statuses.set(status.workflowUUID, status);
+            }
+        }
+        return statuses;
+    }
     // Only used in tests
     async setWorkflowStatus(workflowID, status, resetRecoveryAttempts, internalOptions) {
         const client = await this.pool.connect();
         try {
             await this.updateWorkflowStatus(client, workflowID, status, {
-                update: { resetRecoveryAttempts, resetNameTo: internalOptions?.updateName },
+                update: {
+                    resetRecoveryAttempts,
+                    resetNameTo: internalOptions?.updateName,
+                    queueName: internalOptions?.queueName,
+                    resetStartedAtEpochMs: internalOptions?.resetStartedAtEpochMs,
+                },
             });
         }
         finally {
@@ -1023,15 +1029,15 @@ class SystemDatabase {
     }
     async setWorkflowPriority(workflowID, priority) {
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
-       SET priority = $1, updated_at = $2
-       WHERE workflow_uuid = $3
-         AND status IN ($4, $5)`, [priority, Date.now(), workflowID, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED]);
+       SET priority = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
+       WHERE workflow_uuid = $2
+         AND status IN ($3, $4)`, [priority, workflowID, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED]);
     }
     async setWorkflowDelay(workflowID, delayUntilEpochMS) {
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
-       SET delay_until_epoch_ms = $1, updated_at = $2
-       WHERE workflow_uuid = $3
-         AND status = $4`, [delayUntilEpochMS, Date.now(), workflowID, workflow_1.StatusString.DELAYED]);
+       SET delay_until_epoch_ms = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
+       WHERE workflow_uuid = $2
+         AND status = $3`, [delayUntilEpochMS, workflowID, workflow_1.StatusString.DELAYED]);
     }
     /**
      * Extend an existing debounced DELAYED workflow's delay and update its inputs, atomically.
@@ -1071,7 +1077,6 @@ class SystemDatabase {
             params.delayUntilEpochMS,
             params.input,
             params.serialization,
-            Date.now(),
             params.workflowName,
             classNameOrNull,
             params.queueName,
@@ -1087,12 +1092,13 @@ class SystemDatabase {
              THEN debounce_deadline_epoch_ms
              ELSE $1
            END,
-           inputs = $2, serialization = $3, updated_at = $4,
+           inputs = $2, serialization = $3,
+           updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
            -- Claim it for the target, as its dequeue would: left unclaimed, every peer coalesces onto the one workflow and the last inputs win.
-           application_name = COALESCE(application_name, $10)
-       WHERE name = $5 AND class_name IS NOT DISTINCT FROM $6
-         AND queue_name = $7 AND deduplication_id = $8
-         AND status = $9 AND is_debounced = TRUE
+           application_name = COALESCE(application_name, $9)
+       WHERE name = $4 AND class_name IS NOT DISTINCT FROM $5
+         AND queue_name = $6 AND deduplication_id = $7
+         AND status = $8 AND is_debounced = TRUE
          AND ${ownScope}
        RETURNING workflow_uuid`, updateParams);
         if (updated.rows.length > 0) {
@@ -2216,7 +2222,7 @@ class SystemDatabase {
         // Only what this application would dequeue: a peer's debounce key is not ours to clear.
         const scope = this.#appNameFilter('application_name', this.appName, params);
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
-       SET status = $1, updated_at = $2,
+       SET status = $1, updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
            deduplication_id = CASE WHEN is_debounced THEN NULL ELSE deduplication_id END
        WHERE status = $3 AND delay_until_epoch_ms <= $2 AND ${scope}`, params);
     }
@@ -2248,7 +2254,6 @@ class SystemDatabase {
         return rows.map((row) => row.pk);
     }
     async findAndMarkStartableWorkflows(queue, executorID, appVersion, queuePartitionKey) {
-        const startTimeMs = Date.now();
         const limiterPeriodMS = queue.rateLimit ? queue.rateLimit.periodSec * 1000 : 0;
         const claimedIDs = [];
         const localRunningForQueue = this.countRunningWorkflowsForQueue(queue.name, queuePartitionKey);
@@ -2275,7 +2280,7 @@ class SystemDatabase {
                     queue.name,
                     workflow_1.StatusString.ENQUEUED,
                     workflow_1.StatusString.DELAYED,
-                    startTimeMs - limiterPeriodMS,
+                    limiterPeriodMS,
                     ...partitionParams,
                 ];
                 // Count only what this application would dequeue, matching the select below.
@@ -2284,7 +2289,8 @@ class SystemDatabase {
            WHERE queue_name = $1
              AND rate_limited = TRUE
              AND status NOT IN ($2, $3)
-             AND started_at_epoch_ms > $4
+             -- Database clock on both sides, as the claim stamps started_at_epoch_ms with it.
+             AND started_at_epoch_ms > (EXTRACT(epoch FROM now()) * 1000)::bigint - $4
              AND ${scope}
              ${partitionFilter.replace('$PARTITION', '$5')}`, params);
                 numRecentQueries = Number(countResult.rows[0].count);
@@ -2297,9 +2303,13 @@ class SystemDatabase {
             // If there is a global or local concurrency limit N, select only the N oldest enqueued
             // functions, else select all of them.
             let maxTasks = Infinity;
+            if (queue.rateLimit) {
+                // Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
+                maxTasks = Math.max(0, queue.rateLimit.limitPerPeriod - numRecentQueries);
+            }
             if (queue.workerConcurrency !== undefined) {
                 // Use the in-memory registry for this worker's running count — avoids a DB round trip.
-                maxTasks = Math.max(0, queue.workerConcurrency - localRunningForQueue);
+                maxTasks = Math.min(maxTasks, Math.max(0, queue.workerConcurrency - localRunningForQueue));
             }
             if (queue.concurrency !== undefined) {
                 // Global concurrency still requires a DB query since other workers may be running workflows too.
@@ -2327,7 +2337,10 @@ class SystemDatabase {
             const versionClause = isLatestVersion
                 ? '(application_version = $3 OR application_version IS NULL)'
                 : 'application_version = $3';
-            const lockMode = queue.concurrency ? 'FOR UPDATE NOWAIT' : 'FOR UPDATE SKIP LOCKED';
+            // A limit shared across processes needs a consistent view of the table: NOWAIT makes an
+            // overlapping dequeuer abort rather than claim the next rows and spend the same budget twice.
+            const sharedBudget = queue.concurrency !== undefined || queue.rateLimit !== undefined;
+            const lockMode = sharedBudget ? 'FOR UPDATE NOWAIT' : 'FOR UPDATE SKIP LOCKED';
             const limitClause = maxTasks !== Infinity ? `LIMIT ${maxTasks}` : '';
             const selectParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion, ...partitionParams];
             const selectScope = this.#appNameFilter('application_name', this.appName, selectParams);
@@ -2349,43 +2362,41 @@ class SystemDatabase {
             await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_FIND_AND_MARK_AFTER_SELECT);
             // Start the workflows
             const workflowIDs = rows.map((row) => row.workflow_uuid);
-            for (const id of workflowIDs) {
-                // If we have a rate limit, stop starting functions when the number
-                //   of functions started this period exceeds the limit.
-                if (queue.rateLimit && claimedIDs.length + numRecentQueries >= queue.rateLimit.limitPerPeriod) {
-                    break;
-                }
+            if (workflowIDs.length > 0) {
                 // Start the functions by marking them as pending and updating their executor IDs.
-                // Only claim the workflow if the UPDATE actually transitioned an ENQUEUED row —
-                // otherwise another worker won the race and we must not re-dispatch it.
                 const updateParams = [
                     workflow_1.StatusString.PENDING,
                     executorID,
                     appVersion,
-                    startTimeMs,
                     queue.rateLimit !== undefined,
-                    id,
+                    workflowIDs,
                     workflow_1.StatusString.ENQUEUED,
                     // Claim an unclaimed row for this application; a nameless dequeuer leaves ownership untouched.
                     this.appName ?? null,
                 ];
                 // Re-check ownership alongside status, as the partitioned claim guard does.
                 const claimScope = this.#appNameFilter('application_name', this.appName, updateParams);
-                const updateRes = await client.query(`UPDATE "${this.schemaName}".workflow_status
+                // RETURNING reports exactly the rows this statement flipped, so a row another worker won is absent.
+                const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
            SET status = $1,
                executor_id = $2,
                application_version = $3,
-               started_at_epoch_ms = $4,
-               rate_limited = $5,
-               application_name = COALESCE(application_name, $8),
+               started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
+               rate_limited = $4,
+               application_name = COALESCE(application_name, $7),
+               recovery_attempts = recovery_attempts + 1,
+               updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
                workflow_deadline_epoch_ms = CASE
                  WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
                  THEN (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
                  ELSE workflow_deadline_epoch_ms
                END
-           WHERE workflow_uuid = $6 AND status = $7 AND ${claimScope}`, updateParams);
-                if ((updateRes.rowCount ?? 0) > 0) {
-                    claimedIDs.push(id);
+           WHERE workflow_uuid = ANY($5::text[]) AND status = $6 AND ${claimScope}
+           RETURNING workflow_uuid`, updateParams);
+                const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
+                for (const id of workflowIDs) {
+                    if (flippedIDs.has(id))
+                        claimedIDs.push(id);
                 }
             }
             await client.query('COMMIT');
@@ -2408,7 +2419,6 @@ class SystemDatabase {
             throw new error_1.DBOSError(`Batched partitioned dequeue requires a queue with concurrency 1 and no rate limit: ${queue.name}`);
         }
         // workerConcurrency needs no handling here: dispatch routes 0 (the pause-dequeue idiom) to the fallback path, and validation caps any other value at concurrency=1, which the PENDING gate already enforces globally.
-        const startTimeMs = Date.now();
         const client = await this.pool.connect();
         try {
             await client.query('BEGIN');
@@ -2495,7 +2505,6 @@ class SystemDatabase {
                 workflow_1.StatusString.ENQUEUED,
                 queue.name,
                 appVersion,
-                startTimeMs,
                 // Claim the row, as the unpartitioned dequeue does.
                 this.appName ?? null,
             ];
@@ -2505,9 +2514,11 @@ class SystemDatabase {
          SET status = $1,
              executor_id = $2,
              application_version = $6,
-             started_at_epoch_ms = $7,
+             started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
              rate_limited = FALSE,
-             application_name = COALESCE(application_name, $8),
+             application_name = COALESCE(application_name, $7),
+             recovery_attempts = recovery_attempts + 1,
+             updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
              workflow_deadline_epoch_ms = CASE
                WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL
                THEN (EXTRACT(epoch FROM now()) * 1000)::bigint + workflow_timeout_ms
@@ -3550,7 +3561,7 @@ class SystemDatabase {
         return { queues, schedules, versions, workflows: inFlight + terminal, steps };
     }
     // ==================== Internal ====================
-    async insertWorkflowStatus(client, initStatus, ownerXid, incrementAttempts = false) {
+    async insertWorkflowStatus(client, initStatus, ownerXid) {
         try {
             const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
           workflow_uuid,
@@ -3566,9 +3577,7 @@ class SystemDatabase {
           executor_id,
           application_version,
           application_id,
-          created_at,
           recovery_attempts,
-          updated_at,
           workflow_timeout_ms,
           workflow_deadline_epoch_ms,
           inputs,
@@ -3585,21 +3594,16 @@ class SystemDatabase {
           debounce_deadline_epoch_ms,
           is_debounced,
           application_name
-        ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $26, $27, $28, $29, $30, $31, $32, $33)
+        ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)
         ON CONFLICT (workflow_uuid)
           DO UPDATE SET
-            recovery_attempts = CASE
-              WHEN workflow_status.status != '${workflow_1.StatusString.ENQUEUED}' AND workflow_status.status != '${workflow_1.StatusString.DELAYED}'
-              THEN workflow_status.recovery_attempts + $25
-              ELSE workflow_status.recovery_attempts
-            END,
-            updated_at = EXCLUDED.updated_at,
+            updated_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint,
             executor_id = CASE
               WHEN EXCLUDED.status != '${workflow_1.StatusString.ENQUEUED}' AND EXCLUDED.status != '${workflow_1.StatusString.DELAYED}'
               THEN EXCLUDED.executor_id
               ELSE workflow_status.executor_id
             END
-          RETURNING recovery_attempts, status, name, class_name, config_name, queue_name, workflow_deadline_epoch_ms, executor_id, owner_xid, serialization`, [
+          RETURNING status, name, class_name, config_name, queue_name, workflow_deadline_epoch_ms, executor_id, owner_xid, serialization`, [
                 initStatus.workflowUUID,
                 initStatus.status,
                 initStatus.workflowName,
@@ -3614,9 +3618,7 @@ class SystemDatabase {
                 initStatus.executorId,
                 initStatus.applicationVersion ?? null,
                 initStatus.applicationID,
-                initStatus.createdAt,
                 initStatus.status === workflow_1.StatusString.ENQUEUED || initStatus.status === workflow_1.StatusString.DELAYED ? 0 : 1,
-                initStatus.updatedAt ?? Date.now(),
                 initStatus.timeoutMS ?? null,
                 initStatus.deadlineEpochMS ?? null,
                 initStatus.input ?? null,
@@ -3625,7 +3627,6 @@ class SystemDatabase {
                 initStatus.queuePartitionKey ?? null,
                 initStatus.forkedFrom ?? null,
                 initStatus.parentWorkflowID ?? null,
-                (incrementAttempts ?? false) ? 1 : 0,
                 initStatus.serialization,
                 ownerXid,
                 initStatus.delayUntilEpochMS ?? null,
@@ -3957,9 +3958,15 @@ exports.SystemDatabase = SystemDatabase;
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [Object, Object, Object]),
+    __metadata("design:paramtypes", [Object, Object]),
     __metadata("design:returntype", Promise)
 ], SystemDatabase.prototype, "initWorkflowStatus", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array, Number]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "deadLetterWorkflows", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),
@@ -3978,6 +3985,12 @@ __decorate([
     __metadata("design:paramtypes", [String, String, Number]),
     __metadata("design:returntype", Promise)
 ], SystemDatabase.prototype, "getWorkflowStatus", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "fetchWorkflowStatusChunk", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),