@dbos-inc/dbos-sdk 5.4.11-preview → 5.5.2-preview

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -902,6 +902,23 @@ class SystemDatabase {
902
902
  claimGate(_queueName, _partitionKey, _params) {
903
903
  return 'TRUE';
904
904
  }
905
+ /** Statement whose `acquired` says whether a claim took its budget scope's transaction lock; no row means nothing to claim. */
906
+ claimLockQuery(queueName, partitionKey, params) {
907
+ const name = JSON.stringify(['dbos.queue_claim', this.schemaName, queueName, partitionKey ?? null]);
908
+ params.push(advisoryLockKey(name).toString());
909
+ return `SELECT pg_try_advisory_xact_lock($${params.length}) AS acquired`;
910
+ }
911
+ /** Statement whose `held` lists which of `partitionKeys` a sweep's transaction locked; no row means nothing to claim. */
912
+ partitionLockQuery(queueName, partitionKeys, params) {
913
+ const name = JSON.stringify(['dbos.partition_claim', this.schemaName, queueName]);
914
+ // Two-int keys: the queue, then the partition, hashed so every worker computes the same lock.
915
+ const lockIDs = partitionKeys.map((pk) => (0, crypto_1.createHash)('sha256').update(pk).digest().readInt32BE(0));
916
+ params.push(partitionKeys, lockIDs, Number(BigInt.asIntN(32, advisoryLockKey(name))));
917
+ const n = params.length;
918
+ return `SELECT coalesce(array_agg(p.pk), '{}') AS held
919
+ FROM unnest($${n - 2}::text[], $${n - 1}::int4[]) AS p(pk, lock_id)
920
+ WHERE pg_try_advisory_xact_lock($${n}::int4, p.lock_id)`;
921
+ }
905
922
  /** Predicate confining a statement that names no single workflow to the current shard. */
906
923
  scoped(_alias, _params) {
907
924
  return 'TRUE';
@@ -3210,17 +3227,49 @@ class SystemDatabase {
3210
3227
  queue.partitionConcurrency !== undefined ||
3211
3228
  queue.rateLimit !== undefined ||
3212
3229
  queue.partitionRateLimit !== undefined;
3213
- // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it.
3230
+ // Shares that budget across partitions too, so sweeps of different partitions read disjoint rows and could each spend it: lock the whole queue.
3214
3231
  const hasWriteSkew = queuePartitionKey !== undefined && (queue.concurrency !== undefined || queue.rateLimit !== undefined);
3232
+ // Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
3233
+ let maxTasks = Infinity;
3234
+ // The first limit found with no room, which the caller reads as why nothing was claimed.
3235
+ let stopReason = undefined;
3236
+ const limit = (room, reason) => {
3237
+ maxTasks = Math.min(maxTasks, room);
3238
+ if (maxTasks <= 0)
3239
+ stopReason ??= reason;
3240
+ };
3241
+ // This worker's own limits need no database, so a full worker returns before taking the claim lock a peer could use.
3242
+ if (queue.workerConcurrency !== undefined) {
3243
+ // Use the in-memory registry for this worker's running count — avoids a DB round trip.
3244
+ limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
3245
+ }
3246
+ if (queue.partitionWorkerConcurrency !== undefined) {
3247
+ limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
3248
+ }
3249
+ if (maxTasks <= 0) {
3250
+ return { claimedIDs, stopReason };
3251
+ }
3215
3252
  // A sharded database keeps the version row under another key, which the claim's transaction cannot span.
3216
3253
  if (isLatestVersion === undefined && this.sharded) {
3217
3254
  isLatestVersion = await this.isLatestVersion(appVersion);
3218
3255
  }
3256
+ // Row locks only collide when claimers see the same queue head, so serialize claims on the budget's scope; CockroachDB has no advisory locks before 26.3.
3257
+ const useClaimLock = hasSharedBudget && (this.sharded || !(await this.#isCockroach()));
3219
3258
  const client = await this.connect();
3220
3259
  try {
3221
- // Default to READ COMMITTED except with a budget shared across executors
3222
- if (hasSharedBudget) {
3223
- await client.query(`BEGIN ISOLATION LEVEL ${hasWriteSkew ? 'SERIALIZABLE' : 'REPEATABLE READ'}`);
3260
+ if (useClaimLock) {
3261
+ await client.query('BEGIN');
3262
+ const lockParams = [];
3263
+ const { rows } = await client.query(this.claimLockQuery(queue.name, hasWriteSkew ? undefined : queuePartitionKey, lockParams), lockParams);
3264
+ if (rows[0]?.acquired !== true) {
3265
+ await client.query('COMMIT');
3266
+ // Another worker is claiming from this scope, so it still has work; the next poll retries.
3267
+ return rows.length > 0 ? { claimedIDs, stopReason: 'claim_lock' } : { claimedIDs };
3268
+ }
3269
+ }
3270
+ else if (hasSharedBudget) {
3271
+ // CockroachDB, without advisory locks before 26.3: SERIALIZABLE aborts claimers that spend the same budget.
3272
+ await client.query('BEGIN ISOLATION LEVEL SERIALIZABLE');
3224
3273
  }
3225
3274
  else {
3226
3275
  await client.query('BEGIN');
@@ -3254,26 +3303,6 @@ class SystemDatabase {
3254
3303
  WHERE queue_name = $1 AND status = $2 AND ${scope} ${partitionFilter}`, params);
3255
3304
  return Number(rows[0]?.count ?? 0);
3256
3305
  };
3257
- // Compute maxTasks, the number of workflows startable under every flow control limit on this queue.
3258
- let maxTasks = Infinity;
3259
- // The first limit found with no room, which the caller reads as why nothing was claimed.
3260
- let stopReason = undefined;
3261
- const limit = (room, reason) => {
3262
- maxTasks = Math.min(maxTasks, room);
3263
- if (maxTasks <= 0)
3264
- stopReason ??= reason;
3265
- };
3266
- if (queue.workerConcurrency !== undefined) {
3267
- // Use the in-memory registry for this worker's running count — avoids a DB round trip.
3268
- limit(Math.max(0, queue.workerConcurrency - localRunningCount), 'worker_concurrency');
3269
- }
3270
- if (queue.partitionWorkerConcurrency !== undefined) {
3271
- limit(Math.max(0, queue.partitionWorkerConcurrency - partitionLocalRunningCount), 'partition_worker_concurrency');
3272
- }
3273
- if (maxTasks <= 0) {
3274
- await client.query('COMMIT');
3275
- return { claimedIDs, stopReason };
3276
- }
3277
3306
  if (queue.rateLimit !== undefined) {
3278
3307
  // Bound the claim by the limiter's remaining slots so a backlogged queue locks only what it can start.
3279
3308
  limit(await rateLimitRemaining(queue.rateLimit, false), 'limiter');
@@ -3404,7 +3433,9 @@ class SystemDatabase {
3404
3433
  }
3405
3434
  /** Max heads admitted per sweep: bounds dispatch, not the partition walk; an unconstrained sweep takes the lowest keys, so higher keys can wait under sustained load. */
3406
3435
  partitionedDequeueSweepCap = 8192;
3407
- /** Dequeue each partition's head-of-line workflow in one transaction, at most {@link partitionedDequeueSweepCap} per sweep; only valid for partition-concurrency-1 queues with no queue-wide limit. */
3436
+ /** Max partitions a sweep claims per transaction, which bounds the locks it holds at once and for how long. */
3437
+ partitionedDequeueBatchSize = 64;
3438
+ /** Dequeue each partition's head-of-line workflow, at most {@link partitionedDequeueSweepCap} per sweep in transactions of {@link partitionedDequeueBatchSize}; only valid for partition-concurrency-1 queues with no queue-wide limit. */
3408
3439
  async findAndMarkStartablePartitionedWorkflows(queue, executorID, appVersion, maxTasks = Infinity,
3409
3440
  // Written to every claimed row: the dispatches of this claim own their executions.
3410
3441
  ownerXid = null, isLatestVersion) {
@@ -3421,13 +3452,31 @@ class SystemDatabase {
3421
3452
  if (isLatestVersion === undefined && this.sharded) {
3422
3453
  isLatestVersion = await this.isLatestVersion(appVersion);
3423
3454
  }
3455
+ // Sweeps whose heads differ lock disjoint rows, so a claim also takes its partition's lock; CockroachDB, without advisory locks before 26.3, relies on its SERIALIZABLE default.
3456
+ const usePartitionLocks = this.sharded || !(await this.#isCockroach());
3424
3457
  const client = await this.connect();
3425
3458
  try {
3426
- await client.query('BEGIN');
3427
3459
  isLatestVersion ??= await this.isLatestVersion(appVersion, client);
3428
3460
  const versionClause = (n) => isLatestVersion
3429
3461
  ? `(application_version = $${n} OR application_version IS NULL)`
3430
3462
  : `application_version = $${n}`;
3463
+ // The partition's head: $1 queue, $2 ENQUEUED, `version` the app version, `pk` the partition key expression.
3464
+ const headOf = (pk, version, scope) => `SELECT workflow_uuid
3465
+ FROM "${this.schemaName}".workflow_status
3466
+ WHERE queue_name = $1 AND status = $2
3467
+ AND queue_partition_key = ${pk}
3468
+ AND ${versionClause(version)}
3469
+ AND ${scope}
3470
+ -- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
3471
+ ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
3472
+ LIMIT 1`;
3473
+ // No application filter by design: a mutual-exclusion probe must block on any owner's row.
3474
+ const noPending = (pk, pending, scope) => `NOT EXISTS (
3475
+ SELECT 1
3476
+ FROM "${this.schemaName}".workflow_status
3477
+ WHERE ${scope} AND queue_name = $1 AND status = $${pending}
3478
+ AND queue_partition_key IS NOT NULL AND queue_partition_key = ${pk}
3479
+ )`;
3431
3480
  // This worker's own budget bounds the sweep alongside the cap.
3432
3481
  const sweepLimit = Math.min(this.partitionedDequeueSweepCap, maxTasks);
3433
3482
  // When the worker's budget is the binding constraint, probe partitions in random order to prevent starvation.
@@ -3442,6 +3491,7 @@ class SystemDatabase {
3442
3491
  const candidateScope = `${this.scoped(undefined, candidateParams)} AND ${this.appNameFilter('application_name', this.appName, candidateParams)}`;
3443
3492
  const probeScope = this.scoped(undefined, candidateParams);
3444
3493
  // Walk distinct partition keys with a recursive-CTE loose index scan (one seek per key, mirroring getQueuePartitions) so sweep cost scales with partition count, not backlog depth.
3494
+ // The walk takes no lock: each batch below picks its heads again under its partitions' locks.
3445
3495
  const candidateResult = await client.query(`WITH RECURSIVE partitions AS (
3446
3496
  (SELECT MIN(queue_partition_key) AS pk
3447
3497
  FROM "${this.schemaName}".workflow_status
@@ -3457,38 +3507,17 @@ class SystemDatabase {
3457
3507
  SELECT partitions.pk
3458
3508
  FROM partitions
3459
3509
  WHERE partitions.pk IS NOT NULL
3460
- -- No application filter by design: a mutual-exclusion probe must block on any owner's row.
3461
- AND NOT EXISTS (
3462
- SELECT 1
3463
- FROM "${this.schemaName}".workflow_status
3464
- WHERE ${probeScope} AND queue_name = $1 AND status = $4
3465
- AND queue_partition_key IS NOT NULL AND queue_partition_key = partitions.pk
3466
- )
3510
+ AND ${noPending('partitions.pk', 4, probeScope)}
3467
3511
  ORDER BY ${sweepOrder}
3468
3512
  LIMIT $5
3469
3513
  )
3470
- SELECT head.workflow_uuid
3514
+ SELECT chosen.pk
3471
3515
  FROM chosen
3472
- -- LATERAL plans as a tight nested loop; a correlated scalar subquery runs as a slower per-row SubPlan.
3473
- JOIN LATERAL (
3474
- SELECT workflow_uuid
3475
- FROM "${this.schemaName}".workflow_status
3476
- WHERE queue_name = $1 AND status = $2
3477
- AND queue_partition_key = chosen.pk
3478
- AND ${versionClause(3)}
3479
- AND ${candidateScope}
3480
- -- workflow_uuid totalizes the head order (same head for every worker under created_at ties) and the index's trailing workflow_uuid keeps this a pure top-1 probe.
3481
- ORDER BY priority ASC, created_at ASC, workflow_uuid ASC
3482
- LIMIT 1
3483
- ) head ON TRUE
3484
- -- Which partitions were chosen is settled above; order the result so the claim, and the dispatch it feeds, are deterministic.
3516
+ -- Only partitions with a head this worker may run; LATERAL plans as a tight nested loop.
3517
+ JOIN LATERAL (${headOf('chosen.pk', 3, candidateScope)}) head ON TRUE
3518
+ -- Which partitions were chosen is settled above; order them so the claim, and the dispatch it feeds, are deterministic.
3485
3519
  ORDER BY chosen.pk ASC`, candidateParams);
3486
- const candidateIDs = candidateResult.rows.map((row) => row.workflow_uuid);
3487
- if (candidateIDs.length === 0) {
3488
- await client.query('COMMIT');
3489
- return [];
3490
- }
3491
- await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
3520
+ const partitionKeys = candidateResult.rows.map((row) => row.pk);
3492
3521
  // Re-check queue/partition/version alongside status so a row resumeWorkflows moved to another queue mid-sweep is dropped, not hijacked.
3493
3522
  const claimGuard = (ids, status, name, version, scope) => `${ids}
3494
3523
  AND status = $${status}
@@ -3496,68 +3525,108 @@ class SystemDatabase {
3496
3525
  AND queue_partition_key IS NOT NULL
3497
3526
  AND ${versionClause(version)}
3498
3527
  AND ${scope}`;
3499
- const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
3500
- const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
3501
- const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
3502
- // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
3503
- const lockedResult = await client.query(`SELECT workflow_uuid
3504
- FROM "${this.schemaName}".workflow_status
3505
- WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
3506
- FOR UPDATE SKIP LOCKED`, lockedParams);
3507
- const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
3508
- // Preserve partition order for submission.
3509
- const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
3510
- if (claimIDs.length === 0) {
3511
- await client.query('COMMIT');
3512
- return [];
3528
+ /** Claim the heads of these partitions inside the open transaction, returning the IDs it marked PENDING. */
3529
+ const claimBatch = async (batch) => {
3530
+ let heldPks = batch;
3531
+ if (usePartitionLocks) {
3532
+ await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_BEFORE_LOCKS);
3533
+ const lockParams = [];
3534
+ const { rows } = await client.query(this.partitionLockQuery(queue.name, batch, lockParams), lockParams);
3535
+ // Partitions another sweep holds are left to it, or to the next poll.
3536
+ heldPks = rows[0]?.held ?? [];
3537
+ if (heldPks.length === 0)
3538
+ return [];
3539
+ }
3540
+ // Pick the heads again under the locks, so a previous holder's claims and any newly visible head are seen.
3541
+ const headParams = [queue.name, workflow_1.StatusString.ENQUEUED, appVersion, workflow_1.StatusString.PENDING];
3542
+ const headScope = `${this.scoped(undefined, headParams)} AND ${this.appNameFilter('application_name', this.appName, headParams)}`;
3543
+ const headProbeScope = this.scoped(undefined, headParams);
3544
+ headParams.push(heldPks);
3545
+ const headResult = await client.query(`SELECT head.workflow_uuid
3546
+ FROM unnest($${headParams.length}::text[]) AS chosen(pk)
3547
+ JOIN LATERAL (${headOf('chosen.pk', 3, headScope)}) head ON TRUE
3548
+ WHERE ${noPending('chosen.pk', 4, headProbeScope)}
3549
+ ORDER BY chosen.pk ASC`, headParams);
3550
+ const candidateIDs = headResult.rows.map((row) => row.workflow_uuid);
3551
+ if (candidateIDs.length === 0)
3552
+ return [];
3553
+ await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_PARTITIONED_DEQUEUE_AFTER_CANDIDATES);
3554
+ const lockedParams = [workflow_1.StatusString.ENQUEUED, queue.name, appVersion];
3555
+ const lockedIDsFilter = this.wfIn(undefined, candidateIDs, lockedParams);
3556
+ const lockedScope = this.appNameFilter('application_name', this.appName, lockedParams);
3557
+ // Lock the fixed candidate set — never a LIMIT query, whose SKIP LOCKED could slide past a locked head and admit out of order.
3558
+ const lockedResult = await client.query(`SELECT workflow_uuid
3559
+ FROM "${this.schemaName}".workflow_status
3560
+ WHERE ${claimGuard(lockedIDsFilter, 1, 2, 3, lockedScope)}
3561
+ FOR UPDATE SKIP LOCKED`, lockedParams);
3562
+ const lockedIDs = new Set(lockedResult.rows.map((row) => row.workflow_uuid));
3563
+ // Preserve partition order for submission.
3564
+ const claimIDs = candidateIDs.filter((id) => lockedIDs.has(id));
3565
+ if (claimIDs.length === 0)
3566
+ return [];
3567
+ const flipParams = [
3568
+ workflow_1.StatusString.PENDING,
3569
+ executorID,
3570
+ workflow_1.StatusString.ENQUEUED,
3571
+ queue.name,
3572
+ appVersion,
3573
+ // Claim the row, as the unpartitioned dequeue does.
3574
+ this.appName ?? null,
3575
+ ownerXid,
3576
+ startTimeMs,
3577
+ ];
3578
+ const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
3579
+ const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
3580
+ // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
3581
+ const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
3582
+ SET status = $1,
3583
+ owner_xid = $7,
3584
+ executor_id = $2,
3585
+ application_version = $5,
3586
+ started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3587
+ rate_limited = FALSE,
3588
+ application_name = COALESCE(application_name, $6),
3589
+ recovery_attempts = recovery_attempts + 1,
3590
+ updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3591
+ workflow_deadline_epoch_ms = CASE
3592
+ WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
3593
+ CASE
3594
+ WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
3595
+ THEN 9223372036854775807
3596
+ ELSE $8::bigint + workflow_timeout_ms
3597
+ END
3598
+ ELSE workflow_deadline_epoch_ms
3599
+ END
3600
+ WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
3601
+ RETURNING workflow_uuid`, flipParams);
3602
+ const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
3603
+ return claimIDs.filter((id) => flippedIDs.has(id));
3604
+ };
3605
+ // One short transaction per batch, so a sweep holds at most a batch's locks, and only while it claims that batch.
3606
+ const claimedIDs = [];
3607
+ for (let start = 0; start < partitionKeys.length; start += this.partitionedDequeueBatchSize) {
3608
+ const batch = partitionKeys.slice(start, start + this.partitionedDequeueBatchSize);
3609
+ try {
3610
+ await client.query('BEGIN');
3611
+ const batchIDs = await claimBatch(batch);
3612
+ await client.query('COMMIT');
3613
+ claimedIDs.push(...batchIDs);
3614
+ }
3615
+ catch (error) {
3616
+ // Swallowed: a rollback failure must not mask why we are rolling back.
3617
+ await client.query('ROLLBACK').catch(() => undefined);
3618
+ // Earlier batches committed their claims, which must still reach dispatch; the next poll retries the rest.
3619
+ if (claimedIDs.length === 0)
3620
+ throw error;
3621
+ this.logger.warn(`[${queue.name}] stopping a partitioned sweep after ${claimedIDs.length} claim(s): ${String(error)}`);
3622
+ break;
3623
+ }
3513
3624
  }
3514
- const flipParams = [
3515
- workflow_1.StatusString.PENDING,
3516
- executorID,
3517
- workflow_1.StatusString.ENQUEUED,
3518
- queue.name,
3519
- appVersion,
3520
- // Claim the row, as the unpartitioned dequeue does.
3521
- this.appName ?? null,
3522
- ownerXid,
3523
- startTimeMs,
3524
- ];
3525
- const flipIDsFilter = this.wfIn(undefined, claimIDs, flipParams);
3526
- const flipScope = this.appNameFilter('application_name', this.appName, flipParams);
3527
- // Start the workflows by marking them PENDING; RETURNING reports exactly the rows this statement flipped.
3528
- const flippedResult = await client.query(`UPDATE "${this.schemaName}".workflow_status
3529
- SET status = $1,
3530
- owner_xid = $7,
3531
- executor_id = $2,
3532
- application_version = $5,
3533
- started_at_epoch_ms = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3534
- rate_limited = FALSE,
3535
- application_name = COALESCE(application_name, $6),
3536
- recovery_attempts = recovery_attempts + 1,
3537
- updated_at = (EXTRACT(epoch FROM now()) * 1000)::bigint,
3538
- workflow_deadline_epoch_ms = CASE
3539
- WHEN workflow_timeout_ms IS NOT NULL AND workflow_deadline_epoch_ms IS NULL THEN
3540
- CASE
3541
- WHEN workflow_timeout_ms > 9223372036854775807 - $8::bigint
3542
- THEN 9223372036854775807
3543
- ELSE $8::bigint + workflow_timeout_ms
3544
- END
3545
- ELSE workflow_deadline_epoch_ms
3546
- END
3547
- WHERE ${claimGuard(flipIDsFilter, 3, 4, 5, flipScope)}
3548
- RETURNING workflow_uuid`, flipParams);
3549
- await client.query('COMMIT');
3550
- const flippedIDs = new Set(flippedResult.rows.map((row) => row.workflow_uuid));
3551
- const claimedIDs = claimIDs.filter((id) => flippedIDs.has(id));
3552
3625
  if (claimedIDs.length > 0) {
3553
3626
  this.logger.debug(`[${queue.name}] dequeueing ${claimedIDs.length} task(s)`);
3554
3627
  }
3555
3628
  return claimedIDs;
3556
3629
  }
3557
- catch (error) {
3558
- await client.query('ROLLBACK');
3559
- throw error;
3560
- }
3561
3630
  finally {
3562
3631
  client.release();
3563
3632
  }