bunqueue 2.9.0 → 2.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/README.md +7 -3
  2. package/dist/application/background/recovery/active.d.ts +1 -0
  3. package/dist/application/background/recovery/active.js +12 -0
  4. package/dist/application/background/recovery/index.js +3 -1
  5. package/dist/application/background/recovery/pending.js +1 -0
  6. package/dist/application/cleanupTasks.js +4 -16
  7. package/dist/application/contextFactory.js +3 -0
  8. package/dist/application/dlqManager.js +1 -0
  9. package/dist/application/dlqRetry.js +1 -0
  10. package/dist/application/eventsManager.d.ts +8 -1
  11. package/dist/application/eventsManager.js +76 -29
  12. package/dist/application/operations/ackHelpers.d.ts +1 -4
  13. package/dist/application/operations/ackHelpers.js +39 -14
  14. package/dist/application/operations/flowPush.js +1 -0
  15. package/dist/application/operations/flowValidation.js +4 -4
  16. package/dist/application/operations/pull.d.ts +7 -2
  17. package/dist/application/operations/pull.js +15 -57
  18. package/dist/application/operations/pullFinalization.d.ts +6 -0
  19. package/dist/application/operations/pullFinalization.js +69 -0
  20. package/dist/application/operations/pullStateTransition.d.ts +8 -7
  21. package/dist/application/operations/pullStateTransition.js +25 -18
  22. package/dist/application/operations/push.js +3 -0
  23. package/dist/application/operations/pushBatch.js +14 -7
  24. package/dist/application/operations/pushContext.d.ts +2 -1
  25. package/dist/application/postgres-queue-manager/control.d.ts +4 -0
  26. package/dist/application/postgres-queue-manager/control.js +24 -0
  27. package/dist/application/postgres-queue-manager/delivery.d.ts +5 -4
  28. package/dist/application/postgres-queue-manager/delivery.js +13 -10
  29. package/dist/application/postgres-queue-manager/groupQueries.d.ts +10 -0
  30. package/dist/application/postgres-queue-manager/groupQueries.js +40 -0
  31. package/dist/application/postgres-queue-manager/queries.d.ts +2 -2
  32. package/dist/application/postgres-queue-manager/queries.js +2 -2
  33. package/dist/application/queue-manager/control.js +1 -0
  34. package/dist/application/queue-manager/delivery.d.ts +5 -4
  35. package/dist/application/queue-manager/delivery.js +15 -8
  36. package/dist/application/queue-manager/limits.d.ts +11 -0
  37. package/dist/application/queue-manager/limits.js +61 -0
  38. package/dist/application/queue-manager/state.js +1 -1
  39. package/dist/application/queueTelemetryJournal.d.ts +2 -0
  40. package/dist/application/queueTelemetryJournal.js +23 -0
  41. package/dist/application/types/ack.d.ts +3 -1
  42. package/dist/cli/commandRouter.d.ts +1 -1
  43. package/dist/client/flowPush.js +6 -0
  44. package/dist/client/groupId.d.ts +1 -0
  45. package/dist/client/groupId.js +1 -0
  46. package/dist/client/index.d.ts +1 -1
  47. package/dist/client/jobHelpers.js +1 -0
  48. package/dist/client/queue/operations/add/bulk.js +4 -4
  49. package/dist/client/queue/operations/add/payload.d.ts +1 -0
  50. package/dist/client/queue/operations/add/payload.js +7 -1
  51. package/dist/client/queue/operations/add/single.js +3 -3
  52. package/dist/client/queue/operations/groups.d.ts +21 -0
  53. package/dist/client/queue/operations/groups.js +88 -0
  54. package/dist/client/queue/runtime/queries.d.ts +13 -0
  55. package/dist/client/queue/runtime/queries.js +31 -0
  56. package/dist/client/queue/runtime/state.d.ts +4 -0
  57. package/dist/client/queue/runtime/state.js +1 -0
  58. package/dist/client/types/index.d.ts +2 -2
  59. package/dist/client/types/options.d.ts +4 -0
  60. package/dist/client/types/worker.d.ts +8 -0
  61. package/dist/client/worker/runtime/execution.js +1 -0
  62. package/dist/client/worker/runtime/options.js +3 -0
  63. package/dist/client/worker/types/options.d.ts +2 -1
  64. package/dist/client/worker/workerPull.d.ts +2 -0
  65. package/dist/client/worker/workerPull.js +6 -4
  66. package/dist/domain/job/groupFifoOrder.d.ts +9 -0
  67. package/dist/domain/job/groupFifoOrder.js +26 -0
  68. package/dist/domain/queue/groupLimiterManager.d.ts +23 -0
  69. package/dist/domain/queue/groupLimiterManager.js +107 -0
  70. package/dist/domain/queue/groupScheduler.d.ts +31 -0
  71. package/dist/domain/queue/groupScheduler.js +256 -0
  72. package/dist/domain/queue/groupSchedulerOrder.d.ts +6 -0
  73. package/dist/domain/queue/groupSchedulerOrder.js +30 -0
  74. package/dist/domain/queue/priorityQueue.d.ts +13 -23
  75. package/dist/domain/queue/priorityQueue.js +53 -42
  76. package/dist/domain/queue/shard/keys.d.ts +1 -0
  77. package/dist/domain/queue/shard/keys.js +25 -2
  78. package/dist/domain/queue/shard/lifecycle.js +4 -0
  79. package/dist/domain/queue/shard/limits.d.ts +20 -1
  80. package/dist/domain/queue/shard/limits.js +56 -0
  81. package/dist/domain/queue/shard/state.d.ts +5 -0
  82. package/dist/domain/queue/shard/state.js +9 -1
  83. package/dist/domain/types/commands/core.d.ts +3 -0
  84. package/dist/domain/types/commands/limits.d.ts +41 -0
  85. package/dist/domain/types/commands/union.d.ts +1 -1
  86. package/dist/domain/types/group.d.ts +21 -0
  87. package/dist/domain/types/group.js +78 -0
  88. package/dist/domain/types/priorityQueue.d.ts +1 -0
  89. package/dist/infrastructure/persistence/jobOptionsBlob.d.ts +1 -0
  90. package/dist/infrastructure/persistence/jobOptionsBlob.js +10 -2
  91. package/dist/infrastructure/persistence/migrations.js +11 -0
  92. package/dist/infrastructure/persistence/postgres/admission.js +5 -2
  93. package/dist/infrastructure/persistence/postgres/batchAdmission.js +22 -4
  94. package/dist/infrastructure/persistence/postgres/claimBatch.d.ts +3 -0
  95. package/dist/infrastructure/persistence/postgres/claimBatch.js +16 -3
  96. package/dist/infrastructure/persistence/postgres/claimSelection.d.ts +8 -1
  97. package/dist/infrastructure/persistence/postgres/claimSelection.js +73 -131
  98. package/dist/infrastructure/persistence/postgres/claims.d.ts +8 -1
  99. package/dist/infrastructure/persistence/postgres/claims.js +19 -3
  100. package/dist/infrastructure/persistence/postgres/eventCatchupCursors.d.ts +32 -0
  101. package/dist/infrastructure/persistence/postgres/eventCatchupCursors.js +66 -0
  102. package/dist/infrastructure/persistence/postgres/eventJournalSchema.d.ts +2 -2
  103. package/dist/infrastructure/persistence/postgres/eventJournalSchema.js +16 -2
  104. package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.d.ts +2 -0
  105. package/dist/infrastructure/persistence/postgres/eventPruneWatermarks.js +21 -3
  106. package/dist/infrastructure/persistence/postgres/events.d.ts +3 -3
  107. package/dist/infrastructure/persistence/postgres/events.js +44 -17
  108. package/dist/infrastructure/persistence/postgres/groupClaims.d.ts +15 -0
  109. package/dist/infrastructure/persistence/postgres/groupClaims.js +95 -0
  110. package/dist/infrastructure/persistence/postgres/groupSchema.d.ts +2 -0
  111. package/dist/infrastructure/persistence/postgres/groupSchema.js +155 -0
  112. package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.d.ts +3 -0
  113. package/dist/infrastructure/persistence/postgres/groupSchemaFingerprint.js +127 -0
  114. package/dist/infrastructure/persistence/postgres/groupStateRetention.d.ts +3 -0
  115. package/dist/infrastructure/persistence/postgres/groupStateRetention.js +93 -0
  116. package/dist/infrastructure/persistence/postgres/groups.d.ts +11 -0
  117. package/dist/infrastructure/persistence/postgres/groups.js +117 -0
  118. package/dist/infrastructure/persistence/postgres/queueDestruction.js +4 -0
  119. package/dist/infrastructure/persistence/postgres/runtime.d.ts +0 -1
  120. package/dist/infrastructure/persistence/postgres/runtime.js +3 -1
  121. package/dist/infrastructure/persistence/postgres/schema.d.ts +2 -2
  122. package/dist/infrastructure/persistence/postgres/schema.js +8 -4
  123. package/dist/infrastructure/persistence/postgres/schemaInitialization.js +6 -1
  124. package/dist/infrastructure/persistence/postgres/store.d.ts +11 -1
  125. package/dist/infrastructure/persistence/postgres/store.js +47 -3
  126. package/dist/infrastructure/persistence/postgres/telemetry.d.ts +2 -0
  127. package/dist/infrastructure/persistence/postgres/telemetry.js +33 -21
  128. package/dist/infrastructure/persistence/postgres/types.d.ts +1 -0
  129. package/dist/infrastructure/persistence/schema.d.ts +2 -2
  130. package/dist/infrastructure/persistence/schema.js +12 -1
  131. package/dist/infrastructure/persistence/sqlite/control.d.ts +14 -0
  132. package/dist/infrastructure/persistence/sqlite/control.js +69 -0
  133. package/dist/infrastructure/persistence/sqlite/jobs.d.ts +12 -0
  134. package/dist/infrastructure/persistence/sqlite/jobs.js +36 -0
  135. package/dist/infrastructure/persistence/sqlite/telemetry.d.ts +2 -1
  136. package/dist/infrastructure/persistence/sqlite/telemetry.js +5 -64
  137. package/dist/infrastructure/persistence/sqlite/telemetryWrites.d.ts +4 -0
  138. package/dist/infrastructure/persistence/sqlite/telemetryWrites.js +123 -0
  139. package/dist/infrastructure/persistence/sqliteSerializer.js +2 -0
  140. package/dist/infrastructure/server/handler-routes/control.js +13 -0
  141. package/dist/infrastructure/server/handlers/core.d.ts +0 -5
  142. package/dist/infrastructure/server/handlers/core.js +15 -18
  143. package/dist/infrastructure/server/handlers/groups.d.ts +11 -0
  144. package/dist/infrastructure/server/handlers/groups.js +55 -0
  145. package/dist/infrastructure/server/handlers/pushBatchValidation.js +4 -1
  146. package/dist/infrastructure/server/protocol/validation.d.ts +1 -0
  147. package/dist/infrastructure/server/protocol/validation.js +1 -0
  148. package/dist/infrastructure/server/protocol.d.ts +1 -1
  149. package/dist/infrastructure/server/protocol.js +1 -1
  150. package/package.json +2 -1
@@ -4,7 +4,68 @@ import { shardIndex, SHARD_COUNT } from '../../shared/hash';
4
4
  import * as dlqOps from '../dlqManager';
5
5
  import * as queryOps from '../operations/queryOperations';
6
6
  import { QueueManagerControl } from './control';
7
+ import { assertGroupId, assertPositiveSafeInteger, } from '../../domain/types/group';
7
8
  export class QueueManagerLimits extends QueueManagerControl {
9
+ getGroupJobsCount(queue, groupId) {
10
+ assertGroupId(groupId);
11
+ return this.shards[shardIndex(queue)].getGroupJobsCount(queue, groupId);
12
+ }
13
+ getGroupsJobsCount(queue) {
14
+ return this.shards[shardIndex(queue)].getGroupsJobsCount(queue);
15
+ }
16
+ getGroupActiveCount(queue, groupId) {
17
+ assertGroupId(groupId);
18
+ return this.shards[shardIndex(queue)].getGroupActiveCount(queue, groupId);
19
+ }
20
+ setGroupRateLimit(queue, groupId, max, duration) {
21
+ assertGroupId(groupId);
22
+ assertPositiveSafeInteger(max, 'max');
23
+ assertPositiveSafeInteger(duration, 'duration');
24
+ const shard = this.shards[shardIndex(queue)];
25
+ this.storage?.saveGroupRateLimit(queue, groupId, max, duration);
26
+ shard.setGroupRateLimit(queue, groupId, max, duration);
27
+ shard.notifyBatch(queue, Math.max(1, shard.getGroupJobsCount(queue, groupId)));
28
+ }
29
+ getGroupRateLimit(queue, groupId) {
30
+ assertGroupId(groupId);
31
+ return this.shards[shardIndex(queue)].getGroupRateLimit(queue, groupId);
32
+ }
33
+ removeGroupRateLimit(queue, groupId) {
34
+ assertGroupId(groupId);
35
+ const shard = this.shards[shardIndex(queue)];
36
+ if (!shard.getGroupRateLimit(queue, groupId))
37
+ return 0;
38
+ this.storage?.removeGroupRateLimit(queue, groupId);
39
+ const removed = shard.removeGroupRateLimit(queue, groupId);
40
+ shard.notifyBatch(queue, Math.max(1, shard.getGroupJobsCount(queue, groupId)));
41
+ return removed;
42
+ }
43
+ getGroupRateLimitTtl(queue, groupId, maxJobs) {
44
+ assertGroupId(groupId);
45
+ return this.shards[shardIndex(queue)].getGroupRateLimitTtl(queue, groupId, maxJobs);
46
+ }
47
+ setGroupConcurrency(queue, groupId, concurrency) {
48
+ assertGroupId(groupId);
49
+ assertPositiveSafeInteger(concurrency, 'concurrency');
50
+ const shard = this.shards[shardIndex(queue)];
51
+ this.storage?.saveGroupConcurrency(queue, groupId, concurrency);
52
+ shard.setGroupConcurrency(queue, groupId, concurrency);
53
+ shard.notifyBatch(queue, Math.max(1, shard.getGroupJobsCount(queue, groupId)));
54
+ }
55
+ getGroupConcurrency(queue, groupId) {
56
+ assertGroupId(groupId);
57
+ return this.shards[shardIndex(queue)].getGroupConcurrency(queue, groupId);
58
+ }
59
+ removeGroupConcurrency(queue, groupId) {
60
+ assertGroupId(groupId);
61
+ const shard = this.shards[shardIndex(queue)];
62
+ if (shard.getGroupConcurrency(queue, groupId) === null)
63
+ return 0;
64
+ this.storage?.removeGroupConcurrency(queue, groupId);
65
+ const removed = shard.removeGroupConcurrency(queue, groupId);
66
+ shard.notifyBatch(queue, Math.max(1, shard.getGroupJobsCount(queue, groupId)));
67
+ return removed;
68
+ }
8
69
  getCountsPerPriority(queue) {
9
70
  const counts = this.shards[shardIndex(queue)].getCountsPerPriority(queue);
10
71
  return Object.fromEntries(counts);
@@ -135,7 +135,7 @@ export class QueueManagerState {
135
135
  this.workerManager = new WorkerManager();
136
136
  this.cronScheduler.setWorkerCheckCallback((queue) => this.workerManager.getForQueue(queue).length > 0);
137
137
  this.eventsManager = new EventsManager(this.webhookManager);
138
- this.eventsManager.subscribe((event) => this.telemetryJournal.record(event));
138
+ this.eventsManager.subscribe((event) => this.telemetryJournal.record(event), (events) => this.telemetryJournal.recordBatch(events));
139
139
  this.contextFactory = new ContextFactory(createContextDependencies(managerRuntime(this)), createContextCallbacks(runtime));
140
140
  bgTasks.recover(this.contextFactory.getBackgroundContext());
141
141
  if (this.storage) {
@@ -10,6 +10,8 @@ export declare class QueueTelemetryJournal {
10
10
  private readonly maxMetricDataPoints;
11
11
  constructor(storage: SqliteStorage | null, maxEvents: number, maxMetricDataPoints: number);
12
12
  record(event: JobEvent): void;
13
+ /** Persist a lifecycle batch atomically, with scalar fallback on write failure. */
14
+ recordBatch(events: readonly JobEvent[]): void;
13
15
  getMetrics(queue: string, type: QueueMetricType, start?: number, end?: number): QueueMetrics;
14
16
  trimEvents(queue: string, maxLength: number): number;
15
17
  clearQueue(queue: string): void;
@@ -55,6 +55,29 @@ export class QueueTelemetryJournal {
55
55
  queueMetrics.set(type, state);
56
56
  this.metrics.set(event.queue, queueMetrics);
57
57
  }
58
+ /** Persist a lifecycle batch atomically, with scalar fallback on write failure. */
59
+ recordBatch(events) {
60
+ if (events.length === 0)
61
+ return;
62
+ if (this.storage) {
63
+ try {
64
+ this.storage.recordQueueEventsBatch(events, this.maxEvents, this.maxMetricDataPoints);
65
+ }
66
+ catch {
67
+ for (const event of events) {
68
+ try {
69
+ this.storage.recordQueueEvent(event, this.maxEvents, this.maxMetricDataPoints);
70
+ }
71
+ catch {
72
+ // Match scalar subscriber isolation and continue with later events.
73
+ }
74
+ }
75
+ }
76
+ return;
77
+ }
78
+ for (const event of events)
79
+ this.record(event);
80
+ }
58
81
  getMetrics(queue, type, start = 0, end = -1) {
59
82
  assertNonNegativeInteger(start, 'start');
60
83
  if (end !== -1)
@@ -2,7 +2,7 @@ import type { Shard } from '../../domain/queue/shard';
2
2
  import type { FlowFailureRecord } from '../../domain/types/flow';
3
3
  import type { FailureReason } from '../../domain/types/dlq';
4
4
  import type { Job, JobId } from '../../domain/types/job';
5
- import type { EventType, JobLocation } from '../../domain/types/queue';
5
+ import type { EventType, JobEvent, JobLocation } from '../../domain/types/queue';
6
6
  import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
7
7
  import type { RWLock } from '../../shared/lock';
8
8
  import type { MapLike, SetLike } from '../../shared/lru';
@@ -43,6 +43,7 @@ export interface AckContext {
43
43
  prev?: string;
44
44
  terminal?: boolean;
45
45
  }) => void;
46
+ broadcastBatch?: (events: readonly JobEvent[]) => void;
46
47
  onJobCompleted: (jobId: JobId) => void;
47
48
  onJobFailed?: (jobId: JobId) => void;
48
49
  onJobsCompleted?: (jobIds: JobId[]) => void;
@@ -107,6 +108,7 @@ export interface FinalizeContext {
107
108
  timestamp: number;
108
109
  data?: unknown;
109
110
  }) => void;
111
+ broadcastBatch?: (events: readonly JobEvent[]) => void;
110
112
  onJobCompleted: (jobId: JobId) => void;
111
113
  onJobsCompleted?: (jobIds: JobId[]) => void;
112
114
  needsBroadcast?: () => boolean;
@@ -1,3 +1,3 @@
1
- export declare const CLI_NETWORK_COMMANDS: readonly ("push" | "queue" | "cron" | "dlq" | "job" | "fail" | "worker" | "concurrency" | "stats" | "ping" | "metrics" | "pull" | "ack" | "rate-limit" | "health" | "webhook")[];
1
+ export declare const CLI_NETWORK_COMMANDS: readonly ("push" | "queue" | "cron" | "concurrency" | "job" | "dlq" | "fail" | "worker" | "stats" | "ping" | "metrics" | "pull" | "ack" | "rate-limit" | "health" | "webhook")[];
2
2
  /** Build the protocol command for a routed network CLI command. */
3
3
  export declare function buildCommand(command: string, args: string[]): Promise<Record<string, unknown> | null>;
@@ -4,6 +4,10 @@
4
4
  */
5
5
  import { getSharedManager } from './manager';
6
6
  import { jobId } from '../domain/types/job';
7
+ import { normalizeGroupId } from './groupId';
8
+ function groupId(opts) {
9
+ return opts.group ? normalizeGroupId(opts.group.id) : undefined;
10
+ }
7
11
  /** Parse removeOnComplete/removeOnFail options */
8
12
  function parseRemoveOptions(opts) {
9
13
  return {
@@ -67,6 +71,7 @@ function managerOptions(opts) {
67
71
  removeDependencyOnFailure: opts.removeDependencyOnFailure,
68
72
  ignoreDependencyOnFailure: opts.ignoreDependencyOnFailure,
69
73
  continueParentOnFailure: opts.continueParentOnFailure,
74
+ groupId: groupId(opts),
70
75
  };
71
76
  }
72
77
  /** Full option set for the TCP PUSH command, mirroring buildPushPayload in add.ts (#90). */
@@ -94,6 +99,7 @@ function tcpOptions(opts) {
94
99
  removeDependencyOnFailure: opts.removeDependencyOnFailure,
95
100
  ignoreDependencyOnFailure: opts.ignoreDependencyOnFailure,
96
101
  continueParentOnFailure: opts.continueParentOnFailure,
102
+ groupId: groupId(opts),
97
103
  };
98
104
  }
99
105
  /** Push a job via embedded manager or TCP */
@@ -0,0 +1 @@
1
+ export { normalizeGroupId } from '../domain/types/group';
@@ -0,0 +1 @@
1
+ export { normalizeGroupId } from '../domain/types/group';
@@ -38,7 +38,7 @@ export { shutdownManager } from './manager';
38
38
  export { closeSharedTcpClient } from './tcpClient';
39
39
  export type { ConnectionHealth } from './tcpClient';
40
40
  export { TcpConnectionPool, getSharedPool, closeAllSharedPools } from './tcpPool';
41
- export type { Job, JobOptions, JobJson, JobJsonRaw, QueueOptions, WorkerOptions, Processor, FlowJobData, StallConfig, DlqConfig, DlqEntry, DlqStats, DlqFilter, FailureReason, ConnectionOptions, ParentOpts, RateLimiterOptions, ChangePriorityOpts, GetDependenciesOpts, JobDependencies, JobDependenciesCount, QueueMetrics, QueueMetricsMeta, QueueMetricType, } from './types';
41
+ export type { Job, JobOptions, GroupJobOptions, GroupWorkerOptions, JobJson, JobJsonRaw, QueueOptions, WorkerOptions, Processor, FlowJobData, StallConfig, DlqConfig, DlqEntry, DlqStats, DlqFilter, FailureReason, ConnectionOptions, ParentOpts, RateLimiterOptions, ChangePriorityOpts, GetDependenciesOpts, JobDependencies, JobDependenciesCount, QueueMetrics, QueueMetricsMeta, QueueMetricType, } from './types';
42
42
  export type { FlowStep, FlowResult, FlowJob, JobNode, FlowProducerOptions, FlowOpts } from './flow';
43
43
  export type { RepeatOpts, JobTemplate, SchedulerInfo } from './queue/scheduler';
44
44
  export type { SandboxedWorkerOptions } from './sandboxedWorker';
@@ -97,6 +97,7 @@ export function buildJobOpts(job) {
97
97
  debounce: job.debounceId && job.debounceTtl !== null
98
98
  ? { id: job.debounceId, ttl: job.debounceTtl }
99
99
  : undefined,
100
+ group: job.groupId ? { id: job.groupId } : undefined,
100
101
  };
101
102
  }
102
103
  /** Build parent key from job */
@@ -1,7 +1,7 @@
1
1
  import { getSharedManager } from '../../../manager';
2
2
  import { jobId } from '../../../../domain/types/job';
3
3
  import { createJobProxy, createSimpleJob } from '../../jobProxy';
4
- import { buildJobData, buildRepeatOptions, compact, reflectionMeta } from './payload';
4
+ import { buildJobData, buildRepeatOptions, compact, reflectionMeta, resolveGroupId, } from './payload';
5
5
  export async function addBulk(context, jobs) {
6
6
  if (jobs.length === 0)
7
7
  return [];
@@ -11,7 +11,7 @@ export async function addBulk(context, jobs) {
11
11
  ...opts,
12
12
  }));
13
13
  if (context.embedded) {
14
- const manager = getSharedManager();
14
+ const manager = getSharedManager(context.opts.dataPath);
15
15
  const inputs = jobs.map(({ name, data }, index) => {
16
16
  const options = merged[index];
17
17
  const removeOnComplete = typeof options.removeOnComplete === 'boolean' ? options.removeOnComplete : false;
@@ -30,7 +30,7 @@ export async function addBulk(context, jobs) {
30
30
  dependsOn: options.dependsOn?.map((id) => jobId(id)),
31
31
  parentId: options.parent ? jobId(options.parent.id) : undefined,
32
32
  tags: options.tags,
33
- groupId: options.groupId,
33
+ groupId: resolveGroupId(options),
34
34
  stallTimeout: options.stallTimeout,
35
35
  timestamp: options.timestamp,
36
36
  removeOnComplete,
@@ -84,7 +84,7 @@ export async function addBulk(context, jobs) {
84
84
  ttl: options.ttl,
85
85
  customId: options.jobId,
86
86
  tags: options.tags,
87
- groupId: options.groupId,
87
+ groupId: resolveGroupId(options),
88
88
  dependsOn: options.dependsOn?.map((id) => jobId(id)),
89
89
  parentId: options.parent ? jobId(options.parent.id) : undefined,
90
90
  uniqueKey: options.deduplication?.id,
@@ -16,6 +16,7 @@ export declare function buildRepeatOptions(repeat: NonNullable<JobOptions['repea
16
16
  jobId: string | undefined;
17
17
  };
18
18
  export declare function buildPushPayload(queue: string, name: string, data: unknown, options: ExtendedJobOptions): Record<string, unknown>;
19
+ export declare function resolveGroupId(options: ExtendedJobOptions): string | undefined;
19
20
  export declare function buildJobData(data: unknown, options: ExtendedJobOptions): unknown;
20
21
  export declare function reflectionMeta(options: ExtendedJobOptions): {
21
22
  priority?: number;
@@ -1,3 +1,4 @@
1
+ import { normalizeGroupId } from '../../../groupId';
1
2
  export function compact(object) {
2
3
  const output = {};
3
4
  for (const key in object) {
@@ -51,7 +52,7 @@ export function buildPushPayload(queue, name, data, options) {
51
52
  : undefined,
52
53
  dependsOn: options.dependsOn,
53
54
  tags: options.tags,
54
- groupId: options.groupId,
55
+ groupId: resolveGroupId(options),
55
56
  lifo: options.lifo,
56
57
  removeOnComplete: typeof options.removeOnComplete === 'boolean' ? options.removeOnComplete : undefined,
57
58
  removeOnFail: typeof options.removeOnFail === 'boolean' ? options.removeOnFail : undefined,
@@ -71,6 +72,11 @@ export function buildPushPayload(queue, name, data, options) {
71
72
  parentId: options.parent?.id,
72
73
  });
73
74
  }
75
+ export function resolveGroupId(options) {
76
+ if (options.group)
77
+ return normalizeGroupId(options.group.id);
78
+ return options.groupId === undefined ? undefined : normalizeGroupId(options.groupId);
79
+ }
74
80
  export function buildJobData(data, options) {
75
81
  if (!options.parent)
76
82
  return data;
@@ -3,12 +3,12 @@ import { removeJobDeduplicationKey } from '../../../jobDeduplication';
3
3
  import { toPublicJob } from '../../../types';
4
4
  import { jobId } from '../../../../domain/types/job';
5
5
  import { createJobProxy } from '../../jobProxy';
6
- import { buildJobData, buildPushPayload, buildRepeatOptions } from './payload';
6
+ import { buildJobData, buildPushPayload, buildRepeatOptions, resolveGroupId } from './payload';
7
7
  export async function add(context, jobName, data, options = {}) {
8
8
  const merged = { ...context.opts.defaultJobOptions, ...options };
9
9
  const jobData = buildJobData(data, merged);
10
10
  if (context.embedded) {
11
- const manager = getSharedManager();
11
+ const manager = getSharedManager(context.opts.dataPath);
12
12
  const removeOnComplete = typeof merged.removeOnComplete === 'boolean' ? merged.removeOnComplete : false;
13
13
  const removeOnFail = typeof merged.removeOnFail === 'boolean' ? merged.removeOnFail : false;
14
14
  const repeat = merged.repeat ? buildRepeatOptions(merged.repeat) : undefined;
@@ -25,7 +25,7 @@ export async function add(context, jobName, data, options = {}) {
25
25
  customId: merged.jobId,
26
26
  dependsOn: merged.dependsOn?.map((id) => jobId(id)),
27
27
  tags: merged.tags,
28
- groupId: merged.groupId,
28
+ groupId: resolveGroupId(merged),
29
29
  dedup: merged.deduplication
30
30
  ? {
31
31
  ttl: merged.deduplication.ttl,
@@ -0,0 +1,21 @@
1
+ import type { TcpConnectionPool } from '../../tcpPool';
2
+ interface GroupContext {
3
+ name: string;
4
+ embedded: boolean;
5
+ tcp: TcpConnectionPool | null;
6
+ dataPath: string | undefined;
7
+ }
8
+ export declare function getGroupJobsCount(ctx: GroupContext, groupId: string): Promise<number>;
9
+ export declare function getGroupsJobsCount(ctx: GroupContext, maxCount?: number): Promise<number>;
10
+ export declare function getGroupActiveCount(ctx: GroupContext, groupId: string): Promise<number>;
11
+ export declare function setGroupRateLimit(ctx: GroupContext, groupId: string, max: number, duration: number): Promise<void>;
12
+ export declare function getGroupRateLimit(ctx: GroupContext, groupId: string): Promise<{
13
+ max: number;
14
+ duration: number;
15
+ } | null>;
16
+ export declare function removeGroupRateLimit(ctx: GroupContext, groupId: string): Promise<number>;
17
+ export declare function getGroupRateLimitTtl(ctx: GroupContext, groupId: string, maxJobs?: number): Promise<number>;
18
+ export declare function setGroupConcurrency(ctx: GroupContext, groupId: string, concurrency: number): Promise<void>;
19
+ export declare function getGroupConcurrency(ctx: GroupContext, groupId: string): Promise<number | null>;
20
+ export declare function removeGroupConcurrency(ctx: GroupContext, groupId: string): Promise<number>;
21
+ export {};
@@ -0,0 +1,88 @@
1
+ import { getSharedManager } from '../../manager';
2
+ import { normalizeGroupId } from '../../groupId';
3
+ function manager(ctx) {
4
+ return getSharedManager(ctx.dataPath);
5
+ }
6
+ async function send(ctx, command) {
7
+ if (!ctx.tcp)
8
+ throw new Error('TCP connection is unavailable for group operation');
9
+ const response = await ctx.tcp.send({ ...command, queue: ctx.name });
10
+ if (!response.ok)
11
+ throw new Error(String(response.error ?? 'Group operation failed'));
12
+ const data = typeof response.data === 'object' && response.data !== null
13
+ ? response.data
14
+ : {};
15
+ return { ...response, ...data };
16
+ }
17
+ export async function getGroupJobsCount(ctx, groupId) {
18
+ const id = normalizeGroupId(groupId);
19
+ if (ctx.embedded)
20
+ return await manager(ctx).getGroupJobsCount(ctx.name, id);
21
+ const response = await send(ctx, { cmd: 'GetGroupJobsCount', groupId: id });
22
+ return Number(response.count ?? 0);
23
+ }
24
+ export async function getGroupsJobsCount(ctx, maxCount = 100) {
25
+ if (ctx.embedded)
26
+ return await manager(ctx).getGroupsJobsCount(ctx.name);
27
+ const response = await send(ctx, { cmd: 'GetGroupsJobsCount', maxCount });
28
+ return Number(response.count ?? 0);
29
+ }
30
+ export async function getGroupActiveCount(ctx, groupId) {
31
+ const id = normalizeGroupId(groupId);
32
+ if (ctx.embedded)
33
+ return await manager(ctx).getGroupActiveCount(ctx.name, id);
34
+ const response = await send(ctx, { cmd: 'GetGroupActiveCount', groupId: id });
35
+ return Number(response.count ?? 0);
36
+ }
37
+ export async function setGroupRateLimit(ctx, groupId, max, duration) {
38
+ const id = normalizeGroupId(groupId);
39
+ if (ctx.embedded) {
40
+ await manager(ctx).setGroupRateLimit(ctx.name, id, max, duration);
41
+ return;
42
+ }
43
+ await send(ctx, { cmd: 'SetGroupRateLimit', groupId: id, max, duration });
44
+ }
45
+ export async function getGroupRateLimit(ctx, groupId) {
46
+ const id = normalizeGroupId(groupId);
47
+ if (ctx.embedded)
48
+ return await manager(ctx).getGroupRateLimit(ctx.name, id);
49
+ const response = await send(ctx, { cmd: 'GetGroupRateLimit', groupId: id });
50
+ return response.limit ?? null;
51
+ }
52
+ export async function removeGroupRateLimit(ctx, groupId) {
53
+ const id = normalizeGroupId(groupId);
54
+ if (ctx.embedded)
55
+ return await manager(ctx).removeGroupRateLimit(ctx.name, id);
56
+ const response = await send(ctx, { cmd: 'RemoveGroupRateLimit', groupId: id });
57
+ return Number(response.removed ?? 0);
58
+ }
59
+ export async function getGroupRateLimitTtl(ctx, groupId, maxJobs) {
60
+ const id = normalizeGroupId(groupId);
61
+ if (ctx.embedded) {
62
+ return await manager(ctx).getGroupRateLimitTtl(ctx.name, id, maxJobs);
63
+ }
64
+ const response = await send(ctx, { cmd: 'GetGroupRateLimitTtl', groupId: id, maxJobs });
65
+ return Number(response.ttl ?? -2);
66
+ }
67
+ export async function setGroupConcurrency(ctx, groupId, concurrency) {
68
+ const id = normalizeGroupId(groupId);
69
+ if (ctx.embedded) {
70
+ await manager(ctx).setGroupConcurrency(ctx.name, id, concurrency);
71
+ return;
72
+ }
73
+ await send(ctx, { cmd: 'SetGroupConcurrency', groupId: id, concurrency });
74
+ }
75
+ export async function getGroupConcurrency(ctx, groupId) {
76
+ const id = normalizeGroupId(groupId);
77
+ if (ctx.embedded)
78
+ return await manager(ctx).getGroupConcurrency(ctx.name, id);
79
+ const response = await send(ctx, { cmd: 'GetGroupConcurrency', groupId: id });
80
+ return typeof response.concurrency === 'number' ? response.concurrency : null;
81
+ }
82
+ export async function removeGroupConcurrency(ctx, groupId) {
83
+ const id = normalizeGroupId(groupId);
84
+ if (ctx.embedded)
85
+ return await manager(ctx).removeGroupConcurrency(ctx.name, id);
86
+ const response = await send(ctx, { cmd: 'RemoveGroupConcurrency', groupId: id });
87
+ return Number(response.removed ?? 0);
88
+ }
@@ -3,6 +3,19 @@ import * as countsOps from '../operations/counts';
3
3
  import { QueueState } from './state';
4
4
  /** Job creation, lookup, state-listing, and count operations. */
5
5
  export declare class QueueQueries<T> extends QueueState<T> {
6
+ getGroupJobsCount(groupId: string): Promise<number>;
7
+ getGroupsJobsCount(maxCount?: number): Promise<number>;
8
+ getGroupActiveCount(groupId: string): Promise<number>;
9
+ setGroupRateLimit(groupId: string, max: number, duration: number): Promise<void>;
10
+ getGroupRateLimit(groupId: string): Promise<{
11
+ max: number;
12
+ duration: number;
13
+ } | null>;
14
+ removeGroupRateLimit(groupId: string): Promise<number>;
15
+ getGroupRateLimitTtl(groupId: string, maxJobs?: number): Promise<number>;
16
+ setGroupConcurrency(groupId: string, concurrency: number): Promise<void>;
17
+ getGroupConcurrency(groupId: string): Promise<number | null>;
18
+ removeGroupConcurrency(groupId: string): Promise<number>;
6
19
  add(name: string, data: T, opts?: JobOptions): Promise<Job<T>>;
7
20
  addBulk(jobs: Array<{
8
21
  name: string;
@@ -2,9 +2,40 @@ import * as countsOps from '../operations/counts';
2
2
  import * as addOps from '../operations/add';
3
3
  import * as queryOps from '../operations/query';
4
4
  import * as queryStateOps from '../operations/queryStates';
5
+ import * as groupOps from '../operations/groups';
5
6
  import { QueueState } from './state';
6
7
  /** Job creation, lookup, state-listing, and count operations. */
7
8
  export class QueueQueries extends QueueState {
9
+ getGroupJobsCount(groupId) {
10
+ return groupOps.getGroupJobsCount(this.ctx, groupId);
11
+ }
12
+ getGroupsJobsCount(maxCount) {
13
+ return groupOps.getGroupsJobsCount(this.ctx, maxCount);
14
+ }
15
+ getGroupActiveCount(groupId) {
16
+ return groupOps.getGroupActiveCount(this.ctx, groupId);
17
+ }
18
+ setGroupRateLimit(groupId, max, duration) {
19
+ return groupOps.setGroupRateLimit(this.ctx, groupId, max, duration);
20
+ }
21
+ getGroupRateLimit(groupId) {
22
+ return groupOps.getGroupRateLimit(this.ctx, groupId);
23
+ }
24
+ removeGroupRateLimit(groupId) {
25
+ return groupOps.removeGroupRateLimit(this.ctx, groupId);
26
+ }
27
+ getGroupRateLimitTtl(groupId, maxJobs) {
28
+ return groupOps.getGroupRateLimitTtl(this.ctx, groupId, maxJobs);
29
+ }
30
+ setGroupConcurrency(groupId, concurrency) {
31
+ return groupOps.setGroupConcurrency(this.ctx, groupId, concurrency);
32
+ }
33
+ getGroupConcurrency(groupId) {
34
+ return groupOps.getGroupConcurrency(this.ctx, groupId);
35
+ }
36
+ removeGroupConcurrency(groupId) {
37
+ return groupOps.removeGroupConcurrency(this.ctx, groupId);
38
+ }
8
39
  add(name, data, opts) {
9
40
  if (this.addBatcher && !opts?.durable) {
10
41
  return this.addBatcher.enqueue(name, data, opts);
@@ -18,6 +18,7 @@ export declare class QueueState<T> {
18
18
  embedded: boolean;
19
19
  tcp: TcpConnectionPool | null;
20
20
  prefixKey: string;
21
+ dataPath: string | undefined;
21
22
  };
22
23
  protected get addCtx(): {
23
24
  opts: QueueOptions;
@@ -55,6 +56,7 @@ export declare class QueueState<T> {
55
56
  embedded: boolean;
56
57
  tcp: TcpConnectionPool | null;
57
58
  prefixKey: string;
59
+ dataPath: string | undefined;
58
60
  };
59
61
  protected get queryCtx(): {
60
62
  getJobState: (id: string) => Promise<import("../../types").JobStateType>;
@@ -84,6 +86,7 @@ export declare class QueueState<T> {
84
86
  embedded: boolean;
85
87
  tcp: TcpConnectionPool | null;
86
88
  prefixKey: string;
89
+ dataPath: string | undefined;
87
90
  };
88
91
  protected get moveCtx(): {
89
92
  getJobState: (id: string) => Promise<import("../../types").JobStateType>;
@@ -92,6 +95,7 @@ export declare class QueueState<T> {
92
95
  embedded: boolean;
93
96
  tcp: TcpConnectionPool | null;
94
97
  prefixKey: string;
98
+ dataPath: string | undefined;
95
99
  };
96
100
  protected releaseConnection(): void;
97
101
  }
@@ -78,6 +78,7 @@ export class QueueState {
78
78
  embedded: this.embedded,
79
79
  tcp: this.tcpPool,
80
80
  prefixKey: this.prefixKey,
81
+ dataPath: this.opts.dataPath,
81
82
  };
82
83
  }
83
84
  get addCtx() {
@@ -4,7 +4,7 @@ export type { ActiveEvent, CompletedEvent, DelayedEvent, DrainedEvent, Duplicate
4
4
  export type { FlowJobData, Processor, QueueEventType } from './flow';
5
5
  export type { ChangePriorityOpts, GetDependenciesOpts, Job, JobDependencies, JobDependenciesCount, JobJson, JobJsonRaw, JobStateType, } from './job';
6
6
  export type { QueueMetrics, QueueMetricsMeta, QueueMetricType } from './metrics';
7
- export type { BackoffOptions, DebounceOptions, DeduplicationOptions, JobOptions, KeepJobs, ParentOpts, RepeatOptions, } from './options';
8
- export type { RateLimiterOptions, StallConfig, WorkerOptions } from './worker';
7
+ export type { BackoffOptions, DebounceOptions, DeduplicationOptions, GroupJobOptions, JobOptions, KeepJobs, ParentOpts, RepeatOptions, } from './options';
8
+ export type { GroupWorkerOptions, RateLimiterOptions, StallConfig, WorkerOptions } from './worker';
9
9
  export { createPublicJob, toDlqEntry, toPublicJob } from '../jobConversion';
10
10
  export type { CreatePublicJobOptions, ToPublicJobOptions } from '../jobConversion';
@@ -33,6 +33,9 @@ export interface RepeatOptions {
33
33
  offset?: number;
34
34
  jobId?: string;
35
35
  }
36
+ export interface GroupJobOptions {
37
+ id: string | number;
38
+ }
36
39
  /** Options accepted when adding a job. */
37
40
  export interface JobOptions {
38
41
  priority?: number;
@@ -59,4 +62,5 @@ export interface JobOptions {
59
62
  timestamp?: number;
60
63
  deduplication?: DeduplicationOptions;
61
64
  debounce?: DebounceOptions;
65
+ group?: GroupJobOptions;
62
66
  }
@@ -4,6 +4,13 @@ export interface RateLimiterOptions {
4
4
  duration: number;
5
5
  groupKey?: string;
6
6
  }
7
+ export interface GroupWorkerOptions {
8
+ concurrency?: number;
9
+ limit?: {
10
+ max: number;
11
+ duration: number;
12
+ };
13
+ }
7
14
  export interface WorkerOptions {
8
15
  concurrency?: number;
9
16
  autorun?: boolean;
@@ -16,6 +23,7 @@ export interface WorkerOptions {
16
23
  pollTimeout?: number;
17
24
  useLocks?: boolean;
18
25
  limiter?: RateLimiterOptions;
26
+ group?: GroupWorkerOptions;
19
27
  lockDuration?: number;
20
28
  maxStalledCount?: number;
21
29
  skipStalledCheck?: boolean;
@@ -113,6 +113,7 @@ export class WorkerExecution extends WorkerPolling {
113
113
  useLocks: this.opts.useLocks,
114
114
  pollTimeout: this.opts.pollTimeout,
115
115
  lockDuration: this.opts.lockDuration,
116
+ group: this.opts.group,
116
117
  };
117
118
  }
118
119
  }
@@ -1,7 +1,9 @@
1
+ import { assertGroupPullOptions } from '../../../domain/types/group';
1
2
  import { resolveToken } from '../../resolveToken';
2
3
  import { TcpConnectionPool } from '../../tcpPool';
3
4
  import { WORKER_CONSTANTS } from '../constants';
4
5
  export function resolveWorkerOptions(options, embedded) {
6
+ assertGroupPullOptions(options.group);
5
7
  return {
6
8
  concurrency: options.concurrency ?? 1,
7
9
  autorun: options.autorun ?? true,
@@ -18,6 +20,7 @@ export function resolveWorkerOptions(options, embedded) {
18
20
  removeOnComplete: options.removeOnComplete,
19
21
  removeOnFail: options.removeOnFail,
20
22
  connection: options.connection,
23
+ group: options.group,
21
24
  };
22
25
  }
23
26
  export function createTcpPool(options, concurrency) {
@@ -1,4 +1,4 @@
1
- import type { ConnectionOptions } from '../../types';
1
+ import type { ConnectionOptions, GroupWorkerOptions } from '../../types';
2
2
  export interface ExtendedWorkerOptions {
3
3
  concurrency: number;
4
4
  autorun: boolean;
@@ -21,4 +21,5 @@ export interface ExtendedWorkerOptions {
21
21
  count?: number;
22
22
  };
23
23
  connection?: ConnectionOptions;
24
+ group?: GroupWorkerOptions;
24
25
  }
@@ -4,6 +4,7 @@
4
4
  */
5
5
  import type { Job as InternalJob } from '../../domain/types/job';
6
6
  import type { TcpConnection } from './types';
7
+ import type { GroupWorkerOptions } from '../types';
7
8
  export interface PullConfig {
8
9
  readonly name: string;
9
10
  readonly workerId: string;
@@ -11,6 +12,7 @@ export interface PullConfig {
11
12
  readonly pollTimeout: number;
12
13
  /** Lock TTL in ms to request from the server on a lock-based pull. */
13
14
  readonly lockDuration?: number;
15
+ readonly group?: GroupWorkerOptions;
14
16
  }
15
17
  export declare function pullEmbedded(config: PullConfig, count: number): Promise<Array<{
16
18
  job: InternalJob;