@portqueue/core 1.2.1 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/dist/conformance/index.cjs +82 -2
  2. package/dist/conformance/index.cjs.map +1 -1
  3. package/dist/conformance/index.mjs +83 -3
  4. package/dist/conformance/index.mjs.map +1 -1
  5. package/dist/consumer/completion-batcher.d.ts +29 -0
  6. package/dist/consumer/completion.service.d.ts +5 -3
  7. package/dist/consumer/execution.service.d.ts +7 -2
  8. package/dist/consumer/worker.d.ts +16 -0
  9. package/dist/drivers/driver.interface.d.ts +26 -2
  10. package/dist/drivers/mysql/index.cjs +50 -3
  11. package/dist/drivers/mysql/index.cjs.map +1 -1
  12. package/dist/drivers/mysql/index.mjs +50 -3
  13. package/dist/drivers/mysql/index.mjs.map +1 -1
  14. package/dist/drivers/mysql/mysql-driver.d.ts +16 -3
  15. package/dist/drivers/oracle/index.cjs +50 -3
  16. package/dist/drivers/oracle/index.cjs.map +1 -1
  17. package/dist/drivers/oracle/index.mjs +50 -3
  18. package/dist/drivers/oracle/index.mjs.map +1 -1
  19. package/dist/drivers/oracle/oracle-driver.d.ts +14 -3
  20. package/dist/drivers/pglite/index.cjs +6 -3
  21. package/dist/drivers/pglite/index.cjs.map +1 -1
  22. package/dist/drivers/pglite/index.mjs +6 -3
  23. package/dist/drivers/pglite/index.mjs.map +1 -1
  24. package/dist/drivers/pglite/migrations.d.ts +2 -3
  25. package/dist/drivers/postgresql/index.cjs +1 -1
  26. package/dist/drivers/postgresql/index.mjs +1 -1
  27. package/dist/drivers/postgresql/migrations.d.ts +21 -0
  28. package/dist/drivers/postgresql/postgresql-driver.d.ts +28 -3
  29. package/dist/drivers/shared/base-driver.d.ts +28 -2
  30. package/dist/drivers/sqlite/index.cjs +36 -4
  31. package/dist/drivers/sqlite/index.cjs.map +1 -1
  32. package/dist/drivers/sqlite/index.mjs +36 -4
  33. package/dist/drivers/sqlite/index.mjs.map +1 -1
  34. package/dist/drivers/sqlite/sqlite-driver.d.ts +14 -3
  35. package/dist/index.cjs +107 -10
  36. package/dist/index.cjs.map +1 -1
  37. package/dist/index.d.ts +2 -2
  38. package/dist/index.mjs +108 -11
  39. package/dist/index.mjs.map +1 -1
  40. package/dist/jobs/job-record.d.ts +8 -0
  41. package/dist/{migration-Dx8eUthW.cjs → migration-CX7j4wVB.cjs} +59 -1
  42. package/dist/migration-CX7j4wVB.cjs.map +1 -0
  43. package/dist/{migration-D0IRP2oq.js → migration-VkNQT3a_.js} +60 -2
  44. package/dist/migration-VkNQT3a_.js.map +1 -0
  45. package/dist/{postgresql-driver-CPHv5SUT.js → postgresql-driver-DQRzg2NA.js} +179 -57
  46. package/dist/postgresql-driver-DQRzg2NA.js.map +1 -0
  47. package/dist/{postgresql-driver-DRT68TKb.cjs → postgresql-driver-xfqy9mi8.cjs} +179 -57
  48. package/dist/postgresql-driver-xfqy9mi8.cjs.map +1 -0
  49. package/package.json +1 -1
  50. package/dist/migration-D0IRP2oq.js.map +0 -1
  51. package/dist/migration-Dx8eUthW.cjs.map +0 -1
  52. package/dist/postgresql-driver-CPHv5SUT.js.map +0 -1
  53. package/dist/postgresql-driver-DRT68TKb.cjs.map +0 -1
@@ -2,8 +2,8 @@ import { DatabaseEngine } from '../../options/enums';
2
2
  import { JobStatus } from '../../jobs/job-state.enum';
3
3
  import { DriverCapabilitySet } from '../driver-capabilities';
4
4
  import { BaseDriver, RawResult, Session } from '../shared/base-driver';
5
- import { Driver, DriverProbe, InstanceListFilter, JobListFilter, JobRef, MigrationStep, PageWindow, QueueListFilter, RequeuedJobRef, ScheduleListFilter, ScheduleRef, StalledJobRef, Subscription } from '../driver.interface';
6
- import { InsertedJobRow, InstanceMetrics, InstanceRow, JobRecord, NewJobRecord, NewScheduleRow, QueueRow, ScheduleRow } from '../../jobs/job-record';
5
+ import { CompletedJobInput, Driver, DriverProbe, InstanceListFilter, JobListFilter, JobRef, MigrationStep, PageWindow, QueueListFilter, RequeuedJobRef, ScheduleListFilter, ScheduleRef, StalledJobRef, Subscription } from '../driver.interface';
6
+ import { ClaimedJob, InsertedJobRow, InstanceMetrics, InstanceRow, JobRecord, NewJobRecord, NewScheduleRow, QueueRow, ScheduleRow } from '../../jobs/job-record';
7
7
  import { QueueOptions } from '../../options/queue-options';
8
8
  import { TransactionHandle } from '../../tx/transaction-handle';
9
9
  export interface SqliteDriverOptions {
@@ -136,7 +136,7 @@ export declare class SqliteDriver extends BaseDriver implements Driver {
136
136
  lockDurationSeconds: number;
137
137
  now: number;
138
138
  instanceKey?: string | null;
139
- }): Promise<JobRecord[]>;
139
+ }): Promise<ClaimedJob[]>;
140
140
  /**
141
141
  * The fence a completion is guarded on: the row must still be `ACTIVE` and, when the caller names
142
142
  * the claim it is settling, still carry that claim's token.
@@ -160,6 +160,17 @@ export declare class SqliteDriver extends BaseDriver implements Driver {
160
160
  lockToken?: string | null;
161
161
  tx?: TransactionHandle;
162
162
  }): Promise<boolean>;
163
+ /**
164
+ * One guarded `UPDATE … RETURNING id` per claim token (and per id chunk), fenced per row like
165
+ * {@link completeJob}. Parameters are pushed in the order their `?` placeholders appear.
166
+ */
167
+ completeJobs(options: {
168
+ jobs: CompletedJobInput[];
169
+ now: number;
170
+ tx?: TransactionHandle;
171
+ }): Promise<string[]>;
172
+ /** `NULL` when no job in the chunk returned anything, otherwise a `CASE` on the id. */
173
+ private returnValueExpression;
163
174
  retryJob(options: {
164
175
  jobId: string;
165
176
  nextRunAt: number;
package/dist/index.cjs CHANGED
@@ -1,7 +1,7 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
3
3
  const storage_constants = require("./storage.constants-BKeXcSQe.cjs");
4
- const migration = require("./migration-Dx8eUthW.cjs");
4
+ const migration = require("./migration-CX7j4wVB.cjs");
5
5
  const cronParser = require("cron-parser");
6
6
  const node_os = require("node:os");
7
7
  const node_perf_hooks = require("node:perf_hooks");
@@ -1120,16 +1120,66 @@ function computeBackoffSeconds(input) {
1120
1120
  }
1121
1121
  return Math.max(0, delay);
1122
1122
  }
1123
+ class CompletionBatcher {
1124
+ constructor(driver, clock) {
1125
+ this.driver = driver;
1126
+ this.clock = clock;
1127
+ }
1128
+ pending = [];
1129
+ scheduled = false;
1130
+ flushing = false;
1131
+ /** Settle one claimed job as `COMPLETED`; resolves with whether the row was actually settled. */
1132
+ complete(job) {
1133
+ if (!this.driver.completeJobs) {
1134
+ return this.driver.completeJob({ ...job, now: this.clock.now() });
1135
+ }
1136
+ return new Promise((resolve, reject) => {
1137
+ this.pending.push({ job, resolve, reject });
1138
+ this.schedule();
1139
+ });
1140
+ }
1141
+ schedule() {
1142
+ if (this.scheduled || this.flushing) return;
1143
+ this.scheduled = true;
1144
+ setImmediate(() => {
1145
+ this.scheduled = false;
1146
+ void this.flush();
1147
+ });
1148
+ }
1149
+ async flush() {
1150
+ this.flushing = true;
1151
+ try {
1152
+ while (this.pending.length) {
1153
+ const batch = this.pending.splice(0, storage_constants.MAX_BATCH_IDS);
1154
+ try {
1155
+ const settled = new Set(
1156
+ await this.driver.completeJobs({
1157
+ jobs: batch.map((entry) => entry.job),
1158
+ now: this.clock.now()
1159
+ })
1160
+ );
1161
+ for (const entry of batch) entry.resolve(settled.has(entry.job.jobId));
1162
+ } catch (error) {
1163
+ for (const entry of batch) entry.reject(error);
1164
+ }
1165
+ }
1166
+ } finally {
1167
+ this.flushing = false;
1168
+ }
1169
+ }
1170
+ }
1123
1171
  class CompletionService {
1124
1172
  constructor(runtime) {
1125
1173
  this.runtime = runtime;
1174
+ this.batcher = new CompletionBatcher(runtime.driver, runtime.clock);
1126
1175
  }
1176
+ /** Successful settles are batched (see {@link CompletionBatcher}); failures and retries are not. */
1177
+ batcher;
1127
1178
  async complete(record, queueName, returnValue, startTs) {
1128
1179
  const now = this.runtime.clock.now();
1129
- const settled = await this.runtime.driver.completeJob({
1180
+ const settled = await this.batcher.complete({
1130
1181
  jobId: record.id,
1131
1182
  returnValue,
1132
- now,
1133
1183
  lockToken: record.lockToken
1134
1184
  });
1135
1185
  if (!settled) return this.claimLost(record, queueName, { kind: "completed", returnValue });
@@ -1405,7 +1455,9 @@ class SchedulerService {
1405
1455
  }
1406
1456
  computeNext(schedule, now) {
1407
1457
  if (schedule.cron) {
1408
- return new CronExpression(schedule.cron, schedule.timezone ?? DEFAULT_CRON_TIMEZONE).next(now);
1458
+ return new CronExpression(schedule.cron, schedule.timezone ?? DEFAULT_CRON_TIMEZONE).next(
1459
+ now
1460
+ );
1409
1461
  }
1410
1462
  if (schedule.everySeconds != null) {
1411
1463
  return now + migration.wholeSecondsToMs(schedule.everySeconds, "schedule.startEverySeconds");
@@ -2301,7 +2353,13 @@ class ExecutionService {
2301
2353
  }
2302
2354
  /** Active jobs on this instance → their controller and claim token (for cancel/shutdown). */
2303
2355
  active = /* @__PURE__ */ new Map();
2304
- async run(record) {
2356
+ /**
2357
+ * Run one claimed job to its settled outcome. `onHandlerDone` fires the moment the handler has
2358
+ * returned or thrown — before its outcome is written — so the worker can hand the slot to the next
2359
+ * job while the settle is still in flight; the returned promise resolves once it is written.
2360
+ */
2361
+ async run(record, onHandlerDone = () => {
2362
+ }) {
2305
2363
  const startTs = this.runtime.clock.now();
2306
2364
  const controller = new AbortController();
2307
2365
  this.active.set(record.id, { controller, lockToken: record.lockToken });
@@ -2316,10 +2374,16 @@ class ExecutionService {
2316
2374
  const heartbeat = this.startHeartbeat(record, controller);
2317
2375
  try {
2318
2376
  await this.runtime.hooks.runBeforeProcess(queueName, job);
2319
- const returnValue = await this.config.handler(job);
2377
+ let returnValue;
2378
+ try {
2379
+ returnValue = await this.config.handler(job);
2380
+ } finally {
2381
+ onHandlerDone();
2382
+ }
2320
2383
  const outcome = await this.completion.complete(record, queueName, returnValue, startTs);
2321
2384
  await this.runtime.hooks.runAfterProcess(queueName, job, outcome);
2322
2385
  } catch (err) {
2386
+ onHandlerDone();
2323
2387
  const outcome = await this.completion.failOrRetry(
2324
2388
  record,
2325
2389
  queueName,
@@ -2398,7 +2462,17 @@ class Worker {
2398
2462
  }
2399
2463
  execution;
2400
2464
  limiter;
2465
+ /** Claimed jobs not yet settled: their handler is running, or its outcome is being written. */
2401
2466
  activeCount = 0;
2467
+ /**
2468
+ * Handlers actually executing — what `concurrency` caps. A slot frees the moment a handler
2469
+ * returns, not when its outcome has been written: settles are batched, and holding the slot
2470
+ * through the write would halve every claim (half the slots would always be waiting on a settle).
2471
+ * Shutdown still drains on {@link activeCount}, so no outcome is ever abandoned mid-write.
2472
+ */
2473
+ runningCount = 0;
2474
+ /** A slot-freed pump is already queued for the next turn (see {@link schedulePump}). */
2475
+ pumpScheduled = false;
2402
2476
  stopping = false;
2403
2477
  started = false;
2404
2478
  queueName = null;
@@ -2439,7 +2513,7 @@ class Worker {
2439
2513
  this.pumping = true;
2440
2514
  try {
2441
2515
  while (!this.stopping) {
2442
- const capacity = this.config.concurrency - this.activeCount;
2516
+ const capacity = this.config.concurrency - this.runningCount;
2443
2517
  if (capacity <= 0) break;
2444
2518
  if (this.isPressureHigh()) break;
2445
2519
  if (this.runtime.registry.pauseState(this.config.queue)?.processingPaused) break;
@@ -2463,11 +2537,19 @@ class Worker {
2463
2537
  if (jobs.length === 0) break;
2464
2538
  for (const record of jobs) {
2465
2539
  this.activeCount += 1;
2466
- void this.execution.run(record).catch((err) => {
2540
+ this.runningCount += 1;
2541
+ let slotFreed = false;
2542
+ const freeSlot = () => {
2543
+ if (slotFreed) return;
2544
+ slotFreed = true;
2545
+ this.runningCount -= 1;
2546
+ this.schedulePump();
2547
+ };
2548
+ void this.execution.run(record, freeSlot).catch((err) => {
2467
2549
  this.runtime.events.emit(PortQueueEvent.ERROR, { error: err, source: "execution" });
2468
2550
  }).finally(() => {
2469
2551
  this.activeCount -= 1;
2470
- this.runtime.background.run(() => this.pump());
2552
+ freeSlot();
2471
2553
  });
2472
2554
  }
2473
2555
  }
@@ -2477,6 +2559,19 @@ class Worker {
2477
2559
  this.pumping = false;
2478
2560
  }
2479
2561
  }
2562
+ /**
2563
+ * Pump on the next turn of the event loop, at most once per turn. Handlers that finish together
2564
+ * (a claimed batch of short jobs does) free their slots in the same turn; pumping on the first of
2565
+ * them claimed one row, and the rest a moment later — two statements where one does.
2566
+ */
2567
+ schedulePump() {
2568
+ if (this.pumpScheduled) return;
2569
+ this.pumpScheduled = true;
2570
+ setImmediate(() => {
2571
+ this.pumpScheduled = false;
2572
+ this.runtime.background.run(() => this.pump());
2573
+ });
2574
+ }
2480
2575
  /** Re-pump once the rate limiter has a token. At most one such timer is ever outstanding. */
2481
2576
  scheduleTokenWakeup(delayMs) {
2482
2577
  if (this.tokenTimer || this.stopping) return;
@@ -2851,7 +2946,8 @@ class PortQueue {
2851
2946
  now: this.runtime.clock.now(),
2852
2947
  ...tx
2853
2948
  });
2854
- for (const queue of new Set(requeued.map((r) => r.queueName))) this.runtime.wakeup.signal(queue);
2949
+ for (const queue of new Set(requeued.map((r) => r.queueName)))
2950
+ this.runtime.wakeup.signal(queue);
2855
2951
  for (const ref of requeued) {
2856
2952
  this.events.emit(PortQueueEvent.JOB_ADDED, {
2857
2953
  jobId: ref.jobId,
@@ -3281,6 +3377,7 @@ exports.MigrationMode = storage_constants.MigrationMode;
3281
3377
  exports.PauseMode = storage_constants.PauseMode;
3282
3378
  exports.capabilities = storage_constants.capabilities;
3283
3379
  exports.BaseDriver = migration.BaseDriver;
3380
+ exports.CLAIM_COLUMNS = migration.CLAIM_COLUMNS;
3284
3381
  exports.DriverConnectionError = migration.DriverConnectionError;
3285
3382
  exports.DriverContext = migration.DriverContext;
3286
3383
  exports.DriverQueryError = migration.DriverQueryError;