@portqueue/core 1.2.1 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/conformance/index.cjs +82 -2
- package/dist/conformance/index.cjs.map +1 -1
- package/dist/conformance/index.mjs +83 -3
- package/dist/conformance/index.mjs.map +1 -1
- package/dist/consumer/completion-batcher.d.ts +29 -0
- package/dist/consumer/completion.service.d.ts +5 -3
- package/dist/consumer/execution.service.d.ts +7 -2
- package/dist/consumer/worker.d.ts +16 -0
- package/dist/drivers/driver.interface.d.ts +26 -2
- package/dist/drivers/mysql/index.cjs +50 -3
- package/dist/drivers/mysql/index.cjs.map +1 -1
- package/dist/drivers/mysql/index.mjs +50 -3
- package/dist/drivers/mysql/index.mjs.map +1 -1
- package/dist/drivers/mysql/mysql-driver.d.ts +16 -3
- package/dist/drivers/oracle/index.cjs +50 -3
- package/dist/drivers/oracle/index.cjs.map +1 -1
- package/dist/drivers/oracle/index.mjs +50 -3
- package/dist/drivers/oracle/index.mjs.map +1 -1
- package/dist/drivers/oracle/oracle-driver.d.ts +14 -3
- package/dist/drivers/pglite/index.cjs +6 -3
- package/dist/drivers/pglite/index.cjs.map +1 -1
- package/dist/drivers/pglite/index.mjs +6 -3
- package/dist/drivers/pglite/index.mjs.map +1 -1
- package/dist/drivers/pglite/migrations.d.ts +2 -3
- package/dist/drivers/postgresql/index.cjs +1 -1
- package/dist/drivers/postgresql/index.mjs +1 -1
- package/dist/drivers/postgresql/migrations.d.ts +21 -0
- package/dist/drivers/postgresql/postgresql-driver.d.ts +28 -3
- package/dist/drivers/shared/base-driver.d.ts +28 -2
- package/dist/drivers/sqlite/index.cjs +36 -4
- package/dist/drivers/sqlite/index.cjs.map +1 -1
- package/dist/drivers/sqlite/index.mjs +36 -4
- package/dist/drivers/sqlite/index.mjs.map +1 -1
- package/dist/drivers/sqlite/sqlite-driver.d.ts +14 -3
- package/dist/index.cjs +107 -10
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.mjs +108 -11
- package/dist/index.mjs.map +1 -1
- package/dist/jobs/job-record.d.ts +8 -0
- package/dist/{migration-Dx8eUthW.cjs → migration-CX7j4wVB.cjs} +59 -1
- package/dist/migration-CX7j4wVB.cjs.map +1 -0
- package/dist/{migration-D0IRP2oq.js → migration-VkNQT3a_.js} +60 -2
- package/dist/migration-VkNQT3a_.js.map +1 -0
- package/dist/{postgresql-driver-CPHv5SUT.js → postgresql-driver-DQRzg2NA.js} +179 -57
- package/dist/postgresql-driver-DQRzg2NA.js.map +1 -0
- package/dist/{postgresql-driver-DRT68TKb.cjs → postgresql-driver-xfqy9mi8.cjs} +179 -57
- package/dist/postgresql-driver-xfqy9mi8.cjs.map +1 -0
- package/package.json +1 -1
- package/dist/migration-D0IRP2oq.js.map +0 -1
- package/dist/migration-Dx8eUthW.cjs.map +0 -1
- package/dist/postgresql-driver-CPHv5SUT.js.map +0 -1
- package/dist/postgresql-driver-DRT68TKb.cjs.map +0 -1
package/dist/index.d.ts
CHANGED
|
@@ -18,12 +18,12 @@ export { secondsToMs, wholeSeconds, wholeSecondsToMs, msToSeconds, MS_PER_SECOND
|
|
|
18
18
|
export type { AddJobResult, BatchItemResult, InstanceBatchItem, JobBatchItem, QueueBatchItem, RetriedJobItem, ScheduleBatchItem, JobSnapshot, ScheduleInfo, QueueInfo, InstanceInfo, QueueStats, GetJobsQuery, GetSchedulesQuery, GetInstancesQuery, GetQueuesQuery, } from './producer/results';
|
|
19
19
|
export type { PortQueueEventPayloads, EventListener, Unsubscribe, ErrorSource, } from './events/event-name.enum';
|
|
20
20
|
export type { HookRegistration, HookHandler, BeforeAddResult, ProcessOutcome, } from './events/hooks';
|
|
21
|
-
export type { Driver, DriverProbe, Subscription, MigrationStep, PageWindow, JobListFilter, JobRef, StalledJobRef, RequeuedJobRef, ScheduleRef, QueueListFilter, ScheduleListFilter, InstanceListFilter, } from './drivers/driver.interface';
|
|
21
|
+
export type { Driver, DriverProbe, Subscription, MigrationStep, PageWindow, JobListFilter, JobRef, CompletedJobInput, StalledJobRef, RequeuedJobRef, ScheduleRef, QueueListFilter, ScheduleListFilter, InstanceListFilter, } from './drivers/driver.interface';
|
|
22
22
|
export type { DriverCapabilitySet } from './drivers/driver-capabilities';
|
|
23
23
|
export { capabilities } from './drivers/driver-capabilities';
|
|
24
24
|
export { DriverContext } from './drivers/driver-context';
|
|
25
25
|
export type { StorageOptions } from './drivers/driver-context';
|
|
26
|
-
export { BaseDriver, JOB_COLUMNS, SCHEDULE_INSERT_COLUMNS } from './drivers/shared/base-driver';
|
|
26
|
+
export { BaseDriver, CLAIM_COLUMNS, JOB_COLUMNS, SCHEDULE_INSERT_COLUMNS, } from './drivers/shared/base-driver';
|
|
27
27
|
export type { RawResult, Row, Session } from './drivers/shared/base-driver';
|
|
28
28
|
export { toMigrationSteps, latestVersion } from './drivers/shared/migration';
|
|
29
29
|
export type { FlavourMigration, MigrationNames } from './drivers/shared/migration';
|
package/dist/index.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { P as PauseMode, B as BackoffStrategy, m as FailureDetail, M as MAX_BATCH_IDS, J as JobStatus, g as emptyStatusCounts, b as DeduplicationReason, D as DriverCapability, n as MigrationMode, o as DEFAULT_HEADLESS_MIGRATION_MODE, p as DEFAULT_MIGRATION_MODE, q as BatchItemOutcome, C as CancelOutcome } from "./storage.constants-DYEQUmc8.js";
|
|
2
2
|
import { h, i, j, l, k, d, f, e, a, c } from "./storage.constants-DYEQUmc8.js";
|
|
3
|
-
import { w as wholeSecondsToMs, f as wholeSeconds, s as secondsToMs, g as QueueNotFoundError, h as InvalidOptionsError, i as QueuePausedError, m as msToSeconds, L as LockLostError, j as matchesInstance, l as latestVersion, k as SchemaVersionMismatchError, n as MigrationFailedError, o as ShutdownTimeoutError, p as JobNotFoundError, q as QueueInUseError, u as InstanceActiveError, N as NotStartedError } from "./migration-
|
|
4
|
-
import { B, D, v, b, I, J, c as c2, e as e2, a as a2, M, P, x, Q, d as d2, S, T, U, t } from "./migration-
|
|
3
|
+
import { w as wholeSecondsToMs, f as wholeSeconds, s as secondsToMs, g as QueueNotFoundError, h as InvalidOptionsError, i as QueuePausedError, m as msToSeconds, L as LockLostError, j as matchesInstance, l as latestVersion, k as SchemaVersionMismatchError, n as MigrationFailedError, o as ShutdownTimeoutError, p as JobNotFoundError, q as QueueInUseError, u as InstanceActiveError, N as NotStartedError } from "./migration-VkNQT3a_.js";
|
|
4
|
+
import { B, C, D, v, b, I, J, c as c2, e as e2, a as a2, M, P, x, Q, d as d2, S, T, U, t } from "./migration-VkNQT3a_.js";
|
|
5
5
|
import { CronExpressionParser } from "cron-parser";
|
|
6
6
|
import { availableParallelism } from "node:os";
|
|
7
7
|
import { performance, monitorEventLoopDelay } from "node:perf_hooks";
|
|
@@ -1120,16 +1120,66 @@ function computeBackoffSeconds(input) {
|
|
|
1120
1120
|
}
|
|
1121
1121
|
return Math.max(0, delay);
|
|
1122
1122
|
}
|
|
1123
|
+
class CompletionBatcher {
|
|
1124
|
+
constructor(driver, clock) {
|
|
1125
|
+
this.driver = driver;
|
|
1126
|
+
this.clock = clock;
|
|
1127
|
+
}
|
|
1128
|
+
pending = [];
|
|
1129
|
+
scheduled = false;
|
|
1130
|
+
flushing = false;
|
|
1131
|
+
/** Settle one claimed job as `COMPLETED`; resolves with whether the row was actually settled. */
|
|
1132
|
+
complete(job) {
|
|
1133
|
+
if (!this.driver.completeJobs) {
|
|
1134
|
+
return this.driver.completeJob({ ...job, now: this.clock.now() });
|
|
1135
|
+
}
|
|
1136
|
+
return new Promise((resolve, reject) => {
|
|
1137
|
+
this.pending.push({ job, resolve, reject });
|
|
1138
|
+
this.schedule();
|
|
1139
|
+
});
|
|
1140
|
+
}
|
|
1141
|
+
schedule() {
|
|
1142
|
+
if (this.scheduled || this.flushing) return;
|
|
1143
|
+
this.scheduled = true;
|
|
1144
|
+
setImmediate(() => {
|
|
1145
|
+
this.scheduled = false;
|
|
1146
|
+
void this.flush();
|
|
1147
|
+
});
|
|
1148
|
+
}
|
|
1149
|
+
async flush() {
|
|
1150
|
+
this.flushing = true;
|
|
1151
|
+
try {
|
|
1152
|
+
while (this.pending.length) {
|
|
1153
|
+
const batch = this.pending.splice(0, MAX_BATCH_IDS);
|
|
1154
|
+
try {
|
|
1155
|
+
const settled = new Set(
|
|
1156
|
+
await this.driver.completeJobs({
|
|
1157
|
+
jobs: batch.map((entry) => entry.job),
|
|
1158
|
+
now: this.clock.now()
|
|
1159
|
+
})
|
|
1160
|
+
);
|
|
1161
|
+
for (const entry of batch) entry.resolve(settled.has(entry.job.jobId));
|
|
1162
|
+
} catch (error) {
|
|
1163
|
+
for (const entry of batch) entry.reject(error);
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
} finally {
|
|
1167
|
+
this.flushing = false;
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1123
1171
|
class CompletionService {
|
|
1124
1172
|
constructor(runtime) {
|
|
1125
1173
|
this.runtime = runtime;
|
|
1174
|
+
this.batcher = new CompletionBatcher(runtime.driver, runtime.clock);
|
|
1126
1175
|
}
|
|
1176
|
+
/** Successful settles are batched (see {@link CompletionBatcher}); failures and retries are not. */
|
|
1177
|
+
batcher;
|
|
1127
1178
|
async complete(record, queueName, returnValue, startTs) {
|
|
1128
1179
|
const now = this.runtime.clock.now();
|
|
1129
|
-
const settled = await this.
|
|
1180
|
+
const settled = await this.batcher.complete({
|
|
1130
1181
|
jobId: record.id,
|
|
1131
1182
|
returnValue,
|
|
1132
|
-
now,
|
|
1133
1183
|
lockToken: record.lockToken
|
|
1134
1184
|
});
|
|
1135
1185
|
if (!settled) return this.claimLost(record, queueName, { kind: "completed", returnValue });
|
|
@@ -1405,7 +1455,9 @@ class SchedulerService {
|
|
|
1405
1455
|
}
|
|
1406
1456
|
computeNext(schedule, now) {
|
|
1407
1457
|
if (schedule.cron) {
|
|
1408
|
-
return new CronExpression(schedule.cron, schedule.timezone ?? DEFAULT_CRON_TIMEZONE).next(
|
|
1458
|
+
return new CronExpression(schedule.cron, schedule.timezone ?? DEFAULT_CRON_TIMEZONE).next(
|
|
1459
|
+
now
|
|
1460
|
+
);
|
|
1409
1461
|
}
|
|
1410
1462
|
if (schedule.everySeconds != null) {
|
|
1411
1463
|
return now + wholeSecondsToMs(schedule.everySeconds, "schedule.startEverySeconds");
|
|
@@ -2301,7 +2353,13 @@ class ExecutionService {
|
|
|
2301
2353
|
}
|
|
2302
2354
|
/** Active jobs on this instance → their controller and claim token (for cancel/shutdown). */
|
|
2303
2355
|
active = /* @__PURE__ */ new Map();
|
|
2304
|
-
|
|
2356
|
+
/**
|
|
2357
|
+
* Run one claimed job to its settled outcome. `onHandlerDone` fires the moment the handler has
|
|
2358
|
+
* returned or thrown — before its outcome is written — so the worker can hand the slot to the next
|
|
2359
|
+
* job while the settle is still in flight; the returned promise resolves once it is written.
|
|
2360
|
+
*/
|
|
2361
|
+
async run(record, onHandlerDone = () => {
|
|
2362
|
+
}) {
|
|
2305
2363
|
const startTs = this.runtime.clock.now();
|
|
2306
2364
|
const controller = new AbortController();
|
|
2307
2365
|
this.active.set(record.id, { controller, lockToken: record.lockToken });
|
|
@@ -2316,10 +2374,16 @@ class ExecutionService {
|
|
|
2316
2374
|
const heartbeat = this.startHeartbeat(record, controller);
|
|
2317
2375
|
try {
|
|
2318
2376
|
await this.runtime.hooks.runBeforeProcess(queueName, job);
|
|
2319
|
-
|
|
2377
|
+
let returnValue;
|
|
2378
|
+
try {
|
|
2379
|
+
returnValue = await this.config.handler(job);
|
|
2380
|
+
} finally {
|
|
2381
|
+
onHandlerDone();
|
|
2382
|
+
}
|
|
2320
2383
|
const outcome = await this.completion.complete(record, queueName, returnValue, startTs);
|
|
2321
2384
|
await this.runtime.hooks.runAfterProcess(queueName, job, outcome);
|
|
2322
2385
|
} catch (err) {
|
|
2386
|
+
onHandlerDone();
|
|
2323
2387
|
const outcome = await this.completion.failOrRetry(
|
|
2324
2388
|
record,
|
|
2325
2389
|
queueName,
|
|
@@ -2398,7 +2462,17 @@ class Worker {
|
|
|
2398
2462
|
}
|
|
2399
2463
|
execution;
|
|
2400
2464
|
limiter;
|
|
2465
|
+
/** Claimed jobs not yet settled: their handler is running, or its outcome is being written. */
|
|
2401
2466
|
activeCount = 0;
|
|
2467
|
+
/**
|
|
2468
|
+
* Handlers actually executing — what `concurrency` caps. A slot frees the moment a handler
|
|
2469
|
+
* returns, not when its outcome has been written: settles are batched, and holding the slot
|
|
2470
|
+
* through the write would halve every claim (half the slots would always be waiting on a settle).
|
|
2471
|
+
* Shutdown still drains on {@link activeCount}, so no outcome is ever abandoned mid-write.
|
|
2472
|
+
*/
|
|
2473
|
+
runningCount = 0;
|
|
2474
|
+
/** A slot-freed pump is already queued for the next turn (see {@link schedulePump}). */
|
|
2475
|
+
pumpScheduled = false;
|
|
2402
2476
|
stopping = false;
|
|
2403
2477
|
started = false;
|
|
2404
2478
|
queueName = null;
|
|
@@ -2439,7 +2513,7 @@ class Worker {
|
|
|
2439
2513
|
this.pumping = true;
|
|
2440
2514
|
try {
|
|
2441
2515
|
while (!this.stopping) {
|
|
2442
|
-
const capacity = this.config.concurrency - this.
|
|
2516
|
+
const capacity = this.config.concurrency - this.runningCount;
|
|
2443
2517
|
if (capacity <= 0) break;
|
|
2444
2518
|
if (this.isPressureHigh()) break;
|
|
2445
2519
|
if (this.runtime.registry.pauseState(this.config.queue)?.processingPaused) break;
|
|
@@ -2463,11 +2537,19 @@ class Worker {
|
|
|
2463
2537
|
if (jobs.length === 0) break;
|
|
2464
2538
|
for (const record of jobs) {
|
|
2465
2539
|
this.activeCount += 1;
|
|
2466
|
-
|
|
2540
|
+
this.runningCount += 1;
|
|
2541
|
+
let slotFreed = false;
|
|
2542
|
+
const freeSlot = () => {
|
|
2543
|
+
if (slotFreed) return;
|
|
2544
|
+
slotFreed = true;
|
|
2545
|
+
this.runningCount -= 1;
|
|
2546
|
+
this.schedulePump();
|
|
2547
|
+
};
|
|
2548
|
+
void this.execution.run(record, freeSlot).catch((err) => {
|
|
2467
2549
|
this.runtime.events.emit(PortQueueEvent.ERROR, { error: err, source: "execution" });
|
|
2468
2550
|
}).finally(() => {
|
|
2469
2551
|
this.activeCount -= 1;
|
|
2470
|
-
|
|
2552
|
+
freeSlot();
|
|
2471
2553
|
});
|
|
2472
2554
|
}
|
|
2473
2555
|
}
|
|
@@ -2477,6 +2559,19 @@ class Worker {
|
|
|
2477
2559
|
this.pumping = false;
|
|
2478
2560
|
}
|
|
2479
2561
|
}
|
|
2562
|
+
/**
|
|
2563
|
+
* Pump on the next turn of the event loop, at most once per turn. Handlers that finish together
|
|
2564
|
+
* (a claimed batch of short jobs does) free their slots in the same turn; pumping on the first of
|
|
2565
|
+
* them claimed one row, and the rest a moment later — two statements where one does.
|
|
2566
|
+
*/
|
|
2567
|
+
schedulePump() {
|
|
2568
|
+
if (this.pumpScheduled) return;
|
|
2569
|
+
this.pumpScheduled = true;
|
|
2570
|
+
setImmediate(() => {
|
|
2571
|
+
this.pumpScheduled = false;
|
|
2572
|
+
this.runtime.background.run(() => this.pump());
|
|
2573
|
+
});
|
|
2574
|
+
}
|
|
2480
2575
|
/** Re-pump once the rate limiter has a token. At most one such timer is ever outstanding. */
|
|
2481
2576
|
scheduleTokenWakeup(delayMs) {
|
|
2482
2577
|
if (this.tokenTimer || this.stopping) return;
|
|
@@ -2851,7 +2946,8 @@ class PortQueue {
|
|
|
2851
2946
|
now: this.runtime.clock.now(),
|
|
2852
2947
|
...tx
|
|
2853
2948
|
});
|
|
2854
|
-
for (const queue of new Set(requeued.map((r) => r.queueName)))
|
|
2949
|
+
for (const queue of new Set(requeued.map((r) => r.queueName)))
|
|
2950
|
+
this.runtime.wakeup.signal(queue);
|
|
2855
2951
|
for (const ref of requeued) {
|
|
2856
2952
|
this.events.emit(PortQueueEvent.JOB_ADDED, {
|
|
2857
2953
|
jobId: ref.jobId,
|
|
@@ -3268,6 +3364,7 @@ export {
|
|
|
3268
3364
|
BackoffStrategy,
|
|
3269
3365
|
B as BaseDriver,
|
|
3270
3366
|
BatchItemOutcome,
|
|
3367
|
+
C as CLAIM_COLUMNS,
|
|
3271
3368
|
CancelOutcome,
|
|
3272
3369
|
DASHBOARD_MAX_PAGE_LIMIT,
|
|
3273
3370
|
DEFAULT_BACKOFF_FACTOR,
|