@monque/core 1.5.1 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/CHANGELOG.md +20 -0
- package/dist/index.cjs +266 -67
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +21 -0
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.mts +21 -0
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +266 -67
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -5
- package/src/scheduler/monque.ts +25 -1
- package/src/scheduler/services/change-stream-handler.ts +222 -51
- package/src/scheduler/services/job-manager.ts +4 -1
- package/src/scheduler/services/job-processor.ts +43 -6
- package/src/scheduler/services/job-scheduler.ts +18 -4
- package/src/scheduler/services/lifecycle-manager.ts +72 -17
- package/src/scheduler/services/types.ts +4 -0
- package/src/scheduler/types.ts +14 -0
package/dist/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# @monque/core
|
|
2
2
|
|
|
3
|
+
## 1.6.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- [#232](https://github.com/ueberBrot/monque/pull/232) [`c3d2c83`](https://github.com/ueberBrot/monque/commit/c3d2c83b89d3fc8e77ec1958695d05b68f357d8d) Thanks [@ueberBrot](https://github.com/ueberBrot)! - Add adaptive poll scheduling and targeted change stream processing
|
|
8
|
+
|
|
9
|
+
- **Adaptive polling**: When change streams are active, safety polling runs at `safetyPollInterval` (default 30s) instead of the fast `pollInterval`. Falls back to `pollInterval` when change streams are unavailable.
|
|
10
|
+
- **Targeted polling**: Change stream events now leverage the full document to poll only the specific worker(s) for the affected job type, skipping unrelated workers.
|
|
11
|
+
- **Wakeup timers**: Future-dated jobs (`nextRunAt > now`) get a precise wakeup timer instead of waiting for the next poll cycle.
|
|
12
|
+
- **Local pending-job notifications**: Jobs created or moved back to `pending` by the local scheduler now trigger the same targeted polling and wakeup-timer path immediately, avoiding startup races before the change stream cursor is fully ready.
|
|
13
|
+
- **Slot-freed re-polling**: When a job completes or permanently fails, a targeted re-poll immediately picks up the next waiting job for that worker.
|
|
14
|
+
- **Re-poll queuing**: Poll requests arriving while a poll is running are queued and executed after, preventing silently dropped change-stream-triggered polls.
|
|
15
|
+
- New configuration option: `safetyPollInterval` (default: 30000ms).
|
|
16
|
+
|
|
17
|
+
## 1.5.2
|
|
18
|
+
|
|
19
|
+
### Patch Changes
|
|
20
|
+
|
|
21
|
+
- [#227](https://github.com/ueberBrot/monque/pull/227) [`e9208ca`](https://github.com/ueberBrot/monque/commit/e9208ca5c985d84d023161560d5d3ba195394fe1) Thanks [@ueberBrot](https://github.com/ueberBrot)! - Prevent change stream reconnection attempts from running after the scheduler stops. This clears pending reconnect timers during shutdown and adds coverage for the stop-during-backoff scenario.
|
|
22
|
+
|
|
3
23
|
## 1.5.1
|
|
4
24
|
|
|
5
25
|
### Patch Changes
|
package/dist/index.cjs
CHANGED
|
@@ -714,12 +714,20 @@ function decodeCursor(cursor) {
|
|
|
714
714
|
}
|
|
715
715
|
//#endregion
|
|
716
716
|
//#region src/scheduler/services/change-stream-handler.ts
|
|
717
|
+
/** Minimum poll interval floor to prevent tight loops (ms) */
|
|
718
|
+
const MIN_POLL_INTERVAL = 100;
|
|
719
|
+
/** Grace period after nextRunAt before scheduling a wakeup poll (ms) */
|
|
720
|
+
const POLL_GRACE_PERIOD = 200;
|
|
717
721
|
/**
|
|
718
722
|
* Internal service for MongoDB Change Stream lifecycle.
|
|
719
723
|
*
|
|
720
724
|
* Provides real-time job notifications when available, with automatic
|
|
721
725
|
* reconnection and graceful fallback to polling-only mode.
|
|
722
726
|
*
|
|
727
|
+
* Leverages the full document from change stream events to:
|
|
728
|
+
* - Trigger **targeted polls** for specific workers (using the job `name`)
|
|
729
|
+
* - Schedule **precise wakeup timers** for future-dated jobs (using `nextRunAt`)
|
|
730
|
+
*
|
|
723
731
|
* @internal Not part of public API.
|
|
724
732
|
*/
|
|
725
733
|
var ChangeStreamHandler = class {
|
|
@@ -735,6 +743,12 @@ var ChangeStreamHandler = class {
|
|
|
735
743
|
reconnectTimer = null;
|
|
736
744
|
/** Whether the scheduler is currently using change streams */
|
|
737
745
|
usingChangeStreams = false;
|
|
746
|
+
/** Job names collected during the current debounce window for targeted polling */
|
|
747
|
+
pendingTargetNames = /* @__PURE__ */ new Set();
|
|
748
|
+
/** Wakeup timer for the earliest known future job */
|
|
749
|
+
wakeupTimer = null;
|
|
750
|
+
/** Time of the currently scheduled wakeup */
|
|
751
|
+
wakeupTime = null;
|
|
738
752
|
constructor(ctx, onPoll) {
|
|
739
753
|
this.ctx = ctx;
|
|
740
754
|
this.onPoll = onPoll;
|
|
@@ -755,10 +769,11 @@ var ChangeStreamHandler = class {
|
|
|
755
769
|
*/
|
|
756
770
|
setup() {
|
|
757
771
|
if (!this.ctx.isRunning()) return;
|
|
772
|
+
this.clearReconnectTimer();
|
|
758
773
|
try {
|
|
759
774
|
this.changeStream = this.ctx.collection.watch([{ $match: { $or: [{ operationType: "insert" }, {
|
|
760
775
|
operationType: "update",
|
|
761
|
-
"updateDescription.updatedFields.status": { $exists: true }
|
|
776
|
+
$or: [{ "updateDescription.updatedFields.status": { $exists: true } }, { "updateDescription.updatedFields.nextRunAt": { $exists: true } }]
|
|
762
777
|
}] } }], { fullDocument: "updateLookup" });
|
|
763
778
|
this.changeStream.on("change", (change) => {
|
|
764
779
|
this.handleEvent(change);
|
|
@@ -777,11 +792,20 @@ var ChangeStreamHandler = class {
|
|
|
777
792
|
}
|
|
778
793
|
}
|
|
779
794
|
/**
|
|
780
|
-
* Handle a change stream event
|
|
795
|
+
* Handle a change stream event using the full document for intelligent routing.
|
|
796
|
+
*
|
|
797
|
+
* For **immediate jobs** (`nextRunAt <= now`): collects the job name and triggers
|
|
798
|
+
* a debounced targeted poll for only the relevant workers.
|
|
799
|
+
*
|
|
800
|
+
* For **future jobs** (`nextRunAt > now`): schedules a precise wakeup timer so
|
|
801
|
+
* the job is picked up near its scheduled time without blind polling.
|
|
781
802
|
*
|
|
782
|
-
*
|
|
783
|
-
*
|
|
784
|
-
*
|
|
803
|
+
* For **completed/failed jobs** (slot freed): triggers a targeted re-poll for that
|
|
804
|
+
* worker so the next pending job is picked up immediately, maintaining continuous
|
|
805
|
+
* throughput without waiting for the safety poll interval.
|
|
806
|
+
*
|
|
807
|
+
* Falls back to a full poll (no target names) if the document is missing
|
|
808
|
+
* required fields.
|
|
785
809
|
*
|
|
786
810
|
* @param change - The change stream event document
|
|
787
811
|
*/
|
|
@@ -789,16 +813,81 @@ var ChangeStreamHandler = class {
|
|
|
789
813
|
if (!this.ctx.isRunning()) return;
|
|
790
814
|
const isInsert = change.operationType === "insert";
|
|
791
815
|
const isUpdate = change.operationType === "update";
|
|
792
|
-
const
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
816
|
+
const fullDocument = "fullDocument" in change ? change.fullDocument : void 0;
|
|
817
|
+
const currentStatus = fullDocument?.["status"];
|
|
818
|
+
const isPendingStatus = currentStatus === JobStatus.PENDING;
|
|
819
|
+
const isSlotFreed = isUpdate && (currentStatus === JobStatus.COMPLETED || currentStatus === JobStatus.FAILED);
|
|
820
|
+
if (!(isInsert || isUpdate && isPendingStatus || isSlotFreed)) return;
|
|
821
|
+
if (isSlotFreed) {
|
|
822
|
+
const jobName = fullDocument?.["name"];
|
|
823
|
+
if (jobName) this.pendingTargetNames.add(jobName);
|
|
824
|
+
this.debouncedPoll();
|
|
825
|
+
return;
|
|
826
|
+
}
|
|
827
|
+
const jobName = fullDocument?.["name"];
|
|
828
|
+
const nextRunAt = fullDocument?.["nextRunAt"];
|
|
829
|
+
if (jobName && nextRunAt) {
|
|
830
|
+
this.notifyPendingJob(jobName, nextRunAt);
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
833
|
+
if (jobName) this.pendingTargetNames.add(jobName);
|
|
834
|
+
this.debouncedPoll();
|
|
835
|
+
}
|
|
836
|
+
/**
|
|
837
|
+
* Notify the handler about a pending job created or updated by this process.
|
|
838
|
+
*
|
|
839
|
+
* Reuses the same routing logic as change stream events so local writes don't
|
|
840
|
+
* depend on the MongoDB change stream cursor already being fully ready.
|
|
841
|
+
*
|
|
842
|
+
* @param jobName - Worker name for targeted polling
|
|
843
|
+
* @param nextRunAt - When the job becomes eligible for processing
|
|
844
|
+
*/
|
|
845
|
+
notifyPendingJob(jobName, nextRunAt) {
|
|
846
|
+
if (!this.ctx.isRunning()) return;
|
|
847
|
+
if (nextRunAt.getTime() > Date.now()) {
|
|
848
|
+
this.scheduleWakeup(nextRunAt);
|
|
849
|
+
return;
|
|
801
850
|
}
|
|
851
|
+
this.pendingTargetNames.add(jobName);
|
|
852
|
+
this.debouncedPoll();
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* Schedule a debounced poll with collected target names.
|
|
856
|
+
*
|
|
857
|
+
* Collects job names from multiple change stream events during the debounce
|
|
858
|
+
* window, then triggers a single targeted poll for only those workers.
|
|
859
|
+
*/
|
|
860
|
+
debouncedPoll() {
|
|
861
|
+
if (this.debounceTimer) clearTimeout(this.debounceTimer);
|
|
862
|
+
this.debounceTimer = setTimeout(() => {
|
|
863
|
+
this.debounceTimer = null;
|
|
864
|
+
const names = this.pendingTargetNames.size > 0 ? new Set(this.pendingTargetNames) : void 0;
|
|
865
|
+
this.pendingTargetNames.clear();
|
|
866
|
+
this.onPoll(names).catch((error) => {
|
|
867
|
+
this.ctx.emit("job:error", { error: toError(error) });
|
|
868
|
+
});
|
|
869
|
+
}, 100);
|
|
870
|
+
}
|
|
871
|
+
/**
|
|
872
|
+
* Schedule a wakeup timer for a future-dated job.
|
|
873
|
+
*
|
|
874
|
+
* Maintains a single timer set to the earliest known future job's `nextRunAt`.
|
|
875
|
+
* When the timer fires, triggers a full poll to pick up all due jobs.
|
|
876
|
+
*
|
|
877
|
+
* @param nextRunAt - When the future job should become ready
|
|
878
|
+
*/
|
|
879
|
+
scheduleWakeup(nextRunAt) {
|
|
880
|
+
if (this.wakeupTime && nextRunAt >= this.wakeupTime) return;
|
|
881
|
+
this.clearWakeupTimer();
|
|
882
|
+
this.wakeupTime = nextRunAt;
|
|
883
|
+
const delay = Math.max(nextRunAt.getTime() - Date.now() + POLL_GRACE_PERIOD, MIN_POLL_INTERVAL);
|
|
884
|
+
this.wakeupTimer = setTimeout(() => {
|
|
885
|
+
this.wakeupTime = null;
|
|
886
|
+
this.wakeupTimer = null;
|
|
887
|
+
this.onPoll().catch((error) => {
|
|
888
|
+
this.ctx.emit("job:error", { error: toError(error) });
|
|
889
|
+
});
|
|
890
|
+
}, delay);
|
|
802
891
|
}
|
|
803
892
|
/**
|
|
804
893
|
* Handle change stream errors with exponential backoff reconnection.
|
|
@@ -812,52 +901,74 @@ var ChangeStreamHandler = class {
|
|
|
812
901
|
handleError(error) {
|
|
813
902
|
if (!this.ctx.isRunning()) return;
|
|
814
903
|
this.reconnectAttempts++;
|
|
904
|
+
this.resetActiveState();
|
|
905
|
+
this.closeChangeStream();
|
|
815
906
|
if (this.reconnectAttempts > this.maxReconnectAttempts) {
|
|
816
|
-
this.
|
|
817
|
-
if (this.reconnectTimer) {
|
|
818
|
-
clearTimeout(this.reconnectTimer);
|
|
819
|
-
this.reconnectTimer = null;
|
|
820
|
-
}
|
|
821
|
-
if (this.changeStream) {
|
|
822
|
-
this.changeStream.close().catch(() => {});
|
|
823
|
-
this.changeStream = null;
|
|
824
|
-
}
|
|
907
|
+
this.clearReconnectTimer();
|
|
825
908
|
this.ctx.emit("changestream:fallback", { reason: `Exhausted ${this.maxReconnectAttempts} reconnection attempts: ${error.message}` });
|
|
826
909
|
return;
|
|
827
910
|
}
|
|
828
911
|
const delay = 2 ** (this.reconnectAttempts - 1) * 1e3;
|
|
829
|
-
|
|
912
|
+
this.clearReconnectTimer();
|
|
913
|
+
if (!this.ctx.isRunning()) return;
|
|
830
914
|
this.reconnectTimer = setTimeout(() => {
|
|
831
|
-
this.
|
|
832
|
-
|
|
833
|
-
if (this.changeStream) {
|
|
834
|
-
this.changeStream.close().catch(() => {});
|
|
835
|
-
this.changeStream = null;
|
|
836
|
-
}
|
|
837
|
-
this.setup();
|
|
838
|
-
}
|
|
915
|
+
this.clearReconnectTimer();
|
|
916
|
+
this.reconnect();
|
|
839
917
|
}, delay);
|
|
840
918
|
}
|
|
919
|
+
reconnect() {
|
|
920
|
+
if (!this.ctx.isRunning()) return;
|
|
921
|
+
this.closeChangeStream();
|
|
922
|
+
if (!this.ctx.isRunning()) return;
|
|
923
|
+
this.setup();
|
|
924
|
+
}
|
|
925
|
+
clearReconnectTimer() {
|
|
926
|
+
if (!this.reconnectTimer) return;
|
|
927
|
+
clearTimeout(this.reconnectTimer);
|
|
928
|
+
this.reconnectTimer = null;
|
|
929
|
+
}
|
|
841
930
|
/**
|
|
842
|
-
*
|
|
931
|
+
* Reset all active change stream state: clear debounce timer, wakeup timer,
|
|
932
|
+
* pending target names, and mark as inactive.
|
|
933
|
+
*
|
|
934
|
+
* Does NOT close the cursor (callers handle sync vs async close) or clear
|
|
935
|
+
* the reconnect timer/attempts (callers manage reconnection lifecycle).
|
|
843
936
|
*/
|
|
844
|
-
|
|
937
|
+
resetActiveState() {
|
|
845
938
|
if (this.debounceTimer) {
|
|
846
939
|
clearTimeout(this.debounceTimer);
|
|
847
940
|
this.debounceTimer = null;
|
|
848
941
|
}
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
942
|
+
this.pendingTargetNames.clear();
|
|
943
|
+
this.clearWakeupTimer();
|
|
944
|
+
this.usingChangeStreams = false;
|
|
945
|
+
}
|
|
946
|
+
clearWakeupTimer() {
|
|
947
|
+
if (this.wakeupTimer) {
|
|
948
|
+
clearTimeout(this.wakeupTimer);
|
|
949
|
+
this.wakeupTimer = null;
|
|
852
950
|
}
|
|
951
|
+
this.wakeupTime = null;
|
|
952
|
+
}
|
|
953
|
+
closeChangeStream() {
|
|
954
|
+
if (!this.changeStream) return;
|
|
955
|
+
this.changeStream.close().catch(() => {});
|
|
956
|
+
this.changeStream = null;
|
|
957
|
+
}
|
|
958
|
+
/**
|
|
959
|
+
* Close the change stream cursor and emit closed event.
|
|
960
|
+
*/
|
|
961
|
+
async close() {
|
|
962
|
+
const wasActive = this.usingChangeStreams;
|
|
963
|
+
this.resetActiveState();
|
|
964
|
+
this.clearReconnectTimer();
|
|
853
965
|
if (this.changeStream) {
|
|
854
966
|
try {
|
|
855
967
|
await this.changeStream.close();
|
|
856
968
|
} catch {}
|
|
857
969
|
this.changeStream = null;
|
|
858
|
-
if (
|
|
970
|
+
if (wasActive) this.ctx.emit("changestream:closed", void 0);
|
|
859
971
|
}
|
|
860
|
-
this.usingChangeStreams = false;
|
|
861
972
|
this.reconnectAttempts = 0;
|
|
862
973
|
}
|
|
863
974
|
/**
|
|
@@ -962,6 +1073,7 @@ var JobManager = class {
|
|
|
962
1073
|
}, { returnDocument: "after" });
|
|
963
1074
|
if (!result) throw new JobStateError("Job status changed during retry attempt", jobId, "unknown", "retry");
|
|
964
1075
|
const job = this.ctx.documentToPersistedJob(result);
|
|
1076
|
+
this.ctx.notifyPendingJob(job.name, job.nextRunAt);
|
|
965
1077
|
this.ctx.emit("job:retried", {
|
|
966
1078
|
job,
|
|
967
1079
|
previousStatus
|
|
@@ -998,7 +1110,9 @@ var JobManager = class {
|
|
|
998
1110
|
updatedAt: /* @__PURE__ */ new Date()
|
|
999
1111
|
} }, { returnDocument: "after" });
|
|
1000
1112
|
if (!result) throw new JobStateError("Job status changed during reschedule attempt", jobId, "unknown", "reschedule");
|
|
1001
|
-
|
|
1113
|
+
const job = this.ctx.documentToPersistedJob(result);
|
|
1114
|
+
this.ctx.notifyPendingJob(job.name, job.nextRunAt);
|
|
1115
|
+
return job;
|
|
1002
1116
|
}
|
|
1003
1117
|
/**
|
|
1004
1118
|
* Permanently delete a job.
|
|
@@ -1172,6 +1286,8 @@ var JobManager = class {
|
|
|
1172
1286
|
var JobProcessor = class {
|
|
1173
1287
|
/** Guard flag to prevent concurrent poll() execution */
|
|
1174
1288
|
_isPolling = false;
|
|
1289
|
+
/** Flag to request a re-poll after the current poll finishes */
|
|
1290
|
+
_repollRequested = false;
|
|
1175
1291
|
constructor(ctx) {
|
|
1176
1292
|
this.ctx = ctx;
|
|
1177
1293
|
}
|
|
@@ -1204,12 +1320,27 @@ var JobProcessor = class {
|
|
|
1204
1320
|
* attempts to acquire jobs up to the worker's available concurrency slots.
|
|
1205
1321
|
* Aborts early if the scheduler is stopping (`isRunning` is false) or if
|
|
1206
1322
|
* the instance-level `instanceConcurrency` limit is reached.
|
|
1323
|
+
*
|
|
1324
|
+
* If a poll is requested while one is already running, it is queued and
|
|
1325
|
+
* executed as a full poll after the current one finishes. This prevents
|
|
1326
|
+
* change-stream-triggered polls from being silently dropped.
|
|
1327
|
+
*
|
|
1328
|
+
* @param targetNames - Optional set of worker names to poll. When provided, only the
|
|
1329
|
+
* specified workers are checked. Used by change stream handler for targeted polling.
|
|
1207
1330
|
*/
|
|
1208
|
-
async poll() {
|
|
1209
|
-
if (!this.ctx.isRunning()
|
|
1331
|
+
async poll(targetNames) {
|
|
1332
|
+
if (!this.ctx.isRunning()) return;
|
|
1333
|
+
if (this._isPolling) {
|
|
1334
|
+
this._repollRequested = true;
|
|
1335
|
+
return;
|
|
1336
|
+
}
|
|
1210
1337
|
this._isPolling = true;
|
|
1211
1338
|
try {
|
|
1212
|
-
|
|
1339
|
+
do {
|
|
1340
|
+
this._repollRequested = false;
|
|
1341
|
+
await this._doPoll(targetNames);
|
|
1342
|
+
targetNames = void 0;
|
|
1343
|
+
} while (this._repollRequested && this.ctx.isRunning());
|
|
1213
1344
|
} finally {
|
|
1214
1345
|
this._isPolling = false;
|
|
1215
1346
|
}
|
|
@@ -1217,10 +1348,11 @@ var JobProcessor = class {
|
|
|
1217
1348
|
/**
|
|
1218
1349
|
* Internal poll implementation.
|
|
1219
1350
|
*/
|
|
1220
|
-
async _doPoll() {
|
|
1351
|
+
async _doPoll(targetNames) {
|
|
1221
1352
|
const { instanceConcurrency } = this.ctx.options;
|
|
1222
1353
|
if (instanceConcurrency !== void 0 && this.getTotalActiveJobs() >= instanceConcurrency) return;
|
|
1223
1354
|
for (const [name, worker] of this.ctx.workers) {
|
|
1355
|
+
if (targetNames && !targetNames.has(name)) continue;
|
|
1224
1356
|
const workerAvailableSlots = worker.concurrency - worker.activeJobs.size;
|
|
1225
1357
|
if (workerAvailableSlots <= 0) continue;
|
|
1226
1358
|
const availableSlots = this.getGloballyAvailableSlots(workerAvailableSlots);
|
|
@@ -1357,7 +1489,10 @@ var JobProcessor = class {
|
|
|
1357
1489
|
failReason: ""
|
|
1358
1490
|
}
|
|
1359
1491
|
}, { returnDocument: "after" });
|
|
1360
|
-
|
|
1492
|
+
if (!result) return null;
|
|
1493
|
+
const persistedJob = this.ctx.documentToPersistedJob(result);
|
|
1494
|
+
this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
|
|
1495
|
+
return persistedJob;
|
|
1361
1496
|
}
|
|
1362
1497
|
const result = await this.ctx.collection.findOneAndUpdate({
|
|
1363
1498
|
_id: job._id,
|
|
@@ -1376,7 +1511,8 @@ var JobProcessor = class {
|
|
|
1376
1511
|
failReason: ""
|
|
1377
1512
|
}
|
|
1378
1513
|
}, { returnDocument: "after" });
|
|
1379
|
-
|
|
1514
|
+
if (!result) return null;
|
|
1515
|
+
return this.ctx.documentToPersistedJob(result);
|
|
1380
1516
|
}
|
|
1381
1517
|
/**
|
|
1382
1518
|
* Handle job failure with exponential backoff retry logic using an atomic status transition.
|
|
@@ -1869,13 +2005,17 @@ var JobScheduler = class {
|
|
|
1869
2005
|
returnDocument: "after"
|
|
1870
2006
|
});
|
|
1871
2007
|
if (!result) throw new ConnectionError("Failed to enqueue job: findOneAndUpdate returned no document");
|
|
1872
|
-
|
|
2008
|
+
const persistedJob = this.ctx.documentToPersistedJob(result);
|
|
2009
|
+
if (persistedJob.status === JobStatus.PENDING) this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
|
|
2010
|
+
return persistedJob;
|
|
1873
2011
|
}
|
|
1874
2012
|
const result = await this.ctx.collection.insertOne(job);
|
|
1875
|
-
|
|
2013
|
+
const persistedJob = {
|
|
1876
2014
|
...job,
|
|
1877
2015
|
_id: result.insertedId
|
|
1878
2016
|
};
|
|
2017
|
+
this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
|
|
2018
|
+
return persistedJob;
|
|
1879
2019
|
} catch (error) {
|
|
1880
2020
|
if (error instanceof ConnectionError) throw error;
|
|
1881
2021
|
throw new ConnectionError(`Failed to enqueue job: ${error instanceof Error ? error.message : "Unknown error during enqueue"}`, error instanceof Error ? { cause: error } : void 0);
|
|
@@ -1983,13 +2123,17 @@ var JobScheduler = class {
|
|
|
1983
2123
|
returnDocument: "after"
|
|
1984
2124
|
});
|
|
1985
2125
|
if (!result) throw new ConnectionError("Failed to schedule job: findOneAndUpdate returned no document");
|
|
1986
|
-
|
|
2126
|
+
const persistedJob = this.ctx.documentToPersistedJob(result);
|
|
2127
|
+
if (persistedJob.status === JobStatus.PENDING) this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
|
|
2128
|
+
return persistedJob;
|
|
1987
2129
|
}
|
|
1988
2130
|
const result = await this.ctx.collection.insertOne(job);
|
|
1989
|
-
|
|
2131
|
+
const persistedJob = {
|
|
1990
2132
|
...job,
|
|
1991
2133
|
_id: result.insertedId
|
|
1992
2134
|
};
|
|
2135
|
+
this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
|
|
2136
|
+
return persistedJob;
|
|
1993
2137
|
} catch (error) {
|
|
1994
2138
|
if (error instanceof MonqueError) throw error;
|
|
1995
2139
|
throw new ConnectionError(`Failed to schedule job: ${error instanceof Error ? error.message : "Unknown error during schedule"}`, error instanceof Error ? { cause: error } : void 0);
|
|
@@ -2005,14 +2149,19 @@ const DEFAULT_RETENTION_INTERVAL = 36e5;
|
|
|
2005
2149
|
/**
|
|
2006
2150
|
* Manages scheduler lifecycle timers and job cleanup.
|
|
2007
2151
|
*
|
|
2008
|
-
* Owns poll
|
|
2152
|
+
* Owns poll scheduling, heartbeat interval, cleanup interval, and the
|
|
2009
2153
|
* cleanupJobs logic. Extracted from Monque to keep the facade thin.
|
|
2010
2154
|
*
|
|
2155
|
+
* Uses adaptive poll scheduling: when change streams are active, polls at
|
|
2156
|
+
* `safetyPollInterval` (safety net only). When change streams are inactive,
|
|
2157
|
+
* polls at `pollInterval` (primary discovery mechanism).
|
|
2158
|
+
*
|
|
2011
2159
|
* @internal Not part of public API.
|
|
2012
2160
|
*/
|
|
2013
2161
|
var LifecycleManager = class {
|
|
2014
2162
|
ctx;
|
|
2015
|
-
|
|
2163
|
+
callbacks = null;
|
|
2164
|
+
pollTimeoutId = null;
|
|
2016
2165
|
heartbeatIntervalId = null;
|
|
2017
2166
|
cleanupIntervalId = null;
|
|
2018
2167
|
constructor(ctx) {
|
|
@@ -2021,17 +2170,13 @@ var LifecycleManager = class {
|
|
|
2021
2170
|
/**
|
|
2022
2171
|
* Start all lifecycle timers.
|
|
2023
2172
|
*
|
|
2024
|
-
* Sets up poll
|
|
2173
|
+
* Sets up adaptive poll scheduling, heartbeat interval, and (if configured)
|
|
2025
2174
|
* cleanup interval. Runs an initial poll immediately.
|
|
2026
2175
|
*
|
|
2027
2176
|
* @param callbacks - Functions to invoke on each timer tick
|
|
2028
2177
|
*/
|
|
2029
2178
|
startTimers(callbacks) {
|
|
2030
|
-
this.
|
|
2031
|
-
callbacks.poll().catch((error) => {
|
|
2032
|
-
this.ctx.emit("job:error", { error: toError(error) });
|
|
2033
|
-
});
|
|
2034
|
-
}, this.ctx.options.pollInterval);
|
|
2179
|
+
this.callbacks = callbacks;
|
|
2035
2180
|
this.heartbeatIntervalId = setInterval(() => {
|
|
2036
2181
|
callbacks.updateHeartbeats().catch((error) => {
|
|
2037
2182
|
this.ctx.emit("job:error", { error: toError(error) });
|
|
@@ -2048,23 +2193,22 @@ var LifecycleManager = class {
|
|
|
2048
2193
|
});
|
|
2049
2194
|
}, interval);
|
|
2050
2195
|
}
|
|
2051
|
-
|
|
2052
|
-
this.ctx.emit("job:error", { error: toError(error) });
|
|
2053
|
-
});
|
|
2196
|
+
this.executePollAndScheduleNext();
|
|
2054
2197
|
}
|
|
2055
2198
|
/**
|
|
2056
2199
|
* Stop all lifecycle timers.
|
|
2057
2200
|
*
|
|
2058
|
-
* Clears poll, heartbeat, and cleanup
|
|
2201
|
+
* Clears poll timeout, heartbeat interval, and cleanup interval.
|
|
2059
2202
|
*/
|
|
2060
2203
|
stopTimers() {
|
|
2204
|
+
this.callbacks = null;
|
|
2061
2205
|
if (this.cleanupIntervalId) {
|
|
2062
2206
|
clearInterval(this.cleanupIntervalId);
|
|
2063
2207
|
this.cleanupIntervalId = null;
|
|
2064
2208
|
}
|
|
2065
|
-
if (this.
|
|
2066
|
-
|
|
2067
|
-
this.
|
|
2209
|
+
if (this.pollTimeoutId) {
|
|
2210
|
+
clearTimeout(this.pollTimeoutId);
|
|
2211
|
+
this.pollTimeoutId = null;
|
|
2068
2212
|
}
|
|
2069
2213
|
if (this.heartbeatIntervalId) {
|
|
2070
2214
|
clearInterval(this.heartbeatIntervalId);
|
|
@@ -2072,6 +2216,43 @@ var LifecycleManager = class {
|
|
|
2072
2216
|
}
|
|
2073
2217
|
}
|
|
2074
2218
|
/**
|
|
2219
|
+
* Reset the poll timer to reschedule the next poll.
|
|
2220
|
+
*
|
|
2221
|
+
* Called after change-stream-triggered polls to ensure the safety poll timer
|
|
2222
|
+
* is recalculated (not fired redundantly from an old schedule).
|
|
2223
|
+
*/
|
|
2224
|
+
resetPollTimer() {
|
|
2225
|
+
this.scheduleNextPoll();
|
|
2226
|
+
}
|
|
2227
|
+
/**
|
|
2228
|
+
* Execute a poll and schedule the next one adaptively.
|
|
2229
|
+
*/
|
|
2230
|
+
executePollAndScheduleNext() {
|
|
2231
|
+
if (!this.callbacks) return;
|
|
2232
|
+
this.callbacks.poll().catch((error) => {
|
|
2233
|
+
this.ctx.emit("job:error", { error: toError(error) });
|
|
2234
|
+
}).then(() => {
|
|
2235
|
+
this.scheduleNextPoll();
|
|
2236
|
+
});
|
|
2237
|
+
}
|
|
2238
|
+
/**
|
|
2239
|
+
* Schedule the next poll using adaptive timing.
|
|
2240
|
+
*
|
|
2241
|
+
* When change streams are active, uses `safetyPollInterval` (longer, safety net only).
|
|
2242
|
+
* When change streams are inactive, uses `pollInterval` (shorter, primary discovery).
|
|
2243
|
+
*/
|
|
2244
|
+
scheduleNextPoll() {
|
|
2245
|
+
if (this.pollTimeoutId) {
|
|
2246
|
+
clearTimeout(this.pollTimeoutId);
|
|
2247
|
+
this.pollTimeoutId = null;
|
|
2248
|
+
}
|
|
2249
|
+
if (!this.ctx.isRunning() || !this.callbacks) return;
|
|
2250
|
+
const delay = this.callbacks.isChangeStreamActive() ? this.ctx.options.safetyPollInterval : this.ctx.options.pollInterval;
|
|
2251
|
+
this.pollTimeoutId = setTimeout(() => {
|
|
2252
|
+
this.executePollAndScheduleNext();
|
|
2253
|
+
}, delay);
|
|
2254
|
+
}
|
|
2255
|
+
/**
|
|
2075
2256
|
* Clean up old completed and failed jobs based on retention policy.
|
|
2076
2257
|
*
|
|
2077
2258
|
* - Removes completed jobs older than `jobRetention.completed`
|
|
@@ -2111,6 +2292,7 @@ var LifecycleManager = class {
|
|
|
2111
2292
|
const DEFAULTS = {
|
|
2112
2293
|
collectionName: "monque_jobs",
|
|
2113
2294
|
pollInterval: 1e3,
|
|
2295
|
+
safetyPollInterval: 3e4,
|
|
2114
2296
|
maxRetries: 10,
|
|
2115
2297
|
baseRetryInterval: 1e3,
|
|
2116
2298
|
shutdownTimeout: 3e4,
|
|
@@ -2203,6 +2385,7 @@ var Monque = class extends node_events.EventEmitter {
|
|
|
2203
2385
|
this.options = {
|
|
2204
2386
|
collectionName: options.collectionName ?? DEFAULTS.collectionName,
|
|
2205
2387
|
pollInterval: options.pollInterval ?? DEFAULTS.pollInterval,
|
|
2388
|
+
safetyPollInterval: options.safetyPollInterval ?? DEFAULTS.safetyPollInterval,
|
|
2206
2389
|
maxRetries: options.maxRetries ?? DEFAULTS.maxRetries,
|
|
2207
2390
|
baseRetryInterval: options.baseRetryInterval ?? DEFAULTS.baseRetryInterval,
|
|
2208
2391
|
shutdownTimeout: options.shutdownTimeout ?? DEFAULTS.shutdownTimeout,
|
|
@@ -2237,7 +2420,7 @@ var Monque = class extends node_events.EventEmitter {
|
|
|
2237
2420
|
this._manager = new JobManager(ctx);
|
|
2238
2421
|
this._query = new JobQueryService(ctx);
|
|
2239
2422
|
this._processor = new JobProcessor(ctx);
|
|
2240
|
-
this._changeStreamHandler = new ChangeStreamHandler(ctx, () => this.
|
|
2423
|
+
this._changeStreamHandler = new ChangeStreamHandler(ctx, (targetNames) => this.handleChangeStreamPoll(targetNames));
|
|
2241
2424
|
this._lifecycleManager = new LifecycleManager(ctx);
|
|
2242
2425
|
this.isInitialized = true;
|
|
2243
2426
|
} catch (error) {
|
|
@@ -2275,6 +2458,17 @@ var Monque = class extends node_events.EventEmitter {
|
|
|
2275
2458
|
return this._lifecycleManager;
|
|
2276
2459
|
}
|
|
2277
2460
|
/**
|
|
2461
|
+
* Handle a change-stream-triggered poll and reset the safety poll timer.
|
|
2462
|
+
*
|
|
2463
|
+
* Used as the `onPoll` callback for {@link ChangeStreamHandler}. Runs a
|
|
2464
|
+
* targeted poll for the given worker names, then resets the adaptive safety
|
|
2465
|
+
* poll timer so it doesn't fire redundantly.
|
|
2466
|
+
*/
|
|
2467
|
+
async handleChangeStreamPoll(targetNames) {
|
|
2468
|
+
await this.processor.poll(targetNames);
|
|
2469
|
+
this.lifecycleManager.resetPollTimer();
|
|
2470
|
+
}
|
|
2471
|
+
/**
|
|
2278
2472
|
* Build the shared context for internal services.
|
|
2279
2473
|
*/
|
|
2280
2474
|
buildContext() {
|
|
@@ -2286,6 +2480,10 @@ var Monque = class extends node_events.EventEmitter {
|
|
|
2286
2480
|
workers: this.workers,
|
|
2287
2481
|
isRunning: () => this.isRunning,
|
|
2288
2482
|
emit: (event, payload) => this.emit(event, payload),
|
|
2483
|
+
notifyPendingJob: (name, nextRunAt) => {
|
|
2484
|
+
if (!this.isRunning || !this._changeStreamHandler) return;
|
|
2485
|
+
this._changeStreamHandler.notifyPendingJob(name, nextRunAt);
|
|
2486
|
+
},
|
|
2289
2487
|
documentToPersistedJob: (doc) => documentToPersistedJob(doc)
|
|
2290
2488
|
};
|
|
2291
2489
|
}
|
|
@@ -2986,7 +3184,8 @@ var Monque = class extends node_events.EventEmitter {
|
|
|
2986
3184
|
this.changeStreamHandler.setup();
|
|
2987
3185
|
this.lifecycleManager.startTimers({
|
|
2988
3186
|
poll: () => this.processor.poll(),
|
|
2989
|
-
updateHeartbeats: () => this.processor.updateHeartbeats()
|
|
3187
|
+
updateHeartbeats: () => this.processor.updateHeartbeats(),
|
|
3188
|
+
isChangeStreamActive: () => this.changeStreamHandler.isActive()
|
|
2990
3189
|
});
|
|
2991
3190
|
}
|
|
2992
3191
|
/**
|