@happyvertical/smrt-jobs 0.37.2 → 0.37.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunks/runner-DLsWfeY-.js +1536 -0
- package/dist/chunks/runner-DLsWfeY-.js.map +1 -0
- package/dist/chunks/worker-liveness-C1Gjnhax.js +58 -0
- package/dist/chunks/worker-liveness-C1Gjnhax.js.map +1 -0
- package/dist/index.js +671 -862
- package/dist/index.js.map +1 -1
- package/dist/manifest.json +2 -2
- package/dist/playground.js +162 -174
- package/dist/playground.js.map +1 -1
- package/dist/runner.js +2 -16
- package/dist/smrt-knowledge.json +11 -11
- package/dist/ui.js +67 -67
- package/dist/ui.js.map +1 -1
- package/dist/worker-liveness-thread.js +48 -56
- package/dist/worker-liveness-thread.js.map +1 -1
- package/package.json +19 -19
- package/dist/chunks/runner-2zRlEef7.js +0 -1707
- package/dist/chunks/runner-2zRlEef7.js.map +0 -1
- package/dist/chunks/worker-liveness-DOTjoIjr.js +0 -65
- package/dist/chunks/worker-liveness-DOTjoIjr.js.map +0 -1
- package/dist/runner.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,619 +1,523 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { exponential } from "@happyvertical/jobs";
|
|
1
|
+
import { S as markBackgroundEligible, _ as assertWithinTenantCreationCap, a as SmrtWorker, b as getBackgroundEligibleMethods, c as SmrtJobEventCollection, d as JobContextLogger, f as redactErrorForPersistence, g as TenantJobCapExceededError, h as MAX_JOB_RETRIES, i as DEFAULT_TASK_HEARTBEAT_INTERVAL_MS, l as SmrtJob, m as DEFAULT_TENANT_JOB_CAP, n as TaskRunner, o as SmrtWorkerCollection, p as redactErrorMessage, r as createTaskRunner, s as SmrtJobEvent, t as JobTimeoutError, u as SmrtJobCollection, v as backgroundEligible, x as isBackgroundEligibleMethod, y as clampRetries } from "./chunks/runner-DLsWfeY-.js";
|
|
2
|
+
import { c as unregisterLiveWorker, i as registerLiveWorker, n as isWorkerAlive, t as createWorkerKey } from "./chunks/worker-liveness-C1Gjnhax.js";
|
|
4
3
|
import { ObjectRegistry } from "@happyvertical/smrt-core";
|
|
4
|
+
import { exponential } from "@happyvertical/jobs";
|
|
5
5
|
import { EventEmitter } from "node:events";
|
|
6
6
|
import { createLogger } from "@happyvertical/logger";
|
|
7
7
|
import { createId } from "@happyvertical/utils";
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
async isRunning() {
|
|
88
|
-
const status = await this.status();
|
|
89
|
-
return status === "pending" || status === "running";
|
|
90
|
-
}
|
|
91
|
-
/**
|
|
92
|
-
* Check if the job has completed (successfully or not)
|
|
93
|
-
*/
|
|
94
|
-
async isDone() {
|
|
95
|
-
const status = await this.status();
|
|
96
|
-
return status === "completed" || status === "failed" || status === "cancelled";
|
|
97
|
-
}
|
|
98
|
-
}
|
|
8
|
+
//#region src/job-handle.ts
|
|
9
|
+
var JobHandle = class {
|
|
10
|
+
constructor(id, collection) {
|
|
11
|
+
this.id = id;
|
|
12
|
+
this.collection = collection;
|
|
13
|
+
}
|
|
14
|
+
id;
|
|
15
|
+
collection;
|
|
16
|
+
/**
|
|
17
|
+
* Get the current job status
|
|
18
|
+
*/
|
|
19
|
+
async status() {
|
|
20
|
+
return (await this.getJob()).status;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Get the full job object
|
|
24
|
+
*/
|
|
25
|
+
async getJob() {
|
|
26
|
+
const job = await this.collection.get({ id: this.id });
|
|
27
|
+
if (!job) throw new Error(`Job not found: ${this.id}`);
|
|
28
|
+
return job;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Wait for the job to complete
|
|
32
|
+
*
|
|
33
|
+
* @param options - Wait configuration
|
|
34
|
+
* @returns The job result
|
|
35
|
+
* @throws Error if the job fails or times out
|
|
36
|
+
*/
|
|
37
|
+
async wait(options = {}) {
|
|
38
|
+
const { timeout = 6e4, pollInterval = 100 } = options;
|
|
39
|
+
const startTime = Date.now();
|
|
40
|
+
while (true) {
|
|
41
|
+
const job = await this.getJob();
|
|
42
|
+
if (job.status === "completed") return {
|
|
43
|
+
success: true,
|
|
44
|
+
resultPointer: job.resultPointer
|
|
45
|
+
};
|
|
46
|
+
if (job.status === "failed") return {
|
|
47
|
+
success: false,
|
|
48
|
+
error: job.lastError ?? "Job failed"
|
|
49
|
+
};
|
|
50
|
+
if (job.status === "cancelled") return {
|
|
51
|
+
success: false,
|
|
52
|
+
error: "Job was cancelled"
|
|
53
|
+
};
|
|
54
|
+
if (Date.now() - startTime >= timeout) throw new Error(`Timeout waiting for job ${this.id}`);
|
|
55
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Cancel the job
|
|
60
|
+
*/
|
|
61
|
+
async cancel() {
|
|
62
|
+
await (await this.getJob()).cancel();
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Retry a failed job
|
|
66
|
+
*/
|
|
67
|
+
async retry() {
|
|
68
|
+
await (await this.getJob()).retry();
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Check if the job is still running
|
|
72
|
+
*/
|
|
73
|
+
async isRunning() {
|
|
74
|
+
const status = await this.status();
|
|
75
|
+
return status === "pending" || status === "running";
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Check if the job has completed (successfully or not)
|
|
79
|
+
*/
|
|
80
|
+
async isDone() {
|
|
81
|
+
const status = await this.status();
|
|
82
|
+
return status === "completed" || status === "failed" || status === "cancelled";
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
//#endregion
|
|
86
|
+
//#region src/job-builder.ts
|
|
99
87
|
function priorityToNumber(priority) {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
case "low":
|
|
109
|
-
return 25;
|
|
110
|
-
default:
|
|
111
|
-
return 50;
|
|
112
|
-
}
|
|
88
|
+
if (typeof priority === "number") return priority;
|
|
89
|
+
switch (priority) {
|
|
90
|
+
case "critical": return 100;
|
|
91
|
+
case "high": return 75;
|
|
92
|
+
case "normal": return 50;
|
|
93
|
+
case "low": return 25;
|
|
94
|
+
default: return 50;
|
|
95
|
+
}
|
|
113
96
|
}
|
|
114
97
|
function parseDelay(delay) {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
case "s":
|
|
131
|
-
return value * 1e3;
|
|
132
|
-
case "m":
|
|
133
|
-
return value * 60 * 1e3;
|
|
134
|
-
case "h":
|
|
135
|
-
return value * 60 * 60 * 1e3;
|
|
136
|
-
case "d":
|
|
137
|
-
return value * 24 * 60 * 60 * 1e3;
|
|
138
|
-
default:
|
|
139
|
-
return value;
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
class JobBuilder {
|
|
143
|
-
constructor(objectType, objectId, method, args, collection) {
|
|
144
|
-
this.objectType = objectType;
|
|
145
|
-
this.objectId = objectId;
|
|
146
|
-
this.method = method;
|
|
147
|
-
this.args = args;
|
|
148
|
-
this.collection = collection;
|
|
149
|
-
}
|
|
150
|
-
objectType;
|
|
151
|
-
objectId;
|
|
152
|
-
method;
|
|
153
|
-
args;
|
|
154
|
-
collection;
|
|
155
|
-
_queue = "default";
|
|
156
|
-
_delay = 0;
|
|
157
|
-
_retries = 3;
|
|
158
|
-
_priority = 50;
|
|
159
|
-
_timeout = 3e5;
|
|
160
|
-
_timeoutBehavior = "fail";
|
|
161
|
-
_retryStrategy = exponential();
|
|
162
|
-
_tenantJobCap = DEFAULT_TENANT_JOB_CAP;
|
|
163
|
-
/**
|
|
164
|
-
* Set the queue name
|
|
165
|
-
*/
|
|
166
|
-
queue(name) {
|
|
167
|
-
this._queue = name;
|
|
168
|
-
return this;
|
|
169
|
-
}
|
|
170
|
-
/**
|
|
171
|
-
* Set a delay before the job runs
|
|
172
|
-
* @param delay - Delay as milliseconds or string like '5m', '1h', '30s'
|
|
173
|
-
*/
|
|
174
|
-
delay(delay) {
|
|
175
|
-
this._delay = parseDelay(delay);
|
|
176
|
-
return this;
|
|
177
|
-
}
|
|
178
|
-
/**
|
|
179
|
-
* Set when the job should run
|
|
180
|
-
*/
|
|
181
|
-
runAt(date) {
|
|
182
|
-
this._delay = date.getTime() - Date.now();
|
|
183
|
-
return this;
|
|
184
|
-
}
|
|
185
|
-
/**
|
|
186
|
-
* Set the maximum number of retry attempts.
|
|
187
|
-
*
|
|
188
|
-
* Clamped to {@link MAX_JOB_RETRIES} so a misconfigured caller cannot pin a
|
|
189
|
-
* worker on a poison job indefinitely (S5 audit #1402).
|
|
190
|
-
*/
|
|
191
|
-
retries(count) {
|
|
192
|
-
this._retries = clampRetries(count);
|
|
193
|
-
return this;
|
|
194
|
-
}
|
|
195
|
-
/**
|
|
196
|
-
* Set the retry strategy
|
|
197
|
-
*/
|
|
198
|
-
retryStrategy(strategy) {
|
|
199
|
-
this._retryStrategy = strategy;
|
|
200
|
-
return this;
|
|
201
|
-
}
|
|
202
|
-
/**
|
|
203
|
-
* Set the job priority
|
|
204
|
-
*/
|
|
205
|
-
priority(level) {
|
|
206
|
-
this._priority = priorityToNumber(level);
|
|
207
|
-
return this;
|
|
208
|
-
}
|
|
209
|
-
/**
|
|
210
|
-
* Set the job timeout in milliseconds
|
|
211
|
-
*/
|
|
212
|
-
timeout(ms) {
|
|
213
|
-
this._timeout = ms;
|
|
214
|
-
return this;
|
|
215
|
-
}
|
|
216
|
-
/**
|
|
217
|
-
* Set what happens when the job times out
|
|
218
|
-
*/
|
|
219
|
-
timeoutBehavior(behavior) {
|
|
220
|
-
this._timeoutBehavior = behavior;
|
|
221
|
-
return this;
|
|
222
|
-
}
|
|
223
|
-
/**
|
|
224
|
-
* Override the per-tenant in-flight job cap for this enqueue.
|
|
225
|
-
*
|
|
226
|
-
* Defaults to {@link DEFAULT_TENANT_JOB_CAP}. Pass `0` (or a negative value)
|
|
227
|
-
* to disable the cap for trusted internal callers (S5 audit #1402).
|
|
228
|
-
*/
|
|
229
|
-
tenantJobCap(max) {
|
|
230
|
-
this._tenantJobCap = max;
|
|
231
|
-
return this;
|
|
232
|
-
}
|
|
233
|
-
/**
|
|
234
|
-
* Enqueue the job and return a handle
|
|
235
|
-
*/
|
|
236
|
-
async enqueue() {
|
|
237
|
-
const runAt = new Date(Date.now() + this._delay);
|
|
238
|
-
const retryConfig = "toConfig" in this._retryStrategy ? this._retryStrategy.toConfig() : this._retryStrategy;
|
|
239
|
-
const job = await this.collection.enqueueJob(
|
|
240
|
-
{
|
|
241
|
-
queue: this._queue,
|
|
242
|
-
objectType: this.objectType,
|
|
243
|
-
objectId: this.objectId,
|
|
244
|
-
method: this.method,
|
|
245
|
-
args: this.args,
|
|
246
|
-
runAt,
|
|
247
|
-
priority: this._priority,
|
|
248
|
-
maxAttempts: this._retries,
|
|
249
|
-
timeout: this._timeout,
|
|
250
|
-
timeoutBehavior: this._timeoutBehavior,
|
|
251
|
-
retryStrategy: retryConfig
|
|
252
|
-
},
|
|
253
|
-
{ tenantJobCap: this._tenantJobCap }
|
|
254
|
-
);
|
|
255
|
-
const jobId = job.id;
|
|
256
|
-
if (!jobId) {
|
|
257
|
-
throw new Error("Job was created but has no ID");
|
|
258
|
-
}
|
|
259
|
-
return new JobHandle(jobId, this.collection);
|
|
260
|
-
}
|
|
98
|
+
if (typeof delay === "number") {
|
|
99
|
+
if (!Number.isFinite(delay)) throw new Error(`Invalid delay value: ${delay}`);
|
|
100
|
+
return delay;
|
|
101
|
+
}
|
|
102
|
+
const match = delay.match(/^(\d+)(ms|s|m|h|d)?$/);
|
|
103
|
+
if (!match) throw new Error(`Invalid delay format: ${delay}`);
|
|
104
|
+
const value = parseInt(match[1], 10);
|
|
105
|
+
switch (match[2] || "ms") {
|
|
106
|
+
case "ms": return value;
|
|
107
|
+
case "s": return value * 1e3;
|
|
108
|
+
case "m": return value * 60 * 1e3;
|
|
109
|
+
case "h": return value * 60 * 60 * 1e3;
|
|
110
|
+
case "d": return value * 24 * 60 * 60 * 1e3;
|
|
111
|
+
default: return value;
|
|
112
|
+
}
|
|
261
113
|
}
|
|
262
|
-
|
|
114
|
+
var JobBuilder = class {
|
|
115
|
+
constructor(objectType, objectId, method, args, collection) {
|
|
116
|
+
this.objectType = objectType;
|
|
117
|
+
this.objectId = objectId;
|
|
118
|
+
this.method = method;
|
|
119
|
+
this.args = args;
|
|
120
|
+
this.collection = collection;
|
|
121
|
+
}
|
|
122
|
+
objectType;
|
|
123
|
+
objectId;
|
|
124
|
+
method;
|
|
125
|
+
args;
|
|
126
|
+
collection;
|
|
127
|
+
_queue = "default";
|
|
128
|
+
_delay = 0;
|
|
129
|
+
_retries = 3;
|
|
130
|
+
_priority = 50;
|
|
131
|
+
_timeout = 3e5;
|
|
132
|
+
_timeoutBehavior = "fail";
|
|
133
|
+
_retryStrategy = exponential();
|
|
134
|
+
_tenantJobCap = DEFAULT_TENANT_JOB_CAP;
|
|
135
|
+
/**
|
|
136
|
+
* Set the queue name
|
|
137
|
+
*/
|
|
138
|
+
queue(name) {
|
|
139
|
+
this._queue = name;
|
|
140
|
+
return this;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Set a delay before the job runs
|
|
144
|
+
* @param delay - Delay as milliseconds or string like '5m', '1h', '30s'
|
|
145
|
+
*/
|
|
146
|
+
delay(delay) {
|
|
147
|
+
this._delay = parseDelay(delay);
|
|
148
|
+
return this;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Set when the job should run
|
|
152
|
+
*/
|
|
153
|
+
runAt(date) {
|
|
154
|
+
this._delay = date.getTime() - Date.now();
|
|
155
|
+
return this;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Set the maximum number of retry attempts.
|
|
159
|
+
*
|
|
160
|
+
* Clamped to {@link MAX_JOB_RETRIES} so a misconfigured caller cannot pin a
|
|
161
|
+
* worker on a poison job indefinitely (S5 audit #1402).
|
|
162
|
+
*/
|
|
163
|
+
retries(count) {
|
|
164
|
+
this._retries = clampRetries(count);
|
|
165
|
+
return this;
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Set the retry strategy
|
|
169
|
+
*/
|
|
170
|
+
retryStrategy(strategy) {
|
|
171
|
+
this._retryStrategy = strategy;
|
|
172
|
+
return this;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Set the job priority
|
|
176
|
+
*/
|
|
177
|
+
priority(level) {
|
|
178
|
+
this._priority = priorityToNumber(level);
|
|
179
|
+
return this;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Set the job timeout in milliseconds
|
|
183
|
+
*/
|
|
184
|
+
timeout(ms) {
|
|
185
|
+
this._timeout = ms;
|
|
186
|
+
return this;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Set what happens when the job times out
|
|
190
|
+
*/
|
|
191
|
+
timeoutBehavior(behavior) {
|
|
192
|
+
this._timeoutBehavior = behavior;
|
|
193
|
+
return this;
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Override the per-tenant in-flight job cap for this enqueue.
|
|
197
|
+
*
|
|
198
|
+
* Defaults to {@link DEFAULT_TENANT_JOB_CAP}. Pass `0` (or a negative value)
|
|
199
|
+
* to disable the cap for trusted internal callers (S5 audit #1402).
|
|
200
|
+
*/
|
|
201
|
+
tenantJobCap(max) {
|
|
202
|
+
this._tenantJobCap = max;
|
|
203
|
+
return this;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Enqueue the job and return a handle
|
|
207
|
+
*/
|
|
208
|
+
async enqueue() {
|
|
209
|
+
const runAt = new Date(Date.now() + this._delay);
|
|
210
|
+
const retryConfig = "toConfig" in this._retryStrategy ? this._retryStrategy.toConfig() : this._retryStrategy;
|
|
211
|
+
const jobId = (await this.collection.enqueueJob({
|
|
212
|
+
queue: this._queue,
|
|
213
|
+
objectType: this.objectType,
|
|
214
|
+
objectId: this.objectId,
|
|
215
|
+
method: this.method,
|
|
216
|
+
args: this.args,
|
|
217
|
+
runAt,
|
|
218
|
+
priority: this._priority,
|
|
219
|
+
maxAttempts: this._retries,
|
|
220
|
+
timeout: this._timeout,
|
|
221
|
+
timeoutBehavior: this._timeoutBehavior,
|
|
222
|
+
retryStrategy: retryConfig
|
|
223
|
+
}, { tenantJobCap: this._tenantJobCap })).id;
|
|
224
|
+
if (!jobId) throw new Error("Job was created but has no ID");
|
|
225
|
+
return new JobHandle(jobId, this.collection);
|
|
226
|
+
}
|
|
227
|
+
};
|
|
228
|
+
//#endregion
|
|
229
|
+
//#region src/object-extension.ts
|
|
230
|
+
var collectionCache = /* @__PURE__ */ new WeakMap();
|
|
263
231
|
async function getJobCollection(db) {
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
232
|
+
let collection = collectionCache.get(db);
|
|
233
|
+
if (!collection) {
|
|
234
|
+
collection = await SmrtJobCollection.create({ db: {
|
|
235
|
+
type: "sqlite",
|
|
236
|
+
url: ":memory:"
|
|
237
|
+
} });
|
|
238
|
+
collection._db = db;
|
|
239
|
+
collectionCache.set(db, collection);
|
|
240
|
+
}
|
|
241
|
+
return collection;
|
|
274
242
|
}
|
|
275
243
|
function getObjectTypeName(instance) {
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
const className = instance.constructor.name;
|
|
281
|
-
return ObjectRegistry.getClass(className)?.qualifiedName || className;
|
|
244
|
+
const metaType = instance._meta_type;
|
|
245
|
+
if (typeof metaType === "string" && metaType.length > 0) return metaType;
|
|
246
|
+
const className = instance.constructor.name;
|
|
247
|
+
return ObjectRegistry.getClass(className)?.qualifiedName || className;
|
|
282
248
|
}
|
|
283
249
|
function requireObjectDb(instance) {
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
}
|
|
288
|
-
return db;
|
|
250
|
+
const db = instance._db;
|
|
251
|
+
if (!db) throw new Error("Object not initialized. Call initialize() first.");
|
|
252
|
+
return db;
|
|
289
253
|
}
|
|
290
254
|
async function bgImpl(method, args = {}, options = {}) {
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
);
|
|
300
|
-
if (options.queue) builder.queue(options.queue);
|
|
301
|
-
if (options.priority) builder.priority(options.priority);
|
|
302
|
-
if (options.delay) builder.delay(options.delay);
|
|
303
|
-
if (options.retries !== void 0) builder.retries(options.retries);
|
|
304
|
-
if (options.timeout) builder.timeout(options.timeout);
|
|
305
|
-
return builder.enqueue();
|
|
255
|
+
const collection = await getJobCollection(requireObjectDb(this));
|
|
256
|
+
const builder = new JobBuilder(getObjectTypeName(this), this.id ?? null, method, args, collection);
|
|
257
|
+
if (options.queue) builder.queue(options.queue);
|
|
258
|
+
if (options.priority) builder.priority(options.priority);
|
|
259
|
+
if (options.delay) builder.delay(options.delay);
|
|
260
|
+
if (options.retries !== void 0) builder.retries(options.retries);
|
|
261
|
+
if (options.timeout) builder.timeout(options.timeout);
|
|
262
|
+
return builder.enqueue();
|
|
306
263
|
}
|
|
307
264
|
function backgroundImpl(method, args = {}) {
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
builder.retries(this._retries);
|
|
370
|
-
builder.priority(this._priority);
|
|
371
|
-
builder.timeout(this._timeout);
|
|
372
|
-
builder.timeoutBehavior(this._timeoutBehavior);
|
|
373
|
-
if (this._tenantJobCap !== void 0) {
|
|
374
|
-
builder.tenantJobCap(this._tenantJobCap);
|
|
375
|
-
}
|
|
376
|
-
if (this._retryStrategy) {
|
|
377
|
-
builder.retryStrategy(
|
|
378
|
-
this._retryStrategy
|
|
379
|
-
);
|
|
380
|
-
}
|
|
381
|
-
return builder.enqueue();
|
|
382
|
-
}
|
|
383
|
-
};
|
|
384
|
-
return lazyBuilder;
|
|
265
|
+
const db = requireObjectDb(this);
|
|
266
|
+
const objectType = getObjectTypeName(this);
|
|
267
|
+
const objectId = this.id ?? null;
|
|
268
|
+
return {
|
|
269
|
+
_queue: "default",
|
|
270
|
+
_delay: 0,
|
|
271
|
+
_retries: 3,
|
|
272
|
+
_priority: 50,
|
|
273
|
+
_timeout: 3e5,
|
|
274
|
+
_timeoutBehavior: "fail",
|
|
275
|
+
_retryStrategy: null,
|
|
276
|
+
_tenantJobCap: void 0,
|
|
277
|
+
queue(name) {
|
|
278
|
+
this._queue = name;
|
|
279
|
+
return this;
|
|
280
|
+
},
|
|
281
|
+
delay(d) {
|
|
282
|
+
this._delay = parseDelay(d);
|
|
283
|
+
return this;
|
|
284
|
+
},
|
|
285
|
+
runAt(date) {
|
|
286
|
+
this._delay = date.getTime() - Date.now();
|
|
287
|
+
return this;
|
|
288
|
+
},
|
|
289
|
+
retries(count) {
|
|
290
|
+
this._retries = count;
|
|
291
|
+
return this;
|
|
292
|
+
},
|
|
293
|
+
retryStrategy(strategy) {
|
|
294
|
+
this._retryStrategy = strategy;
|
|
295
|
+
return this;
|
|
296
|
+
},
|
|
297
|
+
priority(level) {
|
|
298
|
+
this._priority = priorityToNumber(level);
|
|
299
|
+
return this;
|
|
300
|
+
},
|
|
301
|
+
timeout(ms) {
|
|
302
|
+
this._timeout = ms;
|
|
303
|
+
return this;
|
|
304
|
+
},
|
|
305
|
+
timeoutBehavior(behavior) {
|
|
306
|
+
this._timeoutBehavior = behavior;
|
|
307
|
+
return this;
|
|
308
|
+
},
|
|
309
|
+
tenantJobCap(max) {
|
|
310
|
+
this._tenantJobCap = max;
|
|
311
|
+
return this;
|
|
312
|
+
},
|
|
313
|
+
async enqueue() {
|
|
314
|
+
const builder = new JobBuilder(objectType, objectId, method, args, await getJobCollection(db));
|
|
315
|
+
builder.queue(this._queue);
|
|
316
|
+
builder.delay(this._delay);
|
|
317
|
+
builder.retries(this._retries);
|
|
318
|
+
builder.priority(this._priority);
|
|
319
|
+
builder.timeout(this._timeout);
|
|
320
|
+
builder.timeoutBehavior(this._timeoutBehavior);
|
|
321
|
+
if (this._tenantJobCap !== void 0) builder.tenantJobCap(this._tenantJobCap);
|
|
322
|
+
if (this._retryStrategy) builder.retryStrategy(this._retryStrategy);
|
|
323
|
+
return builder.enqueue();
|
|
324
|
+
}
|
|
325
|
+
};
|
|
385
326
|
}
|
|
386
327
|
function withBackgroundJobs(BaseClass) {
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
configurable: true
|
|
400
|
-
});
|
|
401
|
-
}
|
|
402
|
-
return BaseClass;
|
|
328
|
+
const prototype = BaseClass.prototype;
|
|
329
|
+
if (typeof prototype.bg !== "function") Object.defineProperty(prototype, "bg", {
|
|
330
|
+
value: bgImpl,
|
|
331
|
+
writable: true,
|
|
332
|
+
configurable: true
|
|
333
|
+
});
|
|
334
|
+
if (typeof prototype.background !== "function") Object.defineProperty(prototype, "background", {
|
|
335
|
+
value: backgroundImpl,
|
|
336
|
+
writable: true,
|
|
337
|
+
configurable: true
|
|
338
|
+
});
|
|
339
|
+
return BaseClass;
|
|
403
340
|
}
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
341
|
+
//#endregion
|
|
342
|
+
//#region src/schedule-runner.ts
|
|
343
|
+
var DEFAULT_CONFIG = {
|
|
344
|
+
id: "",
|
|
345
|
+
pollInterval: 6e4,
|
|
346
|
+
batchSize: 50,
|
|
347
|
+
staleJobThresholdMs: 9e4,
|
|
348
|
+
taskHeartbeatInterval: DEFAULT_TASK_HEARTBEAT_INTERVAL_MS
|
|
411
349
|
};
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
const safeErrorMessage = redactErrorMessage(
|
|
481
|
-
errorMessage ?? "Unknown error"
|
|
482
|
-
);
|
|
483
|
-
try {
|
|
484
|
-
if (success) {
|
|
485
|
-
await this.db.query(
|
|
486
|
-
`UPDATE _smrt_agent_schedules
|
|
350
|
+
var ScheduleRunner = class extends EventEmitter {
|
|
351
|
+
id;
|
|
352
|
+
config;
|
|
353
|
+
jobCollection = null;
|
|
354
|
+
workerCollection = null;
|
|
355
|
+
running = false;
|
|
356
|
+
pollTimer = null;
|
|
357
|
+
db = null;
|
|
358
|
+
logger = createLogger(true);
|
|
359
|
+
constructor(config = {}) {
|
|
360
|
+
super();
|
|
361
|
+
this.config = {
|
|
362
|
+
...DEFAULT_CONFIG,
|
|
363
|
+
...config,
|
|
364
|
+
id: config.id || `schedule_${createId().slice(0, 8)}`
|
|
365
|
+
};
|
|
366
|
+
this.id = this.config.id;
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* Initialize the runner with database connection
|
|
370
|
+
*/
|
|
371
|
+
async initialize(db) {
|
|
372
|
+
this.db = db;
|
|
373
|
+
this.jobCollection = await SmrtJobCollection.create({ db });
|
|
374
|
+
this.workerCollection = await SmrtWorkerCollection.create({ db });
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Start processing schedules
|
|
378
|
+
*/
|
|
379
|
+
async start() {
|
|
380
|
+
if (this.running) return;
|
|
381
|
+
if (!this.db) throw new Error("ScheduleRunner not initialized. Call initialize() first.");
|
|
382
|
+
this.running = true;
|
|
383
|
+
this.startPolling();
|
|
384
|
+
this.emit("runner:started");
|
|
385
|
+
this.logger.info("ScheduleRunner started", { id: this.id });
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* Stop processing schedules
|
|
389
|
+
*/
|
|
390
|
+
async stop() {
|
|
391
|
+
if (!this.running) return;
|
|
392
|
+
this.running = false;
|
|
393
|
+
if (this.pollTimer) {
|
|
394
|
+
clearTimeout(this.pollTimer);
|
|
395
|
+
this.pollTimer = null;
|
|
396
|
+
}
|
|
397
|
+
this.emit("runner:stopped");
|
|
398
|
+
this.logger.info("ScheduleRunner stopped", { id: this.id });
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Check if runner is running
|
|
402
|
+
*/
|
|
403
|
+
isRunning() {
|
|
404
|
+
return this.running;
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Handle job completion for a scheduled job.
|
|
408
|
+
*
|
|
409
|
+
* Call this from TaskRunner's job:completed / job:failed events
|
|
410
|
+
* when the job has a `_scheduleId` in its args.
|
|
411
|
+
*/
|
|
412
|
+
async handleJobCompletion(scheduleId, success, errorMessage) {
|
|
413
|
+
if (!this.db) return;
|
|
414
|
+
const safeErrorMessage = redactErrorMessage(errorMessage ?? "Unknown error");
|
|
415
|
+
try {
|
|
416
|
+
if (success) {
|
|
417
|
+
await this.db.query(`UPDATE _smrt_agent_schedules
|
|
487
418
|
SET running_count = CASE WHEN COALESCE(running_count, 0) > 0 THEN running_count - 1 ELSE 0 END,
|
|
488
419
|
last_run = ?,
|
|
489
420
|
last_status = 'success',
|
|
490
421
|
last_error = NULL,
|
|
491
422
|
run_count = COALESCE(run_count, 0) + 1,
|
|
492
423
|
success_count = COALESCE(success_count, 0) + 1
|
|
493
|
-
WHERE id = ?`,
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
this.emit("schedule:completed", scheduleId);
|
|
498
|
-
} else {
|
|
499
|
-
await this.db.query(
|
|
500
|
-
`UPDATE _smrt_agent_schedules
|
|
424
|
+
WHERE id = ?`, (/* @__PURE__ */ new Date()).toISOString(), scheduleId);
|
|
425
|
+
this.emit("schedule:completed", scheduleId);
|
|
426
|
+
} else {
|
|
427
|
+
await this.db.query(`UPDATE _smrt_agent_schedules
|
|
501
428
|
SET running_count = CASE WHEN COALESCE(running_count, 0) > 0 THEN running_count - 1 ELSE 0 END,
|
|
502
429
|
last_run = ?,
|
|
503
430
|
last_status = 'failed',
|
|
504
431
|
last_error = ?,
|
|
505
432
|
run_count = COALESCE(run_count, 0) + 1,
|
|
506
433
|
failure_count = COALESCE(failure_count, 0) + 1
|
|
507
|
-
WHERE id = ?`,
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
*/
|
|
542
|
-
async poll() {
|
|
543
|
-
if (!this.db || !this.jobCollection) return;
|
|
544
|
-
await this.recoverStaleScheduleState();
|
|
545
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
546
|
-
const result = await this.db.query(
|
|
547
|
-
`SELECT * FROM _smrt_agent_schedules
|
|
434
|
+
WHERE id = ?`, (/* @__PURE__ */ new Date()).toISOString(), safeErrorMessage, scheduleId);
|
|
435
|
+
this.emit("schedule:failed", scheduleId, safeErrorMessage);
|
|
436
|
+
}
|
|
437
|
+
} catch (err) {
|
|
438
|
+
this.logger.error("Failed to update schedule after job completion", {
|
|
439
|
+
scheduleId,
|
|
440
|
+
error: err
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Start the polling loop
|
|
446
|
+
*/
|
|
447
|
+
startPolling() {
|
|
448
|
+
const poll = async () => {
|
|
449
|
+
if (!this.running) return;
|
|
450
|
+
try {
|
|
451
|
+
await this.poll();
|
|
452
|
+
} catch (error) {
|
|
453
|
+
this.emit("runner:error", error);
|
|
454
|
+
this.logger.error("ScheduleRunner poll error", { error });
|
|
455
|
+
}
|
|
456
|
+
if (this.running) this.pollTimer = setTimeout(poll, this.config.pollInterval);
|
|
457
|
+
};
|
|
458
|
+
poll();
|
|
459
|
+
}
|
|
460
|
+
/**
|
|
461
|
+
* Poll for due schedules and create jobs
|
|
462
|
+
*/
|
|
463
|
+
async poll() {
|
|
464
|
+
if (!this.db || !this.jobCollection) return;
|
|
465
|
+
await this.recoverStaleScheduleState();
|
|
466
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
467
|
+
const result = await this.db.query(`SELECT * FROM _smrt_agent_schedules
|
|
548
468
|
WHERE enabled = true
|
|
549
469
|
AND status = 'active'
|
|
550
470
|
AND next_run <= ?
|
|
551
471
|
AND COALESCE(running_count, 0) < COALESCE(max_concurrent, 1)
|
|
552
472
|
ORDER BY next_run ASC
|
|
553
|
-
LIMIT ?`,
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
* its entire correctness mechanism.
|
|
573
|
-
*/
|
|
574
|
-
async recoverStaleScheduleState() {
|
|
575
|
-
if (!this.db || !this.workerCollection) return;
|
|
576
|
-
const schedulesResult = await this.db.query(
|
|
577
|
-
`SELECT id, running_count
|
|
473
|
+
LIMIT ?`, now, this.config.batchSize);
|
|
474
|
+
for (const row of result.rows) await this.triggerSchedule(row);
|
|
475
|
+
}
|
|
476
|
+
/**
|
|
477
|
+
* Reconcile stuck schedule slots against running jobs.
|
|
478
|
+
*
|
|
479
|
+
* This handles two failure modes:
|
|
480
|
+
* - a running job's owning worker is no longer alive (dead/restarted)
|
|
481
|
+
* - a schedule slot remains occupied even though no running job still exists
|
|
482
|
+
*
|
|
483
|
+
* Staleness keys on worker *liveness* (issue #1474), not per-job heartbeat
|
|
484
|
+
* freshness: a job whose `worker_id` is live in this process or holds a fresh
|
|
485
|
+
* lease in `_smrt_workers` is healthy even if its handler is holding the loop
|
|
486
|
+
* synchronously. ScheduleRunner has no in-process active-job set, so this is
|
|
487
|
+
* its entire correctness mechanism.
|
|
488
|
+
*/
|
|
489
|
+
async recoverStaleScheduleState() {
|
|
490
|
+
if (!this.db || !this.workerCollection) return;
|
|
491
|
+
const schedules = (await this.db.query(`SELECT id, running_count
|
|
578
492
|
FROM _smrt_agent_schedules
|
|
579
|
-
WHERE COALESCE(running_count, 0) > 0`
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
const freshLeaseKeys = workersReady ? await this.workerCollection.freshLeaseWorkerKeys() : /* @__PURE__ */ new Set();
|
|
585
|
-
const jobsResult = await this.db.query(
|
|
586
|
-
`SELECT id, args, worker_id
|
|
493
|
+
WHERE COALESCE(running_count, 0) > 0`)).rows;
|
|
494
|
+
if (schedules.length === 0) return;
|
|
495
|
+
const workersReady = await this.workerCollection.tableReady();
|
|
496
|
+
const freshLeaseKeys = workersReady ? await this.workerCollection.freshLeaseWorkerKeys() : /* @__PURE__ */ new Set();
|
|
497
|
+
const jobRows = (await this.db.query(`SELECT id, args, worker_id
|
|
587
498
|
FROM _smrt_jobs
|
|
588
|
-
WHERE status = 'running'`
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
return state?.staleJobIds ?? [];
|
|
611
|
-
});
|
|
612
|
-
const recoveredJobIds = /* @__PURE__ */ new Set();
|
|
613
|
-
if (staleJobIds.length > 0) {
|
|
614
|
-
const placeholders = staleJobIds.map(() => "?").join(", ");
|
|
615
|
-
const result = await this.db.query(
|
|
616
|
-
`UPDATE _smrt_jobs
|
|
499
|
+
WHERE status = 'running'`)).rows;
|
|
500
|
+
const stateBySchedule = /* @__PURE__ */ new Map();
|
|
501
|
+
for (const schedule of schedules) stateBySchedule.set(schedule.id, {
|
|
502
|
+
live: 0,
|
|
503
|
+
staleJobIds: []
|
|
504
|
+
});
|
|
505
|
+
for (const row of jobRows) {
|
|
506
|
+
const scheduleId = this.getScheduleIdFromJobArgs(row.args);
|
|
507
|
+
if (!scheduleId) continue;
|
|
508
|
+
const state = stateBySchedule.get(scheduleId);
|
|
509
|
+
if (!state) continue;
|
|
510
|
+
if (!(workersReady ? isWorkerAlive(row.worker_id, freshLeaseKeys) : true)) state.staleJobIds.push(row.id);
|
|
511
|
+
else state.live += 1;
|
|
512
|
+
}
|
|
513
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
514
|
+
const staleJobIds = schedules.flatMap((schedule) => {
|
|
515
|
+
return stateBySchedule.get(schedule.id)?.staleJobIds ?? [];
|
|
516
|
+
});
|
|
517
|
+
const recoveredJobIds = /* @__PURE__ */ new Set();
|
|
518
|
+
if (staleJobIds.length > 0) {
|
|
519
|
+
const placeholders = staleJobIds.map(() => "?").join(", ");
|
|
520
|
+
const result = await this.db.query(`UPDATE _smrt_jobs
|
|
617
521
|
SET status = 'failed',
|
|
618
522
|
completed_at = ?,
|
|
619
523
|
last_error = ?,
|
|
@@ -621,311 +525,216 @@ class ScheduleRunner extends EventEmitter {
|
|
|
621
525
|
worker_heartbeat = NULL
|
|
622
526
|
WHERE status = 'running'
|
|
623
527
|
AND id IN (${placeholders})
|
|
624
|
-
RETURNING id`,
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
if (!state) continue;
|
|
636
|
-
const desiredRunningCount = state.live;
|
|
637
|
-
const recoveredCount = state.staleJobIds.filter(
|
|
638
|
-
(id) => recoveredJobIds.has(id)
|
|
639
|
-
).length;
|
|
640
|
-
if (Number(schedule.running_count) === desiredRunningCount && recoveredCount === 0) {
|
|
641
|
-
continue;
|
|
642
|
-
}
|
|
643
|
-
if (recoveredCount > 0) {
|
|
644
|
-
await this.db.query(
|
|
645
|
-
`UPDATE _smrt_agent_schedules
|
|
528
|
+
RETURNING id`, now, "Recovered orphaned scheduled job: its owning worker is no longer alive (no fresh liveness lease in _smrt_workers and not running in this process).", ...staleJobIds);
|
|
529
|
+
for (const row of result.rows) if (typeof row.id === "string") recoveredJobIds.add(row.id);
|
|
530
|
+
}
|
|
531
|
+
for (const schedule of schedules) {
|
|
532
|
+
const state = stateBySchedule.get(schedule.id);
|
|
533
|
+
if (!state) continue;
|
|
534
|
+
const desiredRunningCount = state.live;
|
|
535
|
+
const recoveredCount = state.staleJobIds.filter((id) => recoveredJobIds.has(id)).length;
|
|
536
|
+
if (Number(schedule.running_count) === desiredRunningCount && recoveredCount === 0) continue;
|
|
537
|
+
if (recoveredCount > 0) {
|
|
538
|
+
await this.db.query(`UPDATE _smrt_agent_schedules
|
|
646
539
|
SET running_count = ?,
|
|
647
540
|
last_run = ?,
|
|
648
541
|
last_status = 'failed',
|
|
649
542
|
last_error = ?,
|
|
650
543
|
run_count = COALESCE(run_count, 0) + ?,
|
|
651
544
|
failure_count = COALESCE(failure_count, 0) + ?
|
|
652
|
-
WHERE id = ?`,
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
`Recovered ${recoveredCount} orphaned scheduled job(s) from dead worker(s)`,
|
|
656
|
-
recoveredCount,
|
|
657
|
-
recoveredCount,
|
|
658
|
-
schedule.id
|
|
659
|
-
);
|
|
660
|
-
this.emit(
|
|
661
|
-
"schedule:failed",
|
|
662
|
-
schedule.id,
|
|
663
|
-
`Recovered ${recoveredCount} orphaned scheduled job(s)`
|
|
664
|
-
);
|
|
665
|
-
} else {
|
|
666
|
-
await this.db.query(
|
|
667
|
-
`UPDATE _smrt_agent_schedules
|
|
545
|
+
WHERE id = ?`, desiredRunningCount, now, `Recovered ${recoveredCount} orphaned scheduled job(s) from dead worker(s)`, recoveredCount, recoveredCount, schedule.id);
|
|
546
|
+
this.emit("schedule:failed", schedule.id, `Recovered ${recoveredCount} orphaned scheduled job(s)`);
|
|
547
|
+
} else await this.db.query(`UPDATE _smrt_agent_schedules
|
|
668
548
|
SET running_count = ?
|
|
669
|
-
WHERE id = ?`,
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
args._agentConfig = agentConfig;
|
|
721
|
-
}
|
|
722
|
-
const job = await this.jobCollection.enqueueJob({
|
|
723
|
-
tenantId: typeof schedule.tenant_id === "string" && schedule.tenant_id.length > 0 ? schedule.tenant_id : null,
|
|
724
|
-
queue: "agents",
|
|
725
|
-
objectType: canonicalAgentType,
|
|
726
|
-
objectId: schedule.agent_id,
|
|
727
|
-
method: schedule.method || "run",
|
|
728
|
-
args,
|
|
729
|
-
priority: 75,
|
|
730
|
-
// High priority for scheduled agents
|
|
731
|
-
maxAttempts: 3,
|
|
732
|
-
timeout: schedule.timeout || 36e5
|
|
733
|
-
});
|
|
734
|
-
await this.db.query(
|
|
735
|
-
`UPDATE _smrt_agent_schedules
|
|
549
|
+
WHERE id = ?`, desiredRunningCount, schedule.id);
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
getScheduleIdFromJobArgs(args) {
|
|
553
|
+
if (!args) return null;
|
|
554
|
+
let parsedArgs = args;
|
|
555
|
+
if (typeof parsedArgs === "string") try {
|
|
556
|
+
parsedArgs = JSON.parse(parsedArgs);
|
|
557
|
+
} catch {
|
|
558
|
+
return null;
|
|
559
|
+
}
|
|
560
|
+
if (!parsedArgs || typeof parsedArgs !== "object" || Array.isArray(parsedArgs)) return null;
|
|
561
|
+
const scheduleId = parsedArgs._scheduleId;
|
|
562
|
+
return typeof scheduleId === "string" && scheduleId.length > 0 ? scheduleId : null;
|
|
563
|
+
}
|
|
564
|
+
/**
|
|
565
|
+
* Trigger a schedule by creating a job
|
|
566
|
+
*/
|
|
567
|
+
async triggerSchedule(schedule) {
|
|
568
|
+
if (!this.db || !this.jobCollection) return;
|
|
569
|
+
const rawAgentType = schedule.agent_type;
|
|
570
|
+
const canonicalAgentType = ObjectRegistry.getClass(rawAgentType)?.qualifiedName || rawAgentType;
|
|
571
|
+
const scheduleInfo = {
|
|
572
|
+
id: schedule.id,
|
|
573
|
+
agentType: canonicalAgentType,
|
|
574
|
+
agentId: schedule.agent_id,
|
|
575
|
+
cron: schedule.cron
|
|
576
|
+
};
|
|
577
|
+
try {
|
|
578
|
+
let methodArgs = {};
|
|
579
|
+
if (schedule.method_args) methodArgs = typeof schedule.method_args === "string" ? JSON.parse(schedule.method_args) : schedule.method_args;
|
|
580
|
+
let agentConfig = {};
|
|
581
|
+
if (schedule.agent_config) agentConfig = typeof schedule.agent_config === "string" ? JSON.parse(schedule.agent_config) : schedule.agent_config;
|
|
582
|
+
const nextRun = getNextCronDate(schedule.cron);
|
|
583
|
+
const args = {
|
|
584
|
+
...methodArgs,
|
|
585
|
+
_scheduleId: schedule.id
|
|
586
|
+
};
|
|
587
|
+
if (Object.keys(agentConfig).length > 0) args._agentConfig = agentConfig;
|
|
588
|
+
const job = await this.jobCollection.enqueueJob({
|
|
589
|
+
tenantId: typeof schedule.tenant_id === "string" && schedule.tenant_id.length > 0 ? schedule.tenant_id : null,
|
|
590
|
+
queue: "agents",
|
|
591
|
+
objectType: canonicalAgentType,
|
|
592
|
+
objectId: schedule.agent_id,
|
|
593
|
+
method: schedule.method || "run",
|
|
594
|
+
args,
|
|
595
|
+
priority: 75,
|
|
596
|
+
maxAttempts: 3,
|
|
597
|
+
timeout: schedule.timeout || 36e5
|
|
598
|
+
});
|
|
599
|
+
await this.db.query(`UPDATE _smrt_agent_schedules
|
|
736
600
|
SET agent_type = ?,
|
|
737
601
|
running_count = running_count + 1,
|
|
738
602
|
next_run = ?
|
|
739
|
-
WHERE id = ?`,
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
nextRun: nextRun.toISOString()
|
|
750
|
-
});
|
|
751
|
-
} catch (error) {
|
|
752
|
-
await this.db.query(
|
|
753
|
-
`UPDATE _smrt_agent_schedules
|
|
603
|
+
WHERE id = ?`, canonicalAgentType, nextRun.toISOString(), schedule.id);
|
|
604
|
+
this.emit("schedule:triggered", scheduleInfo);
|
|
605
|
+
this.logger.info("Schedule triggered", {
|
|
606
|
+
scheduleId: schedule.id,
|
|
607
|
+
agentType: canonicalAgentType,
|
|
608
|
+
jobId: job.id,
|
|
609
|
+
nextRun: nextRun.toISOString()
|
|
610
|
+
});
|
|
611
|
+
} catch (error) {
|
|
612
|
+
await this.db.query(`UPDATE _smrt_agent_schedules
|
|
754
613
|
SET last_error = ?
|
|
755
|
-
WHERE id = ?`,
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
614
|
+
WHERE id = ?`, redactErrorForPersistence(error), schedule.id);
|
|
615
|
+
this.emit("schedule:error", scheduleInfo, error);
|
|
616
|
+
this.logger.error("Schedule trigger failed", {
|
|
617
|
+
scheduleId: schedule.id,
|
|
618
|
+
error
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
};
|
|
623
|
+
var CRON_FIELD_RANGES = [
|
|
624
|
+
{
|
|
625
|
+
name: "minute",
|
|
626
|
+
min: 0,
|
|
627
|
+
max: 59
|
|
628
|
+
},
|
|
629
|
+
{
|
|
630
|
+
name: "hour",
|
|
631
|
+
min: 0,
|
|
632
|
+
max: 23
|
|
633
|
+
},
|
|
634
|
+
{
|
|
635
|
+
name: "day-of-month",
|
|
636
|
+
min: 1,
|
|
637
|
+
max: 31
|
|
638
|
+
},
|
|
639
|
+
{
|
|
640
|
+
name: "month",
|
|
641
|
+
min: 1,
|
|
642
|
+
max: 12
|
|
643
|
+
},
|
|
644
|
+
{
|
|
645
|
+
name: "day-of-week",
|
|
646
|
+
min: 0,
|
|
647
|
+
max: 7
|
|
648
|
+
}
|
|
775
649
|
];
|
|
776
650
|
function validateCronField(expr, range) {
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
}
|
|
809
|
-
const [startStr, endStr] = rangeParts;
|
|
810
|
-
if (startStr === "" || endStr === "") {
|
|
811
|
-
reject("has an empty range part");
|
|
812
|
-
}
|
|
813
|
-
const start = Number(startStr);
|
|
814
|
-
const end = Number(endStr);
|
|
815
|
-
assertInRange(start);
|
|
816
|
-
assertInRange(end);
|
|
817
|
-
if (start > end) reject("has an inverted range");
|
|
818
|
-
} else {
|
|
819
|
-
assertInRange(Number(body));
|
|
820
|
-
}
|
|
821
|
-
}
|
|
651
|
+
if (expr === "*") return;
|
|
652
|
+
const reject = (detail) => {
|
|
653
|
+
throw new Error(`Invalid cron expression: ${range.name} field "${expr}" ${detail} (valid range ${range.min}-${range.max})`);
|
|
654
|
+
};
|
|
655
|
+
const assertInRange = (value) => {
|
|
656
|
+
if (!Number.isInteger(value) || value < range.min || value > range.max) reject("is out of range");
|
|
657
|
+
};
|
|
658
|
+
for (const term of expr.split(",")) {
|
|
659
|
+
if (term === "") reject("contains an empty value");
|
|
660
|
+
let body = term;
|
|
661
|
+
if (body.includes("/")) {
|
|
662
|
+
const stepParts = body.split("/");
|
|
663
|
+
if (stepParts.length !== 2) reject("has malformed step syntax");
|
|
664
|
+
const [rangePart, stepStr] = stepParts;
|
|
665
|
+
const step = Number(stepStr);
|
|
666
|
+
if (!Number.isInteger(step) || step <= 0) reject("has an invalid step");
|
|
667
|
+
body = rangePart;
|
|
668
|
+
if (body === "*") continue;
|
|
669
|
+
}
|
|
670
|
+
if (body.includes("-")) {
|
|
671
|
+
const rangeParts = body.split("-");
|
|
672
|
+
if (rangeParts.length !== 2) reject("has malformed range syntax");
|
|
673
|
+
const [startStr, endStr] = rangeParts;
|
|
674
|
+
if (startStr === "" || endStr === "") reject("has an empty range part");
|
|
675
|
+
const start = Number(startStr);
|
|
676
|
+
const end = Number(endStr);
|
|
677
|
+
assertInRange(start);
|
|
678
|
+
assertInRange(end);
|
|
679
|
+
if (start > end) reject("has an inverted range");
|
|
680
|
+
} else assertInRange(Number(body));
|
|
681
|
+
}
|
|
822
682
|
}
|
|
823
683
|
function validateCronExpression(cron) {
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
parts.forEach((field, index) => {
|
|
831
|
-
validateCronField(field, CRON_FIELD_RANGES[index]);
|
|
832
|
-
});
|
|
833
|
-
return parts;
|
|
684
|
+
const parts = cron.trim().split(/\s+/);
|
|
685
|
+
if (parts.length !== 5) throw new Error(`Invalid cron expression: expected 5 fields, got ${parts.length}`);
|
|
686
|
+
parts.forEach((field, index) => {
|
|
687
|
+
validateCronField(field, CRON_FIELD_RANGES[index]);
|
|
688
|
+
});
|
|
689
|
+
return parts;
|
|
834
690
|
}
|
|
835
691
|
function getNextCronDate(cron) {
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
dayOfMonthOrWeekMatches = true;
|
|
858
|
-
}
|
|
859
|
-
if (matchesCronField(candidate.getMonth() + 1, monthExpr) && dayOfMonthOrWeekMatches && matchesCronField(candidate.getHours(), hourExpr) && matchesCronField(candidate.getMinutes(), minuteExpr)) {
|
|
860
|
-
return candidate;
|
|
861
|
-
}
|
|
862
|
-
candidate.setMinutes(candidate.getMinutes() + 1);
|
|
863
|
-
}
|
|
864
|
-
throw new Error(`Could not find next run date for cron: ${cron}`);
|
|
692
|
+
const [minuteExpr, hourExpr, dayExpr, monthExpr, dowExpr] = validateCronExpression(cron);
|
|
693
|
+
const candidate = /* @__PURE__ */ new Date(/* @__PURE__ */ new Date());
|
|
694
|
+
candidate.setSeconds(0);
|
|
695
|
+
candidate.setMilliseconds(0);
|
|
696
|
+
candidate.setMinutes(candidate.getMinutes() + 1);
|
|
697
|
+
const dayIsWildcard = dayExpr === "*";
|
|
698
|
+
const dowIsWildcard = dowExpr === "*";
|
|
699
|
+
const maxIterations = 525600;
|
|
700
|
+
for (let i = 0; i < maxIterations; i++) {
|
|
701
|
+
const dayMatches = matchesCronField(candidate.getDate(), dayExpr);
|
|
702
|
+
const dow = candidate.getDay();
|
|
703
|
+
const dowMatches = matchesCronField(dow, dowExpr) || dow === 0 && matchesCronField(7, dowExpr);
|
|
704
|
+
let dayOfMonthOrWeekMatches;
|
|
705
|
+
if (!dayIsWildcard && !dowIsWildcard) dayOfMonthOrWeekMatches = dayMatches || dowMatches;
|
|
706
|
+
else if (!dayIsWildcard) dayOfMonthOrWeekMatches = dayMatches;
|
|
707
|
+
else if (!dowIsWildcard) dayOfMonthOrWeekMatches = dowMatches;
|
|
708
|
+
else dayOfMonthOrWeekMatches = true;
|
|
709
|
+
if (matchesCronField(candidate.getMonth() + 1, monthExpr) && dayOfMonthOrWeekMatches && matchesCronField(candidate.getHours(), hourExpr) && matchesCronField(candidate.getMinutes(), minuteExpr)) return candidate;
|
|
710
|
+
candidate.setMinutes(candidate.getMinutes() + 1);
|
|
711
|
+
}
|
|
712
|
+
throw new Error(`Could not find next run date for cron: ${cron}`);
|
|
865
713
|
}
|
|
866
714
|
function matchesCronField(value, expr) {
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
}
|
|
886
|
-
if (expr.includes(",")) {
|
|
887
|
-
const values = expr.split(",").map((v) => parseInt(v.trim(), 10));
|
|
888
|
-
return values.includes(value);
|
|
889
|
-
}
|
|
890
|
-
return value === parseInt(expr, 10);
|
|
715
|
+
if (expr === "*") return true;
|
|
716
|
+
if (expr.includes("/")) {
|
|
717
|
+
const [range, stepStr] = expr.split("/");
|
|
718
|
+
const step = parseInt(stepStr, 10);
|
|
719
|
+
if (range === "*") return value % step === 0;
|
|
720
|
+
if (range.includes("-")) {
|
|
721
|
+
const [startStr, endStr] = range.split("-");
|
|
722
|
+
const start = parseInt(startStr, 10);
|
|
723
|
+
if (value < start || value > parseInt(endStr, 10)) return false;
|
|
724
|
+
return (value - start) % step === 0;
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
if (expr.includes("-")) {
|
|
728
|
+
const [startStr, endStr] = expr.split("-");
|
|
729
|
+
return value >= parseInt(startStr, 10) && value <= parseInt(endStr, 10);
|
|
730
|
+
}
|
|
731
|
+
if (expr.includes(",")) return expr.split(",").map((v) => parseInt(v.trim(), 10)).includes(value);
|
|
732
|
+
return value === parseInt(expr, 10);
|
|
891
733
|
}
|
|
892
734
|
function createScheduleRunner(config) {
|
|
893
|
-
|
|
735
|
+
return new ScheduleRunner(config);
|
|
894
736
|
}
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
JobHandle,
|
|
900
|
-
e as JobTimeoutError,
|
|
901
|
-
M as MAX_JOB_RETRIES,
|
|
902
|
-
ScheduleRunner,
|
|
903
|
-
f as SmrtJob,
|
|
904
|
-
SmrtJobCollection,
|
|
905
|
-
g as SmrtJobEvent,
|
|
906
|
-
h as SmrtJobEventCollection,
|
|
907
|
-
i as SmrtWorker,
|
|
908
|
-
SmrtWorkerCollection,
|
|
909
|
-
T as TaskRunner,
|
|
910
|
-
j as TenantJobCapExceededError,
|
|
911
|
-
k as assertWithinTenantCreationCap,
|
|
912
|
-
l as backgroundEligible,
|
|
913
|
-
clampRetries,
|
|
914
|
-
createScheduleRunner,
|
|
915
|
-
m as createTaskRunner,
|
|
916
|
-
c as createWorkerKey,
|
|
917
|
-
n as getBackgroundEligibleMethods,
|
|
918
|
-
getNextCronDate,
|
|
919
|
-
o as isBackgroundEligibleMethod,
|
|
920
|
-
isWorkerAlive,
|
|
921
|
-
p as markBackgroundEligible,
|
|
922
|
-
parseDelay,
|
|
923
|
-
priorityToNumber,
|
|
924
|
-
redactErrorForPersistence,
|
|
925
|
-
redactErrorMessage,
|
|
926
|
-
r as registerLiveWorker,
|
|
927
|
-
u as unregisterLiveWorker,
|
|
928
|
-
validateCronExpression,
|
|
929
|
-
withBackgroundJobs
|
|
930
|
-
};
|
|
931
|
-
//# sourceMappingURL=index.js.map
|
|
737
|
+
//#endregion
|
|
738
|
+
export { DEFAULT_TENANT_JOB_CAP, JobBuilder, JobContextLogger, JobHandle, JobTimeoutError, MAX_JOB_RETRIES, ScheduleRunner, SmrtJob, SmrtJobCollection, SmrtJobEvent, SmrtJobEventCollection, SmrtWorker, SmrtWorkerCollection, TaskRunner, TenantJobCapExceededError, assertWithinTenantCreationCap, backgroundEligible, clampRetries, createScheduleRunner, createTaskRunner, createWorkerKey, getBackgroundEligibleMethods, getNextCronDate, isBackgroundEligibleMethod, isWorkerAlive, markBackgroundEligible, parseDelay, priorityToNumber, redactErrorForPersistence, redactErrorMessage, registerLiveWorker, unregisterLiveWorker, validateCronExpression, withBackgroundJobs };
|
|
739
|
+
|
|
740
|
+
//# sourceMappingURL=index.js.map
|