@happyvertical/smrt-jobs 0.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/AGENTS.md +71 -0
  2. package/CLAUDE.md +1 -0
  3. package/LICENSE +7 -0
  4. package/README.md +151 -0
  5. package/dist/__smrt-register__.d.ts +2 -0
  6. package/dist/__smrt-register__.d.ts.map +1 -0
  7. package/dist/background-policy.d.ts +121 -0
  8. package/dist/background-policy.d.ts.map +1 -0
  9. package/dist/chunks/runner-DV8FBO0y.js +1642 -0
  10. package/dist/chunks/runner-DV8FBO0y.js.map +1 -0
  11. package/dist/chunks/worker-liveness-DOTjoIjr.js +65 -0
  12. package/dist/chunks/worker-liveness-DOTjoIjr.js.map +1 -0
  13. package/dist/error-redaction.d.ts +48 -0
  14. package/dist/error-redaction.d.ts.map +1 -0
  15. package/dist/index.d.ts +13 -0
  16. package/dist/index.d.ts.map +1 -0
  17. package/dist/index.js +926 -0
  18. package/dist/index.js.map +1 -0
  19. package/dist/job-builder.d.ts +94 -0
  20. package/dist/job-builder.d.ts.map +1 -0
  21. package/dist/job-handle.d.ts +71 -0
  22. package/dist/job-handle.d.ts.map +1 -0
  23. package/dist/logger-extension.d.ts +58 -0
  24. package/dist/logger-extension.d.ts.map +1 -0
  25. package/dist/manifest.json +1327 -0
  26. package/dist/object-extension.d.ts +68 -0
  27. package/dist/object-extension.d.ts.map +1 -0
  28. package/dist/playground.d.ts +2 -0
  29. package/dist/playground.d.ts.map +1 -0
  30. package/dist/playground.js +179 -0
  31. package/dist/playground.js.map +1 -0
  32. package/dist/runner.d.ts +189 -0
  33. package/dist/runner.d.ts.map +1 -0
  34. package/dist/runner.js +15 -0
  35. package/dist/runner.js.map +1 -0
  36. package/dist/schedule-runner.d.ts +151 -0
  37. package/dist/schedule-runner.d.ts.map +1 -0
  38. package/dist/smrt-job-event.d.ts +54 -0
  39. package/dist/smrt-job-event.d.ts.map +1 -0
  40. package/dist/smrt-job.d.ts +215 -0
  41. package/dist/smrt-job.d.ts.map +1 -0
  42. package/dist/smrt-knowledge.json +508 -0
  43. package/dist/smrt-worker.d.ts +72 -0
  44. package/dist/smrt-worker.d.ts.map +1 -0
  45. package/dist/stale-recovery.d.ts +34 -0
  46. package/dist/stale-recovery.d.ts.map +1 -0
  47. package/dist/svelte/components/JobActions.svelte +103 -0
  48. package/dist/svelte/components/JobActions.svelte.d.ts +23 -0
  49. package/dist/svelte/components/JobActions.svelte.d.ts.map +1 -0
  50. package/dist/svelte/components/JobDashboard.svelte +199 -0
  51. package/dist/svelte/components/JobDashboard.svelte.d.ts +27 -0
  52. package/dist/svelte/components/JobDashboard.svelte.d.ts.map +1 -0
  53. package/dist/svelte/components/JobDetail.svelte +256 -0
  54. package/dist/svelte/components/JobDetail.svelte.d.ts +17 -0
  55. package/dist/svelte/components/JobDetail.svelte.d.ts.map +1 -0
  56. package/dist/svelte/components/JobList.svelte +360 -0
  57. package/dist/svelte/components/JobList.svelte.d.ts +28 -0
  58. package/dist/svelte/components/JobList.svelte.d.ts.map +1 -0
  59. package/dist/svelte/components/JobStats.svelte +242 -0
  60. package/dist/svelte/components/JobStats.svelte.d.ts +15 -0
  61. package/dist/svelte/components/JobStats.svelte.d.ts.map +1 -0
  62. package/dist/svelte/components/JobStatusBadge.svelte +23 -0
  63. package/dist/svelte/components/JobStatusBadge.svelte.d.ts +9 -0
  64. package/dist/svelte/components/JobStatusBadge.svelte.d.ts.map +1 -0
  65. package/dist/svelte/components/types.d.ts +9 -0
  66. package/dist/svelte/components/types.d.ts.map +1 -0
  67. package/dist/svelte/components/types.js +8 -0
  68. package/dist/svelte/i18n.d.ts +22 -0
  69. package/dist/svelte/i18n.d.ts.map +1 -0
  70. package/dist/svelte/i18n.js +22 -0
  71. package/dist/svelte/index.d.ts +25 -0
  72. package/dist/svelte/index.d.ts.map +1 -0
  73. package/dist/svelte/index.js +28 -0
  74. package/dist/svelte/playground.d.ts +329 -0
  75. package/dist/svelte/playground.d.ts.map +1 -0
  76. package/dist/svelte/playground.js +174 -0
  77. package/dist/svelte/types.d.ts +191 -0
  78. package/dist/svelte/types.d.ts.map +1 -0
  79. package/dist/svelte/types.js +87 -0
  80. package/dist/ui.d.ts +10 -0
  81. package/dist/ui.d.ts.map +1 -0
  82. package/dist/ui.js +69 -0
  83. package/dist/ui.js.map +1 -0
  84. package/dist/worker-liveness-thread.d.ts +2 -0
  85. package/dist/worker-liveness-thread.d.ts.map +1 -0
  86. package/dist/worker-liveness-thread.js +66 -0
  87. package/dist/worker-liveness-thread.js.map +1 -0
  88. package/dist/worker-liveness-ticker.d.ts +30 -0
  89. package/dist/worker-liveness-ticker.d.ts.map +1 -0
  90. package/dist/worker-liveness.d.ts +71 -0
  91. package/dist/worker-liveness.d.ts.map +1 -0
  92. package/package.json +93 -0
package/dist/index.js ADDED
@@ -0,0 +1,926 @@
1
+ import { D as DEFAULT_TENANT_JOB_CAP, c as clampRetries, S as SmrtJobCollection, a as DEFAULT_TASK_HEARTBEAT_INTERVAL_MS, b as SmrtWorkerCollection, r as redactErrorMessage, d as redactErrorForPersistence } from "./chunks/runner-DV8FBO0y.js";
2
+ import { J, M, e, f, g, h, T, i, j, k, l, m, n, o } from "./chunks/runner-DV8FBO0y.js";
3
+ import { exponential } from "@happyvertical/jobs";
4
+ import { ObjectRegistry } from "@happyvertical/smrt-core";
5
+ import { EventEmitter } from "node:events";
6
+ import { createLogger } from "@happyvertical/logger";
7
+ import { createId } from "@happyvertical/utils";
8
+ import { i as isWorkerAlive } from "./chunks/worker-liveness-DOTjoIjr.js";
9
+ import { c, r, u } from "./chunks/worker-liveness-DOTjoIjr.js";
10
+ class JobHandle {
11
+ constructor(id, collection) {
12
+ this.id = id;
13
+ this.collection = collection;
14
+ }
15
+ id;
16
+ collection;
17
+ /**
18
+ * Get the current job status
19
+ */
20
+ async status() {
21
+ const job = await this.getJob();
22
+ return job.status;
23
+ }
24
+ /**
25
+ * Get the full job object
26
+ */
27
+ async getJob() {
28
+ const job = await this.collection.get({ id: this.id });
29
+ if (!job) {
30
+ throw new Error(`Job not found: ${this.id}`);
31
+ }
32
+ return job;
33
+ }
34
+ /**
35
+ * Wait for the job to complete
36
+ *
37
+ * @param options - Wait configuration
38
+ * @returns The job result
39
+ * @throws Error if the job fails or times out
40
+ */
41
+ async wait(options = {}) {
42
+ const { timeout = 6e4, pollInterval = 100 } = options;
43
+ const startTime = Date.now();
44
+ while (true) {
45
+ const job = await this.getJob();
46
+ if (job.status === "completed") {
47
+ return {
48
+ success: true,
49
+ resultPointer: job.resultPointer
50
+ };
51
+ }
52
+ if (job.status === "failed") {
53
+ return {
54
+ success: false,
55
+ error: job.lastError ?? "Job failed"
56
+ };
57
+ }
58
+ if (job.status === "cancelled") {
59
+ return {
60
+ success: false,
61
+ error: "Job was cancelled"
62
+ };
63
+ }
64
+ if (Date.now() - startTime >= timeout) {
65
+ throw new Error(`Timeout waiting for job ${this.id}`);
66
+ }
67
+ await new Promise((resolve) => setTimeout(resolve, pollInterval));
68
+ }
69
+ }
70
+ /**
71
+ * Cancel the job
72
+ */
73
+ async cancel() {
74
+ const job = await this.getJob();
75
+ await job.cancel();
76
+ }
77
+ /**
78
+ * Retry a failed job
79
+ */
80
+ async retry() {
81
+ const job = await this.getJob();
82
+ await job.retry();
83
+ }
84
+ /**
85
+ * Check if the job is still running
86
+ */
87
+ async isRunning() {
88
+ const status = await this.status();
89
+ return status === "pending" || status === "running";
90
+ }
91
+ /**
92
+ * Check if the job has completed (successfully or not)
93
+ */
94
+ async isDone() {
95
+ const status = await this.status();
96
+ return status === "completed" || status === "failed" || status === "cancelled";
97
+ }
98
+ }
99
+ function priorityToNumber(priority) {
100
+ if (typeof priority === "number") return priority;
101
+ switch (priority) {
102
+ case "critical":
103
+ return 100;
104
+ case "high":
105
+ return 75;
106
+ case "normal":
107
+ return 50;
108
+ case "low":
109
+ return 25;
110
+ default:
111
+ return 50;
112
+ }
113
+ }
114
+ function parseDelay(delay) {
115
+ if (typeof delay === "number") return delay;
116
+ const match = delay.match(/^(\d+)(ms|s|m|h|d)?$/);
117
+ if (!match) {
118
+ throw new Error(`Invalid delay format: ${delay}`);
119
+ }
120
+ const value = parseInt(match[1], 10);
121
+ const unit = match[2] || "ms";
122
+ switch (unit) {
123
+ case "ms":
124
+ return value;
125
+ case "s":
126
+ return value * 1e3;
127
+ case "m":
128
+ return value * 60 * 1e3;
129
+ case "h":
130
+ return value * 60 * 60 * 1e3;
131
+ case "d":
132
+ return value * 24 * 60 * 60 * 1e3;
133
+ default:
134
+ return value;
135
+ }
136
+ }
137
+ class JobBuilder {
138
+ constructor(objectType, objectId, method, args, collection) {
139
+ this.objectType = objectType;
140
+ this.objectId = objectId;
141
+ this.method = method;
142
+ this.args = args;
143
+ this.collection = collection;
144
+ }
145
+ objectType;
146
+ objectId;
147
+ method;
148
+ args;
149
+ collection;
150
+ _queue = "default";
151
+ _delay = 0;
152
+ _retries = 3;
153
+ _priority = 50;
154
+ _timeout = 3e5;
155
+ _timeoutBehavior = "fail";
156
+ _retryStrategy = exponential();
157
+ _tenantJobCap = DEFAULT_TENANT_JOB_CAP;
158
+ /**
159
+ * Set the queue name
160
+ */
161
+ queue(name) {
162
+ this._queue = name;
163
+ return this;
164
+ }
165
+ /**
166
+ * Set a delay before the job runs
167
+ * @param delay - Delay as milliseconds or string like '5m', '1h', '30s'
168
+ */
169
+ delay(delay) {
170
+ this._delay = parseDelay(delay);
171
+ return this;
172
+ }
173
+ /**
174
+ * Set when the job should run
175
+ */
176
+ runAt(date) {
177
+ this._delay = date.getTime() - Date.now();
178
+ return this;
179
+ }
180
+ /**
181
+ * Set the maximum number of retry attempts.
182
+ *
183
+ * Clamped to {@link MAX_JOB_RETRIES} so a misconfigured caller cannot pin a
184
+ * worker on a poison job indefinitely (S5 audit #1402).
185
+ */
186
+ retries(count) {
187
+ this._retries = clampRetries(count);
188
+ return this;
189
+ }
190
+ /**
191
+ * Set the retry strategy
192
+ */
193
+ retryStrategy(strategy) {
194
+ this._retryStrategy = strategy;
195
+ return this;
196
+ }
197
+ /**
198
+ * Set the job priority
199
+ */
200
+ priority(level) {
201
+ this._priority = priorityToNumber(level);
202
+ return this;
203
+ }
204
+ /**
205
+ * Set the job timeout in milliseconds
206
+ */
207
+ timeout(ms) {
208
+ this._timeout = ms;
209
+ return this;
210
+ }
211
+ /**
212
+ * Set what happens when the job times out
213
+ */
214
+ timeoutBehavior(behavior) {
215
+ this._timeoutBehavior = behavior;
216
+ return this;
217
+ }
218
+ /**
219
+ * Override the per-tenant in-flight job cap for this enqueue.
220
+ *
221
+ * Defaults to {@link DEFAULT_TENANT_JOB_CAP}. Pass `0` (or a negative value)
222
+ * to disable the cap for trusted internal callers (S5 audit #1402).
223
+ */
224
+ tenantJobCap(max) {
225
+ this._tenantJobCap = max;
226
+ return this;
227
+ }
228
+ /**
229
+ * Enqueue the job and return a handle
230
+ */
231
+ async enqueue() {
232
+ const runAt = new Date(Date.now() + this._delay);
233
+ const retryConfig = "toConfig" in this._retryStrategy ? this._retryStrategy.toConfig() : this._retryStrategy;
234
+ const job = await this.collection.enqueueJob(
235
+ {
236
+ queue: this._queue,
237
+ objectType: this.objectType,
238
+ objectId: this.objectId,
239
+ method: this.method,
240
+ args: this.args,
241
+ runAt,
242
+ priority: this._priority,
243
+ maxAttempts: this._retries,
244
+ timeout: this._timeout,
245
+ timeoutBehavior: this._timeoutBehavior,
246
+ retryStrategy: retryConfig
247
+ },
248
+ { tenantJobCap: this._tenantJobCap }
249
+ );
250
+ const jobId = job.id;
251
+ if (!jobId) {
252
+ throw new Error("Job was created but has no ID");
253
+ }
254
+ return new JobHandle(jobId, this.collection);
255
+ }
256
+ }
257
+ const collectionCache = /* @__PURE__ */ new WeakMap();
258
+ async function getJobCollection(db) {
259
+ let collection = collectionCache.get(db);
260
+ if (!collection) {
261
+ collection = await SmrtJobCollection.create({
262
+ db: { type: "sqlite", url: ":memory:" }
263
+ // Placeholder
264
+ });
265
+ collection._db = db;
266
+ collectionCache.set(db, collection);
267
+ }
268
+ return collection;
269
+ }
270
+ function getObjectTypeName(instance) {
271
+ const metaType = instance._meta_type;
272
+ if (typeof metaType === "string" && metaType.length > 0) {
273
+ return metaType;
274
+ }
275
+ const className = instance.constructor.name;
276
+ return ObjectRegistry.getClass(className)?.qualifiedName || className;
277
+ }
278
+ function requireObjectDb(instance) {
279
+ const db = instance._db;
280
+ if (!db) {
281
+ throw new Error("Object not initialized. Call initialize() first.");
282
+ }
283
+ return db;
284
+ }
285
+ async function bgImpl(method, args = {}, options = {}) {
286
+ const db = requireObjectDb(this);
287
+ const collection = await getJobCollection(db);
288
+ const builder = new JobBuilder(
289
+ getObjectTypeName(this),
290
+ this.id ?? null,
291
+ method,
292
+ args,
293
+ collection
294
+ );
295
+ if (options.queue) builder.queue(options.queue);
296
+ if (options.priority) builder.priority(options.priority);
297
+ if (options.delay) builder.delay(options.delay);
298
+ if (options.retries !== void 0) builder.retries(options.retries);
299
+ if (options.timeout) builder.timeout(options.timeout);
300
+ return builder.enqueue();
301
+ }
302
+ function backgroundImpl(method, args = {}) {
303
+ const db = requireObjectDb(this);
304
+ const objectType = getObjectTypeName(this);
305
+ const objectId = this.id ?? null;
306
+ const lazyBuilder = {
307
+ _queue: "default",
308
+ _delay: 0,
309
+ _retries: 3,
310
+ _priority: 50,
311
+ _timeout: 3e5,
312
+ _timeoutBehavior: "fail",
313
+ _retryStrategy: null,
314
+ // `undefined` => fall through to JobBuilder's DEFAULT_TENANT_JOB_CAP. A
315
+ // caller can override (incl. `0` to disable) via tenantJobCap() below.
316
+ _tenantJobCap: void 0,
317
+ queue(name) {
318
+ this._queue = name;
319
+ return this;
320
+ },
321
+ delay(d) {
322
+ this._delay = parseDelay(d);
323
+ return this;
324
+ },
325
+ runAt(date) {
326
+ this._delay = date.getTime() - Date.now();
327
+ return this;
328
+ },
329
+ retries(count) {
330
+ this._retries = count;
331
+ return this;
332
+ },
333
+ retryStrategy(strategy) {
334
+ this._retryStrategy = strategy;
335
+ return this;
336
+ },
337
+ priority(level) {
338
+ this._priority = priorityToNumber(level);
339
+ return this;
340
+ },
341
+ timeout(ms) {
342
+ this._timeout = ms;
343
+ return this;
344
+ },
345
+ timeoutBehavior(behavior) {
346
+ this._timeoutBehavior = behavior;
347
+ return this;
348
+ },
349
+ tenantJobCap(max) {
350
+ this._tenantJobCap = max;
351
+ return this;
352
+ },
353
+ async enqueue() {
354
+ const collection = await getJobCollection(db);
355
+ const builder = new JobBuilder(
356
+ objectType,
357
+ objectId,
358
+ method,
359
+ args,
360
+ collection
361
+ );
362
+ builder.queue(this._queue);
363
+ builder.delay(this._delay);
364
+ builder.retries(this._retries);
365
+ builder.priority(this._priority);
366
+ builder.timeout(this._timeout);
367
+ builder.timeoutBehavior(this._timeoutBehavior);
368
+ if (this._tenantJobCap !== void 0) {
369
+ builder.tenantJobCap(this._tenantJobCap);
370
+ }
371
+ if (this._retryStrategy) {
372
+ builder.retryStrategy(
373
+ this._retryStrategy
374
+ );
375
+ }
376
+ return builder.enqueue();
377
+ }
378
+ };
379
+ return lazyBuilder;
380
+ }
381
+ function withBackgroundJobs(BaseClass) {
382
+ const prototype = BaseClass.prototype;
383
+ if (typeof prototype.bg !== "function") {
384
+ Object.defineProperty(prototype, "bg", {
385
+ value: bgImpl,
386
+ writable: true,
387
+ configurable: true
388
+ });
389
+ }
390
+ if (typeof prototype.background !== "function") {
391
+ Object.defineProperty(prototype, "background", {
392
+ value: backgroundImpl,
393
+ writable: true,
394
+ configurable: true
395
+ });
396
+ }
397
+ return BaseClass;
398
+ }
399
+ const DEFAULT_CONFIG = {
400
+ id: "",
401
+ pollInterval: 6e4,
402
+ // 1 minute
403
+ batchSize: 50,
404
+ staleJobThresholdMs: 9e4,
405
+ taskHeartbeatInterval: DEFAULT_TASK_HEARTBEAT_INTERVAL_MS
406
+ };
407
+ class ScheduleRunner extends EventEmitter {
408
+ id;
409
+ config;
410
+ jobCollection = null;
411
+ workerCollection = null;
412
+ running = false;
413
+ pollTimer = null;
414
+ db = null;
415
+ logger = createLogger(true);
416
+ constructor(config = {}) {
417
+ super();
418
+ this.config = {
419
+ ...DEFAULT_CONFIG,
420
+ ...config,
421
+ id: config.id || `schedule_${createId().slice(0, 8)}`
422
+ };
423
+ this.id = this.config.id;
424
+ }
425
+ /**
426
+ * Initialize the runner with database connection
427
+ */
428
+ async initialize(db) {
429
+ this.db = db;
430
+ this.jobCollection = await SmrtJobCollection.create({ db });
431
+ this.workerCollection = await SmrtWorkerCollection.create({ db });
432
+ }
433
+ /**
434
+ * Start processing schedules
435
+ */
436
+ async start() {
437
+ if (this.running) return;
438
+ if (!this.db) {
439
+ throw new Error(
440
+ "ScheduleRunner not initialized. Call initialize() first."
441
+ );
442
+ }
443
+ this.running = true;
444
+ this.startPolling();
445
+ this.emit("runner:started");
446
+ this.logger.info("ScheduleRunner started", { id: this.id });
447
+ }
448
+ /**
449
+ * Stop processing schedules
450
+ */
451
+ async stop() {
452
+ if (!this.running) return;
453
+ this.running = false;
454
+ if (this.pollTimer) {
455
+ clearTimeout(this.pollTimer);
456
+ this.pollTimer = null;
457
+ }
458
+ this.emit("runner:stopped");
459
+ this.logger.info("ScheduleRunner stopped", { id: this.id });
460
+ }
461
+ /**
462
+ * Check if runner is running
463
+ */
464
+ isRunning() {
465
+ return this.running;
466
+ }
467
+ /**
468
+ * Handle job completion for a scheduled job.
469
+ *
470
+ * Call this from TaskRunner's job:completed / job:failed events
471
+ * when the job has a `_scheduleId` in its args.
472
+ */
473
+ async handleJobCompletion(scheduleId, success, errorMessage) {
474
+ if (!this.db) return;
475
+ const safeErrorMessage = redactErrorMessage(
476
+ errorMessage ?? "Unknown error"
477
+ );
478
+ try {
479
+ if (success) {
480
+ await this.db.query(
481
+ `UPDATE _smrt_agent_schedules
482
+ SET running_count = CASE WHEN COALESCE(running_count, 0) > 0 THEN running_count - 1 ELSE 0 END,
483
+ last_run = ?,
484
+ last_status = 'success',
485
+ last_error = NULL,
486
+ run_count = COALESCE(run_count, 0) + 1,
487
+ success_count = COALESCE(success_count, 0) + 1
488
+ WHERE id = ?`,
489
+ (/* @__PURE__ */ new Date()).toISOString(),
490
+ scheduleId
491
+ );
492
+ this.emit("schedule:completed", scheduleId);
493
+ } else {
494
+ await this.db.query(
495
+ `UPDATE _smrt_agent_schedules
496
+ SET running_count = CASE WHEN COALESCE(running_count, 0) > 0 THEN running_count - 1 ELSE 0 END,
497
+ last_run = ?,
498
+ last_status = 'failed',
499
+ last_error = ?,
500
+ run_count = COALESCE(run_count, 0) + 1,
501
+ failure_count = COALESCE(failure_count, 0) + 1
502
+ WHERE id = ?`,
503
+ (/* @__PURE__ */ new Date()).toISOString(),
504
+ safeErrorMessage,
505
+ scheduleId
506
+ );
507
+ this.emit("schedule:failed", scheduleId, safeErrorMessage);
508
+ }
509
+ } catch (err) {
510
+ this.logger.error("Failed to update schedule after job completion", {
511
+ scheduleId,
512
+ error: err
513
+ });
514
+ }
515
+ }
516
+ /**
517
+ * Start the polling loop
518
+ */
519
+ startPolling() {
520
+ const poll = async () => {
521
+ if (!this.running) return;
522
+ try {
523
+ await this.poll();
524
+ } catch (error) {
525
+ this.emit("runner:error", error);
526
+ this.logger.error("ScheduleRunner poll error", { error });
527
+ }
528
+ if (this.running) {
529
+ this.pollTimer = setTimeout(poll, this.config.pollInterval);
530
+ }
531
+ };
532
+ poll();
533
+ }
534
+ /**
535
+ * Poll for due schedules and create jobs
536
+ */
537
+ async poll() {
538
+ if (!this.db || !this.jobCollection) return;
539
+ await this.recoverStaleScheduleState();
540
+ const now = (/* @__PURE__ */ new Date()).toISOString();
541
+ const result = await this.db.query(
542
+ `SELECT * FROM _smrt_agent_schedules
543
+ WHERE enabled = true
544
+ AND status = 'active'
545
+ AND next_run <= ?
546
+ AND COALESCE(running_count, 0) < COALESCE(max_concurrent, 1)
547
+ ORDER BY next_run ASC
548
+ LIMIT ?`,
549
+ now,
550
+ this.config.batchSize
551
+ );
552
+ for (const row of result.rows) {
553
+ await this.triggerSchedule(row);
554
+ }
555
+ }
556
+ /**
557
+ * Reconcile stuck schedule slots against running jobs.
558
+ *
559
+ * This handles two failure modes:
560
+ * - a running job's owning worker is no longer alive (dead/restarted)
561
+ * - a schedule slot remains occupied even though no running job still exists
562
+ *
563
+ * Staleness keys on worker *liveness* (issue #1474), not per-job heartbeat
564
+ * freshness: a job whose `worker_id` is live in this process or holds a fresh
565
+ * lease in `_smrt_workers` is healthy even if its handler is holding the loop
566
+ * synchronously. ScheduleRunner has no in-process active-job set, so this is
567
+ * its entire correctness mechanism.
568
+ */
569
+ async recoverStaleScheduleState() {
570
+ if (!this.db || !this.workerCollection) return;
571
+ const schedulesResult = await this.db.query(
572
+ `SELECT id, running_count
573
+ FROM _smrt_agent_schedules
574
+ WHERE COALESCE(running_count, 0) > 0`
575
+ );
576
+ const schedules = schedulesResult.rows;
577
+ if (schedules.length === 0) return;
578
+ const workersReady = await this.workerCollection.tableReady();
579
+ const freshLeaseKeys = workersReady ? await this.workerCollection.freshLeaseWorkerKeys() : /* @__PURE__ */ new Set();
580
+ const jobsResult = await this.db.query(
581
+ `SELECT id, args, worker_id
582
+ FROM _smrt_jobs
583
+ WHERE status = 'running'`
584
+ );
585
+ const jobRows = jobsResult.rows;
586
+ const stateBySchedule = /* @__PURE__ */ new Map();
587
+ for (const schedule of schedules) {
588
+ stateBySchedule.set(schedule.id, { live: 0, staleJobIds: [] });
589
+ }
590
+ for (const row of jobRows) {
591
+ const scheduleId = this.getScheduleIdFromJobArgs(row.args);
592
+ if (!scheduleId) continue;
593
+ const state = stateBySchedule.get(scheduleId);
594
+ if (!state) continue;
595
+ const alive = workersReady ? isWorkerAlive(row.worker_id, freshLeaseKeys) : true;
596
+ if (!alive) {
597
+ state.staleJobIds.push(row.id);
598
+ } else {
599
+ state.live += 1;
600
+ }
601
+ }
602
+ const now = (/* @__PURE__ */ new Date()).toISOString();
603
+ const staleJobIds = schedules.flatMap((schedule) => {
604
+ const state = stateBySchedule.get(schedule.id);
605
+ return state?.staleJobIds ?? [];
606
+ });
607
+ const recoveredJobIds = /* @__PURE__ */ new Set();
608
+ if (staleJobIds.length > 0) {
609
+ const placeholders = staleJobIds.map(() => "?").join(", ");
610
+ const result = await this.db.query(
611
+ `UPDATE _smrt_jobs
612
+ SET status = 'failed',
613
+ completed_at = ?,
614
+ last_error = ?,
615
+ worker_id = NULL,
616
+ worker_heartbeat = NULL
617
+ WHERE status = 'running'
618
+ AND id IN (${placeholders})
619
+ RETURNING id`,
620
+ now,
621
+ "Recovered orphaned scheduled job: its owning worker is no longer alive (no fresh liveness lease in _smrt_workers and not running in this process).",
622
+ ...staleJobIds
623
+ );
624
+ for (const row of result.rows) {
625
+ if (typeof row.id === "string") recoveredJobIds.add(row.id);
626
+ }
627
+ }
628
+ for (const schedule of schedules) {
629
+ const state = stateBySchedule.get(schedule.id);
630
+ if (!state) continue;
631
+ const desiredRunningCount = state.live;
632
+ const recoveredCount = state.staleJobIds.filter(
633
+ (id) => recoveredJobIds.has(id)
634
+ ).length;
635
+ if (Number(schedule.running_count) === desiredRunningCount && recoveredCount === 0) {
636
+ continue;
637
+ }
638
+ if (recoveredCount > 0) {
639
+ await this.db.query(
640
+ `UPDATE _smrt_agent_schedules
641
+ SET running_count = ?,
642
+ last_run = ?,
643
+ last_status = 'failed',
644
+ last_error = ?,
645
+ run_count = COALESCE(run_count, 0) + ?,
646
+ failure_count = COALESCE(failure_count, 0) + ?
647
+ WHERE id = ?`,
648
+ desiredRunningCount,
649
+ now,
650
+ `Recovered ${recoveredCount} orphaned scheduled job(s) from dead worker(s)`,
651
+ recoveredCount,
652
+ recoveredCount,
653
+ schedule.id
654
+ );
655
+ this.emit(
656
+ "schedule:failed",
657
+ schedule.id,
658
+ `Recovered ${recoveredCount} orphaned scheduled job(s)`
659
+ );
660
+ } else {
661
+ await this.db.query(
662
+ `UPDATE _smrt_agent_schedules
663
+ SET running_count = ?
664
+ WHERE id = ?`,
665
+ desiredRunningCount,
666
+ schedule.id
667
+ );
668
+ }
669
+ }
670
+ }
671
+ getScheduleIdFromJobArgs(args) {
672
+ if (!args) return null;
673
+ let parsedArgs = args;
674
+ if (typeof parsedArgs === "string") {
675
+ try {
676
+ parsedArgs = JSON.parse(parsedArgs);
677
+ } catch {
678
+ return null;
679
+ }
680
+ }
681
+ if (!parsedArgs || typeof parsedArgs !== "object" || Array.isArray(parsedArgs)) {
682
+ return null;
683
+ }
684
+ const scheduleId = parsedArgs._scheduleId;
685
+ return typeof scheduleId === "string" && scheduleId.length > 0 ? scheduleId : null;
686
+ }
687
+ /**
688
+ * Trigger a schedule by creating a job
689
+ */
690
+ async triggerSchedule(schedule) {
691
+ if (!this.db || !this.jobCollection) return;
692
+ const rawAgentType = schedule.agent_type;
693
+ const canonicalAgentType = ObjectRegistry.getClass(rawAgentType)?.qualifiedName || rawAgentType;
694
+ const scheduleInfo = {
695
+ id: schedule.id,
696
+ agentType: canonicalAgentType,
697
+ agentId: schedule.agent_id,
698
+ cron: schedule.cron
699
+ };
700
+ try {
701
+ let methodArgs = {};
702
+ if (schedule.method_args) {
703
+ methodArgs = typeof schedule.method_args === "string" ? JSON.parse(schedule.method_args) : schedule.method_args;
704
+ }
705
+ let agentConfig = {};
706
+ if (schedule.agent_config) {
707
+ agentConfig = typeof schedule.agent_config === "string" ? JSON.parse(schedule.agent_config) : schedule.agent_config;
708
+ }
709
+ const nextRun = getNextCronDate(schedule.cron);
710
+ await this.db.query(
711
+ `UPDATE _smrt_agent_schedules
712
+ SET agent_type = ?,
713
+ running_count = running_count + 1,
714
+ next_run = ?
715
+ WHERE id = ?`,
716
+ canonicalAgentType,
717
+ nextRun.toISOString(),
718
+ schedule.id
719
+ );
720
+ const args = {
721
+ ...methodArgs,
722
+ _scheduleId: schedule.id
723
+ };
724
+ if (Object.keys(agentConfig).length > 0) {
725
+ args._agentConfig = agentConfig;
726
+ }
727
+ const job = await this.jobCollection.enqueueJob({
728
+ tenantId: typeof schedule.tenant_id === "string" && schedule.tenant_id.length > 0 ? schedule.tenant_id : null,
729
+ queue: "agents",
730
+ objectType: canonicalAgentType,
731
+ objectId: schedule.agent_id,
732
+ method: schedule.method || "run",
733
+ args,
734
+ priority: 75,
735
+ // High priority for scheduled agents
736
+ maxAttempts: 3,
737
+ timeout: schedule.timeout || 36e5
738
+ });
739
+ this.emit("schedule:triggered", scheduleInfo);
740
+ this.logger.info("Schedule triggered", {
741
+ scheduleId: schedule.id,
742
+ agentType: canonicalAgentType,
743
+ jobId: job.id,
744
+ nextRun: nextRun.toISOString()
745
+ });
746
+ } catch (error) {
747
+ await this.db.query(
748
+ `UPDATE _smrt_agent_schedules
749
+ SET running_count = running_count - 1,
750
+ status = 'error',
751
+ last_error = ?
752
+ WHERE id = ?`,
753
+ // Tolerate non-Error throwables: a thrown string/object has no
754
+ // `.message`, which would otherwise persist an empty `last_error`.
755
+ redactErrorForPersistence(error),
756
+ schedule.id
757
+ );
758
+ this.emit("schedule:error", scheduleInfo, error);
759
+ this.logger.error("Schedule trigger failed", {
760
+ scheduleId: schedule.id,
761
+ error
762
+ });
763
+ }
764
+ }
765
+ }
766
+ const CRON_FIELD_RANGES = [
767
+ { name: "minute", min: 0, max: 59 },
768
+ { name: "hour", min: 0, max: 23 },
769
+ { name: "day-of-month", min: 1, max: 31 },
770
+ { name: "month", min: 1, max: 12 },
771
+ { name: "day-of-week", min: 0, max: 7 }
772
+ ];
773
+ function validateCronField(expr, range) {
774
+ if (expr === "*") return;
775
+ const reject = (detail) => {
776
+ throw new Error(
777
+ `Invalid cron expression: ${range.name} field "${expr}" ${detail} (valid range ${range.min}-${range.max})`
778
+ );
779
+ };
780
+ const assertInRange = (value) => {
781
+ if (!Number.isInteger(value) || value < range.min || value > range.max) {
782
+ reject("is out of range");
783
+ }
784
+ };
785
+ for (const term of expr.split(",")) {
786
+ if (term === "") reject("contains an empty value");
787
+ let body = term;
788
+ if (body.includes("/")) {
789
+ const stepParts = body.split("/");
790
+ if (stepParts.length !== 2) {
791
+ reject("has malformed step syntax");
792
+ }
793
+ const [rangePart, stepStr] = stepParts;
794
+ const step = Number(stepStr);
795
+ if (!Number.isInteger(step) || step <= 0) {
796
+ reject("has an invalid step");
797
+ }
798
+ body = rangePart;
799
+ if (body === "*") continue;
800
+ }
801
+ if (body.includes("-")) {
802
+ const rangeParts = body.split("-");
803
+ if (rangeParts.length !== 2) {
804
+ reject("has malformed range syntax");
805
+ }
806
+ const [startStr, endStr] = rangeParts;
807
+ if (startStr === "" || endStr === "") {
808
+ reject("has an empty range part");
809
+ }
810
+ const start = Number(startStr);
811
+ const end = Number(endStr);
812
+ assertInRange(start);
813
+ assertInRange(end);
814
+ if (start > end) reject("has an inverted range");
815
+ } else {
816
+ assertInRange(Number(body));
817
+ }
818
+ }
819
+ }
820
+ function validateCronExpression(cron) {
821
+ const parts = cron.trim().split(/\s+/);
822
+ if (parts.length !== 5) {
823
+ throw new Error(
824
+ `Invalid cron expression: expected 5 fields, got ${parts.length}`
825
+ );
826
+ }
827
+ parts.forEach((field, index) => {
828
+ validateCronField(field, CRON_FIELD_RANGES[index]);
829
+ });
830
+ return parts;
831
+ }
832
+ function getNextCronDate(cron) {
833
+ const [minuteExpr, hourExpr, dayExpr, monthExpr, dowExpr] = validateCronExpression(cron);
834
+ const now = /* @__PURE__ */ new Date();
835
+ const candidate = new Date(now);
836
+ candidate.setSeconds(0);
837
+ candidate.setMilliseconds(0);
838
+ candidate.setMinutes(candidate.getMinutes() + 1);
839
+ const dayIsWildcard = dayExpr === "*";
840
+ const dowIsWildcard = dowExpr === "*";
841
+ const maxIterations = 525600;
842
+ for (let i2 = 0; i2 < maxIterations; i2++) {
843
+ const dayMatches = matchesCronField(candidate.getDate(), dayExpr);
844
+ const dow = candidate.getDay();
845
+ const dowMatches = matchesCronField(dow, dowExpr) || dow === 0 && matchesCronField(7, dowExpr);
846
+ let dayOfMonthOrWeekMatches;
847
+ if (!dayIsWildcard && !dowIsWildcard) {
848
+ dayOfMonthOrWeekMatches = dayMatches || dowMatches;
849
+ } else if (!dayIsWildcard) {
850
+ dayOfMonthOrWeekMatches = dayMatches;
851
+ } else if (!dowIsWildcard) {
852
+ dayOfMonthOrWeekMatches = dowMatches;
853
+ } else {
854
+ dayOfMonthOrWeekMatches = true;
855
+ }
856
+ if (matchesCronField(candidate.getMonth() + 1, monthExpr) && dayOfMonthOrWeekMatches && matchesCronField(candidate.getHours(), hourExpr) && matchesCronField(candidate.getMinutes(), minuteExpr)) {
857
+ return candidate;
858
+ }
859
+ candidate.setMinutes(candidate.getMinutes() + 1);
860
+ }
861
+ throw new Error(`Could not find next run date for cron: ${cron}`);
862
+ }
863
+ function matchesCronField(value, expr) {
864
+ if (expr === "*") return true;
865
+ if (expr.includes("/")) {
866
+ const [range, stepStr] = expr.split("/");
867
+ const step = parseInt(stepStr, 10);
868
+ if (range === "*") return value % step === 0;
869
+ if (range.includes("-")) {
870
+ const [startStr, endStr] = range.split("-");
871
+ const start = parseInt(startStr, 10);
872
+ const end = parseInt(endStr, 10);
873
+ if (value < start || value > end) return false;
874
+ return (value - start) % step === 0;
875
+ }
876
+ }
877
+ if (expr.includes("-")) {
878
+ const [startStr, endStr] = expr.split("-");
879
+ const start = parseInt(startStr, 10);
880
+ const end = parseInt(endStr, 10);
881
+ return value >= start && value <= end;
882
+ }
883
+ if (expr.includes(",")) {
884
+ const values = expr.split(",").map((v) => parseInt(v.trim(), 10));
885
+ return values.includes(value);
886
+ }
887
+ return value === parseInt(expr, 10);
888
+ }
889
+ function createScheduleRunner(config) {
890
+ return new ScheduleRunner(config);
891
+ }
892
+ export {
893
+ DEFAULT_TENANT_JOB_CAP,
894
+ JobBuilder,
895
+ J as JobContextLogger,
896
+ JobHandle,
897
+ M as MAX_JOB_RETRIES,
898
+ ScheduleRunner,
899
+ e as SmrtJob,
900
+ SmrtJobCollection,
901
+ f as SmrtJobEvent,
902
+ g as SmrtJobEventCollection,
903
+ h as SmrtWorker,
904
+ SmrtWorkerCollection,
905
+ T as TaskRunner,
906
+ i as TenantJobCapExceededError,
907
+ j as assertWithinTenantCreationCap,
908
+ k as backgroundEligible,
909
+ clampRetries,
910
+ createScheduleRunner,
911
+ l as createTaskRunner,
912
+ c as createWorkerKey,
913
+ m as getBackgroundEligibleMethods,
914
+ n as isBackgroundEligibleMethod,
915
+ isWorkerAlive,
916
+ o as markBackgroundEligible,
917
+ parseDelay,
918
+ priorityToNumber,
919
+ redactErrorForPersistence,
920
+ redactErrorMessage,
921
+ r as registerLiveWorker,
922
+ u as unregisterLiveWorker,
923
+ validateCronExpression,
924
+ withBackgroundJobs
925
+ };
926
+ //# sourceMappingURL=index.js.map