@happyvertical/smrt-jobs 0.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/AGENTS.md +71 -0
  2. package/CLAUDE.md +1 -0
  3. package/LICENSE +7 -0
  4. package/README.md +151 -0
  5. package/dist/__smrt-register__.d.ts +2 -0
  6. package/dist/__smrt-register__.d.ts.map +1 -0
  7. package/dist/background-policy.d.ts +121 -0
  8. package/dist/background-policy.d.ts.map +1 -0
  9. package/dist/chunks/runner-DV8FBO0y.js +1642 -0
  10. package/dist/chunks/runner-DV8FBO0y.js.map +1 -0
  11. package/dist/chunks/worker-liveness-DOTjoIjr.js +65 -0
  12. package/dist/chunks/worker-liveness-DOTjoIjr.js.map +1 -0
  13. package/dist/error-redaction.d.ts +48 -0
  14. package/dist/error-redaction.d.ts.map +1 -0
  15. package/dist/index.d.ts +13 -0
  16. package/dist/index.d.ts.map +1 -0
  17. package/dist/index.js +926 -0
  18. package/dist/index.js.map +1 -0
  19. package/dist/job-builder.d.ts +94 -0
  20. package/dist/job-builder.d.ts.map +1 -0
  21. package/dist/job-handle.d.ts +71 -0
  22. package/dist/job-handle.d.ts.map +1 -0
  23. package/dist/logger-extension.d.ts +58 -0
  24. package/dist/logger-extension.d.ts.map +1 -0
  25. package/dist/manifest.json +1327 -0
  26. package/dist/object-extension.d.ts +68 -0
  27. package/dist/object-extension.d.ts.map +1 -0
  28. package/dist/playground.d.ts +2 -0
  29. package/dist/playground.d.ts.map +1 -0
  30. package/dist/playground.js +179 -0
  31. package/dist/playground.js.map +1 -0
  32. package/dist/runner.d.ts +189 -0
  33. package/dist/runner.d.ts.map +1 -0
  34. package/dist/runner.js +15 -0
  35. package/dist/runner.js.map +1 -0
  36. package/dist/schedule-runner.d.ts +151 -0
  37. package/dist/schedule-runner.d.ts.map +1 -0
  38. package/dist/smrt-job-event.d.ts +54 -0
  39. package/dist/smrt-job-event.d.ts.map +1 -0
  40. package/dist/smrt-job.d.ts +215 -0
  41. package/dist/smrt-job.d.ts.map +1 -0
  42. package/dist/smrt-knowledge.json +508 -0
  43. package/dist/smrt-worker.d.ts +72 -0
  44. package/dist/smrt-worker.d.ts.map +1 -0
  45. package/dist/stale-recovery.d.ts +34 -0
  46. package/dist/stale-recovery.d.ts.map +1 -0
  47. package/dist/svelte/components/JobActions.svelte +103 -0
  48. package/dist/svelte/components/JobActions.svelte.d.ts +23 -0
  49. package/dist/svelte/components/JobActions.svelte.d.ts.map +1 -0
  50. package/dist/svelte/components/JobDashboard.svelte +199 -0
  51. package/dist/svelte/components/JobDashboard.svelte.d.ts +27 -0
  52. package/dist/svelte/components/JobDashboard.svelte.d.ts.map +1 -0
  53. package/dist/svelte/components/JobDetail.svelte +256 -0
  54. package/dist/svelte/components/JobDetail.svelte.d.ts +17 -0
  55. package/dist/svelte/components/JobDetail.svelte.d.ts.map +1 -0
  56. package/dist/svelte/components/JobList.svelte +360 -0
  57. package/dist/svelte/components/JobList.svelte.d.ts +28 -0
  58. package/dist/svelte/components/JobList.svelte.d.ts.map +1 -0
  59. package/dist/svelte/components/JobStats.svelte +242 -0
  60. package/dist/svelte/components/JobStats.svelte.d.ts +15 -0
  61. package/dist/svelte/components/JobStats.svelte.d.ts.map +1 -0
  62. package/dist/svelte/components/JobStatusBadge.svelte +23 -0
  63. package/dist/svelte/components/JobStatusBadge.svelte.d.ts +9 -0
  64. package/dist/svelte/components/JobStatusBadge.svelte.d.ts.map +1 -0
  65. package/dist/svelte/components/types.d.ts +9 -0
  66. package/dist/svelte/components/types.d.ts.map +1 -0
  67. package/dist/svelte/components/types.js +8 -0
  68. package/dist/svelte/i18n.d.ts +22 -0
  69. package/dist/svelte/i18n.d.ts.map +1 -0
  70. package/dist/svelte/i18n.js +22 -0
  71. package/dist/svelte/index.d.ts +25 -0
  72. package/dist/svelte/index.d.ts.map +1 -0
  73. package/dist/svelte/index.js +28 -0
  74. package/dist/svelte/playground.d.ts +329 -0
  75. package/dist/svelte/playground.d.ts.map +1 -0
  76. package/dist/svelte/playground.js +174 -0
  77. package/dist/svelte/types.d.ts +191 -0
  78. package/dist/svelte/types.d.ts.map +1 -0
  79. package/dist/svelte/types.js +87 -0
  80. package/dist/ui.d.ts +10 -0
  81. package/dist/ui.d.ts.map +1 -0
  82. package/dist/ui.js +69 -0
  83. package/dist/ui.js.map +1 -0
  84. package/dist/worker-liveness-thread.d.ts +2 -0
  85. package/dist/worker-liveness-thread.d.ts.map +1 -0
  86. package/dist/worker-liveness-thread.js +66 -0
  87. package/dist/worker-liveness-thread.js.map +1 -0
  88. package/dist/worker-liveness-ticker.d.ts +30 -0
  89. package/dist/worker-liveness-ticker.d.ts.map +1 -0
  90. package/dist/worker-liveness.d.ts +71 -0
  91. package/dist/worker-liveness.d.ts.map +1 -0
  92. package/package.json +93 -0
@@ -0,0 +1,508 @@
1
+ {
2
+ "schemaVersion": 1,
3
+ "generatedAt": "2026-06-23T01:11:19.800Z",
4
+ "packageName": "@happyvertical/smrt-jobs",
5
+ "packageVersion": "0.30.0",
6
+ "sourceManifestPath": "dist/manifest.json",
7
+ "agentDocPath": "AGENTS.md",
8
+ "sourceHashes": {
9
+ "manifest": "8bed9aec574feed7d15e21164393b59c9f976f2533e62f588445f314e8326ed4",
10
+ "packageJson": "3b3e6a6efac6ef8f88e0a8c91b899c37d21a90cc922fe6d4d05d6d40f23f073c",
11
+ "agents": "407e0ecd3f0ce3ae02d7dc84e86e457ddb6e75677d682b017768b4155b5a68ac"
12
+ },
13
+ "exports": [
14
+ ".",
15
+ "./manifest",
16
+ "./manifest.json",
17
+ "./playground",
18
+ "./runner",
19
+ "./svelte",
20
+ "./ui",
21
+ "./worker-liveness-thread"
22
+ ],
23
+ "dependencies": {
24
+ "@happyvertical/jobs": "catalog:",
25
+ "@happyvertical/logger": "catalog:",
26
+ "@happyvertical/smrt-config": "workspace:*",
27
+ "@happyvertical/smrt-core": "workspace:*",
28
+ "@happyvertical/smrt-tenancy": "workspace:*",
29
+ "@happyvertical/smrt-types": "workspace:*",
30
+ "@happyvertical/smrt-ui": "workspace:*",
31
+ "@happyvertical/sql": "catalog:",
32
+ "@happyvertical/utils": "catalog:",
33
+ "@happyvertical/smrt-vitest": "workspace:*",
34
+ "@sveltejs/package": "^2.5.7",
35
+ "@sveltejs/vite-plugin-svelte": "^6.2.4",
36
+ "@types/node": "25.0.9",
37
+ "svelte": "^5.18.0",
38
+ "svelte-check": "^4.3.5",
39
+ "typescript": "^5.9.3",
40
+ "vite": "^7.3.1",
41
+ "vitest": "^4.0.17"
42
+ },
43
+ "smrtDependencies": [
44
+ "@happyvertical/smrt-config",
45
+ "@happyvertical/smrt-core",
46
+ "@happyvertical/smrt-tenancy",
47
+ "@happyvertical/smrt-types",
48
+ "@happyvertical/smrt-ui",
49
+ "@happyvertical/smrt-vitest"
50
+ ],
51
+ "sdkDependencies": [
52
+ "@happyvertical/jobs",
53
+ "@happyvertical/logger",
54
+ "@happyvertical/sql",
55
+ "@happyvertical/utils"
56
+ ],
57
+ "tags": [],
58
+ "risks": [],
59
+ "objects": [
60
+ {
61
+ "name": "SmrtJobEvent",
62
+ "qualifiedName": "@happyvertical/smrt-jobs:SmrtJobEvent",
63
+ "collection": "smrtjobevents",
64
+ "tableName": "_smrt_job_events",
65
+ "packageName": "@happyvertical/smrt-jobs",
66
+ "extends": "SmrtObject",
67
+ "fields": [
68
+ {
69
+ "name": "tenantId",
70
+ "type": "text",
71
+ "required": false,
72
+ "columnType": "UUID"
73
+ },
74
+ {
75
+ "name": "jobId",
76
+ "type": "foreignKey",
77
+ "required": true,
78
+ "related": "SmrtJob",
79
+ "columnType": "UUID"
80
+ },
81
+ {
82
+ "name": "type",
83
+ "type": "text",
84
+ "required": true,
85
+ "columnType": "TEXT"
86
+ },
87
+ {
88
+ "name": "level",
89
+ "type": "text",
90
+ "required": true,
91
+ "columnType": "TEXT"
92
+ },
93
+ {
94
+ "name": "stage",
95
+ "type": "text",
96
+ "required": false,
97
+ "columnType": "TEXT"
98
+ },
99
+ {
100
+ "name": "progress",
101
+ "type": "integer",
102
+ "required": false,
103
+ "columnType": "INTEGER"
104
+ },
105
+ {
106
+ "name": "message",
107
+ "type": "text",
108
+ "required": true,
109
+ "columnType": "TEXT"
110
+ },
111
+ {
112
+ "name": "data",
113
+ "type": "json",
114
+ "required": false,
115
+ "columnType": "JSON"
116
+ },
117
+ {
118
+ "name": "createdAt",
119
+ "type": "datetime",
120
+ "required": true,
121
+ "columnType": "TIMESTAMP"
122
+ }
123
+ ],
124
+ "relationships": [
125
+ {
126
+ "name": "jobId",
127
+ "type": "foreignKey",
128
+ "required": true,
129
+ "related": "SmrtJob",
130
+ "columnType": "UUID"
131
+ }
132
+ ],
133
+ "methods": [
134
+ "toCursor"
135
+ ],
136
+ "surfaces": [
137
+ {
138
+ "kind": "cli",
139
+ "name": "smrtjobevent_list",
140
+ "operation": "list",
141
+ "objectName": "@happyvertical/smrt-jobs:SmrtJobEvent"
142
+ },
143
+ {
144
+ "kind": "cli",
145
+ "name": "smrtjobevent_get",
146
+ "operation": "get",
147
+ "objectName": "@happyvertical/smrt-jobs:SmrtJobEvent"
148
+ }
149
+ ],
150
+ "relationshipFeatures": [
151
+ "foreignKey",
152
+ "uuidColumns"
153
+ ],
154
+ "tags": [],
155
+ "risks": []
156
+ },
157
+ {
158
+ "name": "SmrtJobEventCollection",
159
+ "qualifiedName": "@happyvertical/smrt-jobs:SmrtJobEventCollection",
160
+ "collection": "smrtjobevents",
161
+ "tableName": "_smrt_job_events",
162
+ "packageName": "@happyvertical/smrt-jobs",
163
+ "extends": "SmrtCollection",
164
+ "fields": [],
165
+ "relationships": [],
166
+ "methods": [
167
+ "append",
168
+ "initialize",
169
+ "latestProgressByJobIds",
170
+ "listByJob",
171
+ "listSinceCursor"
172
+ ],
173
+ "surfaces": [],
174
+ "relationshipFeatures": [
175
+ "uuidColumns"
176
+ ],
177
+ "tags": [],
178
+ "risks": []
179
+ },
180
+ {
181
+ "name": "SmrtJob",
182
+ "qualifiedName": "@happyvertical/smrt-jobs:SmrtJob",
183
+ "collection": "smrtjobs",
184
+ "tableName": "_smrt_jobs",
185
+ "packageName": "@happyvertical/smrt-jobs",
186
+ "extends": "SmrtObject",
187
+ "fields": [
188
+ {
189
+ "name": "tenantId",
190
+ "type": "text",
191
+ "required": false,
192
+ "columnType": "UUID"
193
+ },
194
+ {
195
+ "name": "queue",
196
+ "type": "text",
197
+ "required": true,
198
+ "columnType": "TEXT"
199
+ },
200
+ {
201
+ "name": "objectType",
202
+ "type": "text",
203
+ "required": true,
204
+ "columnType": "TEXT"
205
+ },
206
+ {
207
+ "name": "objectId",
208
+ "type": "text",
209
+ "required": false,
210
+ "columnType": "TEXT"
211
+ },
212
+ {
213
+ "name": "method",
214
+ "type": "text",
215
+ "required": true,
216
+ "columnType": "TEXT"
217
+ },
218
+ {
219
+ "name": "args",
220
+ "type": "json",
221
+ "required": false,
222
+ "columnType": "JSON"
223
+ },
224
+ {
225
+ "name": "runAt",
226
+ "type": "datetime",
227
+ "required": true,
228
+ "columnType": "TIMESTAMP"
229
+ },
230
+ {
231
+ "name": "priority",
232
+ "type": "integer",
233
+ "required": true,
234
+ "columnType": "INTEGER"
235
+ },
236
+ {
237
+ "name": "status",
238
+ "type": "text",
239
+ "required": true,
240
+ "columnType": "TEXT"
241
+ },
242
+ {
243
+ "name": "attempts",
244
+ "type": "integer",
245
+ "required": true,
246
+ "columnType": "INTEGER"
247
+ },
248
+ {
249
+ "name": "maxAttempts",
250
+ "type": "integer",
251
+ "required": true,
252
+ "columnType": "INTEGER"
253
+ },
254
+ {
255
+ "name": "timeout",
256
+ "type": "integer",
257
+ "required": true,
258
+ "columnType": "INTEGER"
259
+ },
260
+ {
261
+ "name": "timeoutBehavior",
262
+ "type": "text",
263
+ "required": true,
264
+ "columnType": "TEXT"
265
+ },
266
+ {
267
+ "name": "startedAt",
268
+ "type": "datetime",
269
+ "required": false,
270
+ "columnType": "TIMESTAMP"
271
+ },
272
+ {
273
+ "name": "completedAt",
274
+ "type": "datetime",
275
+ "required": false,
276
+ "columnType": "TIMESTAMP"
277
+ },
278
+ {
279
+ "name": "lastError",
280
+ "type": "text",
281
+ "required": false,
282
+ "columnType": "TEXT"
283
+ },
284
+ {
285
+ "name": "resultPointer",
286
+ "type": "text",
287
+ "required": false,
288
+ "columnType": "TEXT"
289
+ },
290
+ {
291
+ "name": "retryStrategy",
292
+ "type": "json",
293
+ "required": false,
294
+ "columnType": "JSON"
295
+ },
296
+ {
297
+ "name": "workerId",
298
+ "type": "text",
299
+ "required": false,
300
+ "columnType": "TEXT"
301
+ },
302
+ {
303
+ "name": "workerHeartbeat",
304
+ "type": "datetime",
305
+ "required": false,
306
+ "columnType": "TIMESTAMP"
307
+ }
308
+ ],
309
+ "relationships": [],
310
+ "methods": [
311
+ "cancel",
312
+ "getDescription",
313
+ "retry",
314
+ "save"
315
+ ],
316
+ "surfaces": [
317
+ {
318
+ "kind": "cli",
319
+ "name": "smrtjob_list",
320
+ "operation": "list",
321
+ "objectName": "@happyvertical/smrt-jobs:SmrtJob"
322
+ },
323
+ {
324
+ "kind": "cli",
325
+ "name": "smrtjob_get",
326
+ "operation": "get",
327
+ "objectName": "@happyvertical/smrt-jobs:SmrtJob"
328
+ },
329
+ {
330
+ "kind": "cli",
331
+ "name": "smrtjob_retry",
332
+ "operation": "retry",
333
+ "objectName": "@happyvertical/smrt-jobs:SmrtJob"
334
+ },
335
+ {
336
+ "kind": "cli",
337
+ "name": "smrtjob_cancel",
338
+ "operation": "cancel",
339
+ "objectName": "@happyvertical/smrt-jobs:SmrtJob"
340
+ }
341
+ ],
342
+ "relationshipFeatures": [
343
+ "uuidColumns"
344
+ ],
345
+ "tags": [],
346
+ "risks": []
347
+ },
348
+ {
349
+ "name": "SmrtJobCollection",
350
+ "qualifiedName": "@happyvertical/smrt-jobs:SmrtJobCollection",
351
+ "collection": "smrtjobs",
352
+ "tableName": "_smrt_jobs",
353
+ "packageName": "@happyvertical/smrt-jobs",
354
+ "extends": "SmrtCollection",
355
+ "fields": [],
356
+ "relationships": [],
357
+ "methods": [
358
+ "claimReady",
359
+ "cleanup",
360
+ "countInFlightForTenant",
361
+ "enqueueJob",
362
+ "initialize",
363
+ "listByStatus",
364
+ "listReady",
365
+ "stats"
366
+ ],
367
+ "surfaces": [],
368
+ "relationshipFeatures": [
369
+ "uuidColumns"
370
+ ],
371
+ "tags": [],
372
+ "risks": []
373
+ },
374
+ {
375
+ "name": "SmrtWorker",
376
+ "qualifiedName": "@happyvertical/smrt-jobs:SmrtWorker",
377
+ "collection": "smrtworkers",
378
+ "tableName": "_smrt_workers",
379
+ "packageName": "@happyvertical/smrt-jobs",
380
+ "extends": "SmrtObject",
381
+ "fields": [
382
+ {
383
+ "name": "workerId",
384
+ "type": "text",
385
+ "required": true,
386
+ "columnType": "TEXT"
387
+ },
388
+ {
389
+ "name": "pid",
390
+ "type": "integer",
391
+ "required": false,
392
+ "columnType": "INTEGER"
393
+ },
394
+ {
395
+ "name": "hostname",
396
+ "type": "text",
397
+ "required": false,
398
+ "columnType": "TEXT"
399
+ },
400
+ {
401
+ "name": "startedAt",
402
+ "type": "datetime",
403
+ "required": false,
404
+ "columnType": "TIMESTAMP"
405
+ },
406
+ {
407
+ "name": "heartbeatAt",
408
+ "type": "datetime",
409
+ "required": false,
410
+ "columnType": "TIMESTAMP"
411
+ },
412
+ {
413
+ "name": "leaseExpiresAt",
414
+ "type": "datetime",
415
+ "required": false,
416
+ "columnType": "TIMESTAMP"
417
+ },
418
+ {
419
+ "name": "status",
420
+ "type": "text",
421
+ "required": true,
422
+ "columnType": "TEXT"
423
+ }
424
+ ],
425
+ "relationships": [],
426
+ "methods": [],
427
+ "surfaces": [],
428
+ "relationshipFeatures": [
429
+ "uuidColumns"
430
+ ],
431
+ "tags": [],
432
+ "risks": []
433
+ },
434
+ {
435
+ "name": "SmrtWorkerCollection",
436
+ "qualifiedName": "@happyvertical/smrt-jobs:SmrtWorkerCollection",
437
+ "collection": "smrtworkers",
438
+ "tableName": "_smrt_workers",
439
+ "packageName": "@happyvertical/smrt-jobs",
440
+ "extends": "SmrtCollection",
441
+ "fields": [],
442
+ "relationships": [],
443
+ "methods": [
444
+ "assertReady",
445
+ "expireWorker",
446
+ "freshLeaseWorkerKeys",
447
+ "pruneExpired",
448
+ "registerWorker",
449
+ "renewLease",
450
+ "tableReady"
451
+ ],
452
+ "surfaces": [],
453
+ "relationshipFeatures": [
454
+ "uuidColumns"
455
+ ],
456
+ "tags": [],
457
+ "risks": []
458
+ }
459
+ ],
460
+ "surfaces": [
461
+ {
462
+ "kind": "cli",
463
+ "name": "smrtjobevent_list",
464
+ "operation": "list",
465
+ "objectName": "@happyvertical/smrt-jobs:SmrtJobEvent"
466
+ },
467
+ {
468
+ "kind": "cli",
469
+ "name": "smrtjobevent_get",
470
+ "operation": "get",
471
+ "objectName": "@happyvertical/smrt-jobs:SmrtJobEvent"
472
+ },
473
+ {
474
+ "kind": "cli",
475
+ "name": "smrtjob_list",
476
+ "operation": "list",
477
+ "objectName": "@happyvertical/smrt-jobs:SmrtJob"
478
+ },
479
+ {
480
+ "kind": "cli",
481
+ "name": "smrtjob_get",
482
+ "operation": "get",
483
+ "objectName": "@happyvertical/smrt-jobs:SmrtJob"
484
+ },
485
+ {
486
+ "kind": "cli",
487
+ "name": "smrtjob_retry",
488
+ "operation": "retry",
489
+ "objectName": "@happyvertical/smrt-jobs:SmrtJob"
490
+ },
491
+ {
492
+ "kind": "cli",
493
+ "name": "smrtjob_cancel",
494
+ "operation": "cancel",
495
+ "objectName": "@happyvertical/smrt-jobs:SmrtJob"
496
+ }
497
+ ],
498
+ "prompts": [],
499
+ "relationshipsV2": {
500
+ "foreignKeyFields": 1,
501
+ "crossPackageRefFields": 0,
502
+ "junctionCollections": 0,
503
+ "hierarchicalObjects": 0,
504
+ "polymorphicAssociations": 0,
505
+ "uuidColumns": 9
506
+ },
507
+ "agentDoc": "# @happyvertical/smrt-jobs\n\nBackground job execution with persistent queue, scheduling, and fluent builder API.\n\n## Architecture\n\n```\nSmrtObject.bg('method') → SmrtJob (in _smrt_jobs) → TaskRunner picks up → executes via ObjectRegistry\nAgentSchedule (cron) → ScheduleRunner creates SmrtJob → TaskRunner executes → ScheduleRunner updates\nTaskRunner.start() → SmrtWorker lease (in _smrt_workers) → recovery keys on worker liveness, not heartbeat\n```\n\n## SmrtJob\n\nPersistent in `_smrt_jobs`. Fields: `queue` (default), `objectType`, `objectId`, `method`, `args`, `runAt`, `priority` (higher=sooner), `status`, `attempts`/`maxAttempts`, `timeout` (default 5min), `retryStrategy`, `workerId` (the owning runner's incarnation key), `workerHeartbeat` (telemetry only — no longer gates recovery).\n\nStatus: `pending → running → completed/failed/cancelled`.\n\n## TaskRunner\n\nPolling-based execution engine. Config: `concurrency` (5), `pollInterval` (1s), `heartbeatInterval` (30s, telemetry only), `leaseTtlMs` (30s), `leaseTickMs` (10s), `shutdownTimeout` (30s).\n\n1. `start()` calls `assertReady()` (fail fast if `_smrt_workers` unmigrated), registers a seeded `SmrtWorker` lease, and adds its worker key to the process-global live set — all **before** polling\n2. Polls `claimReady()` to atomically claim pending jobs (`runAt <= NOW`, ordered by `priority DESC, runAt ASC, created_at ASC, id ASC`)\n3. Claim sets `status='running'`, `workerId=<incarnation key>`, heartbeat/start timestamps, and increments `attempts`\n4. Resolves class via `ObjectRegistry.getClass(objectType)`, creates instance, calls method\n5. **Internal args**: `_agentConfig` and `_scheduleId` stripped from args before calling method\n6. Terminal/retry writes are **conditional** (`WHERE worker_id=? AND status='running'`) so a recovered row is never stomped\n7. Retry: uses strategy from `@happyvertical/jobs`, schedules future `runAt` on failure\n8. Events: `job:started`, `job:completed`, `job:failed`, `job:retrying`, `runner:started/stopped`\n\n## Worker liveness & recovery (#1474)\n\nRecovery keys on **worker-process liveness**, never per-job heartbeat freshness (a CPU-bound synchronous handler used to starve the heartbeat and false-recover its own running jobs).\n\n- **`SmrtWorker` / `_smrt_workers`**: one lease row per runner *incarnation* (`workerId` is per-incarnation unique via `createWorkerKey`, so a restart never looks like it still owns the previous crash's jobs). `leaseExpiresAt` is a `datetime` (an integer epoch-ms column overflows `int4`/`INT32` on Postgres/DuckDB). Stage 1 writes/compares it against the host clock — the same approach the old heartbeat recovery used, so it's no more skew-sensitive than the code it replaced.\n- **Process-global live set** (`worker-liveness.ts`, `globalThis.__smrtLiveWorkers`): checked synchronously, so it can't be starved by a blocked loop. Covers all same-process topologies.\n- **Off-loop ticker** (`worker-liveness-ticker.ts` + `worker-liveness-thread.ts`): for engines a second connection can reach (Postgres, file-backed SQLite — `offLoopEligible()`), `start()` spawns a `node:worker_threads` ticker that renews the lease on its own thread, so a CPU-bound synchronous handler on the main loop can't starve it. In-memory SQLite / DuckDB, a thread-spawn failure, a start-handshake timeout, or the thread dying mid-run all fall back to main-loop renewal; the in-process live set keeps same-process correct regardless. The worker entry is a separate build entry resolved via `import.meta.resolve('@happyvertical/smrt-jobs/worker-liveness-thread')`.\n- **Recovery rule** (both runners): a `running` job is orphaned iff its worker is *not alive* = not in the live set **and** no fresh `_smrt_workers` lease. The live set takes precedence over a stale lease. TaskRunner also never recovers a job in its own `activeJobs`. If `_smrt_workers` is absent, recovery skips lease checks (never mass-recovers). Recovery is swept at most once per lease tick, and terminal/recovery writes use `RETURNING id` (not `rowCount`, which DuckDB/JSON adapters always report as ≥1).\n- **Lease clock**: the lease is compared against the host clock (same as the old heartbeat; fine with NTP + a 30s TTL). A dead process stops renewing and the lease expires within its TTL — that is how recovery detects death. (Instant cross-process detection via Postgres session advisory locks was prototyped on `@happyvertical/sql`'s `acquireSession()` but deferred — treating a free lock as proof-of-death false-recovers any worker legitimately in main-loop fallback mode.)\n\n## ScheduleRunner\n\nPolls `_smrt_agent_schedules` every 60s for due entries. Creates SmrtJob with `queue='agents'`, `priority=75`. Wires to TaskRunner events for completion/failure tracking. Slot reconciliation keys on worker liveness (it has no in-process active-job set, so the lease/live-set is its whole mechanism).\n\nCustom cron parser: 5-field (minute hour dom month dow). `*`, ranges, lists, steps supported. **Not timezone-aware** (UTC).\n\n## JobBuilder — Fluent API\n\n```typescript\nconst handle = await doc.background('analyze', { detailed: true })\n .delay('5m').priority('high').retries(5).queue('analysis').timeout(600000).enqueue();\n\nawait handle.wait({ timeout: 60000, pollInterval: 100 }); // polling-based\n```\n\n`bg()` is shorthand: `await doc.bg('analyze', args)` → enqueues immediately, returns JobHandle.\n\n## withBackgroundJobs(Class)\n\nMixin that adds `bg()` and `background()` to any SmrtObject. Uses WeakMap for collection caching per DB instance.\n\n## Gotchas\n\n- **Cron not timezone-aware**: all times treated as UTC\n- **No dead letter queue**: failed jobs stay in DB with `status='failed'` — manual intervention\n- **Result storage**: `resultPointer` is just a string — app must implement result backend\n- **Lazy builder**: `background()` returns builder — nothing happens until `enqueue()`\n- **wait() is polling**: JobHandle.wait() polls DB every 100ms (configurable)\n- **Migrate before start()**: `TaskRunner.start()` throws if `_smrt_workers` is missing — run `smrt db:migrate` after upgrading. Tables are never created at runtime.\n- **Recovery is liveness-based**: don't reintroduce heartbeat-threshold recovery; a blocked event loop must not look dead (see Worker liveness section, #1474)\n"
508
+ }
@@ -0,0 +1,72 @@
1
+ import { SmrtCollection, SmrtObject } from '@happyvertical/smrt-core';
2
+ /**
3
+ * Liveness record for a single TaskRunner / ScheduleRunner incarnation,
4
+ * stored in the `_smrt_workers` system table.
5
+ *
6
+ * @remarks
7
+ * Job recovery asks "is this job's owning worker alive?" rather than "is this
8
+ * job's heartbeat fresh?" (issue #1474). Each running worker keeps a row here
9
+ * and renews `leaseExpiresAt` on a fixed cadence; a worker that dies stops
10
+ * renewing and its lease expires, so its `running` jobs are recovered.
11
+ *
12
+ * `workerId` is unique per *incarnation* (a restarted runner gets a new key —
13
+ * see `createWorkerKey`), which is what lets recovery distinguish a crashed
14
+ * worker's orphaned jobs from an identically-configured restart.
15
+ *
16
+ * `leaseExpiresAt` is a `datetime` so it maps to a real timestamp column on
17
+ * every engine (an integer epoch-ms column overflows `int4`/`INT32` on
18
+ * Postgres and DuckDB). Stage 1 writes/compares it against the host clock —
19
+ * the same approach the previous heartbeat recovery used; Stage 2 will move to
20
+ * database-side time once an off-loop writer exists.
21
+ */
22
+ export declare class SmrtWorker extends SmrtObject {
23
+ /** Per-incarnation-unique worker key (also stored on owned jobs' workerId). */
24
+ workerId: string;
25
+ /** OS process id of the owning runner (diagnostic). */
26
+ pid: number | null;
27
+ /** Hostname of the owning runner (diagnostic). */
28
+ hostname: string | null;
29
+ /** When this incarnation started. */
30
+ startedAt: Date | null;
31
+ /** Last lease renewal time (diagnostic; liveness uses leaseExpiresAt). */
32
+ heartbeatAt: Date | null;
33
+ /** Lease expiry — the worker is alive while this is in the future. */
34
+ leaseExpiresAt: Date | null;
35
+ /** Lifecycle status (`running` while the runner is processing). */
36
+ status: string;
37
+ }
38
+ export interface RegisterWorkerInput {
39
+ workerKey: string;
40
+ pid?: number | null;
41
+ hostname?: string | null;
42
+ leaseTtlMs: number;
43
+ }
44
+ /**
45
+ * Collection for managing `_smrt_workers` liveness rows.
46
+ */
47
+ export declare class SmrtWorkerCollection extends SmrtCollection<SmrtWorker> {
48
+ static readonly _itemClass: typeof SmrtWorker;
49
+ /**
50
+ * Fail fast if the `_smrt_workers` table has not been migrated.
51
+ *
52
+ * The framework never creates application/system tables at runtime; the
53
+ * table is created by `smrt db:migrate` (or `getTestDatabase`). A consumer
54
+ * that upgrades smrt-jobs without migrating must get a clear, actionable
55
+ * error at `start()` rather than a confusing recovery failure later.
56
+ */
57
+ assertReady(): Promise<void>;
58
+ /** Whether the `_smrt_workers` table exists (recovery skips lease checks if not). */
59
+ tableReady(): Promise<boolean>;
60
+ /** Register a worker incarnation with its lease seeded to `now + ttl`. */
61
+ registerWorker(input: RegisterWorkerInput): Promise<void>;
62
+ /** Renew a worker's lease to `now + ttl`. */
63
+ renewLease(workerKey: string, leaseTtlMs: number): Promise<void>;
64
+ /** Remove a worker incarnation (graceful shutdown). */
65
+ expireWorker(workerKey: string): Promise<void>;
66
+ /** Worker keys whose database lease is still fresh (alive cross-process). */
67
+ freshLeaseWorkerKeys(): Promise<Set<string>>;
68
+ /** Delete worker rows whose lease expired more than `graceMs` ago. */
69
+ pruneExpired(graceMs: number): Promise<void>;
70
+ }
71
+ export default SmrtWorker;
72
+ //# sourceMappingURL=smrt-worker.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"smrt-worker.d.ts","sourceRoot":"","sources":["../src/smrt-worker.ts"],"names":[],"mappings":"AAEA,OAAO,wBAAwB,CAAC;AAEhC,OAAO,EAEL,cAAc,EACd,UAAU,EAEX,MAAM,0BAA0B,CAAC;AAElC;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAOa,UAAW,SAAQ,UAAU;IACxC,+EAA+E;IAE/E,QAAQ,EAAE,MAAM,CAAM;IAEtB,uDAAuD;IAEvD,GAAG,EAAE,MAAM,GAAG,IAAI,CAAQ;IAE1B,kDAAkD;IAElD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAQ;IAE/B,qCAAqC;IAErC,SAAS,EAAE,IAAI,GAAG,IAAI,CAAQ;IAE9B,0EAA0E;IAE1E,WAAW,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEhC,sEAAsE;IAEtE,cAAc,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEnC,mEAAmE;IAEnE,MAAM,EAAE,MAAM,CAAa;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,qBAAa,oBAAqB,SAAQ,cAAc,CAAC,UAAU,CAAC;IAClE,MAAM,CAAC,QAAQ,CAAC,UAAU,oBAAc;IAExC;;;;;;;OAOG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAYlC,qFAAqF;IAC/E,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IASpC,0EAA0E;IACpE,cAAc,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAe/D,6CAA6C;IACvC,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAatE,uDAAuD;IACjD,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOpD,6EAA6E;IACvE,oBAAoB,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAelD,sEAAsE;IAChE,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CASnD;AAED,eAAe,UAAU,CAAC"}
@@ -0,0 +1,34 @@
1
+ export declare const DEFAULT_TASK_HEARTBEAT_INTERVAL_MS = 30000;
2
+ /**
3
+ * Default cadence for renewing a worker's liveness lease.
4
+ *
5
+ * Liveness is a per-*worker* lease (see {@link ../smrt-worker.js}), not a
6
+ * per-job heartbeat. The runner renews its lease on this interval; a dead
7
+ * worker stops renewing and its lease expires after {@link DEFAULT_LEASE_TTL_MS}.
8
+ */
9
+ export declare const DEFAULT_LEASE_TICK_MS = 10000;
10
+ /**
11
+ * Default time-to-live for a worker liveness lease.
12
+ *
13
+ * A `running` job is only recovered when its owning worker is neither live in
14
+ * this process nor holding a fresh lease in the database. The TTL is the
15
+ * cross-process detection latency for a genuinely dead worker.
16
+ */
17
+ export declare const DEFAULT_LEASE_TTL_MS = 30000;
18
+ /**
19
+ * Keep stale-job recovery aligned with the actual heartbeat cadence.
20
+ *
21
+ * @deprecated Recovery no longer keys on per-job heartbeat staleness (#1474);
22
+ * it keys on worker liveness. Retained for one release for any external caller.
23
+ * Use {@link getEffectiveLeaseTtlMs} with the lease tick instead.
24
+ */
25
+ export declare function getEffectiveStaleJobThresholdMs(staleJobThresholdMs: number, heartbeatIntervalMs: number): number;
26
+ /**
27
+ * Never let a lease expire in fewer than three renewal ticks.
28
+ *
29
+ * A single missed renewal (GC pause, a slow database round-trip) must not be
30
+ * enough to declare a healthy worker dead. Mirrors the floor applied by
31
+ * {@link getEffectiveStaleJobThresholdMs} for the legacy heartbeat path.
32
+ */
33
+ export declare function getEffectiveLeaseTtlMs(leaseTtlMs: number, leaseTickMs: number): number;
34
+ //# sourceMappingURL=stale-recovery.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"stale-recovery.d.ts","sourceRoot":"","sources":["../src/stale-recovery.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,kCAAkC,QAAQ,CAAC;AAGxD;;;;;;GAMG;AACH,eAAO,MAAM,qBAAqB,QAAQ,CAAC;AAE3C;;;;;;GAMG;AACH,eAAO,MAAM,oBAAoB,QAAQ,CAAC;AAI1C;;;;;;GAMG;AACH,wBAAgB,+BAA+B,CAC7C,mBAAmB,EAAE,MAAM,EAC3B,mBAAmB,EAAE,MAAM,GAC1B,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,GAClB,MAAM,CAER"}