@cat-factory/node-server 0.153.0 → 0.154.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/config.d.ts.map +1 -1
  2. package/dist/config.js +6 -2
  3. package/dist/config.js.map +1 -1
  4. package/dist/container-core-deps.d.ts.map +1 -1
  5. package/dist/container-core-deps.js +9 -1
  6. package/dist/container-core-deps.js.map +1 -1
  7. package/dist/container-executor-deps.d.ts.map +1 -1
  8. package/dist/container-executor-deps.js +4 -1
  9. package/dist/container-executor-deps.js.map +1 -1
  10. package/dist/container-options.d.ts +15 -1
  11. package/dist/container-options.d.ts.map +1 -1
  12. package/dist/container-options.js.map +1 -1
  13. package/dist/db/schema-slack.d.ts +262 -0
  14. package/dist/db/schema-slack.d.ts.map +1 -0
  15. package/dist/db/schema-slack.js +42 -0
  16. package/dist/db/schema-slack.js.map +1 -0
  17. package/dist/db/schema.d.ts +16 -261
  18. package/dist/db/schema.d.ts.map +1 -1
  19. package/dist/db/schema.js +7 -32
  20. package/dist/db/schema.js.map +1 -1
  21. package/dist/environments.d.ts +4 -2
  22. package/dist/environments.d.ts.map +1 -1
  23. package/dist/environments.js +4 -1
  24. package/dist/environments.js.map +1 -1
  25. package/dist/execution/bootstrapRunner.d.ts.map +1 -1
  26. package/dist/execution/bootstrapRunner.js +2 -1
  27. package/dist/execution/bootstrapRunner.js.map +1 -1
  28. package/dist/execution/deadLetter.d.ts +16 -0
  29. package/dist/execution/deadLetter.d.ts.map +1 -0
  30. package/dist/execution/deadLetter.js +40 -0
  31. package/dist/execution/deadLetter.js.map +1 -0
  32. package/dist/execution/envConfigRepairRunner.d.ts.map +1 -1
  33. package/dist/execution/envConfigRepairRunner.js +2 -1
  34. package/dist/execution/envConfigRepairRunner.js.map +1 -1
  35. package/dist/execution/envTestRunner.d.ts +10 -3
  36. package/dist/execution/envTestRunner.d.ts.map +1 -1
  37. package/dist/execution/envTestRunner.js +16 -2
  38. package/dist/execution/envTestRunner.js.map +1 -1
  39. package/dist/execution/githubSyncRunner.d.ts.map +1 -1
  40. package/dist/execution/githubSyncRunner.js +2 -1
  41. package/dist/execution/githubSyncRunner.js.map +1 -1
  42. package/dist/execution/pgBossRunner.d.ts +7 -3
  43. package/dist/execution/pgBossRunner.d.ts.map +1 -1
  44. package/dist/execution/pgBossRunner.js +31 -2
  45. package/dist/execution/pgBossRunner.js.map +1 -1
  46. package/dist/execution/trackerSyncRunner.d.ts.map +1 -1
  47. package/dist/execution/trackerSyncRunner.js +2 -1
  48. package/dist/execution/trackerSyncRunner.js.map +1 -1
  49. package/dist/foundationalServices.d.ts +4 -2
  50. package/dist/foundationalServices.d.ts.map +1 -1
  51. package/dist/foundationalServices.js +4 -1
  52. package/dist/foundationalServices.js.map +1 -1
  53. package/dist/githubReconcile.d.ts +4 -2
  54. package/dist/githubReconcile.d.ts.map +1 -1
  55. package/dist/githubReconcile.js +4 -1
  56. package/dist/githubReconcile.js.map +1 -1
  57. package/dist/index.d.ts +1 -0
  58. package/dist/index.d.ts.map +1 -1
  59. package/dist/index.js +7 -0
  60. package/dist/index.js.map +1 -1
  61. package/dist/infraReachability.d.ts +4 -2
  62. package/dist/infraReachability.d.ts.map +1 -1
  63. package/dist/infraReachability.js +5 -2
  64. package/dist/infraReachability.js.map +1 -1
  65. package/dist/initiativeLoop.d.ts +4 -2
  66. package/dist/initiativeLoop.d.ts.map +1 -1
  67. package/dist/initiativeLoop.js +4 -1
  68. package/dist/initiativeLoop.js.map +1 -1
  69. package/dist/kaizen.d.ts +4 -2
  70. package/dist/kaizen.d.ts.map +1 -1
  71. package/dist/kaizen.js +4 -1
  72. package/dist/kaizen.js.map +1 -1
  73. package/dist/notifications.d.ts +4 -2
  74. package/dist/notifications.d.ts.map +1 -1
  75. package/dist/notifications.js +5 -2
  76. package/dist/notifications.js.map +1 -1
  77. package/dist/platformHealth.d.ts +4 -2
  78. package/dist/platformHealth.d.ts.map +1 -1
  79. package/dist/platformHealth.js +5 -2
  80. package/dist/platformHealth.js.map +1 -1
  81. package/dist/platformMetrics.d.ts +17 -3
  82. package/dist/platformMetrics.d.ts.map +1 -1
  83. package/dist/platformMetrics.js +23 -3
  84. package/dist/platformMetrics.js.map +1 -1
  85. package/dist/queueDepth.d.ts +38 -0
  86. package/dist/queueDepth.d.ts.map +1 -0
  87. package/dist/queueDepth.js +69 -0
  88. package/dist/queueDepth.js.map +1 -0
  89. package/dist/readiness.d.ts +16 -0
  90. package/dist/readiness.d.ts.map +1 -1
  91. package/dist/readiness.js +34 -9
  92. package/dist/readiness.js.map +1 -1
  93. package/dist/recurring.d.ts +4 -2
  94. package/dist/recurring.d.ts.map +1 -1
  95. package/dist/recurring.js +4 -1
  96. package/dist/recurring.js.map +1 -1
  97. package/dist/repositories/drizzle/execution.d.ts +1 -0
  98. package/dist/repositories/drizzle/execution.d.ts.map +1 -1
  99. package/dist/repositories/drizzle/execution.js +12 -0
  100. package/dist/repositories/drizzle/execution.js.map +1 -1
  101. package/dist/retention.d.ts +22 -6
  102. package/dist/retention.d.ts.map +1 -1
  103. package/dist/retention.js +37 -23
  104. package/dist/retention.js.map +1 -1
  105. package/dist/server.d.ts +9 -0
  106. package/dist/server.d.ts.map +1 -1
  107. package/dist/server.js +54 -18
  108. package/dist/server.js.map +1 -1
  109. package/dist/sweeper.d.ts +10 -0
  110. package/dist/sweeper.d.ts.map +1 -1
  111. package/dist/sweeper.js +8 -2
  112. package/dist/sweeper.js.map +1 -1
  113. package/drizzle/20260803043617_bent_spitfire/migration.sql +1 -0
  114. package/drizzle/20260803043617_bent_spitfire/snapshot.json +18998 -0
  115. package/package.json +20 -20
package/dist/kaizen.d.ts CHANGED
@@ -1,9 +1,11 @@
1
1
  import type { Clock } from '@cat-factory/kernel';
2
- import type { Logger, ServerContainer } from '@cat-factory/server';
2
+ import type { Logger, ServerContainer, SweepHealthTracker } from '@cat-factory/server';
3
3
  /**
4
4
  * Start the periodic Kaizen grading sweep. Runs once immediately then on the interval,
5
5
  * non-overlapping + best-effort (see {@link startSweeper}). Returns a stop function that
6
6
  * clears the timer.
7
7
  */
8
- export declare function startKaizenSweeper(container: ServerContainer, clock: Clock, log: Logger): () => void;
8
+ export declare function startKaizenSweeper(container: ServerContainer, clock: Clock, log: Logger,
9
+ /** Records each pass's outcome under this sweep's name (see {@link startSweeper}). */
10
+ health: SweepHealthTracker): () => void;
9
11
  //# sourceMappingURL=kaizen.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"kaizen.d.ts","sourceRoot":"","sources":["../src/kaizen.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AAelE;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,MAAM,GACV,MAAM,IAAI,CAgBZ"}
1
+ {"version":3,"file":"kaizen.d.ts","sourceRoot":"","sources":["../src/kaizen.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AAetF;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,MAAM;AACX,sFAAsF;AACtF,MAAM,EAAE,kBAAkB,GACzB,MAAM,IAAI,CAiBZ"}
package/dist/kaizen.js CHANGED
@@ -14,7 +14,9 @@ const KAIZEN_SWEEP_BATCH = 5;
14
14
  * non-overlapping + best-effort (see {@link startSweeper}). Returns a stop function that
15
15
  * clears the timer.
16
16
  */
17
- export function startKaizenSweeper(container, clock, log) {
17
+ export function startKaizenSweeper(container, clock, log,
18
+ /** Records each pass's outcome under this sweep's name (see {@link startSweeper}). */
19
+ health) {
18
20
  const kaizen = container.kaizen;
19
21
  if (!kaizen)
20
22
  return () => { };
@@ -22,6 +24,7 @@ export function startKaizenSweeper(container, clock, log) {
22
24
  name: 'kaizen',
23
25
  intervalMs: KAIZEN_SWEEP_INTERVAL_MS,
24
26
  log,
27
+ health,
25
28
  failureMessage: 'kaizen sweep failed',
26
29
  tick: async () => {
27
30
  const processed = await kaizen.service.runPending(clock.now() - KAIZEN_STALE_MS, KAIZEN_SWEEP_BATCH);
@@ -1 +1 @@
1
- {"version":3,"file":"kaizen.js","sourceRoot":"","sources":["../src/kaizen.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,mFAAmF;AACnF,gFAAgF;AAChF,qFAAqF;AACrF,wFAAwF;AAExF,2FAA2F;AAC3F,MAAM,wBAAwB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAA;AAC9C,uFAAuF;AACvF,MAAM,eAAe,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AACtC,gFAAgF;AAChF,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAE5B;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,SAA0B,EAC1B,KAAY,EACZ,GAAW;IAEX,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAA;IAC/B,IAAI,CAAC,MAAM;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IAC5B,OAAO,YAAY,CAAC;QAClB,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,wBAAwB;QACpC,GAAG;QACH,cAAc,EAAE,qBAAqB;QACrC,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,UAAU,CAC/C,KAAK,CAAC,GAAG,EAAE,GAAG,eAAe,EAC7B,kBAAkB,CACnB,CAAA;YACD,IAAI,SAAS,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QAC3E,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
1
+ {"version":3,"file":"kaizen.js","sourceRoot":"","sources":["../src/kaizen.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,mFAAmF;AACnF,gFAAgF;AAChF,qFAAqF;AACrF,wFAAwF;AAExF,2FAA2F;AAC3F,MAAM,wBAAwB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAA;AAC9C,uFAAuF;AACvF,MAAM,eAAe,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AACtC,gFAAgF;AAChF,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAE5B;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,SAA0B,EAC1B,KAAY,EACZ,GAAW;AACX,sFAAsF;AACtF,MAA0B;IAE1B,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAA;IAC/B,IAAI,CAAC,MAAM;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IAC5B,OAAO,YAAY,CAAC;QAClB,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,wBAAwB;QACpC,GAAG;QACH,MAAM;QACN,cAAc,EAAE,qBAAqB;QACrC,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,UAAU,CAC/C,KAAK,CAAC,GAAG,EAAE,GAAG,eAAe,EAC7B,kBAAkB,CACnB,CAAA;YACD,IAAI,SAAS,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QAC3E,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
@@ -1,9 +1,11 @@
1
1
  import type { Clock } from '@cat-factory/kernel';
2
- import { type Logger, type ServerContainer } from '@cat-factory/server';
2
+ import { type Logger, type ServerContainer, type SweepHealthTracker } from '@cat-factory/server';
3
3
  /**
4
4
  * Start the periodic notification-escalation sweep. Runs once immediately then on the
5
5
  * interval, non-overlapping + best-effort (see {@link startSweeper}). Returns a stop
6
6
  * function that clears the timer.
7
7
  */
8
- export declare function startNotificationEscalationSweeper(container: ServerContainer, clock: Clock, log: Logger): () => void;
8
+ export declare function startNotificationEscalationSweeper(container: ServerContainer, clock: Clock, log: Logger,
9
+ /** Records each pass's outcome under this sweep's name (see {@link startSweeper}). */
10
+ health: SweepHealthTracker): () => void;
9
11
  //# sourceMappingURL=notifications.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"notifications.d.ts","sourceRoot":"","sources":["../src/notifications.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,EAA8B,KAAK,MAAM,EAAE,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAA;AAYnG;;;;GAIG;AACH,wBAAgB,kCAAkC,CAChD,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,MAAM,GACV,MAAM,IAAI,CAYZ"}
1
+ {"version":3,"file":"notifications.d.ts","sourceRoot":"","sources":["../src/notifications.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,EAEL,KAAK,MAAM,EACX,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAA;AAY5B;;;;GAIG;AACH,wBAAgB,kCAAkC,CAChD,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,MAAM;AACX,sFAAsF;AACtF,MAAM,EAAE,kBAAkB,GACzB,MAAM,IAAI,CAaZ"}
@@ -1,4 +1,4 @@
1
- import { escalateStaleNotifications } from '@cat-factory/server';
1
+ import { escalateStaleNotifications, } from '@cat-factory/server';
2
2
  import { startSweeper } from './sweeper.js';
3
3
  // Periodic notification-escalation sweep for the Node facade — the analogue of the
4
4
  // Worker's every-2-min cron call to `escalateStaleNotifications`. Runs no longer time out
@@ -12,13 +12,16 @@ const NOTIFICATION_ESCALATION_INTERVAL_MS = 60 * 1000;
12
12
  * interval, non-overlapping + best-effort (see {@link startSweeper}). Returns a stop
13
13
  * function that clears the timer.
14
14
  */
15
- export function startNotificationEscalationSweeper(container, clock, log) {
15
+ export function startNotificationEscalationSweeper(container, clock, log,
16
+ /** Records each pass's outcome under this sweep's name (see {@link startSweeper}). */
17
+ health) {
16
18
  if (!container.notifications)
17
19
  return () => { };
18
20
  return startSweeper({
19
21
  name: 'notification-escalation',
20
22
  intervalMs: NOTIFICATION_ESCALATION_INTERVAL_MS,
21
23
  log,
24
+ health,
22
25
  failureMessage: 'notification escalation failed',
23
26
  tick: async () => {
24
27
  const escalated = await escalateStaleNotifications(container, clock.now());
@@ -1 +1 @@
1
- {"version":3,"file":"notifications.js","sourceRoot":"","sources":["../src/notifications.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAqC,MAAM,qBAAqB,CAAA;AACnG,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,mFAAmF;AACnF,0FAA0F;AAC1F,8EAA8E;AAC9E,0FAA0F;AAC1F,kFAAkF;AAElF,uEAAuE;AACvE,MAAM,mCAAmC,GAAG,EAAE,GAAG,IAAI,CAAA;AAErD;;;;GAIG;AACH,MAAM,UAAU,kCAAkC,CAChD,SAA0B,EAC1B,KAAY,EACZ,GAAW;IAEX,IAAI,CAAC,SAAS,CAAC,aAAa;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IAC7C,OAAO,YAAY,CAAC;QAClB,IAAI,EAAE,yBAAyB;QAC/B,UAAU,EAAE,mCAAmC;QAC/C,GAAG;QACH,cAAc,EAAE,gCAAgC;QAChD,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,SAAS,GAAG,MAAM,0BAA0B,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAA;YAC1E,IAAI,SAAS,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QACvE,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
1
+ {"version":3,"file":"notifications.js","sourceRoot":"","sources":["../src/notifications.ts"],"names":[],"mappings":"AACA,OAAO,EACL,0BAA0B,GAI3B,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,mFAAmF;AACnF,0FAA0F;AAC1F,8EAA8E;AAC9E,0FAA0F;AAC1F,kFAAkF;AAElF,uEAAuE;AACvE,MAAM,mCAAmC,GAAG,EAAE,GAAG,IAAI,CAAA;AAErD;;;;GAIG;AACH,MAAM,UAAU,kCAAkC,CAChD,SAA0B,EAC1B,KAAY,EACZ,GAAW;AACX,sFAAsF;AACtF,MAA0B;IAE1B,IAAI,CAAC,SAAS,CAAC,aAAa;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IAC7C,OAAO,YAAY,CAAC;QAClB,IAAI,EAAE,yBAAyB;QAC/B,UAAU,EAAE,mCAAmC;QAC/C,GAAG;QACH,MAAM;QACN,cAAc,EAAE,gCAAgC;QAChD,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,SAAS,GAAG,MAAM,0BAA0B,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAA;YAC1E,IAAI,SAAS,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QACvE,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
@@ -1,10 +1,12 @@
1
1
  import type { Clock } from '@cat-factory/kernel';
2
- import { type Logger, type ServerContainer } from '@cat-factory/server';
2
+ import { type Logger, type ServerContainer, type SweepHealthTracker } from '@cat-factory/server';
3
3
  /**
4
4
  * Start the periodic platform-health alert sweep. Runs once immediately then on the interval,
5
5
  * non-overlapping + best-effort (see {@link startSweeper}). A NO-OP (returns a no-op stop)
6
6
  * unless alerting is opted in — so a deployment that hasn't set `PLATFORM_ALERTS` pays nothing.
7
7
  * Returns a stop function that clears the timer.
8
8
  */
9
- export declare function startPlatformHealthSweeper(container: ServerContainer, clock: Clock, log: Logger): () => void;
9
+ export declare function startPlatformHealthSweeper(container: ServerContainer, clock: Clock, log: Logger,
10
+ /** Records each pass's outcome under this sweep's name (see {@link startSweeper}). */
11
+ health: SweepHealthTracker): () => void;
10
12
  //# sourceMappingURL=platformHealth.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"platformHealth.d.ts","sourceRoot":"","sources":["../src/platformHealth.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,eAAe,EAAuB,MAAM,qBAAqB,CAAA;AAW5F;;;;;GAKG;AACH,wBAAgB,0BAA0B,CACxC,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,MAAM,GACV,MAAM,IAAI,CAaZ"}
1
+ {"version":3,"file":"platformHealth.d.ts","sourceRoot":"","sources":["../src/platformHealth.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,EACL,KAAK,MAAM,EACX,KAAK,eAAe,EACpB,KAAK,kBAAkB,EAExB,MAAM,qBAAqB,CAAA;AAW5B;;;;;GAKG;AACH,wBAAgB,0BAA0B,CACxC,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,MAAM;AACX,sFAAsF;AACtF,MAAM,EAAE,kBAAkB,GACzB,MAAM,IAAI,CAcZ"}
@@ -1,4 +1,4 @@
1
- import { sweepPlatformHealth } from '@cat-factory/server';
1
+ import { sweepPlatformHealth, } from '@cat-factory/server';
2
2
  import { startSweeper } from './sweeper.js';
3
3
  // Periodic platform-health ALERT sweep for the Node facade — the analogue of the Worker's
4
4
  // every-2-min cron call to `sweepPlatformHealth`. It evaluates each account's aggregate
@@ -13,7 +13,9 @@ import { startSweeper } from './sweeper.js';
13
13
  * unless alerting is opted in — so a deployment that hasn't set `PLATFORM_ALERTS` pays nothing.
14
14
  * Returns a stop function that clears the timer.
15
15
  */
16
- export function startPlatformHealthSweeper(container, clock, log) {
16
+ export function startPlatformHealthSweeper(container, clock, log,
17
+ /** Records each pass's outcome under this sweep's name (see {@link startSweeper}). */
18
+ health) {
17
19
  const cfg = container.config.platformAlerts;
18
20
  if (!cfg.enabled)
19
21
  return () => { };
@@ -21,6 +23,7 @@ export function startPlatformHealthSweeper(container, clock, log) {
21
23
  name: 'platform-health',
22
24
  intervalMs: cfg.intervalMs,
23
25
  log,
26
+ health,
24
27
  failureMessage: 'platform health sweep failed',
25
28
  tick: async () => {
26
29
  const { raised, cleared } = await sweepPlatformHealth(container, log);
@@ -1 +1 @@
1
- {"version":3,"file":"platformHealth.js","sourceRoot":"","sources":["../src/platformHealth.ts"],"names":[],"mappings":"AACA,OAAO,EAAqC,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AAC5F,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,0FAA0F;AAC1F,wFAAwF;AACxF,gGAAgG;AAChG,kFAAkF;AAClF,+FAA+F;AAC/F,4FAA4F;AAC5F,sFAAsF;AAEtF;;;;;GAKG;AACH,MAAM,UAAU,0BAA0B,CACxC,SAA0B,EAC1B,KAAY,EACZ,GAAW;IAEX,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,cAAc,CAAA;IAC3C,IAAI,CAAC,GAAG,CAAC,OAAO;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IACjC,OAAO,YAAY,CAAC;QAClB,IAAI,EAAE,iBAAiB;QACvB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,GAAG;QACH,cAAc,EAAE,8BAA8B;QAC9C,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;YACrE,IAAI,MAAM,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;QACvF,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
1
+ {"version":3,"file":"platformHealth.js","sourceRoot":"","sources":["../src/platformHealth.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,mBAAmB,GACpB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,0FAA0F;AAC1F,wFAAwF;AACxF,gGAAgG;AAChG,kFAAkF;AAClF,+FAA+F;AAC/F,4FAA4F;AAC5F,sFAAsF;AAEtF;;;;;GAKG;AACH,MAAM,UAAU,0BAA0B,CACxC,SAA0B,EAC1B,KAAY,EACZ,GAAW;AACX,sFAAsF;AACtF,MAA0B;IAE1B,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,cAAc,CAAA;IAC3C,IAAI,CAAC,GAAG,CAAC,OAAO;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IACjC,OAAO,YAAY,CAAC;QAClB,IAAI,EAAE,iBAAiB;QACvB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,GAAG;QACH,MAAM;QACN,cAAc,EAAE,8BAA8B;QAC9C,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;YACrE,IAAI,MAAM,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;QACvF,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
@@ -1,10 +1,16 @@
1
- import type { Clock, WorkspaceRepository } from '@cat-factory/kernel';
1
+ import type { Clock, OperationalGaugeSample, OperationalMetricsCollector, WorkspaceRepository } from '@cat-factory/kernel';
2
2
  import { type PlatformObservabilityService } from '@cat-factory/orchestration';
3
- import type { Logger, OtelConfig } from '@cat-factory/server';
3
+ import type { Logger, OtelConfig, SweepHealthTracker } from '@cat-factory/server';
4
4
  export interface PlatformMetricsSweeperDeps {
5
5
  otel: OtelConfig;
6
6
  platformObservability: PlatformObservabilityService;
7
7
  workspaceRepository: Pick<WorkspaceRepository, 'listVisible'>;
8
+ /**
9
+ * Reads every pg-boss queue's depth in ONE call. Omitted ⇒ no queue gauge is emitted at
10
+ * all, which is the honest answer for a deployment with no durable substrate (mothership
11
+ * mode) — a zero there would claim an empty queue where there is no queue.
12
+ */
13
+ probeQueueDepth?: () => Promise<OperationalGaugeSample[]>;
8
14
  }
9
15
  /**
10
16
  * Start the periodic platform-metrics push. A NO-OP (returns a no-op stop) unless the base
@@ -13,5 +19,13 @@ export interface PlatformMetricsSweeperDeps {
13
19
  * interval; best-effort per account (a failed summarize/export is logged, never thrown).
14
20
  * Returns a stop function to halt the job on shutdown.
15
21
  */
16
- export declare function startPlatformMetricsSweeper(deps: PlatformMetricsSweeperDeps, clock: Clock, log: Logger): () => void;
22
+ export declare function startPlatformMetricsSweeper(deps: PlatformMetricsSweeperDeps, clock: Clock, log: Logger,
23
+ /**
24
+ * The process-wide collector, DRAINED by this sweep. Node holds ONE for the life of the
25
+ * process, so draining on this interval loses nothing; the Worker cannot do the same and
26
+ * flushes per invocation.
27
+ */
28
+ metrics: OperationalMetricsCollector,
29
+ /** Records each pass's outcome under this sweep's name (see {@link startSweeper}). */
30
+ health: SweepHealthTracker): () => void;
17
31
  //# sourceMappingURL=platformMetrics.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"platformMetrics.d.ts","sourceRoot":"","sources":["../src/platformMetrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AACrE,OAAO,EACL,KAAK,4BAA4B,EAGlC,MAAM,4BAA4B,CAAA;AAEnC,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAU7D,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,UAAU,CAAA;IAChB,qBAAqB,EAAE,4BAA4B,CAAA;IACnD,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAA;CAC9D;AAED;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,0BAA0B,EAChC,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,MAAM,GACV,MAAM,IAAI,CA2BZ"}
1
+ {"version":3,"file":"platformMetrics.d.ts","sourceRoot":"","sources":["../src/platformMetrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,KAAK,EACL,sBAAsB,EACtB,2BAA2B,EAC3B,mBAAmB,EACpB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,KAAK,4BAA4B,EAIlC,MAAM,4BAA4B,CAAA;AAEnC,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AAUjF,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,UAAU,CAAA;IAChB,qBAAqB,EAAE,4BAA4B,CAAA;IACnD,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAA;IAC7D;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAAA;CAC1D;AAED;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,0BAA0B,EAChC,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,MAAM;AACX;;;;GAIG;AACH,OAAO,EAAE,2BAA2B;AACpC,sFAAsF;AACtF,MAAM,EAAE,kBAAkB,GACzB,MAAM,IAAI,CAsCZ"}
@@ -1,4 +1,4 @@
1
- import { distinctAccountIds, sweepPlatformMetrics, } from '@cat-factory/orchestration';
1
+ import { distinctAccountIds, flushOperationalMetrics, sweepPlatformMetrics, } from '@cat-factory/orchestration';
2
2
  import { createPlatformMetricsOtelExporter } from '@cat-factory/observability-otel';
3
3
  import { startSweeper } from './sweeper.js';
4
4
  /**
@@ -8,8 +8,16 @@ import { startSweeper } from './sweeper.js';
8
8
  * interval; best-effort per account (a failed summarize/export is logged, never thrown).
9
9
  * Returns a stop function to halt the job on shutdown.
10
10
  */
11
- export function startPlatformMetricsSweeper(deps, clock, log) {
12
- const { otel, platformObservability, workspaceRepository } = deps;
11
+ export function startPlatformMetricsSweeper(deps, clock, log,
12
+ /**
13
+ * The process-wide collector, DRAINED by this sweep. Node holds ONE for the life of the
14
+ * process, so draining on this interval loses nothing; the Worker cannot do the same and
15
+ * flushes per invocation.
16
+ */
17
+ metrics,
18
+ /** Records each pass's outcome under this sweep's name (see {@link startSweeper}). */
19
+ health) {
20
+ const { otel, platformObservability, workspaceRepository, probeQueueDepth } = deps;
13
21
  if (!otel.platformMetrics.enabled || !otel.endpoint)
14
22
  return () => { };
15
23
  const exporter = createPlatformMetricsOtelExporter({
@@ -22,6 +30,7 @@ export function startPlatformMetricsSweeper(deps, clock, log) {
22
30
  name: 'platform-metrics',
23
31
  intervalMs: otel.platformMetrics.intervalMs,
24
32
  log,
33
+ health,
25
34
  failureMessage: 'platform metrics sweep failed',
26
35
  tick: async () => {
27
36
  const exported = await sweepPlatformMetrics({
@@ -33,6 +42,17 @@ export function startPlatformMetricsSweeper(deps, clock, log) {
33
42
  });
34
43
  if (exported > 0)
35
44
  log.info('exported platform metrics', { exported });
45
+ // The OPERATIONAL half, on the same tick and through the same exporter: the counters
46
+ // this process accumulated since the last flush, plus a live queue-depth reading.
47
+ const flushed = await flushOperationalMetrics({
48
+ collector: metrics,
49
+ sink: exporter,
50
+ ...(probeQueueDepth ? { probeGauges: probeQueueDepth } : {}),
51
+ now: clock.now(),
52
+ logger: log,
53
+ });
54
+ if (flushed > 0)
55
+ log.debug('exported operational metrics', { samples: flushed });
36
56
  },
37
57
  });
38
58
  }
@@ -1 +1 @@
1
- {"version":3,"file":"platformMetrics.js","sourceRoot":"","sources":["../src/platformMetrics.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,4BAA4B,CAAA;AACnC,OAAO,EAAE,iCAAiC,EAAE,MAAM,iCAAiC,CAAA;AAEnF,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAe3C;;;;;;GAMG;AACH,MAAM,UAAU,2BAA2B,CACzC,IAAgC,EAChC,KAAY,EACZ,GAAW;IAEX,MAAM,EAAE,IAAI,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,GAAG,IAAI,CAAA;IACjE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IAEpE,MAAM,QAAQ,GAAG,iCAAiC,CAAC;QACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,MAAM,EAAE,GAAG;KACZ,CAAC,CAAA;IAEF,OAAO,YAAY,CAAC;QAClB,IAAI,EAAE,kBAAkB;QACxB,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,UAAU;QAC3C,GAAG;QACH,cAAc,EAAE,+BAA+B;QAC/C,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC;gBAC1C,cAAc,EAAE,KAAK,IAAI,EAAE,CAAC,kBAAkB,CAAC,MAAM,mBAAmB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC3F,SAAS,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC,qBAAqB,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC;gBACpF,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACnC,MAAM,EAAE,GAAG;aACZ,CAAC,CAAA;YACF,IAAI,QAAQ,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAA;QACvE,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
1
+ {"version":3,"file":"platformMetrics.js","sourceRoot":"","sources":["../src/platformMetrics.ts"],"names":[],"mappings":"AAMA,OAAO,EAEL,kBAAkB,EAClB,uBAAuB,EACvB,oBAAoB,GACrB,MAAM,4BAA4B,CAAA;AACnC,OAAO,EAAE,iCAAiC,EAAE,MAAM,iCAAiC,CAAA;AAEnF,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAqB3C;;;;;;GAMG;AACH,MAAM,UAAU,2BAA2B,CACzC,IAAgC,EAChC,KAAY,EACZ,GAAW;AACX;;;;GAIG;AACH,OAAoC;AACpC,sFAAsF;AACtF,MAA0B;IAE1B,MAAM,EAAE,IAAI,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,eAAe,EAAE,GAAG,IAAI,CAAA;IAClF,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IAEpE,MAAM,QAAQ,GAAG,iCAAiC,CAAC;QACjD,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,MAAM,EAAE,GAAG;KACZ,CAAC,CAAA;IAEF,OAAO,YAAY,CAAC;QAClB,IAAI,EAAE,kBAAkB;QACxB,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,UAAU;QAC3C,GAAG;QACH,MAAM;QACN,cAAc,EAAE,+BAA+B;QAC/C,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC;gBAC1C,cAAc,EAAE,KAAK,IAAI,EAAE,CAAC,kBAAkB,CAAC,MAAM,mBAAmB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC3F,SAAS,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC,qBAAqB,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC;gBACpF,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACnC,MAAM,EAAE,GAAG;aACZ,CAAC,CAAA;YACF,IAAI,QAAQ,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAA;YACrE,qFAAqF;YACrF,kFAAkF;YAClF,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC;gBAC5C,SAAS,EAAE,OAAO;gBAClB,IAAI,EAAE,QAAQ;gBACd,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5D,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE;gBAChB,MAAM,EAAE,GAAG;aACZ,CAAC,CAAA;YACF,IAAI,OAAO,GAAG,CAAC;gBAAE,GAAG,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAA;QAClF,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
@@ -0,0 +1,38 @@
1
+ import type { OperationalGaugeSample } from '@cat-factory/kernel';
2
+ import type { PgBoss } from 'pg-boss';
3
+ /**
4
+ * Read every pg-boss queue's depth as gauge samples. The QUEUE NAME is a bounded dimension
5
+ * (the facade creates a fixed set at boot), so it is safe on a metric series and is the split
6
+ * that matters — "the backlog is on `execution.advance`" and "the backlog is on `github.sync`"
7
+ * are different incidents.
8
+ *
9
+ * Dead-letter queues are reported under their own `state` rather than being folded into the
10
+ * live counts: a job sitting in a DLQ is not backlog waiting to drain, it is work that was
11
+ * GIVEN UP ON, and averaging the two hides exactly the one an operator must act on.
12
+ */
13
+ export declare function probePgBossQueueDepth(boss: PgBoss): Promise<OperationalGaugeSample[]>;
14
+ /** How often the dead-letter sweep looks. Hourly: a dead-lettered job is not time-critical. */
15
+ export declare const DEAD_LETTER_SWEEP_INTERVAL_MS: number;
16
+ /**
17
+ * Report what is sitting in the dead-letter queues. pg-boss moves a job here once it has
18
+ * exhausted `retryLimit`, and NOTHING reads these tables — before this, a webhook delivery or
19
+ * a run start that could never succeed simply stopped existing, with its last failure buried
20
+ * in whichever log line happened to catch it.
21
+ *
22
+ * Deliberately a REPORT, not a re-drive: a job that failed every retry will fail again, and an
23
+ * automatic replay would turn a bounded loss into an unbounded loop. What it produces is a log
24
+ * line naming the queue, so the decision to replay stays a human one.
25
+ *
26
+ * It deliberately emits NO metric of its own. The depth is already carried by
27
+ * `queue.depth{state=dead_letter}` above, and that is the honest shape: `totalCount` is a
28
+ * standing level, so feeding it to a delta counter here re-reported the same jobs on every
29
+ * hourly pass. This sweep exists for the LOG LINE — the metric says how many, the line says
30
+ * which source queue to go and look at.
31
+ *
32
+ * Returns the total across all DLQs (0 when there are none), so a caller can log only when
33
+ * there is something to say.
34
+ */
35
+ export declare function sweepDeadLetterQueues(boss: PgBoss, log: {
36
+ warn(msg: string, fields?: Record<string, unknown>): void;
37
+ }): Promise<number>;
38
+ //# sourceMappingURL=queueDepth.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"queueDepth.d.ts","sourceRoot":"","sources":["../src/queueDepth.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAA;AACjE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AA8BrC;;;;;;;;;GASG;AACH,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAkB3F;AAED,+FAA+F;AAC/F,eAAO,MAAM,6BAA6B,QAAiB,CAAA;AAE3D;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE;IAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;CAAE,GACjE,OAAO,CAAC,MAAM,CAAC,CAejB"}
@@ -0,0 +1,69 @@
1
+ import { DEAD_LETTER_SUFFIX, isDeadLetterQueue } from './execution/deadLetter.js';
2
+ /** One `queue.depth` sample, with the `state` dimension pinned to {@link QueueDepthState}. */
3
+ function depthSample(queue, state, value) {
4
+ return { gauge: 'queue.depth', dimensions: { queue, state }, value };
5
+ }
6
+ /**
7
+ * Read every pg-boss queue's depth as gauge samples. The QUEUE NAME is a bounded dimension
8
+ * (the facade creates a fixed set at boot), so it is safe on a metric series and is the split
9
+ * that matters — "the backlog is on `execution.advance`" and "the backlog is on `github.sync`"
10
+ * are different incidents.
11
+ *
12
+ * Dead-letter queues are reported under their own `state` rather than being folded into the
13
+ * live counts: a job sitting in a DLQ is not backlog waiting to drain, it is work that was
14
+ * GIVEN UP ON, and averaging the two hides exactly the one an operator must act on.
15
+ */
16
+ export async function probePgBossQueueDepth(boss) {
17
+ const queues = await boss.getQueues();
18
+ const samples = [];
19
+ for (const queue of queues) {
20
+ if (isDeadLetterQueue(queue.name)) {
21
+ // A DLQ's whole contents are the signal — every job in it exhausted its retries. This is
22
+ // also the ONLY series that reports dead-lettering: it is a level, and a level belongs on
23
+ // a gauge (see the note on `OperationalCounter` for why the counter it replaced lied).
24
+ samples.push(depthSample(queue.name, 'dead_letter', queue.totalCount));
25
+ continue;
26
+ }
27
+ samples.push(depthSample(queue.name, 'ready', queue.readyCount), depthSample(queue.name, 'active', queue.activeCount), depthSample(queue.name, 'failed', queue.failedCount));
28
+ }
29
+ return samples;
30
+ }
31
+ /** How often the dead-letter sweep looks. Hourly: a dead-lettered job is not time-critical. */
32
+ export const DEAD_LETTER_SWEEP_INTERVAL_MS = 60 * 60 * 1000;
33
+ /**
34
+ * Report what is sitting in the dead-letter queues. pg-boss moves a job here once it has
35
+ * exhausted `retryLimit`, and NOTHING reads these tables — before this, a webhook delivery or
36
+ * a run start that could never succeed simply stopped existing, with its last failure buried
37
+ * in whichever log line happened to catch it.
38
+ *
39
+ * Deliberately a REPORT, not a re-drive: a job that failed every retry will fail again, and an
40
+ * automatic replay would turn a bounded loss into an unbounded loop. What it produces is a log
41
+ * line naming the queue, so the decision to replay stays a human one.
42
+ *
43
+ * It deliberately emits NO metric of its own. The depth is already carried by
44
+ * `queue.depth{state=dead_letter}` above, and that is the honest shape: `totalCount` is a
45
+ * standing level, so feeding it to a delta counter here re-reported the same jobs on every
46
+ * hourly pass. This sweep exists for the LOG LINE — the metric says how many, the line says
47
+ * which source queue to go and look at.
48
+ *
49
+ * Returns the total across all DLQs (0 when there are none), so a caller can log only when
50
+ * there is something to say.
51
+ */
52
+ export async function sweepDeadLetterQueues(boss, log) {
53
+ const queues = await boss.getQueues();
54
+ let total = 0;
55
+ for (const queue of queues) {
56
+ if (!isDeadLetterQueue(queue.name) || queue.totalCount === 0)
57
+ continue;
58
+ total += queue.totalCount;
59
+ log.warn('jobs are sitting in a dead-letter queue', {
60
+ scope: 'dead-letter',
61
+ queue: queue.name,
62
+ // The SOURCE queue is what an operator needs to look at; the DLQ name is derived from it.
63
+ sourceQueue: queue.name.slice(0, -DEAD_LETTER_SUFFIX.length),
64
+ jobs: queue.totalCount,
65
+ });
66
+ }
67
+ return total;
68
+ }
69
+ //# sourceMappingURL=queueDepth.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"queueDepth.js","sourceRoot":"","sources":["../src/queueDepth.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAA;AAwBjF,8FAA8F;AAC9F,SAAS,WAAW,CAAC,KAAa,EAAE,KAAsB,EAAE,KAAa;IACvE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,CAAA;AACtE,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAAY;IACtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAA;IACrC,MAAM,OAAO,GAA6B,EAAE,CAAA;IAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,yFAAyF;YACzF,0FAA0F;YAC1F,uFAAuF;YACvF,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,aAAa,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAA;YACtE,SAAQ;QACV,CAAC;QACD,OAAO,CAAC,IAAI,CACV,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,EAClD,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,EACpD,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,CACrD,CAAA;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,+FAA+F;AAC/F,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAE3D;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAY,EACZ,GAAkE;IAElE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAA;IACrC,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC;YAAE,SAAQ;QACtE,KAAK,IAAI,KAAK,CAAC,UAAU,CAAA;QACzB,GAAG,CAAC,IAAI,CAAC,yCAAyC,EAAE;YAClD,KAAK,EAAE,aAAa;YACpB,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,0FAA0F;YAC1F,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,MAAM,CAAC;YAC5D,IAAI,EAAE,KAAK,CAAC,UAAU;SACvB,CAAC,CAAA;IACJ,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC"}
@@ -18,6 +18,22 @@ export interface ReadinessProbeDeps {
18
18
  ping: () => Promise<void>;
19
19
  /** Whether pg-boss is started and has not emitted `stopped` (a flag the boot sequence owns). */
20
20
  pgBossHealthy: () => boolean;
21
+ /**
22
+ * Round-trips PG-BOSS's OWN connection (a metadata read through its pool) — resolves on
23
+ * success, throws on failure. The flag above only observes the GRACEFUL `stopped`
24
+ * transition, so a boss whose connection died without emitting it read healthy forever; this
25
+ * is what turns that into a real check, and it probes a pool the app's own `ping` does not
26
+ * touch (pg-boss keeps its own).
27
+ *
28
+ * What it still does NOT prove, stated rather than implied: that the worker LOOP is
29
+ * consuming. A boss whose connection is fine but whose workers have stopped picking up jobs
30
+ * passes this. That failure is visible as unbounded queue depth, which is why slice 4.1's
31
+ * `queue.depth` gauge is the signal for it — draining a replica on it would be wrong anyway,
32
+ * since the replica still serves HTTP perfectly well.
33
+ *
34
+ * Omitted ⇒ only the flag is consulted (an embedded/test wiring with no boss to probe).
35
+ */
36
+ pgBossPing?: () => Promise<void>;
21
37
  /** True once graceful shutdown has begun, so the probe drains immediately. Optional (default: not draining). */
22
38
  isDraining?: () => boolean;
23
39
  /** Bounds the DB probe so a wedged pool can't hang the health check. Default 2000ms. */
@@ -1 +1 @@
1
- {"version":3,"file":"readiness.d.ts","sourceRoot":"","sources":["../src/readiness.ts"],"names":[],"mappings":"AAiBA,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,OAAO,CAAA;IACX;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,OAAO,CAAA;IACd,kGAAkG;IAClG,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;CACvC;AAED,MAAM,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,CAAA;AAE3D,MAAM,WAAW,kBAAkB;IACjC,wGAAwG;IACxG,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IACzB,gGAAgG;IAChG,aAAa,EAAE,MAAM,OAAO,CAAA;IAC5B,gHAAgH;IAChH,UAAU,CAAC,EAAE,MAAM,OAAO,CAAA;IAC1B,wFAAwF;IACxF,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AA4BD;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,eAAe,CAAC,CAavF;AAED,uGAAuG;AACvG,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,kBAAkB,GAAG,cAAc,CAE3E"}
1
+ {"version":3,"file":"readiness.d.ts","sourceRoot":"","sources":["../src/readiness.ts"],"names":[],"mappings":"AA8BA,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,OAAO,CAAA;IACX;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,OAAO,CAAA;IACd,kGAAkG;IAClG,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;CACvC;AAED,MAAM,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,CAAA;AAE3D,MAAM,WAAW,kBAAkB;IACjC,wGAAwG;IACxG,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IACzB,gGAAgG;IAChG,aAAa,EAAE,MAAM,OAAO,CAAA;IAC5B;;;;;;;;;;;;;;OAcG;IACH,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAChC,gHAAgH;IAChH,UAAU,CAAC,EAAE,MAAM,OAAO,CAAA;IAC1B,wFAAwF;IACxF,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AA4BD;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,eAAe,CAAC,CAyBvF;AAED,uGAAuG;AACvG,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,kBAAkB,GAAG,cAAc,CAE3E"}
package/dist/readiness.js CHANGED
@@ -14,6 +14,19 @@
14
14
  // request is a fresh isolate), so it has no readiness concept. Local mothership mode has no local
15
15
  // Postgres/pg-boss either (org state is served remotely), so it wires no probe and `/ready` simply
16
16
  // mirrors `/health`.
17
+ //
18
+ // TWO dependencies are deliberately NOT probed here, and both decisions are the point rather
19
+ // than an omission:
20
+ //
21
+ // - REDIS. Probing it per request would mean either opening a connection each time (worse
22
+ // than the gap it closes) or widening the propagator's adapter seam to expose a client's
23
+ // internal state. And the verdict would be wrong either way: a dead Redis degrades
24
+ // cross-node real-time, which a replica should keep serving HTTP through, not drain on.
25
+ // Its boot probe (`redisProbe.ts`) and the propagator's own per-publish warnings own it.
26
+ // - THE TELEMETRY STORE. On this facade it is a SCHEMA in the same Postgres database, so the
27
+ // `SELECT 1` above already covers reaching it; a second query would test the same
28
+ // connection and report a second, correlated opinion of it. (It is a physically separate
29
+ // D1 database only on Cloudflare, which has no `/ready` at all — see below.)
17
30
  function message(err) {
18
31
  return err instanceof Error ? err.message : String(err);
19
32
  }
@@ -50,16 +63,28 @@ export async function checkReadiness(deps) {
50
63
  if (deps.isDraining?.()) {
51
64
  return { ready: false, checks: { shutdown: { ok: false, error: 'draining' } } };
52
65
  }
66
+ const timeoutMs = deps.timeoutMs ?? 2_000;
67
+ const probe = async (run) => {
68
+ try {
69
+ await withTimeout(run(), timeoutMs);
70
+ return { ok: true };
71
+ }
72
+ catch (err) {
73
+ return { ok: false, error: message(err) };
74
+ }
75
+ };
53
76
  const checks = {};
54
- try {
55
- await withTimeout(deps.ping(), deps.timeoutMs ?? 2_000);
56
- checks.database = { ok: true };
57
- }
58
- catch (err) {
59
- checks.database = { ok: false, error: message(err) };
60
- }
61
- checks.pgBoss = deps.pgBossHealthy() ? { ok: true } : { ok: false, error: 'pg-boss not running' };
62
- return { ready: checks.database.ok && checks.pgBoss.ok, checks };
77
+ checks.database = await probe(deps.ping);
78
+ // pg-boss: the graceful-stop flag AND a real round-trip through its own connection. The flag
79
+ // alone was the whole check, and it is only ever flipped by a clean `stopped` — so the one
80
+ // failure a readiness probe exists to catch (the substrate died under a running process)
81
+ // reported healthy. Ordered flag-first because it is free and definitive when it says no.
82
+ checks.pgBoss = !deps.pgBossHealthy()
83
+ ? { ok: false, error: 'pg-boss not running' }
84
+ : deps.pgBossPing
85
+ ? await probe(deps.pgBossPing)
86
+ : { ok: true };
87
+ return { ready: Object.values(checks).every((check) => check.ok), checks };
63
88
  }
64
89
  /** Bind {@link checkReadiness} to a fixed set of probes — the shape `createApp` mounts on `/ready`. */
65
90
  export function makeReadinessProbe(deps) {
@@ -1 +1 @@
1
- {"version":3,"file":"readiness.js","sourceRoot":"","sources":["../src/readiness.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAC7D,EAAE;AACF,6FAA6F;AAC7F,6FAA6F;AAC7F,6FAA6F;AAC7F,EAAE;AACF,gGAAgG;AAChG,iGAAiG;AACjG,8FAA8F;AAC9F,8FAA8F;AAC9F,iDAAiD;AACjD,EAAE;AACF,iGAAiG;AACjG,kGAAkG;AAClG,mGAAmG;AACnG,qBAAqB;AA+BrB,SAAS,OAAO,CAAC,GAAY;IAC3B,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;AACzD,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,WAAW,CAAC,OAAsB,EAAE,EAAU;IAC3D,IAAI,KAAgD,CAAA;IACpD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;QACrD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC1E,+FAA+F;QAC/F,KAAK,CAAC,KAAK,EAAE,EAAE,CAAA;IACjB,CAAC,CAAC,CAAA;IACF,kGAAkG;IAClG,iGAAiG;IACjG,iGAAiG;IACjG,gGAAgG;IAChG,2FAA2F;IAC3F,2FAA2F;IAC3F,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IACvB,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAA;IACxC,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAA;IAChC,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAwB;IAC3D,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE,EAAE,CAAC;QACxB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAA;IACjF,CAAC;IACD,MAAM,MAAM,GAAmC,EAAE,CAAA;IACjD,IAAI,CAAC;QACH,MAAM,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,CAAA;QACvD,MAAM,CAAC,QAAQ,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,CAAA;IAChC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,QAAQ,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAA;IACtD,CAAC;IACD,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAA;IACjG,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAA;AAClE,CAAC;AAED,uGAAuG;AACvG,MAAM,UAAU,kBAAkB,CAAC,IAAwB;IACzD,OAAO,GAAG,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;AACnC,CAAC"}
1
+ {"version":3,"file":"readiness.js","sourceRoot":"","sources":["../src/readiness.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAC7D,EAAE;AACF,6FAA6F;AAC7F,6FAA6F;AAC7F,6FAA6F;AAC7F,EAAE;AACF,gGAAgG;AAChG,iGAAiG;AACjG,8FAA8F;AAC9F,8FAA8F;AAC9F,iDAAiD;AACjD,EAAE;AACF,iGAAiG;AACjG,kGAAkG;AAClG,mGAAmG;AACnG,qBAAqB;AACrB,EAAE;AACF,6FAA6F;AAC7F,oBAAoB;AACpB,EAAE;AACF,4FAA4F;AAC5F,6FAA6F;AAC7F,uFAAuF;AACvF,4FAA4F;AAC5F,6FAA6F;AAC7F,+FAA+F;AAC/F,sFAAsF;AACtF,6FAA6F;AAC7F,iFAAiF;AA+CjF,SAAS,OAAO,CAAC,GAAY;IAC3B,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;AACzD,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,WAAW,CAAC,OAAsB,EAAE,EAAU;IAC3D,IAAI,KAAgD,CAAA;IACpD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;QACrD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC1E,+FAA+F;QAC/F,KAAK,CAAC,KAAK,EAAE,EAAE,CAAA;IACjB,CAAC,CAAC,CAAA;IACF,kGAAkG;IAClG,iGAAiG;IACjG,iGAAiG;IACjG,gGAAgG;IAChG,2FAA2F;IAC3F,2FAA2F;IAC3F,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IACvB,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAA;IACxC,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAA;IAChC,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAwB;IAC3D,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE,EAAE,CAAC;QACxB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAA;IACjF,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAA;IACzC,MAAM,KAAK,GAAG,KAAK,EAAE,GAAwB,EAA2B,EAAE;QACxE,IAAI,CAAC;YACH,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,SAAS,CAAC,CAAA;YACnC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAA;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAA;QAC3C,CAAC;IACH,CAAC,CAAA;IACD,MAAM,MAAM,GAAmC,EAAE,CAAA;IACjD,MAAM,CAAC,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxC,6FAA6F;IAC7F,2FAA2F;IAC3F,yFAAyF;IACzF,0FAA0F;IAC1F,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE;QACnC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,qBAAqB,EAAE;QAC7C,CAAC,CAAC,IAAI,CAAC,UAAU;YACf,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;YAC9B,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAA;IAClB,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAA;AAC5E,CAAC;AAED,uGAAuG;AACvG,MAAM,UAAU,kBAAkB,CAAC,IAAwB;IACzD,OAAO,GAAG,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;AACnC,CAAC"}
@@ -1,9 +1,11 @@
1
1
  import type { Clock } from '@cat-factory/kernel';
2
- import type { Logger, ServerContainer } from '@cat-factory/server';
2
+ import type { Logger, ServerContainer, SweepHealthTracker } from '@cat-factory/server';
3
3
  /**
4
4
  * Start the periodic schedule sweep. Runs once immediately then on the interval,
5
5
  * non-overlapping + best-effort (see {@link startSweeper}). Returns a stop function that
6
6
  * clears the timer.
7
7
  */
8
- export declare function startScheduleSweeper(container: ServerContainer, clock: Clock, log: Logger): () => void;
8
+ export declare function startScheduleSweeper(container: ServerContainer, clock: Clock, log: Logger,
9
+ /** Records each pass's outcome under this sweep's name (see {@link startSweeper}). */
10
+ health: SweepHealthTracker): () => void;
9
11
  //# sourceMappingURL=recurring.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"recurring.d.ts","sourceRoot":"","sources":["../src/recurring.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AAYlE;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,MAAM,GACV,MAAM,IAAI,CAeZ"}
1
+ {"version":3,"file":"recurring.d.ts","sourceRoot":"","sources":["../src/recurring.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AAYtF;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,MAAM;AACX,sFAAsF;AACtF,MAAM,EAAE,kBAAkB,GACzB,MAAM,IAAI,CAgBZ"}
package/dist/recurring.js CHANGED
@@ -11,7 +11,9 @@ const SCHEDULE_SWEEP_INTERVAL_MS = 60 * 1000;
11
11
  * non-overlapping + best-effort (see {@link startSweeper}). Returns a stop function that
12
12
  * clears the timer.
13
13
  */
14
- export function startScheduleSweeper(container, clock, log) {
14
+ export function startScheduleSweeper(container, clock, log,
15
+ /** Records each pass's outcome under this sweep's name (see {@link startSweeper}). */
16
+ health) {
15
17
  const recurring = container.recurring;
16
18
  if (!recurring)
17
19
  return () => { };
@@ -19,6 +21,7 @@ export function startScheduleSweeper(container, clock, log) {
19
21
  name: 'recurring-pipelines',
20
22
  intervalMs: SCHEDULE_SWEEP_INTERVAL_MS,
21
23
  log,
24
+ health,
22
25
  failureMessage: 'recurring-pipeline sweep failed',
23
26
  tick: async () => {
24
27
  const { fired, skipped } = await recurring.service.runDue(clock.now());
@@ -1 +1 @@
1
- {"version":3,"file":"recurring.js","sourceRoot":"","sources":["../src/recurring.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,8EAA8E;AAC9E,iFAAiF;AACjF,oFAAoF;AACpF,mFAAmF;AACnF,eAAe;AAEf,2DAA2D;AAC3D,MAAM,0BAA0B,GAAG,EAAE,GAAG,IAAI,CAAA;AAE5C;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAClC,SAA0B,EAC1B,KAAY,EACZ,GAAW;IAEX,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAA;IACrC,IAAI,CAAC,SAAS;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IAC/B,OAAO,YAAY,CAAC;QAClB,IAAI,EAAE,qBAAqB;QAC3B,UAAU,EAAE,0BAA0B;QACtC,GAAG;QACH,cAAc,EAAE,iCAAiC;QACjD,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAA;YACtE,IAAI,KAAK,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAC7B,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;YAC3D,CAAC;QACH,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
1
+ {"version":3,"file":"recurring.js","sourceRoot":"","sources":["../src/recurring.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,8EAA8E;AAC9E,iFAAiF;AACjF,oFAAoF;AACpF,mFAAmF;AACnF,eAAe;AAEf,2DAA2D;AAC3D,MAAM,0BAA0B,GAAG,EAAE,GAAG,IAAI,CAAA;AAE5C;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAClC,SAA0B,EAC1B,KAAY,EACZ,GAAW;AACX,sFAAsF;AACtF,MAA0B;IAE1B,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAA;IACrC,IAAI,CAAC,SAAS;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IAC/B,OAAO,YAAY,CAAC;QAClB,IAAI,EAAE,qBAAqB;QAC3B,UAAU,EAAE,0BAA0B;QACtC,GAAG;QACH,MAAM;QACN,cAAc,EAAE,iCAAiC;QACjD,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAA;YACtE,IAAI,KAAK,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAC7B,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;YAC3D,CAAC;QACH,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
@@ -58,6 +58,7 @@ export declare class DrizzleAgentRunRepository implements AgentRunRepository {
58
58
  listStale(olderThanEpochMs: number): Promise<StaleAgentRun[]>;
59
59
  listPausedExecutions(): Promise<AgentRunRef[]>;
60
60
  liveRunIds(ids: string[]): Promise<string[]>;
61
+ recordRedrive(workspaceId: string, id: string): Promise<number>;
61
62
  }
62
63
  /**
63
64
  * Deployment-level rollups over `agent_runs`, scoped to an account by a sub-select on
@@ -1 +1 @@
1
- {"version":3,"file":"execution.d.ts","sourceRoot":"","sources":["../../../src/repositories/drizzle/execution.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,YAAY,EACZ,WAAW,EACX,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,QAAQ,EACR,kBAAkB,EAClB,gBAAgB,EAChB,0BAA0B,EAC1B,qBAAqB,EACrB,oBAAoB,EACpB,kBAAkB,EAClB,yBAAyB,EACzB,kBAAkB,EAClB,qBAAqB,EAErB,MAAM,EACN,WAAW,EAEX,aAAa,EACd,MAAM,qBAAqB,CAAA;AAe5B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AAUnD,qBAAa,yBAA0B,YAAW,kBAAkB;IACtD,OAAO,CAAC,QAAQ,CAAC,EAAE;IAA/B,YAA6B,EAAE,EAAE,SAAS,EAAI;IAExC,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAW9D;IAEK,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAMnE;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBnE;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CA0BnE;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI3D;CACF;AAED,sFAAsF;AAEtF,qBAAa,0BAA2B,YAAW,mBAAmB;IAElE,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,KAAK;IAFxB,YACmB,EAAE,EAAE,SAAS,EACb,KAAK,EAAE,KAAK,EAC3B;IAEJ,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAkC;IAI9D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0D;IAE3E,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAYvE;IAEK,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAc7D;IAEK,sBAAsB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQjE;IAEK,YAAY,CAChB,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAA;QACb,MAAM,CAAC,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAA;SAAE,CAAA;QAC1C,QAAQ,CAAC,EAAE,eAAe,EAAE,CAAA;QAC5B,KAAK,CAAC,EAAE,MAAM,CAAA;KACf,GACA,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAsC9B;IAEK,UAAU,CACd,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAA;QACb,MAAM,CAAC,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAA;SAAE,CAAA;QAC1C,QAAQ,CAAC,EAAE,eAAe,EAAE,CAAA;QAC5B,KAAK,CAAC,EAAE,MAAM,CAAA;KACf,GACA,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAgC9B;IAEK,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAWnE;IAEK,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAmBvE;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAQ9D;IAEK,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAM5E;IAEK,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAYxF;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwC7E;IAEK,UAAU,CACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,iBAAiB,EAC5B,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAC5B,OAAO,CAAC,OAAO,CAAC,CAyDlB;IAEK,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,CA6BxF;IAEK,aAAa,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUvE;IAEK,SAAS,CAAC,gBAAgB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAa3D;IAEK,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiCtF;CACF;AAED,qBAAa,yBAA0B,YAAW,kBAAkB;IACtD,OAAO,CAAC,QAAQ,CAAC,EAAE;IAA/B,YAA6B,EAAE,EAAE,SAAS,EAAI;IAExC,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAgBzE;IAEK,SAAS,CAAC,gBAAgB,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAqBlE;IAEK,oBAAoB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAOnD;IAEK,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAiBjD;CACF;AAED;;;;;GAKG;AACH,qBAAa,gCAAiC,YAAW,yBAAyB;IACpE,OAAO,CAAC,QAAQ,CAAC,EAAE;IAA/B,YAA6B,EAAE,EAAE,SAAS,EAAI;IAE9C,gGAAgG;IAChG,OAAO,CAAC,YAAY;IAUd,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAW7F;IAEK,eAAe,CACnB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAsBlC;IAEK,oBAAoB,CACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAejC;IAEK,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAgB1E;IAEK,kBAAkB,CACtB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,qBAAqB,CAAC,CAmChC;CACF;AAsDD,qBAAa,iCAAkC,YAAW,0BAA0B;IACtE,OAAO,CAAC,QAAQ,CAAC,EAAE;IAA/B,YAA6B,EAAE,EAAE,SAAS,EAAI;IAE9C,OAAO,CAAC,MAAM;IAyBR,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAM3E;IAEK,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAWvF;IAEK,IAAI,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAO3D;IAEK,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAOlE;IAEK,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAatE;IAEK,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAalD;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CA0B3E;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI3D;IAEK,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWpE;IAEK,SAAS,CACb,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,GAAG,YAAY,GAAG,SAAS,GAAG,aAAa,CAAC,CAAC,GACrF,OAAO,CAAC,IAAI,CAAC,CAaf;IAEK,QAAQ,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAY9E;IAEK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMrD;CACF"}
1
+ {"version":3,"file":"execution.d.ts","sourceRoot":"","sources":["../../../src/repositories/drizzle/execution.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,YAAY,EACZ,WAAW,EACX,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,cAAc,EACd,QAAQ,EACR,kBAAkB,EAClB,gBAAgB,EAChB,0BAA0B,EAC1B,qBAAqB,EACrB,oBAAoB,EACpB,kBAAkB,EAClB,yBAAyB,EACzB,kBAAkB,EAClB,qBAAqB,EAErB,MAAM,EACN,WAAW,EAEX,aAAa,EACd,MAAM,qBAAqB,CAAA;AAe5B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AAUnD,qBAAa,yBAA0B,YAAW,kBAAkB;IACtD,OAAO,CAAC,QAAQ,CAAC,EAAE;IAA/B,YAA6B,EAAE,EAAE,SAAS,EAAI;IAExC,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAW9D;IAEK,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAMnE;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBnE;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CA0BnE;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI3D;CACF;AAED,sFAAsF;AAEtF,qBAAa,0BAA2B,YAAW,mBAAmB;IAElE,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,KAAK;IAFxB,YACmB,EAAE,EAAE,SAAS,EACb,KAAK,EAAE,KAAK,EAC3B;IAEJ,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAkC;IAI9D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0D;IAE3E,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAYvE;IAEK,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAc7D;IAEK,sBAAsB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQjE;IAEK,YAAY,CAChB,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAA;QACb,MAAM,CAAC,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAA;SAAE,CAAA;QAC1C,QAAQ,CAAC,EAAE,eAAe,EAAE,CAAA;QAC5B,KAAK,CAAC,EAAE,MAAM,CAAA;KACf,GACA,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAsC9B;IAEK,UAAU,CACd,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAA;QACb,MAAM,CAAC,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,EAAE,EAAE,MAAM,CAAA;SAAE,CAAA;QAC1C,QAAQ,CAAC,EAAE,eAAe,EAAE,CAAA;QAC5B,KAAK,CAAC,EAAE,MAAM,CAAA;KACf,GACA,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAgC9B;IAEK,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAWnE;IAEK,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAmBvE;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAQ9D;IAEK,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAM5E;IAEK,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAYxF;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwC7E;IAEK,UAAU,CACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,iBAAiB,EAC5B,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAC5B,OAAO,CAAC,OAAO,CAAC,CAyDlB;IAEK,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,CA6BxF;IAEK,aAAa,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUvE;IAEK,SAAS,CAAC,gBAAgB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAa3D;IAEK,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiCtF;CACF;AAED,qBAAa,yBAA0B,YAAW,kBAAkB;IACtD,OAAO,CAAC,QAAQ,CAAC,EAAE;IAA/B,YAA6B,EAAE,EAAE,SAAS,EAAI;IAExC,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAgBzE;IAEK,SAAS,CAAC,gBAAgB,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAuBlE;IAEK,oBAAoB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAOnD;IAEK,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAiBjD;IAEK,aAAa,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CASpE;CACF;AAED;;;;;GAKG;AACH,qBAAa,gCAAiC,YAAW,yBAAyB;IACpE,OAAO,CAAC,QAAQ,CAAC,EAAE;IAA/B,YAA6B,EAAE,EAAE,SAAS,EAAI;IAE9C,gGAAgG;IAChG,OAAO,CAAC,YAAY;IAUd,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAW7F;IAEK,eAAe,CACnB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAsBlC;IAEK,oBAAoB,CACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAejC;IAEK,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAgB1E;IAEK,kBAAkB,CACtB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,qBAAqB,CAAC,CAmChC;CACF;AAsDD,qBAAa,iCAAkC,YAAW,0BAA0B;IACtE,OAAO,CAAC,QAAQ,CAAC,EAAE;IAA/B,YAA6B,EAAE,EAAE,SAAS,EAAI;IAE9C,OAAO,CAAC,MAAM;IAyBR,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAM3E;IAEK,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAWvF;IAEK,IAAI,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAO3D;IAEK,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAOlE;IAEK,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAatE;IAEK,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAalD;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CA0B3E;IAEK,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI3D;IAEK,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWpE;IAEK,SAAS,CACb,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,GAAG,YAAY,GAAG,SAAS,GAAG,aAAa,CAAC,CAAC,GACrF,OAAO,CAAC,IAAI,CAAC,CAaf;IAEK,QAAQ,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAY9E;IAEK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMrD;CACF"}
@@ -419,6 +419,7 @@ export class DrizzleAgentRunRepository {
419
419
  id: agentRuns.id,
420
420
  kind: agentRuns.kind,
421
421
  updatedAt: agentRuns.updated_at,
422
+ redriveCount: agentRuns.redrive_count,
422
423
  })
423
424
  .from(agentRuns)
424
425
  .where(and(eq(agentRuns.status, 'running'), lt(agentRuns.updated_at, olderThanEpochMs)))
@@ -427,6 +428,7 @@ export class DrizzleAgentRunRepository {
427
428
  workspaceId: r.workspaceId,
428
429
  id: r.id,
429
430
  updatedAt: r.updatedAt,
431
+ redriveCount: r.redriveCount ?? 0,
430
432
  kind: decodeEnum(agentRunKindSchema, r.kind, {
431
433
  table: 'agent_runs',
432
434
  column: 'kind',
@@ -457,6 +459,16 @@ export class DrizzleAgentRunRepository {
457
459
  }
458
460
  return live;
459
461
  }
462
+ async recordRedrive(workspaceId, id) {
463
+ // One statement: increment and read back the new total, so nothing races between a read
464
+ // and a write. The row is absent (0 returned) only for a run that has since been deleted.
465
+ const rows = await this.db
466
+ .update(agentRuns)
467
+ .set({ redrive_count: sql `${agentRuns.redrive_count} + 1` })
468
+ .where(and(eq(agentRuns.workspace_id, workspaceId), eq(agentRuns.id, id)))
469
+ .returning({ redriveCount: agentRuns.redrive_count });
470
+ return rows[0]?.redriveCount ?? 0;
471
+ }
460
472
  }
461
473
  /**
462
474
  * Deployment-level rollups over `agent_runs`, scoped to an account by a sub-select on