@cat-factory/worker 0.137.2 → 0.138.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/dist/app.d.ts.map +1 -1
  2. package/dist/app.js +33 -0
  3. package/dist/app.js.map +1 -1
  4. package/dist/index.d.ts +2 -1
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/index.js +111 -118
  7. package/dist/index.js.map +1 -1
  8. package/dist/infrastructure/config/index.d.ts.map +1 -1
  9. package/dist/infrastructure/config/index.js +4 -0
  10. package/dist/infrastructure/config/index.js.map +1 -1
  11. package/dist/infrastructure/container-account-settings.d.ts +13 -0
  12. package/dist/infrastructure/container-account-settings.d.ts.map +1 -0
  13. package/dist/infrastructure/container-account-settings.js +34 -0
  14. package/dist/infrastructure/container-account-settings.js.map +1 -0
  15. package/dist/infrastructure/container-artifact-storage.d.ts +16 -0
  16. package/dist/infrastructure/container-artifact-storage.d.ts.map +1 -0
  17. package/dist/infrastructure/container-artifact-storage.js +40 -0
  18. package/dist/infrastructure/container-artifact-storage.js.map +1 -0
  19. package/dist/infrastructure/container-assembly.d.ts.map +1 -1
  20. package/dist/infrastructure/container-assembly.js +5 -1
  21. package/dist/infrastructure/container-assembly.js.map +1 -1
  22. package/dist/infrastructure/container-executor-deps.d.ts.map +1 -1
  23. package/dist/infrastructure/container-executor-deps.js +4 -1
  24. package/dist/infrastructure/container-executor-deps.js.map +1 -1
  25. package/dist/infrastructure/container-shared-services.d.ts +1 -1
  26. package/dist/infrastructure/container-shared-services.d.ts.map +1 -1
  27. package/dist/infrastructure/container-shared-services.js +3 -1
  28. package/dist/infrastructure/container-shared-services.js.map +1 -1
  29. package/dist/infrastructure/container.d.ts +5 -21
  30. package/dist/infrastructure/container.d.ts.map +1 -1
  31. package/dist/infrastructure/container.js +12 -57
  32. package/dist/infrastructure/container.js.map +1 -1
  33. package/dist/infrastructure/env.d.ts +4 -0
  34. package/dist/infrastructure/env.d.ts.map +1 -1
  35. package/dist/infrastructure/env.js.map +1 -1
  36. package/dist/infrastructure/observability/cronSweep.d.ts +33 -0
  37. package/dist/infrastructure/observability/cronSweep.d.ts.map +1 -0
  38. package/dist/infrastructure/observability/cronSweep.js +41 -0
  39. package/dist/infrastructure/observability/cronSweep.js.map +1 -0
  40. package/dist/infrastructure/observability/operationalFlush.d.ts +15 -0
  41. package/dist/infrastructure/observability/operationalFlush.d.ts.map +1 -0
  42. package/dist/infrastructure/observability/operationalFlush.js +64 -0
  43. package/dist/infrastructure/observability/operationalFlush.js.map +1 -0
  44. package/dist/infrastructure/repositories/D1AgentRunRepository.d.ts +1 -0
  45. package/dist/infrastructure/repositories/D1AgentRunRepository.d.ts.map +1 -1
  46. package/dist/infrastructure/repositories/D1AgentRunRepository.js +14 -1
  47. package/dist/infrastructure/repositories/D1AgentRunRepository.js.map +1 -1
  48. package/dist/infrastructure/workflows/retention.d.ts +18 -4
  49. package/dist/infrastructure/workflows/retention.d.ts.map +1 -1
  50. package/dist/infrastructure/workflows/retention.js +22 -19
  51. package/dist/infrastructure/workflows/retention.js.map +1 -1
  52. package/dist/infrastructure/workflows/sweeper.d.ts +11 -2
  53. package/dist/infrastructure/workflows/sweeper.d.ts.map +1 -1
  54. package/dist/infrastructure/workflows/sweeper.js +11 -1
  55. package/dist/infrastructure/workflows/sweeper.js.map +1 -1
  56. package/migrations/0076_agent_runs_redrive_count.sql +20 -0
  57. package/package.json +18 -18
package/dist/app.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAU/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAA;AAU3B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,6BAA6B,CAAA;AAEzD,MAAM,WAAW,gBAAgB;IAC/B,+EAA+E;IAC/E,SAAS,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAA;IACrC,mFAAmF;IACnF,uBAAuB,CAAC,EAAE,OAAO,CAAA;IACjC,gFAAgF;IAChF,aAAa,CAAC,EAAE,qBAAqB,CAAA;CACtC;AAqBD;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,IAAI,CAAC,MAAM,CAAC,CA6EtE"}
1
+ {"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAU/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAA;AAU3B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,6BAA6B,CAAA;AAEzD,MAAM,WAAW,gBAAgB;IAC/B,+EAA+E;IAC/E,SAAS,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAA;IACrC,mFAAmF;IACnF,uBAAuB,CAAC,EAAE,OAAO,CAAA;IACjC,gFAAgF;IAChF,aAAa,CAAC,EAAE,qBAAqB,CAAA;CACtC;AAqBD;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,IAAI,CAAC,MAAM,CAAC,CA8GtE"}
package/dist/app.js CHANGED
@@ -82,6 +82,39 @@ export function createApp(options = {}) {
82
82
  await next();
83
83
  });
84
84
  app.get('/health', (c) => c.json({ status: 'ok' }));
85
+ // Readiness (slice 4.3). The Worker has no long-lived process to drain and no rotation to be
86
+ // taken out of — Cloudflare routes every request to a fresh isolate and there is nothing an
87
+ // operator could do with a red answer — so `/ready` here is deliberately NOT the Node
88
+ // facade's drain signal. What it IS: a bindings probe, answering the one question an operator
89
+ // actually asks of a fresh deployment ("is D1 reachable, is TELEMETRY_DB bound"), which on
90
+ // this runtime is a genuinely separate database rather than a schema in the same one.
91
+ //
92
+ // Reported per binding rather than as one boolean, and a failure is 503 so a smoke test can
93
+ // assert on the status alone. Public, before the auth gate, like `/health`.
94
+ app.get('/ready', async (c) => {
95
+ const env = c.env;
96
+ const probe = async (run) => {
97
+ try {
98
+ await run();
99
+ return { ok: true };
100
+ }
101
+ catch (err) {
102
+ // Kept to a short diagnostic: this endpoint is unauthenticated, so it must never carry
103
+ // a connection string or a binding's internals — the same rule as the Node twin's.
104
+ return { ok: false, error: err instanceof Error ? err.message : String(err) };
105
+ }
106
+ };
107
+ const checks = {
108
+ database: await probe(() => c.env.DB.prepare('SELECT 1').first()),
109
+ };
110
+ // TELEMETRY_DB is REQUIRED (the build fails without it), so an absent binding is a real
111
+ // failure to report rather than a capability to skip.
112
+ checks.telemetry = env.TELEMETRY_DB
113
+ ? await probe(() => env.TELEMETRY_DB.prepare('SELECT 1').first())
114
+ : { ok: false, error: 'TELEMETRY_DB is not bound' };
115
+ const ready = Object.values(checks).every((check) => check.ok);
116
+ return c.json({ status: ready ? 'ready' : 'not_ready', checks }, ready ? 200 : 503);
117
+ });
85
118
  // Default-deny session gate + per-workspace authz, shared verbatim with the Node
86
119
  // service (one implementation in @cat-factory/server so the runtimes can't drift).
87
120
  mountAuthGate(app);
package/dist/app.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,0BAA0B,EAC1B,uBAAuB,EACvB,MAAM,EACN,aAAa,EACb,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAA;AAC3B,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,8BAA8B,CAAA;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAA;AAYhE,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,iFAAiF;AACjF,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAA;AAC1C,SAAS,oBAAoB,CAAC,QAAyB;IACrD,MAAM,SAAS,GAAG,QAAQ;SACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SACjB,IAAI,EAAE;SACN,IAAI,CAAC,GAAG,CAAC,CAAA;IACZ,IAAI,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAM;IAC3C,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IAC/B,MAAM,CAAC,KAAK,CACV,yFAAyF;QACvF,0EAA0E,EAC5E,EAAE,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CACzC,CAAA;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,SAAS,CAAC,OAAO,GAAqB,EAAE;IACtD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAU,CAAA;IAE9B,4FAA4F;IAC5F,6FAA6F;IAC7F,kCAAkC;IAClC,mBAAmB,CAAC,GAAG,CAAC,CAAA;IAExB,6EAA6E;IAC7E,yEAAyE;IACzE,mFAAmF;IACnF,2EAA2E;IAC3E,yFAAyF;IACzF,GAAG,CAAC,GAAG,CACL,GAAG,EACH,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CACpB,iBAAiB,CACf,MAAM,EACN,CAAC,CAAC,GAAG,CAAC,oBAAoB,EAC1B,qBAAqB,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CACzC;QACH,iFAAiF;QACjF,mFAAmF;QACnF,kFAAkF;QAClF,wEAAwE;QACxE,YAAY,EAAE,CAAC,GAAG,oBAAoB,CAAC;QACvC,uFAAuF;QACvF,aAAa,EAAE,CAAC,GAAG,oBAAoB,CAAC;KACzC,CAAC,CACH,CAAA;IACD,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;QAC7B,IAAI,CAAC;YACH,CAAC,CAAC,GAAG,CACH,WAAW,EACX,cAAc,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE;gBACvC,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;gBACxD,aAAa,EAAE,OAAO,CAAC,aAAa;aACrC,CAAC,CACH,CAAA;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6FAA6F;YAC7F,8FAA8F;YAC9F,+FAA+F;YAC/F,sFAAsF;YACtF,IAAI,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;gBAClC,MAAM,GAAG,GAAG,0BAA0B,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAA;gBACjF,yEAAyE;gBACzE,0FAA0F;gBAC1F,2FAA2F;gBAC3F,yFAAyF;gBACzF,0FAA0F;gBAC1F,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;gBACrC,IAAI,MAAM;oBAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,MAAM,CAAC,CAAA;gBAClE,OAAO,GAAG,CAAA;YACZ,CAAC;YACD,MAAM,GAAG,CAAA;QACX,CAAC;QACD,MAAM,IAAI,EAAE,CAAA;IACd,CAAC,CAAC,CAAA;IAEF,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;IAEnD,iFAAiF;IACjF,mFAAmF;IACnF,aAAa,CAAC,GAAG,CAAC,CAAA;IAElB,mFAAmF;IACnF,mFAAmF;IACnF,8EAA8E;IAC9E,gCAAgC;IAChC,uBAAuB,CAAC,GAAG,CAAC,CAAA;IAE5B,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAExB,OAAO,GAAG,CAAA;AACZ,CAAC"}
1
+ {"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,0BAA0B,EAC1B,uBAAuB,EACvB,MAAM,EACN,aAAa,EACb,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAA;AAC3B,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,8BAA8B,CAAA;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAA;AAYhE,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,iFAAiF;AACjF,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAA;AAC1C,SAAS,oBAAoB,CAAC,QAAyB;IACrD,MAAM,SAAS,GAAG,QAAQ;SACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SACjB,IAAI,EAAE;SACN,IAAI,CAAC,GAAG,CAAC,CAAA;IACZ,IAAI,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAM;IAC3C,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IAC/B,MAAM,CAAC,KAAK,CACV,yFAAyF;QACvF,0EAA0E,EAC5E,EAAE,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CACzC,CAAA;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,SAAS,CAAC,OAAO,GAAqB,EAAE;IACtD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAU,CAAA;IAE9B,4FAA4F;IAC5F,6FAA6F;IAC7F,kCAAkC;IAClC,mBAAmB,CAAC,GAAG,CAAC,CAAA;IAExB,6EAA6E;IAC7E,yEAAyE;IACzE,mFAAmF;IACnF,2EAA2E;IAC3E,yFAAyF;IACzF,GAAG,CAAC,GAAG,CACL,GAAG,EACH,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CACpB,iBAAiB,CACf,MAAM,EACN,CAAC,CAAC,GAAG,CAAC,oBAAoB,EAC1B,qBAAqB,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CACzC;QACH,iFAAiF;QACjF,mFAAmF;QACnF,kFAAkF;QAClF,wEAAwE;QACxE,YAAY,EAAE,CAAC,GAAG,oBAAoB,CAAC;QACvC,uFAAuF;QACvF,aAAa,EAAE,CAAC,GAAG,oBAAoB,CAAC;KACzC,CAAC,CACH,CAAA;IACD,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;QAC7B,IAAI,CAAC;YACH,CAAC,CAAC,GAAG,CACH,WAAW,EACX,cAAc,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE;gBACvC,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;gBACxD,aAAa,EAAE,OAAO,CAAC,aAAa;aACrC,CAAC,CACH,CAAA;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6FAA6F;YAC7F,8FAA8F;YAC9F,+FAA+F;YAC/F,sFAAsF;YACtF,IAAI,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;gBAClC,MAAM,GAAG,GAAG,0BAA0B,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAA;gBACjF,yEAAyE;gBACzE,0FAA0F;gBAC1F,2FAA2F;gBAC3F,yFAAyF;gBACzF,0FAA0F;gBAC1F,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;gBACrC,IAAI,MAAM;oBAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,MAAM,CAAC,CAAA;gBAClE,OAAO,GAAG,CAAA;YACZ,CAAC;YACD,MAAM,GAAG,CAAA;QACX,CAAC;QACD,MAAM,IAAI,EAAE,CAAA;IACd,CAAC,CAAC,CAAA;IAEF,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;IAEnD,6FAA6F;IAC7F,4FAA4F;IAC5F,sFAAsF;IACtF,8FAA8F;IAC9F,2FAA2F;IAC3F,sFAAsF;IACtF,EAAE;IACF,4FAA4F;IAC5F,4EAA4E;IAC5E,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAA;QACjB,MAAM,KAAK,GAAG,KAAK,EAAE,GAA2B,EAA4C,EAAE;YAC5F,IAAI,CAAC;gBACH,MAAM,GAAG,EAAE,CAAA;gBACX,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAA;YACrB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,uFAAuF;gBACvF,mFAAmF;gBACnF,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;YAC/E,CAAC;QACH,CAAC,CAAA;QACD,MAAM,MAAM,GAAoD;YAC9D,QAAQ,EAAE,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,CAAC;SAClE,CAAA;QACD,wFAAwF;QACxF,sDAAsD;QACtD,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,YAAY;YACjC,CAAC,CAAC,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,YAAa,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,CAAC;YAClE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,2BAA2B,EAAE,CAAA;QACrD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACrF,CAAC,CAAC,CAAA;IAEF,iFAAiF;IACjF,mFAAmF;IACnF,aAAa,CAAC,GAAG,CAAC,CAAA;IAElB,mFAAmF;IACnF,mFAAmF;IACnF,8EAA8E;IAC9E,gCAAgC;IAChC,uBAAuB,CAAC,GAAG,CAAC,CAAA;IAE5B,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAExB,OAAO,GAAG,CAAA;AACZ,CAAC"}
package/dist/index.d.ts CHANGED
@@ -16,12 +16,13 @@ export { InitiativePresetRegistry, type InitiativePresetRegistration } from '@ca
16
16
  export { PipelineRegistry, defaultPipelineRegistry } from '@cat-factory/kernel';
17
17
  export { TaskTypeRegistry, defaultTaskTypeRegistry } from '@cat-factory/kernel';
18
18
  export { FoundationalServiceRegistry, type FoundationalServiceDefinition, defaultFoundationalServiceRegistry, } from '@cat-factory/kernel';
19
+ export { BinaryGeneratorRegistry, type BinaryGeneratorDefinition, defaultBinaryGeneratorRegistry, } from '@cat-factory/kernel';
19
20
  export type { CreateAppOptions } from './app';
20
21
  export { DEFAULT_MODEL_PRESET_ID, MODEL_PRESET_SEED_IDS } from '@cat-factory/kernel';
21
22
  /** The cron sweeper. Module-level (not closed over the app) so {@link createWorker} stays thin. */
22
23
  declare function handleScheduled(controller: ScheduledController, env: Env, ctx: ExecutionContext): Promise<void>;
23
24
  /** The queue consumer multiplexing the three queues. Module-level, like {@link handleScheduled}. */
24
- declare function handleQueue(batch: MessageBatch<ExecutionStartMessage | GitHubSyncMessage | TrackerSyncMessage>, env: Env): Promise<void>;
25
+ declare function handleQueue(batch: MessageBatch<ExecutionStartMessage | GitHubSyncMessage | TrackerSyncMessage>, env: Env, ctx: ExecutionContext): Promise<void>;
25
26
  /** The Worker's exported handler shape (fetch + the cron and queue consumers). */
26
27
  export type WorkerHandler = ExportedHandler<Env, ExecutionStartMessage | GitHubSyncMessage | TrackerSyncMessage>;
27
28
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,eAAe,EACf,YAAY,EACZ,mBAAmB,EACpB,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,KAAK,gBAAgB,EAAa,MAAM,OAAO,CAAA;AAExD,OAAO,KAAK,EACV,GAAG,EACH,qBAAqB,EACrB,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,sBAAsB,CAAA;AA+E7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAA;AAChF,OAAO,EAAE,sBAAsB,EAAE,MAAM,mDAAmD,CAAA;AAC1F,OAAO,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAA;AAChF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oDAAoD,CAAA;AAC5F,OAAO,EAAE,uBAAuB,EAAE,MAAM,oDAAoD,CAAA;AAE5F,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAA;AAGnF,OAAO,EAAE,eAAe,EAAE,MAAM,6CAA6C,CAAA;AAE7E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qDAAqD,CAAA;AAKxF,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,KAAK,oBAAoB,GAC1B,MAAM,gCAAgC,CAAA;AAOvC,OAAO,EACL,iBAAiB,EACjB,wBAAwB,EACxB,KAAK,mBAAmB,GACzB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAAE,+BAA+B,EAAE,MAAM,qBAAqB,CAAA;AACrE,OAAO,EAAE,wBAAwB,EAAE,KAAK,4BAA4B,EAAE,MAAM,qBAAqB,CAAA;AAIjG,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAA;AAK/E,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAA;AAO/E,OAAO,EACL,2BAA2B,EAC3B,KAAK,6BAA6B,EAClC,kCAAkC,GACnC,MAAM,qBAAqB,CAAA;AAG5B,YAAY,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAA;AAK7C,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AAumBpF,mGAAmG;AACnG,iBAAe,eAAe,CAC5B,UAAU,EAAE,mBAAmB,EAC/B,GAAG,EAAE,GAAG,EACR,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED,oGAAoG;AACpG,iBAAe,WAAW,CACxB,KAAK,EAAE,YAAY,CAAC,qBAAqB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC,EACnF,GAAG,EAAE,GAAG,GACP,OAAO,CAAC,IAAI,CAAC,CA0Cf;AAED,kFAAkF;AAClF,MAAM,MAAM,aAAa,GAAG,eAAe,CACzC,GAAG,EACH,qBAAqB,GAAG,iBAAiB,GAAG,kBAAkB,CAC/D,CAAA;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,YAAY,CAAC,OAAO,GAAE,gBAAqB,GAAG,aAAa,CA0B1E;AAiBD;;;;;;GAMG;;qBAIgB,UAAU,CAAC,WAAW,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,eAAe,EACf,YAAY,EACZ,mBAAmB,EACpB,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,KAAK,gBAAgB,EAAa,MAAM,OAAO,CAAA;AAExD,OAAO,KAAK,EACV,GAAG,EACH,qBAAqB,EACrB,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,sBAAsB,CAAA;AAmF7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAA;AAChF,OAAO,EAAE,sBAAsB,EAAE,MAAM,mDAAmD,CAAA;AAC1F,OAAO,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAA;AAChF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oDAAoD,CAAA;AAC5F,OAAO,EAAE,uBAAuB,EAAE,MAAM,oDAAoD,CAAA;AAE5F,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAA;AAGnF,OAAO,EAAE,eAAe,EAAE,MAAM,6CAA6C,CAAA;AAE7E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qDAAqD,CAAA;AAKxF,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,KAAK,oBAAoB,GAC1B,MAAM,gCAAgC,CAAA;AAOvC,OAAO,EACL,iBAAiB,EACjB,wBAAwB,EACxB,KAAK,mBAAmB,GACzB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAAE,+BAA+B,EAAE,MAAM,qBAAqB,CAAA;AACrE,OAAO,EAAE,wBAAwB,EAAE,KAAK,4BAA4B,EAAE,MAAM,qBAAqB,CAAA;AAIjG,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAA;AAK/E,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAA;AAO/E,OAAO,EACL,2BAA2B,EAC3B,KAAK,6BAA6B,EAClC,kCAAkC,GACnC,MAAM,qBAAqB,CAAA;AAO5B,OAAO,EACL,uBAAuB,EACvB,KAAK,yBAAyB,EAC9B,8BAA8B,GAC/B,MAAM,qBAAqB,CAAA;AAG5B,YAAY,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAA;AAK7C,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AAyiBpF,mGAAmG;AACnG,iBAAe,eAAe,CAC5B,UAAU,EAAE,mBAAmB,EAC/B,GAAG,EAAE,GAAG,EACR,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC,CAsBf;AAwBD,oGAAoG;AACpG,iBAAe,WAAW,CACxB,KAAK,EAAE,YAAY,CAAC,qBAAqB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC,EACnF,GAAG,EAAE,GAAG,EACR,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC,CAKf;AAiDD,kFAAkF;AAClF,MAAM,MAAM,aAAa,GAAG,eAAe,CACzC,GAAG,EACH,qBAAqB,GAAG,iBAAiB,GAAG,kBAAkB,CAC/D,CAAA;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,YAAY,CAAC,OAAO,GAAE,gBAAqB,GAAG,aAAa,CA8B1E;AAiBD;;;;;;GAMG;;qBAIgB,UAAU,CAAC,WAAW,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC"}
package/dist/index.js CHANGED
@@ -17,7 +17,7 @@ import { D1SubscriptionQuotaCycleRepository } from './infrastructure/repositorie
17
17
  import { D1PasswordResetTokenRepository } from './infrastructure/repositories/D1PasswordResetTokenRepository';
18
18
  import { D1NotificationRepository } from './infrastructure/repositories/D1NotificationRepository';
19
19
  import { buildContainer, buildCloudflareArtifactStoreResolver } from './infrastructure/container';
20
- import { GITHUB_RECONCILE_STALE_MS, escalateStaleNotifications, FOUNDATIONAL_SOURCE_STALE_MS, shouldRunReachabilityPass, sweepFoundationalSources, sweepInfraReachability, sweepPlatformHealth, } from '@cat-factory/server';
20
+ import { GITHUB_RECONCILE_STALE_MS, escalateStaleNotifications, FOUNDATIONAL_SOURCE_STALE_MS, shouldRunReachabilityPass, sweepFoundationalSources, sweepInfraReachability, sweepPlatformHealth, operationalMetrics, } from '@cat-factory/server';
21
21
  import { CryptoIdGenerator, SystemClock } from './infrastructure/runtime';
22
22
  import { WorkflowsWorkRunner } from './infrastructure/workflows/WorkflowsWorkRunner';
23
23
  import { WorkflowsBootstrapRunner } from './infrastructure/workflows/WorkflowsBootstrapRunner';
@@ -30,10 +30,12 @@ import { handleGitHubSyncBatch, handleTrackerSyncBatch, reconcileStaleRepos, } f
30
30
  import { sweepExpiredEnvironments } from './infrastructure/environments/sweep';
31
31
  import { logger } from './infrastructure/observability/logger';
32
32
  import { runPlatformMetricsSweep } from './infrastructure/observability/platformMetrics';
33
+ import { flushOperationalMetricsForIsolate } from './infrastructure/observability/operationalFlush';
34
+ import { SweepTick } from './infrastructure/observability/cronSweep';
33
35
  import { defaultJudgeRegistry, defaultStepResolverRegistry, sweepBinaryArtifactRetention, validateRegistrationsOnce, } from '@cat-factory/orchestration';
34
36
  import { defaultAgentKindRegistry, defaultInitiativePresetRegistry } from '@cat-factory/agents';
35
37
  import { gateRegistryWithBuiltins } from '@cat-factory/gates';
36
- import { DEFAULT_WORKSPACE_SETTINGS, defaultFoundationalServiceRegistry, defaultPipelineRegistry, defaultTaskTypeRegistry, describeError, } from '@cat-factory/kernel';
38
+ import { DEFAULT_WORKSPACE_SETTINGS, defaultBinaryGeneratorRegistry, defaultFoundationalServiceRegistry, defaultPipelineRegistry, defaultTaskTypeRegistry, describeError, } from '@cat-factory/kernel';
37
39
  import { D1WorkspaceRepository } from './infrastructure/repositories/D1WorkspaceRepository';
38
40
  import { D1WorkspaceSettingsRepository } from './infrastructure/repositories/D1WorkspaceSettingsRepository';
39
41
  import { D1KeyFingerprintStore } from './infrastructure/repositories/D1KeyFingerprintStore';
@@ -87,6 +89,13 @@ export { TaskTypeRegistry, defaultTaskTypeRegistry } from '@cat-factory/kernel';
87
89
  // against the estate from its first request and an account/board can still override or suppress
88
90
  // any of them. See backend/docs/adr/0031-foundational-services.md.
89
91
  export { FoundationalServiceRegistry, defaultFoundationalServiceRegistry, } from '@cat-factory/kernel';
92
+ // Installation-level extension point for GENERATIVE BINARY INTEGRATIONS (the same DI seam once
93
+ // more): a deployment news a `defaultBinaryGeneratorRegistry()`, registers the image / music /
94
+ // video generation APIs it pays for on it by reference, and injects it via the
95
+ // `binaryGeneratorRegistry` override. A pipeline step whose kind carries the `binary-output`
96
+ // trait then SELECTS from them (`stepOptions.binaryOutput.generatorIds`), and the engine briefs
97
+ // the agent on each one's content types, contract and credential variable.
98
+ export { BinaryGeneratorRegistry, defaultBinaryGeneratorRegistry, } from '@cat-factory/kernel';
90
99
  // The built-in model-preset ids + the catalog fallback default. A custom Worker entry that builds
91
100
  // its own app can seed a different out-of-the-box default with
92
101
  // `createApp({ overrides: { defaultModelPresetId: MODEL_PRESET_SEED_IDS.claude } })` (a
@@ -125,6 +134,10 @@ function resolveEntryRegistries(overrides) {
125
134
  // capabilities); registered ones are the `builtin` tier of every workspace catalog, and a
126
135
  // malformed contract document fails boot rather than a design dispatch.
127
136
  foundationalServiceRegistry: overrides.foundationalServiceRegistry ?? defaultFoundationalServiceRegistry(),
137
+ // Generative binary integrations (empty by default — the platform ships none). Registered
138
+ // ones are what a binary-generating step may produce with, and a malformed definition or a
139
+ // cleartext endpoint fails boot rather than a dispatch.
140
+ binaryGeneratorRegistry: overrides.binaryGeneratorRegistry ?? defaultBinaryGeneratorRegistry(),
128
141
  };
129
142
  }
130
143
  /** The boot/cron key-fingerprint check's logger, tagged so its lines are greppable by cron. */
@@ -179,29 +192,26 @@ const FREQUENT_CRON_PERIOD_MS = 2 * 60_000;
179
192
  * windows. The tables exist regardless of whether GitHub/agents are
180
193
  * configured, so this runs unconditionally; an unused table reclaims nothing.
181
194
  */
182
- function runDailyRetentionSweeps(env, ctx, clock) {
195
+ function runDailyRetentionSweeps(env, tick, clock) {
183
196
  // ADR 0026 D6.1: the Worker has no boot moment, so the O(1) ENCRYPTION_KEY drift check
184
197
  // rides the daily cron. It seeds the fingerprint on first run and logs a definitive
185
198
  // drift signal on a key change. Independent of (and cheaper than) the retention work.
186
199
  if (env.ENCRYPTION_KEY) {
187
200
  const encryptionKey = env.ENCRYPTION_KEY;
188
- ctx.waitUntil(checkKeyFingerprint({
201
+ tick.run({ name: 'key-fingerprint', failureMessage: 'key fingerprint check failed' }, checkKeyFingerprint({
189
202
  store: new D1KeyFingerprintStore({ db: env.DB }),
190
203
  masterKeyBase64: encryptionKey,
191
204
  logger: keyFingerprintLogger,
192
- }).catch((error) => logger.error('key fingerprint check failed', {
193
- cron: 'key-fingerprint',
194
- ...describeError(error),
195
- })));
205
+ }));
196
206
  // ADR 0026 D6.2: the drift sweep — decrypt every sealed credential and raise/clear ONE
197
207
  // `key_drift` card per affected workspace. Rides the same daily cron as the fingerprint.
198
- ctx.waitUntil(sweepKeyDriftAndRaise(buildContainer(env), (info) => new WebCryptoSecretCipher({ masterKeyBase64: encryptionKey, info }), keyFingerprintLogger).catch((error) => logger.error('key drift sweep failed', { cron: 'key-drift', ...describeError(error) })));
208
+ tick.run({ name: 'key-drift', failureMessage: 'key drift sweep failed' }, sweepKeyDriftAndRaise(buildContainer(env), (info) => new WebCryptoSecretCipher({ masterKeyBase64: encryptionKey, info }), keyFingerprintLogger));
199
209
  }
200
210
  // This branch never calls buildContainer (no request container is built for the
201
211
  // sweep), so do the same fail-fast the build does: a clear error beats an opaque
202
212
  // NPE deep in a telemetry repo when the binding is unbound.
203
213
  const telemetryDb = requireTelemetryDb(env);
204
- ctx.waitUntil(sweepRetention({
214
+ tick.run({ name: 'retention', failureMessage: 'retention sweep failed' }, sweepRetention({
205
215
  tokenUsageRepository: new D1TokenUsageRepository({ db: env.DB }),
206
216
  rateLimitRepository: new D1RateLimitRepository({
207
217
  db: env.DB,
@@ -229,16 +239,25 @@ function runDailyRetentionSweeps(env, ctx, clock) {
229
239
  : {}),
230
240
  clock,
231
241
  policy: loadConfig(env).retention,
232
- })
233
- .then((result) => logger.info('retention sweep complete', { cron: 'retention', ...result }))
234
- .catch((error) => logger.error('retention sweep failed', { cron: 'retention', ...describeError(error) })));
242
+ logger,
243
+ }).then(({ failedTables, ...reclaimed }) => {
244
+ logger.info('retention sweep complete', { cron: 'retention', ...reclaimed });
245
+ // Each failure was already warned at its table; this names the SET, so a table that
246
+ // has been failing every night is one greppable line rather than a pattern to notice.
247
+ if (failedTables.length > 0) {
248
+ logger.warn('retention sweep could not prune every table', {
249
+ cron: 'retention',
250
+ failedTables,
251
+ });
252
+ }
253
+ }));
235
254
  // Binary-artifact retention (UI screenshots + reference designs) is per-workspace, and
236
255
  // the blob backend is per-account (R2 or S3), so it resolves each workspace's store. Run
237
256
  // whenever storage could be configured: the R2 default (ARTIFACT_BUCKET) OR a per-account
238
257
  // S3 backend (which needs the encryption key to unseal its credentials).
239
258
  if (env.ARTIFACT_BUCKET || env.ENCRYPTION_KEY) {
240
259
  const settingsRepo = new D1WorkspaceSettingsRepository({ db: env.DB });
241
- ctx.waitUntil(sweepBinaryArtifactRetention({
260
+ tick.run({ name: 'artifact-retention', failureMessage: 'artifact retention sweep failed' }, sweepBinaryArtifactRetention({
242
261
  resolveStore: buildCloudflareArtifactStoreResolver(env, env.DB, clock, new CryptoIdGenerator()),
243
262
  listWorkspaceIds: () => new D1WorkspaceRepository({ db: env.DB })
244
263
  .listVisible(null)
@@ -247,14 +266,9 @@ function runDailyRetentionSweeps(env, ctx, clock) {
247
266
  .get(workspaceId)
248
267
  .then((s) => s?.artifactRetentionDays ?? DEFAULT_WORKSPACE_SETTINGS.artifactRetentionDays),
249
268
  now: clock.now(),
250
- })
251
- .then((removed) => logger.info('artifact retention sweep complete', {
269
+ }).then((removed) => logger.info('artifact retention sweep complete', {
252
270
  cron: 'retention',
253
271
  binaryArtifacts: removed,
254
- }))
255
- .catch((error) => logger.error('artifact retention sweep failed', {
256
- cron: 'retention',
257
- ...describeError(error),
258
272
  })));
259
273
  }
260
274
  }
@@ -262,7 +276,7 @@ function runDailyRetentionSweeps(env, ctx, clock) {
262
276
  * Re-drive any agent run — execution OR bootstrap — whose Workflows instance
263
277
  * died. One sweep over the unified agent_runs table dispatches by kind.
264
278
  */
265
- function redriveStuckAgentRuns(env, ctx, clock) {
279
+ function redriveStuckAgentRuns(env, tick, clock) {
266
280
  if (env.EXECUTION_WORKFLOW || env.BOOTSTRAP_WORKFLOW || env.ENV_CONFIG_REPAIR_WORKFLOW) {
267
281
  const execLookup = env.EXECUTION_WORKFLOW ? new WorkflowsLookup(env.EXECUTION_WORKFLOW) : null;
268
282
  const bootLookup = env.BOOTSTRAP_WORKFLOW ? new WorkflowsLookup(env.BOOTSTRAP_WORKFLOW) : null;
@@ -278,7 +292,7 @@ function redriveStuckAgentRuns(env, ctx, clock) {
278
292
  const repairRunner = env.ENV_CONFIG_REPAIR_WORKFLOW
279
293
  ? new WorkflowsEnvConfigRepairRunner(env.ENV_CONFIG_REPAIR_WORKFLOW)
280
294
  : null;
281
- ctx.waitUntil(sweepStuckRuns({
295
+ tick.run({ name: 'stale-run', failureMessage: 'run sweep failed' }, sweepStuckRuns({
282
296
  agentRunRepository: new D1AgentRunRepository({ db: env.DB }),
283
297
  instanceState: (ref) => {
284
298
  const lookup = ref.kind === 'bootstrap'
@@ -336,15 +350,16 @@ function redriveStuckAgentRuns(env, ctx, clock) {
336
350
  leaseMs: SWEEP_LEASE_MS,
337
351
  hardStallMs: SWEEP_HARD_STALL_MS,
338
352
  orphanedSince: runSweepOrphanedSince,
353
+ metrics: operationalMetrics,
354
+ logger,
339
355
  })
340
356
  // Surface what the sweep did — the key signal for "are runs getting stuck?"
341
357
  // Only log when it actually acted.
342
358
  .then(({ redriven, finalized, stalled }) => {
343
359
  if (redriven > 0 || finalized > 0 || stalled > 0) {
344
- logger.warn('swept stuck runs', { cron: 'run-sweeper', redriven, finalized, stalled });
360
+ logger.warn('swept stuck runs', { cron: 'stale-run', redriven, finalized, stalled });
345
361
  }
346
- })
347
- .catch((error) => logger.error('run sweep failed', { cron: 'run-sweeper', ...describeError(error) })));
362
+ }));
348
363
  }
349
364
  }
350
365
  /**
@@ -352,11 +367,11 @@ function redriveStuckAgentRuns(env, ctx, clock) {
352
367
  * sweep never sees them — this sibling sweep re-drives a run whose Workflows instance
353
368
  * was lost and finalizes (cleanup + failed) one whose instance is terminal.
354
369
  */
355
- function redriveStuckEnvTests(env, ctx, clock) {
370
+ function redriveStuckEnvTests(env, tick, clock) {
356
371
  if (env.ENV_TEST_WORKFLOW) {
357
372
  const envTestLookup = new WorkflowsLookup(env.ENV_TEST_WORKFLOW);
358
373
  const envTestRunner = new WorkflowsEnvironmentTestRunner(env.ENV_TEST_WORKFLOW);
359
- ctx.waitUntil(sweepStuckEnvTests({
374
+ tick.run({ name: 'env-test-sweeper', failureMessage: 'env-test sweep failed' }, sweepStuckEnvTests({
360
375
  repository: new D1EnvironmentTestRunRepository({ db: env.DB }),
361
376
  instanceState: (runId) => envTestLookup.instanceState(runId),
362
377
  redrive: (workspaceId, runId) => envTestRunner.startRun(workspaceId, runId),
@@ -366,8 +381,7 @@ function redriveStuckEnvTests(env, ctx, clock) {
366
381
  },
367
382
  clock,
368
383
  leaseMs: SWEEP_LEASE_MS,
369
- })
370
- .then(({ redriven, finalized }) => {
384
+ }).then(({ redriven, finalized }) => {
371
385
  if (redriven > 0 || finalized > 0) {
372
386
  logger.warn('swept stuck env-test runs', {
373
387
  cron: 'env-test-sweeper',
@@ -375,11 +389,7 @@ function redriveStuckEnvTests(env, ctx, clock) {
375
389
  finalized,
376
390
  });
377
391
  }
378
- })
379
- .catch((error) => logger.error('env-test sweep failed', {
380
- cron: 'env-test-sweeper',
381
- ...describeError(error),
382
- })));
392
+ }));
383
393
  }
384
394
  }
385
395
  /**
@@ -388,18 +398,12 @@ function redriveStuckEnvTests(env, ctx, clock) {
388
398
  * bounds standing exposure and a finished run's rows are deleted at completion, but
389
399
  * this backstop also clears any that outlived their TTL. The table always exists.
390
400
  */
391
- function reclaimExpiredActivations(env, ctx, clock) {
401
+ function reclaimExpiredActivations(env, tick, clock) {
392
402
  const activations = new D1SubscriptionActivationRepository({ db: env.DB });
393
- ctx.waitUntil(activations
394
- .deleteExpired(clock.now())
395
- .then((reclaimed) => {
403
+ tick.run({ name: 'activation-sweeper', failureMessage: 'activation sweep failed' }, activations.deleteExpired(clock.now()).then((reclaimed) => {
396
404
  if (reclaimed > 0)
397
405
  logger.info('reclaimed activations', { cron: 'activation-sweeper', reclaimed });
398
- })
399
- .catch((error) => logger.error('activation sweep failed', {
400
- cron: 'activation-sweeper',
401
- ...describeError(error),
402
- })));
406
+ }));
403
407
  }
404
408
  /**
405
409
  * Instance-level container reaper: kill any per-run container that outlived its
@@ -410,20 +414,14 @@ function reclaimExpiredActivations(env, ctx, clock) {
410
414
  * EXEC_CONTAINER binding (no Cloudflare API token). With normal runs now self-
411
415
  * reclaiming, a reaped container is a genuine leak — the registry logs each loudly.
412
416
  */
413
- function reapStaleContainers(env, ctx, clock) {
417
+ function reapStaleContainers(env, tick, clock) {
414
418
  if (env.EXEC_CONTAINER) {
415
419
  const reaper = new ContainerInstanceRegistry(env.EXEC_CONTAINER, new D1LiveContainerRepository({ db: env.DB }), clock);
416
420
  const maxAgeMs = loadConfig(env).execution.containerMaxAgeMs;
417
- ctx.waitUntil(reaper
418
- .reapStaleBefore(clock.now() - maxAgeMs)
419
- .then(({ reaped }) => {
421
+ tick.run({ name: 'container-reaper', failureMessage: 'container reap failed' }, reaper.reapStaleBefore(clock.now() - maxAgeMs).then(({ reaped }) => {
420
422
  if (reaped > 0)
421
423
  logger.warn('reaped leaked containers', { cron: 'container-reaper', reaped });
422
- })
423
- .catch((error) => logger.error('container reap failed', {
424
- cron: 'container-reaper',
425
- ...describeError(error),
426
- })));
424
+ }));
427
425
  }
428
426
  }
429
427
  /**
@@ -431,45 +429,30 @@ function reapStaleContainers(env, ctx, clock) {
431
429
  * the initiative loop, Kaizen gradings, GitHub reconcile, environment teardown, and the
432
430
  * platform observability/health sweeps. Each is an independent, no-op-when-unwired sweep.
433
431
  */
434
- function runPeriodicBackstops(env, ctx, clock, scheduledTime) {
432
+ function runPeriodicBackstops(env, tick, clock, scheduledTime) {
435
433
  // Escalate long-waiting notifications yellow → red (every 2 min). Runs no longer
436
434
  // time out waiting for a human, so the escalating notification — past each
437
435
  // workspace's `waitingEscalationMinutes` threshold — is the overdue-human signal.
438
- ctx.waitUntil(escalateStaleNotifications(buildContainer(env), clock.now())
439
- .then((escalated) => {
436
+ tick.run({ name: 'notification-escalation', failureMessage: 'notification escalation failed' }, escalateStaleNotifications(buildContainer(env), clock.now()).then((escalated) => {
440
437
  if (escalated > 0)
441
438
  logger.info('escalated notifications', { cron: 'notification-escalation', escalated });
442
- })
443
- .catch((error) => logger.error('notification escalation failed', {
444
- cron: 'notification-escalation',
445
- ...describeError(error),
446
- })));
439
+ }));
447
440
  // Fire any due recurring pipelines (every 2 min; the actual cadence is hours).
448
441
  // Each due schedule starts its pipeline against its reused block, skipping any
449
442
  // whose block already has an active run. No-op when the feature isn't wired.
450
- ctx.waitUntil(Promise.resolve(buildContainer(env).recurring?.service.runDue(clock.now()))
451
- .then((result) => {
443
+ tick.run({ name: 'recurring-pipelines', failureMessage: 'recurring-pipeline sweep failed' }, Promise.resolve(buildContainer(env).recurring?.service.runDue(clock.now())).then((result) => {
452
444
  if (result && (result.fired > 0 || result.skipped > 0)) {
453
445
  logger.info('fired recurring pipelines', { cron: 'recurring-pipelines', ...result });
454
446
  }
455
- })
456
- .catch((error) => logger.error('recurring-pipeline sweep failed', {
457
- cron: 'recurring-pipelines',
458
- ...describeError(error),
459
- })));
447
+ }));
460
448
  // Tick the initiative execution loop (every 2 min): reconcile each executing initiative's
461
449
  // spawned tasks and spawn the next wave up to its concurrency cap. Terminal child runs poke
462
450
  // the loop directly, so this is the backstop cadence. No-op when initiatives aren't wired.
463
- ctx.waitUntil(Promise.resolve(buildContainer(env).initiatives?.loop.runDue(clock.now()))
464
- .then((result) => {
451
+ tick.run({ name: 'initiative-loop', failureMessage: 'initiative-loop sweep failed' }, Promise.resolve(buildContainer(env).initiatives?.loop.runDue(clock.now())).then((result) => {
465
452
  if (result && (result.spawned > 0 || result.completed > 0)) {
466
453
  logger.info('ticked initiative loop', { cron: 'initiative-loop', ...result });
467
454
  }
468
- })
469
- .catch((error) => logger.error('initiative-loop sweep failed', {
470
- cron: 'initiative-loop',
471
- ...describeError(error),
472
- })));
455
+ }));
473
456
  // Run any pending Kaizen gradings (every 2 min): the engine only inserts `scheduled`
474
457
  // rows at run completion, so this background pass does the actual LLM grading (and
475
458
  // re-drives `running` rows orphaned by a crashed sweep). Bounded per pass to stay
@@ -477,29 +460,26 @@ function runPeriodicBackstops(env, ctx, clock, scheduledTime) {
477
460
  // model is resolved per-workspace (Model Configuration), so this is workspace-wide.
478
461
  if (!kaizenSweeping) {
479
462
  kaizenSweeping = true;
480
- ctx.waitUntil(Promise.resolve(buildContainer(env).kaizen?.service.runPending(clock.now() - KAIZEN_STALE_MS, KAIZEN_SWEEP_BATCH))
463
+ tick.run({ name: 'kaizen-sweeper', failureMessage: 'kaizen sweep failed' }, Promise.resolve(buildContainer(env).kaizen?.service.runPending(clock.now() - KAIZEN_STALE_MS, KAIZEN_SWEEP_BATCH))
481
464
  .then((processed) => {
482
465
  if (processed && processed > 0)
483
466
  logger.info('ran pending kaizen gradings', { cron: 'kaizen-sweeper', processed });
484
467
  })
485
- .catch((error) => logger.error('kaizen sweep failed', { cron: 'kaizen-sweeper', ...describeError(error) }))
468
+ // Released on BOTH paths, and it has to stay inside the promise handed to `tick.run`:
469
+ // the in-flight latch is what stops a second tick piling onto a slow grading pass, so a
470
+ // failed pass that never cleared it would wedge Kaizen until the isolate was recycled.
486
471
  .finally(() => {
487
472
  kaizenSweeping = false;
488
473
  }));
489
474
  }
490
475
  // Reconcile GitHub projections that may have missed a webhook (no-op unless
491
476
  // the integration is configured).
492
- ctx.waitUntil(reconcileStaleRepos(env, clock, GITHUB_RECONCILE_STALE_MS)
493
- .then((scheduled) => {
477
+ tick.run({ name: 'github-reconcile', failureMessage: 'github reconcile failed' }, reconcileStaleRepos(env, clock, GITHUB_RECONCILE_STALE_MS).then((scheduled) => {
494
478
  if (scheduled > 0)
495
479
  // `sweep:` (not `cron:`) so the summary shares a field with the pass's
496
480
  // per-repo lines, which the shared reconcile core emits on both facades.
497
481
  logger.info('scheduled repo resyncs', { sweep: 'github-reconcile', scheduled });
498
- })
499
- .catch((error) => logger.error('github reconcile failed', {
500
- sweep: 'github-reconcile',
501
- ...describeError(error),
502
- })));
482
+ }));
503
483
  // Refresh repo-linked FOUNDATIONAL-SERVICE sources whose last sync has aged out, so a merged
504
484
  // OpenAPI change reaches the catalog without anyone opening the management surface. Runs on the
505
485
  // same hourly cadence as its staleness window through the stateless window gate — a cron isolate
@@ -507,14 +487,11 @@ function runPeriodicBackstops(env, ctx, clock, scheduledTime) {
507
487
  // by 30 for a source that can only go stale once an hour. No-op unless the catalog + GitHub are
508
488
  // both wired (nothing is linked to refresh).
509
489
  if (shouldRunReachabilityPass(scheduledTime, FREQUENT_CRON_PERIOD_MS, FOUNDATIONAL_SOURCE_STALE_MS)) {
510
- ctx.waitUntil(sweepFoundationalSources(buildContainer(env).foundationalServices, logger).catch((error) => logger.error('foundational-source sweep failed', {
511
- sweep: 'foundational-sources',
512
- ...describeError(error),
513
- })));
490
+ tick.run({ name: 'foundational-sources', failureMessage: 'foundational-source sweep failed' }, sweepFoundationalSources(buildContainer(env).foundationalServices, logger));
514
491
  }
515
492
  // Tear down ephemeral environments whose TTL has elapsed (no-op unless the
516
493
  // environment integration is configured).
517
- ctx.waitUntil(sweepExpiredEnvironments(env, clock).catch((error) => logger.error('environment sweep failed', { cron: 'env-sweeper', ...describeError(error) })));
494
+ tick.run({ name: 'env-sweeper', failureMessage: 'environment sweep failed' }, sweepExpiredEnvironments(env, clock));
518
495
  // Push deployment-level (platform-operator) observability aggregates to the OTLP
519
496
  // endpoint as OpenTelemetry gauge metrics, once per cron tick. Opt-in on top of the base
520
497
  // OTel exporter (OTEL_PLATFORM_METRICS); a no-op otherwise. Per account, enumerated from
@@ -531,7 +508,7 @@ function runPeriodicBackstops(env, ctx, clock, scheduledTime) {
531
508
  logger,
532
509
  });
533
510
  if (sweep)
534
- ctx.waitUntil(sweep);
511
+ tick.run({ name: 'platform-metrics', failureMessage: 'platform metrics sweep failed' }, sweep);
535
512
  }
536
513
  // Probe each workspace's CONFIGURED infrastructure connections and report a dead one as
537
514
  // `unreachable` — raising/clearing an `infra_unreachable` card and pushing an `infraSetup` event
@@ -544,55 +521,67 @@ function runPeriodicBackstops(env, ctx, clock, scheduledTime) {
544
521
  const reachability = loadConfig(env).infraReachability;
545
522
  if (reachability.enabled &&
546
523
  shouldRunReachabilityPass(scheduledTime, FREQUENT_CRON_PERIOD_MS, reachability.intervalMs)) {
547
- ctx.waitUntil(sweepInfraReachability(buildContainer(env), logger)
548
- .then(({ raised, cleared }) => {
524
+ tick.run({ name: 'infra-reachability', failureMessage: 'infra reachability sweep failed' }, sweepInfraReachability(buildContainer(env), logger).then(({ raised, cleared }) => {
549
525
  if (raised > 0 || cleared > 0)
550
- logger.info('infra reachability sweep', {
551
- cron: 'infra-reachability',
552
- raised,
553
- cleared,
554
- });
555
- })
556
- .catch((error) => logger.error('infra reachability sweep failed', {
557
- cron: 'infra-reachability',
558
- ...describeError(error),
559
- })));
526
+ logger.info('infra reachability sweep', { cron: 'infra-reachability', raised, cleared });
527
+ }));
560
528
  }
561
529
  // Raise/clear `platform_health` notifications when the deployment's OWN run health crosses
562
530
  // an operator threshold, per account (the push counterpart to the operator dashboard read).
563
531
  // Opt-in (`PLATFORM_ALERTS`); the container (hence the platform-observability read) is built
564
532
  // only when opted in so a deployment that hasn't opted in pays nothing.
565
533
  if (loadConfig(env).platformAlerts.enabled) {
566
- ctx.waitUntil(sweepPlatformHealth(buildContainer(env), logger)
567
- .then(({ raised, cleared }) => {
534
+ tick.run({ name: 'platform-health', failureMessage: 'platform health sweep failed' }, sweepPlatformHealth(buildContainer(env), logger).then(({ raised, cleared }) => {
568
535
  if (raised > 0 || cleared > 0)
569
536
  logger.info('platform health sweep', { cron: 'platform-health', raised, cleared });
570
- })
571
- .catch((error) => logger.error('platform health sweep failed', {
572
- cron: 'platform-health',
573
- ...describeError(error),
574
- })));
537
+ }));
575
538
  }
576
539
  }
577
540
  /** The cron sweeper. Module-level (not closed over the app) so {@link createWorker} stays thin. */
578
541
  async function handleScheduled(controller, env, ctx) {
579
542
  setLogLevel(parseLogLevel(env.LOG_LEVEL));
580
543
  const clock = new SystemClock();
544
+ const tick = new SweepTick(ctx);
581
545
  // Daily pass: prune the unbounded ledgers/projections to their retention windows.
582
546
  if (controller.cron === RETENTION_CRON) {
583
- runDailyRetentionSweeps(env, ctx, clock);
584
- return;
547
+ runDailyRetentionSweeps(env, tick, clock);
548
+ }
549
+ else {
550
+ // Frequent pass (every 2 min): time-sensitive backstops.
551
+ redriveStuckAgentRuns(env, tick, clock);
552
+ redriveStuckEnvTests(env, tick, clock);
553
+ reclaimExpiredActivations(env, tick, clock);
554
+ reapStaleContainers(env, tick, clock);
555
+ runPeriodicBackstops(env, tick, clock, controller.scheduledTime);
585
556
  }
586
- // Frequent pass (every 2 min): time-sensitive backstops.
587
- redriveStuckAgentRuns(env, ctx, clock);
588
- redriveStuckEnvTests(env, ctx, clock);
589
- reclaimExpiredActivations(env, ctx, clock);
590
- reapStaleContainers(env, ctx, clock);
591
- runPeriodicBackstops(env, ctx, clock, controller.scheduledTime);
557
+ // This tick's own counters, flushed once its passes have SETTLED. The ordering is the whole
558
+ // point: the collector is per ISOLATE, so a counter a cron pass records can only be exported
559
+ // by a flush that runs after it. Draining while the passes were still in flight meant a cron
560
+ // tick drained an empty collector and left its own counters waiting for a next tick in the
561
+ // same isolate — which for the daily retention cron is a tick that never comes.
562
+ flushOperationalMetricsAfter(tick.settled(), env, ctx, clock);
563
+ }
564
+ /**
565
+ * Flush THIS ISOLATE's operational counters once `work` settles, as a post-response
566
+ * `waitUntil`. Every entry point calls it, because a Worker's collector is per-isolate and an
567
+ * isolate is discarded without warning — see `observability/operationalFlush.ts` for why that
568
+ * makes per-invocation flushing the correct shape rather than a wasteful one.
569
+ *
570
+ * `allSettled` rather than a catch: a FAILED invocation is exactly the one whose counters
571
+ * matter, and its rejection is the caller's to propagate, not this bookkeeping's to observe.
572
+ */
573
+ function flushOperationalMetricsAfter(work, env, ctx, clock) {
574
+ ctx.waitUntil(Promise.allSettled([work]).then(() => flushOperationalMetricsForIsolate(loadConfig(env).otel, clock.now()) ?? undefined));
592
575
  }
593
576
  /** The queue consumer multiplexing the three queues. Module-level, like {@link handleScheduled}. */
594
- async function handleQueue(batch, env) {
577
+ async function handleQueue(batch, env, ctx) {
595
578
  setLogLevel(parseLogLevel(env.LOG_LEVEL));
579
+ const work = routeQueueBatch(batch, env);
580
+ flushOperationalMetricsAfter(work, env, ctx, new SystemClock());
581
+ await work;
582
+ }
583
+ /** The routing half of {@link handleQueue}, split out so the flush can bracket the whole batch. */
584
+ async function routeQueueBatch(batch, env) {
596
585
  // Route by source queue — the single handler serves all three queues.
597
586
  if (batch.queue === GITHUB_SYNC_QUEUE_NAME) {
598
587
  await handleGitHubSyncBatch(batch, env);
@@ -673,9 +662,13 @@ export function createWorker(options = {}) {
673
662
  pipelineRegistry: registries.pipelineRegistry,
674
663
  taskTypeRegistry: registries.taskTypeRegistry,
675
664
  foundationalServiceRegistry: registries.foundationalServiceRegistry,
665
+ binaryGeneratorRegistry: registries.binaryGeneratorRegistry,
676
666
  onWarn: (problem) => logger.warn(problem.message, { code: problem.code }),
677
667
  });
678
- return app.fetch(request, env, ctx);
668
+ const response = Promise.resolve(app.fetch(request, env, ctx));
669
+ // Flush whatever THIS isolate counted while serving the request, after the response.
670
+ flushOperationalMetricsAfter(response, env, ctx, new SystemClock());
671
+ return response;
679
672
  },
680
673
  scheduled: handleScheduled,
681
674
  queue: handleQueue,