@cat-factory/worker 0.137.0 → 0.138.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +33 -0
- package/dist/app.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +111 -118
- package/dist/index.js.map +1 -1
- package/dist/infrastructure/config/agents.js +2 -2
- package/dist/infrastructure/config/index.d.ts.map +1 -1
- package/dist/infrastructure/config/index.js +4 -0
- package/dist/infrastructure/config/index.js.map +1 -1
- package/dist/infrastructure/container-account-settings.d.ts +13 -0
- package/dist/infrastructure/container-account-settings.d.ts.map +1 -0
- package/dist/infrastructure/container-account-settings.js +34 -0
- package/dist/infrastructure/container-account-settings.js.map +1 -0
- package/dist/infrastructure/container-artifact-storage.d.ts +16 -0
- package/dist/infrastructure/container-artifact-storage.d.ts.map +1 -0
- package/dist/infrastructure/container-artifact-storage.js +40 -0
- package/dist/infrastructure/container-artifact-storage.js.map +1 -0
- package/dist/infrastructure/container-assembly.d.ts.map +1 -1
- package/dist/infrastructure/container-assembly.js +5 -1
- package/dist/infrastructure/container-assembly.js.map +1 -1
- package/dist/infrastructure/container-executor-deps.d.ts.map +1 -1
- package/dist/infrastructure/container-executor-deps.js +4 -1
- package/dist/infrastructure/container-executor-deps.js.map +1 -1
- package/dist/infrastructure/container-shared-services.d.ts +1 -1
- package/dist/infrastructure/container-shared-services.d.ts.map +1 -1
- package/dist/infrastructure/container-shared-services.js +3 -1
- package/dist/infrastructure/container-shared-services.js.map +1 -1
- package/dist/infrastructure/container.d.ts +5 -21
- package/dist/infrastructure/container.d.ts.map +1 -1
- package/dist/infrastructure/container.js +12 -57
- package/dist/infrastructure/container.js.map +1 -1
- package/dist/infrastructure/env.d.ts +4 -0
- package/dist/infrastructure/env.d.ts.map +1 -1
- package/dist/infrastructure/env.js.map +1 -1
- package/dist/infrastructure/observability/cronSweep.d.ts +33 -0
- package/dist/infrastructure/observability/cronSweep.d.ts.map +1 -0
- package/dist/infrastructure/observability/cronSweep.js +41 -0
- package/dist/infrastructure/observability/cronSweep.js.map +1 -0
- package/dist/infrastructure/observability/operationalFlush.d.ts +15 -0
- package/dist/infrastructure/observability/operationalFlush.d.ts.map +1 -0
- package/dist/infrastructure/observability/operationalFlush.js +64 -0
- package/dist/infrastructure/observability/operationalFlush.js.map +1 -0
- package/dist/infrastructure/repositories/D1AgentRunRepository.d.ts +1 -0
- package/dist/infrastructure/repositories/D1AgentRunRepository.d.ts.map +1 -1
- package/dist/infrastructure/repositories/D1AgentRunRepository.js +14 -1
- package/dist/infrastructure/repositories/D1AgentRunRepository.js.map +1 -1
- package/dist/infrastructure/workflows/retention.d.ts +18 -4
- package/dist/infrastructure/workflows/retention.d.ts.map +1 -1
- package/dist/infrastructure/workflows/retention.js +22 -19
- package/dist/infrastructure/workflows/retention.js.map +1 -1
- package/dist/infrastructure/workflows/sweeper.d.ts +11 -2
- package/dist/infrastructure/workflows/sweeper.d.ts.map +1 -1
- package/dist/infrastructure/workflows/sweeper.js +11 -1
- package/dist/infrastructure/workflows/sweeper.js.map +1 -1
- package/migrations/0076_agent_runs_redrive_count.sql +20 -0
- package/package.json +18 -18
package/dist/app.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAU/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAA;AAU3B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,6BAA6B,CAAA;AAEzD,MAAM,WAAW,gBAAgB;IAC/B,+EAA+E;IAC/E,SAAS,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAA;IACrC,mFAAmF;IACnF,uBAAuB,CAAC,EAAE,OAAO,CAAA;IACjC,gFAAgF;IAChF,aAAa,CAAC,EAAE,qBAAqB,CAAA;CACtC;AAqBD;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,IAAI,CAAC,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAU/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAA;AAU3B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,6BAA6B,CAAA;AAEzD,MAAM,WAAW,gBAAgB;IAC/B,+EAA+E;IAC/E,SAAS,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAA;IACrC,mFAAmF;IACnF,uBAAuB,CAAC,EAAE,OAAO,CAAA;IACjC,gFAAgF;IAChF,aAAa,CAAC,EAAE,qBAAqB,CAAA;CACtC;AAqBD;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,IAAI,CAAC,MAAM,CAAC,CA8GtE"}
|
package/dist/app.js
CHANGED
|
@@ -82,6 +82,39 @@ export function createApp(options = {}) {
|
|
|
82
82
|
await next();
|
|
83
83
|
});
|
|
84
84
|
app.get('/health', (c) => c.json({ status: 'ok' }));
|
|
85
|
+
// Readiness (slice 4.3). The Worker has no long-lived process to drain and no rotation to be
|
|
86
|
+
// taken out of — Cloudflare routes every request to a fresh isolate and there is nothing an
|
|
87
|
+
// operator could do with a red answer — so `/ready` here is deliberately NOT the Node
|
|
88
|
+
// facade's drain signal. What it IS: a bindings probe, answering the one question an operator
|
|
89
|
+
// actually asks of a fresh deployment ("is D1 reachable, is TELEMETRY_DB bound"), which on
|
|
90
|
+
// this runtime is a genuinely separate database rather than a schema in the same one.
|
|
91
|
+
//
|
|
92
|
+
// Reported per binding rather than as one boolean, and a failure is 503 so a smoke test can
|
|
93
|
+
// assert on the status alone. Public, before the auth gate, like `/health`.
|
|
94
|
+
app.get('/ready', async (c) => {
|
|
95
|
+
const env = c.env;
|
|
96
|
+
const probe = async (run) => {
|
|
97
|
+
try {
|
|
98
|
+
await run();
|
|
99
|
+
return { ok: true };
|
|
100
|
+
}
|
|
101
|
+
catch (err) {
|
|
102
|
+
// Kept to a short diagnostic: this endpoint is unauthenticated, so it must never carry
|
|
103
|
+
// a connection string or a binding's internals — the same rule as the Node twin's.
|
|
104
|
+
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
const checks = {
|
|
108
|
+
database: await probe(() => c.env.DB.prepare('SELECT 1').first()),
|
|
109
|
+
};
|
|
110
|
+
// TELEMETRY_DB is REQUIRED (the build fails without it), so an absent binding is a real
|
|
111
|
+
// failure to report rather than a capability to skip.
|
|
112
|
+
checks.telemetry = env.TELEMETRY_DB
|
|
113
|
+
? await probe(() => env.TELEMETRY_DB.prepare('SELECT 1').first())
|
|
114
|
+
: { ok: false, error: 'TELEMETRY_DB is not bound' };
|
|
115
|
+
const ready = Object.values(checks).every((check) => check.ok);
|
|
116
|
+
return c.json({ status: ready ? 'ready' : 'not_ready', checks }, ready ? 200 : 503);
|
|
117
|
+
});
|
|
85
118
|
// Default-deny session gate + per-workspace authz, shared verbatim with the Node
|
|
86
119
|
// service (one implementation in @cat-factory/server so the runtimes can't drift).
|
|
87
120
|
mountAuthGate(app);
|
package/dist/app.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,0BAA0B,EAC1B,uBAAuB,EACvB,MAAM,EACN,aAAa,EACb,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAA;AAC3B,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,8BAA8B,CAAA;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAA;AAYhE,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,iFAAiF;AACjF,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAA;AAC1C,SAAS,oBAAoB,CAAC,QAAyB;IACrD,MAAM,SAAS,GAAG,QAAQ;SACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SACjB,IAAI,EAAE;SACN,IAAI,CAAC,GAAG,CAAC,CAAA;IACZ,IAAI,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAM;IAC3C,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IAC/B,MAAM,CAAC,KAAK,CACV,yFAAyF;QACvF,0EAA0E,EAC5E,EAAE,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CACzC,CAAA;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,SAAS,CAAC,OAAO,GAAqB,EAAE;IACtD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAU,CAAA;IAE9B,4FAA4F;IAC5F,6FAA6F;IAC7F,kCAAkC;IAClC,mBAAmB,CAAC,GAAG,CAAC,CAAA;IAExB,6EAA6E;IAC7E,yEAAyE;IACzE,mFAAmF;IACnF,2EAA2E;IAC3E,yFAAyF;IACzF,GAAG,CAAC,GAAG,CACL,GAAG,EACH,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CACpB,iBAAiB,CACf,MAAM,EACN,CAAC,CAAC,GAAG,CAAC,oBAAoB,EAC1B,qBAAqB,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CACzC;QACH,iFAAiF;QACjF,mFAAmF;QACnF,kFAAkF;QAClF,wEAAwE;QACxE,YAAY,EAAE,CAAC,GAAG,oBAAoB,CAAC;QACvC,uFAAuF;QACvF,aAAa,EAAE,CAAC,GAAG,oBAAoB,CAAC;KACzC,CAAC,CACH,CAAA;IACD,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;QAC7B,IAAI,CAAC;YACH,CAAC,CAAC,GAAG,CACH,WAAW,EACX,cAAc,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE;gBACvC,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;gBACxD,aAAa,EAAE,OAAO,CAAC,aAAa;aACrC,CAAC,CACH,CAAA;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6FAA6F;YAC7F,8FAA8F;YAC9F,+FAA+F;YAC/F,sFAAsF;YACtF,IAAI,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;gBAClC,MAAM,GAAG,GAAG,0BAA0B,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAA;gBACjF,yEAAyE;gBACzE,0FAA0F;gBAC1F,2FAA2F;gBAC3F,yFAAyF;gBACzF,0FAA0F;gBAC1F,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;gBACrC,IAAI,MAAM;oBAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,MAAM,CAAC,CAAA;gBAClE,OAAO,GAAG,CAAA;YACZ,CAAC;YACD,MAAM,GAAG,CAAA;QACX,CAAC;QACD,MAAM,IAAI,EAAE,CAAA;IACd,CAAC,CAAC,CAAA;IAEF,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;IAEnD,iFAAiF;IACjF,mFAAmF;IACnF,aAAa,CAAC,GAAG,CAAC,CAAA;IAElB,mFAAmF;IACnF,mFAAmF;IACnF,8EAA8E;IAC9E,gCAAgC;IAChC,uBAAuB,CAAC,GAAG,CAAC,CAAA;IAE5B,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAExB,OAAO,GAAG,CAAA;AACZ,CAAC"}
|
|
1
|
+
{"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,0BAA0B,EAC1B,uBAAuB,EACvB,MAAM,EACN,aAAa,EACb,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAA;AAC3B,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,8BAA8B,CAAA;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAA;AAYhE,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,iFAAiF;AACjF,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAA;AAC1C,SAAS,oBAAoB,CAAC,QAAyB;IACrD,MAAM,SAAS,GAAG,QAAQ;SACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SACjB,IAAI,EAAE;SACN,IAAI,CAAC,GAAG,CAAC,CAAA;IACZ,IAAI,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAM;IAC3C,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IAC/B,MAAM,CAAC,KAAK,CACV,yFAAyF;QACvF,0EAA0E,EAC5E,EAAE,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CACzC,CAAA;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,SAAS,CAAC,OAAO,GAAqB,EAAE;IACtD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAU,CAAA;IAE9B,4FAA4F;IAC5F,6FAA6F;IAC7F,kCAAkC;IAClC,mBAAmB,CAAC,GAAG,CAAC,CAAA;IAExB,6EAA6E;IAC7E,yEAAyE;IACzE,mFAAmF;IACnF,2EAA2E;IAC3E,yFAAyF;IACzF,GAAG,CAAC,GAAG,CACL,GAAG,EACH,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CACpB,iBAAiB,CACf,MAAM,EACN,CAAC,CAAC,GAAG,CAAC,oBAAoB,EAC1B,qBAAqB,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CACzC;QACH,iFAAiF;QACjF,mFAAmF;QACnF,kFAAkF;QAClF,wEAAwE;QACxE,YAAY,EAAE,CAAC,GAAG,oBAAoB,CAAC;QACvC,uFAAuF;QACvF,aAAa,EAAE,CAAC,GAAG,oBAAoB,CAAC;KACzC,CAAC,CACH,CAAA;IACD,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;QAC7B,IAAI,CAAC;YACH,CAAC,CAAC,GAAG,CACH,WAAW,EACX,cAAc,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE;gBACvC,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;gBACxD,aAAa,EAAE,OAAO,CAAC,aAAa;aACrC,CAAC,CACH,CAAA;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6FAA6F;YAC7F,8FAA8F;YAC9F,+FAA+F;YAC/F,sFAAsF;YACtF,IAAI,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;gBAClC,MAAM,GAAG,GAAG,0BAA0B,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAA;gBACjF,yEAAyE;gBACzE,0FAA0F;gBAC1F,2FAA2F;gBAC3F,yFAAyF;gBACzF,0FAA0F;gBAC1F,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;gBACrC,IAAI,MAAM;oBAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,MAAM,CAAC,CAAA;gBAClE,OAAO,GAAG,CAAA;YACZ,CAAC;YACD,MAAM,GAAG,CAAA;QACX,CAAC;QACD,MAAM,IAAI,EAAE,CAAA;IACd,CAAC,CAAC,CAAA;IAEF,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;IAEnD,6FAA6F;IAC7F,4FAA4F;IAC5F,sFAAsF;IACtF,8FAA8F;IAC9F,2FAA2F;IAC3F,sFAAsF;IACtF,EAAE;IACF,4FAA4F;IAC5F,4EAA4E;IAC5E,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAA;QACjB,MAAM,KAAK,GAAG,KAAK,EAAE,GAA2B,EAA4C,EAAE;YAC5F,IAAI,CAAC;gBACH,MAAM,GAAG,EAAE,CAAA;gBACX,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAA;YACrB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,uFAAuF;gBACvF,mFAAmF;gBACnF,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;YAC/E,CAAC;QACH,CAAC,CAAA;QACD,MAAM,MAAM,GAAoD;YAC9D,QAAQ,EAAE,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,CAAC;SAClE,CAAA;QACD,wFAAwF;QACxF,sDAAsD;QACtD,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,YAAY;YACjC,CAAC,CAAC,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,YAAa,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,CAAC;YAClE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,2BAA2B,EAAE,CAAA;QACrD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACrF,CAAC,CAAC,CAAA;IAEF,iFAAiF;IACjF,mFAAmF;IACnF,aAAa,CAAC,GAAG,CAAC,CAAA;IAElB,mFAAmF;IACnF,mFAAmF;IACnF,8EAA8E;IAC9E,gCAAgC;IAChC,uBAAuB,CAAC,GAAG,CAAC,CAAA;IAE5B,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAExB,OAAO,GAAG,CAAA;AACZ,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -16,12 +16,13 @@ export { InitiativePresetRegistry, type InitiativePresetRegistration } from '@ca
|
|
|
16
16
|
export { PipelineRegistry, defaultPipelineRegistry } from '@cat-factory/kernel';
|
|
17
17
|
export { TaskTypeRegistry, defaultTaskTypeRegistry } from '@cat-factory/kernel';
|
|
18
18
|
export { FoundationalServiceRegistry, type FoundationalServiceDefinition, defaultFoundationalServiceRegistry, } from '@cat-factory/kernel';
|
|
19
|
+
export { BinaryGeneratorRegistry, type BinaryGeneratorDefinition, defaultBinaryGeneratorRegistry, } from '@cat-factory/kernel';
|
|
19
20
|
export type { CreateAppOptions } from './app';
|
|
20
21
|
export { DEFAULT_MODEL_PRESET_ID, MODEL_PRESET_SEED_IDS } from '@cat-factory/kernel';
|
|
21
22
|
/** The cron sweeper. Module-level (not closed over the app) so {@link createWorker} stays thin. */
|
|
22
23
|
declare function handleScheduled(controller: ScheduledController, env: Env, ctx: ExecutionContext): Promise<void>;
|
|
23
24
|
/** The queue consumer multiplexing the three queues. Module-level, like {@link handleScheduled}. */
|
|
24
|
-
declare function handleQueue(batch: MessageBatch<ExecutionStartMessage | GitHubSyncMessage | TrackerSyncMessage>, env: Env): Promise<void>;
|
|
25
|
+
declare function handleQueue(batch: MessageBatch<ExecutionStartMessage | GitHubSyncMessage | TrackerSyncMessage>, env: Env, ctx: ExecutionContext): Promise<void>;
|
|
25
26
|
/** The Worker's exported handler shape (fetch + the cron and queue consumers). */
|
|
26
27
|
export type WorkerHandler = ExportedHandler<Env, ExecutionStartMessage | GitHubSyncMessage | TrackerSyncMessage>;
|
|
27
28
|
/**
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,eAAe,EACf,YAAY,EACZ,mBAAmB,EACpB,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,KAAK,gBAAgB,EAAa,MAAM,OAAO,CAAA;AAExD,OAAO,KAAK,EACV,GAAG,EACH,qBAAqB,EACrB,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,sBAAsB,CAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,eAAe,EACf,YAAY,EACZ,mBAAmB,EACpB,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,KAAK,gBAAgB,EAAa,MAAM,OAAO,CAAA;AAExD,OAAO,KAAK,EACV,GAAG,EACH,qBAAqB,EACrB,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,sBAAsB,CAAA;AAmF7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAA;AAChF,OAAO,EAAE,sBAAsB,EAAE,MAAM,mDAAmD,CAAA;AAC1F,OAAO,EAAE,iBAAiB,EAAE,MAAM,8CAA8C,CAAA;AAChF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oDAAoD,CAAA;AAC5F,OAAO,EAAE,uBAAuB,EAAE,MAAM,oDAAoD,CAAA;AAE5F,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAA;AAGnF,OAAO,EAAE,eAAe,EAAE,MAAM,6CAA6C,CAAA;AAE7E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qDAAqD,CAAA;AAKxF,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,KAAK,oBAAoB,GAC1B,MAAM,gCAAgC,CAAA;AAOvC,OAAO,EACL,iBAAiB,EACjB,wBAAwB,EACxB,KAAK,mBAAmB,GACzB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAAE,+BAA+B,EAAE,MAAM,qBAAqB,CAAA;AACrE,OAAO,EAAE,wBAAwB,EAAE,KAAK,4BAA4B,EAAE,MAAM,qBAAqB,CAAA;AAIjG,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAA;AAK/E,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAA;AAO/E,OAAO,EACL,2BAA2B,EAC3B,KAAK,6BAA6B,EAClC,kCAAkC,GACnC,MAAM,qBAAqB,CAAA;AAO5B,OAAO,EACL,uBAAuB,EACvB,KAAK,yBAAyB,EAC9B,8BAA8B,GAC/B,MAAM,qBAAqB,CAAA;AAG5B,YAAY,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAA;AAK7C,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AAyiBpF,mGAAmG;AACnG,iBAAe,eAAe,CAC5B,UAAU,EAAE,mBAAmB,EAC/B,GAAG,EAAE,GAAG,EACR,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC,CAsBf;AAwBD,oGAAoG;AACpG,iBAAe,WAAW,CACxB,KAAK,EAAE,YAAY,CAAC,qBAAqB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC,EACnF,GAAG,EAAE,GAAG,EACR,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC,CAKf;AAiDD,kFAAkF;AAClF,MAAM,MAAM,aAAa,GAAG,eAAe,CACzC,GAAG,EACH,qBAAqB,GAAG,iBAAiB,GAAG,kBAAkB,CAC/D,CAAA;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,YAAY,CAAC,OAAO,GAAE,gBAAqB,GAAG,aAAa,CA8B1E;AAiBD;;;;;;GAMG;;qBAIgB,UAAU,CAAC,WAAW,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -17,7 +17,7 @@ import { D1SubscriptionQuotaCycleRepository } from './infrastructure/repositorie
|
|
|
17
17
|
import { D1PasswordResetTokenRepository } from './infrastructure/repositories/D1PasswordResetTokenRepository';
|
|
18
18
|
import { D1NotificationRepository } from './infrastructure/repositories/D1NotificationRepository';
|
|
19
19
|
import { buildContainer, buildCloudflareArtifactStoreResolver } from './infrastructure/container';
|
|
20
|
-
import { GITHUB_RECONCILE_STALE_MS, escalateStaleNotifications, FOUNDATIONAL_SOURCE_STALE_MS, shouldRunReachabilityPass, sweepFoundationalSources, sweepInfraReachability, sweepPlatformHealth, } from '@cat-factory/server';
|
|
20
|
+
import { GITHUB_RECONCILE_STALE_MS, escalateStaleNotifications, FOUNDATIONAL_SOURCE_STALE_MS, shouldRunReachabilityPass, sweepFoundationalSources, sweepInfraReachability, sweepPlatformHealth, operationalMetrics, } from '@cat-factory/server';
|
|
21
21
|
import { CryptoIdGenerator, SystemClock } from './infrastructure/runtime';
|
|
22
22
|
import { WorkflowsWorkRunner } from './infrastructure/workflows/WorkflowsWorkRunner';
|
|
23
23
|
import { WorkflowsBootstrapRunner } from './infrastructure/workflows/WorkflowsBootstrapRunner';
|
|
@@ -30,10 +30,12 @@ import { handleGitHubSyncBatch, handleTrackerSyncBatch, reconcileStaleRepos, } f
|
|
|
30
30
|
import { sweepExpiredEnvironments } from './infrastructure/environments/sweep';
|
|
31
31
|
import { logger } from './infrastructure/observability/logger';
|
|
32
32
|
import { runPlatformMetricsSweep } from './infrastructure/observability/platformMetrics';
|
|
33
|
+
import { flushOperationalMetricsForIsolate } from './infrastructure/observability/operationalFlush';
|
|
34
|
+
import { SweepTick } from './infrastructure/observability/cronSweep';
|
|
33
35
|
import { defaultJudgeRegistry, defaultStepResolverRegistry, sweepBinaryArtifactRetention, validateRegistrationsOnce, } from '@cat-factory/orchestration';
|
|
34
36
|
import { defaultAgentKindRegistry, defaultInitiativePresetRegistry } from '@cat-factory/agents';
|
|
35
37
|
import { gateRegistryWithBuiltins } from '@cat-factory/gates';
|
|
36
|
-
import { DEFAULT_WORKSPACE_SETTINGS, defaultFoundationalServiceRegistry, defaultPipelineRegistry, defaultTaskTypeRegistry, describeError, } from '@cat-factory/kernel';
|
|
38
|
+
import { DEFAULT_WORKSPACE_SETTINGS, defaultBinaryGeneratorRegistry, defaultFoundationalServiceRegistry, defaultPipelineRegistry, defaultTaskTypeRegistry, describeError, } from '@cat-factory/kernel';
|
|
37
39
|
import { D1WorkspaceRepository } from './infrastructure/repositories/D1WorkspaceRepository';
|
|
38
40
|
import { D1WorkspaceSettingsRepository } from './infrastructure/repositories/D1WorkspaceSettingsRepository';
|
|
39
41
|
import { D1KeyFingerprintStore } from './infrastructure/repositories/D1KeyFingerprintStore';
|
|
@@ -87,6 +89,13 @@ export { TaskTypeRegistry, defaultTaskTypeRegistry } from '@cat-factory/kernel';
|
|
|
87
89
|
// against the estate from its first request and an account/board can still override or suppress
|
|
88
90
|
// any of them. See backend/docs/adr/0031-foundational-services.md.
|
|
89
91
|
export { FoundationalServiceRegistry, defaultFoundationalServiceRegistry, } from '@cat-factory/kernel';
|
|
92
|
+
// Installation-level extension point for GENERATIVE BINARY INTEGRATIONS (the same DI seam once
|
|
93
|
+
// more): a deployment news a `defaultBinaryGeneratorRegistry()`, registers the image / music /
|
|
94
|
+
// video generation APIs it pays for on it by reference, and injects it via the
|
|
95
|
+
// `binaryGeneratorRegistry` override. A pipeline step whose kind carries the `binary-output`
|
|
96
|
+
// trait then SELECTS from them (`stepOptions.binaryOutput.generatorIds`), and the engine briefs
|
|
97
|
+
// the agent on each one's content types, contract and credential variable.
|
|
98
|
+
export { BinaryGeneratorRegistry, defaultBinaryGeneratorRegistry, } from '@cat-factory/kernel';
|
|
90
99
|
// The built-in model-preset ids + the catalog fallback default. A custom Worker entry that builds
|
|
91
100
|
// its own app can seed a different out-of-the-box default with
|
|
92
101
|
// `createApp({ overrides: { defaultModelPresetId: MODEL_PRESET_SEED_IDS.claude } })` (a
|
|
@@ -125,6 +134,10 @@ function resolveEntryRegistries(overrides) {
|
|
|
125
134
|
// capabilities); registered ones are the `builtin` tier of every workspace catalog, and a
|
|
126
135
|
// malformed contract document fails boot rather than a design dispatch.
|
|
127
136
|
foundationalServiceRegistry: overrides.foundationalServiceRegistry ?? defaultFoundationalServiceRegistry(),
|
|
137
|
+
// Generative binary integrations (empty by default — the platform ships none). Registered
|
|
138
|
+
// ones are what a binary-generating step may produce with, and a malformed definition or a
|
|
139
|
+
// cleartext endpoint fails boot rather than a dispatch.
|
|
140
|
+
binaryGeneratorRegistry: overrides.binaryGeneratorRegistry ?? defaultBinaryGeneratorRegistry(),
|
|
128
141
|
};
|
|
129
142
|
}
|
|
130
143
|
/** The boot/cron key-fingerprint check's logger, tagged so its lines are greppable by cron. */
|
|
@@ -179,29 +192,26 @@ const FREQUENT_CRON_PERIOD_MS = 2 * 60_000;
|
|
|
179
192
|
* windows. The tables exist regardless of whether GitHub/agents are
|
|
180
193
|
* configured, so this runs unconditionally; an unused table reclaims nothing.
|
|
181
194
|
*/
|
|
182
|
-
function runDailyRetentionSweeps(env,
|
|
195
|
+
function runDailyRetentionSweeps(env, tick, clock) {
|
|
183
196
|
// ADR 0026 D6.1: the Worker has no boot moment, so the O(1) ENCRYPTION_KEY drift check
|
|
184
197
|
// rides the daily cron. It seeds the fingerprint on first run and logs a definitive
|
|
185
198
|
// drift signal on a key change. Independent of (and cheaper than) the retention work.
|
|
186
199
|
if (env.ENCRYPTION_KEY) {
|
|
187
200
|
const encryptionKey = env.ENCRYPTION_KEY;
|
|
188
|
-
|
|
201
|
+
tick.run({ name: 'key-fingerprint', failureMessage: 'key fingerprint check failed' }, checkKeyFingerprint({
|
|
189
202
|
store: new D1KeyFingerprintStore({ db: env.DB }),
|
|
190
203
|
masterKeyBase64: encryptionKey,
|
|
191
204
|
logger: keyFingerprintLogger,
|
|
192
|
-
})
|
|
193
|
-
cron: 'key-fingerprint',
|
|
194
|
-
...describeError(error),
|
|
195
|
-
})));
|
|
205
|
+
}));
|
|
196
206
|
// ADR 0026 D6.2: the drift sweep — decrypt every sealed credential and raise/clear ONE
|
|
197
207
|
// `key_drift` card per affected workspace. Rides the same daily cron as the fingerprint.
|
|
198
|
-
|
|
208
|
+
tick.run({ name: 'key-drift', failureMessage: 'key drift sweep failed' }, sweepKeyDriftAndRaise(buildContainer(env), (info) => new WebCryptoSecretCipher({ masterKeyBase64: encryptionKey, info }), keyFingerprintLogger));
|
|
199
209
|
}
|
|
200
210
|
// This branch never calls buildContainer (no request container is built for the
|
|
201
211
|
// sweep), so do the same fail-fast the build does: a clear error beats an opaque
|
|
202
212
|
// NPE deep in a telemetry repo when the binding is unbound.
|
|
203
213
|
const telemetryDb = requireTelemetryDb(env);
|
|
204
|
-
|
|
214
|
+
tick.run({ name: 'retention', failureMessage: 'retention sweep failed' }, sweepRetention({
|
|
205
215
|
tokenUsageRepository: new D1TokenUsageRepository({ db: env.DB }),
|
|
206
216
|
rateLimitRepository: new D1RateLimitRepository({
|
|
207
217
|
db: env.DB,
|
|
@@ -229,16 +239,25 @@ function runDailyRetentionSweeps(env, ctx, clock) {
|
|
|
229
239
|
: {}),
|
|
230
240
|
clock,
|
|
231
241
|
policy: loadConfig(env).retention,
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
242
|
+
logger,
|
|
243
|
+
}).then(({ failedTables, ...reclaimed }) => {
|
|
244
|
+
logger.info('retention sweep complete', { cron: 'retention', ...reclaimed });
|
|
245
|
+
// Each failure was already warned at its table; this names the SET, so a table that
|
|
246
|
+
// has been failing every night is one greppable line rather than a pattern to notice.
|
|
247
|
+
if (failedTables.length > 0) {
|
|
248
|
+
logger.warn('retention sweep could not prune every table', {
|
|
249
|
+
cron: 'retention',
|
|
250
|
+
failedTables,
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
}));
|
|
235
254
|
// Binary-artifact retention (UI screenshots + reference designs) is per-workspace, and
|
|
236
255
|
// the blob backend is per-account (R2 or S3), so it resolves each workspace's store. Run
|
|
237
256
|
// whenever storage could be configured: the R2 default (ARTIFACT_BUCKET) OR a per-account
|
|
238
257
|
// S3 backend (which needs the encryption key to unseal its credentials).
|
|
239
258
|
if (env.ARTIFACT_BUCKET || env.ENCRYPTION_KEY) {
|
|
240
259
|
const settingsRepo = new D1WorkspaceSettingsRepository({ db: env.DB });
|
|
241
|
-
|
|
260
|
+
tick.run({ name: 'artifact-retention', failureMessage: 'artifact retention sweep failed' }, sweepBinaryArtifactRetention({
|
|
242
261
|
resolveStore: buildCloudflareArtifactStoreResolver(env, env.DB, clock, new CryptoIdGenerator()),
|
|
243
262
|
listWorkspaceIds: () => new D1WorkspaceRepository({ db: env.DB })
|
|
244
263
|
.listVisible(null)
|
|
@@ -247,14 +266,9 @@ function runDailyRetentionSweeps(env, ctx, clock) {
|
|
|
247
266
|
.get(workspaceId)
|
|
248
267
|
.then((s) => s?.artifactRetentionDays ?? DEFAULT_WORKSPACE_SETTINGS.artifactRetentionDays),
|
|
249
268
|
now: clock.now(),
|
|
250
|
-
})
|
|
251
|
-
.then((removed) => logger.info('artifact retention sweep complete', {
|
|
269
|
+
}).then((removed) => logger.info('artifact retention sweep complete', {
|
|
252
270
|
cron: 'retention',
|
|
253
271
|
binaryArtifacts: removed,
|
|
254
|
-
}))
|
|
255
|
-
.catch((error) => logger.error('artifact retention sweep failed', {
|
|
256
|
-
cron: 'retention',
|
|
257
|
-
...describeError(error),
|
|
258
272
|
})));
|
|
259
273
|
}
|
|
260
274
|
}
|
|
@@ -262,7 +276,7 @@ function runDailyRetentionSweeps(env, ctx, clock) {
|
|
|
262
276
|
* Re-drive any agent run — execution OR bootstrap — whose Workflows instance
|
|
263
277
|
* died. One sweep over the unified agent_runs table dispatches by kind.
|
|
264
278
|
*/
|
|
265
|
-
function redriveStuckAgentRuns(env,
|
|
279
|
+
function redriveStuckAgentRuns(env, tick, clock) {
|
|
266
280
|
if (env.EXECUTION_WORKFLOW || env.BOOTSTRAP_WORKFLOW || env.ENV_CONFIG_REPAIR_WORKFLOW) {
|
|
267
281
|
const execLookup = env.EXECUTION_WORKFLOW ? new WorkflowsLookup(env.EXECUTION_WORKFLOW) : null;
|
|
268
282
|
const bootLookup = env.BOOTSTRAP_WORKFLOW ? new WorkflowsLookup(env.BOOTSTRAP_WORKFLOW) : null;
|
|
@@ -278,7 +292,7 @@ function redriveStuckAgentRuns(env, ctx, clock) {
|
|
|
278
292
|
const repairRunner = env.ENV_CONFIG_REPAIR_WORKFLOW
|
|
279
293
|
? new WorkflowsEnvConfigRepairRunner(env.ENV_CONFIG_REPAIR_WORKFLOW)
|
|
280
294
|
: null;
|
|
281
|
-
|
|
295
|
+
tick.run({ name: 'stale-run', failureMessage: 'run sweep failed' }, sweepStuckRuns({
|
|
282
296
|
agentRunRepository: new D1AgentRunRepository({ db: env.DB }),
|
|
283
297
|
instanceState: (ref) => {
|
|
284
298
|
const lookup = ref.kind === 'bootstrap'
|
|
@@ -336,15 +350,16 @@ function redriveStuckAgentRuns(env, ctx, clock) {
|
|
|
336
350
|
leaseMs: SWEEP_LEASE_MS,
|
|
337
351
|
hardStallMs: SWEEP_HARD_STALL_MS,
|
|
338
352
|
orphanedSince: runSweepOrphanedSince,
|
|
353
|
+
metrics: operationalMetrics,
|
|
354
|
+
logger,
|
|
339
355
|
})
|
|
340
356
|
// Surface what the sweep did — the key signal for "are runs getting stuck?"
|
|
341
357
|
// Only log when it actually acted.
|
|
342
358
|
.then(({ redriven, finalized, stalled }) => {
|
|
343
359
|
if (redriven > 0 || finalized > 0 || stalled > 0) {
|
|
344
|
-
logger.warn('swept stuck runs', { cron: 'run
|
|
360
|
+
logger.warn('swept stuck runs', { cron: 'stale-run', redriven, finalized, stalled });
|
|
345
361
|
}
|
|
346
|
-
})
|
|
347
|
-
.catch((error) => logger.error('run sweep failed', { cron: 'run-sweeper', ...describeError(error) })));
|
|
362
|
+
}));
|
|
348
363
|
}
|
|
349
364
|
}
|
|
350
365
|
/**
|
|
@@ -352,11 +367,11 @@ function redriveStuckAgentRuns(env, ctx, clock) {
|
|
|
352
367
|
* sweep never sees them — this sibling sweep re-drives a run whose Workflows instance
|
|
353
368
|
* was lost and finalizes (cleanup + failed) one whose instance is terminal.
|
|
354
369
|
*/
|
|
355
|
-
function redriveStuckEnvTests(env,
|
|
370
|
+
function redriveStuckEnvTests(env, tick, clock) {
|
|
356
371
|
if (env.ENV_TEST_WORKFLOW) {
|
|
357
372
|
const envTestLookup = new WorkflowsLookup(env.ENV_TEST_WORKFLOW);
|
|
358
373
|
const envTestRunner = new WorkflowsEnvironmentTestRunner(env.ENV_TEST_WORKFLOW);
|
|
359
|
-
|
|
374
|
+
tick.run({ name: 'env-test-sweeper', failureMessage: 'env-test sweep failed' }, sweepStuckEnvTests({
|
|
360
375
|
repository: new D1EnvironmentTestRunRepository({ db: env.DB }),
|
|
361
376
|
instanceState: (runId) => envTestLookup.instanceState(runId),
|
|
362
377
|
redrive: (workspaceId, runId) => envTestRunner.startRun(workspaceId, runId),
|
|
@@ -366,8 +381,7 @@ function redriveStuckEnvTests(env, ctx, clock) {
|
|
|
366
381
|
},
|
|
367
382
|
clock,
|
|
368
383
|
leaseMs: SWEEP_LEASE_MS,
|
|
369
|
-
})
|
|
370
|
-
.then(({ redriven, finalized }) => {
|
|
384
|
+
}).then(({ redriven, finalized }) => {
|
|
371
385
|
if (redriven > 0 || finalized > 0) {
|
|
372
386
|
logger.warn('swept stuck env-test runs', {
|
|
373
387
|
cron: 'env-test-sweeper',
|
|
@@ -375,11 +389,7 @@ function redriveStuckEnvTests(env, ctx, clock) {
|
|
|
375
389
|
finalized,
|
|
376
390
|
});
|
|
377
391
|
}
|
|
378
|
-
})
|
|
379
|
-
.catch((error) => logger.error('env-test sweep failed', {
|
|
380
|
-
cron: 'env-test-sweeper',
|
|
381
|
-
...describeError(error),
|
|
382
|
-
})));
|
|
392
|
+
}));
|
|
383
393
|
}
|
|
384
394
|
}
|
|
385
395
|
/**
|
|
@@ -388,18 +398,12 @@ function redriveStuckEnvTests(env, ctx, clock) {
|
|
|
388
398
|
* bounds standing exposure and a finished run's rows are deleted at completion, but
|
|
389
399
|
* this backstop also clears any that outlived their TTL. The table always exists.
|
|
390
400
|
*/
|
|
391
|
-
function reclaimExpiredActivations(env,
|
|
401
|
+
function reclaimExpiredActivations(env, tick, clock) {
|
|
392
402
|
const activations = new D1SubscriptionActivationRepository({ db: env.DB });
|
|
393
|
-
|
|
394
|
-
.deleteExpired(clock.now())
|
|
395
|
-
.then((reclaimed) => {
|
|
403
|
+
tick.run({ name: 'activation-sweeper', failureMessage: 'activation sweep failed' }, activations.deleteExpired(clock.now()).then((reclaimed) => {
|
|
396
404
|
if (reclaimed > 0)
|
|
397
405
|
logger.info('reclaimed activations', { cron: 'activation-sweeper', reclaimed });
|
|
398
|
-
})
|
|
399
|
-
.catch((error) => logger.error('activation sweep failed', {
|
|
400
|
-
cron: 'activation-sweeper',
|
|
401
|
-
...describeError(error),
|
|
402
|
-
})));
|
|
406
|
+
}));
|
|
403
407
|
}
|
|
404
408
|
/**
|
|
405
409
|
* Instance-level container reaper: kill any per-run container that outlived its
|
|
@@ -410,20 +414,14 @@ function reclaimExpiredActivations(env, ctx, clock) {
|
|
|
410
414
|
* EXEC_CONTAINER binding (no Cloudflare API token). With normal runs now self-
|
|
411
415
|
* reclaiming, a reaped container is a genuine leak — the registry logs each loudly.
|
|
412
416
|
*/
|
|
413
|
-
function reapStaleContainers(env,
|
|
417
|
+
function reapStaleContainers(env, tick, clock) {
|
|
414
418
|
if (env.EXEC_CONTAINER) {
|
|
415
419
|
const reaper = new ContainerInstanceRegistry(env.EXEC_CONTAINER, new D1LiveContainerRepository({ db: env.DB }), clock);
|
|
416
420
|
const maxAgeMs = loadConfig(env).execution.containerMaxAgeMs;
|
|
417
|
-
|
|
418
|
-
.reapStaleBefore(clock.now() - maxAgeMs)
|
|
419
|
-
.then(({ reaped }) => {
|
|
421
|
+
tick.run({ name: 'container-reaper', failureMessage: 'container reap failed' }, reaper.reapStaleBefore(clock.now() - maxAgeMs).then(({ reaped }) => {
|
|
420
422
|
if (reaped > 0)
|
|
421
423
|
logger.warn('reaped leaked containers', { cron: 'container-reaper', reaped });
|
|
422
|
-
})
|
|
423
|
-
.catch((error) => logger.error('container reap failed', {
|
|
424
|
-
cron: 'container-reaper',
|
|
425
|
-
...describeError(error),
|
|
426
|
-
})));
|
|
424
|
+
}));
|
|
427
425
|
}
|
|
428
426
|
}
|
|
429
427
|
/**
|
|
@@ -431,45 +429,30 @@ function reapStaleContainers(env, ctx, clock) {
|
|
|
431
429
|
* the initiative loop, Kaizen gradings, GitHub reconcile, environment teardown, and the
|
|
432
430
|
* platform observability/health sweeps. Each is an independent, no-op-when-unwired sweep.
|
|
433
431
|
*/
|
|
434
|
-
function runPeriodicBackstops(env,
|
|
432
|
+
function runPeriodicBackstops(env, tick, clock, scheduledTime) {
|
|
435
433
|
// Escalate long-waiting notifications yellow → red (every 2 min). Runs no longer
|
|
436
434
|
// time out waiting for a human, so the escalating notification — past each
|
|
437
435
|
// workspace's `waitingEscalationMinutes` threshold — is the overdue-human signal.
|
|
438
|
-
|
|
439
|
-
.then((escalated) => {
|
|
436
|
+
tick.run({ name: 'notification-escalation', failureMessage: 'notification escalation failed' }, escalateStaleNotifications(buildContainer(env), clock.now()).then((escalated) => {
|
|
440
437
|
if (escalated > 0)
|
|
441
438
|
logger.info('escalated notifications', { cron: 'notification-escalation', escalated });
|
|
442
|
-
})
|
|
443
|
-
.catch((error) => logger.error('notification escalation failed', {
|
|
444
|
-
cron: 'notification-escalation',
|
|
445
|
-
...describeError(error),
|
|
446
|
-
})));
|
|
439
|
+
}));
|
|
447
440
|
// Fire any due recurring pipelines (every 2 min; the actual cadence is hours).
|
|
448
441
|
// Each due schedule starts its pipeline against its reused block, skipping any
|
|
449
442
|
// whose block already has an active run. No-op when the feature isn't wired.
|
|
450
|
-
|
|
451
|
-
.then((result) => {
|
|
443
|
+
tick.run({ name: 'recurring-pipelines', failureMessage: 'recurring-pipeline sweep failed' }, Promise.resolve(buildContainer(env).recurring?.service.runDue(clock.now())).then((result) => {
|
|
452
444
|
if (result && (result.fired > 0 || result.skipped > 0)) {
|
|
453
445
|
logger.info('fired recurring pipelines', { cron: 'recurring-pipelines', ...result });
|
|
454
446
|
}
|
|
455
|
-
})
|
|
456
|
-
.catch((error) => logger.error('recurring-pipeline sweep failed', {
|
|
457
|
-
cron: 'recurring-pipelines',
|
|
458
|
-
...describeError(error),
|
|
459
|
-
})));
|
|
447
|
+
}));
|
|
460
448
|
// Tick the initiative execution loop (every 2 min): reconcile each executing initiative's
|
|
461
449
|
// spawned tasks and spawn the next wave up to its concurrency cap. Terminal child runs poke
|
|
462
450
|
// the loop directly, so this is the backstop cadence. No-op when initiatives aren't wired.
|
|
463
|
-
|
|
464
|
-
.then((result) => {
|
|
451
|
+
tick.run({ name: 'initiative-loop', failureMessage: 'initiative-loop sweep failed' }, Promise.resolve(buildContainer(env).initiatives?.loop.runDue(clock.now())).then((result) => {
|
|
465
452
|
if (result && (result.spawned > 0 || result.completed > 0)) {
|
|
466
453
|
logger.info('ticked initiative loop', { cron: 'initiative-loop', ...result });
|
|
467
454
|
}
|
|
468
|
-
})
|
|
469
|
-
.catch((error) => logger.error('initiative-loop sweep failed', {
|
|
470
|
-
cron: 'initiative-loop',
|
|
471
|
-
...describeError(error),
|
|
472
|
-
})));
|
|
455
|
+
}));
|
|
473
456
|
// Run any pending Kaizen gradings (every 2 min): the engine only inserts `scheduled`
|
|
474
457
|
// rows at run completion, so this background pass does the actual LLM grading (and
|
|
475
458
|
// re-drives `running` rows orphaned by a crashed sweep). Bounded per pass to stay
|
|
@@ -477,29 +460,26 @@ function runPeriodicBackstops(env, ctx, clock, scheduledTime) {
|
|
|
477
460
|
// model is resolved per-workspace (Model Configuration), so this is workspace-wide.
|
|
478
461
|
if (!kaizenSweeping) {
|
|
479
462
|
kaizenSweeping = true;
|
|
480
|
-
|
|
463
|
+
tick.run({ name: 'kaizen-sweeper', failureMessage: 'kaizen sweep failed' }, Promise.resolve(buildContainer(env).kaizen?.service.runPending(clock.now() - KAIZEN_STALE_MS, KAIZEN_SWEEP_BATCH))
|
|
481
464
|
.then((processed) => {
|
|
482
465
|
if (processed && processed > 0)
|
|
483
466
|
logger.info('ran pending kaizen gradings', { cron: 'kaizen-sweeper', processed });
|
|
484
467
|
})
|
|
485
|
-
|
|
468
|
+
// Released on BOTH paths, and it has to stay inside the promise handed to `tick.run`:
|
|
469
|
+
// the in-flight latch is what stops a second tick piling onto a slow grading pass, so a
|
|
470
|
+
// failed pass that never cleared it would wedge Kaizen until the isolate was recycled.
|
|
486
471
|
.finally(() => {
|
|
487
472
|
kaizenSweeping = false;
|
|
488
473
|
}));
|
|
489
474
|
}
|
|
490
475
|
// Reconcile GitHub projections that may have missed a webhook (no-op unless
|
|
491
476
|
// the integration is configured).
|
|
492
|
-
|
|
493
|
-
.then((scheduled) => {
|
|
477
|
+
tick.run({ name: 'github-reconcile', failureMessage: 'github reconcile failed' }, reconcileStaleRepos(env, clock, GITHUB_RECONCILE_STALE_MS).then((scheduled) => {
|
|
494
478
|
if (scheduled > 0)
|
|
495
479
|
// `sweep:` (not `cron:`) so the summary shares a field with the pass's
|
|
496
480
|
// per-repo lines, which the shared reconcile core emits on both facades.
|
|
497
481
|
logger.info('scheduled repo resyncs', { sweep: 'github-reconcile', scheduled });
|
|
498
|
-
})
|
|
499
|
-
.catch((error) => logger.error('github reconcile failed', {
|
|
500
|
-
sweep: 'github-reconcile',
|
|
501
|
-
...describeError(error),
|
|
502
|
-
})));
|
|
482
|
+
}));
|
|
503
483
|
// Refresh repo-linked FOUNDATIONAL-SERVICE sources whose last sync has aged out, so a merged
|
|
504
484
|
// OpenAPI change reaches the catalog without anyone opening the management surface. Runs on the
|
|
505
485
|
// same hourly cadence as its staleness window through the stateless window gate — a cron isolate
|
|
@@ -507,14 +487,11 @@ function runPeriodicBackstops(env, ctx, clock, scheduledTime) {
|
|
|
507
487
|
// by 30 for a source that can only go stale once an hour. No-op unless the catalog + GitHub are
|
|
508
488
|
// both wired (nothing is linked to refresh).
|
|
509
489
|
if (shouldRunReachabilityPass(scheduledTime, FREQUENT_CRON_PERIOD_MS, FOUNDATIONAL_SOURCE_STALE_MS)) {
|
|
510
|
-
|
|
511
|
-
sweep: 'foundational-sources',
|
|
512
|
-
...describeError(error),
|
|
513
|
-
})));
|
|
490
|
+
tick.run({ name: 'foundational-sources', failureMessage: 'foundational-source sweep failed' }, sweepFoundationalSources(buildContainer(env).foundationalServices, logger));
|
|
514
491
|
}
|
|
515
492
|
// Tear down ephemeral environments whose TTL has elapsed (no-op unless the
|
|
516
493
|
// environment integration is configured).
|
|
517
|
-
|
|
494
|
+
tick.run({ name: 'env-sweeper', failureMessage: 'environment sweep failed' }, sweepExpiredEnvironments(env, clock));
|
|
518
495
|
// Push deployment-level (platform-operator) observability aggregates to the OTLP
|
|
519
496
|
// endpoint as OpenTelemetry gauge metrics, once per cron tick. Opt-in on top of the base
|
|
520
497
|
// OTel exporter (OTEL_PLATFORM_METRICS); a no-op otherwise. Per account, enumerated from
|
|
@@ -531,7 +508,7 @@ function runPeriodicBackstops(env, ctx, clock, scheduledTime) {
|
|
|
531
508
|
logger,
|
|
532
509
|
});
|
|
533
510
|
if (sweep)
|
|
534
|
-
|
|
511
|
+
tick.run({ name: 'platform-metrics', failureMessage: 'platform metrics sweep failed' }, sweep);
|
|
535
512
|
}
|
|
536
513
|
// Probe each workspace's CONFIGURED infrastructure connections and report a dead one as
|
|
537
514
|
// `unreachable` — raising/clearing an `infra_unreachable` card and pushing an `infraSetup` event
|
|
@@ -544,55 +521,67 @@ function runPeriodicBackstops(env, ctx, clock, scheduledTime) {
|
|
|
544
521
|
const reachability = loadConfig(env).infraReachability;
|
|
545
522
|
if (reachability.enabled &&
|
|
546
523
|
shouldRunReachabilityPass(scheduledTime, FREQUENT_CRON_PERIOD_MS, reachability.intervalMs)) {
|
|
547
|
-
|
|
548
|
-
.then(({ raised, cleared }) => {
|
|
524
|
+
tick.run({ name: 'infra-reachability', failureMessage: 'infra reachability sweep failed' }, sweepInfraReachability(buildContainer(env), logger).then(({ raised, cleared }) => {
|
|
549
525
|
if (raised > 0 || cleared > 0)
|
|
550
|
-
logger.info('infra reachability sweep', {
|
|
551
|
-
|
|
552
|
-
raised,
|
|
553
|
-
cleared,
|
|
554
|
-
});
|
|
555
|
-
})
|
|
556
|
-
.catch((error) => logger.error('infra reachability sweep failed', {
|
|
557
|
-
cron: 'infra-reachability',
|
|
558
|
-
...describeError(error),
|
|
559
|
-
})));
|
|
526
|
+
logger.info('infra reachability sweep', { cron: 'infra-reachability', raised, cleared });
|
|
527
|
+
}));
|
|
560
528
|
}
|
|
561
529
|
// Raise/clear `platform_health` notifications when the deployment's OWN run health crosses
|
|
562
530
|
// an operator threshold, per account (the push counterpart to the operator dashboard read).
|
|
563
531
|
// Opt-in (`PLATFORM_ALERTS`); the container (hence the platform-observability read) is built
|
|
564
532
|
// only when opted in so a deployment that hasn't opted in pays nothing.
|
|
565
533
|
if (loadConfig(env).platformAlerts.enabled) {
|
|
566
|
-
|
|
567
|
-
.then(({ raised, cleared }) => {
|
|
534
|
+
tick.run({ name: 'platform-health', failureMessage: 'platform health sweep failed' }, sweepPlatformHealth(buildContainer(env), logger).then(({ raised, cleared }) => {
|
|
568
535
|
if (raised > 0 || cleared > 0)
|
|
569
536
|
logger.info('platform health sweep', { cron: 'platform-health', raised, cleared });
|
|
570
|
-
})
|
|
571
|
-
.catch((error) => logger.error('platform health sweep failed', {
|
|
572
|
-
cron: 'platform-health',
|
|
573
|
-
...describeError(error),
|
|
574
|
-
})));
|
|
537
|
+
}));
|
|
575
538
|
}
|
|
576
539
|
}
|
|
577
540
|
/** The cron sweeper. Module-level (not closed over the app) so {@link createWorker} stays thin. */
|
|
578
541
|
async function handleScheduled(controller, env, ctx) {
|
|
579
542
|
setLogLevel(parseLogLevel(env.LOG_LEVEL));
|
|
580
543
|
const clock = new SystemClock();
|
|
544
|
+
const tick = new SweepTick(ctx);
|
|
581
545
|
// Daily pass: prune the unbounded ledgers/projections to their retention windows.
|
|
582
546
|
if (controller.cron === RETENTION_CRON) {
|
|
583
|
-
runDailyRetentionSweeps(env,
|
|
584
|
-
|
|
547
|
+
runDailyRetentionSweeps(env, tick, clock);
|
|
548
|
+
}
|
|
549
|
+
else {
|
|
550
|
+
// Frequent pass (every 2 min): time-sensitive backstops.
|
|
551
|
+
redriveStuckAgentRuns(env, tick, clock);
|
|
552
|
+
redriveStuckEnvTests(env, tick, clock);
|
|
553
|
+
reclaimExpiredActivations(env, tick, clock);
|
|
554
|
+
reapStaleContainers(env, tick, clock);
|
|
555
|
+
runPeriodicBackstops(env, tick, clock, controller.scheduledTime);
|
|
585
556
|
}
|
|
586
|
-
//
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
557
|
+
// This tick's own counters, flushed once its passes have SETTLED. The ordering is the whole
|
|
558
|
+
// point: the collector is per ISOLATE, so a counter a cron pass records can only be exported
|
|
559
|
+
// by a flush that runs after it. Draining while the passes were still in flight meant a cron
|
|
560
|
+
// tick drained an empty collector and left its own counters waiting for a next tick in the
|
|
561
|
+
// same isolate — which for the daily retention cron is a tick that never comes.
|
|
562
|
+
flushOperationalMetricsAfter(tick.settled(), env, ctx, clock);
|
|
563
|
+
}
|
|
564
|
+
/**
|
|
565
|
+
* Flush THIS ISOLATE's operational counters once `work` settles, as a post-response
|
|
566
|
+
* `waitUntil`. Every entry point calls it, because a Worker's collector is per-isolate and an
|
|
567
|
+
* isolate is discarded without warning — see `observability/operationalFlush.ts` for why that
|
|
568
|
+
* makes per-invocation flushing the correct shape rather than a wasteful one.
|
|
569
|
+
*
|
|
570
|
+
* `allSettled` rather than a catch: a FAILED invocation is exactly the one whose counters
|
|
571
|
+
* matter, and its rejection is the caller's to propagate, not this bookkeeping's to observe.
|
|
572
|
+
*/
|
|
573
|
+
function flushOperationalMetricsAfter(work, env, ctx, clock) {
|
|
574
|
+
ctx.waitUntil(Promise.allSettled([work]).then(() => flushOperationalMetricsForIsolate(loadConfig(env).otel, clock.now()) ?? undefined));
|
|
592
575
|
}
|
|
593
576
|
/** The queue consumer multiplexing the three queues. Module-level, like {@link handleScheduled}. */
|
|
594
|
-
async function handleQueue(batch, env) {
|
|
577
|
+
async function handleQueue(batch, env, ctx) {
|
|
595
578
|
setLogLevel(parseLogLevel(env.LOG_LEVEL));
|
|
579
|
+
const work = routeQueueBatch(batch, env);
|
|
580
|
+
flushOperationalMetricsAfter(work, env, ctx, new SystemClock());
|
|
581
|
+
await work;
|
|
582
|
+
}
|
|
583
|
+
/** The routing half of {@link handleQueue}, split out so the flush can bracket the whole batch. */
|
|
584
|
+
async function routeQueueBatch(batch, env) {
|
|
596
585
|
// Route by source queue — the single handler serves all three queues.
|
|
597
586
|
if (batch.queue === GITHUB_SYNC_QUEUE_NAME) {
|
|
598
587
|
await handleGitHubSyncBatch(batch, env);
|
|
@@ -673,9 +662,13 @@ export function createWorker(options = {}) {
|
|
|
673
662
|
pipelineRegistry: registries.pipelineRegistry,
|
|
674
663
|
taskTypeRegistry: registries.taskTypeRegistry,
|
|
675
664
|
foundationalServiceRegistry: registries.foundationalServiceRegistry,
|
|
665
|
+
binaryGeneratorRegistry: registries.binaryGeneratorRegistry,
|
|
676
666
|
onWarn: (problem) => logger.warn(problem.message, { code: problem.code }),
|
|
677
667
|
});
|
|
678
|
-
|
|
668
|
+
const response = Promise.resolve(app.fetch(request, env, ctx));
|
|
669
|
+
// Flush whatever THIS isolate counted while serving the request, after the response.
|
|
670
|
+
flushOperationalMetricsAfter(response, env, ctx, new SystemClock());
|
|
671
|
+
return response;
|
|
679
672
|
},
|
|
680
673
|
scheduled: handleScheduled,
|
|
681
674
|
queue: handleQueue,
|