@cat-factory/worker 0.137.2 → 0.138.0

package/dist/infrastructure/workflows/retention.d.ts

@@ -1,4 +1,4 @@
-import type { AgentContextSnapshotRepository, AgentSearchQueryRepository, Clock, CommitProjectionRepository, LlmCallMetricRepository, NotificationRepository, PasswordResetTokenRepository, PipelineScheduleRepository, ProvisioningLogRepository, RateLimitRepository, SubscriptionQuotaCycleRepository, TokenUsageRepository } from '@cat-factory/kernel';
+import type { AgentContextSnapshotRepository, AgentSearchQueryRepository, Clock, CommitProjectionRepository, LlmCallMetricRepository, NotificationRepository, PasswordResetTokenRepository, PipelineScheduleRepository, ProvisioningLogRepository, RateLimitRepository, SubscriptionQuotaCycleRepository, TokenUsageRepository, Logger } from '@cat-factory/kernel';
 /** Per-table retention ages in milliseconds; 0 (or less) disables that table's pass. */
 interface RetentionPolicy {
     tokenUsageMs: number;
@@ -36,8 +36,10 @@ export interface RetentionDeps {
     notificationRepository: NotificationRepository;
     clock: Clock;
     policy: RetentionPolicy;
+    /** Names the table behind each isolated prune failure. Absent ⇒ the failures are silent. */
+    logger?: Logger;
 }
-/** Rows reclaimed from each table, for logging. */
+/** Rows reclaimed from each table, plus the tables the pass could not prune. */
 export interface RetentionResult {
     tokenUsage: number;
     rateLimits: number;
@@ -50,12 +52,24 @@ export interface RetentionResult {
     provisioningLog: number;
     passwordResetTokens: number;
     notifications: number;
+    /**
+     * The tables whose prune threw this pass. EMPTY on a clean pass. Reported separately from
+     * the counts because a failed prune and an empty table both reclaim 0 rows, and only one of
+     * them means the table is still growing.
+     */
+    failedTables: string[];
 }
 /**
  * Prune each unbounded table to its retention window. The deletes are
  * range-scans on indexed columns and usually reclaim nothing, so this is cheap
- * to run on the every-2-min cron. Returns the counts removed per table.
+ * to run on the every-2-min cron. Returns the counts removed per table plus the tables that
+ * failed.
+ *
+ * Every table is pruned in ISOLATION (slice 4.4): the passes used to be a chain of bare
+ * `await`s, so the first failing `deleteOlderThan` aborted every later one — indefinitely,
+ * since the same table failed on every subsequent pass too. Shared with the Node twin via
+ * `createRetentionPass`, so the two facades cannot drift on it.
  */
-export declare function sweepRetention({ tokenUsageRepository, rateLimitRepository, commitRepository, llmCallMetricRepository, agentContextSnapshotRepository, agentSearchQueryRepository, subscriptionQuotaCycleRepository, pipelineScheduleRepository, provisioningLogRepository, passwordResetTokenRepository, notificationRepository, clock, policy, }: RetentionDeps): Promise<RetentionResult>;
+export declare function sweepRetention({ tokenUsageRepository, rateLimitRepository, commitRepository, llmCallMetricRepository, agentContextSnapshotRepository, agentSearchQueryRepository, subscriptionQuotaCycleRepository, pipelineScheduleRepository, provisioningLogRepository, passwordResetTokenRepository, notificationRepository, clock, policy, logger, }: RetentionDeps): Promise<RetentionResult>;
 export {};
 //# sourceMappingURL=retention.d.ts.map

package/dist/infrastructure/workflows/retention.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"retention.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/workflows/retention.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,8BAA8B,EAC9B,0BAA0B,EAC1B,KAAK,EACL,0BAA0B,EAC1B,uBAAuB,EACvB,sBAAsB,EACtB,4BAA4B,EAC5B,0BAA0B,EAC1B,yBAAyB,EACzB,mBAAmB,EACnB,gCAAgC,EAChC,oBAAoB,EACrB,MAAM,qBAAqB,CAAA;AAoB5B,wFAAwF;AACxF,UAAU,eAAe;IACvB,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,gBAAgB,EAAE,MAAM,CAAA;IACxB;;;;;OAKG;IACH,iBAAiB,EAAE,MAAM,CAAA;IACzB,6EAA6E;IAC7E,eAAe,EAAE,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,oBAAoB,EAAE,oBAAoB,CAAA;IAC1C,mBAAmB,EAAE,mBAAmB,CAAA;IACxC,gBAAgB,EAAE,0BAA0B,CAAA;IAC5C,uBAAuB,EAAE,uBAAuB,CAAA;IAChD,oFAAoF;IACpF,8BAA8B,EAAE,8BAA8B,CAAA;IAC9D,iFAAiF;IACjF,0BAA0B,EAAE,0BAA0B,CAAA;IACtD,uFAAuF;IACvF,gCAAgC,EAAE,gCAAgC,CAAA;IAClE,4FAA4F;IAC5F,0BAA0B,CAAC,EAAE,0BAA0B,CAAA;IACvD,+FAA+F;IAC/F,yBAAyB,CAAC,EAAE,yBAAyB,CAAA;IACrD,mFAAmF;IACnF,4BAA4B,CAAC,EAAE,4BAA4B,CAAA;IAC3D,sFAAsF;IACtF,sBAAsB,EAAE,sBAAsB,CAAA;IAC9C,KAAK,EAAE,KAAK,CAAA;IACZ,MAAM,EAAE,eAAe,CAAA;CACxB;AAED,mDAAmD;AACnD,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,CAAA;IACf,cAAc,EAAE,MAAM,CAAA;IACtB,qBAAqB,EAAE,MAAM,CAAA;IAC7B,kBAAkB,EAAE,MAAM,CAAA;IAC1B,uBAAuB,EAAE,MAAM,CAAA;IAC/B,YAAY,EAAE,MAAM,CAAA;IACpB,eAAe,EAAE,MAAM,CAAA;IACvB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,aAAa,EAAE,MAAM,CAAA;CACtB;AAYD;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,EACnC,oBAAoB,EACpB,mBAAmB,EACnB,gBAAgB,EAChB,uBAAuB,EACvB,8BAA8B,EAC9B,0BAA0B,EAC1B,gCAAgC,EAChC,0BAA0B,EAC1B,yBAAyB,EACzB,4BAA4B,EAC5B,sBAAsB,EACtB,KAAK,EACL,MAAM,GACP,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC,CA0C1C"}
+{"version":3,"file":"retention.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/workflows/retention.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,8BAA8B,EAC9B,0BAA0B,EAC1B,KAAK,EACL,0BAA0B,EAC1B,uBAAuB,EACvB,sBAAsB,EACtB,4BAA4B,EAC5B,0BAA0B,EAC1B,yBAAyB,EACzB,mBAAmB,EACnB,gCAAgC,EAChC,oBAAoB,EACpB,MAAM,EACP,MAAM,qBAAqB,CAAA;AAqB5B,wFAAwF;AACxF,UAAU,eAAe;IACvB,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,gBAAgB,EAAE,MAAM,CAAA;IACxB;;;;;OAKG;IACH,iBAAiB,EAAE,MAAM,CAAA;IACzB,6EAA6E;IAC7E,eAAe,EAAE,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,oBAAoB,EAAE,oBAAoB,CAAA;IAC1C,mBAAmB,EAAE,mBAAmB,CAAA;IACxC,gBAAgB,EAAE,0BAA0B,CAAA;IAC5C,uBAAuB,EAAE,uBAAuB,CAAA;IAChD,oFAAoF;IACpF,8BAA8B,EAAE,8BAA8B,CAAA;IAC9D,iFAAiF;IACjF,0BAA0B,EAAE,0BAA0B,CAAA;IACtD,uFAAuF;IACvF,gCAAgC,EAAE,gCAAgC,CAAA;IAClE,4FAA4F;IAC5F,0BAA0B,CAAC,EAAE,0BAA0B,CAAA;IACvD,+FAA+F;IAC/F,yBAAyB,CAAC,EAAE,yBAAyB,CAAA;IACrD,mFAAmF;IACnF,4BAA4B,CAAC,EAAE,4BAA4B,CAAA;IAC3D,sFAAsF;IACtF,sBAAsB,EAAE,sBAAsB,CAAA;IAC9C,KAAK,EAAE,KAAK,CAAA;IACZ,MAAM,EAAE,eAAe,CAAA;IACvB,4FAA4F;IAC5F,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,gFAAgF;AAChF,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,CAAA;IACf,cAAc,EAAE,MAAM,CAAA;IACtB,qBAAqB,EAAE,MAAM,CAAA;IAC7B,kBAAkB,EAAE,MAAM,CAAA;IAC1B,uBAAuB,EAAE,MAAM,CAAA;IAC/B,YAAY,EAAE,MAAM,CAAA;IACpB,eAAe,EAAE,MAAM,CAAA;IACvB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB;;;;OAIG;IACH,YAAY,EAAE,MAAM,EAAE,CAAA;CACvB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAAC,EACnC,oBAAoB,EACpB,mBAAmB,EACnB,gBAAgB,EAChB,uBAAuB,EACvB,8BAA8B,EAC9B,0BAA0B,EAC1B,gCAAgC,EAChC,0BAA0B,EAC1B,yBAAyB,EACzB,4BAA4B,EAC5B,sBAAsB,EACtB,KAAK,EACL,MAAM,EACN,MAAM,GACP,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC,CA2D1C"}

package/dist/infrastructure/workflows/retention.js

@@ -1,3 +1,4 @@
+import { createRetentionPass } from '@cat-factory/orchestration';
 /** Recurring-pipeline run history is kept ~1 week (the inspector's window). */
 const SCHEDULE_RUN_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
 /**
@@ -7,42 +8,44 @@ const SCHEDULE_RUN_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
  * 30 days has long since reset and only holds stale counters.
  */
 const SUBSCRIPTION_QUOTA_CYCLE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
-/** Delete rows older than `now - windowMs`, treating a non-positive window as "disabled". */
-async function prune(windowMs, now, del) {
-    if (windowMs <= 0)
-        return 0;
-    return del(now - windowMs);
-}
 /**
  * Prune each unbounded table to its retention window. The deletes are
  * range-scans on indexed columns and usually reclaim nothing, so this is cheap
- * to run on the every-2-min cron. Returns the counts removed per table.
+ * to run on the every-2-min cron. Returns the counts removed per table plus the tables that
+ * failed.
+ *
+ * Every table is pruned in ISOLATION (slice 4.4): the passes used to be a chain of bare
+ * `await`s, so the first failing `deleteOlderThan` aborted every later one — indefinitely,
+ * since the same table failed on every subsequent pass too. Shared with the Node twin via
+ * `createRetentionPass`, so the two facades cannot drift on it.
  */
-export async function sweepRetention({ tokenUsageRepository, rateLimitRepository, commitRepository, llmCallMetricRepository, agentContextSnapshotRepository, agentSearchQueryRepository, subscriptionQuotaCycleRepository, pipelineScheduleRepository, provisioningLogRepository, passwordResetTokenRepository, notificationRepository, clock, policy, }) {
+export async function sweepRetention({ tokenUsageRepository, rateLimitRepository, commitRepository, llmCallMetricRepository, agentContextSnapshotRepository, agentSearchQueryRepository, subscriptionQuotaCycleRepository, pipelineScheduleRepository, provisioningLogRepository, passwordResetTokenRepository, notificationRepository, clock, policy, logger, }) {
     const now = clock.now();
+    const pass = createRetentionPass(logger);
     return {
-        tokenUsage: await prune(policy.tokenUsageMs, now, (c) => tokenUsageRepository.deleteOlderThan(c)),
-        rateLimits: await prune(policy.rateLimitMs, now, (c) => rateLimitRepository.deleteOlderThan(c)),
-        commits: await prune(policy.commitMs, now, (c) => commitRepository.deleteOlderThan(c)),
-        llmCallMetrics: await prune(policy.llmCallMetricsMs, now, (c) => llmCallMetricRepository.deleteOlderThan(c)),
+        tokenUsage: await pass.prune('token_usage', policy.tokenUsageMs, now, (c) => tokenUsageRepository.deleteOlderThan(c)),
+        rateLimits: await pass.prune('github_rate_limits', policy.rateLimitMs, now, (c) => rateLimitRepository.deleteOlderThan(c)),
+        commits: await pass.prune('github_commits', policy.commitMs, now, (c) => commitRepository.deleteOlderThan(c)),
+        llmCallMetrics: await pass.prune('llm_call_metrics', policy.llmCallMetricsMs, now, (c) => llmCallMetricRepository.deleteOlderThan(c)),
         // Same window as the LLM call telemetry (heavy prompt + injected-file bodies).
-        agentContextSnapshots: await prune(policy.llmCallMetricsMs, now, (c) => agentContextSnapshotRepository.deleteOlderThan(c)),
+        agentContextSnapshots: await pass.prune('agent_context_snapshots', policy.llmCallMetricsMs, now, (c) => agentContextSnapshotRepository.deleteOlderThan(c)),
         // Same window as the LLM call telemetry (performed web-search queries).
-        agentSearchQueries: await prune(policy.llmCallMetricsMs, now, (c) => agentSearchQueryRepository.deleteOlderThan(c)),
+        agentSearchQueries: await pass.prune('agent_search_queries', policy.llmCallMetricsMs, now, (c) => agentSearchQueryRepository.deleteOlderThan(c)),
         // Idle quota cycles past the fixed 30-day window (well beyond the weekly one).
-        subscriptionQuotaCycles: await prune(SUBSCRIPTION_QUOTA_CYCLE_RETENTION_MS, now, (c) => subscriptionQuotaCycleRepository.deleteOlderThan(c)),
+        subscriptionQuotaCycles: await pass.prune('subscription_quota_cycles', SUBSCRIPTION_QUOTA_CYCLE_RETENTION_MS, now, (c) => subscriptionQuotaCycleRepository.deleteOlderThan(c)),
         scheduleRuns: pipelineScheduleRepository
-            ? await prune(SCHEDULE_RUN_RETENTION_MS, now, (c) => pipelineScheduleRepository.pruneRunsBefore(c))
+            ? await pass.prune('pipeline_schedule_runs', SCHEDULE_RUN_RETENTION_MS, now, (c) => pipelineScheduleRepository.pruneRunsBefore(c))
             : 0,
         provisioningLog: provisioningLogRepository
-            ? await prune(policy.provisioningLogMs, now, (c) => provisioningLogRepository.deleteOlderThan(c))
+            ? await pass.prune('provisioning_log', policy.provisioningLogMs, now, (c) => provisioningLogRepository.deleteOlderThan(c))
             : 0,
         // Reset tokens past their own expiry — `now`, not a window.
         passwordResetTokens: passwordResetTokenRepository
-            ? await passwordResetTokenRepository.deleteExpired(now)
+            ? await pass.expire('password_reset_tokens', () => passwordResetTokenRepository.deleteExpired(now))
             : 0,
         // Resolved (acted/dismissed) notifications past the window; open cards untouched.
-        notifications: await prune(policy.notificationsMs, now, (c) => notificationRepository.deleteResolvedOlderThan(c)),
+        notifications: await pass.prune('notifications', policy.notificationsMs, now, (c) => notificationRepository.deleteResolvedOlderThan(c)),
+        failedTables: pass.failed,
     };
 }
 //# sourceMappingURL=retention.js.map

package/dist/infrastructure/workflows/retention.js.map

@@ -1 +1 @@
-{"version":3,"file":"retention.js","sourceRoot":"","sources":["../../../src/infrastructure/workflows/retention.ts"],"names":[],"mappings":"AAeA,+EAA+E;AAC/E,MAAM,yBAAyB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAEzD;;;;;GAKG;AACH,MAAM,qCAAqC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAgEtE,6FAA6F;AAC7F,KAAK,UAAU,KAAK,CAClB,QAAgB,EAChB,GAAW,EACX,GAAwC;IAExC,IAAI,QAAQ,IAAI,CAAC;QAAE,OAAO,CAAC,CAAA;IAC3B,OAAO,GAAG,CAAC,GAAG,GAAG,QAAQ,CAAC,CAAA;AAC5B,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EACnC,oBAAoB,EACpB,mBAAmB,EACnB,gBAAgB,EAChB,uBAAuB,EACvB,8BAA8B,EAC9B,0BAA0B,EAC1B,gCAAgC,EAChC,0BAA0B,EAC1B,yBAAyB,EACzB,4BAA4B,EAC5B,sBAAsB,EACtB,KAAK,EACL,MAAM,GACQ;IACd,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAA;IACvB,OAAO;QACL,UAAU,EAAE,MAAM,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CACtD,oBAAoB,CAAC,eAAe,CAAC,CAAC,CAAC,CACxC;QACD,UAAU,EAAE,MAAM,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/F,OAAO,EAAE,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACtF,cAAc,EAAE,MAAM,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAC9D,uBAAuB,CAAC,eAAe,CAAC,CAAC,CAAC,CAC3C;QACD,+EAA+E;QAC/E,qBAAqB,EAAE,MAAM,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CACrE,8BAA8B,CAAC,eAAe,CAAC,CAAC,CAAC,CAClD;QACD,wEAAwE;QACxE,kBAAkB,EAAE,MAAM,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAClE,0BAA0B,CAAC,eAAe,CAAC,CAAC,CAAC,CAC9C;QACD,+EAA+E;QAC/E,uBAAuB,EAAE,MAAM,KAAK,CAAC,qCAAqC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CACrF,gCAAgC,CAAC,eAAe,CAAC,CAAC,CAAC,CACpD;QACD,YAAY,EAAE,0BAA0B;YACtC,CAAC,CAAC,MAAM,KAAK,CAAC,yBAAyB,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAChD,0BAA0B,CAAC,eAAe,CAAC,CAAC,CAAC,CAC9C;YACH,CAAC,CAAC,CAAC;QACL,eAAe,EAAE,yBAAyB;YACxC,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAC/C,yBAAyB,CAAC,eAAe,CAAC,CAAC,CAAC,CAC7C;YACH,CAAC,CAAC,CAAC;QACL,4DAA4D;QAC5D,mBAAmB,EAAE,4BAA4B;YAC/C,CAAC,CAAC,MAAM,4BAA4B,CAAC,aAAa,CAAC,GAAG,CAAC;YACvD,CAAC,CAAC,CAAC;QACL,kFAAkF;QAClF,aAAa,EAAE,MAAM,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5D,sBAAsB,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAClD;KACF,CAAA;AACH,CAAC"}
+{"version":3,"file":"retention.js","sourceRoot":"","sources":["../../../src/infrastructure/workflows/retention.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAEhE,+EAA+E;AAC/E,MAAM,yBAAyB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAEzD;;;;;GAKG;AACH,MAAM,qCAAqC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAwEtE;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EACnC,oBAAoB,EACpB,mBAAmB,EACnB,gBAAgB,EAChB,uBAAuB,EACvB,8BAA8B,EAC9B,0BAA0B,EAC1B,gCAAgC,EAChC,0BAA0B,EAC1B,yBAAyB,EACzB,4BAA4B,EAC5B,sBAAsB,EACtB,KAAK,EACL,MAAM,EACN,MAAM,GACQ;IACd,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAA;IACvB,MAAM,IAAI,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACxC,OAAO;QACL,UAAU,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,MAAM,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAC1E,oBAAoB,CAAC,eAAe,CAAC,CAAC,CAAC,CACxC;QACD,UAAU,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,oBAAoB,EAAE,MAAM,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAChF,mBAAmB,CAAC,eAAe,CAAC,CAAC,CAAC,CACvC;QACD,OAAO,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CACtE,gBAAgB,CAAC,eAAe,CAAC,CAAC,CAAC,CACpC;QACD,cAAc,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,MAAM,CAAC,gBAAgB,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CACvF,uBAAuB,CAAC,eAAe,CAAC,CAAC,CAAC,CAC3C;QACD,+EAA+E;QAC/E,qBAAqB,EAAE,MAAM,IAAI,CAAC,KAAK,CACrC,yBAAyB,EACzB,MAAM,CAAC,gBAAgB,EACvB,GAAG,EACH,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,eAAe,CAAC,CAAC,CAAC,CACzD;QACD,wEAAwE;QACxE,kBAAkB,EAAE,MAAM,IAAI,CAAC,KAAK,CAClC,sBAAsB,EACtB,MAAM,CAAC,gBAAgB,EACvB,GAAG,EACH,CAAC,CAAC,EAAE,EAAE,CAAC,0BAA0B,CAAC,eAAe,CAAC,CAAC,CAAC,CACrD;QACD,+EAA+E;QAC/E,uBAAuB,EAAE,MAAM,IAAI,CAAC,KAAK,CACvC,2BAA2B,EAC3B,qCAAqC,EACrC,GAAG,EACH,CAAC,CAAC,EAAE,EAAE,CAAC,gCAAgC,CAAC,eAAe,CAAC,CAAC,CAAC,CAC3D;QACD,YAAY,EAAE,0BAA0B;YACtC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,yBAAyB,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAC/E,0BAA0B,CAAC,eAAe,CAAC,CAAC,CAAC,CAC9C;YACH,CAAC,CAAC,CAAC;QACL,eAAe,EAAE,yBAAyB;YACxC,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,MAAM,CAAC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CACxE,yBAAyB,CAAC,eAAe,CAAC,CAAC,CAAC,CAC7C;YACH,CAAC,CAAC,CAAC;QACL,4DAA4D;QAC5D,mBAAmB,EAAE,4BAA4B;YAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,uBAAuB,EAAE,GAAG,EAAE,CAC9C,4BAA4B,CAAC,aAAa,CAAC,GAAG,CAAC,CAChD;YACH,CAAC,CAAC,CAAC;QACL,kFAAkF;QAClF,aAAa,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,MAAM,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAClF,sBAAsB,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAClD;QACD,YAAY,EAAE,IAAI,CAAC,MAAM;KAC1B,CAAA;AACH,CAAC"}

package/dist/infrastructure/workflows/sweeper.d.ts

@@ -1,4 +1,4 @@
-import type { AgentRunRef, AgentRunRepository, Clock, EnvironmentTestRunRecord } from '@cat-factory/kernel';
+import type { AgentRunRef, AgentRunRepository, Clock, EnvironmentTestRunRecord, Logger, OperationalMetrics } from '@cat-factory/kernel';
 import type { Workflow } from '@cloudflare/workers-types';
 /**
  * A run's durable instance, as the sweeper needs to classify it:
@@ -54,6 +54,15 @@ export interface SweepDeps {
      * (the degenerate unit-test case), never wrongly on a first observation.
      */
     orphanedSince?: Map<string, number>;
+    /**
+     * Counts each disposition (re-drive / finalize / stall). The per-sweep RESULT below already
+     * carries these numbers, but only for this tick's log line: nothing accumulates them, so
+     * "has the re-drive rate tripled since the deploy" was unanswerable except by grepping.
+     * Absent ⇒ uncounted (a unit test).
+     */
+    metrics?: OperationalMetrics;
+    /** Reports the best-effort re-drive bookkeeping below. Absent ⇒ silent (a unit test). */
+    logger?: Logger;
 }
 /** What a sweep did, for logging. */
 export interface SweepResult {
@@ -75,7 +84,7 @@ export interface SweepResult {
  *   - `alive`    → leave it.
  * Pure orchestration over its ports so it is unit-testable with fakes.
  */
-export declare function sweepStuckRuns({ agentRunRepository, instanceState, redrive, finalizeOrphan, failStalled, clock, leaseMs, hardStallMs, orphanedSince, }: SweepDeps): Promise<SweepResult>;
+export declare function sweepStuckRuns({ agentRunRepository, instanceState, redrive, finalizeOrphan, failStalled, clock, leaseMs, hardStallMs, orphanedSince, metrics, logger, }: SweepDeps): Promise<SweepResult>;
 export interface EnvTestSweepDeps {
     repository: {
         listStale(cutoffMs: number): Promise<EnvironmentTestRunRecord[]>;

package/dist/infrastructure/workflows/sweeper.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sweeper.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/workflows/sweeper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,WAAW,EACX,kBAAkB,EAClB,KAAK,EACL,wBAAwB,EACzB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AAEzD;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAA;AAE5D,+DAA+D;AAC/D,UAAU,cAAc;IACtB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAA;CACrD;AAED,0DAA0D;AAC1D,qBAAa,eAAgB,YAAW,cAAc;IACxC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAArC,YAA6B,QAAQ,EAAE,QAAQ,EAAI;IAE7C,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAqBzD;CACF;AAED,MAAM,WAAW,SAAS;IACxB,kBAAkB,EAAE,kBAAkB,CAAA;IACtC,gEAAgE;IAChE,aAAa,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAAA;IACvD,4EAA4E;IAC5E,OAAO,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACxC;;;;OAIG;IACH,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/C;;;;OAIG;IACH,WAAW,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC5C,KAAK,EAAE,KAAK,CAAA;IACZ,yEAAyE;IACzE,OAAO,EAAE,MAAM,CAAA;IACf,6FAA6F;IAC7F,WAAW,EAAE,MAAM,CAAA;IACnB;;;;;;;;;;OAUG;IACH,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACpC;AAED,qCAAqC;AACrC,MAAM,WAAW,WAAW;IAC1B,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAA;IAChB,sEAAsE;IACtE,SAAS,EAAE,MAAM,CAAA;IACjB,6EAA6E;IAC7E,OAAO,EAAE,MAAM,CAAA;CAChB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAAC,EACnC,kBAAkB,EAClB,aAAa,EACb,OAAO,EACP,cAAc,EACd,WAAW,EACX,KAAK,EACL,OAAO,EACP,WAAW,EACX,aAAyC,GAC1C,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CAiDlC;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE;QAAE,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,EAAE,CAAC,CAAA;KAAE,CAAA;IAChF,2DAA2D;IAC3D,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAA;IACpD,4EAA4E;IAC5E,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC1D;;;OAGG;IACH,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjE,KAAK,EAAE,KAAK,CAAA;IACZ,mFAAmF;IACnF,OAAO,EAAE,MAAM,CAAA;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,UAAU,EACV,aAAa,EACb,OAAO,EACP,cAAc,EACd,KAAK,EACL,OAAO,GACR,EAAE,gBAAgB,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAgBrE"}
+{"version":3,"file":"sweeper.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/workflows/sweeper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,WAAW,EACX,kBAAkB,EAClB,KAAK,EACL,wBAAwB,EACxB,MAAM,EACN,kBAAkB,EACnB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AAEzD;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAA;AAE5D,+DAA+D;AAC/D,UAAU,cAAc;IACtB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAA;CACrD;AAED,0DAA0D;AAC1D,qBAAa,eAAgB,YAAW,cAAc;IACxC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAArC,YAA6B,QAAQ,EAAE,QAAQ,EAAI;IAE7C,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAqBzD;CACF;AAED,MAAM,WAAW,SAAS;IACxB,kBAAkB,EAAE,kBAAkB,CAAA;IACtC,gEAAgE;IAChE,aAAa,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAAA;IACvD,4EAA4E;IAC5E,OAAO,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACxC;;;;OAIG;IACH,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/C;;;;OAIG;IACH,WAAW,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC5C,KAAK,EAAE,KAAK,CAAA;IACZ,yEAAyE;IACzE,OAAO,EAAE,MAAM,CAAA;IACf,6FAA6F;IAC7F,WAAW,EAAE,MAAM,CAAA;IACnB;;;;;;;;;;OAUG;IACH,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACnC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,kBAAkB,CAAA;IAC5B,yFAAyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,qCAAqC;AACrC,MAAM,WAAW,WAAW;IAC1B,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAA;IAChB,sEAAsE;IACtE,SAAS,EAAE,MAAM,CAAA;IACjB,6EAA6E;IAC7E,OAAO,EAAE,MAAM,CAAA;CAChB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAAC,EACnC,kBAAkB,EAClB,aAAa,EACb,OAAO,EACP,cAAc,EACd,WAAW,EACX,KAAK,EACL,OAAO,EACP,WAAW,EACX,aAAyC,EACzC,OAAgC,EAChC,MAAmB,GACpB,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CA4DlC;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE;QAAE,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,EAAE,CAAC,CAAA;KAAE,CAAA;IAChF,2DAA2D;IAC3D,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAA;IACpD,4EAA4E;IAC5E,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC1D;;;OAGG;IACH,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjE,KAAK,EAAE,KAAK,CAAA;IACZ,mFAAmF;IACnF,OAAO,EAAE,MAAM,CAAA;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAsB,kBAAkB,CAAC,EACvC,UAAU,EACV,aAAa,EACb,OAAO,EACP,cAAc,EACd,KAAK,EACL,OAAO,GACR,EAAE,gBAAgB,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAgBrE"}

package/dist/infrastructure/workflows/sweeper.js

@@ -1,3 +1,4 @@
+import { noopLogger, noopOperationalMetrics, runBestEffort } from '@cat-factory/kernel';
 /** WorkflowLookup over a Cloudflare Workflows binding. */
 export class WorkflowsLookup {
     workflow;
@@ -40,7 +41,7 @@ export class WorkflowsLookup {
  *   - `alive`    → leave it.
  * Pure orchestration over its ports so it is unit-testable with fakes.
  */
-export async function sweepStuckRuns({ agentRunRepository, instanceState, redrive, finalizeOrphan, failStalled, clock, leaseMs, hardStallMs, orphanedSince = new Map(), }) {
+export async function sweepStuckRuns({ agentRunRepository, instanceState, redrive, finalizeOrphan, failStalled, clock, leaseMs, hardStallMs, orphanedSince = new Map(), metrics = noopOperationalMetrics, logger = noopLogger, }) {
     const now = clock.now();
     const stale = await agentRunRepository.listStale(now - leaseMs);
     let redriven = 0;
@@ -60,6 +61,9 @@ export async function sweepStuckRuns({ agentRunRepository, instanceState, redriv
             orphanedSince.delete(ref.id);
             await finalizeOrphan(ref);
             finalized++;
+            // The run KIND is a bounded enum and the split that matters: bootstrap runs and
+            // execution runs are lost for different reasons and fixed in different places.
+            metrics.increment('sweep.run_finalized', { kind: ref.kind });
             continue;
         }
         // `missing`: the instance was lost (eviction, a missed event) and can be re-created.
@@ -76,12 +80,18 @@ export async function sweepStuckRuns({ agentRunRepository, instanceState, redriv
         if (ref.kind === 'execution' && now - firstSeenOrphaned > hardStallMs) {
             await failStalled(ref);
             stalled++;
+            metrics.increment('sweep.run_stalled', { kind: ref.kind });
             orphanedSince.delete(ref.id);
             stillOrphaned.delete(ref.id);
             continue;
         }
         await redrive(ref);
         redriven++;
+        metrics.increment('sweep.run_redriven', { kind: ref.kind });
+        // Persist the per-run count AFTER the re-drive and best-effort: this is bookkeeping about
+        // the recovery, so it must never be able to fail one. Ordered after for the same reason —
+        // a counter claiming a run was re-driven when it was not is worse than one that missed.
+        await runBestEffort(logger, 'sweep.recordRedrive', () => agentRunRepository.recordRedrive(ref.workspaceId, ref.id));
     }
     // Forget runs that recovered (bumped their lease → left the stale set) or went terminal, so
     // their per-process orphaned clock restarts if they ever stall again.

package/dist/infrastructure/workflows/sweeper.js.map

@@ -1 +1 @@
-{"version":3,"file":"sweeper.js","sourceRoot":"","sources":["../../../src/infrastructure/workflows/sweeper.ts"],"names":[],"mappings":"AAuBA,0DAA0D;AAC1D,MAAM,OAAO,eAAe;IACG,QAAQ;IAArC,YAA6B,QAAkB;wBAAlB,QAAQ;IAAa,CAAC;IAEnD,KAAK,CAAC,aAAa,CAAC,KAAa;QAC/B,IAAI,QAAQ,CAAA;QACZ,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,kEAAkE;YAClE,OAAO,SAAS,CAAA;QAClB,CAAC;QACD,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,CAAA;YAC1C,OAAO,MAAM,KAAK,SAAS;gBACzB,MAAM,KAAK,QAAQ;gBACnB,MAAM,KAAK,SAAS;gBACpB,MAAM,KAAK,QAAQ;gBACnB,CAAC,CAAC,OAAO;gBACT,CAAC,CAAC,UAAU,CAAA;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,8EAA8E;YAC9E,8EAA8E;YAC9E,OAAO,SAAS,CAAA;QAClB,CAAC;IACH,CAAC;CACF;AAiDD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EACnC,kBAAkB,EAClB,aAAa,EACb,OAAO,EACP,cAAc,EACd,WAAW,EACX,KAAK,EACL,OAAO,EACP,WAAW,EACX,aAAa,GAAG,IAAI,GAAG,EAAkB,GAC/B;IACV,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAA;IACvB,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,SAAS,CAAC,GAAG,GAAG,OAAO,CAAC,CAAA;IAC/D,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,oFAAoF;IACpF,wFAAwF;IACxF,2BAA2B;IAC3B,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAA;IACvC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAA;QACtC,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACtB,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAC5B,SAAQ;QACV,CAAC;QACD,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YACzB,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAC5B,MAAM,cAAc,CAAC,GAAG,CAAC,CAAA;YACzB,SAAS,EAAE,CAAA;YACX,SAAQ;QACV,CAAC;QACD,qFAAqF;QACrF,mFAAmF;QACnF,uFAAuF;QACvF,wFAAwF;QACxF,0CAA0C;QAC1C,MAAM,iBAAiB,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,CAAA;QAC1D,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAAA;QAC5C,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACzB,yFAAyF;QACzF,qFAAqF;QACrF,kBAAkB;QAClB,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,GAAG,iBAAiB,GAAG,WAAW,EAAE,CAAC;YACtE,MAAM,WAAW,CAAC,GAAG,CAAC,CAAA;YACtB,OAAO,EAAE,CAAA;YACT,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAC5B,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAC5B,SAAQ;QACV,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,CAAA;QAClB,QAAQ,EAAE,CAAA;IACZ,CAAC;IACD,4FAA4F;IAC5F,sEAAsE;IACtE,KAAK,MAAM,EAAE,IAAI,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC;QACtC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACtD,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,CAAA;AACzC,CAAC;AAkBD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,EACvC,UAAU,EACV,aAAa,EACb,OAAO,EACP,cAAc,EACd,KAAK,EACL,OAAO,GACU;IACjB,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAA;IAC/D,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACzC,IAAI,KAAK,KAAK,OAAO;YAAE,SAAQ;QAC/B,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YACzB,MAAM,cAAc,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC7C,SAAS,EAAE,CAAA;YACX,SAAQ;QACV,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;QACtC,QAAQ,EAAE,CAAA;IACZ,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAA;AAChC,CAAC"}
+{"version":3,"file":"sweeper.js","sourceRoot":"","sources":["../../../src/infrastructure/workflows/sweeper.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AAkBvF,0DAA0D;AAC1D,MAAM,OAAO,eAAe;IACG,QAAQ;IAArC,YAA6B,QAAkB;wBAAlB,QAAQ;IAAa,CAAC;IAEnD,KAAK,CAAC,aAAa,CAAC,KAAa;QAC/B,IAAI,QAAQ,CAAA;QACZ,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,kEAAkE;YAClE,OAAO,SAAS,CAAA;QAClB,CAAC;QACD,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,CAAA;YAC1C,OAAO,MAAM,KAAK,SAAS;gBACzB,MAAM,KAAK,QAAQ;gBACnB,MAAM,KAAK,SAAS;gBACpB,MAAM,KAAK,QAAQ;gBACnB,CAAC,CAAC,OAAO;gBACT,CAAC,CAAC,UAAU,CAAA;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,8EAA8E;YAC9E,8EAA8E;YAC9E,OAAO,SAAS,CAAA;QAClB,CAAC;IACH,CAAC;CACF;AA0DD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EACnC,kBAAkB,EAClB,aAAa,EACb,OAAO,EACP,cAAc,EACd,WAAW,EACX,KAAK,EACL,OAAO,EACP,WAAW,EACX,aAAa,GAAG,IAAI,GAAG,EAAkB,EACzC,OAAO,GAAG,sBAAsB,EAChC,MAAM,GAAG,UAAU,GACT;IACV,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAA;IACvB,MAAM,KAAK,GAAG,MAAM,kBAAkB,CAAC,SAAS,CAAC,GAAG,GAAG,OAAO,CAAC,CAAA;IAC/D,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,oFAAoF;IACpF,wFAAwF;IACxF,2BAA2B;IAC3B,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAA;IACvC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAA;QACtC,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACtB,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAC5B,SAAQ;QACV,CAAC;QACD,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YACzB,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAC5B,MAAM,cAAc,CAAC,GAAG,CAAC,CAAA;YACzB,SAAS,EAAE,CAAA;YACX,gFAAgF;YAChF,+EAA+E;YAC/E,OAAO,CAAC,SAAS,CAAC,qBAAqB,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YAC5D,SAAQ;QACV,CAAC;QACD,qFAAqF;QACrF,mFAAmF;QACnF,uFAAuF;QACvF,wFAAwF;QACxF,0CAA0C;QAC1C,MAAM,iBAAiB,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,CAAA;QAC1D,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAAA;QAC5C,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACzB,yFAAyF;QACzF,qFAAqF;QACrF,kBAAkB;QAClB,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,GAAG,iBAAiB,GAAG,WAAW,EAAE,CAAC;YACtE,MAAM,WAAW,CAAC,GAAG,CAAC,CAAA;YACtB,OAAO,EAAE,CAAA;YACT,OAAO,CAAC,SAAS,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YAC1D,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAC5B,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAC5B,SAAQ;QACV,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,CAAA;QAClB,QAAQ,EAAE,CAAA;QACV,OAAO,CAAC,SAAS,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;QAC3D,0FAA0F;QAC1F,0FAA0F;QAC1F,wFAAwF;QACxF,MAAM,aAAa,CAAC,MAAM,EAAE,qBAAqB,EAAE,GAAG,EAAE,CACtD,kBAAkB,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAC1D,CAAA;IACH,CAAC;IACD,4FAA4F;IAC5F,sEAAsE;IACtE,KAAK,MAAM,EAAE,IAAI,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC;QACtC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACtD,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,CAAA;AACzC,CAAC;AAkBD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,EACvC,UAAU,EACV,aAAa,EACb,OAAO,EACP,cAAc,EACd,KAAK,EACL,OAAO,GACU;IACjB,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAA;IAC/D,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACzC,IAAI,KAAK,KAAK,OAAO;YAAE,SAAQ;QAC/B,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YACzB,MAAM,cAAc,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC7C,SAAS,EAAE,CAAA;YACX,SAAQ;QACV,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;QACtC,QAAQ,EAAE,CAAA;IACZ,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAA;AAChC,CAAC"}

package/migrations/0076_agent_runs_redrive_count.sql

@@ -0,0 +1,20 @@
+-- Per-run re-drive count (docs/initiatives/observability-logging-gaps.md, D4 / slice 4.1).
+--
+-- The sweepers already knew how many runs they re-drove per pass, but only for that pass's log
+-- line: the per-run history lived in an in-memory `orphanedSince` map holding a timestamp and
+-- nothing else. That map dies with the process on Node and with the isolate on Cloudflare, where
+-- the sweep additionally logs only aggregates (`{redriven: 3}`, no run ids) — so "was this run
+-- re-driven three times or is this the first?" had no answer anywhere.
+--
+-- Deliberately NOT rev-guarded and deliberately outside the re-drive's own write: this counts
+-- something that happened TO the run rather than deriving a value FROM it, so two racing sweepers
+-- double-counting is a far better outcome than a lost update, and a `rev` bump here would collide
+-- with the driver's own writes and let bookkeeping fail a re-drive.
+--
+-- Default 0 is honest for every existing row: a run that was re-driven before this column existed
+-- has no recoverable count, and 0 is what "we do not know of any" has to read as. The counter is
+-- kept when a run recovers — needing three re-drives is exactly what an operator wants to see
+-- about a run that eventually succeeded.
+
+ALTER TABLE agent_runs
+  ADD COLUMN redrive_count INTEGER NOT NULL DEFAULT 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cat-factory/worker",
-  "version": "0.137.2",
+  "version": "0.138.0",
   "description": "Reusable Cloudflare Worker library (Hono + D1) exposing the Agent Architecture Board core: the `createApp()` Hono factory, the default fetch/scheduled/queue handler, and the Durable Object + Workflow classes. Deployments (see deploy/backend) supply the wrangler.toml + config and re-export these.",
   "repository": {
     "type": "git",
@@ -42,22 +42,22 @@
     "pino": "^10.3.1",
     "valibot": "^1.4.2",
     "workers-ai-provider": "^4.0.0",
-    "@cat-factory/agents": "0.104.3",
-    "@cat-factory/caching": "0.12.12",
-    "@cat-factory/consensus": "0.13.25",
-    "@cat-factory/contracts": "0.211.0",
-    "@cat-factory/eks": "0.1.199",
-    "@cat-factory/gates": "0.8.45",
-    "@cat-factory/gitlab": "0.15.4",
-    "@cat-factory/integrations": "0.116.4",
-    "@cat-factory/kernel": "0.214.1",
-    "@cat-factory/observability-langfuse": "0.9.42",
-    "@cat-factory/observability-otel": "0.4.42",
-    "@cat-factory/orchestration": "0.185.2",
-    "@cat-factory/prompt-fragments": "0.15.36",
-    "@cat-factory/provider-cloudflare": "0.7.351",
-    "@cat-factory/server": "0.194.0",
-    "@cat-factory/spend": "0.13.1"
+    "@cat-factory/agents": "0.105.0",
+    "@cat-factory/caching": "0.13.0",
+    "@cat-factory/eks": "0.1.200",
+    "@cat-factory/contracts": "0.212.0",
+    "@cat-factory/consensus": "0.13.26",
+    "@cat-factory/gates": "0.8.46",
+    "@cat-factory/observability-langfuse": "0.9.43",
+    "@cat-factory/kernel": "0.215.0",
+    "@cat-factory/observability-otel": "0.5.0",
+    "@cat-factory/orchestration": "0.186.0",
+    "@cat-factory/prompt-fragments": "0.15.37",
+    "@cat-factory/server": "0.195.0",
+    "@cat-factory/spend": "0.13.2",
+    "@cat-factory/integrations": "0.117.0",
+    "@cat-factory/gitlab": "0.15.5",
+    "@cat-factory/provider-cloudflare": "0.7.352"
   },
   "devDependencies": {
     "@cloudflare/vitest-pool-workers": "^0.20.1",
@@ -65,7 +65,7 @@
     "typescript": "7.0.2",
     "vitest": "^4.1.10",
     "wrangler": "^4.118.0",
-    "@cat-factory/conformance": "0.20.10"
+    "@cat-factory/conformance": "0.20.11"
   },
   "scripts": {
     "build": "tsc -b tsconfig.build.json",