@tickernelz/paperclip-pro-server 2026.1001.0 → 2026.1001.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/dist/build-info.json +1 -1
  2. package/dist/index.js +3 -0
  3. package/dist/index.js.map +1 -1
  4. package/dist/services/heartbeat.d.ts +18 -2
  5. package/dist/services/heartbeat.d.ts.map +1 -1
  6. package/dist/services/heartbeat.js +194 -30
  7. package/dist/services/heartbeat.js.map +1 -1
  8. package/dist/services/hot-restart.d.ts +4 -3
  9. package/dist/services/hot-restart.d.ts.map +1 -1
  10. package/dist/services/hot-restart.js +1 -1
  11. package/dist/services/hot-restart.js.map +1 -1
  12. package/dist/services/recovery/service.d.ts +3 -0
  13. package/dist/services/recovery/service.d.ts.map +1 -1
  14. package/dist/services/recovery/service.js +154 -102
  15. package/dist/services/recovery/service.js.map +1 -1
  16. package/package.json +21 -21
  17. package/ui-dist/assets/{Agents.production-fNeT2seP.js → Agents.production-tjOEQe3l.js} +2 -2
  18. package/ui-dist/assets/{CompanyActivity.production-CEF0hx9W.js → CompanyActivity.production-DeT7xKMQ.js} +1 -1
  19. package/ui-dist/assets/{CompanyExport-DT63ZLv6.js → CompanyExport-BLyfRuJ5.js} +1 -1
  20. package/ui-dist/assets/{CompanySkills.production-74w4FgIl.js → CompanySkills.production-BQbzHmtn.js} +1 -1
  21. package/ui-dist/assets/{Costs.production-B0j-0g6E.js → Costs.production-5RBWzUGB.js} +1 -1
  22. package/ui-dist/assets/{OrgChart.production-BdkDUcmQ.js → OrgChart.production-D_5tQ3tx.js} +1 -1
  23. package/ui-dist/assets/{RoutineDetail.production-grakkTMs.js → RoutineDetail.production-Dd163Q10.js} +1 -1
  24. package/ui-dist/assets/{Routines.production-qCZmmUZQ.js → Routines.production-g9pJjOvz.js} +1 -1
  25. package/ui-dist/assets/{definition-6ehunDEH.js → definition-BtNjkL-Z.js} +1 -1
  26. package/ui-dist/assets/{index-DrB9-JGQ.js → index-B_VCIODd.js} +4 -4
  27. package/ui-dist/index.html +1 -1
  28. package/ui-dist/sw.js +1 -1
@@ -211,6 +211,51 @@ function readConfigurationIncompleteFingerprint(latestRun) {
211
211
  function readNonEmptyString(value) {
212
212
  return typeof value === "string" && value.trim().length > 0 ? value : null;
213
213
  }
214
+ const DATABASE_SESSION_FAILURE_CODES = {
215
+ CONNECT_TIMEOUT: true,
216
+ CONNECTION_CLOSED: true,
217
+ CONNECTION_ENDED: true,
218
+ CONNECTION_DESTROYED: true,
219
+ ECONNREFUSED: true,
220
+ ECONNRESET: true,
221
+ ETIMEDOUT: true,
222
+ EPIPE: true,
223
+ EHOSTUNREACH: true,
224
+ ENETUNREACH: true,
225
+ ENOTFOUND: true,
226
+ EAI_AGAIN: true,
227
+ };
228
+ const DATABASE_SESSION_FAILURE_SQLSTATE_RE = /^(?:(?:08|25|53)[0-9A-Z]{3}|57P[0-9A-Z]{2})$/;
229
+ function isDatabaseSessionFailure(error) {
230
+ let current = error;
231
+ for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
232
+ const code = "code" in current ? current.code : undefined;
233
+ if (typeof code === "string" &&
234
+ (DATABASE_SESSION_FAILURE_CODES[code] || DATABASE_SESSION_FAILURE_SQLSTATE_RE.test(code))) {
235
+ return true;
236
+ }
237
+ current = "cause" in current ? current.cause : undefined;
238
+ }
239
+ return false;
240
+ }
241
+ function issueScope(issue) {
242
+ return { issueId: issue.id, companyId: issue.companyId };
243
+ }
244
+ async function reconcileEachCandidate(loop, candidates, scopeOf, reconcile) {
245
+ let errored = 0;
246
+ for (const candidate of candidates) {
247
+ try {
248
+ await reconcile(candidate);
249
+ }
250
+ catch (error) {
251
+ if (isDatabaseSessionFailure(error))
252
+ throw error;
253
+ errored += 1;
254
+ logger.error({ err: error, loop, ...scopeOf(candidate) }, "recovery loop failed for one candidate; continuing with the next");
255
+ }
256
+ }
257
+ return errored;
258
+ }
214
259
  function summarizeRunFailureForIssueComment(run) {
215
260
  if (!run)
216
261
  return null;
@@ -1242,21 +1287,21 @@ export function recoveryService(db, deps) {
1242
1287
  let skipped = 0;
1243
1288
  const issueIds = [];
1244
1289
  const seen = new Set();
1245
- for (const candidate of candidates) {
1290
+ const errored = await reconcileEachCandidate("unassigned_blocking_issues", candidates, issueScope, async (candidate) => {
1246
1291
  if (seen.has(candidate.id))
1247
- continue;
1292
+ return;
1248
1293
  seen.add(candidate.id);
1249
1294
  const creatorAgentId = candidate.createdByAgentId;
1250
1295
  if (!creatorAgentId) {
1251
1296
  skipped += 1;
1252
- continue;
1297
+ return;
1253
1298
  }
1254
1299
  const creatorAgent = await getAgent(creatorAgentId);
1255
1300
  if (!creatorAgent ||
1256
1301
  creatorAgent.companyId !== candidate.companyId ||
1257
1302
  !(await isAgentInvokable(creatorAgent))) {
1258
1303
  skipped += 1;
1259
- continue;
1304
+ return;
1260
1305
  }
1261
1306
  const relations = await issuesSvc.getRelationSummaries(candidate.id);
1262
1307
  const blockingLinks = formatIssueLinksForComment(relations.blocks);
@@ -1266,7 +1311,7 @@ export function recoveryService(db, deps) {
1266
1311
  });
1267
1312
  if (!updated) {
1268
1313
  skipped += 1;
1269
- continue;
1314
+ return;
1270
1315
  }
1271
1316
  await issuesSvc.addComment(candidate.id, [
1272
1317
  "## Assigned Orphan Blocker",
@@ -1315,8 +1360,8 @@ export function recoveryService(db, deps) {
1315
1360
  else {
1316
1361
  skipped += 1;
1317
1362
  }
1318
- }
1319
- return { assigned, skipped, issueIds };
1363
+ });
1364
+ return { assigned, skipped, errored, issueIds };
1320
1365
  }
1321
1366
  async function getCompanyIssuePrefix(companyId) {
1322
1367
  return db
@@ -2181,15 +2226,16 @@ export function recoveryService(db, deps) {
2181
2226
  escalated: 0,
2182
2227
  resolved: 0,
2183
2228
  skipped: 0,
2229
+ errored: 0,
2184
2230
  issueIds: [],
2185
2231
  };
2186
- for (const { action, issue } of rows) {
2232
+ result.errored = await reconcileEachCandidate("active_recovery_actions", rows, ({ issue }) => issueScope(issue), async ({ action, issue }) => {
2187
2233
  const wakePolicy = parseObject(action.wakePolicy);
2188
2234
  const wakePolicyType = readNonEmptyString(wakePolicy.type);
2189
2235
  if (wakePolicyType !== "bounded_recovery_owner" &&
2190
2236
  wakePolicyType !== "bounded_owner_disposition_repair" &&
2191
2237
  action.ownerType !== "board") {
2192
- continue;
2238
+ return;
2193
2239
  }
2194
2240
  if (issue.status === "done" || issue.status === "cancelled") {
2195
2241
  const resolved = await recoveryActionsSvc.resolveActiveForIssue({
@@ -2204,13 +2250,13 @@ export function recoveryService(db, deps) {
2204
2250
  result.resolved += 1;
2205
2251
  result.issueIds.push(issue.id);
2206
2252
  }
2207
- continue;
2253
+ return;
2208
2254
  }
2209
2255
  // A queued comment or healthy child cannot establish what the stopped
2210
2256
  // provider already did. Only execution reconciliation can clear this hold.
2211
2257
  if (requiresExecutionReconciliation(action.cause)) {
2212
2258
  result.skipped += 1;
2213
- continue;
2259
+ return;
2214
2260
  }
2215
2261
  const [sourceState, healthyChildren, hasNewSourcePath] = await Promise.all([
2216
2262
  collectDispositionRepairSourceState(db, { issue }),
@@ -2249,12 +2295,12 @@ export function recoveryService(db, deps) {
2249
2295
  result.resolved += 1;
2250
2296
  result.issueIds.push(issue.id);
2251
2297
  }
2252
- continue;
2298
+ return;
2253
2299
  }
2254
2300
  if (wakePolicyType === "bounded_owner_disposition_repair") {
2255
2301
  if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc)) {
2256
2302
  result.skipped += 1;
2257
- continue;
2303
+ return;
2258
2304
  }
2259
2305
  const latestRun = await latestRecoveryActionRun(action);
2260
2306
  const persistedAttempt = Math.max(action.attemptCount, Math.max(0, Math.floor(asNumber(wakePolicy.attempt, action.attemptCount))));
@@ -2272,14 +2318,14 @@ export function recoveryService(db, deps) {
2272
2318
  else {
2273
2319
  result.skipped += 1;
2274
2320
  }
2275
- continue;
2321
+ return;
2276
2322
  }
2277
2323
  if (action.ownerType === "board")
2278
- continue;
2324
+ return;
2279
2325
  // Legacy takeover actions remain readable and resolvable, but recovery no
2280
2326
  // longer schedules another agent-owned wake for them.
2281
2327
  result.skipped += 1;
2282
- }
2328
+ });
2283
2329
  return result;
2284
2330
  }
2285
2331
  async function escalateDispositionRepair(input) {
@@ -2845,6 +2891,7 @@ export function recoveryService(db, deps) {
2845
2891
  assigneeNotSchedulableExempted: 0,
2846
2892
  onboardingFirstTaskExempted: 0,
2847
2893
  skipped: 0,
2894
+ errored: 0,
2848
2895
  issueIds: [],
2849
2896
  };
2850
2897
  const candidateIssueIds = candidates.map((issue) => issue.id);
@@ -2862,14 +2909,14 @@ export function recoveryService(db, deps) {
2862
2909
  unfinishedGoalBindings.add(`${goal.companyId}:${goal.taskKey}:${goal.agentId}`);
2863
2910
  }
2864
2911
  }
2865
- for (const issue of candidates) {
2912
+ result.errored = await reconcileEachCandidate("stranded_assigned_issues", candidates, issueScope, async (issue) => {
2866
2913
  if (issue.originKind === "chat_channel") {
2867
2914
  await settleSlackConversation(db, issue.companyId, issue.id);
2868
2915
  const [current] = await db.select({ externalConversationState: externalConversationStateSql() })
2869
2916
  .from(issues).where(and(eq(issues.companyId, issue.companyId), eq(issues.id, issue.id)));
2870
2917
  if (current?.externalConversationState === "waiting") {
2871
2918
  result.skipped += 1;
2872
- continue;
2919
+ return;
2873
2920
  }
2874
2921
  }
2875
2922
  if (issue.conversationAgentId) {
@@ -2883,7 +2930,7 @@ export function recoveryService(db, deps) {
2883
2930
  }
2884
2931
  if (!(await instanceSettingsService(db).getExperimental()).enableAgentChat) {
2885
2932
  result.skipped += 1;
2886
- continue;
2933
+ return;
2887
2934
  }
2888
2935
  {
2889
2936
  await deliverConversationComments(db, issue, deps.enqueueWakeup);
@@ -2891,7 +2938,7 @@ export function recoveryService(db, deps) {
2891
2938
  }
2892
2939
  if (isWaitingConversation(issue)) {
2893
2940
  result.skipped += 1;
2894
- continue;
2941
+ return;
2895
2942
  }
2896
2943
  const executionState = issue.status === "in_review"
2897
2944
  ? parseIssueExecutionState(issue.executionState)
@@ -2908,7 +2955,7 @@ export function recoveryService(db, deps) {
2908
2955
  : issue.assigneeAgentId;
2909
2956
  if (!agentId) {
2910
2957
  result.skipped += 1;
2911
- continue;
2958
+ return;
2912
2959
  }
2913
2960
  // An unfinished durable session goal owns its continuation lifecycle.
2914
2961
  // A paused goal waits for explicit resume; an active goal is handled by
@@ -2916,7 +2963,7 @@ export function recoveryService(db, deps) {
2916
2963
  // either authority and can replace the provider session owning the goal.
2917
2964
  if (unfinishedGoalBindings.has(`${issue.companyId}:${issue.id}:${agentId}`)) {
2918
2965
  result.skipped += 1;
2919
- continue;
2966
+ return;
2920
2967
  }
2921
2968
  let latestRun = await getLatestIssueRun(issue.companyId, issue.id);
2922
2969
  // A native chat can finish between the earlier settlement read and this
@@ -2927,7 +2974,7 @@ export function recoveryService(db, deps) {
2927
2974
  latestRun?.status === "succeeded" &&
2928
2975
  parseObject(latestRun.resultJson).finalizationReasonCode === "conversation_turn_finished") {
2929
2976
  result.skipped += 1;
2930
- continue;
2977
+ return;
2931
2978
  }
2932
2979
  const agent = await getAgent(agentId);
2933
2980
  const agentInvokability = agent && agent.companyId === issue.companyId
@@ -2938,7 +2985,7 @@ export function recoveryService(db, deps) {
2938
2985
  agent.companyId === issue.companyId &&
2939
2986
  (await hasCurrentNativePassiveWait(issue, latestRun))) {
2940
2987
  result.skipped += 1;
2941
- continue;
2988
+ return;
2942
2989
  }
2943
2990
  if (issue.status !== "in_review" && !agentInvokable) {
2944
2991
  const classification = classifyContinuationFailure(latestRun);
@@ -2972,15 +3019,15 @@ export function recoveryService(db, deps) {
2972
3019
  result.skipped += 1;
2973
3020
  }
2974
3021
  }
2975
- continue;
3022
+ return;
2976
3023
  }
2977
3024
  if (await hasActiveExecutionPath(issue.companyId, issue.id, issue.status === "in_review" ? agentId : null)) {
2978
3025
  result.skipped += 1;
2979
- continue;
3026
+ return;
2980
3027
  }
2981
3028
  if (await hasPendingWakeInteraction(issue.companyId, issue.id)) {
2982
3029
  result.skipped += 1;
2983
- continue;
3030
+ return;
2984
3031
  }
2985
3032
  // A board-owned recovery action is already the durable, human-owned
2986
3033
  // continuation path. Generic stranded-work recovery must not race that
@@ -2989,11 +3036,11 @@ export function recoveryService(db, deps) {
2989
3036
  const activeRecoveryAction = await recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id);
2990
3037
  if (activeRecoveryAction?.ownerType === "board") {
2991
3038
  result.skipped += 1;
2992
- continue;
3039
+ return;
2993
3040
  }
2994
3041
  if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc)) {
2995
3042
  result.skipped += 1;
2996
- continue;
3043
+ return;
2997
3044
  }
2998
3045
  const participantLatestRunForRecovery = issue.status === "in_review" && participantAgentId
2999
3046
  ? await getLatestIssueRunForAgent(issue.companyId, issue.id, participantAgentId)
@@ -3009,11 +3056,11 @@ export function recoveryService(db, deps) {
3009
3056
  sourceRunId: executionRecoverySource.id,
3010
3057
  })).kind !== "clear") {
3011
3058
  result.skipped += 1;
3012
- continue;
3059
+ return;
3013
3060
  }
3014
3061
  if (isOperatorCancelledRun(executionRecoverySource, agentId)) {
3015
3062
  result.operatorCancelExempted += 1;
3016
- continue;
3063
+ return;
3017
3064
  }
3018
3065
  if (executionRecoverySource &&
3019
3066
  executionRecoverySource.agentId === agentId &&
@@ -3031,7 +3078,7 @@ export function recoveryService(db, deps) {
3031
3078
  });
3032
3079
  result.escalated += 1;
3033
3080
  result.issueIds.push(issue.id);
3034
- continue;
3081
+ return;
3035
3082
  }
3036
3083
  }
3037
3084
  if (await isInvocationBudgetBlocked(issue, agentId)) {
@@ -3066,7 +3113,7 @@ export function recoveryService(db, deps) {
3066
3113
  result.skipped += 1;
3067
3114
  }
3068
3115
  }
3069
- continue;
3116
+ return;
3070
3117
  }
3071
3118
  const nativeUnblockAction = await nativeBlockedUnblockAction(issue, latestRun);
3072
3119
  if (nativeUnblockAction) {
@@ -3082,12 +3129,12 @@ export function recoveryService(db, deps) {
3082
3129
  else {
3083
3130
  result.skipped += 1;
3084
3131
  }
3085
- continue;
3132
+ return;
3086
3133
  }
3087
3134
  if (latestRun?.status === "succeeded" &&
3088
3135
  (await hasPersistedDurableWaitPath(issue, latestRun))) {
3089
3136
  result.skipped += 1;
3090
- continue;
3137
+ return;
3091
3138
  }
3092
3139
  const recoveryNow = new Date();
3093
3140
  const providerQuotaMonitorRun = issue.status === "in_review"
@@ -3095,7 +3142,7 @@ export function recoveryService(db, deps) {
3095
3142
  : latestRun;
3096
3143
  if (hasPendingProviderQuotaRecoveryMonitor(issue, providerQuotaMonitorRun, recoveryNow)) {
3097
3144
  result.skipped += 1;
3098
- continue;
3145
+ return;
3099
3146
  }
3100
3147
  if (isStrandedIssueRecoveryIssue(issue) &&
3101
3148
  isUnsuccessfulTerminalIssueRun(latestRun)) {
@@ -3111,7 +3158,7 @@ export function recoveryService(db, deps) {
3111
3158
  else {
3112
3159
  result.skipped += 1;
3113
3160
  }
3114
- continue;
3161
+ return;
3115
3162
  }
3116
3163
  const adapterFailureClassification = issue.status !== "in_review" &&
3117
3164
  latestRun &&
@@ -3122,7 +3169,7 @@ export function recoveryService(db, deps) {
3122
3169
  const targetAgentId = getAdapterFailureRecoveryTargetAgentId(issue);
3123
3170
  if (!targetAgentId || latestRun.agentId !== targetAgentId) {
3124
3171
  result.skipped += 1;
3125
- continue;
3172
+ return;
3126
3173
  }
3127
3174
  if (adapterFailureClassification.kind === "provider_quota") {
3128
3175
  const monitored = await scheduleProviderQuotaRecoveryMonitor({
@@ -3134,10 +3181,10 @@ export function recoveryService(db, deps) {
3134
3181
  latestRun = await persistAdapterFailureRecoveryClassification(latestRun, adapterFailureClassification);
3135
3182
  result.providerQuotaMonitored += 1;
3136
3183
  result.issueIds.push(issue.id);
3137
- continue;
3184
+ return;
3138
3185
  }
3139
3186
  result.skipped += 1;
3140
- continue;
3187
+ return;
3141
3188
  }
3142
3189
  else {
3143
3190
  const updated = await escalateStrandedAssignedIssue({
@@ -3156,7 +3203,7 @@ export function recoveryService(db, deps) {
3156
3203
  else {
3157
3204
  result.skipped += 1;
3158
3205
  }
3159
- continue;
3206
+ return;
3160
3207
  }
3161
3208
  }
3162
3209
  const acceptedContinuationInteraction = await getLatestAcceptedContinuationInteraction(issue.companyId, issue.id);
@@ -3172,15 +3219,15 @@ export function recoveryService(db, deps) {
3172
3219
  if (!successfulRunSinceResolution) {
3173
3220
  if (!agentInvokable) {
3174
3221
  result.skipped += 1;
3175
- continue;
3222
+ return;
3176
3223
  }
3177
3224
  if (await hasQueuedIssueWake(issue.companyId, issue.id, agentId)) {
3178
3225
  result.skipped += 1;
3179
- continue;
3226
+ return;
3180
3227
  }
3181
3228
  if (await isInvocationBudgetBlocked(issue, agentId)) {
3182
3229
  result.skipped += 1;
3183
- continue;
3230
+ return;
3184
3231
  }
3185
3232
  const latestPostResolutionRun = await getLatestIssueRunSince(issue.companyId, issue.id, agentId, acceptedInteractionResolvedAt);
3186
3233
  if (classifyContinuationFailure(latestPostResolutionRun).kind ===
@@ -3189,7 +3236,7 @@ export function recoveryService(db, deps) {
3189
3236
  if (resolved) {
3190
3237
  result.waitingOnReviewResolved += 1;
3191
3238
  result.issueIds.push(issue.id);
3192
- continue;
3239
+ return;
3193
3240
  }
3194
3241
  const outcome = await reconcileDispositionRepair(issue, latestPostResolutionRun, {
3195
3242
  historicalAttemptCount: legacyReviewParkAttempts.consecutive,
@@ -3206,7 +3253,7 @@ export function recoveryService(db, deps) {
3206
3253
  else {
3207
3254
  result.skipped += 1;
3208
3255
  }
3209
- continue;
3256
+ return;
3210
3257
  }
3211
3258
  const { consecutive } = legacyReviewParkAttempts;
3212
3259
  if (consecutive >= INTERACTION_CONTINUATION_REQUEUE_MAX_ATTEMPTS &&
@@ -3215,7 +3262,7 @@ export function recoveryService(db, deps) {
3215
3262
  if (resolved) {
3216
3263
  result.waitingOnReviewResolved += 1;
3217
3264
  result.issueIds.push(issue.id);
3218
- continue;
3265
+ return;
3219
3266
  }
3220
3267
  const updated = await escalateStrandedAssignedIssue({
3221
3268
  issue,
@@ -3232,7 +3279,7 @@ export function recoveryService(db, deps) {
3232
3279
  else {
3233
3280
  result.skipped += 1;
3234
3281
  }
3235
- continue;
3282
+ return;
3236
3283
  }
3237
3284
  const queued = await enqueueStrandedIssueRecovery({
3238
3285
  issueId: issue.id,
@@ -3260,20 +3307,20 @@ export function recoveryService(db, deps) {
3260
3307
  else {
3261
3308
  result.skipped += 1;
3262
3309
  }
3263
- continue;
3310
+ return;
3264
3311
  }
3265
3312
  }
3266
3313
  if (issue.status === "in_review") {
3267
3314
  if (!participantAgentId || !pendingExecutionState) {
3268
3315
  result.skipped += 1;
3269
- continue;
3316
+ return;
3270
3317
  }
3271
3318
  const participantLatestRun = participantLatestRunForRecovery;
3272
3319
  if (!participantLatestRun ||
3273
3320
  !isTerminalIssueRun(participantLatestRun)) {
3274
3321
  if (!agentInvokable && isAssigneeLifecycleBlocked(agentInvokability)) {
3275
3322
  result.assigneeNotSchedulableExempted += 1;
3276
- continue;
3323
+ return;
3277
3324
  }
3278
3325
  if (!agentInvokable) {
3279
3326
  const updated = await escalateStrandedAssignedIssue({
@@ -3294,7 +3341,7 @@ export function recoveryService(db, deps) {
3294
3341
  else {
3295
3342
  result.skipped += 1;
3296
3343
  }
3297
- continue;
3344
+ return;
3298
3345
  }
3299
3346
  const participantAdapterFailureClassification = isUnsuccessfulTerminalIssueRun(participantLatestRun)
3300
3347
  ? classifyAdapterFailureForRecovery(participantLatestRun, recoveryNow)
@@ -3313,7 +3360,7 @@ export function recoveryService(db, deps) {
3313
3360
  else {
3314
3361
  result.skipped += 1;
3315
3362
  }
3316
- continue;
3363
+ return;
3317
3364
  }
3318
3365
  if (participantAdapterFailureClassification?.kind ===
3319
3366
  "configuration_incomplete") {
@@ -3334,7 +3381,7 @@ export function recoveryService(db, deps) {
3334
3381
  else {
3335
3382
  result.skipped += 1;
3336
3383
  }
3337
- continue;
3384
+ return;
3338
3385
  }
3339
3386
  if (!agentInvokable) {
3340
3387
  const updated = await escalateStrandedAssignedIssue({
@@ -3351,7 +3398,7 @@ export function recoveryService(db, deps) {
3351
3398
  else {
3352
3399
  result.skipped += 1;
3353
3400
  }
3354
- continue;
3401
+ return;
3355
3402
  }
3356
3403
  if (didAutomaticRecoveryFail(participantLatestRun, EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON)) {
3357
3404
  const updated = await escalateStrandedAssignedIssue({
@@ -3368,15 +3415,15 @@ export function recoveryService(db, deps) {
3368
3415
  else {
3369
3416
  result.skipped += 1;
3370
3417
  }
3371
- continue;
3418
+ return;
3372
3419
  }
3373
3420
  if (await hasQueuedIssueWake(issue.companyId, issue.id, participantAgentId)) {
3374
3421
  result.skipped += 1;
3375
- continue;
3422
+ return;
3376
3423
  }
3377
3424
  if (await isInvocationBudgetBlocked(issue, participantAgentId)) {
3378
3425
  result.skipped += 1;
3379
- continue;
3426
+ return;
3380
3427
  }
3381
3428
  const reviewOutcome = {};
3382
3429
  const queued = await enqueueStrandedIssueRecovery({
@@ -3424,7 +3471,7 @@ export function recoveryService(db, deps) {
3424
3471
  else {
3425
3472
  result.skipped += 1;
3426
3473
  }
3427
- continue;
3474
+ return;
3428
3475
  }
3429
3476
  if (issue.status === "todo") {
3430
3477
  if (!latestRun) {
@@ -3436,15 +3483,15 @@ export function recoveryService(db, deps) {
3436
3483
  // as stranded).
3437
3484
  if (await isOnboardingFirstTaskAwaitingUser(issue)) {
3438
3485
  result.onboardingFirstTaskExempted += 1;
3439
- continue;
3486
+ return;
3440
3487
  }
3441
3488
  if (await hasQueuedIssueWake(issue.companyId, issue.id)) {
3442
3489
  result.skipped += 1;
3443
- continue;
3490
+ return;
3444
3491
  }
3445
3492
  if (await isInvocationBudgetBlocked(issue, agentId)) {
3446
3493
  result.skipped += 1;
3447
- continue;
3494
+ return;
3448
3495
  }
3449
3496
  const queued = await enqueueInitialAssignedTodoDispatch(issue, agentId);
3450
3497
  if (queued) {
@@ -3454,12 +3501,12 @@ export function recoveryService(db, deps) {
3454
3501
  else {
3455
3502
  result.skipped += 1;
3456
3503
  }
3457
- continue;
3504
+ return;
3458
3505
  }
3459
3506
  if (latestRun.status === "succeeded" &&
3460
3507
  !(await wasTodoHandedBackDuringOrAfterLatestRun(issue, latestRun))) {
3461
3508
  result.skipped += 1;
3462
- continue;
3509
+ return;
3463
3510
  }
3464
3511
  if (didAutomaticRecoveryFail(latestRun, "assignment_recovery")) {
3465
3512
  const updated = await escalateStrandedAssignedIssue({
@@ -3481,11 +3528,11 @@ export function recoveryService(db, deps) {
3481
3528
  else {
3482
3529
  result.skipped += 1;
3483
3530
  }
3484
- continue;
3531
+ return;
3485
3532
  }
3486
3533
  if (await isInvocationBudgetBlocked(issue, agentId)) {
3487
3534
  result.skipped += 1;
3488
- continue;
3535
+ return;
3489
3536
  }
3490
3537
  const dispatchOutcome = {};
3491
3538
  const queued = await enqueueStrandedIssueRecovery({
@@ -3531,11 +3578,11 @@ export function recoveryService(db, deps) {
3531
3578
  else {
3532
3579
  result.skipped += 1;
3533
3580
  }
3534
- continue;
3581
+ return;
3535
3582
  }
3536
3583
  if (!latestRun && !issue.checkoutRunId && !issue.executionRunId) {
3537
3584
  result.skipped += 1;
3538
- continue;
3585
+ return;
3539
3586
  }
3540
3587
  if (readDispositionRepairAttempt(latestRun)) {
3541
3588
  const outcome = await reconcileDispositionRepair(issue, latestRun);
@@ -3551,17 +3598,17 @@ export function recoveryService(db, deps) {
3551
3598
  else {
3552
3599
  result.skipped += 1;
3553
3600
  }
3554
- continue;
3601
+ return;
3555
3602
  }
3556
3603
  const handoffEvidence = isExhaustedSuccessfulRunHandoff(latestRun);
3557
3604
  if (handoffEvidence) {
3558
3605
  if (isPluginManagedIssueLifecycle(issue)) {
3559
3606
  result.skipped += 1;
3560
- continue;
3607
+ return;
3561
3608
  }
3562
3609
  if (!handoffEvidence.exhausted) {
3563
3610
  result.skipped += 1;
3564
- continue;
3611
+ return;
3565
3612
  }
3566
3613
  // An interrupted corrective run is not evidence that the agent could
3567
3614
  // not choose a disposition: a graceful server shutdown (a deploy
@@ -3579,7 +3626,7 @@ export function recoveryService(db, deps) {
3579
3626
  if (latestRun?.status === "interrupted") {
3580
3627
  if (await isInvocationBudgetBlocked(issue, agentId)) {
3581
3628
  result.skipped += 1;
3582
- continue;
3629
+ return;
3583
3630
  }
3584
3631
  const retried = await enqueueStrandedIssueRecovery({
3585
3632
  issueId: issue.id,
@@ -3592,7 +3639,7 @@ export function recoveryService(db, deps) {
3592
3639
  if (retried) {
3593
3640
  result.successfulRunHandoffRetried += 1;
3594
3641
  result.issueIds.push(issue.id);
3595
- continue;
3642
+ return;
3596
3643
  }
3597
3644
  }
3598
3645
  const updated = await escalateStrandedAssignedIssue({
@@ -3609,7 +3656,7 @@ export function recoveryService(db, deps) {
3609
3656
  else {
3610
3657
  result.skipped += 1;
3611
3658
  }
3612
- continue;
3659
+ return;
3613
3660
  }
3614
3661
  if (isSuccessfulInProgressContinuationRun(latestRun)) {
3615
3662
  const successfulRun = latestRun;
@@ -3621,12 +3668,12 @@ export function recoveryService(db, deps) {
3621
3668
  if (workspace.mode === "shared_workspace" && (await healthyOpenChildIssues(issue, true)).length > 0) {
3622
3669
  result.productiveContinuationObserved += 1;
3623
3670
  result.skipped += 1;
3624
- continue;
3671
+ return;
3625
3672
  }
3626
3673
  if (!isProductiveContinuationRun(successfulRun)) {
3627
3674
  result.successfulContinuationObserved += 1;
3628
3675
  result.skipped += 1;
3629
- continue;
3676
+ return;
3630
3677
  }
3631
3678
  if (isRepeatedProductiveContinuationRecovery(successfulRun)) {
3632
3679
  // GGU-809: skip escalation if the assignee has shown visible progress
@@ -3649,13 +3696,13 @@ export function recoveryService(db, deps) {
3649
3696
  else {
3650
3697
  result.skipped += 1;
3651
3698
  }
3652
- continue;
3699
+ return;
3653
3700
  }
3654
3701
  result.recentProgressExempted += 1;
3655
3702
  }
3656
3703
  if (await isInvocationBudgetBlocked(issue, agentId)) {
3657
3704
  result.skipped += 1;
3658
- continue;
3705
+ return;
3659
3706
  }
3660
3707
  const queued = await enqueueStrandedIssueRecovery({
3661
3708
  issueId: issue.id,
@@ -3672,7 +3719,7 @@ export function recoveryService(db, deps) {
3672
3719
  else {
3673
3720
  result.skipped += 1;
3674
3721
  }
3675
- continue;
3722
+ return;
3676
3723
  }
3677
3724
  if (isUnsuccessfulTerminalIssueRun(latestRun)) {
3678
3725
  const classification = classifyContinuationFailure(latestRun);
@@ -3681,7 +3728,7 @@ export function recoveryService(db, deps) {
3681
3728
  if (resolved) {
3682
3729
  result.waitingOnReviewResolved += 1;
3683
3730
  result.issueIds.push(issue.id);
3684
- continue;
3731
+ return;
3685
3732
  }
3686
3733
  const outcome = await reconcileDispositionRepair(issue, latestRun);
3687
3734
  if (outcome === "queued") {
@@ -3696,7 +3743,7 @@ export function recoveryService(db, deps) {
3696
3743
  else {
3697
3744
  result.skipped += 1;
3698
3745
  }
3699
- continue;
3746
+ return;
3700
3747
  }
3701
3748
  if (classification.kind === "non_retryable") {
3702
3749
  const updated = await escalateStrandedAssignedIssue({
@@ -3718,7 +3765,7 @@ export function recoveryService(db, deps) {
3718
3765
  else {
3719
3766
  result.skipped += 1;
3720
3767
  }
3721
- continue;
3768
+ return;
3722
3769
  }
3723
3770
  if (didAutomaticRecoveryFail(latestRun, "issue_continuation_needed")) {
3724
3771
  const { consecutive, latestFinishedAt } = await summarizeRecentContinuationRetries(issue.companyId, issue.id, agentId, classification.errorCode);
@@ -3743,7 +3790,7 @@ export function recoveryService(db, deps) {
3743
3790
  else {
3744
3791
  result.skipped += 1;
3745
3792
  }
3746
- continue;
3793
+ return;
3747
3794
  }
3748
3795
  if (classification.baseBackoffMs > 0 && latestFinishedAt) {
3749
3796
  const elapsed = Date.now() - latestFinishedAt.getTime();
@@ -3751,14 +3798,14 @@ export function recoveryService(db, deps) {
3751
3798
  Math.pow(2, Math.max(0, consecutive - 1));
3752
3799
  if (elapsed < requiredDelay) {
3753
3800
  result.skipped += 1;
3754
- continue;
3801
+ return;
3755
3802
  }
3756
3803
  }
3757
3804
  }
3758
3805
  }
3759
3806
  if (await isInvocationBudgetBlocked(issue, agentId)) {
3760
3807
  result.skipped += 1;
3761
- continue;
3808
+ return;
3762
3809
  }
3763
3810
  const recoveryOutcome = {};
3764
3811
  const queued = await enqueueStrandedIssueRecovery({
@@ -3805,15 +3852,17 @@ export function recoveryService(db, deps) {
3805
3852
  else {
3806
3853
  result.skipped += 1;
3807
3854
  }
3808
- }
3855
+ });
3809
3856
  const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues();
3810
3857
  result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
3811
3858
  result.skipped += orphanBlockerRecovery.skipped;
3859
+ result.errored += orphanBlockerRecovery.errored;
3812
3860
  result.issueIds.push(...orphanBlockerRecovery.issueIds);
3813
3861
  const activeRecovery = await reconcileActiveRecoveryActions();
3814
3862
  result.continuationRequeued += activeRecovery.requeued;
3815
3863
  result.escalated += activeRecovery.escalated;
3816
3864
  result.skipped += activeRecovery.skipped;
3865
+ result.errored += activeRecovery.errored;
3817
3866
  result.issueIds.push(...activeRecovery.issueIds);
3818
3867
  result.issueIds = [...new Set(result.issueIds)];
3819
3868
  return result;
@@ -3830,6 +3879,7 @@ export function recoveryService(db, deps) {
3830
3879
  candidateLimitSkipped: 0,
3831
3880
  deferredOrFailed: 0,
3832
3881
  enqueueFailed: 0,
3882
+ errored: 0,
3833
3883
  issueIds: [],
3834
3884
  };
3835
3885
  const source = opts?.source ?? "issue_graph_liveness.backstop";
@@ -3920,10 +3970,10 @@ export function recoveryService(db, deps) {
3920
3970
  }
3921
3971
  for (const [companyId, companyCandidates,] of candidatesByCompany.entries()) {
3922
3972
  const readinessMap = await issuesSvc.listDependencyReadiness(companyId, companyCandidates.map((candidate) => candidate.id));
3923
- for (const candidate of companyCandidates) {
3973
+ result.errored += await reconcileEachCandidate("resolved_dependency_wake_backstop", companyCandidates, issueScope, async (candidate) => {
3924
3974
  const agentId = candidate.assigneeAgentId;
3925
3975
  if (!agentId)
3926
- continue;
3976
+ return;
3927
3977
  const readiness = readinessMap.get(candidate.id);
3928
3978
  const resolvedBlockerIssueId = readiness?.blockerIssueIds[0] ?? null;
3929
3979
  if (!readiness ||
@@ -3931,7 +3981,7 @@ export function recoveryService(db, deps) {
3931
3981
  readiness.blockerIssueIds.length === 0 ||
3932
3982
  !resolvedBlockerIssueId) {
3933
3983
  result.notReadySkipped += 1;
3934
- continue;
3984
+ return;
3935
3985
  }
3936
3986
  // Level-triggered dedup: key on the full blocker set (the current ready
3937
3987
  // state), not on any single resolved edge. An older completed per-edge
@@ -3951,7 +4001,7 @@ export function recoveryService(db, deps) {
3951
4001
  });
3952
4002
  if (existingWake) {
3953
4003
  result.existingWakeSkipped += 1;
3954
- continue;
4004
+ return;
3955
4005
  }
3956
4006
  await releaseDependencyGateRecoveryHold(db, {
3957
4007
  companyId,
@@ -3964,15 +4014,15 @@ export function recoveryService(db, deps) {
3964
4014
  })) ||
3965
4015
  (await hasQueuedIssueWake(companyId, candidate.id, agentId))) {
3966
4016
  result.livePathSkipped += 1;
3967
- continue;
4017
+ return;
3968
4018
  }
3969
4019
  if (await hasPendingWakeInteraction(companyId, candidate.id)) {
3970
4020
  result.interactionSkipped += 1;
3971
- continue;
4021
+ return;
3972
4022
  }
3973
4023
  if (await isAutomaticRecoverySuppressedByPauseHold(db, companyId, candidate.id, treeControlSvc)) {
3974
4024
  result.pauseHoldSkipped += 1;
3975
- continue;
4025
+ return;
3976
4026
  }
3977
4027
  try {
3978
4028
  const wake = await deps.enqueueWakeup(agentId, {
@@ -4002,7 +4052,7 @@ export function recoveryService(db, deps) {
4002
4052
  // such as disabled wake-on-demand or concurrency gating. That is
4003
4053
  // not an enqueue error, but the backstop still did not heal now.
4004
4054
  result.deferredOrFailed += 1;
4005
- continue;
4055
+ return;
4006
4056
  }
4007
4057
  result.healed += 1;
4008
4058
  result.issueIds.push(candidate.id);
@@ -4029,7 +4079,7 @@ export function recoveryService(db, deps) {
4029
4079
  result.enqueueFailed += 1;
4030
4080
  logger.warn({ err, issueId: candidate.id, agentId, idempotencyKey, source }, "failed to enqueue dependency wake from issue graph liveness backstop");
4031
4081
  }
4032
- }
4082
+ });
4033
4083
  }
4034
4084
  if (result.healed > 0) {
4035
4085
  logger.warn({
@@ -4278,6 +4328,7 @@ export function recoveryService(db, deps) {
4278
4328
  const result = {
4279
4329
  cleared: 0,
4280
4330
  issueIds: [],
4331
+ errored: 0,
4281
4332
  terminalizedRunIds: [],
4282
4333
  };
4283
4334
  const candidates = await db
@@ -4343,7 +4394,7 @@ export function recoveryService(db, deps) {
4343
4394
  // can no longer reach a terminal status on its own. This lets the sweep
4344
4395
  // clear the lock in the same pass instead of waiting for the run to reach a
4345
4396
  // terminal status by another route.
4346
- for (const row of runRows) {
4397
+ result.errored += await reconcileEachCandidate("stale_issue_lock_run_terminalization", runRows, (row) => ({ runId: row.id, companyId: row.companyId }), async (row) => {
4347
4398
  const outcome = await terminalizeOrphanedRunningRun(row, {
4348
4399
  referencingIssueTerminalStatus: issueTerminalStatusByRunId.get(row.id) ?? null,
4349
4400
  runReferencedByActiveIssue: runIdsReferencedByActiveIssue.has(row.id),
@@ -4351,7 +4402,7 @@ export function recoveryService(db, deps) {
4351
4402
  runStatusById.set(row.id, outcome.status);
4352
4403
  if (outcome.terminalized)
4353
4404
  result.terminalizedRunIds.push(row.id);
4354
- }
4405
+ });
4355
4406
  const isCleanable = (runId) => {
4356
4407
  if (!runId)
4357
4408
  return true;
@@ -4360,10 +4411,10 @@ export function recoveryService(db, deps) {
4360
4411
  return true; // missing run row → no real claim
4361
4412
  return TERMINAL_HEARTBEAT_RUN_STATUSES.has(status);
4362
4413
  };
4363
- for (const issue of candidates) {
4414
+ result.errored += await reconcileEachCandidate("stale_issue_lock_clear", candidates, issueScope, async (issue) => {
4364
4415
  if (!isCleanable(issue.checkoutRunId) ||
4365
4416
  !isCleanable(issue.executionRunId)) {
4366
- continue;
4417
+ return;
4367
4418
  }
4368
4419
  const updated = await db
4369
4420
  .update(issues)
@@ -4382,7 +4433,7 @@ export function recoveryService(db, deps) {
4382
4433
  .returning({ id: issues.id })
4383
4434
  .then((rows) => rows[0] ?? null);
4384
4435
  if (!updated)
4385
- continue;
4436
+ return;
4386
4437
  result.cleared += 1;
4387
4438
  result.issueIds.push(updated.id);
4388
4439
  await logActivity(db, {
@@ -4401,12 +4452,13 @@ export function recoveryService(db, deps) {
4401
4452
  referencedRunStatuses: Object.fromEntries(runStatusById),
4402
4453
  },
4403
4454
  });
4404
- }
4405
- if (result.cleared > 0 || result.terminalizedRunIds.length > 0) {
4455
+ });
4456
+ if (result.cleared > 0 || result.terminalizedRunIds.length > 0 || result.errored > 0) {
4406
4457
  logger.warn({
4407
4458
  cleared: result.cleared,
4408
4459
  issueIds: result.issueIds,
4409
4460
  terminalizedRunIds: result.terminalizedRunIds,
4461
+ errored: result.errored,
4410
4462
  }, "swept stale issue lock columns");
4411
4463
  }
4412
4464
  return result;