@tickernelz/paperclip-pro-server 2026.1001.0 → 2026.1001.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/services/heartbeat.d.ts +18 -2
- package/dist/services/heartbeat.d.ts.map +1 -1
- package/dist/services/heartbeat.js +194 -30
- package/dist/services/heartbeat.js.map +1 -1
- package/dist/services/hot-restart.d.ts +4 -3
- package/dist/services/hot-restart.d.ts.map +1 -1
- package/dist/services/hot-restart.js +1 -1
- package/dist/services/hot-restart.js.map +1 -1
- package/dist/services/recovery/service.d.ts +3 -0
- package/dist/services/recovery/service.d.ts.map +1 -1
- package/dist/services/recovery/service.js +154 -102
- package/dist/services/recovery/service.js.map +1 -1
- package/package.json +21 -21
- package/ui-dist/assets/{Agents.production-fNeT2seP.js → Agents.production-tjOEQe3l.js} +2 -2
- package/ui-dist/assets/{CompanyActivity.production-CEF0hx9W.js → CompanyActivity.production-DeT7xKMQ.js} +1 -1
- package/ui-dist/assets/{CompanyExport-DT63ZLv6.js → CompanyExport-BLyfRuJ5.js} +1 -1
- package/ui-dist/assets/{CompanySkills.production-74w4FgIl.js → CompanySkills.production-BQbzHmtn.js} +1 -1
- package/ui-dist/assets/{Costs.production-B0j-0g6E.js → Costs.production-5RBWzUGB.js} +1 -1
- package/ui-dist/assets/{OrgChart.production-BdkDUcmQ.js → OrgChart.production-D_5tQ3tx.js} +1 -1
- package/ui-dist/assets/{RoutineDetail.production-grakkTMs.js → RoutineDetail.production-Dd163Q10.js} +1 -1
- package/ui-dist/assets/{Routines.production-qCZmmUZQ.js → Routines.production-g9pJjOvz.js} +1 -1
- package/ui-dist/assets/{definition-6ehunDEH.js → definition-BtNjkL-Z.js} +1 -1
- package/ui-dist/assets/{index-DrB9-JGQ.js → index-B_VCIODd.js} +4 -4
- package/ui-dist/index.html +1 -1
- package/ui-dist/sw.js +1 -1
|
@@ -211,6 +211,51 @@ function readConfigurationIncompleteFingerprint(latestRun) {
|
|
|
211
211
|
function readNonEmptyString(value) {
|
|
212
212
|
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
|
213
213
|
}
|
|
214
|
+
const DATABASE_SESSION_FAILURE_CODES = {
|
|
215
|
+
CONNECT_TIMEOUT: true,
|
|
216
|
+
CONNECTION_CLOSED: true,
|
|
217
|
+
CONNECTION_ENDED: true,
|
|
218
|
+
CONNECTION_DESTROYED: true,
|
|
219
|
+
ECONNREFUSED: true,
|
|
220
|
+
ECONNRESET: true,
|
|
221
|
+
ETIMEDOUT: true,
|
|
222
|
+
EPIPE: true,
|
|
223
|
+
EHOSTUNREACH: true,
|
|
224
|
+
ENETUNREACH: true,
|
|
225
|
+
ENOTFOUND: true,
|
|
226
|
+
EAI_AGAIN: true,
|
|
227
|
+
};
|
|
228
|
+
const DATABASE_SESSION_FAILURE_SQLSTATE_RE = /^(?:(?:08|25|53)[0-9A-Z]{3}|57P[0-9A-Z]{2})$/;
|
|
229
|
+
function isDatabaseSessionFailure(error) {
|
|
230
|
+
let current = error;
|
|
231
|
+
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
|
|
232
|
+
const code = "code" in current ? current.code : undefined;
|
|
233
|
+
if (typeof code === "string" &&
|
|
234
|
+
(DATABASE_SESSION_FAILURE_CODES[code] || DATABASE_SESSION_FAILURE_SQLSTATE_RE.test(code))) {
|
|
235
|
+
return true;
|
|
236
|
+
}
|
|
237
|
+
current = "cause" in current ? current.cause : undefined;
|
|
238
|
+
}
|
|
239
|
+
return false;
|
|
240
|
+
}
|
|
241
|
+
function issueScope(issue) {
|
|
242
|
+
return { issueId: issue.id, companyId: issue.companyId };
|
|
243
|
+
}
|
|
244
|
+
async function reconcileEachCandidate(loop, candidates, scopeOf, reconcile) {
|
|
245
|
+
let errored = 0;
|
|
246
|
+
for (const candidate of candidates) {
|
|
247
|
+
try {
|
|
248
|
+
await reconcile(candidate);
|
|
249
|
+
}
|
|
250
|
+
catch (error) {
|
|
251
|
+
if (isDatabaseSessionFailure(error))
|
|
252
|
+
throw error;
|
|
253
|
+
errored += 1;
|
|
254
|
+
logger.error({ err: error, loop, ...scopeOf(candidate) }, "recovery loop failed for one candidate; continuing with the next");
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
return errored;
|
|
258
|
+
}
|
|
214
259
|
function summarizeRunFailureForIssueComment(run) {
|
|
215
260
|
if (!run)
|
|
216
261
|
return null;
|
|
@@ -1242,21 +1287,21 @@ export function recoveryService(db, deps) {
|
|
|
1242
1287
|
let skipped = 0;
|
|
1243
1288
|
const issueIds = [];
|
|
1244
1289
|
const seen = new Set();
|
|
1245
|
-
|
|
1290
|
+
const errored = await reconcileEachCandidate("unassigned_blocking_issues", candidates, issueScope, async (candidate) => {
|
|
1246
1291
|
if (seen.has(candidate.id))
|
|
1247
|
-
|
|
1292
|
+
return;
|
|
1248
1293
|
seen.add(candidate.id);
|
|
1249
1294
|
const creatorAgentId = candidate.createdByAgentId;
|
|
1250
1295
|
if (!creatorAgentId) {
|
|
1251
1296
|
skipped += 1;
|
|
1252
|
-
|
|
1297
|
+
return;
|
|
1253
1298
|
}
|
|
1254
1299
|
const creatorAgent = await getAgent(creatorAgentId);
|
|
1255
1300
|
if (!creatorAgent ||
|
|
1256
1301
|
creatorAgent.companyId !== candidate.companyId ||
|
|
1257
1302
|
!(await isAgentInvokable(creatorAgent))) {
|
|
1258
1303
|
skipped += 1;
|
|
1259
|
-
|
|
1304
|
+
return;
|
|
1260
1305
|
}
|
|
1261
1306
|
const relations = await issuesSvc.getRelationSummaries(candidate.id);
|
|
1262
1307
|
const blockingLinks = formatIssueLinksForComment(relations.blocks);
|
|
@@ -1266,7 +1311,7 @@ export function recoveryService(db, deps) {
|
|
|
1266
1311
|
});
|
|
1267
1312
|
if (!updated) {
|
|
1268
1313
|
skipped += 1;
|
|
1269
|
-
|
|
1314
|
+
return;
|
|
1270
1315
|
}
|
|
1271
1316
|
await issuesSvc.addComment(candidate.id, [
|
|
1272
1317
|
"## Assigned Orphan Blocker",
|
|
@@ -1315,8 +1360,8 @@ export function recoveryService(db, deps) {
|
|
|
1315
1360
|
else {
|
|
1316
1361
|
skipped += 1;
|
|
1317
1362
|
}
|
|
1318
|
-
}
|
|
1319
|
-
return { assigned, skipped, issueIds };
|
|
1363
|
+
});
|
|
1364
|
+
return { assigned, skipped, errored, issueIds };
|
|
1320
1365
|
}
|
|
1321
1366
|
async function getCompanyIssuePrefix(companyId) {
|
|
1322
1367
|
return db
|
|
@@ -2181,15 +2226,16 @@ export function recoveryService(db, deps) {
|
|
|
2181
2226
|
escalated: 0,
|
|
2182
2227
|
resolved: 0,
|
|
2183
2228
|
skipped: 0,
|
|
2229
|
+
errored: 0,
|
|
2184
2230
|
issueIds: [],
|
|
2185
2231
|
};
|
|
2186
|
-
|
|
2232
|
+
result.errored = await reconcileEachCandidate("active_recovery_actions", rows, ({ issue }) => issueScope(issue), async ({ action, issue }) => {
|
|
2187
2233
|
const wakePolicy = parseObject(action.wakePolicy);
|
|
2188
2234
|
const wakePolicyType = readNonEmptyString(wakePolicy.type);
|
|
2189
2235
|
if (wakePolicyType !== "bounded_recovery_owner" &&
|
|
2190
2236
|
wakePolicyType !== "bounded_owner_disposition_repair" &&
|
|
2191
2237
|
action.ownerType !== "board") {
|
|
2192
|
-
|
|
2238
|
+
return;
|
|
2193
2239
|
}
|
|
2194
2240
|
if (issue.status === "done" || issue.status === "cancelled") {
|
|
2195
2241
|
const resolved = await recoveryActionsSvc.resolveActiveForIssue({
|
|
@@ -2204,13 +2250,13 @@ export function recoveryService(db, deps) {
|
|
|
2204
2250
|
result.resolved += 1;
|
|
2205
2251
|
result.issueIds.push(issue.id);
|
|
2206
2252
|
}
|
|
2207
|
-
|
|
2253
|
+
return;
|
|
2208
2254
|
}
|
|
2209
2255
|
// A queued comment or healthy child cannot establish what the stopped
|
|
2210
2256
|
// provider already did. Only execution reconciliation can clear this hold.
|
|
2211
2257
|
if (requiresExecutionReconciliation(action.cause)) {
|
|
2212
2258
|
result.skipped += 1;
|
|
2213
|
-
|
|
2259
|
+
return;
|
|
2214
2260
|
}
|
|
2215
2261
|
const [sourceState, healthyChildren, hasNewSourcePath] = await Promise.all([
|
|
2216
2262
|
collectDispositionRepairSourceState(db, { issue }),
|
|
@@ -2249,12 +2295,12 @@ export function recoveryService(db, deps) {
|
|
|
2249
2295
|
result.resolved += 1;
|
|
2250
2296
|
result.issueIds.push(issue.id);
|
|
2251
2297
|
}
|
|
2252
|
-
|
|
2298
|
+
return;
|
|
2253
2299
|
}
|
|
2254
2300
|
if (wakePolicyType === "bounded_owner_disposition_repair") {
|
|
2255
2301
|
if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc)) {
|
|
2256
2302
|
result.skipped += 1;
|
|
2257
|
-
|
|
2303
|
+
return;
|
|
2258
2304
|
}
|
|
2259
2305
|
const latestRun = await latestRecoveryActionRun(action);
|
|
2260
2306
|
const persistedAttempt = Math.max(action.attemptCount, Math.max(0, Math.floor(asNumber(wakePolicy.attempt, action.attemptCount))));
|
|
@@ -2272,14 +2318,14 @@ export function recoveryService(db, deps) {
|
|
|
2272
2318
|
else {
|
|
2273
2319
|
result.skipped += 1;
|
|
2274
2320
|
}
|
|
2275
|
-
|
|
2321
|
+
return;
|
|
2276
2322
|
}
|
|
2277
2323
|
if (action.ownerType === "board")
|
|
2278
|
-
|
|
2324
|
+
return;
|
|
2279
2325
|
// Legacy takeover actions remain readable and resolvable, but recovery no
|
|
2280
2326
|
// longer schedules another agent-owned wake for them.
|
|
2281
2327
|
result.skipped += 1;
|
|
2282
|
-
}
|
|
2328
|
+
});
|
|
2283
2329
|
return result;
|
|
2284
2330
|
}
|
|
2285
2331
|
async function escalateDispositionRepair(input) {
|
|
@@ -2845,6 +2891,7 @@ export function recoveryService(db, deps) {
|
|
|
2845
2891
|
assigneeNotSchedulableExempted: 0,
|
|
2846
2892
|
onboardingFirstTaskExempted: 0,
|
|
2847
2893
|
skipped: 0,
|
|
2894
|
+
errored: 0,
|
|
2848
2895
|
issueIds: [],
|
|
2849
2896
|
};
|
|
2850
2897
|
const candidateIssueIds = candidates.map((issue) => issue.id);
|
|
@@ -2862,14 +2909,14 @@ export function recoveryService(db, deps) {
|
|
|
2862
2909
|
unfinishedGoalBindings.add(`${goal.companyId}:${goal.taskKey}:${goal.agentId}`);
|
|
2863
2910
|
}
|
|
2864
2911
|
}
|
|
2865
|
-
|
|
2912
|
+
result.errored = await reconcileEachCandidate("stranded_assigned_issues", candidates, issueScope, async (issue) => {
|
|
2866
2913
|
if (issue.originKind === "chat_channel") {
|
|
2867
2914
|
await settleSlackConversation(db, issue.companyId, issue.id);
|
|
2868
2915
|
const [current] = await db.select({ externalConversationState: externalConversationStateSql() })
|
|
2869
2916
|
.from(issues).where(and(eq(issues.companyId, issue.companyId), eq(issues.id, issue.id)));
|
|
2870
2917
|
if (current?.externalConversationState === "waiting") {
|
|
2871
2918
|
result.skipped += 1;
|
|
2872
|
-
|
|
2919
|
+
return;
|
|
2873
2920
|
}
|
|
2874
2921
|
}
|
|
2875
2922
|
if (issue.conversationAgentId) {
|
|
@@ -2883,7 +2930,7 @@ export function recoveryService(db, deps) {
|
|
|
2883
2930
|
}
|
|
2884
2931
|
if (!(await instanceSettingsService(db).getExperimental()).enableAgentChat) {
|
|
2885
2932
|
result.skipped += 1;
|
|
2886
|
-
|
|
2933
|
+
return;
|
|
2887
2934
|
}
|
|
2888
2935
|
{
|
|
2889
2936
|
await deliverConversationComments(db, issue, deps.enqueueWakeup);
|
|
@@ -2891,7 +2938,7 @@ export function recoveryService(db, deps) {
|
|
|
2891
2938
|
}
|
|
2892
2939
|
if (isWaitingConversation(issue)) {
|
|
2893
2940
|
result.skipped += 1;
|
|
2894
|
-
|
|
2941
|
+
return;
|
|
2895
2942
|
}
|
|
2896
2943
|
const executionState = issue.status === "in_review"
|
|
2897
2944
|
? parseIssueExecutionState(issue.executionState)
|
|
@@ -2908,7 +2955,7 @@ export function recoveryService(db, deps) {
|
|
|
2908
2955
|
: issue.assigneeAgentId;
|
|
2909
2956
|
if (!agentId) {
|
|
2910
2957
|
result.skipped += 1;
|
|
2911
|
-
|
|
2958
|
+
return;
|
|
2912
2959
|
}
|
|
2913
2960
|
// An unfinished durable session goal owns its continuation lifecycle.
|
|
2914
2961
|
// A paused goal waits for explicit resume; an active goal is handled by
|
|
@@ -2916,7 +2963,7 @@ export function recoveryService(db, deps) {
|
|
|
2916
2963
|
// either authority and can replace the provider session owning the goal.
|
|
2917
2964
|
if (unfinishedGoalBindings.has(`${issue.companyId}:${issue.id}:${agentId}`)) {
|
|
2918
2965
|
result.skipped += 1;
|
|
2919
|
-
|
|
2966
|
+
return;
|
|
2920
2967
|
}
|
|
2921
2968
|
let latestRun = await getLatestIssueRun(issue.companyId, issue.id);
|
|
2922
2969
|
// A native chat can finish between the earlier settlement read and this
|
|
@@ -2927,7 +2974,7 @@ export function recoveryService(db, deps) {
|
|
|
2927
2974
|
latestRun?.status === "succeeded" &&
|
|
2928
2975
|
parseObject(latestRun.resultJson).finalizationReasonCode === "conversation_turn_finished") {
|
|
2929
2976
|
result.skipped += 1;
|
|
2930
|
-
|
|
2977
|
+
return;
|
|
2931
2978
|
}
|
|
2932
2979
|
const agent = await getAgent(agentId);
|
|
2933
2980
|
const agentInvokability = agent && agent.companyId === issue.companyId
|
|
@@ -2938,7 +2985,7 @@ export function recoveryService(db, deps) {
|
|
|
2938
2985
|
agent.companyId === issue.companyId &&
|
|
2939
2986
|
(await hasCurrentNativePassiveWait(issue, latestRun))) {
|
|
2940
2987
|
result.skipped += 1;
|
|
2941
|
-
|
|
2988
|
+
return;
|
|
2942
2989
|
}
|
|
2943
2990
|
if (issue.status !== "in_review" && !agentInvokable) {
|
|
2944
2991
|
const classification = classifyContinuationFailure(latestRun);
|
|
@@ -2972,15 +3019,15 @@ export function recoveryService(db, deps) {
|
|
|
2972
3019
|
result.skipped += 1;
|
|
2973
3020
|
}
|
|
2974
3021
|
}
|
|
2975
|
-
|
|
3022
|
+
return;
|
|
2976
3023
|
}
|
|
2977
3024
|
if (await hasActiveExecutionPath(issue.companyId, issue.id, issue.status === "in_review" ? agentId : null)) {
|
|
2978
3025
|
result.skipped += 1;
|
|
2979
|
-
|
|
3026
|
+
return;
|
|
2980
3027
|
}
|
|
2981
3028
|
if (await hasPendingWakeInteraction(issue.companyId, issue.id)) {
|
|
2982
3029
|
result.skipped += 1;
|
|
2983
|
-
|
|
3030
|
+
return;
|
|
2984
3031
|
}
|
|
2985
3032
|
// A board-owned recovery action is already the durable, human-owned
|
|
2986
3033
|
// continuation path. Generic stranded-work recovery must not race that
|
|
@@ -2989,11 +3036,11 @@ export function recoveryService(db, deps) {
|
|
|
2989
3036
|
const activeRecoveryAction = await recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id);
|
|
2990
3037
|
if (activeRecoveryAction?.ownerType === "board") {
|
|
2991
3038
|
result.skipped += 1;
|
|
2992
|
-
|
|
3039
|
+
return;
|
|
2993
3040
|
}
|
|
2994
3041
|
if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id, treeControlSvc)) {
|
|
2995
3042
|
result.skipped += 1;
|
|
2996
|
-
|
|
3043
|
+
return;
|
|
2997
3044
|
}
|
|
2998
3045
|
const participantLatestRunForRecovery = issue.status === "in_review" && participantAgentId
|
|
2999
3046
|
? await getLatestIssueRunForAgent(issue.companyId, issue.id, participantAgentId)
|
|
@@ -3009,11 +3056,11 @@ export function recoveryService(db, deps) {
|
|
|
3009
3056
|
sourceRunId: executionRecoverySource.id,
|
|
3010
3057
|
})).kind !== "clear") {
|
|
3011
3058
|
result.skipped += 1;
|
|
3012
|
-
|
|
3059
|
+
return;
|
|
3013
3060
|
}
|
|
3014
3061
|
if (isOperatorCancelledRun(executionRecoverySource, agentId)) {
|
|
3015
3062
|
result.operatorCancelExempted += 1;
|
|
3016
|
-
|
|
3063
|
+
return;
|
|
3017
3064
|
}
|
|
3018
3065
|
if (executionRecoverySource &&
|
|
3019
3066
|
executionRecoverySource.agentId === agentId &&
|
|
@@ -3031,7 +3078,7 @@ export function recoveryService(db, deps) {
|
|
|
3031
3078
|
});
|
|
3032
3079
|
result.escalated += 1;
|
|
3033
3080
|
result.issueIds.push(issue.id);
|
|
3034
|
-
|
|
3081
|
+
return;
|
|
3035
3082
|
}
|
|
3036
3083
|
}
|
|
3037
3084
|
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
|
@@ -3066,7 +3113,7 @@ export function recoveryService(db, deps) {
|
|
|
3066
3113
|
result.skipped += 1;
|
|
3067
3114
|
}
|
|
3068
3115
|
}
|
|
3069
|
-
|
|
3116
|
+
return;
|
|
3070
3117
|
}
|
|
3071
3118
|
const nativeUnblockAction = await nativeBlockedUnblockAction(issue, latestRun);
|
|
3072
3119
|
if (nativeUnblockAction) {
|
|
@@ -3082,12 +3129,12 @@ export function recoveryService(db, deps) {
|
|
|
3082
3129
|
else {
|
|
3083
3130
|
result.skipped += 1;
|
|
3084
3131
|
}
|
|
3085
|
-
|
|
3132
|
+
return;
|
|
3086
3133
|
}
|
|
3087
3134
|
if (latestRun?.status === "succeeded" &&
|
|
3088
3135
|
(await hasPersistedDurableWaitPath(issue, latestRun))) {
|
|
3089
3136
|
result.skipped += 1;
|
|
3090
|
-
|
|
3137
|
+
return;
|
|
3091
3138
|
}
|
|
3092
3139
|
const recoveryNow = new Date();
|
|
3093
3140
|
const providerQuotaMonitorRun = issue.status === "in_review"
|
|
@@ -3095,7 +3142,7 @@ export function recoveryService(db, deps) {
|
|
|
3095
3142
|
: latestRun;
|
|
3096
3143
|
if (hasPendingProviderQuotaRecoveryMonitor(issue, providerQuotaMonitorRun, recoveryNow)) {
|
|
3097
3144
|
result.skipped += 1;
|
|
3098
|
-
|
|
3145
|
+
return;
|
|
3099
3146
|
}
|
|
3100
3147
|
if (isStrandedIssueRecoveryIssue(issue) &&
|
|
3101
3148
|
isUnsuccessfulTerminalIssueRun(latestRun)) {
|
|
@@ -3111,7 +3158,7 @@ export function recoveryService(db, deps) {
|
|
|
3111
3158
|
else {
|
|
3112
3159
|
result.skipped += 1;
|
|
3113
3160
|
}
|
|
3114
|
-
|
|
3161
|
+
return;
|
|
3115
3162
|
}
|
|
3116
3163
|
const adapterFailureClassification = issue.status !== "in_review" &&
|
|
3117
3164
|
latestRun &&
|
|
@@ -3122,7 +3169,7 @@ export function recoveryService(db, deps) {
|
|
|
3122
3169
|
const targetAgentId = getAdapterFailureRecoveryTargetAgentId(issue);
|
|
3123
3170
|
if (!targetAgentId || latestRun.agentId !== targetAgentId) {
|
|
3124
3171
|
result.skipped += 1;
|
|
3125
|
-
|
|
3172
|
+
return;
|
|
3126
3173
|
}
|
|
3127
3174
|
if (adapterFailureClassification.kind === "provider_quota") {
|
|
3128
3175
|
const monitored = await scheduleProviderQuotaRecoveryMonitor({
|
|
@@ -3134,10 +3181,10 @@ export function recoveryService(db, deps) {
|
|
|
3134
3181
|
latestRun = await persistAdapterFailureRecoveryClassification(latestRun, adapterFailureClassification);
|
|
3135
3182
|
result.providerQuotaMonitored += 1;
|
|
3136
3183
|
result.issueIds.push(issue.id);
|
|
3137
|
-
|
|
3184
|
+
return;
|
|
3138
3185
|
}
|
|
3139
3186
|
result.skipped += 1;
|
|
3140
|
-
|
|
3187
|
+
return;
|
|
3141
3188
|
}
|
|
3142
3189
|
else {
|
|
3143
3190
|
const updated = await escalateStrandedAssignedIssue({
|
|
@@ -3156,7 +3203,7 @@ export function recoveryService(db, deps) {
|
|
|
3156
3203
|
else {
|
|
3157
3204
|
result.skipped += 1;
|
|
3158
3205
|
}
|
|
3159
|
-
|
|
3206
|
+
return;
|
|
3160
3207
|
}
|
|
3161
3208
|
}
|
|
3162
3209
|
const acceptedContinuationInteraction = await getLatestAcceptedContinuationInteraction(issue.companyId, issue.id);
|
|
@@ -3172,15 +3219,15 @@ export function recoveryService(db, deps) {
|
|
|
3172
3219
|
if (!successfulRunSinceResolution) {
|
|
3173
3220
|
if (!agentInvokable) {
|
|
3174
3221
|
result.skipped += 1;
|
|
3175
|
-
|
|
3222
|
+
return;
|
|
3176
3223
|
}
|
|
3177
3224
|
if (await hasQueuedIssueWake(issue.companyId, issue.id, agentId)) {
|
|
3178
3225
|
result.skipped += 1;
|
|
3179
|
-
|
|
3226
|
+
return;
|
|
3180
3227
|
}
|
|
3181
3228
|
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
|
3182
3229
|
result.skipped += 1;
|
|
3183
|
-
|
|
3230
|
+
return;
|
|
3184
3231
|
}
|
|
3185
3232
|
const latestPostResolutionRun = await getLatestIssueRunSince(issue.companyId, issue.id, agentId, acceptedInteractionResolvedAt);
|
|
3186
3233
|
if (classifyContinuationFailure(latestPostResolutionRun).kind ===
|
|
@@ -3189,7 +3236,7 @@ export function recoveryService(db, deps) {
|
|
|
3189
3236
|
if (resolved) {
|
|
3190
3237
|
result.waitingOnReviewResolved += 1;
|
|
3191
3238
|
result.issueIds.push(issue.id);
|
|
3192
|
-
|
|
3239
|
+
return;
|
|
3193
3240
|
}
|
|
3194
3241
|
const outcome = await reconcileDispositionRepair(issue, latestPostResolutionRun, {
|
|
3195
3242
|
historicalAttemptCount: legacyReviewParkAttempts.consecutive,
|
|
@@ -3206,7 +3253,7 @@ export function recoveryService(db, deps) {
|
|
|
3206
3253
|
else {
|
|
3207
3254
|
result.skipped += 1;
|
|
3208
3255
|
}
|
|
3209
|
-
|
|
3256
|
+
return;
|
|
3210
3257
|
}
|
|
3211
3258
|
const { consecutive } = legacyReviewParkAttempts;
|
|
3212
3259
|
if (consecutive >= INTERACTION_CONTINUATION_REQUEUE_MAX_ATTEMPTS &&
|
|
@@ -3215,7 +3262,7 @@ export function recoveryService(db, deps) {
|
|
|
3215
3262
|
if (resolved) {
|
|
3216
3263
|
result.waitingOnReviewResolved += 1;
|
|
3217
3264
|
result.issueIds.push(issue.id);
|
|
3218
|
-
|
|
3265
|
+
return;
|
|
3219
3266
|
}
|
|
3220
3267
|
const updated = await escalateStrandedAssignedIssue({
|
|
3221
3268
|
issue,
|
|
@@ -3232,7 +3279,7 @@ export function recoveryService(db, deps) {
|
|
|
3232
3279
|
else {
|
|
3233
3280
|
result.skipped += 1;
|
|
3234
3281
|
}
|
|
3235
|
-
|
|
3282
|
+
return;
|
|
3236
3283
|
}
|
|
3237
3284
|
const queued = await enqueueStrandedIssueRecovery({
|
|
3238
3285
|
issueId: issue.id,
|
|
@@ -3260,20 +3307,20 @@ export function recoveryService(db, deps) {
|
|
|
3260
3307
|
else {
|
|
3261
3308
|
result.skipped += 1;
|
|
3262
3309
|
}
|
|
3263
|
-
|
|
3310
|
+
return;
|
|
3264
3311
|
}
|
|
3265
3312
|
}
|
|
3266
3313
|
if (issue.status === "in_review") {
|
|
3267
3314
|
if (!participantAgentId || !pendingExecutionState) {
|
|
3268
3315
|
result.skipped += 1;
|
|
3269
|
-
|
|
3316
|
+
return;
|
|
3270
3317
|
}
|
|
3271
3318
|
const participantLatestRun = participantLatestRunForRecovery;
|
|
3272
3319
|
if (!participantLatestRun ||
|
|
3273
3320
|
!isTerminalIssueRun(participantLatestRun)) {
|
|
3274
3321
|
if (!agentInvokable && isAssigneeLifecycleBlocked(agentInvokability)) {
|
|
3275
3322
|
result.assigneeNotSchedulableExempted += 1;
|
|
3276
|
-
|
|
3323
|
+
return;
|
|
3277
3324
|
}
|
|
3278
3325
|
if (!agentInvokable) {
|
|
3279
3326
|
const updated = await escalateStrandedAssignedIssue({
|
|
@@ -3294,7 +3341,7 @@ export function recoveryService(db, deps) {
|
|
|
3294
3341
|
else {
|
|
3295
3342
|
result.skipped += 1;
|
|
3296
3343
|
}
|
|
3297
|
-
|
|
3344
|
+
return;
|
|
3298
3345
|
}
|
|
3299
3346
|
const participantAdapterFailureClassification = isUnsuccessfulTerminalIssueRun(participantLatestRun)
|
|
3300
3347
|
? classifyAdapterFailureForRecovery(participantLatestRun, recoveryNow)
|
|
@@ -3313,7 +3360,7 @@ export function recoveryService(db, deps) {
|
|
|
3313
3360
|
else {
|
|
3314
3361
|
result.skipped += 1;
|
|
3315
3362
|
}
|
|
3316
|
-
|
|
3363
|
+
return;
|
|
3317
3364
|
}
|
|
3318
3365
|
if (participantAdapterFailureClassification?.kind ===
|
|
3319
3366
|
"configuration_incomplete") {
|
|
@@ -3334,7 +3381,7 @@ export function recoveryService(db, deps) {
|
|
|
3334
3381
|
else {
|
|
3335
3382
|
result.skipped += 1;
|
|
3336
3383
|
}
|
|
3337
|
-
|
|
3384
|
+
return;
|
|
3338
3385
|
}
|
|
3339
3386
|
if (!agentInvokable) {
|
|
3340
3387
|
const updated = await escalateStrandedAssignedIssue({
|
|
@@ -3351,7 +3398,7 @@ export function recoveryService(db, deps) {
|
|
|
3351
3398
|
else {
|
|
3352
3399
|
result.skipped += 1;
|
|
3353
3400
|
}
|
|
3354
|
-
|
|
3401
|
+
return;
|
|
3355
3402
|
}
|
|
3356
3403
|
if (didAutomaticRecoveryFail(participantLatestRun, EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON)) {
|
|
3357
3404
|
const updated = await escalateStrandedAssignedIssue({
|
|
@@ -3368,15 +3415,15 @@ export function recoveryService(db, deps) {
|
|
|
3368
3415
|
else {
|
|
3369
3416
|
result.skipped += 1;
|
|
3370
3417
|
}
|
|
3371
|
-
|
|
3418
|
+
return;
|
|
3372
3419
|
}
|
|
3373
3420
|
if (await hasQueuedIssueWake(issue.companyId, issue.id, participantAgentId)) {
|
|
3374
3421
|
result.skipped += 1;
|
|
3375
|
-
|
|
3422
|
+
return;
|
|
3376
3423
|
}
|
|
3377
3424
|
if (await isInvocationBudgetBlocked(issue, participantAgentId)) {
|
|
3378
3425
|
result.skipped += 1;
|
|
3379
|
-
|
|
3426
|
+
return;
|
|
3380
3427
|
}
|
|
3381
3428
|
const reviewOutcome = {};
|
|
3382
3429
|
const queued = await enqueueStrandedIssueRecovery({
|
|
@@ -3424,7 +3471,7 @@ export function recoveryService(db, deps) {
|
|
|
3424
3471
|
else {
|
|
3425
3472
|
result.skipped += 1;
|
|
3426
3473
|
}
|
|
3427
|
-
|
|
3474
|
+
return;
|
|
3428
3475
|
}
|
|
3429
3476
|
if (issue.status === "todo") {
|
|
3430
3477
|
if (!latestRun) {
|
|
@@ -3436,15 +3483,15 @@ export function recoveryService(db, deps) {
|
|
|
3436
3483
|
// as stranded).
|
|
3437
3484
|
if (await isOnboardingFirstTaskAwaitingUser(issue)) {
|
|
3438
3485
|
result.onboardingFirstTaskExempted += 1;
|
|
3439
|
-
|
|
3486
|
+
return;
|
|
3440
3487
|
}
|
|
3441
3488
|
if (await hasQueuedIssueWake(issue.companyId, issue.id)) {
|
|
3442
3489
|
result.skipped += 1;
|
|
3443
|
-
|
|
3490
|
+
return;
|
|
3444
3491
|
}
|
|
3445
3492
|
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
|
3446
3493
|
result.skipped += 1;
|
|
3447
|
-
|
|
3494
|
+
return;
|
|
3448
3495
|
}
|
|
3449
3496
|
const queued = await enqueueInitialAssignedTodoDispatch(issue, agentId);
|
|
3450
3497
|
if (queued) {
|
|
@@ -3454,12 +3501,12 @@ export function recoveryService(db, deps) {
|
|
|
3454
3501
|
else {
|
|
3455
3502
|
result.skipped += 1;
|
|
3456
3503
|
}
|
|
3457
|
-
|
|
3504
|
+
return;
|
|
3458
3505
|
}
|
|
3459
3506
|
if (latestRun.status === "succeeded" &&
|
|
3460
3507
|
!(await wasTodoHandedBackDuringOrAfterLatestRun(issue, latestRun))) {
|
|
3461
3508
|
result.skipped += 1;
|
|
3462
|
-
|
|
3509
|
+
return;
|
|
3463
3510
|
}
|
|
3464
3511
|
if (didAutomaticRecoveryFail(latestRun, "assignment_recovery")) {
|
|
3465
3512
|
const updated = await escalateStrandedAssignedIssue({
|
|
@@ -3481,11 +3528,11 @@ export function recoveryService(db, deps) {
|
|
|
3481
3528
|
else {
|
|
3482
3529
|
result.skipped += 1;
|
|
3483
3530
|
}
|
|
3484
|
-
|
|
3531
|
+
return;
|
|
3485
3532
|
}
|
|
3486
3533
|
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
|
3487
3534
|
result.skipped += 1;
|
|
3488
|
-
|
|
3535
|
+
return;
|
|
3489
3536
|
}
|
|
3490
3537
|
const dispatchOutcome = {};
|
|
3491
3538
|
const queued = await enqueueStrandedIssueRecovery({
|
|
@@ -3531,11 +3578,11 @@ export function recoveryService(db, deps) {
|
|
|
3531
3578
|
else {
|
|
3532
3579
|
result.skipped += 1;
|
|
3533
3580
|
}
|
|
3534
|
-
|
|
3581
|
+
return;
|
|
3535
3582
|
}
|
|
3536
3583
|
if (!latestRun && !issue.checkoutRunId && !issue.executionRunId) {
|
|
3537
3584
|
result.skipped += 1;
|
|
3538
|
-
|
|
3585
|
+
return;
|
|
3539
3586
|
}
|
|
3540
3587
|
if (readDispositionRepairAttempt(latestRun)) {
|
|
3541
3588
|
const outcome = await reconcileDispositionRepair(issue, latestRun);
|
|
@@ -3551,17 +3598,17 @@ export function recoveryService(db, deps) {
|
|
|
3551
3598
|
else {
|
|
3552
3599
|
result.skipped += 1;
|
|
3553
3600
|
}
|
|
3554
|
-
|
|
3601
|
+
return;
|
|
3555
3602
|
}
|
|
3556
3603
|
const handoffEvidence = isExhaustedSuccessfulRunHandoff(latestRun);
|
|
3557
3604
|
if (handoffEvidence) {
|
|
3558
3605
|
if (isPluginManagedIssueLifecycle(issue)) {
|
|
3559
3606
|
result.skipped += 1;
|
|
3560
|
-
|
|
3607
|
+
return;
|
|
3561
3608
|
}
|
|
3562
3609
|
if (!handoffEvidence.exhausted) {
|
|
3563
3610
|
result.skipped += 1;
|
|
3564
|
-
|
|
3611
|
+
return;
|
|
3565
3612
|
}
|
|
3566
3613
|
// An interrupted corrective run is not evidence that the agent could
|
|
3567
3614
|
// not choose a disposition: a graceful server shutdown (a deploy
|
|
@@ -3579,7 +3626,7 @@ export function recoveryService(db, deps) {
|
|
|
3579
3626
|
if (latestRun?.status === "interrupted") {
|
|
3580
3627
|
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
|
3581
3628
|
result.skipped += 1;
|
|
3582
|
-
|
|
3629
|
+
return;
|
|
3583
3630
|
}
|
|
3584
3631
|
const retried = await enqueueStrandedIssueRecovery({
|
|
3585
3632
|
issueId: issue.id,
|
|
@@ -3592,7 +3639,7 @@ export function recoveryService(db, deps) {
|
|
|
3592
3639
|
if (retried) {
|
|
3593
3640
|
result.successfulRunHandoffRetried += 1;
|
|
3594
3641
|
result.issueIds.push(issue.id);
|
|
3595
|
-
|
|
3642
|
+
return;
|
|
3596
3643
|
}
|
|
3597
3644
|
}
|
|
3598
3645
|
const updated = await escalateStrandedAssignedIssue({
|
|
@@ -3609,7 +3656,7 @@ export function recoveryService(db, deps) {
|
|
|
3609
3656
|
else {
|
|
3610
3657
|
result.skipped += 1;
|
|
3611
3658
|
}
|
|
3612
|
-
|
|
3659
|
+
return;
|
|
3613
3660
|
}
|
|
3614
3661
|
if (isSuccessfulInProgressContinuationRun(latestRun)) {
|
|
3615
3662
|
const successfulRun = latestRun;
|
|
@@ -3621,12 +3668,12 @@ export function recoveryService(db, deps) {
|
|
|
3621
3668
|
if (workspace.mode === "shared_workspace" && (await healthyOpenChildIssues(issue, true)).length > 0) {
|
|
3622
3669
|
result.productiveContinuationObserved += 1;
|
|
3623
3670
|
result.skipped += 1;
|
|
3624
|
-
|
|
3671
|
+
return;
|
|
3625
3672
|
}
|
|
3626
3673
|
if (!isProductiveContinuationRun(successfulRun)) {
|
|
3627
3674
|
result.successfulContinuationObserved += 1;
|
|
3628
3675
|
result.skipped += 1;
|
|
3629
|
-
|
|
3676
|
+
return;
|
|
3630
3677
|
}
|
|
3631
3678
|
if (isRepeatedProductiveContinuationRecovery(successfulRun)) {
|
|
3632
3679
|
// GGU-809: skip escalation if the assignee has shown visible progress
|
|
@@ -3649,13 +3696,13 @@ export function recoveryService(db, deps) {
|
|
|
3649
3696
|
else {
|
|
3650
3697
|
result.skipped += 1;
|
|
3651
3698
|
}
|
|
3652
|
-
|
|
3699
|
+
return;
|
|
3653
3700
|
}
|
|
3654
3701
|
result.recentProgressExempted += 1;
|
|
3655
3702
|
}
|
|
3656
3703
|
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
|
3657
3704
|
result.skipped += 1;
|
|
3658
|
-
|
|
3705
|
+
return;
|
|
3659
3706
|
}
|
|
3660
3707
|
const queued = await enqueueStrandedIssueRecovery({
|
|
3661
3708
|
issueId: issue.id,
|
|
@@ -3672,7 +3719,7 @@ export function recoveryService(db, deps) {
|
|
|
3672
3719
|
else {
|
|
3673
3720
|
result.skipped += 1;
|
|
3674
3721
|
}
|
|
3675
|
-
|
|
3722
|
+
return;
|
|
3676
3723
|
}
|
|
3677
3724
|
if (isUnsuccessfulTerminalIssueRun(latestRun)) {
|
|
3678
3725
|
const classification = classifyContinuationFailure(latestRun);
|
|
@@ -3681,7 +3728,7 @@ export function recoveryService(db, deps) {
|
|
|
3681
3728
|
if (resolved) {
|
|
3682
3729
|
result.waitingOnReviewResolved += 1;
|
|
3683
3730
|
result.issueIds.push(issue.id);
|
|
3684
|
-
|
|
3731
|
+
return;
|
|
3685
3732
|
}
|
|
3686
3733
|
const outcome = await reconcileDispositionRepair(issue, latestRun);
|
|
3687
3734
|
if (outcome === "queued") {
|
|
@@ -3696,7 +3743,7 @@ export function recoveryService(db, deps) {
|
|
|
3696
3743
|
else {
|
|
3697
3744
|
result.skipped += 1;
|
|
3698
3745
|
}
|
|
3699
|
-
|
|
3746
|
+
return;
|
|
3700
3747
|
}
|
|
3701
3748
|
if (classification.kind === "non_retryable") {
|
|
3702
3749
|
const updated = await escalateStrandedAssignedIssue({
|
|
@@ -3718,7 +3765,7 @@ export function recoveryService(db, deps) {
|
|
|
3718
3765
|
else {
|
|
3719
3766
|
result.skipped += 1;
|
|
3720
3767
|
}
|
|
3721
|
-
|
|
3768
|
+
return;
|
|
3722
3769
|
}
|
|
3723
3770
|
if (didAutomaticRecoveryFail(latestRun, "issue_continuation_needed")) {
|
|
3724
3771
|
const { consecutive, latestFinishedAt } = await summarizeRecentContinuationRetries(issue.companyId, issue.id, agentId, classification.errorCode);
|
|
@@ -3743,7 +3790,7 @@ export function recoveryService(db, deps) {
|
|
|
3743
3790
|
else {
|
|
3744
3791
|
result.skipped += 1;
|
|
3745
3792
|
}
|
|
3746
|
-
|
|
3793
|
+
return;
|
|
3747
3794
|
}
|
|
3748
3795
|
if (classification.baseBackoffMs > 0 && latestFinishedAt) {
|
|
3749
3796
|
const elapsed = Date.now() - latestFinishedAt.getTime();
|
|
@@ -3751,14 +3798,14 @@ export function recoveryService(db, deps) {
|
|
|
3751
3798
|
Math.pow(2, Math.max(0, consecutive - 1));
|
|
3752
3799
|
if (elapsed < requiredDelay) {
|
|
3753
3800
|
result.skipped += 1;
|
|
3754
|
-
|
|
3801
|
+
return;
|
|
3755
3802
|
}
|
|
3756
3803
|
}
|
|
3757
3804
|
}
|
|
3758
3805
|
}
|
|
3759
3806
|
if (await isInvocationBudgetBlocked(issue, agentId)) {
|
|
3760
3807
|
result.skipped += 1;
|
|
3761
|
-
|
|
3808
|
+
return;
|
|
3762
3809
|
}
|
|
3763
3810
|
const recoveryOutcome = {};
|
|
3764
3811
|
const queued = await enqueueStrandedIssueRecovery({
|
|
@@ -3805,15 +3852,17 @@ export function recoveryService(db, deps) {
|
|
|
3805
3852
|
else {
|
|
3806
3853
|
result.skipped += 1;
|
|
3807
3854
|
}
|
|
3808
|
-
}
|
|
3855
|
+
});
|
|
3809
3856
|
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues();
|
|
3810
3857
|
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
|
|
3811
3858
|
result.skipped += orphanBlockerRecovery.skipped;
|
|
3859
|
+
result.errored += orphanBlockerRecovery.errored;
|
|
3812
3860
|
result.issueIds.push(...orphanBlockerRecovery.issueIds);
|
|
3813
3861
|
const activeRecovery = await reconcileActiveRecoveryActions();
|
|
3814
3862
|
result.continuationRequeued += activeRecovery.requeued;
|
|
3815
3863
|
result.escalated += activeRecovery.escalated;
|
|
3816
3864
|
result.skipped += activeRecovery.skipped;
|
|
3865
|
+
result.errored += activeRecovery.errored;
|
|
3817
3866
|
result.issueIds.push(...activeRecovery.issueIds);
|
|
3818
3867
|
result.issueIds = [...new Set(result.issueIds)];
|
|
3819
3868
|
return result;
|
|
@@ -3830,6 +3879,7 @@ export function recoveryService(db, deps) {
|
|
|
3830
3879
|
candidateLimitSkipped: 0,
|
|
3831
3880
|
deferredOrFailed: 0,
|
|
3832
3881
|
enqueueFailed: 0,
|
|
3882
|
+
errored: 0,
|
|
3833
3883
|
issueIds: [],
|
|
3834
3884
|
};
|
|
3835
3885
|
const source = opts?.source ?? "issue_graph_liveness.backstop";
|
|
@@ -3920,10 +3970,10 @@ export function recoveryService(db, deps) {
|
|
|
3920
3970
|
}
|
|
3921
3971
|
for (const [companyId, companyCandidates,] of candidatesByCompany.entries()) {
|
|
3922
3972
|
const readinessMap = await issuesSvc.listDependencyReadiness(companyId, companyCandidates.map((candidate) => candidate.id));
|
|
3923
|
-
|
|
3973
|
+
result.errored += await reconcileEachCandidate("resolved_dependency_wake_backstop", companyCandidates, issueScope, async (candidate) => {
|
|
3924
3974
|
const agentId = candidate.assigneeAgentId;
|
|
3925
3975
|
if (!agentId)
|
|
3926
|
-
|
|
3976
|
+
return;
|
|
3927
3977
|
const readiness = readinessMap.get(candidate.id);
|
|
3928
3978
|
const resolvedBlockerIssueId = readiness?.blockerIssueIds[0] ?? null;
|
|
3929
3979
|
if (!readiness ||
|
|
@@ -3931,7 +3981,7 @@ export function recoveryService(db, deps) {
|
|
|
3931
3981
|
readiness.blockerIssueIds.length === 0 ||
|
|
3932
3982
|
!resolvedBlockerIssueId) {
|
|
3933
3983
|
result.notReadySkipped += 1;
|
|
3934
|
-
|
|
3984
|
+
return;
|
|
3935
3985
|
}
|
|
3936
3986
|
// Level-triggered dedup: key on the full blocker set (the current ready
|
|
3937
3987
|
// state), not on any single resolved edge. An older completed per-edge
|
|
@@ -3951,7 +4001,7 @@ export function recoveryService(db, deps) {
|
|
|
3951
4001
|
});
|
|
3952
4002
|
if (existingWake) {
|
|
3953
4003
|
result.existingWakeSkipped += 1;
|
|
3954
|
-
|
|
4004
|
+
return;
|
|
3955
4005
|
}
|
|
3956
4006
|
await releaseDependencyGateRecoveryHold(db, {
|
|
3957
4007
|
companyId,
|
|
@@ -3964,15 +4014,15 @@ export function recoveryService(db, deps) {
|
|
|
3964
4014
|
})) ||
|
|
3965
4015
|
(await hasQueuedIssueWake(companyId, candidate.id, agentId))) {
|
|
3966
4016
|
result.livePathSkipped += 1;
|
|
3967
|
-
|
|
4017
|
+
return;
|
|
3968
4018
|
}
|
|
3969
4019
|
if (await hasPendingWakeInteraction(companyId, candidate.id)) {
|
|
3970
4020
|
result.interactionSkipped += 1;
|
|
3971
|
-
|
|
4021
|
+
return;
|
|
3972
4022
|
}
|
|
3973
4023
|
if (await isAutomaticRecoverySuppressedByPauseHold(db, companyId, candidate.id, treeControlSvc)) {
|
|
3974
4024
|
result.pauseHoldSkipped += 1;
|
|
3975
|
-
|
|
4025
|
+
return;
|
|
3976
4026
|
}
|
|
3977
4027
|
try {
|
|
3978
4028
|
const wake = await deps.enqueueWakeup(agentId, {
|
|
@@ -4002,7 +4052,7 @@ export function recoveryService(db, deps) {
|
|
|
4002
4052
|
// such as disabled wake-on-demand or concurrency gating. That is
|
|
4003
4053
|
// not an enqueue error, but the backstop still did not heal now.
|
|
4004
4054
|
result.deferredOrFailed += 1;
|
|
4005
|
-
|
|
4055
|
+
return;
|
|
4006
4056
|
}
|
|
4007
4057
|
result.healed += 1;
|
|
4008
4058
|
result.issueIds.push(candidate.id);
|
|
@@ -4029,7 +4079,7 @@ export function recoveryService(db, deps) {
|
|
|
4029
4079
|
result.enqueueFailed += 1;
|
|
4030
4080
|
logger.warn({ err, issueId: candidate.id, agentId, idempotencyKey, source }, "failed to enqueue dependency wake from issue graph liveness backstop");
|
|
4031
4081
|
}
|
|
4032
|
-
}
|
|
4082
|
+
});
|
|
4033
4083
|
}
|
|
4034
4084
|
if (result.healed > 0) {
|
|
4035
4085
|
logger.warn({
|
|
@@ -4278,6 +4328,7 @@ export function recoveryService(db, deps) {
|
|
|
4278
4328
|
const result = {
|
|
4279
4329
|
cleared: 0,
|
|
4280
4330
|
issueIds: [],
|
|
4331
|
+
errored: 0,
|
|
4281
4332
|
terminalizedRunIds: [],
|
|
4282
4333
|
};
|
|
4283
4334
|
const candidates = await db
|
|
@@ -4343,7 +4394,7 @@ export function recoveryService(db, deps) {
|
|
|
4343
4394
|
// can no longer reach a terminal status on its own. This lets the sweep
|
|
4344
4395
|
// clear the lock in the same pass instead of waiting for the run to reach a
|
|
4345
4396
|
// terminal status by another route.
|
|
4346
|
-
|
|
4397
|
+
result.errored += await reconcileEachCandidate("stale_issue_lock_run_terminalization", runRows, (row) => ({ runId: row.id, companyId: row.companyId }), async (row) => {
|
|
4347
4398
|
const outcome = await terminalizeOrphanedRunningRun(row, {
|
|
4348
4399
|
referencingIssueTerminalStatus: issueTerminalStatusByRunId.get(row.id) ?? null,
|
|
4349
4400
|
runReferencedByActiveIssue: runIdsReferencedByActiveIssue.has(row.id),
|
|
@@ -4351,7 +4402,7 @@ export function recoveryService(db, deps) {
|
|
|
4351
4402
|
runStatusById.set(row.id, outcome.status);
|
|
4352
4403
|
if (outcome.terminalized)
|
|
4353
4404
|
result.terminalizedRunIds.push(row.id);
|
|
4354
|
-
}
|
|
4405
|
+
});
|
|
4355
4406
|
const isCleanable = (runId) => {
|
|
4356
4407
|
if (!runId)
|
|
4357
4408
|
return true;
|
|
@@ -4360,10 +4411,10 @@ export function recoveryService(db, deps) {
|
|
|
4360
4411
|
return true; // missing run row → no real claim
|
|
4361
4412
|
return TERMINAL_HEARTBEAT_RUN_STATUSES.has(status);
|
|
4362
4413
|
};
|
|
4363
|
-
|
|
4414
|
+
result.errored += await reconcileEachCandidate("stale_issue_lock_clear", candidates, issueScope, async (issue) => {
|
|
4364
4415
|
if (!isCleanable(issue.checkoutRunId) ||
|
|
4365
4416
|
!isCleanable(issue.executionRunId)) {
|
|
4366
|
-
|
|
4417
|
+
return;
|
|
4367
4418
|
}
|
|
4368
4419
|
const updated = await db
|
|
4369
4420
|
.update(issues)
|
|
@@ -4382,7 +4433,7 @@ export function recoveryService(db, deps) {
|
|
|
4382
4433
|
.returning({ id: issues.id })
|
|
4383
4434
|
.then((rows) => rows[0] ?? null);
|
|
4384
4435
|
if (!updated)
|
|
4385
|
-
|
|
4436
|
+
return;
|
|
4386
4437
|
result.cleared += 1;
|
|
4387
4438
|
result.issueIds.push(updated.id);
|
|
4388
4439
|
await logActivity(db, {
|
|
@@ -4401,12 +4452,13 @@ export function recoveryService(db, deps) {
|
|
|
4401
4452
|
referencedRunStatuses: Object.fromEntries(runStatusById),
|
|
4402
4453
|
},
|
|
4403
4454
|
});
|
|
4404
|
-
}
|
|
4405
|
-
if (result.cleared > 0 || result.terminalizedRunIds.length > 0) {
|
|
4455
|
+
});
|
|
4456
|
+
if (result.cleared > 0 || result.terminalizedRunIds.length > 0 || result.errored > 0) {
|
|
4406
4457
|
logger.warn({
|
|
4407
4458
|
cleared: result.cleared,
|
|
4408
4459
|
issueIds: result.issueIds,
|
|
4409
4460
|
terminalizedRunIds: result.terminalizedRunIds,
|
|
4461
|
+
errored: result.errored,
|
|
4410
4462
|
}, "swept stale issue lock columns");
|
|
4411
4463
|
}
|
|
4412
4464
|
return result;
|