agent-dealer 1.2.15 → 1.2.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle/server/dist/adapters/agent-health.js +21 -0
- package/bundle/server/dist/coordinator/auto-merge.js +8 -1
- package/bundle/server/dist/coordinator/commands.js +153 -13
- package/bundle/server/dist/coordinator/human-resolution.js +4 -0
- package/bundle/server/dist/coordinator/human-resolution.test.js +292 -2
- package/bundle/server/dist/coordinator/projection.js +26 -3
- package/bundle/server/dist/coordinator/projection.test.js +8 -1
- package/bundle/server/dist/coordinator/routing.js +96 -0
- package/bundle/server/dist/coordinator/routing.test.js +163 -0
- package/bundle/server/dist/coordinator/runtime-auth-park.js +97 -0
- package/bundle/server/dist/coordinator/worker-loop.js +15 -0
- package/bundle/server/dist/docs-host-awake.test.js +16 -0
- package/bundle/server/dist/index.js +14 -1
- package/bundle/server/dist/power/host-awake-lifecycle.js +30 -0
- package/bundle/server/dist/power/host-awake-lifecycle.test.js +181 -0
- package/bundle/server/dist/power/host-awake.js +162 -0
- package/bundle/server/dist/power/host-awake.test.js +192 -0
- package/bundle/server/dist/power/sleep-timer.js +86 -0
- package/bundle/server/dist/power/sleep-timer.test.js +79 -0
- package/bundle/server/dist/power/status.js +31 -0
- package/bundle/server/dist/power/status.test.js +60 -0
- package/bundle/server/dist/routes/index.js +3 -0
- package/bundle/server/package.json +4 -3
- package/bundle/server/static-ui/assets/index-DL8mEXzN.css +1 -0
- package/bundle/server/static-ui/assets/{index-DVyqblie.js → index-FCHzrnLr.js} +13 -13
- package/bundle/server/static-ui/index.html +2 -2
- package/bundle/shared/package.json +1 -1
- package/dist/bin.js +3 -0
- package/dist/node-hardening.d.ts +9 -0
- package/dist/node-hardening.js +38 -0
- package/dist/ports.js +54 -14
- package/dist/stop-einval.test.d.ts +1 -0
- package/dist/stop-einval.test.js +215 -0
- package/dist/stop.d.ts +6 -1
- package/dist/stop.js +18 -7
- package/dist/version-output.test.js +23 -16
- package/package.json +1 -1
- package/bundle/server/static-ui/assets/index-COYJxD1y.css +0 -1
|
@@ -29,7 +29,13 @@ export function projectDeveloperRoute(route, currentStatus, round) {
|
|
|
29
29
|
: `Developer retrying (infra attempt) — ${route.reason}`,
|
|
30
30
|
events: ["worker.failed"],
|
|
31
31
|
},
|
|
32
|
-
effect: {
|
|
32
|
+
effect: {
|
|
33
|
+
kind: "enqueue",
|
|
34
|
+
workItem: "developer",
|
|
35
|
+
retryReason: route.reason,
|
|
36
|
+
// NOT-368: mark the work item so a second high-confidence auth failure parks.
|
|
37
|
+
...(route.authTransientRetry ? { authTransientRetry: true } : {}),
|
|
38
|
+
},
|
|
33
39
|
advance: isCiRepair ? "ci" : "infra",
|
|
34
40
|
};
|
|
35
41
|
}
|
|
@@ -67,6 +73,9 @@ export function projectDeveloperRoute(route, currentStatus, round) {
|
|
|
67
73
|
? { pushDivergence: route.pushDivergence }
|
|
68
74
|
: {}),
|
|
69
75
|
...(route.blockerFingerprint ? { blockerFingerprint: route.blockerFingerprint } : {}),
|
|
76
|
+
...("runtimeAuthPark" in route && route.runtimeAuthPark
|
|
77
|
+
? { runtimeAuthPark: route.runtimeAuthPark }
|
|
78
|
+
: {}),
|
|
70
79
|
},
|
|
71
80
|
advance: "none",
|
|
72
81
|
};
|
|
@@ -140,7 +149,14 @@ export function projectReviewerRoute(route, round, hasVerdict) {
|
|
|
140
149
|
currentIntent: `Reviewer retrying (infra attempt) at ${route.headSha.slice(0, 8)} — ${route.reason}`,
|
|
141
150
|
events: ["worker.failed"],
|
|
142
151
|
},
|
|
143
|
-
effect: {
|
|
152
|
+
effect: {
|
|
153
|
+
kind: "enqueue",
|
|
154
|
+
workItem: "reviewer",
|
|
155
|
+
atHeadSha: route.headSha,
|
|
156
|
+
retryReason: route.reason,
|
|
157
|
+
// NOT-368: mark the work item so a second high-confidence auth failure parks.
|
|
158
|
+
...(route.authTransientRetry ? { authTransientRetry: true } : {}),
|
|
159
|
+
},
|
|
144
160
|
advance: "infra",
|
|
145
161
|
hasVerdict,
|
|
146
162
|
};
|
|
@@ -152,7 +168,14 @@ export function projectReviewerRoute(route, round, hasVerdict) {
|
|
|
152
168
|
currentIntent: route.reason,
|
|
153
169
|
events: [hasVerdict ? "worker.completed" : "worker.failed", ...verdictEvents],
|
|
154
170
|
},
|
|
155
|
-
effect: {
|
|
171
|
+
effect: {
|
|
172
|
+
kind: "human_action",
|
|
173
|
+
actionType: route.actionType,
|
|
174
|
+
reason: route.reason,
|
|
175
|
+
...("runtimeAuthPark" in route && route.runtimeAuthPark
|
|
176
|
+
? { runtimeAuthPark: route.runtimeAuthPark }
|
|
177
|
+
: {}),
|
|
178
|
+
},
|
|
156
179
|
advance: "none",
|
|
157
180
|
hasVerdict,
|
|
158
181
|
};
|
|
@@ -113,7 +113,14 @@ test("infra-class retries — including a stale re-review — spend the infra bu
|
|
|
113
113
|
const reviewerRetry = projectReviewerRoute(routeReviewerOutcome({ kind: "publish_failed" }, REVIEW_ROUNDS_LEFT, PINNED_HEAD), 1, false);
|
|
114
114
|
assert.equal(reviewerRetry.advance, "infra");
|
|
115
115
|
// The infra retry re-pins the SAME head — it is not a "the head moved" stale re-review.
|
|
116
|
-
|
|
116
|
+
// retryReason is threaded so the next reviewer session (and auth-transient marker) can
|
|
117
|
+
// show why this infra attempt was spent.
|
|
118
|
+
assert.deepStrictEqual(reviewerRetry.effect, {
|
|
119
|
+
kind: "enqueue",
|
|
120
|
+
workItem: "reviewer",
|
|
121
|
+
atHeadSha: PINNED_HEAD,
|
|
122
|
+
retryReason: "Review publication to GitHub failed.",
|
|
123
|
+
});
|
|
117
124
|
// A stale re-review (the head genuinely moved) is also bounded on the infra budget —
|
|
118
125
|
// an unbounded chain of these could otherwise spawn reviewer sessions indefinitely.
|
|
119
126
|
const stale = projectReviewerRoute(routeReviewerOutcome({ kind: "stale", currentHeadSha: "new" }, REVIEW_ROUNDS_LEFT, PINNED_HEAD), 1, false);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// packages/server/src/coordinator/routing.ts
|
|
2
2
|
import { DEFAULT_MAX_CI_ATTEMPTS } from "@agent-dealer/shared";
|
|
3
3
|
import { normalizeReviewerResult } from "./reviewer-result.js";
|
|
4
|
+
import { authParkReason, MAX_CONSECUTIVE_AUTH_PARKS, repeatedAuthEscalationReason, } from "./runtime-auth-park.js";
|
|
4
5
|
/** Default N for NOT-147: escalate empty-tip timeout/crash after this many failures (one auto-retry). */
|
|
5
6
|
export const DEFAULT_NO_PROGRESS_INFRA_ATTEMPTS = 2;
|
|
6
7
|
function roundsRemain(limits) {
|
|
@@ -148,6 +149,11 @@ export function routeDeveloperOutcome(outcome, limits) {
|
|
|
148
149
|
case "timed_out": {
|
|
149
150
|
// Same infra budget as above, plus NOT-147: empty tip after N timeout/crashes → human
|
|
150
151
|
// gate so we do not burn attempts 3–4 on a branch that never moved.
|
|
152
|
+
// NOT-368: a high-confidence runtime auth failure with a still-failing (or already
|
|
153
|
+
// spent) probe parks for login remediation instead of charging infra.
|
|
154
|
+
const authPark = routeHighConfidenceAuthFailure(limits);
|
|
155
|
+
if (authPark)
|
|
156
|
+
return authPark;
|
|
151
157
|
const reason = infraFailureReason(outcome);
|
|
152
158
|
if (!infraAttemptsRemain(limits)) {
|
|
153
159
|
return {
|
|
@@ -217,6 +223,72 @@ function infraFailureReason(outcome) {
|
|
|
217
223
|
function checksFailedDetails(outcome) {
|
|
218
224
|
return outcome.details?.trim() ? outcome.details : "Developer's PR checks failed.";
|
|
219
225
|
}
|
|
226
|
+
/**
|
|
227
|
+
* NOT-368: high-confidence attributed auth failure → live-probe tie-breaker.
|
|
228
|
+
* Probe still failing (or the one transient retry already spent) → park with remediation,
|
|
229
|
+
* no infra charge. Probe passing on the first such failure → one infra retry. After
|
|
230
|
+
* {@link MAX_CONSECUTIVE_AUTH_PARKS} parks, the next confirmed failure escalates as a
|
|
231
|
+
* distinct "repeated login failures" policy_escalation (not another auth park).
|
|
232
|
+
* Medium / unattributed auth never sets `limits.authFailure`, so those stay on the
|
|
233
|
+
* generic infra-retry path below.
|
|
234
|
+
*/
|
|
235
|
+
function routeHighConfidenceAuthFailure(limits) {
|
|
236
|
+
const auth = limits.authFailure;
|
|
237
|
+
if (!auth || auth.confidence !== "high")
|
|
238
|
+
return null;
|
|
239
|
+
if (auth.consecutiveAuthParks >= MAX_CONSECUTIVE_AUTH_PARKS) {
|
|
240
|
+
return {
|
|
241
|
+
next: "human_action",
|
|
242
|
+
actionType: "policy_escalation",
|
|
243
|
+
reason: repeatedAuthEscalationReason({
|
|
244
|
+
runtime: auth.runtime,
|
|
245
|
+
remediation: auth.remediation,
|
|
246
|
+
rawCause: auth.rawCause,
|
|
247
|
+
priorParks: auth.consecutiveAuthParks,
|
|
248
|
+
}),
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
const shouldPark = auth.probeStillFailing || auth.authTransientRetrySpent;
|
|
252
|
+
if (shouldPark) {
|
|
253
|
+
const consecutivePark = auth.consecutiveAuthParks + 1;
|
|
254
|
+
return {
|
|
255
|
+
next: "human_action",
|
|
256
|
+
actionType: "policy_escalation",
|
|
257
|
+
reason: authParkReason({
|
|
258
|
+
runtime: auth.runtime,
|
|
259
|
+
remediation: auth.remediation,
|
|
260
|
+
rawCause: auth.rawCause,
|
|
261
|
+
}),
|
|
262
|
+
runtimeAuthPark: {
|
|
263
|
+
runtime: auth.runtime,
|
|
264
|
+
remediation: auth.remediation,
|
|
265
|
+
rawCause: auth.rawCause,
|
|
266
|
+
consecutivePark,
|
|
267
|
+
},
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
// Probe passed and the one transient retry is still available — charge infra like today.
|
|
271
|
+
if (!infraAttemptsRemain(limits)) {
|
|
272
|
+
return {
|
|
273
|
+
next: "human_action",
|
|
274
|
+
actionType: "policy_escalation",
|
|
275
|
+
reason: `${authParkReason({
|
|
276
|
+
runtime: auth.runtime,
|
|
277
|
+
remediation: auth.remediation,
|
|
278
|
+
rawCause: auth.rawCause,
|
|
279
|
+
})} (infra-attempt limit reached).`,
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
return {
|
|
283
|
+
next: "retry_developer",
|
|
284
|
+
reason: authParkReason({
|
|
285
|
+
runtime: auth.runtime,
|
|
286
|
+
remediation: auth.remediation,
|
|
287
|
+
rawCause: auth.rawCause,
|
|
288
|
+
}),
|
|
289
|
+
authTransientRetry: true,
|
|
290
|
+
};
|
|
291
|
+
}
|
|
220
292
|
/**
|
|
221
293
|
* NOT-147: when a timeout/crash left the branch with zero commits ahead of base, stop
|
|
222
294
|
* burning full agent spawns after N failures (default 2 = one auto-retry). Only trips when
|
|
@@ -254,6 +326,30 @@ export function routeReviewerOutcome(outcome, limits, pinnedHeadSha) {
|
|
|
254
326
|
case "session_failed":
|
|
255
327
|
case "deck_failure":
|
|
256
328
|
case "publish_failed": {
|
|
329
|
+
// NOT-368: reviewer login deaths use the same auth-park tie-breaker as developers.
|
|
330
|
+
// Only session_failed carries auth (timed_out is folded into session_failed for reviewers).
|
|
331
|
+
if (outcome.kind === "session_failed") {
|
|
332
|
+
const authPark = routeHighConfidenceAuthFailure(limits);
|
|
333
|
+
if (authPark) {
|
|
334
|
+
if (authPark.next === "human_action") {
|
|
335
|
+
return {
|
|
336
|
+
next: "human_action",
|
|
337
|
+
actionType: authPark.actionType,
|
|
338
|
+
reason: authPark.reason,
|
|
339
|
+
...("runtimeAuthPark" in authPark && authPark.runtimeAuthPark
|
|
340
|
+
? { runtimeAuthPark: authPark.runtimeAuthPark }
|
|
341
|
+
: {}),
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
// Transient auth retry → fresh reviewer at the pinned head (not a developer round).
|
|
345
|
+
return {
|
|
346
|
+
next: "retry_reviewer",
|
|
347
|
+
headSha: pinnedHeadSha,
|
|
348
|
+
reason: authPark.reason,
|
|
349
|
+
...(authPark.authTransientRetry ? { authTransientRetry: true } : {}),
|
|
350
|
+
};
|
|
351
|
+
}
|
|
352
|
+
}
|
|
257
353
|
const reason = outcome.kind === "deck_failure"
|
|
258
354
|
? `Agent Deck ${outcome.reason}`
|
|
259
355
|
: outcome.kind === "session_failed"
|
|
@@ -1,7 +1,36 @@
|
|
|
1
1
|
// packages/server/src/coordinator/routing.test.ts
|
|
2
2
|
import { test } from "node:test";
|
|
3
3
|
import assert from "node:assert/strict";
|
|
4
|
+
import { readFileSync } from "node:fs";
|
|
5
|
+
import { dirname, join } from "node:path";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
7
|
+
import { CLAUDE_AUTH_REMEDIATION, CODEX_AUTH_REMEDIATION, CURSOR_KEYCHAIN_REMEDIATION, MUSE_AUTH_REMEDIATION, runtimeAuthClassificationForLog, } from "@agent-dealer/shared";
|
|
4
8
|
import { routeDeveloperOutcome, routeReviewerOutcome, } from "./routing.js";
|
|
9
|
+
import { MAX_CONSECUTIVE_AUTH_PARKS } from "./runtime-auth-park.js";
|
|
10
|
+
const FIXTURE_DIR = join(dirname(fileURLToPath(import.meta.url)), "../../../shared/src/fixtures/runtime-auth");
|
|
11
|
+
function fixtureText(name) {
|
|
12
|
+
return readFileSync(join(FIXTURE_DIR, name), "utf8");
|
|
13
|
+
}
|
|
14
|
+
/** NOT-114 keychain has no capture file — same reconstruction used by runtime-auth-health tests. */
|
|
15
|
+
const CURSOR_KEYCHAIN_STDERR = `Cursor couldn't save your login to the macOS keychain (errSecDuplicateItem, security exit code 45).
|
|
16
|
+
The keychain item is stuck. Delete it and sign in again:
|
|
17
|
+
security delete-generic-password -s cursor-access-token -a cursor-user
|
|
18
|
+
agent login
|
|
19
|
+
`;
|
|
20
|
+
function highConfAuthFromLog(runtime, logText, opts = {}) {
|
|
21
|
+
const classified = runtimeAuthClassificationForLog(logText, runtime);
|
|
22
|
+
assert.ok(classified, `expected auth classification for ${runtime}`);
|
|
23
|
+
assert.ok(classified.runtime, `expected attributed runtime for ${runtime}`);
|
|
24
|
+
return {
|
|
25
|
+
confidence: "high",
|
|
26
|
+
runtime: classified.runtime,
|
|
27
|
+
remediation: opts.remediation ?? classified.issue.message,
|
|
28
|
+
rawCause: logText.trim().slice(0, 500),
|
|
29
|
+
probeStillFailing: opts.probeStillFailing ?? true,
|
|
30
|
+
consecutiveAuthParks: opts.consecutiveAuthParks ?? 0,
|
|
31
|
+
authTransientRetrySpent: opts.authTransientRetrySpent ?? false,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
5
34
|
const REVIEW_ROUNDS_LEFT = { currentRound: 1, maxReviewRounds: 3, infraAttempts: 0, maxInfraAttempts: 3 };
|
|
6
35
|
const REVIEW_AT_LIMIT = { currentRound: 3, maxReviewRounds: 3, infraAttempts: 0, maxInfraAttempts: 3 };
|
|
7
36
|
// NOT-313: CI-repair budget fixtures — infra counters stay at 0/3 throughout so
|
|
@@ -531,3 +560,137 @@ test("muse_cron_used escalates to the operator without spending any budget, even
|
|
|
531
560
|
});
|
|
532
561
|
}
|
|
533
562
|
});
|
|
563
|
+
// --- NOT-368: high-confidence runtime auth park (probe tie-breaker) ---
|
|
564
|
+
const AUTH_PARK_CASES = [
|
|
565
|
+
{
|
|
566
|
+
runtime: "muse_code",
|
|
567
|
+
log: fixtureText("muse-exec-missing-credentials.txt"),
|
|
568
|
+
remediation: MUSE_AUTH_REMEDIATION,
|
|
569
|
+
},
|
|
570
|
+
{
|
|
571
|
+
runtime: "codex_local",
|
|
572
|
+
log: fixtureText("codex-exec-logged-out.txt"),
|
|
573
|
+
remediation: CODEX_AUTH_REMEDIATION,
|
|
574
|
+
},
|
|
575
|
+
{
|
|
576
|
+
runtime: "claude_code",
|
|
577
|
+
log: fixtureText("claude-print-logged-out.txt"),
|
|
578
|
+
remediation: CLAUDE_AUTH_REMEDIATION,
|
|
579
|
+
},
|
|
580
|
+
{
|
|
581
|
+
runtime: "cursor_local",
|
|
582
|
+
log: CURSOR_KEYCHAIN_STDERR,
|
|
583
|
+
remediation: CURSOR_KEYCHAIN_REMEDIATION,
|
|
584
|
+
},
|
|
585
|
+
];
|
|
586
|
+
test("NOT-368: high-confidence auth + probe still failing parks with remediation; infraAttempts unchanged", () => {
|
|
587
|
+
const infraBefore = INFRA_ATTEMPTS_LEFT.infraAttempts;
|
|
588
|
+
for (const { runtime, log, remediation } of AUTH_PARK_CASES) {
|
|
589
|
+
const authFailure = highConfAuthFromLog(runtime, log, { probeStillFailing: true });
|
|
590
|
+
const outcome = { kind: "session_failed", reason: log.trim().slice(0, 200) };
|
|
591
|
+
const result = routeDeveloperOutcome(outcome, { ...INFRA_ATTEMPTS_LEFT, authFailure });
|
|
592
|
+
assert.equal(result.next, "human_action", runtime);
|
|
593
|
+
assert.equal(result.actionType, "policy_escalation", runtime);
|
|
594
|
+
const reason = result.reason;
|
|
595
|
+
assert.match(reason, new RegExp(remediation.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")), runtime);
|
|
596
|
+
assert.ok(result.runtimeAuthPark, runtime);
|
|
597
|
+
assert.equal(result.runtimeAuthPark.remediation, remediation, runtime);
|
|
598
|
+
// Routing never mutates limits — park means projection advance:none (no infra charge).
|
|
599
|
+
assert.equal(INFRA_ATTEMPTS_LEFT.infraAttempts, infraBefore, runtime);
|
|
600
|
+
}
|
|
601
|
+
});
|
|
602
|
+
test("NOT-368: high-confidence auth + probe passing takes one infra retry; second parks even when probe passes", () => {
|
|
603
|
+
const log = fixtureText("muse-exec-missing-credentials.txt");
|
|
604
|
+
const outcome = { kind: "session_failed", reason: "missing meta credentials" };
|
|
605
|
+
const first = routeDeveloperOutcome(outcome, {
|
|
606
|
+
...INFRA_ATTEMPTS_LEFT,
|
|
607
|
+
authFailure: highConfAuthFromLog("muse_code", log, {
|
|
608
|
+
probeStillFailing: false,
|
|
609
|
+
authTransientRetrySpent: false,
|
|
610
|
+
}),
|
|
611
|
+
});
|
|
612
|
+
assert.equal(first.next, "retry_developer");
|
|
613
|
+
assert.equal(first.authTransientRetry, true);
|
|
614
|
+
// Projection would increment infraAttempts by exactly 1 on retry_developer (advance:infra).
|
|
615
|
+
const second = routeDeveloperOutcome(outcome, {
|
|
616
|
+
...INFRA_ATTEMPTS_LEFT,
|
|
617
|
+
infraAttempts: INFRA_ATTEMPTS_LEFT.infraAttempts + 1,
|
|
618
|
+
authFailure: highConfAuthFromLog("muse_code", log, {
|
|
619
|
+
probeStillFailing: false,
|
|
620
|
+
authTransientRetrySpent: true,
|
|
621
|
+
}),
|
|
622
|
+
});
|
|
623
|
+
assert.equal(second.next, "human_action");
|
|
624
|
+
assert.equal(second.actionType, "policy_escalation");
|
|
625
|
+
assert.ok(second.runtimeAuthPark);
|
|
626
|
+
assert.match(second.reason, /muse login|META_API_KEY/i);
|
|
627
|
+
});
|
|
628
|
+
test("NOT-368: medium-confidence / unattributed auth keeps retry_developer and infra charge", () => {
|
|
629
|
+
// Shared "Not logged in" with no vendor anchor and no recorded runtime → medium cause;
|
|
630
|
+
// prepareAuthFailureRouting omits authFailure, so routing is unchanged from today.
|
|
631
|
+
const outcome = { kind: "session_failed", reason: "Not logged in" };
|
|
632
|
+
const result = routeDeveloperOutcome(outcome, INFRA_ATTEMPTS_LEFT);
|
|
633
|
+
assert.deepStrictEqual(result, {
|
|
634
|
+
next: "retry_developer",
|
|
635
|
+
reason: "Not logged in",
|
|
636
|
+
});
|
|
637
|
+
});
|
|
638
|
+
test("NOT-368: 4th consecutive confirmed auth park escalates as repeated login failures", () => {
|
|
639
|
+
const log = fixtureText("muse-exec-missing-credentials.txt");
|
|
640
|
+
const outcome = { kind: "session_failed", reason: "missing meta credentials" };
|
|
641
|
+
const result = routeDeveloperOutcome(outcome, {
|
|
642
|
+
...INFRA_ATTEMPTS_LEFT,
|
|
643
|
+
authFailure: highConfAuthFromLog("muse_code", log, {
|
|
644
|
+
probeStillFailing: true,
|
|
645
|
+
consecutiveAuthParks: MAX_CONSECUTIVE_AUTH_PARKS,
|
|
646
|
+
}),
|
|
647
|
+
});
|
|
648
|
+
assert.equal(result.next, "human_action");
|
|
649
|
+
assert.equal(result.actionType, "policy_escalation");
|
|
650
|
+
assert.match(result.reason, /Repeated .* login failures/i);
|
|
651
|
+
assert.equal(result.runtimeAuthPark, undefined);
|
|
652
|
+
});
|
|
653
|
+
test("NOT-368: reviewer high-confidence auth + probe still failing parks with remediation (no infra retry)", () => {
|
|
654
|
+
for (const { runtime, log, remediation } of AUTH_PARK_CASES) {
|
|
655
|
+
const authFailure = highConfAuthFromLog(runtime, log, { probeStillFailing: true });
|
|
656
|
+
const outcome = { kind: "session_failed", reason: log.trim().slice(0, 200) };
|
|
657
|
+
const result = routeReviewerOutcome(outcome, { ...INFRA_ATTEMPTS_LEFT, authFailure }, PINNED_HEAD);
|
|
658
|
+
assert.equal(result.next, "human_action", runtime);
|
|
659
|
+
assert.equal(result.actionType, "policy_escalation", runtime);
|
|
660
|
+
assert.match(result.reason, new RegExp(remediation.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")), runtime);
|
|
661
|
+
assert.ok(result.runtimeAuthPark, runtime);
|
|
662
|
+
}
|
|
663
|
+
});
|
|
664
|
+
test("NOT-368: reviewer high-confidence auth + probe passing takes one infra retry_reviewer; second parks", () => {
|
|
665
|
+
const log = fixtureText("muse-exec-missing-credentials.txt");
|
|
666
|
+
const outcome = { kind: "session_failed", reason: "missing meta credentials" };
|
|
667
|
+
const first = routeReviewerOutcome(outcome, {
|
|
668
|
+
...INFRA_ATTEMPTS_LEFT,
|
|
669
|
+
authFailure: highConfAuthFromLog("muse_code", log, {
|
|
670
|
+
probeStillFailing: false,
|
|
671
|
+
authTransientRetrySpent: false,
|
|
672
|
+
}),
|
|
673
|
+
}, PINNED_HEAD);
|
|
674
|
+
assert.equal(first.next, "retry_reviewer");
|
|
675
|
+
assert.equal(first.headSha, PINNED_HEAD);
|
|
676
|
+
assert.equal(first.authTransientRetry, true);
|
|
677
|
+
const second = routeReviewerOutcome(outcome, {
|
|
678
|
+
...INFRA_ATTEMPTS_LEFT,
|
|
679
|
+
infraAttempts: INFRA_ATTEMPTS_LEFT.infraAttempts + 1,
|
|
680
|
+
authFailure: highConfAuthFromLog("muse_code", log, {
|
|
681
|
+
probeStillFailing: false,
|
|
682
|
+
authTransientRetrySpent: true,
|
|
683
|
+
}),
|
|
684
|
+
}, PINNED_HEAD);
|
|
685
|
+
assert.equal(second.next, "human_action");
|
|
686
|
+
assert.ok(second.runtimeAuthPark);
|
|
687
|
+
});
|
|
688
|
+
test("NOT-368: reviewer medium-confidence / unattributed auth keeps retry_reviewer", () => {
|
|
689
|
+
const outcome = { kind: "session_failed", reason: "Not logged in" };
|
|
690
|
+
const result = routeReviewerOutcome(outcome, INFRA_ATTEMPTS_LEFT, PINNED_HEAD);
|
|
691
|
+
assert.deepStrictEqual(result, {
|
|
692
|
+
next: "retry_reviewer",
|
|
693
|
+
headSha: PINNED_HEAD,
|
|
694
|
+
reason: "Not logged in",
|
|
695
|
+
});
|
|
696
|
+
});
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { CLAUDE_AUTH_REMEDIATION, CODEX_AUTH_REMEDIATION, CURSOR_AUTH_REMEDIATION, CURSOR_KEYCHAIN_REMEDIATION, MUSE_AUTH_REMEDIATION, RUNTIME_AUTH_LABEL, } from "@agent-dealer/shared";
|
|
2
|
+
/** Cap on consecutive confirmed auth parks per issue before a distinct escalation. */
|
|
3
|
+
export const MAX_CONSECUTIVE_AUTH_PARKS = 3;
|
|
4
|
+
/** Evidence key on a policy_escalation raised for a confirmed runtime login failure. */
|
|
5
|
+
export const RUNTIME_AUTH_PARK_EVIDENCE_KEY = "runtimeAuthPark";
|
|
6
|
+
/** Work-item payload flag: this developer/reviewer retry spent the one transient auth retry. */
|
|
7
|
+
export const AUTH_TRANSIENT_RETRY_PAYLOAD_KEY = "authTransientRetry";
|
|
8
|
+
const REMEDIATION_BY_RUNTIME = {
|
|
9
|
+
cursor_local: CURSOR_AUTH_REMEDIATION,
|
|
10
|
+
codex_local: CODEX_AUTH_REMEDIATION,
|
|
11
|
+
claude_code: CLAUDE_AUTH_REMEDIATION,
|
|
12
|
+
muse_code: MUSE_AUTH_REMEDIATION,
|
|
13
|
+
};
|
|
14
|
+
/** True when the live auth probe still says the runtime cannot authenticate. */
|
|
15
|
+
export function authProbeConfirmsFailure(issues) {
|
|
16
|
+
return issues.some((i) => i.code === "runtime_auth" || i.code === "cursor_keychain");
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Prefer the probe's own remediation text when present; else the classified one.
|
|
20
|
+
*
|
|
21
|
+
* Exception (NOT-368 repair): a classified / parked Cursor keychain remediation must
|
|
22
|
+
* not be replaced by ordinary `runtime_auth` probe copy. `cursor-agent status` often
|
|
23
|
+
* reports a plain login failure while the stuck keychain still blocks `agent login` —
|
|
24
|
+
* swapping in generic login instructions tells the operator to do exactly what remains
|
|
25
|
+
* impossible. Only a probe that itself reports `cursor_keychain` may update that text.
|
|
26
|
+
*/
|
|
27
|
+
export function remediationFromProbe(issues, fallback) {
|
|
28
|
+
const keychainHit = issues.find((i) => i.code === "cursor_keychain");
|
|
29
|
+
if (keychainHit?.message?.trim())
|
|
30
|
+
return keychainHit.message;
|
|
31
|
+
if (fallback.trim() === CURSOR_KEYCHAIN_REMEDIATION) {
|
|
32
|
+
return fallback;
|
|
33
|
+
}
|
|
34
|
+
const authHit = issues.find((i) => i.code === "runtime_auth");
|
|
35
|
+
return authHit?.message?.trim() ? authHit.message : fallback;
|
|
36
|
+
}
|
|
37
|
+
export function defaultRemediationForRuntime(runtime, keychain = false) {
|
|
38
|
+
if (runtime === "cursor_local" && keychain)
|
|
39
|
+
return CURSOR_KEYCHAIN_REMEDIATION;
|
|
40
|
+
return REMEDIATION_BY_RUNTIME[runtime];
|
|
41
|
+
}
|
|
42
|
+
export function parseRuntimeAuthParkEvidence(evidenceJson) {
|
|
43
|
+
if (!evidenceJson)
|
|
44
|
+
return null;
|
|
45
|
+
try {
|
|
46
|
+
const parsed = JSON.parse(evidenceJson);
|
|
47
|
+
const raw = parsed[RUNTIME_AUTH_PARK_EVIDENCE_KEY];
|
|
48
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
49
|
+
return null;
|
|
50
|
+
const ev = raw;
|
|
51
|
+
if ((ev.runtime !== "cursor_local" &&
|
|
52
|
+
ev.runtime !== "codex_local" &&
|
|
53
|
+
ev.runtime !== "claude_code" &&
|
|
54
|
+
ev.runtime !== "muse_code") ||
|
|
55
|
+
typeof ev.remediation !== "string" ||
|
|
56
|
+
typeof ev.rawCause !== "string" ||
|
|
57
|
+
typeof ev.consecutivePark !== "number") {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
return {
|
|
61
|
+
runtime: ev.runtime,
|
|
62
|
+
remediation: ev.remediation,
|
|
63
|
+
rawCause: ev.rawCause,
|
|
64
|
+
consecutivePark: ev.consecutivePark,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
export function isRuntimeAuthParkAction(action) {
|
|
72
|
+
return parseRuntimeAuthParkEvidence(action.evidenceJson) != null;
|
|
73
|
+
}
|
|
74
|
+
export function gateRuntimeAuthParkResume(opts) {
|
|
75
|
+
if (!opts.probeStillFailing)
|
|
76
|
+
return { proceed: true };
|
|
77
|
+
const remediation = opts.probeRemediation?.trim() || opts.evidence.remediation;
|
|
78
|
+
const label = RUNTIME_AUTH_LABEL[opts.evidence.runtime];
|
|
79
|
+
return {
|
|
80
|
+
proceed: false,
|
|
81
|
+
remediation,
|
|
82
|
+
message: `${label} is still not authenticated — ${remediation}`,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
/** Operator-facing reason for a confirmed auth park (parks 1–3). */
|
|
86
|
+
export function authParkReason(opts) {
|
|
87
|
+
const label = RUNTIME_AUTH_LABEL[opts.runtime];
|
|
88
|
+
return (`${label} is not logged in — ${opts.remediation}` +
|
|
89
|
+
(opts.rawCause.trim() ? `\n\nCause: ${opts.rawCause.trim()}` : ""));
|
|
90
|
+
}
|
|
91
|
+
/** Distinct escalation after MAX consecutive auth parks (the 4th confirmed failure). */
|
|
92
|
+
export function repeatedAuthEscalationReason(opts) {
|
|
93
|
+
const label = RUNTIME_AUTH_LABEL[opts.runtime];
|
|
94
|
+
return (`Repeated ${label} login failures after ${opts.priorParks} parks — ` +
|
|
95
|
+
`fix authentication before resuming. ${opts.remediation}` +
|
|
96
|
+
(opts.rawCause.trim() ? `\n\nCause: ${opts.rawCause.trim()}` : ""));
|
|
97
|
+
}
|
|
@@ -24,6 +24,7 @@ import { observeClockJump } from "./clock-jump.js";
|
|
|
24
24
|
import { admitNext, checkRoleAgentHealthy } from "./admission.js";
|
|
25
25
|
import { workerSessionPayload } from "./session-progress.js";
|
|
26
26
|
import { runtimeAvailability } from "../repository/runtime-availability.js";
|
|
27
|
+
import { runHeldWork } from "../power/host-awake-lifecycle.js";
|
|
27
28
|
import { deferLeasedWorkItemForUsageCap, deferLeasedWorkItemForAgentUnhealthy, } from "./usage-cap-defer.js";
|
|
28
29
|
import { outcomeShouldRecordError, reasonForWorkerFailedEvent } from "./failure-reason.js";
|
|
29
30
|
import { checkAgentDeckHealth } from "../adapters/agent-deck.js";
|
|
@@ -123,10 +124,24 @@ function handleEffectFailure(itemId, leaseToken, error) {
|
|
|
123
124
|
}
|
|
124
125
|
})();
|
|
125
126
|
}
|
|
127
|
+
/** Test seam: replace the body inside the host-awake hold (lifecycle tests). */
|
|
128
|
+
let processWorkItemHeldForTests = null;
|
|
129
|
+
export function setProcessWorkItemHeldForTests(fn) {
|
|
130
|
+
processWorkItemHeldForTests = fn;
|
|
131
|
+
}
|
|
126
132
|
async function processWorkItem(claimed) {
|
|
127
133
|
const leaseToken = claimed.leaseToken;
|
|
128
134
|
if (!leaseToken)
|
|
129
135
|
return; // not actually leased — defensive
|
|
136
|
+
// NOT-369: hold idle-sleep for the whole leased attempt (session + publish).
|
|
137
|
+
// runHeldWork releases on every outcome — clean exit, crash, timeout, kill.
|
|
138
|
+
await runHeldWork(() => (processWorkItemHeldForTests ?? processWorkItemHeld)(claimed, leaseToken));
|
|
139
|
+
}
|
|
140
|
+
/** Drive the real processWorkItem path from lifecycle tests (fake held body). */
|
|
141
|
+
export async function processWorkItemForTests(claimed) {
|
|
142
|
+
await processWorkItem(claimed);
|
|
143
|
+
}
|
|
144
|
+
async function processWorkItemHeld(claimed, leaseToken) {
|
|
130
145
|
const issue = getIssue(claimed.issueId);
|
|
131
146
|
const instance = getWorkflowInstance(claimed.workflowInstanceId);
|
|
132
147
|
if (!issue || !instance) {
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// NOT-369: product docs describe the caffeinate hold, pmset advice, and limits.
|
|
2
|
+
import { test } from "node:test";
|
|
3
|
+
import assert from "node:assert/strict";
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
7
|
+
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
|
|
8
|
+
const doc = fs.readFileSync(path.join(repoRoot, "docs/TROUBLESHOOTING.md"), "utf8");
|
|
9
|
+
test("TROUBLESHOOTING documents caffeinate hold and pmset advice", () => {
|
|
10
|
+
assert.match(doc, /caffeinate/);
|
|
11
|
+
assert.match(doc, /pmset/);
|
|
12
|
+
assert.match(doc, /sudo pmset -c sleep 0/);
|
|
13
|
+
assert.match(doc, /Closing the lid/);
|
|
14
|
+
assert.match(doc, /Logging out/);
|
|
15
|
+
assert.match(doc, /NOT-369/);
|
|
16
|
+
});
|
|
@@ -19,6 +19,8 @@ import { runDeveloperEffect } from "./coordinator/developer-effect.js";
|
|
|
19
19
|
import { runReviewerEffect } from "./coordinator/reviewer-effect.js";
|
|
20
20
|
import { registerStaticUi } from "./static-ui.js";
|
|
21
21
|
import { cleanupOrphanedWorkerMcpConfig } from "./paths.js";
|
|
22
|
+
import { registerHostAwakeShutdownCleanup } from "./power/host-awake-lifecycle.js";
|
|
23
|
+
import { ensureSleepTimerCheckedAtStartup } from "./power/sleep-timer.js";
|
|
22
24
|
import net from "node:net";
|
|
23
25
|
const { mode, envFile } = loadAgentDealerEnv();
|
|
24
26
|
console.log(formatEnvStartupLine(mode, envFile));
|
|
@@ -59,6 +61,10 @@ async function main() {
|
|
|
59
61
|
// Best-effort: shutdown must never block process exit.
|
|
60
62
|
}
|
|
61
63
|
};
|
|
64
|
+
// NOT-369: drop any idle-sleep assertion so a clean exit does not leave caffeinate behind.
|
|
65
|
+
// (caffeinate -w also dies if this process dies uncleanly.)
|
|
66
|
+
// Registered via the lifecycle seam so tests can emit SIGINT/SIGTERM/exit on a fake process.
|
|
67
|
+
registerHostAwakeShutdownCleanup(process);
|
|
62
68
|
process.on("SIGINT", () => {
|
|
63
69
|
removeServerPidFile();
|
|
64
70
|
void shutdownCapacityHosts().finally(() => process.exit(0));
|
|
@@ -67,9 +73,16 @@ async function main() {
|
|
|
67
73
|
removeServerPidFile();
|
|
68
74
|
void shutdownCapacityHosts().finally(() => process.exit(0));
|
|
69
75
|
});
|
|
70
|
-
process.on("exit",
|
|
76
|
+
process.on("exit", () => {
|
|
77
|
+
removeServerPidFile();
|
|
78
|
+
});
|
|
71
79
|
migrate();
|
|
72
80
|
cleanupOrphanedWorkerMcpConfig();
|
|
81
|
+
// NOT-369: one-time AC sleep-timer advice (never changes settings; never sudo).
|
|
82
|
+
const sleepNotice = ensureSleepTimerCheckedAtStartup();
|
|
83
|
+
if (sleepNotice) {
|
|
84
|
+
console.warn(`[startup] ${sleepNotice.message}`);
|
|
85
|
+
}
|
|
73
86
|
const app = Fastify({ logger: true });
|
|
74
87
|
await app.register(cors, { origin: true });
|
|
75
88
|
await registerRoutes(app);
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// NOT-369: production lifecycle seam for host-awake holds.
|
|
2
|
+
//
|
|
3
|
+
// Worker-loop / auto-merge call `runHeldWork`. Server startup calls
|
|
4
|
+
// `registerHostAwakeShutdownCleanup`. Lifecycle tests drive those same
|
|
5
|
+
// entry points (via processWorkItem / finalizeAutoMerge / a fake process)
|
|
6
|
+
// so a removed finally or an unwired shutdown handler fails the suite.
|
|
7
|
+
import { releaseAllHostAwake, withHostAwakeHold } from "./host-awake.js";
|
|
8
|
+
/** Acquire → work → release. Used by worker-loop and auto-merge. */
|
|
9
|
+
export async function runHeldWork(work) {
|
|
10
|
+
return withHostAwakeHold(work);
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Register releaseAll on SIGINT, SIGTERM, and exit — the same events index.ts
|
|
14
|
+
* must cover so a clean server stop does not leave caffeinate behind.
|
|
15
|
+
* Returns the cleanup function (idempotent; safe to invoke more than once).
|
|
16
|
+
*/
|
|
17
|
+
export function registerHostAwakeShutdownCleanup(target = process) {
|
|
18
|
+
const cleanup = () => {
|
|
19
|
+
try {
|
|
20
|
+
releaseAllHostAwake();
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
// Best-effort — never block process exit.
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
target.on("SIGINT", cleanup);
|
|
27
|
+
target.on("SIGTERM", cleanup);
|
|
28
|
+
target.on("exit", cleanup);
|
|
29
|
+
return cleanup;
|
|
30
|
+
}
|