@bongos/core 1.19.650 → 1.19.652
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +35 -25
- package/clients/bongos-client/index.d.ts +1 -1
- package/docs/adr/0277-a-box-is-in-use-only-while-a-human-is-attached.md +53 -0
- package/docs/adr/README.md +1 -0
- package/docs/api/openapi.json +4 -1
- package/docs/api-reference.md +1 -1
- package/docs/module-api-changelog.md +4 -0
- package/migrations/core_238_box_unattended_cap.sql +57 -0
- package/modules/dev-box/boxes.js +112 -8
- package/modules/dev-box/routes/box.js +14 -6
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/scripts/gds/box-infra.js +8 -0
- package/scripts/gds/box.js +35 -9
- package/scripts/gds/ship-land.js +34 -1
- package/scripts/gds/ship-merge.js +76 -10
- package/scripts/gds/ship.js +13 -1
- package/src/module-api.js +1 -1
- package/tests/boxes.mjs +160 -3
- package/tests/ship_ci_deploy.mjs +106 -1
package/scripts/gds/box-infra.js
CHANGED
|
@@ -62,6 +62,14 @@ const CONFIG = {
|
|
|
62
62
|
size: process.env.BOX_SIZE || 's-2vcpu-4gb',
|
|
63
63
|
baseImage: process.env.BOX_BASE_IMAGE || 'ubuntu-24-04-x64',
|
|
64
64
|
idleMinutes: numEnv('BOX_IDLE_MINUTES', 10),
|
|
65
|
+
// task 1003507 — the longest a box may stay active without evidence that a
|
|
66
|
+
// HUMAN was attached to it. One number bounds both proxies that used to keep a
|
|
67
|
+
// box alive on their own (a running `claude` process; load above the floor), so
|
|
68
|
+
// an abandoned session can no longer pin a droplet forever. 12h is deliberately
|
|
69
|
+
// generous — longer than any interactive session, long enough for a real
|
|
70
|
+
// overnight autonomous run — because being wrong costs one wake, while having
|
|
71
|
+
// no cap at all cost $20.21 on a single box.
|
|
72
|
+
unattendedMaxHours: numEnv('BOX_UNATTENDED_MAX_HOURS', 12),
|
|
65
73
|
dormantDays: numEnv('BOX_DORMANT_DAYS', 14),
|
|
66
74
|
// task 1002727 — how many days BEFORE the reclaim a builder gets warned.
|
|
67
75
|
dormantWarnLeadDays: numEnv('BOX_DORMANT_WARN_LEAD_DAYS', 3),
|
package/scripts/gds/box.js
CHANGED
|
@@ -28,7 +28,8 @@
|
|
|
28
28
|
//
|
|
29
29
|
// Config (env): DO_API_TOKEN (live only), BOX_PROVISION_MIN_RANK (default xenos),
|
|
30
30
|
// BOX_REGION (nyc3), BOX_SIZE (s-2vcpu-4gb), BOX_BASE_IMAGE (ubuntu-24-04-x64),
|
|
31
|
-
// BOX_IDLE_MINUTES (10),
|
|
31
|
+
// BOX_IDLE_MINUTES (10), BOX_UNATTENDED_MAX_HOURS (12), BOX_DORMANT_DAYS (14),
|
|
32
|
+
// BOX_SSH_KEY_IDS (comma list),
|
|
32
33
|
// BOX_DNS_HOOK (optional command, gets BOX_HOSTNAME/BOX_IP in env), BOX_TAG.
|
|
33
34
|
|
|
34
35
|
const fs = require('node:fs');
|
|
@@ -109,6 +110,21 @@ async function preflightCookieStripScope() {
|
|
|
109
110
|
return scope;
|
|
110
111
|
}
|
|
111
112
|
|
|
113
|
+
// Lazily-built handles to the heavy dependencies (DB pool, DO/CF clients), so a
|
|
114
|
+
// dry-run or a --help never opens a pool or requires a network client.
|
|
115
|
+
//
|
|
116
|
+
// task 1003507 — this was UNDECLARED. `_deps = {...}` further down is an
|
|
117
|
+
// assignment, which in sloppy mode would have created a global, but
|
|
118
|
+
// `if (_deps)` READS it first, and reading an undeclared identifier is a
|
|
119
|
+
// ReferenceError. So loadDeps() threw on its first call, every time, and every
|
|
120
|
+
// command that reaches makeDo()/makeCf() went with it — including
|
|
121
|
+
// `sweep-idle --apply`, which resolves its DO client before the park loop.
|
|
122
|
+
// The idle sweep therefore could not park ANY box, which is a second and
|
|
123
|
+
// entirely separate reason the abandoned box in this task was never reaped.
|
|
124
|
+
// No test caught it because the only apply:true test relied on the
|
|
125
|
+
// claude_active veto emptying the idle set before makeDo() was reached.
|
|
126
|
+
let _deps = null;
|
|
127
|
+
|
|
112
128
|
function loadDeps() {
|
|
113
129
|
if (_deps) return _deps;
|
|
114
130
|
// eslint-disable-next-line global-require
|
|
@@ -559,6 +575,10 @@ async function parkOne(pool, boxes, doApi, box, { actor }) {
|
|
|
559
575
|
await boxes.setBoxState(pool, box.builder_id, 'parked', {
|
|
560
576
|
snapshot_id: snapshotId, parked_at: new Date(now),
|
|
561
577
|
droplet_id: null, ip: null, active_since: null,
|
|
578
|
+
// task 1003507 — the droplet is gone, so there is no `claude` process to
|
|
579
|
+
// report. Clearing the flag AND its clock here means a wake starts a fresh
|
|
580
|
+
// session rather than inheriting an expired latch and being reaped at once.
|
|
581
|
+
claude_active: false, claude_active_since: null,
|
|
562
582
|
});
|
|
563
583
|
await boxes.recordEvent(pool, {
|
|
564
584
|
boxId: box.id, builderId: box.builder_id, event: 'park',
|
|
@@ -610,6 +630,10 @@ async function cmdWake(pool, boxes, ref, { apply }) {
|
|
|
610
630
|
const now = new Date();
|
|
611
631
|
await boxes.setBoxState(pool, builder.id, 'active', {
|
|
612
632
|
droplet_id: String(droplet.id), ip, active_since: now, last_activity_at: now, parked_at: null,
|
|
633
|
+
// task 1003507 — a woken box has reported nothing yet. Start the latch and
|
|
634
|
+
// its clock clean so the first real heartbeat sets both, rather than the
|
|
635
|
+
// box re-entering the sweep carrying a stale claim from its last session.
|
|
636
|
+
claude_active: false, claude_active_since: null,
|
|
613
637
|
});
|
|
614
638
|
await boxes.recordEvent(pool, { boxId: box.id, builderId: builder.id, event: 'wake', detail: `droplet ${droplet.id} ${ip || ''}`, actor: actorTag() });
|
|
615
639
|
// #602: the box just ended a PARKED interval (snapshot retained) — accrue its
|
|
@@ -724,7 +748,7 @@ async function deprovisionOne(pool, boxes, doApi, box, { actor, login, sleepFn }
|
|
|
724
748
|
if (CONFIG.dnsHook && dnsHost) runDnsHookBlackhole(dnsHost);
|
|
725
749
|
await boxes.setBoxState(pool, box.builder_id, 'destroyed', {
|
|
726
750
|
destroyed_at: new Date(now), droplet_id: null, ip: null, snapshot_id: null,
|
|
727
|
-
active_since: null, parked_at: null, claude_active: false,
|
|
751
|
+
active_since: null, parked_at: null, claude_active: false, claude_active_since: null,
|
|
728
752
|
// idea 310: a rebuilt box mints fresh SSH host keys, but the GDS row kept the
|
|
729
753
|
// OLD keys until the new box re-reported (~10 min). In that window GET /box/me
|
|
730
754
|
// served a stale identity and the desktop app pinned the WRONG key — the
|
|
@@ -811,18 +835,20 @@ async function cmdDeprovision(pool, boxes, ref, { apply }) {
|
|
|
811
835
|
|
|
812
836
|
async function cmdSweepIdle(pool, boxes, { apply }) {
|
|
813
837
|
const rows = await boxes.listBoxes(pool, { state: 'active' });
|
|
814
|
-
const
|
|
815
|
-
|
|
838
|
+
const sweepNow = Date.now();
|
|
839
|
+
const sel = { idleMinutes: CONFIG.idleMinutes, nowMs: sweepNow, unattendedMaxHours: CONFIG.unattendedMaxHours };
|
|
840
|
+
const idle = boxes.selectIdleBoxes(rows, sel);
|
|
841
|
+
console.log(`idle sweep (threshold=${CONFIG.idleMinutes}m, unattended cap=${CONFIG.unattendedMaxHours}h${apply ? '' : ', dry-run'}): ${idle.length} of ${rows.length} active box(es) idle`);
|
|
816
842
|
if (idle.length === 0) { console.log(' nothing idle — exit clean'); return { parked: 0 }; }
|
|
817
843
|
const doApi = apply ? makeDo(apply) : null;
|
|
818
844
|
let parked = 0, errors = 0;
|
|
819
845
|
for (const box of idle) {
|
|
820
|
-
const
|
|
821
|
-
if (!apply) { console.log(` would park @${box.github_login}'s box (
|
|
846
|
+
const why = boxes.idleReason(box, sel);
|
|
847
|
+
if (!apply) { console.log(` would park @${box.github_login}'s box (${why}, droplet ${box.droplet_id})`); continue; }
|
|
822
848
|
try {
|
|
823
849
|
await parkOne(pool, boxes, doApi, box, { actor: 'sweep:idle' });
|
|
824
850
|
parked++;
|
|
825
|
-
console.log(` ✓ parked @${box.github_login} (
|
|
851
|
+
console.log(` ✓ parked @${box.github_login} (${why} — snapshot kept, compute stopped)`);
|
|
826
852
|
} catch (e) {
|
|
827
853
|
// A park that fails must NOT fall through to a destroy — leaving the box
|
|
828
854
|
// at 'error' with its droplet intact is strictly safer than reclaiming it,
|
|
@@ -1058,7 +1084,7 @@ async function cmdReconcileDrift(pool, boxes, { apply }) {
|
|
|
1058
1084
|
await teardownEdgeForLogin(g.login, { apply });
|
|
1059
1085
|
await boxes.setBoxState(pool, g.builderId, 'destroyed', {
|
|
1060
1086
|
destroyed_at: new Date(nowMs), droplet_id: null, ip: null, snapshot_id: null,
|
|
1061
|
-
active_since: null, parked_at: null, claude_active: false,
|
|
1087
|
+
active_since: null, parked_at: null, claude_active: false, claude_active_since: null,
|
|
1062
1088
|
host_keys: null, host_keys_at: null,
|
|
1063
1089
|
});
|
|
1064
1090
|
if (row) {
|
|
@@ -1198,7 +1224,7 @@ Commands:
|
|
|
1198
1224
|
deprovision <builder> destroy droplet AND snapshot (full reclaim)
|
|
1199
1225
|
run-intents drain the auto-provision/wake-on-connect queue (#701):
|
|
1200
1226
|
execute each pending box_intents row (provision/wake)
|
|
1201
|
-
sweep-idle park active boxes idle > ${CONFIG.idleMinutes}m
|
|
1227
|
+
sweep-idle park active boxes idle > ${CONFIG.idleMinutes}m, or unattended > ${CONFIG.unattendedMaxHours}h
|
|
1202
1228
|
sweep-dormant warn at ${Math.max(CONFIG.dormantDays - CONFIG.dormantWarnLeadDays, 0)}d, then deprovision boxes parked > ${CONFIG.dormantDays}d
|
|
1203
1229
|
(never reclaims a box that was not warned first)
|
|
1204
1230
|
reconcile source-access clawback: deprovision below-floor/inactive
|
package/scripts/gds/ship-land.js
CHANGED
|
@@ -35,9 +35,37 @@ const { normalizeApiError } = require('./ship-io.js');
|
|
|
35
35
|
// Only landBailed makes the card say NOT landed. Purely additive: no bail site
|
|
36
36
|
// changes what it DOES, only what it now reports back.
|
|
37
37
|
// ADR 0184: an EMPTY ship has no commit to prove, so it must DECLARE itself.
|
|
38
|
+
// task 1002572 (PART 2): a bail also carries an optional `bailId` — the stable
|
|
39
|
+
// name of WHICH refusal fired, so a caller can route the recoverable ones to the
|
|
40
|
+
// push+PR path instead of stranding. `null` for every un-tagged bail, which keeps
|
|
41
|
+
// them terminal (fail-safe: an untagged bail never gains a silent fallback).
|
|
38
42
|
function landed(o = {}) { return { ok: true, pending: false, reason: null, nextStep: null, noArtifact: o.noArtifact === true, noArtifactReason: o.noArtifactReason || null }; }
|
|
39
43
|
function landPending() { return { ok: false, pending: true, reason: null, nextStep: null }; }
|
|
40
|
-
function landBailed(reason, nextStep = null) { return { ok: false, pending: false, reason, nextStep }; }
|
|
44
|
+
function landBailed(reason, nextStep = null, bailId = null) { return { ok: false, pending: false, reason, nextStep, bailId }; }
|
|
45
|
+
|
|
46
|
+
// ---------- which bails a PR can still land (task 1002572, part 2) ----------
|
|
47
|
+
//
|
|
48
|
+
// A builder in a SINGLE main checkout can never complete a local merge: the tree
|
|
49
|
+
// that would RECEIVE the merge is the very tree holding the feature branch. That
|
|
50
|
+
// is a property of the machine, not of the work — the branch itself is perfectly
|
|
51
|
+
// landable, just not from here. So that one bail must fall back to the push+PR
|
|
52
|
+
// path (ciLand) rather than parking the task at 'confirmed' unpaid (ADR 0120).
|
|
53
|
+
//
|
|
54
|
+
// Everything else stays LOUD. A merge conflict, a red post-merge smoke, a failed
|
|
55
|
+
// push, an unreachable origin: those are real failures, and a PR would carry the
|
|
56
|
+
// same breakage to main. Membership is opt-in by id for exactly that reason —
|
|
57
|
+
// blanket-converting bails into PRs is the way this fix goes wrong.
|
|
58
|
+
const LAND_BAIL_MAIN_CHECKOUT_ON_BRANCH = 'main-checkout-on-branch';
|
|
59
|
+
const LAND_BAIL_NO_TASK_BRANCH = 'no-task-branch';
|
|
60
|
+
const PR_RECOVERABLE_BAILS = new Set([LAND_BAIL_MAIN_CHECKOUT_ON_BRANCH]);
|
|
61
|
+
|
|
62
|
+
// landBailRecoverableByPr — PURE. Can this bail still be landed by opening a PR?
|
|
63
|
+
// `no-task-branch` is deliberately NOT in the set: standing on main there is no
|
|
64
|
+
// head branch to open a PR FROM, so the PR path has nothing to offer it (its cure
|
|
65
|
+
// is to go back to the task branch and re-ship, which now works).
|
|
66
|
+
function landBailRecoverableByPr(bailId) {
|
|
67
|
+
return PR_RECOVERABLE_BAILS.has(bailId);
|
|
68
|
+
}
|
|
41
69
|
|
|
42
70
|
// The short COMMAND form of strandNextStep (which is prose, sized for the log).
|
|
43
71
|
// The card has room for one command, not a sentence.
|
|
@@ -639,6 +667,11 @@ module.exports = {
|
|
|
639
667
|
ciLand,
|
|
640
668
|
classifyLandOutcome,
|
|
641
669
|
formatPublishFailure,
|
|
670
|
+
// task 1002572 (PART 2): the bail ids + the pure "can a PR still land this?"
|
|
671
|
+
// decision, so the single-checkout fallback is opt-in per bail and unit-tested.
|
|
672
|
+
LAND_BAIL_MAIN_CHECKOUT_ON_BRANCH,
|
|
673
|
+
LAND_BAIL_NO_TASK_BRANCH,
|
|
674
|
+
landBailRecoverableByPr,
|
|
642
675
|
landBailed,
|
|
643
676
|
landPending,
|
|
644
677
|
landPendingLines,
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
// scripts/gds/ship-merge.js — autoMerge — the merge/regen/push/land chain that puts the branch on main.
|
|
2
2
|
//
|
|
3
3
|
// Carved out of scripts/gds/ship.js by task 1003205 (goal 1000079, the R03
|
|
4
|
-
// de-monolith plan) — a STRUCTURAL move, not a rewrite
|
|
5
|
-
//
|
|
6
|
-
// re-exports every name and owns the CLI entry point, so the ship contract and
|
|
4
|
+
// de-monolith plan) — a STRUCTURAL move, not a rewrite. ship.js remains the facade
|
|
5
|
+
// that re-exports every name and owns the CLI entry point, so the ship contract and
|
|
7
6
|
// every flag are untouched, and the other scripts that import from it are too.
|
|
7
|
+
//
|
|
8
|
+
// task 1002572 (PART 2) split the entry point in two: `mergeLocally` is the original
|
|
9
|
+
// merge/regen/push/deploy chain, and `autoMerge` is a thin wrapper that routes the one
|
|
10
|
+
// bail a PR can still land — a single checkout whose main worktree holds the feature
|
|
11
|
+
// branch — into the push+PR path instead of stranding the task at 'confirmed'.
|
|
8
12
|
|
|
9
13
|
'use strict';
|
|
10
14
|
|
|
@@ -13,7 +17,14 @@ const { cleanupMacFinderDupRefs } = require('../../modules/lifecycle/ship-prefli
|
|
|
13
17
|
const mergeLock = require('../../modules/lifecycle/merge-lock');
|
|
14
18
|
const { buildMergeArgv, gitArgvOk, gitOk, shellOk } = require('./ship-git.js');
|
|
15
19
|
const { mainIsProtected, pushVia, readDeployConfigRaw, resolveDeployMode, resolveDeployTarget, safeBranding, shouldBailOnFetchFailure } = require('./ship-deploy-target.js');
|
|
16
|
-
const {
|
|
20
|
+
const {
|
|
21
|
+
LAND_BAIL_MAIN_CHECKOUT_ON_BRANCH,
|
|
22
|
+
LAND_BAIL_NO_TASK_BRANCH,
|
|
23
|
+
ciLand,
|
|
24
|
+
landBailRecoverableByPr,
|
|
25
|
+
landBailed,
|
|
26
|
+
landed,
|
|
27
|
+
} = require('./ship-land.js');
|
|
17
28
|
const { runSmoke } = require('./ship-grade.js');
|
|
18
29
|
const { currentBranch, mainWorktreeBusy, regenerateApiClient, regenerateApiDocs, regenerateDiagrams, regenerateFileMap, regenerateRepoMap, regenerateSessionIndex } = require('./ship-regen.js');
|
|
19
30
|
const { REPO_ROOT } = require('./ship-state.js');
|
|
@@ -23,7 +34,49 @@ function zeroCommitVerdict(commitsAhead, dbOnly) {
|
|
|
23
34
|
return dbOnly ? 'db-only' : 'refuse';
|
|
24
35
|
}
|
|
25
36
|
|
|
26
|
-
|
|
37
|
+
// onMainNextStep — PURE. The recovery command for the other half of the
|
|
38
|
+
// single-checkout catch-22 (task 1002572): the builder is standing ON main, so
|
|
39
|
+
// there is no head branch to merge FROM and none to open a PR from either. Since
|
|
40
|
+
// mergeLocally's sibling bail now falls back to the PR path, shipping from the
|
|
41
|
+
// task branch WORKS even in a single checkout — so the cure is "go back to the
|
|
42
|
+
// branch and re-ship", not the circular /merge-mode this used to advise. Only
|
|
43
|
+
// name the branch when it actually exists locally; otherwise fall back to
|
|
44
|
+
// /merge-mode rather than inventing a checkout target.
|
|
45
|
+
function onMainNextStep(taskId, taskBranchExists) {
|
|
46
|
+
return taskBranchExists
|
|
47
|
+
? `git checkout task-${taskId} && node scripts/gds/ship.js ${taskId}`
|
|
48
|
+
: '/merge-mode';
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// localTaskBranchExists — does this checkout still hold the task's own branch? The
|
|
52
|
+
// discriminator onMainNextStep needs, so the advice only names a branch that is
|
|
53
|
+
// really there (a builder who deleted it gets /merge-mode, not a failing checkout).
|
|
54
|
+
function localTaskBranchExists(taskId) {
|
|
55
|
+
return gitArgvOk(['rev-parse', '--verify', '--quiet', `refs/heads/task-${taskId}`], { stdio: 'ignore' });
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// autoMerge — the land entry point. Runs the local merge chain, then (task 1002572
|
|
59
|
+
// PART 2) routes the ONE bail a PR can still land into the push+PR path instead of
|
|
60
|
+
// leaving the task at 'confirmed'. The fallback deliberately sits OUTSIDE
|
|
61
|
+
// mergeLocally: its `finally` releases the shared merge lock and normalizes the main
|
|
62
|
+
// worktree, and the PR poll that follows can run for minutes — holding the lock
|
|
63
|
+
// across it would wedge every other ship on this machine.
|
|
64
|
+
// `deps` is a test seam ONLY — the routing is the whole deliverable and it cannot
|
|
65
|
+
// be reached from a unit test otherwise (mergeLocally needs a real second checkout
|
|
66
|
+
// and an authed gh). Callers pass nothing; the defaults are the real functions.
|
|
67
|
+
async function autoMerge(taskId, valueSummary, opts = {}, deps = {}) {
|
|
68
|
+
const merge = deps.mergeLocally || mergeLocally;
|
|
69
|
+
const land = deps.ciLand || ciLand;
|
|
70
|
+
const branchOf = deps.currentBranch || currentBranch;
|
|
71
|
+
const outcome = await merge(taskId, valueSummary, opts);
|
|
72
|
+
if (!landBailRecoverableByPr(outcome && outcome.bailId)) return outcome;
|
|
73
|
+
const branch = branchOf();
|
|
74
|
+
console.log(' auto-merge: a local merge is impossible from this checkout, but the branch is still landable —');
|
|
75
|
+
console.log(' falling back to the push+PR land path. Nothing is stranded; the server lands the PR.');
|
|
76
|
+
return await land(taskId, branch, valueSummary);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function mergeLocally(taskId, valueSummary, { dbOnly = false } = {}) {
|
|
27
80
|
const branch = currentBranch();
|
|
28
81
|
if (!branch || branch === 'main' || branch === 'HEAD') {
|
|
29
82
|
// task 1002723: an --allow-empty / --db-only ship legitimately runs FROM the
|
|
@@ -40,8 +93,15 @@ async function autoMerge(taskId, valueSummary, { dbOnly = false } = {}) {
|
|
|
40
93
|
console.log(` auto-merge: skipped (current branch is ${branch})`);
|
|
41
94
|
// The merge source IS the current branch, so on main there is nothing to
|
|
42
95
|
// merge. Half of the single-checkout catch-22 this task documents: the other
|
|
43
|
-
// half is the "main worktree is on <branch>, not main" bail below
|
|
44
|
-
|
|
96
|
+
// half is the "main worktree is on <branch>, not main" bail below, which now
|
|
97
|
+
// falls back to a PR. This half CANNOT — a PR needs a head branch and there
|
|
98
|
+
// is none here — so it stays a bail, but with a next step that actually
|
|
99
|
+
// recovers instead of pointing back at the same dead end (task 1002572).
|
|
100
|
+
return landBailed(
|
|
101
|
+
`nothing to merge — this checkout is on ${branch}, not a task branch`,
|
|
102
|
+
onMainNextStep(taskId, localTaskBranchExists(taskId)),
|
|
103
|
+
LAND_BAIL_NO_TASK_BRANCH,
|
|
104
|
+
);
|
|
45
105
|
}
|
|
46
106
|
|
|
47
107
|
// Refresh the data-driven diagrams before computing commits-ahead, so a
|
|
@@ -153,14 +213,17 @@ async function autoMerge(taskId, valueSummary, { dbOnly = false } = {}) {
|
|
|
153
213
|
// The main worktree must currently have main checked out.
|
|
154
214
|
const onMain = gitOk('git rev-parse --abbrev-ref HEAD', { cwd: REPO_ROOT });
|
|
155
215
|
if (onMain !== 'main') {
|
|
156
|
-
console.log(` auto-merge: main worktree is on ${onMain}, not main
|
|
216
|
+
console.log(` auto-merge: main worktree is on ${onMain}, not main — a local merge is impossible here`);
|
|
157
217
|
// The other half of the single-checkout catch-22: the merge target tree is
|
|
158
218
|
// the one holding the feature branch, so there is nowhere to merge INTO.
|
|
159
|
-
//
|
|
160
|
-
//
|
|
219
|
+
// task 1002572 (PART 2): this is a property of the MACHINE, not of the work
|
|
220
|
+
// — the branch lands fine via a PR. Tagged PR-recoverable so autoMerge falls
|
|
221
|
+
// back instead of parking the task at 'confirmed' unpaid. The reason +
|
|
222
|
+
// nextStep are kept for the case where the fallback itself bails.
|
|
161
223
|
return landBailed(
|
|
162
224
|
`the main checkout is on ${onMain}, not main — work a task in its own worktree so the main checkout stays on main`,
|
|
163
225
|
'/merge-mode',
|
|
226
|
+
LAND_BAIL_MAIN_CHECKOUT_ON_BRANCH,
|
|
164
227
|
);
|
|
165
228
|
}
|
|
166
229
|
|
|
@@ -293,5 +356,8 @@ async function autoMerge(taskId, valueSummary, { dbOnly = false } = {}) {
|
|
|
293
356
|
|
|
294
357
|
module.exports = {
|
|
295
358
|
autoMerge,
|
|
359
|
+
// task 1002572 (PART 2): the pure recovery command for a ship run from main —
|
|
360
|
+
// exported so the "don't point back at the dead end" wording is pinned.
|
|
361
|
+
onMainNextStep,
|
|
296
362
|
zeroCommitVerdict,
|
|
297
363
|
};
|
package/scripts/gds/ship.js
CHANGED
|
@@ -105,6 +105,9 @@ const {
|
|
|
105
105
|
const {
|
|
106
106
|
classifyLandOutcome,
|
|
107
107
|
formatPublishFailure,
|
|
108
|
+
LAND_BAIL_MAIN_CHECKOUT_ON_BRANCH,
|
|
109
|
+
LAND_BAIL_NO_TASK_BRANCH,
|
|
110
|
+
landBailRecoverableByPr,
|
|
108
111
|
landBailed,
|
|
109
112
|
landPending,
|
|
110
113
|
landPendingLines,
|
|
@@ -140,7 +143,7 @@ const {
|
|
|
140
143
|
isLinkifiableDocPath,
|
|
141
144
|
parseDeletedUpstreamPaths,
|
|
142
145
|
} = require('./ship-preflight-steps.js');
|
|
143
|
-
const { zeroCommitVerdict } = require('./ship-merge.js');
|
|
146
|
+
const { onMainNextStep, zeroCommitVerdict } = require('./ship-merge.js');
|
|
144
147
|
const {
|
|
145
148
|
buildGradePayload,
|
|
146
149
|
buildPriorRoundContext,
|
|
@@ -303,6 +306,15 @@ module.exports = {
|
|
|
303
306
|
// ship unless --db-only is explicitly passed. Exported pure so the gate is
|
|
304
307
|
// unit-tested (has-commits / db-only / refuse) without git.
|
|
305
308
|
zeroCommitVerdict,
|
|
309
|
+
// task 1002572 (PART 2): the single-checkout land fallback, pure halves only.
|
|
310
|
+
// landBailRecoverableByPr is the whole decision — which auto-merge bail a PR can
|
|
311
|
+
// still land (the main checkout holding the branch) versus which are REAL
|
|
312
|
+
// failures a PR would only carry to main. onMainNextStep is the recovery command
|
|
313
|
+
// for the other half of the catch-22, where no head branch exists to PR from.
|
|
314
|
+
LAND_BAIL_MAIN_CHECKOUT_ON_BRANCH,
|
|
315
|
+
LAND_BAIL_NO_TASK_BRANCH,
|
|
316
|
+
landBailRecoverableByPr,
|
|
317
|
+
onMainNextStep,
|
|
306
318
|
// touches[] cleanup [3/8] (task 878): folder-tag derivation, now over the
|
|
307
319
|
// committed diff — exported pure so the regression test can assert it.
|
|
308
320
|
deriveTagsFromPaths,
|
package/src/module-api.js
CHANGED
|
@@ -71,7 +71,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
|
|
|
71
71
|
// there. scripts/gds/bump-version.js still rewrites the literal below; it appends
|
|
72
72
|
// the entry to that file. Look for a version's history there, not here.
|
|
73
73
|
// ---------------------------------------------------------------------------
|
|
74
|
-
const CORE_VERSION = '1.19.
|
|
74
|
+
const CORE_VERSION = '1.19.652'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
|
|
75
75
|
|
|
76
76
|
// A namespaced logger so a module's log lines are attributable + consistent.
|
|
77
77
|
// Usage: const log = api.logger('dev-box'); log.info('mounted');
|
package/tests/boxes.mjs
CHANGED
|
@@ -170,7 +170,7 @@ t('selectIdleBoxes: guards bad inputs', () => {
|
|
|
170
170
|
assert.deepEqual(boxes.selectIdleBoxes([], { idleMinutes: 30, nowMs: NaN }), []);
|
|
171
171
|
});
|
|
172
172
|
|
|
173
|
-
t('selectIdleBoxes:
|
|
173
|
+
t('selectIdleBoxes: with NO cap configured, claude_active=true skips regardless of age (pre-1003507 path)', () => {
|
|
174
174
|
const rows = [
|
|
175
175
|
// Old + claude running → kept alive.
|
|
176
176
|
{ state: 'active', github_login: 'clauding', last_activity_at: new Date(NOW - 60 * 60000).toISOString(), claude_active: true },
|
|
@@ -183,6 +183,114 @@ t('selectIdleBoxes: skips boxes where claude_active=true regardless of age', ()
|
|
|
183
183
|
assert.deepEqual(idle.map((r) => r.github_login).sort(), ['idle', 'legacy']);
|
|
184
184
|
});
|
|
185
185
|
|
|
186
|
+
// --- task 1003507: the unattended cap ---------------------------------------
|
|
187
|
+
//
|
|
188
|
+
// The bug had TWO legs and either one alone kept an abandoned box alive:
|
|
189
|
+
// (1) claude_active is a latch only a ping can clear, and the sweep honoured it
|
|
190
|
+
// with no time bound at all -> unreapable at every threshold, forever;
|
|
191
|
+
// (2) load > 0.2 kept last_activity_at fresh, so the idle clock never expired.
|
|
192
|
+
// The reported box had BOTH, which is why these tests model a FRESH heartbeat:
|
|
193
|
+
// a fix that only bounds (1) leaves the box alive through (2) and looks green
|
|
194
|
+
// against a stale-heartbeat fixture.
|
|
195
|
+
const CAP = 12;
|
|
196
|
+
const HOURS = (n) => n * 60 * 60000;
|
|
197
|
+
const at = (ms) => new Date(ms).toISOString();
|
|
198
|
+
|
|
199
|
+
// The reported incident, to scale: droplet up 5 days, heartbeat 5 min old, a
|
|
200
|
+
// `claude` process running, nobody attached for 21h.
|
|
201
|
+
const abandonedBox = () => ({
|
|
202
|
+
state: 'active', github_login: 'abandoned', claude_active: true,
|
|
203
|
+
claude_active_since: at(NOW - HOURS(21)),
|
|
204
|
+
last_activity_at: at(NOW - 5 * 60000),
|
|
205
|
+
last_attached_at: at(NOW - HOURS(21)),
|
|
206
|
+
active_since: at(NOW - 5 * 24 * 60 * 60000),
|
|
207
|
+
});
|
|
208
|
+
const swept = (rows, over = {}) => boxes.selectIdleBoxes(rows, {
|
|
209
|
+
idleMinutes: 30, nowMs: NOW, unattendedMaxHours: CAP, ...over,
|
|
210
|
+
}).map((r) => r.github_login);
|
|
211
|
+
|
|
212
|
+
t('selectIdleBoxes: an unattended box past the cap is reaped even with a FRESH heartbeat', () => {
|
|
213
|
+
// The regression that matters: last_activity_at is 5 min old, so the ordinary
|
|
214
|
+
// idle rule can never fire. Only the attachment clock catches this box.
|
|
215
|
+
assert.deepEqual(swept([abandonedBox()]), ['abandoned']);
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
t('selectIdleBoxes: a box with a human attached is spared regardless of the cap', () => {
|
|
219
|
+
assert.deepEqual(swept([{ ...abandonedBox(), last_attached_at: at(NOW - 10 * 60000) }]), []);
|
|
220
|
+
assert.deepEqual(swept([{ ...abandonedBox(), last_attached_at: at(NOW - HOURS(11)) }]), []);
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
t('selectIdleBoxes: the cap boundary', () => {
|
|
224
|
+
assert.deepEqual(swept([{ ...abandonedBox(), last_attached_at: at(NOW - HOURS(12) + 60000) }]), []);
|
|
225
|
+
assert.deepEqual(swept([{ ...abandonedBox(), last_attached_at: at(NOW - HOURS(12) - 60000) }]), ['abandoned']);
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
t('selectIdleBoxes: a latched claude_active box that stopped heartbeating is reaped', () => {
|
|
229
|
+
// Leg (1) on its own: silence cannot clear the flag, so before this fix the
|
|
230
|
+
// row survived every sweep at every threshold.
|
|
231
|
+
assert.deepEqual(swept([{
|
|
232
|
+
state: 'active', github_login: 'zombie', claude_active: true,
|
|
233
|
+
claude_active_since: at(NOW - 365 * 24 * 60 * 60000),
|
|
234
|
+
last_activity_at: at(NOW - 365 * 24 * 60 * 60000),
|
|
235
|
+
last_attached_at: null,
|
|
236
|
+
}]), ['zombie']);
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
t('selectIdleBoxes: a box with NO attachment data keeps the old behaviour', () => {
|
|
240
|
+
// A box still running the pre-1003507 heartbeat never reports `attached`, so
|
|
241
|
+
// last_attached_at stays NULL. NULL must read as "no data", never as
|
|
242
|
+
// "unattended" — otherwise this fix parks every un-upgraded box mid-work.
|
|
243
|
+
// (core_238 deliberately does not backfill the column for the same reason.)
|
|
244
|
+
assert.deepEqual(swept([{
|
|
245
|
+
state: 'active', github_login: 'legacy', claude_active: true,
|
|
246
|
+
claude_active_since: at(NOW - HOURS(2)),
|
|
247
|
+
last_activity_at: at(NOW - 5 * 60000),
|
|
248
|
+
last_attached_at: null,
|
|
249
|
+
}]), []);
|
|
250
|
+
// ...but it is still reapable the ordinary way once it goes quiet.
|
|
251
|
+
assert.deepEqual(swept([{
|
|
252
|
+
state: 'active', github_login: 'legacy2', claude_active: false,
|
|
253
|
+
last_activity_at: at(NOW - HOURS(1)), last_attached_at: null,
|
|
254
|
+
}]), ['legacy2']);
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
t('selectIdleBoxes: omitting the cap preserves the unbounded veto exactly', () => {
|
|
258
|
+
assert.deepEqual(swept([abandonedBox()], { unattendedMaxHours: undefined }), []);
|
|
259
|
+
assert.deepEqual(swept([abandonedBox()], { unattendedMaxHours: 0 }), []);
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
t('attendedRecently: NULL is unknown (true), not unattended', () => {
|
|
263
|
+
assert.equal(boxes.attendedRecently({ last_attached_at: null }, { nowMs: NOW, unattendedMaxHours: CAP }), true);
|
|
264
|
+
assert.equal(boxes.attendedRecently({ last_attached_at: at(NOW - HOURS(1)) }, { nowMs: NOW, unattendedMaxHours: CAP }), true);
|
|
265
|
+
assert.equal(boxes.attendedRecently({ last_attached_at: at(NOW - HOURS(20)) }, { nowMs: NOW, unattendedMaxHours: CAP }), false);
|
|
266
|
+
// No cap configured -> nothing is ever "unattended".
|
|
267
|
+
assert.equal(boxes.attendedRecently({ last_attached_at: at(NOW - HOURS(999)) }, { nowMs: NOW }), true);
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
t('claudeVetoHolds: a true flag with no recorded start is refused', () => {
|
|
271
|
+
// An unknown latch age is exactly the forever-latch being fixed, so it must
|
|
272
|
+
// NOT be honoured — core_238 stamps every box that is claude_active on deploy.
|
|
273
|
+
assert.equal(boxes.claudeVetoHolds(
|
|
274
|
+
{ claude_active: true, claude_active_since: null }, { nowMs: NOW, unattendedMaxHours: CAP }
|
|
275
|
+
), false);
|
|
276
|
+
assert.equal(boxes.claudeVetoHolds(
|
|
277
|
+
{ claude_active: true, claude_active_since: at(NOW - HOURS(2)) }, { nowMs: NOW, unattendedMaxHours: CAP }
|
|
278
|
+
), true);
|
|
279
|
+
assert.equal(boxes.claudeVetoHolds(
|
|
280
|
+
{ claude_active: true, claude_active_since: at(NOW - HOURS(20)) }, { nowMs: NOW, unattendedMaxHours: CAP }
|
|
281
|
+
), false);
|
|
282
|
+
assert.equal(boxes.claudeVetoHolds({ claude_active: false }, { nowMs: NOW, unattendedMaxHours: CAP }), false);
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
t('idleReason: names the clock that actually selected the box', () => {
|
|
286
|
+
const opts = { nowMs: NOW, idleMinutes: 30, unattendedMaxHours: CAP };
|
|
287
|
+
assert.match(boxes.idleReason(abandonedBox(), opts), /unattended 21h/);
|
|
288
|
+
assert.match(boxes.idleReason({
|
|
289
|
+
state: 'active', claude_active: false,
|
|
290
|
+
last_activity_at: at(NOW - HOURS(1)), last_attached_at: at(NOW - 60000),
|
|
291
|
+
}, opts), /idle 60m/);
|
|
292
|
+
});
|
|
293
|
+
|
|
186
294
|
t('selectDormantBoxes: picks parked boxes past the dormant threshold', () => {
|
|
187
295
|
const rows = [
|
|
188
296
|
{ state: 'parked', github_login: 'gone', parked_at: new Date(NOW - 20 * DAY).toISOString() },
|
|
@@ -1264,6 +1372,7 @@ function sweepHarness({ idleRows, parkThrows = null }) {
|
|
|
1264
1372
|
async listBoxes() { return idleRows; },
|
|
1265
1373
|
selectIdleBoxes: boxes.selectIdleBoxes,
|
|
1266
1374
|
effectiveActivityMs: boxes.effectiveActivityMs,
|
|
1375
|
+
idleReason: boxes.idleReason,
|
|
1267
1376
|
async setBoxState(_db, _bid, state, patch) { states.push({ state, patch }); return {}; },
|
|
1268
1377
|
async recordComputeCost() { return 0; },
|
|
1269
1378
|
async recordEvent(_db, e) { events.push(e); },
|
|
@@ -1321,17 +1430,65 @@ await ta('cmdSweepIdle: dry-run reports "would park" and touches nothing', async
|
|
|
1321
1430
|
assert.equal(h.states.length, 0, 'no state written in dry-run');
|
|
1322
1431
|
});
|
|
1323
1432
|
|
|
1324
|
-
await ta('cmdSweepIdle: a box
|
|
1433
|
+
await ta('cmdSweepIdle: a box mid-session is never parked', async () => {
|
|
1434
|
+
// Genuinely in use: Claude started 20 min ago and somebody is attached.
|
|
1325
1435
|
const busy = {
|
|
1326
1436
|
id: 4, builder_id: 4, state: 'active', droplet_id: '1', github_login: 'busy',
|
|
1327
1437
|
last_activity_at: new Date(Date.now() - 99 * 3600_000).toISOString(), claude_active: true,
|
|
1438
|
+
claude_active_since: new Date(Date.now() - 20 * 60_000).toISOString(),
|
|
1439
|
+
last_attached_at: new Date(Date.now() - 20 * 60_000).toISOString(),
|
|
1328
1440
|
};
|
|
1329
1441
|
const h = sweepHarness({ idleRows: [busy] });
|
|
1330
1442
|
const res = await boxCli.cmdSweepIdle({}, h.fakeBoxes, { apply: true });
|
|
1331
|
-
assert.equal(res.parked, 0, '
|
|
1443
|
+
assert.equal(res.parked, 0, 'a live session still holds the box open');
|
|
1332
1444
|
assert.equal(h.states.length, 0);
|
|
1333
1445
|
});
|
|
1334
1446
|
|
|
1447
|
+
await ta('cmdSweepIdle: task 1003507 — a box claiming claude_active for 99h with nobody attached IS parked', async () => {
|
|
1448
|
+
// The reported incident, end to end through the command. Before this task the
|
|
1449
|
+
// sweep skipped the row on claude_active alone and reported 0 idle; it now
|
|
1450
|
+
// parks it (snapshot kept), and the log names the clock that selected it.
|
|
1451
|
+
const abandoned = {
|
|
1452
|
+
id: 5, builder_id: 5, state: 'active', droplet_id: '591743808', github_login: 'abandoned',
|
|
1453
|
+
last_activity_at: new Date(Date.now() - 5 * 60_000).toISOString(),
|
|
1454
|
+
claude_active: true,
|
|
1455
|
+
claude_active_since: new Date(Date.now() - 99 * 3600_000).toISOString(),
|
|
1456
|
+
last_attached_at: new Date(Date.now() - 99 * 3600_000).toISOString(),
|
|
1457
|
+
};
|
|
1458
|
+
const h = sweepHarness({ idleRows: [abandoned] });
|
|
1459
|
+
const lines = [];
|
|
1460
|
+
const origLog = console.log;
|
|
1461
|
+
console.log = (...a) => lines.push(a.join(' '));
|
|
1462
|
+
let res;
|
|
1463
|
+
try {
|
|
1464
|
+
res = await boxCli.cmdSweepIdle({}, h.fakeBoxes, { apply: false });
|
|
1465
|
+
} finally { console.log = origLog; }
|
|
1466
|
+
assert.equal(res.parked, 0, 'dry-run writes nothing');
|
|
1467
|
+
assert.ok(lines.some((l) => /would park .*abandoned/.test(l)), 'the abandoned box is selected');
|
|
1468
|
+
assert.ok(lines.some((l) => /unattended 99h/.test(l)), 'the log names the unattended clock, not the idle one');
|
|
1469
|
+
});
|
|
1470
|
+
|
|
1471
|
+
await ta('loadDeps: task 1003507 — resolving deps does not throw ReferenceError', async () => {
|
|
1472
|
+
// _deps was UNDECLARED, so loadDeps() threw on its first call and every
|
|
1473
|
+
// command reaching makeDo()/makeCf() died with it — including
|
|
1474
|
+
// `sweep-idle --apply`, which builds its DO client before the park loop, so
|
|
1475
|
+
// the sweep could never park anything. Reached here through the same
|
|
1476
|
+
// apply:true path that hid the bug: the assertion is that we get PAST dep
|
|
1477
|
+
// resolution, whatever the fake DO client then does.
|
|
1478
|
+
const row = {
|
|
1479
|
+
id: 6, builder_id: 6, state: 'active', droplet_id: '1', github_login: 'reap',
|
|
1480
|
+
last_activity_at: new Date(Date.now() - 99 * 3600_000).toISOString(), claude_active: false,
|
|
1481
|
+
};
|
|
1482
|
+
const h = sweepHarness({ idleRows: [row] });
|
|
1483
|
+
const origLog = console.log;
|
|
1484
|
+
console.log = () => {};
|
|
1485
|
+
let err = null;
|
|
1486
|
+
try {
|
|
1487
|
+
await boxCli.cmdSweepIdle({}, h.fakeBoxes, { apply: true });
|
|
1488
|
+
} catch (e) { err = e; } finally { console.log = origLog; }
|
|
1489
|
+
assert.ok(!(err && /_deps is not defined/.test(err.message)), `loadDeps threw: ${err && err.message}`);
|
|
1490
|
+
});
|
|
1491
|
+
|
|
1335
1492
|
console.log('\ntask 1002727 — the dormant sweep warns before it takes:');
|
|
1336
1493
|
|
|
1337
1494
|
const DORMANT_DAY = 86_400_000;
|