@bongos/core 1.19.876 → 1.19.877
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +93 -38
- package/.gitattributes +6 -2
- package/clients/bongos-client/README.md +1 -1
- package/clients/bongos-client/bongos-client.global.js +4 -0
- package/clients/bongos-client/index.cjs +4 -0
- package/clients/bongos-client/index.d.ts +7 -0
- package/clients/bongos-client/index.mjs +4 -0
- package/docs/adr/0307-autobongos-a-deterministic-supervisor-and-agentic-workers.md +12 -1
- package/docs/api/openapi.json +149 -3
- package/docs/api-reference.md +4 -2
- package/docs/copy-inventory.md +7 -6
- package/docs/copy-registry.json +16 -7
- package/docs/module-api-changelog.md +2 -0
- package/docs/recipes/autobongos-windows-host.md +138 -0
- package/docs/recipes/instance-deploy.md +29 -3
- package/modules/autonomy/cadence.js +276 -0
- package/modules/autonomy/db.js +47 -0
- package/modules/autonomy/migrations/autonomy_002_runner_heartbeat.sql +93 -0
- package/modules/autonomy/routes/autonomy.js +69 -0
- package/modules/autonomy/runner-health.js +97 -0
- package/modules/dev-box/app/src/vendor/bongos-client.cjs +4 -0
- package/modules/hall-ui/public/gate.html +4 -1
- package/modules/hall-ui/public/gate.js +51 -2
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/scripts/gds/autobongos-run.js +368 -11
- package/scripts/gds/autobongos-service.cmd +38 -0
- package/scripts/gds/autobongos-service.ps1 +129 -0
- package/scripts/gds/deploy-main-audit-gate.sh +90 -0
- package/scripts/gds/main-audit.js +26 -4
- package/scripts/gds/run-unit-tests.js +16 -1
- package/src/module-api.js +1 -1
- package/tests/autobongos_cadence.mjs +602 -0
- package/tests/autonomy_runner_health.mjs +135 -0
- package/tests/bump_version.mjs +12 -1
- package/tests/cli_package.mjs +11 -1
- package/tests/deploy_audit_gate.mjs +154 -0
- package/tests/deploy_divergence_line.mjs +47 -1
- package/tests/helpers.mjs +61 -0
- package/tests/main_audit_db_roster.mjs +97 -0
- package/tests/publish_watch.mjs +9 -0
- package/tests/setup_device_flow_fallback.mjs +2 -1
|
@@ -254,7 +254,7 @@
|
|
|
254
254
|
const el = document.getElementById('fence-state');
|
|
255
255
|
el.innerHTML = `
|
|
256
256
|
<div class="ov-foot" style="display:flex;align-items:center;gap:12px;flex-wrap:wrap;">
|
|
257
|
-
<
|
|
257
|
+
<span class="fact-pill ${on ? 'fact-pill--ok' : 'fact-pill--warn'}">${on ? 'Allowed to work' : 'Stopped'}</span>
|
|
258
258
|
<span>${on ? 'The runner may claim and work allowlisted goals.' : escapeHtml(fence.paused_reason || 'The runner is doing nothing.')}</span>
|
|
259
259
|
<button type="button" id="fence-toggle" class="btn">${on ? 'Stop the runner' : 'Start the runner'}</button>
|
|
260
260
|
</div>`;
|
|
@@ -317,6 +317,54 @@
|
|
|
317
317
|
}
|
|
318
318
|
}
|
|
319
319
|
|
|
320
|
+
// The runner's liveness (task 1003905). Rendered ABOVE the switch on purpose:
|
|
321
|
+
// "is it running at all" is the first question, and an owner who has just
|
|
322
|
+
// flipped the switch on wants to see the machine notice.
|
|
323
|
+
//
|
|
324
|
+
// The verdict is the SERVER's — modules/autonomy/runner-health.js decides what
|
|
325
|
+
// dead means and the route attaches it — so this page and any future alert
|
|
326
|
+
// cannot drift into two definitions of the same word.
|
|
327
|
+
// This page's OWN class vocabulary (style.css `.fact-pill--{ok,warn,danger}`),
|
|
328
|
+
// the same one renderRow spends for a PR's verdict. The first cut reached for
|
|
329
|
+
// inline `--color-text-*` tokens from a different design system: they do not
|
|
330
|
+
// exist here, resolved to inherited grey, and drew a dead runner in exactly the
|
|
331
|
+
// same colour as a healthy one. Named classes cannot fail that way silently —
|
|
332
|
+
// a missing class is visible in the stylesheet, a missing var is not.
|
|
333
|
+
const HEALTH_CLASS = { alive: 'fact-pill--ok', late: 'fact-pill--warn', dead: 'fact-pill--danger', never_seen: 'fact-pill--info' };
|
|
334
|
+
|
|
335
|
+
async function loadRunners() {
|
|
336
|
+
const el = document.getElementById('fence-runner');
|
|
337
|
+
if (!el) return;
|
|
338
|
+
let resp;
|
|
339
|
+
try {
|
|
340
|
+
resp = await fetchJson('/autonomy/runners');
|
|
341
|
+
} catch (err) {
|
|
342
|
+
if (err && err.kind === '401') return;
|
|
343
|
+
el.innerHTML = emptyStateHtml('Could not read the runner\u2019s check-ins just now.');
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
const runners = resp.runners || [];
|
|
347
|
+
if (!runners.length) {
|
|
348
|
+
// never_seen, and the server said so rather than this page guessing. Not an
|
|
349
|
+
// alarm: nobody has set the host up yet.
|
|
350
|
+
el.innerHTML = `<p class="ov-foot">${escapeHtml(resp.none ? resp.none.detail : 'No runner has ever checked in.')}</p>`;
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
el.innerHTML = runners.map((r) => {
|
|
354
|
+
const h = r.health || {};
|
|
355
|
+
const cls = HEALTH_CLASS[h.state] || 'fact-pill--info';
|
|
356
|
+
// NOT the word "running": the switch below already uses it for a different
|
|
357
|
+
// thing (allowed to work), and the two lines sit one above the other. A
|
|
358
|
+
// reader should never have to work out which sense is meant.
|
|
359
|
+
const label = { dead: 'Not checking in', late: 'Quiet', alive: 'Checking in' }[h.state] || 'Unknown';
|
|
360
|
+
return `<p class="ov-foot" style="display:flex;align-items:baseline;gap:10px;flex-wrap:wrap;">
|
|
361
|
+
<span class="fact-pill ${cls}">${escapeHtml(label)}</span>
|
|
362
|
+
<span>${escapeHtml(h.detail || '')}</span>
|
|
363
|
+
<span class="ov-fact">last check-in ${escapeHtml(h.age || 'unknown')} ago</span>
|
|
364
|
+
</p>`;
|
|
365
|
+
}).join('');
|
|
366
|
+
}
|
|
367
|
+
|
|
320
368
|
async function loadFence() {
|
|
321
369
|
let fence;
|
|
322
370
|
try {
|
|
@@ -356,7 +404,8 @@
|
|
|
356
404
|
document.getElementById('gate-body').hidden = false;
|
|
357
405
|
if (rank === 'archon') {
|
|
358
406
|
document.getElementById('fence-panel').hidden = false;
|
|
359
|
-
|
|
407
|
+
// Independent reads, so one round trip rather than two.
|
|
408
|
+
await Promise.all([loadRunners(), loadFence()]);
|
|
360
409
|
}
|
|
361
410
|
await load();
|
|
362
411
|
}
|
package/package-lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.19.
|
|
3
|
+
"version": "1.19.877",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@bongos/core",
|
|
9
|
-
"version": "1.19.
|
|
9
|
+
"version": "1.19.877",
|
|
10
10
|
"license": "AGPL-3.0-or-later",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"express": "^4.21.2",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.19.
|
|
3
|
+
"version": "1.19.877",
|
|
4
4
|
"description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
|
|
5
5
|
"license": "AGPL-3.0-or-later",
|
|
6
6
|
"main": "src/platform-server.js",
|
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
// one worker → record. No model anywhere in this file; every branch it takes is
|
|
7
7
|
// a branch a person can read.
|
|
8
8
|
//
|
|
9
|
-
// node scripts/gds/autobongos-run.js
|
|
10
|
-
// node scripts/gds/autobongos-run.js
|
|
9
|
+
// node scripts/gds/autobongos-run.js --forever # the daemon shape
|
|
10
|
+
// node scripts/gds/autobongos-run.js # one task, then exit
|
|
11
|
+
// node scripts/gds/autobongos-run.js --goal 1000119 # narrow to one goal
|
|
11
12
|
// node scripts/gds/autobongos-run.js --max-tasks 5
|
|
12
13
|
// node scripts/gds/autobongos-run.js --dry-run # pick only, claim nothing
|
|
13
14
|
//
|
|
@@ -22,13 +23,19 @@
|
|
|
22
23
|
// 1003903), called at step 6. A worker's `shipped` is a CLAIM — recorded here
|
|
23
24
|
// as `claims_shipped` so nothing can mistake it for a fact — and the ledger
|
|
24
25
|
// read is what turns it into one, or into a released claim and a blocker.
|
|
25
|
-
// ·
|
|
26
|
-
//
|
|
26
|
+
// · IT NEVER TERMINALLY STOPS (task 1003904, ADR 0307 §5). `--forever` runs the
|
|
27
|
+
// cadence state machine in modules/autonomy/cadence.js: three straight
|
|
28
|
+
// failures drop to `probing` — no new work, a growing wait, then exactly one
|
|
29
|
+
// task — and one success returns it to full speed. Nothing exits the loop,
|
|
30
|
+
// not even the kill switch, because a machine nobody is watching must not be
|
|
31
|
+
// able to look idle while it is dead. On boot it reconciles every claim a
|
|
32
|
+
// previous run left behind.
|
|
27
33
|
|
|
28
34
|
const { execFile, spawn } = require('node:child_process');
|
|
29
35
|
const fs = require('node:fs');
|
|
30
36
|
const path = require('node:path');
|
|
31
37
|
const crypto = require('node:crypto');
|
|
38
|
+
const os = require('node:os');
|
|
32
39
|
|
|
33
40
|
const seq = require('./sequence.js');
|
|
34
41
|
const gauge = require('./autonomy-gauge.js');
|
|
@@ -42,11 +49,12 @@ const fence = require('../../modules/autonomy/fence.js');
|
|
|
42
49
|
// where it would not be inside modules/ — which is exactly why fence.js takes the
|
|
43
50
|
// hits as an argument instead of matching them itself.
|
|
44
51
|
const { matchProtected } = require('../../src/bongos/permission-path-check');
|
|
52
|
+
const cadence = require('../../modules/autonomy/cadence.js');
|
|
45
53
|
|
|
46
54
|
const REPO_ROOT = path.resolve(__dirname, '..', '..');
|
|
47
55
|
|
|
48
56
|
function readArgv(argv) {
|
|
49
|
-
const out = { goals: [], maxTasks: 1, dryRun: false, json: false, model: DEFAULT_MODEL, timeoutMs: DEFAULT_WORKER_TIMEOUT_MS };
|
|
57
|
+
const out = { goals: [], maxTasks: 1, dryRun: false, json: false, forever: false, maxLoops: Infinity, model: DEFAULT_MODEL, timeoutMs: DEFAULT_WORKER_TIMEOUT_MS };
|
|
50
58
|
for (let i = 0; i < argv.length; i++) {
|
|
51
59
|
const a = argv[i];
|
|
52
60
|
if (a === '--goal') { out.goals.push(Number(argv[++i])); continue; }
|
|
@@ -56,6 +64,12 @@ function readArgv(argv) {
|
|
|
56
64
|
if (a === '--timeout-min') { out.timeoutMs = Number(argv[++i]) * 60_000; continue; }
|
|
57
65
|
if (a === '--dry-run') { out.dryRun = true; continue; }
|
|
58
66
|
if (a === '--json') { out.json = true; continue; }
|
|
67
|
+
// --forever is the DAEMON shape: never returns, decides its own speed.
|
|
68
|
+
if (a === '--forever') { out.forever = true; continue; }
|
|
69
|
+
// --max-loops bounds the forever loop. It exists so a test (and an operator
|
|
70
|
+
// trying the thing out) can run the machine to completion; production passes
|
|
71
|
+
// it never, and Infinity is the honest default.
|
|
72
|
+
if (a === '--max-loops') { out.maxLoops = Number(argv[++i]); continue; }
|
|
59
73
|
}
|
|
60
74
|
if (!out.goals.length && process.env.AUTOBONGOS_GOALS) {
|
|
61
75
|
out.goals = String(process.env.AUTOBONGOS_GOALS).split(',').map((g) => Number(g.trim())).filter(Boolean);
|
|
@@ -111,6 +125,12 @@ async function pickTask(goals, deps = {}) {
|
|
|
111
125
|
}
|
|
112
126
|
const claimableIds = await seq.fetchClaimableIds(api);
|
|
113
127
|
|
|
128
|
+
// The skipped lists from EVERY goal are kept, not just the last one. An empty
|
|
129
|
+
// queue and a queue where every task halts both yield no task, and telling them
|
|
130
|
+
// apart is the whole of `unworkable_queue` (task 1003904): calling the second
|
|
131
|
+
// one "idle" is how a goal full of blocked work sits for a week with the runner
|
|
132
|
+
// reporting itself perfectly healthy.
|
|
133
|
+
const allSkipped = [];
|
|
114
134
|
for (const goalId of goals) {
|
|
115
135
|
let byId;
|
|
116
136
|
try { byId = await seq.fetchGoalTasks(api, goalId, seq.GOAL_PAGE); }
|
|
@@ -120,8 +140,9 @@ async function pickTask(goals, deps = {}) {
|
|
|
120
140
|
const { order } = seq.topoOrder(candidates);
|
|
121
141
|
const { next, skipped } = seq.pickNext(order, { rank, claimableIds });
|
|
122
142
|
if (next) return { task: next, goalId, skipped };
|
|
143
|
+
for (const sk of skipped) allSkipped.push({ ...sk, goal_id: goalId });
|
|
123
144
|
}
|
|
124
|
-
return { none: true };
|
|
145
|
+
return { none: true, skipped: allSkipped };
|
|
125
146
|
}
|
|
126
147
|
|
|
127
148
|
// WORKER_ENV_ALLOW — what a worker is allowed to inherit (grader finding, task
|
|
@@ -277,7 +298,14 @@ async function iteration(opts, deps = {}) {
|
|
|
277
298
|
// for. gate.goals is already the intersection.
|
|
278
299
|
const picked = await pick(gate.goals, deps.pickDeps || {});
|
|
279
300
|
if (picked.error) return log({ event: 'pick_failed', reason: picked.error });
|
|
280
|
-
if (picked.none)
|
|
301
|
+
if (picked.none) {
|
|
302
|
+
return log({
|
|
303
|
+
event: 'nothing_claimable', goals: gate.goals,
|
|
304
|
+
// Carried so the cadence can tell an EMPTY queue from one where every task
|
|
305
|
+
// halts — the reasons are the matrix's own codes, safe to print.
|
|
306
|
+
skipped: (picked.skipped || []).map((sk) => ({ id: sk.id, goal_id: sk.goal_id, reasons: (sk.reasons || []).map((r) => r.code) })),
|
|
307
|
+
});
|
|
308
|
+
}
|
|
281
309
|
const task = picked.task;
|
|
282
310
|
|
|
283
311
|
// 2b. The per-TASK half of the fence. sequence.js's halt matrix already refuses
|
|
@@ -364,19 +392,343 @@ async function iteration(opts, deps = {}) {
|
|
|
364
392
|
return log(row);
|
|
365
393
|
}
|
|
366
394
|
|
|
395
|
+
// ---------------------------------------------------------------------------
|
|
396
|
+
// Boot recovery, and the forever loop (task 1003904, ADR 0307 §5)
|
|
397
|
+
// ---------------------------------------------------------------------------
|
|
398
|
+
|
|
399
|
+
// reconcileLeftoverClaims — the FIRST thing a run does.
|
|
400
|
+
//
|
|
401
|
+
// "A silently locked task is the worst outcome this loop can produce" is the
|
|
402
|
+
// task's own wording, and cutting power mid-task is exactly how it happens: the
|
|
403
|
+
// claim is held by a session that no longer exists anywhere, and nothing times it
|
|
404
|
+
// out. So before taking any new work, every claim a previous run left behind is
|
|
405
|
+
// read back from the ledger and resolved.
|
|
406
|
+
//
|
|
407
|
+
// The reconciliation is autobongos-verify.js, unchanged (task 1003903). That file
|
|
408
|
+
// already answers "did this actually ship, and if not what do I do about it" for
|
|
409
|
+
// a worker that just exited; a worker that exited three days ago in a power cut
|
|
410
|
+
// is the same question with a longer gap. Writing a second, subtly different
|
|
411
|
+
// answer here is how two code paths end up disagreeing about what `confirmed`
|
|
412
|
+
// means.
|
|
413
|
+
//
|
|
414
|
+
// ONLY TREES THE RUNNER ITSELF MADE, matched against the EXACT shape
|
|
415
|
+
// worktreeName() produces — not a loose prefix.
|
|
416
|
+
//
|
|
417
|
+
// FOUND THE HARD WAY, twice. `autobongos-` as a prefix reads as "ours", and it is
|
|
418
|
+
// not: a person naming a worktree `autobongos-resume-1003904-d29f25` while
|
|
419
|
+
// working on this very feature matched it, and reconciliation released their live
|
|
420
|
+
// claim out from under them. A human at a keyboard picks readable names; the
|
|
421
|
+
// runner picks `autobongos-<task id>-<6 hex>` and nothing else. Matching the
|
|
422
|
+
// generator's own shape is the difference between "probably ours" and "ours".
|
|
423
|
+
//
|
|
424
|
+
// Derived FROM worktreeName so the two cannot drift: change the name format and
|
|
425
|
+
// this regex is what stops the next reconciliation from quietly matching nothing.
|
|
426
|
+
const CLAIM_PREFIX = 'autobongos-';
|
|
427
|
+
const RUNNER_TREE_RE = /^autobongos-\d+-[0-9a-f]{6}$/;
|
|
428
|
+
const isRunnerTree = (name) => RUNNER_TREE_RE.test(String(name || ''));
|
|
429
|
+
|
|
430
|
+
// ── the heartbeat, and why reconciliation needs one ──────────────────────────
|
|
431
|
+
//
|
|
432
|
+
// FOUND BY RUNNING IT. The first live run of reconcileLeftoverClaims released a
|
|
433
|
+
// claim that was very much alive — the one held by the session writing this code.
|
|
434
|
+
// It was right by its own rules (an `autobongos-` tree, a task sitting at
|
|
435
|
+
// `active`), and that is the problem: THE LEDGER CANNOT TELL A LIVE RUN FROM A
|
|
436
|
+
// DEAD ONE. Both look like a held claim. A reconciler that guesses wrong in one
|
|
437
|
+
// direction leaves a task locked forever; in the other it yanks a claim out from
|
|
438
|
+
// under a worker mid-build and strands whatever it had committed.
|
|
439
|
+
//
|
|
440
|
+
// So the runner says, on disk, that it is alive. A heartbeat file carries the pid
|
|
441
|
+
// and a timestamp; reconciliation refuses to touch anything while one is FRESH
|
|
442
|
+
// and its pid still exists. Both halves are needed — a stale file from a power
|
|
443
|
+
// cut must not veto recovery forever, and a fresh file whose process is gone is
|
|
444
|
+
// just litter. This is also what stops two runners on one machine from fighting
|
|
445
|
+
// over each other's claims.
|
|
446
|
+
const HEARTBEAT_STALE_MS = 10 * 60 * 1000;
|
|
447
|
+
|
|
448
|
+
function heartbeatPath() {
|
|
449
|
+
if (process.env.AUTOBONGOS_HEARTBEAT_FILE) return process.env.AUTOBONGOS_HEARTBEAT_FILE;
|
|
450
|
+
try { return require('../../src/instance-config.js').configPath('autobongos-heartbeat.json'); }
|
|
451
|
+
catch (_) { return path.join(REPO_ROOT, '.autobongos-heartbeat.json'); }
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// Best-effort in both directions: a runner that cannot write its heartbeat must
|
|
455
|
+
// still run (it only loses the guard), and one that cannot read it must assume
|
|
456
|
+
// nothing is alive — the file's absence is the common case, on every first boot.
|
|
457
|
+
//
|
|
458
|
+
// DO NOT point AUTOBONGOS_HEARTBEAT_FILE UNDER /proc TO TEST THE FAILURE PATH (task
|
|
459
|
+
// 1004052). The mkdirSync below is recursive, and on Linux a recursive mkdir under
|
|
460
|
+
// /proc never returns — measured 10s+ with no result, where the same call throws
|
|
461
|
+
// EACCES instantly under /sys or /, and the NON-recursive form throws ENOENT instantly
|
|
462
|
+
// under /proc itself. macOS has no /proc, so it fails fast there. A test that used it
|
|
463
|
+
// was green on every developer machine and hung the Ubuntu CI runner for 90s with no
|
|
464
|
+
// output, reddening `unit` on main and blocking every PR. Use a regular FILE as the
|
|
465
|
+
// parent directory instead: mkdir under it is ENOTDIR immediately, everywhere, and it
|
|
466
|
+
// does not depend on the caller's permissions.
|
|
467
|
+
function writeHeartbeat(now = Date.now()) {
|
|
468
|
+
try {
|
|
469
|
+
const p = heartbeatPath();
|
|
470
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
471
|
+
fs.writeFileSync(p, JSON.stringify({ pid: process.pid, at: now }));
|
|
472
|
+
return true;
|
|
473
|
+
} catch (_) { return false; }
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
// postHeartbeat — tell the SERVER we are alive (task 1003905).
|
|
477
|
+
//
|
|
478
|
+
// The file above guards against two runners on this machine; this is the half an
|
|
479
|
+
// owner can see from a web page in another country. Both matter and neither
|
|
480
|
+
// replaces the other: a file is unreadable on exactly the day the machine is off.
|
|
481
|
+
//
|
|
482
|
+
// FAILS SILENTLY, ON PURPOSE. A runner that cannot check in must keep working —
|
|
483
|
+
// letting the watching mechanism take down the thing it watches is a strictly
|
|
484
|
+
// worse outcome than a gap in the log. The gap reports itself anyway: the hall
|
|
485
|
+
// reads the AGE of the last check-in, so a failed post shows up as silence, which
|
|
486
|
+
// is the signal.
|
|
487
|
+
const RUNNER_STARTED_AT = new Date().toISOString();
|
|
488
|
+
|
|
489
|
+
async function postHeartbeat(fields = {}, deps = {}) {
|
|
490
|
+
try {
|
|
491
|
+
const api = deps.api || await (require('./cli-lib').cliClient)();
|
|
492
|
+
await api.autonomy.postAutonomyHeartbeat({
|
|
493
|
+
body: {
|
|
494
|
+
host: os.hostname(),
|
|
495
|
+
pid: process.pid,
|
|
496
|
+
started_at: RUNNER_STARTED_AT,
|
|
497
|
+
...fields,
|
|
498
|
+
},
|
|
499
|
+
});
|
|
500
|
+
return true;
|
|
501
|
+
} catch (_) { return false; }
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
function readHeartbeat() {
|
|
505
|
+
try { return JSON.parse(fs.readFileSync(heartbeatPath(), 'utf8')); }
|
|
506
|
+
catch (_) { return null; }
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
// pidAlive — `kill(pid, 0)` signals nothing and throws ESRCH when the process is
|
|
510
|
+
// gone. EPERM means it exists and belongs to someone else, which still counts as
|
|
511
|
+
// alive: the question is "is something running", not "is it mine".
|
|
512
|
+
function pidAlive(pid) {
|
|
513
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
514
|
+
try { process.kill(pid, 0); return true; }
|
|
515
|
+
catch (e) { return e && e.code === 'EPERM'; }
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
// anotherRunnerIsAlive — the veto. Fresh file AND a live pid AND not us.
|
|
519
|
+
function anotherRunnerIsAlive({ now = Date.now(), hb = readHeartbeat() } = {}) {
|
|
520
|
+
if (!hb || !Number.isFinite(Number(hb.at))) return false;
|
|
521
|
+
if (Number(hb.pid) === process.pid) return false; // our own beat
|
|
522
|
+
if (now - Number(hb.at) > HEARTBEAT_STALE_MS) return false; // a power cut leaves a stale file
|
|
523
|
+
return pidAlive(Number(hb.pid));
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
async function reconcileLeftoverClaims(deps = {}) {
|
|
527
|
+
const api = deps.api || await (require('./cli-lib').cliClient)();
|
|
528
|
+
const log = deps.record || record;
|
|
529
|
+
const verify = deps.verifyShip || verifyShip;
|
|
530
|
+
const runCmd = deps.run || run;
|
|
531
|
+
const isAlive = deps.anotherRunnerIsAlive || anotherRunnerIsAlive;
|
|
532
|
+
|
|
533
|
+
// Refuse outright while another runner is beating. Releasing a claim a live
|
|
534
|
+
// worker is building against strands whatever it has committed — the same harm
|
|
535
|
+
// as leaving one locked, pointed the other way.
|
|
536
|
+
if (isAlive()) return log({ event: 'boot_reconcile', ok: true, skipped: 'another runner is alive', leftover: null });
|
|
537
|
+
|
|
538
|
+
let claims = [];
|
|
539
|
+
try {
|
|
540
|
+
const me = await api.me.getMe();
|
|
541
|
+
if (!me || !me.ok) return log({ event: 'boot_reconcile', ok: false, reason: `GET /me → ${me ? me.status : 'no response'}` });
|
|
542
|
+
claims = (me.data.active_claims || []).filter((c) => isRunnerTree(c.worktree_name));
|
|
543
|
+
} catch (e) {
|
|
544
|
+
// Unreadable is NOT "nothing to reconcile". Say so and let the cadence treat
|
|
545
|
+
// it as a failure, rather than starting work on top of an unknown lock.
|
|
546
|
+
return log({ event: 'boot_reconcile', ok: false, reason: e && e.message ? e.message : String(e) });
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
if (!claims.length) return log({ event: 'boot_reconcile', ok: true, leftover: 0 });
|
|
550
|
+
|
|
551
|
+
const resolved = [];
|
|
552
|
+
for (const c of claims) {
|
|
553
|
+
const wtPath = path.join(REPO_ROOT, '.claude', 'worktrees', c.worktree_name);
|
|
554
|
+
const out = await verify(
|
|
555
|
+
{ taskId: c.task_id, taskTitle: c.title, worker: { outcome: 'interrupted', reason: 'a previous run did not finish this claim' } },
|
|
556
|
+
{
|
|
557
|
+
api,
|
|
558
|
+
// The tree may be gone — a power cut leaves the claim, not necessarily the
|
|
559
|
+
// checkout. verifyShip only uses cwd for the artifact probe, which degrades
|
|
560
|
+
// to `checked: false` rather than failing, so a missing path is survivable.
|
|
561
|
+
cwd: fs.existsSync(wtPath) ? wtPath : REPO_ROOT,
|
|
562
|
+
release: (id, reason) => runCmd(process.execPath,
|
|
563
|
+
[path.join(REPO_ROOT, 'scripts', 'gds', 'release.js'), String(id), '--reason', reason],
|
|
564
|
+
{ cwd: fs.existsSync(wtPath) ? wtPath : REPO_ROOT }),
|
|
565
|
+
},
|
|
566
|
+
);
|
|
567
|
+
resolved.push({ task_id: String(c.task_id), worktree: c.worktree_name, shape: out.shape, released: !!out.released, blocker_id: out.blocker?.blockerId ?? null });
|
|
568
|
+
}
|
|
569
|
+
return log({ event: 'boot_reconcile', ok: true, leftover: claims.length, resolved });
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
// pollSignals — the cheap facts whose change makes waiting pointless.
|
|
573
|
+
//
|
|
574
|
+
// Three reads, all small, all failing SILENTLY to null. A failed probe is not a
|
|
575
|
+
// signal: treating it as one would turn an unreachable instance into a busy loop
|
|
576
|
+
// hammering it, which is the exact opposite of backing off.
|
|
577
|
+
async function pollSignals(deps = {}) {
|
|
578
|
+
const api = deps.api || await (require('./cli-lib').cliClient)();
|
|
579
|
+
let head = null; let fenceUpdatedAt = null; let claimableCount = null;
|
|
580
|
+
try {
|
|
581
|
+
const r = await runFetchHead(deps);
|
|
582
|
+
if (r) head = r;
|
|
583
|
+
} catch (_) { /* null */ }
|
|
584
|
+
try {
|
|
585
|
+
const f = await api.autonomy.getAutonomyFence();
|
|
586
|
+
if (f && f.ok) fenceUpdatedAt = f.data.updated_at || null;
|
|
587
|
+
} catch (_) { /* null */ }
|
|
588
|
+
try {
|
|
589
|
+
const ids = await seq.fetchClaimableIds(api);
|
|
590
|
+
if (ids) claimableCount = ids.size;
|
|
591
|
+
} catch (_) { /* null */ }
|
|
592
|
+
return cadence.signalFingerprint({ head, fenceUpdatedAt, claimableCount });
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
// The head of origin/main, or null. `git ls-remote` rather than `fetch`: it moves
|
|
596
|
+
// no refs and writes nothing, so polling it every few minutes cannot interfere
|
|
597
|
+
// with a worker building in a tree beside it.
|
|
598
|
+
async function runFetchHead(deps = {}) {
|
|
599
|
+
const runCmd = deps.run || run;
|
|
600
|
+
const r = await runCmd('git', ['ls-remote', 'origin', 'refs/heads/main']);
|
|
601
|
+
if (!r.ok) return null;
|
|
602
|
+
const m = /^([0-9a-f]{40})/.exec(String(r.stdout).trim());
|
|
603
|
+
return m ? m[1] : null;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
// waitOrJump — sleep, but wake early if the world moved.
|
|
607
|
+
//
|
|
608
|
+
// Polls at most every `pollEveryS`, so a thirty-minute backoff costs six cheap
|
|
609
|
+
// probes rather than a spin. `sleep` is injected so a test can drive an hour of
|
|
610
|
+
// wall clock in a millisecond.
|
|
611
|
+
async function waitOrJump(waitS, deps = {}, cfg = {}) {
|
|
612
|
+
if (!(waitS > 0)) return { waited: 0, jumped: false };
|
|
613
|
+
const pollEveryS = cfg.pollEveryS ?? 60;
|
|
614
|
+
const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
615
|
+
const poll = deps.pollSignals || pollSignals;
|
|
616
|
+
|
|
617
|
+
const before = await poll(deps);
|
|
618
|
+
let waited = 0;
|
|
619
|
+
while (waited < waitS) {
|
|
620
|
+
const chunk = Math.min(pollEveryS, waitS - waited);
|
|
621
|
+
await sleep(chunk * 1000);
|
|
622
|
+
waited += chunk;
|
|
623
|
+
// Beat DURING the wait. A thirty-minute backoff is the runner working exactly
|
|
624
|
+
// as designed, and it must not be indistinguishable from a machine that fell
|
|
625
|
+
// over at the start of it.
|
|
626
|
+
if (deps.beat) { try { await deps.beat({ last_event: 'waiting' }); } catch (_) { /* never break a wait */ } }
|
|
627
|
+
if (waited >= waitS) break;
|
|
628
|
+
const now = await poll(deps);
|
|
629
|
+
if (cadence.shouldJumpWait(before, now)) return { waited, jumped: true, signal: now };
|
|
630
|
+
}
|
|
631
|
+
return { waited, jumped: false };
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
// forever — the loop that has no terminal stop (task 1003904, ADR 0307 §5).
|
|
635
|
+
//
|
|
636
|
+
// It is a `while (true)` on purpose and the exceptions are the interesting part:
|
|
637
|
+
// NOTHING in here returns. Not an unreachable instance, not an expired token, not
|
|
638
|
+
// the kill switch. The only exits are a signal from the operating system and
|
|
639
|
+
// `--max-loops`, which exists so a test can run the machine to completion.
|
|
640
|
+
//
|
|
641
|
+
// What varies is the SPEED, and the speed is decided entirely by
|
|
642
|
+
// modules/autonomy/cadence.js from the row the last iteration recorded. This
|
|
643
|
+
// function holds no policy — it turns a `waitS` into a wait and a `takeWork` into
|
|
644
|
+
// an iteration, so the rules stay somewhere a test can reach without a clock.
|
|
645
|
+
async function forever(opts, deps = {}) {
|
|
646
|
+
const log = deps.record || record;
|
|
647
|
+
const iterate = deps.iteration || iteration;
|
|
648
|
+
const reconcile = deps.reconcileLeftoverClaims || reconcileLeftoverClaims;
|
|
649
|
+
const wait = deps.waitOrJump || waitOrJump;
|
|
650
|
+
const emit = deps.emit || ((row) => {
|
|
651
|
+
if (opts.json) console.log(JSON.stringify(row));
|
|
652
|
+
else console.log(`${row.event}${row.task_id ? ` task ${row.task_id}` : ''}${row.outcome ? ` → ${row.outcome}` : ''}${row.reason ? `: ${row.reason}` : ''}`);
|
|
653
|
+
});
|
|
654
|
+
|
|
655
|
+
// Claim the machine BEFORE reconciling: a second runner starting a moment later
|
|
656
|
+
// must see this one alive rather than deciding its claims are litter.
|
|
657
|
+
// Both halves of "I am alive": the local file (which stops a second runner
|
|
658
|
+
// treating this one's claims as litter) and the server row (which is the only
|
|
659
|
+
// one an owner can see from anywhere else).
|
|
660
|
+
const beatFile = deps.writeHeartbeat || writeHeartbeat;
|
|
661
|
+
const beatServer = deps.postHeartbeat || postHeartbeat;
|
|
662
|
+
// WRAPPED AT THE CALL SITE, not just inside postHeartbeat. The guarantee is
|
|
663
|
+
// "reporting that I am alive can never be what kills me", and that has to hold
|
|
664
|
+
// for whatever beat implementation is in play — a future one, an injected one —
|
|
665
|
+
// not only for the well-behaved default. Letting the watching mechanism take
|
|
666
|
+
// down the thing it watches is strictly worse than a gap in the log, and the gap
|
|
667
|
+
// reports itself anyway: the hall reads the AGE of the last check-in, so a
|
|
668
|
+
// failed beat shows up as silence, which is the signal.
|
|
669
|
+
const beat = async (fields) => {
|
|
670
|
+
try { beatFile(); } catch (_) { /* a runner that cannot write its file still runs */ }
|
|
671
|
+
try { await beatServer(fields, deps); } catch (_) { /* and one that cannot reach the instance still runs */ }
|
|
672
|
+
};
|
|
673
|
+
await beat({ last_event: 'boot' });
|
|
674
|
+
|
|
675
|
+
// BEFORE ANY WORK. A claim left locked by a previous run is the worst thing
|
|
676
|
+
// this loop produces, and taking new work on top of one compounds it.
|
|
677
|
+
emit(await reconcile(deps));
|
|
678
|
+
|
|
679
|
+
let state = { ...cadence.START };
|
|
680
|
+
const maxLoops = Number.isFinite(Number(opts.maxLoops)) ? Number(opts.maxLoops) : Infinity;
|
|
681
|
+
|
|
682
|
+
for (let loop = 0; loop < maxLoops; loop++) {
|
|
683
|
+
// Every loop, so a crash leaves a heartbeat that goes stale on its own — the
|
|
684
|
+
// runner never has to notice it is dying for its death to become visible.
|
|
685
|
+
await beat({ mode: state.mode, consecutive_failures: state.consecutiveFailures, last_event: 'loop' });
|
|
686
|
+
// In `probing` the runner takes exactly ONE task and only after its wait, so
|
|
687
|
+
// a broken window costs a handful of cheap requests rather than a night of
|
|
688
|
+
// failing spawns. `maxTasks` still bounds a full-speed burst.
|
|
689
|
+
const budget = state.mode === 'probing' ? 1 : opts.maxTasks;
|
|
690
|
+
let row = null;
|
|
691
|
+
|
|
692
|
+
for (let i = 0; i < budget; i++) {
|
|
693
|
+
row = await iterate(opts, deps);
|
|
694
|
+
emit(row);
|
|
695
|
+
// Carry the task id so the hall can tell "quiet because it is building
|
|
696
|
+
// task 1234" from "quiet because it is dead". A worker may hold the loop for
|
|
697
|
+
// ninety minutes, and without this every honest hour of work looks like a
|
|
698
|
+
// corpse.
|
|
699
|
+
await beat({ mode: state.mode, last_event: row.event, working_task_id: Number(row.task_id) || undefined });
|
|
700
|
+
const seen = cadence.classifyEvent(row);
|
|
701
|
+
state = cadence.nextCadence(state, seen, opts.cadence);
|
|
702
|
+
log({ event: 'cadence', mode: state.mode, consecutive_failures: state.consecutiveFailures, wait_s: state.waitS, class: seen.class, why: state.why });
|
|
703
|
+
// A burst continues only while work is actually landing. Anything else ends
|
|
704
|
+
// the burst and hands the decision to the cadence, which is the only thing
|
|
705
|
+
// allowed to say how long to wait.
|
|
706
|
+
if (seen.class !== 'progress') break;
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
const { waited, jumped } = await wait(state.waitS, { ...deps, beat }, opts.cadence);
|
|
710
|
+
if (jumped) log({ event: 'cadence', mode: state.mode, jumped: true, waited_s: waited, why: 'a new signal arrived — main moved, the fence changed, or the queue did' });
|
|
711
|
+
}
|
|
712
|
+
return 0;
|
|
713
|
+
}
|
|
714
|
+
|
|
367
715
|
async function main(argv) {
|
|
368
716
|
const opts = readArgv(argv);
|
|
369
717
|
// No "name a goal" refusal here any more. It used to be the only allowlist, and
|
|
370
718
|
// it lived on the machine being fenced; the database owns that now, and an empty
|
|
371
719
|
// `--goal` means "every allowlisted goal" rather than "anything claimable". The
|
|
372
720
|
// refusal did not disappear — it moved somewhere the runner host cannot edit.
|
|
721
|
+
if (opts.forever) return forever(opts);
|
|
722
|
+
|
|
723
|
+
// The one-shot path, kept for an operator running a single task by hand. It
|
|
724
|
+
// still reconciles first: a leftover claim is a leftover claim whoever is
|
|
725
|
+
// watching.
|
|
726
|
+
const boot = await reconcileLeftoverClaims();
|
|
727
|
+
if (opts.json) console.log(JSON.stringify(boot));
|
|
373
728
|
for (let i = 0; i < opts.maxTasks; i++) {
|
|
374
729
|
const row = await iteration(opts);
|
|
375
730
|
if (opts.json) console.log(JSON.stringify(row));
|
|
376
731
|
else console.log(`${row.event}${row.task_id ? ` task ${row.task_id}` : ''}${row.outcome ? ` → ${row.outcome}` : ''}${row.reason ? `: ${row.reason}` : ''}`);
|
|
377
|
-
// Anything that is not a completed piece of work ends this run — including
|
|
378
|
-
// `verify_halt`, which is the ledger being unreadable. Deciding when to come
|
|
379
|
-
// back is task 1003904's job, not this one's.
|
|
380
732
|
if (row.event !== 'worked') break;
|
|
381
733
|
}
|
|
382
734
|
return 0;
|
|
@@ -389,4 +741,9 @@ if (require.main === module) {
|
|
|
389
741
|
});
|
|
390
742
|
}
|
|
391
743
|
|
|
392
|
-
module.exports = {
|
|
744
|
+
module.exports = {
|
|
745
|
+
readArgv, pickTask, iteration, record, logPath, spawnWorker, workerEnv, WORKER_ENV_ALLOW,
|
|
746
|
+
worktreeName, readFence, reconcileLeftoverClaims, pollSignals, waitOrJump, forever, CLAIM_PREFIX,
|
|
747
|
+
heartbeatPath, writeHeartbeat, readHeartbeat, pidAlive, anotherRunnerIsAlive, HEARTBEAT_STALE_MS,
|
|
748
|
+
isRunnerTree, RUNNER_TREE_RE, postHeartbeat, RUNNER_STARTED_AT,
|
|
749
|
+
};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
@echo off
|
|
2
|
+
REM scripts/gds/autobongos-service.cmd -- what the scheduled task actually runs
|
|
3
|
+
REM (task 1003905). A .cmd, not a direct node invocation, for three reasons the
|
|
4
|
+
REM scheduler cannot do itself.
|
|
5
|
+
REM
|
|
6
|
+
REM 1. THE CHECKOUT IS BROUGHT CURRENT FIRST. A runner pinned to whatever was on
|
|
7
|
+
REM disk the day it was installed drifts silently, and the first symptom is
|
|
8
|
+
REM workers building against a stale fence. `git fetch` + `reset --hard` on the
|
|
9
|
+
REM MAIN checkout only -- worker trees are separate worktrees and are never
|
|
10
|
+
REM touched here.
|
|
11
|
+
REM 2. OUTPUT IS APPENDED TO ONE FILE the owner can read remotely. The scheduler
|
|
12
|
+
REM keeps only an exit code, which cannot answer "what was it doing".
|
|
13
|
+
REM 3. A NON-ZERO EXIT IS RETURNED so the scheduler's restart rule can see it.
|
|
14
|
+
REM
|
|
15
|
+
REM The runner itself never exits on failure (task 1003904) -- it backs off and
|
|
16
|
+
REM probes -- so reaching the end of this file at all means something the inner
|
|
17
|
+
REM loop could not survive.
|
|
18
|
+
|
|
19
|
+
setlocal
|
|
20
|
+
set "REPO=%~dp0..\.."
|
|
21
|
+
set "LOGDIR=%LOCALAPPDATA%\cloudbongos"
|
|
22
|
+
set "LOG=%LOGDIR%\autobongos.log"
|
|
23
|
+
if not exist "%LOGDIR%" mkdir "%LOGDIR%"
|
|
24
|
+
|
|
25
|
+
cd /d "%REPO%" || exit /b 1
|
|
26
|
+
|
|
27
|
+
echo. >> "%LOG%"
|
|
28
|
+
echo ==== autobongos starting %DATE% %TIME% ==== >> "%LOG%"
|
|
29
|
+
|
|
30
|
+
REM Best effort: a fetch that fails must not stop the runner from starting on the
|
|
31
|
+
REM code it already has. Being a commit behind beats not running at all.
|
|
32
|
+
git fetch --quiet origin main >> "%LOG%" 2>&1
|
|
33
|
+
git reset --quiet --hard origin/main >> "%LOG%" 2>&1
|
|
34
|
+
|
|
35
|
+
node scripts\gds\autobongos-run.js --forever --json >> "%LOG%" 2>&1
|
|
36
|
+
set EXITCODE=%ERRORLEVEL%
|
|
37
|
+
echo ==== autobongos exited %EXITCODE% at %DATE% %TIME% ==== >> "%LOG%"
|
|
38
|
+
exit /b %EXITCODE%
|