@bongos/core 1.19.876 → 1.19.877

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/.bongos-core.json +93 -38
  2. package/.gitattributes +6 -2
  3. package/clients/bongos-client/README.md +1 -1
  4. package/clients/bongos-client/bongos-client.global.js +4 -0
  5. package/clients/bongos-client/index.cjs +4 -0
  6. package/clients/bongos-client/index.d.ts +7 -0
  7. package/clients/bongos-client/index.mjs +4 -0
  8. package/docs/adr/0307-autobongos-a-deterministic-supervisor-and-agentic-workers.md +12 -1
  9. package/docs/api/openapi.json +149 -3
  10. package/docs/api-reference.md +4 -2
  11. package/docs/copy-inventory.md +7 -6
  12. package/docs/copy-registry.json +16 -7
  13. package/docs/module-api-changelog.md +2 -0
  14. package/docs/recipes/autobongos-windows-host.md +138 -0
  15. package/docs/recipes/instance-deploy.md +29 -3
  16. package/modules/autonomy/cadence.js +276 -0
  17. package/modules/autonomy/db.js +47 -0
  18. package/modules/autonomy/migrations/autonomy_002_runner_heartbeat.sql +93 -0
  19. package/modules/autonomy/routes/autonomy.js +69 -0
  20. package/modules/autonomy/runner-health.js +97 -0
  21. package/modules/dev-box/app/src/vendor/bongos-client.cjs +4 -0
  22. package/modules/hall-ui/public/gate.html +4 -1
  23. package/modules/hall-ui/public/gate.js +51 -2
  24. package/package-lock.json +2 -2
  25. package/package.json +1 -1
  26. package/scripts/gds/autobongos-run.js +368 -11
  27. package/scripts/gds/autobongos-service.cmd +38 -0
  28. package/scripts/gds/autobongos-service.ps1 +129 -0
  29. package/scripts/gds/deploy-main-audit-gate.sh +90 -0
  30. package/scripts/gds/main-audit.js +26 -4
  31. package/scripts/gds/run-unit-tests.js +16 -1
  32. package/src/module-api.js +1 -1
  33. package/tests/autobongos_cadence.mjs +602 -0
  34. package/tests/autonomy_runner_health.mjs +135 -0
  35. package/tests/bump_version.mjs +12 -1
  36. package/tests/cli_package.mjs +11 -1
  37. package/tests/deploy_audit_gate.mjs +154 -0
  38. package/tests/deploy_divergence_line.mjs +47 -1
  39. package/tests/helpers.mjs +61 -0
  40. package/tests/main_audit_db_roster.mjs +97 -0
  41. package/tests/publish_watch.mjs +9 -0
  42. package/tests/setup_device_flow_fallback.mjs +2 -1
@@ -254,7 +254,7 @@
254
254
  const el = document.getElementById('fence-state');
255
255
  el.innerHTML = `
256
256
  <div class="ov-foot" style="display:flex;align-items:center;gap:12px;flex-wrap:wrap;">
257
- <strong>${on ? 'Running' : 'Stopped'}</strong>
257
+ <span class="fact-pill ${on ? 'fact-pill--ok' : 'fact-pill--warn'}">${on ? 'Allowed to work' : 'Stopped'}</span>
258
258
  <span>${on ? 'The runner may claim and work allowlisted goals.' : escapeHtml(fence.paused_reason || 'The runner is doing nothing.')}</span>
259
259
  <button type="button" id="fence-toggle" class="btn">${on ? 'Stop the runner' : 'Start the runner'}</button>
260
260
  </div>`;
@@ -317,6 +317,54 @@
317
317
  }
318
318
  }
319
319
 
320
+ // The runner's liveness (task 1003905). Rendered ABOVE the switch on purpose:
321
+ // "is it running at all" is the first question, and an owner who has just
322
+ // flipped the switch on wants to see the machine notice.
323
+ //
324
+ // The verdict is the SERVER's — modules/autonomy/runner-health.js decides what
325
+ // dead means and the route attaches it — so this page and any future alert
326
+ // cannot drift into two definitions of the same word.
327
+ // This page's OWN class vocabulary (style.css `.fact-pill--{ok,warn,danger}`),
328
+ // the same one renderRow spends for a PR's verdict. The first cut reached for
329
+ // inline `--color-text-*` tokens from a different design system: they do not
330
+ // exist here, resolved to inherited grey, and drew a dead runner in exactly the
331
+ // same colour as a healthy one. Named classes cannot fail that way silently —
332
+ // a missing class is visible in the stylesheet, a missing var is not.
333
+ const HEALTH_CLASS = { alive: 'fact-pill--ok', late: 'fact-pill--warn', dead: 'fact-pill--danger', never_seen: 'fact-pill--info' };
334
+
335
+ async function loadRunners() {
336
+ const el = document.getElementById('fence-runner');
337
+ if (!el) return;
338
+ let resp;
339
+ try {
340
+ resp = await fetchJson('/autonomy/runners');
341
+ } catch (err) {
342
+ if (err && err.kind === '401') return;
343
+ el.innerHTML = emptyStateHtml('Could not read the runner\u2019s check-ins just now.');
344
+ return;
345
+ }
346
+ const runners = resp.runners || [];
347
+ if (!runners.length) {
348
+ // never_seen, and the server said so rather than this page guessing. Not an
349
+ // alarm: nobody has set the host up yet.
350
+ el.innerHTML = `<p class="ov-foot">${escapeHtml(resp.none ? resp.none.detail : 'No runner has ever checked in.')}</p>`;
351
+ return;
352
+ }
353
+ el.innerHTML = runners.map((r) => {
354
+ const h = r.health || {};
355
+ const cls = HEALTH_CLASS[h.state] || 'fact-pill--info';
356
+ // NOT the word "running": the switch below already uses it for a different
357
+ // thing (allowed to work), and the two lines sit one above the other. A
358
+ // reader should never have to work out which sense is meant.
359
+ const label = { dead: 'Not checking in', late: 'Quiet', alive: 'Checking in' }[h.state] || 'Unknown';
360
+ return `<p class="ov-foot" style="display:flex;align-items:baseline;gap:10px;flex-wrap:wrap;">
361
+ <span class="fact-pill ${cls}">${escapeHtml(label)}</span>
362
+ <span>${escapeHtml(h.detail || '')}</span>
363
+ <span class="ov-fact">last check-in ${escapeHtml(h.age || 'unknown')} ago</span>
364
+ </p>`;
365
+ }).join('');
366
+ }
367
+
320
368
  async function loadFence() {
321
369
  let fence;
322
370
  try {
@@ -356,7 +404,8 @@
356
404
  document.getElementById('gate-body').hidden = false;
357
405
  if (rank === 'archon') {
358
406
  document.getElementById('fence-panel').hidden = false;
359
- await loadFence();
407
+ // Independent reads, so one round trip rather than two.
408
+ await Promise.all([loadRunners(), loadFence()]);
360
409
  }
361
410
  await load();
362
411
  }
package/package-lock.json CHANGED
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "@bongos/core",
3
- "version": "1.19.876",
3
+ "version": "1.19.877",
4
4
  "lockfileVersion": 3,
5
5
  "requires": true,
6
6
  "packages": {
7
7
  "": {
8
8
  "name": "@bongos/core",
9
- "version": "1.19.876",
9
+ "version": "1.19.877",
10
10
  "license": "AGPL-3.0-or-later",
11
11
  "dependencies": {
12
12
  "express": "^4.21.2",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bongos/core",
3
- "version": "1.19.876",
3
+ "version": "1.19.877",
4
4
  "description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
5
5
  "license": "AGPL-3.0-or-later",
6
6
  "main": "src/platform-server.js",
@@ -6,8 +6,9 @@
6
6
  // one worker → record. No model anywhere in this file; every branch it takes is
7
7
  // a branch a person can read.
8
8
  //
9
- // node scripts/gds/autobongos-run.js # every allowlisted goal
10
- // node scripts/gds/autobongos-run.js --goal 1000119 # narrow to one of them
9
+ // node scripts/gds/autobongos-run.js --forever # the daemon shape
10
+ // node scripts/gds/autobongos-run.js # one task, then exit
11
+ // node scripts/gds/autobongos-run.js --goal 1000119 # narrow to one goal
11
12
  // node scripts/gds/autobongos-run.js --max-tasks 5
12
13
  // node scripts/gds/autobongos-run.js --dry-run # pick only, claim nothing
13
14
  //
@@ -22,13 +23,19 @@
22
23
  // 1003903), called at step 6. A worker's `shipped` is a CLAIM — recorded here
23
24
  // as `claims_shipped` so nothing can mistake it for a fact — and the ledger
24
25
  // read is what turns it into one, or into a released claim and a blocker.
25
- // · BACKOFF and never-terminally-stopping is task 1003904. This exits after
26
- // its budget; it does not yet decide when to come back.
26
+ // · IT NEVER TERMINALLY STOPS (task 1003904, ADR 0307 §5). `--forever` runs the
27
+ // cadence state machine in modules/autonomy/cadence.js: three straight
28
+ // failures drop to `probing` — no new work, a growing wait, then exactly one
29
+ // task — and one success returns it to full speed. Nothing exits the loop,
30
+ // not even the kill switch, because a machine nobody is watching must not be
31
+ // able to look idle while it is dead. On boot it reconciles every claim a
32
+ // previous run left behind.
27
33
 
28
34
  const { execFile, spawn } = require('node:child_process');
29
35
  const fs = require('node:fs');
30
36
  const path = require('node:path');
31
37
  const crypto = require('node:crypto');
38
+ const os = require('node:os');
32
39
 
33
40
  const seq = require('./sequence.js');
34
41
  const gauge = require('./autonomy-gauge.js');
@@ -42,11 +49,12 @@ const fence = require('../../modules/autonomy/fence.js');
42
49
  // where it would not be inside modules/ — which is exactly why fence.js takes the
43
50
  // hits as an argument instead of matching them itself.
44
51
  const { matchProtected } = require('../../src/bongos/permission-path-check');
52
+ const cadence = require('../../modules/autonomy/cadence.js');
45
53
 
46
54
  const REPO_ROOT = path.resolve(__dirname, '..', '..');
47
55
 
48
56
  function readArgv(argv) {
49
- const out = { goals: [], maxTasks: 1, dryRun: false, json: false, model: DEFAULT_MODEL, timeoutMs: DEFAULT_WORKER_TIMEOUT_MS };
57
+ const out = { goals: [], maxTasks: 1, dryRun: false, json: false, forever: false, maxLoops: Infinity, model: DEFAULT_MODEL, timeoutMs: DEFAULT_WORKER_TIMEOUT_MS };
50
58
  for (let i = 0; i < argv.length; i++) {
51
59
  const a = argv[i];
52
60
  if (a === '--goal') { out.goals.push(Number(argv[++i])); continue; }
@@ -56,6 +64,12 @@ function readArgv(argv) {
56
64
  if (a === '--timeout-min') { out.timeoutMs = Number(argv[++i]) * 60_000; continue; }
57
65
  if (a === '--dry-run') { out.dryRun = true; continue; }
58
66
  if (a === '--json') { out.json = true; continue; }
67
+ // --forever is the DAEMON shape: never returns, decides its own speed.
68
+ if (a === '--forever') { out.forever = true; continue; }
69
+ // --max-loops bounds the forever loop. It exists so a test (and an operator
70
+ // trying the thing out) can run the machine to completion; production passes
71
+ // it never, and Infinity is the honest default.
72
+ if (a === '--max-loops') { out.maxLoops = Number(argv[++i]); continue; }
59
73
  }
60
74
  if (!out.goals.length && process.env.AUTOBONGOS_GOALS) {
61
75
  out.goals = String(process.env.AUTOBONGOS_GOALS).split(',').map((g) => Number(g.trim())).filter(Boolean);
@@ -111,6 +125,12 @@ async function pickTask(goals, deps = {}) {
111
125
  }
112
126
  const claimableIds = await seq.fetchClaimableIds(api);
113
127
 
128
+ // The skipped lists from EVERY goal are kept, not just the last one. An empty
129
+ // queue and a queue where every task halts both yield no task, and telling them
130
+ // apart is the whole of `unworkable_queue` (task 1003904): calling the second
131
+ // one "idle" is how a goal full of blocked work sits for a week with the runner
132
+ // reporting itself perfectly healthy.
133
+ const allSkipped = [];
114
134
  for (const goalId of goals) {
115
135
  let byId;
116
136
  try { byId = await seq.fetchGoalTasks(api, goalId, seq.GOAL_PAGE); }
@@ -120,8 +140,9 @@ async function pickTask(goals, deps = {}) {
120
140
  const { order } = seq.topoOrder(candidates);
121
141
  const { next, skipped } = seq.pickNext(order, { rank, claimableIds });
122
142
  if (next) return { task: next, goalId, skipped };
143
+ for (const sk of skipped) allSkipped.push({ ...sk, goal_id: goalId });
123
144
  }
124
- return { none: true };
145
+ return { none: true, skipped: allSkipped };
125
146
  }
126
147
 
127
148
  // WORKER_ENV_ALLOW — what a worker is allowed to inherit (grader finding, task
@@ -277,7 +298,14 @@ async function iteration(opts, deps = {}) {
277
298
  // for. gate.goals is already the intersection.
278
299
  const picked = await pick(gate.goals, deps.pickDeps || {});
279
300
  if (picked.error) return log({ event: 'pick_failed', reason: picked.error });
280
- if (picked.none) return log({ event: 'nothing_claimable', goals: opts.goals });
301
+ if (picked.none) {
302
+ return log({
303
+ event: 'nothing_claimable', goals: gate.goals,
304
+ // Carried so the cadence can tell an EMPTY queue from one where every task
305
+ // halts — the reasons are the matrix's own codes, safe to print.
306
+ skipped: (picked.skipped || []).map((sk) => ({ id: sk.id, goal_id: sk.goal_id, reasons: (sk.reasons || []).map((r) => r.code) })),
307
+ });
308
+ }
281
309
  const task = picked.task;
282
310
 
283
311
  // 2b. The per-TASK half of the fence. sequence.js's halt matrix already refuses
@@ -364,19 +392,343 @@ async function iteration(opts, deps = {}) {
364
392
  return log(row);
365
393
  }
366
394
 
395
+ // ---------------------------------------------------------------------------
396
+ // Boot recovery, and the forever loop (task 1003904, ADR 0307 §5)
397
+ // ---------------------------------------------------------------------------
398
+
399
+ // reconcileLeftoverClaims — the FIRST thing a run does.
400
+ //
401
+ // "A silently locked task is the worst outcome this loop can produce" is the
402
+ // task's own wording, and cutting power mid-task is exactly how it happens: the
403
+ // claim is held by a session that no longer exists anywhere, and nothing times it
404
+ // out. So before taking any new work, every claim a previous run left behind is
405
+ // read back from the ledger and resolved.
406
+ //
407
+ // The reconciliation is autobongos-verify.js, unchanged (task 1003903). That file
408
+ // already answers "did this actually ship, and if not what do I do about it" for
409
+ // a worker that just exited; a worker that exited three days ago in a power cut
410
+ // is the same question with a longer gap. Writing a second, subtly different
411
+ // answer here is how two code paths end up disagreeing about what `confirmed`
412
+ // means.
413
+ //
414
+ // ONLY TREES THE RUNNER ITSELF MADE, matched against the EXACT shape
415
+ // worktreeName() produces — not a loose prefix.
416
+ //
417
+ // FOUND THE HARD WAY, twice. `autobongos-` as a prefix reads as "ours", and it is
418
+ // not: a person naming a worktree `autobongos-resume-1003904-d29f25` while
419
+ // working on this very feature matched it, and reconciliation released their live
420
+ // claim out from under them. A human at a keyboard picks readable names; the
421
+ // runner picks `autobongos-<task id>-<6 hex>` and nothing else. Matching the
422
+ // generator's own shape is the difference between "probably ours" and "ours".
423
+ //
424
+ // Derived FROM worktreeName so the two cannot drift: change the name format and
425
+ // this regex is what stops the next reconciliation from quietly matching nothing.
426
+ const CLAIM_PREFIX = 'autobongos-';
427
+ const RUNNER_TREE_RE = /^autobongos-\d+-[0-9a-f]{6}$/;
428
+ const isRunnerTree = (name) => RUNNER_TREE_RE.test(String(name || ''));
429
+
430
+ // ── the heartbeat, and why reconciliation needs one ──────────────────────────
431
+ //
432
+ // FOUND BY RUNNING IT. The first live run of reconcileLeftoverClaims released a
433
+ // claim that was very much alive — the one held by the session writing this code.
434
+ // It was right by its own rules (an `autobongos-` tree, a task sitting at
435
+ // `active`), and that is the problem: THE LEDGER CANNOT TELL A LIVE RUN FROM A
436
+ // DEAD ONE. Both look like a held claim. A reconciler that guesses wrong in one
437
+ // direction leaves a task locked forever; in the other it yanks a claim out from
438
+ // under a worker mid-build and strands whatever it had committed.
439
+ //
440
+ // So the runner says, on disk, that it is alive. A heartbeat file carries the pid
441
+ // and a timestamp; reconciliation refuses to touch anything while one is FRESH
442
+ // and its pid still exists. Both halves are needed — a stale file from a power
443
+ // cut must not veto recovery forever, and a fresh file whose process is gone is
444
+ // just litter. This is also what stops two runners on one machine from fighting
445
+ // over each other's claims.
446
+ const HEARTBEAT_STALE_MS = 10 * 60 * 1000;
447
+
448
+ function heartbeatPath() {
449
+ if (process.env.AUTOBONGOS_HEARTBEAT_FILE) return process.env.AUTOBONGOS_HEARTBEAT_FILE;
450
+ try { return require('../../src/instance-config.js').configPath('autobongos-heartbeat.json'); }
451
+ catch (_) { return path.join(REPO_ROOT, '.autobongos-heartbeat.json'); }
452
+ }
453
+
454
+ // Best-effort in both directions: a runner that cannot write its heartbeat must
455
+ // still run (it only loses the guard), and one that cannot read it must assume
456
+ // nothing is alive — the file's absence is the common case, on every first boot.
457
+ //
458
+ // DO NOT point AUTOBONGOS_HEARTBEAT_FILE UNDER /proc TO TEST THE FAILURE PATH (task
459
+ // 1004052). The mkdirSync below is recursive, and on Linux a recursive mkdir under
460
+ // /proc never returns — measured 10s+ with no result, where the same call throws
461
+ // EACCES instantly under /sys or /, and the NON-recursive form throws ENOENT instantly
462
+ // under /proc itself. macOS has no /proc, so it fails fast there. A test that used it
463
+ // was green on every developer machine and hung the Ubuntu CI runner for 90s with no
464
+ // output, reddening `unit` on main and blocking every PR. Use a regular FILE as the
465
+ // parent directory instead: mkdir under it is ENOTDIR immediately, everywhere, and it
466
+ // does not depend on the caller's permissions.
467
+ function writeHeartbeat(now = Date.now()) {
468
+ try {
469
+ const p = heartbeatPath();
470
+ fs.mkdirSync(path.dirname(p), { recursive: true });
471
+ fs.writeFileSync(p, JSON.stringify({ pid: process.pid, at: now }));
472
+ return true;
473
+ } catch (_) { return false; }
474
+ }
475
+
476
+ // postHeartbeat — tell the SERVER we are alive (task 1003905).
477
+ //
478
+ // The file above guards against two runners on this machine; this is the half an
479
+ // owner can see from a web page in another country. Both matter and neither
480
+ // replaces the other: a file is unreadable on exactly the day the machine is off.
481
+ //
482
+ // FAILS SILENTLY, ON PURPOSE. A runner that cannot check in must keep working —
483
+ // letting the watching mechanism take down the thing it watches is a strictly
484
+ // worse outcome than a gap in the log. The gap reports itself anyway: the hall
485
+ // reads the AGE of the last check-in, so a failed post shows up as silence, which
486
+ // is the signal.
487
+ const RUNNER_STARTED_AT = new Date().toISOString();
488
+
489
+ async function postHeartbeat(fields = {}, deps = {}) {
490
+ try {
491
+ const api = deps.api || await (require('./cli-lib').cliClient)();
492
+ await api.autonomy.postAutonomyHeartbeat({
493
+ body: {
494
+ host: os.hostname(),
495
+ pid: process.pid,
496
+ started_at: RUNNER_STARTED_AT,
497
+ ...fields,
498
+ },
499
+ });
500
+ return true;
501
+ } catch (_) { return false; }
502
+ }
503
+
504
+ function readHeartbeat() {
505
+ try { return JSON.parse(fs.readFileSync(heartbeatPath(), 'utf8')); }
506
+ catch (_) { return null; }
507
+ }
508
+
509
+ // pidAlive — `kill(pid, 0)` signals nothing and throws ESRCH when the process is
510
+ // gone. EPERM means it exists and belongs to someone else, which still counts as
511
+ // alive: the question is "is something running", not "is it mine".
512
+ function pidAlive(pid) {
513
+ if (!Number.isInteger(pid) || pid <= 0) return false;
514
+ try { process.kill(pid, 0); return true; }
515
+ catch (e) { return e && e.code === 'EPERM'; }
516
+ }
517
+
518
+ // anotherRunnerIsAlive — the veto. Fresh file AND a live pid AND not us.
519
+ function anotherRunnerIsAlive({ now = Date.now(), hb = readHeartbeat() } = {}) {
520
+ if (!hb || !Number.isFinite(Number(hb.at))) return false;
521
+ if (Number(hb.pid) === process.pid) return false; // our own beat
522
+ if (now - Number(hb.at) > HEARTBEAT_STALE_MS) return false; // a power cut leaves a stale file
523
+ return pidAlive(Number(hb.pid));
524
+ }
525
+
526
+ async function reconcileLeftoverClaims(deps = {}) {
527
+ const api = deps.api || await (require('./cli-lib').cliClient)();
528
+ const log = deps.record || record;
529
+ const verify = deps.verifyShip || verifyShip;
530
+ const runCmd = deps.run || run;
531
+ const isAlive = deps.anotherRunnerIsAlive || anotherRunnerIsAlive;
532
+
533
+ // Refuse outright while another runner is beating. Releasing a claim a live
534
+ // worker is building against strands whatever it has committed — the same harm
535
+ // as leaving one locked, pointed the other way.
536
+ if (isAlive()) return log({ event: 'boot_reconcile', ok: true, skipped: 'another runner is alive', leftover: null });
537
+
538
+ let claims = [];
539
+ try {
540
+ const me = await api.me.getMe();
541
+ if (!me || !me.ok) return log({ event: 'boot_reconcile', ok: false, reason: `GET /me → ${me ? me.status : 'no response'}` });
542
+ claims = (me.data.active_claims || []).filter((c) => isRunnerTree(c.worktree_name));
543
+ } catch (e) {
544
+ // Unreadable is NOT "nothing to reconcile". Say so and let the cadence treat
545
+ // it as a failure, rather than starting work on top of an unknown lock.
546
+ return log({ event: 'boot_reconcile', ok: false, reason: e && e.message ? e.message : String(e) });
547
+ }
548
+
549
+ if (!claims.length) return log({ event: 'boot_reconcile', ok: true, leftover: 0 });
550
+
551
+ const resolved = [];
552
+ for (const c of claims) {
553
+ const wtPath = path.join(REPO_ROOT, '.claude', 'worktrees', c.worktree_name);
554
+ const out = await verify(
555
+ { taskId: c.task_id, taskTitle: c.title, worker: { outcome: 'interrupted', reason: 'a previous run did not finish this claim' } },
556
+ {
557
+ api,
558
+ // The tree may be gone — a power cut leaves the claim, not necessarily the
559
+ // checkout. verifyShip only uses cwd for the artifact probe, which degrades
560
+ // to `checked: false` rather than failing, so a missing path is survivable.
561
+ cwd: fs.existsSync(wtPath) ? wtPath : REPO_ROOT,
562
+ release: (id, reason) => runCmd(process.execPath,
563
+ [path.join(REPO_ROOT, 'scripts', 'gds', 'release.js'), String(id), '--reason', reason],
564
+ { cwd: fs.existsSync(wtPath) ? wtPath : REPO_ROOT }),
565
+ },
566
+ );
567
+ resolved.push({ task_id: String(c.task_id), worktree: c.worktree_name, shape: out.shape, released: !!out.released, blocker_id: out.blocker?.blockerId ?? null });
568
+ }
569
+ return log({ event: 'boot_reconcile', ok: true, leftover: claims.length, resolved });
570
+ }
571
+
572
+ // pollSignals — the cheap facts whose change makes waiting pointless.
573
+ //
574
+ // Three reads, all small, all failing SILENTLY to null. A failed probe is not a
575
+ // signal: treating it as one would turn an unreachable instance into a busy loop
576
+ // hammering it, which is the exact opposite of backing off.
577
+ async function pollSignals(deps = {}) {
578
+ const api = deps.api || await (require('./cli-lib').cliClient)();
579
+ let head = null; let fenceUpdatedAt = null; let claimableCount = null;
580
+ try {
581
+ const r = await runFetchHead(deps);
582
+ if (r) head = r;
583
+ } catch (_) { /* null */ }
584
+ try {
585
+ const f = await api.autonomy.getAutonomyFence();
586
+ if (f && f.ok) fenceUpdatedAt = f.data.updated_at || null;
587
+ } catch (_) { /* null */ }
588
+ try {
589
+ const ids = await seq.fetchClaimableIds(api);
590
+ if (ids) claimableCount = ids.size;
591
+ } catch (_) { /* null */ }
592
+ return cadence.signalFingerprint({ head, fenceUpdatedAt, claimableCount });
593
+ }
594
+
595
+ // The head of origin/main, or null. `git ls-remote` rather than `fetch`: it moves
596
+ // no refs and writes nothing, so polling it every few minutes cannot interfere
597
+ // with a worker building in a tree beside it.
598
+ async function runFetchHead(deps = {}) {
599
+ const runCmd = deps.run || run;
600
+ const r = await runCmd('git', ['ls-remote', 'origin', 'refs/heads/main']);
601
+ if (!r.ok) return null;
602
+ const m = /^([0-9a-f]{40})/.exec(String(r.stdout).trim());
603
+ return m ? m[1] : null;
604
+ }
605
+
606
+ // waitOrJump — sleep, but wake early if the world moved.
607
+ //
608
+ // Polls at most every `pollEveryS`, so a thirty-minute backoff costs six cheap
609
+ // probes rather than a spin. `sleep` is injected so a test can drive an hour of
610
+ // wall clock in a millisecond.
611
+ async function waitOrJump(waitS, deps = {}, cfg = {}) {
612
+ if (!(waitS > 0)) return { waited: 0, jumped: false };
613
+ const pollEveryS = cfg.pollEveryS ?? 60;
614
+ const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
615
+ const poll = deps.pollSignals || pollSignals;
616
+
617
+ const before = await poll(deps);
618
+ let waited = 0;
619
+ while (waited < waitS) {
620
+ const chunk = Math.min(pollEveryS, waitS - waited);
621
+ await sleep(chunk * 1000);
622
+ waited += chunk;
623
+ // Beat DURING the wait. A thirty-minute backoff is the runner working exactly
624
+ // as designed, and it must not be indistinguishable from a machine that fell
625
+ // over at the start of it.
626
+ if (deps.beat) { try { await deps.beat({ last_event: 'waiting' }); } catch (_) { /* never break a wait */ } }
627
+ if (waited >= waitS) break;
628
+ const now = await poll(deps);
629
+ if (cadence.shouldJumpWait(before, now)) return { waited, jumped: true, signal: now };
630
+ }
631
+ return { waited, jumped: false };
632
+ }
633
+
634
+ // forever — the loop that has no terminal stop (task 1003904, ADR 0307 §5).
635
+ //
636
+ // It is a `while (true)` on purpose and the exceptions are the interesting part:
637
+ // NOTHING in here returns. Not an unreachable instance, not an expired token, not
638
+ // the kill switch. The only exits are a signal from the operating system and
639
+ // `--max-loops`, which exists so a test can run the machine to completion.
640
+ //
641
+ // What varies is the SPEED, and the speed is decided entirely by
642
+ // modules/autonomy/cadence.js from the row the last iteration recorded. This
643
+ // function holds no policy — it turns a `waitS` into a wait and a `takeWork` into
644
+ // an iteration, so the rules stay somewhere a test can reach without a clock.
645
+ async function forever(opts, deps = {}) {
646
+ const log = deps.record || record;
647
+ const iterate = deps.iteration || iteration;
648
+ const reconcile = deps.reconcileLeftoverClaims || reconcileLeftoverClaims;
649
+ const wait = deps.waitOrJump || waitOrJump;
650
+ const emit = deps.emit || ((row) => {
651
+ if (opts.json) console.log(JSON.stringify(row));
652
+ else console.log(`${row.event}${row.task_id ? ` task ${row.task_id}` : ''}${row.outcome ? ` → ${row.outcome}` : ''}${row.reason ? `: ${row.reason}` : ''}`);
653
+ });
654
+
655
+ // Claim the machine BEFORE reconciling: a second runner starting a moment later
656
+ // must see this one alive rather than deciding its claims are litter.
657
+ // Both halves of "I am alive": the local file (which stops a second runner
658
+ // treating this one's claims as litter) and the server row (which is the only
659
+ // one an owner can see from anywhere else).
660
+ const beatFile = deps.writeHeartbeat || writeHeartbeat;
661
+ const beatServer = deps.postHeartbeat || postHeartbeat;
662
+ // WRAPPED AT THE CALL SITE, not just inside postHeartbeat. The guarantee is
663
+ // "reporting that I am alive can never be what kills me", and that has to hold
664
+ // for whatever beat implementation is in play — a future one, an injected one —
665
+ // not only for the well-behaved default. Letting the watching mechanism take
666
+ // down the thing it watches is strictly worse than a gap in the log, and the gap
667
+ // reports itself anyway: the hall reads the AGE of the last check-in, so a
668
+ // failed beat shows up as silence, which is the signal.
669
+ const beat = async (fields) => {
670
+ try { beatFile(); } catch (_) { /* a runner that cannot write its file still runs */ }
671
+ try { await beatServer(fields, deps); } catch (_) { /* and one that cannot reach the instance still runs */ }
672
+ };
673
+ await beat({ last_event: 'boot' });
674
+
675
+ // BEFORE ANY WORK. A claim left locked by a previous run is the worst thing
676
+ // this loop produces, and taking new work on top of one compounds it.
677
+ emit(await reconcile(deps));
678
+
679
+ let state = { ...cadence.START };
680
+ const maxLoops = Number.isFinite(Number(opts.maxLoops)) ? Number(opts.maxLoops) : Infinity;
681
+
682
+ for (let loop = 0; loop < maxLoops; loop++) {
683
+ // Every loop, so a crash leaves a heartbeat that goes stale on its own — the
684
+ // runner never has to notice it is dying for its death to become visible.
685
+ await beat({ mode: state.mode, consecutive_failures: state.consecutiveFailures, last_event: 'loop' });
686
+ // In `probing` the runner takes exactly ONE task and only after its wait, so
687
+ // a broken window costs a handful of cheap requests rather than a night of
688
+ // failing spawns. `maxTasks` still bounds a full-speed burst.
689
+ const budget = state.mode === 'probing' ? 1 : opts.maxTasks;
690
+ let row = null;
691
+
692
+ for (let i = 0; i < budget; i++) {
693
+ row = await iterate(opts, deps);
694
+ emit(row);
695
+ // Carry the task id so the hall can tell "quiet because it is building
696
+ // task 1234" from "quiet because it is dead". A worker may hold the loop for
697
+ // ninety minutes, and without this every honest hour of work looks like a
698
+ // corpse.
699
+ await beat({ mode: state.mode, last_event: row.event, working_task_id: Number(row.task_id) || undefined });
700
+ const seen = cadence.classifyEvent(row);
701
+ state = cadence.nextCadence(state, seen, opts.cadence);
702
+ log({ event: 'cadence', mode: state.mode, consecutive_failures: state.consecutiveFailures, wait_s: state.waitS, class: seen.class, why: state.why });
703
+ // A burst continues only while work is actually landing. Anything else ends
704
+ // the burst and hands the decision to the cadence, which is the only thing
705
+ // allowed to say how long to wait.
706
+ if (seen.class !== 'progress') break;
707
+ }
708
+
709
+ const { waited, jumped } = await wait(state.waitS, { ...deps, beat }, opts.cadence);
710
+ if (jumped) log({ event: 'cadence', mode: state.mode, jumped: true, waited_s: waited, why: 'a new signal arrived — main moved, the fence changed, or the queue did' });
711
+ }
712
+ return 0;
713
+ }
714
+
367
715
  async function main(argv) {
368
716
  const opts = readArgv(argv);
369
717
  // No "name a goal" refusal here any more. It used to be the only allowlist, and
370
718
  // it lived on the machine being fenced; the database owns that now, and an empty
371
719
  // `--goal` means "every allowlisted goal" rather than "anything claimable". The
372
720
  // refusal did not disappear — it moved somewhere the runner host cannot edit.
721
+ if (opts.forever) return forever(opts);
722
+
723
+ // The one-shot path, kept for an operator running a single task by hand. It
724
+ // still reconciles first: a leftover claim is a leftover claim whoever is
725
+ // watching.
726
+ const boot = await reconcileLeftoverClaims();
727
+ if (opts.json) console.log(JSON.stringify(boot));
373
728
  for (let i = 0; i < opts.maxTasks; i++) {
374
729
  const row = await iteration(opts);
375
730
  if (opts.json) console.log(JSON.stringify(row));
376
731
  else console.log(`${row.event}${row.task_id ? ` task ${row.task_id}` : ''}${row.outcome ? ` → ${row.outcome}` : ''}${row.reason ? `: ${row.reason}` : ''}`);
377
- // Anything that is not a completed piece of work ends this run — including
378
- // `verify_halt`, which is the ledger being unreadable. Deciding when to come
379
- // back is task 1003904's job, not this one's.
380
732
  if (row.event !== 'worked') break;
381
733
  }
382
734
  return 0;
@@ -389,4 +741,9 @@ if (require.main === module) {
389
741
  });
390
742
  }
391
743
 
392
- module.exports = { readArgv, pickTask, iteration, record, logPath, spawnWorker, workerEnv, WORKER_ENV_ALLOW, worktreeName, readFence };
744
+ module.exports = {
745
+ readArgv, pickTask, iteration, record, logPath, spawnWorker, workerEnv, WORKER_ENV_ALLOW,
746
+ worktreeName, readFence, reconcileLeftoverClaims, pollSignals, waitOrJump, forever, CLAIM_PREFIX,
747
+ heartbeatPath, writeHeartbeat, readHeartbeat, pidAlive, anotherRunnerIsAlive, HEARTBEAT_STALE_MS,
748
+ isRunnerTree, RUNNER_TREE_RE, postHeartbeat, RUNNER_STARTED_AT,
749
+ };
@@ -0,0 +1,38 @@
1
+ @echo off
2
+ REM scripts/gds/autobongos-service.cmd -- what the scheduled task actually runs
3
+ REM (task 1003905). A .cmd, not a direct node invocation, for three reasons the
4
+ REM scheduler cannot do itself.
5
+ REM
6
+ REM 1. THE CHECKOUT IS BROUGHT CURRENT FIRST. A runner pinned to whatever was on
7
+ REM disk the day it was installed drifts silently, and the first symptom is
8
+ REM workers building against a stale fence. `git fetch` + `reset --hard` on the
9
+ REM MAIN checkout only -- worker trees are separate worktrees and are never
10
+ REM touched here.
11
+ REM 2. OUTPUT IS APPENDED TO ONE FILE the owner can read remotely. The scheduler
12
+ REM keeps only an exit code, which cannot answer "what was it doing".
13
+ REM 3. A NON-ZERO EXIT IS RETURNED so the scheduler's restart rule can see it.
14
+ REM
15
+ REM The runner itself never exits on failure (task 1003904) -- it backs off and
16
+ REM probes -- so reaching the end of this file at all means something the inner
17
+ REM loop could not survive.
18
+
19
+ setlocal
20
+ set "REPO=%~dp0..\.."
21
+ set "LOGDIR=%LOCALAPPDATA%\cloudbongos"
22
+ set "LOG=%LOGDIR%\autobongos.log"
23
+ if not exist "%LOGDIR%" mkdir "%LOGDIR%"
24
+
25
+ cd /d "%REPO%" || exit /b 1
26
+
27
+ echo. >> "%LOG%"
28
+ echo ==== autobongos starting %DATE% %TIME% ==== >> "%LOG%"
29
+
30
+ REM Best effort: a fetch that fails must not stop the runner from starting on the
31
+ REM code it already has. Being a commit behind beats not running at all.
32
+ git fetch --quiet origin main >> "%LOG%" 2>&1
33
+ git reset --quiet --hard origin/main >> "%LOG%" 2>&1
34
+
35
+ node scripts\gds\autobongos-run.js --forever --json >> "%LOG%" 2>&1
36
+ set EXITCODE=%ERRORLEVEL%
37
+ echo ==== autobongos exited %EXITCODE% at %DATE% %TIME% ==== >> "%LOG%"
38
+ exit /b %EXITCODE%