@bongos/core 1.19.876 → 1.19.878

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/.bongos-core.json +110 -40
  2. package/.gitattributes +6 -2
  3. package/clients/bongos-client/README.md +1 -1
  4. package/clients/bongos-client/bongos-client.global.js +4 -0
  5. package/clients/bongos-client/index.cjs +4 -0
  6. package/clients/bongos-client/index.d.ts +7 -0
  7. package/clients/bongos-client/index.mjs +4 -0
  8. package/docs/adr/0307-autobongos-a-deterministic-supervisor-and-agentic-workers.md +12 -1
  9. package/docs/api/openapi.json +149 -3
  10. package/docs/api-reference.md +4 -2
  11. package/docs/copy-inventory.md +7 -6
  12. package/docs/copy-registry.json +16 -7
  13. package/docs/module-api-changelog.md +4 -0
  14. package/docs/recipes/autobongos-windows-host.md +163 -0
  15. package/docs/recipes/instance-deploy.md +29 -3
  16. package/modules/autonomy/cadence.js +276 -0
  17. package/modules/autonomy/db.js +47 -0
  18. package/modules/autonomy/digest.js +344 -0
  19. package/modules/autonomy/migrations/autonomy_002_runner_heartbeat.sql +93 -0
  20. package/modules/autonomy/routes/autonomy.js +69 -0
  21. package/modules/autonomy/runner-health.js +97 -0
  22. package/modules/dev-box/app/src/vendor/bongos-client.cjs +4 -0
  23. package/modules/hall-ui/public/gate.html +4 -1
  24. package/modules/hall-ui/public/gate.js +51 -2
  25. package/package-lock.json +2 -2
  26. package/package.json +1 -1
  27. package/scripts/gds/autobongos-digest.js +123 -0
  28. package/scripts/gds/autobongos-run.js +375 -11
  29. package/scripts/gds/autobongos-service.cmd +38 -0
  30. package/scripts/gds/autobongos-service.ps1 +135 -0
  31. package/scripts/gds/deploy-main-audit-gate.sh +90 -0
  32. package/scripts/gds/main-audit.js +26 -4
  33. package/scripts/gds/provision-core-upgrade.js +23 -3
  34. package/scripts/gds/run-unit-tests.js +16 -1
  35. package/src/module-api.js +1 -1
  36. package/tests/autobongos_cadence.mjs +602 -0
  37. package/tests/autonomy_digest.mjs +302 -0
  38. package/tests/autonomy_runner_health.mjs +135 -0
  39. package/tests/bump_version.mjs +12 -1
  40. package/tests/cli_package.mjs +11 -1
  41. package/tests/core_upgrade_runner.mjs +43 -1
  42. package/tests/deploy_audit_gate.mjs +154 -0
  43. package/tests/deploy_divergence_line.mjs +47 -1
  44. package/tests/helpers.mjs +61 -0
  45. package/tests/main_audit_db_roster.mjs +97 -0
  46. package/tests/publish_watch.mjs +9 -0
  47. package/tests/setup_device_flow_fallback.mjs +2 -1
@@ -6,8 +6,9 @@
6
6
  // one worker → record. No model anywhere in this file; every branch it takes is
7
7
  // a branch a person can read.
8
8
  //
9
- // node scripts/gds/autobongos-run.js # every allowlisted goal
10
- // node scripts/gds/autobongos-run.js --goal 1000119 # narrow to one of them
9
+ // node scripts/gds/autobongos-run.js --forever # the daemon shape
10
+ // node scripts/gds/autobongos-run.js # one task, then exit
11
+ // node scripts/gds/autobongos-run.js --goal 1000119 # narrow to one goal
11
12
  // node scripts/gds/autobongos-run.js --max-tasks 5
12
13
  // node scripts/gds/autobongos-run.js --dry-run # pick only, claim nothing
13
14
  //
@@ -22,13 +23,19 @@
22
23
  // 1003903), called at step 6. A worker's `shipped` is a CLAIM — recorded here
23
24
  // as `claims_shipped` so nothing can mistake it for a fact — and the ledger
24
25
  // read is what turns it into one, or into a released claim and a blocker.
25
- // · BACKOFF and never-terminally-stopping is task 1003904. This exits after
26
- // its budget; it does not yet decide when to come back.
26
+ // · IT NEVER TERMINALLY STOPS (task 1003904, ADR 0307 §5). `--forever` runs the
27
+ // cadence state machine in modules/autonomy/cadence.js: three straight
28
+ // failures drop to `probing` — no new work, a growing wait, then exactly one
29
+ // task — and one success returns it to full speed. Nothing exits the loop,
30
+ // not even the kill switch, because a machine nobody is watching must not be
31
+ // able to look idle while it is dead. On boot it reconciles every claim a
32
+ // previous run left behind.
27
33
 
28
34
  const { execFile, spawn } = require('node:child_process');
29
35
  const fs = require('node:fs');
30
36
  const path = require('node:path');
31
37
  const crypto = require('node:crypto');
38
+ const os = require('node:os');
32
39
 
33
40
  const seq = require('./sequence.js');
34
41
  const gauge = require('./autonomy-gauge.js');
@@ -42,11 +49,12 @@ const fence = require('../../modules/autonomy/fence.js');
42
49
  // where it would not be inside modules/ — which is exactly why fence.js takes the
43
50
  // hits as an argument instead of matching them itself.
44
51
  const { matchProtected } = require('../../src/bongos/permission-path-check');
52
+ const cadence = require('../../modules/autonomy/cadence.js');
45
53
 
46
54
  const REPO_ROOT = path.resolve(__dirname, '..', '..');
47
55
 
48
56
  function readArgv(argv) {
49
- const out = { goals: [], maxTasks: 1, dryRun: false, json: false, model: DEFAULT_MODEL, timeoutMs: DEFAULT_WORKER_TIMEOUT_MS };
57
+ const out = { goals: [], maxTasks: 1, dryRun: false, json: false, forever: false, maxLoops: Infinity, model: DEFAULT_MODEL, timeoutMs: DEFAULT_WORKER_TIMEOUT_MS };
50
58
  for (let i = 0; i < argv.length; i++) {
51
59
  const a = argv[i];
52
60
  if (a === '--goal') { out.goals.push(Number(argv[++i])); continue; }
@@ -56,6 +64,12 @@ function readArgv(argv) {
56
64
  if (a === '--timeout-min') { out.timeoutMs = Number(argv[++i]) * 60_000; continue; }
57
65
  if (a === '--dry-run') { out.dryRun = true; continue; }
58
66
  if (a === '--json') { out.json = true; continue; }
67
+ // --forever is the DAEMON shape: never returns, decides its own speed.
68
+ if (a === '--forever') { out.forever = true; continue; }
69
+ // --max-loops bounds the forever loop. It exists so a test (and an operator
70
+ // trying the thing out) can run the machine to completion; production passes
71
+ // it never, and Infinity is the honest default.
72
+ if (a === '--max-loops') { out.maxLoops = Number(argv[++i]); continue; }
59
73
  }
60
74
  if (!out.goals.length && process.env.AUTOBONGOS_GOALS) {
61
75
  out.goals = String(process.env.AUTOBONGOS_GOALS).split(',').map((g) => Number(g.trim())).filter(Boolean);
@@ -111,6 +125,12 @@ async function pickTask(goals, deps = {}) {
111
125
  }
112
126
  const claimableIds = await seq.fetchClaimableIds(api);
113
127
 
128
+ // The skipped lists from EVERY goal are kept, not just the last one. An empty
129
+ // queue and a queue where every task halts both yield no task, and telling them
130
+ // apart is the whole of `unworkable_queue` (task 1003904): calling the second
131
+ // one "idle" is how a goal full of blocked work sits for a week with the runner
132
+ // reporting itself perfectly healthy.
133
+ const allSkipped = [];
114
134
  for (const goalId of goals) {
115
135
  let byId;
116
136
  try { byId = await seq.fetchGoalTasks(api, goalId, seq.GOAL_PAGE); }
@@ -120,8 +140,9 @@ async function pickTask(goals, deps = {}) {
120
140
  const { order } = seq.topoOrder(candidates);
121
141
  const { next, skipped } = seq.pickNext(order, { rank, claimableIds });
122
142
  if (next) return { task: next, goalId, skipped };
143
+ for (const sk of skipped) allSkipped.push({ ...sk, goal_id: goalId });
123
144
  }
124
- return { none: true };
145
+ return { none: true, skipped: allSkipped };
125
146
  }
126
147
 
127
148
  // WORKER_ENV_ALLOW — what a worker is allowed to inherit (grader finding, task
@@ -277,7 +298,14 @@ async function iteration(opts, deps = {}) {
277
298
  // for. gate.goals is already the intersection.
278
299
  const picked = await pick(gate.goals, deps.pickDeps || {});
279
300
  if (picked.error) return log({ event: 'pick_failed', reason: picked.error });
280
- if (picked.none) return log({ event: 'nothing_claimable', goals: opts.goals });
301
+ if (picked.none) {
302
+ return log({
303
+ event: 'nothing_claimable', goals: gate.goals,
304
+ // Carried so the cadence can tell an EMPTY queue from one where every task
305
+ // halts — the reasons are the matrix's own codes, safe to print.
306
+ skipped: (picked.skipped || []).map((sk) => ({ id: sk.id, goal_id: sk.goal_id, reasons: (sk.reasons || []).map((r) => r.code) })),
307
+ });
308
+ }
281
309
  const task = picked.task;
282
310
 
283
311
  // 2b. The per-TASK half of the fence. sequence.js's halt matrix already refuses
@@ -353,6 +381,13 @@ async function iteration(opts, deps = {}) {
353
381
  // What the LEDGER says, kept separate from what the worker said, so the run
354
382
  // log can be read afterwards without having to trust either one alone.
355
383
  ledger_status: ledger.status, ledger_shape: ledger.shape, verified: !!ledger.verified,
384
+ // The LEDGER's own explanation, kept beside the worker's. `reason` above is
385
+ // verdict.reason — what the WORKER said happened — and it was the only reason
386
+ // in the row, so anything reading the log afterwards was quoting the thing
387
+ // being checked rather than the check. The morning digest did exactly that
388
+ // until the grader caught it (task 1003906).
389
+ ledger_reason: ledger.reason || null,
390
+ ledger_remedy: ledger.remedy || null,
356
391
  artifact_on_main: ledger.artifact && ledger.artifact.checked ? !!ledger.artifact.onMain : null,
357
392
  claim_released: !!ledger.released,
358
393
  blocker_id: ledger.blocker && ledger.blocker.blockerId ? ledger.blocker.blockerId : null,
@@ -364,19 +399,343 @@ async function iteration(opts, deps = {}) {
364
399
  return log(row);
365
400
  }
366
401
 
402
+ // ---------------------------------------------------------------------------
403
+ // Boot recovery, and the forever loop (task 1003904, ADR 0307 §5)
404
+ // ---------------------------------------------------------------------------
405
+
406
+ // reconcileLeftoverClaims — the FIRST thing a run does.
407
+ //
408
+ // "A silently locked task is the worst outcome this loop can produce" is the
409
+ // task's own wording, and cutting power mid-task is exactly how it happens: the
410
+ // claim is held by a session that no longer exists anywhere, and nothing times it
411
+ // out. So before taking any new work, every claim a previous run left behind is
412
+ // read back from the ledger and resolved.
413
+ //
414
+ // The reconciliation is autobongos-verify.js, unchanged (task 1003903). That file
415
+ // already answers "did this actually ship, and if not what do I do about it" for
416
+ // a worker that just exited; a worker that exited three days ago in a power cut
417
+ // is the same question with a longer gap. Writing a second, subtly different
418
+ // answer here is how two code paths end up disagreeing about what `confirmed`
419
+ // means.
420
+ //
421
+ // ONLY TREES THE RUNNER ITSELF MADE, matched against the EXACT shape
422
+ // worktreeName() produces — not a loose prefix.
423
+ //
424
+ // FOUND THE HARD WAY, twice. `autobongos-` as a prefix reads as "ours", and it is
425
+ // not: a person naming a worktree `autobongos-resume-1003904-d29f25` while
426
+ // working on this very feature matched it, and reconciliation released their live
427
+ // claim out from under them. A human at a keyboard picks readable names; the
428
+ // runner picks `autobongos-<task id>-<6 hex>` and nothing else. Matching the
429
+ // generator's own shape is the difference between "probably ours" and "ours".
430
+ //
431
+ // Derived FROM worktreeName so the two cannot drift: change the name format and
432
+ // this regex is what stops the next reconciliation from quietly matching nothing.
433
+ const CLAIM_PREFIX = 'autobongos-';
434
+ const RUNNER_TREE_RE = /^autobongos-\d+-[0-9a-f]{6}$/;
435
+ const isRunnerTree = (name) => RUNNER_TREE_RE.test(String(name || ''));
436
+
437
+ // ── the heartbeat, and why reconciliation needs one ──────────────────────────
438
+ //
439
+ // FOUND BY RUNNING IT. The first live run of reconcileLeftoverClaims released a
440
+ // claim that was very much alive — the one held by the session writing this code.
441
+ // It was right by its own rules (an `autobongos-` tree, a task sitting at
442
+ // `active`), and that is the problem: THE LEDGER CANNOT TELL A LIVE RUN FROM A
443
+ // DEAD ONE. Both look like a held claim. A reconciler that guesses wrong in one
444
+ // direction leaves a task locked forever; in the other it yanks a claim out from
445
+ // under a worker mid-build and strands whatever it had committed.
446
+ //
447
+ // So the runner says, on disk, that it is alive. A heartbeat file carries the pid
448
+ // and a timestamp; reconciliation refuses to touch anything while one is FRESH
449
+ // and its pid still exists. Both halves are needed — a stale file from a power
450
+ // cut must not veto recovery forever, and a fresh file whose process is gone is
451
+ // just litter. This is also what stops two runners on one machine from fighting
452
+ // over each other's claims.
453
+ const HEARTBEAT_STALE_MS = 10 * 60 * 1000;
454
+
455
+ function heartbeatPath() {
456
+ if (process.env.AUTOBONGOS_HEARTBEAT_FILE) return process.env.AUTOBONGOS_HEARTBEAT_FILE;
457
+ try { return require('../../src/instance-config.js').configPath('autobongos-heartbeat.json'); }
458
+ catch (_) { return path.join(REPO_ROOT, '.autobongos-heartbeat.json'); }
459
+ }
460
+
461
+ // Best-effort in both directions: a runner that cannot write its heartbeat must
462
+ // still run (it only loses the guard), and one that cannot read it must assume
463
+ // nothing is alive — the file's absence is the common case, on every first boot.
464
+ //
465
+ // DO NOT point AUTOBONGOS_HEARTBEAT_FILE UNDER /proc TO TEST THE FAILURE PATH (task
466
+ // 1004052). The mkdirSync below is recursive, and on Linux a recursive mkdir under
467
+ // /proc never returns — measured 10s+ with no result, where the same call throws
468
+ // EACCES instantly under /sys or /, and the NON-recursive form throws ENOENT instantly
469
+ // under /proc itself. macOS has no /proc, so it fails fast there. A test that used it
470
+ // was green on every developer machine and hung the Ubuntu CI runner for 90s with no
471
+ // output, reddening `unit` on main and blocking every PR. Use a regular FILE as the
472
+ // parent directory instead: mkdir under it is ENOTDIR immediately, everywhere, and it
473
+ // does not depend on the caller's permissions.
474
+ function writeHeartbeat(now = Date.now()) {
475
+ try {
476
+ const p = heartbeatPath();
477
+ fs.mkdirSync(path.dirname(p), { recursive: true });
478
+ fs.writeFileSync(p, JSON.stringify({ pid: process.pid, at: now }));
479
+ return true;
480
+ } catch (_) { return false; }
481
+ }
482
+
483
+ // postHeartbeat — tell the SERVER we are alive (task 1003905).
484
+ //
485
+ // The file above guards against two runners on this machine; this is the half an
486
+ // owner can see from a web page in another country. Both matter and neither
487
+ // replaces the other: a file is unreadable on exactly the day the machine is off.
488
+ //
489
+ // FAILS SILENTLY, ON PURPOSE. A runner that cannot check in must keep working —
490
+ // letting the watching mechanism take down the thing it watches is a strictly
491
+ // worse outcome than a gap in the log. The gap reports itself anyway: the hall
492
+ // reads the AGE of the last check-in, so a failed post shows up as silence, which
493
+ // is the signal.
494
+ const RUNNER_STARTED_AT = new Date().toISOString();
495
+
496
+ async function postHeartbeat(fields = {}, deps = {}) {
497
+ try {
498
+ const api = deps.api || await (require('./cli-lib').cliClient)();
499
+ await api.autonomy.postAutonomyHeartbeat({
500
+ body: {
501
+ host: os.hostname(),
502
+ pid: process.pid,
503
+ started_at: RUNNER_STARTED_AT,
504
+ ...fields,
505
+ },
506
+ });
507
+ return true;
508
+ } catch (_) { return false; }
509
+ }
510
+
511
+ function readHeartbeat() {
512
+ try { return JSON.parse(fs.readFileSync(heartbeatPath(), 'utf8')); }
513
+ catch (_) { return null; }
514
+ }
515
+
516
+ // pidAlive — `kill(pid, 0)` signals nothing and throws ESRCH when the process is
517
+ // gone. EPERM means it exists and belongs to someone else, which still counts as
518
+ // alive: the question is "is something running", not "is it mine".
519
+ function pidAlive(pid) {
520
+ if (!Number.isInteger(pid) || pid <= 0) return false;
521
+ try { process.kill(pid, 0); return true; }
522
+ catch (e) { return e && e.code === 'EPERM'; }
523
+ }
524
+
525
+ // anotherRunnerIsAlive — the veto. Fresh file AND a live pid AND not us.
526
+ function anotherRunnerIsAlive({ now = Date.now(), hb = readHeartbeat() } = {}) {
527
+ if (!hb || !Number.isFinite(Number(hb.at))) return false;
528
+ if (Number(hb.pid) === process.pid) return false; // our own beat
529
+ if (now - Number(hb.at) > HEARTBEAT_STALE_MS) return false; // a power cut leaves a stale file
530
+ return pidAlive(Number(hb.pid));
531
+ }
532
+
533
+ async function reconcileLeftoverClaims(deps = {}) {
534
+ const api = deps.api || await (require('./cli-lib').cliClient)();
535
+ const log = deps.record || record;
536
+ const verify = deps.verifyShip || verifyShip;
537
+ const runCmd = deps.run || run;
538
+ const isAlive = deps.anotherRunnerIsAlive || anotherRunnerIsAlive;
539
+
540
+ // Refuse outright while another runner is beating. Releasing a claim a live
541
+ // worker is building against strands whatever it has committed — the same harm
542
+ // as leaving one locked, pointed the other way.
543
+ if (isAlive()) return log({ event: 'boot_reconcile', ok: true, skipped: 'another runner is alive', leftover: null });
544
+
545
+ let claims = [];
546
+ try {
547
+ const me = await api.me.getMe();
548
+ if (!me || !me.ok) return log({ event: 'boot_reconcile', ok: false, reason: `GET /me → ${me ? me.status : 'no response'}` });
549
+ claims = (me.data.active_claims || []).filter((c) => isRunnerTree(c.worktree_name));
550
+ } catch (e) {
551
+ // Unreadable is NOT "nothing to reconcile". Say so and let the cadence treat
552
+ // it as a failure, rather than starting work on top of an unknown lock.
553
+ return log({ event: 'boot_reconcile', ok: false, reason: e && e.message ? e.message : String(e) });
554
+ }
555
+
556
+ if (!claims.length) return log({ event: 'boot_reconcile', ok: true, leftover: 0 });
557
+
558
+ const resolved = [];
559
+ for (const c of claims) {
560
+ const wtPath = path.join(REPO_ROOT, '.claude', 'worktrees', c.worktree_name);
561
+ const out = await verify(
562
+ { taskId: c.task_id, taskTitle: c.title, worker: { outcome: 'interrupted', reason: 'a previous run did not finish this claim' } },
563
+ {
564
+ api,
565
+ // The tree may be gone — a power cut leaves the claim, not necessarily the
566
+ // checkout. verifyShip only uses cwd for the artifact probe, which degrades
567
+ // to `checked: false` rather than failing, so a missing path is survivable.
568
+ cwd: fs.existsSync(wtPath) ? wtPath : REPO_ROOT,
569
+ release: (id, reason) => runCmd(process.execPath,
570
+ [path.join(REPO_ROOT, 'scripts', 'gds', 'release.js'), String(id), '--reason', reason],
571
+ { cwd: fs.existsSync(wtPath) ? wtPath : REPO_ROOT }),
572
+ },
573
+ );
574
+ resolved.push({ task_id: String(c.task_id), worktree: c.worktree_name, shape: out.shape, released: !!out.released, blocker_id: out.blocker?.blockerId ?? null });
575
+ }
576
+ return log({ event: 'boot_reconcile', ok: true, leftover: claims.length, resolved });
577
+ }
578
+
579
+ // pollSignals — the cheap facts whose change makes waiting pointless.
580
+ //
581
+ // Three reads, all small, all failing SILENTLY to null. A failed probe is not a
582
+ // signal: treating it as one would turn an unreachable instance into a busy loop
583
+ // hammering it, which is the exact opposite of backing off.
584
+ async function pollSignals(deps = {}) {
585
+ const api = deps.api || await (require('./cli-lib').cliClient)();
586
+ let head = null; let fenceUpdatedAt = null; let claimableCount = null;
587
+ try {
588
+ const r = await runFetchHead(deps);
589
+ if (r) head = r;
590
+ } catch (_) { /* null */ }
591
+ try {
592
+ const f = await api.autonomy.getAutonomyFence();
593
+ if (f && f.ok) fenceUpdatedAt = f.data.updated_at || null;
594
+ } catch (_) { /* null */ }
595
+ try {
596
+ const ids = await seq.fetchClaimableIds(api);
597
+ if (ids) claimableCount = ids.size;
598
+ } catch (_) { /* null */ }
599
+ return cadence.signalFingerprint({ head, fenceUpdatedAt, claimableCount });
600
+ }
601
+
602
+ // The head of origin/main, or null. `git ls-remote` rather than `fetch`: it moves
603
+ // no refs and writes nothing, so polling it every few minutes cannot interfere
604
+ // with a worker building in a tree beside it.
605
+ async function runFetchHead(deps = {}) {
606
+ const runCmd = deps.run || run;
607
+ const r = await runCmd('git', ['ls-remote', 'origin', 'refs/heads/main']);
608
+ if (!r.ok) return null;
609
+ const m = /^([0-9a-f]{40})/.exec(String(r.stdout).trim());
610
+ return m ? m[1] : null;
611
+ }
612
+
613
+ // waitOrJump — sleep, but wake early if the world moved.
614
+ //
615
+ // Polls at most every `pollEveryS`, so a thirty-minute backoff costs six cheap
616
+ // probes rather than a spin. `sleep` is injected so a test can drive an hour of
617
+ // wall clock in a millisecond.
618
+ async function waitOrJump(waitS, deps = {}, cfg = {}) {
619
+ if (!(waitS > 0)) return { waited: 0, jumped: false };
620
+ const pollEveryS = cfg.pollEveryS ?? 60;
621
+ const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
622
+ const poll = deps.pollSignals || pollSignals;
623
+
624
+ const before = await poll(deps);
625
+ let waited = 0;
626
+ while (waited < waitS) {
627
+ const chunk = Math.min(pollEveryS, waitS - waited);
628
+ await sleep(chunk * 1000);
629
+ waited += chunk;
630
+ // Beat DURING the wait. A thirty-minute backoff is the runner working exactly
631
+ // as designed, and it must not be indistinguishable from a machine that fell
632
+ // over at the start of it.
633
+ if (deps.beat) { try { await deps.beat({ last_event: 'waiting' }); } catch (_) { /* never break a wait */ } }
634
+ if (waited >= waitS) break;
635
+ const now = await poll(deps);
636
+ if (cadence.shouldJumpWait(before, now)) return { waited, jumped: true, signal: now };
637
+ }
638
+ return { waited, jumped: false };
639
+ }
640
+
641
+ // forever — the loop that has no terminal stop (task 1003904, ADR 0307 §5).
642
+ //
643
+ // It is a `while (true)` on purpose and the exceptions are the interesting part:
644
+ // NOTHING in here returns. Not an unreachable instance, not an expired token, not
645
+ // the kill switch. The only exits are a signal from the operating system and
646
+ // `--max-loops`, which exists so a test can run the machine to completion.
647
+ //
648
+ // What varies is the SPEED, and the speed is decided entirely by
649
+ // modules/autonomy/cadence.js from the row the last iteration recorded. This
650
+ // function holds no policy — it turns a `waitS` into a wait and a `takeWork` into
651
+ // an iteration, so the rules stay somewhere a test can reach without a clock.
652
+ async function forever(opts, deps = {}) {
653
+ const log = deps.record || record;
654
+ const iterate = deps.iteration || iteration;
655
+ const reconcile = deps.reconcileLeftoverClaims || reconcileLeftoverClaims;
656
+ const wait = deps.waitOrJump || waitOrJump;
657
+ const emit = deps.emit || ((row) => {
658
+ if (opts.json) console.log(JSON.stringify(row));
659
+ else console.log(`${row.event}${row.task_id ? ` task ${row.task_id}` : ''}${row.outcome ? ` → ${row.outcome}` : ''}${row.reason ? `: ${row.reason}` : ''}`);
660
+ });
661
+
662
+ // Claim the machine BEFORE reconciling: a second runner starting a moment later
663
+ // must see this one alive rather than deciding its claims are litter.
664
+ // Both halves of "I am alive": the local file (which stops a second runner
665
+ // treating this one's claims as litter) and the server row (which is the only
666
+ // one an owner can see from anywhere else).
667
+ const beatFile = deps.writeHeartbeat || writeHeartbeat;
668
+ const beatServer = deps.postHeartbeat || postHeartbeat;
669
+ // WRAPPED AT THE CALL SITE, not just inside postHeartbeat. The guarantee is
670
+ // "reporting that I am alive can never be what kills me", and that has to hold
671
+ // for whatever beat implementation is in play — a future one, an injected one —
672
+ // not only for the well-behaved default. Letting the watching mechanism take
673
+ // down the thing it watches is strictly worse than a gap in the log, and the gap
674
+ // reports itself anyway: the hall reads the AGE of the last check-in, so a
675
+ // failed beat shows up as silence, which is the signal.
676
+ const beat = async (fields) => {
677
+ try { beatFile(); } catch (_) { /* a runner that cannot write its file still runs */ }
678
+ try { await beatServer(fields, deps); } catch (_) { /* and one that cannot reach the instance still runs */ }
679
+ };
680
+ await beat({ last_event: 'boot' });
681
+
682
+ // BEFORE ANY WORK. A claim left locked by a previous run is the worst thing
683
+ // this loop produces, and taking new work on top of one compounds it.
684
+ emit(await reconcile(deps));
685
+
686
+ let state = { ...cadence.START };
687
+ const maxLoops = Number.isFinite(Number(opts.maxLoops)) ? Number(opts.maxLoops) : Infinity;
688
+
689
+ for (let loop = 0; loop < maxLoops; loop++) {
690
+ // Every loop, so a crash leaves a heartbeat that goes stale on its own — the
691
+ // runner never has to notice it is dying for its death to become visible.
692
+ await beat({ mode: state.mode, consecutive_failures: state.consecutiveFailures, last_event: 'loop' });
693
+ // In `probing` the runner takes exactly ONE task and only after its wait, so
694
+ // a broken window costs a handful of cheap requests rather than a night of
695
+ // failing spawns. `maxTasks` still bounds a full-speed burst.
696
+ const budget = state.mode === 'probing' ? 1 : opts.maxTasks;
697
+ let row = null;
698
+
699
+ for (let i = 0; i < budget; i++) {
700
+ row = await iterate(opts, deps);
701
+ emit(row);
702
+ // Carry the task id so the hall can tell "quiet because it is building
703
+ // task 1234" from "quiet because it is dead". A worker may hold the loop for
704
+ // ninety minutes, and without this every honest hour of work looks like a
705
+ // corpse.
706
+ await beat({ mode: state.mode, last_event: row.event, working_task_id: Number(row.task_id) || undefined });
707
+ const seen = cadence.classifyEvent(row);
708
+ state = cadence.nextCadence(state, seen, opts.cadence);
709
+ log({ event: 'cadence', mode: state.mode, consecutive_failures: state.consecutiveFailures, wait_s: state.waitS, class: seen.class, why: state.why });
710
+ // A burst continues only while work is actually landing. Anything else ends
711
+ // the burst and hands the decision to the cadence, which is the only thing
712
+ // allowed to say how long to wait.
713
+ if (seen.class !== 'progress') break;
714
+ }
715
+
716
+ const { waited, jumped } = await wait(state.waitS, { ...deps, beat }, opts.cadence);
717
+ if (jumped) log({ event: 'cadence', mode: state.mode, jumped: true, waited_s: waited, why: 'a new signal arrived — main moved, the fence changed, or the queue did' });
718
+ }
719
+ return 0;
720
+ }
721
+
367
722
  async function main(argv) {
368
723
  const opts = readArgv(argv);
369
724
  // No "name a goal" refusal here any more. It used to be the only allowlist, and
370
725
  // it lived on the machine being fenced; the database owns that now, and an empty
371
726
  // `--goal` means "every allowlisted goal" rather than "anything claimable". The
372
727
  // refusal did not disappear — it moved somewhere the runner host cannot edit.
728
+ if (opts.forever) return forever(opts);
729
+
730
+ // The one-shot path, kept for an operator running a single task by hand. It
731
+ // still reconciles first: a leftover claim is a leftover claim whoever is
732
+ // watching.
733
+ const boot = await reconcileLeftoverClaims();
734
+ if (opts.json) console.log(JSON.stringify(boot));
373
735
  for (let i = 0; i < opts.maxTasks; i++) {
374
736
  const row = await iteration(opts);
375
737
  if (opts.json) console.log(JSON.stringify(row));
376
738
  else console.log(`${row.event}${row.task_id ? ` task ${row.task_id}` : ''}${row.outcome ? ` → ${row.outcome}` : ''}${row.reason ? `: ${row.reason}` : ''}`);
377
- // Anything that is not a completed piece of work ends this run — including
378
- // `verify_halt`, which is the ledger being unreadable. Deciding when to come
379
- // back is task 1003904's job, not this one's.
380
739
  if (row.event !== 'worked') break;
381
740
  }
382
741
  return 0;
@@ -389,4 +748,9 @@ if (require.main === module) {
389
748
  });
390
749
  }
391
750
 
392
- module.exports = { readArgv, pickTask, iteration, record, logPath, spawnWorker, workerEnv, WORKER_ENV_ALLOW, worktreeName, readFence };
751
+ module.exports = {
752
+ readArgv, pickTask, iteration, record, logPath, spawnWorker, workerEnv, WORKER_ENV_ALLOW,
753
+ worktreeName, readFence, reconcileLeftoverClaims, pollSignals, waitOrJump, forever, CLAIM_PREFIX,
754
+ heartbeatPath, writeHeartbeat, readHeartbeat, pidAlive, anotherRunnerIsAlive, HEARTBEAT_STALE_MS,
755
+ isRunnerTree, RUNNER_TREE_RE, postHeartbeat, RUNNER_STARTED_AT,
756
+ };
@@ -0,0 +1,38 @@
1
+ @echo off
2
+ REM scripts/gds/autobongos-service.cmd -- what the scheduled task actually runs
3
+ REM (task 1003905). A .cmd, not a direct node invocation, for three reasons the
4
+ REM scheduler cannot do itself.
5
+ REM
6
+ REM 1. THE CHECKOUT IS BROUGHT CURRENT FIRST. A runner pinned to whatever was on
7
+ REM disk the day it was installed drifts silently, and the first symptom is
8
+ REM workers building against a stale fence. `git fetch` + `reset --hard` on the
9
+ REM MAIN checkout only -- worker trees are separate worktrees and are never
10
+ REM touched here.
11
+ REM 2. OUTPUT IS APPENDED TO ONE FILE the owner can read remotely. The scheduler
12
+ REM keeps only an exit code, which cannot answer "what was it doing".
13
+ REM 3. A NON-ZERO EXIT IS RETURNED so the scheduler's restart rule can see it.
14
+ REM
15
+ REM The runner itself never exits on failure (task 1003904) -- it backs off and
16
+ REM probes -- so reaching the end of this file at all means something the inner
17
+ REM loop could not survive.
18
+
19
+ setlocal
20
+ set "REPO=%~dp0..\.."
21
+ set "LOGDIR=%LOCALAPPDATA%\cloudbongos"
22
+ set "LOG=%LOGDIR%\autobongos.log"
23
+ if not exist "%LOGDIR%" mkdir "%LOGDIR%"
24
+
25
+ cd /d "%REPO%" || exit /b 1
26
+
27
+ echo. >> "%LOG%"
28
+ echo ==== autobongos starting %DATE% %TIME% ==== >> "%LOG%"
29
+
30
+ REM Best effort: a fetch that fails must not stop the runner from starting on the
31
+ REM code it already has. Being a commit behind beats not running at all.
32
+ git fetch --quiet origin main >> "%LOG%" 2>&1
33
+ git reset --quiet --hard origin/main >> "%LOG%" 2>&1
34
+
35
+ node scripts\gds\autobongos-run.js --forever --json >> "%LOG%" 2>&1
36
+ set EXITCODE=%ERRORLEVEL%
37
+ echo ==== autobongos exited %EXITCODE% at %DATE% %TIME% ==== >> "%LOG%"
38
+ exit /b %EXITCODE%
@@ -0,0 +1,135 @@
1
+ # scripts/gds/autobongos-service.ps1 -- install the Autobongos runner so it
2
+ # outlives the reboot (task 1003905, ADR 0307 section 5).
3
+ #
4
+ # powershell -ExecutionPolicy Bypass -File scripts\gds\autobongos-service.ps1 install
5
+ # powershell -ExecutionPolicy Bypass -File scripts\gds\autobongos-service.ps1 status
6
+ # powershell -ExecutionPolicy Bypass -File scripts\gds\autobongos-service.ps1 digest
7
+ # powershell -ExecutionPolicy Bypass -File scripts\gds\autobongos-service.ps1 start|stop|uninstall
8
+ #
9
+ # WHY TASK SCHEDULER AND NOT A REAL SERVICE. Windows services speak a protocol
10
+ # (SCM handshakes, stop signals) that node does not, so every "node as a service"
11
+ # route needs a third-party wrapper: nssm, winsw, node-windows. Each is a binary
12
+ # to download, verify, update and explain -- a permanent dependency bought for one
13
+ # machine. Task Scheduler is already on the box, runs at boot with no session,
14
+ # survives sign-out, and restarts a crashed task natively. The one thing it does
15
+ # not give is the Services panel, which nobody here needs.
16
+ #
17
+ # WHAT ACTUALLY KEEPS IT ALIVE IS THE INNER LOOP, and that is the important part
18
+ # to understand before reading the RestartCount settings below. `--forever` never
19
+ # returns: it backs off and probes instead of exiting (task 1003904), so the
20
+ # scheduler's restart is the OUTER ring, for the cases the loop cannot survive at
21
+ # all -- a reboot, a power cut, a node crash, an OOM kill. If this ring is doing
22
+ # work regularly, something is wrong that the backoff should have absorbed, and
23
+ # the run log will say so.
24
+ #
25
+ # UNPRIVILEGED ON PURPOSE. It runs as the logged-in user, not SYSTEM: the runner
26
+ # needs that user's git credential, npm config and Claude session, and SYSTEM has
27
+ # none of them. It is also the smaller blast radius for a process that spawns
28
+ # workers with blanket bypassPermissions.
29
+
30
+ [CmdletBinding()]
31
+ param(
32
+ [Parameter(Position = 0)]
33
+ [ValidateSet('install', 'uninstall', 'start', 'stop', 'status', 'digest')]
34
+ [string]$Action = 'status',
35
+
36
+ # The checkout to run from. Defaults to the repo this script lives in, so the
37
+ # common case needs no argument and cannot point at the wrong tree by typo.
38
+ [string]$RepoRoot,
39
+
40
+ [string]$TaskName = 'Autobongos'
41
+ )
42
+
43
+ $ErrorActionPreference = 'Stop'
44
+
45
+ if (-not $RepoRoot) {
46
+ $RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path
47
+ }
48
+ $LogDir = Join-Path $env:LOCALAPPDATA 'cloudbongos'
49
+ $LogFile = Join-Path $LogDir 'autobongos.log'
50
+ $Wrapper = Join-Path $RepoRoot 'scripts\gds\autobongos-service.cmd'
51
+
52
+ function Assert-Prereqs {
53
+ if (-not (Test-Path (Join-Path $RepoRoot 'scripts\gds\autobongos-run.js'))) {
54
+ throw "No autobongos-run.js under $RepoRoot -- pass -RepoRoot <path to the checkout>."
55
+ }
56
+ foreach ($exe in @('node', 'git', 'claude')) {
57
+ if (-not (Get-Command $exe -ErrorAction SilentlyContinue)) {
58
+ # Named, not guessed at: a runner that starts without `claude` on PATH will
59
+ # spawn-fail every worker and look like a mysterious outage in the log.
60
+ throw "'$exe' is not on PATH for this user. The runner needs node, git and claude."
61
+ }
62
+ }
63
+ }
64
+
65
+ function Show-Status {
66
+ $t = Get-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue
67
+ if (-not $t) { Write-Host "not installed (task '$TaskName' does not exist)"; return }
68
+ $info = Get-ScheduledTaskInfo -TaskName $TaskName
69
+ Write-Host "task : $TaskName"
70
+ Write-Host "state : $($t.State)"
71
+ Write-Host "last run : $($info.LastRunTime) (result $($info.LastTaskResult))"
72
+ Write-Host "next run : $($info.NextRunTime)"
73
+ Write-Host "log : $LogFile"
74
+ if (Test-Path $LogFile) {
75
+ Write-Host "log size : $([math]::Round((Get-Item $LogFile).Length / 1KB)) KB"
76
+ Write-Host ''
77
+ Write-Host '--- last 15 lines ---'
78
+ Get-Content $LogFile -Tail 15
79
+ } else {
80
+ Write-Host 'log : (not written yet)'
81
+ }
82
+ Write-Host ''
83
+ Write-Host 'The scheduler only says whether the PROCESS is up. Whether the runner'
84
+ Write-Host 'is actually working is the hall: it reads the heartbeat this posts.'
85
+ }
86
+
87
+ switch ($Action) {
88
+ 'install' {
89
+ Assert-Prereqs
90
+ New-Item -ItemType Directory -Force -Path $LogDir | Out-Null
91
+
92
+ $action = New-ScheduledTaskAction -Execute $Wrapper -WorkingDirectory $RepoRoot
93
+ # AtStartup covers the cold reboot the done-when actually names. AtLogOn is
94
+ # NOT used: it would wait for someone to sign in, which is the one thing that
95
+ # must not be required.
96
+ $trigger = New-ScheduledTaskTrigger -AtStartup
97
+ $principal = New-ScheduledTaskPrincipal -UserId "$env:USERDOMAIN\$env:USERNAME" -LogonType S4U -RunLevel Limited
98
+ $settings = New-ScheduledTaskSettingsSet `
99
+ -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries `
100
+ -StartWhenAvailable `
101
+ -RestartCount 999 -RestartInterval (New-TimeSpan -Minutes 1) `
102
+ -ExecutionTimeLimit ([TimeSpan]::Zero) `
103
+ -MultipleInstances IgnoreNew
104
+ # ExecutionTimeLimit ZERO = never kill it. The default is three days, and a
105
+ # daemon silently executed after three days is exactly the invisible death
106
+ # this whole task exists to prevent.
107
+ # MultipleInstances IgnoreNew = the scheduler will not start a second copy
108
+ # over a live one; the heartbeat file is the second guard, inside the runner.
109
+
110
+ Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger `
111
+ -Principal $principal -Settings $settings -Force | Out-Null
112
+
113
+ Write-Host "installed '$TaskName' -- starts at boot, restarts on crash, logs to $LogFile"
114
+ Write-Host ''
115
+ Write-Host 'Start it now with: .\scripts\gds\autobongos-service.ps1 start'
116
+ Write-Host 'Then confirm from the hall (/builders/gate) that it is checking in.'
117
+ }
118
+ 'uninstall' {
119
+ Unregister-ScheduledTask -TaskName $TaskName -Confirm:$false -ErrorAction SilentlyContinue
120
+ Write-Host "removed '$TaskName'. The log at $LogFile is left in place."
121
+ }
122
+ 'digest' {
123
+ # The night in one page (task 1003906). On the box for convenience; the same
124
+ # five answers are what the Gate page and a mailed digest would carry.
125
+ & node (Join-Path $RepoRoot 'scripts\gds\autobongos-digest.js') @args
126
+ }
127
+ 'start' { Start-ScheduledTask -TaskName $TaskName; Write-Host "started '$TaskName'" }
128
+ 'stop' {
129
+ Stop-ScheduledTask -TaskName $TaskName
130
+ Write-Host "stopped '$TaskName' -- but note this only stops the PROCESS on this machine."
131
+ Write-Host 'The durable way to stop the runner is the kill switch in the hall, which'
132
+ Write-Host 'survives a reboot and cannot be undone by anything running here.'
133
+ }
134
+ default { Show-Status }
135
+ }