@bongos/core 1.19.876 → 1.19.878

package/tests/autonomy_digest.mjs

@@ -0,0 +1,302 @@
+// tests/autonomy_digest.mjs — the night, readable afterwards (task 1003906).
+//
+// The done-when is "a night produces one digest answering all five questions
+// without anyone opening a log". So the spine of this file is one test that
+// feeds a realistic night through and asserts all five answers are present and
+// correct — because five separately-passing sections can still add up to a page
+// that does not answer the question.
+
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const { buildDigest, renderDigest, windowsFromGauge, humanDuration } = require('../modules/autonomy/digest.js');
+const cli = require('../scripts/gds/autobongos-digest.js');
+
+const T0 = Date.parse('2026-09-19T02:00:00.000Z');
+const at = (m) => new Date(T0 + m * 60000).toISOString();
+const NOW_S = Math.floor((T0 + 8 * 3600e3) / 1000);
+
+const night = () => [
+  { at: at(0), event: 'boot_reconcile', ok: true, leftover: 0 },
+  { at: at(1), event: 'worked', task_id: 111, title: 'A shipped thing', goal_id: 1000119,
+    duration_s: 2820, cost_usd: 3.41, verified: true, ledger_shape: 'shipped',
+    undetermined_decisions: ['Cached for 60s rather than 5m'] },
+  { at: at(50), event: 'worked', task_id: 222, title: 'A stranded thing', goal_id: 1000119,
+    duration_s: 5400, cost_usd: 6.8, verified: false, ledger_shape: 'unlanded',
+    reason: 'the grade passed and the land never finished', blocker_id: '1000161',
+    undetermined_decisions: ['Sorted inactive last', 'Kept the id tie-break', 'Chose ascending'] },
+  { at: at(120), event: 'fenced', code: 'grader_unavailable', reason: 'the grader is bypassed' },
+  { at: at(121), event: 'fenced', code: 'grader_unavailable', reason: 'the grader is bypassed' },
+  { at: at(122), event: 'cadence', mode: 'probing', class: 'transient', consecutive_failures: 3, wait_s: 60 },
+  { at: at(200), event: 'nothing_claimable', goals: [1000119],
+    skipped: [{ id: 333, goal_id: 1000119, reasons: ['needs_migration'] }] },
+  { at: at(201), event: 'hold', reason: 'seven-day pace: 78% used' },
+];
+const gauge = {
+  five_hour: { used_percentage: 41, resets_at: NOW_S + 3600 },
+  seven_day: { used_percentage: 78, resets_at: NOW_S + 240000 },
+};
+
+// ── the done-when, in one test ─────────────────────────────────────────────────
+
+test('one night produces one page answering all five questions', () => {
+  const text = renderDigest(buildDigest(night(), { gauge, nowEpochS: NOW_S }));
+
+  // 1. what shipped — and ONLY what the ledger agreed shipped.
+  assert.match(text, /SHIPPED — 1 task/);
+  assert.match(text, /111\s+A shipped thing/);
+  assert.doesNotMatch(text.split('ON HOLD')[0], /222/, 'a task that ran but did not ship is not "shipped"');
+
+  // 2. what is on hold, and why.
+  assert.match(text, /ON HOLD/);
+  assert.match(text, /222\s+ran but did not ship — unlanded/);
+  assert.match(text, /blocker 1000161/);
+  assert.match(text, /fence: grader_unavailable ×2/);
+  assert.match(text, /every one halted/);
+  assert.match(text, /capacity: held 1 time/);
+
+  // 3. what each worker decided that its spec did not determine.
+  assert.match(text, /DECIDED FOR YOU — 4 decisions across 2 tasks/);
+  assert.match(text, /Cached for 60s rather than 5m/);
+  assert.match(text, /sentence the task description should have contained/);
+  assert.match(text, /Thinnest spec: 222/);
+
+  // 4. what it cost.
+  assert.match(text, /COST/);
+  assert.match(text, /\$10\.21 across 2 worker runs/);
+
+  // 5. where both limit windows stand.
+  assert.match(text, /Five-hour window\s+41% used, resets in 60m/);
+  assert.match(text, /Seven-day window\s+78% used/);
+});
+
+test('every section prints even when the night was empty', () => {
+  // A digest whose quiet sections vanish teaches its reader that absence means
+  // "fine" — and then the morning a section is missing because the DATA is
+  // missing reads exactly the same.
+  const text = renderDigest(buildDigest([], { gauge, nowEpochS: NOW_S }));
+  for (const heading of ['SHIPPED', 'ON HOLD', 'DECIDED FOR YOU', 'COST', 'LIMITS', 'RUNNER']) {
+    assert.ok(text.includes(heading), `${heading} must print on an empty night`);
+  }
+  assert.match(text, /nothing shipped/);
+  assert.match(text, /nothing is waiting on you/);
+  assert.match(text, /no worker had to fill a gap/);
+});
+
+// ── the answers, individually ──────────────────────────────────────────────────
+
+test('shipped is the LEDGER’s answer, never the worker’s', () => {
+  const d = buildDigest([
+    { at: at(0), event: 'worked', task_id: 1, verified: true, ledger_shape: 'shipped' },
+    { at: at(1), event: 'worked', task_id: 2, verified: false, ledger_shape: 'still_held' },
+    { at: at(2), event: 'worked', task_id: 3, outcome: 'claims_shipped', ledger_shape: 'shipped_no_artifact', verified: false },
+  ], { nowEpochS: NOW_S });
+  assert.deepEqual(d.shipped.map((s) => s.taskId), ['1']);
+  assert.deepEqual(d.onHold.ranButDidNotShip.map((r) => r.shape), ['still_held', 'shipped_no_artifact']);
+});
+
+test('an unworkable queue is reported as a queue problem, not as breakage', () => {
+  const d = buildDigest([{ at: at(0), event: 'nothing_claimable', skipped: [{ id: 9, reasons: ['needs_migration'] }] }], { nowEpochS: NOW_S });
+  assert.equal(d.onHold.unworkable.length, 1);
+  const text = renderDigest(d);
+  assert.match(text, /this is not a runner failure/);
+});
+
+test('the decisions list groups by task and names the thinnest spec', () => {
+  // The SHAPE is the signal: one task with six decisions is a thin spec, six
+  // tasks with one each is a normal night.
+  const d = buildDigest([
+    { at: at(0), event: 'worked', task_id: 1, verified: true, undetermined_decisions: ['a'] },
+    { at: at(1), event: 'worked', task_id: 2, verified: true, undetermined_decisions: ['b', 'c', 'd'] },
+    { at: at(2), event: 'worked', task_id: 3, verified: true, undetermined_decisions: [] },
+  ], { nowEpochS: NOW_S });
+  assert.equal(d.decisionCount, 4);
+  assert.equal(d.decisions.length, 2, 'a task with no decisions is not listed');
+  assert.equal(d.thinnestSpec.taskId, '2');
+});
+
+test('a decision from a task that did NOT ship is still surfaced', () => {
+  // These are the ones that matter most: the judgement was made, the work is not
+  // in, and the call is sitting in a branch nobody has read.
+  const text = renderDigest(buildDigest([
+    { at: at(0), event: 'worked', task_id: 7, title: 'T', verified: false, ledger_shape: 'unlanded', undetermined_decisions: ['picked a default'] },
+  ], { nowEpochS: NOW_S }));
+  assert.match(text, /picked a default/);
+  assert.match(text, /\[did not ship\]/);
+});
+
+test('an unreported cost is excluded and counted, never coerced to zero', () => {
+  const d = buildDigest([
+    { at: at(0), event: 'worked', task_id: 1, verified: true, cost_usd: 2.5 },
+    { at: at(1), event: 'worked', task_id: 2, verified: true, cost_usd: null },
+    { at: at(2), event: 'worked', task_id: 3, verified: true },
+  ], { nowEpochS: NOW_S });
+  assert.equal(d.cost.totalUsd, 2.5, 'a worker whose cost was not reported must not make the night look cheaper');
+  assert.equal(d.cost.reported, 1);
+  assert.equal(d.cost.unreported, 2);
+  assert.match(renderDigest(d), /2 did not report a cost/);
+});
+
+// ── the limit windows ──────────────────────────────────────────────────────────
+
+test('a window whose reset has PASSED is reported as stale, not as headroom', () => {
+  // "41% used, resets in 0s" invites a decision about headroom that no longer
+  // exists: the percentage was true of a window that has since rolled.
+  const [five] = windowsFromGauge({ five_hour: { used_percentage: 41, resets_at: NOW_S - 60 } }, NOW_S);
+  assert.equal(five.known, false);
+  assert.equal(five.stale, true);
+  assert.match(five.detail, /has since reset/);
+});
+
+test('a window the gauge never reported says so rather than reading as zero', () => {
+  const [five, seven] = windowsFromGauge({ seven_day: { used_percentage: 12, resets_at: NOW_S + 999 } }, NOW_S);
+  assert.equal(five.known, false);
+  assert.equal(five.detail, 'not reported');
+  assert.equal(seven.known, true);
+  for (const g of [null, undefined, {}, 'nope']) {
+    assert.deepEqual(windowsFromGauge(g, NOW_S).map((w) => w.known), [false, false], JSON.stringify(g));
+  }
+});
+
+// ── the runner footer ──────────────────────────────────────────────────────────
+
+test('a night that ended probing says so — "nothing shipped" has two causes', () => {
+  const text = renderDigest(buildDigest([
+    { at: at(0), event: 'cadence', mode: 'full', class: 'quiet' },
+    { at: at(1), event: 'cadence', mode: 'probing', class: 'transient' },
+  ], { nowEpochS: NOW_S }));
+  assert.match(text, /Ended the night PROBING/);
+});
+
+test('an event this digest does not know about is counted, never dropped', () => {
+  // A digest that silently ignores what it does not understand is how a new
+  // failure mode stays invisible for a week.
+  const d = buildDigest([{ at: at(0), event: 'something_new_entirely' }], { nowEpochS: NOW_S });
+  assert.equal(d.runner.unrecognised, 1);
+  assert.equal(d.runner.byEvent.something_new_entirely, 1);
+  assert.match(renderDigest(d), /a kind this digest does not know about/);
+});
+
+test('malformed rows never take the night down', () => {
+  const d = buildDigest([null, undefined, 42, 'nope', {}, { event: 'worked', verified: true }], { nowEpochS: NOW_S });
+  assert.equal(d.shipped.length, 1);
+  assert.doesNotThrow(() => renderDigest(d));
+});
+
+// ── the CLI half ───────────────────────────────────────────────────────────────
+
+test('--since accepts a span and defaults to local midnight', () => {
+  const now = new Date('2026-09-19T14:30:00');
+  assert.equal(cli.sinceMs('24h', now), now.getTime() - 86400e3);
+  assert.equal(cli.sinceMs('90m', now), now.getTime() - 5400e3);
+  assert.equal(cli.sinceMs('3d', now), now.getTime() - 3 * 86400e3);
+  // Local midnight, because "last night" is a local idea — a UTC cut would split
+  // an owner's evening in half depending on where they live.
+  const midnight = new Date(now); midnight.setHours(0, 0, 0, 0);
+  assert.equal(cli.sinceMs(null, now), midnight.getTime());
+  assert.equal(cli.sinceMs('soon', now), null, 'an unparseable span must refuse, not guess');
+});
+
+test('a torn last line costs one row, not the night', async () => {
+  // A JSONL written by a process that can be killed mid-write has a torn last
+  // line more often than never.
+  const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path');
+  const f = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'ab-digest-')), 'runs.jsonl');
+  fs.writeFileSync(f, `${JSON.stringify({ at: at(0), event: 'worked', task_id: 1, verified: true })}\n{"at":"x","event":"wor`);
+  const { rows, skipped, missing } = cli.readRows(f);
+  assert.equal(missing, false);
+  assert.equal(rows.length, 1);
+  assert.equal(skipped, 1);
+});
+
+test('a missing log is reported, not mistaken for a quiet night', async () => {
+  const { rows, missing } = cli.readRows('/definitely/not/here/runs.jsonl');
+  assert.deepEqual(rows, []);
+  assert.equal(missing, true);
+});
+
+test('humanDuration reads like a person wrote it', () => {
+  assert.equal(humanDuration(45), '45s');
+  assert.equal(humanDuration(2820), '47m');
+  assert.equal(humanDuration(10080), '2.8h');
+  assert.equal(humanDuration(null), null);
+});
+
+// ── the grader's three findings (task 1003906) ─────────────────────────────────
+
+test('the "did not ship" reason is the LEDGER’s, not the worker’s', () => {
+  // THE INVERSION THIS GOAL EXISTS TO UNDO. `row.reason` is verdict.reason — the
+  // WORKER's account of itself — and quoting it here means the digest reports the
+  // thing being checked rather than the check. A worker that stopped dead needing
+  // a decision reports success; the ledger says what actually happened.
+  const d = buildDigest([{
+    at: at(0), event: 'worked', task_id: 9, title: 'T', verified: false, ledger_shape: 'unlanded',
+    reason: 'I finished the work and shipped it',
+    ledger_reason: 'the grade passed and the land never finished',
+    ledger_remedy: '/merge-mode is the documented manual fallback',
+  }], { nowEpochS: NOW_S });
+
+  assert.equal(d.onHold.ranButDidNotShip[0].reason, 'the grade passed and the land never finished');
+  assert.equal(d.onHold.ranButDidNotShip[0].workerSaid, 'I finished the work and shipped it');
+
+  const text = renderDigest(d);
+  assert.match(text, /the grade passed and the land never finished/);
+  assert.match(text, /next: \/merge-mode/);
+  // The disagreement is the single most useful line on the page.
+  assert.match(text, /the worker said: I finished the work and shipped it/);
+});
+
+test('the worker’s account is printed only when it disagrees', () => {
+  // A worker that agreed with the ledger tells the reader nothing twice.
+  const same = 'the grade parked this';
+  const text = renderDigest(buildDigest([{
+    at: at(0), event: 'worked', task_id: 9, verified: false, ledger_shape: 'ungraded', reason: same, ledger_reason: same,
+  }], { nowEpochS: NOW_S }));
+  assert.equal((text.match(/the grade parked this/g) || []).length, 1);
+  assert.doesNotMatch(text, /the worker said:/);
+});
+
+test('a row from before ledger_reason existed still shows something', () => {
+  const d = buildDigest([{ at: at(0), event: 'worked', task_id: 9, verified: false, ledger_shape: 'unlanded', reason: 'older row' }], { nowEpochS: NOW_S });
+  assert.equal(d.onHold.ranButDidNotShip[0].reason, 'older row');
+});
+
+test('the digest calls a gauge stale on the SAME rule the runner holds on', () => {
+  // A digest that applied a laxer test would print live headroom about a window
+  // the runner had already decided it could not vouch for — two surfaces
+  // disagreeing about one file, with the reader believing the wrong one.
+  const gaugeMod = require('../modules/autonomy/gauge.js');
+  const tooOld = gaugeMod.DEFAULT_STALE_AFTER_S + 60;
+
+  const [fresh] = windowsFromGauge(
+    { five_hour: { used_percentage: 41, resets_at: NOW_S + 7200, observed_at: NOW_S - 60 } }, NOW_S);
+  assert.equal(fresh.known, true, 'a recent reading is usable');
+
+  const [stale] = windowsFromGauge(
+    { five_hour: { used_percentage: 41, resets_at: NOW_S + 7200, observed_at: NOW_S - tooOld } }, NOW_S);
+  assert.equal(stale.known, false);
+  assert.equal(stale.stale, true);
+  assert.match(stale.detail, /older than the runner trusts/);
+  assert.doesNotMatch(stale.detail, /resets in/, 'never offer headroom from a reading the runner rejected');
+});
+
+test('a window with no observed_at falls back to the file’s written_at', () => {
+  // One window can go unreported while the other is carried forward — the exact
+  // shape gauge.js reads.
+  const gaugeMod = require('../modules/autonomy/gauge.js');
+  const old = NOW_S - gaugeMod.DEFAULT_STALE_AFTER_S - 60;
+  const [w] = windowsFromGauge({ written_at: old, five_hour: { used_percentage: 41, resets_at: NOW_S + 7200 } }, NOW_S);
+  assert.equal(w.stale, true);
+});
+
+test('the period bounds survive a log too long to spread as arguments', () => {
+  // `Math.min(...times)` throws past the engine's argument cap, and a reporting
+  // CLI that dies on a big log fails exactly when the log is worth reading.
+  const many = Array.from({ length: 200000 }, (_, i) => ({ at: at(i % 1000), event: 'cadence', mode: 'full' }));
+  let d;
+  assert.doesNotThrow(() => { d = buildDigest(many, { nowEpochS: NOW_S }); });
+  assert.equal(d.period.rows, 200000);
+  assert.equal(d.period.fromMs, T0);
+});

package/tests/autonomy_runner_health.mjs

@@ -0,0 +1,135 @@
+// tests/autonomy_runner_health.mjs — a dead runner is visible, not silent (task 1003905).
+//
+// The done-when's second half: "a runner that has stopped heartbeating is visible
+// from the hall." The asymmetry is the whole design — a running machine can say
+// so, a dead one cannot say anything — so the signal is the AGE of the last
+// check-in, never a status the runner writes. These pin the four answers and, more
+// importantly, the three DIFFERENT kinds of quiet.
+
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const { describeRunner, humanAge, ageS, ALIVE_S, LATE_S } = require('../modules/autonomy/runner-health.js');
+
+const NOW = Date.parse('2026-09-19T12:00:00.000Z');
+const agoS = (s) => new Date(NOW - s * 1000).toISOString();
+const hb = (over = {}) => ({ host: 'winbox', pid: 4242, mode: 'full', last_seen_at: agoS(30), ...over });
+
+test('no row at all is NEVER SEEN, and that is not a failure', () => {
+  // Telling an owner their runner "died" before it ever ran is how a status light
+  // trains people to ignore it.
+  for (const none of [null, undefined, 'nope', 42]) {
+    const d = describeRunner(none, { nowMs: NOW });
+    assert.equal(d.state, 'never_seen', String(none));
+    assert.equal(d.ok, true, 'never set up is not broken');
+    assert.match(d.detail, /has ever checked in/);
+  }
+});
+
+test('the four states follow the age of the last check-in, nothing else', () => {
+  assert.equal(describeRunner(hb({ last_seen_at: agoS(10) }), { nowMs: NOW }).state, 'alive');
+  assert.equal(describeRunner(hb({ last_seen_at: agoS(ALIVE_S - 1) }), { nowMs: NOW }).state, 'alive');
+  assert.equal(describeRunner(hb({ last_seen_at: agoS(ALIVE_S + 1) }), { nowMs: NOW }).state, 'late');
+  assert.equal(describeRunner(hb({ last_seen_at: agoS(LATE_S - 1) }), { nowMs: NOW }).state, 'late');
+  assert.equal(describeRunner(hb({ last_seen_at: agoS(LATE_S + 1) }), { nowMs: NOW }).state, 'dead');
+  assert.equal(describeRunner(hb({ last_seen_at: agoS(86400) }), { nowMs: NOW }).state, 'dead');
+});
+
+test('a runner mid-worker goes quiet legitimately, and LATE says so rather than crying dead', () => {
+  // A worker may hold the loop for ninety minutes. Reporting every honest hour of
+  // work as a corpse is worse than saying nothing, because it teaches the owner to
+  // ignore the one that matters.
+  const building = describeRunner(hb({ last_seen_at: agoS(70 * 60), working_task_id: '1003905' }), { nowMs: NOW });
+  assert.equal(building.state, 'late');
+  assert.equal(building.ok, true, 'late is not an alarm');
+  assert.match(building.detail, /expected while task 1003905 builds/);
+  assert.equal(building.workingTaskId, '1003905');
+});
+
+test('a gap longer than any worker takes IS the alarm', () => {
+  const d = describeRunner(hb({ last_seen_at: agoS(5 * 3600), working_task_id: '1003905' }), { nowMs: NOW });
+  assert.equal(d.state, 'dead');
+  assert.equal(d.ok, false);
+  assert.match(d.detail, /longer than any worker takes/);
+  assert.match(d.detail, /winbox/, 'name the host — an owner with two machines needs to know which');
+});
+
+test('an unreadable check-in raises the alarm rather than explaining it away', () => {
+  // A row that exists but cannot be read is a runner we heard from once and cannot
+  // account for now. On a liveness check the fail direction is to raise the alarm.
+  for (const bad of ['', 'not a date', null, undefined]) {
+    const d = describeRunner(hb({ last_seen_at: bad }), { nowMs: NOW });
+    assert.equal(d.state, 'dead', JSON.stringify(bad));
+    assert.equal(d.ok, false);
+  }
+  // And it is NOT never_seen — the row exists, so something ran once.
+  assert.notEqual(describeRunner(hb({ last_seen_at: 'x' }), { nowMs: NOW }).state, 'never_seen');
+});
+
+test('a probing runner reads as alive, because probing is working', () => {
+  const d = describeRunner(hb({ mode: 'probing', last_seen_at: agoS(20) }), { nowMs: NOW });
+  assert.equal(d.state, 'alive');
+  assert.equal(d.mode, 'probing');
+  assert.match(d.detail, /probing after failures/);
+});
+
+test('a clock skew into the future never reads as ancient', () => {
+  const d = describeRunner(hb({ last_seen_at: new Date(NOW + 60_000).toISOString() }), { nowMs: NOW });
+  assert.equal(d.ageS, 0);
+  assert.equal(d.state, 'alive');
+});
+
+test('every answer carries a readable age and a sentence', () => {
+  for (const s of [0, 30, 300, ALIVE_S + 60, LATE_S + 60, 86400 * 3]) {
+    const d = describeRunner(hb({ last_seen_at: agoS(s) }), { nowMs: NOW });
+    assert.ok(typeof d.detail === 'string' && d.detail.length, `${s}s needs a sentence`);
+    assert.ok(typeof d.age === 'string' && d.age.length, `${s}s needs a readable age`);
+    assert.equal(d.ageS, s);
+  }
+});
+
+test('humanAge reads like a person wrote it', () => {
+  assert.equal(humanAge(45), '45s');
+  assert.equal(humanAge(300), '5m');
+  assert.equal(humanAge(7200), '2h');
+  assert.equal(humanAge(86400 * 3), '3d');
+  assert.equal(humanAge(NaN), 'unknown');
+});
+
+test('ageS refuses to invent a number it cannot compute', () => {
+  assert.equal(ageS('nope', NOW), null);
+  assert.equal(ageS(agoS(10), NaN), null);
+  assert.equal(ageS(agoS(10), NOW), 10);
+});
+
+// ── the forgery the grader found (task 1003905) ─────────────────────────────────
+
+test('a heartbeat row is keyed by the CALLER, so liveness cannot be forged', async () => {
+  // THE ATTACK: with `host` alone as the primary key, any authenticated builder
+  // could POST a heartbeat naming the real runner's hostname and keep the signal
+  // reading "alive" forever — defeating the one property this table exists for,
+  // that a dead machine holding broad spawn permissions becomes visible.
+  //
+  // An ownership check in the handler would also close it, and would be one `if`
+  // away from removal by a later edit that looked reasonable. The caller is in the
+  // KEY instead, so the forgery is impossible rather than merely checked for.
+  const fs = require('node:fs');
+  const path = require('node:path');
+  const sql = fs.readFileSync(path.join(process.cwd(), 'modules/autonomy/migrations/autonomy_002_runner_heartbeat.sql'), 'utf8');
+  assert.match(sql, /PRIMARY KEY \(builder_id, host\)/, 'the caller must be half the key');
+  assert.doesNotMatch(sql, /host\s+text PRIMARY KEY/, 'host alone as the key is the forgeable shape');
+
+  const db = fs.readFileSync(path.join(process.cwd(), 'modules/autonomy/db.js'), 'utf8');
+  assert.match(db, /ON CONFLICT \(builder_id, host\)/, 'the upsert must collide per builder, not per host');
+  assert.match(db, /readRunnerHeartbeats\(builderId\)/, 'the read is scoped to one builder');
+  assert.match(db, /WHERE builder_id = \$1/);
+
+  const routes = fs.readFileSync(path.join(process.cwd(), 'modules/autonomy/routes/autonomy.js'), 'utf8');
+  // The identity must come from the SESSION. A `host` from the body is fine; a
+  // builder id from the body would hand the attacker the key back.
+  assert.match(routes, /builderId: req\.builder\.id/);
+  assert.doesNotMatch(routes, /builderId: (b|req\.body)\./, 'never take the caller’s identity from the body');
+  assert.match(routes, /readRunnerHeartbeats\(req\.builder\.id\)/);
+});

package/tests/bump_version.mjs

@@ -16,9 +16,11 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { createRequire } from 'node:module';
+import { shallowCloneReason } from './helpers.mjs';
 
 const require = createRequire(import.meta.url);
 const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
+
 const bump = require(path.join(ROOT, 'scripts', 'gds', 'bump-version.js'));
 const { bumpPatch, CONST_RE } = bump;
 // The LIVE test below reads this repo's real release history (task 1003202).
@@ -347,11 +349,20 @@ test('a hand-bumped branch passes the LIVE check its own PR runs', () => {
   assert.equal(bump.laneReleasedVersions(subjects).has('1.20.0'), true);
 });
 
-test('LIVE: the version HEAD declares is one this lane already released', () => {
+test('LIVE: the version HEAD declares is one this lane already released', (t) => {
   // The steady state the lane depends on: HEAD carries the version set by the
   // previous release commit, so `taken` is true and the auto-bump fires. If this
   // ever reads false on a normal main checkout, the lane would try to republish
   // an existing version.
+  //
+  // This reads the whole release history, so it cannot run on a shallow clone — a
+  // cloud session sees ~46 release commits instead of hundreds and the assertion
+  // below fails on the clone, not on the code (task 1004049). Skip with the reason
+  // printed, never a silent return: an unrunnable case that reports as a pass is the
+  // defect task 1003993 is open on.
+  const shallow = shallowCloneReason(ROOT);
+  if (shallow) { t.skip(shallow); return; }
+
   const headVersion = /^const CORE_VERSION = '([0-9.]+)';/m
     .exec(fs.readFileSync(path.join(ROOT, 'src', 'module-api.js'), 'utf8'))[1];
   const subjects = execFileSync('git', ['log', '--format=%s', 'HEAD'], { encoding: 'utf8', cwd: ROOT }).split('\n');

package/tests/cli_package.mjs

@@ -19,6 +19,7 @@ import os from 'node:os';
 import path from 'node:path';
 import { spawnSync } from 'node:child_process';
 import { fileURLToPath, pathToFileURL } from 'node:url';
+import { childEnv } from './helpers.mjs';
 
 const REPO_ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
 
@@ -34,8 +35,17 @@ const REPO_ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
 // rather than inlined twice: this is the third site in the repo to need it (after
 // tests/cli_sessions.mjs in task 1003760 and tests/cli_surface.mjs in task 1003941),
 // and every one of them got it half-right first.
+//
+// AND AN INHERITED API BASE IS NOT ISOLATION EITHER (task 1004049). The same
+// shape of hole, one env var over: resolveApiBase() reads the CANONICAL
+// <PREFIX>_API_BASE before any legacy spelling, so the `CLOUDBONGOS_API_BASE:
+// 'http://127.0.0.1:45999'` the verbs below rely on to be a DEAD port is silently
+// shadowed on any box that exports the canonical one — every Bongos session box.
+// The verbs then made real calls to the live instance instead of failing fast
+// against a closed port, which is why this test ran for minutes and timed out
+// rather than failing with something readable. childEnv strips every spelling.
 function homelessEnv(home, extra = {}) {
-  return { ...process.env, HOME: home, USERPROFILE: home, HOMEDRIVE: '', HOMEPATH: home, ...extra };
+  return childEnv({ HOME: home, USERPROFILE: home, HOMEDRIVE: '', HOMEPATH: home, ...extra });
 }
 const BUILDER = path.join(REPO_ROOT, 'scripts', 'gds', 'build-cli-package.js');
 // Running npm from a test (task 1003529). On Windows `npm` is a .cmd shim: bare

package/tests/core_upgrade_runner.mjs

@@ -28,6 +28,8 @@
 
 import { strict as assert } from 'node:assert';
 import { createRequire } from 'node:module';
+import fs from 'node:fs';
+import path from 'node:path';
 
 const require = createRequire(import.meta.url);
 const R = require('../scripts/gds/provision-core-upgrade.js');
@@ -46,13 +48,19 @@ const t = async (name, fn) => {
 // instant. `apply:true` because the interesting behaviour is the applied path.
 function makeDeps({ shape = 'control-plane', served = '1.19.760', available = ['1.19.758', '1.19.760', '1.19.762'], installed = '1.19.760', queries = [] } = {}) {
   const ran = [];
-  const exec = (cmd) => {
+  // task 1004053: the CWD is half of what makes a relative CLI path resolve, and the
+  // bug that shipped was invisible to every assertion that read only the command
+  // string. Captured alongside, so a test can check the pair.
+  const ranWith = [];
+  const exec = (cmd, opts) => {
     ran.push(cmd);
+    ranWith.push({ cmd, cwd: opts && opts.cwd });
     if (/\/version/.test(cmd)) return { ok: true, stdout: served == null ? '' : JSON.stringify({ coreVersion: served }) };
     return { ok: true, stdout: '' };
   };
   return {
     ran,
+    ranWith,
     queries,
     db: { query: async (sql, params) => { queries.push({ sql, params }); return { rows: [] }; } },
     provisioning: { recordEvent: async () => {}, clearErrorNote: async () => {} },
@@ -245,6 +253,40 @@ await t('drives upgrade.js with the registry, the health gate, the version read-
   assert.ok(cmd.includes('--commit-pin'));
 });
 
+await t('the upgrade CLI is run somewhere it ACTUALLY EXISTS — the bug that broke every standalone tenant (task 1004053)', async () => {
+  // The whole failure was a path that resolved nowhere:
+  //   Error: Cannot find module '/srv/<base>/<slug>/scripts/gds/upgrade.js'
+  // No assertion about FLAGS could have caught it, so this one resolves the composed
+  // cwd + relative CLI against the real filesystem, for every shape the door admits.
+  for (const shape of ['control-plane', 'standalone', 'co-tenant']) {
+    const d = makeDeps({ served: '1.19.762' });
+    await R.coreUpgradeInstance(inst({ hosting_shape: shape, slug: 'hermeslines-marketing', port: 3011 }), d, intent());
+    const call = d.ranWith.find((c) => /upgrade\.js/.test(c.cmd));
+    assert.ok(call, `${shape}: it ran the upgrade CLI`);
+    const rel = (call.cmd.match(/node (\S*scripts\/gds\/upgrade\.js)/) || [])[1];
+    assert.ok(rel, `${shape}: the command names the CLI`);
+    assert.ok(fs.existsSync(path.join(call.cwd, rel)),
+      `${shape}: ${path.join(call.cwd, rel)} must exist — this is the assertion the original bug would have failed`);
+  }
+});
+
+await t('a standalone tenant is NAMED with --instance, because the CLI no longer runs inside it', async () => {
+  const d = makeDeps({ served: '1.19.762' });
+  await R.coreUpgradeInstance(inst({ hosting_shape: 'standalone', slug: 'hermeslines-marketing', port: 3011 }), d, intent());
+  const cmd = d.ran.find((c) => /upgrade\.js/.test(c));
+  assert.match(cmd, /--instance \S*hermeslines-marketing/, 'the target is named explicitly');
+  assert.ok(cmd.includes('--service hermeslines-marketing'), 'and its own unit is still the one restarted');
+});
+
+await t('the control-plane row carries NO --instance — its target IS the checkout the CLI runs in', async () => {
+  // Naming it would be harmless but wrong, and the shapes that already worked must not
+  // change behaviour on the way to fixing the one that never did.
+  const d = makeDeps({ served: '1.19.762' });
+  await R.coreUpgradeInstance(inst(), d, intent());
+  const cmd = d.ran.find((c) => /upgrade\.js/.test(c));
+  assert.ok(!cmd.includes('--instance'), 'unchanged for the platform itself');
+});
+
 await t('the health URL and the version URL are LOOPBACK — the apex is behind basic_auth and a CDN (ADR 0293 D6)', async () => {
   const d = makeDeps();
   await R.coreUpgradeInstance(inst(), d, intent());