create-walle 0.10.2 → 0.10.3

package/template/coding-task-manager/lib/landing/batch-runner.js

@@ -0,0 +1,366 @@
+'use strict';
+// WP-1 (landing pipeline design 2026-10-30 §5 D-1..D-3, §8 WP-1): the batched runner — one train,
+// one gate, one CAS for several queued jobs, bisected on FAIL. Kept out of bin/land.js (which sits at
+// its file-size ceiling) behind the same `configure(deps)` seam the server shards use: the runner's
+// own helpers (`runJob`, `progress`, …) are handed in by bin/land.js, never re-implemented here.
+
+const fs = require('fs');
+const path = require('path');
+
+const { revParse, mergeTree, isAncestor } = require('./git');
+const queue = require('./queue');
+const train = require('./train');
+const gate = require('./gate');
+const proof = require('./proof');
+const landingWindow = require('./window');
+const { loadTargetLandingConfig, gatesHash } = require('./config');
+
+// D-3: how often a lone job re-checks the queue while it waits `landing.batch.waitSeconds` for company.
+const BATCH_WAIT_POLL_MS = 2000;
+
+// runJob, progress, submitterWorkdir, gateSelectionTelemetry, sweepConflictPreviews, branchRef, delay
+let deps = {};
+
+function configure(next) {
+  deps = { ...deps, ...next };
+}
+
+// D-1/D-3 (§9 S-8): the window for the NEXT train, decided BEFORE anything is marked `in_progress`.
+async function chooseWindow(ctx, targetTip) {
+  const batchConfig = batchConfigOf(ctx);
+  const runnable = queue.peekRunnable(ctx.root, { targetTip });
+  // D-3: the window is persisted state, clamped to `landing.batch.max` (a config the operator
+  // lowers takes effect on the next train, never after a restart).
+  let window = Math.min(landingWindow.readWindow(ctx.root).size, batchConfig.max);
+  // D-1 last sentence (§9 S-8): a job whose tip is proven or whose tree is cached lands ALONE,
+  // exactly as today, and is never a batch member.
+  const head = runnable.length ? queue.readJob(ctx.root, runnable[0]) : null;
+  if (head && window > 1 && fastPathApplies(ctx, ctx.cwd, batchConfig.config, head)) {
+    deps.progress(`job ${head.id}: proven or cached — landing alone, never a batch member`);
+    window = 1;
+  }
+  // D-3: `landing.batch.waitSeconds` — how long a lone job waits for company before gating
+  // alone. 0 (the default) never waits.
+  if (batchConfig.waitSeconds > 0 && window > 1 && runnable.length === 1) {
+    const deadline = Date.now() + batchConfig.waitSeconds * 1000;
+    let company = runnable.length;
+    while (Date.now() < deadline && company < 2) {
+      await deps.delay(BATCH_WAIT_POLL_MS); // eslint-disable-line no-await-in-loop
+      company = queue.peekRunnable(ctx.root, { targetTip }).length;
+    }
+    if (company < 2) deps.progress(`job ${queue.peekRunnable(ctx.root, { targetTip })[0] || '?'}: waited ${batchConfig.waitSeconds}s for company; gating alone`);
+  }
+  return window;
+}
+
+// WP-1 D-1 (§9 S-8): the fast-path question, asked BEFORE a job can become a batch member — is this
+// tip already proven (D-1 gate-once) or is its train tree already carrying a cached PASS? The
+// decisions mirror `runJobInner`'s (the proof store first, then the tree-keyed verdict cache) so the
+// fast path and the real path can never disagree about which job lands alone.
+function fastPathApplies(ctx, workdir, config, job) {
+  if (!job || !config) return false;
+  let base;
+  try {
+    base = revParse(workdir, deps.branchRef(config.target));
+  } catch (error) {
+    /* REPORT: an unreadable target is `runJobInner`'s refusal to make, not the scheduler's */
+    return false;
+  }
+  const merged = mergeTree(workdir, base, job.tip);
+  if (merged.conflicts.length) return false;
+  if (gate.readCachedVerdict(ctx.root, merged.tree, gatesHash(config.gates))) return true;
+  if (!isAncestor(workdir, base, job.tip)) return false;
+  return Boolean(proof.proofChain(workdir, ctx.root, {
+    base, tip: job.tip, gatesHash: gatesHash(config.gates),
+  }).ok);
+}
+
+// D-3 (§6): the batch configuration, read fresh each iteration so a landing that changes it is
+// honoured on the next train. A policy this process cannot read falls back to the serial defaults —
+// `runJobInner` refuses the job itself a moment later, with the error the operator needs to see.
+function batchConfigOf(ctx) {
+  try {
+    const config = loadTargetLandingConfig(ctx.cwd);
+    return { max: config.batch.max, waitSeconds: config.batch.waitSeconds, config };
+  } catch (error) {
+    /* REPORT: unreadable policy — no batching this iteration; the job's own load enforces it */
+    return { max: 1, waitSeconds: 0, config: null };
+  }
+}
+
+// WP-1 D-1/D-2 (§5, §6, §8 WP-1, §9 S-1..S-8): the batched runner.
+//
+// One train, one gate, one CAS. `runTrain` owns the whole bisection sequence: a batch that FAILs is
+// halved, the first half becomes the NEXT train immediately (same priority slot, `attempt + 1`) and
+// the second half goes back to the queue ahead of everything else — so a batch is never re-gated as a
+// whole and the extra gates a single culprit costs are ⌈log₂N⌉ + 1, not N.
+//
+// A train of one is `runJob` — literally, not approximately — so the singleton path (triage,
+// base_broken, R-3, the exit codes) is byte-for-byte what it was before this change.
+async function runTrain(ctx, members, token, signal) {
+  let current = members;
+  for (;;) {
+    try {
+      if (current.length <= 1) return [await runSingle(ctx, current[0], token, signal)];
+      const outcome = await runBatchOnce(ctx, current, token, signal);
+      if (!outcome.bisect) return outcome.jobs;
+      current = outcome.bisect;
+    } catch (error) {
+      const code = typeof error.code === 'string' ? error.code : 'internal_error';
+      deps.progress(`batch ${members[0].id}: ${code}: ${error.message}`);
+      return current.map((member) => failMember(ctx, member, code, error));
+    }
+  }
+}
+
+// D-3: a singleton that LANDS is a PASS for the window too. Without this the window could never
+// leave 1 — at 1 every train is a singleton, and only a batch verdict would ever move it.
+async function runSingle(ctx, job, token, signal) {
+  const result = await deps.runJob(ctx, job, token, signal);
+  if (result && result.state === 'landed') {
+    landingWindow.bumpWindow(ctx.root, 'PASS', batchConfigOf(ctx).max);
+    // WP-1 R-9 (§6.3 step 6): the SINGLE place a landing reaches the origin half — a `kind:"sync"`
+    // job pushes what it landed, an ordinary landing auto-publishes when `landing.publish.auto`.
+    await deps.afterTrainLanded(ctx, [result]);
+  }
+  return result;
+}
+
+// Anything unexpected still leaves the queue decidable: every member that has not been settled is
+// `failed` with the typed code recorded, never stuck `in_progress` for a later runner to reap.
+function failMember(ctx, member, code, error) {
+  const fresh = queue.readJob(ctx.root, member.id);
+  if (!fresh || queue.TERMINAL_STATES.includes(fresh.state)) return fresh;
+  deps.progress(`job ${member.id}: ${code}: ${error.message}`);
+  return queue.finishJob(ctx.root, member.id, {
+    state: 'failed',
+    reason: null,
+    error: { code, message: error.message },
+    timestamps: { ...(fresh.timestamps || {}), gated: queue.nowIso() },
+  });
+}
+
+// D-7 (WP-1): a stop request is honoured at a TRAIN BOUNDARY, never by aborting the others' gate. A
+// cancel never lands; a preempt goes back to the queue, where its urgent successor sorts to the head.
+function dropStoppedMembers(ctx, members) {
+  const live = [];
+  for (const member of members) {
+    const fresh = queue.readJob(ctx.root, member.id);
+    const request = fresh && fresh.stopRequest;
+    if (!request) {
+      if (fresh) live.push(fresh);
+      continue;
+    }
+    if (request.kind === 'cancel') {
+      deps.progress(`job ${member.id}: stopped before the train was built`);
+      queue.finishJob(ctx.root, member.id, {
+        state: 'canceled',
+        reason: 'stopped',
+        canceledBy: request.by,
+        stopRequest: null,
+        timestamps: { ...(fresh.timestamps || {}), canceled: queue.nowIso() },
+      });
+    } else {
+      deps.progress(`job ${member.id}: preempted by ${request.forJob}; back to the queue`);
+      queue.requeuePreempted(ctx.root, member.id);
+    }
+  }
+  return live;
+}
+
+// The train's synthetic job record: the gate and the refresh entries are handed LAND_TIP/LAND_JOB, and
+// for a batch both name the TRAIN (the chain head and the batch id), not whichever member happened to
+// be first — a member id there would make one member's log the batch's identity.
+function batchJob(batchId, built, members) {
+  return {
+    id: batchId,
+    tip: built.train,
+    branch: `batch(${batchId})`,
+    title: `${members.length} members`,
+  };
+}
+
+async function runBatchOnce(ctx, members, token, signal) {
+  const batchId = (members[0].batch && members[0].batch.id) || queue.newBatchId();
+  const attempt = (members[0].batch && members[0].batch.attempt) || 1;
+  const workdir = deps.submitterWorkdir(ctx, members[0]);
+  // D-1/I-5: the policy is the `.ctm.json` committed at the TARGET, exactly as for one job.
+  const config = loadTargetLandingConfig(workdir);
+  const trainsRoot = train.trainsRoot(workdir, ctx.common).dir;
+  const trainDir = train.trainDirFor(trainsRoot, batchId);
+  // §9 S-4: the gate log, the selection record, the shadow record and the train worktree are all keyed
+  // by the TRAIN id, so a batch has one log every member points at (`gateLog`), not N copies.
+  const logFile = path.join(queue.ensureLogsDir(ctx.root), `${batchId}.log`);
+  const targetWorktree = train.targetWorktreeFor(workdir, config.target);
+  const sourceDir = targetWorktree ? targetWorktree.path : workdir;
+  const windowBefore = landingWindow.readWindow(ctx.root).size;
+
+  const live = dropStoppedMembers(ctx, members);
+  if (!live.length) return { jobs: [], bisect: null };
+  if (live.length === 1) return { jobs: [await runSingle(ctx, live[0], token, signal)], bisect: null };
+
+  const built = train.buildBatchTrain(workdir, config.target, live, {
+    onSkip: (member, paths, head) => {
+      deps.progress(`job ${member.id}: conflicts with the train chain at ${String(head).slice(0, 12)} — skipped (${paths.join(', ')})`);
+      queue.updateJob(ctx.root, member.id, {
+        state: 'queued',
+        conflictPreview: { base: head, paths, checkedAt: queue.nowIso() },
+      });
+    },
+  });
+  const landing = built.members;
+  deps.progress(`batch ${batchId} attempt ${attempt}: ${built.kind} on ${built.base.slice(0, 12)} — ${landing.length} member(s), ${built.skipped.length} skipped`);
+  if (!landing.length) return { jobs: [], bisect: null };
+
+  let finalBuilt = built;
+  const refreshEntries = train.matchingRefreshEntries(config);
+  if (built.kind === 'train' && refreshEntries.length) {
+    finalBuilt = await train.refreshTrain({
+      cwd: workdir, dir: trainDir, job: batchJob(batchId, built, landing), built, entries: refreshEntries, log: logFile, signal,
+    });
+    if (finalBuilt.refreshed && finalBuilt.refreshed.length) deps.progress(`batch ${batchId}: refreshed ${finalBuilt.refreshed.join(', ')}`);
+  }
+
+  let verdict;
+  let reason = null;
+  let gateSeconds = null;
+  let gateTelemetry = null;
+  const cached = gate.readCachedVerdict(ctx.root, finalBuilt.tree, gatesHash(config.gates));
+  if (cached) {
+    verdict = cached.verdict;
+    gateSeconds = cached.seconds;
+    gateTelemetry = {
+      mode: 'cached', planBase: null, selected: null, executed: null, cached: null,
+      missReasons: null, fallbackReason: null, intersection: null,
+    };
+    deps.progress(`batch ${batchId}: verdict CACHED (${verdict})`);
+    if (fs.existsSync(trainDir)) train.removeTrainWorktree(workdir, trainDir);
+  } else {
+    const result = await gate.runGate({
+      cwd: workdir, dir: trainDir, config, job: batchJob(batchId, finalBuilt, landing), trainId: batchId,
+      built: finalBuilt, token, root: ctx.root, logFile, sourceDir, signal,
+    });
+    verdict = result.verdict;
+    reason = result.reason;
+    gateSeconds = result.seconds;
+    gateTelemetry = deps.gateSelectionTelemetry(result.selection);
+    deps.progress(`batch ${batchId}: gate ${verdict === null ? 'stopped' : verdict}${reason ? ` (${reason})` : ''} in ${result.seconds}s`);
+    if (result.verdict === 'PASS') {
+      // WP-2 (§5 D-1): the train's PASS is a proof for the NEXT train built on this tree, under the
+      // same D-1 refusals the hook's own PASS goes through. A refusal is reported, never a change of
+      // this batch's outcome.
+      try {
+        const refused = proof.proofRefusal(workdir, {
+          planBase: finalBuilt.base, tree: finalBuilt.tree, logText: fs.readFileSync(result.logPath, 'utf8'),
+        });
+        const fatal = refused && refused !== 'no_pass' ? refused : null;
+        if (fatal) deps.progress(`batch ${batchId}: proof not recorded (${fatal})`);
+        else {
+          proof.recordProof(ctx.root, {
+            tree: finalBuilt.tree,
+            planBaseTree: revParse(workdir, `${finalBuilt.base}^{tree}`),
+            gatesHash: gatesHash(config.gates),
+            verdict: 'PASS',
+            source: 'train',
+            commit: finalBuilt.train,
+            seconds: result.seconds,
+            at: new Date().toISOString(),
+            logPath: result.logPath,
+            selection: result.selection,
+          });
+        }
+      } catch (error) {
+        deps.progress(`batch ${batchId}: proof not recorded (${error.code || 'error'}: ${error.message})`);
+      }
+    }
+  }
+
+  // D-3/D-5 (WP-1) post-gate checkpoint: a member stopped while the gate ran is settled here and the
+  // run is VOIDED — the chain this gate judged was built from every member (train.js), so it carries
+  // code that will not land and no outcome of it can be used (P-1). The survivors go back to the queue
+  // at the SAME attempt: a withdrawal is not a bisection round, and the window is not touched (D-4).
+  const landingNow = dropStoppedMembers(ctx, landing);
+  const dropped = landing.length - landingNow.length;
+
+  if (reason === 'gate_stopped' || dropped > 0 || !landingNow.length) {
+    if (landingNow.length) queue.requeueHalf(ctx.root, landingNow, { batchId, attempt, batchMembers: landingNow.map((m) => m.id), reason: dropped > 0 ? 'withdrawn' : 'bisected' });
+    if (dropped > 0) deps.progress(`batch ${batchId}: ${dropped} member(s) withdrew — not landing; ${landingNow.length} requeued at attempt ${attempt}`);
+    return { jobs: [], bisect: null };
+  }
+
+  if (verdict !== 'PASS') {
+    // D-2: halve. The window halves FIRST (D-3), so the next train is built at the smaller window.
+    const windowAfter = landingWindow.bumpWindow(ctx.root, 'FAIL', config.batch.max).size;
+    queue.recordBatchMetric(ctx.root, {
+      batchId, members: landing.map((m) => m.id), verdict: 'FAIL', reason, gateSeconds, windowBefore, windowAfter,
+    });
+    const half = Math.ceil(landingNow.length / 2);
+    const first = landingNow.slice(0, half);
+    const second = landingNow.slice(half);
+    if (second.length) queue.requeueHalf(ctx.root, second, { batchId, attempt: attempt + 1, batchMembers: landingNow.map((m) => m.id) });
+    for (const member of first) {
+      queue.updateJob(ctx.root, member.id, {
+        batch: { id: batchId, members: landingNow.map((m) => m.id), attempt: attempt + 1, index: first.indexOf(member) },
+      });
+    }
+    deps.progress(`batch ${batchId}: FAIL (${reason || 'gate'}) — bisecting ${landingNow.length} → ${first.length} now + ${second.length} requeued`);
+    return { jobs: [], bisect: first.map((member) => queue.readJob(ctx.root, member.id)).filter(Boolean) };
+  }
+
+  // D-1/D-11: ONE CAS for the whole chain, comparing against the commit the train was built on.
+  let landed;
+  try {
+    landed = train.landTrain(workdir, config.target, finalBuilt.train, finalBuilt.base, { env: { LAND_TOKEN: token } });
+  } catch (error) {
+    if (error && (error.code === 'cas_lost' || error.code === 'target_dirty')) {
+      // A target that moved (or is dirty) is not this batch's failure: every member goes back to the
+      // queue and the next drain iteration rebuilds the chain on the target as it is now.
+      deps.progress(`batch ${batchId}: ${error.code}; requeueing ${landingNow.length} member(s)`);
+      queue.requeueHalf(ctx.root, landingNow, { batchId, attempt: attempt + 1, batchMembers: landingNow.map((m) => m.id) });
+      return { jobs: [], bisect: null };
+    }
+    throw error;
+  }
+  deps.progress(`batch ${batchId}: landed ${String(finalBuilt.train).slice(0, 12)} via ${landed.via} (${landingNow.length} member(s))`);
+  const windowAfter = landingWindow.bumpWindow(ctx.root, 'PASS', config.batch.max).size;
+  const results = queue.finishBatch(ctx.root, batchId, landingNow, (member) => ({
+    state: 'landed',
+    reason: null,
+    verdict: 'PASS',
+    base: finalBuilt.base,
+    tree: finalBuilt.tree,
+    train: finalBuilt.train,
+    landedVia: landed.via,
+    gateLog: logFile,
+    gate: gateTelemetry,
+    batch: member.batch,
+    timestamps: { ...(member.timestamps || {}), landed: queue.nowIso() },
+  }), {
+    gateSeconds,
+    gate: gateTelemetry,
+    // §6/D-8: the batch's ONE row rides the SAME hold as the members it describes, so a reader that
+    // can see the landing can see the verdict that produced it.
+    metric: {
+      members: landingNow.map((m) => m.id),
+      verdict: 'PASS',
+      gateSeconds,
+      windowBefore,
+      windowAfter,
+      gate: gateTelemetry,
+    },
+  });
+  deps.sweepConflictPreviews(ctx, workdir, config);
+  // WP-1 R-9: a BATCH lands N members with one gate, so it publishes once — for the train, not per
+  // member. A sync job is never a batch member in practice (it is enqueued while the queue drains),
+  // and `afterTrainLanded` reads the landed set either way.
+  await deps.afterTrainLanded(ctx, results);
+  return { jobs: results, bisect: null };
+}
+
+module.exports = {
+  configure,
+  chooseWindow,
+  runTrain,
+  fastPathApplies,
+  batchConfigOf,
+};