dorfl 0.13.0 → 0.13.2

package/src/pi-harness.ts

@@ -1,5 +1,5 @@
 import {spawn, spawnSync} from 'node:child_process';
-import {existsSync, readFileSync} from 'node:fs';
+import {existsSync, readFileSync, writeSync} from 'node:fs';
 import {
 	NullHarness,
 	pidAlive,
@@ -66,6 +66,97 @@ import type {HarnessAdapter} from './config.js';
 /** The default pi CLI binary name (resolved on `PATH`). */
 export const DEFAULT_PI_BIN = 'pi';
 
+/**
+ * **A runner MUST NOT exit while an async launch is unsettled** (observation
+ * `deadline-reap-lets-node-exit-0-before-the-checkpoint-runs`).
+ *
+ * An `await` is not a handle: node keeps a process alive for referenced HANDLES
+ * (timers, sockets, child processes), and a suspended promise is none of those.
+ * {@link PiHarness.launchAsync} deliberately drops every handle it owns the
+ * moment pi exits (it destroys the stdio pipes and `unref`s the child so a
+ * leaked grandchild's inherited FDs cannot pin the loop), and on the deadline
+ * path it then keeps the promise pending across the process-group reap. In the
+ * field that combination let the event loop go EMPTY mid-reap: node exited
+ * normally with code 0, the suspended pipeline (checkpoint save, branch push,
+ * lock release, writer-sentinel release, job-record update) simply never ran,
+ * and the run reported SUCCESS while leaving the item locked and 90 minutes of
+ * agent work uncommitted.
+ *
+ * So the launch holds an explicit REFERENCED keep-alive for exactly as long as
+ * it is in flight, and an exit guard turns any remaining way of exiting mid-
+ * launch into a LOUD, non-zero failure instead of a silent success. The two are
+ * deliberately independent: the keep-alive prevents the known mechanism, the
+ * guard refuses to let any future variant of it be mistaken for a clean run.
+ */
+let inFlightLaunches = 0;
+/** The referenced handle that keeps the loop alive while launches are in flight. */
+let inFlightKeepAlive: NodeJS.Timeout | undefined;
+let inFlightExitGuardInstalled = false;
+
+/**
+ * The keep-alive tick. Long, because it exists ONLY to be a referenced handle:
+ * it never does work, and it is cleared the moment the last launch settles.
+ */
+const KEEPALIVE_TICK_MS = 60_000;
+
+/** Report an exit that happened with a launch still in flight, LOUDLY. */
+function installInFlightExitGuard(): void {
+	if (inFlightExitGuardInstalled) {
+		return;
+	}
+	inFlightExitGuardInstalled = true;
+	process.on('exit', (code) => {
+		if (inFlightLaunches === 0) {
+			return;
+		}
+		// `writeSync` on fd 2, NOT console.error: stderr to a pipe is asynchronous,
+		// and an `exit` listener is the last synchronous moment there is, so a
+		// buffered write would be dropped exactly when it matters most.
+		writeSync(
+			2,
+			`>> INTERNAL ERROR: dorfl is exiting while ${inFlightLaunches} agent ` +
+				'launch(es) are still in flight, so the run STOPPED between the agent ' +
+				'and its outcome: nothing was committed, pushed, surfaced or released, ' +
+				'and any item lock is still held. This is a dorfl defect, not a task ' +
+				'failure. Recover with `dorfl requeue <slug>` (the work branch/worktree ' +
+				'is kept) and please report it.\n',
+		);
+		if (code === 0) {
+			// NEVER report this as success: a caller (CI leg, driving loop, `run`
+			// tick) that only checks the status must see a failure here.
+			process.exitCode = 1;
+		}
+	});
+}
+
+/** Mark one async launch as started (holds the loop open + arms the guard). */
+function launchStarted(): void {
+	inFlightLaunches += 1;
+	installInFlightExitGuard();
+	if (inFlightKeepAlive === undefined) {
+		// REFERENCED on purpose: this is the handle that keeps the runner alive
+		// across the window where it owns no other one.
+		inFlightKeepAlive = setInterval(() => {}, KEEPALIVE_TICK_MS);
+	}
+}
+
+/** Mark one async launch as settled (releases the keep-alive when the last one lands). */
+function launchSettled(): void {
+	inFlightLaunches = Math.max(0, inFlightLaunches - 1);
+	if (inFlightLaunches === 0 && inFlightKeepAlive !== undefined) {
+		clearInterval(inFlightKeepAlive);
+		inFlightKeepAlive = undefined;
+	}
+}
+
+/**
+ * How many async launches are currently unsettled. Exposed for the regression
+ * test that pins the keep-alive/guard invariant; not part of the harness seam.
+ */
+export function inFlightLaunchCount(): number {
+	return inFlightLaunches;
+}
+
 /**
  * The grace period between a deadline SIGTERM and the follow-up SIGKILL in
  * {@link PiHarness.launchAsync} (spec `graceful-pre-timeout-wip-checkpoint`).
@@ -233,7 +324,11 @@ export class PiHarness implements Harness {
 			command: [this.piBin, ...args].join(' '),
 			session: sessionFile,
 		};
-		return new Promise<LaunchResult>((resolve, reject) => {
+		// IN FLIGHT from here until the promise settles: hold the loop open and arm
+		// the exit guard, so the runner can never quietly disappear between the
+		// agent and its outcome (see the keep-alive block above).
+		launchStarted();
+		const launch = new Promise<LaunchResult>((resolve, reject) => {
 			const child = spawn(this.piBin, args, {
 				// Same as `launch`: spawn in the repo/worktree dir so the session
 				// header `cwd` groups the dashboard correctly (invariant #3).
@@ -436,6 +531,11 @@ export class PiHarness implements Harness {
 			}
 			child.stdin?.end();
 		});
+		// Release the keep-alive on BOTH outcomes (resolve AND reject) — a failed
+		// spawn must not pin the loop open for the rest of the process's life.
+		return launch.finally(() => {
+			launchSettled();
+		});
 	}
 
 	/**

package/src/reap-agent-tree.ts

@@ -72,11 +72,27 @@ export interface ReapResult {
 	detail: string;
 }
 
-/** Sleep helper (injectable clock is not needed: callers inject `wait` in tests). */
+/**
+ * Sleep helper (injectable clock is not needed: callers inject `wait` in tests).
+ *
+ * **The timer is deliberately REFERENCED — do not `unref()` it.** This sleep is
+ * the ONLY pending handle the runner holds while it waits for a signalled group
+ * to die: by the time {@link reapProcessGroup} runs, the harness has already
+ * destroyed the child's stdio and `unref`'d the child handle, and a pending
+ * promise is not a handle. An `unref`'d timer here therefore left the event loop
+ * with NOTHING referenced, so node did the correct thing with an empty loop and
+ * EXITED 0 mid-reap — abandoning the suspended `await` and with it the whole
+ * deadline checkpoint (no WIP commit, no branch push, no lock release, no
+ * sentinel release), while reporting success (observation
+ * `deadline-reap-lets-node-exit-0-before-the-checkpoint-runs`).
+ *
+ * Referencing it cannot hang a runner: this loop is bounded by construction
+ * (`sigtermGraceMs + sigkillTimeoutMs`), which is the same property that lets
+ * the launch resolve-on-`exit` discipline stay safe.
+ */
 function sleep(ms: number): Promise<void> {
 	return new Promise((resolve) => {
-		const timer = setTimeout(resolve, ms);
-		timer.unref?.();
+		setTimeout(resolve, ms);
 	});
 }
 

package/src/tasking.ts

@@ -536,10 +536,59 @@ export async function performTask(
 	}
 	if (!agent.ok) {
 		const detail = agent.detail ?? `the agent failed to task '${slug}'.`;
-		const message = `Agent failed tasking '${slug}' (${detail}).`;
+		let message = `Agent failed tasking '${slug}' (${detail}).`;
+		// RELEASE THE LOCK THE FAILED RUN TOOK (observation
+		// `crashed-do-spec-strands-a-tasking-lock-no-verb-releases`).
+		//
+		// The pre-fix behaviour left the lock HELD on the arbiter, justified by a
+		// comment claiming "surfacing it is the review/edit loop's job". That is
+		// UNREACHABLE from here: the review/edit loop runs at step 3.5, strictly
+		// AFTER this early return, so an agent that dies at step 3 is surfaced by
+		// NOBODY. The lock outlived every process that knew about it and each retry
+		// of `do spec:<slug>` lost the create-only CAS against a holder that no
+		// longer existed (`'spec-<slug>' is already locked (held by another)`).
+		// Field trigger: three consecutive model-API faults (`Connection error.`,
+		// `overloaded_error`, `api_error`).
+		//
+		// WHY A PLAIN RELEASE AND NOT A `stuck` SURFACE. The sibling failure just
+		// below (`ReviewParseError`) routes through `surfaceTaskingBlock`, which
+		// marks the spec `needsAnswers:true` + writes a question sidecar. That is
+		// correct THERE (a review verdict is a JUDGEMENT a human must resolve) and
+		// wrong HERE: an agent/transport crash carries no judgement to record, and
+		// surfacing it would take the spec OUT of the taskable pool behind a
+		// contentless question — turning a retryable blip into mandatory human
+		// paperwork. A crashed agent is RETRYABLE, so the recovery is to return the
+		// spec to the pool and let `do spec:<slug>` simply be re-run.
+		//
+		// WHY THIS DISCARDS NOTHING. The tasking work branch is created by
+		// `switchToWorkBranch` with a LOCAL `git switch -C` and is never pushed
+		// before the integrate band; the durable `specs/ready → specs/tasked` move
+		// also happens only at integrate. A run that died at step 3 therefore
+		// published NOTHING to the arbiter, so releasing its lock cannot lose work.
+		// The release is deliberately no more cautious than that risk warrants.
+		//
+		// A release FAULT is reported but never masks the agent failure: the agent
+		// outcome is the terminal one, and `release-lock spec:<slug>` remains the
+		// human backstop for the case where this process is itself killed.
+		if (useLock) {
+			const released = await lock.release({
+				slug,
+				cwd,
+				arbiter,
+				lockedBlob,
+				env,
+				note,
+			});
+			message =
+				released.exitCode === 0
+					? `${message} Released the tasking lock for '${slug}' — the spec is ` +
+						'back in the taskable pool, so re-run `dorfl do spec:' +
+						`${slug}\` to retry (nothing was published, so nothing was lost).`
+					: `${message} WARNING: could not release the tasking lock for ` +
+						`'${slug}' (${released.message}). Clear it with \`dorfl ` +
+						`release-lock spec:${slug}\` before retrying.`;
+		}
 		note(message);
-		// The lock stays held (the runner did not release it): a stuck tasking is
-		// recoverable / re-runnable. Surfacing it is the review/edit loop's job.
 		return {exitCode: 1, outcome: 'agent-failed', slug, message};
 	}