dorfl 0.11.0 → 0.11.2

package/src/reap-agent-tree.ts

@@ -0,0 +1,221 @@
+/**
+ * **Reap a stopped agent's whole PROCESS TREE, and VERIFY it is gone** (spec
+ * `graceful-pre-timeout-wip-checkpoint`, observation
+ * `checkpoint-releases-lock-while-predecessor-agent-still-writes`).
+ *
+ * ## The gap this closes
+ *
+ * The deadline checkpoint used to `child.kill('SIGTERM')` the agent process and
+ * treat the signal as if it were the outcome: the moment pi's own `exit` fired,
+ * the runner saved the WIP, RELEASED the item lock, and let the next tick
+ * dispatch a CONTINUATION agent into the SAME worktree. But `child.kill` signals
+ * exactly ONE pid, and a modern agent is a TREE (subagent processes, MCP servers,
+ * model proxies, tool subshells). Those descendants survive the parent's SIGTERM,
+ * and once pi exits they are re-parented to init — so they can no longer even be
+ * FOUND by walking `ppid`, while they keep writing into the worktree.
+ *
+ * Observed on a real run: the checkpoint saved WIP at 02:13, a continuation agent
+ * onboarded into the same worktree at ~02:15, and the predecessor's session log
+ * kept being written until 02:19:29 — four minutes INTO the successor's run,
+ * whose opening `git status` had already read the tree as clean. Nothing was lost
+ * only because the two happened to touch different files. The shape is the
+ * defect: one lock, one working tree, two live writers. A write landing after the
+ * successor's `git status` is invisible to it; a write landing during its edits
+ * can be clobbered either way; and the successor can commit the predecessor's
+ * half-finished edits as its own, under a message describing something else.
+ *
+ * ## Why the PROCESS GROUP is the handle
+ *
+ * A pid-tree walk cannot work here: the descendants we must reap are precisely
+ * the ones that OUTLIVE the parent, and an orphan's `ppid` is gone. A process
+ * GROUP id, by contrast, is inherited by every descendant and is NOT changed by
+ * re-parenting. So if the agent is spawned as a group LEADER (`detached: true`,
+ * making its pgid equal its pid), `kill(-pgid, …)` reaches the entire tree,
+ * orphans included — which is why {@link reapProcessGroup} takes a pgid and why
+ * `pi-harness.ts` spawns the deadline-capable async launch detached.
+ *
+ * ## Signal, then VERIFY — never assume
+ *
+ * The point of the whole module is that sending a signal is not evidence that
+ * anything died. So: SIGTERM the group, POLL until it is actually gone, escalate
+ * to SIGKILL after a grace, keep polling, and if it STILL will not die, say so
+ * LOUDLY and let the caller refuse to release the lock. A checkpoint that cannot
+ * prove the predecessor is dead must not hand the worktree to a successor.
+ */
+
+/** Poll interval while waiting for a signalled group to actually exit. */
+const REAP_POLL_MS = 50;
+
+/**
+ * How long to wait after SIGTERM before escalating to SIGKILL. Matches
+ * `pi-harness.ts`'s `DEADLINE_SIGKILL_GRACE_MS` intent: enough for an agent to
+ * flush its session log and exit cleanly, short enough not to stall a CI leg.
+ */
+export const REAP_SIGTERM_GRACE_MS = 10_000;
+
+/** How long to keep waiting after SIGKILL before declaring the reap FAILED. */
+export const REAP_SIGKILL_TIMEOUT_MS = 5_000;
+
+/** The outcome of a {@link reapProcessGroup} attempt. */
+export interface ReapResult {
+	/**
+	 * True iff the group is VERIFIED gone (observed non-existent, not merely
+	 * signalled). Only a `true` here licenses releasing the item lock and
+	 * dispatching a successor into the same worktree.
+	 */
+	reaped: boolean;
+	/** True iff SIGKILL was needed (the tree ignored SIGTERM) — worth reporting. */
+	escalatedToSigkill: boolean;
+	/** Total wall-clock ms spent waiting for the tree to die. */
+	waitedMs: number;
+	/** A human-readable account, always populated (the LOUD failure text). */
+	detail: string;
+}
+
+/** Sleep helper (injectable clock is not needed: callers inject `wait` in tests). */
+function sleep(ms: number): Promise<void> {
+	return new Promise((resolve) => {
+		const timer = setTimeout(resolve, ms);
+		timer.unref?.();
+	});
+}
+
+/**
+ * Is any process still alive in process group `pgid`?
+ *
+ * `process.kill(-pgid, 0)` is the liveness probe: signal 0 performs the
+ * permission/existence check WITHOUT delivering a signal (the same technique
+ * `harness.ts`'s {@link pidAlive} uses for a single pid, widened to the group).
+ *
+ *  - It THROWS `ESRCH` when no process in the group exists ⇒ the group is gone.
+ *  - It THROWS `EPERM` when the group exists but we may not signal it. That is
+ *    still "alive", and reporting it as dead would be the very assumption this
+ *    module exists to remove — so `EPERM` reads as ALIVE.
+ *  - It succeeds ⇒ alive.
+ */
+export function processGroupAlive(pgid: number): boolean {
+	if (!Number.isInteger(pgid) || pgid <= 1) {
+		// pgid 0/1 (or a bogus value) would mean "our own group" / init — signalling
+		// those would be catastrophic, so never claim they are ours to reap.
+		return false;
+	}
+	try {
+		process.kill(-pgid, 0);
+		return true;
+	} catch (err) {
+		const code = (err as NodeJS.ErrnoException).code;
+		if (code === 'EPERM') {
+			return true; // exists, just not signallable by us.
+		}
+		return false; // ESRCH (or anything else): treat as gone.
+	}
+}
+
+/** Signal a whole process group, tolerating an already-dead group. */
+function signalGroup(pgid: number, signal: NodeJS.Signals): void {
+	try {
+		process.kill(-pgid, signal);
+	} catch {
+		// ESRCH: already gone — the wait loop below observes that and succeeds.
+	}
+}
+
+/**
+ * SIGTERM process group `pgid`, wait for it to ACTUALLY exit, escalate to
+ * SIGKILL after {@link REAP_SIGTERM_GRACE_MS}, and report whether the tree is
+ * VERIFIED gone.
+ *
+ * Bounded by construction: at worst `sigtermGraceMs + sigkillTimeoutMs` before it
+ * returns `reaped: false` with a loud `detail`. It never waits indefinitely, so
+ * it cannot reintroduce the runner-hang the async launch's resolve-on-`exit`
+ * discipline exists to avoid.
+ */
+export async function reapProcessGroup(params: {
+	/** The process GROUP id to reap (the group leader's pid). */
+	pgid: number;
+	sigtermGraceMs?: number;
+	sigkillTimeoutMs?: number;
+	/** Injectable sleep so tests need not burn real seconds. */
+	wait?: (ms: number) => Promise<void>;
+	/**
+	 * Injectable liveness probe (default {@link processGroupAlive}). Exists so the
+	 * REFUSAL path — a tree that survives SIGTERM *and* SIGKILL — can be tested
+	 * deterministically. There is no portable way to create a genuinely unkillable
+	 * process, and the alternative (a group we truly cannot signal) would risk the
+	 * test process itself, so the probe is the seam.
+	 */
+	alive?: (pgid: number) => boolean;
+}): Promise<ReapResult> {
+	const {
+		pgid,
+		sigtermGraceMs = REAP_SIGTERM_GRACE_MS,
+		sigkillTimeoutMs = REAP_SIGKILL_TIMEOUT_MS,
+		wait = sleep,
+		alive = processGroupAlive,
+	} = params;
+	const started = Date.now();
+
+	if (!alive(pgid)) {
+		return {
+			reaped: true,
+			escalatedToSigkill: false,
+			waitedMs: 0,
+			detail: `agent process group ${pgid} was already gone (nothing to reap).`,
+		};
+	}
+
+	// 1. SOFT: ask the whole tree to stop, then WAIT for it to be observably gone.
+	signalGroup(pgid, 'SIGTERM');
+	while (Date.now() - started < sigtermGraceMs) {
+		if (!alive(pgid)) {
+			const waitedMs = Date.now() - started;
+			return {
+				reaped: true,
+				escalatedToSigkill: false,
+				waitedMs,
+				detail:
+					`agent process group ${pgid} exited on SIGTERM after ${waitedMs}ms ` +
+					'(verified gone).',
+			};
+		}
+		await wait(REAP_POLL_MS);
+	}
+
+	// 2. HARD: the tree ignored SIGTERM through the grace. SIGKILL it and keep
+	//    verifying — a wedged agent still holding the worktree must not survive
+	//    into the successor's run.
+	signalGroup(pgid, 'SIGKILL');
+	const killDeadline = Date.now() + sigkillTimeoutMs;
+	while (Date.now() < killDeadline) {
+		if (!alive(pgid)) {
+			const waitedMs = Date.now() - started;
+			return {
+				reaped: true,
+				escalatedToSigkill: true,
+				waitedMs,
+				detail:
+					`agent process group ${pgid} ignored SIGTERM and was SIGKILLed; ` +
+					`exited after ${waitedMs}ms (verified gone).`,
+			};
+		}
+		await wait(REAP_POLL_MS);
+	}
+
+	// 3. LOUD FAILURE: we cannot prove the predecessor is dead. Say exactly that,
+	//    and exactly what the caller must not do as a result.
+	const waitedMs = Date.now() - started;
+	return {
+		reaped: false,
+		escalatedToSigkill: true,
+		waitedMs,
+		detail:
+			`REFUSING TO PROCEED: agent process group ${pgid} is STILL ALIVE ${waitedMs}ms ` +
+			'after SIGTERM and SIGKILL (an unkillable/uninterruptible descendant — e.g. a ' +
+			'process wedged in a kernel call, or one we lack permission to signal). It may ' +
+			'still be WRITING to the worktree, so the item lock must NOT be released and no ' +
+			'successor agent may onboard here: two live writers in one working tree can ' +
+			'silently clobber each other and let a successor commit the predecessor’s ' +
+			`half-finished edits. Inspect and kill it by hand (\`ps -g ${pgid}\`, ` +
+			`\`kill -9 -${pgid}\`) before re-running this item.`,
+	};
+}

package/src/repo-config.ts

@@ -2,6 +2,7 @@ import {readFileSync, existsSync} from 'node:fs';
 import {join} from 'node:path';
 import {
 	mergeConfig,
+	applyModelFallbacks,
 	validateDeadlineConfig,
 	validateDorflCmdConfig,
 	warnDeprecatedConfigKeys,
@@ -209,8 +210,12 @@ export const REPO_ALLOWED_KEYS = [
 	// `model` (which model this repo's work runs on) and `harness` (which adapter)
 	// are legitimate repo properties (ADR §13) — model is routing intent, not auth,
 	// and a repo may prefer a given harness. `piBin`/`agentCmd` stay host-only
-	// (machine paths/commands), so they are rejected below.
+	// (machine paths/commands), so they are rejected below. `buildModel` is the
+	// builder-specific override of the general `model` (falls back to `model` when
+	// unset at all levels — see `applyModelFallbacks`); like `model` it is routing
+	// intent (not auth), so it is repo-appropriate.
 	'model',
+	'buildModel',
 	'harness',
 	// Gate 2 (PR/code review) policy is a genuine repo property (GATES spec
 	// `work/specs/tasked/review.md`), resolved per-repo like `integration`/`autoBuild`:
@@ -230,6 +235,14 @@ export const REPO_ALLOWED_KEYS = [
 	'taskerLoop',
 	'taskerLoopMax',
 	'taskerLoopModel',
+	// `triageModel` (the model the advance lifecycle gates — surface, apply,
+	// triage — run on) is a genuine repo property like `reviewModel`/
+	// `taskerLoopModel`: routing intent (not auth), resolved per-repo through the
+	// same chain. `intakeModel` (the front-door intake decision agent's model) is
+	// likewise repo-appropriate. Each falls back to `model` when unset at all
+	// levels (see `applyModelFallbacks`).
+	'triageModel',
+	'intakeModel',
 	// `freshWorktreeGate` (run the acceptance gate against the REBASED tip in a
 	// clean throwaway worktree, ON by default) is a genuine repo property exactly
 	// like `verify`/`prepare`/`review`: whether this repo's gate tests the merged
@@ -646,6 +659,11 @@ export function resolveRepoConfigFromLoaded(
 	// (flag / env / per-repo / global) surfaces the same clear error (ADR
 	// `dorfl-cmd-repo-settable-exception-to-host-only`).
 	validateDorflCmdConfig(config);
+	// Apply the per-role model fallbacks: `buildModel`, `reviewModel`, and
+	// `taskerLoopModel` each inherit the general `model` when unset at EVERY level
+	// (flag / env / per-repo / global). Applied HERE — the resolution FINAL point —
+	// so the fallback uses the FULLY resolved `model`, never an intermediate layer.
+	applyModelFallbacks(config);
 	return {
 		config,
 		rejected: repo.rejected,

package/src/run.ts

@@ -834,8 +834,9 @@ async function runOneItem(
 
 		// 4. Run the agent — via the injected runner (tests) or the harness seam
 		//    (null adapter by default), shelling out to the configured agentCmd. The
-		//    resolved per-repo `model` (ADR §13) flows through the seam to the adapter;
-		//    a `{model}`-in-agentCmd misconfiguration surfaces as agent-failed.
+		//    resolved per-repo `buildModel` (ADR §13, which inherits the general
+		//    `model` when unset) flows through the seam to the adapter; a
+		//    `{model}`-in-agentCmd misconfiguration surfaces as agent-failed.
 		let agent: {ok: boolean; detail?: string; output?: string};
 		try {
 			agent = runAgent(
@@ -844,7 +845,7 @@ async function runOneItem(
 				prompt,
 				slug,
 				config.agentCmd,
-				config.model,
+				config.buildModel,
 				config.sessionsDir,
 			);
 		} catch (err) {

package/src/worktree-writer-lock.ts

@@ -0,0 +1,217 @@
+import {
+	existsSync,
+	mkdirSync,
+	readFileSync,
+	rmSync,
+	writeFileSync,
+} from 'node:fs';
+import {dirname, join} from 'node:path';
+import {run} from './git.js';
+import {pidAlive} from './harness.js';
+import {processGroupAlive} from './reap-agent-tree.js';
+
+/**
+ * **The per-WORKING-TREE writer sentinel** (observation
+ * `checkpoint-releases-lock-while-predecessor-agent-still-writes`).
+ *
+ * The per-item lock (`item-lock.ts`) guards the ITEM: it answers "who owns this
+ * task?" and it is what claim / requeue / bounce move around. Nothing guarded the
+ * WORKING TREE. Those are different resources, and the deadline checkpoint is
+ * exactly where they come apart: the checkpoint releases the item lock so the
+ * next tick can continue the task, but the tree the previous agent was editing is
+ * reused by the successor. If the predecessor is still alive, two agents write to
+ * one tree.
+ *
+ * The primary fix is to reap the predecessor and VERIFY it is gone before the
+ * lock moves (`reap-agent-tree.ts`). This sentinel is the INDEPENDENT backstop:
+ * even if a live writer survives by some route the reap did not cover (a
+ * deliberately `setsid`-ed grandchild, a stale run from a crashed runner, an
+ * operator manually re-driving an item), a second agent physically cannot onboard
+ * into a tree that already has a LIVE holder. It is deliberately keyed on the
+ * TREE, not on the item: two different items sharing one worktree is just as
+ * unsafe as two attempts at the same item.
+ *
+ * ## Where the sentinel lives, and why not in the tree
+ *
+ * It is written to the worktree's PRIVATE git directory (`git rev-parse
+ * --absolute-git-dir`, which for a linked worktree is
+ * `.../.git/worktrees/<name>/`), NOT to a file inside the working tree. That
+ * placement is load-bearing:
+ *
+ *  - it is per-worktree (linked worktrees each get their own git dir), which is
+ *    precisely the granularity we are guarding;
+ *  - it can never appear in `git status`, so it cannot be mistaken for agent work,
+ *    cannot be swept into a commit by a `git add -A`, and needs no new exclusion
+ *    in the empty-diff backstop / `gc`'s cleanliness predicate (unlike
+ *    `.dorfl-job.json`, which each of those has to filter out by name);
+ *  - it is removed with the worktree, so it cannot outlive what it guards.
+ *
+ * ## Liveness, not presence
+ *
+ * A pid file that only records presence becomes a permanent blocker the first
+ * time a runner is `kill -9`ed. So the holder is checked for LIVENESS (its
+ * process group first, falling back to its pid) and a dead holder's sentinel is
+ * treated as stale and taken over. Only a genuinely live foreign writer refuses.
+ */
+
+/** The sentinel filename inside the worktree's private git directory. */
+export const WRITER_SENTINEL_FILENAME = 'dorfl-writer.json';
+
+/** The recorded holder of a worktree's writer sentinel. */
+export interface WorktreeWriter {
+	/** The runner process that owns the agent writing in this tree. */
+	pid: number;
+	/** The agent's process GROUP, when the harness spawned a killable one. */
+	pgid?: number;
+	/** The item being built in this tree (diagnostics: names the other writer). */
+	slug: string;
+	/** ISO timestamp of acquisition (diagnostics: how long it has been held). */
+	startedAt: string;
+}
+
+/** The outcome of trying to become a worktree's sole writer. */
+export type WorktreeWriterLock =
+	| {
+			acquired: true;
+			/** Release the sentinel. Idempotent, and safe if it was already stolen. */
+			release(): void;
+	  }
+	| {
+			acquired: false;
+			/** The LIVE holder that refused us (when it could be parsed). */
+			holder?: WorktreeWriter;
+			/** Human-readable refusal, naming the other writer. */
+			reason: string;
+	  };
+
+/**
+ * The worktree's PRIVATE git directory, or `undefined` when `dir` is not a git
+ * worktree (in which case there is no sentinel location and the caller proceeds
+ * unguarded rather than failing — this is a backstop, not a gate).
+ */
+function worktreeGitDir(
+	dir: string,
+	env: NodeJS.ProcessEnv | undefined,
+): string | undefined {
+	const result = run('git', ['rev-parse', '--absolute-git-dir'], dir, {env});
+	if (result.status !== 0) {
+		return undefined;
+	}
+	const path = result.stdout.trim();
+	return path === '' ? undefined : path;
+}
+
+/** The sentinel path for `dir`, or `undefined` when `dir` is not a worktree. */
+export function writerSentinelPath(
+	dir: string,
+	env?: NodeJS.ProcessEnv,
+): string | undefined {
+	const gitDir = worktreeGitDir(dir, env);
+	return gitDir === undefined
+		? undefined
+		: join(gitDir, WRITER_SENTINEL_FILENAME);
+}
+
+/** Read + parse the sentinel, or `undefined` when absent/corrupt. */
+export function readWorktreeWriter(
+	dir: string,
+	env?: NodeJS.ProcessEnv,
+): WorktreeWriter | undefined {
+	const path = writerSentinelPath(dir, env);
+	if (path === undefined || !existsSync(path)) {
+		return undefined;
+	}
+	try {
+		const parsed = JSON.parse(readFileSync(path, 'utf8')) as WorktreeWriter;
+		return typeof parsed?.pid === 'number' ? parsed : undefined;
+	} catch {
+		// A corrupt sentinel records nothing we can trust; treat it as absent so it
+		// self-heals on the next acquire rather than wedging the worktree forever.
+		return undefined;
+	}
+}
+
+/**
+ * Is the recorded holder still running? Prefers the agent's process GROUP (which
+ * survives the group leader's death and so catches exactly the orphaned-writer
+ * case this exists for), and falls back to the runner pid.
+ */
+export function writerAlive(holder: WorktreeWriter): boolean {
+	if (holder.pgid !== undefined && processGroupAlive(holder.pgid)) {
+		return true;
+	}
+	return pidAlive(holder.pid);
+}
+
+/**
+ * Claim `dir` as the SOLE agent-writable working tree for `slug`.
+ *
+ * Refuses when a DIFFERENT, still-LIVE writer holds it — the second-agent case
+ * the observation describes. A dead holder's sentinel is stale and is taken over
+ * silently (a `kill -9`ed runner must not poison the worktree forever), and our
+ * OWN pid re-acquiring is a no-op re-entry rather than a refusal.
+ *
+ * When `dir` is not a git worktree there is nowhere private to record the
+ * sentinel; that is reported as acquired with a no-op release, because this is a
+ * defence-in-depth backstop and must never become a new way for a legitimate run
+ * to fail.
+ */
+export function acquireWorktreeWriterLock(params: {
+	dir: string;
+	slug: string;
+	/** The agent's process group, when known (the strongest liveness anchor). */
+	pgid?: number;
+	env?: NodeJS.ProcessEnv;
+}): WorktreeWriterLock {
+	const {dir, slug, pgid, env} = params;
+	const path = writerSentinelPath(dir, env);
+	if (path === undefined) {
+		return {acquired: true, release: () => {}};
+	}
+
+	const existing = readWorktreeWriter(dir, env);
+	if (
+		existing !== undefined &&
+		existing.pid !== process.pid &&
+		writerAlive(existing)
+	) {
+		return {
+			acquired: false,
+			holder: existing,
+			reason:
+				`worktree ${dir} already has a LIVE agent writer: pid ${existing.pid}` +
+				(existing.pgid !== undefined ? ` (group ${existing.pgid})` : '') +
+				` building '${existing.slug}' since ${existing.startedAt}. Refusing to ` +
+				`onboard '${slug}' into the same working tree: two live agents in one ` +
+				'tree can clobber each other’s edits, and the second can commit the ' +
+				'first’s half-finished work under a message describing something else. ' +
+				'Wait for it to exit, or kill it, then retry.',
+		};
+	}
+
+	const record: WorktreeWriter = {
+		pid: process.pid,
+		...(pgid !== undefined ? {pgid} : {}),
+		slug,
+		startedAt: new Date().toISOString(),
+	};
+	mkdirSync(dirname(path), {recursive: true});
+	writeFileSync(path, `${JSON.stringify(record, null, 2)}\n`, 'utf8');
+
+	let released = false;
+	return {
+		acquired: true,
+		release: (): void => {
+			if (released) {
+				return;
+			}
+			released = true;
+			// Only remove a sentinel that is still OURS: if it was stolen as stale by
+			// another runner, deleting it would silently un-guard that runner's tree.
+			const current = readWorktreeWriter(dir, env);
+			if (current === undefined || current.pid === process.pid) {
+				rmSync(path, {force: true});
+			}
+		},
+	};
+}