headlesscode 1.0.2 → 1.2.0

package/src/tools/executor.ts

@@ -190,6 +190,8 @@ export interface ToolExecutorOptions {
 	permissions?: PermissionsConfig
 	/** See ToolContext.guardLargeOverwrites (types.ts) for the full writeup. */
 	guardLargeOverwrites?: boolean
+	/** See ToolContext.disableReadFileCache (types.ts) for the full writeup. */
+	disableReadFileCache?: boolean
 	/**
 	 * Live worker monitoring: fired at the same lifecycle points where the
 	 * `.harness.needs-decision` marker is written/cleared, so the session can
@@ -359,6 +361,7 @@ export class ToolExecutor {
 				workspaceRoot: this.workspaceRoot,
 				permissions: this.permissions,
 				guardLargeOverwrites: this.options.guardLargeOverwrites,
+				disableReadFileCache: this.options.disableReadFileCache,
 				decisionTimeoutMs: this.options.decisionTimeoutMs,
 				decisionPollIntervalMs: this.options.decisionPollIntervalMs,
 				pauseBudgetClock: this.pauseBudgetClock,
@@ -799,7 +802,32 @@ function readFileHandler(
 		// cached hash can be reused without re-reading the file; any mismatch
 		// (including a same-length rewrite, which changes mtime) falls back to
 		// a full read + sha256 below.
+		//
+		// 2026-09-02: real, confirmed, live-observed failure mode of this
+		// short-circuit against a local model -- ctx.disableReadFileCache
+		// skips both cache-hit checks in this function entirely, always
+		// serving real content. A session's edit_file call failed ("no
+		// match found"), the error told it to re-read and retry, it DID
+		// call read_file again exactly as instructed, and got back
+		// "[cache] this file is unchanged... re-read the earlier tool
+		// result" instead of the actual content -- correct per this
+		// mechanism's own design (the safety valve is "the SECOND
+		// consecutive identical call serves real content again"), but the
+		// model never made that second call: it read the cache-hit message
+		// as "you already have what you need", gave up, and called
+		// attempt_completion claiming the endpoint worked -- a genuine
+		// fabrication directly caused by this response, not a model
+		// hallucination from nothing. This short-circuit's own rationale
+		// (avoid paying full token cost for content the conversation
+		// already has) is a real concern for a REMOTE model's per-token API
+		// bill; re-serving a few hundred lines of file content costs a
+		// local session near-nothing (it's prefill, not generation, so it
+		// barely affects wall-clock time either) against a GPU with no
+		// per-token price. Cheap insurance against a much more expensive
+		// failure mode locally; the cloud sessions this genuinely saves
+		// money for are unaffected (disableReadFileCache stays unset there).
 		if (
+			!ctx.disableReadFileCache &&
 			cache !== undefined &&
 			!cache.toldUnchanged &&
 			cache.size === stat.size &&
@@ -818,8 +846,9 @@ function readFileHandler(
 
 		// Cache-check the CURRENT on-disk content (never "no write tool was
 		// called"): identical args + identical hash => byte-identical output.
+		// See the disableReadFileCache comment on the check above.
 		const currentHash = hashFileContent(content)
-		if (cache !== undefined && cache.hash === currentHash && !cache.toldUnchanged) {
+		if (!ctx.disableReadFileCache && cache !== undefined && cache.hash === currentHash && !cache.toldUnchanged) {
 			cache.toldUnchanged = true
 			return ok(READ_FILE_CACHE_HIT_MESSAGE)
 		}
@@ -1565,9 +1594,48 @@ function executeCommandHandler(args: Record<string, unknown>, ctx: ToolContext):
 
 	return (async () => {
 		let cwd = ctx.workspaceRoot
-		if (args.cwd != null && args.cwd !== "") {
+		// 2026-09-02: real, confirmed, fully deterministic bug -- a local
+		// LoRA backend's tool-call parser was found sending the literal
+		// 4-character STRING "null" (or "None") for an unset optional
+		// `cwd`, not a real absent/null value. `args.cwd != null` is only
+		// false for the JS primitive null -- a non-empty string "null"
+		// sails through this check as if it were a real requested cwd,
+		// resolving to "<workspaceRoot>/null", which (almost) never
+		// exists. That was fixed at the source (the LoRA server's own
+		// parser), but defend here too: any backend/model can make the
+		// same JSON-serialization slip, and treating the literal text
+		// null/None the same as a real null is cheap and unambiguous --
+		// no real directory is ever named exactly "null" or "None".
+		const isNullish = args.cwd == null || args.cwd === "" || args.cwd === "null" || args.cwd === "None"
+		if (!isNullish) {
 			const cwdArg = requireString(args, "cwd")
 			cwd = safeTarget(ctx, cwdArg)
+			// Validate BEFORE spawning rather than let a bad cwd reach
+			// spawn(): Node's child_process misattributes a chdir()
+			// failure (nonexistent cwd) to the SHELL itself -- "spawn
+			// /bin/bash ENOENT" -- with no mention of cwd anywhere in the
+			// error, which is exactly the "infrastructure spawn failure"
+			// pattern several ground-truth eval runs hit today (verified
+			// directly: spawning with a nonexistent cwd reproduces that
+			// precise error string/code/path). That misattribution isn't
+			// specific to the null-string bug above -- ANY nonexistent
+			// cwd the model supplies (a stale/hallucinated path) hits it
+			// the same way. Checking here turns a misleading spawn crash
+			// into a clear, actionable, model-facing error instead, and
+			// correctly counts it as a real mistake (it's the model's
+			// bad path, not infrastructure).
+			let cwdStat: fs.Stats | undefined
+			try {
+				cwdStat = fs.statSync(cwd)
+			} catch {
+				cwdStat = undefined
+			}
+			if (cwdStat === undefined || !cwdStat.isDirectory()) {
+				const rel = path.relative(ctx.workspaceRoot, cwd).toPosix() || path.basename(cwd)
+				return err(
+					`execute_command: cwd '${rel}' does not exist in this workspace.\n\nRecovery suggestions:\n1. Use list_files to confirm the real directory structure before setting cwd\n2. Omit cwd (or pass null) to run in the workspace root\n3. If you meant a path from a different task/workspace, it doesn't exist here`,
+				)
+			}
 		}
 
 		// Permissions gate (command allow/deny + dangerous substitution +
@@ -1695,6 +1763,7 @@ function executeCommandHandler(args: Record<string, unknown>, ctx: ToolContext):
 						env: { ...process.env, LANG: "en_US.UTF-8", LC_ALL: "en_US.UTF-8" },
 					})
 				} catch (error) {
+					logSpawnDiagnostics(command, attempt, "sync-catch", error)
 					if (attempt < MAX_SPAWN_ATTEMPTS && isBashSpawnEnoent(error)) {
 						// A live standalone repro of one of these exact failures spawned
 						// cleanly on the first try outside the long-running harness
@@ -1757,6 +1826,7 @@ function executeCommandHandler(args: Record<string, unknown>, ctx: ToolContext):
 					attemptSettled = true
 					clearTimeout(timer)
 					backgroundCommands.delete(child)
+					logSpawnDiagnostics(command, attempt, "error-event", error)
 					if (attempt < MAX_SPAWN_ATTEMPTS && isBashSpawnEnoent(error)) {
 						stdout = ""
 						stderr = ""
@@ -1787,11 +1857,17 @@ function executeCommandHandler(args: Record<string, unknown>, ctx: ToolContext):
 				// no other real scenario (a genuine command exit code is always
 				// 0-255), so treat any negative code on the first attempt the same
 				// way: retry once before surfacing anything to the model.
-				if (attempt < MAX_SPAWN_ATTEMPTS && code !== null && code < 0) {
-					stdout = ""
-					stderr = ""
+				if (code !== null && code < 0) {
+					// The same bash-spawn-ENOENT failure surfaced via `close`
+					// (no `error` event). Snapshot system state at the moment of
+					// the failure, before any retry backoff begins (issue #1).
+					logSpawnDiagnostics(command, attempt, "close-negative-code", undefined, code)
+					if (attempt < MAX_SPAWN_ATTEMPTS) {
+						stdout = ""
+						stderr = ""
 						setTimeout(() => spawnAttempt(attempt + 1), SPAWN_RETRY_DELAY_MS * attempt)
-					return
+						return
+					}
 				}
 				if (signal === "SIGKILL") {
 					// Defensive path only: a timeout never sends SIGKILL anymore,
@@ -1881,7 +1957,15 @@ function listFilesHandler(
 		const key = `${target} ${recursive}`
 		const entry = calls?.get(key)
 
-		if (calls !== undefined && generation !== undefined && entry !== undefined && entry.generation === generation) {
+		// See readFileHandler's disableReadFileCache comment for the full
+		// rationale (same short-circuit shape, same local-backend risk).
+		if (
+			!ctx.disableReadFileCache &&
+			calls !== undefined &&
+			generation !== undefined &&
+			entry !== undefined &&
+			entry.generation === generation
+		) {
 			if (!entry.toldUnchanged) {
 				entry.toldUnchanged = true
 				return ok(
@@ -2330,6 +2414,147 @@ function errorMessage(error: unknown): string {
 	return error instanceof Error ? error.message : String(error)
 }
 
+/**
+ * Spawn-failure diagnostics snapshot (issue #1).
+ *
+ * The bash-spawn-ENOENT failure ("spawn /bin/bash ENOENT", and the
+ * close-event variant with a negative code) has one CONFIRMED contributing
+ * factor — system memory commit pressure: when /proc/meminfo's Committed_AS
+ * sits above ~95-98% of CommitLimit, spawn failure rates spike — plus several
+ * unconfirmed candidates (open-fd exhaustion, the ulimit -u process-count
+ * ceiling, zombie/defunct accumulation, RSS growth in the long-running
+ * harness process). Rather than infer the cause after the fact from separate
+ * /proc snapshots, capture a one-shot snapshot AT the moment of each spawn
+ * failure. Every read is best-effort: a missing /proc file (non-Linux host,
+ * restricted container) yields "n/a" for that field instead of throwing, and
+ * a failure here must never abort the spawn-retry logic this is diagnosing.
+ */
+export function collectSpawnDiagnostics(): Record<string, string> {
+	const diag: Record<string, string> = {}
+
+	// Memory commit pressure — the confirmed contributor. Values in kB.
+	let committedAsKb = "n/a"
+	let commitLimitKb = "n/a"
+	try {
+		const meminfo = fs.readFileSync("/proc/meminfo", "utf8")
+		for (const line of meminfo.split("\n")) {
+			if (line.startsWith("Committed_AS:")) {
+				committedAsKb = (line.split(/\s+/)[1] ?? "n/a").trim()
+			} else if (line.startsWith("CommitLimit:")) {
+				commitLimitKb = (line.split(/\s+/)[1] ?? "n/a").trim()
+			}
+		}
+	} catch {
+		// non-Linux or /proc not mounted — fields stay "n/a"
+	}
+	diag.committedAsKb = committedAsKb
+	diag.commitLimitKb = commitLimitKb
+	const committed = Number(committedAsKb)
+	const limit = Number(commitLimitKb)
+	diag.committedPct =
+		Number.isFinite(committed) && Number.isFinite(limit) && limit > 0
+			? `${((committed / limit) * 100).toFixed(1)}%`
+			: "n/a"
+
+	// Open fd count for THIS process — fd exhaustion near the soft limit is a
+	// candidate contributor (posix_spawn needs fds for the child's stdio).
+	try {
+		diag.openFds = String(fs.readdirSync("/proc/self/fd").length)
+	} catch {
+		diag.openFds = "n/a"
+	}
+
+	// System process count + zombie/defunct count + the ulimit -u ceiling.
+	// The zombie scan is bounded to the first 1024 pids so the diagnostic
+	// stays lightweight even on a box with tens of thousands of processes —
+	// this runs ON a spawn-failure path and must never make it slower.
+	let procs = 0
+	let zombies = 0
+	try {
+		const pids = fs.readdirSync("/proc").filter((e) => /^\d+$/.test(e))
+		procs = pids.length
+		for (const pid of pids.slice(0, 1024)) {
+			try {
+				const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8")
+				// comm can contain spaces and parens — the state char is the
+				// first field after the LAST ')', two chars later.
+				const closeParen = stat.lastIndexOf(")")
+				if (closeParen !== -1 && closeParen + 2 < stat.length && stat[closeParen + 2] === "Z") {
+					zombies++
+				}
+			} catch {
+				// pid exited between readdir and read — not a zombie
+			}
+		}
+	} catch {
+		// non-Linux — both stay at their defaults
+	}
+	diag.systemProcs = procs > 0 ? String(procs) : "n/a"
+	diag.zombies = procs > 0 ? String(zombies) : "n/a"
+	let maxProcs = "n/a"
+	try {
+		const limits = fs.readFileSync("/proc/self/limits", "utf8")
+		const m = limits.match(/Max processes\s+(\d+)/)
+		if (m) {
+			maxProcs = m[1]!
+		}
+	} catch {
+		// non-Linux
+	}
+	diag.maxProcs = maxProcs
+
+	// The long-running harness's own footprint — RSS climbs slowly over hours
+	// (the observed llama-server pattern); heap/uptime come from process.*.
+	let rssMb = "n/a"
+	try {
+		const status = fs.readFileSync("/proc/self/status", "utf8")
+		const m = status.match(/VmRSS:\s+(\d+) kB/)
+		if (m) {
+			rssMb = `${(Number(m[1]!) / 1024).toFixed(0)}`
+		}
+	} catch {
+		// non-Linux
+	}
+	diag.rssMb = rssMb
+	diag.heapMb = `${Math.round(process.memoryUsage().heapUsed / (1024 * 1024))}`
+	diag.uptimeS = `${Math.round(process.uptime())}`
+
+	return diag
+}
+
+/** Render a diagnostics snapshot as one grep-friendly `key=value` line. */
+export function formatSpawnDiagnostics(diag: Record<string, string>): string {
+	return [
+		`committed=${diag.committedPct} (${diag.committedAsKb}/${diag.commitLimitKb} kB)`,
+		`openFds=${diag.openFds}`,
+		`procs=${diag.systemProcs}`,
+		`zombies=${diag.zombies}`,
+		`maxProcs=${diag.maxProcs}`,
+		`rss=${diag.rssMb}MB`,
+		`heap=${diag.heapMb}MB`,
+		`uptime=${diag.uptimeS}s`,
+	].join(" ")
+}
+
+/**
+ * One-line, always-on stderr log emitted at the moment of a spawn failure.
+ * Unconditional (NOT HEADLESSCODE_DEBUG-gated): the whole point is that the
+ * NEXT real-session occurrence gets captured without anyone having to
+ * remember to enable a flag first.
+ */
+function logSpawnDiagnostics(command: string, attempt: number, shape: string, error?: unknown, code?: number | null): void {
+	const detail =
+		error !== undefined
+			? ` error=${errorMessage(error)}`
+			: code !== undefined
+				? ` code=${code}`
+				: ""
+	const shown = command.length > 200 ? `${command.slice(0, 200)}…` : command
+	process.stderr.write(
+		`[execute_command spawn-diagnostics] shape=${shape} attempt=${attempt}${detail} cmd=${JSON.stringify(shown)} ${formatSpawnDiagnostics(collectSpawnDiagnostics())}\n`,
+	)
+}
+
 /**
  * Lazily construct the session's local summarizer. Returns undefined when the
  * feature is off (never construct an Ollama client for a non-opted-in