codebay 0.9.1 → 0.10.1

package/src/lib/instances.server.ts

@@ -41,17 +41,32 @@ import {
 	devcontainerCliAvailable,
 	devcontainerUp,
 	INJECTIONS_DONE_FILE,
+	launchCommandFor,
 	readDeclaredContainerPorts,
+	TERMINAL_LAUNCHED_MARKER,
 	writeOverrideConfig
 } from './devcontainer.server.ts';
 import { writeContainerFile } from './container-files.server.ts';
+import { execInContainer, type ExecTarget } from './exec.server.ts';
 import { clearAttention, getAttention } from './bridge.server.ts';
 import { proxyPathFor } from './proxy.server.ts';
-import { resolveInjections } from './injections.server.ts';
+import { resolveInjectionStages } from './injections.server.ts';
 import { cloneRepo, readGitBranch } from './git.server.ts';
 import { isRepoUrl, parseRepoUrl } from './repo-url.ts';
-import { currentHealthSnapshots, stopHealthMonitor, syncHealthMonitors } from './health.server.ts';
-import { pickFreePort } from './ports.server.ts';
+import { getClaudePermissionMode } from '../container-injections/claude-permission-mode.ts';
+import {
+	customEnvVarsConfig,
+	customEnvVarValues
+} from '../container-injections/custom-env-vars.ts';
+import {
+	currentHealthSnapshots,
+	stopHealthMonitor,
+	surfaceAccessible,
+	syncHealthMonitors
+} from './health.server.ts';
+import { runCapturePass, stopLogCapture, syncLogCapture } from './log-capture.server.ts';
+import { isHostPortBindable, pickBindablePort } from './ports.server.ts';
+import { pickAvatar, pickUniqueAvatar } from '../avatars/pick.server.ts';
 import type { ServerWebSocket } from 'bun';
 import {
 	isInstanceFilter,
@@ -61,13 +76,17 @@ import {
 	type InstanceHealth
 } from '../types.ts';
 
+/** The sprite a row shows: its persisted choice, or the id hash for rows created before avatars were stored. */
+const effectiveAvatarName = (row: InstanceRow): string => row.avatar ?? pickAvatar(row.id).name;
+
 /**
- * `bridge_token` authenticates the no-Basic-Auth `/api/bridge/` endpoint, so it
- * must never reach the browser — every row handed to a route goes through here.
+ * `bridge_token` authenticates the no-Basic-Auth `/api/bridge/` endpoint, so it must never reach the
+ * browser — every row handed to a route goes through here. Also resolves the avatar name so the
+ * client renders by name (`findAvatar`) and never needs a hash function of its own.
  */
 export function sanitizeInstance(row: InstanceRow): Omit<InstanceRow, 'bridge_token'> {
 	const { bridge_token: _token, ...rest } = row;
-	return rest;
+	return { ...rest, avatar: effectiveAvatarName(row) };
 }
 
 interface LiveState {
@@ -78,6 +97,27 @@ interface LiveState {
 const globalForReg = globalThis as unknown as { __codebayRegistry?: Map<string, LiveState> };
 const registry: Map<string, LiveState> = (globalForReg.__codebayRegistry ??= new Map());
 
+/**
+ * The ids `provision()` is currently building. A `creating` row is only left alone while its boot
+ * is actually in flight — otherwise a server restart mid-boot would strand it there forever, since
+ * reconcile skips `creating` and `rebuildInstance` refuses to touch it.
+ */
+const globalForBoots = globalThis as unknown as { __codebayProvisioning?: Set<string> };
+const inFlight: Set<string> = (globalForBoots.__codebayProvisioning ??= new Set());
+
+/**
+ * Whether reconcile may re-derive a row's status from Docker. A live boot owns its own row until it
+ * opens the instance itself; an orphaned `creating` row (the manager restarted mid-boot) must not
+ * be left there, since nothing else would ever move it and Rebuild refuses to touch a booting row.
+ */
+export function reconcilesFromDocker(
+	row: Pick<InstanceRow, 'id' | 'status'>,
+	building: ReadonlySet<string>
+): boolean {
+	if (row.status === 'error') return false;
+	return row.status !== 'creating' || !building.has(row.id);
+}
+
 function live(id: string): LiveState {
 	let state = registry.get(id);
 	if (!state) {
@@ -87,7 +127,36 @@ function live(id: string): LiveState {
 	return state;
 }
 
+/**
+ * Cached so the redaction on every log chunk doesn't re-read the DB. Rebuilt lazily after the
+ * settings action invalidates it; values shorter than 4 chars are skipped so a trivially short
+ * secret can't blank out unrelated log text, and longest-first avoids a short value masking a
+ * substring of a longer one.
+ */
+let secretValuesCache: string[] | null = null;
+
+function getSecretValues(): string[] {
+	if (secretValuesCache === null) {
+		secretValuesCache = customEnvVarValues()
+			.filter((v) => v.length >= 4)
+			.sort((a, b) => b.length - a.length);
+	}
+	return secretValuesCache;
+}
+
+/** Call after the custom-env-vars setting changes so newly-added values redact from the next boot. */
+export function invalidateSecretValues(): void {
+	secretValuesCache = null;
+}
+
+/** Exported for tests; the boot flow reaches it only through `appendLog`. */
+export function redactSecrets(text: string): string {
+	for (const value of getSecretValues()) text = text.split(value).join('••••');
+	return text;
+}
+
 function appendLog(id: string, chunk: string): void {
+	chunk = redactSecrets(chunk);
 	const state = live(id);
 	state.logs.push(chunk);
 	if (state.logs.length > 2000) state.logs.splice(0, state.logs.length - 2000);
@@ -255,7 +324,10 @@ async function allocatePort(): Promise<number> {
 	for (const [port, reservedAt] of reservedPorts) {
 		if (dbPorts.has(port) || now - reservedAt > RESERVATION_TTL_MS) reservedPorts.delete(port);
 	}
-	const port = pickFreePort([dbPorts, dockerPorts, new Set(reservedPorts.keys())]);
+	const port = await pickBindablePort(
+		[dbPorts, dockerPorts, new Set(reservedPorts.keys())],
+		isHostPortBindable
+	);
 	reservedPorts.set(port, now);
 	return port;
 }
@@ -270,6 +342,10 @@ async function assertDir(path: string): Promise<void> {
 	if (!info.isDirectory()) throw new Error(`Not a folder: ${path}`);
 }
 
+function elapsed(startMs: number): string {
+	return `${((Date.now() - startMs) / 1000).toFixed(1)}s`;
+}
+
 function failInstance(id: string, err: unknown): void {
 	const message = (err as Error).message;
 	updateInstance(id, { status: 'error', error: message });
@@ -278,19 +354,26 @@ function failInstance(id: string, err: unknown): void {
 }
 
 async function boot(row: InstanceRow, opts: { branch?: string } = {}): Promise<void> {
+	// Claims the row for the copy/clone phase, which runs ahead of the build; provision() re-claims
+	// it and owns the release from there.
+	inFlight.add(row.id);
 	try {
+		const sourceStart = Date.now();
 		if (isRepoUrl(row.source_path)) {
 			appendLog(row.id, `Cloning ${row.source_path} → ${row.workspace_path}\n`);
 			await cloneRepo(row.source_path, row.workspace_path, (chunk) => appendLog(row.id, chunk), {
 				branch: opts.branch
 			});
+			appendLog(row.id, `⏱ clone: ${elapsed(sourceStart)}\n`);
 		} else {
 			appendLog(row.id, `Copying ${row.source_path} → ${row.workspace_path}\n`);
 			const ignore = parseCopyIgnore(getOption('copy_ignore_patterns') ?? DEFAULT_COPY_IGNORE);
 			await copyWorkspace(row.source_path, row.workspace_path, ignore);
+			appendLog(row.id, `⏱ copy: ${elapsed(sourceStart)}\n`);
 		}
 		await seedDeclaredPorts(row);
 	} catch (err) {
+		inFlight.delete(row.id);
 		failInstance(row.id, err);
 		return;
 	}
@@ -314,22 +397,55 @@ async function seedDeclaredPorts(row: InstanceRow): Promise<void> {
 	}
 }
 
+const surfaceLabel = (mode: InstanceMode) =>
+	mode === 'terminal' ? 'ttyd terminal' : 'code-server';
+
+/**
+ * Moves an instance off a host port that something outside Docker has taken over, which otherwise
+ * strands it forever — the container publishes fine but nothing reaches it from the host. Requires
+ * the port to be *both* unbindable and dead, because a rebuild doesn't stop the old container
+ * first: a healthy instance's port is always unbindable here, and its surface still answers.
+ */
+async function rescueHijackedPort(row: InstanceRow): Promise<void> {
+	if (await isHostPortBindable(row.host_port)) return;
+	if (await surfaceAccessible(row.host_port)) return;
+	const stale = row.host_port;
+	const port = await allocatePort();
+	updateInstance(row.id, { host_port: port });
+	row.host_port = port;
+	appendLog(
+		row.id,
+		`⚠ Host port ${stale} is held by another process and isn't reachable — moving to ${port}\n`
+	);
+}
+
 /** Never re-copies the workspace, so in-container edits survive a rebuild. */
 async function provision(row: InstanceRow, opts: { noCache?: boolean } = {}): Promise<void> {
+	// Held so the `catch` can release the container-side launcher's wait on a boot that dies.
+	let containerId: string | null = null;
+	inFlight.add(row.id);
 	try {
+		// On a rebuild `devcontainer up --remove-existing-container` discards the old container's
+		// ~/.claude, so drain it first; a first boot has no container_id and skips straight past.
+		if (row.container_id) {
+			stopLogCapture(row.id);
+			await runCapturePass(row).catch(() => undefined);
+		}
+		await rescueHijackedPort(row);
 		const forwards = listForwards(row.id).map((f) => ({
 			container_port: f.container_port,
 			host_port: f.host_port
 		}));
-		const surface = row.mode === 'terminal' ? 'ttyd terminal' : 'code-server';
-		appendLog(row.id, `Injecting ${surface} (host port ${row.host_port})\n`);
+		appendLog(row.id, `Injecting ${surfaceLabel(row.mode)} (host port ${row.host_port})\n`);
 		const defaultImage = getOption('default_image') ?? DEFAULT_IMAGE;
 		const { imageSource } = await writeOverrideConfig(
 			row.workspace_path,
 			row.host_port,
 			forwards,
 			defaultImage,
-			row.mode
+			row.mode,
+			getClaudePermissionMode(),
+			customEnvVarsConfig()?.vars ?? []
 		);
 		updateInstance(row.id, { image_source: imageSource });
 
@@ -337,9 +453,11 @@ async function provision(row: InstanceRow, opts: { noCache?: boolean } = {}): Pr
 		if (noCache) appendLog(row.id, `Building without cache (--build-no-cache)\n`);
 
 		appendLog(row.id, `Starting devcontainer…\n`);
+		const upStart = Date.now();
 		const result = await devcontainerUp(row.workspace_path, (chunk) => appendLog(row.id, chunk), {
 			noCache
 		});
+		appendLog(row.id, `⏱ devcontainer up: ${elapsed(upStart)}\n`);
 
 		if (result.outcome !== 'success' || !result.containerId) {
 			throw new Error(
@@ -347,11 +465,13 @@ async function provision(row: InstanceRow, opts: { noCache?: boolean } = {}): Pr
 			);
 		}
 
+		containerId = result.containerId;
+		// The row deliberately stays `creating` until the injections below finish: `running` is what
+		// opens the IDE/terminal to the user, and a container whose setup is still landing isn't ready.
 		updateInstance(row.id, {
 			container_id: result.containerId,
 			remote_workspace_folder: result.remoteWorkspaceFolder ?? null,
 			remote_user: result.remoteUser ?? null,
-			status: 'running',
 			error: null
 		});
 
@@ -360,34 +480,66 @@ async function provision(row: InstanceRow, opts: { noCache?: boolean } = {}): Pr
 		row.container_id = result.containerId;
 		row.remote_workspace_folder = result.remoteWorkspaceFolder ?? null;
 		row.remote_user = result.remoteUser ?? null;
-		row.status = 'running';
 
 		const target = {
 			containerId: result.containerId,
 			remoteUser: result.remoteUser,
 			instance: row
 		};
-		// Swallow per-injection failures so one bad injection can't abort the rest of provisioning.
-		for (const injection of resolveInjections(row.mode)) {
-			try {
-				await injection.apply(target, (msg) => appendLog(row.id, msg));
-			} catch (err) {
-				appendLog(row.id, `⚠ ${injection.label} injection failed: ${(err as Error).message}\n`);
-			}
+		// Stages parallelize independent injections; per-injection failures are still swallowed
+		// so one bad injection can't abort the rest of provisioning.
+		const injectionsStart = Date.now();
+		for (const stage of resolveInjectionStages(row.mode)) {
+			await Promise.all(
+				stage.map(async (injection) => {
+					try {
+						await injection.apply(target, (msg) => appendLog(row.id, msg));
+					} catch (err) {
+						appendLog(row.id, `⚠ ${injection.label} injection failed: ${(err as Error).message}\n`);
+					}
+				})
+			);
 		}
+		appendLog(row.id, `⏱ injections: ${elapsed(injectionsStart)}\n`);
 
 		// Unblocks the terminal launcher's wait; written even after ⚠s — its timeout is the fallback.
-		await writeContainerFile(target, { dir: '$HOME', name: INJECTIONS_DONE_FILE }, 'done\n').catch(
-			() => undefined
-		);
+		await writeInjectionsDone(row.id, target);
+
+		// postStartCommand ran before the injections above, and its launcher bails when the binary
+		// is missing — so a container whose build-time ttyd install failed got it too late to be
+		// started. No-ops (one accessibility probe) whenever postStart already brought the port up.
+		await relaunchSurface(row);
+
+		// Everything the instance needs is in place, so open it to the user: this is what lights up
+		// Open IDE/terminal, the proxy, and the health and log-capture monitors.
+		updateInstance(row.id, { status: 'running' });
+		row.status = 'running';
 
 		appendLog(row.id, `\n✓ Instance running — open it via the proxy at ${proxyPathFor(row.id)}\n`);
 		triggerReconcile();
 	} catch (err) {
+		// The launcher waits minutes for the sentinel, so a boot that will never write it must say so.
+		if (containerId)
+			await writeInjectionsDone(row.id, { containerId, remoteUser: row.remote_user });
 		failInstance(row.id, err);
+	} finally {
+		inFlight.delete(row.id);
 	}
 }
 
+/**
+ * `writeContainerFile` reports failure by return value rather than throwing, and a sentinel that
+ * never lands leaves the launcher waiting out its whole bound — so surface it in the boot log.
+ */
+async function writeInjectionsDone(id: string, target: ExecTarget): Promise<void> {
+	const res = await writeContainerFile(
+		target,
+		{ dir: '$HOME', name: INJECTIONS_DONE_FILE },
+		'done\n'
+	).catch((err: Error) => ({ ok: false, error: err.message }));
+	if (!res.ok) appendLog(id, `⚠ Could not mark setup complete in the container: ${res.error}\n`);
+}
+
 /** The first instance keeps the bare name; later collisions get a `#2`, `#3`, … suffix. */
 function uniqueName(desired: string, excludeId?: string): string {
 	const taken = new Set(
@@ -437,6 +589,8 @@ export async function createInstance(
 		bridge_token: crypto.randomUUID().replace(/-/g, ''),
 		remote_user: null,
 		image_source: null,
+		// Kept synchronous through insertInstance so two concurrent creations can't claim the same free sprite.
+		avatar: pickUniqueAvatar(id, allInstances().map(effectiveAvatarName)).name,
 		mode: opts.mode ?? getDefaultMode(),
 		terminal_split: 0
 	};
@@ -455,10 +609,19 @@ export async function listInstances(): Promise<Instance[]> {
 	const branches = new Map<string, string | null>();
 	await Promise.all(
 		rows.map(async (row) => {
-			if (!row.container_id || row.status === 'creating' || row.status === 'error') return;
+			if (!row.container_id || !reconcilesFromDocker(row, inFlight)) return;
 			const running = await isRunning(row.container_id);
 			const next: InstanceStatus = running ? 'running' : 'stopped';
 			if (next !== row.status) {
+				// Only reachable for a `creating` row whose boot died with the manager, and its
+				// injections will never finish — so stop the container-side launcher waiting on them.
+				if (row.status === 'creating') {
+					appendLog(row.id, `\n⚠ Setup was interrupted — Rebuild to run it again\n`);
+					void writeInjectionsDone(row.id, {
+						containerId: row.container_id,
+						remoteUser: row.remote_user
+					});
+				}
 				updateInstance(row.id, { status: next });
 				row.status = next;
 			}
@@ -468,6 +631,7 @@ export async function listInstances(): Promise<Instance[]> {
 	);
 	// Reconcile is the only place that knows the current running set.
 	syncHealthMonitors(rows);
+	syncLogCapture(rows);
 	// Distinguishes a forward that's live from one still pending a rebuild.
 	const openPorts = new Map<string, Set<number>>();
 	for (const { id, health } of currentHealthSnapshots()) {
@@ -488,6 +652,9 @@ export async function listInstances(): Promise<Instance[]> {
 	}
 	return rows.map((row) => ({
 		...sanitizeInstance(row),
+		// Resolve legacy null rows to their hashed sprite here so the client only sees an
+		// unresolvable name when a persisted sprite was actually removed from the catalog.
+		avatar: effectiveAvatarName(row),
 		git_branch: branches.get(row.id) ?? null,
 		attention: getAttention(row.id),
 		forwarded_ports: forwards.get(row.id) ?? []
@@ -560,7 +727,10 @@ export function removeForwardedPort(id: string, containerPort: number): Instance
 export function rebuildInstance(id: string, opts: { noCache?: boolean } = {}): InstanceRow {
 	const row = getInstance(id);
 	if (!row) throw new Error('Instance not found');
-	if (row.status === 'creating') return row; // a build is already in flight
+	if (inFlight.has(id)) return row; // a build is already in flight
+	// Claimed before the broadcast below, which reconciles synchronously: an unclaimed `creating`
+	// row reads as orphaned, and reconcile would flip it straight back off `creating`.
+	inFlight.add(id);
 	updateInstance(id, { status: 'creating', error: null });
 	triggerReconcile();
 	const fresh = getInstance(id)!;
@@ -579,11 +749,51 @@ export function rebuildRunningInstancesNoCache(): number {
 	return running.length;
 }
 
+/** Single-quote a path for the shell, so a workspace folder with spaces survives the `cd`. */
+function quote(value: string): string {
+	return `'${value.replaceAll("'", `'\\''`)}'`;
+}
+
+/**
+ * `postStartCommand` is a devcontainer-CLI lifecycle hook, not a container entrypoint, so a plain
+ * `docker start` never re-runs it and the ttyd daemon it backgrounded is gone. IDE mode only
+ * survives because the code-server feature ships an entrypoint of its own; ttyd has no equivalent.
+ * Both launchers are guarded (ttyd by process name, code-server by its port), so re-running one
+ * against a healthy container is a no-op.
+ */
+export async function relaunchSurface(row: InstanceRow): Promise<void> {
+	if (!row.container_id) return;
+
+	const steps: string[] = [];
+	// The folderOpen task's run-once gate is meant to span one container run, but the marker file
+	// outlives it — left in place, a restarted IDE container never reopens the Claude terminal.
+	if (row.mode !== 'terminal') steps.push(`rm -f "$HOME/${TERMINAL_LAUNCHED_MARKER}"`);
+	// code-server comes back on its own (its feature ships a container entrypoint), so only launch
+	// a surface that is actually down rather than racing the entrypoint for the port.
+	if (!(await surfaceAccessible(row.host_port))) steps.push(launchCommandFor(row.mode));
+	if (!steps.length) return;
+
+	// Both launchers resolve paths off `$PWD`, which under postStartCommand is the workspace
+	// folder; an exec only inherits the image's WorkingDir, so re-establish it explicitly.
+	const cd = row.remote_workspace_folder
+		? `cd ${quote(row.remote_workspace_folder)} || exit 1; `
+		: '';
+	const res = await execInContainer(
+		{ containerId: row.container_id, remoteUser: row.remote_user },
+		{ script: cd + steps.join('; ') }
+	);
+	if (!res.ok) {
+		appendLog(row.id, `⚠ Could not relaunch ${surfaceLabel(row.mode)}: ${res.error}\n`);
+	}
+}
+
 export async function startInstance(id: string): Promise<InstanceRow> {
 	const row = getInstance(id);
 	if (!row) throw new Error('Instance not found');
 	if (!row.container_id) throw new Error('Instance has no container yet');
 	const ok = await startContainer(row.container_id);
+	// Awaited so the first health tick after the reconcile below already sees a listening port.
+	if (ok) await relaunchSurface(row);
 	updateInstance(id, {
 		status: ok ? 'running' : 'error',
 		error: ok ? null : 'Failed to start container'
@@ -595,6 +805,9 @@ export async function startInstance(id: string): Promise<InstanceRow> {
 export async function stopInstance(id: string): Promise<InstanceRow> {
 	const row = getInstance(id);
 	if (!row) throw new Error('Instance not found');
+	// A later delete can't exec into a stopped container, so this is the last chance to mirror.
+	stopLogCapture(id);
+	if (row.container_id) await runCapturePass(row).catch(() => undefined);
 	const ok = row.container_id ? await stopContainer(row.container_id) : true;
 	updateInstance(id, {
 		status: ok ? 'stopped' : 'error',
@@ -608,12 +821,19 @@ export async function stopInstance(id: string): Promise<InstanceRow> {
 export async function deleteInstance(id: string): Promise<void> {
 	const row = getInstance(id);
 	if (!row) return;
-	if (row.container_id) await removeContainer(row.container_id);
+	stopHealthMonitor(id);
+	// Mirror the tail of the session before the transcripts die with the container; the periodic
+	// chain is stopped first so it can't race this last pass. Retention outlives the instance —
+	// <LOGS_DIR>/*-<id>.jsonl is deliberately left on disk.
+	stopLogCapture(id);
+	if (row.container_id) {
+		await runCapturePass(row).catch(() => undefined);
+		await removeContainer(row.container_id);
+	}
 	await rm(join(INSTANCES_DIR, id), { recursive: true, force: true });
 	deleteForwards(id);
 	deleteInstanceRow(id);
 	registry.delete(id);
-	stopHealthMonitor(id);
 	clearAttention(id);
 	triggerReconcile();
 }