@roj-ai/sdk 0.1.33 → 0.1.35

package/src/plugins/services/pid-registry.test.ts

@@ -0,0 +1,180 @@
+import { describe, expect, it } from 'bun:test'
+import { spawn } from 'node:child_process'
+import { readdir, readFile, mkdtemp, rm, writeFile, mkdir } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { silentLogger } from '~/lib/logger/logger.js'
+import { createNodePlatform } from '~/testing/node-platform.js'
+import { ServicePidRegistry } from './pid-registry.js'
+import { getProcessStartTime } from './service.js'
+
+const fs = createNodePlatform().fs
+
+async function withDataDir<T>(fn: (dataPath: string) => Promise<T>): Promise<T> {
+	const dataPath = await mkdtemp(join(tmpdir(), 'pid-registry-'))
+	try {
+		return await fn(dataPath)
+	} finally {
+		await rm(dataPath, { recursive: true, force: true })
+	}
+}
+
+/** A detached `sleep`, so `kill(-pid)` has a real process group to take down. */
+function spawnVictim(): { pid: number; exited: Promise<void> } {
+	const child = spawn('/bin/sh', ['-c', 'sleep 60'], { detached: true, stdio: 'ignore' })
+	child.unref()
+	if (!child.pid) throw new Error('failed to spawn victim')
+	const exited = new Promise<void>(resolve => child.on('close', () => resolve()))
+	return { pid: child.pid, exited }
+}
+
+function isAlive(pid: number): boolean {
+	try {
+		process.kill(pid, 0)
+		return true
+	} catch {
+		return false
+	}
+}
+
+/** A pid the kernel cannot have handed out, so it reads as a dead agent. */
+async function unusablePid(): Promise<number> {
+	const max = Number((await readFile('/proc/sys/kernel/pid_max', 'utf-8')).trim())
+	return max + 1
+}
+
+async function writeRecord(dataPath: string, record: Record<string, unknown>): Promise<string> {
+	const dir = join(dataPath, 'service-pids')
+	await mkdir(dir, { recursive: true })
+	const path = join(dir, `${record.sessionId}__${record.serviceType}.json`)
+	await writeFile(path, JSON.stringify(record))
+	return path
+}
+
+const makeRegistry = (dataPath: string) => new ServicePidRegistry(fs, silentLogger, dataPath)
+
+describe('ServicePidRegistry', () => {
+	it('kills a service left behind by an agent that is gone, and drops the record', async () => {
+		await withDataDir(async dataPath => {
+			const victim = spawnVictim()
+			await writeRecord(dataPath, {
+				sessionId: 'sess-1',
+				serviceType: 'astro-dev',
+				pid: victim.pid,
+				pidStartTime: await getProcessStartTime(fs, victim.pid),
+				command: 'sleep 60',
+				agentPid: await unusablePid(),
+				agentStartTime: 12345,
+			})
+
+			await makeRegistry(dataPath).sweepOrphans()
+
+			await victim.exited
+			expect(isAlive(victim.pid)).toBe(false)
+			expect(await readdir(join(dataPath, 'service-pids'))).toEqual([])
+		})
+	})
+
+	it('leaves a service whose agent is still running', async () => {
+		await withDataDir(async dataPath => {
+			const victim = spawnVictim()
+			try {
+				// This test process stands in for a second, still-live agent.
+				await writeRecord(dataPath, {
+					sessionId: 'sess-1',
+					serviceType: 'astro-dev',
+					pid: victim.pid,
+					pidStartTime: await getProcessStartTime(fs, victim.pid),
+					command: 'sleep 60',
+					agentPid: process.pid,
+					agentStartTime: await getProcessStartTime(fs, process.pid),
+				})
+
+				await makeRegistry(dataPath).sweepOrphans()
+
+				expect(isAlive(victim.pid)).toBe(true)
+				expect(await readdir(join(dataPath, 'service-pids'))).toHaveLength(1)
+			} finally {
+				process.kill(-victim.pid, 'SIGKILL')
+			}
+		})
+	})
+
+	it('refuses to kill when the pid has been recycled', async () => {
+		await withDataDir(async dataPath => {
+			const victim = spawnVictim()
+			try {
+				const actual = await getProcessStartTime(fs, victim.pid)
+				await writeRecord(dataPath, {
+					sessionId: 'sess-1',
+					serviceType: 'astro-dev',
+					pid: victim.pid,
+					// Deliberately not the process now holding this pid.
+					pidStartTime: (actual ?? 0) + 999,
+					command: 'sleep 60',
+					agentPid: await unusablePid(),
+				})
+
+				await makeRegistry(dataPath).sweepOrphans()
+
+				expect(isAlive(victim.pid)).toBe(true)
+				// The record still goes — whatever holds that pid now, it is not ours to track.
+				expect(await readdir(join(dataPath, 'service-pids'))).toEqual([])
+			} finally {
+				process.kill(-victim.pid, 'SIGKILL')
+			}
+		})
+	})
+
+	it('never sweeps what this very process started', async () => {
+		await withDataDir(async dataPath => {
+			const registry = makeRegistry(dataPath)
+			const victim = spawnVictim()
+			try {
+				await registry.record({
+					sessionId: 'sess-1',
+					serviceType: 'astro-dev',
+					pid: victim.pid,
+					pidStartTime: await getProcessStartTime(fs, victim.pid),
+					command: 'sleep 60',
+				})
+
+				await registry.sweepOrphans()
+
+				expect(isAlive(victim.pid)).toBe(true)
+			} finally {
+				process.kill(-victim.pid, 'SIGKILL')
+			}
+		})
+	})
+
+	it('drops an unreadable record instead of throwing', async () => {
+		await withDataDir(async dataPath => {
+			const dir = join(dataPath, 'service-pids')
+			await mkdir(dir, { recursive: true })
+			await writeFile(join(dir, 'sess-1__astro-dev.json'), 'not json')
+
+			await makeRegistry(dataPath).sweepOrphans()
+
+			expect(await readdir(dir)).toEqual([])
+		})
+	})
+
+	it('does nothing when no service has ever been recorded', async () => {
+		await withDataDir(async dataPath => {
+			await makeRegistry(dataPath).sweepOrphans()
+		})
+	})
+
+	it('forget removes the record for a service that has exited', async () => {
+		await withDataDir(async dataPath => {
+			const registry = makeRegistry(dataPath)
+			await registry.record({ sessionId: 'sess-1', serviceType: 'astro-dev', pid: 1, pidStartTime: 1, command: 'x' })
+			expect(await readdir(join(dataPath, 'service-pids'))).toHaveLength(1)
+
+			await registry.forget('sess-1', 'astro-dev')
+
+			expect(await readdir(join(dataPath, 'service-pids'))).toEqual([])
+		})
+	})
+})

package/src/plugins/services/pid-registry.ts

@@ -0,0 +1,191 @@
+/**
+ * ServicePidRegistry — on-disk record of the service processes this host has spawned,
+ * so a later agent can find and kill the ones nobody owns any more.
+ *
+ * The session projection cannot do this job. It remembers one pid per service type, so
+ * every start overwrites the previous generation, and its reconcile runs from
+ * `onSessionReady` — which fires lazily, when a session is loaded. A session nobody
+ * reopens after a restart therefore never reconciles at all, and its dev server keeps
+ * running until the box runs out of memory. That is not theoretical: a production
+ * sandbox on 2026-08-05 held four dev servers at ~500 MB each across three sessions on
+ * 4 GB with no swap, and the kernel OOM-killer fired repeatedly.
+ *
+ * A file per (session, service type) survives both an agent restart and a VM
+ * pause/resume, and is enumerable without knowing which sessions exist — which is what
+ * makes a host-wide sweep at boot possible.
+ */
+
+import { join } from 'node:path'
+import type { FileSystem } from '~/platform/fs.js'
+import type { Logger } from '../../lib/logger/logger.js'
+import { getProcessStartTime } from './service.js'
+
+/**
+ * What we need in order to decide, later and from another process, whether a recorded
+ * pid may be killed.
+ *
+ * `pidStartTime` is /proc's start time in clock ticks: paired with the pid it identifies
+ * one process, so a mismatch means the kernel recycled the pid and killing would hit an
+ * unrelated process. `agentPid`/`agentStartTime` identify the agent that spawned it — a
+ * booting agent must not kill the services of another agent that is still running.
+ */
+export interface ServicePidRecord {
+	sessionId: string
+	serviceType: string
+	pid: number
+	pidStartTime?: number
+	command: string
+	agentPid: number
+	agentStartTime?: number
+}
+
+const REGISTRY_DIR = 'service-pids'
+
+export class ServicePidRegistry {
+	constructor(
+		private readonly fs: FileSystem,
+		private readonly logger: Logger,
+		/** Agent data directory. Never the session workspace — that is a git worktree. */
+		private readonly dataPath: string,
+	) {}
+
+	private get root(): string {
+		return join(this.dataPath, REGISTRY_DIR)
+	}
+
+	private recordPath(sessionId: string, serviceType: string): string {
+		// Service types are plugin-configured identifiers (`astro-dev`, `cms-sidecar`) and
+		// session ids are uuids, so neither needs escaping to be a safe file name.
+		return join(this.root, `${sessionId}__${serviceType}.json`)
+	}
+
+	/**
+	 * Record a freshly spawned service process. Best-effort: failing to record costs us a
+	 * later sweep, so it must never take down the start that just succeeded.
+	 */
+	async record(record: Omit<ServicePidRecord, 'agentPid' | 'agentStartTime'>): Promise<void> {
+		try {
+			await this.fs.mkdir(this.root, { recursive: true })
+			const full: ServicePidRecord = {
+				...record,
+				agentPid: process.pid,
+				agentStartTime: await getProcessStartTime(this.fs, process.pid),
+			}
+			await this.fs.writeFile(this.recordPath(record.sessionId, record.serviceType), JSON.stringify(full))
+		} catch (error) {
+			this.logger.debug('Could not record service pid', { sessionId: record.sessionId, serviceType: record.serviceType, error })
+		}
+	}
+
+	/** Drop the record for a service whose process is gone. */
+	async forget(sessionId: string, serviceType: string): Promise<void> {
+		try {
+			await this.fs.unlink(this.recordPath(sessionId, serviceType))
+		} catch {
+			// Already gone, or never recorded.
+		}
+	}
+
+	/**
+	 * Kill every recorded service process that no live agent owns, and clear its record.
+	 *
+	 * Meant to run once at agent boot, before any session loads: at that moment this agent
+	 * tracks nothing, so every survivor of a previous incarnation is by definition an
+	 * orphan. Records left by an agent that is still running are skipped — during an
+	 * overlapping restart the outgoing agent's services are still its own.
+	 */
+	async sweepOrphans(): Promise<void> {
+		let files: string[]
+		try {
+			files = await this.fs.readdir(this.root)
+		} catch {
+			return // No registry yet — nothing has ever been spawned here.
+		}
+
+		for (const file of files) {
+			if (!file.endsWith('.json')) continue
+			const path = join(this.root, file)
+			const record = await this.readRecord(path)
+			if (!record) {
+				await this.fs.unlink(path).catch(() => {})
+				continue
+			}
+
+			if (await this.isAgentAlive(record)) {
+				this.logger.debug('Leaving service to its still-running agent', {
+					sessionId: record.sessionId,
+					serviceType: record.serviceType,
+					agentPid: record.agentPid,
+				})
+				continue
+			}
+
+			await this.killIfStillOurs(record)
+			await this.fs.unlink(path).catch(() => {})
+		}
+	}
+
+	private async readRecord(path: string): Promise<ServicePidRecord | null> {
+		try {
+			const parsed: unknown = JSON.parse(await this.fs.readFile(path, 'utf-8'))
+			if (typeof parsed !== 'object' || parsed === null) return null
+			const record = parsed as Partial<ServicePidRecord>
+			if (typeof record.pid !== 'number' || typeof record.sessionId !== 'string' || typeof record.serviceType !== 'string') {
+				return null
+			}
+			return {
+				sessionId: record.sessionId,
+				serviceType: record.serviceType,
+				pid: record.pid,
+				pidStartTime: record.pidStartTime,
+				command: typeof record.command === 'string' ? record.command : '',
+				agentPid: typeof record.agentPid === 'number' ? record.agentPid : 0,
+				agentStartTime: record.agentStartTime,
+			}
+		} catch {
+			return null
+		}
+	}
+
+	/**
+	 * Whether a live agent still owns this record. Ourselves counts: a record carrying our own
+	 * pid describes a service this very process started, which the sweep must never touch —
+	 * that is what makes the sweep safe to run at any time, not only before the first start.
+	 */
+	private async isAgentAlive(record: ServicePidRecord): Promise<boolean> {
+		if (record.agentPid === process.pid) return true
+		if (record.agentPid === 0) return false
+		const currentStartTime = await getProcessStartTime(this.fs, record.agentPid)
+		if (currentStartTime === undefined) return false
+		return record.agentStartTime === undefined || currentStartTime === record.agentStartTime
+	}
+
+	private async killIfStillOurs(record: ServicePidRecord): Promise<void> {
+		const currentStartTime = await getProcessStartTime(this.fs, record.pid)
+		if (currentStartTime === undefined) return // Process is gone.
+		if (record.pidStartTime !== undefined && currentStartTime !== record.pidStartTime) {
+			this.logger.warn('PID reuse detected during orphan sweep — refusing to kill', {
+				sessionId: record.sessionId,
+				serviceType: record.serviceType,
+				pid: record.pid,
+				storedStartTime: record.pidStartTime,
+				currentStartTime,
+			})
+			return
+		}
+
+		try {
+			// Negative pid: services are spawned detached, so this takes the whole group —
+			// the `/bin/sh -c` wrapper and the runtime it exec'd underneath it.
+			process.kill(-record.pid, 'SIGKILL')
+			this.logger.info('Killed orphaned service process group', {
+				sessionId: record.sessionId,
+				serviceType: record.serviceType,
+				pid: record.pid,
+				command: record.command,
+			})
+		} catch (error) {
+			this.logger.debug('Orphaned service process already gone', { pid: record.pid, error })
+		}
+	}
+}

package/src/plugins/services/plugin.ts

@@ -8,6 +8,7 @@ import { Err, Ok } from '~/lib/utils/result.js'
 import type { PortPool } from './port-pool.js'
 import { buildServiceStatusMessage } from './prompt.js'
 import type { ServiceConfig, ServiceEntry } from './schema.js'
+import type { ServicePidRegistry } from './pid-registry.js'
 import { getProcessStartTime, ServiceExecutor } from './service.js'
 
 export const serviceEvents = createEventsFactory({
@@ -21,6 +22,14 @@ export const serviceEvents = createEventsFactory({
 			command: z.string().optional(),
 			pid: z.number().optional(),
 			pidStartTime: z.number().optional(),
+			/**
+			 * Set on a `failed` change when the `restartPolicy` has already queued a
+			 * revival, so that no consumer — agent prompt, SPA, platform — mistakes it
+			 * for a terminal failure. Cleared by the next status change.
+			 */
+			restartAt: z.number().optional(),
+			restartAttempt: z.number().optional(),
+			restartMaxRetries: z.number().optional(),
 		}),
 	},
 })
@@ -33,6 +42,8 @@ export type ServiceStatusChangedEvent = (typeof serviceEvents)['Events']['servic
 export interface ServicePluginConfig {
 	services: ServiceConfig[]
 	portPool: PortPool
+	/** Durable pid record swept at agent boot. Absent for embedders without a data dir. */
+	pidRegistry?: ServicePidRegistry
 }
 
 /**
@@ -70,6 +81,10 @@ export const servicePlugin = definePlugin('services')
 						const updated: ServiceEntry = {
 							...existing,
 							status: event.toStatus,
+							// A queued revival lives only until the next status change.
+							restartAt: event.restartAt,
+							restartAttempt: event.restartAttempt,
+							restartMaxRetries: event.restartMaxRetries,
 						}
 						if (event.toStatus === 'starting') {
 							updated.startedAt = event.timestamp
@@ -112,17 +127,25 @@ export const servicePlugin = definePlugin('services')
 					let changed = false
 					const newServices = new Map(services)
 					for (const [serviceType, entry] of services) {
-						if (entry.status === 'starting' || entry.status === 'ready') {
-							newServices.set(serviceType, {
-								...entry,
-								status: 'stopped',
-								port: undefined,
-								pid: undefined,
-								pidStartTime: undefined,
-								stoppedAt: event.timestamp,
-							})
-							changed = true
+						const running = entry.status === 'starting' || entry.status === 'ready'
+						// Any queued revival died with the runtime that held its timer.
+						if (!running && entry.restartAt === undefined) continue
+
+						const updated: ServiceEntry = {
+							...entry,
+							restartAt: undefined,
+							restartAttempt: undefined,
+							restartMaxRetries: undefined,
+						}
+						if (running) {
+							updated.status = 'stopped'
+							updated.port = undefined
+							updated.pid = undefined
+							updated.pidStartTime = undefined
+							updated.stoppedAt = event.timestamp
 						}
+						newServices.set(serviceType, updated)
+						changed = true
 					}
 					return changed ? newServices : services
 				}
@@ -133,7 +156,11 @@ export const servicePlugin = definePlugin('services')
 		},
 	})
 	.context(async (ctx, pluginConfig) => {
-		const executor = new ServiceExecutor(ctx.logger, pluginConfig.portPool, { fs: ctx.platform.fs, process: ctx.platform.process })
+		const executor = new ServiceExecutor(ctx.logger, pluginConfig.portPool, {
+			fs: ctx.platform.fs,
+			process: ctx.platform.process,
+			pidRegistry: pluginConfig.pidRegistry,
+		})
 		executor.onStatusChanged = (sessionId, serviceType, status, details) => {
 			void ctx.emitEvent(serviceEvents.create('service_status_changed', {
 				serviceType,
@@ -144,9 +171,22 @@ export const servicePlugin = definePlugin('services')
 				command: details.command,
 				pid: details.pid,
 				pidStartTime: details.pidStartTime,
+				restartAt: details.restartAt,
+				restartAttempt: details.restartAttempt,
+				restartMaxRetries: details.restartMaxRetries,
 			}))
-			// Broadcast service status to connected clients (DO → SPA) via WS
-			ctx.notify('serviceStatus', { sessionId: String(sessionId), serviceType, status, port: details.port })
+			// Broadcast service status to connected clients (DO → SPA) via WS.
+			// restartAt rides along so a consumer can hold on to a preview URL it
+			// would otherwise tear down on `failed`.
+			ctx.notify('serviceStatus', {
+				sessionId: String(sessionId),
+				serviceType,
+				status,
+				port: details.port,
+				restartAt: details.restartAt,
+				restartAttempt: details.restartAttempt,
+				restartMaxRetries: details.restartMaxRetries,
+			})
 		}
 		return { executor }
 	})
@@ -371,7 +411,10 @@ export const servicePlugin = definePlugin('services')
 	.sessionHook('onSessionClose', async (ctx) => {
 		for (const svcConfig of ctx.pluginConfig.services) {
 			const status = ctx.pluginContext.executor.getStatus(svcConfig.type)
-			if (status === 'ready' || status === 'starting') {
+			// A queued revival outlives the session otherwise: its timer holds the
+			// executor alive and spawns a process into a session that is already gone.
+			const revivalQueued = ctx.pluginContext.executor.hasScheduledRestart(svcConfig.type)
+			if (status === 'ready' || status === 'starting' || revivalQueued) {
 				await ctx.pluginContext.executor.stop(svcConfig.type, ctx.sessionId)
 			}
 		}
@@ -388,7 +431,7 @@ export const servicePlugin = definePlugin('services')
 			createTool({
 				name: 'service_start',
 				description:
-					`Start a stopped or failed session service. Only call this if the session context shows the service is not running — if it is already "ready", you do not need to start it. Available services: ${serviceList}`,
+					`Start a stopped or failed session service. Only call this if the session context shows the service is not running — if it is already "ready", or marked as restarting automatically, you do not need to start it. Available services: ${serviceList}`,
 				input: z.object({
 					serviceType: z.string().describe('Service type identifier'),
 				}),

package/src/plugins/services/prompt.ts

@@ -42,6 +42,14 @@ export function buildServiceStatusMessage(
 		if (entry?.error) {
 			line += ` (error: ${entry.error})`
 		}
+		// A queued revival means the service comes back on its own — say so, or the
+		// agent burns a turn restarting by hand what was about to restart anyway.
+		if (entry?.restartAt !== undefined) {
+			const attempt = entry.restartAttempt !== undefined && entry.restartMaxRetries !== undefined
+				? ` (attempt ${entry.restartAttempt}/${entry.restartMaxRetries})`
+				: ''
+			line += ` — restarting automatically${attempt}, no action needed`
+		}
 		// Deliberately no cwd: it is a real host path, while sandboxed agents see the
 		// world through virtual roots — leaking it sends them chasing paths that
 		// don't exist in their namespace (see webmaster path-identity incident).

package/src/plugins/services/schema.ts

@@ -101,6 +101,28 @@ export interface ServiceConfig {
 	logBufferSize?: number
 	/** Startup timeout in ms — service marked as failed if not ready within this time (default: 30000) */
 	startupTimeoutMs?: number
+	/**
+	 * Bring the service back after an unexpected exit, on the port it already
+	 * holds. Omitted means what has always happened: the service is left
+	 * `failed` and waits for someone to call restart.
+	 *
+	 * For a long-running service nobody is watching — a dev server behind a
+	 * preview URL — a crash otherwise means the URL stays dead until a human
+	 * or an agent notices. The retry budget is per consecutive failure and
+	 * resets once the service has stayed up for `healthyAfterMs`, so a process
+	 * that dies repeatedly at startup still ends up `failed` instead of
+	 * looping forever.
+	 */
+	restartPolicy?: {
+		/** Consecutive automatic restarts before giving up (default: 3) */
+		maxRetries?: number
+		/** Delay before the first retry; doubles per attempt (default: 1000) */
+		initialDelayMs?: number
+		/** Ceiling for the doubling (default: 30000) */
+		maxDelayMs?: number
+		/** Uptime that counts as healthy and resets the budget (default: 60000) */
+		healthyAfterMs?: number
+	}
 	/** Auto-pause configuration (interface prepared, not yet implemented) */
 	autoPause?: { inactivityMs: number }
 	/**
@@ -133,6 +155,17 @@ export interface ServiceEntry {
 	startedAt?: number
 	readyAt?: number
 	stoppedAt?: number
+	/**
+	 * When a `restartPolicy` revival is queued, the epoch ms at which it fires.
+	 * Present only while the revival is pending — any later status change clears
+	 * it — so `failed` with `restartAt` set means "down, but coming back on its
+	 * own" and nobody downstream has to treat it as terminal.
+	 */
+	restartAt?: number
+	/** 1-based attempt number of the queued revival. */
+	restartAttempt?: number
+	/** Retry budget the queued revival counts against. */
+	restartMaxRetries?: number
 	/** Process group PID (tracked for orphan cleanup after restart) */
 	pid?: number
 	/**