@roj-ai/sdk 0.1.36 → 0.2.1

package/src/plugins/services/plugin.ts

@@ -9,19 +9,26 @@ import type { PortPool } from './port-pool.js'
 import { buildServiceStatusMessage } from './prompt.js'
 import type { ServiceConfig, ServiceEntry } from './schema.js'
 import type { ServicePidRegistry } from './pid-registry.js'
-import { getProcessStartTime, ServiceExecutor } from './service.js'
+import { ServiceExecutor } from './service.js'
+import type { RuntimeLeaseRelease } from '~/core/sessions/runtime-activity.js'
 
 export const serviceEvents = createEventsFactory({
 	events: {
 		service_status_changed: z.object({
 			serviceType: z.string(),
-			toStatus: z.enum(['stopped', 'starting', 'ready', 'stopping', 'failed', 'paused']),
+			toStatus: z.enum(['stopped', 'starting', 'ready', 'stopping', 'failed']),
 			port: z.number().optional(),
 			error: z.string().optional(),
 			cwd: z.string().optional(),
 			command: z.string().optional(),
 			pid: z.number().optional(),
 			pidStartTime: z.number().optional(),
+			/**
+			 * Who asked for the stop, on a `stopped` change. Absent from logs written
+			 * before the discriminator existed — the reducer reads those as `agent`,
+			 * the only default that cannot resurrect a service meant to stay down.
+			 */
+			stoppedBy: z.enum(['agent', 'eviction']).optional(),
 			/**
 			 * Set on a `failed` change when the `restartPolicy` has already queued a
 			 * revival, so that no consumer — agent prompt, SPA, platform — mistakes it
@@ -36,6 +43,17 @@ export const serviceEvents = createEventsFactory({
 
 export type ServiceStatusChangedEvent = (typeof serviceEvents)['Events']['service_status_changed']
 
+/** Ephemeral status broadcast — mirrors the durable event, minus the fields no client reads. */
+const serviceStatusNotificationSchema = z.object({
+	sessionId: z.string(),
+	serviceType: z.string(),
+	status: z.enum(['stopped', 'starting', 'ready', 'stopping', 'failed']),
+	port: z.number().optional(),
+	restartAt: z.number().optional(),
+	restartAttempt: z.number().optional(),
+	restartMaxRetries: z.number().optional(),
+})
+
 /**
  * Session-wide service configuration.
  */
@@ -54,7 +72,9 @@ export interface ServiceAgentConfig {
 }
 
 export const servicePlugin = definePlugin('services')
+	.order(100)
 	.pluginConfig<ServicePluginConfig>()
+	.notification('serviceStatus', { schema: serviceStatusNotificationSchema })
 	.isSessionEnabled(({ pluginConfig }) => pluginConfig !== undefined && pluginConfig.services.length > 0)
 	.events([serviceEvents, sessionEvents])
 	.state({
@@ -70,6 +90,7 @@ export const servicePlugin = definePlugin('services')
 						newServices.set(event.serviceType, {
 							serviceType: event.serviceType,
 							status: event.toStatus,
+							stoppedBy: 'never',
 							port: event.port,
 							cwd: event.cwd,
 							command: event.command,
@@ -88,6 +109,7 @@ export const servicePlugin = definePlugin('services')
 						}
 						if (event.toStatus === 'starting') {
 							updated.startedAt = event.timestamp
+							updated.stoppedBy = 'never'
 							updated.error = undefined
 							updated.port = event.port
 							updated.cwd = event.cwd
@@ -107,13 +129,17 @@ export const servicePlugin = definePlugin('services')
 								updated.command = event.command
 							}
 						}
-						if (event.toStatus === 'failed' && event.error) {
-							updated.error = event.error
-							updated.pid = undefined
-							updated.pidStartTime = undefined
+						if (event.toStatus === 'failed') {
+							if (event.error) updated.error = event.error
+							// `pid` is a claim on a process we spawned and have not seen exit — the
+							// only handle a later boot has on it. A `failed` change carries it while
+							// the process may still be alive, and omits it once the exit is confirmed.
+							updated.pid = event.pid
+							updated.pidStartTime = event.pidStartTime
 						}
 						if (event.toStatus === 'stopped') {
 							updated.stoppedAt = event.timestamp
+							updated.stoppedBy = event.stoppedBy ?? 'agent'
 							updated.pid = undefined
 							updated.pidStartTime = undefined
 						}
@@ -127,9 +153,11 @@ export const servicePlugin = definePlugin('services')
 					let changed = false
 					const newServices = new Map(services)
 					for (const [serviceType, entry] of services) {
-						const running = entry.status === 'starting' || entry.status === 'ready'
+						// `stopping` counts as live: a runtime that died mid-stop leaves it
+						// behind and no other path ever clears it.
+						const live = entry.status === 'starting' || entry.status === 'ready' || entry.status === 'stopping'
 						// Any queued revival died with the runtime that held its timer.
-						if (!running && entry.restartAt === undefined) continue
+						if (!live && entry.restartAt === undefined) continue
 
 						const updated: ServiceEntry = {
 							...entry,
@@ -137,12 +165,15 @@ export const servicePlugin = definePlugin('services')
 							restartAttempt: undefined,
 							restartMaxRetries: undefined,
 						}
-						if (running) {
+						if (live) {
 							updated.status = 'stopped'
+							// The runtime died under it — nobody decided to stop it.
+							updated.stoppedBy = 'eviction'
 							updated.port = undefined
-							updated.pid = undefined
-							updated.pidStartTime = undefined
 							updated.stoppedAt = event.timestamp
+							// `pid` deliberately survives: nothing killed that process, and this is
+							// the only handle left to find it with. This event is emitted immediately
+							// before `onSessionReady`, whose reconcile is what reclaims it.
 						}
 						newServices.set(serviceType, updated)
 						changed = true
@@ -156,39 +187,159 @@ export const servicePlugin = definePlugin('services')
 		},
 	})
 	.context(async (ctx, pluginConfig) => {
-		const executor = new ServiceExecutor(ctx.logger, pluginConfig.portPool, {
+		const logger = ctx.logger
+		const emitEvent = ctx.emitEvent
+		const notify = ctx.notify
+		const runtimeActivity = ctx.runtimeActivity
+		interface ServiceLease {
+			release: RuntimeLeaseRelease
+			terminalPending: boolean
+		}
+		const serviceLeases = new Map<string, ServiceLease>()
+		/** Status publications still in flight per type — a lease has to outlive them. */
+		const pendingPublications = new Map<string, number>()
+		const statusEffects = new Set<Promise<void>>()
+		let statusEffectTail = Promise.resolve()
+		let publicationEnabled = true
+		const executor = new ServiceExecutor(logger, pluginConfig.portPool, {
 			fs: ctx.platform.fs,
 			process: ctx.platform.process,
 			pidRegistry: pluginConfig.pidRegistry,
 		})
+		const beginLifecycle = (serviceType: string): ServiceLease | undefined => {
+			const existing = serviceLeases.get(serviceType)
+			if (existing && !existing.terminalPending) return existing
+
+			// Undefined once the runtime unloads — the close tail that runs then is
+			// already awaited by disposal, so nothing is left unguarded.
+			const release = runtimeActivity.tryAcquire(`service:${serviceType}`)
+			if (!release) return undefined
+			const lease: ServiceLease = { release, terminalPending: false }
+			serviceLeases.set(serviceType, lease)
+			return lease
+		}
+		const releaseLease = (serviceType: string, lease: ServiceLease): void => {
+			if (serviceLeases.get(serviceType) === lease) serviceLeases.delete(serviceType)
+			lease.release()
+		}
+		/**
+		 * A healthy service must not pin its runtime: eviction stops services and the
+		 * rebuild starts them again, so a dev server sitting at `ready` for hours is
+		 * exactly what eviction is for. Only work that cannot survive disposal keeps
+		 * the lease — a transition in progress, a queued revival, an unpublished status.
+		 */
+		const needsRuntime = (serviceType: string): boolean => {
+			if ((pendingPublications.get(serviceType) ?? 0) > 0) return true
+			const status = executor.getStatus(serviceType)
+			return status === 'starting' || status === 'stopping' || executor.hasScheduledRestart(serviceType)
+		}
+		const releaseLeaseWhenIdle = (serviceType: string, lease: ServiceLease): void => {
+			if (serviceLeases.get(serviceType) !== lease || lease.terminalPending) return
+			if (!needsRuntime(serviceType)) releaseLease(serviceType, lease)
+		}
+		const startService = async (
+			config: ServiceConfig,
+			sessionId: Parameters<ServiceExecutor['start']>[1],
+			workspaceDir?: string,
+			preferredPort?: number,
+		) => {
+			const lease = beginLifecycle(config.type)
+			try {
+				return await executor.start(config, sessionId, workspaceDir, preferredPort)
+			} finally {
+				if (lease) releaseLeaseWhenIdle(config.type, lease)
+			}
+		}
+		const restartService = async (
+			config: ServiceConfig,
+			sessionId: Parameters<ServiceExecutor['restart']>[1],
+			workspaceDir?: string,
+			preferredPort?: number,
+		) => {
+			const lease = beginLifecycle(config.type)
+			try {
+				return await executor.restart(config, sessionId, workspaceDir, preferredPort)
+			} finally {
+				if (lease) releaseLeaseWhenIdle(config.type, lease)
+			}
+		}
+		const waitForStatusEffects = async (): Promise<void> => {
+			while (statusEffects.size > 0) {
+				await Promise.allSettled([...statusEffects])
+			}
+		}
+		executor.onStartSettled = (serviceType) => {
+			const lease = serviceLeases.get(serviceType)
+			if (lease) releaseLeaseWhenIdle(serviceType, lease)
+		}
 		executor.onStatusChanged = (sessionId, serviceType, status, details) => {
-			void ctx.emitEvent(serviceEvents.create('service_status_changed', {
-				serviceType,
-				toStatus: status,
-				port: details.port,
-				error: details.error,
-				cwd: details.cwd,
-				command: details.command,
-				pid: details.pid,
-				pidStartTime: details.pidStartTime,
-				restartAt: details.restartAt,
-				restartAttempt: details.restartAttempt,
-				restartMaxRetries: details.restartMaxRetries,
-			}))
-			// Broadcast service status to connected clients (DO → SPA) via WS.
-			// restartAt rides along so a consumer can hold on to a preview URL it
-			// would otherwise tear down on `failed`.
-			ctx.notify('serviceStatus', {
-				sessionId: String(sessionId),
-				serviceType,
-				status,
-				port: details.port,
-				restartAt: details.restartAt,
-				restartAttempt: details.restartAttempt,
-				restartMaxRetries: details.restartMaxRetries,
+			if (!publicationEnabled) return
+			const terminal = status === 'stopped' || (status === 'failed' && details.restartAt === undefined)
+			const lease = terminal ? serviceLeases.get(serviceType) : beginLifecycle(serviceType)
+			if (terminal && lease) lease.terminalPending = true
+			pendingPublications.set(serviceType, (pendingPublications.get(serviceType) ?? 0) + 1)
+
+			const effect = statusEffectTail.then(async () => {
+				try {
+					await emitEvent(serviceEvents.create('service_status_changed', {
+						serviceType,
+						toStatus: status,
+						port: details.port,
+						error: details.error,
+						cwd: details.cwd,
+						command: details.command,
+						pid: details.pid,
+						pidStartTime: details.pidStartTime,
+						stoppedBy: details.stoppedBy,
+						restartAt: details.restartAt,
+						restartAttempt: details.restartAttempt,
+						restartMaxRetries: details.restartMaxRetries,
+					}))
+					// Broadcast only after the durable projection catches up.
+					notify('serviceStatus', {
+						sessionId: String(sessionId),
+						serviceType,
+						status,
+						port: details.port,
+						restartAt: details.restartAt,
+						restartAttempt: details.restartAttempt,
+						restartMaxRetries: details.restartMaxRetries,
+					})
+				} catch (error) {
+					logger.error('Failed to publish service status', error instanceof Error ? error : new Error(String(error)), {
+						serviceType,
+						status,
+					})
+				} finally {
+					const remaining = (pendingPublications.get(serviceType) ?? 1) - 1
+					if (remaining > 0) pendingPublications.set(serviceType, remaining)
+					else pendingPublications.delete(serviceType)
+					if (lease) {
+						if (terminal) releaseLease(serviceType, lease)
+						else releaseLeaseWhenIdle(serviceType, lease)
+					}
+				}
 			})
+			statusEffectTail = effect.then(() => undefined, () => undefined)
+			statusEffects.add(effect)
+			const removeEffect = () => statusEffects.delete(effect)
+			void effect.then(removeEffect, removeEffect)
 		}
-		return { executor }
+		const close = async (
+			sessionId: Parameters<ServiceExecutor['close']>[0],
+			reason: Parameters<ServiceExecutor['close']>[1],
+		): Promise<void> => {
+			try {
+				await executor.close(sessionId, reason)
+			} finally {
+				publicationEnabled = false
+				executor.onStatusChanged = undefined
+				executor.onStartSettled = undefined
+				await waitForStatusEffects()
+				for (const [serviceType, lease] of serviceLeases) releaseLease(serviceType, lease)
+			}
+		}
+		return { executor, startService, restartService, close }
 	})
 	.agentConfig<ServiceAgentConfig>()
 	.method('start', {
@@ -214,7 +365,7 @@ export const servicePlugin = definePlugin('services')
 				}
 
 				const preferredPort = ctx.pluginState.get(input.serviceType)?.port
-				const startResult = await ctx.pluginContext.executor.start(svcConfig, ctx.sessionId, ctx.sessionState.workspaceDir, preferredPort)
+				const startResult = await ctx.pluginContext.startService(svcConfig, ctx.sessionId, ctx.sessionState.workspaceDir, preferredPort)
 				if (!startResult.ok) return Err(ValidationErrors.invalid(startResult.error.message))
 				started.push(input.serviceType)
 			} else {
@@ -229,7 +380,7 @@ export const servicePlugin = definePlugin('services')
 							}
 						} else {
 							const preferredPort = ctx.pluginState.get(svcConfig.type)?.port
-							const startResult = await ctx.pluginContext.executor.start(svcConfig, ctx.sessionId, ctx.sessionState.workspaceDir, preferredPort)
+							const startResult = await ctx.pluginContext.startService(svcConfig, ctx.sessionId, ctx.sessionState.workspaceDir, preferredPort)
 							if (!startResult.ok) return Err(ValidationErrors.invalid(startResult.error.message))
 							started.push(svcConfig.type)
 						}
@@ -269,7 +420,7 @@ export const servicePlugin = definePlugin('services')
 			if (!svcConfig) return Err(ValidationErrors.invalid(`Service ${input.serviceType} not found`))
 
 			const preferredPort = ctx.pluginState.get(input.serviceType)?.port
-			const result = await ctx.pluginContext.executor.restart(svcConfig, ctx.sessionId, ctx.sessionState.workspaceDir, preferredPort)
+			const result = await ctx.pluginContext.restartService(svcConfig, ctx.sessionId, ctx.sessionState.workspaceDir, preferredPort)
 			if (!result.ok) return Err(ValidationErrors.invalid(result.error.message))
 			return Ok({})
 		},
@@ -347,78 +498,83 @@ export const servicePlugin = definePlugin('services')
 		},
 	})
 	.sessionHook('onSessionReady', async (ctx) => {
-		// Reconcile: kill orphaned process groups from previous server instance
-		// and mark corresponding services as stopped. Port is preserved in state
-		// so the next start() reuses it via preferredPort.
-		// Also re-notify running services so DO re-registers their URLs.
+		// Reconcile what a previous runtime left behind: kill orphaned process groups and
+		// settle the entries that claimed them. Port is preserved in state so the next
+		// start() reuses it via preferredPort. Also re-notify running services so DO
+		// re-registers their URLs.
+		//
+		// Driven off the recorded pid rather than a list of running statuses: a process we
+		// spawned and never saw exit is an orphan whatever the entry says, and `failed` or
+		// `stopped` reaches the next boot still owning one just as often as `ready` does.
 		for (const [serviceType, entry] of ctx.pluginState) {
-			if (entry.status === 'starting' || entry.status === 'ready' || entry.status === 'paused') {
-				const executorStatus = ctx.pluginContext.executor.getStatus(serviceType)
-				if (!executorStatus) {
-					if (entry.pid !== undefined) {
-						// PID-reuse guard: only kill if we can confirm this PID still
-						// belongs to the process we spawned. If we stored a pidStartTime,
-						// the current start time must match — a mismatch means the kernel
-						// recycled the PID for an unrelated process and SIGKILL would be
-						// disastrous. On non-Linux (no /proc), start times are always
-						// undefined and we fall back to the pre-existing kill-and-hope path.
-						const currentStartTime = await getProcessStartTime(ctx.platform.fs, entry.pid)
-						const pidReused = entry.pidStartTime !== undefined
-							&& currentStartTime !== undefined
-							&& currentStartTime !== entry.pidStartTime
-
-						if (pidReused) {
-							ctx.logger.warn('PID reuse detected during orphan reconcile — refusing to kill', {
-								serviceType,
-								pid: entry.pid,
-								storedStartTime: entry.pidStartTime,
-								currentStartTime,
-							})
-						} else {
-							try {
-								process.kill(-entry.pid, 'SIGKILL')
-								ctx.logger.info('Killed orphaned service process group', { serviceType, pid: entry.pid })
-							} catch (err) {
-								ctx.logger.debug('Orphaned service process already gone', { serviceType, pid: entry.pid, err })
-							}
-						}
-					}
-					await ctx.emitEvent(serviceEvents.create('service_status_changed', {
-						serviceType,
-						toStatus: 'stopped',
-					}))
-				} else if (executorStatus === 'ready' && entry.port) {
+			// `stopping` counts as live: a runtime that died mid-stop leaves it behind, and
+			// it is reachable by no other recovery path.
+			const live = entry.status === 'starting' || entry.status === 'ready' || entry.status === 'stopping'
+			if (!live && entry.pid === undefined) continue
+
+			const executorStatus = ctx.pluginContext.executor.getStatus(serviceType)
+			if (executorStatus) {
+				if (executorStatus === 'ready' && entry.port) {
 					// Re-notify so DO can re-register service URL after reconnect
 					ctx.notify('serviceStatus', { sessionId: String(ctx.sessionId), serviceType, status: 'ready', port: entry.port })
 				}
+				continue
 			}
-		}
 
-		// Auto-start services configured with autoStart
-		for (const svcConfig of ctx.pluginConfig.services) {
-			if (svcConfig.autoStart) {
-				const status = ctx.pluginContext.executor.getStatus(svcConfig.type)
-				if (status !== 'ready' && status !== 'starting') {
-					const preferredPort = ctx.pluginState.get(svcConfig.type)?.port
-					const startResult = await ctx.pluginContext.executor.start(svcConfig, ctx.sessionId, ctx.sessionState.workspaceDir, preferredPort)
-					if (!startResult.ok) {
-						ctx.logger.debug('Auto-start service skipped', { serviceType: svcConfig.type, error: startResult.error.message })
-					}
+			if (entry.pid !== undefined) {
+				// Guarded and loud when it fails: a signal that did not land must not look
+				// like one that did.
+				const reaped = await ctx.pluginContext.executor.reapProcessGroup(
+					{ pid: entry.pid, pidStartTime: entry.pidStartTime },
+					{ serviceType },
+				)
+				if (!reaped.ok) {
+					ctx.logger.warn('Could not kill orphaned service process group', {
+						serviceType,
+						pid: entry.pid,
+						error: reaped.error.message,
+					})
+				} else if (reaped.value === 'killed') {
+					ctx.logger.info('Killed orphaned service process group', { serviceType, pid: entry.pid })
 				}
 			}
+
+			// A `failed` entry keeps its status and the error that explains it — only its
+			// claim on a process is taken away. Everything else settles as stopped.
+			await ctx.emitEvent(serviceEvents.create('service_status_changed', entry.status === 'failed'
+				? { serviceType, toStatus: 'failed' }
+				: {
+					serviceType,
+					toStatus: 'stopped',
+					// The runtime died under it — nobody decided to stop it, so autoStart may revive it.
+					stoppedBy: 'eviction',
+				}))
 		}
-	})
-	.sessionHook('onSessionClose', async (ctx) => {
+
+		// Auto-start services configured with autoStart
 		for (const svcConfig of ctx.pluginConfig.services) {
+			if (!svcConfig.autoStart) continue
 			const status = ctx.pluginContext.executor.getStatus(svcConfig.type)
-			// A queued revival outlives the session otherwise: its timer holds the
-			// executor alive and spawns a process into a session that is already gone.
-			const revivalQueued = ctx.pluginContext.executor.hasScheduledRestart(svcConfig.type)
-			if (status === 'ready' || status === 'starting' || revivalQueued) {
-				await ctx.pluginContext.executor.stop(svcConfig.type, ctx.sessionId)
+			if (status === 'ready' || status === 'starting') continue
+
+			// On a fresh runtime the executor knows nothing, so the persisted decision is
+			// the only thing that can tell an evicted service (bring it back) from one the
+			// agent stopped on purpose (leave it down).
+			const entry = ctx.pluginState.get(svcConfig.type)
+			if (entry?.status === 'stopped' && entry.stoppedBy === 'agent') {
+				ctx.logger.debug('Auto-start skipped — the agent stopped this service', { serviceType: svcConfig.type })
+				continue
+			}
+
+			const startResult = await ctx.pluginContext.startService(svcConfig, ctx.sessionId, ctx.sessionState.workspaceDir, entry?.port)
+			if (!startResult.ok) {
+				ctx.logger.debug('Auto-start service skipped', { serviceType: svcConfig.type, error: startResult.error.message })
 			}
 		}
 	})
+	.sessionHook('onSessionClose', async (ctx) => {
+		await ctx.pluginContext.close(ctx.sessionId, ctx.reason)
+	})
 	.tools((ctx) => {
 		const serviceMap = new Map(ctx.pluginConfig.services.map((svc) => [svc.type, svc]))
 		const visibleServices = ctx.pluginAgentConfig?.services ?? []

package/src/plugins/services/prompt.ts

@@ -31,12 +31,7 @@ export function buildServiceStatusMessage(
 		const status = entry?.status ?? 'stopped'
 		let line = `- ${config.type}: ${config.description} — ${status}`
 
-		if (
-			entry?.port !== undefined
-			&& entry.status !== 'stopped'
-			&& entry.status !== 'failed'
-			&& entry.status !== 'paused'
-		) {
+		if (entry?.port !== undefined && entry.status !== 'stopped' && entry.status !== 'failed') {
 			line += ` (port ${entry.port})`
 		}
 		if (entry?.error) {

package/src/plugins/services/schema.ts

@@ -13,7 +13,17 @@
 /**
  * Service lifecycle status.
  */
-export type ServiceStatus = 'stopped' | 'starting' | 'ready' | 'stopping' | 'failed' | 'paused'
+export type ServiceStatus = 'stopped' | 'starting' | 'ready' | 'stopping' | 'failed'
+
+/**
+ * Who asked for a stop. Every runtime-lifecycle stop — idle eviction, session
+ * close, server shutdown — records `eviction`, because none of them is the
+ * agent's decision and all of them may be undone by the next runtime.
+ */
+export type ServiceStopSource = 'agent' | 'eviction'
+
+/** `never` means the service has not been stopped since its last start. */
+export type ServiceStoppedBy = ServiceStopSource | 'never'
 
 // ============================================================================
 // Service Config (declared in presets)
@@ -123,8 +133,6 @@ export interface ServiceConfig {
 		/** Uptime that counts as healthy and resets the budget (default: 60000) */
 		healthyAfterMs?: number
 	}
-	/** Auto-pause configuration (interface prepared, not yet implemented) */
-	autoPause?: { inactivityMs: number }
 	/**
 	 * When false, the service is hidden from the owning agent: excluded from its
 	 * service_* tools and the session-context status block. It still runs at the
@@ -146,6 +154,12 @@ export interface ServiceConfig {
 export interface ServiceEntry {
 	serviceType: string
 	status: ServiceStatus
+	/**
+	 * Who stopped the service. An idle eviction stops services and the rebuilt
+	 * runtime starts them again, so autoStart needs this to tell a parked
+	 * service from one the agent deliberately shut down and expects to stay down.
+	 */
+	stoppedBy: ServiceStoppedBy
 	port?: number
 	error?: string
 	/** Resolved working directory used for the current/last start. */