lopata 0.18.4 → 0.19.0

package/src/api/handlers/workflows.ts

@@ -1,6 +1,5 @@
 import type { SQLQueryBindings } from 'bun:sqlite'
 import type { SqliteWorkflowBinding } from '../../bindings/workflow'
-import { getWaitingEventTypes, isInstanceSleeping } from '../../bindings/workflow'
 import { getDatabase } from '../../db'
 import type { HandlerContext, OkResponse, WorkflowDetail, WorkflowInstance, WorkflowSummary } from '../types'
 import { getAllConfigs } from '../types'
@@ -66,7 +65,7 @@ export const handlers = {
 		return db.prepare(query).all(...params) as WorkflowInstance[]
 	},
 
-	'workflows.getInstance'({ id }: { name: string; id: string }): WorkflowDetail {
+	async 'workflows.getInstance'({ name, id }: { name: string; id: string }, ctx: HandlerContext): Promise<WorkflowDetail> {
 		const db = getDatabase()
 		const instance = db.query<Record<string, unknown>, [string]>(
 			'SELECT * FROM workflow_instances WHERE id = ?',
@@ -95,10 +94,28 @@ export const handlers = {
 			'SELECT id, event_type, payload, created_at FROM workflow_events WHERE instance_id = ? ORDER BY created_at',
 		).all(id)
 
-		// Compute active sleep: check if the instance is currently sleeping (in-memory)
+		// The "sleeping" / "waiting for events" introspection reads live-process
+		// in-memory registries. In thread mode those registries are populated in
+		// the worker, so route the reads through the binding (the thread router
+		// forwards them); in-process the binding runs them locally. Resolve the
+		// binding best-effort — a stopped/unknown worker just yields the defaults.
+		let sleeping = false
+		let waitingForEvents: string[] = []
+		try {
+			const binding = getWorkflowBinding(ctx, name)
+			if (instance.status === 'running') {
+				const r = await binding.executeControl({ kind: 'isSleeping', instanceId: id })
+				if (r.kind === 'isSleeping') sleeping = r.value
+			}
+			if (instance.status === 'waiting') {
+				const r = await binding.executeControl({ kind: 'waitingEventTypes', instanceId: id })
+				if (r.kind === 'waitingEventTypes') waitingForEvents = r.value
+			}
+		} catch {}
+
+		// Compute active sleep: find the latest sleep/sleepUntil step "until" time.
 		let activeSleep: WorkflowDetail['activeSleep'] = null
-		if (instance.status === 'running' && isInstanceSleeping(id)) {
-			// Find the latest sleep/sleepUntil step to get the "until" time
+		if (sleeping) {
 			for (let i = steps.length - 1; i >= 0; i--) {
 				const s = steps[i]!
 				if ((s.step_name.startsWith('sleep:') || s.step_name.startsWith('sleepUntil:')) && s.output) {
@@ -114,51 +131,37 @@ export const handlers = {
 			}
 		}
 
-		// Compute waiting event types from in-memory registry
-		const waitingForEvents = instance.status === 'waiting' ? getWaitingEventTypes(id) : []
-
 		return { ...instance, steps, stepAttempts, events, activeSleep, waitingForEvents } as WorkflowDetail
 	},
 
 	async 'workflows.terminate'({ name, id }: { name: string; id: string }, ctx: HandlerContext): Promise<OkResponse> {
-		const binding = getWorkflowBinding(ctx, name)
-		const instance = await binding.get(id)
-		await instance.terminate()
+		await getWorkflowBinding(ctx, name).executeControl({ kind: 'terminate', instanceId: id })
 		return { ok: true }
 	},
 
 	async 'workflows.create'({ name, params }: { name: string; params: string }, ctx: HandlerContext): Promise<{ ok: true; id: string }> {
-		const binding = getWorkflowBinding(ctx, name)
-		const parsed = JSON.parse(params)
-		const instance = await binding.create({ params: parsed })
-		return { ok: true, id: instance.id }
+		const result = await getWorkflowBinding(ctx, name).executeControl({ kind: 'create', params: JSON.parse(params) })
+		if (result.kind !== 'create') throw new Error('Unexpected workflow control result for create')
+		return { ok: true, id: result.id }
 	},
 
 	async 'workflows.pause'({ name, id }: { name: string; id: string }, ctx: HandlerContext): Promise<OkResponse> {
-		const binding = getWorkflowBinding(ctx, name)
-		const instance = await binding.get(id)
-		await instance.pause()
+		await getWorkflowBinding(ctx, name).executeControl({ kind: 'pause', instanceId: id })
 		return { ok: true }
 	},
 
 	async 'workflows.resume'({ name, id }: { name: string; id: string }, ctx: HandlerContext): Promise<OkResponse> {
-		const binding = getWorkflowBinding(ctx, name)
-		const instance = await binding.get(id)
-		await instance.resume()
+		await getWorkflowBinding(ctx, name).executeControl({ kind: 'resume', instanceId: id })
 		return { ok: true }
 	},
 
 	async 'workflows.restart'({ name, id, fromStep }: { name: string; id: string; fromStep?: string }, ctx: HandlerContext): Promise<OkResponse> {
-		const binding = getWorkflowBinding(ctx, name)
-		const instance = await binding.get(id)
-		await instance.restart(fromStep ? { fromStep } : undefined)
+		await getWorkflowBinding(ctx, name).executeControl({ kind: 'restart', instanceId: id, fromStep })
 		return { ok: true }
 	},
 
 	async 'workflows.skipSleep'({ name, id }: { name: string; id: string }, ctx: HandlerContext): Promise<OkResponse> {
-		const binding = getWorkflowBinding(ctx, name)
-		const instance = await binding.get(id)
-		await instance.skipSleep()
+		await getWorkflowBinding(ctx, name).executeControl({ kind: 'skipSleep', instanceId: id })
 		return { ok: true }
 	},
 
@@ -166,22 +169,24 @@ export const handlers = {
 		{ name, id, type, payload }: { name: string; id: string; type: string; payload?: string },
 		ctx: HandlerContext,
 	): Promise<OkResponse> {
-		const binding = getWorkflowBinding(ctx, name)
-		const instance = await binding.get(id)
-		const parsed = payload ? JSON.parse(payload) : undefined
-		await instance.sendEvent({ type, payload: parsed })
+		await getWorkflowBinding(ctx, name).executeControl({
+			kind: 'sendEvent',
+			instanceId: id,
+			eventType: type,
+			payload: payload ? JSON.parse(payload) : undefined,
+		})
 		return { ok: true }
 	},
 
 	async 'workflows.duplicate'({ name, id }: { name: string; id: string }, ctx: HandlerContext): Promise<{ ok: true; id: string }> {
-		const binding = getWorkflowBinding(ctx, name)
 		const db = getDatabase()
 		const row = db.query<{ params: string | null }, [string]>(
 			'SELECT params FROM workflow_instances WHERE id = ?',
 		).get(id)
 		if (!row) throw new Error('Workflow instance not found')
 		const params = row.params !== null ? JSON.parse(row.params) : {}
-		const newInstance = await binding.create({ params })
-		return { ok: true, id: newInstance.id }
+		const result = await getWorkflowBinding(ctx, name).executeControl({ kind: 'create', params })
+		if (result.kind !== 'create') throw new Error('Unexpected workflow control result for duplicate')
+		return { ok: true, id: result.id }
 	},
 }

package/src/bindings/container-cleanup.ts

@@ -0,0 +1,125 @@
+/**
+ * Main-side registry of active Docker containers spawned by lopata.
+ *
+ * Containers come from two places: the in-process path (dashboard API
+ * handlers + the legacy/vite-plugin executor in `env.ts`) and the DO worker
+ * threads. DO workers call `register` / `unregister` via postMessage
+ * (`container-registered` / `container-removed` in `do-executor-worker.ts`);
+ * the in-process callers invoke the helpers directly through the
+ * `DockerManager` constructor callbacks.
+ *
+ * Two safety nets:
+ *
+ *   1. An `exit` listener `docker rm -f`s anything still tracked at process
+ *      shutdown. Synchronous (`Bun.spawnSync`) since `exit` doesn't await.
+ *
+ *   2. `reapOrphanContainers()` scans for containers labeled `lopata=1`
+ *      whose `lopata.pid` is no longer alive — cleans up leftovers from
+ *      crashes / SIGKILL where (1) never ran. Invoked once at lopata
+ *      startup from `cli/dev.ts`.
+ */
+
+import { $ } from 'bun'
+
+/** Marker label on every lopata-managed container. */
+export const LOPATA_LABEL_KEY = 'lopata'
+export const LOPATA_LABEL_VALUE = '1'
+/** Pid sub-label so the reaper can detect orphans of dead processes. */
+export const LOPATA_PID_LABEL_KEY = 'lopata.pid'
+
+/** Labels passed to `DockerManager` (constructor) for every `docker run`. */
+export function containerLabels(): Record<string, string> {
+	return {
+		[LOPATA_LABEL_KEY]: LOPATA_LABEL_VALUE,
+		[LOPATA_PID_LABEL_KEY]: String(process.pid),
+	}
+}
+
+const activeContainers = new Set<string>()
+let exitHandlerRegistered = false
+
+function ensureExitHandlerRegistered(): void {
+	if (exitHandlerRegistered) return
+	exitHandlerRegistered = true
+	process.on('exit', () => {
+		if (activeContainers.size === 0) return
+		// `exit` is synchronous-only; a single batched call keeps shutdown
+		// snappy even with many containers. `docker rm -f` accepts a list.
+		try {
+			Bun.spawnSync(['docker', 'rm', '-f', ...activeContainers])
+		} catch {
+			// best-effort cleanup
+		}
+		activeContainers.clear()
+	})
+}
+
+export function registerContainer(name: string): void {
+	ensureExitHandlerRegistered()
+	activeContainers.add(name)
+}
+
+export function unregisterContainer(name: string): void {
+	activeContainers.delete(name)
+}
+
+/** @internal Test hook — current tracked container set. */
+export function _activeContainersSnapshot(): string[] {
+	return [...activeContainers]
+}
+
+/**
+ * Reap containers labeled `lopata=1` whose `lopata.pid` is no longer alive.
+ * Returns the number of orphans removed. Silently returns 0 if docker isn't
+ * available so non-container users aren't penalised.
+ */
+export async function reapOrphanContainers(): Promise<number> {
+	let listOut: string
+	try {
+		const format = `{{.Names}}|{{.Label "${LOPATA_PID_LABEL_KEY}"}}`
+		const result = await $`docker ps -a --filter label=${LOPATA_LABEL_KEY}=${LOPATA_LABEL_VALUE} --format ${format}`.quiet().nothrow()
+		if (result.exitCode !== 0) return 0
+		listOut = result.stdout.toString()
+	} catch {
+		return 0
+	}
+
+	const orphans = _selectOrphanNames(listOut, isProcessAlive)
+	if (orphans.length === 0) return 0
+	await $`docker rm -f ${orphans}`.quiet().nothrow()
+	return orphans.length
+}
+
+/**
+ * @internal Pure-function core of {@link reapOrphanContainers}, exported for
+ * tests. Parses the `docker ps` `{{.Names}}|{{.Label ...}}` table and returns
+ * the container names whose owner pid is unknown, unparseable, or dead.
+ */
+export function _selectOrphanNames(listOut: string, alive: (pid: number) => boolean): string[] {
+	const orphans: string[] = []
+	for (const line of listOut.split('\n')) {
+		if (!line) continue
+		const sep = line.indexOf('|')
+		const name = sep === -1 ? line : line.slice(0, sep)
+		const pidStr = sep === -1 ? '' : line.slice(sep + 1)
+		if (!name) continue
+		const pid = pidStr ? Number.parseInt(pidStr, 10) : Number.NaN
+		// Live pid → another lopata owns it. Missing/unparseable/dead/≤0 → orphan.
+		if (Number.isFinite(pid) && pid > 0 && alive(pid)) continue
+		orphans.push(name)
+	}
+	return orphans
+}
+
+function isProcessAlive(pid: number): boolean {
+	// `process.kill(0, 0)` on POSIX signals the whole process group, which
+	// succeeds whenever the caller has a group — so pid 0 must NOT be treated
+	// as "alive". Negative pids are invalid; treat both as dead.
+	if (pid <= 0) return false
+	try {
+		process.kill(pid, 0)
+		return true
+	} catch {
+		return false
+	}
+}

package/src/bindings/container-docker.ts

@@ -15,36 +15,9 @@ export interface DockerContainerInfo {
 	state: string // "running", "exited", "created", etc.
 	exitCode: number | null
 	ports: Record<string, string>
-}
-
-// Track active containers for cleanup on process exit
-const activeContainers = new Set<string>()
-let cleanupRegistered = false
-
-function registerCleanup() {
-	if (cleanupRegistered) return
-	cleanupRegistered = true
-
-	const cleanup = () => {
-		for (const name of activeContainers) {
-			try {
-				Bun.spawnSync(['docker', 'rm', '-f', name])
-			} catch {
-				// best-effort cleanup
-			}
-		}
-		activeContainers.clear()
-	}
-
-	process.on('SIGINT', () => {
-		cleanup()
-		process.exit(130)
-	})
-	process.on('SIGTERM', () => {
-		cleanup()
-		process.exit(143)
-	})
-	process.on('exit', cleanup)
+	/** Container labels (e.g. `lopata.pid`) — used to detect a container left by a
+	 *  crashed/foreign lopata process before adopting it by name. */
+	labels: Record<string, string>
 }
 
 // Image build cache: tag -> { mtime }
@@ -52,7 +25,29 @@ const imageCache = new Map<string, { mtime: number }>()
 
 const DOCKER_JSON_FORMAT = '{{json .}}'
 
+export interface DockerManagerOptions {
+	/** Called after a container is successfully created via `run()`. */
+	onRegister?: (name: string) => void
+	/** Called after a container is removed via `remove()`. */
+	onRemove?: (name: string) => void
+	/**
+	 * Labels passed to every `docker run` (`--label key=value`).
+	 * Cleanup uses these to detect orphans from crashed processes.
+	 */
+	labels?: Record<string, string>
+}
+
 export class DockerManager {
+	private _onRegister?: (name: string) => void
+	private _onRemove?: (name: string) => void
+	private _labels?: Record<string, string>
+
+	constructor(options?: DockerManagerOptions) {
+		this._onRegister = options?.onRegister
+		this._onRemove = options?.onRemove
+		this._labels = options?.labels
+	}
+
 	/**
 	 * Build an image from a Dockerfile, with lazy mtime-based caching.
 	 * Skips rebuild if the Dockerfile hasn't changed since last build.
@@ -80,10 +75,15 @@ export class DockerManager {
 	 * Run a container and return its container ID.
 	 */
 	async run(options: DockerRunOptions): Promise<string> {
-		registerCleanup()
-
 		const args: string[] = ['docker', 'run', '-d', '--name', options.name]
 
+		// Labels (orphan reaping etc.)
+		if (this._labels) {
+			for (const [key, value] of Object.entries(this._labels)) {
+				args.push('--label', `${key}=${value}`)
+			}
+		}
+
 		// Port mappings
 		for (const [containerPort, hostPort] of options.ports) {
 			args.push('-p', `${hostPort}:${containerPort}`)
@@ -120,10 +120,19 @@ export class DockerManager {
 		}
 
 		const containerId = result.stdout.toString().trim()
-		activeContainers.add(options.name)
+		this._onRegister?.(options.name)
 		return containerId
 	}
 
+	/**
+	 * Fire the `onRegister` callback for a container we didn't create via
+	 * `run()` — e.g. one adopted at startup because it was already running.
+	 * Lets cleanup hooks (process-exit handler) track it the same way.
+	 */
+	registerExisting(name: string): void {
+		this._onRegister?.(name)
+	}
+
 	/**
 	 * Stop a running container gracefully.
 	 */
@@ -172,12 +181,18 @@ export class DockerManager {
 			}
 
 			const state = data.State?.Status ?? 'unknown'
+			const rawLabels = data.Config?.Labels ?? {}
+			const labels: Record<string, string> = {}
+			for (const [k, v] of Object.entries(rawLabels)) {
+				if (typeof v === 'string') labels[k] = v
+			}
 			return {
 				id: data.Id ?? '',
 				name: (data.Name ?? '').replace(/^\//, ''),
 				state,
 				exitCode: state === 'running' ? null : (data.State?.ExitCode ?? null),
 				ports,
+				labels,
 			}
 		} catch {
 			return null
@@ -202,7 +217,7 @@ export class DockerManager {
 	 */
 	async remove(name: string): Promise<void> {
 		await $`docker rm -f ${name}`.quiet().nothrow()
-		activeContainers.delete(name)
+		this._onRemove?.(name)
 	}
 
 	/**

package/src/bindings/container.ts

@@ -1,3 +1,4 @@
+import { LOPATA_PID_LABEL_KEY } from './container-cleanup'
 import { DockerManager, type DockerRunOptions } from './container-docker'
 import { DurableObjectBase, type DurableObjectStateImpl } from './durable-object'
 
@@ -97,20 +98,32 @@ export class ContainerRuntime {
 
 		this._transition('running')
 
-		// Check if a container with this name already exists (e.g. after process crash)
+		// Check if a container with this name already exists (e.g. survived a reload
+		// in this same process, or was left behind by a crashed/foreign lopata run).
 		const existing = await this._docker.inspect(this._containerName)
 		if (existing) {
-			if (existing.state === 'running') {
+			// Only adopt a running container THIS process created (matching pid
+			// label). A stale label means the creator crashed/exited — its config,
+			// env and port mappings may not match what we'd build now, so removing
+			// and recreating is safer than silently adopting foreign state. (Labels
+			// are immutable, so an adopted container keeps the dead creator's pid.)
+			const ownerPid = existing.labels[LOPATA_PID_LABEL_KEY]
+			const ownedByThisProcess = ownerPid === String(process.pid)
+			if (existing.state === 'running' && ownedByThisProcess) {
 				// Recover port mappings from the running container
 				this._hostPorts.clear()
 				this._recoverPortMappings(existing.ports)
+				// `run()` would have fired onRegister; do the same for adoption so
+				// the process-exit cleanup tracks recovered containers too.
+				this._docker.registerExisting(this._containerName)
 				await this.onStart?.()
 				this._startHealthCheck()
 				this._startMonitor()
 				this.renewActivityTimeout()
 				return
 			}
-			// Container exists but isn't running — remove it before creating a new one
+			// Not running, or owned by a different/dead lopata process — remove it
+			// before creating a fresh one.
 			await this._docker.remove(this._containerName)
 		}
 
@@ -292,15 +305,17 @@ export class ContainerRuntime {
 	// ─── Private ────────────────────────────────────────────────────────────
 
 	/**
-	 * Recover host port mappings from docker inspect Ports object.
-	 * Format: { "8080/tcp": [{ "HostIp": "...", "HostPort": "49152" }], ... }
+	 * Recover host port mappings from the flattened ports map returned by
+	 * `DockerManager.inspect()`: { "8080/tcp": "0.0.0.0:32768", ... }.
+	 * (inspect() already flattens docker's raw `[{HostIp,HostPort}]` shape, so
+	 * the value here is a "host:port" string, not an array.)
 	 */
-	private _recoverPortMappings(ports: Record<string, unknown>) {
-		for (const [key, bindings] of Object.entries(ports)) {
-			if (!Array.isArray(bindings) || bindings.length === 0) continue
+	private _recoverPortMappings(ports: Record<string, string>) {
+		for (const [key, value] of Object.entries(ports)) {
+			if (typeof value !== 'string') continue
 			const containerPort = parseInt(key, 10)
 			if (Number.isNaN(containerPort)) continue
-			const hostPort = parseInt(bindings[0].HostPort, 10)
+			const hostPort = parseInt(value.split(':').pop() ?? '', 10)
 			if (Number.isNaN(hostPort)) continue
 			this._hostPorts.set(containerPort, hostPort)
 		}

package/src/bindings/do-executor-inprocess.ts

@@ -1,7 +1,6 @@
 import { warnInvalidRpcArgs } from '../rpc-validate'
 import type { DOExecutor, DOExecutorFactory, ExecutorConfig } from './do-executor'
 import { type DurableObjectBase, DurableObjectStateImpl } from './durable-object'
-import { createRpcFunctionStub, wrapRpcReturnValue } from './rpc-stub'
 
 export class InProcessExecutor implements DOExecutor {
 	private _state: DurableObjectStateImpl
@@ -63,8 +62,11 @@ export class InProcessExecutor implements DOExecutor {
 		try {
 			const val = (this._instance as unknown as Record<string, unknown>)[method]
 			if (typeof val === 'function') {
-				const result = await (val as (...a: unknown[]) => unknown).call(this._instance, ...args)
-				return wrapRpcReturnValue(result, method)
+				// Return the raw result — the namespace `get()` stub wraps it once via
+				// wrapRpcReturnValue, matching the worker-thread executor (whose
+				// executeRpc also returns a raw value). Wrapping here too would
+				// double-wrap and emit warnInvalidRpcReturn twice.
+				return await (val as (...a: unknown[]) => unknown).call(this._instance, ...args)
 			}
 			throw new Error(`"${method}" is not a method on the Durable Object`)
 		} finally {
@@ -77,9 +79,11 @@ export class InProcessExecutor implements DOExecutor {
 		try {
 			const val = (this._instance as unknown as Record<string, unknown>)[prop]
 			if (typeof val === 'function') {
-				return createRpcFunctionStub(val as Function, this._instance)
+				// Re-dispatching callable (mirrors WorkerExecutor.executeRpcGet); the
+				// stub wraps it once. Avoids the double function-stub wrap.
+				return (...args: unknown[]) => this.executeRpc(prop, args)
 			}
-			return wrapRpcReturnValue(val, prop)
+			return val
 		} finally {
 			this._state._exit()
 		}