@bbigbang/core 0.1.0

package/dist/services/workspaceToolActionRunner.js

@@ -0,0 +1,1582 @@
+import { randomUUID } from 'node:crypto';
+import { log } from '@bbigbang/runtime-acp';
+import { actionRequiresCommand, parseWorkspaceToolManifest, resolveWorkspaceToolActionMode, } from './workspaceToolManifest.js';
+import { WorkspaceToolServiceError } from './workspaceToolErrors.js';
+import { buildWorkspaceToolStartupCommand, buildWorkspaceToolRuntimeCommand, CLI_STOP_CLEANUP_TIMEOUT_MS, closeTerminalBestEffort, deriveOneShotRunCompletion, deriveTerminalRunCompletion, extractWorkspaceToolBusinessStatus, getPublishedWorkspaceToolBundleRoot, isJsonlEventEnvelope, normalizeActionParams, normalizeStateRow, parseStateRowFromJsonlEvent, parseStateRowFromOutput, resolveToolCwd, resolveToolPath, rewriteWorkspaceToolCommandPaths, serializeJson, sleep, validateStateRowForManifest, waitForTerminalExit, } from './workspaceToolExecutionUtils.js';
+import { WorkspaceToolRunLedger } from './workspaceToolRunLedger.js';
+import { PersistentRuntimeJsonlClient } from './persistentRuntimeJsonlClient.js';
+import { WorkspaceToolTerminalLifecycle, defaultPersistentRuntimeIdleTtlSeconds, } from './workspaceToolTerminalLifecycle.js';
+import { WorkspaceToolUploadMaterializer, } from './workspaceToolUploadMaterializer.js';
+const PERSISTENT_STARTUP_OUTPUT_SETTLE_MS = 1000;
+const PERSISTENT_STARTUP_OUTPUT_POLL_MS = 50;
+const PERSISTENT_STARTUP_SNAPSHOT_FALLBACK_MS = 2000;
+const PERSISTENT_STARTUP_DUPLICATE_IGNORE_IDLE_MS = 1000;
+const PERSISTENT_RUNTIME_ACTION_TIMEOUT_MS = 120_000;
+function workspaceToolErrorWithRunId(error, runId) {
+    if (error instanceof WorkspaceToolServiceError) {
+        if (!error.runId)
+            error.runId = runId;
+        return error;
+    }
+    const wrapped = new WorkspaceToolServiceError(String(error?.message ?? error), 500, { runId });
+    wrapped.cause = error;
+    return wrapped;
+}
+function loadNodeHostname(db, nodeId) {
+    const row = db.prepare(`SELECT hostname
+       FROM nodes
+      WHERE node_id = ?
+      LIMIT 1`).get(nodeId);
+    return row?.hostname ?? null;
+}
+function isMissingTerminalError(error) {
+    const message = String(error?.message ?? error).toLowerCase();
+    return message.includes('not_found:terminal not found')
+        || message.includes('terminal not found')
+        || message.includes('unknown terminal');
+}
+export class WorkspaceToolActionRunner {
+    deps;
+    /*
+     * Run completion contract:
+     * - completed: one-shot terminal exitCode=0, persistent_runtime result event,
+     *   or a stop action whose optional cleanup command exits successfully.
+     * - failed: one-shot non-zero exit/validation error, persistent_runtime error
+     *   event/read failure, missing terminal while sending/closing, or cleanup
+     *   failure that is not preserving an earlier action failure.
+     * - timed_out: persistent runtime action exceeded the JSONL result/error
+     *   deadline, maxRunSeconds, or idle TTL without a protocol terminal event.
+     * - cancelled: user stop/restart/delete closes a previously active runtime
+     *   terminal; missing already-closed terminals are recorded with
+     *   signal='missing_terminal'.
+     *
+     * Latest-state writes are flushed before returning tool info to callers. Run
+     * completion rows are ledger facts; status-only snapshot sync after terminal
+     * lifecycle completion is best-effort and must not strand a completed run.
+     */
+    ignoredRealtimeStartupLineTimeouts = new Map();
+    observedPersistentStartupOutput = new Set();
+    startupStateReplaySuppressedRunIds = new Set();
+    ignoredRealtimeStartupLines = new Map();
+    runLedger;
+    uploadMaterializer;
+    latestStateProjector;
+    jsonlClient;
+    terminalLifecycle;
+    constructor(deps) {
+        this.deps = deps;
+        this.runLedger = new WorkspaceToolRunLedger({
+            db: this.deps.db,
+            loadUsername: (userId) => this.deps.loadUsername(userId),
+            isAdminUser: (userId) => this.deps.isAdminUser(userId),
+        });
+        this.uploadMaterializer = new WorkspaceToolUploadMaterializer({
+            db: this.deps.db,
+            isAdminUser: (userId) => this.deps.isAdminUser(userId),
+            hasRunRow: (runId) => this.runLedger.hasRunRow(runId),
+            writeWorkspaceFile: (nodeId, workspaceRoot, relativePath, content, mode, options) => this.deps.writeWorkspaceFile(nodeId, workspaceRoot, relativePath, content, mode, options),
+            deleteWorkspaceFile: (nodeId, workspaceRoot, relativePath) => this.deps.deleteWorkspaceFile(nodeId, workspaceRoot, relativePath),
+        });
+        this.latestStateProjector = deps.latestStateProjector;
+        this.jsonlClient = new PersistentRuntimeJsonlClient({
+            readWorkspaceFile: (nodeId, workspaceRoot, relativePath, options) => this.deps.readWorkspaceFile(nodeId, workspaceRoot, relativePath, options),
+            getActiveTerminalId: (toolId) => this.loadActiveTerminalId(toolId),
+        });
+        const activeTerminalBuffers = new Map();
+        this.terminalLifecycle = new WorkspaceToolTerminalLifecycle({
+            db: this.deps.db,
+            runLedger: this.runLedger,
+            latestStateProjector: this.latestStateProjector,
+            terminalBroker: this.deps.terminalBroker,
+            enqueueToolMutation: (toolId, work) => this.deps.enqueueToolMutation(toolId, work),
+            onBeforeFinish: (terminalId, runId) => {
+                this.flushRealtimeStateBuffer(terminalId);
+                this.clearPersistentStartupOutputObserved(runId);
+                this.clearIgnoredRealtimeStartupLines(terminalId);
+            },
+            maxActivePersistentRunsPerAgent: this.deps.maxActivePersistentRunsPerAgent,
+        }, activeTerminalBuffers);
+    }
+    processTerminalOutput(terminalId, chunk) {
+        const ctx = this.terminalLifecycle.getContext(terminalId);
+        if (!ctx)
+            return;
+        this.terminalLifecycle.resetPersistentIdleTimeout(terminalId);
+        this.processRealtimeStateChunk(ctx, terminalId, chunk);
+    }
+    processRealtimeStateChunk(ctx, terminalId, chunk) {
+        chunk = this.prepareRealtimeChunkAfterSnapshotFallback(ctx, terminalId, chunk);
+        if (!chunk)
+            return;
+        ctx.lineBuffer += chunk;
+        const lines = ctx.lineBuffer.split('\n');
+        ctx.lineBuffer = lines.pop() ?? '';
+        for (const line of lines) {
+            this.processRealtimeStateLine(ctx, terminalId, line);
+        }
+    }
+    processRealtimeStateLine(ctx, terminalId, line, options = {}) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            return false;
+        const candidate = extractWorkspaceToolJsonLineCandidate(trimmed);
+        if (options.consumeIgnoredRealtimeLine !== false && (this.consumeIgnoredRealtimeStartupLine(terminalId, trimmed)
+            || (candidate ? this.consumeIgnoredRealtimeStartupLine(terminalId, candidate) : false))) {
+            return true;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(candidate ?? trimmed);
+        }
+        catch {
+            return false;
+        }
+        try {
+            const isEvent = isJsonlEventEnvelope(parsed);
+            const state = isEvent
+                ? parseStateRowFromJsonlEvent(parsed)
+                : normalizeStateRow(parsed, { rejectUnsupportedMediaKinds: true });
+            if (!state && isEvent) {
+                if (ctx.runId) {
+                    const inserted = this.insertRunEventFromEnvelope(ctx.runId, ctx.toolId, parsed);
+                    if (inserted) {
+                        this.markPersistentStartupOutputObserved(ctx.runId);
+                    }
+                    return inserted;
+                }
+            }
+            if (!state)
+                return false;
+            validateStateRowForManifest(ctx.manifest, state);
+            if (options.allowStateUpdate === false)
+                return true;
+            this.scheduleRealtimeStateUpdate(ctx, state);
+            if (ctx.runId) {
+                this.markPersistentStartupOutputObserved(ctx.runId);
+            }
+            return true;
+        }
+        catch (error) {
+            log.warn('[workspace-tools] ignoring invalid realtime tool state line', {
+                toolId: ctx.toolId,
+                terminalId,
+                error: String(error?.message ?? error),
+            });
+            return false;
+        }
+    }
+    scheduleRealtimeStateUpdate(ctx, state) {
+        this.latestStateProjector.scheduleRealtimeStateUpdate(ctx, state);
+    }
+    flushPendingRealtimeStateForTool(toolId) {
+        this.latestStateProjector.flushPendingRealtimeStateForTool(toolId);
+    }
+    clearPendingRealtimeStateForTool(toolId) {
+        this.latestStateProjector.clearPendingRealtimeStateForTool(toolId);
+    }
+    flushRealtimeStateBuffer(terminalId) {
+        const ctx = this.terminalLifecycle.getContext(terminalId);
+        if (!ctx)
+            return;
+        if (ctx.lineBuffer.trim()) {
+            this.processRealtimeStateLine(ctx, terminalId, ctx.lineBuffer);
+            ctx.lineBuffer = '';
+        }
+        this.latestStateProjector.flushPendingRealtimeStateForTool(ctx.toolId);
+    }
+    getToolInfoForActionViewerAfterPendingRealtimeFlush(params) {
+        return this.latestStateProjector.getToolInfoForActionViewerAfterPendingRealtimeFlush({
+            viewerUserId: params.viewerUserId,
+            toolId: params.toolRow.toolId,
+            allowAdminBypass: params.allowAdminBypass,
+        });
+    }
+    async invokeAction(params) {
+        const manifest = parseWorkspaceToolManifest(params.toolRow.manifestJson);
+        const action = manifest.actions.find((item) => item.actionId === params.actionId);
+        if (!action) {
+            throw new WorkspaceToolServiceError('Tool action not found.', 404);
+        }
+        const actionMode = resolveWorkspaceToolActionMode(action);
+        const actionParams = normalizeActionParams(action, params.inputParams ?? {});
+        if (actionMode === 'notify_agent') {
+            return this.deps.notifyAgentForToolAction({
+                toolRow: params.toolRow,
+                action,
+                viewerUserId: params.viewerUserId,
+                allowAdminBypass: params.allowAdminBypass,
+                requestedByAgentId: params.requestedByAgentId,
+                actionParams,
+            });
+        }
+        const agent = this.deps.conversationManager.getAgent(params.toolRow.agentId);
+        if (!agent?.nodeId || !agent.workspacePath) {
+            throw new WorkspaceToolServiceError('Agent workspace is unavailable for tool execution.', 409);
+        }
+        const agentNodeId = agent.nodeId;
+        const workspaceRoot = agent.workspacePath;
+        const executionHostname = loadNodeHostname(this.deps.db, agentNodeId);
+        const publishedBundleRoot = getPublishedWorkspaceToolBundleRoot(params.toolRow.toolId, params.toolRow.revision);
+        const actionCwdAbsolute = resolveToolCwd(workspaceRoot, manifest, action, params.toolRow.bundleRoot, publishedBundleRoot);
+        const runId = randomUUID();
+        const runStartedAt = Date.now();
+        const usesPersistentRuntime = manifest.runtime?.mode === 'persistent_runtime';
+        const hasExecutionEntry = manifest.cli?.entry?.trim() || (usesPersistentRuntime && manifest.runtime?.entry?.trim());
+        if (actionRequiresCommand(action) && !action.command?.trim() && !hasExecutionEntry) {
+            throw new WorkspaceToolServiceError(`Action "${action.actionId}" requires a command or manifest cli.entry.`, 400);
+        }
+        let materializedRun = null;
+        const getMaterializedRun = async () => {
+            if (materializedRun)
+                return materializedRun;
+            const materializedActionParams = await this.uploadMaterializer.materializeFileActionParams({
+                action,
+                actionParams,
+                runId,
+                toolId: params.toolRow.toolId,
+                viewerUserId: params.viewerUserId,
+                agentNodeId,
+                workspaceRoot,
+            });
+            const requesterAgent = params.requestedByAgentId
+                ? this.deps.conversationManager.getAgent(params.requestedByAgentId)
+                : null;
+            const requestedByUsername = requesterAgent
+                ? requesterAgent.name
+                : this.deps.loadUsername(params.viewerUserId);
+            const runBase = {
+                runId,
+                toolId: params.toolRow.toolId,
+                actionId: action.actionId,
+                terminalId: null,
+                status: 'running',
+                paramsJson: serializeJson(materializedActionParams),
+                exitCode: null,
+                signal: null,
+                startedAt: runStartedAt,
+                endedAt: null,
+                requestedByUserId: requesterAgent ? null : params.viewerUserId,
+                requestedByUsername,
+                requestedByAgentId: requesterAgent ? params.requestedByAgentId : null,
+                requestedByAgentName: requesterAgent ? requesterAgent.name : null,
+                toolRevision: params.toolRow.revision,
+                sourceBundleRoot: params.toolRow.bundleRoot,
+                actionCwdAbsolute,
+                runtimeLogDir: usesPersistentRuntime ? this.jsonlClient.normalizeLogDir(params.toolRow.toolId, manifest) : null,
+                executionTargetKind: 'owner_agent',
+                executionAgentId: params.toolRow.agentId,
+                executionNodeId: agentNodeId,
+                executionHostname,
+                executionWorkspacePath: workspaceRoot,
+            };
+            materializedRun = {
+                actionParams: materializedActionParams,
+                run: runBase,
+                stagedUploadPaths: this.uploadMaterializer.collectMaterializedUploadPaths(materializedActionParams),
+            };
+            return materializedRun;
+        };
+        if (usesPersistentRuntime) {
+            const materialized = await getMaterializedRun();
+            const { actionParams: materializedActionParams, run: runBase } = materialized;
+            return this.uploadMaterializer.invokeWithMaterializedUploads({ runId: materialized.run.runId, agentNodeId, workspaceRoot, stagedUploadPaths: materialized.stagedUploadPaths }, () => this.runPersistentRuntimeAction({
+                toolRow: params.toolRow,
+                manifest,
+                action,
+                viewerUserId: params.viewerUserId,
+                allowAdminBypass: params.allowAdminBypass,
+                agentNodeId,
+                workspaceRoot,
+                sourceBundleRoot: params.toolRow.bundleRoot,
+                publishedBundleRoot,
+                actionParams: materializedActionParams,
+                run: runBase,
+            }));
+        }
+        if (action.kind === 'start') {
+            if (params.toolRow.activeTerminalId) {
+                throw new WorkspaceToolServiceError('Tool already has an active persistent terminal.', 409);
+            }
+            return this.deps.enqueueAgentPersistentStart(params.toolRow.agentId, async () => {
+                this.terminalLifecycle.assertAgentPersistentRunQuota(params.toolRow.agentId);
+                const materialized = await getMaterializedRun();
+                const { actionParams: materializedActionParams, run: runBase } = materialized;
+                return this.uploadMaterializer.invokeWithMaterializedUploads({ runId: materialized.run.runId, agentNodeId, workspaceRoot, stagedUploadPaths: materialized.stagedUploadPaths }, () => this.startPersistentAction({
+                    toolRow: params.toolRow,
+                    manifest,
+                    action,
+                    viewerUserId: params.viewerUserId,
+                    allowAdminBypass: params.allowAdminBypass,
+                    agentNodeId,
+                    workspaceRoot,
+                    sourceBundleRoot: params.toolRow.bundleRoot,
+                    publishedBundleRoot,
+                    actionParams: materializedActionParams,
+                    run: runBase,
+                }));
+            });
+        }
+        if (action.kind === 'restart') {
+            const materialized = await getMaterializedRun();
+            const { actionParams: materializedActionParams, run: runBase } = materialized;
+            if (params.toolRow.activeTerminalId) {
+                await this.deps.terminalBroker.closeTerminal(agentNodeId, params.toolRow.activeTerminalId);
+                this.finishActivePersistentRun(params.toolRow.toolId, params.toolRow.activeTerminalId, { status: 'cancelled' });
+            }
+            return this.deps.enqueueAgentPersistentStart(params.toolRow.agentId, async () => {
+                this.terminalLifecycle.assertAgentPersistentRunQuota(params.toolRow.agentId);
+                return this.uploadMaterializer.invokeWithMaterializedUploads({ runId: materialized.run.runId, agentNodeId, workspaceRoot, stagedUploadPaths: materialized.stagedUploadPaths }, () => this.startPersistentAction({
+                    toolRow: params.toolRow,
+                    manifest,
+                    action,
+                    viewerUserId: params.viewerUserId,
+                    allowAdminBypass: params.allowAdminBypass,
+                    agentNodeId,
+                    workspaceRoot,
+                    sourceBundleRoot: params.toolRow.bundleRoot,
+                    publishedBundleRoot,
+                    actionParams: materializedActionParams,
+                    run: runBase,
+                }));
+            });
+        }
+        if (action.kind === 'stop') {
+            if (!params.toolRow.activeTerminalId) {
+                throw new WorkspaceToolServiceError('Tool is not running.', 409);
+            }
+            const materialized = await getMaterializedRun();
+            const { actionParams: materializedActionParams, run: runBase } = materialized;
+            if (action.command?.trim()) {
+                return this.uploadMaterializer.invokeWithMaterializedUploads({ runId: materialized.run.runId, agentNodeId, workspaceRoot, stagedUploadPaths: materialized.stagedUploadPaths }, () => this.runOneShotAction({
+                    toolRow: params.toolRow,
+                    manifest,
+                    action,
+                    viewerUserId: params.viewerUserId,
+                    allowAdminBypass: params.allowAdminBypass,
+                    agentNodeId,
+                    workspaceRoot,
+                    sourceBundleRoot: params.toolRow.bundleRoot,
+                    publishedBundleRoot,
+                    actionParams: materializedActionParams,
+                    run: runBase,
+                    strictStatusJson: false,
+                }));
+            }
+            return this.uploadMaterializer.invokeWithMaterializedUploads({ runId: materialized.run.runId, agentNodeId, workspaceRoot, stagedUploadPaths: materialized.stagedUploadPaths }, async () => {
+                await this.deps.terminalBroker.closeTerminal(agentNodeId, params.toolRow.activeTerminalId);
+                this.finishActivePersistentRun(params.toolRow.toolId, params.toolRow.activeTerminalId, { status: 'cancelled' });
+                const runTerminalId = params.toolRow.activeTerminalId;
+                this.insertRun({
+                    ...runBase,
+                    terminalId: runTerminalId,
+                });
+                let cleanupResult = null;
+                if (manifest.cli?.entry?.trim()) {
+                    cleanupResult = await this.runCliStopCleanupBestEffort({
+                        toolRow: params.toolRow,
+                        manifest,
+                        action,
+                        agentNodeId,
+                        workspaceRoot,
+                        sourceBundleRoot: params.toolRow.bundleRoot,
+                        publishedBundleRoot,
+                        actionParams: materializedActionParams,
+                        runId: runBase.runId,
+                    });
+                }
+                const completedAt = Date.now();
+                const cleanupCompletion = cleanupResult?.completion ?? {
+                    status: 'completed',
+                    exitCode: 0,
+                    signal: null,
+                };
+                this.completeRun(runBase.runId, {
+                    status: cleanupCompletion.status,
+                    endedAt: completedAt,
+                    exitCode: cleanupCompletion.exitCode,
+                    signal: cleanupCompletion.signal,
+                });
+                await this.refreshLatestStateFromStatusActionSilently({
+                    toolRow: params.toolRow,
+                    manifest,
+                    agentNodeId,
+                    workspaceRoot,
+                    sourceBundleRoot: params.toolRow.bundleRoot,
+                    publishedBundleRoot,
+                });
+                const tool = this.deps.getToolInfoForActionViewer(params.viewerUserId, params.toolRow.toolId, params.allowAdminBypass);
+                const run = this.runInfoForActionViewer({
+                    ...runBase,
+                    terminalId: runTerminalId,
+                    status: cleanupCompletion.status,
+                    endedAt: completedAt,
+                    exitCode: cleanupCompletion.exitCode,
+                    signal: cleanupCompletion.signal,
+                }, action.kind, params.viewerUserId, params.toolRow.userId);
+                await this.deps.recordPlatformExecNotification({
+                    toolRow: params.toolRow,
+                    action,
+                    run,
+                    viewerUserId: params.viewerUserId,
+                    actionParams: materializedActionParams,
+                    outputText: null,
+                });
+                return {
+                    mode: 'platform_exec',
+                    tool,
+                    run,
+                    latestState: tool.latestState ?? null,
+                    outputText: null,
+                };
+            });
+        }
+        const materialized = await getMaterializedRun();
+        const { actionParams: materializedActionParams, run: runBase } = materialized;
+        return this.uploadMaterializer.invokeWithMaterializedUploads({ runId: materialized.run.runId, agentNodeId, workspaceRoot, stagedUploadPaths: materialized.stagedUploadPaths }, () => this.runOneShotAction({
+            toolRow: params.toolRow,
+            manifest,
+            action,
+            viewerUserId: params.viewerUserId,
+            allowAdminBypass: params.allowAdminBypass,
+            agentNodeId,
+            workspaceRoot,
+            sourceBundleRoot: params.toolRow.bundleRoot,
+            publishedBundleRoot,
+            actionParams: materializedActionParams,
+            run: runBase,
+            strictStatusJson: action.kind === 'status',
+        }));
+    }
+    async runCliStopCleanupBestEffort(params) {
+        this.insertCleanupRunEvent(params.runId, params.toolRow.toolId, 'cleanup_started', {
+            summary: 'Cleanup started',
+            actionId: params.action.actionId,
+            timeoutMs: CLI_STOP_CLEANUP_TIMEOUT_MS,
+        });
+        const cwd = resolveToolCwd(params.workspaceRoot, params.manifest, params.action, params.sourceBundleRoot, params.publishedBundleRoot);
+        const command = rewriteWorkspaceToolCommandPaths(params.workspaceRoot, buildWorkspaceToolStartupCommand({
+            manifest: params.manifest,
+            action: params.action,
+            actionParams: params.actionParams,
+            workspaceRoot: params.workspaceRoot,
+            publishedBundleRoot: params.publishedBundleRoot,
+        }), params.sourceBundleRoot, params.publishedBundleRoot);
+        let terminalId = null;
+        let latestObservedOutputText = '';
+        try {
+            const terminal = await this.deps.terminalBroker.createTerminal(params.agentNodeId, {
+                workspaceRoot: params.workspaceRoot,
+                cwd,
+                name: `${params.toolRow.name}: ${params.action.label} (cleanup)`,
+                startupCommand: command,
+                closeOnStartupCommand: true,
+            });
+            terminalId = terminal.terminalId;
+            const snapshot = await waitForTerminalExit(this.deps.terminalBroker, params.agentNodeId, terminalId, {
+                timeoutMs: CLI_STOP_CLEANUP_TIMEOUT_MS,
+                timeoutMessage: 'Timed out waiting for workspace-tool CLI stop cleanup to finish.',
+                onSnapshot: (current) => {
+                    latestObservedOutputText = current.buffer ?? '';
+                },
+            });
+            const outputText = snapshot.buffer ?? '';
+            this.recordRunEventsFromOutput(params.runId, params.toolRow.toolId, outputText);
+            const completion = deriveOneShotRunCompletion(snapshot);
+            if (completion.status === 'completed') {
+                this.insertCleanupRunEvent(params.runId, params.toolRow.toolId, 'cleanup_completed', {
+                    summary: 'Cleanup completed',
+                    actionId: params.action.actionId,
+                    exitCode: completion.exitCode,
+                    signal: completion.signal,
+                });
+            }
+            else {
+                this.insertCleanupRunEvent(params.runId, params.toolRow.toolId, 'cleanup_failed', {
+                    summary: 'Cleanup failed',
+                    actionId: params.action.actionId,
+                    exitCode: completion.exitCode,
+                    signal: completion.signal,
+                });
+            }
+            return {
+                outputText,
+                completion,
+            };
+        }
+        catch (error) {
+            let outputText = latestObservedOutputText;
+            if (terminalId) {
+                outputText = outputText || await this.captureTerminalOutputBestEffort(params.agentNodeId, terminalId);
+                this.recordRunEventsFromOutput(params.runId, params.toolRow.toolId, outputText);
+            }
+            log.warn('[workspace-tools] CLI stop cleanup failed', {
+                toolId: params.toolRow.toolId,
+                actionId: params.action.actionId,
+                error: String(error?.message ?? error),
+            });
+            const timedOut = error instanceof WorkspaceToolServiceError && error.statusCode === 504;
+            this.insertCleanupRunEvent(params.runId, params.toolRow.toolId, timedOut ? 'cleanup_timed_out' : 'cleanup_failed', {
+                summary: timedOut ? 'Cleanup timed out' : 'Cleanup failed',
+                actionId: params.action.actionId,
+                timeoutMs: timedOut ? CLI_STOP_CLEANUP_TIMEOUT_MS : undefined,
+                error: String(error?.message ?? error),
+            });
+            return {
+                outputText,
+                completion: {
+                    status: 'failed',
+                    exitCode: null,
+                    signal: null,
+                },
+            };
+        }
+        finally {
+            if (terminalId) {
+                await closeTerminalBestEffort(this.deps.terminalBroker, params.agentNodeId, terminalId);
+            }
+        }
+    }
+    insertCleanupRunEvent(runId, toolId, type, payload) {
+        this.runLedger.insertCleanupRunEvent(runId, toolId, type, payload);
+    }
+    insertMissingTerminalRunEvent(runId, toolId, payload) {
+        this.runLedger.insertMissingTerminalRunEvent(runId, toolId, payload);
+    }
+    insertPersistentRuntimeActionTimeoutRunEvent(runId, toolId, payload) {
+        this.runLedger.insertPersistentRuntimeActionTimeoutRunEvent(runId, toolId, payload);
+    }
+    findPlatformStatusAction(manifest) {
+        const statusActions = manifest.actions.filter((candidate) => {
+            if (candidate.kind !== 'status')
+                return false;
+            if (resolveWorkspaceToolActionMode(candidate) !== 'platform_exec')
+                return false;
+            if (actionRequiresCommand(candidate) && !candidate.command?.trim() && !manifest.cli?.entry?.trim())
+                return false;
+            return true;
+        });
+        return statusActions.find((candidate) => candidate.actionId === 'status') ?? statusActions[0] ?? null;
+    }
+    async refreshLatestStateFromStatusActionSilently(params) {
+        const statusAction = this.findPlatformStatusAction(params.manifest);
+        if (!statusAction)
+            return null;
+        const cwd = resolveToolCwd(params.workspaceRoot, params.manifest, statusAction, params.sourceBundleRoot, params.publishedBundleRoot);
+        const command = rewriteWorkspaceToolCommandPaths(params.workspaceRoot, buildWorkspaceToolStartupCommand({
+            manifest: params.manifest,
+            action: statusAction,
+            actionParams: normalizeActionParams(statusAction, {}),
+            workspaceRoot: params.workspaceRoot,
+            publishedBundleRoot: params.publishedBundleRoot,
+        }), params.sourceBundleRoot, params.publishedBundleRoot);
+        let terminalId = null;
+        try {
+            const terminal = await this.deps.terminalBroker.createTerminal(params.agentNodeId, {
+                workspaceRoot: params.workspaceRoot,
+                cwd,
+                name: `${params.toolRow.name}: ${statusAction.label} (silent refresh)`,
+                startupCommand: command,
+                closeOnStartupCommand: true,
+            });
+            terminalId = terminal.terminalId;
+            const snapshot = await waitForTerminalExit(this.deps.terminalBroker, params.agentNodeId, terminalId);
+            const completion = deriveOneShotRunCompletion(snapshot);
+            if (completion.status !== 'completed')
+                return null;
+            const parsedState = parseStateRowFromOutput(snapshot.buffer ?? '', true);
+            if (!parsedState)
+                return null;
+            validateStateRowForManifest(params.manifest, parsedState);
+            this.latestStateProjector.updateLatestState({
+                toolId: params.toolRow.toolId,
+                userId: params.toolRow.userId,
+                agentId: params.toolRow.agentId,
+                maintenanceConversationId: params.toolRow.maintenanceConversationId,
+                state: parsedState,
+            });
+            return parsedState;
+        }
+        catch (error) {
+            log.warn('[workspace-tools] silent status refresh failed', {
+                toolId: params.toolRow.toolId,
+                actionId: statusAction.actionId,
+                error: String(error?.message ?? error),
+            });
+            return null;
+        }
+        finally {
+            if (terminalId) {
+                await closeTerminalBestEffort(this.deps.terminalBroker, params.agentNodeId, terminalId);
+            }
+        }
+    }
+    async refreshLatestStateAfterPersistentStart(params) {
+        const maxAttempts = 5;
+        for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+            const parsedState = await this.refreshLatestStateFromStatusActionSilently(params);
+            const businessStatus = extractWorkspaceToolBusinessStatus(parsedState);
+            if (!parsedState) {
+                return false;
+            }
+            if (businessStatus !== 'stopped') {
+                this.suppressPersistentStartupStateReplay(params.runId);
+                return true;
+            }
+            const latestRow = this.deps.db.prepare(`SELECT active_terminal_id as activeTerminalId
+           FROM workspace_tools
+          WHERE tool_id = ?
+            AND deleted_at IS NULL
+          LIMIT 1`).get(params.toolRow.toolId);
+            if (!latestRow?.activeTerminalId) {
+                this.suppressPersistentStartupStateReplay(params.runId);
+                return true;
+            }
+            if (attempt >= maxAttempts - 1) {
+                return true;
+            }
+            if (this.deps.silentStatusRefreshRetryDelayMs > 0) {
+                await sleep(this.deps.silentStatusRefreshRetryDelayMs);
+            }
+        }
+        return false;
+    }
+    async startPersistentAction(params) {
+        const cwd = resolveToolCwd(params.workspaceRoot, params.manifest, params.action, params.sourceBundleRoot, params.publishedBundleRoot);
+        const command = rewriteWorkspaceToolCommandPaths(params.workspaceRoot, buildWorkspaceToolStartupCommand({
+            manifest: params.manifest,
+            action: params.action,
+            actionParams: params.actionParams,
+            workspaceRoot: params.workspaceRoot,
+            publishedBundleRoot: params.publishedBundleRoot,
+        }), params.sourceBundleRoot, params.publishedBundleRoot);
+        let terminalId = null;
+        let runInserted = false;
+        try {
+            const terminal = await this.deps.terminalBroker.createTerminal(params.agentNodeId, {
+                workspaceRoot: params.workspaceRoot,
+                cwd,
+                name: `${params.toolRow.name}: ${params.action.label}`,
+                startupCommand: command,
+                bufferOutputUntilTaken: true,
+            });
+            terminalId = terminal.terminalId;
+            this.insertRun({ ...params.run, terminalId });
+            runInserted = true;
+            this.deps.db.prepare(`UPDATE workspace_tools
+            SET active_terminal_id = ?,
+                updated_at = ?
+          WHERE tool_id = ?`).run(terminalId, Date.now(), params.toolRow.toolId);
+            this.latestStateProjector.syncStatus(params.toolRow.toolId, 'running');
+            this.terminalLifecycle.setContext(terminalId, {
+                toolId: params.toolRow.toolId,
+                runId: params.run.runId,
+                userId: params.toolRow.userId,
+                agentId: params.toolRow.agentId,
+                agentNodeId: params.agentNodeId,
+                maintenanceConversationId: params.toolRow.maintenanceConversationId,
+                manifest: params.manifest,
+                lineBuffer: '',
+                idleTimeoutSeconds: params.action.idleTimeoutSeconds,
+            });
+            this.schedulePersistentRunTimeout({
+                toolId: params.toolRow.toolId,
+                agentNodeId: params.agentNodeId,
+                terminalId,
+                maxRunSeconds: params.action.maxRunSeconds,
+            });
+            const startedCtx = this.terminalLifecycle.getContext(terminalId);
+            if (startedCtx) {
+                this.terminalLifecycle.resetPersistentIdleTimeout(terminalId);
+            }
+            const realtimeCtx = this.terminalLifecycle.getContext(terminalId);
+            if (realtimeCtx) {
+                if (params.manifest.cli?.entry && !params.action.command) {
+                    const shouldScheduleStartupFallback = await this.capturePersistentStartupOutput({
+                        terminalId,
+                        agentNodeId: params.agentNodeId,
+                        runId: params.run.runId,
+                    });
+                    if (shouldScheduleStartupFallback) {
+                        this.schedulePersistentStartupSnapshotFallback({
+                            terminalId,
+                            agentNodeId: params.agentNodeId,
+                            runId: params.run.runId,
+                        });
+                    }
+                }
+                else {
+                    const earlyOutput = this.deps.terminalBroker.takeBufferedOutput(terminalId);
+                    if (earlyOutput) {
+                        this.terminalLifecycle.resetPersistentIdleTimeout(terminalId);
+                        this.processRealtimeStateChunk(realtimeCtx, terminalId, earlyOutput);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            if (runInserted) {
+                this.completeRun(params.run.runId, {
+                    status: 'failed',
+                    endedAt: Date.now(),
+                });
+            }
+            if (terminalId) {
+                try {
+                    this.deps.db.prepare(`UPDATE workspace_tools
+                SET active_terminal_id = NULL,
+                    updated_at = ?
+              WHERE tool_id = ?
+                AND active_terminal_id = ?`).run(Date.now(), params.toolRow.toolId, terminalId);
+                }
+                catch {
+                    // Preserve the startup failure while still closing the leaked terminal.
+                }
+                this.terminalLifecycle.deleteContext(terminalId);
+                this.clearPendingRealtimeStateForTool(params.toolRow.toolId);
+                this.clearPersistentRunTimeout(terminalId);
+                this.clearPersistentIdleTimeout(terminalId);
+                this.clearPersistentStartupCaptureTimeout(terminalId);
+                this.clearPersistentStartupOutputObserved(params.run.runId);
+                this.clearIgnoredRealtimeStartupLines(terminalId);
+                await closeTerminalBestEffort(this.deps.terminalBroker, params.agentNodeId, terminalId);
+            }
+            throw workspaceToolErrorWithRunId(error, params.run.runId);
+        }
+        if (!terminalId) {
+            throw new WorkspaceToolServiceError('Persistent tool terminal was not created.', 500);
+        }
+        const didRefreshLatestState = await this.refreshLatestStateAfterPersistentStart({
+            runId: params.run.runId,
+            toolRow: params.toolRow,
+            manifest: params.manifest,
+            agentNodeId: params.agentNodeId,
+            workspaceRoot: params.workspaceRoot,
+            sourceBundleRoot: params.sourceBundleRoot,
+            publishedBundleRoot: params.publishedBundleRoot,
+        });
+        const tool = this.deps.getToolInfoForActionViewer(params.viewerUserId, params.toolRow.toolId, params.allowAdminBypass);
+        const run = this.requireRun(params.run.runId, params.action.kind, params.viewerUserId, params.toolRow.userId);
+        await this.deps.recordPlatformExecNotification({
+            toolRow: params.toolRow,
+            action: params.action,
+            run,
+            viewerUserId: params.viewerUserId,
+            actionParams: params.actionParams,
+            outputText: null,
+        });
+        return {
+            mode: 'platform_exec',
+            tool,
+            run,
+            latestState: tool.latestState ?? null,
+            outputText: null,
+        };
+    }
+    async capturePersistentStartupOutput(params) {
+        const drainBufferedOutput = () => {
+            const ctx = this.getActivePersistentStartupContext(params.terminalId, params.runId);
+            if (!ctx)
+                return null;
+            const output = this.deps.terminalBroker.takeBufferedOutput(params.terminalId);
+            if (!output)
+                return ctx;
+            this.terminalLifecycle.resetPersistentIdleTimeout(params.terminalId);
+            this.processRealtimeStateChunk(ctx, params.terminalId, output);
+            return ctx;
+        };
+        let ctx = drainBufferedOutput();
+        if (!ctx)
+            return false;
+        const deadline = Date.now() + PERSISTENT_STARTUP_OUTPUT_SETTLE_MS;
+        while (!this.hasObservedPersistentStartupOutput(ctx, params.runId) && Date.now() < deadline) {
+            await sleep(PERSISTENT_STARTUP_OUTPUT_POLL_MS);
+            ctx = drainBufferedOutput();
+            if (!ctx)
+                return false;
+        }
+        if (this.hasObservedPersistentStartupOutput(ctx, params.runId)) {
+            return false;
+        }
+        const snapshot = await this.captureTerminalSnapshotBestEffort(params.agentNodeId, params.terminalId);
+        if (!snapshot)
+            return true;
+        if (snapshot.exited)
+            return false;
+        if (!snapshot.buffer)
+            return true;
+        ctx = this.getActivePersistentStartupContext(params.terminalId, params.runId);
+        if (!ctx || this.hasObservedPersistentStartupOutput(ctx, params.runId)) {
+            return false;
+        }
+        this.terminalLifecycle.resetPersistentIdleTimeout(params.terminalId);
+        this.processRealtimeStateSnapshotFallback(ctx, params.terminalId, snapshot.buffer, {
+            allowStateUpdate: !this.isPersistentStartupStateReplaySuppressed(params.runId),
+        });
+        this.flushPendingRealtimeStateForTool(ctx.toolId);
+        return !this.hasObservedPersistentStartupOutput(ctx, params.runId);
+    }
+    schedulePersistentStartupSnapshotFallback(params) {
+        if (this.hasObservedPersistentStartupOutputForTerminal(params.terminalId, params.runId)) {
+            return;
+        }
+        this.terminalLifecycle.schedulePersistentStartupSnapshotFallback({
+            terminalId: params.terminalId,
+            delayMs: PERSISTENT_STARTUP_SNAPSHOT_FALLBACK_MS,
+            onFallback: () => this.capturePersistentStartupSnapshotFallback(params),
+        });
+    }
+    async capturePersistentStartupSnapshotFallback(params) {
+        const ctx = this.getActivePersistentStartupContext(params.terminalId, params.runId);
+        if (!ctx || this.hasObservedPersistentStartupOutput(ctx, params.runId)) {
+            return;
+        }
+        const snapshot = await this.captureTerminalSnapshotBestEffort(params.agentNodeId, params.terminalId);
+        if (!snapshot?.buffer || snapshot.exited)
+            return;
+        const latestCtx = this.getActivePersistentStartupContext(params.terminalId, params.runId);
+        if (!latestCtx || this.hasObservedPersistentStartupOutput(latestCtx, params.runId)) {
+            return;
+        }
+        this.terminalLifecycle.resetPersistentIdleTimeout(params.terminalId);
+        this.processRealtimeStateSnapshotFallback(latestCtx, params.terminalId, snapshot.buffer, {
+            allowStateUpdate: !this.isPersistentStartupStateReplaySuppressed(params.runId),
+        });
+        this.flushPendingRealtimeStateForTool(latestCtx.toolId);
+    }
+    processRealtimeStateSnapshotFallback(ctx, terminalId, outputText, options = {}) {
+        for (const line of outputText.split('\n')) {
+            const candidate = extractWorkspaceToolJsonLineCandidate(line);
+            if (!candidate)
+                continue;
+            const accepted = this.processRealtimeStateLine(ctx, terminalId, candidate, {
+                consumeIgnoredRealtimeLine: false,
+                allowStateUpdate: options.allowStateUpdate,
+            });
+            if (accepted) {
+                this.recordIgnoredRealtimeStartupLine(terminalId, candidate);
+            }
+        }
+    }
+    prepareRealtimeChunkAfterSnapshotFallback(ctx, terminalId, chunk) {
+        if (!ctx.lineBuffer || !chunk)
+            return chunk;
+        const ignored = this.ignoredRealtimeStartupLines.get(terminalId);
+        if (!ignored?.size)
+            return chunk;
+        const buffered = ctx.lineBuffer;
+        const bufferedTrimmed = buffered.trimStart();
+        const bufferedJsonFragment = extractWorkspaceToolJsonFragment(buffered);
+        const chunkJsonFragment = extractWorkspaceToolJsonFragment(chunk);
+        for (const line of ignored.keys()) {
+            const prefix = line.startsWith(buffered)
+                ? buffered
+                : bufferedTrimmed.length > 0 && line.startsWith(bufferedTrimmed)
+                    ? bufferedTrimmed
+                    : bufferedJsonFragment && line.startsWith(bufferedJsonFragment)
+                        ? bufferedJsonFragment
+                        : null;
+            if (prefix == null)
+                continue;
+            const tail = line.slice(prefix.length);
+            if (tail && chunk.startsWith(tail)) {
+                this.consumeIgnoredRealtimeStartupLine(terminalId, line);
+                ctx.lineBuffer = '';
+                this.scheduleIgnoredRealtimeStartupLinesExpiryIfIdle(ctx, terminalId);
+                return chunk.slice(tail.length);
+            }
+            if (chunkJsonFragment) {
+                this.consumeIgnoredRealtimeStartupLine(terminalId, line);
+                ctx.lineBuffer = '';
+                this.scheduleIgnoredRealtimeStartupLinesExpiryIfIdle(ctx, terminalId);
+                return chunk;
+            }
+        }
+        return chunk;
+    }
+    recordIgnoredRealtimeStartupLine(terminalId, line) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            return;
+        const existing = this.ignoredRealtimeStartupLines.get(terminalId) ?? new Map();
+        existing.set(trimmed, (existing.get(trimmed) ?? 0) + 1);
+        this.ignoredRealtimeStartupLines.set(terminalId, existing);
+        this.scheduleIgnoredRealtimeStartupLinesExpiryIfIdle(this.terminalLifecycle.getContext(terminalId) ?? null, terminalId);
+    }
+    consumeIgnoredRealtimeStartupLine(terminalId, line) {
+        const lines = this.ignoredRealtimeStartupLines.get(terminalId);
+        const count = lines?.get(line) ?? 0;
+        if (!lines || count <= 0)
+            return false;
+        if (count === 1) {
+            lines.delete(line);
+        }
+        else {
+            lines.set(line, count - 1);
+        }
+        if (lines.size === 0) {
+            this.ignoredRealtimeStartupLines.delete(terminalId);
+            this.clearIgnoredRealtimeStartupLinesExpiry(terminalId);
+        }
+        return true;
+    }
+    hasObservedPersistentStartupOutputForTerminal(terminalId, runId) {
+        const ctx = this.terminalLifecycle.getContext(terminalId);
+        return ctx ? this.hasObservedPersistentStartupOutput(ctx, runId) : true;
+    }
+    suppressPersistentStartupStateReplay(runId) {
+        this.startupStateReplaySuppressedRunIds.add(runId);
+    }
+    isPersistentStartupStateReplaySuppressed(runId) {
+        return this.startupStateReplaySuppressedRunIds.has(runId);
+    }
+    scheduleIgnoredRealtimeStartupLinesExpiryIfIdle(ctx, terminalId) {
+        if (ctx?.lineBuffer)
+            return;
+        this.clearIgnoredRealtimeStartupLinesExpiry(terminalId);
+        const timer = setTimeout(() => {
+            this.ignoredRealtimeStartupLineTimeouts.delete(terminalId);
+            this.ignoredRealtimeStartupLines.delete(terminalId);
+        }, PERSISTENT_STARTUP_DUPLICATE_IGNORE_IDLE_MS);
+        timer.unref?.();
+        this.ignoredRealtimeStartupLineTimeouts.set(terminalId, timer);
+    }
+    clearIgnoredRealtimeStartupLines(terminalId) {
+        this.clearIgnoredRealtimeStartupLinesExpiry(terminalId);
+        this.ignoredRealtimeStartupLines.delete(terminalId);
+    }
+    clearIgnoredRealtimeStartupLinesExpiry(terminalId) {
+        const handle = this.ignoredRealtimeStartupLineTimeouts.get(terminalId);
+        if (handle) {
+            clearTimeout(handle);
+            this.ignoredRealtimeStartupLineTimeouts.delete(terminalId);
+        }
+    }
+    getActivePersistentStartupContext(terminalId, runId) {
+        if (this.terminalLifecycle.isTerminating(terminalId))
+            return null;
+        const ctx = this.terminalLifecycle.getContext(terminalId);
+        return ctx?.runId === runId ? ctx : null;
+    }
+    async runPersistentRuntimeAction(params) {
+        const currentActiveTerminalId = this.loadActiveTerminalId(params.toolRow.toolId);
+        if (params.action.kind === 'stop') {
+            if (!currentActiveTerminalId) {
+                throw new WorkspaceToolServiceError('Tool runtime is not running.', 409);
+            }
+            return this.stopPersistentRuntimeAction({ ...params, terminalId: currentActiveTerminalId });
+        }
+        let terminalId = currentActiveTerminalId;
+        let startedRuntime = false;
+        if (!terminalId) {
+            terminalId = await this.deps.enqueueAgentPersistentStart(params.toolRow.agentId, async () => {
+                const latestActiveTerminalId = this.loadActiveTerminalId(params.toolRow.toolId);
+                if (latestActiveTerminalId)
+                    return latestActiveTerminalId;
+                this.terminalLifecycle.assertAgentPersistentRunQuota(params.toolRow.agentId);
+                return this.startPersistentRuntimeTerminal(params);
+            });
+            startedRuntime = true;
+        }
+        const ctx = this.terminalLifecycle.getContext(terminalId);
+        if (ctx) {
+            ctx.runId = null;
+            ctx.idleTimeoutSeconds = undefined;
+            ctx.lineBuffer = '';
+            this.clearPersistentIdleTimeout(terminalId);
+        }
+        const logDir = this.jsonlClient.normalizeLogDir(params.toolRow.toolId, params.manifest);
+        const logPath = this.jsonlClient.buildRunLogPath(logDir, params.run.runId);
+        this.insertRun({ ...params.run, terminalId });
+        try {
+            const request = this.jsonlClient.buildStdinRequest({
+                runId: params.run.runId,
+                actionId: params.action.actionId,
+                actionParams: params.actionParams,
+                toolId: params.toolRow.toolId,
+                revision: params.toolRow.revision,
+            });
+            try {
+                await this.deps.terminalBroker.sendInput(params.agentNodeId, terminalId, this.jsonlClient.stdinRequestToLine(request));
+            }
+            catch (error) {
+                if (isMissingTerminalError(error)) {
+                    this.finishActivePersistentRun(params.toolRow.toolId, terminalId, { status: 'failed', signal: 'missing_terminal' });
+                }
+                throw error;
+            }
+            const completion = await this.jsonlClient.syncRunLog({
+                runId: params.run.runId,
+                toolId: params.toolRow.toolId,
+                terminalId,
+                agentNodeId: params.agentNodeId,
+                workspaceRoot: params.workspaceRoot,
+                logPath,
+                actionId: params.action.actionId,
+                manifest: params.manifest,
+                timeoutMs: this.jsonlClient.actionTimeoutMs(params.action),
+                onStateUpdate: (state) => this.latestStateProjector.updateLatestState({
+                    toolId: params.toolRow.toolId,
+                    userId: params.toolRow.userId,
+                    agentId: params.toolRow.agentId,
+                    maintenanceConversationId: params.toolRow.maintenanceConversationId,
+                    state,
+                }),
+                onRunEvent: (envelope) => this.runLedger.insertRunEventFromEnvelope(params.run.runId, params.toolRow.toolId, envelope),
+                onTimeoutEvent: (timeoutPayload) => this.runLedger.insertPersistentRuntimeActionTimeoutRunEvent(params.run.runId, params.toolRow.toolId, timeoutPayload),
+            });
+            const endedAt = Date.now();
+            this.completeRun(params.run.runId, {
+                status: completion.status,
+                endedAt,
+                exitCode: completion.exitCode,
+                signal: completion.signal,
+            });
+            this.resetPersistentRuntimeIdleTimeout(terminalId, params);
+        }
+        catch (error) {
+            const timedOut = error instanceof WorkspaceToolServiceError && error.statusCode === 504;
+            const missingTerminal = isMissingTerminalError(error);
+            const completion = deriveTerminalRunCompletion(timedOut ? { kind: 'timeout' }
+                : missingTerminal ? { kind: 'missing_terminal' }
+                    : { kind: 'error_event' });
+            if (missingTerminal) {
+                this.insertMissingTerminalRunEvent(params.run.runId, params.toolRow.toolId, {
+                    actionId: params.action.actionId,
+                    error: String(error?.message ?? error),
+                });
+            }
+            this.completeRun(params.run.runId, {
+                status: completion.status,
+                endedAt: Date.now(),
+                exitCode: completion.exitCode,
+                signal: completion.signal,
+            });
+            if (completion.status === 'timed_out' || startedRuntime || missingTerminal) {
+                await closeTerminalBestEffort(this.deps.terminalBroker, params.agentNodeId, terminalId);
+                this.finishActivePersistentRun(params.toolRow.toolId, terminalId, completion);
+            }
+            else {
+                this.resetPersistentRuntimeIdleTimeout(terminalId, params);
+            }
+            const failedRun = this.requireRun(params.run.runId, params.action.kind, params.viewerUserId, params.toolRow.userId);
+            await this.deps.recordPlatformExecNotification({
+                toolRow: params.toolRow,
+                action: params.action,
+                run: failedRun,
+                viewerUserId: params.viewerUserId,
+                actionParams: params.actionParams,
+                outputText: null,
+            });
+            throw workspaceToolErrorWithRunId(error, params.run.runId);
+        }
+        const tool = this.getToolInfoForActionViewerAfterPendingRealtimeFlush({
+            viewerUserId: params.viewerUserId,
+            toolRow: params.toolRow,
+            allowAdminBypass: params.allowAdminBypass,
+        });
+        const run = this.requireRun(params.run.runId, params.action.kind, params.viewerUserId, params.toolRow.userId);
+        await this.deps.recordPlatformExecNotification({
+            toolRow: params.toolRow,
+            action: params.action,
+            run,
+            viewerUserId: params.viewerUserId,
+            actionParams: params.actionParams,
+            outputText: null,
+        });
+        return {
+            mode: 'platform_exec',
+            tool,
+            run,
+            latestState: tool.latestState ?? null,
+            outputText: null,
+        };
+    }
+    async stopPersistentRuntimeAction(params) {
+        this.insertRun({ ...params.run, terminalId: params.terminalId });
+        this.insertCleanupRunEvent(params.run.runId, params.toolRow.toolId, 'cleanup_started', {
+            summary: 'Runtime stop requested',
+            actionId: params.action.actionId,
+        });
+        let finalStatus = 'completed';
+        let finalExitCode = 0;
+        let finalSignal = null;
+        let actionError = null;
+        const shouldRequestRuntimeStop = typeof params.action.maxRunSeconds === 'number';
+        if (shouldRequestRuntimeStop) {
+            const logDir = this.jsonlClient.normalizeLogDir(params.toolRow.toolId, params.manifest);
+            const logPath = this.jsonlClient.buildRunLogPath(logDir, params.run.runId);
+            try {
+                const request = this.jsonlClient.buildStdinRequest({
+                    runId: params.run.runId,
+                    actionId: params.action.actionId,
+                    actionParams: params.actionParams,
+                    toolId: params.toolRow.toolId,
+                    revision: params.toolRow.revision,
+                });
+                await this.deps.terminalBroker.sendInput(params.agentNodeId, params.terminalId, this.jsonlClient.stdinRequestToLine(request));
+                const completion = await this.jsonlClient.syncRunLog({
+                    runId: params.run.runId,
+                    toolId: params.toolRow.toolId,
+                    terminalId: params.terminalId,
+                    agentNodeId: params.agentNodeId,
+                    workspaceRoot: params.workspaceRoot,
+                    logPath,
+                    actionId: params.action.actionId,
+                    manifest: params.manifest,
+                    timeoutMs: this.jsonlClient.actionTimeoutMs(params.action),
+                    onStateUpdate: (state) => this.latestStateProjector.updateLatestState({
+                        toolId: params.toolRow.toolId,
+                        userId: params.toolRow.userId,
+                        agentId: params.toolRow.agentId,
+                        maintenanceConversationId: params.toolRow.maintenanceConversationId,
+                        state,
+                    }),
+                    onRunEvent: (envelope) => this.runLedger.insertRunEventFromEnvelope(params.run.runId, params.toolRow.toolId, envelope),
+                    onTimeoutEvent: (timeoutPayload) => this.runLedger.insertPersistentRuntimeActionTimeoutRunEvent(params.run.runId, params.toolRow.toolId, timeoutPayload),
+                });
+                finalStatus = completion.status;
+                finalExitCode = completion.exitCode;
+                finalSignal = completion.signal;
+                if (completion.status === 'failed') {
+                    actionError = new WorkspaceToolServiceError('Persistent runtime stop action reported an error event.', 500, {
+                        runId: params.run.runId,
+                    });
+                }
+            }
+            catch (error) {
+                const timedOut = error instanceof WorkspaceToolServiceError && error.statusCode === 504;
+                const missingTerminal = isMissingTerminalError(error);
+                const completion = deriveTerminalRunCompletion(timedOut ? { kind: 'timeout' }
+                    : missingTerminal ? { kind: 'missing_terminal' }
+                        : { kind: 'error_event' });
+                finalStatus = completion.status;
+                finalExitCode = completion.exitCode;
+                finalSignal = completion.signal;
+                if (missingTerminal) {
+                    this.insertMissingTerminalRunEvent(params.run.runId, params.toolRow.toolId, {
+                        actionId: params.action.actionId,
+                        error: String(error?.message ?? error),
+                    });
+                }
+                actionError = error;
+            }
+        }
+        try {
+            await this.deps.terminalBroker.closeTerminal(params.agentNodeId, params.terminalId);
+            this.finishActivePersistentRun(params.toolRow.toolId, params.terminalId, { status: 'cancelled' });
+        }
+        catch (error) {
+            if (isMissingTerminalError(error)) {
+                this.finishActivePersistentRun(params.toolRow.toolId, params.terminalId, { status: 'cancelled', signal: 'missing_terminal' });
+                this.insertCleanupRunEvent(params.run.runId, params.toolRow.toolId, 'cleanup_completed', {
+                    summary: 'Runtime terminal was already gone',
+                    actionId: params.action.actionId,
+                    exitCode: null,
+                    signal: 'missing_terminal',
+                });
+                const finalEndedAt = Date.now();
+                this.completeRun(params.run.runId, {
+                    status: finalStatus,
+                    endedAt: finalEndedAt,
+                    exitCode: finalExitCode,
+                    signal: finalSignal ?? 'missing_terminal',
+                });
+                const tool = this.getToolInfoForActionViewerAfterPendingRealtimeFlush({
+                    viewerUserId: params.viewerUserId,
+                    toolRow: params.toolRow,
+                    allowAdminBypass: params.allowAdminBypass,
+                });
+                const run = this.requireRun(params.run.runId, params.action.kind, params.viewerUserId, params.toolRow.userId);
+                await this.deps.recordPlatformExecNotification({
+                    toolRow: params.toolRow,
+                    action: params.action,
+                    run,
+                    viewerUserId: params.viewerUserId,
+                    actionParams: params.actionParams,
+                    outputText: null,
+                });
+                if (actionError && finalStatus !== 'completed') {
+                    throw workspaceToolErrorWithRunId(actionError, params.run.runId);
+                }
+                return {
+                    mode: 'platform_exec',
+                    tool,
+                    run,
+                    latestState: tool.latestState ?? null,
+                    outputText: null,
+                };
+            }
+            this.insertCleanupRunEvent(params.run.runId, params.toolRow.toolId, 'cleanup_failed', {
+                summary: 'Runtime terminal close failed',
+                actionId: params.action.actionId,
+                error: String(error?.message ?? error),
+            });
+            const preserveActionFailure = Boolean(actionError && finalStatus !== 'completed');
+            this.completeRun(params.run.runId, {
+                status: preserveActionFailure ? finalStatus : 'failed',
+                endedAt: Date.now(),
+                exitCode: preserveActionFailure ? finalExitCode : null,
+                signal: preserveActionFailure ? finalSignal : null,
+            });
+            throw workspaceToolErrorWithRunId(preserveActionFailure ? actionError : error, params.run.runId);
+        }
+        this.insertCleanupRunEvent(params.run.runId, params.toolRow.toolId, 'cleanup_completed', {
+            summary: 'Runtime terminal closed',
+            actionId: params.action.actionId,
+            exitCode: finalExitCode,
+            signal: finalSignal,
+        });
+        this.completeRun(params.run.runId, {
+            status: finalStatus,
+            endedAt: Date.now(),
+            exitCode: finalExitCode,
+            signal: finalSignal,
+        });
+        const tool = this.getToolInfoForActionViewerAfterPendingRealtimeFlush({
+            viewerUserId: params.viewerUserId,
+            toolRow: params.toolRow,
+            allowAdminBypass: params.allowAdminBypass,
+        });
+        const run = this.requireRun(params.run.runId, params.action.kind, params.viewerUserId, params.toolRow.userId);
+        await this.deps.recordPlatformExecNotification({
+            toolRow: params.toolRow,
+            action: params.action,
+            run,
+            viewerUserId: params.viewerUserId,
+            actionParams: params.actionParams,
+            outputText: null,
+        });
+        if (actionError && finalStatus !== 'completed') {
+            throw workspaceToolErrorWithRunId(actionError, params.run.runId);
+        }
+        return {
+            mode: 'platform_exec',
+            tool,
+            run,
+            latestState: tool.latestState ?? null,
+            outputText: null,
+        };
+    }
+    async startPersistentRuntimeTerminal(params) {
+        const logDir = this.jsonlClient.normalizeLogDir(params.toolRow.toolId, params.manifest);
+        const command = rewriteWorkspaceToolCommandPaths(params.workspaceRoot, buildWorkspaceToolRuntimeCommand({
+            manifest: params.manifest,
+            workspaceRoot: params.workspaceRoot,
+            publishedBundleRoot: params.publishedBundleRoot,
+            toolId: params.toolRow.toolId,
+            revision: params.toolRow.revision,
+            logDir,
+        }), params.sourceBundleRoot, params.publishedBundleRoot);
+        if (!command.trim()) {
+            throw new WorkspaceToolServiceError('Persistent tool runtime requires runtime.entry or cli.entry.', 400);
+        }
+        let terminalId = null;
+        try {
+            const terminal = await this.deps.terminalBroker.createTerminal(params.agentNodeId, {
+                workspaceRoot: params.workspaceRoot,
+                cwd: resolveToolPath(params.workspaceRoot, params.publishedBundleRoot),
+                name: `${params.toolRow.name}: runtime`,
+                startupCommand: command,
+            });
+            terminalId = terminal.terminalId;
+            this.deps.db.prepare(`UPDATE workspace_tools
+            SET active_terminal_id = ?,
+                updated_at = ?
+          WHERE tool_id = ?`).run(terminal.terminalId, Date.now(), params.toolRow.toolId);
+            this.latestStateProjector.syncStatus(params.toolRow.toolId, 'running');
+            this.terminalLifecycle.setContext(terminal.terminalId, {
+                toolId: params.toolRow.toolId,
+                runId: null,
+                userId: params.toolRow.userId,
+                agentId: params.toolRow.agentId,
+                agentNodeId: params.agentNodeId,
+                maintenanceConversationId: params.toolRow.maintenanceConversationId,
+                manifest: params.manifest,
+                lineBuffer: '',
+            });
+            return terminal.terminalId;
+        }
+        catch (error) {
+            if (terminalId) {
+                try {
+                    this.deps.db.prepare(`UPDATE workspace_tools
+                SET active_terminal_id = NULL,
+                    updated_at = ?
+              WHERE tool_id = ?
+                AND active_terminal_id = ?`).run(Date.now(), params.toolRow.toolId, terminalId);
+                }
+                catch {
+                    // Preserve the startup failure while still closing the leaked terminal.
+                }
+                this.terminalLifecycle.deleteContext(terminalId);
+                this.clearPersistentRunTimeout(terminalId);
+                this.clearPersistentIdleTimeout(terminalId);
+                this.clearPersistentStartupCaptureTimeout(terminalId);
+                this.clearIgnoredRealtimeStartupLines(terminalId);
+                await closeTerminalBestEffort(this.deps.terminalBroker, params.agentNodeId, terminalId);
+            }
+            throw error;
+        }
+    }
+    resetPersistentRuntimeIdleTimeout(terminalId, params) {
+        const ctx = this.terminalLifecycle.getContext(terminalId);
+        if (!ctx)
+            return;
+        ctx.runId = null;
+        ctx.idleTimeoutSeconds = params.manifest.runtime?.idleTtlSeconds ?? defaultPersistentRuntimeIdleTtlSeconds();
+        this.terminalLifecycle.resetPersistentIdleTimeout(terminalId);
+    }
+    loadActiveTerminalId(toolId) {
+        const row = this.deps.db.prepare(`SELECT active_terminal_id as activeTerminalId
+         FROM workspace_tools
+        WHERE tool_id = ?
+          AND deleted_at IS NULL`).get(toolId);
+        return row?.activeTerminalId ?? null;
+    }
+    hasObservedPersistentStartupOutput(ctx, runId) {
+        if (this.observedPersistentStartupOutput.has(runId))
+            return true;
+        if (this.latestStateProjector.hasPendingRealtimeStateUpdate(ctx.toolId))
+            return true;
+        const row = this.deps.db.prepare(`SELECT 1
+         FROM workspace_tool_run_events
+        WHERE run_id = ?
+        LIMIT 1`).get(runId);
+        return Boolean(row);
+    }
+    async runOneShotAction(params) {
+        const cwd = resolveToolCwd(params.workspaceRoot, params.manifest, params.action, params.sourceBundleRoot, params.publishedBundleRoot);
+        const command = rewriteWorkspaceToolCommandPaths(params.workspaceRoot, buildWorkspaceToolStartupCommand({
+            manifest: params.manifest,
+            action: params.action,
+            actionParams: params.actionParams,
+            workspaceRoot: params.workspaceRoot,
+            publishedBundleRoot: params.publishedBundleRoot,
+        }), params.sourceBundleRoot, params.publishedBundleRoot);
+        let terminalId = null;
+        try {
+            const terminal = await this.deps.terminalBroker.createTerminal(params.agentNodeId, {
+                workspaceRoot: params.workspaceRoot,
+                cwd,
+                name: `${params.toolRow.name}: ${params.action.label}`,
+                startupCommand: command,
+                closeOnStartupCommand: true,
+            });
+            terminalId = terminal.terminalId;
+            this.insertRun({ ...params.run, terminalId });
+        }
+        catch (error) {
+            if (terminalId) {
+                await closeTerminalBestEffort(this.deps.terminalBroker, params.agentNodeId, terminalId);
+            }
+            throw error;
+        }
+        if (!terminalId) {
+            throw new WorkspaceToolServiceError('Tool action terminal was not created.', 500);
+        }
+        let outputText = '';
+        let latestObservedOutputText = '';
+        let completion = null;
+        try {
+            const snapshot = await waitForTerminalExit(this.deps.terminalBroker, params.agentNodeId, terminalId, {
+                onSnapshot: (current) => {
+                    latestObservedOutputText = current.buffer ?? '';
+                },
+            });
+            const endedAt = Date.now();
+            outputText = snapshot.buffer ?? '';
+            completion = deriveOneShotRunCompletion(snapshot);
+            this.recordRunEventsFromOutput(params.run.runId, params.toolRow.toolId, outputText);
+            if (params.action.kind === 'status') {
+                const parsedState = parseStateRowFromOutput(outputText, true);
+                if (!parsedState) {
+                    throw new WorkspaceToolServiceError('Status action output must be a single JSON object.', 400);
+                }
+                validateStateRowForManifest(params.manifest, parsedState);
+                this.latestStateProjector.updateLatestState({
+                    toolId: params.toolRow.toolId,
+                    userId: params.toolRow.userId,
+                    agentId: params.toolRow.agentId,
+                    maintenanceConversationId: params.toolRow.maintenanceConversationId,
+                    state: parsedState,
+                });
+            }
+            else {
+                const parsedState = parseStateRowFromOutput(outputText, false);
+                if (parsedState) {
+                    validateStateRowForManifest(params.manifest, parsedState);
+                    this.latestStateProjector.updateLatestState({
+                        toolId: params.toolRow.toolId,
+                        userId: params.toolRow.userId,
+                        agentId: params.toolRow.agentId,
+                        maintenanceConversationId: params.toolRow.maintenanceConversationId,
+                        state: parsedState,
+                    });
+                }
+            }
+            this.completeRun(params.run.runId, {
+                status: completion.status,
+                endedAt,
+                exitCode: completion.exitCode,
+                signal: completion.signal,
+            });
+        }
+        catch (error) {
+            if (!outputText) {
+                outputText = latestObservedOutputText || await this.captureTerminalOutputBestEffort(params.agentNodeId, terminalId);
+                this.recordRunEventsFromOutput(params.run.runId, params.toolRow.toolId, outputText);
+            }
+            const failedEndedAt = Date.now();
+            const failedExitCode = completion?.exitCode ?? null;
+            const failedSignal = completion?.signal ?? null;
+            this.completeRun(params.run.runId, {
+                status: 'failed',
+                endedAt: failedEndedAt,
+                exitCode: failedExitCode,
+                signal: failedSignal,
+            });
+            const failedRun = this.runInfoForActionViewer({
+                ...params.run,
+                terminalId,
+                status: 'failed',
+                endedAt: failedEndedAt,
+                exitCode: failedExitCode,
+                signal: failedSignal,
+            }, params.action.kind, params.viewerUserId, params.toolRow.userId);
+            await this.deps.recordPlatformExecNotification({
+                toolRow: params.toolRow,
+                action: params.action,
+                run: failedRun,
+                viewerUserId: params.viewerUserId,
+                actionParams: params.actionParams,
+                outputText,
+            });
+            throw workspaceToolErrorWithRunId(error, params.run.runId);
+        }
+        finally {
+            if (terminalId) {
+                await closeTerminalBestEffort(this.deps.terminalBroker, params.agentNodeId, terminalId);
+            }
+        }
+        if (params.action.kind !== 'status') {
+            await this.refreshLatestStateFromStatusActionSilently({
+                toolRow: params.toolRow,
+                manifest: params.manifest,
+                agentNodeId: params.agentNodeId,
+                workspaceRoot: params.workspaceRoot,
+                sourceBundleRoot: params.sourceBundleRoot,
+                publishedBundleRoot: params.publishedBundleRoot,
+            });
+        }
+        const tool = this.deps.getToolInfoForActionViewer(params.viewerUserId, params.toolRow.toolId, params.allowAdminBypass);
+        const run = this.requireRun(params.run.runId, params.action.kind, params.viewerUserId, params.toolRow.userId);
+        await this.deps.recordPlatformExecNotification({
+            toolRow: params.toolRow,
+            action: params.action,
+            run,
+            viewerUserId: params.viewerUserId,
+            actionParams: params.actionParams,
+            outputText,
+        });
+        return {
+            mode: 'platform_exec',
+            tool,
+            run,
+            latestState: tool.latestState ?? null,
+            outputText,
+        };
+    }
+    insertRun(run) {
+        this.runLedger.insertRun(run);
+    }
+    runInfoForActionViewer(row, actionKind, viewerUserId, ownerUserId) {
+        return this.runLedger.runInfoForActionViewer(row, actionKind, viewerUserId, ownerUserId);
+    }
+    recordRunEventsFromOutput(runId, toolId, outputText) {
+        this.runLedger.recordRunEventsFromOutput(runId, toolId, outputText);
+    }
+    async captureTerminalOutputBestEffort(agentNodeId, terminalId) {
+        const snapshot = await this.captureTerminalSnapshotBestEffort(agentNodeId, terminalId);
+        return snapshot?.buffer ?? '';
+    }
+    async captureTerminalSnapshotBestEffort(agentNodeId, terminalId) {
+        try {
+            const snapshot = await this.deps.terminalBroker.snapshotTerminal(agentNodeId, terminalId);
+            return {
+                buffer: snapshot.buffer ?? '',
+                exited: snapshot.terminal.exited,
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    insertRunEventFromEnvelope(runId, toolId, value) {
+        return this.runLedger.insertRunEventFromEnvelope(runId, toolId, value);
+    }
+    completeRun(runId, params) {
+        this.runLedger.completeRun(runId, params);
+    }
+    requireRun(runId, actionKind, viewerUserId, ownerUserId) {
+        return this.runLedger.requireRun(runId, actionKind, viewerUserId, ownerUserId);
+    }
+    finishActivePersistentRun(toolId, terminalId, completion) {
+        return this.terminalLifecycle.finishActivePersistentRun(toolId, terminalId, completion);
+    }
+    reattachActivePersistentTerminal(row, nodeId) {
+        return this.terminalLifecycle.reattachActivePersistentTerminal(row, nodeId);
+    }
+    async handleTerminalTermination(event) {
+        return this.terminalLifecycle.handleTerminalTermination(event);
+    }
+    schedulePersistentRunTimeout(params) {
+        this.terminalLifecycle.schedulePersistentRunTimeout(params);
+    }
+    schedulePersistentIdleTimeout(params) {
+        this.terminalLifecycle.schedulePersistentIdleTimeout(params);
+    }
+    clearPersistentRunTimeout(terminalId) {
+        this.terminalLifecycle.clearPersistentRunTimeout(terminalId);
+    }
+    clearPersistentIdleTimeout(terminalId) {
+        this.terminalLifecycle.clearPersistentIdleTimeout(terminalId);
+    }
+    clearPersistentStartupCaptureTimeout(terminalId) {
+        this.terminalLifecycle.clearPersistentStartupCaptureTimeout(terminalId);
+    }
+    markPersistentStartupOutputObserved(runId) {
+        this.observedPersistentStartupOutput.add(runId);
+    }
+    clearPersistentStartupOutputObserved(runId) {
+        if (runId) {
+            this.observedPersistentStartupOutput.delete(runId);
+            this.startupStateReplaySuppressedRunIds.delete(runId);
+        }
+    }
+}
+function extractWorkspaceToolJsonLineCandidate(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    for (let index = trimmed.indexOf('{'); index >= 0; index = trimmed.indexOf('{', index + 1)) {
+        const candidate = trimmed.slice(index);
+        try {
+            JSON.parse(candidate);
+            return candidate;
+        }
+        catch {
+            // Keep scanning; prompts or shell echoes can contain braces before the JSON object.
+        }
+    }
+    return null;
+}
+function extractWorkspaceToolJsonFragment(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    const quotedObjectIndex = trimmed.indexOf('{"');
+    if (quotedObjectIndex >= 0) {
+        return trimmed.slice(quotedObjectIndex);
+    }
+    return trimmed.startsWith('{') ? trimmed : null;
+}