@revisium/orchestrator 0.1.0-alpha.8 → 0.1.0

package/dist/task-control-plane/task-control-plane-api.service.js

@@ -18,6 +18,7 @@ import { Inject, Injectable } from '@nestjs/common';
 import { baseUrl, getConfig, isAlive, isHealthy, readRuntime } from '../cli/config.js';
 import { AgentObservabilityService } from '../observability/agent-observability.service.js';
 import { ControlPlaneError } from '../control-plane/errors.js';
+import { fnv1a64Hex } from '../control-plane/steps.js';
 import { DbosService } from '../engine/dbos.service.js';
 import { PipelineService } from '../pipeline/pipeline.service.js';
 import { RUN_PROGRESS_EVENT_KEY } from '../pipeline/data-driven-task.workflow.js';
@@ -27,6 +28,7 @@ import { InboxService } from '../revisium/inbox.service.js';
 import { PlaybooksService } from '../revisium/playbooks.service.js';
 import { RolesService } from '../revisium/roles.service.js';
 import { RunService } from '../revisium/run.service.js';
+import { CreateRunWorkflowError, previewCreateRunIds, } from '../run/create-run.js';
 import { PrReadinessService } from './pr-readiness.service.js';
 const execFileAsync = promisify(execFile);
 const GATE_TOPICS = new Set(['plan', 'merge', 'question']);
@@ -120,6 +122,40 @@ function summarizeAttempts(attempts) {
 function normalizedRunStatus(status) {
     return status === 'paused' ? 'blocked' : status;
 }
+function hasWorkflowProgress(events) {
+    return events.some((event) => (event.type.startsWith('step_')
+        || event.type.startsWith('attempt_')
+        || event.type.startsWith('gate_')
+        || event.type === 'pipeline_blocked'
+        || event.type === 'pr_polled'
+        || event.type === 'integrate_succeeded'));
+}
+function observedRunStatus(rowStatus, workflowStatus, events) {
+    if (rowStatus !== 'ready')
+        return rowStatus;
+    if (workflowStatus === 'SUCCESS')
+        return 'completed';
+    if (workflowStatus === 'ERROR')
+        return 'failed';
+    if (workflowStatus && workflowStatus !== 'NOT_STARTED')
+        return 'running';
+    if (hasWorkflowProgress(events))
+        return 'running';
+    return rowStatus;
+}
+function observedTaskStatus(rowStatus, runStatus) {
+    if (rowStatus !== 'ready')
+        return rowStatus;
+    if (runStatus === 'running' || runStatus === 'completed' || runStatus === 'failed' || runStatus === 'cancelled') {
+        return runStatus;
+    }
+    return rowStatus;
+}
+function latestEventPulse(events) {
+    const latest = [...events]
+        .sort((left, right) => dateOrEpoch(right.createdAt).getTime() - dateOrEpoch(left.createdAt).getTime())[0];
+    return latest ? { latestEventAt: latest.createdAt, latestEventType: latest.type } : {};
+}
 function payloadRecord(event) {
     return asRecord(event.payload);
 }
@@ -138,6 +174,72 @@ function eventSummary(event) {
         return output.slice(0, 180);
     return event.type;
 }
+function blockedReasonFromEvent(event) {
+    const reason = payloadRecord(event)?.reason;
+    return typeof reason === 'string' && reason ? reason : undefined;
+}
+function latestPipelineBlockedEvent(events) {
+    return [...events].reverse().find((event) => event.type === 'pipeline_blocked');
+}
+function recoveryCreatedForBlockedEvent(events, blockedEventId) {
+    return [...events]
+        .reverse()
+        .find((event) => {
+        if (event.type !== 'run_recovery_created')
+            return false;
+        return payloadRecord(event)?.blockedEventId === blockedEventId;
+    });
+}
+function requiredString(value, field, runId) {
+    if (typeof value === 'string' && value.trim())
+        return value;
+    throw new ControlPlaneError('VALIDATION_FAILURE', `Cannot recover run ${runId}: parent ${field} is missing`);
+}
+function optionalString(value) {
+    return typeof value === 'string' ? value : undefined;
+}
+function optionalInteger(value) {
+    return typeof value === 'number' && Number.isSafeInteger(value) ? value : undefined;
+}
+function requiredRecord(value, field, runId) {
+    const record = asRecord(value);
+    if (record)
+        return record;
+    throw new ControlPlaneError('VALIDATION_FAILURE', `Cannot recover run ${runId}: parent ${field} is not a record`);
+}
+function optionalRecord(value) {
+    return asRecord(value) ?? {};
+}
+function firstRepoRef(value, runId) {
+    if (Array.isArray(value) && typeof value[0] === 'string' && value[0].trim())
+        return value[0];
+    throw new ControlPlaneError('VALIDATION_FAILURE', `Cannot recover run ${runId}: parent repo is missing`);
+}
+function makeRecoveryLineage(parentRunId, recoveryRunId, blockedEventId) {
+    return { parentRunId, recoveryRunId, blockedEventId, reason: 'preflight' };
+}
+function recoveryLineagePayload(lineage) {
+    return { parentRunId: lineage.parentRunId, recoveryRunId: lineage.recoveryRunId, blockedEventId: lineage.blockedEventId, reason: lineage.reason };
+}
+function recoveryLineageEvent(lineage, runId, taskId, type) {
+    const payload = recoveryLineagePayload(lineage);
+    return { runId, taskId, stepId: '', stepKey: 'recovery', type, idempotencyKey: lineage.blockedEventId, payload, actor: 'orchestrator' };
+}
+function recoverableStartRunResponse(runId, route, recoverable) {
+    return {
+        runId,
+        workflowID: recoverable.workflowID,
+        alreadyStarted: true,
+        recoverable: true,
+        retryStarted: false,
+        nextAction: 'resume_run',
+        blockedEventId: recoverable.blockedEventId,
+        blockedReason: recoverable.reason,
+        workflowStatus: recoverable.workflowStatus,
+        route,
+        engine: 'data-driven',
+    };
+}
 function stepKeyBase(stepKey) {
     return stepKey.split('#')[0] ?? stepKey;
 }
@@ -208,6 +310,7 @@ function mapRunForWorkflow(run) {
         scope: run.scope,
         repos: run.repos ?? [],
         createdAt: dateOrEpoch(run.createdAt),
+        ...(run.issueRef ? { issueRef: run.issueRef } : {}),
     };
 }
 function workflowNodeStatus(input) {
@@ -407,6 +510,7 @@ let TaskControlPlaneApiService = class TaskControlPlaneApiService {
             playbookId: input.playbookId,
             pipelineId: input.pipelineId,
             params: input.params,
+            issueRef: input.issueRef,
             executionProfile: input.executionProfile,
             source: input.pipelineId ? 'explicit' : 'deterministic-installed-playbook',
         });
@@ -420,6 +524,7 @@ let TaskControlPlaneApiService = class TaskControlPlaneApiService {
             playbookId: route.playbookId,
             pipelineId: route.pipelineId,
             params: route.params,
+            issueRef: route.params.issueRef,
             routeDecision: route,
             executionProfile: route.executionProfile,
         });
@@ -436,8 +541,11 @@ let TaskControlPlaneApiService = class TaskControlPlaneApiService {
         const run = await this.runs.getRun(input.runId);
         if (!run)
             throw new ControlPlaneError('ROW_NOT_FOUND', `run not found: ${input.runId}`);
-        const route = input.route ?? await this.routeForRun(run);
         const existingStatus = await this.dbos.getWorkflowStatus(input.runId);
+        const route = input.route ?? await this.routeForRun(run);
+        const recoverable = await this.detectRecoverablePreflightBlock(input.runId, run.data.status, existingStatus);
+        if (recoverable)
+            return recoverableStartRunResponse(input.runId, route, recoverable);
         // CUTOVER (plan 0015 slice 3): the data-driven engine is the SOLE pipeline engine. EVERY pipeline
         // routes to the data-driven workflow, executing the state-machine template carried in its
         // execution_policy (template_json). A pipeline lacking a valid template FAILS LOUD here
@@ -457,8 +565,115 @@ let TaskControlPlaneApiService = class TaskControlPlaneApiService {
             engine: 'data-driven',
         };
     }
-    resumeRun(input) {
-        return this.startRun(input);
+    async resumeRun(input) {
+        const run = await this.runs.getRun(input.runId);
+        if (!run)
+            throw new ControlPlaneError('ROW_NOT_FOUND', `run not found: ${input.runId}`);
+        const workflow = await this.dbos.getWorkflowStatus(input.runId);
+        const recoverable = await this.detectRecoverablePreflightBlock(input.runId, run.data.status, workflow);
+        if (!recoverable)
+            return this.startRun(input);
+        const recovery = await this.createOrReusePreflightRecoveryRun(input.runId, run.data, recoverable.blockedEvent);
+        const started = await this.startRun({
+            runId: recovery.recoveryRunId,
+            runnerMode: input.runnerMode,
+        });
+        return {
+            ...started,
+            recovered: true,
+            recovery,
+        };
+    }
+    async detectRecoverablePreflightBlock(runId, runStatus, workflow) {
+        if (runStatus !== 'paused' || workflow?.status !== 'SUCCESS')
+            return null;
+        const events = await this.runs.listRunEvents(runId, { type: 'pipeline_blocked', limit: 500 });
+        const blockedEvent = latestPipelineBlockedEvent(events);
+        if (!blockedEvent || blockedReasonFromEvent(blockedEvent) !== 'preflight')
+            return null;
+        return {
+            blockedEvent,
+            blockedEventId: blockedEvent.eventId,
+            reason: 'preflight',
+            workflowID: workflow.workflowID,
+            workflowStatus: 'SUCCESS',
+        };
+    }
+    async createOrReusePreflightRecoveryRun(parentRunId, parentData, blockedEvent) {
+        const existing = await this.existingPreflightRecoveryRun(parentRunId, blockedEvent);
+        if (existing)
+            return existing;
+        const parentDetail = await this.runs.showRun(parentRunId);
+        const parentTask = parentDetail?.tasks[0];
+        if (!parentTask) {
+            throw new ControlPlaneError('ROW_NOT_FOUND', `Cannot recover run ${parentRunId}: parent task is missing`);
+        }
+        const createInput = this.buildPreflightRecoveryCreateInput(parentRunId, parentData, parentTask.roleHint, blockedEvent);
+        const expected = previewCreateRunIds(createInput);
+        const created = await this.createRecoveryRunOrReuseExpected(createInput, expected);
+        const lineage = makeRecoveryLineage(parentRunId, created.runId, blockedEvent.eventId);
+        await this.recordRecoveryLineage(lineage);
+        return lineage;
+    }
+    async existingPreflightRecoveryRun(parentRunId, blockedEvent) {
+        const events = await this.runs.listRunEvents(parentRunId, { limit: 500 });
+        const event = recoveryCreatedForBlockedEvent(events, blockedEvent.eventId);
+        if (!event)
+            return null;
+        const recoveryRunId = payloadRecord(event)?.recoveryRunId;
+        if (typeof recoveryRunId !== 'string' || !recoveryRunId)
+            return null;
+        const recovery = await this.runs.showRun(recoveryRunId);
+        if (!recovery) {
+            throw new ControlPlaneError('ROW_NOT_FOUND', `Cannot recover run ${parentRunId}: recovery run ${recoveryRunId} referenced by lineage event is missing`);
+        }
+        const lineage = makeRecoveryLineage(parentRunId, recoveryRunId, blockedEvent.eventId);
+        await this.recordRecoveryLineage(lineage);
+        return lineage;
+    }
+    buildPreflightRecoveryCreateInput(parentRunId, parentData, role, blockedEvent) {
+        const now = dateOrEpoch(blockedEvent.createdAt);
+        const title = requiredString(parentData.title, 'title', parentRunId);
+        const repo = firstRepoRef(parentData.repos, parentRunId);
+        const description = optionalString(parentData.description);
+        const scope = optionalString(parentData.scope);
+        const priority = optionalInteger(parentData.priority);
+        const recoveryRole = requiredString(role, 'task role_hint', parentRunId);
+        const playbookId = optionalString(parentData.playbook_id);
+        const pipelineId = optionalString(parentData.pipeline_id);
+        const params = optionalRecord(parentData.params);
+        const routeDecision = parentData.route_decision;
+        if (!isRouteDecision(routeDecision)) {
+            throw new ControlPlaneError('VALIDATION_FAILURE', `Cannot recover run ${parentRunId}: parent route_decision is invalid`);
+        }
+        const executionProfile = requiredRecord(parentData.execution_profile, 'execution_profile', parentRunId);
+        const idSuffix = fnv1a64Hex(`${parentRunId}|${blockedEvent.eventId}`).slice(0, 8);
+        return { title, repo, description, scope, priority, role: recoveryRole, playbookId, pipelineId, params, routeDecision, executionProfile, now, idSuffix };
+    }
+    /* node:coverage disable */
+    async createRecoveryRunOrReuseExpected(input, expected) {
+        try {
+            /* node:coverage enable */
+            return await this.runs.createRun(input);
+        }
+        catch (error) {
+            if (!(error instanceof CreateRunWorkflowError))
+                throw error;
+            const existing = await this.runs.showRun(expected.runId);
+            if (existing?.tasks[0])
+                return expected;
+            throw error;
+        }
+    }
+    async recordRecoveryLineage(lineage) {
+        const [parentDetail, recoveryDetail] = await Promise.all([
+            this.runs.showRun(lineage.parentRunId),
+            this.runs.showRun(lineage.recoveryRunId),
+        ]);
+        const parentTaskId = parentDetail?.tasks[0]?.taskId ?? '';
+        const recoveryTaskId = recoveryDetail?.tasks[0]?.taskId ?? '';
+        await this.runs.appendEvent(recoveryLineageEvent(lineage, lineage.parentRunId, parentTaskId, 'run_recovery_created'));
+        await this.runs.appendEvent(recoveryLineageEvent(lineage, lineage.recoveryRunId, recoveryTaskId, 'run_recovery_parent'));
     }
     async cancelRun(runId) {
         const result = await this.runs.cancelRun(runId, { actor: 'mcp', source: 'mcp-cancel' });
@@ -480,7 +695,7 @@ let TaskControlPlaneApiService = class TaskControlPlaneApiService {
         };
     }
     getRunEvents(input) {
-        return this.runs.listRunEvents(input.runId, { type: input.type, limit: input.limit });
+        return this.runs.listRunEvents(input.runId, { type: input.type, limit: input.limit, expand: input.expand });
     }
     async getRunProgress(runId) {
         const detail = await this.runs.showRun(runId);
@@ -626,17 +841,22 @@ let TaskControlPlaneApiService = class TaskControlPlaneApiService {
         const detail = await this.runs.showRun(runId);
         if (!detail)
             throw new ControlPlaneError('ROW_NOT_FOUND', `run not found: ${runId}`);
-        const [events, attempts, inbox] = await Promise.all([
+        const [events, attempts, inbox, workflow] = await Promise.all([
             this.runs.listRunEvents(runId, { limit: 20 }),
             this.runs.listRunAttempts(runId, { limit: 100 }),
             this.inbox.listInbox({ runId, status: 'pending', limit: 50 }),
+            this.dbos.getWorkflowStatus(runId).catch(() => null),
         ]);
+        const latestBlockingEvent = latestPipelineBlockedEvent(events);
+        const blockedReason = latestBlockingEvent ? blockedReasonFromEvent(latestBlockingEvent) : undefined;
+        const runStatus = observedRunStatus(detail.run.status, workflow?.status ?? '', events);
         return {
-            run: detail.run,
-            tasks: detail.tasks,
+            run: { ...detail.run, status: runStatus },
+            tasks: detail.tasks.map((task) => ({ ...task, status: observedTaskStatus(task.status, runStatus) })),
             pendingInbox: inbox,
             latestEvents: events.slice(-10),
             usage: summarizeAttempts(attempts),
+            ...(blockedReason !== undefined ? { blockedReason } : {}),
         };
     }
     async waitForRun(input) {
@@ -650,24 +870,36 @@ let TaskControlPlaneApiService = class TaskControlPlaneApiService {
             await new Promise((resolve) => setTimeout(resolve, intervalMs));
         }
     }
+    /**
+     * Resolve a single run to its actionable state. Public so `RunWatchService` (slice 141 D2) can
+     * fan it out across many runs; it is a point-in-time level read (gate = `inbox.find(approval)`),
+     * not an event cursor — the watch primitive layers at-least-once + idempotent delivery on top.
+     */
     async resolveRunState(runId) {
         const detail = await this.runs.showRun(runId);
         if (!detail)
             throw new ControlPlaneError('ROW_NOT_FOUND', `run not found: ${runId}`);
+        const issueRef = detail.run.issueRef;
+        const issueRefPart = issueRef ? { issueRef } : {};
         const [inbox, events, workflow] = await Promise.all([
             this.inbox.listInbox({ runId, status: 'pending', limit: 20 }),
             this.runs.listRunEvents(runId, { limit: 20 }),
             this.dbos.getWorkflowStatus(runId),
         ]);
+        const workflowStatus = workflow?.status ?? '';
+        const runStatus = observedRunStatus(detail.run.status, workflowStatus, events);
+        const eventPulsePart = latestEventPulse(events);
         const gate = inbox.find((item) => item.kind === 'approval');
         if (gate) {
             return {
                 runId,
                 state: 'pending_gate',
                 nextAction: 'resolve approval with approve_gate or reject_gate',
-                runStatus: detail.run.status,
-                workflowStatus: workflow?.status ?? '',
+                runStatus,
+                workflowStatus,
                 inbox: gate,
+                ...issueRefPart,
+                ...eventPulsePart,
             };
         }
         const question = inbox.find((item) => item.kind === 'question');
@@ -676,46 +908,66 @@ let TaskControlPlaneApiService = class TaskControlPlaneApiService {
                 runId,
                 state: 'question',
                 nextAction: 'answer question with answer_question',
-                runStatus: detail.run.status,
-                workflowStatus: workflow?.status ?? '',
+                runStatus,
+                workflowStatus,
                 inbox: question,
+                ...issueRefPart,
+                ...eventPulsePart,
             };
         }
         if (detail.run.status === 'completed') {
-            return { runId, state: 'completed', nextAction: 'none', runStatus: detail.run.status, workflowStatus: workflow?.status ?? '' };
+            return { runId, state: 'completed', nextAction: 'none', runStatus, workflowStatus, ...issueRefPart, ...eventPulsePart };
         }
         if (detail.run.status === 'failed') {
-            return { runId, state: 'failed', nextAction: 'inspect get_run_events/get_run_log', runStatus: detail.run.status, workflowStatus: workflow?.status ?? '' };
+            return { runId, state: 'failed', nextAction: 'inspect get_run_events/get_run_log', runStatus, workflowStatus, ...issueRefPart, ...eventPulsePart };
         }
         if (detail.run.status === 'cancelled') {
-            return { runId, state: 'blocked', nextAction: 'run was cancelled; create or resume a different run', runStatus: detail.run.status, workflowStatus: workflow?.status ?? '' };
+            return { runId, state: 'blocked', nextAction: 'run was cancelled; create or resume a different run', runStatus, workflowStatus, ...issueRefPart, ...eventPulsePart };
         }
+        const blockingEvent = latestPipelineBlockedEvent(events);
+        const blockedReason = blockingEvent ? blockedReasonFromEvent(blockingEvent) : undefined;
         if (detail.run.status === 'paused') {
-            return { runId, state: 'blocked', nextAction: 'inspect blocking event and decide whether to create a follow-up run', runStatus: detail.run.status, workflowStatus: workflow?.status ?? '' };
+            return {
+                runId,
+                state: 'blocked',
+                nextAction: 'inspect blocking event and decide whether to create a follow-up run',
+                runStatus,
+                workflowStatus,
+                ...(blockedReason !== undefined ? { blockedReason } : {}),
+                ...issueRefPart,
+                ...eventPulsePart,
+            };
         }
-        const blocked = [...events].reverse().find((event) => event.type === 'pipeline_blocked');
-        if (blocked) {
+        if (blockingEvent) {
             return {
                 runId,
                 state: 'blocked',
                 nextAction: 'inspect blocking event and decide whether to create a follow-up run',
-                runStatus: detail.run.status,
-                workflowStatus: workflow?.status ?? '',
-                latestBlockingEvent: blocked,
+                runStatus,
+                workflowStatus,
+                latestBlockingEvent: blockingEvent,
+                ...(blockedReason !== undefined ? { blockedReason } : {}),
+                ...issueRefPart,
+                ...eventPulsePart,
             };
         }
         if (workflow?.status === 'ERROR') {
-            return { runId, state: 'failed', nextAction: 'inspect get_run_events/get_run_log', runStatus: detail.run.status, workflowStatus: workflow.status };
+            return { runId, state: 'failed', nextAction: 'inspect get_run_events/get_run_log', runStatus, workflowStatus, ...issueRefPart, ...eventPulsePart };
         }
         if (workflow?.status === 'SUCCESS') {
-            return { runId, state: 'completed', nextAction: 'none', runStatus: detail.run.status, workflowStatus: workflow.status };
+            return { runId, state: 'completed', nextAction: 'none', runStatus, workflowStatus, ...issueRefPart, ...eventPulsePart };
+        }
+        if (runStatus === 'ready') {
+            return { runId, state: 'ready', nextAction: 'start_run', runStatus, workflowStatus, ...issueRefPart, ...eventPulsePart };
         }
         return {
             runId,
             state: 'running',
             nextAction: 'wait_for_run again or inspect get_run_digest',
-            runStatus: detail.run.status,
-            workflowStatus: workflow?.status ?? '',
+            runStatus,
+            workflowStatus,
+            ...issueRefPart,
+            ...eventPulsePart,
         };
     }
     listInbox(filter) {
@@ -855,7 +1107,7 @@ let TaskControlPlaneApiService = class TaskControlPlaneApiService {
         });
     }
     async resolveRouteDecision(input) {
-        const params = normalizeParams(input.params);
+        const params = normalizeParams(input.params, input.issueRef);
         const executionProfile = normalizeExecutionProfile(input.executionProfile);
         const playbook = await this.playbooks.resolvePlaybook(input.playbookId);
         const pipeline = input.pipelineId